Creates a Filebeat pipeline to ingest Laravel Monolog/log lines.

README.md

Ingest Laravel Log lines in ElasticSearch/Filebeat

Set up the pipeline

Run Create_Laravel_Pipeline.txt in Kibana or manually through the command line. Run Simulate_Pipeline.txt to check if it works and parses the document properly.

Configure Filebeat

Configure Filebeat to add a new prospector log and ship it to straight to the pipeline we created:

- type: log
  
  enabled: true
  
  # Paths that should be crawled and fetched. Glob based paths.

  paths:
    - /path/to/laravel/root/storage/logs/*.log

  exclude_files: ['laravel-worker']
  
  ### Multiline options

  multiline.pattern: '^\[[0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}:[0-9]{2}\]'

  multiline.negate: true

  multiline.match: after
  
 output.elasticsearch:

  hosts: ["localhost:9200"]
  
  pipeline: laravel

Create_Laravel_Pipeline.txt

PUT _ingest/pipeline/laravel
{
    "description": "Parses Laravel log files.",
    "processors": [
        {
          "rename": {
            "field": "message",
            "target_field": "event.original"
          }
        },
        {
            "grok": {
                "field": "event.original",
                "patterns": [
                    "\\[%{TIMESTAMP_ISO8601:timestamp}\\] %{DATA:laravel.environment}\\.%{DATA:laravel.severity}( \\[%{IP:destination.ip}\\])?: %{GREEDYDATA:message}\n?(?m)%{GREEDYDATA:stacktrace}?"
                ]
            }
        },
        {
            "date": {
                "field": "timestamp",
                "formats": [
                    "yyyy-MM-dd HH:mm:ss",
                    "ISO8601"  
                ],
                "timezone": "America/Toronto"
            }
        },
        {
            "remove": {
                "field": ["timestamp"]
            }
        },
        {
            "set": {
                "field": "event.module",
                "value": "laravel"
            }
        },
        {
            "set": {
                "field": "event.dataset",
                "value": "laravel.log"
            }
        }
    ]
}

Simulate_Pipeline.txt

POST _ingest/pipeline/laravel/_simulate
{
    "docs": [
    {
      "_source": {
        "message": "[2018-11-26 22:23:30] local.ERROR: Error executing \"ReceiveMessage\" on \"https:\/\/sqs.ca-central-1.amazonaws.com\/aaaa\/test\"; AWS HTTP error: cURL error 6: Could not resolve host: sqs.ca-central-1.amazonaws.com; Unknown error (see http:\/\/curl.haxx.se\/libcurl\/c\/libcurl-errors.html) {\"exception\":\"[object] (Aws\\\\Sqs\\\\Exception\\\\SqsException(code: 0): Error executing \\\"ReceiveMessage\\\" on \\\"https:\/\/sqs.ca-central-1.amazonaws.com\/653837051153\/test\\\"; AWS HTTP error: cURL error 6: Could not resolve host: sqs.ca-central-1.amazonaws.com; Unknown error (see http:\/\/curl.haxx.se\/libcurl\/c\/libcurl-errors.html) at \/usr\/share\/nginx\/venngo\/vendor\/aws\/aws-sdk-php\/src\/WrappedHttpHandler.php:191, GuzzleHttp\\\\Exception\\\\ConnectException(code: 0): cURL error 6: Could not resolve host: sqs.ca-central-1.amazonaws.com; Unknown error (see http:\/\/curl.haxx.se\/libcurl\/c\/libcurl-errors.html) at \/usr\/share\/nginx\/venngo\/vendor\/guzzlehttp\/guzzle\/src\/Handler\/CurlFactory.php:185)"
      }
    }
  ]
}