Last active
March 19, 2017 09:30
-
-
Save stefanvangastel/058f66b466e2772d223253af8f9800b4 to your computer and use it in GitHub Desktop.
CakePHP 3 + Syslog + Logstash + Elasticsearch, these gists are source for a blogpost regarding the out of the box syslog engine for CakePHP 3 and using logstash to push them to elasticsearch
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
# Change default log engine at the end of bootstrap.php | |
# See https://book.cakephp.org/3.0/en/core-libraries/logging.html#logging-to-syslog | |
Log::config('default', [ | |
'engine' => 'Syslog', | |
'flag' => LOG_ODELAY | LOG_PERROR, | |
'facility' => LOG_LOCAL7, // Use local7 as dedicated syslog facilty for this app (https://en.wikipedia.org/wiki/Syslog#Facility) | |
'prefix' => 'MY_APP' | |
]); |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
{ | |
"_index": "logstash-2017.01.04", | |
"_type": "syslog", | |
"_id": "AVlp9VLfjrqyulk7dql7", | |
"_score": 1, | |
"_source": { | |
"@timestamp": "2017-01-04T14:50:01.685Z", | |
"message": "<187>Jan 4 15:50:00 ubuntu MY_APP: error: [Cake\\Network\\Exception\\InternalErrorException] No user account found in header or REMOTE_USER var. Is proxy forwarding working ok?\n<187>Jan 4 15:50:00 ubuntu MY_APP: error: Request URL: /some-url\n<187>Jan 4 15:50:00 ubuntu MY_APP: error: Stack Trace:\n<187>Jan 4 15:50:00 ubuntu MY_APP: error: #0 /var/www/vendor/cakephp/cakephp/src/Controller/Component/AuthComponent.php(697): Example\\LdapAuthenticate\\Auth\\LdapAuthenticate->getUser(Object(Cake\\Network\\Request))\n<187>Jan 4 15:50:00 ubuntu MY_APP: error: #1 /var/www/vendor/cakephp/cakephp/src/Controller/Component/AuthComponent.php(292): Cake\\Controller\\Component\\AuthComponent->_getUser()\n<187>Jan 4 15:50:00 ubuntu MY_APP: error: #2 /var/www/vendor/cakephp/cakephp/src/Controller/Component/AuthComponent.php(258): Cake\\Controller\\Component\\AuthComponent->authCheck(Object(Cake\\Event\\Event))\n<187>Jan 4 15:50:00 ubuntu MY_APP: error: #3 /var/www/vendor/cakephp/cakephp/src/Event/EventManager.php(385): Cake\\Controller\\Component\\AuthComponent->startup(Object(Cake\\Event\\Event))\n<187>Jan 4 15:50:00 ubuntu MY_APP: error: #4 /var/www/vendor/cakephp/cakephp/src/Event/EventManager.php(355): Cake\\Event\\EventManager->_callListener(Array, Object(Cake\\Event\\Event))\n<187>Jan 4 15:50:00 ubuntu MY_APP: error: #5 /var/www/vendor/cakephp/cakephp/src/Event/EventDispatcherTrait.php(78): Cake\\Event\\EventManager->dispatch(Object(Cake\\Event\\Event))\n<187>Jan 4 15:50:00 ubuntu MY_APP: error: #6 /var/www/vendor/cakephp/cakephp/src/Controller/Controller.php(495): Cake\\Controller\\Controller->dispatchEvent('Controller.star...')\n<187>Jan 4 15:50:00 ubuntu MY_APP: error: #7 /var/www/vendor/cakephp/cakephp/src/Routing/Dispatcher.php(109): Cake\\Controller\\Controller->startupProcess()\n<187>Jan 4 15:50:00 ubuntu MY_APP: error: #8 /var/www/vendor/cakephp/cakephp/src/Routing/Dispatcher.php(87): Cake\\Routing\\Dispatcher->_invoke(Object(App\\Controller\\AppController))\n<187>Jan 4 15:50:00 ubuntu MY_APP: error: #9 /var/www/webroot/index.php(37): Cake\\Routing\\Dispatcher->dispatch(Object(Cake\\Network\\Request), Object(Cake\\Network\\Response))\n<187>Jan 4 15:50:00 ubuntu MY_APP: error: #10 {main}\n<187>Jan 4 15:50:00 ubuntu MY_APP: error: \n<187>Jan 4 15:38:10 ubuntu MY_APP: error: ", | |
"@version": "1", | |
"tags": [ | |
"multiline", | |
"cakephp_log" | |
], | |
"host": "192.168.0.99", | |
"port": 60677, | |
"type": "syslog", | |
"timestamp": "Jan 4 15:50:00", | |
"logsource": "ubuntu", | |
"program": "MY_APP", | |
"loglevel": "error", | |
"exception": "Cake\\Network\\Exception\\InternalErrorException", | |
"path": "/some-url", | |
"stacktrace": "\n#0 /var/www/vendor/cakephp/cakephp/src/Controller/Component/AuthComponent.php(697): Example\\LdapAuthenticate\\Auth\\LdapAuthenticate->getUser(Object(Cake\\Network\\Request))\n#1 /var/www/vendor/cakephp/cakephp/src/Controller/Component/AuthComponent.php(292): Cake\\Controller\\Component\\AuthComponent->_getUser()\n#2 /var/www/vendor/cakephp/cakephp/src/Controller/Component/AuthComponent.php(258): Cake\\Controller\\Component\\AuthComponent->authCheck(Object(Cake\\Event\\Event))\n#3 /var/www/vendor/cakephp/cakephp/src/Event/EventManager.php(385): Cake\\Controller\\Component\\AuthComponent->startup(Object(Cake\\Event\\Event))\n#4 /var/www/vendor/cakephp/cakephp/src/Event/EventManager.php(355): Cake\\Event\\EventManager->_callListener(Array, Object(Cake\\Event\\Event))\n#5 /var/www/vendor/cakephp/cakephp/src/Event/EventDispatcherTrait.php(78): Cake\\Event\\EventManager->dispatch(Object(Cake\\Event\\Event))\n#6 /var/www/vendor/cakephp/cakephp/src/Controller/Controller.php(495): Cake\\Controller\\Controller->dispatchEvent('Controller.star...')\n#7 /var/www/vendor/cakephp/cakephp/src/Routing/Dispatcher.php(109): Cake\\Controller\\Controller->startupProcess()\n#8 /var/www/vendor/cakephp/cakephp/src/Routing/Dispatcher.php(87): Cake\\Routing\\Dispatcher->_invoke(Object(App\\Controller\\AppController))\n#9 /var/www/webroot/index.php(37): Cake\\Routing\\Dispatcher->dispatch(Object(Cake\\Network\\Request), Object(Cake\\Network\\Response))\n#10 {main}\n\n" | |
}, | |
"fields": { | |
"@timestamp": [ | |
1483541401685 | |
] | |
} | |
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
# File: /etc/logstash/conf.d/my_app.conf | |
input { | |
tcp { | |
codec => multiline { | |
# Merge lines based on an exception | |
pattern => "\[%{GREEDYDATA}\]" | |
negate => "true" | |
what => "previous" | |
} | |
port => 5140 | |
type => "syslog" | |
} | |
} | |
filter{ | |
grok{ | |
match => {"message"=>"%{SYSLOGBASE2} %{LOGLEVEL:loglevel}: \[%{DATA:exception}\] %{GREEDYDATA} Request URL: %{URIPATHPARAM:path}%{GREEDYDATA}Stack Trace:%{GREEDYDATA:stacktrace}"} | |
overwrite =>["message"] | |
add_tag => ["cakephp_log"] | |
} | |
mutate { | |
# Remove the prefixed syslog base (e.g.: 'Jan 4 15:39:35 ubuntu my_app: error:') | |
gsub => ["stacktrace","...\ ..\ ..\:..\:..\ .*\ .*\:\ error: ",""] | |
# Remove the syslog prefix (e.g.: '<187>') | |
gsub => ["stacktrace","<[0-9]+>",""] | |
} | |
} | |
output { | |
# Send to Elasticsearch | |
elasticsearch { | |
hosts => "es.mydomain.com" | |
index => "logstash-%{+YYYY.MM.dd}" | |
} | |
# Debug | |
stdout { codec => rubydebug } | |
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
FILE: /var/log/my_app.log | |
Jan 4 15:50:01 ubuntu MY_APP: error: [Cake\Network\Exception\InternalErrorException] Holy cow, something went wrong! | |
Jan 4 15:50:01 ubuntu MY_APP: error: Request URL: /some-url | |
Jan 4 15:50:01 ubuntu MY_APP: error: Stack Trace: | |
Jan 4 15:50:01 ubuntu MY_APP: error: #0 /var/www/vendor/cakephp/cakephp/src/Controller/Component/AuthComponent.php(697): Example\LdapAuthenticate\Auth\LdapAuthenticate->getUser(Object(Cake\Network\Request)) | |
Jan 4 15:50:01 ubuntu MY_APP: error: #1 /var/www/vendor/cakephp/cakephp/src/Controller/Component/AuthComponent.php(292): Cake\Controller\Component\AuthComponent->_getUser() | |
Jan 4 15:50:01 ubuntu MY_APP: error: #2 /var/www/vendor/cakephp/cakephp/src/Controller/Component/AuthComponent.php(258): Cake\Controller\Component\AuthComponent->authCheck(Object(Cake\Event\Event)) | |
Jan 4 15:50:01 ubuntu MY_APP: error: #3 /var/www/vendor/cakephp/cakephp/src/Event/EventManager.php(385): Cake\Controller\Component\AuthComponent->startup(Object(Cake\Event\Event)) | |
Jan 4 15:50:01 ubuntu MY_APP: error: #4 /var/www/vendor/cakephp/cakephp/src/Event/EventManager.php(355): Cake\Event\EventManager->_callListener(Array, Object(Cake\Event\Event)) | |
Jan 4 15:50:01 ubuntu MY_APP: error: #5 /var/www/vendor/cakephp/cakephp/src/Event/EventDispatcherTrait.php(78): Cake\Event\EventManager->dispatch(Object(Cake\Event\Event)) | |
Jan 4 15:50:01 ubuntu MY_APP: error: #6 /var/www/vendor/cakephp/cakephp/src/Controller/Controller.php(495): Cake\Controller\Controller->dispatchEvent('Controller.star...') | |
Jan 4 15:50:01 ubuntu MY_APP: error: #7 /var/www/vendor/cakephp/cakephp/src/Routing/Dispatcher.php(109): Cake\Controller\Controller->startupProcess() | |
Jan 4 15:50:01 ubuntu MY_APP: error: #8 /var/www/vendor/cakephp/cakephp/src/Routing/Dispatcher.php(87): Cake\Routing\Dispatcher->_invoke(Object(App\Controller\AppController)) | |
Jan 4 15:50:01 ubuntu MY_APP: error: #9 /var/www/webroot/index.php(37): Cake\Routing\Dispatcher->dispatch(Object(Cake\Network\Request), Object(Cake\Network\Response)) | |
Jan 4 15:50:01 ubuntu MY_APP: error: #10 {main} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
# File: /etc/rsyslog.d/my_app.conf | |
# Use local7 as log facility for my_app, send the logs to both a Logstash server over TCP (@@) and a local file in /var/log | |
local7.* @@logstash.mydomain.com:5140 | |
local7.* /var/log/my_app.log |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment