-
-
Save eperdeme/ca2b69f8c7acc7f88132 to your computer and use it in GitHub Desktop.
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| input { | |
| file { | |
| path => "/var/log/nginx/access.log" | |
| type => "nginx-access" | |
| sincedb_path => "/var/log/.nginxaccesssincedb" | |
| } | |
| } | |
| input { | |
| file { | |
| path => "/var/log/nginx/error.log" | |
| type => "nginx-error" | |
| sincedb_path => "/var/log/.nginxerrorsincedb" | |
| } | |
| } | |
| input { | |
| redis { | |
| host => "127.0.0.1" | |
| data_type => "list" | |
| key => "logstash" | |
| } | |
| } | |
| input { | |
| tcp { | |
| type => "syslog" | |
| port => "514" | |
| } | |
| } | |
| input { | |
| tcp { | |
| type => "VMware" | |
| port => "1514" | |
| } | |
| } | |
| input { | |
| tcp { | |
| type => "vCenter" | |
| port => "1515" | |
| } | |
| } | |
| input { | |
| tcp { | |
| type => "Netscaler" | |
| port => "1517" | |
| } | |
| } | |
| input { | |
| tcp { | |
| type => "eventlog" | |
| port => "3515" | |
| format => "json" | |
| } | |
| } | |
| input { | |
| tcp { | |
| type => "iis" | |
| port => "3525" | |
| codec => "json_lines" | |
| } | |
| } | |
| filter { | |
| if [type] == "syslog" { | |
| dns { | |
| reverse => [ "host" ] action => "replace" | |
| } | |
| mutate { | |
| add_tag => [ "syslog-TCP", "syslog" ] | |
| } | |
| } | |
| if [type] == "VMware" { | |
| mutate { | |
| add_tag => "VMware" | |
| } | |
| } | |
| if [type] == "vCenter" { | |
| mutate { | |
| add_tag => "vCenter" | |
| } | |
| } | |
| if [type] == "PFsense" { | |
| mutate { | |
| add_tag => "PFsense" | |
| } | |
| } | |
| if [type] == "Netscaler" { | |
| mutate { | |
| add_tag => "Netscaler" | |
| } | |
| } | |
| if [type] == "eventlog" { | |
| mutate { | |
| add_tag => [ "WindowsEventLog" ] | |
| } | |
| } | |
| if [type] == "apache" { | |
| mutate { | |
| add_tag => [ "apache" ] | |
| } | |
| } | |
| if [type] =~ "nginx" { | |
| mutate { | |
| add_tag => [ "nginx" ] | |
| } | |
| } | |
| if [type] == "iis" { | |
| mutate { | |
| add_tag => [ "IIS" ] | |
| } | |
| } | |
| } | |
| filter { | |
| if "syslog" in [tags] { | |
| grok { | |
| match => { "message" => "<%{POSINT:syslog_pri}>%{SYSLOGTIMESTAMP:syslog_timestamp} %{SYSLOGHOST:syslog_hostname} %{DATA:syslog_program}(?:\[%{POSINT:syslog_pid}\])?: %{GREEDYDATA:syslog_message}" } | |
| add_field => [ "received_at", "%{@timestamp}" ] | |
| add_field => [ "received_from", "%{host}" ] | |
| } | |
| syslog_pri { } | |
| date { | |
| match => [ "syslog_timestamp", "MMM d HH:mm:ss", "MMM dd HH:mm:ss" ] | |
| } | |
| if !("_grokparsefailure" in [tags]) { | |
| mutate { | |
| replace => [ "@source_host", "%{syslog_hostname}" ] | |
| replace => [ "@message", "%{syslog_message}" ] | |
| } | |
| } | |
| mutate { | |
| remove_field => [ "syslog_hostname", "syslog_message" ] | |
| } | |
| if "_grokparsefailure" in [tags] { | |
| drop { } | |
| } | |
| } | |
| } | |
| filter { | |
| if "syslog" in [tags] { | |
| if "IPTables" in [message] { | |
| grok { | |
| match => { "message" => "%{IPTABLES}" } | |
| patterns_dir => [ "/opt/logstash/patterns" ] | |
| } | |
| mutate { | |
| add_tag => [ "IPTABLES" ] | |
| } | |
| geoip { | |
| source => "src_ip" | |
| target => "geoip" | |
| add_field => [ "[geoip][coordinates]", "%{[geoip][longitude]}" ] | |
| add_field => [ "[geoip][coordinates]", "%{[geoip][latitude]}" ] | |
| } | |
| mutate { | |
| convert => [ "[geoip][coordinates]", "float" ] | |
| } | |
| } | |
| } | |
| } | |
| filter { | |
| if "IPTABLES" in [tags] { | |
| grok { | |
| match => [ | |
| "message", "IPTables-%{WORD:iptables_action}" | |
| ] | |
| } | |
| grok { | |
| match => [ | |
| "message", "PROTO=%{WORD:iptables_proto}" | |
| ] | |
| } | |
| mutate { | |
| remove_field => [ "proto" ] | |
| } | |
| mutate { | |
| rename => [ "iptables_proto", "proto" ] | |
| } | |
| } | |
| } | |
| filter { | |
| if "syslog" in [tags] { | |
| if [syslog_program] == "haproxy" { | |
| grok { | |
| break_on_match => false | |
| match => [ | |
| "message", "%{HAPROXYHTTP}", | |
| "message", "%{HAPROXYTCP}" | |
| ] | |
| add_tag => [ "HAProxy" ] | |
| } | |
| geoip { | |
| source => "client_ip" | |
| target => "geoip" | |
| add_field => [ "[geoip][coordinates]", "%{[geoip][longitude]}" ] | |
| add_field => [ "[geoip][coordinates]", "%{[geoip][latitude]}" ] | |
| } | |
| mutate { | |
| convert => [ "[geoip][coordinates]", "float" ] | |
| } | |
| mutate { | |
| replace => [ "host", "%{@source_host}" ] | |
| } | |
| mutate { | |
| rename => [ "http_status_code", "response" ] | |
| } | |
| mutate { | |
| rename => [ "http_request", "request" ] | |
| } | |
| mutate { | |
| rename => [ "client_ip", "clientip" ] | |
| } | |
| } | |
| } | |
| } | |
| filter { | |
| if "syslog" in [tags] { | |
| if [syslog_program] =~ /Keepalived/ { | |
| mutate { | |
| add_tag => [ "KeepAliveD" ] | |
| } | |
| } | |
| } | |
| } | |
| filter { | |
| if "VMware" in [tags] { | |
| grok { | |
| break_on_match => false | |
| match => [ | |
| "message", "<%{POSINT:syslog_pri}>%{TIMESTAMP_ISO8601:@timestamp} %{SYSLOGHOST:hostname} %{SYSLOGPROG:message_program}: (?<message-body>(?<message_system_info>(?:\[%{DATA:message_thread_id} %{DATA:syslog_level} \'%{DATA:message_service}\'\ ?%{DATA:message_opID}])) \[%{DATA:message_service_info}]\ (?<message-syslog>(%{GREEDYDATA})))", | |
| "message", "<%{POSINT:syslog_pri}>%{TIMESTAMP_ISO8601:@timestamp} %{SYSLOGHOST:hostname} %{SYSLOGPROG:message_program}: (?<message-body>(?<message_system_info>(?:\[%{DATA:message_thread_id} %{DATA:syslog_level} \'%{DATA:message_service}\'\ ?%{DATA:message_opID}])) (?<message-syslog>(%{GREEDYDATA})))", | |
| "message", "<%{POSINT:syslog_pri}>%{TIMESTAMP_ISO8601:@timestamp} %{SYSLOGHOST:hostname} %{SYSLOGPROG:message_program}: %{GREEDYDATA:message-syslog}" | |
| ] | |
| } | |
| syslog_pri { } | |
| mutate { | |
| replace => [ "@source_host", "%{hostname}" ] | |
| } | |
| mutate { | |
| replace => [ "@message", "%{message-syslog}" ] | |
| } | |
| if "Device naa" in [message] { | |
| grok { | |
| break_on_match => false | |
| match => [ | |
| "message", "Device naa.%{WORD:device_naa} performance has %{WORD:device_status}%{GREEDYDATA} of %{INT:datastore_latency_from}%{GREEDYDATA} to %{INT:datastore_latency_to}", | |
| "message", "Device naa.%{WORD:device_naa} performance has %{WORD:device_status}%{GREEDYDATA} from %{INT:datastore_latency_from}%{GREEDYDATA} to %{INT:datastore_latency_to}" | |
| ] | |
| } | |
| } | |
| if "connectivity issues" in [message] { | |
| grok { | |
| match => [ | |
| "message", "Hostd: %{GREEDYDATA} : %{DATA:device_access} to volume %{DATA:device_id} %{DATA:datastore} (following|due to)" | |
| ] | |
| } | |
| } | |
| if "WARNING" in [message] { | |
| grok { | |
| match => [ | |
| "message", "WARNING: %{GREEDYDATA:vmware_warning_msg}" | |
| ] | |
| } | |
| } | |
| } | |
| if "_grokparsefailure" in [tags] { | |
| if "VMware" in [tags] { | |
| grok { | |
| break_on_match => false | |
| match => [ | |
| "message", "<%{POSINT:syslog_pri}>%{DATA:message_system_info}, (?<message-body>(%{SYSLOGHOST:hostname} %{SYSLOGPROG:message_program}: %{GREEDYDATA:message-syslog}))", | |
| "message", "" | |
| ] | |
| } | |
| } | |
| } | |
| } | |
| filter { | |
| if "vCenter" in [tags] { | |
| grok { | |
| break_on_match => false | |
| match => [ | |
| "message", "<%{INT:syslog_pri}>%{SYSLOGTIMESTAMP} %{IPORHOST:syslog_source} %{TIMESTAMP_ISO8601:@timestamp} (?<message-body>(?<message_system_info>(?:\[%{DATA:message_thread_id} %{DATA:syslog_level} \'%{DATA:message_service}\'\ ?%{DATA:message_opID}])) \[%{DATA:message_service_info}]\ (?<message-syslog>(%{GREEDYDATA})))", | |
| "message", "<%{INT:syslog_pri}>%{SYSLOGTIMESTAMP} %{IPORHOST:syslog_source} %{TIMESTAMP_ISO8601:@timestamp} (?<message-body>(?<message_system_info>(?:\[%{DATA:message_thread_id} %{DATA:syslog_level} \'%{DATA:message_service}\'\ ?%{DATA:message_opID}])) (?<message-syslog>(%{GREEDYDATA})))", | |
| "message", "<%{INT:syslog_pri}>%{SYSLOGTIMESTAMP} %{IPORHOST:syslog_source} %{TIMESTAMP_ISO8601:@timestamp} %{GREEDYDATA:message-syslog}" | |
| ] | |
| } | |
| if "_grokparsefailure" in [tags] { | |
| grok { | |
| break_on_match => false | |
| match => [ | |
| "message", "" | |
| ] | |
| } | |
| } | |
| syslog_pri { } | |
| mutate { | |
| replace => [ "@message", "%{message-syslog}" ] | |
| rename => [ "syslog_source", "@source_host" ] | |
| rename => [ "hostname", "syslog_source-hostname" ] | |
| rename => [ "program", "message_program" ] | |
| rename => [ "message_vce_server", "syslog_source-hostname" ] | |
| remove_field => [ "@version", "type", "path" ] | |
| } | |
| } | |
| } | |
| filter { | |
| if "PFSense" in [tags] { | |
| grok { | |
| add_tag => [ "firewall" ] | |
| match => [ "message", "<(?<evtid>.*)>(?<datetime>(?:Jan(?:uary)?|Feb(?:ruary)?|Mar(?:ch)?|Apr(?:il)?|May|Jun(?:e)?|Jul(?:y)?|Aug(?:ust)?|Sep(?:tember)?|Oct(?:ober)?|Nov(?:ember)?|Dec(?:ember)?)\s+(?:(?:0[1-9])|(?:[12][0-9])|(?:3[01])|[1-9]) (?:2[0123]|[01]?[0-9]):(?:[0-5][0-9]):(?:[0-5][0-9])) (?<prog>.*?): (?<msg>.*)" ] | |
| } | |
| mutate { | |
| gsub => ["datetime"," "," "] | |
| } | |
| date { | |
| match => [ "datetime", "MMM dd HH:mm:ss" ] | |
| } | |
| mutate { | |
| replace => [ "message", "%{msg}" ] | |
| } | |
| mutate { | |
| remove_field => [ "msg", "datetime" ] | |
| } | |
| } | |
| if [prog] =~ /^pf$/ { | |
| mutate { | |
| add_tag => [ "packetfilter" ] | |
| } | |
| multiline { | |
| pattern => "^\s+|^\t\s+" | |
| what => "previous" | |
| } | |
| mutate { | |
| remove_field => [ "msg", "datetime" ] | |
| remove_tag => [ "multiline" ] | |
| } | |
| grok { | |
| match => [ "message", "rule (?<rule>.*)\(.*\): (?<action>pass|block) .* on (?<iface>.*): .* proto (?<proto>TCP|UDP|IGMP|ICMP) .*\n\s*(?<src_ip>(\d+\.\d+\.\d+\.\d+))\.?(?<src_port>(\d*)) [<|>] (?<dest_ip>(\d+\.\d+\.\d+\.\d+))\.?(?<dest_port>(\d*)):" ] | |
| } | |
| } | |
| if [prog] =~ /^dhcpd$/ { | |
| if [message] =~ /^DHCPACK|^DHCPREQUEST|^DHCPOFFER/ { | |
| grok { | |
| match => [ "message", "(?<action>.*) (on|for|to) (?<src_ip>[0-2]?[0-9]?[0-9]\.[0-2]?[0-9]?[0-9]\.[0-2]?[0-9]?[0-9]\.[0-2]?[0-9]?[0-9]) .*(?<mac_address>[0-9a-fA-F][0-9a-fA-F]:[0-9a-fA-F][0-9a-fA-F]:[0-9a-fA-F][0-9a-fA-F]:[0-9a-fA-F][0-9a-fA-F]:[0-9a-fA-F][0-9a-fA-F]:[0-9a-fA-F][0-9a-fA-F]).* via (?<iface>.*)" ] | |
| } | |
| } | |
| if [message] =~ /^DHCPDISCOVER/ { | |
| grok { | |
| match => [ "message", "(?<action>.*) from (?<mac_address>[0-9a-fA-F][0-9a-fA-F]:[0-9a-fA-F][0-9a-fA-F]:[0-9a-fA-F][0-9a-fA-F]:[0-9a-fA-F][0-9a-fA-F]:[0-9a-fA-F][0-9a-fA-F]:[0-9a-fA-F][0-9a-fA-F]).* via (?<iface>.*)" ] | |
| } | |
| } | |
| if [message] =~ /^DHCPINFORM/ { | |
| grok { | |
| match => [ "message", "(?<action>.*) from (?<src_ip>.*).* via (?<iface>.*)" ] | |
| } | |
| } | |
| } | |
| if "_grokparsefailure" in [tags] { | |
| drop { } | |
| } | |
| } | |
| filter { | |
| if "PFSense" in [tags] { | |
| mutate { | |
| replace => [ "@source_host", "%{host}" ] | |
| } | |
| mutate { | |
| replace => [ "@message", "%{message}" ] | |
| } | |
| } | |
| } | |
| filter { | |
| if "Netscaler" in [tags] { | |
| grok { | |
| break_on_match => true | |
| match => [ | |
| "message", "<%{POSINT:syslog_pri}> %{DATE_US}:%{TIME} GMT %{SYSLOGHOST:syslog_hostname} %{GREEDYDATA:netscaler_message} : %{DATA} %{INT:netscaler_spcbid} - %{DATA} %{IP:clientip} - %{DATA} %{INT:netscaler_client_port} - %{DATA} %{IP:netscaler_vserver_ip} - %{DATA} %{INT:netscaler_vserver_port} %{GREEDYDATA:netscaler_message} - %{DATA} %{WORD:netscaler_session_type}", | |
| "message", "<%{POSINT:syslog_pri}> %{DATE_US}:%{TIME} GMT %{SYSLOGHOST:syslog_hostname} %{GREEDYDATA:netscaler_message}" | |
| ] | |
| } | |
| syslog_pri { } | |
| mutate { | |
| replace => [ "@source_host", "%{host}" ] | |
| } | |
| mutate { | |
| replace => [ "@message", "%{netscaler_message}" ] | |
| } | |
| geoip { | |
| source => "clientip" | |
| target => "geoip" | |
| add_field => [ "[geoip][coordinates]", "%{[geoip][longitude]}" ] | |
| add_field => [ "[geoip][coordinates]", "%{[geoip][latitude]}" ] | |
| } | |
| mutate { | |
| convert => [ "[geoip][coordinates]", "float" ] | |
| } | |
| } | |
| } | |
| filter { | |
| if [type] == "apache" { | |
| grok { | |
| pattern => "%{COMBINEDAPACHELOG}" | |
| } | |
| geoip { | |
| source => "clientip" | |
| target => "geoip" | |
| add_field => [ "[geoip][coordinates]", "%{[geoip][longitude]}" ] | |
| add_field => [ "[geoip][coordinates]", "%{[geoip][latitude]}" ] | |
| } | |
| mutate { | |
| convert => [ "[geoip][coordinates]", "float" ] | |
| } | |
| mutate { | |
| replace => [ "@source_host", "%{host}" ] | |
| } | |
| mutate { | |
| replace => [ "@message", "%{message}" ] | |
| } | |
| mutate { | |
| rename => [ "verb" , "method" ] | |
| } | |
| grok { | |
| match => [ | |
| "message", "%{DATA:apache_vhost} " | |
| ] | |
| } | |
| } | |
| } | |
| filter { | |
| if [type] =~ "nginx" { | |
| grok { | |
| pattern => "%{COMBINEDAPACHELOG}" | |
| } | |
| geoip { | |
| source => "clientip" | |
| target => "geoip" | |
| add_field => [ "[geoip][coordinates]", "%{[geoip][longitude]}" ] | |
| add_field => [ "[geoip][coordinates]", "%{[geoip][latitude]}" ] | |
| } | |
| mutate { | |
| convert => [ "[geoip][coordinates]", "float" ] | |
| } | |
| mutate { | |
| replace => [ "@source_host", "%{host}" ] | |
| } | |
| mutate { | |
| replace => [ "@message", "%{message}" ] | |
| } | |
| mutate { | |
| rename => [ "verb" , "method" ] | |
| } | |
| grok { | |
| match => [ | |
| "message", "%{DATA:apache_vhost} " | |
| ] | |
| } | |
| } | |
| } | |
| filter { | |
| if [type] == "eventlog" { | |
| grep { | |
| match => { "EventReceivedTime" => "\d+"} | |
| } | |
| mutate { | |
| lowercase => [ "EventType", "FileName", "Hostname", "Severity" ] | |
| } | |
| mutate { | |
| rename => [ "Hostname", "@source_host" ] | |
| } | |
| date { | |
| match => [ "EventReceivedTime", "UNIX" ] | |
| } | |
| mutate { | |
| rename => [ "Message", "@message" ] | |
| rename => [ "Severity", "eventlog_severity" ] | |
| rename => [ "SeverityValue", "eventlog_severity_code" ] | |
| rename => [ "Channel", "eventlog_channel" ] | |
| rename => [ "SourceName", "eventlog_program" ] | |
| rename => [ "SourceModuleName", "nxlog_input" ] | |
| rename => [ "Category", "eventlog_category" ] | |
| rename => [ "EventID", "eventlog_id" ] | |
| rename => [ "RecordNumber", "eventlog_record_number" ] | |
| rename => [ "ProcessID", "eventlog_pid" ] | |
| } | |
| mutate { | |
| remove => [ "SourceModuleType", "EventTimeWritten", "EventTime", "EventReceivedTime", "EventType" ] | |
| } | |
| } | |
| } | |
| filter { | |
| if [type] == "iis" { | |
| if [message] =~ "^#" { | |
| drop {} | |
| } | |
| grok { | |
| match => [ | |
| "message", "%{TIMESTAMP_ISO8601:logtime} %{IPORHOST:hostname} %{URIPROTO:cs_method} %{URIPATH:cs_stem} (?:%{NOTSPACE:cs_query}|-) %{NUMBER:src_port} %{NOTSPACE:cs_username} %{IP:clientip} %{NOTSPACE:cs_useragent} %{NUMBER:sc_status} %{NUMBER:sc_subresponse} %{NUMBER:sc_win32_status} %{NUMBER:timetaken}" | |
| ] | |
| } | |
| date { | |
| match => [ "logtime", "YYYY-MM-dd HH:mm:ss" ] | |
| timezone => "UTC" | |
| } | |
| mutate { | |
| replace => [ "@source_host", "%{hostname}" ] | |
| } | |
| mutate { | |
| replace => [ "@message", "%{message}" ] | |
| } | |
| geoip { | |
| source => "clientip" | |
| target => "geoip" | |
| add_field => [ "[geoip][coordinates]", "%{[geoip][longitude]}" ] | |
| add_field => [ "[geoip][coordinates]", "%{[geoip][latitude]}" ] | |
| } | |
| mutate { | |
| convert => [ "[geoip][coordinates]", "float" ] | |
| } | |
| mutate { | |
| rename => [ "cs_method", "method" ] | |
| rename => [ "cs_stem", "request" ] | |
| rename => [ "cs_useragent", "agent" ] | |
| rename => [ "cs_username", "username" ] | |
| rename => [ "sc_status", "response" ] | |
| rename => [ "timetaken", "time_request" ] | |
| } | |
| } | |
| } | |
| filter { | |
| if [type] == "mysql-slowquery" { | |
| multiline { | |
| what => previous | |
| pattern => "^\s" | |
| } | |
| grok { pattern => "^%{NUMBER:date} *%{NOTSPACE:time}" } | |
| mutate { replace => [ "time", "%{date} %{time}" ] } | |
| date { | |
| match => [ "YYMMdd H:mm:ss", "YYMMdd HH:mm:ss" ] | |
| } | |
| mutate { remove => [ "time", "date" ] } | |
| split { } | |
| } | |
| } | |
| output { | |
| elasticsearch { | |
| cluster => "logstash-cluster" | |
| flush_size => 1 | |
| manage_template => true | |
| template => "/opt/logstash/lib/logstash/outputs/elasticsearch/elasticsearch-template.json" | |
| } | |
| } |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment