Created
July 12, 2018 12:33
-
-
Save tkuther/b432827283f360293a87b5be49594f91 to your computer and use it in GitHub Desktop.
Revisions
-
tkuther created this gist
Jul 12, 2018 .There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode charactersOriginal file line number Diff line number Diff line change @@ -0,0 +1,242 @@ apiVersion: extensions/v1beta1 kind: DaemonSet metadata: name: filebeat namespace: kube-system labels: k8s-app: filebeat spec: template: metadata: labels: k8s-app: filebeat spec: serviceAccountName: filebeat terminationGracePeriodSeconds: 30 containers: - name: filebeat image: docker.elastic.co/beats/filebeat:6.3.1 args: [ "-c", "/etc/filebeat.yml", "-e", ] securityContext: runAsUser: 0 resources: limits: memory: 200Mi requests: cpu: 100m memory: 100Mi volumeMounts: - name: config mountPath: /etc/filebeat.yml readOnly: true subPath: filebeat.yml - name: inputs mountPath: /usr/share/filebeat/inputs.d readOnly: true - name: data mountPath: /usr/share/filebeat/data - name: varlibdockercontainers mountPath: /var/lib/docker/containers readOnly: true - name: config mountPath: /usr/share/filebeat/module/nginx/access/ingest/default.json readOnly: true subPath: default.json volumes: - name: config configMap: defaultMode: 0600 name: filebeat-config - name: varlibdockercontainers hostPath: path: /var/lib/docker/containers - name: inputs configMap: defaultMode: 0600 name: filebeat-inputs - name: data hostPath: path: /var/lib/filebeat --- apiVersion: rbac.authorization.k8s.io/v1beta1 kind: ClusterRoleBinding metadata: name: filebeat subjects: - kind: ServiceAccount name: filebeat namespace: kube-system roleRef: kind: ClusterRole name: filebeat apiGroup: rbac.authorization.k8s.io --- apiVersion: rbac.authorization.k8s.io/v1beta1 kind: ClusterRole metadata: name: filebeat labels: k8s-app: filebeat rules: - apiGroups: [""] # "" indicates the core API group resources: - namespaces - pods verbs: - get - watch - list --- apiVersion: v1 kind: ServiceAccount metadata: name: filebeat namespace: kube-system labels: k8s-app: filebeat --- apiVersion: v1 kind: ConfigMap metadata: name: filebeat-inputs namespace: kube-system labels: k8s-app: filebeat data: kubernetes.yml: |- - type: docker containers.ids: - "*" processors: - add_kubernetes_metadata: in_cluster: true - drop_event: when: equals: kubernetes.labels.app: nginx-ingress exclude_lines: ['kube-probe'] --- apiVersion: v1 kind: ConfigMap metadata: name: filebeat-config namespace: kube-system labels: k8s-app: filebeat data: filebeat.yml: |- filebeat.config: inputs: # Mounted `filebeat-inputs` configmap: path: ${path.config}/inputs.d/*.yml # Reload inputs configs as they change: reload.enabled: false modules: path: ${path.config}/modules.d/*.yml # Reload module configs as they change: reload.enabled: false filebeat.autodiscover: providers: - type: kubernetes templates: - condition: equals: kubernetes.labels.app: nginx-ingress config: - module: nginx access: input: type: docker containers.ids: - "${data.kubernetes.container.id}" processors: - add_cloud_metadata: fields: logtype: kubernetes kubernetes.cluster.name: aks-ae-01 fields_under_root: true output.elasticsearch: hosts: ["es-master.elasticsearch.svc.cluster.local:9200"] default.json: |- { "description": "Pipeline for parsing Nginx access logs. Requires the geoip and user_agent plugins.", "processors": [{ "grok": { "field": "message", "patterns":[ "\"?%{IP_LIST:nginx.access.remote_ip_list} - \\[%{IP_LIST:nginx.access.x_forwarded_for}\\] - %{DATA:nginx.access.user_name} \\[%{HTTPDATE:nginx.access.time}\\] \"%{GREEDYDATA:nginx.access.info}\" %{NUMBER:nginx.access.response_code} (?:%{NUMBER:nginx.access.body_sent.bytes}|-) (?:\"(?:%{DATA:nginx.access.referrer}|-)\") \"%{DATA:nginx.access.agent}\" %{NUMBER:nginx.access.request_length} %{NUMBER:nginx.access.response_time} %{DATA:nginx.access.upstream.proxy} %{NUMBER:nginx.access.upstream.bytes_sent} %{NUMBER:nginx.access.upstream.response_time} %{NUMBER:nginx.access.upstream.response}", "\"?%{IP_LIST:nginx.access.remote_ip_list} - %{DATA:nginx.access.user_name} \\[%{HTTPDATE:nginx.access.time}\\] \"%{GREEDYDATA:nginx.access.info}\" %{NUMBER:nginx.access.response_code} %{NUMBER:nginx.access.body_sent.bytes} \"%{DATA:nginx.access.referrer}\" \"%{DATA:nginx.access.agent}\"" ], "pattern_definitions": { "IP_LIST": "%{IP}(\"?,?\\s*%{IP})*" }, "ignore_missing": true } }, { "grok": { "field": "nginx.access.info", "patterns": [ "%{WORD:nginx.access.method} %{DATA:nginx.access.url} HTTP/%{NUMBER:nginx.access.http_version}", "" ], "ignore_missing": true } }, { "remove": { "field": "nginx.access.info" } }, { "split": { "field": "nginx.access.remote_ip_list", "separator": "\"?,?\\s+" } }, { "script": { "lang": "painless", "inline": "boolean isPrivate(def ip) { try { StringTokenizer tok = new StringTokenizer(ip, '.'); int firstByte = Integer.parseInt(tok.nextToken()); int secondByte = Integer.parseInt(tok.nextToken());if (firstByte == 10) { return true; } if (firstByte == 192 && secondByte == 168) { return true; } if (firstByte == 172 && secondByte >= 16 && secondByte <= 31) { return true; } if (firstByte == 127) { return true; } return false; } catch (Exception e) { return false; } } def found = false; for (def item : ctx.nginx.access.remote_ip_list) { if (!isPrivate(item)) { ctx.nginx.access.remote_ip = item; found = true; break; } } if (!found) { ctx.nginx.access.remote_ip = ctx.nginx.access.remote_ip_list[0]; }" } }, { "rename": { "field": "@timestamp", "target_field": "read_timestamp" } }, { "date": { "field": "nginx.access.time", "target_field": "@timestamp", "formats": ["dd/MMM/YYYY:H:m:s Z"] } }, { "remove": { "field": "nginx.access.time" } }, { "user_agent": { "field": "nginx.access.agent", "target_field": "nginx.access.user_agent" } }, { "remove": { "field": "nginx.access.agent" } }, { "geoip": { "field": "nginx.access.remote_ip", "target_field": "nginx.access.geoip" } }], "on_failure" : [{ "set" : { "field" : "error.message", "value" : "{{ _ingest.on_failure_message }}" } }] }