Skip to content

Instantly share code, notes, and snippets.

@jerrac

jerrac/gitlab2elk.yml

Created March 23, 2015 22:19
Show Gist options

  • Select an option

    Learn more about clone URLs
  • Save jerrac/d19e49e77819ef82de6e to your computer and use it in GitHub Desktop.

Select an option

Learn more about clone URLs
Save jerrac/d19e49e77819ef82de6e to your computer and use it in GitHub Desktop.
Download ZIP

Revisions

  1. jerrac created this gist Mar 23, 2015.
    127 changes: 127 additions & 0 deletions gitlab2elk.yml
    View file
    Original file line number Diff line number Diff line change
    @@ -0,0 +1,127 @@
    # YAML config for these Ansible roles:
    # https://github.com/LaneCommunityCollege/aspects_logstash
    # https://github.com/LaneCommunityCollege/aspects_logstash_forwarder
    #
    # Since it's just straight config blocks, you should be able to just copy and paste what you need if you don't use
    # those Ansible roles.
    #
    # End result is multiline logs combined into one, and dates are parsed correctly.

    aspects_logstash_rules:
    patterns:
    gitlabpatterns: |
    GLAPPLOGTIME (?:Jan(?:uary)?|Feb(?:ruary)?|Mar(?:ch)?|Apr(?:il)?|May|Jun(?:e)?|Jul(?:y)?|Aug(?:ust)?|Sep(?:tember)?|Oct(?:ober)?|Nov(?:ember)?|Dec(?:ember)?)\s(?:(?:0[1-9])|(?:[12][0-9])|(?:3[01])|[1-9]),\s\d{4}\s(?:2[0123]|[01]?[0-9]):(?:[0-5][0-9])
    GLABTIME %{YEAR}-%{MONTHNUM}-%{MONTHDAY} %{HOUR}:%{MINUTE}:%{SECOND}
    filters:
    gitlabapplicationlog: |
    if [type] == "gitlabapplicationlog"{
    grok {
    match => ["message", "%{GLAPPLOGTIME:timestamp}"]
    patterns_dir => ["/etc/logstash/patterns"]
    }
    date {
    match => [ "timestamp", "MMM dd',' yyyy HH:mm" ]
    target => "@timestamp"
    }
    }
    gitlabhostlog: |
    if [type] == "gitlabhostlog"{
    multiline {
    pattern => "^%{GLAPPLOGTIME} "
    negate => true
    what => previous
    patterns_dir => ["/etc/logstash/patterns"]
    }
    grok {
    match => ["message", "%{GLAPPLOGTIME:timestamp}"]
    patterns_dir => ["/etc/logstash/patterns"]
    }
    date {
    match => [ "timestamp", "MMM dd',' yyyy HH:mm" ]
    target => "@timestamp"
    }
    }
    gitlabproductionlog: |
    if [type] == "gitlabproductionlog"{
    multiline {
    pattern => "Started"
    negate => true
    what => previous
    }
    grok { match => ["message", "%{TIMESTAMP_ISO8601:timestamp}"]
    patterns_dir => ["/etc/logstash/patterns"] }
    date {
    match => [ "timestamp", "YYYY-MM-dd HH:mm:ss" ]
    target => "@timestamp"
    }
    }
    gitlabsatlog: |
    if [type] == "gitlabsatlog"{
    grok {
    match => ["message", "%{GLAPPLOGTIME:timestamp}"]
    patterns_dir => ["/etc/logstash/patterns"]
    }
    date {
    match => [ "timestamp", "MMM dd',' yyyy HH:mm" ]
    target => "@timestamp"
    }
    }
    gitlabsidekiqlog: |
    if [type] == "gitlabsidekiqlog"{
    multiline {
    pattern => "^%{TIMESTAMP_ISO8601}"
    negate => true
    what => previous
    }
    grok { match => ["message", "%{TIMESTAMP_ISO8601:timestamp}"] }
    date {
    match => [ "timestamp", "YYYY-MM-dd'T'HH:mm:ss.SSSZ" ]
    target => "@timestamp"
    }
    }
    gitlabunicornerrlog: |
    if [type] == "gitlabunicornerrlog"{
    grok { match => ["message", "%{TIMESTAMP_ISO8601:timestamp}"] }
    date {
    match => [ "timestamp", "YYYY-MM-dd'T'HH:mm:ss.SSSSSS" ]
    target => "@timestamp"
    }
    }
    gitlabunicornoutlog: |
    if [type] == "gitlabunicornoutlog"{
    grok { match => ["message", "%{TIMESTAMP_ISO8601:timestamp}"] }
    date {
    match => [ "timestamp", "YYYY-MM-dd'T'HH:mm:ssZ" ]
    target => "@timestamp"
    }
    }
    aspects_logstash_forwarder_inputs:
    gitlabproductionlog: |
    {
    "paths": ["/home/git/gitlab/log/production.log"], "fields": { "type": "gitlabproductionlog"}
    }
    gitlabunicornerrlog: |
    {
    "paths": ["/home/git/gitlab/log/unicorn.stderr.log"], "fields": { "type": "gitlabunicornerrlog"}
    }
    gitlabunicornoutlog: |
    {
    "paths": ["/home/git/gitlab/log/unicorn.stdout.log"], "fields": { "type": "gitlabunicornoutlog"}
    }
    gitlabhostlog: |
    {
    "paths": ["/home/git/gitlab/log/githost.log"], "fields": { "type": "gitlabhostlog"}
    }
    gitlabsidekiqlog: |
    {
    "paths": ["/home/git/gitlab/log/sidekiq.log"], "fields": { "type": "gitlabsidekiqlog"}
    }
    gitlabsatlog: |
    {
    "paths": ["/home/git/gitlab/log/satellites.log"], "fields": { "type": "gitlabsatlog"}
    }
    gitlabapplicationlog: |
    {
    "paths": ["/home/git/gitlab/log/application.log"], "fields": { "type": "gitlabapplicationlog"}
    }