Skip to content

Instantly share code, notes, and snippets.

@spinscale

spinscale/ingest node example

Created January 18, 2017 11:55
Show Gist options

  • Select an option

    Learn more about clone URLs
  • Save spinscale/ff87ee3b604973cbd042a617dc4dfc5a to your computer and use it in GitHub Desktop.

Select an option

Learn more about clone URLs
Save spinscale/ff87ee3b604973cbd042a617dc4dfc5a to your computer and use it in GitHub Desktop.
Download ZIP

Revisions

  1. spinscale created this gist Jan 18, 2017.
    198 changes: 198 additions & 0 deletions ingest node example
    View file
    Original file line number Diff line number Diff line change
    @@ -0,0 +1,198 @@
    DELETE _all

    PUT _ingest/pipeline/rename_hostname
    {
    "processors": [
    {
    "rename": {
    "field": "hostname",
    "target_field": "host",
    "ignore_missing": true
    }
    }
    ]
    }

    PUT foo/bar/1
    {
    "hostname" : "host-001.example.org"
    }

    GET foo/bar/1

    PUT foo/bar/1?pipeline=rename_hostname
    {
    "hostname" : "host-001.example.org"
    }

    GET _ingest/pipeline/rename_hostname

    DELETE _ingest/pipeline/rename_hostname

    POST _ingest/pipeline/_simulate
    {
    "pipeline": {
    "description": "Ingest pipeline for Combined Log Format",
    "processors": [
    {
    "grok": {
    "field": "message",
    "patterns": [
    "%{IPORHOST:clientip} %{USER:ident} %{USER:auth} \\[%{HTTPDATE:timestamp}\\] \"%{WORD:verb} %{DATA:request} HTTP/%{NUMBER:httpversion}\" %{NUMBER:response:int} (?:-|%{NUMBER:bytes:int}) %{QS:referrer} %{QS:agent}"
    ]
    }
    },
    {
    "date": {
    "field": "timestamp",
    "formats": [
    "dd/MMM/YYYY:HH:mm:ss Z"
    ]
    }
    },
    {
    "geoip": {
    "field": "clientip"
    }
    },
    {
    "user_agent": {
    "field": "agent"
    }
    }
    ]
    },
    "docs": [
    {
    "_source": {
    "message": "212.87.37.154 - - [12/Sep/2016:16:21:15 +0000] \"GET /favicon.ico HTTP/1.1\" 200 3638 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/52.0.2743.116 Safari/537.36\""
    }
    }
    ]
    }

    # Script processor
    POST _ingest/pipeline/_simulate
    {
    "pipeline": {
    "processors": [
    {
    "script": {
    "inline": "ctx.bytes_total = ctx.bytes_in + ctx.bytes_out"

    }
    }
    ]
    },
    "docs": [
    {
    "_source": {
    "bytes_in": 1234,
    "bytes_out": 4321
    }
    }
    ]
    }


    # Foreach processor
    # Don't forget to show _ingest field
    POST _ingest/pipeline/_simulate
    {
    "pipeline": {
    "processors": [
    {
    "foreach": {
    "field" : "values",
    "processor" : {
    "convert" : {
    "field" : "_ingest._value.id",
    "type" : "integer"
    }
    }
    }
    }
    ]
    },
    "docs": [
    {
    "_source": {
    "values": [
    {"name": "first", "id": "1" },
    {"name": "second", "id": "2" },
    {"name": "third", "id": "3" }
    ]
    }
    }
    ]
    }


    # Setting metadata
    POST _ingest/pipeline/_simulate
    {
    "pipeline": {
    "processors": [
    {
    "set": {
    "field": "_id",
    "value": "foo"
    }
    },
    {
    "date_index_name": {
    "field": "_ingest.timestamp",
    "index_name_prefix": "whatever-",
    "date_rounding": "d"
    }
    }
    ]
    },
    "docs": [
    {
    "_source": {
    "foo": "bar"
    }
    }
    ]
    }


    # Handling failures, dead letter queue
    POST _ingest/pipeline/_simulate
    {
    "pipeline": {
    "description": "Ingest pipeline for Combined Log Format",
    "on_failure": [
    {
    "set": {
    "field": "_index",
    "value": "failed-{{ _index }}"
    }
    }
    ],
    "processors": [
    {
    "convert": {
    "field": "non-existing",
    "type": "integer"
    }
    }
    ]
    },
    "docs": [
    {
    "_index": "products",
    "_source": {
    "foo": "bar"
    }
    }
    ]
    }



    # Get stats
    GET /_nodes/stats/ingest