Created
August 20, 2025 13:38
-
-
Save rafael-gumiero/89b98694e8418df4c972491e5c3218dc to your computer and use it in GitHub Desktop.
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| version: "2" | |
| otlp-pipeline: | |
| source: | |
| otel_logs_source: | |
| path: "/${pipelineName}/v1/logs" | |
| processor: | |
| # 1. Rename timestamp fields | |
| - rename_keys: | |
| entries: | |
| - from_key: "time" | |
| to_key: "@timestamp" | |
| - from_key: "observedTimestamp" | |
| to_key: "observedTimestamp" | |
| # 2. Create severity object structure | |
| - rename_keys: | |
| entries: | |
| - from_key: "severityText" | |
| to_key: "severity/text" | |
| - from_key: "severityNumber" | |
| to_key: "severity/number" | |
| # 3. Parse JSON fields for module and runtime that are in attributes | |
| - parse_json: | |
| source: "attributes/log.attributes.module" | |
| destination: "attributes/log.attributes.module" | |
| overwrite_if_destination_exists: true | |
| delete_source: false | |
| normalize_keys: false | |
| tags_on_failure: ["parse_module_failed"] | |
| handle_failed_events: "skip" | |
| - parse_json: | |
| source: "attributes/log.attributes.runtime" | |
| destination: "attributes/log.attributes.runtime" | |
| overwrite_if_destination_exists: true | |
| delete_source: false | |
| normalize_keys: false | |
| tags_on_failure: ["parse_runtime_failed"] | |
| handle_failed_events: "skip" | |
| # 4. Move ALL log.attributes fields to temp_attributes (SS4O structure) | |
| - rename_keys: | |
| entries: | |
| - from_key: "attributes/log.attributes.status_code" | |
| to_key: "temp_attributes/status_code" | |
| - from_key: "attributes/log.attributes.module" | |
| to_key: "temp_attributes/module" | |
| - from_key: "attributes/log.attributes.network_type" | |
| to_key: "temp_attributes/network_type" | |
| - from_key: "attributes/log.attributes.severity" | |
| to_key: "temp_attributes/severity" | |
| - from_key: "attributes/log.attributes.duration" | |
| to_key: "temp_attributes/duration" | |
| - from_key: "attributes/log.attributes.x-request-id" | |
| to_key: "temp_attributes/x-request-id" | |
| - from_key: "attributes/log.attributes.url_path" | |
| to_key: "temp_attributes/url_path" | |
| - from_key: "attributes/log.attributes.request_method" | |
| to_key: "temp_attributes/request_method" | |
| - from_key: "attributes/log.attributes.consumed_at" | |
| to_key: "temp_attributes/consumed_at" | |
| - from_key: "attributes/log.attributes.server_address" | |
| to_key: "temp_attributes/server_address" | |
| - from_key: "attributes/log.attributes.session_id" | |
| to_key: "temp_attributes/session_id" | |
| - from_key: "attributes/log.attributes.created_at" | |
| to_key: "temp_attributes/created_at" | |
| - from_key: "attributes/log.attributes.producer_at" | |
| to_key: "temp_attributes/producer_at" | |
| - from_key: "attributes/log.attributes.completion_type" | |
| to_key: "temp_attributes/completion_type" | |
| - from_key: "attributes/log.attributes.runtime" | |
| to_key: "temp_attributes/runtime" | |
| - from_key: "attributes/log.attributes.enduser@id" | |
| to_key: "temp_attributes/enduser.id" | |
| - from_key: "attributes/log.attributes.k8s@cluster@name" | |
| to_key: "temp_attributes/k8s.cluster.name" | |
| - from_key: "attributes/log.attributes.trackback@stage" | |
| to_key: "temp_attributes/trackback.stage" | |
| - from_key: "attributes/log.attributes.trackback@result" | |
| to_key: "temp_attributes/trackback.result" | |
| - from_key: "attributes/log.attributes.trackback@label" | |
| to_key: "temp_attributes/trackback.label" | |
| - from_key: "attributes/log.attributes.trackback@scope" | |
| to_key: "temp_attributes/trackback.scope" | |
| - from_key: "attributes/log.attributes.exception_message" | |
| to_key: "temp_attributes/exception_message" | |
| # 5. Move ALL resource.attributes fields to resource/ (SS4O structure) | |
| - rename_keys: | |
| entries: | |
| - from_key: "attributes/resource.attributes.service_version_patch" | |
| to_key: "resource/service_version_patch" | |
| - from_key: "attributes/resource.attributes.service_distribution" | |
| to_key: "resource/service_distribution" | |
| - from_key: "attributes/resource.attributes.service_installation_id" | |
| to_key: "resource/service_installation_id" | |
| - from_key: "attributes/resource.attributes.service_environment" | |
| to_key: "resource/service_environment" | |
| - from_key: "attributes/resource.attributes.service_version_build" | |
| to_key: "resource/service_version_build" | |
| - from_key: "attributes/resource.attributes.service_version_major" | |
| to_key: "resource/service_version_major" | |
| - from_key: "attributes/resource.attributes.service_stack" | |
| to_key: "resource/service_stack" | |
| - from_key: "attributes/resource.attributes.service_version_minor" | |
| to_key: "resource/service_version_minor" | |
| - from_key: "attributes/resource.attributes.os_version_major" | |
| to_key: "resource/os_version_major" | |
| - from_key: "attributes/resource.attributes.os_version_minor" | |
| to_key: "resource/os_version_minor" | |
| - from_key: "attributes/resource.attributes.service_bundle_id" | |
| to_key: "resource/service_bundle_id" | |
| - from_key: "attributes/resource.attributes.os_api" | |
| to_key: "resource/os_api" | |
| - from_key: "attributes/resource.attributes.os_version_patch" | |
| to_key: "resource/os_version_patch" | |
| - from_key: "attributes/resource.attributes.device@manufacturer" | |
| to_key: "resource/device.manufacturer" | |
| - from_key: "attributes/resource.attributes.device@model@name" | |
| to_key: "resource/device.model.name" | |
| - from_key: "attributes/resource.attributes.device@score" | |
| to_key: "resource/device.score" | |
| - from_key: "attributes/resource.attributes.device@rooted" | |
| to_key: "resource/device.rooted" | |
| - from_key: "attributes/resource.attributes.service@name" | |
| to_key: "resource/service.name" | |
| - from_key: "attributes/resource.attributes.service@version" | |
| to_key: "resource/service.version" | |
| - from_key: "attributes/resource.attributes.event@domain" | |
| to_key: "resource/event.domain" | |
| - from_key: "attributes/resource.attributes.os@name" | |
| to_key: "resource/os.name" | |
| - from_key: "attributes/resource.attributes.os@version" | |
| to_key: "resource/os.version" | |
| # 6. Move instrumentationScope fields (to empty attributes completely) | |
| - rename_keys: | |
| entries: | |
| - from_key: "attributes/instrumentationScope.name" | |
| to_key: "instrumentationScope/name" | |
| - from_key: "attributes/instrumentationScope.version" | |
| to_key: "instrumentationScope/version" | |
| - from_key: "schemaUrl" | |
| to_key: "instrumentationScope/schemaUrl" | |
| # 7. Now attributes should be empty - rename temp_attributes to attributes (pending final solution) | |
| #- rename_keys: | |
| # entries: | |
| # - from_key: "temp_attributes" | |
| # to_key: "attributes" | |
| # 8. Final cleanup | |
| - delete_entries: | |
| with_keys: | |
| - "serviceName" | |
| - "schemaUrl" | |
| sink: | |
| - pipeline: | |
| name: "otel-logs-pipeline" | |
| otel-logs-pipeline: | |
| source: | |
| pipeline: | |
| name: "otlp-pipeline" | |
| sink: | |
| - opensearch: | |
| hosts: [ "https://vpc-xxx.us-east-1.es.amazonaws.com" ] | |
| aws: | |
| sts_role_arn: "arn:aws:iam::xxxx:role/role-osi-pipeline-otel-logs-aos" | |
| region: "us-east-1" | |
| serverless: false | |
| index_type: "management_disabled" | |
| index: "eks-otel-logs-use1-hom" | |
| action: "create" | |
| dlq: | |
| s3: | |
| bucket: "aws-s3-xxxx" | |
| key_path_prefix: "otel-logs-pipeline/logs/dlq" | |
| region: "us-east-1" | |
| sts_role_arn: "arn:aws:iam::xxxx:role/role-osi-pipeline-otel-logs-aos" |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment