-
-
Save anecula/e4a40a7eb473a39340b7518224ce9bad to your computer and use it in GitHub Desktop.
Pull down cloudwatch logs with boto
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # IF YOU INCUR HUGE COSTS WITH THIS OR IT BREAKS DON'T BLAME ME License | |
| # This is a throw-away script I wrote to pull the json events for all of the streams from a cloudwatch log | |
| # For some reason, the naive way to do vpc network logging does logging to different streams in a cloudwatch | |
| # log based on interface. | |
| # Great for diagnosing lots of things, and generating verbose logs, but for the broad-stroke analysis I was doing, | |
| # all I really wanted was the basic data. This would have been easier if I had logged to s3, but I did not see a | |
| # way to do that in 2 clicks. | |
| group_name = 'CHANGEME' | |
| import boto3, json, time | |
| client = boto3.client('logs') | |
| all_streams = [] | |
| stream_batch = client.describe_log_streams(logGroupName=group_name) | |
| all_streams += stream_batch['logStreams'] | |
| while 'nextToken' in stream_batch: | |
| stream_batch = client.describe_log_streams(logGroupName=group_name,nextToken=stream_batch['nextToken']) | |
| all_streams += stream_batch['logStreams'] | |
| print(len(all_streams)) | |
| stream_names = [stream['logStreamName'] for stream in all_streams] | |
| out_to = open(group_name + str(time.time()) + "cloud_logs.txt", 'w') | |
| for stream in stream_names: | |
| logs_batch = client.get_log_events(logGroupName=group_name, logStreamName=stream) | |
| for event in logs_batch['events']: | |
| event.update({'group': group_name, 'stream':stream }) | |
| out_to.write(json.dumps(event) + '\n') | |
| print(stream, ":", len(logs_batch['events'])) | |
| while 'nextToken' in logs_batch: | |
| logs_batch = client.get_log_events(logGroupName=group_name, logStreamName=stream, nextToken=logs_batch['nextToken']) | |
| for event in logs_batch['events']: | |
| event.update({'group': group_name, 'stream':stream }) | |
| out_to.write(json.dumps(event) + '\n') |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
Did you try export them to S3? or Feeding the logs to ES?