#!/usr/lib64/fluent/ruby/bin/ruby require 'json' require 'aws-sdk' require 'msgpack' require 'logger' #log = Logger.new("/tmp/debug.log", 3) #log.level = Logger::DEBUG def gunzip(data) sio = StringIO.new(data) gz = Zlib::GzipReader.new(sio) read_data = gz.read gz.close read_data end def get_trail_log(line) raw_log = JSON.load(line) json_log = raw_log['body'] trail_log = JSON.load(json_log) trail_row_log = trail_log['Message'] trail_row_log.each_line do |record| if record != "CloudTrail validation message." file = JSON.parse(record) gz_log = file['s3ObjectKey'].join AWS.config( :access_key_id => 'AKxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx', :secret_access_key => 'xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx', ) s3 = AWS::S3.new obj = s3.buckets['your_backet_name'].objects["#{gz_log}"] obj.read do |raw| trail_logs = JSON.parse(gunzip(raw)) return trail_logs end end end end while line = STDIN.gets.chomp trail_logs = get_trail_log(line) logs = trail_logs['Records'] log.info("#{logs}") logs.each do |log| parsed_log = JSON.generate(log) print parsed_log + "\n" end end