Skip to content

Instantly share code, notes, and snippets.

@foton

foton/bi_operations_concern.rb

Last active June 10, 2021 19:42
Show Gist options

  • Select an option

    Learn more about clone URLs
  • Save foton/a3846202363c768715ad8e446950e929 to your computer and use it in GitHub Desktop.

Select an option

Learn more about clone URLs
Save foton/a3846202363c768715ad8e446950e929 to your computer and use it in GitHub Desktop.
Download ZIP

Revisions

  1. foton revised this gist Jun 10, 2021. 1 changed file with 0 additions and 2 deletions.
    2 changes: 0 additions & 2 deletions big_query_ads_insert_job.rb
    View file
    Original file line number Diff line number Diff line change
    @@ -1,5 +1,3 @@

    ############################################## JOB UNDER TEST ###################################################
    # frozen_string_literal: true

    module Mmspektrum
  2. foton created this gist Jun 10, 2021.
    50 changes: 50 additions & 0 deletions bi_operations_concern.rb
    View file
    Original file line number Diff line number Diff line change
    @@ -0,0 +1,50 @@
    # frozen_string_literal: true

    module Mmspektrum
    module BiOperations
    # ENV["BIGQUERY_PROJECT"] = "mmspektrum-bi"
    ENV["BIGQUERY_CREDENTIALS"] = ENV.fetch("GOOGLE_CLOUD_BIGQUERY_KEYFILE")

    class BqMockTable
    def insert(_data_rows)
    OpenStruct.new(insert_errors: [])
    end
    end

    module Banner
    extend ActiveSupport::Concern

    included do
    end

    def to_bq
    {
    banner_id: self.id,
    banner_company_id: self.mmspektrum_company_id,
    banner_company_title: self.company.title,
    banner_code: self.code,
    banner_target_url: self.target_url
    }
    end
    end

    # Common data helper methods
    def self.bq_datetime(time)
    time ? time.strftime("%Y-%m-%d %H:%M:%S") : nil
    end

    def self.bq_table(table:, dataset:)
    if Rails.env.development? || Rails.env.test?
    bq_table_mock
    else
    bigquery = Google::Cloud::Bigquery.new
    dataset = bigquery.dataset dataset
    dataset.table table
    end
    end

    def self.bq_table_mock
    BqMockTable.new
    end
    end
    end
    55 changes: 55 additions & 0 deletions big_query_ads_insert_job.rb
    View file
    Original file line number Diff line number Diff line change
    @@ -0,0 +1,55 @@

    ############################################## JOB UNDER TEST ###################################################
    # frozen_string_literal: true

    module Mmspektrum
    module BigQuery
    module Ads
    class InsertJob < ApplicationJob
    def perform(banners, request_data)
    table = Mmspektrum::BiOperations.bq_table(table: "displayed_ads", dataset: ENV["GOOGLE_CLOUD_BIGQUERY_PROJECT"])
    request_bq = {
    request_url: request_data[:url],
    device: request_data[:device],
    visit_id: request_data[:visit_id],
    timestamp: Mmspektrum::BiOperations.bq_datetime(request_data[:timestamp])
    }

    rows = banners.collect { |banner| banner.to_bq.merge(request_bq) }

    response = table.insert(rows)

    if response.insert_errors.present?
    bq_log_insert_errors(response.insert_errors, "bigquery:#{table.table_id}")
    end
    end

    private
    def bq_log_insert_errors(errors, script_name)
    error_rows = errors.collect do |i_err_row|
    i_err_row.errors.collect do |error|
    {
    script: script_name,
    location: error["location"],
    message: error["message"],
    created_at: Mmspektrum::BiOperations.bq_datetime(Time.current)
    }
    end
    end.flatten

    table = Mmspektrum::BiOperations.bq_table(table: "insert_logs", dataset: ENV.fetch("BIGQUERY_LOG_DATASET"))
    response = table.insert(error_rows)

    if response.insert_errors.present?
    puts error_rows
    response.insert_errors.each do |row|
    row.errors.each do |e|
    puts "ERROR: #{e["location"]} - #{e["message"]}"
    end
    end
    end
    end
    end
    end
    end
    end
    133 changes: 133 additions & 0 deletions job_test.rb
    View file
    Original file line number Diff line number Diff line change
    @@ -0,0 +1,133 @@
    # frozen_string_literal: true

    require "test_helper"
    require "minitest/mock"
    require "google/cloud/bigquery"

    module BigQuery
    module Ads
    class InsertJobTest < ActiveJob::TestCase
    attr_reader :table, :banners, :request_data

    setup do
    @banners = [
    create(:mmspektrum_ads_banner),
    create(:mmspektrum_ads_banner, :article_a)
    ]
    @request_data = { url: "htps://mmspektrum.cz/hoo",
    device: :phone, # :desktop , : tablet
    timestamp: Time.current - 1.minute,
    visit_id: "123" }
    end

    test "inserts data abouts Ads displayed" do
    mocks = insert_table_building_mocks(bq_return_value_ok, [expected_insert_bq_rows(banners, request_data)])

    Mmspektrum::BiOperations.stub(:bq_table, mocks.last) do
    perform_enqueued_jobs(only: Mmspektrum::BigQuery::Ads::InsertJob) do
    Mmspektrum::BigQuery::Ads::InsertJob.perform_later(banners, request_data)
    end
    end

    mocks.each(&:verify)
    end

    test "report errors" do
    logged_at_time = Time.current

    insert_mocks = insert_table_building_mocks(bq_return_value_bad, [expected_insert_bq_rows(banners, request_data)])
    error_logging_mocks = log_table_building_mocks(bq_return_value_ok, [expected_error_log_rows("bigquery:displayed_ads", logged_at_time)])

    bg_table_calls_return_values = [insert_mocks.last, error_logging_mocks.last]

    # one final stub => loging errors work with Time.current
    Time.stub(:current, logged_at_time) do
    # now we can stub the selecting method on module
    Mmspektrum::BiOperations.stub(:bq_table, -> (arguments) { bg_table_calls_return_values.shift.call(arguments) }) do
    perform_enqueued_jobs(only: Mmspektrum::BigQuery::Ads::InsertJob) do
    Mmspektrum::BigQuery::Ads::InsertJob.perform_later(banners, request_data)
    end
    end
    end

    [insert_mocks + error_logging_mocks].flatten.each(&:verify)
    end

    private
    def insert_table_building_mocks(insert_response, arguments, arguments_for_error_logging = nil)
    mocks = []
    mocks << insert_table_mock = mock(:insert, insert_response, arguments)

    # on errors for each `response.insert_errors` one call to table.table_id is done => we have to mock it
    insert_table_mock.expect(:table_id, "displayed_ads") if insert_response.insert_errors.present?

    # mocking class method `.bq_table`, not instance method => :call + stub(:class method)
    mocks << _insert_table_building_mock = mock(:call,
    insert_table_mock,
    [{ table: "displayed_ads",
    dataset: ENV.fetch("GOOGLE_CLOUD_BIGQUERY_PROJECT") }])
    mocks
    end

    def log_table_building_mocks(log_response, arguments, arguments_for_error_logging = nil)
    mocks = []
    mocks << log_table_mock = mock(:insert, log_response, arguments)

    # mocking class method `.bq_table`, not instance method => :call + stub(:class method)
    mocks << _log_table_building_mock = mock(:call,
    log_table_mock,
    [{ table: "insert_logs",
    dataset: ENV.fetch("BIGQUERY_LOG_DATASET") }])
    mocks
    end

    def mock(method, response, arguments)
    m = Minitest::Mock.new
    m.expect(method, response, arguments)
    m
    end

    def expected_insert_bq_rows(banners, request_data)
    banners.collect do |banner|
    {
    request_url: request_data[:url],
    device: request_data[:device],
    visit_id: request_data[:visit_id],
    timestamp: request_data[:timestamp].strftime("%Y-%m-%d %H:%M:%S"),
    banner_id: banner.id,
    banner_company_id: banner.mmspektrum_company_id,
    banner_company_title: banner.company.title,
    banner_code: banner.code,
    banner_target_url: banner.target_url
    }
    end
    end

    def bq_return_value_ok
    OpenStruct.new(insert_errors: [])
    end

    def bq_return_value_bad
    rows = [OpenStruct.new(errors: [errors.first, errors.second]), OpenStruct.new(errors: [errors.third])]
    OpenStruct.new(insert_errors: rows)
    end

    def errors
    [{ "location": "location_here", "message": "Something went wrong" },
    { "location": "location_somwhere_else", "message": "Ooops!" },
    { "location": "over_there", "message": "Another one bites the dust" }]
    end

    def expected_error_log_rows(script_name, logged_at_time)
    errors.collect do |error|
    {
    script: script_name,
    location: error["location"],
    message: error["message"],
    created_at: Mmspektrum::BiOperations.bq_datetime(logged_at_time)
    }
    end
    end
    end
    end
    end