Deep Mocking and stubbing with Minitest

bi_operations_concern.rb

# frozen_string_literal: true

module Mmspektrum
  module BiOperations
    # ENV["BIGQUERY_PROJECT"]     = "mmspektrum-bi"
    ENV["BIGQUERY_CREDENTIALS"] = ENV.fetch("GOOGLE_CLOUD_BIGQUERY_KEYFILE")

    class BqMockTable
      def insert(_data_rows)
        OpenStruct.new(insert_errors: [])
      end
    end

    module Banner
      extend ActiveSupport::Concern

      included do
      end

      def to_bq
        {
          banner_id: self.id,
          banner_company_id: self.mmspektrum_company_id,
          banner_company_title: self.company.title,
          banner_code: self.code,
          banner_target_url: self.target_url
        }
      end
    end

    # Common data helper methods
    def self.bq_datetime(time)
      time ? time.strftime("%Y-%m-%d %H:%M:%S") : nil
    end

    def self.bq_table(table:, dataset:)
      if Rails.env.development? || Rails.env.test?
        bq_table_mock
      else
        bigquery = Google::Cloud::Bigquery.new
        dataset = bigquery.dataset dataset
        dataset.table table
      end
    end

    def self.bq_table_mock
      BqMockTable.new
    end
  end
end

big_query_ads_insert_job.rb

# frozen_string_literal: true

module Mmspektrum
  module BigQuery
    module Ads
      class InsertJob < ApplicationJob
        def perform(banners, request_data)
          table = Mmspektrum::BiOperations.bq_table(table: "displayed_ads", dataset: ENV["GOOGLE_CLOUD_BIGQUERY_PROJECT"])
          request_bq = {
            request_url: request_data[:url],
            device: request_data[:device],
            visit_id: request_data[:visit_id],
            timestamp: Mmspektrum::BiOperations.bq_datetime(request_data[:timestamp])
          }

          rows = banners.collect { |banner| banner.to_bq.merge(request_bq) }

          response = table.insert(rows)

          if response.insert_errors.present?
            bq_log_insert_errors(response.insert_errors, "bigquery:#{table.table_id}")
          end
        end

        private
          def bq_log_insert_errors(errors, script_name)
            error_rows = errors.collect do |i_err_row|
              i_err_row.errors.collect do |error|
                {
                  script: script_name,
                  location: error["location"],
                  message: error["message"],
                  created_at: Mmspektrum::BiOperations.bq_datetime(Time.current)
                }
              end
            end.flatten

            table = Mmspektrum::BiOperations.bq_table(table: "insert_logs", dataset: ENV.fetch("BIGQUERY_LOG_DATASET"))
            response = table.insert(error_rows)

            if response.insert_errors.present?
              puts error_rows
              response.insert_errors.each do |row|
                row.errors.each do |e|
                  puts "ERROR: #{e["location"]} - #{e["message"]}"
                end
              end
            end
          end
      end
    end
  end
end

job_test.rb

# frozen_string_literal: true

require "test_helper"
require "minitest/mock"
require "google/cloud/bigquery"

module BigQuery
  module Ads
    class InsertJobTest < ActiveJob::TestCase
      attr_reader :table, :banners, :request_data

      setup do
        @banners = [
          create(:mmspektrum_ads_banner),
          create(:mmspektrum_ads_banner, :article_a)
        ]
        @request_data = { url: "htps://mmspektrum.cz/hoo",
                          device: :phone, # :desktop , : tablet
                          timestamp: Time.current - 1.minute,
                          visit_id:  "123" }
      end

      test "inserts data abouts Ads displayed" do
        mocks = insert_table_building_mocks(bq_return_value_ok, [expected_insert_bq_rows(banners, request_data)])

        Mmspektrum::BiOperations.stub(:bq_table, mocks.last) do
          perform_enqueued_jobs(only: Mmspektrum::BigQuery::Ads::InsertJob) do
            Mmspektrum::BigQuery::Ads::InsertJob.perform_later(banners, request_data)
          end
        end

        mocks.each(&:verify)
      end

      test "report errors" do
        logged_at_time = Time.current

        insert_mocks = insert_table_building_mocks(bq_return_value_bad, [expected_insert_bq_rows(banners, request_data)])
        error_logging_mocks = log_table_building_mocks(bq_return_value_ok, [expected_error_log_rows("bigquery:displayed_ads", logged_at_time)])

        bg_table_calls_return_values = [insert_mocks.last, error_logging_mocks.last]

        # one final stub => loging errors work with Time.current
        Time.stub(:current, logged_at_time) do
          # now we can stub the selecting method on module
          Mmspektrum::BiOperations.stub(:bq_table, -> (arguments) { bg_table_calls_return_values.shift.call(arguments) }) do
            perform_enqueued_jobs(only: Mmspektrum::BigQuery::Ads::InsertJob) do
              Mmspektrum::BigQuery::Ads::InsertJob.perform_later(banners, request_data)
            end
          end
        end

        [insert_mocks + error_logging_mocks].flatten.each(&:verify)
      end

      private
        def insert_table_building_mocks(insert_response, arguments, arguments_for_error_logging = nil)
          mocks = []
          mocks << insert_table_mock = mock(:insert, insert_response, arguments)

          # on errors for each `response.insert_errors` one call to table.table_id is done => we have to mock it
          insert_table_mock.expect(:table_id, "displayed_ads") if insert_response.insert_errors.present?

          # mocking class method `.bq_table`, not instance method => :call + stub(:class method)
          mocks << _insert_table_building_mock = mock(:call,
                                                      insert_table_mock,
                                                      [{ table: "displayed_ads",
                                                         dataset: ENV.fetch("GOOGLE_CLOUD_BIGQUERY_PROJECT") }])
          mocks
        end

        def log_table_building_mocks(log_response, arguments, arguments_for_error_logging = nil)
          mocks = []
          mocks << log_table_mock = mock(:insert, log_response, arguments)

          # mocking class method `.bq_table`, not instance method => :call + stub(:class method)
          mocks << _log_table_building_mock = mock(:call,
                                                   log_table_mock,
                                                   [{ table: "insert_logs",
                                                      dataset: ENV.fetch("BIGQUERY_LOG_DATASET") }])
          mocks
        end

        def mock(method, response, arguments)
          m = Minitest::Mock.new
          m.expect(method, response, arguments)
          m
        end

        def expected_insert_bq_rows(banners, request_data)
          banners.collect do |banner|
            {
              request_url: request_data[:url],
              device: request_data[:device],
              visit_id: request_data[:visit_id],
              timestamp: request_data[:timestamp].strftime("%Y-%m-%d %H:%M:%S"),
              banner_id: banner.id,
              banner_company_id: banner.mmspektrum_company_id,
              banner_company_title: banner.company.title,
              banner_code: banner.code,
              banner_target_url: banner.target_url
            }
          end
        end

        def bq_return_value_ok
          OpenStruct.new(insert_errors: [])
        end

        def bq_return_value_bad
          rows = [OpenStruct.new(errors: [errors.first, errors.second]), OpenStruct.new(errors: [errors.third])]
          OpenStruct.new(insert_errors: rows)
        end

        def errors
          [{ "location": "location_here", "message": "Something went wrong" },
           { "location": "location_somwhere_else", "message": "Ooops!" },
           { "location": "over_there", "message": "Another one bites the dust" }]
        end

        def expected_error_log_rows(script_name, logged_at_time)
          errors.collect do |error|
            {
              script: script_name,
              location: error["location"],
              message: error["message"],
              created_at: Mmspektrum::BiOperations.bq_datetime(logged_at_time)
            }
          end
        end
    end
  end
end