spec/gitlab/qa/component/specs_spec.rb

# frozen_string_literal: true

describe Gitlab::QA::Component::Specs do
  let(:docker_command) { spy('docker command') }
  let(:suite) { spy('suite') }
  let(:stubbed_env) { {} }

  before do
    stub_const('Gitlab::QA::Docker::Command', docker_command)
    allow(Gitlab::QA::Runtime::Scenario.attributes).to receive(:include?).and_return(false)
    allow(Gitlab::QA::Runtime::Env).to receive(:variables_to_mask).and_return(nil)
  end

  around do |example|
    # In CI, QA_IMAGE could be set and modify the tests' behavior. Ensure we reset it to nil.
    ClimateControl.modify(QA_IMAGE: nil, **stubbed_env) { example.run }
  end

  describe '#perform' do
    it 'bind-mounts a docker socket' do
      described_class.perform do |specs|
        specs.suite = suite
        specs.release = spy('release', login_params: nil)
      end

      expect(docker_command).to have_received(:volume)
        .with('/var/run/docker.sock', '/var/run/docker.sock')
    end

    it 'bind-mounds volumes' do
      allow(SecureRandom).to receive(:hex).and_return('def456')
      allow(Gitlab::QA::Runtime::Env)
        .to receive_messages(host_artifacts_dir: '/tmp/gitlab-qa/gitlab-qa-run-2018-07-11-10-00-00-abc123',
          qa_rspec_report_path: '/qa/rspec')

      release = instance_double(
        Gitlab::QA::Release,
        edition: :ce,
        project_name: 'gitlab-ce',
        qa_image: 'gitlab-ce-qa',
        qa_tag: 'latest'
      )
      allow(release)
        .to receive(:login_params)
        .and_return(nil)

      described_class.perform do |specs|
        specs.suite = suite
        specs.release = release
      end

      expect(docker_command).to have_received(:volume)
        .with('/var/run/docker.sock', '/var/run/docker.sock')
      expect(docker_command).to have_received(:volume)
        .with('/tmp/gitlab-qa/gitlab-qa-run-2018-07-11-10-00-00-abc123/gitlab-ce-qa-def456', File.join(
          Gitlab::QA::Docker::Volumes::QA_CONTAINER_WORKDIR, 'tmp'))
      expect(docker_command).to have_received(:volume)
        .with('/qa/rspec', File.join(Gitlab::QA::Docker::Volumes::QA_CONTAINER_WORKDIR, 'rspec'))
    end

    context 'with test suite class environment variable set' do
      let(:stubbed_env) { { "QA_SUITE_CLASS_NAME" => "Test::Instance::Create" } }
      let(:docker_engine) { spy('docker engine') }

      before do
        stub_const('Gitlab::QA::Docker::Engine', docker_engine)
      end

      it 'uses scenario class from environment variable' do
        described_class.perform do |specs|
          specs.suite = suite
          specs.release = spy('release', login_params: nil)
        end

        expect(docker_engine).to have_received(:run)
          .with(a_hash_including(args: ["Test::Instance::Create"]))
      end
    end

    context 'when preparing QA image' do
      let(:docker_engine) { spy('docker engine') }
      let(:qa_image) { 'gitlab/gitlab-ce-qa' }
      let(:qa_tag) { 'latest' }
      let(:release) do
        double('release', project_name: 'gitlab-ce', qa_image: qa_image, qa_tag: qa_tag, login_params: nil)
      end

      before do
        stub_const('Gitlab::QA::Docker::Engine', docker_engine)
      end

      context 'when not skipping Docker pulls' do
        before do
          allow(Gitlab::QA::Runtime::Env).to receive(:skip_pull?).and_return(false)
        end

        it 'pulls the QA image' do
          described_class.perform do |specs|
            specs.suite = suite
            specs.release = release
          end

          expect(docker_engine).to have_received(:pull)
            .with(image: "#{qa_image}:#{qa_tag}")
        end
      end

      context 'when skipping Docker pulls' do
        before do
          allow(Gitlab::QA::Runtime::Env).to receive(:skip_pull?).and_return(true)
        end

        it 'does not pull QA image' do
          described_class.perform do |specs|
            specs.suite = suite
            specs.release = release
          end

          expect(docker_engine).not_to have_received(:pull)
        end
      end

      context 'when Runtime::Scenario.qa_image is set' do
        let(:custom_qa_image) { 'custom-qa-image:v1' }

        before do
          stub_const('Gitlab::QA::Runtime::Scenario', spy)
          allow(Gitlab::QA::Runtime::Scenario.attributes).to receive(:include?).and_return(true)
          allow(Gitlab::QA::Runtime::Scenario).to receive(:qa_image).and_return(custom_qa_image)
          allow(Gitlab::QA::Runtime::Env).to receive(:skip_pull?).and_return(false)
        end

        it 'pulls the custom QA image' do
          described_class.perform do |specs|
            specs.suite = suite
            specs.release = release
          end

          expect(docker_engine).to have_received(:pull)
            .with(image: custom_qa_image)
        end
      end
    end

    describe 'Docker::Engine#run arguments' do
      let(:docker_engine) { spy('docker engine') }

      before do
        stub_const('Gitlab::QA::Docker::Engine', docker_engine)
      end

      it 'accepts a GitLab image' do
        described_class.perform do |specs|
          specs.suite = suite
          specs.release = Gitlab::QA::Release.new('gitlab/gitlab-ce:foobar')
        end

        expect(docker_engine).to have_received(:run)
          .with(a_hash_including(image: 'gitlab/gitlab-ce-qa:foobar', args: [suite]))
      end

      it 'accepts a GitLab QA image' do
        described_class.perform do |specs|
          specs.suite = suite
          specs.release = Gitlab::QA::Release.new('gitlab/gitlab-ce-qa:foobar')
        end

        expect(docker_engine).to have_received(:run)
          .with(a_hash_including(image: 'gitlab/gitlab-ce-qa:foobar', args: [suite]))
      end

      context 'when Runtime::Scenario.qa_image is set' do
        let(:custom_qa_image) { 'custom-qa-image:v1' }

        before do
          stub_const('Gitlab::QA::Runtime::Scenario', spy)
          allow(Gitlab::QA::Runtime::Scenario.attributes).to receive(:include?).and_return(true)
          allow(Gitlab::QA::Runtime::Scenario).to receive(:qa_image).and_return(custom_qa_image)
        end

        it 'runs the custom QA image' do
          described_class.perform do |specs|
            specs.suite = suite
            specs.release = Gitlab::QA::Release.new('ce')
          end

          expect(docker_engine).to have_received(:run)
            .with(a_hash_including(image: custom_qa_image, args: [suite]))
        end
      end

      context 'when release.login_params are present' do
        it 'log into the container registry' do
          release = double('release', project_name: 'gitlab-ce', qa_image: 'gitlab/gitlab-ce-qa', qa_tag: 'latest',
            login_params: { foo: :bar })
          described_class.perform do |specs|
            specs.suite = suite
            specs.release = release
          end

          expect(docker_engine).to have_received(:login).with(release.login_params)
          expect(docker_engine).to have_received(:run)
            .with(a_hash_including(image: 'gitlab/gitlab-ce-qa:latest', args: [suite]))
        end
      end

      context 'when --no-tests is passed' do
        let(:scenario_spy) { spy('scenario') }

        before do
          stub_const('Gitlab::QA::Runtime::Scenario', scenario_spy)

          allow(scenario_spy).to receive(:run_tests).and_return(false)
        end

        it 'skips tests' do
          described_class.perform do |specs|
            specs.suite = suite
            specs.release = spy('release')
          end

          expect(docker_engine).not_to have_received(:run)
        end
      end

      context 'when --enable-feature is passed in args' do
        let(:args) { %w[http://abc.test --enable-feature a] }

        it 'mentions the feature flag and runs exactly once' do
          allow(Gitlab::QA::Runtime::Logger.logger).to receive(:info)

          described_class.perform do |specs|
            specs.suite = suite
            specs.args = args
            specs.release = Gitlab::QA::Release.new('EE')
          end

          expect(Gitlab::QA::Runtime::Logger.logger).to have_received(:info).with(/Running with feature flag/)
          expect(docker_engine).to have_received(:run).once
        end
      end

      context 'when --enable-feature, --disable-feature, and --set-feature-flags are passed in args' do
        let(:args) { %w[http://abc.test --disable-feature a --enable-feature b --set-feature-flags c=enable,d=disable] }
        let(:image) { 'gitlab/gitlab-ee-qa:nightly' }

        it 'runs once for each option' do
          described_class.perform do |specs|
            specs.suite = suite
            specs.args = args
            specs.release = Gitlab::QA::Release.new('EE')
          end

          expect(docker_engine).to have_received(:run)
            .with(a_hash_including(image: image, args: [suite, 'http://abc.test', '--disable-feature', 'a']))
          expect(docker_engine).to have_received(:run)
            .with(a_hash_including(image: image, args: [suite, 'http://abc.test', '--enable-feature', 'b']))
          expect(docker_engine).to have_received(:run)
            .with(a_hash_including(image: image, args: [suite, 'http://abc.test', '--set-feature-flags',
              'c=enable,d=disable']))
        end
      end

      context 'when rspec args are specified and no feature flags passed' do
        let(:args) { %w[http://abc.test -- file/path --tag focus] }

        it 'passes the args' do
          described_class.perform do |specs|
            specs.suite = suite
            specs.args = args
            specs.release = Gitlab::QA::Release.new('EE')
          end
          expect(docker_engine).to have_received(:run)
            .with(a_hash_including(
              image: 'gitlab/gitlab-ee-qa:nightly',
              args: [suite, 'http://abc.test', '--', 'file/path', '--tag', 'focus']
            ))
        end

        it 'does not mention feature flags' do
          allow(Gitlab::QA::Runtime::Logger.logger).to receive(:info)

          described_class.perform do |specs|
            specs.suite = suite
            specs.release = Gitlab::QA::Release.new('EE')
            specs.args = args
          end

          expect(Gitlab::QA::Runtime::Logger.logger).not_to have_received(:info).with(/Running with feature flag/)
        end
      end

      context 'when hostname is toggled' do
        let(:hostname) { 'test.test' }

        before do
          allow(docker_engine).to receive(:run).and_yield(docker_command)
        end

        it 'passes to the container when set' do
          described_class.perform do |specs|
            specs.suite = suite
            specs.hostname = hostname
            specs.release = Gitlab::QA::Release.new('EE')
          end

          expect(docker_command).to have_received(:<<).with("--hostname #{hostname}")
          expect(docker_command).to have_received(:env).with('QA_HOSTNAME', hostname)
        end

        it 'does not pass to the container when unset' do
          described_class.perform do |specs|
            specs.suite = suite
            specs.release = Gitlab::QA::Release.new('EE')
          end

          expect(docker_command).not_to have_received(:<<).with("--hostname #{hostname}")
          expect(docker_command).not_to have_received(:env).with('QA_HOSTNAME', hostname)
        end
      end

      context 'when docker add hosts' do
        before do
          allow(docker_engine).to receive(:run).and_yield(docker_command)
        end

        it 'does not pass --add-hosts when Runtime::Env.docker_add_hosts is not set' do
          described_class.perform do |specs|
            specs.suite = suite
            specs.release = Gitlab::QA::Release.new('EE')
          end

          expect(docker_command).not_to have_received(:<<).with(/add-host/)
        end

        it 'passes --add-hosts when Runtime::Env.docker_add_hosts is set' do
          allow(Gitlab::QA::Runtime::Env)
            .to receive(:docker_add_hosts)
                  .and_return(%w[docker:93.184.216.34 gravatar.com:10.0.0.1])

          described_class.perform do |specs|
            specs.suite = suite
            specs.release = Gitlab::QA::Release.new('EE')
          end

          expect(docker_command).to have_received(:<<)
                                      .at_most(1)
                                      .with("--add-host docker:93.184.216.34 --add-host gravatar.com:10.0.0.1 ")
        end
      end

      context 'when file variables are set' do
        let(:secrets) { %w[secret private_stuff] }

        before do
          allow(docker_engine).to receive(:run).and_yield(docker_command)
          allow(Gitlab::QA::Runtime::Env).to receive(:variables_to_mask).and_return(secrets)
        end

        it 'masks values specified' do
          described_class.perform do |specs|
            specs.suite = suite
            specs.release = Gitlab::QA::Release.new('EE')
          end

          expect(docker_engine).to have_received(:run).with(a_hash_including(mask_secrets: secrets))
        end
      end
    end

    describe 'with retry enabled' do
      let(:docker_engine) { instance_double(Gitlab::QA::Docker::Engine, pull: nil, run: nil) }
      let(:artifacts_dir) { '/tmp/gitlab-qa/qa-run' }
      let(:args) { ['--', 'file/path', '--tag', 'some_tag'] }
      let(:spec_status) { 'failed' }

      let(:release) do
        instance_double(
          Gitlab::QA::Release,
          login_params: nil,
          edition: :ce,
          project_name: 'gitlab-ce',
          qa_image: 'gitlab-ce-qa',
          qa_tag: 'latest'
        )
      end

      def run_tests
        described_class.perform do |specs|
          specs.suite = suite
          specs.release = release
          specs.args = args
          specs.retry_failed_specs = true
        end
      end

      before do
        allow(File).to receive(:exist?).and_call_original
        allow(File).to receive(:exist?).with(/examples.txt$/).and_return(true)
        allow(File).to receive(:read).with(/examples.txt$/).and_return(<<~EXAMPLES)
          example_id            | status          | run_time     |
          --------------------- | --------------- | ------------ |
          ./some_spec.rb[1:1:1] | #{spec_status}  | 22.7 seconds |
        EXAMPLES

        allow(Gitlab::QA::Runtime::Env).to receive(:host_artifacts_dir).and_return(artifacts_dir)

        allow(Gitlab::QA::Docker::Engine).to receive(:new).and_return(docker_engine)
        allow(docker_engine).to receive(:run)
          .with(image: "#{release.qa_image}:#{release.qa_tag}", args: [suite, *args], mask_secrets: nil)
          .and_raise(Gitlab::QA::Support::ShellCommand::StatusError)
        allow(docker_engine).to receive(:run)
          .with(
            image: "#{release.qa_image}:#{release.qa_tag}", args: [suite, '--', '--only-failures'], mask_secrets: nil
          )
          .and_yield(docker_command)
      end

      context 'with failures in last run file' do
        it 'retries failed specs', :aggregate_failures do
          run_tests

          expect(docker_command).to have_received(:env).with('QA_RSPEC_RETRIED', 'true')
          expect(docker_command).to have_received(:env).with('NO_KNAPSACK', 'true')
          expect(docker_command).to have_received(:env).with(
            'RSPEC_LAST_RUN_RESULTS_FILE', '/home/gitlab/qa/tmp/examples.txt'
          )

          expect(docker_command).to have_received(:volume).with(
            %r{#{artifacts_dir}/\S+-retry}, '/home/gitlab/qa/tmp'
          )
          expect(docker_command).to have_received(:volume).with(
            %r{#{artifacts_dir}/\S+/examples.txt}, '/home/gitlab/qa/tmp/examples.txt'
          )
          expect(docker_engine).to have_received(:run).with(
            image: "#{release.qa_image}:#{release.qa_tag}", args: [suite, '--', '--only-failures'], mask_secrets: nil
          )
        end
      end

      context 'without failures in last run file' do
        let(:spec_status) { 'unknown' }

        it 'does not retry failed specs' do
          expect { run_tests }.to raise_error(Gitlab::QA::Support::ShellCommand::StatusError)
        end
      end
    end
  end
end