Merge pull request #14660 from Homebrew/faster_tests

workflows/tests: faster, more reliable tests.
2025-07-14 16:09:03 +08:00 · 2023-02-17 15:42:33 +00:00 · 2023-02-17 15:42:33 +00:00 · c9e0971a84
commit c9e0971a84
parent a8453c1dc0 a743e9277f
12 changed files with 134 additions and 216 deletions
--- a/.github/workflows/tests.yml
+++ b/.github/workflows/tests.yml
@ -13,6 +13,7 @@ env:
  HOMEBREW_DEVELOPER: 1
  HOMEBREW_NO_AUTO_UPDATE: 1
  HOMEBREW_NO_ENV_HINTS: 1
+  HOMEBREW_BOOTSNAP: 1

 concurrency:
  group: "${{ github.ref }}"
@ -70,13 +71,6 @@ jobs:

      - run: brew doctor

-      - name: Run brew update-tests
-        if: github.event_name == 'pull_request'
-        run: |
-          brew update-test
-          brew update-test --to-tag
-          brew update-test --commit=HEAD
-
      - name: Run brew readall on all taps
        run: brew readall --eval-all --aliases

@ -135,6 +129,7 @@ jobs:

  vendored-gems:
    name: vendored gems
+    needs: syntax
    runs-on: ubuntu-22.04
    steps:
      - name: Set up Homebrew
@ -189,9 +184,10 @@ jobs:
          docker push "homebrew/ubuntu22.04:master"

      - name: Build deprecated 16.04 Docker image
+        if: github.ref == 'refs/heads/master'
        working-directory: ${{ steps.set-up-homebrew.outputs.repository-path }}
        run: |
-          echo "homebrew/ubuntu16.04:master is deprecated and will soon be retired. Use homebrew/ubuntu22.04:master or homebrew/ubuntu16.04 or homebrew/brew. For CI, homebrew/ubuntu22.04:master is recommended." > .docker-deprecate
+          echo "homebrew/ubuntu16.04:master is deprecated and will soon be retired. Use homebrew/ubuntu22.04:master or homebrew/ubuntu18.04 or homebrew/brew. For CI, homebrew/ubuntu22.04:master is recommended." > .docker-deprecate
          docker build -t brew-deprecated --build-arg=version=16.04 \
               --label org.opencontainers.image.created="$(date --rfc-3339=seconds --utc)" \
               --label org.opencontainers.image.url="https://brew.sh" \
@ -211,6 +207,29 @@ jobs:
          docker tag brew-deprecated "homebrew/ubuntu16.04:master"
          docker push "homebrew/ubuntu16.04:master"

+  update-test:
+    name: ${{ matrix.name }}
+    runs-on: ${{ matrix.runs-on }}
+    needs: syntax
+    if: github.event_name == 'pull_request'
+    strategy:
+      matrix:
+        include:
+          - name: update-test (Ubuntu 22.04)
+            runs-on: ubuntu-22.04
+          - name: update-test (macOS 12)
+            runs-on: macos-12
+    steps:
+      - name: Set up Homebrew
+        id: set-up-homebrew
+        uses: Homebrew/actions/setup-homebrew@master
+
+      - name: Run brew update-tests
+        run: |
+          brew update-test
+          brew update-test --to-tag
+          brew update-test --commit=HEAD
+
  tests:
    name: ${{ matrix.name }}
    needs: syntax
@ -218,6 +237,9 @@ jobs:
    strategy:
      matrix:
        include:
+          - name: tests (online)
+            test-flags: --online --coverage
+            runs-on: ubuntu-22.04
          - name: tests (no-compatibility mode)
            test-flags: --no-compat --coverage
            runs-on: ubuntu-22.04
@ -225,11 +247,14 @@ jobs:
            test-flags: --generic --coverage
            runs-on: ubuntu-22.04
          - name: tests (Ubuntu 22.04)
-            test-flags: --online --coverage
+            test-flags: --coverage
            runs-on: ubuntu-22.04
          - name: tests (Ubuntu 18.04)
            test-flags: --coverage
            runs-on: ubuntu-18.04
+          - name: tests (macOS 12)
+            test-flags: --coverage
+            runs-on: macos-12
    steps:
      - name: Set up Homebrew
        id: set-up-homebrew
@ -256,118 +281,31 @@ jobs:
          restore-keys: ${{ runner.os }}-${{ matrix.test-flags }}-parallel_runtime_rspec-

      - name: Install brew tests dependencies
-        run: brew install curl
-
-      - name: Run brew tests
-        run: |
-          # brew tests doesn't like world writable directories
-          sudo chmod -R g-w,o-w /home/linuxbrew/.linuxbrew/Homebrew
-
-          brew tests ${{ matrix.test-flags }}
-        env:
-          HOMEBREW_GITHUB_API_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-
-      - uses: codecov/codecov-action@d9f34f8cd5cb3b3eb79b3e4b5dae3a16df499a70
-        with:
-          directory: ${{ steps.set-up-homebrew.outputs.repository-path }}
-          files: Library/Homebrew/test/coverage/coverage.xml
-
-  test-default-formula-linux:
-    name: ${{ matrix.name }}
-    if: startsWith(github.repository, 'Homebrew/')
-    runs-on: ${{ matrix.runs-on }}
-    env:
-      HOMEBREW_BOOTSNAP: 1
-    strategy:
-      matrix:
-        include:
-          - name: test default formula (Ubuntu 22.04)
-            test-flags: --online --coverage
-            runs-on: ubuntu-22.04
-          - name: test default formula (Ubuntu 18.04)
-            test-flags: --online --coverage
-            runs-on: ubuntu-18.04
-    steps:
-      - name: Set up Homebrew
-        id: set-up-homebrew
-        uses: Homebrew/actions/setup-homebrew@master
-
-      - run: brew test-bot --only-cleanup-before
-
-      - run: brew test-bot --only-formulae --only-json-tab --test-default-formula
-
-  test-everything:
-    name: test everything (macOS)
-    needs: syntax
-    if: startsWith(github.repository, 'Homebrew/')
-    runs-on: macos-12
-    env:
-      HOMEBREW_BOOTSNAP: 1
-    steps:
-      - name: Set up Homebrew
-        id: set-up-homebrew
-        uses: Homebrew/actions/setup-homebrew@master
-
-      - run: brew test-bot --only-cleanup-before
-
-      - run: brew config
-
-      # Can't cache this because we need to check that it doesn't fail the
-      # "uncommitted RubyGems" step with a cold cache.
-      - name: Install Bundler RubyGems
-        run: brew install-bundler-gems --groups=sorbet
-
-      - name: Check for uncommitted RubyGems
-        working-directory: ${{ steps.set-up-homebrew.outputs.repository-path }}
-        run: git diff --stat --exit-code Library/Homebrew/vendor/bundle/ruby
-
-      - run: brew doctor
-
-      - name: Run brew update-tests
-        if: github.event_name == 'pull_request'
-        run: |
-          brew update-test
-          brew update-test --to-tag
-          brew update-test --commit=HEAD
-
-      - name: Set up all Homebrew taps
-        run: |
-          brew tap homebrew/cask
-          brew tap homebrew/cask-drivers
-          brew tap homebrew/cask-fonts
-          brew tap homebrew/cask-versions
-          brew update-reset Library/Taps/homebrew/homebrew-bundle \
-                            Library/Taps/homebrew/homebrew-cask \
-                            Library/Taps/homebrew/homebrew-cask-versions \
-                            Library/Taps/homebrew/homebrew-services
-
-      - name: Run brew readall on all taps
-        run: brew readall --eval-all --aliases
-
-      - name: Install brew tests dependencies
+        if: matrix.name == 'tests (online)'
        run: brew install subversion curl

-      - name: Create parallel test log directory
-        run: mkdir tests
-
-      - name: Cache parallel tests log
-        uses: actions/cache@v1
-        with:
-          path: tests
-          key: ${{ runner.os }}-parallel_runtime_rspec-${{ github.sha }}
-          restore-keys: ${{ runner.os }}-parallel_runtime_rspec-
+        # brew tests doesn't like world writable directories
+      - name: Cleanup permissions
+        if: runner.os == 'Linux'
+        run: sudo chmod -R g-w,o-w /home/linuxbrew/.linuxbrew/Homebrew

      - name: Run brew tests
        run: |
+          # Don't attempt to use BuildPulse for generic tests.
+          # Fails for no apparent reason.
+          if [[ "${{ matrix.test-flags }}" =~ "--generic" ]]
+          then
+            unset HOMEBREW_BUILDPULSE_ACCESS_KEY_ID
+          fi
+
          # Retry multiple times when using BuildPulse to detect and submit
          # flakiness (because rspec-retry is disabled).
          if [[ -n "${HOMEBREW_BUILDPULSE_ACCESS_KEY_ID}" ]]
          then
-            brew tests --online --coverage ||
-            brew tests --online --coverage ||
-            brew tests --online --coverage
+            brew tests ${{ matrix.test-flags }} ||
+            brew tests ${{ matrix.test-flags }}
          else
-            brew tests --online --coverage
+            brew tests ${{ matrix.test-flags }}
          fi
        env:
          HOMEBREW_GITHUB_API_TOKEN: ${{ secrets.GITHUB_TOKEN }}
@ -378,9 +316,30 @@ jobs:
          HOMEBREW_BUILDPULSE_ACCOUNT_ID: 1503512
          HOMEBREW_BUILDPULSE_REPOSITORY_ID: 53238813

-      - run: brew test-bot --only-formulae --test-default-formula
-
      - uses: codecov/codecov-action@d9f34f8cd5cb3b3eb79b3e4b5dae3a16df499a70
        with:
          directory: ${{ steps.set-up-homebrew.outputs.repository-path }}
          files: Library/Homebrew/test/coverage/coverage.xml
+
+  test-default-formula:
+    name: ${{ matrix.name }}
+    needs: syntax
+    if: startsWith(github.repository, 'Homebrew/')
+    runs-on: ${{ matrix.runs-on }}
+    strategy:
+      matrix:
+        include:
+          - name: test default formula (Ubuntu 22.04)
+            runs-on: ubuntu-22.04
+          - name: test default formula (Ubuntu 18.04)
+            runs-on: ubuntu-18.04
+          - name: test default formula (macOS 12)
+            runs-on: macos-12
+    steps:
+      - name: Set up Homebrew
+        id: set-up-homebrew
+        uses: Homebrew/actions/setup-homebrew@master
+
+      - run: brew test-bot --only-cleanup-before
+
+      - run: brew test-bot --only-formulae --only-json-tab --test-default-formula
--- a/Library/Homebrew/.rspec_parallel
+++ b/Library/Homebrew/.rspec_parallel
@ -1,4 +1,4 @@
--format NoSeedProgressFormatter
+--format QuietProgressFormatter
 --format ParallelTests::RSpec::RuntimeLogger
 --out <%= ENV["PARALLEL_RSPEC_LOG_PATH"] %>
 --format RspecJunitFormatter
--- a/Library/Homebrew/.simplecov
+++ b/Library/Homebrew/.simplecov
@ -14,8 +14,8 @@ SimpleCov.start do
  enable_coverage :branch

  # We manage the result cache ourselves and the default of 10 minutes can be
-  # too low (particularly on Travis CI), causing results from some integration
-  # tests to be dropped. This causes random fluctuations in test coverage.
+  # too low causing results from some integration tests to be dropped. This
+  # causes random fluctuations in test coverage.
  merge_timeout 86400

  at_fork do
@ -30,7 +30,7 @@ SimpleCov.start do

  if ENV["HOMEBREW_INTEGRATION_TEST"]
    # This needs a unique name so it won't be overwritten
-    command_name "i#{ENV.fetch("TEST_ENV_NUMBER", $PROCESS_ID)}"
+    command_name "brew_integration_#{ENV.fetch("TEST_ENV_NUMBER", $PROCESS_ID)}"

    # be quiet, the parent process will be in charge of output and checking coverage totals
    SimpleCov.print_error_status = false
@ -51,7 +51,7 @@ SimpleCov.start do
      raise if $ERROR_INFO.is_a?(SystemExit)
    end
  else
-    command_name "b#{ENV.fetch("TEST_ENV_NUMBER", $PROCESS_ID)}"
+    command_name "brew_#{ENV.fetch("TEST_ENV_NUMBER", $PROCESS_ID)}"

    # Not using this during integration tests makes the tests 4x times faster
    # without changing the coverage.
--- a/Library/Homebrew/dev-cmd/tests.rb
+++ b/Library/Homebrew/dev-cmd/tests.rb
@ -170,6 +170,12 @@ module Homebrew
      end
      # rubocop:enable Homebrew/MoveToExtendOS

+      bundle_args << "--tag" << "~needs_network" unless args.online?
+      unless ENV["CI"]
+        bundle_args << "--tag" << "~needs_ci" \
+                    << "--tag" << "~needs_svn"
+      end
+
      puts "Randomized with seed #{seed}"

      # Submit test flakiness information using BuildPulse
--- a/Library/Homebrew/test/cmd/search_spec.rb
+++ b/Library/Homebrew/test/cmd/search_spec.rb
@ -7,20 +7,7 @@ require "cmd/shared_examples/args_parse"
 describe "brew search" do
  it_behaves_like "parseable arguments"

-  # TODO: consider re-enabling when/if we can make these not flaky.
-  #
-  # it "falls back to a GitHub tap search when no formula is found",
-  #    :integration_test, :needs_macos, :needs_network do
-  #   setup_test_formula "testball"
-  #   setup_remote_tap "homebrew/cask"
-
-  #   expect { brew "search", "homebrew/cask/firefox" }
-  #     .to output(/firefox/).to_stdout
-  #     .and be_a_success
-  # end
-
-  # doesn't actually need Linux but only want one integration test per-OS.
-  it "finds formula in search", :integration_test, :need_linux do
+  it "finds formula in search", :integration_test do
    setup_test_formula "testball"

    expect { brew "search", "testball" }
--- a/Library/Homebrew/test/cmd/services_spec.rb
+++ b/Library/Homebrew/test/cmd/services_spec.rb
@ -1,7 +1,7 @@
 # typed: false
 # frozen_string_literal: true

-describe "brew services", :integration_test, :needs_macos, :needs_network do
+describe "brew services", :integration_test, :needs_network do
  it "allows controlling services" do
    setup_remote_tap "homebrew/services"

--- a/Library/Homebrew/test/dev-cmd/bump_spec.rb
+++ b/Library/Homebrew/test/dev-cmd/bump_spec.rb
@ -7,27 +7,6 @@ describe "brew bump" do
  it_behaves_like "parseable arguments"

  describe "formula", :integration_test, :needs_homebrew_curl, :needs_network do
-    # TODO: consider re-enabling when/if we can make these not flaky.
-    #
-    #   it "returns data for single valid specified formula" do
-    #     install_test_formula "testball"
-
-    #     expect { brew "bump", "testball" }
-    #       .to output.to_stdout
-    #       .and not_to_output.to_stderr
-    #       .and be_a_success
-    #   end
-
-    #   it "returns data for multiple valid specified formula" do
-    #     install_test_formula "testball"
-    #     install_test_formula "testball2"
-
-    #     expect { brew "bump", "testball", "testball2" }
-    #       .to output.to_stdout
-    #       .and not_to_output.to_stderr
-    #       .and be_a_success
-    #   end
-
    it "returns no data and prints a message for HEAD-only formulae" do
      content = <<~RUBY
        desc "HEAD-only test formula"
--- a/Library/Homebrew/test/dev-cmd/prof_spec.rb
+++ b/Library/Homebrew/test/dev-cmd/prof_spec.rb
@ -6,25 +6,23 @@ require "cmd/shared_examples/args_parse"
 describe "brew prof" do
  it_behaves_like "parseable arguments"

-  # TODO: consider re-enabling when/if we can make these not flaky.
-  #
-  # describe "integration tests", :integration_test, :needs_network do
-  #   after do
-  #     FileUtils.rm_rf HOMEBREW_LIBRARY_PATH/"prof"
-  #   end
-
-  #   it "works using ruby-prof (the default)" do
-  #     expect { brew "prof", "help", "HOMEBREW_BROWSER" => "echo" }
-  #       .to output(/^Example usage:/).to_stdout
-  #       .and not_to_output.to_stderr
-  #       .and be_a_success
-  #   end
-
-  #   it "works using stackprof" do
-  #     expect { brew "prof", "--stackprof", "help", "HOMEBREW_BROWSER" => "echo" }
-  #       .to output(/^Example usage:/).to_stdout
-  #       .and not_to_output.to_stderr
-  #       .and be_a_success
-  #   end
-  # end
+  describe "integration tests", :integration_test, :needs_network do
+    after do
+      FileUtils.rm_rf HOMEBREW_LIBRARY_PATH/"prof"
+    end
+
+    it "works using ruby-prof (the default)" do
+      expect { brew "prof", "help", "HOMEBREW_BROWSER" => "echo" }
+        .to output(/^Example usage:/).to_stdout
+        .and not_to_output.to_stderr
+        .and be_a_success
+    end
+
+    it "works using stackprof" do
+      expect { brew "prof", "--stackprof", "help", "HOMEBREW_BROWSER" => "echo" }
+        .to output(/^Example usage:/).to_stdout
+        .and not_to_output.to_stderr
+        .and be_a_success
+    end
+  end
 end
--- a/Library/Homebrew/test/os/mac/pkgconfig_spec.rb
+++ b/Library/Homebrew/test/os/mac/pkgconfig_spec.rb
@ -13,7 +13,7 @@
 # Additionally, libffi version detection cannot be performed on systems running Mojave or earlier.
 #
 # For indeterminable cases, consult https://opensource.apple.com for the version used.
-describe "pkg-config" do
+describe "pkg-config", :needs_ci do
  def pc_version(library)
    path = HOMEBREW_LIBRARY_PATH/"os/mac/pkgconfig/#{MacOS.version}/#{library}.pc"
    version = File.foreach(path)
--- a/Library/Homebrew/test/spec_helper.rb
+++ b/Library/Homebrew/test/spec_helper.rb
@ -38,7 +38,7 @@ $LOAD_PATH.push(File.expand_path("#{ENV.fetch("HOMEBREW_LIBRARY")}/Homebrew/test

 require_relative "../global"

-require "test/support/no_seed_progress_formatter"
+require "test/support/quiet_progress_formatter"
 require "test/support/helper/cask"
 require "test/support/helper/fixtures"
 require "test/support/helper/formula"
@ -72,6 +72,10 @@ RSpec.configure do |config|

  config.silence_filter_announcements = true if ENV["TEST_ENV_NUMBER"]

+  # Improve backtrace formatting
+  config.filter_gems_from_backtrace "rspec-retry", "sorbet-runtime"
+  config.backtrace_exclusion_patterns << %r{test/spec_helper\.rb}
+
  config.expect_with :rspec do |c|
    c.max_formatted_output_length = 200
  end
@ -79,12 +83,6 @@ RSpec.configure do |config|
  # Use rspec-retry to handle flaky tests.
  config.default_sleep_interval = 1

-  # Don't make retries as noisy unless in CI.
-  if ENV["CI"]
-    config.verbose_retry = true
-    config.display_try_failure_messages = true
-  end
-
  # Don't want the nicer default retry behaviour when using BuildPulse to
  # identify flaky tests.
  config.default_retry_count = 2 unless ENV["BUILDPULSE"]
@ -132,12 +130,16 @@ RSpec.configure do |config|
    skip "Not running on macOS." unless OS.mac?
  end

+  config.before(:each, :needs_ci) do
+    skip "Not running on CI." unless ENV["CI"]
+  end
+
  config.before(:each, :needs_java) do
    skip "Java is not installed." unless which("java")
  end

  config.before(:each, :needs_python) do
-    skip "Python is not installed." unless which("python")
+    skip "Python is not installed." if !which("python3") && !which("python")
  end

  config.before(:each, :needs_network) do
--- a/Library/Homebrew/test/support/no_seed_progress_formatter.rb
+++ b/Library/Homebrew/test/support/no_seed_progress_formatter.rb
@ -3,8 +3,10 @@

 require "rspec/core/formatters/progress_formatter"

-class NoSeedProgressFormatter < RSpec::Core::Formatters::ProgressFormatter
+class QuietProgressFormatter < RSpec::Core::Formatters::ProgressFormatter
  RSpec::Core::Formatters.register self, :seed

+  def dump_summary(notification); end
  def seed(notification); end
+  def close(notification); end
 end
--- a/Library/Homebrew/test/utils/analytics_spec.rb
+++ b/Library/Homebrew/test/utils/analytics_spec.rb
@ -5,37 +5,37 @@ require "utils/analytics"
 require "formula_installer"

 describe Utils::Analytics do
-  describe "::label_google" do
  before do
    described_class.clear_additional_tags_cache
  end

+  describe "::label_google" do
    let(:ci) { ", CI" if ENV["CI"] }

    it "returns OS_VERSION and prefix when HOMEBREW_PREFIX is a custom prefix on intel" do
-      allow(Hardware::CPU).to receive(:type).and_return(:intel)
      allow(Hardware::CPU).to receive(:in_rosetta2?).and_return(false)
-      allow(Homebrew).to receive(:default_prefix?).and_return(false)
+      expect(Hardware::CPU).to receive(:type).and_return(:intel).at_least(:once)
+      expect(Homebrew).to receive(:default_prefix?).and_return(false)
      expect(described_class.label_google).to include described_class.custom_prefix_label_google
    end

    it "returns OS_VERSION, ARM and prefix when HOMEBREW_PREFIX is a custom prefix on arm" do
-      allow(Hardware::CPU).to receive(:type).and_return(:arm)
      allow(Hardware::CPU).to receive(:in_rosetta2?).and_return(false)
-      allow(Homebrew).to receive(:default_prefix?).and_return(false)
+      expect(Hardware::CPU).to receive(:type).and_return(:arm).at_least(:once)
+      expect(Homebrew).to receive(:default_prefix?).and_return(false).at_least(:once)
      expect(described_class.label_google).to include described_class.arch_label_google
      expect(described_class.label_google).to include described_class.custom_prefix_label_google
    end

    it "returns OS_VERSION, Rosetta and prefix when HOMEBREW_PREFIX is a custom prefix on Rosetta", :needs_macos do
-      allow(Hardware::CPU).to receive(:type).and_return(:intel)
      allow(Hardware::CPU).to receive(:in_rosetta2?).and_return(true)
-      allow(Homebrew).to receive(:default_prefix?).and_return(false)
+      expect(Hardware::CPU).to receive(:type).and_return(:intel).at_least(:once)
+      expect(Homebrew).to receive(:default_prefix?).and_return(false).at_least(:once)
      expect(described_class.label_google).to include described_class.custom_prefix_label_google
    end

    it "does not include prefix when HOMEBREW_PREFIX is the default prefix" do
-      allow(Homebrew).to receive(:default_prefix?).and_return(true)
+      expect(Homebrew).to receive(:default_prefix?).and_return(true)
      expect(described_class.label_google).not_to include HOMEBREW_PREFIX.to_s
    end

@ -46,24 +46,16 @@ describe Utils::Analytics do
  end

  describe "::additional_tags_influx" do
-    before do
-      described_class.clear_additional_tags_cache
-    end
-
    let(:ci) { ", CI" if ENV["CI"] }

    it "returns OS_VERSION and prefix when HOMEBREW_PREFIX is a custom prefix on intel" do
-      allow(Hardware::CPU).to receive(:type).and_return(:intel)
-      allow(Hardware::CPU).to receive(:in_rosetta2?).and_return(false)
-      allow(Homebrew).to receive(:default_prefix?).and_return(false)
+      expect(Homebrew).to receive(:default_prefix?).and_return(false).at_least(:once)
      expect(described_class.additional_tags_influx).to have_key(:prefix)
      expect(described_class.additional_tags_influx[:prefix]).to eq "custom-prefix"
    end

    it "returns OS_VERSION, ARM and prefix when HOMEBREW_PREFIX is a custom prefix on arm" do
-      allow(Hardware::CPU).to receive(:type).and_return(:arm)
-      allow(Hardware::CPU).to receive(:in_rosetta2?).and_return(false)
-      allow(Homebrew).to receive(:default_prefix?).and_return(false)
+      expect(Homebrew).to receive(:default_prefix?).and_return(false).at_least(:once)
      expect(described_class.additional_tags_influx).to have_key(:arch)
      expect(described_class.additional_tags_influx[:arch]).to eq HOMEBREW_PHYSICAL_PROCESSOR
      expect(described_class.additional_tags_influx).to have_key(:prefix)
@ -71,15 +63,13 @@ describe Utils::Analytics do
    end

    it "returns OS_VERSION, Rosetta and prefix when HOMEBREW_PREFIX is a custom prefix on Rosetta", :needs_macos do
-      allow(Hardware::CPU).to receive(:type).and_return(:intel)
-      allow(Hardware::CPU).to receive(:in_rosetta2?).and_return(true)
-      allow(Homebrew).to receive(:default_prefix?).and_return(false)
+      expect(Homebrew).to receive(:default_prefix?).and_return(false).at_least(:once)
      expect(described_class.additional_tags_influx).to have_key(:prefix)
      expect(described_class.additional_tags_influx[:prefix]).to eq "custom-prefix"
    end

    it "does not include prefix when HOMEBREW_PREFIX is the default prefix" do
-      allow(Homebrew).to receive(:default_prefix?).and_return(true)
+      expect(Homebrew).to receive(:default_prefix?).and_return(true).at_least(:once)
      expect(described_class.additional_tags_influx).to have_key(:prefix)
      expect(described_class.additional_tags_influx[:prefix]).to eq HOMEBREW_PREFIX.to_s
    end
@ -90,7 +80,7 @@ describe Utils::Analytics do
    end

    it "includes developer when ENV['HOMEBREW_DEVELOPER'] is set" do
-      ENV["HOMEBREW_DEVELOPER"] = "1"
+      expect(Homebrew::EnvConfig).to receive(:developer?).and_return(true)
      expect(described_class.additional_tags_influx).to have_key(:developer)
    end
  end
@ -100,10 +90,6 @@ describe Utils::Analytics do
    let(:options) { ["--head"].join }
    let(:action)  { "#{f.full_name} #{options}".strip }

-    before do
-      described_class.clear_additional_tags_cache
-    end
-
    context "when ENV vars is set" do
      it "returns nil when HOMEBREW_NO_ANALYTICS is true" do
        ENV["HOMEBREW_NO_ANALYTICS"] = "true"
@ -133,8 +119,8 @@ describe Utils::Analytics do
    it "passes to the influxdb and google methods" do
      ENV.delete("HOMEBREW_NO_ANALYTICS_THIS_RUN")
      ENV.delete("HOMEBREW_NO_ANALYTICS")
-      ENV.delete("HOMEBREW_DEVELOPER")
      ENV["HOMEBREW_ANALYTICS_DEBUG"] = "true"
+      expect(Homebrew::EnvConfig).to receive(:developer?).and_return(false)
      expect(described_class).to receive(:report_google).with(:event, hash_including(ea: action)).once
      expect(described_class).to receive(:report_influx).with(:install, "formula_name --head", false,
                                                              hash_including(developer: false)).once
@ -144,8 +130,8 @@ describe Utils::Analytics do
    it "sends to google twice on request" do
      ENV.delete("HOMEBREW_NO_ANALYTICS_THIS_RUN")
      ENV.delete("HOMEBREW_NO_ANALYTICS")
-      ENV.delete("HOMEBREW_DEVELOPER")
      ENV["HOMEBREW_ANALYTICS_DEBUG"] = "true"
+      expect(Homebrew::EnvConfig).to receive(:developer?).and_return(false)
      expect(described_class).to receive(:report_google).with(:event, hash_including(ea: action, ec: :install)).once
      expect(described_class).to receive(:report_google).with(:event,
                                                              hash_including(ea: action,
@ -166,7 +152,6 @@ describe Utils::Analytics do
      ENV.delete("HOMEBREW_NO_ANALYTICS_THIS_RUN")
      ENV.delete("HOMEBREW_NO_ANALYTICS")
      ENV["HOMEBREW_ANALYTICS_DEBUG"] = "true"
-      ENV["HOMEBREW_DEVELOPER"] = "1"
      expect(described_class).to receive(:deferred_curl).once
      described_class.report_influx(:install, action, true, developer: true, CI: true)
    end