Merge pull request #14660 from Homebrew/faster_tests

workflows/tests: faster, more reliable tests.

.github/workflows/tests.yml

@@ -13,6 +13,7 @@ env:
   HOMEBREW_DEVELOPER: 1
   HOMEBREW_NO_AUTO_UPDATE: 1
   HOMEBREW_NO_ENV_HINTS: 1
+  HOMEBREW_BOOTSNAP: 1
 
 concurrency:
   group: "${{ github.ref }}"
@@ -70,13 +71,6 @@ jobs:
 
       - run: brew doctor
 
-      - name: Run brew update-tests
-        if: github.event_name == 'pull_request'
-        run: |
-          brew update-test
-          brew update-test --to-tag
-          brew update-test --commit=HEAD
-
       - name: Run brew readall on all taps
         run: brew readall --eval-all --aliases
 
@@ -135,6 +129,7 @@ jobs:
 
   vendored-gems:
     name: vendored gems
+    needs: syntax
     runs-on: ubuntu-22.04
     steps:
       - name: Set up Homebrew
@@ -189,9 +184,10 @@ jobs:
           docker push "homebrew/ubuntu22.04:master"
 
       - name: Build deprecated 16.04 Docker image
+        if: github.ref == 'refs/heads/master'
         working-directory: ${{ steps.set-up-homebrew.outputs.repository-path }}
         run: |
-          echo "homebrew/ubuntu16.04:master is deprecated and will soon be retired. Use homebrew/ubuntu22.04:master or homebrew/ubuntu16.04 or homebrew/brew. For CI, homebrew/ubuntu22.04:master is recommended." > .docker-deprecate
+          echo "homebrew/ubuntu16.04:master is deprecated and will soon be retired. Use homebrew/ubuntu22.04:master or homebrew/ubuntu18.04 or homebrew/brew. For CI, homebrew/ubuntu22.04:master is recommended." > .docker-deprecate
           docker build -t brew-deprecated --build-arg=version=16.04 \
                --label org.opencontainers.image.created="$(date --rfc-3339=seconds --utc)" \
                --label org.opencontainers.image.url="https://brew.sh" \
@@ -211,6 +207,29 @@ jobs:
           docker tag brew-deprecated "homebrew/ubuntu16.04:master"
           docker push "homebrew/ubuntu16.04:master"
 
+  update-test:
+    name: ${{ matrix.name }}
+    runs-on: ${{ matrix.runs-on }}
+    needs: syntax
+    if: github.event_name == 'pull_request'
+    strategy:
+      matrix:
+        include:
+          - name: update-test (Ubuntu 22.04)
+            runs-on: ubuntu-22.04
+          - name: update-test (macOS 12)
+            runs-on: macos-12
+    steps:
+      - name: Set up Homebrew
+        id: set-up-homebrew
+        uses: Homebrew/actions/setup-homebrew@master
+
+      - name: Run brew update-tests
+        run: |
+          brew update-test
+          brew update-test --to-tag
+          brew update-test --commit=HEAD
+
   tests:
     name: ${{ matrix.name }}
     needs: syntax
@@ -218,6 +237,9 @@ jobs:
     strategy:
       matrix:
         include:
+          - name: tests (online)
+            test-flags: --online --coverage
+            runs-on: ubuntu-22.04
           - name: tests (no-compatibility mode)
             test-flags: --no-compat --coverage
             runs-on: ubuntu-22.04
@@ -225,11 +247,14 @@ jobs:
             test-flags: --generic --coverage
             runs-on: ubuntu-22.04
           - name: tests (Ubuntu 22.04)
-            test-flags: --online --coverage
+            test-flags: --coverage
             runs-on: ubuntu-22.04
           - name: tests (Ubuntu 18.04)
             test-flags: --coverage
             runs-on: ubuntu-18.04
+          - name: tests (macOS 12)
+            test-flags: --coverage
+            runs-on: macos-12
     steps:
       - name: Set up Homebrew
         id: set-up-homebrew
@@ -256,118 +281,31 @@ jobs:
           restore-keys: ${{ runner.os }}-${{ matrix.test-flags }}-parallel_runtime_rspec-
 
       - name: Install brew tests dependencies
-        run: brew install curl
+        if: matrix.name == 'tests (online)'
-
-      - name: Run brew tests
-        run: |
-          # brew tests doesn't like world writable directories
-          sudo chmod -R g-w,o-w /home/linuxbrew/.linuxbrew/Homebrew
-
-          brew tests ${{ matrix.test-flags }}
-        env:
-          HOMEBREW_GITHUB_API_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-
-      - uses: codecov/codecov-action@d9f34f8cd5cb3b3eb79b3e4b5dae3a16df499a70
-        with:
-          directory: ${{ steps.set-up-homebrew.outputs.repository-path }}
-          files: Library/Homebrew/test/coverage/coverage.xml
-
-  test-default-formula-linux:
-    name: ${{ matrix.name }}
-    if: startsWith(github.repository, 'Homebrew/')
-    runs-on: ${{ matrix.runs-on }}
-    env:
-      HOMEBREW_BOOTSNAP: 1
-    strategy:
-      matrix:
-        include:
-          - name: test default formula (Ubuntu 22.04)
-            test-flags: --online --coverage
-            runs-on: ubuntu-22.04
-          - name: test default formula (Ubuntu 18.04)
-            test-flags: --online --coverage
-            runs-on: ubuntu-18.04
-    steps:
-      - name: Set up Homebrew
-        id: set-up-homebrew
-        uses: Homebrew/actions/setup-homebrew@master
-
-      - run: brew test-bot --only-cleanup-before
-
-      - run: brew test-bot --only-formulae --only-json-tab --test-default-formula
-
-  test-everything:
-    name: test everything (macOS)
-    needs: syntax
-    if: startsWith(github.repository, 'Homebrew/')
-    runs-on: macos-12
-    env:
-      HOMEBREW_BOOTSNAP: 1
-    steps:
-      - name: Set up Homebrew
-        id: set-up-homebrew
-        uses: Homebrew/actions/setup-homebrew@master
-
-      - run: brew test-bot --only-cleanup-before
-
-      - run: brew config
-
-      # Can't cache this because we need to check that it doesn't fail the
-      # "uncommitted RubyGems" step with a cold cache.
-      - name: Install Bundler RubyGems
-        run: brew install-bundler-gems --groups=sorbet
-
-      - name: Check for uncommitted RubyGems
-        working-directory: ${{ steps.set-up-homebrew.outputs.repository-path }}
-        run: git diff --stat --exit-code Library/Homebrew/vendor/bundle/ruby
-
-      - run: brew doctor
-
-      - name: Run brew update-tests
-        if: github.event_name == 'pull_request'
-        run: |
-          brew update-test
-          brew update-test --to-tag
-          brew update-test --commit=HEAD
-
-      - name: Set up all Homebrew taps
-        run: |
-          brew tap homebrew/cask
-          brew tap homebrew/cask-drivers
-          brew tap homebrew/cask-fonts
-          brew tap homebrew/cask-versions
-          brew update-reset Library/Taps/homebrew/homebrew-bundle \
-                            Library/Taps/homebrew/homebrew-cask \
-                            Library/Taps/homebrew/homebrew-cask-versions \
-                            Library/Taps/homebrew/homebrew-services
-
-      - name: Run brew readall on all taps
-        run: brew readall --eval-all --aliases
-
-      - name: Install brew tests dependencies
         run: brew install subversion curl
 
-      - name: Create parallel test log directory
+        # brew tests doesn't like world writable directories
-        run: mkdir tests
+      - name: Cleanup permissions
-
+        if: runner.os == 'Linux'
-      - name: Cache parallel tests log
+        run: sudo chmod -R g-w,o-w /home/linuxbrew/.linuxbrew/Homebrew
-        uses: actions/cache@v1
-        with:
-          path: tests
-          key: ${{ runner.os }}-parallel_runtime_rspec-${{ github.sha }}
-          restore-keys: ${{ runner.os }}-parallel_runtime_rspec-
 
       - name: Run brew tests
         run: |
+          # Don't attempt to use BuildPulse for generic tests.
+          # Fails for no apparent reason.
+          if [[ "${{ matrix.test-flags }}" =~ "--generic" ]]
+          then
+            unset HOMEBREW_BUILDPULSE_ACCESS_KEY_ID
+          fi
+
           # Retry multiple times when using BuildPulse to detect and submit
           # flakiness (because rspec-retry is disabled).
           if [[ -n "${HOMEBREW_BUILDPULSE_ACCESS_KEY_ID}" ]]
           then
-            brew tests --online --coverage ||
+            brew tests ${{ matrix.test-flags }} ||
-            brew tests --online --coverage ||
+            brew tests ${{ matrix.test-flags }}
-            brew tests --online --coverage
           else
-            brew tests --online --coverage
+            brew tests ${{ matrix.test-flags }}
           fi
         env:
           HOMEBREW_GITHUB_API_TOKEN: ${{ secrets.GITHUB_TOKEN }}
@@ -378,9 +316,30 @@ jobs:
           HOMEBREW_BUILDPULSE_ACCOUNT_ID: 1503512
           HOMEBREW_BUILDPULSE_REPOSITORY_ID: 53238813
 
-      - run: brew test-bot --only-formulae --test-default-formula
-
       - uses: codecov/codecov-action@d9f34f8cd5cb3b3eb79b3e4b5dae3a16df499a70
         with:
           directory: ${{ steps.set-up-homebrew.outputs.repository-path }}
           files: Library/Homebrew/test/coverage/coverage.xml
+
+  test-default-formula:
+    name: ${{ matrix.name }}
+    needs: syntax
+    if: startsWith(github.repository, 'Homebrew/')
+    runs-on: ${{ matrix.runs-on }}
+    strategy:
+      matrix:
+        include:
+          - name: test default formula (Ubuntu 22.04)
+            runs-on: ubuntu-22.04
+          - name: test default formula (Ubuntu 18.04)
+            runs-on: ubuntu-18.04
+          - name: test default formula (macOS 12)
+            runs-on: macos-12
+    steps:
+      - name: Set up Homebrew
+        id: set-up-homebrew
+        uses: Homebrew/actions/setup-homebrew@master
+
+      - run: brew test-bot --only-cleanup-before
+
+      - run: brew test-bot --only-formulae --only-json-tab --test-default-formula

Library/Homebrew/.rspec_parallel

@@ -1,4 +1,4 @@
---format NoSeedProgressFormatter
+--format QuietProgressFormatter
 --format ParallelTests::RSpec::RuntimeLogger
 --out <%= ENV["PARALLEL_RSPEC_LOG_PATH"] %>
 --format RspecJunitFormatter

Library/Homebrew/.simplecov

@@ -14,8 +14,8 @@ SimpleCov.start do
   enable_coverage :branch
 
   # We manage the result cache ourselves and the default of 10 minutes can be
-  # too low (particularly on Travis CI), causing results from some integration
+  # too low causing results from some integration tests to be dropped. This
-  # tests to be dropped. This causes random fluctuations in test coverage.
+  # causes random fluctuations in test coverage.
   merge_timeout 86400
 
   at_fork do
@@ -30,7 +30,7 @@ SimpleCov.start do
 
   if ENV["HOMEBREW_INTEGRATION_TEST"]
     # This needs a unique name so it won't be overwritten
-    command_name "i#{ENV.fetch("TEST_ENV_NUMBER", $PROCESS_ID)}"
+    command_name "brew_integration_#{ENV.fetch("TEST_ENV_NUMBER", $PROCESS_ID)}"
 
     # be quiet, the parent process will be in charge of output and checking coverage totals
     SimpleCov.print_error_status = false
@@ -51,7 +51,7 @@ SimpleCov.start do
       raise if $ERROR_INFO.is_a?(SystemExit)
     end
   else
-    command_name "b#{ENV.fetch("TEST_ENV_NUMBER", $PROCESS_ID)}"
+    command_name "brew_#{ENV.fetch("TEST_ENV_NUMBER", $PROCESS_ID)}"
 
     # Not using this during integration tests makes the tests 4x times faster
     # without changing the coverage.

Library/Homebrew/dev-cmd/tests.rb

@@ -170,6 +170,12 @@ module Homebrew
       end
       # rubocop:enable Homebrew/MoveToExtendOS
 
+      bundle_args << "--tag" << "~needs_network" unless args.online?
+      unless ENV["CI"]
+        bundle_args << "--tag" << "~needs_ci" \
+                    << "--tag" << "~needs_svn"
+      end
+
       puts "Randomized with seed #{seed}"
 
       # Submit test flakiness information using BuildPulse

Library/Homebrew/test/cmd/search_spec.rb

@@ -7,20 +7,7 @@ require "cmd/shared_examples/args_parse"
 describe "brew search" do
   it_behaves_like "parseable arguments"
 
-  # TODO: consider re-enabling when/if we can make these not flaky.
+  it "finds formula in search", :integration_test do
-  #
-  # it "falls back to a GitHub tap search when no formula is found",
-  #    :integration_test, :needs_macos, :needs_network do
-  #   setup_test_formula "testball"
-  #   setup_remote_tap "homebrew/cask"
-
-  #   expect { brew "search", "homebrew/cask/firefox" }
-  #     .to output(/firefox/).to_stdout
-  #     .and be_a_success
-  # end
-
-  # doesn't actually need Linux but only want one integration test per-OS.
-  it "finds formula in search", :integration_test, :need_linux do
     setup_test_formula "testball"
 
     expect { brew "search", "testball" }

Library/Homebrew/test/cmd/services_spec.rb

@@ -1,7 +1,7 @@
 # typed: false
 # frozen_string_literal: true
 
-describe "brew services", :integration_test, :needs_macos, :needs_network do
+describe "brew services", :integration_test, :needs_network do
   it "allows controlling services" do
     setup_remote_tap "homebrew/services"
 

Library/Homebrew/test/dev-cmd/bump_spec.rb

@@ -7,27 +7,6 @@ describe "brew bump" do
   it_behaves_like "parseable arguments"
 
   describe "formula", :integration_test, :needs_homebrew_curl, :needs_network do
-    # TODO: consider re-enabling when/if we can make these not flaky.
-    #
-    #   it "returns data for single valid specified formula" do
-    #     install_test_formula "testball"
-
-    #     expect { brew "bump", "testball" }
-    #       .to output.to_stdout
-    #       .and not_to_output.to_stderr
-    #       .and be_a_success
-    #   end
-
-    #   it "returns data for multiple valid specified formula" do
-    #     install_test_formula "testball"
-    #     install_test_formula "testball2"
-
-    #     expect { brew "bump", "testball", "testball2" }
-    #       .to output.to_stdout
-    #       .and not_to_output.to_stderr
-    #       .and be_a_success
-    #   end
-
     it "returns no data and prints a message for HEAD-only formulae" do
       content = <<~RUBY
         desc "HEAD-only test formula"

Library/Homebrew/test/dev-cmd/prof_spec.rb

@@ -6,25 +6,23 @@ require "cmd/shared_examples/args_parse"
 describe "brew prof" do
   it_behaves_like "parseable arguments"
 
-  # TODO: consider re-enabling when/if we can make these not flaky.
+  describe "integration tests", :integration_test, :needs_network do
-  #
+    after do
-  # describe "integration tests", :integration_test, :needs_network do
+      FileUtils.rm_rf HOMEBREW_LIBRARY_PATH/"prof"
-  #   after do
+    end
-  #     FileUtils.rm_rf HOMEBREW_LIBRARY_PATH/"prof"
+
-  #   end
+    it "works using ruby-prof (the default)" do
-
+      expect { brew "prof", "help", "HOMEBREW_BROWSER" => "echo" }
-  #   it "works using ruby-prof (the default)" do
+        .to output(/^Example usage:/).to_stdout
-  #     expect { brew "prof", "help", "HOMEBREW_BROWSER" => "echo" }
+        .and not_to_output.to_stderr
-  #       .to output(/^Example usage:/).to_stdout
+        .and be_a_success
-  #       .and not_to_output.to_stderr
+    end
-  #       .and be_a_success
+
-  #   end
+    it "works using stackprof" do
-
+      expect { brew "prof", "--stackprof", "help", "HOMEBREW_BROWSER" => "echo" }
-  #   it "works using stackprof" do
+        .to output(/^Example usage:/).to_stdout
-  #     expect { brew "prof", "--stackprof", "help", "HOMEBREW_BROWSER" => "echo" }
+        .and not_to_output.to_stderr
-  #       .to output(/^Example usage:/).to_stdout
+        .and be_a_success
-  #       .and not_to_output.to_stderr
+    end
-  #       .and be_a_success
+  end
-  #   end
-  # end
 end

Library/Homebrew/test/os/mac/pkgconfig_spec.rb

@@ -13,7 +13,7 @@
 # Additionally, libffi version detection cannot be performed on systems running Mojave or earlier.
 #
 # For indeterminable cases, consult https://opensource.apple.com for the version used.
-describe "pkg-config" do
+describe "pkg-config", :needs_ci do
   def pc_version(library)
     path = HOMEBREW_LIBRARY_PATH/"os/mac/pkgconfig/#{MacOS.version}/#{library}.pc"
     version = File.foreach(path)

Library/Homebrew/test/spec_helper.rb

@@ -38,7 +38,7 @@ $LOAD_PATH.push(File.expand_path("#{ENV.fetch("HOMEBREW_LIBRARY")}/Homebrew/test
 
 require_relative "../global"
 
-require "test/support/no_seed_progress_formatter"
+require "test/support/quiet_progress_formatter"
 require "test/support/helper/cask"
 require "test/support/helper/fixtures"
 require "test/support/helper/formula"
@@ -72,6 +72,10 @@ RSpec.configure do |config|
 
   config.silence_filter_announcements = true if ENV["TEST_ENV_NUMBER"]
 
+  # Improve backtrace formatting
+  config.filter_gems_from_backtrace "rspec-retry", "sorbet-runtime"
+  config.backtrace_exclusion_patterns << %r{test/spec_helper\.rb}
+
   config.expect_with :rspec do |c|
     c.max_formatted_output_length = 200
   end
@@ -79,12 +83,6 @@ RSpec.configure do |config|
   # Use rspec-retry to handle flaky tests.
   config.default_sleep_interval = 1
 
-  # Don't make retries as noisy unless in CI.
-  if ENV["CI"]
-    config.verbose_retry = true
-    config.display_try_failure_messages = true
-  end
-
   # Don't want the nicer default retry behaviour when using BuildPulse to
   # identify flaky tests.
   config.default_retry_count = 2 unless ENV["BUILDPULSE"]
@@ -132,12 +130,16 @@ RSpec.configure do |config|
     skip "Not running on macOS." unless OS.mac?
   end
 
+  config.before(:each, :needs_ci) do
+    skip "Not running on CI." unless ENV["CI"]
+  end
+
   config.before(:each, :needs_java) do
     skip "Java is not installed." unless which("java")
   end
 
   config.before(:each, :needs_python) do
-    skip "Python is not installed." unless which("python")
+    skip "Python is not installed." if !which("python3") && !which("python")
   end
 
   config.before(:each, :needs_network) do

Library/Homebrew/test/support/quiet_progress_formatter.rb

@@ -3,8 +3,10 @@
 
 require "rspec/core/formatters/progress_formatter"
 
-class NoSeedProgressFormatter < RSpec::Core::Formatters::ProgressFormatter
+class QuietProgressFormatter < RSpec::Core::Formatters::ProgressFormatter
   RSpec::Core::Formatters.register self, :seed
 
+  def dump_summary(notification); end
   def seed(notification); end
+  def close(notification); end
 end

Library/Homebrew/test/utils/analytics_spec.rb

@@ -5,37 +5,37 @@ require "utils/analytics"
 require "formula_installer"
 
 describe Utils::Analytics do
-  describe "::label_google" do
   before do
     described_class.clear_additional_tags_cache
   end
 
+  describe "::label_google" do
     let(:ci) { ", CI" if ENV["CI"] }
 
     it "returns OS_VERSION and prefix when HOMEBREW_PREFIX is a custom prefix on intel" do
-      allow(Hardware::CPU).to receive(:type).and_return(:intel)
       allow(Hardware::CPU).to receive(:in_rosetta2?).and_return(false)
-      allow(Homebrew).to receive(:default_prefix?).and_return(false)
+      expect(Hardware::CPU).to receive(:type).and_return(:intel).at_least(:once)
+      expect(Homebrew).to receive(:default_prefix?).and_return(false)
       expect(described_class.label_google).to include described_class.custom_prefix_label_google
     end
 
     it "returns OS_VERSION, ARM and prefix when HOMEBREW_PREFIX is a custom prefix on arm" do
-      allow(Hardware::CPU).to receive(:type).and_return(:arm)
       allow(Hardware::CPU).to receive(:in_rosetta2?).and_return(false)
-      allow(Homebrew).to receive(:default_prefix?).and_return(false)
+      expect(Hardware::CPU).to receive(:type).and_return(:arm).at_least(:once)
+      expect(Homebrew).to receive(:default_prefix?).and_return(false).at_least(:once)
       expect(described_class.label_google).to include described_class.arch_label_google
       expect(described_class.label_google).to include described_class.custom_prefix_label_google
     end
 
     it "returns OS_VERSION, Rosetta and prefix when HOMEBREW_PREFIX is a custom prefix on Rosetta", :needs_macos do
-      allow(Hardware::CPU).to receive(:type).and_return(:intel)
       allow(Hardware::CPU).to receive(:in_rosetta2?).and_return(true)
-      allow(Homebrew).to receive(:default_prefix?).and_return(false)
+      expect(Hardware::CPU).to receive(:type).and_return(:intel).at_least(:once)
+      expect(Homebrew).to receive(:default_prefix?).and_return(false).at_least(:once)
       expect(described_class.label_google).to include described_class.custom_prefix_label_google
     end
 
     it "does not include prefix when HOMEBREW_PREFIX is the default prefix" do
-      allow(Homebrew).to receive(:default_prefix?).and_return(true)
+      expect(Homebrew).to receive(:default_prefix?).and_return(true)
       expect(described_class.label_google).not_to include HOMEBREW_PREFIX.to_s
     end
 
@@ -46,24 +46,16 @@ describe Utils::Analytics do
   end
 
   describe "::additional_tags_influx" do
-    before do
-      described_class.clear_additional_tags_cache
-    end
-
     let(:ci) { ", CI" if ENV["CI"] }
 
     it "returns OS_VERSION and prefix when HOMEBREW_PREFIX is a custom prefix on intel" do
-      allow(Hardware::CPU).to receive(:type).and_return(:intel)
+      expect(Homebrew).to receive(:default_prefix?).and_return(false).at_least(:once)
-      allow(Hardware::CPU).to receive(:in_rosetta2?).and_return(false)
-      allow(Homebrew).to receive(:default_prefix?).and_return(false)
       expect(described_class.additional_tags_influx).to have_key(:prefix)
       expect(described_class.additional_tags_influx[:prefix]).to eq "custom-prefix"
     end
 
     it "returns OS_VERSION, ARM and prefix when HOMEBREW_PREFIX is a custom prefix on arm" do
-      allow(Hardware::CPU).to receive(:type).and_return(:arm)
+      expect(Homebrew).to receive(:default_prefix?).and_return(false).at_least(:once)
-      allow(Hardware::CPU).to receive(:in_rosetta2?).and_return(false)
-      allow(Homebrew).to receive(:default_prefix?).and_return(false)
       expect(described_class.additional_tags_influx).to have_key(:arch)
       expect(described_class.additional_tags_influx[:arch]).to eq HOMEBREW_PHYSICAL_PROCESSOR
       expect(described_class.additional_tags_influx).to have_key(:prefix)
@@ -71,15 +63,13 @@ describe Utils::Analytics do
     end
 
     it "returns OS_VERSION, Rosetta and prefix when HOMEBREW_PREFIX is a custom prefix on Rosetta", :needs_macos do
-      allow(Hardware::CPU).to receive(:type).and_return(:intel)
+      expect(Homebrew).to receive(:default_prefix?).and_return(false).at_least(:once)
-      allow(Hardware::CPU).to receive(:in_rosetta2?).and_return(true)
-      allow(Homebrew).to receive(:default_prefix?).and_return(false)
       expect(described_class.additional_tags_influx).to have_key(:prefix)
       expect(described_class.additional_tags_influx[:prefix]).to eq "custom-prefix"
     end
 
     it "does not include prefix when HOMEBREW_PREFIX is the default prefix" do
-      allow(Homebrew).to receive(:default_prefix?).and_return(true)
+      expect(Homebrew).to receive(:default_prefix?).and_return(true).at_least(:once)
       expect(described_class.additional_tags_influx).to have_key(:prefix)
       expect(described_class.additional_tags_influx[:prefix]).to eq HOMEBREW_PREFIX.to_s
     end
@@ -90,7 +80,7 @@ describe Utils::Analytics do
     end
 
     it "includes developer when ENV['HOMEBREW_DEVELOPER'] is set" do
-      ENV["HOMEBREW_DEVELOPER"] = "1"
+      expect(Homebrew::EnvConfig).to receive(:developer?).and_return(true)
       expect(described_class.additional_tags_influx).to have_key(:developer)
     end
   end
@@ -100,10 +90,6 @@ describe Utils::Analytics do
     let(:options) { ["--head"].join }
     let(:action)  { "#{f.full_name} #{options}".strip }
 
-    before do
-      described_class.clear_additional_tags_cache
-    end
-
     context "when ENV vars is set" do
       it "returns nil when HOMEBREW_NO_ANALYTICS is true" do
         ENV["HOMEBREW_NO_ANALYTICS"] = "true"
@@ -133,8 +119,8 @@ describe Utils::Analytics do
     it "passes to the influxdb and google methods" do
       ENV.delete("HOMEBREW_NO_ANALYTICS_THIS_RUN")
       ENV.delete("HOMEBREW_NO_ANALYTICS")
-      ENV.delete("HOMEBREW_DEVELOPER")
       ENV["HOMEBREW_ANALYTICS_DEBUG"] = "true"
+      expect(Homebrew::EnvConfig).to receive(:developer?).and_return(false)
       expect(described_class).to receive(:report_google).with(:event, hash_including(ea: action)).once
       expect(described_class).to receive(:report_influx).with(:install, "formula_name --head", false,
                                                               hash_including(developer: false)).once
@@ -144,8 +130,8 @@ describe Utils::Analytics do
     it "sends to google twice on request" do
       ENV.delete("HOMEBREW_NO_ANALYTICS_THIS_RUN")
       ENV.delete("HOMEBREW_NO_ANALYTICS")
-      ENV.delete("HOMEBREW_DEVELOPER")
       ENV["HOMEBREW_ANALYTICS_DEBUG"] = "true"
+      expect(Homebrew::EnvConfig).to receive(:developer?).and_return(false)
       expect(described_class).to receive(:report_google).with(:event, hash_including(ea: action, ec: :install)).once
       expect(described_class).to receive(:report_google).with(:event,
                                                               hash_including(ea: action,
@@ -166,7 +152,6 @@ describe Utils::Analytics do
       ENV.delete("HOMEBREW_NO_ANALYTICS_THIS_RUN")
       ENV.delete("HOMEBREW_NO_ANALYTICS")
       ENV["HOMEBREW_ANALYTICS_DEBUG"] = "true"
-      ENV["HOMEBREW_DEVELOPER"] = "1"
       expect(described_class).to receive(:deferred_curl).once
       described_class.report_influx(:install, action, true, developer: true, CI: true)
     end