diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 55389578cb..49929681f0 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -307,10 +307,6 @@ jobs: key: ${{ runner.os }}-${{ matrix.test-flags }}-parallel_runtime_rspec-${{ github.sha }} restore-keys: ${{ runner.os }}-${{ matrix.test-flags }}-parallel_runtime_rspec- - - name: Install brew tests dependencies - if: matrix.name != 'tests (generic OS)' - run: brew install buildpulse-test-reporter - - name: Install brew tests --online dependencies if: matrix.name == 'tests (online)' run: brew install subversion curl @@ -330,18 +326,8 @@ jobs: - name: Run brew tests if: github.event_name == 'pull_request' || matrix.name != 'tests (online)' run: | - # brew tests - - # Don't attempt to use BuildPulse for generic tests. - # Fails for no apparent reason. - if [[ "${{ matrix.name }}" =~ "generic" ]] - then - unset HOMEBREW_BUILDPULSE_ACCESS_KEY_ID - fi - - # Retry multiple times when using BuildPulse to detect and submit - # flakiness (because rspec-retry is disabled). - if [[ -n "${HOMEBREW_BUILDPULSE_ACCESS_KEY_ID-}" ]] + # Retry multiple times to detect and submit flakiness to CodeCov (because rspec-retry is disabled). + if [[ -n "${CODECOV_TOKEN-}" ]] then brew tests ${{ matrix.test-flags }} || brew tests ${{ matrix.test-flags }} @@ -352,10 +338,7 @@ jobs: HOMEBREW_GITHUB_API_TOKEN: ${{ secrets.GITHUB_TOKEN }} # These cannot be queried at the macOS level on GitHub Actions. HOMEBREW_LANGUAGES: en-GB - HOMEBREW_BUILDPULSE_ACCESS_KEY_ID: ${{ secrets.BUILDPULSE_ACCESS_KEY_ID }} - HOMEBREW_BUILDPULSE_SECRET_ACCESS_KEY: ${{ secrets.BUILDPULSE_SECRET_ACCESS_KEY }} - HOMEBREW_BUILDPULSE_ACCOUNT_ID: 1503512 - HOMEBREW_BUILDPULSE_REPOSITORY_ID: 53238813 + CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} - name: Get RSpec JUnit XML filenames id: junit_xml diff --git a/Library/Homebrew/dev-cmd/tests.rb b/Library/Homebrew/dev-cmd/tests.rb index b45df9ac94..ec512f72cb 100644 --- a/Library/Homebrew/dev-cmd/tests.rb +++ b/Library/Homebrew/dev-cmd/tests.rb @@ -137,13 +137,6 @@ module Homebrew ENV["HOMEBREW_DEBUG"] = "1" if args.debug? # Used in spec_helper.rb to require the "debug" gem. - # Submit test flakiness information using BuildPulse - # BUILDPULSE used in spec_helper.rb - if use_buildpulse? - ENV["BUILDPULSE"] = "1" - ohai "Running tests with BuildPulse-friendly settings" - end - # Workaround for: # # ``` @@ -158,8 +151,6 @@ module Homebrew end success = $CHILD_STATUS.success? - run_buildpulse if use_buildpulse? - return if success Homebrew.failed = true @@ -168,38 +159,6 @@ module Homebrew private - sig { returns(T.nilable(T::Boolean)) } - def use_buildpulse? - return @use_buildpulse if defined?(@use_buildpulse) - - @use_buildpulse = T.let(ENV["HOMEBREW_BUILDPULSE_ACCESS_KEY_ID"].present? && - ENV["HOMEBREW_BUILDPULSE_SECRET_ACCESS_KEY"].present? && - ENV["HOMEBREW_BUILDPULSE_ACCOUNT_ID"].present? && - ENV["HOMEBREW_BUILDPULSE_REPOSITORY_ID"].present?, T.nilable(T::Boolean)) - end - - sig { void } - def run_buildpulse - require "formula" - - with_env(HOMEBREW_NO_AUTO_UPDATE: "1", HOMEBREW_NO_BOOTSNAP: "1") do - ensure_formula_installed!("buildpulse-test-reporter", - reason: "reporting test flakiness") - end - - ENV["BUILDPULSE_ACCESS_KEY_ID"] = ENV.fetch("HOMEBREW_BUILDPULSE_ACCESS_KEY_ID") - ENV["BUILDPULSE_SECRET_ACCESS_KEY"] = ENV.fetch("HOMEBREW_BUILDPULSE_SECRET_ACCESS_KEY") - - ohai "Sending test results to BuildPulse" - - system_command Formula["buildpulse-test-reporter"].opt_bin/"buildpulse-test-reporter", - args: [ - "submit", "#{HOMEBREW_LIBRARY_PATH}/test/junit", - "--account-id", ENV.fetch("HOMEBREW_BUILDPULSE_ACCOUNT_ID"), - "--repository-id", ENV.fetch("HOMEBREW_BUILDPULSE_REPOSITORY_ID") - ] - end - sig { returns(T::Array[String]) } def changed_test_files changed_files = Utils.popen_read("git", "diff", "--name-only", "master") diff --git a/Library/Homebrew/test/spec_helper.rb b/Library/Homebrew/test/spec_helper.rb index 831d8f7404..26857879a5 100644 --- a/Library/Homebrew/test/spec_helper.rb +++ b/Library/Homebrew/test/spec_helper.rb @@ -91,9 +91,9 @@ RSpec.configure do |config| # Use rspec-retry to handle flaky tests. config.default_sleep_interval = 1 - # Don't want the nicer default retry behaviour when using BuildPulse to + # Don't want the nicer default retry behaviour when using CodeCov to # identify flaky tests. - config.default_retry_count = 2 unless ENV["BUILDPULSE"] + config.default_retry_count = 2 unless ENV["CODECOV_TOKEN"] config.expect_with :rspec do |expectations| # This option will default to `true` in RSpec 4. It makes the `description` @@ -122,9 +122,9 @@ RSpec.configure do |config| config.around(:each, :needs_network) do |example| example.metadata[:timeout] ||= 120 - # Don't want the nicer default retry behaviour when using BuildPulse to + # Don't want the nicer default retry behaviour when using CodeCov to # identify flaky tests. - example.metadata[:retry] ||= 4 unless ENV["BUILDPULSE"] + example.metadata[:retry] ||= 4 unless ENV["CODECOV_TOKEN"] example.metadata[:retry_wait] ||= 2 example.metadata[:exponential_backoff] ||= true diff --git a/README.md b/README.md index f75e9c718e..87509366c9 100644 --- a/README.md +++ b/README.md @@ -78,10 +78,6 @@ Secure password storage and syncing is provided by [1Password for Teams](https:/ [1Password](https://1password.com) -Flaky test detection and tracking is provided by [BuildPulse](https://buildpulse.io/). - -[![BuildPulse](https://github.com/Homebrew/brew/assets/1699443/87385e9a-6c47-4e59-b17e-fe083e945709)](https://buildpulse.io) - 's DNS is [resolving with DNSimple](https://dnsimple.com/resolving/homebrew). [![DNSimple](https://cdn.dnsimple.com/assets/resolving-with-us/logo-light.png)](https://dnsimple.com/resolving/homebrew#gh-light-mode-only) diff --git a/docs/Homebrew-brew-Maintainer-Guide.md b/docs/Homebrew-brew-Maintainer-Guide.md index 1968a56a06..966d7f0095 100644 --- a/docs/Homebrew-brew-Maintainer-Guide.md +++ b/docs/Homebrew-brew-Maintainer-Guide.md @@ -42,13 +42,11 @@ A coverage report is generated by Codecov for every PR, and its results are show Codecov should be used as a guide to indicate when more tests are probably needed, but it's unrealistic for every line of code to have a test associated with it, especially when testing would require a slow integration test. For this reason, it's okay to merge PRs that fail the Codecov check if necessary, but this should be avoided if possible. -### `brew tests` and BuildPulse +CodeCov also monitors CI jobs for every push to `Homebrew/brew` to detect flaky tests and track them over time. The reports are available on [CodeCov](https://app.codecov.io/gh/Homebrew/brew/tests/master). -BuildPulse monitors CI jobs for every push to `Homebrew/brew` to detect flaky tests and track them over time. The reports are available to Homebrew maintainers on [buildpulse.io](https://app.buildpulse.io/@Homebrew) and daily summaries are published to [`#buildpulse-health`](https://machomebrew.slack.com/archives/C0268BSJBJ8) in Slack. +CodeCov can be used as a guide to identify which flaky tests are causing the most disruption to the CI suite. To make the biggest improvements to the reliability of the build, we can focus on the most disruptive flaky tests first (i.e. the tests causing the most intermittent failures). -BuildPulse can be used as a guide to identify which flaky tests are causing the most disruption to the CI suite. To make the biggest improvements to the reliability of the build, we can focus on the most disruptive flaky tests first (i.e. the tests causing the most intermittent failures). - -To help find the root cause for a particular flaky test, buildpulse.io provides links to the most recent CI job and commit where the test failed and then passed with no change to the underlying code. You may want to check out the code at that commit to attempt to reproduce the failure locally. You can also see the list of recent failures on [buildpulse.io](https://app.buildpulse.io/@Homebrew) to determine if the test always fails the same way. +To help find the root cause for a particular flaky test, CodeCov provides links to the most recent CI job and commit where the test failed and then passed with no change to the underlying code. You may want to check out the code at that commit to attempt to reproduce the failure locally. You can also see the list of recent failures on CodeCov to determine if the test always fails the same way. ## Manpages and Shell Completions