diff --git a/Library/Homebrew/dev-cmd/tests.rb b/Library/Homebrew/dev-cmd/tests.rb index d13fe70134..fb948c1013 100644 --- a/Library/Homebrew/dev-cmd/tests.rb +++ b/Library/Homebrew/dev-cmd/tests.rb @@ -1,267 +1,271 @@ # typed: true # frozen_string_literal: true +require "abstract_command" require "cli/parser" require "fileutils" require "system_command" module Homebrew - extend SystemCommand::Mixin + module DevCmd + class Tests < AbstractCommand + include SystemCommand::Mixin - sig { returns(CLI::Parser) } - def self.tests_args - Homebrew::CLI::Parser.new do - description <<~EOS - Run Homebrew's unit and integration tests. - EOS - switch "--coverage", - description: "Generate code coverage reports." - switch "--generic", - description: "Run only OS-agnostic tests." - switch "--online", - description: "Include tests that use the GitHub API and tests that use any of the taps for " \ - "official external commands." - switch "--byebug", - description: "Enable debugging using byebug." - switch "--changed", - description: "Only runs tests on files that were changed from the master branch." - switch "--fail-fast", - description: "Exit early on the first failing test." - flag "--only=", - description: "Run only `_spec.rb`. Appending `:` will start at a " \ - "specific line." - flag "--profile=", - description: "Run the test suite serially to find the slowest tests." - flag "--seed=", - description: "Randomise tests with the specified instead of a random seed." + cmd_args do + description <<~EOS + Run Homebrew's unit and integration tests. + EOS + switch "--coverage", + description: "Generate code coverage reports." + switch "--generic", + description: "Run only OS-agnostic tests." + switch "--online", + description: "Include tests that use the GitHub API and tests that use any of the taps for " \ + "official external commands." + switch "--byebug", + description: "Enable debugging using byebug." + switch "--changed", + description: "Only runs tests on files that were changed from the master branch." + switch "--fail-fast", + description: "Exit early on the first failing test." + flag "--only=", + description: "Run only `_spec.rb`. Appending `:` will start at a " \ + "specific line." + flag "--profile=", + description: "Run the test suite serially to find the slowest tests." + flag "--seed=", + description: "Randomise tests with the specified instead of a random seed." - conflicts "--changed", "--only" + conflicts "--changed", "--only" - named_args :none - end - end - - def self.use_buildpulse? - return @use_buildpulse if defined?(@use_buildpulse) - - @use_buildpulse = ENV["HOMEBREW_BUILDPULSE_ACCESS_KEY_ID"].present? && - ENV["HOMEBREW_BUILDPULSE_SECRET_ACCESS_KEY"].present? && - ENV["HOMEBREW_BUILDPULSE_ACCOUNT_ID"].present? && - ENV["HOMEBREW_BUILDPULSE_REPOSITORY_ID"].present? - end - - def self.run_buildpulse - require "formula" - - with_env(HOMEBREW_NO_AUTO_UPDATE: "1", HOMEBREW_NO_BOOTSNAP: "1") do - ensure_formula_installed!("buildpulse-test-reporter", - reason: "reporting test flakiness") - end - - ENV["BUILDPULSE_ACCESS_KEY_ID"] = ENV.fetch("HOMEBREW_BUILDPULSE_ACCESS_KEY_ID") - ENV["BUILDPULSE_SECRET_ACCESS_KEY"] = ENV.fetch("HOMEBREW_BUILDPULSE_SECRET_ACCESS_KEY") - - ohai "Sending test results to BuildPulse" - - system_command Formula["buildpulse-test-reporter"].opt_bin/"buildpulse-test-reporter", - args: [ - "submit", "#{HOMEBREW_LIBRARY_PATH}/test/junit", - "--account-id", ENV.fetch("HOMEBREW_BUILDPULSE_ACCOUNT_ID"), - "--repository-id", ENV.fetch("HOMEBREW_BUILDPULSE_REPOSITORY_ID") - ] - end - - def self.changed_test_files - changed_files = Utils.popen_read("git", "diff", "--name-only", "master") - - raise UsageError, "No files have been changed from the master branch!" if changed_files.blank? - - filestub_regex = %r{Library/Homebrew/([\w/-]+).rb} - changed_files.scan(filestub_regex).map(&:last).filter_map do |filestub| - if filestub.start_with?("test/") - # Only run tests on *_spec.rb files in test/ folder - filestub.end_with?("_spec") ? Pathname("#{filestub}.rb") : nil - else - # For all other changed .rb files guess the associated test file name - Pathname("test/#{filestub}_spec.rb") - end - end.select(&:exist?) - end - - def self.tests - args = tests_args.parse - - # Given we might be testing various commands, we probably want everything (except sorbet-static) - Homebrew.install_bundler_gems!(groups: Homebrew.valid_gem_groups - ["sorbet"]) - - require "byebug" if args.byebug? - - HOMEBREW_LIBRARY_PATH.cd do - setup_environment!(args) - - parallel = true - - files = if args.only - test_name, line = args.only.split(":", 2) - - if line.nil? - Dir.glob("test/{#{test_name},#{test_name}/**/*}_spec.rb") - else - parallel = false - ["test/#{test_name}_spec.rb:#{line}"] - end - elsif args.changed? - changed_test_files - else - Dir.glob("test/**/*_spec.rb") + named_args :none end - if files.blank? - raise UsageError, "The `--only` argument requires a valid file or folder name!" if args.only + sig { override.void } + def run + # Given we might be testing various commands, we probably want everything (except sorbet-static) + Homebrew.install_bundler_gems!(groups: Homebrew.valid_gem_groups - ["sorbet"]) - if args.changed? - opoo "No tests are directly associated with the changed files!" - return + require "byebug" if args.byebug? + + HOMEBREW_LIBRARY_PATH.cd do + setup_environment! + + parallel = true + + files = if args.only + # FIXME: This is safe once args are namespaced by command + test_name, line = T.unsafe(args.only).split(":", 2) + + if line.nil? + Dir.glob("test/{#{test_name},#{test_name}/**/*}_spec.rb") + else + parallel = false + ["test/#{test_name}_spec.rb:#{line}"] + end + elsif args.changed? + changed_test_files + else + Dir.glob("test/**/*_spec.rb") + end + + if files.blank? + raise UsageError, "The `--only` argument requires a valid file or folder name!" if args.only + + if args.changed? + opoo "No tests are directly associated with the changed files!" + return + end + end + + parallel = false if args.profile + + parallel_rspec_log_name = "parallel_runtime_rspec" + parallel_rspec_log_name = "#{parallel_rspec_log_name}.generic" if args.generic? + parallel_rspec_log_name = "#{parallel_rspec_log_name}.online" if args.online? + parallel_rspec_log_name = "#{parallel_rspec_log_name}.log" + + parallel_rspec_log_path = if ENV["CI"] + "tests/#{parallel_rspec_log_name}" + else + "#{HOMEBREW_CACHE}/#{parallel_rspec_log_name}" + end + ENV["PARALLEL_RSPEC_LOG_PATH"] = parallel_rspec_log_path + + parallel_args = if ENV["CI"] + %W[ + --combine-stderr + --serialize-stdout + --runtime-log #{parallel_rspec_log_path} + ] + else + %w[ + --nice + ] + end + + # Generate seed ourselves and output later to avoid multiple different + # seeds being output when running parallel tests. + seed = args.seed || rand(0xFFFF).to_i + + bundle_args = ["-I", HOMEBREW_LIBRARY_PATH/"test"] + bundle_args += %W[ + --seed #{seed} + --color + --require spec_helper + ] + bundle_args << "--fail-fast" if args.fail_fast? + bundle_args << "--profile" << args.profile if args.profile + + # TODO: Refactor and move to extend/os + # rubocop:disable Homebrew/MoveToExtendOS + unless OS.mac? + bundle_args << "--tag" << "~needs_macos" << "--tag" << "~cask" + files = files.grep_v(%r{^test/(os/mac|cask)(/.*|_spec\.rb)$}) + end + + unless OS.linux? + bundle_args << "--tag" << "~needs_linux" + files = files.grep_v(%r{^test/os/linux(/.*|_spec\.rb)$}) + end + # rubocop:enable Homebrew/MoveToExtendOS + + bundle_args << "--tag" << "~needs_network" unless args.online? + unless ENV["CI"] + bundle_args << "--tag" << "~needs_ci" \ + << "--tag" << "~needs_svn" + end + + puts "Randomized with seed #{seed}" + + # Submit test flakiness information using BuildPulse + # BUILDPULSE used in spec_helper.rb + if use_buildpulse? + ENV["BUILDPULSE"] = "1" + ohai "Running tests with BuildPulse-friendly settings" + end + + if parallel + system "bundle", "exec", "parallel_rspec", *parallel_args, "--", *bundle_args, "--", *files + else + system "bundle", "exec", "rspec", *bundle_args, "--", *files + end + success = $CHILD_STATUS.success? + + run_buildpulse if use_buildpulse? + + return if success + + Homebrew.failed = true end end - parallel = false if args.profile + private - parallel_rspec_log_name = "parallel_runtime_rspec" - parallel_rspec_log_name = "#{parallel_rspec_log_name}.generic" if args.generic? - parallel_rspec_log_name = "#{parallel_rspec_log_name}.online" if args.online? - parallel_rspec_log_name = "#{parallel_rspec_log_name}.log" + def use_buildpulse? + return @use_buildpulse if defined?(@use_buildpulse) - parallel_rspec_log_path = if ENV["CI"] - "tests/#{parallel_rspec_log_name}" - else - "#{HOMEBREW_CACHE}/#{parallel_rspec_log_name}" + @use_buildpulse = ENV["HOMEBREW_BUILDPULSE_ACCESS_KEY_ID"].present? && + ENV["HOMEBREW_BUILDPULSE_SECRET_ACCESS_KEY"].present? && + ENV["HOMEBREW_BUILDPULSE_ACCOUNT_ID"].present? && + ENV["HOMEBREW_BUILDPULSE_REPOSITORY_ID"].present? end - ENV["PARALLEL_RSPEC_LOG_PATH"] = parallel_rspec_log_path - parallel_args = if ENV["CI"] - %W[ - --combine-stderr - --serialize-stdout - --runtime-log #{parallel_rspec_log_path} - ] - else - %w[ - --nice + def run_buildpulse + require "formula" + + with_env(HOMEBREW_NO_AUTO_UPDATE: "1", HOMEBREW_NO_BOOTSNAP: "1") do + ensure_formula_installed!("buildpulse-test-reporter", + reason: "reporting test flakiness") + end + + ENV["BUILDPULSE_ACCESS_KEY_ID"] = ENV.fetch("HOMEBREW_BUILDPULSE_ACCESS_KEY_ID") + ENV["BUILDPULSE_SECRET_ACCESS_KEY"] = ENV.fetch("HOMEBREW_BUILDPULSE_SECRET_ACCESS_KEY") + + ohai "Sending test results to BuildPulse" + + system_command Formula["buildpulse-test-reporter"].opt_bin/"buildpulse-test-reporter", + args: [ + "submit", "#{HOMEBREW_LIBRARY_PATH}/test/junit", + "--account-id", ENV.fetch("HOMEBREW_BUILDPULSE_ACCOUNT_ID"), + "--repository-id", ENV.fetch("HOMEBREW_BUILDPULSE_REPOSITORY_ID") + ] + end + + def changed_test_files + changed_files = Utils.popen_read("git", "diff", "--name-only", "master") + + raise UsageError, "No files have been changed from the master branch!" if changed_files.blank? + + filestub_regex = %r{Library/Homebrew/([\w/-]+).rb} + changed_files.scan(filestub_regex).map(&:last).filter_map do |filestub| + if filestub.start_with?("test/") + # Only run tests on *_spec.rb files in test/ folder + filestub.end_with?("_spec") ? Pathname("#{filestub}.rb") : nil + else + # For all other changed .rb files guess the associated test file name + Pathname("test/#{filestub}_spec.rb") + end + end.select(&:exist?) + end + + def setup_environment! + # Cleanup any unwanted user configuration. + allowed_test_env = %w[ + HOMEBREW_GITHUB_API_TOKEN + HOMEBREW_CACHE + HOMEBREW_LOGS + HOMEBREW_TEMP ] + allowed_test_env << "HOMEBREW_USE_RUBY_FROM_PATH" if Homebrew::EnvConfig.developer? + Homebrew::EnvConfig::ENVS.keys.map(&:to_s).each do |env| + next if allowed_test_env.include?(env) + + ENV.delete(env) + end + + # Codespaces HOMEBREW_PREFIX and /tmp are mounted 755 which makes Ruby warn constantly. + if (ENV["HOMEBREW_CODESPACES"] == "true") && (HOMEBREW_TEMP.to_s == "/tmp") + # Need to keep this fairly short to avoid socket paths being too long in tests. + homebrew_prefix_tmp = "/home/linuxbrew/tmp" + ENV["HOMEBREW_TEMP"] = homebrew_prefix_tmp + FileUtils.mkdir_p homebrew_prefix_tmp + system "chmod", "-R", "g-w,o-w", HOMEBREW_PREFIX, homebrew_prefix_tmp + end + + ENV["HOMEBREW_TESTS"] = "1" + ENV["HOMEBREW_NO_AUTO_UPDATE"] = "1" + ENV["HOMEBREW_NO_ANALYTICS_THIS_RUN"] = "1" + ENV["HOMEBREW_TEST_GENERIC_OS"] = "1" if args.generic? + ENV["HOMEBREW_TEST_ONLINE"] = "1" if args.online? + ENV["HOMEBREW_SORBET_RUNTIME"] = "1" + + # TODO: remove this and fix tests when possible. + ENV["HOMEBREW_NO_INSTALL_FROM_API"] = "1" + ENV.delete("HOMEBREW_INTERNAL_JSON_V3") + + ENV["USER"] ||= system_command!("id", args: ["-nu"]).stdout.chomp + + # Avoid local configuration messing with tests, e.g. git being configured + # to use GPG to sign by default + ENV["HOME"] = "#{HOMEBREW_LIBRARY_PATH}/test" + + # Print verbose output when requesting debug or verbose output. + ENV["HOMEBREW_VERBOSE_TESTS"] = "1" if args.debug? || args.verbose? + + if args.coverage? + ENV["HOMEBREW_TESTS_COVERAGE"] = "1" + FileUtils.rm_f "test/coverage/.resultset.json" + end + + # Override author/committer as global settings might be invalid and thus + # will cause silent failure during the setup of dummy Git repositories. + %w[AUTHOR COMMITTER].each do |role| + ENV["GIT_#{role}_NAME"] = "brew tests" + ENV["GIT_#{role}_EMAIL"] = "brew-tests@localhost" + ENV["GIT_#{role}_DATE"] = "Sun Jan 22 19:59:13 2017 +0000" + end end - - # Generate seed ourselves and output later to avoid multiple different - # seeds being output when running parallel tests. - seed = args.seed || rand(0xFFFF).to_i - - bundle_args = ["-I", HOMEBREW_LIBRARY_PATH/"test"] - bundle_args += %W[ - --seed #{seed} - --color - --require spec_helper - ] - bundle_args << "--fail-fast" if args.fail_fast? - bundle_args << "--profile" << args.profile if args.profile - - # TODO: Refactor and move to extend/os - # rubocop:disable Homebrew/MoveToExtendOS - unless OS.mac? - bundle_args << "--tag" << "~needs_macos" << "--tag" << "~cask" - files = files.grep_v(%r{^test/(os/mac|cask)(/.*|_spec\.rb)$}) - end - - unless OS.linux? - bundle_args << "--tag" << "~needs_linux" - files = files.grep_v(%r{^test/os/linux(/.*|_spec\.rb)$}) - end - # rubocop:enable Homebrew/MoveToExtendOS - - bundle_args << "--tag" << "~needs_network" unless args.online? - unless ENV["CI"] - bundle_args << "--tag" << "~needs_ci" \ - << "--tag" << "~needs_svn" - end - - puts "Randomized with seed #{seed}" - - # Submit test flakiness information using BuildPulse - # BUILDPULSE used in spec_helper.rb - if use_buildpulse? - ENV["BUILDPULSE"] = "1" - ohai "Running tests with BuildPulse-friendly settings" - end - - if parallel - system "bundle", "exec", "parallel_rspec", *parallel_args, "--", *bundle_args, "--", *files - else - system "bundle", "exec", "rspec", *bundle_args, "--", *files - end - success = $CHILD_STATUS.success? - - run_buildpulse if use_buildpulse? - - return if success - - Homebrew.failed = true - end - end - - def self.setup_environment!(args) - # Cleanup any unwanted user configuration. - allowed_test_env = %w[ - HOMEBREW_GITHUB_API_TOKEN - HOMEBREW_CACHE - HOMEBREW_LOGS - HOMEBREW_TEMP - ] - allowed_test_env << "HOMEBREW_USE_RUBY_FROM_PATH" if Homebrew::EnvConfig.developer? - Homebrew::EnvConfig::ENVS.keys.map(&:to_s).each do |env| - next if allowed_test_env.include?(env) - - ENV.delete(env) - end - - # Codespaces HOMEBREW_PREFIX and /tmp are mounted 755 which makes Ruby warn constantly. - if (ENV["HOMEBREW_CODESPACES"] == "true") && (HOMEBREW_TEMP.to_s == "/tmp") - # Need to keep this fairly short to avoid socket paths being too long in tests. - homebrew_prefix_tmp = "/home/linuxbrew/tmp" - ENV["HOMEBREW_TEMP"] = homebrew_prefix_tmp - FileUtils.mkdir_p homebrew_prefix_tmp - system "chmod", "-R", "g-w,o-w", HOMEBREW_PREFIX, homebrew_prefix_tmp - end - - ENV["HOMEBREW_TESTS"] = "1" - ENV["HOMEBREW_NO_AUTO_UPDATE"] = "1" - ENV["HOMEBREW_NO_ANALYTICS_THIS_RUN"] = "1" - ENV["HOMEBREW_TEST_GENERIC_OS"] = "1" if args.generic? - ENV["HOMEBREW_TEST_ONLINE"] = "1" if args.online? - ENV["HOMEBREW_SORBET_RUNTIME"] = "1" - - # TODO: remove this and fix tests when possible. - ENV["HOMEBREW_NO_INSTALL_FROM_API"] = "1" - ENV.delete("HOMEBREW_INTERNAL_JSON_V3") - - ENV["USER"] ||= system_command!("id", args: ["-nu"]).stdout.chomp - - # Avoid local configuration messing with tests, e.g. git being configured - # to use GPG to sign by default - ENV["HOME"] = "#{HOMEBREW_LIBRARY_PATH}/test" - - # Print verbose output when requesting debug or verbose output. - ENV["HOMEBREW_VERBOSE_TESTS"] = "1" if args.debug? || args.verbose? - - if args.coverage? - ENV["HOMEBREW_TESTS_COVERAGE"] = "1" - FileUtils.rm_f "test/coverage/.resultset.json" - end - - # Override author/committer as global settings might be invalid and thus - # will cause silent failure during the setup of dummy Git repositories. - %w[AUTHOR COMMITTER].each do |role| - ENV["GIT_#{role}_NAME"] = "brew tests" - ENV["GIT_#{role}_EMAIL"] = "brew-tests@localhost" - ENV["GIT_#{role}_DATE"] = "Sun Jan 22 19:59:13 2017 +0000" end end end diff --git a/Library/Homebrew/test/dev-cmd/tests_spec.rb b/Library/Homebrew/test/dev-cmd/tests_spec.rb new file mode 100644 index 0000000000..eef6e0aec1 --- /dev/null +++ b/Library/Homebrew/test/dev-cmd/tests_spec.rb @@ -0,0 +1,8 @@ +# frozen_string_literal: true + +require "cmd/shared_examples/args_parse" +require "dev-cmd/tests" + +RSpec.describe Homebrew::DevCmd::Tests do + it_behaves_like "parseable arguments" +end