diff --git a/Library/Homebrew/cmd/fetch.rb b/Library/Homebrew/cmd/fetch.rb index 6c12d1c233..c1629f8e59 100644 --- a/Library/Homebrew/cmd/fetch.rb +++ b/Library/Homebrew/cmd/fetch.rb @@ -150,12 +150,7 @@ module Homebrew download_queue.enqueue(resource) end - formula.resources.each do |r| - download_queue.enqueue(r) - r.patches.each { |patch| download_queue.enqueue(patch.resource) if patch.external? } - end - - formula.patchlist.each { |patch| download_queue.enqueue(patch.resource) if patch.external? } + formula.enqueue_resources_and_patches(download_queue:) end end else @@ -190,7 +185,11 @@ module Homebrew sig { returns(Integer) } def concurrency - @concurrency ||= T.let(args.concurrency&.to_i || 1, T.nilable(Integer)) + @concurrency ||= T.let( + # TODO: document this variable when ready to publicly announce it. + args.concurrency&.to_i || ENV.fetch("HOMEBREW_DOWNLOAD_CONCURRENCY", "1").to_i, + T.nilable(Integer), + ) end sig { returns(Integer) } diff --git a/Library/Homebrew/cmd/install.rb b/Library/Homebrew/cmd/install.rb index 60c272ac9e..b15930fd03 100644 --- a/Library/Homebrew/cmd/install.rb +++ b/Library/Homebrew/cmd/install.rb @@ -47,6 +47,7 @@ module Homebrew description: "Ask for confirmation before downloading and installing formulae. " \ "Print download and install sizes of bottles and dependencies.", env: :ask + flag "--concurrency=", description: "Number of concurrent downloads.", hidden: true [ [:switch, "--formula", "--formulae", { description: "Treat all named arguments as formulae.", diff --git a/Library/Homebrew/download_queue.rb b/Library/Homebrew/download_queue.rb index 0de8add528..887d3d1f35 100644 --- a/Library/Homebrew/download_queue.rb +++ b/Library/Homebrew/download_queue.rb @@ -17,7 +17,7 @@ module Homebrew @pool = T.let(Concurrent::FixedThreadPool.new(concurrency), Concurrent::FixedThreadPool) end - sig { params(downloadable: T.any(Resource, Bottle, Cask::Download)).void } + sig { params(downloadable: T.any(Resource, Bottle, Cask::Download, Downloadable)).void } def enqueue(downloadable) downloads[downloadable] ||= Concurrent::Promises.future_on( pool, RetryableDownload.new(downloadable, tries:), force, quiet @@ -135,6 +135,13 @@ module Homebrew end end + sig { void } + def wait + downloads.each do |downloadable, promise| + promise.wait! + end + end + sig { void } def shutdown pool.shutdown @@ -166,9 +173,9 @@ module Homebrew sig { returns(T::Boolean) } attr_reader :quiet - sig { returns(T::Hash[T.any(Resource, Bottle, Cask::Download), Concurrent::Promises::Future]) } + sig { returns(T::Hash[T.any(Resource, Bottle, Cask::Download, Downloadable), Concurrent::Promises::Future]) } def downloads - @downloads ||= T.let({}, T.nilable(T::Hash[T.any(Resource, Bottle, Cask::Download), + @downloads ||= T.let({}, T.nilable(T::Hash[T.any(Resource, Bottle, Cask::Download, Downloadable), Concurrent::Promises::Future])) end diff --git a/Library/Homebrew/formula.rb b/Library/Homebrew/formula.rb index ac85783f92..ee5356193a 100644 --- a/Library/Homebrew/formula.rb +++ b/Library/Homebrew/formula.rb @@ -3202,6 +3202,15 @@ class Formula end end + sig { params(download_queue: Homebrew::DownloadQueue).void } + def enqueue_resources_and_patches(download_queue:) + resources.each do |resource| + download_queue.enqueue(resource) + resource.patches.select(&:external?).each { |patch| download_queue.enqueue(patch.resource) } + end + patchlist.select(&:external?).each { |patch| download_queue.enqueue(patch.resource) } + end + sig { void } def fetch_patches patchlist.select(&:external?).each(&:fetch) diff --git a/Library/Homebrew/formula_installer.rb b/Library/Homebrew/formula_installer.rb index 241863355f..4209250b8a 100644 --- a/Library/Homebrew/formula_installer.rb +++ b/Library/Homebrew/formula_installer.rb @@ -139,6 +139,8 @@ class FormulaInstaller # Take the original formula instance, which might have been swapped from an API instance to a source instance @formula = T.let(T.must(previously_fetched_formula), Formula) if previously_fetched_formula + + @ran_prelude_fetch = T.let(false, T::Boolean) end sig { returns(T::Boolean) } @@ -293,8 +295,8 @@ class FormulaInstaller dep.bottle&.compatible_locations? end - sig { void } - def prelude + sig { params(download_queue: T.nilable(Homebrew::DownloadQueue)).void } + def prelude_fetch(download_queue: nil) deprecate_disable_type = DeprecateDisable.type(formula) if deprecate_disable_type.present? message = "#{formula.full_name} has been #{DeprecateDisable.message(formula)}" @@ -312,8 +314,24 @@ class FormulaInstaller end end + # Needs to be done before expand_dependencies for compute_dependencies + fetch_bottle_tab(download_queue:) if pour_bottle? + + @ran_prelude_fetch = true + end + + sig { params(download_queue: T.nilable(Homebrew::DownloadQueue)).void } + def prelude(download_queue: nil) + prelude_fetch(download_queue:) unless @ran_prelude_fetch + Tab.clear_cache + # Setup bottle_tab_runtime_dependencies for compute_dependencies + @bottle_tab_runtime_dependencies = formula.bottle_tab_attributes + .fetch("runtime_dependencies", []).then { |deps| deps || [] } + .each_with_object({}) { |dep, h| h[dep["full_name"]] = dep } + .freeze + verify_deps_exist unless ignore_deps? forbidden_license_check @@ -321,7 +339,7 @@ class FormulaInstaller forbidden_formula_check check_install_sanity - install_fetch_deps unless ignore_deps? + install_fetch_deps(download_queue:) unless ignore_deps? end sig { void } @@ -450,14 +468,14 @@ class FormulaInstaller sig { params(_formula: Formula).returns(T.nilable(T::Boolean)) } def fresh_install?(_formula) = false - sig { void } - def install_fetch_deps + sig { params(download_queue: T.nilable(Homebrew::DownloadQueue)).void } + def install_fetch_deps(download_queue: nil) return if @compute_dependencies.blank? compute_dependencies(use_cache: false) if @compute_dependencies.any? do |dep, options| next false unless dep.implicit? - fetch_dependencies + fetch_dependencies(download_queue:) install_dependency(dep, options) true end @@ -787,8 +805,8 @@ on_request: installed_on_request?, options:) @show_header = true unless deps.empty? end - sig { params(dep: Dependency).void } - def fetch_dependency(dep) + sig { params(dep: Dependency, download_queue: T.nilable(Homebrew::DownloadQueue)).void } + def fetch_dependency(dep, download_queue: nil) df = dep.to_formula fi = FormulaInstaller.new( df, @@ -808,8 +826,8 @@ on_request: installed_on_request?, options:) quiet: quiet?, verbose: verbose?, ) - fi.prelude - fi.fetch + fi.prelude(download_queue:) + fi.fetch(download_queue:) end sig { params(dep: Dependency, inherited_options: Options).void } @@ -1326,8 +1344,8 @@ on_request: installed_on_request?, options:) @show_summary_heading = true end - sig { void } - def fetch_dependencies + sig { params(download_queue: T.nilable(Homebrew::DownloadQueue)).void } + def fetch_dependencies(download_queue: nil) return if ignore_deps? # Don't output dependencies if we're explicitly installing them. @@ -1337,11 +1355,16 @@ on_request: installed_on_request?, options:) return if deps.empty? - oh1 "Fetching dependencies for #{formula.full_name}: " \ - "#{deps.map(&:first).map { Formatter.identifier(_1) }.to_sentence}", - truncate: false + dependencies_string = deps.map(&:first) + .map { Formatter.identifier(_1) } + .to_sentence + unless download_queue + oh1 "Fetching dependencies for #{formula.full_name}: " \ + "#{dependencies_string}", + truncate: false + end - deps.each { |(dep, _options)| fetch_dependency(dep) } + deps.each { |(dep, _options)| fetch_dependency(dep, download_queue:) } end sig { returns(T.nilable(Formula)) } @@ -1356,54 +1379,67 @@ on_request: installed_on_request?, options:) end end - sig { params(quiet: T::Boolean).void } - def fetch_bottle_tab(quiet: false) + sig { params(download_queue: T.nilable(Homebrew::DownloadQueue), quiet: T::Boolean).void } + def fetch_bottle_tab(download_queue: nil, quiet: false) return if @fetch_bottle_tab - begin - formula.fetch_bottle_tab(quiet: quiet) - @bottle_tab_runtime_dependencies = formula.bottle_tab_attributes - .fetch("runtime_dependencies", []).then { |deps| deps || [] } - .each_with_object({}) { |dep, h| h[dep["full_name"]] = dep } - .freeze - rescue DownloadError, Resource::BottleManifest::Error - # do nothing + if download_queue && (bottle = formula.bottle) && (manifest_resource = bottle.github_packages_manifest_resource) + download_queue.enqueue(manifest_resource) + else + begin + formula.fetch_bottle_tab(quiet: quiet) + rescue DownloadError, Resource::BottleManifest::Error + # do nothing + end end + @fetch_bottle_tab = T.let(true, T.nilable(TrueClass)) end - sig { void } - def fetch + sig { params(download_queue: T.nilable(Homebrew::DownloadQueue)).void } + def fetch(download_queue: nil) return if previously_fetched_formula - fetch_dependencies + fetch_dependencies(download_queue:) return if only_deps? return if formula.local_bottle_path.present? - oh1 "Fetching #{Formatter.identifier(formula.full_name)}".strip + oh1 "Fetching #{Formatter.identifier(formula.full_name)}".strip unless download_queue downloadable_object = downloadable check_attestation = if pour_bottle?(output_warning: true) - fetch_bottle_tab + fetch_bottle_tab(download_queue:) !downloadable_object.cached_download.exist? else @formula = Homebrew::API::Formula.source_download(formula) if formula.loaded_from_api? - formula.fetch_patches - formula.resources.each(&:fetch) + if download_queue + formula.enqueue_resources_and_patches(download_queue:) + else + formula.fetch_patches + formula.resources.each(&:fetch) + end + downloadable_object = downloadable false end - downloadable_object.fetch + + if download_queue + download_queue.enqueue(downloadable_object) + else + downloadable_object.fetch + end # We skip `gh` to avoid a bootstrapping cycle, in the off-chance a user attempts # to explicitly `brew install gh` without already having a version for bootstrapping. # We also skip bottle installs from local bottle paths, as these are done in CI # as part of the build lifecycle before attestations are produced. if check_attestation && + # TODO: support this for download queues at some point + download_queue.nil? && Homebrew::Attestation.enabled? && formula.tap&.core_tap? && formula.name != "gh" diff --git a/Library/Homebrew/install.rb b/Library/Homebrew/install.rb index 24676a26ea..f5d1f9868e 100644 --- a/Library/Homebrew/install.rb +++ b/Library/Homebrew/install.rb @@ -6,6 +6,7 @@ require "fileutils" require "hardware" require "development_tools" require "upgrade" +require "download_queue" module Homebrew # Helper module for performing (pre-)install checks. @@ -314,15 +315,37 @@ module Homebrew skip_link: false ) unless dry_run - formula_installers.each do |fi| - fi.prelude - fi.fetch - rescue CannotInstallFormulaError => e - ofail e.message - next - rescue UnsatisfiedRequirements, DownloadError, ChecksumMismatchError => e - ofail "#{fi.formula}: #{e}" - next + concurrency = ENV.fetch("HOMEBREW_DOWNLOAD_CONCURRENCY", 1).to_i + if concurrency > 1 + retries = 0 + # TODO: disable force when done testing locally!!! + force = true + raise if ENV["CI"] && force == true + + download_queue = Homebrew::DownloadQueue.new(concurrency:, retries:, force:) + end + begin + formula_installers.each do |fi| + fi.prelude_fetch(download_queue:) + download_queue&.start + download_queue&.wait + + fi.prelude(download_queue:) + download_queue&.start + download_queue&.wait + + fi.fetch(download_queue:) + download_queue&.start + download_queue&.wait + rescue CannotInstallFormulaError => e + ofail e.message + next + rescue UnsatisfiedRequirements, DownloadError, ChecksumMismatchError => e + ofail "#{fi.formula}: #{e}" + next + end + ensure + download_queue&.shutdown end end diff --git a/Library/Homebrew/resource.rb b/Library/Homebrew/resource.rb index a774f659df..1c44e44d40 100644 --- a/Library/Homebrew/resource.rb +++ b/Library/Homebrew/resource.rb @@ -91,6 +91,7 @@ class Resource patches.grep(DATAPatch) { |p| p.path = owner.owner.path } end + sig { params(skip_downloaded: T::Boolean).void } def fetch_patches(skip_downloaded: false) external_patches = patches.select(&:external?) external_patches.reject!(&:downloaded?) if skip_downloaded