WIP

Library/Homebrew/cmd/fetch.rb

@@ -150,12 +150,7 @@ module Homebrew
                   download_queue.enqueue(resource)
                 end
 
-                formula.resources.each do |r|
+                formula.enqueue_resources_and_patches(download_queue:)
-                  download_queue.enqueue(r)
-                  r.patches.each { |patch| download_queue.enqueue(patch.resource) if patch.external? }
-                end
-
-                formula.patchlist.each { |patch| download_queue.enqueue(patch.resource) if patch.external? }
               end
             end
           else
@@ -190,7 +185,11 @@ module Homebrew
 
       sig { returns(Integer) }
       def concurrency
-        @concurrency ||= T.let(args.concurrency&.to_i || 1, T.nilable(Integer))
+        @concurrency ||= T.let(
+          # TODO: document this variable when ready to publicly announce it.
+          args.concurrency&.to_i || ENV.fetch("HOMEBREW_DOWNLOAD_CONCURRENCY", "1").to_i,
+          T.nilable(Integer),
+        )
       end
 
       sig { returns(Integer) }

Library/Homebrew/cmd/install.rb

@@ -47,6 +47,7 @@ module Homebrew
                description: "Ask for confirmation before downloading and installing formulae. " \
                             "Print download and install sizes of bottles and dependencies.",
                env:         :ask
+        flag "--concurrency=", description: "Number of concurrent downloads.", hidden: true
         [
           [:switch, "--formula", "--formulae", {
             description: "Treat all named arguments as formulae.",

Library/Homebrew/download_queue.rb

@@ -17,7 +17,7 @@ module Homebrew
       @pool = T.let(Concurrent::FixedThreadPool.new(concurrency), Concurrent::FixedThreadPool)
     end
 
-    sig { params(downloadable: T.any(Resource, Bottle, Cask::Download)).void }
+    sig { params(downloadable: T.any(Resource, Bottle, Cask::Download, Downloadable)).void }
     def enqueue(downloadable)
       downloads[downloadable] ||= Concurrent::Promises.future_on(
         pool, RetryableDownload.new(downloadable, tries:), force, quiet
@@ -135,6 +135,13 @@ module Homebrew
       end
     end
 
+    sig { void }
+    def wait
+      downloads.each do |downloadable, promise|
+        promise.wait!
+      end
+    end
+
     sig { void }
     def shutdown
       pool.shutdown
@@ -166,9 +173,9 @@ module Homebrew
     sig { returns(T::Boolean) }
     attr_reader :quiet
 
-    sig { returns(T::Hash[T.any(Resource, Bottle, Cask::Download), Concurrent::Promises::Future]) }
+    sig { returns(T::Hash[T.any(Resource, Bottle, Cask::Download, Downloadable), Concurrent::Promises::Future]) }
     def downloads
-      @downloads ||= T.let({}, T.nilable(T::Hash[T.any(Resource, Bottle, Cask::Download),
+      @downloads ||= T.let({}, T.nilable(T::Hash[T.any(Resource, Bottle, Cask::Download, Downloadable),
                                                  Concurrent::Promises::Future]))
     end
 

Library/Homebrew/formula.rb

@@ -3202,6 +3202,15 @@ class Formula
     end
   end
 
+  sig { params(download_queue: Homebrew::DownloadQueue).void }
+  def enqueue_resources_and_patches(download_queue:)
+    resources.each do |resource|
+      download_queue.enqueue(resource)
+      resource.patches.select(&:external?).each { |patch| download_queue.enqueue(patch.resource) }
+    end
+    patchlist.select(&:external?).each { |patch| download_queue.enqueue(patch.resource) }
+  end
+
   sig { void }
   def fetch_patches
     patchlist.select(&:external?).each(&:fetch)

Library/Homebrew/formula_installer.rb

@@ -139,6 +139,8 @@ class FormulaInstaller
 
     # Take the original formula instance, which might have been swapped from an API instance to a source instance
     @formula = T.let(T.must(previously_fetched_formula), Formula) if previously_fetched_formula
+
+    @ran_prelude_fetch = T.let(false, T::Boolean)
   end
 
   sig { returns(T::Boolean) }
@@ -293,8 +295,8 @@ class FormulaInstaller
       dep.bottle&.compatible_locations?
   end
 
-  sig { void }
+  sig { params(download_queue: T.nilable(Homebrew::DownloadQueue)).void }
-  def prelude
+  def prelude_fetch(download_queue: nil)
     deprecate_disable_type = DeprecateDisable.type(formula)
     if deprecate_disable_type.present?
       message = "#{formula.full_name} has been #{DeprecateDisable.message(formula)}"
@@ -312,8 +314,24 @@ class FormulaInstaller
       end
     end
 
+    # Needs to be done before expand_dependencies for compute_dependencies
+    fetch_bottle_tab(download_queue:) if pour_bottle?
+
+    @ran_prelude_fetch = true
+  end
+
+  sig { params(download_queue: T.nilable(Homebrew::DownloadQueue)).void }
+  def prelude(download_queue: nil)
+    prelude_fetch(download_queue:) unless @ran_prelude_fetch
+
     Tab.clear_cache
 
+    # Setup bottle_tab_runtime_dependencies for compute_dependencies
+    @bottle_tab_runtime_dependencies = formula.bottle_tab_attributes
+                                              .fetch("runtime_dependencies", []).then { |deps| deps || [] }
+                                              .each_with_object({}) { |dep, h| h[dep["full_name"]] = dep }
+                                              .freeze
+
     verify_deps_exist unless ignore_deps?
 
     forbidden_license_check
@@ -321,7 +339,7 @@ class FormulaInstaller
     forbidden_formula_check
 
     check_install_sanity
-    install_fetch_deps unless ignore_deps?
+    install_fetch_deps(download_queue:) unless ignore_deps?
   end
 
   sig { void }
@@ -450,14 +468,14 @@ class FormulaInstaller
   sig { params(_formula: Formula).returns(T.nilable(T::Boolean)) }
   def fresh_install?(_formula) = false
 
-  sig { void }
+  sig { params(download_queue: T.nilable(Homebrew::DownloadQueue)).void }
-  def install_fetch_deps
+  def install_fetch_deps(download_queue: nil)
     return if @compute_dependencies.blank?
 
     compute_dependencies(use_cache: false) if @compute_dependencies.any? do |dep, options|
       next false unless dep.implicit?
 
-      fetch_dependencies
+      fetch_dependencies(download_queue:)
       install_dependency(dep, options)
       true
     end
@@ -787,8 +805,8 @@ on_request: installed_on_request?, options:)
     @show_header = true unless deps.empty?
   end
 
-  sig { params(dep: Dependency).void }
+  sig { params(dep: Dependency, download_queue: T.nilable(Homebrew::DownloadQueue)).void }
-  def fetch_dependency(dep)
+  def fetch_dependency(dep, download_queue: nil)
     df = dep.to_formula
     fi = FormulaInstaller.new(
       df,
@@ -808,8 +826,8 @@ on_request: installed_on_request?, options:)
       quiet:                      quiet?,
       verbose:                    verbose?,
     )
-    fi.prelude
+    fi.prelude(download_queue:)
-    fi.fetch
+    fi.fetch(download_queue:)
   end
 
   sig { params(dep: Dependency, inherited_options: Options).void }
@@ -1326,8 +1344,8 @@ on_request: installed_on_request?, options:)
     @show_summary_heading = true
   end
 
-  sig { void }
+  sig { params(download_queue: T.nilable(Homebrew::DownloadQueue)).void }
-  def fetch_dependencies
+  def fetch_dependencies(download_queue: nil)
     return if ignore_deps?
 
     # Don't output dependencies if we're explicitly installing them.
@@ -1337,11 +1355,16 @@ on_request: installed_on_request?, options:)
 
     return if deps.empty?
 
-    oh1 "Fetching dependencies for #{formula.full_name}: " \
+    dependencies_string = deps.map(&:first)
-        "#{deps.map(&:first).map { Formatter.identifier(_1) }.to_sentence}",
+                              .map { Formatter.identifier(_1) }
-        truncate: false
+                              .to_sentence
+    unless download_queue
+      oh1 "Fetching dependencies for #{formula.full_name}: " \
+          "#{dependencies_string}",
+          truncate: false
+    end
 
-    deps.each { |(dep, _options)| fetch_dependency(dep) }
+    deps.each { |(dep, _options)| fetch_dependency(dep, download_queue:) }
   end
 
   sig { returns(T.nilable(Formula)) }
@@ -1356,54 +1379,67 @@ on_request: installed_on_request?, options:)
     end
   end
 
-  sig { params(quiet: T::Boolean).void }
+  sig { params(download_queue: T.nilable(Homebrew::DownloadQueue), quiet: T::Boolean).void }
-  def fetch_bottle_tab(quiet: false)
+  def fetch_bottle_tab(download_queue: nil, quiet: false)
     return if @fetch_bottle_tab
 
-    begin
+    if download_queue && (bottle = formula.bottle) && (manifest_resource = bottle.github_packages_manifest_resource)
-      formula.fetch_bottle_tab(quiet: quiet)
+      download_queue.enqueue(manifest_resource)
-      @bottle_tab_runtime_dependencies = formula.bottle_tab_attributes
+    else
-                                                .fetch("runtime_dependencies", []).then { |deps| deps || [] }
+      begin
-                                                .each_with_object({}) { |dep, h| h[dep["full_name"]] = dep }
+        formula.fetch_bottle_tab(quiet: quiet)
-                                                .freeze
+      rescue DownloadError, Resource::BottleManifest::Error
-    rescue DownloadError, Resource::BottleManifest::Error
+        # do nothing
-      # do nothing
+      end
     end
+
     @fetch_bottle_tab = T.let(true, T.nilable(TrueClass))
   end
 
-  sig { void }
+  sig { params(download_queue: T.nilable(Homebrew::DownloadQueue)).void }
-  def fetch
+  def fetch(download_queue: nil)
     return if previously_fetched_formula
 
-    fetch_dependencies
+    fetch_dependencies(download_queue:)
 
     return if only_deps?
     return if formula.local_bottle_path.present?
 
-    oh1 "Fetching #{Formatter.identifier(formula.full_name)}".strip
+    oh1 "Fetching #{Formatter.identifier(formula.full_name)}".strip unless download_queue
 
     downloadable_object = downloadable
     check_attestation = if pour_bottle?(output_warning: true)
-      fetch_bottle_tab
+      fetch_bottle_tab(download_queue:)
 
       !downloadable_object.cached_download.exist?
     else
       @formula = Homebrew::API::Formula.source_download(formula) if formula.loaded_from_api?
 
-      formula.fetch_patches
+      if download_queue
-      formula.resources.each(&:fetch)
+        formula.enqueue_resources_and_patches(download_queue:)
+      else
+        formula.fetch_patches
+        formula.resources.each(&:fetch)
+      end
+
       downloadable_object = downloadable
 
       false
     end
-    downloadable_object.fetch
+
+    if download_queue
+      download_queue.enqueue(downloadable_object)
+    else
+      downloadable_object.fetch
+    end
 
     # We skip `gh` to avoid a bootstrapping cycle, in the off-chance a user attempts
     # to explicitly `brew install gh` without already having a version for bootstrapping.
     # We also skip bottle installs from local bottle paths, as these are done in CI
     # as part of the build lifecycle before attestations are produced.
     if check_attestation &&
+       # TODO: support this for download queues at some point
+       download_queue.nil? &&
        Homebrew::Attestation.enabled? &&
        formula.tap&.core_tap? &&
        formula.name != "gh"

Library/Homebrew/install.rb

@@ -6,6 +6,7 @@ require "fileutils"
 require "hardware"
 require "development_tools"
 require "upgrade"
+require "download_queue"
 
 module Homebrew
   # Helper module for performing (pre-)install checks.
@@ -314,15 +315,37 @@ module Homebrew
         skip_link: false
       )
         unless dry_run
-          formula_installers.each do |fi|
+          concurrency = ENV.fetch("HOMEBREW_DOWNLOAD_CONCURRENCY", 1).to_i
-            fi.prelude
+          if concurrency > 1
-            fi.fetch
+            retries = 0
-          rescue CannotInstallFormulaError => e
+            # TODO: disable force when done testing locally!!!
-            ofail e.message
+            force = true
-            next
+            raise if ENV["CI"] && force == true
-          rescue UnsatisfiedRequirements, DownloadError, ChecksumMismatchError => e
+
-            ofail "#{fi.formula}: #{e}"
+            download_queue = Homebrew::DownloadQueue.new(concurrency:, retries:, force:)
-            next
+          end
+          begin
+            formula_installers.each do |fi|
+              fi.prelude_fetch(download_queue:)
+              download_queue&.start
+              download_queue&.wait
+
+              fi.prelude(download_queue:)
+              download_queue&.start
+              download_queue&.wait
+
+              fi.fetch(download_queue:)
+              download_queue&.start
+              download_queue&.wait
+            rescue CannotInstallFormulaError => e
+              ofail e.message
+              next
+            rescue UnsatisfiedRequirements, DownloadError, ChecksumMismatchError => e
+              ofail "#{fi.formula}: #{e}"
+              next
+            end
+          ensure
+            download_queue&.shutdown
           end
         end
 

Library/Homebrew/resource.rb

@@ -91,6 +91,7 @@ class Resource
     patches.grep(DATAPatch) { |p| p.path = owner.owner.path }
   end
 
+  sig { params(skip_downloaded: T::Boolean).void }
   def fetch_patches(skip_downloaded: false)
     external_patches = patches.select(&:external?)
     external_patches.reject!(&:downloaded?) if skip_downloaded