diff --git a/Library/Homebrew/cmd/fetch.rb b/Library/Homebrew/cmd/fetch.rb index 6c12d1c233..a47ac2158d 100644 --- a/Library/Homebrew/cmd/fetch.rb +++ b/Library/Homebrew/cmd/fetch.rb @@ -26,7 +26,6 @@ module Homebrew "(Pass `all` to download for all architectures.)" flag "--bottle-tag=", description: "Download a bottle for given tag." - flag "--concurrency=", description: "Number of concurrent downloads.", hidden: true switch "--HEAD", description: "Fetch HEAD version instead of stable version." switch "-f", "--force", @@ -150,12 +149,7 @@ module Homebrew download_queue.enqueue(resource) end - formula.resources.each do |r| - download_queue.enqueue(r) - r.patches.each { |patch| download_queue.enqueue(patch.resource) if patch.external? } - end - - formula.patchlist.each { |patch| download_queue.enqueue(patch.resource) if patch.external? } + formula.enqueue_resources_and_patches(download_queue:) end end else @@ -181,18 +175,13 @@ module Homebrew end end - download_queue.start + download_queue.fetch ensure download_queue.shutdown end private - sig { returns(Integer) } - def concurrency - @concurrency ||= T.let(args.concurrency&.to_i || 1, T.nilable(Integer)) - end - sig { returns(Integer) } def retries @retries ||= T.let(args.retry? ? FETCH_MAX_TRIES : 0, T.nilable(Integer)) @@ -201,7 +190,7 @@ module Homebrew sig { returns(DownloadQueue) } def download_queue @download_queue ||= T.let(begin - DownloadQueue.new(concurrency:, retries:, force: args.force?) + DownloadQueue.new(retries:, force: args.force?) end, T.nilable(DownloadQueue)) end end diff --git a/Library/Homebrew/download_queue.rb b/Library/Homebrew/download_queue.rb index 0de8add528..162075944a 100644 --- a/Library/Homebrew/download_queue.rb +++ b/Library/Homebrew/download_queue.rb @@ -8,16 +8,16 @@ require "retryable_download" module Homebrew class DownloadQueue - sig { params(concurrency: Integer, retries: Integer, force: T::Boolean).void } - def initialize(concurrency:, retries:, force:) - @concurrency = concurrency - @quiet = T.let(concurrency > 1, T::Boolean) + sig { params(retries: Integer, force: T::Boolean).void } + def initialize(retries: 0, force: false) + @concurrency = T.let(EnvConfig.download_concurrency, Integer) + @quiet = T.let(@concurrency > 1, T::Boolean) @tries = T.let(retries + 1, Integer) @force = force @pool = T.let(Concurrent::FixedThreadPool.new(concurrency), Concurrent::FixedThreadPool) end - sig { params(downloadable: T.any(Resource, Bottle, Cask::Download)).void } + sig { params(downloadable: Downloadable).void } def enqueue(downloadable) downloads[downloadable] ||= Concurrent::Promises.future_on( pool, RetryableDownload.new(downloadable, tries:), force, quiet @@ -28,7 +28,7 @@ module Homebrew end sig { void } - def start + def fetch if concurrency == 1 downloads.each do |downloadable, promise| promise.wait! @@ -133,6 +133,8 @@ module Homebrew $stdout.flush end end + + downloads.clear end sig { void } @@ -166,10 +168,9 @@ module Homebrew sig { returns(T::Boolean) } attr_reader :quiet - sig { returns(T::Hash[T.any(Resource, Bottle, Cask::Download), Concurrent::Promises::Future]) } + sig { returns(T::Hash[Downloadable, Concurrent::Promises::Future]) } def downloads - @downloads ||= T.let({}, T.nilable(T::Hash[T.any(Resource, Bottle, Cask::Download), - Concurrent::Promises::Future])) + @downloads ||= T.let({}, T.nilable(T::Hash[Downloadable, Concurrent::Promises::Future])) end class Spinner diff --git a/Library/Homebrew/env_config.rb b/Library/Homebrew/env_config.rb index 1af588add6..6a2a558232 100644 --- a/Library/Homebrew/env_config.rb +++ b/Library/Homebrew/env_config.rb @@ -630,5 +630,13 @@ module Homebrew def devcmdrun? Homebrew::Settings.read("devcmdrun") == "true" end + + sig { returns(Integer) } + def download_concurrency + # TODO: document this variable when ready to publicly announce it. + concurrency = ENV.fetch("HOMEBREW_DOWNLOAD_CONCURRENCY", 1).to_i + concurrency = 1 if concurrency <= 1 + concurrency + end end end diff --git a/Library/Homebrew/formula.rb b/Library/Homebrew/formula.rb index ac85783f92..ee5356193a 100644 --- a/Library/Homebrew/formula.rb +++ b/Library/Homebrew/formula.rb @@ -3202,6 +3202,15 @@ class Formula end end + sig { params(download_queue: Homebrew::DownloadQueue).void } + def enqueue_resources_and_patches(download_queue:) + resources.each do |resource| + download_queue.enqueue(resource) + resource.patches.select(&:external?).each { |patch| download_queue.enqueue(patch.resource) } + end + patchlist.select(&:external?).each { |patch| download_queue.enqueue(patch.resource) } + end + sig { void } def fetch_patches patchlist.select(&:external?).each(&:fetch) diff --git a/Library/Homebrew/retryable_download.rb b/Library/Homebrew/retryable_download.rb index 8bcb6e9126..896a008a65 100644 --- a/Library/Homebrew/retryable_download.rb +++ b/Library/Homebrew/retryable_download.rb @@ -5,10 +5,6 @@ module Homebrew class RetryableDownload include Downloadable - sig { returns(Downloadable) } - attr_reader :downloadable - private :downloadable - sig { override.returns(T.any(NilClass, String, URL)) } def url = downloadable.url @@ -92,5 +88,10 @@ module Homebrew sig { override.returns(String) } def download_name = downloadable.download_name + + private + + sig { returns(Downloadable) } + attr_reader :downloadable end end diff --git a/Library/Homebrew/test/cmd/fetch_spec.rb b/Library/Homebrew/test/cmd/fetch_spec.rb index c264a66147..f63c977f9b 100644 --- a/Library/Homebrew/test/cmd/fetch_spec.rb +++ b/Library/Homebrew/test/cmd/fetch_spec.rb @@ -19,7 +19,7 @@ RSpec.describe Homebrew::Cmd::FetchCmd do setup_test_formula "testball1" setup_test_formula "testball2" - expect { brew "fetch", "testball1", "testball2", "--concurrency=2" }.to be_a_success + expect { brew "fetch", "testball1", "testball2", "HOMEBREW_DOWNLOAD_CONCURRENCY" => "2" }.to be_a_success expect(HOMEBREW_CACHE/"testball1--0.1.tbz").to be_a_symlink expect(HOMEBREW_CACHE/"testball1--0.1.tbz").to exist @@ -33,7 +33,7 @@ RSpec.describe Homebrew::Cmd::FetchCmd do setup_test_formula "testball1" setup_test_formula "testball3" - expect { brew "fetch", "testball1", "testball3", "--concurrency=2" }.to be_a_failure + expect { brew "fetch", "testball1", "testball3", "HOMEBREW_DOWNLOAD_CONCURRENCY" => "2" }.to be_a_failure .and output(/Error:.*process has already locked/).to_stderr end end