From 9255a1fd84eb02a31f7a3cb68ae266d525e3d3a3 Mon Sep 17 00:00:00 2001 From: Mike McQuaid Date: Wed, 23 Jul 2025 16:25:44 +0100 Subject: [PATCH 1/4] workflows/tests: set HOMEBREW_DOWNLOAD_CONCURRENCY for test-bot tests. This should give us some basic integration testing for the new download concurrency code before we enabled/test it in homebrew-core's CI. --- .github/workflows/tests.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index d3e05a7718..6ef76a15b0 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -368,6 +368,7 @@ jobs: env: HOMEBREW_TEST_BOT_ANALYTICS: 1 HOMEBREW_ENFORCE_SBOM: 1 + HOMEBREW_DOWNLOAD_CONCURRENCY: 4 steps: - name: Install Homebrew and Homebrew's dependencies # All other images are built from our Homebrew Dockerfile. From d3a9525fd7f12ebe33845f913825df975f864d59 Mon Sep 17 00:00:00 2001 From: Mike McQuaid Date: Wed, 23 Jul 2025 17:00:52 +0100 Subject: [PATCH 2/4] retryable_download: ensure HOMEBREW_CELLAR exists. --- Library/Homebrew/retryable_download.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/Library/Homebrew/retryable_download.rb b/Library/Homebrew/retryable_download.rb index f28e614893..37e9a00661 100644 --- a/Library/Homebrew/retryable_download.rb +++ b/Library/Homebrew/retryable_download.rb @@ -73,6 +73,7 @@ module Homebrew downloadable.verify_download_integrity(download) if verify_download_integrity && !json_download if pour && downloadable.is_a?(Bottle) + HOMEBREW_CELLAR.mkpath UnpackStrategy.detect(download, prioritize_extension: true) .extract_nestedly(to: HOMEBREW_CELLAR) elsif json_download From 0c09ed71d4797db42266380b9859b148aa8a3e2b Mon Sep 17 00:00:00 2001 From: Mike McQuaid Date: Fri, 25 Jul 2025 17:25:41 +0100 Subject: [PATCH 3/4] formula_installer: fix fetch deps and local pour for download queue. --- Library/Homebrew/formula_installer.rb | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/Library/Homebrew/formula_installer.rb b/Library/Homebrew/formula_installer.rb index cf37d3eab4..f2b51c4125 100644 --- a/Library/Homebrew/formula_installer.rb +++ b/Library/Homebrew/formula_installer.rb @@ -320,6 +320,8 @@ class FormulaInstaller # Needs to be done before expand_dependencies for compute_dependencies fetch_bottle_tab if pour_bottle? + fetch_fetch_deps unless ignore_deps? + @ran_prelude_fetch = true end @@ -346,7 +348,9 @@ class FormulaInstaller forbidden_formula_check check_install_sanity - install_fetch_deps unless ignore_deps? + + # with the download queue: these should have already been installed + install_fetch_deps if !ignore_deps? && download_queue.nil? end sig { void } @@ -475,6 +479,18 @@ class FormulaInstaller sig { params(_formula: Formula).returns(T.nilable(T::Boolean)) } def fresh_install?(_formula) = false + sig { void } + def fetch_fetch_deps + return if @compute_dependencies.blank? + + compute_dependencies(use_cache: false) if @compute_dependencies.any? do |dep,| + next false unless dep.implicit? + + fetch_dependencies + true + end + end + sig { void } def install_fetch_deps return if @compute_dependencies.blank? @@ -1455,8 +1471,6 @@ on_request: installed_on_request?, options:) # We also skip bottle installs from local bottle paths, as these are done in CI # as part of the build lifecycle before attestations are produced. if check_attestation && - # TODO: support this for download queues at some point - download_queue.nil? && Homebrew::Attestation.enabled? && formula.tap&.core_tap? && formula.name != "gh" @@ -1545,7 +1559,7 @@ on_request: installed_on_request?, options:) # download queue has already done the actual staging but we'll lie about # pouring now for nicer output ohai "Pouring #{downloadable.downloader.basename}" - downloadable.downloader.stage unless download_queue + downloadable.downloader.stage if download_queue.nil? || !formula.prefix.exist? end Tab.clear_cache From b63669c92b6731504c1fb1fba6baee26da9d6684 Mon Sep 17 00:00:00 2001 From: Mike McQuaid Date: Tue, 29 Jul 2025 12:42:13 +0100 Subject: [PATCH 4/4] download_queue: fix bottle manifest handling. These should be ignored upon failures. --- Library/Homebrew/download_queue.rb | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/Library/Homebrew/download_queue.rb b/Library/Homebrew/download_queue.rb index 7e372f8a91..05dceb0d40 100644 --- a/Library/Homebrew/download_queue.rb +++ b/Library/Homebrew/download_queue.rb @@ -38,6 +38,8 @@ module Homebrew rescue ChecksumMismatchError => e opoo "#{downloadable.download_type} reports different checksum: #{e.expected}" Homebrew.failed = true if downloadable.is_a?(Resource::Patch) + rescue => e + raise e unless bottle_manifest_error?(downloadable, e) end else spinner = Spinner.new @@ -68,6 +70,9 @@ module Homebrew raise future.state.to_s end + exception = future.reason if future.rejected? + next 1 if bottle_manifest_error?(downloadable, exception) + message = "#{downloadable.download_type} #{downloadable.name}" if tty stdout_print_and_flush "#{status} #{message}#{"\n" unless last}" @@ -76,14 +81,13 @@ module Homebrew end if future.rejected? - if (e = future.reason).is_a?(ChecksumMismatchError) - opoo "#{downloadable.download_type} reports different checksum: #{e.expected}" + if exception.is_a?(ChecksumMismatchError) + opoo "#{downloadable.download_type} reports different checksum: #{exception.expected}" Homebrew.failed = true if downloadable.is_a?(Resource::Patch) next 2 else message = future.reason.to_s - onoe message - Homebrew.failed = true + ofail message next message.count("\n") end end @@ -165,6 +169,13 @@ module Homebrew private + sig { params(downloadable: Downloadable, exception: T.nilable(Exception)).returns(T::Boolean) } + def bottle_manifest_error?(downloadable, exception) + return false if exception.nil? + + downloadable.is_a?(Resource::BottleManifest) || exception.is_a?(Resource::BottleManifest::Error) + end + sig { void } def cancel # FIXME: Implement graceful cancellation of running downloads based on