From 36c7f4950cea500c51211a290894f1c40f10374c Mon Sep 17 00:00:00 2001 From: Mike McQuaid Date: Fri, 18 Jul 2025 15:01:34 +0100 Subject: [PATCH] Optionally parallelise API file downloads This assumes that all should be downloaded (at least once) on `brew` commands being run. Requires a certain amount of cleanup and refactoring around our API handling and Tap migration methods (which were both weirdly placed and in some cases broken). Behaviour without `HOMEBREW_DOWNLOAD_CONCURRENCY` set should be unchanged. --- Library/Homebrew/api.rb | 15 +++++++-- Library/Homebrew/api/cask.rb | 31 +++++++++++++++++- Library/Homebrew/api/download.rb | 19 ++++++++--- Library/Homebrew/api/formula.rb | 45 +++++++++++++++----------- Library/Homebrew/brew.rb | 17 ++++++++++ Library/Homebrew/download_queue.rb | 2 ++ Library/Homebrew/retryable_download.rb | 3 ++ Library/Homebrew/tap.rb | 11 ++----- 8 files changed, 106 insertions(+), 37 deletions(-) diff --git a/Library/Homebrew/api.rb b/Library/Homebrew/api.rb index 57051063b5..8e1a3994d0 100644 --- a/Library/Homebrew/api.rb +++ b/Library/Homebrew/api.rb @@ -13,6 +13,7 @@ module Homebrew HOMEBREW_CACHE_API = T.let((HOMEBREW_CACHE/"api").freeze, Pathname) HOMEBREW_CACHE_API_SOURCE = T.let((HOMEBREW_CACHE/"api-source").freeze, Pathname) + TAP_MIGRATIONS_STALE_SECONDS = T.let(86400, Integer) # 1 day sig { params(endpoint: String).returns(T::Hash[String, T.untyped]) } def self.fetch(endpoint) @@ -33,11 +34,11 @@ module Homebrew end sig { - params(endpoint: String, target: Pathname, - stale_seconds: Integer).returns([T.any(T::Array[T.untyped], T::Hash[String, T.untyped]), T::Boolean]) + params(endpoint: String, target: Pathname, stale_seconds: Integer, download_queue: T.nilable(DownloadQueue)) + .returns([T.any(T::Array[T.untyped], T::Hash[String, T.untyped]), T::Boolean]) } def self.fetch_json_api_file(endpoint, target: HOMEBREW_CACHE_API/endpoint, - stale_seconds: Homebrew::EnvConfig.api_auto_update_secs.to_i) + stale_seconds: Homebrew::EnvConfig.api_auto_update_secs.to_i, download_queue: nil) # Lazy-load dependency. require "development_tools" @@ -65,6 +66,14 @@ module Homebrew ((Time.now - stale_seconds) < target.mtime)) skip_download ||= Homebrew.running_as_root_but_not_owned_by_root? + if download_queue + unless skip_download + download = Homebrew::API::Download.new(url, nil, cache: HOMEBREW_CACHE_API, require_checksum: false) + download_queue.enqueue(download) + end + return [{}, false] + end + json_data = begin begin args = curl_args.dup diff --git a/Library/Homebrew/api/cask.rb b/Library/Homebrew/api/cask.rb index c5e6e5a6d0..87323a0ec8 100644 --- a/Library/Homebrew/api/cask.rb +++ b/Library/Homebrew/api/cask.rb @@ -2,7 +2,9 @@ # frozen_string_literal: true require "cachable" +require "api" require "api/download" +require "download_queue" module Homebrew module API @@ -52,9 +54,26 @@ module Homebrew HOMEBREW_CACHE_API/api_filename end + sig { + params(download_queue: T.nilable(::Homebrew::DownloadQueue)) + .returns([T.any(T::Array[T.untyped], T::Hash[String, T.untyped]), T::Boolean]) + } + def self.fetch_api!(download_queue: nil) + Homebrew::API.fetch_json_api_file api_filename, download_queue: + end + + sig { + params(download_queue: T.nilable(::Homebrew::DownloadQueue)) + .returns([T.any(T::Array[T.untyped], T::Hash[String, T.untyped]), T::Boolean]) + } + def self.fetch_tap_migrations!(download_queue: nil) + stale_seconds = Homebrew::API::TAP_MIGRATIONS_STALE_SECONDS + Homebrew::API.fetch_json_api_file "cask_tap_migrations.jws.json", stale_seconds:, download_queue: + end + sig { returns(T::Boolean) } def self.download_and_cache_data! - json_casks, updated = Homebrew::API.fetch_json_api_file api_filename + json_casks, updated = fetch_api! cache["renames"] = {} cache["casks"] = json_casks.to_h do |json_cask| @@ -91,6 +110,16 @@ module Homebrew cache.fetch("renames") end + sig { returns(T::Hash[String, T.untyped]) } + def self.tap_migrations + unless cache.key?("tap_migrations") + json_migrations, = fetch_tap_migrations! + cache["tap_migrations"] = json_migrations + end + + cache.fetch("tap_migrations") + end + sig { params(regenerate: T::Boolean).void } def self.write_names(regenerate: false) download_and_cache_data! unless cache.key?("casks") diff --git a/Library/Homebrew/api/download.rb b/Library/Homebrew/api/download.rb index e848daf328..db20a37d0a 100644 --- a/Library/Homebrew/api/download.rb +++ b/Library/Homebrew/api/download.rb @@ -17,18 +17,20 @@ module Homebrew sig { params( - url: String, - checksum: T.nilable(Checksum), - mirrors: T::Array[String], - cache: T.nilable(Pathname), + url: String, + checksum: T.nilable(Checksum), + mirrors: T::Array[String], + cache: T.nilable(Pathname), + require_checksum: T::Boolean, ).void } - def initialize(url, checksum, mirrors: [], cache: nil) + def initialize(url, checksum, mirrors: [], cache: nil, require_checksum: true) super() @url = T.let(URL.new(url, using: API::DownloadStrategy), URL) @checksum = checksum @mirrors = mirrors @cache = cache + @require_checksum = require_checksum end sig { override.returns(API::DownloadStrategy) } @@ -55,6 +57,13 @@ module Homebrew def symlink_location downloader.symlink_location end + + private + + sig { override.returns(T::Boolean) } + def silence_checksum_missing_error? + !@require_checksum + end end end end diff --git a/Library/Homebrew/api/formula.rb b/Library/Homebrew/api/formula.rb index 44f118bbdb..31cde44e68 100644 --- a/Library/Homebrew/api/formula.rb +++ b/Library/Homebrew/api/formula.rb @@ -2,7 +2,9 @@ # frozen_string_literal: true require "cachable" +require "api" require "api/download" +require "download_queue" module Homebrew module API @@ -52,9 +54,26 @@ module Homebrew HOMEBREW_CACHE_API/api_filename end + sig { + params(download_queue: T.nilable(Homebrew::DownloadQueue)) + .returns([T.any(T::Array[T.untyped], T::Hash[String, T.untyped]), T::Boolean]) + } + def self.fetch_api!(download_queue: nil) + Homebrew::API.fetch_json_api_file api_filename, download_queue: + end + + sig { + params(download_queue: T.nilable(Homebrew::DownloadQueue)) + .returns([T.any(T::Array[T.untyped], T::Hash[String, T.untyped]), T::Boolean]) + } + def self.fetch_tap_migrations!(download_queue: nil) + stale_seconds = Homebrew::API::TAP_MIGRATIONS_STALE_SECONDS + Homebrew::API.fetch_json_api_file "formula_tap_migrations.jws.json", stale_seconds:, download_queue: + end + sig { returns(T::Boolean) } def self.download_and_cache_data! - json_formulae, updated = Homebrew::API.fetch_json_api_file api_filename + json_formulae, updated = fetch_api! cache["aliases"] = {} cache["renames"] = {} @@ -80,7 +99,7 @@ module Homebrew write_names_and_aliases(regenerate: json_updated) end - cache["formulae"] + cache.fetch("formulae") end sig { returns(T::Hash[String, String]) } @@ -90,7 +109,7 @@ module Homebrew write_names_and_aliases(regenerate: json_updated) end - cache["aliases"] + cache.fetch("aliases") end sig { returns(T::Hash[String, String]) } @@ -100,29 +119,17 @@ module Homebrew write_names_and_aliases(regenerate: json_updated) end - cache["renames"] + cache.fetch("renames") end sig { returns(T::Hash[String, T.untyped]) } def self.tap_migrations - # Not sure that we need to reload here. unless cache.key?("tap_migrations") - json_updated = download_and_cache_data! - write_names_and_aliases(regenerate: json_updated) + json_migrations, = fetch_tap_migrations! + cache["tap_migrations"] = json_migrations end - cache["tap_migrations"] - end - - sig { returns(String) } - def self.tap_git_head - # Note sure we need to reload here. - unless cache.key?("tap_git_head") - json_updated = download_and_cache_data! - write_names_and_aliases(regenerate: json_updated) - end - - cache["tap_git_head"] + cache.fetch("tap_migrations") end sig { params(regenerate: T::Boolean).void } diff --git a/Library/Homebrew/brew.rb b/Library/Homebrew/brew.rb index 478517b1be..a86a393ec5 100644 --- a/Library/Homebrew/brew.rb +++ b/Library/Homebrew/brew.rb @@ -88,6 +88,23 @@ begin cmd_class = Homebrew::AbstractCommand.command(cmd) Homebrew.running_command = cmd if cmd_class + if Homebrew::EnvConfig.download_concurrency > 1 + require "download_queue" + require "api" + require "api/formula" + require "api/cask" + download_queue = Homebrew::DownloadQueue.new + Homebrew::API::Formula.fetch_api!(download_queue:) + Homebrew::API::Formula.fetch_tap_migrations!(download_queue:) + Homebrew::API::Cask.fetch_api!(download_queue:) + Homebrew::API::Cask.fetch_tap_migrations!(download_queue:) + begin + download_queue.fetch + ensure + download_queue.shutdown + end + end + command_instance = cmd_class.new require "utils/analytics" diff --git a/Library/Homebrew/download_queue.rb b/Library/Homebrew/download_queue.rb index 50dc8109c2..5787596354 100644 --- a/Library/Homebrew/download_queue.rb +++ b/Library/Homebrew/download_queue.rb @@ -28,6 +28,8 @@ module Homebrew if pour && download.bottle? UnpackStrategy.detect(download.cached_download, prioritize_extension: true) .extract_nestedly(to: HOMEBREW_CELLAR) + elsif download.api? + FileUtils.touch(download.cached_download, mtime: Time.now) end end end diff --git a/Library/Homebrew/retryable_download.rb b/Library/Homebrew/retryable_download.rb index 19a9ed3d02..c9ece5c491 100644 --- a/Library/Homebrew/retryable_download.rb +++ b/Library/Homebrew/retryable_download.rb @@ -92,6 +92,9 @@ module Homebrew sig { returns(T::Boolean) } def bottle? = downloadable.is_a?(Bottle) + sig { returns(T::Boolean) } + def api? = downloadable.is_a?(API::Download) + private sig { returns(Downloadable) } diff --git a/Library/Homebrew/tap.rb b/Library/Homebrew/tap.rb index 31c36dbc04..71eb1a2fd0 100644 --- a/Library/Homebrew/tap.rb +++ b/Library/Homebrew/tap.rb @@ -31,9 +31,6 @@ class Tap HOMEBREW_TAP_STYLE_EXCEPTIONS_DIR = "style_exceptions" private_constant :HOMEBREW_TAP_STYLE_EXCEPTIONS_DIR - TAP_MIGRATIONS_STALE_SECONDS = 86400 # 1 day - private_constant :TAP_MIGRATIONS_STALE_SECONDS - HOMEBREW_TAP_JSON_FILES = %W[ #{HOMEBREW_TAP_FORMULA_RENAMES_FILE} #{HOMEBREW_TAP_CASK_RENAMES_FILE} @@ -1312,9 +1309,7 @@ class CoreTap < AbstractCoreTap ensure_installed! super else - migrations, = Homebrew::API.fetch_json_api_file "formula_tap_migrations.jws.json", - stale_seconds: TAP_MIGRATIONS_STALE_SECONDS - migrations + Homebrew::API::Formula.tap_migrations end end @@ -1471,9 +1466,7 @@ class CoreCaskTap < AbstractCoreTap @tap_migrations ||= if Homebrew::EnvConfig.no_install_from_api? super else - migrations, = Homebrew::API.fetch_json_api_file "cask_tap_migrations.jws.json", - stale_seconds: TAP_MIGRATIONS_STALE_SECONDS - migrations + Homebrew::API::Cask.tap_migrations end end end