admin/brew
1
0
Fork 0
Code Issues Packages Projects Releases 6 Wiki Activity

Optionally parallelise API file downloads

Browse Source 
This assumes that all should be downloaded (at least once) on `brew`
commands being run.

Requires a certain amount of cleanup and refactoring around our API
handling and Tap migration methods (which were both weirdly placed and
in some cases broken).

Behaviour without `HOMEBREW_DOWNLOAD_CONCURRENCY` set should
be unchanged.
This commit is contained in:
Mike McQuaid 2025-07-18 15:01:34 +01:00
parent 5cc6722372
commit 36c7f4950c
No known key found for this signature in database
8 changed files with 106 additions and 37 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
Library/Homebrew/api.rb
View File

@ -13,6 +13,7 @@ module Homebrew
HOMEBREW_CACHE_API = T.let((HOMEBREW_CACHE/"api").freeze, Pathname)
HOMEBREW_CACHE_API_SOURCE = T.let((HOMEBREW_CACHE/"api-source").freeze, Pathname)
TAP_MIGRATIONS_STALE_SECONDS = T.let(86400, Integer) # 1 day
sig { params(endpoint: String).returns(T::Hash[String, T.untyped]) }
def self.fetch(endpoint)
@ -33,11 +34,11 @@ module Homebrew
end
sig {
params(endpoint: String, target: Pathname,
stale_seconds: Integer).returns([T.any(T::Array[T.untyped], T::Hash[String, T.untyped]), T::Boolean])
params(endpoint: String, target: Pathname, stale_seconds: Integer, download_queue: T.nilable(DownloadQueue))
.returns([T.any(T::Array[T.untyped], T::Hash[String, T.untyped]), T::Boolean])
}
def self.fetch_json_api_file(endpoint, target: HOMEBREW_CACHE_API/endpoint,
stale_seconds: Homebrew::EnvConfig.api_auto_update_secs.to_i)
stale_seconds: Homebrew::EnvConfig.api_auto_update_secs.to_i, download_queue: nil)
# Lazy-load dependency.
require "development_tools"
@ -65,6 +66,14 @@ module Homebrew
((Time.now - stale_seconds) < target.mtime))
skip_download ||= Homebrew.running_as_root_but_not_owned_by_root?
if download_queue
unless skip_download
download = Homebrew::API::Download.new(url, nil, cache: HOMEBREW_CACHE_API, require_checksum: false)
download_queue.enqueue(download)
end
return [{}, false]
end
json_data = begin
begin
args = curl_args.dup

31
Library/Homebrew/api/cask.rb
View File

@ -2,7 +2,9 @@
# frozen_string_literal: true
require "cachable"
require "api"
require "api/download"
require "download_queue"
module Homebrew
module API
@ -52,9 +54,26 @@ module Homebrew
HOMEBREW_CACHE_API/api_filename
end
sig {
params(download_queue: T.nilable(::Homebrew::DownloadQueue))
.returns([T.any(T::Array[T.untyped], T::Hash[String, T.untyped]), T::Boolean])
}
def self.fetch_api!(download_queue: nil)
Homebrew::API.fetch_json_api_file api_filename, download_queue:
end
sig {
params(download_queue: T.nilable(::Homebrew::DownloadQueue))
.returns([T.any(T::Array[T.untyped], T::Hash[String, T.untyped]), T::Boolean])
}
def self.fetch_tap_migrations!(download_queue: nil)
stale_seconds = Homebrew::API::TAP_MIGRATIONS_STALE_SECONDS
Homebrew::API.fetch_json_api_file "cask_tap_migrations.jws.json", stale_seconds:, download_queue:
end
sig { returns(T::Boolean) }
def self.download_and_cache_data!
json_casks, updated = Homebrew::API.fetch_json_api_file api_filename
json_casks, updated = fetch_api!
cache["renames"] = {}
cache["casks"] = json_casks.to_h do |json_cask|
@ -91,6 +110,16 @@ module Homebrew
cache.fetch("renames")
end
sig { returns(T::Hash[String, T.untyped]) }
def self.tap_migrations
unless cache.key?("tap_migrations")
json_migrations, = fetch_tap_migrations!
cache["tap_migrations"] = json_migrations
end
cache.fetch("tap_migrations")
end
sig { params(regenerate: T::Boolean).void }
def self.write_names(regenerate: false)
download_and_cache_data! unless cache.key?("casks")

19
Library/Homebrew/api/download.rb
View File

@ -17,18 +17,20 @@ module Homebrew
sig {
params(
url: String,
checksum: T.nilable(Checksum),
mirrors: T::Array[String],
cache: T.nilable(Pathname),
url: String,
checksum: T.nilable(Checksum),
mirrors: T::Array[String],
cache: T.nilable(Pathname),
require_checksum: T::Boolean,
).void
}
def initialize(url, checksum, mirrors: [], cache: nil)
def initialize(url, checksum, mirrors: [], cache: nil, require_checksum: true)
super()
@url = T.let(URL.new(url, using: API::DownloadStrategy), URL)
@checksum = checksum
@mirrors = mirrors
@cache = cache
@require_checksum = require_checksum
end
sig { override.returns(API::DownloadStrategy) }
@ -55,6 +57,13 @@ module Homebrew
def symlink_location
downloader.symlink_location
end
private
sig { override.returns(T::Boolean) }
def silence_checksum_missing_error?
!@require_checksum
end
end
end
end

45
Library/Homebrew/api/formula.rb
View File

@ -2,7 +2,9 @@
# frozen_string_literal: true
require "cachable"
require "api"
require "api/download"
require "download_queue"
module Homebrew
module API
@ -52,9 +54,26 @@ module Homebrew
HOMEBREW_CACHE_API/api_filename
end
sig {
params(download_queue: T.nilable(Homebrew::DownloadQueue))
.returns([T.any(T::Array[T.untyped], T::Hash[String, T.untyped]), T::Boolean])
}
def self.fetch_api!(download_queue: nil)
Homebrew::API.fetch_json_api_file api_filename, download_queue:
end
sig {
params(download_queue: T.nilable(Homebrew::DownloadQueue))
.returns([T.any(T::Array[T.untyped], T::Hash[String, T.untyped]), T::Boolean])
}
def self.fetch_tap_migrations!(download_queue: nil)
stale_seconds = Homebrew::API::TAP_MIGRATIONS_STALE_SECONDS
Homebrew::API.fetch_json_api_file "formula_tap_migrations.jws.json", stale_seconds:, download_queue:
end
sig { returns(T::Boolean) }
def self.download_and_cache_data!
json_formulae, updated = Homebrew::API.fetch_json_api_file api_filename
json_formulae, updated = fetch_api!
cache["aliases"] = {}
cache["renames"] = {}
@ -80,7 +99,7 @@ module Homebrew
write_names_and_aliases(regenerate: json_updated)
end
cache["formulae"]
cache.fetch("formulae")
end
sig { returns(T::Hash[String, String]) }
@ -90,7 +109,7 @@ module Homebrew
write_names_and_aliases(regenerate: json_updated)
end
cache["aliases"]
cache.fetch("aliases")
end
sig { returns(T::Hash[String, String]) }
@ -100,29 +119,17 @@ module Homebrew
write_names_and_aliases(regenerate: json_updated)
end
cache["renames"]
cache.fetch("renames")
end
sig { returns(T::Hash[String, T.untyped]) }
def self.tap_migrations
# Not sure that we need to reload here.
unless cache.key?("tap_migrations")
json_updated = download_and_cache_data!
write_names_and_aliases(regenerate: json_updated)
json_migrations, = fetch_tap_migrations!
cache["tap_migrations"] = json_migrations
end
cache["tap_migrations"]
end
sig { returns(String) }
def self.tap_git_head
# Note sure we need to reload here.
unless cache.key?("tap_git_head")
json_updated = download_and_cache_data!
write_names_and_aliases(regenerate: json_updated)
end
cache["tap_git_head"]
cache.fetch("tap_migrations")
end
sig { params(regenerate: T::Boolean).void }

17
Library/Homebrew/brew.rb
View File

@ -88,6 +88,23 @@ begin
cmd_class = Homebrew::AbstractCommand.command(cmd)
Homebrew.running_command = cmd
if cmd_class
if Homebrew::EnvConfig.download_concurrency > 1
require "download_queue"
require "api"
require "api/formula"
require "api/cask"
download_queue = Homebrew::DownloadQueue.new
Homebrew::API::Formula.fetch_api!(download_queue:)
Homebrew::API::Formula.fetch_tap_migrations!(download_queue:)
Homebrew::API::Cask.fetch_api!(download_queue:)
Homebrew::API::Cask.fetch_tap_migrations!(download_queue:)
begin
download_queue.fetch
ensure
download_queue.shutdown
end
end
command_instance = cmd_class.new
require "utils/analytics"

2
Library/Homebrew/download_queue.rb
View File

@ -28,6 +28,8 @@ module Homebrew
if pour && download.bottle?
UnpackStrategy.detect(download.cached_download, prioritize_extension: true)
.extract_nestedly(to: HOMEBREW_CELLAR)
elsif download.api?
FileUtils.touch(download.cached_download, mtime: Time.now)
end
end
end

3
Library/Homebrew/retryable_download.rb
View File

@ -92,6 +92,9 @@ module Homebrew
sig { returns(T::Boolean) }
def bottle? = downloadable.is_a?(Bottle)
sig { returns(T::Boolean) }
def api? = downloadable.is_a?(API::Download)
private
sig { returns(Downloadable) }

11
Library/Homebrew/tap.rb
View File

@ -31,9 +31,6 @@ class Tap
HOMEBREW_TAP_STYLE_EXCEPTIONS_DIR = "style_exceptions"
private_constant :HOMEBREW_TAP_STYLE_EXCEPTIONS_DIR
TAP_MIGRATIONS_STALE_SECONDS = 86400 # 1 day
private_constant :TAP_MIGRATIONS_STALE_SECONDS
HOMEBREW_TAP_JSON_FILES = %W[
#{HOMEBREW_TAP_FORMULA_RENAMES_FILE}
#{HOMEBREW_TAP_CASK_RENAMES_FILE}
@ -1312,9 +1309,7 @@ class CoreTap < AbstractCoreTap
ensure_installed!
super
else
migrations, = Homebrew::API.fetch_json_api_file "formula_tap_migrations.jws.json",
stale_seconds: TAP_MIGRATIONS_STALE_SECONDS
migrations
Homebrew::API::Formula.tap_migrations
end
end
@ -1471,9 +1466,7 @@ class CoreCaskTap < AbstractCoreTap
@tap_migrations ||= if Homebrew::EnvConfig.no_install_from_api?
super
else
migrations, = Homebrew::API.fetch_json_api_file "cask_tap_migrations.jws.json",
stale_seconds: TAP_MIGRATIONS_STALE_SECONDS
migrations
Homebrew::API::Cask.tap_migrations
end
end
end