Merge pull request #20273 from Homebrew/api_download_queue

Optionally parallelise API file downloads

Library/Homebrew/api.rb

@@ -13,6 +13,7 @@ module Homebrew
 
     HOMEBREW_CACHE_API = T.let((HOMEBREW_CACHE/"api").freeze, Pathname)
     HOMEBREW_CACHE_API_SOURCE = T.let((HOMEBREW_CACHE/"api-source").freeze, Pathname)
+    TAP_MIGRATIONS_STALE_SECONDS = T.let(86400, Integer) # 1 day
 
     sig { params(endpoint: String).returns(T::Hash[String, T.untyped]) }
     def self.fetch(endpoint)
@@ -33,11 +34,11 @@ module Homebrew
     end
 
     sig {
-      params(endpoint: String, target: Pathname,
+      params(endpoint: String, target: Pathname, stale_seconds: Integer, download_queue: T.nilable(DownloadQueue))
-             stale_seconds: Integer).returns([T.any(T::Array[T.untyped], T::Hash[String, T.untyped]), T::Boolean])
+        .returns([T.any(T::Array[T.untyped], T::Hash[String, T.untyped]), T::Boolean])
     }
     def self.fetch_json_api_file(endpoint, target: HOMEBREW_CACHE_API/endpoint,
-                                 stale_seconds: Homebrew::EnvConfig.api_auto_update_secs.to_i)
+                                 stale_seconds: Homebrew::EnvConfig.api_auto_update_secs.to_i, download_queue: nil)
       # Lazy-load dependency.
       require "development_tools"
 
@@ -65,6 +66,14 @@ module Homebrew
                       ((Time.now - stale_seconds) < target.mtime))
       skip_download ||= Homebrew.running_as_root_but_not_owned_by_root?
 
+      if download_queue
+        unless skip_download
+          download = Homebrew::API::Download.new(url, nil, cache: HOMEBREW_CACHE_API, require_checksum: false)
+          download_queue.enqueue(download)
+        end
+        return [{}, false]
+      end
+
       json_data = begin
         begin
           args = curl_args.dup

Library/Homebrew/api/cask.rb

@@ -2,7 +2,9 @@
 # frozen_string_literal: true
 
 require "cachable"
+require "api"
 require "api/download"
+require "download_queue"
 
 module Homebrew
   module API
@@ -52,9 +54,26 @@ module Homebrew
         HOMEBREW_CACHE_API/api_filename
       end
 
+      sig {
+        params(download_queue: T.nilable(::Homebrew::DownloadQueue))
+          .returns([T.any(T::Array[T.untyped], T::Hash[String, T.untyped]), T::Boolean])
+      }
+      def self.fetch_api!(download_queue: nil)
+        Homebrew::API.fetch_json_api_file api_filename, download_queue:
+      end
+
+      sig {
+        params(download_queue: T.nilable(::Homebrew::DownloadQueue))
+          .returns([T.any(T::Array[T.untyped], T::Hash[String, T.untyped]), T::Boolean])
+      }
+      def self.fetch_tap_migrations!(download_queue: nil)
+        stale_seconds = Homebrew::API::TAP_MIGRATIONS_STALE_SECONDS
+        Homebrew::API.fetch_json_api_file "cask_tap_migrations.jws.json", stale_seconds:, download_queue:
+      end
+
       sig { returns(T::Boolean) }
       def self.download_and_cache_data!
-        json_casks, updated = Homebrew::API.fetch_json_api_file api_filename
+        json_casks, updated = fetch_api!
 
         cache["renames"] = {}
         cache["casks"] = json_casks.to_h do |json_cask|
@@ -91,6 +110,16 @@ module Homebrew
         cache.fetch("renames")
       end
 
+      sig { returns(T::Hash[String, T.untyped]) }
+      def self.tap_migrations
+        unless cache.key?("tap_migrations")
+          json_migrations, = fetch_tap_migrations!
+          cache["tap_migrations"] = json_migrations
+        end
+
+        cache.fetch("tap_migrations")
+      end
+
       sig { params(regenerate: T::Boolean).void }
       def self.write_names(regenerate: false)
         download_and_cache_data! unless cache.key?("casks")

Library/Homebrew/api/download.rb

@@ -17,18 +17,20 @@ module Homebrew
 
       sig {
         params(
-          url:      String,
+          url:              String,
-          checksum: T.nilable(Checksum),
+          checksum:         T.nilable(Checksum),
-          mirrors:  T::Array[String],
+          mirrors:          T::Array[String],
-          cache:    T.nilable(Pathname),
+          cache:            T.nilable(Pathname),
+          require_checksum: T::Boolean,
         ).void
       }
-      def initialize(url, checksum, mirrors: [], cache: nil)
+      def initialize(url, checksum, mirrors: [], cache: nil, require_checksum: true)
         super()
         @url = T.let(URL.new(url, using: API::DownloadStrategy), URL)
         @checksum = checksum
         @mirrors = mirrors
         @cache = cache
+        @require_checksum = require_checksum
       end
 
       sig { override.returns(API::DownloadStrategy) }
@@ -55,6 +57,13 @@ module Homebrew
       def symlink_location
         downloader.symlink_location
       end
+
+      private
+
+      sig { override.returns(T::Boolean) }
+      def silence_checksum_missing_error?
+        !@require_checksum
+      end
     end
   end
 end

Library/Homebrew/api/formula.rb

@@ -2,7 +2,9 @@
 # frozen_string_literal: true
 
 require "cachable"
+require "api"
 require "api/download"
+require "download_queue"
 
 module Homebrew
   module API
@@ -52,9 +54,26 @@ module Homebrew
         HOMEBREW_CACHE_API/api_filename
       end
 
+      sig {
+        params(download_queue: T.nilable(Homebrew::DownloadQueue))
+          .returns([T.any(T::Array[T.untyped], T::Hash[String, T.untyped]), T::Boolean])
+      }
+      def self.fetch_api!(download_queue: nil)
+        Homebrew::API.fetch_json_api_file api_filename, download_queue:
+      end
+
+      sig {
+        params(download_queue: T.nilable(Homebrew::DownloadQueue))
+          .returns([T.any(T::Array[T.untyped], T::Hash[String, T.untyped]), T::Boolean])
+      }
+      def self.fetch_tap_migrations!(download_queue: nil)
+        stale_seconds = Homebrew::API::TAP_MIGRATIONS_STALE_SECONDS
+        Homebrew::API.fetch_json_api_file "formula_tap_migrations.jws.json", stale_seconds:, download_queue:
+      end
+
       sig { returns(T::Boolean) }
       def self.download_and_cache_data!
-        json_formulae, updated = Homebrew::API.fetch_json_api_file api_filename
+        json_formulae, updated = fetch_api!
 
         cache["aliases"] = {}
         cache["renames"] = {}
@@ -80,7 +99,7 @@ module Homebrew
           write_names_and_aliases(regenerate: json_updated)
         end
 
-        cache["formulae"]
+        cache.fetch("formulae")
       end
 
       sig { returns(T::Hash[String, String]) }
@@ -90,7 +109,7 @@ module Homebrew
           write_names_and_aliases(regenerate: json_updated)
         end
 
-        cache["aliases"]
+        cache.fetch("aliases")
       end
 
       sig { returns(T::Hash[String, String]) }
@@ -100,29 +119,17 @@ module Homebrew
           write_names_and_aliases(regenerate: json_updated)
         end
 
-        cache["renames"]
+        cache.fetch("renames")
       end
 
       sig { returns(T::Hash[String, T.untyped]) }
       def self.tap_migrations
-        # Not sure that we need to reload here.
         unless cache.key?("tap_migrations")
-          json_updated = download_and_cache_data!
+          json_migrations, = fetch_tap_migrations!
-          write_names_and_aliases(regenerate: json_updated)
+          cache["tap_migrations"] = json_migrations
         end
 
-        cache["tap_migrations"]
+        cache.fetch("tap_migrations")
-      end
-
-      sig { returns(String) }
-      def self.tap_git_head
-        # Note sure we need to reload here.
-        unless cache.key?("tap_git_head")
-          json_updated = download_and_cache_data!
-          write_names_and_aliases(regenerate: json_updated)
-        end
-
-        cache["tap_git_head"]
       end
 
       sig { params(regenerate: T::Boolean).void }

Library/Homebrew/brew.rb

@@ -88,6 +88,23 @@ begin
     cmd_class = Homebrew::AbstractCommand.command(cmd)
     Homebrew.running_command = cmd
     if cmd_class
+      if Homebrew::EnvConfig.download_concurrency > 1
+        require "download_queue"
+        require "api"
+        require "api/formula"
+        require "api/cask"
+        download_queue = Homebrew::DownloadQueue.new
+        Homebrew::API::Formula.fetch_api!(download_queue:)
+        Homebrew::API::Formula.fetch_tap_migrations!(download_queue:)
+        Homebrew::API::Cask.fetch_api!(download_queue:)
+        Homebrew::API::Cask.fetch_tap_migrations!(download_queue:)
+        begin
+          download_queue.fetch
+        ensure
+          download_queue.shutdown
+        end
+      end
+
       command_instance = cmd_class.new
 
       require "utils/analytics"

Library/Homebrew/download_queue.rb

@@ -28,6 +28,8 @@ module Homebrew
         if pour && download.bottle?
           UnpackStrategy.detect(download.cached_download, prioritize_extension: true)
                         .extract_nestedly(to: HOMEBREW_CELLAR)
+        elsif download.api?
+          FileUtils.touch(download.cached_download, mtime: Time.now)
         end
       end
     end

Library/Homebrew/retryable_download.rb

@@ -92,6 +92,9 @@ module Homebrew
     sig { returns(T::Boolean) }
     def bottle? = downloadable.is_a?(Bottle)
 
+    sig { returns(T::Boolean) }
+    def api? = downloadable.is_a?(API::Download)
+
     private
 
     sig { returns(Downloadable) }

Library/Homebrew/tap.rb

@@ -31,9 +31,6 @@ class Tap
   HOMEBREW_TAP_STYLE_EXCEPTIONS_DIR = "style_exceptions"
   private_constant :HOMEBREW_TAP_STYLE_EXCEPTIONS_DIR
 
-  TAP_MIGRATIONS_STALE_SECONDS = 86400 # 1 day
-  private_constant :TAP_MIGRATIONS_STALE_SECONDS
-
   HOMEBREW_TAP_JSON_FILES = %W[
     #{HOMEBREW_TAP_FORMULA_RENAMES_FILE}
     #{HOMEBREW_TAP_CASK_RENAMES_FILE}
@@ -1312,9 +1309,7 @@ class CoreTap < AbstractCoreTap
       ensure_installed!
       super
     else
-      migrations, = Homebrew::API.fetch_json_api_file "formula_tap_migrations.jws.json",
+      Homebrew::API::Formula.tap_migrations
-                                                      stale_seconds: TAP_MIGRATIONS_STALE_SECONDS
-      migrations
     end
   end
 
@@ -1471,9 +1466,7 @@ class CoreCaskTap < AbstractCoreTap
     @tap_migrations ||= if Homebrew::EnvConfig.no_install_from_api?
       super
     else
-      migrations, = Homebrew::API.fetch_json_api_file "cask_tap_migrations.jws.json",
+      Homebrew::API::Cask.tap_migrations
-                                                      stale_seconds: TAP_MIGRATIONS_STALE_SECONDS
-      migrations
     end
   end
 end