From 27dda52a5d6e96799031fd52a64847130861a20d Mon Sep 17 00:00:00 2001 From: Mike McQuaid Date: Wed, 5 Jul 2023 17:18:12 +0100 Subject: [PATCH] Download `tap_migrations.json` files from the API Download the previously stored tap migrations files for homebrew/core and homebrew/cask from the formulae.brew.sh API. This adds a much longer stale time (24 hours) to decide whether or not the migrations files need downloaded from the API in Ruby land. `brew update` will still update them every time. Requires https://github.com/Homebrew/brew/pull/15628 Fixes https://github.com/Homebrew/brew/issues/14897 --- Library/Homebrew/api.rb | 9 ++++--- Library/Homebrew/api/cask.rb | 3 +-- Library/Homebrew/api/formula.rb | 3 +-- Library/Homebrew/cmd/update.sh | 42 +++++++++++++++++++-------------- Library/Homebrew/tap.rb | 14 +++++++++-- 5 files changed, 44 insertions(+), 27 deletions(-) diff --git a/Library/Homebrew/api.rb b/Library/Homebrew/api.rb index 3b13359f72..5508b0a2ba 100644 --- a/Library/Homebrew/api.rb +++ b/Library/Homebrew/api.rb @@ -34,8 +34,11 @@ module Homebrew raise ArgumentError, "Invalid JSON file: #{Tty.underline}#{api_url}#{Tty.reset}" end - sig { params(endpoint: String, target: Pathname).returns([T.any(Array, Hash), T::Boolean]) } - def self.fetch_json_api_file(endpoint, target:) + sig { + params(endpoint: String, target: Pathname, stale_seconds: Integer).returns([T.any(Array, Hash), T::Boolean]) + } + def self.fetch_json_api_file(endpoint, target: HOMEBREW_CACHE_API/endpoint, + stale_seconds: Homebrew::EnvConfig.api_auto_update_secs.to_i) retry_count = 0 url = "#{Homebrew::EnvConfig.api_domain}/#{endpoint}" default_url = "#{HOMEBREW_API_DEFAULT_DOMAIN}/#{endpoint}" @@ -59,7 +62,7 @@ module Homebrew !target.empty? && (!Homebrew.auto_update_command? || Homebrew::EnvConfig.no_auto_update? || - ((Time.now - Homebrew::EnvConfig.api_auto_update_secs.to_i) < target.mtime)) + ((Time.now - stale_seconds) < target.mtime)) skip_download ||= Homebrew.running_as_root_but_not_owned_by_root? json_data = begin diff --git a/Library/Homebrew/api/cask.rb b/Library/Homebrew/api/cask.rb index b65d05cb59..ba538d465a 100644 --- a/Library/Homebrew/api/cask.rb +++ b/Library/Homebrew/api/cask.rb @@ -43,8 +43,7 @@ module Homebrew sig { returns(T::Boolean) } def download_and_cache_data! - json_casks, updated = Homebrew::API.fetch_json_api_file "cask.jws.json", - target: HOMEBREW_CACHE_API/"cask.jws.json" + json_casks, updated = Homebrew::API.fetch_json_api_file "cask.jws.json" cache["renames"] = {} cache["casks"] = json_casks.to_h do |json_cask| diff --git a/Library/Homebrew/api/formula.rb b/Library/Homebrew/api/formula.rb index 8f5ce040df..a8d2215295 100644 --- a/Library/Homebrew/api/formula.rb +++ b/Library/Homebrew/api/formula.rb @@ -40,8 +40,7 @@ module Homebrew sig { returns(T::Boolean) } def download_and_cache_data! - json_formulae, updated = Homebrew::API.fetch_json_api_file "formula.jws.json", - target: HOMEBREW_CACHE_API/"formula.jws.json" + json_formulae, updated = Homebrew::API.fetch_json_api_file "formula.jws.json" cache["aliases"] = {} cache["renames"] = {} diff --git a/Library/Homebrew/cmd/update.sh b/Library/Homebrew/cmd/update.sh index 7108374449..9d2aa59f2a 100644 --- a/Library/Homebrew/cmd/update.sh +++ b/Library/Homebrew/cmd/update.sh @@ -781,34 +781,37 @@ EOS if [[ -z "${HOMEBREW_NO_INSTALL_FROM_API}" ]] then - mkdir -p "${HOMEBREW_CACHE}/api" + local api_cache="${HOMEBREW_CACHE}/api" + mkdir -p "${api_cache}" - for formula_or_cask in formula cask + for json in formula cask formula_tap_migrations cask_tap_migrations do - if [[ -f "${HOMEBREW_CACHE}/api/${formula_or_cask}.jws.json" ]] + local filename="${json}.jws.json" + local cache_path="${api_cache}/${filename}" + if [[ -f "${cache_path}" ]] then - INITIAL_JSON_BYTESIZE="$(wc -c "${HOMEBREW_CACHE}"/api/"${formula_or_cask}".jws.json)" + INITIAL_JSON_BYTESIZE="$(wc -c "${cache_path}")" fi JSON_URLS=() if [[ -n "${HOMEBREW_API_DOMAIN}" && "${HOMEBREW_API_DOMAIN}" != "${HOMEBREW_API_DEFAULT_DOMAIN}" ]] then - JSON_URLS=("${HOMEBREW_API_DOMAIN}/${formula_or_cask}.jws.json") + JSON_URLS=("${HOMEBREW_API_DOMAIN}/${filename}") fi - JSON_URLS+=("${HOMEBREW_API_DEFAULT_DOMAIN}/${formula_or_cask}.jws.json") + JSON_URLS+=("${HOMEBREW_API_DEFAULT_DOMAIN}/${filename}") for json_url in "${JSON_URLS[@]}" do time_cond=() - if [[ -s "${HOMEBREW_CACHE}/api/${formula_or_cask}.jws.json" ]] + if [[ -s "${cache_path}" ]] then - time_cond=("--time-cond" "${HOMEBREW_CACHE}/api/${formula_or_cask}.jws.json") + time_cond=("--time-cond" "${cache_path}") fi curl \ "${CURL_DISABLE_CURLRC_ARGS[@]}" \ --fail --compressed --silent \ --speed-limit "${HOMEBREW_CURL_SPEED_LIMIT}" --speed-time "${HOMEBREW_CURL_SPEED_TIME}" \ - --location --remote-time --output "${HOMEBREW_CACHE}/api/${formula_or_cask}.jws.json" \ + --location --remote-time --output "${cache_path}" \ "${time_cond[@]}" \ --user-agent "${HOMEBREW_USER_AGENT_CURL}" \ "${json_url}" @@ -816,23 +819,25 @@ EOS [[ ${curl_exit_code} -eq 0 ]] && break done - if [[ -f "${HOMEBREW_CACHE}/api/${formula_or_cask}_names.txt" ]] + if [[ "${json}" == "formula" ]] && [[ -f "${api_cache}/formula_names.txt" ]] then - mv -f "${HOMEBREW_CACHE}/api/${formula_or_cask}_names.txt" \ - "${HOMEBREW_CACHE}/api/${formula_or_cask}_names.before.txt" + mv -f "${api_cache}/formula_names.txt" "${api_cache}/formula_names.before.txt" + elif [[ "${json}" == "cask" ]] && [[ -f "${api_cache}/cask_names.txt" ]] + then + mv -f "${api_cache}/cask_names.txt" "${api_cache}/cask_names.before.txt" fi if [[ ${curl_exit_code} -eq 0 ]] then - touch "${HOMEBREW_CACHE}/api/${formula_or_cask}.jws.json" + touch "${cache_path}" - CURRENT_JSON_BYTESIZE="$(wc -c "${HOMEBREW_CACHE}"/api/"${formula_or_cask}".jws.json)" + CURRENT_JSON_BYTESIZE="$(wc -c "${cache_path}")" if [[ "${INITIAL_JSON_BYTESIZE}" != "${CURRENT_JSON_BYTESIZE}" ]] then - if [[ "${formula_or_cask}" == "formula" ]] + if [[ "${json}" == "formula" ]] then - rm -f "${HOMEBREW_CACHE}/api/formula_aliases.txt" + rm -f "${api_cache}/formula_aliases.txt" fi HOMEBREW_UPDATED="1" fi @@ -840,9 +845,10 @@ EOS echo "Failed to download ${json_url}!" >>"${update_failed_file}" fi - # Not a typo, this is the file we used to download that we should cleanup. - rm -f "${HOMEBREW_CACHE}/api/${formula_or_cask}.json" done + + # Not a typo, these are the files we used to download that no longer need so should cleanup. + rm -f "${HOMEBREW_CACHE}/api/formula.json" "${HOMEBREW_CACHE}/api/cask.json" fi if [[ -f "${update_failed_file}" ]] diff --git a/Library/Homebrew/tap.rb b/Library/Homebrew/tap.rb index d5839c5409..fb0f8284ae 100644 --- a/Library/Homebrew/tap.rb +++ b/Library/Homebrew/tap.rb @@ -25,6 +25,8 @@ class Tap HOMEBREW_TAP_STYLE_EXCEPTIONS_DIR = "style_exceptions" HOMEBREW_TAP_PYPI_FORMULA_MAPPINGS = "pypi_formula_mappings.json" + TAP_MIGRATIONS_STALE_SECONDS = 86400 # 1 day + HOMEBREW_TAP_JSON_FILES = %W[ #{HOMEBREW_TAP_FORMULA_RENAMES_FILE} #{HOMEBREW_TAP_CASK_RENAMES_FILE} @@ -714,7 +716,11 @@ class Tap # Hash with tap migrations. sig { returns(Hash) } def tap_migrations - @tap_migrations ||= if (migration_file = path/HOMEBREW_TAP_MIGRATIONS_FILE).file? + @tap_migrations ||= if name == "homebrew/cask" && !Homebrew::EnvConfig.no_install_from_api? + migrations, = Homebrew::API.fetch_json_api_file "cask_tap_migrations.jws.json", + stale_seconds: TAP_MIGRATIONS_STALE_SECONDS + migrations + elsif (migration_file = path/HOMEBREW_TAP_MIGRATIONS_FILE).file? JSON.parse(migration_file.read) else {} @@ -958,9 +964,13 @@ class CoreTap < Tap # @private sig { returns(Hash) } def tap_migrations - @tap_migrations ||= begin + @tap_migrations ||= if Homebrew::EnvConfig.no_install_from_api? self.class.ensure_installed! super + else + migrations, = Homebrew::API.fetch_json_api_file "formula_tap_migrations.jws.json", + stale_seconds: TAP_MIGRATIONS_STALE_SECONDS + migrations end end