From ba3bccf9edec05fad50beda6ada56eebdf880755 Mon Sep 17 00:00:00 2001 From: Xuehai Pan Date: Fri, 3 Feb 2023 14:10:40 +0000 Subject: [PATCH] api: download from `HOMEBREW_API_DOMAIN` --- Library/Homebrew/.rubocop.yml | 2 +- Library/Homebrew/api.rb | 26 ++++++++++++++----- Library/Homebrew/brew.sh | 2 ++ Library/Homebrew/cmd/update.sh | 26 +++++++++++++------ Library/Homebrew/env_config.rb | 7 +++++ Library/Homebrew/global.rb | 1 + .../sorbet/rbi/hidden-definitions/hidden.rbi | 3 +++ 7 files changed, 52 insertions(+), 15 deletions(-) diff --git a/Library/Homebrew/.rubocop.yml b/Library/Homebrew/.rubocop.yml index 3e5a81360f..ec475b77c8 100644 --- a/Library/Homebrew/.rubocop.yml +++ b/Library/Homebrew/.rubocop.yml @@ -41,7 +41,7 @@ Metrics/PerceivedComplexity: Metrics/MethodLength: Max: 232 Metrics/ModuleLength: - Max: 466 + Max: 473 Exclude: # TODO: extract more of the bottling logic - "dev-cmd/bottle.rb" diff --git a/Library/Homebrew/api.rb b/Library/Homebrew/api.rb index ec42b3f12c..2537e75c71 100644 --- a/Library/Homebrew/api.rb +++ b/Library/Homebrew/api.rb @@ -17,7 +17,6 @@ module Homebrew module_function - API_DOMAIN = "https://formulae.brew.sh/api" HOMEBREW_CACHE_API = (HOMEBREW_CACHE/"api").freeze # Set a longer timeout just for large(r) files. @@ -27,8 +26,13 @@ module Homebrew def fetch(endpoint) return cache[endpoint] if cache.present? && cache.key?(endpoint) - api_url = "#{API_DOMAIN}/#{endpoint}" + api_url = "#{Homebrew::EnvConfig.api_domain}/#{endpoint}" output = Utils::Curl.curl_output("--fail", api_url) + if !output.success? && Homebrew::EnvConfig.api_domain != HOMEBREW_API_DEFAULT_DOMAIN + # Fall back to the default API domain and try again + api_url = "#{HOMEBREW_API_DEFAULT_DOMAIN}/#{endpoint}" + output = Utils::Curl.curl_output("--fail", api_url) + end raise ArgumentError, "No file found at #{Tty.underline}#{api_url}#{Tty.reset}" unless output.success? cache[endpoint] = JSON.parse(output.stdout) @@ -39,8 +43,9 @@ module Homebrew sig { params(endpoint: String, target: Pathname).returns(Hash) } def fetch_json_api_file(endpoint, target:) retry_count = 0 - url = "#{API_DOMAIN}/#{endpoint}" - curl_args = %W[--compressed --silent #{url}] + url = "#{Homebrew::EnvConfig.api_domain}/#{endpoint}" + default_url = "#{HOMEBREW_API_DEFAULT_DOMAIN}/#{endpoint}" + curl_args = %w[--compressed --silent] curl_args.prepend("--time-cond", target) if target.exist? && !target.empty? begin @@ -48,8 +53,17 @@ module Homebrew # Disable retries here, we handle them ourselves below. Utils::Curl.curl_download(*curl_args, to: target, max_time: JSON_API_MAX_TIME, retries: 0) rescue ErrorDuringExecution - raise unless target.exist? - raise if target.empty? + if url == default_url + raise unless target.exist? + raise if target.empty? + elsif retry_count.zero? || !target.exist? || target.empty? + # Fall back to the default API domain and try again + # This block will be executed only once, because we set `url` to `default_url` + url = default_url + target.unlink if target.exist? && target.empty? + + retry + end opoo "#{target.basename}: update failed, falling back to cached version." end diff --git a/Library/Homebrew/brew.sh b/Library/Homebrew/brew.sh index c3a85cc7fd..05aa804b33 100644 --- a/Library/Homebrew/brew.sh +++ b/Library/Homebrew/brew.sh @@ -601,6 +601,7 @@ then unset HOMEBREW_BOTTLE_DOMAIN fi +HOMEBREW_API_DEFAULT_DOMAIN="https://formulae.brew.sh/api" HOMEBREW_BOTTLE_DEFAULT_DOMAIN="https://ghcr.io/v2/homebrew/core" HOMEBREW_USER_AGENT="${HOMEBREW_PRODUCT}/${HOMEBREW_USER_AGENT_VERSION} (${HOMEBREW_SYSTEM}; ${HOMEBREW_PROCESSOR} ${HOMEBREW_OS_USER_AGENT_VERSION})" @@ -633,6 +634,7 @@ export HOMEBREW_MACOS_VERSION export HOMEBREW_MACOS_VERSION_NUMERIC export HOMEBREW_USER_AGENT export HOMEBREW_USER_AGENT_CURL +export HOMEBREW_API_DEFAULT_DOMAIN export HOMEBREW_BOTTLE_DEFAULT_DOMAIN export HOMEBREW_MACOS_SYSTEM_RUBY_NEW_ENOUGH diff --git a/Library/Homebrew/cmd/update.sh b/Library/Homebrew/cmd/update.sh index 63a5a97953..dfd6d3f49e 100644 --- a/Library/Homebrew/cmd/update.sh +++ b/Library/Homebrew/cmd/update.sh @@ -778,14 +778,24 @@ EOS then INITIAL_JSON_BYTESIZE="$(wc -c "${HOMEBREW_CACHE}"/api/"${formula_or_cask}".json)" fi - curl \ - "${CURL_DISABLE_CURLRC_ARGS[@]}" \ - --fail --compressed --silent --max-time 10 \ - --location --remote-time --output "${HOMEBREW_CACHE}/api/${formula_or_cask}.json" \ - --time-cond "${HOMEBREW_CACHE}/api/${formula_or_cask}.json" \ - --user-agent "${HOMEBREW_USER_AGENT_CURL}" \ - "https://formulae.brew.sh/api/${formula_or_cask}.json" - curl_exit_code=$? + JSON_URLS=() + if [[ -n "${HOMEBREW_API_DOMAIN}" && "${HOMEBREW_API_DOMAIN}" != "${HOMEBREW_API_DEFAULT_DOMAIN}" ]] + then + JSON_URLS=("${HOMEBREW_API_DOMAIN}/${formula_or_cask}.json") + fi + JSON_URLS+=("${HOMEBREW_API_DEFAULT_DOMAIN}/${formula_or_cask}.json") + for json_url in "${JSON_URLS[@]}" + do + curl \ + "${CURL_DISABLE_CURLRC_ARGS[@]}" \ + --fail --compressed --silent --max-time 10 \ + --location --remote-time --output "${HOMEBREW_CACHE}/api/${formula_or_cask}.json" \ + --time-cond "${HOMEBREW_CACHE}/api/${formula_or_cask}.json" \ + --user-agent "${HOMEBREW_USER_AGENT_CURL}" \ + "${json_url}" + curl_exit_code=$? + [[ ${curl_exit_code} -eq 0 ]] && break + done if [[ ${curl_exit_code} -eq 0 ]] then CURRENT_JSON_BYTESIZE="$(wc -c "${HOMEBREW_CACHE}"/api/"${formula_or_cask}".json)" diff --git a/Library/Homebrew/env_config.rb b/Library/Homebrew/env_config.rb index 0324f6facf..e37aeed288 100644 --- a/Library/Homebrew/env_config.rb +++ b/Library/Homebrew/env_config.rb @@ -15,6 +15,13 @@ module Homebrew description: "Additional Google Analytics tracking ID to emit user behaviour analytics to. " \ "For more information, see: ", }, + HOMEBREW_API_DOMAIN: { + description: "Use this URL as the download mirror for Homebrew JSON API. " \ + "If metadata files at that URL are temporarily unavailable, " \ + "the default API domain will be used as a fallback mirror.", + default_text: "`https://formulae.brew.sh/api`.", + default: HOMEBREW_API_DEFAULT_DOMAIN, + }, HOMEBREW_ARCH: { description: "Linux only: Pass this value to a type name representing the compiler's `-march` option.", default: "native", diff --git a/Library/Homebrew/global.rb b/Library/Homebrew/global.rb index 6291f4b17b..50da2acf26 100644 --- a/Library/Homebrew/global.rb +++ b/Library/Homebrew/global.rb @@ -33,6 +33,7 @@ ActiveSupport::Inflector.inflections(:en) do |inflect| inflect.irregular "it", "they" end +HOMEBREW_API_DEFAULT_DOMAIN = ENV.fetch("HOMEBREW_API_DEFAULT_DOMAIN").freeze HOMEBREW_BOTTLE_DEFAULT_DOMAIN = ENV.fetch("HOMEBREW_BOTTLE_DEFAULT_DOMAIN").freeze HOMEBREW_BREW_DEFAULT_GIT_REMOTE = ENV.fetch("HOMEBREW_BREW_DEFAULT_GIT_REMOTE").freeze HOMEBREW_CORE_DEFAULT_GIT_REMOTE = ENV.fetch("HOMEBREW_CORE_DEFAULT_GIT_REMOTE").freeze diff --git a/Library/Homebrew/sorbet/rbi/hidden-definitions/hidden.rbi b/Library/Homebrew/sorbet/rbi/hidden-definitions/hidden.rbi index 4ed023810a..1ab4e60cf9 100644 --- a/Library/Homebrew/sorbet/rbi/hidden-definitions/hidden.rbi +++ b/Library/Homebrew/sorbet/rbi/hidden-definitions/hidden.rbi @@ -4404,6 +4404,8 @@ module Homebrew::EnvConfig def self.all_proxy(); end + def self.api_domain(); end + def self.arch(); end def self.artifact_domain(); end @@ -5458,6 +5460,7 @@ class Object FORMULA_COMPONENT_PRECEDENCE_LIST = ::T.let(nil, ::T.untyped) GZIP_BUFFER_SIZE = ::T.let(nil, ::T.untyped) HIDDEN_DESC_PLACEHOLDER = ::T.let(nil, ::T.untyped) + HOMEBREW_API_DEFAULT_DOMAIN = ::T.let(nil, ::T.untyped) HOMEBREW_BOTTLES_EXTNAME_REGEX = ::T.let(nil, ::T.untyped) HOMEBREW_BOTTLE_DEFAULT_DOMAIN = ::T.let(nil, ::T.untyped) HOMEBREW_BREWED_CURL_PATH = ::T.let(nil, ::T.untyped)