From 737dd1654be77d4b80b5db4e717c7237fd95c92b Mon Sep 17 00:00:00 2001 From: Rylan Polster Date: Fri, 6 Aug 2021 02:30:44 -0400 Subject: [PATCH 1/7] Refactor API methods --- Library/Homebrew/api.rb | 39 ++++++ Library/Homebrew/api/analytics.rb | 28 +++++ Library/Homebrew/api/bottle.rb | 95 ++++++++++++++ Library/Homebrew/api/cask.rb | 20 +++ Library/Homebrew/api/formula.rb | 28 +++++ Library/Homebrew/api/versions.rb | 52 ++++++++ Library/Homebrew/bottle_api.rb | 118 ------------------ Library/Homebrew/bottle_api.rbi | 5 - Library/Homebrew/cli/named_args.rb | 7 +- Library/Homebrew/cmd/info.rb | 5 +- Library/Homebrew/cmd/outdated.rb | 4 +- Library/Homebrew/cmd/reinstall.rb | 6 +- Library/Homebrew/cmd/upgrade.rb | 6 +- Library/Homebrew/dev-cmd/unbottled.rb | 3 +- Library/Homebrew/extend/os/api/analytics.rb | 4 + Library/Homebrew/extend/os/api/bottle.rb | 4 + Library/Homebrew/extend/os/api/formula.rb | 4 + .../Homebrew/extend/os/linux/api/analytics.rb | 17 +++ .../Homebrew/extend/os/linux/api/bottle.rb | 17 +++ .../Homebrew/extend/os/linux/api/formula.rb | 17 +++ .../extend/os/linux/utils/analytics.rb | 23 ---- Library/Homebrew/extend/os/utils/analytics.rb | 1 - Library/Homebrew/formula.rb | 7 +- .../bottle_spec.rb} | 46 ++----- Library/Homebrew/test/api/versions_spec.rb | 55 ++++++++ Library/Homebrew/test/api_spec.rb | 39 ++++++ Library/Homebrew/utils/analytics.rb | 51 +++----- 27 files changed, 469 insertions(+), 232 deletions(-) create mode 100644 Library/Homebrew/api.rb create mode 100644 Library/Homebrew/api/analytics.rb create mode 100644 Library/Homebrew/api/bottle.rb create mode 100644 Library/Homebrew/api/cask.rb create mode 100644 Library/Homebrew/api/formula.rb create mode 100644 Library/Homebrew/api/versions.rb delete mode 100644 Library/Homebrew/bottle_api.rb delete mode 100644 Library/Homebrew/bottle_api.rbi create mode 100644 Library/Homebrew/extend/os/api/analytics.rb create mode 100644 Library/Homebrew/extend/os/api/bottle.rb create mode 100644 Library/Homebrew/extend/os/api/formula.rb create mode 100644 Library/Homebrew/extend/os/linux/api/analytics.rb create mode 100644 Library/Homebrew/extend/os/linux/api/bottle.rb create mode 100644 Library/Homebrew/extend/os/linux/api/formula.rb delete mode 100644 Library/Homebrew/extend/os/linux/utils/analytics.rb rename Library/Homebrew/test/{bottle_api_spec.rb => api/bottle_spec.rb} (71%) create mode 100644 Library/Homebrew/test/api/versions_spec.rb create mode 100644 Library/Homebrew/test/api_spec.rb diff --git a/Library/Homebrew/api.rb b/Library/Homebrew/api.rb new file mode 100644 index 0000000000..ac21298f64 --- /dev/null +++ b/Library/Homebrew/api.rb @@ -0,0 +1,39 @@ +# typed: false +# frozen_string_literal: true + +require "api/analytics" +require "api/bottle" +require "api/cask" +require "api/formula" +require "api/versions" + +module Homebrew + # Helper functions for using Homebrew's formulae.brew.sh API. + # + # @api private + module API + extend T::Sig + + module_function + + API_DOMAIN = "https://formulae.brew.sh/api" + + sig { params(endpoint: String, json: T::Boolean).returns(T.any(String, Hash)) } + def fetch(endpoint, json: false) + return @cache[endpoint] if @cache.present? && @cache.key?(endpoint) + + api_url = "#{API_DOMAIN}/#{endpoint}" + output = Utils::Curl.curl_output("--fail", "--max-time", "5", api_url) + raise ArgumentError, "No file found at #{Tty.underline}#{api_url}#{Tty.reset}" unless output.success? + + @cache ||= {} + @cache[endpoint] = if json + JSON.parse(output.stdout) + else + output.stdout + end + rescue JSON::ParserError + raise ArgumentError, "Invalid JSON file: #{Tty.underline}#{api_url}#{Tty.reset}" + end + end +end diff --git a/Library/Homebrew/api/analytics.rb b/Library/Homebrew/api/analytics.rb new file mode 100644 index 0000000000..f7967d4d1d --- /dev/null +++ b/Library/Homebrew/api/analytics.rb @@ -0,0 +1,28 @@ +# typed: false +# frozen_string_literal: true + +module Homebrew + module API + # Helper functions for using the analytics JSON API. + # + # @api private + module Analytics + extend T::Sig + + module_function + + sig { returns(String) } + def analytics_api_path + "analytics" + end + alias generic_analytics_api_path analytics_api_path + + sig { params(category: String, days: T.any(Integer, String)).returns(Hash) } + def fetch(category, days) + Homebrew::API.fetch "#{analytics_api_path}/#{category}/#{days}d.json", json: true + end + end + end +end + +require "extend/os/api/analytics" diff --git a/Library/Homebrew/api/bottle.rb b/Library/Homebrew/api/bottle.rb new file mode 100644 index 0000000000..ad832f2254 --- /dev/null +++ b/Library/Homebrew/api/bottle.rb @@ -0,0 +1,95 @@ +# typed: false +# frozen_string_literal: true + +require "github_packages" + +module Homebrew + module API + # Helper functions for using the Bottle JSON API. + # + # @api private + module Bottle + extend T::Sig + + module_function + + sig { returns(String) } + def bottle_api_path + "bottle" + end + alias generic_bottle_api_path bottle_api_path + + GITHUB_PACKAGES_SHA256_REGEX = %r{#{GitHubPackages::URL_REGEX}.*/blobs/sha256:(?\h{64})$}.freeze + + sig { params(name: String).returns(Hash) } + def fetch(name) + Homebrew::API.fetch "#{bottle_api_path}/#{name}.json", json: true + end + + sig { params(name: String).returns(T::Boolean) } + def available?(name) + fetch name + true + rescue ArgumentError + false + end + + sig { params(name: String).void } + def fetch_bottles(name) + hash = fetch(name) + bottle_tag = Utils::Bottles.tag.to_s + + if !hash["bottles"].key?(bottle_tag) && !hash["bottles"].key?("all") + odie "No bottle available for #{name} on the current OS" + end + + download_bottle(hash, bottle_tag) + + hash["dependencies"].each do |dep_hash| + existing_formula = begin + Formulary.factory dep_hash["name"] + rescue FormulaUnavailableError + # The formula might not exist if it's not installed and homebrew/core isn't tapped + nil + end + + next if existing_formula.present? && existing_formula.latest_version_installed? + + download_bottle(dep_hash, bottle_tag) + end + end + + sig { params(url: String).returns(T.nilable(String)) } + def checksum_from_url(url) + match = url.match GITHUB_PACKAGES_SHA256_REGEX + return if match.blank? + + match[:sha256] + end + + sig { params(hash: Hash, tag: Symbol).void } + def download_bottle(hash, tag) + bottle = hash["bottles"][tag] + bottle ||= hash["bottles"]["all"] + return if bottle.blank? + + sha256 = bottle["sha256"] || checksum_from_url(bottle["url"]) + bottle_filename = ::Bottle::Filename.new(hash["name"], hash["pkg_version"], tag, hash["rebuild"]) + + resource = Resource.new hash["name"] + resource.url bottle["url"] + resource.sha256 sha256 + resource.version hash["pkg_version"] + resource.downloader.resolved_basename = bottle_filename + + resource.fetch + + # Map the name of this formula to the local bottle path to allow the + # formula to be loaded by passing just the name to `Formulary::factory`. + Formulary.map_formula_name_to_local_bottle_path hash["name"], resource.downloader.cached_location + end + end + end +end + +require "extend/os/api/bottle" diff --git a/Library/Homebrew/api/cask.rb b/Library/Homebrew/api/cask.rb new file mode 100644 index 0000000000..c1dd4ddcd6 --- /dev/null +++ b/Library/Homebrew/api/cask.rb @@ -0,0 +1,20 @@ +# typed: false +# frozen_string_literal: true + +module Homebrew + module API + # Helper functions for using the cask JSON API. + # + # @api private + module Cask + extend T::Sig + + module_function + + sig { params(name: String).returns(Hash) } + def fetch(name) + Homebrew::API.fetch "cask/#{name}.json", json: true + end + end + end +end diff --git a/Library/Homebrew/api/formula.rb b/Library/Homebrew/api/formula.rb new file mode 100644 index 0000000000..329f80bfde --- /dev/null +++ b/Library/Homebrew/api/formula.rb @@ -0,0 +1,28 @@ +# typed: false +# frozen_string_literal: true + +module Homebrew + module API + # Helper functions for using the formula JSON API. + # + # @api private + module Formula + extend T::Sig + + module_function + + sig { returns(String) } + def formula_api_path + "formula" + end + alias generic_formula_api_path formula_api_path + + sig { params(name: String).returns(Hash) } + def fetch(name) + Homebrew::API.fetch "#{formula_api_path}/#{name}.json", json: true + end + end + end +end + +require "extend/os/api/formula" diff --git a/Library/Homebrew/api/versions.rb b/Library/Homebrew/api/versions.rb new file mode 100644 index 0000000000..0ad01b93c4 --- /dev/null +++ b/Library/Homebrew/api/versions.rb @@ -0,0 +1,52 @@ +# typed: false +# frozen_string_literal: true + +module Homebrew + module API + # Helper functions for using the versions JSON API. + # + # @api private + module Versions + extend T::Sig + + module_function + + def formulae + # The result is cached by Homebrew::API.fetch + Homebrew::API.fetch "versions-formulae.json", json: true + end + + def linux + # The result is cached by Homebrew::API.fetch + Homebrew::API.fetch "versions-linux.json", json: true + end + + def casks + # The result is cached by Homebrew::API.fetch + Homebrew::API.fetch "versions-casks.json", json: true + end + + sig { params(name: String).returns(T.nilable(PkgVersion)) } + def latest_formula_version(name) + versions = if OS.mac? || Homebrew::EnvConfig.force_homebrew_on_linux? + formulae + else + linux + end + + return unless versions.key? name + + version = Version.new(versions[name]["version"]) + revision = versions[name]["revision"] + PkgVersion.new(version, revision) + end + + sig { params(token: String).returns(T.nilable(Version)) } + def latest_cask_version(token) + return unless casks.key? token + + Version.new(casks[token]["version"]) + end + end + end +end diff --git a/Library/Homebrew/bottle_api.rb b/Library/Homebrew/bottle_api.rb deleted file mode 100644 index 8ee7f6c95f..0000000000 --- a/Library/Homebrew/bottle_api.rb +++ /dev/null @@ -1,118 +0,0 @@ -# typed: true -# frozen_string_literal: true - -require "github_packages" - -# Helper functions for using the Bottle JSON API. -# -# @api private -module BottleAPI - extend T::Sig - - module_function - - FORMULAE_BREW_SH_BOTTLE_API_DOMAIN = if OS.mac? - "https://formulae.brew.sh/api/bottle" - else - "https://formulae.brew.sh/api/bottle-linux" - end.freeze - - FORMULAE_BREW_SH_VERSIONS_API_URL = if OS.mac? - "https://formulae.brew.sh/api/versions-formulae.json" - else - "https://formulae.brew.sh/api/versions-linux.json" - end.freeze - - GITHUB_PACKAGES_SHA256_REGEX = %r{#{GitHubPackages::URL_REGEX}.*/blobs/sha256:(?\h{64})$}.freeze - - sig { params(name: String).returns(Hash) } - def fetch(name) - return @cache[name] if @cache.present? && @cache.key?(name) - - api_url = "#{FORMULAE_BREW_SH_BOTTLE_API_DOMAIN}/#{name}.json" - output = Utils::Curl.curl_output("--fail", api_url) - raise ArgumentError, "No JSON file found at #{Tty.underline}#{api_url}#{Tty.reset}" unless output.success? - - @cache ||= {} - @cache[name] = JSON.parse(output.stdout) - rescue JSON::ParserError - raise ArgumentError, "Invalid JSON file: #{Tty.underline}#{api_url}#{Tty.reset}" - end - - sig { params(name: String).returns(T.nilable(PkgVersion)) } - def latest_pkg_version(name) - @formula_versions ||= begin - output = Utils::Curl.curl_output("--fail", FORMULAE_BREW_SH_VERSIONS_API_URL) - JSON.parse(output.stdout) - end - - return unless @formula_versions.key? name - - version = Version.new(@formula_versions[name]["version"]) - revision = @formula_versions[name]["revision"] - PkgVersion.new(version, revision) - end - - sig { params(name: String).returns(T::Boolean) } - def bottle_available?(name) - fetch name - true - rescue ArgumentError - false - end - - sig { params(name: String).void } - def fetch_bottles(name) - hash = fetch(name) - bottle_tag = Utils::Bottles.tag.to_s - - if !hash["bottles"].key?(bottle_tag) && !hash["bottles"].key?("all") - odie "No bottle available for #{name} on the current OS" - end - - download_bottle(hash, bottle_tag) - - hash["dependencies"].each do |dep_hash| - existing_formula = begin - Formulary.factory dep_hash["name"] - rescue FormulaUnavailableError - # The formula might not exist if it's not installed and homebrew/core isn't tapped - nil - end - - next if existing_formula.present? && existing_formula.latest_version_installed? - - download_bottle(dep_hash, bottle_tag) - end - end - - sig { params(url: String).returns(T.nilable(String)) } - def checksum_from_url(url) - match = url.match GITHUB_PACKAGES_SHA256_REGEX - return if match.blank? - - match[:sha256] - end - - sig { params(hash: Hash, tag: Symbol).void } - def download_bottle(hash, tag) - bottle = hash["bottles"][tag] - bottle ||= hash["bottles"]["all"] - return if bottle.blank? - - sha256 = bottle["sha256"] || checksum_from_url(bottle["url"]) - bottle_filename = Bottle::Filename.new(hash["name"], hash["pkg_version"], tag, hash["rebuild"]) - - resource = Resource.new hash["name"] - resource.url bottle["url"] - resource.sha256 sha256 - resource.version hash["pkg_version"] - resource.downloader.resolved_basename = bottle_filename - - resource.fetch - - # Map the name of this formula to the local bottle path to allow the - # formula to be loaded by passing just the name to `Formulary::factory`. - Formulary.map_formula_name_to_local_bottle_path hash["name"], resource.downloader.cached_location - end -end diff --git a/Library/Homebrew/bottle_api.rbi b/Library/Homebrew/bottle_api.rbi deleted file mode 100644 index 6fefceb732..0000000000 --- a/Library/Homebrew/bottle_api.rbi +++ /dev/null @@ -1,5 +0,0 @@ -# typed: strict - -module BottleAPI - include Kernel -end diff --git a/Library/Homebrew/cli/named_args.rb b/Library/Homebrew/cli/named_args.rb index cd49751f48..a1413cf002 100644 --- a/Library/Homebrew/cli/named_args.rb +++ b/Library/Homebrew/cli/named_args.rb @@ -2,7 +2,7 @@ # frozen_string_literal: true require "delegate" -require "bottle_api" +require "api" require "cli/args" module Homebrew @@ -94,8 +94,9 @@ module Homebrew unreadable_error = nil if only != :cask - if prefer_loading_from_json && ENV["HOMEBREW_JSON_CORE"].present? && BottleAPI.bottle_available?(name) - BottleAPI.fetch_bottles(name) + if prefer_loading_from_json && ENV["HOMEBREW_JSON_CORE"].present? && + Homebrew::API::Bottle.available?(name) + Homebrew::API::Bottle.fetch_bottles(name) end begin diff --git a/Library/Homebrew/cmd/info.rb b/Library/Homebrew/cmd/info.rb index 06ad93735e..fec292ee77 100644 --- a/Library/Homebrew/cmd/info.rb +++ b/Library/Homebrew/cmd/info.rb @@ -11,6 +11,7 @@ require "tab" require "json" require "utils/spdx" require "deprecate_disable" +require "api" module Homebrew extend T::Sig @@ -243,8 +244,8 @@ module Homebrew def info_formula(f, args:) specs = [] - if ENV["HOMEBREW_JSON_CORE"].present? && BottleAPI.bottle_available?(f.name) - info = BottleAPI.fetch(f.name) + if ENV["HOMEBREW_JSON_CORE"].present? && Homebrew::API::Bottle.available?(f.name) + info = Homebrew::API::Bottle.fetch(f.name) latest_version = info["pkg_version"].split("_").first bottle_exists = info["bottles"].key?(Utils::Bottles.tag.to_s) || info["bottles"].key?("all") diff --git a/Library/Homebrew/cmd/outdated.rb b/Library/Homebrew/cmd/outdated.rb index 26c455972a..e6999fd641 100644 --- a/Library/Homebrew/cmd/outdated.rb +++ b/Library/Homebrew/cmd/outdated.rb @@ -6,7 +6,7 @@ require "keg" require "cli/parser" require "cask/cmd" require "cask/caskroom" -require "bottle_api" +require "api" module Homebrew extend T::Sig @@ -99,7 +99,7 @@ module Homebrew outdated_kegs = f.outdated_kegs(fetch_head: args.fetch_HEAD?) current_version = if ENV["HOMEBREW_JSON_CORE"].present? && (f.core_formula? || f.tap.blank?) - BottleAPI.latest_pkg_version(f.name)&.to_s || f.pkg_version.to_s + Homebrew::API::Versions.latest_formula_version(f.name)&.to_s || f.pkg_version.to_s elsif f.alias_changed? && !f.latest_formula.latest_version_installed? latest = f.latest_formula "#{latest.name} (#{latest.pkg_version})" diff --git a/Library/Homebrew/cmd/reinstall.rb b/Library/Homebrew/cmd/reinstall.rb index f1dfac9caf..c95693917e 100644 --- a/Library/Homebrew/cmd/reinstall.rb +++ b/Library/Homebrew/cmd/reinstall.rb @@ -12,7 +12,7 @@ require "cask/cmd" require "cask/utils" require "cask/macos" require "upgrade" -require "bottle_api" +require "api" module Homebrew extend T::Sig @@ -90,9 +90,9 @@ module Homebrew formula = Formulary.factory(name) next unless formula.any_version_installed? next if formula.tap.present? && !formula.core_formula? - next unless BottleAPI.bottle_available?(name) + next unless Homebrew::API::Bottle.available?(name) - BottleAPI.fetch_bottles(name) + Homebrew::API::Bottle.fetch_bottles(name) rescue FormulaUnavailableError next end diff --git a/Library/Homebrew/cmd/upgrade.rb b/Library/Homebrew/cmd/upgrade.rb index 9c05976a51..cbc10fb6b1 100644 --- a/Library/Homebrew/cmd/upgrade.rb +++ b/Library/Homebrew/cmd/upgrade.rb @@ -8,7 +8,7 @@ require "upgrade" require "cask/cmd" require "cask/utils" require "cask/macos" -require "bottle_api" +require "api" module Homebrew extend T::Sig @@ -163,9 +163,9 @@ module Homebrew if ENV["HOMEBREW_JSON_CORE"].present? formulae_to_install.map! do |formula| next formula if formula.tap.present? && !formula.core_formula? - next formula unless BottleAPI.bottle_available?(formula.name) + next formula unless Homebrew::API::Bottle.available?(formula.name) - BottleAPI.fetch_bottles(formula.name) + Homebrew::API::Bottle.fetch_bottles(formula.name) Formulary.factory(formula.name) rescue FormulaUnavailableError formula diff --git a/Library/Homebrew/dev-cmd/unbottled.rb b/Library/Homebrew/dev-cmd/unbottled.rb index c29a027f71..94e0c5b5e3 100644 --- a/Library/Homebrew/dev-cmd/unbottled.rb +++ b/Library/Homebrew/dev-cmd/unbottled.rb @@ -3,6 +3,7 @@ require "cli/parser" require "formula" +require "api" module Homebrew extend T::Sig @@ -87,7 +88,7 @@ module Homebrew formula_installs = {} ohai "Getting analytics data..." - analytics = Utils::Analytics.formulae_brew_sh_json("analytics/install/90d.json") + analytics = Homebrew::API::Analytics.fetch "install", 90 if analytics.blank? raise UsageError, diff --git a/Library/Homebrew/extend/os/api/analytics.rb b/Library/Homebrew/extend/os/api/analytics.rb new file mode 100644 index 0000000000..970ac1c2cc --- /dev/null +++ b/Library/Homebrew/extend/os/api/analytics.rb @@ -0,0 +1,4 @@ +# typed: strict +# frozen_string_literal: true + +require "extend/os/linux/api/analytics" if OS.linux? diff --git a/Library/Homebrew/extend/os/api/bottle.rb b/Library/Homebrew/extend/os/api/bottle.rb new file mode 100644 index 0000000000..13e6d9c42b --- /dev/null +++ b/Library/Homebrew/extend/os/api/bottle.rb @@ -0,0 +1,4 @@ +# typed: strict +# frozen_string_literal: true + +require "extend/os/linux/api/bottle" if OS.linux? diff --git a/Library/Homebrew/extend/os/api/formula.rb b/Library/Homebrew/extend/os/api/formula.rb new file mode 100644 index 0000000000..6f5536d1d9 --- /dev/null +++ b/Library/Homebrew/extend/os/api/formula.rb @@ -0,0 +1,4 @@ +# typed: strict +# frozen_string_literal: true + +require "extend/os/linux/api/formula" if OS.linux? diff --git a/Library/Homebrew/extend/os/linux/api/analytics.rb b/Library/Homebrew/extend/os/linux/api/analytics.rb new file mode 100644 index 0000000000..5d486078e5 --- /dev/null +++ b/Library/Homebrew/extend/os/linux/api/analytics.rb @@ -0,0 +1,17 @@ +# typed: false +# frozen_string_literal: true + +module Homebrew + module API + module Analytics + class << self + sig { returns(String) } + def analytics_api_path + return generic_analytics_api_path if Homebrew::EnvConfig.force_homebrew_on_linux? + + "analytics-linux" + end + end + end + end +end diff --git a/Library/Homebrew/extend/os/linux/api/bottle.rb b/Library/Homebrew/extend/os/linux/api/bottle.rb new file mode 100644 index 0000000000..6ac1ea34b1 --- /dev/null +++ b/Library/Homebrew/extend/os/linux/api/bottle.rb @@ -0,0 +1,17 @@ +# typed: false +# frozen_string_literal: true + +module Homebrew + module API + module Bottle + class << self + sig { returns(String) } + def bottle_api_path + return generic_bottle_api_path if Homebrew::EnvConfig.force_homebrew_on_linux? + + "bottle-linux" + end + end + end + end +end diff --git a/Library/Homebrew/extend/os/linux/api/formula.rb b/Library/Homebrew/extend/os/linux/api/formula.rb new file mode 100644 index 0000000000..1ca27b5e66 --- /dev/null +++ b/Library/Homebrew/extend/os/linux/api/formula.rb @@ -0,0 +1,17 @@ +# typed: false +# frozen_string_literal: true + +module Homebrew + module API + module Formula + class << self + sig { returns(String) } + def formula_api_path + return generic_formula_api_path if Homebrew::EnvConfig.force_homebrew_on_linux? + + "formula-linux" + end + end + end + end +end diff --git a/Library/Homebrew/extend/os/linux/utils/analytics.rb b/Library/Homebrew/extend/os/linux/utils/analytics.rb deleted file mode 100644 index c183cbf614..0000000000 --- a/Library/Homebrew/extend/os/linux/utils/analytics.rb +++ /dev/null @@ -1,23 +0,0 @@ -# typed: strict -# frozen_string_literal: true - -module Utils - module Analytics - class << self - extend T::Sig - sig { returns(String) } - def formula_path - return generic_formula_path if Homebrew::EnvConfig.force_homebrew_on_linux? - - "formula-linux" - end - - sig { returns(String) } - def analytics_path - return generic_analytics_path if Homebrew::EnvConfig.force_homebrew_on_linux? - - "analytics-linux" - end - end - end -end diff --git a/Library/Homebrew/extend/os/utils/analytics.rb b/Library/Homebrew/extend/os/utils/analytics.rb index f0042d78c2..3cdaffacb6 100644 --- a/Library/Homebrew/extend/os/utils/analytics.rb +++ b/Library/Homebrew/extend/os/utils/analytics.rb @@ -1,5 +1,4 @@ # typed: strict # frozen_string_literal: true -require "extend/os/linux/utils/analytics" if OS.linux? require "extend/os/mac/utils/analytics" if OS.mac? diff --git a/Library/Homebrew/formula.rb b/Library/Homebrew/formula.rb index c656b39e46..7adf1236ac 100644 --- a/Library/Homebrew/formula.rb +++ b/Library/Homebrew/formula.rb @@ -29,7 +29,7 @@ require "mktemp" require "find" require "utils/spdx" require "extend/on_os" -require "bottle_api" +require "api" # A formula provides instructions and metadata for Homebrew to install a piece # of software. Every Homebrew formula is a {Formula}. @@ -520,7 +520,8 @@ class Formula # exists and is not empty. # @private def latest_version_installed? - latest_prefix = if ENV["HOMEBREW_JSON_CORE"].present? && (latest_pkg_version = BottleAPI.latest_pkg_version(name)) + latest_prefix = if ENV["HOMEBREW_JSON_CORE"].present? && + (latest_pkg_version = Homebrew::API::Versions.latest_formula_version(name)) prefix latest_pkg_version else latest_installed_prefix @@ -1340,7 +1341,7 @@ class Formula all_kegs = [] current_version = T.let(false, T::Boolean) latest_version = if ENV["HOMEBREW_JSON_CORE"].present? && (core_formula? || tap.blank?) - BottleAPI.latest_pkg_version(name) || pkg_version + Homebrew::API::Versions.latest_formula_version(name) || pkg_version else pkg_version end diff --git a/Library/Homebrew/test/bottle_api_spec.rb b/Library/Homebrew/test/api/bottle_spec.rb similarity index 71% rename from Library/Homebrew/test/bottle_api_spec.rb rename to Library/Homebrew/test/api/bottle_spec.rb index 556e24dcc5..dee8e05649 100644 --- a/Library/Homebrew/test/bottle_api_spec.rb +++ b/Library/Homebrew/test/api/bottle_spec.rb @@ -1,10 +1,12 @@ # typed: false # frozen_string_literal: true -describe BottleAPI do - before do - ENV["HOMEBREW_JSON_CORE"] = "1" - end +require "api" + +describe Homebrew::API::Bottle do + # before do + # ENV["HOMEBREW_JSON_CORE"] = "1" + # end let(:bottle_json) { <<~EOS @@ -28,14 +30,6 @@ describe BottleAPI do EOS } let(:bottle_hash) { JSON.parse(bottle_json) } - let(:versions_json) { - <<~EOS - { - "foo":{"version":"1.2.3","revision":0}, - "bar":{"version":"1.2","revision":4} - } - EOS - } def mock_curl_output(stdout: "", success: true) curl_output = OpenStruct.new(stdout: stdout, success?: success) @@ -51,7 +45,7 @@ describe BottleAPI do it "raises an error if the formula does not exist" do mock_curl_output success: false - expect { described_class.fetch("bar") }.to raise_error(ArgumentError, /No JSON file found/) + expect { described_class.fetch("bar") }.to raise_error(ArgumentError, /No file found/) end it "raises an error if the bottle JSON is invalid" do @@ -60,35 +54,15 @@ describe BottleAPI do end end - describe "::latest_pkg_version" do - it "returns the expected `PkgVersion` when the revision is 0" do - mock_curl_output stdout: versions_json - pkg_version = described_class.latest_pkg_version("foo") - expect(pkg_version.to_s).to eq "1.2.3" - end - - it "returns the expected `PkgVersion` when the revision is not 0" do - mock_curl_output stdout: versions_json - pkg_version = described_class.latest_pkg_version("bar") - expect(pkg_version.to_s).to eq "1.2_4" - end - - it "returns `nil` when the formula is not in the JSON file" do - mock_curl_output stdout: versions_json - pkg_version = described_class.latest_pkg_version("baz") - expect(pkg_version).to be_nil - end - end - - describe "::bottle_available?" do + describe "::available?" do it "returns `true` if `fetch` succeeds" do allow(described_class).to receive(:fetch) - expect(described_class.bottle_available?("foo")).to eq true + expect(described_class.available?("foo")).to eq true end it "returns `false` if `fetch` fails" do allow(described_class).to receive(:fetch).and_raise ArgumentError - expect(described_class.bottle_available?("foo")).to eq false + expect(described_class.available?("foo")).to eq false end end diff --git a/Library/Homebrew/test/api/versions_spec.rb b/Library/Homebrew/test/api/versions_spec.rb new file mode 100644 index 0000000000..28b66d65f6 --- /dev/null +++ b/Library/Homebrew/test/api/versions_spec.rb @@ -0,0 +1,55 @@ +# typed: false +# frozen_string_literal: true + +require "api" + +describe Homebrew::API::Versions do + let(:versions_formulae_json) { + <<~EOS + { + "foo":{"version":"1.2.3","revision":0}, + "bar":{"version":"1.2","revision":4} + } + EOS + } + let(:versions_casks_json) { '{"foo":{"version":"1.2.3"}}' } + + def mock_curl_output(stdout: "", success: true) + curl_output = OpenStruct.new(stdout: stdout, success?: success) + allow(Utils::Curl).to receive(:curl_output).and_return curl_output + end + + describe "::latest_formula_version" do + it "returns the expected `PkgVersion` when the revision is 0" do + mock_curl_output stdout: versions_formulae_json + pkg_version = described_class.latest_formula_version("foo") + expect(pkg_version.to_s).to eq "1.2.3" + end + + it "returns the expected `PkgVersion` when the revision is not 0" do + mock_curl_output stdout: versions_formulae_json + pkg_version = described_class.latest_formula_version("bar") + expect(pkg_version.to_s).to eq "1.2_4" + end + + it "returns `nil` when the formula is not in the JSON file" do + mock_curl_output stdout: versions_formulae_json + pkg_version = described_class.latest_formula_version("baz") + expect(pkg_version).to be_nil + end + end + + describe "::latest_cask_version" do + it "returns the expected `Version`" do + mock_curl_output stdout: versions_casks_json + version = described_class.latest_cask_version("foo") + expect(version.to_s).to eq "1.2.3" + end + + it "returns `nil` when the cask is not in the JSON file" do + mock_curl_output stdout: versions_casks_json + version = described_class.latest_cask_version("bar") + expect(version).to be_nil + end + end +end diff --git a/Library/Homebrew/test/api_spec.rb b/Library/Homebrew/test/api_spec.rb new file mode 100644 index 0000000000..c8dffd19a9 --- /dev/null +++ b/Library/Homebrew/test/api_spec.rb @@ -0,0 +1,39 @@ +# typed: false +# frozen_string_literal: true + +require "api" + +describe Homebrew::API do + let(:text) { "foo" } + let(:json) { '{"foo":"bar"}' } + let(:json_hash) { JSON.parse(json) } + + def mock_curl_output(stdout: "", success: true) + curl_output = OpenStruct.new(stdout: stdout, success?: success) + allow(Utils::Curl).to receive(:curl_output).and_return curl_output + end + + describe "::fetch" do + it "fetches a text file" do + mock_curl_output stdout: text + fetched_text = described_class.fetch("foo.txt") + expect(fetched_text).to eq text + end + + it "fetches a JSON file" do + mock_curl_output stdout: json + fetched_json = described_class.fetch("foo.json", json: true) + expect(fetched_json).to eq json_hash + end + + it "raises an error if the file does not exist" do + mock_curl_output success: false + expect { described_class.fetch("bar.txt") }.to raise_error(ArgumentError, /No file found/) + end + + it "raises an error if the JSON file is invalid" do + mock_curl_output stdout: text + expect { described_class.fetch("baz.txt", json: true) }.to raise_error(ArgumentError, /Invalid JSON file/) + end + end +end diff --git a/Library/Homebrew/utils/analytics.rb b/Library/Homebrew/utils/analytics.rb index a1efb5b9bd..e5c8e6d640 100644 --- a/Library/Homebrew/utils/analytics.rb +++ b/Library/Homebrew/utils/analytics.rb @@ -4,6 +4,7 @@ require "context" require "erb" require "settings" +require "api" module Utils # Helper module for fetching and reporting analytics data. @@ -129,7 +130,12 @@ module Utils def output(args:, filter: nil) days = args.days || "30" category = args.category || "install" - json = formulae_brew_sh_json("analytics/#{category}/#{days}d.json") + begin + json = Homebrew::API::Analytics.fetch category, days + rescue ArgumentError + # Ignore failed API requests + return + end return if json.blank? || json["items"].blank? os_version = category == "os-version" @@ -182,17 +188,27 @@ module Utils end def formula_output(f, args:) - json = formulae_brew_sh_json("#{formula_path}/#{f}.json") + return if Homebrew::EnvConfig.no_analytics? || Homebrew::EnvConfig.no_github_api? + + json = Homebrew::API::Formula.fetch f return if json.blank? || json["analytics"].blank? get_analytics(json, args: args) + rescue ArgumentError + # Ignore failed API requests + nil end def cask_output(cask, args:) - json = formulae_brew_sh_json("#{cask_path}/#{cask}.json") + return if Homebrew::EnvConfig.no_analytics? || Homebrew::EnvConfig.no_github_api? + + json = Homebrew::API::Cask.fetch cask return if json.blank? || json["analytics"].blank? get_analytics(json, args: args) + rescue ArgumentError + # Ignore failed API requests + nil end sig { returns(String) } @@ -317,18 +333,6 @@ module Utils Homebrew::Settings.read(key) == "true" end - def formulae_brew_sh_json(endpoint) - return if Homebrew::EnvConfig.no_analytics? || Homebrew::EnvConfig.no_github_api? - - output, = curl_output("--max-time", "5", - "https://formulae.brew.sh/api/#{endpoint}") - return if output.blank? - - JSON.parse(output) - rescue JSON::ParserError - nil - end - def format_count(count) count.to_s.reverse.gsub(/(\d{3})(?=\d)/, '\\1,').reverse end @@ -336,23 +340,6 @@ module Utils def format_percent(percent) format("%.2f", percent: percent) end - - sig { returns(String) } - def formula_path - "formula" - end - alias generic_formula_path formula_path - - sig { returns(String) } - def analytics_path - "analytics" - end - alias generic_analytics_path analytics_path - - sig { returns(String) } - def cask_path - "cask" - end end end end From 2afbd38dde4f371ec4a120aa46f77e7738bce5f2 Mon Sep 17 00:00:00 2001 From: Rylan Polster Date: Fri, 6 Aug 2021 04:35:45 -0400 Subject: [PATCH 2/7] Remove extra comment --- Library/Homebrew/api/bottle.rb | 2 +- Library/Homebrew/test/api/bottle_spec.rb | 4 ---- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/Library/Homebrew/api/bottle.rb b/Library/Homebrew/api/bottle.rb index ad832f2254..1e6a595d86 100644 --- a/Library/Homebrew/api/bottle.rb +++ b/Library/Homebrew/api/bottle.rb @@ -5,7 +5,7 @@ require "github_packages" module Homebrew module API - # Helper functions for using the Bottle JSON API. + # Helper functions for using the bottle JSON API. # # @api private module Bottle diff --git a/Library/Homebrew/test/api/bottle_spec.rb b/Library/Homebrew/test/api/bottle_spec.rb index dee8e05649..89760e45eb 100644 --- a/Library/Homebrew/test/api/bottle_spec.rb +++ b/Library/Homebrew/test/api/bottle_spec.rb @@ -4,10 +4,6 @@ require "api" describe Homebrew::API::Bottle do - # before do - # ENV["HOMEBREW_JSON_CORE"] = "1" - # end - let(:bottle_json) { <<~EOS { From f84265f9a200c693525b16260386f9c2b523aa48 Mon Sep 17 00:00:00 2001 From: Rylan Polster Date: Fri, 6 Aug 2021 04:40:22 -0400 Subject: [PATCH 3/7] Remove extra type signatures --- Library/Homebrew/extend/os/linux/api/analytics.rb | 1 - Library/Homebrew/extend/os/linux/api/bottle.rb | 1 - Library/Homebrew/extend/os/linux/api/formula.rb | 1 - 3 files changed, 3 deletions(-) diff --git a/Library/Homebrew/extend/os/linux/api/analytics.rb b/Library/Homebrew/extend/os/linux/api/analytics.rb index 5d486078e5..7f89d52eab 100644 --- a/Library/Homebrew/extend/os/linux/api/analytics.rb +++ b/Library/Homebrew/extend/os/linux/api/analytics.rb @@ -5,7 +5,6 @@ module Homebrew module API module Analytics class << self - sig { returns(String) } def analytics_api_path return generic_analytics_api_path if Homebrew::EnvConfig.force_homebrew_on_linux? diff --git a/Library/Homebrew/extend/os/linux/api/bottle.rb b/Library/Homebrew/extend/os/linux/api/bottle.rb index 6ac1ea34b1..43336dbde8 100644 --- a/Library/Homebrew/extend/os/linux/api/bottle.rb +++ b/Library/Homebrew/extend/os/linux/api/bottle.rb @@ -5,7 +5,6 @@ module Homebrew module API module Bottle class << self - sig { returns(String) } def bottle_api_path return generic_bottle_api_path if Homebrew::EnvConfig.force_homebrew_on_linux? diff --git a/Library/Homebrew/extend/os/linux/api/formula.rb b/Library/Homebrew/extend/os/linux/api/formula.rb index 1ca27b5e66..80818c823e 100644 --- a/Library/Homebrew/extend/os/linux/api/formula.rb +++ b/Library/Homebrew/extend/os/linux/api/formula.rb @@ -5,7 +5,6 @@ module Homebrew module API module Formula class << self - sig { returns(String) } def formula_api_path return generic_formula_api_path if Homebrew::EnvConfig.force_homebrew_on_linux? From b1215800d407a9a17a45e62e34d055845547e0ed Mon Sep 17 00:00:00 2001 From: Rylan Polster Date: Fri, 6 Aug 2021 11:42:55 -0400 Subject: [PATCH 4/7] Fix tests --- Library/Homebrew/dev-cmd/bottle.rb | 5 ++--- Library/Homebrew/test/api/bottle_spec.rb | 4 ++++ 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/Library/Homebrew/dev-cmd/bottle.rb b/Library/Homebrew/dev-cmd/bottle.rb index b803e5e6fa..1af1cf4f68 100644 --- a/Library/Homebrew/dev-cmd/bottle.rb +++ b/Library/Homebrew/dev-cmd/bottle.rb @@ -11,6 +11,7 @@ require "utils/inreplace" require "erb" require "archive" require "zlib" +require "api" BOTTLE_ERB = <<-EOS bottle do @@ -333,8 +334,6 @@ module Homebrew root_url = args.root_url - formulae_brew_sh_path = Utils::Analytics.formula_path - relocatable = T.let(false, T::Boolean) skip_relocation = T.let(false, T::Boolean) @@ -561,7 +560,7 @@ module Homebrew "filename" => filename.url_encode, "local_filename" => filename.to_s, "sha256" => sha256, - "formulae_brew_sh_path" => formulae_brew_sh_path, + "formulae_brew_sh_path" => Homebrew::API::Formula.formula_api_path, "tab" => tab.to_bottle_hash, }, }, diff --git a/Library/Homebrew/test/api/bottle_spec.rb b/Library/Homebrew/test/api/bottle_spec.rb index 89760e45eb..e6bc6e1fa7 100644 --- a/Library/Homebrew/test/api/bottle_spec.rb +++ b/Library/Homebrew/test/api/bottle_spec.rb @@ -4,6 +4,10 @@ require "api" describe Homebrew::API::Bottle do + before do + ENV["HOMEBREW_JSON_CORE"] = "1" + end + let(:bottle_json) { <<~EOS { From a8c03c1cc6571e2551da489ee097eb5c3ca50488 Mon Sep 17 00:00:00 2001 From: Rylan Polster Date: Fri, 6 Aug 2021 11:52:21 -0400 Subject: [PATCH 5/7] Restrict `HOMEBREW_JSON_CORE` use in tests --- Library/Homebrew/test/api/bottle_spec.rb | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/Library/Homebrew/test/api/bottle_spec.rb b/Library/Homebrew/test/api/bottle_spec.rb index e6bc6e1fa7..17f9b1a066 100644 --- a/Library/Homebrew/test/api/bottle_spec.rb +++ b/Library/Homebrew/test/api/bottle_spec.rb @@ -4,10 +4,6 @@ require "api" describe Homebrew::API::Bottle do - before do - ENV["HOMEBREW_JSON_CORE"] = "1" - end - let(:bottle_json) { <<~EOS { @@ -68,6 +64,7 @@ describe Homebrew::API::Bottle do describe "::fetch_bottles" do before do + ENV["HOMEBREW_JSON_CORE"] = "1" allow(described_class).to receive(:fetch).and_return bottle_hash end From eab0f88c3cf9dfc2396f35281022d14592d7b8dc Mon Sep 17 00:00:00 2001 From: Rylan Polster Date: Mon, 9 Aug 2021 10:29:55 -0400 Subject: [PATCH 6/7] Remove `json` argument and extend `Cachable` --- Library/Homebrew/api.rb | 16 +++++++--------- Library/Homebrew/api/analytics.rb | 2 +- Library/Homebrew/api/bottle.rb | 2 +- Library/Homebrew/api/cask.rb | 2 +- Library/Homebrew/api/formula.rb | 2 +- Library/Homebrew/api/versions.rb | 6 +++--- 6 files changed, 14 insertions(+), 16 deletions(-) diff --git a/Library/Homebrew/api.rb b/Library/Homebrew/api.rb index ac21298f64..38d2748db5 100644 --- a/Library/Homebrew/api.rb +++ b/Library/Homebrew/api.rb @@ -6,6 +6,7 @@ require "api/bottle" require "api/cask" require "api/formula" require "api/versions" +require "extend/cachable" module Homebrew # Helper functions for using Homebrew's formulae.brew.sh API. @@ -14,24 +15,21 @@ module Homebrew module API extend T::Sig + extend Cachable + module_function API_DOMAIN = "https://formulae.brew.sh/api" - sig { params(endpoint: String, json: T::Boolean).returns(T.any(String, Hash)) } - def fetch(endpoint, json: false) - return @cache[endpoint] if @cache.present? && @cache.key?(endpoint) + sig { params(endpoint: String).returns(T.any(String, Hash)) } + def fetch(endpoint) + return cache[endpoint] if cache.present? && cache.key?(endpoint) api_url = "#{API_DOMAIN}/#{endpoint}" output = Utils::Curl.curl_output("--fail", "--max-time", "5", api_url) raise ArgumentError, "No file found at #{Tty.underline}#{api_url}#{Tty.reset}" unless output.success? - @cache ||= {} - @cache[endpoint] = if json - JSON.parse(output.stdout) - else - output.stdout - end + cache[endpoint] = JSON.parse(output.stdout) rescue JSON::ParserError raise ArgumentError, "Invalid JSON file: #{Tty.underline}#{api_url}#{Tty.reset}" end diff --git a/Library/Homebrew/api/analytics.rb b/Library/Homebrew/api/analytics.rb index f7967d4d1d..bdec017f10 100644 --- a/Library/Homebrew/api/analytics.rb +++ b/Library/Homebrew/api/analytics.rb @@ -19,7 +19,7 @@ module Homebrew sig { params(category: String, days: T.any(Integer, String)).returns(Hash) } def fetch(category, days) - Homebrew::API.fetch "#{analytics_api_path}/#{category}/#{days}d.json", json: true + Homebrew::API.fetch "#{analytics_api_path}/#{category}/#{days}d.json" end end end diff --git a/Library/Homebrew/api/bottle.rb b/Library/Homebrew/api/bottle.rb index 1e6a595d86..54f32cdc30 100644 --- a/Library/Homebrew/api/bottle.rb +++ b/Library/Homebrew/api/bottle.rb @@ -23,7 +23,7 @@ module Homebrew sig { params(name: String).returns(Hash) } def fetch(name) - Homebrew::API.fetch "#{bottle_api_path}/#{name}.json", json: true + Homebrew::API.fetch "#{bottle_api_path}/#{name}.json" end sig { params(name: String).returns(T::Boolean) } diff --git a/Library/Homebrew/api/cask.rb b/Library/Homebrew/api/cask.rb index c1dd4ddcd6..8a89ae0187 100644 --- a/Library/Homebrew/api/cask.rb +++ b/Library/Homebrew/api/cask.rb @@ -13,7 +13,7 @@ module Homebrew sig { params(name: String).returns(Hash) } def fetch(name) - Homebrew::API.fetch "cask/#{name}.json", json: true + Homebrew::API.fetch "cask/#{name}.json" end end end diff --git a/Library/Homebrew/api/formula.rb b/Library/Homebrew/api/formula.rb index 329f80bfde..4582ba7a76 100644 --- a/Library/Homebrew/api/formula.rb +++ b/Library/Homebrew/api/formula.rb @@ -19,7 +19,7 @@ module Homebrew sig { params(name: String).returns(Hash) } def fetch(name) - Homebrew::API.fetch "#{formula_api_path}/#{name}.json", json: true + Homebrew::API.fetch "#{formula_api_path}/#{name}.json" end end end diff --git a/Library/Homebrew/api/versions.rb b/Library/Homebrew/api/versions.rb index 0ad01b93c4..b4f971b4a1 100644 --- a/Library/Homebrew/api/versions.rb +++ b/Library/Homebrew/api/versions.rb @@ -13,17 +13,17 @@ module Homebrew def formulae # The result is cached by Homebrew::API.fetch - Homebrew::API.fetch "versions-formulae.json", json: true + Homebrew::API.fetch "versions-formulae.json" end def linux # The result is cached by Homebrew::API.fetch - Homebrew::API.fetch "versions-linux.json", json: true + Homebrew::API.fetch "versions-linux.json" end def casks # The result is cached by Homebrew::API.fetch - Homebrew::API.fetch "versions-casks.json", json: true + Homebrew::API.fetch "versions-casks.json" end sig { params(name: String).returns(T.nilable(PkgVersion)) } From ab8aea06521e0998c2fe781bd985143eb597c770 Mon Sep 17 00:00:00 2001 From: Rylan Polster Date: Mon, 9 Aug 2021 11:00:00 -0400 Subject: [PATCH 7/7] Fix tests --- Library/Homebrew/test/api_spec.rb | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/Library/Homebrew/test/api_spec.rb b/Library/Homebrew/test/api_spec.rb index c8dffd19a9..68a01074fc 100644 --- a/Library/Homebrew/test/api_spec.rb +++ b/Library/Homebrew/test/api_spec.rb @@ -14,15 +14,9 @@ describe Homebrew::API do end describe "::fetch" do - it "fetches a text file" do - mock_curl_output stdout: text - fetched_text = described_class.fetch("foo.txt") - expect(fetched_text).to eq text - end - it "fetches a JSON file" do mock_curl_output stdout: json - fetched_json = described_class.fetch("foo.json", json: true) + fetched_json = described_class.fetch("foo.json") expect(fetched_json).to eq json_hash end @@ -33,7 +27,7 @@ describe Homebrew::API do it "raises an error if the JSON file is invalid" do mock_curl_output stdout: text - expect { described_class.fetch("baz.txt", json: true) }.to raise_error(ArgumentError, /Invalid JSON file/) + expect { described_class.fetch("baz.txt") }.to raise_error(ArgumentError, /Invalid JSON file/) end end end