From 22f986b89a95184126c0dca3de70fcbfc3af0f2f Mon Sep 17 00:00:00 2001 From: Rylan Polster Date: Sat, 3 Jul 2021 13:28:56 -0400 Subject: [PATCH 1/9] Install formulae from JSON files --- Library/Homebrew/cli/named_args.rb | 14 +++- Library/Homebrew/cmd/install.rb | 4 +- Library/Homebrew/cmd/outdated.rb | 4 +- Library/Homebrew/cmd/reinstall.rb | 13 ++++ Library/Homebrew/cmd/upgrade.rb | 14 ++++ Library/Homebrew/formula.rb | 9 ++- Library/Homebrew/utils.rb | 1 + Library/Homebrew/utils/bottle_api.rb | 103 ++++++++++++++++++++++++++ Library/Homebrew/utils/bottle_api.rbi | 7 ++ manpages/brew.1 | 2 +- 10 files changed, 163 insertions(+), 8 deletions(-) create mode 100644 Library/Homebrew/utils/bottle_api.rb create mode 100644 Library/Homebrew/utils/bottle_api.rbi diff --git a/Library/Homebrew/cli/named_args.rb b/Library/Homebrew/cli/named_args.rb index f99cea17d7..71ff3306c4 100644 --- a/Library/Homebrew/cli/named_args.rb +++ b/Library/Homebrew/cli/named_args.rb @@ -49,12 +49,14 @@ module Homebrew ignore_unavailable: T.nilable(T::Boolean), method: T.nilable(Symbol), uniq: T::Boolean, + load_from_json: T::Boolean, ).returns(T::Array[T.any(Formula, Keg, Cask::Cask)]) } - def to_formulae_and_casks(only: parent&.only_formula_or_cask, ignore_unavailable: nil, method: nil, uniq: true) + def to_formulae_and_casks(only: parent&.only_formula_or_cask, ignore_unavailable: nil, method: nil, uniq: true, + load_from_json: false) @to_formulae_and_casks ||= {} @to_formulae_and_casks[only] ||= downcased_unique_named.flat_map do |name| - load_formula_or_cask(name, only: only, method: method) + load_formula_or_cask(name, only: only, method: method, load_from_json: load_from_json) rescue FormulaUnreadableError, FormulaClassUnavailableError, TapFormulaUnreadableError, TapFormulaClassUnavailableError, Cask::CaskUnreadableError @@ -88,7 +90,7 @@ module Homebrew end.uniq.freeze end - def load_formula_or_cask(name, only: nil, method: nil) + def load_formula_or_cask(name, only: nil, method: nil, load_from_json: false) unreadable_error = nil if only != :cask @@ -121,6 +123,12 @@ module Homebrew # The formula was found, but there's a problem with its implementation unreadable_error ||= e rescue NoSuchKegError, FormulaUnavailableError => e + if load_from_json && ENV["HOMEBREW_JSON_CORE"].present? && !CoreTap.instance.installed? && + Utils::BottleAPI.bottle_available?(name) + Utils::BottleAPI.download_bottles(name) + retry + end + raise e if only == :formula end end diff --git a/Library/Homebrew/cmd/install.rb b/Library/Homebrew/cmd/install.rb index 276700e6bc..3582770b30 100644 --- a/Library/Homebrew/cmd/install.rb +++ b/Library/Homebrew/cmd/install.rb @@ -154,8 +154,10 @@ module Homebrew EOS end + allow_loading_from_json = ENV["HOMEBREW_JSON_CORE"].present? && !CoreTap.instance.installed? + begin - formulae, casks = args.named.to_formulae_and_casks + formulae, casks = args.named.to_formulae_and_casks(load_from_json: allow_loading_from_json) .partition { |formula_or_cask| formula_or_cask.is_a?(Formula) } rescue FormulaOrCaskUnavailableError, Cask::CaskUnavailableError => e retry if Tap.install_default_cask_tap_if_necessary(force: args.cask?) diff --git a/Library/Homebrew/cmd/outdated.rb b/Library/Homebrew/cmd/outdated.rb index fb5246be21..b888dcd8cc 100644 --- a/Library/Homebrew/cmd/outdated.rb +++ b/Library/Homebrew/cmd/outdated.rb @@ -97,8 +97,10 @@ module Homebrew elsif f.head? && outdated_kegs.any? { |k| k.version.to_s == f.pkg_version.to_s } # There is a newer HEAD but the version number has not changed. "latest HEAD" - else + elsif f.tap.present? f.pkg_version.to_s + else + Utils::BottleAPI.latest_pkg_version(f.name).to_s end outdated_versions = outdated_kegs.group_by { |keg| Formulary.from_keg(keg).full_name } diff --git a/Library/Homebrew/cmd/reinstall.rb b/Library/Homebrew/cmd/reinstall.rb index 062119eb18..1bfec15091 100644 --- a/Library/Homebrew/cmd/reinstall.rb +++ b/Library/Homebrew/cmd/reinstall.rb @@ -84,6 +84,19 @@ module Homebrew def reinstall args = reinstall_args.parse + if ENV["HOMEBREW_JSON_CORE"].present? && !CoreTap.instance.installed? + args.named.each do |name| + formula = Formulary.factory(name) + next unless formula.any_version_installed? + next if formula.tap.present? && !formula.tap.installed? + next unless Utils::BottleAPI.bottle_available?(name) + + Utils::BottleAPI.download_bottles(name) + rescue FormulaUnavailableError + next + end + end + formulae, casks = args.named.to_formulae_and_casks(method: :resolve) .partition { |o| o.is_a?(Formula) } diff --git a/Library/Homebrew/cmd/upgrade.rb b/Library/Homebrew/cmd/upgrade.rb index bd296a6f3d..8f5e8ab71e 100644 --- a/Library/Homebrew/cmd/upgrade.rb +++ b/Library/Homebrew/cmd/upgrade.rb @@ -159,6 +159,20 @@ module Homebrew puts pinned.map { |f| "#{f.full_specified_name} #{f.pkg_version}" } * ", " end + if ENV["HOMEBREW_JSON_CORE"].present? && !CoreTap.instance.installed? + formulae_to_install.map! do |formula| + next formula if formula.tap.present? && formula.tap.installed? + next formula unless Utils::BottleAPI.bottle_available?(formula.name) + + Utils::BottleAPI.download_bottles(formula.name) + Formulary.factory(formula.name) + rescue FormulaUnavailableError + formula + end + end + + opoo formulae_to_install + if formulae_to_install.empty? oh1 "No packages to upgrade" else diff --git a/Library/Homebrew/formula.rb b/Library/Homebrew/formula.rb index d7793cdfaf..db7e8faf43 100644 --- a/Library/Homebrew/formula.rb +++ b/Library/Homebrew/formula.rb @@ -1325,6 +1325,11 @@ class Formula Formula.cache[:outdated_kegs][cache_key] ||= begin all_kegs = [] current_version = T.let(false, T::Boolean) + latest_version = if tap.present? + pkg_version + else + Utils::BottleAPI.latest_pkg_version name + end installed_kegs.each do |keg| all_kegs << keg @@ -1332,8 +1337,8 @@ class Formula next if version.head? tab = Tab.for_keg(keg) - next if version_scheme > tab.version_scheme && pkg_version != version - next if version_scheme == tab.version_scheme && pkg_version > version + next if version_scheme > tab.version_scheme && latest_version != version + next if version_scheme == tab.version_scheme && latest_version > version # don't consider this keg current if there's a newer formula available next if follow_installed_alias? && new_formula_available? diff --git a/Library/Homebrew/utils.rb b/Library/Homebrew/utils.rb index 361cc285f7..5b4de1c9de 100644 --- a/Library/Homebrew/utils.rb +++ b/Library/Homebrew/utils.rb @@ -4,6 +4,7 @@ require "time" require "utils/analytics" +require "utils/bottle_api" require "utils/curl" require "utils/fork" require "utils/formatter" diff --git a/Library/Homebrew/utils/bottle_api.rb b/Library/Homebrew/utils/bottle_api.rb new file mode 100644 index 0000000000..0df65cba62 --- /dev/null +++ b/Library/Homebrew/utils/bottle_api.rb @@ -0,0 +1,103 @@ +# typed: true +# frozen_string_literal: true + +require "github_packages" + +module Utils + # Helper functions for using the Bottle JSON API. + # + # @api private + module BottleAPI + extend T::Sig + + module_function + + FORMULAE_BREW_SH_BOTTLE_API_DOMAIN = if OS.mac? + "https://formulae.brew.sh/api/bottle" + else + "https://formulae.brew.sh/api/bottle-linux" + end.freeze + + FORMULAE_BREW_SH_VERSIONS_API_URL = if OS.mac? + "https://formulae.brew.sh/api/versions.json" + else + "https://formulae.brew.sh/api/versions-linux.json" + end.freeze + + GITHUB_PACKAGES_SHA256_REGEX = %r{#{GitHubPackages::URL_REGEX}.*/blobs/sha256:(?\h{64})$}.freeze + + sig { params(name: String).returns(Hash) } + def fetch(name) + return @cache[name] if @cache.present? && @cache.key?(name) + + api_url = "#{FORMULAE_BREW_SH_BOTTLE_API_DOMAIN}/#{name}.json" + output = Utils::Curl.curl_output("--fail", api_url) + raise ArgumentError, "No JSON file found at #{Tty.underline}#{api_url}#{Tty.reset}" unless output.success? + + @cache ||= {} + @cache[name] = JSON.parse(output.stdout) + rescue JSON::ParserError + raise ArgumentError, "Invalid JSON file: #{Tty.underline}#{api_url}#{Tty.reset}" + end + + sig { params(name: String).returns(PkgVersion) } + def latest_pkg_version(name) + @formula_versions ||= begin + output = Utils::Curl.curl_output("--fail", FORMULAE_BREW_SH_VERSIONS_API_URL) + JSON.parse(output.stdout) + end + PkgVersion.new(@formula_versions[name]["version"], @formula_versions[name]["revision"]) + end + + sig { params(name: String).returns(T::Boolean) } + def bottle_available?(name) + fetch name + true + rescue ArgumentError + false + end + + sig { params(name: String).void } + def download_bottles(name) + hash = fetch(name) + bottle_tag = Utils::Bottles.tag.to_s + + odie "No bottle availabe for current OS" unless hash["bottles"].key? bottle_tag + + download_bottle(hash, bottle_tag) + + hash["dependencies"].each do |dep_hash| + download_bottle(dep_hash, bottle_tag) + end + end + + sig { params(url: String).returns(T.nilable(String)) } + def checksum_from_url(url) + match = url.match GITHUB_PACKAGES_SHA256_REGEX + return if match.blank? + + match[:sha256] + end + + sig { params(hash: Hash, tag: Symbol).void } + def download_bottle(hash, tag) + bottle = hash["bottles"][tag] + return if bottle.blank? + + sha256 = bottle["sha256"] || checksum_from_url(bottle["url"]) + bottle_filename = Bottle::Filename.new(hash["name"], hash["pkg_version"], tag, hash["rebuild"]) + + resource = Resource.new hash["name"] + resource.url bottle["url"] + resource.sha256 sha256 + resource.version hash["pkg_version"] + resource.downloader.resolved_basename = bottle_filename + + resource.fetch + + # Map the name of this formula to the local bottle path to allow the + # formula to be loaded by passing just the name to `Formulary::factory`. + Formulary.map_formula_name_to_local_bottle_path hash["name"], resource.downloader.cached_location + end + end +end diff --git a/Library/Homebrew/utils/bottle_api.rbi b/Library/Homebrew/utils/bottle_api.rbi new file mode 100644 index 0000000000..403c511f59 --- /dev/null +++ b/Library/Homebrew/utils/bottle_api.rbi @@ -0,0 +1,7 @@ +# typed: strict + +module Utils + module BottleAPI + include Kernel + end +end diff --git a/manpages/brew.1 b/manpages/brew.1 index 20d2deda05..267aedf461 100644 --- a/manpages/brew.1 +++ b/manpages/brew.1 @@ -1,7 +1,7 @@ .\" generated with Ronn/v0.7.3 .\" http://github.com/rtomayko/ronn/tree/0.7.3 . -.TH "BREW" "1" "June 2021" "Homebrew" "brew" +.TH "BREW" "1" "July 2021" "Homebrew" "brew" . .SH "NAME" \fBbrew\fR \- The Missing Package Manager for macOS (or Linux) From 36dd69dd60f48a7b7e498b09ed00f3a9b68aeac7 Mon Sep 17 00:00:00 2001 From: Rylan Polster Date: Mon, 5 Jul 2021 10:49:19 -0400 Subject: [PATCH 2/9] Rename `download_bottles` to `fetch_bottles` --- Library/Homebrew/cli/named_args.rb | 2 +- Library/Homebrew/cmd/reinstall.rb | 2 +- Library/Homebrew/cmd/upgrade.rb | 4 +--- Library/Homebrew/utils/bottle_api.rb | 6 +++--- 4 files changed, 6 insertions(+), 8 deletions(-) diff --git a/Library/Homebrew/cli/named_args.rb b/Library/Homebrew/cli/named_args.rb index 71ff3306c4..3160fb2252 100644 --- a/Library/Homebrew/cli/named_args.rb +++ b/Library/Homebrew/cli/named_args.rb @@ -125,7 +125,7 @@ module Homebrew rescue NoSuchKegError, FormulaUnavailableError => e if load_from_json && ENV["HOMEBREW_JSON_CORE"].present? && !CoreTap.instance.installed? && Utils::BottleAPI.bottle_available?(name) - Utils::BottleAPI.download_bottles(name) + Utils::BottleAPI.fetch_bottles(name) retry end diff --git a/Library/Homebrew/cmd/reinstall.rb b/Library/Homebrew/cmd/reinstall.rb index 1bfec15091..e03d522e1d 100644 --- a/Library/Homebrew/cmd/reinstall.rb +++ b/Library/Homebrew/cmd/reinstall.rb @@ -91,7 +91,7 @@ module Homebrew next if formula.tap.present? && !formula.tap.installed? next unless Utils::BottleAPI.bottle_available?(name) - Utils::BottleAPI.download_bottles(name) + Utils::BottleAPI.fetch_bottles(name) rescue FormulaUnavailableError next end diff --git a/Library/Homebrew/cmd/upgrade.rb b/Library/Homebrew/cmd/upgrade.rb index 8f5e8ab71e..64c06aedd5 100644 --- a/Library/Homebrew/cmd/upgrade.rb +++ b/Library/Homebrew/cmd/upgrade.rb @@ -164,15 +164,13 @@ module Homebrew next formula if formula.tap.present? && formula.tap.installed? next formula unless Utils::BottleAPI.bottle_available?(formula.name) - Utils::BottleAPI.download_bottles(formula.name) + Utils::BottleAPI.fetch_bottles(formula.name) Formulary.factory(formula.name) rescue FormulaUnavailableError formula end end - opoo formulae_to_install - if formulae_to_install.empty? oh1 "No packages to upgrade" else diff --git a/Library/Homebrew/utils/bottle_api.rb b/Library/Homebrew/utils/bottle_api.rb index 0df65cba62..3d42ac083d 100644 --- a/Library/Homebrew/utils/bottle_api.rb +++ b/Library/Homebrew/utils/bottle_api.rb @@ -19,7 +19,7 @@ module Utils end.freeze FORMULAE_BREW_SH_VERSIONS_API_URL = if OS.mac? - "https://formulae.brew.sh/api/versions.json" + "https://formulae.brew.sh/api/versions-formulae.json" else "https://formulae.brew.sh/api/versions-linux.json" end.freeze @@ -58,11 +58,11 @@ module Utils end sig { params(name: String).void } - def download_bottles(name) + def fetch_bottles(name) hash = fetch(name) bottle_tag = Utils::Bottles.tag.to_s - odie "No bottle availabe for current OS" unless hash["bottles"].key? bottle_tag + odie "No bottle available for current OS" unless hash["bottles"].key? bottle_tag download_bottle(hash, bottle_tag) From ca5f6026eddbc0037f875b3db4a72a606a7d3c42 Mon Sep 17 00:00:00 2001 From: Rylan Polster Date: Mon, 5 Jul 2021 11:29:27 -0400 Subject: [PATCH 3/9] Fix tests --- Library/Homebrew/cmd/outdated.rb | 2 +- Library/Homebrew/cmd/reinstall.rb | 2 +- Library/Homebrew/cmd/upgrade.rb | 2 +- Library/Homebrew/formula.rb | 2 +- Library/Homebrew/utils/bottle_api.rb | 3 +++ 5 files changed, 7 insertions(+), 4 deletions(-) diff --git a/Library/Homebrew/cmd/outdated.rb b/Library/Homebrew/cmd/outdated.rb index b888dcd8cc..801b3e8370 100644 --- a/Library/Homebrew/cmd/outdated.rb +++ b/Library/Homebrew/cmd/outdated.rb @@ -100,7 +100,7 @@ module Homebrew elsif f.tap.present? f.pkg_version.to_s else - Utils::BottleAPI.latest_pkg_version(f.name).to_s + Utils::BottleAPI.latest_pkg_version(f.name)&.to_s || f.pkg_version.to_s end outdated_versions = outdated_kegs.group_by { |keg| Formulary.from_keg(keg).full_name } diff --git a/Library/Homebrew/cmd/reinstall.rb b/Library/Homebrew/cmd/reinstall.rb index e03d522e1d..9dc0b25e40 100644 --- a/Library/Homebrew/cmd/reinstall.rb +++ b/Library/Homebrew/cmd/reinstall.rb @@ -88,7 +88,7 @@ module Homebrew args.named.each do |name| formula = Formulary.factory(name) next unless formula.any_version_installed? - next if formula.tap.present? && !formula.tap.installed? + next if formula.tap.present? next unless Utils::BottleAPI.bottle_available?(name) Utils::BottleAPI.fetch_bottles(name) diff --git a/Library/Homebrew/cmd/upgrade.rb b/Library/Homebrew/cmd/upgrade.rb index 64c06aedd5..ded57bc74d 100644 --- a/Library/Homebrew/cmd/upgrade.rb +++ b/Library/Homebrew/cmd/upgrade.rb @@ -161,7 +161,7 @@ module Homebrew if ENV["HOMEBREW_JSON_CORE"].present? && !CoreTap.instance.installed? formulae_to_install.map! do |formula| - next formula if formula.tap.present? && formula.tap.installed? + next formula if formula.tap.present? next formula unless Utils::BottleAPI.bottle_available?(formula.name) Utils::BottleAPI.fetch_bottles(formula.name) diff --git a/Library/Homebrew/formula.rb b/Library/Homebrew/formula.rb index db7e8faf43..8c6bda7216 100644 --- a/Library/Homebrew/formula.rb +++ b/Library/Homebrew/formula.rb @@ -1328,7 +1328,7 @@ class Formula latest_version = if tap.present? pkg_version else - Utils::BottleAPI.latest_pkg_version name + Utils::BottleAPI.latest_pkg_version(name) || pkg_version end installed_kegs.each do |keg| diff --git a/Library/Homebrew/utils/bottle_api.rb b/Library/Homebrew/utils/bottle_api.rb index 3d42ac083d..838367f57b 100644 --- a/Library/Homebrew/utils/bottle_api.rb +++ b/Library/Homebrew/utils/bottle_api.rb @@ -46,6 +46,9 @@ module Utils output = Utils::Curl.curl_output("--fail", FORMULAE_BREW_SH_VERSIONS_API_URL) JSON.parse(output.stdout) end + + return unless @formula_versions.key? name + PkgVersion.new(@formula_versions[name]["version"], @formula_versions[name]["revision"]) end From e316c4f01376b80781f3e1a9afe022fccd78c546 Mon Sep 17 00:00:00 2001 From: Rylan Polster Date: Mon, 5 Jul 2021 11:43:34 -0400 Subject: [PATCH 4/9] Rename `Utils::BottleAPI` to `BottleAPI` --- Library/Homebrew/bottle_api.rb | 104 +++++++++++++++++++++++++ Library/Homebrew/bottle_api.rbi | 5 ++ Library/Homebrew/cli/named_args.rb | 6 +- Library/Homebrew/cmd/outdated.rb | 3 +- Library/Homebrew/cmd/reinstall.rb | 5 +- Library/Homebrew/cmd/upgrade.rb | 5 +- Library/Homebrew/formula.rb | 3 +- Library/Homebrew/utils.rb | 1 - Library/Homebrew/utils/bottle_api.rb | 106 -------------------------- Library/Homebrew/utils/bottle_api.rbi | 7 -- 10 files changed, 122 insertions(+), 123 deletions(-) create mode 100644 Library/Homebrew/bottle_api.rb create mode 100644 Library/Homebrew/bottle_api.rbi delete mode 100644 Library/Homebrew/utils/bottle_api.rb delete mode 100644 Library/Homebrew/utils/bottle_api.rbi diff --git a/Library/Homebrew/bottle_api.rb b/Library/Homebrew/bottle_api.rb new file mode 100644 index 0000000000..2c29bb1086 --- /dev/null +++ b/Library/Homebrew/bottle_api.rb @@ -0,0 +1,104 @@ +# typed: true +# frozen_string_literal: true + +require "github_packages" + +# Helper functions for using the Bottle JSON API. +# +# @api private +module BottleAPI + extend T::Sig + + module_function + + FORMULAE_BREW_SH_BOTTLE_API_DOMAIN = if OS.mac? + "https://formulae.brew.sh/api/bottle" + else + "https://formulae.brew.sh/api/bottle-linux" + end.freeze + + FORMULAE_BREW_SH_VERSIONS_API_URL = if OS.mac? + "https://formulae.brew.sh/api/versions-formulae.json" + else + "https://formulae.brew.sh/api/versions-linux.json" + end.freeze + + GITHUB_PACKAGES_SHA256_REGEX = %r{#{GitHubPackages::URL_REGEX}.*/blobs/sha256:(?\h{64})$}.freeze + + sig { params(name: String).returns(Hash) } + def fetch(name) + return @cache[name] if @cache.present? && @cache.key?(name) + + api_url = "#{FORMULAE_BREW_SH_BOTTLE_API_DOMAIN}/#{name}.json" + output = Utils::Curl.curl_output("--fail", api_url) + raise ArgumentError, "No JSON file found at #{Tty.underline}#{api_url}#{Tty.reset}" unless output.success? + + @cache ||= {} + @cache[name] = JSON.parse(output.stdout) + rescue JSON::ParserError + raise ArgumentError, "Invalid JSON file: #{Tty.underline}#{api_url}#{Tty.reset}" + end + + sig { params(name: String).returns(T.nilable(PkgVersion)) } + def latest_pkg_version(name) + @formula_versions ||= begin + output = Utils::Curl.curl_output("--fail", FORMULAE_BREW_SH_VERSIONS_API_URL) + JSON.parse(output.stdout) + end + + return unless @formula_versions.key? name + + PkgVersion.new(@formula_versions[name]["version"], @formula_versions[name]["revision"]) + end + + sig { params(name: String).returns(T::Boolean) } + def bottle_available?(name) + fetch name + true + rescue ArgumentError + false + end + + sig { params(name: String).void } + def fetch_bottles(name) + hash = fetch(name) + bottle_tag = Utils::Bottles.tag.to_s + + odie "No bottle available for current OS" unless hash["bottles"].key? bottle_tag + + download_bottle(hash, bottle_tag) + + hash["dependencies"].each do |dep_hash| + download_bottle(dep_hash, bottle_tag) + end + end + + sig { params(url: String).returns(T.nilable(String)) } + def checksum_from_url(url) + match = url.match GITHUB_PACKAGES_SHA256_REGEX + return if match.blank? + + match[:sha256] + end + + sig { params(hash: Hash, tag: Symbol).void } + def download_bottle(hash, tag) + bottle = hash["bottles"][tag] + return if bottle.blank? + + sha256 = bottle["sha256"] || checksum_from_url(bottle["url"]) + bottle_filename = Bottle::Filename.new(hash["name"], hash["pkg_version"], tag, hash["rebuild"]) + + resource = Resource.new hash["name"] + resource.url bottle["url"] + resource.sha256 sha256 + resource.version hash["pkg_version"] + resource.downloader.resolved_basename = bottle_filename + + resource.fetch + + # Map the name of this formula to the local bottle path to allow the + # formula to be loaded by passing just the name to `Formulary::factory`. + Formulary.map_formula_name_to_local_bottle_path hash["name"], resource.downloader.cached_location + end +end diff --git a/Library/Homebrew/bottle_api.rbi b/Library/Homebrew/bottle_api.rbi new file mode 100644 index 0000000000..6fefceb732 --- /dev/null +++ b/Library/Homebrew/bottle_api.rbi @@ -0,0 +1,5 @@ +# typed: strict + +module BottleAPI + include Kernel +end diff --git a/Library/Homebrew/cli/named_args.rb b/Library/Homebrew/cli/named_args.rb index 3160fb2252..4a8fbbd847 100644 --- a/Library/Homebrew/cli/named_args.rb +++ b/Library/Homebrew/cli/named_args.rb @@ -2,7 +2,7 @@ # frozen_string_literal: true require "delegate" - +require "bottle_api" require "cli/args" module Homebrew @@ -124,8 +124,8 @@ module Homebrew unreadable_error ||= e rescue NoSuchKegError, FormulaUnavailableError => e if load_from_json && ENV["HOMEBREW_JSON_CORE"].present? && !CoreTap.instance.installed? && - Utils::BottleAPI.bottle_available?(name) - Utils::BottleAPI.fetch_bottles(name) + BottleAPI.bottle_available?(name) + BottleAPI.fetch_bottles(name) retry end diff --git a/Library/Homebrew/cmd/outdated.rb b/Library/Homebrew/cmd/outdated.rb index 801b3e8370..dc49defd60 100644 --- a/Library/Homebrew/cmd/outdated.rb +++ b/Library/Homebrew/cmd/outdated.rb @@ -6,6 +6,7 @@ require "keg" require "cli/parser" require "cask/cmd" require "cask/caskroom" +require "bottle_api" module Homebrew extend T::Sig @@ -100,7 +101,7 @@ module Homebrew elsif f.tap.present? f.pkg_version.to_s else - Utils::BottleAPI.latest_pkg_version(f.name)&.to_s || f.pkg_version.to_s + BottleAPI.latest_pkg_version(f.name)&.to_s || f.pkg_version.to_s end outdated_versions = outdated_kegs.group_by { |keg| Formulary.from_keg(keg).full_name } diff --git a/Library/Homebrew/cmd/reinstall.rb b/Library/Homebrew/cmd/reinstall.rb index 9dc0b25e40..bd64922c22 100644 --- a/Library/Homebrew/cmd/reinstall.rb +++ b/Library/Homebrew/cmd/reinstall.rb @@ -12,6 +12,7 @@ require "cask/cmd" require "cask/utils" require "cask/macos" require "upgrade" +require "bottle_api" module Homebrew extend T::Sig @@ -89,9 +90,9 @@ module Homebrew formula = Formulary.factory(name) next unless formula.any_version_installed? next if formula.tap.present? - next unless Utils::BottleAPI.bottle_available?(name) + next unless BottleAPI.bottle_available?(name) - Utils::BottleAPI.fetch_bottles(name) + BottleAPI.fetch_bottles(name) rescue FormulaUnavailableError next end diff --git a/Library/Homebrew/cmd/upgrade.rb b/Library/Homebrew/cmd/upgrade.rb index ded57bc74d..3e8ff5c935 100644 --- a/Library/Homebrew/cmd/upgrade.rb +++ b/Library/Homebrew/cmd/upgrade.rb @@ -8,6 +8,7 @@ require "upgrade" require "cask/cmd" require "cask/utils" require "cask/macos" +require "bottle_api" module Homebrew extend T::Sig @@ -162,9 +163,9 @@ module Homebrew if ENV["HOMEBREW_JSON_CORE"].present? && !CoreTap.instance.installed? formulae_to_install.map! do |formula| next formula if formula.tap.present? - next formula unless Utils::BottleAPI.bottle_available?(formula.name) + next formula unless BottleAPI.bottle_available?(formula.name) - Utils::BottleAPI.fetch_bottles(formula.name) + BottleAPI.fetch_bottles(formula.name) Formulary.factory(formula.name) rescue FormulaUnavailableError formula diff --git a/Library/Homebrew/formula.rb b/Library/Homebrew/formula.rb index 8c6bda7216..719aec9c07 100644 --- a/Library/Homebrew/formula.rb +++ b/Library/Homebrew/formula.rb @@ -29,6 +29,7 @@ require "mktemp" require "find" require "utils/spdx" require "extend/on_os" +require "bottle_api" # A formula provides instructions and metadata for Homebrew to install a piece # of software. Every Homebrew formula is a {Formula}. @@ -1328,7 +1329,7 @@ class Formula latest_version = if tap.present? pkg_version else - Utils::BottleAPI.latest_pkg_version(name) || pkg_version + BottleAPI.latest_pkg_version(name) || pkg_version end installed_kegs.each do |keg| diff --git a/Library/Homebrew/utils.rb b/Library/Homebrew/utils.rb index 5b4de1c9de..361cc285f7 100644 --- a/Library/Homebrew/utils.rb +++ b/Library/Homebrew/utils.rb @@ -4,7 +4,6 @@ require "time" require "utils/analytics" -require "utils/bottle_api" require "utils/curl" require "utils/fork" require "utils/formatter" diff --git a/Library/Homebrew/utils/bottle_api.rb b/Library/Homebrew/utils/bottle_api.rb deleted file mode 100644 index 838367f57b..0000000000 --- a/Library/Homebrew/utils/bottle_api.rb +++ /dev/null @@ -1,106 +0,0 @@ -# typed: true -# frozen_string_literal: true - -require "github_packages" - -module Utils - # Helper functions for using the Bottle JSON API. - # - # @api private - module BottleAPI - extend T::Sig - - module_function - - FORMULAE_BREW_SH_BOTTLE_API_DOMAIN = if OS.mac? - "https://formulae.brew.sh/api/bottle" - else - "https://formulae.brew.sh/api/bottle-linux" - end.freeze - - FORMULAE_BREW_SH_VERSIONS_API_URL = if OS.mac? - "https://formulae.brew.sh/api/versions-formulae.json" - else - "https://formulae.brew.sh/api/versions-linux.json" - end.freeze - - GITHUB_PACKAGES_SHA256_REGEX = %r{#{GitHubPackages::URL_REGEX}.*/blobs/sha256:(?\h{64})$}.freeze - - sig { params(name: String).returns(Hash) } - def fetch(name) - return @cache[name] if @cache.present? && @cache.key?(name) - - api_url = "#{FORMULAE_BREW_SH_BOTTLE_API_DOMAIN}/#{name}.json" - output = Utils::Curl.curl_output("--fail", api_url) - raise ArgumentError, "No JSON file found at #{Tty.underline}#{api_url}#{Tty.reset}" unless output.success? - - @cache ||= {} - @cache[name] = JSON.parse(output.stdout) - rescue JSON::ParserError - raise ArgumentError, "Invalid JSON file: #{Tty.underline}#{api_url}#{Tty.reset}" - end - - sig { params(name: String).returns(PkgVersion) } - def latest_pkg_version(name) - @formula_versions ||= begin - output = Utils::Curl.curl_output("--fail", FORMULAE_BREW_SH_VERSIONS_API_URL) - JSON.parse(output.stdout) - end - - return unless @formula_versions.key? name - - PkgVersion.new(@formula_versions[name]["version"], @formula_versions[name]["revision"]) - end - - sig { params(name: String).returns(T::Boolean) } - def bottle_available?(name) - fetch name - true - rescue ArgumentError - false - end - - sig { params(name: String).void } - def fetch_bottles(name) - hash = fetch(name) - bottle_tag = Utils::Bottles.tag.to_s - - odie "No bottle available for current OS" unless hash["bottles"].key? bottle_tag - - download_bottle(hash, bottle_tag) - - hash["dependencies"].each do |dep_hash| - download_bottle(dep_hash, bottle_tag) - end - end - - sig { params(url: String).returns(T.nilable(String)) } - def checksum_from_url(url) - match = url.match GITHUB_PACKAGES_SHA256_REGEX - return if match.blank? - - match[:sha256] - end - - sig { params(hash: Hash, tag: Symbol).void } - def download_bottle(hash, tag) - bottle = hash["bottles"][tag] - return if bottle.blank? - - sha256 = bottle["sha256"] || checksum_from_url(bottle["url"]) - bottle_filename = Bottle::Filename.new(hash["name"], hash["pkg_version"], tag, hash["rebuild"]) - - resource = Resource.new hash["name"] - resource.url bottle["url"] - resource.sha256 sha256 - resource.version hash["pkg_version"] - resource.downloader.resolved_basename = bottle_filename - - resource.fetch - - # Map the name of this formula to the local bottle path to allow the - # formula to be loaded by passing just the name to `Formulary::factory`. - Formulary.map_formula_name_to_local_bottle_path hash["name"], resource.downloader.cached_location - end - end -end diff --git a/Library/Homebrew/utils/bottle_api.rbi b/Library/Homebrew/utils/bottle_api.rbi deleted file mode 100644 index 403c511f59..0000000000 --- a/Library/Homebrew/utils/bottle_api.rbi +++ /dev/null @@ -1,7 +0,0 @@ -# typed: strict - -module Utils - module BottleAPI - include Kernel - end -end From b68e17552cc8c78c2f7b187a783e6e419f2cc74e Mon Sep 17 00:00:00 2001 From: Rylan Polster Date: Tue, 6 Jul 2021 09:33:16 -0400 Subject: [PATCH 5/9] Only use `BottleAPI::latest_pkg_version` with env var --- Library/Homebrew/cmd/outdated.rb | 6 +++--- Library/Homebrew/formula.rb | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/Library/Homebrew/cmd/outdated.rb b/Library/Homebrew/cmd/outdated.rb index dc49defd60..f2079fae56 100644 --- a/Library/Homebrew/cmd/outdated.rb +++ b/Library/Homebrew/cmd/outdated.rb @@ -98,10 +98,10 @@ module Homebrew elsif f.head? && outdated_kegs.any? { |k| k.version.to_s == f.pkg_version.to_s } # There is a newer HEAD but the version number has not changed. "latest HEAD" - elsif f.tap.present? - f.pkg_version.to_s - else + elsif f.tap.blank? && ENV["HOMEBREW_JSON_CORE"].present? BottleAPI.latest_pkg_version(f.name)&.to_s || f.pkg_version.to_s + else + f.pkg_version.to_s end outdated_versions = outdated_kegs.group_by { |keg| Formulary.from_keg(keg).full_name } diff --git a/Library/Homebrew/formula.rb b/Library/Homebrew/formula.rb index 719aec9c07..ec14f8d450 100644 --- a/Library/Homebrew/formula.rb +++ b/Library/Homebrew/formula.rb @@ -1326,10 +1326,10 @@ class Formula Formula.cache[:outdated_kegs][cache_key] ||= begin all_kegs = [] current_version = T.let(false, T::Boolean) - latest_version = if tap.present? - pkg_version - else + latest_version = if tap.blank? && ENV["HOMEBREW_JSON_CORE"].present? BottleAPI.latest_pkg_version(name) || pkg_version + else + pkg_version end installed_kegs.each do |keg| From 92af768211ccae19ab6d26bdad020e5544a32029 Mon Sep 17 00:00:00 2001 From: Rylan Polster Date: Tue, 6 Jul 2021 09:33:41 -0400 Subject: [PATCH 6/9] Fix `BottleAPI::latest_pkg_version` comparison --- Library/Homebrew/bottle_api.rb | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Library/Homebrew/bottle_api.rb b/Library/Homebrew/bottle_api.rb index 2c29bb1086..214e62a73d 100644 --- a/Library/Homebrew/bottle_api.rb +++ b/Library/Homebrew/bottle_api.rb @@ -48,7 +48,9 @@ module BottleAPI return unless @formula_versions.key? name - PkgVersion.new(@formula_versions[name]["version"], @formula_versions[name]["revision"]) + version = Version.new(@formula_versions[name]["version"]) + revision = @formula_versions[name]["revision"] + PkgVersion.new(version, revision) end sig { params(name: String).returns(T::Boolean) } From 68c2be5c17a4e105053afda9ab0a8d9b182276ba Mon Sep 17 00:00:00 2001 From: Rylan Polster Date: Mon, 12 Jul 2021 03:03:29 -0400 Subject: [PATCH 7/9] BottleAPI: add tests --- Library/Homebrew/test/bottle_api_spec.rb | 120 +++++++++++++++++++++++ 1 file changed, 120 insertions(+) create mode 100644 Library/Homebrew/test/bottle_api_spec.rb diff --git a/Library/Homebrew/test/bottle_api_spec.rb b/Library/Homebrew/test/bottle_api_spec.rb new file mode 100644 index 0000000000..9be228e05a --- /dev/null +++ b/Library/Homebrew/test/bottle_api_spec.rb @@ -0,0 +1,120 @@ +# typed: false +# frozen_string_literal: true + +describe BottleAPI do + let(:bottle_json) { + <<~EOS + { + "name": "hello", + "pkg_version": "2.10", + "rebuild": 0, + "bottles": { + "arm64_big_sur": { + "url": "https://ghcr.io/v2/homebrew/core/hello/blobs/sha256:b3b083db0807ff92c6e289a298f378198354b7727fb9ba9f4d550b8e08f90a60" + }, + "big_sur": { + "url": "https://ghcr.io/v2/homebrew/core/hello/blobs/sha256:69489ae397e4645127aa7773211310f81ebb6c99e1f8e3e22c5cdb55333f5408" + }, + "x86_64_linux": { + "url": "https://ghcr.io/v2/homebrew/core/hello/blobs/sha256:e6980196298e0a9cfe4fa4e328a71a1869a4d5e1d31c38442150ed784cfc0e29" + } + }, + "dependencies": [] + } + EOS + } + let(:bottle_hash) { JSON.parse(bottle_json) } + let(:versions_json) { + <<~EOS + { + "foo":{"version":"1.2.3","revision":0}, + "bar":{"version":"1.2","revision":4} + } + EOS + } + + def mock_curl_output(stdout: "", success: true) + curl_output = OpenStruct.new(stdout: stdout, success?: success) + allow(Utils::Curl).to receive(:curl_output).and_return curl_output + end + + describe "::fetch" do + it "fetches the bottle JSON for a formula that exists" do + mock_curl_output stdout: bottle_json + fetched_hash = described_class.fetch("foo") + expect(fetched_hash).to eq bottle_hash + end + + it "raises an error if the formula does not exist" do + mock_curl_output success: false + expect { described_class.fetch("bar") }.to raise_error(ArgumentError, /No JSON file found/) + end + + it "raises an error if the bottle JSON is invalid" do + mock_curl_output stdout: "foo" + expect { described_class.fetch("baz") }.to raise_error(ArgumentError, /Invalid JSON file/) + end + end + + describe "::latest_pkg_version" do + it "returns the expected `PkgVersion` when the revision is 0" do + mock_curl_output stdout: versions_json + pkg_version = described_class.latest_pkg_version("foo") + expect(pkg_version.to_s).to eq "1.2.3" + end + + it "returns the expected `PkgVersion` when the revision is not 0" do + mock_curl_output stdout: versions_json + pkg_version = described_class.latest_pkg_version("bar") + expect(pkg_version.to_s).to eq "1.2_4" + end + + it "returns `nil` when the formula is not in the JSON file" do + mock_curl_output stdout: versions_json + pkg_version = described_class.latest_pkg_version("baz") + expect(pkg_version).to be_nil + end + end + + describe "::bottle_available?" do + it "returns `true` if `fetch` succeeds" do + allow(described_class).to receive(:fetch) + expect(described_class.bottle_available?("foo")).to eq true + end + + it "returns `false` if `fetch` fails" do + allow(described_class).to receive(:fetch).and_raise ArgumentError + expect(described_class.bottle_available?("foo")).to eq false + end + end + + describe "::fetch_bottles" do + before do + allow(described_class).to receive(:fetch).and_return bottle_hash + end + + it "fetches bottles if a bottle is available" do + allow(Utils::Bottles).to receive(:tag).and_return :arm64_big_sur + expect { described_class.fetch_bottles("hello") }.not_to raise_error + end + + it "raises an error if no bottle is available" do + allow(Utils::Bottles).to receive(:tag).and_return :catalina + expect { described_class.fetch_bottles("hello") }.to raise_error(SystemExit) + end + end + + describe "::checksum_from_url" do + let(:sha256) { "b3b083db0807ff92c6e289a298f378198354b7727fb9ba9f4d550b8e08f90a60" } + let(:url) { "https://ghcr.io/v2/homebrew/core/hello/blobs/sha256:#{sha256}" } + let(:non_ghp_url) { "https://formulae.brew.sh/api/formula/hello.json" } + + it "returns the `sha256` for a GitHub packages URL" do + expect(described_class.checksum_from_url(url)).to eq sha256 + end + + it "returns `nil` for a non-GitHub packages URL" do + expect(described_class.checksum_from_url(non_ghp_url)).to be_nil + end + end +end From 1e34b69cd4385cee62a4333fea37dc94b9f4fbe8 Mon Sep 17 00:00:00 2001 From: Rylan Polster Date: Mon, 12 Jul 2021 03:23:42 -0400 Subject: [PATCH 8/9] Prioritize installing from JSON if env var is set --- Library/Homebrew/cli/named_args.rb | 26 ++++++++++++-------------- Library/Homebrew/cmd/install.rb | 4 +--- Library/Homebrew/cmd/outdated.rb | 6 +++--- Library/Homebrew/cmd/reinstall.rb | 4 ++-- Library/Homebrew/cmd/upgrade.rb | 4 ++-- Library/Homebrew/formula.rb | 2 +- 6 files changed, 21 insertions(+), 25 deletions(-) diff --git a/Library/Homebrew/cli/named_args.rb b/Library/Homebrew/cli/named_args.rb index 4a8fbbd847..cd49751f48 100644 --- a/Library/Homebrew/cli/named_args.rb +++ b/Library/Homebrew/cli/named_args.rb @@ -45,18 +45,18 @@ module Homebrew # the formula and prints a warning unless `only` is specified. sig { params( - only: T.nilable(Symbol), - ignore_unavailable: T.nilable(T::Boolean), - method: T.nilable(Symbol), - uniq: T::Boolean, - load_from_json: T::Boolean, + only: T.nilable(Symbol), + ignore_unavailable: T.nilable(T::Boolean), + method: T.nilable(Symbol), + uniq: T::Boolean, + prefer_loading_from_json: T::Boolean, ).returns(T::Array[T.any(Formula, Keg, Cask::Cask)]) } def to_formulae_and_casks(only: parent&.only_formula_or_cask, ignore_unavailable: nil, method: nil, uniq: true, - load_from_json: false) + prefer_loading_from_json: false) @to_formulae_and_casks ||= {} @to_formulae_and_casks[only] ||= downcased_unique_named.flat_map do |name| - load_formula_or_cask(name, only: only, method: method, load_from_json: load_from_json) + load_formula_or_cask(name, only: only, method: method, prefer_loading_from_json: prefer_loading_from_json) rescue FormulaUnreadableError, FormulaClassUnavailableError, TapFormulaUnreadableError, TapFormulaClassUnavailableError, Cask::CaskUnreadableError @@ -90,10 +90,14 @@ module Homebrew end.uniq.freeze end - def load_formula_or_cask(name, only: nil, method: nil, load_from_json: false) + def load_formula_or_cask(name, only: nil, method: nil, prefer_loading_from_json: false) unreadable_error = nil if only != :cask + if prefer_loading_from_json && ENV["HOMEBREW_JSON_CORE"].present? && BottleAPI.bottle_available?(name) + BottleAPI.fetch_bottles(name) + end + begin formula = case method when nil, :factory @@ -123,12 +127,6 @@ module Homebrew # The formula was found, but there's a problem with its implementation unreadable_error ||= e rescue NoSuchKegError, FormulaUnavailableError => e - if load_from_json && ENV["HOMEBREW_JSON_CORE"].present? && !CoreTap.instance.installed? && - BottleAPI.bottle_available?(name) - BottleAPI.fetch_bottles(name) - retry - end - raise e if only == :formula end end diff --git a/Library/Homebrew/cmd/install.rb b/Library/Homebrew/cmd/install.rb index 3582770b30..e3fd35a1b1 100644 --- a/Library/Homebrew/cmd/install.rb +++ b/Library/Homebrew/cmd/install.rb @@ -154,10 +154,8 @@ module Homebrew EOS end - allow_loading_from_json = ENV["HOMEBREW_JSON_CORE"].present? && !CoreTap.instance.installed? - begin - formulae, casks = args.named.to_formulae_and_casks(load_from_json: allow_loading_from_json) + formulae, casks = args.named.to_formulae_and_casks(prefer_loading_from_json: true) .partition { |formula_or_cask| formula_or_cask.is_a?(Formula) } rescue FormulaOrCaskUnavailableError, Cask::CaskUnavailableError => e retry if Tap.install_default_cask_tap_if_necessary(force: args.cask?) diff --git a/Library/Homebrew/cmd/outdated.rb b/Library/Homebrew/cmd/outdated.rb index f2079fae56..af7fb7a915 100644 --- a/Library/Homebrew/cmd/outdated.rb +++ b/Library/Homebrew/cmd/outdated.rb @@ -92,14 +92,14 @@ module Homebrew if verbose? outdated_kegs = f.outdated_kegs(fetch_head: args.fetch_HEAD?) - current_version = if f.alias_changed? && !f.latest_formula.latest_version_installed? + current_version = if ENV["HOMEBREW_JSON_CORE"].present? && (f.core_formula? || f.tap.blank?) + BottleAPI.latest_pkg_version(f.name)&.to_s || f.pkg_version.to_s + elsif f.alias_changed? && !f.latest_formula.latest_version_installed? latest = f.latest_formula "#{latest.name} (#{latest.pkg_version})" elsif f.head? && outdated_kegs.any? { |k| k.version.to_s == f.pkg_version.to_s } # There is a newer HEAD but the version number has not changed. "latest HEAD" - elsif f.tap.blank? && ENV["HOMEBREW_JSON_CORE"].present? - BottleAPI.latest_pkg_version(f.name)&.to_s || f.pkg_version.to_s else f.pkg_version.to_s end diff --git a/Library/Homebrew/cmd/reinstall.rb b/Library/Homebrew/cmd/reinstall.rb index bd64922c22..f1dfac9caf 100644 --- a/Library/Homebrew/cmd/reinstall.rb +++ b/Library/Homebrew/cmd/reinstall.rb @@ -85,11 +85,11 @@ module Homebrew def reinstall args = reinstall_args.parse - if ENV["HOMEBREW_JSON_CORE"].present? && !CoreTap.instance.installed? + if ENV["HOMEBREW_JSON_CORE"].present? args.named.each do |name| formula = Formulary.factory(name) next unless formula.any_version_installed? - next if formula.tap.present? + next if formula.tap.present? && !formula.core_formula? next unless BottleAPI.bottle_available?(name) BottleAPI.fetch_bottles(name) diff --git a/Library/Homebrew/cmd/upgrade.rb b/Library/Homebrew/cmd/upgrade.rb index 3e8ff5c935..9c05976a51 100644 --- a/Library/Homebrew/cmd/upgrade.rb +++ b/Library/Homebrew/cmd/upgrade.rb @@ -160,9 +160,9 @@ module Homebrew puts pinned.map { |f| "#{f.full_specified_name} #{f.pkg_version}" } * ", " end - if ENV["HOMEBREW_JSON_CORE"].present? && !CoreTap.instance.installed? + if ENV["HOMEBREW_JSON_CORE"].present? formulae_to_install.map! do |formula| - next formula if formula.tap.present? + next formula if formula.tap.present? && !formula.core_formula? next formula unless BottleAPI.bottle_available?(formula.name) BottleAPI.fetch_bottles(formula.name) diff --git a/Library/Homebrew/formula.rb b/Library/Homebrew/formula.rb index ec14f8d450..370551988b 100644 --- a/Library/Homebrew/formula.rb +++ b/Library/Homebrew/formula.rb @@ -1326,7 +1326,7 @@ class Formula Formula.cache[:outdated_kegs][cache_key] ||= begin all_kegs = [] current_version = T.let(false, T::Boolean) - latest_version = if tap.blank? && ENV["HOMEBREW_JSON_CORE"].present? + latest_version = if ENV["HOMEBREW_JSON_CORE"].present? && (core_formula? || tap.blank?) BottleAPI.latest_pkg_version(name) || pkg_version else pkg_version From 1973e72015b25cd79d9e887d6d9284391d2a6366 Mon Sep 17 00:00:00 2001 From: Rylan Polster Date: Tue, 13 Jul 2021 02:29:03 -0400 Subject: [PATCH 9/9] BottleAPI: fix tests --- Library/Homebrew/test/bottle_api_spec.rb | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Library/Homebrew/test/bottle_api_spec.rb b/Library/Homebrew/test/bottle_api_spec.rb index 9be228e05a..556e24dcc5 100644 --- a/Library/Homebrew/test/bottle_api_spec.rb +++ b/Library/Homebrew/test/bottle_api_spec.rb @@ -2,6 +2,10 @@ # frozen_string_literal: true describe BottleAPI do + before do + ENV["HOMEBREW_JSON_CORE"] = "1" + end + let(:bottle_json) { <<~EOS {