From 22f986b89a95184126c0dca3de70fcbfc3af0f2f Mon Sep 17 00:00:00 2001 From: Rylan Polster Date: Sat, 3 Jul 2021 13:28:56 -0400 Subject: [PATCH] Install formulae from JSON files --- Library/Homebrew/cli/named_args.rb | 14 +++- Library/Homebrew/cmd/install.rb | 4 +- Library/Homebrew/cmd/outdated.rb | 4 +- Library/Homebrew/cmd/reinstall.rb | 13 ++++ Library/Homebrew/cmd/upgrade.rb | 14 ++++ Library/Homebrew/formula.rb | 9 ++- Library/Homebrew/utils.rb | 1 + Library/Homebrew/utils/bottle_api.rb | 103 ++++++++++++++++++++++++++ Library/Homebrew/utils/bottle_api.rbi | 7 ++ manpages/brew.1 | 2 +- 10 files changed, 163 insertions(+), 8 deletions(-) create mode 100644 Library/Homebrew/utils/bottle_api.rb create mode 100644 Library/Homebrew/utils/bottle_api.rbi diff --git a/Library/Homebrew/cli/named_args.rb b/Library/Homebrew/cli/named_args.rb index f99cea17d7..71ff3306c4 100644 --- a/Library/Homebrew/cli/named_args.rb +++ b/Library/Homebrew/cli/named_args.rb @@ -49,12 +49,14 @@ module Homebrew ignore_unavailable: T.nilable(T::Boolean), method: T.nilable(Symbol), uniq: T::Boolean, + load_from_json: T::Boolean, ).returns(T::Array[T.any(Formula, Keg, Cask::Cask)]) } - def to_formulae_and_casks(only: parent&.only_formula_or_cask, ignore_unavailable: nil, method: nil, uniq: true) + def to_formulae_and_casks(only: parent&.only_formula_or_cask, ignore_unavailable: nil, method: nil, uniq: true, + load_from_json: false) @to_formulae_and_casks ||= {} @to_formulae_and_casks[only] ||= downcased_unique_named.flat_map do |name| - load_formula_or_cask(name, only: only, method: method) + load_formula_or_cask(name, only: only, method: method, load_from_json: load_from_json) rescue FormulaUnreadableError, FormulaClassUnavailableError, TapFormulaUnreadableError, TapFormulaClassUnavailableError, Cask::CaskUnreadableError @@ -88,7 +90,7 @@ module Homebrew end.uniq.freeze end - def load_formula_or_cask(name, only: nil, method: nil) + def load_formula_or_cask(name, only: nil, method: nil, load_from_json: false) unreadable_error = nil if only != :cask @@ -121,6 +123,12 @@ module Homebrew # The formula was found, but there's a problem with its implementation unreadable_error ||= e rescue NoSuchKegError, FormulaUnavailableError => e + if load_from_json && ENV["HOMEBREW_JSON_CORE"].present? && !CoreTap.instance.installed? && + Utils::BottleAPI.bottle_available?(name) + Utils::BottleAPI.download_bottles(name) + retry + end + raise e if only == :formula end end diff --git a/Library/Homebrew/cmd/install.rb b/Library/Homebrew/cmd/install.rb index 276700e6bc..3582770b30 100644 --- a/Library/Homebrew/cmd/install.rb +++ b/Library/Homebrew/cmd/install.rb @@ -154,8 +154,10 @@ module Homebrew EOS end + allow_loading_from_json = ENV["HOMEBREW_JSON_CORE"].present? && !CoreTap.instance.installed? + begin - formulae, casks = args.named.to_formulae_and_casks + formulae, casks = args.named.to_formulae_and_casks(load_from_json: allow_loading_from_json) .partition { |formula_or_cask| formula_or_cask.is_a?(Formula) } rescue FormulaOrCaskUnavailableError, Cask::CaskUnavailableError => e retry if Tap.install_default_cask_tap_if_necessary(force: args.cask?) diff --git a/Library/Homebrew/cmd/outdated.rb b/Library/Homebrew/cmd/outdated.rb index fb5246be21..b888dcd8cc 100644 --- a/Library/Homebrew/cmd/outdated.rb +++ b/Library/Homebrew/cmd/outdated.rb @@ -97,8 +97,10 @@ module Homebrew elsif f.head? && outdated_kegs.any? { |k| k.version.to_s == f.pkg_version.to_s } # There is a newer HEAD but the version number has not changed. "latest HEAD" - else + elsif f.tap.present? f.pkg_version.to_s + else + Utils::BottleAPI.latest_pkg_version(f.name).to_s end outdated_versions = outdated_kegs.group_by { |keg| Formulary.from_keg(keg).full_name } diff --git a/Library/Homebrew/cmd/reinstall.rb b/Library/Homebrew/cmd/reinstall.rb index 062119eb18..1bfec15091 100644 --- a/Library/Homebrew/cmd/reinstall.rb +++ b/Library/Homebrew/cmd/reinstall.rb @@ -84,6 +84,19 @@ module Homebrew def reinstall args = reinstall_args.parse + if ENV["HOMEBREW_JSON_CORE"].present? && !CoreTap.instance.installed? + args.named.each do |name| + formula = Formulary.factory(name) + next unless formula.any_version_installed? + next if formula.tap.present? && !formula.tap.installed? + next unless Utils::BottleAPI.bottle_available?(name) + + Utils::BottleAPI.download_bottles(name) + rescue FormulaUnavailableError + next + end + end + formulae, casks = args.named.to_formulae_and_casks(method: :resolve) .partition { |o| o.is_a?(Formula) } diff --git a/Library/Homebrew/cmd/upgrade.rb b/Library/Homebrew/cmd/upgrade.rb index bd296a6f3d..8f5e8ab71e 100644 --- a/Library/Homebrew/cmd/upgrade.rb +++ b/Library/Homebrew/cmd/upgrade.rb @@ -159,6 +159,20 @@ module Homebrew puts pinned.map { |f| "#{f.full_specified_name} #{f.pkg_version}" } * ", " end + if ENV["HOMEBREW_JSON_CORE"].present? && !CoreTap.instance.installed? + formulae_to_install.map! do |formula| + next formula if formula.tap.present? && formula.tap.installed? + next formula unless Utils::BottleAPI.bottle_available?(formula.name) + + Utils::BottleAPI.download_bottles(formula.name) + Formulary.factory(formula.name) + rescue FormulaUnavailableError + formula + end + end + + opoo formulae_to_install + if formulae_to_install.empty? oh1 "No packages to upgrade" else diff --git a/Library/Homebrew/formula.rb b/Library/Homebrew/formula.rb index d7793cdfaf..db7e8faf43 100644 --- a/Library/Homebrew/formula.rb +++ b/Library/Homebrew/formula.rb @@ -1325,6 +1325,11 @@ class Formula Formula.cache[:outdated_kegs][cache_key] ||= begin all_kegs = [] current_version = T.let(false, T::Boolean) + latest_version = if tap.present? + pkg_version + else + Utils::BottleAPI.latest_pkg_version name + end installed_kegs.each do |keg| all_kegs << keg @@ -1332,8 +1337,8 @@ class Formula next if version.head? tab = Tab.for_keg(keg) - next if version_scheme > tab.version_scheme && pkg_version != version - next if version_scheme == tab.version_scheme && pkg_version > version + next if version_scheme > tab.version_scheme && latest_version != version + next if version_scheme == tab.version_scheme && latest_version > version # don't consider this keg current if there's a newer formula available next if follow_installed_alias? && new_formula_available? diff --git a/Library/Homebrew/utils.rb b/Library/Homebrew/utils.rb index 361cc285f7..5b4de1c9de 100644 --- a/Library/Homebrew/utils.rb +++ b/Library/Homebrew/utils.rb @@ -4,6 +4,7 @@ require "time" require "utils/analytics" +require "utils/bottle_api" require "utils/curl" require "utils/fork" require "utils/formatter" diff --git a/Library/Homebrew/utils/bottle_api.rb b/Library/Homebrew/utils/bottle_api.rb new file mode 100644 index 0000000000..0df65cba62 --- /dev/null +++ b/Library/Homebrew/utils/bottle_api.rb @@ -0,0 +1,103 @@ +# typed: true +# frozen_string_literal: true + +require "github_packages" + +module Utils + # Helper functions for using the Bottle JSON API. + # + # @api private + module BottleAPI + extend T::Sig + + module_function + + FORMULAE_BREW_SH_BOTTLE_API_DOMAIN = if OS.mac? + "https://formulae.brew.sh/api/bottle" + else + "https://formulae.brew.sh/api/bottle-linux" + end.freeze + + FORMULAE_BREW_SH_VERSIONS_API_URL = if OS.mac? + "https://formulae.brew.sh/api/versions.json" + else + "https://formulae.brew.sh/api/versions-linux.json" + end.freeze + + GITHUB_PACKAGES_SHA256_REGEX = %r{#{GitHubPackages::URL_REGEX}.*/blobs/sha256:(?\h{64})$}.freeze + + sig { params(name: String).returns(Hash) } + def fetch(name) + return @cache[name] if @cache.present? && @cache.key?(name) + + api_url = "#{FORMULAE_BREW_SH_BOTTLE_API_DOMAIN}/#{name}.json" + output = Utils::Curl.curl_output("--fail", api_url) + raise ArgumentError, "No JSON file found at #{Tty.underline}#{api_url}#{Tty.reset}" unless output.success? + + @cache ||= {} + @cache[name] = JSON.parse(output.stdout) + rescue JSON::ParserError + raise ArgumentError, "Invalid JSON file: #{Tty.underline}#{api_url}#{Tty.reset}" + end + + sig { params(name: String).returns(PkgVersion) } + def latest_pkg_version(name) + @formula_versions ||= begin + output = Utils::Curl.curl_output("--fail", FORMULAE_BREW_SH_VERSIONS_API_URL) + JSON.parse(output.stdout) + end + PkgVersion.new(@formula_versions[name]["version"], @formula_versions[name]["revision"]) + end + + sig { params(name: String).returns(T::Boolean) } + def bottle_available?(name) + fetch name + true + rescue ArgumentError + false + end + + sig { params(name: String).void } + def download_bottles(name) + hash = fetch(name) + bottle_tag = Utils::Bottles.tag.to_s + + odie "No bottle availabe for current OS" unless hash["bottles"].key? bottle_tag + + download_bottle(hash, bottle_tag) + + hash["dependencies"].each do |dep_hash| + download_bottle(dep_hash, bottle_tag) + end + end + + sig { params(url: String).returns(T.nilable(String)) } + def checksum_from_url(url) + match = url.match GITHUB_PACKAGES_SHA256_REGEX + return if match.blank? + + match[:sha256] + end + + sig { params(hash: Hash, tag: Symbol).void } + def download_bottle(hash, tag) + bottle = hash["bottles"][tag] + return if bottle.blank? + + sha256 = bottle["sha256"] || checksum_from_url(bottle["url"]) + bottle_filename = Bottle::Filename.new(hash["name"], hash["pkg_version"], tag, hash["rebuild"]) + + resource = Resource.new hash["name"] + resource.url bottle["url"] + resource.sha256 sha256 + resource.version hash["pkg_version"] + resource.downloader.resolved_basename = bottle_filename + + resource.fetch + + # Map the name of this formula to the local bottle path to allow the + # formula to be loaded by passing just the name to `Formulary::factory`. + Formulary.map_formula_name_to_local_bottle_path hash["name"], resource.downloader.cached_location + end + end +end diff --git a/Library/Homebrew/utils/bottle_api.rbi b/Library/Homebrew/utils/bottle_api.rbi new file mode 100644 index 0000000000..403c511f59 --- /dev/null +++ b/Library/Homebrew/utils/bottle_api.rbi @@ -0,0 +1,7 @@ +# typed: strict + +module Utils + module BottleAPI + include Kernel + end +end diff --git a/manpages/brew.1 b/manpages/brew.1 index 20d2deda05..267aedf461 100644 --- a/manpages/brew.1 +++ b/manpages/brew.1 @@ -1,7 +1,7 @@ .\" generated with Ronn/v0.7.3 .\" http://github.com/rtomayko/ronn/tree/0.7.3 . -.TH "BREW" "1" "June 2021" "Homebrew" "brew" +.TH "BREW" "1" "July 2021" "Homebrew" "brew" . .SH "NAME" \fBbrew\fR \- The Missing Package Manager for macOS (or Linux)