diff --git a/Library/Homebrew/bottle_api.rb b/Library/Homebrew/bottle_api.rb new file mode 100644 index 0000000000..214e62a73d --- /dev/null +++ b/Library/Homebrew/bottle_api.rb @@ -0,0 +1,106 @@ +# typed: true +# frozen_string_literal: true + +require "github_packages" + +# Helper functions for using the Bottle JSON API. +# +# @api private +module BottleAPI + extend T::Sig + + module_function + + FORMULAE_BREW_SH_BOTTLE_API_DOMAIN = if OS.mac? + "https://formulae.brew.sh/api/bottle" + else + "https://formulae.brew.sh/api/bottle-linux" + end.freeze + + FORMULAE_BREW_SH_VERSIONS_API_URL = if OS.mac? + "https://formulae.brew.sh/api/versions-formulae.json" + else + "https://formulae.brew.sh/api/versions-linux.json" + end.freeze + + GITHUB_PACKAGES_SHA256_REGEX = %r{#{GitHubPackages::URL_REGEX}.*/blobs/sha256:(?\h{64})$}.freeze + + sig { params(name: String).returns(Hash) } + def fetch(name) + return @cache[name] if @cache.present? && @cache.key?(name) + + api_url = "#{FORMULAE_BREW_SH_BOTTLE_API_DOMAIN}/#{name}.json" + output = Utils::Curl.curl_output("--fail", api_url) + raise ArgumentError, "No JSON file found at #{Tty.underline}#{api_url}#{Tty.reset}" unless output.success? + + @cache ||= {} + @cache[name] = JSON.parse(output.stdout) + rescue JSON::ParserError + raise ArgumentError, "Invalid JSON file: #{Tty.underline}#{api_url}#{Tty.reset}" + end + + sig { params(name: String).returns(T.nilable(PkgVersion)) } + def latest_pkg_version(name) + @formula_versions ||= begin + output = Utils::Curl.curl_output("--fail", FORMULAE_BREW_SH_VERSIONS_API_URL) + JSON.parse(output.stdout) + end + + return unless @formula_versions.key? name + + version = Version.new(@formula_versions[name]["version"]) + revision = @formula_versions[name]["revision"] + PkgVersion.new(version, revision) + end + + sig { params(name: String).returns(T::Boolean) } + def bottle_available?(name) + fetch name + true + rescue ArgumentError + false + end + + sig { params(name: String).void } + def fetch_bottles(name) + hash = fetch(name) + bottle_tag = Utils::Bottles.tag.to_s + + odie "No bottle available for current OS" unless hash["bottles"].key? bottle_tag + + download_bottle(hash, bottle_tag) + + hash["dependencies"].each do |dep_hash| + download_bottle(dep_hash, bottle_tag) + end + end + + sig { params(url: String).returns(T.nilable(String)) } + def checksum_from_url(url) + match = url.match GITHUB_PACKAGES_SHA256_REGEX + return if match.blank? + + match[:sha256] + end + + sig { params(hash: Hash, tag: Symbol).void } + def download_bottle(hash, tag) + bottle = hash["bottles"][tag] + return if bottle.blank? + + sha256 = bottle["sha256"] || checksum_from_url(bottle["url"]) + bottle_filename = Bottle::Filename.new(hash["name"], hash["pkg_version"], tag, hash["rebuild"]) + + resource = Resource.new hash["name"] + resource.url bottle["url"] + resource.sha256 sha256 + resource.version hash["pkg_version"] + resource.downloader.resolved_basename = bottle_filename + + resource.fetch + + # Map the name of this formula to the local bottle path to allow the + # formula to be loaded by passing just the name to `Formulary::factory`. + Formulary.map_formula_name_to_local_bottle_path hash["name"], resource.downloader.cached_location + end +end diff --git a/Library/Homebrew/bottle_api.rbi b/Library/Homebrew/bottle_api.rbi new file mode 100644 index 0000000000..6fefceb732 --- /dev/null +++ b/Library/Homebrew/bottle_api.rbi @@ -0,0 +1,5 @@ +# typed: strict + +module BottleAPI + include Kernel +end diff --git a/Library/Homebrew/cli/named_args.rb b/Library/Homebrew/cli/named_args.rb index f99cea17d7..cd49751f48 100644 --- a/Library/Homebrew/cli/named_args.rb +++ b/Library/Homebrew/cli/named_args.rb @@ -2,7 +2,7 @@ # frozen_string_literal: true require "delegate" - +require "bottle_api" require "cli/args" module Homebrew @@ -45,16 +45,18 @@ module Homebrew # the formula and prints a warning unless `only` is specified. sig { params( - only: T.nilable(Symbol), - ignore_unavailable: T.nilable(T::Boolean), - method: T.nilable(Symbol), - uniq: T::Boolean, + only: T.nilable(Symbol), + ignore_unavailable: T.nilable(T::Boolean), + method: T.nilable(Symbol), + uniq: T::Boolean, + prefer_loading_from_json: T::Boolean, ).returns(T::Array[T.any(Formula, Keg, Cask::Cask)]) } - def to_formulae_and_casks(only: parent&.only_formula_or_cask, ignore_unavailable: nil, method: nil, uniq: true) + def to_formulae_and_casks(only: parent&.only_formula_or_cask, ignore_unavailable: nil, method: nil, uniq: true, + prefer_loading_from_json: false) @to_formulae_and_casks ||= {} @to_formulae_and_casks[only] ||= downcased_unique_named.flat_map do |name| - load_formula_or_cask(name, only: only, method: method) + load_formula_or_cask(name, only: only, method: method, prefer_loading_from_json: prefer_loading_from_json) rescue FormulaUnreadableError, FormulaClassUnavailableError, TapFormulaUnreadableError, TapFormulaClassUnavailableError, Cask::CaskUnreadableError @@ -88,10 +90,14 @@ module Homebrew end.uniq.freeze end - def load_formula_or_cask(name, only: nil, method: nil) + def load_formula_or_cask(name, only: nil, method: nil, prefer_loading_from_json: false) unreadable_error = nil if only != :cask + if prefer_loading_from_json && ENV["HOMEBREW_JSON_CORE"].present? && BottleAPI.bottle_available?(name) + BottleAPI.fetch_bottles(name) + end + begin formula = case method when nil, :factory diff --git a/Library/Homebrew/cmd/install.rb b/Library/Homebrew/cmd/install.rb index 4067b981ae..e04ced8c99 100644 --- a/Library/Homebrew/cmd/install.rb +++ b/Library/Homebrew/cmd/install.rb @@ -155,7 +155,7 @@ module Homebrew end begin - formulae, casks = args.named.to_formulae_and_casks + formulae, casks = args.named.to_formulae_and_casks(prefer_loading_from_json: true) .partition { |formula_or_cask| formula_or_cask.is_a?(Formula) } rescue FormulaOrCaskUnavailableError, Cask::CaskUnavailableError => e retry if Tap.install_default_cask_tap_if_necessary(force: args.cask?) diff --git a/Library/Homebrew/cmd/outdated.rb b/Library/Homebrew/cmd/outdated.rb index fb5246be21..af7fb7a915 100644 --- a/Library/Homebrew/cmd/outdated.rb +++ b/Library/Homebrew/cmd/outdated.rb @@ -6,6 +6,7 @@ require "keg" require "cli/parser" require "cask/cmd" require "cask/caskroom" +require "bottle_api" module Homebrew extend T::Sig @@ -91,7 +92,9 @@ module Homebrew if verbose? outdated_kegs = f.outdated_kegs(fetch_head: args.fetch_HEAD?) - current_version = if f.alias_changed? && !f.latest_formula.latest_version_installed? + current_version = if ENV["HOMEBREW_JSON_CORE"].present? && (f.core_formula? || f.tap.blank?) + BottleAPI.latest_pkg_version(f.name)&.to_s || f.pkg_version.to_s + elsif f.alias_changed? && !f.latest_formula.latest_version_installed? latest = f.latest_formula "#{latest.name} (#{latest.pkg_version})" elsif f.head? && outdated_kegs.any? { |k| k.version.to_s == f.pkg_version.to_s } diff --git a/Library/Homebrew/cmd/reinstall.rb b/Library/Homebrew/cmd/reinstall.rb index 062119eb18..f1dfac9caf 100644 --- a/Library/Homebrew/cmd/reinstall.rb +++ b/Library/Homebrew/cmd/reinstall.rb @@ -12,6 +12,7 @@ require "cask/cmd" require "cask/utils" require "cask/macos" require "upgrade" +require "bottle_api" module Homebrew extend T::Sig @@ -84,6 +85,19 @@ module Homebrew def reinstall args = reinstall_args.parse + if ENV["HOMEBREW_JSON_CORE"].present? + args.named.each do |name| + formula = Formulary.factory(name) + next unless formula.any_version_installed? + next if formula.tap.present? && !formula.core_formula? + next unless BottleAPI.bottle_available?(name) + + BottleAPI.fetch_bottles(name) + rescue FormulaUnavailableError + next + end + end + formulae, casks = args.named.to_formulae_and_casks(method: :resolve) .partition { |o| o.is_a?(Formula) } diff --git a/Library/Homebrew/cmd/upgrade.rb b/Library/Homebrew/cmd/upgrade.rb index bd296a6f3d..9c05976a51 100644 --- a/Library/Homebrew/cmd/upgrade.rb +++ b/Library/Homebrew/cmd/upgrade.rb @@ -8,6 +8,7 @@ require "upgrade" require "cask/cmd" require "cask/utils" require "cask/macos" +require "bottle_api" module Homebrew extend T::Sig @@ -159,6 +160,18 @@ module Homebrew puts pinned.map { |f| "#{f.full_specified_name} #{f.pkg_version}" } * ", " end + if ENV["HOMEBREW_JSON_CORE"].present? + formulae_to_install.map! do |formula| + next formula if formula.tap.present? && !formula.core_formula? + next formula unless BottleAPI.bottle_available?(formula.name) + + BottleAPI.fetch_bottles(formula.name) + Formulary.factory(formula.name) + rescue FormulaUnavailableError + formula + end + end + if formulae_to_install.empty? oh1 "No packages to upgrade" else diff --git a/Library/Homebrew/formula.rb b/Library/Homebrew/formula.rb index 4dd7cfd2f1..ade304980e 100644 --- a/Library/Homebrew/formula.rb +++ b/Library/Homebrew/formula.rb @@ -29,6 +29,7 @@ require "mktemp" require "find" require "utils/spdx" require "extend/on_os" +require "bottle_api" # A formula provides instructions and metadata for Homebrew to install a piece # of software. Every Homebrew formula is a {Formula}. @@ -1325,6 +1326,11 @@ class Formula Formula.cache[:outdated_kegs][cache_key] ||= begin all_kegs = [] current_version = T.let(false, T::Boolean) + latest_version = if ENV["HOMEBREW_JSON_CORE"].present? && (core_formula? || tap.blank?) + BottleAPI.latest_pkg_version(name) || pkg_version + else + pkg_version + end installed_kegs.each do |keg| all_kegs << keg @@ -1332,8 +1338,8 @@ class Formula next if version.head? tab = Tab.for_keg(keg) - next if version_scheme > tab.version_scheme && pkg_version != version - next if version_scheme == tab.version_scheme && pkg_version > version + next if version_scheme > tab.version_scheme && latest_version != version + next if version_scheme == tab.version_scheme && latest_version > version # don't consider this keg current if there's a newer formula available next if follow_installed_alias? && new_formula_available? diff --git a/Library/Homebrew/test/bottle_api_spec.rb b/Library/Homebrew/test/bottle_api_spec.rb new file mode 100644 index 0000000000..556e24dcc5 --- /dev/null +++ b/Library/Homebrew/test/bottle_api_spec.rb @@ -0,0 +1,124 @@ +# typed: false +# frozen_string_literal: true + +describe BottleAPI do + before do + ENV["HOMEBREW_JSON_CORE"] = "1" + end + + let(:bottle_json) { + <<~EOS + { + "name": "hello", + "pkg_version": "2.10", + "rebuild": 0, + "bottles": { + "arm64_big_sur": { + "url": "https://ghcr.io/v2/homebrew/core/hello/blobs/sha256:b3b083db0807ff92c6e289a298f378198354b7727fb9ba9f4d550b8e08f90a60" + }, + "big_sur": { + "url": "https://ghcr.io/v2/homebrew/core/hello/blobs/sha256:69489ae397e4645127aa7773211310f81ebb6c99e1f8e3e22c5cdb55333f5408" + }, + "x86_64_linux": { + "url": "https://ghcr.io/v2/homebrew/core/hello/blobs/sha256:e6980196298e0a9cfe4fa4e328a71a1869a4d5e1d31c38442150ed784cfc0e29" + } + }, + "dependencies": [] + } + EOS + } + let(:bottle_hash) { JSON.parse(bottle_json) } + let(:versions_json) { + <<~EOS + { + "foo":{"version":"1.2.3","revision":0}, + "bar":{"version":"1.2","revision":4} + } + EOS + } + + def mock_curl_output(stdout: "", success: true) + curl_output = OpenStruct.new(stdout: stdout, success?: success) + allow(Utils::Curl).to receive(:curl_output).and_return curl_output + end + + describe "::fetch" do + it "fetches the bottle JSON for a formula that exists" do + mock_curl_output stdout: bottle_json + fetched_hash = described_class.fetch("foo") + expect(fetched_hash).to eq bottle_hash + end + + it "raises an error if the formula does not exist" do + mock_curl_output success: false + expect { described_class.fetch("bar") }.to raise_error(ArgumentError, /No JSON file found/) + end + + it "raises an error if the bottle JSON is invalid" do + mock_curl_output stdout: "foo" + expect { described_class.fetch("baz") }.to raise_error(ArgumentError, /Invalid JSON file/) + end + end + + describe "::latest_pkg_version" do + it "returns the expected `PkgVersion` when the revision is 0" do + mock_curl_output stdout: versions_json + pkg_version = described_class.latest_pkg_version("foo") + expect(pkg_version.to_s).to eq "1.2.3" + end + + it "returns the expected `PkgVersion` when the revision is not 0" do + mock_curl_output stdout: versions_json + pkg_version = described_class.latest_pkg_version("bar") + expect(pkg_version.to_s).to eq "1.2_4" + end + + it "returns `nil` when the formula is not in the JSON file" do + mock_curl_output stdout: versions_json + pkg_version = described_class.latest_pkg_version("baz") + expect(pkg_version).to be_nil + end + end + + describe "::bottle_available?" do + it "returns `true` if `fetch` succeeds" do + allow(described_class).to receive(:fetch) + expect(described_class.bottle_available?("foo")).to eq true + end + + it "returns `false` if `fetch` fails" do + allow(described_class).to receive(:fetch).and_raise ArgumentError + expect(described_class.bottle_available?("foo")).to eq false + end + end + + describe "::fetch_bottles" do + before do + allow(described_class).to receive(:fetch).and_return bottle_hash + end + + it "fetches bottles if a bottle is available" do + allow(Utils::Bottles).to receive(:tag).and_return :arm64_big_sur + expect { described_class.fetch_bottles("hello") }.not_to raise_error + end + + it "raises an error if no bottle is available" do + allow(Utils::Bottles).to receive(:tag).and_return :catalina + expect { described_class.fetch_bottles("hello") }.to raise_error(SystemExit) + end + end + + describe "::checksum_from_url" do + let(:sha256) { "b3b083db0807ff92c6e289a298f378198354b7727fb9ba9f4d550b8e08f90a60" } + let(:url) { "https://ghcr.io/v2/homebrew/core/hello/blobs/sha256:#{sha256}" } + let(:non_ghp_url) { "https://formulae.brew.sh/api/formula/hello.json" } + + it "returns the `sha256` for a GitHub packages URL" do + expect(described_class.checksum_from_url(url)).to eq sha256 + end + + it "returns `nil` for a non-GitHub packages URL" do + expect(described_class.checksum_from_url(non_ghp_url)).to be_nil + end + end +end diff --git a/manpages/brew.1 b/manpages/brew.1 index 00b783f854..1b95231bc3 100644 --- a/manpages/brew.1 +++ b/manpages/brew.1 @@ -1,7 +1,7 @@ .\" generated with Ronn/v0.7.3 .\" http://github.com/rtomayko/ronn/tree/0.7.3 . -.TH "BREW" "1" "June 2021" "Homebrew" "brew" +.TH "BREW" "1" "July 2021" "Homebrew" "brew" . .SH "NAME" \fBbrew\fR \- The Missing Package Manager for macOS (or Linux)