Merge pull request #11648 from Rylan12/homebrew-json

Install formulae from JSON files

Library/Homebrew/bottle_api.rb

@@ -0,0 +1,106 @@
+# typed: true
+# frozen_string_literal: true
+
+require "github_packages"
+
+# Helper functions for using the Bottle JSON API.
+#
+# @api private
+module BottleAPI
+  extend T::Sig
+
+  module_function
+
+  FORMULAE_BREW_SH_BOTTLE_API_DOMAIN = if OS.mac?
+    "https://formulae.brew.sh/api/bottle"
+  else
+    "https://formulae.brew.sh/api/bottle-linux"
+  end.freeze
+
+  FORMULAE_BREW_SH_VERSIONS_API_URL = if OS.mac?
+    "https://formulae.brew.sh/api/versions-formulae.json"
+  else
+    "https://formulae.brew.sh/api/versions-linux.json"
+  end.freeze
+
+  GITHUB_PACKAGES_SHA256_REGEX = %r{#{GitHubPackages::URL_REGEX}.*/blobs/sha256:(?<sha256>\h{64})$}.freeze
+
+  sig { params(name: String).returns(Hash) }
+  def fetch(name)
+    return @cache[name] if @cache.present? && @cache.key?(name)
+
+    api_url = "#{FORMULAE_BREW_SH_BOTTLE_API_DOMAIN}/#{name}.json"
+    output = Utils::Curl.curl_output("--fail", api_url)
+    raise ArgumentError, "No JSON file found at #{Tty.underline}#{api_url}#{Tty.reset}" unless output.success?
+
+    @cache ||= {}
+    @cache[name] = JSON.parse(output.stdout)
+  rescue JSON::ParserError
+    raise ArgumentError, "Invalid JSON file: #{Tty.underline}#{api_url}#{Tty.reset}"
+  end
+
+  sig { params(name: String).returns(T.nilable(PkgVersion)) }
+  def latest_pkg_version(name)
+    @formula_versions ||= begin
+      output = Utils::Curl.curl_output("--fail", FORMULAE_BREW_SH_VERSIONS_API_URL)
+      JSON.parse(output.stdout)
+    end
+
+    return unless @formula_versions.key? name
+
+    version = Version.new(@formula_versions[name]["version"])
+    revision = @formula_versions[name]["revision"]
+    PkgVersion.new(version, revision)
+  end
+
+  sig { params(name: String).returns(T::Boolean) }
+  def bottle_available?(name)
+    fetch name
+    true
+  rescue ArgumentError
+    false
+  end
+
+  sig { params(name: String).void }
+  def fetch_bottles(name)
+    hash = fetch(name)
+    bottle_tag = Utils::Bottles.tag.to_s
+
+    odie "No bottle available for current OS" unless hash["bottles"].key? bottle_tag
+
+    download_bottle(hash, bottle_tag)
+
+    hash["dependencies"].each do |dep_hash|
+      download_bottle(dep_hash, bottle_tag)
+    end
+  end
+
+  sig { params(url: String).returns(T.nilable(String)) }
+  def checksum_from_url(url)
+    match = url.match GITHUB_PACKAGES_SHA256_REGEX
+    return if match.blank?
+
+    match[:sha256]
+  end
+
+  sig { params(hash: Hash, tag: Symbol).void }
+  def download_bottle(hash, tag)
+    bottle = hash["bottles"][tag]
+    return if bottle.blank?
+
+    sha256 = bottle["sha256"] || checksum_from_url(bottle["url"])
+    bottle_filename = Bottle::Filename.new(hash["name"], hash["pkg_version"], tag, hash["rebuild"])
+
+    resource = Resource.new hash["name"]
+    resource.url bottle["url"]
+    resource.sha256 sha256
+    resource.version hash["pkg_version"]
+    resource.downloader.resolved_basename = bottle_filename
+
+    resource.fetch
+
+    # Map the name of this formula to the local bottle path to allow the
+    # formula to be loaded by passing just the name to `Formulary::factory`.
+    Formulary.map_formula_name_to_local_bottle_path hash["name"], resource.downloader.cached_location
+  end
+end

Library/Homebrew/bottle_api.rbi

@@ -0,0 +1,5 @@
+# typed: strict
+
+module BottleAPI
+  include Kernel
+end

Library/Homebrew/cli/named_args.rb

@@ -2,7 +2,7 @@
 # frozen_string_literal: true
 
 require "delegate"
-
+require "bottle_api"
 require "cli/args"
 
 module Homebrew
@@ -45,16 +45,18 @@ module Homebrew
       # the formula and prints a warning unless `only` is specified.
       sig {
         params(
-          only:               T.nilable(Symbol),
+          only:                     T.nilable(Symbol),
-          ignore_unavailable: T.nilable(T::Boolean),
+          ignore_unavailable:       T.nilable(T::Boolean),
-          method:             T.nilable(Symbol),
+          method:                   T.nilable(Symbol),
-          uniq:               T::Boolean,
+          uniq:                     T::Boolean,
+          prefer_loading_from_json: T::Boolean,
         ).returns(T::Array[T.any(Formula, Keg, Cask::Cask)])
       }
-      def to_formulae_and_casks(only: parent&.only_formula_or_cask, ignore_unavailable: nil, method: nil, uniq: true)
+      def to_formulae_and_casks(only: parent&.only_formula_or_cask, ignore_unavailable: nil, method: nil, uniq: true,
+                                prefer_loading_from_json: false)
         @to_formulae_and_casks ||= {}
         @to_formulae_and_casks[only] ||= downcased_unique_named.flat_map do |name|
-          load_formula_or_cask(name, only: only, method: method)
+          load_formula_or_cask(name, only: only, method: method, prefer_loading_from_json: prefer_loading_from_json)
         rescue FormulaUnreadableError, FormulaClassUnavailableError,
                TapFormulaUnreadableError, TapFormulaClassUnavailableError,
                Cask::CaskUnreadableError
@@ -88,10 +90,14 @@ module Homebrew
         end.uniq.freeze
       end
 
-      def load_formula_or_cask(name, only: nil, method: nil)
+      def load_formula_or_cask(name, only: nil, method: nil, prefer_loading_from_json: false)
         unreadable_error = nil
 
         if only != :cask
+          if prefer_loading_from_json && ENV["HOMEBREW_JSON_CORE"].present? && BottleAPI.bottle_available?(name)
+            BottleAPI.fetch_bottles(name)
+          end
+
           begin
             formula = case method
             when nil, :factory

Library/Homebrew/cmd/install.rb

@@ -155,7 +155,7 @@ module Homebrew
     end
 
     begin
-      formulae, casks = args.named.to_formulae_and_casks
+      formulae, casks = args.named.to_formulae_and_casks(prefer_loading_from_json: true)
                             .partition { |formula_or_cask| formula_or_cask.is_a?(Formula) }
     rescue FormulaOrCaskUnavailableError, Cask::CaskUnavailableError => e
       retry if Tap.install_default_cask_tap_if_necessary(force: args.cask?)

Library/Homebrew/cmd/outdated.rb

@@ -6,6 +6,7 @@ require "keg"
 require "cli/parser"
 require "cask/cmd"
 require "cask/caskroom"
+require "bottle_api"
 
 module Homebrew
   extend T::Sig
@@ -91,7 +92,9 @@ module Homebrew
         if verbose?
           outdated_kegs = f.outdated_kegs(fetch_head: args.fetch_HEAD?)
 
-          current_version = if f.alias_changed? && !f.latest_formula.latest_version_installed?
+          current_version = if ENV["HOMEBREW_JSON_CORE"].present? && (f.core_formula? || f.tap.blank?)
+            BottleAPI.latest_pkg_version(f.name)&.to_s || f.pkg_version.to_s
+          elsif f.alias_changed? && !f.latest_formula.latest_version_installed?
             latest = f.latest_formula
             "#{latest.name} (#{latest.pkg_version})"
           elsif f.head? && outdated_kegs.any? { |k| k.version.to_s == f.pkg_version.to_s }

Library/Homebrew/cmd/reinstall.rb

@@ -12,6 +12,7 @@ require "cask/cmd"
 require "cask/utils"
 require "cask/macos"
 require "upgrade"
+require "bottle_api"
 
 module Homebrew
   extend T::Sig
@@ -84,6 +85,19 @@ module Homebrew
   def reinstall
     args = reinstall_args.parse
 
+    if ENV["HOMEBREW_JSON_CORE"].present?
+      args.named.each do |name|
+        formula = Formulary.factory(name)
+        next unless formula.any_version_installed?
+        next if formula.tap.present? && !formula.core_formula?
+        next unless BottleAPI.bottle_available?(name)
+
+        BottleAPI.fetch_bottles(name)
+      rescue FormulaUnavailableError
+        next
+      end
+    end
+
     formulae, casks = args.named.to_formulae_and_casks(method: :resolve)
                           .partition { |o| o.is_a?(Formula) }
 

Library/Homebrew/cmd/upgrade.rb

@@ -8,6 +8,7 @@ require "upgrade"
 require "cask/cmd"
 require "cask/utils"
 require "cask/macos"
+require "bottle_api"
 
 module Homebrew
   extend T::Sig
@@ -159,6 +160,18 @@ module Homebrew
       puts pinned.map { |f| "#{f.full_specified_name} #{f.pkg_version}" } * ", "
     end
 
+    if ENV["HOMEBREW_JSON_CORE"].present?
+      formulae_to_install.map! do |formula|
+        next formula if formula.tap.present? && !formula.core_formula?
+        next formula unless BottleAPI.bottle_available?(formula.name)
+
+        BottleAPI.fetch_bottles(formula.name)
+        Formulary.factory(formula.name)
+      rescue FormulaUnavailableError
+        formula
+      end
+    end
+
     if formulae_to_install.empty?
       oh1 "No packages to upgrade"
     else

Library/Homebrew/formula.rb

@@ -29,6 +29,7 @@ require "mktemp"
 require "find"
 require "utils/spdx"
 require "extend/on_os"
+require "bottle_api"
 
 # A formula provides instructions and metadata for Homebrew to install a piece
 # of software. Every Homebrew formula is a {Formula}.
@@ -1325,6 +1326,11 @@ class Formula
     Formula.cache[:outdated_kegs][cache_key] ||= begin
       all_kegs = []
       current_version = T.let(false, T::Boolean)
+      latest_version = if ENV["HOMEBREW_JSON_CORE"].present? && (core_formula? || tap.blank?)
+        BottleAPI.latest_pkg_version(name) || pkg_version
+      else
+        pkg_version
+      end
 
       installed_kegs.each do |keg|
         all_kegs << keg
@@ -1332,8 +1338,8 @@ class Formula
         next if version.head?
 
         tab = Tab.for_keg(keg)
-        next if version_scheme > tab.version_scheme && pkg_version != version
+        next if version_scheme > tab.version_scheme && latest_version != version
-        next if version_scheme == tab.version_scheme && pkg_version > version
+        next if version_scheme == tab.version_scheme && latest_version > version
 
         # don't consider this keg current if there's a newer formula available
         next if follow_installed_alias? && new_formula_available?

Library/Homebrew/test/bottle_api_spec.rb

@@ -0,0 +1,124 @@
+# typed: false
+# frozen_string_literal: true
+
+describe BottleAPI do
+  before do
+    ENV["HOMEBREW_JSON_CORE"] = "1"
+  end
+
+  let(:bottle_json) {
+    <<~EOS
+      {
+        "name": "hello",
+        "pkg_version": "2.10",
+        "rebuild": 0,
+        "bottles": {
+          "arm64_big_sur": {
+            "url": "https://ghcr.io/v2/homebrew/core/hello/blobs/sha256:b3b083db0807ff92c6e289a298f378198354b7727fb9ba9f4d550b8e08f90a60"
+          },
+          "big_sur": {
+            "url": "https://ghcr.io/v2/homebrew/core/hello/blobs/sha256:69489ae397e4645127aa7773211310f81ebb6c99e1f8e3e22c5cdb55333f5408"
+          },
+          "x86_64_linux": {
+            "url": "https://ghcr.io/v2/homebrew/core/hello/blobs/sha256:e6980196298e0a9cfe4fa4e328a71a1869a4d5e1d31c38442150ed784cfc0e29"
+          }
+        },
+        "dependencies": []
+      }
+    EOS
+  }
+  let(:bottle_hash) { JSON.parse(bottle_json) }
+  let(:versions_json) {
+    <<~EOS
+      {
+        "foo":{"version":"1.2.3","revision":0},
+        "bar":{"version":"1.2","revision":4}
+      }
+    EOS
+  }
+
+  def mock_curl_output(stdout: "", success: true)
+    curl_output = OpenStruct.new(stdout: stdout, success?: success)
+    allow(Utils::Curl).to receive(:curl_output).and_return curl_output
+  end
+
+  describe "::fetch" do
+    it "fetches the bottle JSON for a formula that exists" do
+      mock_curl_output stdout: bottle_json
+      fetched_hash = described_class.fetch("foo")
+      expect(fetched_hash).to eq bottle_hash
+    end
+
+    it "raises an error if the formula does not exist" do
+      mock_curl_output success: false
+      expect { described_class.fetch("bar") }.to raise_error(ArgumentError, /No JSON file found/)
+    end
+
+    it "raises an error if the bottle JSON is invalid" do
+      mock_curl_output stdout: "foo"
+      expect { described_class.fetch("baz") }.to raise_error(ArgumentError, /Invalid JSON file/)
+    end
+  end
+
+  describe "::latest_pkg_version" do
+    it "returns the expected `PkgVersion` when the revision is 0" do
+      mock_curl_output stdout: versions_json
+      pkg_version = described_class.latest_pkg_version("foo")
+      expect(pkg_version.to_s).to eq "1.2.3"
+    end
+
+    it "returns the expected `PkgVersion` when the revision is not 0" do
+      mock_curl_output stdout: versions_json
+      pkg_version = described_class.latest_pkg_version("bar")
+      expect(pkg_version.to_s).to eq "1.2_4"
+    end
+
+    it "returns `nil` when the formula is not in the JSON file" do
+      mock_curl_output stdout: versions_json
+      pkg_version = described_class.latest_pkg_version("baz")
+      expect(pkg_version).to be_nil
+    end
+  end
+
+  describe "::bottle_available?" do
+    it "returns `true` if `fetch` succeeds" do
+      allow(described_class).to receive(:fetch)
+      expect(described_class.bottle_available?("foo")).to eq true
+    end
+
+    it "returns `false` if `fetch` fails" do
+      allow(described_class).to receive(:fetch).and_raise ArgumentError
+      expect(described_class.bottle_available?("foo")).to eq false
+    end
+  end
+
+  describe "::fetch_bottles" do
+    before do
+      allow(described_class).to receive(:fetch).and_return bottle_hash
+    end
+
+    it "fetches bottles if a bottle is available" do
+      allow(Utils::Bottles).to receive(:tag).and_return :arm64_big_sur
+      expect { described_class.fetch_bottles("hello") }.not_to raise_error
+    end
+
+    it "raises an error if no bottle is available" do
+      allow(Utils::Bottles).to receive(:tag).and_return :catalina
+      expect { described_class.fetch_bottles("hello") }.to raise_error(SystemExit)
+    end
+  end
+
+  describe "::checksum_from_url" do
+    let(:sha256) { "b3b083db0807ff92c6e289a298f378198354b7727fb9ba9f4d550b8e08f90a60" }
+    let(:url) { "https://ghcr.io/v2/homebrew/core/hello/blobs/sha256:#{sha256}" }
+    let(:non_ghp_url) { "https://formulae.brew.sh/api/formula/hello.json" }
+
+    it "returns the `sha256` for a GitHub packages URL" do
+      expect(described_class.checksum_from_url(url)).to eq sha256
+    end
+
+    it "returns `nil` for a non-GitHub packages URL" do
+      expect(described_class.checksum_from_url(non_ghp_url)).to be_nil
+    end
+  end
+end

manpages/brew.1

@@ -1,7 +1,7 @@
 .\" generated with Ronn/v0.7.3
 .\" http://github.com/rtomayko/ronn/tree/0.7.3
 .
-.TH "BREW" "1" "June 2021" "Homebrew" "brew"
+.TH "BREW" "1" "July 2021" "Homebrew" "brew"
 .
 .SH "NAME"
 \fBbrew\fR \- The Missing Package Manager for macOS (or Linux)