Merge pull request #14733 from Bo98/api-security

Security enhancements to API

Library/Homebrew/api.rb

@@ -56,7 +56,7 @@ module Homebrew
                       (Homebrew::EnvConfig.no_auto_update? ||
                       ((Time.now - Homebrew::EnvConfig.api_auto_update_secs.to_i) < target.mtime))
 
-      begin
+      json_data = begin
         begin
           args = curl_args.dup
           args.prepend("--time-cond", target.to_s) if target.exist? && !target.empty?
@@ -83,7 +83,7 @@ module Homebrew
         end
 
         FileUtils.touch(target) unless skip_download
-        [JSON.parse(target.read), !skip_download]
+        JSON.parse(target.read)
       rescue JSON::ParserError
         target.unlink
         retry_count += 1
@@ -92,10 +92,26 @@ module Homebrew
 
         retry
       end
+
+      if endpoint.end_with?(".jws.json")
+        success, data = verify_and_parse_jws(json_data)
+        unless success
+          target.unlink
+          odie <<~EOS
+            Failed to verify integrity (#{data}) of:
+              #{url}
+            Potential MITM attempt detected. Please run `brew update` and try again.
+          EOS
+        end
+        [data, !skip_download]
+      else
+        [json_data, !skip_download]
+      end
     end
 
-    sig { params(name: String, git_head: T.nilable(String)).returns(String) }
+    sig { params(name: String, git_head: T.nilable(String), sha256: T.nilable(String)).returns(String) }
-    def self.fetch_homebrew_cask_source(name, git_head: nil)
+    def self.fetch_homebrew_cask_source(name, git_head: nil, sha256: nil)
+      # TODO: unify with formula logic (https://github.com/Homebrew/brew/issues/14746)
       git_head = "master" if git_head.blank?
       raw_endpoint = "#{git_head}/Casks/#{name}.rb"
       return cache[raw_endpoint] if cache.present? && cache.key?(raw_endpoint)
@@ -103,10 +119,13 @@ module Homebrew
       # This API sometimes returns random 404s so needs a fallback at formulae.brew.sh.
       raw_source_url = "https://raw.githubusercontent.com/Homebrew/homebrew-cask/#{raw_endpoint}"
       api_source_url = "#{HOMEBREW_API_DEFAULT_DOMAIN}/cask-source/#{name}.rb"
-      output = Utils::Curl.curl_output("--fail", raw_source_url)
+
+      url = raw_source_url
+      output = Utils::Curl.curl_output("--fail", url)
 
       if !output.success? || output.blank?
-        output = Utils::Curl.curl_output("--fail", api_source_url)
+        url = api_source_url
+        output = Utils::Curl.curl_output("--fail", url)
         if !output.success? || output.blank?
           raise ArgumentError, <<~EOS
             No valid file found at either of:
@@ -116,7 +135,20 @@ module Homebrew
         end
       end
 
-      cache[raw_endpoint] = output.stdout
+      cask_source = output.stdout
+      actual_sha256 = Digest::SHA256.hexdigest(cask_source)
+      if sha256 && actual_sha256 != sha256
+        raise ArgumentError, <<~EOS
+          SHA256 mismatch
+          Expected: #{Formatter.success(sha256.to_s)}
+            Actual: #{Formatter.error(actual_sha256.to_s)}
+               URL: #{url}
+          Check if you can access the URL in your browser.
+          Regardless, try again in a few minutes.
+        EOS
+      end
+
+      cache[raw_endpoint] = cask_source
     end
 
     sig { params(json: Hash).returns(Hash) }
@@ -140,5 +172,31 @@ module Homebrew
 
       false
     end
+
+    sig { params(json_data: Hash).returns([T::Boolean, T.any(String, Array, Hash)]) }
+    private_class_method def self.verify_and_parse_jws(json_data)
+      signatures = json_data["signatures"]
+      homebrew_signature = signatures&.find { |sig| sig.dig("header", "kid") == "homebrew-1" }
+      return false, "key not found" if homebrew_signature.nil?
+
+      header = JSON.parse(Base64.urlsafe_decode64(homebrew_signature["protected"]))
+      if header["alg"] != "PS512" || header["b64"] != false # NOTE: nil has a meaning of true
+        return false, "invalid algorithm"
+      end
+
+      require "openssl"
+
+      pubkey = OpenSSL::PKey::RSA.new((HOMEBREW_LIBRARY_PATH/"api/homebrew-1.pem").read)
+      signing_input = "#{homebrew_signature["protected"]}.#{json_data["payload"]}"
+      unless pubkey.verify_pss("SHA512",
+                               Base64.urlsafe_decode64(homebrew_signature["signature"]),
+                               signing_input,
+                               salt_length: :digest,
+                               mgf1_hash:   "SHA512")
+        return false, "signature mismatch"
+      end
+
+      [true, JSON.parse(json_data["payload"])]
+    end
   end
 end

Library/Homebrew/api/cask.rb

@@ -20,15 +20,15 @@ module Homebrew
           Homebrew::API.fetch "cask/#{token}.json"
         end
 
-        sig { params(token: String, git_head: T.nilable(String)).returns(String) }
+        sig { params(token: String, git_head: T.nilable(String), sha256: T.nilable(String)).returns(String) }
-        def fetch_source(token, git_head: nil)
+        def fetch_source(token, git_head: nil, sha256: nil)
-          Homebrew::API.fetch_homebrew_cask_source token, git_head: git_head
+          Homebrew::API.fetch_homebrew_cask_source token, git_head: git_head, sha256: sha256
         end
 
         sig { returns(T::Boolean) }
         def download_and_cache_data!
-          json_casks, updated = Homebrew::API.fetch_json_api_file "cask.json",
+          json_casks, updated = Homebrew::API.fetch_json_api_file "cask.jws.json",
-                                                                  target: HOMEBREW_CACHE_API/"cask.json"
+                                                                  target: HOMEBREW_CACHE_API/"cask.jws.json"
 
           cache["casks"] = json_casks.to_h do |json_cask|
             [json_cask["token"], json_cask.except("token")]

Library/Homebrew/api/formula.rb

@@ -22,8 +22,8 @@ module Homebrew
 
         sig { returns(T::Boolean) }
         def download_and_cache_data!
-          json_formulae, updated = Homebrew::API.fetch_json_api_file "formula.json",
+          json_formulae, updated = Homebrew::API.fetch_json_api_file "formula.jws.json",
-                                                                     target: HOMEBREW_CACHE_API/"formula.json"
+                                                                     target: HOMEBREW_CACHE_API/"formula.jws.json"
 
           cache["aliases"] = {}
           cache["formulae"] = json_formulae.to_h do |json_formula|

Library/Homebrew/api/homebrew-1.pem

@@ -0,0 +1,14 @@
+-----BEGIN PUBLIC KEY-----
+MIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEAyKoOYzp1rhwXISRi61BY
+XBEr2PalSK8lEVOL2USy7mpy0OubOlFyujawyQcBcCn+uPOJ/WaK+POhNWcLLoiK
+L2m8GViaQm7SMwdLKUXFgKSPHcG/1m6Vu+TNBKTfQqT60PjEYIrn5NW9ZrM0cUhK
+REmsbeAMBevdSaW9UwY9iIhprrgovvT8SzKhF8ZOIZKXfJX4VNk0y/7VJYNuGGqH
+3npxV7OKd4yTGRGqFcC9kJ84me3thiu0yqlOjASmfWIwIwcfp4j6BEM2LuqKd7yX
+h51/O+MTthkuxV36moDKfdgdOFsvlCFkziaYLScCX9lOlmZHtOfJTAOXxTmM7qGr
+wTGK0vhvTi8k9dBmH/dccredQBtPOfM/FEdeyakGLoTcDguiBS/4El3I2KtF6B2h
+OGoBumR915/cI4drr5yPMduZ7gjs7ZEZnVkeVzic24TfUHpnOYzrhucNJtHMBDj9
+6d1Gk82AhtuF9KlusLmCb6qXCWQSp/A4RZpN37E/p9q8rLp/7B/zp8X2TVvecPNy
+BdMagdktdEqK7WPlYMcUp56JaOph8vqYoU+oGyCpWoLvcXFb75o4eefuu6Rs5SyM
+c9JCCJ0DDFPjCRFnGPkvsKxFCzMFqH1jpWH0RQIrgmNVM5PO84iRH9YJsSPQzpMj
+KvK/ZH4YgR9wNkBNagFo7lsCAwEAAQ==
+-----END PUBLIC KEY-----

Library/Homebrew/cask/cask.rb

@@ -83,11 +83,12 @@ module Cask
       @tap
     end
 
-    def initialize(token, sourcefile_path: nil, source: nil, tap: nil, config: nil,
+    def initialize(token, sourcefile_path: nil, source: nil, source_checksum: nil, tap: nil,
-                   allow_reassignment: false, loaded_from_api: false, loader: nil, &block)
+                   config: nil, allow_reassignment: false, loaded_from_api: false, loader: nil, &block)
       @token = token
       @sourcefile_path = sourcefile_path
       @source = source
+      @ruby_source_checksum = source_checksum
       @tap = tap
       @allow_reassignment = allow_reassignment
       @loaded_from_api = loaded_from_api
@@ -277,27 +278,28 @@ module Cask
       end
 
       {
-        "token"          => token,
+        "token"                => token,
-        "full_token"     => full_name,
+        "full_token"           => full_name,
-        "tap"            => tap&.name,
+        "tap"                  => tap&.name,
-        "name"           => name,
+        "name"                 => name,
-        "desc"           => desc,
+        "desc"                 => desc,
-        "homepage"       => homepage,
+        "homepage"             => homepage,
-        "url"            => url,
+        "url"                  => url,
-        "appcast"        => appcast,
+        "appcast"              => appcast,
-        "version"        => version,
+        "version"              => version,
-        "versions"       => os_versions,
+        "versions"             => os_versions,
-        "installed"      => versions.last,
+        "installed"            => versions.last,
-        "outdated"       => outdated?,
+        "outdated"             => outdated?,
-        "sha256"         => sha256,
+        "sha256"               => sha256,
-        "artifacts"      => artifacts_list,
+        "artifacts"            => artifacts_list,
-        "caveats"        => (caveats unless caveats.empty?),
+        "caveats"              => (caveats unless caveats.empty?),
-        "depends_on"     => depends_on,
+        "depends_on"           => depends_on,
-        "conflicts_with" => conflicts_with,
+        "conflicts_with"       => conflicts_with,
-        "container"      => container&.pairs,
+        "container"            => container&.pairs,
-        "auto_updates"   => auto_updates,
+        "auto_updates"         => auto_updates,
-        "tap_git_head"   => tap&.git_head,
+        "tap_git_head"         => tap&.git_head,
-        "languages"      => languages,
+        "languages"            => languages,
+        "ruby_source_checksum" => ruby_source_checksum,
       }
     end
 
@@ -349,6 +351,12 @@ module Cask
       hash
     end
 
+    def ruby_source_checksum
+      @ruby_source_checksum ||= {
+        "sha256" => Digest::SHA256.file(sourcefile_path).hexdigest,
+      }
+    end
+
     def artifacts_list
       artifacts.map do |artifact|
         case artifact

Library/Homebrew/cask/cask_loader.rb

@@ -233,13 +233,21 @@ module Cask
         # Use the cask-source API if there are any `*flight` blocks or the cask has multiple languages
         if json_cask[:artifacts].any? { |artifact| FLIGHT_STANZAS.include?(artifact.keys.first) } ||
            json_cask[:languages].any?
-          cask_source = Homebrew::API::Cask.fetch_source(token, git_head: json_cask[:tap_git_head])
+          cask_source = Homebrew::API::Cask.fetch_source(token,
+                                                         git_head: json_cask[:tap_git_head],
+                                                         sha256:   json_cask.dig(:ruby_source_checksum, :sha256))
           return FromContentLoader.new(cask_source).load(config: config)
         end
 
         tap = Tap.fetch(json_cask[:tap]) if json_cask[:tap].to_s.include?("/")
 
-        Cask.new(token, tap: tap, source: cask_source, config: config, loaded_from_api: true, loader: self) do
+        Cask.new(token,
+                 tap:             tap,
+                 source:          cask_source,
+                 source_checksum: json_cask[:ruby_source_checksum],
+                 config:          config,
+                 loaded_from_api: true,
+                 loader:          self) do
           version json_cask[:version]
 
           if json_cask[:sha256] == "no_check"

Library/Homebrew/cmd/update.sh

@@ -766,28 +766,28 @@ EOS
 
     for formula_or_cask in formula cask
     do
-      if [[ -f "${HOMEBREW_CACHE}/api/${formula_or_cask}.json" ]]
+      if [[ -f "${HOMEBREW_CACHE}/api/${formula_or_cask}.jws.json" ]]
       then
-        INITIAL_JSON_BYTESIZE="$(wc -c "${HOMEBREW_CACHE}"/api/"${formula_or_cask}".json)"
+        INITIAL_JSON_BYTESIZE="$(wc -c "${HOMEBREW_CACHE}"/api/"${formula_or_cask}".jws.json)"
       fi
       JSON_URLS=()
       if [[ -n "${HOMEBREW_API_DOMAIN}" && "${HOMEBREW_API_DOMAIN}" != "${HOMEBREW_API_DEFAULT_DOMAIN}" ]]
       then
-        JSON_URLS=("${HOMEBREW_API_DOMAIN}/${formula_or_cask}.json")
+        JSON_URLS=("${HOMEBREW_API_DOMAIN}/${formula_or_cask}.jws.json")
       fi
-      JSON_URLS+=("${HOMEBREW_API_DEFAULT_DOMAIN}/${formula_or_cask}.json")
+      JSON_URLS+=("${HOMEBREW_API_DEFAULT_DOMAIN}/${formula_or_cask}.jws.json")
       for json_url in "${JSON_URLS[@]}"
       do
         time_cond=()
-        if [[ -s "${HOMEBREW_CACHE}/api/${formula_or_cask}.json" ]]
+        if [[ -s "${HOMEBREW_CACHE}/api/${formula_or_cask}.jws.json" ]]
         then
-          time_cond=("--time-cond" "${HOMEBREW_CACHE}/api/${formula_or_cask}.json")
+          time_cond=("--time-cond" "${HOMEBREW_CACHE}/api/${formula_or_cask}.jws.json")
         fi
         curl \
           "${CURL_DISABLE_CURLRC_ARGS[@]}" \
           --fail --compressed --silent \
           --speed-limit "${HOMEBREW_CURL_SPEED_LIMIT}" --speed-time "${HOMEBREW_CURL_SPEED_TIME}" \
-          --location --remote-time --output "${HOMEBREW_CACHE}/api/${formula_or_cask}.json" \
+          --location --remote-time --output "${HOMEBREW_CACHE}/api/${formula_or_cask}.jws.json" \
           "${time_cond[@]}" \
           --user-agent "${HOMEBREW_USER_AGENT_CURL}" \
           "${json_url}"
@@ -796,8 +796,8 @@ EOS
       done
       if [[ ${curl_exit_code} -eq 0 ]]
       then
-        touch "${HOMEBREW_CACHE}/api/${formula_or_cask}.json"
+        touch "${HOMEBREW_CACHE}/api/${formula_or_cask}.jws.json"
-        CURRENT_JSON_BYTESIZE="$(wc -c "${HOMEBREW_CACHE}"/api/"${formula_or_cask}".json)"
+        CURRENT_JSON_BYTESIZE="$(wc -c "${HOMEBREW_CACHE}"/api/"${formula_or_cask}".jws.json)"
         if [[ "${INITIAL_JSON_BYTESIZE}" != "${CURRENT_JSON_BYTESIZE}" ]]
         then
           rm -f "${HOMEBREW_CACHE}/api/${formula_or_cask}_names.txt"

Library/Homebrew/formula.rb

@@ -2133,6 +2133,7 @@ class Formula
       "disable_date"             => disable_date,
       "disable_reason"           => disable_reason,
       "tap_git_head"             => tap_git_head,
+      "ruby_source_checksum"     => {},
     }
 
     if stable
@@ -2183,6 +2184,16 @@ class Formula
       }
     end
 
+    if self.class.loaded_from_api && active_spec.resource_defined?("ruby-source")
+      hsh["ruby_source_checksum"] = {
+        "sha256" => resource("ruby-source").checksum.hexdigest,
+      }
+    elsif !self.class.loaded_from_api && path.exist?
+      hsh["ruby_source_checksum"] = {
+        "sha256" => Digest::SHA256.file(path).hexdigest,
+      }
+    end
+
     hsh
   end
 

Library/Homebrew/formula.rbi

@@ -17,7 +17,7 @@ class Formula
   def service?; end
   def version; end
 
-  def resource; end
+  def resource(name); end
   def deps; end
   def uses_from_macos_elements; end
   def requirements; end

Library/Homebrew/formula_installer.rb

@@ -1182,9 +1182,12 @@ class FormulaInstaller
     if pour_bottle?(output_warning: true)
       formula.fetch_bottle_tab
     else
-      if formula.core_formula? && Homebrew::EnvConfig.install_from_api?
+      if formula.class.loaded_from_api
-        url = "https://raw.githubusercontent.com/#{formula.tap.full_name}/#{formula.tap_git_head}/Formula/#{formula.name}.rb"
+        # TODO: unify with cask logic (https://github.com/Homebrew/brew/issues/14746)
-        @formula = Formulary.factory(url, formula.active_spec_sym,
+        resource = formula.resource("ruby-source")
+        resource.fetch
+        @formula = Formulary.factory(resource.cached_download,
+                                     formula.active_spec_sym,
                                      alias_path: formula.alias_path,
                                      flags:      formula.class.build_flags,
                                      from:       :formula_installer)

Library/Homebrew/formulary.rb

@@ -252,6 +252,13 @@ module Formulary
         link_overwrite overwrite_path
       end
 
+      resource "ruby-source" do
+        url "https://raw.githubusercontent.com/Homebrew/homebrew-core/#{json_formula["tap_git_head"]}/Formula/#{name}.rb"
+        if (ruby_source_sha256 = json_formula.dig("ruby_source_checksum", "sha256")).present?
+          sha256 ruby_source_sha256
+        end
+      end
+
       def install
         raise "Cannot build from source from abstract formula."
       end
@@ -446,7 +453,13 @@ module Formulary
   class FromPathLoader < FormulaLoader
     def initialize(path)
       path = Pathname.new(path).expand_path
-      super path.basename(".rb").to_s, path
+      name = path.basename(".rb").to_s
+
+      # For files we've downloaded, they will be prefixed with `{URL MD5}--`.
+      # Remove that prefix to get the original filename.
+      name = name.split("--", 2).last if path.dirname == HOMEBREW_CACHE/"downloads"
+
+      super name, path
     end
   end
 

Library/Homebrew/test/api/cask_spec.rb

@@ -9,12 +9,16 @@ describe Homebrew::API::Cask do
   before do
     stub_const("Homebrew::API::HOMEBREW_CACHE_API", cache_dir)
     Homebrew::API.clear_cache
+    described_class.clear_cache
   end
 
   def mock_curl_download(stdout:)
     allow(Utils::Curl).to receive(:curl_download) do |*_args, **kwargs|
       kwargs[:to].write stdout
     end
+    allow(Homebrew::API).to receive(:verify_and_parse_jws) do |json_data|
+      [true, json_data]
+    end
   end
 
   describe "::all_casks" do

Library/Homebrew/test/api/formula_spec.rb

@@ -8,12 +8,16 @@ describe Homebrew::API::Formula do
 
   before do
     stub_const("Homebrew::API::HOMEBREW_CACHE_API", cache_dir)
+    described_class.clear_cache
   end
 
   def mock_curl_download(stdout:)
     allow(Utils::Curl).to receive(:curl_download) do |*_args, **kwargs|
       kwargs[:to].write stdout
     end
+    allow(Homebrew::API).to receive(:verify_and_parse_jws) do |json_data|
+      [true, json_data]
+    end
   end
 
   describe "::all_formulae" do

Library/Homebrew/test/cask/cmd/list_spec.rb

@@ -131,7 +131,10 @@ describe Cask::Cmd::List, :cask do
             "tap_git_head": null,
             "languages": [
 
-            ]
+            ],
+            "ruby_source_checksum": {
+              "sha256": "#{Digest::SHA256.file(Tap.default_cask_tap.cask_dir/"local-caffeine.rb").hexdigest}"
+            }
           },
           {
             "token": "local-transmission",
@@ -166,7 +169,10 @@ describe Cask::Cmd::List, :cask do
             "tap_git_head": null,
             "languages": [
 
-            ]
+            ],
+            "ruby_source_checksum": {
+              "sha256": "#{Digest::SHA256.file(Tap.default_cask_tap.cask_dir/"local-transmission.rb").hexdigest}"
+            }
           },
           {
             "token": "multiple-versions",
@@ -204,7 +210,10 @@ describe Cask::Cmd::List, :cask do
             "tap_git_head": null,
             "languages": [
 
-            ]
+            ],
+            "ruby_source_checksum": {
+              "sha256": "#{Digest::SHA256.file(Tap.default_cask_tap.cask_dir/"multiple-versions.rb").hexdigest}"
+            }
           },
           {
             "token": "third-party-cask",
@@ -239,7 +248,10 @@ describe Cask::Cmd::List, :cask do
             "tap_git_head": null,
             "languages": [
 
-            ]
+            ],
+            "ruby_source_checksum": {
+              "sha256": "#{Digest::SHA256.file(Tap.fetch("third-party", "tap").cask_dir/"third-party-cask.rb").hexdigest}"
+            }
           },
           {
             "token": "with-languages",
@@ -275,7 +287,10 @@ describe Cask::Cmd::List, :cask do
             "languages": [
               "zh",
               "en-US"
-            ]
+            ],
+            "ruby_source_checksum": {
+              "sha256": "#{Digest::SHA256.file(Tap.default_cask_tap.cask_dir/"with-languages.rb").hexdigest}"
+            }
           }
         ]
       EOS