Further improvements to API handling in shell

Library/Homebrew/api.rb

@@ -35,7 +35,7 @@ module Homebrew
       raise ArgumentError, "Invalid JSON file: #{Tty.underline}#{api_url}#{Tty.reset}"
     end
 
-    sig { params(endpoint: String, target: Pathname).returns(T.any(Array, Hash)) }
+    sig { params(endpoint: String, target: Pathname).returns([T.any(Array, Hash), T::Boolean]) }
     def self.fetch_json_api_file(endpoint, target:)
       retry_count = 0
       url = "#{Homebrew::EnvConfig.api_domain}/#{endpoint}"
@@ -83,7 +83,7 @@ module Homebrew
         end
 
         FileUtils.touch(target) unless skip_download
-        JSON.parse(target.read)
+        [JSON.parse(target.read), !skip_download]
       rescue JSON::ParserError
         target.unlink
         retry_count += 1
@@ -129,5 +129,16 @@ module Homebrew
 
       json.except("variations")
     end
+
+    sig { params(names: T::Array[String], type: String, regenerate: T::Boolean).returns(T::Boolean) }
+    def self.write_names_file(names, type, regenerate:)
+      names_path = HOMEBREW_CACHE_API/"#{type}_names.txt"
+      if !names_path.exist? || regenerate
+        names_path.write(names.join("\n"))
+        return true
+      end
+
+      false
+    end
   end
 end

Library/Homebrew/api/cask.rb

@@ -1,6 +1,8 @@
 # typed: true
 # frozen_string_literal: true
 
+require "extend/cachable"
+
 module Homebrew
   module API
     # Helper functions for using the cask JSON API.
@@ -8,8 +10,11 @@ module Homebrew
     # @api private
     module Cask
       class << self
+        include Cachable
         extend T::Sig
 
+        private :cache
+
         sig { params(token: String).returns(Hash) }
         def fetch(token)
           Homebrew::API.fetch "cask/#{token}.json"
@@ -20,16 +25,34 @@ module Homebrew
           Homebrew::API.fetch_homebrew_cask_source token, git_head: git_head
         end
 
-        sig { returns(Hash) }
+        sig { returns(T::Boolean) }
-        def all_casks
+        def download_and_cache_data!
-          @all_casks ||= begin
+          json_casks, updated = Homebrew::API.fetch_json_api_file "cask.json",
-            json_casks = Homebrew::API.fetch_json_api_file "cask.json",
                                                                   target: HOMEBREW_CACHE_API/"cask.json"
 
-            json_casks.to_h do |json_cask|
+          cache["casks"] = json_casks.to_h do |json_cask|
             [json_cask["token"], json_cask.except("token")]
           end
+
+          updated
         end
+        private :download_and_cache_data!
+
+        sig { returns(Hash) }
+        def all_casks
+          unless cache.key?("casks")
+            json_updated = download_and_cache_data!
+            write_names(regenerate: json_updated)
+          end
+
+          cache["casks"]
+        end
+
+        sig { params(regenerate: T::Boolean).void }
+        def write_names(regenerate: false)
+          download_and_cache_data! unless cache.key?("casks")
+
+          Homebrew::API.write_names_file(all_casks.keys, "cask", regenerate: regenerate)
         end
       end
     end

Library/Homebrew/api/formula.rb

@@ -1,6 +1,8 @@
 # typed: true
 # frozen_string_literal: true
 
+require "extend/cachable"
+
 module Homebrew
   module API
     # Helper functions for using the formula JSON API.
@@ -8,35 +10,65 @@ module Homebrew
     # @api private
     module Formula
       class << self
+        include Cachable
         extend T::Sig
 
+        private :cache
+
         sig { params(name: String).returns(Hash) }
         def fetch(name)
           Homebrew::API.fetch "formula/#{name}.json"
         end
 
-        sig { returns(Hash) }
+        sig { returns(T::Boolean) }
-        def all_formulae
+        def download_and_cache_data!
-          @all_formulae ||= begin
+          json_formulae, updated = Homebrew::API.fetch_json_api_file "formula.json",
-            json_formulae = Homebrew::API.fetch_json_api_file "formula.json",
                                                                      target: HOMEBREW_CACHE_API/"formula.json"
 
-            @all_aliases = {}
+          cache["aliases"] = {}
-            json_formulae.to_h do |json_formula|
+          cache["formulae"] = json_formulae.to_h do |json_formula|
             json_formula["aliases"].each do |alias_name|
-                @all_aliases[alias_name] = json_formula["name"]
+              cache["aliases"][alias_name] = json_formula["name"]
             end
 
             [json_formula["name"], json_formula.except("name")]
           end
+
+          updated
         end
+        private :download_and_cache_data!
+
+        sig { returns(Hash) }
+        def all_formulae
+          unless cache.key?("formulae")
+            json_updated = download_and_cache_data!
+            write_names_and_aliases(regenerate: json_updated)
+          end
+
+          cache["formulae"]
         end
 
         sig { returns(Hash) }
         def all_aliases
-          all_formulae if @all_aliases.blank?
+          unless cache.key?("aliases")
+            json_updated = download_and_cache_data!
+            write_names_and_aliases(regenerate: json_updated)
+          end
 
-          @all_aliases
+          cache["aliases"]
+        end
+
+        sig { params(regenerate: T::Boolean).void }
+        def write_names_and_aliases(regenerate: false)
+          download_and_cache_data! unless cache.key?("formulae")
+
+          return unless Homebrew::API.write_names_file(all_formulae.keys, "formula", regenerate: regenerate)
+
+          (HOMEBREW_CACHE_API/"formula_aliases.txt").open("w") do |file|
+            all_aliases.each do |alias_name, real_name|
+              file.puts "#{alias_name}|#{real_name}"
+            end
+          end
         end
       end
     end

Library/Homebrew/cmd/casks.sh

@@ -8,5 +8,14 @@
 source "${HOMEBREW_LIBRARY}/Homebrew/items.sh"
 
 homebrew-casks() {
+  # HOMEBREW_CACHE is set by brew.sh
+  # shellcheck disable=SC2154
+  if [[ -z "${HOMEBREW_NO_INSTALL_FROM_API}" &&
+        -f "${HOMEBREW_CACHE}/api/cask_names.txt" ]]
+  then
+    cat "${HOMEBREW_CACHE}/api/cask_names.txt"
+    echo
+  else
     homebrew-items '*/Casks/*\.rb' '' 's|/Casks/|/|' '^homebrew/cask'
+  fi
 }

Library/Homebrew/cmd/formulae.sh

@@ -8,18 +8,14 @@
 source "${HOMEBREW_LIBRARY}/Homebrew/items.sh"
 
 homebrew-formulae() {
-  local formulae
-  formulae="$(homebrew-items '*\.rb' 'Casks' 's|/Formula/|/|' '^homebrew/core')"
-
   # HOMEBREW_CACHE is set by brew.sh
   # shellcheck disable=SC2154
   if [[ -z "${HOMEBREW_NO_INSTALL_FROM_API}" &&
-        -f "${HOMEBREW_CACHE}/api/formula.json" ]]
+        -f "${HOMEBREW_CACHE}/api/formula_names.txt" ]]
   then
-    local api_formulae
+    cat "${HOMEBREW_CACHE}/api/formula_names.txt"
-    api_formulae="$(ruby -e "require 'json'; JSON.parse(File.read('${HOMEBREW_CACHE}/api/formula.json')).each { |f| puts f['name'] }" 2>/dev/null)"
+    echo
-    formulae="$(echo -e "${formulae}\n${api_formulae}" | sort -uf | grep .)"
+  else
+    homebrew-items '*\.rb' 'Casks' 's|/Formula/|/|' '^homebrew/core'
   fi
-
-  echo "${formulae}"
 }

Library/Homebrew/cmd/update-report.rb

@@ -146,6 +146,12 @@ module Homebrew
       end
     end
 
+    # Check if we can parse the JSON and do any Ruby-side follow-up.
+    if Homebrew::EnvConfig.install_from_api?
+      Homebrew::API::Formula.write_names_and_aliases
+      Homebrew::API::Cask.write_names
+    end
+
     Homebrew.failed = true if ENV["HOMEBREW_UPDATE_FAILED"]
     return if Homebrew::EnvConfig.disable_load_formula?
 

Library/Homebrew/cmd/update.sh

@@ -796,9 +796,15 @@ EOS
       done
       if [[ ${curl_exit_code} -eq 0 ]]
       then
+        touch "${HOMEBREW_CACHE}/api/${formula_or_cask}.json"
         CURRENT_JSON_BYTESIZE="$(wc -c "${HOMEBREW_CACHE}"/api/"${formula_or_cask}".json)"
         if [[ "${INITIAL_JSON_BYTESIZE}" != "${CURRENT_JSON_BYTESIZE}" ]]
         then
+          rm -f "${HOMEBREW_CACHE}/api/${formula_or_cask}_names.txt"
+          if [[ "${formula_or_cask}" == "formula" ]]
+          then
+            rm -f "${HOMEBREW_CACHE}/api/formula_aliases.txt"
+          fi
           HOMEBREW_UPDATED="1"
         fi
       else

Library/Homebrew/prefix.sh

@@ -39,16 +39,26 @@ homebrew-prefix() {
   fi
 
   if [[ -z "${formula_exists}" &&
-        -z "${HOMEBREW_NO_INSTALL_FROM_API}" &&
+        -z "${HOMEBREW_NO_INSTALL_FROM_API}" ]]
-        -f "${HOMEBREW_CACHE}/api/formula.json" ]]
   then
-    formula_exists="$(
+    if [[ -f "${HOMEBREW_CACHE}/api/formula_names.txt" ]] &&
-      ruby -rjson <<RUBY 2>/dev/null
+       grep -Fxq "${formula}" "${HOMEBREW_CACHE}/api/formula_names.txt"
-puts 1 if JSON.parse(File.read("${HOMEBREW_CACHE}/api/formula.json")).any? do |f|
+    then
-  f["name"] == "${formula}"
+      formula_exists="1"
-end
+    elif [[ -f "${HOMEBREW_CACHE}/api/formula_aliases.txt" ]]
-RUBY
+    then
-    )"
+      while IFS="|" read -r alias_name real_name
+      do
+        case "${alias_name}" in
+          "${formula}")
+            formula_exists="1"
+            formula="${real_name}"
+            break
+            ;;
+          *) ;;
+        esac
+      done <"${HOMEBREW_CACHE}/api/formula_aliases.txt"
+    fi
   fi
 
   [[ -z "${formula_exists}" ]] && return 1

Library/Homebrew/test/api_spec.rb

@@ -51,13 +51,13 @@ describe Homebrew::API do
 
     it "fetches a JSON file" do
       mock_curl_download stdout: json
-      fetched_json = described_class.fetch_json_api_file("foo.json", target: cache_dir/"foo.json")
+      fetched_json, = described_class.fetch_json_api_file("foo.json", target: cache_dir/"foo.json")
       expect(fetched_json).to eq json_hash
     end
 
     it "updates an existing JSON file" do
       mock_curl_download stdout: json
-      fetched_json = described_class.fetch_json_api_file("bar.json", target: cache_dir/"bar.json")
+      fetched_json, = described_class.fetch_json_api_file("bar.json", target: cache_dir/"bar.json")
       expect(fetched_json).to eq json_hash
     end