From c2342eca91ec4befd903393ed28a79aee1a50df2 Mon Sep 17 00:00:00 2001 From: Bo Anderson Date: Thu, 16 Feb 2023 21:49:03 +0000 Subject: [PATCH] Further improvements to API handling in shell --- Library/Homebrew/api.rb | 15 ++++++- Library/Homebrew/api/cask.rb | 37 +++++++++++++---- Library/Homebrew/api/formula.rb | 60 ++++++++++++++++++++------- Library/Homebrew/cmd/casks.sh | 11 ++++- Library/Homebrew/cmd/formulae.sh | 14 +++---- Library/Homebrew/cmd/update-report.rb | 6 +++ Library/Homebrew/cmd/update.sh | 6 +++ Library/Homebrew/prefix.sh | 28 +++++++++---- Library/Homebrew/test/api_spec.rb | 4 +- 9 files changed, 137 insertions(+), 44 deletions(-) diff --git a/Library/Homebrew/api.rb b/Library/Homebrew/api.rb index d9e2bbbab1..89bf6cba38 100644 --- a/Library/Homebrew/api.rb +++ b/Library/Homebrew/api.rb @@ -35,7 +35,7 @@ module Homebrew raise ArgumentError, "Invalid JSON file: #{Tty.underline}#{api_url}#{Tty.reset}" end - sig { params(endpoint: String, target: Pathname).returns(T.any(Array, Hash)) } + sig { params(endpoint: String, target: Pathname).returns([T.any(Array, Hash), T::Boolean]) } def self.fetch_json_api_file(endpoint, target:) retry_count = 0 url = "#{Homebrew::EnvConfig.api_domain}/#{endpoint}" @@ -83,7 +83,7 @@ module Homebrew end FileUtils.touch(target) unless skip_download - JSON.parse(target.read) + [JSON.parse(target.read), !skip_download] rescue JSON::ParserError target.unlink retry_count += 1 @@ -129,5 +129,16 @@ module Homebrew json.except("variations") end + + sig { params(names: T::Array[String], type: String, regenerate: T::Boolean).returns(T::Boolean) } + def self.write_names_file(names, type, regenerate:) + names_path = HOMEBREW_CACHE_API/"#{type}_names.txt" + if !names_path.exist? || regenerate + names_path.write(names.join("\n")) + return true + end + + false + end end end diff --git a/Library/Homebrew/api/cask.rb b/Library/Homebrew/api/cask.rb index a4dac3540e..d5308372ed 100644 --- a/Library/Homebrew/api/cask.rb +++ b/Library/Homebrew/api/cask.rb @@ -1,6 +1,8 @@ # typed: true # frozen_string_literal: true +require "extend/cachable" + module Homebrew module API # Helper functions for using the cask JSON API. @@ -8,8 +10,11 @@ module Homebrew # @api private module Cask class << self + include Cachable extend T::Sig + private :cache + sig { params(token: String).returns(Hash) } def fetch(token) Homebrew::API.fetch "cask/#{token}.json" @@ -20,16 +25,34 @@ module Homebrew Homebrew::API.fetch_homebrew_cask_source token, git_head: git_head end + sig { returns(T::Boolean) } + def download_and_cache_data! + json_casks, updated = Homebrew::API.fetch_json_api_file "cask.json", + target: HOMEBREW_CACHE_API/"cask.json" + + cache["casks"] = json_casks.to_h do |json_cask| + [json_cask["token"], json_cask.except("token")] + end + + updated + end + private :download_and_cache_data! + sig { returns(Hash) } def all_casks - @all_casks ||= begin - json_casks = Homebrew::API.fetch_json_api_file "cask.json", - target: HOMEBREW_CACHE_API/"cask.json" - - json_casks.to_h do |json_cask| - [json_cask["token"], json_cask.except("token")] - end + unless cache.key?("casks") + json_updated = download_and_cache_data! + write_names(regenerate: json_updated) end + + cache["casks"] + end + + sig { params(regenerate: T::Boolean).void } + def write_names(regenerate: false) + download_and_cache_data! unless cache.key?("casks") + + Homebrew::API.write_names_file(all_casks.keys, "cask", regenerate: regenerate) end end end diff --git a/Library/Homebrew/api/formula.rb b/Library/Homebrew/api/formula.rb index 2ad31b137b..f8ef3a9809 100644 --- a/Library/Homebrew/api/formula.rb +++ b/Library/Homebrew/api/formula.rb @@ -1,6 +1,8 @@ # typed: true # frozen_string_literal: true +require "extend/cachable" + module Homebrew module API # Helper functions for using the formula JSON API. @@ -8,35 +10,65 @@ module Homebrew # @api private module Formula class << self + include Cachable extend T::Sig + private :cache + sig { params(name: String).returns(Hash) } def fetch(name) Homebrew::API.fetch "formula/#{name}.json" end + sig { returns(T::Boolean) } + def download_and_cache_data! + json_formulae, updated = Homebrew::API.fetch_json_api_file "formula.json", + target: HOMEBREW_CACHE_API/"formula.json" + + cache["aliases"] = {} + cache["formulae"] = json_formulae.to_h do |json_formula| + json_formula["aliases"].each do |alias_name| + cache["aliases"][alias_name] = json_formula["name"] + end + + [json_formula["name"], json_formula.except("name")] + end + + updated + end + private :download_and_cache_data! + sig { returns(Hash) } def all_formulae - @all_formulae ||= begin - json_formulae = Homebrew::API.fetch_json_api_file "formula.json", - target: HOMEBREW_CACHE_API/"formula.json" - - @all_aliases = {} - json_formulae.to_h do |json_formula| - json_formula["aliases"].each do |alias_name| - @all_aliases[alias_name] = json_formula["name"] - end - - [json_formula["name"], json_formula.except("name")] - end + unless cache.key?("formulae") + json_updated = download_and_cache_data! + write_names_and_aliases(regenerate: json_updated) end + + cache["formulae"] end sig { returns(Hash) } def all_aliases - all_formulae if @all_aliases.blank? + unless cache.key?("aliases") + json_updated = download_and_cache_data! + write_names_and_aliases(regenerate: json_updated) + end - @all_aliases + cache["aliases"] + end + + sig { params(regenerate: T::Boolean).void } + def write_names_and_aliases(regenerate: false) + download_and_cache_data! unless cache.key?("formulae") + + return unless Homebrew::API.write_names_file(all_formulae.keys, "formula", regenerate: regenerate) + + (HOMEBREW_CACHE_API/"formula_aliases.txt").open("w") do |file| + all_aliases.each do |alias_name, real_name| + file.puts "#{alias_name}|#{real_name}" + end + end end end end diff --git a/Library/Homebrew/cmd/casks.sh b/Library/Homebrew/cmd/casks.sh index b39285b1de..58e3acbff2 100644 --- a/Library/Homebrew/cmd/casks.sh +++ b/Library/Homebrew/cmd/casks.sh @@ -8,5 +8,14 @@ source "${HOMEBREW_LIBRARY}/Homebrew/items.sh" homebrew-casks() { - homebrew-items '*/Casks/*\.rb' '' 's|/Casks/|/|' '^homebrew/cask' + # HOMEBREW_CACHE is set by brew.sh + # shellcheck disable=SC2154 + if [[ -z "${HOMEBREW_NO_INSTALL_FROM_API}" && + -f "${HOMEBREW_CACHE}/api/cask_names.txt" ]] + then + cat "${HOMEBREW_CACHE}/api/cask_names.txt" + echo + else + homebrew-items '*/Casks/*\.rb' '' 's|/Casks/|/|' '^homebrew/cask' + fi } diff --git a/Library/Homebrew/cmd/formulae.sh b/Library/Homebrew/cmd/formulae.sh index 7da00f0980..94a0f66a3e 100644 --- a/Library/Homebrew/cmd/formulae.sh +++ b/Library/Homebrew/cmd/formulae.sh @@ -8,18 +8,14 @@ source "${HOMEBREW_LIBRARY}/Homebrew/items.sh" homebrew-formulae() { - local formulae - formulae="$(homebrew-items '*\.rb' 'Casks' 's|/Formula/|/|' '^homebrew/core')" - # HOMEBREW_CACHE is set by brew.sh # shellcheck disable=SC2154 if [[ -z "${HOMEBREW_NO_INSTALL_FROM_API}" && - -f "${HOMEBREW_CACHE}/api/formula.json" ]] + -f "${HOMEBREW_CACHE}/api/formula_names.txt" ]] then - local api_formulae - api_formulae="$(ruby -e "require 'json'; JSON.parse(File.read('${HOMEBREW_CACHE}/api/formula.json')).each { |f| puts f['name'] }" 2>/dev/null)" - formulae="$(echo -e "${formulae}\n${api_formulae}" | sort -uf | grep .)" + cat "${HOMEBREW_CACHE}/api/formula_names.txt" + echo + else + homebrew-items '*\.rb' 'Casks' 's|/Formula/|/|' '^homebrew/core' fi - - echo "${formulae}" } diff --git a/Library/Homebrew/cmd/update-report.rb b/Library/Homebrew/cmd/update-report.rb index 5dfa445033..bbd0508896 100644 --- a/Library/Homebrew/cmd/update-report.rb +++ b/Library/Homebrew/cmd/update-report.rb @@ -146,6 +146,12 @@ module Homebrew end end + # Check if we can parse the JSON and do any Ruby-side follow-up. + if Homebrew::EnvConfig.install_from_api? + Homebrew::API::Formula.write_names_and_aliases + Homebrew::API::Cask.write_names + end + Homebrew.failed = true if ENV["HOMEBREW_UPDATE_FAILED"] return if Homebrew::EnvConfig.disable_load_formula? diff --git a/Library/Homebrew/cmd/update.sh b/Library/Homebrew/cmd/update.sh index d59bde3527..0d158b306c 100644 --- a/Library/Homebrew/cmd/update.sh +++ b/Library/Homebrew/cmd/update.sh @@ -796,9 +796,15 @@ EOS done if [[ ${curl_exit_code} -eq 0 ]] then + touch "${HOMEBREW_CACHE}/api/${formula_or_cask}.json" CURRENT_JSON_BYTESIZE="$(wc -c "${HOMEBREW_CACHE}"/api/"${formula_or_cask}".json)" if [[ "${INITIAL_JSON_BYTESIZE}" != "${CURRENT_JSON_BYTESIZE}" ]] then + rm -f "${HOMEBREW_CACHE}/api/${formula_or_cask}_names.txt" + if [[ "${formula_or_cask}" == "formula" ]] + then + rm -f "${HOMEBREW_CACHE}/api/formula_aliases.txt" + fi HOMEBREW_UPDATED="1" fi else diff --git a/Library/Homebrew/prefix.sh b/Library/Homebrew/prefix.sh index 86a4990dea..dc55b996ad 100644 --- a/Library/Homebrew/prefix.sh +++ b/Library/Homebrew/prefix.sh @@ -39,16 +39,26 @@ homebrew-prefix() { fi if [[ -z "${formula_exists}" && - -z "${HOMEBREW_NO_INSTALL_FROM_API}" && - -f "${HOMEBREW_CACHE}/api/formula.json" ]] + -z "${HOMEBREW_NO_INSTALL_FROM_API}" ]] then - formula_exists="$( - ruby -rjson </dev/null -puts 1 if JSON.parse(File.read("${HOMEBREW_CACHE}/api/formula.json")).any? do |f| - f["name"] == "${formula}" -end -RUBY - )" + if [[ -f "${HOMEBREW_CACHE}/api/formula_names.txt" ]] && + grep -Fxq "${formula}" "${HOMEBREW_CACHE}/api/formula_names.txt" + then + formula_exists="1" + elif [[ -f "${HOMEBREW_CACHE}/api/formula_aliases.txt" ]] + then + while IFS="|" read -r alias_name real_name + do + case "${alias_name}" in + "${formula}") + formula_exists="1" + formula="${real_name}" + break + ;; + *) ;; + esac + done <"${HOMEBREW_CACHE}/api/formula_aliases.txt" + fi fi [[ -z "${formula_exists}" ]] && return 1 diff --git a/Library/Homebrew/test/api_spec.rb b/Library/Homebrew/test/api_spec.rb index e001f269fb..bc09640624 100644 --- a/Library/Homebrew/test/api_spec.rb +++ b/Library/Homebrew/test/api_spec.rb @@ -51,13 +51,13 @@ describe Homebrew::API do it "fetches a JSON file" do mock_curl_download stdout: json - fetched_json = described_class.fetch_json_api_file("foo.json", target: cache_dir/"foo.json") + fetched_json, = described_class.fetch_json_api_file("foo.json", target: cache_dir/"foo.json") expect(fetched_json).to eq json_hash end it "updates an existing JSON file" do mock_curl_download stdout: json - fetched_json = described_class.fetch_json_api_file("bar.json", target: cache_dir/"bar.json") + fetched_json, = described_class.fetch_json_api_file("bar.json", target: cache_dir/"bar.json") expect(fetched_json).to eq json_hash end