From 98f3258ff4539ce66518968d5c730869ffe96175 Mon Sep 17 00:00:00 2001 From: Sam Ford <1584702+samford@users.noreply.github.com> Date: Thu, 8 Feb 2024 09:59:21 -0500 Subject: [PATCH 1/2] Livecheck: Add Crate strategy We discussed the idea of adding a livecheck strategy to check crate versions years ago but decided to put it off because it would have only applied to one formula at the time (and it wasn't clear that a crate was necessary in that case). We now have a few formulae that use a crate in the `stable` URL (`cargo-llvm-cov`, `pngquant`, `oakc`) and another formula with a crate resource (`deno`), so there's some value to the idea now. I established a standard approach for checking crate versions in a somewhat recent `pngquant` `livecheck` block update and this commit reworks it into a strategy, so we won't have to duplicate that `livecheck` block in these cases. With this strategy, we usually won't even need a `livecheck` block at all. Under normal circumstances, a regex and/or strategy block shouldn't be necessary but the strategy supports them when needed. The response from the crates.io API is a JSON object, so this uses `Json#versions_from_content` internally and a `strategy` block will receive the parsed `json` object and a regex (the strategy default or the regex from the `livecheck` block). --- Library/Homebrew/.rubocop.yml | 1 + Library/Homebrew/livecheck/strategy.rb | 1 + Library/Homebrew/livecheck/strategy/crate.rb | 115 +++++++++++++ .../test/livecheck/strategy/crate_spec.rb | 152 ++++++++++++++++++ docs/Brew-Livecheck.md | 18 +++ 5 files changed, 287 insertions(+) create mode 100644 Library/Homebrew/livecheck/strategy/crate.rb create mode 100644 Library/Homebrew/test/livecheck/strategy/crate_spec.rb diff --git a/Library/Homebrew/.rubocop.yml b/Library/Homebrew/.rubocop.yml index 69df9b23f3..c14a7bc83e 100644 --- a/Library/Homebrew/.rubocop.yml +++ b/Library/Homebrew/.rubocop.yml @@ -40,6 +40,7 @@ Style/Documentation: - livecheck/strategy/apache.rb - livecheck/strategy/bitbucket.rb - livecheck/strategy/cpan.rb + - livecheck/strategy/crate.rb - livecheck/strategy/extract_plist.rb - livecheck/strategy/git.rb - livecheck/strategy/github_latest.rb diff --git a/Library/Homebrew/livecheck/strategy.rb b/Library/Homebrew/livecheck/strategy.rb index e17e13a646..1fd2c556ec 100644 --- a/Library/Homebrew/livecheck/strategy.rb +++ b/Library/Homebrew/livecheck/strategy.rb @@ -266,6 +266,7 @@ end require_relative "strategy/apache" require_relative "strategy/bitbucket" require_relative "strategy/cpan" +require_relative "strategy/crate" require_relative "strategy/electron_builder" require_relative "strategy/extract_plist" require_relative "strategy/git" diff --git a/Library/Homebrew/livecheck/strategy/crate.rb b/Library/Homebrew/livecheck/strategy/crate.rb new file mode 100644 index 0000000000..2b5d04898e --- /dev/null +++ b/Library/Homebrew/livecheck/strategy/crate.rb @@ -0,0 +1,115 @@ +# typed: true +# frozen_string_literal: true + +module Homebrew + module Livecheck + module Strategy + # The {Crate} strategy identifies versions of a Rust crate by checking + # the information from the `versions` API endpoint. + # + # Crate URLs have the following format: + # `https://static.crates.io/crates/example/example-1.2.3.crate` + # + # The default regex identifies versions like `1.2.3`/`v1.2.3` from the + # version `num` field. This is a common version format but a different + # regex can be provided in a `livecheck` block to override the default + # if a package uses a different format (e.g. `1.2.3d`, `1.2.3-4`, etc.). + # + # @api public + class Crate + # The default regex used to identify versions when a regex isn't + # provided. + DEFAULT_REGEX = /^v?(\d+(?:\.\d+)+)$/i + + # The default `strategy` block used to extract version information when + # a `strategy` block isn't provided. + DEFAULT_BLOCK = proc do |json, regex| + json["versions"]&.map do |version| + next if version["yanked"] == true + + match = version["num"]&.match(regex) + next if match.blank? + + match[1] + end + end.freeze + + # The `Regexp` used to determine if the strategy applies to the URL. + URL_MATCH_REGEX = %r{ + ^https?://static\.crates\.io/crates + /(?[^/]+) # The name of the package + /.+\.crate # The crate filename + }ix + + # Whether the strategy can be applied to the provided URL. + # + # @param url [String] the URL to match against + # @return [Boolean] + sig { params(url: String).returns(T::Boolean) } + def self.match?(url) + URL_MATCH_REGEX.match?(url) + end + + # Extracts information from a provided URL and uses it to generate + # various input values used by the strategy to check for new versions. + # + # @param url [String] the URL used to generate values + # @return [Hash] + sig { params(url: String).returns(T::Hash[Symbol, T.untyped]) } + def self.generate_input_values(url) + values = {} + + match = url.match(URL_MATCH_REGEX) + return values if match.blank? + + values[:url] = "https://crates.io/api/v1/crates/#{match[:package]}/versions" + + values + end + + # Generates a URL and checks the content at the URL for new versions + # using {Json#versions_from_content}. + # + # @param url [String] the URL of the content to check + # @param regex [Regexp, nil] a regex for matching versions in content + # @param provided_content [String, nil] content to check instead of + # fetching + # @param homebrew_curl [Boolean] whether to use brewed curl with the URL + # @return [Hash] + sig { + params( + url: String, + regex: T.nilable(Regexp), + provided_content: T.nilable(String), + homebrew_curl: T::Boolean, + _unused: T.nilable(T::Hash[Symbol, T.untyped]), + block: T.nilable(Proc), + ).returns(T::Hash[Symbol, T.untyped]) + } + def self.find_versions(url:, regex: nil, provided_content: nil, homebrew_curl: false, **_unused, &block) + match_data = { matches: {}, regex: regex, url: url } + match_data[:cached] = true if provided_content.is_a?(String) + + generated = generate_input_values(url) + return match_data if generated.blank? + + match_data[:url] = generated[:url] + + content = if provided_content.is_a?(String) + provided_content + else + match_data.merge!(Strategy.page_content(match_data[:url], homebrew_curl: homebrew_curl)) + match_data[:content] + end + return match_data if content.blank? + + Json.versions_from_content(content, regex || DEFAULT_REGEX, &block || DEFAULT_BLOCK).each do |match_text| + match_data[:matches][match_text] = Version.new(match_text) + end + + match_data + end + end + end + end +end diff --git a/Library/Homebrew/test/livecheck/strategy/crate_spec.rb b/Library/Homebrew/test/livecheck/strategy/crate_spec.rb new file mode 100644 index 0000000000..0f3dd87c4a --- /dev/null +++ b/Library/Homebrew/test/livecheck/strategy/crate_spec.rb @@ -0,0 +1,152 @@ +# frozen_string_literal: true + +require "livecheck/strategy" + +describe Homebrew::Livecheck::Strategy::Crate do + subject(:crate) { described_class } + + let(:crate_url) { "https://static.crates.io/crates/example/example-0.1.0.crate" } + let(:non_crate_url) { "https://brew.sh/test" } + + let(:regex) { /^v?(\d+(?:\.\d+)+)$/i } + + let(:generated) do + { url: "https://crates.io/api/v1/crates/example/versions" } + end + + # This is a limited subset of a `versions` response object, for the sake of + # testing. + let(:content) do + <<~EOS + { + "versions": [ + { + "crate": "example", + "created_at": "2023-01-03T00:00:00.000000+00:00", + "num": "1.0.2", + "updated_at": "2023-01-03T00:00:00.000000+00:00", + "yanked": true + }, + { + "crate": "example", + "created_at": "2023-01-02T00:00:00.000000+00:00", + "num": "1.0.1", + "updated_at": "2023-01-02T00:00:00.000000+00:00", + "yanked": false + }, + { + "crate": "example", + "created_at": "2023-01-01T00:00:00.000000+00:00", + "num": "1.0.0", + "updated_at": "2023-01-01T00:00:00.000000+00:00", + "yanked": false + } + ] + } + EOS + end + + let(:matches) { ["1.0.0", "1.0.1"] } + + let(:find_versions_return_hash) do + { + matches: { + "1.0.1" => Version.new("1.0.1"), + "1.0.0" => Version.new("1.0.0"), + }, + regex: regex, + url: generated[:url], + } + end + + let(:find_versions_cached_return_hash) do + find_versions_return_hash.merge({ cached: true }) + end + + describe "::match?" do + it "returns true for a crate URL" do + expect(crate.match?(crate_url)).to be true + end + + it "returns false for a non-crate URL" do + expect(crate.match?(non_crate_url)).to be false + end + end + + describe "::generate_input_values" do + it "returns a hash containing url for a crate URL" do + expect(crate.generate_input_values(crate_url)).to eq(generated) + end + + it "returns an empty hash for a non-crate URL" do + expect(crate.generate_input_values(non_crate_url)).to eq({}) + end + end + + describe "::find_versions" do + let(:match_data) do + cached = { + matches: matches.to_h { |v| [v, Version.new(v)] }, + regex: nil, + url: generated[:url], + cached: true, + } + + { + cached: cached, + cached_default: cached.merge({ matches: {} }), + } + end + + it "finds versions in provided content" do + expect(crate.find_versions(url: crate_url, regex: regex, provided_content: content)) + .to eq(match_data[:cached].merge({ regex: regex })) + + expect(crate.find_versions(url: crate_url, provided_content: content)) + .to eq(match_data[:cached]) + end + + it "finds versions in provided content using a block" do + expect(crate.find_versions(url: crate_url, regex: regex, provided_content: content) do |json, regex| + json["versions"]&.map do |version| + next if version["yanked"] == true + next if (match = version["num"]&.match(regex)).blank? + + match[1] + end + end).to eq(match_data[:cached].merge({ regex: regex })) + + expect(crate.find_versions(url: crate_url, provided_content: content) do |json| + json["versions"]&.map do |version| + next if version["yanked"] == true + next if (match = version["num"]&.match(regex)).blank? + + match[1] + end + end).to eq(match_data[:cached]) + end + + it "returns default match_data when block doesn't return version information" do + no_match_regex = /will_not_match/i + + expect(crate.find_versions(url: crate_url, provided_content: '{"other":true}')) + .to eq(match_data[:cached_default]) + expect(crate.find_versions(url: crate_url, provided_content: '{"versions":[{}]}')) + .to eq(match_data[:cached_default]) + expect(crate.find_versions(url: crate_url, regex: no_match_regex, provided_content: content)) + .to eq(match_data[:cached_default].merge({ regex: no_match_regex })) + end + + it "returns default match_data when url is blank" do + expect(crate.find_versions(url: "") { "1.2.3" }) + .to eq({ matches: {}, regex: nil, url: "" }) + end + + it "returns default match_data when content is blank" do + expect(crate.find_versions(url: crate_url, provided_content: "{}") { "1.2.3" }) + .to eq(match_data[:cached_default]) + expect(crate.find_versions(url: crate_url, provided_content: "") { "1.2.3" }) + .to eq(match_data[:cached_default]) + end + end +end diff --git a/docs/Brew-Livecheck.md b/docs/Brew-Livecheck.md index 34004bdf5d..07ca49b159 100644 --- a/docs/Brew-Livecheck.md +++ b/docs/Brew-Livecheck.md @@ -239,6 +239,24 @@ end You can find more information on the response JSON from this API endpoint in the related [GitHub REST API documentation](https://docs.github.com/en/rest/releases/releases?apiVersion=latest#list-releases). +#### `Crate` `strategy` block + +A `strategy` block for `Crate` receives parsed JSON data from the registry API's `versions` endpoint and either the provided or default strategy regex. The strategy uses the following logic by default, so this `strategy` block may be a good starting point for a modified approach: + +```ruby +livecheck do + url :stable + strategy :crate do |json, regex| + json["versions"]&.map do |version| + next if version["yanked"] == true + next if (match = version["num"]&.match(regex)).blank? + + match[1] + end + end +end +``` + #### `ElectronBuilder` `strategy` block A `strategy` block for `ElectronBuilder` fetches content at a URL and parses it as an electron-builder appcast in YAML format. It's used for casks of macOS applications built using the Electron framework. From 55ec4c483cdb2ad0d4ffb8cf507ef2412414b160 Mon Sep 17 00:00:00 2001 From: Sam Ford <1584702+samford@users.noreply.github.com> Date: Thu, 8 Feb 2024 17:30:11 -0500 Subject: [PATCH 2/2] Crate: Rework conditions Co-authored-by: Douglas Eichelberger Co-authored-by: Markus Reiter --- Library/Homebrew/livecheck/strategy/crate.rb | 14 +++++--------- docs/Brew-Livecheck.md | 4 ++-- 2 files changed, 7 insertions(+), 11 deletions(-) diff --git a/Library/Homebrew/livecheck/strategy/crate.rb b/Library/Homebrew/livecheck/strategy/crate.rb index 2b5d04898e..b9bfd37a6e 100644 --- a/Library/Homebrew/livecheck/strategy/crate.rb +++ b/Library/Homebrew/livecheck/strategy/crate.rb @@ -25,10 +25,8 @@ module Homebrew # a `strategy` block isn't provided. DEFAULT_BLOCK = proc do |json, regex| json["versions"]&.map do |version| - next if version["yanked"] == true - - match = version["num"]&.match(regex) - next if match.blank? + next if version["yanked"] + next unless (match = version["num"]&.match(regex)) match[1] end @@ -58,9 +56,7 @@ module Homebrew sig { params(url: String).returns(T::Hash[Symbol, T.untyped]) } def self.generate_input_values(url) values = {} - - match = url.match(URL_MATCH_REGEX) - return values if match.blank? + return values unless (match = url.match(URL_MATCH_REGEX)) values[:url] = "https://crates.io/api/v1/crates/#{match[:package]}/versions" @@ -95,13 +91,13 @@ module Homebrew match_data[:url] = generated[:url] - content = if provided_content.is_a?(String) + content = if provided_content provided_content else match_data.merge!(Strategy.page_content(match_data[:url], homebrew_curl: homebrew_curl)) match_data[:content] end - return match_data if content.blank? + return match_data unless content Json.versions_from_content(content, regex || DEFAULT_REGEX, &block || DEFAULT_BLOCK).each do |match_text| match_data[:matches][match_text] = Version.new(match_text) diff --git a/docs/Brew-Livecheck.md b/docs/Brew-Livecheck.md index 07ca49b159..36cfce54bf 100644 --- a/docs/Brew-Livecheck.md +++ b/docs/Brew-Livecheck.md @@ -248,8 +248,8 @@ livecheck do url :stable strategy :crate do |json, regex| json["versions"]&.map do |version| - next if version["yanked"] == true - next if (match = version["num"]&.match(regex)).blank? + next if version["yanked"] + next unless (match = version["num"]&.match(regex)) match[1] end