From f5d311490c79f8e76352711ad8cc9302b6a52ab9 Mon Sep 17 00:00:00 2001 From: Markus Reiter Date: Sat, 12 Dec 2020 21:56:07 +0100 Subject: [PATCH 01/70] Add `page_headers` and `page_contents` helpers. --- Library/Homebrew/livecheck/strategy.rb | 14 ++++++++++++++ Library/Homebrew/livecheck/strategy/page_match.rb | 9 ++++----- 2 files changed, 18 insertions(+), 5 deletions(-) diff --git a/Library/Homebrew/livecheck/strategy.rb b/Library/Homebrew/livecheck/strategy.rb index 4115432951..f368479843 100644 --- a/Library/Homebrew/livecheck/strategy.rb +++ b/Library/Homebrew/livecheck/strategy.rb @@ -80,6 +80,20 @@ module Homebrew (strategy.const_defined?(:PRIORITY) ? -strategy::PRIORITY : -DEFAULT_PRIORITY) end end + + def self.page_headers(url) + @headers ||= {} + @headers[url] ||= curl_output("--head", "--location", url).stdout + .split("\r\n\r\n", 2).first + .split("\r\n").drop(1) + .map { |header| header.split(/:\s*/, 2) } + .to_h.transform_keys(&:downcase) + end + + def self.page_contents(url) + @page_contents ||= {} + @page_contents[url] ||= URI.parse(url).open.read + end end end end diff --git a/Library/Homebrew/livecheck/strategy/page_match.rb b/Library/Homebrew/livecheck/strategy/page_match.rb index 49961eff59..478fa2bcda 100644 --- a/Library/Homebrew/livecheck/strategy/page_match.rb +++ b/Library/Homebrew/livecheck/strategy/page_match.rb @@ -30,8 +30,8 @@ module Homebrew URL_MATCH_REGEX = %r{^https?://}i.freeze # Whether the strategy can be applied to the provided URL. - # PageMatch will technically match any HTTP URL but it's only usable - # when the formula has a `livecheck` block containing a regex. + # PageMatch will technically match any HTTP URL but is only + # usable with a `livecheck` block containing a regex. # # @param url [String] the URL to match against # @return [Boolean] @@ -47,9 +47,8 @@ module Homebrew # content # @return [Array] def self.page_matches(url, regex) - page = URI.parse(url).open.read - matches = page.scan(regex) - matches.map(&:first).uniq + page = Strategy.page_contents(url) + page.scan(regex).map(&:first).uniq end # Checks the content at the URL for new versions, using the provided From b293acc89bc1dbc1c8bde6c40cd7dd0bc0cc745c Mon Sep 17 00:00:00 2001 From: Markus Reiter Date: Sat, 12 Dec 2020 21:56:33 +0100 Subject: [PATCH 02/70] Add `FollowRedirection` livecheck strategy. --- Library/Homebrew/livecheck/strategy.rb | 1 + .../livecheck/strategy/follow_redirection.rb | 47 +++++++++++++++++++ 2 files changed, 48 insertions(+) create mode 100644 Library/Homebrew/livecheck/strategy/follow_redirection.rb diff --git a/Library/Homebrew/livecheck/strategy.rb b/Library/Homebrew/livecheck/strategy.rb index f368479843..158fede7bd 100644 --- a/Library/Homebrew/livecheck/strategy.rb +++ b/Library/Homebrew/livecheck/strategy.rb @@ -110,5 +110,6 @@ require_relative "strategy/launchpad" require_relative "strategy/npm" require_relative "strategy/page_match" require_relative "strategy/pypi" +require_relative "strategy/follow_redirection" require_relative "strategy/sourceforge" require_relative "strategy/xorg" diff --git a/Library/Homebrew/livecheck/strategy/follow_redirection.rb b/Library/Homebrew/livecheck/strategy/follow_redirection.rb new file mode 100644 index 0000000000..f27e6176d3 --- /dev/null +++ b/Library/Homebrew/livecheck/strategy/follow_redirection.rb @@ -0,0 +1,47 @@ +# typed: false +# frozen_string_literal: true + +require_relative "page_match" + +module Homebrew + module Livecheck + module Strategy + # The {FollowRedirection} strategy follows all URL redirections and scans + # the final URL for matching text using the provided regex. + # + # @api private + class FollowRedirection + extend T::Sig + + NICE_NAME = "Follow HTTP Redirection" + + # We set the priority to zero since this cannot + # be detected automatically. + PRIORITY = 0 + + # Whether the strategy can be applied to the provided URL. + # FollowRedirection will technically match any HTTP URL but is + # only usable with a `livecheck` block containing a regex. + sig { params(url: String).returns(T::Boolean) } + def self.match?(url) + url.match?(%r{^https?://}) + end + + # Checks the final URL for new versions after following all redirections, + # using the provided regex for matching. + sig { params(url: String, regex: T.nilable(Regexp)).returns(T::Hash[Symbol, T.untyped]) } + def self.find_versions(url, regex) + raise ArgumentError, "A regular expression is required for the #{NICE_NAME} strategy." if regex.nil? + + match_data = { matches: {}, regex: regex, url: url } + + if (location = Strategy.page_headers(url)["location"]) && (match = location[regex, 1]) + match_data[:matches][match] = Version.new(match) + end + + match_data + end + end + end + end +end From c24af82a25f9ce625a81dbe19a93b0242989ae11 Mon Sep 17 00:00:00 2001 From: Markus Reiter Date: Sat, 12 Dec 2020 21:59:04 +0100 Subject: [PATCH 03/70] Add `Sparkle` livecheck strategy. --- Library/Homebrew/livecheck/strategy.rb | 1 + .../Homebrew/livecheck/strategy/sparkle.rb | 56 +++++++++++++++++++ 2 files changed, 57 insertions(+) create mode 100644 Library/Homebrew/livecheck/strategy/sparkle.rb diff --git a/Library/Homebrew/livecheck/strategy.rb b/Library/Homebrew/livecheck/strategy.rb index 158fede7bd..1fa154a96d 100644 --- a/Library/Homebrew/livecheck/strategy.rb +++ b/Library/Homebrew/livecheck/strategy.rb @@ -112,4 +112,5 @@ require_relative "strategy/page_match" require_relative "strategy/pypi" require_relative "strategy/follow_redirection" require_relative "strategy/sourceforge" +require_relative "strategy/sparkle" require_relative "strategy/xorg" diff --git a/Library/Homebrew/livecheck/strategy/sparkle.rb b/Library/Homebrew/livecheck/strategy/sparkle.rb new file mode 100644 index 0000000000..5c3015defd --- /dev/null +++ b/Library/Homebrew/livecheck/strategy/sparkle.rb @@ -0,0 +1,56 @@ +# typed: false +# frozen_string_literal: true + +require_relative "page_match" + +module Homebrew + module Livecheck + module Strategy + # The {Sparkle} strategy fetches content at a URL and parses + # its contents as a Sparkle appcast in XML format. + # + # @api private + class Sparkle + extend T::Sig + + NICE_NAME = "Sparkle" + + PRIORITY = 1 + + # Whether the strategy can be applied to the provided URL. + sig { params(url: String).returns(T::Boolean) } + def self.match?(url) + url.match?(%r{^https?://}) && + ["application/xml", "text/xml"].include?(Strategy.page_headers(url)["content-type"]) && + Strategy.page_contents(url).include?("http://www.andymatuschak.org/xml-namespaces/sparkle") + end + + # Checks the content at the URL for new versions. + sig { params(url: String, regex: T.nilable(Regexp)).returns(T::Hash[Symbol, T.untyped]) } + def self.find_versions(url, regex) + raise ArgumentError, "The #{NICE_NAME} strategy does not support regular expressions." if regex + + require "nokogiri" + + match_data = { matches: {}, regex: regex, url: url } + + contents = Strategy.page_contents(url) + + xml = Nokogiri.parse(contents) + xml.remove_namespaces! + + match = xml.xpath("//rss//channel//item//enclosure") + .map { |enclosure| [*enclosure["shortVersionString"], *enclosure["version"]].uniq } + .reject(&:empty?) + .uniq + .max_by { |versions| versions.map { |v| Version.new(v) } } + &.join(",") + + match_data[:matches][match] = Version.new(match) if match + + match_data + end + end + end + end +end From 51274fb02a4fe30275652f3474016723e75fe26f Mon Sep 17 00:00:00 2001 From: Markus Reiter Date: Sat, 12 Dec 2020 21:59:52 +0100 Subject: [PATCH 04/70] Increase priority of `GithubLatest` livecheck strategy. --- Library/Homebrew/livecheck/strategy/git.rb | 2 +- Library/Homebrew/livecheck/strategy/github_latest.rb | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Library/Homebrew/livecheck/strategy/git.rb b/Library/Homebrew/livecheck/strategy/git.rb index 4cd216bf58..94a42fdab7 100644 --- a/Library/Homebrew/livecheck/strategy/git.rb +++ b/Library/Homebrew/livecheck/strategy/git.rb @@ -26,7 +26,7 @@ module Homebrew class Git # The priority of the strategy on an informal scale of 1 to 10 (from # lowest to highest). - PRIORITY = 8 + PRIORITY = 7 # Fetches a remote Git repository's tags using `git ls-remote --tags` # and parses the command's output. If a regex is provided, it will be diff --git a/Library/Homebrew/livecheck/strategy/github_latest.rb b/Library/Homebrew/livecheck/strategy/github_latest.rb index d3afcddbf4..4868ead895 100644 --- a/Library/Homebrew/livecheck/strategy/github_latest.rb +++ b/Library/Homebrew/livecheck/strategy/github_latest.rb @@ -37,7 +37,7 @@ module Homebrew # A priority of zero causes livecheck to skip the strategy. We do this # for {GithubLatest} so we can selectively apply the strategy using # `strategy :github_latest` in a `livecheck` block. - PRIORITY = 0 + PRIORITY = 8 # The `Regexp` used to determine if the strategy applies to the URL. URL_MATCH_REGEX = %r{//github\.com(?:/downloads)?(?:/[^/]+){2}}i.freeze From 05971c096581cbf68e00b42a474cffccceb798b8 Mon Sep 17 00:00:00 2001 From: Markus Reiter Date: Sun, 13 Dec 2020 12:18:37 +0100 Subject: [PATCH 05/70] Fix order of `livecheck` in casks. --- Library/Homebrew/rubocops/cask/constants/stanza.rb | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Library/Homebrew/rubocops/cask/constants/stanza.rb b/Library/Homebrew/rubocops/cask/constants/stanza.rb index 93a9b67d55..7e8874eb3d 100644 --- a/Library/Homebrew/rubocops/cask/constants/stanza.rb +++ b/Library/Homebrew/rubocops/cask/constants/stanza.rb @@ -8,8 +8,7 @@ module RuboCop STANZA_GROUPS = [ [:version, :sha256], [:language], - [:url, :appcast, :name, :desc, :homepage], - [:livecheck], + [:url, :appcast, :livecheck, :name, :desc, :homepage], [ :auto_updates, :conflicts_with, From 7762a597d66e8c0e906bad9a483e0520fadd2441 Mon Sep 17 00:00:00 2001 From: Markus Reiter Date: Sun, 13 Dec 2020 12:19:46 +0100 Subject: [PATCH 06/70] Don't preprocess URL for Sparkle strategy. --- Library/Homebrew/livecheck/livecheck.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/Library/Homebrew/livecheck/livecheck.rb b/Library/Homebrew/livecheck/livecheck.rb index 9025a1fb1e..8bc6b69a09 100644 --- a/Library/Homebrew/livecheck/livecheck.rb +++ b/Library/Homebrew/livecheck/livecheck.rb @@ -29,6 +29,7 @@ module Homebrew STRATEGY_SYMBOLS_TO_SKIP_PREPROCESS_URL = [ :github_latest, + :sparkle, :page_match, ].freeze From 10b5548eace342387afc97f5a809792dbd94b4b5 Mon Sep 17 00:00:00 2001 From: Markus Reiter Date: Sun, 13 Dec 2020 12:21:59 +0100 Subject: [PATCH 07/70] Fix `page_headers` method for multiple headers. --- Library/Homebrew/livecheck/strategy.rb | 26 +++++++++++++++++++++----- 1 file changed, 21 insertions(+), 5 deletions(-) diff --git a/Library/Homebrew/livecheck/strategy.rb b/Library/Homebrew/livecheck/strategy.rb index 1fa154a96d..6a566ad16b 100644 --- a/Library/Homebrew/livecheck/strategy.rb +++ b/Library/Homebrew/livecheck/strategy.rb @@ -83,11 +83,27 @@ module Homebrew def self.page_headers(url) @headers ||= {} - @headers[url] ||= curl_output("--head", "--location", url).stdout - .split("\r\n\r\n", 2).first - .split("\r\n").drop(1) - .map { |header| header.split(/:\s*/, 2) } - .to_h.transform_keys(&:downcase) + + return @headers[url] if @headers.key?(url) + + stdout, _, status = curl_output( + "--head", "--request", "GET", "--silent", "--location", + "--connect-timeout", 5, "--retry-max-time", 15, "--max-time", 10, + url + ) + return {} unless status.success? + + headers = {} + + while stdout.match?(/\AHTTP.*\r$/) + h, stdout = stdout.split("\r\n\r\n", 2) + + headers = headers.merge(h.split("\r\n").drop(1) + .map { |header| header.split(/:\s*/, 2) } + .to_h.transform_keys(&:downcase)) + end + + @headers[url] = headers end def self.page_contents(url) From b3c46ba2b931aa8d6e99eaf1733f0349f2c09a19 Mon Sep 17 00:00:00 2001 From: Markus Reiter Date: Sun, 13 Dec 2020 12:23:20 +0100 Subject: [PATCH 08/70] Allow extracting URL in Sparkle strategy. --- Library/Homebrew/livecheck.rb | 6 ++++- Library/Homebrew/livecheck/livecheck.rb | 2 +- .../Homebrew/livecheck/strategy/sparkle.rb | 24 ++++++++++++------- 3 files changed, 22 insertions(+), 10 deletions(-) diff --git a/Library/Homebrew/livecheck.rb b/Library/Homebrew/livecheck.rb index 7fb6c3bb2b..2560b80737 100644 --- a/Library/Homebrew/livecheck.rb +++ b/Library/Homebrew/livecheck.rb @@ -67,7 +67,9 @@ class Livecheck # # @param symbol [Symbol] symbol for the desired strategy # @return [Symbol, nil] - def strategy(symbol = nil) + def strategy(symbol = nil, &block) + @strategy_block = block if block + case symbol when nil @strategy @@ -78,6 +80,8 @@ class Livecheck end end + attr_reader :strategy_block + # Sets the `@url` instance variable to the provided argument or returns the # `@url` instance variable when no argument is provided. The argument can be # a `String` (a URL) or a supported `Symbol` corresponding to a URL in the diff --git a/Library/Homebrew/livecheck/livecheck.rb b/Library/Homebrew/livecheck/livecheck.rb index 8bc6b69a09..1b31956e1a 100644 --- a/Library/Homebrew/livecheck/livecheck.rb +++ b/Library/Homebrew/livecheck/livecheck.rb @@ -480,7 +480,7 @@ module Homebrew next if strategy.blank? - strategy_data = strategy.find_versions(url, livecheck_regex) + strategy_data = strategy.find_versions(url, livecheck_regex, &livecheck.strategy_block) match_version_map = strategy_data[:matches] regex = strategy_data[:regex] diff --git a/Library/Homebrew/livecheck/strategy/sparkle.rb b/Library/Homebrew/livecheck/strategy/sparkle.rb index 5c3015defd..3372008723 100644 --- a/Library/Homebrew/livecheck/strategy/sparkle.rb +++ b/Library/Homebrew/livecheck/strategy/sparkle.rb @@ -1,6 +1,7 @@ # typed: false # frozen_string_literal: true +require "bundle_version" require_relative "page_match" module Homebrew @@ -27,7 +28,7 @@ module Homebrew # Checks the content at the URL for new versions. sig { params(url: String, regex: T.nilable(Regexp)).returns(T::Hash[Symbol, T.untyped]) } - def self.find_versions(url, regex) + def self.find_versions(url, regex, &block) raise ArgumentError, "The #{NICE_NAME} strategy does not support regular expressions." if regex require "nokogiri" @@ -39,14 +40,21 @@ module Homebrew xml = Nokogiri.parse(contents) xml.remove_namespaces! - match = xml.xpath("//rss//channel//item//enclosure") - .map { |enclosure| [*enclosure["shortVersionString"], *enclosure["version"]].uniq } - .reject(&:empty?) - .uniq - .max_by { |versions| versions.map { |v| Version.new(v) } } - &.join(",") + enclosure = + xml.xpath("//rss//channel//item//enclosure") + .map { |e| { url: e["url"], version: BundleVersion.new(e["shortVersionString"], e["version"]) } } + .max_by { |e| e[:version] } - match_data[:matches][match] = Version.new(match) if match + if enclosure + match = if block + enclosure[:version] = Cask::DSL::Version.new(enclosure[:version].nice_version) + block.call(enclosure).to_s + else + enclosure[:version].nice_version + end + + match_data[:matches][match] = Version.new(match) + end match_data end From df702964032f0a48ff4a6b838ec9552e3392e0f2 Mon Sep 17 00:00:00 2001 From: Markus Reiter Date: Sun, 13 Dec 2020 12:23:59 +0100 Subject: [PATCH 09/70] Allow extracting file name in `FollowRedirection` strategy. --- .../livecheck/strategy/follow_redirection.rb | 32 ++++++++++++++++--- 1 file changed, 27 insertions(+), 5 deletions(-) diff --git a/Library/Homebrew/livecheck/strategy/follow_redirection.rb b/Library/Homebrew/livecheck/strategy/follow_redirection.rb index f27e6176d3..d50b4c8eb3 100644 --- a/Library/Homebrew/livecheck/strategy/follow_redirection.rb +++ b/Library/Homebrew/livecheck/strategy/follow_redirection.rb @@ -30,15 +30,37 @@ module Homebrew # Checks the final URL for new versions after following all redirections, # using the provided regex for matching. sig { params(url: String, regex: T.nilable(Regexp)).returns(T::Hash[Symbol, T.untyped]) } - def self.find_versions(url, regex) - raise ArgumentError, "A regular expression is required for the #{NICE_NAME} strategy." if regex.nil? - + def self.find_versions(url, regex, &block) match_data = { matches: {}, regex: regex, url: url } - if (location = Strategy.page_headers(url)["location"]) && (match = location[regex, 1]) - match_data[:matches][match] = Version.new(match) + data = { headers: Strategy.page_headers(url) } + + if (filename = data[:headers]["content-disposition"]) + if regex + data[:version] ||= location[regex, 1] + else + v = Version.parse(filename, detected_from_url: true) + data[:version] ||= v.to_s unless v.null? + end end + if (location = data[:headers]["location"]) + if regex + data[:version] ||= location[regex, 1] + else + v = Version.parse(location, detected_from_url: true) + data[:version] ||= v.to_s unless v.null? + end + end + + version = if block + block.call(data) + else + data[:version] + end + + match_data[:matches][version] = Version.new(version) if version + match_data end end From aba4eac8ab3468a581c91c1e814483a1185151d4 Mon Sep 17 00:00:00 2001 From: Markus Reiter Date: Sun, 13 Dec 2020 12:24:25 +0100 Subject: [PATCH 10/70] Support multiple capture groups in `PageMatch` strategy. --- Library/Homebrew/livecheck/strategy/page_match.rb | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/Library/Homebrew/livecheck/strategy/page_match.rb b/Library/Homebrew/livecheck/strategy/page_match.rb index 478fa2bcda..0249c3629b 100644 --- a/Library/Homebrew/livecheck/strategy/page_match.rb +++ b/Library/Homebrew/livecheck/strategy/page_match.rb @@ -48,7 +48,19 @@ module Homebrew # @return [Array] def self.page_matches(url, regex) page = Strategy.page_contents(url) - page.scan(regex).map(&:first).uniq + + matches = page.scan(regex) + + regex_names = regex.names.map(&:to_sym) + + if regex_names.count > 1 + matches.map do |match| + match_data = regex_names.zip(match).to_h + regex_names.sort.map { |name| match_data[name] }.join(",") + end.uniq + else + matches.map(&:first).uniq + end end # Checks the content at the URL for new versions, using the provided From 1857e0ebfac0a9d8f62b71362e64a23b1bc590a1 Mon Sep 17 00:00:00 2001 From: Markus Reiter Date: Sun, 13 Dec 2020 17:09:25 +0100 Subject: [PATCH 11/70] Skip discontinued, latest and unversioned casks. --- Library/Homebrew/livecheck/livecheck.rb | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/Library/Homebrew/livecheck/livecheck.rb b/Library/Homebrew/livecheck/livecheck.rb index 1b31956e1a..8af477178a 100644 --- a/Library/Homebrew/livecheck/livecheck.rb +++ b/Library/Homebrew/livecheck/livecheck.rb @@ -269,6 +269,7 @@ module Homebrew # @return [Hash, nil, Boolean] def skip_conditions(formula_or_cask, json: false, full_name: false, quiet: false, verbose: false) formula = formula_or_cask if formula_or_cask.is_a?(Formula) + cask = formula_or_cask if formula_or_cask.is_a?(Cask::Cask) if formula&.deprecated? && !formula.livecheckable? return status_hash(formula, "deprecated", full_name: full_name, verbose: verbose) if json @@ -277,6 +278,13 @@ module Homebrew return end + if cask&.discontinued? && !cask.livecheckable? + return status_hash(cask, "discontinued", args: args) if args.json? + + puts "#{Tty.red}#{cask_name(cask, args: args)}#{Tty.reset} : discontinued" unless args.quiet? + return + end + if formula&.disabled? && !formula.livecheckable? return status_hash(formula, "disabled", full_name: full_name, verbose: verbose) if json @@ -291,6 +299,20 @@ module Homebrew return end + if cask&.version&.latest? && !cask.livecheckable? + return status_hash(cask, "latest", args: args) if args.json? + + puts "#{Tty.red}#{cask_name(cask, args: args)}#{Tty.reset} : latest" unless args.quiet? + return + end + + if cask&.url&.unversioned? && !cask.livecheckable? + return status_hash(cask, "unversioned", args: args) if args.json? + + puts "#{Tty.red}#{cask_name(cask, args: args)}#{Tty.reset} : unversioned" unless args.quiet? + return + end + if formula&.head_only? && !formula.any_version_installed? head_only_msg = "HEAD only formula must be installed to be livecheckable" return status_hash(formula, "error", [head_only_msg], full_name: full_name, verbose: verbose) if json From 7555556be806656ff6b4fb1899d9a8f53678b237 Mon Sep 17 00:00:00 2001 From: Markus Reiter Date: Mon, 14 Dec 2020 02:07:07 +0100 Subject: [PATCH 12/70] Support block in `PageMatch` strategy. --- .../Homebrew/livecheck/strategy/page_match.rb | 29 ++++++++++--------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/Library/Homebrew/livecheck/strategy/page_match.rb b/Library/Homebrew/livecheck/strategy/page_match.rb index 0249c3629b..4e778bae6f 100644 --- a/Library/Homebrew/livecheck/strategy/page_match.rb +++ b/Library/Homebrew/livecheck/strategy/page_match.rb @@ -46,21 +46,22 @@ module Homebrew # @param regex [Regexp] a regex used for matching versions in the # content # @return [Array] - def self.page_matches(url, regex) + def self.page_matches(url, regex, &block) page = Strategy.page_contents(url) - matches = page.scan(regex) - - regex_names = regex.names.map(&:to_sym) - - if regex_names.count > 1 - matches.map do |match| - match_data = regex_names.zip(match).to_h - regex_names.sort.map { |name| match_data[name] }.join(",") - end.uniq - else - matches.map(&:first).uniq + if block + data = { page: page } + case (value = block.call(data)) + when String + return [value] + when Array + return value + else + raise TypeError, "Return value of `strategy :page_match` block must be a string or array." + end end + + page.scan(regex).map(&:first).uniq end # Checks the content at the URL for new versions, using the provided @@ -69,10 +70,10 @@ module Homebrew # @param url [String] the URL of the content to check # @param regex [Regexp] a regex used for matching versions in content # @return [Hash] - def self.find_versions(url, regex) + def self.find_versions(url, regex, &block) match_data = { matches: {}, regex: regex, url: url } - page_matches(url, regex).each do |match| + page_matches(url, regex, &block).each do |match| match_data[:matches][match] = Version.new(match) end From 6a125401c7787db228da0835dd3507fe0df456fc Mon Sep 17 00:00:00 2001 From: Markus Reiter Date: Mon, 14 Dec 2020 02:08:00 +0100 Subject: [PATCH 13/70] Unescape URL before parsing a version from it. --- Library/Homebrew/version.rb | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/Library/Homebrew/version.rb b/Library/Homebrew/version.rb index 8c1735e9a1..c3c86456ac 100644 --- a/Library/Homebrew/version.rb +++ b/Library/Homebrew/version.rb @@ -318,11 +318,13 @@ class Version end def self.parse(spec, detected_from_url: false) - version = _parse(spec) + version = _parse(spec, detected_from_url: detected_from_url) version.nil? ? NULL : new(version, detected_from_url: detected_from_url) end - def self._parse(spec) + def self._parse(spec, detected_from_url:) + spec = CGI.unescape(spec.to_s) if detected_from_url + spec = Pathname.new(spec) unless spec.is_a? Pathname spec_s = spec.to_s @@ -465,7 +467,6 @@ class Version m = /[-.vV]?((?:\d+\.)+\d+(?:[-_.]?(?i:alpha|beta|pre|rc)\.?\d{,2})?)/.match(spec_s) return m.captures.first unless m.nil? end - private_class_method :_parse def initialize(val, detected_from_url: false) From 9017778326faa87ac1d01399aef0cebe58a0f9d7 Mon Sep 17 00:00:00 2001 From: Markus Reiter Date: Mon, 14 Dec 2020 02:08:35 +0100 Subject: [PATCH 14/70] Don't use `Cask::DSL::Version` in `Sparkle` strategy. --- Library/Homebrew/livecheck/strategy/sparkle.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Library/Homebrew/livecheck/strategy/sparkle.rb b/Library/Homebrew/livecheck/strategy/sparkle.rb index 3372008723..fa80a7a5f8 100644 --- a/Library/Homebrew/livecheck/strategy/sparkle.rb +++ b/Library/Homebrew/livecheck/strategy/sparkle.rb @@ -47,7 +47,7 @@ module Homebrew if enclosure match = if block - enclosure[:version] = Cask::DSL::Version.new(enclosure[:version].nice_version) + enclosure[:version] = enclosure[:version].nice_version block.call(enclosure).to_s else enclosure[:version].nice_version From 0fe3bf7c7fe2ad76fdf803ed7f2d8353f1a98795 Mon Sep 17 00:00:00 2001 From: Markus Reiter Date: Mon, 14 Dec 2020 02:09:23 +0100 Subject: [PATCH 15/70] Only use `Sparkle` strategy if URL is specified explicitly. --- Library/Homebrew/livecheck/livecheck.rb | 8 +++++--- Library/Homebrew/livecheck/strategy.rb | 8 ++++++-- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/Library/Homebrew/livecheck/livecheck.rb b/Library/Homebrew/livecheck/livecheck.rb index 8af477178a..f29c62144c 100644 --- a/Library/Homebrew/livecheck/livecheck.rb +++ b/Library/Homebrew/livecheck/livecheck.rb @@ -435,9 +435,9 @@ module Homebrew has_livecheckable = formula_or_cask.livecheckable? livecheck = formula_or_cask.livecheck + livecheck_url = livecheck.url livecheck_regex = livecheck.regex livecheck_strategy = livecheck.strategy - livecheck_url = livecheck.url urls = [livecheck_url] if livecheck_url.present? urls ||= checkable_urls(formula_or_cask) @@ -475,7 +475,9 @@ module Homebrew strategies = Strategy.from_url( url, livecheck_strategy: livecheck_strategy, + url_provided: livecheck_url.present?, regex_provided: livecheck_regex.present?, + block_provided: livecheck.strategy_block.present?, ) strategy = Strategy.from_symbol(livecheck_strategy) strategy ||= strategies.first @@ -490,8 +492,8 @@ module Homebrew puts "Regex: #{livecheck_regex.inspect}" if livecheck_regex.present? end - if livecheck_strategy == :page_match && livecheck_regex.blank? - odebug "#{strategy_name} strategy requires a regex" + if livecheck_strategy == :page_match && (livecheck_regex.blank? && livecheck.strategy_block.blank?) + odebug "#{strategy_name} strategy requires a regex or block" next end diff --git a/Library/Homebrew/livecheck/strategy.rb b/Library/Homebrew/livecheck/strategy.rb index 6a566ad16b..fecfc44a08 100644 --- a/Library/Homebrew/livecheck/strategy.rb +++ b/Library/Homebrew/livecheck/strategy.rb @@ -57,12 +57,16 @@ module Homebrew # @param regex_provided [Boolean] whether a regex is provided in the # `livecheck` block # @return [Array] - def from_url(url, livecheck_strategy: nil, regex_provided: nil) + def from_url(url, livecheck_strategy: nil, url_provided: nil, regex_provided: nil, block_provided: nil) usable_strategies = strategies.values.select do |strategy| if strategy == PageMatch # Only treat the `PageMatch` strategy as usable if a regex is # present in the `livecheck` block - next unless regex_provided + next unless (regex_provided || block_provided) + elsif strategy == Sparkle && (livecheck_strategy || !url_provided) + # Skip the `Sparkle` strategy if a strategy is specified explicitly + # or if the URL is not specified explicitly. + next elsif strategy.const_defined?(:PRIORITY) && !strategy::PRIORITY.positive? && from_symbol(livecheck_strategy) != strategy From b9741dd1e69f5a3dd59005ef4f89a63867347e3a Mon Sep 17 00:00:00 2001 From: Markus Reiter Date: Mon, 14 Dec 2020 02:10:38 +0100 Subject: [PATCH 16/70] Try getting headers with both user agents. --- Library/Homebrew/livecheck/strategy.rb | 30 +++++++++++++++----------- 1 file changed, 17 insertions(+), 13 deletions(-) diff --git a/Library/Homebrew/livecheck/strategy.rb b/Library/Homebrew/livecheck/strategy.rb index fecfc44a08..91b32c0fa3 100644 --- a/Library/Homebrew/livecheck/strategy.rb +++ b/Library/Homebrew/livecheck/strategy.rb @@ -90,24 +90,28 @@ module Homebrew return @headers[url] if @headers.key?(url) - stdout, _, status = curl_output( - "--head", "--request", "GET", "--silent", "--location", - "--connect-timeout", 5, "--retry-max-time", 15, "--max-time", 10, - url - ) - return {} unless status.success? - headers = {} - while stdout.match?(/\AHTTP.*\r$/) - h, stdout = stdout.split("\r\n\r\n", 2) + [:default, :browser].each do |user_agent| + stdout, _, status = curl_output( + "--fail", "--head", "--request", "GET", "--silent", "--location", + "--connect-timeout", 5, "--retry-max-time", 15, "--max-time", 10, + url, + user_agent: user_agent + ) - headers = headers.merge(h.split("\r\n").drop(1) - .map { |header| header.split(/:\s*/, 2) } - .to_h.transform_keys(&:downcase)) + while stdout.match?(/\AHTTP.*\r$/) + h, stdout = stdout.split("\r\n\r\n", 2) + + headers = headers.merge(h.split("\r\n").drop(1) + .map { |header| header.split(/:\s*/, 2) } + .to_h.transform_keys(&:downcase)) + end + + return headers if status.success? end - @headers[url] = headers + headers end def self.page_contents(url) From 775c2fd245c5a2eb9976885778ff91d7b6c66b35 Mon Sep 17 00:00:00 2001 From: Markus Reiter Date: Mon, 14 Dec 2020 02:16:01 +0100 Subject: [PATCH 17/70] Clarify error message. --- Library/Homebrew/livecheck/strategy/page_match.rb | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Library/Homebrew/livecheck/strategy/page_match.rb b/Library/Homebrew/livecheck/strategy/page_match.rb index 4e778bae6f..067a61c2f3 100644 --- a/Library/Homebrew/livecheck/strategy/page_match.rb +++ b/Library/Homebrew/livecheck/strategy/page_match.rb @@ -49,6 +49,8 @@ module Homebrew def self.page_matches(url, regex, &block) page = Strategy.page_contents(url) + odebug "Page Contents", page + if block data = { page: page } case (value = block.call(data)) @@ -57,7 +59,7 @@ module Homebrew when Array return value else - raise TypeError, "Return value of `strategy :page_match` block must be a string or array." + raise TypeError, "Return value of `strategy :page_match` block must be a string or array of strings." end end From 8b7f38faf0e770c8446f328364e639e78307f062 Mon Sep 17 00:00:00 2001 From: Markus Reiter Date: Mon, 14 Dec 2020 02:16:08 +0100 Subject: [PATCH 18/70] Fix code style. --- Library/Homebrew/livecheck/livecheck.rb | 2 +- Library/Homebrew/livecheck/strategy.rb | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Library/Homebrew/livecheck/livecheck.rb b/Library/Homebrew/livecheck/livecheck.rb index f29c62144c..e6f3bede10 100644 --- a/Library/Homebrew/livecheck/livecheck.rb +++ b/Library/Homebrew/livecheck/livecheck.rb @@ -477,7 +477,7 @@ module Homebrew livecheck_strategy: livecheck_strategy, url_provided: livecheck_url.present?, regex_provided: livecheck_regex.present?, - block_provided: livecheck.strategy_block.present?, + block_provided: livecheck.strategy_block.present?, ) strategy = Strategy.from_symbol(livecheck_strategy) strategy ||= strategies.first diff --git a/Library/Homebrew/livecheck/strategy.rb b/Library/Homebrew/livecheck/strategy.rb index 91b32c0fa3..5527166bb9 100644 --- a/Library/Homebrew/livecheck/strategy.rb +++ b/Library/Homebrew/livecheck/strategy.rb @@ -62,7 +62,7 @@ module Homebrew if strategy == PageMatch # Only treat the `PageMatch` strategy as usable if a regex is # present in the `livecheck` block - next unless (regex_provided || block_provided) + next unless regex_provided || block_provided elsif strategy == Sparkle && (livecheck_strategy || !url_provided) # Skip the `Sparkle` strategy if a strategy is specified explicitly # or if the URL is not specified explicitly. From f76b5f105f69eea12913eeda228b74ba3e591560 Mon Sep 17 00:00:00 2001 From: Markus Reiter Date: Mon, 14 Dec 2020 02:35:26 +0100 Subject: [PATCH 19/70] Skip fetching headers if URL has `.xml` extension. --- Library/Homebrew/livecheck/strategy/sparkle.rb | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/Library/Homebrew/livecheck/strategy/sparkle.rb b/Library/Homebrew/livecheck/strategy/sparkle.rb index fa80a7a5f8..1f1b8a8ade 100644 --- a/Library/Homebrew/livecheck/strategy/sparkle.rb +++ b/Library/Homebrew/livecheck/strategy/sparkle.rb @@ -21,9 +21,18 @@ module Homebrew # Whether the strategy can be applied to the provided URL. sig { params(url: String).returns(T::Boolean) } def self.match?(url) - url.match?(%r{^https?://}) && - ["application/xml", "text/xml"].include?(Strategy.page_headers(url)["content-type"]) && - Strategy.page_contents(url).include?("http://www.andymatuschak.org/xml-namespaces/sparkle") + return false unless url.match?(%r{^https?://}) + + xml = url.end_with?('.xml') + xml ||= begin + headers = Strategy.page_headers(url) + content_type = headers["content-type"]&.split(';', 2)&.first + ["application/xml", "text/xml"].include?(content_type) + end + return false unless xml + + contents = Strategy.page_contents(url) + contents.include?("http://www.andymatuschak.org/xml-namespaces/sparkle") end # Checks the content at the URL for new versions. From cf40f7b8f648a28313f82c18b0059cd01a667e55 Mon Sep 17 00:00:00 2001 From: Markus Reiter Date: Mon, 14 Dec 2020 02:49:32 +0100 Subject: [PATCH 20/70] Rename `FollowRedirection` to `HeaderMatch`. --- Library/Homebrew/livecheck/strategy.rb | 2 +- .../{follow_redirection.rb => header_match.rb} | 13 +++++++------ 2 files changed, 8 insertions(+), 7 deletions(-) rename Library/Homebrew/livecheck/strategy/{follow_redirection.rb => header_match.rb} (85%) diff --git a/Library/Homebrew/livecheck/strategy.rb b/Library/Homebrew/livecheck/strategy.rb index 5527166bb9..a340b53446 100644 --- a/Library/Homebrew/livecheck/strategy.rb +++ b/Library/Homebrew/livecheck/strategy.rb @@ -130,11 +130,11 @@ require_relative "strategy/github_latest" require_relative "strategy/gnome" require_relative "strategy/gnu" require_relative "strategy/hackage" +require_relative "strategy/header_match" require_relative "strategy/launchpad" require_relative "strategy/npm" require_relative "strategy/page_match" require_relative "strategy/pypi" -require_relative "strategy/follow_redirection" require_relative "strategy/sourceforge" require_relative "strategy/sparkle" require_relative "strategy/xorg" diff --git a/Library/Homebrew/livecheck/strategy/follow_redirection.rb b/Library/Homebrew/livecheck/strategy/header_match.rb similarity index 85% rename from Library/Homebrew/livecheck/strategy/follow_redirection.rb rename to Library/Homebrew/livecheck/strategy/header_match.rb index d50b4c8eb3..7aedecac1c 100644 --- a/Library/Homebrew/livecheck/strategy/follow_redirection.rb +++ b/Library/Homebrew/livecheck/strategy/header_match.rb @@ -6,22 +6,23 @@ require_relative "page_match" module Homebrew module Livecheck module Strategy - # The {FollowRedirection} strategy follows all URL redirections and scans - # the final URL for matching text using the provided regex. + # The {HeaderMatch} strategy follows all URL redirections and scans + # the resulting headers for matching text using the provided regex. # # @api private - class FollowRedirection + class HeaderMatch extend T::Sig - NICE_NAME = "Follow HTTP Redirection" + NICE_NAME = "Match HTTP Headers" # We set the priority to zero since this cannot # be detected automatically. PRIORITY = 0 # Whether the strategy can be applied to the provided URL. - # FollowRedirection will technically match any HTTP URL but is - # only usable with a `livecheck` block containing a regex. + # The strategy will technically match any HTTP URL but is + # only usable with a `livecheck` block containing a regex + # or block. sig { params(url: String).returns(T::Boolean) } def self.match?(url) url.match?(%r{^https?://}) From 82535696fad17f62c6f82fc4d7f43b9f02225da2 Mon Sep 17 00:00:00 2001 From: Markus Reiter Date: Mon, 14 Dec 2020 04:35:26 +0100 Subject: [PATCH 21/70] Fix code style. --- Library/Homebrew/livecheck/strategy/sparkle.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Library/Homebrew/livecheck/strategy/sparkle.rb b/Library/Homebrew/livecheck/strategy/sparkle.rb index 1f1b8a8ade..b6147e5611 100644 --- a/Library/Homebrew/livecheck/strategy/sparkle.rb +++ b/Library/Homebrew/livecheck/strategy/sparkle.rb @@ -23,10 +23,10 @@ module Homebrew def self.match?(url) return false unless url.match?(%r{^https?://}) - xml = url.end_with?('.xml') + xml = url.end_with?(".xml") xml ||= begin headers = Strategy.page_headers(url) - content_type = headers["content-type"]&.split(';', 2)&.first + content_type = headers["content-type"]&.split(";", 2)&.first ["application/xml", "text/xml"].include?(content_type) end return false unless xml From 5ae726e2b4be5f078db376b1071181d6af6d6628 Mon Sep 17 00:00:00 2001 From: Markus Reiter Date: Mon, 14 Dec 2020 04:36:05 +0100 Subject: [PATCH 22/70] Match Sparkle appcasts with HTTPS namespace. --- Library/Homebrew/livecheck/strategy/sparkle.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Library/Homebrew/livecheck/strategy/sparkle.rb b/Library/Homebrew/livecheck/strategy/sparkle.rb index b6147e5611..2aaf3f31c9 100644 --- a/Library/Homebrew/livecheck/strategy/sparkle.rb +++ b/Library/Homebrew/livecheck/strategy/sparkle.rb @@ -32,7 +32,7 @@ module Homebrew return false unless xml contents = Strategy.page_contents(url) - contents.include?("http://www.andymatuschak.org/xml-namespaces/sparkle") + contents.match?(%r{https?://www.andymatuschak.org/xml-namespaces/sparkle}) end # Checks the content at the URL for new versions. From a8d893f1347a742ac30643c503818a3a849034da Mon Sep 17 00:00:00 2001 From: Markus Reiter Date: Mon, 14 Dec 2020 05:04:14 +0100 Subject: [PATCH 23/70] Handle Sparkle appcast which specify the version next to the enclosure. --- .../Homebrew/livecheck/strategy/sparkle.rb | 32 +++++++++++++------ 1 file changed, 23 insertions(+), 9 deletions(-) diff --git a/Library/Homebrew/livecheck/strategy/sparkle.rb b/Library/Homebrew/livecheck/strategy/sparkle.rb index 2aaf3f31c9..e5bc6da8e7 100644 --- a/Library/Homebrew/livecheck/strategy/sparkle.rb +++ b/Library/Homebrew/livecheck/strategy/sparkle.rb @@ -46,20 +46,34 @@ module Homebrew contents = Strategy.page_contents(url) - xml = Nokogiri.parse(contents) + xml = Nokogiri::XML(contents) xml.remove_namespaces! - enclosure = - xml.xpath("//rss//channel//item//enclosure") - .map { |e| { url: e["url"], version: BundleVersion.new(e["shortVersionString"], e["version"]) } } - .max_by { |e| e[:version] } + items = xml.xpath("//rss//channel//item").map do |item| + enclosure = (item > "enclosure").first - if enclosure + next unless enclosure + + short_version ||= enclosure["shortVersionString"] + version ||= enclosure["version"] + + short_version ||= (item > "shortVersionString").first&.text + version ||= (item > "version").first&.text + + { + url: enclosure["url"], + version: BundleVersion.new(short_version, version), + } + end.compact + + item = items.max_by { |e| e[:version] } + + if item match = if block - enclosure[:version] = enclosure[:version].nice_version - block.call(enclosure).to_s + item[:version] = item[:version].nice_version + block.call(item).to_s else - enclosure[:version].nice_version + item[:version].nice_version end match_data[:matches][match] = Version.new(match) From 347a58f164266e0632099a780bb3c16d87db5813 Mon Sep 17 00:00:00 2001 From: Markus Reiter Date: Mon, 14 Dec 2020 05:04:50 +0100 Subject: [PATCH 24/70] Don't skip Sparkle strategy if explicitly specified. --- Library/Homebrew/livecheck/strategy.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Library/Homebrew/livecheck/strategy.rb b/Library/Homebrew/livecheck/strategy.rb index a340b53446..ede94b9708 100644 --- a/Library/Homebrew/livecheck/strategy.rb +++ b/Library/Homebrew/livecheck/strategy.rb @@ -63,7 +63,7 @@ module Homebrew # Only treat the `PageMatch` strategy as usable if a regex is # present in the `livecheck` block next unless regex_provided || block_provided - elsif strategy == Sparkle && (livecheck_strategy || !url_provided) + elsif strategy == Sparkle && (from_symbol(livecheck_strategy) != Sparkle || !url_provided) # Skip the `Sparkle` strategy if a strategy is specified explicitly # or if the URL is not specified explicitly. next From d5e1d004e876cb2185e4ad5f3f918213a0e6e4cf Mon Sep 17 00:00:00 2001 From: Markus Reiter Date: Mon, 14 Dec 2020 05:41:41 +0100 Subject: [PATCH 25/70] Fix Sparkle strategy when only URLs but no versions are found. --- .../Homebrew/livecheck/strategy/sparkle.rb | 22 +++++++++++++------ 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/Library/Homebrew/livecheck/strategy/sparkle.rb b/Library/Homebrew/livecheck/strategy/sparkle.rb index e5bc6da8e7..f22bd0fc4d 100644 --- a/Library/Homebrew/livecheck/strategy/sparkle.rb +++ b/Library/Homebrew/livecheck/strategy/sparkle.rb @@ -32,7 +32,13 @@ module Homebrew return false unless xml contents = Strategy.page_contents(url) - contents.match?(%r{https?://www.andymatuschak.org/xml-namespaces/sparkle}) + + return true if contents.match?(%r{https?://www.andymatuschak.org/xml-namespaces/sparkle}) + + contents.include?("rss") && + contents.include?("channel") && + contents.include?("item") && + contents.include?("enclosure") end # Checks the content at the URL for new versions. @@ -60,23 +66,25 @@ module Homebrew short_version ||= (item > "shortVersionString").first&.text version ||= (item > "version").first&.text - { + data = { url: enclosure["url"], - version: BundleVersion.new(short_version, version), - } + version: short_version || version ? BundleVersion.new(short_version, version) : nil, + }.compact + + data unless data.empty? end.compact item = items.max_by { |e| e[:version] } if item match = if block - item[:version] = item[:version].nice_version + item[:version] = item[:version]&.nice_version block.call(item).to_s else - item[:version].nice_version + item[:version]&.nice_version end - match_data[:matches][match] = Version.new(match) + match_data[:matches][match] = Version.new(match) if match end match_data From 605c33c70ce5e57d6a7762afc0c52faf70d4a692 Mon Sep 17 00:00:00 2001 From: Markus Reiter Date: Mon, 14 Dec 2020 06:06:49 +0100 Subject: [PATCH 26/70] Improve matching of Sparkle strategy. --- Library/Homebrew/livecheck/strategy.rb | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/Library/Homebrew/livecheck/strategy.rb b/Library/Homebrew/livecheck/strategy.rb index ede94b9708..d82b1e1560 100644 --- a/Library/Homebrew/livecheck/strategy.rb +++ b/Library/Homebrew/livecheck/strategy.rb @@ -63,10 +63,11 @@ module Homebrew # Only treat the `PageMatch` strategy as usable if a regex is # present in the `livecheck` block next unless regex_provided || block_provided - elsif strategy == Sparkle && (from_symbol(livecheck_strategy) != Sparkle || !url_provided) - # Skip the `Sparkle` strategy if a strategy is specified explicitly + elsif strategy == Sparkle + # Skip the `Sparkle` strategy if another strategy is specified explicitly # or if the URL is not specified explicitly. - next + next unless url_provided + next if livecheck_strategy && from_symbol(livecheck_strategy) != strategy elsif strategy.const_defined?(:PRIORITY) && !strategy::PRIORITY.positive? && from_symbol(livecheck_strategy) != strategy From 54a3eb2adba1e5c41a88197bd74bddacc8b19790 Mon Sep 17 00:00:00 2001 From: Markus Reiter Date: Mon, 14 Dec 2020 09:55:01 +0100 Subject: [PATCH 27/70] Include item title in Sparkle strategy. --- Library/Homebrew/livecheck/strategy/sparkle.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/Library/Homebrew/livecheck/strategy/sparkle.rb b/Library/Homebrew/livecheck/strategy/sparkle.rb index f22bd0fc4d..48f52d1dab 100644 --- a/Library/Homebrew/livecheck/strategy/sparkle.rb +++ b/Library/Homebrew/livecheck/strategy/sparkle.rb @@ -67,6 +67,7 @@ module Homebrew version ||= (item > "version").first&.text data = { + title: (item > "title").first&.text, url: enclosure["url"], version: short_version || version ? BundleVersion.new(short_version, version) : nil, }.compact From 126842531e8f6fcae593882f0b5cf2f1c61797c6 Mon Sep 17 00:00:00 2001 From: Markus Reiter Date: Mon, 14 Dec 2020 09:57:51 +0100 Subject: [PATCH 28/70] Print backtrace in debug mode. --- Library/Homebrew/livecheck/livecheck.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/Library/Homebrew/livecheck/livecheck.rb b/Library/Homebrew/livecheck/livecheck.rb index e6f3bede10..d1c0c57f88 100644 --- a/Library/Homebrew/livecheck/livecheck.rb +++ b/Library/Homebrew/livecheck/livecheck.rb @@ -201,6 +201,7 @@ module Homebrew status_hash(formula_or_cask, "error", [e.to_s], full_name: full_name, verbose: verbose) elsif !quiet onoe "#{Tty.blue}#{formula_or_cask_name(formula_or_cask, full_name: full_name)}#{Tty.reset}: #{e}" + $stderr.puts e.backtrace if args.debug? nil end end From d3e2a98136385197f52f6f979378efed6cdad202 Mon Sep 17 00:00:00 2001 From: Markus Reiter Date: Mon, 14 Dec 2020 09:58:00 +0100 Subject: [PATCH 29/70] Remove debug output. --- Library/Homebrew/livecheck/strategy/page_match.rb | 2 -- 1 file changed, 2 deletions(-) diff --git a/Library/Homebrew/livecheck/strategy/page_match.rb b/Library/Homebrew/livecheck/strategy/page_match.rb index 067a61c2f3..c1dc2cd001 100644 --- a/Library/Homebrew/livecheck/strategy/page_match.rb +++ b/Library/Homebrew/livecheck/strategy/page_match.rb @@ -49,8 +49,6 @@ module Homebrew def self.page_matches(url, regex, &block) page = Strategy.page_contents(url) - odebug "Page Contents", page - if block data = { page: page } case (value = block.call(data)) From 02849cf3e7ecab5d09195cb598da3e7753e7bd1d Mon Sep 17 00:00:00 2001 From: Markus Reiter Date: Mon, 14 Dec 2020 10:34:14 +0100 Subject: [PATCH 30/70] Don't pass `nice_version` to block. --- Library/Homebrew/livecheck/strategy/sparkle.rb | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/Library/Homebrew/livecheck/strategy/sparkle.rb b/Library/Homebrew/livecheck/strategy/sparkle.rb index 48f52d1dab..dc17081e38 100644 --- a/Library/Homebrew/livecheck/strategy/sparkle.rb +++ b/Library/Homebrew/livecheck/strategy/sparkle.rb @@ -69,20 +69,21 @@ module Homebrew data = { title: (item > "title").first&.text, url: enclosure["url"], - version: short_version || version ? BundleVersion.new(short_version, version) : nil, + bundle_version: short_version || version ? BundleVersion.new(short_version, version) : nil, }.compact data unless data.empty? end.compact - item = items.max_by { |e| e[:version] } + item = items.max_by { |e| e[:bundle_version] } if item match = if block - item[:version] = item[:version]&.nice_version + item[:short_version] = item[:bundle_version]&.short_version + item[:version] = item[:bundle_version]&.version block.call(item).to_s else - item[:version]&.nice_version + item[:bundle_version]&.nice_version end match_data[:matches][match] = Version.new(match) if match From 3aa5395f75acb52cc5ae752a70080a19fbd37af6 Mon Sep 17 00:00:00 2001 From: Markus Reiter Date: Mon, 14 Dec 2020 11:34:27 +0100 Subject: [PATCH 31/70] Look in more places for URL, short version and version. --- .../Homebrew/livecheck/strategy/sparkle.rb | 21 ++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/Library/Homebrew/livecheck/strategy/sparkle.rb b/Library/Homebrew/livecheck/strategy/sparkle.rb index dc17081e38..87fd431479 100644 --- a/Library/Homebrew/livecheck/strategy/sparkle.rb +++ b/Library/Homebrew/livecheck/strategy/sparkle.rb @@ -58,17 +58,24 @@ module Homebrew items = xml.xpath("//rss//channel//item").map do |item| enclosure = (item > "enclosure").first - next unless enclosure + url = enclosure&.attr("url") + short_version = enclosure&.attr("shortVersionString") + version = enclosure&.attr("version") - short_version ||= enclosure["shortVersionString"] - version ||= enclosure["version"] + url ||= (item > "link").first&.text + short_version ||= (item > "shortVersionString").first&.text&.strip + version ||= (item > "version").first&.text&.strip - short_version ||= (item > "shortVersionString").first&.text - version ||= (item > "version").first&.text + title = (item > "title").first&.text&.strip + + if match = title&.match(/(\d+(?:\.\d+)*)\s*(\([^)]+\))?\Z/) + short_version ||= match[1] + version ||= match[2] + end data = { - title: (item > "title").first&.text, - url: enclosure["url"], + title: title, + url: url, bundle_version: short_version || version ? BundleVersion.new(short_version, version) : nil, }.compact From 3c4e855f4d0596ced3ed0525004adaca6aa0d118 Mon Sep 17 00:00:00 2001 From: Markus Reiter Date: Mon, 14 Dec 2020 21:09:09 +0100 Subject: [PATCH 32/70] Revert `livecheck` position. --- Library/Homebrew/rubocops/cask/constants/stanza.rb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Library/Homebrew/rubocops/cask/constants/stanza.rb b/Library/Homebrew/rubocops/cask/constants/stanza.rb index 7e8874eb3d..93a9b67d55 100644 --- a/Library/Homebrew/rubocops/cask/constants/stanza.rb +++ b/Library/Homebrew/rubocops/cask/constants/stanza.rb @@ -8,7 +8,8 @@ module RuboCop STANZA_GROUPS = [ [:version, :sha256], [:language], - [:url, :appcast, :livecheck, :name, :desc, :homepage], + [:url, :appcast, :name, :desc, :homepage], + [:livecheck], [ :auto_updates, :conflicts_with, From c58fbe000e5de5b5d6df6cc4805b866544a56ca9 Mon Sep 17 00:00:00 2001 From: Markus Reiter Date: Tue, 15 Dec 2020 10:25:26 +0100 Subject: [PATCH 33/70] Revert priority of `GithubLatest` strategy. --- Library/Homebrew/livecheck/strategy/github_latest.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Library/Homebrew/livecheck/strategy/github_latest.rb b/Library/Homebrew/livecheck/strategy/github_latest.rb index 4868ead895..d3afcddbf4 100644 --- a/Library/Homebrew/livecheck/strategy/github_latest.rb +++ b/Library/Homebrew/livecheck/strategy/github_latest.rb @@ -37,7 +37,7 @@ module Homebrew # A priority of zero causes livecheck to skip the strategy. We do this # for {GithubLatest} so we can selectively apply the strategy using # `strategy :github_latest` in a `livecheck` block. - PRIORITY = 8 + PRIORITY = 0 # The `Regexp` used to determine if the strategy applies to the URL. URL_MATCH_REGEX = %r{//github\.com(?:/downloads)?(?:/[^/]+){2}}i.freeze From 2a3ee11c07f245fd6b59642c438f74408a6357c5 Mon Sep 17 00:00:00 2001 From: Markus Reiter Date: Tue, 15 Dec 2020 17:11:18 +0100 Subject: [PATCH 34/70] Fix typo. --- Library/Homebrew/bundle_version.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Library/Homebrew/bundle_version.rb b/Library/Homebrew/bundle_version.rb index ca63aa36b8..69abe5cfbd 100644 --- a/Library/Homebrew/bundle_version.rb +++ b/Library/Homebrew/bundle_version.rb @@ -59,7 +59,7 @@ module Homebrew [other.version, other.short_version].map { |v| v&.yield_self(&Version.public_method(:new)) } end - # Create a nicely formatted version (on a best effor basis). + # Create a nicely formatted version (on a best effort basis). sig { returns(String) } def nice_version nice_parts.join(",") From 3a4c7223df13161c834d85258ba8567c5e55a722 Mon Sep 17 00:00:00 2001 From: Markus Reiter Date: Tue, 15 Dec 2020 17:25:16 +0100 Subject: [PATCH 35/70] Allow accessing version in `livecheck` blocks. --- Library/Homebrew/livecheck.rb | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/Library/Homebrew/livecheck.rb b/Library/Homebrew/livecheck.rb index 2560b80737..43288cdb3c 100644 --- a/Library/Homebrew/livecheck.rb +++ b/Library/Homebrew/livecheck.rb @@ -8,6 +8,8 @@ # This information is used by the `brew livecheck` command to control its # behavior. class Livecheck + extend Forwardable + # A very brief description of why the formula/cask is skipped (e.g. `No longer # developed or maintained`). # @return [String, nil] @@ -107,6 +109,9 @@ class Livecheck end end + delegate version: :@formula_or_cask + private :version + # Returns a `Hash` of all instance variable values. # @return [Hash] def to_hash From 8a583a7aeebbc5992167e9cd6e58fd0af1a28100 Mon Sep 17 00:00:00 2001 From: Markus Reiter Date: Tue, 15 Dec 2020 17:36:50 +0100 Subject: [PATCH 36/70] Remove `args`. --- Library/Homebrew/livecheck/livecheck.rb | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/Library/Homebrew/livecheck/livecheck.rb b/Library/Homebrew/livecheck/livecheck.rb index d1c0c57f88..2fa30748ae 100644 --- a/Library/Homebrew/livecheck/livecheck.rb +++ b/Library/Homebrew/livecheck/livecheck.rb @@ -201,7 +201,7 @@ module Homebrew status_hash(formula_or_cask, "error", [e.to_s], full_name: full_name, verbose: verbose) elsif !quiet onoe "#{Tty.blue}#{formula_or_cask_name(formula_or_cask, full_name: full_name)}#{Tty.reset}: #{e}" - $stderr.puts e.backtrace if args.debug? + $stderr.puts e.backtrace if debug nil end end @@ -280,9 +280,9 @@ module Homebrew end if cask&.discontinued? && !cask.livecheckable? - return status_hash(cask, "discontinued", args: args) if args.json? + return status_hash(cask, "discontinued", full_name: full_name, verbose: verbose) if json - puts "#{Tty.red}#{cask_name(cask, args: args)}#{Tty.reset} : discontinued" unless args.quiet? + puts "#{Tty.red}#{cask_name(cask, full_name: full_name)}#{Tty.reset} : discontinued" unless quiet return end @@ -301,16 +301,16 @@ module Homebrew end if cask&.version&.latest? && !cask.livecheckable? - return status_hash(cask, "latest", args: args) if args.json? + return status_hash(cask, "latest", full_name: full_name, verbose: verbose) if json - puts "#{Tty.red}#{cask_name(cask, args: args)}#{Tty.reset} : latest" unless args.quiet? + puts "#{Tty.red}#{cask_name(cask, full_name: full_name)}#{Tty.reset} : latest" unless quiet return end if cask&.url&.unversioned? && !cask.livecheckable? - return status_hash(cask, "unversioned", args: args) if args.json? + return status_hash(cask, "unversioned", full_name: full_name, verbose: verbose) if json - puts "#{Tty.red}#{cask_name(cask, args: args)}#{Tty.reset} : unversioned" unless args.quiet? + puts "#{Tty.red}#{cask_name(cask, full_name: full_name)}#{Tty.reset} : unversioned" unless quiet return end From 0deceac28cf190900ca861f7da27f05cf82f8b61 Mon Sep 17 00:00:00 2001 From: Markus Reiter Date: Tue, 15 Dec 2020 20:08:53 +0100 Subject: [PATCH 37/70] Allow regex without capture groups. --- Library/Homebrew/livecheck/strategy/page_match.rb | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/Library/Homebrew/livecheck/strategy/page_match.rb b/Library/Homebrew/livecheck/strategy/page_match.rb index c1dc2cd001..92fa626b0a 100644 --- a/Library/Homebrew/livecheck/strategy/page_match.rb +++ b/Library/Homebrew/livecheck/strategy/page_match.rb @@ -61,7 +61,14 @@ module Homebrew end end - page.scan(regex).map(&:first).uniq + page.scan(regex).map do |match| + case match + when String + match + else + match.first + end + end.uniq end # Checks the content at the URL for new versions, using the provided From 75eb6d17807c1c70736818d956ef167739f9f8ce Mon Sep 17 00:00:00 2001 From: Markus Reiter Date: Tue, 15 Dec 2020 20:26:19 +0100 Subject: [PATCH 38/70] Add `Item` for Sparkle strategy. --- .../Homebrew/livecheck/strategy/sparkle.rb | 45 ++++++++++++------- 1 file changed, 29 insertions(+), 16 deletions(-) diff --git a/Library/Homebrew/livecheck/strategy/sparkle.rb b/Library/Homebrew/livecheck/strategy/sparkle.rb index 87fd431479..7822bb73ac 100644 --- a/Library/Homebrew/livecheck/strategy/sparkle.rb +++ b/Library/Homebrew/livecheck/strategy/sparkle.rb @@ -46,13 +46,30 @@ module Homebrew def self.find_versions(url, regex, &block) raise ArgumentError, "The #{NICE_NAME} strategy does not support regular expressions." if regex - require "nokogiri" - match_data = { matches: {}, regex: regex, url: url } contents = Strategy.page_contents(url) - xml = Nokogiri::XML(contents) + if (item = item_from_content(contents)) + match = if block + item[:short_version] = item[:bundle_version]&.short_version + item[:version] = item[:bundle_version]&.version + block.call(item).to_s + else + item.bundle_version&.nice_version + end + + match_data[:matches][match] = Version.new(match) if match + end + + match_data + end + + sig { params(content).returns(T.nilable(Item)) } + def self.item_from_content(content) + require "nokogiri" + + xml = Nokogiri::XML(content) xml.remove_namespaces! items = xml.xpath("//rss//channel//item").map do |item| @@ -79,24 +96,20 @@ module Homebrew bundle_version: short_version || version ? BundleVersion.new(short_version, version) : nil, }.compact - data unless data.empty? + Item.new(**data) unless data.empty? end.compact - item = items.max_by { |e| e[:bundle_version] } + item = items.max_by(&:bundle_version) + end + private_class_method :item_from_content - if item - match = if block - item[:short_version] = item[:bundle_version]&.short_version - item[:version] = item[:bundle_version]&.version - block.call(item).to_s - else - item[:bundle_version]&.nice_version - end + Item = Struct.new(:title, :url, :bundle_version, :short_version, :version, keyword_init: true) do + extend T::Sig - match_data[:matches][match] = Version.new(match) if match - end + extend Forwardable - match_data + delegate version: :bundle_version + delegate short_version: :bundle_version end end end From 71759035dd82b40b7ad8a4a1f30e917b4306e6c8 Mon Sep 17 00:00:00 2001 From: Markus Reiter Date: Tue, 15 Dec 2020 20:30:56 +0100 Subject: [PATCH 39/70] Fix Sparkle detection if content type is missing. --- Library/Homebrew/livecheck/strategy/sparkle.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Library/Homebrew/livecheck/strategy/sparkle.rb b/Library/Homebrew/livecheck/strategy/sparkle.rb index 7822bb73ac..b37b41ce4b 100644 --- a/Library/Homebrew/livecheck/strategy/sparkle.rb +++ b/Library/Homebrew/livecheck/strategy/sparkle.rb @@ -26,8 +26,8 @@ module Homebrew xml = url.end_with?(".xml") xml ||= begin headers = Strategy.page_headers(url) - content_type = headers["content-type"]&.split(";", 2)&.first - ["application/xml", "text/xml"].include?(content_type) + content_type = headers["content-type"] + content_type.blank? || content_type.include?("xml") end return false unless xml From 4aaa2de6c4cb1cafc421c0dffaad0bde0f771016 Mon Sep 17 00:00:00 2001 From: Markus Reiter Date: Thu, 17 Dec 2020 15:40:23 +0100 Subject: [PATCH 40/70] Fix syntax. --- Library/Homebrew/livecheck/strategy/sparkle.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Library/Homebrew/livecheck/strategy/sparkle.rb b/Library/Homebrew/livecheck/strategy/sparkle.rb index b37b41ce4b..9ccfb23aa3 100644 --- a/Library/Homebrew/livecheck/strategy/sparkle.rb +++ b/Library/Homebrew/livecheck/strategy/sparkle.rb @@ -65,7 +65,7 @@ module Homebrew match_data end - sig { params(content).returns(T.nilable(Item)) } + sig { params(content: String).returns(T.nilable(Item)) } def self.item_from_content(content) require "nokogiri" @@ -99,7 +99,7 @@ module Homebrew Item.new(**data) unless data.empty? end.compact - item = items.max_by(&:bundle_version) + items.max_by(&:bundle_version) end private_class_method :item_from_content From 3723cd7decc4d94ad772e3e6948262ceff35ee94 Mon Sep 17 00:00:00 2001 From: Markus Reiter Date: Fri, 18 Dec 2020 17:58:21 +0100 Subject: [PATCH 41/70] Remove detection for Sparkle strategy. --- .../Homebrew/livecheck/strategy/sparkle.rb | 29 ++----------------- 1 file changed, 2 insertions(+), 27 deletions(-) diff --git a/Library/Homebrew/livecheck/strategy/sparkle.rb b/Library/Homebrew/livecheck/strategy/sparkle.rb index 9ccfb23aa3..7fad9e885a 100644 --- a/Library/Homebrew/livecheck/strategy/sparkle.rb +++ b/Library/Homebrew/livecheck/strategy/sparkle.rb @@ -1,4 +1,4 @@ -# typed: false +# typed: true # frozen_string_literal: true require "bundle_version" @@ -11,36 +11,11 @@ module Homebrew # its contents as a Sparkle appcast in XML format. # # @api private - class Sparkle + class Sparkle < PageMatch extend T::Sig NICE_NAME = "Sparkle" - PRIORITY = 1 - - # Whether the strategy can be applied to the provided URL. - sig { params(url: String).returns(T::Boolean) } - def self.match?(url) - return false unless url.match?(%r{^https?://}) - - xml = url.end_with?(".xml") - xml ||= begin - headers = Strategy.page_headers(url) - content_type = headers["content-type"] - content_type.blank? || content_type.include?("xml") - end - return false unless xml - - contents = Strategy.page_contents(url) - - return true if contents.match?(%r{https?://www.andymatuschak.org/xml-namespaces/sparkle}) - - contents.include?("rss") && - contents.include?("channel") && - contents.include?("item") && - contents.include?("enclosure") - end - # Checks the content at the URL for new versions. sig { params(url: String, regex: T.nilable(Regexp)).returns(T::Hash[Symbol, T.untyped]) } def self.find_versions(url, regex, &block) From 8fa5a8e8329965fb78d04bc6dfb5091de2da4de2 Mon Sep 17 00:00:00 2001 From: Markus Reiter Date: Fri, 18 Dec 2020 18:15:50 +0100 Subject: [PATCH 42/70] Don't suggest adding appcast if livecheck exists. --- Library/Homebrew/cask/audit.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Library/Homebrew/cask/audit.rb b/Library/Homebrew/cask/audit.rb index 0e54c062dd..6121c1ac00 100644 --- a/Library/Homebrew/cask/audit.rb +++ b/Library/Homebrew/cask/audit.rb @@ -298,7 +298,7 @@ module Cask end def check_hosting_with_appcast - return if cask.appcast + return if cask.appcast || cask.livecheckable? add_appcast = "please add an appcast. See https://github.com/Homebrew/homebrew-cask/blob/HEAD/doc/cask_language_reference/stanzas/appcast.md" From 1a18742366df86a6de4109e34902559e639aba34 Mon Sep 17 00:00:00 2001 From: Markus Reiter Date: Fri, 18 Dec 2020 18:31:07 +0100 Subject: [PATCH 43/70] Pass headers directly instead of hash containing `:headers`. --- Library/Homebrew/livecheck/strategy/header_match.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Library/Homebrew/livecheck/strategy/header_match.rb b/Library/Homebrew/livecheck/strategy/header_match.rb index 7aedecac1c..f07103cc9a 100644 --- a/Library/Homebrew/livecheck/strategy/header_match.rb +++ b/Library/Homebrew/livecheck/strategy/header_match.rb @@ -55,7 +55,7 @@ module Homebrew end version = if block - block.call(data) + block.call(data[:headers]) else data[:version] end From 71104a27efffbc0badaeee2c4928cfc5f8b1f75f Mon Sep 17 00:00:00 2001 From: Markus Reiter Date: Fri, 18 Dec 2020 19:14:11 +0100 Subject: [PATCH 44/70] Pass page directly instead of hash containing `:page`. --- Library/Homebrew/livecheck/strategy/page_match.rb | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Library/Homebrew/livecheck/strategy/page_match.rb b/Library/Homebrew/livecheck/strategy/page_match.rb index 92fa626b0a..e739f2a839 100644 --- a/Library/Homebrew/livecheck/strategy/page_match.rb +++ b/Library/Homebrew/livecheck/strategy/page_match.rb @@ -50,8 +50,7 @@ module Homebrew page = Strategy.page_contents(url) if block - data = { page: page } - case (value = block.call(data)) + case (value = block.call(page)) when String return [value] when Array From c306577575e637aaa39837f334b53e00d20c66b6 Mon Sep 17 00:00:00 2001 From: Markus Reiter Date: Fri, 18 Dec 2020 20:07:40 +0100 Subject: [PATCH 45/70] Skip Sparkle strategy only if URL is not specified explicitly. --- Library/Homebrew/livecheck/strategy.rb | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/Library/Homebrew/livecheck/strategy.rb b/Library/Homebrew/livecheck/strategy.rb index d82b1e1560..2c986a46fc 100644 --- a/Library/Homebrew/livecheck/strategy.rb +++ b/Library/Homebrew/livecheck/strategy.rb @@ -64,10 +64,8 @@ module Homebrew # present in the `livecheck` block next unless regex_provided || block_provided elsif strategy == Sparkle - # Skip the `Sparkle` strategy if another strategy is specified explicitly - # or if the URL is not specified explicitly. + # Skip the `Sparkle` strategy if the URL is not specified explicitly. next unless url_provided - next if livecheck_strategy && from_symbol(livecheck_strategy) != strategy elsif strategy.const_defined?(:PRIORITY) && !strategy::PRIORITY.positive? && from_symbol(livecheck_strategy) != strategy From b8de4e8b21e7daf40c805e1f1058eae7393d06e1 Mon Sep 17 00:00:00 2001 From: Markus Reiter Date: Fri, 18 Dec 2020 21:17:55 +0100 Subject: [PATCH 46/70] Simplify strategies. --- .../livecheck/strategy/header_match.rb | 46 +++++++++---------- .../Homebrew/livecheck/strategy/sparkle.rb | 10 ++-- 2 files changed, 29 insertions(+), 27 deletions(-) diff --git a/Library/Homebrew/livecheck/strategy/header_match.rb b/Library/Homebrew/livecheck/strategy/header_match.rb index f07103cc9a..e307f9dec1 100644 --- a/Library/Homebrew/livecheck/strategy/header_match.rb +++ b/Library/Homebrew/livecheck/strategy/header_match.rb @@ -34,33 +34,33 @@ module Homebrew def self.find_versions(url, regex, &block) match_data = { matches: {}, regex: regex, url: url } - data = { headers: Strategy.page_headers(url) } + headers = Strategy.page_headers(url) - if (filename = data[:headers]["content-disposition"]) - if regex - data[:version] ||= location[regex, 1] - else - v = Version.parse(filename, detected_from_url: true) - data[:version] ||= v.to_s unless v.null? - end - end - - if (location = data[:headers]["location"]) - if regex - data[:version] ||= location[regex, 1] - else - v = Version.parse(location, detected_from_url: true) - data[:version] ||= v.to_s unless v.null? - end - end - - version = if block - block.call(data[:headers]) + if block + match = block.call(headers) else - data[:version] + match = nil + + if (filename = headers["content-disposition"]) + if regex + match ||= location[regex, 1] + else + v = Version.parse(filename, detected_from_url: true) + match ||= v.to_s unless v.null? + end + end + + if (location = headers["location"]) + if regex + match ||= location[regex, 1] + else + v = Version.parse(location, detected_from_url: true) + match ||= v.to_s unless v.null? + end + end end - match_data[:matches][version] = Version.new(version) if version + match_data[:matches][match] = Version.new(match) if match match_data end diff --git a/Library/Homebrew/livecheck/strategy/sparkle.rb b/Library/Homebrew/livecheck/strategy/sparkle.rb index 7fad9e885a..eb0877c279 100644 --- a/Library/Homebrew/livecheck/strategy/sparkle.rb +++ b/Library/Homebrew/livecheck/strategy/sparkle.rb @@ -27,9 +27,7 @@ module Homebrew if (item = item_from_content(contents)) match = if block - item[:short_version] = item[:bundle_version]&.short_version - item[:version] = item[:bundle_version]&.version - block.call(item).to_s + block.call(item)&.to_s else item.bundle_version&.nice_version end @@ -65,10 +63,14 @@ module Homebrew version ||= match[2] end + bundle_version = BundleVersion.new(short_version, version) if short_version || version + data = { title: title, url: url, - bundle_version: short_version || version ? BundleVersion.new(short_version, version) : nil, + bundle_version: bundle_version, + short_version: bundle_version&.short_version, + version: bundle_version&.version, }.compact Item.new(**data) unless data.empty? From 0fd0b1dad0f15646d2dc63ac636160db13d224c9 Mon Sep 17 00:00:00 2001 From: Markus Reiter Date: Fri, 18 Dec 2020 21:29:58 +0100 Subject: [PATCH 47/70] Actually cache headers. --- Library/Homebrew/livecheck/strategy.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Library/Homebrew/livecheck/strategy.rb b/Library/Homebrew/livecheck/strategy.rb index 2c986a46fc..08f2dbd63a 100644 --- a/Library/Homebrew/livecheck/strategy.rb +++ b/Library/Homebrew/livecheck/strategy.rb @@ -107,7 +107,7 @@ module Homebrew .to_h.transform_keys(&:downcase)) end - return headers if status.success? + return (@headers[url] = headers) if status.success? end headers From 0d7c8c8c61914f5e28c019be852773284548b7c8 Mon Sep 17 00:00:00 2001 From: Markus Reiter Date: Fri, 18 Dec 2020 21:34:00 +0100 Subject: [PATCH 48/70] Revert priority for Git strategy. --- Library/Homebrew/livecheck/strategy/git.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Library/Homebrew/livecheck/strategy/git.rb b/Library/Homebrew/livecheck/strategy/git.rb index 94a42fdab7..4cd216bf58 100644 --- a/Library/Homebrew/livecheck/strategy/git.rb +++ b/Library/Homebrew/livecheck/strategy/git.rb @@ -26,7 +26,7 @@ module Homebrew class Git # The priority of the strategy on an informal scale of 1 to 10 (from # lowest to highest). - PRIORITY = 7 + PRIORITY = 8 # Fetches a remote Git repository's tags using `git ls-remote --tags` # and parses the command's output. If a regex is provided, it will be From f5774c1af430b8798b6be7ebb8cffec7ba644549 Mon Sep 17 00:00:00 2001 From: Sam Ford <1584702+samford@users.noreply.github.com> Date: Fri, 18 Dec 2020 22:12:10 -0500 Subject: [PATCH 49/70] Sparkle: Use demodulized class name in error text --- Library/Homebrew/livecheck/strategy/sparkle.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Library/Homebrew/livecheck/strategy/sparkle.rb b/Library/Homebrew/livecheck/strategy/sparkle.rb index eb0877c279..ea4903732b 100644 --- a/Library/Homebrew/livecheck/strategy/sparkle.rb +++ b/Library/Homebrew/livecheck/strategy/sparkle.rb @@ -19,7 +19,7 @@ module Homebrew # Checks the content at the URL for new versions. sig { params(url: String, regex: T.nilable(Regexp)).returns(T::Hash[Symbol, T.untyped]) } def self.find_versions(url, regex, &block) - raise ArgumentError, "The #{NICE_NAME} strategy does not support regular expressions." if regex + raise ArgumentError, "The #{name.demodulize} strategy does not support regular expressions." if regex match_data = { matches: {}, regex: regex, url: url } From a55686a0e2b9c0a087bd3524cab4237076696a37 Mon Sep 17 00:00:00 2001 From: Sam Ford <1584702+samford@users.noreply.github.com> Date: Fri, 18 Dec 2020 22:11:36 -0500 Subject: [PATCH 50/70] Enforce strategy url requirement in latest_version --- Library/Homebrew/livecheck/livecheck.rb | 5 +++++ Library/Homebrew/livecheck/strategy.rb | 3 --- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/Library/Homebrew/livecheck/livecheck.rb b/Library/Homebrew/livecheck/livecheck.rb index 2fa30748ae..f9c1df8bc1 100644 --- a/Library/Homebrew/livecheck/livecheck.rb +++ b/Library/Homebrew/livecheck/livecheck.rb @@ -498,6 +498,11 @@ module Homebrew next end + if livecheck_strategy.present? && livecheck_url.blank? + odebug "#{strategy_name} strategy requires a url" + next + end + if livecheck_strategy.present? && strategies.exclude?(strategy) odebug "#{strategy_name} strategy does not apply to this URL" next diff --git a/Library/Homebrew/livecheck/strategy.rb b/Library/Homebrew/livecheck/strategy.rb index 08f2dbd63a..5dd94e4111 100644 --- a/Library/Homebrew/livecheck/strategy.rb +++ b/Library/Homebrew/livecheck/strategy.rb @@ -63,9 +63,6 @@ module Homebrew # Only treat the `PageMatch` strategy as usable if a regex is # present in the `livecheck` block next unless regex_provided || block_provided - elsif strategy == Sparkle - # Skip the `Sparkle` strategy if the URL is not specified explicitly. - next unless url_provided elsif strategy.const_defined?(:PRIORITY) && !strategy::PRIORITY.positive? && from_symbol(livecheck_strategy) != strategy From 1bd2be5e047ce7b1e2b68b48b1b2c6c2539cd89d Mon Sep 17 00:00:00 2001 From: Sam Ford <1584702+samford@users.noreply.github.com> Date: Sat, 19 Dec 2020 00:21:29 -0500 Subject: [PATCH 51/70] Rename page_contents to page_content --- Library/Homebrew/livecheck/strategy.rb | 6 +++--- Library/Homebrew/livecheck/strategy/page_match.rb | 2 +- Library/Homebrew/livecheck/strategy/sparkle.rb | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Library/Homebrew/livecheck/strategy.rb b/Library/Homebrew/livecheck/strategy.rb index 5dd94e4111..78d85acacf 100644 --- a/Library/Homebrew/livecheck/strategy.rb +++ b/Library/Homebrew/livecheck/strategy.rb @@ -110,9 +110,9 @@ module Homebrew headers end - def self.page_contents(url) - @page_contents ||= {} - @page_contents[url] ||= URI.parse(url).open.read + def self.page_content(url) + @page_content ||= {} + @page_content[url] ||= URI.parse(url).open.read end end end diff --git a/Library/Homebrew/livecheck/strategy/page_match.rb b/Library/Homebrew/livecheck/strategy/page_match.rb index e739f2a839..361cf04b97 100644 --- a/Library/Homebrew/livecheck/strategy/page_match.rb +++ b/Library/Homebrew/livecheck/strategy/page_match.rb @@ -47,7 +47,7 @@ module Homebrew # content # @return [Array] def self.page_matches(url, regex, &block) - page = Strategy.page_contents(url) + page = Strategy.page_content(url) if block case (value = block.call(page)) diff --git a/Library/Homebrew/livecheck/strategy/sparkle.rb b/Library/Homebrew/livecheck/strategy/sparkle.rb index ea4903732b..24b27c738f 100644 --- a/Library/Homebrew/livecheck/strategy/sparkle.rb +++ b/Library/Homebrew/livecheck/strategy/sparkle.rb @@ -23,7 +23,7 @@ module Homebrew match_data = { matches: {}, regex: regex, url: url } - contents = Strategy.page_contents(url) + contents = Strategy.page_content(url) if (item = item_from_content(contents)) match = if block From 7f40198506fdfe14172608de73147edc21f59f64 Mon Sep 17 00:00:00 2001 From: Sam Ford <1584702+samford@users.noreply.github.com> Date: Sat, 19 Dec 2020 00:46:18 -0500 Subject: [PATCH 52/70] Make Strategy#page_headers more general purpose --- Library/Homebrew/livecheck/strategy.rb | 8 ++++---- Library/Homebrew/livecheck/strategy/header_match.rb | 10 +++++++--- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/Library/Homebrew/livecheck/strategy.rb b/Library/Homebrew/livecheck/strategy.rb index 78d85acacf..de46bfd582 100644 --- a/Library/Homebrew/livecheck/strategy.rb +++ b/Library/Homebrew/livecheck/strategy.rb @@ -86,7 +86,7 @@ module Homebrew return @headers[url] if @headers.key?(url) - headers = {} + headers = [] [:default, :browser].each do |user_agent| stdout, _, status = curl_output( @@ -99,9 +99,9 @@ module Homebrew while stdout.match?(/\AHTTP.*\r$/) h, stdout = stdout.split("\r\n\r\n", 2) - headers = headers.merge(h.split("\r\n").drop(1) - .map { |header| header.split(/:\s*/, 2) } - .to_h.transform_keys(&:downcase)) + headers << h.split("\r\n").drop(1) + .map { |header| header.split(/:\s*/, 2) } + .to_h.transform_keys(&:downcase) end return (@headers[url] = headers) if status.success? diff --git a/Library/Homebrew/livecheck/strategy/header_match.rb b/Library/Homebrew/livecheck/strategy/header_match.rb index e307f9dec1..af58672425 100644 --- a/Library/Homebrew/livecheck/strategy/header_match.rb +++ b/Library/Homebrew/livecheck/strategy/header_match.rb @@ -36,12 +36,16 @@ module Homebrew headers = Strategy.page_headers(url) + # Merge the headers from all responses into one hash + merged_headers = {} + headers.each { |resp_headers| merged_headers.merge!(resp_headers) } + if block - match = block.call(headers) + match = block.call(merged_headers) else match = nil - if (filename = headers["content-disposition"]) + if (filename = merged_headers["content-disposition"]) if regex match ||= location[regex, 1] else @@ -50,7 +54,7 @@ module Homebrew end end - if (location = headers["location"]) + if (location = merged_headers["location"]) if regex match ||= location[regex, 1] else From 8b40d959bfe5cbf950ea8614e10092563c7dec3f Mon Sep 17 00:00:00 2001 From: Sam Ford <1584702+samford@users.noreply.github.com> Date: Sat, 19 Dec 2020 00:47:21 -0500 Subject: [PATCH 53/70] HeaderMatch: Fix content-disposition regex logic --- Library/Homebrew/livecheck/strategy/header_match.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Library/Homebrew/livecheck/strategy/header_match.rb b/Library/Homebrew/livecheck/strategy/header_match.rb index af58672425..94d1706df7 100644 --- a/Library/Homebrew/livecheck/strategy/header_match.rb +++ b/Library/Homebrew/livecheck/strategy/header_match.rb @@ -47,7 +47,7 @@ module Homebrew if (filename = merged_headers["content-disposition"]) if regex - match ||= location[regex, 1] + match ||= filename[regex, 1] else v = Version.parse(filename, detected_from_url: true) match ||= v.to_s unless v.null? From 0148ee870eedc0d850c25fd7c6fb3e0e02893653 Mon Sep 17 00:00:00 2001 From: Sam Ford <1584702+samford@users.noreply.github.com> Date: Sat, 19 Dec 2020 00:51:54 -0500 Subject: [PATCH 54/70] Livecheck: Use alphabetical order in symbols array --- Library/Homebrew/livecheck/livecheck.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Library/Homebrew/livecheck/livecheck.rb b/Library/Homebrew/livecheck/livecheck.rb index f9c1df8bc1..91588946a0 100644 --- a/Library/Homebrew/livecheck/livecheck.rb +++ b/Library/Homebrew/livecheck/livecheck.rb @@ -29,8 +29,8 @@ module Homebrew STRATEGY_SYMBOLS_TO_SKIP_PREPROCESS_URL = [ :github_latest, - :sparkle, :page_match, + :sparkle, ].freeze UNSTABLE_VERSION_KEYWORDS = %w[ From 9fdc902630065dc1aef8c50f5783279ad343d22b Mon Sep 17 00:00:00 2001 From: Sam Ford <1584702+samford@users.noreply.github.com> Date: Sat, 19 Dec 2020 01:07:56 -0500 Subject: [PATCH 55/70] Sparkle: Reorganize methods, etc. --- .../Homebrew/livecheck/strategy/sparkle.rb | 46 +++++++++---------- 1 file changed, 23 insertions(+), 23 deletions(-) diff --git a/Library/Homebrew/livecheck/strategy/sparkle.rb b/Library/Homebrew/livecheck/strategy/sparkle.rb index 24b27c738f..72e5d607e7 100644 --- a/Library/Homebrew/livecheck/strategy/sparkle.rb +++ b/Library/Homebrew/livecheck/strategy/sparkle.rb @@ -16,26 +16,13 @@ module Homebrew NICE_NAME = "Sparkle" - # Checks the content at the URL for new versions. - sig { params(url: String, regex: T.nilable(Regexp)).returns(T::Hash[Symbol, T.untyped]) } - def self.find_versions(url, regex, &block) - raise ArgumentError, "The #{name.demodulize} strategy does not support regular expressions." if regex + Item = Struct.new(:title, :url, :bundle_version, :short_version, :version, keyword_init: true) do + extend T::Sig - match_data = { matches: {}, regex: regex, url: url } + extend Forwardable - contents = Strategy.page_content(url) - - if (item = item_from_content(contents)) - match = if block - block.call(item)&.to_s - else - item.bundle_version&.nice_version - end - - match_data[:matches][match] = Version.new(match) if match - end - - match_data + delegate version: :bundle_version + delegate short_version: :bundle_version end sig { params(content: String).returns(T.nilable(Item)) } @@ -80,13 +67,26 @@ module Homebrew end private_class_method :item_from_content - Item = Struct.new(:title, :url, :bundle_version, :short_version, :version, keyword_init: true) do - extend T::Sig + # Checks the content at the URL for new versions. + sig { params(url: String, regex: T.nilable(Regexp)).returns(T::Hash[Symbol, T.untyped]) } + def self.find_versions(url, regex, &block) + raise ArgumentError, "The #{name.demodulize} strategy does not support regular expressions." if regex - extend Forwardable + match_data = { matches: {}, regex: regex, url: url } - delegate version: :bundle_version - delegate short_version: :bundle_version + contents = Strategy.page_content(url) + + if (item = item_from_content(contents)) + match = if block + block.call(item)&.to_s + else + item.bundle_version&.nice_version + end + + match_data[:matches][match] = Version.new(match) if match + end + + match_data end end end From 740d5d3bcd900e4f851379cb7228c2bef5c56842 Mon Sep 17 00:00:00 2001 From: Sam Ford <1584702+samford@users.noreply.github.com> Date: Sat, 19 Dec 2020 01:11:45 -0500 Subject: [PATCH 56/70] Sparkle: Add test --- .../test/livecheck/strategy/sparkle_spec.rb | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 Library/Homebrew/test/livecheck/strategy/sparkle_spec.rb diff --git a/Library/Homebrew/test/livecheck/strategy/sparkle_spec.rb b/Library/Homebrew/test/livecheck/strategy/sparkle_spec.rb new file mode 100644 index 0000000000..f9a878c1d4 --- /dev/null +++ b/Library/Homebrew/test/livecheck/strategy/sparkle_spec.rb @@ -0,0 +1,16 @@ +# typed: false +# frozen_string_literal: true + +require "livecheck/strategy/sparkle" + +describe Homebrew::Livecheck::Strategy::Sparkle do + subject(:sparkle) { described_class } + + let(:url) { "https://www.example.com/example/appcast.xml" } + + describe "::match?" do + it "returns true for any URL" do + expect(sparkle.match?(url)).to be true + end + end +end From 6bcb0fd4b8750d5c165e02a9dd11dc8f1ac752ef Mon Sep 17 00:00:00 2001 From: Sam Ford <1584702+samford@users.noreply.github.com> Date: Sat, 19 Dec 2020 01:14:23 -0500 Subject: [PATCH 57/70] HeaderMatch: Add test --- .../test/livecheck/strategy/header_match_spec.rb | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 Library/Homebrew/test/livecheck/strategy/header_match_spec.rb diff --git a/Library/Homebrew/test/livecheck/strategy/header_match_spec.rb b/Library/Homebrew/test/livecheck/strategy/header_match_spec.rb new file mode 100644 index 0000000000..fbb38b8366 --- /dev/null +++ b/Library/Homebrew/test/livecheck/strategy/header_match_spec.rb @@ -0,0 +1,16 @@ +# typed: false +# frozen_string_literal: true + +require "livecheck/strategy/header_match" + +describe Homebrew::Livecheck::Strategy::HeaderMatch do + subject(:header_match) { described_class } + + let(:url) { "https://www.example.com/" } + + describe "::match?" do + it "returns true for any URL" do + expect(header_match.match?(url)).to be true + end + end +end From 57995f936f6b80ee3d45a5b31b368981e8ebbfca Mon Sep 17 00:00:00 2001 From: Sam Ford <1584702+samford@users.noreply.github.com> Date: Sat, 19 Dec 2020 01:16:16 -0500 Subject: [PATCH 58/70] Sparkle: Remove unnecessary NICE_NAME --- Library/Homebrew/livecheck/strategy/sparkle.rb | 2 -- 1 file changed, 2 deletions(-) diff --git a/Library/Homebrew/livecheck/strategy/sparkle.rb b/Library/Homebrew/livecheck/strategy/sparkle.rb index 72e5d607e7..a5d277317f 100644 --- a/Library/Homebrew/livecheck/strategy/sparkle.rb +++ b/Library/Homebrew/livecheck/strategy/sparkle.rb @@ -14,8 +14,6 @@ module Homebrew class Sparkle < PageMatch extend T::Sig - NICE_NAME = "Sparkle" - Item = Struct.new(:title, :url, :bundle_version, :short_version, :version, keyword_init: true) do extend T::Sig From 8ea9ec81764e9790dbe37c06ddf68aeebc99402f Mon Sep 17 00:00:00 2001 From: Sam Ford <1584702+samford@users.noreply.github.com> Date: Sat, 19 Dec 2020 01:20:00 -0500 Subject: [PATCH 59/70] HeaderMatch: Align NICE_NAME with PageMatch --- Library/Homebrew/livecheck/strategy/header_match.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Library/Homebrew/livecheck/strategy/header_match.rb b/Library/Homebrew/livecheck/strategy/header_match.rb index 94d1706df7..bd0406089d 100644 --- a/Library/Homebrew/livecheck/strategy/header_match.rb +++ b/Library/Homebrew/livecheck/strategy/header_match.rb @@ -13,7 +13,7 @@ module Homebrew class HeaderMatch extend T::Sig - NICE_NAME = "Match HTTP Headers" + NICE_NAME = "Header match" # We set the priority to zero since this cannot # be detected automatically. From f93ecd7d555e590d78eb7a444b18b238c0182350 Mon Sep 17 00:00:00 2001 From: Sam Ford <1584702+samford@users.noreply.github.com> Date: Sat, 19 Dec 2020 02:22:03 -0500 Subject: [PATCH 60/70] Satisfy brew typecheck and style --- Library/Homebrew/livecheck/strategy/sparkle.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Library/Homebrew/livecheck/strategy/sparkle.rb b/Library/Homebrew/livecheck/strategy/sparkle.rb index a5d277317f..3e202ceb9f 100644 --- a/Library/Homebrew/livecheck/strategy/sparkle.rb +++ b/Library/Homebrew/livecheck/strategy/sparkle.rb @@ -68,7 +68,7 @@ module Homebrew # Checks the content at the URL for new versions. sig { params(url: String, regex: T.nilable(Regexp)).returns(T::Hash[Symbol, T.untyped]) } def self.find_versions(url, regex, &block) - raise ArgumentError, "The #{name.demodulize} strategy does not support regular expressions." if regex + raise ArgumentError, "The #{T.must(name).demodulize} strategy does not support a regex." if regex match_data = { matches: {}, regex: regex, url: url } From 921485c6722e71310036c8562487bedca2d5e45c Mon Sep 17 00:00:00 2001 From: Markus Reiter Date: Sat, 19 Dec 2020 20:36:41 +0100 Subject: [PATCH 61/70] =?UTF-8?q?Don't=20print=20backtrace=20for=20?= =?UTF-8?q?=E2=80=9CUnable=20to=20get=20versions=E2=80=9D=20error.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Library/Homebrew/livecheck/error.rb | 12 ++++++++++++ Library/Homebrew/livecheck/livecheck.rb | 5 +++-- 2 files changed, 15 insertions(+), 2 deletions(-) create mode 100644 Library/Homebrew/livecheck/error.rb diff --git a/Library/Homebrew/livecheck/error.rb b/Library/Homebrew/livecheck/error.rb new file mode 100644 index 0000000000..a10875551b --- /dev/null +++ b/Library/Homebrew/livecheck/error.rb @@ -0,0 +1,12 @@ +# typed: true +# frozen_string_literal: true + +module Homebrew + module Livecheck + # Error during a livecheck run. + # + # @api private + class Error < RuntimeError + end + end +end diff --git a/Library/Homebrew/livecheck/livecheck.rb b/Library/Homebrew/livecheck/livecheck.rb index 91588946a0..ece9c04656 100644 --- a/Library/Homebrew/livecheck/livecheck.rb +++ b/Library/Homebrew/livecheck/livecheck.rb @@ -1,6 +1,7 @@ # typed: false # frozen_string_literal: true +require "livecheck/error" require "livecheck/strategy" require "ruby-progressbar" require "uri" @@ -145,7 +146,7 @@ module Homebrew if latest.blank? no_versions_msg = "Unable to get versions" - raise TypeError, no_versions_msg unless json + raise Livecheck::Error, no_versions_msg unless json next version_info if version_info.is_a?(Hash) && version_info[:status] && version_info[:messages] @@ -201,7 +202,7 @@ module Homebrew status_hash(formula_or_cask, "error", [e.to_s], full_name: full_name, verbose: verbose) elsif !quiet onoe "#{Tty.blue}#{formula_or_cask_name(formula_or_cask, full_name: full_name)}#{Tty.reset}: #{e}" - $stderr.puts e.backtrace if debug + $stderr.puts e.backtrace if debug && !e.is_a?(Livecheck::Error) nil end end From edf6a11be601918ae1374124407ea244a35e2824 Mon Sep 17 00:00:00 2001 From: Markus Reiter Date: Sat, 19 Dec 2020 20:40:29 +0100 Subject: [PATCH 62/70] Simplify creation of `merged_headers`. --- Library/Homebrew/livecheck/strategy/header_match.rb | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Library/Homebrew/livecheck/strategy/header_match.rb b/Library/Homebrew/livecheck/strategy/header_match.rb index bd0406089d..1971709a9e 100644 --- a/Library/Homebrew/livecheck/strategy/header_match.rb +++ b/Library/Homebrew/livecheck/strategy/header_match.rb @@ -37,8 +37,7 @@ module Homebrew headers = Strategy.page_headers(url) # Merge the headers from all responses into one hash - merged_headers = {} - headers.each { |resp_headers| merged_headers.merge!(resp_headers) } + merged_headers = headers.reduce(&:merge) if block match = block.call(merged_headers) From cc5cd4bf591ec8261d86fcac26b45d081848a598 Mon Sep 17 00:00:00 2001 From: Sam Ford <1584702+samford@users.noreply.github.com> Date: Sat, 19 Dec 2020 11:29:56 -0500 Subject: [PATCH 63/70] HeaderMatch: Align with PageMatch --- Library/Homebrew/livecheck/strategy/header_match.rb | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/Library/Homebrew/livecheck/strategy/header_match.rb b/Library/Homebrew/livecheck/strategy/header_match.rb index 1971709a9e..22d3ea7e49 100644 --- a/Library/Homebrew/livecheck/strategy/header_match.rb +++ b/Library/Homebrew/livecheck/strategy/header_match.rb @@ -15,17 +15,22 @@ module Homebrew NICE_NAME = "Header match" - # We set the priority to zero since this cannot - # be detected automatically. + # A priority of zero causes livecheck to skip the strategy. We only + # apply {HeaderMatch} using `strategy :header_match` in a `livecheck` + # block, as we can't automatically determine when this can be + # successfully applied to a URL. PRIORITY = 0 + # The `Regexp` used to determine if the strategy applies to the URL. + URL_MATCH_REGEX = %r{^https?://}i.freeze + # Whether the strategy can be applied to the provided URL. # The strategy will technically match any HTTP URL but is # only usable with a `livecheck` block containing a regex # or block. sig { params(url: String).returns(T::Boolean) } def self.match?(url) - url.match?(%r{^https?://}) + URL_MATCH_REGEX.match?(url) end # Checks the final URL for new versions after following all redirections, From 3933b866b697e3b488988536fa71984c3b392861 Mon Sep 17 00:00:00 2001 From: Sam Ford <1584702+samford@users.noreply.github.com> Date: Sat, 19 Dec 2020 11:36:16 -0500 Subject: [PATCH 64/70] Sparkle: Align with PageMatch This inlines the `PRIORITY` and `#match?` logic from `PageMatch` into `Sparkle` instead of subclassing `PageMatch`. `Sparkle` doesn't really make sense as a subclass of `PageMatch` (since it functions in a completely different manner), so it's better to simply copy over these parts of `PageMatch`. This also helps to separate the strategies, so any changes to `PRIORITY` or `#match?` in `PageMatch` won't affect `Sparkle`. --- .../Homebrew/livecheck/strategy/sparkle.rb | 20 ++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/Library/Homebrew/livecheck/strategy/sparkle.rb b/Library/Homebrew/livecheck/strategy/sparkle.rb index 3e202ceb9f..c90649ceef 100644 --- a/Library/Homebrew/livecheck/strategy/sparkle.rb +++ b/Library/Homebrew/livecheck/strategy/sparkle.rb @@ -11,9 +11,27 @@ module Homebrew # its contents as a Sparkle appcast in XML format. # # @api private - class Sparkle < PageMatch + class Sparkle extend T::Sig + # A priority of zero causes livecheck to skip the strategy. We only + # apply {Sparkle} using `strategy :sparkle` in a `livecheck` block, + # as we can't automatically determine when this can be successfully + # applied to a URL without fetching the content. + PRIORITY = 0 + + # The `Regexp` used to determine if the strategy applies to the URL. + URL_MATCH_REGEX = %r{^https?://}i.freeze + + # Whether the strategy can be applied to the provided URL. + # The strategy will technically match any HTTP URL but is + # only usable with a `livecheck` block containing a regex + # or block. + sig { params(url: String).returns(T::Boolean) } + def self.match?(url) + URL_MATCH_REGEX.match?(url) + end + Item = Struct.new(:title, :url, :bundle_version, :short_version, :version, keyword_init: true) do extend T::Sig From f299d3ea6e5b5ce56a152c6f34ff1aaacc07184f Mon Sep 17 00:00:00 2001 From: Sam Ford <1584702+samford@users.noreply.github.com> Date: Sat, 19 Dec 2020 18:17:18 -0500 Subject: [PATCH 65/70] Strategy#page_headers: Disable debug in curl --- Library/Homebrew/livecheck/strategy.rb | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/Library/Homebrew/livecheck/strategy.rb b/Library/Homebrew/livecheck/strategy.rb index de46bfd582..5d66765ee8 100644 --- a/Library/Homebrew/livecheck/strategy.rb +++ b/Library/Homebrew/livecheck/strategy.rb @@ -89,12 +89,22 @@ module Homebrew headers = [] [:default, :browser].each do |user_agent| - stdout, _, status = curl_output( - "--fail", "--head", "--request", "GET", "--silent", "--location", - "--connect-timeout", 5, "--retry-max-time", 15, "--max-time", 10, - url, - user_agent: user_agent + args = [ + "--head", # Only work with the response headers + "--request", "GET", # Use a GET request (instead of HEAD) + "--silent", # Silent mode + "--location", # Follow redirects + "--connect-timeout", "5", # Max time allowed for connection (secs) + "--max-time", "10" # Max time allowed for transfer (secs) + ] + + result = curl_with_workarounds( + *args, url, + print_stdout: false, print_stderr: false, + debug: false, verbose: false, + user_agent: user_agent, retry: false ) + stdout = result.stdout while stdout.match?(/\AHTTP.*\r$/) h, stdout = stdout.split("\r\n\r\n", 2) @@ -104,7 +114,7 @@ module Homebrew .to_h.transform_keys(&:downcase) end - return (@headers[url] = headers) if status.success? + return (@headers[url] = headers) if result.success? end headers From 91ee5e0a6592a45a5b0cdce8a2f00f41f7c84c55 Mon Sep 17 00:00:00 2001 From: Markus Reiter Date: Sun, 20 Dec 2020 02:23:30 +0100 Subject: [PATCH 66/70] Simplify assignment of `curl` result. --- Library/Homebrew/livecheck/strategy.rb | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/Library/Homebrew/livecheck/strategy.rb b/Library/Homebrew/livecheck/strategy.rb index 5d66765ee8..c999fce0d3 100644 --- a/Library/Homebrew/livecheck/strategy.rb +++ b/Library/Homebrew/livecheck/strategy.rb @@ -98,13 +98,12 @@ module Homebrew "--max-time", "10" # Max time allowed for transfer (secs) ] - result = curl_with_workarounds( + stdout, _, status = curl_with_workarounds( *args, url, print_stdout: false, print_stderr: false, debug: false, verbose: false, user_agent: user_agent, retry: false ) - stdout = result.stdout while stdout.match?(/\AHTTP.*\r$/) h, stdout = stdout.split("\r\n\r\n", 2) @@ -114,7 +113,7 @@ module Homebrew .to_h.transform_keys(&:downcase) end - return (@headers[url] = headers) if result.success? + return (@headers[url] = headers) if status.success? end headers From 3261bbc0bdb10c3d5d1f97b0367e1fe15ddb3fc5 Mon Sep 17 00:00:00 2001 From: Sam Ford <1584702+samford@users.noreply.github.com> Date: Sat, 19 Dec 2020 22:26:37 -0500 Subject: [PATCH 67/70] Reorganize livecheck/livecheck test variables --- .../Homebrew/test/livecheck/livecheck_spec.rb | 59 ++++++++++--------- 1 file changed, 30 insertions(+), 29 deletions(-) diff --git a/Library/Homebrew/test/livecheck/livecheck_spec.rb b/Library/Homebrew/test/livecheck/livecheck_spec.rb index 5873f7ae78..64ae501f8d 100644 --- a/Library/Homebrew/test/livecheck/livecheck_spec.rb +++ b/Library/Homebrew/test/livecheck/livecheck_spec.rb @@ -20,6 +20,24 @@ describe Homebrew::Livecheck do end end + let(:c) do + Cask::CaskLoader.load(+<<-RUBY) + cask "test" do + version "0.0.1,2" + + url "https://brew.sh/test-0.0.1.tgz" + name "Test" + desc "Test cask" + homepage "https://brew.sh" + + livecheck do + url "https://formulae.brew.sh/api/formula/ruby.json" + regex(/"stable":"(\d+(?:\.\d+)+)"/i) + end + end + RUBY + end + let(:f_deprecated) do formula("test_deprecated") do desc "Deprecated test formula" @@ -38,11 +56,11 @@ describe Homebrew::Livecheck do end end - let(:f_gist) do - formula("test_gist") do - desc "Gist test formula" + let(:f_versioned) do + formula("test@0.0.1") do + desc "Versioned test formula" homepage "https://brew.sh" - url "https://gist.github.com/Homebrew/0000000000" + url "https://brew.sh/test-0.0.1.tgz" end end @@ -54,6 +72,14 @@ describe Homebrew::Livecheck do end end + let(:f_gist) do + formula("test_gist") do + desc "Gist test formula" + homepage "https://brew.sh" + url "https://gist.github.com/Homebrew/0000000000" + end + end + let(:f_skip) do formula("test_skip") do desc "Skipped test formula" @@ -66,31 +92,6 @@ describe Homebrew::Livecheck do end end - let(:f_versioned) do - formula("test@0.0.1") do - desc "Versioned test formula" - homepage "https://brew.sh" - url "https://brew.sh/test-0.0.1.tgz" - end - end - - let(:c) do - Cask::CaskLoader.load(+<<-RUBY) - cask "test" do - version "0.0.1,2" - - url "https://brew.sh/test-0.0.1.tgz" - name "Test" - homepage "https://brew.sh" - - livecheck do - url "https://formulae.brew.sh/api/formula/ruby.json" - regex(/"stable":"(\d+(?:\.\d+)+)"/i) - end - end - RUBY - end - describe "::formula_name" do it "returns the name of the formula" do expect(livecheck.formula_name(f)).to eq("test") From 732e67d5db0c972804adea594ca16b8a7cdf2f44 Mon Sep 17 00:00:00 2001 From: Sam Ford <1584702+samford@users.noreply.github.com> Date: Sat, 19 Dec 2020 23:45:26 -0500 Subject: [PATCH 68/70] Expand livecheck/livecheck tests --- .../Homebrew/test/livecheck/livecheck_spec.rb | 64 +++++++++++++++++++ 1 file changed, 64 insertions(+) diff --git a/Library/Homebrew/test/livecheck/livecheck_spec.rb b/Library/Homebrew/test/livecheck/livecheck_spec.rb index 64ae501f8d..a4cdb6bc4b 100644 --- a/Library/Homebrew/test/livecheck/livecheck_spec.rb +++ b/Library/Homebrew/test/livecheck/livecheck_spec.rb @@ -47,6 +47,24 @@ describe Homebrew::Livecheck do end end + let(:c_discontinued) do + Cask::CaskLoader.load(+<<-RUBY) + cask "test_discontinued" do + version "0.0.1" + sha256 :no_check + + url "https://brew.sh/test-0.0.1.tgz" + name "Test Discontinued" + desc "Discontinued test cask" + homepage "https://brew.sh" + + caveats do + discontinued + end + end + RUBY + end + let(:f_disabled) do formula("test_disabled") do desc "Disabled test formula" @@ -64,6 +82,34 @@ describe Homebrew::Livecheck do end end + let(:c_latest) do + Cask::CaskLoader.load(+<<-RUBY) + cask "test_latest" do + version :latest + sha256 :no_check + + url "https://brew.sh/test-0.0.1.tgz" + name "Test Latest" + desc "Latest test cask" + homepage "https://brew.sh" + end + RUBY + end + + # `URL#unversioned?` doesn't work properly when using the + # `Cask::CaskLoader.load` setup above, so we use `Cask::Cask.new` instead. + let(:c_unversioned) do + Cask::Cask.new "test_unversioned" do + version "1.2.3" + sha256 :no_check + + url "https://brew.sh/test.tgz" + name "Test Unversioned" + desc "Unversioned test cask" + homepage "https://brew.sh" + end + end + let(:f_head_only) do formula("test_head_only") do desc "HEAD-only test formula" @@ -133,6 +179,12 @@ describe Homebrew::Livecheck do .and not_to_output.to_stderr end + it "skips a discontinued cask without a livecheckable" do + expect { livecheck.skip_conditions(c_discontinued) } + .to output("test_discontinued : discontinued\n").to_stdout + .and not_to_output.to_stderr + end + it "skips a disabled formula without a livecheckable" do expect { livecheck.skip_conditions(f_disabled) } .to output("test_disabled : disabled\n").to_stdout @@ -145,6 +197,18 @@ describe Homebrew::Livecheck do .and not_to_output.to_stderr end + it "skips a cask containing `version :latest` without a livecheckable" do + expect { livecheck.skip_conditions(c_latest) } + .to output("test_latest : latest\n").to_stdout + .and not_to_output.to_stderr + end + + it "skips a cask containing an unversioned URL without a livecheckable" do + expect { livecheck.skip_conditions(c_unversioned) } + .to output("test_unversioned : unversioned\n").to_stdout + .and not_to_output.to_stderr + end + it "skips a HEAD-only formula if not installed" do expect { livecheck.skip_conditions(f_head_only) } .to output("test_head_only : HEAD only formula must be installed to be livecheckable\n").to_stdout From c6c303acfe0db2acf7acdda876badd8524ee2b63 Mon Sep 17 00:00:00 2001 From: Sam Ford <1584702+samford@users.noreply.github.com> Date: Sun, 20 Dec 2020 00:23:22 -0500 Subject: [PATCH 69/70] Sparkle: Rename contents to content --- Library/Homebrew/livecheck/strategy/sparkle.rb | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Library/Homebrew/livecheck/strategy/sparkle.rb b/Library/Homebrew/livecheck/strategy/sparkle.rb index c90649ceef..6ec8ae4c28 100644 --- a/Library/Homebrew/livecheck/strategy/sparkle.rb +++ b/Library/Homebrew/livecheck/strategy/sparkle.rb @@ -8,7 +8,7 @@ module Homebrew module Livecheck module Strategy # The {Sparkle} strategy fetches content at a URL and parses - # its contents as a Sparkle appcast in XML format. + # it as a Sparkle appcast in XML format. # # @api private class Sparkle @@ -90,9 +90,9 @@ module Homebrew match_data = { matches: {}, regex: regex, url: url } - contents = Strategy.page_content(url) + content = Strategy.page_content(url) - if (item = item_from_content(contents)) + if (item = item_from_content(content)) match = if block block.call(item)&.to_s else From 99ea95c28df20d0132e22320ca64b89a2ead466e Mon Sep 17 00:00:00 2001 From: Sam Ford <1584702+samford@users.noreply.github.com> Date: Sun, 20 Dec 2020 01:56:54 -0500 Subject: [PATCH 70/70] Sparkle: Expand tests --- .../Homebrew/livecheck/strategy/sparkle.rb | 1 - .../test/livecheck/strategy/sparkle_spec.rb | 58 +++++++++++++++++++ 2 files changed, 58 insertions(+), 1 deletion(-) diff --git a/Library/Homebrew/livecheck/strategy/sparkle.rb b/Library/Homebrew/livecheck/strategy/sparkle.rb index 6ec8ae4c28..63d61a9a9e 100644 --- a/Library/Homebrew/livecheck/strategy/sparkle.rb +++ b/Library/Homebrew/livecheck/strategy/sparkle.rb @@ -81,7 +81,6 @@ module Homebrew items.max_by(&:bundle_version) end - private_class_method :item_from_content # Checks the content at the URL for new versions. sig { params(url: String, regex: T.nilable(Regexp)).returns(T::Hash[Symbol, T.untyped]) } diff --git a/Library/Homebrew/test/livecheck/strategy/sparkle_spec.rb b/Library/Homebrew/test/livecheck/strategy/sparkle_spec.rb index f9a878c1d4..f98e43ef00 100644 --- a/Library/Homebrew/test/livecheck/strategy/sparkle_spec.rb +++ b/Library/Homebrew/test/livecheck/strategy/sparkle_spec.rb @@ -8,9 +8,67 @@ describe Homebrew::Livecheck::Strategy::Sparkle do let(:url) { "https://www.example.com/example/appcast.xml" } + let(:appcast_data) { + { + title: "Version 1.2.3", + url: "https://www.example.com/example/example.tar.gz", + bundle_version: Homebrew::BundleVersion.new("1.2.3", "1234"), + short_version: "1.2.3", + version: "1234", + } + } + + let(:appcast_item) { + Homebrew::Livecheck::Strategy::Sparkle::Item.new( + { + title: appcast_data[:title], + url: appcast_data[:url], + bundle_version: appcast_data[:bundle_version], + short_version: appcast_data[:bundle_version]&.short_version, + version: appcast_data[:bundle_version]&.version, + }, + ) + } + + let(:appcast_xml) { + <<~EOS + + + + Example Changelog + #{url} + Most recent changes with links to updates. + en + + #{appcast_data[:title]} + 10.10 + https://www.example.com/example/1.2.3.html + + + + + EOS + } + describe "::match?" do it "returns true for any URL" do expect(sparkle.match?(url)).to be true end end + + describe "::item_from_content" do + let(:item_from_appcast_xml) { sparkle.item_from_content(appcast_xml) } + + it "returns nil if content is blank" do + expect(sparkle.item_from_content("")).to be nil + end + + it "returns an Item when given XML data" do + expect(item_from_appcast_xml).to be_a(Homebrew::Livecheck::Strategy::Sparkle::Item) + expect(item_from_appcast_xml.title).to eq(appcast_item.title) + expect(item_from_appcast_xml.url).to eq(appcast_item.url) + expect(item_from_appcast_xml.bundle_version.short_version).to eq(appcast_item.bundle_version.short_version) + expect(item_from_appcast_xml.bundle_version.version).to eq(appcast_item.bundle_version.version) + end + end end