From 59adb553384847dd893e20fdd77c4f5288124e8e Mon Sep 17 00:00:00 2001 From: Dario Vladovic Date: Sat, 7 Nov 2020 03:18:42 +0100 Subject: [PATCH 1/8] livecheck: refactor url preprocessing --- Library/Homebrew/livecheck/livecheck.rb | 49 ++++++++----------------- 1 file changed, 16 insertions(+), 33 deletions(-) diff --git a/Library/Homebrew/livecheck/livecheck.rb b/Library/Homebrew/livecheck/livecheck.rb index fae6e91804..94e9c550db 100644 --- a/Library/Homebrew/livecheck/livecheck.rb +++ b/Library/Homebrew/livecheck/livecheck.rb @@ -3,6 +3,7 @@ require "livecheck/strategy" require "ruby-progressbar" +require "uri" module Homebrew # The {Livecheck} module consists of methods used by the `brew livecheck` @@ -13,20 +14,6 @@ module Homebrew module Livecheck module_function - GITHUB_SPECIAL_CASES = %w[ - api.github.com - /latest - mednafen - camlp5 - kotlin - osrm-backend - prometheus - pyenv-virtualenv - sysdig - shairport-sync - yuicompressor - ].freeze - UNSTABLE_VERSION_KEYWORDS = %w[ alpha beta @@ -316,27 +303,23 @@ module Homebrew # Preprocesses and returns the URL used by livecheck. # @return [String] def preprocess_url(url) - # Check for GitHub repos on github.com, not AWS - url = url.sub("github.s3.amazonaws.com", "github.com") if url.include?("github") + uri = URI.parse url + host = uri.host + path = uri.path.delete_prefix("/").delete_suffix(".git") - # Use repo from GitHub or GitLab inferred from download URL - if url.include?("github.com") && GITHUB_SPECIAL_CASES.none? { |sc| url.include? sc } - if url.include? "archive" - url = url.sub(%r{/archive/.*}, ".git") if url.include? "github" - elsif url.include? "releases" - url = url.sub(%r{/releases/.*}, ".git") - elsif url.include? "downloads" - url = "#{Pathname.new(url.sub(%r{/downloads(.*)}, "\\1")).dirname}.git" - elsif !url.end_with?(".git") - # Truncate the URL at the user/repo part, if possible - %r{(?(?:[a-z]+://)?github.com/[^/]+/[^/#]+)} =~ url - url = github_repo_url if github_repo_url.present? + if host == "github.s3.amazonaws.com" + owner, repo = path.sub(%r{^downloads/}, "").split("/") + return "https://github.com/#{owner}/#{repo}.git" + end - url.delete_suffix!("/") if url.end_with?("/") - url += ".git" - end - elsif url.include?("/-/archive/") - url = url.sub(%r{/-/archive/.*$}i, ".git") + if ["github.com", "tildegit.org", "gitlab.com"].include? host + return url if path.match? %r{/releases/latest/?$} + + owner, repo = path.split("/") + url = "https://#{host}/#{owner}/#{repo}.git" + elsif host == "git.sr.ht" + owner, repo = path.split("/") + url = "https://#{host}/#{owner}/#{repo}" end url From 1f103f88d09f996b666325a19fadf315d1e8a6ba Mon Sep 17 00:00:00 2001 From: Dario Vladovic Date: Sat, 7 Nov 2020 03:19:26 +0100 Subject: [PATCH 2/8] livecheck: enable git strategy for sourcehut --- Library/Homebrew/download_strategy.rb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Library/Homebrew/download_strategy.rb b/Library/Homebrew/download_strategy.rb index 81dcf445fc..a2deeed57f 100644 --- a/Library/Homebrew/download_strategy.rb +++ b/Library/Homebrew/download_strategy.rb @@ -1225,7 +1225,8 @@ class DownloadStrategyDetector when %r{^https?://github\.com/[^/]+/[^/]+\.git$} GitHubGitDownloadStrategy when %r{^https?://.+\.git$}, - %r{^git://} + %r{^git://}, + %r{^https?://git\.sr\.ht/[^/]+/[^/]+$} GitDownloadStrategy when %r{^https?://www\.apache\.org/dyn/closer\.cgi}, %r{^https?://www\.apache\.org/dyn/closer\.lua} From 74fd700445c94b76a72fc67452fec934645e0389 Mon Sep 17 00:00:00 2001 From: Dario Vladovic Date: Wed, 11 Nov 2020 02:32:44 +0100 Subject: [PATCH 3/8] livecheck: fix github & gitlab url processing - support both `github.com/downloads//` and `github.s3.amazonaws.com//` URL patterns - support self-hosted GitLab installations (with project groups) - support _well-known_ Gitea and Gogs instances --- Library/Homebrew/livecheck/livecheck.rb | 35 +++++++++++++++++++------ 1 file changed, 27 insertions(+), 8 deletions(-) diff --git a/Library/Homebrew/livecheck/livecheck.rb b/Library/Homebrew/livecheck/livecheck.rb index 94e9c550db..fad448d0c3 100644 --- a/Library/Homebrew/livecheck/livecheck.rb +++ b/Library/Homebrew/livecheck/livecheck.rb @@ -14,6 +14,17 @@ module Homebrew module Livecheck module_function + GITEA_INSTANCES = %w[ + codeberg.org + gitea.com + opendev.org + tildegit.org + ].freeze + + GOGS_INSTANCES = %w[ + lolg.it + ].freeze + UNSTABLE_VERSION_KEYWORDS = %w[ alpha beta @@ -304,22 +315,30 @@ module Homebrew # @return [String] def preprocess_url(url) uri = URI.parse url - host = uri.host + host = uri.host == "github.s3.amazonaws.com" ? "github.com" : uri.host path = uri.path.delete_prefix("/").delete_suffix(".git") + scheme = uri.scheme - if host == "github.s3.amazonaws.com" - owner, repo = path.sub(%r{^downloads/}, "").split("/") - return "https://github.com/#{owner}/#{repo}.git" - end + if host == "github.com" + return url if path.match? %r{/releases/latest/?$} - if ["github.com", "tildegit.org", "gitlab.com"].include? host + owner, repo = path.delete_prefix("downloads/").split("/") + url = "#{scheme}://#{host}/#{owner}/#{repo}.git" + elsif GITEA_INSTANCES.include? host return url if path.match? %r{/releases/latest/?$} owner, repo = path.split("/") - url = "https://#{host}/#{owner}/#{repo}.git" + url = "#{scheme}://#{host}/#{owner}/#{repo}.git" + elsif GOGS_INSTANCES.include? host + owner, repo = path.split("/") + url = "#{scheme}://#{host}/#{owner}/#{repo}.git" + # sourcehut elsif host == "git.sr.ht" owner, repo = path.split("/") - url = "https://#{host}/#{owner}/#{repo}" + url = "#{scheme}://#{host}/#{owner}/#{repo}" + # gitlab + elsif path.include?("/-/archive/") + url = url.sub(%r{/-/archive/.*$}i, ".git") end url From 1f40d84ab125f7744a1415aafbccd9bfb143e30d Mon Sep 17 00:00:00 2001 From: Sam Ford <1584702+samford@users.noreply.github.com> Date: Thu, 26 Nov 2020 09:50:00 -0500 Subject: [PATCH 4/8] livecheck: expand #preprocess_url tests --- .../Homebrew/test/livecheck/livecheck_spec.rb | 25 +++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/Library/Homebrew/test/livecheck/livecheck_spec.rb b/Library/Homebrew/test/livecheck/livecheck_spec.rb index edf2d27216..4bef7a8992 100644 --- a/Library/Homebrew/test/livecheck/livecheck_spec.rb +++ b/Library/Homebrew/test/livecheck/livecheck_spec.rb @@ -200,5 +200,30 @@ describe Homebrew::Livecheck do expect(livecheck.preprocess_url("https://brew.sh/Homebrew/brew/-/archive/1.0.0/brew-1.0.0.tar.gz")) .to eq("https://brew.sh/Homebrew/brew.git") end + + it "returns the Git repository URL for a Codeberg archive URL" do + expect(livecheck.preprocess_url("https://codeberg.org/Homebrew/brew/archive/brew-1.0.0.tar.gz")) + .to eq("https://codeberg.org/Homebrew/brew.git") + end + + it "returns the Git repository URL for a Gitea archive URL" do + expect(livecheck.preprocess_url("https://gitea.com/Homebrew/brew/archive/brew-1.0.0.tar.gz")) + .to eq("https://gitea.com/Homebrew/brew.git") + end + + it "returns the Git repository URL for an Opendev archive URL" do + expect(livecheck.preprocess_url("https://opendev.org/Homebrew/brew/archive/brew-1.0.0.tar.gz")) + .to eq("https://opendev.org/Homebrew/brew.git") + end + + it "returns the Git repository URL for a tildegit archive URL" do + expect(livecheck.preprocess_url("https://tildegit.org/Homebrew/brew/archive/brew-1.0.0.tar.gz")) + .to eq("https://tildegit.org/Homebrew/brew.git") + end + + it "returns the Git repository URL for a LOL Git archive URL" do + expect(livecheck.preprocess_url("https://lolg.it/Homebrew/brew/archive/brew-1.0.0.tar.gz")) + .to eq("https://lolg.it/Homebrew/brew.git") + end end end From d07590442c49c13257e6927481fd928f235905d9 Mon Sep 17 00:00:00 2001 From: Sam Ford <1584702+samford@users.noreply.github.com> Date: Thu, 26 Nov 2020 10:31:38 -0500 Subject: [PATCH 5/8] livecheck: handle URI#parse URI::InvalidURIError --- Library/Homebrew/livecheck/livecheck.rb | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/Library/Homebrew/livecheck/livecheck.rb b/Library/Homebrew/livecheck/livecheck.rb index fad448d0c3..0ce4a1d6e7 100644 --- a/Library/Homebrew/livecheck/livecheck.rb +++ b/Library/Homebrew/livecheck/livecheck.rb @@ -314,7 +314,12 @@ module Homebrew # Preprocesses and returns the URL used by livecheck. # @return [String] def preprocess_url(url) - uri = URI.parse url + begin + uri = URI.parse url + rescue URI::InvalidURIError + return url + end + host = uri.host == "github.s3.amazonaws.com" ? "github.com" : uri.host path = uri.path.delete_prefix("/").delete_suffix(".git") scheme = uri.scheme From 5332e877b2e9250cb604ba76c1d368b6c6ed5fa0 Mon Sep 17 00:00:00 2001 From: Sam Ford <1584702+samford@users.noreply.github.com> Date: Thu, 26 Nov 2020 10:32:55 -0500 Subject: [PATCH 6/8] livecheck: improve host string comparison --- Library/Homebrew/livecheck/livecheck.rb | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Library/Homebrew/livecheck/livecheck.rb b/Library/Homebrew/livecheck/livecheck.rb index 0ce4a1d6e7..4ed490337b 100644 --- a/Library/Homebrew/livecheck/livecheck.rb +++ b/Library/Homebrew/livecheck/livecheck.rb @@ -324,21 +324,21 @@ module Homebrew path = uri.path.delete_prefix("/").delete_suffix(".git") scheme = uri.scheme - if host == "github.com" + if host.end_with?("github.com") return url if path.match? %r{/releases/latest/?$} owner, repo = path.delete_prefix("downloads/").split("/") url = "#{scheme}://#{host}/#{owner}/#{repo}.git" - elsif GITEA_INSTANCES.include? host + elsif host.end_with?(*GITEA_INSTANCES) return url if path.match? %r{/releases/latest/?$} owner, repo = path.split("/") url = "#{scheme}://#{host}/#{owner}/#{repo}.git" - elsif GOGS_INSTANCES.include? host + elsif host.end_with?(*GOGS_INSTANCES) owner, repo = path.split("/") url = "#{scheme}://#{host}/#{owner}/#{repo}.git" # sourcehut - elsif host == "git.sr.ht" + elsif host.end_with?("git.sr.ht") owner, repo = path.split("/") url = "#{scheme}://#{host}/#{owner}/#{repo}" # gitlab From 417bb2ebfdca8d97325520bf191a86f845786be3 Mon Sep 17 00:00:00 2001 From: Sam Ford <1584702+samford@users.noreply.github.com> Date: Thu, 26 Nov 2020 10:33:20 -0500 Subject: [PATCH 7/8] livecheck: clean up comment --- Library/Homebrew/livecheck/livecheck.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Library/Homebrew/livecheck/livecheck.rb b/Library/Homebrew/livecheck/livecheck.rb index 4ed490337b..1c4d6d3699 100644 --- a/Library/Homebrew/livecheck/livecheck.rb +++ b/Library/Homebrew/livecheck/livecheck.rb @@ -341,7 +341,7 @@ module Homebrew elsif host.end_with?("git.sr.ht") owner, repo = path.split("/") url = "#{scheme}://#{host}/#{owner}/#{repo}" - # gitlab + # GitLab (gitlab.com or self-hosted) elsif path.include?("/-/archive/") url = url.sub(%r{/-/archive/.*$}i, ".git") end From 5a007a4ec6eed7df2c255e1175ff918fa1f1b00e Mon Sep 17 00:00:00 2001 From: Sam Ford <1584702+samford@users.noreply.github.com> Date: Thu, 26 Nov 2020 13:34:16 -0500 Subject: [PATCH 8/8] livecheck: expand #preprocess_url tests more --- Library/Homebrew/test/livecheck/livecheck_spec.rb | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/Library/Homebrew/test/livecheck/livecheck_spec.rb b/Library/Homebrew/test/livecheck/livecheck_spec.rb index 4bef7a8992..261896abd9 100644 --- a/Library/Homebrew/test/livecheck/livecheck_spec.rb +++ b/Library/Homebrew/test/livecheck/livecheck_spec.rb @@ -156,6 +156,12 @@ describe Homebrew::Livecheck do describe "::preprocess_url" do let(:github_git_url_with_extension) { "https://github.com/Homebrew/brew.git" } + it "returns the unmodified URL for an unparseable URL" do + # Modeled after the `head` URL in the `ncp` formula + expect(livecheck.preprocess_url(":something:cvs:@cvs.brew.sh:/cvs")) + .to eq(":something:cvs:@cvs.brew.sh:/cvs") + end + it "returns the unmodified URL for a GitHub URL ending in .git" do expect(livecheck.preprocess_url(github_git_url_with_extension)) .to eq(github_git_url_with_extension) @@ -225,5 +231,10 @@ describe Homebrew::Livecheck do expect(livecheck.preprocess_url("https://lolg.it/Homebrew/brew/archive/brew-1.0.0.tar.gz")) .to eq("https://lolg.it/Homebrew/brew.git") end + + it "returns the Git repository URL for a sourcehut archive URL" do + expect(livecheck.preprocess_url("https://git.sr.ht/~Homebrew/brew/archive/1.0.0.tar.gz")) + .to eq("https://git.sr.ht/~Homebrew/brew") + end end end