From 29006396dd4fd75602b44ebc81d7a30181b2d55f Mon Sep 17 00:00:00 2001 From: EricFromCanada Date: Wed, 19 Feb 2020 11:48:54 -0500 Subject: [PATCH 1/6] audit: check apache.org URLs --- Library/Homebrew/rubocops/urls.rb | 12 +++++++++++- Library/Homebrew/test/rubocops/urls_spec.rb | 5 +++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/Library/Homebrew/rubocops/urls.rb b/Library/Homebrew/rubocops/urls.rb index af8d299901..1391ae95b3 100644 --- a/Library/Homebrew/rubocops/urls.rb +++ b/Library/Homebrew/rubocops/urls.rb @@ -41,6 +41,11 @@ module RuboCop problem "Please don't use fossies.org in the url (using as a mirror is fine)" end + apache_pattern = %r{^https?://(?:[^/]*\.)?apache\.org/(?:dyn/closer\.cgi\?path=/?|dist/)(.*)}i + audit_urls(urls, apache_pattern) do |match, url| + problem "#{url} should be `https://www.apache.org/dyn/closer.lua?path=#{match[1]}`" + end + audit_urls(mirrors, /.*/) do |_, mirror| urls.each do |url| url_string = string_content(parameters(url).first) @@ -58,7 +63,7 @@ module RuboCop %r{^http://ftpmirror\.gnu\.org/}, %r{^http://download\.savannah\.gnu\.org/}, %r{^http://download-mirror\.savannah\.gnu\.org/}, - %r{^http://[^/]*\.apache\.org/}, + %r{^http://(?:[^/]*\.)?apache\.org/}, %r{^http://code\.google\.com/}, %r{^http://fossies\.org/}, %r{^http://mirrors\.kernel\.org/}, @@ -78,6 +83,11 @@ module RuboCop problem "Please use https:// for #{url}" end + apache_mirror_pattern = %r{^https?://(?:[^/]*\.)?apache\.org/dyn/closer\.(?:cgi|lua)\?path=/?(.*)}i + audit_urls(mirrors, apache_mirror_pattern) do |match, mirror| + problem "Please use `https://archive.apache.org/dist/#{match[1]}` as a mirror instead of #{mirror}." + end + cpan_pattern = %r{^http://search\.mcpan\.org/CPAN/(.*)}i audit_urls(urls, cpan_pattern) do |match, url| problem "#{url} should be `https://cpan.metacpan.org/#{match[1]}`" diff --git a/Library/Homebrew/test/rubocops/urls_spec.rb b/Library/Homebrew/test/rubocops/urls_spec.rb index abedeeaa37..c82bdd3a33 100644 --- a/Library/Homebrew/test/rubocops/urls_spec.rb +++ b/Library/Homebrew/test/rubocops/urls_spec.rb @@ -18,6 +18,11 @@ describe RuboCop::Cop::FormulaAudit::Urls do "url" => "http://tools.ietf.org/tools/rfcmarkup/rfcmarkup-1.119.tgz", "msg" => "Please use https:// for http://tools.ietf.org/tools/rfcmarkup/rfcmarkup-1.119.tgz", "col" => 2, + }, { + "url" => "https://apache.org/dyn/closer.cgi?path=/apr/apr-1.7.0.tar.bz2", + "msg" => "https://apache.org/dyn/closer.cgi?path=/apr/apr-1.7.0.tar.bz2 should be " \ + "`https://www.apache.org/dyn/closer.lua?path=apr/apr-1.7.0.tar.bz2`", + "col" => 2, }, { "url" => "http://search.mcpan.org/CPAN/authors/id/Z/ZE/ZEFRAM/Perl4-CoreLibs-0.003.tar.gz", "msg" => "http://search.mcpan.org/CPAN/authors/id/Z/ZE/ZEFRAM/Perl4-CoreLibs-0.003.tar.gz should be " \ From 50cc9db0f279fb199e9a08e7c5922038ca0b8c88 Mon Sep 17 00:00:00 2001 From: EricFromCanada Date: Tue, 18 Feb 2020 18:09:53 -0500 Subject: [PATCH 2/6] bump-formula-pr: also infer mirrors when a checksum is provided --- Library/Homebrew/dev-cmd/bump-formula-pr.rb | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/Library/Homebrew/dev-cmd/bump-formula-pr.rb b/Library/Homebrew/dev-cmd/bump-formula-pr.rb index 545819f569..e9017856e5 100644 --- a/Library/Homebrew/dev-cmd/bump-formula-pr.rb +++ b/Library/Homebrew/dev-cmd/bump-formula-pr.rb @@ -168,7 +168,13 @@ module Homebrew new_hash = args[hash_type] if hash_type new_tag = args.tag new_revision = args.revision - new_mirror = args.mirror + new_mirror ||= args.mirror + new_mirror ||= case new_url + when requested_spec != :devel && %r{.*ftp.gnu.org/gnu.*} + new_url.sub "ftp.gnu.org/gnu", "ftpmirror.gnu.org" + when %r{.*mirrors.ocf.berkeley.edu/debian.*} + new_url.sub "mirrors.ocf.berkeley.edu/debian", "mirrorservice.org/sites/ftp.debian.org/debian" + end forced_version = args.version new_url_hash = if new_url && new_hash true @@ -179,12 +185,6 @@ module Homebrew elsif !new_url odie "#{formula}: no --url= argument specified!" else - new_mirror ||= case new_url - when requested_spec != :devel && %r{.*ftp.gnu.org/gnu.*} - new_url.sub "ftp.gnu.org/gnu", "ftpmirror.gnu.org" - when %r{.*mirrors.ocf.berkeley.edu/debian.*} - new_url.sub "mirrors.ocf.berkeley.edu/debian", "mirrorservice.org/sites/ftp.debian.org/debian" - end resource = Resource.new { @url = new_url } resource.download_strategy = DownloadStrategyDetector.detect_from_url(new_url) resource.owner = Resource.new(formula.name) From 0132b905bff01b1d1cc493d664398d54c20ccc54 Mon Sep 17 00:00:00 2001 From: EricFromCanada Date: Tue, 18 Feb 2020 18:41:02 -0500 Subject: [PATCH 3/6] bump-formula-pr: automatically mirror Apache & GNU projects --- Library/Homebrew/dev-cmd/bump-formula-pr.rb | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Library/Homebrew/dev-cmd/bump-formula-pr.rb b/Library/Homebrew/dev-cmd/bump-formula-pr.rb index e9017856e5..ba6b4c7bc1 100644 --- a/Library/Homebrew/dev-cmd/bump-formula-pr.rb +++ b/Library/Homebrew/dev-cmd/bump-formula-pr.rb @@ -172,6 +172,10 @@ module Homebrew new_mirror ||= case new_url when requested_spec != :devel && %r{.*ftp.gnu.org/gnu.*} new_url.sub "ftp.gnu.org/gnu", "ftpmirror.gnu.org" + when %r{.*download.savannah.gnu.org/*} + new_url.sub "download.savannah.gnu.org", "download-mirror.savannah.gnu.org" + when %r{.*www.apache.org/dyn/closer.lua\?path=.*} + new_url.sub "www.apache.org/dyn/closer.lua?path=", "archive.apache.org/dist/" when %r{.*mirrors.ocf.berkeley.edu/debian.*} new_url.sub "mirrors.ocf.berkeley.edu/debian", "mirrorservice.org/sites/ftp.debian.org/debian" end From 7ec2ffda93d297477a624cf2fbee3df068e79530 Mon Sep 17 00:00:00 2001 From: EricFromCanada Date: Tue, 18 Feb 2020 18:12:00 -0500 Subject: [PATCH 4/6] bump-formula-pr: warn when removing mirrors --- Library/Homebrew/dev-cmd/bump-formula-pr.rb | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/Library/Homebrew/dev-cmd/bump-formula-pr.rb b/Library/Homebrew/dev-cmd/bump-formula-pr.rb index ba6b4c7bc1..2ba246a8cc 100644 --- a/Library/Homebrew/dev-cmd/bump-formula-pr.rb +++ b/Library/Homebrew/dev-cmd/bump-formula-pr.rb @@ -312,6 +312,17 @@ module Homebrew new_formula_version = formula_version(formula, requested_spec, new_contents) + if !new_mirror && !formula_spec.mirrors.empty? + if Homebrew.args.force? + opoo "#{formula}: Removing all mirrors because a --mirror= argument was not specified." + else + odie <<~EOS + #{formula}: a --mirror= argument for updating the mirror URL was not specified. + Use --force to remove all mirrors. + EOS + end + end + if new_formula_version < old_formula_version formula.path.atomic_write(backup_file) unless args.dry_run? odie <<~EOS From a24477dad64ed8c7069811b6f0e3390973b9c9cf Mon Sep 17 00:00:00 2001 From: EricFromCanada Date: Wed, 19 Feb 2020 14:55:13 -0500 Subject: [PATCH 5/6] bump-formula-pr: fix output when listing guesses --- Library/Homebrew/dev-cmd/bump-formula-pr.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Library/Homebrew/dev-cmd/bump-formula-pr.rb b/Library/Homebrew/dev-cmd/bump-formula-pr.rb index 2ba246a8cc..f969c245c9 100644 --- a/Library/Homebrew/dev-cmd/bump-formula-pr.rb +++ b/Library/Homebrew/dev-cmd/bump-formula-pr.rb @@ -146,7 +146,7 @@ module Homebrew if guesses.count == 1 formula = guesses.shift elsif guesses.count > 1 - odie "Couldn't guess formula for sure; could be one of these:\n#{guesses}" + odie "Couldn't guess formula for sure; could be one of these:\n#{guesses.map(&:name).join(", ")}" end end raise FormulaUnspecifiedError unless formula From 6f5cd0e7fe87ac7a49b142540ac3ccdfcda58d8b Mon Sep 17 00:00:00 2001 From: EricFromCanada Date: Wed, 19 Feb 2020 16:27:34 -0500 Subject: [PATCH 6/6] bump-formula-pr: support add multiple mirror URLs with --mirror --- Library/Homebrew/dev-cmd/bump-formula-pr.rb | 18 ++++++++++-------- docs/Manpage.md | 2 +- manpages/brew.1 | 2 +- 3 files changed, 12 insertions(+), 10 deletions(-) diff --git a/Library/Homebrew/dev-cmd/bump-formula-pr.rb b/Library/Homebrew/dev-cmd/bump-formula-pr.rb index f969c245c9..d3a8098dab 100644 --- a/Library/Homebrew/dev-cmd/bump-formula-pr.rb +++ b/Library/Homebrew/dev-cmd/bump-formula-pr.rb @@ -40,8 +40,9 @@ module Homebrew description: "Print the pull request URL instead of opening in a browser." switch "--no-fork", description: "Don't try to fork the repository." - flag "--mirror=", - description: "Use the specified as a mirror URL." + comma_array "--mirror=", + description: "Use the specified as a mirror URL. If is a comma-separated list "\ + "of URLs, multiple mirrors will be added." flag "--version=", description: "Use the specified to override the value parsed from the URL or tag. Note "\ "that `--version=0` can be used to delete an existing version override from a "\ @@ -168,7 +169,7 @@ module Homebrew new_hash = args[hash_type] if hash_type new_tag = args.tag new_revision = args.revision - new_mirror ||= args.mirror + new_mirrors ||= args.mirror new_mirror ||= case new_url when requested_spec != :devel && %r{.*ftp.gnu.org/gnu.*} new_url.sub "ftp.gnu.org/gnu", "ftpmirror.gnu.org" @@ -179,6 +180,7 @@ module Homebrew when %r{.*mirrors.ocf.berkeley.edu/debian.*} new_url.sub "mirrors.ocf.berkeley.edu/debian", "mirrorservice.org/sites/ftp.debian.org/debian" end + new_mirrors ||= [new_mirror] unless new_mirror.nil? forced_version = args.version new_url_hash = if new_url && new_hash true @@ -253,10 +255,10 @@ module Homebrew backup_file = File.read(formula.path) unless args.dry_run? - if new_mirror + if new_mirrors replacement_pairs << [ /^( +)(url \"#{Regexp.escape(new_url)}\"\n)/m, - "\\1\\2\\1mirror \"#{new_mirror}\"\n", + "\\1\\2\\1mirror \"#{new_mirrors.join("\"\n\\1mirror \"")}\"\n", ] end @@ -278,9 +280,9 @@ module Homebrew old_formula_version.to_s, forced_version, ] - elsif new_mirror + elsif new_mirrors replacement_pairs << [ - /^( +)(mirror \"#{new_mirror}\"\n)/m, + /^( +)(mirror \"#{new_mirrors.last}\"\n)/m, "\\1\\2\\1version \"#{forced_version}\"\n", ] else @@ -312,7 +314,7 @@ module Homebrew new_formula_version = formula_version(formula, requested_spec, new_contents) - if !new_mirror && !formula_spec.mirrors.empty? + if !new_mirrors && !formula_spec.mirrors.empty? if Homebrew.args.force? opoo "#{formula}: Removing all mirrors because a --mirror= argument was not specified." else diff --git a/docs/Manpage.md b/docs/Manpage.md index 0f21851f56..0560e0bea5 100644 --- a/docs/Manpage.md +++ b/docs/Manpage.md @@ -767,7 +767,7 @@ uses. * `--no-fork`: Don't try to fork the repository. * `--mirror`: - Use the specified *`URL`* as a mirror URL. + Use the specified *`URL`* as a mirror URL. If *`URL`* is a comma-separated list of URLs, multiple mirrors will be added. * `--version`: Use the specified *`version`* to override the value parsed from the URL or tag. Note that `--version=0` can be used to delete an existing version override from a formula if it has become redundant. * `--message`: diff --git a/manpages/brew.1 b/manpages/brew.1 index e9bf2853ad..bb74d1aaf2 100644 --- a/manpages/brew.1 +++ b/manpages/brew.1 @@ -975,7 +975,7 @@ Don\'t try to fork the repository\. . .TP \fB\-\-mirror\fR -Use the specified \fIURL\fR as a mirror URL\. +Use the specified \fIURL\fR as a mirror URL\. If \fIURL\fR is a comma\-separated list of URLs, multiple mirrors will be added\. . .TP \fB\-\-version\fR