From b0bd15fb41b711782e04d24b307dc515ef65aaa2 Mon Sep 17 00:00:00 2001 From: Shaun Jackman Date: Sun, 21 Feb 2021 22:12:17 -0800 Subject: [PATCH 1/8] pr-upload: Upload bottles to Archive.org --archive-item specifies the item identifier. HOMEBREW_ARCHIVE_KEY=access:secret specifies the S3 key. --- Library/Homebrew/archive.rb | 185 ++++++++++++++++++++++++++ Library/Homebrew/dev-cmd/pr-upload.rb | 52 ++++++-- Library/Homebrew/env_config.rb | 4 + 3 files changed, 229 insertions(+), 12 deletions(-) create mode 100644 Library/Homebrew/archive.rb diff --git a/Library/Homebrew/archive.rb b/Library/Homebrew/archive.rb new file mode 100644 index 0000000000..a3d49b00bd --- /dev/null +++ b/Library/Homebrew/archive.rb @@ -0,0 +1,185 @@ +# typed: false +# frozen_string_literal: true + +require "digest/md5" +require "utils/curl" + +# Archive API client. +# +# @api private +class Archive + extend T::Sig + + include Context + include Utils::Curl + + class Error < RuntimeError + end + + sig { returns(String) } + def inspect + "#" + end + + sig { params(item: T.nilable(String)).void } + def initialize(item: "homebrew") + @archive_item = item + + raise UsageError, "Must set the Archive item!" unless @archive_item + + ENV["HOMEBREW_FORCE_HOMEBREW_ON_LINUX"] = "1" if @archive_item == "homebrew" && !OS.mac? + end + + def open_api(url, *args, auth: true) + if auth + raise UsageError, "HOMEBREW_ARCHIVE_KEY is unset." unless (key = Homebrew::EnvConfig.archive_key) + + if key.exclude?(":") + raise UsageError, + "Use HOMEBREW_ARCHIVE_KEY=access:secret. See https://archive.org/account/s3.php" + end + + args += ["--header", "Authorization: AWS #{key}"] + end + + curl(*args, url, print_stdout: false, secrets: key) + end + + sig { + params(local_file: String, + dir: String, + remote_file: String, + warn_on_error: T.nilable(T::Boolean)).void + } + def upload(local_file, dir:, remote_file:, warn_on_error: false) + unless File.exist? local_file + msg = "#{local_file} for upload doesn't exist!" + raise Error, msg unless warn_on_error + + # Warn and return early here since we know this upload is going to fail. + opoo msg + return + end + + md5_base64 = Digest::MD5.base64digest(File.read(local_file)) + url = "https://#{@archive_item}.s3.us.archive.org/#{dir}/#{remote_file}" + args = ["--upload-file", local_file, "--header", "Content-MD5: #{md5_base64}"] + args << "--fail" unless warn_on_error + result = T.unsafe(self).open_api(url, *args) + return if result.success? && result.stdout.exclude?("Error") + + msg = "Bottle upload failed: #{result.stdout}" + raise msg unless warn_on_error + + opoo msg + end + + sig { params(url: String).returns(T::Boolean) } + def stable_mirrored?(url) + headers, = curl_output("--connect-timeout", "15", "--location", "--head", url) + status_code = headers.scan(%r{^HTTP/.* (\d+)}).last.first + status_code.start_with?("2") + end + + sig { + params(formula: Formula, + dir: String, + warn_on_error: T::Boolean).returns(String) + } + def mirror_formula(formula, dir: "mirror", warn_on_error: false) + formula.downloader.fetch + + filename = ERB::Util.url_encode(formula.downloader.basename) + destination_url = "https://archive.org/download/#{@archive_item}/#{dir}/#{filename}" + + odebug "Uploading to #{destination_url}" + + upload( + formula.downloader.cached_location, + dir: dir, + remote_file: filename, + warn_on_error: warn_on_error, + ) + + destination_url + end + + # Gets the MD5 hash of the specified remote file. + # + # @return the hash, the empty string (if the file doesn't have a hash), nil (if the file doesn't exist) + sig { params(dir: String, remote_file: String).returns(T.nilable(String)) } + def remote_md5(dir:, remote_file:) + url = "https://#{@archive_item}.s3.us.archive.org/#{dir}/#{remote_file}" + result = curl_output "--fail", "--silent", "--head", "--location", url + if result.success? + result.stdout.match(/^ETag: "(\h{32})"/)&.values_at(1)&.first || "" + else + raise Error if result.status.exitstatus != 22 && result.stderr.exclude?("404 Not Found") + + nil + end + end + + sig { params(directory: String, filename: String).returns(String) } + def file_delete_instructions(directory, filename) + <<~EOS + Run: + curl -X DELETE -H "Authorization: AWS $HOMEBREW_ARCHIVE_KEY" https://#{@archive_item}.s3.us.archive.org/#{directory}/#{filename} + Or run: + ia delete #{@archive_item} #{directory}/#{filename} + EOS + end + + sig { + params(bottles_hash: T::Hash[String, T.untyped], + warn_on_error: T.nilable(T::Boolean)).void + } + def upload_bottles(bottles_hash, warn_on_error: false) + bottles_hash.each do |_formula_name, bottle_hash| + directory = bottle_hash["bintray"]["repository"] + bottle_count = bottle_hash["bottle"]["tags"].length + + bottle_hash["bottle"]["tags"].each do |_tag, tag_hash| + filename = tag_hash["filename"] # URL encoded in Bottle::Filename#archive + delete_instructions = file_delete_instructions(directory, filename) + + local_filename = tag_hash["local_filename"] + md5 = Digest::MD5.hexdigest(File.read(local_filename)) + + odebug "Checking remote file #{@archive_item}/#{directory}/#{filename}" + result = remote_md5(dir: directory, remote_file: filename) + case result + when nil + # File doesn't exist. + odebug "Uploading #{@archive_item}/#{directory}/#{filename}" + upload(local_filename, + dir: directory, + remote_file: filename, + warn_on_error: warn_on_error) + when md5 + # File exists, hash matches. + odebug "#{filename} is already published with matching hash." + bottle_count -= 1 + when "" + # File exists, but can't find hash + failed_message = "#{filename} is already published!" + raise Error, "#{failed_message}\n#{delete_instructions}" unless warn_on_error + + opoo failed_message + else + # File exists, but hash either doesn't exist or is mismatched. + failed_message = <<~EOS + #{filename} is already published with a mismatched hash! + Expected: #{md5} + Actual: #{result} + EOS + raise Error, "#{failed_message}#{delete_instructions}" unless warn_on_error + + opoo failed_message + end + end + + odebug "Uploaded #{bottle_count} bottles" + end + end +end diff --git a/Library/Homebrew/dev-cmd/pr-upload.rb b/Library/Homebrew/dev-cmd/pr-upload.rb index 951d24fef0..5006d29498 100644 --- a/Library/Homebrew/dev-cmd/pr-upload.rb +++ b/Library/Homebrew/dev-cmd/pr-upload.rb @@ -2,6 +2,7 @@ # frozen_string_literal: true require "cli/parser" +require "archive" require "bintray" module Homebrew @@ -27,6 +28,8 @@ module Homebrew switch "--warn-on-upload-failure", description: "Warn instead of raising an error if the bottle upload fails. "\ "Useful for repairing bottle uploads that previously failed." + flag "--archive-item=", + description: "Upload to the specified Archive item (default: `homebrew`)." flag "--bintray-org=", description: "Upload to the specified Bintray organisation (default: `homebrew`)." flag "--root-url=", @@ -47,6 +50,18 @@ module Homebrew end end + def archive?(bottles_hash) + @archive ||= bottles_hash.values.all? do |bottle_hash| + bottle_hash["bottle"]["root_url"].start_with? "https://archive.com/" + end + end + + def bintray?(bottles_hash) + @bintray ||= bottles_hash.values.all? do |bottle_hash| + bottle_hash["bottle"]["root_url"].match? %r{^https://[\w-]+\.bintray\.com/} + end + end + def github_releases?(bottles_hash) @github_releases ||= bottles_hash.values.all? do |bottle_hash| root_url = bottle_hash["bottle"]["root_url"] @@ -76,11 +91,16 @@ module Homebrew bottle_args += json_files if args.dry_run? - service = if github_releases?(bottles_hash) - "GitHub Releases" - else - "Bintray" - end + service = + if archive?(bottles_hash) + "Archive.org" + elsif bintray?(bottles_hash) + "Bintray" + elsif github_releases?(bottles_hash) + "GitHub Releases" + else + odie "Service specified by root_url is not recognized" + end puts <<~EOS brew #{bottle_args.join " "} Upload bottles described by these JSON files to #{service}: @@ -102,7 +122,20 @@ module Homebrew safe_system HOMEBREW_BREW_FILE, *audit_args end - if github_releases?(bottles_hash) + if archive?(bottles_hash) + # Handle uploading to Archive.org. + archive_item = args.archive_item || "homebrew" + archive = Archive.new(item: archive_item) + archive.upload_bottles(bottles_hash, + warn_on_error: args.warn_on_upload_failure?) + elsif bintray?(bottles_hash) + # Handle uploading to Bintray. + bintray_org = args.bintray_org || "homebrew" + bintray = Bintray.new(org: bintray_org) + bintray.upload_bottles(bottles_hash, + publish_package: !args.no_publish?, + warn_on_error: args.warn_on_upload_failure?) + elsif github_releases?(bottles_hash) # Handle uploading to GitHub Releases. bottles_hash.each_value do |bottle_hash| root_url = bottle_hash["bottle"]["root_url"] @@ -128,12 +161,7 @@ module Homebrew end end else - # Handle uploading to Bintray. - bintray_org = args.bintray_org || "homebrew" - bintray = Bintray.new(org: bintray_org) - bintray.upload_bottles(bottles_hash, - publish_package: !args.no_publish?, - warn_on_error: args.warn_on_upload_failure?) + odie "Service specified by root_url is not recognized" end end end diff --git a/Library/Homebrew/env_config.rb b/Library/Homebrew/env_config.rb index 8438e1aec4..bcb7903ada 100644 --- a/Library/Homebrew/env_config.rb +++ b/Library/Homebrew/env_config.rb @@ -15,6 +15,10 @@ module Homebrew description: "Linux only: Pass this value to a type name representing the compiler's `-march` option.", default: "native", }, + HOMEBREW_ARCHIVE_KEY: { + description: "Use this API key when accessing the Archive.org API (where bottles are stored). " \ + "The format is access:secret. See https://archive.org/account/s3.php", + }, HOMEBREW_ARTIFACT_DOMAIN: { description: "Prefix all download URLs, including those for bottles, with this value. " \ "For example, `HOMEBREW_ARTIFACT_DOMAIN=http://localhost:8080` will cause a " \ From 2794641a28b8f914d8e28a50d505284d8a10e4f3 Mon Sep 17 00:00:00 2001 From: Shaun Jackman Date: Mon, 22 Feb 2021 17:24:05 -0800 Subject: [PATCH 2/8] Update man pages --- completions/bash/brew | 1 + completions/fish/brew.fish | 1 + completions/zsh/_brew | 1 + docs/Manpage.md | 5 +++++ manpages/brew.1 | 10 ++++++++++ 5 files changed, 18 insertions(+) diff --git a/completions/bash/brew b/completions/bash/brew index badc67053b..5ab2314c7d 100644 --- a/completions/bash/brew +++ b/completions/bash/brew @@ -1507,6 +1507,7 @@ _brew_pr_upload() { case "$cur" in -*) __brewcomp " + --archive-item --bintray-org --debug --dry-run diff --git a/completions/fish/brew.fish b/completions/fish/brew.fish index bdeea8667a..058f9a5b22 100644 --- a/completions/fish/brew.fish +++ b/completions/fish/brew.fish @@ -1077,6 +1077,7 @@ __fish_brew_complete_arg 'pr-pull' -l workflows -d 'Retrieve artifacts from the __fish_brew_complete_cmd 'pr-upload' 'Apply the bottle commit and publish bottles to Bintray or GitHub Releases' +__fish_brew_complete_arg 'pr-upload' -l archive-item -d 'Upload to the specified Archive item (default: `homebrew`)' __fish_brew_complete_arg 'pr-upload' -l bintray-org -d 'Upload to the specified Bintray organisation (default: `homebrew`)' __fish_brew_complete_arg 'pr-upload' -l debug -d 'Display any debugging information' __fish_brew_complete_arg 'pr-upload' -l dry-run -d 'Print what would be done rather than doing it' diff --git a/completions/zsh/_brew b/completions/zsh/_brew index a0b7f98a26..52493950f3 100644 --- a/completions/zsh/_brew +++ b/completions/zsh/_brew @@ -1255,6 +1255,7 @@ _brew_pr_pull() { # brew pr-upload _brew_pr_upload() { _arguments \ + '--archive-item[Upload to the specified Archive item (default: `homebrew`)]' \ '--bintray-org[Upload to the specified Bintray organisation (default: `homebrew`)]' \ '--debug[Display any debugging information]' \ '--dry-run[Print what would be done rather than doing it]' \ diff --git a/docs/Manpage.md b/docs/Manpage.md index 95e9f04901..9d82a72731 100644 --- a/docs/Manpage.md +++ b/docs/Manpage.md @@ -1208,6 +1208,8 @@ Apply the bottle commit and publish bottles to Bintray or GitHub Releases. Do not generate a new commit before uploading. * `--warn-on-upload-failure`: Warn instead of raising an error if the bottle upload fails. Useful for repairing bottle uploads that previously failed. +* `--archive-item`: + Upload to the specified Archive item (default: `homebrew`). * `--bintray-org`: Upload to the specified Bintray organisation (default: `homebrew`). * `--root-url`: @@ -1698,6 +1700,9 @@ example, run `export HOMEBREW_NO_INSECURE_REDIRECT=1` rather than just *Default:* `native`. +- `HOMEBREW_ARCHIVE_KEY` +
Use this API key when accessing the Archive.org API (where bottles are stored). The format is access:secret. See https://archive.org/account/s3.php + - `HOMEBREW_ARTIFACT_DOMAIN`
Prefix all download URLs, including those for bottles, with this value. For example, `HOMEBREW_ARTIFACT_DOMAIN=http://localhost:8080` will cause a formula with the URL `https://example.com/foo.tar.gz` to instead download from `http://localhost:8080/example.com/foo.tar.gz`. diff --git a/manpages/brew.1 b/manpages/brew.1 index 733c78cc81..6bf3ad5257 100644 --- a/manpages/brew.1 +++ b/manpages/brew.1 @@ -1692,6 +1692,10 @@ Do not generate a new commit before uploading\. Warn instead of raising an error if the bottle upload fails\. Useful for repairing bottle uploads that previously failed\. . .TP +\fB\-\-archive\-item\fR +Upload to the specified Archive item (default: \fBhomebrew\fR)\. +. +.TP \fB\-\-bintray\-org\fR Upload to the specified Bintray organisation (default: \fBhomebrew\fR)\. . @@ -2353,6 +2357,12 @@ Linux only: Pass this value to a type name representing the compiler\'s \fB\-mar \fIDefault:\fR \fBnative\fR\. . .TP +\fBHOMEBREW_ARCHIVE_KEY\fR +. +.br +Use this API key when accessing the Archive\.org API (where bottles are stored)\. The format is access:secret\. See https://archive\.org/account/s3\.php +. +.TP \fBHOMEBREW_ARTIFACT_DOMAIN\fR . .br From c11692ba780d7a5663aa693107d98d99a83d1050 Mon Sep 17 00:00:00 2001 From: Shaun Jackman Date: Mon, 22 Feb 2021 17:56:26 -0800 Subject: [PATCH 3/8] pr-pull: Upload bottles to Archive.org Add option --archive-item --- Library/Homebrew/dev-cmd/pr-pull.rb | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/Library/Homebrew/dev-cmd/pr-pull.rb b/Library/Homebrew/dev-cmd/pr-pull.rb index 86a20aa26c..f4aaf5f141 100644 --- a/Library/Homebrew/dev-cmd/pr-pull.rb +++ b/Library/Homebrew/dev-cmd/pr-pull.rb @@ -49,6 +49,8 @@ module Homebrew description: "Message to include when autosquashing revision bumps, deletions, and rebuilds." flag "--artifact=", description: "Download artifacts with the specified name (default: `bottles`)." + flag "--archive-item=", + description: "Upload to the specified Archive item (default: `homebrew`)." flag "--bintray-org=", description: "Upload to the specified Bintray organisation (default: `homebrew`)." flag "--tap=", @@ -65,6 +67,7 @@ module Homebrew description: "Comma-separated list of workflows which can be ignored if they have not been run." conflicts "--clean", "--autosquash" + conflicts "--archive-item", "--bintray-org" named_args :pull_request, min: 1 end @@ -357,6 +360,7 @@ module Homebrew workflows = args.workflows.presence || ["tests.yml"] artifact = args.artifact || "bottles" + archive_item = args.archive_item bintray_org = args.bintray_org || "homebrew" mirror_repo = args.bintray_mirror || "mirror" tap = Tap.fetch(args.tap || CoreTap.instance.name) @@ -424,7 +428,11 @@ module Homebrew upload_args << "--keep-old" if args.keep_old? upload_args << "--warn-on-upload-failure" if args.warn_on_upload_failure? upload_args << "--root-url=#{args.root_url}" if args.root_url - upload_args << "--bintray-org=#{bintray_org}" + upload_args << if archive_item.present? + "--archive-item=#{archive_item}" + else + "--bintray-org=#{bintray_org}" + end safe_system HOMEBREW_BREW_FILE, *upload_args end end From 7fd6d2166b93d8b86710b7642c094c7164f20dd7 Mon Sep 17 00:00:00 2001 From: Shaun Jackman Date: Mon, 22 Feb 2021 17:58:13 -0800 Subject: [PATCH 4/8] Update man pages --- completions/bash/brew | 1 + completions/fish/brew.fish | 1 + completions/zsh/_brew | 3 ++- docs/Manpage.md | 2 ++ manpages/brew.1 | 4 ++++ 5 files changed, 10 insertions(+), 1 deletion(-) diff --git a/completions/bash/brew b/completions/bash/brew index 5ab2314c7d..504a665c38 100644 --- a/completions/bash/brew +++ b/completions/bash/brew @@ -1475,6 +1475,7 @@ _brew_pr_pull() { case "$cur" in -*) __brewcomp " + --archive-item --artifact --autosquash --bintray-mirror diff --git a/completions/fish/brew.fish b/completions/fish/brew.fish index 058f9a5b22..1debb25df8 100644 --- a/completions/fish/brew.fish +++ b/completions/fish/brew.fish @@ -1053,6 +1053,7 @@ __fish_brew_complete_arg 'pr-publish' -l workflow -d 'Target workflow filename ( __fish_brew_complete_cmd 'pr-pull' 'Download and publish bottles, and apply the bottle commit from a pull request with artifacts generated by GitHub Actions' +__fish_brew_complete_arg 'pr-pull' -l archive-item -d 'Upload to the specified Archive item (default: `homebrew`)' __fish_brew_complete_arg 'pr-pull' -l artifact -d 'Download artifacts with the specified name (default: `bottles`)' __fish_brew_complete_arg 'pr-pull' -l autosquash -d 'Automatically reformat and reword commits in the pull request to our preferred format' __fish_brew_complete_arg 'pr-pull' -l bintray-mirror -d 'Use the specified Bintray repository to automatically mirror stable URLs defined in the formulae (default: `mirror`)' diff --git a/completions/zsh/_brew b/completions/zsh/_brew index 52493950f3..1f645fcccc 100644 --- a/completions/zsh/_brew +++ b/completions/zsh/_brew @@ -1229,10 +1229,11 @@ _brew_pr_publish() { # brew pr-pull _brew_pr_pull() { _arguments \ + '(--bintray-org)--archive-item[Upload to the specified Archive item (default: `homebrew`)]' \ '--artifact[Download artifacts with the specified name (default: `bottles`)]' \ '(--clean)--autosquash[Automatically reformat and reword commits in the pull request to our preferred format]' \ '--bintray-mirror[Use the specified Bintray repository to automatically mirror stable URLs defined in the formulae (default: `mirror`)]' \ - '--bintray-org[Upload to the specified Bintray organisation (default: `homebrew`)]' \ + '(--archive-item)--bintray-org[Upload to the specified Bintray organisation (default: `homebrew`)]' \ '--branch-okay[Do not warn if pulling to a branch besides the repository default (useful for testing)]' \ '(--autosquash)--clean[Do not amend the commits from pull requests]' \ '--debug[Display any debugging information]' \ diff --git a/docs/Manpage.md b/docs/Manpage.md index 9d82a72731..b5e26897c8 100644 --- a/docs/Manpage.md +++ b/docs/Manpage.md @@ -1181,6 +1181,8 @@ Requires write access to the repository. Message to include when autosquashing revision bumps, deletions, and rebuilds. * `--artifact`: Download artifacts with the specified name (default: `bottles`). +* `--archive-item`: + Upload to the specified Archive item (default: `homebrew`). * `--bintray-org`: Upload to the specified Bintray organisation (default: `homebrew`). * `--tap`: diff --git a/manpages/brew.1 b/manpages/brew.1 index 6bf3ad5257..e79201ba38 100644 --- a/manpages/brew.1 +++ b/manpages/brew.1 @@ -1645,6 +1645,10 @@ Message to include when autosquashing revision bumps, deletions, and rebuilds\. Download artifacts with the specified name (default: \fBbottles\fR)\. . .TP +\fB\-\-archive\-item\fR +Upload to the specified Archive item (default: \fBhomebrew\fR)\. +. +.TP \fB\-\-bintray\-org\fR Upload to the specified Bintray organisation (default: \fBhomebrew\fR)\. . From 3d715ca3f06903f8404ebecd78bac3caf4717e83 Mon Sep 17 00:00:00 2001 From: Shaun Jackman Date: Mon, 22 Feb 2021 18:06:44 -0800 Subject: [PATCH 5/8] archive.rb: Rename dir to directory --- Library/Homebrew/archive.rb | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/Library/Homebrew/archive.rb b/Library/Homebrew/archive.rb index a3d49b00bd..b4ad85a804 100644 --- a/Library/Homebrew/archive.rb +++ b/Library/Homebrew/archive.rb @@ -47,11 +47,11 @@ class Archive sig { params(local_file: String, - dir: String, + directory: String, remote_file: String, warn_on_error: T.nilable(T::Boolean)).void } - def upload(local_file, dir:, remote_file:, warn_on_error: false) + def upload(local_file, directory:, remote_file:, warn_on_error: false) unless File.exist? local_file msg = "#{local_file} for upload doesn't exist!" raise Error, msg unless warn_on_error @@ -62,7 +62,7 @@ class Archive end md5_base64 = Digest::MD5.base64digest(File.read(local_file)) - url = "https://#{@archive_item}.s3.us.archive.org/#{dir}/#{remote_file}" + url = "https://#{@archive_item}.s3.us.archive.org/#{directory}/#{remote_file}" args = ["--upload-file", local_file, "--header", "Content-MD5: #{md5_base64}"] args << "--fail" unless warn_on_error result = T.unsafe(self).open_api(url, *args) @@ -83,20 +83,20 @@ class Archive sig { params(formula: Formula, - dir: String, + directory: String, warn_on_error: T::Boolean).returns(String) } - def mirror_formula(formula, dir: "mirror", warn_on_error: false) + def mirror_formula(formula, directory: "mirror", warn_on_error: false) formula.downloader.fetch filename = ERB::Util.url_encode(formula.downloader.basename) - destination_url = "https://archive.org/download/#{@archive_item}/#{dir}/#{filename}" + destination_url = "https://archive.org/download/#{@archive_item}/#{directory}/#{filename}" odebug "Uploading to #{destination_url}" upload( formula.downloader.cached_location, - dir: dir, + directory: directory, remote_file: filename, warn_on_error: warn_on_error, ) @@ -107,9 +107,9 @@ class Archive # Gets the MD5 hash of the specified remote file. # # @return the hash, the empty string (if the file doesn't have a hash), nil (if the file doesn't exist) - sig { params(dir: String, remote_file: String).returns(T.nilable(String)) } - def remote_md5(dir:, remote_file:) - url = "https://#{@archive_item}.s3.us.archive.org/#{dir}/#{remote_file}" + sig { params(directory: String, remote_file: String).returns(T.nilable(String)) } + def remote_md5(directory:, remote_file:) + url = "https://#{@archive_item}.s3.us.archive.org/#{directory}/#{remote_file}" result = curl_output "--fail", "--silent", "--head", "--location", url if result.success? result.stdout.match(/^ETag: "(\h{32})"/)&.values_at(1)&.first || "" @@ -147,13 +147,13 @@ class Archive md5 = Digest::MD5.hexdigest(File.read(local_filename)) odebug "Checking remote file #{@archive_item}/#{directory}/#{filename}" - result = remote_md5(dir: directory, remote_file: filename) + result = remote_md5(directory: directory, remote_file: filename) case result when nil # File doesn't exist. odebug "Uploading #{@archive_item}/#{directory}/#{filename}" upload(local_filename, - dir: directory, + directory: directory, remote_file: filename, warn_on_error: warn_on_error) when md5 From 0f2c47dbe3bca661cf44875b214195dd3f2d451e Mon Sep 17 00:00:00 2001 From: Shaun Jackman Date: Mon, 22 Feb 2021 18:07:39 -0800 Subject: [PATCH 6/8] Add test/archive_spec.rb --- Library/Homebrew/test/archive_spec.rb | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 Library/Homebrew/test/archive_spec.rb diff --git a/Library/Homebrew/test/archive_spec.rb b/Library/Homebrew/test/archive_spec.rb new file mode 100644 index 0000000000..877aa7e630 --- /dev/null +++ b/Library/Homebrew/test/archive_spec.rb @@ -0,0 +1,20 @@ +# typed: false +# frozen_string_literal: true + +require "archive" + +describe Archive, :needs_network do + subject(:archive) { described_class.new(item: "homebrew") } + + describe "::remote_checksum" do + it "detects a published file" do + hash = archive.remote_md5(directory: ".", remote_file: "cmake-3.1.2.yosemite.bottle.tar.gz") + expect(hash).to eq("c6e525d472124670b0b635800488f438") + end + + it "fails on a non-existent file" do + hash = archive.remote_md5(directory: "bottles", remote_file: "my-fake-bottle-1.0.snow_hyena.tar.gz") + expect(hash).to be nil + end + end +end From 641f134274f6bb8eac05c60e2e624ce042492fcd Mon Sep 17 00:00:00 2001 From: Shaun Jackman Date: Tue, 23 Feb 2021 11:22:55 -0800 Subject: [PATCH 7/8] Apply Mike's suggestions --- Library/Homebrew/archive.rb | 30 ++++++++++----------------- Library/Homebrew/dev-cmd/pr-pull.rb | 2 +- Library/Homebrew/dev-cmd/pr-upload.rb | 6 +++--- Library/Homebrew/env_config.rb | 8 +++---- 4 files changed, 19 insertions(+), 27 deletions(-) diff --git a/Library/Homebrew/archive.rb b/Library/Homebrew/archive.rb index b4ad85a804..3e43dcc272 100644 --- a/Library/Homebrew/archive.rb +++ b/Library/Homebrew/archive.rb @@ -4,7 +4,7 @@ require "digest/md5" require "utils/curl" -# Archive API client. +# The Internet Archive API client. # # @api private class Archive @@ -23,20 +23,18 @@ class Archive sig { params(item: T.nilable(String)).void } def initialize(item: "homebrew") + raise UsageError, "Must set the Archive item!" unless item + @archive_item = item - - raise UsageError, "Must set the Archive item!" unless @archive_item - - ENV["HOMEBREW_FORCE_HOMEBREW_ON_LINUX"] = "1" if @archive_item == "homebrew" && !OS.mac? end def open_api(url, *args, auth: true) if auth - raise UsageError, "HOMEBREW_ARCHIVE_KEY is unset." unless (key = Homebrew::EnvConfig.archive_key) + key = Homebrew::EnvConfig.internet_archive_key + raise UsageError, "HOMEBREW_INTERNET_ARCHIVE_KEY is unset." if key.blank? if key.exclude?(":") - raise UsageError, - "Use HOMEBREW_ARCHIVE_KEY=access:secret. See https://archive.org/account/s3.php" + raise UsageError, "Use HOMEBREW_INTERNET_ARCHIVE_KEY=access:secret. See https://archive.org/account/s3.php" end args += ["--header", "Authorization: AWS #{key}"] @@ -52,7 +50,8 @@ class Archive warn_on_error: T.nilable(T::Boolean)).void } def upload(local_file, directory:, remote_file:, warn_on_error: false) - unless File.exist? local_file + local_file = Pathname.new(local_file) + unless local_file.exist? msg = "#{local_file} for upload doesn't exist!" raise Error, msg unless warn_on_error @@ -61,7 +60,7 @@ class Archive return end - md5_base64 = Digest::MD5.base64digest(File.read(local_file)) + md5_base64 = Digest::MD5.base64digest(local_file.read) url = "https://#{@archive_item}.s3.us.archive.org/#{directory}/#{remote_file}" args = ["--upload-file", local_file, "--header", "Content-MD5: #{md5_base64}"] args << "--fail" unless warn_on_error @@ -74,13 +73,6 @@ class Archive opoo msg end - sig { params(url: String).returns(T::Boolean) } - def stable_mirrored?(url) - headers, = curl_output("--connect-timeout", "15", "--location", "--head", url) - status_code = headers.scan(%r{^HTTP/.* (\d+)}).last.first - status_code.start_with?("2") - end - sig { params(formula: Formula, directory: String, @@ -124,7 +116,7 @@ class Archive def file_delete_instructions(directory, filename) <<~EOS Run: - curl -X DELETE -H "Authorization: AWS $HOMEBREW_ARCHIVE_KEY" https://#{@archive_item}.s3.us.archive.org/#{directory}/#{filename} + curl -X DELETE -H "Authorization: AWS $HOMEBREW_INTERNET_ARCHIVE_KEY" https://#{@archive_item}.s3.us.archive.org/#{directory}/#{filename} Or run: ia delete #{@archive_item} #{directory}/#{filename} EOS @@ -139,7 +131,7 @@ class Archive directory = bottle_hash["bintray"]["repository"] bottle_count = bottle_hash["bottle"]["tags"].length - bottle_hash["bottle"]["tags"].each do |_tag, tag_hash| + bottle_hash["bottle"]["tags"].each_value do |tag_hash| filename = tag_hash["filename"] # URL encoded in Bottle::Filename#archive delete_instructions = file_delete_instructions(directory, filename) diff --git a/Library/Homebrew/dev-cmd/pr-pull.rb b/Library/Homebrew/dev-cmd/pr-pull.rb index f4aaf5f141..fb4e67d4cc 100644 --- a/Library/Homebrew/dev-cmd/pr-pull.rb +++ b/Library/Homebrew/dev-cmd/pr-pull.rb @@ -50,7 +50,7 @@ module Homebrew flag "--artifact=", description: "Download artifacts with the specified name (default: `bottles`)." flag "--archive-item=", - description: "Upload to the specified Archive item (default: `homebrew`)." + description: "Upload to the specified Internet Archive item (default: `homebrew`)." flag "--bintray-org=", description: "Upload to the specified Bintray organisation (default: `homebrew`)." flag "--tap=", diff --git a/Library/Homebrew/dev-cmd/pr-upload.rb b/Library/Homebrew/dev-cmd/pr-upload.rb index 5006d29498..f82645f2d8 100644 --- a/Library/Homebrew/dev-cmd/pr-upload.rb +++ b/Library/Homebrew/dev-cmd/pr-upload.rb @@ -29,7 +29,7 @@ module Homebrew description: "Warn instead of raising an error if the bottle upload fails. "\ "Useful for repairing bottle uploads that previously failed." flag "--archive-item=", - description: "Upload to the specified Archive item (default: `homebrew`)." + description: "Upload to the specified Internet Archive item (default: `homebrew`)." flag "--bintray-org=", description: "Upload to the specified Bintray organisation (default: `homebrew`)." flag "--root-url=", @@ -93,7 +93,7 @@ module Homebrew if args.dry_run? service = if archive?(bottles_hash) - "Archive.org" + "Internet Archive" elsif bintray?(bottles_hash) "Bintray" elsif github_releases?(bottles_hash) @@ -123,7 +123,7 @@ module Homebrew end if archive?(bottles_hash) - # Handle uploading to Archive.org. + # Handle uploading to the Internet Archive. archive_item = args.archive_item || "homebrew" archive = Archive.new(item: archive_item) archive.upload_bottles(bottles_hash, diff --git a/Library/Homebrew/env_config.rb b/Library/Homebrew/env_config.rb index bcb7903ada..20a9a9cd6b 100644 --- a/Library/Homebrew/env_config.rb +++ b/Library/Homebrew/env_config.rb @@ -15,10 +15,6 @@ module Homebrew description: "Linux only: Pass this value to a type name representing the compiler's `-march` option.", default: "native", }, - HOMEBREW_ARCHIVE_KEY: { - description: "Use this API key when accessing the Archive.org API (where bottles are stored). " \ - "The format is access:secret. See https://archive.org/account/s3.php", - }, HOMEBREW_ARTIFACT_DOMAIN: { description: "Prefix all download URLs, including those for bottles, with this value. " \ "For example, `HOMEBREW_ARTIFACT_DOMAIN=http://localhost:8080` will cause a " \ @@ -183,6 +179,10 @@ module Homebrew default_text: 'The "Beer Mug" emoji.', default: "🍺", }, + HOMEBREW_INTERNET_ARCHIVE_KEY: { + description: "Use this API key when accessing the Internet Archive S3 API, where bottles are stored. " \ + "The format is access:secret. See https://archive.org/account/s3.php", + }, HOMEBREW_LIVECHECK_WATCHLIST: { description: "Consult this file for the list of formulae to check by default when no formula argument " \ "is passed to `brew livecheck`.", From be11a22d97c1af2018789485286f4ed9c558c29f Mon Sep 17 00:00:00 2001 From: Shaun Jackman Date: Tue, 23 Feb 2021 11:31:53 -0800 Subject: [PATCH 8/8] Update man pages --- completions/fish/brew.fish | 4 ++-- completions/zsh/_brew | 4 ++-- docs/Manpage.md | 10 +++++----- manpages/brew.1 | 16 ++++++++-------- 4 files changed, 17 insertions(+), 17 deletions(-) diff --git a/completions/fish/brew.fish b/completions/fish/brew.fish index 1debb25df8..d20366c055 100644 --- a/completions/fish/brew.fish +++ b/completions/fish/brew.fish @@ -1053,7 +1053,7 @@ __fish_brew_complete_arg 'pr-publish' -l workflow -d 'Target workflow filename ( __fish_brew_complete_cmd 'pr-pull' 'Download and publish bottles, and apply the bottle commit from a pull request with artifacts generated by GitHub Actions' -__fish_brew_complete_arg 'pr-pull' -l archive-item -d 'Upload to the specified Archive item (default: `homebrew`)' +__fish_brew_complete_arg 'pr-pull' -l archive-item -d 'Upload to the specified Internet Archive item (default: `homebrew`)' __fish_brew_complete_arg 'pr-pull' -l artifact -d 'Download artifacts with the specified name (default: `bottles`)' __fish_brew_complete_arg 'pr-pull' -l autosquash -d 'Automatically reformat and reword commits in the pull request to our preferred format' __fish_brew_complete_arg 'pr-pull' -l bintray-mirror -d 'Use the specified Bintray repository to automatically mirror stable URLs defined in the formulae (default: `mirror`)' @@ -1078,7 +1078,7 @@ __fish_brew_complete_arg 'pr-pull' -l workflows -d 'Retrieve artifacts from the __fish_brew_complete_cmd 'pr-upload' 'Apply the bottle commit and publish bottles to Bintray or GitHub Releases' -__fish_brew_complete_arg 'pr-upload' -l archive-item -d 'Upload to the specified Archive item (default: `homebrew`)' +__fish_brew_complete_arg 'pr-upload' -l archive-item -d 'Upload to the specified Internet Archive item (default: `homebrew`)' __fish_brew_complete_arg 'pr-upload' -l bintray-org -d 'Upload to the specified Bintray organisation (default: `homebrew`)' __fish_brew_complete_arg 'pr-upload' -l debug -d 'Display any debugging information' __fish_brew_complete_arg 'pr-upload' -l dry-run -d 'Print what would be done rather than doing it' diff --git a/completions/zsh/_brew b/completions/zsh/_brew index 1f645fcccc..bd00632419 100644 --- a/completions/zsh/_brew +++ b/completions/zsh/_brew @@ -1229,7 +1229,7 @@ _brew_pr_publish() { # brew pr-pull _brew_pr_pull() { _arguments \ - '(--bintray-org)--archive-item[Upload to the specified Archive item (default: `homebrew`)]' \ + '(--bintray-org)--archive-item[Upload to the specified Internet Archive item (default: `homebrew`)]' \ '--artifact[Download artifacts with the specified name (default: `bottles`)]' \ '(--clean)--autosquash[Automatically reformat and reword commits in the pull request to our preferred format]' \ '--bintray-mirror[Use the specified Bintray repository to automatically mirror stable URLs defined in the formulae (default: `mirror`)]' \ @@ -1256,7 +1256,7 @@ _brew_pr_pull() { # brew pr-upload _brew_pr_upload() { _arguments \ - '--archive-item[Upload to the specified Archive item (default: `homebrew`)]' \ + '--archive-item[Upload to the specified Internet Archive item (default: `homebrew`)]' \ '--bintray-org[Upload to the specified Bintray organisation (default: `homebrew`)]' \ '--debug[Display any debugging information]' \ '--dry-run[Print what would be done rather than doing it]' \ diff --git a/docs/Manpage.md b/docs/Manpage.md index b5e26897c8..048c24edcb 100644 --- a/docs/Manpage.md +++ b/docs/Manpage.md @@ -1182,7 +1182,7 @@ Requires write access to the repository. * `--artifact`: Download artifacts with the specified name (default: `bottles`). * `--archive-item`: - Upload to the specified Archive item (default: `homebrew`). + Upload to the specified Internet Archive item (default: `homebrew`). * `--bintray-org`: Upload to the specified Bintray organisation (default: `homebrew`). * `--tap`: @@ -1211,7 +1211,7 @@ Apply the bottle commit and publish bottles to Bintray or GitHub Releases. * `--warn-on-upload-failure`: Warn instead of raising an error if the bottle upload fails. Useful for repairing bottle uploads that previously failed. * `--archive-item`: - Upload to the specified Archive item (default: `homebrew`). + Upload to the specified Internet Archive item (default: `homebrew`). * `--bintray-org`: Upload to the specified Bintray organisation (default: `homebrew`). * `--root-url`: @@ -1702,9 +1702,6 @@ example, run `export HOMEBREW_NO_INSECURE_REDIRECT=1` rather than just *Default:* `native`. -- `HOMEBREW_ARCHIVE_KEY` -
Use this API key when accessing the Archive.org API (where bottles are stored). The format is access:secret. See https://archive.org/account/s3.php - - `HOMEBREW_ARTIFACT_DOMAIN`
Prefix all download URLs, including those for bottles, with this value. For example, `HOMEBREW_ARTIFACT_DOMAIN=http://localhost:8080` will cause a formula with the URL `https://example.com/foo.tar.gz` to instead download from `http://localhost:8080/example.com/foo.tar.gz`. @@ -1841,6 +1838,9 @@ example, run `export HOMEBREW_NO_INSECURE_REDIRECT=1` rather than just *Default:* The "Beer Mug" emoji. +- `HOMEBREW_INTERNET_ARCHIVE_KEY` +
Use this API key when accessing the Internet Archive S3 API, where bottles are stored. The format is access:secret. See https://archive.org/account/s3.php + - `HOMEBREW_LIVECHECK_WATCHLIST`
Consult this file for the list of formulae to check by default when no formula argument is passed to `brew livecheck`. diff --git a/manpages/brew.1 b/manpages/brew.1 index e79201ba38..e7121f303a 100644 --- a/manpages/brew.1 +++ b/manpages/brew.1 @@ -1646,7 +1646,7 @@ Download artifacts with the specified name (default: \fBbottles\fR)\. . .TP \fB\-\-archive\-item\fR -Upload to the specified Archive item (default: \fBhomebrew\fR)\. +Upload to the specified Internet Archive item (default: \fBhomebrew\fR)\. . .TP \fB\-\-bintray\-org\fR @@ -1697,7 +1697,7 @@ Warn instead of raising an error if the bottle upload fails\. Useful for repairi . .TP \fB\-\-archive\-item\fR -Upload to the specified Archive item (default: \fBhomebrew\fR)\. +Upload to the specified Internet Archive item (default: \fBhomebrew\fR)\. . .TP \fB\-\-bintray\-org\fR @@ -2361,12 +2361,6 @@ Linux only: Pass this value to a type name representing the compiler\'s \fB\-mar \fIDefault:\fR \fBnative\fR\. . .TP -\fBHOMEBREW_ARCHIVE_KEY\fR -. -.br -Use this API key when accessing the Archive\.org API (where bottles are stored)\. The format is access:secret\. See https://archive\.org/account/s3\.php -. -.TP \fBHOMEBREW_ARTIFACT_DOMAIN\fR . .br @@ -2622,6 +2616,12 @@ Print this text before the installation summary of each successful build\. \fIDefault:\fR The "Beer Mug" emoji\. . .TP +\fBHOMEBREW_INTERNET_ARCHIVE_KEY\fR +. +.br +Use this API key when accessing the Internet Archive S3 API, where bottles are stored\. The format is access:secret\. See https://archive\.org/account/s3\.php +. +.TP \fBHOMEBREW_LIVECHECK_WATCHLIST\fR . .br