diff --git a/Library/Homebrew/archive.rb b/Library/Homebrew/archive.rb new file mode 100644 index 0000000000..3e43dcc272 --- /dev/null +++ b/Library/Homebrew/archive.rb @@ -0,0 +1,177 @@ +# typed: false +# frozen_string_literal: true + +require "digest/md5" +require "utils/curl" + +# The Internet Archive API client. +# +# @api private +class Archive + extend T::Sig + + include Context + include Utils::Curl + + class Error < RuntimeError + end + + sig { returns(String) } + def inspect + "#" + end + + sig { params(item: T.nilable(String)).void } + def initialize(item: "homebrew") + raise UsageError, "Must set the Archive item!" unless item + + @archive_item = item + end + + def open_api(url, *args, auth: true) + if auth + key = Homebrew::EnvConfig.internet_archive_key + raise UsageError, "HOMEBREW_INTERNET_ARCHIVE_KEY is unset." if key.blank? + + if key.exclude?(":") + raise UsageError, "Use HOMEBREW_INTERNET_ARCHIVE_KEY=access:secret. See https://archive.org/account/s3.php" + end + + args += ["--header", "Authorization: AWS #{key}"] + end + + curl(*args, url, print_stdout: false, secrets: key) + end + + sig { + params(local_file: String, + directory: String, + remote_file: String, + warn_on_error: T.nilable(T::Boolean)).void + } + def upload(local_file, directory:, remote_file:, warn_on_error: false) + local_file = Pathname.new(local_file) + unless local_file.exist? + msg = "#{local_file} for upload doesn't exist!" + raise Error, msg unless warn_on_error + + # Warn and return early here since we know this upload is going to fail. + opoo msg + return + end + + md5_base64 = Digest::MD5.base64digest(local_file.read) + url = "https://#{@archive_item}.s3.us.archive.org/#{directory}/#{remote_file}" + args = ["--upload-file", local_file, "--header", "Content-MD5: #{md5_base64}"] + args << "--fail" unless warn_on_error + result = T.unsafe(self).open_api(url, *args) + return if result.success? && result.stdout.exclude?("Error") + + msg = "Bottle upload failed: #{result.stdout}" + raise msg unless warn_on_error + + opoo msg + end + + sig { + params(formula: Formula, + directory: String, + warn_on_error: T::Boolean).returns(String) + } + def mirror_formula(formula, directory: "mirror", warn_on_error: false) + formula.downloader.fetch + + filename = ERB::Util.url_encode(formula.downloader.basename) + destination_url = "https://archive.org/download/#{@archive_item}/#{directory}/#{filename}" + + odebug "Uploading to #{destination_url}" + + upload( + formula.downloader.cached_location, + directory: directory, + remote_file: filename, + warn_on_error: warn_on_error, + ) + + destination_url + end + + # Gets the MD5 hash of the specified remote file. + # + # @return the hash, the empty string (if the file doesn't have a hash), nil (if the file doesn't exist) + sig { params(directory: String, remote_file: String).returns(T.nilable(String)) } + def remote_md5(directory:, remote_file:) + url = "https://#{@archive_item}.s3.us.archive.org/#{directory}/#{remote_file}" + result = curl_output "--fail", "--silent", "--head", "--location", url + if result.success? + result.stdout.match(/^ETag: "(\h{32})"/)&.values_at(1)&.first || "" + else + raise Error if result.status.exitstatus != 22 && result.stderr.exclude?("404 Not Found") + + nil + end + end + + sig { params(directory: String, filename: String).returns(String) } + def file_delete_instructions(directory, filename) + <<~EOS + Run: + curl -X DELETE -H "Authorization: AWS $HOMEBREW_INTERNET_ARCHIVE_KEY" https://#{@archive_item}.s3.us.archive.org/#{directory}/#{filename} + Or run: + ia delete #{@archive_item} #{directory}/#{filename} + EOS + end + + sig { + params(bottles_hash: T::Hash[String, T.untyped], + warn_on_error: T.nilable(T::Boolean)).void + } + def upload_bottles(bottles_hash, warn_on_error: false) + bottles_hash.each do |_formula_name, bottle_hash| + directory = bottle_hash["bintray"]["repository"] + bottle_count = bottle_hash["bottle"]["tags"].length + + bottle_hash["bottle"]["tags"].each_value do |tag_hash| + filename = tag_hash["filename"] # URL encoded in Bottle::Filename#archive + delete_instructions = file_delete_instructions(directory, filename) + + local_filename = tag_hash["local_filename"] + md5 = Digest::MD5.hexdigest(File.read(local_filename)) + + odebug "Checking remote file #{@archive_item}/#{directory}/#{filename}" + result = remote_md5(directory: directory, remote_file: filename) + case result + when nil + # File doesn't exist. + odebug "Uploading #{@archive_item}/#{directory}/#{filename}" + upload(local_filename, + directory: directory, + remote_file: filename, + warn_on_error: warn_on_error) + when md5 + # File exists, hash matches. + odebug "#{filename} is already published with matching hash." + bottle_count -= 1 + when "" + # File exists, but can't find hash + failed_message = "#{filename} is already published!" + raise Error, "#{failed_message}\n#{delete_instructions}" unless warn_on_error + + opoo failed_message + else + # File exists, but hash either doesn't exist or is mismatched. + failed_message = <<~EOS + #{filename} is already published with a mismatched hash! + Expected: #{md5} + Actual: #{result} + EOS + raise Error, "#{failed_message}#{delete_instructions}" unless warn_on_error + + opoo failed_message + end + end + + odebug "Uploaded #{bottle_count} bottles" + end + end +end diff --git a/Library/Homebrew/dev-cmd/pr-pull.rb b/Library/Homebrew/dev-cmd/pr-pull.rb index 86a20aa26c..fb4e67d4cc 100644 --- a/Library/Homebrew/dev-cmd/pr-pull.rb +++ b/Library/Homebrew/dev-cmd/pr-pull.rb @@ -49,6 +49,8 @@ module Homebrew description: "Message to include when autosquashing revision bumps, deletions, and rebuilds." flag "--artifact=", description: "Download artifacts with the specified name (default: `bottles`)." + flag "--archive-item=", + description: "Upload to the specified Internet Archive item (default: `homebrew`)." flag "--bintray-org=", description: "Upload to the specified Bintray organisation (default: `homebrew`)." flag "--tap=", @@ -65,6 +67,7 @@ module Homebrew description: "Comma-separated list of workflows which can be ignored if they have not been run." conflicts "--clean", "--autosquash" + conflicts "--archive-item", "--bintray-org" named_args :pull_request, min: 1 end @@ -357,6 +360,7 @@ module Homebrew workflows = args.workflows.presence || ["tests.yml"] artifact = args.artifact || "bottles" + archive_item = args.archive_item bintray_org = args.bintray_org || "homebrew" mirror_repo = args.bintray_mirror || "mirror" tap = Tap.fetch(args.tap || CoreTap.instance.name) @@ -424,7 +428,11 @@ module Homebrew upload_args << "--keep-old" if args.keep_old? upload_args << "--warn-on-upload-failure" if args.warn_on_upload_failure? upload_args << "--root-url=#{args.root_url}" if args.root_url - upload_args << "--bintray-org=#{bintray_org}" + upload_args << if archive_item.present? + "--archive-item=#{archive_item}" + else + "--bintray-org=#{bintray_org}" + end safe_system HOMEBREW_BREW_FILE, *upload_args end end diff --git a/Library/Homebrew/dev-cmd/pr-upload.rb b/Library/Homebrew/dev-cmd/pr-upload.rb index 951d24fef0..f82645f2d8 100644 --- a/Library/Homebrew/dev-cmd/pr-upload.rb +++ b/Library/Homebrew/dev-cmd/pr-upload.rb @@ -2,6 +2,7 @@ # frozen_string_literal: true require "cli/parser" +require "archive" require "bintray" module Homebrew @@ -27,6 +28,8 @@ module Homebrew switch "--warn-on-upload-failure", description: "Warn instead of raising an error if the bottle upload fails. "\ "Useful for repairing bottle uploads that previously failed." + flag "--archive-item=", + description: "Upload to the specified Internet Archive item (default: `homebrew`)." flag "--bintray-org=", description: "Upload to the specified Bintray organisation (default: `homebrew`)." flag "--root-url=", @@ -47,6 +50,18 @@ module Homebrew end end + def archive?(bottles_hash) + @archive ||= bottles_hash.values.all? do |bottle_hash| + bottle_hash["bottle"]["root_url"].start_with? "https://archive.com/" + end + end + + def bintray?(bottles_hash) + @bintray ||= bottles_hash.values.all? do |bottle_hash| + bottle_hash["bottle"]["root_url"].match? %r{^https://[\w-]+\.bintray\.com/} + end + end + def github_releases?(bottles_hash) @github_releases ||= bottles_hash.values.all? do |bottle_hash| root_url = bottle_hash["bottle"]["root_url"] @@ -76,11 +91,16 @@ module Homebrew bottle_args += json_files if args.dry_run? - service = if github_releases?(bottles_hash) - "GitHub Releases" - else - "Bintray" - end + service = + if archive?(bottles_hash) + "Internet Archive" + elsif bintray?(bottles_hash) + "Bintray" + elsif github_releases?(bottles_hash) + "GitHub Releases" + else + odie "Service specified by root_url is not recognized" + end puts <<~EOS brew #{bottle_args.join " "} Upload bottles described by these JSON files to #{service}: @@ -102,7 +122,20 @@ module Homebrew safe_system HOMEBREW_BREW_FILE, *audit_args end - if github_releases?(bottles_hash) + if archive?(bottles_hash) + # Handle uploading to the Internet Archive. + archive_item = args.archive_item || "homebrew" + archive = Archive.new(item: archive_item) + archive.upload_bottles(bottles_hash, + warn_on_error: args.warn_on_upload_failure?) + elsif bintray?(bottles_hash) + # Handle uploading to Bintray. + bintray_org = args.bintray_org || "homebrew" + bintray = Bintray.new(org: bintray_org) + bintray.upload_bottles(bottles_hash, + publish_package: !args.no_publish?, + warn_on_error: args.warn_on_upload_failure?) + elsif github_releases?(bottles_hash) # Handle uploading to GitHub Releases. bottles_hash.each_value do |bottle_hash| root_url = bottle_hash["bottle"]["root_url"] @@ -128,12 +161,7 @@ module Homebrew end end else - # Handle uploading to Bintray. - bintray_org = args.bintray_org || "homebrew" - bintray = Bintray.new(org: bintray_org) - bintray.upload_bottles(bottles_hash, - publish_package: !args.no_publish?, - warn_on_error: args.warn_on_upload_failure?) + odie "Service specified by root_url is not recognized" end end end diff --git a/Library/Homebrew/env_config.rb b/Library/Homebrew/env_config.rb index 8438e1aec4..20a9a9cd6b 100644 --- a/Library/Homebrew/env_config.rb +++ b/Library/Homebrew/env_config.rb @@ -179,6 +179,10 @@ module Homebrew default_text: 'The "Beer Mug" emoji.', default: "🍺", }, + HOMEBREW_INTERNET_ARCHIVE_KEY: { + description: "Use this API key when accessing the Internet Archive S3 API, where bottles are stored. " \ + "The format is access:secret. See https://archive.org/account/s3.php", + }, HOMEBREW_LIVECHECK_WATCHLIST: { description: "Consult this file for the list of formulae to check by default when no formula argument " \ "is passed to `brew livecheck`.", diff --git a/Library/Homebrew/test/archive_spec.rb b/Library/Homebrew/test/archive_spec.rb new file mode 100644 index 0000000000..877aa7e630 --- /dev/null +++ b/Library/Homebrew/test/archive_spec.rb @@ -0,0 +1,20 @@ +# typed: false +# frozen_string_literal: true + +require "archive" + +describe Archive, :needs_network do + subject(:archive) { described_class.new(item: "homebrew") } + + describe "::remote_checksum" do + it "detects a published file" do + hash = archive.remote_md5(directory: ".", remote_file: "cmake-3.1.2.yosemite.bottle.tar.gz") + expect(hash).to eq("c6e525d472124670b0b635800488f438") + end + + it "fails on a non-existent file" do + hash = archive.remote_md5(directory: "bottles", remote_file: "my-fake-bottle-1.0.snow_hyena.tar.gz") + expect(hash).to be nil + end + end +end diff --git a/completions/bash/brew b/completions/bash/brew index badc67053b..504a665c38 100644 --- a/completions/bash/brew +++ b/completions/bash/brew @@ -1475,6 +1475,7 @@ _brew_pr_pull() { case "$cur" in -*) __brewcomp " + --archive-item --artifact --autosquash --bintray-mirror @@ -1507,6 +1508,7 @@ _brew_pr_upload() { case "$cur" in -*) __brewcomp " + --archive-item --bintray-org --debug --dry-run diff --git a/completions/fish/brew.fish b/completions/fish/brew.fish index bdeea8667a..d20366c055 100644 --- a/completions/fish/brew.fish +++ b/completions/fish/brew.fish @@ -1053,6 +1053,7 @@ __fish_brew_complete_arg 'pr-publish' -l workflow -d 'Target workflow filename ( __fish_brew_complete_cmd 'pr-pull' 'Download and publish bottles, and apply the bottle commit from a pull request with artifacts generated by GitHub Actions' +__fish_brew_complete_arg 'pr-pull' -l archive-item -d 'Upload to the specified Internet Archive item (default: `homebrew`)' __fish_brew_complete_arg 'pr-pull' -l artifact -d 'Download artifacts with the specified name (default: `bottles`)' __fish_brew_complete_arg 'pr-pull' -l autosquash -d 'Automatically reformat and reword commits in the pull request to our preferred format' __fish_brew_complete_arg 'pr-pull' -l bintray-mirror -d 'Use the specified Bintray repository to automatically mirror stable URLs defined in the formulae (default: `mirror`)' @@ -1077,6 +1078,7 @@ __fish_brew_complete_arg 'pr-pull' -l workflows -d 'Retrieve artifacts from the __fish_brew_complete_cmd 'pr-upload' 'Apply the bottle commit and publish bottles to Bintray or GitHub Releases' +__fish_brew_complete_arg 'pr-upload' -l archive-item -d 'Upload to the specified Internet Archive item (default: `homebrew`)' __fish_brew_complete_arg 'pr-upload' -l bintray-org -d 'Upload to the specified Bintray organisation (default: `homebrew`)' __fish_brew_complete_arg 'pr-upload' -l debug -d 'Display any debugging information' __fish_brew_complete_arg 'pr-upload' -l dry-run -d 'Print what would be done rather than doing it' diff --git a/completions/zsh/_brew b/completions/zsh/_brew index a0b7f98a26..bd00632419 100644 --- a/completions/zsh/_brew +++ b/completions/zsh/_brew @@ -1229,10 +1229,11 @@ _brew_pr_publish() { # brew pr-pull _brew_pr_pull() { _arguments \ + '(--bintray-org)--archive-item[Upload to the specified Internet Archive item (default: `homebrew`)]' \ '--artifact[Download artifacts with the specified name (default: `bottles`)]' \ '(--clean)--autosquash[Automatically reformat and reword commits in the pull request to our preferred format]' \ '--bintray-mirror[Use the specified Bintray repository to automatically mirror stable URLs defined in the formulae (default: `mirror`)]' \ - '--bintray-org[Upload to the specified Bintray organisation (default: `homebrew`)]' \ + '(--archive-item)--bintray-org[Upload to the specified Bintray organisation (default: `homebrew`)]' \ '--branch-okay[Do not warn if pulling to a branch besides the repository default (useful for testing)]' \ '(--autosquash)--clean[Do not amend the commits from pull requests]' \ '--debug[Display any debugging information]' \ @@ -1255,6 +1256,7 @@ _brew_pr_pull() { # brew pr-upload _brew_pr_upload() { _arguments \ + '--archive-item[Upload to the specified Internet Archive item (default: `homebrew`)]' \ '--bintray-org[Upload to the specified Bintray organisation (default: `homebrew`)]' \ '--debug[Display any debugging information]' \ '--dry-run[Print what would be done rather than doing it]' \ diff --git a/docs/Manpage.md b/docs/Manpage.md index 95e9f04901..048c24edcb 100644 --- a/docs/Manpage.md +++ b/docs/Manpage.md @@ -1181,6 +1181,8 @@ Requires write access to the repository. Message to include when autosquashing revision bumps, deletions, and rebuilds. * `--artifact`: Download artifacts with the specified name (default: `bottles`). +* `--archive-item`: + Upload to the specified Internet Archive item (default: `homebrew`). * `--bintray-org`: Upload to the specified Bintray organisation (default: `homebrew`). * `--tap`: @@ -1208,6 +1210,8 @@ Apply the bottle commit and publish bottles to Bintray or GitHub Releases. Do not generate a new commit before uploading. * `--warn-on-upload-failure`: Warn instead of raising an error if the bottle upload fails. Useful for repairing bottle uploads that previously failed. +* `--archive-item`: + Upload to the specified Internet Archive item (default: `homebrew`). * `--bintray-org`: Upload to the specified Bintray organisation (default: `homebrew`). * `--root-url`: @@ -1834,6 +1838,9 @@ example, run `export HOMEBREW_NO_INSECURE_REDIRECT=1` rather than just *Default:* The "Beer Mug" emoji. +- `HOMEBREW_INTERNET_ARCHIVE_KEY` +
Use this API key when accessing the Internet Archive S3 API, where bottles are stored. The format is access:secret. See https://archive.org/account/s3.php + - `HOMEBREW_LIVECHECK_WATCHLIST`
Consult this file for the list of formulae to check by default when no formula argument is passed to `brew livecheck`. diff --git a/manpages/brew.1 b/manpages/brew.1 index 733c78cc81..e7121f303a 100644 --- a/manpages/brew.1 +++ b/manpages/brew.1 @@ -1645,6 +1645,10 @@ Message to include when autosquashing revision bumps, deletions, and rebuilds\. Download artifacts with the specified name (default: \fBbottles\fR)\. . .TP +\fB\-\-archive\-item\fR +Upload to the specified Internet Archive item (default: \fBhomebrew\fR)\. +. +.TP \fB\-\-bintray\-org\fR Upload to the specified Bintray organisation (default: \fBhomebrew\fR)\. . @@ -1692,6 +1696,10 @@ Do not generate a new commit before uploading\. Warn instead of raising an error if the bottle upload fails\. Useful for repairing bottle uploads that previously failed\. . .TP +\fB\-\-archive\-item\fR +Upload to the specified Internet Archive item (default: \fBhomebrew\fR)\. +. +.TP \fB\-\-bintray\-org\fR Upload to the specified Bintray organisation (default: \fBhomebrew\fR)\. . @@ -2608,6 +2616,12 @@ Print this text before the installation summary of each successful build\. \fIDefault:\fR The "Beer Mug" emoji\. . .TP +\fBHOMEBREW_INTERNET_ARCHIVE_KEY\fR +. +.br +Use this API key when accessing the Internet Archive S3 API, where bottles are stored\. The format is access:secret\. See https://archive\.org/account/s3\.php +. +.TP \fBHOMEBREW_LIVECHECK_WATCHLIST\fR . .br