From b0bd15fb41b711782e04d24b307dc515ef65aaa2 Mon Sep 17 00:00:00 2001 From: Shaun Jackman Date: Sun, 21 Feb 2021 22:12:17 -0800 Subject: [PATCH] pr-upload: Upload bottles to Archive.org --archive-item specifies the item identifier. HOMEBREW_ARCHIVE_KEY=access:secret specifies the S3 key. --- Library/Homebrew/archive.rb | 185 ++++++++++++++++++++++++++ Library/Homebrew/dev-cmd/pr-upload.rb | 52 ++++++-- Library/Homebrew/env_config.rb | 4 + 3 files changed, 229 insertions(+), 12 deletions(-) create mode 100644 Library/Homebrew/archive.rb diff --git a/Library/Homebrew/archive.rb b/Library/Homebrew/archive.rb new file mode 100644 index 0000000000..a3d49b00bd --- /dev/null +++ b/Library/Homebrew/archive.rb @@ -0,0 +1,185 @@ +# typed: false +# frozen_string_literal: true + +require "digest/md5" +require "utils/curl" + +# Archive API client. +# +# @api private +class Archive + extend T::Sig + + include Context + include Utils::Curl + + class Error < RuntimeError + end + + sig { returns(String) } + def inspect + "#" + end + + sig { params(item: T.nilable(String)).void } + def initialize(item: "homebrew") + @archive_item = item + + raise UsageError, "Must set the Archive item!" unless @archive_item + + ENV["HOMEBREW_FORCE_HOMEBREW_ON_LINUX"] = "1" if @archive_item == "homebrew" && !OS.mac? + end + + def open_api(url, *args, auth: true) + if auth + raise UsageError, "HOMEBREW_ARCHIVE_KEY is unset." unless (key = Homebrew::EnvConfig.archive_key) + + if key.exclude?(":") + raise UsageError, + "Use HOMEBREW_ARCHIVE_KEY=access:secret. See https://archive.org/account/s3.php" + end + + args += ["--header", "Authorization: AWS #{key}"] + end + + curl(*args, url, print_stdout: false, secrets: key) + end + + sig { + params(local_file: String, + dir: String, + remote_file: String, + warn_on_error: T.nilable(T::Boolean)).void + } + def upload(local_file, dir:, remote_file:, warn_on_error: false) + unless File.exist? local_file + msg = "#{local_file} for upload doesn't exist!" + raise Error, msg unless warn_on_error + + # Warn and return early here since we know this upload is going to fail. + opoo msg + return + end + + md5_base64 = Digest::MD5.base64digest(File.read(local_file)) + url = "https://#{@archive_item}.s3.us.archive.org/#{dir}/#{remote_file}" + args = ["--upload-file", local_file, "--header", "Content-MD5: #{md5_base64}"] + args << "--fail" unless warn_on_error + result = T.unsafe(self).open_api(url, *args) + return if result.success? && result.stdout.exclude?("Error") + + msg = "Bottle upload failed: #{result.stdout}" + raise msg unless warn_on_error + + opoo msg + end + + sig { params(url: String).returns(T::Boolean) } + def stable_mirrored?(url) + headers, = curl_output("--connect-timeout", "15", "--location", "--head", url) + status_code = headers.scan(%r{^HTTP/.* (\d+)}).last.first + status_code.start_with?("2") + end + + sig { + params(formula: Formula, + dir: String, + warn_on_error: T::Boolean).returns(String) + } + def mirror_formula(formula, dir: "mirror", warn_on_error: false) + formula.downloader.fetch + + filename = ERB::Util.url_encode(formula.downloader.basename) + destination_url = "https://archive.org/download/#{@archive_item}/#{dir}/#{filename}" + + odebug "Uploading to #{destination_url}" + + upload( + formula.downloader.cached_location, + dir: dir, + remote_file: filename, + warn_on_error: warn_on_error, + ) + + destination_url + end + + # Gets the MD5 hash of the specified remote file. + # + # @return the hash, the empty string (if the file doesn't have a hash), nil (if the file doesn't exist) + sig { params(dir: String, remote_file: String).returns(T.nilable(String)) } + def remote_md5(dir:, remote_file:) + url = "https://#{@archive_item}.s3.us.archive.org/#{dir}/#{remote_file}" + result = curl_output "--fail", "--silent", "--head", "--location", url + if result.success? + result.stdout.match(/^ETag: "(\h{32})"/)&.values_at(1)&.first || "" + else + raise Error if result.status.exitstatus != 22 && result.stderr.exclude?("404 Not Found") + + nil + end + end + + sig { params(directory: String, filename: String).returns(String) } + def file_delete_instructions(directory, filename) + <<~EOS + Run: + curl -X DELETE -H "Authorization: AWS $HOMEBREW_ARCHIVE_KEY" https://#{@archive_item}.s3.us.archive.org/#{directory}/#{filename} + Or run: + ia delete #{@archive_item} #{directory}/#{filename} + EOS + end + + sig { + params(bottles_hash: T::Hash[String, T.untyped], + warn_on_error: T.nilable(T::Boolean)).void + } + def upload_bottles(bottles_hash, warn_on_error: false) + bottles_hash.each do |_formula_name, bottle_hash| + directory = bottle_hash["bintray"]["repository"] + bottle_count = bottle_hash["bottle"]["tags"].length + + bottle_hash["bottle"]["tags"].each do |_tag, tag_hash| + filename = tag_hash["filename"] # URL encoded in Bottle::Filename#archive + delete_instructions = file_delete_instructions(directory, filename) + + local_filename = tag_hash["local_filename"] + md5 = Digest::MD5.hexdigest(File.read(local_filename)) + + odebug "Checking remote file #{@archive_item}/#{directory}/#{filename}" + result = remote_md5(dir: directory, remote_file: filename) + case result + when nil + # File doesn't exist. + odebug "Uploading #{@archive_item}/#{directory}/#{filename}" + upload(local_filename, + dir: directory, + remote_file: filename, + warn_on_error: warn_on_error) + when md5 + # File exists, hash matches. + odebug "#{filename} is already published with matching hash." + bottle_count -= 1 + when "" + # File exists, but can't find hash + failed_message = "#{filename} is already published!" + raise Error, "#{failed_message}\n#{delete_instructions}" unless warn_on_error + + opoo failed_message + else + # File exists, but hash either doesn't exist or is mismatched. + failed_message = <<~EOS + #{filename} is already published with a mismatched hash! + Expected: #{md5} + Actual: #{result} + EOS + raise Error, "#{failed_message}#{delete_instructions}" unless warn_on_error + + opoo failed_message + end + end + + odebug "Uploaded #{bottle_count} bottles" + end + end +end diff --git a/Library/Homebrew/dev-cmd/pr-upload.rb b/Library/Homebrew/dev-cmd/pr-upload.rb index 951d24fef0..5006d29498 100644 --- a/Library/Homebrew/dev-cmd/pr-upload.rb +++ b/Library/Homebrew/dev-cmd/pr-upload.rb @@ -2,6 +2,7 @@ # frozen_string_literal: true require "cli/parser" +require "archive" require "bintray" module Homebrew @@ -27,6 +28,8 @@ module Homebrew switch "--warn-on-upload-failure", description: "Warn instead of raising an error if the bottle upload fails. "\ "Useful for repairing bottle uploads that previously failed." + flag "--archive-item=", + description: "Upload to the specified Archive item (default: `homebrew`)." flag "--bintray-org=", description: "Upload to the specified Bintray organisation (default: `homebrew`)." flag "--root-url=", @@ -47,6 +50,18 @@ module Homebrew end end + def archive?(bottles_hash) + @archive ||= bottles_hash.values.all? do |bottle_hash| + bottle_hash["bottle"]["root_url"].start_with? "https://archive.com/" + end + end + + def bintray?(bottles_hash) + @bintray ||= bottles_hash.values.all? do |bottle_hash| + bottle_hash["bottle"]["root_url"].match? %r{^https://[\w-]+\.bintray\.com/} + end + end + def github_releases?(bottles_hash) @github_releases ||= bottles_hash.values.all? do |bottle_hash| root_url = bottle_hash["bottle"]["root_url"] @@ -76,11 +91,16 @@ module Homebrew bottle_args += json_files if args.dry_run? - service = if github_releases?(bottles_hash) - "GitHub Releases" - else - "Bintray" - end + service = + if archive?(bottles_hash) + "Archive.org" + elsif bintray?(bottles_hash) + "Bintray" + elsif github_releases?(bottles_hash) + "GitHub Releases" + else + odie "Service specified by root_url is not recognized" + end puts <<~EOS brew #{bottle_args.join " "} Upload bottles described by these JSON files to #{service}: @@ -102,7 +122,20 @@ module Homebrew safe_system HOMEBREW_BREW_FILE, *audit_args end - if github_releases?(bottles_hash) + if archive?(bottles_hash) + # Handle uploading to Archive.org. + archive_item = args.archive_item || "homebrew" + archive = Archive.new(item: archive_item) + archive.upload_bottles(bottles_hash, + warn_on_error: args.warn_on_upload_failure?) + elsif bintray?(bottles_hash) + # Handle uploading to Bintray. + bintray_org = args.bintray_org || "homebrew" + bintray = Bintray.new(org: bintray_org) + bintray.upload_bottles(bottles_hash, + publish_package: !args.no_publish?, + warn_on_error: args.warn_on_upload_failure?) + elsif github_releases?(bottles_hash) # Handle uploading to GitHub Releases. bottles_hash.each_value do |bottle_hash| root_url = bottle_hash["bottle"]["root_url"] @@ -128,12 +161,7 @@ module Homebrew end end else - # Handle uploading to Bintray. - bintray_org = args.bintray_org || "homebrew" - bintray = Bintray.new(org: bintray_org) - bintray.upload_bottles(bottles_hash, - publish_package: !args.no_publish?, - warn_on_error: args.warn_on_upload_failure?) + odie "Service specified by root_url is not recognized" end end end diff --git a/Library/Homebrew/env_config.rb b/Library/Homebrew/env_config.rb index 8438e1aec4..bcb7903ada 100644 --- a/Library/Homebrew/env_config.rb +++ b/Library/Homebrew/env_config.rb @@ -15,6 +15,10 @@ module Homebrew description: "Linux only: Pass this value to a type name representing the compiler's `-march` option.", default: "native", }, + HOMEBREW_ARCHIVE_KEY: { + description: "Use this API key when accessing the Archive.org API (where bottles are stored). " \ + "The format is access:secret. See https://archive.org/account/s3.php", + }, HOMEBREW_ARTIFACT_DOMAIN: { description: "Prefix all download URLs, including those for bottles, with this value. " \ "For example, `HOMEBREW_ARTIFACT_DOMAIN=http://localhost:8080` will cause a " \