diff --git a/Library/Homebrew/archive.rb b/Library/Homebrew/archive.rb deleted file mode 100644 index 7ed40d91f7..0000000000 --- a/Library/Homebrew/archive.rb +++ /dev/null @@ -1,186 +0,0 @@ -# typed: false -# frozen_string_literal: true - -require "digest/md5" -require "utils/curl" - -# The Internet Archive API client. -# -# @api private -class Archive - extend T::Sig - - include Context - include Utils::Curl - - class Error < RuntimeError - end - - URL_PREFIX = "https://archive.org" - S3_DOMAIN = "s3.us.archive.org" - - sig { returns(String) } - def inspect - "#" - end - - sig { params(item: T.nilable(String)).void } - def initialize(item: "homebrew") - raise UsageError, "Must set the Archive item!" unless item - - @archive_item = item - end - - def open_api(url, *args, auth: true) - if auth - key = Homebrew::EnvConfig.internet_archive_key - raise UsageError, "HOMEBREW_INTERNET_ARCHIVE_KEY is unset." if key.blank? - - if key.exclude?(":") - raise UsageError, "Use HOMEBREW_INTERNET_ARCHIVE_KEY=access:secret. See #{URL_PREFIX}/account/s3.php" - end - - args += ["--header", "Authorization: AWS #{key}"] - end - - curl(*args, url, print_stdout: false, secrets: key) - end - - sig { - params(local_file: String, - directory: String, - remote_file: String, - warn_on_error: T.nilable(T::Boolean)).void - } - def upload(local_file, directory:, remote_file:, warn_on_error: false) - local_file = Pathname.new(local_file) - unless local_file.exist? - msg = "#{local_file} for upload doesn't exist!" - raise Error, msg unless warn_on_error - - # Warn and return early here since we know this upload is going to fail. - opoo msg - return - end - - md5_base64 = Digest::MD5.base64digest(local_file.read) - url = "https://#{@archive_item}.#{S3_DOMAIN}/#{directory}/#{remote_file}" - args = ["--upload-file", local_file, "--header", "Content-MD5: #{md5_base64}"] - args << "--fail" unless warn_on_error - result = T.unsafe(self).open_api(url, *args) - return if result.success? && result.stdout.exclude?("Error") - - msg = "Bottle upload failed: #{result.stdout}" - raise msg unless warn_on_error - - opoo msg - end - - sig { - params(formula: Formula, - directory: String, - warn_on_error: T::Boolean).returns(String) - } - def mirror_formula(formula, directory: "mirror", warn_on_error: false) - formula.downloader.fetch - - filename = ERB::Util.url_encode(formula.downloader.basename) - destination_url = "#{URL_PREFIX}/download/#{@archive_item}/#{directory}/#{filename}" - - odebug "Uploading to #{destination_url}" - - upload( - formula.downloader.cached_location, - directory: directory, - remote_file: filename, - warn_on_error: warn_on_error, - ) - - destination_url - end - - # Gets the MD5 hash of the specified remote file. - # - # @return the hash, the empty string (if the file doesn't have a hash), nil (if the file doesn't exist) - sig { params(directory: String, remote_file: String).returns(T.nilable(String)) } - def remote_md5(directory:, remote_file:) - url = "https://#{@archive_item}.#{S3_DOMAIN}/#{directory}/#{remote_file}" - result = curl_output "--fail", "--silent", "--head", "--location", url - if result.success? - result.stdout.match(/^ETag: "(\h{32})"/)&.values_at(1)&.first || "" - else - raise Error if result.status.exitstatus != 22 && result.stderr.exclude?("404 Not Found") - - nil - end - end - - sig { params(directory: String, filename: String).returns(String) } - def file_delete_instructions(directory, filename) - <<~EOS - Run: - curl -X DELETE -H "Authorization: AWS $HOMEBREW_INTERNET_ARCHIVE_KEY" https://#{@archive_item}.#{S3_DOMAIN}/#{directory}/#{filename} - Or run: - ia delete #{@archive_item} #{directory}/#{filename} - EOS - end - - sig { - params(bottles_hash: T::Hash[String, T.untyped], - warn_on_error: T.nilable(T::Boolean)).void - } - def upload_bottles(bottles_hash, warn_on_error: false) - bottles_hash.each do |formula_full_name, bottle_hash| - _, formula_tap_repo, = formula_full_name.split("/") - directory = if formula_tap_repo - "bottles-#{tap.repo}" - else - "bottles" - end - - bottle_count = bottle_hash["bottle"]["tags"].length - - bottle_hash["bottle"]["tags"].each_value do |tag_hash| - filename = tag_hash["filename"] # URL encoded in Bottle::Filename#archive - delete_instructions = file_delete_instructions(directory, filename) - - local_filename = tag_hash["local_filename"] - md5 = Digest::MD5.hexdigest(File.read(local_filename)) - - odebug "Checking remote file #{@archive_item}/#{directory}/#{filename}" - result = remote_md5(directory: directory, remote_file: filename) - case result - when nil - # File doesn't exist. - odebug "Uploading #{@archive_item}/#{directory}/#{filename}" - upload(local_filename, - directory: directory, - remote_file: filename, - warn_on_error: warn_on_error) - when md5 - # File exists, hash matches. - odebug "#{filename} is already published with matching hash." - bottle_count -= 1 - when "" - # File exists, but can't find hash - failed_message = "#{filename} is already published!" - raise Error, "#{failed_message}\n#{delete_instructions}" unless warn_on_error - - opoo failed_message - else - # File exists, but hash either doesn't exist or is mismatched. - failed_message = <<~EOS - #{filename} is already published with a mismatched hash! - Expected: #{md5} - Actual: #{result} - EOS - raise Error, "#{failed_message}#{delete_instructions}" unless warn_on_error - - opoo failed_message - end - end - - odebug "Uploaded #{bottle_count} bottles" - end - end -end diff --git a/Library/Homebrew/dev-cmd/bottle.rb b/Library/Homebrew/dev-cmd/bottle.rb index a5cf751af2..5ae9ec5961 100644 --- a/Library/Homebrew/dev-cmd/bottle.rb +++ b/Library/Homebrew/dev-cmd/bottle.rb @@ -9,7 +9,6 @@ require "formula_versions" require "cli/parser" require "utils/inreplace" require "erb" -require "archive" require "zlib" require "api" diff --git a/Library/Homebrew/dev-cmd/pr-pull.rb b/Library/Homebrew/dev-cmd/pr-pull.rb index 849999f0ae..f6b431f625 100644 --- a/Library/Homebrew/dev-cmd/pr-pull.rb +++ b/Library/Homebrew/dev-cmd/pr-pull.rb @@ -49,8 +49,6 @@ module Homebrew description: "Message to include when autosquashing revision bumps, deletions, and rebuilds." flag "--artifact=", description: "Download artifacts with the specified name (default: `bottles`)." - flag "--archive-item=", - description: "Upload to the specified Internet Archive item (default: `homebrew`)." flag "--tap=", description: "Target tap repository (default: `homebrew/core`)." flag "--root-url=", @@ -340,7 +338,6 @@ module Homebrew workflows = args.workflows.presence || ["tests.yml"] artifact = args.artifact || "bottles" - archive_item = args.archive_item tap = Tap.fetch(args.tap || CoreTap.instance.name) Utils::Git.set_name_email!(committer: args.committer.blank?) @@ -411,7 +408,6 @@ module Homebrew upload_args << "--committer=#{args.committer}" if args.committer upload_args << "--root-url=#{args.root_url}" if args.root_url upload_args << "--root-url-using=#{args.root_url_using}" if args.root_url_using - upload_args << "--archive-item=#{archive_item}" if archive_item.present? safe_system HOMEBREW_BREW_FILE, *upload_args end end diff --git a/Library/Homebrew/dev-cmd/pr-upload.rb b/Library/Homebrew/dev-cmd/pr-upload.rb index 4bf3333cbb..8e51d0d375 100644 --- a/Library/Homebrew/dev-cmd/pr-upload.rb +++ b/Library/Homebrew/dev-cmd/pr-upload.rb @@ -2,7 +2,6 @@ # frozen_string_literal: true require "cli/parser" -require "archive" require "github_packages" require "github_releases" @@ -29,8 +28,6 @@ module Homebrew "Useful for repairing bottle uploads that previously failed." flag "--committer=", description: "Specify a committer name and email in `git`'s standard author format." - flag "--archive-item=", - description: "Upload to the specified Internet Archive item (default: `homebrew`)." flag "--github-org=", description: "Upload to the specified GitHub organisation's GitHub Packages (default: `homebrew`)." flag "--root-url=", @@ -54,12 +51,6 @@ module Homebrew end end - def internet_archive?(bottles_hash) - @internet_archive ||= bottles_hash.values.all? do |bottle_hash| - bottle_hash["bottle"]["root_url"].start_with? "#{Archive::URL_PREFIX}/" - end - end - def github_releases?(bottles_hash) @github_releases ||= bottles_hash.values.all? do |bottle_hash| root_url = bottle_hash["bottle"]["root_url"] @@ -113,8 +104,6 @@ module Homebrew dry_run_service = if github_packages?(bottles_hash) # GitHub Packages has its own --dry-run handling. nil - elsif internet_archive?(bottles_hash) - "Internet Archive" elsif github_releases?(bottles_hash) "GitHub Releases" else @@ -158,12 +147,7 @@ module Homebrew safe_system HOMEBREW_BREW_FILE, *audit_args end - if internet_archive?(bottles_hash) - archive_item = args.archive_item || "homebrew" - archive = Archive.new(item: archive_item) - archive.upload_bottles(bottles_hash, - warn_on_error: args.warn_on_upload_failure?) - elsif github_releases?(bottles_hash) + if github_releases?(bottles_hash) github_releases = GitHubReleases.new github_releases.upload_bottles(bottles_hash) elsif github_packages?(bottles_hash) diff --git a/Library/Homebrew/env_config.rb b/Library/Homebrew/env_config.rb index 7ae86692e7..26ef31b11a 100644 --- a/Library/Homebrew/env_config.rb +++ b/Library/Homebrew/env_config.rb @@ -187,10 +187,6 @@ module Homebrew default_text: 'The "Beer Mug" emoji.', default: "🍺", }, - HOMEBREW_INTERNET_ARCHIVE_KEY: { - description: "Use this API key when accessing the Internet Archive S3 API, where bottles are stored. " \ - "The format is access:secret. See https://archive.org/account/s3.php", - }, HOMEBREW_LIVECHECK_WATCHLIST: { description: "Consult this file for the list of formulae to check by default when no formula argument " \ "is passed to `brew livecheck`.", diff --git a/Library/Homebrew/test/archive_spec.rb b/Library/Homebrew/test/archive_spec.rb deleted file mode 100644 index 877aa7e630..0000000000 --- a/Library/Homebrew/test/archive_spec.rb +++ /dev/null @@ -1,20 +0,0 @@ -# typed: false -# frozen_string_literal: true - -require "archive" - -describe Archive, :needs_network do - subject(:archive) { described_class.new(item: "homebrew") } - - describe "::remote_checksum" do - it "detects a published file" do - hash = archive.remote_md5(directory: ".", remote_file: "cmake-3.1.2.yosemite.bottle.tar.gz") - expect(hash).to eq("c6e525d472124670b0b635800488f438") - end - - it "fails on a non-existent file" do - hash = archive.remote_md5(directory: "bottles", remote_file: "my-fake-bottle-1.0.snow_hyena.tar.gz") - expect(hash).to be nil - end - end -end