Merge pull request #12130 from FnControlOption/patch-4

archive: delete

Library/Homebrew/archive.rb

@@ -1,186 +0,0 @@
-# typed: false
-# frozen_string_literal: true
-
-require "digest/md5"
-require "utils/curl"
-
-# The Internet Archive API client.
-#
-# @api private
-class Archive
-  extend T::Sig
-
-  include Context
-  include Utils::Curl
-
-  class Error < RuntimeError
-  end
-
-  URL_PREFIX = "https://archive.org"
-  S3_DOMAIN = "s3.us.archive.org"
-
-  sig { returns(String) }
-  def inspect
-    "#<Archive: item=#{@archive_item}>"
-  end
-
-  sig { params(item: T.nilable(String)).void }
-  def initialize(item: "homebrew")
-    raise UsageError, "Must set the Archive item!" unless item
-
-    @archive_item = item
-  end
-
-  def open_api(url, *args, auth: true)
-    if auth
-      key = Homebrew::EnvConfig.internet_archive_key
-      raise UsageError, "HOMEBREW_INTERNET_ARCHIVE_KEY is unset." if key.blank?
-
-      if key.exclude?(":")
-        raise UsageError, "Use HOMEBREW_INTERNET_ARCHIVE_KEY=access:secret. See #{URL_PREFIX}/account/s3.php"
-      end
-
-      args += ["--header", "Authorization: AWS #{key}"]
-    end
-
-    curl(*args, url, print_stdout: false, secrets: key)
-  end
-
-  sig {
-    params(local_file:    String,
-           directory:     String,
-           remote_file:   String,
-           warn_on_error: T.nilable(T::Boolean)).void
-  }
-  def upload(local_file, directory:, remote_file:, warn_on_error: false)
-    local_file = Pathname.new(local_file)
-    unless local_file.exist?
-      msg = "#{local_file} for upload doesn't exist!"
-      raise Error, msg unless warn_on_error
-
-      # Warn and return early here since we know this upload is going to fail.
-      opoo msg
-      return
-    end
-
-    md5_base64 = Digest::MD5.base64digest(local_file.read)
-    url = "https://#{@archive_item}.#{S3_DOMAIN}/#{directory}/#{remote_file}"
-    args = ["--upload-file", local_file, "--header", "Content-MD5: #{md5_base64}"]
-    args << "--fail" unless warn_on_error
-    result = T.unsafe(self).open_api(url, *args)
-    return if result.success? && result.stdout.exclude?("Error")
-
-    msg = "Bottle upload failed: #{result.stdout}"
-    raise msg unless warn_on_error
-
-    opoo msg
-  end
-
-  sig {
-    params(formula:       Formula,
-           directory:     String,
-           warn_on_error: T::Boolean).returns(String)
-  }
-  def mirror_formula(formula, directory: "mirror", warn_on_error: false)
-    formula.downloader.fetch
-
-    filename = ERB::Util.url_encode(formula.downloader.basename)
-    destination_url = "#{URL_PREFIX}/download/#{@archive_item}/#{directory}/#{filename}"
-
-    odebug "Uploading to #{destination_url}"
-
-    upload(
-      formula.downloader.cached_location,
-      directory:     directory,
-      remote_file:   filename,
-      warn_on_error: warn_on_error,
-    )
-
-    destination_url
-  end
-
-  # Gets the MD5 hash of the specified remote file.
-  #
-  # @return the hash, the empty string (if the file doesn't have a hash), nil (if the file doesn't exist)
-  sig { params(directory: String, remote_file: String).returns(T.nilable(String)) }
-  def remote_md5(directory:, remote_file:)
-    url = "https://#{@archive_item}.#{S3_DOMAIN}/#{directory}/#{remote_file}"
-    result = curl_output "--fail", "--silent", "--head", "--location", url
-    if result.success?
-      result.stdout.match(/^ETag: "(\h{32})"/)&.values_at(1)&.first || ""
-    else
-      raise Error if result.status.exitstatus != 22 && result.stderr.exclude?("404 Not Found")
-
-      nil
-    end
-  end
-
-  sig { params(directory: String, filename: String).returns(String) }
-  def file_delete_instructions(directory, filename)
-    <<~EOS
-      Run:
-        curl -X DELETE -H "Authorization: AWS $HOMEBREW_INTERNET_ARCHIVE_KEY" https://#{@archive_item}.#{S3_DOMAIN}/#{directory}/#{filename}
-      Or run:
-        ia delete #{@archive_item} #{directory}/#{filename}
-    EOS
-  end
-
-  sig {
-    params(bottles_hash:  T::Hash[String, T.untyped],
-           warn_on_error: T.nilable(T::Boolean)).void
-  }
-  def upload_bottles(bottles_hash, warn_on_error: false)
-    bottles_hash.each do |formula_full_name, bottle_hash|
-      _, formula_tap_repo, = formula_full_name.split("/")
-      directory = if formula_tap_repo
-        "bottles-#{tap.repo}"
-      else
-        "bottles"
-      end
-
-      bottle_count = bottle_hash["bottle"]["tags"].length
-
-      bottle_hash["bottle"]["tags"].each_value do |tag_hash|
-        filename = tag_hash["filename"] # URL encoded in Bottle::Filename#archive
-        delete_instructions = file_delete_instructions(directory, filename)
-
-        local_filename = tag_hash["local_filename"]
-        md5 = Digest::MD5.hexdigest(File.read(local_filename))
-
-        odebug "Checking remote file #{@archive_item}/#{directory}/#{filename}"
-        result = remote_md5(directory: directory, remote_file: filename)
-        case result
-        when nil
-          # File doesn't exist.
-          odebug "Uploading #{@archive_item}/#{directory}/#{filename}"
-          upload(local_filename,
-                 directory:     directory,
-                 remote_file:   filename,
-                 warn_on_error: warn_on_error)
-        when md5
-          # File exists, hash matches.
-          odebug "#{filename} is already published with matching hash."
-          bottle_count -= 1
-        when ""
-          # File exists, but can't find hash
-          failed_message = "#{filename} is already published!"
-          raise Error, "#{failed_message}\n#{delete_instructions}" unless warn_on_error
-
-          opoo failed_message
-        else
-          # File exists, but hash either doesn't exist or is mismatched.
-          failed_message = <<~EOS
-            #{filename} is already published with a mismatched hash!
-              Expected: #{md5}
-              Actual:   #{result}
-          EOS
-          raise Error, "#{failed_message}#{delete_instructions}" unless warn_on_error
-
-          opoo failed_message
-        end
-      end
-
-      odebug "Uploaded #{bottle_count} bottles"
-    end
-  end
-end

Library/Homebrew/dev-cmd/bottle.rb

@@ -9,7 +9,6 @@ require "formula_versions"
 require "cli/parser"
 require "utils/inreplace"
 require "erb"
-require "archive"
 require "zlib"
 require "api"
 

Library/Homebrew/dev-cmd/pr-pull.rb

@@ -49,8 +49,6 @@ module Homebrew
              description: "Message to include when autosquashing revision bumps, deletions, and rebuilds."
       flag   "--artifact=",
              description: "Download artifacts with the specified name (default: `bottles`)."
-      flag   "--archive-item=",
-             description: "Upload to the specified Internet Archive item (default: `homebrew`)."
       flag   "--tap=",
              description: "Target tap repository (default: `homebrew/core`)."
       flag   "--root-url=",
@@ -340,7 +338,6 @@ module Homebrew
 
     workflows = args.workflows.presence || ["tests.yml"]
     artifact = args.artifact || "bottles"
-    archive_item = args.archive_item
     tap = Tap.fetch(args.tap || CoreTap.instance.name)
 
     Utils::Git.set_name_email!(committer: args.committer.blank?)
@@ -411,7 +408,6 @@ module Homebrew
           upload_args << "--committer=#{args.committer}" if args.committer
           upload_args << "--root-url=#{args.root_url}" if args.root_url
           upload_args << "--root-url-using=#{args.root_url_using}" if args.root_url_using
-          upload_args << "--archive-item=#{archive_item}" if archive_item.present?
           safe_system HOMEBREW_BREW_FILE, *upload_args
         end
       end

Library/Homebrew/dev-cmd/pr-upload.rb

@@ -2,7 +2,6 @@
 # frozen_string_literal: true
 
 require "cli/parser"
-require "archive"
 require "github_packages"
 require "github_releases"
 
@@ -29,8 +28,6 @@ module Homebrew
                           "Useful for repairing bottle uploads that previously failed."
       flag   "--committer=",
              description: "Specify a committer name and email in `git`'s standard author format."
-      flag   "--archive-item=",
-             description: "Upload to the specified Internet Archive item (default: `homebrew`)."
       flag   "--github-org=",
              description: "Upload to the specified GitHub organisation's GitHub Packages (default: `homebrew`)."
       flag   "--root-url=",
@@ -54,12 +51,6 @@ module Homebrew
     end
   end
 
-  def internet_archive?(bottles_hash)
-    @internet_archive ||= bottles_hash.values.all? do |bottle_hash|
-      bottle_hash["bottle"]["root_url"].start_with? "#{Archive::URL_PREFIX}/"
-    end
-  end
-
   def github_releases?(bottles_hash)
     @github_releases ||= bottles_hash.values.all? do |bottle_hash|
       root_url = bottle_hash["bottle"]["root_url"]
@@ -113,8 +104,6 @@ module Homebrew
       dry_run_service = if github_packages?(bottles_hash)
         # GitHub Packages has its own --dry-run handling.
         nil
-      elsif internet_archive?(bottles_hash)
-        "Internet Archive"
       elsif github_releases?(bottles_hash)
         "GitHub Releases"
       else
@@ -158,12 +147,7 @@ module Homebrew
       safe_system HOMEBREW_BREW_FILE, *audit_args
     end
 
-    if internet_archive?(bottles_hash)
+    if github_releases?(bottles_hash)
-      archive_item = args.archive_item || "homebrew"
-      archive = Archive.new(item: archive_item)
-      archive.upload_bottles(bottles_hash,
-                             warn_on_error: args.warn_on_upload_failure?)
-    elsif github_releases?(bottles_hash)
       github_releases = GitHubReleases.new
       github_releases.upload_bottles(bottles_hash)
     elsif github_packages?(bottles_hash)

Library/Homebrew/env_config.rb

@@ -187,10 +187,6 @@ module Homebrew
         default_text: 'The "Beer Mug" emoji.',
         default:      "🍺",
       },
-      HOMEBREW_INTERNET_ARCHIVE_KEY:              {
-        description: "Use this API key when accessing the Internet Archive S3 API, where bottles are stored. " \
-                     "The format is access:secret. See https://archive.org/account/s3.php",
-      },
       HOMEBREW_LIVECHECK_WATCHLIST:               {
         description: "Consult this file for the list of formulae to check by default when no formula argument " \
                      "is passed to `brew livecheck`.",

Library/Homebrew/test/archive_spec.rb

@@ -1,20 +0,0 @@
-# typed: false
-# frozen_string_literal: true
-
-require "archive"
-
-describe Archive, :needs_network do
-  subject(:archive) { described_class.new(item: "homebrew") }
-
-  describe "::remote_checksum" do
-    it "detects a published file" do
-      hash = archive.remote_md5(directory: ".", remote_file: "cmake-3.1.2.yosemite.bottle.tar.gz")
-      expect(hash).to eq("c6e525d472124670b0b635800488f438")
-    end
-
-    it "fails on a non-existent file" do
-      hash = archive.remote_md5(directory: "bottles", remote_file: "my-fake-bottle-1.0.snow_hyena.tar.gz")
-      expect(hash).to be nil
-    end
-  end
-end