admin/brew
1
0
Fork 0
Code Issues Packages Projects Releases 6 Wiki Activity

Merge pull request #12130 from FnControlOption/patch-4

Browse Source 
archive: delete
This commit is contained in:
Mike McQuaid 2021-10-01 12:58:28 +01:00 committed by GitHub
commit 04b54df776
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 1 additions and 232 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

186
Library/Homebrew/archive.rb
View File

@ -1,186 +0,0 @@
# typed: false
# frozen_string_literal: true
require "digest/md5"
require "utils/curl"
# The Internet Archive API client.
#
# @api private
class Archive
extend T::Sig
include Context
include Utils::Curl
class Error < RuntimeError
end
URL_PREFIX = "https://archive.org"
S3_DOMAIN = "s3.us.archive.org"
sig { returns(String) }
def inspect
"#<Archive: item=#{@archive_item}>"
end
sig { params(item: T.nilable(String)).void }
def initialize(item: "homebrew")
raise UsageError, "Must set the Archive item!" unless item
@archive_item = item
end
def open_api(url, *args, auth: true)
if auth
key = Homebrew::EnvConfig.internet_archive_key
raise UsageError, "HOMEBREW_INTERNET_ARCHIVE_KEY is unset." if key.blank?
if key.exclude?(":")
raise UsageError, "Use HOMEBREW_INTERNET_ARCHIVE_KEY=access:secret. See #{URL_PREFIX}/account/s3.php"
end
args += ["--header", "Authorization: AWS #{key}"]
end
curl(*args, url, print_stdout: false, secrets: key)
end
sig {
params(local_file: String,
directory: String,
remote_file: String,
warn_on_error: T.nilable(T::Boolean)).void
}
def upload(local_file, directory:, remote_file:, warn_on_error: false)
local_file = Pathname.new(local_file)
unless local_file.exist?
msg = "#{local_file} for upload doesn't exist!"
raise Error, msg unless warn_on_error
# Warn and return early here since we know this upload is going to fail.
opoo msg
return
end
md5_base64 = Digest::MD5.base64digest(local_file.read)
url = "https://#{@archive_item}.#{S3_DOMAIN}/#{directory}/#{remote_file}"
args = ["--upload-file", local_file, "--header", "Content-MD5: #{md5_base64}"]
args << "--fail" unless warn_on_error
result = T.unsafe(self).open_api(url, *args)
return if result.success? && result.stdout.exclude?("Error")
msg = "Bottle upload failed: #{result.stdout}"
raise msg unless warn_on_error
opoo msg
end
sig {
params(formula: Formula,
directory: String,
warn_on_error: T::Boolean).returns(String)
}
def mirror_formula(formula, directory: "mirror", warn_on_error: false)
formula.downloader.fetch
filename = ERB::Util.url_encode(formula.downloader.basename)
destination_url = "#{URL_PREFIX}/download/#{@archive_item}/#{directory}/#{filename}"
odebug "Uploading to #{destination_url}"
upload(
formula.downloader.cached_location,
directory: directory,
remote_file: filename,
warn_on_error: warn_on_error,
)
destination_url
end
# Gets the MD5 hash of the specified remote file.
#
# @return the hash, the empty string (if the file doesn't have a hash), nil (if the file doesn't exist)
sig { params(directory: String, remote_file: String).returns(T.nilable(String)) }
def remote_md5(directory:, remote_file:)
url = "https://#{@archive_item}.#{S3_DOMAIN}/#{directory}/#{remote_file}"
result = curl_output "--fail", "--silent", "--head", "--location", url
if result.success?
result.stdout.match(/^ETag: "(\h{32})"/)&.values_at(1)&.first || ""
else
raise Error if result.status.exitstatus != 22 && result.stderr.exclude?("404 Not Found")
nil
end
end
sig { params(directory: String, filename: String).returns(String) }
def file_delete_instructions(directory, filename)
<<~EOS
Run:
curl -X DELETE -H "Authorization: AWS $HOMEBREW_INTERNET_ARCHIVE_KEY" https://#{@archive_item}.#{S3_DOMAIN}/#{directory}/#{filename}
Or run:
ia delete #{@archive_item} #{directory}/#{filename}
EOS
end
sig {
params(bottles_hash: T::Hash[String, T.untyped],
warn_on_error: T.nilable(T::Boolean)).void
}
def upload_bottles(bottles_hash, warn_on_error: false)
bottles_hash.each do |formula_full_name, bottle_hash|
_, formula_tap_repo, = formula_full_name.split("/")
directory = if formula_tap_repo
"bottles-#{tap.repo}"
else
"bottles"
end
bottle_count = bottle_hash["bottle"]["tags"].length
bottle_hash["bottle"]["tags"].each_value do |tag_hash|
filename = tag_hash["filename"] # URL encoded in Bottle::Filename#archive
delete_instructions = file_delete_instructions(directory, filename)
local_filename = tag_hash["local_filename"]
md5 = Digest::MD5.hexdigest(File.read(local_filename))
odebug "Checking remote file #{@archive_item}/#{directory}/#{filename}"
result = remote_md5(directory: directory, remote_file: filename)
case result
when nil
# File doesn't exist.
odebug "Uploading #{@archive_item}/#{directory}/#{filename}"
upload(local_filename,
directory: directory,
remote_file: filename,
warn_on_error: warn_on_error)
when md5
# File exists, hash matches.
odebug "#{filename} is already published with matching hash."
bottle_count -= 1
when ""
# File exists, but can't find hash
failed_message = "#{filename} is already published!"
raise Error, "#{failed_message}\n#{delete_instructions}" unless warn_on_error
opoo failed_message
else
# File exists, but hash either doesn't exist or is mismatched.
failed_message = <<~EOS
#{filename} is already published with a mismatched hash!
Expected: #{md5}
Actual: #{result}
EOS
raise Error, "#{failed_message}#{delete_instructions}" unless warn_on_error
opoo failed_message
end
end
odebug "Uploaded #{bottle_count} bottles"
end
end
end

1
Library/Homebrew/dev-cmd/bottle.rb
View File

@ -9,7 +9,6 @@ require "formula_versions"
require "cli/parser"
require "utils/inreplace"
require "erb"
require "archive"
require "zlib"
require "api"

4
Library/Homebrew/dev-cmd/pr-pull.rb
View File

@ -49,8 +49,6 @@ module Homebrew
description: "Message to include when autosquashing revision bumps, deletions, and rebuilds."
flag "--artifact=",
description: "Download artifacts with the specified name (default: `bottles`)."
flag "--archive-item=",
description: "Upload to the specified Internet Archive item (default: `homebrew`)."
flag "--tap=",
description: "Target tap repository (default: `homebrew/core`)."
flag "--root-url=",
@ -340,7 +338,6 @@ module Homebrew
workflows = args.workflows.presence || ["tests.yml"]
artifact = args.artifact || "bottles"
archive_item = args.archive_item
tap = Tap.fetch(args.tap || CoreTap.instance.name)
Utils::Git.set_name_email!(committer: args.committer.blank?)
@ -411,7 +408,6 @@ module Homebrew
upload_args << "--committer=#{args.committer}" if args.committer
upload_args << "--root-url=#{args.root_url}" if args.root_url
upload_args << "--root-url-using=#{args.root_url_using}" if args.root_url_using
upload_args << "--archive-item=#{archive_item}" if archive_item.present?
safe_system HOMEBREW_BREW_FILE, *upload_args
end
end

18
Library/Homebrew/dev-cmd/pr-upload.rb
View File

@ -2,7 +2,6 @@
# frozen_string_literal: true
require "cli/parser"
require "archive"
require "github_packages"
require "github_releases"
@ -29,8 +28,6 @@ module Homebrew
"Useful for repairing bottle uploads that previously failed."
flag "--committer=",
description: "Specify a committer name and email in `git`'s standard author format."
flag "--archive-item=",
description: "Upload to the specified Internet Archive item (default: `homebrew`)."
flag "--github-org=",
description: "Upload to the specified GitHub organisation's GitHub Packages (default: `homebrew`)."
flag "--root-url=",
@ -54,12 +51,6 @@ module Homebrew
end
end
def internet_archive?(bottles_hash)
@internet_archive ||= bottles_hash.values.all? do |bottle_hash|
bottle_hash["bottle"]["root_url"].start_with? "#{Archive::URL_PREFIX}/"
end
end
def github_releases?(bottles_hash)
@github_releases ||= bottles_hash.values.all? do |bottle_hash|
root_url = bottle_hash["bottle"]["root_url"]
@ -113,8 +104,6 @@ module Homebrew
dry_run_service = if github_packages?(bottles_hash)
# GitHub Packages has its own --dry-run handling.
nil
elsif internet_archive?(bottles_hash)
"Internet Archive"
elsif github_releases?(bottles_hash)
"GitHub Releases"
else
@ -158,12 +147,7 @@ module Homebrew
safe_system HOMEBREW_BREW_FILE, *audit_args
end
if internet_archive?(bottles_hash)
archive_item = args.archive_item || "homebrew"
archive = Archive.new(item: archive_item)
archive.upload_bottles(bottles_hash,
warn_on_error: args.warn_on_upload_failure?)
elsif github_releases?(bottles_hash)
if github_releases?(bottles_hash)
github_releases = GitHubReleases.new
github_releases.upload_bottles(bottles_hash)
elsif github_packages?(bottles_hash)

4
Library/Homebrew/env_config.rb
View File

@ -187,10 +187,6 @@ module Homebrew
default_text: 'The "Beer Mug" emoji.',
default: "🍺",
},
HOMEBREW_INTERNET_ARCHIVE_KEY: {
description: "Use this API key when accessing the Internet Archive S3 API, where bottles are stored. " \
"The format is access:secret. See https://archive.org/account/s3.php",
},
HOMEBREW_LIVECHECK_WATCHLIST: {
description: "Consult this file for the list of formulae to check by default when no formula argument " \
"is passed to `brew livecheck`.",

20
Library/Homebrew/test/archive_spec.rb
View File

@ -1,20 +0,0 @@
# typed: false
# frozen_string_literal: true
require "archive"
describe Archive, :needs_network do
subject(:archive) { described_class.new(item: "homebrew") }
describe "::remote_checksum" do
it "detects a published file" do
hash = archive.remote_md5(directory: ".", remote_file: "cmake-3.1.2.yosemite.bottle.tar.gz")
expect(hash).to eq("c6e525d472124670b0b635800488f438")
end
it "fails on a non-existent file" do
hash = archive.remote_md5(directory: "bottles", remote_file: "my-fake-bottle-1.0.snow_hyena.tar.gz")
expect(hash).to be nil
end
end
end