pull: extract bottle publishing code

and add tests
This commit is contained in:
Michka Popoff 2019-04-27 00:17:48 +02:00
parent fb643a28f0
commit a70179d8b7
5 changed files with 333 additions and 227 deletions

View File

@ -0,0 +1,162 @@
# frozen_string_literal: true
require "utils"
require "formula_info"
class BottlePublisher
def initialize(tap, changed_formulae_names, bintray_org, no_publish, warn_on_publish_failure)
@tap = tap
@changed_formulae_names = changed_formulae_names
@no_publish = no_publish
@bintray_org = bintray_org
@warn_on_publish_failure = warn_on_publish_failure
end
def publish_and_check_bottles
# Formulae with affected bottles that were published
bintray_published_formulae = []
# Publish bottles on Bintray
unless @no_publish
published = publish_changed_formula_bottles
bintray_published_formulae.concat(published)
end
# Verify bintray publishing after all patches have been applied
bintray_published_formulae.uniq!
verify_bintray_published(bintray_published_formulae)
end
def publish_changed_formula_bottles
raise "Need to load formulae to publish them!" if ENV["HOMEBREW_DISABLE_LOAD_FORMULA"]
published = []
bintray_creds = { user: ENV["HOMEBREW_BINTRAY_USER"], key: ENV["HOMEBREW_BINTRAY_KEY"] }
if bintray_creds[:user] && bintray_creds[:key]
@changed_formulae_names.each do |name|
f = Formula[name]
next if f.bottle_unneeded? || f.bottle_disabled?
bintray_org = @bintray_org || @tap.user.downcase
next unless publish_bottle_file_on_bintray(f, bintray_org, bintray_creds)
published << f.full_name
end
else
opoo "You must set HOMEBREW_BINTRAY_USER and HOMEBREW_BINTRAY_KEY to add or update bottles on Bintray!"
end
published
end
# Publishes the current bottle files for a given formula to Bintray
def publish_bottle_file_on_bintray(f, bintray_org, creds)
repo = Utils::Bottles::Bintray.repository(f.tap)
package = Utils::Bottles::Bintray.package(f.name)
info = FormulaInfo.lookup(f.full_name)
raise "Failed publishing bottle: failed reading formula info for #{f.full_name}" if info.nil?
unless info.bottle_info_any
opoo "No bottle defined in formula #{package}"
return false
end
version = info.pkg_version
ohai "Publishing on Bintray: #{package} #{version}"
curl "--write-out", '\n', "--silent", "--fail",
"--user", "#{creds[:user]}:#{creds[:key]}", "--request", "POST",
"--header", "Content-Type: application/json",
"--data", '{"publish_wait_for_secs": 0}',
"https://api.bintray.com/content/#{bintray_org}/#{repo}/#{package}/#{version}/publish"
true
rescue => e
raise unless @warn_on_publish_failure
onoe e
false
end
# Verifies that formulae have been published on Bintray by downloading a bottle file
# for each one. Blocks until the published files are available.
# Raises an error if the verification fails.
# This does not currently work for `brew pull`, because it may have cached the old
# version of a formula.
def verify_bintray_published(formulae_names)
return if formulae_names.empty?
raise "Need to load formulae to verify their publication!" if ENV["HOMEBREW_DISABLE_LOAD_FORMULA"]
ohai "Verifying bottles published on Bintray"
formulae = formulae_names.map { |n| Formula[n] }
max_retries = 300 # shared among all bottles
poll_retry_delay_seconds = 2
HOMEBREW_CACHE.cd do
formulae.each do |f|
retry_count = 0
wrote_dots = false
# Choose arbitrary bottle just to get the host/port for Bintray right
jinfo = FormulaInfo.lookup(f.full_name)
unless jinfo
opoo "Cannot publish bottle: Failed reading info for formula #{f.full_name}"
next
end
bottle_info = jinfo.bottle_info_any
unless bottle_info
opoo "No bottle defined in formula #{f.full_name}"
next
end
# Poll for publication completion using a quick partial HEAD, to avoid spurious error messages
# 401 error is normal while file is still in async publishing process
url = URI(bottle_info.url)
puts "Verifying bottle: #{File.basename(url.path)}"
http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true
retry_count = 0
http.start do
loop do
req = Net::HTTP::Head.new bottle_info.url
req.initialize_http_header "User-Agent" => HOMEBREW_USER_AGENT_RUBY
res = http.request req
break if res.is_a?(Net::HTTPSuccess) || res.code == "302"
unless res.is_a?(Net::HTTPClientError)
raise "Failed to find published #{f} bottle at #{url} (#{res.code} #{res.message})!"
end
raise "Failed to find published #{f} bottle at #{url}!" if retry_count >= max_retries
print(wrote_dots ? "." : "Waiting on Bintray.")
wrote_dots = true
sleep poll_retry_delay_seconds
retry_count += 1
end
end
# Actual download and verification
# We do a retry on this, too, because sometimes the external curl will fail even
# when the prior HEAD has succeeded.
puts "\n" if wrote_dots
filename = File.basename(url.path)
curl_retry_delay_seconds = 4
max_curl_retries = 1
retry_count = 0
# We're in the cache; make sure to force re-download
loop do
begin
curl_download url, to: filename
break
rescue
raise "Failed to download #{f} bottle from #{url}!" if retry_count >= max_curl_retries
puts "curl download failed; retrying in #{curl_retry_delay_seconds} sec"
sleep curl_retry_delay_seconds
curl_retry_delay_seconds *= 2
retry_count += 1
end
end
checksum = Checksum.new(:sha256, bottle_info.sha256)
Pathname.new(filename).verify_checksum(checksum)
end
end
end
end

View File

@ -8,6 +8,8 @@ require "formula"
require "formulary"
require "version"
require "pkg_version"
require "bottle_publisher"
require "formula_info"
module GitHub
module_function
@ -253,7 +255,13 @@ module Homebrew
else
fetch_bottles_patch(bottle_commit_url, args, bottle_branch, branch, orig_revision)
end
publish_and_check_bottles(tap, args, changed_formulae_names)
BottlePublisher.new(
tap,
changed_formulae_names,
args.bintray_org,
args.no_publish?,
args.warn_on_publish_failure?,
).publish_and_check_bottles
elsif merge_commit
fetch_merge_patch(url, args, issue)
end
@ -301,46 +309,6 @@ module Homebrew
PatchPuller.new(url, args, "merge commit").pull_merge_commit(issue)
end
def publish_and_check_bottles(tap, args, changed_formulae_names)
# Formulae with affected bottles that were published
bintray_published_formulae = []
# Publish bottles on Bintray
unless args.no_publish?
published = publish_changed_formula_bottles(tap, changed_formulae_names)
bintray_published_formulae.concat(published)
end
# Verify bintray publishing after all patches have been applied
bintray_published_formulae.uniq!
verify_bintray_published(bintray_published_formulae)
end
def force_utf8!(str)
str.force_encoding("UTF-8") if str.respond_to?(:force_encoding)
end
def publish_changed_formula_bottles(tap, changed_formulae_names)
raise "Need to load formulae to publish them!" if ENV["HOMEBREW_DISABLE_LOAD_FORMULA"]
published = []
bintray_creds = { user: ENV["HOMEBREW_BINTRAY_USER"], key: ENV["HOMEBREW_BINTRAY_KEY"] }
if bintray_creds[:user] && bintray_creds[:key]
changed_formulae_names.each do |name|
f = Formula[name]
next if f.bottle_unneeded? || f.bottle_disabled?
bintray_org = args.bintray_org || tap.user.downcase
next unless publish_bottle_file_on_bintray(f, bintray_org, bintray_creds)
published << f.full_name
end
else
opoo "You must set HOMEBREW_BINTRAY_USER and HOMEBREW_BINTRAY_KEY to add or update bottles on Bintray!"
end
published
end
class PatchPuller
attr_reader :base_url
attr_reader :patch_url
@ -440,7 +408,7 @@ module Homebrew
# Returns info as a hash (type => version), for pull.rb's internal use.
# Uses special key `:nonexistent => true` for nonexistent formulae.
def current_versions_from_info_external(formula_name)
info = FormulaInfoFromJson.lookup(formula_name)
info = FormulaInfo.lookup(formula_name)
versions = {}
if info
[:stable, :devel, :head].each do |spec_type|
@ -495,191 +463,6 @@ module Homebrew
Utils.popen_write("pbcopy") { |io| io.write text }
end
# Publishes the current bottle files for a given formula to Bintray
def publish_bottle_file_on_bintray(f, bintray_org, creds)
repo = Utils::Bottles::Bintray.repository(f.tap)
package = Utils::Bottles::Bintray.package(f.name)
info = FormulaInfoFromJson.lookup(f.full_name)
raise "Failed publishing bottle: failed reading formula info for #{f.full_name}" if info.nil?
unless info.bottle_info_any
opoo "No bottle defined in formula #{package}"
return false
end
version = info.pkg_version
ohai "Publishing on Bintray: #{package} #{version}"
curl "--write-out", '\n', "--silent", "--fail",
"--user", "#{creds[:user]}:#{creds[:key]}", "--request", "POST",
"--header", "Content-Type: application/json",
"--data", '{"publish_wait_for_secs": 0}',
"https://api.bintray.com/content/#{bintray_org}/#{repo}/#{package}/#{version}/publish"
true
rescue => e
raise unless @args.warn_on_publish_failure?
onoe e
false
end
# Formula info drawn from an external `brew info --json` call
class FormulaInfoFromJson
# The whole info structure parsed from the JSON
attr_accessor :info
def initialize(info)
@info = info
end
# Looks up formula on disk and reads its info.
# Returns nil if formula is absent or if there was an error reading it.
def self.lookup(name)
json = Utils.popen_read(HOMEBREW_BREW_FILE, "info", "--json=v1", name)
return unless $CHILD_STATUS.success?
Homebrew.force_utf8!(json)
FormulaInfoFromJson.new(JSON.parse(json)[0])
end
def bottle_tags
return [] unless info["bottle"]["stable"]
info["bottle"]["stable"]["files"].keys
end
def bottle_info(my_bottle_tag = Utils::Bottles.tag)
tag_s = my_bottle_tag.to_s
return unless info["bottle"]["stable"]
btl_info = info["bottle"]["stable"]["files"][tag_s]
return unless btl_info
BottleInfo.new(btl_info["url"], btl_info["sha256"])
end
def bottle_info_any
bottle_info(any_bottle_tag)
end
def any_bottle_tag
tag = Utils::Bottles.tag.to_s
# Prefer native bottles as a convenience for download caching
bottle_tags.include?(tag) ? tag : bottle_tags.first
end
def version(spec_type)
version_str = info["versions"][spec_type.to_s]
version_str && Version.create(version_str)
end
def pkg_version(spec_type = :stable)
PkgVersion.new(version(spec_type), revision)
end
def revision
info["revision"]
end
end
# Bottle info as used internally by pull, with alternate platform support.
class BottleInfo
# URL of bottle as string
attr_accessor :url
# Expected SHA-256 as string
attr_accessor :sha256
def initialize(url, sha256)
@url = url
@sha256 = sha256
end
end
# Verifies that formulae have been published on Bintray by downloading a bottle file
# for each one. Blocks until the published files are available.
# Raises an error if the verification fails.
# This does not currently work for `brew pull`, because it may have cached the old
# version of a formula.
def verify_bintray_published(formulae_names)
return if formulae_names.empty?
raise "Need to load formulae to verify their publication!" if ENV["HOMEBREW_DISABLE_LOAD_FORMULA"]
ohai "Verifying bottles published on Bintray"
formulae = formulae_names.map { |n| Formula[n] }
max_retries = 300 # shared among all bottles
poll_retry_delay_seconds = 2
HOMEBREW_CACHE.cd do
formulae.each do |f|
retry_count = 0
wrote_dots = false
# Choose arbitrary bottle just to get the host/port for Bintray right
jinfo = FormulaInfoFromJson.lookup(f.full_name)
unless jinfo
opoo "Cannot publish bottle: Failed reading info for formula #{f.full_name}"
next
end
bottle_info = jinfo.bottle_info_any
unless bottle_info
opoo "No bottle defined in formula #{f.full_name}"
next
end
# Poll for publication completion using a quick partial HEAD, to avoid spurious error messages
# 401 error is normal while file is still in async publishing process
url = URI(bottle_info.url)
puts "Verifying bottle: #{File.basename(url.path)}"
http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true
retry_count = 0
http.start do
loop do
req = Net::HTTP::Head.new bottle_info.url
req.initialize_http_header "User-Agent" => HOMEBREW_USER_AGENT_RUBY
res = http.request req
break if res.is_a?(Net::HTTPSuccess) || res.code == "302"
unless res.is_a?(Net::HTTPClientError)
raise "Failed to find published #{f} bottle at #{url} (#{res.code} #{res.message})!"
end
raise "Failed to find published #{f} bottle at #{url}!" if retry_count >= max_retries
print(wrote_dots ? "." : "Waiting on Bintray.")
wrote_dots = true
sleep poll_retry_delay_seconds
retry_count += 1
end
end
# Actual download and verification
# We do a retry on this, too, because sometimes the external curl will fail even
# when the prior HEAD has succeeded.
puts "\n" if wrote_dots
filename = File.basename(url.path)
curl_retry_delay_seconds = 4
max_curl_retries = 1
retry_count = 0
# We're in the cache; make sure to force re-download
loop do
begin
curl_download url, to: filename
break
rescue
raise "Failed to download #{f} bottle from #{url}!" if retry_count >= max_curl_retries
puts "curl download failed; retrying in #{curl_retry_delay_seconds} sec"
sleep curl_retry_delay_seconds
curl_retry_delay_seconds *= 2
retry_count += 1
end
end
checksum = Checksum.new(:sha256, bottle_info.sha256)
Pathname.new(filename).verify_checksum(checksum)
end
end
end
def check_bintray_mirror(name, url)
headers, = curl_output("--connect-timeout", "15", "--location", "--head", url)
status_code = headers.scan(%r{^HTTP\/.* (\d+)}).last.first

View File

@ -0,0 +1,66 @@
# frozen_string_literal: true
# Formula info drawn from an external `brew info --json` call
class FormulaInfo
# The whole info structure parsed from the JSON
attr_accessor :info
def initialize(info)
@info = info
end
# Looks up formula on disk and reads its info.
# Returns nil if formula is absent or if there was an error reading it.
def self.lookup(name)
json = Utils.popen_read(HOMEBREW_BREW_FILE, "info", "--json=v1", name)
return unless $CHILD_STATUS.success?
force_utf8!(json)
FormulaInfo.new(JSON.parse(json)[0])
end
def bottle_tags
return [] unless info["bottle"]["stable"]
info["bottle"]["stable"]["files"].keys
end
def bottle_info(my_bottle_tag = Utils::Bottles.tag)
tag_s = my_bottle_tag.to_s
return unless info["bottle"]["stable"]
btl_info = info["bottle"]["stable"]["files"][tag_s]
return unless btl_info
{ "url" => btl_info["url"], "sha256" => btl_info["sha256"] }
end
def bottle_info_any
bottle_info(any_bottle_tag)
end
def any_bottle_tag
tag = Utils::Bottles.tag.to_s
# Prefer native bottles as a convenience for download caching
bottle_tags.include?(tag) ? tag : bottle_tags.first
end
def version(spec_type)
version_str = info["versions"][spec_type.to_s]
version_str && Version.create(version_str)
end
def pkg_version(spec_type = :stable)
PkgVersion.new(version(spec_type), revision)
end
def revision
info["revision"]
end
def self.force_utf8!(str)
str.force_encoding("UTF-8") if str.respond_to?(:force_encoding)
end
end

View File

@ -0,0 +1,48 @@
# frozen_string_literal: true
require "bottle_publisher"
describe BottlePublisher do
subject(:bottle_publisher) {
described_class.new(
CoreTap.instance, ["#{CoreTap.instance.name}/hello.rb"], "homebrew", false, false
)
}
let(:tap) { CoreTap.new }
describe "publish_and_check_bottles" do
it "fails if HOMEBREW_DISABLE_LOAD_FORMULA is set to 1" do
ENV["HOMEBREW_DISABLE_LOAD_FORMULA"] = "1"
expect { bottle_publisher.publish_and_check_bottles }
.to raise_error("Need to load formulae to publish them!")
end
it "returns nil because HOMEBREW_BINTRAY_USER and HOMEBREW_BINTRAY_KEY are not set" do
ENV["HOMEBREW_BINTRAY_USER"] = nil
ENV["HOMEBREW_BINTRAY_KEY"] = nil
expect(bottle_publisher.publish_and_check_bottles)
.to eq nil
end
end
describe "verify_bintray_published" do
it "returns nil if no formula has been defined" do
expect(bottle_publisher.verify_bintray_published([]))
.to eq nil
end
it "fails if HOMEBREW_DISABLE_LOAD_FORMULA is set to 1" do
ENV["HOMEBREW_DISABLE_LOAD_FORMULA"] = "1"
stub_formula_loader(formula("foo") { url "foo-1.0" })
expect { bottle_publisher.verify_bintray_published(["foo"]) }
.to raise_error("Need to load formulae to verify their publication!")
end
it "checks if a bottle has been published" do
stub_formula_loader(formula("foo") { url "foo-1.0" })
expect { bottle_publisher.verify_bintray_published(["foo"]) }
.to output("Warning: Cannot publish bottle: Failed reading info for formula foo\n").to_stderr
end
end
end

View File

@ -0,0 +1,47 @@
# frozen_string_literal: true
require "formula_info"
require "global"
describe FormulaInfo, :integration_test do
it "tests the FormulaInfo class" do
install_test_formula "testball"
expect(
described_class.lookup(Formula["testball"].path)
.revision,
).to eq(0)
expect(
described_class.lookup(Formula["testball"].path)
.bottle_tags,
).to eq([])
expect(
described_class.lookup(Formula["testball"].path)
.bottle_info,
).to eq(nil)
expect(
described_class.lookup(Formula["testball"].path)
.bottle_info_any,
).to eq(nil)
expect(
described_class.lookup(Formula["testball"].path)
.any_bottle_tag,
).to eq(nil)
expect(
described_class.lookup(Formula["testball"].path)
.version(:stable).to_s,
).to eq("0.1")
version = described_class.lookup(Formula["testball"].path)
.version(:stable)
expect(
described_class.lookup(Formula["testball"].path)
.pkg_version,
).to eq(PkgVersion.new(version, 0))
end
end