diff --git a/Library/Homebrew/bottle_publisher.rb b/Library/Homebrew/bottle_publisher.rb new file mode 100644 index 0000000000..a6c34a3801 --- /dev/null +++ b/Library/Homebrew/bottle_publisher.rb @@ -0,0 +1,162 @@ +# frozen_string_literal: true + +require "utils" +require "formula_info" + +class BottlePublisher + def initialize(tap, changed_formulae_names, bintray_org, no_publish, warn_on_publish_failure) + @tap = tap + @changed_formulae_names = changed_formulae_names + @no_publish = no_publish + @bintray_org = bintray_org + @warn_on_publish_failure = warn_on_publish_failure + end + + def publish_and_check_bottles + # Formulae with affected bottles that were published + bintray_published_formulae = [] + + # Publish bottles on Bintray + unless @no_publish + published = publish_changed_formula_bottles + bintray_published_formulae.concat(published) + end + + # Verify bintray publishing after all patches have been applied + bintray_published_formulae.uniq! + verify_bintray_published(bintray_published_formulae) + end + + def publish_changed_formula_bottles + raise "Need to load formulae to publish them!" if ENV["HOMEBREW_DISABLE_LOAD_FORMULA"] + + published = [] + bintray_creds = { user: ENV["HOMEBREW_BINTRAY_USER"], key: ENV["HOMEBREW_BINTRAY_KEY"] } + if bintray_creds[:user] && bintray_creds[:key] + @changed_formulae_names.each do |name| + f = Formula[name] + next if f.bottle_unneeded? || f.bottle_disabled? + + bintray_org = @bintray_org || @tap.user.downcase + next unless publish_bottle_file_on_bintray(f, bintray_org, bintray_creds) + + published << f.full_name + end + else + opoo "You must set HOMEBREW_BINTRAY_USER and HOMEBREW_BINTRAY_KEY to add or update bottles on Bintray!" + end + published + end + + # Publishes the current bottle files for a given formula to Bintray + def publish_bottle_file_on_bintray(f, bintray_org, creds) + repo = Utils::Bottles::Bintray.repository(f.tap) + package = Utils::Bottles::Bintray.package(f.name) + info = FormulaInfo.lookup(f.full_name) + raise "Failed publishing bottle: failed reading formula info for #{f.full_name}" if info.nil? + + unless info.bottle_info_any + opoo "No bottle defined in formula #{package}" + return false + end + version = info.pkg_version + ohai "Publishing on Bintray: #{package} #{version}" + curl "--write-out", '\n', "--silent", "--fail", + "--user", "#{creds[:user]}:#{creds[:key]}", "--request", "POST", + "--header", "Content-Type: application/json", + "--data", '{"publish_wait_for_secs": 0}', + "https://api.bintray.com/content/#{bintray_org}/#{repo}/#{package}/#{version}/publish" + true + rescue => e + raise unless @warn_on_publish_failure + + onoe e + false + end + + # Verifies that formulae have been published on Bintray by downloading a bottle file + # for each one. Blocks until the published files are available. + # Raises an error if the verification fails. + # This does not currently work for `brew pull`, because it may have cached the old + # version of a formula. + def verify_bintray_published(formulae_names) + return if formulae_names.empty? + + raise "Need to load formulae to verify their publication!" if ENV["HOMEBREW_DISABLE_LOAD_FORMULA"] + + ohai "Verifying bottles published on Bintray" + formulae = formulae_names.map { |n| Formula[n] } + max_retries = 300 # shared among all bottles + poll_retry_delay_seconds = 2 + + HOMEBREW_CACHE.cd do + formulae.each do |f| + retry_count = 0 + wrote_dots = false + # Choose arbitrary bottle just to get the host/port for Bintray right + jinfo = FormulaInfo.lookup(f.full_name) + unless jinfo + opoo "Cannot publish bottle: Failed reading info for formula #{f.full_name}" + next + end + bottle_info = jinfo.bottle_info_any + unless bottle_info + opoo "No bottle defined in formula #{f.full_name}" + next + end + + # Poll for publication completion using a quick partial HEAD, to avoid spurious error messages + # 401 error is normal while file is still in async publishing process + url = URI(bottle_info.url) + puts "Verifying bottle: #{File.basename(url.path)}" + http = Net::HTTP.new(url.host, url.port) + http.use_ssl = true + retry_count = 0 + http.start do + loop do + req = Net::HTTP::Head.new bottle_info.url + req.initialize_http_header "User-Agent" => HOMEBREW_USER_AGENT_RUBY + res = http.request req + break if res.is_a?(Net::HTTPSuccess) || res.code == "302" + + unless res.is_a?(Net::HTTPClientError) + raise "Failed to find published #{f} bottle at #{url} (#{res.code} #{res.message})!" + end + + raise "Failed to find published #{f} bottle at #{url}!" if retry_count >= max_retries + + print(wrote_dots ? "." : "Waiting on Bintray.") + wrote_dots = true + sleep poll_retry_delay_seconds + retry_count += 1 + end + end + + # Actual download and verification + # We do a retry on this, too, because sometimes the external curl will fail even + # when the prior HEAD has succeeded. + puts "\n" if wrote_dots + filename = File.basename(url.path) + curl_retry_delay_seconds = 4 + max_curl_retries = 1 + retry_count = 0 + # We're in the cache; make sure to force re-download + loop do + begin + curl_download url, to: filename + break + rescue + raise "Failed to download #{f} bottle from #{url}!" if retry_count >= max_curl_retries + + puts "curl download failed; retrying in #{curl_retry_delay_seconds} sec" + sleep curl_retry_delay_seconds + curl_retry_delay_seconds *= 2 + retry_count += 1 + end + end + checksum = Checksum.new(:sha256, bottle_info.sha256) + Pathname.new(filename).verify_checksum(checksum) + end + end + end +end diff --git a/Library/Homebrew/dev-cmd/pull.rb b/Library/Homebrew/dev-cmd/pull.rb index b686a1f9dc..aa316f11aa 100644 --- a/Library/Homebrew/dev-cmd/pull.rb +++ b/Library/Homebrew/dev-cmd/pull.rb @@ -8,6 +8,8 @@ require "formula" require "formulary" require "version" require "pkg_version" +require "bottle_publisher" +require "formula_info" module GitHub module_function @@ -253,7 +255,13 @@ module Homebrew else fetch_bottles_patch(bottle_commit_url, args, bottle_branch, branch, orig_revision) end - publish_and_check_bottles(tap, args, changed_formulae_names) + BottlePublisher.new( + tap, + changed_formulae_names, + args.bintray_org, + args.no_publish?, + args.warn_on_publish_failure?, + ).publish_and_check_bottles elsif merge_commit fetch_merge_patch(url, args, issue) end @@ -301,46 +309,6 @@ module Homebrew PatchPuller.new(url, args, "merge commit").pull_merge_commit(issue) end - def publish_and_check_bottles(tap, args, changed_formulae_names) - # Formulae with affected bottles that were published - bintray_published_formulae = [] - - # Publish bottles on Bintray - unless args.no_publish? - published = publish_changed_formula_bottles(tap, changed_formulae_names) - bintray_published_formulae.concat(published) - end - - # Verify bintray publishing after all patches have been applied - bintray_published_formulae.uniq! - verify_bintray_published(bintray_published_formulae) - end - - def force_utf8!(str) - str.force_encoding("UTF-8") if str.respond_to?(:force_encoding) - end - - def publish_changed_formula_bottles(tap, changed_formulae_names) - raise "Need to load formulae to publish them!" if ENV["HOMEBREW_DISABLE_LOAD_FORMULA"] - - published = [] - bintray_creds = { user: ENV["HOMEBREW_BINTRAY_USER"], key: ENV["HOMEBREW_BINTRAY_KEY"] } - if bintray_creds[:user] && bintray_creds[:key] - changed_formulae_names.each do |name| - f = Formula[name] - next if f.bottle_unneeded? || f.bottle_disabled? - - bintray_org = args.bintray_org || tap.user.downcase - next unless publish_bottle_file_on_bintray(f, bintray_org, bintray_creds) - - published << f.full_name - end - else - opoo "You must set HOMEBREW_BINTRAY_USER and HOMEBREW_BINTRAY_KEY to add or update bottles on Bintray!" - end - published - end - class PatchPuller attr_reader :base_url attr_reader :patch_url @@ -440,7 +408,7 @@ module Homebrew # Returns info as a hash (type => version), for pull.rb's internal use. # Uses special key `:nonexistent => true` for nonexistent formulae. def current_versions_from_info_external(formula_name) - info = FormulaInfoFromJson.lookup(formula_name) + info = FormulaInfo.lookup(formula_name) versions = {} if info [:stable, :devel, :head].each do |spec_type| @@ -495,191 +463,6 @@ module Homebrew Utils.popen_write("pbcopy") { |io| io.write text } end - # Publishes the current bottle files for a given formula to Bintray - def publish_bottle_file_on_bintray(f, bintray_org, creds) - repo = Utils::Bottles::Bintray.repository(f.tap) - package = Utils::Bottles::Bintray.package(f.name) - info = FormulaInfoFromJson.lookup(f.full_name) - raise "Failed publishing bottle: failed reading formula info for #{f.full_name}" if info.nil? - - unless info.bottle_info_any - opoo "No bottle defined in formula #{package}" - return false - end - version = info.pkg_version - ohai "Publishing on Bintray: #{package} #{version}" - curl "--write-out", '\n', "--silent", "--fail", - "--user", "#{creds[:user]}:#{creds[:key]}", "--request", "POST", - "--header", "Content-Type: application/json", - "--data", '{"publish_wait_for_secs": 0}', - "https://api.bintray.com/content/#{bintray_org}/#{repo}/#{package}/#{version}/publish" - true - rescue => e - raise unless @args.warn_on_publish_failure? - - onoe e - false - end - - # Formula info drawn from an external `brew info --json` call - class FormulaInfoFromJson - # The whole info structure parsed from the JSON - attr_accessor :info - - def initialize(info) - @info = info - end - - # Looks up formula on disk and reads its info. - # Returns nil if formula is absent or if there was an error reading it. - def self.lookup(name) - json = Utils.popen_read(HOMEBREW_BREW_FILE, "info", "--json=v1", name) - - return unless $CHILD_STATUS.success? - - Homebrew.force_utf8!(json) - FormulaInfoFromJson.new(JSON.parse(json)[0]) - end - - def bottle_tags - return [] unless info["bottle"]["stable"] - - info["bottle"]["stable"]["files"].keys - end - - def bottle_info(my_bottle_tag = Utils::Bottles.tag) - tag_s = my_bottle_tag.to_s - return unless info["bottle"]["stable"] - - btl_info = info["bottle"]["stable"]["files"][tag_s] - return unless btl_info - - BottleInfo.new(btl_info["url"], btl_info["sha256"]) - end - - def bottle_info_any - bottle_info(any_bottle_tag) - end - - def any_bottle_tag - tag = Utils::Bottles.tag.to_s - # Prefer native bottles as a convenience for download caching - bottle_tags.include?(tag) ? tag : bottle_tags.first - end - - def version(spec_type) - version_str = info["versions"][spec_type.to_s] - version_str && Version.create(version_str) - end - - def pkg_version(spec_type = :stable) - PkgVersion.new(version(spec_type), revision) - end - - def revision - info["revision"] - end - end - - # Bottle info as used internally by pull, with alternate platform support. - class BottleInfo - # URL of bottle as string - attr_accessor :url - # Expected SHA-256 as string - attr_accessor :sha256 - - def initialize(url, sha256) - @url = url - @sha256 = sha256 - end - end - - # Verifies that formulae have been published on Bintray by downloading a bottle file - # for each one. Blocks until the published files are available. - # Raises an error if the verification fails. - # This does not currently work for `brew pull`, because it may have cached the old - # version of a formula. - def verify_bintray_published(formulae_names) - return if formulae_names.empty? - - raise "Need to load formulae to verify their publication!" if ENV["HOMEBREW_DISABLE_LOAD_FORMULA"] - - ohai "Verifying bottles published on Bintray" - formulae = formulae_names.map { |n| Formula[n] } - max_retries = 300 # shared among all bottles - poll_retry_delay_seconds = 2 - - HOMEBREW_CACHE.cd do - formulae.each do |f| - retry_count = 0 - wrote_dots = false - # Choose arbitrary bottle just to get the host/port for Bintray right - jinfo = FormulaInfoFromJson.lookup(f.full_name) - unless jinfo - opoo "Cannot publish bottle: Failed reading info for formula #{f.full_name}" - next - end - bottle_info = jinfo.bottle_info_any - unless bottle_info - opoo "No bottle defined in formula #{f.full_name}" - next - end - - # Poll for publication completion using a quick partial HEAD, to avoid spurious error messages - # 401 error is normal while file is still in async publishing process - url = URI(bottle_info.url) - puts "Verifying bottle: #{File.basename(url.path)}" - http = Net::HTTP.new(url.host, url.port) - http.use_ssl = true - retry_count = 0 - http.start do - loop do - req = Net::HTTP::Head.new bottle_info.url - req.initialize_http_header "User-Agent" => HOMEBREW_USER_AGENT_RUBY - res = http.request req - break if res.is_a?(Net::HTTPSuccess) || res.code == "302" - - unless res.is_a?(Net::HTTPClientError) - raise "Failed to find published #{f} bottle at #{url} (#{res.code} #{res.message})!" - end - - raise "Failed to find published #{f} bottle at #{url}!" if retry_count >= max_retries - - print(wrote_dots ? "." : "Waiting on Bintray.") - wrote_dots = true - sleep poll_retry_delay_seconds - retry_count += 1 - end - end - - # Actual download and verification - # We do a retry on this, too, because sometimes the external curl will fail even - # when the prior HEAD has succeeded. - puts "\n" if wrote_dots - filename = File.basename(url.path) - curl_retry_delay_seconds = 4 - max_curl_retries = 1 - retry_count = 0 - # We're in the cache; make sure to force re-download - loop do - begin - curl_download url, to: filename - break - rescue - raise "Failed to download #{f} bottle from #{url}!" if retry_count >= max_curl_retries - - puts "curl download failed; retrying in #{curl_retry_delay_seconds} sec" - sleep curl_retry_delay_seconds - curl_retry_delay_seconds *= 2 - retry_count += 1 - end - end - checksum = Checksum.new(:sha256, bottle_info.sha256) - Pathname.new(filename).verify_checksum(checksum) - end - end - end - def check_bintray_mirror(name, url) headers, = curl_output("--connect-timeout", "15", "--location", "--head", url) status_code = headers.scan(%r{^HTTP\/.* (\d+)}).last.first diff --git a/Library/Homebrew/formula_info.rb b/Library/Homebrew/formula_info.rb new file mode 100644 index 0000000000..42a81bf8a5 --- /dev/null +++ b/Library/Homebrew/formula_info.rb @@ -0,0 +1,66 @@ +# frozen_string_literal: true + +# Formula info drawn from an external `brew info --json` call + +class FormulaInfo + # The whole info structure parsed from the JSON + attr_accessor :info + + def initialize(info) + @info = info + end + + # Looks up formula on disk and reads its info. + # Returns nil if formula is absent or if there was an error reading it. + def self.lookup(name) + json = Utils.popen_read(HOMEBREW_BREW_FILE, "info", "--json=v1", name) + + return unless $CHILD_STATUS.success? + + force_utf8!(json) + FormulaInfo.new(JSON.parse(json)[0]) + end + + def bottle_tags + return [] unless info["bottle"]["stable"] + + info["bottle"]["stable"]["files"].keys + end + + def bottle_info(my_bottle_tag = Utils::Bottles.tag) + tag_s = my_bottle_tag.to_s + return unless info["bottle"]["stable"] + + btl_info = info["bottle"]["stable"]["files"][tag_s] + return unless btl_info + + { "url" => btl_info["url"], "sha256" => btl_info["sha256"] } + end + + def bottle_info_any + bottle_info(any_bottle_tag) + end + + def any_bottle_tag + tag = Utils::Bottles.tag.to_s + # Prefer native bottles as a convenience for download caching + bottle_tags.include?(tag) ? tag : bottle_tags.first + end + + def version(spec_type) + version_str = info["versions"][spec_type.to_s] + version_str && Version.create(version_str) + end + + def pkg_version(spec_type = :stable) + PkgVersion.new(version(spec_type), revision) + end + + def revision + info["revision"] + end + + def self.force_utf8!(str) + str.force_encoding("UTF-8") if str.respond_to?(:force_encoding) + end +end diff --git a/Library/Homebrew/test/bottle_publisher_spec.rb b/Library/Homebrew/test/bottle_publisher_spec.rb new file mode 100644 index 0000000000..ebcea24638 --- /dev/null +++ b/Library/Homebrew/test/bottle_publisher_spec.rb @@ -0,0 +1,48 @@ +# frozen_string_literal: true + +require "bottle_publisher" + +describe BottlePublisher do + subject(:bottle_publisher) { + described_class.new( + CoreTap.instance, ["#{CoreTap.instance.name}/hello.rb"], "homebrew", false, false + ) + } + + let(:tap) { CoreTap.new } + + describe "publish_and_check_bottles" do + it "fails if HOMEBREW_DISABLE_LOAD_FORMULA is set to 1" do + ENV["HOMEBREW_DISABLE_LOAD_FORMULA"] = "1" + expect { bottle_publisher.publish_and_check_bottles } + .to raise_error("Need to load formulae to publish them!") + end + + it "returns nil because HOMEBREW_BINTRAY_USER and HOMEBREW_BINTRAY_KEY are not set" do + ENV["HOMEBREW_BINTRAY_USER"] = nil + ENV["HOMEBREW_BINTRAY_KEY"] = nil + expect(bottle_publisher.publish_and_check_bottles) + .to eq nil + end + end + + describe "verify_bintray_published" do + it "returns nil if no formula has been defined" do + expect(bottle_publisher.verify_bintray_published([])) + .to eq nil + end + + it "fails if HOMEBREW_DISABLE_LOAD_FORMULA is set to 1" do + ENV["HOMEBREW_DISABLE_LOAD_FORMULA"] = "1" + stub_formula_loader(formula("foo") { url "foo-1.0" }) + expect { bottle_publisher.verify_bintray_published(["foo"]) } + .to raise_error("Need to load formulae to verify their publication!") + end + + it "checks if a bottle has been published" do + stub_formula_loader(formula("foo") { url "foo-1.0" }) + expect { bottle_publisher.verify_bintray_published(["foo"]) } + .to output("Warning: Cannot publish bottle: Failed reading info for formula foo\n").to_stderr + end + end +end diff --git a/Library/Homebrew/test/formula_info_spec.rb b/Library/Homebrew/test/formula_info_spec.rb new file mode 100644 index 0000000000..c936dbef76 --- /dev/null +++ b/Library/Homebrew/test/formula_info_spec.rb @@ -0,0 +1,47 @@ +# frozen_string_literal: true + +require "formula_info" +require "global" + +describe FormulaInfo, :integration_test do + it "tests the FormulaInfo class" do + install_test_formula "testball" + + expect( + described_class.lookup(Formula["testball"].path) + .revision, + ).to eq(0) + + expect( + described_class.lookup(Formula["testball"].path) + .bottle_tags, + ).to eq([]) + + expect( + described_class.lookup(Formula["testball"].path) + .bottle_info, + ).to eq(nil) + + expect( + described_class.lookup(Formula["testball"].path) + .bottle_info_any, + ).to eq(nil) + + expect( + described_class.lookup(Formula["testball"].path) + .any_bottle_tag, + ).to eq(nil) + + expect( + described_class.lookup(Formula["testball"].path) + .version(:stable).to_s, + ).to eq("0.1") + + version = described_class.lookup(Formula["testball"].path) + .version(:stable) + expect( + described_class.lookup(Formula["testball"].path) + .pkg_version, + ).to eq(PkgVersion.new(version, 0)) + end +end