From 3b0359706f2d08719b826c71f88c6232a84bc007 Mon Sep 17 00:00:00 2001 From: Dawid Dziurla Date: Mon, 8 Jun 2020 17:48:28 +0200 Subject: [PATCH] bintray: automatically run brew mirror if needed --- Library/Homebrew/bintray.rb | 32 +++++++++++++++++++++++++++++ Library/Homebrew/dev-cmd/mirror.rb | 31 +++------------------------- Library/Homebrew/dev-cmd/pr-pull.rb | 30 +++++++++++++++++++++++---- 3 files changed, 61 insertions(+), 32 deletions(-) diff --git a/Library/Homebrew/bintray.rb b/Library/Homebrew/bintray.rb index 8d4a0ea154..c9efcd47ca 100644 --- a/Library/Homebrew/bintray.rb +++ b/Library/Homebrew/bintray.rb @@ -56,6 +56,38 @@ class Bintray %w[homebrew linuxbrew].include? org end + def stable_mirrored?(url) + headers, = curl_output("--connect-timeout", "15", "--location", "--head", url) + status_code = headers.scan(%r{^HTTP/.* (\d+)}).last.first + status_code.start_with?("2") + end + + def mirror_formula(formula, repo: "mirror") + package = Utils::Bottles::Bintray.package formula.name + + create_package(repo: repo, package: package) unless package_exists?(repo: repo, package: package) + + formula.downloader.fetch + + version = ERB::Util.url_encode(formula.pkg_version) + filename = ERB::Util.url_encode(formula.downloader.basename) + destination_url = "https://dl.bintray.com/#{@bintray_org}/#{repo}/#{filename}" + + odebug "Uploading to #{destination_url}" + + upload( + formula.downloader.cached_location, + repo: repo, + package: package, + version: version, + sha256: formula.stable.checksum, + remote_file: filename, + ) + publish(repo: repo, package: package, version: version) + + destination_url + end + def create_package(repo:, package:, **extra_data_args) url = "#{API_URL}/packages/#{@bintray_org}/#{repo}" data = { name: package, public_download_numbers: true } diff --git a/Library/Homebrew/dev-cmd/mirror.rb b/Library/Homebrew/dev-cmd/mirror.rb index 9abb83302b..6ef150f922 100644 --- a/Library/Homebrew/dev-cmd/mirror.rb +++ b/Library/Homebrew/dev-cmd/mirror.rb @@ -26,37 +26,12 @@ module Homebrew mirror_args.parse bintray_org = args.bintray_org || "homebrew" - bintray_repo = "mirror" bintray = Bintray.new(org: bintray_org) - args.formulae.each do |f| - bintray_package = Utils::Bottles::Bintray.package f.name - - unless bintray.package_exists?(repo: bintray_repo, package: bintray_package) - bintray.create_package repo: bintray_repo, package: bintray_package - end - - downloader = f.downloader - - downloader.fetch - - filename = ERB::Util.url_encode(downloader.basename) - - destination_url = "https://dl.bintray.com/#{bintray_org}/#{bintray_repo}/#{filename}" - ohai "Uploading to #{destination_url}" - - version = ERB::Util.url_encode(f.pkg_version) - bintray.upload( - downloader.cached_location, - repo: bintray_repo, - package: bintray_package, - version: version, - sha256: f.stable.checksum, - remote_file: filename, - ) - bintray.publish(repo: bintray_repo, package: bintray_package, version: version) - ohai "Mirrored #{filename}!" + args.formulae.each do |formula| + mirror_url = bintray.mirror_formula(formula) + ohai "Mirrored #{formula.full_name} to #{mirror_url}!" end end end diff --git a/Library/Homebrew/dev-cmd/pr-pull.rb b/Library/Homebrew/dev-cmd/pr-pull.rb index 48495a2b5d..06b190dc56 100644 --- a/Library/Homebrew/dev-cmd/pr-pull.rb +++ b/Library/Homebrew/dev-cmd/pr-pull.rb @@ -204,18 +204,40 @@ module Homebrew url = GitHub.get_artifact_url(user, repo, pr, workflow_id: workflow, artifact_name: artifact) download_artifact(url, dir, pr) + json_files = Dir["*.json"] + if Homebrew.args.dry_run? - puts "brew bottle --merge --write #{Dir["*.json"].join " "}" + puts "brew bottle --merge --write #{json_files.join " "}" else - quiet_system "#{HOMEBREW_PREFIX}/bin/brew", "bottle", "--merge", "--write", *Dir["*.json"] + quiet_system "#{HOMEBREW_PREFIX}/bin/brew", "bottle", "--merge", "--write", *json_files end next if args.no_upload? if Homebrew.args.dry_run? - puts "Upload bottles described by these JSON files to Bintray:\n #{Dir["*.json"].join("\n ")}" + puts "Upload bottles described by these JSON files to Bintray:\n #{json_files.join("\n ")}" else - bintray.upload_bottle_json Dir["*.json"], publish_package: !args.no_publish? + bintray.upload_bottle_json json_files, publish_package: !args.no_publish? + end + + bottles_hash = json_files.reduce({}) do |hash, json_file| + hash.deep_merge(JSON.parse(IO.read(json_file))) + end + + bottles_hash.each do |formula_name, _| + formula = Formula[formula_name] + stable_urls = [formula.stable.url] + formula.stable.mirrors + stable_urls.grep(%r{^https://dl.bintray.com/homebrew/mirror/}) do |mirror_url| + if Homebrew.args.dry_run? + puts "Mirror formulae sources described by these JSON files to Bintray:\n #{json_files.join("\n ")}" + next + end + + next if bintray.stable_mirrored?(mirror_url) + + mirror_url = bintray.mirror_formula(formula) + ohai "Mirrored #{formula.full_name} to #{mirror_url}!" + end end end end