Merge pull request #7236 from jonchang/pr-pull

Adds a new `pr-pull` command.

Library/Homebrew/bintray.rb

@@ -0,0 +1,127 @@
+# frozen_string_literal: true
+
+require "utils/curl"
+require "json"
+
+class Bintray
+  API_URL = "https://api.bintray.com"
+
+  class Error < RuntimeError
+  end
+
+  def inspect
+    "#<Bintray: user=#{@bintray_user} org=#{@bintray_org} key=***>"
+  end
+
+  def initialize(user: ENV["HOMEBREW_BINTRAY_USER"], key: ENV["HOMEBREW_BINTRAY_KEY"], org: "homebrew", clear: true)
+    @bintray_user = user
+    @bintray_key = key
+    @bintray_org = org
+
+    if !@bintray_user || !@bintray_key
+      unless Homebrew.args.dry_run?
+        raise UsageError, "Missing HOMEBREW_BINTRAY_USER or HOMEBREW_BINTRAY_KEY variables!"
+      end
+    end
+
+    ENV["HOMEBREW_FORCE_HOMEBREW_ON_LINUX"] = "1" if @bintray_org == "homebrew" && !OS.mac?
+
+    ENV.clear_sensitive_environment! if clear
+  end
+
+  def open_api(url, *extra_curl_args, auth: true)
+    args = extra_curl_args
+    args += ["--user", "#{@bintray_user}:#{@bintray_key}"] if auth
+    curl(*args, url,
+         show_output: Homebrew.args.verbose?,
+         secrets:     @bintray_key)
+  end
+
+  def upload(local_file, repo:, package:, version:, remote_file:, sha256: nil)
+    url = "#{API_URL}/content/#{@bintray_org}/#{repo}/#{package}/#{version}/#{remote_file}"
+    args = ["--upload-file", local_file]
+    args += ["--header", "X-Checksum-Sha2: #{sha256}"] unless sha256.blank?
+    open_api url, *args
+  end
+
+  def publish(repo:, package:, version:)
+    url = "#{API_URL}/content/#{@bintray_org}/#{repo}/#{package}/#{version}/publish"
+    open_api url, "--request", "POST"
+  end
+
+  def official_org?(org: @bintray_org)
+    %w[homebrew linuxbrew].include? org
+  end
+
+  def create_package(repo:, package:, **extra_data_args)
+    url = "#{API_URL}/packages/#{@bintray_org}/#{repo}/#{package}"
+    data = { name: package, public_download_numbers: true }
+    data[:public_stats] = official_org?
+    data.merge! extra_data_args
+    open_api url, "--request", "POST", "--data", data.to_json
+  end
+
+  def package_exists?(repo:, package:)
+    url = "#{API_URL}/packages/#{@bintray_org}/#{repo}/#{package}"
+    open_api url, "--output", "/dev/null", auth: false
+  end
+
+  def file_published?(repo:, remote_file:)
+    url = "https://dl.bintray.com/#{@bintray_org}/#{repo}/#{remote_file}"
+    begin
+      curl "--silent", "--head", "--output", "/dev/null", url
+    rescue ErrorDuringExecution => e
+      stderr = e.output.select { |type,| type == :stderr }
+                .map { |_, line| line }
+                .join
+      raise if e.status.exitstatus != 22 && !stderr.include?("404 Not Found")
+
+      false
+    else
+      true
+    end
+  end
+
+  def upload_bottle_json(json_files, publish_package: false)
+    bottles_hash = json_files.reduce({}) do |hash, json_file|
+      hash.deep_merge(JSON.parse(IO.read(json_file)))
+    end
+
+    formula_packaged = {}
+
+    bottles_hash.each do |formula_name, bottle_hash|
+      version = bottle_hash["formula"]["pkg_version"]
+      bintray_package = bottle_hash["bintray"]["package"]
+      bintray_repo = bottle_hash["bintray"]["repository"]
+
+      bottle_hash["bottle"]["tags"].each do |_tag, tag_hash|
+        filename = tag_hash["filename"]
+        sha256 = tag_hash["sha256"]
+
+        if file_published? repo: bintray_repo, remote_file: filename
+          raise Error, <<~EOS
+            #{filename} is already published.
+            Please remove it manually from:
+              https://bintray.com/#{@bintray_org}/#{bintray_repo}/#{bintray_package}/view#files
+            Or run:
+              curl -X DELETE -u $HOMEBREW_BINTRAY_USER:$HOMEBREW_BINTRAY_KEY \\
+              https://api.bintray.com/content/#{@bintray_org}/#{bintray_repo}/#{filename}
+          EOS
+        end
+
+        if !formula_packaged[formula_name] && !package_exists?(repo: bintray_repo, package: bintray_package)
+          create_package repo: bintray_repo, package: bintray_package
+          formula_packaged[formula_name] = true
+        end
+
+        upload(tag_hash["local_filename"],
+               repo:        bintray_repo,
+               package:     bintray_package,
+               version:     version,
+               remote_file: filename,
+               sha256:      sha256)
+      end
+      publish repo: bintray_repo, package: bintray_package, version: version if publish_package
+    end
+  end
+end

Library/Homebrew/dev-cmd/pr-pull.rb

@@ -0,0 +1,143 @@
+# frozen_string_literal: true
+
+require "cli/parser"
+require "utils/github"
+require "tmpdir"
+require "bintray"
+
+module Homebrew
+  module_function
+
+  def pr_pull_args
+    Homebrew::CLI::Parser.new do
+      usage_banner <<~EOS
+        `pr-pull` <pull_request>
+
+        Download and publish bottles, and apply the bottle commit from a
+        pull request with artifacts generated from GitHub Actions.
+        Requires write access to the repository.
+      EOS
+      switch "--no-publish",
+             description: "Download the bottles, apply the bottle commit, and "\
+                          "upload the bottles to Bintray, but don't publish them."
+      switch "--no-upload",
+             description: "Download the bottles and apply the bottle commit, "\
+                          "but don't upload to Bintray."
+      switch "--dry-run", "-n",
+             description: "Print what would be done rather than doing it."
+      switch "--clean",
+             description: "Do not amend the commits from pull requests."
+      switch "--branch-okay",
+             description: "Do not warn if pulling to a branch besides master (useful for testing)."
+      switch "--resolve",
+             description: "When a patch fails to apply, leave in progress and allow user to resolve, instead "\
+                          "of aborting."
+      flag "--workflow=",
+           description: "Retrieve artifacts from the specified workflow (default: tests.yml)."
+      flag "--artifact=",
+           description: "Download artifacts with the specified name (default: bottles)."
+      flag "--bintray-org=",
+           description: "Upload to the specified Bintray organisation."
+      switch :verbose
+      switch :debug
+      min_named 1
+    end
+  end
+
+  def signoff!(pr, path: ".", dry_run: false)
+    message = Utils.popen_read "git", "-C", path, "log", "-1", "--pretty=%B"
+    close_message = "Closes ##{pr}."
+    message += "\n#{close_message}" unless message.include? close_message
+    if dry_run
+      puts "git commit --amend --signoff -m $message"
+    else
+      safe_system "git", "-C", path, "commit", "--amend", "--signoff", "--allow-empty", "-q", "-m", message
+    end
+  end
+
+  def cherry_pick_pr!(pr, path: ".", dry_run: false)
+    if dry_run
+      puts <<~EOS
+        git fetch --force origin +refs/pull/#{pr}/head
+        git merge-base HEAD FETCH_HEAD
+        git cherry-pick --ff --allow-empty $merge_base..FETCH_HEAD
+      EOS
+    else
+      safe_system "git", "-C", path, "fetch", "--quiet", "--force", "origin", "+refs/pull/#{pr}/head"
+      merge_base = Utils.popen_read("git", "-C", path, "merge-base", "HEAD", "FETCH_HEAD").strip
+      commit_count = Utils.popen_read("git", "-C", path, "rev-list", "#{merge_base}..FETCH_HEAD").lines.count
+
+      # git cherry-pick unfortunately has no quiet option
+      ohai "Cherry-picking #{commit_count} commit#{"s" unless commit_count == 1} from ##{pr}"
+      cherry_pick_args = "git", "-C", path, "cherry-pick", "--ff", "--allow-empty", "#{merge_base}..FETCH_HEAD"
+      result = Homebrew.args.verbose? ? system(*cherry_pick_args) : quiet_system(*cherry_pick_args)
+
+      unless result
+        if Homebrew.args.resolve?
+          odie "Cherry-pick failed: try to resolve it."
+        else
+          system "git", "-C", path, "cherry-pick", "--abort"
+          odie "Cherry-pick failed!"
+        end
+      end
+    end
+  end
+
+  def check_branch(path, ref)
+    branch = Utils.popen_read("git", "-C", path, "symbolic-ref", "--short", "HEAD").strip
+
+    return if branch == ref || args.clean? || args.branch_okay?
+
+    opoo "Current branch is #{branch}: do you need to pull inside #{ref}?"
+  end
+
+  def pr_pull
+    pr_pull_args.parse
+
+    bintray_user = ENV["HOMEBREW_BINTRAY_USER"]
+    bintray_key = ENV["HOMEBREW_BINTRAY_KEY"]
+    ENV.clear_sensitive_environment!
+
+    if bintray_user.blank? || bintray_key.blank?
+      odie "Missing HOMEBREW_BINTRAY_USER or HOMEBREW_BINTRAY_KEY variables!" if !args.dry_run? && !args.no_upload?
+    else
+      bintray = Bintray.new(user: bintray_user, key: bintray_key, org: args.bintray_org)
+    end
+
+    workflow = args.workflow || "tests.yml"
+    artifact = args.artifact || "bottles"
+
+    args.named.each do |arg|
+      arg = "#{CoreTap.instance.default_remote}/pull/#{arg}" if arg.to_i.positive?
+      url_match = arg.match HOMEBREW_PULL_OR_COMMIT_URL_REGEX
+      _, user, repo, pr = *url_match
+      tap = Tap.fetch(user, repo) if repo.match?(HOMEBREW_OFFICIAL_REPO_PREFIXES_REGEX)
+      odie "Not a GitHub pull request: #{arg}" unless pr
+
+      check_branch tap.path, "master"
+
+      ohai "Fetching #{tap} pull request ##{pr}"
+      Dir.mktmpdir pr do |dir|
+        cd dir do
+          GitHub.fetch_artifact(user, repo, pr, dir, workflow_id: workflow, artifact_name: artifact)
+          cherry_pick_pr! pr, path: tap.path, dry_run: args.dry_run?
+          signoff! pr, path: tap.path, dry_run: args.dry_run? unless args.clean?
+
+          if args.dry_run?
+            puts "brew bottle --merge --write #{Dir["*.json"].join " "}"
+          else
+            quiet_system "#{HOMEBREW_PREFIX}/bin/brew", "bottle", "--merge", "--write", *Dir["*.json"]
+          end
+
+          next if args.no_upload?
+
+          if args.dry_run?
+            puts "Upload bottles described by these JSON files to Bintray:\n  #{Dir["*.json"].join("\n  ")}"
+          else
+            bintray.upload_bottle_json Dir["*.json"], publish_package: !args.no_publish?
+          end
+        end
+      end
+    end
+  end
+end

Library/Homebrew/test/bintray_spec.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+require "bintray"
+
+describe Bintray, :needs_network do
+  bintray = described_class.new(user: "BrewTestBot", key: "deadbeef", org: "homebrew")
+  describe "::file_published?" do
+    it "detects a published file" do
+      results = bintray.file_published?(repo: "bottles", remote_file: "hello-2.10.catalina.bottle.tar.gz")
+      expect(results).to be true
+    end
+
+    it "fails on a non-existant file" do
+      results = bintray.file_published?(repo: "bottles", remote_file: "my-fake-bottle-1.0.snow_hyena.tar.gz")
+      expect(results).to be false
+    end
+  end
+
+  describe "::package_exists?" do
+    it "detects a package" do
+      results = bintray.package_exists?(repo: "bottles", package: "hello")
+      expect(results.status.exitstatus).to be 0
+    end
+  end
+end

Library/Homebrew/test/dev-cmd/pr-pull_spec.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+require "cmd/shared_examples/args_parse"
+
+describe "Homebrew.pr_pull_args" do
+  it_behaves_like "parseable arguments"
+end

Library/Homebrew/test/utils/github_spec.rb

@@ -41,4 +41,18 @@ describe GitHub do
       expect(results.first["title"]).to eq("Shall we run `brew update` automatically?")
     end
   end
+
+  describe "::fetch_artifact", :needs_network do
+    it "fails to find a nonexistant workflow" do
+      expect {
+        subject.fetch_artifact("Homebrew", "homebrew-core", 1, ".")
+      }.to raise_error(/No matching workflow run found/)
+    end
+
+    it "fails to find artifacts that don't exist" do
+      expect {
+        subject.fetch_artifact("Homebrew", "homebrew-core", 51971, ".", artifact_name: "false_bottles")
+      }.to raise_error(/No artifact .+ was found/)
+    end
+  end
 end

Library/Homebrew/utils/github.rb

@@ -1,7 +1,8 @@
 # frozen_string_literal: true
 
-require "uri"
+require "download_strategy"
 require "tempfile"
+require "uri"
 
 module GitHub
   module_function
@@ -438,6 +439,70 @@ module GitHub
                   scopes:         CREATE_ISSUE_FORK_OR_PR_SCOPES)
   end
 
+  def fetch_artifact(user, repo, pr, dir, workflow_id: "tests.yml", artifact_name: "bottles")
+    base_url = "#{API_URL}/repos/#{user}/#{repo}"
+    pr_payload = open_api("#{base_url}/pulls/#{pr}")
+    pr_sha = pr_payload["head"]["sha"]
+    pr_branch = pr_payload["head"]["ref"]
+
+    workflow = open_api("#{base_url}/actions/workflows/#{workflow_id}/runs?branch=#{pr_branch}")
+    workflow_run = workflow["workflow_runs"].select do |run|
+      run["head_sha"] == pr_sha
+    end
+
+    if workflow_run.empty?
+      raise Error, <<~EOS
+        No matching workflow run found for these criteria!
+          Commit SHA:   #{pr_sha}
+          Branch ref:   #{pr_branch}
+          Pull request: #{pr}
+          Workflow:     #{workflow_id}
+      EOS
+    end
+
+    status = workflow_run.first["status"].sub("_", " ")
+    if status != "completed"
+      raise Error, <<~EOS
+        The newest workflow run for ##{pr} is still #{status}!
+          #{Formatter.url workflow_run.first["html_url"]}
+      EOS
+    end
+
+    artifacts = open_api(workflow_run.first["artifacts_url"])
+
+    artifact = artifacts["artifacts"].select do |art|
+      art["name"] == artifact_name
+    end
+
+    if artifact.empty?
+      raise Error, <<~EOS
+        No artifact with the name `#{artifact_name}` was found!
+          #{Formatter.url workflow_run.first["html_url"]}
+      EOS
+    end
+
+    artifact_url = artifact.first["archive_download_url"]
+
+    token, username = api_credentials
+    case api_credentials_type
+    when :env_username_password, :keychain_username_password
+      curl_args = { user: "#{username}:#{token}" }
+    when :env_token
+      curl_args = { header: "Authorization: token #{token}" }
+    end
+
+    # Download the artifact as a zip file and unpack it into `dir`. This is
+    # preferred over system `curl` and `tar` as this leverages the Homebrew
+    # cache to avoid repeated downloads of (possibly large) bottles.
+    FileUtils.chdir dir do
+      curl_args[:cache] = Pathname.new(dir)
+      curl_args[:secrets] = [token]
+      downloader = CurlDownloadStrategy.new(artifact_url, "artifact", pr, **curl_args)
+      downloader.fetch
+      downloader.stage
+    end
+  end
+
   def api_errors
     [GitHub::AuthenticationFailedError, GitHub::HTTPNotFoundError,
      GitHub::RateLimitExceededError, GitHub::Error, JSON::ParserError].freeze

docs/Manpage.md

@@ -887,6 +887,31 @@ Generate Homebrew's manpages.
 Publishes bottles for a pull request with GitHub Actions. Requires write access
 to the repository.
 
+### `pr-pull` *`pull_request`*
+
+Download and publish bottles, and apply the bottle commit from a pull request
+with artifacts generated from GitHub Actions. Requires write access to the
+repository.
+
+* `--no-publish`:
+  Download the bottles, apply the bottle commit, and upload the bottles to Bintray, but don't publish them.
+* `--no-upload`:
+  Download the bottles and apply the bottle commit, but don't upload to Bintray.
+* `-n`, `--dry-run`:
+  Print what would be done rather than doing it.
+* `--clean`:
+  Do not amend the commits from pull requests.
+* `--branch-okay`:
+  Do not warn if pulling to a branch besides master (useful for testing).
+* `--resolve`:
+  When a patch fails to apply, leave in progress and allow user to resolve, instead of aborting.
+* `--workflow`:
+  Retrieve artifacts from the specified workflow (default: tests.yml).
+* `--artifact`:
+  Download artifacts with the specified name (default: bottles).
+* `--bintray-org`:
+  Upload to the specified Bintray organisation.
+
 ### `prof` *`command`*
 
 Run Homebrew with the Ruby profiler, e.g. `brew prof readall`.

manpages/brew.1

@@ -1127,6 +1127,45 @@ This is now done automatically by \fBbrew update\fR\.
 .SS "\fBpr\-publish\fR \fIpull_request\fR"
 Publishes bottles for a pull request with GitHub Actions\. Requires write access to the repository\.
 .
+.SS "\fBpr\-pull\fR \fIpull_request\fR"
+Download and publish bottles, and apply the bottle commit from a pull request with artifacts generated from GitHub Actions\. Requires write access to the repository\.
+.
+.TP
+\fB\-\-no\-publish\fR
+Download the bottles, apply the bottle commit, and upload the bottles to Bintray, but don\'t publish them\.
+.
+.TP
+\fB\-\-no\-upload\fR
+Download the bottles and apply the bottle commit, but don\'t upload to Bintray\.
+.
+.TP
+\fB\-n\fR, \fB\-\-dry\-run\fR
+Print what would be done rather than doing it\.
+.
+.TP
+\fB\-\-clean\fR
+Do not amend the commits from pull requests\.
+.
+.TP
+\fB\-\-branch\-okay\fR
+Do not warn if pulling to a branch besides master (useful for testing)\.
+.
+.TP
+\fB\-\-resolve\fR
+When a patch fails to apply, leave in progress and allow user to resolve, instead of aborting\.
+.
+.TP
+\fB\-\-workflow\fR
+Retrieve artifacts from the specified workflow (default: tests\.yml)\.
+.
+.TP
+\fB\-\-artifact\fR
+Download artifacts with the specified name (default: bottles)\.
+.
+.TP
+\fB\-\-bintray\-org\fR
+Upload to the specified Bintray organisation\.
+.
 .SS "\fBprof\fR \fIcommand\fR"
 Run Homebrew with the Ruby profiler, e\.g\. \fBbrew prof readall\fR\.
 .