From 5b1d0786a45e69b4af1d71d433cc11bb676fd6b6 Mon Sep 17 00:00:00 2001 From: Jonathan Chang Date: Mon, 30 Mar 2020 00:47:13 +1100 Subject: [PATCH 1/8] utils/github: add fetch_artifact API --- Library/Homebrew/utils/github.rb | 67 +++++++++++++++++++++++++++++++- 1 file changed, 66 insertions(+), 1 deletion(-) diff --git a/Library/Homebrew/utils/github.rb b/Library/Homebrew/utils/github.rb index eb6289d7a1..787e3c0647 100644 --- a/Library/Homebrew/utils/github.rb +++ b/Library/Homebrew/utils/github.rb @@ -1,7 +1,8 @@ # frozen_string_literal: true -require "uri" +require "download_strategy" require "tempfile" +require "uri" module GitHub module_function @@ -438,6 +439,70 @@ module GitHub scopes: CREATE_ISSUE_FORK_OR_PR_SCOPES) end + def fetch_artifact(user, repo, pr, dir, workflow_id: "tests.yml", artifact_name: "bottles") + base_url = "#{API_URL}/repos/#{user}/#{repo}" + pr_payload = open_api("#{base_url}/pulls/#{pr}") + pr_sha = pr_payload["head"]["sha"] + pr_branch = pr_payload["head"]["ref"] + + workflow = open_api("#{base_url}/actions/workflows/#{workflow_id}/runs?branch=#{pr_branch}") + workflow_run = workflow["workflow_runs"].select do |run| + run["head_sha"] == pr_sha + end + + if workflow_run.empty? + raise Error, <<~EOS + No matching workflow run found for these criteria! + Commit SHA: #{pr_sha} + Branch ref: #{pr_branch} + Pull request: #{pr} + Workflow: #{workflow_id} + EOS + end + + status = workflow_run.first["status"].sub("_", " ") + if status != "completed" + raise Error, <<~EOS + The newest workflow run for ##{pr} is still #{status}! + #{Formatter.url workflow_run.first["html_url"]} + EOS + end + + artifacts = open_api(workflow_run.first["artifacts_url"]) + + artifact = artifacts["artifacts"].select do |art| + art["name"] == artifact_name + end + + if artifact.empty? + raise Error, <<~EOS + No artifact with the name `#{artifact_name}` was found! + #{Formatter.url workflow_run.first["html_url"]} + EOS + end + + artifact_url = artifact.first["archive_download_url"] + + token, username = api_credentials + case api_credentials_type + when :env_username_password, :keychain_username_password + curl_args = { user: "#{username}:#{token}" } + when :env_token + curl_args = { header: "Authorization: token #{token}" } + end + + # Download the artifact as a zip file and unpack it into `dir`. This is + # preferred over system `curl` and `tar` as this leverages the Homebrew + # cache to avoid repeated downloads of (possibly large) bottles. + FileUtils.chdir dir do + curl_args[:cache] = Pathname.new(dir) + curl_args[:secrets] = [token] + downloader = CurlDownloadStrategy.new(artifact_url, "artifact", pr, **curl_args) + downloader.fetch + downloader.stage + end + end + def api_errors [GitHub::AuthenticationFailedError, GitHub::HTTPNotFoundError, GitHub::RateLimitExceededError, GitHub::Error, JSON::ParserError].freeze From 49c5a4393a679bff908799b96b1df5a276cb1ca1 Mon Sep 17 00:00:00 2001 From: Jonathan Chang Date: Mon, 30 Mar 2020 19:35:54 +1100 Subject: [PATCH 2/8] bintray: add Bintray API functions --- Library/Homebrew/bintray.rb | 125 ++++++++++++++++++++++++++++++++++++ 1 file changed, 125 insertions(+) create mode 100644 Library/Homebrew/bintray.rb diff --git a/Library/Homebrew/bintray.rb b/Library/Homebrew/bintray.rb new file mode 100644 index 0000000000..990b3ed3da --- /dev/null +++ b/Library/Homebrew/bintray.rb @@ -0,0 +1,125 @@ +# frozen_string_literal: true + +require "utils/curl" +require "json" + +class Bintray + API_URL = "https://api.bintray.com" + + class Error < RuntimeError + end + + def inspect + "#" + end + + def initialize(user: nil, key: nil, org: nil, clear: true) + @bintray_user = user || ENV["HOMEBREW_BINTRAY_USER"] + @bintray_key = key || ENV["HOMEBREW_BINTRAY_KEY"] + + if !@bintray_user || !@bintray_key + raise Error, "Missing HOMEBREW_BINTRAY_USER or HOMEBREW_BINTRAY_KEY variables!" unless Homebrew.args.dry_run? + end + + @bintray_org = org || "homebrew" + ENV["HOMEBREW_FORCE_HOMEBREW_ON_LINUX"] = "1" if @bintray_org == "homebrew" && !OS.mac? + + ENV.clear_sensitive_environment! if clear + end + + def open_api(url, *extra_curl_args, auth: true) + args = extra_curl_args + args += ["--user", "#{@bintray_user}:#{@bintray_key}"] if auth + curl(*args, url, + show_output: Homebrew.args.verbose?, + secrets: @bintray_key) + end + + def upload(local_file, repo:, package:, version:, remote_file:, sha256: nil) + url = "#{API_URL}/content/#{@bintray_org}/#{repo}/#{package}/#{version}/#{remote_file}" + args = ["--upload-file", local_file] + args += ["--header", "X-Checksum-Sha2: #{sha256}"] unless sha256.blank? + open_api url, *args + end + + def publish(repo:, package:, version:) + url = "#{API_URL}/content/#{@bintray_org}/#{repo}/#{package}/#{version}/publish" + open_api url, "--request", "POST" + end + + def official_org?(org: @bintray_org) + %w[homebrew linuxbrew].include? org + end + + def create_package(repo:, package:, **extra_data_args) + url = "#{API_URL}/packages/#{@bintray_org}/#{repo}/#{package}" + data = { name: package, public_download_numbers: true } + data[:public_stats] = official_org? + data.merge! extra_data_args + open_api url, "--request", "POST", "--data", data.to_json + end + + def package_exists?(repo:, package:) + url = "#{API_URL}/packages/#{@bintray_org}/#{repo}/#{package}" + open_api url, "--output", "/dev/null", auth: false + end + + def file_published?(repo:, remote_file:) + url = "https://dl.bintray.com/#{@bintray_org}/#{repo}/#{remote_file}" + begin + curl "--silent", "--head", "--output", "/dev/null", url + rescue ErrorDuringExecution => e + stderr = e.output.select { |type,| type == :stderr } + .map { |_, line| line } + .join + raise if e.status.exitstatus != 22 && !stderr.include?("404 Not Found") + + false + else + true + end + end + + def upload_bottle_json(json_files, publish_package: false) + bottles_hash = json_files.reduce({}) do |hash, json_file| + hash.deep_merge(JSON.parse(IO.read(json_file))) + end + + formula_packaged = {} + + bottles_hash.each do |formula_name, bottle_hash| + version = bottle_hash["formula"]["pkg_version"] + bintray_package = bottle_hash["bintray"]["package"] + bintray_repo = bottle_hash["bintray"]["repository"] + + bottle_hash["bottle"]["tags"].each do |_tag, tag_hash| + filename = tag_hash["filename"] + sha256 = tag_hash["sha256"] + + if file_published? repo: bintray_repo, remote_file: filename + raise Error, <<~EOS + #{filename} is already published. + Please remove it manually from: + https://bintray.com/#{@bintray_org}/#{bintray_repo}/#{bintray_package}/view#files + Or run: + curl -X DELETE -u $HOMEBREW_BINTRAY_USER:$HOMEBREW_BINTRAY_KEY \\ + https://api.bintray.com/content/#{@bintray_org}/#{bintray_repo}/#{filename} + EOS + end + + if !formula_packaged[formula_name] && !package_exists?(repo: bintray_repo, package: bintray_package) + create_package repo: bintray_repo, package: bintray_package + formula_packaged[formula_name] = true + end + + upload(tag_hash["local_filename"], + repo: bintray_repo, + package: bintray_package, + version: version, + remote_file: filename, + sha256: sha256) + end + publish repo: bintray_repo, package: bintray_package, version: version if publish_package + end + end +end From 0574ba436bb193eb899c22cc36799367364469e7 Mon Sep 17 00:00:00 2001 From: Jonathan Chang Date: Mon, 30 Mar 2020 00:47:38 +1100 Subject: [PATCH 3/8] dev-cmd: add pr-pull command --- Library/Homebrew/dev-cmd/pr-pull.rb | 143 ++++++++++++++++++++++++++++ 1 file changed, 143 insertions(+) create mode 100644 Library/Homebrew/dev-cmd/pr-pull.rb diff --git a/Library/Homebrew/dev-cmd/pr-pull.rb b/Library/Homebrew/dev-cmd/pr-pull.rb new file mode 100644 index 0000000000..fe15b54c27 --- /dev/null +++ b/Library/Homebrew/dev-cmd/pr-pull.rb @@ -0,0 +1,143 @@ +# frozen_string_literal: true + +require "cli/parser" +require "utils/github" +require "tmpdir" +require "bintray" + +module Homebrew + module_function + + def pr_pull_args + Homebrew::CLI::Parser.new do + usage_banner <<~EOS + `pr-pull` + + Download and publish bottles, and apply the bottle commit from a + pull request with artifacts generated from GitHub Actions. + Requires write access to the repository. + EOS + switch "--no-publish", + description: "Download the bottles, apply the bottle commit, and "\ + "upload the bottles to Bintray, but don't publish them." + switch "--no-upload", + description: "Download the bottles and apply the bottle commit, "\ + "but don't upload to Bintray." + switch "--dry-run", "-n", + description: "Print what would be done rather than doing it." + switch "--clean", + description: "Do not amend the commits from pull requests." + switch "--branch-okay", + description: "Do not warn if pulling to a branch besides master (useful for testing)." + switch "--resolve", + description: "When a patch fails to apply, leave in progress and allow user to resolve, instead "\ + "of aborting." + flag "--workflow=", + description: "Retrieve artifacts from the specified workflow (default: tests.yml)." + flag "--artifact=", + description: "Download artifacts with the specified name (default: bottles)." + flag "--bintray-org=", + description: "Upload to the specified Bintray organisation." + switch :verbose + switch :debug + min_named 1 + end + end + + def signoff!(pr, path: ".", dry_run: false) + message = Utils.popen_read "git", "-C", path, "log", "-1", "--pretty=%B" + close_message = "Closes ##{pr}." + message += "\n#{close_message}" unless message.include? close_message + if dry_run + puts "git commit --amend --signoff -m $message" + else + safe_system "git", "-C", path, "commit", "--amend", "--signoff", "--allow-empty", "-q", "-m", message + end + end + + def cherry_pick_pr!(pr, path: ".", dry_run: false) + if dry_run + puts <<~EOS + git fetch --force origin +refs/pull/#{pr}/head + git merge-base HEAD FETCH_HEAD + git cherry-pick --ff --allow-empty $merge_base..FETCH_HEAD + EOS + else + safe_system "git", "-C", path, "fetch", "--quiet", "--force", "origin", "+refs/pull/#{pr}/head" + merge_base = Utils.popen_read("git", "-C", path, "merge-base", "HEAD", "FETCH_HEAD").strip + commit_count = Utils.popen_read("git", "-C", path, "rev-list", "#{merge_base}..FETCH_HEAD").lines.count + + # git cherry-pick unfortunately has no quiet option + ohai "Cherry-picking #{commit_count} commit#{"s" unless commit_count == 1} from ##{pr}" + cherry_pick_args = "git", "-C", path, "cherry-pick", "--ff", "--allow-empty", "#{merge_base}..FETCH_HEAD" + result = Homebrew.args.verbose? ? system(*cherry_pick_args) : quiet_system(*cherry_pick_args) + + unless result + if Homebrew.args.resolve? + odie "Cherry-pick failed: try to resolve it." + else + system "git", "-C", path, "cherry-pick", "--abort" + odie "Cherry-pick failed!" + end + end + end + end + + def check_branch(path, ref) + branch = Utils.popen_read("git", "-C", path, "symbolic-ref", "--short", "HEAD").strip + + return if branch == ref || args.clean? || args.branch_okay? + + opoo "Current branch is #{branch}: do you need to pull inside #{ref}?" + end + + def pr_pull + pr_pull_args.parse + + bintray_user = ENV["HOMEBREW_BINTRAY_USER"] + bintray_key = ENV["HOMEBREW_BINTRAY_KEY"] + ENV.clear_sensitive_environment! + + if bintray_user.blank? || bintray_key.blank? + odie "Missing HOMEBREW_BINTRAY_USER or HOMEBREW_BINTRAY_KEY variables!" if !args.dry_run? && !args.no_upload? + else + bintray = Bintray.new(user: bintray_user, key: bintray_key, org: args.bintray_org) + end + + workflow = args.workflow || "tests.yml" + artifact = args.artifact || "bottles" + + args.named.each do |arg| + arg = "#{CoreTap.instance.default_remote}/pull/#{arg}" if arg.to_i.positive? + url_match = arg.match HOMEBREW_PULL_OR_COMMIT_URL_REGEX + _, user, repo, pr = *url_match + tap = Tap.fetch(user, repo) if repo.match?(HOMEBREW_OFFICIAL_REPO_PREFIXES_REGEX) + odie "Not a GitHub pull request: #{arg}" unless pr + + check_branch tap.path, "master" + + ohai "Fetching #{tap} pull request ##{pr}" + Dir.mktmpdir pr do |dir| + cd dir do + GitHub.fetch_artifact(user, repo, pr, dir, workflow_id: workflow, artifact_name: artifact) + cherry_pick_pr! pr, path: tap.path, dry_run: args.dry_run? + signoff! pr, path: tap.path, dry_run: args.dry_run? unless args.clean? + + if args.dry_run? + puts "brew bottle --merge --write #{Dir["*.json"].join " "}" + else + quiet_system "#{HOMEBREW_PREFIX}/bin/brew", "bottle", "--merge", "--write", *Dir["*.json"] + end + + next if args.no_upload? + + if args.dry_run? + puts "Upload bottles described by these JSON files to Bintray:\n #{Dir["*.json"].join("\n ")}" + else + bintray.upload_bottle_json Dir["*.json"], publish_package: !args.no_publish? + end + end + end + end + end +end From d459718bc4e79984879abd31879ccaf0dc1b2265 Mon Sep 17 00:00:00 2001 From: Jonathan Chang Date: Mon, 30 Mar 2020 19:37:12 +1100 Subject: [PATCH 4/8] manpages: update for new pr-pull command --- docs/Manpage.md | 25 +++++++++++++++++++++++++ manpages/brew.1 | 39 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 64 insertions(+) diff --git a/docs/Manpage.md b/docs/Manpage.md index ace12b9b16..e503d4f954 100644 --- a/docs/Manpage.md +++ b/docs/Manpage.md @@ -887,6 +887,31 @@ Generate Homebrew's manpages. Publishes bottles for a pull request with GitHub Actions. Requires write access to the repository. +### `pr-pull` *`pull_request`* + +Download and publish bottles, and apply the bottle commit from a pull request +with artifacts generated from GitHub Actions. Requires write access to the +repository. + +* `--no-publish`: + Download the bottles, apply the bottle commit, and upload the bottles to Bintray, but don't publish them. +* `--no-upload`: + Download the bottles and apply the bottle commit, but don't upload to Bintray. +* `-n`, `--dry-run`: + Print what would be done rather than doing it. +* `--clean`: + Do not amend the commits from pull requests. +* `--branch-okay`: + Do not warn if pulling to a branch besides master (useful for testing). +* `--resolve`: + When a patch fails to apply, leave in progress and allow user to resolve, instead of aborting. +* `--workflow`: + Retrieve artifacts from the specified workflow (default: tests.yml). +* `--artifact`: + Download artifacts with the specified name (default: bottles). +* `--bintray-org`: + Upload to the specified Bintray organisation. + ### `prof` *`command`* Run Homebrew with the Ruby profiler, e.g. `brew prof readall`. diff --git a/manpages/brew.1 b/manpages/brew.1 index 228d6853db..7501f2cf09 100644 --- a/manpages/brew.1 +++ b/manpages/brew.1 @@ -1127,6 +1127,45 @@ This is now done automatically by \fBbrew update\fR\. .SS "\fBpr\-publish\fR \fIpull_request\fR" Publishes bottles for a pull request with GitHub Actions\. Requires write access to the repository\. . +.SS "\fBpr\-pull\fR \fIpull_request\fR" +Download and publish bottles, and apply the bottle commit from a pull request with artifacts generated from GitHub Actions\. Requires write access to the repository\. +. +.TP +\fB\-\-no\-publish\fR +Download the bottles, apply the bottle commit, and upload the bottles to Bintray, but don\'t publish them\. +. +.TP +\fB\-\-no\-upload\fR +Download the bottles and apply the bottle commit, but don\'t upload to Bintray\. +. +.TP +\fB\-n\fR, \fB\-\-dry\-run\fR +Print what would be done rather than doing it\. +. +.TP +\fB\-\-clean\fR +Do not amend the commits from pull requests\. +. +.TP +\fB\-\-branch\-okay\fR +Do not warn if pulling to a branch besides master (useful for testing)\. +. +.TP +\fB\-\-resolve\fR +When a patch fails to apply, leave in progress and allow user to resolve, instead of aborting\. +. +.TP +\fB\-\-workflow\fR +Retrieve artifacts from the specified workflow (default: tests\.yml)\. +. +.TP +\fB\-\-artifact\fR +Download artifacts with the specified name (default: bottles)\. +. +.TP +\fB\-\-bintray\-org\fR +Upload to the specified Bintray organisation\. +. .SS "\fBprof\fR \fIcommand\fR" Run Homebrew with the Ruby profiler, e\.g\. \fBbrew prof readall\fR\. . From 282bfbf70adc3e343bfddb192b130771f8dba6ea Mon Sep 17 00:00:00 2001 From: Jonathan Chang Date: Mon, 30 Mar 2020 21:07:45 +1100 Subject: [PATCH 5/8] bintray: add tests --- Library/Homebrew/test/bintray_spec.rb | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 Library/Homebrew/test/bintray_spec.rb diff --git a/Library/Homebrew/test/bintray_spec.rb b/Library/Homebrew/test/bintray_spec.rb new file mode 100644 index 0000000000..0e1eceaf29 --- /dev/null +++ b/Library/Homebrew/test/bintray_spec.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +require "bintray" + +describe Bintray, :needs_network do + bintray = described_class.new(user: "BrewTestBot", key: "deadbeef", org: "homebrew") + describe "::file_published?" do + it "detects a published file" do + results = bintray.file_published?(repo: "bottles", remote_file: "hello-2.10.catalina.bottle.tar.gz") + expect(results).to be true + end + + it "fails on a non-existant file" do + results = bintray.file_published?(repo: "bottles", remote_file: "my-fake-bottle-1.0.snow_hyena.tar.gz") + expect(results).to be false + end + end + + describe "::package_exists?" do + it "detects a package" do + results = bintray.package_exists?(repo: "bottles", package: "hello") + expect(results.status.exitstatus).to be 0 + end + end +end From bab564f2030680bcde3a66664bf88e445c1ff258 Mon Sep 17 00:00:00 2001 From: Jonathan Chang Date: Mon, 30 Mar 2020 21:07:56 +1100 Subject: [PATCH 6/8] pr-pull: add tests --- Library/Homebrew/test/dev-cmd/pr-pull_spec.rb | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 Library/Homebrew/test/dev-cmd/pr-pull_spec.rb diff --git a/Library/Homebrew/test/dev-cmd/pr-pull_spec.rb b/Library/Homebrew/test/dev-cmd/pr-pull_spec.rb new file mode 100644 index 0000000000..99dfa041e9 --- /dev/null +++ b/Library/Homebrew/test/dev-cmd/pr-pull_spec.rb @@ -0,0 +1,7 @@ +# frozen_string_literal: true + +require "cmd/shared_examples/args_parse" + +describe "Homebrew.pr_pull_args" do + it_behaves_like "parseable arguments" +end From 0d3a4d98c0ebb7e5a97cb8151f7c348e6955002a Mon Sep 17 00:00:00 2001 From: Jonathan Chang Date: Mon, 30 Mar 2020 22:17:41 +1100 Subject: [PATCH 7/8] utils/github: add test for fetch_artifact --- Library/Homebrew/test/utils/github_spec.rb | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/Library/Homebrew/test/utils/github_spec.rb b/Library/Homebrew/test/utils/github_spec.rb index 936ae03ae6..c2dc27b443 100644 --- a/Library/Homebrew/test/utils/github_spec.rb +++ b/Library/Homebrew/test/utils/github_spec.rb @@ -41,4 +41,18 @@ describe GitHub do expect(results.first["title"]).to eq("Shall we run `brew update` automatically?") end end + + describe "::fetch_artifact", :needs_network do + it "fails to find a nonexistant workflow" do + expect { + subject.fetch_artifact("Homebrew", "homebrew-core", 1, ".") + }.to raise_error(/No matching workflow run found/) + end + + it "fails to find artifacts that don't exist" do + expect { + subject.fetch_artifact("Homebrew", "homebrew-core", 51971, ".", artifact_name: "false_bottles") + }.to raise_error(/No artifact .+ was found/) + end + end end From e601c3f87cb144afdedb69a15dec300da26feb3e Mon Sep 17 00:00:00 2001 From: Jonathan Chang Date: Mon, 30 Mar 2020 22:29:31 +1100 Subject: [PATCH 8/8] bintray: streamline initialization Co-Authored-By: Mike McQuaid --- Library/Homebrew/bintray.rb | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/Library/Homebrew/bintray.rb b/Library/Homebrew/bintray.rb index 990b3ed3da..5338d4efb6 100644 --- a/Library/Homebrew/bintray.rb +++ b/Library/Homebrew/bintray.rb @@ -13,15 +13,17 @@ class Bintray "#" end - def initialize(user: nil, key: nil, org: nil, clear: true) - @bintray_user = user || ENV["HOMEBREW_BINTRAY_USER"] - @bintray_key = key || ENV["HOMEBREW_BINTRAY_KEY"] + def initialize(user: ENV["HOMEBREW_BINTRAY_USER"], key: ENV["HOMEBREW_BINTRAY_KEY"], org: "homebrew", clear: true) + @bintray_user = user + @bintray_key = key + @bintray_org = org if !@bintray_user || !@bintray_key - raise Error, "Missing HOMEBREW_BINTRAY_USER or HOMEBREW_BINTRAY_KEY variables!" unless Homebrew.args.dry_run? + unless Homebrew.args.dry_run? + raise UsageError, "Missing HOMEBREW_BINTRAY_USER or HOMEBREW_BINTRAY_KEY variables!" + end end - @bintray_org = org || "homebrew" ENV["HOMEBREW_FORCE_HOMEBREW_ON_LINUX"] = "1" if @bintray_org == "homebrew" && !OS.mac? ENV.clear_sensitive_environment! if clear