admin/brew
1
0
Fork 0
Code Issues Packages Projects Releases 6 Wiki Activity

Merge pull request #10626 from nandahkrishna/refactor-github-api

Browse Source 
utils/github: split GitHub module
This commit is contained in:
Nanda H Krishna 2021-02-18 01:20:01 +05:30 committed by GitHub
commit 7dc8025934
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
15 changed files with 391 additions and 368 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
Library/Homebrew/.rubocop.yml
View File

@ -32,9 +32,8 @@ Metrics/PerceivedComplexity:
Max: 90
Metrics/MethodLength:
Max: 260
# TODO: Reduce to 600 after refactoring utils/github
Metrics/ModuleLength:
Max: 620
Max: 600
Exclude:
- "test/**/*"

20
Library/Homebrew/cmd/gist-logs.rb
View File

@ -55,7 +55,7 @@ module Homebrew
files["00.tap.out"] = { content: tap }
end
odie "`brew gist-logs` requires HOMEBREW_GITHUB_API_TOKEN to be set!" if GitHub.api_credentials_type == :none
odie "`brew gist-logs` requires HOMEBREW_GITHUB_API_TOKEN to be set!" if GitHub::API.credentials_type == :none
# Description formatted to work well as page title when viewing gist
descr = if f.core_formula?
@ -63,9 +63,9 @@ module Homebrew
else
"#{f.name} (#{f.full_name}) on #{OS_VERSION} - Homebrew build logs"
end
url = create_gist(files, descr, private: args.private?)
url = GitHub.create_gist(files, descr, private: args.private?)
url = create_issue(f.tap, "#{f.name} failed to build on #{MacOS.full_version}", url) if args.new_issue?
url = GitHub.create_issue(f.tap, "#{f.name} failed to build on #{MacOS.full_version}", url) if args.new_issue?
puts url if url
end
@ -108,20 +108,6 @@ module Homebrew
logs
end
def create_gist(files, description, private:)
url = "https://api.github.com/gists"
data = { "public" => !private, "files" => files, "description" => description }
scopes = GitHub::CREATE_GIST_SCOPES
GitHub.open_api(url, data: data, scopes: scopes)["html_url"]
end
def create_issue(repo, title, body)
url = "https://api.github.com/repos/#{repo}/issues"
data = { "title" => title, "body" => body }
scopes = GitHub::CREATE_ISSUE_FORK_OR_PR_SCOPES
GitHub.open_api(url, data: data, scopes: scopes)["html_url"]
end
def gist_logs
args = gist_logs_args.parse

4
Library/Homebrew/dev-cmd/pr-pull.rb
View File

@ -337,9 +337,9 @@ module Homebrew
end
def download_artifact(url, dir, pr)
odie "Credentials must be set to access the Artifacts API" if GitHub.api_credentials_type == :none
odie "Credentials must be set to access the Artifacts API" if GitHub::API.credentials_type == :none
token = GitHub.api_credentials
token = GitHub::API.credentials
curl_args = ["--header", "Authorization: token #{token}"]
# Download the artifact as a zip file and unpack it into `dir`. This is

2
Library/Homebrew/dev-cmd/pr-upload.rb
View File

@ -114,7 +114,7 @@ module Homebrew
rel = GitHub.get_release user, repo, tag
odebug "Existing GitHub release \"#{tag}\" found"
rel
rescue GitHub::HTTPNotFoundError
rescue GitHub::API::HTTPNotFoundError
odebug "Creating new GitHub release \"#{tag}\""
GitHub.create_or_update_release user, repo, tag
end

6
Library/Homebrew/dev-cmd/release.rb
View File

@ -39,7 +39,7 @@ module Homebrew
begin
latest_release = GitHub.get_latest_release "Homebrew", "brew"
rescue GitHub::HTTPNotFoundError
rescue GitHub::API::HTTPNotFoundError
odie "No existing releases found!"
end
latest_version = Version.new latest_release["tag_name"]
@ -48,7 +48,7 @@ module Homebrew
one_month_ago = Date.today << 1
latest_major_minor_release = begin
GitHub.get_release "Homebrew", "brew", "#{latest_version.major_minor}.0"
rescue GitHub::HTTPNotFoundError
rescue GitHub::API::HTTPNotFoundError
nil
end
@ -89,7 +89,7 @@ module Homebrew
begin
release = GitHub.create_or_update_release "Homebrew", "brew", new_version, body: release_notes, draft: true
rescue *GitHub::API_ERRORS => e
rescue *GitHub::API::ERRORS => e
odie "Unable to create release: #{e.message}!"
end

2
Library/Homebrew/exceptions.rb
View File

@ -423,7 +423,7 @@ class BuildError < RuntimeError
def fetch_issues
GitHub.issues_for_formula(formula.name, tap: formula.tap, state: "open")
rescue GitHub::RateLimitExceededError => e
rescue GitHub::API::RateLimitExceededError => e
opoo e.message
[]
end

2
Library/Homebrew/formula_creator.rb
View File

@ -79,7 +79,7 @@ module Homebrew
@desc = metadata["description"]
@homepage = metadata["homepage"]
@license = metadata["license"]["spdx_id"] if metadata["license"]
rescue GitHub::HTTPNotFoundError
rescue GitHub::API::HTTPNotFoundError
# If there was no repository found assume the network connection is at
# fault rather than the input URL.
nil

2
Library/Homebrew/search.rb
View File

@ -48,7 +48,7 @@ module Homebrew
filename: query,
extension: "rb",
)
rescue GitHub::Error => e
rescue GitHub::API::Error => e
opoo "Error searching on GitHub: #{e}\n"
nil
end

4
Library/Homebrew/tap.rb
View File

@ -692,9 +692,9 @@ class Tap
else
GitHub.private_repo?(full_name)
end
rescue GitHub::HTTPNotFoundError
rescue GitHub::API::HTTPNotFoundError
true
rescue GitHub::Error
rescue GitHub::API::Error
false
end
end

4
Library/Homebrew/test/search_spec.rb
View File

@ -21,7 +21,7 @@ describe Homebrew::Search do
end
it "does not raise if the network fails" do
allow(GitHub).to receive(:open_api).and_raise(GitHub::Error)
allow(GitHub::API).to receive(:open_rest).and_raise(GitHub::API::Error)
expect(mod.search_taps("some-formula"))
.to match(formulae: [], casks: [])
@ -45,7 +45,7 @@ describe Homebrew::Search do
],
}
allow(GitHub).to receive(:open_api).and_yield(json_response)
allow(GitHub::API).to receive(:open_rest).and_yield(json_response)
expect(mod.search_taps("some-formula"))
.to match(formulae: ["homebrew/foo/some-formula"], casks: ["homebrew/bar/some-cask"])

2
Library/Homebrew/test/tap_spec.rb
View File

@ -216,7 +216,7 @@ describe Tap do
end
specify "#private?" do
skip "HOMEBREW_GITHUB_API_TOKEN is required" unless GitHub.api_credentials
skip "HOMEBREW_GITHUB_API_TOKEN is required" unless GitHub::API.credentials
expect(homebrew_foo_tap).to be_private
end

387
Library/Homebrew/utils/github.rb
View File

@ -1,12 +1,11 @@
# typed: false
# frozen_string_literal: true
require "tempfile"
require "uri"
require "utils/github/actions"
require "utils/shell"
require "utils/github/api"
# Helper functions for interacting with the GitHub API.
# Wrapper functions for the GitHub API.
#
# @api private
module GitHub
@ -14,288 +13,10 @@ module GitHub
module_function
API_URL = "https://api.github.com"
API_MAX_PAGES = 50
API_MAX_ITEMS = 5000
CREATE_GIST_SCOPES = ["gist"].freeze
CREATE_ISSUE_FORK_OR_PR_SCOPES = ["public_repo"].freeze
CREATE_WORKFLOW_SCOPES = ["workflow"].freeze
ALL_SCOPES = (CREATE_GIST_SCOPES + CREATE_ISSUE_FORK_OR_PR_SCOPES + CREATE_WORKFLOW_SCOPES).freeze
ALL_SCOPES_URL = Formatter.url(
"https://github.com/settings/tokens/new?scopes=#{ALL_SCOPES.join(",")}&description=Homebrew",
).freeze
CREATE_GITHUB_PAT_MESSAGE = <<~EOS
Create a GitHub personal access token:
#{ALL_SCOPES_URL}
#{Utils::Shell.set_variable_in_profile("HOMEBREW_GITHUB_API_TOKEN", "your_token_here")}
EOS
# Generic API error.
class Error < RuntimeError
attr_reader :github_message
end
# Error when the requested URL is not found.
class HTTPNotFoundError < Error
def initialize(github_message)
@github_message = github_message
super
end
end
# Error when the API rate limit is exceeded.
class RateLimitExceededError < Error
def initialize(reset, github_message)
@github_message = github_message
super <<~EOS
GitHub API Error: #{github_message}
Try again in #{pretty_ratelimit_reset(reset)}, or:
#{CREATE_GITHUB_PAT_MESSAGE}
EOS
end
def pretty_ratelimit_reset(reset)
pretty_duration(Time.at(reset) - Time.now)
end
end
# Error when authentication fails.
class AuthenticationFailedError < Error
def initialize(github_message)
@github_message = github_message
message = +"GitHub #{github_message}:"
message << if Homebrew::EnvConfig.github_api_token
<<~EOS
HOMEBREW_GITHUB_API_TOKEN may be invalid or expired; check:
#{Formatter.url("https://github.com/settings/tokens")}
EOS
else
<<~EOS
The GitHub credentials in the macOS keychain may be invalid.
Clear them with:
printf "protocol=https\\nhost=github.com\\n" | git credential-osxkeychain erase
#{CREATE_GITHUB_PAT_MESSAGE}
EOS
end
super message.freeze
end
end
# Error when the user has no GitHub API credentials set at all (macOS keychain or envvar).
class MissingAuthenticationError < Error
def initialize
message = +"No GitHub credentials found in macOS Keychain or environment.\n"
message << CREATE_GITHUB_PAT_MESSAGE
super message
end
end
# Error when the API returns a validation error.
class ValidationFailedError < Error
def initialize(github_message, errors)
@github_message = if errors.empty?
github_message
else
"#{github_message}: #{errors}"
end
super(@github_message)
end
end
API_ERRORS = [
AuthenticationFailedError,
HTTPNotFoundError,
RateLimitExceededError,
Error,
JSON::ParserError,
].freeze
# Gets the password field from `git-credential-osxkeychain` for github.com,
# but only if that password looks like a GitHub Personal Access Token.
sig { returns(T.nilable(String)) }
def keychain_username_password
github_credentials = Utils.popen(["git", "credential-osxkeychain", "get"], "w+") do |pipe|
pipe.write "protocol=https\nhost=github.com\n"
pipe.close_write
pipe.read
end
github_username = github_credentials[/username=(.+)/, 1]
github_password = github_credentials[/password=(.+)/, 1]
return unless github_username
# Don't use passwords from the keychain unless they look like
# GitHub Personal Access Tokens:
# https://github.com/Homebrew/brew/issues/6862#issuecomment-572610344
return unless /^[a-f0-9]{40}$/i.match?(github_password)
github_password
rescue Errno::EPIPE
# The above invocation via `Utils.popen` can fail, causing the pipe to be
# prematurely closed (before we can write to it) and thus resulting in a
# broken pipe error. The root cause is usually a missing or malfunctioning
# `git-credential-osxkeychain` helper.
nil
end
def api_credentials
@api_credentials ||= begin
Homebrew::EnvConfig.github_api_token || keychain_username_password
end
end
sig { returns(Symbol) }
def api_credentials_type
if Homebrew::EnvConfig.github_api_token
:env_token
elsif keychain_username_password
:keychain_username_password
else
:none
end
end
# Given an API response from GitHub, warn the user if their credentials
# have insufficient permissions.
def api_credentials_error_message(response_headers, needed_scopes)
return if response_headers.empty?
scopes = response_headers["x-accepted-oauth-scopes"].to_s.split(", ")
needed_scopes = Set.new(scopes || needed_scopes)
credentials_scopes = response_headers["x-oauth-scopes"]
return if needed_scopes.subset?(Set.new(credentials_scopes.to_s.split(", ")))
needed_scopes = needed_scopes.to_a.join(", ").presence || "none"
credentials_scopes = "none" if credentials_scopes.blank?
what = case api_credentials_type
when :keychain_username_password
"macOS keychain GitHub"
when :env_token
"HOMEBREW_GITHUB_API_TOKEN"
end
@api_credentials_error_message ||= onoe <<~EOS
Your #{what} credentials do not have sufficient scope!
Scopes required: #{needed_scopes}
Scopes present: #{credentials_scopes}
#{CREATE_GITHUB_PAT_MESSAGE}
EOS
end
def open_api(url, data: nil, data_binary_path: nil, request_method: nil, scopes: [].freeze, parse_json: true)
# This is a no-op if the user is opting out of using the GitHub API.
return block_given? ? yield({}) : {} if Homebrew::EnvConfig.no_github_api?
args = ["--header", "Accept: application/vnd.github.v3+json", "--write-out", "\n%\{http_code}"]
args += ["--header", "Accept: application/vnd.github.antiope-preview+json"]
token = api_credentials
args += ["--header", "Authorization: token #{token}"] unless api_credentials_type == :none
data_tmpfile = nil
if data
begin
data = JSON.generate data
data_tmpfile = Tempfile.new("github_api_post", HOMEBREW_TEMP)
rescue JSON::ParserError => e
raise Error, "Failed to parse JSON request:\n#{e.message}\n#{data}", e.backtrace
end
end
if data_binary_path.present?
args += ["--data-binary", "@#{data_binary_path}"]
args += ["--header", "Content-Type: application/gzip"]
end
headers_tmpfile = Tempfile.new("github_api_headers", HOMEBREW_TEMP)
begin
if data
data_tmpfile.write data
data_tmpfile.close
args += ["--data", "@#{data_tmpfile.path}"]
args += ["--request", request_method.to_s] if request_method
end
args += ["--dump-header", headers_tmpfile.path]
output, errors, status = curl_output("--location", url.to_s, *args, secrets: [token])
output, _, http_code = output.rpartition("\n")
output, _, http_code = output.rpartition("\n") if http_code == "000"
headers = headers_tmpfile.read
ensure
if data_tmpfile
data_tmpfile.close
data_tmpfile.unlink
end
headers_tmpfile.close
headers_tmpfile.unlink
end
begin
raise_api_error(output, errors, http_code, headers, scopes) if !http_code.start_with?("2") || !status.success?
return if http_code == "204" # No Content
output = JSON.parse output if parse_json
if block_given?
yield output
else
output
end
rescue JSON::ParserError => e
raise Error, "Failed to parse JSON response\n#{e.message}", e.backtrace
end
end
def open_graphql(query, scopes: [].freeze)
data = { query: query }
result = open_api("https://api.github.com/graphql", scopes: scopes, data: data, request_method: "POST")
raise Error, result["errors"].map { |e| "#{e["type"]}: #{e["message"]}" }.join("\n") if result["errors"].present?
result["data"]
end
def raise_api_error(output, errors, http_code, headers, scopes)
json = begin
JSON.parse(output)
rescue
nil
end
message = json&.[]("message") || "curl failed! #{errors}"
meta = {}
headers.lines.each do |l|
key, _, value = l.delete(":").partition(" ")
key = key.downcase.strip
next if key.empty?
meta[key] = value.strip
end
if meta.fetch("x-ratelimit-remaining", 1).to_i <= 0
reset = meta.fetch("x-ratelimit-reset").to_i
raise RateLimitExceededError.new(reset, message)
end
api_credentials_error_message(meta, scopes)
case http_code
when "401", "403"
raise AuthenticationFailedError, message
when "404"
raise MissingAuthenticationError if api_credentials_type == :none && scopes.present?
raise HTTPNotFoundError, message
when "422"
errors = json&.[]("errors") || []
raise ValidationFailedError.new(message, errors)
else
raise Error, message
end
odeprecated "GitHub.open_api", "GitHub::API.open_rest"
API.open_rest(url, data: data, data_binary_path: data_binary_path, request_method: request_method,
scopes: scopes, parse_json: parse_json)
end
def check_runs(repo: nil, commit: nil, pr: nil)
@ -304,19 +25,31 @@ module GitHub
commit = pr.fetch("head").fetch("sha")
end
open_api(url_to("repos", repo, "commits", commit, "check-runs"))
API.open_rest(url_to("repos", repo, "commits", commit, "check-runs"))
end
def create_check_run(repo:, data:)
open_api(url_to("repos", repo, "check-runs"), data: data)
API.open_rest(url_to("repos", repo, "check-runs"), data: data)
end
def search_issues(query, **qualifiers)
search("issues", query, **qualifiers)
end
def create_gist(files, description, private:)
url = "https://api.github.com/gists"
data = { "public" => !private, "files" => files, "description" => description }
API.open_rest(url, data: data, scopes: CREATE_GIST_SCOPES)["html_url"]
end
def create_issue(repo, title, body)
url = "https://api.github.com/repos/#{repo}/issues"
data = { "title" => title, "body" => body }
API.open_rest(url, data: data, scopes: CREATE_ISSUE_FORK_OR_PR_SCOPES)["html_url"]
end
def repository(user, repo)
open_api(url_to("repos", user, repo))
API.open_rest(url_to("repos", user, repo))
end
def search_code(**qualifiers)
@ -335,11 +68,11 @@ module GitHub
end
def user
@user ||= open_api("#{API_URL}/user")
@user ||= API.open_rest("#{API_URL}/user")
end
def permission(repo, user)
open_api("#{API_URL}/repos/#{repo}/collaborators/#{user}/permission")
API.open_rest("#{API_URL}/repos/#{repo}/collaborators/#{user}/permission")
end
def write_access?(repo, user = nil)
@ -349,14 +82,14 @@ module GitHub
def pull_requests(repo, **options)
url = "#{API_URL}/repos/#{repo}/pulls?#{URI.encode_www_form(options)}"
open_api(url)
API.open_rest(url)
end
def merge_pull_request(repo, number:, sha:, merge_method:, commit_message: nil)
url = "#{API_URL}/repos/#{repo}/pulls/#{number}/merge"
data = { sha: sha, merge_method: merge_method }
data[:commit_message] = commit_message if commit_message
open_api(url, data: data, request_method: :PUT, scopes: CREATE_ISSUE_FORK_OR_PR_SCOPES)
API.open_rest(url, data: data, request_method: :PUT, scopes: CREATE_ISSUE_FORK_OR_PR_SCOPES)
end
def print_pull_requests_matching(query, only = nil)
@ -386,14 +119,14 @@ module GitHub
url = "#{API_URL}/repos/#{repo}/forks"
data = {}
scopes = CREATE_ISSUE_FORK_OR_PR_SCOPES
open_api(url, data: data, scopes: scopes)
API.open_rest(url, data: data, scopes: scopes)
end
def check_fork_exists(repo)
_, reponame = repo.split("/")
username = open_api(url_to("user")) { |json| json["login"] }
json = open_api(url_to("repos", username, reponame))
username = API.open_rest(url_to("user")) { |json| json["login"] }
json = API.open_rest(url_to("repos", username, reponame))
return false if json["message"] == "Not Found"
@ -404,12 +137,12 @@ module GitHub
url = "#{API_URL}/repos/#{repo}/pulls"
data = { title: title, head: head, base: base, body: body }
scopes = CREATE_ISSUE_FORK_OR_PR_SCOPES
open_api(url, data: data, scopes: scopes)
API.open_rest(url, data: data, scopes: scopes)
end
def private_repo?(full_name)
uri = url_to "repos", full_name
open_api(uri) { |json| json["private"] }
API.open_rest(uri) { |json| json["private"] }
end
def query_string(*main_params, **qualifiers)
@ -429,7 +162,7 @@ module GitHub
def search(entity, *queries, **qualifiers)
uri = url_to "search", entity
uri.query = query_string(*queries, **qualifiers)
open_api(uri) { |json| json.fetch("items", []) }
API.open_rest(uri) { |json| json.fetch("items", []) }
end
def approved_reviews(user, repo, pr, commit: nil)
@ -451,7 +184,7 @@ module GitHub
}
EOS
result = open_graphql(query, scopes: ["user:email"])
result = API.open_graphql(query, scopes: ["user:email"])
reviews = result["repository"]["pullRequest"]["reviews"]["nodes"]
valid_associations = %w[MEMBER OWNER]
@ -475,26 +208,26 @@ module GitHub
def dispatch_event(user, repo, event, **payload)
url = "#{API_URL}/repos/#{user}/#{repo}/dispatches"
open_api(url, data: { event_type: event, client_payload: payload },
API.open_rest(url, data: { event_type: event, client_payload: payload },
request_method: :POST,
scopes: CREATE_ISSUE_FORK_OR_PR_SCOPES)
end
def workflow_dispatch_event(user, repo, workflow, ref, **inputs)
url = "#{API_URL}/repos/#{user}/#{repo}/actions/workflows/#{workflow}/dispatches"
open_api(url, data: { ref: ref, inputs: inputs },
API.open_rest(url, data: { ref: ref, inputs: inputs },
request_method: :POST,
scopes: CREATE_ISSUE_FORK_OR_PR_SCOPES)
end
def get_release(user, repo, tag)
url = "#{API_URL}/repos/#{user}/#{repo}/releases/tags/#{tag}"
open_api(url, request_method: :GET)
API.open_rest(url, request_method: :GET)
end
def get_latest_release(user, repo)
url = "#{API_URL}/repos/#{user}/#{repo}/releases/latest"
open_api(url, request_method: :GET)
API.open_rest(url, request_method: :GET)
end
def create_or_update_release(user, repo, tag, id: nil, name: nil, body: nil, draft: false)
@ -511,24 +244,24 @@ module GitHub
draft: draft,
}
data[:body] = body if body.present?
open_api(url, data: data, request_method: method, scopes: CREATE_ISSUE_FORK_OR_PR_SCOPES)
API.open_rest(url, data: data, request_method: method, scopes: CREATE_ISSUE_FORK_OR_PR_SCOPES)
end
def upload_release_asset(user, repo, id, local_file: nil, remote_file: nil)
url = "https://uploads.github.com/repos/#{user}/#{repo}/releases/#{id}/assets"
url += "?name=#{remote_file}" if remote_file
open_api(url, data_binary_path: local_file, request_method: :POST, scopes: CREATE_ISSUE_FORK_OR_PR_SCOPES)
API.open_rest(url, data_binary_path: local_file, request_method: :POST, scopes: CREATE_ISSUE_FORK_OR_PR_SCOPES)
end
def get_workflow_run(user, repo, pr, workflow_id: "tests.yml", artifact_name: "bottles")
scopes = CREATE_ISSUE_FORK_OR_PR_SCOPES
base_url = "#{API_URL}/repos/#{user}/#{repo}"
pr_payload = open_api("#{base_url}/pulls/#{pr}", scopes: scopes)
pr_payload = API.open_rest("#{base_url}/pulls/#{pr}", scopes: scopes)
pr_sha = pr_payload["head"]["sha"]
pr_branch = URI.encode_www_form_component(pr_payload["head"]["ref"])
parameters = "event=pull_request&branch=#{pr_branch}"
workflow = open_api("#{base_url}/actions/workflows/#{workflow_id}/runs?#{parameters}", scopes: scopes)
workflow = API.open_rest("#{base_url}/actions/workflows/#{workflow_id}/runs?#{parameters}", scopes: scopes)
workflow_run = workflow["workflow_runs"].select do |run|
run["head_sha"] == pr_sha
end
@ -539,7 +272,7 @@ module GitHub
def get_artifact_url(workflow_array)
workflow_run, pr_sha, pr_branch, pr, workflow_id, scopes, artifact_name = *workflow_array
if workflow_run.empty?
raise Error, <<~EOS
raise API::Error, <<~EOS
No matching workflow run found for these criteria!
Commit SHA: #{pr_sha}
Branch ref: #{pr_branch}
@ -550,20 +283,20 @@ module GitHub
status = workflow_run.first["status"].sub("_", " ")
if status != "completed"
raise Error, <<~EOS
raise API::Error, <<~EOS
The newest workflow run for ##{pr} is still #{status}!
#{Formatter.url workflow_run.first["html_url"]}
EOS
end
artifacts = open_api(workflow_run.first["artifacts_url"], scopes: scopes)
artifacts = API.open_rest(workflow_run.first["artifacts_url"], scopes: scopes)
artifact = artifacts["artifacts"].select do |art|
art["name"] == artifact_name
end
if artifact.empty?
raise Error, <<~EOS
raise API::Error, <<~EOS
No artifact with the name `#{artifact_name}` was found!
#{Formatter.url workflow_run.first["html_url"]}
EOS
@ -577,7 +310,7 @@ module GitHub
members = []
(1..API_MAX_PAGES).each do |page|
result = open_api("#{url}&page=#{page}").map { |member| member["login"] }
result = API.open_rest("#{url}&page=#{page}").map { |member| member["login"] }
members.concat(result)
return members if result.length < per_page
@ -602,13 +335,13 @@ module GitHub
}
}
EOS
result = open_graphql(query, scopes: ["read:org", "user"])
result = API.open_graphql(query, scopes: ["read:org", "user"])
if result["organization"]["teams"]["nodes"].blank?
raise Error,
raise API::Error,
"Your token needs the 'read:org' scope to access this API"
end
raise Error, "The team #{org}/#{team} does not exist" if result["organization"]["team"].blank?
raise API::Error, "The team #{org}/#{team} does not exist" if result["organization"]["team"].blank?
result["organization"]["team"]["members"]["nodes"].map { |member| [member["login"], member["name"]] }.to_h
end
@ -639,13 +372,13 @@ module GitHub
}
}
EOS
result = open_graphql(query, scopes: ["admin:org", "user"])
result = API.open_graphql(query, scopes: ["admin:org", "user"])
tiers = result["organization"]["sponsorsListing"]["tiers"]["nodes"]
tiers.map do |t|
tier = t["monthlyPriceInDollars"]
raise Error, "Your token needs the 'admin:org' scope to access this API" if t["adminInfo"].nil?
raise API::Error, "Your token needs the 'admin:org' scope to access this API" if t["adminInfo"].nil?
sponsorships = t["adminInfo"]["sponsorships"]
count = sponsorships["totalCount"]
@ -669,11 +402,11 @@ module GitHub
end
def get_repo_license(user, repo)
response = open_api("#{API_URL}/repos/#{user}/#{repo}/license")
response = API.open_rest("#{API_URL}/repos/#{user}/#{repo}/license")
return unless response.key?("license")
response["license"]["spdx_id"]
rescue HTTPNotFoundError
rescue API::HTTPNotFoundError
nil
end
@ -688,14 +421,14 @@ module GitHub
issues_for_formula(query, tap_full_name: tap_full_name, state: state).select do |pr|
pr["html_url"].include?("/pull/") && regex.match?(pr["title"])
end
rescue RateLimitExceededError => e
rescue API::RateLimitExceededError => e
opoo e.message
[]
end
def check_for_duplicate_pull_requests(name, tap_full_name, state:, file:, args:, version: nil)
pull_requests = fetch_pull_requests(name, tap_full_name, state: state, version: version).select do |pr|
pr_files = open_api(url_to("repos", tap_full_name, "pulls", pr["number"], "files"))
pr_files = API.open_rest(url_to("repos", tap_full_name, "pulls", pr["number"], "files"))
pr_files.any? { |f| f["filename"] == file }
end
return if pull_requests.blank?
@ -772,7 +505,7 @@ module GitHub
else
begin
remote_url, username = forked_repo_info!(tap_full_name)
rescue *API_ERRORS => e
rescue *API::ERRORS => e
sourcefile_path.atomic_write(old_contents)
odie "Unable to fork: #{e.message}!"
end
@ -812,7 +545,7 @@ module GitHub
else
exec_browser url
end
rescue *API_ERRORS => e
rescue *API::ERRORS => e
odie "Unable to open pull request: #{e.message}!"
end
end
@ -820,29 +553,29 @@ module GitHub
end
def pull_request_commits(user, repo, pr, per_page: 100)
pr_data = open_api(url_to("repos", user, repo, "pulls", pr))
pr_data = API.open_rest(url_to("repos", user, repo, "pulls", pr))
commits_api = pr_data["commits_url"]
commit_count = pr_data["commits"]
commits = []
if commit_count > API_MAX_ITEMS
raise Error, "Getting #{commit_count} commits would exceed limit of #{API_MAX_ITEMS} API items!"
raise API::Error, "Getting #{commit_count} commits would exceed limit of #{API_MAX_ITEMS} API items!"
end
(1..API_MAX_PAGES).each do |page|
result = open_api(commits_api + "?per_page=#{per_page}&page=#{page}")
result = API.open_rest(commits_api + "?per_page=#{per_page}&page=#{page}")
commits.concat(result.map { |c| c["sha"] })
return commits if commits.length == commit_count
if result.empty? || page * per_page >= commit_count
raise Error, "Expected #{commit_count} commits but actually got #{commits.length}!"
raise API::Error, "Expected #{commit_count} commits but actually got #{commits.length}!"
end
end
end
def pull_request_labels(user, repo, pr)
pr_data = open_api(url_to("repos", user, repo, "pulls", pr))
pr_data = API.open_rest(url_to("repos", user, repo, "pulls", pr))
pr_data["labels"].map { |label| label["name"] }
end
end

304
Library/Homebrew/utils/github/api.rb Normal file
View File

@ -0,0 +1,304 @@
# typed: false
# frozen_string_literal: true
require "tempfile"
require "utils/shell"
module GitHub
API_URL = "https://api.github.com"
API_MAX_PAGES = 50
API_MAX_ITEMS = 5000
CREATE_GIST_SCOPES = ["gist"].freeze
CREATE_ISSUE_FORK_OR_PR_SCOPES = ["public_repo"].freeze
CREATE_WORKFLOW_SCOPES = ["workflow"].freeze
ALL_SCOPES = (CREATE_GIST_SCOPES + CREATE_ISSUE_FORK_OR_PR_SCOPES + CREATE_WORKFLOW_SCOPES).freeze
ALL_SCOPES_URL = Formatter.url(
"https://github.com/settings/tokens/new?scopes=#{ALL_SCOPES.join(",")}&description=Homebrew",
).freeze
CREATE_GITHUB_PAT_MESSAGE = <<~EOS
Create a GitHub personal access token:
#{ALL_SCOPES_URL}
#{Utils::Shell.set_variable_in_profile("HOMEBREW_GITHUB_API_TOKEN", "your_token_here")}
EOS
# Helper functions to access the GitHub API.
#
# @api private
module API
extend T::Sig
module_function
# Generic API error.
class Error < RuntimeError
attr_reader :github_message
end
# Error when the requested URL is not found.
class HTTPNotFoundError < Error
def initialize(github_message)
@github_message = github_message
super
end
end
# Error when the API rate limit is exceeded.
class RateLimitExceededError < Error
def initialize(reset, github_message)
@github_message = github_message
super <<~EOS
GitHub API Error: #{github_message}
Try again in #{pretty_ratelimit_reset(reset)}, or:
#{CREATE_GITHUB_PAT_MESSAGE}
EOS
end
def pretty_ratelimit_reset(reset)
pretty_duration(Time.at(reset) - Time.now)
end
end
# Error when authentication fails.
class AuthenticationFailedError < Error
def initialize(github_message)
@github_message = github_message
message = +"GitHub #{github_message}:"
message << if Homebrew::EnvConfig.github_api_token
<<~EOS
HOMEBREW_GITHUB_API_TOKEN may be invalid or expired; check:
#{Formatter.url("https://github.com/settings/tokens")}
EOS
else
<<~EOS
The GitHub credentials in the macOS keychain may be invalid.
Clear them with:
printf "protocol=https\\nhost=github.com\\n" | git credential-osxkeychain erase
#{CREATE_GITHUB_PAT_MESSAGE}
EOS
end
super message.freeze
end
end
# Error when the user has no GitHub API credentials set at all (macOS keychain or envvar).
class MissingAuthenticationError < Error
def initialize
message = +"No GitHub credentials found in macOS Keychain or environment.\n"
message << CREATE_GITHUB_PAT_MESSAGE
super message
end
end
# Error when the API returns a validation error.
class ValidationFailedError < Error
def initialize(github_message, errors)
@github_message = if errors.empty?
github_message
else
"#{github_message}: #{errors}"
end
super(@github_message)
end
end
ERRORS = [
AuthenticationFailedError,
HTTPNotFoundError,
RateLimitExceededError,
Error,
JSON::ParserError,
].freeze
# Gets the password field from `git-credential-osxkeychain` for github.com,
# but only if that password looks like a GitHub Personal Access Token.
sig { returns(T.nilable(String)) }
def keychain_username_password
github_credentials = Utils.popen(["git", "credential-osxkeychain", "get"], "w+") do |pipe|
pipe.write "protocol=https\nhost=github.com\n"
pipe.close_write
pipe.read
end
github_username = github_credentials[/username=(.+)/, 1]
github_password = github_credentials[/password=(.+)/, 1]
return unless github_username
# Don't use passwords from the keychain unless they look like
# GitHub Personal Access Tokens:
# https://github.com/Homebrew/brew/issues/6862#issuecomment-572610344
return unless /^[a-f0-9]{40}$/i.match?(github_password)
github_password
rescue Errno::EPIPE
# The above invocation via `Utils.popen` can fail, causing the pipe to be
# prematurely closed (before we can write to it) and thus resulting in a
# broken pipe error. The root cause is usually a missing or malfunctioning
# `git-credential-osxkeychain` helper.
nil
end
def credentials
@credentials ||= begin
Homebrew::EnvConfig.github_api_token || keychain_username_password
end
end
sig { returns(Symbol) }
def credentials_type
if Homebrew::EnvConfig.github_api_token
:env_token
elsif keychain_username_password
:keychain_username_password
else
:none
end
end
# Given an API response from GitHub, warn the user if their credentials
# have insufficient permissions.
def credentials_error_message(response_headers, needed_scopes)
return if response_headers.empty?
scopes = response_headers["x-accepted-oauth-scopes"].to_s.split(", ")
needed_scopes = Set.new(scopes || needed_scopes)
credentials_scopes = response_headers["x-oauth-scopes"]
return if needed_scopes.subset?(Set.new(credentials_scopes.to_s.split(", ")))
needed_scopes = needed_scopes.to_a.join(", ").presence || "none"
credentials_scopes = "none" if credentials_scopes.blank?
what = case credentials_type
when :keychain_username_password
"macOS keychain GitHub"
when :env_token
"HOMEBREW_GITHUB_API_TOKEN"
end
@credentials_error_message ||= onoe <<~EOS
Your #{what} credentials do not have sufficient scope!
Scopes required: #{needed_scopes}
Scopes present: #{credentials_scopes}
#{CREATE_GITHUB_PAT_MESSAGE}
EOS
end
def open_rest(url, data: nil, data_binary_path: nil, request_method: nil, scopes: [].freeze, parse_json: true)
# This is a no-op if the user is opting out of using the GitHub API.
return block_given? ? yield({}) : {} if Homebrew::EnvConfig.no_github_api?
args = ["--header", "Accept: application/vnd.github.v3+json", "--write-out", "\n%\{http_code}"]
args += ["--header", "Accept: application/vnd.github.antiope-preview+json"]
token = credentials
args += ["--header", "Authorization: token #{token}"] unless credentials_type == :none
data_tmpfile = nil
if data
begin
data = JSON.generate data
data_tmpfile = Tempfile.new("github_api_post", HOMEBREW_TEMP)
rescue JSON::ParserError => e
raise Error, "Failed to parse JSON request:\n#{e.message}\n#{data}", e.backtrace
end
end
if data_binary_path.present?
args += ["--data-binary", "@#{data_binary_path}"]
args += ["--header", "Content-Type: application/gzip"]
end
headers_tmpfile = Tempfile.new("github_api_headers", HOMEBREW_TEMP)
begin
if data
data_tmpfile.write data
data_tmpfile.close
args += ["--data", "@#{data_tmpfile.path}"]
args += ["--request", request_method.to_s] if request_method
end
args += ["--dump-header", headers_tmpfile.path]
output, errors, status = curl_output("--location", url.to_s, *args, secrets: [token])
output, _, http_code = output.rpartition("\n")
output, _, http_code = output.rpartition("\n") if http_code == "000"
headers = headers_tmpfile.read
ensure
if data_tmpfile
data_tmpfile.close
data_tmpfile.unlink
end
headers_tmpfile.close
headers_tmpfile.unlink
end
begin
raise_error(output, errors, http_code, headers, scopes) if !http_code.start_with?("2") || !status.success?
return if http_code == "204" # No Content
output = JSON.parse output if parse_json
if block_given?
yield output
else
output
end
rescue JSON::ParserError => e
raise Error, "Failed to parse JSON response\n#{e.message}", e.backtrace
end
end
def open_graphql(query, scopes: [].freeze)
data = { query: query }
result = open_rest("https://api.github.com/graphql", scopes: scopes, data: data, request_method: "POST")
if result["errors"].present?
raise Error, result["errors"].map { |e|
"#{e["type"]}: #{e["message"]}"
}.join("\n")
end
result["data"]
end
def raise_error(output, errors, http_code, headers, scopes)
json = begin
JSON.parse(output)
rescue
nil
end
message = json&.[]("message") || "curl failed! #{errors}"
meta = {}
headers.lines.each do |l|
key, _, value = l.delete(":").partition(" ")
key = key.downcase.strip
next if key.empty?
meta[key] = value.strip
end
if meta.fetch("x-ratelimit-remaining", 1).to_i <= 0
reset = meta.fetch("x-ratelimit-reset").to_i
raise RateLimitExceededError.new(reset, message)
end
credentials_error_message(meta, scopes)
case http_code
when "401", "403"
raise AuthenticationFailedError, message
when "404"
raise MissingAuthenticationError if credentials_type == :none && scopes.present?
raise HTTPNotFoundError, message
when "422"
errors = json&.[]("errors") || []
raise ValidationFailedError.new(message, errors)
else
raise Error, message
end
end
end
end

7
Library/Homebrew/utils/shared_audits.rb
View File

@ -17,17 +17,18 @@ module SharedAudits
@github_repo_data["#{user}/#{repo}"] ||= GitHub.repository(user, repo)
@github_repo_data["#{user}/#{repo}"]
rescue GitHub::HTTPNotFoundError
rescue GitHub::API::HTTPNotFoundError
nil
end
def github_release_data(user, repo, tag)
id = "#{user}/#{repo}/#{tag}"
url = "#{GitHub::API_URL}/repos/#{user}/#{repo}/releases/tags/#{tag}"
@github_release_data ||= {}
@github_release_data[id] ||= GitHub.open_api("#{GitHub::API_URL}/repos/#{user}/#{repo}/releases/tags/#{tag}")
@github_release_data[id] ||= GitHub::API.open_rest(url)
@github_release_data[id]
rescue GitHub::HTTPNotFoundError
rescue GitHub::API::HTTPNotFoundError
nil
end

2
Library/Homebrew/utils/spdx.rb
View File

@ -29,7 +29,7 @@ module SPDX
end
def latest_tag
@latest_tag ||= GitHub.open_api(API_URL)["tag_name"]
@latest_tag ||= GitHub::API.open_rest(API_URL)["tag_name"]
end
def download_latest_license_data!(to: DATA_PATH)