diff --git a/Library/Homebrew/cmd/gist-logs.rb b/Library/Homebrew/cmd/gist-logs.rb index 543bd6289a..bdb0fa9b03 100644 --- a/Library/Homebrew/cmd/gist-logs.rb +++ b/Library/Homebrew/cmd/gist-logs.rb @@ -7,12 +7,10 @@ #: If `--new-issue` is passed, automatically create a new issue in the appropriate #: GitHub repository as well as creating the Gist. #: -#: If no logs are found, an error message is presented. +#: If no logs are found, an error message is presented. require "formula" require "system_config" -require "net/http" -require "net/https" require "stringio" require "socket" @@ -46,17 +44,14 @@ module Homebrew url = create_gist(files, descr) if ARGV.include?("--new-issue") || ARGV.switch?("n") - auth = :AUTH_TOKEN - if GitHub.api_credentials_type == :none puts "You can create a personal access token: https://github.com/settings/tokens" puts "and then set HOMEBREW_GITHUB_API_TOKEN as authentication method." puts - - auth = :AUTH_USER_LOGIN + login! end - url = new_issue(f.tap, "#{f.name} failed to build on #{MacOS.full_version}", url, auth) + url = new_issue(f.tap, "#{f.name} failed to build on #{MacOS.full_version}", url) end puts url if url @@ -84,13 +79,12 @@ module Homebrew result end - def login(request) + def login! print "GitHub User: " - user = $stdin.gets.chomp + ENV["HOMEBREW_GITHUB_API_USERNAME"] = $stdin.gets.chomp print "Password: " - password = noecho_gets.chomp + ENV["HOMEBREW_GITHUB_API_PASSWORD"] = noecho_gets.chomp puts - request.basic_auth(user, password) end def load_logs(dir) @@ -103,77 +97,19 @@ module Homebrew logs end - def create_gist(files, descr) - post("/gists", { "public" => true, "files" => files, "description" => descr })["html_url"] + def create_gist(files, description) + data = { "public" => true, "files" => files, "description" => description } + GitHub.open("https://api.github.com/gists", data)["html_url"] end - def new_issue(repo, title, body, auth) - post("/repos/#{repo}/issues", { "title" => title, "body" => body }, auth)["html_url"] - end - - def http - @http ||= begin - uri = URI.parse("https://api.github.com") - p = ENV["http_proxy"] ? URI.parse(ENV["http_proxy"]) : nil - if p.class == URI::HTTP || p.class == URI::HTTPS - @http = Net::HTTP.new(uri.host, uri.port, p.host, p.port, p.user, p.password) - else - @http = Net::HTTP.new(uri.host, uri.port) - end - @http.use_ssl = true - @http - end - end - - def make_request(path, data, auth) - headers = GitHub.api_headers - headers["Content-Type"] = "application/json" - - basic_auth_credentials = nil - if auth != :AUTH_USER_LOGIN - token, username = GitHub.api_credentials - case GitHub.api_credentials_type - when :keychain - basic_auth_credentials = [username, token] - when :environment - headers["Authorization"] = "token #{token}" - end - end - - request = Net::HTTP::Post.new(path, headers) - request.basic_auth(*basic_auth_credentials) if basic_auth_credentials - - login(request) if auth == :AUTH_USER_LOGIN - - request.body = Utils::JSON.dump(data) - request - end - - def post(path, data, auth = nil) - request = make_request(path, data, auth) - - case response = http.request(request) - when Net::HTTPCreated - Utils::JSON.load get_body(response) - else - GitHub.api_credentials_error_message(response) - raise "HTTP #{response.code} #{response.message} (expected 201)" - end - end - - def get_body(response) - if !response.body.respond_to?(:force_encoding) - response.body - elsif response["Content-Type"].downcase == "application/json; charset=utf-8" - response.body.dup.force_encoding(Encoding::UTF_8) - else - response.body.encode(Encoding::UTF_8, :undef => :replace) - end + def new_issue(repo, title, body) + data = { "title" => title, "body" => body } + GitHub.open("https://api.github.com/repos/MikeMcQuaid/test/issues", data)["html_url"] end def gist_logs raise FormulaUnspecifiedError if ARGV.resolved_formulae.length != 1 - gistify_logs(ARGV.resolved_formulae[0]) + gistify_logs(ARGV.resolved_formulae.first) end end diff --git a/Library/Homebrew/cmd/pull.rb b/Library/Homebrew/cmd/pull.rb index 0aca8a1ec5..3a90e9229a 100644 --- a/Library/Homebrew/cmd/pull.rb +++ b/Library/Homebrew/cmd/pull.rb @@ -503,6 +503,7 @@ module Homebrew url = URI(bottle_info.url) puts "Verifying bottle: #{File.basename(url.path)}" http = Net::HTTP.new(url.host, url.port) + http.initialize_http_header "User-Agent" => HOMEBREW_USER_AGENT_RUBY http.use_ssl = true retry_count = 0 http.start do diff --git a/Library/Homebrew/global.rb b/Library/Homebrew/global.rb index 4a147272f4..52bee21660 100644 --- a/Library/Homebrew/global.rb +++ b/Library/Homebrew/global.rb @@ -30,7 +30,13 @@ RUBY_BIN = RUBY_PATH.dirname HOMEBREW_USER_AGENT_CURL = ENV["HOMEBREW_USER_AGENT_CURL"] HOMEBREW_USER_AGENT_RUBY = "#{ENV["HOMEBREW_USER_AGENT"]} ruby/#{RUBY_VERSION}-p#{RUBY_PATCHLEVEL}" -HOMEBREW_CURL_ARGS = "-f#RLA" +HOMEBREW_CURL_ARGS = [ + "--fail", + "--progress-bar", + "--remote-time", + "--location", + "--user-agent", HOMEBREW_USER_AGENT_CURL +].freeze require "tap_constants" diff --git a/Library/Homebrew/utils.rb b/Library/Homebrew/utils.rb index 09bab2c540..ef7776bcad 100644 --- a/Library/Homebrew/utils.rb +++ b/Library/Homebrew/utils.rb @@ -7,7 +7,8 @@ require "utils/popen" require "utils/fork" require "utils/git" require "utils/analytics" -require "open-uri" +require "utils/github" +require "utils/curl" class Tty class << self @@ -343,21 +344,6 @@ def quiet_system(cmd, *args) end end -def curl(*args) - curl = Pathname.new ENV["HOMEBREW_CURL"] - curl = Pathname.new "/usr/bin/curl" unless curl.exist? - raise "#{curl} is not executable" unless curl.exist? && curl.executable? - - flags = HOMEBREW_CURL_ARGS - flags = flags.delete("#") if ARGV.verbose? - - args = [flags, HOMEBREW_USER_AGENT_CURL, *args] - args << "--verbose" if ENV["HOMEBREW_CURL_VERBOSE"] - args << "--silent" if !$stdout.tty? || ENV["TRAVIS"] - - safe_system curl, *args -end - def puts_columns(items) return if items.empty? @@ -519,230 +505,6 @@ def shell_profile end end -module GitHub - extend self - ISSUES_URI = URI.parse("https://api.github.com/search/issues") - - Error = Class.new(RuntimeError) - HTTPNotFoundError = Class.new(Error) - - class RateLimitExceededError < Error - def initialize(reset, error) - super <<-EOS.undent - GitHub API Error: #{error} - Try again in #{pretty_ratelimit_reset(reset)}, or create a personal access token: - #{Tty.em}https://github.com/settings/tokens/new?scopes=&description=Homebrew#{Tty.reset} - and then set the token as: export HOMEBREW_GITHUB_API_TOKEN="your_new_token" - EOS - end - - def pretty_ratelimit_reset(reset) - pretty_duration(Time.at(reset) - Time.now) - end - end - - class AuthenticationFailedError < Error - def initialize(error) - message = "GitHub #{error}\n" - if ENV["HOMEBREW_GITHUB_API_TOKEN"] - message << <<-EOS.undent - HOMEBREW_GITHUB_API_TOKEN may be invalid or expired; check: - #{Tty.em}https://github.com/settings/tokens#{Tty.reset} - EOS - else - message << <<-EOS.undent - The GitHub credentials in the OS X keychain may be invalid. - Clear them with: - printf "protocol=https\\nhost=github.com\\n" | git credential-osxkeychain erase - Or create a personal access token: - #{Tty.em}https://github.com/settings/tokens/new?scopes=&description=Homebrew#{Tty.reset} - and then set the token as: export HOMEBREW_GITHUB_API_TOKEN="your_new_token" - EOS - end - super message - end - end - - def api_credentials - @api_credentials ||= begin - if ENV["HOMEBREW_GITHUB_API_TOKEN"] - ENV["HOMEBREW_GITHUB_API_TOKEN"] - else - github_credentials = Utils.popen("git credential-osxkeychain get", "w+") do |io| - io.puts "protocol=https\nhost=github.com" - io.close_write - io.read - end - github_username = github_credentials[/username=(.+)/, 1] - github_password = github_credentials[/password=(.+)/, 1] - if github_username && github_password - [github_password, github_username] - else - [] - end - end - end - end - - def api_credentials_type - token, username = api_credentials - if token && !token.empty? - if username && !username.empty? - :keychain - else - :environment - end - else - :none - end - end - - def api_credentials_error_message(response_headers) - @api_credentials_error_message_printed ||= begin - unauthorized = (response_headers["status"] == "401 Unauthorized") - scopes = response_headers["x-accepted-oauth-scopes"].to_s.split(", ") - if !unauthorized && scopes.empty? - credentials_scopes = response_headers["x-oauth-scopes"].to_s.split(", ") - - case GitHub.api_credentials_type - when :keychain - onoe <<-EOS.undent - Your OS X keychain GitHub credentials do not have sufficient scope! - Scopes they have: #{credentials_scopes} - Create a personal access token: https://github.com/settings/tokens - and then set HOMEBREW_GITHUB_API_TOKEN as the authentication method instead. - EOS - when :environment - onoe <<-EOS.undent - Your HOMEBREW_GITHUB_API_TOKEN does not have sufficient scope! - Scopes it has: #{credentials_scopes} - Create a new personal access token: https://github.com/settings/tokens - and then set the new HOMEBREW_GITHUB_API_TOKEN as the authentication method instead. - EOS - end - end - true - end - end - - def api_headers - { - "User-Agent" => HOMEBREW_USER_AGENT_RUBY, - "Accept" => "application/vnd.github.v3+json" - } - end - - def open(url, &_block) - # This is a no-op if the user is opting out of using the GitHub API. - return if ENV["HOMEBREW_NO_GITHUB_API"] - - require "net/https" - - headers = api_headers - token, username = api_credentials - case api_credentials_type - when :keychain - headers[:http_basic_authentication] = [username, token] - when :environment - headers["Authorization"] = "token #{token}" - end - - begin - Kernel.open(url, headers) { |f| yield Utils::JSON.load(f.read) } - rescue OpenURI::HTTPError => e - handle_api_error(e) - rescue EOFError, SocketError, OpenSSL::SSL::SSLError => e - raise Error, "Failed to connect to: #{url}\n#{e.message}", e.backtrace - rescue Utils::JSON::Error => e - raise Error, "Failed to parse JSON response\n#{e.message}", e.backtrace - end - end - - def handle_api_error(e) - if e.io.meta.fetch("x-ratelimit-remaining", 1).to_i <= 0 - reset = e.io.meta.fetch("x-ratelimit-reset").to_i - error = Utils::JSON.load(e.io.read)["message"] - raise RateLimitExceededError.new(reset, error) - end - - GitHub.api_credentials_error_message(e.io.meta) - - case e.io.status.first - when "401", "403" - raise AuthenticationFailedError.new(e.message) - when "404" - raise HTTPNotFoundError, e.message, e.backtrace - else - error = Utils::JSON.load(e.io.read)["message"] rescue nil - raise Error, [e.message, error].compact.join("\n"), e.backtrace - end - end - - def issues_matching(query, qualifiers = {}) - uri = ISSUES_URI.dup - uri.query = build_query_string(query, qualifiers) - open(uri) { |json| json["items"] } - end - - def repository(user, repo) - open(URI.parse("https://api.github.com/repos/#{user}/#{repo}")) { |j| j } - end - - def build_query_string(query, qualifiers) - s = "q=#{uri_escape(query)}+" - s << build_search_qualifier_string(qualifiers) - s << "&per_page=100" - end - - def build_search_qualifier_string(qualifiers) - { - :repo => "Homebrew/homebrew-core", - :in => "title" - }.update(qualifiers).map do |qualifier, value| - "#{qualifier}:#{value}" - end.join("+") - end - - def uri_escape(query) - if URI.respond_to?(:encode_www_form_component) - URI.encode_www_form_component(query) - else - require "erb" - ERB::Util.url_encode(query) - end - end - - def issues_for_formula(name, options = {}) - tap = options[:tap] || CoreTap.instance - issues_matching(name, :state => "open", :repo => "#{tap.user}/homebrew-#{tap.repo}") - end - - def print_pull_requests_matching(query) - return [] if ENV["HOMEBREW_NO_GITHUB_API"] - ohai "Searching pull requests..." - - open_or_closed_prs = issues_matching(query, :type => "pr") - - open_prs = open_or_closed_prs.select { |i| i["state"] == "open" } - if open_prs.any? - puts "Open pull requests:" - prs = open_prs - elsif open_or_closed_prs.any? - puts "Closed pull requests:" - prs = open_or_closed_prs - else - return - end - - prs.each { |i| puts "#{i["title"]} (#{i["html_url"]})" } - end - - def private_repo?(user, repo) - uri = URI.parse("https://api.github.com/repos/#{user}/#{repo}") - open(uri) { |json| json["private"] } - end -end - def disk_usage_readable(size_in_bytes) if size_in_bytes >= 1_073_741_824 size = size_in_bytes.to_f / 1_073_741_824 diff --git a/Library/Homebrew/utils/curl.rb b/Library/Homebrew/utils/curl.rb new file mode 100644 index 0000000000..a1653d46ca --- /dev/null +++ b/Library/Homebrew/utils/curl.rb @@ -0,0 +1,24 @@ +require "pathname" + +def curl_args(extra_args=[]) + curl = Pathname.new ENV["HOMEBREW_CURL"] + curl = Pathname.new "/usr/bin/curl" unless curl.exist? + raise "#{curl} is not executable" unless curl.exist? && curl.executable? + + flags = HOMEBREW_CURL_ARGS + flags -= ["--progress-bar"] if ARGV.verbose? + + args = ["#{curl}"] + flags + extra_args + args << "--verbose" if ENV["HOMEBREW_CURL_VERBOSE"] + args << "--silent" if !$stdout.tty? || ENV["TRAVIS"] + args +end + +def curl(*args) + safe_system(*curl_args(args)) +end + +def curl_output(*args) + curl_args = curl_args(args) - ["--fail"] + Utils.popen_read_text(*curl_args) +end diff --git a/Library/Homebrew/utils/github.rb b/Library/Homebrew/utils/github.rb new file mode 100644 index 0000000000..7fd4842110 --- /dev/null +++ b/Library/Homebrew/utils/github.rb @@ -0,0 +1,267 @@ +require "uri" +require "tempfile" + +module GitHub + extend self + ISSUES_URI = URI.parse("https://api.github.com/search/issues") + + Error = Class.new(RuntimeError) + HTTPNotFoundError = Class.new(Error) + + class RateLimitExceededError < Error + def initialize(reset, error) + super <<-EOS.undent + GitHub API Error: #{error} + Try again in #{pretty_ratelimit_reset(reset)}, or create a personal access token: + #{Tty.em}https://github.com/settings/tokens/new?scopes=&description=Homebrew#{Tty.reset} + and then set the token as: export HOMEBREW_GITHUB_API_TOKEN="your_new_token" + EOS + end + + def pretty_ratelimit_reset(reset) + pretty_duration(Time.at(reset) - Time.now) + end + end + + class AuthenticationFailedError < Error + def initialize(error) + message = "GitHub #{error}\n" + if ENV["HOMEBREW_GITHUB_API_TOKEN"] + message << <<-EOS.undent + HOMEBREW_GITHUB_API_TOKEN may be invalid or expired; check: + #{Tty.em}https://github.com/settings/tokens#{Tty.reset} + EOS + else + message << <<-EOS.undent + The GitHub credentials in the OS X keychain may be invalid. + Clear them with: + printf "protocol=https\\nhost=github.com\\n" | git credential-osxkeychain erase + Or create a personal access token: + #{Tty.em}https://github.com/settings/tokens/new?scopes=&description=Homebrew#{Tty.reset} + and then set the token as: export HOMEBREW_GITHUB_API_TOKEN="your_new_token" + EOS + end + super message + end + end + + def api_credentials + @api_credentials ||= begin + if ENV["HOMEBREW_GITHUB_API_TOKEN"] + ENV["HOMEBREW_GITHUB_API_TOKEN"] + elsif ENV["HOMEBREW_GITHUB_API_USERNAME"] && ENV["HOMEBREW_GITHUB_API_PASSWORD"] + [ENV["HOMEBREW_GITHUB_API_USERNAME"], ENV["HOMEBREW_GITHUB_API_PASSWORD"]] + else + github_credentials = Utils.popen("git credential-osxkeychain get", "w+") do |io| + io.puts "protocol=https\nhost=github.com" + io.close_write + io.read + end + github_username = github_credentials[/username=(.+)/, 1] + github_password = github_credentials[/password=(.+)/, 1] + if github_username && github_password + [github_password, github_username] + else + [] + end + end + end + end + + def api_credentials_type + token, username = api_credentials + if token && !token.empty? + if username && !username.empty? + :keychain + else + :environment + end + else + :none + end + end + + def api_credentials_error_message(response_headers) + return if response_headers.empty? + + @api_credentials_error_message_printed ||= begin + unauthorized = (response_headers["http/1.1"] == "401 Unauthorized") + scopes = response_headers["x-accepted-oauth-scopes"].to_s.split(", ") + if !unauthorized && scopes.empty? + credentials_scopes = response_headers["x-oauth-scopes"].to_s.split(", ") + + case GitHub.api_credentials_type + when :keychain + onoe <<-EOS.undent + Your OS X keychain GitHub credentials do not have sufficient scope! + Scopes they have: #{credentials_scopes} + Create a personal access token: https://github.com/settings/tokens + and then set HOMEBREW_GITHUB_API_TOKEN as the authentication method instead. + EOS + when :environment + onoe <<-EOS.undent + Your HOMEBREW_GITHUB_API_TOKEN does not have sufficient scope! + Scopes it has: #{credentials_scopes} + Create a new personal access token: https://github.com/settings/tokens + and then set the new HOMEBREW_GITHUB_API_TOKEN as the authentication method instead. + EOS + end + end + true + end + end + + def open(url, data=nil) + # This is a no-op if the user is opting out of using the GitHub API. + return if ENV["HOMEBREW_NO_GITHUB_API"] + + args = %W[--header application/vnd.github.v3+json --write-out \n%{http_code}] + args += curl_args + + token, username = api_credentials + case api_credentials_type + when :keychain + args += %W[--user #{username}:#{token}] + when :environment + args += ["--header", "Authorization: token #{token}"] + end + + data_tmpfile = nil + if data + begin + data = Utils::JSON.dump data + data_tmpfile = Tempfile.new("github_api_post", HOMEBREW_TEMP) + rescue Utils::JSON::Error => e + raise Error, "Failed to parse JSON request:\n#{e.message}\n#{data}", e.backtrace + end + end + + headers_tmpfile = Tempfile.new("github_api_headers", HOMEBREW_TEMP) + begin + if data + data_tmpfile.write data + args += ["--data", "@#{data_tmpfile.path}"] + end + + args += ["--dump-header", "#{headers_tmpfile.path}"] + + output, _, http_code = curl_output(url.to_s, *args).rpartition("\n") + output, _, http_code = output.rpartition("\n") if http_code == "000" + headers = headers_tmpfile.read + ensure + if data_tmpfile + data_tmpfile.close + data_tmpfile.unlink + end + headers_tmpfile.close + headers_tmpfile.unlink + end + + begin + if !http_code.start_with?("2") && !$?.success? + raise_api_error(output, http_code, headers) + end + json = Utils::JSON.load output + if block_given? + yield json + else + json + end + rescue Utils::JSON::Error => e + raise Error, "Failed to parse JSON response\n#{e.message}", e.backtrace + end + end + + def raise_api_error(output, http_code, headers) + meta = {} + headers.lines.each do |l| + key, _, value = l.delete(":").partition(" ") + key = key.downcase.strip + next if key.empty? + meta[key] = value.strip + end + + if meta.fetch("x-ratelimit-remaining", 1).to_i <= 0 + reset = meta.fetch("x-ratelimit-reset").to_i + error = Utils::JSON.load(output)["message"] + raise RateLimitExceededError.new(reset, error) + end + + GitHub.api_credentials_error_message(meta) + + case http_code + when "401", "403" + raise AuthenticationFailedError.new(output) + when "404" + raise HTTPNotFoundError, output + else + error = Utils::JSON.load(output)["message"] rescue nil + error ||= output + raise Error, error + end + end + + def issues_matching(query, qualifiers = {}) + uri = ISSUES_URI.dup + uri.query = build_query_string(query, qualifiers) + open(uri) { |json| json["items"] } + end + + def repository(user, repo) + open(URI.parse("https://api.github.com/repos/#{user}/#{repo}")) { |j| j } + end + + def build_query_string(query, qualifiers) + s = "q=#{uri_escape(query)}+" + s << build_search_qualifier_string(qualifiers) + s << "&per_page=100" + end + + def build_search_qualifier_string(qualifiers) + { + :repo => "Homebrew/homebrew-core", + :in => "title" + }.update(qualifiers).map do |qualifier, value| + "#{qualifier}:#{value}" + end.join("+") + end + + def uri_escape(query) + if URI.respond_to?(:encode_www_form_component) + URI.encode_www_form_component(query) + else + require "erb" + ERB::Util.url_encode(query) + end + end + + def issues_for_formula(name, options = {}) + tap = options[:tap] || CoreTap.instance + issues_matching(name, :state => "open", :repo => "#{tap.user}/homebrew-#{tap.repo}") + end + + def print_pull_requests_matching(query) + return [] if ENV["HOMEBREW_NO_GITHUB_API"] + ohai "Searching pull requests..." + + open_or_closed_prs = issues_matching(query, :type => "pr") + + open_prs = open_or_closed_prs.select { |i| i["state"] == "open" } + if open_prs.any? + puts "Open pull requests:" + prs = open_prs + elsif open_or_closed_prs.any? + puts "Closed pull requests:" + prs = open_or_closed_prs + else + return + end + + prs.each { |i| puts "#{i["title"]} (#{i["html_url"]})" } + end + + def private_repo?(user, repo) + uri = URI.parse("https://api.github.com/repos/#{user}/#{repo}") + open(uri) { |json| json["private"] } + end +end