From ff653571b19d7716de12c82bfec606aac347c409 Mon Sep 17 00:00:00 2001 From: Markus Reiter Date: Sat, 10 Oct 2020 15:23:03 +0200 Subject: [PATCH] Create `Utils::Curl` module and explicitly include it. --- Library/Homebrew/download_strategy.rb | 4 + .../Homebrew/sorbet/rbi/utils/notability.rbi | 15 - Library/Homebrew/utils/curl.rb | 512 +++++++++--------- Library/Homebrew/utils/shared_audits.rb | 3 + Library/Homebrew/utils/spdx.rb | 3 + 5 files changed, 274 insertions(+), 263 deletions(-) delete mode 100644 Library/Homebrew/sorbet/rbi/utils/notability.rbi diff --git a/Library/Homebrew/download_strategy.rb b/Library/Homebrew/download_strategy.rb index 35950630b2..6192391b78 100644 --- a/Library/Homebrew/download_strategy.rb +++ b/Library/Homebrew/download_strategy.rb @@ -11,6 +11,8 @@ require "lock_file" require "mechanize/version" require "mechanize/http/content_disposition_parser" +require "utils/curl" + # @abstract Abstract superclass for all download strategies. # # @api private @@ -328,6 +330,8 @@ end # # @api public class CurlDownloadStrategy < AbstractFileDownloadStrategy + include Utils::Curl + attr_reader :mirrors def initialize(url, name, version, **meta) diff --git a/Library/Homebrew/sorbet/rbi/utils/notability.rbi b/Library/Homebrew/sorbet/rbi/utils/notability.rbi deleted file mode 100644 index 546b4c8c62..0000000000 --- a/Library/Homebrew/sorbet/rbi/utils/notability.rbi +++ /dev/null @@ -1,15 +0,0 @@ -# typed: strict - -module SharedAudits - def github(user, repo) - end - - def gitlab(user, repo) - end - - def bitbucket(user, repo) - end - - def curl_output(*args, secrets: [], **options) - end -end diff --git a/Library/Homebrew/utils/curl.rb b/Library/Homebrew/utils/curl.rb index db53efb9ee..7229bd2e2a 100644 --- a/Library/Homebrew/utils/curl.rb +++ b/Library/Homebrew/utils/curl.rb @@ -3,259 +3,275 @@ require "open3" -def curl_executable - @curl ||= [ - ENV["HOMEBREW_CURL"], - which("curl"), - "/usr/bin/curl", - ].compact.map { |c| Pathname(c) }.find(&:executable?) - raise "no executable curl was found" unless @curl +module Utils + # Helper function for interacting with `curl`. + # + # @api private + module Curl + module_function - @curl -end + def curl_executable + @curl ||= [ + ENV["HOMEBREW_CURL"], + which("curl"), + "/usr/bin/curl", + ].compact.map { |c| Pathname(c) }.find(&:executable?) + raise "no executable curl was found" unless @curl -def curl_args(*extra_args, show_output: false, user_agent: :default) - args = [] - - # do not load .curlrc unless requested (must be the first argument) - args << "--disable" unless Homebrew::EnvConfig.curlrc? - - args << "--globoff" - - args << "--show-error" - - args << "--user-agent" << case user_agent - when :browser, :fake - HOMEBREW_USER_AGENT_FAKE_SAFARI - when :default - HOMEBREW_USER_AGENT_CURL - else - user_agent - end - - args << "--header" << "Accept-Language: en" - - unless show_output - args << "--fail" - args << "--progress-bar" unless Context.current.verbose? - args << "--verbose" if Homebrew::EnvConfig.curl_verbose? - args << "--silent" unless $stdout.tty? - end - - args << "--retry" << Homebrew::EnvConfig.curl_retries - - args + extra_args -end - -def curl_with_workarounds(*args, secrets: nil, print_stdout: nil, print_stderr: nil, verbose: nil, env: {}, **options) - command_options = { - secrets: secrets, - print_stdout: print_stdout, - print_stderr: print_stderr, - verbose: verbose, - }.compact - - # SSL_CERT_FILE can be incorrectly set by users or portable-ruby and screw - # with SSL downloads so unset it here. - result = system_command curl_executable, - args: curl_args(*args, **options), - env: { "SSL_CERT_FILE" => nil }.merge(env), - **command_options - - if !result.success? && !args.include?("--http1.1") - # This is a workaround for https://github.com/curl/curl/issues/1618. - if result.status.exitstatus == 56 # Unexpected EOF - out = curl_output("-V").stdout - - # If `curl` doesn't support HTTP2, the exception is unrelated to this bug. - return result unless out.include?("HTTP2") - - # The bug is fixed in `curl` >= 7.60.0. - curl_version = out[/curl (\d+(\.\d+)+)/, 1] - return result if Gem::Version.new(curl_version) >= Gem::Version.new("7.60.0") - - return curl_with_workarounds(*args, "--http1.1", **command_options, **options) + @curl end - if result.status.exitstatus == 16 # Error in the HTTP2 framing layer - return curl_with_workarounds(*args, "--http1.1", **command_options, **options) + def curl_args(*extra_args, show_output: false, user_agent: :default) + args = [] + + # do not load .curlrc unless requested (must be the first argument) + args << "--disable" unless Homebrew::EnvConfig.curlrc? + + args << "--globoff" + + args << "--show-error" + + args << "--user-agent" << case user_agent + when :browser, :fake + HOMEBREW_USER_AGENT_FAKE_SAFARI + when :default + HOMEBREW_USER_AGENT_CURL + else + user_agent + end + + args << "--header" << "Accept-Language: en" + + unless show_output + args << "--fail" + args << "--progress-bar" unless Context.current.verbose? + args << "--verbose" if Homebrew::EnvConfig.curl_verbose? + args << "--silent" unless $stdout.tty? + end + + args << "--retry" << Homebrew::EnvConfig.curl_retries + + args + extra_args + end + + def curl_with_workarounds( + *args, secrets: nil, print_stdout: nil, print_stderr: nil, verbose: nil, env: {}, **options + ) + command_options = { + secrets: secrets, + print_stdout: print_stdout, + print_stderr: print_stderr, + verbose: verbose, + }.compact + + # SSL_CERT_FILE can be incorrectly set by users or portable-ruby and screw + # with SSL downloads so unset it here. + result = system_command curl_executable, + args: curl_args(*args, **options), + env: { "SSL_CERT_FILE" => nil }.merge(env), + **command_options + + if !result.success? && !args.include?("--http1.1") + # This is a workaround for https://github.com/curl/curl/issues/1618. + if result.status.exitstatus == 56 # Unexpected EOF + out = curl_output("-V").stdout + + # If `curl` doesn't support HTTP2, the exception is unrelated to this bug. + return result unless out.include?("HTTP2") + + # The bug is fixed in `curl` >= 7.60.0. + curl_version = out[/curl (\d+(\.\d+)+)/, 1] + return result if Gem::Version.new(curl_version) >= Gem::Version.new("7.60.0") + + return curl_with_workarounds(*args, "--http1.1", **command_options, **options) + end + + if result.status.exitstatus == 16 # Error in the HTTP2 framing layer + return curl_with_workarounds(*args, "--http1.1", **command_options, **options) + end + end + + result + end + + def curl(*args, print_stdout: true, **options) + result = curl_with_workarounds(*args, print_stdout: print_stdout, **options) + result.assert_success! + result + end + + def curl_download(*args, to: nil, partial: true, **options) + destination = Pathname(to) + destination.dirname.mkpath + + if partial + range_stdout = curl_output("--location", "--range", "0-1", + "--dump-header", "-", + "--write-out", "%\{http_code}", + "--output", "/dev/null", *args, **options).stdout + headers, _, http_status = range_stdout.partition("\r\n\r\n") + + supports_partial_download = http_status.to_i == 206 # Partial Content + if supports_partial_download && + destination.exist? && + destination.size == %r{^.*Content-Range: bytes \d+-\d+/(\d+)\r\n.*$}m.match(headers)&.[](1)&.to_i + return # We've already downloaded all the bytes + end + else + supports_partial_download = false + end + + continue_at = if destination.exist? && supports_partial_download + "-" + else + 0 + end + + curl( + "--location", "--remote-time", "--continue-at", continue_at.to_s, "--output", destination, *args, **options + ) + end + + def curl_output(*args, **options) + curl_with_workarounds(*args, print_stderr: false, show_output: true, **options) + end + + # Check if a URL is protected by CloudFlare (e.g. badlion.net and jaxx.io). + def url_protected_by_cloudflare?(details) + [403, 503].include?(details[:status].to_i) && + details[:headers].match?(/^Set-Cookie: __cfduid=/i) && + details[:headers].match?(/^Server: cloudflare/i) + end + + # Check if a URL is protected by Incapsula (e.g. corsair.com). + def url_protected_by_incapsula?(details) + details[:status].to_i == 403 && + details[:headers].match?(/^Set-Cookie: visid_incap_/i) && + details[:headers].match?(/^Set-Cookie: incap_ses_/i) + end + + def curl_check_http_content(url, user_agents: [:default], check_content: false, strict: false) + return unless url.start_with? "http" + + details = nil + user_agent = nil + hash_needed = url.start_with?("http:") + user_agents.each do |ua| + details = curl_http_content_headers_and_checksum(url, hash_needed: hash_needed, user_agent: ua) + user_agent = ua + break if http_status_ok?(details[:status]) + end + + unless details[:status] + # Hack around https://github.com/Homebrew/brew/issues/3199 + return if MacOS.version == :el_capitan + + return "The URL #{url} is not reachable" + end + + unless http_status_ok?(details[:status]) + return if url_protected_by_cloudflare?(details) || url_protected_by_incapsula?(details) + + return "The URL #{url} is not reachable (HTTP status code #{details[:status]})" + end + + if url.start_with?("https://") && Homebrew::EnvConfig.no_insecure_redirect? && + !details[:final_url].start_with?("https://") + return "The URL #{url} redirects back to HTTP" + end + + return unless hash_needed + + secure_url = url.sub "http", "https" + secure_details = + curl_http_content_headers_and_checksum(secure_url, hash_needed: true, user_agent: user_agent) + + if !http_status_ok?(details[:status]) || + !http_status_ok?(secure_details[:status]) + return + end + + etag_match = details[:etag] && + details[:etag] == secure_details[:etag] + content_length_match = + details[:content_length] && + details[:content_length] == secure_details[:content_length] + file_match = details[:file_hash] == secure_details[:file_hash] + + if (etag_match || content_length_match || file_match) && + secure_details[:final_url].start_with?("https://") && + url.start_with?("http://") + return "The URL #{url} should use HTTPS rather than HTTP" + end + + return unless check_content + + no_protocol_file_contents = %r{https?:\\?/\\?/} + details[:file] = details[:file].gsub(no_protocol_file_contents, "/") + secure_details[:file] = secure_details[:file].gsub(no_protocol_file_contents, "/") + + # Check for the same content after removing all protocols + if (details[:file] == secure_details[:file]) && + secure_details[:final_url].start_with?("https://") && + url.start_with?("http://") + return "The URL #{url} should use HTTPS rather than HTTP" + end + + return unless strict + + # Same size, different content after normalization + # (typical causes: Generated ID, Timestamp, Unix time) + if details[:file].length == secure_details[:file].length + return "The URL #{url} may be able to use HTTPS rather than HTTP. Please verify it in a browser." + end + + lenratio = (100 * secure_details[:file].length / details[:file].length).to_i + return unless (90..110).cover?(lenratio) + + "The URL #{url} may be able to use HTTPS rather than HTTP. Please verify it in a browser." + end + + def curl_http_content_headers_and_checksum(url, hash_needed: false, user_agent: :default) + file = Tempfile.new.tap(&:close) + + max_time = hash_needed ? "600" : "25" + output, = curl_output( + "--dump-header", "-", "--output", file.path, "--include", "--location", + "--connect-timeout", "15", "--max-time", max_time, url, + user_agent: user_agent + ) + + status_code = :unknown + while status_code == :unknown || status_code.to_s.start_with?("3") + headers, _, output = output.partition("\r\n\r\n") + status_code = headers[%r{HTTP/.* (\d+)}, 1] + location = headers[/^Location:\s*(.*)$/i, 1] + final_url = location.chomp if location + end + + output_hash = Digest::SHA256.file(file.path) if hash_needed + + final_url ||= url + + { + url: url, + final_url: final_url, + status: status_code, + etag: headers[%r{ETag: ([wW]/)?"(([^"]|\\")*)"}, 2], + content_length: headers[/Content-Length: (\d+)/, 1], + headers: headers, + file_hash: output_hash, + file: output, + } + ensure + file.unlink + end + + def http_status_ok?(status) + (100..299).cover?(status.to_i) end end - - result end -def curl(*args, print_stdout: true, **options) - result = curl_with_workarounds(*args, print_stdout: print_stdout, **options) - result.assert_success! - result -end - -def curl_download(*args, to: nil, partial: true, **options) - destination = Pathname(to) - destination.dirname.mkpath - - if partial - range_stdout = curl_output("--location", "--range", "0-1", - "--dump-header", "-", - "--write-out", "%\{http_code}", - "--output", "/dev/null", *args, **options).stdout - headers, _, http_status = range_stdout.partition("\r\n\r\n") - - supports_partial_download = http_status.to_i == 206 # Partial Content - if supports_partial_download && - destination.exist? && - destination.size == %r{^.*Content-Range: bytes \d+-\d+/(\d+)\r\n.*$}m.match(headers)&.[](1)&.to_i - return # We've already downloaded all the bytes - end - else - supports_partial_download = false - end - - continue_at = if destination.exist? && supports_partial_download - "-" - else - 0 - end - - curl("--location", "--remote-time", "--continue-at", continue_at.to_s, "--output", destination, *args, **options) -end - -def curl_output(*args, **options) - curl_with_workarounds(*args, print_stderr: false, show_output: true, **options) -end - -# Check if a URL is protected by CloudFlare (e.g. badlion.net and jaxx.io). -def url_protected_by_cloudflare?(details) - [403, 503].include?(details[:status].to_i) && - details[:headers].match?(/^Set-Cookie: __cfduid=/i) && - details[:headers].match?(/^Server: cloudflare/i) -end - -# Check if a URL is protected by Incapsula (e.g. corsair.com). -def url_protected_by_incapsula?(details) - details[:status].to_i == 403 && - details[:headers].match?(/^Set-Cookie: visid_incap_/i) && - details[:headers].match?(/^Set-Cookie: incap_ses_/i) -end - -def curl_check_http_content(url, user_agents: [:default], check_content: false, strict: false) - return unless url.start_with? "http" - - details = nil - user_agent = nil - hash_needed = url.start_with?("http:") - user_agents.each do |ua| - details = curl_http_content_headers_and_checksum(url, hash_needed: hash_needed, user_agent: ua) - user_agent = ua - break if http_status_ok?(details[:status]) - end - - unless details[:status] - # Hack around https://github.com/Homebrew/brew/issues/3199 - return if MacOS.version == :el_capitan - - return "The URL #{url} is not reachable" - end - - unless http_status_ok?(details[:status]) - return if url_protected_by_cloudflare?(details) || url_protected_by_incapsula?(details) - - return "The URL #{url} is not reachable (HTTP status code #{details[:status]})" - end - - if url.start_with?("https://") && Homebrew::EnvConfig.no_insecure_redirect? && - !details[:final_url].start_with?("https://") - return "The URL #{url} redirects back to HTTP" - end - - return unless hash_needed - - secure_url = url.sub "http", "https" - secure_details = - curl_http_content_headers_and_checksum(secure_url, hash_needed: true, user_agent: user_agent) - - if !http_status_ok?(details[:status]) || - !http_status_ok?(secure_details[:status]) - return - end - - etag_match = details[:etag] && - details[:etag] == secure_details[:etag] - content_length_match = - details[:content_length] && - details[:content_length] == secure_details[:content_length] - file_match = details[:file_hash] == secure_details[:file_hash] - - if (etag_match || content_length_match || file_match) && - secure_details[:final_url].start_with?("https://") && - url.start_with?("http://") - return "The URL #{url} should use HTTPS rather than HTTP" - end - - return unless check_content - - no_protocol_file_contents = %r{https?:\\?/\\?/} - details[:file] = details[:file].gsub(no_protocol_file_contents, "/") - secure_details[:file] = secure_details[:file].gsub(no_protocol_file_contents, "/") - - # Check for the same content after removing all protocols - if (details[:file] == secure_details[:file]) && - secure_details[:final_url].start_with?("https://") && - url.start_with?("http://") - return "The URL #{url} should use HTTPS rather than HTTP" - end - - return unless strict - - # Same size, different content after normalization - # (typical causes: Generated ID, Timestamp, Unix time) - if details[:file].length == secure_details[:file].length - return "The URL #{url} may be able to use HTTPS rather than HTTP. Please verify it in a browser." - end - - lenratio = (100 * secure_details[:file].length / details[:file].length).to_i - return unless (90..110).cover?(lenratio) - - "The URL #{url} may be able to use HTTPS rather than HTTP. Please verify it in a browser." -end - -def curl_http_content_headers_and_checksum(url, hash_needed: false, user_agent: :default) - file = Tempfile.new.tap(&:close) - - max_time = hash_needed ? "600" : "25" - output, = curl_output( - "--dump-header", "-", "--output", file.path, "--include", "--location", - "--connect-timeout", "15", "--max-time", max_time, url, - user_agent: user_agent - ) - - status_code = :unknown - while status_code == :unknown || status_code.to_s.start_with?("3") - headers, _, output = output.partition("\r\n\r\n") - status_code = headers[%r{HTTP/.* (\d+)}, 1] - location = headers[/^Location:\s*(.*)$/i, 1] - final_url = location.chomp if location - end - - output_hash = Digest::SHA256.file(file.path) if hash_needed - - final_url ||= url - - { - url: url, - final_url: final_url, - status: status_code, - etag: headers[%r{ETag: ([wW]/)?"(([^"]|\\")*)"}, 2], - content_length: headers[/Content-Length: (\d+)/, 1], - headers: headers, - file_hash: output_hash, - file: output, - } -ensure - file.unlink -end - -def http_status_ok?(status) - (100..299).cover?(status.to_i) -end +# FIXME: Include `Utils::Curl` explicitly everywhere it is used. +include Utils::Curl # rubocop:disable Style/MixinUsage diff --git a/Library/Homebrew/utils/shared_audits.rb b/Library/Homebrew/utils/shared_audits.rb index 694f659804..ad69344ca3 100644 --- a/Library/Homebrew/utils/shared_audits.rb +++ b/Library/Homebrew/utils/shared_audits.rb @@ -7,6 +7,9 @@ require "utils/curl" # # @api private module SharedAudits + include Utils::Curl + extend Utils::Curl + module_function def github_repo_data(user, repo) diff --git a/Library/Homebrew/utils/spdx.rb b/Library/Homebrew/utils/spdx.rb index 64467c291e..693689f307 100644 --- a/Library/Homebrew/utils/spdx.rb +++ b/Library/Homebrew/utils/spdx.rb @@ -1,12 +1,15 @@ # typed: true # frozen_string_literal: true +require "utils/curl" require "utils/github" # Helper module for updating SPDX license data. # # @api private module SPDX + extend Utils::Curl + module_function DATA_PATH = (HOMEBREW_DATA_PATH/"spdx").freeze