Hello, world!
+ + + EOS + + body[:with_carriage_returns] = body[:default].sub("\n", "\r\n\r\n") + + body[:with_http_status_line] = body[:default].sub("", "HTTP/1.1 200\r\n") + + body + } + describe "curl_args" do let(:args) { ["foo"] } let(:user_agent_string) { "Lorem ipsum dolor sit amet" } @@ -101,4 +268,117 @@ describe "Utils::Curl" do expect(curl_args(*args, show_output: true).join(" ")).not_to include("--fail") end end + + describe "#parse_curl_output" do + it "returns a correct hash when curl output contains response(s) and body" do + expect(parse_curl_output("#{response_text[:ok]}#{body[:default]}")) + .to eq({ responses: [response_hash[:ok]], body: body[:default] }) + expect(parse_curl_output("#{response_text[:ok]}#{body[:with_carriage_returns]}")) + .to eq({ responses: [response_hash[:ok]], body: body[:with_carriage_returns] }) + expect(parse_curl_output("#{response_text[:ok]}#{body[:with_http_status_line]}")) + .to eq({ responses: [response_hash[:ok]], body: body[:with_http_status_line] }) + expect(parse_curl_output("#{response_text[:redirection_to_ok]}#{body[:default]}")) + .to eq({ responses: [response_hash[:redirection], response_hash[:ok]], body: body[:default] }) + expect(parse_curl_output("#{response_text[:redirections_to_ok]}#{body[:default]}")) + .to eq({ + responses: [ + response_hash[:redirection2], + response_hash[:redirection1], + response_hash[:redirection], + response_hash[:ok], + ], + body: body[:default], + }) + end + + it "returns a correct hash when curl output contains HTTP response text and no body" do + expect(parse_curl_output(response_text[:ok])).to eq({ responses: [response_hash[:ok]], body: "" }) + end + + it "returns a correct hash when curl output contains body and no HTTP response text" do + expect(parse_curl_output(body[:default])).to eq({ responses: [], body: body[:default] }) + expect(parse_curl_output(body[:with_carriage_returns])) + .to eq({ responses: [], body: body[:with_carriage_returns] }) + expect(parse_curl_output(body[:with_http_status_line])) + .to eq({ responses: [], body: body[:with_http_status_line] }) + end + + it "returns correct hash when curl output is blank" do + expect(parse_curl_output("")).to eq({ responses: [], body: "" }) + end + end + + describe "#parse_curl_response" do + it "returns a correct hash when given HTTP response text" do + expect(parse_curl_response(response_text[:ok])).to eq(response_hash[:ok]) + expect(parse_curl_response(response_text[:redirection])).to eq(response_hash[:redirection]) + end + + it "returns an empty hash when given an empty string" do + expect(parse_curl_response("")).to eq({}) + end + end + + describe "#curl_response_last_location" do + it "returns the last location header when given an array of HTTP response hashes" do + expect(curl_response_last_location([ + response_hash[:redirection], + response_hash[:ok], + ])).to eq(response_hash[:redirection][:headers]["location"]) + + expect(curl_response_last_location([ + response_hash[:redirection2], + response_hash[:redirection1], + response_hash[:redirection], + response_hash[:ok], + ])).to eq(response_hash[:redirection][:headers]["location"]) + end + + it "returns the location as given, by default or when absolutize is false" do + expect(curl_response_last_location([ + response_hash[:redirection_no_scheme], + response_hash[:ok], + ])).to eq(response_hash[:redirection_no_scheme][:headers]["location"]) + + expect(curl_response_last_location([ + response_hash[:redirection_root_relative], + response_hash[:ok], + ])).to eq(response_hash[:redirection_root_relative][:headers]["location"]) + + expect(curl_response_last_location([ + response_hash[:redirection_parent_relative], + response_hash[:ok], + ])).to eq(response_hash[:redirection_parent_relative][:headers]["location"]) + end + + it "returns an absolute URL when absolutize is true and a base URL is provided" do + expect( + curl_response_last_location( + [response_hash[:redirection_no_scheme], response_hash[:ok]], + absolutize: true, + base_url: "https://brew.sh/test", + ), + ).to eq("https:#{response_hash[:redirection_no_scheme][:headers]["location"]}") + + expect( + curl_response_last_location( + [response_hash[:redirection_root_relative], response_hash[:ok]], + absolutize: true, + base_url: "https://brew.sh/test", + ), + ).to eq("https://brew.sh#{response_hash[:redirection_root_relative][:headers]["location"]}") + + expect( + curl_response_last_location( + [response_hash[:redirection_parent_relative], response_hash[:ok]], + absolutize: true, + base_url: "https://brew.sh/test1/test2", + ), + ).to eq(response_hash[:redirection_parent_relative][:headers]["location"].sub(/^\./, "https://brew.sh/test1")) + end + + it "returns nil when the response hash doesn't contain a location header" do + expect(curl_response_last_location([response_hash[:ok]])).to be_nil + end + end end diff --git a/Library/Homebrew/utils/curl.rb b/Library/Homebrew/utils/curl.rb index 4b08d1fa77..20792f25a5 100644 --- a/Library/Homebrew/utils/curl.rb +++ b/Library/Homebrew/utils/curl.rb @@ -14,6 +14,22 @@ module Utils using TimeRemaining + # This regex is used to extract the part of an ETag within quotation marks, + # ignoring any leading weak validator indicator (`W/`). This simplifies + # ETag comparison in `#curl_check_http_content`. + ETAG_VALUE_REGEX = %r{^(?:[wW]/)?"((?:[^"]|\\")*)"}.freeze + + # HTTP responses and body content are typically separated by a double + # `CRLF` (whereas HTTP header lines are separated by a single `CRLF`). + # In rare cases, this can also be a double newline (`\n\n`). + HTTP_RESPONSE_BODY_SEPARATOR = "\r\n\r\n" + + # This regex is used to isolate the parts of an HTTP status line, namely + # the status code and any following descriptive text (e.g., `Not Found`). + HTTP_STATUS_LINE_REGEX = %r{^HTTP/.* (?\d+)(?: (?[^\r\n]+))?}.freeze
+
+ private_constant :ETAG_VALUE_REGEX, :HTTP_RESPONSE_BODY_SEPARATOR, :HTTP_STATUS_LINE_REGEX
+
module_function
def curl_executable(use_homebrew_curl: false)
@@ -145,23 +161,19 @@ module Utils
result
end
- def parse_headers(headers)
- return {} if headers.blank?
-
- # Skip status code
- headers.split("\r\n")[1..].to_h do |h|
- name, content = h.split(": ")
- [name.downcase, content]
- end
- end
-
def curl_download(*args, to: nil, try_partial: true, **options)
destination = Pathname(to)
destination.dirname.mkpath
if try_partial
range_stdout = curl_output("--location", "--head", *args, **options).stdout
- headers = parse_headers(range_stdout.split("\r\n\r\n").first)
+ parsed_output = parse_curl_output(range_stdout)
+
+ headers = if parsed_output[:responses].present?
+ parsed_output[:responses].last[:headers]
+ else
+ {}
+ end
# Any value for `accept-ranges` other than none indicates that the server supports partial requests.
# Its absence indicates no support.
@@ -187,6 +199,8 @@ module Utils
# Check if a URL is protected by CloudFlare (e.g. badlion.net and jaxx.io).
def url_protected_by_cloudflare?(details)
+ return false if details[:headers].blank?
+
[403, 503].include?(details[:status].to_i) &&
details[:headers].match?(/^Set-Cookie: (__cfduid|__cf_bm)=/i) &&
details[:headers].match?(/^Server: cloudflare/i)
@@ -194,6 +208,8 @@ module Utils
# Check if a URL is protected by Incapsula (e.g. corsair.com).
def url_protected_by_incapsula?(details)
+ return false if details[:headers].blank?
+
details[:status].to_i == 403 &&
details[:headers].match?(/^Set-Cookie: visid_incap_/i) &&
details[:headers].match?(/^Set-Cookie: incap_ses_/i)
@@ -255,7 +271,7 @@ module Utils
end
if url.start_with?("https://") && Homebrew::EnvConfig.no_insecure_redirect? &&
- !details[:final_url].start_with?("https://")
+ (details[:final_url].present? && !details[:final_url].start_with?("https://"))
return "The #{url_type} #{url} redirects back to HTTP"
end
@@ -270,9 +286,11 @@ module Utils
details[:content_length] == secure_details[:content_length]
file_match = details[:file_hash] == secure_details[:file_hash]
- if (etag_match || content_length_match || file_match) &&
- secure_details[:final_url].start_with?("https://") &&
- url.start_with?("http://")
+ http_with_https_available =
+ url.start_with?("http://") &&
+ (secure_details[:final_url].present? && secure_details[:final_url].start_with?("https://"))
+
+ if (etag_match || content_length_match || file_match) && http_with_https_available
return "The #{url_type} #{url} should use HTTPS rather than HTTP"
end
@@ -283,8 +301,7 @@ module Utils
https_content = secure_details[:file]&.gsub(no_protocol_file_contents, "/")
# Check for the same content after removing all protocols
- if (http_content && https_content) && (http_content == https_content) &&
- url.start_with?("http://") && secure_details[:final_url].start_with?("https://")
+ if (http_content && https_content) && (http_content == https_content) && http_with_https_available
return "The #{url_type} #{url} should use HTTPS rather than HTTP"
end
@@ -328,30 +345,33 @@ module Utils
user_agent: user_agent
)
- status_code = :unknown
- while status_code == :unknown || status_code.to_s.start_with?("3")
- headers, _, output = output.partition("\r\n\r\n")
- status_code = headers[%r{HTTP/.* (\d+)}, 1]
- location = headers[/^Location:\s*(.*)$/i, 1]
- final_url = location.chomp if location
- end
-
if status.success?
+ parsed_output = parse_curl_output(output)
+ responses = parsed_output[:responses]
+
+ final_url = curl_response_last_location(responses)
+ headers = if responses.last.present?
+ status_code = responses.last[:status_code]
+ responses.last[:headers]
+ else
+ {}
+ end
+ etag = headers["etag"][ETAG_VALUE_REGEX, 1] if headers["etag"].present?
+ content_length = headers["content-length"]
+
file_contents = File.read(file.path)
file_hash = Digest::SHA2.hexdigest(file_contents) if hash_needed
end
- final_url ||= url
-
{
url: url,
final_url: final_url,
status: status_code,
- etag: headers[%r{ETag: ([wW]/)?"(([^"]|\\")*)"}, 2],
- content_length: headers[/Content-Length: (\d+)/, 1],
headers: headers,
- file_hash: file_hash,
+ etag: etag,
+ content_length: content_length,
file: file_contents,
+ file_hash: file_hash,
}
ensure
file.unlink
@@ -367,6 +387,95 @@ module Utils
def http_status_ok?(status)
(100..299).cover?(status.to_i)
end
+
+ # Separates the output text from `curl` into an array of HTTP responses and
+ # the final response body (i.e. content). Response hashes contain the
+ # `:status_code`, `:status_text`, and `:headers`.
+ # @param output [String] The output text from `curl` containing HTTP
+ # responses, body content, or both.
+ # @return [Hash] A hash containing an array of response hashes and the body
+ # content, if found.
+ sig { params(output: String).returns(T::Hash[Symbol, T.untyped]) }
+ def parse_curl_output(output)
+ responses = []
+
+ max_iterations = 5
+ iterations = 0
+ output = output.lstrip
+ while output.match?(%r{\AHTTP/[\d.]+ \d+}) && output.include?(HTTP_RESPONSE_BODY_SEPARATOR)
+ iterations += 1
+ raise "Too many redirects (max = #{max_iterations})" if iterations > max_iterations
+
+ response_text, _, output = output.partition(HTTP_RESPONSE_BODY_SEPARATOR)
+ output = output.lstrip
+ next if response_text.blank?
+
+ response_text.chomp!
+ response = parse_curl_response(response_text)
+ responses << response if response.present?
+ end
+
+ { responses: responses, body: output }
+ end
+
+ # Returns the URL from the last location header found in cURL responses,
+ # if any.
+ # @param responses [Array] An array of hashes containing response
+ # status information and headers from `#parse_curl_response`.
+ # @param absolutize [true, false] Whether to make the location URL absolute.
+ # @param base_url [String, nil] The URL to use as a base for making the
+ # `location` URL absolute.
+ # @return [String, nil] The URL from the last-occurring `location` header
+ # in the responses or `nil` (if no `location` headers found).
+ sig {
+ params(
+ responses: T::Array[T::Hash[Symbol, T.untyped]],
+ absolutize: T::Boolean,
+ base_url: T.nilable(String),
+ ).returns(T.nilable(String))
+ }
+ def curl_response_last_location(responses, absolutize: false, base_url: nil)
+ responses.reverse_each do |response|
+ next if response[:headers].blank?
+
+ location = response[:headers]["location"]
+ next if location.blank?
+
+ absolute_url = URI.join(base_url, location).to_s if absolutize && base_url.present?
+ return absolute_url || location
+ end
+
+ nil
+ end
+
+ private
+
+ # Parses HTTP response text from `curl` output into a hash containing the
+ # information from the status line (status code and, optionally,
+ # descriptive text) and headers.
+ # @param response_text [String] The text of a `curl` response, consisting
+ # of a status line followed by header lines.
+ # @return [Hash] A hash containing the response status information and
+ # headers (as a hash with header names as keys).
+ sig { params(response_text: String).returns(T::Hash[Symbol, T.untyped]) }
+ def parse_curl_response(response_text)
+ response = {}
+ return response unless response_text.match?(HTTP_STATUS_LINE_REGEX)
+
+ # Parse the status line and remove it
+ match = response_text.match(HTTP_STATUS_LINE_REGEX)
+ response[:status_code] = match["code"] if match["code"].present?
+ response[:status_text] = match["text"] if match["text"].present?
+ response_text = response_text.sub(%r{^HTTP/.* (\d+).*$\s*}, "")
+
+ # Create a hash from the header lines
+ response[:headers] =
+ response_text.split("\r\n")
+ .to_h { |header| header.split(/:\s*/, 2) }
+ .transform_keys(&:downcase)
+
+ response
+ end
end
end