Merge pull request #13326 from samford/curl/check-all-responses-for-protected-cookies

Curl: Check all responses for protected cookies

Library/Homebrew/test/utils/curl_spec.rb

@@ -14,7 +14,7 @@ describe "Utils::Curl" do
     details[:normal][:no_cookie] = {
       url:            "https://www.example.com/",
       final_url:      nil,
-      status:         "403",
+      status_code:    "403",
       headers:        {
         "age"            => "123456",
         "cache-control"  => "max-age=604800",
@@ -35,7 +35,7 @@ describe "Utils::Curl" do
     }
 
     details[:normal][:ok] = Marshal.load(Marshal.dump(details[:normal][:no_cookie]))
-    details[:normal][:ok][:status] = "200"
+    details[:normal][:ok][:status_code] = "200"
 
     details[:normal][:single_cookie] = Marshal.load(Marshal.dump(details[:normal][:no_cookie]))
     details[:normal][:single_cookie][:headers]["set-cookie"] = "a_cookie=for_testing"
@@ -52,7 +52,7 @@ describe "Utils::Curl" do
     details[:cloudflare][:single_cookie] = {
       url:            "https://www.example.com/",
       final_url:      nil,
-      status:         "403",
+      status_code:    "403",
       headers:        {
         "date"            => "Wed, 1 Jan 2020 01:23:45 GMT",
         "content-type"    => "text/plain; charset=UTF-8",

Library/Homebrew/utils/curl.rb

@@ -198,21 +198,20 @@ module Utils
     end
 
     # Check if a URL is protected by CloudFlare (e.g. badlion.net and jaxx.io).
-    # @param details [Hash] Response information from
+    # @param response [Hash] A response hash from `#parse_curl_response`.
-    #  `#curl_http_content_headers_and_checksum`.
     # @return [true, false] Whether a response contains headers indicating that
     #   the URL is protected by Cloudflare.
-    sig { params(details: T::Hash[Symbol, T.untyped]).returns(T::Boolean) }
+    sig { params(response: T::Hash[Symbol, T.untyped]).returns(T::Boolean) }
-    def url_protected_by_cloudflare?(details)
+    def url_protected_by_cloudflare?(response)
-      return false if details[:headers].blank?
+      return false if response[:headers].blank?
-      return false unless [403, 503].include?(details[:status].to_i)
+      return false unless [403, 503].include?(response[:status_code].to_i)
 
-      set_cookie_header = Array(details[:headers]["set-cookie"])
+      set_cookie_header = Array(response[:headers]["set-cookie"])
       has_cloudflare_cookie_header = set_cookie_header.compact.any? do |cookie|
         cookie.match?(/^(__cfduid|__cf_bm)=/i)
       end
 
-      server_header = Array(details[:headers]["server"])
+      server_header = Array(response[:headers]["server"])
       has_cloudflare_server = server_header.compact.any? do |server|
         server.match?(/^cloudflare/i)
       end
@@ -221,16 +220,15 @@ module Utils
     end
 
     # Check if a URL is protected by Incapsula (e.g. corsair.com).
-    # @param details [Hash] Response information from
+    # @param response [Hash] A response hash from `#parse_curl_response`.
-    #  `#curl_http_content_headers_and_checksum`.
     # @return [true, false] Whether a response contains headers indicating that
     #   the URL is protected by Incapsula.
-    sig { params(details: T::Hash[Symbol, T.untyped]).returns(T::Boolean) }
+    sig { params(response: T::Hash[Symbol, T.untyped]).returns(T::Boolean) }
-    def url_protected_by_incapsula?(details)
+    def url_protected_by_incapsula?(response)
-      return false if details[:headers].blank?
+      return false if response[:headers].blank?
-      return false if details[:status].to_i != 403
+      return false if response[:status_code].to_i != 403
 
-      set_cookie_header = Array(details[:headers]["set-cookie"])
+      set_cookie_header = Array(response[:headers]["set-cookie"])
       set_cookie_header.compact.any? { |cookie| cookie.match?(/^(visid_incap|incap_ses)_/i) }
     end
 
@@ -255,7 +253,7 @@ module Utils
             next
           end
 
-          next unless http_status_ok?(secure_details[:status])
+          next unless http_status_ok?(secure_details[:status_code])
 
           hash_needed = true
           user_agents = [user_agent]
@@ -273,20 +271,22 @@ module Utils
             use_homebrew_curl: use_homebrew_curl,
             user_agent:        user_agent,
           )
-        break if http_status_ok?(details[:status])
+        break if http_status_ok?(details[:status_code])
       end
 
-      unless details[:status]
+      unless details[:status_code]
         # Hack around https://github.com/Homebrew/brew/issues/3199
         return if MacOS.version == :el_capitan
 
         return "The #{url_type} #{url} is not reachable"
       end
 
-      unless http_status_ok?(details[:status])
+      unless http_status_ok?(details[:status_code])
-        return if url_protected_by_cloudflare?(details) || url_protected_by_incapsula?(details)
+        return if details[:responses].any? do |response|
+          url_protected_by_cloudflare?(response) || url_protected_by_incapsula?(response)
+        end
 
-        return "The #{url_type} #{url} is not reachable (HTTP status code #{details[:status]})"
+        return "The #{url_type} #{url} is not reachable (HTTP status code #{details[:status_code]})"
       end
 
       if url.start_with?("https://") && Homebrew::EnvConfig.no_insecure_redirect? &&
@@ -296,7 +296,7 @@ module Utils
 
       return unless secure_details
 
-      return if !http_status_ok?(details[:status]) || !http_status_ok?(secure_details[:status])
+      return if !http_status_ok?(details[:status_code]) || !http_status_ok?(secure_details[:status_code])
 
       etag_match = details[:etag] &&
                    details[:etag] == secure_details[:etag]
@@ -397,12 +397,13 @@ module Utils
       {
         url:            url,
         final_url:      final_url,
-        status:         status_code,
+        status_code:    status_code,
         headers:        headers,
         etag:           etag,
         content_length: content_length,
         file:           file_contents,
         file_hash:      file_hash,
+        responses:      responses,
       }
     ensure
       file.unlink