Merge pull request #1725 from MikeMcQuaid/curl-homepage-reliablity

audit: improve reliability of homepage audit.

Library/Homebrew/dev-cmd/audit.rb

@@ -169,6 +169,33 @@ class FormulaAuditor
     @specs = %w[stable devel head].map { |s| formula.send(s) }.compact
   end
 
+  def url_status_code(url, range: false)
+    # The system Curl is too old and unreliable with HTTPS homepages on
+    # Yosemite and below.
+    return "200" unless DevelopmentTools.curl_handles_most_https_homepages?
+
+    extra_args = [
+      "--connect-timeout", "15",
+      "--output", "/dev/null",
+      "--write-out", "%{http_code}"
+    ]
+    extra_args << "--range" << "0-0" if range
+    extra_args << url
+
+    args = curl_args(
+      extra_args: extra_args,
+      show_output: true,
+      default_user_agent: true
+    )
+    retries = 3
+    status_code = nil
+    retries.times do
+      status_code = Open3.popen3(*args) { |_, stdout, _, _| stdout.read }
+      break if status_code.start_with? "20"
+    end
+    status_code
+  end
+
   def audit_style
     return unless @style_offenses
     display_cop_names = ARGV.include?("--display-cop-names")
@@ -569,8 +596,8 @@ class FormulaAuditor
     end
 
     return unless @online
-    status_code, = curl_output "--connect-timeout", "15", "--output", "/dev/null", "--range", "0-0",
+
-                               "--write-out", "%{http_code}", homepage
+    status_code = url_status_code(homepage)
     return if status_code.start_with? "20"
     problem "The homepage #{homepage} is not reachable (HTTP status code #{status_code})"
   end

Library/Homebrew/development_tools.rb

@@ -120,8 +120,8 @@ class DevelopmentTools
       @non_apple_gcc_version = {}
     end
 
-    def tar_supports_xz?
+    def curl_handles_most_https_homepages?
-      false
+      true
     end
   end
 end

Library/Homebrew/extend/os/mac/development_tools.rb

@@ -77,8 +77,9 @@ class DevelopmentTools
       end
     end
 
-    def tar_supports_xz?
+    def curl_handles_most_https_homepages?
-      false
+      # The system Curl is too old for some modern HTTPS homepages on Yosemite.
+      MacOS.version >= :el_capitan
     end
   end
 end

Library/Homebrew/global.rb

@@ -27,14 +27,6 @@ RUBY_BIN = RUBY_PATH.dirname
 HOMEBREW_USER_AGENT_CURL = ENV["HOMEBREW_USER_AGENT_CURL"]
 HOMEBREW_USER_AGENT_RUBY = "#{ENV["HOMEBREW_USER_AGENT"]} ruby/#{RUBY_VERSION}-p#{RUBY_PATCHLEVEL}".freeze
 
-HOMEBREW_CURL_ARGS = [
-  "--fail",
-  "--progress-bar",
-  "--remote-time",
-  "--location",
-  "--user-agent", HOMEBREW_USER_AGENT_CURL
-].freeze
-
 require "tap_constants"
 
 module Homebrew

Library/Homebrew/utils/curl.rb

@@ -1,27 +1,38 @@
 require "pathname"
 require "open3"
 
-def curl_args(extra_args = [])
+def curl_args(options = {})
   curl = Pathname.new ENV["HOMEBREW_CURL"]
   curl = Pathname.new "/usr/bin/curl" unless curl.exist?
   raise "#{curl} is not executable" unless curl.exist? && curl.executable?
 
-  flags = HOMEBREW_CURL_ARGS
+  args = [
-  flags -= ["--progress-bar"] if ARGV.verbose?
+    curl.to_s,
+    "--remote-time",
+    "--location",
+  ]
 
-  args = [curl.to_s] + flags + extra_args
+  unless options[:default_user_agent]
-  args << "--verbose" if ENV["HOMEBREW_CURL_VERBOSE"]
+    args << "--user-agent" << HOMEBREW_USER_AGENT_CURL
-  args << "--silent" if !$stdout.tty? || ENV["TRAVIS"]
+  end
+
+  unless options[:show_output]
+    args << "--progress-bar" unless ARGV.verbose?
+    args << "--verbose" if ENV["HOMEBREW_CURL_VERBOSE"]
+    args << "--fail"
+    args << "--silent" if !$stdout.tty? || ENV["TRAVIS"]
+  end
+
+  args += options[:extra_args] if options[:extra_args]
   args
 end
 
 def curl(*args)
-  safe_system(*curl_args(args))
+  safe_system(*curl_args(extra_args: args))
 end
 
 def curl_output(*args)
-  curl_args = curl_args(args)
+  curl_args = curl_args(extra_args: args, show_output: true)
-  curl_args -= ["--fail", "--silent"]
   Open3.popen3(*curl_args) do |_, stdout, stderr, wait_thread|
     [stdout.read, stderr.read, wait_thread.value]
   end