From 6f44dc41d5316e26a3ac533019199da502f430fc Mon Sep 17 00:00:00 2001 From: Mike McQuaid Date: Thu, 29 Dec 2016 12:50:41 +0000 Subject: [PATCH 1/4] development_tools: remove unused method. --- Library/Homebrew/development_tools.rb | 4 ---- Library/Homebrew/extend/os/mac/development_tools.rb | 4 ---- 2 files changed, 8 deletions(-) diff --git a/Library/Homebrew/development_tools.rb b/Library/Homebrew/development_tools.rb index ea7f5837d0..bee2b86dea 100644 --- a/Library/Homebrew/development_tools.rb +++ b/Library/Homebrew/development_tools.rb @@ -119,10 +119,6 @@ class DevelopmentTools @clang_version = @clang_build_version = nil @non_apple_gcc_version = {} end - - def tar_supports_xz? - false - end end end diff --git a/Library/Homebrew/extend/os/mac/development_tools.rb b/Library/Homebrew/extend/os/mac/development_tools.rb index 7c97b9d69e..381b26e661 100644 --- a/Library/Homebrew/extend/os/mac/development_tools.rb +++ b/Library/Homebrew/extend/os/mac/development_tools.rb @@ -76,9 +76,5 @@ class DevelopmentTools end end end - - def tar_supports_xz? - false - end end end From 59180ec370560d461d9fa44a3e510e9f1ea68375 Mon Sep 17 00:00:00 2001 From: Mike McQuaid Date: Fri, 23 Dec 2016 20:29:56 +0000 Subject: [PATCH 2/4] audit: improve reliability of homepage audit. - Don't run on Yosemite where the system Curl is too old for some modern HTTPS homepages - Try up to 3 times in case of transient failures. --- Library/Homebrew/dev-cmd/audit.rb | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/Library/Homebrew/dev-cmd/audit.rb b/Library/Homebrew/dev-cmd/audit.rb index ef22cbb7a4..757cc0df4b 100644 --- a/Library/Homebrew/dev-cmd/audit.rb +++ b/Library/Homebrew/dev-cmd/audit.rb @@ -569,9 +569,20 @@ class FormulaAuditor end return unless @online - status_code, = curl_output "--connect-timeout", "15", "--output", "/dev/null", "--range", "0-0", - "--write-out", "%{http_code}", homepage - return if status_code.start_with? "20" + + # The system Curl is too old and unreliable with HTTPS homepages on + # Yosemite and below. + return unless MacOS.version >= :el_capitan + + retries = 3 + retries.times do + status_code, = curl_output "--connect-timeout", "15", + "--output", "/dev/null", + "--range", "0-0", + "--write-out", "%{http_code}", + homepage + return if status_code.start_with? "20" + end problem "The homepage #{homepage} is not reachable (HTTP status code #{status_code})" end From e6fb3c3114ea842884a49c7f2ddfe1a9c4e3eee0 Mon Sep 17 00:00:00 2001 From: Mike McQuaid Date: Sun, 25 Dec 2016 23:01:40 +0000 Subject: [PATCH 3/4] curl: make curl_args more configurable. Allow configuring whether output should be shown or the default the default user agent is used. --- Library/Homebrew/global.rb | 8 -------- Library/Homebrew/utils/curl.rb | 29 ++++++++++++++++++++--------- 2 files changed, 20 insertions(+), 17 deletions(-) diff --git a/Library/Homebrew/global.rb b/Library/Homebrew/global.rb index eabc4c164c..9a2f0c794e 100644 --- a/Library/Homebrew/global.rb +++ b/Library/Homebrew/global.rb @@ -27,14 +27,6 @@ RUBY_BIN = RUBY_PATH.dirname HOMEBREW_USER_AGENT_CURL = ENV["HOMEBREW_USER_AGENT_CURL"] HOMEBREW_USER_AGENT_RUBY = "#{ENV["HOMEBREW_USER_AGENT"]} ruby/#{RUBY_VERSION}-p#{RUBY_PATCHLEVEL}".freeze -HOMEBREW_CURL_ARGS = [ - "--fail", - "--progress-bar", - "--remote-time", - "--location", - "--user-agent", HOMEBREW_USER_AGENT_CURL -].freeze - require "tap_constants" module Homebrew diff --git a/Library/Homebrew/utils/curl.rb b/Library/Homebrew/utils/curl.rb index 00c3a591cd..eab623c404 100644 --- a/Library/Homebrew/utils/curl.rb +++ b/Library/Homebrew/utils/curl.rb @@ -1,27 +1,38 @@ require "pathname" require "open3" -def curl_args(extra_args = []) +def curl_args(options = {}) curl = Pathname.new ENV["HOMEBREW_CURL"] curl = Pathname.new "/usr/bin/curl" unless curl.exist? raise "#{curl} is not executable" unless curl.exist? && curl.executable? - flags = HOMEBREW_CURL_ARGS - flags -= ["--progress-bar"] if ARGV.verbose? + args = [ + curl.to_s, + "--remote-time", + "--location", + ] - args = [curl.to_s] + flags + extra_args - args << "--verbose" if ENV["HOMEBREW_CURL_VERBOSE"] - args << "--silent" if !$stdout.tty? || ENV["TRAVIS"] + unless options[:default_user_agent] + args << "--user-agent" << HOMEBREW_USER_AGENT_CURL + end + + unless options[:show_output] + args << "--progress-bar" unless ARGV.verbose? + args << "--verbose" if ENV["HOMEBREW_CURL_VERBOSE"] + args << "--fail" + args << "--silent" if !$stdout.tty? || ENV["TRAVIS"] + end + + args += options[:extra_args] if options[:extra_args] args end def curl(*args) - safe_system(*curl_args(args)) + safe_system(*curl_args(extra_args: args)) end def curl_output(*args) - curl_args = curl_args(args) - curl_args -= ["--fail", "--silent"] + curl_args = curl_args(extra_args: args, show_output: true) Open3.popen3(*curl_args) do |_, stdout, stderr, wait_thread| [stdout.read, stderr.read, wait_thread.value] end From b3c6334d3cde6d653427ae29892e3a14af9c5bd9 Mon Sep 17 00:00:00 2001 From: Mike McQuaid Date: Fri, 30 Dec 2016 20:17:34 +0000 Subject: [PATCH 4/4] audit: use new curl_args form. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This will use Curl’s default user agent to reduce homepage errors and provides a function that can be used for other audits to perform similar tests on URLs. --- Library/Homebrew/dev-cmd/audit.rb | 42 +++++++++++++------ Library/Homebrew/development_tools.rb | 4 ++ .../extend/os/mac/development_tools.rb | 5 +++ 3 files changed, 38 insertions(+), 13 deletions(-) diff --git a/Library/Homebrew/dev-cmd/audit.rb b/Library/Homebrew/dev-cmd/audit.rb index 757cc0df4b..74ba987f63 100644 --- a/Library/Homebrew/dev-cmd/audit.rb +++ b/Library/Homebrew/dev-cmd/audit.rb @@ -169,6 +169,33 @@ class FormulaAuditor @specs = %w[stable devel head].map { |s| formula.send(s) }.compact end + def url_status_code(url, range: false) + # The system Curl is too old and unreliable with HTTPS homepages on + # Yosemite and below. + return "200" unless DevelopmentTools.curl_handles_most_https_homepages? + + extra_args = [ + "--connect-timeout", "15", + "--output", "/dev/null", + "--write-out", "%{http_code}" + ] + extra_args << "--range" << "0-0" if range + extra_args << url + + args = curl_args( + extra_args: extra_args, + show_output: true, + default_user_agent: true + ) + retries = 3 + status_code = nil + retries.times do + status_code = Open3.popen3(*args) { |_, stdout, _, _| stdout.read } + break if status_code.start_with? "20" + end + status_code + end + def audit_style return unless @style_offenses display_cop_names = ARGV.include?("--display-cop-names") @@ -570,19 +597,8 @@ class FormulaAuditor return unless @online - # The system Curl is too old and unreliable with HTTPS homepages on - # Yosemite and below. - return unless MacOS.version >= :el_capitan - - retries = 3 - retries.times do - status_code, = curl_output "--connect-timeout", "15", - "--output", "/dev/null", - "--range", "0-0", - "--write-out", "%{http_code}", - homepage - return if status_code.start_with? "20" - end + status_code = url_status_code(homepage) + return if status_code.start_with? "20" problem "The homepage #{homepage} is not reachable (HTTP status code #{status_code})" end diff --git a/Library/Homebrew/development_tools.rb b/Library/Homebrew/development_tools.rb index bee2b86dea..0a2f127296 100644 --- a/Library/Homebrew/development_tools.rb +++ b/Library/Homebrew/development_tools.rb @@ -119,6 +119,10 @@ class DevelopmentTools @clang_version = @clang_build_version = nil @non_apple_gcc_version = {} end + + def curl_handles_most_https_homepages? + true + end end end diff --git a/Library/Homebrew/extend/os/mac/development_tools.rb b/Library/Homebrew/extend/os/mac/development_tools.rb index 381b26e661..8c0c48a516 100644 --- a/Library/Homebrew/extend/os/mac/development_tools.rb +++ b/Library/Homebrew/extend/os/mac/development_tools.rb @@ -76,5 +76,10 @@ class DevelopmentTools end end end + + def curl_handles_most_https_homepages? + # The system Curl is too old for some modern HTTPS homepages on Yosemite. + MacOS.version >= :el_capitan + end end end