From 923c84d4f7c4e0cf09bfb417deb83bff2364c199 Mon Sep 17 00:00:00 2001 From: Viktor Szakats Date: Sun, 27 Aug 2017 09:39:28 +0000 Subject: [PATCH 1/7] add some heuristics to https upgrade check --- Library/Homebrew/dev-cmd/audit.rb | 36 ++++++++++++++++++++++++++----- 1 file changed, 31 insertions(+), 5 deletions(-) diff --git a/Library/Homebrew/dev-cmd/audit.rb b/Library/Homebrew/dev-cmd/audit.rb index 2c93364812..4fadae1a40 100644 --- a/Library/Homebrew/dev-cmd/audit.rb +++ b/Library/Homebrew/dev-cmd/audit.rb @@ -201,7 +201,7 @@ class FormulaAuditor @specs = %w[stable devel head].map { |s| formula.send(s) }.compact end - def self.check_http_content(url, name, user_agents: [:default]) + def self.check_http_content(type, url, name, user_agents: [:default]) return unless url.start_with? "http" details = nil @@ -236,8 +236,31 @@ class FormulaAuditor details[:content_length] == secure_details[:content_length] file_match = details[:file_hash] == secure_details[:file_hash] - return if !etag_match && !content_length_match && !file_match - "The URL #{url} could use HTTPS rather than HTTP" + if etag_match || content_length_match || file_match + return "The URL #{url} could use HTTPS rather than HTTP" + end + + if type == "homepage" + + details[:file] = details[:file].gsub(/https?:\\?\/\\?\//, '/') + secure_details[:file] = secure_details[:file].gsub(/https?:\\?\/\\?\//, '/') + + # Same content after normalization + if details[:file] == secure_details[:file] + return "The URL #{url} could use HTTPS rather than HTTP" + end + + # Same size, different content after normalization + # (typical causes: Generated ID, Timestamp, Unix time) + if details[:file].length == secure_details[:file].length + return "The URL #{url} could use HTTPS rather than HTTP" + end + + lenratio = (100 * secure_details[:file].length / details[:file].length).to_i + if lenratio >= 90 && lenratio <= 120 + return "The URL #{url} may be able to use HTTPS rather than HTTP. Please verify it in a browser." + end + end end def self.http_content_headers_and_checksum(url, hash_needed: false, user_agent: :default) @@ -260,6 +283,7 @@ class FormulaAuditor etag: headers[%r{ETag: ([wW]\/)?"(([^"]|\\")*)"}, 2], content_length: headers[/Content-Length: (\d+)/, 1], file_hash: output_hash, + file: output, } end @@ -564,7 +588,8 @@ class FormulaAuditor return unless @online return unless DevelopmentTools.curl_handles_most_https_homepages? - if http_content_problem = FormulaAuditor.check_http_content(homepage, + if http_content_problem = FormulaAuditor.check_http_content("homepage", + homepage, formula.name, user_agents: [:browser, :default]) problem http_content_problem @@ -1219,7 +1244,8 @@ class ResourceAuditor # A `brew mirror`'ed URL is usually not yet reachable at the time of # pull request. next if url =~ %r{^https://dl.bintray.com/homebrew/mirror/} - if http_content_problem = FormulaAuditor.check_http_content(url, name) + if http_content_problem = FormulaAuditor.check_http_content("url", + url, name) problem http_content_problem end elsif strategy <= GitDownloadStrategy From 11b267a7cfb3411c74c4fc21732463d543fc4fb3 Mon Sep 17 00:00:00 2001 From: Viktor Szakats Date: Sun, 27 Aug 2017 17:52:26 +0000 Subject: [PATCH 2/7] try addressing style issues --- Library/Homebrew/dev-cmd/audit.rb | 31 +++++++++++++++---------------- 1 file changed, 15 insertions(+), 16 deletions(-) diff --git a/Library/Homebrew/dev-cmd/audit.rb b/Library/Homebrew/dev-cmd/audit.rb index 4fadae1a40..97fc93db9e 100644 --- a/Library/Homebrew/dev-cmd/audit.rb +++ b/Library/Homebrew/dev-cmd/audit.rb @@ -240,26 +240,25 @@ class FormulaAuditor return "The URL #{url} could use HTTPS rather than HTTP" end - if type == "homepage" + return if type != "homepage" - details[:file] = details[:file].gsub(/https?:\\?\/\\?\//, '/') - secure_details[:file] = secure_details[:file].gsub(/https?:\\?\/\\?\//, '/') + details[:file] = details[:file].gsub(%r{https?:\\?\/\\?\/}, "/") + secure_details[:file] = secure_details[:file].gsub(%r{https?:\\?\/\\?\/}, "/") - # Same content after normalization - if details[:file] == secure_details[:file] - return "The URL #{url} could use HTTPS rather than HTTP" - end + # Same content after normalization + if details[:file] == secure_details[:file] + return "The URL #{url} could use HTTPS rather than HTTP" + end - # Same size, different content after normalization - # (typical causes: Generated ID, Timestamp, Unix time) - if details[:file].length == secure_details[:file].length - return "The URL #{url} could use HTTPS rather than HTTP" - end + # Same size, different content after normalization + # (typical causes: Generated ID, Timestamp, Unix time) + if details[:file].length == secure_details[:file].length + return "The URL #{url} could use HTTPS rather than HTTP" + end - lenratio = (100 * secure_details[:file].length / details[:file].length).to_i - if lenratio >= 90 && lenratio <= 120 - return "The URL #{url} may be able to use HTTPS rather than HTTP. Please verify it in a browser." - end + lenratio = (100 * secure_details[:file].length / details[:file].length).to_i + if lenratio >= 90 && lenratio <= 120 + return "The URL #{url} may be able to use HTTPS rather than HTTP. Please verify it in a browser." end end From 1c2c390c6fae78062c28a8ed1610936a4c2dd412 Mon Sep 17 00:00:00 2001 From: Viktor Szakats Date: Sun, 27 Aug 2017 18:00:59 +0000 Subject: [PATCH 3/7] try addressing style issues #2 --- Library/Homebrew/dev-cmd/audit.rb | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/Library/Homebrew/dev-cmd/audit.rb b/Library/Homebrew/dev-cmd/audit.rb index 97fc93db9e..d468d69cb5 100644 --- a/Library/Homebrew/dev-cmd/audit.rb +++ b/Library/Homebrew/dev-cmd/audit.rb @@ -257,9 +257,8 @@ class FormulaAuditor end lenratio = (100 * secure_details[:file].length / details[:file].length).to_i - if lenratio >= 90 && lenratio <= 120 - return "The URL #{url} may be able to use HTTPS rather than HTTP. Please verify it in a browser." - end + return if lenratio < 90 || lenratio > 120 + "The URL #{url} may be able to use HTTPS rather than HTTP. Please verify it in a browser." end def self.http_content_headers_and_checksum(url, hash_needed: false, user_agent: :default) From c30b94135853b809eda34c5c347c549bbf08a42d Mon Sep 17 00:00:00 2001 From: Viktor Szakats Date: Tue, 29 Aug 2017 12:31:07 +0000 Subject: [PATCH 4/7] review follow-up #1 --- Library/Homebrew/dev-cmd/audit.rb | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/Library/Homebrew/dev-cmd/audit.rb b/Library/Homebrew/dev-cmd/audit.rb index d468d69cb5..5a86c0adfd 100644 --- a/Library/Homebrew/dev-cmd/audit.rb +++ b/Library/Homebrew/dev-cmd/audit.rb @@ -201,7 +201,7 @@ class FormulaAuditor @specs = %w[stable devel head].map { |s| formula.send(s) }.compact end - def self.check_http_content(type, url, name, user_agents: [:default]) + def self.check_http_content(url, name, user_agents: [:default], check_content: false) return unless url.start_with? "http" details = nil @@ -237,23 +237,24 @@ class FormulaAuditor file_match = details[:file_hash] == secure_details[:file_hash] if etag_match || content_length_match || file_match - return "The URL #{url} could use HTTPS rather than HTTP" + return "The URL #{url} should use HTTPS rather than HTTP" end - return if type != "homepage" + return unless check_content - details[:file] = details[:file].gsub(%r{https?:\\?\/\\?\/}, "/") - secure_details[:file] = secure_details[:file].gsub(%r{https?:\\?\/\\?\/}, "/") + no_protocol_file_contents = %r{https?:\\?/\\?/} + details[:file] = details[:file].gsub(no_protocol_file_contents, "/") + secure_details[:file] = secure_details[:file].gsub(no_protocol_file_contents, "/") - # Same content after normalization + # Check for the same content after removing all protocols if details[:file] == secure_details[:file] - return "The URL #{url} could use HTTPS rather than HTTP" + return "The URL #{url} should use HTTPS rather than HTTP" end # Same size, different content after normalization # (typical causes: Generated ID, Timestamp, Unix time) if details[:file].length == secure_details[:file].length - return "The URL #{url} could use HTTPS rather than HTTP" + return "The URL #{url} may be able to use HTTPS rather than HTTP. Please verify it in a browser." end lenratio = (100 * secure_details[:file].length / details[:file].length).to_i @@ -586,10 +587,10 @@ class FormulaAuditor return unless @online return unless DevelopmentTools.curl_handles_most_https_homepages? - if http_content_problem = FormulaAuditor.check_http_content("homepage", - homepage, + if http_content_problem = FormulaAuditor.check_http_content(homepage, formula.name, - user_agents: [:browser, :default]) + user_agents: [:browser, :default], + check_content: true) problem http_content_problem end end @@ -1242,8 +1243,7 @@ class ResourceAuditor # A `brew mirror`'ed URL is usually not yet reachable at the time of # pull request. next if url =~ %r{^https://dl.bintray.com/homebrew/mirror/} - if http_content_problem = FormulaAuditor.check_http_content("url", - url, name) + if http_content_problem = FormulaAuditor.check_http_content(url, name) problem http_content_problem end elsif strategy <= GitDownloadStrategy From 56ccf10efaac5f97bae6b2e5aef9ef87d7529328 Mon Sep 17 00:00:00 2001 From: Viktor Szakats Date: Tue, 29 Aug 2017 12:53:45 +0000 Subject: [PATCH 5/7] limit some heuristics to strict mode --- Library/Homebrew/dev-cmd/audit.rb | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/Library/Homebrew/dev-cmd/audit.rb b/Library/Homebrew/dev-cmd/audit.rb index 5a86c0adfd..17948ea2cc 100644 --- a/Library/Homebrew/dev-cmd/audit.rb +++ b/Library/Homebrew/dev-cmd/audit.rb @@ -201,7 +201,7 @@ class FormulaAuditor @specs = %w[stable devel head].map { |s| formula.send(s) }.compact end - def self.check_http_content(url, name, user_agents: [:default], check_content: false) + def self.check_http_content(url, name, user_agents: [:default], check_content: false, strict: false) return unless url.start_with? "http" details = nil @@ -251,6 +251,8 @@ class FormulaAuditor return "The URL #{url} should use HTTPS rather than HTTP" end + return unless strict + # Same size, different content after normalization # (typical causes: Generated ID, Timestamp, Unix time) if details[:file].length == secure_details[:file].length @@ -590,7 +592,8 @@ class FormulaAuditor if http_content_problem = FormulaAuditor.check_http_content(homepage, formula.name, user_agents: [:browser, :default], - check_content: true) + check_content: true, + strict: @strict) problem http_content_problem end end From 18f5b43d9001955784e235ce7f87068038175a82 Mon Sep 17 00:00:00 2001 From: Viktor Szakats Date: Tue, 29 Aug 2017 17:02:27 +0000 Subject: [PATCH 6/7] fix length ratio range --- Library/Homebrew/dev-cmd/audit.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Library/Homebrew/dev-cmd/audit.rb b/Library/Homebrew/dev-cmd/audit.rb index 17948ea2cc..93aab63841 100644 --- a/Library/Homebrew/dev-cmd/audit.rb +++ b/Library/Homebrew/dev-cmd/audit.rb @@ -260,7 +260,7 @@ class FormulaAuditor end lenratio = (100 * secure_details[:file].length / details[:file].length).to_i - return if lenratio < 90 || lenratio > 120 + return if lenratio < 90 || lenratio > 110 "The URL #{url} may be able to use HTTPS rather than HTTP. Please verify it in a browser." end From 42e2c71dbc9c744fdabab70bf51dce106a76f0a6 Mon Sep 17 00:00:00 2001 From: Viktor Szakats Date: Fri, 1 Sep 2017 16:47:31 +0000 Subject: [PATCH 7/7] cleanup range check --- Library/Homebrew/dev-cmd/audit.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Library/Homebrew/dev-cmd/audit.rb b/Library/Homebrew/dev-cmd/audit.rb index 93aab63841..f6a8b10a6b 100644 --- a/Library/Homebrew/dev-cmd/audit.rb +++ b/Library/Homebrew/dev-cmd/audit.rb @@ -260,7 +260,7 @@ class FormulaAuditor end lenratio = (100 * secure_details[:file].length / details[:file].length).to_i - return if lenratio < 90 || lenratio > 110 + return unless (90..110).cover?(lenratio) "The URL #{url} may be able to use HTTPS rather than HTTP. Please verify it in a browser." end