From 545a332fef7fa87e00f9893f92250d5d981f62f8 Mon Sep 17 00:00:00 2001 From: Markus Reiter Date: Tue, 2 May 2023 23:58:51 +0200 Subject: [PATCH 1/2] Use `curl_head` and `curl_output` for `Livecheck` strategies. --- Library/Homebrew/download_strategy.rb | 2 +- Library/Homebrew/livecheck/strategy.rb | 21 ++++++++------------- Library/Homebrew/utils/curl.rb | 8 ++++---- 3 files changed, 13 insertions(+), 18 deletions(-) diff --git a/Library/Homebrew/download_strategy.rb b/Library/Homebrew/download_strategy.rb index 699228d9f0..a49b47e709 100644 --- a/Library/Homebrew/download_strategy.rb +++ b/Library/Homebrew/download_strategy.rb @@ -473,7 +473,7 @@ class CurlDownloadStrategy < AbstractFileDownloadStrategy url = url.sub(%r{^https?://#{GitHubPackages::URL_DOMAIN}/}o, "#{domain.chomp("/")}/") end - parsed_output = curl_head(url.to_s, timeout: timeout) + parsed_output = curl_head(url.to_s, wanted_headers: ["content-disposition"], timeout: timeout) parsed_headers = parsed_output.fetch(:responses).map { |r| r.fetch(:headers) } final_url = curl_response_follow_redirections(parsed_output.fetch(:responses), url) diff --git a/Library/Homebrew/livecheck/strategy.rb b/Library/Homebrew/livecheck/strategy.rb index 65431ccf40..8ac14dd951 100644 --- a/Library/Homebrew/livecheck/strategy.rb +++ b/Library/Homebrew/livecheck/strategy.rb @@ -10,6 +10,8 @@ module Homebrew # # @api private module Strategy + extend Utils::Curl + module_function # {Strategy} priorities informally range from 1 to 10, where 10 is the @@ -53,14 +55,6 @@ module Homebrew "--silent" ].freeze - # `curl` arguments used in `Strategy#page_headers` method. - PAGE_HEADERS_CURL_ARGS = ([ - # We only need the response head (not the body) - "--head", - # Some servers may not allow a HEAD request, so we use GET - "--request", "GET" - ] + DEFAULT_CURL_ARGS).freeze - # `curl` arguments used in `Strategy#page_content` method. PAGE_CONTENT_CURL_ARGS = ([ "--compressed", @@ -188,11 +182,12 @@ module Homebrew headers = [] [:default, :browser].each do |user_agent| - output, _, status = curl_with_workarounds( - *PAGE_HEADERS_CURL_ARGS, url, - **DEFAULT_CURL_OPTIONS, + output, _, status = curl_head( + url, + wanted_headers: ["location", "content-disposition"], use_homebrew_curl: homebrew_curl, - user_agent: user_agent + user_agent: user_agent, + **DEFAULT_CURL_OPTIONS, ) next unless status.success? @@ -216,7 +211,7 @@ module Homebrew def self.page_content(url, homebrew_curl: false) stderr = T.let(nil, T.nilable(String)) [:default, :browser].each do |user_agent| - stdout, stderr, status = curl_with_workarounds( + stdout, stderr, status = curl_output( *PAGE_CONTENT_CURL_ARGS, url, **DEFAULT_CURL_OPTIONS, use_homebrew_curl: homebrew_curl, diff --git a/Library/Homebrew/utils/curl.rb b/Library/Homebrew/utils/curl.rb index b32da61da0..4a6ca0cedd 100644 --- a/Library/Homebrew/utils/curl.rb +++ b/Library/Homebrew/utils/curl.rb @@ -205,7 +205,7 @@ module Utils curl_with_workarounds(*args, print_stderr: false, show_output: true, **options) end - def curl_head(*args, **options) + def curl_head(*args, wanted_headers: [], **options) [[], ["--request", "GET"]].each do |request_args| result = curl_output( "--fail", "--location", "--silent", "--head", *request_args, *args, @@ -216,9 +216,9 @@ module Utils if result.success? || result.exit_status == 22 parsed_output = parse_curl_output(result.stdout) - # If we didn't get a `Content-Disposition` header yet, retry using `GET`. - next if request_args.empty? && - parsed_output.fetch(:responses).none? { |r| r.fetch(:headers).key?("content-disposition") } + # If we didn't get any wanted header yet, retry using `GET`. + next if request_args.empty? && wanted_headers.any? && + parsed_output.fetch(:responses).none? { |r| (r.fetch(:headers).keys & wanted_headers).any? } return parsed_output if result.success? end From 353818f508808908d301efde1a6be53e64f90fe6 Mon Sep 17 00:00:00 2001 From: Markus Reiter Date: Sat, 6 May 2023 03:41:35 +0200 Subject: [PATCH 2/2] Rename `curl_head` to `curl_headers`. --- Library/Homebrew/download_strategy.rb | 2 +- Library/Homebrew/livecheck/strategy.rb | 2 +- Library/Homebrew/test/download_strategies/curl_spec.rb | 2 +- Library/Homebrew/utils/curl.rb | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Library/Homebrew/download_strategy.rb b/Library/Homebrew/download_strategy.rb index a49b47e709..2b2797d620 100644 --- a/Library/Homebrew/download_strategy.rb +++ b/Library/Homebrew/download_strategy.rb @@ -473,7 +473,7 @@ class CurlDownloadStrategy < AbstractFileDownloadStrategy url = url.sub(%r{^https?://#{GitHubPackages::URL_DOMAIN}/}o, "#{domain.chomp("/")}/") end - parsed_output = curl_head(url.to_s, wanted_headers: ["content-disposition"], timeout: timeout) + parsed_output = curl_headers(url.to_s, wanted_headers: ["content-disposition"], timeout: timeout) parsed_headers = parsed_output.fetch(:responses).map { |r| r.fetch(:headers) } final_url = curl_response_follow_redirections(parsed_output.fetch(:responses), url) diff --git a/Library/Homebrew/livecheck/strategy.rb b/Library/Homebrew/livecheck/strategy.rb index 8ac14dd951..72df2f4934 100644 --- a/Library/Homebrew/livecheck/strategy.rb +++ b/Library/Homebrew/livecheck/strategy.rb @@ -182,7 +182,7 @@ module Homebrew headers = [] [:default, :browser].each do |user_agent| - output, _, status = curl_head( + output, _, status = curl_headers( url, wanted_headers: ["location", "content-disposition"], use_homebrew_curl: homebrew_curl, diff --git a/Library/Homebrew/test/download_strategies/curl_spec.rb b/Library/Homebrew/test/download_strategies/curl_spec.rb index fc3dfb1df7..106eaf93bf 100644 --- a/Library/Homebrew/test/download_strategies/curl_spec.rb +++ b/Library/Homebrew/test/download_strategies/curl_spec.rb @@ -12,7 +12,7 @@ describe CurlDownloadStrategy do let(:artifact_domain) { nil } before do - allow(strategy).to receive(:curl_head).and_return({ responses: [{ headers: {} }] }) + allow(strategy).to receive(:curl_headers).and_return({ responses: [{ headers: {} }] }) end it "parses the opts and sets the corresponding args" do diff --git a/Library/Homebrew/utils/curl.rb b/Library/Homebrew/utils/curl.rb index 4a6ca0cedd..4b2d3b227f 100644 --- a/Library/Homebrew/utils/curl.rb +++ b/Library/Homebrew/utils/curl.rb @@ -205,7 +205,7 @@ module Utils curl_with_workarounds(*args, print_stderr: false, show_output: true, **options) end - def curl_head(*args, wanted_headers: [], **options) + def curl_headers(*args, wanted_headers: [], **options) [[], ["--request", "GET"]].each do |request_args| result = curl_output( "--fail", "--location", "--silent", "--head", *request_args, *args,