Merge pull request #19077 from Homebrew/curl-typed-strict

Curl: Use `typed: strict`
This commit is contained in:
Sam Ford 2025-01-15 11:54:43 +00:00 committed by GitHub
commit 256e826c3a
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
13 changed files with 336 additions and 80 deletions

View File

@ -204,7 +204,7 @@ module Cask
begin begin
ohai "Downloading #{url}" ohai "Downloading #{url}"
::Utils::Curl.curl_download url, to: path ::Utils::Curl.curl_download url.to_s, to: path
rescue ErrorDuringExecution rescue ErrorDuringExecution
raise CaskUnavailableError.new(token, "Failed to download #{Formatter.url(url)}.") raise CaskUnavailableError.new(token, "Failed to download #{Formatter.url(url)}.")
end end

View File

@ -99,21 +99,26 @@ module Cask
class BlockDSL class BlockDSL
# Allow accessing the URL associated with page contents. # Allow accessing the URL associated with page contents.
module PageWithURL class PageWithURL < SimpleDelegator
# Get the URL of the fetched page. # Get the URL of the fetched page.
# #
# ### Example # ### Example
# #
# ```ruby # ```ruby
# url "https://example.org/download" do |page| # url "https://example.org/download" do |page|
# file = page[/href="([^"]+.dmg)"/, 1] # file_path = page[/href="([^"]+\.dmg)"/, 1]
# URL.join(page.url, file) # URI.join(page.url, file_path)
# end # end
# ``` # ```
# #
# @api public # @api public
sig { returns(URI::Generic) } sig { returns(URI::Generic) }
attr_accessor :url attr_accessor :url
def initialize(str, url)
super(str)
@url = url
end
end end
sig { sig {
@ -135,13 +140,10 @@ module Cask
sig { returns(T.any(T.any(URI::Generic, String), [T.any(URI::Generic, String), Hash])) } sig { returns(T.any(T.any(URI::Generic, String), [T.any(URI::Generic, String), Hash])) }
def call def call
if @uri if @uri
result = ::Utils::Curl.curl_output("--fail", "--silent", "--location", @uri) result = ::Utils::Curl.curl_output("--fail", "--silent", "--location", @uri.to_s)
result.assert_success! result.assert_success!
page = result.stdout page = PageWithURL.new(result.stdout, URI(@uri))
page.extend PageWithURL
page.url = URI(@uri)
instance_exec(page, &@block) instance_exec(page, &@block)
else else
instance_exec(&@block) instance_exec(&@block)

View File

@ -723,7 +723,7 @@ module Formulary
end end
HOMEBREW_CACHE_FORMULA.mkpath HOMEBREW_CACHE_FORMULA.mkpath
FileUtils.rm_f(path) FileUtils.rm_f(path)
Utils::Curl.curl_download url, to: path Utils::Curl.curl_download url.to_s, to: path
super super
rescue MethodDeprecatedError => e rescue MethodDeprecatedError => e
if (match_data = url.match(%r{github.com/(?<user>[\w-]+)/(?<repo>[\w-]+)/}).presence) if (match_data = url.match(%r{github.com/(?<user>[\w-]+)/(?<repo>[\w-]+)/}).presence)

View File

@ -169,7 +169,7 @@ class GitHubPackages
# Going forward, this should probably be pinned to tags. # Going forward, this should probably be pinned to tags.
# We currently use features newer than the last one (v1.0.2). # We currently use features newer than the last one (v1.0.2).
url = "https://raw.githubusercontent.com/opencontainers/image-spec/170393e57ed656f7f81c3070bfa8c3346eaa0a5a/schema/#{basename}.json" url = "https://raw.githubusercontent.com/opencontainers/image-spec/170393e57ed656f7f81c3070bfa8c3346eaa0a5a/schema/#{basename}.json"
out, = Utils::Curl.curl_output(url) out = Utils::Curl.curl_output(url).stdout
json = JSON.parse(out) json = JSON.parse(out)
@schema_json ||= {} @schema_json ||= {}

View File

@ -37,7 +37,7 @@ RSpec.describe CurlDownloadStrategy do
it "calls curl with default arguments" do it "calls curl with default arguments" do
expect(strategy).to receive(:curl).with( expect(strategy).to receive(:curl).with(
"--remote-time", "--remote-time",
"--output", an_instance_of(Pathname), "--output", an_instance_of(String),
# example.com supports partial requests. # example.com supports partial requests.
"--continue-at", "-", "--continue-at", "-",
"--location", "--location",

View File

@ -138,6 +138,12 @@ RSpec.describe "Utils::Curl" do
}, },
} }
response_hash[:ok_no_status_text] = response_hash[:ok].deep_dup
response_hash[:ok_no_status_text].delete(:status_text)
response_hash[:ok_blank_header_value] = response_hash[:ok].deep_dup
response_hash[:ok_blank_header_value][:headers]["range"] = ""
response_hash[:redirection] = { response_hash[:redirection] = {
status_code: "301", status_code: "301",
status_text: "Moved Permanently", status_text: "Moved Permanently",
@ -257,6 +263,16 @@ RSpec.describe "Utils::Curl" do
\r \r
EOS EOS
response_text[:ok_no_status_text] = response_text[:ok].sub(" #{response_hash[:ok][:status_text]}", "")
response_text[:ok_blank_header_name] = response_text[:ok].sub(
"#{response_hash[:ok][:headers]["date"]}\r\n",
"#{response_hash[:ok][:headers]["date"]}\r\n: Test\r\n",
)
response_text[:ok_blank_header_value] = response_text[:ok].sub(
"#{response_hash[:ok][:headers]["date"]}\r\n",
"#{response_hash[:ok][:headers]["date"]}\r\nRange:\r\n",
)
response_text[:redirection] = response_text[:ok].sub( response_text[:redirection] = response_text[:ok].sub(
"HTTP/1.1 #{response_hash[:ok][:status_code]} #{response_hash[:ok][:status_text]}\r", "HTTP/1.1 #{response_hash[:ok][:status_code]} #{response_hash[:ok][:status_text]}\r",
"HTTP/1.1 #{response_hash[:redirection][:status_code]} #{response_hash[:redirection][:status_text]}\r\n" \ "HTTP/1.1 #{response_hash[:redirection][:status_code]} #{response_hash[:redirection][:status_text]}\r\n" \
@ -306,7 +322,27 @@ RSpec.describe "Utils::Curl" do
body body
end end
describe "curl_args" do describe "::curl_executable" do
it "returns `HOMEBREW_BREWED_CURL_PATH` when `use_homebrew_curl` is `true`" do
expect(curl_executable(use_homebrew_curl: true)).to eq(HOMEBREW_BREWED_CURL_PATH)
end
it "returns curl shim path when `use_homebrew_curl` is `false` or omitted" do
curl_shim_path = HOMEBREW_SHIMS_PATH/"shared/curl"
expect(curl_executable(use_homebrew_curl: false)).to eq(curl_shim_path)
expect(curl_executable).to eq(curl_shim_path)
end
end
describe "::curl_path" do
it "returns a curl path string" do
expect(curl_path).to match(%r{[^/]+(?:/[^/]+)*})
end
end
describe "::curl_args" do
include Context
let(:args) { ["foo"] } let(:args) { ["foo"] }
let(:user_agent_string) { "Lorem ipsum dolor sit amet" } let(:user_agent_string) { "Lorem ipsum dolor sit amet" }
@ -388,6 +424,11 @@ RSpec.describe "Utils::Curl" do
expect { curl_args(*args, retry_max_time: "test") }.to raise_error(TypeError) expect { curl_args(*args, retry_max_time: "test") }.to raise_error(TypeError)
end end
it "uses `--show-error` when :show_error is `true`" do
expect(curl_args(*args, show_error: true)).to include("--show-error")
expect(curl_args(*args, show_error: false)).not_to include("--show-error")
end
it "uses `--referer` when :referer is present" do it "uses `--referer` when :referer is present" do
expect(curl_args(*args, referer: "https://brew.sh").join(" ")).to include("--referer https://brew.sh") expect(curl_args(*args, referer: "https://brew.sh").join(" ")).to include("--referer https://brew.sh")
end end
@ -426,9 +467,62 @@ RSpec.describe "Utils::Curl" do
expect(curl_args(*args).join(" ")).to include("--fail") expect(curl_args(*args).join(" ")).to include("--fail")
expect(curl_args(*args, show_output: true).join(" ")).not_to include("--fail") expect(curl_args(*args, show_output: true).join(" ")).not_to include("--fail")
end end
it "uses `--progress-bar` outside of a `--verbose` context" do
expect(curl_args(*args).join(" ")).to include("--progress-bar")
with_context verbose: true do
expect(curl_args(*args).join(" ")).not_to include("--progress-bar")
end
end end
describe "url_protected_by_cloudflare?" do context "when `EnvConfig.curl_verbose?` is `true`" do
before do
allow(Homebrew::EnvConfig).to receive(:curl_verbose?).and_return(true)
end
it "uses `--verbose`" do
expect(curl_args(*args).join(" ")).to include("--verbose")
end
end
context "when `EnvConfig.curl_verbose?` is `false`" do
before do
allow(Homebrew::EnvConfig).to receive(:curl_verbose?).and_return(false)
end
it "doesn't use `--verbose`" do
expect(curl_args(*args).join(" ")).not_to include("--verbose")
end
end
context "when `$stdout.tty?` is `false`" do
before do
allow($stdout).to receive(:tty?).and_return(false)
end
it "uses `--silent`" do
expect(curl_args(*args).join(" ")).to include("--silent")
end
end
context "when `$stdout.tty?` is `true`" do
before do
allow($stdout).to receive(:tty?).and_return(true)
end
it "doesn't use `--silent` outside of a `--quiet` context" do
with_context quiet: false do
expect(curl_args(*args).join(" ")).not_to include("--silent")
end
with_context quiet: true do
expect(curl_args(*args).join(" ")).to include("--silent")
end
end
end
end
describe "::url_protected_by_cloudflare?" do
it "returns `true` when a URL is protected by Cloudflare" do it "returns `true` when a URL is protected by Cloudflare" do
expect(url_protected_by_cloudflare?(details[:cloudflare][:single_cookie])).to be(true) expect(url_protected_by_cloudflare?(details[:cloudflare][:single_cookie])).to be(true)
expect(url_protected_by_cloudflare?(details[:cloudflare][:multiple_cookies])).to be(true) expect(url_protected_by_cloudflare?(details[:cloudflare][:multiple_cookies])).to be(true)
@ -448,7 +542,7 @@ RSpec.describe "Utils::Curl" do
end end
end end
describe "url_protected_by_incapsula?" do describe "::url_protected_by_incapsula?" do
it "returns `true` when a URL is protected by Cloudflare" do it "returns `true` when a URL is protected by Cloudflare" do
expect(url_protected_by_incapsula?(details[:incapsula][:single_cookie_visid_incap])).to be(true) expect(url_protected_by_incapsula?(details[:incapsula][:single_cookie_visid_incap])).to be(true)
expect(url_protected_by_incapsula?(details[:incapsula][:single_cookie_incap_ses])).to be(true) expect(url_protected_by_incapsula?(details[:incapsula][:single_cookie_incap_ses])).to be(true)
@ -468,7 +562,54 @@ RSpec.describe "Utils::Curl" do
end end
end end
describe "#parse_curl_output" do describe "::curl_version" do
it "returns a curl version string" do
expect(curl_version).to match(/^v?(\d+(?:\.\d+)+)$/)
end
end
describe "::curl_supports_fail_with_body?" do
it "returns `true` if curl version is 7.76.0 or higher" do
allow_any_instance_of(Utils::Curl).to receive(:curl_version).and_return(Version.new("7.76.0"))
expect(curl_supports_fail_with_body?).to be(true)
allow_any_instance_of(Utils::Curl).to receive(:curl_version).and_return(Version.new("7.76.1"))
expect(curl_supports_fail_with_body?).to be(true)
end
it "returns `false` if curl version is lower than 7.76.0" do
allow_any_instance_of(Utils::Curl).to receive(:curl_version).and_return(Version.new("7.75.0"))
expect(curl_supports_fail_with_body?).to be(false)
end
end
describe "::curl_supports_tls13?" do
it "returns `true` if curl command is successful" do
allow_any_instance_of(Kernel).to receive(:quiet_system).and_return(true)
expect(curl_supports_tls13?).to be(true)
end
it "returns `false` if curl command is not successful" do
allow_any_instance_of(Kernel).to receive(:quiet_system).and_return(false)
expect(curl_supports_tls13?).to be(false)
end
end
describe "::http_status_ok?" do
it "returns `true` when `status` is 1xx or 2xx" do
expect(http_status_ok?("200")).to be(true)
end
it "returns `false` when `status` is not 1xx or 2xx" do
expect(http_status_ok?("301")).to be(false)
end
it "returns `false` when `status` is `nil`" do
expect(http_status_ok?(nil)).to be(false)
end
end
describe "::parse_curl_output" do
it "returns a correct hash when curl output contains response(s) and body" do it "returns a correct hash when curl output contains response(s) and body" do
expect(parse_curl_output("#{response_text[:ok]}#{body[:default]}")) expect(parse_curl_output("#{response_text[:ok]}#{body[:default]}"))
.to eq({ responses: [response_hash[:ok]], body: body[:default] }) .to eq({ responses: [response_hash[:ok]], body: body[:default] })
@ -505,21 +646,33 @@ RSpec.describe "Utils::Curl" do
it "returns correct hash when curl output is blank" do it "returns correct hash when curl output is blank" do
expect(parse_curl_output("")).to eq({ responses: [], body: "" }) expect(parse_curl_output("")).to eq({ responses: [], body: "" })
end end
it "errors if response count exceeds `max_iterations`" do
expect do
parse_curl_output(response_text[:redirections_to_ok], max_iterations: 1)
end.to raise_error("Too many redirects (max = 1)")
end
end end
describe "#parse_curl_response" do describe "::parse_curl_response" do
it "returns a correct hash when given HTTP response text" do it "returns a correct hash when given HTTP response text" do
expect(parse_curl_response(response_text[:ok])).to eq(response_hash[:ok]) expect(parse_curl_response(response_text[:ok])).to eq(response_hash[:ok])
expect(parse_curl_response(response_text[:ok_no_status_text])).to eq(response_hash[:ok_no_status_text])
expect(parse_curl_response(response_text[:ok_blank_header_value])).to eq(response_hash[:ok_blank_header_value])
expect(parse_curl_response(response_text[:redirection])).to eq(response_hash[:redirection]) expect(parse_curl_response(response_text[:redirection])).to eq(response_hash[:redirection])
expect(parse_curl_response(response_text[:duplicate_header])).to eq(response_hash[:duplicate_header]) expect(parse_curl_response(response_text[:duplicate_header])).to eq(response_hash[:duplicate_header])
end end
it "skips over response header lines with blank header name" do
expect(parse_curl_response(response_text[:ok_blank_header_name])).to eq(response_hash[:ok])
end
it "returns an empty hash when given an empty string" do it "returns an empty hash when given an empty string" do
expect(parse_curl_response("")).to eq({}) expect(parse_curl_response("")).to eq({})
end end
end end
describe "#curl_response_last_location" do describe "::curl_response_last_location" do
it "returns the last location header when given an array of HTTP response hashes" do it "returns the last location header when given an array of HTTP response hashes" do
expect(curl_response_last_location([ expect(curl_response_last_location([
response_hash[:redirection], response_hash[:redirection],
@ -577,12 +730,20 @@ RSpec.describe "Utils::Curl" do
).to eq(response_hash[:redirection_parent_relative][:headers]["location"].sub(/^\./, "https://brew.sh/test1")) ).to eq(response_hash[:redirection_parent_relative][:headers]["location"].sub(/^\./, "https://brew.sh/test1"))
end end
it "skips response hashes without a `:headers` value" do
expect(curl_response_last_location([
response_hash[:redirection],
{ status_code: "404", status_text: "Not Found" },
response_hash[:ok],
])).to eq(response_hash[:redirection][:headers]["location"])
end
it "returns nil when the response hash doesn't contain a location header" do it "returns nil when the response hash doesn't contain a location header" do
expect(curl_response_last_location([response_hash[:ok]])).to be_nil expect(curl_response_last_location([response_hash[:ok]])).to be_nil
end end
end end
describe "#curl_response_follow_redirections" do describe "::curl_response_follow_redirections" do
it "returns the original URL when there are no location headers" do it "returns the original URL when there are no location headers" do
expect( expect(
curl_response_follow_redirections( curl_response_follow_redirections(
@ -634,5 +795,18 @@ RSpec.describe "Utils::Curl" do
), ),
).to eq("#{location_urls[0]}example/") ).to eq("#{location_urls[0]}example/")
end end
it "skips response hashes without a `:headers` value" do
expect(
curl_response_follow_redirections(
[
response_hash[:redirection_root_relative],
{ status_code: "404", status_text: "Not Found" },
response_hash[:ok],
],
"https://brew.sh/test1/test2",
),
).to eq("https://brew.sh/example/")
end
end end
end end

View File

@ -289,7 +289,7 @@ module Utils
formula_version_urls = output.stdout formula_version_urls = output.stdout
.scan(%r{/orgs/Homebrew/packages/#{formula_url_suffix}\d+\?tag=[^"]+}) .scan(%r{/orgs/Homebrew/packages/#{formula_url_suffix}\d+\?tag=[^"]+})
.map do |url| .map do |url|
url.sub("/orgs/Homebrew/packages/", "/Homebrew/homebrew-core/pkgs/") T.cast(url, String).sub("/orgs/Homebrew/packages/", "/Homebrew/homebrew-core/pkgs/")
end end
return if formula_version_urls.empty? return if formula_version_urls.empty?
@ -304,9 +304,9 @@ module Utils
) )
next if last_thirty_days_match.blank? next if last_thirty_days_match.blank?
last_thirty_days_downloads = last_thirty_days_match.captures.first.tr(",", "") last_thirty_days_downloads = T.must(last_thirty_days_match.captures.first).tr(",", "")
thirty_day_download_count += if (millions_match = last_thirty_days_downloads.match(/(\d+\.\d+)M/).presence) thirty_day_download_count += if (millions_match = last_thirty_days_downloads.match(/(\d+\.\d+)M/).presence)
millions_match.captures.first.to_f * 1_000_000 (millions_match.captures.first.to_f * 1_000_000).to_i
else else
last_thirty_days_downloads.to_i last_thirty_days_downloads.to_i
end end

View File

@ -1,4 +1,4 @@
# typed: true # rubocop:todo Sorbet/StrictSigil # typed: strict
# frozen_string_literal: true # frozen_string_literal: true
require "open3" require "open3"
@ -48,23 +48,29 @@ module Utils
module_function module_function
sig { params(use_homebrew_curl: T::Boolean).returns(T.any(Pathname, String)) }
def curl_executable(use_homebrew_curl: false) def curl_executable(use_homebrew_curl: false)
return HOMEBREW_BREWED_CURL_PATH if use_homebrew_curl return HOMEBREW_BREWED_CURL_PATH if use_homebrew_curl
@curl_executable ||= HOMEBREW_SHIMS_PATH/"shared/curl" @curl_executable ||= T.let(HOMEBREW_SHIMS_PATH/"shared/curl", T.nilable(T.any(Pathname, String)))
end end
sig { returns(String) }
def curl_path def curl_path
@curl_path ||= Utils.popen_read(curl_executable, "--homebrew=print-path").chomp.presence @curl_path ||= T.let(
Utils.popen_read(curl_executable, "--homebrew=print-path").chomp.presence,
T.nilable(String),
)
end end
sig { void }
def clear_path_cache def clear_path_cache
@curl_path = nil @curl_path = nil
end end
sig { sig {
params( params(
extra_args: T.untyped, extra_args: String,
connect_timeout: T.any(Integer, Float, NilClass), connect_timeout: T.any(Integer, Float, NilClass),
max_time: T.any(Integer, Float, NilClass), max_time: T.any(Integer, Float, NilClass),
retries: T.nilable(Integer), retries: T.nilable(Integer),
@ -140,9 +146,23 @@ module Utils
(args + extra_args).map(&:to_s) (args + extra_args).map(&:to_s)
end end
sig {
params(
args: String,
secrets: T.any(String, T::Array[String]),
print_stdout: T.any(T::Boolean, Symbol),
print_stderr: T.any(T::Boolean, Symbol),
debug: T.nilable(T::Boolean),
verbose: T.nilable(T::Boolean),
env: T::Hash[String, String],
timeout: T.nilable(T.any(Integer, Float)),
use_homebrew_curl: T::Boolean,
options: T.untyped,
).returns(SystemCommand::Result)
}
def curl_with_workarounds( def curl_with_workarounds(
*args, *args,
secrets: nil, print_stdout: nil, print_stderr: nil, debug: nil, secrets: [], print_stdout: false, print_stderr: false, debug: nil,
verbose: nil, env: {}, timeout: nil, use_homebrew_curl: false, **options verbose: nil, env: {}, timeout: nil, use_homebrew_curl: false, **options
) )
end_time = Time.now + timeout if timeout end_time = Time.now + timeout if timeout
@ -190,13 +210,28 @@ module Utils
result result
end end
sig {
params(
args: String,
print_stdout: T.any(T::Boolean, Symbol),
options: T.untyped,
).returns(SystemCommand::Result)
}
def curl(*args, print_stdout: true, **options) def curl(*args, print_stdout: true, **options)
result = curl_with_workarounds(*args, print_stdout:, **options) result = curl_with_workarounds(*args, print_stdout:, **options)
result.assert_success! result.assert_success!
result result
end end
def curl_download(*args, to: nil, try_partial: false, **options) sig {
params(
args: String,
to: T.any(Pathname, String),
try_partial: T::Boolean,
options: T.untyped,
).returns(T.nilable(SystemCommand::Result))
}
def curl_download(*args, to:, try_partial: false, **options)
destination = Pathname(to) destination = Pathname(to)
destination.dirname.mkpath destination.dirname.mkpath
@ -224,15 +259,23 @@ module Utils
end end
end end
args = ["--remote-time", "--output", destination, *args] args = ["--remote-time", "--output", destination.to_s, *args]
curl(*args, **options) curl(*args, **options)
end end
sig { params(args: String, options: T.untyped).returns(SystemCommand::Result) }
def curl_output(*args, **options) def curl_output(*args, **options)
curl_with_workarounds(*args, print_stderr: false, show_output: true, **options) curl_with_workarounds(*args, print_stderr: false, show_output: true, **options)
end end
sig {
params(
args: String,
wanted_headers: T::Array[String],
options: T.untyped,
).returns(T::Hash[Symbol, T.untyped])
}
def curl_headers(*args, wanted_headers: [], **options) def curl_headers(*args, wanted_headers: [], **options)
get_retry_args = ["--request", "GET"] get_retry_args = ["--request", "GET"]
# This is a workaround for https://github.com/Homebrew/brew/issues/18213 # This is a workaround for https://github.com/Homebrew/brew/issues/18213
@ -268,6 +311,8 @@ module Utils
result.assert_success! result.assert_success!
end end
{}
end end
# Check if a URL is protected by CloudFlare (e.g. badlion.net and jaxx.io). # Check if a URL is protected by CloudFlare (e.g. badlion.net and jaxx.io).
@ -295,6 +340,18 @@ module Utils
set_cookie_header.compact.any? { |cookie| cookie.match?(/^(visid_incap|incap_ses)_/i) } set_cookie_header.compact.any? { |cookie| cookie.match?(/^(visid_incap|incap_ses)_/i) }
end end
sig {
params(
url: String,
url_type: String,
specs: T::Hash[Symbol, String],
user_agents: T::Array[Symbol],
referer: T.nilable(String),
check_content: T::Boolean,
strict: T::Boolean,
use_homebrew_curl: T::Boolean,
).returns(T.nilable(String))
}
def curl_check_http_content(url, url_type, specs: {}, user_agents: [:default], referer: nil, def curl_check_http_content(url, url_type, specs: {}, user_agents: [:default], referer: nil,
check_content: false, strict: false, use_homebrew_curl: false) check_content: false, strict: false, use_homebrew_curl: false)
return unless url.start_with? "http" return unless url.start_with? "http"
@ -325,7 +382,7 @@ module Utils
end end
end end
details = T.let(nil, T.nilable(T::Hash[Symbol, T.untyped])) details = T.let({}, T::Hash[Symbol, T.untyped])
attempts = 0 attempts = 0
user_agents.each do |user_agent| user_agents.each do |user_agent|
loop do loop do
@ -373,7 +430,9 @@ module Utils
return "The #{url_type} #{url} is not reachable (HTTP status code #{details[:status_code]})" return "The #{url_type} #{url} is not reachable (HTTP status code #{details[:status_code]})"
end end
"Unable to find homepage" if SharedAudits.github_repo_data(repo_details[:user], repo_details[:repo]).nil? if SharedAudits.github_repo_data(T.must(repo_details[:user]), T.must(repo_details[:repo])).nil?
"Unable to find homepage"
end
end end
if url.start_with?("https://") && Homebrew::EnvConfig.no_insecure_redirect? && if url.start_with?("https://") && Homebrew::EnvConfig.no_insecure_redirect? &&
@ -425,6 +484,16 @@ module Utils
"The #{url_type} #{url} may be able to use HTTPS rather than HTTP. Please verify it in a browser." "The #{url_type} #{url} may be able to use HTTPS rather than HTTP. Please verify it in a browser."
end end
sig {
params(
url: String,
specs: T::Hash[Symbol, String],
hash_needed: T::Boolean,
use_homebrew_curl: T::Boolean,
user_agent: Symbol,
referer: T.nilable(String),
).returns(T::Hash[Symbol, T.untyped])
}
def curl_http_content_headers_and_checksum( def curl_http_content_headers_and_checksum(
url, specs: {}, hash_needed: false, url, specs: {}, hash_needed: false,
use_homebrew_curl: false, user_agent: :default, referer: nil use_homebrew_curl: false, user_agent: :default, referer: nil
@ -504,26 +573,32 @@ module Utils
T.must(file).unlink T.must(file).unlink
end end
sig { returns(Version) }
def curl_version def curl_version
@curl_version ||= {} @curl_version ||= T.let({}, T.nilable(T::Hash[String, Version]))
@curl_version[curl_path] ||= Version.new(curl_output("-V").stdout[/curl (\d+(\.\d+)+)/, 1]) @curl_version[curl_path] ||= Version.new(T.must(curl_output("-V").stdout[/curl (\d+(\.\d+)+)/, 1]))
end end
sig { returns(T::Boolean) }
def curl_supports_fail_with_body? def curl_supports_fail_with_body?
@curl_supports_fail_with_body ||= Hash.new do |h, key| @curl_supports_fail_with_body ||= T.let(Hash.new do |h, key|
h[key] = curl_version >= Version.new("7.76.0") h[key] = curl_version >= Version.new("7.76.0")
end end, T.nilable(T::Hash[T.any(Pathname, String), T::Boolean]))
@curl_supports_fail_with_body[curl_path] @curl_supports_fail_with_body[curl_path]
end end
sig { returns(T::Boolean) }
def curl_supports_tls13? def curl_supports_tls13?
@curl_supports_tls13 ||= Hash.new do |h, key| @curl_supports_tls13 ||= T.let(Hash.new do |h, key|
h[key] = quiet_system(curl_executable, "--tlsv1.3", "--head", "https://brew.sh/") h[key] = quiet_system(curl_executable, "--tlsv1.3", "--head", "https://brew.sh/")
end end, T.nilable(T::Hash[T.any(Pathname, String), T::Boolean]))
@curl_supports_tls13[curl_path] @curl_supports_tls13[curl_path]
end end
sig { params(status: T.nilable(String)).returns(T::Boolean) }
def http_status_ok?(status) def http_status_ok?(status)
return false if status.nil?
(100..299).cover?(status.to_i) (100..299).cover?(status.to_i)
end end
@ -627,11 +702,10 @@ module Utils
sig { params(response_text: String).returns(T::Hash[Symbol, T.untyped]) } sig { params(response_text: String).returns(T::Hash[Symbol, T.untyped]) }
def parse_curl_response(response_text) def parse_curl_response(response_text)
response = {} response = {}
return response unless response_text.match?(HTTP_STATUS_LINE_REGEX) return response unless (match = response_text.match(HTTP_STATUS_LINE_REGEX))
# Parse the status line and remove it # Parse the status line and remove it
match = T.must(response_text.match(HTTP_STATUS_LINE_REGEX)) response[:status_code] = match["code"]
response[:status_code] = match["code"] if match["code"].present?
response[:status_text] = match["text"] if match["text"].present? response[:status_text] = match["text"] if match["text"].present?
response_text = response_text.sub(%r{^HTTP/.* (\d+).*$\s*}, "") response_text = response_text.sub(%r{^HTTP/.* (\d+).*$\s*}, "")
@ -639,18 +713,18 @@ module Utils
response[:headers] = {} response[:headers] = {}
response_text.split("\r\n").each do |line| response_text.split("\r\n").each do |line|
header_name, header_value = line.split(/:\s*/, 2) header_name, header_value = line.split(/:\s*/, 2)
next if header_name.blank? next if header_name.blank? || header_value.nil?
header_name = header_name.strip.downcase header_name = header_name.strip.downcase
header_value&.strip! header_value.strip!
case response[:headers][header_name] case response[:headers][header_name]
when nil
response[:headers][header_name] = header_value
when String when String
response[:headers][header_name] = [response[:headers][header_name], header_value] response[:headers][header_name] = [response[:headers][header_name], header_value]
when Array when Array
response[:headers][header_name].push(header_value) response[:headers][header_name].push(header_value)
else
response[:headers][header_name] = header_value
end end
response[:headers][header_name] response[:headers][header_name]

View File

@ -828,15 +828,15 @@ module GitHub
return if Homebrew::EnvConfig.no_github_api? return if Homebrew::EnvConfig.no_github_api?
require "utils/curl" require "utils/curl"
output, _, status = Utils::Curl.curl_output( result = Utils::Curl.curl_output(
"--silent", "--head", "--location", "--silent", "--head", "--location",
"--header", "Accept: application/vnd.github.sha", "--header", "Accept: application/vnd.github.sha",
url_to("repos", user, repo, "commits", ref).to_s url_to("repos", user, repo, "commits", ref).to_s
) )
return unless status.success? return unless result.status.success?
commit = output[/^ETag: "(\h+)"/, 1] commit = result.stdout[/^ETag: "(\h+)"/, 1]
return if commit.blank? return if commit.blank?
version.update_commit(commit) version.update_commit(commit)
@ -847,14 +847,14 @@ module GitHub
return false if Homebrew::EnvConfig.no_github_api? return false if Homebrew::EnvConfig.no_github_api?
require "utils/curl" require "utils/curl"
output, _, status = Utils::Curl.curl_output( result = Utils::Curl.curl_output(
"--silent", "--head", "--location", "--silent", "--head", "--location",
"--header", "Accept: application/vnd.github.sha", "--header", "Accept: application/vnd.github.sha",
url_to("repos", user, repo, "commits", commit).to_s url_to("repos", user, repo, "commits", commit).to_s
) )
return true unless status.success? return true unless result.status.success?
return true if output.blank? return true if (output = result.stdout).blank?
output[/^Status: (200)/, 1] != "200" output[/^Status: (200)/, 1] != "200"
end end

View File

@ -274,8 +274,8 @@ module GitHub
args += ["--dump-header", T.must(headers_tmpfile.path)] args += ["--dump-header", T.must(headers_tmpfile.path)]
require "utils/curl" require "utils/curl"
output, errors, status = Utils::Curl.curl_output("--location", url.to_s, *args, secrets: [token]) result = Utils::Curl.curl_output("--location", url.to_s, *args, secrets: [token])
output, _, http_code = output.rpartition("\n") output, _, http_code = result.stdout.rpartition("\n")
output, _, http_code = output.rpartition("\n") if http_code == "000" output, _, http_code = output.rpartition("\n") if http_code == "000"
headers = headers_tmpfile.read headers = headers_tmpfile.read
ensure ensure
@ -288,7 +288,9 @@ module GitHub
end end
begin begin
raise_error(output, errors, http_code, headers, scopes) if !http_code.start_with?("2") || !status.success? if !http_code.start_with?("2") || !result.status.success?
raise_error(output, result.stderr, http_code, headers, scopes)
end
return if http_code == "204" # No Content return if http_code == "204" # No Content

View File

@ -65,12 +65,12 @@ module PyPI
else else
"https://pypi.org/pypi/#{name}/json" "https://pypi.org/pypi/#{name}/json"
end end
out, _, status = Utils::Curl.curl_output metadata_url, "--location", "--fail" result = Utils::Curl.curl_output(metadata_url, "--location", "--fail")
return unless status.success? return unless result.status.success?
begin begin
json = JSON.parse out json = JSON.parse(result.stdout)
rescue JSON::ParserError rescue JSON::ParserError
return return
end end

View File

@ -15,14 +15,16 @@ module Repology
last_package_in_response += "/" if last_package_in_response.present? last_package_in_response += "/" if last_package_in_response.present?
url = "https://repology.org/api/v1/projects/#{last_package_in_response}?inrepo=#{repository}&outdated=1" url = "https://repology.org/api/v1/projects/#{last_package_in_response}?inrepo=#{repository}&outdated=1"
output, errors, = Utils::Curl.curl_output(url.to_s, "--silent", result = Utils::Curl.curl_output(
use_homebrew_curl: !Utils::Curl.curl_supports_tls13?) "--silent", url.to_s,
JSON.parse(output) use_homebrew_curl: !Utils::Curl.curl_supports_tls13?
)
JSON.parse(result.stdout)
rescue rescue
if Homebrew::EnvConfig.developer? if Homebrew::EnvConfig.developer?
$stderr.puts errors $stderr.puts result&.stderr
else else
odebug errors odebug result&.stderr
end end
raise raise
@ -32,14 +34,16 @@ module Repology
def self.single_package_query(name, repository:) def self.single_package_query(name, repository:)
url = "https://repology.org/api/v1/project/#{name}" url = "https://repology.org/api/v1/project/#{name}"
output, errors, = Utils::Curl.curl_output("--location", "--silent", url.to_s, result = Utils::Curl.curl_output(
use_homebrew_curl: !Utils::Curl.curl_supports_tls13?) "--location", "--silent", url.to_s,
use_homebrew_curl: !Utils::Curl.curl_supports_tls13?
)
data = JSON.parse(output) data = JSON.parse(result.stdout)
{ name => data } { name => data }
rescue => e rescue => e
require "utils/backtrace" require "utils/backtrace"
error_output = [errors, "#{e.class}: #{e}", Utils::Backtrace.clean(e)].compact error_output = [result&.stderr, "#{e.class}: #{e}", Utils::Backtrace.clean(e)].compact
if Homebrew::EnvConfig.developer? if Homebrew::EnvConfig.developer?
$stderr.puts(*error_output) $stderr.puts(*error_output)
else else

View File

@ -12,8 +12,8 @@ module SharedAudits
def self.eol_data(product, cycle) def self.eol_data(product, cycle)
@eol_data ||= T.let({}, T.nilable(T::Hash[String, T.untyped])) @eol_data ||= T.let({}, T.nilable(T::Hash[String, T.untyped]))
@eol_data["#{product}/#{cycle}"] ||= begin @eol_data["#{product}/#{cycle}"] ||= begin
out, _, status = Utils::Curl.curl_output("--location", "https://endoflife.date/api/#{product}/#{cycle}.json") result = Utils::Curl.curl_output("--location", "https://endoflife.date/api/#{product}/#{cycle}.json")
json = JSON.parse(out) if status.success? json = JSON.parse(result.stdout) if result.status.success?
json = nil if json&.dig("message")&.include?("Product not found") json = nil if json&.dig("message")&.include?("Product not found")
json json
end end
@ -75,8 +75,8 @@ module SharedAudits
def self.gitlab_repo_data(user, repo) def self.gitlab_repo_data(user, repo)
@gitlab_repo_data ||= T.let({}, T.nilable(T::Hash[String, T.untyped])) @gitlab_repo_data ||= T.let({}, T.nilable(T::Hash[String, T.untyped]))
@gitlab_repo_data["#{user}/#{repo}"] ||= begin @gitlab_repo_data["#{user}/#{repo}"] ||= begin
out, _, status = Utils::Curl.curl_output("https://gitlab.com/api/v4/projects/#{user}%2F#{repo}") result = Utils::Curl.curl_output("https://gitlab.com/api/v4/projects/#{user}%2F#{repo}")
json = JSON.parse(out) if status.success? json = JSON.parse(result.stdout) if result.status.success?
json = nil if json&.dig("message")&.include?("404 Project Not Found") json = nil if json&.dig("message")&.include?("404 Project Not Found")
json json
end end
@ -87,10 +87,10 @@ module SharedAudits
id = "#{user}/#{repo}/#{tag}" id = "#{user}/#{repo}/#{tag}"
@gitlab_release_data ||= T.let({}, T.nilable(T::Hash[String, T.untyped])) @gitlab_release_data ||= T.let({}, T.nilable(T::Hash[String, T.untyped]))
@gitlab_release_data[id] ||= begin @gitlab_release_data[id] ||= begin
out, _, status = Utils::Curl.curl_output( result = Utils::Curl.curl_output(
"https://gitlab.com/api/v4/projects/#{user}%2F#{repo}/releases/#{tag}", "--fail" "https://gitlab.com/api/v4/projects/#{user}%2F#{repo}/releases/#{tag}", "--fail"
) )
JSON.parse(out) if status.success? JSON.parse(result.stdout) if result.status.success?
end end
end end
@ -154,10 +154,10 @@ module SharedAudits
sig { params(user: String, repo: String).returns(T.nilable(String)) } sig { params(user: String, repo: String).returns(T.nilable(String)) }
def self.bitbucket(user, repo) def self.bitbucket(user, repo)
api_url = "https://api.bitbucket.org/2.0/repositories/#{user}/#{repo}" api_url = "https://api.bitbucket.org/2.0/repositories/#{user}/#{repo}"
out, _, status = Utils::Curl.curl_output("--request", "GET", api_url) result = Utils::Curl.curl_output("--request", "GET", api_url)
return unless status.success? return unless result.status.success?
metadata = JSON.parse(out) metadata = JSON.parse(result.stdout)
return if metadata.nil? return if metadata.nil?
return "Uses deprecated Mercurial support in Bitbucket" if metadata["scm"] == "hg" return "Uses deprecated Mercurial support in Bitbucket" if metadata["scm"] == "hg"
@ -166,16 +166,16 @@ module SharedAudits
return "Bitbucket repository too new (<30 days old)" if Date.parse(metadata["created_on"]) >= (Date.today - 30) return "Bitbucket repository too new (<30 days old)" if Date.parse(metadata["created_on"]) >= (Date.today - 30)
forks_out, _, forks_status = Utils::Curl.curl_output("--request", "GET", "#{api_url}/forks") forks_result = Utils::Curl.curl_output("--request", "GET", "#{api_url}/forks")
return unless forks_status.success? return unless forks_result.status.success?
watcher_out, _, watcher_status = Utils::Curl.curl_output("--request", "GET", "#{api_url}/watchers") watcher_result = Utils::Curl.curl_output("--request", "GET", "#{api_url}/watchers")
return unless watcher_status.success? return unless watcher_result.status.success?
forks_metadata = JSON.parse(forks_out) forks_metadata = JSON.parse(forks_result.stdout)
return if forks_metadata.nil? return if forks_metadata.nil?
watcher_metadata = JSON.parse(watcher_out) watcher_metadata = JSON.parse(watcher_result.stdout)
return if watcher_metadata.nil? return if watcher_metadata.nil?
return if forks_metadata["size"] >= 30 || watcher_metadata["size"] >= 75 return if forks_metadata["size"] >= 30 || watcher_metadata["size"] >= 75