From cf22382921e9181e3cb55646f2c34a1873f4282e Mon Sep 17 00:00:00 2001 From: Sam Ford <1584702+samford@users.noreply.github.com> Date: Fri, 10 Jan 2025 21:37:20 -0500 Subject: [PATCH] Curl: use `typed: strict` This upgrades `utils/curl.rb` to `typed: strict`, which requires a number of changes to pass `brew typecheck`. The most straightforward are adding type signatures to methods, adding type annotations (e.g., `T.let`) to variables that need them, and ensuring that methods always use the expected return type. I had to refactor areas where we call a `Utils::Curl` method and use array destructuring on a `SystemCommand::Result` return value (e.g., `output, errors, status = curl_output(...)`), as Sorbet doesn't understand implicit array conversion. As suggested by Markus, I've switched these areas to use `#stdout`, `#stderr`, and `#status`. This requires the use of an intermediate variable (`result`) in some cases but this was a fairly straightforward substitution. I also had to refactor how `Cask::URL::BlockDSL::PageWithURL` works. It currently uses `page.extend PageWithURL` to add a `url` attribute but this reworks it to subclass `SimpleDelegator` and use an `initialize` method instead. This achieves the same goal but in a way that Sorbet can understand. --- Library/Homebrew/cask/cask_loader.rb | 2 +- Library/Homebrew/cask/url.rb | 18 +-- Library/Homebrew/formulary.rb | 2 +- Library/Homebrew/github_packages.rb | 2 +- .../test/download_strategies/curl_spec.rb | 2 +- Library/Homebrew/utils/analytics.rb | 6 +- Library/Homebrew/utils/curl.rb | 105 +++++++++++++++--- Library/Homebrew/utils/github.rb | 12 +- Library/Homebrew/utils/github/api.rb | 8 +- Library/Homebrew/utils/pypi.rb | 6 +- Library/Homebrew/utils/repology.rb | 22 ++-- Library/Homebrew/utils/shared_audits.rb | 30 ++--- 12 files changed, 149 insertions(+), 66 deletions(-) diff --git a/Library/Homebrew/cask/cask_loader.rb b/Library/Homebrew/cask/cask_loader.rb index a91882a6e5..bdf4bded72 100644 --- a/Library/Homebrew/cask/cask_loader.rb +++ b/Library/Homebrew/cask/cask_loader.rb @@ -204,7 +204,7 @@ module Cask begin ohai "Downloading #{url}" - ::Utils::Curl.curl_download url, to: path + ::Utils::Curl.curl_download url.to_s, to: path rescue ErrorDuringExecution raise CaskUnavailableError.new(token, "Failed to download #{Formatter.url(url)}.") end diff --git a/Library/Homebrew/cask/url.rb b/Library/Homebrew/cask/url.rb index 5bcee1de9f..bec7cf6f90 100644 --- a/Library/Homebrew/cask/url.rb +++ b/Library/Homebrew/cask/url.rb @@ -99,21 +99,26 @@ module Cask class BlockDSL # Allow accessing the URL associated with page contents. - module PageWithURL + class PageWithURL < SimpleDelegator # Get the URL of the fetched page. # # ### Example # # ```ruby # url "https://example.org/download" do |page| - # file = page[/href="([^"]+.dmg)"/, 1] - # URL.join(page.url, file) + # file_path = page[/href="([^"]+\.dmg)"/, 1] + # URI.join(page.url, file_path) # end # ``` # # @api public sig { returns(URI::Generic) } attr_accessor :url + + def initialize(str, url) + super(str) + @url = url + end end sig { @@ -135,13 +140,10 @@ module Cask sig { returns(T.any(T.any(URI::Generic, String), [T.any(URI::Generic, String), Hash])) } def call if @uri - result = ::Utils::Curl.curl_output("--fail", "--silent", "--location", @uri) + result = ::Utils::Curl.curl_output("--fail", "--silent", "--location", @uri.to_s) result.assert_success! - page = result.stdout - page.extend PageWithURL - page.url = URI(@uri) - + page = PageWithURL.new(result.stdout, URI(@uri)) instance_exec(page, &@block) else instance_exec(&@block) diff --git a/Library/Homebrew/formulary.rb b/Library/Homebrew/formulary.rb index 823a51f4dd..0529671e9a 100644 --- a/Library/Homebrew/formulary.rb +++ b/Library/Homebrew/formulary.rb @@ -723,7 +723,7 @@ module Formulary end HOMEBREW_CACHE_FORMULA.mkpath FileUtils.rm_f(path) - Utils::Curl.curl_download url, to: path + Utils::Curl.curl_download url.to_s, to: path super rescue MethodDeprecatedError => e if (match_data = url.match(%r{github.com/(?[\w-]+)/(?[\w-]+)/}).presence) diff --git a/Library/Homebrew/github_packages.rb b/Library/Homebrew/github_packages.rb index 68ad1649bf..43f2556292 100644 --- a/Library/Homebrew/github_packages.rb +++ b/Library/Homebrew/github_packages.rb @@ -169,7 +169,7 @@ class GitHubPackages # Going forward, this should probably be pinned to tags. # We currently use features newer than the last one (v1.0.2). url = "https://raw.githubusercontent.com/opencontainers/image-spec/170393e57ed656f7f81c3070bfa8c3346eaa0a5a/schema/#{basename}.json" - out, = Utils::Curl.curl_output(url) + out = Utils::Curl.curl_output(url).stdout json = JSON.parse(out) @schema_json ||= {} diff --git a/Library/Homebrew/test/download_strategies/curl_spec.rb b/Library/Homebrew/test/download_strategies/curl_spec.rb index a71caf5ef1..a7c5deee52 100644 --- a/Library/Homebrew/test/download_strategies/curl_spec.rb +++ b/Library/Homebrew/test/download_strategies/curl_spec.rb @@ -37,7 +37,7 @@ RSpec.describe CurlDownloadStrategy do it "calls curl with default arguments" do expect(strategy).to receive(:curl).with( "--remote-time", - "--output", an_instance_of(Pathname), + "--output", an_instance_of(String), # example.com supports partial requests. "--continue-at", "-", "--location", diff --git a/Library/Homebrew/utils/analytics.rb b/Library/Homebrew/utils/analytics.rb index cbf79f1f67..f7fb2e6564 100644 --- a/Library/Homebrew/utils/analytics.rb +++ b/Library/Homebrew/utils/analytics.rb @@ -289,7 +289,7 @@ module Utils formula_version_urls = output.stdout .scan(%r{/orgs/Homebrew/packages/#{formula_url_suffix}\d+\?tag=[^"]+}) .map do |url| - url.sub("/orgs/Homebrew/packages/", "/Homebrew/homebrew-core/pkgs/") + T.cast(url, String).sub("/orgs/Homebrew/packages/", "/Homebrew/homebrew-core/pkgs/") end return if formula_version_urls.empty? @@ -304,9 +304,9 @@ module Utils ) next if last_thirty_days_match.blank? - last_thirty_days_downloads = last_thirty_days_match.captures.first.tr(",", "") + last_thirty_days_downloads = T.must(last_thirty_days_match.captures.first).tr(",", "") thirty_day_download_count += if (millions_match = last_thirty_days_downloads.match(/(\d+\.\d+)M/).presence) - millions_match.captures.first.to_f * 1_000_000 + (millions_match.captures.first.to_f * 1_000_000).to_i else last_thirty_days_downloads.to_i end diff --git a/Library/Homebrew/utils/curl.rb b/Library/Homebrew/utils/curl.rb index 39b0a8fc72..1b07afa72d 100644 --- a/Library/Homebrew/utils/curl.rb +++ b/Library/Homebrew/utils/curl.rb @@ -1,4 +1,4 @@ -# typed: true # rubocop:todo Sorbet/StrictSigil +# typed: strict # frozen_string_literal: true require "open3" @@ -48,23 +48,29 @@ module Utils module_function + sig { params(use_homebrew_curl: T::Boolean).returns(T.any(Pathname, String)) } def curl_executable(use_homebrew_curl: false) return HOMEBREW_BREWED_CURL_PATH if use_homebrew_curl - @curl_executable ||= HOMEBREW_SHIMS_PATH/"shared/curl" + @curl_executable ||= T.let(HOMEBREW_SHIMS_PATH/"shared/curl", T.nilable(T.any(Pathname, String))) end + sig { returns(String) } def curl_path - @curl_path ||= Utils.popen_read(curl_executable, "--homebrew=print-path").chomp.presence + @curl_path ||= T.let( + Utils.popen_read(curl_executable, "--homebrew=print-path").chomp.presence, + T.nilable(String), + ) end + sig { void } def clear_path_cache @curl_path = nil end sig { params( - extra_args: T.untyped, + extra_args: String, connect_timeout: T.any(Integer, Float, NilClass), max_time: T.any(Integer, Float, NilClass), retries: T.nilable(Integer), @@ -140,9 +146,23 @@ module Utils (args + extra_args).map(&:to_s) end + sig { + params( + args: String, + secrets: T.any(String, T::Array[String]), + print_stdout: T.any(T::Boolean, Symbol), + print_stderr: T.any(T::Boolean, Symbol), + debug: T.nilable(T::Boolean), + verbose: T.nilable(T::Boolean), + env: T::Hash[String, String], + timeout: T.nilable(T.any(Integer, Float)), + use_homebrew_curl: T::Boolean, + options: T.untyped, + ).returns(SystemCommand::Result) + } def curl_with_workarounds( *args, - secrets: nil, print_stdout: nil, print_stderr: nil, debug: nil, + secrets: [], print_stdout: false, print_stderr: false, debug: nil, verbose: nil, env: {}, timeout: nil, use_homebrew_curl: false, **options ) end_time = Time.now + timeout if timeout @@ -190,13 +210,28 @@ module Utils result end + sig { + params( + args: String, + print_stdout: T.any(T::Boolean, Symbol), + options: T.untyped, + ).returns(SystemCommand::Result) + } def curl(*args, print_stdout: true, **options) result = curl_with_workarounds(*args, print_stdout:, **options) result.assert_success! result end - def curl_download(*args, to: nil, try_partial: false, **options) + sig { + params( + args: String, + to: T.any(Pathname, String), + try_partial: T::Boolean, + options: T.untyped, + ).returns(T.nilable(SystemCommand::Result)) + } + def curl_download(*args, to:, try_partial: false, **options) destination = Pathname(to) destination.dirname.mkpath @@ -224,15 +259,23 @@ module Utils end end - args = ["--remote-time", "--output", destination, *args] + args = ["--remote-time", "--output", destination.to_s, *args] curl(*args, **options) end + sig { params(args: String, options: T.untyped).returns(SystemCommand::Result) } def curl_output(*args, **options) curl_with_workarounds(*args, print_stderr: false, show_output: true, **options) end + sig { + params( + args: String, + wanted_headers: T::Array[String], + options: T.untyped, + ).returns(T::Hash[Symbol, T.untyped]) + } def curl_headers(*args, wanted_headers: [], **options) get_retry_args = ["--request", "GET"] # This is a workaround for https://github.com/Homebrew/brew/issues/18213 @@ -268,6 +311,8 @@ module Utils result.assert_success! end + + {} end # Check if a URL is protected by CloudFlare (e.g. badlion.net and jaxx.io). @@ -295,6 +340,18 @@ module Utils set_cookie_header.compact.any? { |cookie| cookie.match?(/^(visid_incap|incap_ses)_/i) } end + sig { + params( + url: String, + url_type: String, + specs: T::Hash[Symbol, String], + user_agents: T::Array[Symbol], + referer: T.nilable(String), + check_content: T::Boolean, + strict: T::Boolean, + use_homebrew_curl: T::Boolean, + ).returns(T.nilable(String)) + } def curl_check_http_content(url, url_type, specs: {}, user_agents: [:default], referer: nil, check_content: false, strict: false, use_homebrew_curl: false) return unless url.start_with? "http" @@ -325,7 +382,7 @@ module Utils end end - details = T.let(nil, T.nilable(T::Hash[Symbol, T.untyped])) + details = T.let({}, T::Hash[Symbol, T.untyped]) attempts = 0 user_agents.each do |user_agent| loop do @@ -373,7 +430,9 @@ module Utils return "The #{url_type} #{url} is not reachable (HTTP status code #{details[:status_code]})" end - "Unable to find homepage" if SharedAudits.github_repo_data(repo_details[:user], repo_details[:repo]).nil? + if SharedAudits.github_repo_data(T.must(repo_details[:user]), T.must(repo_details[:repo])).nil? + "Unable to find homepage" + end end if url.start_with?("https://") && Homebrew::EnvConfig.no_insecure_redirect? && @@ -425,6 +484,16 @@ module Utils "The #{url_type} #{url} may be able to use HTTPS rather than HTTP. Please verify it in a browser." end + sig { + params( + url: String, + specs: T::Hash[Symbol, String], + hash_needed: T::Boolean, + use_homebrew_curl: T::Boolean, + user_agent: Symbol, + referer: T.nilable(String), + ).returns(T::Hash[Symbol, T.untyped]) + } def curl_http_content_headers_and_checksum( url, specs: {}, hash_needed: false, use_homebrew_curl: false, user_agent: :default, referer: nil @@ -504,26 +573,32 @@ module Utils T.must(file).unlink end + sig { returns(Version) } def curl_version - @curl_version ||= {} - @curl_version[curl_path] ||= Version.new(curl_output("-V").stdout[/curl (\d+(\.\d+)+)/, 1]) + @curl_version ||= T.let({}, T.nilable(T::Hash[String, Version])) + @curl_version[curl_path] ||= Version.new(T.must(curl_output("-V").stdout[/curl (\d+(\.\d+)+)/, 1])) end + sig { returns(T::Boolean) } def curl_supports_fail_with_body? - @curl_supports_fail_with_body ||= Hash.new do |h, key| + @curl_supports_fail_with_body ||= T.let(Hash.new do |h, key| h[key] = curl_version >= Version.new("7.76.0") - end + end, T.nilable(T::Hash[T.any(Pathname, String), T::Boolean])) @curl_supports_fail_with_body[curl_path] end + sig { returns(T::Boolean) } def curl_supports_tls13? - @curl_supports_tls13 ||= Hash.new do |h, key| + @curl_supports_tls13 ||= T.let(Hash.new do |h, key| h[key] = quiet_system(curl_executable, "--tlsv1.3", "--head", "https://brew.sh/") - end + end, T.nilable(T::Hash[T.any(Pathname, String), T::Boolean])) @curl_supports_tls13[curl_path] end + sig { params(status: T.nilable(String)).returns(T::Boolean) } def http_status_ok?(status) + return false if status.nil? + (100..299).cover?(status.to_i) end diff --git a/Library/Homebrew/utils/github.rb b/Library/Homebrew/utils/github.rb index fc5f934bbe..156c76b445 100644 --- a/Library/Homebrew/utils/github.rb +++ b/Library/Homebrew/utils/github.rb @@ -828,15 +828,15 @@ module GitHub return if Homebrew::EnvConfig.no_github_api? require "utils/curl" - output, _, status = Utils::Curl.curl_output( + result = Utils::Curl.curl_output( "--silent", "--head", "--location", "--header", "Accept: application/vnd.github.sha", url_to("repos", user, repo, "commits", ref).to_s ) - return unless status.success? + return unless result.status.success? - commit = output[/^ETag: "(\h+)"/, 1] + commit = result.stdout[/^ETag: "(\h+)"/, 1] return if commit.blank? version.update_commit(commit) @@ -847,14 +847,14 @@ module GitHub return false if Homebrew::EnvConfig.no_github_api? require "utils/curl" - output, _, status = Utils::Curl.curl_output( + result = Utils::Curl.curl_output( "--silent", "--head", "--location", "--header", "Accept: application/vnd.github.sha", url_to("repos", user, repo, "commits", commit).to_s ) - return true unless status.success? - return true if output.blank? + return true unless result.status.success? + return true if (output = result.stdout).blank? output[/^Status: (200)/, 1] != "200" end diff --git a/Library/Homebrew/utils/github/api.rb b/Library/Homebrew/utils/github/api.rb index 26b4e3d0aa..f9a5219316 100644 --- a/Library/Homebrew/utils/github/api.rb +++ b/Library/Homebrew/utils/github/api.rb @@ -274,8 +274,8 @@ module GitHub args += ["--dump-header", T.must(headers_tmpfile.path)] require "utils/curl" - output, errors, status = Utils::Curl.curl_output("--location", url.to_s, *args, secrets: [token]) - output, _, http_code = output.rpartition("\n") + result = Utils::Curl.curl_output("--location", url.to_s, *args, secrets: [token]) + output, _, http_code = result.stdout.rpartition("\n") output, _, http_code = output.rpartition("\n") if http_code == "000" headers = headers_tmpfile.read ensure @@ -288,7 +288,9 @@ module GitHub end begin - raise_error(output, errors, http_code, headers, scopes) if !http_code.start_with?("2") || !status.success? + if !http_code.start_with?("2") || !result.status.success? + raise_error(output, result.stderr, http_code, headers, scopes) + end return if http_code == "204" # No Content diff --git a/Library/Homebrew/utils/pypi.rb b/Library/Homebrew/utils/pypi.rb index 18487c31c7..6432d8b8c4 100644 --- a/Library/Homebrew/utils/pypi.rb +++ b/Library/Homebrew/utils/pypi.rb @@ -65,12 +65,12 @@ module PyPI else "https://pypi.org/pypi/#{name}/json" end - out, _, status = Utils::Curl.curl_output metadata_url, "--location", "--fail" + result = Utils::Curl.curl_output(metadata_url, "--location", "--fail") - return unless status.success? + return unless result.status.success? begin - json = JSON.parse out + json = JSON.parse(result.stdout) rescue JSON::ParserError return end diff --git a/Library/Homebrew/utils/repology.rb b/Library/Homebrew/utils/repology.rb index c2681327d2..54bbcb1a6f 100644 --- a/Library/Homebrew/utils/repology.rb +++ b/Library/Homebrew/utils/repology.rb @@ -15,14 +15,16 @@ module Repology last_package_in_response += "/" if last_package_in_response.present? url = "https://repology.org/api/v1/projects/#{last_package_in_response}?inrepo=#{repository}&outdated=1" - output, errors, = Utils::Curl.curl_output(url.to_s, "--silent", - use_homebrew_curl: !Utils::Curl.curl_supports_tls13?) - JSON.parse(output) + result = Utils::Curl.curl_output( + "--silent", url.to_s, + use_homebrew_curl: !Utils::Curl.curl_supports_tls13? + ) + JSON.parse(result.stdout) rescue if Homebrew::EnvConfig.developer? - $stderr.puts errors + $stderr.puts result&.stderr else - odebug errors + odebug result&.stderr end raise @@ -32,14 +34,16 @@ module Repology def self.single_package_query(name, repository:) url = "https://repology.org/api/v1/project/#{name}" - output, errors, = Utils::Curl.curl_output("--location", "--silent", url.to_s, - use_homebrew_curl: !Utils::Curl.curl_supports_tls13?) + result = Utils::Curl.curl_output( + "--location", "--silent", url.to_s, + use_homebrew_curl: !Utils::Curl.curl_supports_tls13? + ) - data = JSON.parse(output) + data = JSON.parse(result.stdout) { name => data } rescue => e require "utils/backtrace" - error_output = [errors, "#{e.class}: #{e}", Utils::Backtrace.clean(e)].compact + error_output = [result&.stderr, "#{e.class}: #{e}", Utils::Backtrace.clean(e)].compact if Homebrew::EnvConfig.developer? $stderr.puts(*error_output) else diff --git a/Library/Homebrew/utils/shared_audits.rb b/Library/Homebrew/utils/shared_audits.rb index 2ef6d5a016..9ba8c9f9ac 100644 --- a/Library/Homebrew/utils/shared_audits.rb +++ b/Library/Homebrew/utils/shared_audits.rb @@ -12,8 +12,8 @@ module SharedAudits def self.eol_data(product, cycle) @eol_data ||= T.let({}, T.nilable(T::Hash[String, T.untyped])) @eol_data["#{product}/#{cycle}"] ||= begin - out, _, status = Utils::Curl.curl_output("--location", "https://endoflife.date/api/#{product}/#{cycle}.json") - json = JSON.parse(out) if status.success? + result = Utils::Curl.curl_output("--location", "https://endoflife.date/api/#{product}/#{cycle}.json") + json = JSON.parse(result.stdout) if result.status.success? json = nil if json&.dig("message")&.include?("Product not found") json end @@ -75,8 +75,8 @@ module SharedAudits def self.gitlab_repo_data(user, repo) @gitlab_repo_data ||= T.let({}, T.nilable(T::Hash[String, T.untyped])) @gitlab_repo_data["#{user}/#{repo}"] ||= begin - out, _, status = Utils::Curl.curl_output("https://gitlab.com/api/v4/projects/#{user}%2F#{repo}") - json = JSON.parse(out) if status.success? + result = Utils::Curl.curl_output("https://gitlab.com/api/v4/projects/#{user}%2F#{repo}") + json = JSON.parse(result.stdout) if result.status.success? json = nil if json&.dig("message")&.include?("404 Project Not Found") json end @@ -87,10 +87,10 @@ module SharedAudits id = "#{user}/#{repo}/#{tag}" @gitlab_release_data ||= T.let({}, T.nilable(T::Hash[String, T.untyped])) @gitlab_release_data[id] ||= begin - out, _, status = Utils::Curl.curl_output( + result = Utils::Curl.curl_output( "https://gitlab.com/api/v4/projects/#{user}%2F#{repo}/releases/#{tag}", "--fail" ) - JSON.parse(out) if status.success? + JSON.parse(result.stdout) if result.status.success? end end @@ -154,10 +154,10 @@ module SharedAudits sig { params(user: String, repo: String).returns(T.nilable(String)) } def self.bitbucket(user, repo) api_url = "https://api.bitbucket.org/2.0/repositories/#{user}/#{repo}" - out, _, status = Utils::Curl.curl_output("--request", "GET", api_url) - return unless status.success? + result = Utils::Curl.curl_output("--request", "GET", api_url) + return unless result.status.success? - metadata = JSON.parse(out) + metadata = JSON.parse(result.stdout) return if metadata.nil? return "Uses deprecated Mercurial support in Bitbucket" if metadata["scm"] == "hg" @@ -166,16 +166,16 @@ module SharedAudits return "Bitbucket repository too new (<30 days old)" if Date.parse(metadata["created_on"]) >= (Date.today - 30) - forks_out, _, forks_status = Utils::Curl.curl_output("--request", "GET", "#{api_url}/forks") - return unless forks_status.success? + forks_result = Utils::Curl.curl_output("--request", "GET", "#{api_url}/forks") + return unless forks_result.status.success? - watcher_out, _, watcher_status = Utils::Curl.curl_output("--request", "GET", "#{api_url}/watchers") - return unless watcher_status.success? + watcher_result = Utils::Curl.curl_output("--request", "GET", "#{api_url}/watchers") + return unless watcher_result.status.success? - forks_metadata = JSON.parse(forks_out) + forks_metadata = JSON.parse(forks_result.stdout) return if forks_metadata.nil? - watcher_metadata = JSON.parse(watcher_out) + watcher_metadata = JSON.parse(watcher_result.stdout) return if watcher_metadata.nil? return if forks_metadata["size"] >= 30 || watcher_metadata["size"] >= 75