From 956b71eeed0553de2f225b3fa23816212213a1f9 Mon Sep 17 00:00:00 2001 From: Adrian Ho Date: Thu, 6 Feb 2025 21:20:55 +0800 Subject: [PATCH] update-python-resources: add option to ignore errors This is particularly useful for third-party Python formulae that have a ton of resources, not all of which may adhere to homebrew/core's strict policies. See #19240 for context. I've also added logic that ignores `--ignore-errors` on `homebrew/core`, although I personally think this new behavior is also useful for mainline formula creation. Before: error out on a single non-conforming resource, zero resource blocks added to formula, scary stacktrace. After: all conforming resources added, all non-conforming resources identified in comments, error message at end, `brew` exits non-zero without scary stacktrace:- ``` % brew update-python-resources --ignore-errors gromgit/test/auto-coder || echo OOPS ==> Retrieving PyPI dependencies for "auto-coder==0.1.243"... ==> Retrieving PyPI dependencies for excluded ""... ==> Getting PyPI info for "aiohappyeyeballs==2.4.4" [200+ resource lines elided] ==> Getting PyPI info for "zhipuai==2.1.5.20250106" ==> Updating resource blocks Error: Unable to resolve some dependencies. Please check /opt/homebrew/Library/Taps/gromgit/homebrew-test/Formula/auto-coder.rb for RESOURCE-ERROR comments. OOPS % brew cat gromgit/test/auto-coder | ggrep -C10 RESOURCE-ERROR license "Apache-2.0" depends_on "python@3.11" # Additional dependency # resource "" do # url "" # sha256 "" # end # RESOURCE-ERROR: Unable to resolve "azure-cognitiveservices-speech==1.42.0" (no suitable source distribution on PyPI) # RESOURCE-ERROR: Unable to resolve "ray==2.42.0" (no suitable source distribution on PyPI) resource "aiohappyeyeballs" do url "https://files.pythonhosted.org/packages/7f/55/e4373e888fdacb15563ef6fa9fa8c8252476ea071e96fb46defac9f18bf2/aiohappyeyeballs-2.4.4.tar.gz" sha256 "5fdd7d87889c63183afc18ce9271f9b0a7d32c2303e394468dd45d514a757745" end resource "aiohttp" do url "https://files.pythonhosted.org/packages/37/4b/952d49c73084fb790cb5c6ead50848c8e96b4980ad806cf4d2ad341eaa03/aiohttp-3.11.12.tar.gz" sha256 "7603ca26d75b1b86160ce1bbe2787a0b706e592af5b2504e12caa88a217767b0" end ``` --- .../dev-cmd/update-python-resources.rb | 9 +++ .../dev_cmd/update_python_resources.rbi | 3 + .../sorbet/tapioca/compilers/args_spec.rb | 2 +- Library/Homebrew/utils/pypi.rb | 67 +++++++++++++------ 4 files changed, 61 insertions(+), 20 deletions(-) diff --git a/Library/Homebrew/dev-cmd/update-python-resources.rb b/Library/Homebrew/dev-cmd/update-python-resources.rb index 865103698f..c80cf01c20 100644 --- a/Library/Homebrew/dev-cmd/update-python-resources.rb +++ b/Library/Homebrew/dev-cmd/update-python-resources.rb @@ -15,6 +15,9 @@ module Homebrew description: "Print the updated resource blocks instead of changing ." switch "-s", "--silent", description: "Suppress any output." + switch "--ignore-errors", + description: "Record all discovered resources, even those that can't be resolved successfully. " \ + "This option is ignored for homebrew/core formulae." switch "--ignore-non-pypi-packages", description: "Don't fail if is not a PyPI package." switch "--install-dependencies", @@ -36,6 +39,11 @@ module Homebrew sig { override.void } def run args.named.to_formulae.each do |formula| + ignore_errors = if T.must(formula.tap).name == "homebrew/core" + false + else + args.ignore_errors? + end PyPI.update_python_resources! formula, version: args.version, package_name: args.package_name, @@ -45,6 +53,7 @@ module Homebrew print_only: args.print_only?, silent: args.silent?, verbose: args.verbose?, + ignore_errors: ignore_errors, ignore_non_pypi_packages: args.ignore_non_pypi_packages? end end diff --git a/Library/Homebrew/sorbet/rbi/dsl/homebrew/dev_cmd/update_python_resources.rbi b/Library/Homebrew/sorbet/rbi/dsl/homebrew/dev_cmd/update_python_resources.rbi index f5adb35b4f..a6f96b01a0 100644 --- a/Library/Homebrew/sorbet/rbi/dsl/homebrew/dev_cmd/update_python_resources.rbi +++ b/Library/Homebrew/sorbet/rbi/dsl/homebrew/dev_cmd/update_python_resources.rbi @@ -17,6 +17,9 @@ class Homebrew::DevCmd::UpdatePythonResources::Args < Homebrew::CLI::Args sig { returns(T.nilable(T::Array[String])) } def extra_packages; end + sig { returns(T::Boolean) } + def ignore_errors?; end + sig { returns(T::Boolean) } def ignore_non_pypi_packages?; end diff --git a/Library/Homebrew/test/sorbet/tapioca/compilers/args_spec.rb b/Library/Homebrew/test/sorbet/tapioca/compilers/args_spec.rb index 7761721b29..9f50418353 100644 --- a/Library/Homebrew/test/sorbet/tapioca/compilers/args_spec.rb +++ b/Library/Homebrew/test/sorbet/tapioca/compilers/args_spec.rb @@ -27,7 +27,7 @@ RSpec.describe Tapioca::Compilers::Args do it "returns a mapping of update-python-resources args to default values" do expect(compiler.args_table(update_python_resources_parser)).to contain_exactly( - :d?, :debug?, :exclude_packages, :extra_packages, :h?, :help?, :ignore_non_pypi_packages?, + :d?, :debug?, :exclude_packages, :extra_packages, :h?, :help?, :ignore_errors?, :ignore_non_pypi_packages?, :install_dependencies?, :p?, :package_name, :print_only?, :q?, :quiet?, :s?, :silent?, :v?, :verbose?, :version ) diff --git a/Library/Homebrew/utils/pypi.rb b/Library/Homebrew/utils/pypi.rb index d29582c9ad..41bdaa4f2c 100644 --- a/Library/Homebrew/utils/pypi.rb +++ b/Library/Homebrew/utils/pypi.rb @@ -54,8 +54,11 @@ module PyPI # Get name, URL, SHA-256 checksum and latest version for a given package. # This only works for packages from PyPI or from a PyPI URL; packages # derived from non-PyPI URLs will produce `nil` here. - sig { params(new_version: T.nilable(T.any(String, Version))).returns(T.nilable(T::Array[String])) } - def pypi_info(new_version: nil) + sig { + params(new_version: T.nilable(T.any(String, Version)), + ignore_errors: T.nilable(T::Boolean)).returns(T.nilable(T::Array[String])) + } + def pypi_info(new_version: nil, ignore_errors: false) return unless valid_pypi_package? return @pypi_info if @pypi_info.present? && new_version.blank? @@ -87,6 +90,8 @@ module PyPI end if dist.nil? + return ["", "", "", "", "no suitable source distribution on PyPI"] if ignore_errors + onoe "#{name} exists on PyPI but lacks a suitable source distribution" return end @@ -218,13 +223,14 @@ module PyPI print_only: T.nilable(T::Boolean), silent: T.nilable(T::Boolean), verbose: T.nilable(T::Boolean), + ignore_errors: T.nilable(T::Boolean), ignore_non_pypi_packages: T.nilable(T::Boolean), ).returns(T.nilable(T::Boolean)) } def self.update_python_resources!(formula, version: nil, package_name: nil, extra_packages: nil, exclude_packages: nil, dependencies: nil, install_dependencies: false, print_only: false, silent: false, verbose: false, - ignore_non_pypi_packages: false) + ignore_errors: false, ignore_non_pypi_packages: false) auto_update_list = formula.tap&.pypi_formula_mappings if auto_update_list.present? && auto_update_list.key?(formula.full_name) && package_name.blank? && extra_packages.blank? && exclude_packages.blank? @@ -350,6 +356,7 @@ module PyPI end new_resource_blocks = "" + package_errors = "" found_packages.sort.each do |package| if exclude_packages.include? package ohai "Excluding \"#{package}\"" if show_info @@ -358,31 +365,48 @@ module PyPI end ohai "Getting PyPI info for \"#{package}\"" if show_info - name, url, checksum = package.pypi_info - # Fail if unable to find name, url or checksum for any resource - if name.blank? - odie "Unable to resolve some dependencies. Please update the resources for \"#{formula.name}\" manually." - elsif url.blank? || checksum.blank? - odie <<~EOS - Unable to find the URL and/or sha256 for the "#{name}" resource. - Please update the resources for "#{formula.name}" manually. - EOS + name, url, checksum, _, package_error = package.pypi_info(ignore_errors: ignore_errors) + if package_error.blank? + # Fail if unable to find name, url or checksum for any resource + if name.blank? + if ignore_errors + package_error = "unknown failure" + else + odie "Unable to resolve some dependencies. Please update the resources for \"#{formula.name}\" manually." + end + elsif url.blank? || checksum.blank? + if ignore_errors + package_error = "unable to find URL and/or sha256" + else + odie <<~EOS + Unable to find the URL and/or sha256 for the "#{name}" resource. + Please update the resources for "#{formula.name}" manually. + EOS + end + end end - # Append indented resource block - new_resource_blocks += <<-EOS + if package_error.blank? + # Append indented resource block + new_resource_blocks += <<-EOS resource "#{name}" do url "#{url}" sha256 "#{checksum}" end - EOS + EOS + else + # Leave a placeholder for formula author to investigate + package_errors += " # RESOURCE-ERROR: Unable to resolve \"#{package}\" (#{package_error})\n" + end end + resource_section = "#{package_errors}\n#{new_resource_blocks}" + odie "Excluded superfluous packages: #{exclude_packages.join(", ")}" if exclude_packages.any? if print_only - puts new_resource_blocks.chomp + puts resource_section.chomp return end @@ -390,11 +414,12 @@ module PyPI if formula.resources.all? { |resource| resource.name.start_with?("homebrew-") } # Place resources above install method inreplace_regex = / def install/ - new_resource_blocks += " def install" + resource_section += " def install" else # Replace existing resource blocks with new resource blocks inreplace_regex = / \ \ ( + (\#\ RESOURCE-ERROR:\ .*\s+)* resource\ .*\ do\s+ url\ .*\s+ sha256\ .*\s+ @@ -405,7 +430,7 @@ module PyPI end\s+)* end\s+)+ /x - new_resource_blocks += " " + resource_section += " " end ohai "Updating resource blocks" unless silent @@ -413,7 +438,11 @@ module PyPI if T.must(s.inreplace_string.split(/^ test do\b/, 2).first).scan(inreplace_regex).length > 1 odie "Unable to update resource blocks for \"#{formula.name}\" automatically. Please update them manually." end - s.sub! inreplace_regex, new_resource_blocks + s.sub! inreplace_regex, resource_section + end + + if package_errors.present? + ofail "Unable to resolve some dependencies. Please check #{formula.path} for RESOURCE-ERROR comments." end true