update-python-resources: add option to ignore errors

This is particularly useful for third-party Python formulae that have a ton of resources, not all of which may adhere to homebrew/core's strict policies. See #19240 for context.

I've also added logic that ignores `--ignore-errors` on `homebrew/core`, although I personally think this new behavior is also useful for mainline formula creation.

Before: error out on a single non-conforming resource, zero resource blocks added to formula, scary stacktrace.

After: all conforming resources added, all non-conforming resources identified in comments, error message at end, `brew` exits non-zero without scary stacktrace:-
```
% brew update-python-resources --ignore-errors gromgit/test/auto-coder || echo OOPS
==> Retrieving PyPI dependencies for "auto-coder==0.1.243"...
==> Retrieving PyPI dependencies for excluded ""...
==> Getting PyPI info for "aiohappyeyeballs==2.4.4"
[200+ resource lines elided]
==> Getting PyPI info for "zhipuai==2.1.5.20250106"
==> Updating resource blocks
Error: Unable to resolve some dependencies. Please check /opt/homebrew/Library/Taps/gromgit/homebrew-test/Formula/auto-coder.rb for RESOURCE-ERROR comments.
OOPS

% brew cat gromgit/test/auto-coder | ggrep -C10 RESOURCE-ERROR
  license "Apache-2.0"

  depends_on "python@3.11"

  # Additional dependency
  # resource "" do
  #   url ""
  #   sha256 ""
  # end

  # RESOURCE-ERROR: Unable to resolve "azure-cognitiveservices-speech==1.42.0" (no suitable source distribution on PyPI)
  # RESOURCE-ERROR: Unable to resolve "ray==2.42.0" (no suitable source distribution on PyPI)

  resource "aiohappyeyeballs" do
    url "e4373e888f/aiohappyeyeballs-2.4.4.tar.gz"
    sha256 "5fdd7d87889c63183afc18ce9271f9b0a7d32c2303e394468dd45d514a757745"
  end

  resource "aiohttp" do
    url "952d49c730/aiohttp-3.11.12.tar.gz"
    sha256 "7603ca26d75b1b86160ce1bbe2787a0b706e592af5b2504e12caa88a217767b0"
  end
```
This commit is contained in:
Adrian Ho 2025-02-06 21:20:55 +08:00
parent fd92510c71
commit 956b71eeed
4 changed files with 61 additions and 20 deletions

View File

@ -15,6 +15,9 @@ module Homebrew
description: "Print the updated resource blocks instead of changing <formula>." description: "Print the updated resource blocks instead of changing <formula>."
switch "-s", "--silent", switch "-s", "--silent",
description: "Suppress any output." description: "Suppress any output."
switch "--ignore-errors",
description: "Record all discovered resources, even those that can't be resolved successfully. " \
"This option is ignored for homebrew/core formulae."
switch "--ignore-non-pypi-packages", switch "--ignore-non-pypi-packages",
description: "Don't fail if <formula> is not a PyPI package." description: "Don't fail if <formula> is not a PyPI package."
switch "--install-dependencies", switch "--install-dependencies",
@ -36,6 +39,11 @@ module Homebrew
sig { override.void } sig { override.void }
def run def run
args.named.to_formulae.each do |formula| args.named.to_formulae.each do |formula|
ignore_errors = if T.must(formula.tap).name == "homebrew/core"
false
else
args.ignore_errors?
end
PyPI.update_python_resources! formula, PyPI.update_python_resources! formula,
version: args.version, version: args.version,
package_name: args.package_name, package_name: args.package_name,
@ -45,6 +53,7 @@ module Homebrew
print_only: args.print_only?, print_only: args.print_only?,
silent: args.silent?, silent: args.silent?,
verbose: args.verbose?, verbose: args.verbose?,
ignore_errors: ignore_errors,
ignore_non_pypi_packages: args.ignore_non_pypi_packages? ignore_non_pypi_packages: args.ignore_non_pypi_packages?
end end
end end

View File

@ -17,6 +17,9 @@ class Homebrew::DevCmd::UpdatePythonResources::Args < Homebrew::CLI::Args
sig { returns(T.nilable(T::Array[String])) } sig { returns(T.nilable(T::Array[String])) }
def extra_packages; end def extra_packages; end
sig { returns(T::Boolean) }
def ignore_errors?; end
sig { returns(T::Boolean) } sig { returns(T::Boolean) }
def ignore_non_pypi_packages?; end def ignore_non_pypi_packages?; end

View File

@ -27,7 +27,7 @@ RSpec.describe Tapioca::Compilers::Args do
it "returns a mapping of update-python-resources args to default values" do it "returns a mapping of update-python-resources args to default values" do
expect(compiler.args_table(update_python_resources_parser)).to contain_exactly( expect(compiler.args_table(update_python_resources_parser)).to contain_exactly(
:d?, :debug?, :exclude_packages, :extra_packages, :h?, :help?, :ignore_non_pypi_packages?, :d?, :debug?, :exclude_packages, :extra_packages, :h?, :help?, :ignore_errors?, :ignore_non_pypi_packages?,
:install_dependencies?, :p?, :package_name, :print_only?, :q?, :quiet?, :s?, :silent?, :v?, :verbose?, :install_dependencies?, :p?, :package_name, :print_only?, :q?, :quiet?, :s?, :silent?, :v?, :verbose?,
:version :version
) )

View File

@ -54,8 +54,11 @@ module PyPI
# Get name, URL, SHA-256 checksum and latest version for a given package. # Get name, URL, SHA-256 checksum and latest version for a given package.
# This only works for packages from PyPI or from a PyPI URL; packages # This only works for packages from PyPI or from a PyPI URL; packages
# derived from non-PyPI URLs will produce `nil` here. # derived from non-PyPI URLs will produce `nil` here.
sig { params(new_version: T.nilable(T.any(String, Version))).returns(T.nilable(T::Array[String])) } sig {
def pypi_info(new_version: nil) params(new_version: T.nilable(T.any(String, Version)),
ignore_errors: T.nilable(T::Boolean)).returns(T.nilable(T::Array[String]))
}
def pypi_info(new_version: nil, ignore_errors: false)
return unless valid_pypi_package? return unless valid_pypi_package?
return @pypi_info if @pypi_info.present? && new_version.blank? return @pypi_info if @pypi_info.present? && new_version.blank?
@ -87,6 +90,8 @@ module PyPI
end end
if dist.nil? if dist.nil?
return ["", "", "", "", "no suitable source distribution on PyPI"] if ignore_errors
onoe "#{name} exists on PyPI but lacks a suitable source distribution" onoe "#{name} exists on PyPI but lacks a suitable source distribution"
return return
end end
@ -218,13 +223,14 @@ module PyPI
print_only: T.nilable(T::Boolean), print_only: T.nilable(T::Boolean),
silent: T.nilable(T::Boolean), silent: T.nilable(T::Boolean),
verbose: T.nilable(T::Boolean), verbose: T.nilable(T::Boolean),
ignore_errors: T.nilable(T::Boolean),
ignore_non_pypi_packages: T.nilable(T::Boolean), ignore_non_pypi_packages: T.nilable(T::Boolean),
).returns(T.nilable(T::Boolean)) ).returns(T.nilable(T::Boolean))
} }
def self.update_python_resources!(formula, version: nil, package_name: nil, extra_packages: nil, def self.update_python_resources!(formula, version: nil, package_name: nil, extra_packages: nil,
exclude_packages: nil, dependencies: nil, install_dependencies: false, exclude_packages: nil, dependencies: nil, install_dependencies: false,
print_only: false, silent: false, verbose: false, print_only: false, silent: false, verbose: false,
ignore_non_pypi_packages: false) ignore_errors: false, ignore_non_pypi_packages: false)
auto_update_list = formula.tap&.pypi_formula_mappings auto_update_list = formula.tap&.pypi_formula_mappings
if auto_update_list.present? && auto_update_list.key?(formula.full_name) && if auto_update_list.present? && auto_update_list.key?(formula.full_name) &&
package_name.blank? && extra_packages.blank? && exclude_packages.blank? package_name.blank? && extra_packages.blank? && exclude_packages.blank?
@ -350,6 +356,7 @@ module PyPI
end end
new_resource_blocks = "" new_resource_blocks = ""
package_errors = ""
found_packages.sort.each do |package| found_packages.sort.each do |package|
if exclude_packages.include? package if exclude_packages.include? package
ohai "Excluding \"#{package}\"" if show_info ohai "Excluding \"#{package}\"" if show_info
@ -358,17 +365,28 @@ module PyPI
end end
ohai "Getting PyPI info for \"#{package}\"" if show_info ohai "Getting PyPI info for \"#{package}\"" if show_info
name, url, checksum = package.pypi_info name, url, checksum, _, package_error = package.pypi_info(ignore_errors: ignore_errors)
if package_error.blank?
# Fail if unable to find name, url or checksum for any resource # Fail if unable to find name, url or checksum for any resource
if name.blank? if name.blank?
if ignore_errors
package_error = "unknown failure"
else
odie "Unable to resolve some dependencies. Please update the resources for \"#{formula.name}\" manually." odie "Unable to resolve some dependencies. Please update the resources for \"#{formula.name}\" manually."
end
elsif url.blank? || checksum.blank? elsif url.blank? || checksum.blank?
if ignore_errors
package_error = "unable to find URL and/or sha256"
else
odie <<~EOS odie <<~EOS
Unable to find the URL and/or sha256 for the "#{name}" resource. Unable to find the URL and/or sha256 for the "#{name}" resource.
Please update the resources for "#{formula.name}" manually. Please update the resources for "#{formula.name}" manually.
EOS EOS
end end
end
end
if package_error.blank?
# Append indented resource block # Append indented resource block
new_resource_blocks += <<-EOS new_resource_blocks += <<-EOS
resource "#{name}" do resource "#{name}" do
@ -377,12 +395,18 @@ module PyPI
end end
EOS EOS
else
# Leave a placeholder for formula author to investigate
package_errors += " # RESOURCE-ERROR: Unable to resolve \"#{package}\" (#{package_error})\n"
end end
end
resource_section = "#{package_errors}\n#{new_resource_blocks}"
odie "Excluded superfluous packages: #{exclude_packages.join(", ")}" if exclude_packages.any? odie "Excluded superfluous packages: #{exclude_packages.join(", ")}" if exclude_packages.any?
if print_only if print_only
puts new_resource_blocks.chomp puts resource_section.chomp
return return
end end
@ -390,11 +414,12 @@ module PyPI
if formula.resources.all? { |resource| resource.name.start_with?("homebrew-") } if formula.resources.all? { |resource| resource.name.start_with?("homebrew-") }
# Place resources above install method # Place resources above install method
inreplace_regex = / def install/ inreplace_regex = / def install/
new_resource_blocks += " def install" resource_section += " def install"
else else
# Replace existing resource blocks with new resource blocks # Replace existing resource blocks with new resource blocks
inreplace_regex = / inreplace_regex = /
\ \ ( \ \ (
(\#\ RESOURCE-ERROR:\ .*\s+)*
resource\ .*\ do\s+ resource\ .*\ do\s+
url\ .*\s+ url\ .*\s+
sha256\ .*\s+ sha256\ .*\s+
@ -405,7 +430,7 @@ module PyPI
end\s+)* end\s+)*
end\s+)+ end\s+)+
/x /x
new_resource_blocks += " " resource_section += " "
end end
ohai "Updating resource blocks" unless silent ohai "Updating resource blocks" unless silent
@ -413,7 +438,11 @@ module PyPI
if T.must(s.inreplace_string.split(/^ test do\b/, 2).first).scan(inreplace_regex).length > 1 if T.must(s.inreplace_string.split(/^ test do\b/, 2).first).scan(inreplace_regex).length > 1
odie "Unable to update resource blocks for \"#{formula.name}\" automatically. Please update them manually." odie "Unable to update resource blocks for \"#{formula.name}\" automatically. Please update them manually."
end end
s.sub! inreplace_regex, new_resource_blocks s.sub! inreplace_regex, resource_section
end
if package_errors.present?
ofail "Unable to resolve some dependencies. Please check #{formula.path} for RESOURCE-ERROR comments."
end end
true true