update-python-resources: add option to ignore errors
This is particularly useful for third-party Python formulae that have a ton of resources, not all of which may adhere to homebrew/core's strict policies. See #19240 for context. I've also added logic that ignores `--ignore-errors` on `homebrew/core`, although I personally think this new behavior is also useful for mainline formula creation. Before: error out on a single non-conforming resource, zero resource blocks added to formula, scary stacktrace. After: all conforming resources added, all non-conforming resources identified in comments, error message at end, `brew` exits non-zero without scary stacktrace:- ``` % brew update-python-resources --ignore-errors gromgit/test/auto-coder || echo OOPS ==> Retrieving PyPI dependencies for "auto-coder==0.1.243"... ==> Retrieving PyPI dependencies for excluded ""... ==> Getting PyPI info for "aiohappyeyeballs==2.4.4" [200+ resource lines elided] ==> Getting PyPI info for "zhipuai==2.1.5.20250106" ==> Updating resource blocks Error: Unable to resolve some dependencies. Please check /opt/homebrew/Library/Taps/gromgit/homebrew-test/Formula/auto-coder.rb for RESOURCE-ERROR comments. OOPS % brew cat gromgit/test/auto-coder | ggrep -C10 RESOURCE-ERROR license "Apache-2.0" depends_on "python@3.11" # Additional dependency # resource "" do # url "" # sha256 "" # end # RESOURCE-ERROR: Unable to resolve "azure-cognitiveservices-speech==1.42.0" (no suitable source distribution on PyPI) # RESOURCE-ERROR: Unable to resolve "ray==2.42.0" (no suitable source distribution on PyPI) resource "aiohappyeyeballs" do url "e4373e888f/aiohappyeyeballs-2.4.4.tar.gz" sha256 "5fdd7d87889c63183afc18ce9271f9b0a7d32c2303e394468dd45d514a757745" end resource "aiohttp" do url "952d49c730/aiohttp-3.11.12.tar.gz" sha256 "7603ca26d75b1b86160ce1bbe2787a0b706e592af5b2504e12caa88a217767b0" end ```
This commit is contained in:
parent
fd92510c71
commit
956b71eeed
@ -15,6 +15,9 @@ module Homebrew
|
|||||||
description: "Print the updated resource blocks instead of changing <formula>."
|
description: "Print the updated resource blocks instead of changing <formula>."
|
||||||
switch "-s", "--silent",
|
switch "-s", "--silent",
|
||||||
description: "Suppress any output."
|
description: "Suppress any output."
|
||||||
|
switch "--ignore-errors",
|
||||||
|
description: "Record all discovered resources, even those that can't be resolved successfully. " \
|
||||||
|
"This option is ignored for homebrew/core formulae."
|
||||||
switch "--ignore-non-pypi-packages",
|
switch "--ignore-non-pypi-packages",
|
||||||
description: "Don't fail if <formula> is not a PyPI package."
|
description: "Don't fail if <formula> is not a PyPI package."
|
||||||
switch "--install-dependencies",
|
switch "--install-dependencies",
|
||||||
@ -36,6 +39,11 @@ module Homebrew
|
|||||||
sig { override.void }
|
sig { override.void }
|
||||||
def run
|
def run
|
||||||
args.named.to_formulae.each do |formula|
|
args.named.to_formulae.each do |formula|
|
||||||
|
ignore_errors = if T.must(formula.tap).name == "homebrew/core"
|
||||||
|
false
|
||||||
|
else
|
||||||
|
args.ignore_errors?
|
||||||
|
end
|
||||||
PyPI.update_python_resources! formula,
|
PyPI.update_python_resources! formula,
|
||||||
version: args.version,
|
version: args.version,
|
||||||
package_name: args.package_name,
|
package_name: args.package_name,
|
||||||
@ -45,6 +53,7 @@ module Homebrew
|
|||||||
print_only: args.print_only?,
|
print_only: args.print_only?,
|
||||||
silent: args.silent?,
|
silent: args.silent?,
|
||||||
verbose: args.verbose?,
|
verbose: args.verbose?,
|
||||||
|
ignore_errors: ignore_errors,
|
||||||
ignore_non_pypi_packages: args.ignore_non_pypi_packages?
|
ignore_non_pypi_packages: args.ignore_non_pypi_packages?
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|||||||
@ -17,6 +17,9 @@ class Homebrew::DevCmd::UpdatePythonResources::Args < Homebrew::CLI::Args
|
|||||||
sig { returns(T.nilable(T::Array[String])) }
|
sig { returns(T.nilable(T::Array[String])) }
|
||||||
def extra_packages; end
|
def extra_packages; end
|
||||||
|
|
||||||
|
sig { returns(T::Boolean) }
|
||||||
|
def ignore_errors?; end
|
||||||
|
|
||||||
sig { returns(T::Boolean) }
|
sig { returns(T::Boolean) }
|
||||||
def ignore_non_pypi_packages?; end
|
def ignore_non_pypi_packages?; end
|
||||||
|
|
||||||
|
|||||||
@ -27,7 +27,7 @@ RSpec.describe Tapioca::Compilers::Args do
|
|||||||
|
|
||||||
it "returns a mapping of update-python-resources args to default values" do
|
it "returns a mapping of update-python-resources args to default values" do
|
||||||
expect(compiler.args_table(update_python_resources_parser)).to contain_exactly(
|
expect(compiler.args_table(update_python_resources_parser)).to contain_exactly(
|
||||||
:d?, :debug?, :exclude_packages, :extra_packages, :h?, :help?, :ignore_non_pypi_packages?,
|
:d?, :debug?, :exclude_packages, :extra_packages, :h?, :help?, :ignore_errors?, :ignore_non_pypi_packages?,
|
||||||
:install_dependencies?, :p?, :package_name, :print_only?, :q?, :quiet?, :s?, :silent?, :v?, :verbose?,
|
:install_dependencies?, :p?, :package_name, :print_only?, :q?, :quiet?, :s?, :silent?, :v?, :verbose?,
|
||||||
:version
|
:version
|
||||||
)
|
)
|
||||||
|
|||||||
@ -54,8 +54,11 @@ module PyPI
|
|||||||
# Get name, URL, SHA-256 checksum and latest version for a given package.
|
# Get name, URL, SHA-256 checksum and latest version for a given package.
|
||||||
# This only works for packages from PyPI or from a PyPI URL; packages
|
# This only works for packages from PyPI or from a PyPI URL; packages
|
||||||
# derived from non-PyPI URLs will produce `nil` here.
|
# derived from non-PyPI URLs will produce `nil` here.
|
||||||
sig { params(new_version: T.nilable(T.any(String, Version))).returns(T.nilable(T::Array[String])) }
|
sig {
|
||||||
def pypi_info(new_version: nil)
|
params(new_version: T.nilable(T.any(String, Version)),
|
||||||
|
ignore_errors: T.nilable(T::Boolean)).returns(T.nilable(T::Array[String]))
|
||||||
|
}
|
||||||
|
def pypi_info(new_version: nil, ignore_errors: false)
|
||||||
return unless valid_pypi_package?
|
return unless valid_pypi_package?
|
||||||
return @pypi_info if @pypi_info.present? && new_version.blank?
|
return @pypi_info if @pypi_info.present? && new_version.blank?
|
||||||
|
|
||||||
@ -87,6 +90,8 @@ module PyPI
|
|||||||
end
|
end
|
||||||
|
|
||||||
if dist.nil?
|
if dist.nil?
|
||||||
|
return ["", "", "", "", "no suitable source distribution on PyPI"] if ignore_errors
|
||||||
|
|
||||||
onoe "#{name} exists on PyPI but lacks a suitable source distribution"
|
onoe "#{name} exists on PyPI but lacks a suitable source distribution"
|
||||||
return
|
return
|
||||||
end
|
end
|
||||||
@ -218,13 +223,14 @@ module PyPI
|
|||||||
print_only: T.nilable(T::Boolean),
|
print_only: T.nilable(T::Boolean),
|
||||||
silent: T.nilable(T::Boolean),
|
silent: T.nilable(T::Boolean),
|
||||||
verbose: T.nilable(T::Boolean),
|
verbose: T.nilable(T::Boolean),
|
||||||
|
ignore_errors: T.nilable(T::Boolean),
|
||||||
ignore_non_pypi_packages: T.nilable(T::Boolean),
|
ignore_non_pypi_packages: T.nilable(T::Boolean),
|
||||||
).returns(T.nilable(T::Boolean))
|
).returns(T.nilable(T::Boolean))
|
||||||
}
|
}
|
||||||
def self.update_python_resources!(formula, version: nil, package_name: nil, extra_packages: nil,
|
def self.update_python_resources!(formula, version: nil, package_name: nil, extra_packages: nil,
|
||||||
exclude_packages: nil, dependencies: nil, install_dependencies: false,
|
exclude_packages: nil, dependencies: nil, install_dependencies: false,
|
||||||
print_only: false, silent: false, verbose: false,
|
print_only: false, silent: false, verbose: false,
|
||||||
ignore_non_pypi_packages: false)
|
ignore_errors: false, ignore_non_pypi_packages: false)
|
||||||
auto_update_list = formula.tap&.pypi_formula_mappings
|
auto_update_list = formula.tap&.pypi_formula_mappings
|
||||||
if auto_update_list.present? && auto_update_list.key?(formula.full_name) &&
|
if auto_update_list.present? && auto_update_list.key?(formula.full_name) &&
|
||||||
package_name.blank? && extra_packages.blank? && exclude_packages.blank?
|
package_name.blank? && extra_packages.blank? && exclude_packages.blank?
|
||||||
@ -350,6 +356,7 @@ module PyPI
|
|||||||
end
|
end
|
||||||
|
|
||||||
new_resource_blocks = ""
|
new_resource_blocks = ""
|
||||||
|
package_errors = ""
|
||||||
found_packages.sort.each do |package|
|
found_packages.sort.each do |package|
|
||||||
if exclude_packages.include? package
|
if exclude_packages.include? package
|
||||||
ohai "Excluding \"#{package}\"" if show_info
|
ohai "Excluding \"#{package}\"" if show_info
|
||||||
@ -358,17 +365,28 @@ module PyPI
|
|||||||
end
|
end
|
||||||
|
|
||||||
ohai "Getting PyPI info for \"#{package}\"" if show_info
|
ohai "Getting PyPI info for \"#{package}\"" if show_info
|
||||||
name, url, checksum = package.pypi_info
|
name, url, checksum, _, package_error = package.pypi_info(ignore_errors: ignore_errors)
|
||||||
|
if package_error.blank?
|
||||||
# Fail if unable to find name, url or checksum for any resource
|
# Fail if unable to find name, url or checksum for any resource
|
||||||
if name.blank?
|
if name.blank?
|
||||||
|
if ignore_errors
|
||||||
|
package_error = "unknown failure"
|
||||||
|
else
|
||||||
odie "Unable to resolve some dependencies. Please update the resources for \"#{formula.name}\" manually."
|
odie "Unable to resolve some dependencies. Please update the resources for \"#{formula.name}\" manually."
|
||||||
|
end
|
||||||
elsif url.blank? || checksum.blank?
|
elsif url.blank? || checksum.blank?
|
||||||
|
if ignore_errors
|
||||||
|
package_error = "unable to find URL and/or sha256"
|
||||||
|
else
|
||||||
odie <<~EOS
|
odie <<~EOS
|
||||||
Unable to find the URL and/or sha256 for the "#{name}" resource.
|
Unable to find the URL and/or sha256 for the "#{name}" resource.
|
||||||
Please update the resources for "#{formula.name}" manually.
|
Please update the resources for "#{formula.name}" manually.
|
||||||
EOS
|
EOS
|
||||||
end
|
end
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
if package_error.blank?
|
||||||
# Append indented resource block
|
# Append indented resource block
|
||||||
new_resource_blocks += <<-EOS
|
new_resource_blocks += <<-EOS
|
||||||
resource "#{name}" do
|
resource "#{name}" do
|
||||||
@ -377,12 +395,18 @@ module PyPI
|
|||||||
end
|
end
|
||||||
|
|
||||||
EOS
|
EOS
|
||||||
|
else
|
||||||
|
# Leave a placeholder for formula author to investigate
|
||||||
|
package_errors += " # RESOURCE-ERROR: Unable to resolve \"#{package}\" (#{package_error})\n"
|
||||||
end
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
resource_section = "#{package_errors}\n#{new_resource_blocks}"
|
||||||
|
|
||||||
odie "Excluded superfluous packages: #{exclude_packages.join(", ")}" if exclude_packages.any?
|
odie "Excluded superfluous packages: #{exclude_packages.join(", ")}" if exclude_packages.any?
|
||||||
|
|
||||||
if print_only
|
if print_only
|
||||||
puts new_resource_blocks.chomp
|
puts resource_section.chomp
|
||||||
return
|
return
|
||||||
end
|
end
|
||||||
|
|
||||||
@ -390,11 +414,12 @@ module PyPI
|
|||||||
if formula.resources.all? { |resource| resource.name.start_with?("homebrew-") }
|
if formula.resources.all? { |resource| resource.name.start_with?("homebrew-") }
|
||||||
# Place resources above install method
|
# Place resources above install method
|
||||||
inreplace_regex = / def install/
|
inreplace_regex = / def install/
|
||||||
new_resource_blocks += " def install"
|
resource_section += " def install"
|
||||||
else
|
else
|
||||||
# Replace existing resource blocks with new resource blocks
|
# Replace existing resource blocks with new resource blocks
|
||||||
inreplace_regex = /
|
inreplace_regex = /
|
||||||
\ \ (
|
\ \ (
|
||||||
|
(\#\ RESOURCE-ERROR:\ .*\s+)*
|
||||||
resource\ .*\ do\s+
|
resource\ .*\ do\s+
|
||||||
url\ .*\s+
|
url\ .*\s+
|
||||||
sha256\ .*\s+
|
sha256\ .*\s+
|
||||||
@ -405,7 +430,7 @@ module PyPI
|
|||||||
end\s+)*
|
end\s+)*
|
||||||
end\s+)+
|
end\s+)+
|
||||||
/x
|
/x
|
||||||
new_resource_blocks += " "
|
resource_section += " "
|
||||||
end
|
end
|
||||||
|
|
||||||
ohai "Updating resource blocks" unless silent
|
ohai "Updating resource blocks" unless silent
|
||||||
@ -413,7 +438,11 @@ module PyPI
|
|||||||
if T.must(s.inreplace_string.split(/^ test do\b/, 2).first).scan(inreplace_regex).length > 1
|
if T.must(s.inreplace_string.split(/^ test do\b/, 2).first).scan(inreplace_regex).length > 1
|
||||||
odie "Unable to update resource blocks for \"#{formula.name}\" automatically. Please update them manually."
|
odie "Unable to update resource blocks for \"#{formula.name}\" automatically. Please update them manually."
|
||||||
end
|
end
|
||||||
s.sub! inreplace_regex, new_resource_blocks
|
s.sub! inreplace_regex, resource_section
|
||||||
|
end
|
||||||
|
|
||||||
|
if package_errors.present?
|
||||||
|
ofail "Unable to resolve some dependencies. Please check #{formula.path} for RESOURCE-ERROR comments."
|
||||||
end
|
end
|
||||||
|
|
||||||
true
|
true
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user