Merge pull request #19252 from gromgit/update-python-resources/ignore-errors

update-python-resources: add option to ignore errors

Library/Homebrew/dev-cmd/update-python-resources.rb

@@ -15,6 +15,9 @@ module Homebrew
                description: "Print the updated resource blocks instead of changing <formula>."
         switch "-s", "--silent",
                description: "Suppress any output."
+        switch "--ignore-errors",
+               description: "Record all discovered resources, even those that can't be resolved successfully. " \
+                            "This option is ignored for homebrew/core formulae."
         switch "--ignore-non-pypi-packages",
                description: "Don't fail if <formula> is not a PyPI package."
         switch "--install-dependencies",
@@ -36,6 +39,11 @@ module Homebrew
       sig { override.void }
       def run
         args.named.to_formulae.each do |formula|
+          ignore_errors = if T.must(formula.tap).name == "homebrew/core"
+            false
+          else
+            args.ignore_errors?
+          end
           PyPI.update_python_resources! formula,
                                         version:                  args.version,
                                         package_name:             args.package_name,
@@ -45,6 +53,7 @@ module Homebrew
                                         print_only:               args.print_only?,
                                         silent:                   args.silent?,
                                         verbose:                  args.verbose?,
+                                        ignore_errors:            ignore_errors,
                                         ignore_non_pypi_packages: args.ignore_non_pypi_packages?
         end
       end

Library/Homebrew/sorbet/rbi/dsl/homebrew/dev_cmd/update_python_resources.rbi

@@ -17,6 +17,9 @@ class Homebrew::DevCmd::UpdatePythonResources::Args < Homebrew::CLI::Args
   sig { returns(T.nilable(T::Array[String])) }
   def extra_packages; end
 
+  sig { returns(T::Boolean) }
+  def ignore_errors?; end
+
   sig { returns(T::Boolean) }
   def ignore_non_pypi_packages?; end
 

Library/Homebrew/test/sorbet/tapioca/compilers/args_spec.rb

@@ -27,7 +27,7 @@ RSpec.describe Tapioca::Compilers::Args do
 
     it "returns a mapping of update-python-resources args to default values" do
       expect(compiler.args_table(update_python_resources_parser)).to contain_exactly(
-        :d?, :debug?, :exclude_packages, :extra_packages, :h?, :help?, :ignore_non_pypi_packages?,
+        :d?, :debug?, :exclude_packages, :extra_packages, :h?, :help?, :ignore_errors?, :ignore_non_pypi_packages?,
         :install_dependencies?, :p?, :package_name, :print_only?, :q?, :quiet?, :s?, :silent?, :v?, :verbose?,
         :version
       )

Library/Homebrew/utils/pypi.rb

@@ -54,8 +54,11 @@ module PyPI
     # Get name, URL, SHA-256 checksum and latest version for a given package.
     # This only works for packages from PyPI or from a PyPI URL; packages
     # derived from non-PyPI URLs will produce `nil` here.
-    sig { params(new_version: T.nilable(T.any(String, Version))).returns(T.nilable(T::Array[String])) }
+    sig {
-    def pypi_info(new_version: nil)
+      params(new_version:   T.nilable(T.any(String, Version)),
+             ignore_errors: T.nilable(T::Boolean)).returns(T.nilable(T::Array[String]))
+    }
+    def pypi_info(new_version: nil, ignore_errors: false)
       return unless valid_pypi_package?
       return @pypi_info if @pypi_info.present? && new_version.blank?
 
@@ -87,6 +90,8 @@ module PyPI
       end
 
       if dist.nil?
+        return ["", "", "", "", "no suitable source distribution on PyPI"] if ignore_errors
+
         onoe "#{name} exists on PyPI but lacks a suitable source distribution"
         return
       end
@@ -218,13 +223,14 @@ module PyPI
       print_only:               T.nilable(T::Boolean),
       silent:                   T.nilable(T::Boolean),
       verbose:                  T.nilable(T::Boolean),
+      ignore_errors:            T.nilable(T::Boolean),
       ignore_non_pypi_packages: T.nilable(T::Boolean),
     ).returns(T.nilable(T::Boolean))
   }
   def self.update_python_resources!(formula, version: nil, package_name: nil, extra_packages: nil,
                                     exclude_packages: nil, dependencies: nil, install_dependencies: false,
                                     print_only: false, silent: false, verbose: false,
-                                    ignore_non_pypi_packages: false)
+                                    ignore_errors: false, ignore_non_pypi_packages: false)
     auto_update_list = formula.tap&.pypi_formula_mappings
     if auto_update_list.present? && auto_update_list.key?(formula.full_name) &&
        package_name.blank? && extra_packages.blank? && exclude_packages.blank?
@@ -350,6 +356,7 @@ module PyPI
     end
 
     new_resource_blocks = ""
+    package_errors = ""
     found_packages.sort.each do |package|
       if exclude_packages.include? package
         ohai "Excluding \"#{package}\"" if show_info
@@ -358,31 +365,48 @@ module PyPI
       end
 
       ohai "Getting PyPI info for \"#{package}\"" if show_info
-      name, url, checksum = package.pypi_info
+      name, url, checksum, _, package_error = package.pypi_info(ignore_errors: ignore_errors)
-      # Fail if unable to find name, url or checksum for any resource
+      if package_error.blank?
-      if name.blank?
+        # Fail if unable to find name, url or checksum for any resource
-        odie "Unable to resolve some dependencies. Please update the resources for \"#{formula.name}\" manually."
+        if name.blank?
-      elsif url.blank? || checksum.blank?
+          if ignore_errors
-        odie <<~EOS
+            package_error = "unknown failure"
-          Unable to find the URL and/or sha256 for the "#{name}" resource.
+          else
-          Please update the resources for "#{formula.name}" manually.
+            odie "Unable to resolve some dependencies. Please update the resources for \"#{formula.name}\" manually."
-        EOS
+          end
+        elsif url.blank? || checksum.blank?
+          if ignore_errors
+            package_error = "unable to find URL and/or sha256"
+          else
+            odie <<~EOS
+              Unable to find the URL and/or sha256 for the "#{name}" resource.
+              Please update the resources for "#{formula.name}" manually.
+            EOS
+          end
+        end
       end
 
-      # Append indented resource block
+      if package_error.blank?
-      new_resource_blocks += <<-EOS
+        # Append indented resource block
+        new_resource_blocks += <<-EOS
   resource "#{name}" do
     url "#{url}"
     sha256 "#{checksum}"
   end
 
-      EOS
+        EOS
+      else
+        # Leave a placeholder for formula author to investigate
+        package_errors += "  # RESOURCE-ERROR: Unable to resolve \"#{package}\" (#{package_error})\n"
+      end
     end
 
+    resource_section = "#{package_errors}\n#{new_resource_blocks}"
+
     odie "Excluded superfluous packages: #{exclude_packages.join(", ")}" if exclude_packages.any?
 
     if print_only
-      puts new_resource_blocks.chomp
+      puts resource_section.chomp
       return
     end
 
@@ -390,11 +414,12 @@ module PyPI
     if formula.resources.all? { |resource| resource.name.start_with?("homebrew-") }
       # Place resources above install method
       inreplace_regex = /  def install/
-      new_resource_blocks += "  def install"
+      resource_section += "  def install"
     else
       # Replace existing resource blocks with new resource blocks
       inreplace_regex = /
         \ \ (
+        (\#\ RESOURCE-ERROR:\ .*\s+)*
         resource\ .*\ do\s+
           url\ .*\s+
           sha256\ .*\s+
@@ -405,7 +430,7 @@ module PyPI
           end\s+)*
         end\s+)+
       /x
-      new_resource_blocks += "  "
+      resource_section += "  "
     end
 
     ohai "Updating resource blocks" unless silent
@@ -413,7 +438,11 @@ module PyPI
       if T.must(s.inreplace_string.split(/^  test do\b/, 2).first).scan(inreplace_regex).length > 1
         odie "Unable to update resource blocks for \"#{formula.name}\" automatically. Please update them manually."
       end
-      s.sub! inreplace_regex, new_resource_blocks
+      s.sub! inreplace_regex, resource_section
+    end
+
+    if package_errors.present?
+      ofail "Unable to resolve some dependencies. Please check #{formula.path} for RESOURCE-ERROR comments."
     end
 
     true