diff --git a/Library/Homebrew/dev-cmd/update-python-resources.rb b/Library/Homebrew/dev-cmd/update-python-resources.rb new file mode 100644 index 0000000000..d41b77a72d --- /dev/null +++ b/Library/Homebrew/dev-cmd/update-python-resources.rb @@ -0,0 +1,37 @@ +# frozen_string_literal: true + +require "cli/parser" +require "utils/pypi" + +module Homebrew + module_function + + def update_python_resources_args + Homebrew::CLI::Parser.new do + usage_banner <<~EOS + `update-python-resources` [] + + Update versions for PyPI resource blocks in . + EOS + switch "-p", "--print-only", + description: "Print the updated resource blocks instead of changing ." + switch "-s", "--silent", + description: "Suppress any output." + switch "--ignore-non-pypi-packages", + description: "Don't fail if is not a PyPI package." + flag "--version=", + description: "Use the specified when finding resources for . "\ + "If no version is specified, the current version for will be used." + min_named :formula + end + end + + def update_python_resources + args = update_python_resources_args.parse + + args.formulae.each do |formula| + PyPI.update_python_resources! formula, args.version, print_only: args.print_only?, silent: args.silent?, + ignore_non_pypi_packages: args.ignore_non_pypi_packages? + end + end +end diff --git a/Library/Homebrew/utils/pypi.rb b/Library/Homebrew/utils/pypi.rb new file mode 100644 index 0000000000..486462be19 --- /dev/null +++ b/Library/Homebrew/utils/pypi.rb @@ -0,0 +1,120 @@ +# frozen_string_literal: true + +module PyPI + module_function + + PYTHONHOSTED_URL_PREFIX = "https://files.pythonhosted.org/packages/" + + @pipgrip_installed = nil + + # Get name, url, and version for a given pypi package + def get_pypi_info(package, version) + metadata_url = "https://pypi.org/pypi/#{package}/#{version}/json" + out, _, status = curl_output metadata_url, "--location" + + return unless status.success? + + begin + json = JSON.parse out + rescue JSON::ParserError + return + end + + sdist = json["urls"].find { |url| url["packagetype"] == "sdist" } + [json["info"]["name"], sdist["url"], sdist["digests"]["sha256"]] + end + + def update_python_resources!(formula, version = nil, print_only: false, silent: false, + ignore_non_pypi_packages: false) + + @pipgrip_installed ||= Formula["pipgrip"].any_version_installed? + odie '"pipgrip" must be installed (`brew install pipgrip`)' unless @pipgrip_installed + + # PyPI package name isn't always the same as the formula name. Try to infer from the URL. + pypi_name = if formula.stable.url.start_with?(PYTHONHOSTED_URL_PREFIX) + File.basename(formula.stable.url).match(/^(.+)-[a-z\d.]+$/)[1] + else + formula.name + end + + version ||= formula.version + + if get_pypi_info(pypi_name, version).blank? + odie "\"#{pypi_name}\" at version #{version} is not available on PyPI." unless ignore_non_pypi_packages + return + end + + non_pypi_resources = formula.resources.reject do |resource| + resource.url.start_with? PYTHONHOSTED_URL_PREFIX + end + + if non_pypi_resources.present? && !print_only + odie "\"#{formula.name}\" contains non-PyPI resources. Please update the resources manually." + end + + ohai "Retrieving PyPI dependencies for \"#{pypi_name}==#{version}\"" if !print_only && !silent + pipgrip_output = Utils.popen_read Formula["pipgrip"].bin/"pipgrip", "--json", "#{pypi_name}==#{version}" + unless $CHILD_STATUS.success? + odie <<~EOS + Unable to determine dependencies for \"#{pypi_name}\" because of a failure when running + `pipgrip --json #{pypi_name}==#{version}`. Please update the resources for \"#{formula.name}\" manually. + EOS + end + + packages = JSON.parse(pipgrip_output).sort.to_h + + # Remove extra packages that may be included in pipgrip output + exclude_list = %W[#{pypi_name} argparse pip setuptools wheel wsgiref] + packages.delete_if do |package| + exclude_list.include? package + end + + new_resource_blocks = "" + packages.each do |package, package_version| + name, url, checksum = get_pypi_info package, package_version + # Fail if unable to find name, url or checksum for any resource + if name.blank? + odie "Unable to resolve some dependencies. Please update the resources for \"#{formula.name}\" manually." + elsif url.blank? || checksum.blank? + odie <<~EOS + Unable to find the URL and/or sha256 for the \"#{name}\" resource. + Please update the resources for \"#{formula.name}\" manually. + EOS + end + + # Append indented resource block + new_resource_blocks += <<-EOS + resource "#{name}" do + url "#{url}" + sha256 "#{checksum}" + end + + EOS + end + + if print_only + puts new_resource_blocks.chomp + return + end + + # Check whether resources already exist (excluding homebrew-virtualenv) + if formula.resources.blank? || + (formula.resources.length == 1 && formula.resources.first.name == "homebrew-virtualenv") + # Place resources above install method + inreplace_regex = / def install/ + new_resource_blocks += " def install" + else + # Replace existing resource blocks with new resource blocks + inreplace_regex = / (resource .* do\s+url .*\s+sha256 .*\s+ end\s*)+/ + new_resource_blocks += " " + end + + ohai "Updating resource blocks" unless silent + Utils::Inreplace.inreplace formula.path do |s| + if s.inreplace_string.scan(inreplace_regex).length > 1 + odie "Unable to update resource blocks for \"#{formula.name}\" automatically. Please update them manually." + end + s.sub! inreplace_regex, new_resource_blocks + end + end +end diff --git a/completions/internal_commands_list.txt b/completions/internal_commands_list.txt index 3beacf0b09..aeb66a2818 100644 --- a/completions/internal_commands_list.txt +++ b/completions/internal_commands_list.txt @@ -88,6 +88,7 @@ untap up update update-license-data +update-python-resources update-report update-reset update-test diff --git a/docs/Manpage.md b/docs/Manpage.md index 22cca4677c..aa331bfb4c 100644 --- a/docs/Manpage.md +++ b/docs/Manpage.md @@ -1081,6 +1081,19 @@ directory. * `--commit`: Commit changes to the SPDX license data. +### `update-python-resources` [*`options`*] *`formula`* + +Update versions for PyPI resource blocks in *`formula`*. + +* `-p`, `--print-only`: + Print the updated resource blocks instead of changing *`formula`*. +* `-s`, `--silent`: + Suppress any output. +* `--ignore-non-pypi-packages`: + Don't fail if *`formula`* is not a PyPI package. +* `--version`: + Use the specified *`version`* when finding resources for *`formula`*. If no version is specified, the current version for *`formula`* will be used. + ### `update-test` [*`options`*] Run a test of `brew update` with a new repository clone. If no options are diff --git a/manpages/brew.1 b/manpages/brew.1 index a9aad74da5..52aa949546 100644 --- a/manpages/brew.1 +++ b/manpages/brew.1 @@ -1416,6 +1416,25 @@ Return a failing status code if current license data\'s version is the same as t \fB\-\-commit\fR Commit changes to the SPDX license data\. . +.SS "\fBupdate\-python\-resources\fR [\fIoptions\fR] \fIformula\fR" +Update versions for PyPI resource blocks in \fIformula\fR\. +. +.TP +\fB\-p\fR, \fB\-\-print\-only\fR +Print the updated resource blocks instead of changing \fIformula\fR\. +. +.TP +\fB\-s\fR, \fB\-\-silent\fR +Suppress any output\. +. +.TP +\fB\-\-ignore\-non\-pypi\-packages\fR +Don\'t fail if \fIformula\fR is not a PyPI package\. +. +.TP +\fB\-\-version\fR +Use the specified \fIversion\fR when finding resources for \fIformula\fR\. If no version is specified, the current version for \fIformula\fR will be used\. +. .SS "\fBupdate\-test\fR [\fIoptions\fR]" Run a test of \fBbrew update\fR with a new repository clone\. If no options are passed, use \fBorigin/master\fR as the start commit\. .