diff --git a/Library/Homebrew/test/utils/pypi_spec.rb b/Library/Homebrew/test/utils/pypi_spec.rb new file mode 100644 index 0000000000..021a2215c8 --- /dev/null +++ b/Library/Homebrew/test/utils/pypi_spec.rb @@ -0,0 +1,171 @@ +# typed: false +# frozen_string_literal: true + +require "utils/pypi" + +describe PyPI do + let(:package_url) do + "https://files.pythonhosted.org/packages/b0/3f/2e1dad67eb172b6443b5eb37eb885a054a55cfd733393071499514140282/"\ + "snakemake-5.29.0.tar.gz" + end + let(:old_package_url) do + "https://files.pythonhosted.org/packages/6f/c4/da52bfdd6168ea46a0fe2b7c983b6c34c377a8733ec177cc00b197a96a9f/"\ + "snakemake-5.28.0.tar.gz" + end + + describe PyPI::Package do + let(:package_checksum) { "47417307d08ecb0707b3b29effc933bd63d8c8e3ab15509c62b685b7614c6568" } + let(:old_package_checksum) { "2367ce91baf7f8fa7738d33aff9670ffdf5410bbac49aeb209f73b45a3425046" } + + let(:package) { described_class.new("snakemake") } + let(:package_with_version) { described_class.new("snakemake==5.28.0") } + let(:package_with_different_version) { described_class.new("snakemake==5.29.0") } + let(:package_with_extra) { described_class.new("snakemake[foo]") } + let(:package_with_extra_and_version) { described_class.new("snakemake[foo]==5.28.0") } + let(:package_from_url) { described_class.new(package_url, is_url: true) } + let(:other_package) { described_class.new("virtualenv==20.2.0") } + + describe "initialize" do + it "initializes name" do + expect(described_class.new("foo").name).to eq "foo" + end + + it "initializes name with extra" do + expect(described_class.new("foo[bar]").name).to eq "foo" + end + + it "initializes extra" do + expect(described_class.new("foo[bar]").extras).to eq ["bar"] + end + + it "initializes multiple extras" do + expect(described_class.new("foo[bar,baz]").extras).to eq ["bar", "baz"] + end + + it "initializes name with version" do + expect(described_class.new("foo==1.2.3").name).to eq "foo" + end + + it "initializes version" do + expect(described_class.new("foo==1.2.3").version).to eq "1.2.3" + end + + it "initializes extra with version" do + expect(described_class.new("foo[bar]==1.2.3").extras).to eq ["bar"] + end + + it "initializes multiple extras with version" do + expect(described_class.new("foo[bar,baz]==1.2.3").extras).to eq ["bar", "baz"] + end + + it "initializes version with extra" do + expect(described_class.new("foo[bar]==1.2.3").version).to eq "1.2.3" + end + + it "initializes version with multiple extras" do + expect(described_class.new("foo[bar,baz]==1.2.3").version).to eq "1.2.3" + end + + it "initializes name from url" do + expect(described_class.new(package_url, is_url: true).name).to eq "snakemake" + end + + it "initializes version from url" do + expect(described_class.new(package_url, is_url: true).version).to eq "5.29.0" + end + end + + describe ".pypi_info", :needs_network do + it "gets pypi info from a package name" do + expect(package.pypi_info.first).to eq "snakemake" + end + + it "gets pypi info from a package name and specified version" do + expect(package.pypi_info(version: "5.29.0")).to eq ["snakemake", package_url, package_checksum, "5.29.0"] + end + + it "gets pypi info from a package name with extra" do + expect(package_with_extra.pypi_info.first).to eq "snakemake" + end + + it "gets pypi info from a package name and version" do + expect(package_with_version.pypi_info).to eq ["snakemake", old_package_url, old_package_checksum, "5.28.0"] + end + + it "gets pypi info from a package name with overriden version" do + expected_result = ["snakemake", package_url, package_checksum, "5.29.0"] + expect(package_with_version.pypi_info(version: "5.29.0")).to eq expected_result + end + + it "gets pypi info from a package name, extras, and version" do + expected_result = ["snakemake", old_package_url, old_package_checksum, "5.28.0"] + expect(package_with_extra_and_version.pypi_info).to eq expected_result + end + + it "gets pypi info from a url" do + expect(package_from_url.pypi_info).to eq ["snakemake", package_url, package_checksum, "5.29.0"] + end + + it "gets pypi info from a url with overriden version" do + expected_result = ["snakemake", old_package_url, old_package_checksum, "5.28.0"] + expect(package_from_url.pypi_info(version: "5.28.0")).to eq expected_result + end + end + + describe ".to_s" do + it "returns string representation of package name" do + expect(package.to_s).to eq "snakemake" + end + + it "returns string representation of package with version" do + expect(package_with_version.to_s).to eq "snakemake==5.28.0" + end + + it "returns string representation of package with extra" do + expect(package_with_extra.to_s).to eq "snakemake[foo]" + end + + it "returns string representation of package with extra and version" do + expect(package_with_extra_and_version.to_s).to eq "snakemake[foo]==5.28.0" + end + + it "returns string representation of package from url" do + expect(package_from_url.to_s).to eq "snakemake==5.29.0" + end + end + + describe ".same_package?" do + it "returns false for different packages" do + expect(package.same_package?(other_package)).to eq false + end + + it "returns true for the same package" do + expect(package.same_package?(package_with_version)).to eq true + end + + it "returns true for the same package with different versions" do + expect(package_with_version.same_package?(package_with_different_version)).to eq true + end + end + + describe "<=>" do + it "returns -1" do + expect(package <=> other_package).to eq((-1)) + end + + it "returns 0" do + expect(package <=> package_with_version).to eq 0 + end + + it "returns 1" do + expect(other_package <=> package_with_extra_and_version).to eq 1 + end + end + end + + describe "update_pypi_url", :needs_network do + it "updates url to new version" do + expect(described_class.update_pypi_url(old_package_url, "5.29.0")).to eq package_url + end + end +end diff --git a/Library/Homebrew/utils/pypi.rb b/Library/Homebrew/utils/pypi.rb index 4c15a03140..d69291b069 100644 --- a/Library/Homebrew/utils/pypi.rb +++ b/Library/Homebrew/utils/pypi.rb @@ -5,6 +5,8 @@ # # @api private module PyPI + extend T::Sig + module_function PYTHONHOSTED_URL_PREFIX = "https://files.pythonhosted.org/packages/" @@ -12,45 +14,119 @@ module PyPI @pipgrip_installed = nil - def url_to_pypi_package_name(url) - return unless url.start_with? PYTHONHOSTED_URL_PREFIX + # PyPI Package + # + # @api private + class Package + extend T::Sig - File.basename(url).match(/^(.+)-[a-z\d.]+$/)[1] + attr_accessor :name + attr_accessor :extras + attr_accessor :version + + sig { params(package_string: String, is_url: T::Boolean).void } + def initialize(package_string, is_url: false) + @pypi_info = nil + + if is_url + unless package_string.start_with?(PYTHONHOSTED_URL_PREFIX) && + match = File.basename(package_string).match(/^(.+)-([a-z\d.]+?)(?:.tar.gz|.zip)$/) + raise ArgumentError, "package should be a valid PyPI url" + end + + @name = match[1] + @version = match[2] + return + end + + @name = package_string + @name, @version = @name.split("==") if @name.include? "==" + + return unless match = @name.match(/^(.*?)\[(.+)\]$/) + + @name = match[1] + @extras = match[2].split "," + end + + # Get name, URL, SHA-256 checksum, and latest version for a given PyPI package. + sig { params(version: T.nilable(T.any(String, Version))).returns(T.nilable(T::Array[String])) } + def pypi_info(version: nil) + return @pypi_info if @pypi_info.present? && version.blank? + + version ||= @version + metadata_url = if version.present? + "https://pypi.org/pypi/#{@name}/#{version}/json" + else + "https://pypi.org/pypi/#{@name}/json" + end + out, _, status = curl_output metadata_url, "--location" + + return unless status.success? + + begin + json = JSON.parse out + rescue JSON::ParserError + return + end + + sdist = json["urls"].find { |url| url["packagetype"] == "sdist" } + return json["info"]["name"] if sdist.nil? + + @pypi_info = [json["info"]["name"], sdist["url"], sdist["digests"]["sha256"], json["info"]["version"]] + end + + sig { returns(T::Boolean) } + def valid_pypi_package? + info = pypi_info + info.present? && info.is_a?(Array) + end + + sig { returns(String) } + def to_s + out = @name + out += "[#{@extras.join(",")}]" if @extras.present? + out += "==#{@version}" if @version.present? + out + end + + sig { params(other: Package).returns(T::Boolean) } + def same_package?(other) + @name.tr("_", "-") == other.name.tr("_", "-") + end + + # Compare only names so we can use .include? on a Package array + sig { params(other: Package).returns(T::Boolean) } + def ==(other) + same_package?(other) + end + + sig { params(other: Package).returns(T.nilable(Integer)) } + def <=>(other) + @name <=> other.name + end end + sig { params(url: String, version: T.any(String, Version)).returns(T.nilable(String)) } def update_pypi_url(url, version) - package = url_to_pypi_package_name url - return if package.nil? + package = Package.new url, is_url: true - _, url = get_pypi_info(package, version) + _, url = package.pypi_info(version: version) url end - # Get name, URL, SHA-256 checksum, and latest version for a given PyPI package. - def get_pypi_info(package, version = nil) - package = package.split("[").first - metadata_url = if version.present? - "https://pypi.org/pypi/#{package}/#{version}/json" - else - "https://pypi.org/pypi/#{package}/json" - end - out, _, status = curl_output metadata_url, "--location" - - return unless status.success? - - begin - json = JSON.parse out - rescue JSON::ParserError - return - end - - sdist = json["urls"].find { |url| url["packagetype"] == "sdist" } - return json["info"]["name"] if sdist.nil? - - [json["info"]["name"], sdist["url"], sdist["digests"]["sha256"], json["info"]["version"]] - end - # Return true if resources were checked (even if no change). + sig do + params( + formula: Formula, + version: T.nilable(String), + package_name: T.nilable(String), + extra_packages: T.nilable(T::Array[String]), + exclude_packages: T.nilable(T::Array[String]), + print_only: T::Boolean, + silent: T::Boolean, + ignore_non_pypi_packages: T::Boolean, + ).returns(T.nilable(T::Boolean)) + end def update_python_resources!(formula, version: nil, package_name: nil, extra_packages: nil, exclude_packages: nil, print_only: false, silent: false, ignore_non_pypi_packages: false) @@ -73,12 +149,17 @@ module PyPI end end - version ||= formula.version if package_name.blank? - package_name ||= url_to_pypi_package_name formula.stable.url - extra_packages ||= [] - exclude_packages ||= [] + main_package = if package_name.present? + Package.new(package_name) + else + begin + Package.new(formula.stable.url, is_url: true) + rescue ArgumentError + nil + end + end - if package_name.blank? + if main_package.blank? return if ignore_non_pypi_packages odie <<~EOS @@ -87,76 +168,81 @@ module PyPI EOS end - input_package_names = { package_name => version } - extra_packages.each do |extra| - extra_name, extra_version = extra.split "==" + unless main_package.valid_pypi_package? + return if ignore_non_pypi_packages - if input_package_names.key?(extra_name) && input_package_names[extra_name] != extra_version - odie "Conflicting versions specified for the `#{extra_name}` package: "\ - "#{input_package_names[extra_name]}, #{extra_version}" + odie "\"#{main_package}\" is not available on PyPI." + end + + main_package.version = version if version.present? + + extra_packages = (extra_packages || []).map { |p| Package.new p } + exclude_packages = (exclude_packages || []).map { |p| Package.new p } + + input_packages = [main_package] + extra_packages.each do |extra_package| + if !extra_package.valid_pypi_package? && !ignore_non_pypi_packages + odie "\"#{extra_package}\" is not available on PyPI." end - input_package_names[extra_name] = extra_version + input_packages.each do |existing_package| + if existing_package.same_package?(extra_package) && existing_package.version != extra_package.version + odie "Conflicting versions specified for the `#{extra_package.name}` package: "\ + "#{existing_package.version}, #{extra_package.version}" + end + end + + input_packages << extra_package unless input_packages.include? extra_package end - input_package_names.each do |name, package_version| - name = name.split("[").first - next if get_pypi_info(name, package_version).present? - - version_string = " at version #{package_version}" if package_version.present? - odie "\"#{name}\"#{version_string} is not available on PyPI." unless ignore_non_pypi_packages - end - - non_pypi_resources = formula.resources.reject do |resource| - resource.url.start_with? PYTHONHOSTED_URL_PREFIX - end - - if non_pypi_resources.present? && !print_only - odie "\"#{formula.name}\" contains non-PyPI resources. Please update the resources manually." + formula.resources.each do |resource| + if !print_only && !resource.url.start_with?(PYTHONHOSTED_URL_PREFIX) + odie "\"#{formula.name}\" contains non-PyPI resources. Please update the resources manually." + end end @pipgrip_installed ||= Formula["pipgrip"].any_version_installed? odie '"pipgrip" must be installed (`brew install pipgrip`)' unless @pipgrip_installed - found_packages = {} - input_package_names.each do |name, package_version| - pypi_package_string = if package_version.present? - "#{name}==#{package_version}" - else - name - end - - ohai "Retrieving PyPI dependencies for \"#{pypi_package_string}\"..." if !print_only && !silent - pipgrip_output = Utils.popen_read Formula["pipgrip"].bin/"pipgrip", "--json", "--no-cache-dir", - pypi_package_string + found_packages = [] + input_packages.each do |package| + ohai "Retrieving PyPI dependencies for \"#{package}\"..." if !print_only && !silent + pipgrip_output = Utils.popen_read Formula["pipgrip"].bin/"pipgrip", "--json", "--no-cache-dir", package.to_s unless $CHILD_STATUS.success? odie <<~EOS - Unable to determine dependencies for \"#{name}\" because of a failure when running - `pipgrip --json --no-cache-dir #{pypi_package_string}`. + Unable to determine dependencies for \"#{package}\" because of a failure when running + `pipgrip --json --no-cache-dir #{package}`. Please update the resources for \"#{formula.name}\" manually. EOS end - found_packages.merge!(JSON.parse(pipgrip_output).to_h) do |conflicting_package, old_version, new_version| - next old_version if old_version == new_version + JSON.parse(pipgrip_output).to_h.each do |new_name, new_version| + new_package = Package.new("#{new_name}==#{new_version}") - odie "Conflicting versions found for the `#{conflicting_package}` resource: #{old_version}, #{new_version}" + found_packages.each do |existing_package| + if existing_package.same_package?(new_package) && existing_package.version != new_package.version + odie "Conflicting versions found for the `#{new_package.name}` resource: "\ + "#{existing_package.version}, #{new_package.version}" + end + end + + found_packages << new_package unless found_packages.include? new_package end end # Remove extra packages that may be included in pipgrip output - exclude_list = %W[#{package_name.split("[").first.downcase} argparse pip setuptools wheel wsgiref] + exclude_list = %W[#{main_package.name} argparse pip setuptools wheel wsgiref].map { |p| Package.new p } found_packages.delete_if { |package| exclude_list.include? package } new_resource_blocks = "" - found_packages.sort.each do |package, package_version| + found_packages.sort.each do |package| if exclude_packages.include? package - ohai "Excluding \"#{package}==#{package_version}\"" if !print_only && !silent + ohai "Excluding \"#{package}\"" if !print_only && !silent next end - ohai "Getting PyPI info for \"#{package}==#{package_version}\"" if !print_only && !silent - name, url, checksum = get_pypi_info package, package_version + ohai "Getting PyPI info for \"#{package}\"" if !print_only && !silent + name, url, checksum = package.pypi_info # Fail if unable to find name, url or checksum for any resource if name.blank? odie "Unable to resolve some dependencies. Please update the resources for \"#{formula.name}\" manually."