From ab7af6355d5225473c57b680a481ff1e27403c76 Mon Sep 17 00:00:00 2001 From: William Woodruff Date: Mon, 24 Apr 2023 17:05:51 -0600 Subject: [PATCH 01/12] [WIP] pypi: cut out pipgrip Signed-off-by: William Woodruff --- Library/Homebrew/utils/pypi.rb | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/Library/Homebrew/utils/pypi.rb b/Library/Homebrew/utils/pypi.rb index c65576a186..eae1f1d451 100644 --- a/Library/Homebrew/utils/pypi.rb +++ b/Library/Homebrew/utils/pypi.rb @@ -212,11 +212,11 @@ module PyPI end end - ensure_formula_installed!("pipgrip") + ensure_formula_installed!("python") ohai "Retrieving PyPI dependencies for \"#{input_packages.join(" ")}\"..." if !print_only && !silent command = - [Formula["pipgrip"].opt_bin/"pipgrip", "--json", "--tree", "--no-cache-dir", *input_packages.map(&:to_s)] + [Formula["python"].bin/"python", "-m", "pip", "install", "-q", "--dry-run", "--ignore-installed", "--report", "/dev/stdout", *input_packages.map(&:to_s)] pipgrip_output = Utils.popen_read(*command) unless $CHILD_STATUS.success? odie <<~EOS @@ -284,6 +284,10 @@ module PyPI true end + def self.normalize_python_package(name) + name.gsub(/[-_.]+/, "-").downcase + end + def self.json_to_packages(json_tree, main_package, exclude_packages) return [] if json_tree.blank? From 953c24a7c4579006b9c33578b602bf1d0648a2a0 Mon Sep 17 00:00:00 2001 From: William Woodruff Date: Mon, 24 Apr 2023 17:23:22 -0600 Subject: [PATCH 02/12] [WIP] pypi: fix python executable Signed-off-by: William Woodruff --- Library/Homebrew/utils/pypi.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Library/Homebrew/utils/pypi.rb b/Library/Homebrew/utils/pypi.rb index eae1f1d451..17db37add5 100644 --- a/Library/Homebrew/utils/pypi.rb +++ b/Library/Homebrew/utils/pypi.rb @@ -216,7 +216,7 @@ module PyPI ohai "Retrieving PyPI dependencies for \"#{input_packages.join(" ")}\"..." if !print_only && !silent command = - [Formula["python"].bin/"python", "-m", "pip", "install", "-q", "--dry-run", "--ignore-installed", "--report", "/dev/stdout", *input_packages.map(&:to_s)] + [Formula["python"].bin/"python3", "-m", "pip", "install", "-q", "--dry-run", "--ignore-installed", "--report", "/dev/stdout", *input_packages.map(&:to_s)] pipgrip_output = Utils.popen_read(*command) unless $CHILD_STATUS.success? odie <<~EOS From 0316b9659f9d5dbc089a5858ef95684a70e88fda Mon Sep 17 00:00:00 2001 From: William Woodruff Date: Mon, 24 Apr 2023 17:28:53 -0600 Subject: [PATCH 03/12] [WIP] pypi: hackety hack Signed-off-by: William Woodruff --- Library/Homebrew/utils/pypi.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Library/Homebrew/utils/pypi.rb b/Library/Homebrew/utils/pypi.rb index 17db37add5..d0d3a9e4a2 100644 --- a/Library/Homebrew/utils/pypi.rb +++ b/Library/Homebrew/utils/pypi.rb @@ -217,7 +217,7 @@ module PyPI ohai "Retrieving PyPI dependencies for \"#{input_packages.join(" ")}\"..." if !print_only && !silent command = [Formula["python"].bin/"python3", "-m", "pip", "install", "-q", "--dry-run", "--ignore-installed", "--report", "/dev/stdout", *input_packages.map(&:to_s)] - pipgrip_output = Utils.popen_read(*command) + pipgrip_output = Utils.popen_read(*command, env: { "PIP_REQUIRE_VIRTUALENV" => "false" }) unless $CHILD_STATUS.success? odie <<~EOS Unable to determine dependencies for "#{input_packages.join(" ")}" because of a failure when running From 48a604e31c2e6360534423c86730b554a70fb2bf Mon Sep 17 00:00:00 2001 From: William Woodruff Date: Mon, 24 Apr 2023 17:33:17 -0600 Subject: [PATCH 04/12] [WIP] pypi: hackety hack Signed-off-by: William Woodruff --- Library/Homebrew/utils/pypi.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Library/Homebrew/utils/pypi.rb b/Library/Homebrew/utils/pypi.rb index d0d3a9e4a2..ee2351cc06 100644 --- a/Library/Homebrew/utils/pypi.rb +++ b/Library/Homebrew/utils/pypi.rb @@ -217,7 +217,7 @@ module PyPI ohai "Retrieving PyPI dependencies for \"#{input_packages.join(" ")}\"..." if !print_only && !silent command = [Formula["python"].bin/"python3", "-m", "pip", "install", "-q", "--dry-run", "--ignore-installed", "--report", "/dev/stdout", *input_packages.map(&:to_s)] - pipgrip_output = Utils.popen_read(*command, env: { "PIP_REQUIRE_VIRTUALENV" => "false" }) + pip_output = Utils.popen_read({ "PIP_REQUIRE_VIRTUALENV" => "false" }, *command) unless $CHILD_STATUS.success? odie <<~EOS Unable to determine dependencies for "#{input_packages.join(" ")}" because of a failure when running @@ -226,7 +226,7 @@ module PyPI EOS end - found_packages = json_to_packages(JSON.parse(pipgrip_output), main_package, exclude_packages).uniq + found_packages = json_to_packages(JSON.parse(pip_output), main_package, exclude_packages).uniq new_resource_blocks = "" found_packages.sort.each do |package| From 727a88d0257a9bc2655952abde62996445e72d56 Mon Sep 17 00:00:00 2001 From: William Woodruff Date: Mon, 24 Apr 2023 17:53:09 -0600 Subject: [PATCH 05/12] [WIP] pypi: hackety hack Signed-off-by: William Woodruff --- Library/Homebrew/utils/pypi.rb | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/Library/Homebrew/utils/pypi.rb b/Library/Homebrew/utils/pypi.rb index ee2351cc06..26572ea67a 100644 --- a/Library/Homebrew/utils/pypi.rb +++ b/Library/Homebrew/utils/pypi.rb @@ -226,7 +226,7 @@ module PyPI EOS end - found_packages = json_to_packages(JSON.parse(pip_output), main_package, exclude_packages).uniq + found_packages = pip_report_to_packages(JSON.parse(pip_output), main_package, exclude_packages).uniq new_resource_blocks = "" found_packages.sort.each do |package| @@ -288,17 +288,16 @@ module PyPI name.gsub(/[-_.]+/, "-").downcase end - def self.json_to_packages(json_tree, main_package, exclude_packages) - return [] if json_tree.blank? + def self.pip_report_to_packages(report, main_package, exclude_packages) + return [] if report.blank? - json_tree.flat_map do |package_json| - package = Package.new("#{package_json["name"]}==#{package_json["version"]}") - dependencies = if package == main_package || exclude_packages.exclude?(package) - json_to_packages(package_json["dependencies"], main_package, exclude_packages) - else - [] - end - [package] + dependencies + report["install"].filter_map do |package| + name = normalize_python_package(package["metadata"]["name"]) + version = package["metadata"]["version"] + + package = "#{name}==#{version}" + + package if exclude_packages.exclude? package end end end From 92225e9ec0f2196d87996fb7c6eb7075ffb52ae5 Mon Sep 17 00:00:00 2001 From: William Woodruff Date: Mon, 24 Apr 2023 17:55:18 -0600 Subject: [PATCH 06/12] [WIP] pypi: filter_map sadness Signed-off-by: William Woodruff --- Library/Homebrew/utils/pypi.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Library/Homebrew/utils/pypi.rb b/Library/Homebrew/utils/pypi.rb index 26572ea67a..621375c9e0 100644 --- a/Library/Homebrew/utils/pypi.rb +++ b/Library/Homebrew/utils/pypi.rb @@ -291,13 +291,13 @@ module PyPI def self.pip_report_to_packages(report, main_package, exclude_packages) return [] if report.blank? - report["install"].filter_map do |package| + report["install"].map do |package| name = normalize_python_package(package["metadata"]["name"]) version = package["metadata"]["version"] package = "#{name}==#{version}" package if exclude_packages.exclude? package - end + end.compact end end From 6e798a7ab826f3cc87638063b2ea85c827100c3c Mon Sep 17 00:00:00 2001 From: William Woodruff Date: Mon, 24 Apr 2023 17:56:33 -0600 Subject: [PATCH 07/12] [WIP] pypi: hackety hack Signed-off-by: William Woodruff --- Library/Homebrew/utils/pypi.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Library/Homebrew/utils/pypi.rb b/Library/Homebrew/utils/pypi.rb index 621375c9e0..527ebcc09c 100644 --- a/Library/Homebrew/utils/pypi.rb +++ b/Library/Homebrew/utils/pypi.rb @@ -295,7 +295,7 @@ module PyPI name = normalize_python_package(package["metadata"]["name"]) version = package["metadata"]["version"] - package = "#{name}==#{version}" + package = Package.new "#{name}==#{version}" package if exclude_packages.exclude? package end.compact From ce8834e43e798ba543aa3cfe919f9a09c4037ccd Mon Sep 17 00:00:00 2001 From: William Woodruff Date: Mon, 24 Apr 2023 20:11:24 -0600 Subject: [PATCH 08/12] [WIP] pypi: more name normalization Signed-off-by: William Woodruff --- Library/Homebrew/utils/pypi.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Library/Homebrew/utils/pypi.rb b/Library/Homebrew/utils/pypi.rb index 527ebcc09c..2162dc4922 100644 --- a/Library/Homebrew/utils/pypi.rb +++ b/Library/Homebrew/utils/pypi.rb @@ -28,7 +28,7 @@ module PyPI end raise ArgumentError, "Package should be a valid PyPI URL" if match.blank? - @name = match[1] + @name = normalize_python_package(match[1]) @version = match[2] return end From 57af1abb6c424ee0d1035efdaf89b57cf62c24cd Mon Sep 17 00:00:00 2001 From: William Woodruff Date: Mon, 24 Apr 2023 20:12:09 -0600 Subject: [PATCH 09/12] pypi: namespacing Signed-off-by: William Woodruff --- Library/Homebrew/utils/pypi.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Library/Homebrew/utils/pypi.rb b/Library/Homebrew/utils/pypi.rb index 2162dc4922..87082e783e 100644 --- a/Library/Homebrew/utils/pypi.rb +++ b/Library/Homebrew/utils/pypi.rb @@ -28,7 +28,7 @@ module PyPI end raise ArgumentError, "Package should be a valid PyPI URL" if match.blank? - @name = normalize_python_package(match[1]) + @name = PyPI.normalize_python_package(match[1]) @version = match[2] return end From 12613ea2b0160a9f147bf042fa8f2e24b6687d34 Mon Sep 17 00:00:00 2001 From: William Woodruff Date: Mon, 24 Apr 2023 20:27:31 -0600 Subject: [PATCH 10/12] pypi: unused arg Signed-off-by: William Woodruff --- Library/Homebrew/utils/pypi.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Library/Homebrew/utils/pypi.rb b/Library/Homebrew/utils/pypi.rb index 87082e783e..03b4a56a99 100644 --- a/Library/Homebrew/utils/pypi.rb +++ b/Library/Homebrew/utils/pypi.rb @@ -226,7 +226,7 @@ module PyPI EOS end - found_packages = pip_report_to_packages(JSON.parse(pip_output), main_package, exclude_packages).uniq + found_packages = pip_report_to_packages(JSON.parse(pip_output), exclude_packages).uniq new_resource_blocks = "" found_packages.sort.each do |package| @@ -288,7 +288,7 @@ module PyPI name.gsub(/[-_.]+/, "-").downcase end - def self.pip_report_to_packages(report, main_package, exclude_packages) + def self.pip_report_to_packages(report, exclude_packages) return [] if report.blank? report["install"].map do |package| From 1f671862d5a17b92e0e55b88f47a5deacfbb2daa Mon Sep 17 00:00:00 2001 From: William Woodruff Date: Mon, 24 Apr 2023 20:28:49 -0600 Subject: [PATCH 11/12] pypi: `brew style --fix` Signed-off-by: William Woodruff --- Library/Homebrew/utils/pypi.rb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Library/Homebrew/utils/pypi.rb b/Library/Homebrew/utils/pypi.rb index 03b4a56a99..d85ecc7800 100644 --- a/Library/Homebrew/utils/pypi.rb +++ b/Library/Homebrew/utils/pypi.rb @@ -216,7 +216,8 @@ module PyPI ohai "Retrieving PyPI dependencies for \"#{input_packages.join(" ")}\"..." if !print_only && !silent command = - [Formula["python"].bin/"python3", "-m", "pip", "install", "-q", "--dry-run", "--ignore-installed", "--report", "/dev/stdout", *input_packages.map(&:to_s)] + [Formula["python"].bin/"python3", "-m", "pip", "install", "-q", "--dry-run", "--ignore-installed", "--report", + "/dev/stdout", *input_packages.map(&:to_s)] pip_output = Utils.popen_read({ "PIP_REQUIRE_VIRTUALENV" => "false" }, *command) unless $CHILD_STATUS.success? odie <<~EOS From 62df91f09c23c533a881ce82ccd6b748c5c07932 Mon Sep 17 00:00:00 2001 From: William Woodruff Date: Mon, 24 Apr 2023 20:52:35 -0600 Subject: [PATCH 12/12] pypi: document normalization Signed-off-by: William Woodruff --- Library/Homebrew/utils/pypi.rb | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Library/Homebrew/utils/pypi.rb b/Library/Homebrew/utils/pypi.rb index d85ecc7800..3193698b8a 100644 --- a/Library/Homebrew/utils/pypi.rb +++ b/Library/Homebrew/utils/pypi.rb @@ -286,6 +286,8 @@ module PyPI end def self.normalize_python_package(name) + # This normalization is defined in the PyPA packaging specifications; + # https://packaging.python.org/en/latest/specifications/name-normalization/#name-normalization name.gsub(/[-_.]+/, "-").downcase end