Merge pull request #11150 from MikeMcQuaid/bottle_reproducibility-tweaks

Bottle reproducibility tweaks

Library/Homebrew/dev-cmd/bottle.rb

@@ -325,24 +325,12 @@ module Homebrew
       bottle_path = f.local_bottle_path
       local_filename = bottle_path.basename.to_s
 
-      tab_path = Utils::Bottles.receipt_path(f.local_bottle_path)
+      tab_path = Utils::Bottles.receipt_path(bottle_path)
       raise "This bottle does not contain the file INSTALL_RECEIPT.json: #{bottle_path}" unless tab_path
 
-      tab_json = Utils.safe_popen_read("tar", "xfO", f.local_bottle_path, tab_path)
+      tab_json = Utils::Bottles.file_from_bottle(bottle_path, tab_path)
       tab = Tab.from_file_content(tab_json, tab_path)
 
-      # TODO: most of this logic can be removed when we're done with bulk GitHub Packages bottle uploading
-      tap_git_revision = tab["source"]["tap_git_head"]
-      if tap.core_tap?
-        if bottle_tag.to_s.end_with?("_linux")
-          tap_git_remote = "https://github.com/Homebrew/linuxbrew-core"
-          formulae_brew_sh_path = "formula-linux"
-        else
-          tap_git_remote = "https://github.com/Homebrew/homebrew-core"
-          formulae_brew_sh_path = "formula"
-        end
-      end
-
       _, _, bottle_cellar = Formula[f.name].bottle_specification.checksum_for(bottle_tag, no_older_versions: true)
       relocatable = [:any, :any_skip_relocation].include?(bottle_cellar)
       skip_relocation = bottle_cellar == :any_skip_relocation
@@ -389,6 +377,7 @@ module Homebrew
         end
 
         keg.find do |file|
+          # Set the times for reproducible bottles.
           if file.symlink?
             File.lutime(tab.source_modified_time, tab.source_modified_time, file)
           else
@@ -398,8 +387,11 @@ module Homebrew
 
         cd cellar do
           sudo_purge
-          safe_system "tar", "cf", tar_path, "#{f.name}/#{f.pkg_version}"
+          # Unset the owner/group for reproducible bottles.
+          # Tar then gzip for reproducible bottles.
+          safe_system "tar", "--create", "--numeric-owner", "--file", tar_path, "#{f.name}/#{f.pkg_version}"
           sudo_purge
+          # Set more times for reproducible bottles.
           tar_path.utime(tab.source_modified_time, tab.source_modified_time)
           relocatable_tar_path = "#{f}-bottle.tar"
           mv tar_path, relocatable_tar_path

Library/Homebrew/test/unpack_strategy_spec.rb

@@ -32,7 +32,7 @@ describe UnpackStrategy do
         (mktmpdir/"file.tar").tap do |path|
           mktmpdir do |dir|
             (dir/directories).mkpath
-            system "tar", "-c", "-f", path, "-C", dir, "A/"
+            system "tar", "--create", "--file", path, "--directory", dir, "A/"
           end
         end
       }
@@ -49,7 +49,7 @@ describe UnpackStrategy do
         (mktmpdir/basename).tap do |path|
           mktmpdir do |dir|
             FileUtils.touch dir/"file.txt"
-            system "tar", "-c", "-f", path, "-C", dir, "file.txt"
+            system "tar", "--create", "--file", path, "--directory", dir, "file.txt"
           end
         end
       }

Library/Homebrew/unpack_strategy/tar.rb

@@ -30,7 +30,7 @@ module UnpackStrategy
       return false unless [Bzip2, Gzip, Lzip, Xz].any? { |s| s.can_extract?(path) }
 
       # Check if `tar` can list the contents, then it can also extract it.
-      stdout, _, status = system_command("tar", args: ["tf", path], print_stderr: false)
+      stdout, _, status = system_command("tar", args: ["--list", "--file", path], print_stderr: false)
       status.success? && !stdout.empty?
     end
 
@@ -48,7 +48,9 @@ module UnpackStrategy
         end
 
         system_command! "tar",
-                        args:    ["xof", tar_path, "-C", unpack_dir],
+                        args:    ["--extract", "--no-same-owner",
+                                  "--file", tar_path,
+                                  "--directory", unpack_dir],
                         verbose: verbose
       end
     end

Library/Homebrew/utils/bottles.rb

@@ -40,23 +40,20 @@ module Utils
         HOMEBREW_BOTTLES_EXTNAME_REGEX.match(filename).to_a
       end
 
-      def bottle_file_list(bottle_file)
-        @bottle_file_list ||= {}
-        @bottle_file_list[bottle_file] ||= Utils.popen_read("tar", "-tzf", bottle_file)
-                                                .lines
-                                                .map(&:chomp)
-      end
-
       def receipt_path(bottle_file)
         bottle_file_list(bottle_file).find do |line|
           line =~ %r{.+/.+/INSTALL_RECEIPT.json}
         end
       end
 
+      def file_from_bottle(bottle_file, file_path)
+        Utils.popen_read("tar", "--extract", "--to-stdout", "--file", bottle_file, file_path)
+      end
+
       def resolve_formula_names(bottle_file)
         name = bottle_file_list(bottle_file).first.to_s.split("/").first
         full_name = if (receipt_file_path = receipt_path(bottle_file))
-          receipt_file = Utils.popen_read("tar", "-xOzf", bottle_file, receipt_file_path)
+          receipt_file = file_from_bottle(bottle_file, receipt_file_path)
           tap = Tab.from_file_content(receipt_file, "#{bottle_file}/#{receipt_file_path}").tap
           "#{tap}/#{name}" if tap.present? && !tap.core_tap?
         elsif (bottle_json_path = Pathname(bottle_file.sub(/\.tar\.gz$/, ".json"))) &&
@@ -80,7 +77,7 @@ module Utils
                            name: resolve_formula_names(bottle_file)[0])
         bottle_version = resolve_version bottle_file
         formula_path = "#{name}/#{bottle_version}/.brew/#{name}.rb"
-        contents = Utils.popen_read "tar", "-xOzf", bottle_file, formula_path
+        contents = file_from_bottle(bottle_file, formula_path)
         raise BottleFormulaUnavailableError.new(bottle_file, formula_path) unless $CHILD_STATUS.success?
 
         contents
@@ -94,6 +91,15 @@ module Utils
           filename&.url_encode
         end
       end
+
+      private
+
+      def bottle_file_list(bottle_file)
+        @bottle_file_list ||= {}
+        @bottle_file_list[bottle_file] ||= Utils.popen_read("tar", "--list", "--file", bottle_file)
+                                                .lines
+                                                .map(&:chomp)
+      end
     end
 
     # Denotes the arch and OS of a bottle.

Library/Homebrew/utils/tar.rb

@@ -26,7 +26,7 @@ module Utils
 
         path = Pathname.new(path)
         return unless TAR_FILE_EXTENSIONS.include? path.extname
-        return if Utils.popen_read(executable, "-tf", path).match?(%r{/.*\.})
+        return if Utils.popen_read(executable, "--list", "--file", path).match?(%r{/.*\.})
 
         odie "#{path} is not a valid tar file!"
       end