From 5b3bbb76c9d224c08ae498677868b69aab71ff7c Mon Sep 17 00:00:00 2001 From: Markus Reiter Date: Sun, 1 Jul 2018 23:35:29 +0200 Subject: [PATCH] Separate staging from download. --- Library/Homebrew/download_strategy.rb | 250 +++++-------- Library/Homebrew/extend/pathname.rb | 41 --- Library/Homebrew/formula_installer.rb | 2 + Library/Homebrew/sandbox.rb | 9 + .../Homebrew/test/download_strategies_spec.rb | 19 - .../Homebrew/test/support/fixtures/test.jar | Bin 0 -> 397 bytes .../Homebrew/test/support/fixtures/test.lha | Bin 0 -> 51 bytes .../Homebrew/test/support/fixtures/test.lz | Bin 0 -> 36 bytes Library/Homebrew/test/unpack_strategy_spec.rb | 127 +++++++ Library/Homebrew/unpack_strategy.rb | 344 ++++++++++++++++++ 10 files changed, 582 insertions(+), 210 deletions(-) create mode 100644 Library/Homebrew/test/support/fixtures/test.jar create mode 100644 Library/Homebrew/test/support/fixtures/test.lha create mode 100644 Library/Homebrew/test/support/fixtures/test.lz create mode 100644 Library/Homebrew/test/unpack_strategy_spec.rb create mode 100644 Library/Homebrew/unpack_strategy.rb diff --git a/Library/Homebrew/download_strategy.rb b/Library/Homebrew/download_strategy.rb index c7a2739019..3a8f3d24a6 100644 --- a/Library/Homebrew/download_strategy.rb +++ b/Library/Homebrew/download_strategy.rb @@ -1,6 +1,7 @@ require "json" require "rexml/document" require "time" +require "unpack_strategy" class AbstractDownloadStrategy extend Forwardable @@ -45,7 +46,10 @@ class AbstractDownloadStrategy # Unpack {#cached_location} into the current working directory, and possibly # chdir into the newly-unpacked directory. # Unlike {Resource#stage}, this does not take a block. - def stage; end + def stage + UnpackStrategy.detect(cached_location) + .extract(basename: basename_without_params) + end # @!attribute [r] cached_location # The path to the cached file or directory associated with the resource. @@ -63,22 +67,6 @@ class AbstractDownloadStrategy rm_rf(cached_location) end - def expand_safe_system_args(args) - args = args.dup - args.each_with_index do |arg, ii| - next unless arg.is_a? Hash - if ARGV.verbose? - args.delete_at ii - else - args[ii] = arg[:quiet_flag] - end - return args - end - # 2 as default because commands are eg. svn up, git pull - args.insert(2, "-q") unless ARGV.verbose? - args - end - def safe_system(*args) if @shutup quiet_system(*args) || raise(ErrorDuringExecution.new(args.shift, args)) @@ -87,8 +75,9 @@ class AbstractDownloadStrategy end end - def quiet_safe_system(*args) - safe_system(*expand_safe_system_args(args)) + def basename_without_params + # Strip any ?thing=wad out of .c?thing=wad style extensions + File.basename(@url)[/[^?]+/] end end @@ -152,7 +141,7 @@ class VCSDownloadStrategy < AbstractDownloadStrategy private def cache_tag - "__UNKNOWN__" + raise NotImplementedError end def cache_filename @@ -160,7 +149,7 @@ class VCSDownloadStrategy < AbstractDownloadStrategy end def repo_valid? - true + raise NotImplementedError end def clone_repo; end @@ -177,40 +166,8 @@ end class AbstractFileDownloadStrategy < AbstractDownloadStrategy def stage - path = cached_location - unpack_dir = Pathname.pwd - - case type = path.compression_type - when :zip - safe_system "unzip", "-qq", path, "-d", unpack_dir - chdir - when :gzip_only - FileUtils.cp path, unpack_dir, preserve: true - safe_system "gunzip", "-q", "-N", unpack_dir/path.basename - when :bzip2_only - FileUtils.cp path, unpack_dir, preserve: true - safe_system "bunzip2", "-q", unpack_dir/path.basename - when :gzip, :bzip2, :xz, :compress, :tar - if type == :xz && DependencyCollector.tar_needs_xz_dependency? - pipe_to_tar "#{HOMEBREW_PREFIX}/opt/xz/bin/xz", unpack_dir - else - safe_system "tar", "xf", path, "-C", unpack_dir - end - chdir - when :lzip - pipe_to_tar "#{HOMEBREW_PREFIX}/opt/lzip/bin/lzip", unpack_dir - chdir - when :lha - safe_system "#{HOMEBREW_PREFIX}/opt/lha/bin/lha", "xq2w=#{unpack_dir}", path - when :xar - safe_system "xar", "-x", "-f", path, "-C", unpack_dir - when :rar - safe_system "unrar", "x", "-inul", path, unpack_dir - when :p7zip - safe_system "7zr", "x", "-y", "-bd", "-bso0", path, "-o#{unpack_dir}" - else - cp path, unpack_dir/basename_without_params, preserve: true - end + super + chdir end private @@ -227,22 +184,6 @@ class AbstractFileDownloadStrategy < AbstractDownloadStrategy end end - def pipe_to_tar(tool, unpack_dir) - path = cached_location - - Utils.popen_read(tool, "-dc", path) do |rd| - Utils.popen_write("tar", "xf", "-", "-C", unpack_dir) do |wr| - buf = "" - wr.write(buf) while rd.read(16384, buf) - end - end - end - - def basename_without_params - # Strip any ?thing=wad out of .c?thing=wad style extensions - File.basename(@url)[/[^?]+/] - end - def ext # We need a Pathname because we've monkeypatched extname to support double # extensions (e.g. tar.gz). @@ -384,7 +325,8 @@ end # Useful for installing jars. class NoUnzipCurlDownloadStrategy < CurlDownloadStrategy def stage - cp cached_location, basename_without_params, preserve: true + UncompressedUnpackStrategy.new(cached_location) + .extract(basename: basename_without_params) end end @@ -607,11 +549,6 @@ class SubversionDownloadStrategy < VCSDownloadStrategy super end - def stage - super - safe_system "svn", "export", "--force", cached_location, Dir.pwd - end - def source_modified_time xml = REXML::Document.new(Utils.popen_read("svn", "info", "--xml", cached_location.to_s)) Time.parse REXML::XPath.first(xml, "//date/text()").to_s @@ -647,7 +584,7 @@ class SubversionDownloadStrategy < VCSDownloadStrategy args << "-r" << revision end args << "--ignore-externals" if ignore_externals - quiet_safe_system(*args) + safe_system(*args) end def cache_tag @@ -692,11 +629,6 @@ class GitDownloadStrategy < VCSDownloadStrategy @shallow = meta.fetch(:shallow) { true } end - def stage - super - cp_r File.join(cached_location, "."), Dir.pwd, preserve: true - end - def source_modified_time Time.parse Utils.popen_read("git", "--git-dir", git_dir, "show", "-s", "--format=%cD") end @@ -929,10 +861,6 @@ class CVSDownloadStrategy < VCSDownloadStrategy end end - def cvspath - @cvspath ||= which("cvs", PATH.new("/usr/bin", Formula["cvs"].opt_bin, ENV["PATH"])) - end - def source_modified_time # Filter CVS's files because the timestamp for each of them is the moment # of clone. @@ -946,10 +874,6 @@ class CVSDownloadStrategy < VCSDownloadStrategy max_mtime end - def stage - cp_r File.join(cached_location, "."), Dir.pwd, preserve: true - end - private def cache_tag @@ -960,16 +884,23 @@ class CVSDownloadStrategy < VCSDownloadStrategy (cached_location/"CVS").directory? end + def quiet_flag + "-Q" unless ARGV.verbose? + end + def clone_repo - HOMEBREW_CACHE.cd do + with_cvs_env do # Login is only needed (and allowed) with pserver; skip for anoncvs. - quiet_safe_system cvspath, { quiet_flag: "-Q" }, "-d", @url, "login" if @url.include? "pserver" - quiet_safe_system cvspath, { quiet_flag: "-Q" }, "-d", @url, "checkout", "-d", cache_filename, @module + safe_system "cvs", *quiet_flag, "-d", @url, "login" if @url.include? "pserver" + safe_system "cvs", *quiet_flag, "-d", @url, "checkout", "-d", cached_location.basename, @module, + chdir: cached_location.dirname end end def update - cached_location.cd { quiet_safe_system cvspath, { quiet_flag: "-Q" }, "up" } + with_cvs_env do + safe_system "cvs", *quiet_flag, "update", chdir: cached_location + end end def split_url(in_url) @@ -978,6 +909,12 @@ class CVSDownloadStrategy < VCSDownloadStrategy url = parts.join(":") [mod, url] end + + def with_cvs_env + with_env PATH => PATH.new("/usr/bin", Formula["cvs"].opt_bin, ENV["PATH"]) do + yield + end + end end class MercurialDownloadStrategy < VCSDownloadStrategy @@ -986,30 +923,16 @@ class MercurialDownloadStrategy < VCSDownloadStrategy @url = @url.sub(%r{^hg://}, "") end - def hgpath - @hgpath ||= which("hg", PATH.new(Formula["mercurial"].opt_bin, ENV["PATH"])) - end - - def stage - super - - dst = Dir.getwd - cached_location.cd do - if @ref_type && @ref - ohai "Checking out #{@ref_type} #{@ref}" if @ref_type && @ref - safe_system hgpath, "archive", "--subrepos", "-y", "-r", @ref, "-t", "files", dst - else - safe_system hgpath, "archive", "--subrepos", "-y", "-t", "files", dst - end + def source_modified_time + with_hg_env do + Time.parse Utils.popen_read("hg", "tip", "--template", "{date|isodate}", "-R", cached_location.to_s) end end - def source_modified_time - Time.parse Utils.popen_read(hgpath, "tip", "--template", "{date|isodate}", "-R", cached_location.to_s) - end - def last_commit - Utils.popen_read(hgpath, "parent", "--template", "{node|short}", "-R", cached_location.to_s) + with_hg_env do + Utils.popen_read("hg", "parent", "--template", "{node|short}", "-R", cached_location.to_s) + end end private @@ -1023,12 +946,29 @@ class MercurialDownloadStrategy < VCSDownloadStrategy end def clone_repo - safe_system hgpath, "clone", @url, cached_location + with_hg_env do + safe_system "hg", "clone", @url, cached_location + end end def update - cached_location.cd do - safe_system hgpath, "pull", "--update" + with_hg_env do + safe_system "hg", "--cwd", cached_location, "pull", "--update" + + update_args = if @ref_type && @ref + ohai "Checking out #{@ref_type} #{@ref}" + [@ref] + else + ["--clean"] + end + + safe_system "hg", "--cwd", cached_location, "update", *update_args + end + end + + def with_hg_env + with_env PATH => PATH.new(Formula["mercurial"].opt_bin, ENV["PATH"]) do + yield end end end @@ -1040,25 +980,18 @@ class BazaarDownloadStrategy < VCSDownloadStrategy ENV["BZR_HOME"] = HOMEBREW_TEMP end - def bzrpath - @bzrpath ||= which("bzr", PATH.new(Formula["bazaar"].opt_bin, ENV["PATH"])) - end - - def stage - # The export command doesn't work on checkouts - # See https://bugs.launchpad.net/bzr/+bug/897511 - cp_r File.join(cached_location, "."), Dir.pwd, preserve: true - rm_r ".bzr" - end - def source_modified_time - timestamp = Utils.popen_read(bzrpath, "log", "-l", "1", "--timezone=utc", cached_location.to_s)[/^timestamp: (.+)$/, 1] + timestamp = with_bazaar_env do + Utils.popen_read("bzr", "log", "-l", "1", "--timezone=utc", cached_location.to_s)[/^timestamp: (.+)$/, 1] + end raise "Could not get any timestamps from bzr!" if timestamp.to_s.empty? Time.parse timestamp end def last_commit - Utils.popen_read(bzrpath, "revno", cached_location.to_s).chomp + with_bazaar_env do + Utils.popen_read("bzr", "revno", cached_location.to_s).chomp + end end private @@ -1072,13 +1005,21 @@ class BazaarDownloadStrategy < VCSDownloadStrategy end def clone_repo - # "lightweight" means history-less - safe_system bzrpath, "checkout", "--lightweight", @url, cached_location + with_bazaar_env do + # "lightweight" means history-less + safe_system "bzr", "checkout", "--lightweight", @url, cached_location + end end def update - cached_location.cd do - safe_system bzrpath, "update" + with_bazaar_env do + safe_system "bzr", "update", chdir: cached_location + end + end + + def with_bazaar_env + with_env "PATH" => PATH.new(Formula["bazaar"].opt_bin, ENV["PATH"]) do + yield end end end @@ -1089,23 +1030,22 @@ class FossilDownloadStrategy < VCSDownloadStrategy @url = @url.sub(%r{^fossil://}, "") end - def fossilpath - @fossilpath ||= which("fossil", PATH.new(Formula["fossil"].opt_bin, ENV["PATH"])) - end - - def stage - super - args = [fossilpath, "open", cached_location] - args << @ref if @ref_type && @ref - safe_system(*args) - end - def source_modified_time - Time.parse Utils.popen_read(fossilpath, "info", "tip", "-R", cached_location.to_s)[/^uuid: +\h+ (.+)$/, 1] + with_fossil_env do + Time.parse Utils.popen_read("fossil", "info", "tip", "-R", cached_location.to_s)[/^uuid: +\h+ (.+)$/, 1] + end end def last_commit - Utils.popen_read(fossilpath, "info", "tip", "-R", cached_location.to_s)[/^uuid: +(\h+) .+$/, 1] + with_fossil_env do + Utils.popen_read("fossil", "info", "tip", "-R", cached_location.to_s)[/^uuid: +(\h+) .+$/, 1] + end + end + + def repo_valid? + with_fossil_env do + quiet_system "fossil", "branch", "-R", cached_location + end end private @@ -1115,11 +1055,21 @@ class FossilDownloadStrategy < VCSDownloadStrategy end def clone_repo - safe_system fossilpath, "clone", @url, cached_location + with_fossil_env do + safe_system "fossil", "clone", @url, cached_location + end end def update - safe_system fossilpath, "pull", "-R", cached_location + with_fossil_env do + safe_system "fossil", "pull", "-R", cached_location + end + end + + def with_fossil_env + with_env "PATH" => PATH.new(Formula["fossil"].opt_bin, ENV["PATH"]) do + yield + end end end diff --git a/Library/Homebrew/extend/pathname.rb b/Library/Homebrew/extend/pathname.rb index a138be3e25..3f26cc1833 100644 --- a/Library/Homebrew/extend/pathname.rb +++ b/Library/Homebrew/extend/pathname.rb @@ -263,47 +263,6 @@ class Pathname Version.parse(basename) end - # @private - def compression_type - case extname - when ".jar", ".war" - # Don't treat jars or wars as compressed - return - when ".gz" - # If the filename ends with .gz not preceded by .tar - # then we want to gunzip but not tar - return :gzip_only - when ".bz2" - return :bzip2_only - when ".lha", ".lzh" - return :lha - end - - # Get enough of the file to detect common file types - # POSIX tar magic has a 257 byte offset - # magic numbers stolen from /usr/share/file/magic/ - case open("rb") { |f| f.read(262) } - when /^PK\003\004/n then :zip - when /^\037\213/n then :gzip - when /^BZh/n then :bzip2 - when /^\037\235/n then :compress - when /^.{257}ustar/n then :tar - when /^\xFD7zXZ\x00/n then :xz - when /^LZIP/n then :lzip - when /^Rar!/n then :rar - when /^7z\xBC\xAF\x27\x1C/n then :p7zip - when /^xar!/n then :xar - when /^\xed\xab\xee\xdb/n then :rpm - else - # This code so that bad-tarballs and zips produce good error messages - # when they don't unarchive properly. - case extname - when ".tar.gz", ".tgz", ".tar.bz2", ".tbz" then :tar - when ".zip" then :zip - end - end - end - # @private def text_executable? /^#!\s*\S+/ =~ open("r") { |f| f.read(1024) } diff --git a/Library/Homebrew/formula_installer.rb b/Library/Homebrew/formula_installer.rb index 9b1e7a00b2..40c4de5e1c 100644 --- a/Library/Homebrew/formula_installer.rb +++ b/Library/Homebrew/formula_installer.rb @@ -728,6 +728,8 @@ class FormulaInstaller sandbox.allow_write_path(ENV["HOME"]) if ARGV.interactive? sandbox.allow_write_temp_and_cache sandbox.allow_write_log(formula) + sandbox.allow_cvs + sandbox.allow_fossil sandbox.allow_write_xcode sandbox.allow_write_cellar(formula) sandbox.exec(*args) diff --git a/Library/Homebrew/sandbox.rb b/Library/Homebrew/sandbox.rb index bda767970b..6ec24a8367 100644 --- a/Library/Homebrew/sandbox.rb +++ b/Library/Homebrew/sandbox.rb @@ -54,6 +54,15 @@ class Sandbox allow_write_path HOMEBREW_CACHE end + def allow_cvs + allow_write_path "/Users/#{ENV["USER"]}/.cvspass" + end + + def allow_fossil + allow_write_path "/Users/#{ENV["USER"]}/.fossil" + allow_write_path "/Users/#{ENV["USER"]}/.fossil-journal" + end + def allow_write_cellar(formula) allow_write_path formula.rack allow_write_path formula.etc diff --git a/Library/Homebrew/test/download_strategies_spec.rb b/Library/Homebrew/test/download_strategies_spec.rb index f4857787ef..6f410358ed 100644 --- a/Library/Homebrew/test/download_strategies_spec.rb +++ b/Library/Homebrew/test/download_strategies_spec.rb @@ -9,25 +9,6 @@ describe AbstractDownloadStrategy do let(:resource) { double(Resource, url: url, mirrors: [], specs: specs, version: nil) } let(:args) { %w[foo bar baz] } - describe "#expand_safe_system_args" do - it "works with an explicit quiet flag" do - args << { quiet_flag: "--flag" } - expanded_args = subject.expand_safe_system_args(args) - expect(expanded_args).to eq(%w[foo bar baz --flag]) - end - - it "adds an implicit quiet flag" do - expanded_args = subject.expand_safe_system_args(args) - expect(expanded_args).to eq(%w[foo bar -q baz]) - end - - it "does not mutate the arguments" do - result = subject.expand_safe_system_args(args) - expect(args).to eq(%w[foo bar baz]) - expect(result).not_to be args - end - end - specify "#source_modified_time" do FileUtils.mktemp "mtime" do FileUtils.touch "foo", mtime: Time.now - 10 diff --git a/Library/Homebrew/test/support/fixtures/test.jar b/Library/Homebrew/test/support/fixtures/test.jar new file mode 100644 index 0000000000000000000000000000000000000000..15a8adbd567eb307f490597a6e31fd459e00df3e GIT binary patch literal 397 zcmWIWW@h1H0D-f!ANqh9P=b>|hQZf0#8KDN&rLrxgp+}JBfn+zb|5aT;AUWC`O3(^ zz#;-v8~`)|M00?Rs5{Dc-xkQT17Z+$DojJcb$l!|cgCadIUmZ{1i>0erKJk9x`NAV=@~2y9a&eI8oX_1{pSp#-8?sDy z0-eIhB*%=)KN3JEfq($RTSpL$ PATH.new(Formula["mercurial"].opt_bin, ENV["PATH"]) do + safe_system "hg", "--cwd", path, "archive", "--subrepos", "-y", "-t", "files", unpack_dir + end + end +end + +class FossilUnpackStrategy < UnpackStrategy + def self.can_extract?(path:, magic_number:) + return false unless magic_number.match?(/\ASQLite format 3\000/n) + + # Fossil database is made up of artifacts, so the `artifact` table must exist. + query = "select count(*) from sqlite_master where type = 'view' and name = 'artifact'" + Utils.popen_read("sqlite3", path, query).to_i == 1 + end + + private + + def extract_to_dir(unpack_dir, basename:) + args = if @ref_type && @ref + [@ref] + else + [] + end + + with_env "PATH" => PATH.new(Formula["fossil"].opt_bin, ENV["PATH"]) do + safe_system "fossil", "open", path, *args, chdir: unpack_dir + end + end +end + +class BazaarUnpackStrategy < DirectoryUnpackStrategy + def self.can_extract?(path:, magic_number:) + super && (path/".bzr").directory? + end + + private + + def extract_to_dir(unpack_dir, basename:) + super + + # The export command doesn't work on checkouts (see https://bugs.launchpad.net/bzr/+bug/897511). + FileUtils.rm_r unpack_dir/".bzr" + end +end