Merge pull request #4552 from reitermarkus/reorder-unpack-strategies

Reorder unpack strategies.
This commit is contained in:
Markus Reiter 2018-08-02 05:44:12 +02:00 committed by GitHub
commit 4183dd3564
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
35 changed files with 338 additions and 132 deletions

View File

@ -47,8 +47,11 @@ class AbstractDownloadStrategy
# chdir into the newly-unpacked directory.
# Unlike {Resource#stage}, this does not take a block.
def stage
UnpackStrategy.detect(cached_location, ref_type: @ref_type, ref: @ref)
UnpackStrategy.detect(cached_location,
extension_only: true,
ref_type: @ref_type, ref: @ref)
.extract_nestedly(basename: basename_without_params,
extension_only: true,
verbose: ARGV.verbose? && !shutup)
end

View File

@ -1,36 +1,60 @@
module UnpackStrategy
# length of the longest regex (currently Tar)
MAX_MAGIC_NUMBER_LENGTH = 262
private_constant :MAX_MAGIC_NUMBER_LENGTH
module Magic
# length of the longest regex (currently Tar)
MAX_MAGIC_NUMBER_LENGTH = 262
refine Pathname do
def magic_number
@magic_number ||= if directory?
""
else
binread(MAX_MAGIC_NUMBER_LENGTH) || ""
end
end
def file_type
@file_type ||= system_command("file", args: ["-b", self], print_stderr: false)
.stdout.chomp
end
def zipinfo
@zipinfo ||= system_command("zipinfo", args: ["-1", self], print_stderr: false)
.stdout
.encode(Encoding::UTF_8, invalid: :replace)
.split("\n")
end
end
end
private_constant :Magic
def self.strategies
@strategies ||= [
Pkg,
Ttf,
Otf,
Air,
Executable,
SelfExtractingExecutable,
Jar,
LuaRock,
MicrosoftOfficeXml,
Zip,
Dmg,
Xar,
Compress,
Tar,
Bzip2,
Tar, # needs to be before Bzip2/Gzip/Xz/Lzma
Gzip,
Lzma,
Xz,
Lzip,
Air, # needs to be before Zip
Jar, # needs to be before Zip
LuaRock, # needs to be before Zip
MicrosoftOfficeXml, # needs to be before Zip
Zip,
Pkg, # needs to be before Xar
Xar,
Ttf,
Otf,
Git,
Mercurial,
Subversion,
Cvs,
SelfExtractingExecutable, # needs to be before Cab
Cab,
Executable,
Dmg, # needs to be before Bzip2
Bzip2,
Fossil,
Bazaar,
Cab,
Compress,
P7Zip,
Sit,
Rar,
@ -42,6 +66,7 @@ module UnpackStrategy
def self.from_type(type)
type = {
naked: :uncompressed,
nounzip: :uncompressed,
seven_zip: :p7zip,
}.fetch(type, type)
@ -52,32 +77,30 @@ module UnpackStrategy
end
end
def self.from_path(path)
magic_number = if path.directory?
""
else
File.binread(path, MAX_MAGIC_NUMBER_LENGTH) || ""
end
strategy = strategies.detect do |s|
s.can_extract?(path: path, magic_number: magic_number)
end
# This is so that bad files produce good error messages.
strategy ||= case path.extname
when ".tar", ".tar.gz", ".tgz", ".tar.bz2", ".tbz", ".tar.xz", ".txz"
Tar
when ".zip"
Zip
else
Uncompressed
end
strategy
def self.from_extension(extension)
strategies.sort_by { |s| s.extensions.map(&:length).max(0) }
.reverse
.detect { |s| s.extensions.any? { |ext| extension.end_with?(ext) } }
end
def self.detect(path, type: nil, ref_type: nil, ref: nil)
strategy = type ? from_type(type) : from_path(path)
def self.from_magic(path)
strategies.detect { |s| s.can_extract?(path) }
end
def self.detect(path, extension_only: false, type: nil, ref_type: nil, ref: nil)
strategy = from_type(type) if type
if extension_only
strategy ||= from_extension(path.extname)
strategy ||= strategies.select { |s| s < Directory || s == Fossil }
.detect { |s| s.can_extract?(path) }
else
strategy ||= from_magic(path)
strategy ||= from_extension(path.extname)
end
strategy ||= Uncompressed
strategy.new(path, ref_type: ref_type, ref: ref)
end
@ -96,7 +119,7 @@ module UnpackStrategy
extract_to_dir(unpack_dir, basename: basename, verbose: verbose)
end
def extract_nestedly(to: nil, basename: nil, verbose: false)
def extract_nestedly(to: nil, basename: nil, verbose: false, extension_only: false)
Dir.mktmpdir do |tmp_unpack_dir|
tmp_unpack_dir = Pathname(tmp_unpack_dir)
@ -105,9 +128,9 @@ module UnpackStrategy
children = tmp_unpack_dir.children
if children.count == 1 && !children.first.directory?
s = UnpackStrategy.detect(children.first)
s = UnpackStrategy.detect(children.first, extension_only: extension_only)
s.extract_nestedly(to: to, verbose: verbose)
s.extract_nestedly(to: to, verbose: verbose, extension_only: extension_only)
next
end

View File

@ -2,8 +2,15 @@ module UnpackStrategy
class Air
include UnpackStrategy
def self.can_extract?(path:, magic_number:)
path.extname == ".air"
using Magic
def self.extensions
[".air"]
end
def self.can_extract?(path)
mime_type = "application/vnd.adobe.air-application-installer-package+zip"
path.magic_number.match?(/.{59}#{Regexp.escape(mime_type)}/)
end
def dependencies

View File

@ -2,7 +2,9 @@ require_relative "directory"
module UnpackStrategy
class Bazaar < Directory
def self.can_extract?(path:, magic_number:)
using Magic
def self.can_extract?(path)
super && (path/".bzr").directory?
end

View File

@ -2,8 +2,14 @@ module UnpackStrategy
class Bzip2
include UnpackStrategy
def self.can_extract?(path:, magic_number:)
magic_number.match?(/\ABZh/n)
using Magic
def self.extensions
[".bz2"]
end
def self.can_extract?(path)
path.magic_number.match?(/\ABZh/n)
end
private

View File

@ -2,8 +2,14 @@ module UnpackStrategy
class Cab
include UnpackStrategy
def self.can_extract?(path:, magic_number:)
magic_number.match?(/\AMSCF/n)
using Magic
def self.extensions
[".cab"]
end
def self.can_extract?(path)
path.magic_number.match?(/\AMSCF/n)
end
def extract_to_dir(unpack_dir, basename:, verbose:)

View File

@ -2,8 +2,14 @@ require_relative "tar"
module UnpackStrategy
class Compress < Tar
def self.can_extract?(path:, magic_number:)
magic_number.match?(/\A\037\235/n)
using Magic
def self.extensions
[".compress"]
end
def self.can_extract?(path)
path.magic_number.match?(/\A\037\235/n)
end
end
end

View File

@ -2,7 +2,9 @@ require_relative "directory"
module UnpackStrategy
class Cvs < Directory
def self.can_extract?(path:, magic_number:)
using Magic
def self.can_extract?(path)
super && (path/"CVS").directory?
end
end

View File

@ -2,7 +2,13 @@ module UnpackStrategy
class Directory
include UnpackStrategy
def self.can_extract?(path:, magic_number:)
using Magic
def self.extensions
[]
end
def self.can_extract?(path)
path.directory?
end

View File

@ -4,6 +4,8 @@ module UnpackStrategy
class Dmg
include UnpackStrategy
using Magic
module Bom
DMG_METADATA = Set.new %w[
.background
@ -83,12 +85,18 @@ module UnpackStrategy
end
system_command! "ditto", args: ["--bom", bomfile.path, "--", path, unpack_dir]
FileUtils.chmod "u+w", Pathname.glob(unpack_dir/"**/*").reject(&:symlink?)
end
end
end
private_constant :Mount
def self.can_extract?(path:, magic_number:)
def self.extensions
[".dmg"]
end
def self.can_extract?(path)
imageinfo = system_command("hdiutil",
args: ["imageinfo", path],
print_stderr: false).stdout

View File

@ -2,8 +2,15 @@ require_relative "uncompressed"
module UnpackStrategy
class Executable < Uncompressed
def self.can_extract?(path:, magic_number:)
magic_number.match?(/\A#!\s*\S+/n)
using Magic
def self.extensions
[".sh", ".bash"]
end
def self.can_extract?(path)
path.magic_number.match?(/\A#!\s*\S+/n) ||
path.magic_number.match?(/\AMZ/n)
end
end
end

View File

@ -2,8 +2,14 @@ module UnpackStrategy
class Fossil
include UnpackStrategy
def self.can_extract?(path:, magic_number:)
return false unless magic_number.match?(/\ASQLite format 3\000/n)
using Magic
def self.extensions
[]
end
def self.can_extract?(path)
return false unless path.magic_number.match?(/\ASQLite format 3\000/n)
# Fossil database is made up of artifacts, so the `artifact` table must exist.
query = "select count(*) from sqlite_master where type = 'view' and name = 'artifact'"

View File

@ -2,7 +2,13 @@ module UnpackStrategy
class GenericUnar
include UnpackStrategy
def self.can_extract?(path:, magic_number:)
using Magic
def self.extensions
[]
end
def self.can_extract?(_path)
false
end

View File

@ -2,7 +2,9 @@ require_relative "directory"
module UnpackStrategy
class Git < Directory
def self.can_extract?(path:, magic_number:)
using Magic
def self.can_extract?(path)
super && (path/".git").directory?
end
end

View File

@ -2,8 +2,14 @@ module UnpackStrategy
class Gzip
include UnpackStrategy
def self.can_extract?(path:, magic_number:)
magic_number.match?(/\A\037\213/n)
using Magic
def self.extensions
[".gz"]
end
def self.can_extract?(path)
path.magic_number.match?(/\A\037\213/n)
end
private

View File

@ -2,14 +2,17 @@ require_relative "uncompressed"
module UnpackStrategy
class Jar < Uncompressed
def self.can_extract?(path:, magic_number:)
return false unless Zip.can_extract?(path: path, magic_number: magic_number)
using Magic
def self.extensions
[".apk", ".jar"]
end
def self.can_extract?(path)
return false unless Zip.can_extract?(path)
# Check further if the ZIP is a JAR/WAR.
out, = Open3.capture3("zipinfo", "-1", path)
out.encode(Encoding::UTF_8, invalid: :replace)
.split("\n")
.include?("META-INF/MANIFEST.MF")
path.zipinfo.include?("META-INF/MANIFEST.MF")
end
end
end

View File

@ -2,8 +2,14 @@ module UnpackStrategy
class Lha
include UnpackStrategy
def self.can_extract?(path:, magic_number:)
magic_number.match?(/\A..-(lh0|lh1|lz4|lz5|lzs|lh\\40|lhd|lh2|lh3|lh4|lh5)-/n)
using Magic
def self.extensions
[".lha", ".lzh"]
end
def self.can_extract?(path)
path.magic_number.match?(/\A..-(lh0|lh1|lz4|lz5|lzs|lh\\40|lhd|lh2|lh3|lh4|lh5)-/n)
end
def dependencies

View File

@ -2,14 +2,17 @@ require_relative "uncompressed"
module UnpackStrategy
class LuaRock < Uncompressed
def self.can_extract?(path:, magic_number:)
return false unless Zip.can_extract?(path: path, magic_number: magic_number)
using Magic
def self.extensions
[".rock"]
end
def self.can_extract?(path)
return false unless Zip.can_extract?(path)
# Check further if the ZIP is a LuaRocks package.
out, = Open3.capture3("zipinfo", "-1", path)
out.encode(Encoding::UTF_8, invalid: :replace)
.split("\n")
.any? { |line| line.match?(%r{\A[^/]+.rockspec\Z}) }
path.zipinfo.grep(%r{\A[^/]+.rockspec\Z}).any?
end
end
end

View File

@ -2,8 +2,14 @@ module UnpackStrategy
class Lzip
include UnpackStrategy
def self.can_extract?(path:, magic_number:)
magic_number.match?(/\ALZIP/n)
using Magic
def self.extensions
[".lz"]
end
def self.can_extract?(path)
path.magic_number.match?(/\ALZIP/n)
end
def dependencies

View File

@ -2,8 +2,14 @@ module UnpackStrategy
class Lzma
include UnpackStrategy
def self.can_extract?(path:, magic_number:)
magic_number.match?(/\A\]\000\000\200\000/n)
using Magic
def self.extensions
[".lzma"]
end
def self.can_extract?(path)
path.magic_number.match?(/\A\]\000\000\200\000/n)
end
def extract_to_dir(unpack_dir, basename:, verbose:)

View File

@ -2,7 +2,9 @@ require_relative "directory"
module UnpackStrategy
class Mercurial < Directory
def self.can_extract?(path:, magic_number:)
using Magic
def self.can_extract?(path)
super && (path/".hg").directory?
end

View File

@ -2,12 +2,22 @@ require_relative "uncompressed"
module UnpackStrategy
class MicrosoftOfficeXml < Uncompressed
def self.can_extract?(path:, magic_number:)
return false unless Zip.can_extract?(path: path, magic_number: magic_number)
using Magic
def self.extensions
[
".doc", ".docx",
".ppt", ".pptx",
".xls", ".xlsx"
]
end
def self.can_extract?(path)
return false unless Zip.can_extract?(path)
# Check further if the ZIP is a Microsoft Office XML document.
magic_number.match?(/\APK\003\004/n) &&
magic_number.match?(%r{\A.{30}(\[Content_Types\]\.xml|_rels/\.rels)}n)
path.magic_number.match?(/\APK\003\004/n) &&
path.magic_number.match?(%r{\A.{30}(\[Content_Types\]\.xml|_rels/\.rels)}n)
end
end
end

View File

@ -2,8 +2,14 @@ require_relative "uncompressed"
module UnpackStrategy
class Otf < Uncompressed
def self.can_extract?(path:, magic_number:)
magic_number.match?(/\AOTTO/n)
using Magic
def self.extensions
[".otf"]
end
def self.can_extract?(path)
path.magic_number.match?(/\AOTTO/n)
end
end
end

View File

@ -2,8 +2,14 @@ module UnpackStrategy
class P7Zip
include UnpackStrategy
def self.can_extract?(path:, magic_number:)
magic_number.match?(/\A7z\xBC\xAF\x27\x1C/n)
using Magic
def self.extensions
[".7z"]
end
def self.can_extract?(path)
path.magic_number.match?(/\A7z\xBC\xAF\x27\x1C/n)
end
def dependencies

View File

@ -2,9 +2,15 @@ require_relative "uncompressed"
module UnpackStrategy
class Pkg < Uncompressed
def self.can_extract?(path:, magic_number:)
using Magic
def self.extensions
[".pkg", ".mkpg"]
end
def self.can_extract?(path)
path.extname.match?(/\A.m?pkg\Z/) &&
(path.directory? || magic_number.match?(/\Axar!/n))
(path.directory? || path.magic_number.match?(/\Axar!/n))
end
end
end

View File

@ -2,8 +2,14 @@ module UnpackStrategy
class Rar
include UnpackStrategy
def self.can_extract?(path:, magic_number:)
magic_number.match?(/\ARar!/n)
using Magic
def self.extensions
[".rar"]
end
def self.can_extract?(path)
path.magic_number.match?(/\ARar!/n)
end
def dependencies

View File

@ -2,12 +2,15 @@ require_relative "generic_unar"
module UnpackStrategy
class SelfExtractingExecutable < GenericUnar
def self.can_extract?(path:, magic_number:)
return false unless magic_number.match?(/\AMZ/n)
using Magic
system_command("file",
args: [path],
print_stderr: false).stdout.include?("self-extracting")
def self.extensions
[]
end
def self.can_extract?(path)
path.magic_number.match?(/\AMZ/n) &&
path.file_type.include?("self-extracting archive")
end
end
end

View File

@ -2,8 +2,14 @@ require_relative "generic_unar"
module UnpackStrategy
class Sit < GenericUnar
def self.can_extract?(path:, magic_number:)
magic_number.match?(/\AStuffIt/n)
using Magic
def self.extensions
[".sit"]
end
def self.can_extract?(path)
path.magic_number.match?(/\AStuffIt/n)
end
end
end

View File

@ -2,7 +2,9 @@ require_relative "directory"
module UnpackStrategy
class Subversion < Directory
def self.can_extract?(path:, magic_number:)
using Magic
def self.can_extract?(path)
super && (path/".svn").directory?
end

View File

@ -2,8 +2,24 @@ module UnpackStrategy
class Tar
include UnpackStrategy
def self.can_extract?(path:, magic_number:)
return true if magic_number.match?(/\A.{257}ustar/n)
using Magic
def self.extensions
[
".tar",
".tbz", ".tbz2", ".tar.bz2",
".tgz", ".tar.gz",
".tlz", ".tar.lz",
".txz", ".tar.xz"
]
end
def self.can_extract?(path)
return true if path.magic_number.match?(/\A.{257}ustar/n)
unless [Bzip2, Gzip, Lzip, Xz].any? { |s| s.can_extract?(path) }
return false
end
# Check if `tar` can list the contents, then it can also extract it.
IO.popen(["tar", "tf", path], err: File::NULL) do |stdout|
@ -14,7 +30,17 @@ module UnpackStrategy
private
def extract_to_dir(unpack_dir, basename:, verbose:)
system_command! "tar", args: ["xf", path, "-C", unpack_dir]
Dir.mktmpdir do |tmpdir|
tar_path = path
if DependencyCollector.tar_needs_xz_dependency? && Xz.can_extract?(path)
tmpdir = Pathname(tmpdir)
Xz.new(path).extract(to: tmpdir)
tar_path = tmpdir.children.first
end
system_command! "tar", args: ["xf", tar_path, "-C", unpack_dir]
end
end
end
end

View File

@ -2,11 +2,17 @@ require_relative "uncompressed"
module UnpackStrategy
class Ttf < Uncompressed
def self.can_extract?(path:, magic_number:)
using Magic
def self.extensions
[".ttc", ".ttf"]
end
def self.can_extract?(path)
# TrueType Font
magic_number.match?(/\A\000\001\000\000\000/n) ||
path.magic_number.match?(/\A\000\001\000\000\000/n) ||
# Truetype Font Collection
magic_number.match?(/\Attcf/n)
path.magic_number.match?(/\Attcf/n)
end
end
end

View File

@ -2,7 +2,9 @@ module UnpackStrategy
class Uncompressed
include UnpackStrategy
alias extract_nestedly extract
def extract_nestedly(extension_only: false, **options)
extract(**options)
end
private

View File

@ -2,8 +2,14 @@ module UnpackStrategy
class Xar
include UnpackStrategy
def self.can_extract?(path:, magic_number:)
magic_number.match?(/\Axar!/n)
using Magic
def self.extensions
[".xar"]
end
def self.can_extract?(path)
path.magic_number.match?(/\Axar!/n)
end
private

View File

@ -2,8 +2,14 @@ module UnpackStrategy
class Xz
include UnpackStrategy
def self.can_extract?(path:, magic_number:)
magic_number.match?(/\A\xFD7zXZ\x00/n)
using Magic
def self.extensions
[".xz"]
end
def self.can_extract?(path)
path.magic_number.match?(/\A\xFD7zXZ\x00/n)
end
def dependencies
@ -18,19 +24,6 @@ module UnpackStrategy
system_command! "unxz",
args: [*quiet_flags, "-T0", "--", unpack_dir/basename],
env: { "PATH" => PATH.new(Formula["xz"].opt_bin, ENV["PATH"]) }
extract_nested_tar(unpack_dir)
end
def extract_nested_tar(unpack_dir)
return unless DependencyCollector.tar_needs_xz_dependency?
return if (children = unpack_dir.children).count != 1
return if (tar = children.first).extname != ".tar"
Dir.mktmpdir do |tmpdir|
tmpdir = Pathname(tmpdir)
FileUtils.mv tar, tmpdir/tar.basename
Tar.new(tmpdir/tar.basename).extract(to: unpack_dir)
end
end
end
end

View File

@ -2,8 +2,14 @@ module UnpackStrategy
class Zip
include UnpackStrategy
def self.can_extract?(path:, magic_number:)
magic_number.match?(/\APK(\003\004|\005\006)/n)
using Magic
def self.extensions
[".zip"]
end
def self.can_extract?(path)
path.magic_number.match?(/\APK(\003\004|\005\006)/n)
end
private