brew/Library/Homebrew/download_strategy.rb
Adam Vandenberg 13c604035b Retry downloads if a server doesn't support byte ranges
If an incomplete download exists, and the HTTP server doesn't support
byte ranges, delete the incomplete download and retry once.

Tested with freeling, whose server does not support byte ranges.

Closes Homebrew/homebrew#19757.
2013-05-13 07:32:03 -07:00

733 lines
20 KiB
Ruby

require 'open-uri'
require 'vendor/multi_json'
class AbstractDownloadStrategy
def initialize name, package
@url = package.url
specs = package.specs
@spec, @ref = specs.dup.shift unless specs.empty?
end
def expand_safe_system_args args
args = args.dup
args.each_with_index do |arg, ii|
if arg.is_a? Hash
unless ARGV.verbose?
args[ii] = arg[:quiet_flag]
else
args.delete_at ii
end
return args
end
end
# 2 as default because commands are eg. svn up, git pull
args.insert(2, '-q') unless ARGV.verbose?
args
end
def quiet_safe_system *args
safe_system(*expand_safe_system_args(args))
end
end
class CurlDownloadStrategy < AbstractDownloadStrategy
attr_reader :tarball_path
attr_accessor :local_bottle_path
def initialize name, package
super
if name.to_s.empty? || name == '__UNKNOWN__'
@tarball_path = Pathname.new("#{HOMEBREW_CACHE}/#{File.basename(@url)}")
else
@tarball_path = Pathname.new("#{HOMEBREW_CACHE}/#{name}-#{package.version}#{ext}")
end
@mirrors = package.mirrors
@temporary_path = Pathname.new("#@tarball_path.incomplete")
@local_bottle_path = nil
end
def cached_location
@tarball_path
end
def downloaded_size
@temporary_path.size? or 0
end
# Private method, can be overridden if needed.
def _fetch
curl @url, '-C', downloaded_size, '-o', @temporary_path
end
def fetch
if @local_bottle_path
@tarball_path = @local_bottle_path
return @local_bottle_path
end
ohai "Downloading #{@url}"
unless @tarball_path.exist?
had_incomplete_download = @temporary_path.exist?
begin
_fetch
rescue ErrorDuringExecution
# 33 == range not supported
# try wiping the incomplete download and retrying once
if $?.exitstatus == 33 && had_incomplete_download
ohai "Trying a full download"
@temporary_path.unlink
had_incomplete_download = false
retry
else
raise CurlDownloadStrategyError, "Download failed: #{@url}"
end
end
ignore_interrupts { @temporary_path.rename(@tarball_path) }
else
puts "Already downloaded: #{@tarball_path}"
end
rescue CurlDownloadStrategyError
raise if @mirrors.empty?
puts "Trying a mirror..."
@url = @mirrors.shift
retry
else
@tarball_path
end
def stage
ohai "Pouring #{File.basename(@tarball_path)}" if @tarball_path.to_s.match bottle_regex
case @tarball_path.compression_type
when :zip
with_system_path { quiet_safe_system 'unzip', {:quiet_flag => '-qq'}, @tarball_path }
chdir
when :gzip, :bzip2, :compress, :tar
# Assume these are also tarred
# TODO check if it's really a tar archive
with_system_path { safe_system 'tar', 'xf', @tarball_path }
chdir
when :xz
raise "You must install XZutils: brew install xz" unless which "xz"
safe_system "xz -dc \"#{@tarball_path}\" | /usr/bin/tar xf -"
chdir
when :pkg
safe_system '/usr/sbin/pkgutil', '--expand', @tarball_path, File.basename(@url)
chdir
when :rar
raise "You must install unrar: brew install unrar" unless which "unrar"
quiet_safe_system 'unrar', 'x', {:quiet_flag => '-inul'}, @tarball_path
when :p7zip
raise "You must install 7zip: brew install p7zip" unless which "7zr"
safe_system '7zr', 'x', @tarball_path
else
# we are assuming it is not an archive, use original filename
# this behaviour is due to ScriptFileFormula expectations
# So I guess we should cp, but we mv, for this historic reason
# HOWEVER if this breaks some expectation you had we *will* change the
# behaviour, just open an issue at github
# We also do this for jar files, as they are in fact zip files, but
# we don't want to unzip them
FileUtils.cp @tarball_path, File.basename(@url)
end
end
private
def chdir
entries=Dir['*']
case entries.length
when 0 then raise "Empty archive"
when 1 then Dir.chdir entries.first rescue nil
end
end
def ext
# GitHub uses odd URLs for zip files, so check for those
rx=%r[https?://(www\.)?github\.com/.*/(zip|tar)ball/]
if rx.match @url
if $2 == 'zip'
'.zip'
else
'.tgz'
end
else
Pathname.new(@url).extname
end
end
end
# Detect and download from Apache Mirror
class CurlApacheMirrorDownloadStrategy < CurlDownloadStrategy
def _fetch
mirrors = MultiJson.decode(open("#{@url}&asjson=1").read)
url = mirrors.fetch('preferred') + mirrors.fetch('path_info')
ohai "Best Mirror #{url}"
curl url, '-C', downloaded_size, '-o', @temporary_path
rescue IndexError, MultiJson::DecodeError
raise "Couldn't determine mirror. Try again later."
end
end
# Download via an HTTP POST.
# Query parameters on the URL are converted into POST parameters
class CurlPostDownloadStrategy < CurlDownloadStrategy
def _fetch
base_url,data = @url.split('?')
curl base_url, '-d', data, '-C', downloaded_size, '-o', @temporary_path
end
end
# Use this strategy to download but not unzip a file.
# Useful for installing jars.
class NoUnzipCurlDownloadStrategy < CurlDownloadStrategy
def stage
FileUtils.cp @tarball_path, File.basename(@url)
end
end
# Normal strategy tries to untar as well
class GzipOnlyDownloadStrategy < CurlDownloadStrategy
def stage
FileUtils.mv @tarball_path, File.basename(@url)
with_system_path { safe_system 'gunzip', '-f', File.basename(@url) }
end
end
# This Download Strategy is provided for use with sites that
# only provide HTTPS and also have a broken cert.
# Try not to need this, as we probably won't accept the formula.
class CurlUnsafeDownloadStrategy < CurlDownloadStrategy
def _fetch
curl @url, '--insecure', '-C', downloaded_size, '-o', @temporary_path
end
end
# This strategy extracts our binary packages.
class CurlBottleDownloadStrategy < CurlDownloadStrategy
def initialize name, package
super
@tarball_path = HOMEBREW_CACHE/"#{name}-#{package.version}#{ext}"
mirror = ENV['HOMEBREW_SOURCEFORGE_MIRROR']
@url = "#{@url}?use_mirror=#{mirror}" if mirror
end
end
class SubversionDownloadStrategy < AbstractDownloadStrategy
def initialize name, package
super
@@svn ||= 'svn'
if name.to_s.empty? || name == '__UNKNOWN__'
raise NotImplementedError, "strategy requires a name parameter"
else
@co = Pathname.new("#{HOMEBREW_CACHE}/#{name}--svn")
end
@co = Pathname.new(@co.to_s + '-HEAD') if ARGV.build_head?
end
def cached_location
@co
end
def fetch
@url.sub!(/^svn\+/, '') if @url =~ %r[^svn\+http://]
ohai "Checking out #{@url}"
if @spec == :revision
fetch_repo @co, @url, @ref
elsif @spec == :revisions
# nil is OK for main_revision, as fetch_repo will then get latest
main_revision = @ref.delete :trunk
fetch_repo @co, @url, main_revision, true
get_externals do |external_name, external_url|
fetch_repo @co+external_name, external_url, @ref[external_name], true
end
else
fetch_repo @co, @url
end
end
def stage
quiet_safe_system @@svn, 'export', '--force', @co, Dir.pwd
end
def shell_quote str
# Oh god escaping shell args.
# See http://notetoself.vrensk.com/2008/08/escaping-single-quotes-in-ruby-harder-than-expected/
str.gsub(/\\|'/) { |c| "\\#{c}" }
end
def get_externals
`'#{shell_quote(svn)}' propget svn:externals '#{shell_quote(@url)}'`.chomp.each_line do |line|
name, url = line.split(/\s+/)
yield name, url
end
end
def fetch_repo target, url, revision=nil, ignore_externals=false
# Use "svn up" when the repository already exists locally.
# This saves on bandwidth and will have a similar effect to verifying the
# cache as it will make any changes to get the right revision.
svncommand = target.exist? ? 'up' : 'checkout'
args = [@@svn, svncommand]
# SVN shipped with XCode 3.1.4 can't force a checkout.
args << '--force' unless MacOS.version == :leopard and @@svn == '/usr/bin/svn'
args << url if !target.exist?
args << target
args << '-r' << revision if revision
args << '--ignore-externals' if ignore_externals
quiet_safe_system(*args)
end
end
# Require a newer version of Subversion than 1.4.x (Leopard-provided version)
class StrictSubversionDownloadStrategy < SubversionDownloadStrategy
def find_svn
exe = `svn -print-path`
`#{exe} --version` =~ /version (\d+\.\d+(\.\d+)*)/
svn_version = $1
version_tuple=svn_version.split(".").collect {|v|Integer(v)}
if version_tuple[0] == 1 and version_tuple[1] <= 4
onoe "Detected Subversion (#{exe}, version #{svn_version}) is too old."
puts "Subversion 1.4.x will not export externals correctly for this formula."
puts "You must either `brew install subversion` or set HOMEBREW_SVN to the path"
puts "of a newer svn binary."
end
return exe
end
end
# Download from SVN servers with invalid or self-signed certs
class UnsafeSubversionDownloadStrategy < SubversionDownloadStrategy
def fetch_repo target, url, revision=nil, ignore_externals=false
# Use "svn up" when the repository already exists locally.
# This saves on bandwidth and will have a similar effect to verifying the
# cache as it will make any changes to get the right revision.
svncommand = target.exist? ? 'up' : 'checkout'
args = [@@svn, svncommand, '--non-interactive', '--trust-server-cert', '--force']
args << url if !target.exist?
args << target
args << '-r' << revision if revision
args << '--ignore-externals' if ignore_externals
quiet_safe_system(*args)
end
end
class GitDownloadStrategy < AbstractDownloadStrategy
def initialize name, package
super
@@git ||= 'git'
if name.to_s.empty? || name == '__UNKNOWN__'
raise NotImplementedError, "strategy requires a name parameter"
else
@clone = Pathname.new("#{HOMEBREW_CACHE}/#{name}--git")
end
end
def cached_location
@clone
end
def fetch
raise "You must: brew install git" unless which "git"
ohai "Cloning #@url"
if @clone.exist? && repo_valid?
puts "Updating #@clone"
Dir.chdir(@clone) do
config_repo
update_repo
checkout
reset
update_submodules if submodules?
end
elsif @clone.exist?
puts "Removing invalid .git repo from cache"
FileUtils.rm_rf @clone
clone_repo
else
clone_repo
end
end
def stage
dst = Dir.getwd
Dir.chdir @clone do
if @spec and @ref
ohai "Checking out #@spec #@ref"
else
reset
end
# http://stackoverflow.com/questions/160608/how-to-do-a-git-export-like-svn-export
safe_system @@git, 'checkout-index', '-a', '-f', "--prefix=#{dst}/"
checkout_submodules(dst) if submodules?
end
end
private
def git_dir
@clone.join(".git")
end
def has_ref?
quiet_system @@git, '--git-dir', git_dir, 'rev-parse', '-q', '--verify', @ref
end
def support_depth?
@spec != :revision and host_supports_depth?
end
def host_supports_depth?
@url =~ %r{git://} or @url =~ %r{https://github.com/}
end
def repo_valid?
quiet_system @@git, "--git-dir", git_dir, "status", "-s"
end
def submodules?
@clone.join(".gitmodules").exist?
end
def clone_args
args = %w{clone}
args << '--depth' << '1' if support_depth?
case @spec
when :branch, :tag then args << '--branch' << @ref
end
args << @url << @clone
end
def refspec
case @spec
when :branch then "+refs/heads/#@ref:refs/remotes/origin/#@ref"
when :tag then "+refs/tags/#@ref:refs/tags/#@ref"
else "+refs/heads/master:refs/remotes/origin/master"
end
end
def config_repo
safe_system @@git, 'config', 'remote.origin.url', @url
safe_system @@git, 'config', 'remote.origin.fetch', refspec
end
def update_repo
unless @spec == :tag && has_ref?
quiet_safe_system @@git, 'fetch', 'origin'
end
end
def clone_repo
safe_system @@git, *clone_args
@clone.cd { update_submodules } if submodules?
end
def checkout_args
ref = case @spec
when :branch, :tag, :revision then @ref
else `git symbolic-ref refs/remotes/origin/HEAD`.strip.split("/").last
end
args = %w{checkout -f}
args << { :quiet_flag => '-q' }
args << ref
end
def checkout
nostdout { quiet_safe_system @@git, *checkout_args }
end
def reset_args
ref = case @spec
when :branch then "origin/#@ref"
when :revision, :tag then @ref
else "origin/HEAD"
end
args = %w{reset}
args << { :quiet_flag => "-q" }
args << "--hard" << ref
end
def reset
quiet_safe_system @@git, *reset_args
end
def update_submodules
safe_system @@git, 'submodule', 'update', '--init'
end
def checkout_submodules(dst)
sub_cmd = %W{#@@git checkout-index -a -f --prefix=#{dst}/$path/}
safe_system @@git, 'submodule', '--quiet', 'foreach', '--recursive', *sub_cmd
end
end
class CVSDownloadStrategy < AbstractDownloadStrategy
def initialize name, package
super
if name.to_s.empty? || name == '__UNKNOWN__'
raise NotImplementedError, "strategy requires a name parameter"
else
@unique_token = "#{name}--cvs"
@co = Pathname.new("#{HOMEBREW_CACHE}/#{@unique_token}")
end
end
def cached_location; @co; end
def fetch
ohai "Checking out #{@url}"
# URL of cvs cvs://:pserver:anoncvs@www.gccxml.org:/cvsroot/GCC_XML:gccxml
# will become:
# cvs -d :pserver:anoncvs@www.gccxml.org:/cvsroot/GCC_XML login
# cvs -d :pserver:anoncvs@www.gccxml.org:/cvsroot/GCC_XML co gccxml
mod, url = split_url(@url)
unless @co.exist?
Dir.chdir HOMEBREW_CACHE do
safe_system '/usr/bin/cvs', '-d', url, 'login'
safe_system '/usr/bin/cvs', '-d', url, 'checkout', '-d', @unique_token, mod
end
else
puts "Updating #{@co}"
Dir.chdir(@co) { safe_system '/usr/bin/cvs', 'up' }
end
end
def stage
FileUtils.cp_r Dir[@co+"{.}"], Dir.pwd
require 'find'
Find.find(Dir.pwd) do |path|
if FileTest.directory?(path) && File.basename(path) == "CVS"
Find.prune
FileUtil.rm_r path, :force => true
end
end
end
private
def split_url(in_url)
parts=in_url.sub(%r[^cvs://], '').split(/:/)
mod=parts.pop
url=parts.join(':')
[ mod, url ]
end
end
class MercurialDownloadStrategy < AbstractDownloadStrategy
def initialize name, package
super
if name.to_s.empty? || name == '__UNKNOWN__'
raise NotImplementedError, "strategy requires a name parameter"
else
@clone = Pathname.new("#{HOMEBREW_CACHE}/#{name}--hg")
end
end
def cached_location; @clone; end
def hgpath
@path ||= %W[
#{which("hg")}
#{HOMEBREW_PREFIX}/bin/hg
#{HOMEBREW_PREFIX}/share/python/hg
].find { |p| File.executable? p }
end
def fetch
raise "You must: brew install mercurial" unless hgpath
ohai "Cloning #{@url}"
unless @clone.exist?
url=@url.sub(%r[^hg://], '')
safe_system hgpath, 'clone', url, @clone
else
puts "Updating #{@clone}"
Dir.chdir(@clone) do
safe_system hgpath, 'pull'
safe_system hgpath, 'update'
end
end
end
def stage
dst=Dir.getwd
Dir.chdir @clone do
if @spec and @ref
ohai "Checking out #{@spec} #{@ref}"
safe_system hgpath, 'archive', '--subrepos', '-y', '-r', @ref, '-t', 'files', dst
else
safe_system hgpath, 'archive', '--subrepos', '-y', '-t', 'files', dst
end
end
end
end
class BazaarDownloadStrategy < AbstractDownloadStrategy
def initialize name, package
super
if name.to_s.empty? || name == '__UNKNOWN__'
raise NotImplementedError, "strategy requires a name parameter"
else
@clone = Pathname.new("#{HOMEBREW_CACHE}/#{name}--bzr")
end
end
def cached_location; @clone; end
def bzrpath
@path ||= %W[
#{which("bzr")}
#{HOMEBREW_PREFIX}/bin/bzr
].find { |p| File.executable? p }
end
def fetch
raise "You must: brew install bazaar" unless bzrpath
ohai "Cloning #{@url}"
unless @clone.exist?
url=@url.sub(%r[^bzr://], '')
# 'lightweight' means history-less
safe_system bzrpath, 'checkout', '--lightweight', url, @clone
else
puts "Updating #{@clone}"
Dir.chdir(@clone) { safe_system bzrpath, 'update' }
end
end
def stage
# FIXME: The export command doesn't work on checkouts
# See https://bugs.launchpad.net/bzr/+bug/897511
FileUtils.cp_r Dir[@clone+"{.}"], Dir.pwd
FileUtils.rm_r Dir[Dir.pwd+"/.bzr"]
#dst=Dir.getwd
#Dir.chdir @clone do
# if @spec and @ref
# ohai "Checking out #{@spec} #{@ref}"
# Dir.chdir @clone do
# safe_system bzrpath, 'export', '-r', @ref, dst
# end
# else
# safe_system bzrpath, 'export', dst
# end
#end
end
end
class FossilDownloadStrategy < AbstractDownloadStrategy
def initialize name, package
super
if name.to_s.empty? || name == '__UNKNOWN__'
raise NotImplementedError, "strategy requires a name parameter"
else
@clone = Pathname.new("#{HOMEBREW_CACHE}/#{name}--fossil")
end
end
def cached_location; @clone; end
def fossilpath
@path ||= %W[
#{which("fossil")}
#{HOMEBREW_PREFIX}/bin/fossil
].find { |p| File.executable? p }
end
def fetch
raise "You must: brew install fossil" unless fossilpath
ohai "Cloning #{@url}"
unless @clone.exist?
url=@url.sub(%r[^fossil://], '')
safe_system fossilpath, 'clone', url, @clone
else
puts "Updating #{@clone}"
safe_system fossilpath, 'pull', '-R', @clone
end
end
def stage
# TODO: The 'open' and 'checkout' commands are very noisy and have no '-q' option.
safe_system fossilpath, 'open', @clone
if @spec and @ref
ohai "Checking out #{@spec} #{@ref}"
safe_system fossilpath, 'checkout', @ref
end
end
end
class DownloadStrategyDetector
def self.detect(url, strategy=nil)
if strategy.is_a? Class and strategy.ancestors.include? AbstractDownloadStrategy
strategy
elsif strategy.is_a? Symbol
detect_from_symbol(strategy)
else
detect_from_url(url)
end
end
def self.detect_from_url(url)
case url
# We use a special URL pattern for cvs
when %r[^cvs://] then CVSDownloadStrategy
# Standard URLs
when %r[^bzr://] then BazaarDownloadStrategy
when %r[^git://] then GitDownloadStrategy
when %r[^https?://.+\.git$] then GitDownloadStrategy
when %r[^hg://] then MercurialDownloadStrategy
when %r[^svn://] then SubversionDownloadStrategy
when %r[^svn\+http://] then SubversionDownloadStrategy
when %r[^fossil://] then FossilDownloadStrategy
# Some well-known source hosts
when %r[^https?://(.+?\.)?googlecode\.com/hg] then MercurialDownloadStrategy
when %r[^https?://(.+?\.)?googlecode\.com/svn] then SubversionDownloadStrategy
when %r[^https?://(.+?\.)?sourceforge\.net/svnroot/] then SubversionDownloadStrategy
when %r[^http://svn.apache.org/repos/] then SubversionDownloadStrategy
when %r[^http://www.apache.org/dyn/closer.cgi] then CurlApacheMirrorDownloadStrategy
# Common URL patterns
when %r[^https?://svn\.] then SubversionDownloadStrategy
when bottle_native_regex, bottle_regex
CurlBottleDownloadStrategy
# Otherwise just try to download
else CurlDownloadStrategy
end
end
def self.detect_from_symbol(symbol)
case symbol
when :bzr then BazaarDownloadStrategy
when :curl then CurlDownloadStrategy
when :cvs then CVSDownloadStrategy
when :git then GitDownloadStrategy
when :hg then MercurialDownloadStrategy
when :nounzip then NoUnzipCurlDownloadStrategy
when :post then CurlPostDownloadStrategy
when :svn then SubversionDownloadStrategy
else
raise "Unknown download strategy #{strategy} was requested."
end
end
end