Merge pull request #14035 from hmarr/git-partial-clone-sparse-checkout

Support git partial clones with sparse checkouts (take 2)

Library/Homebrew/cask/artifact/relocated.rb

@@ -51,7 +51,9 @@ module Cask
         target = target_hash[:target]
         @source_string = source.to_s
         @target_string = target.to_s
-        source = cask.staged_path.join(source)
+        base_path = cask.staged_path
+        base_path = base_path.join(cask.url.only_path) if cask.url&.only_path.present?
+        source = base_path.join(source)
         @source = source
         target ||= source.basename
         @target = resolve_target(target)

Library/Homebrew/cask/url.rb

@@ -15,7 +15,7 @@ class URL < Delegator
                 :verified, :using,
                 :tag, :branch, :revisions, :revision,
                 :trust_cert, :cookies, :referer, :header, :user_agent,
-                :data
+                :data, :only_path
 
     extend Forwardable
     def_delegators :uri, :path, :scheme, :to_s
@@ -36,6 +36,7 @@ class URL < Delegator
         header:     T.nilable(String),
         user_agent: T.nilable(T.any(Symbol, String)),
         data:       T.nilable(T::Hash[String, String]),
+        only_path:  T.nilable(String),
       ).void
     }
     def initialize(
@@ -51,7 +52,8 @@ class URL < Delegator
       referer: nil,
       header: nil,
       user_agent: nil,
-      data: nil
+      data: nil,
+      only_path: nil
     )
 
       @uri = URI(uri)
@@ -69,6 +71,7 @@ class URL < Delegator
       specs[:header]     = @header     = header
       specs[:user_agent] = @user_agent = user_agent || :default
       specs[:data]       = @data       = data
+      specs[:only_path]  = @only_path  = only_path
 
       @specs = specs.compact
     end
@@ -156,6 +159,7 @@ class URL < Delegator
       header:          T.nilable(String),
       user_agent:      T.nilable(T.any(Symbol, String)),
       data:            T.nilable(T::Hash[String, String]),
+      only_path:       T.nilable(String),
       caller_location: Thread::Backtrace::Location,
       dsl:             T.nilable(Cask::DSL),
       block:           T.nilable(T.proc.params(arg0: T.all(String, BlockDSL::PageWithURL)).returns(T.untyped)),
@@ -175,6 +179,7 @@ class URL < Delegator
     header: nil,
     user_agent: nil,
     data: nil,
+    only_path: nil,
     caller_location: T.must(caller_locations).fetch(0),
     dsl: nil,
     &block
@@ -202,6 +207,7 @@ class URL < Delegator
         header:     header,
         user_agent: user_agent,
         data:       data,
+        only_path:  only_path,
       )
     end
     )

Library/Homebrew/download_strategy.rb

@@ -811,6 +811,10 @@ end
 # @api public
 class GitDownloadStrategy < VCSDownloadStrategy
   def initialize(url, name, version, **meta)
+    # Needs to be before the call to `super`, as the VCSDownloadStrategy's
+    # constructor calls `cache_tag` and sets the cache path.
+    @only_path = meta[:only_path]
+
     super
     @ref_type ||= :branch
     @ref ||= "master"
@@ -836,7 +840,11 @@ class GitDownloadStrategy < VCSDownloadStrategy
 
   sig { returns(String) }
   def cache_tag
-    "git"
+    if partial_clone_sparse_checkout?
+      "git-sparse"
+    else
+      "git"
+    end
   end
 
   sig { returns(Integer) }
@@ -880,6 +888,12 @@ class GitDownloadStrategy < VCSDownloadStrategy
     (cached_location/".gitmodules").exist?
   end
 
+  def partial_clone_sparse_checkout?
+    return false if @only_path.blank?
+
+    Utils::Git.supports_partial_clone_sparse_checkout?
+  end
+
   sig { returns(T::Array[String]) }
   def clone_args
     args = %w[clone]
@@ -889,6 +903,8 @@ class GitDownloadStrategy < VCSDownloadStrategy
       args << "--branch" << @ref
     end
 
+    args << "--no-checkout" << "--filter=blob:none" if partial_clone_sparse_checkout?
+
     args << "-c" << "advice.detachedHead=false" # silences detached head warning
     args << @url << cached_location
   end
@@ -922,6 +938,13 @@ class GitDownloadStrategy < VCSDownloadStrategy
     command! "git",
              args:  ["config", "advice.detachedHead", "false"],
              chdir: cached_location
+
+    return unless partial_clone_sparse_checkout?
+
+    command! "git",
+             args:  ["config", "origin.partialclonefilter", "blob:none"],
+             chdir: cached_location
+    configure_sparse_checkout
   end
 
   sig { params(timeout: T.nilable(Time)).void }
@@ -950,6 +973,9 @@ class GitDownloadStrategy < VCSDownloadStrategy
              args:    ["config", "homebrew.cacheversion", cache_version],
              chdir:   cached_location,
              timeout: timeout&.remaining
+
+    configure_sparse_checkout if partial_clone_sparse_checkout?
+
     checkout(timeout: timeout)
     update_submodules(timeout: timeout) if submodules?
   end
@@ -1020,6 +1046,14 @@ class GitDownloadStrategy < VCSDownloadStrategy
       dot_git.atomic_write("gitdir: #{relative_git_dir}\n")
     end
   end
+
+  def configure_sparse_checkout
+    command! "git",
+             args:  ["config", "core.sparseCheckout", "true"],
+             chdir: cached_location
+
+    (git_dir/"info"/"sparse-checkout").atomic_write("#{@only_path}\n")
+  end
 end
 
 # Strategy for downloading a Git repository from GitHub.

Library/Homebrew/utils/git.rb

@@ -141,5 +141,11 @@ module Utils
         raise ErrorDuringExecution.new(cmd, status: $CHILD_STATUS, output: [[:stdout, output]])
       end
     end
+
+    def supports_partial_clone_sparse_checkout?
+      # There is some support for partial clones prior to 2.20, but we avoid using it
+      # due to performance issues
+      Version.create(version) >= Version.create("2.20.0")
+    end
   end
 end

docs/Cask-Cookbook.md

@@ -1138,6 +1138,18 @@ In rare cases, a distribution may not be available over ordinary HTTP/S. Subvers
 | `revision:`        | a string identifying the subversion revision to download
 | `trust_cert:`      | set to `true` to automatically trust the certificate presented by the server (avoiding an interactive prompt)
 
+#### Git URLs
+
+Artifacts also may be distributed via git repositories. URLs that end in `.git` are automatically assumed to be git repositories, and the following key/value pairs may be appended to `url`:
+
+| key                | value       |
+| ------------------ | ----------- |
+| `using:`           | the symbol `:git` is the only legal value
+| `tag:`             | a string identifying the git tag to download
+| `revision:`        | a string identifying the git revision to download
+| `branch:`          | a string identifying the git branch to download
+| `only_path:`       | a path within the repository to limit the checkout to. If only a single directory of a large repository is required, using this option can signficantly speed up downloads. If provided, artifact paths are relative to this path.
+
 #### SourceForge/OSDN URLs
 
 SourceForge and OSDN (formerly `SourceForge.JP`) projects are common ways to distribute binaries, but they provide many different styles of URLs to get to the goods.