From 7df90eb7e1d3c1d32b99daf9206d57591d2a5a93 Mon Sep 17 00:00:00 2001 From: Harry Marr Date: Mon, 2 May 2022 19:53:44 -0400 Subject: [PATCH 1/9] Support git partial clones with sparse checkouts --- Library/Homebrew/cask/url.rb | 10 +++++++-- Library/Homebrew/download_strategy.rb | 30 +++++++++++++++++++++++++++ 2 files changed, 38 insertions(+), 2 deletions(-) diff --git a/Library/Homebrew/cask/url.rb b/Library/Homebrew/cask/url.rb index 6221f0414e..f692b0f0e5 100644 --- a/Library/Homebrew/cask/url.rb +++ b/Library/Homebrew/cask/url.rb @@ -15,7 +15,7 @@ class URL < Delegator :verified, :using, :tag, :branch, :revisions, :revision, :trust_cert, :cookies, :referer, :header, :user_agent, - :data + :data, :only_paths extend Forwardable def_delegators :uri, :path, :scheme, :to_s @@ -36,6 +36,7 @@ class URL < Delegator header: T.nilable(String), user_agent: T.nilable(T.any(Symbol, String)), data: T.nilable(T::Hash[String, String]), + only_paths: T.nilable(T::Array[String]), ).void } def initialize( @@ -51,7 +52,8 @@ class URL < Delegator referer: nil, header: nil, user_agent: nil, - data: nil + data: nil, + only_paths: nil ) @uri = URI(uri) @@ -69,6 +71,7 @@ class URL < Delegator specs[:header] = @header = header specs[:user_agent] = @user_agent = user_agent || :default specs[:data] = @data = data + specs[:only_paths] = @only_paths = only_paths @specs = specs.compact end @@ -156,6 +159,7 @@ class URL < Delegator header: T.nilable(String), user_agent: T.nilable(T.any(Symbol, String)), data: T.nilable(T::Hash[String, String]), + only_paths: T.nilable(T::Array[String]), caller_location: Thread::Backtrace::Location, dsl: T.nilable(Cask::DSL), block: T.nilable(T.proc.params(arg0: T.all(String, BlockDSL::PageWithURL)).returns(T.untyped)), @@ -175,6 +179,7 @@ class URL < Delegator header: nil, user_agent: nil, data: nil, + only_paths: nil, caller_location: T.must(caller_locations).fetch(0), dsl: nil, &block @@ -202,6 +207,7 @@ class URL < Delegator header: header, user_agent: user_agent, data: data, + only_paths: only_paths, ) end ) diff --git a/Library/Homebrew/download_strategy.rb b/Library/Homebrew/download_strategy.rb index c53e62d99b..d3aead0c4c 100644 --- a/Library/Homebrew/download_strategy.rb +++ b/Library/Homebrew/download_strategy.rb @@ -814,6 +814,7 @@ class GitDownloadStrategy < VCSDownloadStrategy super @ref_type ||= :branch @ref ||= "master" + @only_paths = meta[:only_paths] end # @see AbstractDownloadStrategy#source_modified_time @@ -880,6 +881,14 @@ class GitDownloadStrategy < VCSDownloadStrategy (cached_location/".gitmodules").exist? end + def partial_clone_sparse_checkout? + return false if @only_paths.nil? + + # There is some support for partial clones prior to 2.20, but we avoid using it + # due to performance issues + Version.create(Utils::Git.version) >= Version.create("2.20.0") + end + sig { returns(T::Array[String]) } def clone_args args = %w[clone] @@ -889,6 +898,8 @@ class GitDownloadStrategy < VCSDownloadStrategy args << "--branch" << @ref end + args << "--no-checkout" << "--filter=blob:none" if partial_clone_sparse_checkout? + args << "-c" << "advice.detachedHead=false" # silences detached head warning args << @url << cached_location end @@ -922,6 +933,13 @@ class GitDownloadStrategy < VCSDownloadStrategy command! "git", args: ["config", "advice.detachedHead", "false"], chdir: cached_location + + if partial_clone_sparse_checkout? + command! "git", + args: ["config", "origin.partialclonefilter", "blob:none"], + chdir: cached_location + configure_sparse_checkout + end end sig { params(timeout: T.nilable(Time)).void } @@ -950,6 +968,9 @@ class GitDownloadStrategy < VCSDownloadStrategy args: ["config", "homebrew.cacheversion", cache_version], chdir: cached_location, timeout: timeout&.remaining + + configure_sparse_checkout if partial_clone_sparse_checkout? + checkout(timeout: timeout) update_submodules(timeout: timeout) if submodules? end @@ -1020,6 +1041,15 @@ class GitDownloadStrategy < VCSDownloadStrategy dot_git.atomic_write("gitdir: #{relative_git_dir}\n") end end + + def configure_sparse_checkout + command! "git", + args: ["config", "core.sparseCheckout", "true"], + chdir: cached_location + + sparse_checkout_paths = @only_paths.join("\n") + "\n" + (git_dir/"info"/"sparse-checkout").atomic_write(sparse_checkout_paths) + end end # Strategy for downloading a Git repository from GitHub. From 1472259e1d0eb52fcac2a860bbd6eb8b5aa64cf8 Mon Sep 17 00:00:00 2001 From: Harry Marr Date: Mon, 2 May 2022 19:58:06 -0400 Subject: [PATCH 2/9] Appease rubocop --- Library/Homebrew/download_strategy.rb | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/Library/Homebrew/download_strategy.rb b/Library/Homebrew/download_strategy.rb index d3aead0c4c..181be0a3dd 100644 --- a/Library/Homebrew/download_strategy.rb +++ b/Library/Homebrew/download_strategy.rb @@ -934,12 +934,12 @@ class GitDownloadStrategy < VCSDownloadStrategy args: ["config", "advice.detachedHead", "false"], chdir: cached_location - if partial_clone_sparse_checkout? - command! "git", - args: ["config", "origin.partialclonefilter", "blob:none"], - chdir: cached_location - configure_sparse_checkout - end + return unless partial_clone_sparse_checkout? + + command! "git", + args: ["config", "origin.partialclonefilter", "blob:none"], + chdir: cached_location + configure_sparse_checkout end sig { params(timeout: T.nilable(Time)).void } @@ -1047,7 +1047,7 @@ class GitDownloadStrategy < VCSDownloadStrategy args: ["config", "core.sparseCheckout", "true"], chdir: cached_location - sparse_checkout_paths = @only_paths.join("\n") + "\n" + sparse_checkout_paths = "#{@only_paths.join("\n")}\n" (git_dir/"info"/"sparse-checkout").atomic_write(sparse_checkout_paths) end end From 99504c44746d7b9b6f55164de389581b3d0ebe97 Mon Sep 17 00:00:00 2001 From: Harry Marr Date: Wed, 4 May 2022 11:13:48 -0400 Subject: [PATCH 3/9] Use `blank?` rather than `nil?` Co-authored-by: Mike McQuaid --- Library/Homebrew/download_strategy.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Library/Homebrew/download_strategy.rb b/Library/Homebrew/download_strategy.rb index 181be0a3dd..37607e0ed9 100644 --- a/Library/Homebrew/download_strategy.rb +++ b/Library/Homebrew/download_strategy.rb @@ -882,7 +882,7 @@ class GitDownloadStrategy < VCSDownloadStrategy end def partial_clone_sparse_checkout? - return false if @only_paths.nil? + return false if @only_paths.blank? # There is some support for partial clones prior to 2.20, but we avoid using it # due to performance issues From 49e2cf9f929101fa3bf109b326a6eec98697e925 Mon Sep 17 00:00:00 2001 From: Harry Marr Date: Wed, 4 May 2022 11:22:32 -0400 Subject: [PATCH 4/9] Extract `supports_partial_clone_sparse_checkout?` --- Library/Homebrew/download_strategy.rb | 4 +--- Library/Homebrew/utils/git.rb | 6 ++++++ 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/Library/Homebrew/download_strategy.rb b/Library/Homebrew/download_strategy.rb index 37607e0ed9..b062ca362f 100644 --- a/Library/Homebrew/download_strategy.rb +++ b/Library/Homebrew/download_strategy.rb @@ -884,9 +884,7 @@ class GitDownloadStrategy < VCSDownloadStrategy def partial_clone_sparse_checkout? return false if @only_paths.blank? - # There is some support for partial clones prior to 2.20, but we avoid using it - # due to performance issues - Version.create(Utils::Git.version) >= Version.create("2.20.0") + Utils::Git.supports_partial_clone_sparse_checkout? end sig { returns(T::Array[String]) } diff --git a/Library/Homebrew/utils/git.rb b/Library/Homebrew/utils/git.rb index c81435a0b1..eb492c2832 100644 --- a/Library/Homebrew/utils/git.rb +++ b/Library/Homebrew/utils/git.rb @@ -141,5 +141,11 @@ module Utils raise ErrorDuringExecution.new(cmd, status: $CHILD_STATUS, output: [[:stdout, output]]) end end + + def supports_partial_clone_sparse_checkout? + # There is some support for partial clones prior to 2.20, but we avoid using it + # due to performance issues + Version.create(version) >= Version.create("2.20.0") + end end end From 6219cd0a9279989caf56eb1f6c530c69ae5dd713 Mon Sep 17 00:00:00 2001 From: Harry Marr Date: Sun, 12 Jun 2022 10:48:47 -0400 Subject: [PATCH 5/9] Use different cache tag for sparse checkouts --- Library/Homebrew/download_strategy.rb | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/Library/Homebrew/download_strategy.rb b/Library/Homebrew/download_strategy.rb index b062ca362f..3d917eb7f6 100644 --- a/Library/Homebrew/download_strategy.rb +++ b/Library/Homebrew/download_strategy.rb @@ -811,10 +811,13 @@ end # @api public class GitDownloadStrategy < VCSDownloadStrategy def initialize(url, name, version, **meta) + # Needs to be before the call to `super`, as the VCSDownloadStrategy's + # constructor calls `cache_tag` and sets the cache path. + @only_paths = meta[:only_paths] + super @ref_type ||= :branch @ref ||= "master" - @only_paths = meta[:only_paths] end # @see AbstractDownloadStrategy#source_modified_time @@ -837,7 +840,11 @@ class GitDownloadStrategy < VCSDownloadStrategy sig { returns(String) } def cache_tag - "git" + if partial_clone_sparse_checkout? + "git-sparse" + else + "git" + end end sig { returns(Integer) } From b11df3bef825417b4d67ed9b5639f206e20851bf Mon Sep 17 00:00:00 2001 From: Harry Marr Date: Sat, 22 Oct 2022 10:53:02 -0400 Subject: [PATCH 6/9] Singularise `only_paths` (now `only_path`) --- Library/Homebrew/cask/url.rb | 14 +++++++------- Library/Homebrew/download_strategy.rb | 7 +++---- 2 files changed, 10 insertions(+), 11 deletions(-) diff --git a/Library/Homebrew/cask/url.rb b/Library/Homebrew/cask/url.rb index f692b0f0e5..a59e1f07b1 100644 --- a/Library/Homebrew/cask/url.rb +++ b/Library/Homebrew/cask/url.rb @@ -15,7 +15,7 @@ class URL < Delegator :verified, :using, :tag, :branch, :revisions, :revision, :trust_cert, :cookies, :referer, :header, :user_agent, - :data, :only_paths + :data, :only_path extend Forwardable def_delegators :uri, :path, :scheme, :to_s @@ -36,7 +36,7 @@ class URL < Delegator header: T.nilable(String), user_agent: T.nilable(T.any(Symbol, String)), data: T.nilable(T::Hash[String, String]), - only_paths: T.nilable(T::Array[String]), + only_path: T.nilable(String), ).void } def initialize( @@ -53,7 +53,7 @@ class URL < Delegator header: nil, user_agent: nil, data: nil, - only_paths: nil + only_path: nil ) @uri = URI(uri) @@ -71,7 +71,7 @@ class URL < Delegator specs[:header] = @header = header specs[:user_agent] = @user_agent = user_agent || :default specs[:data] = @data = data - specs[:only_paths] = @only_paths = only_paths + specs[:only_path] = @only_path = only_path @specs = specs.compact end @@ -159,7 +159,7 @@ class URL < Delegator header: T.nilable(String), user_agent: T.nilable(T.any(Symbol, String)), data: T.nilable(T::Hash[String, String]), - only_paths: T.nilable(T::Array[String]), + only_path: T.nilable(String), caller_location: Thread::Backtrace::Location, dsl: T.nilable(Cask::DSL), block: T.nilable(T.proc.params(arg0: T.all(String, BlockDSL::PageWithURL)).returns(T.untyped)), @@ -179,7 +179,7 @@ class URL < Delegator header: nil, user_agent: nil, data: nil, - only_paths: nil, + only_path: nil, caller_location: T.must(caller_locations).fetch(0), dsl: nil, &block @@ -207,7 +207,7 @@ class URL < Delegator header: header, user_agent: user_agent, data: data, - only_paths: only_paths, + only_path: only_path, ) end ) diff --git a/Library/Homebrew/download_strategy.rb b/Library/Homebrew/download_strategy.rb index 3d917eb7f6..3ce7303d47 100644 --- a/Library/Homebrew/download_strategy.rb +++ b/Library/Homebrew/download_strategy.rb @@ -813,7 +813,7 @@ class GitDownloadStrategy < VCSDownloadStrategy def initialize(url, name, version, **meta) # Needs to be before the call to `super`, as the VCSDownloadStrategy's # constructor calls `cache_tag` and sets the cache path. - @only_paths = meta[:only_paths] + @only_path = meta[:only_path] super @ref_type ||= :branch @@ -889,7 +889,7 @@ class GitDownloadStrategy < VCSDownloadStrategy end def partial_clone_sparse_checkout? - return false if @only_paths.blank? + return false if @only_path.blank? Utils::Git.supports_partial_clone_sparse_checkout? end @@ -1052,8 +1052,7 @@ class GitDownloadStrategy < VCSDownloadStrategy args: ["config", "core.sparseCheckout", "true"], chdir: cached_location - sparse_checkout_paths = "#{@only_paths.join("\n")}\n" - (git_dir/"info"/"sparse-checkout").atomic_write(sparse_checkout_paths) + (git_dir/"info"/"sparse-checkout").atomic_write("#{@only_path}\n") end end From 2d68e27d7b1f15f52e88909280777a38ec287d75 Mon Sep 17 00:00:00 2001 From: Harry Marr Date: Sat, 22 Oct 2022 10:54:51 -0400 Subject: [PATCH 7/9] Prefix artifact source with `only_path` --- Library/Homebrew/cask/artifact/relocated.rb | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Library/Homebrew/cask/artifact/relocated.rb b/Library/Homebrew/cask/artifact/relocated.rb index d2a6716914..c6618c6d00 100644 --- a/Library/Homebrew/cask/artifact/relocated.rb +++ b/Library/Homebrew/cask/artifact/relocated.rb @@ -51,7 +51,9 @@ module Cask target = target_hash[:target] @source_string = source.to_s @target_string = target.to_s - source = cask.staged_path.join(source) + base_path = cask.staged_path + base_path = base_path.join(cask.url.only_path) if cask.url && cask.url.only_path.present? + source = base_path.join(source) @source = source target ||= source.basename @target = resolve_target(target) From 237a4181c08de37d87966a68f3f7aeaf9ce9c414 Mon Sep 17 00:00:00 2001 From: Harry Marr Date: Sat, 22 Oct 2022 12:01:13 -0400 Subject: [PATCH 8/9] Add docs for casks that use git URLs --- docs/Cask-Cookbook.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/docs/Cask-Cookbook.md b/docs/Cask-Cookbook.md index e1d26d6fe4..23e048ad9b 100644 --- a/docs/Cask-Cookbook.md +++ b/docs/Cask-Cookbook.md @@ -1138,6 +1138,18 @@ In rare cases, a distribution may not be available over ordinary HTTP/S. Subvers | `revision:` | a string identifying the subversion revision to download | `trust_cert:` | set to `true` to automatically trust the certificate presented by the server (avoiding an interactive prompt) +#### Git URLs + +Artifacts also may be distributed via git repositories. URLs that end in `.git` are automatically assumed to be git repositories, and the following key/value pairs may be appended to `url`: + +| key | value | +| ------------------ | ----------- | +| `using:` | the symbol `:git` is the only legal value +| `tag:` | a string identifying the git tag to download +| `revision:` | a string identifying the git revision to download +| `branch:` | a string identifying the git branch to download +| `only_path:` | a path within the repository to limit the checkout to. If only a single directory of a large repository is required, using this option can signficantly speed up downloads. If provided, artifact paths are relative to this path. + #### SourceForge/OSDN URLs SourceForge and OSDN (formerly `SourceForge.JP`) projects are common ways to distribute binaries, but they provide many different styles of URLs to get to the goods. From 4e3a21b380d4248de239f269048304a02a8556bd Mon Sep 17 00:00:00 2001 From: Mike McQuaid Date: Tue, 25 Oct 2022 09:00:54 +0100 Subject: [PATCH 9/9] cask/artifact/relocated: tweak syntax. --- Library/Homebrew/cask/artifact/relocated.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Library/Homebrew/cask/artifact/relocated.rb b/Library/Homebrew/cask/artifact/relocated.rb index c6618c6d00..536b26be6e 100644 --- a/Library/Homebrew/cask/artifact/relocated.rb +++ b/Library/Homebrew/cask/artifact/relocated.rb @@ -52,7 +52,7 @@ module Cask @source_string = source.to_s @target_string = target.to_s base_path = cask.staged_path - base_path = base_path.join(cask.url.only_path) if cask.url && cask.url.only_path.present? + base_path = base_path.join(cask.url.only_path) if cask.url&.only_path.present? source = base_path.join(source) @source = source target ||= source.basename