From 599ecc9b5ad7951b8ddc51490ebe93a976d43b29 Mon Sep 17 00:00:00 2001 From: Mike McQuaid Date: Sun, 14 Oct 2018 21:10:15 -0700 Subject: [PATCH] Deprecate unused download strategies Download the unused, private download strategies. These are a better fit for being declared in the formulae and/or taps that use them rather than in Homebrew/brew where they are unused by Homebrew maintainers or official taps/formulae. --- Library/Homebrew/compat.rb | 1 + Library/Homebrew/compat/download_strategy.rb | 254 ++++++++++++++++++ Library/Homebrew/download_strategy.rb | 200 -------------- .../Homebrew/test/download_strategies_spec.rb | 22 +- docs/Formula-Cookbook.md | 14 +- 5 files changed, 270 insertions(+), 221 deletions(-) create mode 100644 Library/Homebrew/compat/download_strategy.rb diff --git a/Library/Homebrew/compat.rb b/Library/Homebrew/compat.rb index cf048d1beb..7c444615b2 100644 --- a/Library/Homebrew/compat.rb +++ b/Library/Homebrew/compat.rb @@ -1,6 +1,7 @@ require "compat/os/mac" require "compat/dependable" require "compat/dependency_collector" +require "compat/download_strategy" require "compat/fileutils" require "compat/formula_support" require "compat/cask" diff --git a/Library/Homebrew/compat/download_strategy.rb b/Library/Homebrew/compat/download_strategy.rb new file mode 100644 index 0000000000..d4b476a260 --- /dev/null +++ b/Library/Homebrew/compat/download_strategy.rb @@ -0,0 +1,254 @@ +require "download_strategy" + +# S3DownloadStrategy downloads tarballs from AWS S3. +# To use it, add `:using => :s3` to the URL section of your +# formula. This download strategy uses AWS access tokens (in the +# environment variables `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY`) +# to sign the request. This strategy is good in a corporate setting, +# because it lets you use a private S3 bucket as a repo for internal +# distribution. (It will work for public buckets as well.) +class S3DownloadStrategy < CurlDownloadStrategy + def initialize(url, name, version, **meta) + odeprecated("S3DownloadStrategy", + "maintaining S3DownloadStrategy in your own formula or tap") + super + end + + def _fetch(url:, resolved_url:) + if url !~ %r{^https?://([^.].*)\.s3\.amazonaws\.com/(.+)$} && + url !~ %r{^s3://([^.].*?)/(.+)$} + raise "Bad S3 URL: " + url + end + + bucket = Regexp.last_match(1) + key = Regexp.last_match(2) + + ENV["AWS_ACCESS_KEY_ID"] = ENV["HOMEBREW_AWS_ACCESS_KEY_ID"] + ENV["AWS_SECRET_ACCESS_KEY"] = ENV["HOMEBREW_AWS_SECRET_ACCESS_KEY"] + + begin + signer = Aws::S3::Presigner.new + s3url = signer.presigned_url :get_object, bucket: bucket, key: key + rescue Aws::Sigv4::Errors::MissingCredentialsError + ohai "AWS credentials missing, trying public URL instead." + s3url = url + end + + curl_download s3url, to: temporary_path + end +end + +# GitHubPrivateRepositoryDownloadStrategy downloads contents from GitHub +# Private Repository. To use it, add +# `:using => :github_private_repo` to the URL section of +# your formula. This download strategy uses GitHub access tokens (in the +# environment variables `HOMEBREW_GITHUB_API_TOKEN`) to sign the request. This +# strategy is suitable for corporate use just like S3DownloadStrategy, because +# it lets you use a private GitHub repository for internal distribution. It +# works with public one, but in that case simply use CurlDownloadStrategy. +class GitHubPrivateRepositoryDownloadStrategy < CurlDownloadStrategy + require "utils/formatter" + require "utils/github" + + def initialize(url, name, version, **meta) + odeprecated("GitHubPrivateRepositoryDownloadStrategy", + "maintaining GitHubPrivateRepositoryDownloadStrategy in your own formula or tap") + super + parse_url_pattern + set_github_token + end + + def parse_url_pattern + unless match = url.match(%r{https://github.com/([^/]+)/([^/]+)/(\S+)}) + raise CurlDownloadStrategyError, "Invalid url pattern for GitHub Repository." + end + + _, @owner, @repo, @filepath = *match + end + + def download_url + "https://#{@github_token}@github.com/#{@owner}/#{@repo}/#{@filepath}" + end + + private + + def _fetch(url:, resolved_url:) + curl_download download_url, to: temporary_path + end + + def set_github_token + @github_token = ENV["HOMEBREW_GITHUB_API_TOKEN"] + unless @github_token + raise CurlDownloadStrategyError, "Environmental variable HOMEBREW_GITHUB_API_TOKEN is required." + end + + validate_github_repository_access! + end + + def validate_github_repository_access! + # Test access to the repository + GitHub.repository(@owner, @repo) + rescue GitHub::HTTPNotFoundError + # We only handle HTTPNotFoundError here, + # becase AuthenticationFailedError is handled within util/github. + message = <<~EOS + HOMEBREW_GITHUB_API_TOKEN can not access the repository: #{@owner}/#{@repo} + This token may not have permission to access the repository or the url of formula may be incorrect. + EOS + raise CurlDownloadStrategyError, message + end +end + +# GitHubPrivateRepositoryReleaseDownloadStrategy downloads tarballs from GitHub +# Release assets. To use it, add `:using => :github_private_release` to the URL section +# of your formula. This download strategy uses GitHub access tokens (in the +# environment variables HOMEBREW_GITHUB_API_TOKEN) to sign the request. +class GitHubPrivateRepositoryReleaseDownloadStrategy < GitHubPrivateRepositoryDownloadStrategy + def initialize(url, name, version, **meta) + odeprecated("GitHubPrivateRepositoryReleaseDownloadStrategy", + "maintaining GitHubPrivateRepositoryReleaseDownloadStrategy in your own formula or tap") + super + end + + def parse_url_pattern + url_pattern = %r{https://github.com/([^/]+)/([^/]+)/releases/download/([^/]+)/(\S+)} + unless @url =~ url_pattern + raise CurlDownloadStrategyError, "Invalid url pattern for GitHub Release." + end + + _, @owner, @repo, @tag, @filename = *@url.match(url_pattern) + end + + def download_url + "https://#{@github_token}@api.github.com/repos/#{@owner}/#{@repo}/releases/assets/#{asset_id}" + end + + private + + def _fetch(url:, resolved_url:) + # HTTP request header `Accept: application/octet-stream` is required. + # Without this, the GitHub API will respond with metadata, not binary. + curl_download download_url, "--header", "Accept: application/octet-stream", to: temporary_path + end + + def asset_id + @asset_id ||= resolve_asset_id + end + + def resolve_asset_id + release_metadata = fetch_release_metadata + assets = release_metadata["assets"].select { |a| a["name"] == @filename } + raise CurlDownloadStrategyError, "Asset file not found." if assets.empty? + + assets.first["id"] + end + + def fetch_release_metadata + release_url = "https://api.github.com/repos/#{@owner}/#{@repo}/releases/tags/#{@tag}" + GitHub.open_api(release_url) + end +end + +# ScpDownloadStrategy downloads files using ssh via scp. To use it, add +# `:using => :scp` to the URL section of your formula or +# provide a URL starting with scp://. This strategy uses ssh credentials for +# authentication. If a public/private keypair is configured, it will not +# prompt for a password. +# +# @example +# class Abc < Formula +# url "scp://example.com/src/abc.1.0.tar.gz" +# ... +class ScpDownloadStrategy < AbstractFileDownloadStrategy + def initialize(url, name, version, **meta) + odeprecated("ScpDownloadStrategy", + "maintaining ScpDownloadStrategy in your own formula or tap") + super + parse_url_pattern + end + + def parse_url_pattern + url_pattern = %r{scp://([^@]+@)?([^@:/]+)(:\d+)?/(\S+)} + if @url !~ url_pattern + raise ScpDownloadStrategyError, "Invalid URL for scp: #{@url}" + end + + _, @user, @host, @port, @path = *@url.match(url_pattern) + end + + def fetch + ohai "Downloading #{@url}" + + if cached_location.exist? + puts "Already downloaded: #{cached_location}" + else + system_command! "scp", args: [scp_source, temporary_path.to_s] + ignore_interrupts { temporary_path.rename(cached_location) } + end + end + + def clear_cache + super + rm_rf(temporary_path) + end + + private + + def scp_source + path_prefix = "/" unless @path.start_with?("~") + port_arg = "-P #{@port[1..-1]} " if @port + "#{port_arg}#{@user}#{@host}:#{path_prefix}#{@path}" + end +end + +class DownloadStrategyDetector + class << self + module Compat + def detect(url, using = nil) + strategy = super + require_aws_sdk if strategy == S3DownloadStrategy + strategy + end + + def detect_from_url(url) + case url + when %r{^s3://} + odeprecated("s3://", + "maintaining S3DownloadStrategy in your own formula or tap") + S3DownloadStrategy + when %r{^scp://} + odeprecated("scp://", + "maintaining ScpDownloadStrategy in your own formula or tap") + ScpDownloadStrategy + else + super(url) + end + end + + def detect_from_symbol(symbol) + case symbol + when :github_private_repo + odeprecated(":github_private_repo", + "maintaining GitHubPrivateRepositoryDownloadStrategy in your own formula or tap") + GitHubPrivateRepositoryDownloadStrategy + when :github_private_release + odeprecated(":github_private_repo", + "maintaining GitHubPrivateRepositoryReleaseDownloadStrategy in your own formula or tap") + GitHubPrivateRepositoryReleaseDownloadStrategy + when :s3 + odeprecated(":s3", + "maintaining S3DownloadStrategy in your own formula or tap") + S3DownloadStrategy + when :scp + odeprecated(":scp", + "maintaining ScpDownloadStrategy in your own formula or tap") + ScpDownloadStrategy + else + super(symbol) + end + end + end + + prepend Compat + end +end diff --git a/Library/Homebrew/download_strategy.rb b/Library/Homebrew/download_strategy.rb index fb803c899e..4afcc16032 100644 --- a/Library/Homebrew/download_strategy.rb +++ b/Library/Homebrew/download_strategy.rb @@ -474,191 +474,6 @@ class LocalBottleDownloadStrategy < AbstractFileDownloadStrategy end end -# S3DownloadStrategy downloads tarballs from AWS S3. -# To use it, add `:using => :s3` to the URL section of your -# formula. This download strategy uses AWS access tokens (in the -# environment variables `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY`) -# to sign the request. This strategy is good in a corporate setting, -# because it lets you use a private S3 bucket as a repo for internal -# distribution. (It will work for public buckets as well.) -class S3DownloadStrategy < CurlDownloadStrategy - def _fetch(url:, resolved_url:) - if url !~ %r{^https?://([^.].*)\.s3\.amazonaws\.com/(.+)$} && - url !~ %r{^s3://([^.].*?)/(.+)$} - raise "Bad S3 URL: " + url - end - - bucket = Regexp.last_match(1) - key = Regexp.last_match(2) - - ENV["AWS_ACCESS_KEY_ID"] = ENV["HOMEBREW_AWS_ACCESS_KEY_ID"] - ENV["AWS_SECRET_ACCESS_KEY"] = ENV["HOMEBREW_AWS_SECRET_ACCESS_KEY"] - - begin - signer = Aws::S3::Presigner.new - s3url = signer.presigned_url :get_object, bucket: bucket, key: key - rescue Aws::Sigv4::Errors::MissingCredentialsError - ohai "AWS credentials missing, trying public URL instead." - s3url = url - end - - curl_download s3url, to: temporary_path - end -end - -# GitHubPrivateRepositoryDownloadStrategy downloads contents from GitHub -# Private Repository. To use it, add -# `:using => :github_private_repo` to the URL section of -# your formula. This download strategy uses GitHub access tokens (in the -# environment variables `HOMEBREW_GITHUB_API_TOKEN`) to sign the request. This -# strategy is suitable for corporate use just like S3DownloadStrategy, because -# it lets you use a private GitHub repository for internal distribution. It -# works with public one, but in that case simply use CurlDownloadStrategy. -class GitHubPrivateRepositoryDownloadStrategy < CurlDownloadStrategy - require "utils/formatter" - require "utils/github" - - def initialize(url, name, version, **meta) - super - parse_url_pattern - set_github_token - end - - def parse_url_pattern - unless match = url.match(%r{https://github.com/([^/]+)/([^/]+)/(\S+)}) - raise CurlDownloadStrategyError, "Invalid url pattern for GitHub Repository." - end - - _, @owner, @repo, @filepath = *match - end - - def download_url - "https://#{@github_token}@github.com/#{@owner}/#{@repo}/#{@filepath}" - end - - private - - def _fetch(url:, resolved_url:) - curl_download download_url, to: temporary_path - end - - def set_github_token - @github_token = ENV["HOMEBREW_GITHUB_API_TOKEN"] - unless @github_token - raise CurlDownloadStrategyError, "Environmental variable HOMEBREW_GITHUB_API_TOKEN is required." - end - - validate_github_repository_access! - end - - def validate_github_repository_access! - # Test access to the repository - GitHub.repository(@owner, @repo) - rescue GitHub::HTTPNotFoundError - # We only handle HTTPNotFoundError here, - # becase AuthenticationFailedError is handled within util/github. - message = <<~EOS - HOMEBREW_GITHUB_API_TOKEN can not access the repository: #{@owner}/#{@repo} - This token may not have permission to access the repository or the url of formula may be incorrect. - EOS - raise CurlDownloadStrategyError, message - end -end - -# GitHubPrivateRepositoryReleaseDownloadStrategy downloads tarballs from GitHub -# Release assets. To use it, add `:using => :github_private_release` to the URL section -# of your formula. This download strategy uses GitHub access tokens (in the -# environment variables `HOMEBREW_GITHUB_API_TOKEN`) to sign the request. -class GitHubPrivateRepositoryReleaseDownloadStrategy < GitHubPrivateRepositoryDownloadStrategy - def parse_url_pattern - url_pattern = %r{https://github.com/([^/]+)/([^/]+)/releases/download/([^/]+)/(\S+)} - unless @url =~ url_pattern - raise CurlDownloadStrategyError, "Invalid url pattern for GitHub Release." - end - - _, @owner, @repo, @tag, @filename = *@url.match(url_pattern) - end - - def download_url - "https://#{@github_token}@api.github.com/repos/#{@owner}/#{@repo}/releases/assets/#{asset_id}" - end - - private - - def _fetch(url:, resolved_url:) - # HTTP request header `Accept: application/octet-stream` is required. - # Without this, the GitHub API will respond with metadata, not binary. - curl_download download_url, "--header", "Accept: application/octet-stream", to: temporary_path - end - - def asset_id - @asset_id ||= resolve_asset_id - end - - def resolve_asset_id - release_metadata = fetch_release_metadata - assets = release_metadata["assets"].select { |a| a["name"] == @filename } - raise CurlDownloadStrategyError, "Asset file not found." if assets.empty? - - assets.first["id"] - end - - def fetch_release_metadata - release_url = "https://api.github.com/repos/#{@owner}/#{@repo}/releases/tags/#{@tag}" - GitHub.open_api(release_url) - end -end - -# ScpDownloadStrategy downloads files using ssh via scp. To use it, add -# `:using => :scp` to the URL section of your formula or -# provide a URL starting with scp://. This strategy uses ssh credentials for -# authentication. If a public/private keypair is configured, it will not -# prompt for a password. -# -# @example -# class Abc < Formula -# url "scp://example.com/src/abc.1.0.tar.gz" -# ... -class ScpDownloadStrategy < AbstractFileDownloadStrategy - def initialize(url, name, version, **meta) - super - parse_url_pattern - end - - def parse_url_pattern - url_pattern = %r{scp://([^@]+@)?([^@:/]+)(:\d+)?/(\S+)} - if @url !~ url_pattern - raise ScpDownloadStrategyError, "Invalid URL for scp: #{@url}" - end - - _, @user, @host, @port, @path = *@url.match(url_pattern) - end - - def fetch - ohai "Downloading #{@url}" - - if cached_location.exist? - puts "Already downloaded: #{cached_location}" - else - system_command! "scp", args: [scp_source, temporary_path.to_s] - ignore_interrupts { temporary_path.rename(cached_location) } - end - end - - def clear_cache - super - rm_rf(temporary_path) - end - - private - - def scp_source - path_prefix = "/" unless @path.start_with?("~") - port_arg = "-P #{@port[1..-1]} " if @port - "#{port_arg}#{@user}#{@host}:#{path_prefix}#{@path}" - end -end - class SubversionDownloadStrategy < VCSDownloadStrategy def initialize(url, name, version, **meta) super @@ -1220,8 +1035,6 @@ class DownloadStrategyDetector "Unknown download strategy specification #{strategy.inspect}" end - require_aws_sdk if strategy == S3DownloadStrategy - strategy end @@ -1254,10 +1067,6 @@ class DownloadStrategyDetector SubversionDownloadStrategy when %r{^https?://(.+?\.)?sourceforge\.net/hgweb/} MercurialDownloadStrategy - when %r{^s3://} - S3DownloadStrategy - when %r{^scp://} - ScpDownloadStrategy else CurlDownloadStrategy end @@ -1268,11 +1077,7 @@ class DownloadStrategyDetector when :hg then MercurialDownloadStrategy when :nounzip then NoUnzipCurlDownloadStrategy when :git then GitDownloadStrategy - when :github_private_repo then GitHubPrivateRepositoryDownloadStrategy - when :github_private_release then GitHubPrivateRepositoryReleaseDownloadStrategy when :bzr then BazaarDownloadStrategy - when :s3 then S3DownloadStrategy - when :scp then ScpDownloadStrategy when :svn then SubversionDownloadStrategy when :curl then CurlDownloadStrategy when :cvs then CVSDownloadStrategy @@ -1282,9 +1087,4 @@ class DownloadStrategyDetector raise "Unknown download strategy #{symbol} was requested." end end - - def self.require_aws_sdk - Homebrew.install_gem! "aws-sdk-s3", "~> 1.8" - require "aws-sdk-s3" - end end diff --git a/Library/Homebrew/test/download_strategies_spec.rb b/Library/Homebrew/test/download_strategies_spec.rb index f7c1da2970..c85cab0118 100644 --- a/Library/Homebrew/test/download_strategies_spec.rb +++ b/Library/Homebrew/test/download_strategies_spec.rb @@ -46,8 +46,8 @@ describe VCSDownloadStrategy do end end -describe GitHubPrivateRepositoryDownloadStrategy do - subject { described_class.new(url, "foo", version) } +describe "GitHubPrivateRepositoryDownloadStrategy", :needs_compat do + subject { GitHubPrivateRepositoryDownloadStrategy.new(url, "foo", version) } let(:url) { "https://github.com/owner/repo/archive/1.1.5.tar.gz" } let(:version) { nil } @@ -70,8 +70,8 @@ describe GitHubPrivateRepositoryDownloadStrategy do its(:download_url) { is_expected.to eq("https://token@github.com/owner/repo/archive/1.1.5.tar.gz") } end -describe GitHubPrivateRepositoryReleaseDownloadStrategy do - subject { described_class.new(url, "foo", version) } +describe "GitHubPrivateRepositoryReleaseDownloadStrategy", :needs_compat do + subject { GitHubPrivateRepositoryReleaseDownloadStrategy.new(url, "foo", version) } let(:url) { "https://github.com/owner/repo/releases/download/tag/foo_v0.1.0_darwin_amd64.tar.gz" } let(:version) { nil } @@ -200,8 +200,8 @@ describe GitDownloadStrategy do end end -describe S3DownloadStrategy do - subject { described_class.new(url, name, version) } +describe "S3DownloadStrategy", :needs_compat do + subject { S3DownloadStrategy.new(url, name, version) } let(:name) { "foo" } let(:url) { "https://bucket.s3.amazonaws.com/foo.tar.gz" } @@ -427,8 +427,8 @@ describe CurlPostDownloadStrategy do end end -describe ScpDownloadStrategy do - subject { described_class.new(url, name, version) } +describe "ScpDownloadStrategy", :needs_compat do + subject { ScpDownloadStrategy.new(url, name, version) } let(:name) { "foo" } let(:url) { "scp://example.com/foo.tar.gz" } @@ -565,7 +565,7 @@ describe DownloadStrategyDetector do it { is_expected.to eq(GitHubGitDownloadStrategy) } end - context "when given an S3 URL" do + context "when given an S3 URL", :needs_compat do let(:url) { "s3://bucket/homebrew/brew.tar.gz" } it "returns S3DownloadStrategy" do @@ -574,7 +574,7 @@ describe DownloadStrategyDetector do end end - context "when given strategy = S3DownloadStrategy" do + context "when given strategy = S3DownloadStrategy", :needs_compat do let(:url) { "https://bkt.s3.amazonaws.com/key.tar.gz" } let(:strategy) { S3DownloadStrategy } @@ -584,7 +584,7 @@ describe DownloadStrategyDetector do end end - context "when given an scp URL" do + context "when given an scp URL", :needs_compat do let(:url) { "scp://example.com/brew.tar.gz" } it { is_expected.to eq(ScpDownloadStrategy) } diff --git a/docs/Formula-Cookbook.md b/docs/Formula-Cookbook.md index a4e5e2e73f..f8db639391 100644 --- a/docs/Formula-Cookbook.md +++ b/docs/Formula-Cookbook.md @@ -546,7 +546,7 @@ class Python3 < Formula head "https://hg.python.org/cpython", :using => :hg ``` -Homebrew offers both anonymous and authenticated download strategies. +Homebrew offers anonymous download strategies. | `:using` value | download strategy | |----------------|-------------------------------| @@ -561,19 +561,13 @@ Homebrew offers both anonymous and authenticated download strategies. | `:svn` | `SubversionDownloadStrategy` | |----------------|-------------------------------| -| `:using` value | download strategy | authentication source | -|---------------------------|--------------------------------------------------|----------------------------------------------| -| `:github_private_release` | `GitHubPrivateRepositoryReleaseDownloadStrategy` | `HOMEBREW_GITHUB_API_TOKEN` | -| `:github_private_repo` | `GitHubPrivateRepositoryDownloadStrategy` | `HOMEBREW_GITHUB_API_TOKEN` | -| `:s3` | `S3DownloadStrategy` | `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY` | -| `:scp` | `ScpDownloadStrategy` | SSH key pair | -|---------------------------|--------------------------------------------------|----------------------------------------------| - If you need more control over the way files are downloaded and staged, you can create a custom download strategy and specify it using the [`url`](https://www.rubydoc.info/github/Homebrew/brew/master/Formula#url-class_method) method's `:using` option: ```ruby class MyDownloadStrategy < SomeHomebrewDownloadStrategy - # Does something cool + def fetch + # downloads output to `temporary_path` + end end class Foo < Formula