Merge pull request #10977 from MikeMcQuaid/github_packages_tab_download

GitHub Packages Tab download

Library/Homebrew/cmd/fetch.rb

@@ -88,6 +88,7 @@ module Homebrew
         fetched_bottle = false
         if fetch_bottle?(f, args: args)
           begin
+            f.fetch_bottle_tab
             fetch_formula(f.bottle, args: args)
           rescue Interrupt
             raise

Library/Homebrew/dev-cmd/bottle.rb

@@ -540,6 +540,7 @@ module Homebrew
           "prefix"   => bottle.prefix,
           "cellar"   => bottle.cellar.to_s,
           "rebuild"  => bottle.rebuild,
+          "date"     => Time.now.strftime("%F"),
           "tags"     => {
             bottle_tag.to_s => {
               "filename"              => filename.bintray,

Library/Homebrew/download_strategy.rb

@@ -426,7 +426,7 @@ class CurlDownloadStrategy < AbstractFileDownloadStrategy
       url = url.sub(%r{^((ht|f)tps?://)?}, "#{domain.chomp("/")}/")
     end
 
-    out, _, status= curl_output("--location", "--silent", "--head", "--request", "GET", url.to_s)
+    out, _, status = curl_output("--location", "--silent", "--head", "--request", "GET", url.to_s)
 
     lines = status.success? ? out.lines.map(&:chomp) : []
 
@@ -533,20 +533,18 @@ end
 #
 # @api public
 class CurlGitHubPackagesDownloadStrategy < CurlDownloadStrategy
-  attr_accessor :checksum, :name
+  attr_writer :resolved_basename
+
+  def initialize(url, name, version, **meta)
+    meta ||= {}
+    meta[:header] = "Authorization: Bearer"
+    super(url, name, version, meta)
+  end
 
   private
 
-  def _fetch(url:, resolved_url:)
-    raise CurlDownloadStrategyError, "Empty checksum" if checksum.blank?
-    raise CurlDownloadStrategyError, "Empty name" if name.blank?
-
-    _, org, repo, = *url.match(GitHubPackages::URL_REGEX)
-
-    # remove redundant repo prefix for a shorter name
-    repo = repo.delete_prefix("homebrew-")
-    blob_url = "#{GitHubPackages::URL_PREFIX}#{org}/#{repo}/#{name}/blobs/sha256:#{checksum}"
-    curl_download(blob_url, "--header", "Authorization: Bearer", to: temporary_path)
+  def resolved_basename
+    @resolved_basename.presence || super
   end
 end
 

Library/Homebrew/formula.rb

@@ -2211,6 +2211,20 @@ class Formula
     patchlist.select(&:external?).each(&:fetch)
   end
 
+  sig { void }
+  def fetch_bottle_tab
+    return unless bottled?
+
+    T.must(bottle).fetch_tab
+  end
+
+  sig { returns(Hash) }
+  def bottle_tab_attributes
+    return {} unless bottled?
+
+    T.must(bottle).tab_attributes
+  end
+
   private
 
   def prepare_patches

Library/Homebrew/formula_installer.rb

@@ -1100,7 +1100,9 @@ class FormulaInstaller
 
     return if only_deps?
 
-    unless pour_bottle?(output_warning: true)
+    if pour_bottle?(output_warning: true)
+      formula.fetch_bottle_tab
+    else
       formula.fetch_patches
       formula.resources.each(&:fetch)
     end
@@ -1124,14 +1126,17 @@ class FormulaInstaller
     end
 
     keg = Keg.new(formula.prefix)
-    tab = Tab.for_keg(keg)
     Tab.clear_cache
 
+    tab = if (tab_attributes = formula.bottle_tab_attributes.presence)
+      Tab.from_file_content(tab_attributes.to_json, keg/Tab::FILENAME)
+    else
+      Tab.for_keg(keg)
+    end
+
     skip_linkage = formula.bottle_specification.skip_relocation?
     keg.replace_placeholders_with_locations tab.changed_files, skip_linkage: skip_linkage
 
-    tab = Tab.for_keg(keg)
-
     unless ignore_deps?
       CxxStdlib.check_compatibility(
         formula, formula.recursive_dependencies,

Library/Homebrew/formulary.rb

@@ -192,8 +192,8 @@ module Formulary
     end
 
     def get_formula(spec, force_bottle: false, flags: [], **)
-      contents = Utils::Bottles.formula_contents @bottle_filename, name: name
       formula = begin
+        contents = Utils::Bottles.formula_contents @bottle_filename, name: name
         Formulary.from_contents(name, path, contents, spec, force_bottle: force_bottle, flags: flags)
       rescue FormulaUnreadableError => e
         opoo <<~EOS
@@ -201,6 +201,12 @@ module Formulary
           #{e}
         EOS
         super
+      rescue BottleFormulaUnavailableError => e
+        opoo <<~EOS
+          #{e}
+          Falling back to non-bottle formula.
+        EOS
+        super
       end
       formula.local_bottle_path = @bottle_filename
       formula

Library/Homebrew/github_packages.rb

@@ -68,6 +68,32 @@ class GitHubPackages
     end
   end
 
+  def self.version_rebuild(version, rebuild, bottle_tag = nil)
+    bottle_tag = (".#{bottle_tag}" if bottle_tag.present?)
+
+    rebuild = if rebuild.to_i.positive?
+      if bottle_tag
+        ".#{rebuild}"
+      else
+        "-#{rebuild}"
+      end
+    end
+
+    "#{version}#{bottle_tag}#{rebuild}"
+  end
+
+  def self.repo_without_prefix(repo)
+    # remove redundant repo prefix for a shorter name
+    repo.delete_prefix("homebrew-")
+  end
+
+  def self.root_url(org, repo, prefix = URL_PREFIX)
+    # docker/skopeo insist on lowercase org ("repository name")
+    org = org.downcase
+
+    "#{prefix}#{org}/#{repo_without_prefix(repo)}"
+  end
+
   private
 
   IMAGE_CONFIG_SCHEMA_URI = "https://opencontainers.org/schema/image/config"
@@ -138,10 +164,8 @@ class GitHubPackages
     repo = "homebrew-#{repo}" unless HOMEBREW_OFFICIAL_REPO_PREFIXES_REGEX.match?(repo)
 
     version = bottle_hash["formula"]["pkg_version"]
-    rebuild = if (rebuild = bottle_hash["bottle"]["rebuild"]).positive?
-      ".#{rebuild}"
-    end
-    version_rebuild = "#{version}#{rebuild}"
+    rebuild = bottle_hash["bottle"]["rebuild"]
+    version_rebuild = GitHubPackages.version_rebuild(version, rebuild)
     root = Pathname("#{formula_name}--#{version_rebuild}")
     FileUtils.rm_rf root
 
@@ -161,8 +185,9 @@ class GitHubPackages
       remote
     end
 
+    created_date = bottle_hash["bottle"]["date"]
     formula_annotations_hash = {
-      "org.opencontainers.image.created"       => Time.now.strftime("%F"),
+      "org.opencontainers.image.created"       => created_date,
       "org.opencontainers.image.description"   => bottle_hash["formula"]["desc"],
       "org.opencontainers.image.documentation" => documentation,
       "org.opencontainers.image.license"       => bottle_hash["formula"]["license"],
@@ -185,16 +210,34 @@ class GitHubPackages
       tar_gz_sha256 = write_tar_gz(local_file, blobs)
 
       tab = tag_hash["tab"]
-      architecture = TAB_ARCH_TO_PLATFORM_ARCHITECTURE[tab["arch"]]
+      architecture = if tab["arch"].present?
+        TAB_ARCH_TO_PLATFORM_ARCHITECTURE[tab["arch"]]
+      elsif bottle_tag.to_s.start_with?("arm64")
+        "arm64"
+      else
+        "amd64"
+      end
       raise TypeError, "unknown tab['arch']: #{tab["arch"]}" if architecture.blank?
 
-      os = BUILT_ON_OS_TO_PLATFORM_OS[tab["built_on"]["os"]]
+      os = if tab["built_on"].present? && tab["built_on"]["os"].present?
+        BUILT_ON_OS_TO_PLATFORM_OS[tab["built_on"]["os"]]
+      elsif bottle_tag.to_s.end_with?("_linux")
+        "linux"
+      else
+        "darwin"
+      end
       raise TypeError, "unknown tab['built_on']['os']: #{tab["built_on"]["os"]}" if os.blank?
 
+      os_version = if tab["built_on"].present? && tab["built_on"]["os_version"].present?
+        tab["built_on"]["os_version"]
+      else
+        MacOS::Version.from_symbol(bottle_tag).to_s
+      end
+
       platform_hash = {
         architecture: architecture,
         os: os,
-        "os.version" => tab["built_on"]["os_version"],
+        "os.version" => os_version,
       }
       tar_sha256 = Digest::SHA256.hexdigest(
         Utils.safe_popen_read("gunzip", "--stdout", "--decompress", local_file),
@@ -205,10 +248,10 @@ class GitHubPackages
       formulae_dir = tag_hash["formulae_brew_sh_path"]
       documentation = "https://formulae.brew.sh/#{formulae_dir}/#{formula_name}" if formula_core_tap
 
-      tag = "#{version}.#{bottle_tag}#{rebuild}"
+      tag = GitHubPackages.version_rebuild(version, rebuild, bottle_tag)
 
       annotations_hash = formula_annotations_hash.merge({
-        "org.opencontainers.image.created"       => Time.at(tag_hash["tab"]["source_modified_time"]).strftime("%F"),
+        "org.opencontainers.image.created"       => created_date,
         "org.opencontainers.image.documentation" => documentation,
         "org.opencontainers.image.ref.name"      => tag,
         "org.opencontainers.image.title"         => "#{formula_full_name} #{tag}",
@@ -255,11 +298,8 @@ class GitHubPackages
     write_index_json(index_json_sha256, index_json_size, root,
                      "org.opencontainers.image.ref.name" => version_rebuild)
 
-    # docker/skopeo insist on lowercase org ("repository name")
-    org_prefix = "#{DOCKER_PREFIX}#{org.downcase}"
-    # remove redundant repo prefix for a shorter name
-    package_name = "#{repo.delete_prefix("homebrew-")}/#{formula_name}"
-    image_tag = "#{org_prefix}/#{package_name}:#{version_rebuild}"
+    image_tag = "#{GitHubPackages.root_url(org, repo, DOCKER_PREFIX)}/#{formula_name}:#{version_rebuild}"
+
     puts
     args = ["copy", "--all", "oci:#{root}", image_tag.to_s]
     if dry_run
@@ -267,6 +307,7 @@ class GitHubPackages
     else
       args << "--dest-creds=#{user}:#{token}"
       system_command!(skopeo, verbose: true, print_stdout: true, args: args)
+      package_name = "#{GitHubPackages.repo_without_prefix(repo)}/#{formula_name}"
       ohai "Uploaded to https://github.com/orgs/Homebrew/packages/container/package/#{package_name}"
     end
   end

Library/Homebrew/software_spec.rb

@@ -304,9 +304,18 @@ class Bottle
 
     checksum, tag, cellar = spec.checksum_for(Utils::Bottles.tag)
 
-    filename = Filename.create(formula, tag, spec.rebuild)
-    @resource.url("#{spec.root_url}/#{filename.bintray}",
-                  select_download_strategy(spec.root_url_specs))
+    filename = Filename.create(formula, tag, spec.rebuild).bintray
+
+    # TODO: this will need adjusted when if we use GitHub Packages by default
+    path, resolved_basename = if (bottle_domain = Homebrew::EnvConfig.bottle_domain.presence) &&
+                                 bottle_domain.start_with?(GitHubPackages::URL_PREFIX)
+      ["#{@name}/blobs/sha256:#{checksum}", filename]
+    else
+      filename
+    end
+
+    @resource.url("#{spec.root_url}/#{path}", select_download_strategy(spec.root_url_specs))
+    @resource.downloader.resolved_basename = resolved_basename if resolved_basename.present?
     @resource.version = formula.pkg_version
     @resource.checksum = checksum
     @prefix = spec.prefix
@@ -316,16 +325,12 @@ class Bottle
 
   def fetch(verify_download_integrity: true)
     # add the default bottle domain as a fallback mirror
-    # TODO: this may need adjusted when if we use GitHub Packages by default
     if @resource.download_strategy == CurlDownloadStrategy &&
        @resource.url.start_with?(Homebrew::EnvConfig.bottle_domain)
       fallback_url = @resource.url
                               .sub(/^#{Regexp.escape(Homebrew::EnvConfig.bottle_domain)}/,
                                    HOMEBREW_BOTTLE_DEFAULT_DOMAIN)
       @resource.mirror(fallback_url) if [@resource.url, *@resource.mirrors].exclude?(fallback_url)
-    elsif @resource.download_strategy == CurlGitHubPackagesDownloadStrategy
-      @resource.downloader.name = @name
-      @resource.downloader.checksum = @resource.checksum.hexdigest
     end
     @resource.fetch(verify_download_integrity: verify_download_integrity)
   end
@@ -343,8 +348,62 @@ class Bottle
     resource.downloader.stage
   end
 
+  def fetch_tab
+    # a checksum is used later identifying the correct tab but we do not have the checksum for the manifest/tab
+    github_packages_manifest_resource&.fetch(verify_download_integrity: false)
+  end
+
+  def tab_attributes
+    return {} unless github_packages_manifest_resource&.downloaded?
+
+    manifest_json = github_packages_manifest_resource.cached_download.read
+
+    json = begin
+      JSON.parse(manifest_json)
+    rescue JSON::ParserError
+      raise ArgumentError, "Couldn't parse manifest JSON."
+    end
+
+    manifests = json["manifests"]
+    raise ArgumentError, "Missing 'manifests' section." if manifests.blank?
+
+    manifests_annotations = manifests.map { |m| m["annotations"] }
+    raise ArgumentError, "Missing 'annotations' section." if manifests_annotations.blank?
+
+    bottle_checksum = @resource.checksum.hexdigest
+    manifest_annotations = manifests_annotations.find do |m|
+      m["sh.brew.bottle.checksum"] == bottle_checksum
+    end
+    raise ArgumentError, "Couldn't find manifest matching bottle checksum." if manifest_annotations.blank?
+
+    tab = manifest_annotations["sh.brew.tab"]
+    raise ArgumentError, "Couldn't find tab from manifest." if tab.blank?
+
+    begin
+      JSON.parse(tab)
+    rescue JSON::ParserError
+      raise ArgumentError, "Couldn't parse tab JSON."
+    end
+  end
+
   private
 
+  def github_packages_manifest_resource
+    return if @resource.download_strategy != CurlGitHubPackagesDownloadStrategy
+
+    @github_packages_manifest_resource ||= begin
+      resource = Resource.new("#{name}_bottle_manifest")
+
+      version_rebuild = GitHubPackages.version_rebuild(@resource.version, rebuild)
+      resource.version(version_rebuild)
+
+      resource.url("#{@spec.root_url}/#{name}/manifests/#{version_rebuild}",
+                   using: CurlGitHubPackagesDownloadStrategy)
+      resource.downloader.resolved_basename = "#{name}-#{version_rebuild}.bottle_manifest.json"
+      resource
+    end
+  end
+
   def select_download_strategy(specs)
     specs[:using] ||= DownloadStrategyDetector.detect(@spec.root_url)
     specs
@@ -380,7 +439,7 @@ class BottleSpecification
   def root_url(var = nil, specs = {})
     if var.nil?
       @root_url ||= if Homebrew::EnvConfig.bottle_domain.start_with?(GitHubPackages::URL_PREFIX)
-        "#{GitHubPackages::URL_PREFIX}#{tap.full_name}"
+        GitHubPackages.root_url(tap.user, tap.repo).to_s
       else
         "#{Homebrew::EnvConfig.bottle_domain}/#{Utils::Bottles::Bintray.repository(tap)}"
       end

Library/Homebrew/utils/bottles.rb

@@ -39,12 +39,6 @@ module Utils
         HOMEBREW_BOTTLES_EXTNAME_REGEX.match(filename).to_a
       end
 
-      # TODO: remove when removed from brew-test-bot
-      sig { returns(Regexp) }
-      def native_regex
-        /(\.#{Regexp.escape(tag.to_s)}\.bottle\.(\d+\.)?tar\.gz)$/o
-      end
-
       def receipt_path(bottle_file)
         path = Utils.popen_read("tar", "-tzf", bottle_file).lines.map(&:chomp).find do |line|
           line =~ %r{.+/.+/INSTALL_RECEIPT.json}