github_packages: use skopeo.

`skopeo` allows replacing `docker` and `oras` with a single tool that
behaves in a way that's a little more intuitive for us (we build the
directory structure on disk and upload that). It seems to be better at
preserving our metadata, too.

Take the opportunity for writing out more metadata ourselves to split
out more logic into separate functions.

While we're here, fix the `download_strategy` timeout handling for
GitHub Packages.

Library/Homebrew/download_strategy.rb

@@ -548,14 +548,14 @@ class CurlGitHubPackagesDownloadStrategy < CurlDownloadStrategy
 
   private
 
-  def _fetch(url:, resolved_url:)
+  def _fetch(url:, resolved_url:, timeout:)
     raise "Empty checksum" if checksum.blank?
     raise "Empty name" if name.blank?
 
     _, org, repo, = *url.match(GitHubPackages::URL_REGEX)
 
     blob_url = "https://ghcr.io/v2/#{org}/#{repo}/#{name}/blobs/sha256:#{checksum}"
-    curl_download(blob_url, "--header", "Authorization: Bearer", to: temporary_path)
+    curl_download(blob_url, "--header", "Authorization: Bearer", to: temporary_path, timeout: timeout)
   end
 end
 

Library/Homebrew/github_packages.rb

@@ -11,7 +11,6 @@ class GitHubPackages
   extend T::Sig
 
   include Context
-  include Utils::Curl
 
   URL_DOMAIN = "ghcr.io"
   URL_PREFIX = "https://#{URL_DOMAIN}/v2/"
@@ -39,72 +38,67 @@ class GitHubPackages
     raise UsageError, "HOMEBREW_GITHUB_PACKAGES_USER is unset." if user.blank?
     raise UsageError, "HOMEBREW_GITHUB_PACKAGES_TOKEN is unset." if token.blank?
 
-    docker = HOMEBREW_PREFIX/"bin/docker"
+    skopeo = HOMEBREW_PREFIX/"bin/skopeo"
-    unless docker.exist?
+    unless skopeo.exist?
-      ohai "Installing `docker` for upload..."
+      ohai "Installing `skopeo` for upload..."
-      safe_system HOMEBREW_BREW_FILE, "install", "--formula", "docker"
+      safe_system HOMEBREW_BREW_FILE, "install", "--formula", "skopeo"
-      docker = Formula["docker"].opt_bin/"docker"
+      skopeo = Formula["skopeo"].opt_bin/"skopeo"
-    end
-
-    puts
-    system_command!(docker, verbose: true, print_stdout: true, input: token, args: [
-      "login", "--username", user, "--password-stdin", URL_DOMAIN
-    ])
-
-    oras = HOMEBREW_PREFIX/"bin/oras"
-    unless oras.exist?
-      ohai "Installing `oras` for upload..."
-      safe_system HOMEBREW_BREW_FILE, "install", "oras"
-      oras = Formula["oras"].opt_bin/"oras"
     end
 
     bottles_hash.each do |formula_name, bottle_hash|
+      upload_bottle(user, token, skopeo, formula_name, bottle_hash)
+    end
+  end
+
+  private
+
+  def upload_bottle(user, token, skopeo, formula_name, bottle_hash)
     _, org, repo, = *bottle_hash["bottle"]["root_url"].match(URL_REGEX)
 
-      # docker CLI insists on lowercase org ("repository name")
+    # docker/skopeo insist on lowercase org ("repository name")
     org = org.downcase
-      image = "#{URL_DOMAIN}/#{org}/#{repo}/#{formula_name}"
 
     version = bottle_hash["formula"]["pkg_version"]
     rebuild = if (rebuild = bottle_hash["bottle"]["rebuild"]).positive?
       ".#{rebuild}"
     end
+    version_rebuild = "#{version}#{rebuild}"
+    root = Pathname("#{formula_name}-#{version_rebuild}")
+
+    write_oci_layout(root)
+
+    blobs = root/"blobs/sha256"
+    blobs.mkpath
 
     formula_path = HOMEBREW_REPOSITORY/bottle_hash["formula"]["path"]
     formula = Formulary.factory(formula_path)
 
-      image_tags = bottle_hash["bottle"]["tags"].map do |bottle_tag, tag_hash|
-        local_file = tag_hash["local_filename"]
-        odebug "Uploading #{local_file}"
-
-        tag = "#{version}.#{bottle_tag}#{rebuild}"
-
-        tab = Tab.from_file_content(
-          Utils.safe_popen_read("tar", "xfO", local_file, "#{formula_name}/#{version}/INSTALL_RECEIPT.json"),
-          "#{local_file}/#{formula_name}/#{version}",
-        )
-        created_time = tab.source_modified_time
-        created_time ||= Time.now
-
     # TODO: ideally most/all of these attributes would be stored in the
     # bottle JSON rather than reading them from the formula.
     git_revision = formula.tap.git_head
     git_path = formula_path.to_s.delete_prefix("#{formula.tap.path}/")
-        manifest_hash = {
+    source = "https://github.com/#{org}/#{repo}/blob/#{git_revision}/#{git_path}"
-          "org.opencontainers.image.title"    => formula.full_name,
+
-          "org.opencontainers.image.url"      => formula.homepage,
+    formula_annotations_hash = {
-          "org.opencontainers.image.version"  => version,
+      "org.opencontainers.image.description" => formula.desc,
+      "org.opencontainers.image.license"     => formula.license,
       "org.opencontainers.image.revision"    => git_revision,
-          "org.opencontainers.image.source"   => "https://github.com/#{org}/#{repo}/blob/#{git_revision}/#{git_path}",
+      "org.opencontainers.image.source"      => source,
-          "org.opencontainers.image.created"  => created_time.strftime("%F"),
+      "org.opencontainers.image.url"         => formula.homepage,
+      "org.opencontainers.image.vendor"      => org,
+      "org.opencontainers.image.version"     => version,
     }
-        manifest_hash["org.opencontainers.image.description"] = formula.desc if formula.desc.present?
-        manifest_hash["org.opencontainers.image.license"] = formula.license if formula.license.present?
 
-        manifest_annotations = Pathname("#{formula_name}.#{tag}.annotations.json")
+    manifests = bottle_hash["bottle"]["tags"].map do |bottle_tag, tag_hash|
-        manifest_annotations.unlink if manifest_annotations.exist?
+      local_file = tag_hash["local_filename"]
-        manifest_annotations.write({ "$manifest" => manifest_hash }.to_json)
+      odebug "Uploading #{local_file}"
 
+      tar_gz_sha256 = write_tar_gz(local_file, blobs)
+
+      tab = Tab.from_file_content(
+        Utils.safe_popen_read("tar", "xfO", local_file, "#{formula_name}/#{version}/INSTALL_RECEIPT.json"),
+        "#{local_file}/#{formula_name}/#{version}",
+      )
       os_version = if tab.built_on.present?
         /(\d+\.)*\d+/ =~ tab.built_on["os_version"]
         Regexp.last_match(0)
@@ -112,8 +106,8 @@ class GitHubPackages
 
       # TODO: ideally most/all of these attributes would be stored in the
       # bottle JSON rather than reading them from the formula.
-        os, arch = if @bottle_tag.to_s.end_with?("_linux")
+      os, arch, formulae_dir = if @bottle_tag.to_s.end_with?("_linux")
-          ["linux", "amd64"]
+        ["linux", "amd64", "formula-linux"]
       else
         os = "darwin"
         macos_version = MacOS::Version.from_symbol(bottle_tag.to_sym)
@@ -123,63 +117,120 @@ class GitHubPackages
         else
           "amd64"
         end
-          [os, arch]
+        [os, arch, "formula"]
       end
 
+      platform_hash = {
+        architecture: arch,
+        os: os,
+        "os.version" => os_version,
+      }
       tar_sha256 = Digest::SHA256.hexdigest(
         Utils.safe_popen_read("gunzip", "--stdout", "--decompress", local_file),
       )
 
-        config_hash = {
+      config_json_sha256, config_json_size = write_config(platform_hash, tar_sha256, blobs)
-          "architecture" => arch,
+
-          "os"           => os,
+      created_time = tab.source_modified_time
-          "os.version"   => os_version,
+      created_time ||= Time.now
-          "rootfs"       => {
+      documentation = "https://formulae.brew.sh/#{formulae_dir}/#{formula_name}" if formula.tap.core_tap?
-            "type"     => "layers",
+      tag = "#{version}.#{bottle_tag}#{rebuild}"
-            "diff_ids" => ["sha256:#{tar_sha256}"],
+      title = "#{formula.full_name} #{tag}"
+
+      annotations_hash = formula_annotations_hash.merge({
+        "org.opencontainers.image.created"       => created_time.strftime("%F"),
+        "org.opencontainers.image.documentation" => documentation,
+        "org.opencontainers.image.ref.name"      => tag,
+        "org.opencontainers.image.title"         => title,
+      }).sort.to_h
+      annotations_hash.each do |key, value|
+        annotations_hash.delete(key) if value.blank?
+      end
+
+      manifest_json_sha256, manifest_json_size = write_hash(blobs, {
+        schemaVersion: 2,
+        config:        {
+          mediaType: "application/vnd.oci.image.config.v1+json",
+          digest:    "sha256:#{config_json_sha256}",
+          size:      config_json_size,
         },
+        layers:        [{
+          mediaType:   "application/vnd.oci.image.layer.v1.tar+gzip",
+          digest:      "sha256:#{tar_gz_sha256}",
+          size:        File.size(local_file),
+          annotations: {
+            "org.opencontainers.image.title": local_file,
+          },
+        }],
+        annotations:   annotations_hash,
+      })
+
+      {
+        mediaType: "application/vnd.oci.image.manifest.v1+json",
+        digest:    "sha256:#{manifest_json_sha256}",
+        size:      manifest_json_size,
+        platform:  platform_hash,
       }
-
-        manifest_config = Pathname("#{formula_name}.#{tag}.config.json")
-        manifest_config.unlink if manifest_config.exist?
-        manifest_config.write(config_hash.to_json)
-
-        # TODO: If we push the architecture-specific images to the tag :latest,
-        # then we don't need to delete the architecture-specific tags.
-        image_tag = "#{image}:#{tag}"
-        puts
-        system_command!(oras, verbose: true, print_stdout: true, args: [
-          "push", image_tag,
-          "--verbose",
-          "--manifest-annotations=#{manifest_annotations}",
-          "--manifest-config=#{manifest_config}:application/vnd.oci.image.config.v1+json",
-          "--username", user,
-          "--password", token,
-          "#{local_file}:application/vnd.oci.image.layer.v1.tar+gzip"
-        ])
-
-        image_tag
     end
 
-      image_tag = "#{image}:#{version}#{rebuild}"
+    index_json_sha256, index_json_size = write_index(manifests, blobs)
-      puts
-      system_command!(docker, verbose: true, print_stdout: true, args: [
-        "buildx", "imagetools", "create", "--tag", image_tag, *image_tags
-      ])
 
-      # TODO: once the main image metadata is working correctly delete the package using:
+    write_index_json(index_json_sha256, index_json_size, root)
-      # `curl -X DELETE -u $HOMEBREW_GITHUB_PACKAGES_USER:$HOMEBREW_GITHUB_PACKAGES_TOKEN
+
-      #  https://api.github.com/orgs/Homebrew/packages/container/homebrew-core%2F$PACKAGE/versions/$VERSION`
+    image = "#{URL_DOMAIN}/#{org}/#{repo}/#{formula_name}"
-      # Alternatively, if we push the architecture-specific images to the tag :latest,
+    image_tag = "#{image}:#{version_rebuild}"
-      # then we don't need to delete the architecture-specific tags.
-      # Alternatively, remove all usage of `docker` here instead.
-    end
-  ensure
-    if docker
     puts
-      system_command!(docker, verbose: true, print_stdout: true, args: [
+    system_command!(skopeo, verbose: true, print_stdout: true, args: [
-        "logout", URL_DOMAIN
+      "copy", "--dest-creds=#{user}:#{token}",
+      "oci:#{root}", "docker://#{image_tag}"
     ])
   end
+
+  def write_oci_layout(root)
+    write_hash(root, { imageLayoutVersion: "1.0.0" }, "oci-layout")
+  end
+
+  def write_tar_gz(local_file, blobs)
+    tar_gz_sha256 = Digest::SHA256.file(local_file)
+                                  .hexdigest
+    FileUtils.cp local_file, blobs/tar_gz_sha256
+    tar_gz_sha256
+  end
+
+  def write_config(platform_hash, tar_sha256, blobs)
+    write_hash(blobs, platform_hash.merge({
+      rootfs: {
+        type:     "layers",
+        diff_ids: ["sha256:#{tar_sha256}"],
+      },
+    }))
+  end
+
+  def write_index(manifests, blobs)
+    write_hash(blobs, {
+      schemaVersion: 2,
+      manifests:     manifests,
+    })
+  end
+
+  def write_index_json(index_json_sha256, index_json_size, root)
+    write_hash(root, {
+      schemaVersion: 2,
+      manifests:     [{
+        mediaType: "application/vnd.oci.image.index.v1+json",
+        digest:    "sha256:#{index_json_sha256}",
+        size:      index_json_size,
+      }],
+    }, "index.json")
+  end
+
+  def write_hash(directory, hash, _filename = nil)
+    json = hash.to_json
+    sha256 = Digest::SHA256.hexdigest(json)
+    path = directory/sha256
+    path.unlink if path.exist?
+    path.write(json)
+
+    [sha256, json.size]
   end
 end