From 74f31af8d4842ab98f1ce76d761847d30f2de073 Mon Sep 17 00:00:00 2001 From: Mike McQuaid Date: Fri, 19 Mar 2021 21:09:18 +0000 Subject: [PATCH] github_packages: use skopeo. `skopeo` allows replacing `docker` and `oras` with a single tool that behaves in a way that's a little more intuitive for us (we build the directory structure on disk and upload that). It seems to be better at preserving our metadata, too. Take the opportunity for writing out more metadata ourselves to split out more logic into separate functions. While we're here, fix the `download_strategy` timeout handling for GitHub Packages. --- Library/Homebrew/download_strategy.rb | 4 +- Library/Homebrew/github_packages.rb | 329 +++++++++++++++----------- 2 files changed, 192 insertions(+), 141 deletions(-) diff --git a/Library/Homebrew/download_strategy.rb b/Library/Homebrew/download_strategy.rb index 76e5b5302f..fffb0ad86a 100644 --- a/Library/Homebrew/download_strategy.rb +++ b/Library/Homebrew/download_strategy.rb @@ -548,14 +548,14 @@ class CurlGitHubPackagesDownloadStrategy < CurlDownloadStrategy private - def _fetch(url:, resolved_url:) + def _fetch(url:, resolved_url:, timeout:) raise "Empty checksum" if checksum.blank? raise "Empty name" if name.blank? _, org, repo, = *url.match(GitHubPackages::URL_REGEX) blob_url = "https://ghcr.io/v2/#{org}/#{repo}/#{name}/blobs/sha256:#{checksum}" - curl_download(blob_url, "--header", "Authorization: Bearer", to: temporary_path) + curl_download(blob_url, "--header", "Authorization: Bearer", to: temporary_path, timeout: timeout) end end diff --git a/Library/Homebrew/github_packages.rb b/Library/Homebrew/github_packages.rb index ff54e49282..9178b2724d 100644 --- a/Library/Homebrew/github_packages.rb +++ b/Library/Homebrew/github_packages.rb @@ -11,7 +11,6 @@ class GitHubPackages extend T::Sig include Context - include Utils::Curl URL_DOMAIN = "ghcr.io" URL_PREFIX = "https://#{URL_DOMAIN}/v2/" @@ -39,147 +38,199 @@ class GitHubPackages raise UsageError, "HOMEBREW_GITHUB_PACKAGES_USER is unset." if user.blank? raise UsageError, "HOMEBREW_GITHUB_PACKAGES_TOKEN is unset." if token.blank? - docker = HOMEBREW_PREFIX/"bin/docker" - unless docker.exist? - ohai "Installing `docker` for upload..." - safe_system HOMEBREW_BREW_FILE, "install", "--formula", "docker" - docker = Formula["docker"].opt_bin/"docker" - end - - puts - system_command!(docker, verbose: true, print_stdout: true, input: token, args: [ - "login", "--username", user, "--password-stdin", URL_DOMAIN - ]) - - oras = HOMEBREW_PREFIX/"bin/oras" - unless oras.exist? - ohai "Installing `oras` for upload..." - safe_system HOMEBREW_BREW_FILE, "install", "oras" - oras = Formula["oras"].opt_bin/"oras" + skopeo = HOMEBREW_PREFIX/"bin/skopeo" + unless skopeo.exist? + ohai "Installing `skopeo` for upload..." + safe_system HOMEBREW_BREW_FILE, "install", "--formula", "skopeo" + skopeo = Formula["skopeo"].opt_bin/"skopeo" end bottles_hash.each do |formula_name, bottle_hash| - _, org, repo, = *bottle_hash["bottle"]["root_url"].match(URL_REGEX) - - # docker CLI insists on lowercase org ("repository name") - org = org.downcase - image = "#{URL_DOMAIN}/#{org}/#{repo}/#{formula_name}" - - version = bottle_hash["formula"]["pkg_version"] - rebuild = if (rebuild = bottle_hash["bottle"]["rebuild"]).positive? - ".#{rebuild}" - end - - formula_path = HOMEBREW_REPOSITORY/bottle_hash["formula"]["path"] - formula = Formulary.factory(formula_path) - - image_tags = bottle_hash["bottle"]["tags"].map do |bottle_tag, tag_hash| - local_file = tag_hash["local_filename"] - odebug "Uploading #{local_file}" - - tag = "#{version}.#{bottle_tag}#{rebuild}" - - tab = Tab.from_file_content( - Utils.safe_popen_read("tar", "xfO", local_file, "#{formula_name}/#{version}/INSTALL_RECEIPT.json"), - "#{local_file}/#{formula_name}/#{version}", - ) - created_time = tab.source_modified_time - created_time ||= Time.now - - # TODO: ideally most/all of these attributes would be stored in the - # bottle JSON rather than reading them from the formula. - git_revision = formula.tap.git_head - git_path = formula_path.to_s.delete_prefix("#{formula.tap.path}/") - manifest_hash = { - "org.opencontainers.image.title" => formula.full_name, - "org.opencontainers.image.url" => formula.homepage, - "org.opencontainers.image.version" => version, - "org.opencontainers.image.revision" => git_revision, - "org.opencontainers.image.source" => "https://github.com/#{org}/#{repo}/blob/#{git_revision}/#{git_path}", - "org.opencontainers.image.created" => created_time.strftime("%F"), - } - manifest_hash["org.opencontainers.image.description"] = formula.desc if formula.desc.present? - manifest_hash["org.opencontainers.image.license"] = formula.license if formula.license.present? - - manifest_annotations = Pathname("#{formula_name}.#{tag}.annotations.json") - manifest_annotations.unlink if manifest_annotations.exist? - manifest_annotations.write({ "$manifest" => manifest_hash }.to_json) - - os_version = if tab.built_on.present? - /(\d+\.)*\d+/ =~ tab.built_on["os_version"] - Regexp.last_match(0) - end - - # TODO: ideally most/all of these attributes would be stored in the - # bottle JSON rather than reading them from the formula. - os, arch = if @bottle_tag.to_s.end_with?("_linux") - ["linux", "amd64"] - else - os = "darwin" - macos_version = MacOS::Version.from_symbol(bottle_tag.to_sym) - os_version ||= macos_version.to_f.to_s - arch = if macos_version.arch == :arm64 - "arm64" - else - "amd64" - end - [os, arch] - end - - tar_sha256 = Digest::SHA256.hexdigest( - Utils.safe_popen_read("gunzip", "--stdout", "--decompress", local_file), - ) - - config_hash = { - "architecture" => arch, - "os" => os, - "os.version" => os_version, - "rootfs" => { - "type" => "layers", - "diff_ids" => ["sha256:#{tar_sha256}"], - }, - } - - manifest_config = Pathname("#{formula_name}.#{tag}.config.json") - manifest_config.unlink if manifest_config.exist? - manifest_config.write(config_hash.to_json) - - # TODO: If we push the architecture-specific images to the tag :latest, - # then we don't need to delete the architecture-specific tags. - image_tag = "#{image}:#{tag}" - puts - system_command!(oras, verbose: true, print_stdout: true, args: [ - "push", image_tag, - "--verbose", - "--manifest-annotations=#{manifest_annotations}", - "--manifest-config=#{manifest_config}:application/vnd.oci.image.config.v1+json", - "--username", user, - "--password", token, - "#{local_file}:application/vnd.oci.image.layer.v1.tar+gzip" - ]) - - image_tag - end - - image_tag = "#{image}:#{version}#{rebuild}" - puts - system_command!(docker, verbose: true, print_stdout: true, args: [ - "buildx", "imagetools", "create", "--tag", image_tag, *image_tags - ]) - - # TODO: once the main image metadata is working correctly delete the package using: - # `curl -X DELETE -u $HOMEBREW_GITHUB_PACKAGES_USER:$HOMEBREW_GITHUB_PACKAGES_TOKEN - # https://api.github.com/orgs/Homebrew/packages/container/homebrew-core%2F$PACKAGE/versions/$VERSION` - # Alternatively, if we push the architecture-specific images to the tag :latest, - # then we don't need to delete the architecture-specific tags. - # Alternatively, remove all usage of `docker` here instead. - end - ensure - if docker - puts - system_command!(docker, verbose: true, print_stdout: true, args: [ - "logout", URL_DOMAIN - ]) + upload_bottle(user, token, skopeo, formula_name, bottle_hash) end end + + private + + def upload_bottle(user, token, skopeo, formula_name, bottle_hash) + _, org, repo, = *bottle_hash["bottle"]["root_url"].match(URL_REGEX) + + # docker/skopeo insist on lowercase org ("repository name") + org = org.downcase + + version = bottle_hash["formula"]["pkg_version"] + rebuild = if (rebuild = bottle_hash["bottle"]["rebuild"]).positive? + ".#{rebuild}" + end + version_rebuild = "#{version}#{rebuild}" + root = Pathname("#{formula_name}-#{version_rebuild}") + + write_oci_layout(root) + + blobs = root/"blobs/sha256" + blobs.mkpath + + formula_path = HOMEBREW_REPOSITORY/bottle_hash["formula"]["path"] + formula = Formulary.factory(formula_path) + + # TODO: ideally most/all of these attributes would be stored in the + # bottle JSON rather than reading them from the formula. + git_revision = formula.tap.git_head + git_path = formula_path.to_s.delete_prefix("#{formula.tap.path}/") + source = "https://github.com/#{org}/#{repo}/blob/#{git_revision}/#{git_path}" + + formula_annotations_hash = { + "org.opencontainers.image.description" => formula.desc, + "org.opencontainers.image.license" => formula.license, + "org.opencontainers.image.revision" => git_revision, + "org.opencontainers.image.source" => source, + "org.opencontainers.image.url" => formula.homepage, + "org.opencontainers.image.vendor" => org, + "org.opencontainers.image.version" => version, + } + + manifests = bottle_hash["bottle"]["tags"].map do |bottle_tag, tag_hash| + local_file = tag_hash["local_filename"] + odebug "Uploading #{local_file}" + + tar_gz_sha256 = write_tar_gz(local_file, blobs) + + tab = Tab.from_file_content( + Utils.safe_popen_read("tar", "xfO", local_file, "#{formula_name}/#{version}/INSTALL_RECEIPT.json"), + "#{local_file}/#{formula_name}/#{version}", + ) + os_version = if tab.built_on.present? + /(\d+\.)*\d+/ =~ tab.built_on["os_version"] + Regexp.last_match(0) + end + + # TODO: ideally most/all of these attributes would be stored in the + # bottle JSON rather than reading them from the formula. + os, arch, formulae_dir = if @bottle_tag.to_s.end_with?("_linux") + ["linux", "amd64", "formula-linux"] + else + os = "darwin" + macos_version = MacOS::Version.from_symbol(bottle_tag.to_sym) + os_version ||= macos_version.to_f.to_s + arch = if macos_version.arch == :arm64 + "arm64" + else + "amd64" + end + [os, arch, "formula"] + end + + platform_hash = { + architecture: arch, + os: os, + "os.version" => os_version, + } + tar_sha256 = Digest::SHA256.hexdigest( + Utils.safe_popen_read("gunzip", "--stdout", "--decompress", local_file), + ) + + config_json_sha256, config_json_size = write_config(platform_hash, tar_sha256, blobs) + + created_time = tab.source_modified_time + created_time ||= Time.now + documentation = "https://formulae.brew.sh/#{formulae_dir}/#{formula_name}" if formula.tap.core_tap? + tag = "#{version}.#{bottle_tag}#{rebuild}" + title = "#{formula.full_name} #{tag}" + + annotations_hash = formula_annotations_hash.merge({ + "org.opencontainers.image.created" => created_time.strftime("%F"), + "org.opencontainers.image.documentation" => documentation, + "org.opencontainers.image.ref.name" => tag, + "org.opencontainers.image.title" => title, + }).sort.to_h + annotations_hash.each do |key, value| + annotations_hash.delete(key) if value.blank? + end + + manifest_json_sha256, manifest_json_size = write_hash(blobs, { + schemaVersion: 2, + config: { + mediaType: "application/vnd.oci.image.config.v1+json", + digest: "sha256:#{config_json_sha256}", + size: config_json_size, + }, + layers: [{ + mediaType: "application/vnd.oci.image.layer.v1.tar+gzip", + digest: "sha256:#{tar_gz_sha256}", + size: File.size(local_file), + annotations: { + "org.opencontainers.image.title": local_file, + }, + }], + annotations: annotations_hash, + }) + + { + mediaType: "application/vnd.oci.image.manifest.v1+json", + digest: "sha256:#{manifest_json_sha256}", + size: manifest_json_size, + platform: platform_hash, + } + end + + index_json_sha256, index_json_size = write_index(manifests, blobs) + + write_index_json(index_json_sha256, index_json_size, root) + + image = "#{URL_DOMAIN}/#{org}/#{repo}/#{formula_name}" + image_tag = "#{image}:#{version_rebuild}" + puts + system_command!(skopeo, verbose: true, print_stdout: true, args: [ + "copy", "--dest-creds=#{user}:#{token}", + "oci:#{root}", "docker://#{image_tag}" + ]) + end + + def write_oci_layout(root) + write_hash(root, { imageLayoutVersion: "1.0.0" }, "oci-layout") + end + + def write_tar_gz(local_file, blobs) + tar_gz_sha256 = Digest::SHA256.file(local_file) + .hexdigest + FileUtils.cp local_file, blobs/tar_gz_sha256 + tar_gz_sha256 + end + + def write_config(platform_hash, tar_sha256, blobs) + write_hash(blobs, platform_hash.merge({ + rootfs: { + type: "layers", + diff_ids: ["sha256:#{tar_sha256}"], + }, + })) + end + + def write_index(manifests, blobs) + write_hash(blobs, { + schemaVersion: 2, + manifests: manifests, + }) + end + + def write_index_json(index_json_sha256, index_json_size, root) + write_hash(root, { + schemaVersion: 2, + manifests: [{ + mediaType: "application/vnd.oci.image.index.v1+json", + digest: "sha256:#{index_json_sha256}", + size: index_json_size, + }], + }, "index.json") + end + + def write_hash(directory, hash, _filename = nil) + json = hash.to_json + sha256 = Digest::SHA256.hexdigest(json) + path = directory/sha256 + path.unlink if path.exist? + path.write(json) + + [sha256, json.size] + end end