github_packages: use skopeo.
`skopeo` allows replacing `docker` and `oras` with a single tool that behaves in a way that's a little more intuitive for us (we build the directory structure on disk and upload that). It seems to be better at preserving our metadata, too. Take the opportunity for writing out more metadata ourselves to split out more logic into separate functions. While we're here, fix the `download_strategy` timeout handling for GitHub Packages.
This commit is contained in:
Mike McQuaid
parent
97d4527eba
commit
74f31af8d4
@ -548,14 +548,14 @@ class CurlGitHubPackagesDownloadStrategy < CurlDownloadStrategy
|
||||
|
||||
private
|
||||
|
||||
def _fetch(url:, resolved_url:)
|
||||
def _fetch(url:, resolved_url:, timeout:)
|
||||
raise "Empty checksum" if checksum.blank?
|
||||
raise "Empty name" if name.blank?
|
||||
|
||||
_, org, repo, = *url.match(GitHubPackages::URL_REGEX)
|
||||
|
||||
blob_url = "https://ghcr.io/v2/#{org}/#{repo}/#{name}/blobs/sha256:#{checksum}"
|
||||
curl_download(blob_url, "--header", "Authorization: Bearer", to: temporary_path)
|
||||
curl_download(blob_url, "--header", "Authorization: Bearer", to: temporary_path, timeout: timeout)
|
||||
end
|
||||
end
|
||||
|
||||
|
||||
@ -11,7 +11,6 @@ class GitHubPackages
|
||||
extend T::Sig
|
||||
|
||||
include Context
|
||||
include Utils::Curl
|
||||
|
||||
URL_DOMAIN = "ghcr.io"
|
||||
URL_PREFIX = "https://#{URL_DOMAIN}/v2/"
|
||||
@ -39,72 +38,67 @@ class GitHubPackages
|
||||
raise UsageError, "HOMEBREW_GITHUB_PACKAGES_USER is unset." if user.blank?
|
||||
raise UsageError, "HOMEBREW_GITHUB_PACKAGES_TOKEN is unset." if token.blank?
|
||||
|
||||
docker = HOMEBREW_PREFIX/"bin/docker"
|
||||
unless docker.exist?
|
||||
ohai "Installing `docker` for upload..."
|
||||
safe_system HOMEBREW_BREW_FILE, "install", "--formula", "docker"
|
||||
docker = Formula["docker"].opt_bin/"docker"
|
||||
end
|
||||
|
||||
puts
|
||||
system_command!(docker, verbose: true, print_stdout: true, input: token, args: [
|
||||
"login", "--username", user, "--password-stdin", URL_DOMAIN
|
||||
])
|
||||
|
||||
oras = HOMEBREW_PREFIX/"bin/oras"
|
||||
unless oras.exist?
|
||||
ohai "Installing `oras` for upload..."
|
||||
safe_system HOMEBREW_BREW_FILE, "install", "oras"
|
||||
oras = Formula["oras"].opt_bin/"oras"
|
||||
skopeo = HOMEBREW_PREFIX/"bin/skopeo"
|
||||
unless skopeo.exist?
|
||||
ohai "Installing `skopeo` for upload..."
|
||||
safe_system HOMEBREW_BREW_FILE, "install", "--formula", "skopeo"
|
||||
skopeo = Formula["skopeo"].opt_bin/"skopeo"
|
||||
end
|
||||
|
||||
bottles_hash.each do |formula_name, bottle_hash|
|
||||
upload_bottle(user, token, skopeo, formula_name, bottle_hash)
|
||||
end
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def upload_bottle(user, token, skopeo, formula_name, bottle_hash)
|
||||
_, org, repo, = *bottle_hash["bottle"]["root_url"].match(URL_REGEX)
|
||||
|
||||
# docker CLI insists on lowercase org ("repository name")
|
||||
# docker/skopeo insist on lowercase org ("repository name")
|
||||
org = org.downcase
|
||||
image = "#{URL_DOMAIN}/#{org}/#{repo}/#{formula_name}"
|
||||
|
||||
version = bottle_hash["formula"]["pkg_version"]
|
||||
rebuild = if (rebuild = bottle_hash["bottle"]["rebuild"]).positive?
|
||||
".#{rebuild}"
|
||||
end
|
||||
version_rebuild = "#{version}#{rebuild}"
|
||||
root = Pathname("#{formula_name}-#{version_rebuild}")
|
||||
|
||||
write_oci_layout(root)
|
||||
|
||||
blobs = root/"blobs/sha256"
|
||||
blobs.mkpath
|
||||
|
||||
formula_path = HOMEBREW_REPOSITORY/bottle_hash["formula"]["path"]
|
||||
formula = Formulary.factory(formula_path)
|
||||
|
||||
image_tags = bottle_hash["bottle"]["tags"].map do |bottle_tag, tag_hash|
|
||||
local_file = tag_hash["local_filename"]
|
||||
odebug "Uploading #{local_file}"
|
||||
|
||||
tag = "#{version}.#{bottle_tag}#{rebuild}"
|
||||
|
||||
tab = Tab.from_file_content(
|
||||
Utils.safe_popen_read("tar", "xfO", local_file, "#{formula_name}/#{version}/INSTALL_RECEIPT.json"),
|
||||
"#{local_file}/#{formula_name}/#{version}",
|
||||
)
|
||||
created_time = tab.source_modified_time
|
||||
created_time ||= Time.now
|
||||
|
||||
# TODO: ideally most/all of these attributes would be stored in the
|
||||
# bottle JSON rather than reading them from the formula.
|
||||
git_revision = formula.tap.git_head
|
||||
git_path = formula_path.to_s.delete_prefix("#{formula.tap.path}/")
|
||||
manifest_hash = {
|
||||
"org.opencontainers.image.title" => formula.full_name,
|
||||
"org.opencontainers.image.url" => formula.homepage,
|
||||
"org.opencontainers.image.version" => version,
|
||||
source = "https://github.com/#{org}/#{repo}/blob/#{git_revision}/#{git_path}"
|
||||
|
||||
formula_annotations_hash = {
|
||||
"org.opencontainers.image.description" => formula.desc,
|
||||
"org.opencontainers.image.license" => formula.license,
|
||||
"org.opencontainers.image.revision" => git_revision,
|
||||
"org.opencontainers.image.source" => "https://github.com/#{org}/#{repo}/blob/#{git_revision}/#{git_path}",
|
||||
"org.opencontainers.image.created" => created_time.strftime("%F"),
|
||||
"org.opencontainers.image.source" => source,
|
||||
"org.opencontainers.image.url" => formula.homepage,
|
||||
"org.opencontainers.image.vendor" => org,
|
||||
"org.opencontainers.image.version" => version,
|
||||
}
|
||||
manifest_hash["org.opencontainers.image.description"] = formula.desc if formula.desc.present?
|
||||
manifest_hash["org.opencontainers.image.license"] = formula.license if formula.license.present?
|
||||
|
||||
manifest_annotations = Pathname("#{formula_name}.#{tag}.annotations.json")
|
||||
manifest_annotations.unlink if manifest_annotations.exist?
|
||||
manifest_annotations.write({ "$manifest" => manifest_hash }.to_json)
|
||||
manifests = bottle_hash["bottle"]["tags"].map do |bottle_tag, tag_hash|
|
||||
local_file = tag_hash["local_filename"]
|
||||
odebug "Uploading #{local_file}"
|
||||
|
||||
tar_gz_sha256 = write_tar_gz(local_file, blobs)
|
||||
|
||||
tab = Tab.from_file_content(
|
||||
Utils.safe_popen_read("tar", "xfO", local_file, "#{formula_name}/#{version}/INSTALL_RECEIPT.json"),
|
||||
"#{local_file}/#{formula_name}/#{version}",
|
||||
)
|
||||
os_version = if tab.built_on.present?
|
||||
/(\d+\.)*\d+/ =~ tab.built_on["os_version"]
|
||||
Regexp.last_match(0)
|
||||
@ -112,8 +106,8 @@ class GitHubPackages
|
||||
|
||||
# TODO: ideally most/all of these attributes would be stored in the
|
||||
# bottle JSON rather than reading them from the formula.
|
||||
os, arch = if @bottle_tag.to_s.end_with?("_linux")
|
||||
["linux", "amd64"]
|
||||
os, arch, formulae_dir = if @bottle_tag.to_s.end_with?("_linux")
|
||||
["linux", "amd64", "formula-linux"]
|
||||
else
|
||||
os = "darwin"
|
||||
macos_version = MacOS::Version.from_symbol(bottle_tag.to_sym)
|
||||
@ -123,63 +117,120 @@ class GitHubPackages
|
||||
else
|
||||
"amd64"
|
||||
end
|
||||
[os, arch]
|
||||
[os, arch, "formula"]
|
||||
end
|
||||
|
||||
platform_hash = {
|
||||
architecture: arch,
|
||||
os: os,
|
||||
"os.version" => os_version,
|
||||
}
|
||||
tar_sha256 = Digest::SHA256.hexdigest(
|
||||
Utils.safe_popen_read("gunzip", "--stdout", "--decompress", local_file),
|
||||
)
|
||||
|
||||
config_hash = {
|
||||
"architecture" => arch,
|
||||
"os" => os,
|
||||
"os.version" => os_version,
|
||||
"rootfs" => {
|
||||
"type" => "layers",
|
||||
"diff_ids" => ["sha256:#{tar_sha256}"],
|
||||
config_json_sha256, config_json_size = write_config(platform_hash, tar_sha256, blobs)
|
||||
|
||||
created_time = tab.source_modified_time
|
||||
created_time ||= Time.now
|
||||
documentation = "https://formulae.brew.sh/#{formulae_dir}/#{formula_name}" if formula.tap.core_tap?
|
||||
tag = "#{version}.#{bottle_tag}#{rebuild}"
|
||||
title = "#{formula.full_name} #{tag}"
|
||||
|
||||
annotations_hash = formula_annotations_hash.merge({
|
||||
"org.opencontainers.image.created" => created_time.strftime("%F"),
|
||||
"org.opencontainers.image.documentation" => documentation,
|
||||
"org.opencontainers.image.ref.name" => tag,
|
||||
"org.opencontainers.image.title" => title,
|
||||
}).sort.to_h
|
||||
annotations_hash.each do |key, value|
|
||||
annotations_hash.delete(key) if value.blank?
|
||||
end
|
||||
|
||||
manifest_json_sha256, manifest_json_size = write_hash(blobs, {
|
||||
schemaVersion: 2,
|
||||
config: {
|
||||
mediaType: "application/vnd.oci.image.config.v1+json",
|
||||
digest: "sha256:#{config_json_sha256}",
|
||||
size: config_json_size,
|
||||
},
|
||||
layers: [{
|
||||
mediaType: "application/vnd.oci.image.layer.v1.tar+gzip",
|
||||
digest: "sha256:#{tar_gz_sha256}",
|
||||
size: File.size(local_file),
|
||||
annotations: {
|
||||
"org.opencontainers.image.title": local_file,
|
||||
},
|
||||
}],
|
||||
annotations: annotations_hash,
|
||||
})
|
||||
|
||||
{
|
||||
mediaType: "application/vnd.oci.image.manifest.v1+json",
|
||||
digest: "sha256:#{manifest_json_sha256}",
|
||||
size: manifest_json_size,
|
||||
platform: platform_hash,
|
||||
}
|
||||
|
||||
manifest_config = Pathname("#{formula_name}.#{tag}.config.json")
|
||||
manifest_config.unlink if manifest_config.exist?
|
||||
manifest_config.write(config_hash.to_json)
|
||||
|
||||
# TODO: If we push the architecture-specific images to the tag :latest,
|
||||
# then we don't need to delete the architecture-specific tags.
|
||||
image_tag = "#{image}:#{tag}"
|
||||
puts
|
||||
system_command!(oras, verbose: true, print_stdout: true, args: [
|
||||
"push", image_tag,
|
||||
"--verbose",
|
||||
"--manifest-annotations=#{manifest_annotations}",
|
||||
"--manifest-config=#{manifest_config}:application/vnd.oci.image.config.v1+json",
|
||||
"--username", user,
|
||||
"--password", token,
|
||||
"#{local_file}:application/vnd.oci.image.layer.v1.tar+gzip"
|
||||
])
|
||||
|
||||
image_tag
|
||||
end
|
||||
|
||||
image_tag = "#{image}:#{version}#{rebuild}"
|
||||
puts
|
||||
system_command!(docker, verbose: true, print_stdout: true, args: [
|
||||
"buildx", "imagetools", "create", "--tag", image_tag, *image_tags
|
||||
])
|
||||
index_json_sha256, index_json_size = write_index(manifests, blobs)
|
||||
|
||||
# TODO: once the main image metadata is working correctly delete the package using:
|
||||
# `curl -X DELETE -u $HOMEBREW_GITHUB_PACKAGES_USER:$HOMEBREW_GITHUB_PACKAGES_TOKEN
|
||||
# https://api.github.com/orgs/Homebrew/packages/container/homebrew-core%2F$PACKAGE/versions/$VERSION`
|
||||
# Alternatively, if we push the architecture-specific images to the tag :latest,
|
||||
# then we don't need to delete the architecture-specific tags.
|
||||
# Alternatively, remove all usage of `docker` here instead.
|
||||
end
|
||||
ensure
|
||||
if docker
|
||||
write_index_json(index_json_sha256, index_json_size, root)
|
||||
|
||||
image = "#{URL_DOMAIN}/#{org}/#{repo}/#{formula_name}"
|
||||
image_tag = "#{image}:#{version_rebuild}"
|
||||
puts
|
||||
system_command!(docker, verbose: true, print_stdout: true, args: [
|
||||
"logout", URL_DOMAIN
|
||||
system_command!(skopeo, verbose: true, print_stdout: true, args: [
|
||||
"copy", "--dest-creds=#{user}:#{token}",
|
||||
"oci:#{root}", "docker://#{image_tag}"
|
||||
])
|
||||
end
|
||||
|
||||
def write_oci_layout(root)
|
||||
write_hash(root, { imageLayoutVersion: "1.0.0" }, "oci-layout")
|
||||
end
|
||||
|
||||
def write_tar_gz(local_file, blobs)
|
||||
tar_gz_sha256 = Digest::SHA256.file(local_file)
|
||||
.hexdigest
|
||||
FileUtils.cp local_file, blobs/tar_gz_sha256
|
||||
tar_gz_sha256
|
||||
end
|
||||
|
||||
def write_config(platform_hash, tar_sha256, blobs)
|
||||
write_hash(blobs, platform_hash.merge({
|
||||
rootfs: {
|
||||
type: "layers",
|
||||
diff_ids: ["sha256:#{tar_sha256}"],
|
||||
},
|
||||
}))
|
||||
end
|
||||
|
||||
def write_index(manifests, blobs)
|
||||
write_hash(blobs, {
|
||||
schemaVersion: 2,
|
||||
manifests: manifests,
|
||||
})
|
||||
end
|
||||
|
||||
def write_index_json(index_json_sha256, index_json_size, root)
|
||||
write_hash(root, {
|
||||
schemaVersion: 2,
|
||||
manifests: [{
|
||||
mediaType: "application/vnd.oci.image.index.v1+json",
|
||||
digest: "sha256:#{index_json_sha256}",
|
||||
size: index_json_size,
|
||||
}],
|
||||
}, "index.json")
|
||||
end
|
||||
|
||||
def write_hash(directory, hash, _filename = nil)
|
||||
json = hash.to_json
|
||||
sha256 = Digest::SHA256.hexdigest(json)
|
||||
path = directory/sha256
|
||||
path.unlink if path.exist?
|
||||
path.write(json)
|
||||
|
||||
[sha256, json.size]
|
||||
end
|
||||
end
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user