Merge pull request #10891 from MikeMcQuaid/github_packages_skopeo
github_packages: use skopeo.
This commit is contained in:
Mike McQuaid
GitHub
commit
0c98e3756d
@ -548,14 +548,14 @@ class CurlGitHubPackagesDownloadStrategy < CurlDownloadStrategy
|
|||||||
|
|
||||||
private
|
private
|
||||||
|
|
||||||
def _fetch(url:, resolved_url:)
|
def _fetch(url:, resolved_url:, timeout:)
|
||||||
raise "Empty checksum" if checksum.blank?
|
raise "Empty checksum" if checksum.blank?
|
||||||
raise "Empty name" if name.blank?
|
raise "Empty name" if name.blank?
|
||||||
|
|
||||||
_, org, repo, = *url.match(GitHubPackages::URL_REGEX)
|
_, org, repo, = *url.match(GitHubPackages::URL_REGEX)
|
||||||
|
|
||||||
blob_url = "https://ghcr.io/v2/#{org}/#{repo}/#{name}/blobs/sha256:#{checksum}"
|
blob_url = "https://ghcr.io/v2/#{org}/#{repo}/#{name}/blobs/sha256:#{checksum}"
|
||||||
curl_download(blob_url, "--header", "Authorization: Bearer", to: temporary_path)
|
curl_download(blob_url, "--header", "Authorization: Bearer", to: temporary_path, timeout: timeout)
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|
||||||
|
|||||||
@ -11,7 +11,6 @@ class GitHubPackages
|
|||||||
extend T::Sig
|
extend T::Sig
|
||||||
|
|
||||||
include Context
|
include Context
|
||||||
include Utils::Curl
|
|
||||||
|
|
||||||
URL_DOMAIN = "ghcr.io"
|
URL_DOMAIN = "ghcr.io"
|
||||||
URL_PREFIX = "https://#{URL_DOMAIN}/v2/"
|
URL_PREFIX = "https://#{URL_DOMAIN}/v2/"
|
||||||
@ -39,147 +38,199 @@ class GitHubPackages
|
|||||||
raise UsageError, "HOMEBREW_GITHUB_PACKAGES_USER is unset." if user.blank?
|
raise UsageError, "HOMEBREW_GITHUB_PACKAGES_USER is unset." if user.blank?
|
||||||
raise UsageError, "HOMEBREW_GITHUB_PACKAGES_TOKEN is unset." if token.blank?
|
raise UsageError, "HOMEBREW_GITHUB_PACKAGES_TOKEN is unset." if token.blank?
|
||||||
|
|
||||||
docker = HOMEBREW_PREFIX/"bin/docker"
|
skopeo = HOMEBREW_PREFIX/"bin/skopeo"
|
||||||
unless docker.exist?
|
unless skopeo.exist?
|
||||||
ohai "Installing `docker` for upload..."
|
ohai "Installing `skopeo` for upload..."
|
||||||
safe_system HOMEBREW_BREW_FILE, "install", "--formula", "docker"
|
safe_system HOMEBREW_BREW_FILE, "install", "--formula", "skopeo"
|
||||||
docker = Formula["docker"].opt_bin/"docker"
|
skopeo = Formula["skopeo"].opt_bin/"skopeo"
|
||||||
end
|
|
||||||
|
|
||||||
puts
|
|
||||||
system_command!(docker, verbose: true, print_stdout: true, input: token, args: [
|
|
||||||
"login", "--username", user, "--password-stdin", URL_DOMAIN
|
|
||||||
])
|
|
||||||
|
|
||||||
oras = HOMEBREW_PREFIX/"bin/oras"
|
|
||||||
unless oras.exist?
|
|
||||||
ohai "Installing `oras` for upload..."
|
|
||||||
safe_system HOMEBREW_BREW_FILE, "install", "oras"
|
|
||||||
oras = Formula["oras"].opt_bin/"oras"
|
|
||||||
end
|
end
|
||||||
|
|
||||||
bottles_hash.each do |formula_name, bottle_hash|
|
bottles_hash.each do |formula_name, bottle_hash|
|
||||||
_, org, repo, = *bottle_hash["bottle"]["root_url"].match(URL_REGEX)
|
upload_bottle(user, token, skopeo, formula_name, bottle_hash)
|
||||||
|
|
||||||
# docker CLI insists on lowercase org ("repository name")
|
|
||||||
org = org.downcase
|
|
||||||
image = "#{URL_DOMAIN}/#{org}/#{repo}/#{formula_name}"
|
|
||||||
|
|
||||||
version = bottle_hash["formula"]["pkg_version"]
|
|
||||||
rebuild = if (rebuild = bottle_hash["bottle"]["rebuild"]).positive?
|
|
||||||
".#{rebuild}"
|
|
||||||
end
|
|
||||||
|
|
||||||
formula_path = HOMEBREW_REPOSITORY/bottle_hash["formula"]["path"]
|
|
||||||
formula = Formulary.factory(formula_path)
|
|
||||||
|
|
||||||
image_tags = bottle_hash["bottle"]["tags"].map do |bottle_tag, tag_hash|
|
|
||||||
local_file = tag_hash["local_filename"]
|
|
||||||
odebug "Uploading #{local_file}"
|
|
||||||
|
|
||||||
tag = "#{version}.#{bottle_tag}#{rebuild}"
|
|
||||||
|
|
||||||
tab = Tab.from_file_content(
|
|
||||||
Utils.safe_popen_read("tar", "xfO", local_file, "#{formula_name}/#{version}/INSTALL_RECEIPT.json"),
|
|
||||||
"#{local_file}/#{formula_name}/#{version}",
|
|
||||||
)
|
|
||||||
created_time = tab.source_modified_time
|
|
||||||
created_time ||= Time.now
|
|
||||||
|
|
||||||
# TODO: ideally most/all of these attributes would be stored in the
|
|
||||||
# bottle JSON rather than reading them from the formula.
|
|
||||||
git_revision = formula.tap.git_head
|
|
||||||
git_path = formula_path.to_s.delete_prefix("#{formula.tap.path}/")
|
|
||||||
manifest_hash = {
|
|
||||||
"org.opencontainers.image.title" => formula.full_name,
|
|
||||||
"org.opencontainers.image.url" => formula.homepage,
|
|
||||||
"org.opencontainers.image.version" => version,
|
|
||||||
"org.opencontainers.image.revision" => git_revision,
|
|
||||||
"org.opencontainers.image.source" => "https://github.com/#{org}/#{repo}/blob/#{git_revision}/#{git_path}",
|
|
||||||
"org.opencontainers.image.created" => created_time.strftime("%F"),
|
|
||||||
}
|
|
||||||
manifest_hash["org.opencontainers.image.description"] = formula.desc if formula.desc.present?
|
|
||||||
manifest_hash["org.opencontainers.image.license"] = formula.license if formula.license.present?
|
|
||||||
|
|
||||||
manifest_annotations = Pathname("#{formula_name}.#{tag}.annotations.json")
|
|
||||||
manifest_annotations.unlink if manifest_annotations.exist?
|
|
||||||
manifest_annotations.write({ "$manifest" => manifest_hash }.to_json)
|
|
||||||
|
|
||||||
os_version = if tab.built_on.present?
|
|
||||||
/(\d+\.)*\d+/ =~ tab.built_on["os_version"]
|
|
||||||
Regexp.last_match(0)
|
|
||||||
end
|
|
||||||
|
|
||||||
# TODO: ideally most/all of these attributes would be stored in the
|
|
||||||
# bottle JSON rather than reading them from the formula.
|
|
||||||
os, arch = if @bottle_tag.to_s.end_with?("_linux")
|
|
||||||
["linux", "amd64"]
|
|
||||||
else
|
|
||||||
os = "darwin"
|
|
||||||
macos_version = MacOS::Version.from_symbol(bottle_tag.to_sym)
|
|
||||||
os_version ||= macos_version.to_f.to_s
|
|
||||||
arch = if macos_version.arch == :arm64
|
|
||||||
"arm64"
|
|
||||||
else
|
|
||||||
"amd64"
|
|
||||||
end
|
|
||||||
[os, arch]
|
|
||||||
end
|
|
||||||
|
|
||||||
tar_sha256 = Digest::SHA256.hexdigest(
|
|
||||||
Utils.safe_popen_read("gunzip", "--stdout", "--decompress", local_file),
|
|
||||||
)
|
|
||||||
|
|
||||||
config_hash = {
|
|
||||||
"architecture" => arch,
|
|
||||||
"os" => os,
|
|
||||||
"os.version" => os_version,
|
|
||||||
"rootfs" => {
|
|
||||||
"type" => "layers",
|
|
||||||
"diff_ids" => ["sha256:#{tar_sha256}"],
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
manifest_config = Pathname("#{formula_name}.#{tag}.config.json")
|
|
||||||
manifest_config.unlink if manifest_config.exist?
|
|
||||||
manifest_config.write(config_hash.to_json)
|
|
||||||
|
|
||||||
# TODO: If we push the architecture-specific images to the tag :latest,
|
|
||||||
# then we don't need to delete the architecture-specific tags.
|
|
||||||
image_tag = "#{image}:#{tag}"
|
|
||||||
puts
|
|
||||||
system_command!(oras, verbose: true, print_stdout: true, args: [
|
|
||||||
"push", image_tag,
|
|
||||||
"--verbose",
|
|
||||||
"--manifest-annotations=#{manifest_annotations}",
|
|
||||||
"--manifest-config=#{manifest_config}:application/vnd.oci.image.config.v1+json",
|
|
||||||
"--username", user,
|
|
||||||
"--password", token,
|
|
||||||
"#{local_file}:application/vnd.oci.image.layer.v1.tar+gzip"
|
|
||||||
])
|
|
||||||
|
|
||||||
image_tag
|
|
||||||
end
|
|
||||||
|
|
||||||
image_tag = "#{image}:#{version}#{rebuild}"
|
|
||||||
puts
|
|
||||||
system_command!(docker, verbose: true, print_stdout: true, args: [
|
|
||||||
"buildx", "imagetools", "create", "--tag", image_tag, *image_tags
|
|
||||||
])
|
|
||||||
|
|
||||||
# TODO: once the main image metadata is working correctly delete the package using:
|
|
||||||
# `curl -X DELETE -u $HOMEBREW_GITHUB_PACKAGES_USER:$HOMEBREW_GITHUB_PACKAGES_TOKEN
|
|
||||||
# https://api.github.com/orgs/Homebrew/packages/container/homebrew-core%2F$PACKAGE/versions/$VERSION`
|
|
||||||
# Alternatively, if we push the architecture-specific images to the tag :latest,
|
|
||||||
# then we don't need to delete the architecture-specific tags.
|
|
||||||
# Alternatively, remove all usage of `docker` here instead.
|
|
||||||
end
|
|
||||||
ensure
|
|
||||||
if docker
|
|
||||||
puts
|
|
||||||
system_command!(docker, verbose: true, print_stdout: true, args: [
|
|
||||||
"logout", URL_DOMAIN
|
|
||||||
])
|
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|
||||||
|
private
|
||||||
|
|
||||||
|
def upload_bottle(user, token, skopeo, formula_name, bottle_hash)
|
||||||
|
_, org, repo, = *bottle_hash["bottle"]["root_url"].match(URL_REGEX)
|
||||||
|
|
||||||
|
# docker/skopeo insist on lowercase org ("repository name")
|
||||||
|
org = org.downcase
|
||||||
|
|
||||||
|
version = bottle_hash["formula"]["pkg_version"]
|
||||||
|
rebuild = if (rebuild = bottle_hash["bottle"]["rebuild"]).positive?
|
||||||
|
".#{rebuild}"
|
||||||
|
end
|
||||||
|
version_rebuild = "#{version}#{rebuild}"
|
||||||
|
root = Pathname("#{formula_name}-#{version_rebuild}")
|
||||||
|
|
||||||
|
write_oci_layout(root)
|
||||||
|
|
||||||
|
blobs = root/"blobs/sha256"
|
||||||
|
blobs.mkpath
|
||||||
|
|
||||||
|
formula_path = HOMEBREW_REPOSITORY/bottle_hash["formula"]["path"]
|
||||||
|
formula = Formulary.factory(formula_path)
|
||||||
|
|
||||||
|
# TODO: ideally most/all of these attributes would be stored in the
|
||||||
|
# bottle JSON rather than reading them from the formula.
|
||||||
|
git_revision = formula.tap.git_head
|
||||||
|
git_path = formula_path.to_s.delete_prefix("#{formula.tap.path}/")
|
||||||
|
source = "https://github.com/#{org}/#{repo}/blob/#{git_revision}/#{git_path}"
|
||||||
|
|
||||||
|
formula_annotations_hash = {
|
||||||
|
"org.opencontainers.image.description" => formula.desc,
|
||||||
|
"org.opencontainers.image.license" => formula.license,
|
||||||
|
"org.opencontainers.image.revision" => git_revision,
|
||||||
|
"org.opencontainers.image.source" => source,
|
||||||
|
"org.opencontainers.image.url" => formula.homepage,
|
||||||
|
"org.opencontainers.image.vendor" => org,
|
||||||
|
"org.opencontainers.image.version" => version,
|
||||||
|
}
|
||||||
|
|
||||||
|
manifests = bottle_hash["bottle"]["tags"].map do |bottle_tag, tag_hash|
|
||||||
|
local_file = tag_hash["local_filename"]
|
||||||
|
odebug "Uploading #{local_file}"
|
||||||
|
|
||||||
|
tar_gz_sha256 = write_tar_gz(local_file, blobs)
|
||||||
|
|
||||||
|
tab = Tab.from_file_content(
|
||||||
|
Utils.safe_popen_read("tar", "xfO", local_file, "#{formula_name}/#{version}/INSTALL_RECEIPT.json"),
|
||||||
|
"#{local_file}/#{formula_name}/#{version}",
|
||||||
|
)
|
||||||
|
os_version = if tab.built_on.present?
|
||||||
|
/(\d+\.)*\d+/ =~ tab.built_on["os_version"]
|
||||||
|
Regexp.last_match(0)
|
||||||
|
end
|
||||||
|
|
||||||
|
# TODO: ideally most/all of these attributes would be stored in the
|
||||||
|
# bottle JSON rather than reading them from the formula.
|
||||||
|
os, arch, formulae_dir = if @bottle_tag.to_s.end_with?("_linux")
|
||||||
|
["linux", "amd64", "formula-linux"]
|
||||||
|
else
|
||||||
|
os = "darwin"
|
||||||
|
macos_version = MacOS::Version.from_symbol(bottle_tag.to_sym)
|
||||||
|
os_version ||= macos_version.to_f.to_s
|
||||||
|
arch = if macos_version.arch == :arm64
|
||||||
|
"arm64"
|
||||||
|
else
|
||||||
|
"amd64"
|
||||||
|
end
|
||||||
|
[os, arch, "formula"]
|
||||||
|
end
|
||||||
|
|
||||||
|
platform_hash = {
|
||||||
|
architecture: arch,
|
||||||
|
os: os,
|
||||||
|
"os.version" => os_version,
|
||||||
|
}
|
||||||
|
tar_sha256 = Digest::SHA256.hexdigest(
|
||||||
|
Utils.safe_popen_read("gunzip", "--stdout", "--decompress", local_file),
|
||||||
|
)
|
||||||
|
|
||||||
|
config_json_sha256, config_json_size = write_config(platform_hash, tar_sha256, blobs)
|
||||||
|
|
||||||
|
created_time = tab.source_modified_time
|
||||||
|
created_time ||= Time.now
|
||||||
|
documentation = "https://formulae.brew.sh/#{formulae_dir}/#{formula_name}" if formula.tap.core_tap?
|
||||||
|
tag = "#{version}.#{bottle_tag}#{rebuild}"
|
||||||
|
title = "#{formula.full_name} #{tag}"
|
||||||
|
|
||||||
|
annotations_hash = formula_annotations_hash.merge({
|
||||||
|
"org.opencontainers.image.created" => created_time.strftime("%F"),
|
||||||
|
"org.opencontainers.image.documentation" => documentation,
|
||||||
|
"org.opencontainers.image.ref.name" => tag,
|
||||||
|
"org.opencontainers.image.title" => title,
|
||||||
|
}).sort.to_h
|
||||||
|
annotations_hash.each do |key, value|
|
||||||
|
annotations_hash.delete(key) if value.blank?
|
||||||
|
end
|
||||||
|
|
||||||
|
manifest_json_sha256, manifest_json_size = write_hash(blobs, {
|
||||||
|
schemaVersion: 2,
|
||||||
|
config: {
|
||||||
|
mediaType: "application/vnd.oci.image.config.v1+json",
|
||||||
|
digest: "sha256:#{config_json_sha256}",
|
||||||
|
size: config_json_size,
|
||||||
|
},
|
||||||
|
layers: [{
|
||||||
|
mediaType: "application/vnd.oci.image.layer.v1.tar+gzip",
|
||||||
|
digest: "sha256:#{tar_gz_sha256}",
|
||||||
|
size: File.size(local_file),
|
||||||
|
annotations: {
|
||||||
|
"org.opencontainers.image.title": local_file,
|
||||||
|
},
|
||||||
|
}],
|
||||||
|
annotations: annotations_hash,
|
||||||
|
})
|
||||||
|
|
||||||
|
{
|
||||||
|
mediaType: "application/vnd.oci.image.manifest.v1+json",
|
||||||
|
digest: "sha256:#{manifest_json_sha256}",
|
||||||
|
size: manifest_json_size,
|
||||||
|
platform: platform_hash,
|
||||||
|
}
|
||||||
|
end
|
||||||
|
|
||||||
|
index_json_sha256, index_json_size = write_index(manifests, blobs)
|
||||||
|
|
||||||
|
write_index_json(index_json_sha256, index_json_size, root)
|
||||||
|
|
||||||
|
image = "#{URL_DOMAIN}/#{org}/#{repo}/#{formula_name}"
|
||||||
|
image_tag = "#{image}:#{version_rebuild}"
|
||||||
|
puts
|
||||||
|
system_command!(skopeo, verbose: true, print_stdout: true, args: [
|
||||||
|
"copy", "--dest-creds=#{user}:#{token}",
|
||||||
|
"oci:#{root}", "docker://#{image_tag}"
|
||||||
|
])
|
||||||
|
end
|
||||||
|
|
||||||
|
def write_oci_layout(root)
|
||||||
|
write_hash(root, { imageLayoutVersion: "1.0.0" }, "oci-layout")
|
||||||
|
end
|
||||||
|
|
||||||
|
def write_tar_gz(local_file, blobs)
|
||||||
|
tar_gz_sha256 = Digest::SHA256.file(local_file)
|
||||||
|
.hexdigest
|
||||||
|
FileUtils.cp local_file, blobs/tar_gz_sha256
|
||||||
|
tar_gz_sha256
|
||||||
|
end
|
||||||
|
|
||||||
|
def write_config(platform_hash, tar_sha256, blobs)
|
||||||
|
write_hash(blobs, platform_hash.merge({
|
||||||
|
rootfs: {
|
||||||
|
type: "layers",
|
||||||
|
diff_ids: ["sha256:#{tar_sha256}"],
|
||||||
|
},
|
||||||
|
}))
|
||||||
|
end
|
||||||
|
|
||||||
|
def write_index(manifests, blobs)
|
||||||
|
write_hash(blobs, {
|
||||||
|
schemaVersion: 2,
|
||||||
|
manifests: manifests,
|
||||||
|
})
|
||||||
|
end
|
||||||
|
|
||||||
|
def write_index_json(index_json_sha256, index_json_size, root)
|
||||||
|
write_hash(root, {
|
||||||
|
schemaVersion: 2,
|
||||||
|
manifests: [{
|
||||||
|
mediaType: "application/vnd.oci.image.index.v1+json",
|
||||||
|
digest: "sha256:#{index_json_sha256}",
|
||||||
|
size: index_json_size,
|
||||||
|
}],
|
||||||
|
}, "index.json")
|
||||||
|
end
|
||||||
|
|
||||||
|
def write_hash(directory, hash, _filename = nil)
|
||||||
|
json = hash.to_json
|
||||||
|
sha256 = Digest::SHA256.hexdigest(json)
|
||||||
|
path = directory/sha256
|
||||||
|
path.unlink if path.exist?
|
||||||
|
path.write(json)
|
||||||
|
|
||||||
|
[sha256, json.size]
|
||||||
|
end
|
||||||
end
|
end
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user