From c3ea073a07cdf14b40102c72bf65419af83bb6ef Mon Sep 17 00:00:00 2001 From: Mike McQuaid Date: Fri, 26 Mar 2021 20:36:26 +0000 Subject: [PATCH 1/6] utils/gems: add dependencies to LOAD_PATH. --- Library/Homebrew/utils/gems.rb | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Library/Homebrew/utils/gems.rb b/Library/Homebrew/utils/gems.rb index a8ef197b22..bc2337834c 100644 --- a/Library/Homebrew/utils/gems.rb +++ b/Library/Homebrew/utils/gems.rb @@ -88,7 +88,10 @@ module Homebrew specs = Gem.install name, version, document: [] end - # Add the new specs to the $LOAD_PATH. + specs += specs.flat_map(&:runtime_dependencies) + .flat_map(&:to_specs) + + # Add the specs to the $LOAD_PATH. specs.each do |spec| spec.require_paths.each do |path| full_path = File.join(spec.full_gem_path, path) From 86130efd589455eed33d701756f0efd55ff5ce47 Mon Sep 17 00:00:00 2001 From: Mike McQuaid Date: Mon, 29 Mar 2021 14:35:16 +0100 Subject: [PATCH 2/6] download_strategy: use GitHubPackages constant. --- Library/Homebrew/download_strategy.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Library/Homebrew/download_strategy.rb b/Library/Homebrew/download_strategy.rb index 9b6292a3ef..a184483139 100644 --- a/Library/Homebrew/download_strategy.rb +++ b/Library/Homebrew/download_strategy.rb @@ -543,7 +543,7 @@ class CurlGitHubPackagesDownloadStrategy < CurlDownloadStrategy _, org, repo, = *url.match(GitHubPackages::URL_REGEX) - blob_url = "https://ghcr.io/v2/#{org}/#{repo}/#{name}/blobs/sha256:#{checksum}" + blob_url = "#{GitHubPackages::URL_PREFIX}#{org}/#{repo}/#{name}/blobs/sha256:#{checksum}" curl_download(blob_url, "--header", "Authorization: Bearer", to: temporary_path) end end From 5f31ebd8d4e617aed9bc73f5a831981590ad2824 Mon Sep 17 00:00:00 2001 From: Mike McQuaid Date: Fri, 26 Mar 2021 20:37:36 +0000 Subject: [PATCH 3/6] github_packages: use json_schemer gem. Instead of the deprecated json-schema. --- .gitignore | 6 +++-- Library/Homebrew/github_packages.rb | 39 +++++++++++++++++++--------- Library/Homebrew/sorbet/rbi/todo.rbi | 3 +-- 3 files changed, 32 insertions(+), 16 deletions(-) diff --git a/.gitignore b/.gitignore index bb179fb7f3..bdd631f909 100644 --- a/.gitignore +++ b/.gitignore @@ -84,7 +84,6 @@ **/vendor/bundle/ruby/*/gems/thread_safe-*/lib/thread_safe/util # Ignore dependencies we don't wish to vendor -**/vendor/bundle/ruby/*/gems/addressable-*/ **/vendor/bundle/ruby/*/gems/ast-*/ **/vendor/bundle/ruby/*/gems/bootsnap-*/ **/vendor/bundle/ruby/*/gems/bundler-*/ @@ -97,11 +96,14 @@ **/vendor/bundle/ruby/*/gems/diff-lcs-*/ **/vendor/bundle/ruby/*/gems/docile-*/ **/vendor/bundle/ruby/*/gems/domain_name-*/ +**/vendor/bundle/ruby/*/gems/ecma-re-validator-*/ +**/vendor/bundle/ruby/*/gems/hana-*/ **/vendor/bundle/ruby/*/gems/highline-*/ **/vendor/bundle/ruby/*/gems/http-cookie-*/ **/vendor/bundle/ruby/*/gems/hpricot-*/ **/vendor/bundle/ruby/*/gems/jaro_winkler-*/ **/vendor/bundle/ruby/*/gems/json-*/ +**/vendor/bundle/ruby/*/gems/json_schemer-*/ **/vendor/bundle/ruby/*/gems/method_source-*/ **/vendor/bundle/ruby/*/gems/mime-types-data-*/ **/vendor/bundle/ruby/*/gems/mime-types-*/ @@ -120,7 +122,6 @@ **/vendor/bundle/ruby/*/gems/powerpack-*/ **/vendor/bundle/ruby/*/gems/psych-*/ **/vendor/bundle/ruby/*/gems/pry-*/ -**/vendor/bundle/ruby/*/gems/public_suffix-*/ **/vendor/bundle/ruby/*/gems/racc-*/ **/vendor/bundle/ruby/*/gems/rainbow-*/ **/vendor/bundle/ruby/*/gems/rdiscount-*/ @@ -152,6 +153,7 @@ **/vendor/bundle/ruby/*/gems/unf_ext-*/ **/vendor/bundle/ruby/*/gems/unf-*/ **/vendor/bundle/ruby/*/gems/unicode-display_width-*/ +**/vendor/bundle/ruby/*/gems/uri_template-*/ **/vendor/bundle/ruby/*/gems/webrobots-*/ # Ignore `bin` contents (again). diff --git a/Library/Homebrew/github_packages.rb b/Library/Homebrew/github_packages.rb index 41a3acf057..a73262b97e 100644 --- a/Library/Homebrew/github_packages.rb +++ b/Library/Homebrew/github_packages.rb @@ -46,10 +46,8 @@ class GitHubPackages end # TODO: these dependencies are installed but cannot be required automatically. - Homebrew.install_gem!("public_suffix") - Homebrew.install_gem!("addressable") - Homebrew.install_gem!("json-schema") - require "json-schema" + Homebrew.install_gem!("json_schemer") + require "json_schemer" load_schemas! @@ -97,13 +95,30 @@ class GitHubPackages out, = curl_output(url) json = JSON.parse(out) + @schema_json ||= {} Array(uris).each do |uri| - schema = JSON::Schema.new(json, uri) - schema.uri = uri - JSON::Validator.add_schema(schema) + @schema_json[uri] = json end end + def schema_resolver(uri) + @schema_json[uri.to_s.gsub(/#.*/, "")] + end + + def validate_schema!(schema_uri, json) + schema = JSONSchemer.schema(@schema_json[schema_uri], ref_resolver: method(:schema_resolver)) + json = json.deep_stringify_keys + return if schema.valid?(json) + + puts + ofail "#{Formatter.url(schema_uri)} JSON schema validation failed!" + oh1 "Errors" + pp schema.validate(json).to_a + oh1 "JSON" + pp json + exit 1 + end + def upload_bottle(user, token, skopeo, formula_name, bottle_hash) _, org, repo, = *bottle_hash["bottle"]["root_url"].match(URL_REGEX) @@ -217,7 +232,7 @@ class GitHubPackages }], annotations: annotations_hash, } - JSON::Validator.validate!(IMAGE_MANIFEST_SCHEMA_URI, image_manifest) + validate_schema!(IMAGE_MANIFEST_SCHEMA_URI, image_manifest) manifest_json_sha256, manifest_json_size = write_hash(blobs, image_manifest) { @@ -244,7 +259,7 @@ class GitHubPackages def write_image_layout(root) image_layout = { imageLayoutVersion: "1.0.0" } - JSON::Validator.validate!(IMAGE_LAYOUT_SCHEMA_URI, image_layout) + validate_schema!(IMAGE_LAYOUT_SCHEMA_URI, image_layout) write_hash(root, image_layout, "oci-layout") end @@ -262,7 +277,7 @@ class GitHubPackages diff_ids: ["sha256:#{tar_sha256}"], }, }) - JSON::Validator.validate!(IMAGE_CONFIG_SCHEMA_URI, image_config) + validate_schema!(IMAGE_CONFIG_SCHEMA_URI, image_config) write_hash(blobs, image_config) end @@ -272,7 +287,7 @@ class GitHubPackages manifests: manifests, annotations: {}, } - JSON::Validator.validate!(IMAGE_INDEX_SCHEMA_URI, image_index) + validate_schema!(IMAGE_INDEX_SCHEMA_URI, image_index) write_hash(blobs, image_index) end @@ -287,7 +302,7 @@ class GitHubPackages }], annotations: {}, } - JSON::Validator.validate!(IMAGE_INDEX_SCHEMA_URI, index_json) + validate_schema!(IMAGE_INDEX_SCHEMA_URI, index_json) write_hash(root, index_json, "index.json") end diff --git a/Library/Homebrew/sorbet/rbi/todo.rbi b/Library/Homebrew/sorbet/rbi/todo.rbi index 50a72ebfcd..ca35a66298 100644 --- a/Library/Homebrew/sorbet/rbi/todo.rbi +++ b/Library/Homebrew/sorbet/rbi/todo.rbi @@ -4,8 +4,7 @@ # typed: strong module ::StackProf; end module DependencyCollector::Compat; end -module JSON::Schema; end -module JSON::Validator; end +module GitHubPackages::JSONSchemer; end module OS::Mac::Version::NULL; end module T::InterfaceWrapper::Helpers; end module T::Private::Abstract::Hooks; end From 490ffb70de3662b8887d2e2314318958d547830d Mon Sep 17 00:00:00 2001 From: Mike McQuaid Date: Mon, 29 Mar 2021 14:34:10 +0100 Subject: [PATCH 4/6] github_packages: import fixes from #10947 --- Library/Homebrew/github_packages.rb | 30 ++++++++++++++++++----------- 1 file changed, 19 insertions(+), 11 deletions(-) diff --git a/Library/Homebrew/github_packages.rb b/Library/Homebrew/github_packages.rb index a73262b97e..76953087e0 100644 --- a/Library/Homebrew/github_packages.rb +++ b/Library/Homebrew/github_packages.rb @@ -51,8 +51,8 @@ class GitHubPackages load_schemas! - bottles_hash.each do |formula_name, bottle_hash| - upload_bottle(user, token, skopeo, formula_name, bottle_hash) + bottles_hash.each_value do |bottle_hash| + upload_bottle(user, token, skopeo, bottle_hash) end end @@ -119,7 +119,11 @@ class GitHubPackages exit 1 end - def upload_bottle(user, token, skopeo, formula_name, bottle_hash) + def upload_bottle(user, token, skopeo, bottle_hash) + formula_path = HOMEBREW_REPOSITORY/bottle_hash["formula"]["path"] + formula = Formulary.factory(formula_path) + formula_name = formula.name + _, org, repo, = *bottle_hash["bottle"]["root_url"].match(URL_REGEX) # docker/skopeo insist on lowercase org ("repository name") @@ -138,9 +142,6 @@ class GitHubPackages blobs = root/"blobs/sha256" blobs.mkpath - formula_path = HOMEBREW_REPOSITORY/bottle_hash["formula"]["path"] - formula = Formulary.factory(formula_path) - # TODO: ideally most/all of these attributes would be stored in the # bottle JSON rather than reading them from the formula. git_revision = formula.tap.git_head @@ -148,6 +149,7 @@ class GitHubPackages source = "https://github.com/#{org}/#{repo}/blob/#{git_revision}/#{git_path}" formula_annotations_hash = { + "org.opencontainers.image.created" => Time.now.strftime("%F"), "org.opencontainers.image.description" => formula.desc, "org.opencontainers.image.license" => formula.license, "org.opencontainers.image.revision" => git_revision, @@ -156,6 +158,9 @@ class GitHubPackages "org.opencontainers.image.vendor" => org, "org.opencontainers.image.version" => version, } + formula_annotations_hash.each do |key, value| + formula_annotations_hash.delete(key) if value.blank? + end manifests = bottle_hash["bottle"]["tags"].map do |bottle_tag, tag_hash| local_file = tag_hash["local_filename"] @@ -174,7 +179,7 @@ class GitHubPackages # TODO: ideally most/all of these attributes would be stored in the # bottle JSON rather than reading them from the formula. - os, arch, formulae_dir = if @bottle_tag.to_s.end_with?("_linux") + os, arch, formulae_dir = if bottle_tag.to_s.end_with?("_linux") ["linux", "amd64", "formula-linux"] else os = "darwin" @@ -240,11 +245,13 @@ class GitHubPackages digest: "sha256:#{manifest_json_sha256}", size: manifest_json_size, platform: platform_hash, - annotations: {}, + annotations: { + "org.opencontainers.image.ref.name" => tag, + }, } end - index_json_sha256, index_json_size = write_image_index(manifests, blobs) + index_json_sha256, index_json_size = write_image_index(manifests, blobs, formula_annotations_hash) write_index_json(index_json_sha256, index_json_size, root) @@ -281,11 +288,12 @@ class GitHubPackages write_hash(blobs, image_config) end - def write_image_index(manifests, blobs) + def write_image_index(manifests, blobs, annotations) image_index = { + mediaType: "application/vnd.docker.distribution.manifest.list.v2+json", schemaVersion: 2, manifests: manifests, - annotations: {}, + annotations: annotations, } validate_schema!(IMAGE_INDEX_SCHEMA_URI, image_index) write_hash(blobs, image_index) From 725f2828b4ec27e43c4cc0086cc8b2894c91f7d2 Mon Sep 17 00:00:00 2001 From: Mike McQuaid Date: Mon, 29 Mar 2021 14:37:43 +0100 Subject: [PATCH 5/6] github_packages: add mediaType comment. --- Library/Homebrew/github_packages.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/Library/Homebrew/github_packages.rb b/Library/Homebrew/github_packages.rb index 76953087e0..47ff830b1c 100644 --- a/Library/Homebrew/github_packages.rb +++ b/Library/Homebrew/github_packages.rb @@ -290,6 +290,7 @@ class GitHubPackages def write_image_index(manifests, blobs, annotations) image_index = { + # Currently needed for correct multi-arch display in GitHub Packages UI mediaType: "application/vnd.docker.distribution.manifest.list.v2+json", schemaVersion: 2, manifests: manifests, From 7f1ac4cdae4ba46c3586d02438c7471455387c30 Mon Sep 17 00:00:00 2001 From: Mike McQuaid Date: Mon, 29 Mar 2021 14:37:51 +0100 Subject: [PATCH 6/6] github_packages: tweak package name and output URL --- Library/Homebrew/github_packages.rb | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/Library/Homebrew/github_packages.rb b/Library/Homebrew/github_packages.rb index 47ff830b1c..d1f6508570 100644 --- a/Library/Homebrew/github_packages.rb +++ b/Library/Homebrew/github_packages.rb @@ -126,9 +126,6 @@ class GitHubPackages _, org, repo, = *bottle_hash["bottle"]["root_url"].match(URL_REGEX) - # docker/skopeo insist on lowercase org ("repository name") - org = org.downcase - version = bottle_hash["formula"]["pkg_version"] rebuild = if (rebuild = bottle_hash["bottle"]["rebuild"]).positive? ".#{rebuild}" @@ -255,13 +252,17 @@ class GitHubPackages write_index_json(index_json_sha256, index_json_size, root) - image = "#{URL_DOMAIN}/#{org}/#{repo}/#{formula_name}" - image_tag = "#{image}:#{version_rebuild}" + # docker/skopeo insist on lowercase org ("repository name") + org_prefix = "#{URL_DOMAIN}/#{org.downcase}" + # remove redundant repo prefix for a shorter name + package_name = "#{repo.delete_prefix("homebrew-")}/#{formula_name}" + image_tag = "#{org_prefix}/#{package_name}:#{version_rebuild}" puts system_command!(skopeo, verbose: true, print_stdout: true, args: [ "copy", "--dest-creds=#{user}:#{token}", "oci:#{root}", "docker://#{image_tag}" ]) + ohai "Uploaded to https://github.com/orgs/Homebrew/packages/container/package/#{package_name}" end def write_image_layout(root)