diff --git a/Library/Homebrew/cmd/fetch.rb b/Library/Homebrew/cmd/fetch.rb index b6ce4a8ca6..7ef29695ee 100644 --- a/Library/Homebrew/cmd/fetch.rb +++ b/Library/Homebrew/cmd/fetch.rb @@ -88,6 +88,7 @@ module Homebrew fetched_bottle = false if fetch_bottle?(f, args: args) begin + f.fetch_bottle_tab fetch_formula(f.bottle, args: args) rescue Interrupt raise diff --git a/Library/Homebrew/dev-cmd/bottle.rb b/Library/Homebrew/dev-cmd/bottle.rb index 3924598f05..2d30e39b31 100644 --- a/Library/Homebrew/dev-cmd/bottle.rb +++ b/Library/Homebrew/dev-cmd/bottle.rb @@ -540,6 +540,7 @@ module Homebrew "prefix" => bottle.prefix, "cellar" => bottle.cellar.to_s, "rebuild" => bottle.rebuild, + "date" => Time.now.strftime("%F"), "tags" => { bottle_tag.to_s => { "filename" => filename.bintray, diff --git a/Library/Homebrew/download_strategy.rb b/Library/Homebrew/download_strategy.rb index bd6ddc00cd..088dbb6b71 100644 --- a/Library/Homebrew/download_strategy.rb +++ b/Library/Homebrew/download_strategy.rb @@ -426,7 +426,7 @@ class CurlDownloadStrategy < AbstractFileDownloadStrategy url = url.sub(%r{^((ht|f)tps?://)?}, "#{domain.chomp("/")}/") end - out, _, status= curl_output("--location", "--silent", "--head", "--request", "GET", url.to_s) + out, _, status = curl_output("--location", "--silent", "--head", "--request", "GET", url.to_s) lines = status.success? ? out.lines.map(&:chomp) : [] @@ -533,20 +533,18 @@ end # # @api public class CurlGitHubPackagesDownloadStrategy < CurlDownloadStrategy - attr_accessor :checksum, :name + attr_writer :resolved_basename + + def initialize(url, name, version, **meta) + meta ||= {} + meta[:header] = "Authorization: Bearer" + super(url, name, version, meta) + end private - def _fetch(url:, resolved_url:) - raise CurlDownloadStrategyError, "Empty checksum" if checksum.blank? - raise CurlDownloadStrategyError, "Empty name" if name.blank? - - _, org, repo, = *url.match(GitHubPackages::URL_REGEX) - - # remove redundant repo prefix for a shorter name - repo = repo.delete_prefix("homebrew-") - blob_url = "#{GitHubPackages::URL_PREFIX}#{org}/#{repo}/#{name}/blobs/sha256:#{checksum}" - curl_download(blob_url, "--header", "Authorization: Bearer", to: temporary_path) + def resolved_basename + @resolved_basename.presence || super end end diff --git a/Library/Homebrew/formula.rb b/Library/Homebrew/formula.rb index bd4d1dd6da..bd49409185 100644 --- a/Library/Homebrew/formula.rb +++ b/Library/Homebrew/formula.rb @@ -2211,6 +2211,20 @@ class Formula patchlist.select(&:external?).each(&:fetch) end + sig { void } + def fetch_bottle_tab + return unless bottled? + + T.must(bottle).fetch_tab + end + + sig { returns(Hash) } + def bottle_tab_attributes + return {} unless bottled? + + T.must(bottle).tab_attributes + end + private def prepare_patches diff --git a/Library/Homebrew/formula_installer.rb b/Library/Homebrew/formula_installer.rb index d09970819c..67fd981421 100644 --- a/Library/Homebrew/formula_installer.rb +++ b/Library/Homebrew/formula_installer.rb @@ -1100,7 +1100,9 @@ class FormulaInstaller return if only_deps? - unless pour_bottle?(output_warning: true) + if pour_bottle?(output_warning: true) + formula.fetch_bottle_tab + else formula.fetch_patches formula.resources.each(&:fetch) end @@ -1124,14 +1126,17 @@ class FormulaInstaller end keg = Keg.new(formula.prefix) - tab = Tab.for_keg(keg) Tab.clear_cache + tab = if (tab_attributes = formula.bottle_tab_attributes.presence) + Tab.from_file_content(tab_attributes.to_json, keg/Tab::FILENAME) + else + Tab.for_keg(keg) + end + skip_linkage = formula.bottle_specification.skip_relocation? keg.replace_placeholders_with_locations tab.changed_files, skip_linkage: skip_linkage - tab = Tab.for_keg(keg) - unless ignore_deps? CxxStdlib.check_compatibility( formula, formula.recursive_dependencies, diff --git a/Library/Homebrew/formulary.rb b/Library/Homebrew/formulary.rb index 7c61238115..b6dfe53522 100644 --- a/Library/Homebrew/formulary.rb +++ b/Library/Homebrew/formulary.rb @@ -192,8 +192,8 @@ module Formulary end def get_formula(spec, force_bottle: false, flags: [], **) - contents = Utils::Bottles.formula_contents @bottle_filename, name: name formula = begin + contents = Utils::Bottles.formula_contents @bottle_filename, name: name Formulary.from_contents(name, path, contents, spec, force_bottle: force_bottle, flags: flags) rescue FormulaUnreadableError => e opoo <<~EOS @@ -201,6 +201,12 @@ module Formulary #{e} EOS super + rescue BottleFormulaUnavailableError => e + opoo <<~EOS + #{e} + Falling back to non-bottle formula. + EOS + super end formula.local_bottle_path = @bottle_filename formula diff --git a/Library/Homebrew/github_packages.rb b/Library/Homebrew/github_packages.rb index f3d628a34a..309efc6751 100644 --- a/Library/Homebrew/github_packages.rb +++ b/Library/Homebrew/github_packages.rb @@ -68,6 +68,32 @@ class GitHubPackages end end + def self.version_rebuild(version, rebuild, bottle_tag = nil) + bottle_tag = (".#{bottle_tag}" if bottle_tag.present?) + + rebuild = if rebuild.to_i.positive? + if bottle_tag + ".#{rebuild}" + else + "-#{rebuild}" + end + end + + "#{version}#{bottle_tag}#{rebuild}" + end + + def self.repo_without_prefix(repo) + # remove redundant repo prefix for a shorter name + repo.delete_prefix("homebrew-") + end + + def self.root_url(org, repo, prefix = URL_PREFIX) + # docker/skopeo insist on lowercase org ("repository name") + org = org.downcase + + "#{prefix}#{org}/#{repo_without_prefix(repo)}" + end + private IMAGE_CONFIG_SCHEMA_URI = "https://opencontainers.org/schema/image/config" @@ -138,10 +164,8 @@ class GitHubPackages repo = "homebrew-#{repo}" unless HOMEBREW_OFFICIAL_REPO_PREFIXES_REGEX.match?(repo) version = bottle_hash["formula"]["pkg_version"] - rebuild = if (rebuild = bottle_hash["bottle"]["rebuild"]).positive? - ".#{rebuild}" - end - version_rebuild = "#{version}#{rebuild}" + rebuild = bottle_hash["bottle"]["rebuild"] + version_rebuild = GitHubPackages.version_rebuild(version, rebuild) root = Pathname("#{formula_name}--#{version_rebuild}") FileUtils.rm_rf root @@ -161,8 +185,9 @@ class GitHubPackages remote end + created_date = bottle_hash["bottle"]["date"] formula_annotations_hash = { - "org.opencontainers.image.created" => Time.now.strftime("%F"), + "org.opencontainers.image.created" => created_date, "org.opencontainers.image.description" => bottle_hash["formula"]["desc"], "org.opencontainers.image.documentation" => documentation, "org.opencontainers.image.license" => bottle_hash["formula"]["license"], @@ -185,16 +210,34 @@ class GitHubPackages tar_gz_sha256 = write_tar_gz(local_file, blobs) tab = tag_hash["tab"] - architecture = TAB_ARCH_TO_PLATFORM_ARCHITECTURE[tab["arch"]] + architecture = if tab["arch"].present? + TAB_ARCH_TO_PLATFORM_ARCHITECTURE[tab["arch"]] + elsif bottle_tag.to_s.start_with?("arm64") + "arm64" + else + "amd64" + end raise TypeError, "unknown tab['arch']: #{tab["arch"]}" if architecture.blank? - os = BUILT_ON_OS_TO_PLATFORM_OS[tab["built_on"]["os"]] + os = if tab["built_on"].present? && tab["built_on"]["os"].present? + BUILT_ON_OS_TO_PLATFORM_OS[tab["built_on"]["os"]] + elsif bottle_tag.to_s.end_with?("_linux") + "linux" + else + "darwin" + end raise TypeError, "unknown tab['built_on']['os']: #{tab["built_on"]["os"]}" if os.blank? + os_version = if tab["built_on"].present? && tab["built_on"]["os_version"].present? + tab["built_on"]["os_version"] + else + MacOS::Version.from_symbol(bottle_tag).to_s + end + platform_hash = { architecture: architecture, os: os, - "os.version" => tab["built_on"]["os_version"], + "os.version" => os_version, } tar_sha256 = Digest::SHA256.hexdigest( Utils.safe_popen_read("gunzip", "--stdout", "--decompress", local_file), @@ -205,10 +248,10 @@ class GitHubPackages formulae_dir = tag_hash["formulae_brew_sh_path"] documentation = "https://formulae.brew.sh/#{formulae_dir}/#{formula_name}" if formula_core_tap - tag = "#{version}.#{bottle_tag}#{rebuild}" + tag = GitHubPackages.version_rebuild(version, rebuild, bottle_tag) annotations_hash = formula_annotations_hash.merge({ - "org.opencontainers.image.created" => Time.at(tag_hash["tab"]["source_modified_time"]).strftime("%F"), + "org.opencontainers.image.created" => created_date, "org.opencontainers.image.documentation" => documentation, "org.opencontainers.image.ref.name" => tag, "org.opencontainers.image.title" => "#{formula_full_name} #{tag}", @@ -255,11 +298,8 @@ class GitHubPackages write_index_json(index_json_sha256, index_json_size, root, "org.opencontainers.image.ref.name" => version_rebuild) - # docker/skopeo insist on lowercase org ("repository name") - org_prefix = "#{DOCKER_PREFIX}#{org.downcase}" - # remove redundant repo prefix for a shorter name - package_name = "#{repo.delete_prefix("homebrew-")}/#{formula_name}" - image_tag = "#{org_prefix}/#{package_name}:#{version_rebuild}" + image_tag = "#{GitHubPackages.root_url(org, repo, DOCKER_PREFIX)}/#{formula_name}:#{version_rebuild}" + puts args = ["copy", "--all", "oci:#{root}", image_tag.to_s] if dry_run @@ -267,6 +307,7 @@ class GitHubPackages else args << "--dest-creds=#{user}:#{token}" system_command!(skopeo, verbose: true, print_stdout: true, args: args) + package_name = "#{GitHubPackages.repo_without_prefix(repo)}/#{formula_name}" ohai "Uploaded to https://github.com/orgs/Homebrew/packages/container/package/#{package_name}" end end diff --git a/Library/Homebrew/software_spec.rb b/Library/Homebrew/software_spec.rb index 51839f3d88..1cf353a6f1 100644 --- a/Library/Homebrew/software_spec.rb +++ b/Library/Homebrew/software_spec.rb @@ -304,9 +304,18 @@ class Bottle checksum, tag, cellar = spec.checksum_for(Utils::Bottles.tag) - filename = Filename.create(formula, tag, spec.rebuild) - @resource.url("#{spec.root_url}/#{filename.bintray}", - select_download_strategy(spec.root_url_specs)) + filename = Filename.create(formula, tag, spec.rebuild).bintray + + # TODO: this will need adjusted when if we use GitHub Packages by default + path, resolved_basename = if (bottle_domain = Homebrew::EnvConfig.bottle_domain.presence) && + bottle_domain.start_with?(GitHubPackages::URL_PREFIX) + ["#{@name}/blobs/sha256:#{checksum}", filename] + else + filename + end + + @resource.url("#{spec.root_url}/#{path}", select_download_strategy(spec.root_url_specs)) + @resource.downloader.resolved_basename = resolved_basename if resolved_basename.present? @resource.version = formula.pkg_version @resource.checksum = checksum @prefix = spec.prefix @@ -316,16 +325,12 @@ class Bottle def fetch(verify_download_integrity: true) # add the default bottle domain as a fallback mirror - # TODO: this may need adjusted when if we use GitHub Packages by default if @resource.download_strategy == CurlDownloadStrategy && @resource.url.start_with?(Homebrew::EnvConfig.bottle_domain) fallback_url = @resource.url .sub(/^#{Regexp.escape(Homebrew::EnvConfig.bottle_domain)}/, HOMEBREW_BOTTLE_DEFAULT_DOMAIN) @resource.mirror(fallback_url) if [@resource.url, *@resource.mirrors].exclude?(fallback_url) - elsif @resource.download_strategy == CurlGitHubPackagesDownloadStrategy - @resource.downloader.name = @name - @resource.downloader.checksum = @resource.checksum.hexdigest end @resource.fetch(verify_download_integrity: verify_download_integrity) end @@ -343,8 +348,62 @@ class Bottle resource.downloader.stage end + def fetch_tab + # a checksum is used later identifying the correct tab but we do not have the checksum for the manifest/tab + github_packages_manifest_resource&.fetch(verify_download_integrity: false) + end + + def tab_attributes + return {} unless github_packages_manifest_resource&.downloaded? + + manifest_json = github_packages_manifest_resource.cached_download.read + + json = begin + JSON.parse(manifest_json) + rescue JSON::ParserError + raise ArgumentError, "Couldn't parse manifest JSON." + end + + manifests = json["manifests"] + raise ArgumentError, "Missing 'manifests' section." if manifests.blank? + + manifests_annotations = manifests.map { |m| m["annotations"] } + raise ArgumentError, "Missing 'annotations' section." if manifests_annotations.blank? + + bottle_checksum = @resource.checksum.hexdigest + manifest_annotations = manifests_annotations.find do |m| + m["sh.brew.bottle.checksum"] == bottle_checksum + end + raise ArgumentError, "Couldn't find manifest matching bottle checksum." if manifest_annotations.blank? + + tab = manifest_annotations["sh.brew.tab"] + raise ArgumentError, "Couldn't find tab from manifest." if tab.blank? + + begin + JSON.parse(tab) + rescue JSON::ParserError + raise ArgumentError, "Couldn't parse tab JSON." + end + end + private + def github_packages_manifest_resource + return if @resource.download_strategy != CurlGitHubPackagesDownloadStrategy + + @github_packages_manifest_resource ||= begin + resource = Resource.new("#{name}_bottle_manifest") + + version_rebuild = GitHubPackages.version_rebuild(@resource.version, rebuild) + resource.version(version_rebuild) + + resource.url("#{@spec.root_url}/#{name}/manifests/#{version_rebuild}", + using: CurlGitHubPackagesDownloadStrategy) + resource.downloader.resolved_basename = "#{name}-#{version_rebuild}.bottle_manifest.json" + resource + end + end + def select_download_strategy(specs) specs[:using] ||= DownloadStrategyDetector.detect(@spec.root_url) specs @@ -380,7 +439,7 @@ class BottleSpecification def root_url(var = nil, specs = {}) if var.nil? @root_url ||= if Homebrew::EnvConfig.bottle_domain.start_with?(GitHubPackages::URL_PREFIX) - "#{GitHubPackages::URL_PREFIX}#{tap.full_name}" + GitHubPackages.root_url(tap.user, tap.repo).to_s else "#{Homebrew::EnvConfig.bottle_domain}/#{Utils::Bottles::Bintray.repository(tap)}" end diff --git a/Library/Homebrew/utils/bottles.rb b/Library/Homebrew/utils/bottles.rb index 7c2150c315..d01b8582a6 100644 --- a/Library/Homebrew/utils/bottles.rb +++ b/Library/Homebrew/utils/bottles.rb @@ -39,12 +39,6 @@ module Utils HOMEBREW_BOTTLES_EXTNAME_REGEX.match(filename).to_a end - # TODO: remove when removed from brew-test-bot - sig { returns(Regexp) } - def native_regex - /(\.#{Regexp.escape(tag.to_s)}\.bottle\.(\d+\.)?tar\.gz)$/o - end - def receipt_path(bottle_file) path = Utils.popen_read("tar", "-tzf", bottle_file).lines.map(&:chomp).find do |line| line =~ %r{.+/.+/INSTALL_RECEIPT.json}