From 442aa9170422ca830c3f75889bae3ef4890e2f79 Mon Sep 17 00:00:00 2001 From: Mike McQuaid Date: Thu, 9 May 2024 13:10:35 +0100 Subject: [PATCH] SBOM improvements - write a schema when installing formulae (if not already present) - cache the schema on disk rather than downloading it every time - make more methods/attributes `private` - allow validation to be optional, only enable for Homebrew developers at installation time - use the tab for more, correct information - ensure that dependencies/bottles are written correctly - use new SBOM 3 schema URL - improve test coverage --- Library/Homebrew/dev-cmd/bottle.rb | 6 +- Library/Homebrew/formula_installer.rb | 9 + Library/Homebrew/sbom.rb | 223 ++++++++++-------- .../Homebrew/test/formula_installer_spec.rb | 1 + Library/Homebrew/test/sbom_spec.rb | 66 +++--- 5 files changed, 175 insertions(+), 130 deletions(-) diff --git a/Library/Homebrew/dev-cmd/bottle.rb b/Library/Homebrew/dev-cmd/bottle.rb index 3245c516cc..fa87397904 100644 --- a/Library/Homebrew/dev-cmd/bottle.rb +++ b/Library/Homebrew/dev-cmd/bottle.rb @@ -96,13 +96,13 @@ module Homebrew sig { override.void } def run - Homebrew.install_bundler_gems!(groups: ["bottle"]) - if args.merge? Homebrew.install_bundler_gems!(groups: ["ast"]) return merge end + Homebrew.install_bundler_gems!(groups: ["bottle"]) + gnu_tar_formula_ensure_installed_if_needed! args.named.to_resolved_formulae(uniq: false).each do |formula| @@ -508,7 +508,7 @@ module Homebrew tab.write end - sbom = SBOM.create(formula) + sbom = SBOM.create(formula, tab) sbom.write keg.consistent_reproducible_symlink_permissions! diff --git a/Library/Homebrew/formula_installer.rb b/Library/Homebrew/formula_installer.rb index 1475a595ff..b26c77ca3a 100644 --- a/Library/Homebrew/formula_installer.rb +++ b/Library/Homebrew/formula_installer.rb @@ -23,6 +23,7 @@ require "deprecate_disable" require "unlink" require "service" require "attestation" +require "sbom" # Installer for a formula. class FormulaInstaller @@ -828,6 +829,12 @@ on_request: installed_on_request?, options:) tab.runtime_dependencies = Tab.runtime_deps_hash(formula, f_runtime_deps) tab.write + # write a SBOM file (if we don't already have one and aren't bottling) + if !build_bottle? && !SBOM.exist?(formula) + sbom = SBOM.create(formula, tab) + sbom.write(validate: Homebrew::EnvConfig.developer?) + end + # let's reset Utils::Git.available? if we just installed git Utils::Git.clear_available_cache if formula.name == "git" @@ -1216,6 +1223,8 @@ on_request: installed_on_request?, options:) def fetch return if previously_fetched_formula + SBOM.fetch_schema! if Homebrew::EnvConfig.developer? + fetch_dependencies return if only_deps? diff --git a/Library/Homebrew/sbom.rb b/Library/Homebrew/sbom.rb index 68971bcc91..8741a86431 100644 --- a/Library/Homebrew/sbom.rb +++ b/Library/Homebrew/sbom.rb @@ -12,37 +12,25 @@ class SBOM extend Cachable FILENAME = "sbom.spdx.json" - SCHEMA = "https://raw.githubusercontent.com/spdx/spdx-spec/v2.3/schemas/spdx-schema.json" - - attr_accessor :homebrew_version, :spdxfile, :built_as_bottle, :installed_as_dependency, :installed_on_request, - :changed_files, :poured_from_bottle, :loaded_from_api, :time, :stdlib, :aliases, :arch, :source, - :built_on, :license, :name - attr_writer :compiler, :runtime_dependencies, :source_modified_time + SCHEMA_URL = "https://spdx.github.io/spdx-3-model/model.jsonld" + SCHEMA_FILENAME = "sbom.spdx.schema.3.json" + SCHEMA_CACHE_TARGET = (HOMEBREW_CACHE/"sbom/#{SCHEMA_FILENAME}").freeze # Instantiates a {SBOM} for a new installation of a formula. - sig { params(formula: Formula, compiler: T.nilable(String), stdlib: T.nilable(String)).returns(T.attached_class) } - def self.create(formula, compiler: nil, stdlib: nil) - runtime_deps = formula.runtime_formula_dependencies(undeclared: false) - + sig { params(formula: Formula, tab: Tab).returns(T.attached_class) } + def self.create(formula, tab) attributes = { - name: formula.name, - homebrew_version: HOMEBREW_VERSION, - spdxfile: formula.prefix/FILENAME, - built_as_bottle: formula.build.bottle?, - installed_as_dependency: false, - installed_on_request: false, - poured_from_bottle: false, - loaded_from_api: false, - time: Time.now.to_i, - source_modified_time: formula.source_modified_time.to_i, - compiler:, - stdlib:, - aliases: formula.aliases, - runtime_dependencies: SBOM.runtime_deps_hash(runtime_deps), - arch: Hardware::CPU.arch, - license: SPDX.license_expression_to_string(formula.license), - built_on: DevelopmentTools.build_system_info, - source: { + name: formula.name, + homebrew_version: HOMEBREW_VERSION, + spdxfile: SBOM.spdxfile(formula), + time: Time.now.to_i, + source_modified_time: tab.source_modified_time.to_i, + compiler: tab.compiler, + stdlib: tab.stdlib, + runtime_dependencies: SBOM.runtime_deps_hash(Array(tab.runtime_dependencies)), + license: SPDX.license_expression_to_string(formula.license), + built_on: DevelopmentTools.build_system_info, + source: { path: formula.specified_path.to_s, tap: formula.tap&.name, tap_git_head: nil, # Filled in later if possible @@ -63,44 +51,109 @@ class SBOM new(attributes) end - sig { params(attributes: Hash).void } - def initialize(attributes = {}) - attributes.each { |key, value| instance_variable_set(:"@#{key}", value) } + sig { params(formula: Formula).returns(Pathname) } + def self.spdxfile(formula) + formula.prefix/FILENAME + end + + sig { params(deps: T::Array[T::Hash[String, String]]).returns(T::Array[T::Hash[String, String]]) } + def self.runtime_deps_hash(deps) + deps.map do |dep| + full_name = dep.fetch("full_name") + dep_formula = Formula[full_name] + { + "full_name" => full_name, + "pkg_version" => dep.fetch("pkg_version"), + "name" => dep_formula.name, + "license" => SPDX.license_expression_to_string(dep_formula.license), + "bottle" => dep_formula.bottle_hash, + "formula_pkg_version" => dep_formula.pkg_version.to_s, + } + end + end + + sig { params(formula: Formula).returns(T::Boolean) } + def self.exist?(formula) + spdxfile(formula).exist? + end + + sig { returns(T::Hash[String, String]) } + def self.fetch_schema! + return @schema if @schema.present? + + url = SCHEMA_URL + target = SCHEMA_CACHE_TARGET + quieter = target.exist? && !target.empty? + + curl_args = Utils::Curl.curl_args(retries: 0) + curl_args += ["--silent", "--time-cond", target.to_s] if quieter + + begin + unless quieter + oh1 "Fetching SBOM schema" + ohai "Downloading #{url}" + end + Utils::Curl.curl_download(*curl_args, url, to: target, retries: 0) + FileUtils.touch(target, mtime: Time.now) + rescue ErrorDuringExecution + target.unlink if target.exist? && target.empty? + + if target.exist? + opoo "SBOM schema update failed, falling back to cached version." + else + opoo "Failed to fetch SBOM schema, cannot perform SBOM validation!" + + return {} + end + end + + @schema = begin + JSON.parse(target.read, freeze: true) + rescue JSON::ParserError + target.unlink + opoo "Failed to fetch SBOM schema, cached version corrupted, cannot perform SBOM validation!" + {} + end end sig { returns(T::Boolean) } def valid? + unless require? "json_schemer" + error_message = "Need json_schemer to validate SBOM, run `brew install-bundler-gems --add-groups=bottle`!" + odie error_message if ENV["HOMEBREW_ENFORCE_SBOM"] + return false + end + + schema = SBOM.fetch_schema! + if schema.blank? + error_message = "Could not fetch JSON schema to validate SBOM!" + ENV["HOMEBREW_ENFORCE_SBOM"] ? odie(error_message) : opoo(error_message) + return false + end + + schemer = JSONSchemer.schema(schema) data = to_spdx_sbom - - schema_string, _, status = Utils::Curl.curl_output(SCHEMA) - - opoo "Failed to fetch schema!" unless status.success? - - require "json_schemer" - - schemer = JSONSchemer.schema(schema_string) - return true if schemer.valid?(data) opoo "SBOM validation errors:" schemer.validate(data).to_a.each do |error| - ohai error["error"] + puts error["error"] end - odie "Failed to validate SBOM agains schema!" if ENV["HOMEBREW_ENFORCE_SBOM"] + odie "Failed to validate SBOM against JSON schema!" if ENV["HOMEBREW_ENFORCE_SBOM"] false end - sig { void } - def write + sig { params(validate: T::Boolean).void } + def write(validate: true) # If this is a new installation, the cache of installed formulae # will no longer be valid. Formula.clear_cache unless spdxfile.exist? self.class.cache[spdxfile] = self - unless valid? + if validate && !valid? opoo "SBOM is not valid, not writing to disk!" return end @@ -108,6 +161,16 @@ class SBOM spdxfile.atomic_write(JSON.pretty_generate(to_spdx_sbom)) end + private + + attr_reader :name, :homebrew_version, :time, :stdlib, :source, :built_on, :license + attr_accessor :spdxfile + + sig { params(attributes: Hash).void } + def initialize(attributes = {}) + attributes.each { |key, value| instance_variable_set(:"@#{key}", value) } + end + sig { params(runtime_dependency_declaration: T::Array[Hash], compiler_declaration: Hash).returns(T::Array[Hash]) } def generate_relations_json(runtime_dependency_declaration, compiler_declaration) runtime = runtime_dependency_declaration.map do |dependency| @@ -139,7 +202,7 @@ class SBOM ] if compiler_declaration["SPDXRef-Stdlib"].present? - base += { + base << { spdxElementId: "SPDXRef-Stdlib", relationshipType: "DEPENDENCY_OF", relatedSpdxElement: "SPDXRef-Bottle-#{name}", @@ -157,7 +220,7 @@ class SBOM } def generate_packages_json(runtime_dependency_declaration, compiler_declaration) bottle = [] - if get_bottle_info(source[:bottle]) + if (bottle_info = get_bottle_info(source[:bottle])) bottle << { SPDXID: "SPDXRef-Bottle-#{name}", name: name.to_s, @@ -166,7 +229,7 @@ class SBOM licenseDeclared: assert_value(nil), builtDate: source_modified_time.to_s, licenseConcluded: license, - downloadLocation: T.must(get_bottle_info(source[:bottle]))["url"], + downloadLocation: bottle_info.fetch("url"), copyrightText: assert_value(nil), externalRefs: [ { @@ -178,7 +241,7 @@ class SBOM checksums: [ { algorithm: "SHA256", - checksumValue: T.must(get_bottle_info(source[:bottle]))["sha256"], + checksumValue: bottle_info.fetch("sha256"), }, ], } @@ -216,25 +279,28 @@ class SBOM bottle_info = get_bottle_info(dependency["bottle"]) next unless bottle_info.present? + # Only set bottle URL if the dependency is the same version as the formula/bottle. + bottle_url = bottle_info["url"] if dependency["pkg_version"] == dependency["formula_pkg_version"] + { - SPDXID: "SPDXRef-Package-SPDXRef-#{dependency["name"].tr("/", "-")}-#{dependency["version"]}", + SPDXID: "SPDXRef-Package-SPDXRef-#{dependency["name"].tr("/", "-")}-#{dependency["pkg_version"]}", name: dependency["name"], versionInfo: dependency["pkg_version"], filesAnalyzed: false, licenseDeclared: assert_value(nil), licenseConcluded: assert_value(dependency["license"]), - downloadLocation: assert_value(bottle_info.present? ? bottle_info["url"] : nil), + downloadLocation: assert_value(bottle_url), copyrightText: assert_value(nil), checksums: [ { algorithm: "SHA256", - checksumValue: assert_value(bottle_info.present? ? bottle_info["sha256"] : nil), + checksumValue: assert_value(bottle_info["sha256"]), }, ], externalRefs: [ { referenceCategory: "PACKAGE-MANAGER", - referenceLocator: "pkg:brew/#{dependency["full_name"]}@#{dependency["version"]}", + referenceLocator: "pkg:brew/#{dependency["full_name"]}@#{dependency["pkg_version"]}", referenceType: :purl, }, ], @@ -294,57 +360,19 @@ class SBOM } end - sig { params(deps: T::Array[Formula]).returns(T::Array[T::Hash[Symbol, String]]) } - def self.runtime_deps_hash(deps) - deps.map do |dep| - { - full_name: dep.full_name, - name: dep.name, - version: dep.version.to_s, - revision: dep.revision, - pkg_version: dep.pkg_version.to_s, - declared_directly: true, - license: SPDX.license_expression_to_string(dep.license), - bottle: dep.bottle_hash, - } - end - end - - private - sig { params(base: T.nilable(T::Hash[String, Hash])).returns(T.nilable(T::Hash[String, String])) } def get_bottle_info(base) return unless base.present? - return unless base.key?("files") - T.must(base["files"])[Utils::Bottles.tag.to_sym] - end + files = base["files"].presence + return unless files - sig { returns(T::Boolean) } - def stable? - spec == :stable + files[Utils::Bottles.tag.to_sym] || files[:all] end sig { returns(Symbol) } def compiler - @compiler || DevelopmentTools.default_compiler - end - - sig { returns(CxxStdlib) } - def cxxstdlib - # Older sboms won't have these values, so provide sensible defaults - lib = stdlib.to_sym if stdlib - CxxStdlib.create(lib, compiler.to_sym) - end - - sig { returns(T::Boolean) } - def built_bottle? - built_as_bottle && !poured_from_bottle - end - - sig { returns(T::Boolean) } - def bottle? - built_as_bottle + @compiler.presence&.to_sym || DevelopmentTools.default_compiler end sig { returns(T.nilable(Tap)) } @@ -353,11 +381,6 @@ class SBOM Tap.fetch(tap_name) if tap_name end - sig { returns(Symbol) } - def spec - source[:spec].to_sym - end - sig { returns(T.nilable(Version)) } def stable_version source[:stable][:version] diff --git a/Library/Homebrew/test/formula_installer_spec.rb b/Library/Homebrew/test/formula_installer_spec.rb index d4b6284b3d..9e6b47be63 100644 --- a/Library/Homebrew/test/formula_installer_spec.rb +++ b/Library/Homebrew/test/formula_installer_spec.rb @@ -443,6 +443,7 @@ RSpec.describe FormulaInstaller do it "shows audit problems if HOMEBREW_DEVELOPER is set" do ENV["HOMEBREW_DEVELOPER"] = "1" + expect(SBOM).to receive(:fetch_schema!).and_return({}) formula_installer.fetch formula_installer.install expect(formula_installer).to receive(:audit_installed).and_call_original diff --git a/Library/Homebrew/test/sbom_spec.rb b/Library/Homebrew/test/sbom_spec.rb index 8858f6fa5a..c270b2a0fd 100644 --- a/Library/Homebrew/test/sbom_spec.rb +++ b/Library/Homebrew/test/sbom_spec.rb @@ -4,18 +4,26 @@ require "sbom" RSpec.describe SBOM, :needs_network do describe "#valid?" do - it "returns true if the SBOM is valid" do - f = formula do - url "foo-1.0" - end - - sbom = described_class.create(f) + it "returns true if a minimal SBOM is valid" do + f = formula { url "foo-1.0" } + sbom = described_class.create(f, Tab.new) expect(sbom).to be_valid end - it "returns true if the SBOM is valid with dependencies" do + it "returns true if a maximal SBOM is valid" do f = formula do - url "foo-1.0" + homepage "https://brew.sh" + + url "https://brew.sh/test-0.1.tbz" + sha256 TEST_SHA256 + + patch do + url "patch_macos" + end + + bottle do + sha256 all: "9befdad158e59763fb0622083974a6252878019702d8c961e1bec3a5f5305339" + end # some random dependencies to test with depends_on "cmake" => :build @@ -27,33 +35,37 @@ RSpec.describe SBOM, :needs_network do beanstalkd = formula "beanstalkd" do url "one-1.1" + + bottle do + sha256 all: "ac4c0330b70dae06eaa8065bfbea78dda277699d1ae8002478017a1bd9cf1908" + end end zlib = formula "zlib" do url "two-1.1" - end - allow(f).to receive_messages( - runtime_formula_dependencies: [beanstalkd, zlib], - ) - - sbom = described_class.create(f) - expect(sbom).to be_valid - end - - it "returns true if SBOM is valid with patches" do - f = formula do - homepage "https://brew.sh" - - url "https://brew.sh/test-0.1.tbz" - sha256 TEST_SHA256 - - patch do - url "patch_macos" + bottle do + sha256 all: "6a4642964fe5c4d1cc8cd3507541736d5b984e34a303a814ef550d4f2f8242f9" end end - sbom = described_class.create(f) + runtime_dependencies = [beanstalkd, zlib] + runtime_deps_hash = runtime_dependencies.map do |dep| + { + "full_name" => dep.full_name, + "version" => dep.version.to_s, + "revision" => dep.revision, + "pkg_version" => dep.pkg_version.to_s, + "declared_directly" => true, + } + end + expect(Tab).to receive(:runtime_deps_hash).and_return(runtime_deps_hash) + tab = Tab.create(f, DevelopmentTools.default_compiler, :libcxx) + + expect(Formulary).to receive(:factory).with("beanstalkd").and_return(beanstalkd) + expect(Formulary).to receive(:factory).with("zlib").and_return(zlib) + + sbom = described_class.create(f, tab) expect(sbom).to be_valid end end