Merge pull request #17254 from Homebrew/sbom_tweaks

SBOM improvements

Library/Homebrew/dev-cmd/bottle.rb

@@ -96,13 +96,13 @@ module Homebrew
 
       sig { override.void }
       def run
-        Homebrew.install_bundler_gems!(groups: ["bottle"])
-
         if args.merge?
           Homebrew.install_bundler_gems!(groups: ["ast"])
           return merge
         end
 
+        Homebrew.install_bundler_gems!(groups: ["bottle"])
+
         gnu_tar_formula_ensure_installed_if_needed!
 
         args.named.to_resolved_formulae(uniq: false).each do |formula|
@@ -508,7 +508,7 @@ module Homebrew
               tab.write
             end
 
-            sbom = SBOM.create(formula)
+            sbom = SBOM.create(formula, tab)
             sbom.write
 
             keg.consistent_reproducible_symlink_permissions!

Library/Homebrew/formula_installer.rb

@@ -23,6 +23,7 @@ require "deprecate_disable"
 require "unlink"
 require "service"
 require "attestation"
+require "sbom"
 
 # Installer for a formula.
 class FormulaInstaller
@@ -828,6 +829,12 @@ on_request: installed_on_request?, options:)
     tab.runtime_dependencies = Tab.runtime_deps_hash(formula, f_runtime_deps)
     tab.write
 
+    # write a SBOM file (if we don't already have one and aren't bottling)
+    if !build_bottle? && !SBOM.exist?(formula)
+      sbom = SBOM.create(formula, tab)
+      sbom.write(validate: Homebrew::EnvConfig.developer?)
+    end
+
     # let's reset Utils::Git.available? if we just installed git
     Utils::Git.clear_available_cache if formula.name == "git"
 
@@ -1216,6 +1223,8 @@ on_request: installed_on_request?, options:)
   def fetch
     return if previously_fetched_formula
 
+    SBOM.fetch_schema! if Homebrew::EnvConfig.developer?
+
     fetch_dependencies
 
     return if only_deps?

Library/Homebrew/sbom.rb

@@ -12,37 +12,25 @@ class SBOM
   extend Cachable
 
   FILENAME = "sbom.spdx.json"
-  SCHEMA = "https://raw.githubusercontent.com/spdx/spdx-spec/v2.3/schemas/spdx-schema.json"
+  SCHEMA_URL = "https://spdx.github.io/spdx-3-model/model.jsonld"
-
+  SCHEMA_FILENAME = "sbom.spdx.schema.3.json"
-  attr_accessor :homebrew_version, :spdxfile, :built_as_bottle, :installed_as_dependency, :installed_on_request,
+  SCHEMA_CACHE_TARGET = (HOMEBREW_CACHE/"sbom/#{SCHEMA_FILENAME}").freeze
-                :changed_files, :poured_from_bottle, :loaded_from_api, :time, :stdlib, :aliases, :arch, :source,
-                :built_on, :license, :name
-  attr_writer :compiler, :runtime_dependencies, :source_modified_time
 
   # Instantiates a {SBOM} for a new installation of a formula.
-  sig { params(formula: Formula, compiler: T.nilable(String), stdlib: T.nilable(String)).returns(T.attached_class) }
+  sig { params(formula: Formula, tab: Tab).returns(T.attached_class) }
-  def self.create(formula, compiler: nil, stdlib: nil)
+  def self.create(formula, tab)
-    runtime_deps = formula.runtime_formula_dependencies(undeclared: false)
-
     attributes = {
-      name:                    formula.name,
+      name:                 formula.name,
-      homebrew_version:        HOMEBREW_VERSION,
+      homebrew_version:     HOMEBREW_VERSION,
-      spdxfile:                formula.prefix/FILENAME,
+      spdxfile:             SBOM.spdxfile(formula),
-      built_as_bottle:         formula.build.bottle?,
+      time:                 Time.now.to_i,
-      installed_as_dependency: false,
+      source_modified_time: tab.source_modified_time.to_i,
-      installed_on_request:    false,
+      compiler:             tab.compiler,
-      poured_from_bottle:      false,
+      stdlib:               tab.stdlib,
-      loaded_from_api:         false,
+      runtime_dependencies: SBOM.runtime_deps_hash(Array(tab.runtime_dependencies)),
-      time:                    Time.now.to_i,
+      license:              SPDX.license_expression_to_string(formula.license),
-      source_modified_time:    formula.source_modified_time.to_i,
+      built_on:             DevelopmentTools.build_system_info,
-      compiler:,
+      source:               {
-      stdlib:,
-      aliases:                 formula.aliases,
-      runtime_dependencies:    SBOM.runtime_deps_hash(runtime_deps),
-      arch:                    Hardware::CPU.arch,
-      license:                 SPDX.license_expression_to_string(formula.license),
-      built_on:                DevelopmentTools.build_system_info,
-      source:                  {
         path:         formula.specified_path.to_s,
         tap:          formula.tap&.name,
         tap_git_head: nil, # Filled in later if possible
@@ -63,44 +51,109 @@ class SBOM
     new(attributes)
   end
 
-  sig { params(attributes: Hash).void }
+  sig { params(formula: Formula).returns(Pathname) }
-  def initialize(attributes = {})
+  def self.spdxfile(formula)
-    attributes.each { |key, value| instance_variable_set(:"@#{key}", value) }
+    formula.prefix/FILENAME
+  end
+
+  sig { params(deps: T::Array[T::Hash[String, String]]).returns(T::Array[T::Hash[String, String]]) }
+  def self.runtime_deps_hash(deps)
+    deps.map do |dep|
+      full_name = dep.fetch("full_name")
+      dep_formula = Formula[full_name]
+      {
+        "full_name"           => full_name,
+        "pkg_version"         => dep.fetch("pkg_version"),
+        "name"                => dep_formula.name,
+        "license"             => SPDX.license_expression_to_string(dep_formula.license),
+        "bottle"              => dep_formula.bottle_hash,
+        "formula_pkg_version" => dep_formula.pkg_version.to_s,
+      }
+    end
+  end
+
+  sig { params(formula: Formula).returns(T::Boolean) }
+  def self.exist?(formula)
+    spdxfile(formula).exist?
+  end
+
+  sig { returns(T::Hash[String, String]) }
+  def self.fetch_schema!
+    return @schema if @schema.present?
+
+    url = SCHEMA_URL
+    target = SCHEMA_CACHE_TARGET
+    quieter = target.exist? && !target.empty?
+
+    curl_args = Utils::Curl.curl_args(retries: 0)
+    curl_args += ["--silent", "--time-cond", target.to_s] if quieter
+
+    begin
+      unless quieter
+        oh1 "Fetching SBOM schema"
+        ohai "Downloading #{url}"
+      end
+      Utils::Curl.curl_download(*curl_args, url, to: target, retries: 0)
+      FileUtils.touch(target, mtime: Time.now)
+    rescue ErrorDuringExecution
+      target.unlink if target.exist? && target.empty?
+
+      if target.exist?
+        opoo "SBOM schema update failed, falling back to cached version."
+      else
+        opoo "Failed to fetch SBOM schema, cannot perform SBOM validation!"
+
+        return {}
+      end
+    end
+
+    @schema = begin
+      JSON.parse(target.read, freeze: true)
+    rescue JSON::ParserError
+      target.unlink
+      opoo "Failed to fetch SBOM schema, cached version corrupted, cannot perform SBOM validation!"
+      {}
+    end
   end
 
   sig { returns(T::Boolean) }
   def valid?
+    unless require? "json_schemer"
+      error_message = "Need json_schemer to validate SBOM, run `brew install-bundler-gems --add-groups=bottle`!"
+      odie error_message if ENV["HOMEBREW_ENFORCE_SBOM"]
+      return false
+    end
+
+    schema = SBOM.fetch_schema!
+    if schema.blank?
+      error_message = "Could not fetch JSON schema to validate SBOM!"
+      ENV["HOMEBREW_ENFORCE_SBOM"] ? odie(error_message) : opoo(error_message)
+      return false
+    end
+
+    schemer = JSONSchemer.schema(schema)
     data = to_spdx_sbom
-
-    schema_string, _, status = Utils::Curl.curl_output(SCHEMA)
-
-    opoo "Failed to fetch schema!" unless status.success?
-
-    require "json_schemer"
-
-    schemer = JSONSchemer.schema(schema_string)
-
     return true if schemer.valid?(data)
 
     opoo "SBOM validation errors:"
     schemer.validate(data).to_a.each do |error|
-      ohai error["error"]
+      puts error["error"]
     end
 
-    odie "Failed to validate SBOM agains schema!" if ENV["HOMEBREW_ENFORCE_SBOM"]
+    odie "Failed to validate SBOM against JSON schema!" if ENV["HOMEBREW_ENFORCE_SBOM"]
 
     false
   end
 
-  sig { void }
+  sig { params(validate: T::Boolean).void }
-  def write
+  def write(validate: true)
     # If this is a new installation, the cache of installed formulae
     # will no longer be valid.
     Formula.clear_cache unless spdxfile.exist?
 
     self.class.cache[spdxfile] = self
 
-    unless valid?
+    if validate && !valid?
       opoo "SBOM is not valid, not writing to disk!"
       return
     end
@@ -108,6 +161,16 @@ class SBOM
     spdxfile.atomic_write(JSON.pretty_generate(to_spdx_sbom))
   end
 
+  private
+
+  attr_reader :name, :homebrew_version, :time, :stdlib, :source, :built_on, :license
+  attr_accessor :spdxfile
+
+  sig { params(attributes: Hash).void }
+  def initialize(attributes = {})
+    attributes.each { |key, value| instance_variable_set(:"@#{key}", value) }
+  end
+
   sig { params(runtime_dependency_declaration: T::Array[Hash], compiler_declaration: Hash).returns(T::Array[Hash]) }
   def generate_relations_json(runtime_dependency_declaration, compiler_declaration)
     runtime = runtime_dependency_declaration.map do |dependency|
@@ -139,7 +202,7 @@ class SBOM
     ]
 
     if compiler_declaration["SPDXRef-Stdlib"].present?
-      base += {
+      base << {
         spdxElementId:      "SPDXRef-Stdlib",
         relationshipType:   "DEPENDENCY_OF",
         relatedSpdxElement: "SPDXRef-Bottle-#{name}",
@@ -157,7 +220,7 @@ class SBOM
   }
   def generate_packages_json(runtime_dependency_declaration, compiler_declaration)
     bottle = []
-    if get_bottle_info(source[:bottle])
+    if (bottle_info = get_bottle_info(source[:bottle]))
       bottle << {
         SPDXID:           "SPDXRef-Bottle-#{name}",
         name:             name.to_s,
@@ -166,7 +229,7 @@ class SBOM
         licenseDeclared:  assert_value(nil),
         builtDate:        source_modified_time.to_s,
         licenseConcluded: license,
-        downloadLocation: T.must(get_bottle_info(source[:bottle]))["url"],
+        downloadLocation: bottle_info.fetch("url"),
         copyrightText:    assert_value(nil),
         externalRefs:     [
           {
@@ -178,7 +241,7 @@ class SBOM
         checksums:        [
           {
             algorithm:     "SHA256",
-            checksumValue: T.must(get_bottle_info(source[:bottle]))["sha256"],
+            checksumValue: bottle_info.fetch("sha256"),
           },
         ],
       }
@@ -216,25 +279,28 @@ class SBOM
       bottle_info = get_bottle_info(dependency["bottle"])
       next unless bottle_info.present?
 
+      # Only set bottle URL if the dependency is the same version as the formula/bottle.
+      bottle_url = bottle_info["url"] if dependency["pkg_version"] == dependency["formula_pkg_version"]
+
       {
-        SPDXID:           "SPDXRef-Package-SPDXRef-#{dependency["name"].tr("/", "-")}-#{dependency["version"]}",
+        SPDXID:           "SPDXRef-Package-SPDXRef-#{dependency["name"].tr("/", "-")}-#{dependency["pkg_version"]}",
         name:             dependency["name"],
         versionInfo:      dependency["pkg_version"],
         filesAnalyzed:    false,
         licenseDeclared:  assert_value(nil),
         licenseConcluded: assert_value(dependency["license"]),
-        downloadLocation: assert_value(bottle_info.present? ? bottle_info["url"] : nil),
+        downloadLocation: assert_value(bottle_url),
         copyrightText:    assert_value(nil),
         checksums:        [
           {
             algorithm:     "SHA256",
-            checksumValue: assert_value(bottle_info.present? ? bottle_info["sha256"] : nil),
+            checksumValue: assert_value(bottle_info["sha256"]),
           },
         ],
         externalRefs:     [
           {
             referenceCategory: "PACKAGE-MANAGER",
-            referenceLocator:  "pkg:brew/#{dependency["full_name"]}@#{dependency["version"]}",
+            referenceLocator:  "pkg:brew/#{dependency["full_name"]}@#{dependency["pkg_version"]}",
             referenceType:     :purl,
           },
         ],
@@ -294,57 +360,19 @@ class SBOM
     }
   end
 
-  sig { params(deps: T::Array[Formula]).returns(T::Array[T::Hash[Symbol, String]]) }
-  def self.runtime_deps_hash(deps)
-    deps.map do |dep|
-      {
-        full_name:         dep.full_name,
-        name:              dep.name,
-        version:           dep.version.to_s,
-        revision:          dep.revision,
-        pkg_version:       dep.pkg_version.to_s,
-        declared_directly: true,
-        license:           SPDX.license_expression_to_string(dep.license),
-        bottle:            dep.bottle_hash,
-      }
-    end
-  end
-
-  private
-
   sig { params(base: T.nilable(T::Hash[String, Hash])).returns(T.nilable(T::Hash[String, String])) }
   def get_bottle_info(base)
     return unless base.present?
-    return unless base.key?("files")
 
-    T.must(base["files"])[Utils::Bottles.tag.to_sym]
+    files = base["files"].presence
-  end
+    return unless files
 
-  sig { returns(T::Boolean) }
+    files[Utils::Bottles.tag.to_sym] || files[:all]
-  def stable?
-    spec == :stable
   end
 
   sig { returns(Symbol) }
   def compiler
-    @compiler || DevelopmentTools.default_compiler
+    @compiler.presence&.to_sym || DevelopmentTools.default_compiler
-  end
-
-  sig { returns(CxxStdlib) }
-  def cxxstdlib
-    # Older sboms won't have these values, so provide sensible defaults
-    lib = stdlib.to_sym if stdlib
-    CxxStdlib.create(lib, compiler.to_sym)
-  end
-
-  sig { returns(T::Boolean) }
-  def built_bottle?
-    built_as_bottle && !poured_from_bottle
-  end
-
-  sig { returns(T::Boolean) }
-  def bottle?
-    built_as_bottle
   end
 
   sig { returns(T.nilable(Tap)) }
@@ -353,11 +381,6 @@ class SBOM
     Tap.fetch(tap_name) if tap_name
   end
 
-  sig { returns(Symbol) }
-  def spec
-    source[:spec].to_sym
-  end
-
   sig { returns(T.nilable(Version)) }
   def stable_version
     source[:stable][:version]

Library/Homebrew/test/formula_installer_spec.rb

@@ -443,6 +443,7 @@ RSpec.describe FormulaInstaller do
 
     it "shows audit problems if HOMEBREW_DEVELOPER is set" do
       ENV["HOMEBREW_DEVELOPER"] = "1"
+      expect(SBOM).to receive(:fetch_schema!).and_return({})
       formula_installer.fetch
       formula_installer.install
       expect(formula_installer).to receive(:audit_installed).and_call_original

Library/Homebrew/test/sbom_spec.rb

@@ -4,18 +4,26 @@ require "sbom"
 
 RSpec.describe SBOM, :needs_network do
   describe "#valid?" do
-    it "returns true if the SBOM is valid" do
+    it "returns true if a minimal SBOM is valid" do
-      f = formula do
+      f = formula { url "foo-1.0" }
-        url "foo-1.0"
+      sbom = described_class.create(f, Tab.new)
-      end
-
-      sbom = described_class.create(f)
       expect(sbom).to be_valid
     end
 
-    it "returns true if the SBOM is valid with dependencies" do
+    it "returns true if a maximal SBOM is valid" do
       f = formula do
-        url "foo-1.0"
+        homepage "https://brew.sh"
+
+        url "https://brew.sh/test-0.1.tbz"
+        sha256 TEST_SHA256
+
+        patch do
+          url "patch_macos"
+        end
+
+        bottle do
+          sha256 all:   "9befdad158e59763fb0622083974a6252878019702d8c961e1bec3a5f5305339"
+        end
 
         # some random dependencies to test with
         depends_on "cmake" => :build
@@ -27,33 +35,37 @@ RSpec.describe SBOM, :needs_network do
 
       beanstalkd = formula "beanstalkd" do
         url "one-1.1"
+
+        bottle do
+          sha256 all:   "ac4c0330b70dae06eaa8065bfbea78dda277699d1ae8002478017a1bd9cf1908"
+        end
       end
 
       zlib = formula "zlib" do
         url "two-1.1"
-      end
 
-      allow(f).to receive_messages(
+        bottle do
-        runtime_formula_dependencies: [beanstalkd, zlib],
+          sha256 all:   "6a4642964fe5c4d1cc8cd3507541736d5b984e34a303a814ef550d4f2f8242f9"
-      )
-
-      sbom = described_class.create(f)
-      expect(sbom).to be_valid
-    end
-
-    it "returns true if SBOM is valid with patches" do
-      f = formula do
-        homepage "https://brew.sh"
-
-        url "https://brew.sh/test-0.1.tbz"
-        sha256 TEST_SHA256
-
-        patch do
-          url "patch_macos"
         end
       end
 
-      sbom = described_class.create(f)
+      runtime_dependencies = [beanstalkd, zlib]
+      runtime_deps_hash = runtime_dependencies.map do |dep|
+        {
+          "full_name"         => dep.full_name,
+          "version"           => dep.version.to_s,
+          "revision"          => dep.revision,
+          "pkg_version"       => dep.pkg_version.to_s,
+          "declared_directly" => true,
+        }
+      end
+      expect(Tab).to receive(:runtime_deps_hash).and_return(runtime_deps_hash)
+      tab = Tab.create(f, DevelopmentTools.default_compiler, :libcxx)
+
+      expect(Formulary).to receive(:factory).with("beanstalkd").and_return(beanstalkd)
+      expect(Formulary).to receive(:factory).with("zlib").and_return(zlib)
+
+      sbom = described_class.create(f, tab)
       expect(sbom).to be_valid
     end
   end