Berkeley db cache optimization for brew linkage command.

Library/Homebrew/dev-cmd/linkage.rb

@@ -1,4 +1,4 @@
-#:  * `linkage` [`--test`] [`--reverse`] <formula>:
+#:  * `linkage` [`--test`] [`--reverse`] [`--rebuild`] <formula>:
 #:    Checks the library links of an installed formula.
 #:
 #:    Only works on installed formulae. An error is raised if it is run on
@@ -9,6 +9,9 @@
 #:
 #:    If `--reverse` is passed, print the dylib followed by the binaries
 #:    which link to it for each library the keg references.
+#:
+#:    If `--rebuild` is passed, flushes the `LinkageStore` cache for each
+#:    'keg.name' and forces a check on the dylibs.
 
 require "os/mac/linkage_checker"
 
@@ -18,7 +21,10 @@ module Homebrew
   def linkage
     ARGV.kegs.each do |keg|
       ohai "Checking #{keg.name} linkage" if ARGV.kegs.size > 1
-      result = LinkageChecker.new(keg)
+      database_cache = DatabaseCache.new("linkage")
+      result = LinkageChecker.new(keg, database_cache)
+      result.flush_cache_and_check_dylibs if ARGV.include?("--rebuild")
+
       if ARGV.include?("--test")
         result.display_test_output
         Homebrew.failed = true if result.broken_dylibs?
@@ -27,6 +33,8 @@ module Homebrew
       else
         result.display_normal_output
       end
+
+      database_cache.close
     end
   end
 end

Library/Homebrew/extend/os/mac/formula_cellar_checks.rb

@@ -64,7 +64,10 @@ module FormulaCellarChecks
   def check_linkage
     return unless formula.prefix.directory?
     keg = Keg.new(formula.prefix)
-    checker = LinkageChecker.new(keg, formula)
+    database_cache = DatabaseCache.new("linkage")
+    checker = LinkageChecker.new(keg, database_cache, formula)
+    checker.flush_cache_and_check_dylibs
+    database_cache.close
 
     return unless checker.broken_dylibs?
     output = <<~EOS

Library/Homebrew/os/mac/cache_store.rb

@@ -0,0 +1,207 @@
+require "dbm"
+require "json"
+
+#
+# `DatabaseCache` is a class acting as an interface to a persistent storage
+# mechanism residing in the `HOMEBREW_CACHE`
+#
+class DatabaseCache
+  # Name of the database cache file located at <HOMEBREW_CACHE>/<name>.db
+  #
+  # @return [String]
+  attr_accessor :name
+
+  # Instantiates new `DatabaseCache` object
+  #
+  # @param  [String] name
+  # @return [nil]
+  def initialize(name)
+    @name = name
+  end
+
+  # Memoized `DBM` database object with on-disk database located in the
+  # `HOMEBREW_CACHE`
+  #
+  # @return [DBM] db
+  def db
+    @db ||= DBM.open("#{HOMEBREW_CACHE}/#{name}", 0666, DBM::WRCREAT)
+  end
+
+  # Close the `DBM` database object after usage
+  #
+  # @return [nil]
+  def close
+    db.close
+  end
+end
+
+#
+# `CacheStore` is an abstract base class which provides methods to mutate and
+# fetch data from a persistent storage mechanism
+#
+# @abstract
+#
+class CacheStore
+  # Instantiates a new `CacheStore` class
+  #
+  # @param  [DatabaseCache] database_cache
+  # @return [nil]
+  def initialize(database_cache)
+    @db = database_cache.db
+  end
+
+  # Inserts new values or updates existing cached values to persistent storage
+  # mechanism
+  #
+  # @abstract
+  # @param  [Any]
+  # @return [nil]
+  def update!(*)
+    raise NotImplementedError
+  end
+
+  # Fetches cached values in persistent storage according to the type of data
+  # stored
+  #
+  # @abstract
+  # @param  [Any]
+  # @return [Any]
+  def fetch(*)
+    raise NotImplementedError
+  end
+
+  # Deletes data from the cache based on a condition defined in a concrete class
+  #
+  # @abstract
+  # @return [nil]
+  def flush_cache!
+    raise NotImplementedError
+  end
+
+  protected
+
+  # A class instance providing access to the `DBM` database object
+  #
+  # @return [DBM]
+  attr_reader :db
+end
+
+#
+# `LinkageStore` is a concrete class providing methods to fetch and mutate
+# linkage-specific data used by the `brew linkage` command
+#
+# If the cache hasn't changed, don't do extra processing in `LinkageChecker`.
+# Instead, just fetch the data stored in the cache
+#
+class LinkageStore < CacheStore
+  # Types of dylibs of the form (label -> array)
+  HASH_LINKAGE_TYPES = %w[brewed_dylibs reverse_links].freeze
+
+  # The keg name for the `LinkageChecker` class
+  #
+  # @return [String]
+  attr_reader :key
+
+  # Initializes new `LinkageStore` class
+  #
+  # @param  [String]        keg_name
+  # @param  [DatabaseCache] database_cache
+  # @return [nil]
+  def initialize(keg_name, database_cache)
+    @key = keg_name
+    super(database_cache)
+  end
+
+  # Inserts new values or updates existing cached values to persistent storage
+  # mechanism according to the type of data
+  #
+  # @param  [Hash] path_values
+  # @param  [Hash] hash_values
+  # @return [nil]
+  def update!(
+    path_values: {
+      "system_dylibs" => %w[], "variable_dylibs" => %w[], "broken_dylibs" => %w[],
+      "indirect_deps" => %w[], "undeclared_deps" => %w[], "unnecessary_deps" => %w[]
+    },
+    hash_values: {
+      "brewed_dylibs" => {}, "reverse_links" => {}
+    }
+  )
+    db[key] = {
+      "path_values" => format_path_values(path_values),
+      "hash_values" => format_hash_values(hash_values),
+    }
+  end
+
+  # Fetches cached values in persistent storage according to the type of data
+  # stored
+  #
+  # @param  [String] type
+  # @return [Any]
+  def fetch(type:)
+    if HASH_LINKAGE_TYPES.include?(type)
+      fetch_hash_values(type: type)
+    else
+      fetch_path_values(type: type)
+    end
+  end
+
+  # A condition for where to flush the cache
+  #
+  # @return [String]
+  def flush_cache!
+    db.delete(key)
+  end
+
+  private
+
+  # Fetches a subset of paths where the name = `key`
+  #
+  # @param  [String] type
+  # @return [Array[String]]
+  def fetch_path_values(type:)
+    return [] unless db.key?(key) && !db[key].nil?
+    string_to_hash(db[key])["path_values"][type]
+  end
+
+  # Fetches a subset of paths and labels where the name = `key`. Formats said
+  # paths/labels into `key => [value]` syntax expected by `LinkageChecker`
+  #
+  # @param  [String] type
+  # @return [Hash]
+  def fetch_hash_values(type:)
+    return {} unless db.key?(key) && !db[key].nil?
+    string_to_hash(db[key])["hash_values"][type]
+  end
+
+  # Parses `DBM` stored `String` into ruby `Hash`
+  #
+  # @param [String] string
+  # @return [Hash]
+  def string_to_hash(string)
+    JSON.parse(string.gsub("=>", ":"))
+  end
+
+  # Formats the linkage data for `path_values` into a kind which can be parsed
+  # by the `string_to_hash` method. Converts ruby `Set`s to `Array`s.
+  #
+  # @param  [Hash(String, Set(String))] hash
+  # @return [Hash(String, Array(String))]
+  def format_path_values(hash)
+    hash.each_with_object({}) { |(k, v), h| h[k] = v.to_a }
+  end
+
+  # Formats the linkage data for `hash_values` into a kind which can be parsed
+  # by the `string_to_hash` method. Converts ruby `Set`s to `Array`s, and
+  # converts ruby `Pathname`s to `String`s
+  #
+  # @param  [Hash(String, Set(Pathname))] hash
+  # @return [Hash(String, Array(String))]
+  def format_hash_values(hash)
+    hash.each_with_object({}) do |(outer_key, outer_values), outer_hash|
+      outer_hash[outer_key] = outer_values.each_with_object({}) do |(k, v), h|
+        h[k] = v.to_a.map(&:to_s)
+      end
+    end
+  end
+end

Library/Homebrew/os/mac/linkage_checker.rb

@@ -1,27 +1,56 @@
 require "set"
 require "keg"
 require "formula"
+require "os/mac/cache_store"
 
 class LinkageChecker
-  attr_reader :keg, :formula
-  attr_reader :brewed_dylibs, :system_dylibs, :broken_dylibs, :variable_dylibs
-  attr_reader :undeclared_deps, :unnecessary_deps, :reverse_links
+  attr_reader :keg, :formula, :store
 
-  def initialize(keg, formula = nil)
+  def initialize(keg, db, formula = nil)
     @keg = keg
     @formula = formula || resolve_formula(keg)
-    @brewed_dylibs = Hash.new { |h, k| h[k] = Set.new }
-    @system_dylibs = Set.new
-    @broken_dylibs = Set.new
-    @variable_dylibs = Set.new
-    @indirect_deps = []
-    @undeclared_deps = []
-    @reverse_links = Hash.new { |h, k| h[k] = Set.new }
-    @unnecessary_deps = []
-    check_dylibs
+    @store = LinkageStore.new(keg.name, db)
   end
 
-  def check_dylibs
+  # 'Hash-type' cache values
+
+  def brewed_dylibs
+    @brewed_dylibs ||= store.fetch(type: "brewed_dylibs")
+  end
+
+  def reverse_links
+    @reverse_links ||= store.fetch(type: "reverse_links")
+  end
+
+  # 'Path-type' cached values
+
+  def system_dylibs
+    @system_dylibs ||= store.fetch(type: "system_dylibs")
+  end
+
+  def broken_dylibs
+    @broken_dylibs ||= store.fetch(type: "broken_dylibs")
+  end
+
+  def variable_dylibs
+    @variable_dylibs ||= store.fetch(type: "variable_dylibs")
+  end
+
+  def undeclared_deps
+    @undeclared_deps ||= store.fetch(type: "undeclared_deps")
+  end
+
+  def indirect_deps
+    @indirect_deps ||= store.fetch(type: "indirect_deps")
+  end
+
+  def unnecessary_deps
+    @unnecessary_deps ||= store.fetch(type: "unnecessary_deps")
+  end
+
+  def flush_cache_and_check_dylibs
+    reset_dylibs!
+
     @keg.find do |file|
       next if file.symlink? || file.directory?
       next unless file.dylib? || file.binary_executable? || file.mach_o_bundle?
@@ -54,6 +83,7 @@ class LinkageChecker
     end
 
     @indirect_deps, @undeclared_deps, @unnecessary_deps = check_undeclared_deps if formula
+    store_dylibs!
   end
 
   def check_undeclared_deps
@@ -99,18 +129,18 @@ class LinkageChecker
   end
 
   def display_normal_output
-    display_items "System libraries", @system_dylibs
-    display_items "Homebrew libraries", @brewed_dylibs
-    display_items "Indirect dependencies with linkage", @indirect_deps
-    display_items "Variable-referenced libraries", @variable_dylibs
-    display_items "Missing libraries", @broken_dylibs
-    display_items "Undeclared dependencies with linkage", @undeclared_deps
-    display_items "Dependencies with no linkage", @unnecessary_deps
+    display_items "System libraries", system_dylibs
+    display_items "Homebrew libraries", brewed_dylibs
+    display_items "Indirect dependencies with linkage", indirect_deps
+    display_items "Variable-referenced libraries", variable_dylibs
+    display_items "Missing libraries", broken_dylibs
+    display_items "Undeclared dependencies with linkage", undeclared_deps
+    display_items "Dependencies with no linkage", unnecessary_deps
   end
 
   def display_reverse_output
-    return if @reverse_links.empty?
-    sorted = @reverse_links.sort
+    return if reverse_links.empty?
+    sorted = reverse_links.sort
     sorted.each do |dylib, files|
       puts dylib
       files.each do |f|
@@ -122,21 +152,21 @@ class LinkageChecker
   end
 
   def display_test_output
-    display_items "Missing libraries", @broken_dylibs
-    display_items "Possible unnecessary dependencies", @unnecessary_deps
-    puts "No broken dylib links" if @broken_dylibs.empty?
+    display_items "Missing libraries", broken_dylibs
+    display_items "Possible unnecessary dependencies", unnecessary_deps
+    puts "No broken dylib links" if broken_dylibs.empty?
   end
 
   def broken_dylibs?
-    !@broken_dylibs.empty?
+    !broken_dylibs.empty?
   end
 
   def undeclared_deps?
-    !@undeclared_deps.empty?
+    !undeclared_deps.empty?
   end
 
   def unnecessary_deps?
-    !@unnecessary_deps.empty?
+    !unnecessary_deps.empty?
   end
 
   private
@@ -175,4 +205,39 @@ class LinkageChecker
   rescue FormulaUnavailableError
     opoo "Formula unavailable: #{keg.name}"
   end
+
+  # Helper function to reset dylib values when building cache
+  #
+  # @return [nil]
+  def reset_dylibs!
+    store.flush_cache!
+    @system_dylibs    = Set.new
+    @broken_dylibs    = Set.new
+    @variable_dylibs  = Set.new
+    @brewed_dylibs    = Hash.new { |h, k| h[k] = Set.new }
+    @reverse_links    = Hash.new { |h, k| h[k] = Set.new }
+    @indirect_deps    = []
+    @undeclared_deps  = []
+    @unnecessary_deps = []
+  end
+
+  # Updates data store with package path values
+  #
+  # @return [nil]
+  def store_dylibs!
+    store.update!(
+      path_values: {
+        "system_dylibs"    => @system_dylibs,
+        "variable_dylibs"  => @variable_dylibs,
+        "broken_dylibs"    => @broken_dylibs,
+        "indirect_deps"    => @indirect_deps,
+        "undeclared_deps"  => @undeclared_deps,
+        "unnecessary_deps" => @unnecessary_deps,
+      },
+      hash_values: {
+        "brewed_dylibs"    => @brewed_dylibs,
+        "reverse_links"    => @reverse_links,
+      },
+    )
+  end
 end