Merge pull request #4382 from sjackman/keg_relocate

Relocate bottles on Linux using patchelf

Library/Homebrew/dev-cmd/bottle.rb

@@ -87,11 +87,20 @@ module Homebrew
     end
 
     return merge if args.merge?
+    ensure_relocation_formulae_installed!
     ARGV.resolved_formulae.each do |f|
       bottle_formula f
     end
   end
 
+  def ensure_relocation_formulae_installed!
+    Keg.relocation_formulae.each do |f|
+      next if Formula[f].installed?
+      ohai "Installing #{f}..."
+      safe_system HOMEBREW_BREW_FILE, "install", f
+    end
+  end
+
   def keg_contain?(string, keg, ignores)
     @put_string_exists_header, @put_filenames = nil
 

Library/Homebrew/extend/os/keg_relocate.rb

@@ -1 +1,5 @@
-require "extend/os/mac/keg_relocate" if OS.mac?
+if OS.mac?
+  require "extend/os/mac/keg_relocate"
+elsif OS.linux?
+  require "extend/os/linux/keg_relocate"
+end

Library/Homebrew/extend/os/linux/keg_relocate.rb

@@ -0,0 +1,81 @@
+class Keg
+  def relocate_dynamic_linkage(relocation)
+    # Patching patchelf using itself fails with "Text file busy" or SIGBUS.
+    return if name == "patchelf"
+
+    elf_files.each do |file|
+      file.ensure_writable do
+        change_rpath(file, relocation.old_prefix, relocation.new_prefix)
+      end
+    end
+  end
+
+  def change_rpath(file, old_prefix, new_prefix)
+    return if !file.elf? || !file.dynamic_elf?
+
+    patchelf = DevelopmentTools.locate "patchelf"
+    cmd_rpath = [patchelf, "--print-rpath", file]
+    old_rpath = Utils.popen_read(*cmd_rpath, err: :out).strip
+
+    # patchelf requires that the ELF file have a .dynstr section.
+    # Skip ELF files that do not have a .dynstr section.
+    return if ["cannot find section .dynstr", "strange: no string table"].include?(old_rpath)
+    raise ErrorDuringExecution, "#{cmd_rpath}\n#{old_rpath}" unless $CHILD_STATUS.success?
+
+    rpath = old_rpath
+            .split(":")
+            .map { |x| x.sub(old_prefix, new_prefix) }
+            .select { |x| x.start_with?(new_prefix, "$ORIGIN") }
+
+    lib_path = "#{new_prefix}/lib"
+    rpath << lib_path unless rpath.include? lib_path
+    new_rpath = rpath.join(":")
+    cmd = [patchelf, "--force-rpath", "--set-rpath", new_rpath]
+
+    if file.binary_executable?
+      old_interpreter = Utils.safe_popen_read(patchelf, "--print-interpreter", file).strip
+      new_interpreter = if File.readable? "#{new_prefix}/lib/ld.so"
+        "#{new_prefix}/lib/ld.so"
+      else
+        old_interpreter.sub old_prefix, new_prefix
+      end
+      cmd << "--set-interpreter" << new_interpreter if old_interpreter != new_interpreter
+    end
+
+    return if old_rpath == new_rpath && old_interpreter == new_interpreter
+    safe_system(*cmd, file)
+  end
+
+  def detect_cxx_stdlibs(options = {})
+    skip_executables = options.fetch(:skip_executables, false)
+    results = Set.new
+    elf_files.each do |file|
+      next unless file.dynamic_elf?
+      next if file.binary_executable? && skip_executables
+      dylibs = file.dynamically_linked_libraries
+      results << :libcxx if dylibs.any? { |s| s.include? "libc++.so" }
+      results << :libstdcxx if dylibs.any? { |s| s.include? "libstdc++.so" }
+    end
+    results.to_a
+  end
+
+  def elf_files
+    hardlinks = Set.new
+    elf_files = []
+    path.find do |pn|
+      next if pn.symlink? || pn.directory?
+      next if !pn.dylib? && !pn.binary_executable?
+
+      # If we've already processed a file, ignore its hardlinks (which have the
+      # same dev ID and inode). This prevents relocations from being performed
+      # on a binary more than once.
+      next unless hardlinks.add? [pn.stat.dev, pn.stat.ino]
+      elf_files << pn
+    end
+    elf_files
+  end
+
+  def self.relocation_formulae
+    ["patchelf"]
+  end
+end

Library/Homebrew/formula_installer.rb

@@ -473,6 +473,7 @@ class FormulaInstaller
 
   def expand_dependencies(deps)
     inherited_options = Hash.new { |hash, key| hash[key] = Options.new }
+    pour_bottle = pour_bottle?
 
     expanded_deps = Dependency.expand(formula, deps) do |dependent, dep|
       inherited_options[dep.name] |= inherited_options_for(dep)
@@ -480,6 +481,7 @@ class FormulaInstaller
         dependent,
         inherited_options.fetch(dependent.name, []),
       )
+      pour_bottle = true if install_bottle_for?(dep.to_formula, build)
 
       if dep.prune_from_option?(build)
         Dependency.prune
@@ -494,6 +496,16 @@ class FormulaInstaller
       end
     end
 
+    if pour_bottle
+      bottle_deps = Keg.relocation_formulae
+                       .map { |formula| Dependency.new(formula) }
+                       .reject do |dep|
+        inherited_options[dep.name] |= inherited_options_for(dep)
+        dep.satisfied? inherited_options[dep.name]
+      end
+      expanded_deps = Dependency.merge_repeats(bottle_deps + expanded_deps) unless bottle_deps.empty?
+    end
+
     expanded_deps.map { |dep| [dep, inherited_options[dep.name]] }
   end
 

Library/Homebrew/keg_relocate.rb

@@ -175,6 +175,10 @@ class Keg
   def self.file_linked_libraries(_file, _string)
     []
   end
+
+  def self.relocation_formulae
+    []
+  end
 end
 
 require "extend/os/keg_relocate"

Library/Homebrew/test/dev-cmd/bottle_spec.rb

@@ -4,6 +4,9 @@ describe "brew bottle", :integration_test do
       expect { brew "install", "--build-bottle", testball }
         .to be_a_success
 
+      setup_test_formula "patchelf"
+      (HOMEBREW_CELLAR/"patchelf/1.0/bin").mkpath
+
       expect { brew "bottle", "--no-rebuild", testball }
         .to output(/Formula not from core or any taps/).to_stderr
         .and not_to_output.to_stdout

Library/Homebrew/test/formula_installer_bottle_spec.rb

@@ -17,6 +17,9 @@ describe FormulaInstaller do
     expect(formula).to be_bottled
     expect(formula).to pour_bottle
 
+    stub_formula_loader formula
+    stub_formula_loader formula("patchelf") { url "patchelf-1.0" }
+    allow(Formula["patchelf"]).to receive(:installed?).and_return(true)
     described_class.new(formula).install
 
     keg = Keg.new(formula.prefix)

Library/Homebrew/test/formulary_spec.rb

@@ -136,20 +136,26 @@ describe Formulary do
     end
 
     context "with installed Formula" do
-      let(:formula) { described_class.factory(formula_path) }
-      let(:installer) { FormulaInstaller.new(formula) }
+      before do
+        allow(Formulary).to receive(:loader_for).and_call_original
+        stub_formula_loader formula("patchelf") { url "patchelf-1.0" }
+        allow(Formula["patchelf"]).to receive(:installed?).and_return(true)
+      end
+
+      let(:installed_formula) { described_class.factory(formula_path) }
+      let(:installer) { FormulaInstaller.new(installed_formula) }
 
       it "returns a Formula when given a rack" do
         installer.install
 
-        f = described_class.from_rack(formula.rack)
+        f = described_class.from_rack(installed_formula.rack)
         expect(f).to be_kind_of(Formula)
       end
 
       it "returns a Formula when given a Keg" do
         installer.install
 
-        keg = Keg.new(formula.prefix)
+        keg = Keg.new(installed_formula.prefix)
         f = described_class.from_keg(keg)
         expect(f).to be_kind_of(Formula)
       end

Library/Homebrew/test/support/helper/spec/shared_context/integration_test.rb

@@ -157,6 +157,10 @@ RSpec.shared_context "integration test" do
         url "https://example.com/#{name}-1.0"
         depends_on "foo"
       RUBY
+    when "patchelf"
+      content = <<~RUBY
+        url "https://example.com/#{name}-1.0"
+      RUBY
     end
 
     Formulary.core_path(name).tap do |formula_path|

Library/Homebrew/utils/popen.rb

@@ -3,10 +3,22 @@ module Utils
     popen(args, "rb", options, &block)
   end
 
+  def self.safe_popen_read(*args, **options, &block)
+    output = popen_read(*args, **options, &block)
+    raise ErrorDuringExecution, args unless $CHILD_STATUS.success?
+    output
+  end
+
   def self.popen_write(*args, **options, &block)
     popen(args, "wb", options, &block)
   end
 
+  def self.safe_popen_write(*args, **options, &block)
+    output = popen_write(args, **options, &block)
+    raise ErrorDuringExecution, args unless $CHILD_STATUS.success?
+    output
+  end
+
   def self.popen(args, mode, options = {})
     IO.popen("-", mode) do |pipe|
       if pipe