From d271614872279cce7cec6344d3262f5a19315787 Mon Sep 17 00:00:00 2001 From: Michka Popoff Date: Thu, 25 Aug 2022 11:04:37 +0100 Subject: [PATCH] install glibc/gcc automatically if too old. Right now this is done through the gcc@5 formula. See https://github.com/Homebrew/homebrew-core/blob/9692318ca653f58857fec12136381c9cec290aa9/Formula/gcc%405.rb#L33 This is fragile because when we will migrate to gcc@11 we have to think about migrating the installation from one gcc formula to another.. Also, not having the right glibc version results in a non-functional brew installation on an older Linux: the glibc installation needs to be done by brew, and not by a workaround in a specific formula Co-Authored-By: Mike McQuaid Co-Authored-By: Bo Anderson Co-Authored-By: Shaun Jackman --- Library/Homebrew/dependency_collector.rb | 11 ++ Library/Homebrew/development_tools.rb | 10 ++ .../extend/os/dependency_collector.rb | 6 +- .../extend/os/linux/dependency_collector.rb | 76 ++++++++++++ .../extend/os/linux/development_tools.rb | 12 ++ Library/Homebrew/extend/os/linux/formula.rb | 17 +++ .../extend/os/linux/linkage_checker.rb | 4 - Library/Homebrew/formula.rb | 8 +- Library/Homebrew/formula_installer.rb | 28 +---- Library/Homebrew/keg_relocate.rb | 11 -- Library/Homebrew/os/linux/glibc.rb | 5 + Library/Homebrew/software_spec.rb | 5 + .../test/installed_dependents_spec.rb | 109 +++++++++++------- 13 files changed, 215 insertions(+), 87 deletions(-) create mode 100644 Library/Homebrew/extend/os/linux/dependency_collector.rb diff --git a/Library/Homebrew/dependency_collector.rb b/Library/Homebrew/dependency_collector.rb index f4864b9051..c1f4085b9d 100644 --- a/Library/Homebrew/dependency_collector.rb +++ b/Library/Homebrew/dependency_collector.rb @@ -28,6 +28,8 @@ class DependencyCollector def initialize @deps = Dependencies.new @requirements = Requirements.new + + init_global_dep_tree_if_needed! end def initialize_copy(other) @@ -68,6 +70,12 @@ class DependencyCollector parse_spec(spec, Array(tags)) end + sig { params(related_formula_names: T::Array[String]).returns(T.nilable(Dependency)) } + def gcc_dep_if_needed(related_formula_names); end + + sig { params(related_formula_names: T::Array[String]).returns(T.nilable(Dependency)) } + def glibc_dep_if_needed(related_formula_names); end + def git_dep_if_needed(tags) return if Utils::Git.available? @@ -110,6 +118,9 @@ class DependencyCollector private + sig { void } + def init_global_dep_tree_if_needed!; end + def parse_spec(spec, tags) case spec when String diff --git a/Library/Homebrew/development_tools.rb b/Library/Homebrew/development_tools.rb index 6ccda875a4..e1288c9bcb 100644 --- a/Library/Homebrew/development_tools.rb +++ b/Library/Homebrew/development_tools.rb @@ -98,6 +98,16 @@ class DevelopmentTools @gcc_version = {} end + sig { returns(T::Boolean) } + def build_system_too_old? + false + end + + sig { returns(T::Boolean) } + def system_gcc_too_old? + false + end + sig { returns(T::Boolean) } def ca_file_handles_most_https_certificates? # The system CA file is too old for some modern HTTPS certificates on diff --git a/Library/Homebrew/extend/os/dependency_collector.rb b/Library/Homebrew/extend/os/dependency_collector.rb index 32a2466933..58a3f23994 100644 --- a/Library/Homebrew/extend/os/dependency_collector.rb +++ b/Library/Homebrew/extend/os/dependency_collector.rb @@ -1,4 +1,8 @@ # typed: strict # frozen_string_literal: true -require "extend/os/mac/dependency_collector" if OS.mac? +if OS.mac? + require "extend/os/mac/dependency_collector" +elsif OS.linux? + require "extend/os/linux/dependency_collector" +end diff --git a/Library/Homebrew/extend/os/linux/dependency_collector.rb b/Library/Homebrew/extend/os/linux/dependency_collector.rb new file mode 100644 index 0000000000..f7ec7a1c0a --- /dev/null +++ b/Library/Homebrew/extend/os/linux/dependency_collector.rb @@ -0,0 +1,76 @@ +# typed: true +# frozen_string_literal: true + +require "os/linux/glibc" + +class DependencyCollector + extend T::Sig + + undef gcc_dep_if_needed + undef glibc_dep_if_needed + undef init_global_dep_tree_if_needed! + + sig { params(related_formula_names: T::Set[String]).returns(T.nilable(Dependency)) } + def gcc_dep_if_needed(related_formula_names) + return unless DevelopmentTools.system_gcc_too_old? + return if related_formula_names.include?(GCC) + return if global_dep_tree[GCC]&.intersect?(related_formula_names) + return if global_dep_tree[GLIBC]&.intersect?(related_formula_names) # gcc depends on glibc + + Dependency.new(GCC) + end + + sig { params(related_formula_names: T::Set[String]).returns(T.nilable(Dependency)) } + def glibc_dep_if_needed(related_formula_names) + return unless OS::Linux::Glibc.below_ci_version? + return if global_dep_tree[GLIBC]&.intersect?(related_formula_names) + + Dependency.new(GLIBC) + end + + private + + GLIBC = "glibc" + GCC = CompilerSelector.preferred_gcc.freeze + + # Use class variables to avoid this expensive logic needing to be done more + # than once. + # rubocop:disable Style/ClassVars + @@global_dep_tree = {} + + sig { void } + def init_global_dep_tree_if_needed! + return unless DevelopmentTools.build_system_too_old? + return if @@global_dep_tree.present? + + # Defined in precedence order (gcc depends on glibc). + global_deps = [GLIBC, GCC].freeze + + @@global_dep_tree = global_deps.to_h { |name| [name, Set.new([name])] } + + global_deps.each do |global_dep_name| + # This is an arbitrary number picked based on testing the current tree + # depth and just to ensure that this doesn't loop indefinitely if we + # introduce a circular dependency by mistake. + maximum_tree_depth = 10 + current_tree_depth = 0 + + deps = Formula[global_dep_name].deps + while deps.present? + current_tree_depth += 1 + if current_tree_depth > maximum_tree_depth + raise "maximum tree depth (#{maximum_tree_depth}) exceeded calculating #{global_dep_name} dependency tree!" + end + + @@global_dep_tree[global_dep_name].merge(deps.map(&:name)) + deps = deps.flat_map { |dep| dep.to_formula.deps } + end + end + end + + sig { returns(T::Hash[String, T::Set[String]]) } + def global_dep_tree + @@global_dep_tree + end + # rubocop:enable Style/ClassVars +end diff --git a/Library/Homebrew/extend/os/linux/development_tools.rb b/Library/Homebrew/extend/os/linux/development_tools.rb index 2648ebca68..35c83585f9 100644 --- a/Library/Homebrew/extend/os/linux/development_tools.rb +++ b/Library/Homebrew/extend/os/linux/development_tools.rb @@ -21,6 +21,18 @@ class DevelopmentTools :gcc end + sig { returns(T::Boolean) } + def build_system_too_old? + return @build_system_too_old if defined? @build_system_too_old + + @build_system_too_old = (system_gcc_too_old? || OS::Linux::Glibc.below_ci_version?) + end + + sig { returns(T::Boolean) } + def system_gcc_too_old? + gcc_version("gcc") < OS::LINUX_GCC_CI_VERSION + end + sig { returns(T::Hash[String, T.nilable(String)]) } def build_system_info generic_build_system_info.merge({ diff --git a/Library/Homebrew/extend/os/linux/formula.rb b/Library/Homebrew/extend/os/linux/formula.rb index 5f46e710d0..933604b104 100644 --- a/Library/Homebrew/extend/os/linux/formula.rb +++ b/Library/Homebrew/extend/os/linux/formula.rb @@ -5,6 +5,7 @@ class Formula undef shared_library undef loader_path undef deuniversalize_machos + undef add_global_deps_to_spec sig { params(name: String, version: T.nilable(T.any(String, Integer))).returns(String) } def shared_library(name, version = nil) @@ -23,4 +24,20 @@ class Formula sig { params(targets: T.nilable(T.any(Pathname, String))).void } def deuniversalize_machos(*targets); end + + sig { params(spec: SoftwareSpec).void } + def add_global_deps_to_spec(spec) + @global_deps ||= begin + dependency_collector = spec.dependency_collector + related_formula_names = Set.new([ + name, + *versioned_formulae_names, + ]) + [ + dependency_collector.gcc_dep_if_needed(related_formula_names), + dependency_collector.glibc_dep_if_needed(related_formula_names), + ].compact.freeze + end + @global_deps.each { |dep| spec.dependency_collector.add(dep) } + end end diff --git a/Library/Homebrew/extend/os/linux/linkage_checker.rb b/Library/Homebrew/extend/os/linux/linkage_checker.rb index 95146eb1f4..583107d85e 100644 --- a/Library/Homebrew/extend/os/linux/linkage_checker.rb +++ b/Library/Homebrew/extend/os/linux/linkage_checker.rb @@ -80,9 +80,5 @@ class LinkageChecker @unwanted_system_dylibs = @system_dylibs.reject do |s| SYSTEM_LIBRARY_ALLOWLIST.include? File.basename(s) end - # FIXME: Remove this when these dependencies are injected correctly (e.g. through `DependencyCollector`) - # See discussion at - # https://github.com/Homebrew/brew/pull/13577 - @undeclared_deps -= [CompilerSelector.preferred_gcc, "glibc", "gcc"] end end diff --git a/Library/Homebrew/formula.rb b/Library/Homebrew/formula.rb index 1cc59e76d0..e3974cbfa6 100644 --- a/Library/Homebrew/formula.rb +++ b/Library/Homebrew/formula.rb @@ -262,9 +262,13 @@ class Formula return unless spec.url spec.owner = self + add_global_deps_to_spec(spec) instance_variable_set("@#{name}", spec) end + sig { params(spec: SoftwareSpec).void } + def add_global_deps_to_spec(spec); end + def determine_active_spec(requested) spec = send(requested) || stable || head spec || raise(FormulaSpecificationError, "formulae require at least a URL") @@ -443,7 +447,7 @@ class Formula # Returns any `@`-versioned formulae names for any formula (including versioned formulae). sig { returns(T::Array[String]) } def versioned_formulae_names - @versioned_formulae_names ||= Pathname.glob(path.to_s.gsub(/(@[\d.]+)?\.rb$/, "@*.rb")).map do |versioned_path| + Pathname.glob(path.to_s.gsub(/(@[\d.]+)?\.rb$/, "@*.rb")).map do |versioned_path| next if versioned_path == path versioned_path.basename(".rb").to_s @@ -453,7 +457,7 @@ class Formula # Returns any `@`-versioned Formula objects for any Formula (including versioned formulae). sig { returns(T::Array[Formula]) } def versioned_formulae - @versioned_formulae ||= versioned_formulae_names.map do |name| + versioned_formulae_names.map do |name| Formula[name] rescue FormulaUnavailableError nil diff --git a/Library/Homebrew/formula_installer.rb b/Library/Homebrew/formula_installer.rb index e895c09215..0f83751c7e 100644 --- a/Library/Homebrew/formula_installer.rb +++ b/Library/Homebrew/formula_installer.rb @@ -581,11 +581,9 @@ class FormulaInstaller end def expand_dependencies_for_formula(formula, inherited_options) - any_bottle_used = false - # Cache for this expansion only. FormulaInstaller has a lot of inputs which can alter expansion. cache_key = "FormulaInstaller-#{formula.full_name}-#{Time.now.to_f}" - expanded_deps = Dependency.expand(formula, cache_key: cache_key) do |dependent, dep| + Dependency.expand(formula, cache_key: cache_key) do |dependent, dep| inherited_options[dep.name] |= inherited_options_for(dep) build = effective_build_options_for( dependent, @@ -601,36 +599,14 @@ class FormulaInstaller Dependency.prune elsif dep.satisfied?(inherited_options[dep.name]) Dependency.skip - else - any_bottle_used ||= install_bottle_for?(dep.to_formula, build) end end - - [expanded_deps, any_bottle_used] end def expand_dependencies inherited_options = Hash.new { |hash, key| hash[key] = Options.new } - any_bottle_used = pour_bottle? - expanded_deps, any_dep_bottle_used = expand_dependencies_for_formula(formula, inherited_options) - any_bottle_used ||= any_dep_bottle_used - - # We require some dependencies (glibc, GCC 5, etc.) if binaries were built. - # Native binaries shouldn't exist in cross-platform `all` bottles. - if any_bottle_used && !formula.bottled?(:all) && !Keg.bottle_dependencies.empty? - all_bottle_deps = Keg.bottle_dependencies.flat_map do |bottle_dep| - bottle_dep.recursive_dependencies.map(&:name) + [bottle_dep.name] - end - - if all_bottle_deps.exclude?(formula.name) - bottle_deps = Keg.bottle_dependencies.flat_map do |bottle_dep| - expanded_bottle_deps, = expand_dependencies_for_formula(bottle_dep, inherited_options) - expanded_bottle_deps - end - expanded_deps = Dependency.merge_repeats(bottle_deps + expanded_deps) - end - end + expanded_deps = expand_dependencies_for_formula(formula, inherited_options) expanded_deps.map { |dep| [dep, inherited_options[dep.name]] } end diff --git a/Library/Homebrew/keg_relocate.rb b/Library/Homebrew/keg_relocate.rb index ae3db9de42..961d3f27fc 100644 --- a/Library/Homebrew/keg_relocate.rb +++ b/Library/Homebrew/keg_relocate.rb @@ -366,17 +366,6 @@ class Keg def self.file_linked_libraries(_file, _string) [] end - - def self.bottle_dependencies - return [] unless Homebrew::SimulateSystem.simulating_or_running_on_linux? - - @bottle_dependencies ||= begin - formulae = [] - gcc = Formulary.factory(CompilerSelector.preferred_gcc) - formulae << gcc if DevelopmentTools.gcc_version("gcc") < gcc.version.to_i - formulae - end - end end require "extend/os/keg_relocate" diff --git a/Library/Homebrew/os/linux/glibc.rb b/Library/Homebrew/os/linux/glibc.rb index 7478701223..476723e741 100644 --- a/Library/Homebrew/os/linux/glibc.rb +++ b/Library/Homebrew/os/linux/glibc.rb @@ -44,6 +44,11 @@ module OS def below_minimum_version? system_version < minimum_version end + + sig { returns(T::Boolean) } + def below_ci_version? + system_version < LINUX_GLIBC_CI_VERSION + end end end end diff --git a/Library/Homebrew/software_spec.rb b/Library/Homebrew/software_spec.rb index 6b4c15981e..1970f02cd4 100644 --- a/Library/Homebrew/software_spec.rb +++ b/Library/Homebrew/software_spec.rb @@ -50,6 +50,11 @@ class SoftwareSpec @uses_from_macos_elements = [] end + def initialize_copy(other) + super + @dependency_collector = @dependency_collector.dup + end + def owner=(owner) @name = owner.name @full_name = owner.full_name diff --git a/Library/Homebrew/test/installed_dependents_spec.rb b/Library/Homebrew/test/installed_dependents_spec.rb index 6326584b09..29be0cf17d 100644 --- a/Library/Homebrew/test/installed_dependents_spec.rb +++ b/Library/Homebrew/test/installed_dependents_spec.rb @@ -6,7 +6,20 @@ require "installed_dependents" describe InstalledDependents do include FileUtils - def setup_test_keg(name, version) + def stub_formula(name, version = "1.0", &block) + f = formula(name) do + url "#{name}-#{version}" + + instance_eval(&block) if block + end + stub_formula_loader f + stub_formula_loader f, "homebrew/core/#{f}" + f + end + + def setup_test_keg(name, version, &block) + stub_formula(name, version, &block) + path = HOMEBREW_CELLAR/name/version (path/"bin").mkpath @@ -18,27 +31,25 @@ describe InstalledDependents do end let!(:keg) { setup_test_keg("foo", "1.0") } + let!(:keg_only_keg) do + setup_test_keg("foo-keg-only", "1.0") do + keg_only "a good reason" + end + end describe "::find_some_installed_dependents" do - def stub_formula_name(name) - f = formula(name) { url "foo-1.0" } - stub_formula_loader f - stub_formula_loader f, "homebrew/core/#{f}" - f - end - - def setup_test_keg(name, version) - f = stub_formula_name(name) + def setup_test_keg(name, version, &block) keg = super - Tab.create(f, DevelopmentTools.default_compiler, :libcxx).write + Tab.create(keg.to_formula, DevelopmentTools.default_compiler, :libcxx).write keg end before do keg.link + keg_only_keg.optlink end - def alter_tab(keg = dependent) + def alter_tab(keg) tab = Tab.for_keg(keg) yield tab tab.write @@ -46,24 +57,26 @@ describe InstalledDependents do # 1.1.6 is the earliest version of Homebrew that generates correct runtime # dependency lists in {Tab}s. - def dependencies(deps, homebrew_version: "1.1.6") - alter_tab do |tab| + def tab_dependencies(keg, deps, homebrew_version: "1.1.6") + alter_tab(keg) do |tab| tab.homebrew_version = homebrew_version - tab.tabfile = dependent/Tab::FILENAME + tab.tabfile = keg/Tab::FILENAME tab.runtime_dependencies = deps end end - def unreliable_dependencies(deps) + def unreliable_tab_dependencies(keg, deps) # 1.1.5 is (hopefully!) the last version of Homebrew that generates # incorrect runtime dependency lists in {Tab}s. - dependencies(deps, homebrew_version: "1.1.5") + tab_dependencies(keg, deps, homebrew_version: "1.1.5") end - let(:dependent) { setup_test_keg("bar", "1.0") } - specify "a dependency with no Tap in Tab" do tap_dep = setup_test_keg("baz", "1.0") + dependent = setup_test_keg("bar", "1.0") do + depends_on "foo" + depends_on "baz" + end # allow tap_dep to be linked too FileUtils.rm_r tap_dep/"bin" @@ -71,83 +84,93 @@ describe InstalledDependents do alter_tab(keg) { |t| t.source["tap"] = nil } - dependencies nil - Formula["bar"].class.depends_on "foo" - Formula["bar"].class.depends_on "baz" + tab_dependencies dependent, nil result = described_class.find_some_installed_dependents([keg, tap_dep]) expect(result).to eq([[keg, tap_dep], ["bar"]]) end specify "no dependencies anywhere" do - dependencies nil + dependent = setup_test_keg("bar", "1.0") + tab_dependencies dependent, nil expect(described_class.find_some_installed_dependents([keg])).to be_nil end specify "missing Formula dependency" do - dependencies nil - Formula["bar"].class.depends_on "foo" + dependent = setup_test_keg("bar", "1.0") do + depends_on "foo" + end + tab_dependencies dependent, nil expect(described_class.find_some_installed_dependents([keg])).to eq([[keg], ["bar"]]) end specify "uninstalling dependent and dependency" do - dependencies nil - Formula["bar"].class.depends_on "foo" + dependent = setup_test_keg("bar", "1.0") do + depends_on "foo" + end + tab_dependencies dependent, nil expect(described_class.find_some_installed_dependents([keg, dependent])).to be_nil end specify "renamed dependency" do - dependencies nil + dependent = setup_test_keg("bar", "1.0") do + depends_on "foo" + end + tab_dependencies dependent, nil stub_formula_loader Formula["foo"], "homebrew/core/foo-old" renamed_path = HOMEBREW_CELLAR/"foo-old" (HOMEBREW_CELLAR/"foo").rename(renamed_path) - renamed_keg = Keg.new(renamed_path/"1.0") - - Formula["bar"].class.depends_on "foo" + renamed_keg = Keg.new(renamed_path/keg.version.to_s) result = described_class.find_some_installed_dependents([renamed_keg]) expect(result).to eq([[renamed_keg], ["bar"]]) end specify "empty dependencies in Tab" do - dependencies [] + dependent = setup_test_keg("bar", "1.0") + tab_dependencies dependent, [] expect(described_class.find_some_installed_dependents([keg])).to be_nil end specify "same name but different version in Tab" do - dependencies [{ "full_name" => "foo", "version" => "1.1" }] + dependent = setup_test_keg("bar", "1.0") + tab_dependencies dependent, [{ "full_name" => keg.name, "version" => "1.1" }] expect(described_class.find_some_installed_dependents([keg])).to eq([[keg], ["bar"]]) end specify "different name and same version in Tab" do - stub_formula_name("baz") - dependencies [{ "full_name" => "baz", "version" => keg.version.to_s }] + stub_formula("baz") + dependent = setup_test_keg("bar", "1.0") + tab_dependencies dependent, [{ "full_name" => "baz", "version" => keg.version.to_s }] expect(described_class.find_some_installed_dependents([keg])).to be_nil end specify "same name and version in Tab" do - dependencies [{ "full_name" => "foo", "version" => "1.0" }] + dependent = setup_test_keg("bar", "1.0") + tab_dependencies dependent, [{ "full_name" => keg.name, "version" => keg.version.to_s }] expect(described_class.find_some_installed_dependents([keg])).to eq([[keg], ["bar"]]) end specify "fallback for old versions" do - unreliable_dependencies [{ "full_name" => "baz", "version" => "1.0" }] - Formula["bar"].class.depends_on "foo" + dependent = setup_test_keg("bar", "1.0") do + depends_on "foo" + end + unreliable_tab_dependencies dependent, [{ "full_name" => "baz", "version" => "1.0" }] expect(described_class.find_some_installed_dependents([keg])).to eq([[keg], ["bar"]]) end specify "non-opt-linked" do keg.remove_opt_record - dependencies [{ "full_name" => "foo", "version" => "1.0" }] + dependent = setup_test_keg("bar", "1.0") + tab_dependencies dependent, [{ "full_name" => keg.name, "version" => keg.version.to_s }] expect(described_class.find_some_installed_dependents([keg])).to be_nil end specify "keg-only" do - keg.unlink - Formula["foo"].class.keg_only "a good reason" - dependencies [{ "full_name" => "foo", "version" => "1.1" }] # different version - expect(described_class.find_some_installed_dependents([keg])).to eq([[keg], ["bar"]]) + dependent = setup_test_keg("bar", "1.0") + tab_dependencies dependent, [{ "full_name" => keg_only_keg.name, "version" => "1.1" }] # different version + expect(described_class.find_some_installed_dependents([keg_only_keg])).to eq([[keg_only_keg], ["bar"]]) end def stub_cask_name(name, version, dependency)