diff --git a/Library/Homebrew/test/dev-cmd/determine-test-runners_spec.rb b/Library/Homebrew/test/dev-cmd/determine-test-runners_spec.rb index 3a0e2ddf30..769a66a65b 100644 --- a/Library/Homebrew/test/dev-cmd/determine-test-runners_spec.rb +++ b/Library/Homebrew/test/dev-cmd/determine-test-runners_spec.rb @@ -9,12 +9,7 @@ describe "brew determine-test-runners" do FileUtils.rm_f github_output end - # TODO: Generate this dynamically based on our supported macOS versions. let(:linux_runner) { "ubuntu-22.04" } - let(:all_runners) { ["11", "11-arm64", "12", "12-arm64", "13", "13-arm64", linux_runner] } - let(:intel_runners) { all_runners.reject { |r| r.end_with? "-arm64" } } - let(:arm64_runners) { all_runners - intel_runners } - let(:macos_runners) { all_runners - [linux_runner] } # We need to make sure we write to a different path for each example. let(:github_output) { "#{TEST_TMPDIR}/github_output#{DetermineRunnerTestHelper.new.number}" } let(:ephemeral_suffix) { "-12345-1" } @@ -26,28 +21,23 @@ describe "brew determine-test-runners" do "GITHUB_RUN_ATTEMPT" => ephemeral_suffix.split("-").third, }.freeze end + let(:all_runners) do + out = [] + MacOSVersions::SYMBOLS.each_value do |v| + macos_version = OS::Mac::Version.new(v) + next if macos_version.unsupported_release? + + out << v + out << "#{v}-arm64" + end + + out << linux_runner + + out + end it_behaves_like "parseable arguments" - it "fails without any arguments", :integration_test do - expect { brew "determine-test-runners" } - .to not_to_output.to_stdout - .and be_a_failure - end - - it "fails when the necessary environment variables are missing", :integration_test do - setup_test_formula "testball" - - runner_env.each_key do |k| - runner_env_dup = runner_env.dup - runner_env_dup[k] = nil - - expect { brew "determine-test-runners", "testball", runner_env_dup } - .to not_to_output.to_stdout - .and be_a_failure - end - end - it "assigns all runners for formulae without any requirements", :integration_test do setup_test_formula "testball" @@ -57,125 +47,7 @@ describe "brew determine-test-runners" do .and be_a_success expect(File.read(github_output)).not_to be_empty - expect(get_runners(github_output)).to eq(all_runners) - end - - it "assigns all runners when there are deleted formulae", :integration_test do - expect { brew "determine-test-runners", "", "testball", runner_env.merge({ "GITHUB_OUTPUT" => github_output }) } - .to not_to_output.to_stdout - .and not_to_output.to_stderr - .and be_a_success - - expect(File.read(github_output)).not_to be_empty - expect(get_runners(github_output)).to eq(all_runners) - end - - it "assigns `ubuntu-latest` when there are no testing formulae and no deleted formulae", :integration_test do - expect { brew "determine-test-runners", "", runner_env.merge({ "GITHUB_OUTPUT" => github_output }) } - .to not_to_output.to_stdout - .and not_to_output.to_stderr - .and be_a_success - - expect(File.read(github_output)).not_to be_empty - expect(get_runners(github_output)).to eq(["ubuntu-latest"]) - end - - it "assigns only Intel runners when a formula `depends_on arch: :x86_64`", :integration_test do - setup_test_formula "intel_depender", <<~RUBY - url "https://brew.sh/intel_depender-1.0.tar.gz" - depends_on arch: :x86_64 - RUBY - - expect { brew "determine-test-runners", "intel_depender", runner_env.merge({ "GITHUB_OUTPUT" => github_output }) } - .to not_to_output.to_stdout - .and not_to_output.to_stderr - .and be_a_success - - expect(File.read(github_output)).not_to be_empty - expect(get_runners(github_output)).to eq(intel_runners) - end - - it "assigns only ARM64 runners when a formula `depends_on arch: :arm64`", :integration_test do - setup_test_formula "fancy-m1-ml-framework", <<~RUBY - url "https://brew.sh/fancy-m1-ml-framework-1.0.tar.gz" - depends_on arch: :arm64 - RUBY - - expect do - brew "determine-test-runners", "fancy-m1-ml-framework", runner_env.merge({ "GITHUB_OUTPUT" => github_output }) - end - .to not_to_output.to_stdout - .and not_to_output.to_stderr - .and be_a_success - - expect(File.read(github_output)).not_to be_empty - expect(get_runners(github_output)).to eq(arm64_runners) - end - - it "assigns only macOS runners when a formula `depends_on :macos`", :integration_test do - setup_test_formula "xcode-helper", <<~RUBY - url "https://brew.sh/xcode-helper-1.0.tar.gz" - depends_on :macos - RUBY - - expect { brew "determine-test-runners", "xcode-helper", runner_env.merge({ "GITHUB_OUTPUT" => github_output }) } - .to not_to_output.to_stdout - .and not_to_output.to_stderr - .and be_a_success - - expect(File.read(github_output)).not_to be_empty - expect(get_runners(github_output)).to eq(macos_runners) - end - - it "assigns only Linux runners when a formula `depends_on :linux`", :integration_test do - setup_test_formula "linux-kernel-requirer", <<~RUBY - url "https://brew.sh/linux-kernel-requirer-1.0.tar.gz" - depends_on :linux - RUBY - - expect do - brew "determine-test-runners", "linux-kernel-requirer", runner_env.merge({ "GITHUB_OUTPUT" => github_output }) - end - .to not_to_output.to_stdout - .and not_to_output.to_stderr - .and be_a_success - - expect(File.read(github_output)).not_to be_empty - expect(get_runners(github_output)).to eq([linux_runner]) - end - - # TODO: Keep this updated to use the newest supported macOS version. - it "assigns only compatible runners when there is a versioned macOS requirement", :integration_test do - setup_test_formula "needs-macos-13", <<~RUBY - url "https://brew.sh/needs-macos-13-1.0.tar.gz" - depends_on macos: :ventura - RUBY - - expect { brew "determine-test-runners", "needs-macos-13", runner_env.merge({ "GITHUB_OUTPUT" => github_output }) } - .to not_to_output.to_stdout - .and not_to_output.to_stderr - .and be_a_success - - expect(File.read(github_output)).not_to be_empty - expect(get_runners(github_output)).to eq(["13", "13-arm64", linux_runner]) - expect(get_runners(github_output)).not_to eq(all_runners) - end - - describe "--dependents" do - it "assignes no runners for formulae with no dependents", :integration_test do - setup_test_formula "testball" - - expect do - brew "determine-test-runners", "--eval-all", "--dependents", "testball", - runner_env.merge({ "GITHUB_OUTPUT" => github_output }) - end - .to not_to_output.to_stdout - .and not_to_output.to_stderr - .and be_a_success - - expect(File.read(github_output)).not_to be_empty - expect(get_runners(github_output)).to eq([]) - end + expect(get_runners(github_output).sort).to eq(all_runners.sort) end end diff --git a/Library/Homebrew/test/github_runner_matrix_spec.rb b/Library/Homebrew/test/github_runner_matrix_spec.rb new file mode 100644 index 0000000000..37e8260f9d --- /dev/null +++ b/Library/Homebrew/test/github_runner_matrix_spec.rb @@ -0,0 +1,288 @@ +# typed: false +# frozen_string_literal: true + +require "github_runner_matrix" +require "test/support/fixtures/testball" + +describe GitHubRunnerMatrix do + before do + allow(ENV).to receive(:fetch).with("HOMEBREW_LINUX_RUNNER").and_return("ubuntu-latest") + allow(ENV).to receive(:fetch).with("HOMEBREW_LINUX_CLEANUP").and_return("false") + allow(ENV).to receive(:fetch).with("GITHUB_RUN_ID").and_return("12345") + allow(ENV).to receive(:fetch).with("GITHUB_RUN_ATTEMPT").and_return("1") + end + + let(:newest_supported_macos) do + MacOSVersions::SYMBOLS.find { |_, v| !OS::Mac::Version.new(v).prerelease? } + end + + let(:testball) { TestRunnerFormula.new(Testball.new) } + let(:testball_depender) { setup_test_runner_formula("testball-depender", ["testball"]) } + let(:testball_depender_linux) { setup_test_runner_formula("testball-depender-linux", ["testball", :linux]) } + let(:testball_depender_macos) { setup_test_runner_formula("testball-depender-macos", ["testball", :macos]) } + let(:testball_depender_intel) do + setup_test_runner_formula("testball-depender-intel", ["testball", { arch: :x86_64 }]) + end + let(:testball_depender_arm) { setup_test_runner_formula("testball-depender-arm", ["testball", { arch: :arm64 }]) } + let(:testball_depender_newest) do + symbol, = newest_supported_macos + setup_test_runner_formula("testball-depender-newest", ["testball", { macos: symbol }]) + end + + describe "#active_runner_specs_hash" do + it "returns an object that responds to `#to_json`" do + expect( + described_class.new([], ["deleted"], dependent_matrix: false) + .active_runner_specs_hash + .respond_to?(:to_json), + ).to be(true) + end + end + + describe "#generate_runners!" do + it "is idempotent" do + matrix = described_class.new([], [], dependent_matrix: false) + runners = matrix.runners.dup + matrix.send(:generate_runners!) + + expect(matrix.runners).to eq(runners) + end + end + + context "when there are no testing formulae and no deleted formulae" do + it "activates no test runners" do + expect(described_class.new([], [], dependent_matrix: false).runners.any?(&:active)) + .to be(false) + end + + it "activates no dependent runners" do + expect(described_class.new([], [], dependent_matrix: true).runners.any?(&:active)) + .to be(false) + end + end + + context "when there are testing formulae and no deleted formulae" do + context "when it is a matrix for the `tests` job" do + context "when testing formulae have no requirements" do + it "activates all runners" do + expect(described_class.new([testball], [], dependent_matrix: false).runners.all?(&:active)) + .to be(true) + end + end + + context "when testing formulae require Linux" do + it "activates only the Linux runner" do + runner_matrix = described_class.new([testball_depender_linux], [], dependent_matrix: false) + + expect(runner_matrix.runners.all?(&:active)).to be(false) + expect(runner_matrix.runners.any?(&:active)).to be(true) + expect(get_runner_names(runner_matrix)).to eq(["Linux"]) + end + end + + context "when testing formulae require macOS" do + it "activates only the macOS runners" do + runner_matrix = described_class.new([testball_depender_macos], [], dependent_matrix: false) + + expect(runner_matrix.runners.all?(&:active)).to be(false) + expect(runner_matrix.runners.any?(&:active)).to be(true) + expect(get_runner_names(runner_matrix)).to eq(get_runner_names(runner_matrix, :macos?)) + end + end + + context "when testing formulae require Intel" do + it "activates only the Intel runners" do + runner_matrix = described_class.new([testball_depender_intel], [], dependent_matrix: false) + + expect(runner_matrix.runners.all?(&:active)).to be(false) + expect(runner_matrix.runners.any?(&:active)).to be(true) + expect(get_runner_names(runner_matrix)).to eq(get_runner_names(runner_matrix, :x86_64?)) + end + end + + context "when testing formulae require ARM" do + it "activates only the ARM runners" do + runner_matrix = described_class.new([testball_depender_arm], [], dependent_matrix: false) + + expect(runner_matrix.runners.all?(&:active)).to be(false) + expect(runner_matrix.runners.any?(&:active)).to be(true) + expect(get_runner_names(runner_matrix)).to eq(get_runner_names(runner_matrix, :arm64?)) + end + end + + context "when testing formulae require a macOS version" do + it "activates the Linux runner and suitable macOS runners" do + _, v = newest_supported_macos + runner_matrix = described_class.new([testball_depender_newest], [], dependent_matrix: false) + + expect(runner_matrix.runners.all?(&:active)).to be(false) + expect(runner_matrix.runners.any?(&:active)).to be(true) + expect(get_runner_names(runner_matrix).sort).to eq(["Linux", "macOS #{v}-arm64", "macOS #{v}-x86_64"]) + end + end + end + + context "when it is a matrix for the `test_deps` job" do + context "when testing formulae have no dependents" do + it "activates no runners" do + allow(Homebrew::EnvConfig).to receive(:eval_all?).and_return(true) + allow(Formula).to receive(:all).and_return([testball].map(&:formula)) + + expect(described_class.new([testball], [], dependent_matrix: true).runners.any?(&:active)) + .to be(false) + end + end + + context "when testing formulae have dependents" do + context "when dependents have no requirements" do + it "activates all runners" do + allow(Homebrew::EnvConfig).to receive(:eval_all?).and_return(true) + allow(Formula).to receive(:all).and_return([testball, testball_depender].map(&:formula)) + + expect(described_class.new([testball], [], dependent_matrix: true).runners.all?(&:active)) + .to be(true) + end + end + + context "when dependents require Linux" do + it "activates only Linux runners" do + allow(Homebrew::EnvConfig).to receive(:eval_all?).and_return(true) + allow(Formula).to receive(:all).and_return([testball, testball_depender_linux].map(&:formula)) + + runner_matrix = described_class.new([testball], [], dependent_matrix: true) + expect(runner_matrix.runners.all?(&:active)).to be(false) + expect(runner_matrix.runners.any?(&:active)).to be(true) + expect(get_runner_names(runner_matrix)).to eq(get_runner_names(runner_matrix, :linux?)) + end + end + + context "when dependents require macOS" do + it "activates only macOS runners" do + allow(Homebrew::EnvConfig).to receive(:eval_all?).and_return(true) + allow(Formula).to receive(:all).and_return([testball, testball_depender_macos].map(&:formula)) + + runner_matrix = described_class.new([testball], [], dependent_matrix: true) + expect(runner_matrix.runners.all?(&:active)).to be(false) + expect(runner_matrix.runners.any?(&:active)).to be(true) + expect(get_runner_names(runner_matrix)).to eq(get_runner_names(runner_matrix, :macos?)) + end + end + + context "when dependents require an Intel architecture" do + it "activates only Intel runners" do + allow(Homebrew::EnvConfig).to receive(:eval_all?).and_return(true) + allow(Formula).to receive(:all).and_return([testball, testball_depender_intel].map(&:formula)) + + runner_matrix = described_class.new([testball], [], dependent_matrix: true) + expect(runner_matrix.runners.all?(&:active)).to be(false) + expect(runner_matrix.runners.any?(&:active)).to be(true) + expect(get_runner_names(runner_matrix)).to eq(get_runner_names(runner_matrix, :x86_64?)) + end + end + + context "when dependents require an ARM architecture" do + it "activates only ARM runners" do + allow(Homebrew::EnvConfig).to receive(:eval_all?).and_return(true) + allow(Formula).to receive(:all).and_return([testball, testball_depender_arm].map(&:formula)) + + runner_matrix = described_class.new([testball], [], dependent_matrix: true) + expect(runner_matrix.runners.all?(&:active)).to be(false) + expect(runner_matrix.runners.any?(&:active)).to be(true) + expect(get_runner_names(runner_matrix)).to eq(get_runner_names(runner_matrix, :arm64?)) + end + end + end + end + end + + context "when there are deleted formulae" do + context "when it is a matrix for the `tests` job" do + it "activates all runners" do + expect(described_class.new([], ["deleted"], dependent_matrix: false).runners.all?(&:active)) + .to be(true) + end + end + + context "when it is a matrix for the `test_deps` job" do + context "when there are no testing formulae" do + it "activates no runners" do + expect(described_class.new([], ["deleted"], dependent_matrix: true).runners.any?(&:active)) + .to be(false) + end + end + + context "when there are testing formulae with no dependents" do + it "activates no runners" do + testing_formulae = [testball] + runner_matrix = described_class.new(testing_formulae, ["deleted"], dependent_matrix: true) + + allow(Homebrew::EnvConfig).to receive(:eval_all?).and_return(true) + allow(Formula).to receive(:all).and_return(testing_formulae.map(&:formula)) + + expect(runner_matrix.runners.none?(&:active)).to be(true) + end + end + + context "when there are testing formulae with dependents" do + context "when dependent formulae have no requirements" do + it "activates the applicable runners" do + allow(Homebrew::EnvConfig).to receive(:eval_all?).and_return(true) + allow(Formula).to receive(:all).and_return([testball, testball_depender].map(&:formula)) + + testing_formulae = [testball] + expect(described_class.new(testing_formulae, ["deleted"], dependent_matrix: true).runners.all?(&:active)) + .to be(true) + end + end + + context "when dependent formulae have requirements" do + context "when dependent formulae require Linux" do + it "activates the applicable runners" do + allow(Homebrew::EnvConfig).to receive(:eval_all?).and_return(true) + allow(Formula).to receive(:all).and_return([testball, testball_depender_linux].map(&:formula)) + + testing_formulae = [testball] + matrix = described_class.new(testing_formulae, ["deleted"], dependent_matrix: true) + expect(get_runner_names(matrix)).to eq(["Linux"]) + end + end + + context "when dependent formulae require macOS" do + it "activates the applicable runners" do + allow(Homebrew::EnvConfig).to receive(:eval_all?).and_return(true) + allow(Formula).to receive(:all).and_return([testball, testball_depender_macos].map(&:formula)) + + testing_formulae = [testball] + matrix = described_class.new(testing_formulae, ["deleted"], dependent_matrix: true) + expect(get_runner_names(matrix)).to eq(get_runner_names(matrix, :macos?)) + end + end + end + end + end + end + + def get_runner_names(runner_matrix, predicate = :active) + runner_matrix.runners + .select(&predicate) + .map(&:spec) + .map(&:name) + end + + def setup_test_runner_formula(name, dependencies = [], **kwargs) + f = formula name do + url "https://brew.sh/#{name}-1.0.tar.gz" + dependencies.each { |dependency| depends_on dependency } + + kwargs.each do |k, v| + send(:"on_#{k}") do + v.each do |dep| + depends_on dep + end + end + end + end + + TestRunnerFormula.new(f) + end +end diff --git a/Library/Homebrew/test/github_runner_spec.rb b/Library/Homebrew/test/github_runner_spec.rb new file mode 100644 index 0000000000..04cfa8d595 --- /dev/null +++ b/Library/Homebrew/test/github_runner_spec.rb @@ -0,0 +1,34 @@ +# typed: false +# frozen_string_literal: true + +require "github_runner" + +describe GitHubRunner do + let(:runner) do + spec = MacOSRunnerSpec.new(name: "macOS 11-arm64", runner: "11-arm64", cleanup: true) + version = OS::Mac::Version.new("11") + described_class.new(platform: :macos, arch: :arm64, spec: spec, macos_version: version) + end + + it "has immutable attributes" do + [:platform, :arch, :spec, :macos_version].each do |attribute| + expect(runner.respond_to?("#{attribute}=")).to be(false) + end + end + + it "is inactive by default" do + expect(runner.active).to be(false) + end + + describe "#macos?" do + it "returns true if the runner is a macOS runner" do + expect(runner.macos?).to be(true) + end + end + + describe "#linux?" do + it "returns false if the runner is a macOS runner" do + expect(runner.linux?).to be(false) + end + end +end diff --git a/Library/Homebrew/test/linux_runner_spec_spec.rb b/Library/Homebrew/test/linux_runner_spec_spec.rb new file mode 100644 index 0000000000..0f3add6594 --- /dev/null +++ b/Library/Homebrew/test/linux_runner_spec_spec.rb @@ -0,0 +1,29 @@ +# typed: false +# frozen_string_literal: true + +require "linux_runner_spec" + +describe LinuxRunnerSpec do + let(:spec) do + described_class.new( + name: "Linux", + runner: "ubuntu-latest", + container: { image: "ghcr.io/homebrew/ubuntu22.04:master", options: "--user=linuxbrew" }, + workdir: "/github/home", + timeout: 360, + cleanup: false, + ) + end + + it "has immutable attributes" do + [:name, :runner, :container, :workdir, :timeout, :cleanup].each do |attribute| + expect(spec.respond_to?("#{attribute}=")).to be(false) + end + end + + describe "#to_h" do + it "returns an object that responds to `#to_json`" do + expect(spec.to_h.respond_to?(:to_json)).to be(true) + end + end +end diff --git a/Library/Homebrew/test/macos_runner_spec_spec.rb b/Library/Homebrew/test/macos_runner_spec_spec.rb new file mode 100644 index 0000000000..3c56c72805 --- /dev/null +++ b/Library/Homebrew/test/macos_runner_spec_spec.rb @@ -0,0 +1,20 @@ +# typed: false +# frozen_string_literal: true + +require "macos_runner_spec" + +describe MacOSRunnerSpec do + let(:spec) { described_class.new(name: "macOS 11-arm64", runner: "11-arm64", cleanup: true) } + + it "has immutable attributes" do + [:name, :runner, :cleanup].each do |attribute| + expect(spec.respond_to?("#{attribute}=")).to be(false) + end + end + + describe "#to_h" do + it "returns an object that responds to `#to_json`" do + expect(spec.to_h.respond_to?(:to_json)).to be(true) + end + end +end