admin/brew
1
0
Fork 0
Code Issues Packages Projects Releases 6 Wiki Activity

Add more tests

Browse Source 
Also, remove most integration tests, because they're slow.
This commit is contained in:
Carlo Cabrera 2023-04-07 22:21:46 +08:00
parent fdc113d85c
commit 5b8ead7bc8
No known key found for this signature in database
GPG Key ID: C74D447FC549A1D0
5 changed files with 386 additions and 143 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

158
Library/Homebrew/test/dev-cmd/determine-test-runners_spec.rb
View File

@ -9,12 +9,7 @@ describe "brew determine-test-runners" do
FileUtils.rm_f github_output
end
# TODO: Generate this dynamically based on our supported macOS versions.
let(:linux_runner) { "ubuntu-22.04" }
let(:all_runners) { ["11", "11-arm64", "12", "12-arm64", "13", "13-arm64", linux_runner] }
let(:intel_runners) { all_runners.reject { |r| r.end_with? "-arm64" } }
let(:arm64_runners) { all_runners - intel_runners }
let(:macos_runners) { all_runners - [linux_runner] }
# We need to make sure we write to a different path for each example.
let(:github_output) { "#{TEST_TMPDIR}/github_output#{DetermineRunnerTestHelper.new.number}" }
let(:ephemeral_suffix) { "-12345-1" }
@ -26,28 +21,23 @@ describe "brew determine-test-runners" do
"GITHUB_RUN_ATTEMPT" => ephemeral_suffix.split("-").third,
}.freeze
end
let(:all_runners) do
out = []
MacOSVersions::SYMBOLS.each_value do |v|
macos_version = OS::Mac::Version.new(v)
next if macos_version.unsupported_release?
out << v
out << "#{v}-arm64"
end
out << linux_runner
out
end
it_behaves_like "parseable arguments"
it "fails without any arguments", :integration_test do
expect { brew "determine-test-runners" }
.to not_to_output.to_stdout
.and be_a_failure
end
it "fails when the necessary environment variables are missing", :integration_test do
setup_test_formula "testball"
runner_env.each_key do |k|
runner_env_dup = runner_env.dup
runner_env_dup[k] = nil
expect { brew "determine-test-runners", "testball", runner_env_dup }
.to not_to_output.to_stdout
.and be_a_failure
end
end
it "assigns all runners for formulae without any requirements", :integration_test do
setup_test_formula "testball"
@ -57,125 +47,7 @@ describe "brew determine-test-runners" do
.and be_a_success
expect(File.read(github_output)).not_to be_empty
expect(get_runners(github_output)).to eq(all_runners)
end
it "assigns all runners when there are deleted formulae", :integration_test do
expect { brew "determine-test-runners", "", "testball", runner_env.merge({ "GITHUB_OUTPUT" => github_output }) }
.to not_to_output.to_stdout
.and not_to_output.to_stderr
.and be_a_success
expect(File.read(github_output)).not_to be_empty
expect(get_runners(github_output)).to eq(all_runners)
end
it "assigns `ubuntu-latest` when there are no testing formulae and no deleted formulae", :integration_test do
expect { brew "determine-test-runners", "", runner_env.merge({ "GITHUB_OUTPUT" => github_output }) }
.to not_to_output.to_stdout
.and not_to_output.to_stderr
.and be_a_success
expect(File.read(github_output)).not_to be_empty
expect(get_runners(github_output)).to eq(["ubuntu-latest"])
end
it "assigns only Intel runners when a formula `depends_on arch: :x86_64`", :integration_test do
setup_test_formula "intel_depender", <<~RUBY
url "https://brew.sh/intel_depender-1.0.tar.gz"
depends_on arch: :x86_64
RUBY
expect { brew "determine-test-runners", "intel_depender", runner_env.merge({ "GITHUB_OUTPUT" => github_output }) }
.to not_to_output.to_stdout
.and not_to_output.to_stderr
.and be_a_success
expect(File.read(github_output)).not_to be_empty
expect(get_runners(github_output)).to eq(intel_runners)
end
it "assigns only ARM64 runners when a formula `depends_on arch: :arm64`", :integration_test do
setup_test_formula "fancy-m1-ml-framework", <<~RUBY
url "https://brew.sh/fancy-m1-ml-framework-1.0.tar.gz"
depends_on arch: :arm64
RUBY
expect do
brew "determine-test-runners", "fancy-m1-ml-framework", runner_env.merge({ "GITHUB_OUTPUT" => github_output })
end
.to not_to_output.to_stdout
.and not_to_output.to_stderr
.and be_a_success
expect(File.read(github_output)).not_to be_empty
expect(get_runners(github_output)).to eq(arm64_runners)
end
it "assigns only macOS runners when a formula `depends_on :macos`", :integration_test do
setup_test_formula "xcode-helper", <<~RUBY
url "https://brew.sh/xcode-helper-1.0.tar.gz"
depends_on :macos
RUBY
expect { brew "determine-test-runners", "xcode-helper", runner_env.merge({ "GITHUB_OUTPUT" => github_output }) }
.to not_to_output.to_stdout
.and not_to_output.to_stderr
.and be_a_success
expect(File.read(github_output)).not_to be_empty
expect(get_runners(github_output)).to eq(macos_runners)
end
it "assigns only Linux runners when a formula `depends_on :linux`", :integration_test do
setup_test_formula "linux-kernel-requirer", <<~RUBY
url "https://brew.sh/linux-kernel-requirer-1.0.tar.gz"
depends_on :linux
RUBY
expect do
brew "determine-test-runners", "linux-kernel-requirer", runner_env.merge({ "GITHUB_OUTPUT" => github_output })
end
.to not_to_output.to_stdout
.and not_to_output.to_stderr
.and be_a_success
expect(File.read(github_output)).not_to be_empty
expect(get_runners(github_output)).to eq([linux_runner])
end
# TODO: Keep this updated to use the newest supported macOS version.
it "assigns only compatible runners when there is a versioned macOS requirement", :integration_test do
setup_test_formula "needs-macos-13", <<~RUBY
url "https://brew.sh/needs-macos-13-1.0.tar.gz"
depends_on macos: :ventura
RUBY
expect { brew "determine-test-runners", "needs-macos-13", runner_env.merge({ "GITHUB_OUTPUT" => github_output }) }
.to not_to_output.to_stdout
.and not_to_output.to_stderr
.and be_a_success
expect(File.read(github_output)).not_to be_empty
expect(get_runners(github_output)).to eq(["13", "13-arm64", linux_runner])
expect(get_runners(github_output)).not_to eq(all_runners)
end
describe "--dependents" do
it "assignes no runners for formulae with no dependents", :integration_test do
setup_test_formula "testball"
expect do
brew "determine-test-runners", "--eval-all", "--dependents", "testball",
runner_env.merge({ "GITHUB_OUTPUT" => github_output })
end
.to not_to_output.to_stdout
.and not_to_output.to_stderr
.and be_a_success
expect(File.read(github_output)).not_to be_empty
expect(get_runners(github_output)).to eq([])
end
expect(get_runners(github_output).sort).to eq(all_runners.sort)
end
end

288
Library/Homebrew/test/github_runner_matrix_spec.rb Normal file
View File

@ -0,0 +1,288 @@
# typed: false
# frozen_string_literal: true
require "github_runner_matrix"
require "test/support/fixtures/testball"
describe GitHubRunnerMatrix do
before do
allow(ENV).to receive(:fetch).with("HOMEBREW_LINUX_RUNNER").and_return("ubuntu-latest")
allow(ENV).to receive(:fetch).with("HOMEBREW_LINUX_CLEANUP").and_return("false")
allow(ENV).to receive(:fetch).with("GITHUB_RUN_ID").and_return("12345")
allow(ENV).to receive(:fetch).with("GITHUB_RUN_ATTEMPT").and_return("1")
end
let(:newest_supported_macos) do
MacOSVersions::SYMBOLS.find { |_, v| !OS::Mac::Version.new(v).prerelease? }
end
let(:testball) { TestRunnerFormula.new(Testball.new) }
let(:testball_depender) { setup_test_runner_formula("testball-depender", ["testball"]) }
let(:testball_depender_linux) { setup_test_runner_formula("testball-depender-linux", ["testball", :linux]) }
let(:testball_depender_macos) { setup_test_runner_formula("testball-depender-macos", ["testball", :macos]) }
let(:testball_depender_intel) do
setup_test_runner_formula("testball-depender-intel", ["testball", { arch: :x86_64 }])
end
let(:testball_depender_arm) { setup_test_runner_formula("testball-depender-arm", ["testball", { arch: :arm64 }]) }
let(:testball_depender_newest) do
symbol, = newest_supported_macos
setup_test_runner_formula("testball-depender-newest", ["testball", { macos: symbol }])
end
describe "#active_runner_specs_hash" do
it "returns an object that responds to `#to_json`" do
expect(
described_class.new([], ["deleted"], dependent_matrix: false)
.active_runner_specs_hash
.respond_to?(:to_json),
).to be(true)
end
end
describe "#generate_runners!" do
it "is idempotent" do
matrix = described_class.new([], [], dependent_matrix: false)
runners = matrix.runners.dup
matrix.send(:generate_runners!)
expect(matrix.runners).to eq(runners)
end
end
context "when there are no testing formulae and no deleted formulae" do
it "activates no test runners" do
expect(described_class.new([], [], dependent_matrix: false).runners.any?(&:active))
.to be(false)
end
it "activates no dependent runners" do
expect(described_class.new([], [], dependent_matrix: true).runners.any?(&:active))
.to be(false)
end
end
context "when there are testing formulae and no deleted formulae" do
context "when it is a matrix for the `tests` job" do
context "when testing formulae have no requirements" do
it "activates all runners" do
expect(described_class.new([testball], [], dependent_matrix: false).runners.all?(&:active))
.to be(true)
end
end
context "when testing formulae require Linux" do
it "activates only the Linux runner" do
runner_matrix = described_class.new([testball_depender_linux], [], dependent_matrix: false)
expect(runner_matrix.runners.all?(&:active)).to be(false)
expect(runner_matrix.runners.any?(&:active)).to be(true)
expect(get_runner_names(runner_matrix)).to eq(["Linux"])
end
end
context "when testing formulae require macOS" do
it "activates only the macOS runners" do
runner_matrix = described_class.new([testball_depender_macos], [], dependent_matrix: false)
expect(runner_matrix.runners.all?(&:active)).to be(false)
expect(runner_matrix.runners.any?(&:active)).to be(true)
expect(get_runner_names(runner_matrix)).to eq(get_runner_names(runner_matrix, :macos?))
end
end
context "when testing formulae require Intel" do
it "activates only the Intel runners" do
runner_matrix = described_class.new([testball_depender_intel], [], dependent_matrix: false)
expect(runner_matrix.runners.all?(&:active)).to be(false)
expect(runner_matrix.runners.any?(&:active)).to be(true)
expect(get_runner_names(runner_matrix)).to eq(get_runner_names(runner_matrix, :x86_64?))
end
end
context "when testing formulae require ARM" do
it "activates only the ARM runners" do
runner_matrix = described_class.new([testball_depender_arm], [], dependent_matrix: false)
expect(runner_matrix.runners.all?(&:active)).to be(false)
expect(runner_matrix.runners.any?(&:active)).to be(true)
expect(get_runner_names(runner_matrix)).to eq(get_runner_names(runner_matrix, :arm64?))
end
end
context "when testing formulae require a macOS version" do
it "activates the Linux runner and suitable macOS runners" do
_, v = newest_supported_macos
runner_matrix = described_class.new([testball_depender_newest], [], dependent_matrix: false)
expect(runner_matrix.runners.all?(&:active)).to be(false)
expect(runner_matrix.runners.any?(&:active)).to be(true)
expect(get_runner_names(runner_matrix).sort).to eq(["Linux", "macOS #{v}-arm64", "macOS #{v}-x86_64"])
end
end
end
context "when it is a matrix for the `test_deps` job" do
context "when testing formulae have no dependents" do
it "activates no runners" do
allow(Homebrew::EnvConfig).to receive(:eval_all?).and_return(true)
allow(Formula).to receive(:all).and_return([testball].map(&:formula))
expect(described_class.new([testball], [], dependent_matrix: true).runners.any?(&:active))
.to be(false)
end
end
context "when testing formulae have dependents" do
context "when dependents have no requirements" do
it "activates all runners" do
allow(Homebrew::EnvConfig).to receive(:eval_all?).and_return(true)
allow(Formula).to receive(:all).and_return([testball, testball_depender].map(&:formula))
expect(described_class.new([testball], [], dependent_matrix: true).runners.all?(&:active))
.to be(true)
end
end
context "when dependents require Linux" do
it "activates only Linux runners" do
allow(Homebrew::EnvConfig).to receive(:eval_all?).and_return(true)
allow(Formula).to receive(:all).and_return([testball, testball_depender_linux].map(&:formula))
runner_matrix = described_class.new([testball], [], dependent_matrix: true)
expect(runner_matrix.runners.all?(&:active)).to be(false)
expect(runner_matrix.runners.any?(&:active)).to be(true)
expect(get_runner_names(runner_matrix)).to eq(get_runner_names(runner_matrix, :linux?))
end
end
context "when dependents require macOS" do
it "activates only macOS runners" do
allow(Homebrew::EnvConfig).to receive(:eval_all?).and_return(true)
allow(Formula).to receive(:all).and_return([testball, testball_depender_macos].map(&:formula))
runner_matrix = described_class.new([testball], [], dependent_matrix: true)
expect(runner_matrix.runners.all?(&:active)).to be(false)
expect(runner_matrix.runners.any?(&:active)).to be(true)
expect(get_runner_names(runner_matrix)).to eq(get_runner_names(runner_matrix, :macos?))
end
end
context "when dependents require an Intel architecture" do
it "activates only Intel runners" do
allow(Homebrew::EnvConfig).to receive(:eval_all?).and_return(true)
allow(Formula).to receive(:all).and_return([testball, testball_depender_intel].map(&:formula))
runner_matrix = described_class.new([testball], [], dependent_matrix: true)
expect(runner_matrix.runners.all?(&:active)).to be(false)
expect(runner_matrix.runners.any?(&:active)).to be(true)
expect(get_runner_names(runner_matrix)).to eq(get_runner_names(runner_matrix, :x86_64?))
end
end
context "when dependents require an ARM architecture" do
it "activates only ARM runners" do
allow(Homebrew::EnvConfig).to receive(:eval_all?).and_return(true)
allow(Formula).to receive(:all).and_return([testball, testball_depender_arm].map(&:formula))
runner_matrix = described_class.new([testball], [], dependent_matrix: true)
expect(runner_matrix.runners.all?(&:active)).to be(false)
expect(runner_matrix.runners.any?(&:active)).to be(true)
expect(get_runner_names(runner_matrix)).to eq(get_runner_names(runner_matrix, :arm64?))
end
end
end
end
end
context "when there are deleted formulae" do
context "when it is a matrix for the `tests` job" do
it "activates all runners" do
expect(described_class.new([], ["deleted"], dependent_matrix: false).runners.all?(&:active))
.to be(true)
end
end
context "when it is a matrix for the `test_deps` job" do
context "when there are no testing formulae" do
it "activates no runners" do
expect(described_class.new([], ["deleted"], dependent_matrix: true).runners.any?(&:active))
.to be(false)
end
end
context "when there are testing formulae with no dependents" do
it "activates no runners" do
testing_formulae = [testball]
runner_matrix = described_class.new(testing_formulae, ["deleted"], dependent_matrix: true)
allow(Homebrew::EnvConfig).to receive(:eval_all?).and_return(true)
allow(Formula).to receive(:all).and_return(testing_formulae.map(&:formula))
expect(runner_matrix.runners.none?(&:active)).to be(true)
end
end
context "when there are testing formulae with dependents" do
context "when dependent formulae have no requirements" do
it "activates the applicable runners" do
allow(Homebrew::EnvConfig).to receive(:eval_all?).and_return(true)
allow(Formula).to receive(:all).and_return([testball, testball_depender].map(&:formula))
testing_formulae = [testball]
expect(described_class.new(testing_formulae, ["deleted"], dependent_matrix: true).runners.all?(&:active))
.to be(true)
end
end
context "when dependent formulae have requirements" do
context "when dependent formulae require Linux" do
it "activates the applicable runners" do
allow(Homebrew::EnvConfig).to receive(:eval_all?).and_return(true)
allow(Formula).to receive(:all).and_return([testball, testball_depender_linux].map(&:formula))
testing_formulae = [testball]
matrix = described_class.new(testing_formulae, ["deleted"], dependent_matrix: true)
expect(get_runner_names(matrix)).to eq(["Linux"])
end
end
context "when dependent formulae require macOS" do
it "activates the applicable runners" do
allow(Homebrew::EnvConfig).to receive(:eval_all?).and_return(true)
allow(Formula).to receive(:all).and_return([testball, testball_depender_macos].map(&:formula))
testing_formulae = [testball]
matrix = described_class.new(testing_formulae, ["deleted"], dependent_matrix: true)
expect(get_runner_names(matrix)).to eq(get_runner_names(matrix, :macos?))
end
end
end
end
end
end
def get_runner_names(runner_matrix, predicate = :active)
runner_matrix.runners
.select(&predicate)
.map(&:spec)
.map(&:name)
end
def setup_test_runner_formula(name, dependencies = [], **kwargs)
f = formula name do
url "https://brew.sh/#{name}-1.0.tar.gz"
dependencies.each { |dependency| depends_on dependency }
kwargs.each do |k, v|
send(:"on_#{k}") do
v.each do |dep|
depends_on dep
end
end
end
end
TestRunnerFormula.new(f)
end
end

34
Library/Homebrew/test/github_runner_spec.rb Normal file
View File

@ -0,0 +1,34 @@
# typed: false
# frozen_string_literal: true
require "github_runner"
describe GitHubRunner do
let(:runner) do
spec = MacOSRunnerSpec.new(name: "macOS 11-arm64", runner: "11-arm64", cleanup: true)
version = OS::Mac::Version.new("11")
described_class.new(platform: :macos, arch: :arm64, spec: spec, macos_version: version)
end
it "has immutable attributes" do
[:platform, :arch, :spec, :macos_version].each do |attribute|
expect(runner.respond_to?("#{attribute}=")).to be(false)
end
end
it "is inactive by default" do
expect(runner.active).to be(false)
end
describe "#macos?" do
it "returns true if the runner is a macOS runner" do
expect(runner.macos?).to be(true)
end
end
describe "#linux?" do
it "returns false if the runner is a macOS runner" do
expect(runner.linux?).to be(false)
end
end
end

29
Library/Homebrew/test/linux_runner_spec_spec.rb Normal file
View File

@ -0,0 +1,29 @@
# typed: false
# frozen_string_literal: true
require "linux_runner_spec"
describe LinuxRunnerSpec do
let(:spec) do
described_class.new(
name: "Linux",
runner: "ubuntu-latest",
container: { image: "ghcr.io/homebrew/ubuntu22.04:master", options: "--user=linuxbrew" },
workdir: "/github/home",
timeout: 360,
cleanup: false,
)
end
it "has immutable attributes" do
[:name, :runner, :container, :workdir, :timeout, :cleanup].each do |attribute|
expect(spec.respond_to?("#{attribute}=")).to be(false)
end
end
describe "#to_h" do
it "returns an object that responds to `#to_json`" do
expect(spec.to_h.respond_to?(:to_json)).to be(true)
end
end
end

20
Library/Homebrew/test/macos_runner_spec_spec.rb Normal file
View File

@ -0,0 +1,20 @@
# typed: false
# frozen_string_literal: true
require "macos_runner_spec"
describe MacOSRunnerSpec do
let(:spec) { described_class.new(name: "macOS 11-arm64", runner: "11-arm64", cleanup: true) }
it "has immutable attributes" do
[:name, :runner, :cleanup].each do |attribute|
expect(spec.respond_to?("#{attribute}=")).to be(false)
end
end
describe "#to_h" do
it "returns an object that responds to `#to_json`" do
expect(spec.to_h.respond_to?(:to_json)).to be(true)
end
end
end