Merge pull request #1041 from penman/parallel_tests

Parallelise test suite

.gitignore

@@ -13,11 +13,13 @@
 /Library/Homebrew/cask/bin
 /Library/Homebrew/cask/vendor
 /Library/Homebrew/cask/coverage
+/Library/Homebrew/cask/tmp
 /Library/Homebrew/test/.bundle
 /Library/Homebrew/test/bin
 /Library/Homebrew/test/vendor
 /Library/Homebrew/test/coverage
 /Library/Homebrew/test/fs_leak_log
+/Library/Homebrew/tmp
 /Library/LinkedKegs
 /Library/Locks
 /Library/PinnedKegs

Library/Homebrew/.simplecov

@@ -1,8 +1,8 @@
 #!/usr/bin/env ruby
 
 SimpleCov.start do
-  coverage_dir File.expand_path("../coverage", File.realpath(__FILE__))
-  root File.expand_path("../..", File.realpath(__FILE__))
+  coverage_dir File.expand_path("../test/coverage", File.realpath(__FILE__))
+  root File.expand_path("..", File.realpath(__FILE__))
 
   # We manage the result cache ourselves and the default of 10 minutes can be
   # too low (particularly on Travis CI), causing results from some integration

Library/Homebrew/cask/.simplecov

@@ -1 +1 @@
-../test/.simplecov
+../.simplecov

Library/Homebrew/cask/Gemfile

@@ -16,6 +16,7 @@ group :test do
   gem "minitest", "5.4.1"
   gem "minitest-reporters"
   gem "mocha", "1.1.0", require: false
+  gem "parallel_tests"
   gem "rspec", "~> 3.0.0"
   gem "rspec-its", require: false
   gem "rspec-wait", require: false

Library/Homebrew/cask/Gemfile.lock

@@ -23,6 +23,9 @@ GEM
       ruby-progressbar
     mocha (1.1.0)
       metaclass (~> 0.0.1)
+    parallel (1.9.0)
+    parallel_tests (2.9.0)
+      parallel
     parser (2.3.1.2)
       ast (~> 2.2)
     powerpack (0.1.1)
@@ -80,6 +83,7 @@ DEPENDENCIES
   minitest (= 5.4.1)
   minitest-reporters
   mocha (= 1.1.0)
+  parallel_tests
   pry
   pry-byebug
   rake

Library/Homebrew/cask/Rakefile

@@ -7,23 +7,9 @@ $LOAD_PATH.unshift(File.expand_path("#{homebrew_repo}/Library/Homebrew"))
 $LOAD_PATH.unshift(File.expand_path("../lib", __FILE__))
 
 namespace :test do
-  Rake::TestTask.new(:minitest) do |t|
-    # TODO: setting the --seed here is an ugly temporary hack, to remain only
-    #       until test-suite glitches are fixed.
-    ENV["TESTOPTS"] = "--seed=14830" if ENV["TRAVIS"]
-    t.pattern = "test/**/*_test.rb"
-    t.libs << "test"
-  end
-
-  RSpec::Core::RakeTask.new(:rspec)
-
-  desc "Run tests for minitest and RSpec with coverage"
-  task :coverage do
-    ENV["HOMEBREW_TESTS_COVERAGE"] = "1"
-
-    Rake::Task[:test].invoke
-
-    if ENV["CODECOV_TOKEN"]
+  namespace :coverage do
+    desc "Upload coverage to Codecov"
+    task :upload do
       require "simplecov"
       require "codecov"
       formatter = SimpleCov::Formatter::Codecov.new
@@ -32,14 +18,11 @@ namespace :test do
   end
 end
 
-desc "Run tests for minitest and RSpec"
-task test: ["test:minitest", "test:rspec"]
-
 RuboCop::RakeTask.new(:rubocop) do |t|
   t.options = ["--force-exclusion"]
 end
 
-task default: [:test, :rubocop]
+task default: [:rubocop]
 
 desc "Open a REPL for debugging and experimentation"
 task :console do

Library/Homebrew/cask/cmd/brew-cask-tests.rb

@@ -1,5 +1,12 @@
 require "English"
 
+def run_tests(executable, files, args = [])
+  opts = []
+  opts << "--serialize-stdout" if ENV["CI"]
+
+  system "bundle", "exec", executable, *opts, "--", *args, "--", *files
+end
+
 repo_root = Pathname(__FILE__).realpath.parent.parent
 repo_root.cd do
   ENV["HOMEBREW_NO_ANALYTICS_THIS_RUN"] = "1"
@@ -9,12 +16,26 @@ repo_root.cd do
     system "bundle", "install", "--path", "vendor/bundle"
   end
 
-  test_task = "test"
-  %w[rspec minitest coverage].each do |subtask|
-    next unless ARGV.flag?("--#{subtask}")
-    test_task = "test:#{subtask}"
+  rspec = ARGV.flag?("--rspec") || !ARGV.flag?("--minitest")
+  minitest = ARGV.flag?("--minitest") || !ARGV.flag?("--rspec")
+
+  ENV["HOMEBREW_TESTS_COVERAGE"] = "1" if ARGV.flag?("--coverage")
+
+  if rspec
+    run_tests "parallel_rspec", Dir["spec/**/*_spec.rb"], %w[
+      --format progress
+      --format ParallelTests::RSpec::RuntimeLogger
+      --out tmp/parallel_runtime_rspec.log
+    ]
+  end
+
+  if minitest
+    run_tests "parallel_test", Dir["test/**/*_test.rb"]
+  end
+
+  if ENV["CODECOV_TOKEN"]
+    system "bundle", "exec", "rake", "test:coverage:upload"
   end
 
-  system "bundle", "exec", "rake", test_task
   Homebrew.failed = !$CHILD_STATUS.success?
 end

Library/Homebrew/cask/test/test_helper.rb

@@ -39,6 +39,8 @@ require "minitest/autorun"
 require "minitest/reporters"
 Minitest::Reporters.use! Minitest::Reporters::DefaultReporter.new(color: true)
 
+require "parallel_tests/test/runtime_logger"
+
 # Force mocha to patch MiniTest since we have both loaded thanks to homebrew's testing_env
 require "mocha/api"
 require "mocha/integration/mini_test"

Library/Homebrew/dev-cmd/tests.rb

@@ -6,7 +6,7 @@ require "tap"
 
 module Homebrew
   def tests
-    (HOMEBREW_LIBRARY/"Homebrew/test").cd do
+    (HOMEBREW_LIBRARY/"Homebrew").cd do
       ENV["HOMEBREW_NO_ANALYTICS_THIS_RUN"] = "1"
       ENV["HOMEBREW_DEVELOPER"] = "1"
       ENV["TESTOPTS"] = "-v" if ARGV.verbose?
@@ -19,9 +19,11 @@ module Homebrew
 
       if ARGV.include? "--coverage"
         ENV["HOMEBREW_TESTS_COVERAGE"] = "1"
-        FileUtils.rm_f "coverage/.resultset.json"
+        FileUtils.rm_f "test/coverage/.resultset.json"
       end
 
+      ENV["BUNDLE_GEMFILE"] = "#{Dir.pwd}/test/Gemfile"
+
       # Override author/committer as global settings might be invalid and thus
       # will cause silent failure during the setup of dummy Git repositories.
       %w[AUTHOR COMMITTER].each do |role|
@@ -37,16 +39,26 @@ module Homebrew
       # Make it easier to reproduce test runs.
       ENV["SEED"] = ARGV.next if ARGV.include? "--seed"
 
+      files = Dir["test/test_*.rb"]
+      files -= Dir["test/test_os_mac_*.rb"] unless OS.mac?
+
+      opts = []
+      opts << "--serialize-stdout" if ENV["CI"]
+
       args = []
       args << "--trace" if ARGV.include? "--trace"
+
       if ARGV.value("only")
         ENV["HOMEBREW_TESTS_ONLY"] = "1"
         test_name, test_method = ARGV.value("only").split("/", 2)
-        args << "TEST=test_#{test_name}.rb"
-        args << "TESTOPTS=--name=test_#{test_method}" if test_method
+        files = ["test/test_#{test_name}.rb"]
+        args << "--name=test_#{test_method}" if test_method
       end
+
       args += ARGV.named.select { |v| v[/^TEST(OPTS)?=/] }
-      system "bundle", "exec", "rake", "test", *args
+
+      system "bundle", "exec", "parallel_test", *opts,
+        "--", *args, "--", *files
 
       Homebrew.failed = !$?.success?
 

Library/Homebrew/test/Gemfile

@@ -3,6 +3,7 @@ source "https://rubygems.org"
 gem "mocha", "~> 1.1"
 gem "minitest", "~> 5.3"
 gem "rake", "~> 10.3"
+gem "parallel_tests", "~> 2.9"
 
 group :coverage do
   # This is SimpleCov v0.12.0 with one PR merged on top, that finally resolves

Library/Homebrew/test/Gemfile.lock

@@ -21,6 +21,9 @@ GEM
     minitest (5.9.0)
     mocha (1.1.0)
       metaclass (~> 0.0.1)
+    parallel (1.9.0)
+    parallel_tests (2.9.0)
+      parallel
     rake (10.5.0)
     simplecov-html (0.10.0)
     url (0.3.2)
@@ -32,8 +35,9 @@ DEPENDENCIES
   codecov
   minitest (~> 5.3)
   mocha (~> 1.1)
+  parallel_tests (~> 2.9)
   rake (~> 10.3)
   simplecov (= 0.12.0)!
 
 BUNDLED WITH
-   1.12.5
+   1.13.1

Library/Homebrew/test/Rakefile

@@ -1,29 +0,0 @@
-require "rake"
-require "rake/testtask"
-
-def mac?
-  return false if ENV["HOMEBREW_TEST_GENERIC_OS"]
-  RUBY_PLATFORM.to_s.downcase.include? "darwin"
-end
-
-TEST_DIRECTORY = File.dirname(File.expand_path(__FILE__))
-TEST_FILES = Dir["#{TEST_DIRECTORY}/test_*.rb"].reject do |f|
-  f.include?("/test_os_mac_") && !mac?
-end
-
-task default: :test
-
-Rake::TestTask.new(:test) do |t|
-  t.libs << TEST_DIRECTORY
-  t.test_files = TEST_FILES
-end
-
-namespace :test do
-  TEST_FILES.each do |file|
-    name = file[/test_(.+)\.rb/, 1]
-    Rake::TestTask.new(name) do |t|
-      t.libs << TEST_DIRECTORY
-      t.test_files = [file]
-    end
-  end
-end

Library/Homebrew/test/test_inreplace.rb

@@ -91,26 +91,29 @@ class InreplaceTest < Homebrew::TestCase
   end
 
   def test_inreplace_errors
+    require "tempfile"
     extend(Utils::Inreplace)
 
-    open("test", "w") { |f| f.write "a\nb\nc\n" }
+    file = Tempfile.new("test")
+
+    file.write "a\nb\nc\n"
 
     assert_raises(Utils::InreplaceError) do
-      inreplace "test", "d", "f"
+      inreplace file.path, "d", "f"
     end
 
     assert_raises(Utils::InreplaceError) do
       # Under current context, we are testing `String#gsub!`, so let's disable rubocop temporarily.
-      inreplace("test") { |s| s.gsub!("d", "f") } # rubocop:disable Performance/StringReplacement
+      inreplace(file.path) { |s| s.gsub!("d", "f") } # rubocop:disable Performance/StringReplacement
     end
 
     assert_raises(Utils::InreplaceError) do
-      inreplace("test") do |s|
+      inreplace(file.path) do |s|
         s.change_make_var! "VAR", "value"
         s.remove_make_var! "VAR2"
       end
     end
   ensure
-    File.unlink("test")
+    file.unlink
   end
 end

Library/Homebrew/test/testing_env.rb

@@ -15,6 +15,7 @@ TEST_DIRECTORY = File.dirname(File.expand_path(__FILE__))
 begin
   require "rubygems"
   require "minitest/autorun"
+  require "parallel_tests/test/runtime_logger"
   require "mocha/setup"
 rescue LoadError
   abort "Run `bundle install` or install the mocha and minitest gems before running the tests"
@@ -42,7 +43,7 @@ module Homebrew
   module FSLeakLogger
     def self.included(klass)
       require "find"
-      @@log = File.open("fs_leak_log", "w")
+      @@log = File.open("#{__dir__}/fs_leak_log", "w")
       klass.make_my_diffs_pretty!
     end