diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 1633277162..464124a2b7 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -320,3 +320,14 @@ jobs: - run: brew test-bot --only-formulae --test-default-formula - uses: codecov/codecov-action@29386c70ef20e286228c72b668a06fd0e8399192 + + - name: Upload test results to BuildPulse for flaky test detection + # Only run this step for PRs where where we have access to secrets. + # Run this step even when the tests fail. Skip if the workflow is cancelled. + if: github.event.pull_request.head.repo.full_name == github.repository && !cancelled() + env: + BUILDPULSE_ACCESS_KEY_ID: ${{ secrets.BUILDPULSE_ACCESS_KEY_ID }} + BUILDPULSE_SECRET_ACCESS_KEY: ${{ secrets.BUILDPULSE_SECRET_ACCESS_KEY }} + run: | + brew install buildpulse-test-reporter + buildpulse-test-reporter submit Library/Homebrew/test/junit --account-id 1503512 --repository-id 53238813 diff --git a/.gitignore b/.gitignore index 9274f245c7..6001fe1574 100644 --- a/.gitignore +++ b/.gitignore @@ -14,6 +14,7 @@ /Library/Homebrew/test/.gem /Library/Homebrew/test/.subversion /Library/Homebrew/test/coverage +/Library/Homebrew/test/junit /Library/Homebrew/test/fs_leak_log /Library/Homebrew/vendor/portable-ruby /Library/Taps @@ -133,6 +134,7 @@ **/vendor/bundle/ruby/*/gems/rspec-*/ **/vendor/bundle/ruby/*/gems/rspec-core-*/ **/vendor/bundle/ruby/*/gems/rspec-expectations-*/ +**/vendor/bundle/ruby/*/gems/rspec_junit_formatter-*/ **/vendor/bundle/ruby/*/gems/rspec-its-*/ **/vendor/bundle/ruby/*/gems/rspec-mocks-*/ **/vendor/bundle/ruby/*/gems/rspec-retry-*/ diff --git a/Library/Homebrew/.rspec_parallel b/Library/Homebrew/.rspec_parallel new file mode 100644 index 0000000000..5200a0136d --- /dev/null +++ b/Library/Homebrew/.rspec_parallel @@ -0,0 +1,6 @@ +--format NoSeedProgressFormatter +--format ParallelTests::RSpec::RuntimeLogger +--out <%= ENV["PARALLEL_RSPEC_LOG_PATH"] %> +--format RspecJunitFormatter +--out test/junit/rspec<%= ENV["TEST_ENV_NUMBER"] %>.xml +<%= "--format RSpec::Github::Formatter" if ENV["GITHUB_ACTIONS"] %> diff --git a/Library/Homebrew/Gemfile b/Library/Homebrew/Gemfile index 70c9b44ba2..7ff09f5ea3 100644 --- a/Library/Homebrew/Gemfile +++ b/Library/Homebrew/Gemfile @@ -12,6 +12,7 @@ gem "ronn", require: false gem "rspec", require: false gem "rspec-github", require: false gem "rspec-its", require: false +gem "rspec_junit_formatter", require: false gem "rspec-retry", require: false gem "rspec-wait", require: false gem "rubocop", require: false diff --git a/Library/Homebrew/Gemfile.lock b/Library/Homebrew/Gemfile.lock index 6bfd4b1237..4e0c4925bc 100644 --- a/Library/Homebrew/Gemfile.lock +++ b/Library/Homebrew/Gemfile.lock @@ -109,6 +109,8 @@ GEM rspec-support (3.10.2) rspec-wait (0.0.9) rspec (>= 3, < 4) + rspec_junit_formatter (0.4.1) + rspec-core (>= 2, < 4, != 2.12.0) rubocop (1.17.0) parallel (~> 1.10) parser (>= 3.0.0.0) @@ -194,6 +196,7 @@ DEPENDENCIES rspec-retry rspec-sorbet rspec-wait + rspec_junit_formatter rubocop rubocop-ast rubocop-performance diff --git a/Library/Homebrew/dev-cmd/tests.rb b/Library/Homebrew/dev-cmd/tests.rb index f35117e56e..57e0223a56 100644 --- a/Library/Homebrew/dev-cmd/tests.rb +++ b/Library/Homebrew/dev-cmd/tests.rb @@ -111,6 +111,7 @@ module Homebrew else "#{HOMEBREW_CACHE}/#{parallel_rspec_log_name}" end + ENV["PARALLEL_RSPEC_LOG_PATH"] = parallel_rspec_log_path parallel_args = if ENV["CI"] %W[ @@ -133,13 +134,8 @@ module Homebrew --seed #{seed} --color --require spec_helper - --format NoSeedProgressFormatter - --format ParallelTests::RSpec::RuntimeLogger - --out #{parallel_rspec_log_path} ] - bundle_args << "--format" << "RSpec::Github::Formatter" if ENV["GITHUB_ACTIONS"] - unless OS.mac? bundle_args << "--tag" << "~needs_macos" << "--tag" << "~cask" files = files.reject { |p| p =~ %r{^test/(os/mac|cask)(/.*|_spec\.rb)$} }