Merge pull request #9311 from MikeMcQuaid/runner_os

workflows/tests: split out jobs.

.github/workflows/tests.yml

@@ -1,169 +1,266 @@
-name: GitHub Actions CI
+name: CI
 on:
   push:
-    branches: master
+    branches:
+      - master
   pull_request:
 env:
   HOMEBREW_DEVELOPER: 1
   HOMEBREW_NO_AUTO_UPDATE: 1
 jobs:
-  tests:
+  syntax:
     if: github.repository == 'Homebrew/brew'
+    runs-on: ubuntu-latest
+    steps:
+      - name: Set up Homebrew
+        id: set-up-homebrew
+        uses: Homebrew/actions/setup-homebrew@master
+
+      - name: Cache Bundler RubyGems
+        uses: actions/cache@v1
+        with:
+          path: ${{ steps.set-up-homebrew.outputs.gems-path }}
+          key: ${{ runner.os }}-rubygems-${{ steps.set-up-homebrew.outputs.gems-hash }}
+          restore-keys: ${{ runner.os }}-rubygems-
+
+      - name: Install Bundler RubyGems
+        run: brew install-bundler-gems
+
+      - run: brew style --display-cop-names
+
+      - run: brew man --fail-if-changed
+
+      - run: brew typecheck
+
+      - name: Run vale for docs linting
+        run: |
+          brew install vale
+          vale docs/
+
+      - name: Lint Dockerfile
+        run: |
+          brew install hadolint
+          hadolint Dockerfile
+
+  tap-syntax:
+    name: ${{ matrix.name }}
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        os: [ubuntu-latest, macOS-latest, macOS-11.0]
         include:
-          - os: ubuntu-latest
+          - name: tap syntax (Linux)
-            core-tap: 'linuxbrew-core'
+            os: ubuntu-latest
-          - os: macOS-latest
+            core-tap: linuxbrew-core
-            core-tap: 'homebrew-core'
+          - name: tap syntax (macOS)
-          - os: macOS-11.0
+            os: macos-latest
-            core-tap: 'homebrew-core'
+            core-tap: homebrew-core
     steps:
-    - name: Set up Homebrew
+      - name: Set up Homebrew
-      id: set-up-homebrew
+        id: set-up-homebrew
-      uses: Homebrew/actions/setup-homebrew@master
+        uses: Homebrew/actions/setup-homebrew@master
 
-    - name: Configure Git user
+      - run: brew test-bot --only-cleanup-before
-      uses: Homebrew/actions/git-user-config@master
-      with:
-        username: BrewTestBot
 
-    - run: brew test-bot --only-cleanup-before
+      - name: Set up Xcode
+        if: matrix.os == 'macos-11.0'
+        run: sudo xcode-select --switch /Applications/Xcode_12.2.app/Contents/Developer
 
-    - name: Set up Xcode
+      - run: brew config
-      if: matrix.os == 'macOS-11.0'
-      run: sudo xcode-select --switch /Applications/Xcode_12.2.app/Contents/Developer
 
-    - run: brew config
+      - name: Cache Bundler RubyGems
+        uses: actions/cache@v1
+        with:
+          path: ${{ steps.set-up-homebrew.outputs.gems-path }}
+          key: ${{ runner.os }}-rubygems-${{ steps.set-up-homebrew.outputs.gems-hash }}
+          restore-keys: ${{ runner.os }}-rubygems-
 
-    # Can't cache this because we need to check that it doesn't fail the
+      - name: Install Bundler RubyGems
-    # "uncommitted RubyGems" step with a cold cache.
+        run: brew install-bundler-gems
-    - name: Install Bundler RubyGems
-      run: brew install-bundler-gems
 
-    - name: Check for uncommitted RubyGems
+      - run: brew doctor
-      run: git diff --stat --exit-code Library/Homebrew/vendor/bundle/ruby
 
-    - run: brew doctor
+      - name: Run brew update-tests
+        if: github.event_name == 'pull_request'
+        run: |
+          brew update-test
+          brew update-test --to-tag
+          brew update-test --commit=HEAD
 
-    - name: Set up Homebrew official command taps
+      - name: Set up all Homebrew taps
-      run: |
+        if: runner.os == 'macOS'
-        # Setup taps needed for 'brew tests' and 'brew man'
+        run: |
-        cd "$(brew --repo)"
+          brew tap homebrew/cask
-
+          brew tap homebrew/cask-drivers
-        if [ "$RUNNER_OS" = "macOS" ]; then
+          brew tap homebrew/cask-fonts
+          brew tap homebrew/cask-versions
           brew update-reset Library/Taps/homebrew/homebrew-bundle \
                             Library/Taps/homebrew/homebrew-cask \
                             Library/Taps/homebrew/homebrew-cask-versions \
                             Library/Taps/homebrew/homebrew-services
-        else
-          brew update-reset Library/Taps/homebrew/homebrew-services
-        fi
 
-    - name: Run brew style on Homebrew/brew
+      - name: Run brew readall on all taps
-      run: brew style --display-cop-names
+        run: brew readall --aliases
 
-    - name: Run brew man
+      - name: Run brew style on ${{ matrix.core-tap }}
-      run: brew man --fail-if-changed
+        run: brew style --display-cop-names homebrew/core
 
-    - name: Install brew tests dependencies
+      - name: Run brew style on official taps
-      if: matrix.os != 'ubuntu-latest'
+        if: runner.os == 'macOS'
-      run: |
+        run: |
-        brew install subversion
+          brew style --display-cop-names homebrew/bundle \
-        Library/Homebrew/shims/scm/svn --homebrew=print-path
+                                         homebrew/services \
-        which svn
+                                         homebrew/test-bot
-        which svnadmin
 
-    - name: Run brew tests
+      - name: Run brew style on cask taps
-      run: |
+        if: runner.os == 'macOS'
-        # brew tests doesn't like world writable directories
+        run: |
-        if [[ "$RUNNER_OS" = "Linux" ]]; then
+          brew style --display-cop-names homebrew/cask \
+                                         homebrew/cask-drivers \
+                                         homebrew/cask-fonts \
+                                         homebrew/cask-versions
+
+      - name: Run brew audit --skip-style on all taps
+        run: brew audit --skip-style
+
+  vendored-gems:
+    name: ${{ matrix.name }}
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        include:
+          - name: vendored gems (Linux)
+            os: ubuntu-latest
+            core-tap: linuxbrew-core
+          - name: vendored gems (macOS)
+            os: macos-latest
+            core-tap: homebrew-core
+    steps:
+      - name: Set up Homebrew
+        id: set-up-homebrew
+        uses: Homebrew/actions/setup-homebrew@master
+
+      - name: Set up Xcode
+        if: matrix.os == 'macos-11.0'
+        run: sudo xcode-select --switch /Applications/Xcode_12.2.app/Contents/Developer
+
+      - name: Configure Git user
+        uses: Homebrew/actions/git-user-config@master
+        with:
+          username: BrewTestBot
+
+      # Can't cache this because we need to check that it doesn't fail the
+      # "uncommitted RubyGems" step with a cold cache.
+      - name: Install Bundler RubyGems
+        run: brew install-bundler-gems
+
+      - name: Check for uncommitted RubyGems
+        run: git diff --stat --exit-code Library/Homebrew/vendor/bundle/ruby
+
+  docker:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Set up Homebrew
+        id: set-up-homebrew
+        uses: Homebrew/actions/setup-homebrew@master
+
+      - name: Build Docker image
+        run: docker build -t brew --build-arg=version=16.04 .
+
+      - name: Deploy the Docker image to GitHub Packages and Docker Hub
+        if: github.ref == 'refs/heads/master'
+        run: |
+          echo ${{secrets.HOMEBREW_GITHUB_PACKAGES_API_TOKEN}} | \
+            docker login ghcr.io -u BrewTestBot --password-stdin
+          docker tag brew "ghcr.io/homebrew/ubuntu16.04:master"
+          docker push "ghcr.io/homebrew/ubuntu16.04:master"
+          echo ${{secrets.DOCKER_TOKEN}} | \
+            docker login -u brewtestbot --password-stdin
+          docker tag brew "homebrew/ubuntu16.04:master"
+          docker push "homebrew/ubuntu16.04:master"
+
+  tests:
+    name: ${{ matrix.name }}
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        include:
+          - name: tests (no-compatibility mode)
+            os: ubuntu-latest
+            test-flags: --no-compat --online --coverage
+          - name: tests (generic OS)
+            os: ubuntu-latest
+            test-flags: --generic --online --coverage
+          - name: tests (Linux)
+            os: ubuntu-latest
+            test-flags: --online --coverage
+          - name: tests (macOS)
+            os: macos-11.0
+            test-flags: --online --coverage
+    steps:
+      - name: Set up Homebrew
+        id: set-up-homebrew
+        uses: Homebrew/actions/setup-homebrew@master
+
+      - name: Set up Xcode
+        if: matrix.os == 'macos-11.0'
+        run: sudo xcode-select --switch /Applications/Xcode_12.2.app/Contents/Developer
+
+      - name: Install brew tests dependencies
+        if: runner.os == 'macOS'
+        run: |
+          brew install subversion
+          Library/Homebrew/shims/scm/svn --homebrew=print-path
+          which svn
+          which svnadmin
+
+      - name: Cache Bundler RubyGems
+        uses: actions/cache@v1
+        with:
+          path: ${{ steps.set-up-homebrew.outputs.gems-path }}
+          key: ${{ runner.os }}-rubygems-${{ steps.set-up-homebrew.outputs.gems-hash }}
+          restore-keys: ${{ runner.os }}-rubygems-
+
+      - name: Install Bundler RubyGems
+        run: brew install-bundler-gems
+
+      - name: Run brew tests
+        run: |
+          if ${{ runner.os == 'Linux' }}; then
+            # brew tests doesn't like world writable directories
             sudo chmod -R g-w,o-w /home/linuxbrew/.linuxbrew/Homebrew
-        fi
+          elif ${{ runner.os == 'macOS' }}; then
+            # These cannot be queried at the macOS level on GitHub Actions.
+            export HOMEBREW_LANGUAGES='en-GB'
+          fi
 
-        # don't bother running all tests on both platforms (for speed)
+          if ${{ contains(matrix.test-flags, '--coverage') }}; then
-        if [ "$RUNNER_OS" = "Linux" ]; then
+            # Set variables for coverage reporting.
-          brew tests --no-compat --online
+            export HOMEBREW_CODECOV_TOKEN='3ea0364c-80ce-47a3-9fba-93a940d4b5d7'
-          brew tests --generic --online
+          fi
-          brew tests --online
-        else
-          brew tests --online --coverage
-        fi
-      env:
-        HOMEBREW_GITHUB_API_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
-        # set variables for coverage reporting
+          brew tests ${{ matrix.test-flags }}
-        HOMEBREW_CODECOV_TOKEN: 3ea0364c-80ce-47a3-9fba-93a940d4b5d7
+        env:
+          HOMEBREW_GITHUB_API_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
-        # These cannot be queried at the macOS level on GitHub Actions.
+  test-default-formula:
-        HOMEBREW_LANGUAGES: en-GB
+    name: ${{ matrix.name }}
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        include:
+          - name: test default formula (Linux)
+            os: ubuntu-latest
+          - name: test default formula (macOS)
+            os: macos-latest
+    steps:
+      - name: Set up Homebrew
+        id: set-up-homebrew
+        uses: Homebrew/actions/setup-homebrew@master
 
-    - name: Run brew update-tests
+      - name: Set up Xcode
-      run: |
+        if: matrix.os == 'macos-11.0'
-        brew update-test
+        run: sudo xcode-select --switch /Applications/Xcode_12.2.app/Contents/Developer
-        brew update-test --to-tag
-        brew update-test --commit=HEAD
-      if: github.event_name == 'pull_request'
 
-    - name: Run brew readall on all taps
+      - run: brew test-bot --only-cleanup-before
-      run: brew readall --aliases
 
-    - name: Run brew style on ${{ matrix.core-tap }}
+      - run: brew test-bot --only-formulae --test-default-formula
-      run: brew style --display-cop-names homebrew/core
-
-    - name: Run brew style on official taps
-      run: brew style --display-cop-names homebrew/bundle homebrew/services homebrew/test-bot
-
-    - name: Run brew style on cask taps
-      if: matrix.os != 'ubuntu-latest'
-      run: |
-        brew tap homebrew/cask
-        brew tap homebrew/cask-drivers
-        brew tap homebrew/cask-fonts
-        brew tap homebrew/cask-versions
-
-        brew style --display-cop-names \
-          homebrew/cask \
-          homebrew/cask-drivers \
-          homebrew/cask-fonts \
-          homebrew/cask-versions
-
-    - name: Run brew audit --skip-style on all taps
-      run: brew audit --skip-style
-
-    - run: brew typecheck
-
-    - name: Run vale for docs linting
-      run: |
-        brew install vale
-        vale docs/
-
-    - name: Lint Dockerfile
-      if: matrix.os == 'ubuntu-latest'
-      run: |
-        brew install hadolint
-        hadolint Dockerfile
-
-    - name: Build Docker image
-      if: matrix.os == 'ubuntu-latest'
-      run: docker build -t brew --build-arg=version=16.04 .
-
-    - name: Run brew test-bot --only-formulae --test-default-formula
-      run: |
-        if [ "$RUNNER_OS" = "Linux" ]; then
-          docker run --rm brew brew test-bot --only-formulae --test-default-formula
-        else
-          brew test-bot --only-formulae --test-default-formula
-        fi
-
-    - name: Deploy the Docker image to GitHub Packages and Docker Hub
-      if: matrix.os == 'ubuntu-latest' && github.ref == 'refs/heads/master'
-      run: |
-        echo ${{secrets.HOMEBREW_GITHUB_PACKAGES_API_TOKEN}} | docker login ghcr.io -u BrewTestBot --password-stdin
-        docker tag brew "ghcr.io/homebrew/ubuntu16.04:master"
-        docker push "ghcr.io/homebrew/ubuntu16.04:master"
-        echo ${{secrets.DOCKER_TOKEN}} | docker login -u brewtestbot --password-stdin
-        docker tag brew "homebrew/ubuntu16.04:master"
-        docker push "homebrew/ubuntu16.04:master"

Library/Homebrew/.simplecov

@@ -29,10 +29,7 @@ SimpleCov.start do
     # be quiet, the parent process will be in charge of output and checking coverage totals
     SimpleCov.print_error_status = false
 
-    at_exit do
+    SimpleCov.at_exit do
-      exit_code = $ERROR_INFO.nil? ? 0 : $ERROR_INFO.status
-      $stdout.reopen("/dev/null")
-
       # Just save result, but don't write formatted output.
       coverage_result = Coverage.result
       # TODO: this method is private, find a better way.
@@ -40,7 +37,10 @@ SimpleCov.start do
       simplecov_result = SimpleCov::Result.new(coverage_result)
       SimpleCov::ResultMerger.store_result(simplecov_result)
 
-      exit! exit_code
+      # If an integration test raises a `SystemExit` exception on exit,
+      # exit immediately using the same status code to avoid reporting
+      # an error when expecting a non-successful exit status.
+      raise if $ERROR_INFO.is_a?(SystemExit)
     end
   else
     command_name "#{command_name} (#{$PROCESS_ID})"

Library/Homebrew/dev-cmd/tests.rb

@@ -42,7 +42,6 @@ module Homebrew
     args = tests_args.parse
 
     Homebrew.install_bundler_gems!
-    gem_user_dir = Gem.user_dir
 
     require "byebug" if args.byebug?
 
@@ -138,6 +137,9 @@ module Homebrew
 
       puts "Randomized with seed #{seed}"
 
+      # Let tests find `bundle` in the actual location.
+      ENV["HOMEBREW_TESTS_GEM_USER_DIR"] = gem_user_dir
+
       # Let `bundle` in PATH find its gem.
       ENV["GEM_PATH"] = "#{ENV["GEM_PATH"]}:#{gem_user_dir}"
 

Library/Homebrew/test/cmd/help_spec.rb

@@ -6,6 +6,7 @@ describe "brew", :integration_test do
     it "prints help for a documented Ruby command" do
       expect { brew "help", "cat" }
         .to output(/^Usage: brew cat/).to_stdout
+        .and not_to_output.to_stderr
         .and be_a_success
     end
   end
@@ -14,6 +15,7 @@ describe "brew", :integration_test do
     it "prints help when no argument is given" do
       expect { brew "cat" }
         .to output(/^Usage: brew cat/).to_stderr
+        .and not_to_output.to_stdout
         .and be_a_failure
     end
   end

Library/Homebrew/test/spec_helper.rb

@@ -80,7 +80,11 @@ RSpec.configure do |config|
   if ENV["CI"]
     config.verbose_retry = true
     config.display_try_failure_messages = true
-    config.default_retry_count = 2
+
+    config.around(:each, :integration_test) do |example|
+      example.metadata[:timeout] ||= 120
+      example.run
+    end
 
     config.around(:each, :needs_network) do |example|
       example.metadata[:timeout] ||= 120

Library/Homebrew/test/support/helper/spec/shared_context/integration_test.rb

@@ -13,7 +13,8 @@ RSpec.shared_context "integration test" do
   matcher :be_a_success do
     match do |actual|
       status = actual.is_a?(Proc) ? actual.call : actual
-      status.respond_to?(:success?) && status.success?
+      expect(status).to respond_to(:success?)
+      status.success?
     end
 
     def supports_block_expectations?

Library/Homebrew/utils/gems.rb

@@ -17,16 +17,20 @@ module Homebrew
     "#{RbConfig::CONFIG["prefix"]}/bin"
   end
 
+  def gem_user_dir
+    ENV["HOMEBREW_TESTS_GEM_USER_DIR"] || Gem.user_dir
+  end
+
   def gem_user_bindir
     require "rubygems"
-    "#{Gem.user_dir}/bin"
+    "#{gem_user_dir}/bin"
   end
 
   def ohai_if_defined(message)
     if defined?(ohai)
-      ohai message
+      $stderr.ohai message
     else
-      puts "==> #{message}"
+      $stderr.puts "==> #{message}"
     end
   end
 
@@ -55,7 +59,7 @@ module Homebrew
     Gem.clear_paths
     Gem::Specification.reset
 
-    # Add necessary Ruby and Gem binary directories to PATH.
+    # Add necessary Ruby and Gem binary directories to `PATH`.
     gem_bindir ||= Gem.bindir
     paths = ENV.fetch("PATH").split(":")
     paths.unshift(gem_bindir) unless paths.include?(gem_bindir)
@@ -92,7 +96,7 @@ module Homebrew
 
   def install_bundler!
     require "rubygems"
-    setup_gem_environment!(gem_home: Gem.user_dir, gem_bindir: gem_user_bindir)
+    setup_gem_environment!(gem_home: gem_user_dir, gem_bindir: gem_user_bindir)
     install_gem_setup_path!(
       "bundler",
       version:               HOMEBREW_BUNDLER_VERSION,