Merge pull request #9311 from MikeMcQuaid/runner_os

workflows/tests: split out jobs.
2020-11-30 22:42:28 +01:00 · 2020-11-30 22:42:28 +01:00 · 8fe91aacc3
commit 8fe91aacc3
parent 7afaebc7f8 c17d55201e
7 changed files with 256 additions and 146 deletions
--- a/.github/workflows/tests.yml
+++ b/.github/workflows/tests.yml
@ -1,137 +1,34 @@
-name: GitHub Actions CI
+name: CI
 on:
  push:
-    branches: master
+    branches:
+      - master
  pull_request:
 env:
  HOMEBREW_DEVELOPER: 1
  HOMEBREW_NO_AUTO_UPDATE: 1
 jobs:
-  tests:
+  syntax:
    if: github.repository == 'Homebrew/brew'
-    runs-on: ${{ matrix.os }}
-    strategy:
-      matrix:
-        os: [ubuntu-latest, macOS-latest, macOS-11.0]
-        include:
-          - os: ubuntu-latest
-            core-tap: 'linuxbrew-core'
-          - os: macOS-latest
-            core-tap: 'homebrew-core'
-          - os: macOS-11.0
-            core-tap: 'homebrew-core'
+    runs-on: ubuntu-latest
    steps:
      - name: Set up Homebrew
        id: set-up-homebrew
        uses: Homebrew/actions/setup-homebrew@master

-    - name: Configure Git user
-      uses: Homebrew/actions/git-user-config@master
+      - name: Cache Bundler RubyGems
+        uses: actions/cache@v1
        with:
-        username: BrewTestBot
+          path: ${{ steps.set-up-homebrew.outputs.gems-path }}
+          key: ${{ runner.os }}-rubygems-${{ steps.set-up-homebrew.outputs.gems-hash }}
+          restore-keys: ${{ runner.os }}-rubygems-

-    - run: brew test-bot --only-cleanup-before
-
-    - name: Set up Xcode
-      if: matrix.os == 'macOS-11.0'
-      run: sudo xcode-select --switch /Applications/Xcode_12.2.app/Contents/Developer
-
-    - run: brew config
-
-    # Can't cache this because we need to check that it doesn't fail the
-    # "uncommitted RubyGems" step with a cold cache.
      - name: Install Bundler RubyGems
        run: brew install-bundler-gems

-    - name: Check for uncommitted RubyGems
-      run: git diff --stat --exit-code Library/Homebrew/vendor/bundle/ruby
+      - run: brew style --display-cop-names

-    - run: brew doctor
-
-    - name: Set up Homebrew official command taps
-      run: |
-        # Setup taps needed for 'brew tests' and 'brew man'
-        cd "$(brew --repo)"
-
-        if [ "$RUNNER_OS" = "macOS" ]; then
-          brew update-reset Library/Taps/homebrew/homebrew-bundle \
-                            Library/Taps/homebrew/homebrew-cask \
-                            Library/Taps/homebrew/homebrew-cask-versions \
-                            Library/Taps/homebrew/homebrew-services
-        else
-          brew update-reset Library/Taps/homebrew/homebrew-services
-        fi
-
-    - name: Run brew style on Homebrew/brew
-      run: brew style --display-cop-names
-
-    - name: Run brew man
-      run: brew man --fail-if-changed
-
-    - name: Install brew tests dependencies
-      if: matrix.os != 'ubuntu-latest'
-      run: |
-        brew install subversion
-        Library/Homebrew/shims/scm/svn --homebrew=print-path
-        which svn
-        which svnadmin
-
-    - name: Run brew tests
-      run: |
-        # brew tests doesn't like world writable directories
-        if [[ "$RUNNER_OS" = "Linux" ]]; then
-            sudo chmod -R g-w,o-w /home/linuxbrew/.linuxbrew/Homebrew
-        fi
-
-        # don't bother running all tests on both platforms (for speed)
-        if [ "$RUNNER_OS" = "Linux" ]; then
-          brew tests --no-compat --online
-          brew tests --generic --online
-          brew tests --online
-        else
-          brew tests --online --coverage
-        fi
-      env:
-        HOMEBREW_GITHUB_API_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-
-        # set variables for coverage reporting
-        HOMEBREW_CODECOV_TOKEN: 3ea0364c-80ce-47a3-9fba-93a940d4b5d7
-
-        # These cannot be queried at the macOS level on GitHub Actions.
-        HOMEBREW_LANGUAGES: en-GB
-
-    - name: Run brew update-tests
-      run: |
-        brew update-test
-        brew update-test --to-tag
-        brew update-test --commit=HEAD
-      if: github.event_name == 'pull_request'
-
-    - name: Run brew readall on all taps
-      run: brew readall --aliases
-
-    - name: Run brew style on ${{ matrix.core-tap }}
-      run: brew style --display-cop-names homebrew/core
-
-    - name: Run brew style on official taps
-      run: brew style --display-cop-names homebrew/bundle homebrew/services homebrew/test-bot
-
-    - name: Run brew style on cask taps
-      if: matrix.os != 'ubuntu-latest'
-      run: |
-        brew tap homebrew/cask
-        brew tap homebrew/cask-drivers
-        brew tap homebrew/cask-fonts
-        brew tap homebrew/cask-versions
-
-        brew style --display-cop-names \
-          homebrew/cask \
-          homebrew/cask-drivers \
-          homebrew/cask-fonts \
-          homebrew/cask-versions
-
-    - name: Run brew audit --skip-style on all taps
-      run: brew audit --skip-style
+      - run: brew man --fail-if-changed

      - run: brew typecheck

@ -141,29 +38,229 @@ jobs:
          vale docs/

      - name: Lint Dockerfile
-      if: matrix.os == 'ubuntu-latest'
        run: |
          brew install hadolint
          hadolint Dockerfile

+  tap-syntax:
+    name: ${{ matrix.name }}
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        include:
+          - name: tap syntax (Linux)
+            os: ubuntu-latest
+            core-tap: linuxbrew-core
+          - name: tap syntax (macOS)
+            os: macos-latest
+            core-tap: homebrew-core
+    steps:
+      - name: Set up Homebrew
+        id: set-up-homebrew
+        uses: Homebrew/actions/setup-homebrew@master
+
+      - run: brew test-bot --only-cleanup-before
+
+      - name: Set up Xcode
+        if: matrix.os == 'macos-11.0'
+        run: sudo xcode-select --switch /Applications/Xcode_12.2.app/Contents/Developer
+
+      - run: brew config
+
+      - name: Cache Bundler RubyGems
+        uses: actions/cache@v1
+        with:
+          path: ${{ steps.set-up-homebrew.outputs.gems-path }}
+          key: ${{ runner.os }}-rubygems-${{ steps.set-up-homebrew.outputs.gems-hash }}
+          restore-keys: ${{ runner.os }}-rubygems-
+
+      - name: Install Bundler RubyGems
+        run: brew install-bundler-gems
+
+      - run: brew doctor
+
+      - name: Run brew update-tests
+        if: github.event_name == 'pull_request'
+        run: |
+          brew update-test
+          brew update-test --to-tag
+          brew update-test --commit=HEAD
+
+      - name: Set up all Homebrew taps
+        if: runner.os == 'macOS'
+        run: |
+          brew tap homebrew/cask
+          brew tap homebrew/cask-drivers
+          brew tap homebrew/cask-fonts
+          brew tap homebrew/cask-versions
+          brew update-reset Library/Taps/homebrew/homebrew-bundle \
+                            Library/Taps/homebrew/homebrew-cask \
+                            Library/Taps/homebrew/homebrew-cask-versions \
+                            Library/Taps/homebrew/homebrew-services
+
+      - name: Run brew readall on all taps
+        run: brew readall --aliases
+
+      - name: Run brew style on ${{ matrix.core-tap }}
+        run: brew style --display-cop-names homebrew/core
+
+      - name: Run brew style on official taps
+        if: runner.os == 'macOS'
+        run: |
+          brew style --display-cop-names homebrew/bundle \
+                                         homebrew/services \
+                                         homebrew/test-bot
+
+      - name: Run brew style on cask taps
+        if: runner.os == 'macOS'
+        run: |
+          brew style --display-cop-names homebrew/cask \
+                                         homebrew/cask-drivers \
+                                         homebrew/cask-fonts \
+                                         homebrew/cask-versions
+
+      - name: Run brew audit --skip-style on all taps
+        run: brew audit --skip-style
+
+  vendored-gems:
+    name: ${{ matrix.name }}
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        include:
+          - name: vendored gems (Linux)
+            os: ubuntu-latest
+            core-tap: linuxbrew-core
+          - name: vendored gems (macOS)
+            os: macos-latest
+            core-tap: homebrew-core
+    steps:
+      - name: Set up Homebrew
+        id: set-up-homebrew
+        uses: Homebrew/actions/setup-homebrew@master
+
+      - name: Set up Xcode
+        if: matrix.os == 'macos-11.0'
+        run: sudo xcode-select --switch /Applications/Xcode_12.2.app/Contents/Developer
+
+      - name: Configure Git user
+        uses: Homebrew/actions/git-user-config@master
+        with:
+          username: BrewTestBot
+
+      # Can't cache this because we need to check that it doesn't fail the
+      # "uncommitted RubyGems" step with a cold cache.
+      - name: Install Bundler RubyGems
+        run: brew install-bundler-gems
+
+      - name: Check for uncommitted RubyGems
+        run: git diff --stat --exit-code Library/Homebrew/vendor/bundle/ruby
+
+  docker:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Set up Homebrew
+        id: set-up-homebrew
+        uses: Homebrew/actions/setup-homebrew@master
+
      - name: Build Docker image
-      if: matrix.os == 'ubuntu-latest'
        run: docker build -t brew --build-arg=version=16.04 .

-    - name: Run brew test-bot --only-formulae --test-default-formula
-      run: |
-        if [ "$RUNNER_OS" = "Linux" ]; then
-          docker run --rm brew brew test-bot --only-formulae --test-default-formula
-        else
-          brew test-bot --only-formulae --test-default-formula
-        fi
-
      - name: Deploy the Docker image to GitHub Packages and Docker Hub
-      if: matrix.os == 'ubuntu-latest' && github.ref == 'refs/heads/master'
+        if: github.ref == 'refs/heads/master'
        run: |
-        echo ${{secrets.HOMEBREW_GITHUB_PACKAGES_API_TOKEN}} | docker login ghcr.io -u BrewTestBot --password-stdin
+          echo ${{secrets.HOMEBREW_GITHUB_PACKAGES_API_TOKEN}} | \
+            docker login ghcr.io -u BrewTestBot --password-stdin
          docker tag brew "ghcr.io/homebrew/ubuntu16.04:master"
          docker push "ghcr.io/homebrew/ubuntu16.04:master"
-        echo ${{secrets.DOCKER_TOKEN}} | docker login -u brewtestbot --password-stdin
+          echo ${{secrets.DOCKER_TOKEN}} | \
+            docker login -u brewtestbot --password-stdin
          docker tag brew "homebrew/ubuntu16.04:master"
          docker push "homebrew/ubuntu16.04:master"
+
+  tests:
+    name: ${{ matrix.name }}
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        include:
+          - name: tests (no-compatibility mode)
+            os: ubuntu-latest
+            test-flags: --no-compat --online --coverage
+          - name: tests (generic OS)
+            os: ubuntu-latest
+            test-flags: --generic --online --coverage
+          - name: tests (Linux)
+            os: ubuntu-latest
+            test-flags: --online --coverage
+          - name: tests (macOS)
+            os: macos-11.0
+            test-flags: --online --coverage
+    steps:
+      - name: Set up Homebrew
+        id: set-up-homebrew
+        uses: Homebrew/actions/setup-homebrew@master
+
+      - name: Set up Xcode
+        if: matrix.os == 'macos-11.0'
+        run: sudo xcode-select --switch /Applications/Xcode_12.2.app/Contents/Developer
+
+      - name: Install brew tests dependencies
+        if: runner.os == 'macOS'
+        run: |
+          brew install subversion
+          Library/Homebrew/shims/scm/svn --homebrew=print-path
+          which svn
+          which svnadmin
+
+      - name: Cache Bundler RubyGems
+        uses: actions/cache@v1
+        with:
+          path: ${{ steps.set-up-homebrew.outputs.gems-path }}
+          key: ${{ runner.os }}-rubygems-${{ steps.set-up-homebrew.outputs.gems-hash }}
+          restore-keys: ${{ runner.os }}-rubygems-
+
+      - name: Install Bundler RubyGems
+        run: brew install-bundler-gems
+
+      - name: Run brew tests
+        run: |
+          if ${{ runner.os == 'Linux' }}; then
+            # brew tests doesn't like world writable directories
+            sudo chmod -R g-w,o-w /home/linuxbrew/.linuxbrew/Homebrew
+          elif ${{ runner.os == 'macOS' }}; then
+            # These cannot be queried at the macOS level on GitHub Actions.
+            export HOMEBREW_LANGUAGES='en-GB'
+          fi
+
+          if ${{ contains(matrix.test-flags, '--coverage') }}; then
+            # Set variables for coverage reporting.
+            export HOMEBREW_CODECOV_TOKEN='3ea0364c-80ce-47a3-9fba-93a940d4b5d7'
+          fi
+
+          brew tests ${{ matrix.test-flags }}
+        env:
+          HOMEBREW_GITHUB_API_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+  test-default-formula:
+    name: ${{ matrix.name }}
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        include:
+          - name: test default formula (Linux)
+            os: ubuntu-latest
+          - name: test default formula (macOS)
+            os: macos-latest
+    steps:
+      - name: Set up Homebrew
+        id: set-up-homebrew
+        uses: Homebrew/actions/setup-homebrew@master
+
+      - name: Set up Xcode
+        if: matrix.os == 'macos-11.0'
+        run: sudo xcode-select --switch /Applications/Xcode_12.2.app/Contents/Developer
+
+      - run: brew test-bot --only-cleanup-before
+
+      - run: brew test-bot --only-formulae --test-default-formula
--- a/Library/Homebrew/.simplecov
+++ b/Library/Homebrew/.simplecov
@ -29,10 +29,7 @@ SimpleCov.start do
    # be quiet, the parent process will be in charge of output and checking coverage totals
    SimpleCov.print_error_status = false

-    at_exit do
-      exit_code = $ERROR_INFO.nil? ? 0 : $ERROR_INFO.status
-      $stdout.reopen("/dev/null")
-
+    SimpleCov.at_exit do
      # Just save result, but don't write formatted output.
      coverage_result = Coverage.result
      # TODO: this method is private, find a better way.
@ -40,7 +37,10 @@ SimpleCov.start do
      simplecov_result = SimpleCov::Result.new(coverage_result)
      SimpleCov::ResultMerger.store_result(simplecov_result)

-      exit! exit_code
+      # If an integration test raises a `SystemExit` exception on exit,
+      # exit immediately using the same status code to avoid reporting
+      # an error when expecting a non-successful exit status.
+      raise if $ERROR_INFO.is_a?(SystemExit)
    end
  else
    command_name "#{command_name} (#{$PROCESS_ID})"
--- a/Library/Homebrew/dev-cmd/tests.rb
+++ b/Library/Homebrew/dev-cmd/tests.rb
@ -42,7 +42,6 @@ module Homebrew
    args = tests_args.parse

    Homebrew.install_bundler_gems!
-    gem_user_dir = Gem.user_dir

    require "byebug" if args.byebug?

@ -138,6 +137,9 @@ module Homebrew

      puts "Randomized with seed #{seed}"

+      # Let tests find `bundle` in the actual location.
+      ENV["HOMEBREW_TESTS_GEM_USER_DIR"] = gem_user_dir
+
      # Let `bundle` in PATH find its gem.
      ENV["GEM_PATH"] = "#{ENV["GEM_PATH"]}:#{gem_user_dir}"

--- a/Library/Homebrew/test/cmd/help_spec.rb
+++ b/Library/Homebrew/test/cmd/help_spec.rb
@ -6,6 +6,7 @@ describe "brew", :integration_test do
    it "prints help for a documented Ruby command" do
      expect { brew "help", "cat" }
        .to output(/^Usage: brew cat/).to_stdout
+        .and not_to_output.to_stderr
        .and be_a_success
    end
  end
@ -14,6 +15,7 @@ describe "brew", :integration_test do
    it "prints help when no argument is given" do
      expect { brew "cat" }
        .to output(/^Usage: brew cat/).to_stderr
+        .and not_to_output.to_stdout
        .and be_a_failure
    end
  end
--- a/Library/Homebrew/test/spec_helper.rb
+++ b/Library/Homebrew/test/spec_helper.rb
@ -80,7 +80,11 @@ RSpec.configure do |config|
  if ENV["CI"]
    config.verbose_retry = true
    config.display_try_failure_messages = true
-    config.default_retry_count = 2
+
+    config.around(:each, :integration_test) do |example|
+      example.metadata[:timeout] ||= 120
+      example.run
+    end

    config.around(:each, :needs_network) do |example|
      example.metadata[:timeout] ||= 120
--- a/Library/Homebrew/test/support/helper/spec/shared_context/integration_test.rb
+++ b/Library/Homebrew/test/support/helper/spec/shared_context/integration_test.rb
@ -13,7 +13,8 @@ RSpec.shared_context "integration test" do
  matcher :be_a_success do
    match do |actual|
      status = actual.is_a?(Proc) ? actual.call : actual
-      status.respond_to?(:success?) && status.success?
+      expect(status).to respond_to(:success?)
+      status.success?
    end

    def supports_block_expectations?
--- a/Library/Homebrew/utils/gems.rb
+++ b/Library/Homebrew/utils/gems.rb
@ -17,16 +17,20 @@ module Homebrew
    "#{RbConfig::CONFIG["prefix"]}/bin"
  end

+  def gem_user_dir
+    ENV["HOMEBREW_TESTS_GEM_USER_DIR"] || Gem.user_dir
+  end
+
  def gem_user_bindir
    require "rubygems"
-    "#{Gem.user_dir}/bin"
+    "#{gem_user_dir}/bin"
  end

  def ohai_if_defined(message)
    if defined?(ohai)
-      ohai message
+      $stderr.ohai message
    else
-      puts "==> #{message}"
+      $stderr.puts "==> #{message}"
    end
  end

@ -55,7 +59,7 @@ module Homebrew
    Gem.clear_paths
    Gem::Specification.reset

-    # Add necessary Ruby and Gem binary directories to PATH.
+    # Add necessary Ruby and Gem binary directories to `PATH`.
    gem_bindir ||= Gem.bindir
    paths = ENV.fetch("PATH").split(":")
    paths.unshift(gem_bindir) unless paths.include?(gem_bindir)
@ -92,7 +96,7 @@ module Homebrew

  def install_bundler!
    require "rubygems"
-    setup_gem_environment!(gem_home: Gem.user_dir, gem_bindir: gem_user_bindir)
+    setup_gem_environment!(gem_home: gem_user_dir, gem_bindir: gem_user_bindir)
    install_gem_setup_path!(
      "bundler",
      version:               HOMEBREW_BUNDLER_VERSION,