Update primer tests to own file, separate batches and concurrent runs

.github/workflows/ci.yaml

@@ -475,71 +475,3 @@ jobs:
         run: |
           . venv/bin/activate
           pytest --benchmark-disable tests/
-
-  pytest-primer-stdlib:
-    name: Run primer tests on stdlib Python ${{ matrix.python-version }} (Linux)
-    runs-on: ubuntu-latest
-    needs: prepare-tests-linux
-    strategy:
-      matrix:
-        python-version: [3.8, 3.9, "3.10"]
-    steps:
-      - name: Check out code from GitHub
-        uses: actions/[email protected]
-      - name: Set up Python ${{ matrix.python-version }}
-        id: python
-        uses: actions/[email protected]
-        with:
-          python-version: ${{ matrix.python-version }}
-      - name: Restore Python virtual environment
-        id: cache-venv
-        uses: actions/[email protected]
-        with:
-          path: venv
-          key:
-            ${{ runner.os }}-${{ steps.python.outputs.python-version }}-${{
-            needs.prepare-tests-linux.outputs.python-key }}
-      - name: Fail job if Python cache restore failed
-        if: steps.cache-venv.outputs.cache-hit != 'true'
-        run: |
-          echo "Failed to restore Python venv from cache"
-          exit 1
-      - name: Run pytest
-        run: |
-          . venv/bin/activate
-          pip install -e .
-          pytest -m primer_stdlib --primer-stdlib -n auto
-
-  pytest-primer-external:
-    name: Run primer tests on external libs Python ${{ matrix.python-version }} (Linux)
-    runs-on: ubuntu-latest
-    needs: prepare-tests-linux
-    strategy:
-      matrix:
-        python-version: [3.8, 3.9, "3.10"]
-    steps:
-      - name: Check out code from GitHub
-        uses: actions/[email protected]
-      - name: Set up Python ${{ matrix.python-version }}
-        id: python
-        uses: actions/[email protected]
-        with:
-          python-version: ${{ matrix.python-version }}
-      - name: Restore Python virtual environment
-        id: cache-venv
-        uses: actions/[email protected]
-        with:
-          path: venv
-          key:
-            ${{ runner.os }}-${{ steps.python.outputs.python-version }}-${{
-            needs.prepare-tests-linux.outputs.python-key }}
-      - name: Fail job if Python cache restore failed
-        if: steps.cache-venv.outputs.cache-hit != 'true'
-        run: |
-          echo "Failed to restore Python venv from cache"
-          exit 1
-      - name: Run pytest
-        run: |
-          . venv/bin/activate
-          pip install -e .
-          pytest -m primer_external --primer-external -n auto

.github/workflows/primer-test.yaml

@@ -0,0 +1,167 @@
+name: Primer tests
+
+on:
+  push:
+    branches:
+      - main
+      - 2.*
+  pull_request:
+    paths:
+      - "pylint/**"
+      - "tests/primer/**"
+
+env:
+  CACHE_VERSION: 3
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  prepare-tests-linux:
+    name: Prepare tests for Python ${{ matrix.python-version }} (Linux)
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: [3.8, 3.9, "3.10"]
+    outputs:
+      python-key: ${{ steps.generate-python-key.outputs.key }}
+    steps:
+      - name: Check out code from GitHub
+        uses: actions/[email protected]
+        with:
+          fetch-depth: 0
+      - name: Set up Python ${{ matrix.python-version }}
+        id: python
+        uses: actions/[email protected]
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Generate partial Python venv restore key
+        id: generate-python-key
+        run: >-
+          echo "::set-output name=key::venv-${{ env.CACHE_VERSION }}-${{
+            hashFiles('setup.cfg', 'requirements_test.txt', 'requirements_test_min.txt')
+          }}"
+      - name: Restore Python virtual environment
+        id: cache-venv
+        uses: actions/[email protected]
+        with:
+          path: venv
+          key: >-
+            ${{ runner.os }}-${{ steps.python.outputs.python-version }}-${{
+            steps.generate-python-key.outputs.key }}
+          restore-keys: |
+            ${{ runner.os }}-${{ steps.python.outputs.python-version }}-venv-${{ env.CACHE_VERSION }}-
+      - name: Create Python virtual environment
+        if: steps.cache-venv.outputs.cache-hit != 'true'
+        run: |
+          python -m venv venv
+          . venv/bin/activate
+          python -m pip install -U pip setuptools wheel
+          pip install -U -r requirements_test.txt
+
+  pytest-primer-stdlib:
+    name: Run primer tests on stdlib Python ${{ matrix.python-version }} (Linux)
+    runs-on: ubuntu-latest
+    needs: prepare-tests-linux
+    strategy:
+      matrix:
+        python-version: [3.8, 3.9, "3.10"]
+    steps:
+      - name: Check out code from GitHub
+        uses: actions/[email protected]
+      - name: Set up Python ${{ matrix.python-version }}
+        id: python
+        uses: actions/[email protected]
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Restore Python virtual environment
+        id: cache-venv
+        uses: actions/[email protected]
+        with:
+          path: venv
+          key:
+            ${{ runner.os }}-${{ steps.python.outputs.python-version }}-${{
+            needs.prepare-tests-linux.outputs.python-key }}
+      - name: Fail job if Python cache restore failed
+        if: steps.cache-venv.outputs.cache-hit != 'true'
+        run: |
+          echo "Failed to restore Python venv from cache"
+          exit 1
+      - name: Run pytest
+        run: |
+          . venv/bin/activate
+          pip install -e .
+          pytest -m primer_stdlib --primer-stdlib -n auto
+
+  pytest-primer-external-batch-one:
+    name:
+      Run primer tests batch one on external libs Python ${{ matrix.python-version }}
+      (Linux)
+    runs-on: ubuntu-latest
+    needs: prepare-tests-linux
+    strategy:
+      matrix:
+        python-version: [3.8, 3.9, "3.10"]
+    steps:
+      - name: Check out code from GitHub
+        uses: actions/[email protected]
+      - name: Set up Python ${{ matrix.python-version }}
+        id: python
+        uses: actions/[email protected]
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Restore Python virtual environment
+        id: cache-venv
+        uses: actions/[email protected]
+        with:
+          path: venv
+          key:
+            ${{ runner.os }}-${{ steps.python.outputs.python-version }}-${{
+            needs.prepare-tests-linux.outputs.python-key }}
+      - name: Fail job if Python cache restore failed
+        if: steps.cache-venv.outputs.cache-hit != 'true'
+        run: |
+          echo "Failed to restore Python venv from cache"
+          exit 1
+      - name: Run pytest
+        run: |
+          . venv/bin/activate
+          pip install -e .
+          pytest -m primer_external_batch_one --primer-external -n auto
+
+  pytest-primer-external-batch-two:
+    name:
+      Run primer tests batch two on external libs Python ${{ matrix.python-version }}
+      (Linux)
+    runs-on: ubuntu-latest
+    needs: prepare-tests-linux
+    strategy:
+      matrix:
+        python-version: [3.8, 3.9, "3.10"]
+    steps:
+      - name: Check out code from GitHub
+        uses: actions/[email protected]
+      - name: Set up Python ${{ matrix.python-version }}
+        id: python
+        uses: actions/[email protected]
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Restore Python virtual environment
+        id: cache-venv
+        uses: actions/[email protected]
+        with:
+          path: venv
+          key:
+            ${{ runner.os }}-${{ steps.python.outputs.python-version }}-${{
+            needs.prepare-tests-linux.outputs.python-key }}
+      - name: Fail job if Python cache restore failed
+        if: steps.cache-venv.outputs.cache-hit != 'true'
+        run: |
+          echo "Failed to restore Python venv from cache"
+          exit 1
+      - name: Run pytest
+        run: |
+          . venv/bin/activate
+          pip install -e .
+          pytest -m primer_external_batch_two --primer-external -n auto

doc/development_guide/testing.rst

@@ -148,7 +148,14 @@ on the ``stdlib`` and a selection of external repositories.
 To run the ``primer`` tests you can add either ``--primer-stdlib`` or ``--primer-external`` to the
 pytest_ command. If you want to only run the ``primer`` you can add either of their marks, for example::
 
-    pytest -m primer_external --primer-external
+    pytest -m primer_stdlib --primer-stdlib
+
+The external ``primer`` has been split up in two marks to speed up our Continuous Integration. You can run
+either of the two batches or run them both::
+
+    pytest -m primer_external_batch_one --primer-external # Runs batch one
+    pytest -m primer_external_batch_two --primer-external # Runs batch two
+    pytest -m "primer_external_batch_one or primer_external_batch_two" --primer-external # Runs both batches
 
 The list of repositories is created on the basis of three criteria: 1) projects need to use a diverse
 range of language features, 2) projects need to be well maintained and 3) projects should not have a codebase

setup.cfg

@@ -72,7 +72,8 @@ python_files = *test_*.py
 addopts = --strict-markers
 markers =
     primer_stdlib: Checks for crashes and errors when running pylint on stdlib
-    primer_external: Checks for crashes and errors when running pylint on external libs
+    primer_external_batch_one: Checks for crashes and errors when running pylint on external libs (batch one)
+    primer_external_batch_two: Checks for crashes and errors when running pylint on external libs (batch two)
     benchmark: Baseline of pylint performance, if this regress something serious happened
 
 [isort]

tests/conftest.py

@@ -75,13 +75,15 @@ def pytest_addoption(parser) -> None:
 
 def pytest_collection_modifyitems(config, items) -> None:
     """Convert command line options to markers"""
-    # Add skip_primer_stdlib mark
+    # Add skip_primer_external mark
     if not config.getoption("--primer-external"):
         skip_primer_external = pytest.mark.skip(
             reason="need --primer-external option to run"
         )
         for item in items:
-            if "primer_external" in item.keywords:
+            if "primer_external_batch_one" in item.keywords:
+                item.add_marker(skip_primer_external)
+            if "primer_external_batch_two" in item.keywords:
                 item.add_marker(skip_primer_external)
 
     # Add skip_primer_stdlib mark

tests/primer/packages_to_lint_batch_one.json

@@ -0,0 +1,33 @@
+{
+  "black": {
+    "branch": "main",
+    "directories": ["src/black/", "src/blackd/", "src/black_primer/", "src/blib2to3/"],
+    "url": "https://github.com/psf/black.git"
+  },
+  "django": {
+    "branch": "main",
+    "directories": ["django"],
+    "url": "https://github.com/django/django.git"
+  },
+  "flask": {
+    "branch": "main",
+    "directories": ["src/flask"],
+    "url": "https://github.com/pallets/flask.git"
+  },
+  "graph-explorer": {
+    "branch": "master",
+    "directories": ["graph_explorer"],
+    "pylintrc_relpath": ".pylintrc",
+    "url": "https://github.com/vimeo/graph-explorer.git"
+  },
+  "home-assistant": {
+    "branch": "dev",
+    "directories": ["homeassistant"],
+    "url": "https://github.com/home-assistant/core.git"
+  },
+  "keras": {
+    "branch": "master",
+    "directories": ["keras"],
+    "url": "https://github.com/keras-team/keras.git"
+  }
+}