Stronger, clearer test suite with cross-engine verification

.codecov.yml

@@ -20,12 +20,16 @@ coverage:
   status:
     project:
       default:
-        target: auto
+        target: 80%
         threshold: 1%
-        # Make project coverage informational (won't block PR)
-        informational: true
+        # Project coverage is the real gate (see ADR
+        # test-suite-and-validation).
+        informational: false
     patch:
       default:
         target: auto
-        # Require patch coverage but with threshold
         threshold: 1%
+        # Non-blocking: patch grades diff lines against unit-only
+        # coverage, so engine/integration-tested code would otherwise
+        # always fail. See ADR test-suite-and-validation.
+        informational: true

.github/workflows/coverage.yml

@@ -26,21 +26,8 @@ env:
   CI_BRANCH: ${{ github.head_ref || github.ref_name }}
 
 jobs:
-  # Job 1: Run docstring coverage
-  docstring-coverage:
-    runs-on: ubuntu-latest
-
-    steps:
-      - name: Check-out repository
-        uses: actions/checkout@v6
-
-      - name: Set up pixi
-        uses: ./.github/actions/setup-pixi
-
-      - name: Run docstring coverage
-        run: pixi run docstring-coverage
-
-  # Job 2: Run unit tests with coverage and upload to Codecov
+  # Job 1: Run unit tests with coverage and upload to Codecov.
+  # Docstring coverage is gated as a static check in lint-format.yml.
   unit-tests-coverage:
     runs-on: ubuntu-latest
 
@@ -65,6 +52,6 @@ jobs:
 
   # Job 4: Build and publish dashboard (reusable workflow)
   run-reusable-workflows:
-    needs: [docstring-coverage, unit-tests-coverage] # depend on the previous jobs
+    needs: [unit-tests-coverage] # depend on the previous job
     uses: ./.github/workflows/dashboard.yml
     secrets: inherit

.github/workflows/lint-format.yml

@@ -79,6 +79,12 @@ jobs:
         shell: bash
         run: pixi run docstring-lint-check
 
+      - name: Check docstring coverage (interrogate)
+        id: docstring_coverage
+        continue-on-error: true
+        shell: bash
+        run: pixi run docstring-coverage
+
       - name: Check formatting of non-Python files (md, toml, etc.)
         id: nonpy_format
         continue-on-error: true
@@ -91,6 +97,30 @@ jobs:
         shell: bash
         run: pixi run notebook-lint-check
 
+      - name: Check unit-test directory mirrors src/ structure
+        id: test_structure
+        continue-on-error: true
+        shell: bash
+        run: pixi run test-structure-check
+
+      - name: Check spelling (codespell)
+        id: spell
+        continue-on-error: true
+        shell: bash
+        run: pixi run spell-check
+
+      - name: Check documentation links (lychee)
+        id: link
+        continue-on-error: true
+        shell: bash
+        run: pixi run link-check
+
+      - name: Build documentation strictly (no tutorial execution)
+        id: docs_build
+        continue-on-error: true
+        shell: bash
+        run: pixi run docs-build
+
       # Add summary
       - name: Add quality checks summary
         if: always()
@@ -106,8 +136,13 @@ jobs:
             echo "| py lint          | ${{ steps.py_lint.outcome == 'success' && '✅' || '❌' }} |"
             echo "| py format        | ${{ steps.py_format.outcome == 'success' && '✅' || '❌' }} |"
             echo "| docstring lint   | ${{ steps.docstring_lint.outcome == 'success' && '✅' || '❌' }} |"
+            echo "| docstring cover  | ${{ steps.docstring_coverage.outcome == 'success' && '✅' || '❌' }} |"
             echo "| nonpy format     | ${{ steps.nonpy_format.outcome == 'success' && '✅' || '❌' }} |"
             echo "| notebooks lint   | ${{ steps.notebook_lint.outcome == 'success' && '✅' || '❌' }} |"
+            echo "| test structure   | ${{ steps.test_structure.outcome == 'success' && '✅' || '❌' }} |"
+            echo "| spelling         | ${{ steps.spell.outcome == 'success' && '✅' || '❌' }} |"
+            echo "| doc links        | ${{ steps.link.outcome == 'success' && '✅' || '❌' }} |"
+            echo "| docs strict build| ${{ steps.docs_build.outcome == 'success' && '✅' || '❌' }} |"
           } >> "$GITHUB_STEP_SUMMARY"
 
       # Fail job if any check failed
@@ -118,7 +153,12 @@ jobs:
           || steps.py_lint.outcome == 'failure'
           || steps.py_format.outcome == 'failure'
           || steps.docstring_lint.outcome == 'failure'
+          || steps.docstring_coverage.outcome == 'failure'
           || steps.nonpy_format.outcome == 'failure'
           || steps.notebook_lint.outcome == 'failure'
+          || steps.test_structure.outcome == 'failure'
+          || steps.spell.outcome == 'failure'
+          || steps.link.outcome == 'failure'
+          || steps.docs_build.outcome == 'failure'
         shell: bash
         run: exit 1

.github/workflows/nightly.yml

@@ -0,0 +1,45 @@
+# Nightly run of the FULL test suite across all cost tiers (default, pr,
+# and nightly), plus the performance benchmarks. The per-push and per-PR
+# runs in test.yml deselect the pr/nightly tiers; the nightly run does
+# not. See ADR test-suite-and-validation.
+
+name: Nightly tests
+
+on:
+  # Run the nightly-tier tests on a daily schedule
+  schedule:
+    - cron: '0 3 * * *' # 03:00 UTC daily
+  # Allow manual runs from the Actions tab
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+# Allow only one concurrent nightly run per ref.
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  nightly-tests:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Check-out repository
+        uses: actions/checkout@v6
+
+      - name: Set up pixi
+        uses: ./.github/actions/setup-pixi
+
+      - name: Run the full test suite (all tiers)
+        run: pixi run test-all
+
+      - name: Run performance benchmarks
+        run: pixi run benchmarks
+
+      - name: Upload benchmark results
+        if: ${{ !cancelled() }}
+        uses: ./.github/actions/upload-artifact
+        with:
+          name: benchmark-results
+          path: benchmark.json

.github/workflows/test.yml

@@ -54,17 +54,29 @@ jobs:
 
     outputs:
       pytest-marks: ${{ steps.set-mark.outputs.pytest_marks }}
+      run-pr-tier: ${{ steps.set-mark.outputs.run_pr_tier }}
 
     steps:
-      # Determine if integration tests should be run fully or only the fast ones
-      # (to save time on branches other than master and develop)
-      - name: Set mark for integration tests
+      # Select the test cost tier by branch (see ADR test-suite-and-validation):
+      # the pr tier runs on develop/master and pull requests; feature-branch
+      # pushes run only the default (fast) tier. The nightly tier runs
+      # separately in nightly.yml.
+      - name: Set test tier marker expression
         id: set-mark
         run: |
-          if [[ "${{ env.CI_BRANCH }}" == "master" || "${{ env.CI_BRANCH }}" == "develop" ]]; then
-            echo "pytest_marks=" >> $GITHUB_OUTPUT
+          # Pull requests and develop/master pushes run the pr tier;
+          # feature-branch pushes run only the default (fast) tier. On a
+          # pull_request event CI_BRANCH is the contributor branch, so the
+          # event type must be checked explicitly. run_pr_tier gates the
+          # integration steps: the integration layer is entirely pr-marked,
+          # so on the fast tier its selection is empty and pytest would exit
+          # 5 ("no tests collected"); skip the step in that case instead.
+          if [[ "${{ github.event_name }}" == "pull_request" || "${{ env.CI_BRANCH }}" == "master" || "${{ env.CI_BRANCH }}" == "develop" ]]; then
+            echo 'pytest_marks=-m "not nightly"' >> "$GITHUB_OUTPUT"
+            echo 'run_pr_tier=true' >> "$GITHUB_OUTPUT"
           else
-            echo "pytest_marks=-m fast" >> $GITHUB_OUTPUT
+            echo 'pytest_marks=-m "not pr and not nightly"' >> "$GITHUB_OUTPUT"
+            echo 'run_pr_tier=false' >> "$GITHUB_OUTPUT"
           fi
 
   # Job 2: Test code
@@ -99,7 +111,7 @@ jobs:
             env="py-$(echo $py_ver | tr -d .)-env" # Converts 3.XX -> py-3XX-env
 
             echo "Running tests in environment: $env"
-            pixi run --environment $env unit-tests
+            pixi run --environment $env unit-tests ${{ needs.env-prepare.outputs.pytest-marks }}
           done
 
       - name: Run functional tests
@@ -114,10 +126,14 @@ jobs:
             env="py-$(echo $py_ver | tr -d .)-env" # Converts 3.XX -> py-3XX-env
 
             echo "Running tests in environment: $env"
-            pixi run --environment $env functional-tests
+            pixi run --environment $env functional-tests ${{ needs.env-prepare.outputs.pytest-marks }}
           done
 
+      # Integration tests are entirely pr-marked, so they only run in the pr
+      # tier (PRs + develop/master). On feature-branch pushes the selection
+      # would be empty; skip the step rather than fail on pytest exit code 5.
       - name: Run integration tests ${{ needs.env-prepare.outputs.pytest-marks }}
+        if: needs.env-prepare.outputs.run-pr-tier == 'true'
         shell: bash
         run: |
           set -euo pipefail
@@ -173,7 +189,8 @@ jobs:
 
   # Job 3: Test the package
   package-test:
-    needs: source-test # depend on previous job
+    # env-prepare provides the tier marker expression; source-test the wheel
+    needs: [env-prepare, source-test]
 
     strategy:
       fail-fast: false
@@ -270,7 +287,7 @@ jobs:
             cd easydiffraction_py$py_ver
 
             echo "Running tests"
-            pixi run python -m pytest ../tests/unit/ --color=yes -v
+            pixi run python -m pytest ../tests/unit/ --color=yes -v ${{ needs.env-prepare.outputs.pytest-marks }}
 
             echo "Exiting pixi project directory"
             cd ..
@@ -289,13 +306,15 @@ jobs:
             cd easydiffraction_py$py_ver
 
             echo "Running tests"
-            pixi run python -m pytest ../tests/functional/ --color=yes -v
+            pixi run python -m pytest ../tests/functional/ --color=yes -v ${{ needs.env-prepare.outputs.pytest-marks }}
 
             echo "Exiting pixi project directory"
             cd ..
           done
 
+      # See the source-test job: integration tests run only in the pr tier.
       - name: Run integration tests ${{ needs.env-prepare.outputs.pytest-marks }}
+        if: needs.env-prepare.outputs.run-pr-tier == 'true'
         shell: bash
         run: |
           set -euo pipefail

.pre-commit-config.yaml

@@ -39,6 +39,13 @@ repos:
         pass_filenames: false
         stages: [manual]
 
+      - id: pixi-docstring-coverage
+        name: pixi run docstring-coverage
+        entry: pixi run docstring-coverage
+        language: system
+        pass_filenames: false
+        stages: [manual]
+
       - id: pixi-nonpy-format-check
         name: pixi run nonpy-format-check
         entry: pixi run nonpy-format-check
@@ -60,16 +67,16 @@ repos:
         pass_filenames: false
         stages: [manual]
 
-      - id: pixi-unit-tests
-        name: pixi run unit-tests
-        entry: pixi run unit-tests
+      - id: pixi-spell-check
+        name: pixi run spell-check
+        entry: pixi run spell-check
         language: system
         pass_filenames: false
         stages: [manual]
 
-      - id: pixi-functional-tests
-        name: pixi run functional-tests
-        entry: pixi run functional-tests
+      - id: pixi-link-check
+        name: pixi run link-check
+        entry: pixi run link-check
         language: system
         pass_filenames: false
         stages: [manual]

docs/dev/adrs/accepted/test-strategy.md

@@ -39,3 +39,12 @@ aliases.
 New features should add focused tests at the lowest useful layer and
 broader tests when behavior crosses module boundaries. The mirrored
 structure makes missing coverage easier to spot.
+
+## Amendments
+
+[Test Suite and Validation Strategy](test-suite-and-validation.md)
+sharpens these layer definitions into strict, testable placement
+criteria and adds test cost tiers, coverage policy, codecov
+configuration, cross-engine verification documentation, and a nightly
+validation harness. The practical placement rules live in the
+[Testing Guide](../../testing-guide.md).