From 0877d12f630d370e0f16f408578212569306c139 Mon Sep 17 00:00:00 2001 From: Ariel Schulz Date: Tue, 9 Jun 2026 14:08:39 +0200 Subject: [PATCH 01/11] Add matrix:generate and deprecate older endpoints --- exasol/toolbox/nox/_ci.py | 113 ++++++++++++++++++++++++++--- exasol/toolbox/nox/tasks.py | 1 + test/unit/nox/_ci_test.py | 141 ++++++++++++++++++++++++++++++++++++ 3 files changed, 244 insertions(+), 11 deletions(-) create mode 100644 test/unit/nox/_ci_test.py diff --git a/exasol/toolbox/nox/_ci.py b/exasol/toolbox/nox/_ci.py index ff3f047dc..af9bfc6fe 100644 --- a/exasol/toolbox/nox/_ci.py +++ b/exasol/toolbox/nox/_ci.py @@ -1,5 +1,7 @@ +import argparse import json -import logging +from collections.abc import Iterable +from typing import Any import nox from nox import Session @@ -9,32 +11,121 @@ PROJECT_CONFIG, ) -_log = logging.getLogger(__name__) +# The relevant nox sessions will be removed in: +# https://github.com/exasol/python-toolbox/issues/859 +MATRIX_DEPRECATION_DATE = "2026-09-15" -def _python_matrix(config: BaseConfig): - return {"python-version": config.python_versions} +def _matrix_keys(config: BaseConfig) -> tuple[str, ...]: + """ + Return the config keys that are valid for matrix generation. + Includes both declared fields and computed fields. + """ -def _exasol_matrix(config: BaseConfig): - return {"exasol-version": config.exasol_versions} + config_class = type(config) + return tuple(config_class.model_fields) + tuple(config_class.model_computed_fields) + + +def _matrix_args(session: Session, config: BaseConfig) -> list[str]: + parser = argparse.ArgumentParser( + prog="nox -s matrix:generate", + usage="nox -s matrix:generate -- [ ...]", + formatter_class=argparse.ArgumentDefaultsHelpFormatter, + ) + parser.add_argument( + "keys", + nargs="+", + choices=sorted(_matrix_keys(config)), + help="BaseConfig keys to include in the generated matrix output", + ) + return parser.parse_args(session.posargs).keys + + +def _dump_matrix(config: BaseConfig, keys: Iterable[str]) -> dict[str, Any]: + """ + Build a JSON-serializable matrix subset from the project's config. + + GitHub Actions matrix values must be arrays. Pydantic already serializes + tuple-based config values to lists, so scalar values are wrapped in a + single-element list here. + """ + + allowed_keys = set(_matrix_keys(config)) + requested_keys = tuple(keys) + for key in requested_keys: + if key not in allowed_keys: + raise KeyError(key) + + matrix = config.model_dump(mode="json", include=set(requested_keys)) + for key, value in matrix.items(): + if not isinstance(value, list): + matrix[key] = [value] + return matrix + + +def _print_deprecated_matrix( + session: Session, + config: BaseConfig, + key_map: dict[str, str], + session_name: str, + replacement_args: str, +) -> None: + matrix = _dump_matrix(config, key_map.values()) + renamed_matrix: dict[str, Any] = {} + for output_key, config_key in key_map.items(): + renamed_matrix[output_key] = matrix[config_key] + print(json.dumps(renamed_matrix)) + + session.warn( + f"Warning: `nox -s {session_name}` is deprecated and will be removed on " + f"{MATRIX_DEPRECATION_DATE}. Use `nox -s matrix:generate -- {replacement_args}` " + "instead." + ) + + +@nox.session(name="matrix:generate", python=False) +def generate_matrix(session: Session) -> None: + """Output selected BaseConfig values as JSON.""" + keys = _matrix_args(session, PROJECT_CONFIG) + matrix = _dump_matrix(PROJECT_CONFIG, keys) + print(json.dumps(matrix)) @nox.session(name="matrix:python", python=False) def python_matrix(session: Session) -> None: """Output the build matrix for Python versions as JSON.""" - print(json.dumps(_python_matrix(PROJECT_CONFIG))) + _print_deprecated_matrix( + session=session, + config=PROJECT_CONFIG, + key_map={"python-version": "python_versions"}, + session_name="matrix:python", + replacement_args="python_versions", + ) @nox.session(name="matrix:exasol", python=False) def exasol_matrix(session: Session) -> None: """Output the build matrix for Exasol versions as JSON.""" - print(json.dumps(_exasol_matrix(PROJECT_CONFIG))) + _print_deprecated_matrix( + session=session, + config=PROJECT_CONFIG, + key_map={"exasol-version": "exasol_versions"}, + session_name="matrix:exasol", + replacement_args="exasol_versions", + ) @nox.session(name="matrix:all", python=False) def full_matrix(session: Session) -> None: """Output the full build matrix for Python & Exasol versions as JSON.""" - matrix = _python_matrix(PROJECT_CONFIG) - matrix.update(_exasol_matrix(PROJECT_CONFIG)) - print(json.dumps(matrix)) + _print_deprecated_matrix( + session=session, + config=PROJECT_CONFIG, + key_map={ + "python-version": "python_versions", + "exasol-version": "exasol_versions", + }, + session_name="matrix:all", + replacement_args="python_versions exasol_versions", + ) diff --git a/exasol/toolbox/nox/tasks.py b/exasol/toolbox/nox/tasks.py index 0fe726fbf..434417020 100644 --- a/exasol/toolbox/nox/tasks.py +++ b/exasol/toolbox/nox/tasks.py @@ -66,6 +66,7 @@ def check(session: Session) -> None: ) from exasol.toolbox.nox._ci import ( + generate_matrix, python_matrix, exasol_matrix, full_matrix, diff --git a/test/unit/nox/_ci_test.py b/test/unit/nox/_ci_test.py new file mode 100644 index 000000000..7fc0adb08 --- /dev/null +++ b/test/unit/nox/_ci_test.py @@ -0,0 +1,141 @@ +from __future__ import annotations + +import json +from unittest.mock import patch + +import pytest +from pydantic import computed_field + +from exasol.toolbox.config import BaseConfig +from exasol.toolbox.nox._ci import ( + _dump_matrix, + exasol_matrix, + full_matrix, + generate_matrix, + python_matrix, +) + + +@pytest.fixture +def nox_session_runner_posargs(request) -> list[str]: + return list(getattr(request, "param", [])) + + +@pytest.fixture +def config(tmp_path) -> BaseConfig: + class Config(BaseConfig): + extra_matrix_value: str = "extra" + + @computed_field # type: ignore[misc] + @property + def computed_matrix_value(self) -> str: + return f"{self.project_name}-computed" + + return Config(root_path=tmp_path, project_name="toolbox") + + +class TestGenerateMatrix: + @staticmethod + @pytest.mark.parametrize( + "nox_session_runner_posargs", + [["computed_matrix_value", "extra_matrix_value"]], + indirect=True, + ) + def test_uses_requested_field_names( + nox_session, + config, + capsys, + nox_session_runner_posargs, + ): + with patch("exasol.toolbox.nox._ci.PROJECT_CONFIG", new=config): + generate_matrix(nox_session) + + assert json.loads(capsys.readouterr().out) == { + "computed_matrix_value": ["toolbox-computed"], + "extra_matrix_value": ["extra"], + } + + @staticmethod + @pytest.mark.parametrize( + "nox_session_runner_posargs", + [["missing_value"]], + indirect=True, + ) + def test_rejects_unknown_field( + nox_session, + config, + capsys, + nox_session_runner_posargs, + ): + with patch("exasol.toolbox.nox._ci.PROJECT_CONFIG", new=config): + with pytest.raises(SystemExit): + generate_matrix(nox_session) + + assert "invalid choice: 'missing_value'" in capsys.readouterr().err + + +class TestDumpMatrix: + @staticmethod + @pytest.mark.parametrize( + ("requested_keys", "expected"), + [ + ( + ("computed_matrix_value",), + {"computed_matrix_value": ["toolbox-computed"]}, + ), + ( + ("computed_matrix_value", "extra_matrix_value"), + { + "computed_matrix_value": ["toolbox-computed"], + "extra_matrix_value": ["extra"], + }, + ), + ], + ) + def test_returns_requested_keys(config, requested_keys, expected): + assert _dump_matrix(config, requested_keys) == expected + + @staticmethod + def test_rejects_unknown_key(config): + with pytest.raises(KeyError, match="missing_matrix_value"): + _dump_matrix(config, ("missing_matrix_value",)) + + +class TestDeprecatedMatrixSessions: + @staticmethod + def test_exasol_session_still_emits_field_names( + nox_session, config, caplog, capsys + ): + with patch("exasol.toolbox.nox._ci.PROJECT_CONFIG", new=config): + exasol_matrix(nox_session) + + captured = capsys.readouterr() + assert json.loads(captured.out) == { + "exasol-version": ["7.1.30", "8.29.13", "2025.1.8"] + } + assert len(caplog.messages) == 1 + + @staticmethod + def test_python_session_still_emits_field_names( + nox_session, config, caplog, capsys + ): + with patch("exasol.toolbox.nox._ci.PROJECT_CONFIG", new=config): + python_matrix(nox_session) + + captured = capsys.readouterr() + assert json.loads(captured.out) == { + "python-version": ["3.10", "3.11", "3.12", "3.13", "3.14"] + } + assert len(caplog.messages) == 1 + + @staticmethod + def test_full_session_still_emits_field_names(nox_session, config, caplog, capsys): + with patch("exasol.toolbox.nox._ci.PROJECT_CONFIG", new=config): + full_matrix(nox_session) + + captured = capsys.readouterr() + assert json.loads(captured.out) == { + "python-version": ["3.10", "3.11", "3.12", "3.13", "3.14"], + "exasol-version": ["7.1.30", "8.29.13", "2025.1.8"], + } + assert len(caplog.messages) == 1 From ec7f6f93a88f761b2fbd925536b6992550299af3 Mon Sep 17 00:00:00 2001 From: Ariel Schulz Date: Tue, 9 Jun 2026 14:18:34 +0200 Subject: [PATCH 02/11] Add matrix.yml and remove older ones --- .github/workflows/matrix-all.yml | 2 + .github/workflows/matrix-exasol.yml | 2 + .github/workflows/matrix-python.yml | 2 + .../features/github_workflows/index.rst | 4 ++ .../github/workflows/matrix-exasol.yml | 40 ------------------- .../github/workflows/matrix-python.yml | 40 ------------------- .../workflows/{matrix-all.yml => matrix.yml} | 18 ++++++--- .../github/workflows/slow-checks.yml | 12 +++--- test/integration/project-template/nox_test.py | 2 +- test/unit/nox/_workflow_test.py | 8 ++-- test/unit/util/workflows/templates_test.py | 4 +- 11 files changed, 34 insertions(+), 100 deletions(-) delete mode 100644 exasol/toolbox/templates/github/workflows/matrix-exasol.yml delete mode 100644 exasol/toolbox/templates/github/workflows/matrix-python.yml rename exasol/toolbox/templates/github/workflows/{matrix-all.yml => matrix.yml} (59%) diff --git a/.github/workflows/matrix-all.yml b/.github/workflows/matrix-all.yml index cc8e849b0..b02124f41 100644 --- a/.github/workflows/matrix-all.yml +++ b/.github/workflows/matrix-all.yml @@ -1,5 +1,7 @@ # Generated and maintained by the exasol-toolbox. # Last generated with exasol-toolbox version 8.1.1. +# This workflow is deprecated and will be removed by 2026-09-15. Use matrix.yml instead: +# https://github.com/exasol/python-toolbox/issues/859 name: Build Matrix (All Versions) on: diff --git a/.github/workflows/matrix-exasol.yml b/.github/workflows/matrix-exasol.yml index bab0ffb02..8d1332662 100644 --- a/.github/workflows/matrix-exasol.yml +++ b/.github/workflows/matrix-exasol.yml @@ -1,5 +1,7 @@ # Generated and maintained by the exasol-toolbox. # Last generated with exasol-toolbox version 8.1.1. +# This workflow is deprecated and will be removed by 2026-09-15. Use matrix.yml instead: +# https://github.com/exasol/python-toolbox/issues/859 name: Build Matrix (Exasol) on: diff --git a/.github/workflows/matrix-python.yml b/.github/workflows/matrix-python.yml index c671c36f3..d33cb9065 100644 --- a/.github/workflows/matrix-python.yml +++ b/.github/workflows/matrix-python.yml @@ -1,5 +1,7 @@ # Generated and maintained by the exasol-toolbox. # Last generated with exasol-toolbox version 8.1.1. +# This workflow is deprecated and will be removed by 2026-09-15. Use matrix.yml instead: +# https://github.com/exasol/python-toolbox/issues/859 name: Build Matrix (Python) on: diff --git a/doc/user_guide/features/github_workflows/index.rst b/doc/user_guide/features/github_workflows/index.rst index 0e699d9a8..0b550ec93 100644 --- a/doc/user_guide/features/github_workflows/index.rst +++ b/doc/user_guide/features/github_workflows/index.rst @@ -84,6 +84,10 @@ Maintained by the PTB - Workflow call - Calls Nox session ``matrix:all``, which typically evaluates ``exasol_versions`` and ``python_versions`` from the ``PROJECT_CONFIG``. + * - ``matrix.yml`` + - Workflow call + - Calls Nox session ``matrix:generate`` with one or more space-separated + ``BaseConfig`` keys to build a custom matrix from the ``PROJECT_CONFIG``. * - ``matrix-exasol.yml`` - Workflow call - Calls Nox session ``matrix:exasol`` to get the ``exasol_versions`` from the diff --git a/exasol/toolbox/templates/github/workflows/matrix-exasol.yml b/exasol/toolbox/templates/github/workflows/matrix-exasol.yml deleted file mode 100644 index 18b3b851b..000000000 --- a/exasol/toolbox/templates/github/workflows/matrix-exasol.yml +++ /dev/null @@ -1,40 +0,0 @@ -(( workflow_header )) -name: Build Matrix (Exasol) - -on: - workflow_call: - outputs: - matrix: - description: "Generates the exasol version build matrix" - value: ${{ jobs.set-matrix-exasol.outputs.matrix }} - -jobs: - set-matrix-exasol: - runs-on: "(( os_version ))" - permissions: - contents: read - steps: - - name: Check out Repository - id: check-out-repository - uses: actions/checkout@v6 - with: - persist-credentials: false - - - name: Set up Python & Poetry Environment - id: set-up-python-and-poetry-environment - uses: exasol/python-toolbox/.github/actions/python-environment@v8 - with: - python-version: "(( minimum_python_version ))" - poetry-version: "(( dependency_manager_version ))" - - - name: Generate Matrix - id: generate-matrix - run: poetry run -- nox -s matrix:exasol - - - name: Set Matrix - id: set-matrix - run: | - echo "matrix=$(poetry run -- nox -s matrix:exasol)" >> $GITHUB_OUTPUT - - outputs: - matrix: ${{ steps.set-matrix.outputs.matrix }} diff --git a/exasol/toolbox/templates/github/workflows/matrix-python.yml b/exasol/toolbox/templates/github/workflows/matrix-python.yml deleted file mode 100644 index 062426ff1..000000000 --- a/exasol/toolbox/templates/github/workflows/matrix-python.yml +++ /dev/null @@ -1,40 +0,0 @@ -(( workflow_header )) -name: Build Matrix (Python) - -on: - workflow_call: - outputs: - matrix: - description: "Generates the python version build matrix" - value: ${{ jobs.set-matrix-python.outputs.matrix }} - -jobs: - set-matrix-python: - runs-on: "(( os_version ))" - permissions: - contents: read - steps: - - name: Check out Repository - id: check-out-repository - uses: actions/checkout@v6 - with: - persist-credentials: false - - - name: Set up Python & Poetry Environment - id: set-up-python-and-poetry-environment - uses: exasol/python-toolbox/.github/actions/python-environment@v8 - with: - python-version: "(( minimum_python_version ))" - poetry-version: "(( dependency_manager_version ))" - - - name: Generate Matrix - id: generate-matrix - run: poetry run -- nox -s matrix:python - - - name: Set Matrix - id: set-matrix - run: | - echo "matrix=$(poetry run -- nox -s matrix:python)" >> $GITHUB_OUTPUT - - outputs: - matrix: ${{ steps.set-matrix.outputs.matrix }} diff --git a/exasol/toolbox/templates/github/workflows/matrix-all.yml b/exasol/toolbox/templates/github/workflows/matrix.yml similarity index 59% rename from exasol/toolbox/templates/github/workflows/matrix-all.yml rename to exasol/toolbox/templates/github/workflows/matrix.yml index c24c2f2db..b1e4b284e 100644 --- a/exasol/toolbox/templates/github/workflows/matrix-all.yml +++ b/exasol/toolbox/templates/github/workflows/matrix.yml @@ -1,15 +1,21 @@ (( workflow_header )) -name: Build Matrix (All Versions) +name: Build Matrix on: workflow_call: + inputs: + matrix_keys: + description: "Space-separated BaseConfig keys to include in the generated matrix output" + default: python_versions exasol_versions + required: false + type: string outputs: matrix: - description: "Generates the all versions build matrix" - value: ${{ jobs.set-matrix-all.outputs.matrix }} + description: "Generates the requested build matrix" + value: ${{ jobs.set-matrix.outputs.matrix }} jobs: - set-matrix-all: + set-matrix: runs-on: "(( os_version ))" permissions: contents: read @@ -29,12 +35,12 @@ jobs: - name: Generate Matrix id: generate-matrix - run: poetry run -- nox -s matrix:all + run: poetry run -- nox -s matrix:generate -- ${{ inputs.matrix_keys }} - name: Set Matrix id: set-matrix run: | - echo "matrix=$(poetry run -- nox -s matrix:all)" >> $GITHUB_OUTPUT + echo "matrix=$(poetry run -- nox -s matrix:generate -- ${{ inputs.matrix_keys }})" >> $GITHUB_OUTPUT outputs: matrix: ${{ steps.set-matrix.outputs.matrix }} diff --git a/exasol/toolbox/templates/github/workflows/slow-checks.yml b/exasol/toolbox/templates/github/workflows/slow-checks.yml index 618e1790a..6e9efa82c 100644 --- a/exasol/toolbox/templates/github/workflows/slow-checks.yml +++ b/exasol/toolbox/templates/github/workflows/slow-checks.yml @@ -8,12 +8,14 @@ on: jobs: build-matrix: name: Build Matrix - uses: ./.github/workflows/matrix-all.yml + uses: ./.github/workflows/matrix.yml + with: + matrix_keys: python_versions exasol_versions permissions: contents: read run-integration-tests: - name: Run Integration Tests (Python-${{ matrix.python-version }}, Exasol-${{ matrix.exasol-version}}) + name: Run Integration Tests (Python-${{ matrix.python_versions }}, Exasol-${{ matrix.exasol_versions}}) needs: - build-matrix runs-on: "(( os_version ))" @@ -35,18 +37,18 @@ jobs: id: set-up-python-and-poetry-environment uses: exasol/python-toolbox/.github/actions/python-environment@v8 with: - python-version: ${{ matrix.python-version }} + python-version: ${{ matrix.python_versions }} poetry-version: "(( dependency_manager_version ))" - name: Run Integration Tests id: run-integration-tests - run: poetry run -- nox -s test:integration -- --coverage --db-version ${{ matrix.exasol-version }} + run: poetry run -- nox -s test:integration -- --coverage --db-version ${{ matrix.exasol_versions }} - name: Upload Artifacts id: upload-artifacts uses: actions/upload-artifact@v7 with: - name: coverage-python${{ matrix.python-version }}-exasol${{ matrix.exasol-version }}-slow + name: coverage-python${{ matrix.python_versions }}-exasol${{ matrix.exasol_versions }}-slow path: .coverage include-hidden-files: true overwrite: false diff --git a/test/integration/project-template/nox_test.py b/test/integration/project-template/nox_test.py index e06fe9f62..994e998fa 100644 --- a/test/integration/project-template/nox_test.py +++ b/test/integration/project-template/nox_test.py @@ -83,4 +83,4 @@ def test_install_github_workflows(self, poetry_path, run_command): assert output.returncode == 0 file_list = run_command(["ls", ".github/workflows"]).stdout.splitlines() - assert len(file_list) == 16 + assert len(file_list) == 14 diff --git a/test/unit/nox/_workflow_test.py b/test/unit/nox/_workflow_test.py index 983d16256..55ec74c10 100644 --- a/test/unit/nox/_workflow_test.py +++ b/test/unit/nox/_workflow_test.py @@ -39,7 +39,7 @@ class TestGenerateWorkflow: @staticmethod @pytest.mark.parametrize( "nox_session_runner_posargs, expected_count", - [(ALL, 16), *[(key, 1) for key in WORKFLOW_TEMPLATE_OPTIONS.keys()]], + [(ALL, 14), *[(key, 1) for key in WORKFLOW_TEMPLATE_OPTIONS.keys()]], indirect=["nox_session_runner_posargs"], ) def test_works_as_expected( @@ -128,7 +128,7 @@ def test_raises_session_quit_when_workflows_are_out_of_date( check_workflow(nox_session) assert str(exc.value) == ( - "\n16 workflows are out of date:\n" + "\n14 workflows are out of date:\n" "- build-and-publish\n" "- cd\n" "- check-release-tag\n" @@ -137,9 +137,7 @@ def test_raises_session_quit_when_workflows_are_out_of_date( "- dependency-update\n" "- fast-tests\n" "- gh-pages\n" - "- matrix-all\n" - "- matrix-exasol\n" - "- matrix-python\n" + "- matrix\n" "- merge-gate\n" "- periodic-validation\n" "- pr-merge\n" diff --git a/test/unit/util/workflows/templates_test.py b/test/unit/util/workflows/templates_test.py index 32a62957a..8969ad658 100644 --- a/test/unit/util/workflows/templates_test.py +++ b/test/unit/util/workflows/templates_test.py @@ -25,9 +25,7 @@ def test_get_workflow_templates(project_config): "fast-tests", "dependency-update", "gh-pages", - "matrix-all", - "matrix-exasol", - "matrix-python", + "matrix", "merge-gate", "periodic-validation", "pr-merge", From 233eba4555681167eee58ac001244bda62b7236b Mon Sep 17 00:00:00 2001 From: Ariel Schulz Date: Tue, 9 Jun 2026 14:23:50 +0200 Subject: [PATCH 03/11] Update PTB's workflows --- .github/workflows/matrix-exasol.yml | 43 ------------------- .github/workflows/matrix-python.yml | 43 ------------------- .../workflows/{matrix-all.yml => matrix.yml} | 20 +++++---- .github/workflows/slow-checks.yml | 10 +++-- 4 files changed, 18 insertions(+), 98 deletions(-) delete mode 100644 .github/workflows/matrix-exasol.yml delete mode 100644 .github/workflows/matrix-python.yml rename .github/workflows/{matrix-all.yml => matrix.yml} (59%) diff --git a/.github/workflows/matrix-exasol.yml b/.github/workflows/matrix-exasol.yml deleted file mode 100644 index 8d1332662..000000000 --- a/.github/workflows/matrix-exasol.yml +++ /dev/null @@ -1,43 +0,0 @@ -# Generated and maintained by the exasol-toolbox. -# Last generated with exasol-toolbox version 8.1.1. -# This workflow is deprecated and will be removed by 2026-09-15. Use matrix.yml instead: -# https://github.com/exasol/python-toolbox/issues/859 -name: Build Matrix (Exasol) - -on: - workflow_call: - outputs: - matrix: - description: "Generates the exasol version build matrix" - value: ${{ jobs.set-matrix-exasol.outputs.matrix }} - -jobs: - set-matrix-exasol: - runs-on: "ubuntu-24.04" - permissions: - contents: read - steps: - - name: Check out Repository - id: check-out-repository - uses: actions/checkout@v6 - with: - persist-credentials: false - - - name: Set up Python & Poetry Environment - id: set-up-python-and-poetry-environment - uses: exasol/python-toolbox/.github/actions/python-environment@v8 - with: - python-version: "3.10" - poetry-version: "2.3.0" - - - name: Generate Matrix - id: generate-matrix - run: poetry run -- nox -s matrix:exasol - - - name: Set Matrix - id: set-matrix - run: | - echo "matrix=$(poetry run -- nox -s matrix:exasol)" >> $GITHUB_OUTPUT - - outputs: - matrix: ${{ steps.set-matrix.outputs.matrix }} diff --git a/.github/workflows/matrix-python.yml b/.github/workflows/matrix-python.yml deleted file mode 100644 index d33cb9065..000000000 --- a/.github/workflows/matrix-python.yml +++ /dev/null @@ -1,43 +0,0 @@ -# Generated and maintained by the exasol-toolbox. -# Last generated with exasol-toolbox version 8.1.1. -# This workflow is deprecated and will be removed by 2026-09-15. Use matrix.yml instead: -# https://github.com/exasol/python-toolbox/issues/859 -name: Build Matrix (Python) - -on: - workflow_call: - outputs: - matrix: - description: "Generates the python version build matrix" - value: ${{ jobs.set-matrix-python.outputs.matrix }} - -jobs: - set-matrix-python: - runs-on: "ubuntu-24.04" - permissions: - contents: read - steps: - - name: Check out Repository - id: check-out-repository - uses: actions/checkout@v6 - with: - persist-credentials: false - - - name: Set up Python & Poetry Environment - id: set-up-python-and-poetry-environment - uses: exasol/python-toolbox/.github/actions/python-environment@v8 - with: - python-version: "3.10" - poetry-version: "2.3.0" - - - name: Generate Matrix - id: generate-matrix - run: poetry run -- nox -s matrix:python - - - name: Set Matrix - id: set-matrix - run: | - echo "matrix=$(poetry run -- nox -s matrix:python)" >> $GITHUB_OUTPUT - - outputs: - matrix: ${{ steps.set-matrix.outputs.matrix }} diff --git a/.github/workflows/matrix-all.yml b/.github/workflows/matrix.yml similarity index 59% rename from .github/workflows/matrix-all.yml rename to .github/workflows/matrix.yml index b02124f41..bd8a7fc86 100644 --- a/.github/workflows/matrix-all.yml +++ b/.github/workflows/matrix.yml @@ -1,18 +1,22 @@ # Generated and maintained by the exasol-toolbox. # Last generated with exasol-toolbox version 8.1.1. -# This workflow is deprecated and will be removed by 2026-09-15. Use matrix.yml instead: -# https://github.com/exasol/python-toolbox/issues/859 -name: Build Matrix (All Versions) +name: Build Matrix on: workflow_call: + inputs: + matrix_keys: + description: "Space-separated BaseConfig keys to include in the generated matrix output" + default: python_versions exasol_versions + required: false + type: string outputs: matrix: - description: "Generates the all versions build matrix" - value: ${{ jobs.set-matrix-all.outputs.matrix }} + description: "Generates the requested build matrix" + value: ${{ jobs.set-matrix.outputs.matrix }} jobs: - set-matrix-all: + set-matrix: runs-on: "ubuntu-24.04" permissions: contents: read @@ -32,12 +36,12 @@ jobs: - name: Generate Matrix id: generate-matrix - run: poetry run -- nox -s matrix:all + run: poetry run -- nox -s matrix:generate -- ${{ inputs.matrix_keys }} - name: Set Matrix id: set-matrix run: | - echo "matrix=$(poetry run -- nox -s matrix:all)" >> $GITHUB_OUTPUT + echo "matrix=$(poetry run -- nox -s matrix:generate -- ${{ inputs.matrix_keys }})" >> $GITHUB_OUTPUT outputs: matrix: ${{ steps.set-matrix.outputs.matrix }} diff --git a/.github/workflows/slow-checks.yml b/.github/workflows/slow-checks.yml index 8c4cd585f..98338cdc1 100644 --- a/.github/workflows/slow-checks.yml +++ b/.github/workflows/slow-checks.yml @@ -6,12 +6,14 @@ on: jobs: build-matrix: name: Build Matrix - uses: ./.github/workflows/matrix-python.yml + uses: ./.github/workflows/matrix.yml + with: + matrix_keys: python_versions permissions: contents: read run-integration-tests: - name: Run Integration Tests (Python-${{ matrix.python-version }}) + name: Run Integration Tests (Python-${{ matrix.python_versions }}) needs: - build-matrix runs-on: "ubuntu-24.04" @@ -31,7 +33,7 @@ jobs: id: set-up-python-and-poetry-environment uses: exasol/python-toolbox/.github/actions/python-environment@v8 with: - python-version: ${{ matrix.python-version }} + python-version: ${{ matrix.python_versions }} poetry-version: "2.3.0" - name: Run Integration Tests @@ -42,7 +44,7 @@ jobs: id: upload-artifacts uses: actions/upload-artifact@v7 with: - name: coverage-python${{ matrix.python-version }}-slow + name: coverage-python${{ matrix.python_versions }}-slow path: .coverage include-hidden-files: true overwrite: false From b0012bd27c83168ca9a48a3e9c71f3e7c32ca8bc Mon Sep 17 00:00:00 2001 From: Ariel Schulz Date: Tue, 9 Jun 2026 14:38:32 +0200 Subject: [PATCH 04/11] Fix missed workflow to matrix.yml --- .github/workflows/test-python-environment.yml | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/.github/workflows/test-python-environment.yml b/.github/workflows/test-python-environment.yml index f05267e9a..1242eda56 100644 --- a/.github/workflows/test-python-environment.yml +++ b/.github/workflows/test-python-environment.yml @@ -41,7 +41,9 @@ jobs: needs: - check-changes if: needs.check-changes.outputs.should_run == 'true' - uses: ./.github/workflows/matrix-all.yml + uses: ./.github/workflows/matrix.yml + with: + matrix_keys: python_versions permissions: contents: read @@ -60,8 +62,8 @@ jobs: - int-linux-x64-4core-gpu-t4-ubuntu24.04-1 - int-linux-x64-4core-ubuntu24.04-1 - int-linux-x64-2core-ubuntu24.04-1 - python-version: ${{ fromJson(needs.build-matrix.outputs.matrix).python-version }} - name: Verify Poetry Setup for ${{ matrix.runner }} (Python-${{ matrix.python-version }}) + python-versions: ${{ fromJson(needs.build-matrix.outputs.matrix).python_versions }} + name: Verify Poetry Setup for ${{ matrix.runner }} (Python-${{ matrix.python-versions }}) runs-on: labels: ${{ matrix.runner }} steps: @@ -75,7 +77,7 @@ jobs: id: set-up-python-and-poetry-environment uses: ./.github/actions/python-environment with: - python-version: "${{ matrix.python-version }}" + python-version: "${{ matrix.python-versions }}" poetry-version: "2.3.0" - name: Check Poetry Version @@ -85,7 +87,7 @@ jobs: - name: Validate Python Version id: validate-python-version env: - PYTHON_VERSION: ${{ matrix.python-version }} + PYTHON_VERSION: ${{ matrix.python-versions }} run: | poetry run which python poetry run python --version From c552e5e2a952511640cf2e6df82826a9e9233d93 Mon Sep 17 00:00:00 2001 From: Ariel Schulz Date: Wed, 10 Jun 2026 14:04:48 +0200 Subject: [PATCH 05/11] Switch poetry.urls to project.urls --- project-template/{{cookiecutter.repo_name}}/pyproject.toml | 2 +- pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/project-template/{{cookiecutter.repo_name}}/pyproject.toml b/project-template/{{cookiecutter.repo_name}}/pyproject.toml index db6d6ca7d..cdf34b5ef 100644 --- a/project-template/{{cookiecutter.repo_name}}/pyproject.toml +++ b/project-template/{{cookiecutter.repo_name}}/pyproject.toml @@ -35,7 +35,7 @@ include = [ [tool.poetry.requires-plugins] poetry-plugin-export = ">=1.8" -[poetry.urls] +[project.urls] repository = "https://github.com/exasol/{{cookiecutter.repo_name}}" homepage = "https://github.com/exasol/{{cookiecutter.repo_name}}" diff --git a/pyproject.toml b/pyproject.toml index c824c1faf..178c9b8b7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -84,7 +84,7 @@ include = [ "exasol/toolbox/templates/**/*" ] -[poetry.urls] +[project.urls] Homepage = "https://www.exasol.com/" Documentation = "https://exasol.github.io/python-toolbox/" Source = "https://github.com/exasol/python-toolbox" From 814f2a0edf95240ccb95739623a037c333a467f7 Mon Sep 17 00:00:00 2001 From: Ariel Schulz Date: Wed, 10 Jun 2026 14:10:03 +0200 Subject: [PATCH 06/11] Switch to a JSON array --- .github/workflows/matrix.yml | 50 +++++++++++++++---- .github/workflows/slow-checks.yml | 2 +- .github/workflows/test-python-environment.yml | 2 +- .../templates/github/workflows/matrix.yml | 50 +++++++++++++++---- .../github/workflows/slow-checks.yml | 2 +- 5 files changed, 85 insertions(+), 21 deletions(-) diff --git a/.github/workflows/matrix.yml b/.github/workflows/matrix.yml index bd8a7fc86..0f73f1285 100644 --- a/.github/workflows/matrix.yml +++ b/.github/workflows/matrix.yml @@ -5,10 +5,9 @@ name: Build Matrix on: workflow_call: inputs: - matrix_keys: - description: "Space-separated BaseConfig keys to include in the generated matrix output" - default: python_versions exasol_versions - required: false + matrix_keys_json: + description: "JSON array of BaseConfig keys to include in the generated matrix output." + required: true type: string outputs: matrix: @@ -34,14 +33,47 @@ jobs: python-version: "3.10" poetry-version: "2.3.0" - - name: Generate Matrix - id: generate-matrix - run: poetry run -- nox -s matrix:generate -- ${{ inputs.matrix_keys }} + - name: Parse Matrix Keys + id: parse-matrix-keys + env: + MATRIX_KEYS_JSON: ${{ inputs.matrix_keys_json }} + run: | + python - <<'PY' + import json + import os + + matrix_keys = json.loads(os.environ["MATRIX_KEYS_JSON"]) + with open(os.environ["GITHUB_OUTPUT"], "a", encoding="utf-8") as output_file: + print(f"matrix_keys_json={json.dumps(matrix_keys)}", file=output_file) + PY - - name: Set Matrix + - name: Generate Matrix id: set-matrix + env: + MATRIX_KEYS_JSON: ${{ steps.parse-matrix-keys.outputs.matrix_keys_json }} run: | - echo "matrix=$(poetry run -- nox -s matrix:generate -- ${{ inputs.matrix_keys }})" >> $GITHUB_OUTPUT + python - <<'PY' + import json + import os + import subprocess + + matrix_keys = json.loads(os.environ["MATRIX_KEYS_JSON"]) + matrix_json = subprocess.check_output( + [ + "poetry", + "run", + "--", + "nox", + "-s", + "matrix:generate", + "--", + *matrix_keys, + ], + text=True, + ).strip() + with open(os.environ["GITHUB_OUTPUT"], "a", encoding="utf-8") as output_file: + print(f"matrix={matrix_json}", file=output_file) + PY outputs: matrix: ${{ steps.set-matrix.outputs.matrix }} diff --git a/.github/workflows/slow-checks.yml b/.github/workflows/slow-checks.yml index 98338cdc1..5e127d1cd 100644 --- a/.github/workflows/slow-checks.yml +++ b/.github/workflows/slow-checks.yml @@ -8,7 +8,7 @@ jobs: name: Build Matrix uses: ./.github/workflows/matrix.yml with: - matrix_keys: python_versions + matrix_keys_json: '["python_versions"]' permissions: contents: read diff --git a/.github/workflows/test-python-environment.yml b/.github/workflows/test-python-environment.yml index 1242eda56..10432180e 100644 --- a/.github/workflows/test-python-environment.yml +++ b/.github/workflows/test-python-environment.yml @@ -43,7 +43,7 @@ jobs: if: needs.check-changes.outputs.should_run == 'true' uses: ./.github/workflows/matrix.yml with: - matrix_keys: python_versions + matrix_keys_json: '["python_versions"]' permissions: contents: read diff --git a/exasol/toolbox/templates/github/workflows/matrix.yml b/exasol/toolbox/templates/github/workflows/matrix.yml index b1e4b284e..e4b6ec331 100644 --- a/exasol/toolbox/templates/github/workflows/matrix.yml +++ b/exasol/toolbox/templates/github/workflows/matrix.yml @@ -4,10 +4,9 @@ name: Build Matrix on: workflow_call: inputs: - matrix_keys: - description: "Space-separated BaseConfig keys to include in the generated matrix output" - default: python_versions exasol_versions - required: false + matrix_keys_json: + description: "JSON array of BaseConfig keys to include in the generated matrix output." + required: true type: string outputs: matrix: @@ -33,14 +32,47 @@ jobs: python-version: "(( minimum_python_version ))" poetry-version: "(( dependency_manager_version ))" - - name: Generate Matrix - id: generate-matrix - run: poetry run -- nox -s matrix:generate -- ${{ inputs.matrix_keys }} + - name: Parse Matrix Keys + id: parse-matrix-keys + env: + MATRIX_KEYS_JSON: ${{ inputs.matrix_keys_json }} + run: | + python - <<'PY' + import json + import os + + matrix_keys = json.loads(os.environ["MATRIX_KEYS_JSON"]) + with open(os.environ["GITHUB_OUTPUT"], "a", encoding="utf-8") as output_file: + print(f"matrix_keys_json={json.dumps(matrix_keys)}", file=output_file) + PY - - name: Set Matrix + - name: Generate Matrix id: set-matrix + env: + MATRIX_KEYS_JSON: ${{ steps.parse-matrix-keys.outputs.matrix_keys_json }} run: | - echo "matrix=$(poetry run -- nox -s matrix:generate -- ${{ inputs.matrix_keys }})" >> $GITHUB_OUTPUT + python - <<'PY' + import json + import os + import subprocess + + matrix_keys = json.loads(os.environ["MATRIX_KEYS_JSON"]) + matrix_json = subprocess.check_output( + [ + "poetry", + "run", + "--", + "nox", + "-s", + "matrix:generate", + "--", + *matrix_keys, + ], + text=True, + ).strip() + with open(os.environ["GITHUB_OUTPUT"], "a", encoding="utf-8") as output_file: + print(f"matrix={matrix_json}", file=output_file) + PY outputs: matrix: ${{ steps.set-matrix.outputs.matrix }} diff --git a/exasol/toolbox/templates/github/workflows/slow-checks.yml b/exasol/toolbox/templates/github/workflows/slow-checks.yml index 6e9efa82c..01b9a6c0c 100644 --- a/exasol/toolbox/templates/github/workflows/slow-checks.yml +++ b/exasol/toolbox/templates/github/workflows/slow-checks.yml @@ -10,7 +10,7 @@ jobs: name: Build Matrix uses: ./.github/workflows/matrix.yml with: - matrix_keys: python_versions exasol_versions + matrix_keys_json: '["python_versions","exasol_versions"]' permissions: contents: read From 3fc086b076d2ecf9edbc0ed2f0ea2790b880a47e Mon Sep 17 00:00:00 2001 From: Ariel Schulz Date: Wed, 10 Jun 2026 14:12:10 +0200 Subject: [PATCH 07/11] Alter so runs when workflow file is modified to reduce delayed errors --- .github/workflows/test-python-environment.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test-python-environment.yml b/.github/workflows/test-python-environment.yml index 10432180e..a765da36f 100644 --- a/.github/workflows/test-python-environment.yml +++ b/.github/workflows/test-python-environment.yml @@ -27,8 +27,8 @@ jobs: # Otherwise, check if any relevant files were changed in this PR else - if git diff --quiet "origin/$BASE_REF...HEAD" -- .github/actions/python-environment; then - echo "No changes in .github/actions/python-environment, skipping." + if git diff --quiet "origin/$BASE_REF...HEAD" -- .github/actions/python-environment .github/workflows/test-python-environment.yml; then + echo "No changes in .github/actions/python-environment or .github/workflows/test-python-environment.yml, skipping." echo "should_run=false" >> $GITHUB_OUTPUT else echo "Changes detected in directory, running." From 15a8d4047ed82c791e7d736a8cc14fa7650f1b57 Mon Sep 17 00:00:00 2001 From: Ariel Schulz Date: Thu, 11 Jun 2026 06:56:36 +0200 Subject: [PATCH 08/11] Switch to jq and bash --- .github/workflows/matrix.yml | 43 +++---------------- .../templates/github/workflows/matrix.yml | 43 +++---------------- 2 files changed, 12 insertions(+), 74 deletions(-) diff --git a/.github/workflows/matrix.yml b/.github/workflows/matrix.yml index 0f73f1285..db5f4fdc1 100644 --- a/.github/workflows/matrix.yml +++ b/.github/workflows/matrix.yml @@ -33,47 +33,16 @@ jobs: python-version: "3.10" poetry-version: "2.3.0" - - name: Parse Matrix Keys - id: parse-matrix-keys - env: - MATRIX_KEYS_JSON: ${{ inputs.matrix_keys_json }} - run: | - python - <<'PY' - import json - import os - - matrix_keys = json.loads(os.environ["MATRIX_KEYS_JSON"]) - with open(os.environ["GITHUB_OUTPUT"], "a", encoding="utf-8") as output_file: - print(f"matrix_keys_json={json.dumps(matrix_keys)}", file=output_file) - PY - - name: Generate Matrix id: set-matrix env: - MATRIX_KEYS_JSON: ${{ steps.parse-matrix-keys.outputs.matrix_keys_json }} + MATRIX_KEYS_JSON: ${{ inputs.matrix_keys_json }} run: | - python - <<'PY' - import json - import os - import subprocess - - matrix_keys = json.loads(os.environ["MATRIX_KEYS_JSON"]) - matrix_json = subprocess.check_output( - [ - "poetry", - "run", - "--", - "nox", - "-s", - "matrix:generate", - "--", - *matrix_keys, - ], - text=True, - ).strip() - with open(os.environ["GITHUB_OUTPUT"], "a", encoding="utf-8") as output_file: - print(f"matrix={matrix_json}", file=output_file) - PY + readarray -t matrix_keys < <(jq -r '.[]' <<< "$MATRIX_KEYS_JSON") + matrix_json="$(poetry run -- nox -s matrix:generate -- "${matrix_keys[@]}")" + echo "Generated matrix JSON:" + jq . <<< "$matrix_json" + echo "matrix=$matrix_json" >> "$GITHUB_OUTPUT" outputs: matrix: ${{ steps.set-matrix.outputs.matrix }} diff --git a/exasol/toolbox/templates/github/workflows/matrix.yml b/exasol/toolbox/templates/github/workflows/matrix.yml index e4b6ec331..91329a819 100644 --- a/exasol/toolbox/templates/github/workflows/matrix.yml +++ b/exasol/toolbox/templates/github/workflows/matrix.yml @@ -32,47 +32,16 @@ jobs: python-version: "(( minimum_python_version ))" poetry-version: "(( dependency_manager_version ))" - - name: Parse Matrix Keys - id: parse-matrix-keys - env: - MATRIX_KEYS_JSON: ${{ inputs.matrix_keys_json }} - run: | - python - <<'PY' - import json - import os - - matrix_keys = json.loads(os.environ["MATRIX_KEYS_JSON"]) - with open(os.environ["GITHUB_OUTPUT"], "a", encoding="utf-8") as output_file: - print(f"matrix_keys_json={json.dumps(matrix_keys)}", file=output_file) - PY - - name: Generate Matrix id: set-matrix env: - MATRIX_KEYS_JSON: ${{ steps.parse-matrix-keys.outputs.matrix_keys_json }} + MATRIX_KEYS_JSON: ${{ inputs.matrix_keys_json }} run: | - python - <<'PY' - import json - import os - import subprocess - - matrix_keys = json.loads(os.environ["MATRIX_KEYS_JSON"]) - matrix_json = subprocess.check_output( - [ - "poetry", - "run", - "--", - "nox", - "-s", - "matrix:generate", - "--", - *matrix_keys, - ], - text=True, - ).strip() - with open(os.environ["GITHUB_OUTPUT"], "a", encoding="utf-8") as output_file: - print(f"matrix={matrix_json}", file=output_file) - PY + readarray -t matrix_keys < <(jq -r '.[]' <<< "$MATRIX_KEYS_JSON") + matrix_json="$(poetry run -- nox -s matrix:generate -- "${matrix_keys[@]}")" + echo "Generated matrix JSON:" + jq . <<< "$matrix_json" + echo "matrix=$matrix_json" >> "$GITHUB_OUTPUT" outputs: matrix: ${{ steps.set-matrix.outputs.matrix }} From ed980e254c03b7715fcbf3f8a5e6f99daec49f97 Mon Sep 17 00:00:00 2001 From: Ariel Schulz Date: Thu, 11 Jun 2026 07:14:19 +0200 Subject: [PATCH 09/11] Put behind slow approval for PR too --- .github/workflows/merge-gate-extension.yml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/.github/workflows/merge-gate-extension.yml b/.github/workflows/merge-gate-extension.yml index c952367ef..eecc3fe0e 100644 --- a/.github/workflows/merge-gate-extension.yml +++ b/.github/workflows/merge-gate-extension.yml @@ -4,8 +4,20 @@ on: workflow_call: jobs: + approve-run-slow-tests: + name: Approve Running Slow Tests? + runs-on: "ubuntu-24.04" + permissions: + contents: read + environment: manual-approval + steps: + - name: Tests + run: echo "Slow tests approved" + test-python-environment: name: Test python-environment Action uses: ./.github/workflows/test-python-environment.yml + needs: + - approve-run-slow-tests permissions: contents: read From b9bf47b64a19ae4775fe5739cabbdcb95bd88216 Mon Sep 17 00:00:00 2001 From: Ariel Schulz Date: Thu, 11 Jun 2026 13:02:23 +0200 Subject: [PATCH 10/11] Add summary description --- doc/changes/unreleased.md | 26 +++++++++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/doc/changes/unreleased.md b/doc/changes/unreleased.md index 6bc5dd92f..a6d3e23e2 100644 --- a/doc/changes/unreleased.md +++ b/doc/changes/unreleased.md @@ -2,8 +2,32 @@ ## Summary +This major version introduces `matrix.yml` as the new nox session `matrix:generate`. + +Projects can extend `BaseConfig` with additional matrix values when they need to expose +more entries to the workflows. + +```python +class Config(BaseConfig): + extra_matrix_value: str = "extra" + + @computed_field # type: ignore[misc] + @property + def computed_matrix_value(self) -> str: + # This can be requested when generating the matrix. If it is a singular value, + # like is shown here, then the code will automatically wrap it in an array. + return f"{self.project_name}-computed" +``` + +The corresponding nox sessions (`matrix:all`, `matrix:exasol`, and `matrix:python`) will +remain available until September 15, 2026, to provide a transition period for existing projects. + +At the same time, the workflows `matrix-all.yml`, `matrix-exasol.yml`, and `matrix-python.yml` +are deprecated and are no longer maintained by the exasol-toolbox. You can still use +these workflows in your project until you can transition fully to using `matrix.yml`. ## Feature * #730: Added support to extend GitHub workflow `cd.yml` -* #864: Modified PTB workflow templates to not persist credentials and to use pinned SHAs \ No newline at end of file +* #864: Modified PTB workflow templates to not persist credentials and to use pinned SHAs +* #654: Added and used general matrix `matrix.yml` for PTB-provided workflows From 9eca9d6107dbf17e9025412cc5bfcec49e1c8fc1 Mon Sep 17 00:00:00 2001 From: Ariel Schulz Date: Thu, 11 Jun 2026 13:06:16 +0200 Subject: [PATCH 11/11] Add documentation --- .../github_workflows/create_and_update.rst | 7 +- .../features/github_workflows/index.rst | 24 +++---- .../github_workflows/template_variables.rst | 15 ---- .../github_workflows/workflow_variables.rst | 68 +++++++++++++++++++ 4 files changed, 83 insertions(+), 31 deletions(-) delete mode 100644 doc/user_guide/features/github_workflows/template_variables.rst create mode 100644 doc/user_guide/features/github_workflows/workflow_variables.rst diff --git a/doc/user_guide/features/github_workflows/create_and_update.rst b/doc/user_guide/features/github_workflows/create_and_update.rst index b1da0db26..e6ab0958a 100644 --- a/doc/user_guide/features/github_workflows/create_and_update.rst +++ b/doc/user_guide/features/github_workflows/create_and_update.rst @@ -8,7 +8,7 @@ update existing workflows. The workflows are based on Jinja templates with variables populated by the PTB. The PTB reads the values from various attributes and properties of your -project's config, see :ref:`template_variables`. +project's config, see :ref:`workflow_templates`. Please note that the PTB only updates the values in the GitHub workflows when *updating* the workflows. So, after updating the :ref:`list of Python versions @@ -49,6 +49,11 @@ The PTB has a default for these versions, but you can override it in the Some workflows are expected to not depend on a specific Python version and will use only the lowest Python version in the list specified above. +If you need more than the built-in matrix inputs, you can extend +:class:`exasol.toolbox.config.BaseConfig` with extra properties or computed fields +and consume them from ``matrix.yml`` via ``matrix:generate``. See +:ref:`workflow_matrix` for an example. + .. _customize_workflows: Customize Workflows for Your Project diff --git a/doc/user_guide/features/github_workflows/index.rst b/doc/user_guide/features/github_workflows/index.rst index 724962196..35db31df4 100644 --- a/doc/user_guide/features/github_workflows/index.rst +++ b/doc/user_guide/features/github_workflows/index.rst @@ -9,7 +9,7 @@ GitHub Workflow Templates github_project_configuration create_and_update - template_variables + workflow_variables workflow_patcher The PTB ships with configurable GitHub workflow templates covering the most common @@ -44,6 +44,12 @@ The PTB has three categories of workflows: Maintained by the PTB ^^^^^^^^^^^^^^^^^^^^^ +.. note:: + The ``matrix.yml`` workflow replaces the older ``matrix-all.yml``, ``matrix-exasol.yml``, + and ``matrix-python.yml`` workflows. The associated nox sessions (``matrix:all``, + ``matrix:exasol``, and ``matrix:python``) are deprecated and will be removed + 2026-09-15. + .. list-table:: :widths: 25 25 50 :header-rows: 1 @@ -79,22 +85,10 @@ Maintained by the PTB * - ``gh-pages.yml`` - Workflow call - Builds the documentation and deploys it to GitHub Pages. - * - ``matrix-all.yml`` - - Workflow call - - Calls Nox session ``matrix:all``, which typically evaluates ``exasol_versions`` - and ``python_versions`` from the ``PROJECT_CONFIG``. * - ``matrix.yml`` - Workflow call - - Calls Nox session ``matrix:generate`` with one or more space-separated - ``BaseConfig`` keys to build a custom matrix from the ``PROJECT_CONFIG``. - * - ``matrix-exasol.yml`` - - Workflow call - - Calls Nox session ``matrix:exasol`` to get the ``exasol_versions`` from the - ``PROJECT_CONFIG``. - * - ``matrix-python.yml`` - - Workflow call - - Calls Nox session ``matrix:python`` to get the ``python_versions`` from the - ``PROJECT_CONFIG``. + - Calls Nox session ``matrix:generate`` to build a custom matrix from the + ``PROJECT_CONFIG``. See :ref:`workflow_matrix`. * - ``merge-gate.yml`` - Workflow call - Acts as a final status check (gatekeeper) to ensure all required CI steps have diff --git a/doc/user_guide/features/github_workflows/template_variables.rst b/doc/user_guide/features/github_workflows/template_variables.rst deleted file mode 100644 index f6ff6078b..000000000 --- a/doc/user_guide/features/github_workflows/template_variables.rst +++ /dev/null @@ -1,15 +0,0 @@ -.. _template_variables: - -Template Variables -================== - -Underlying the CLI, the PTB uses Jinja to dynamically generate project-specific -workflows. The rendering process is supported by the ``github_template_dict`` found in -your ``noxconfig.py::PROJECT_CONFIG``. This dictionary is inherited by default from -:py:attr:`exasol.toolbox.config.BaseConfig.github_template_dict`, ensuring a -standardized baseline that can be easily overridden, if necessary. - -.. literalinclude:: ../../../../exasol/toolbox/config.py - :language: python - :start-at: github_template_dict - :end-before: @computed_field diff --git a/doc/user_guide/features/github_workflows/workflow_variables.rst b/doc/user_guide/features/github_workflows/workflow_variables.rst new file mode 100644 index 000000000..8f38625f8 --- /dev/null +++ b/doc/user_guide/features/github_workflows/workflow_variables.rst @@ -0,0 +1,68 @@ +.. _workflow_variables: + +Workflow Variables +================== + +.. _workflow_templates: + +Workflow Templates +------------------ + +Underlying the CLI, the PTB uses Jinja to dynamically generate project-specific +workflows. The rendering process is supported by the ``github_template_dict`` found in +your ``noxconfig.py::PROJECT_CONFIG``. This dictionary is inherited by default from +:py:attr:`exasol.toolbox.config.BaseConfig.github_template_dict`, ensuring a +standardized baseline that can be easily overridden, if necessary. + +.. literalinclude:: ../../../../exasol/toolbox/config.py + :language: python + :start-at: github_template_dict + :end-before: @computed_field + + +.. _workflow_matrix: + +Matrix Combinations +------------------- + +The ``matrix.yml`` is used to generate different combinations of workflow inputs, such +as Python versions, Exasol versions, and any project-specific values exposed by the +config. This lets the PTB render workflows from a single matrix definition instead of +maintaining separate variants. + +Extending the Matrix +^^^^^^^^^^^^^^^^^^^^ + +If you need to expose additional values via the ``matrix.yml``, you can extend +:class:`exasol.toolbox.config.BaseConfig` as shown below. + +.. code-block:: python + + from pydantic import computed_field + + from exasol.toolbox.config import BaseConfig + + + class Config(BaseConfig): + extra_matrix_value: str = "extra" + + @computed_field # type: ignore[misc] + @property + def computed_matrix_value(self) -> str: + # A single value is automatically wrapped into a list when the matrix is generated. + return f"{self.project_name}-computed" + +You can consume the additional value in a workflow by passing the relevant +``matrix_keys_json`` entries when calling ``matrix.yml``: + +.. code-block:: yaml + + jobs: + build-matrix: + uses: ./.github/workflows/matrix.yml + with: + matrix_keys_json: '["extra_matrix_value","computed_matrix_value"]' + +Singular values such as ``computed_matrix_value`` do not need to be wrapped in an +array in ``PROJECT_CONFIG``. The ``matrix.yml`` generation automatically converts them +into arrays for the workflow matrix.