From b04019aca9b2174808c9dedbc513caff84d1871d Mon Sep 17 00:00:00 2001 From: PranjalManhgaye Date: Sat, 29 Aug 2026 14:14:15 +0530 Subject: [PATCH 1/7] Parallelize release system tests via per-tutorial matrix (#789) Add prepare-matrix job that reads release from tests.yaml and runs one GitHub Actions job per tutorial with per-case suite names. --- .github/workflows/system-tests-manual.yml | 26 ++- .github/workflows/system-tests-nightly.yml | 4 +- .../workflows/system-tests-release-matrix.yml | 61 +++++++ changelog-entries/789.md | 1 + tests/README.md | 7 +- tests/parallel_matrix.py | 152 ++++++++++++++++++ tests/prepare_matrix.py | 68 ++++++++ 7 files changed, 312 insertions(+), 7 deletions(-) create mode 100644 .github/workflows/system-tests-release-matrix.yml create mode 100644 changelog-entries/789.md create mode 100644 tests/parallel_matrix.py create mode 100644 tests/prepare_matrix.py diff --git a/.github/workflows/system-tests-manual.yml b/.github/workflows/system-tests-manual.yml index 2d4fee4a3..52d37fa76 100644 --- a/.github/workflows/system-tests-manual.yml +++ b/.github/workflows/system-tests-manual.yml @@ -53,11 +53,33 @@ on: type: string jobs: - run-system-tests: + run-system-tests-parallel: name: Trigger system tests + if: ${{ inputs.suites == 'release' }} + uses: ./.github/workflows/system-tests-release-matrix.yml + with: + build_args: "TUTORIALS_REF:${{ inputs.ref-tutorials }},\ + PRECICE_REF:${{ inputs.ref-precice }},\ + PYTHON_BINDINGS_REF:${{ inputs.ref-python-bindings }},\ + CALCULIX_ADAPTER_REF:${{ inputs.ref-calculix-adapter }},\ + DEALII_ADAPTER_REF:${{ inputs.ref-dealii-adapter }},\ + DUMUX_ADAPTER_REF:${{ inputs.ref-dumux-adapter }},\ + DUNE_ADAPTER_REF:${{ inputs.ref-dune-adapter }},\ + FENICS_ADAPTER_REF:${{ inputs.ref-fenics-adapter }},\ + FENICSX_ADAPTER_REF:${{ inputs.ref-fenicsx-adapter }},\ + FMI_RUNNER_REF:${{ inputs.ref-fmi-runner }},\ + MICRO_MANAGER_REF:${{ inputs.ref-micro-manager }},\ + OPENFOAM_ADAPTER_REF:${{ inputs.ref-openfoam-adapter }},\ + SU2_ADAPTER_REF:${{ inputs.ref-su2-adapter }}" + system_tests_branch: ${{ inputs.system_tests_branch || 'develop' }} + log_level: "INFO" + + run-system-tests-custom: + name: Trigger system tests + if: ${{ inputs.suites != 'release' }} uses: ./.github/workflows/system-tests.yml with: - suites: ${{ inputs.suites || 'release' }} + suites: ${{ inputs.suites }} build_args: "TUTORIALS_REF:${{ inputs.ref-tutorials }},\ PRECICE_REF:${{ inputs.ref-precice }},\ PYTHON_BINDINGS_REF:${{ inputs.ref-python-bindings }},\ diff --git a/.github/workflows/system-tests-nightly.yml b/.github/workflows/system-tests-nightly.yml index 2f1e973e2..133e42070 100644 --- a/.github/workflows/system-tests-nightly.yml +++ b/.github/workflows/system-tests-nightly.yml @@ -8,6 +8,4 @@ on: jobs: run-system-tests: name: Trigger system tests - uses: ./.github/workflows/system-tests.yml - with: - suites: 'release' + uses: ./.github/workflows/system-tests-release-matrix.yml diff --git a/.github/workflows/system-tests-release-matrix.yml b/.github/workflows/system-tests-release-matrix.yml new file mode 100644 index 000000000..569754b9e --- /dev/null +++ b/.github/workflows/system-tests-release-matrix.yml @@ -0,0 +1,61 @@ +name: System tests (release matrix) + +on: + workflow_call: + inputs: + build_args: + description: 'Build arguments' + type: string + default: '' + system_tests_branch: + description: 'Branch to take the system tests from' + default: 'develop' + type: string + log_level: + description: 'Logging verbosity level used for the systemtests' + default: 'INFO' + type: string + upload_artifacts: + description: 'TRUE or FALSE (FALSE -> Upload only when the workflow fails)' + default: 'TRUE' + type: string + max_parallel: + description: 'Maximum number of parallel matrix jobs' + default: '4' + type: string + +jobs: + prepare-matrix: + name: Prepare release matrix + runs-on: ubuntu-latest + outputs: + matrix: ${{ steps.matrix.outputs.matrix }} + steps: + - name: Check out tutorials + uses: actions/checkout@v7 + with: + ref: ${{ inputs.system_tests_branch }} + - name: Install Python dependencies + run: pip install --user -r tests/requirements.txt + - name: Build matrix from release suite + id: matrix + working-directory: tests + run: | + python3 prepare_matrix.py --source-suite release --format=github > matrix.json + echo "matrix=$(cat matrix.json)" >> "$GITHUB_OUTPUT" + + run-system-tests: + name: Trigger system tests (${{ matrix.tutorial }}) + needs: prepare-matrix + strategy: + fail-fast: false + max-parallel: ${{ fromJSON(inputs.max_parallel) }} + matrix: + include: ${{ fromJson(needs.prepare-matrix.outputs.matrix) }} + uses: ./.github/workflows/system-tests.yml + with: + suites: ${{ matrix.suites }} + build_args: ${{ inputs.build_args }} + system_tests_branch: ${{ inputs.system_tests_branch }} + log_level: ${{ inputs.log_level }} + upload_artifacts: ${{ inputs.upload_artifacts }} diff --git a/changelog-entries/789.md b/changelog-entries/789.md new file mode 100644 index 000000000..17749a997 --- /dev/null +++ b/changelog-entries/789.md @@ -0,0 +1 @@ +- Parallelize scheduled and manual `release` system tests via a per-tutorial GitHub Actions matrix derived from `tests.yaml` [#789](https://github.com/precice/tutorials/issues/789) diff --git a/tests/README.md b/tests/README.md index 0b932319a..c1986098b 100644 --- a/tests/README.md +++ b/tests/README.md @@ -32,7 +32,9 @@ The available test suites are found in [`tests.yaml`](https://github.com/precice The `Use workflow from` is a default option of GitHub Actions that concerns the GHA workflow file itself. -The [System tests (nightly)](https://github.com/precice/tutorials/actions/workflows/system-tests-nightly.yml) executes the `release` test suite every night. The [System tests (weekly)](https://github.com/precice/tutorials/actions/workflows/system-tests-weekly.yml) executes the `extra` test suite once per week. +The [System tests (nightly)](https://github.com/precice/tutorials/actions/workflows/system-tests-nightly.yml) executes the `release` test suite every night as a GitHub Actions matrix with one job per tutorial. The [System tests (weekly)](https://github.com/precice/tutorials/actions/workflows/system-tests-weekly.yml) executes the `extra` test suite once per week. + +When the manual workflow runs `release`, it uses the same per-tutorial matrix. Other suite selections still run as a single job. If a matrix job fails, re-run only that tutorial job from the Actions run page. ### Running from a pull request @@ -77,10 +79,11 @@ This will build and connect Docker containers and run tutorials in the `runs/` d To clean up at the end, you might want to run a `docker system prune -a` and remove the `runs/` directory to save space. -There are also some auxiliary scripts (run without arguments): +There are also some auxiliary scripts: - `print_test_suites.py`: Print all test suites defined in `tests.yaml` - `print_case_combinations.py`: Print all possible combinations of participants in a tutorial, using its `metadata.yaml`. +- `prepare_matrix.py`: Build the per-tutorial matrix used for parallel `release` runs. ## Understanding the logs diff --git a/tests/parallel_matrix.py b/tests/parallel_matrix.py new file mode 100644 index 000000000..81abf90ee --- /dev/null +++ b/tests/parallel_matrix.py @@ -0,0 +1,152 @@ +"""Build a per-tutorial parallel matrix from a meta test suite in tests.yaml.""" + +from __future__ import annotations + +from collections import defaultdict +from dataclasses import dataclass +from pathlib import Path +from typing import Dict, List, Sequence, Tuple + +import yaml + +from paths import PRECICE_TESTS_DIR + +CaseKey = Tuple[str, Tuple[str, ...]] + + +@dataclass(frozen=True) +class MatrixJob: + tutorial: str + suites: str + case_count: int + + +def _tests_yaml_path(tests_yaml: Path | None = None) -> Path: + return tests_yaml if tests_yaml is not None else PRECICE_TESTS_DIR / "tests.yaml" + + +def load_test_suites(tests_yaml: Path | None = None) -> dict: + with open(_tests_yaml_path(tests_yaml), "r", encoding="utf-8") as handle: + data = yaml.safe_load(handle) + return data["test_suites"] + + +def build_case_suite_lookup(test_suites: dict) -> Dict[CaseKey, str]: + """Map (path, case_combination) to the runnable per-case suite name.""" + candidates: Dict[CaseKey, set[str]] = defaultdict(set) + for suite_name, suite_def in test_suites.items(): + tutorials = suite_def.get("tutorials", []) + if len(tutorials) != 1: + continue + entry = tutorials[0] + key = (entry["path"], tuple(entry["case_combination"])) + candidates[key].add(suite_name) + + lookup: Dict[CaseKey, str] = {} + for key, suite_names in candidates.items(): + path, combo = key + canonical = f"{path}_{'_'.join(combo)}" + if canonical in suite_names: + lookup[key] = canonical + else: + lookup[key] = sorted(suite_names)[0] + return lookup + + +def build_external_suite_lookup(test_suites: dict) -> Dict[CaseKey, str]: + """Map external cases to the suite that defines them (e.g. mixedbc).""" + lookup: Dict[CaseKey, str] = {} + for suite_name, suite_def in test_suites.items(): + external = suite_def.get("external", []) + if len(external) != 1: + continue + entry = external[0] + lookup[(entry["path"], tuple(entry["case_combination"]))] = suite_name + return lookup + + +def _release_case_entries( + source_suite: dict, + case_lookup: Dict[CaseKey, str], + external_lookup: Dict[CaseKey, str], +) -> List[Tuple[str, str]]: + """Return (tutorial_path, case_suite_name) pairs for a meta suite definition.""" + entries: List[Tuple[str, str]] = [] + + def add_entry(case_def: dict) -> None: + case_key: CaseKey = (case_def["path"], tuple(case_def["case_combination"])) + tutorial = case_def["path"] + if case_key in case_lookup: + suite_name = case_lookup[case_key] + elif case_key in external_lookup: + suite_name = external_lookup[case_key] + else: + raise KeyError( + f"No runnable suite found for {tutorial} " + f"{case_def['case_combination']}") + entries.append((tutorial, suite_name)) + + for case_def in source_suite.get("tutorials", []): + add_entry(case_def) + for case_def in source_suite.get("external", []): + add_entry(case_def) + + return entries + + +def build_parallel_matrix( + source_suite_name: str = "release", + tests_yaml: Path | None = None, +) -> List[MatrixJob]: + test_suites = load_test_suites(tests_yaml) + if source_suite_name not in test_suites: + raise KeyError(f"Unknown test suite: {source_suite_name}") + + case_lookup = build_case_suite_lookup(test_suites) + external_lookup = build_external_suite_lookup(test_suites) + release_entries = _release_case_entries( + test_suites[source_suite_name], + case_lookup, + external_lookup, + ) + + grouped: Dict[str, List[str]] = defaultdict(list) + for tutorial, case_suite in release_entries: + grouped[tutorial].append(case_suite) + + matrix_jobs: List[MatrixJob] = [] + for tutorial in sorted(grouped): + case_suites = grouped[tutorial] + matrix_jobs.append( + MatrixJob( + tutorial=tutorial, + suites=",".join(case_suites), + case_count=len(case_suites), + ) + ) + return matrix_jobs + + +def matrix_jobs_for_github( + source_suite_name: str = "release", + tests_yaml: Path | None = None, +) -> List[dict]: + return [ + {"tutorial": job.tutorial, "suites": job.suites} + for job in build_parallel_matrix(source_suite_name, tests_yaml) + ] + + +def release_case_suite_names( + source_suite_name: str = "release", + tests_yaml: Path | None = None, +) -> List[str]: + test_suites = load_test_suites(tests_yaml) + case_lookup = build_case_suite_lookup(test_suites) + external_lookup = build_external_suite_lookup(test_suites) + entries = _release_case_entries( + test_suites[source_suite_name], + case_lookup, + external_lookup, + ) + return sorted(suite for _, suite in entries) diff --git a/tests/prepare_matrix.py b/tests/prepare_matrix.py new file mode 100644 index 000000000..f6c273777 --- /dev/null +++ b/tests/prepare_matrix.py @@ -0,0 +1,68 @@ +#!/usr/bin/env python3 +"""Emit a GitHub Actions matrix for parallel release system tests.""" + +from __future__ import annotations + +import argparse +import json +import os +import sys + +from parallel_matrix import build_parallel_matrix, matrix_jobs_for_github + + +def _write_github_step_summary(matrix_jobs) -> None: + summary_path = os.environ.get("GITHUB_STEP_SUMMARY") + if not summary_path: + return + + total_cases = sum(job.case_count for job in matrix_jobs) + with open(summary_path, "w", encoding="utf-8") as summary_file: + print("## Parallel test matrix\n", file=summary_file) + print(f"- Source suite: `release`", file=summary_file) + print(f"- Tutorial jobs: **{len(matrix_jobs)}**", file=summary_file) + print(f"- Total cases: **{total_cases}**\n", file=summary_file) + print("| Tutorial | Cases |", file=summary_file) + print("| --- | ---: |", file=summary_file) + for job in matrix_jobs: + print(f"| `{job.tutorial}` | {job.case_count} |", file=summary_file) + + +def main() -> int: + parser = argparse.ArgumentParser( + description="Build a per-tutorial matrix from a meta test suite.") + parser.add_argument( + "--source-suite", + default="release", + help="Meta suite to expand (default: release).", + ) + parser.add_argument( + "--format", + choices=["github", "pretty"], + default="pretty", + help="Output format (github: JSON for fromJson()).", + ) + args = parser.parse_args() + + try: + matrix_jobs = build_parallel_matrix(args.source_suite) + except (KeyError, FileNotFoundError) as error: + print(error, file=sys.stderr) + return 1 + + _write_github_step_summary(matrix_jobs) + + if args.format == "github": + print(json.dumps(matrix_jobs_for_github(args.source_suite))) + else: + total_cases = sum(job.case_count for job in matrix_jobs) + print( + f"Matrix for {args.source_suite}: " + f"{len(matrix_jobs)} tutorial jobs, {total_cases} cases") + for job in matrix_jobs: + print(f" {job.tutorial}: {job.case_count} case(s)") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) From 2777c44ce95135ecd0b61098e1f136c2085f0986 Mon Sep 17 00:00:00 2001 From: PranjalManhgaye Date: Sun, 25 Oct 2026 11:46:00 +0530 Subject: [PATCH 2/7] Refactor system test matrix workflows per review - Move prepare-matrix into manual and nightly workflows - Drop system-tests-release-matrix.yml - Use one matrix path: serial = 1 job, parallel = per-tutorial - Generalize prepare_matrix.py for any suite selection - Add unit tests for matrix generation --- .github/workflows/system-tests-manual.yml | 82 ++++++------ .github/workflows/system-tests-nightly.yml | 27 +++- .../workflows/system-tests-release-matrix.yml | 57 --------- tests/README.md | 4 +- tests/parallel_matrix.py | 84 +++++++++++++ tests/prepare_matrix.py | 71 +++++++---- tests/test_parallel_matrix.py | 117 ++++++++++++++++++ 7 files changed, 309 insertions(+), 133 deletions(-) delete mode 100644 .github/workflows/system-tests-release-matrix.yml create mode 100644 tests/test_parallel_matrix.py diff --git a/.github/workflows/system-tests-manual.yml b/.github/workflows/system-tests-manual.yml index 3e4124465..b90e5b77f 100644 --- a/.github/workflows/system-tests-manual.yml +++ b/.github/workflows/system-tests-manual.yml @@ -9,7 +9,7 @@ on: required: true type: string parallel: - description: 'Run release as a per-tutorial matrix (serial by default)' + description: 'Split expandable suites into per-tutorial matrix jobs' default: false type: boolean system_tests_branch: @@ -57,55 +57,42 @@ on: type: string jobs: - run-system-tests-serial: - name: Trigger system tests - if: ${{ inputs.suites == 'release' && !inputs.parallel }} - uses: $/.github/workflows/system-tests.yml - with: - suites: release - build_args: "TUTORIALS_REF:${{ inputs.ref-tutorials }},\ - PRECICE_REF:${{ inputs.ref-precice }},\ - PYTHON_BINDINGS_REF:${{ inputs.ref-python-bindings }},\ - CALCULIX_ADAPTER_REF:${{ inputs.ref-calculix-adapter }},\ - DEALII_ADAPTER_REF:${{ inputs.ref-dealii-adapter }},\ - DUMUX_ADAPTER_REF:${{ inputs.ref-dumux-adapter }},\ - DUNE_ADAPTER_REF:${{ inputs.ref-dune-adapter }},\ - FENICS_ADAPTER_REF:${{ inputs.ref-fenics-adapter }},\ - FENICSX_ADAPTER_REF:${{ inputs.ref-fenicsx-adapter }},\ - FMI_RUNNER_REF:${{ inputs.ref-fmi-runner }},\ - MICRO_MANAGER_REF:${{ inputs.ref-micro-manager }},\ - OPENFOAM_ADAPTER_REF:${{ inputs.ref-openfoam-adapter }},\ - SU2_ADAPTER_REF:${{ inputs.ref-su2-adapter }}" - system_tests_branch: ${{ inputs.system_tests_branch || 'develop' }} - log_level: "INFO" - - run-system-tests-parallel: - name: Trigger system tests (parallel) - if: ${{ inputs.suites == 'release' && inputs.parallel }} - uses: $/.github/workflows/system-tests-release-matrix.yml - with: - build_args: "TUTORIALS_REF:${{ inputs.ref-tutorials }},\ - PRECICE_REF:${{ inputs.ref-precice }},\ - PYTHON_BINDINGS_REF:${{ inputs.ref-python-bindings }},\ - CALCULIX_ADAPTER_REF:${{ inputs.ref-calculix-adapter }},\ - DEALII_ADAPTER_REF:${{ inputs.ref-dealii-adapter }},\ - DUMUX_ADAPTER_REF:${{ inputs.ref-dumux-adapter }},\ - DUNE_ADAPTER_REF:${{ inputs.ref-dune-adapter }},\ - FENICS_ADAPTER_REF:${{ inputs.ref-fenics-adapter }},\ - FENICSX_ADAPTER_REF:${{ inputs.ref-fenicsx-adapter }},\ - FMI_RUNNER_REF:${{ inputs.ref-fmi-runner }},\ - MICRO_MANAGER_REF:${{ inputs.ref-micro-manager }},\ - OPENFOAM_ADAPTER_REF:${{ inputs.ref-openfoam-adapter }},\ - SU2_ADAPTER_REF:${{ inputs.ref-su2-adapter }}" - system_tests_branch: ${{ inputs.system_tests_branch || 'develop' }} - log_level: "INFO" + prepare-matrix: + name: Prepare test matrix + runs-on: ubuntu-latest + outputs: + matrix: ${{ steps.matrix.outputs.matrix }} + steps: + - name: Check out tutorials + uses: actions/checkout@v7 + with: + ref: ${{ inputs.system_tests_branch }} + - name: Install Python dependencies + run: pip install --user -r tests/requirements.txt + - name: Build matrix from selected suites + id: matrix + working-directory: tests + env: + SUITES: ${{ inputs.suites }} + PARALLEL: ${{ inputs.parallel }} + run: | + MODE=serial + if [ "$PARALLEL" = "true" ]; then + MODE=parallel + fi + python3 prepare_matrix.py --suites "$SUITES" --mode "$MODE" --format=github > matrix.json + echo "matrix=$(cat matrix.json)" >> "$GITHUB_OUTPUT" - run-system-tests-custom: - name: Trigger system tests - if: ${{ inputs.suites != 'release' }} + run-system-tests: + name: ${{ matrix.tutorial != '' && format('Trigger system tests ({0})', matrix.tutorial) || 'Trigger system tests' }} + needs: prepare-matrix + strategy: + fail-fast: false + matrix: + include: ${{ fromJson(needs.prepare-matrix.outputs.matrix) }} uses: $/.github/workflows/system-tests.yml with: - suites: ${{ inputs.suites }} + suites: ${{ matrix.suites }} build_args: "TUTORIALS_REF:${{ inputs.ref-tutorials }},\ PRECICE_REF:${{ inputs.ref-precice }},\ PYTHON_BINDINGS_REF:${{ inputs.ref-python-bindings }},\ @@ -121,3 +108,4 @@ jobs: SU2_ADAPTER_REF:${{ inputs.ref-su2-adapter }}" system_tests_branch: ${{ inputs.system_tests_branch || 'develop' }} log_level: "INFO" + job_label: ${{ matrix.tutorial }} diff --git a/.github/workflows/system-tests-nightly.yml b/.github/workflows/system-tests-nightly.yml index 6014c4cbb..68189d66c 100644 --- a/.github/workflows/system-tests-nightly.yml +++ b/.github/workflows/system-tests-nightly.yml @@ -6,8 +6,33 @@ on: timezone: "Europe/Berlin" jobs: + prepare-matrix: + name: Prepare test matrix + runs-on: ubuntu-latest + outputs: + matrix: ${{ steps.matrix.outputs.matrix }} + steps: + - name: Check out tutorials + uses: actions/checkout@v7 + with: + ref: develop + - name: Install Python dependencies + run: pip install --user -r tests/requirements.txt + - name: Build serial matrix for release + id: matrix + working-directory: tests + run: | + python3 prepare_matrix.py --suites release --mode serial --format=github > matrix.json + echo "matrix=$(cat matrix.json)" >> "$GITHUB_OUTPUT" + run-system-tests: name: Trigger system tests + needs: prepare-matrix + strategy: + fail-fast: false + matrix: + include: ${{ fromJson(needs.prepare-matrix.outputs.matrix) }} uses: $/.github/workflows/system-tests.yml with: - suites: 'release' + suites: ${{ matrix.suites }} + job_label: ${{ matrix.tutorial }} diff --git a/.github/workflows/system-tests-release-matrix.yml b/.github/workflows/system-tests-release-matrix.yml deleted file mode 100644 index 46f746a0a..000000000 --- a/.github/workflows/system-tests-release-matrix.yml +++ /dev/null @@ -1,57 +0,0 @@ -name: System tests (release matrix) - -on: - workflow_call: - inputs: - build_args: - description: 'Build arguments' - type: string - default: '' - system_tests_branch: - description: 'Branch to take the system tests from' - default: 'develop' - type: string - log_level: - description: 'Logging verbosity level used for the systemtests' - default: 'INFO' - type: string - upload_artifacts: - description: 'TRUE or FALSE (FALSE -> Upload only when the workflow fails)' - default: 'TRUE' - type: string - -jobs: - prepare-matrix: - name: Prepare release matrix - runs-on: ubuntu-latest - outputs: - matrix: ${{ steps.matrix.outputs.matrix }} - steps: - - name: Check out tutorials - uses: actions/checkout@v7 - with: - ref: ${{ inputs.system_tests_branch }} - - name: Install Python dependencies - run: pip install --user -r tests/requirements.txt - - name: Build matrix from release suite - id: matrix - working-directory: tests - run: | - python3 prepare_matrix.py --source-suite release --format=github > matrix.json - echo "matrix=$(cat matrix.json)" >> "$GITHUB_OUTPUT" - - run-system-tests: - name: Trigger system tests (${{ matrix.tutorial }}) - needs: prepare-matrix - strategy: - fail-fast: false - matrix: - include: ${{ fromJson(needs.prepare-matrix.outputs.matrix) }} - uses: $/.github/workflows/system-tests.yml - with: - suites: ${{ matrix.suites }} - build_args: ${{ inputs.build_args }} - system_tests_branch: ${{ inputs.system_tests_branch }} - log_level: ${{ inputs.log_level }} - upload_artifacts: ${{ inputs.upload_artifacts }} - job_label: ${{ matrix.tutorial }} diff --git a/tests/README.md b/tests/README.md index 9d9c92146..c91522f4c 100644 --- a/tests/README.md +++ b/tests/README.md @@ -34,7 +34,7 @@ The `Use workflow from` is a default option of GitHub Actions that concerns the The [System tests (nightly)](https://github.com/precice/tutorials/actions/workflows/system-tests-nightly.yml) executes the `release` test suite every night. The [System tests (weekly)](https://github.com/precice/tutorials/actions/workflows/system-tests-weekly.yml) executes the `extra` test suite once per week. -When the manual workflow runs `release`, it uses a single serial job by default. Enable the `parallel` input to split `release` into one job per tutorial. Other suite selections always run as a single job. If a parallel matrix job fails, re-run only that tutorial job from the Actions run page. +The manual and nightly workflows always run a `prepare-matrix` job first. By default, suites run in a single serial matrix job. Enable the `parallel` input to split expandable suites (such as `release`) into one job per tutorial. If a parallel matrix job fails, re-run only that tutorial job from the Actions run page. ### Running from a pull request @@ -89,7 +89,7 @@ There are also some auxiliary scripts: - `print_test_suites.py`: Print all test suites defined in `tests.yaml` - `print_case_combinations.py`: Print all possible combinations of participants in a tutorial, using its `metadata.yaml`. -- `prepare_matrix.py`: Build the per-tutorial matrix used for parallel `release` runs. +- `prepare_matrix.py`: Build the serial or per-tutorial matrix used by system test workflows. ## Understanding the logs diff --git a/tests/parallel_matrix.py b/tests/parallel_matrix.py index fa50c634a..cc68c2e1c 100644 --- a/tests/parallel_matrix.py +++ b/tests/parallel_matrix.py @@ -94,6 +94,86 @@ def add_entry(case_def: dict) -> None: return entries +def build_serial_matrix(suites: str) -> List[MatrixJob]: + """Return a single matrix job that runs the given suite selection as-is.""" + return [MatrixJob(tutorial="", suites=suites, case_count=0)] + + +def _tutorial_from_suite_name(suite_name: str) -> str: + if "_" in suite_name: + return suite_name.split("_", 1)[0] + return suite_name + + +def _is_expandable_meta_suite(test_suites: dict, suite_name: str) -> bool: + if suite_name not in test_suites: + return False + suite_def = test_suites[suite_name] + return bool(suite_def.get("tutorials") or suite_def.get("external")) + + +def merge_matrix_jobs(jobs: List[MatrixJob]) -> List[MatrixJob]: + """Merge jobs that target the same tutorial, preserving suite order.""" + grouped_suites: Dict[str, List[str]] = defaultdict(list) + for job in jobs: + for suite_name in job.suites.split(","): + if suite_name: + grouped_suites[job.tutorial].append(suite_name) + + merged: List[MatrixJob] = [] + for tutorial in sorted(grouped_suites): + unique_suites: List[str] = [] + seen: set[str] = set() + for suite_name in grouped_suites[tutorial]: + if suite_name in seen: + continue + seen.add(suite_name) + unique_suites.append(suite_name) + merged.append( + MatrixJob( + tutorial=tutorial, + suites=",".join(unique_suites), + case_count=len(unique_suites), + ) + ) + return merged + + +def build_matrix( + suites: str, + parallel: bool, + tests_yaml: Path | None = None, +) -> List[MatrixJob]: + """Build a GitHub Actions matrix for serial or per-tutorial parallel runs.""" + if not parallel: + return build_serial_matrix(suites) + + test_suites = load_test_suites(tests_yaml) + suite_names = [part.strip() for part in suites.split(",") if part.strip()] + if not suite_names: + raise KeyError("No test suites specified") + + matrix_jobs: List[MatrixJob] = [] + for suite_name in suite_names: + if _is_expandable_meta_suite(test_suites, suite_name): + matrix_jobs.extend(build_parallel_matrix(suite_name, tests_yaml)) + continue + if suite_name in test_suites: + matrix_jobs.append( + MatrixJob( + tutorial=_tutorial_from_suite_name(suite_name), + suites=suite_name, + case_count=1, + ) + ) + continue + raise KeyError(f"Unknown test suite: {suite_name}") + + if len(suite_names) == 1: + return matrix_jobs + return merge_matrix_jobs(matrix_jobs) + + def build_parallel_matrix( source_suite_name: str = "release", tests_yaml: Path | None = None, @@ -140,3 +220,7 @@ def matrix_jobs_for_github( {"tutorial": job.tutorial, "suites": job.suites} for job in jobs ] + + +def total_case_count(matrix_jobs: List[MatrixJob]) -> int: + return sum(job.case_count for job in matrix_jobs) diff --git a/tests/prepare_matrix.py b/tests/prepare_matrix.py index 3e6d1ec8d..b999d573f 100644 --- a/tests/prepare_matrix.py +++ b/tests/prepare_matrix.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -"""Emit a GitHub Actions matrix for parallel release system tests.""" +"""Emit a GitHub Actions matrix for system test workflows.""" from __future__ import annotations @@ -8,42 +8,60 @@ import os import sys -from parallel_matrix import build_parallel_matrix, matrix_jobs_for_github +from parallel_matrix import build_matrix, matrix_jobs_for_github -def _write_github_step_summary(matrix_jobs, source_suite: str) -> None: +def _write_github_step_summary(matrix_jobs, suites: str, parallel: bool) -> None: summary_path = os.environ.get("GITHUB_STEP_SUMMARY") if not summary_path: return - total_cases = sum(job.case_count for job in matrix_jobs) job_count = len(matrix_jobs) + total_cases = sum(job.case_count for job in matrix_jobs) with open(summary_path, "w", encoding="utf-8") as summary_file: - print("## Parallel test matrix\n", file=summary_file) - print( - f"Next, **{job_count}** parallel jobs will run on the " - f"self-hosted runner(s), one per tutorial.\n", - file=summary_file, - ) - print(f"- Source suite: `{source_suite}`", file=summary_file) - print(f"- Tutorial jobs: **{job_count}**", file=summary_file) - print(f"- Total cases: **{total_cases}**\n", file=summary_file) - print("| # | Tutorial | Cases |", file=summary_file) - print("| - | --- | ---: |", file=summary_file) - for index, job in enumerate(matrix_jobs, start=1): + print("## System test matrix\n", file=summary_file) + if parallel: print( - f"| {index} | `{job.tutorial}` | {job.case_count} |", + f"Next, **{job_count}** jobs will run on the " + f"self-hosted runner(s), split by tutorial.\n", file=summary_file, ) + else: + print( + f"Next, **{job_count}** serial job will run on the " + f"self-hosted runner(s).\n", + file=summary_file, + ) + print(f"- Suites: `{suites}`", file=summary_file) + print(f"- Mode: `{'parallel' if parallel else 'serial'}`", file=summary_file) + print(f"- Jobs: **{job_count}**", file=summary_file) + if parallel: + print(f"- Total cases: **{total_cases}**\n", file=summary_file) + print("| # | Tutorial | Cases |", file=summary_file) + print("| - | --- | ---: |", file=summary_file) + for index, job in enumerate(matrix_jobs, start=1): + label = job.tutorial or suites + print( + f"| {index} | `{label}` | {job.case_count} |", + file=summary_file, + ) def main() -> int: parser = argparse.ArgumentParser( - description="Build a per-tutorial matrix from a meta test suite.") + description="Build a system test matrix from tests.yaml.") parser.add_argument( + "--suites", "--source-suite", + dest="suites", default="release", - help="Meta suite to expand (default: release).", + help="Suite or comma-separated suites to run (default: release).", + ) + parser.add_argument( + "--mode", + choices=["serial", "parallel"], + default="parallel", + help="serial: one matrix job; parallel: split expandable suites per tutorial.", ) parser.add_argument( "--format", @@ -53,24 +71,25 @@ def main() -> int: ) args = parser.parse_args() + parallel = args.mode == "parallel" try: - matrix_jobs = build_parallel_matrix(args.source_suite) + matrix_jobs = build_matrix(args.suites, parallel=parallel) except (KeyError, FileNotFoundError) as error: print(error, file=sys.stderr) return 1 - _write_github_step_summary(matrix_jobs, args.source_suite) + _write_github_step_summary(matrix_jobs, args.suites, parallel) if args.format == "github": - print(json.dumps(matrix_jobs_for_github( - args.source_suite, matrix_jobs=matrix_jobs))) + print(json.dumps(matrix_jobs_for_github(matrix_jobs=matrix_jobs))) else: total_cases = sum(job.case_count for job in matrix_jobs) print( - f"Matrix for {args.source_suite}: " - f"{len(matrix_jobs)} tutorial jobs, {total_cases} cases") + f"Matrix for {args.suites} ({args.mode}): " + f"{len(matrix_jobs)} job(s), {total_cases} case(s)") for index, job in enumerate(matrix_jobs, start=1): - print(f" {index}. {job.tutorial}: {job.case_count} case(s)") + label = job.tutorial or "(serial)" + print(f" {index}. {label}: {job.case_count} case(s)") return 0 diff --git a/tests/test_parallel_matrix.py b/tests/test_parallel_matrix.py new file mode 100644 index 000000000..b0d618710 --- /dev/null +++ b/tests/test_parallel_matrix.py @@ -0,0 +1,117 @@ +#!/usr/bin/env python3 +"""Tests for system test matrix generation.""" + +from __future__ import annotations + +import json +import subprocess +import sys +import unittest +from pathlib import Path + +from parallel_matrix import ( + build_matrix, + build_parallel_matrix, + build_serial_matrix, + matrix_jobs_for_github, + total_case_count, +) + +TESTS_DIR = Path(__file__).resolve().parent + + +class ParallelMatrixTests(unittest.TestCase): + def test_serial_release_is_single_job(self) -> None: + jobs = build_matrix("release", parallel=False) + self.assertEqual(len(jobs), 1) + self.assertEqual(jobs[0].suites, "release") + self.assertEqual(jobs[0].tutorial, "") + + def test_serial_custom_suites_passthrough(self) -> None: + jobs = build_matrix("quickstart,elastic-tube-1d", parallel=False) + self.assertEqual(len(jobs), 1) + self.assertEqual(jobs[0].suites, "quickstart,elastic-tube-1d") + + def test_parallel_release_splits_by_tutorial(self) -> None: + jobs = build_matrix("release", parallel=True) + self.assertGreater(len(jobs), 1) + tutorials = {job.tutorial for job in jobs} + self.assertIn("elastic-tube-1d", tutorials) + self.assertEqual(total_case_count(jobs), 73) + + def test_parallel_quickstart_is_single_tutorial_job(self) -> None: + jobs = build_matrix("quickstart", parallel=True) + self.assertEqual(len(jobs), 1) + self.assertEqual(jobs[0].tutorial, "quickstart") + self.assertEqual(jobs[0].case_count, 1) + + def test_parallel_elastic_tube_meta_suite(self) -> None: + jobs = build_matrix("elastic-tube-1d", parallel=True) + self.assertEqual(len(jobs), 1) + self.assertEqual(jobs[0].tutorial, "elastic-tube-1d") + self.assertEqual(jobs[0].case_count, 6) + + def test_parallel_unknown_suite_raises(self) -> None: + with self.assertRaises(KeyError): + build_matrix("does-not-exist", parallel=True) + + def test_github_output_is_valid_json(self) -> None: + jobs = build_parallel_matrix("release") + payload = matrix_jobs_for_github(matrix_jobs=jobs) + encoded = json.dumps(payload) + decoded = json.loads(encoded) + self.assertEqual(len(decoded), len(jobs)) + self.assertIn("tutorial", decoded[0]) + self.assertIn("suites", decoded[0]) + + def test_prepare_matrix_cli_serial(self) -> None: + result = subprocess.run( + [ + sys.executable, + "prepare_matrix.py", + "--suites", + "release", + "--mode", + "serial", + "--format", + "github", + ], + cwd=TESTS_DIR, + check=True, + capture_output=True, + text=True, + ) + payload = json.loads(result.stdout.strip()) + self.assertEqual(payload, [{"tutorial": "", "suites": "release"}]) + + def test_prepare_matrix_cli_parallel_release(self) -> None: + result = subprocess.run( + [ + sys.executable, + "prepare_matrix.py", + "--suites", + "release", + "--mode", + "parallel", + "--format", + "github", + ], + cwd=TESTS_DIR, + check=True, + capture_output=True, + text=True, + ) + payload = json.loads(result.stdout.strip()) + self.assertGreater(len(payload), 1) + self.assertEqual( + sum(len(entry["suites"].split(",")) for entry in payload), + 73, + ) + + def test_build_serial_matrix_helper(self) -> None: + jobs = build_serial_matrix("extra") + self.assertEqual(jobs[0].suites, "extra") + + +if __name__ == "__main__": + unittest.main() From 42bf5629e2d31a4ac2dcc2b3daeeb20e370d3657 Mon Sep 17 00:00:00 2001 From: PranjalManhgaye Date: Sun, 25 Oct 2026 11:54:57 +0530 Subject: [PATCH 3/7] Drop unit tests to keep PR scope minimal Matrix generation is covered by manual workflow runs on the VM. --- tests/test_parallel_matrix.py | 117 ---------------------------------- 1 file changed, 117 deletions(-) delete mode 100644 tests/test_parallel_matrix.py diff --git a/tests/test_parallel_matrix.py b/tests/test_parallel_matrix.py deleted file mode 100644 index b0d618710..000000000 --- a/tests/test_parallel_matrix.py +++ /dev/null @@ -1,117 +0,0 @@ -#!/usr/bin/env python3 -"""Tests for system test matrix generation.""" - -from __future__ import annotations - -import json -import subprocess -import sys -import unittest -from pathlib import Path - -from parallel_matrix import ( - build_matrix, - build_parallel_matrix, - build_serial_matrix, - matrix_jobs_for_github, - total_case_count, -) - -TESTS_DIR = Path(__file__).resolve().parent - - -class ParallelMatrixTests(unittest.TestCase): - def test_serial_release_is_single_job(self) -> None: - jobs = build_matrix("release", parallel=False) - self.assertEqual(len(jobs), 1) - self.assertEqual(jobs[0].suites, "release") - self.assertEqual(jobs[0].tutorial, "") - - def test_serial_custom_suites_passthrough(self) -> None: - jobs = build_matrix("quickstart,elastic-tube-1d", parallel=False) - self.assertEqual(len(jobs), 1) - self.assertEqual(jobs[0].suites, "quickstart,elastic-tube-1d") - - def test_parallel_release_splits_by_tutorial(self) -> None: - jobs = build_matrix("release", parallel=True) - self.assertGreater(len(jobs), 1) - tutorials = {job.tutorial for job in jobs} - self.assertIn("elastic-tube-1d", tutorials) - self.assertEqual(total_case_count(jobs), 73) - - def test_parallel_quickstart_is_single_tutorial_job(self) -> None: - jobs = build_matrix("quickstart", parallel=True) - self.assertEqual(len(jobs), 1) - self.assertEqual(jobs[0].tutorial, "quickstart") - self.assertEqual(jobs[0].case_count, 1) - - def test_parallel_elastic_tube_meta_suite(self) -> None: - jobs = build_matrix("elastic-tube-1d", parallel=True) - self.assertEqual(len(jobs), 1) - self.assertEqual(jobs[0].tutorial, "elastic-tube-1d") - self.assertEqual(jobs[0].case_count, 6) - - def test_parallel_unknown_suite_raises(self) -> None: - with self.assertRaises(KeyError): - build_matrix("does-not-exist", parallel=True) - - def test_github_output_is_valid_json(self) -> None: - jobs = build_parallel_matrix("release") - payload = matrix_jobs_for_github(matrix_jobs=jobs) - encoded = json.dumps(payload) - decoded = json.loads(encoded) - self.assertEqual(len(decoded), len(jobs)) - self.assertIn("tutorial", decoded[0]) - self.assertIn("suites", decoded[0]) - - def test_prepare_matrix_cli_serial(self) -> None: - result = subprocess.run( - [ - sys.executable, - "prepare_matrix.py", - "--suites", - "release", - "--mode", - "serial", - "--format", - "github", - ], - cwd=TESTS_DIR, - check=True, - capture_output=True, - text=True, - ) - payload = json.loads(result.stdout.strip()) - self.assertEqual(payload, [{"tutorial": "", "suites": "release"}]) - - def test_prepare_matrix_cli_parallel_release(self) -> None: - result = subprocess.run( - [ - sys.executable, - "prepare_matrix.py", - "--suites", - "release", - "--mode", - "parallel", - "--format", - "github", - ], - cwd=TESTS_DIR, - check=True, - capture_output=True, - text=True, - ) - payload = json.loads(result.stdout.strip()) - self.assertGreater(len(payload), 1) - self.assertEqual( - sum(len(entry["suites"].split(",")) for entry in payload), - 73, - ) - - def test_build_serial_matrix_helper(self) -> None: - jobs = build_serial_matrix("extra") - self.assertEqual(jobs[0].suites, "extra") - - -if __name__ == "__main__": - unittest.main() From f2cf9adcc620a3ab52afafd14e694d6733081491 Mon Sep 17 00:00:00 2001 From: PranjalManhgaye Date: Mon, 26 Oct 2026 14:26:33 +0530 Subject: [PATCH 4/7] Address review: merge matrix logic into prepare_matrix.py Combine parallel_matrix.py into prepare_matrix.py with clearer docs, unified suite lookup, and simplified step summary wording. --- .github/workflows/system-tests-manual.yml | 2 +- tests/README.md | 2 +- tests/parallel_matrix.py | 226 --------------------- tests/prepare_matrix.py | 234 ++++++++++++++++++++-- 4 files changed, 222 insertions(+), 242 deletions(-) delete mode 100644 tests/parallel_matrix.py diff --git a/.github/workflows/system-tests-manual.yml b/.github/workflows/system-tests-manual.yml index b90e5b77f..f6630efed 100644 --- a/.github/workflows/system-tests-manual.yml +++ b/.github/workflows/system-tests-manual.yml @@ -9,7 +9,7 @@ on: required: true type: string parallel: - description: 'Split expandable suites into per-tutorial matrix jobs' + description: 'Split into per-tutorial parallel jobs' default: false type: boolean system_tests_branch: diff --git a/tests/README.md b/tests/README.md index c91522f4c..13ce23c42 100644 --- a/tests/README.md +++ b/tests/README.md @@ -89,7 +89,7 @@ There are also some auxiliary scripts: - `print_test_suites.py`: Print all test suites defined in `tests.yaml` - `print_case_combinations.py`: Print all possible combinations of participants in a tutorial, using its `metadata.yaml`. -- `prepare_matrix.py`: Build the serial or per-tutorial matrix used by system test workflows. +- `prepare_matrix.py`: Build the serial or per-tutorial matrix from `tests.yaml` for system test workflows. ## Understanding the logs diff --git a/tests/parallel_matrix.py b/tests/parallel_matrix.py deleted file mode 100644 index cc68c2e1c..000000000 --- a/tests/parallel_matrix.py +++ /dev/null @@ -1,226 +0,0 @@ -"""Build a per-tutorial parallel matrix from a meta test suite in tests.yaml.""" - -from __future__ import annotations - -from collections import defaultdict -from dataclasses import dataclass -from pathlib import Path -from typing import Dict, List, Tuple - -import yaml - -from paths import PRECICE_TESTS_DIR - -CaseKey = Tuple[str, Tuple[str, ...]] - - -@dataclass(frozen=True) -class MatrixJob: - tutorial: str - suites: str - case_count: int - - -def _tests_yaml_path(tests_yaml: Path | None = None) -> Path: - return tests_yaml if tests_yaml is not None else PRECICE_TESTS_DIR / "tests.yaml" - - -def load_test_suites(tests_yaml: Path | None = None) -> dict: - with open(_tests_yaml_path(tests_yaml), "r", encoding="utf-8") as handle: - data = yaml.safe_load(handle) - return data["test_suites"] - - -def build_case_suite_lookup(test_suites: dict) -> Dict[CaseKey, str]: - """Map (path, case_combination) to the runnable per-case suite name.""" - candidates: Dict[CaseKey, set[str]] = defaultdict(set) - for suite_name, suite_def in test_suites.items(): - tutorials = suite_def.get("tutorials", []) - if len(tutorials) != 1: - continue - entry = tutorials[0] - key = (entry["path"], tuple(entry["case_combination"])) - candidates[key].add(suite_name) - - lookup: Dict[CaseKey, str] = {} - for key, suite_names in candidates.items(): - path, combo = key - canonical = f"{path}_{'_'.join(combo)}" - if canonical in suite_names: - lookup[key] = canonical - else: - lookup[key] = sorted(suite_names)[0] - return lookup - - -def build_external_suite_lookup(test_suites: dict) -> Dict[CaseKey, str]: - """Map external cases to the suite that defines them (e.g. mixedbc).""" - lookup: Dict[CaseKey, str] = {} - for suite_name, suite_def in test_suites.items(): - external = suite_def.get("external", []) - if len(external) != 1: - continue - entry = external[0] - lookup[(entry["path"], tuple(entry["case_combination"]))] = suite_name - return lookup - - -def _release_case_entries( - source_suite: dict, - case_lookup: Dict[CaseKey, str], - external_lookup: Dict[CaseKey, str], -) -> List[Tuple[str, str]]: - """Return (tutorial_path, case_suite_name) pairs for a meta suite definition.""" - entries: List[Tuple[str, str]] = [] - - def add_entry(case_def: dict) -> None: - case_key: CaseKey = (case_def["path"], tuple(case_def["case_combination"])) - tutorial = case_def["path"] - if case_key in case_lookup: - suite_name = case_lookup[case_key] - elif case_key in external_lookup: - suite_name = external_lookup[case_key] - else: - raise KeyError( - f"No runnable suite found for {tutorial} " - f"{case_def['case_combination']}") - entries.append((tutorial, suite_name)) - - for case_def in source_suite.get("tutorials", []): - add_entry(case_def) - for case_def in source_suite.get("external", []): - add_entry(case_def) - - return entries - - -def build_serial_matrix(suites: str) -> List[MatrixJob]: - """Return a single matrix job that runs the given suite selection as-is.""" - return [MatrixJob(tutorial="", suites=suites, case_count=0)] - - -def _tutorial_from_suite_name(suite_name: str) -> str: - if "_" in suite_name: - return suite_name.split("_", 1)[0] - return suite_name - - -def _is_expandable_meta_suite(test_suites: dict, suite_name: str) -> bool: - if suite_name not in test_suites: - return False - suite_def = test_suites[suite_name] - return bool(suite_def.get("tutorials") or suite_def.get("external")) - - -def merge_matrix_jobs(jobs: List[MatrixJob]) -> List[MatrixJob]: - """Merge jobs that target the same tutorial, preserving suite order.""" - grouped_suites: Dict[str, List[str]] = defaultdict(list) - for job in jobs: - for suite_name in job.suites.split(","): - if suite_name: - grouped_suites[job.tutorial].append(suite_name) - - merged: List[MatrixJob] = [] - for tutorial in sorted(grouped_suites): - unique_suites: List[str] = [] - seen: set[str] = set() - for suite_name in grouped_suites[tutorial]: - if suite_name in seen: - continue - seen.add(suite_name) - unique_suites.append(suite_name) - merged.append( - MatrixJob( - tutorial=tutorial, - suites=",".join(unique_suites), - case_count=len(unique_suites), - ) - ) - return merged - - -def build_matrix( - suites: str, - parallel: bool, - tests_yaml: Path | None = None, -) -> List[MatrixJob]: - """Build a GitHub Actions matrix for serial or per-tutorial parallel runs.""" - if not parallel: - return build_serial_matrix(suites) - - test_suites = load_test_suites(tests_yaml) - suite_names = [part.strip() for part in suites.split(",") if part.strip()] - if not suite_names: - raise KeyError("No test suites specified") - - matrix_jobs: List[MatrixJob] = [] - for suite_name in suite_names: - if _is_expandable_meta_suite(test_suites, suite_name): - matrix_jobs.extend(build_parallel_matrix(suite_name, tests_yaml)) - continue - if suite_name in test_suites: - matrix_jobs.append( - MatrixJob( - tutorial=_tutorial_from_suite_name(suite_name), - suites=suite_name, - case_count=1, - ) - ) - continue - raise KeyError(f"Unknown test suite: {suite_name}") - - if len(suite_names) == 1: - return matrix_jobs - return merge_matrix_jobs(matrix_jobs) - - -def build_parallel_matrix( - source_suite_name: str = "release", - tests_yaml: Path | None = None, -) -> List[MatrixJob]: - test_suites = load_test_suites(tests_yaml) - if source_suite_name not in test_suites: - raise KeyError(f"Unknown test suite: {source_suite_name}") - - case_lookup = build_case_suite_lookup(test_suites) - external_lookup = build_external_suite_lookup(test_suites) - release_entries = _release_case_entries( - test_suites[source_suite_name], - case_lookup, - external_lookup, - ) - - grouped: Dict[str, List[str]] = defaultdict(list) - for tutorial, case_suite in release_entries: - grouped[tutorial].append(case_suite) - - matrix_jobs: List[MatrixJob] = [] - for tutorial in sorted(grouped): - case_suites = grouped[tutorial] - matrix_jobs.append( - MatrixJob( - tutorial=tutorial, - suites=",".join(case_suites), - case_count=len(case_suites), - ) - ) - return matrix_jobs - - -def matrix_jobs_for_github( - source_suite_name: str = "release", - tests_yaml: Path | None = None, - matrix_jobs: List[MatrixJob] | None = None, -) -> List[dict]: - jobs = ( - matrix_jobs - if matrix_jobs is not None - else build_parallel_matrix(source_suite_name, tests_yaml)) - return [ - {"tutorial": job.tutorial, "suites": job.suites} - for job in jobs - ] - - -def total_case_count(matrix_jobs: List[MatrixJob]) -> int: - return sum(job.case_count for job in matrix_jobs) diff --git a/tests/prepare_matrix.py b/tests/prepare_matrix.py index b999d573f..b460649d2 100644 --- a/tests/prepare_matrix.py +++ b/tests/prepare_matrix.py @@ -1,5 +1,14 @@ #!/usr/bin/env python3 -"""Emit a GitHub Actions matrix for system test workflows.""" +"""Build a GitHub Actions matrix for system test workflows. + +Reads suite definitions from tests.yaml and emits either: +- serial mode: one job running the requested suite string as-is +- parallel mode: one job per tutorial, with comma-separated per-case suite names + +Naming convention for per-case suites: + __ +Example: quickstart_fluid-openfoam_solid-cpp (tutorial: quickstart) +""" from __future__ import annotations @@ -7,42 +16,237 @@ import json import os import sys +from collections import defaultdict +from dataclasses import dataclass +from pathlib import Path +from typing import Dict, List, Tuple + +import yaml + +from paths import PRECICE_TESTS_DIR + +CaseKey = Tuple[str, Tuple[str, ...]] + + +@dataclass(frozen=True) +class MatrixJob: + tutorial: str + suites: str + case_count: int + + +def load_test_suites(tests_yaml: Path | None = None) -> dict: + path = tests_yaml if tests_yaml is not None else PRECICE_TESTS_DIR / "tests.yaml" + with open(path, "r", encoding="utf-8") as handle: + data = yaml.safe_load(handle) + return data["test_suites"] + + +def build_suite_lookup(test_suites: dict) -> Dict[CaseKey, str]: + """Map a tutorial case to its runnable suite name in tests.yaml. + + Only considers suite definitions with exactly one tutorial or external + entry (per-case suites). Meta suites such as release list many cases and + are skipped here. + + Example: + ("quickstart", ("fluid-openfoam", "solid-cpp")) + -> "quickstart_fluid-openfoam_solid-cpp" + """ + candidates: Dict[CaseKey, set[str]] = defaultdict(set) + for suite_name, suite_def in test_suites.items(): + for section in ("tutorials", "external"): + entries = suite_def.get(section, []) + if len(entries) != 1: + continue + entry = entries[0] + key = (entry["path"], tuple(entry["case_combination"])) + candidates[key].add(suite_name) + + lookup: Dict[CaseKey, str] = {} + for key, suite_names in candidates.items(): + path, combo = key + canonical = f"{path}_{'_'.join(combo)}" + if canonical in suite_names: + lookup[key] = canonical + else: + lookup[key] = sorted(suite_names)[0] + return lookup + + +def _meta_suite_case_entries( + meta_suite: dict, + suite_lookup: Dict[CaseKey, str], +) -> List[Tuple[str, str]]: + """Expand a meta suite (e.g. release) into (tutorial, per-case suite) pairs.""" + entries: List[Tuple[str, str]] = [] + + for section in ("tutorials", "external"): + for case_def in meta_suite.get(section, []): + case_key: CaseKey = ( + case_def["path"], + tuple(case_def["case_combination"]), + ) + tutorial = case_def["path"] + if case_key not in suite_lookup: + raise KeyError( + f"No runnable suite found for {tutorial} " + f"{case_def['case_combination']}") + entries.append((tutorial, suite_lookup[case_key])) + + return entries + + +def _tutorial_from_suite_name(suite_name: str) -> str: + """Derive tutorial path from a per-case suite name. + + Example: channel-transport_fluid-openfoam_transport-openfoam -> channel-transport + """ + if "_" in suite_name: + return suite_name.split("_", 1)[0] + return suite_name + + +def _is_expandable_meta_suite(test_suites: dict, suite_name: str) -> bool: + if suite_name not in test_suites: + return False + suite_def = test_suites[suite_name] + return bool(suite_def.get("tutorials") or suite_def.get("external")) + + +def _build_serial_matrix(suites: str) -> List[MatrixJob]: + return [MatrixJob(tutorial="", suites=suites, case_count=0)] + + +def _expand_meta_suite( + meta_suite_name: str, + tests_yaml: Path | None = None, +) -> List[MatrixJob]: + """Split one meta suite into per-tutorial matrix jobs.""" + test_suites = load_test_suites(tests_yaml) + if meta_suite_name not in test_suites: + raise KeyError(f"Unknown test suite: {meta_suite_name}") + + suite_lookup = build_suite_lookup(test_suites) + case_entries = _meta_suite_case_entries( + test_suites[meta_suite_name], + suite_lookup, + ) + + grouped: Dict[str, List[str]] = defaultdict(list) + for tutorial, case_suite in case_entries: + grouped[tutorial].append(case_suite) + + return [ + MatrixJob( + tutorial=tutorial, + suites=",".join(case_suites), + case_count=len(case_suites), + ) + for tutorial in sorted(grouped) + for case_suites in [grouped[tutorial]] + ] + + +def _merge_matrix_jobs(jobs: List[MatrixJob]) -> List[MatrixJob]: + """Merge jobs for the same tutorial when multiple meta suites were requested.""" + grouped_suites: Dict[str, List[str]] = defaultdict(list) + for job in jobs: + for suite_name in job.suites.split(","): + if suite_name: + grouped_suites[job.tutorial].append(suite_name) + + merged: List[MatrixJob] = [] + for tutorial in sorted(grouped_suites): + unique_suites: List[str] = [] + seen: set[str] = set() + for suite_name in grouped_suites[tutorial]: + if suite_name in seen: + continue + seen.add(suite_name) + unique_suites.append(suite_name) + merged.append( + MatrixJob( + tutorial=tutorial, + suites=",".join(unique_suites), + case_count=len(unique_suites), + ) + ) + return merged + + +def build_matrix( + suites: str, + parallel: bool, + tests_yaml: Path | None = None, +) -> List[MatrixJob]: + if not parallel: + return _build_serial_matrix(suites) + + test_suites = load_test_suites(tests_yaml) + suite_names = [part.strip() for part in suites.split(",") if part.strip()] + if not suite_names: + raise KeyError("No test suites specified") + + matrix_jobs: List[MatrixJob] = [] + for suite_name in suite_names: + if _is_expandable_meta_suite(test_suites, suite_name): + matrix_jobs.extend(_expand_meta_suite(suite_name, tests_yaml)) + continue + if suite_name in test_suites: + matrix_jobs.append( + MatrixJob( + tutorial=_tutorial_from_suite_name(suite_name), + suites=suite_name, + case_count=1, + ) + ) + continue + raise KeyError(f"Unknown test suite: {suite_name}") + + if len(suite_names) == 1: + return matrix_jobs + return _merge_matrix_jobs(matrix_jobs) + -from parallel_matrix import build_matrix, matrix_jobs_for_github +def matrix_jobs_for_github(matrix_jobs: List[MatrixJob]) -> List[dict]: + return [{"tutorial": job.tutorial, "suites": job.suites} for job in matrix_jobs] -def _write_github_step_summary(matrix_jobs, suites: str, parallel: bool) -> None: +def _write_github_step_summary( + matrix_jobs: List[MatrixJob], + suites: str, + parallel: bool, +) -> None: summary_path = os.environ.get("GITHUB_STEP_SUMMARY") if not summary_path: return job_count = len(matrix_jobs) - total_cases = sum(job.case_count for job in matrix_jobs) with open(summary_path, "w", encoding="utf-8") as summary_file: print("## System test matrix\n", file=summary_file) if parallel: print( - f"Next, **{job_count}** jobs will run on the " - f"self-hosted runner(s), split by tutorial.\n", + f"Next, **{job_count}** jobs will run, split by tutorial.\n", file=summary_file, ) else: + assert job_count == 1 print( - f"Next, **{job_count}** serial job will run on the " - f"self-hosted runner(s).\n", + "Next, all test cases will be executed as one serial job.\n", file=summary_file, ) print(f"- Suites: `{suites}`", file=summary_file) print(f"- Mode: `{'parallel' if parallel else 'serial'}`", file=summary_file) print(f"- Jobs: **{job_count}**", file=summary_file) if parallel: + total_cases = sum(job.case_count for job in matrix_jobs) print(f"- Total cases: **{total_cases}**\n", file=summary_file) print("| # | Tutorial | Cases |", file=summary_file) print("| - | --- | ---: |", file=summary_file) for index, job in enumerate(matrix_jobs, start=1): - label = job.tutorial or suites print( - f"| {index} | `{label}` | {job.case_count} |", + f"| {index} | `{job.tutorial}` | {job.case_count} |", file=summary_file, ) @@ -51,9 +255,8 @@ def main() -> int: parser = argparse.ArgumentParser( description="Build a system test matrix from tests.yaml.") parser.add_argument( + "-s", "--suites", - "--source-suite", - dest="suites", default="release", help="Suite or comma-separated suites to run (default: release).", ) @@ -67,7 +270,10 @@ def main() -> int: "--format", choices=["github", "pretty"], default="pretty", - help="Output format (github: JSON for fromJson()).", + help=( + "github: JSON for workflow fromJson(); " + "pretty: human-readable output for local runs." + ), ) args = parser.parse_args() @@ -81,7 +287,7 @@ def main() -> int: _write_github_step_summary(matrix_jobs, args.suites, parallel) if args.format == "github": - print(json.dumps(matrix_jobs_for_github(matrix_jobs=matrix_jobs))) + print(json.dumps(matrix_jobs_for_github(matrix_jobs))) else: total_cases = sum(job.case_count for job in matrix_jobs) print( From edf4c0d2d525cb042da4a511fe24304ce91f6b43 Mon Sep 17 00:00:00 2001 From: PranjalManhgaye Date: Mon, 26 Oct 2026 15:22:09 +0530 Subject: [PATCH 5/7] Document local usage of prepare_matrix.py in tests README Clarify that the script only previews the GitHub Actions matrix and does not run tests, with serial/parallel modes and format options. --- tests/README.md | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/tests/README.md b/tests/README.md index 13ce23c42..9c96cec00 100644 --- a/tests/README.md +++ b/tests/README.md @@ -89,7 +89,25 @@ There are also some auxiliary scripts: - `print_test_suites.py`: Print all test suites defined in `tests.yaml` - `print_case_combinations.py`: Print all possible combinations of participants in a tutorial, using its `metadata.yaml`. -- `prepare_matrix.py`: Build the serial or per-tutorial matrix from `tests.yaml` for system test workflows. +- `prepare_matrix.py`: Preview how the manual and nightly workflows split selected suites into GitHub Actions matrix jobs (see below). + +### Previewing the GitHub Actions matrix (`prepare_matrix.py`) + +The [System tests (manual)](https://github.com/precice/tutorials/actions/workflows/system-tests-manual.yml) and [System tests (nightly)](https://github.com/precice/tutorials/actions/workflows/system-tests-nightly.yml) workflows run `prepare_matrix.py` before triggering `system-tests.yml`. The script reads `tests.yaml` and prints the resulting matrix. It does **not** build Docker images, run simulations, or start any tests. + +Run it from the `tests/` directory (after installing `requirements.txt` as above): + +```bash +python prepare_matrix.py --suites release --mode serial +python prepare_matrix.py --suites release --mode parallel +``` + +- **`--mode serial`**: One matrix job with the suite string passed through unchanged (the default for manual runs unless `parallel` is enabled). +- **`--mode parallel`**: Expand meta suites such as `release` into one job per tutorial, with comma-separated per-case suite names (for example `quickstart_fluid-openfoam_solid-cpp`). +- **`--format pretty`** (default): Human-readable summary of job count and tutorials. +- **`--format github`**: JSON array for `fromJson()` in the workflow (`[{"tutorial": "...", "suites": "..."}, ...]`). + +Use this locally to check how a suite change affects CI parallelism, or to verify that a new per-case suite name resolves correctly, without dispatching a workflow. To actually execute tests, use `systemtests.py` as described above. ## Understanding the logs From 4360a708ac4cadbb3049806a63941db4e75ed656 Mon Sep 17 00:00:00 2001 From: Pranjal Date: Mon, 26 Oct 2026 15:24:51 +0530 Subject: [PATCH 6/7] Update tests/prepare_matrix.py Co-authored-by: Gerasimos Chourdakis --- tests/prepare_matrix.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/prepare_matrix.py b/tests/prepare_matrix.py index b460649d2..de3d4eba8 100644 --- a/tests/prepare_matrix.py +++ b/tests/prepare_matrix.py @@ -98,7 +98,7 @@ def _meta_suite_case_entries( def _tutorial_from_suite_name(suite_name: str) -> str: - """Derive tutorial path from a per-case suite name. + """Derive tutorial name from a per-case suite name. Example: channel-transport_fluid-openfoam_transport-openfoam -> channel-transport """ From b9266d35f037a99b1936fd90a04cf3aadffd2603 Mon Sep 17 00:00:00 2001 From: Gerasimos Chourdakis Date: Mon, 26 Oct 2026 11:14:46 +0100 Subject: [PATCH 7/7] Simplify documentation --- tests/README.md | 24 +++--------------------- 1 file changed, 3 insertions(+), 21 deletions(-) diff --git a/tests/README.md b/tests/README.md index 9c96cec00..fd2b8d040 100644 --- a/tests/README.md +++ b/tests/README.md @@ -87,27 +87,9 @@ To clean up at the end, you might want to run a `docker system prune -a` and rem There are also some auxiliary scripts: -- `print_test_suites.py`: Print all test suites defined in `tests.yaml` -- `print_case_combinations.py`: Print all possible combinations of participants in a tutorial, using its `metadata.yaml`. -- `prepare_matrix.py`: Preview how the manual and nightly workflows split selected suites into GitHub Actions matrix jobs (see below). - -### Previewing the GitHub Actions matrix (`prepare_matrix.py`) - -The [System tests (manual)](https://github.com/precice/tutorials/actions/workflows/system-tests-manual.yml) and [System tests (nightly)](https://github.com/precice/tutorials/actions/workflows/system-tests-nightly.yml) workflows run `prepare_matrix.py` before triggering `system-tests.yml`. The script reads `tests.yaml` and prints the resulting matrix. It does **not** build Docker images, run simulations, or start any tests. - -Run it from the `tests/` directory (after installing `requirements.txt` as above): - -```bash -python prepare_matrix.py --suites release --mode serial -python prepare_matrix.py --suites release --mode parallel -``` - -- **`--mode serial`**: One matrix job with the suite string passed through unchanged (the default for manual runs unless `parallel` is enabled). -- **`--mode parallel`**: Expand meta suites such as `release` into one job per tutorial, with comma-separated per-case suite names (for example `quickstart_fluid-openfoam_solid-cpp`). -- **`--format pretty`** (default): Human-readable summary of job count and tutorials. -- **`--format github`**: JSON array for `fromJson()` in the workflow (`[{"tutorial": "...", "suites": "..."}, ...]`). - -Use this locally to check how a suite change affects CI parallelism, or to verify that a new per-case suite name resolves correctly, without dispatching a workflow. To actually execute tests, use `systemtests.py` as described above. +- `print_test_suites.py`: Print all test suites defined in `tests.yaml`. No arguments are expected. +- `print_case_combinations.py`: Print all possible combinations of participants in a tutorial, using its `metadata.yaml`. No arguments are expected. +- `prepare_matrix.py`: Preview how the manual and nightly workflows split selected suites into GitHub Actions matrix jobs. Run it as `python prepare_matrix.py --suites release --mode [serial|parallel] --format [pretty|github]`. ## Understanding the logs