diff --git a/.github/workflows/system-tests-manual.yml b/.github/workflows/system-tests-manual.yml index 71a325821..f6630efed 100644 --- a/.github/workflows/system-tests-manual.yml +++ b/.github/workflows/system-tests-manual.yml @@ -8,6 +8,10 @@ on: default: 'release' required: true type: string + parallel: + description: 'Split into per-tutorial parallel jobs' + default: false + type: boolean system_tests_branch: description: 'Use the system tests from' default: 'develop' @@ -53,11 +57,42 @@ on: type: string jobs: + prepare-matrix: + name: Prepare test matrix + runs-on: ubuntu-latest + outputs: + matrix: ${{ steps.matrix.outputs.matrix }} + steps: + - name: Check out tutorials + uses: actions/checkout@v7 + with: + ref: ${{ inputs.system_tests_branch }} + - name: Install Python dependencies + run: pip install --user -r tests/requirements.txt + - name: Build matrix from selected suites + id: matrix + working-directory: tests + env: + SUITES: ${{ inputs.suites }} + PARALLEL: ${{ inputs.parallel }} + run: | + MODE=serial + if [ "$PARALLEL" = "true" ]; then + MODE=parallel + fi + python3 prepare_matrix.py --suites "$SUITES" --mode "$MODE" --format=github > matrix.json + echo "matrix=$(cat matrix.json)" >> "$GITHUB_OUTPUT" + run-system-tests: - name: Trigger system tests + name: ${{ matrix.tutorial != '' && format('Trigger system tests ({0})', matrix.tutorial) || 'Trigger system tests' }} + needs: prepare-matrix + strategy: + fail-fast: false + matrix: + include: ${{ fromJson(needs.prepare-matrix.outputs.matrix) }} uses: $/.github/workflows/system-tests.yml with: - suites: ${{ inputs.suites || 'release' }} + suites: ${{ matrix.suites }} build_args: "TUTORIALS_REF:${{ inputs.ref-tutorials }},\ PRECICE_REF:${{ inputs.ref-precice }},\ PYTHON_BINDINGS_REF:${{ inputs.ref-python-bindings }},\ @@ -73,3 +108,4 @@ jobs: SU2_ADAPTER_REF:${{ inputs.ref-su2-adapter }}" system_tests_branch: ${{ inputs.system_tests_branch || 'develop' }} log_level: "INFO" + job_label: ${{ matrix.tutorial }} diff --git a/.github/workflows/system-tests-nightly.yml b/.github/workflows/system-tests-nightly.yml index 6014c4cbb..68189d66c 100644 --- a/.github/workflows/system-tests-nightly.yml +++ b/.github/workflows/system-tests-nightly.yml @@ -6,8 +6,33 @@ on: timezone: "Europe/Berlin" jobs: + prepare-matrix: + name: Prepare test matrix + runs-on: ubuntu-latest + outputs: + matrix: ${{ steps.matrix.outputs.matrix }} + steps: + - name: Check out tutorials + uses: actions/checkout@v7 + with: + ref: develop + - name: Install Python dependencies + run: pip install --user -r tests/requirements.txt + - name: Build serial matrix for release + id: matrix + working-directory: tests + run: | + python3 prepare_matrix.py --suites release --mode serial --format=github > matrix.json + echo "matrix=$(cat matrix.json)" >> "$GITHUB_OUTPUT" + run-system-tests: name: Trigger system tests + needs: prepare-matrix + strategy: + fail-fast: false + matrix: + include: ${{ fromJson(needs.prepare-matrix.outputs.matrix) }} uses: $/.github/workflows/system-tests.yml with: - suites: 'release' + suites: ${{ matrix.suites }} + job_label: ${{ matrix.tutorial }} diff --git a/.github/workflows/system-tests.yml b/.github/workflows/system-tests.yml index 83768bf64..d5aca58ba 100644 --- a/.github/workflows/system-tests.yml +++ b/.github/workflows/system-tests.yml @@ -21,8 +21,13 @@ on: description: 'TRUE or FALSE (FALSE -> Upload only when the workflow fails)' default: 'TRUE' type: string + job_label: + description: 'Optional label for job and artifact names (e.g. tutorial name)' + default: '' + type: string jobs: run_system_tests: + name: ${{ inputs.job_label != '' && format('System tests ({0})', inputs.job_label) || 'System tests' }} runs-on: [self-hosted, linux, x64, precice-tests-vm] steps: - name: Display a quick job summary @@ -90,7 +95,7 @@ jobs: id: archive-logs uses: actions/upload-artifact@v7 with: - name: system_tests_run_${{ github.run_id }}_${{ github.run_attempt }}_logs + name: system_tests_run_${{ github.run_id }}_${{ github.run_attempt }}${{ inputs.job_label != '' && format('_{0}', inputs.job_label) || '' }}_logs path: | runs/*/system-tests-build.log runs/*/system-tests-run.log @@ -103,7 +108,7 @@ jobs: id: archive-diffs uses: actions/upload-artifact@v7 with: - name: system_tests_run_${{ github.run_id }}_${{ github.run_attempt }}_diffs + name: system_tests_run_${{ github.run_id }}_${{ github.run_attempt }}${{ inputs.job_label != '' && format('_{0}', inputs.job_label) || '' }}_diffs path: | runs/*/diff-results/visualizations/**/*.png if-no-files-found: ignore @@ -121,7 +126,7 @@ jobs: if: ${{ failure() || inputs.upload_artifacts == 'TRUE' }} uses: actions/upload-artifact@v7 with: - name: system_tests_run_${{ github.run_id }}_${{ github.run_attempt }}_full + name: system_tests_run_${{ github.run_id }}_${{ github.run_attempt }}${{ inputs.job_label != '' && format('_{0}', inputs.job_label) || '' }}_full path: | runs/* !runs/tests/thirdparty diff --git a/changelog-entries/789.md b/changelog-entries/789.md new file mode 100644 index 000000000..17749a997 --- /dev/null +++ b/changelog-entries/789.md @@ -0,0 +1 @@ +- Parallelize scheduled and manual `release` system tests via a per-tutorial GitHub Actions matrix derived from `tests.yaml` [#789](https://github.com/precice/tutorials/issues/789) diff --git a/tests/README.md b/tests/README.md index a241fe86d..fd2b8d040 100644 --- a/tests/README.md +++ b/tests/README.md @@ -34,6 +34,8 @@ The `Use workflow from` is a default option of GitHub Actions that concerns the The [System tests (nightly)](https://github.com/precice/tutorials/actions/workflows/system-tests-nightly.yml) executes the `release` test suite every night. The [System tests (weekly)](https://github.com/precice/tutorials/actions/workflows/system-tests-weekly.yml) executes the `extra` test suite once per week. +The manual and nightly workflows always run a `prepare-matrix` job first. By default, suites run in a single serial matrix job. Enable the `parallel` input to split expandable suites (such as `release`) into one job per tutorial. If a parallel matrix job fails, re-run only that tutorial job from the Actions run page. + ### Running from a pull request Several repositories include a workflow that allows triggering the system tests by adding the `trigger-system-tests` label to the pull request. The event is triggered only at the moment of adding the label, so you need to remove the label and add it again if needed. @@ -83,10 +85,11 @@ This will build and connect Docker containers and run tutorials in the `runs/` d To clean up at the end, you might want to run a `docker system prune -a` and remove the `runs/` directory to save space. -There are also some auxiliary scripts (run without arguments): +There are also some auxiliary scripts: -- `print_test_suites.py`: Print all test suites defined in `tests.yaml` -- `print_case_combinations.py`: Print all possible combinations of participants in a tutorial, using its `metadata.yaml`. +- `print_test_suites.py`: Print all test suites defined in `tests.yaml`. No arguments are expected. +- `print_case_combinations.py`: Print all possible combinations of participants in a tutorial, using its `metadata.yaml`. No arguments are expected. +- `prepare_matrix.py`: Preview how the manual and nightly workflows split selected suites into GitHub Actions matrix jobs. Run it as `python prepare_matrix.py --suites release --mode [serial|parallel] --format [pretty|github]`. ## Understanding the logs diff --git a/tests/prepare_matrix.py b/tests/prepare_matrix.py new file mode 100644 index 000000000..de3d4eba8 --- /dev/null +++ b/tests/prepare_matrix.py @@ -0,0 +1,303 @@ +#!/usr/bin/env python3 +"""Build a GitHub Actions matrix for system test workflows. + +Reads suite definitions from tests.yaml and emits either: +- serial mode: one job running the requested suite string as-is +- parallel mode: one job per tutorial, with comma-separated per-case suite names + +Naming convention for per-case suites: + __ +Example: quickstart_fluid-openfoam_solid-cpp (tutorial: quickstart) +""" + +from __future__ import annotations + +import argparse +import json +import os +import sys +from collections import defaultdict +from dataclasses import dataclass +from pathlib import Path +from typing import Dict, List, Tuple + +import yaml + +from paths import PRECICE_TESTS_DIR + +CaseKey = Tuple[str, Tuple[str, ...]] + + +@dataclass(frozen=True) +class MatrixJob: + tutorial: str + suites: str + case_count: int + + +def load_test_suites(tests_yaml: Path | None = None) -> dict: + path = tests_yaml if tests_yaml is not None else PRECICE_TESTS_DIR / "tests.yaml" + with open(path, "r", encoding="utf-8") as handle: + data = yaml.safe_load(handle) + return data["test_suites"] + + +def build_suite_lookup(test_suites: dict) -> Dict[CaseKey, str]: + """Map a tutorial case to its runnable suite name in tests.yaml. + + Only considers suite definitions with exactly one tutorial or external + entry (per-case suites). Meta suites such as release list many cases and + are skipped here. + + Example: + ("quickstart", ("fluid-openfoam", "solid-cpp")) + -> "quickstart_fluid-openfoam_solid-cpp" + """ + candidates: Dict[CaseKey, set[str]] = defaultdict(set) + for suite_name, suite_def in test_suites.items(): + for section in ("tutorials", "external"): + entries = suite_def.get(section, []) + if len(entries) != 1: + continue + entry = entries[0] + key = (entry["path"], tuple(entry["case_combination"])) + candidates[key].add(suite_name) + + lookup: Dict[CaseKey, str] = {} + for key, suite_names in candidates.items(): + path, combo = key + canonical = f"{path}_{'_'.join(combo)}" + if canonical in suite_names: + lookup[key] = canonical + else: + lookup[key] = sorted(suite_names)[0] + return lookup + + +def _meta_suite_case_entries( + meta_suite: dict, + suite_lookup: Dict[CaseKey, str], +) -> List[Tuple[str, str]]: + """Expand a meta suite (e.g. release) into (tutorial, per-case suite) pairs.""" + entries: List[Tuple[str, str]] = [] + + for section in ("tutorials", "external"): + for case_def in meta_suite.get(section, []): + case_key: CaseKey = ( + case_def["path"], + tuple(case_def["case_combination"]), + ) + tutorial = case_def["path"] + if case_key not in suite_lookup: + raise KeyError( + f"No runnable suite found for {tutorial} " + f"{case_def['case_combination']}") + entries.append((tutorial, suite_lookup[case_key])) + + return entries + + +def _tutorial_from_suite_name(suite_name: str) -> str: + """Derive tutorial name from a per-case suite name. + + Example: channel-transport_fluid-openfoam_transport-openfoam -> channel-transport + """ + if "_" in suite_name: + return suite_name.split("_", 1)[0] + return suite_name + + +def _is_expandable_meta_suite(test_suites: dict, suite_name: str) -> bool: + if suite_name not in test_suites: + return False + suite_def = test_suites[suite_name] + return bool(suite_def.get("tutorials") or suite_def.get("external")) + + +def _build_serial_matrix(suites: str) -> List[MatrixJob]: + return [MatrixJob(tutorial="", suites=suites, case_count=0)] + + +def _expand_meta_suite( + meta_suite_name: str, + tests_yaml: Path | None = None, +) -> List[MatrixJob]: + """Split one meta suite into per-tutorial matrix jobs.""" + test_suites = load_test_suites(tests_yaml) + if meta_suite_name not in test_suites: + raise KeyError(f"Unknown test suite: {meta_suite_name}") + + suite_lookup = build_suite_lookup(test_suites) + case_entries = _meta_suite_case_entries( + test_suites[meta_suite_name], + suite_lookup, + ) + + grouped: Dict[str, List[str]] = defaultdict(list) + for tutorial, case_suite in case_entries: + grouped[tutorial].append(case_suite) + + return [ + MatrixJob( + tutorial=tutorial, + suites=",".join(case_suites), + case_count=len(case_suites), + ) + for tutorial in sorted(grouped) + for case_suites in [grouped[tutorial]] + ] + + +def _merge_matrix_jobs(jobs: List[MatrixJob]) -> List[MatrixJob]: + """Merge jobs for the same tutorial when multiple meta suites were requested.""" + grouped_suites: Dict[str, List[str]] = defaultdict(list) + for job in jobs: + for suite_name in job.suites.split(","): + if suite_name: + grouped_suites[job.tutorial].append(suite_name) + + merged: List[MatrixJob] = [] + for tutorial in sorted(grouped_suites): + unique_suites: List[str] = [] + seen: set[str] = set() + for suite_name in grouped_suites[tutorial]: + if suite_name in seen: + continue + seen.add(suite_name) + unique_suites.append(suite_name) + merged.append( + MatrixJob( + tutorial=tutorial, + suites=",".join(unique_suites), + case_count=len(unique_suites), + ) + ) + return merged + + +def build_matrix( + suites: str, + parallel: bool, + tests_yaml: Path | None = None, +) -> List[MatrixJob]: + if not parallel: + return _build_serial_matrix(suites) + + test_suites = load_test_suites(tests_yaml) + suite_names = [part.strip() for part in suites.split(",") if part.strip()] + if not suite_names: + raise KeyError("No test suites specified") + + matrix_jobs: List[MatrixJob] = [] + for suite_name in suite_names: + if _is_expandable_meta_suite(test_suites, suite_name): + matrix_jobs.extend(_expand_meta_suite(suite_name, tests_yaml)) + continue + if suite_name in test_suites: + matrix_jobs.append( + MatrixJob( + tutorial=_tutorial_from_suite_name(suite_name), + suites=suite_name, + case_count=1, + ) + ) + continue + raise KeyError(f"Unknown test suite: {suite_name}") + + if len(suite_names) == 1: + return matrix_jobs + return _merge_matrix_jobs(matrix_jobs) + + +def matrix_jobs_for_github(matrix_jobs: List[MatrixJob]) -> List[dict]: + return [{"tutorial": job.tutorial, "suites": job.suites} for job in matrix_jobs] + + +def _write_github_step_summary( + matrix_jobs: List[MatrixJob], + suites: str, + parallel: bool, +) -> None: + summary_path = os.environ.get("GITHUB_STEP_SUMMARY") + if not summary_path: + return + + job_count = len(matrix_jobs) + with open(summary_path, "w", encoding="utf-8") as summary_file: + print("## System test matrix\n", file=summary_file) + if parallel: + print( + f"Next, **{job_count}** jobs will run, split by tutorial.\n", + file=summary_file, + ) + else: + assert job_count == 1 + print( + "Next, all test cases will be executed as one serial job.\n", + file=summary_file, + ) + print(f"- Suites: `{suites}`", file=summary_file) + print(f"- Mode: `{'parallel' if parallel else 'serial'}`", file=summary_file) + print(f"- Jobs: **{job_count}**", file=summary_file) + if parallel: + total_cases = sum(job.case_count for job in matrix_jobs) + print(f"- Total cases: **{total_cases}**\n", file=summary_file) + print("| # | Tutorial | Cases |", file=summary_file) + print("| - | --- | ---: |", file=summary_file) + for index, job in enumerate(matrix_jobs, start=1): + print( + f"| {index} | `{job.tutorial}` | {job.case_count} |", + file=summary_file, + ) + + +def main() -> int: + parser = argparse.ArgumentParser( + description="Build a system test matrix from tests.yaml.") + parser.add_argument( + "-s", + "--suites", + default="release", + help="Suite or comma-separated suites to run (default: release).", + ) + parser.add_argument( + "--mode", + choices=["serial", "parallel"], + default="parallel", + help="serial: one matrix job; parallel: split expandable suites per tutorial.", + ) + parser.add_argument( + "--format", + choices=["github", "pretty"], + default="pretty", + help=( + "github: JSON for workflow fromJson(); " + "pretty: human-readable output for local runs." + ), + ) + args = parser.parse_args() + + parallel = args.mode == "parallel" + try: + matrix_jobs = build_matrix(args.suites, parallel=parallel) + except (KeyError, FileNotFoundError) as error: + print(error, file=sys.stderr) + return 1 + + _write_github_step_summary(matrix_jobs, args.suites, parallel) + + if args.format == "github": + print(json.dumps(matrix_jobs_for_github(matrix_jobs))) + else: + total_cases = sum(job.case_count for job in matrix_jobs) + print( + f"Matrix for {args.suites} ({args.mode}): " + f"{len(matrix_jobs)} job(s), {total_cases} case(s)") + for index, job in enumerate(matrix_jobs, start=1): + label = job.tutorial or "(serial)" + print(f" {index}. {label}: {job.case_count} case(s)") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main())