From 0e93bb6d6500b975bfb46b90497f82481339d91f Mon Sep 17 00:00:00 2001 From: Saqib Khan <11544614+saqibkh@users.noreply.github.com> Date: Fri, 23 Oct 2026 16:26:18 -0500 Subject: [PATCH 1/2] nvidia-gpu-tests: optionally fail the play on a failed DCGM diagnostic Every test in the role runs with ignore_errors, so a failing GPU never fails the play. Add gpu_test_dcgm_fail_on_error (off by default); when set, a non-zero exit from dcgmi diag fails the play. Signed-off-by: Saqib Khan <11544614+saqibkh@users.noreply.github.com> --- roles/nvidia-gpu-tests/README.md | 2 +- roles/nvidia-gpu-tests/defaults/main.yml | 4 ++++ roles/nvidia-gpu-tests/tasks/main.yml | 2 +- 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/roles/nvidia-gpu-tests/README.md b/roles/nvidia-gpu-tests/README.md index 8f5b14231..eee1f3796 100644 --- a/roles/nvidia-gpu-tests/README.md +++ b/roles/nvidia-gpu-tests/README.md @@ -9,7 +9,7 @@ Running this will perform the following: * Install the CUDA toolkit * Download and build cuda-samples * Run the Peer2Peer and MatrixMultiply tests -* Run the DCGM diagnostics +* Run the DCGM diagnostics (set `gpu_test_dcgm_fail_on_error: yes` to fail the play when `dcgmi diag` exits non-zero; by default failures are ignored) * Run a basic Tensorflow DL job diff --git a/roles/nvidia-gpu-tests/defaults/main.yml b/roles/nvidia-gpu-tests/defaults/main.yml index 0012fc28e..e9d647a24 100644 --- a/roles/nvidia-gpu-tests/defaults/main.yml +++ b/roles/nvidia-gpu-tests/defaults/main.yml @@ -14,6 +14,10 @@ gpu_test_build: yes # 1 -> quick test (seconds), 2 -> regular test (minute), 3 -> long test (minutes) gpu_test_dcgm_level: 3 +# Whether a failed DCGM diagnostic (dcgmi exits non-zero) fails the play. Set to +# yes to stop on a failing GPU; by default the failure is ignored and only reported. +gpu_test_dcgm_fail_on_error: no + # Batch size to use for Tensorflow tests # DGX-2 -> (512, 50); DGX-1 -> (256,50); DGX-Station -> (128, 50) gpu_test_tf_batch_size: 128 # (choose from 32, 64, 128, 256, 512) diff --git a/roles/nvidia-gpu-tests/tasks/main.yml b/roles/nvidia-gpu-tests/tasks/main.yml index 5c4592ff8..8b78d5950 100644 --- a/roles/nvidia-gpu-tests/tasks/main.yml +++ b/roles/nvidia-gpu-tests/tasks/main.yml @@ -44,7 +44,7 @@ - name: run dcgm diag for up to 15 minutes shell: "dcgmi diag -r {{ gpu_test_dcgm_level }}" register: dcgm - ignore_errors: yes + ignore_errors: "{{ not gpu_test_dcgm_fail_on_error }}" # Note, this may fail to run depending on Ansible settings # We only look at the last few lines with relevant output From d897e9246b7d7a9cb3275dff80cfd3d801f2040c Mon Sep 17 00:00:00 2001 From: Doug Holt Date: Tue, 27 Oct 2026 09:08:47 -0600 Subject: [PATCH 2/2] fix(gpu-tests): normalize the DCGM failure option as a boolean Honor string false/no values supplied through Ansible extra vars. Exercise the actual diagnostic task with a local stub across default, boolean and string settings, with successful and failing diagnostics. Co-Authored-By: Paperclip Signed-off-by: Doug Holt --- roles/nvidia-gpu-tests/tasks/main.yml | 2 +- tests/nvidia-gpu-tests/test_dcgm_failure.py | 72 +++++++++++++++++++++ 2 files changed, 73 insertions(+), 1 deletion(-) create mode 100644 tests/nvidia-gpu-tests/test_dcgm_failure.py diff --git a/roles/nvidia-gpu-tests/tasks/main.yml b/roles/nvidia-gpu-tests/tasks/main.yml index 8b78d5950..eba3c100a 100644 --- a/roles/nvidia-gpu-tests/tasks/main.yml +++ b/roles/nvidia-gpu-tests/tasks/main.yml @@ -44,7 +44,7 @@ - name: run dcgm diag for up to 15 minutes shell: "dcgmi diag -r {{ gpu_test_dcgm_level }}" register: dcgm - ignore_errors: "{{ not gpu_test_dcgm_fail_on_error }}" + ignore_errors: "{{ not (gpu_test_dcgm_fail_on_error | bool) }}" # Note, this may fail to run depending on Ansible settings # We only look at the last few lines with relevant output diff --git a/tests/nvidia-gpu-tests/test_dcgm_failure.py b/tests/nvidia-gpu-tests/test_dcgm_failure.py new file mode 100644 index 000000000..08fb0a398 --- /dev/null +++ b/tests/nvidia-gpu-tests/test_dcgm_failure.py @@ -0,0 +1,72 @@ +#!/usr/bin/env python3 +"""Exercise the real DCGM task locally with a fake dcgmi; no GPU required. + +Run with: python3 tests/nvidia-gpu-tests/test_dcgm_failure.py +Requires PyYAML and ansible-playbook on PATH. +""" +import os +from pathlib import Path +import subprocess +import tempfile +import unittest + +import yaml + + +ROOT = Path(__file__).resolve().parents[2] +ROLE = ROOT / "roles/nvidia-gpu-tests" + + +class DcgmFailureTest(unittest.TestCase): + def test_exit_policy(self): + tasks = yaml.safe_load((ROLE / "tasks/main.yml").read_text()) + task = next(task for task in tasks if task.get("register") == "dcgm") + defaults = yaml.safe_load((ROLE / "defaults/main.yml").read_text()) + # Removing boolean normalization must break the string-false cases. + cases = [ + ("default", [], False), + ("boolean false", ["-e", '{"gpu_test_dcgm_fail_on_error": false}'], False), + ("boolean true", ["-e", '{"gpu_test_dcgm_fail_on_error": true}'], True), + ("string false", ["-e", "gpu_test_dcgm_fail_on_error=false"], False), + ("string no", ["-e", "gpu_test_dcgm_fail_on_error=no"], False), + ("string true", ["-e", "gpu_test_dcgm_fail_on_error=true"], True), + ("string yes", ["-e", "gpu_test_dcgm_fail_on_error=yes"], True), + ] + with tempfile.TemporaryDirectory(prefix="dcgm-test-") as directory: + temp = Path(directory) + stub = temp / "dcgmi" + stub.write_text('#!/bin/sh\n[ "$*" = "diag -r 3" ] || exit 99\nprintf "diagnostic fixture\\n"\nexit "$DIAG_RC"\n') + stub.chmod(0o755) + config = temp / "ansible.cfg" + config.write_text("[defaults]\nretry_files_enabled = False\nhost_key_checking = False\n") + play = [{ + "hosts": "localhost", "connection": "local", "gather_facts": False, + "vars": defaults, + "environment": {"PATH": str(temp) + os.pathsep + os.environ["PATH"]}, + "tasks": [task, {"name": "Verify diagnostic result", "assert": {"that": [ + "dcgm.rc == expected_rc | int", "'diagnostic fixture' in dcgm.stdout" + ]}}], + }] + playbook = temp / "test.yml" + playbook.write_text(yaml.safe_dump(play)) + env = dict(os.environ, ANSIBLE_CONFIG=str(config), ANSIBLE_HOME=str(temp / "home"), + ANSIBLE_LOCAL_TEMP=str(temp / "local"), ANSIBLE_REMOTE_TEMP=str(temp / "remote"), + ANSIBLE_NOCOLOR="1") + for name, extra, fail_on_error in cases: + for rc in (0, 1): + with self.subTest(setting=name, diagnostic_rc=rc): + result = subprocess.run( + ["ansible-playbook", "-i", "localhost,", str(playbook), + "-e", f"expected_rc={rc}", *extra], + cwd=temp, env=dict(env, DIAG_RC=str(rc)), + text=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, timeout=60, + ) + if rc: + self.assertIn("diagnostic fixture", result.stdout, result.stdout) + self.assertEqual(result.returncode, 2 if rc and fail_on_error else 0, result.stdout) + if not (rc and fail_on_error): + self.assertIn("All assertions passed", result.stdout, result.stdout) + + +if __name__ == "__main__": + unittest.main()