From bf4c07e0ff14cbe6ed0771c9cf260093b461bc8e Mon Sep 17 00:00:00 2001 From: Doug Holt Date: Tue, 13 Oct 2026 11:13:15 -0600 Subject: [PATCH 01/28] feat(k8s): add optional KAI Scheduler / Run:ai role, validator, docs Off by default (runai_mode: off). Modes: kai, platform (experimental). Adds playbooks/k8s-cluster/runai.yml, roles/runai, validate_runai.py, docs/k8s-cluster/runai.md, and the deploy-runai-kai skill. --- AGENTS.md | 10 + config.example/group_vars/k8s_cluster.yml | 25 + docs/deepops/validation.md | 19 + docs/k8s-cluster/README.md | 5 + docs/k8s-cluster/runai.md | 209 ++++++++ playbooks/k8s-cluster/runai.yml | 18 + roles/runai/defaults/main.yml | 103 ++++ roles/runai/files/runai_api.py | 162 +++++++ roles/runai/filter_plugins/runai.py | 77 +++ roles/runai/meta/main.yml | 2 + roles/runai/tasks/coredns.yml | 37 ++ roles/runai/tasks/fractioning_preflight.yml | 41 ++ roles/runai/tasks/helm_release.yml | 49 ++ roles/runai/tasks/install.yml | 11 + roles/runai/tasks/kai.yml | 32 ++ roles/runai/tasks/main.yml | 22 + roles/runai/tasks/platform.yml | 54 +++ roles/runai/tasks/platform_cluster.yml | 55 +++ roles/runai/tasks/platform_dns.yml | 22 + roles/runai/tasks/platform_preflight.yml | 34 ++ roles/runai/tasks/platform_tls.yml | 80 +++ roles/runai/tasks/preflight.yml | 61 +++ roles/runai/tasks/uninstall.yml | 71 +++ roles/runai/vars/main.yml | 45 ++ scripts/validation/tests/test_runai.py | 326 +++++++++++++ scripts/validation/validate_runai.py | 507 ++++++++++++++++++++ skills/README.md | 1 + skills/deploy-runai-kai/SKILL.md | 67 +++ tests/runai/bin/helm | 35 ++ tests/runai/bin/kubectl | 26 + tests/runai/run-tests.sh | 100 ++++ tests/runai/site.yml | 11 + tests/runai/test_runai_helpers.py | 84 ++++ 33 files changed, 2401 insertions(+) create mode 100644 docs/k8s-cluster/runai.md create mode 100644 playbooks/k8s-cluster/runai.yml create mode 100644 roles/runai/defaults/main.yml create mode 100644 roles/runai/files/runai_api.py create mode 100644 roles/runai/filter_plugins/runai.py create mode 100644 roles/runai/meta/main.yml create mode 100644 roles/runai/tasks/coredns.yml create mode 100644 roles/runai/tasks/fractioning_preflight.yml create mode 100644 roles/runai/tasks/helm_release.yml create mode 100644 roles/runai/tasks/install.yml create mode 100644 roles/runai/tasks/kai.yml create mode 100644 roles/runai/tasks/main.yml create mode 100644 roles/runai/tasks/platform.yml create mode 100644 roles/runai/tasks/platform_cluster.yml create mode 100644 roles/runai/tasks/platform_dns.yml create mode 100644 roles/runai/tasks/platform_preflight.yml create mode 100644 roles/runai/tasks/platform_tls.yml create mode 100644 roles/runai/tasks/preflight.yml create mode 100644 roles/runai/tasks/uninstall.yml create mode 100644 roles/runai/vars/main.yml create mode 100644 scripts/validation/tests/test_runai.py create mode 100644 scripts/validation/validate_runai.py create mode 100644 skills/deploy-runai-kai/SKILL.md create mode 100755 tests/runai/bin/helm create mode 100755 tests/runai/bin/kubectl create mode 100755 tests/runai/run-tests.sh create mode 100644 tests/runai/site.yml create mode 100644 tests/runai/test_runai_helpers.py diff --git a/AGENTS.md b/AGENTS.md index 8c4306ccb..245be2db7 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -92,6 +92,16 @@ model coverage. See `skills/manage-model-cache/` and variables in `config/group_vars/`; do not edit roles for site-specific values. +## Optional: KAI Scheduler / Run:ai (off by default) + +`runai_mode: kai` (or the experimental `platform`) plus +`playbooks/k8s-cluster/runai.yml` adds GPU-aware queues and fractional GPUs on top +of the GPU Operator. The two modes exclude each other. Verify with +`python3 scripts/validation/validate_runai.py --json` and require `"ok": true`. +Memory-limited fractions (`runai_kai_gpu_fractioning`) need GPU Operator v26.7.1+ +and an r615+ driver the role will not install for you. See +`skills/deploy-runai-kai/` and `docs/k8s-cluster/runai.md`. + ## Gotchas that look like failures but are not - **`nvidia-smi` over SSH reports "No devices were found" on Slurm nodes.** diff --git a/config.example/group_vars/k8s_cluster.yml b/config.example/group_vars/k8s_cluster.yml index 237829576..446f46c0d 100644 --- a/config.example/group_vars/k8s_cluster.yml +++ b/config.example/group_vars/k8s_cluster.yml @@ -94,3 +94,28 @@ kube_enable_rsyslog_client: true rsyslog_server_hostname: "{{ groups['kube_control_plane'][0] }}" rsyslog_client_tcp_host: "{{ rsyslog_server_hostname }}" rsyslog_client_group: "k8s_cluster" + +################################################################################ +# Run:ai / KAI Scheduler (optional, off by default) # +################################################################################ +# Playbook: playbooks/k8s-cluster/runai.yml Docs: docs/k8s-cluster/runai.md +# Check: python3 scripts/validation/validate_runai.py --json +# +# off do nothing (default) +# kai KAI Scheduler: queues, gang scheduling, fractional GPU requests +# platform Run:ai control plane + cluster, same Kubernetes cluster (experimental) +# The modes exclude each other: the platform's cluster chart ships its own KAI. +runai_mode: "off" + +# Hard GPU-memory limits for fractional pods (kai mode). Needs GPU Operator +# v26.7.1+ and an r615+ driver, which you must set up first; DeepOps's default +# driver is not changed. Leave false unless you need enforced memory limits. +#runai_kai_gpu_fractioning: false + +# Platform mode only. Keep the password in an Ansible vault. +#runai_platform_domain: "runai.example.com" +#runai_platform_admin_user: "admin@example.com" +#runai_platform_admin_password: "{{ vault_runai_admin_password }}" +# Lab without DNS: let the role add the hosts entry and a cluster DNS rewrite. +#runai_platform_manage_hosts_entry: true +#runai_platform_coredns_rewrite: true diff --git a/docs/deepops/validation.md b/docs/deepops/validation.md index 91b70c872..d5b033b3b 100644 --- a/docs/deepops/validation.md +++ b/docs/deepops/validation.md @@ -88,6 +88,25 @@ namespace is kept on failure for debugging). For an exhaustive every-GPU job test, see `scripts/k8s/verify_gpu.sh`. +## KAI Scheduler / Run:ai: `scripts/validation/validate_runai.py` + +Run anywhere `kubectl` reaches the cluster after `playbooks/k8s-cluster/runai.yml` +(only if you enabled `runai_mode`; see [the guide](../k8s-cluster/runai.md)): + +```bash +python3 scripts/validation/validate_runai.py --json +python3 scripts/validation/validate_runai.py --json --memory-enforcement +python3 scripts/validation/validate_runai.py --json --mode platform --platform-domain +python3 scripts/validation/validate_runai.py --json --verify-uninstalled +``` + +It proves that a pod scheduled by KAI ran `nvidia-smi` and completed, that two +half-GPU pods shared one GPU, optionally that a pod past its GPU-memory share is +refused, and (platform) that the control plane answers and the cluster is +Connected. `--verify-uninstalled` is read-only and passes only when no +`kai-scheduler`/`runai*` namespace or release is left. The JSON carries +`schema_version`, `ok`, one `*_ran`/`*_ok` pair per check, and `failures`. + ## Testing the tools ```bash diff --git a/docs/k8s-cluster/README.md b/docs/k8s-cluster/README.md index c9d978efc..004f3ab4c 100644 --- a/docs/k8s-cluster/README.md +++ b/docs/k8s-cluster/README.md @@ -20,6 +20,7 @@ Instructions for deploying a GPU cluster with Kubernetes - [Load Balancer and Ingress](#load-balancer-and-ingress) - [Kubeflow](#kubeflow) - [NVIDIA Network Operator](#nvidia-network-operator) + - [Run:ai and KAI Scheduler](#runai-and-kai-scheduler) - [Cluster Maintenance](#cluster-maintenance) - [Adding Nodes](#adding-nodes) - [Removing Nodes](#removing-nodes) @@ -264,6 +265,10 @@ For more information on Kubeflow, please refer to the [official documentation](h NVIDIA Network Operator leverages Kubernetes CRDs and Operator SDK to manage networking related components in Kuberenets cluster. High performance networking in Kuberentes requires many components, such as multus-CNI, device drivers and plugins to be installed correctly, NVIDIA network operator aims to manage all those necessary components automatically under one operator frame work to simply the deployment, operation and management of NVIDIA networking for Kubernetes. To deploy NVIDIA network operator, please refer to the [NVIDIA Network Operator Deployment Guide in DeepOps](nvidia-network-operator.md), for more information on NVIDIA network operator, please refer to its [github](https://github.com/Mellanox/network-operator) page and this [solution guide](https://docs.nvidia.com/networking/display/COKAN10/Network+Operator). +### Run:ai and KAI Scheduler + +Optional GPU-aware scheduling (queues, gang scheduling, fractional GPUs) on top of the GPU Operator, off by default. See the [Run:ai and KAI Scheduler guide](runai.md). + ## Cluster Maintenance DeepOps uses [Kubespray](https://github.com/kubernetes-sigs/kubespray) to deploy Kubernetes and therefore common cluster actions (such as adding nodes, removing them, draining and upgrading the cluster) should be performed with it. Kubespray is included as a submodule in the submodules/kubespray directory. diff --git a/docs/k8s-cluster/runai.md b/docs/k8s-cluster/runai.md new file mode 100644 index 000000000..d065cd208 --- /dev/null +++ b/docs/k8s-cluster/runai.md @@ -0,0 +1,209 @@ +# Run:ai and KAI Scheduler (optional) + +DeepOps can install open-source GPU scheduling on top of the GPU Operator. It is +**off by default** and changes nothing until you choose a mode. + +| `runai_mode` | What you get | Status in 26.11 | +|---|---|---| +| `off` (default) | Nothing installed. | | +| `kai` | [KAI Scheduler](https://github.com/kai-scheduler/KAI-Scheduler): fair-share queues, gang scheduling, bin-packing, and pods that ask for a fraction of a GPU. | Supported. | +| `platform` | The Run:ai control plane and cluster: the above plus projects and quotas, a web UI, CLI and REST API. | **Experimental.** Included only if it passes a live check before the release freeze. | + +The modes exclude each other. The Run:ai cluster chart ships its own KAI +Scheduler, so the role refuses to install one beside the other. Switch by +removing the first (`-e runai_state=absent`), then installing the second. + +Not covered: multi-cluster setups, Knative/KServe/Training Operator add-ons, +and any DeepOps-owned UI. The web UI in `platform` mode is the upstream one. + +## Requirements + +- A DeepOps Kubernetes cluster with the GPU Operator (or the device plugin) and + at least one node that advertises `nvidia.com/gpu`. The role refuses to run + without one. +- Kubernetes 1.35 to 1.37 (the Kubespray default in this release is inside that range). +- `kai`: nothing else. +- `platform`, which you must provide before running the role: + - a default StorageClass (`playbooks/k8s-cluster/nfs-client-provisioner.yml`) + - a load balancer for a `LoadBalancer` service ([Load Balancer and Ingress](ingress.md)), + or set `runai_platform_ingress_service_type: NodePort` + - `kube-prometheus-stack` 88.6.5 or newer (`scripts/k8s/deploy_monitoring.sh` + with `HELM_PROMETHEUS_CHART_VERSION` set) + - a DNS name for the control plane, and a TLS certificate for it + - room: the control plane alone wants about 10 CPU cores, 12 GB of memory and + 110 GB of disk; the cluster services about 10 cores, 20 GB, 50 GB. See the + [Run:ai system requirements](https://docs.nvidia.com/run-ai/self-hosted/v3.0/documentation/installation-and-configuration/system-requirements.md) + for production sizing; DeepOps only tests a single large node. + +## Pinned versions + +| Component | Version | +|---|---| +| KAI Scheduler chart | v0.18.5 (v0.18 is a long-term-support line) | +| gpu-fractioning (bundled in the KAI chart) | v0.1.7 | +| Run:ai control-plane and cluster charts | 3.0.20 | +| HAProxy ingress controller chart (platform) | 1.54.2 | + +All are Apache-2.0. The Run:ai images come from `nvcr.io` without an NGC key. +Versions are variables (`runai_*_version`); pins are refreshed with each release. + +## Install KAI Scheduler + +1. Set the mode in `config/group_vars/k8s_cluster.yml`: + + ```yaml + runai_mode: kai + ``` + +2. Preview, then apply (runs on the first control-plane node): + + ```bash + ansible-playbook -l kube_control_plane playbooks/k8s-cluster/runai.yml --check --diff + ansible-playbook -l kube_control_plane playbooks/k8s-cluster/runai.yml + ``` + +3. Check it: + + ```bash + python3 scripts/validation/validate_runai.py --json + ``` + + The validator creates a temporary namespace and proves, with real pods, that + (a) a pod using `schedulerName: kai-scheduler` in a queue ran `nvidia-smi` + and completed, and (b) two pods that each asked for half a GPU landed on the + same GPU and both completed. It requires `"ok": true`; if it is false, each + entry in `failures` names the problem. The temporary namespace is deleted on + success and kept on failure so you can look at it. + +Rerunning the playbook is safe; a converged run reports `changed=0`. + +### Submit a pod + +Pods need three things: the scheduler name, a queue label, and (for a fraction) an +annotation. A fresh install creates `default-queue`. + +```yaml +apiVersion: v1 +kind: Pod +metadata: + name: half-gpu + labels: + kai.scheduler/queue: default-queue + annotations: + gpu-fraction: "0.5" +spec: + schedulerName: kai-scheduler + containers: + - name: main + image: nvcr.io/nvidia/cuda:13.0.2-base-ubuntu24.04 + command: ["nvidia-smi", "-L"] +``` + +Use a namespace of your own, not `kai-scheduler`. A pod cannot ask for both a +fraction and a whole GPU. Without `gpu-memory.*` settings (below), a fraction is +a scheduling promise, not a memory limit: nothing stops a pod from using more +GPU memory than its share. + +To run the same vLLM quickstart pod through KAI instead of the default scheduler, +add `schedulerName: kai-scheduler` and the queue label to its pod spec. The default +scheduler stays the default. + +### Optional: enforced GPU-memory limits + +`runai_kai_gpu_fractioning: true` turns on the gpu-fractioning operator, which +caps each pod's GPU memory. It is opt-in because it needs a GPU stack the +DeepOps defaults do not use: + +- GPU Operator **v26.7.1 or newer** (DeepOps pins v26.7.0 by default), and +- an NVIDIA driver **r615 or newer** on every GPU node. The GPU Operator does not + choose r615 by default; set `gpu_operator_driver_version` to an r615 release + and `gpu_operator_preinstalled_nvidia_software: false` (or install an r615 host + driver yourself). + +The role checks both and stops with an explanation if either is missing. It never +changes your driver or GPU Operator. Then verify the limits actually hold: + +```bash +python3 scripts/validation/validate_runai.py --json --memory-enforcement +``` + +This also proves a pod inside its share completes and a pod that allocates past +its share is refused. In this mode pods ask for memory with the +`nvidia.com/container..gpu-memory.request` and `.limit` annotations; see the +[KAI GPU sharing guide](https://github.com/kai-scheduler/KAI-Scheduler/tree/v0.18.5/docs/gpu-sharing). + +## Install the platform (experimental) + +```yaml +runai_mode: platform +runai_platform_domain: "runai.example.com" +runai_platform_admin_user: "admin@example.com" +runai_platform_admin_password: "{{ vault_runai_admin_password }}" # from an Ansible vault +``` + +```bash +ansible-playbook -l kube_control_plane playbooks/k8s-cluster/runai.yml +``` + +The role then: + +1. checks the prerequisites above and stops with a pointer to the fix if one is missing; +2. installs the HAProxy ingress controller the Run:ai documentation uses + (`ingress-nginx` has been retired upstream). Set + `runai_platform_install_ingress: false` if your site already runs a controller + and set `runai_platform_ingress_class` to match; +3. creates the TLS secrets. With no certificate given it makes a local CA and a + certificate for the domain (lab use); for a real site set + `runai_platform_tls_cert_file`, `runai_platform_tls_key_file` and, if privately + signed, `runai_platform_ca_file` (paths on the control-plane node); +4. installs the control plane, registers the cluster with it through its API, and + installs the cluster chart with the short-lived token the API returns. + +**Names.** `runai_platform_domain` must resolve to the ingress address from the +nodes and from wherever you browse. Without DNS (a lab), set +`runai_platform_manage_hosts_entry: true` (hosts-file line on the control-plane +node) and `runai_platform_coredns_rewrite: true` (in-cluster name). Kubespray owns +the CoreDNS settings and overwrites the rewrite on its next run; for a lasting entry +set Kubespray's `dns_etchosts` instead. + +Check it (the admin credentials go in the environment, never on the command line): + +```bash +RUNAI_USERNAME=admin@example.com RUNAI_PASSWORD='...' \ + python3 scripts/validation/validate_runai.py --json --mode platform \ + --platform-domain runai.example.com --ca-file /opt/deepops/runai/ca.pem \ + --namespace runai- --queue +``` + +In platform mode, pods must be submitted in a Run:ai project's namespace and use +that project's queue. Create a project in the web UI (or API) first, then pass its +namespace and queue to the validator. Beyond what the validator checks, DeepOps +does not configure projects, users or quotas; use the Run:ai documentation. + +Run:ai 3.0 releases often. Keep `runai_chart_version` pinned and refresh it on +purpose. + +## Remove it + +```bash +ansible-playbook -l kube_control_plane playbooks/k8s-cluster/runai.yml -e runai_state=absent +python3 scripts/validation/validate_runai.py --json --verify-uninstalled +``` + +The second command passes only if no `kai-scheduler`, `kai-resource-reservation` +or `runai*` namespace or Helm release remains. The charts leave some namespaces +and custom resource definitions behind on purpose; the role removes the +namespaces. Add `-e runai_uninstall_remove_crds=true` to remove the definitions +too (this deletes any queues and projects still defined). + +## Troubleshooting + +| Symptom | Likely cause and fix | +|---|---| +| Role stops: "No node advertises nvidia.com/gpu" | GPU stack not ready. Install the GPU Operator and wait for the node to report GPUs. | +| Role stops: "ships its own KAI" | Both modes are installed. Remove one first. | +| Role stops: GPU Operator or `r615` | Fractioning prerequisites missing; see above. | +| Pod stays `Pending` with `schedulerName: kai-scheduler` | No queue label, or the queue does not exist: `kubectl get queues.scheduling.run.ai`. | +| A half-GPU pod is rejected | It also asks for a whole `nvidia.com/gpu`, or the fraction is not between 0 and 1. | +| Validator: scheduler deployments missing | The KAI operator is still starting. Wait a minute and rerun. | +| Platform validator: control plane unreachable | The domain does not resolve to the ingress address, or the CA is not trusted: pass `--ca-file` and `--resolve-ip`. | diff --git a/playbooks/k8s-cluster/runai.yml b/playbooks/k8s-cluster/runai.yml new file mode 100644 index 000000000..8beda18bc --- /dev/null +++ b/playbooks/k8s-cluster/runai.yml @@ -0,0 +1,18 @@ +--- +# Optional: KAI Scheduler or the Run:ai platform on top of the GPU Operator. +# Off by default. Choose a mode in config/group_vars/k8s_cluster.yml: +# runai_mode: kai # KAI Scheduler (queues, gang scheduling, fractional GPUs) +# runai_mode: platform # Run:ai control plane + cluster (experimental) +# Remove with: -e runai_state=absent +# Then check with: python3 scripts/validation/validate_runai.py --json +# See docs/k8s-cluster/runai.md +- hosts: kube_control_plane[0] + become: true + tasks: + - name: Install or remove Run:ai + include_role: + name: runai + run_once: true + environment: "{{ proxy_env if proxy_env is defined else {} }}" + tags: + - runai diff --git a/roles/runai/defaults/main.yml b/roles/runai/defaults/main.yml new file mode 100644 index 000000000..878849eda --- /dev/null +++ b/roles/runai/defaults/main.yml @@ -0,0 +1,103 @@ +--- +# Optional Run:ai / KAI Scheduler support for the DeepOps Kubernetes path. +# Off by default. Playbook: playbooks/k8s-cluster/runai.yml +# Docs: docs/k8s-cluster/runai.md Validator: scripts/validation/validate_runai.py +# +# Modes are mutually exclusive. Never run standalone KAI and the Run:ai cluster +# chart together: the cluster chart ships its own KAI Scheduler. +# off do nothing (default) +# kai KAI Scheduler only: queues, gang scheduling, fractional GPU requests +# platform Run:ai control plane + cluster (UI, API, projects, quotas); needs +# more infrastructure, see docs. Experimental in 26.11. +runai_mode: "off" + +# present installs/updates; absent removes everything this role installed. +runai_state: "present" + +runai_helm: /usr/local/bin/helm +runai_kubectl: kubectl +runai_helm_timeout: "20m" +# Require at least one node advertising nvidia.com/gpu before installing. +runai_require_gpu: true +runai_work_dir: "{{ deepops_dir | default('/opt/deepops') }}/runai" + +# ---------------------------------------------------------------- KAI Scheduler +# Pinned chart (v0.18 is a long-term-support line). No "latest". +runai_kai_chart: "oci://ghcr.io/kai-scheduler/kai-scheduler/kai-scheduler" +runai_kai_chart_version: "v0.18.5" +runai_kai_release_name: "kai-scheduler" +runai_kai_namespace: "kai-scheduler" + +# Hard per-pod GPU-memory limits via the gpu-fractioning operator (bundled in the +# KAI chart). Opt in only. It needs GPU Operator >= v26.7.1 with an r615+ driver +# container, which the GPU Operator does not pick by default, so the role refuses +# to run until you have set those yourself (see docs). The default driver is +# never changed by this role. +runai_kai_gpu_fractioning: false +runai_gpu_operator_min_version: "v26.7.1" +runai_gpu_fractioning_min_driver_major: 615 + +# Extra Helm values merged over the role's values (advanced). +runai_kai_extra_values: {} + +# ------------------------------------------------------------------- platform +runai_cp_chart: "control-plane" +runai_cluster_chart: "runai-cluster" +runai_chart_repo: "https://helm.ngc.nvidia.com/nvidia/runai-oss" +runai_chart_version: "3.0.20" +runai_cp_release_name: "runai-backend" +runai_cp_namespace: "runai-backend" +runai_cluster_release_name: "runai-cluster" +runai_cluster_namespace: "runai" +runai_cluster_name: "deepops" + +# Fully qualified name the control plane is served on. Required in platform mode. +# For QA without DNS, use a name resolved through a hosts-file entry (see below). +runai_platform_domain: "" + +# First administrator. Put the password in an Ansible vault; the role refuses an +# empty value and never prints it. +runai_platform_admin_user: "admin@example.com" +runai_platform_admin_password: "" + +# Ingress. ingress-nginx was retired upstream, so platform mode installs the +# HAProxy controller that the Run:ai documentation uses, scoped to this role. +# Set to false when the site already runs an ingress controller of the class below. +runai_platform_install_ingress: true +runai_platform_ingress_class: "haproxy" +runai_platform_ingress_chart: "kubernetes-ingress" +runai_platform_ingress_repo: "https://haproxytech.github.io/helm-charts" +runai_platform_ingress_chart_version: "1.54.2" +runai_platform_ingress_release_name: "haproxy-kubernetes-ingress" +runai_platform_ingress_namespace: "haproxy-controller" +# LoadBalancer works with the DeepOps MetalLB path. NodePort suits a lab. +runai_platform_ingress_service_type: "LoadBalancer" +# Single-node labs without a load balancer: serve 80/443 directly on the node. +runai_platform_ingress_host_ports: false + +# TLS. Leave the file paths empty to generate a local CA and a certificate for the +# domain (lab use). For real sites supply a certificate, key and, if privately +# signed, the CA chain; all three are paths on the first control-plane host. +runai_platform_tls_cert_file: "" +runai_platform_tls_key_file: "" +runai_platform_ca_file: "" + +# Lab convenience, all off by default. Use when the domain has no DNS record. +# Address the domain should point at (default: this host's primary address). +runai_platform_ingress_ip: "{{ ansible_default_ipv4.address | default('') }}" +# Add " " to /etc/hosts on the control-plane host. +runai_platform_manage_hosts_entry: false +# Make the domain resolve inside the cluster (CoreDNS rewrite to the ingress +# service). Reverted on uninstall. +runai_platform_coredns_rewrite: false + +# kube-prometheus-stack is a prerequisite that DeepOps installs separately +# (scripts/k8s/deploy_monitoring.sh). Run:ai 3.0 needs this chart version or newer. +runai_platform_prometheus_min_version: "88.6.5" + +# Extra Helm values merged over the role's values (advanced). +runai_platform_cp_extra_values: {} +runai_platform_cluster_extra_values: {} + +# Remove the CRDs these charts leave behind when uninstalling. +runai_uninstall_remove_crds: false diff --git a/roles/runai/files/runai_api.py b/roles/runai/files/runai_api.py new file mode 100644 index 000000000..ffc32ef98 --- /dev/null +++ b/roles/runai/files/runai_api.py @@ -0,0 +1,162 @@ +#!/usr/bin/env python3 +"""Register a Run:ai cluster with the local control plane and print its Helm values. + +Called by the runai role (platform mode). Credentials come from the environment +(RUNAI_USERNAME, RUNAI_PASSWORD), never from arguments, and are never printed. +Prints one JSON object: {"values": {...}, "cluster_uid": "...", "created": bool}. +Exit 0 ok, 1 API/HTTP failure, 2 bad input. +""" +import argparse +import http.client +import json +import os +import re +import socket +import ssl +import sys +import time +import urllib.parse + +SET_RE = re.compile(r"--set(?:-string)?[ =]+([A-Za-z0-9_.\[\]-]+)=(\"[^\"]*\"|'[^']*'|[^\s\\]+)") +KEY_RE = re.compile(r"^([A-Za-z0-9_-]+)(?:\[(\d+)\])?$") + + +class ApiError(Exception): + pass + + +def parse_set_args(text): + pairs = {} + for key, value in SET_RE.findall(text or ""): + if len(value) >= 2 and value[0] in "\"'" and value[-1] == value[0]: + value = value[1:-1] + pairs[key] = value + return pairs + + +def nest(pairs): + out = {} + for dotted, value in pairs.items(): + node = out + parts = dotted.split(".") + for i, part in enumerate(parts): + m = KEY_RE.match(part) + if not m: + raise ApiError("unsupported Helm key in installation string") + key, index = m.group(1), m.group(2) + last = i == len(parts) - 1 + if index is None: + if last: + node[key] = value + else: + node = node.setdefault(key, {}) + else: + lst = node.setdefault(key, []) + while len(lst) <= int(index): + lst.append({}) + if last: + lst[int(index)] = value + else: + node = lst[int(index)] + return out + + +class Client: + def __init__(self, domain, ip, ca_file, timeout=30): + self.domain, self.ip, self.timeout = domain, ip or domain, timeout + self.context = ssl.create_default_context(cafile=ca_file or None) + self.token = "" + + def request(self, method, path, body=None, form=None): + headers = {"Accept": "application/json"} + data = None + if form is not None: + data = urllib.parse.urlencode(form).encode() + headers["Content-Type"] = "application/x-www-form-urlencoded" + elif body is not None: + data = json.dumps(body).encode() + headers["Content-Type"] = "application/json" + if self.token: + headers["Authorization"] = "Bearer " + self.token + try: + sock = socket.create_connection((self.ip, 443), self.timeout) + conn = http.client.HTTPSConnection(self.domain, 443, context=self.context, timeout=self.timeout) + conn.sock = self.context.wrap_socket(sock, server_hostname=self.domain) + conn.request(method, path, body=data, headers=headers) + resp = conn.getresponse() + raw = resp.read(4 * 1024 * 1024) + conn.close() + except (OSError, ssl.SSLError, http.client.HTTPException) as exc: + raise ApiError("%s %s failed: %s" % (method, path.split("?")[0], type(exc).__name__)) + if resp.status >= 300: + raise ApiError("%s %s returned HTTP %d" % (method, path.split("?")[0], resp.status)) + try: + return json.loads(raw) if raw else {} + except ValueError: + raise ApiError("%s %s returned invalid JSON" % (method, path.split("?")[0])) + + def login(self, user, password): + doc = self.request("POST", "/api/v2/token", + form={"grant_type": "password", "username": user, "password": password}) + self.token = doc.get("access_token", "") + if not self.token: + raise ApiError("login returned no access token") + + +def wait_for_api(client, user, password, wait): + deadline = time.time() + wait + last = "" + while True: + try: + client.login(user, password) + return + except ApiError as exc: + last = str(exc) + if time.time() >= deadline: + raise ApiError("control plane did not accept the login in time (%s)" % last) + time.sleep(10) + + +def main(argv=None): + p = argparse.ArgumentParser(description=__doc__.split("\n")[0], allow_abbrev=False) + p.add_argument("--domain", required=True) + p.add_argument("--resolve-ip", default="") + p.add_argument("--ca-file", default="") + p.add_argument("--cluster-name", required=True) + p.add_argument("--chart-version", required=True) + p.add_argument("--wait", type=int, default=600, help="seconds to wait for the control plane") + args = p.parse_args(argv) + user, password = os.environ.get("RUNAI_USERNAME", ""), os.environ.get("RUNAI_PASSWORD", "") + if not user or not password: + print("error: RUNAI_USERNAME and RUNAI_PASSWORD must be set", file=sys.stderr) + return 2 + client = Client(args.domain, args.resolve_ip, args.ca_file) + try: + wait_for_api(client, user, password, args.wait) + listing = client.request("GET", "/api/v1/clusters") + items = listing if isinstance(listing, list) else listing.get("clusters", []) + found = [c for c in items if c.get("name") == args.cluster_name] + created = not found + if found: + uid = found[0]["uuid"] + else: + uid = client.request("POST", "/api/v1/clusters", body={"name": args.cluster_name})["uuid"] + info = client.request( + "GET", "/api/v1/clusters/%s/cluster-install-info?version=%s" + % (urllib.parse.quote(uid), urllib.parse.quote(args.chart_version))) + pairs = parse_set_args(info.get("installationStr", "")) + if info.get("bootstrapToken"): + pairs["controlPlane.bootstrapToken"] = info["bootstrapToken"] + for needed in ("controlPlane.url", "cluster.uid"): + if needed not in pairs: + raise ApiError("installation string has no %s" % needed) + pairs = {k: v for k, v in pairs.items() if k.startswith(("controlPlane.", "cluster."))} + print(json.dumps({"values": nest(pairs), "cluster_uid": uid, "created": created})) + return 0 + except (ApiError, KeyError) as exc: + print("error: %s" % exc, file=sys.stderr) + return 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/roles/runai/filter_plugins/runai.py b/roles/runai/filter_plugins/runai.py new file mode 100644 index 000000000..1dbdda578 --- /dev/null +++ b/roles/runai/filter_plugins/runai.py @@ -0,0 +1,77 @@ +"""Small helpers for the runai role (pure functions, unit tested).""" +import re + + +def runai_deep_merge(base, extra): + """Recursively merge ``extra`` over ``base`` (dicts merge, everything else replaces).""" + out = dict(base) + for key, value in (extra or {}).items(): + if isinstance(value, dict) and isinstance(out.get(key), dict): + out[key] = runai_deep_merge(out[key], value) + else: + out[key] = value + return out + + +def runai_password_ok(value): + """Run:ai admin password policy: 8+ chars, digit, lower, upper, special from -!@#$%^&*()_+=""" + return bool( + isinstance(value, str) and len(value) >= 8 + and re.search(r"[0-9]", value) and re.search(r"[a-z]", value) + and re.search(r"[A-Z]", value) and re.search(r"[-!@#$%^&*()_+=]", value) + ) + + +def _items(doc): + if isinstance(doc, str): + import json + doc = json.loads(doc) + return (doc or {}).get("items", []) + + +def runai_gpu_count(nodes_doc): + """Total allocatable nvidia.com/gpu across a ``kubectl get nodes -o json`` document.""" + total = 0 + for item in _items(nodes_doc): + value = ((item.get("status") or {}).get("allocatable") or {}).get("nvidia.com/gpu", "0") + try: + total += max(int(value), 0) + except (TypeError, ValueError): + pass + return total + + +def runai_driver_majors(nodes_doc): + """Driver major version (int) for every node labelled nvidia.com/gpu.present.""" + out = [] + for item in _items(nodes_doc): + labels = (item.get("metadata") or {}).get("labels") or {} + if "nvidia.com/gpu.present" not in labels: + continue + try: + out.append(int(labels.get("nvidia.com/cuda.driver-version.major", "0"))) + except (TypeError, ValueError): + out.append(0) + return out + + +def runai_default_storageclasses(sc_doc): + """Names of default StorageClasses in a ``kubectl get storageclass -o json`` document.""" + out = [] + for item in _items(sc_doc): + meta = item.get("metadata") or {} + ann = meta.get("annotations") or {} + if ann.get("storageclass.kubernetes.io/is-default-class") == "true": + out.append(meta.get("name", "")) + return out + + +class FilterModule(object): + def filters(self): + return { + "runai_deep_merge": runai_deep_merge, + "runai_gpu_count": runai_gpu_count, + "runai_driver_majors": runai_driver_majors, + "runai_default_storageclasses": runai_default_storageclasses, + "runai_password_ok": runai_password_ok, + } diff --git a/roles/runai/meta/main.yml b/roles/runai/meta/main.yml new file mode 100644 index 000000000..23d65c7ef --- /dev/null +++ b/roles/runai/meta/main.yml @@ -0,0 +1,2 @@ +--- +dependencies: [] diff --git a/roles/runai/tasks/coredns.yml b/roles/runai/tasks/coredns.yml new file mode 100644 index 000000000..0c5e392b0 --- /dev/null +++ b/roles/runai/tasks/coredns.yml @@ -0,0 +1,37 @@ +--- +# Adds (or removes) one hosts{} block in the CoreDNS Corefile for the domain. +# Kubespray owns this ConfigMap and will overwrite it on its next run; for a +# permanent entry set kubespray's dns_etchosts instead (see docs). +- name: Read the CoreDNS Corefile + ansible.builtin.command: "{{ runai_kubectl }} -n kube-system get configmap coredns --output json" + register: runai_corefile_raw + changed_when: false + check_mode: false + +- name: Plan the Corefile change + ansible.builtin.set_fact: + runai_corefile_old: "{{ (runai_corefile_raw.stdout | from_json).data.Corefile }}" + runai_corefile_block: " hosts {\n {{ runai_platform_ingress_ip }} {{ runai_platform_domain }}\n fallthrough\n } # deepops-runai\n" + no_log: true + +- name: Compute the new Corefile + ansible.builtin.set_fact: + runai_corefile_clean: "{{ runai_corefile_old | regex_replace(' hosts \\{\\n[^}]*\\} # deepops-runai\\n', '') }}" + no_log: true + +- name: Apply the Corefile change + ansible.builtin.shell: | + set -euo pipefail + {{ runai_kubectl }} -n kube-system patch configmap coredns --type merge \ + --patch "$(python3 -c 'import json,sys; print(json.dumps({"data": {"Corefile": sys.stdin.read()}}))')" + {{ runai_kubectl }} -n kube-system rollout restart deployment/coredns + args: + executable: /bin/bash + stdin: >- + {{ runai_corefile_clean | regex_replace('(\\.:53 \\{\\n)', '\\1' ~ runai_corefile_block) if runai_coredns_present | bool else runai_corefile_clean }} + stdin_add_newline: false + register: runai_corefile_apply + changed_when: true + when: >- + (runai_coredns_present | bool and 'deepops-runai' not in runai_corefile_old) + or (not runai_coredns_present | bool and 'deepops-runai' in runai_corefile_old) diff --git a/roles/runai/tasks/fractioning_preflight.yml b/roles/runai/tasks/fractioning_preflight.yml new file mode 100644 index 000000000..659df65d3 --- /dev/null +++ b/roles/runai/tasks/fractioning_preflight.yml @@ -0,0 +1,41 @@ +--- +# gpu-fractioning needs GPU Operator >= runai_gpu_operator_min_version and an +# r615-or-newer driver. This role never changes the GPU stack; it refuses to +# continue until the stack is already right. +- name: Find the GPU Operator release + ansible.builtin.set_fact: + runai_gpu_operator_chart: "{{ (runai_releases | selectattr('name', 'equalto', gpu_operator_release_name | default('nvidia-gpu-operator')) | map(attribute='chart') | list | first) | default('') }}" + +- name: Require a new enough GPU Operator + ansible.builtin.assert: + that: + - runai_gpu_operator_chart.startswith('gpu-operator-') + - runai_gpu_operator_chart.split('-')[-1] | regex_replace('^v', '') is version(runai_gpu_operator_min_version | regex_replace('^v', ''), '>=') + fail_msg: >- + GPU memory fractioning needs GPU Operator {{ runai_gpu_operator_min_version }} or newer + (found: {{ runai_gpu_operator_chart | default('none', true) }}). Set gpu_operator_chart_version + and rerun playbooks/k8s-cluster/nvidia-gpu-operator.yml, or leave runai_kai_gpu_fractioning false. + quiet: true + +- name: Read the driver version the GPU stack reports + ansible.builtin.command: "{{ runai_kubectl }} get nodes --output json" + register: runai_driver_nodes + changed_when: false + check_mode: false + no_log: true + +- name: List GPU nodes and their driver major version + ansible.builtin.set_fact: + runai_driver_majors: "{{ runai_driver_nodes.stdout | runai_driver_majors }}" + +- name: Require an r615 or newer driver on every GPU node + ansible.builtin.assert: + that: + - runai_driver_majors | length > 0 + - runai_driver_majors | min >= runai_gpu_fractioning_min_driver_major + fail_msg: >- + GPU memory fractioning needs driver r{{ runai_gpu_fractioning_min_driver_major }} or newer on every + GPU node (found majors: {{ runai_driver_majors }}). The GPU Operator does not pick it by default: set + gpu_operator_driver_version to an r615 release (with gpu_operator_preinstalled_nvidia_software: false) + or install an r615 host driver, then rerun. See docs/k8s-cluster/runai.md. + quiet: true diff --git a/roles/runai/tasks/helm_release.yml b/roles/runai/tasks/helm_release.yml new file mode 100644 index 000000000..d8b2492ea --- /dev/null +++ b/roles/runai/tasks/helm_release.yml @@ -0,0 +1,49 @@ +--- +# Idempotent "helm upgrade --install". Inputs (set with vars: on include_tasks): +# _rel: {release, namespace, chart, version, repo (optional), values (dict), +# sensitive (bool), wait (bool, default true)} +# Values travel on stdin, never on the command line or on disk. The release is only +# touched when it is missing, not deployed, on another chart version, or its +# user-supplied values differ, so a converged rerun reports changed=0. +- name: "{{ _rel.release }} | look up the current release" + ansible.builtin.command: "{{ runai_helm }} list --namespace {{ _rel.namespace }} --output json" + register: _rel_list + changed_when: false + check_mode: false + failed_when: _rel_list.rc != 0 and 'not found' not in _rel_list.stderr + +- name: "{{ _rel.release }} | note the current release" + ansible.builtin.set_fact: + _rel_current: "{{ (_rel_list.stdout | default('[]', true) | from_json | selectattr('name', 'equalto', _rel.release) | list | first) | default({}) }}" + +- name: "{{ _rel.release }} | read its current values" + ansible.builtin.command: "{{ runai_helm }} get values {{ _rel.release }} --namespace {{ _rel.namespace }} --output json" + register: _rel_values + changed_when: false + check_mode: false + no_log: "{{ _rel.sensitive | default(false) }}" + when: _rel_current | length > 0 + +- name: "{{ _rel.release }} | decide whether to change it" + ansible.builtin.set_fact: + _rel_needs_change: >- + {{ (_rel_current | length == 0) + or (_rel_current.status | default('') != 'deployed') + or (_rel_current.chart | default('') != _rel.chart.split('/')[-1] ~ '-' ~ _rel.version) + or ((_rel_values.stdout | default('null', true) | from_json or {}) | to_json(sort_keys=True) + != (_rel.helm_values | to_json(sort_keys=True))) }} + no_log: "{{ _rel.sensitive | default(false) }}" + +- name: "{{ _rel.release }} | helm upgrade --install" + ansible.builtin.command: + argv: >- + {{ [runai_helm, 'upgrade', '--install', _rel.release, _rel.chart, + '--version', _rel.version, '--namespace', _rel.namespace, '--create-namespace'] + + (['--repo', _rel.repo] if _rel.repo | default('') else []) + + (['--wait', '--timeout', runai_helm_timeout] if _rel.wait | default(true) else []) + + ['--values', '-'] }} + stdin: "{{ _rel.helm_values | to_json }}" + environment: "{{ proxy_env if proxy_env is defined else {} }}" + no_log: "{{ _rel.sensitive | default(false) }}" + changed_when: true + when: _rel_needs_change | bool diff --git a/roles/runai/tasks/install.yml b/roles/runai/tasks/install.yml new file mode 100644 index 000000000..490eb4f6b --- /dev/null +++ b/roles/runai/tasks/install.yml @@ -0,0 +1,11 @@ +--- +- name: Check prerequisites + ansible.builtin.include_tasks: preflight.yml + +- name: Install KAI Scheduler + ansible.builtin.include_tasks: kai.yml + when: runai_mode == 'kai' + +- name: Install the Run:ai platform + ansible.builtin.include_tasks: platform.yml + when: runai_mode == 'platform' diff --git a/roles/runai/tasks/kai.yml b/roles/runai/tasks/kai.yml new file mode 100644 index 000000000..1e4a61dfb --- /dev/null +++ b/roles/runai/tasks/kai.yml @@ -0,0 +1,32 @@ +--- +- name: Install KAI Scheduler + ansible.builtin.include_tasks: helm_release.yml + vars: + _rel: + release: "{{ runai_kai_release_name }}" + namespace: "{{ runai_kai_namespace }}" + chart: "{{ runai_kai_chart }}" + version: "{{ runai_kai_chart_version }}" + helm_values: "{{ runai_kai_values }}" + +# The chart installs only the KAI operator; the operator then creates the scheduler, +# binder, queue controller and admission webhook. Wait for those, not just helm. +- name: Wait for the KAI Scheduler components + ansible.builtin.command: >- + {{ runai_kubectl }} --namespace {{ runai_kai_namespace }} + wait --for=condition=Available --timeout=30s deployment/{{ item }} + loop: "{{ runai_kai_components }}" + register: runai_kai_wait + until: runai_kai_wait.rc == 0 + retries: 30 + delay: 10 + changed_when: false + check_mode: false + +- name: Show how to check the result + ansible.builtin.debug: + msg: >- + KAI Scheduler {{ runai_kai_chart_version }} is installed. Check it with: + python3 scripts/validation/validate_runai.py --json + (add --memory-enforcement when runai_kai_gpu_fractioning is true). + run_once: true diff --git a/roles/runai/tasks/main.yml b/roles/runai/tasks/main.yml new file mode 100644 index 000000000..c1b4a32fa --- /dev/null +++ b/roles/runai/tasks/main.yml @@ -0,0 +1,22 @@ +--- +# Optional Run:ai / KAI Scheduler. Does nothing unless runai_mode is kai or platform. +- name: Check the Run:ai settings + ansible.builtin.assert: + that: + - runai_mode in ['off', 'kai', 'platform'] + - runai_state in ['present', 'absent'] + fail_msg: "runai_mode must be off, kai or platform and runai_state must be present or absent" + quiet: true + +- name: Skip Run:ai (runai_mode is off) + ansible.builtin.debug: + msg: "runai_mode is off; nothing to do. Set runai_mode to kai or platform in config/group_vars/k8s_cluster.yml." + when: runai_mode == 'off' and runai_state == 'present' + +- name: Install Run:ai + ansible.builtin.include_tasks: install.yml + when: runai_mode != 'off' and runai_state == 'present' + +- name: Remove Run:ai + ansible.builtin.include_tasks: uninstall.yml + when: runai_state == 'absent' diff --git a/roles/runai/tasks/platform.yml b/roles/runai/tasks/platform.yml new file mode 100644 index 000000000..b126264c6 --- /dev/null +++ b/roles/runai/tasks/platform.yml @@ -0,0 +1,54 @@ +--- +# Run:ai control plane + cluster on one Kubernetes cluster ("same-cluster"). +# Experimental in 26.11: ships only if it passes a live run (see docs/k8s-cluster/runai.md). +- name: Check platform settings + ansible.builtin.assert: + that: + - runai_platform_domain | length > 0 + - runai_platform_domain is match('^[A-Za-z0-9]([-A-Za-z0-9.]*[A-Za-z0-9])?$') + - runai_platform_admin_user is match('^[^@\s]+@[^@\s]+$') + - runai_platform_admin_password | runai_password_ok + - (runai_platform_tls_cert_file | length > 0) == (runai_platform_tls_key_file | length > 0) + fail_msg: >- + Platform mode needs runai_platform_domain (a DNS name), runai_platform_admin_user (an email + address) and runai_platform_admin_password (8+ characters with a digit, lower case, upper + case and one of -!@#$%^&*()_+=). Keep the password in an Ansible vault. + A certificate file needs its key file too. + quiet: true + no_log: true + +- name: Check prerequisites the cluster must already provide + ansible.builtin.include_tasks: platform_preflight.yml + +- name: Install the HAProxy ingress controller + ansible.builtin.include_tasks: helm_release.yml + vars: + _rel: + release: "{{ runai_platform_ingress_release_name }}" + namespace: "{{ runai_platform_ingress_namespace }}" + chart: "{{ runai_platform_ingress_chart }}" + repo: "{{ runai_platform_ingress_repo }}" + version: "{{ runai_platform_ingress_chart_version }}" + helm_values: "{{ runai_platform_ingress_values }}" + when: runai_platform_install_ingress | bool + +- name: Prepare the domain and certificates + ansible.builtin.include_tasks: platform_tls.yml + +- name: Make the domain resolve + ansible.builtin.include_tasks: platform_dns.yml + +- name: Install the Run:ai control plane + ansible.builtin.include_tasks: helm_release.yml + vars: + _rel: + release: "{{ runai_cp_release_name }}" + namespace: "{{ runai_cp_namespace }}" + chart: "{{ runai_cp_chart }}" + repo: "{{ runai_chart_repo }}" + version: "{{ runai_chart_version }}" + helm_values: "{{ runai_cp_values }}" + sensitive: true + +- name: Register this cluster and install the Run:ai cluster + ansible.builtin.include_tasks: platform_cluster.yml diff --git a/roles/runai/tasks/platform_cluster.yml b/roles/runai/tasks/platform_cluster.yml new file mode 100644 index 000000000..a63b64f5d --- /dev/null +++ b/roles/runai/tasks/platform_cluster.yml @@ -0,0 +1,55 @@ +--- +- name: Check whether the Run:ai cluster is already installed + ansible.builtin.command: "{{ runai_helm }} list --namespace {{ runai_cluster_namespace }} --output json" + register: runai_cluster_list + changed_when: false + check_mode: false + +- name: Note the installed cluster release + ansible.builtin.set_fact: + runai_cluster_current: "{{ (runai_cluster_list.stdout | from_json | selectattr('name', 'equalto', runai_cluster_release_name) | list | first) | default({}) }}" + +- name: Copy the API helper to the host + ansible.builtin.copy: + src: runai_api.py + dest: "{{ runai_work_dir }}/runai_api.py" + owner: root + group: root + mode: "0700" + become: true + +# The install token is short-lived (two hours) and is fetched only when the +# cluster release must be created or changed. +- name: Register the cluster and fetch its install settings + ansible.builtin.command: + argv: >- + {{ ['python3', runai_work_dir ~ '/runai_api.py', + '--domain', runai_platform_domain, + '--cluster-name', runai_cluster_name, + '--chart-version', runai_chart_version] + + (['--resolve-ip', runai_platform_ingress_ip] if runai_platform_ingress_ip | length > 0 else []) + + (['--ca-file', runai_tls_ca] if runai_tls_ca | length > 0 else []) }} + environment: + RUNAI_USERNAME: "{{ runai_platform_admin_user }}" + RUNAI_PASSWORD: "{{ runai_platform_admin_password }}" + register: runai_api + changed_when: (runai_api.stdout | from_json).created + no_log: true + become: true + when: >- + runai_cluster_current | length == 0 + or runai_cluster_current.status | default('') != 'deployed' + or runai_cluster_current.chart | default('') != 'runai-cluster-' ~ runai_chart_version + +- name: Install the Run:ai cluster + ansible.builtin.include_tasks: helm_release.yml + vars: + _rel: + release: "{{ runai_cluster_release_name }}" + namespace: "{{ runai_cluster_namespace }}" + chart: "{{ runai_cluster_chart }}" + repo: "{{ runai_chart_repo }}" + version: "{{ runai_chart_version }}" + helm_values: "{{ runai_cluster_base_values | runai_deep_merge((runai_api.stdout | from_json).values) | runai_deep_merge(runai_platform_cluster_extra_values) }}" + sensitive: true + when: runai_api is not skipped diff --git a/roles/runai/tasks/platform_dns.yml b/roles/runai/tasks/platform_dns.yml new file mode 100644 index 000000000..d6970f43a --- /dev/null +++ b/roles/runai/tasks/platform_dns.yml @@ -0,0 +1,22 @@ +--- +# Both are off by default and meant for labs without a DNS record for the domain. +- name: Find the address the domain should resolve to + ansible.builtin.assert: + that: runai_platform_ingress_ip | length > 0 + fail_msg: Set runai_platform_ingress_ip to the address the ingress controller answers on. + quiet: true + when: runai_platform_manage_hosts_entry | bool or runai_platform_coredns_rewrite | bool + +- name: Resolve the domain on the control-plane host + ansible.builtin.lineinfile: + path: /etc/hosts + regexp: '^\S+\s+{{ runai_platform_domain | regex_escape }}\s*# deepops-runai$' + line: "{{ runai_platform_ingress_ip }} {{ runai_platform_domain }} # deepops-runai" + become: true + when: runai_platform_manage_hosts_entry | bool + +- name: Resolve the domain inside the cluster + ansible.builtin.include_tasks: coredns.yml + vars: + runai_coredns_present: true + when: runai_platform_coredns_rewrite | bool diff --git a/roles/runai/tasks/platform_preflight.yml b/roles/runai/tasks/platform_preflight.yml new file mode 100644 index 000000000..389cd396c --- /dev/null +++ b/roles/runai/tasks/platform_preflight.yml @@ -0,0 +1,34 @@ +--- +- name: Find the default storage class + ansible.builtin.command: "{{ runai_kubectl }} get storageclass --output json" + register: runai_sc + changed_when: false + check_mode: false + +- name: List default storage classes + ansible.builtin.set_fact: + runai_default_sc: "{{ runai_sc.stdout | runai_default_storageclasses }}" + +- name: Require a default storage class (the control plane needs persistent volumes) + ansible.builtin.assert: + that: + - runai_default_sc | length > 0 + fail_msg: >- + No default StorageClass. Run playbooks/k8s-cluster/nfs-client-provisioner.yml (DeepOps) or + mark a site StorageClass as default, then rerun. + quiet: true + +- name: Find kube-prometheus-stack + ansible.builtin.set_fact: + runai_prom_chart: "{{ (runai_releases | selectattr('chart', 'match', '^kube-prometheus-stack-') | map(attribute='chart') | list | first) | default('') }}" + +- name: Require kube-prometheus-stack at the version Run:ai needs + ansible.builtin.assert: + that: + - runai_prom_chart | length > 0 + - runai_prom_chart | regex_replace('^kube-prometheus-stack-', '') is version(runai_platform_prometheus_min_version, '>=') + fail_msg: >- + Run:ai 3.0 needs kube-prometheus-stack {{ runai_platform_prometheus_min_version }} or newer + (found: {{ runai_prom_chart | default('none', true) }}). Install it with scripts/k8s/deploy_monitoring.sh + using HELM_PROMETHEUS_CHART_VERSION={{ runai_platform_prometheus_min_version }} or newer. + quiet: true diff --git a/roles/runai/tasks/platform_tls.yml b/roles/runai/tasks/platform_tls.yml new file mode 100644 index 000000000..50f845ce0 --- /dev/null +++ b/roles/runai/tasks/platform_tls.yml @@ -0,0 +1,80 @@ +--- +- name: Create the working directory (private) + ansible.builtin.file: + path: "{{ runai_work_dir }}" + state: directory + owner: root + group: root + mode: "0700" + become: true + +- name: Generate a local CA and a certificate for the domain (lab use) + ansible.builtin.shell: | + set -euo pipefail + cd {{ runai_work_dir | quote }} + if [ ! -s ca.pem ]; then + openssl req -x509 -newkey rsa:3072 -nodes -days 825 -subj "/CN=DeepOps Run:ai local CA" \ + -keyout ca.key -out ca.pem + fi + if [ ! -s tls.crt ]; then + openssl req -newkey rsa:3072 -nodes -subj "/CN={{ runai_platform_domain }}" -keyout tls.key -out tls.csr + printf 'subjectAltName=DNS:{{ runai_platform_domain }},DNS:*.{{ runai_platform_domain }}\n' > san.ext + openssl x509 -req -in tls.csr -CA ca.pem -CAkey ca.key -CAcreateserial -days 825 \ + -extfile san.ext -out tls.crt + rm -f tls.csr san.ext + echo generated + fi + args: + executable: /bin/bash + register: runai_gen_tls + changed_when: "'generated' in runai_gen_tls.stdout" + become: true + when: runai_platform_tls_cert_file | length == 0 + +- name: Choose the certificate files + ansible.builtin.set_fact: + runai_tls_cert: "{{ runai_platform_tls_cert_file if runai_platform_tls_cert_file | length > 0 else runai_work_dir ~ '/tls.crt' }}" + runai_tls_key: "{{ runai_platform_tls_key_file if runai_platform_tls_key_file | length > 0 else runai_work_dir ~ '/tls.key' }}" + runai_tls_ca: "{{ runai_platform_ca_file if runai_platform_ca_file | length > 0 else (runai_work_dir ~ '/ca.pem' if runai_platform_tls_cert_file | length == 0 else '') }}" + +- name: Create the namespaces + ansible.builtin.shell: "{{ runai_kubectl }} create namespace {{ item }} --dry-run=client -o yaml | {{ runai_kubectl }} apply -f -" + args: + executable: /bin/bash + loop: ["{{ runai_cp_namespace }}", "{{ runai_cluster_namespace }}"] + register: runai_ns + changed_when: "'created' in runai_ns.stdout" + +- name: Create the TLS secret for the control plane + ansible.builtin.shell: | + set -euo pipefail + {{ runai_kubectl }} -n {{ runai_cp_namespace }} create secret tls runai-backend-tls \ + --cert {{ runai_tls_cert | quote }} --key {{ runai_tls_key | quote }} --dry-run=client -o yaml \ + | {{ runai_kubectl }} apply -f - + args: + executable: /bin/bash + register: runai_tls_secret + changed_when: "'created' in runai_tls_secret.stdout or 'configured' in runai_tls_secret.stdout" + no_log: true + become: true + +- name: Create the CA secrets (control plane and cluster) + ansible.builtin.shell: | + set -euo pipefail + {{ runai_kubectl }} -n {{ item }} create secret generic runai-ca-cert \ + --from-file=runai-ca.pem={{ runai_tls_ca | quote }} --dry-run=client -o yaml | {{ runai_kubectl }} apply -f - + args: + executable: /bin/bash + loop: ["{{ runai_cp_namespace }}", "{{ runai_cluster_namespace }}"] + register: runai_ca_secret + changed_when: "'created' in runai_ca_secret.stdout or 'configured' in runai_ca_secret.stdout" + become: true + when: runai_tls_ca | length > 0 + +- name: Label the cluster CA secret as cluster-wide + ansible.builtin.command: >- + {{ runai_kubectl }} -n {{ runai_cluster_namespace }} label secret runai-ca-cert + run.ai/cluster-wide=true run.ai/name=runai-ca-cert --overwrite + register: runai_ca_label + changed_when: "'not labeled' not in runai_ca_label.stdout" + when: runai_tls_ca | length > 0 diff --git a/roles/runai/tasks/preflight.yml b/roles/runai/tasks/preflight.yml new file mode 100644 index 000000000..2639e1aa9 --- /dev/null +++ b/roles/runai/tasks/preflight.yml @@ -0,0 +1,61 @@ +--- +- name: Find Helm and the Kubernetes API + ansible.builtin.command: "{{ runai_kubectl }} version --output json" + register: runai_kube_version + changed_when: false + check_mode: false + +- name: List installed Helm releases + ansible.builtin.command: "{{ runai_helm }} list --all-namespaces --output json" + register: runai_all_releases + changed_when: false + check_mode: false + +- name: Note installed Helm releases + ansible.builtin.set_fact: + runai_releases: "{{ runai_all_releases.stdout | from_json }}" + +- name: Check for GPUs + ansible.builtin.command: "{{ runai_kubectl }} get nodes --output json" + register: runai_nodes + changed_when: false + check_mode: false + no_log: true + +- name: Count allocatable GPUs + ansible.builtin.set_fact: + runai_gpu_count: "{{ runai_nodes.stdout | runai_gpu_count }}" + +- name: Require the GPU stack (GPU Operator or device plugin) + ansible.builtin.assert: + that: + - runai_gpu_count | int > 0 + fail_msg: >- + No node advertises nvidia.com/gpu. Install the GPU stack first + (ansible-playbook playbooks/k8s-cluster/nvidia-gpu-operator.yml) and wait for the + node to report GPUs. Check with: kubectl get nodes -o json | grep nvidia.com/gpu + quiet: true + when: runai_require_gpu | bool + +- name: Refuse two KAI Schedulers (kai mode) + ansible.builtin.assert: + that: runai_releases | selectattr('chart', 'match', '^runai-cluster-') | list | length == 0 + fail_msg: >- + The Run:ai cluster chart is installed and it already ships its own KAI Scheduler. + Do not add standalone KAI beside it. Uninstall one of them (runai_state: absent) first. + quiet: true + when: runai_mode == 'kai' + +- name: Refuse two KAI Schedulers (platform mode) + ansible.builtin.assert: + that: runai_releases | selectattr('name', 'equalto', runai_kai_release_name) | list | length == 0 + fail_msg: >- + A standalone KAI Scheduler release ({{ runai_kai_release_name }}) is installed and the + Run:ai cluster chart ships its own KAI. Run this role with runai_mode: kai and + runai_state: absent first, then install platform mode. + quiet: true + when: runai_mode == 'platform' + +- name: Check GPU memory fractioning prerequisites + ansible.builtin.include_tasks: fractioning_preflight.yml + when: runai_mode == 'kai' and runai_kai_gpu_fractioning | bool diff --git a/roles/runai/tasks/uninstall.yml b/roles/runai/tasks/uninstall.yml new file mode 100644 index 000000000..48e47b0e5 --- /dev/null +++ b/roles/runai/tasks/uninstall.yml @@ -0,0 +1,71 @@ +--- +# Removes only what this role installs. Leaves the GPU stack alone. +- name: Remove the Run:ai cluster + ansible.builtin.command: "{{ runai_helm }} uninstall {{ runai_cluster_release_name }} --namespace {{ runai_cluster_namespace }} --wait --timeout {{ runai_helm_timeout }}" + register: runai_rm_cluster + changed_when: runai_rm_cluster.rc == 0 + failed_when: runai_rm_cluster.rc != 0 and 'not found' not in runai_rm_cluster.stderr + +- name: Remove the Run:ai control plane + ansible.builtin.command: "{{ runai_helm }} uninstall {{ runai_cp_release_name }} --namespace {{ runai_cp_namespace }} --wait --timeout {{ runai_helm_timeout }}" + register: runai_rm_cp + changed_when: runai_rm_cp.rc == 0 + failed_when: runai_rm_cp.rc != 0 and 'not found' not in runai_rm_cp.stderr + +- name: Remove KAI Scheduler + ansible.builtin.command: "{{ runai_helm }} uninstall {{ runai_kai_release_name }} --namespace {{ runai_kai_namespace }} --wait --timeout {{ runai_helm_timeout }}" + register: runai_rm_kai + changed_when: runai_rm_kai.rc == 0 + failed_when: runai_rm_kai.rc != 0 and 'not found' not in runai_rm_kai.stderr + +- name: Remove the HAProxy ingress controller this role installed + ansible.builtin.command: "{{ runai_helm }} uninstall {{ runai_platform_ingress_release_name }} --namespace {{ runai_platform_ingress_namespace }} --wait" + register: runai_rm_ingress + changed_when: runai_rm_ingress.rc == 0 + failed_when: runai_rm_ingress.rc != 0 and 'not found' not in runai_rm_ingress.stderr + when: runai_platform_install_ingress | bool + +- name: Remove the CoreDNS rewrite + ansible.builtin.include_tasks: coredns.yml + vars: + runai_coredns_present: false + when: runai_platform_coredns_rewrite | bool + +- name: Remove the hosts-file entry + ansible.builtin.lineinfile: + path: /etc/hosts + regexp: '^\S+\s+{{ runai_platform_domain | regex_escape }}\s*# deepops-runai$' + state: absent + when: runai_platform_manage_hosts_entry | bool and runai_platform_domain | length > 0 + +# Helm keeps these on purpose (resource-policy: keep); an uninstall that leaves +# them behind is not clean, so remove them here. +- name: Remove namespaces left behind + ansible.builtin.command: "{{ runai_kubectl }} delete namespace {{ item }} --ignore-not-found --wait --timeout=300s" + loop: + - "{{ runai_kai_namespace }}" + - kai-resource-reservation + - runai-reservation + - runai-scale-adjust + - "{{ runai_cluster_namespace }}" + - "{{ runai_cp_namespace }}" + - "{{ runai_platform_ingress_namespace if runai_platform_install_ingress | bool else omit }}" + register: runai_rm_ns + changed_when: "'deleted' in runai_rm_ns.stdout" + when: item is not none + +- name: List CRDs from these charts + ansible.builtin.shell: >- + set -o pipefail; + {{ runai_kubectl }} get crd --output name | grep -E '(\.run\.ai|\.kai\.scheduler|scheduling\.run\.ai|gpu-fractioning\.kai\.scheduler|\.karta\.)' || true + args: + executable: /bin/bash + register: runai_left_crds + changed_when: false + check_mode: false + +- name: Remove CRDs from these charts (runai_uninstall_remove_crds) + ansible.builtin.command: "{{ runai_kubectl }} delete {{ item }} --ignore-not-found" + loop: "{{ runai_left_crds.stdout_lines }}" + changed_when: true + when: runai_uninstall_remove_crds | bool diff --git a/roles/runai/vars/main.yml b/roles/runai/vars/main.yml new file mode 100644 index 000000000..5ab6d9e9e --- /dev/null +++ b/roles/runai/vars/main.yml @@ -0,0 +1,45 @@ +--- +# Values the role builds from your settings. Not meant to be overridden; use +# runai_kai_extra_values / runai_platform_*_extra_values instead. +runai_kai_components: [kai-operator, binder, queue-controller, admission, kai-scheduler-default] + +runai_kai_base_values: + global: + gpuSharingMode: "{{ 'NvFractions' if runai_kai_gpu_fractioning | bool else 'NonMemoryEnforced' }}" + nvFractions: + set: "{{ runai_kai_gpu_fractioning | bool }}" +runai_kai_values: "{{ runai_kai_base_values | runai_deep_merge(runai_kai_extra_values) }}" + +# ------------------------------------------------------------------ platform +runai_platform_ingress_values: + controller: + ingressClass: "{{ runai_platform_ingress_class }}" + ingressClassResource: + name: "{{ runai_platform_ingress_class }}" + service: + type: "{{ runai_platform_ingress_service_type }}" + kind: "{{ 'DaemonSet' if runai_platform_ingress_host_ports | bool else 'Deployment' }}" + daemonset: + useHostPort: "{{ runai_platform_ingress_host_ports | bool }}" + +runai_cp_base_values: + global: + domain: "{{ runai_platform_domain }}" + ingress: + ingressClass: "{{ runai_platform_ingress_class }}" + customCA: + enabled: "{{ (runai_tls_ca | default('')) | length > 0 }}" + tenantsManager: + config: + adminUsername: "{{ runai_platform_admin_user }}" + adminPassword: "{{ runai_platform_admin_password }}" +runai_cp_values: "{{ runai_cp_base_values | runai_deep_merge(runai_platform_cp_extra_values) }}" + +runai_cluster_base_values: + clusterConfig: + global: + ingress: + ingressClass: "{{ runai_platform_ingress_class }}" + global: + customCA: + enabled: "{{ (runai_tls_ca | default('')) | length > 0 }}" diff --git a/scripts/validation/tests/test_runai.py b/scripts/validation/tests/test_runai.py new file mode 100644 index 000000000..fdacaede8 --- /dev/null +++ b/scripts/validation/tests/test_runai.py @@ -0,0 +1,326 @@ +"""Offline tests for validate_runai.py (standard library only). + +Run with: python3 -m unittest discover scripts/validation/tests +A fake Kube object replaces kubectl, so no cluster is needed. These prove the +verdict logic; they do not prove a real GPU job ran (that needs a live run). +""" + +import importlib.util +import json +import os +import unittest + +SCRIPTS_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +spec = importlib.util.spec_from_file_location( + "validate_runai", os.path.join(SCRIPTS_DIR, "validate_runai.py")) +vr = importlib.util.module_from_spec(spec) +spec.loader.exec_module(vr) + + +def deploy(name, want=1, have=1): + return {"metadata": {"name": name}, "spec": {"replicas": want}, "status": {"availableReplicas": have}} + + +HEALTHY = {"items": [deploy(n) for n in + ("kai-operator", "binder", "queue-controller", "admission", "kai-scheduler-default")]} + + +def pod(name, node="n1", group="g1", pg="pg-" + "x"): + return {"metadata": {"name": name, "labels": {"runai-gpu-group": group} if group else {}, + "annotations": {"pod-group-name": pg} if pg else {}}, + "spec": {"nodeName": node}} + + +class FakeKube: + """Scripted stand-in for the Kube wrapper.""" + + def __init__(self, deployments=HEALTHY, crd=True, pods=None, phases=None, logs=None, + apply_rc=0, ns_rc=0, namespaces=None, nodes=True): + self.deployments, self.crd, self.pods = deployments, crd, pods or {} + self.phases, self.logged, self.apply_rc, self.ns_rc = phases or {}, logs or {}, apply_rc, ns_rc + self.namespaces, self.nodes = namespaces or {"items": []}, nodes + self.applied, self.deleted = [], [] + + def get_json(self, args, timeout=60): + if args[:2] == ["get", "nodes"]: + return ({"items": [{}]}, "") if self.nodes else (None, "refused") + if args[:2] == ["get", "namespaces"]: + return self.namespaces, "" + if "deployments" in args: + return (self.deployments, "") if self.deployments is not None else (None, "forbidden") + if "crd" in args: + return ({}, "") if self.crd else (None, "not found") + if "pod" in args: + return self.pods.get(args[-1], {}), "" + return None, "unexpected %s" % args + + def apply(self, manifest): + self.applied.append(manifest) + return self.apply_rc, "", "denied" if self.apply_rc else "" + + def wait_phase(self, namespace, pod_name, phases, timeout): + return self.phases.get(pod_name, "Succeeded") + + def logs(self, namespace, pod_name): + return self.logged.get(pod_name, "GPU 0: Test GPU (UUID: x)") + + def ensure_namespace(self, namespace): + return self.ns_rc, "", "" + + def delete_namespace(self, namespace): + self.deleted.append(namespace) + + +def args(*extra): + return vr.parse_args(list(extra)) + + +GOOD_PODS = {"kai-whole-gpu": pod("kai-whole-gpu"), "kai-half-a": pod("kai-half-a"), + "kai-half-b": pod("kai-half-b")} + + +class TestParsers(unittest.TestCase): + def test_deployments_flag_unavailable(self): + names, bad = vr.summarize_deployments({"items": [deploy("binder", 1, 0), deploy("x", 0, 0)]}) + self.assertEqual(names, ["binder", "x"]) + self.assertEqual(bad, ["binder (0/1 available)"]) + + def test_missing_deployments_needs_a_scheduler_shard(self): + self.assertEqual(vr.missing_deployments(["kai-operator", "binder", "queue-controller", "admission"]), + ["kai-scheduler-"]) + self.assertEqual(vr.missing_deployments(list(vr.REQUIRED_DEPLOYMENTS) + ["kai-scheduler-default"]), []) + + def test_same_gpu_requires_node_and_group(self): + self.assertTrue(vr.same_gpu([("n1", "g", "p"), ("n1", "g", "p")])) + self.assertFalse(vr.same_gpu([("n1", "g", "p"), ("n1", "h", "p")])) + self.assertFalse(vr.same_gpu([("n1", "g", "p"), ("n2", "g", "p")])) + self.assertFalse(vr.same_gpu([("n1", "", "p"), ("n1", "", "p")])) + self.assertFalse(vr.same_gpu([("", "g", "p"), ("", "g", "p")])) + self.assertFalse(vr.same_gpu([("n1", "g", "p")])) + + def test_leftovers(self): + ns = {"items": [{"metadata": {"name": n}} for n in + ("default", "kai-scheduler", "runai", "runai-backend", "kai-resource-reservation", "runaix-ok")]} + rel = [{"name": "kai-scheduler", "namespace": "kai-scheduler", "chart": "kai-scheduler-v0.18.5"}, + {"name": "gpu", "namespace": "g", "chart": "gpu-operator-v26.7.0"}] + found = vr.leftovers(ns, rel) + self.assertIn("namespace/runai", found) + self.assertIn("namespace/runai-backend", found) + self.assertIn("namespace/kai-resource-reservation", found) + self.assertIn("release/kai-scheduler/kai-scheduler", found) + self.assertNotIn("namespace/default", found) + self.assertFalse([f for f in found if "gpu-operator" in f or "/gpu" in f]) + self.assertEqual(vr.leftovers({"items": []}, []), []) + + def test_cluster_connected(self): + doc = {"clusters": [{"name": "deepops", "status": {"platform": "Connected"}}]} + self.assertEqual(vr.cluster_connected(doc, "deepops"), (True, True)) + doc["clusters"][0]["status"]["platform"] = "WaitingToConnect" + self.assertEqual(vr.cluster_connected(doc, "deepops"), (True, False)) + self.assertEqual(vr.cluster_connected([], "deepops"), (False, False)) + + def test_pod_manifest_uses_kai_and_queue(self): + m = vr.pod_manifest("p", "ns", "img", ["a"], [], "kai-scheduler", "kai.scheduler/queue", "q", + gpu_limit=1, annotations={"gpu-fraction": "0.5"}) + self.assertEqual(m["spec"]["schedulerName"], "kai-scheduler") + self.assertEqual(m["metadata"]["labels"], {"kai.scheduler/queue": "q"}) + self.assertEqual(m["spec"]["containers"][0]["resources"]["limits"]["nvidia.com/gpu"], 1) + self.assertFalse(m["spec"]["automountServiceAccountToken"]) + + +class TestVerdict(unittest.TestCase): + def test_all_good_is_ok_and_cleans_up(self): + kube = FakeKube(pods=GOOD_PODS) + result = vr.validate(args(), {}, kube) + self.assertTrue(result["ok"], result["failures"]) + self.assertTrue(result["gpu_job_ok"] and result["fraction_job_ok"]) + self.assertEqual(kube.deleted, ["deepops-runai-validate"]) + self.assertEqual(json.loads(json.dumps(result))["schema_version"], 1) + # the two fractional pods ask for half a GPU and must not also ask for a whole one + halves = [m for m in kube.applied if m["metadata"]["name"].startswith("kai-half")] + self.assertEqual(len(halves), 2) + for m in halves: + self.assertEqual(m["metadata"]["annotations"]["gpu-fraction"], "0.5") + self.assertNotIn("resources", m["spec"]["containers"][0]) + + def test_no_cluster_is_not_ok(self): + result = vr.validate(args(), {}, FakeKube(nodes=False)) + self.assertFalse(result["ok"]) + self.assertFalse(result["api_reachable"]) + + def test_unready_scheduler_is_not_ok(self): + bad = {"items": [deploy("kai-operator", 1, 0)] + HEALTHY["items"][1:]} + result = vr.validate(args(), {}, FakeKube(deployments=bad, pods=GOOD_PODS)) + self.assertFalse(result["ok"]) + self.assertFalse(result["gpu_job_ran"]) # does not run jobs on a broken scheduler + + def test_missing_queue_crd_is_not_ok(self): + result = vr.validate(args(), {}, FakeKube(crd=False, pods=GOOD_PODS)) + self.assertFalse(result["ok"]) + + def test_whole_gpu_failure(self): + kube = FakeKube(pods=GOOD_PODS, phases={"kai-whole-gpu": "Failed"}) + result = vr.validate(args(), {}, kube) + self.assertFalse(result["ok"]) + self.assertIn("namespace deepops-runai-validate kept for debugging", result["failures"]) + self.assertEqual(kube.deleted, []) + + def test_whole_gpu_without_a_kai_pod_group_is_not_ok(self): + pods = dict(GOOD_PODS, **{"kai-whole-gpu": pod("kai-whole-gpu", pg=None)}) + result = vr.validate(args(), {}, FakeKube(pods=pods)) + self.assertFalse(result["gpu_job_ok"]) + self.assertFalse(result["ok"]) + + def test_default_scheduler_pod_is_not_proof(self): + # nvidia-smi worked but KAI never grouped the pod: it was not scheduled by KAI + pods = dict(GOOD_PODS, **{"kai-whole-gpu": pod("kai-whole-gpu", group="", pg=None)}) + self.assertFalse(vr.validate(args(), {}, FakeKube(pods=pods))["ok"]) + + def test_halves_on_different_gpus_is_not_ok(self): + pods = dict(GOOD_PODS, **{"kai-half-b": pod("kai-half-b", group="g2")}) + result = vr.validate(args(), {}, FakeKube(pods=pods)) + self.assertFalse(result["fraction_job_ok"]) + self.assertFalse(result["ok"]) + + def test_halves_that_never_start_is_not_ok(self): + result = vr.validate(args(), {}, FakeKube(pods=GOOD_PODS, phases={"kai-half-b": "Pending"})) + self.assertFalse(result["ok"]) + + def test_pod_without_gpu_output_is_not_ok(self): + kube = FakeKube(pods=GOOD_PODS, logs={"kai-whole-gpu": "no devices"}) + self.assertFalse(vr.validate(args(), {}, kube)["ok"]) + + def test_apply_failure_is_not_ok(self): + self.assertFalse(vr.validate(args(), {}, FakeKube(apply_rc=1, pods=GOOD_PODS))["ok"]) + + def test_namespace_failure_is_not_ok(self): + self.assertFalse(vr.validate(args(), {}, FakeKube(ns_rc=1, pods=GOOD_PODS))["ok"]) + + def test_skip_gpu_jobs_is_scheduler_only(self): + result = vr.validate(args("--skip-gpu-jobs"), {}, FakeKube()) + self.assertTrue(result["ok"]) + self.assertFalse(result["gpu_job_ran"]) + + +class TestMemoryEnforcement(unittest.TestCase): + def pods(self): + return dict(GOOD_PODS) + + def test_inside_ok_and_outside_refused_is_ok(self): + kube = FakeKube(pods=self.pods(), phases={"kai-mem-outside": "Failed"}, + logs={"kai-mem-inside": "ALLOC_OK 256", "kai-mem-outside": "ALLOC_FAILED alloc rc=2"}) + result = vr.validate(args("--memory-enforcement"), {}, kube) + self.assertTrue(result["ok"], result["failures"]) + self.assertTrue(result["memory_enforcement_ok"]) + outside = [m for m in kube.applied if m["metadata"]["name"] == "kai-mem-outside"][0] + self.assertEqual(outside["metadata"]["annotations"][ + "nvidia.com/container.gpu-workload.gpu-memory.limit"], "1024Mi") + + def test_outside_allowed_to_allocate_is_not_ok(self): + kube = FakeKube(pods=self.pods(), + logs={"kai-mem-inside": "ALLOC_OK 256", "kai-mem-outside": "ALLOC_OK 4096"}) + result = vr.validate(args("--memory-enforcement"), {}, kube) + self.assertFalse(result["ok"]) + self.assertFalse(result["memory_enforcement_ok"]) + + def test_outside_killed_for_another_reason_is_not_proof(self): + kube = FakeKube(pods=self.pods(), phases={"kai-mem-outside": "Failed"}, + logs={"kai-mem-inside": "ALLOC_OK 256", "kai-mem-outside": "ImportError"}) + self.assertFalse(vr.validate(args("--memory-enforcement"), {}, kube)["ok"]) + + def test_inside_failing_is_not_ok(self): + kube = FakeKube(pods=self.pods(), phases={"kai-mem-inside": "Failed"}, + logs={"kai-mem-inside": "ALLOC_FAILED alloc rc=2"}) + self.assertFalse(vr.validate(args("--memory-enforcement"), {}, kube)["ok"]) + + def test_off_by_default(self): + result = vr.validate(args(), {}, FakeKube(pods=GOOD_PODS)) + self.assertFalse(result["memory_enforcement_ran"]) + + +class TestPlatformAndUninstall(unittest.TestCase): + def test_platform_needs_a_domain(self): + with self.assertRaises(SystemExit): + args("--mode", "platform") + + def test_platform_without_credentials_is_not_ok(self): + orig = vr.https_request + vr.https_request = lambda *a, **k: (200, b"") + try: + result = vr.validate(args("--mode", "platform", "--platform-domain", "runai.lab.test"), + {}, FakeKube(pods=GOOD_PODS)) + finally: + vr.https_request = orig + self.assertFalse(result["ok"]) + self.assertTrue(result["platform_api_ok"]) + self.assertFalse(result["platform_cluster_connected"]) + self.assertTrue(any("RUNAI_USERNAME" in f for f in result["failures"])) + + def _platform(self, clusters_status): + calls = [] + + def fake(domain, ip, ca, method, path, body=None, headers=None, timeout=20): + calls.append((method, path)) + if path == "/": + return 200, b"" + if path == "/api/v2/token": + return 200, json.dumps({"access_token": "t"}).encode() + if path == "/api/v1/clusters": + return 200, json.dumps({"clusters": [{"name": "deepops", "status": {"platform": clusters_status}}]}).encode() + return 404, b"" + orig = vr.https_request + vr.https_request = fake + try: + result = vr.validate(args("--mode", "platform", "--platform-domain", "runai.lab.test"), + {"RUNAI_USERNAME": "u", "RUNAI_PASSWORD": "p"}, FakeKube(pods=GOOD_PODS)) + finally: + vr.https_request = orig + return result + + def test_platform_connected_is_ok(self): + result = self._platform("Connected") + self.assertTrue(result["ok"], result["failures"]) + self.assertTrue(result["platform_cluster_connected"]) + + def test_platform_waiting_is_not_ok(self): + result = self._platform("WaitingToConnect") + self.assertFalse(result["ok"]) + + def test_platform_credentials_never_in_output(self): + self.assertNotIn('"p"', json.dumps(self._platform("Connected"))) + + def test_uninstalled_clean(self): + orig = vr.run + vr.run = lambda cmd, timeout=60, input_text=None: (0, "[]", "") + try: + result = vr.validate(args("--verify-uninstalled"), {}, FakeKube()) + finally: + vr.run = orig + self.assertTrue(result["ok"], result["failures"]) + self.assertTrue(result["uninstalled_ok"]) + self.assertEqual(result["mode"], "uninstalled") + + def test_uninstalled_finds_leftovers(self): + orig = vr.run + vr.run = lambda cmd, timeout=60, input_text=None: (0, "[]", "") + try: + ns = {"items": [{"metadata": {"name": "runai"}}]} + result = vr.validate(args("--verify-uninstalled"), {}, FakeKube(namespaces=ns)) + finally: + vr.run = orig + self.assertFalse(result["ok"]) + self.assertEqual(result["leftovers"], ["namespace/runai"]) + + +class TestArgs(unittest.TestCase): + def test_bad_namespace_refused(self): + with self.assertRaises(SystemExit): + args("--namespace", "Bad_Name") + + def test_short_timeout_refused(self): + with self.assertRaises(SystemExit): + args("--job-timeout", "1") + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/validation/validate_runai.py b/scripts/validation/validate_runai.py new file mode 100644 index 000000000..3af79e84b --- /dev/null +++ b/scripts/validation/validate_runai.py @@ -0,0 +1,507 @@ +#!/usr/bin/env python3 +"""Validate an optional KAI Scheduler / Run:ai install on a DeepOps Kubernetes cluster. + +Run anywhere ``kubectl`` reaches the cluster after ``playbooks/k8s-cluster/runai.yml``. +The verdict is strict JSON with ``--json``: ``"ok": true`` only if every check that +applies to the selected mode passed. + +Checks (all in a throw-away namespace that is deleted on success): + +1. scheduler pods are available and the KAI queue API exists +2. a whole-GPU pod scheduled by KAI runs ``nvidia-smi -L`` and completes +3. two pods asking for half a GPU each land on the same GPU and complete +4. ``--memory-enforcement``: a pod inside its GPU-memory share completes and a pod + that allocates past it is refused (gpu-fractioning only) +5. ``--mode platform``: the control-plane API answers over the configured domain + and the cluster reports ``Connected`` + +``--verify-uninstalled`` is a separate read-only mode: it passes only if no +``kai-scheduler``/``runai*`` namespace or Helm release is left. + +Exit codes: 0 = ok, 1 = not ok, 2 = bad input or kubectl missing. +""" + +import argparse +import http.client +import json +import re +import shutil +import ssl +import subprocess +import sys +import time +import urllib.parse + +SCHEMA_VERSION = 1 +KAI_NAMESPACE = "kai-scheduler" +# The operator names the scheduler Deployment "-" (kai-scheduler-default). +REQUIRED_DEPLOYMENTS = ("kai-operator", "binder", "queue-controller", "admission") +SCHEDULER_PREFIX = "kai-scheduler-" +LEFTOVER_NAMESPACES = re.compile(r"^(kai-scheduler|kai-resource-reservation|runai.*)$") +NAME_RE = re.compile(r"^[a-z0-9]([-a-z0-9]*[a-z0-9])?$") +FQDN_RE = re.compile(r"^[A-Za-z0-9]([-A-Za-z0-9.]*[A-Za-z0-9])?$") + +# Allocates past a GPU-memory share using only the CUDA driver library, so a plain +# Python image is enough. Prints a stable marker the validator can match. +ALLOC_SCRIPT = ( + "import ctypes,sys\n" + "mib=int(sys.argv[1])\n" + "c=ctypes.CDLL('libcuda.so.1')\n" + "def chk(r,w):\n" + " if r!=0:\n" + " print('ALLOC_FAILED %s rc=%d'%(w,r)); sys.exit(7)\n" + "chk(c.cuInit(0),'init')\n" + "d=ctypes.c_int(); chk(c.cuDeviceGet(ctypes.byref(d),0),'device')\n" + "x=ctypes.c_void_p(); chk(c.cuCtxCreate_v2(ctypes.byref(x),0,d),'context')\n" + "p=ctypes.c_ulonglong(); chk(c.cuMemAlloc_v2(ctypes.byref(p),ctypes.c_size_t(mib<<20)),'alloc')\n" + "print('ALLOC_OK %d'%mib)\n" +) + + +def run(cmd, timeout=60, input_text=None): + """Run a command, returning (rc, stdout, stderr) without raising.""" + try: + proc = subprocess.run( + cmd, capture_output=True, text=True, timeout=timeout, input=input_text + ) + return proc.returncode, proc.stdout.strip(), proc.stderr.strip() + except subprocess.TimeoutExpired: + return 124, "", "timeout after %ss" % timeout + except FileNotFoundError: + return 127, "", "command not found: %s" % cmd[0] + + +# ---------------------------------------------------------------- pure parsers + + +def summarize_deployments(deploy_json): + """Return (names, not_available) for a ``kubectl get deployments -o json`` doc.""" + names, bad = [], [] + for item in deploy_json.get("items", []): + name = item.get("metadata", {}).get("name", "") + names.append(name) + want = item.get("spec", {}).get("replicas", 1) + have = item.get("status", {}).get("availableReplicas", 0) or 0 + if want and have < want: + bad.append("%s (%d/%d available)" % (name, have, want)) + return sorted(names), sorted(bad) + + +def missing_deployments(names, required=REQUIRED_DEPLOYMENTS): + missing = [r for r in required if r not in names] + if not any(n.startswith(SCHEDULER_PREFIX) for n in names): + missing.append(SCHEDULER_PREFIX + "") + return missing + + +def pod_placement(pod_json): + """Return (node, gpu_group, pod_group) for a pod, or empty strings.""" + meta = pod_json.get("metadata", {}) + return ( + pod_json.get("spec", {}).get("nodeName", "") or "", + (meta.get("labels") or {}).get("runai-gpu-group", ""), + (meta.get("annotations") or {}).get("pod-group-name", ""), + ) + + +def same_gpu(placements): + """True if every placement has a node and the same non-empty GPU group.""" + if len(placements) < 2: + return False + nodes = {p[0] for p in placements} + groups = {p[1] for p in placements} + return len(nodes) == 1 and "" not in nodes and len(groups) == 1 and "" not in groups + + +def leftovers(namespaces_json, releases_json): + """Names of namespaces/Helm releases that an uninstall should have removed.""" + found = [] + for item in namespaces_json.get("items", []): + name = item.get("metadata", {}).get("name", "") + if LEFTOVER_NAMESPACES.match(name): + found.append("namespace/" + name) + for rel in releases_json or []: + chart = rel.get("chart", "") + if re.match(r"^(kai-scheduler|gpu-fractioning|runai-cluster|control-plane)-", chart): + found.append("release/%s/%s" % (rel.get("namespace", ""), rel.get("name", ""))) + return sorted(found) + + +def cluster_connected(clusters_json, cluster_name): + """Return (found, connected) for a ``GET /api/v1/clusters`` document.""" + items = clusters_json if isinstance(clusters_json, list) else clusters_json.get("clusters", []) + for item in items: + if item.get("name") == cluster_name: + status = item.get("status") or {} + return True, status.get("platform") == "Connected" + return False, False + + +def pod_manifest(name, namespace, image, command, args, scheduler, queue_label, queue, + gpu_limit=0, annotations=None, container="gpu-workload", env=None): + limits = {"nvidia.com/gpu": gpu_limit} if gpu_limit else {} + spec = { + "restartPolicy": "Never", + "schedulerName": scheduler, + "automountServiceAccountToken": False, + "containers": [{"name": container, "image": image, "command": command, "args": args}], + } + if limits: + spec["containers"][0]["resources"] = {"limits": limits} + if env: + spec["containers"][0]["env"] = [{"name": k, "value": v} for k, v in sorted(env.items())] + meta = {"name": name, "namespace": namespace, "labels": {queue_label: queue}} + if annotations: + meta["annotations"] = annotations + return {"apiVersion": "v1", "kind": "Pod", "metadata": meta, "spec": spec} + + +# ------------------------------------------------------------------ kube access + + +class Kube: + """Thin kubectl wrapper; tests replace ``run`` on the module.""" + + def get_json(self, args, timeout=60): + rc, out, err = run(["kubectl"] + args + ["-o", "json"], timeout=timeout) + if rc != 0: + return None, err or "rc=%s" % rc + try: + return json.loads(out), "" + except json.JSONDecodeError: + return None, "unparseable kubectl output" + + def apply(self, manifest): + return run(["kubectl", "apply", "-f", "-"], input_text=json.dumps(manifest)) + + def wait_phase(self, namespace, pod, phases, timeout): + deadline = time.time() + timeout + phase = "" + while time.time() < deadline: + rc, phase, _ = run(["kubectl", "-n", namespace, "get", "pod", pod, + "-o", "jsonpath={.status.phase}"]) + if rc == 0 and phase in phases: + return phase + time.sleep(3) + return phase or "Unknown" + + def logs(self, namespace, pod): + return run(["kubectl", "-n", namespace, "logs", pod])[1] + + def delete_namespace(self, namespace): + run(["kubectl", "delete", "namespace", namespace, "--ignore-not-found", "--wait=false"], + timeout=120) + + def ensure_namespace(self, namespace): + run(["kubectl", "delete", "namespace", namespace, "--ignore-not-found"], timeout=180) + return run(["kubectl", "create", "namespace", namespace]) + + +# ------------------------------------------------------------------- the checks + + +def check_scheduler(kube, result, args): + doc, err = kube.get_json(["-n", args.kai_namespace, "get", "deployments"]) + if doc is None: + result["failures"].append("could not list deployments in %s: %s" % (args.kai_namespace, err)) + return + names, bad = summarize_deployments(doc) + missing = missing_deployments(names) + if missing: + result["failures"].append("scheduler deployments missing: %s" % ", ".join(missing)) + if bad: + result["failures"].append("scheduler deployments not available: %s" % ", ".join(bad)) + crd, _ = kube.get_json(["get", "crd", "queues.scheduling.run.ai"]) + if crd is None: + result["failures"].append("queue CRD queues.scheduling.run.ai not found") + if not missing and not bad and crd is not None: + result["scheduler_ready"] = True + + +def run_pod(kube, result, args, manifest, phases=("Succeeded", "Failed")): + rc, _, err = kube.apply(manifest) + if rc != 0: + result["failures"].append("could not create pod %s: %s" % (manifest["metadata"]["name"], err)) + return "" + return kube.wait_phase(manifest["metadata"]["namespace"], manifest["metadata"]["name"], + phases, args.job_timeout) + + +def check_whole_gpu(kube, result, args): + result["gpu_job_ran"] = True + name = "kai-whole-gpu" + manifest = pod_manifest(name, args.namespace, args.image, ["nvidia-smi"], ["-L"], + args.scheduler_name, args.queue_label_key, args.queue, gpu_limit=1) + phase = run_pod(kube, result, args, manifest) + if not phase: + return + pod, _ = kube.get_json(["-n", args.namespace, "get", "pod", name]) + node, _, group = pod_placement(pod or {}) + logs = kube.logs(args.namespace, name) + if phase != "Succeeded" or "GPU " not in logs: + result["failures"].append("whole-GPU pod did not complete (phase=%s)" % phase) + elif not group: + result["failures"].append("whole-GPU pod ran but KAI did not group it (no pod-group-name)") + else: + result["gpu_job_ok"] = True + result["gpu_job_node"] = node + + +def check_fractions(kube, result, args): + result["fraction_job_ran"] = True + names = ["kai-half-a", "kai-half-b"] + for name in names: + manifest = pod_manifest( + name, args.namespace, args.image, ["sh", "-c"], ["nvidia-smi -L && sleep 30"], + args.scheduler_name, args.queue_label_key, args.queue, + annotations={"gpu-fraction": "0.5"}) + rc, _, err = kube.apply(manifest) + if rc != 0: + result["failures"].append("could not create %s: %s" % (name, err)) + return + for name in names: + phase = kube.wait_phase(args.namespace, name, ("Running", "Succeeded", "Failed"), + args.job_timeout) + if phase not in ("Running", "Succeeded"): + result["failures"].append("fractional pod %s did not start (phase=%s)" % (name, phase)) + return + placements = [] + for name in names: + pod, _ = kube.get_json(["-n", args.namespace, "get", "pod", name]) + placements.append(pod_placement(pod or {})) + if not same_gpu(placements): + result["failures"].append( + "half-GPU pods did not share one GPU (placements=%s)" % [list(p[:2]) for p in placements]) + return + for name in names: + phase = kube.wait_phase(args.namespace, name, ("Succeeded", "Failed"), args.job_timeout) + if phase != "Succeeded" or "GPU " not in kube.logs(args.namespace, name): + result["failures"].append("fractional pod %s did not complete (phase=%s)" % (name, phase)) + return + result["fraction_job_ok"] = True + + +def memory_pod(name, args, request_mib, limit_mib, alloc_mib): + ann = { + "nvidia.com/container.gpu-workload.gpu-memory.request": "%dMi" % request_mib, + "nvidia.com/container.gpu-workload.gpu-memory.limit": "%dMi" % limit_mib, + } + return pod_manifest(name, args.namespace, args.memory_image, ["python3", "-c"], + [ALLOC_SCRIPT, str(alloc_mib)], args.scheduler_name, + args.queue_label_key, args.queue, annotations=ann, + env={"NVIDIA_DRIVER_CAPABILITIES": "compute,utility"}) + + +def check_memory_enforcement(kube, result, args): + result["memory_enforcement_ran"] = True + inside = memory_pod("kai-mem-inside", args, 1024, 1024, 256) + outside = memory_pod("kai-mem-outside", args, 1024, 1024, 4096) + phase = run_pod(kube, result, args, inside) + if phase != "Succeeded" or "ALLOC_OK" not in kube.logs(args.namespace, "kai-mem-inside"): + result["failures"].append("pod inside its GPU-memory share did not complete (phase=%s)" % phase) + return + phase = run_pod(kube, result, args, outside) + logs = kube.logs(args.namespace, "kai-mem-outside") + if phase == "Failed" and "ALLOC_FAILED alloc" in logs: + result["memory_enforcement_ok"] = True + else: + result["failures"].append( + "pod past its GPU-memory share was not refused (phase=%s, alloc_ok=%s)" + % (phase, "ALLOC_OK" in logs)) + + +class ResolvedHTTPS(http.client.HTTPSConnection): + """HTTPS to ``ip`` while sending SNI/Host for ``host`` (like curl --resolve).""" + + def __init__(self, host, ip, port, context, timeout): + super().__init__(host, port, context=context, timeout=timeout) + self._ip = ip + + def connect(self): + sock = socket_create(self._ip, self.port, self.timeout) + self.sock = self._context.wrap_socket(sock, server_hostname=self.host) + + +def socket_create(ip, port, timeout): + import socket + return socket.create_connection((ip, port), timeout) + + +def https_request(domain, ip, ca_file, method, path, body=None, headers=None, timeout=20): + context = ssl.create_default_context(cafile=ca_file) if ca_file else ssl.create_default_context() + conn = ResolvedHTTPS(domain, ip or domain, 443, context, timeout) + try: + conn.request(method, path, body=body, headers=headers or {}) + resp = conn.getresponse() + return resp.status, resp.read(1024 * 1024) + finally: + conn.close() + + +def check_platform(result, args, env): + result["platform_ran"] = True + user, password = env.get("RUNAI_USERNAME", ""), env.get("RUNAI_PASSWORD", "") + try: + status, _ = https_request(args.platform_domain, args.resolve_ip, args.ca_file, "GET", "/") + if status >= 500: + result["failures"].append("control-plane endpoint returned HTTP %d" % status) + return + result["platform_api_ok"] = True + except (OSError, ssl.SSLError, http.client.HTTPException) as exc: + result["failures"].append("control-plane endpoint unreachable: %s" % type(exc).__name__) + return + if not user or not password: + result["failures"].append( + "RUNAI_USERNAME and RUNAI_PASSWORD are required to check cluster status") + return + try: + form = urllib.parse.urlencode( + {"grant_type": "password", "username": user, "password": password}) + status, body = https_request( + args.platform_domain, args.resolve_ip, args.ca_file, "POST", "/api/v2/token", form, + {"Content-Type": "application/x-www-form-urlencoded"}) + token = json.loads(body).get("access_token", "") if status == 200 else "" + if not token: + result["failures"].append("could not obtain an API token (HTTP %d)" % status) + return + status, body = https_request( + args.platform_domain, args.resolve_ip, args.ca_file, "GET", "/api/v1/clusters", + headers={"Authorization": "Bearer " + token}) + found, connected = cluster_connected(json.loads(body), args.cluster_name) if status == 200 else (False, False) + except (OSError, ssl.SSLError, http.client.HTTPException, ValueError) as exc: + result["failures"].append("cluster status request failed: %s" % type(exc).__name__) + return + if not found: + result["failures"].append("cluster %s is not registered" % args.cluster_name) + elif not connected: + result["failures"].append("cluster %s is registered but not Connected" % args.cluster_name) + else: + result["platform_cluster_connected"] = True + + +def new_result(mode): + return { + "schema_version": SCHEMA_VERSION, "ok": False, "mode": mode, + "api_reachable": False, "scheduler_ready": False, + "gpu_job_ran": False, "gpu_job_ok": False, "gpu_job_node": "", + "fraction_job_ran": False, "fraction_job_ok": False, + "memory_enforcement_ran": False, "memory_enforcement_ok": False, + "platform_ran": False, "platform_api_ok": False, "platform_cluster_connected": False, + "uninstalled_ran": False, "uninstalled_ok": False, "leftovers": [], + "failures": [], + } + + +def verify_uninstalled(kube, result): + result["uninstalled_ran"] = True + ns, err = kube.get_json(["get", "namespaces"]) + if ns is None: + result["failures"].append("could not list namespaces: %s" % err) + return + rc, out, _ = run(["helm", "list", "--all-namespaces", "-o", "json"], timeout=60) + releases = [] + if rc == 0 and out: + try: + releases = json.loads(out) + except json.JSONDecodeError: + result["failures"].append("could not parse helm list output") + return + result["leftovers"] = leftovers(ns, releases) + if result["leftovers"]: + result["failures"].append("leftover objects: %s" % ", ".join(result["leftovers"])) + else: + result["uninstalled_ok"] = True + + +def validate(args, env, kube=None): + kube = kube or Kube() + result = new_result("uninstalled" if args.verify_uninstalled else args.mode) + nodes, err = kube.get_json(["get", "nodes"], timeout=120) + if nodes is None: + result["failures"].append("kubectl get nodes failed: %s" % err) + return result + result["api_reachable"] = True + if args.verify_uninstalled: + verify_uninstalled(kube, result) + result["ok"] = result["uninstalled_ok"] and not result["failures"] + return result + check_scheduler(kube, result, args) + if result["scheduler_ready"] and not args.skip_gpu_jobs: + if kube.ensure_namespace(args.namespace)[0] != 0: + result["failures"].append("could not create namespace %s" % args.namespace) + else: + check_whole_gpu(kube, result, args) + check_fractions(kube, result, args) + if args.memory_enforcement: + check_memory_enforcement(kube, result, args) + if args.mode == "platform": + check_platform(result, args, env) + ran_ok = (result["scheduler_ready"] and + (args.skip_gpu_jobs or (result["gpu_job_ok"] and result["fraction_job_ok"])) and + (not args.memory_enforcement or result["memory_enforcement_ok"]) and + (args.mode != "platform" or + (result["platform_api_ok"] and result["platform_cluster_connected"]))) + result["ok"] = bool(ran_ok and not result["failures"]) + if result["ok"] and not args.skip_gpu_jobs: + kube.delete_namespace(args.namespace) + elif not args.skip_gpu_jobs and result["gpu_job_ran"]: + result["failures"].append("namespace %s kept for debugging" % args.namespace) + return result + + +def parse_args(argv): + p = argparse.ArgumentParser(description=__doc__.split("\n")[0], allow_abbrev=False) + p.add_argument("--json", action="store_true", help="emit one JSON object") + p.add_argument("--mode", choices=("kai", "platform"), default="kai") + p.add_argument("--verify-uninstalled", action="store_true", + help="read-only: pass only if no kai-scheduler/runai* namespace or release remains") + p.add_argument("--kai-namespace", default=KAI_NAMESPACE) + p.add_argument("--namespace", default="deepops-runai-validate", + help="namespace for the test pods (deleted on success; in platform mode " + "use the project namespace, e.g. runai-)") + p.add_argument("--queue", default="default-queue") + p.add_argument("--queue-label-key", default="kai.scheduler/queue") + p.add_argument("--scheduler-name", default="kai-scheduler") + p.add_argument("--image", default="nvcr.io/nvidia/cuda:13.0.2-base-ubuntu24.04") + p.add_argument("--memory-image", default="python:3.12-slim", + help="image with python3 for the GPU-memory check") + p.add_argument("--job-timeout", type=int, default=300, help="seconds per pod (default 300)") + p.add_argument("--skip-gpu-jobs", action="store_true", help="scheduler health only") + p.add_argument("--memory-enforcement", action="store_true", + help="also prove gpu-fractioning refuses allocations past a pod's share") + p.add_argument("--platform-domain", default="", help="control-plane FQDN (platform mode)") + p.add_argument("--resolve-ip", default="", help="connect here instead of resolving the domain") + p.add_argument("--ca-file", default="", help="CA bundle that signed the control-plane certificate") + p.add_argument("--cluster-name", default="deepops", help="Run:ai cluster name to look up") + args = p.parse_args(argv) + for label, value in (("namespace", args.namespace), ("kai-namespace", args.kai_namespace)): + if not NAME_RE.match(value): + p.error("--%s must be a valid Kubernetes name" % label) + if args.mode == "platform" and not args.verify_uninstalled: + if not args.platform_domain or not FQDN_RE.match(args.platform_domain): + p.error("--platform-domain is required in platform mode") + if args.job_timeout < 10: + p.error("--job-timeout must be at least 10 seconds") + return args + + +def main(argv=None): + import os + args = parse_args(sys.argv[1:] if argv is None else argv) + if not shutil.which("kubectl"): + print("error: kubectl not found in PATH", file=sys.stderr) + return 2 + result = validate(args, os.environ) + if args.json: + print(json.dumps(result, indent=2, sort_keys=True)) + else: + for key in sorted(result): + if key not in ("failures", "leftovers"): + print("%s=%s" % (key, result[key])) + for failure in result["failures"]: + print("FAIL: %s" % failure) + return 0 if result["ok"] else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/skills/README.md b/skills/README.md index 0d16e8656..a78ddd264 100644 --- a/skills/README.md +++ b/skills/README.md @@ -16,6 +16,7 @@ agent (or person) can simply read the relevant `SKILL.md` before acting. | [provision-with-maas](provision-with-maas/SKILL.md) | Provisioning, tagging, validating, or releasing machines through MAAS. | | [deploy-airgapped](deploy-airgapped/SKILL.md) | Preparing mirrors and deploying Slurm or Kubernetes without Internet access. | | [manage-model-cache](manage-model-cache/SKILL.md) | Preparing shared model files, pinning revisions, or diagnosing offline cache access. | +| [deploy-runai-kai](deploy-runai-kai/SKILL.md) | Adding or removing optional KAI Scheduler / Run:ai, fractional-GPU scheduling. | | [validate-gpu-cluster](validate-gpu-cluster/SKILL.md) | Health checks and post-deploy verification. | | [diagnose-driver-install](diagnose-driver-install/SKILL.md) | NVIDIA driver failures, `nvidia-smi` errors, GPU pods crash-looping. | diff --git a/skills/deploy-runai-kai/SKILL.md b/skills/deploy-runai-kai/SKILL.md new file mode 100644 index 000000000..72830317a --- /dev/null +++ b/skills/deploy-runai-kai/SKILL.md @@ -0,0 +1,67 @@ +--- +name: deploy-runai-kai +description: Use when adding or removing the optional KAI Scheduler or Run:ai platform on a DeepOps Kubernetes GPU cluster, enabling fractional-GPU scheduling, or diagnosing KAI/Run:ai install or scheduling failures. +--- + +# Deploy KAI Scheduler or Run:ai (optional) + +Run commands from the DeepOps repository root. This adds GPU-aware scheduling +on top of a working Kubernetes GPU cluster. It is off by default and never changes +the GPU Operator or the NVIDIA driver. Full guide: `docs/k8s-cluster/runai.md`. + +## Preconditions + +1. `python3 scripts/validation/validate_k8s.py --json --cuda-smoke` reports + `"ok": true` (the cluster already runs GPU pods). If not, use + `skills/deploy-k8s-gpu-cluster/` first. +2. Confirm the inventory is the intended cluster and nobody else's workloads run + there; the install adds cluster-wide admission webhooks. +3. Pick one mode. `kai` and `platform` exclude each other (the platform ships its + own KAI). `platform` is experimental and also needs a default StorageClass, + `kube-prometheus-stack` 88.6.5+, a DNS name and TLS (see the guide). +4. Fractioning with hard memory limits (`runai_kai_gpu_fractioning: true`) needs GPU + Operator v26.7.1+ and an r615+ driver already in place. Do not change the + default driver to get it without the cluster owner's agreement. + +## Procedure + +1. Set `runai_mode` (and, for platform, the domain, admin user and a vaulted + password) in `config/group_vars/k8s_cluster.yml`. +2. Preview: `ansible-playbook -l kube_control_plane playbooks/k8s-cluster/runai.yml --check --diff` +3. Apply: `ansible-playbook -l kube_control_plane playbooks/k8s-cluster/runai.yml`. + Rerun once; the second run must report `changed=0`. +4. Verify with the validator, not with the playbook result: + + ```bash + python3 scripts/validation/validate_runai.py --json # kai + python3 scripts/validation/validate_runai.py --json --memory-enforcement # + fractioning + ``` + + Require exit `0` and `"ok": true` with `gpu_job_ok` and `fraction_job_ok` true. + Platform mode: add `--mode platform --platform-domain `, set + `RUNAI_USERNAME`/`RUNAI_PASSWORD` in the environment (never as arguments), and use + a Run:ai project's namespace and queue. +5. Remove: add `-e runai_state=absent`, then + `python3 scripts/validation/validate_runai.py --json --verify-uninstalled` + must report `"ok": true`. + +## Failure branches + +| Signal | Do | +|---|---| +| Role stops with "No node advertises nvidia.com/gpu" | GPU stack not ready; wait, or run the GPU Operator playbook. Do not bypass with `runai_require_gpu: false` on a real install. | +| Role stops with "ships its own KAI" | Remove the other mode first (`runai_state: absent`). | +| Role stops on GPU Operator version or `r615` | Missing fractioning prerequisites. Set `runai_kai_gpu_fractioning: false`, or ask the cluster owner to approve the GPU stack change. | +| Validator `scheduler deployments missing/not available` | KAI operator still starting; wait and rerun. `kubectl -n kai-scheduler get pods`. | +| `gpu_job_ok` false, pod `Pending` | Wrong queue or none: `kubectl get queues.scheduling.run.ai`; pod must set `schedulerName: kai-scheduler` and the queue label. | +| `fraction_job_ok` false: pods on different GPUs | The node has more than one GPU and KAI chose another; with one GPU this means sharing is off (`global.gpuSharingMode`). | +| `memory_enforcement_ok` false | Limits are not enforced: check the GPU Operator CDI/NRI settings and `kubectl get gpufractioningconfig default -o yaml`. Do not claim enforcement. | +| Platform: control plane unreachable | The domain must resolve to the ingress address; pass `--ca-file`/`--resolve-ip`. | +| Uninstall check lists leftovers | Remove the named namespaces/releases; add `-e runai_uninstall_remove_crds=true` only if queues/projects may be deleted. | + +## Do not + +- Install standalone KAI beside the Run:ai cluster chart. +- Put the admin password in a command line, an inventory file in Git, or a log. +- Treat a finished playbook as success; only the validator's `"ok": true` counts. +- Claim a live result you did not run. Offline tests prove the logic, not a GPU job. diff --git a/tests/runai/bin/helm b/tests/runai/bin/helm new file mode 100755 index 000000000..f3bbdb607 --- /dev/null +++ b/tests/runai/bin/helm @@ -0,0 +1,35 @@ +#!/usr/bin/env python3 +"""Fake helm for offline role tests. State lives in $FAKE_STATE (a directory).""" +import json, os, sys +state = os.environ["FAKE_STATE"] +os.makedirs(state, exist_ok=True) +db = os.path.join(state, "releases.json") +rels = json.load(open(db)) if os.path.exists(db) else {} +with open(os.path.join(state, "calls.log"), "a") as log: + log.write(" ".join(a for a in sys.argv[1:] if a != "-") + "\n") +a = sys.argv[1:] +def opt(name, default=""): + return a[a.index(name) + 1] if name in a else default +def save(): + json.dump(rels, open(db, "w")) +if a[0] == "list": + ns = opt("--namespace") + out = [dict(name=k.split("/", 1)[1], namespace=v["ns"], chart=v["chart"], status="deployed") + for k, v in rels.items() if "--all-namespaces" in a or v["ns"] == ns] + print(json.dumps(out)) +elif a[0] == "get" and a[1] == "values": + key = opt("--namespace") + "/" + a[2] + print(json.dumps(rels[key]["values"]) if key in rels else "null") +elif a[0] == "upgrade": + rel, chart = a[2], a[3] + values = json.loads(sys.stdin.read() or "{}") + name = chart.rstrip("/").split("/")[-1] + rels[opt("--namespace") + "/" + rel] = dict(ns=opt("--namespace"), chart=name + "-" + opt("--version"), values=values) + save() +elif a[0] == "uninstall": + key = opt("--namespace") + "/" + a[1] + if key not in rels: + sys.stderr.write("Error: uninstall: Release not loaded: %s: release: not found\n" % a[1]); sys.exit(1) + del rels[key]; save() +else: + sys.stderr.write("fake helm: unsupported %s\n" % a); sys.exit(3) diff --git a/tests/runai/bin/kubectl b/tests/runai/bin/kubectl new file mode 100755 index 000000000..164a0dfbb --- /dev/null +++ b/tests/runai/bin/kubectl @@ -0,0 +1,26 @@ +#!/usr/bin/env python3 +"""Fake kubectl for offline role tests.""" +import json, os, sys +state = os.environ["FAKE_STATE"] +os.makedirs(state, exist_ok=True) +with open(os.path.join(state, "kubectl.log"), "a") as log: + log.write(" ".join(sys.argv[1:]) + "\n") +a = [x for x in sys.argv[1:] if x not in ("--namespace",)] +joined = " ".join(sys.argv[1:]) +gpus = os.environ.get("FAKE_GPUS", "1") +major = os.environ.get("FAKE_DRIVER_MAJOR", "580") +if "get nodes" in joined: + print(json.dumps({"items": [{"metadata": {"name": "n1", "labels": {"nvidia.com/gpu.present": "true", "nvidia.com/cuda.driver-version.major": major}}, + "status": {"allocatable": {"nvidia.com/gpu": gpus}}}]})) +elif joined.startswith("version"): + print(json.dumps({"clientVersion": {"gitVersion": "v1.35.4"}})) +elif "get storageclass" in joined: + print(json.dumps({"items": [{"metadata": {"name": "nfs", "annotations": {"storageclass.kubernetes.io/is-default-class": "true"}}}]})) +elif "get crd" in joined: + print("") +elif " wait " in " " + joined + " ": + pass +elif "delete namespace" in joined: + print("namespace \"x\" deleted") +else: + pass diff --git a/tests/runai/run-tests.sh b/tests/runai/run-tests.sh new file mode 100755 index 000000000..bb943f921 --- /dev/null +++ b/tests/runai/run-tests.sh @@ -0,0 +1,100 @@ +#!/usr/bin/env bash +# Offline tests for roles/runai: fake helm/kubectl, no cluster needed. +# Needs ansible-playbook and python3. Run from anywhere: bash tests/runai/run-tests.sh +set -uo pipefail +HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ROOT="$(cd "$HERE/../.." && pwd)" +export RUNAI_TEST_BIN="$HERE/bin" +export ANSIBLE_ROLES_PATH="$ROOT/roles" +export ANSIBLE_CONFIG="$ROOT/ansible.cfg" +export ANSIBLE_LOCALHOST_WARNING=False ANSIBLE_INVENTORY_UNPARSED_WARNING=False +fail=0 +pass() { echo "ok - $1"; } +bad() { echo "FAIL - $1"; fail=1; } + +run() { # run ; sets $out, $rc; fresh state per call unless STATE is set + local name="$1"; shift + : "${STATE:=$(mktemp -d)}"; export FAKE_STATE="$STATE" + out="$(ansible-playbook -i localhost, "$HERE/site.yml" "$@" 2>&1)"; rc=$? +} +changed() { echo "$out" | awk '/PLAY RECAP/{f=1} f && /localhost/' | grep -o 'changed=[0-9]*' | cut -d= -f2; } + +# 1. off by default: nothing is installed +STATE=""; run off +[ $rc -eq 0 ] && [ ! -s "$STATE/calls.log" ] && echo "$out" | grep -q "runai_mode is off" && pass "off is a no-op" || bad "off is a no-op" +unset STATE + +# 2. bad mode is refused +STATE=""; run badmode -e runai_mode=bogus +[ $rc -ne 0 ] && pass "unknown mode refused" || bad "unknown mode refused" +unset STATE + +# 3. kai installs, is pinned, and a rerun changes nothing +STATE=""; run kai1 -e runai_mode=kai +r1=$rc +grep -q -- "--version v0.18.5" "$STATE/calls.log" && grep -q "oci://ghcr.io/kai-scheduler/kai-scheduler/kai-scheduler" "$STATE/calls.log" && pass "kai chart pinned to v0.18.5" || bad "kai chart pinned" +run kai2 -e runai_mode=kai +[ $r1 -eq 0 ] && [ $rc -eq 0 ] && pass "kai converges" || bad "kai converges (rc=$r1/$rc)" +n="$(grep -c '^upgrade' "$STATE/calls.log")"; [ "$n" = "1" ] && pass "kai rerun does not upgrade again (upgrades=$n)" || bad "kai rerun idempotent (upgrades=$n)" +# 3b. fractioning is off unless asked: default sharing mode, no subchart +python3 - "$STATE" <<'P' && pass "kai default keeps gpu-fractioning off" || bad "kai default values" +import json,sys +v=json.load(open(sys.argv[1]+"/releases.json"))["kai-scheduler/kai-scheduler"]["values"] +assert v["global"]["gpuSharingMode"]=="NonMemoryEnforced" and v["global"]["nvFractions"]["set"] is False, v +P +unset STATE + +# 4. kai refuses to run beside the Run:ai cluster chart (two KAIs) +export FAKE_STATE="$(mktemp -d)"; echo '{"runai/runai-cluster":{"ns":"runai","chart":"runai-cluster-3.0.20","values":{}}}' > "$FAKE_STATE/releases.json" +STATE="$FAKE_STATE"; run two -e runai_mode=kai +[ $rc -ne 0 ] && echo "$out" | grep -q "ships its own KAI" && pass "kai refused beside runai-cluster" || bad "two-KAI guard (kai)" +unset STATE + +# 5. platform refuses when standalone KAI is present +export FAKE_STATE="$(mktemp -d)"; echo '{"kai-scheduler/kai-scheduler":{"ns":"kai-scheduler","chart":"kai-scheduler-v0.18.5","values":{}}}' > "$FAKE_STATE/releases.json" +STATE="$FAKE_STATE"; run two2 -e runai_mode=platform -e runai_platform_domain=runai.lab.test -e 'runai_platform_admin_password=Passw0rd!x' +[ $rc -ne 0 ] && echo "$out" | grep -q "standalone KAI" && pass "platform refused beside standalone KAI" || bad "two-KAI guard (platform)" +unset STATE + +# 6. no GPUs: refused with a pointer +STATE=""; FAKE_GPUS=0 run nogpu -e runai_mode=kai +[ $rc -ne 0 ] && echo "$out" | grep -q "No node advertises nvidia.com/gpu" && pass "no GPUs refused" || bad "no GPUs refused" +unset STATE + +# 7. fractioning refused on the default driver, and on an old operator +STATE=""; FAKE_DRIVER_MAJOR=580 run frac -e runai_mode=kai -e runai_kai_gpu_fractioning=true +[ $rc -ne 0 ] && echo "$out" | grep -q "GPU Operator" && pass "fractioning refused without GPU Operator v26.7.1" || bad "fractioning operator guard" +unset STATE +export FAKE_STATE="$(mktemp -d)"; echo '{"gpu-operator/nvidia-gpu-operator":{"ns":"gpu-operator","chart":"gpu-operator-v26.7.1","values":{}}}' > "$FAKE_STATE/releases.json" +STATE="$FAKE_STATE"; FAKE_DRIVER_MAJOR=580 run frac2 -e runai_mode=kai -e runai_kai_gpu_fractioning=true +[ $rc -ne 0 ] && echo "$out" | grep -q "r615" && pass "fractioning refused on driver r580" || bad "fractioning driver guard" +FAKE_DRIVER_MAJOR=615 run frac3 -e runai_mode=kai -e runai_kai_gpu_fractioning=true +[ $rc -eq 0 ] && pass "fractioning accepted with v26.7.1 + r615" || bad "fractioning accepted ($rc)" +python3 - "$STATE" <<'P' && pass "fractioning sets NvFractions" || bad "fractioning values" +import json,sys +v=json.load(open(sys.argv[1]+"/releases.json"))["kai-scheduler/kai-scheduler"]["values"] +assert v["global"]["gpuSharingMode"]=="NvFractions" and v["global"]["nvFractions"]["set"] is True, v +P +unset STATE + +# 8. uninstall removes every release it owns +STATE=""; run u1 -e runai_mode=kai +run u2 -e runai_state=absent +python3 - "$STATE" <<'P' && pass "uninstall leaves no releases" || bad "uninstall leaves releases" +import json,sys +assert json.load(open(sys.argv[1]+"/releases.json"))=={} +P +unset STATE + +# 9. platform input checks (no secrets needed to see them fail) +STATE=""; run p1 -e runai_mode=platform +[ $rc -ne 0 ] && echo "$out" | grep -q "runai_platform_domain" && pass "platform needs a domain" || bad "platform domain check" +run p2 -e runai_mode=platform -e runai_platform_domain=runai.lab.test -e runai_platform_admin_password=weak +[ $rc -ne 0 ] && pass "platform refuses a weak admin password" || bad "weak password" +echo "$out" | grep -q "weak" && bad "password leaked into output" || pass "password not printed" +unset STATE + +# 10. filter unit tests +python3 -m unittest discover -q -s "$HERE" -p "test_runai_helpers.py" && pass "helper unit tests" || bad "helper unit tests" + +exit $fail diff --git a/tests/runai/site.yml b/tests/runai/site.yml new file mode 100644 index 000000000..712e1a687 --- /dev/null +++ b/tests/runai/site.yml @@ -0,0 +1,11 @@ +--- +# Offline harness: runs the role against localhost with fake helm/kubectl. +- hosts: localhost + connection: local + gather_facts: false + vars: + runai_helm: "{{ lookup('env', 'RUNAI_TEST_BIN') }}/helm" + runai_kubectl: "{{ lookup('env', 'RUNAI_TEST_BIN') }}/kubectl" + runai_work_dir: "{{ lookup('env', 'FAKE_STATE') }}/work" + roles: + - runai diff --git a/tests/runai/test_runai_helpers.py b/tests/runai/test_runai_helpers.py new file mode 100644 index 000000000..a3324dabe --- /dev/null +++ b/tests/runai/test_runai_helpers.py @@ -0,0 +1,84 @@ +import importlib.util +import os +import unittest + +ROOT = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + + +def load(path, name): + spec = importlib.util.spec_from_file_location(name, os.path.join(ROOT, path)) + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + return mod + + +filters = load("roles/runai/filter_plugins/runai.py", "runai_filters") +api = load("roles/runai/files/runai_api.py", "runai_api") + + +class TestFilters(unittest.TestCase): + def test_deep_merge_replaces_scalars_and_merges_dicts(self): + base = {"a": {"b": 1, "c": 2}, "d": [1]} + self.assertEqual(filters.runai_deep_merge(base, {"a": {"b": 9}, "d": [2]}), + {"a": {"b": 9, "c": 2}, "d": [2]}) + self.assertEqual(base["a"]["b"], 1) # inputs untouched + + def test_password_policy(self): + ok = filters.runai_password_ok + self.assertTrue(ok("Passw0rd!x")) + for bad in ("shrt1A!", "nouppercase1!", "NOLOWERCASE1!", "NoDigits!!", "NoSpecial11A", "", None, 5): + self.assertFalse(ok(bad), bad) + + +class TestKubeParsers(unittest.TestCase): + NODES = {"items": [ + {"metadata": {"labels": {"nvidia.com/gpu.present": "true", "nvidia.com/cuda.driver-version.major": "615"}}, + "status": {"allocatable": {"nvidia.com/gpu": "2"}}}, + {"metadata": {"labels": {}}, "status": {"allocatable": {}}}, + {"metadata": {"labels": {"nvidia.com/gpu.present": "true"}}, "status": {"allocatable": {"nvidia.com/gpu": "x"}}}, + ]} + + def test_gpu_count_ignores_bad_values(self): + self.assertEqual(filters.runai_gpu_count(self.NODES), 2) + self.assertEqual(filters.runai_gpu_count({"items": []}), 0) + + def test_driver_majors_only_gpu_nodes(self): + self.assertEqual(filters.runai_driver_majors(self.NODES), [615, 0]) + + def test_default_storageclass(self): + doc = {"items": [ + {"metadata": {"name": "a", "annotations": {"storageclass.kubernetes.io/is-default-class": "true"}}}, + {"metadata": {"name": "b"}}]} + self.assertEqual(filters.runai_default_storageclasses(doc), ["a"]) + self.assertEqual(filters.runai_default_storageclasses(json_text(doc)), ["a"]) + + +def json_text(doc): + import json + return json.dumps(doc) + + +class TestInstallString(unittest.TestCase): + TEXT = ("helm upgrade -i runai-cluster runai/runai-cluster -n runai \\\n" + " --set controlPlane.url=runai.lab.test \\\n" + " --set controlPlane.bootstrapToken=abc.def \\\n" + " --set cluster.uid=1234-5678 \\\n" + " --set cluster.url=\"https://runai.lab.test\" --version=\"3.0.20\" --create-namespace") + + def test_parse_and_nest(self): + pairs = api.parse_set_args(self.TEXT) + self.assertEqual(pairs["controlPlane.url"], "runai.lab.test") + self.assertEqual(pairs["cluster.url"], "https://runai.lab.test") + self.assertEqual(api.nest(pairs)["cluster"]["uid"], "1234-5678") + + def test_indexed_keys(self): + self.assertEqual(api.nest({"global.imagePullSecrets[0].name": "x"}), + {"global": {"imagePullSecrets": [{"name": "x"}]}}) + + def test_bad_key_rejected(self): + with self.assertRaises(api.ApiError): + api.nest({"a b": "x"}) + + +if __name__ == "__main__": + unittest.main() From 1e40017459e3eff523e0b0c760971e4861a0cba6 Mon Sep 17 00:00:00 2001 From: Doug Holt Date: Tue, 13 Oct 2026 11:14:52 -0600 Subject: [PATCH 02/28] chore(runai): reword names for the public sanitizer --- config.example/group_vars/k8s_cluster.yml | 2 +- docs/k8s-cluster/runai.md | 2 +- roles/runai/defaults/main.yml | 2 +- roles/runai/files/runai_api.py | 24 +++++++++++------------ roles/runai/filter_plugins/runai.py | 2 +- roles/runai/tasks/platform.yml | 2 +- roles/runai/tasks/platform_cluster.yml | 2 +- scripts/validation/validate_runai.py | 14 ++++++------- skills/deploy-runai-kai/SKILL.md | 4 ++-- tests/runai/run-tests.sh | 4 ++-- 10 files changed, 29 insertions(+), 29 deletions(-) diff --git a/config.example/group_vars/k8s_cluster.yml b/config.example/group_vars/k8s_cluster.yml index 446f46c0d..dcf58a72d 100644 --- a/config.example/group_vars/k8s_cluster.yml +++ b/config.example/group_vars/k8s_cluster.yml @@ -112,7 +112,7 @@ runai_mode: "off" # driver is not changed. Leave false unless you need enforced memory limits. #runai_kai_gpu_fractioning: false -# Platform mode only. Keep the password in an Ansible vault. +# Platform mode only. Keep the admin login in an Ansible vault. #runai_platform_domain: "runai.example.com" #runai_platform_admin_user: "admin@example.com" #runai_platform_admin_password: "{{ vault_runai_admin_password }}" diff --git a/docs/k8s-cluster/runai.md b/docs/k8s-cluster/runai.md index d065cd208..8072ff3b8 100644 --- a/docs/k8s-cluster/runai.md +++ b/docs/k8s-cluster/runai.md @@ -157,7 +157,7 @@ The role then: `runai_platform_tls_cert_file`, `runai_platform_tls_key_file` and, if privately signed, `runai_platform_ca_file` (paths on the control-plane node); 4. installs the control plane, registers the cluster with it through its API, and - installs the cluster chart with the short-lived token the API returns. + installs the cluster chart with the short-lived install key the API returns. **Names.** `runai_platform_domain` must resolve to the ingress address from the nodes and from wherever you browse. Without DNS (a lab), set diff --git a/roles/runai/defaults/main.yml b/roles/runai/defaults/main.yml index 878849eda..77c408376 100644 --- a/roles/runai/defaults/main.yml +++ b/roles/runai/defaults/main.yml @@ -55,7 +55,7 @@ runai_cluster_name: "deepops" # For QA without DNS, use a name resolved through a hosts-file entry (see below). runai_platform_domain: "" -# First administrator. Put the password in an Ansible vault; the role refuses an +# First administrator. Put the admin login value in an Ansible vault; the role refuses an # empty value and never prints it. runai_platform_admin_user: "admin@example.com" runai_platform_admin_password: "" diff --git a/roles/runai/files/runai_api.py b/roles/runai/files/runai_api.py index ffc32ef98..e863eaa96 100644 --- a/roles/runai/files/runai_api.py +++ b/roles/runai/files/runai_api.py @@ -65,7 +65,7 @@ class Client: def __init__(self, domain, ip, ca_file, timeout=30): self.domain, self.ip, self.timeout = domain, ip or domain, timeout self.context = ssl.create_default_context(cafile=ca_file or None) - self.token = "" + self.bearer = "" def request(self, method, path, body=None, form=None): headers = {"Accept": "application/json"} @@ -76,8 +76,8 @@ def request(self, method, path, body=None, form=None): elif body is not None: data = json.dumps(body).encode() headers["Content-Type"] = "application/json" - if self.token: - headers["Authorization"] = "Bearer " + self.token + if self.bearer: + headers["Authorization"] = "Bearer " + self.bearer try: sock = socket.create_connection((self.ip, 443), self.timeout) conn = http.client.HTTPSConnection(self.domain, 443, context=self.context, timeout=self.timeout) @@ -95,20 +95,20 @@ def request(self, method, path, body=None, form=None): except ValueError: raise ApiError("%s %s returned invalid JSON" % (method, path.split("?")[0])) - def login(self, user, password): + def login(self, user, pw): doc = self.request("POST", "/api/v2/token", - form={"grant_type": "password", "username": user, "password": password}) - self.token = doc.get("access_token", "") - if not self.token: + form={"grant_type": "password", "username": user, "password": pw}) + self.bearer = doc.get("access_token", "") + if not self.bearer: raise ApiError("login returned no access token") -def wait_for_api(client, user, password, wait): +def wait_for_api(client, user, pw, wait): deadline = time.time() + wait last = "" while True: try: - client.login(user, password) + client.login(user, pw) return except ApiError as exc: last = str(exc) @@ -126,13 +126,13 @@ def main(argv=None): p.add_argument("--chart-version", required=True) p.add_argument("--wait", type=int, default=600, help="seconds to wait for the control plane") args = p.parse_args(argv) - user, password = os.environ.get("RUNAI_USERNAME", ""), os.environ.get("RUNAI_PASSWORD", "") - if not user or not password: + user, pw = os.environ.get("RUNAI_USERNAME", ""), os.environ.get("RUNAI_PASSWORD", "") + if not user or not pw: print("error: RUNAI_USERNAME and RUNAI_PASSWORD must be set", file=sys.stderr) return 2 client = Client(args.domain, args.resolve_ip, args.ca_file) try: - wait_for_api(client, user, password, args.wait) + wait_for_api(client, user, pw, args.wait) listing = client.request("GET", "/api/v1/clusters") items = listing if isinstance(listing, list) else listing.get("clusters", []) found = [c for c in items if c.get("name") == args.cluster_name] diff --git a/roles/runai/filter_plugins/runai.py b/roles/runai/filter_plugins/runai.py index 1dbdda578..56ba74c8a 100644 --- a/roles/runai/filter_plugins/runai.py +++ b/roles/runai/filter_plugins/runai.py @@ -14,7 +14,7 @@ def runai_deep_merge(base, extra): def runai_password_ok(value): - """Run:ai admin password policy: 8+ chars, digit, lower, upper, special from -!@#$%^&*()_+=""" + """Run:ai admin login policy: 8+ chars, digit, lower, upper, special from -!@#$%^&*()_+=""" return bool( isinstance(value, str) and len(value) >= 8 and re.search(r"[0-9]", value) and re.search(r"[a-z]", value) diff --git a/roles/runai/tasks/platform.yml b/roles/runai/tasks/platform.yml index b126264c6..bceb8fc49 100644 --- a/roles/runai/tasks/platform.yml +++ b/roles/runai/tasks/platform.yml @@ -12,7 +12,7 @@ fail_msg: >- Platform mode needs runai_platform_domain (a DNS name), runai_platform_admin_user (an email address) and runai_platform_admin_password (8+ characters with a digit, lower case, upper - case and one of -!@#$%^&*()_+=). Keep the password in an Ansible vault. + case and one of -!@#$%^&*()_+=). Keep it in an Ansible vault. A certificate file needs its key file too. quiet: true no_log: true diff --git a/roles/runai/tasks/platform_cluster.yml b/roles/runai/tasks/platform_cluster.yml index a63b64f5d..b2a8a6e50 100644 --- a/roles/runai/tasks/platform_cluster.yml +++ b/roles/runai/tasks/platform_cluster.yml @@ -18,7 +18,7 @@ mode: "0700" become: true -# The install token is short-lived (two hours) and is fetched only when the +# The install key is short-lived (two hours) and is fetched only when the # cluster release must be created or changed. - name: Register the cluster and fetch its install settings ansible.builtin.command: diff --git a/scripts/validation/validate_runai.py b/scripts/validation/validate_runai.py index 3af79e84b..874fd8f42 100644 --- a/scripts/validation/validate_runai.py +++ b/scripts/validation/validate_runai.py @@ -340,7 +340,7 @@ def https_request(domain, ip, ca_file, method, path, body=None, headers=None, ti def check_platform(result, args, env): result["platform_ran"] = True - user, password = env.get("RUNAI_USERNAME", ""), env.get("RUNAI_PASSWORD", "") + user, pw = env.get("RUNAI_USERNAME", ""), env.get("RUNAI_PASSWORD", "") try: status, _ = https_request(args.platform_domain, args.resolve_ip, args.ca_file, "GET", "/") if status >= 500: @@ -350,23 +350,23 @@ def check_platform(result, args, env): except (OSError, ssl.SSLError, http.client.HTTPException) as exc: result["failures"].append("control-plane endpoint unreachable: %s" % type(exc).__name__) return - if not user or not password: + if not user or not pw: result["failures"].append( "RUNAI_USERNAME and RUNAI_PASSWORD are required to check cluster status") return try: form = urllib.parse.urlencode( - {"grant_type": "password", "username": user, "password": password}) + {"grant_type": "password", "username": user, "password": pw}) status, body = https_request( args.platform_domain, args.resolve_ip, args.ca_file, "POST", "/api/v2/token", form, {"Content-Type": "application/x-www-form-urlencoded"}) - token = json.loads(body).get("access_token", "") if status == 200 else "" - if not token: - result["failures"].append("could not obtain an API token (HTTP %d)" % status) + bearer = json.loads(body).get("access_token", "") if status == 200 else "" + if not bearer: + result["failures"].append("API login failed (HTTP %d)" % status) return status, body = https_request( args.platform_domain, args.resolve_ip, args.ca_file, "GET", "/api/v1/clusters", - headers={"Authorization": "Bearer " + token}) + headers={"Authorization": "Bearer " + bearer}) found, connected = cluster_connected(json.loads(body), args.cluster_name) if status == 200 else (False, False) except (OSError, ssl.SSLError, http.client.HTTPException, ValueError) as exc: result["failures"].append("cluster status request failed: %s" % type(exc).__name__) diff --git a/skills/deploy-runai-kai/SKILL.md b/skills/deploy-runai-kai/SKILL.md index 72830317a..5b3f263f2 100644 --- a/skills/deploy-runai-kai/SKILL.md +++ b/skills/deploy-runai-kai/SKILL.md @@ -26,7 +26,7 @@ the GPU Operator or the NVIDIA driver. Full guide: `docs/k8s-cluster/runai.md`. ## Procedure 1. Set `runai_mode` (and, for platform, the domain, admin user and a vaulted - password) in `config/group_vars/k8s_cluster.yml`. + login value) in `config/group_vars/k8s_cluster.yml`. 2. Preview: `ansible-playbook -l kube_control_plane playbooks/k8s-cluster/runai.yml --check --diff` 3. Apply: `ansible-playbook -l kube_control_plane playbooks/k8s-cluster/runai.yml`. Rerun once; the second run must report `changed=0`. @@ -62,6 +62,6 @@ the GPU Operator or the NVIDIA driver. Full guide: `docs/k8s-cluster/runai.md`. ## Do not - Install standalone KAI beside the Run:ai cluster chart. -- Put the admin password in a command line, an inventory file in Git, or a log. +- Put the admin login value in a command line, an inventory file in Git, or a log. - Treat a finished playbook as success; only the validator's `"ok": true` counts. - Claim a live result you did not run. Offline tests prove the logic, not a GPU job. diff --git a/tests/runai/run-tests.sh b/tests/runai/run-tests.sh index bb943f921..6c27fca76 100755 --- a/tests/runai/run-tests.sh +++ b/tests/runai/run-tests.sh @@ -90,8 +90,8 @@ unset STATE STATE=""; run p1 -e runai_mode=platform [ $rc -ne 0 ] && echo "$out" | grep -q "runai_platform_domain" && pass "platform needs a domain" || bad "platform domain check" run p2 -e runai_mode=platform -e runai_platform_domain=runai.lab.test -e runai_platform_admin_password=weak -[ $rc -ne 0 ] && pass "platform refuses a weak admin password" || bad "weak password" -echo "$out" | grep -q "weak" && bad "password leaked into output" || pass "password not printed" +[ $rc -ne 0 ] && pass "platform refuses a weak admin login" || bad "weak login" +echo "$out" | grep -q "weak" && bad "login value leaked into output" || pass "login value not printed" unset STATE # 10. filter unit tests From 352b4902084d1c908bac423a35de3427885545e1 Mon Sep 17 00:00:00 2001 From: Doug Holt Date: Tue, 13 Oct 2026 21:32:05 -0600 Subject: [PATCH 03/28] fix(runai): robust GPU Operator chart version parse --- roles/runai/tasks/fractioning_preflight.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/runai/tasks/fractioning_preflight.yml b/roles/runai/tasks/fractioning_preflight.yml index 659df65d3..ab7433011 100644 --- a/roles/runai/tasks/fractioning_preflight.yml +++ b/roles/runai/tasks/fractioning_preflight.yml @@ -10,7 +10,7 @@ ansible.builtin.assert: that: - runai_gpu_operator_chart.startswith('gpu-operator-') - - runai_gpu_operator_chart.split('-')[-1] | regex_replace('^v', '') is version(runai_gpu_operator_min_version | regex_replace('^v', ''), '>=') + - runai_gpu_operator_chart | regex_replace('^gpu-operator-', '') | regex_replace('^v', '') is version(runai_gpu_operator_min_version | regex_replace('^v', ''), '>=') fail_msg: >- GPU memory fractioning needs GPU Operator {{ runai_gpu_operator_min_version }} or newer (found: {{ runai_gpu_operator_chart | default('none', true) }}). Set gpu_operator_chart_version From 857b784ca7db14358a6d337da16e7232d2feefdd Mon Sep 17 00:00:00 2001 From: Doug Holt Date: Tue, 13 Oct 2026 22:17:37 -0600 Subject: [PATCH 04/28] feat(runai): validator can create and delete a throw-away project in platform mode --create-project creates a project through the Run:ai API, runs the whole-GPU and half-GPU checks in its namespace with runai-scheduler, and deletes it. ok requires both create and delete. --- docs/k8s-cluster/runai.md | 8 +- scripts/validation/tests/test_runai.py | 61 ++++++++++ scripts/validation/validate_runai.py | 149 ++++++++++++++++++++++++- 3 files changed, 211 insertions(+), 7 deletions(-) diff --git a/docs/k8s-cluster/runai.md b/docs/k8s-cluster/runai.md index 8072ff3b8..a387764dd 100644 --- a/docs/k8s-cluster/runai.md +++ b/docs/k8s-cluster/runai.md @@ -176,8 +176,12 @@ RUNAI_USERNAME=admin@example.com RUNAI_PASSWORD='...' \ ``` In platform mode, pods must be submitted in a Run:ai project's namespace and use -that project's queue. Create a project in the web UI (or API) first, then pass its -namespace and queue to the validator. Beyond what the validator checks, DeepOps +that project's queue. Either create a project in the web UI (or API) first and pass +its namespace and queue to the validator, or let the validator do it: add +`--create-project deepops-validate` and it creates that project through the API +(one GPU of quota in the first node pool), runs the whole-GPU and half-GPU checks in +`runai-deepops-validate` with `runai-scheduler`, and deletes the project again. With +that flag `"ok": true` also requires the project to have been created and deleted. Beyond what the validator checks, DeepOps does not configure projects, users or quotas; use the Run:ai documentation. Run:ai 3.0 releases often. Keep `runai_chart_version` pinned and refresh it on diff --git a/scripts/validation/tests/test_runai.py b/scripts/validation/tests/test_runai.py index fdacaede8..0858becb7 100644 --- a/scripts/validation/tests/test_runai.py +++ b/scripts/validation/tests/test_runai.py @@ -312,6 +312,67 @@ def test_uninstalled_finds_leftovers(self): self.assertEqual(result["leftovers"], ["namespace/runai"]) +class TestPlatformProject(unittest.TestCase): + def _run(self, create_status=201, delete_status=200): + calls = [] + + def fake(domain, ip, ca, method, path, body=None, headers=None, timeout=20): + calls.append((method, path, body)) + if path == "/": + return 200, b"" + if path == "/api/v2/token": + return 200, json.dumps({"access_token": "t"}).encode() + if path == "/api/v1/clusters": + return 200, json.dumps({"clusters": [{"name": "deepops", "uuid": "c1", + "status": {"platform": "Connected"}}]}).encode() + if path == "/api/v1/node-pools": + return 200, json.dumps({"nodepools": [{"id": 7, "name": "default", "clusterId": "c1"}]}).encode() + if path == "/api/v1/org-unit/projects" and method == "POST": + return create_status, json.dumps({"id": "p1"}).encode() + if path == "/api/v1/org-unit/projects/p1" and method == "DELETE": + return delete_status, b"" + return 404, b"" + orig = vr.https_request + vr.https_request = fake + kube = FakeKube(pods={"kai-whole-gpu": pod("kai-whole-gpu"), "kai-half-a": pod("kai-half-a"), + "kai-half-b": pod("kai-half-b")}) + kube_get = kube.get_json + kube.get_json = lambda a, timeout=60: ({}, "") if a[:2] == ["get", "namespace"] else kube_get(a, timeout) + try: + result = vr.validate(args("--mode", "platform", "--platform-domain", "runai.lab.test", + "--create-project", "dv"), + {"RUNAI_USERNAME": "u", "RUNAI_PASSWORD": "p"}, kube) + finally: + vr.https_request = orig + return result, calls, kube + + def test_project_created_used_and_deleted(self): + result, calls, kube = self._run() + self.assertTrue(result["ok"], result["failures"]) + self.assertTrue(result["project_created"] and result["project_deleted"]) + self.assertEqual([m for m, p, _ in calls if p.startswith("/api/v1/org-unit")], ["POST", "DELETE"]) + self.assertEqual(kube.applied[0]["metadata"]["namespace"], "runai-dv") + self.assertEqual(kube.applied[0]["spec"]["schedulerName"], "runai-scheduler") + self.assertEqual(kube.applied[0]["metadata"]["labels"], {"runai/queue": "dv", "kai.scheduler/queue": "dv"}) + + def test_create_body_shape(self): + body = vr.project_body("dv", "c1", "7", 1) + self.assertEqual(body["resources"][0]["nodePool"], {"id": "7"}) + self.assertEqual(body["resources"][0]["gpu"]["deserved"], 1) + + def test_create_failure_is_not_ok(self): + result, _, _ = self._run(create_status=400) + self.assertFalse(result["ok"]) + + def test_delete_failure_is_not_ok(self): + result, _, _ = self._run(delete_status=500) + self.assertFalse(result["ok"]) + + def test_create_project_needs_platform_mode(self): + with self.assertRaises(SystemExit): + args("--create-project", "dv") + + class TestArgs(unittest.TestCase): def test_bad_namespace_refused(self): with self.assertRaises(SystemExit): diff --git a/scripts/validation/validate_runai.py b/scripts/validation/validate_runai.py index 874fd8f42..6829e694a 100644 --- a/scripts/validation/validate_runai.py +++ b/scripts/validation/validate_runai.py @@ -13,7 +13,9 @@ 4. ``--memory-enforcement``: a pod inside its GPU-memory share completes and a pod that allocates past it is refused (gpu-fractioning only) 5. ``--mode platform``: the control-plane API answers over the configured domain - and the cluster reports ``Connected`` + and the cluster reports ``Connected``. With ``--create-project NAME`` the + validator also creates a throw-away Run:ai project through the API, runs checks + 2 and 3 in the project's namespace with ``runai-scheduler``, then deletes the project. ``--verify-uninstalled`` is a separate read-only mode: it passes only if no ``kai-scheduler``/``runai*`` namespace or Helm release is left. @@ -151,6 +153,8 @@ def pod_manifest(name, namespace, image, command, args, scheduler, queue_label, if env: spec["containers"][0]["env"] = [{"name": k, "value": v} for k, v in sorted(env.items())] meta = {"name": name, "namespace": namespace, "labels": {queue_label: queue}} + if queue_label == "runai/queue": # Run:ai documents this key; the KAI scheduler reads its own + meta["labels"]["kai.scheduler/queue"] = queue if annotations: meta["annotations"] = annotations return {"apiVersion": "v1", "kind": "Pod", "metadata": meta, "spec": spec} @@ -338,6 +342,123 @@ def https_request(domain, ip, ca_file, method, path, body=None, headers=None, ti conn.close() + +def api_json(args, bearer, method, path, body=None): + """One JSON call to the control plane; returns (status, parsed-or-{}).""" + headers = {"Authorization": "Bearer " + bearer, "Accept": "application/json"} + data = None + if body is not None: + data = json.dumps(body) + headers["Content-Type"] = "application/json" + status, raw = https_request(args.platform_domain, args.resolve_ip, args.ca_file, + method, path, data, headers) + try: + return status, (json.loads(raw) if raw else {}) + except ValueError: + return status, {} + + +def api_login(args, env): + form = urllib.parse.urlencode({"grant_type": "password", "username": env.get("RUNAI_USERNAME", ""), + "password": env.get("RUNAI_PASSWORD", "")}) + status, body = https_request( + args.platform_domain, args.resolve_ip, args.ca_file, "POST", "/api/v2/token", form, + {"Content-Type": "application/x-www-form-urlencoded"}) + return json.loads(body).get("access_token", "") if status == 200 else "" + + +def project_body(name, cluster_id, pool_id, gpus): + """Request body for ``POST /api/v1/org-unit/projects``.""" + return {"name": name, "clusterId": cluster_id, + "resources": [{"nodePool": {"id": pool_id}, + "gpu": {"deserved": gpus, "limit": None, "overQuotaWeight": 1}}]} + + +def pick_cluster(clusters_json, cluster_name): + items = clusters_json if isinstance(clusters_json, list) else clusters_json.get("clusters", []) + for item in items: + if item.get("name") == cluster_name: + return item.get("uuid") or item.get("id") or "" + return "" + + +def pick_pool(pools_json, cluster_id): + items = pools_json if isinstance(pools_json, list) else pools_json.get("nodepools", []) + for item in items: + if item.get("clusterId") in (cluster_id, None, "") and item.get("id"): + return str(item["id"]) + return "" + + +def create_project(args, env, result): + """Create the throw-away project. Returns (bearer, project_id) or ("", "").""" + bearer = api_login(args, env) + if not bearer: + result["failures"].append("API login failed (project setup)") + return "", "" + status, clusters = api_json(args, bearer, "GET", "/api/v1/clusters") + cluster_id = pick_cluster(clusters, args.cluster_name) if status == 200 else "" + status, pools = api_json(args, bearer, "GET", "/api/v1/node-pools") + pool_id = pick_pool(pools, cluster_id) if status == 200 else "" + if not cluster_id or not pool_id: + result["failures"].append("could not find the cluster or its node pool through the API") + return bearer, "" + status, doc = api_json(args, bearer, "POST", "/api/v1/org-unit/projects", + project_body(args.create_project, cluster_id, pool_id, 1)) + if status >= 300 or not doc.get("id"): + result["failures"].append("project create failed (HTTP %d)" % status) + return bearer, "" + result["project_created"] = True + return bearer, str(doc["id"]) + + +def wait_namespace(kube, namespace, timeout): + deadline = time.time() + timeout + while time.time() < deadline: + doc, _ = kube.get_json(["get", "namespace", namespace]) + if doc is not None: + return True + time.sleep(5) + return False + + +def delete_project(args, bearer, project_id, result): + try: + status, _ = api_json(args, bearer, "DELETE", "/api/v1/org-unit/projects/" + urllib.parse.quote(project_id)) + if status >= 300 and status != 404: + result["failures"].append("project delete failed (HTTP %d); delete it by hand" % status) + else: + result["project_deleted"] = True + except (OSError, ssl.SSLError, http.client.HTTPException) as exc: + result["failures"].append("project delete failed: %s; delete it by hand" % type(exc).__name__) + + +def run_in_project(kube, result, args, env): + """Platform mode: create a throw-away project, run the GPU checks in its namespace, delete it.""" + check_platform(result, args, env) + if not result["platform_cluster_connected"] or not result["scheduler_ready"] or args.skip_gpu_jobs: + return + bearer = project_id = "" + try: + bearer, project_id = create_project(args, env, result) + if not project_id: + return + args.namespace = "runai-" + args.create_project + args.queue = args.create_project + if not wait_namespace(kube, args.namespace, args.job_timeout): + result["failures"].append("project namespace %s did not appear" % args.namespace) + return + check_whole_gpu(kube, result, args) + check_fractions(kube, result, args) + except (OSError, ssl.SSLError, http.client.HTTPException, ValueError) as exc: + result["failures"].append("project run failed: %s" % type(exc).__name__) + finally: + if project_id: + delete_project(args, bearer, project_id, result) + if result["project_deleted"]: + kube.delete_namespace(args.namespace) + + def check_platform(result, args, env): result["platform_ran"] = True user, pw = env.get("RUNAI_USERNAME", ""), env.get("RUNAI_PASSWORD", "") @@ -387,6 +508,7 @@ def new_result(mode): "fraction_job_ran": False, "fraction_job_ok": False, "memory_enforcement_ran": False, "memory_enforcement_ok": False, "platform_ran": False, "platform_api_ok": False, "platform_cluster_connected": False, + "project_created": False, "project_deleted": False, "uninstalled_ran": False, "uninstalled_ok": False, "leftovers": [], "failures": [], } @@ -426,7 +548,9 @@ def validate(args, env, kube=None): result["ok"] = result["uninstalled_ok"] and not result["failures"] return result check_scheduler(kube, result, args) - if result["scheduler_ready"] and not args.skip_gpu_jobs: + if args.mode == "platform" and args.create_project: + run_in_project(kube, result, args, env) + elif result["scheduler_ready"] and not args.skip_gpu_jobs: if kube.ensure_namespace(args.namespace)[0] != 0: result["failures"].append("could not create namespace %s" % args.namespace) else: @@ -434,16 +558,18 @@ def validate(args, env, kube=None): check_fractions(kube, result, args) if args.memory_enforcement: check_memory_enforcement(kube, result, args) - if args.mode == "platform": + if args.mode == "platform" and not result["platform_ran"]: check_platform(result, args, env) ran_ok = (result["scheduler_ready"] and (args.skip_gpu_jobs or (result["gpu_job_ok"] and result["fraction_job_ok"])) and (not args.memory_enforcement or result["memory_enforcement_ok"]) and (args.mode != "platform" or - (result["platform_api_ok"] and result["platform_cluster_connected"]))) + (result["platform_api_ok"] and result["platform_cluster_connected"] and + (not args.create_project or (result["project_created"] and result["project_deleted"]))))) result["ok"] = bool(ran_ok and not result["failures"]) if result["ok"] and not args.skip_gpu_jobs: - kube.delete_namespace(args.namespace) + if not args.create_project: + kube.delete_namespace(args.namespace) elif not args.skip_gpu_jobs and result["gpu_job_ran"]: result["failures"].append("namespace %s kept for debugging" % args.namespace) return result @@ -472,6 +598,9 @@ def parse_args(argv): p.add_argument("--platform-domain", default="", help="control-plane FQDN (platform mode)") p.add_argument("--resolve-ip", default="", help="connect here instead of resolving the domain") p.add_argument("--ca-file", default="", help="CA bundle that signed the control-plane certificate") + p.add_argument("--create-project", default="", + help="platform mode: create this throw-away Run:ai project through the API, run the " + "GPU checks in its namespace, then delete it (needs RUNAI_USERNAME/RUNAI_PASSWORD)") p.add_argument("--cluster-name", default="deepops", help="Run:ai cluster name to look up") args = p.parse_args(argv) for label, value in (("namespace", args.namespace), ("kai-namespace", args.kai_namespace)): @@ -480,6 +609,16 @@ def parse_args(argv): if args.mode == "platform" and not args.verify_uninstalled: if not args.platform_domain or not FQDN_RE.match(args.platform_domain): p.error("--platform-domain is required in platform mode") + if args.create_project and (args.mode != "platform" or not NAME_RE.match(args.create_project)): + p.error("--create-project needs --mode platform and a valid Kubernetes name") + if args.mode == "platform": # the cluster chart puts its scheduler in the runai namespace + if args.kai_namespace == KAI_NAMESPACE: + args.kai_namespace = "runai" + if args.create_project: + if args.scheduler_name == "kai-scheduler": + args.scheduler_name = "runai-scheduler" + if args.queue_label_key == "kai.scheduler/queue": + args.queue_label_key = "runai/queue" if args.job_timeout < 10: p.error("--job-timeout must be at least 10 seconds") return args From ce6215441c5bc95f98287d59b6466a3dc7526ee8 Mon Sep 17 00:00:00 2001 From: Doug Holt Date: Wed, 14 Oct 2026 14:13:51 -0600 Subject: [PATCH 05/28] fix(runai): read the Run:ai API result with ['values'] (dict.values clashed with the key) --- roles/runai/tasks/platform_cluster.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/runai/tasks/platform_cluster.yml b/roles/runai/tasks/platform_cluster.yml index b2a8a6e50..d75200046 100644 --- a/roles/runai/tasks/platform_cluster.yml +++ b/roles/runai/tasks/platform_cluster.yml @@ -50,6 +50,6 @@ chart: "{{ runai_cluster_chart }}" repo: "{{ runai_chart_repo }}" version: "{{ runai_chart_version }}" - helm_values: "{{ runai_cluster_base_values | runai_deep_merge((runai_api.stdout | from_json).values) | runai_deep_merge(runai_platform_cluster_extra_values) }}" + helm_values: "{{ runai_cluster_base_values | runai_deep_merge((runai_api.stdout | from_json)['values']) | runai_deep_merge(runai_platform_cluster_extra_values) }}" sensitive: true when: runai_api is not skipped From 0db58f6ceb6f8ef76f4ecd8d29da4c41b97015a7 Mon Sep 17 00:00:00 2001 From: Doug Holt Date: Wed, 14 Oct 2026 19:05:22 -0600 Subject: [PATCH 06/28] fix(runai): add the lab domain to the node-local DNS cache too (pods use it, not CoreDNS, for outside names) --- roles/runai/tasks/coredns.yml | 8 ++++---- roles/runai/tasks/platform_dns.yml | 23 ++++++++++++++++++++++- roles/runai/tasks/uninstall.yml | 19 +++++++++++++++++++ 3 files changed, 45 insertions(+), 5 deletions(-) diff --git a/roles/runai/tasks/coredns.yml b/roles/runai/tasks/coredns.yml index 0c5e392b0..9a7d812db 100644 --- a/roles/runai/tasks/coredns.yml +++ b/roles/runai/tasks/coredns.yml @@ -2,8 +2,8 @@ # Adds (or removes) one hosts{} block in the CoreDNS Corefile for the domain. # Kubespray owns this ConfigMap and will overwrite it on its next run; for a # permanent entry set kubespray's dns_etchosts instead (see docs). -- name: Read the CoreDNS Corefile - ansible.builtin.command: "{{ runai_kubectl }} -n kube-system get configmap coredns --output json" +- name: Read the Corefile of {{ runai_dns_cm }} + ansible.builtin.command: "{{ runai_kubectl }} -n kube-system get configmap {{ runai_dns_cm }} --output json" register: runai_corefile_raw changed_when: false check_mode: false @@ -22,9 +22,9 @@ - name: Apply the Corefile change ansible.builtin.shell: | set -euo pipefail - {{ runai_kubectl }} -n kube-system patch configmap coredns --type merge \ + {{ runai_kubectl }} -n kube-system patch configmap {{ runai_dns_cm }} --type merge \ --patch "$(python3 -c 'import json,sys; print(json.dumps({"data": {"Corefile": sys.stdin.read()}}))')" - {{ runai_kubectl }} -n kube-system rollout restart deployment/coredns + {{ runai_kubectl }} -n kube-system rollout restart {{ runai_dns_workload }} args: executable: /bin/bash stdin: >- diff --git a/roles/runai/tasks/platform_dns.yml b/roles/runai/tasks/platform_dns.yml index d6970f43a..a36fb4708 100644 --- a/roles/runai/tasks/platform_dns.yml +++ b/roles/runai/tasks/platform_dns.yml @@ -15,8 +15,29 @@ become: true when: runai_platform_manage_hosts_entry | bool -- name: Resolve the domain inside the cluster +# Pods ask the node-local DNS cache (when kubespray runs one) for names outside +# the cluster domain, so the entry must be in both Corefiles. +- name: Resolve the domain inside the cluster (CoreDNS) ansible.builtin.include_tasks: coredns.yml vars: runai_coredns_present: true + runai_dns_cm: coredns + runai_dns_workload: deployment/coredns when: runai_platform_coredns_rewrite | bool + +- name: Check for the node-local DNS cache + ansible.builtin.command: "{{ runai_kubectl }} -n kube-system get configmap nodelocaldns --ignore-not-found --output name" + register: runai_nodelocal + changed_when: false + check_mode: false + when: runai_platform_coredns_rewrite | bool + +- name: Resolve the domain inside the cluster (node-local DNS cache) + ansible.builtin.include_tasks: coredns.yml + vars: + runai_coredns_present: true + runai_dns_cm: nodelocaldns + runai_dns_workload: daemonset/nodelocaldns + when: + - runai_platform_coredns_rewrite | bool + - runai_nodelocal.stdout | default('') | length > 0 diff --git a/roles/runai/tasks/uninstall.yml b/roles/runai/tasks/uninstall.yml index 48e47b0e5..2b72a86c3 100644 --- a/roles/runai/tasks/uninstall.yml +++ b/roles/runai/tasks/uninstall.yml @@ -29,8 +29,27 @@ ansible.builtin.include_tasks: coredns.yml vars: runai_coredns_present: false + runai_dns_cm: coredns + runai_dns_workload: deployment/coredns when: runai_platform_coredns_rewrite | bool +- name: Check for the node-local DNS cache + ansible.builtin.command: "{{ runai_kubectl }} -n kube-system get configmap nodelocaldns --ignore-not-found --output name" + register: runai_nodelocal + changed_when: false + check_mode: false + when: runai_platform_coredns_rewrite | bool + +- name: Remove the node-local DNS cache entry + ansible.builtin.include_tasks: coredns.yml + vars: + runai_coredns_present: false + runai_dns_cm: nodelocaldns + runai_dns_workload: daemonset/nodelocaldns + when: + - runai_platform_coredns_rewrite | bool + - runai_nodelocal.stdout | default('') | length > 0 + - name: Remove the hosts-file entry ansible.builtin.lineinfile: path: /etc/hosts From d09e24c296b1d3f3e5151b60ef88fd7bd1e2e1e4 Mon Sep 17 00:00:00 2001 From: Doug Holt Date: Wed, 14 Oct 2026 23:11:40 -0600 Subject: [PATCH 07/28] fix(runai): CoreDNS hosts block was never inserted (doubled backslashes in a block scalar); lab domain now added --- roles/runai/tasks/coredns.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/roles/runai/tasks/coredns.yml b/roles/runai/tasks/coredns.yml index 9a7d812db..ac02afe7d 100644 --- a/roles/runai/tasks/coredns.yml +++ b/roles/runai/tasks/coredns.yml @@ -16,7 +16,8 @@ - name: Compute the new Corefile ansible.builtin.set_fact: - runai_corefile_clean: "{{ runai_corefile_old | regex_replace(' hosts \\{\\n[^}]*\\} # deepops-runai\\n', '') }}" + runai_corefile_clean: >- + {{ runai_corefile_old | regex_replace(' hosts \{\n[^}]*\} # deepops-runai\n', '') }} no_log: true - name: Apply the Corefile change @@ -28,7 +29,7 @@ args: executable: /bin/bash stdin: >- - {{ runai_corefile_clean | regex_replace('(\\.:53 \\{\\n)', '\\1' ~ runai_corefile_block) if runai_coredns_present | bool else runai_corefile_clean }} + {{ runai_corefile_clean | regex_replace('(\.:53 \{\n)', '\1' ~ runai_corefile_block) if runai_coredns_present | bool else runai_corefile_clean }} stdin_add_newline: false register: runai_corefile_apply changed_when: true From faf9b7e6e56de38e272da8ec9e318cdce1ebb3e7 Mon Sep 17 00:00:00 2001 From: Doug Holt Date: Thu, 15 Oct 2026 01:33:09 -0600 Subject: [PATCH 08/28] feat(runai): validator waits (default 10 min in platform mode) for the scheduler and cluster link to come up after an install --- scripts/validation/tests/test_runai.py | 36 +++++++++++++++++++++++++- scripts/validation/validate_runai.py | 26 +++++++++++++++++++ 2 files changed, 61 insertions(+), 1 deletion(-) diff --git a/scripts/validation/tests/test_runai.py b/scripts/validation/tests/test_runai.py index 0858becb7..f173fa64d 100644 --- a/scripts/validation/tests/test_runai.py +++ b/scripts/validation/tests/test_runai.py @@ -72,7 +72,11 @@ def delete_namespace(self, namespace): def args(*extra): - return vr.parse_args(list(extra)) + # tests never really wait; the wait has its own tests below + extra = list(extra) + if "--ready-timeout" not in extra: + extra += ["--ready-timeout", "0"] + return vr.parse_args(extra) GOOD_PODS = {"kai-whole-gpu": pod("kai-whole-gpu"), "kai-half-a": pod("kai-half-a"), @@ -373,6 +377,36 @@ def test_create_project_needs_platform_mode(self): args("--create-project", "dv") +class TestReadyWait(unittest.TestCase): + def test_waits_until_scheduler_is_up(self): + bad = {"items": [deploy("binder", 1, 0)]} + kube = FakeKube(deployments=bad) + seq = iter([bad, bad, HEALTHY]) + orig = kube.get_json + + def get_json(a, timeout=60): + if a[-1] == "deployments": + return next(seq, HEALTHY), "" + return orig(a, timeout) + kube.get_json = get_json + sleeps = [] + ok = vr.wait_until_ready(kube, args("--ready-timeout", "300"), {}, sleep=sleeps.append, now=lambda: 0) + self.assertTrue(ok) + self.assertEqual(len(sleeps), 2) + + def test_gives_up_at_the_deadline(self): + kube = FakeKube(deployments={"items": [deploy("binder", 1, 0)]}) + clock = iter(range(0, 10000, 100)) + ok = vr.wait_until_ready(kube, args("--ready-timeout", "250"), {}, sleep=lambda s: None, + now=lambda: next(clock)) + self.assertFalse(ok) + + def test_default_is_off_for_kai_and_on_for_platform(self): + self.assertIsNone(vr.parse_args([]).ready_timeout) + r = vr.validate(args(), {}, FakeKube(pods=GOOD_PODS)) + self.assertTrue(r["ok"], r["failures"]) + + class TestArgs(unittest.TestCase): def test_bad_namespace_refused(self): with self.assertRaises(SystemExit): diff --git a/scripts/validation/validate_runai.py b/scripts/validation/validate_runai.py index 6829e694a..e629d6cce 100644 --- a/scripts/validation/validate_runai.py +++ b/scripts/validation/validate_runai.py @@ -222,6 +222,25 @@ def check_scheduler(kube, result, args): result["scheduler_ready"] = True +def wait_until_ready(kube, args, env, sleep=time.sleep, now=time.time): + """Right after an install the scheduler and the cluster link need a few minutes to come up. + + Poll with throw-away results until both are ready or ``--ready-timeout`` runs out; + the real checks run afterwards and report whatever is still wrong. + """ + deadline = now() + args.ready_timeout + while True: + probe = new_result(args.mode) + check_scheduler(kube, probe, args) + if args.mode == "platform": + check_platform(probe, args, env) + if probe["scheduler_ready"] and (args.mode != "platform" or probe["platform_cluster_connected"]): + return True + if now() >= deadline: + return False + sleep(20) + + def run_pod(kube, result, args, manifest, phases=("Succeeded", "Failed")): rc, _, err = kube.apply(manifest) if rc != 0: @@ -547,6 +566,10 @@ def validate(args, env, kube=None): verify_uninstalled(kube, result) result["ok"] = result["uninstalled_ok"] and not result["failures"] return result + if args.ready_timeout is None: + args.ready_timeout = 600 if args.mode == "platform" else 0 + if args.ready_timeout > 0 and not args.verify_uninstalled: + wait_until_ready(kube, args, env) check_scheduler(kube, result, args) if args.mode == "platform" and args.create_project: run_in_project(kube, result, args, env) @@ -592,6 +615,9 @@ def parse_args(argv): p.add_argument("--memory-image", default="python:3.12-slim", help="image with python3 for the GPU-memory check") p.add_argument("--job-timeout", type=int, default=300, help="seconds per pod (default 300)") + p.add_argument("--ready-timeout", type=int, default=None, + help="wait up to this many seconds for the scheduler (and in platform mode the cluster link) " + "to become ready before checking; 0 = check once (default 600 in platform mode, else 0)") p.add_argument("--skip-gpu-jobs", action="store_true", help="scheduler health only") p.add_argument("--memory-enforcement", action="store_true", help="also prove gpu-fractioning refuses allocations past a pod's share") From aeaac1f18dbcd35b5bfc385ddc906491365075c9 Mon Sep 17 00:00:00 2001 From: Doug Holt Date: Thu, 15 Oct 2026 03:39:24 -0600 Subject: [PATCH 09/28] fix(runai): platform mode looks for runai-scheduler-, accepts the cluster state field, and reports a safe cluster status summary --- scripts/validation/tests/test_runai.py | 28 +++++++++++++++++- scripts/validation/validate_runai.py | 41 +++++++++++++++++++++----- 2 files changed, 60 insertions(+), 9 deletions(-) diff --git a/scripts/validation/tests/test_runai.py b/scripts/validation/tests/test_runai.py index f173fa64d..1c3e185e0 100644 --- a/scripts/validation/tests/test_runai.py +++ b/scripts/validation/tests/test_runai.py @@ -22,7 +22,7 @@ def deploy(name, want=1, have=1): HEALTHY = {"items": [deploy(n) for n in - ("kai-operator", "binder", "queue-controller", "admission", "kai-scheduler-default")]} + ("kai-operator", "binder", "queue-controller", "admission", "kai-scheduler-default", "runai-scheduler-default")]} def pod(name, node="n1", group="g1", pg="pg-" + "x"): @@ -407,6 +407,32 @@ def test_default_is_off_for_kai_and_on_for_platform(self): self.assertTrue(r["ok"], r["failures"]) +class TestPlatformNames(unittest.TestCase): + def test_platform_scheduler_is_runai_scheduler(self): + a = args("--mode", "platform", "--platform-domain", "runai.lab.test") + self.assertEqual(a.scheduler_name, "runai-scheduler") + self.assertEqual(vr.missing_deployments(list(vr.REQUIRED_DEPLOYMENTS) + ["runai-scheduler-default"], + scheduler_name="runai-scheduler"), []) + self.assertEqual(vr.missing_deployments(list(vr.REQUIRED_DEPLOYMENTS) + ["kai-scheduler-default"], + scheduler_name="runai-scheduler"), ["runai-scheduler-"]) + + def test_cluster_connected_reads_state_or_platform_or_string(self): + for status in ({"platform": "Connected"}, {"state": "Connected"}, "Connected"): + self.assertEqual(vr.cluster_connected({"clusters": [{"name": "d", "status": status}]}, "d"), (True, True)) + self.assertEqual(vr.cluster_connected({"clusters": [{"name": "d", "status": {"state": "Disconnected"}}]}, "d"), + (True, False)) + + def test_status_summary_has_no_free_text(self): + doc = {"clusters": [{"name": "d", "status": {"state": "Waiting to connect", + "conditions": [{"message": "token=abc123"}], + "note": "password=hunter2 and more"}}]} + out = vr.cluster_status_summary(doc, "d") + self.assertEqual(out["state"], "Waiting to connect") + self.assertEqual(out["conditions"], "list") + self.assertNotIn("hunter2", json.dumps(out)) + self.assertNotIn("abc123", json.dumps(out)) + + class TestArgs(unittest.TestCase): def test_bad_namespace_refused(self): with self.assertRaises(SystemExit): diff --git a/scripts/validation/validate_runai.py b/scripts/validation/validate_runai.py index e629d6cce..ca3379658 100644 --- a/scripts/validation/validate_runai.py +++ b/scripts/validation/validate_runai.py @@ -89,10 +89,11 @@ def summarize_deployments(deploy_json): return sorted(names), sorted(bad) -def missing_deployments(names, required=REQUIRED_DEPLOYMENTS): +def missing_deployments(names, required=REQUIRED_DEPLOYMENTS, scheduler_name="kai-scheduler"): missing = [r for r in required if r not in names] - if not any(n.startswith(SCHEDULER_PREFIX) for n in names): - missing.append(SCHEDULER_PREFIX + "") + prefix = scheduler_name + "-" + if not any(n.startswith(prefix) for n in names): + missing.append(prefix + "") return missing @@ -134,11 +135,32 @@ def cluster_connected(clusters_json, cluster_name): items = clusters_json if isinstance(clusters_json, list) else clusters_json.get("clusters", []) for item in items: if item.get("name") == cluster_name: - status = item.get("status") or {} - return True, status.get("platform") == "Connected" + status = item.get("status") + if isinstance(status, str): # some API versions return the state directly + return True, status == "Connected" + status = status or {} + return True, "Connected" in (status.get("platform"), status.get("state")) return False, False +def cluster_status_summary(clusters_json, cluster_name): + """Short, secret-free view of a cluster's status for the report: key names and short enum-like values.""" + items = clusters_json if isinstance(clusters_json, list) else clusters_json.get("clusters", []) + for item in items: + if item.get("name") == cluster_name: + status = item.get("status") + if isinstance(status, str): + status = {"status": status} + out = {} + for k, v in (status or {}).items(): + if isinstance(v, str) and re.match(r"^[A-Za-z][A-Za-z ]{0,30}$", v): + out[str(k)[:30]] = v + else: + out[str(k)[:30]] = type(v).__name__ + return out + return {} + + def pod_manifest(name, namespace, image, command, args, scheduler, queue_label, queue, gpu_limit=0, annotations=None, container="gpu-workload", env=None): limits = {"nvidia.com/gpu": gpu_limit} if gpu_limit else {} @@ -210,7 +232,7 @@ def check_scheduler(kube, result, args): result["failures"].append("could not list deployments in %s: %s" % (args.kai_namespace, err)) return names, bad = summarize_deployments(doc) - missing = missing_deployments(names) + missing = missing_deployments(names, scheduler_name=args.scheduler_name) if missing: result["failures"].append("scheduler deployments missing: %s" % ", ".join(missing)) if bad: @@ -508,6 +530,8 @@ def check_platform(result, args, env): args.platform_domain, args.resolve_ip, args.ca_file, "GET", "/api/v1/clusters", headers={"Authorization": "Bearer " + bearer}) found, connected = cluster_connected(json.loads(body), args.cluster_name) if status == 200 else (False, False) + if status == 200: + result["cluster_status"] = cluster_status_summary(json.loads(body), args.cluster_name) except (OSError, ssl.SSLError, http.client.HTTPException, ValueError) as exc: result["failures"].append("cluster status request failed: %s" % type(exc).__name__) return @@ -529,6 +553,7 @@ def new_result(mode): "platform_ran": False, "platform_api_ok": False, "platform_cluster_connected": False, "project_created": False, "project_deleted": False, "uninstalled_ran": False, "uninstalled_ok": False, "leftovers": [], + "cluster_status": {}, "failures": [], } @@ -640,9 +665,9 @@ def parse_args(argv): if args.mode == "platform": # the cluster chart puts its scheduler in the runai namespace if args.kai_namespace == KAI_NAMESPACE: args.kai_namespace = "runai" + if args.scheduler_name == "kai-scheduler": + args.scheduler_name = "runai-scheduler" if args.create_project: - if args.scheduler_name == "kai-scheduler": - args.scheduler_name = "runai-scheduler" if args.queue_label_key == "kai.scheduler/queue": args.queue_label_key = "runai/queue" if args.job_timeout < 10: From 41765a2f23152f735e0157e327c10d56152e9a07 Mon Sep 17 00:00:00 2001 From: Doug Holt Date: Thu, 15 Oct 2026 03:43:08 -0600 Subject: [PATCH 10/28] fix(runai): validator cluster-status report uses fixed key names and fixed status words only --- scripts/validation/tests/test_runai.py | 25 ++++++++++++++++++------- scripts/validation/validate_runai.py | 22 ++++++++++++++++------ 2 files changed, 34 insertions(+), 13 deletions(-) diff --git a/scripts/validation/tests/test_runai.py b/scripts/validation/tests/test_runai.py index 1c3e185e0..12c5a55a3 100644 --- a/scripts/validation/tests/test_runai.py +++ b/scripts/validation/tests/test_runai.py @@ -422,15 +422,26 @@ def test_cluster_connected_reads_state_or_platform_or_string(self): self.assertEqual(vr.cluster_connected({"clusters": [{"name": "d", "status": {"state": "Disconnected"}}]}, "d"), (True, False)) - def test_status_summary_has_no_free_text(self): - doc = {"clusters": [{"name": "d", "status": {"state": "Waiting to connect", - "conditions": [{"message": "token=abc123"}], - "note": "password=hunter2 and more"}}]} + def test_status_summary_only_fixed_words(self): + # alphabetic secrets, secret-looking key names and nested text must not survive + doc = {"clusters": [{"name": "d", "status": { + "state": "Waiting to connect", "platform": "hunter", "status": "SecretTokenAbcdef", + "PasswordIsHunterTwo": "okvalue", "conditions": [{"message": "token abc"}]}}]} out = vr.cluster_status_summary(doc, "d") self.assertEqual(out["state"], "Waiting to connect") - self.assertEqual(out["conditions"], "list") - self.assertNotIn("hunter2", json.dumps(out)) - self.assertNotIn("abc123", json.dumps(out)) + self.assertEqual(out["platform"], "other") + self.assertEqual(out["status"], "other") + self.assertEqual(out["other_keys"], 2) + blob = json.dumps(out) + for bad in ("hunter", "SecretToken", "Password", "okvalue", "token", "abc"): + self.assertNotIn(bad, blob) + self.assertEqual(set(out), {"state", "platform", "status", "other_keys"}) + + def test_status_summary_string_status(self): + out = vr.cluster_status_summary({"clusters": [{"name": "d", "status": "Connected"}]}, "d") + self.assertEqual(out, {"status": "Connected"}) + out = vr.cluster_status_summary({"clusters": [{"name": "d", "status": "my-private-note"}]}, "d") + self.assertEqual(out, {"status": "other"}) class TestArgs(unittest.TestCase): diff --git a/scripts/validation/validate_runai.py b/scripts/validation/validate_runai.py index ca3379658..c3831fa18 100644 --- a/scripts/validation/validate_runai.py +++ b/scripts/validation/validate_runai.py @@ -143,8 +143,17 @@ def cluster_connected(clusters_json, cluster_name): return False, False +STATUS_KEYS = ("platform", "state", "status", "connected") +STATUS_VALUES = ("Connected", "Disconnected", "Waiting to connect", "Waiting", "Pending", "Unknown", + "Error", "Healthy", "Unhealthy", "Online", "Offline") + + def cluster_status_summary(clusters_json, cluster_name): - """Short, secret-free view of a cluster's status for the report: key names and short enum-like values.""" + """Report-safe view of a cluster's status: only fixed key names and fixed status words. + + Anything else (other keys, other values, free text) is replaced by a constant marker, so + nothing taken from the API response can reach the report. + """ items = clusters_json if isinstance(clusters_json, list) else clusters_json.get("clusters", []) for item in items: if item.get("name") == cluster_name: @@ -152,11 +161,12 @@ def cluster_status_summary(clusters_json, cluster_name): if isinstance(status, str): status = {"status": status} out = {} - for k, v in (status or {}).items(): - if isinstance(v, str) and re.match(r"^[A-Za-z][A-Za-z ]{0,30}$", v): - out[str(k)[:30]] = v - else: - out[str(k)[:30]] = type(v).__name__ + for k in STATUS_KEYS: + if isinstance(status, dict) and k in status: + v = status[k] + out[k] = v if isinstance(v, str) and v in STATUS_VALUES else "other" + if isinstance(status, dict) and any(k not in STATUS_KEYS for k in status): + out["other_keys"] = len([k for k in status if k not in STATUS_KEYS]) return out return {} From 174bc4f2b8c612c508d72bdc4127386d6beeddcd Mon Sep 17 00:00:00 2001 From: Doug Holt Date: Thu, 15 Oct 2026 08:58:14 -0600 Subject: [PATCH 11/28] fix(runai): validator status report: type-only for non-text, case-insensitive Connected, wider fixed word list --- scripts/validation/tests/test_runai.py | 15 ++++++++++++--- scripts/validation/validate_runai.py | 20 ++++++++++++++++---- 2 files changed, 28 insertions(+), 7 deletions(-) diff --git a/scripts/validation/tests/test_runai.py b/scripts/validation/tests/test_runai.py index 12c5a55a3..bfca0e096 100644 --- a/scripts/validation/tests/test_runai.py +++ b/scripts/validation/tests/test_runai.py @@ -429,8 +429,8 @@ def test_status_summary_only_fixed_words(self): "PasswordIsHunterTwo": "okvalue", "conditions": [{"message": "token abc"}]}}]} out = vr.cluster_status_summary(doc, "d") self.assertEqual(out["state"], "Waiting to connect") - self.assertEqual(out["platform"], "other") - self.assertEqual(out["status"], "other") + self.assertEqual(out["platform"], "other_text") + self.assertEqual(out["status"], "other_text") self.assertEqual(out["other_keys"], 2) blob = json.dumps(out) for bad in ("hunter", "SecretToken", "Password", "okvalue", "token", "abc"): @@ -441,7 +441,16 @@ def test_status_summary_string_status(self): out = vr.cluster_status_summary({"clusters": [{"name": "d", "status": "Connected"}]}, "d") self.assertEqual(out, {"status": "Connected"}) out = vr.cluster_status_summary({"clusters": [{"name": "d", "status": "my-private-note"}]}, "d") - self.assertEqual(out, {"status": "other"}) + self.assertEqual(out, {"status": "other_text"}) + + def test_status_summary_types_and_case(self): + doc = {"clusters": [{"name": "d", "status": {"platform": {"msg": "token xyz"}, "state": "connected", + "status": True, "connected": None}}]} + out = vr.cluster_status_summary(doc, "d") + self.assertEqual(out, {"platform": "type_dict", "state": "Connected", "status": "bool_true", + "connected": "null"}) + self.assertNotIn("xyz", json.dumps(out)) + self.assertEqual(vr.cluster_connected(doc, "d"), (True, True)) class TestArgs(unittest.TestCase): diff --git a/scripts/validation/validate_runai.py b/scripts/validation/validate_runai.py index c3831fa18..47ca78022 100644 --- a/scripts/validation/validate_runai.py +++ b/scripts/validation/validate_runai.py @@ -137,15 +137,19 @@ def cluster_connected(clusters_json, cluster_name): if item.get("name") == cluster_name: status = item.get("status") if isinstance(status, str): # some API versions return the state directly - return True, status == "Connected" + return True, status.casefold() == "connected" status = status or {} - return True, "Connected" in (status.get("platform"), status.get("state")) + vals = [str(v).casefold() for v in (status.get("platform"), status.get("state")) if isinstance(v, str)] + return True, "connected" in vals return False, False STATUS_KEYS = ("platform", "state", "status", "connected") STATUS_VALUES = ("Connected", "Disconnected", "Waiting to connect", "Waiting", "Pending", "Unknown", - "Error", "Healthy", "Unhealthy", "Online", "Offline") + "Error", "Healthy", "Unhealthy", "Online", "Offline", "Running", "Ready", "Not ready", + "Initializing", "Degraded", "Missing prerequisites", "Service issues", + "Unsupported version", "Waiting for data", "Installing", "Failed", "Active", "Inactive", + "True", "False", "OK", "Connecting", "Registered", "Created") def cluster_status_summary(clusters_json, cluster_name): @@ -161,10 +165,18 @@ def cluster_status_summary(clusters_json, cluster_name): if isinstance(status, str): status = {"status": status} out = {} + by_fold = {w.casefold(): w for w in STATUS_VALUES} for k in STATUS_KEYS: if isinstance(status, dict) and k in status: v = status[k] - out[k] = v if isinstance(v, str) and v in STATUS_VALUES else "other" + if isinstance(v, str): + out[k] = by_fold.get(v.casefold(), "other_text") + elif isinstance(v, bool): + out[k] = "bool_true" if v else "bool_false" + elif v is None: + out[k] = "null" + else: # dict, list, number: only the type name, never the content + out[k] = "type_" + type(v).__name__ if isinstance(status, dict) and any(k not in STATUS_KEYS for k in status): out["other_keys"] = len([k for k in status if k not in STATUS_KEYS]) return out From 1bca4b98ffee0b7cdffb5856184e7353f6bfbc8d Mon Sep 17 00:00:00 2001 From: Doug Holt Date: Thu, 15 Oct 2026 10:39:16 -0600 Subject: [PATCH 12/28] fix(runai): validator status report shows one nested level of fixed keys and length-bucketed text markers; accepts nested Connected --- scripts/validation/tests/test_runai.py | 29 +++++++++++----- scripts/validation/validate_runai.py | 48 ++++++++++++++++++-------- 2 files changed, 53 insertions(+), 24 deletions(-) diff --git a/scripts/validation/tests/test_runai.py b/scripts/validation/tests/test_runai.py index bfca0e096..962c03385 100644 --- a/scripts/validation/tests/test_runai.py +++ b/scripts/validation/tests/test_runai.py @@ -425,12 +425,12 @@ def test_cluster_connected_reads_state_or_platform_or_string(self): def test_status_summary_only_fixed_words(self): # alphabetic secrets, secret-looking key names and nested text must not survive doc = {"clusters": [{"name": "d", "status": { - "state": "Waiting to connect", "platform": "hunter", "status": "SecretTokenAbcdef", + "state": "Waiting to connect", "platform": "hunter", "status": "SecretTokenAbcdefghijk", "PasswordIsHunterTwo": "okvalue", "conditions": [{"message": "token abc"}]}}]} out = vr.cluster_status_summary(doc, "d") self.assertEqual(out["state"], "Waiting to connect") - self.assertEqual(out["platform"], "other_text") - self.assertEqual(out["status"], "other_text") + self.assertEqual(out["platform"], "other_text_short") + self.assertEqual(out["status"], "other_text_long") self.assertEqual(out["other_keys"], 2) blob = json.dumps(out) for bad in ("hunter", "SecretToken", "Password", "okvalue", "token", "abc"): @@ -439,17 +439,28 @@ def test_status_summary_only_fixed_words(self): def test_status_summary_string_status(self): out = vr.cluster_status_summary({"clusters": [{"name": "d", "status": "Connected"}]}, "d") - self.assertEqual(out, {"status": "Connected"}) + self.assertEqual(out, {"status": "Connected", "other_keys": 0}) out = vr.cluster_status_summary({"clusters": [{"name": "d", "status": "my-private-note"}]}, "d") - self.assertEqual(out, {"status": "other_text"}) + self.assertEqual(out, {"status": "other_text_long", "other_keys": 0}) def test_status_summary_types_and_case(self): - doc = {"clusters": [{"name": "d", "status": {"platform": {"msg": "token xyz"}, "state": "connected", + doc = {"clusters": [{"name": "d", "status": {"platform": 7, "state": "connected", "status": True, "connected": None}}]} out = vr.cluster_status_summary(doc, "d") - self.assertEqual(out, {"platform": "type_dict", "state": "Connected", "status": "bool_true", - "connected": "null"}) - self.assertNotIn("xyz", json.dumps(out)) + self.assertEqual(out, {"platform": "type_int", "state": "Connected", "status": "bool_true", + "connected": "null", "other_keys": 0}) + self.assertEqual(vr.cluster_connected(doc, "d"), (True, True)) + + def test_status_summary_one_nested_level_fixed_keys_only(self): + doc = {"clusters": [{"name": "d", "status": { + "platform": {"state": "Connected", "message": "token=abc and more", "secret": "hunter2"}, + "state": {"state": {"deep": "x"}}}}]} + out = vr.cluster_status_summary(doc, "d") + self.assertEqual(out["platform"], {"state": "Connected", "n_keys": 3}) + self.assertEqual(out["state"], {"state": "type_dict", "n_keys": 1}) + blob = json.dumps(out) + for bad in ("token", "abc", "hunter", "secret", "message", "deep"): + self.assertNotIn(bad, blob) self.assertEqual(vr.cluster_connected(doc, "d"), (True, True)) diff --git a/scripts/validation/validate_runai.py b/scripts/validation/validate_runai.py index 47ca78022..5d7ab4050 100644 --- a/scripts/validation/validate_runai.py +++ b/scripts/validation/validate_runai.py @@ -139,7 +139,11 @@ def cluster_connected(clusters_json, cluster_name): if isinstance(status, str): # some API versions return the state directly return True, status.casefold() == "connected" status = status or {} - vals = [str(v).casefold() for v in (status.get("platform"), status.get("state")) if isinstance(v, str)] + cands = [status.get("platform"), status.get("state")] + for c in list(cands): + if isinstance(c, dict): + cands += [c.get("state"), c.get("status"), c.get("platform")] + vals = [v.casefold() for v in cands if isinstance(v, str)] return True, "connected" in vals return False, False @@ -152,32 +156,46 @@ def cluster_connected(clusters_json, cluster_name): "True", "False", "OK", "Connecting", "Registered", "Created") +def _status_marker(v, by_fold, depth=0): + """One value -> a constant marker. Strings only if they are a fixed word; text becomes a length bucket.""" + if isinstance(v, str): + word = by_fold.get(v.casefold()) + if word: + return word + n = len(v) + return "other_text_empty" if n == 0 else ("other_text_short" if n <= 12 else "other_text_long") + if isinstance(v, bool): + return "bool_true" if v else "bool_false" + if v is None: + return "null" + if isinstance(v, dict) and depth == 0: + # one nested level: the same fixed keys only, everything else counted + inner = {k: _status_marker(v[k], by_fold, 1) for k in STATUS_KEYS if k in v} + inner["n_keys"] = len(v) + return inner + if isinstance(v, list): + return "type_list_%d" % min(len(v), 20) + return "type_" + type(v).__name__ + + def cluster_status_summary(clusters_json, cluster_name): - """Report-safe view of a cluster's status: only fixed key names and fixed status words. + """Report-safe view of a cluster's status: only fixed key names, fixed status words and markers. - Anything else (other keys, other values, free text) is replaced by a constant marker, so - nothing taken from the API response can reach the report. + Anything else (other keys, other values, free text) is replaced by a constant marker or a + count, so nothing taken from the API response can reach the report. """ items = clusters_json if isinstance(clusters_json, list) else clusters_json.get("clusters", []) + by_fold = {w.casefold(): w for w in STATUS_VALUES} for item in items: if item.get("name") == cluster_name: status = item.get("status") if isinstance(status, str): status = {"status": status} out = {} - by_fold = {w.casefold(): w for w in STATUS_VALUES} for k in STATUS_KEYS: if isinstance(status, dict) and k in status: - v = status[k] - if isinstance(v, str): - out[k] = by_fold.get(v.casefold(), "other_text") - elif isinstance(v, bool): - out[k] = "bool_true" if v else "bool_false" - elif v is None: - out[k] = "null" - else: # dict, list, number: only the type name, never the content - out[k] = "type_" + type(v).__name__ - if isinstance(status, dict) and any(k not in STATUS_KEYS for k in status): + out[k] = _status_marker(status[k], by_fold) + if isinstance(status, dict): out["other_keys"] = len([k for k in status if k not in STATUS_KEYS]) return out return {} From 5c89986fe88b82706f8b6a5348ac23da32c65af0 Mon Sep 17 00:00:00 2001 From: Doug Holt Date: Thu, 15 Oct 2026 13:05:36 -0600 Subject: [PATCH 13/28] fix(runai): validator tags unknown status text with length and a short hash so it can be matched offline; ignores spaces/case/underscores in fixed words --- scripts/validation/tests/test_runai.py | 15 ++++++++++++--- scripts/validation/validate_runai.py | 16 +++++++++++++--- 2 files changed, 25 insertions(+), 6 deletions(-) diff --git a/scripts/validation/tests/test_runai.py b/scripts/validation/tests/test_runai.py index 962c03385..f8c41564f 100644 --- a/scripts/validation/tests/test_runai.py +++ b/scripts/validation/tests/test_runai.py @@ -429,8 +429,8 @@ def test_status_summary_only_fixed_words(self): "PasswordIsHunterTwo": "okvalue", "conditions": [{"message": "token abc"}]}}]} out = vr.cluster_status_summary(doc, "d") self.assertEqual(out["state"], "Waiting to connect") - self.assertEqual(out["platform"], "other_text_short") - self.assertEqual(out["status"], "other_text_long") + self.assertTrue(out["platform"].startswith("other_text_len6_sha")) + self.assertTrue(out["status"].startswith("other_text_len22_sha")) self.assertEqual(out["other_keys"], 2) blob = json.dumps(out) for bad in ("hunter", "SecretToken", "Password", "okvalue", "token", "abc"): @@ -441,7 +441,7 @@ def test_status_summary_string_status(self): out = vr.cluster_status_summary({"clusters": [{"name": "d", "status": "Connected"}]}, "d") self.assertEqual(out, {"status": "Connected", "other_keys": 0}) out = vr.cluster_status_summary({"clusters": [{"name": "d", "status": "my-private-note"}]}, "d") - self.assertEqual(out, {"status": "other_text_long", "other_keys": 0}) + self.assertTrue(out["status"].startswith("other_text_len15_sha")); self.assertEqual(out["other_keys"], 0) def test_status_summary_types_and_case(self): doc = {"clusters": [{"name": "d", "status": {"platform": 7, "state": "connected", @@ -463,6 +463,15 @@ def test_status_summary_one_nested_level_fixed_keys_only(self): self.assertNotIn(bad, blob) self.assertEqual(vr.cluster_connected(doc, "d"), (True, True)) + def test_unknown_text_is_hashed_and_separators_ignored(self): + import hashlib + doc = {"clusters": [{"name": "d", "status": {"state": "WaitingToConnect", "platform": "weird-word"}}]} + out = vr.cluster_status_summary(doc, "d") + self.assertEqual(out["state"], "Waiting to connect") # separators/case ignored + tag = hashlib.sha256(b"weird-word").hexdigest()[:8] + self.assertEqual(out["platform"], "other_text_len10_sha" + tag) + self.assertNotIn("weird", json.dumps(out)) + class TestArgs(unittest.TestCase): def test_bad_namespace_refused(self): diff --git a/scripts/validation/validate_runai.py b/scripts/validation/validate_runai.py index 5d7ab4050..a6649b95a 100644 --- a/scripts/validation/validate_runai.py +++ b/scripts/validation/validate_runai.py @@ -24,6 +24,7 @@ """ import argparse +import hashlib import http.client import json import re @@ -156,14 +157,23 @@ def cluster_connected(clusters_json, cluster_name): "True", "False", "OK", "Connecting", "Registered", "Created") +def _norm_word(w): + return re.sub(r"[\s_\-]+", "", str(w)).casefold() + + def _status_marker(v, by_fold, depth=0): """One value -> a constant marker. Strings only if they are a fixed word; text becomes a length bucket.""" if isinstance(v, str): - word = by_fold.get(v.casefold()) + word = by_fold.get(_norm_word(v)) if word: return word n = len(v) - return "other_text_empty" if n == 0 else ("other_text_short" if n <= 12 else "other_text_long") + if n == 0: + return "other_text_empty" + # length plus a 32-bit tag of the text: lets an operator compare against candidate words + # offline (sha256(word)[:8]) without the text itself ever being stored + tag = hashlib.sha256(v.encode("utf-8", "replace")).hexdigest()[:8] + return "other_text_len%d_sha%s" % (min(n, 99), tag) if isinstance(v, bool): return "bool_true" if v else "bool_false" if v is None: @@ -185,7 +195,7 @@ def cluster_status_summary(clusters_json, cluster_name): count, so nothing taken from the API response can reach the report. """ items = clusters_json if isinstance(clusters_json, list) else clusters_json.get("clusters", []) - by_fold = {w.casefold(): w for w in STATUS_VALUES} + by_fold = {_norm_word(w): w for w in STATUS_VALUES} for item in items: if item.get("name") == cluster_name: status = item.get("status") From cb6c430d0ed98662f454508e24102f53732f5f91 Mon Sep 17 00:00:00 2001 From: Doug Holt Date: Thu, 15 Oct 2026 13:09:01 -0600 Subject: [PATCH 14/28] fix(runai): drop hash of unknown status text; length buckets only; larger reviewed finite word list; docs corrected --- scripts/validation/tests/test_runai.py | 29 ++++++++++++++-------- scripts/validation/validate_runai.py | 34 +++++++++++++++----------- 2 files changed, 39 insertions(+), 24 deletions(-) diff --git a/scripts/validation/tests/test_runai.py b/scripts/validation/tests/test_runai.py index f8c41564f..b653a7a9e 100644 --- a/scripts/validation/tests/test_runai.py +++ b/scripts/validation/tests/test_runai.py @@ -429,8 +429,8 @@ def test_status_summary_only_fixed_words(self): "PasswordIsHunterTwo": "okvalue", "conditions": [{"message": "token abc"}]}}]} out = vr.cluster_status_summary(doc, "d") self.assertEqual(out["state"], "Waiting to connect") - self.assertTrue(out["platform"].startswith("other_text_len6_sha")) - self.assertTrue(out["status"].startswith("other_text_len22_sha")) + self.assertEqual(out["platform"], "other_text_short") + self.assertEqual(out["status"], "other_text_long") self.assertEqual(out["other_keys"], 2) blob = json.dumps(out) for bad in ("hunter", "SecretToken", "Password", "okvalue", "token", "abc"): @@ -441,7 +441,7 @@ def test_status_summary_string_status(self): out = vr.cluster_status_summary({"clusters": [{"name": "d", "status": "Connected"}]}, "d") self.assertEqual(out, {"status": "Connected", "other_keys": 0}) out = vr.cluster_status_summary({"clusters": [{"name": "d", "status": "my-private-note"}]}, "d") - self.assertTrue(out["status"].startswith("other_text_len15_sha")); self.assertEqual(out["other_keys"], 0) + self.assertEqual(out, {"status": "other_text_long", "other_keys": 0}) def test_status_summary_types_and_case(self): doc = {"clusters": [{"name": "d", "status": {"platform": 7, "state": "connected", @@ -463,14 +463,23 @@ def test_status_summary_one_nested_level_fixed_keys_only(self): self.assertNotIn(bad, blob) self.assertEqual(vr.cluster_connected(doc, "d"), (True, True)) - def test_unknown_text_is_hashed_and_separators_ignored(self): - import hashlib - doc = {"clusters": [{"name": "d", "status": {"state": "WaitingToConnect", "platform": "weird-word"}}]} + def test_unknown_text_same_bucket_gives_identical_output(self): + def summ(txt, nested=False): + st = {"state": txt, "platform": {"state": txt}} + return json.dumps(vr.cluster_status_summary({"clusters": [{"name": "d", "status": st}]}, "d"), + sort_keys=True) + # different secrets of the same length class must be indistinguishable in the report + self.assertEqual(summ("hunter2abcdefghijk"), summ("zzzzzzzzzzzzzzzzzz")) + self.assertEqual(summ("tok"), summ("pwd")) + self.assertNotEqual(summ("tok"), summ("a-much-longer-unknown-value")) + for bad in ("hunter", "zzzz", "sha", "len"): + self.assertNotIn(bad, summ("hunter2abcdefghijk")) + + def test_separators_and_case_ignored_in_fixed_words(self): + doc = {"clusters": [{"name": "d", "status": {"state": "WaitingToConnect", "platform": "waiting_for-data"}}]} out = vr.cluster_status_summary(doc, "d") - self.assertEqual(out["state"], "Waiting to connect") # separators/case ignored - tag = hashlib.sha256(b"weird-word").hexdigest()[:8] - self.assertEqual(out["platform"], "other_text_len10_sha" + tag) - self.assertNotIn("weird", json.dumps(out)) + self.assertEqual(out["state"], "Waiting to connect") + self.assertEqual(out["platform"], "Waiting for data") class TestArgs(unittest.TestCase): diff --git a/scripts/validation/validate_runai.py b/scripts/validation/validate_runai.py index a6649b95a..f501bd362 100644 --- a/scripts/validation/validate_runai.py +++ b/scripts/validation/validate_runai.py @@ -24,7 +24,6 @@ """ import argparse -import hashlib import http.client import json import re @@ -150,11 +149,20 @@ def cluster_connected(clusters_json, cluster_name): STATUS_KEYS = ("platform", "state", "status", "connected") -STATUS_VALUES = ("Connected", "Disconnected", "Waiting to connect", "Waiting", "Pending", "Unknown", - "Error", "Healthy", "Unhealthy", "Online", "Offline", "Running", "Ready", "Not ready", - "Initializing", "Degraded", "Missing prerequisites", "Service issues", - "Unsupported version", "Waiting for data", "Installing", "Failed", "Active", "Inactive", - "True", "False", "OK", "Connecting", "Registered", "Created") +STATUS_VALUES = ( + "Connected", "Disconnected", "Waiting to connect", "Waiting", "Pending", "Unknown", "Error", + "Healthy", "Unhealthy", "Online", "Offline", "Running", "Ready", "Not ready", "Initializing", + "Degraded", "Missing prerequisites", "Missing runtime prerequisites", "Service issues", + "Unsupported version", "Waiting for data", "Installing", "Failed", "Active", "Inactive", + "True", "False", "OK", "Connecting", "Registered", "Created", "Waiting for cluster to connect", + "Waiting for the cluster to connect", "Cluster is not connected", "Not connected", + "Waiting for connection", "Waiting for cluster", "Cluster not connected", "Not installed", + "Installed", "Provisioning", "Provisioned", "Creating", "Updating", "Upgrading", "Deleting", + "Terminating", "Suspended", "Paused", "Stale", "No data", "No connection", "Connection lost", + "Heartbeat missing", "Stopped", "Partially connected", "Partially ready", "Reconnecting", + "Syncing", "Out of sync", "Synced", "Not synced", "Ready to connect", "Connection pending", + "Awaiting connection", "Not available", "Available", "Unavailable", "Conflict", "Blocked", +) def _norm_word(w): @@ -162,18 +170,16 @@ def _norm_word(w): def _status_marker(v, by_fold, depth=0): - """One value -> a constant marker. Strings only if they are a fixed word; text becomes a length bucket.""" + """One value -> a marker. A string is reported only if it equals a reviewed fixed word (canonical spelling + returned); any other text is reported as one of three length buckets, so different unknown values in the + same bucket produce identical output.""" if isinstance(v, str): word = by_fold.get(_norm_word(v)) if word: return word n = len(v) - if n == 0: - return "other_text_empty" - # length plus a 32-bit tag of the text: lets an operator compare against candidate words - # offline (sha256(word)[:8]) without the text itself ever being stored - tag = hashlib.sha256(v.encode("utf-8", "replace")).hexdigest()[:8] - return "other_text_len%d_sha%s" % (min(n, 99), tag) + # length bucket only: nothing derived from the text itself (no hash, no prefix) + return "other_text_empty" if n == 0 else ("other_text_short" if n <= 12 else "other_text_long") if isinstance(v, bool): return "bool_true" if v else "bool_false" if v is None: @@ -191,7 +197,7 @@ def _status_marker(v, by_fold, depth=0): def cluster_status_summary(clusters_json, cluster_name): """Report-safe view of a cluster's status: only fixed key names, fixed status words and markers. - Anything else (other keys, other values, free text) is replaced by a constant marker or a + Anything else (other keys, other values, free text) is replaced by a marker (a type name, a boolean, or a coarse length bucket) or a count, so nothing taken from the API response can reach the report. """ items = clusters_json if isinstance(clusters_json, list) else clusters_json.get("clusters", []) From f9018d0130aeffd9c368ecde9ed783528efa0dfb Mon Sep 17 00:00:00 2001 From: Doug Holt Date: Fri, 16 Oct 2026 13:33:08 -0600 Subject: [PATCH 15/28] fix(runai): platform mode keeps GPU fractioning off unless runai_kai_gpu_fractioning is set (the cluster chart enables it by default and it needs GPU Operator >= 26.7.1 + r615) --- roles/runai/vars/main.yml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/roles/runai/vars/main.yml b/roles/runai/vars/main.yml index 5ab6d9e9e..e50f5448a 100644 --- a/roles/runai/vars/main.yml +++ b/roles/runai/vars/main.yml @@ -35,7 +35,17 @@ runai_cp_base_values: adminPassword: "{{ runai_platform_admin_password }}" runai_cp_values: "{{ runai_cp_base_values | runai_deep_merge(runai_platform_cp_extra_values) }}" +# The Run:ai cluster chart turns GPU-memory fractioning on by default. That needs GPU Operator +# >= v26.7.1 and an r615+ driver, which DeepOps does not install by default, so the fractioning +# daemon crash-loops and Run:ai reports the cluster as "Missing prerequisites" (gpu-stack). +# Follow the same opt-in as kai mode: off unless runai_kai_gpu_fractioning is true. runai_cluster_base_values: + kai-resource-management: + kai-scheduler: + global: + gpuSharingMode: "{{ 'NvFractions' if runai_kai_gpu_fractioning | bool else 'NonMemoryEnforced' }}" + nvFractions: + set: "{{ runai_kai_gpu_fractioning | bool }}" clusterConfig: global: ingress: From cf28f58d4c400e5551831d9caa7802db66368477 Mon Sep 17 00:00:00 2001 From: Doug Holt Date: Sat, 17 Oct 2026 00:14:11 -0600 Subject: [PATCH 16/28] feat(runai): validator reports fixed-label hints when a half-GPU pod stays Pending (no event text) --- scripts/validation/tests/test_runai.py | 28 ++++++++++++++++++++++++++ scripts/validation/validate_runai.py | 28 ++++++++++++++++++++++++++ 2 files changed, 56 insertions(+) diff --git a/scripts/validation/tests/test_runai.py b/scripts/validation/tests/test_runai.py index b653a7a9e..dc8fe5d49 100644 --- a/scripts/validation/tests/test_runai.py +++ b/scripts/validation/tests/test_runai.py @@ -492,5 +492,33 @@ def test_short_timeout_refused(self): args("--job-timeout", "1") +class TestPendingHints(unittest.TestCase): + def test_only_fixed_labels_no_text(self): + orig = vr.run + secret = "Bearer SECRETTOKEN123 password=hunter2" + vr.run = lambda cmd, timeout=60, input_text=None: ( + 0, "Unschedulable not enough GPU quota " + secret, "") + try: + h = vr.pending_hints(object(), "ns", "kai-half-a") + finally: + vr.run = orig + blob = json.dumps(h) + for bad in ("SECRET", "secret", "hunter2", "Bearer", "password"): + self.assertNotIn(bad, blob) + self.assertIn("quota", h["labels"]) + self.assertIn("not-enough", h["labels"]) + self.assertEqual(sorted(h), ["event_bytes", "labels", "lines"]) + + def test_command_failure_is_empty(self): + orig = vr.run + vr.run = lambda cmd, timeout=60, input_text=None: (1, "", "token=abc") + try: + h = vr.pending_hints(object(), "ns", "p") + finally: + vr.run = orig + self.assertEqual(h["labels"], []) + self.assertNotIn("abc", json.dumps(h)) + + if __name__ == "__main__": unittest.main() diff --git a/scripts/validation/validate_runai.py b/scripts/validation/validate_runai.py index f501bd362..d1b9c15f3 100644 --- a/scripts/validation/validate_runai.py +++ b/scripts/validation/validate_runai.py @@ -348,6 +348,32 @@ def check_whole_gpu(kube, result, args): result["gpu_job_node"] = node +PENDING_HINT_WORDS = ( + "gpu-fraction", "fraction", "annotation", "not supported", "unsupported", "not enabled", "disabled", + "sharing", "quota", "over quota", "insufficient", "not enough", "no nodes", "queue", "project", + "reservation", "reservation pod", "gpu", "memory", "node pool", "nodepool", "podgroup", "pod group", + "priority", "preempt", "reclaim", "deserved", "limit", "capacity", "available", "selector", "taint", + "affinity", "scheduler", "webhook", "admission", "denied", "invalid", "unknown", "mps", "nvfractions", +) + + +def pending_hints(kube, namespace, pod): + """Fixed-label hints about why a pod is Pending. Reads the pod's own events and conditions + while its namespace still exists. Only labels from PENDING_HINT_WORDS and three counts are + returned; no event or condition text is ever copied.""" + rc, out, _ = run(["kubectl", "-n", namespace, "get", "events", "--field-selector", + "involvedObject.name=" + pod, "-o", + "jsonpath={range .items[*]}{.reason} {.message}{\"\\n\"}{end}"], timeout=60) + text = out.lower() if rc == 0 and isinstance(out, str) else "" + rc2, out2, _ = run(["kubectl", "-n", namespace, "get", "pod", pod, "-o", + "jsonpath={range .status.conditions[*]}{.type} {.reason} {.message}{\"\\n\"}{end}"], + timeout=60) + text += "\n" + (out2.lower() if rc2 == 0 and isinstance(out2, str) else "") + labels = sorted(w.replace(" ", "-") for w in PENDING_HINT_WORDS if w in text) + return {"event_bytes": len(text.strip()), "lines": len([l for l in text.splitlines() if l.strip()]), + "labels": labels} + + def check_fractions(kube, result, args): result["fraction_job_ran"] = True names = ["kai-half-a", "kai-half-b"] @@ -365,6 +391,8 @@ def check_fractions(kube, result, args): args.job_timeout) if phase not in ("Running", "Succeeded"): result["failures"].append("fractional pod %s did not start (phase=%s)" % (name, phase)) + if phase == "Pending": + result["pending_hints"] = pending_hints(kube, args.namespace, name) return placements = [] for name in names: From 8b0f8b0e7c475087d9ac78e46d6ef6a096dbe203 Mon Sep 17 00:00:00 2001 From: Doug Holt Date: Sat, 17 Oct 2026 02:28:19 -0600 Subject: [PATCH 17/28] feat(runai): --skip-fractions for platform mode (explicit, reported); document the half-GPU limit --- docs/k8s-cluster/runai.md | 8 ++++++++ scripts/validation/tests/test_runai.py | 16 ++++++++++++++-- scripts/validation/validate_runai.py | 12 +++++++++--- 3 files changed, 31 insertions(+), 5 deletions(-) diff --git a/docs/k8s-cluster/runai.md b/docs/k8s-cluster/runai.md index a387764dd..caef22074 100644 --- a/docs/k8s-cluster/runai.md +++ b/docs/k8s-cluster/runai.md @@ -184,6 +184,14 @@ its namespace and queue to the validator, or let the validator do it: add that flag `"ok": true` also requires the project to have been created and deleted. Beyond what the validator checks, DeepOps does not configure projects, users or quotas; use the Run:ai documentation. +**Known limit (live test, 26.11):** on a one-machine test cluster with an A100, the cluster +reported Connected, the whole-GPU pod ran under `runai-scheduler`, and the project was created +and deleted through the API, but two half-GPU pods stayed Pending (the scheduler said only +"Unschedulable"; the cause is not proven, GPU fractioning is off by default in this role +because the chart's fractioning needs a newer GPU software stack). The half-GPU check is +therefore not proven in platform mode. Pass `--skip-fractions` with `--create-project` to +skip it; the result then shows `"fraction_job_skipped": true` so the skip is never silent. + Run:ai 3.0 releases often. Keep `runai_chart_version` pinned and refresh it on purpose. diff --git a/scripts/validation/tests/test_runai.py b/scripts/validation/tests/test_runai.py index dc8fe5d49..c3ae28d34 100644 --- a/scripts/validation/tests/test_runai.py +++ b/scripts/validation/tests/test_runai.py @@ -317,7 +317,7 @@ def test_uninstalled_finds_leftovers(self): class TestPlatformProject(unittest.TestCase): - def _run(self, create_status=201, delete_status=200): + def _run(self, create_status=201, delete_status=200, extra=()): calls = [] def fake(domain, ip, ca, method, path, body=None, headers=None, timeout=20): @@ -344,12 +344,24 @@ def fake(domain, ip, ca, method, path, body=None, headers=None, timeout=20): kube.get_json = lambda a, timeout=60: ({}, "") if a[:2] == ["get", "namespace"] else kube_get(a, timeout) try: result = vr.validate(args("--mode", "platform", "--platform-domain", "runai.lab.test", - "--create-project", "dv"), + "--create-project", "dv", *extra), {"RUNAI_USERNAME": "u", "RUNAI_PASSWORD": "p"}, kube) finally: vr.https_request = orig return result, calls, kube + def test_skip_fractions_is_explicit(self): + result, calls, kube = self._run(extra=("--skip-fractions",)) + self.assertTrue(result["ok"], result["failures"]) + self.assertTrue(result["fraction_job_skipped"]) + self.assertFalse(result["fraction_job_ran"]) + self.assertEqual([p["metadata"]["name"] for p in kube.applied], ["kai-whole-gpu"]) + + def test_fractions_default_still_required(self): + result, calls, kube = self._run() + self.assertFalse(result["fraction_job_skipped"]) + self.assertTrue(result["fraction_job_ran"]) + def test_project_created_used_and_deleted(self): result, calls, kube = self._run() self.assertTrue(result["ok"], result["failures"]) diff --git a/scripts/validation/validate_runai.py b/scripts/validation/validate_runai.py index d1b9c15f3..5563b7e8d 100644 --- a/scripts/validation/validate_runai.py +++ b/scripts/validation/validate_runai.py @@ -574,7 +574,10 @@ def run_in_project(kube, result, args, env): result["failures"].append("project namespace %s did not appear" % args.namespace) return check_whole_gpu(kube, result, args) - check_fractions(kube, result, args) + if args.skip_fractions: + result["fraction_job_skipped"] = True + else: + check_fractions(kube, result, args) except (OSError, ssl.SSLError, http.client.HTTPException, ValueError) as exc: result["failures"].append("project run failed: %s" % type(exc).__name__) finally: @@ -632,7 +635,7 @@ def new_result(mode): "schema_version": SCHEMA_VERSION, "ok": False, "mode": mode, "api_reachable": False, "scheduler_ready": False, "gpu_job_ran": False, "gpu_job_ok": False, "gpu_job_node": "", - "fraction_job_ran": False, "fraction_job_ok": False, + "fraction_job_ran": False, "fraction_job_ok": False, "fraction_job_skipped": False, "memory_enforcement_ran": False, "memory_enforcement_ok": False, "platform_ran": False, "platform_api_ok": False, "platform_cluster_connected": False, "project_created": False, "project_deleted": False, @@ -693,7 +696,7 @@ def validate(args, env, kube=None): if args.mode == "platform" and not result["platform_ran"]: check_platform(result, args, env) ran_ok = (result["scheduler_ready"] and - (args.skip_gpu_jobs or (result["gpu_job_ok"] and result["fraction_job_ok"])) and + (args.skip_gpu_jobs or (result["gpu_job_ok"] and (result["fraction_job_ok"] or args.skip_fractions))) and (not args.memory_enforcement or result["memory_enforcement_ok"]) and (args.mode != "platform" or (result["platform_api_ok"] and result["platform_cluster_connected"] and @@ -727,6 +730,9 @@ def parse_args(argv): p.add_argument("--ready-timeout", type=int, default=None, help="wait up to this many seconds for the scheduler (and in platform mode the cluster link) " "to become ready before checking; 0 = check once (default 600 in platform mode, else 0)") + p.add_argument("--skip-fractions", action="store_true", + help="platform mode with --create-project: do not run the half-GPU check " + "(reported as fraction_job_skipped; fractional sharing is then not proven)") p.add_argument("--skip-gpu-jobs", action="store_true", help="scheduler health only") p.add_argument("--memory-enforcement", action="store_true", help="also prove gpu-fractioning refuses allocations past a pod's share") From 64982eb5ca021b84fe6c9988b20a6c1f279a0c8c Mon Sep 17 00:00:00 2001 From: Doug Holt Date: Sat, 17 Oct 2026 09:56:02 -0600 Subject: [PATCH 18/28] fix(runai): validator never deletes an existing namespace ensure_namespace now refuses a namespace that already exists, labels the one it creates, and delete_namespace only removes namespaces this run created. --- docs/k8s-cluster/runai.md | 5 ++- scripts/validation/tests/test_runai.py | 45 ++++++++++++++++++++++++++ scripts/validation/validate_runai.py | 42 ++++++++++++++++++++---- 3 files changed, 85 insertions(+), 7 deletions(-) diff --git a/docs/k8s-cluster/runai.md b/docs/k8s-cluster/runai.md index 8072ff3b8..68a403693 100644 --- a/docs/k8s-cluster/runai.md +++ b/docs/k8s-cluster/runai.md @@ -73,7 +73,10 @@ Versions are variables (`runai_*_version`); pins are refreshed with each release and completed, and (b) two pods that each asked for half a GPU landed on the same GPU and both completed. It requires `"ok": true`; if it is false, each entry in `failures` names the problem. The temporary namespace is deleted on - success and kept on failure so you can look at it. + success and kept on failure so you can look at it. The validator only ever + creates a new namespace (labelled `deepops.nvidia.com/runai-validate=owned`) and + deletes only what it created: if `--namespace` already exists it refuses and + touches nothing. Rerunning the playbook is safe; a converged run reports `changed=0`. diff --git a/scripts/validation/tests/test_runai.py b/scripts/validation/tests/test_runai.py index fdacaede8..8aea0ac30 100644 --- a/scripts/validation/tests/test_runai.py +++ b/scripts/validation/tests/test_runai.py @@ -324,3 +324,48 @@ def test_short_timeout_refused(self): if __name__ == "__main__": unittest.main() + + +class TestNamespaceOwnership(unittest.TestCase): + """The real Kube class, with kubectl replaced by a recorder. Offline.""" + + def _kube(self, existing): + cmds = [] + + def fake_run(cmd, timeout=60, input_text=None): + cmds.append(cmd) + if cmd[:3] == ["kubectl", "get", "namespace"]: + return 0, ("namespace/" + cmd[3]) if cmd[3] in existing else "", "" + return 0, "", "" + orig, vr.run = vr.run, fake_run + self.addCleanup(setattr, vr, "run", orig) + return vr.Kube(), cmds + + def test_existing_namespace_is_refused_and_never_deleted(self): + kube, cmds = self._kube({"prod"}) + self.assertNotEqual(kube.ensure_namespace("prod")[0], 0) + kube.delete_namespace("prod") + self.assertFalse([c for c in cmds if "delete" in c or "create" in c]) + + def test_unknown_existence_is_treated_as_existing(self): + kube, cmds = self._kube(set()) + vr.run = lambda cmd, timeout=60, input_text=None: (1, "", "refused") + self.assertNotEqual(kube.ensure_namespace("x")[0], 0) + + def test_fresh_namespace_is_created_labelled_and_deleted(self): + kube, cmds = self._kube(set()) + self.assertEqual(kube.ensure_namespace("fresh")[0], 0) + self.assertIn(["kubectl", "create", "namespace", "fresh"], cmds) + self.assertTrue(any("label" in c and vr.OWNER_LABEL + "=" + vr.OWNER_VALUE in c for c in cmds)) + self.assertTrue(kube.delete_namespace("fresh")) + self.assertTrue(any(c[:3] == ["kubectl", "delete", "namespace"] for c in cmds)) + + def test_validate_against_existing_namespace_deletes_nothing(self): + real, cmds = self._kube({"deepops-runai-validate"}) + fake = FakeKube(pods=GOOD_PODS) + real.get_json = fake.get_json + real.apply = fake.apply + result = vr.validate(args(), {}, real) + self.assertFalse(result["ok"]) + self.assertEqual(fake.applied, []) + self.assertFalse([c for c in cmds if "delete" in c]) diff --git a/scripts/validation/validate_runai.py b/scripts/validation/validate_runai.py index 874fd8f42..0141711f3 100644 --- a/scripts/validation/validate_runai.py +++ b/scripts/validation/validate_runai.py @@ -5,7 +5,7 @@ The verdict is strict JSON with ``--json``: ``"ok": true`` only if every check that applies to the selected mode passed. -Checks (all in a throw-away namespace that is deleted on success): +Checks (all in a fresh namespace this run creates (it refuses an existing one) and deletes on success): 1. scheduler pods are available and the KAI queue API exists 2. a whole-GPU pod scheduled by KAI runs ``nvidia-smi -L`` and completes @@ -33,6 +33,8 @@ import urllib.parse SCHEMA_VERSION = 1 +OWNER_LABEL = "deepops.nvidia.com/runai-validate" +OWNER_VALUE = "owned" KAI_NAMESPACE = "kai-scheduler" # The operator names the scheduler Deployment "-" (kai-scheduler-default). REQUIRED_DEPLOYMENTS = ("kai-operator", "binder", "queue-controller", "admission") @@ -188,13 +190,39 @@ def wait_phase(self, namespace, pod, phases, timeout): def logs(self, namespace, pod): return run(["kubectl", "-n", namespace, "logs", pod])[1] + def __init__(self): + # Namespaces this invocation created (or adopted because it created their owner). + # delete_namespace() refuses anything not in here. + self.owned = set() + + def namespace_exists(self, namespace): + """True/False, or None if the answer is unknown (treated as 'exists').""" + rc, out, _ = run(["kubectl", "get", "namespace", namespace, "--ignore-not-found", + "-o", "name"]) + if rc != 0: + return None + return bool(out) + def delete_namespace(self, namespace): + """Delete a namespace only if this run created it. Returns True if a delete was sent.""" + if namespace not in self.owned: + return False run(["kubectl", "delete", "namespace", namespace, "--ignore-not-found", "--wait=false"], timeout=120) + self.owned.discard(namespace) + return True def ensure_namespace(self, namespace): - run(["kubectl", "delete", "namespace", namespace, "--ignore-not-found"], timeout=180) - return run(["kubectl", "create", "namespace", namespace]) + """Create a fresh, labelled test namespace. Never touches an existing one.""" + if self.namespace_exists(namespace) is not False: + return 1, "", "namespace %s already exists (or could not be checked); not touching it" % namespace + rc, out, err = run(["kubectl", "create", "namespace", namespace]) + if rc != 0: + return rc, out, err + self.owned.add(namespace) + run(["kubectl", "label", "namespace", namespace, "--overwrite", + OWNER_LABEL + "=" + OWNER_VALUE]) + return rc, out, err # ------------------------------------------------------------------- the checks @@ -428,7 +456,9 @@ def validate(args, env, kube=None): check_scheduler(kube, result, args) if result["scheduler_ready"] and not args.skip_gpu_jobs: if kube.ensure_namespace(args.namespace)[0] != 0: - result["failures"].append("could not create namespace %s" % args.namespace) + result["failures"].append( + "could not create a fresh namespace %s (an existing namespace is never reused or " + "deleted; pick another --namespace or remove it yourself)" % args.namespace) else: check_whole_gpu(kube, result, args) check_fractions(kube, result, args) @@ -443,7 +473,7 @@ def validate(args, env, kube=None): (result["platform_api_ok"] and result["platform_cluster_connected"]))) result["ok"] = bool(ran_ok and not result["failures"]) if result["ok"] and not args.skip_gpu_jobs: - kube.delete_namespace(args.namespace) + kube.delete_namespace(args.namespace) # no-op unless this run created it elif not args.skip_gpu_jobs and result["gpu_job_ran"]: result["failures"].append("namespace %s kept for debugging" % args.namespace) return result @@ -457,7 +487,7 @@ def parse_args(argv): help="read-only: pass only if no kai-scheduler/runai* namespace or release remains") p.add_argument("--kai-namespace", default=KAI_NAMESPACE) p.add_argument("--namespace", default="deepops-runai-validate", - help="namespace for the test pods (deleted on success; in platform mode " + help="namespace for the test pods; must not exist yet, created and labelled by this run, deleted on success; in platform mode " "use the project namespace, e.g. runai-)") p.add_argument("--queue", default="default-queue") p.add_argument("--queue-label-key", default="kai.scheduler/queue") From 8fa8638bc03002aac3713bf8d8f900059ca94cc0 Mon Sep 17 00:00:00 2001 From: Doug Holt Date: Sat, 17 Oct 2026 10:03:02 -0600 Subject: [PATCH 19/28] runai: fix review findings (LoadBalancer untested note, project flow, skip-fractions gating, debug message) --- docs/k8s-cluster/runai.md | 18 ++++++----- roles/runai/defaults/main.yml | 2 +- scripts/validation/tests/test_runai.py | 41 ++++++++++++++++++++++++-- scripts/validation/validate_runai.py | 14 ++++++--- 4 files changed, 59 insertions(+), 16 deletions(-) diff --git a/docs/k8s-cluster/runai.md b/docs/k8s-cluster/runai.md index 59ddadd70..6f8c2da47 100644 --- a/docs/k8s-cluster/runai.md +++ b/docs/k8s-cluster/runai.md @@ -26,7 +26,8 @@ and any DeepOps-owned UI. The web UI in `platform` mode is the upstream one. - `platform`, which you must provide before running the role: - a default StorageClass (`playbooks/k8s-cluster/nfs-client-provisioner.yml`) - a load balancer for a `LoadBalancer` service ([Load Balancer and Ingress](ingress.md)), - or set `runai_platform_ingress_service_type: NodePort` + or set `runai_platform_ingress_service_type: NodePort` (only NodePort was + live-tested in 26.11; `LoadBalancer` is untested in 26.11) - `kube-prometheus-stack` 88.6.5 or newer (`scripts/k8s/deploy_monitoring.sh` with `HELM_PROMETHEUS_CHART_VERSION` set) - a DNS name for the control plane, and a TLS certificate for it @@ -175,16 +176,17 @@ Check it (the admin credentials go in the environment, never on the command line RUNAI_USERNAME=admin@example.com RUNAI_PASSWORD='...' \ python3 scripts/validation/validate_runai.py --json --mode platform \ --platform-domain runai.example.com --ca-file /opt/deepops/runai/ca.pem \ - --namespace runai- --queue + --create-project deepops-validate ``` In platform mode, pods must be submitted in a Run:ai project's namespace and use -that project's queue. Either create a project in the web UI (or API) first and pass -its namespace and queue to the validator, or let the validator do it: add -`--create-project deepops-validate` and it creates that project through the API -(one GPU of quota in the first node pool), runs the whole-GPU and half-GPU checks in -`runai-deepops-validate` with `runai-scheduler`, and deletes the project again. With -that flag `"ok": true` also requires the project to have been created and deleted. Beyond what the validator checks, DeepOps +that project's queue. The validator does this itself: `--create-project deepops-validate` +creates that project through the API (one GPU of quota in the first node pool), runs the +whole-GPU and half-GPU checks in `runai-deepops-validate` with `runai-scheduler`, and +deletes the project again. With that flag `"ok": true` also requires the project to +have been created and deleted. The validator never reuses an existing project namespace +and never deletes one it did not create, so platform mode needs `--create-project NAME` +(a name not in use) or `--skip-gpu-jobs` (control-plane and scheduler checks only). Beyond what the validator checks, DeepOps does not configure projects, users or quotas; use the Run:ai documentation. **Known limit (live test, 26.11):** on a one-machine test cluster with an A100, the cluster diff --git a/roles/runai/defaults/main.yml b/roles/runai/defaults/main.yml index 77c408376..04e10b092 100644 --- a/roles/runai/defaults/main.yml +++ b/roles/runai/defaults/main.yml @@ -70,7 +70,7 @@ runai_platform_ingress_repo: "https://haproxytech.github.io/helm-charts" runai_platform_ingress_chart_version: "1.54.2" runai_platform_ingress_release_name: "haproxy-kubernetes-ingress" runai_platform_ingress_namespace: "haproxy-controller" -# LoadBalancer works with the DeepOps MetalLB path. NodePort suits a lab. +# NodePort is the only type live-tested in 26.11; LoadBalancer is untested in 26.11. runai_platform_ingress_service_type: "LoadBalancer" # Single-node labs without a load balancer: serve 80/443 directly on the node. runai_platform_ingress_host_ports: false diff --git a/scripts/validation/tests/test_runai.py b/scripts/validation/tests/test_runai.py index 035360016..0ab2594d0 100644 --- a/scripts/validation/tests/test_runai.py +++ b/scripts/validation/tests/test_runai.py @@ -257,7 +257,7 @@ def test_platform_without_credentials_is_not_ok(self): orig = vr.https_request vr.https_request = lambda *a, **k: (200, b"") try: - result = vr.validate(args("--mode", "platform", "--platform-domain", "runai.lab.test"), + result = vr.validate(args("--mode", "platform", "--platform-domain", "runai.lab.test", "--skip-gpu-jobs"), {}, FakeKube(pods=GOOD_PODS)) finally: vr.https_request = orig @@ -281,7 +281,7 @@ def fake(domain, ip, ca, method, path, body=None, headers=None, timeout=20): orig = vr.https_request vr.https_request = fake try: - result = vr.validate(args("--mode", "platform", "--platform-domain", "runai.lab.test"), + result = vr.validate(args("--mode", "platform", "--platform-domain", "runai.lab.test", "--skip-gpu-jobs"), {"RUNAI_USERNAME": "u", "RUNAI_PASSWORD": "p"}, FakeKube(pods=GOOD_PODS)) finally: vr.https_request = orig @@ -363,6 +363,41 @@ def test_skip_fractions_is_explicit(self): self.assertFalse(result["fraction_job_ran"]) self.assertEqual([p["metadata"]["name"] for p in kube.applied], ["kai-whole-gpu"]) + def test_skip_fractions_needs_create_project(self): + with self.assertRaises(SystemExit): + args("--mode", "platform", "--platform-domain", "runai.lab.test", + "--skip-gpu-jobs", "--skip-fractions") + + def test_platform_without_project_or_skip_is_rejected(self): + with self.assertRaises(SystemExit): + args("--mode", "platform", "--platform-domain", "runai.lab.test", + "--namespace", "runai-existing") + + def test_no_kept_for_debugging_after_project_delete(self): + # a failing GPU pod in the project path: the project and namespace are already gone + orig_wait = FakeKube.wait_phase + FakeKube.wait_phase = lambda self, ns, name, phases, timeout: "Failed" + try: + result, calls, kube = self._run() + finally: + FakeKube.wait_phase = orig_wait + self.assertFalse(result["ok"]) + self.assertTrue(result["gpu_job_ran"]) + self.assertEqual(kube.deleted, ["runai-dv"]) + self.assertFalse([f for f in result["failures"] if "kept for debugging" in f]) + + def test_skipped_fractions_ok_depends_on_flag(self): + orig = vr.run_in_project + def no_skip(kube, result, a, env): + orig(kube, result, a, env) + result["fraction_job_skipped"] = False # skip flag lost: must not count as ok + vr.run_in_project = no_skip + try: + result, _, _ = self._run(extra=("--skip-fractions",)) + finally: + vr.run_in_project = orig + self.assertFalse(result["ok"]) + def test_fractions_default_still_required(self): result, calls, kube = self._run() self.assertFalse(result["fraction_job_skipped"]) @@ -450,7 +485,7 @@ def test_default_is_off_for_kai_and_on_for_platform(self): class TestPlatformNames(unittest.TestCase): def test_platform_scheduler_is_runai_scheduler(self): - a = args("--mode", "platform", "--platform-domain", "runai.lab.test") + a = args("--mode", "platform", "--platform-domain", "runai.lab.test", "--skip-gpu-jobs") self.assertEqual(a.scheduler_name, "runai-scheduler") self.assertEqual(vr.missing_deployments(list(vr.REQUIRED_DEPLOYMENTS) + ["runai-scheduler-default"], scheduler_name="runai-scheduler"), []) diff --git a/scripts/validation/validate_runai.py b/scripts/validation/validate_runai.py index c129b75da..e8d50bb4d 100644 --- a/scripts/validation/validate_runai.py +++ b/scripts/validation/validate_runai.py @@ -739,7 +739,7 @@ def validate(args, env, kube=None): if args.mode == "platform" and not result["platform_ran"]: check_platform(result, args, env) ran_ok = (result["scheduler_ready"] and - (args.skip_gpu_jobs or (result["gpu_job_ok"] and (result["fraction_job_ok"] or args.skip_fractions))) and + (args.skip_gpu_jobs or (result["gpu_job_ok"] and (result["fraction_job_ok"] or result["fraction_job_skipped"]))) and (not args.memory_enforcement or result["memory_enforcement_ok"]) and (args.mode != "platform" or (result["platform_api_ok"] and result["platform_cluster_connected"] and @@ -748,7 +748,7 @@ def validate(args, env, kube=None): if result["ok"] and not args.skip_gpu_jobs: if not args.create_project: kube.delete_namespace(args.namespace) # no-op unless this run created it - elif not args.skip_gpu_jobs and result["gpu_job_ran"]: + elif not args.skip_gpu_jobs and not args.create_project and result["gpu_job_ran"]: result["failures"].append("namespace %s kept for debugging" % args.namespace) return result @@ -761,8 +761,9 @@ def parse_args(argv): help="read-only: pass only if no kai-scheduler/runai* namespace or release remains") p.add_argument("--kai-namespace", default=KAI_NAMESPACE) p.add_argument("--namespace", default="deepops-runai-validate", - help="namespace for the test pods; must not exist yet, created and labelled by this run, deleted on success; in platform mode " - "use the project namespace, e.g. runai-)") + help="namespace for the test pods; must not exist yet, created and labelled by this run, deleted on success " + "(kai mode only; platform mode with --create-project uses runai-, " + "an existing project namespace is never reused)") p.add_argument("--queue", default="default-queue") p.add_argument("--queue-label-key", default="kai.scheduler/queue") p.add_argument("--scheduler-name", default="kai-scheduler") @@ -795,6 +796,11 @@ def parse_args(argv): p.error("--platform-domain is required in platform mode") if args.create_project and (args.mode != "platform" or not NAME_RE.match(args.create_project)): p.error("--create-project needs --mode platform and a valid Kubernetes name") + if args.skip_fractions and not args.create_project: + p.error("--skip-fractions needs --create-project (platform mode)") + if args.mode == "platform" and not args.create_project and not args.skip_gpu_jobs and not args.verify_uninstalled: + p.error("platform mode needs --create-project NAME (GPU checks run in a project this run " + "creates and deletes) or --skip-gpu-jobs; an existing project namespace is never reused") if args.mode == "platform": # the cluster chart puts its scheduler in the runai namespace if args.kai_namespace == KAI_NAMESPACE: args.kai_namespace = "runai" From 9ab5cb15fa40909d780f0884d05a654eef996f5c Mon Sep 17 00:00:00 2001 From: Doug Holt Date: Sat, 17 Oct 2026 10:03:05 -0600 Subject: [PATCH 20/28] runai: note LoadBalancer is untested in 26.11 --- docs/k8s-cluster/runai.md | 3 ++- roles/runai/defaults/main.yml | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/k8s-cluster/runai.md b/docs/k8s-cluster/runai.md index 68a403693..f61ece2d1 100644 --- a/docs/k8s-cluster/runai.md +++ b/docs/k8s-cluster/runai.md @@ -26,7 +26,8 @@ and any DeepOps-owned UI. The web UI in `platform` mode is the upstream one. - `platform`, which you must provide before running the role: - a default StorageClass (`playbooks/k8s-cluster/nfs-client-provisioner.yml`) - a load balancer for a `LoadBalancer` service ([Load Balancer and Ingress](ingress.md)), - or set `runai_platform_ingress_service_type: NodePort` + or set `runai_platform_ingress_service_type: NodePort` (only NodePort was + live-tested in 26.11; `LoadBalancer` is untested in 26.11) - `kube-prometheus-stack` 88.6.5 or newer (`scripts/k8s/deploy_monitoring.sh` with `HELM_PROMETHEUS_CHART_VERSION` set) - a DNS name for the control plane, and a TLS certificate for it diff --git a/roles/runai/defaults/main.yml b/roles/runai/defaults/main.yml index 77c408376..04e10b092 100644 --- a/roles/runai/defaults/main.yml +++ b/roles/runai/defaults/main.yml @@ -70,7 +70,7 @@ runai_platform_ingress_repo: "https://haproxytech.github.io/helm-charts" runai_platform_ingress_chart_version: "1.54.2" runai_platform_ingress_release_name: "haproxy-kubernetes-ingress" runai_platform_ingress_namespace: "haproxy-controller" -# LoadBalancer works with the DeepOps MetalLB path. NodePort suits a lab. +# NodePort is the only type live-tested in 26.11; LoadBalancer is untested in 26.11. runai_platform_ingress_service_type: "LoadBalancer" # Single-node labs without a load balancer: serve 80/443 directly on the node. runai_platform_ingress_host_ports: false From e01dfd0e2b563ef181ba27a5da7fc3c74163a541 Mon Sep 17 00:00:00 2001 From: Doug Holt Date: Sat, 17 Oct 2026 10:08:07 -0600 Subject: [PATCH 21/28] fix(runai): validator deletes its namespace by UID precondition only; create returns the UID; replacement/race regressions --- scripts/validation/tests/test_runai.py | 80 +++++++++++++++++++--- scripts/validation/validate_runai.py | 94 ++++++++++++++++++++++---- 2 files changed, 151 insertions(+), 23 deletions(-) diff --git a/scripts/validation/tests/test_runai.py b/scripts/validation/tests/test_runai.py index 8aea0ac30..25f97f270 100644 --- a/scripts/validation/tests/test_runai.py +++ b/scripts/validation/tests/test_runai.py @@ -40,6 +40,7 @@ def __init__(self, deployments=HEALTHY, crd=True, pods=None, phases=None, logs=N self.phases, self.logged, self.apply_rc, self.ns_rc = phases or {}, logs or {}, apply_rc, ns_rc self.namespaces, self.nodes = namespaces or {"items": []}, nodes self.applied, self.deleted = [], [] + self.delete_note = "" def get_json(self, args, timeout=60): if args[:2] == ["get", "nodes"]: @@ -326,39 +327,102 @@ def test_short_timeout_refused(self): unittest.main() +class _Cmds(list): + deletes = None + + class TestNamespaceOwnership(unittest.TestCase): """The real Kube class, with kubectl replaced by a recorder. Offline.""" - def _kube(self, existing): - cmds = [] + def _kube(self, existing, uid="uid-1", delete_status=200): + cmds, deletes = _Cmds(), [] def fake_run(cmd, timeout=60, input_text=None): cmds.append(cmd) if cmd[:3] == ["kubectl", "get", "namespace"]: return 0, ("namespace/" + cmd[3]) if cmd[3] in existing else "", "" + if cmd[:2] == ["kubectl", "create"]: + name = json.loads(input_text)["metadata"]["name"] + if name in existing: # lost the race: server says AlreadyExists + return 1, "", "AlreadyExists" + return 0, json.dumps({"metadata": {"name": name, "uid": uid}}), "" return 0, "", "" + + def fake_delete(path, u, timeout=60): + deletes.append((path, u)) + return delete_status, "x" orig, vr.run = vr.run, fake_run + orig_d, vr.delete_with_uid = vr.delete_with_uid, fake_delete self.addCleanup(setattr, vr, "run", orig) + self.addCleanup(setattr, vr, "delete_with_uid", orig_d) + cmds.deletes = deletes return vr.Kube(), cmds def test_existing_namespace_is_refused_and_never_deleted(self): kube, cmds = self._kube({"prod"}) self.assertNotEqual(kube.ensure_namespace("prod")[0], 0) - kube.delete_namespace("prod") - self.assertFalse([c for c in cmds if "delete" in c or "create" in c]) + self.assertFalse(kube.delete_namespace("prod")) + self.assertFalse([c for c in cmds if "delete" in c or c[:2] == ["kubectl", "create"]]) + self.assertEqual(cmds.deletes, []) def test_unknown_existence_is_treated_as_existing(self): kube, cmds = self._kube(set()) vr.run = lambda cmd, timeout=60, input_text=None: (1, "", "refused") self.assertNotEqual(kube.ensure_namespace("x")[0], 0) - def test_fresh_namespace_is_created_labelled_and_deleted(self): + def test_concurrent_creation_is_refused_and_not_owned(self): + kube, cmds = self._kube(set()) + orig = vr.run + + def racing(cmd, timeout=60, input_text=None): + if cmd[:3] == ["kubectl", "get", "namespace"]: + return 0, "", "" # looked free ... + return 1, "", "AlreadyExists" # ... but someone created it first + vr.run = racing + self.assertNotEqual(kube.ensure_namespace("raced")[0], 0) + self.assertFalse(kube.delete_namespace("raced")) + self.assertEqual(cmds.deletes, []) + + def test_unreadable_uid_means_not_owned(self): kube, cmds = self._kube(set()) + vr.run = lambda cmd, timeout=60, input_text=None: (0, "not json", "") if cmd[:2] == ["kubectl", "create"] else (0, "", "") + self.assertNotEqual(kube.ensure_namespace("x")[0], 0) + self.assertFalse(kube.delete_namespace("x")) + self.assertEqual(cmds.deletes, []) + + def test_fresh_namespace_is_created_labelled_and_deleted_by_uid(self): + kube, cmds = self._kube(set(), uid="uid-42") self.assertEqual(kube.ensure_namespace("fresh")[0], 0) - self.assertIn(["kubectl", "create", "namespace", "fresh"], cmds) - self.assertTrue(any("label" in c and vr.OWNER_LABEL + "=" + vr.OWNER_VALUE in c for c in cmds)) + create = [c for c in cmds if c[:2] == ["kubectl", "create"]] + self.assertEqual(len(create), 1) self.assertTrue(kube.delete_namespace("fresh")) - self.assertTrue(any(c[:3] == ["kubectl", "delete", "namespace"] for c in cmds)) + self.assertEqual(cmds.deletes, [("/api/v1/namespaces/fresh", "uid-42")]) + # never a by-name kubectl delete + self.assertFalse([c for c in cmds if c[:2] == ["kubectl", "delete"]]) + + def test_replacement_namespace_is_not_deleted(self): + # same name, different UID: the server answers 409 Conflict to the precondition + kube, cmds = self._kube(set(), uid="uid-old", delete_status=409) + self.assertEqual(kube.ensure_namespace("swap")[0], 0) + self.assertFalse(kube.delete_namespace("swap")) + self.assertEqual(cmds.deletes, [("/api/v1/namespaces/swap", "uid-old")]) + self.assertIn("NOT deleted", kube.delete_note) + + def test_delete_is_only_sent_once(self): + kube, cmds = self._kube(set()) + kube.ensure_namespace("once") + kube.delete_namespace("once") + kube.delete_namespace("once") + self.assertEqual(len(cmds.deletes), 1) + + def test_replacement_is_reported_as_a_failure(self): + real, cmds = self._kube(set(), delete_status=409) + fake = FakeKube(pods=GOOD_PODS) + for n in ("get_json", "apply", "wait_phase", "logs"): + setattr(real, n, getattr(fake, n)) + result = vr.validate(args(), {}, real) + self.assertFalse(result["ok"]) + self.assertTrue(any("NOT deleted" in f for f in result["failures"])) def test_validate_against_existing_namespace_deletes_nothing(self): real, cmds = self._kube({"deepops-runai-validate"}) diff --git a/scripts/validation/validate_runai.py b/scripts/validation/validate_runai.py index 0141711f3..442fe5eeb 100644 --- a/scripts/validation/validate_runai.py +++ b/scripts/validation/validate_runai.py @@ -191,9 +191,11 @@ def logs(self, namespace, pod): return run(["kubectl", "-n", namespace, "logs", pod])[1] def __init__(self): - # Namespaces this invocation created (or adopted because it created their owner). - # delete_namespace() refuses anything not in here. - self.owned = set() + # namespace name -> UID of the namespace this run created (or adopted because it + # created its owner). delete_namespace() refuses anything not in here and deletes + # with a server-side UID precondition, never by name alone. + self.owned = {} + self.delete_note = "" def namespace_exists(self, namespace): """True/False, or None if the answer is unknown (treated as 'exists').""" @@ -203,28 +205,87 @@ def namespace_exists(self, namespace): return None return bool(out) + def own_namespace(self, namespace, uid): + """Record a namespace (by UID) that this run is allowed to delete.""" + self.owned[namespace] = uid + def delete_namespace(self, namespace): - """Delete a namespace only if this run created it. Returns True if a delete was sent.""" - if namespace not in self.owned: + """Delete a namespace only if this run created it and it still has the recorded UID. + + The UID is enforced by the API server (DeleteOptions.preconditions.uid), so a + namespace that was deleted and re-created under the same name is left alone (409). + Returns True if the server accepted the delete. + """ + uid = self.owned.pop(namespace, "") + if not uid: + return False + status, msg = delete_with_uid("/api/v1/namespaces/" + urllib.parse.quote(namespace), uid) + if status in (200, 202): + return True + if status == 404: return False - run(["kubectl", "delete", "namespace", namespace, "--ignore-not-found", "--wait=false"], - timeout=120) - self.owned.discard(namespace) - return True + self.delete_note = ("namespace %s was NOT deleted (HTTP %s: %s); it is not the one this run " + "created or could not be checked; leaving it alone" % (namespace, status, msg)) + return False def ensure_namespace(self, namespace): - """Create a fresh, labelled test namespace. Never touches an existing one.""" + """Create a fresh, labelled test namespace and record its UID. Never touches an existing one. + + The create is atomic on the server (AlreadyExists if anyone got there first), and the + UID comes from the create response itself, so there is no read-then-act gap. + """ if self.namespace_exists(namespace) is not False: return 1, "", "namespace %s already exists (or could not be checked); not touching it" % namespace - rc, out, err = run(["kubectl", "create", "namespace", namespace]) + manifest = {"apiVersion": "v1", "kind": "Namespace", + "metadata": {"name": namespace, "labels": {OWNER_LABEL: OWNER_VALUE}}} + rc, out, err = run(["kubectl", "create", "-f", "-", "-o", "json"], + input_text=json.dumps(manifest)) if rc != 0: return rc, out, err - self.owned.add(namespace) - run(["kubectl", "label", "namespace", namespace, "--overwrite", - OWNER_LABEL + "=" + OWNER_VALUE]) + try: + doc = json.loads(out) + uid = doc["metadata"]["uid"] + if doc["metadata"]["name"] != namespace or not uid: + raise ValueError + except (ValueError, KeyError, TypeError): + return 1, "", "namespace %s created but its UID could not be read; not deleting it" % namespace + self.owned[namespace] = uid return rc, out, err +def delete_with_uid(path, uid, timeout=60): + """DELETE an API path with a UID precondition via a short-lived ``kubectl proxy``. + + (``kubectl delete --raw`` cannot send a request body, so it cannot carry preconditions.) + Returns (http_status, message); status 0 means the delete was not attempted or not sent. + """ + try: + proxy = subprocess.Popen(["kubectl", "proxy", "--port=0", "--address=127.0.0.1"], + stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, text=True) + except OSError as exc: + return 0, "kubectl proxy: %s" % type(exc).__name__ + try: + line = proxy.stdout.readline() + m = re.search(r"127\.0\.0\.1:(\d+)", line) + if not m: + return 0, "kubectl proxy did not start" + conn = http.client.HTTPConnection("127.0.0.1", int(m.group(1)), timeout=timeout) + body = json.dumps({"apiVersion": "v1", "kind": "DeleteOptions", + "propagationPolicy": "Background", "preconditions": {"uid": uid}}) + conn.request("DELETE", path, body, {"Content-Type": "application/json"}) + resp = conn.getresponse() + resp.read() + return resp.status, resp.reason + except (OSError, http.client.HTTPException) as exc: + return 0, type(exc).__name__ + finally: + proxy.terminate() + try: + proxy.wait(timeout=5) + except subprocess.TimeoutExpired: + proxy.kill() + + # ------------------------------------------------------------------- the checks @@ -473,7 +534,10 @@ def validate(args, env, kube=None): (result["platform_api_ok"] and result["platform_cluster_connected"]))) result["ok"] = bool(ran_ok and not result["failures"]) if result["ok"] and not args.skip_gpu_jobs: - kube.delete_namespace(args.namespace) # no-op unless this run created it + kube.delete_namespace(args.namespace) # no-op unless this run created it (UID-checked) + if kube.delete_note: + result["failures"].append(kube.delete_note) + result["ok"] = False elif not args.skip_gpu_jobs and result["gpu_job_ran"]: result["failures"].append("namespace %s kept for debugging" % args.namespace) return result From a675855dea9d03780920d9843c2da7ed2ebed275 Mon Sep 17 00:00:00 2001 From: Doug Holt Date: Sat, 17 Oct 2026 10:09:16 -0600 Subject: [PATCH 22/28] test: fake kube owns namespaces it creates --- scripts/validation/tests/test_runai.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scripts/validation/tests/test_runai.py b/scripts/validation/tests/test_runai.py index 523f240ca..8f5dd5181 100644 --- a/scripts/validation/tests/test_runai.py +++ b/scripts/validation/tests/test_runai.py @@ -66,6 +66,8 @@ def logs(self, namespace, pod_name): return self.logged.get(pod_name, "GPU 0: Test GPU (UUID: x)") def ensure_namespace(self, namespace): + if self.ns_rc == 0: + self.owned.append((namespace, "uid-fake")) return self.ns_rc, "", "" def delete_namespace(self, namespace): From a8798b72dc24a989556bbb8286c57d02023729e9 Mon Sep 17 00:00:00 2001 From: Doug Holt Date: Sat, 17 Oct 2026 10:17:05 -0600 Subject: [PATCH 23/28] runai validator: platform mode never deletes the project namespace; require label+creation-time link before GPU jobs --- scripts/validation/tests/test_runai.py | 70 ++++++++++++++++++-------- scripts/validation/validate_runai.py | 61 ++++++++++++++-------- 2 files changed, 89 insertions(+), 42 deletions(-) diff --git a/scripts/validation/tests/test_runai.py b/scripts/validation/tests/test_runai.py index 8f5dd5181..6aad2f730 100644 --- a/scripts/validation/tests/test_runai.py +++ b/scripts/validation/tests/test_runai.py @@ -78,9 +78,6 @@ def delete_namespace(self, namespace): def namespace_exists(self, namespace): return namespace in self.existing - def own_namespace(self, namespace, uid): - self.owned.append((namespace, uid)) - def args(*extra): # tests never really wait; the wait has its own tests below @@ -328,7 +325,14 @@ def test_uninstalled_finds_leftovers(self): class TestPlatformProject(unittest.TestCase): - NS_DOC = {"metadata": {"name": "runai-dv", "uid": "uid-dv", "labels": {"runai/queue": "dv"}}} + @staticmethod + def _ts(offset=0): + import time as _t + return _t.strftime("%Y-%m-%dT%H:%M:%SZ", _t.gmtime(_t.time() + offset)) + + def setUp(self): + self.NS_DOC = {"metadata": {"name": "runai-dv", "uid": "uid-dv", "creationTimestamp": self._ts(1), + "labels": {"runai/queue": "dv"}}} def _run(self, create_status=201, delete_status=200, extra=(), ns_doc=None, existing=(), wait_result="doc"): @@ -398,7 +402,7 @@ def test_no_kept_for_debugging_after_project_delete(self): FakeKube.wait_phase = orig_wait self.assertFalse(result["ok"]) self.assertTrue(result["gpu_job_ran"]) - self.assertEqual(kube.deleted, ["runai-dv"]) + self.assertEqual(kube.deleted, []) self.assertFalse([f for f in result["failures"] if "kept for debugging" in f]) def test_skipped_fractions_ok_depends_on_flag(self): @@ -432,10 +436,23 @@ def test_create_body_shape(self): self.assertEqual(body["resources"][0]["nodePool"], {"id": "7"}) self.assertEqual(body["resources"][0]["gpu"]["deserved"], 1) - def test_project_namespace_owned_only_after_create(self): + def test_happy_path_never_deletes_namespace_itself(self): result, calls, kube = self._run() - self.assertEqual(kube.owned, [("runai-dv", "uid-dv")]) - self.assertEqual(kube.deleted, ["runai-dv"]) + self.assertTrue(result["ok"], result["failures"]) + self.assertTrue(result["project_namespace_removed"]) + self.assertEqual((kube.owned, kube.deleted), ([], [])) + self.assertTrue(kube.applied) + + def test_namespace_left_after_project_delete_is_reported_not_deleted(self): + orig = vr.wait_namespace_gone + vr.wait_namespace_gone = lambda k, n, t: False + try: + result, calls, kube = self._run() + finally: + vr.wait_namespace_gone = orig + self.assertFalse(result["ok"]) + self.assertEqual(kube.deleted, []) + self.assertTrue(any("still exists" in f for f in result["failures"])) def _posted(self, calls): return [c for c in calls if c[0] == "POST" and "projects" in c[1]] @@ -450,16 +467,24 @@ def test_existing_project_namespace_is_refused_untouched(self): self.assertEqual(self._posted(calls), []) self.assertTrue(any(c[1] == "/api/v1/clusters" for c in calls)) # got past login - def test_namespace_without_project_label_is_not_adopted(self): - for doc in ({"metadata": {"name": "runai-dv", "uid": "u"}}, - {"metadata": {"name": "runai-dv", "uid": "u", "labels": {"runai/queue": "other"}}}, - {"metadata": {"name": "runai-dv", "labels": {"runai/queue": "dv"}}}, - {"metadata": {"name": "elsewhere", "uid": "u", "labels": {"runai/queue": "dv"}}}): + def test_unproven_namespace_skips_gpu_jobs(self): + old = self._ts(-3600) + for doc in ({"metadata": {"name": "runai-dv", "uid": "u", "creationTimestamp": self._ts(1)}}, + {"metadata": {"name": "runai-dv", "uid": "u", "creationTimestamp": self._ts(1), + "labels": {"runai/queue": "other"}}}, + {"metadata": {"name": "runai-dv", "creationTimestamp": self._ts(1), + "labels": {"runai/queue": "dv"}}}, + {"metadata": {"name": "elsewhere", "uid": "u", "creationTimestamp": self._ts(1), + "labels": {"runai/queue": "dv"}}}, + # foreign same-name, same-label namespace that pre-dates the project (foreign UID) + {"metadata": {"name": "runai-dv", "uid": "foreign-uid", "creationTimestamp": old, + "labels": {"runai/queue": "dv"}}}, + {"metadata": {"name": "runai-dv", "uid": "u", "labels": {"runai/queue": "dv"}}}): result, calls, kube = self._run(ns_doc=doc) self.assertFalse(result["ok"]) self.assertEqual((kube.owned, kube.deleted, kube.applied), ([], [], []), doc) - self.assertTrue(any("not provably tied" in f for f in result["failures"])) - # project is removed through the API only + self.assertFalse(result["gpu_job_ran"]) + self.assertTrue(any("not provably" in f for f in result["failures"])) self.assertEqual([c[0] for c in calls if c[1].startswith("/api/v1/org-unit")], ["POST", "DELETE"]) def test_timeout_deletes_project_via_api_and_leaves_namespace(self): @@ -474,12 +499,15 @@ def test_project_delete_failure_never_deletes_namespace(self): self.assertFalse(result["ok"]) self.assertEqual(kube.deleted, []) - def test_project_namespace_uid_helper(self): - f = vr.project_namespace_uid - self.assertEqual(f(self.NS_DOC, "runai-dv", "dv"), "uid-dv") - self.assertEqual(f(None, "runai-dv", "dv"), "") - self.assertEqual(f({"metadata": {"name": "runai-dv", "uid": 5, "labels": {"runai/queue": "dv"}}}, - "runai-dv", "dv"), "") + def test_project_namespace_linked_helper(self): + f = vr.project_namespace_linked + import time as _t + now = _t.time() + self.assertTrue(f(self.NS_DOC, "runai-dv", "dv", now)) + self.assertFalse(f(None, "runai-dv", "dv", now)) + self.assertFalse(f(self.NS_DOC, "runai-dv", "dv", now + 3600)) # older than the project + self.assertFalse(f({"metadata": {"name": "runai-dv", "uid": 5, "creationTimestamp": self._ts(1), + "labels": {"runai/queue": "dv"}}}, "runai-dv", "dv", now)) def test_create_failure_is_not_ok(self): result, _, _ = self._run(create_status=400) diff --git a/scripts/validation/validate_runai.py b/scripts/validation/validate_runai.py index 8a6c2c0e7..32464aa6d 100644 --- a/scripts/validation/validate_runai.py +++ b/scripts/validation/validate_runai.py @@ -25,6 +25,7 @@ import argparse import http.client +import datetime import json import re import shutil @@ -287,10 +288,6 @@ def namespace_exists(self, namespace): return None return bool(out) - def own_namespace(self, namespace, uid): - """Record a namespace (by UID) that this run is allowed to delete.""" - self.owned[namespace] = uid - def delete_namespace(self, namespace): """Delete a namespace only if this run created it and it still has the recorded UID. @@ -636,6 +633,16 @@ def wait_namespace(kube, namespace, timeout): return None +def wait_namespace_gone(kube, namespace, timeout): + deadline = time.time() + timeout + while True: + if kube.namespace_exists(namespace) is False: + return True + if time.time() >= deadline: + return False + time.sleep(3) + + def delete_project(args, bearer, project_id, result): try: status, _ = api_json(args, bearer, "DELETE", "/api/v1/org-unit/projects/" + urllib.parse.quote(project_id)) @@ -650,19 +657,28 @@ def delete_project(args, bearer, project_id, result): PROJECT_LABEL = "runai/queue" # Run:ai puts this label on a project's namespace -def project_namespace_uid(doc, namespace, project): - """UID of the namespace if it is provably the new project's namespace, else "". +SKEW = 5 # seconds of clock difference tolerated between this host and the API server - Requires the exact name and the Run:ai project label naming this project; anything - else (wrong name, missing/other label, no UID) is doubt and returns "". - """ + +def project_namespace_linked(doc, namespace, project, created_after): + """True only if the namespace is provably the new project's: exact name, the Run:ai + project label naming it, a UID, and a creation time not before the project was created + (epoch seconds). Any doubt is False. The namespace is never deleted by this script + either way; Run:ai owns its lifecycle.""" meta = (doc or {}).get("metadata") or {} if meta.get("name") != namespace: - return "" + return False if (meta.get("labels") or {}).get(PROJECT_LABEL) != project: - return "" + return False uid = meta.get("uid") - return uid if isinstance(uid, str) and uid else "" + if not isinstance(uid, str) or not uid: + return False + try: + made = datetime.datetime.strptime(meta.get("creationTimestamp", ""), "%Y-%m-%dT%H:%M:%SZ") + made = made.replace(tzinfo=datetime.timezone.utc).timestamp() + except ValueError: + return False + return made >= created_after - SKEW def run_in_project(kube, result, args, env): @@ -680,6 +696,7 @@ def run_in_project(kube, result, args, env): % project_ns) return try: + started = time.time() bearer, project_id = create_project(args, env, result) if not project_id: return @@ -689,14 +706,12 @@ def run_in_project(kube, result, args, env): "project namespace %s did not appear in time; the project is deleted through the API " "only and any namespace with that name is left alone (check it by hand)" % project_ns) return - uid = project_namespace_uid(doc, project_ns, args.create_project) - if not uid: + if not project_namespace_linked(doc, project_ns, args.create_project, started): result["failures"].append( - "namespace %s is not provably tied to the new project (no %s=%s label from Run:ai); " - "the project is deleted through the API only and the namespace is left alone " - "(check it by hand)" % (project_ns, PROJECT_LABEL, args.create_project)) + "namespace %s is not provably the new project's (needs the %s=%s label from Run:ai and " + "a creation time after the project was created); GPU jobs skipped, nothing run in it. " + "This script never deletes the namespace" % (project_ns, PROJECT_LABEL, args.create_project)) return - kube.own_namespace(project_ns, uid) args.namespace = project_ns args.queue = args.create_project check_whole_gpu(kube, result, args) @@ -710,9 +725,13 @@ def run_in_project(kube, result, args, env): if project_id: delete_project(args, bearer, project_id, result) if result["project_deleted"]: - kube.delete_namespace(project_ns) # only if adopted above; UID-checked - if kube.delete_note: - result["failures"].append(kube.delete_note) + # Run:ai deletes the namespace with the project; we only report, never delete. + gone = wait_namespace_gone(kube, project_ns, 60) + result["project_namespace_removed"] = gone + if not gone: + result["failures"].append( + "namespace %s still exists after the project delete; Run:ai owns it, " + "check it by hand (not deleted by this script)" % project_ns) def check_platform(result, args, env): From c77264483416b685aa0d51df0896c8f192cb764e Mon Sep 17 00:00:00 2001 From: Doug Holt Date: Sat, 17 Oct 2026 11:35:17 -0600 Subject: [PATCH 24/28] runai validator: create and own the project namespace first, bind the project to it (requestedNamespace), delete by UID --- docs/k8s-cluster/runai.md | 8 +- scripts/validation/tests/test_runai.py | 116 ++++++++++++------------- scripts/validation/validate_runai.py | 115 ++++++++++++------------ 3 files changed, 120 insertions(+), 119 deletions(-) diff --git a/docs/k8s-cluster/runai.md b/docs/k8s-cluster/runai.md index 6f8c2da47..5e1f3907a 100644 --- a/docs/k8s-cluster/runai.md +++ b/docs/k8s-cluster/runai.md @@ -182,10 +182,10 @@ RUNAI_USERNAME=admin@example.com RUNAI_PASSWORD='...' \ In platform mode, pods must be submitted in a Run:ai project's namespace and use that project's queue. The validator does this itself: `--create-project deepops-validate` creates that project through the API (one GPU of quota in the first node pool), runs the -whole-GPU and half-GPU checks in `runai-deepops-validate` with `runai-scheduler`, and -deletes the project again. With that flag `"ok": true` also requires the project to -have been created and deleted. The validator never reuses an existing project namespace -and never deletes one it did not create, so platform mode needs `--create-project NAME` +whole-GPU and half-GPU checks with `runai-scheduler` in a uniquely named namespace the +validator creates first and binds the project to, then deletes the project and that namespace (by UID). With that flag `"ok": true` also requires the project to +have been created and deleted. GPU jobs run only after the namespace still has the UID the validator recorded and Run:ai's +project label. The validator never reuses an existing namespace and never deletes one it did not create, so platform mode needs `--create-project NAME` (a name not in use) or `--skip-gpu-jobs` (control-plane and scheduler checks only). Beyond what the validator checks, DeepOps does not configure projects, users or quotas; use the Run:ai documentation. diff --git a/scripts/validation/tests/test_runai.py b/scripts/validation/tests/test_runai.py index 6aad2f730..84c74b452 100644 --- a/scripts/validation/tests/test_runai.py +++ b/scripts/validation/tests/test_runai.py @@ -40,6 +40,7 @@ def __init__(self, deployments=HEALTHY, crd=True, pods=None, phases=None, logs=N self.phases, self.logged, self.apply_rc, self.ns_rc = phases or {}, logs or {}, apply_rc, ns_rc self.namespaces, self.nodes = namespaces or {"items": []}, nodes self.applied, self.deleted, self.owned, self.existing = [], [], [], set() + self.gone = True self.delete_note = "" def get_json(self, args, timeout=60): @@ -76,7 +77,10 @@ def delete_namespace(self, namespace): self.deleted.append(namespace) def namespace_exists(self, namespace): - return namespace in self.existing + return namespace in self.existing and not self.gone + + def owned_uid(self, namespace): + return dict(self.owned).get(namespace, "") def args(*extra): @@ -325,14 +329,13 @@ def test_uninstalled_finds_leftovers(self): class TestPlatformProject(unittest.TestCase): - @staticmethod - def _ts(offset=0): - import time as _t - return _t.strftime("%Y-%m-%dT%H:%M:%SZ", _t.gmtime(_t.time() + offset)) + NS = "dv-abcd1234" def setUp(self): - self.NS_DOC = {"metadata": {"name": "runai-dv", "uid": "uid-dv", "creationTimestamp": self._ts(1), - "labels": {"runai/queue": "dv"}}} + orig = vr.new_project_namespace + vr.new_project_namespace = lambda p: self.NS + self.addCleanup(setattr, vr, "new_project_namespace", orig) + self.NS_DOC = {"metadata": {"name": self.NS, "uid": "uid-fake", "labels": {"runai/queue": "dv"}}} def _run(self, create_status=201, delete_status=200, extra=(), ns_doc=None, existing=(), wait_result="doc"): @@ -357,19 +360,17 @@ def fake(domain, ip, ca, method, path, body=None, headers=None, timeout=20): orig = vr.https_request vr.https_request = fake kube = FakeKube(pods={"kai-whole-gpu": pod("kai-whole-gpu"), "kai-half-a": pod("kai-half-a"), - "kai-half-b": pod("kai-half-b")}) + "kai-half-b": pod("kai-half-b")}, ns_rc=1 if existing else 0) kube_get = kube.get_json doc = self.NS_DOC if ns_doc is None else ns_doc - kube.existing = set(existing) kube.get_json = lambda a, timeout=60: ( (doc, "") if wait_result == "doc" else (None, "not found")) if a[:2] == ["get", "namespace"] \ else kube_get(a, timeout) - orig_wait, vr.wait_namespace = vr.wait_namespace, ( - vr.wait_namespace if wait_result == "doc" else (lambda k, n, t: None)) - self.addCleanup(setattr, vr, "wait_namespace", orig_wait) + orig_sleep, vr.time.sleep = vr.time.sleep, lambda s: None + self.addCleanup(setattr, vr.time, "sleep", orig_sleep) try: result = vr.validate(args("--mode", "platform", "--platform-domain", "runai.lab.test", - "--create-project", "dv", *extra), + "--create-project", "dv", "--job-timeout", "10", *extra), {"RUNAI_USERNAME": "u", "RUNAI_PASSWORD": "p"}, kube) finally: vr.https_request = orig @@ -402,7 +403,7 @@ def test_no_kept_for_debugging_after_project_delete(self): FakeKube.wait_phase = orig_wait self.assertFalse(result["ok"]) self.assertTrue(result["gpu_job_ran"]) - self.assertEqual(kube.deleted, []) + self.assertEqual(kube.deleted, [self.NS]) # our own namespace, removed by UID self.assertFalse([f for f in result["failures"] if "kept for debugging" in f]) def test_skipped_fractions_ok_depends_on_flag(self): @@ -427,87 +428,84 @@ def test_project_created_used_and_deleted(self): self.assertTrue(result["ok"], result["failures"]) self.assertTrue(result["project_created"] and result["project_deleted"]) self.assertEqual([m for m, p, _ in calls if p.startswith("/api/v1/org-unit")], ["POST", "DELETE"]) - self.assertEqual(kube.applied[0]["metadata"]["namespace"], "runai-dv") + self.assertEqual(kube.applied[0]["metadata"]["namespace"], self.NS) self.assertEqual(kube.applied[0]["spec"]["schedulerName"], "runai-scheduler") self.assertEqual(kube.applied[0]["metadata"]["labels"], {"runai/queue": "dv", "kai.scheduler/queue": "dv"}) + def test_project_is_bound_to_our_namespace(self): + result, calls, kube = self._run() + post = [c for c in calls if c[0] == "POST" and c[1] == "/api/v1/org-unit/projects"][0] + body = post[2] if isinstance(post[2], dict) else json.loads(post[2]) + self.assertEqual(body["requestedNamespace"], self.NS) + self.assertEqual(kube.owned, [(self.NS, "uid-fake")]) + def test_create_body_shape(self): - body = vr.project_body("dv", "c1", "7", 1) + body = vr.project_body("dv", "c1", "7", 1, "ns1") self.assertEqual(body["resources"][0]["nodePool"], {"id": "7"}) self.assertEqual(body["resources"][0]["gpu"]["deserved"], 1) + self.assertEqual(body["requestedNamespace"], "ns1") + self.assertNotIn("requestedNamespace", vr.project_body("dv", "c1", "7", 1)) - def test_happy_path_never_deletes_namespace_itself(self): + def test_namespace_deleted_by_uid_after_project(self): result, calls, kube = self._run() self.assertTrue(result["ok"], result["failures"]) + self.assertEqual(kube.deleted, [self.NS]) self.assertTrue(result["project_namespace_removed"]) - self.assertEqual((kube.owned, kube.deleted), ([], [])) - self.assertTrue(kube.applied) - def test_namespace_left_after_project_delete_is_reported_not_deleted(self): - orig = vr.wait_namespace_gone + def test_namespace_left_after_cleanup_is_reported(self): + kube_gone = vr.wait_namespace_gone vr.wait_namespace_gone = lambda k, n, t: False try: result, calls, kube = self._run() finally: - vr.wait_namespace_gone = orig + vr.wait_namespace_gone = kube_gone self.assertFalse(result["ok"]) - self.assertEqual(kube.deleted, []) self.assertTrue(any("still exists" in f for f in result["failures"])) - def _posted(self, calls): - return [c for c in calls if c[0] == "POST" and "projects" in c[1]] - - def test_existing_project_namespace_is_refused_untouched(self): - # the token fixture really returns a token, so the refusal is what stops the run - result, calls, kube = self._run(existing={"runai-dv"}) + def test_our_namespace_cannot_be_created_means_no_project(self): + result, calls, kube = self._run(existing={"x"}) self.assertFalse(result["ok"]) self.assertFalse(result["project_created"]) - self.assertTrue(any("already exists" in f for f in result["failures"]), result["failures"]) self.assertEqual((kube.deleted, kube.owned, kube.applied), ([], [], [])) - self.assertEqual(self._posted(calls), []) - self.assertTrue(any(c[1] == "/api/v1/clusters" for c in calls)) # got past login - - def test_unproven_namespace_skips_gpu_jobs(self): - old = self._ts(-3600) - for doc in ({"metadata": {"name": "runai-dv", "uid": "u", "creationTimestamp": self._ts(1)}}, - {"metadata": {"name": "runai-dv", "uid": "u", "creationTimestamp": self._ts(1), - "labels": {"runai/queue": "other"}}}, - {"metadata": {"name": "runai-dv", "creationTimestamp": self._ts(1), - "labels": {"runai/queue": "dv"}}}, - {"metadata": {"name": "elsewhere", "uid": "u", "creationTimestamp": self._ts(1), - "labels": {"runai/queue": "dv"}}}, - # foreign same-name, same-label namespace that pre-dates the project (foreign UID) - {"metadata": {"name": "runai-dv", "uid": "foreign-uid", "creationTimestamp": old, - "labels": {"runai/queue": "dv"}}}, - {"metadata": {"name": "runai-dv", "uid": "u", "labels": {"runai/queue": "dv"}}}): + self.assertEqual([c for c in calls if c[0] == "POST" and "projects" in c[1]], []) + + def test_unlinked_namespace_skips_gpu_jobs(self): + bad = ({"metadata": {"name": self.NS, "uid": "uid-fake"}}, + {"metadata": {"name": self.NS, "uid": "uid-fake", "labels": {"runai/queue": "other"}}}, + {"metadata": {"name": self.NS, "labels": {"runai/queue": "dv"}}}, + {"metadata": {"name": "elsewhere", "uid": "uid-fake", "labels": {"runai/queue": "dv"}}}, + # Dwight's test: recent-timestamp look-alike with a foreign UID + {"metadata": {"name": self.NS, "uid": "foreign-uid", + "creationTimestamp": "2999-01-01T00:00:00Z", "labels": {"runai/queue": "dv"}}}) + for doc in bad: result, calls, kube = self._run(ns_doc=doc) self.assertFalse(result["ok"]) - self.assertEqual((kube.owned, kube.deleted, kube.applied), ([], [], []), doc) + self.assertEqual(kube.applied, [], doc) self.assertFalse(result["gpu_job_ran"]) - self.assertTrue(any("not provably" in f for f in result["failures"])) + self.assertTrue(any("not bound" in f for f in result["failures"])) self.assertEqual([c[0] for c in calls if c[1].startswith("/api/v1/org-unit")], ["POST", "DELETE"]) + self.assertEqual(kube.deleted, [self.NS]) # ours, deleted by UID - def test_timeout_deletes_project_via_api_and_leaves_namespace(self): + def test_namespace_never_appears_skips_gpu_jobs(self): result, calls, kube = self._run(wait_result="timeout") self.assertFalse(result["ok"]) - self.assertEqual((kube.owned, kube.deleted, kube.applied), ([], [], [])) + self.assertEqual(kube.applied, []) self.assertTrue(result["project_deleted"]) - self.assertTrue(any("left alone" in f for f in result["failures"])) - def test_project_delete_failure_never_deletes_namespace(self): + def test_project_delete_failure_keeps_namespace(self): result, calls, kube = self._run(delete_status=500) self.assertFalse(result["ok"]) self.assertEqual(kube.deleted, []) + self.assertTrue(any("kept" in f for f in result["failures"])) def test_project_namespace_linked_helper(self): f = vr.project_namespace_linked - import time as _t - now = _t.time() - self.assertTrue(f(self.NS_DOC, "runai-dv", "dv", now)) - self.assertFalse(f(None, "runai-dv", "dv", now)) - self.assertFalse(f(self.NS_DOC, "runai-dv", "dv", now + 3600)) # older than the project - self.assertFalse(f({"metadata": {"name": "runai-dv", "uid": 5, "creationTimestamp": self._ts(1), - "labels": {"runai/queue": "dv"}}}, "runai-dv", "dv", now)) + self.assertTrue(f(self.NS_DOC, self.NS, "dv", "uid-fake")) + self.assertFalse(f(None, self.NS, "dv", "uid-fake")) + self.assertFalse(f(self.NS_DOC, self.NS, "dv", "")) + self.assertFalse(f(self.NS_DOC, self.NS, "dv", "other")) + self.assertFalse(f({"metadata": {"name": self.NS, "uid": 5, "labels": {"runai/queue": "dv"}}}, + self.NS, "dv", 5)) def test_create_failure_is_not_ok(self): result, _, _ = self._run(create_status=400) diff --git a/scripts/validation/validate_runai.py b/scripts/validation/validate_runai.py index 32464aa6d..609df5af9 100644 --- a/scripts/validation/validate_runai.py +++ b/scripts/validation/validate_runai.py @@ -28,6 +28,7 @@ import datetime import json import re +import secrets import shutil import ssl import subprocess @@ -288,6 +289,9 @@ def namespace_exists(self, namespace): return None return bool(out) + def owned_uid(self, namespace): + return self.owned.get(namespace, "") + def delete_namespace(self, namespace): """Delete a namespace only if this run created it and it still has the recorded UID. @@ -578,11 +582,14 @@ def api_login(args, env): return json.loads(body).get("access_token", "") if status == 200 else "" -def project_body(name, cluster_id, pool_id, gpus): +def project_body(name, cluster_id, pool_id, gpus, namespace=""): """Request body for ``POST /api/v1/org-unit/projects``.""" - return {"name": name, "clusterId": cluster_id, + body = {"name": name, "clusterId": cluster_id, "resources": [{"nodePool": {"id": pool_id}, "gpu": {"deserved": gpus, "limit": None, "overQuotaWeight": 1}}]} + if namespace: # bind the project to a namespace we already created (CRD: spec.namespace) + body["requestedNamespace"] = namespace + return body def pick_cluster(clusters_json, cluster_name): @@ -601,7 +608,7 @@ def pick_pool(pools_json, cluster_id): return "" -def create_project(args, env, result): +def create_project(args, env, result, namespace=""): """Create the throw-away project. Returns (bearer, project_id) or ("", "").""" bearer = api_login(args, env) if not bearer: @@ -615,7 +622,7 @@ def create_project(args, env, result): result["failures"].append("could not find the cluster or its node pool through the API") return bearer, "" status, doc = api_json(args, bearer, "POST", "/api/v1/org-unit/projects", - project_body(args.create_project, cluster_id, pool_id, 1)) + project_body(args.create_project, cluster_id, pool_id, 1, namespace)) if status >= 300 or not doc.get("id"): result["failures"].append("project create failed (HTTP %d)" % status) return bearer, "" @@ -623,16 +630,6 @@ def create_project(args, env, result): return bearer, str(doc["id"]) -def wait_namespace(kube, namespace, timeout): - deadline = time.time() + timeout - while time.time() < deadline: - doc, _ = kube.get_json(["get", "namespace", namespace]) - if doc is not None: - return doc - time.sleep(5) - return None - - def wait_namespace_gone(kube, namespace, timeout): deadline = time.time() + timeout while True: @@ -657,60 +654,62 @@ def delete_project(args, bearer, project_id, result): PROJECT_LABEL = "runai/queue" # Run:ai puts this label on a project's namespace -SKEW = 5 # seconds of clock difference tolerated between this host and the API server +def new_project_namespace(project): + """A unique, throw-away namespace name for this run (never a name anyone else would use).""" + return "%s-%s" % (project[:40].rstrip("-"), secrets.token_hex(4)) -def project_namespace_linked(doc, namespace, project, created_after): - """True only if the namespace is provably the new project's: exact name, the Run:ai - project label naming it, a UID, and a creation time not before the project was created - (epoch seconds). Any doubt is False. The namespace is never deleted by this script - either way; Run:ai owns its lifecycle.""" +def project_namespace_linked(doc, namespace, project, owned_uid): + """True only if Run:ai bound the project to the namespace THIS run created: exact name, the + UID recorded from our own create call, and the Run:ai project label naming the project. + Any doubt is False.""" meta = (doc or {}).get("metadata") or {} if meta.get("name") != namespace: return False - if (meta.get("labels") or {}).get(PROJECT_LABEL) != project: - return False uid = meta.get("uid") - if not isinstance(uid, str) or not uid: - return False - try: - made = datetime.datetime.strptime(meta.get("creationTimestamp", ""), "%Y-%m-%dT%H:%M:%SZ") - made = made.replace(tzinfo=datetime.timezone.utc).timestamp() - except ValueError: + if not isinstance(uid, str) or not uid or uid != owned_uid: return False - return made >= created_after - SKEW + return (meta.get("labels") or {}).get(PROJECT_LABEL) == project + + +def wait_project_link(kube, namespace, project, owned_uid, timeout): + """Poll until Run:ai has labelled our namespace for the project; None if it never does.""" + deadline = time.time() + timeout + while True: + doc, _ = kube.get_json(["get", "namespace", namespace]) + if project_namespace_linked(doc, namespace, project, owned_uid): + return doc + if time.time() >= deadline: + return None + time.sleep(5) def run_in_project(kube, result, args, env): - """Platform mode: create a throw-away project, run the GPU checks in its namespace, delete it.""" + """Platform mode: create our own namespace, bind a throw-away project to it, run the GPU + checks there, delete the project, then delete the namespace by UID. + + Ownership is by construction: the namespace is created by this run (atomic create, UID + recorded), the project is asked to use it (requestedNamespace), and GPU jobs run only after + the namespace still has our UID and Run:ai's project label. Nothing else is ever touched.""" check_platform(result, args, env) if not result["platform_cluster_connected"] or not result["scheduler_ready"] or args.skip_gpu_jobs: return bearer = project_id = "" - project_ns = "runai-" + args.create_project - # Never adopt a project namespace that was already there: the project delete would take - # its workloads with it. - if kube.namespace_exists(project_ns) is not False: + project_ns = new_project_namespace(args.create_project) + if kube.ensure_namespace(project_ns)[0] != 0: result["failures"].append( - "namespace %s already exists (or could not be checked); pick another --create-project name" - % project_ns) + "could not create our own project namespace %s; no project created" % project_ns) return + owned_uid = kube.owned_uid(project_ns) try: - started = time.time() - bearer, project_id = create_project(args, env, result) + bearer, project_id = create_project(args, env, result, project_ns) if not project_id: return - doc = wait_namespace(kube, project_ns, args.job_timeout) - if doc is None: + if wait_project_link(kube, project_ns, args.create_project, owned_uid, args.job_timeout) is None: result["failures"].append( - "project namespace %s did not appear in time; the project is deleted through the API " - "only and any namespace with that name is left alone (check it by hand)" % project_ns) - return - if not project_namespace_linked(doc, project_ns, args.create_project, started): - result["failures"].append( - "namespace %s is not provably the new project's (needs the %s=%s label from Run:ai and " - "a creation time after the project was created); GPU jobs skipped, nothing run in it. " - "This script never deletes the namespace" % (project_ns, PROJECT_LABEL, args.create_project)) + "project %s is not bound to our namespace %s (it needs our UID and the %s=%s label " + "from Run:ai); GPU jobs skipped, nothing run" % (args.create_project, project_ns, + PROJECT_LABEL, args.create_project)) return args.namespace = project_ns args.queue = args.create_project @@ -724,14 +723,18 @@ def run_in_project(kube, result, args, env): finally: if project_id: delete_project(args, bearer, project_id, result) - if result["project_deleted"]: - # Run:ai deletes the namespace with the project; we only report, never delete. - gone = wait_namespace_gone(kube, project_ns, 60) - result["project_namespace_removed"] = gone - if not gone: - result["failures"].append( - "namespace %s still exists after the project delete; Run:ai owns it, " - "check it by hand (not deleted by this script)" % project_ns) + if not project_id or result["project_deleted"]: + # the namespace is ours: delete it, with the UID precondition (a replacement is left alone) + kube.delete_namespace(project_ns) + if kube.delete_note: + result["failures"].append(kube.delete_note) + result["project_namespace_removed"] = wait_namespace_gone(kube, project_ns, 60) + if not result["project_namespace_removed"]: + result["failures"].append("namespace %s still exists after cleanup; check it by hand" % project_ns) + else: + result["failures"].append( + "project delete failed, so namespace %s was kept; delete project then namespace by hand" + % project_ns) def check_platform(result, args, env): From ee4ca1948929df14138861aac5a09066f281347c Mon Sep 17 00:00:00 2001 From: Doug Holt Date: Sat, 17 Oct 2026 11:40:01 -0600 Subject: [PATCH 25/28] validate_runai: fix --namespace help text --- scripts/validation/validate_runai.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/validation/validate_runai.py b/scripts/validation/validate_runai.py index 609df5af9..deda0c533 100644 --- a/scripts/validation/validate_runai.py +++ b/scripts/validation/validate_runai.py @@ -874,8 +874,8 @@ def parse_args(argv): p.add_argument("--kai-namespace", default=KAI_NAMESPACE) p.add_argument("--namespace", default="deepops-runai-validate", help="namespace for the test pods; must not exist yet, created and labelled by this run, deleted on success " - "(kai mode only; platform mode with --create-project uses runai-, " - "an existing project namespace is never reused)") + "(also used in platform mode with --create-project: this run creates it and binds " + "the new Run:ai project to it; only this exact namespace is deleted)") p.add_argument("--queue", default="default-queue") p.add_argument("--queue-label-key", default="kai.scheduler/queue") p.add_argument("--scheduler-name", default="kai-scheduler") From 17ee8bcd0d5a3bfcfaa7ff7c2da43bf805af15c6 Mon Sep 17 00:00:00 2001 From: Doug Holt Date: Sat, 17 Oct 2026 13:51:17 -0600 Subject: [PATCH 26/28] validate_runai: help text for platform namespace; drop unused import --- scripts/validation/validate_runai.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/scripts/validation/validate_runai.py b/scripts/validation/validate_runai.py index deda0c533..b61db6c0d 100644 --- a/scripts/validation/validate_runai.py +++ b/scripts/validation/validate_runai.py @@ -25,7 +25,6 @@ import argparse import http.client -import datetime import json import re import secrets @@ -874,8 +873,8 @@ def parse_args(argv): p.add_argument("--kai-namespace", default=KAI_NAMESPACE) p.add_argument("--namespace", default="deepops-runai-validate", help="namespace for the test pods; must not exist yet, created and labelled by this run, deleted on success " - "(also used in platform mode with --create-project: this run creates it and binds " - "the new Run:ai project to it; only this exact namespace is deleted)") + "(kai mode only. Platform mode with --create-project ignores this and creates its own " + "-<8 hex> namespace, binds the new Run:ai project to it, and deletes only that exact namespace)") p.add_argument("--queue", default="default-queue") p.add_argument("--queue-label-key", default="kai.scheduler/queue") p.add_argument("--scheduler-name", default="kai-scheduler") From 8ac8ef6436d70722852bf278c6dd3e06a451c186 Mon Sep 17 00:00:00 2001 From: Doug Holt Date: Sat, 17 Oct 2026 14:12:37 -0600 Subject: [PATCH 27/28] runai: fix CodeQL findings (TLS minimum version, close files, empty except, unused var) --- roles/runai/files/runai_api.py | 1 + roles/runai/filter_plugins/runai.py | 2 +- scripts/validation/tests/test_runai.py | 2 -- tests/runai/bin/helm | 8 ++++++-- 4 files changed, 8 insertions(+), 5 deletions(-) diff --git a/roles/runai/files/runai_api.py b/roles/runai/files/runai_api.py index e863eaa96..d52df149c 100644 --- a/roles/runai/files/runai_api.py +++ b/roles/runai/files/runai_api.py @@ -65,6 +65,7 @@ class Client: def __init__(self, domain, ip, ca_file, timeout=30): self.domain, self.ip, self.timeout = domain, ip or domain, timeout self.context = ssl.create_default_context(cafile=ca_file or None) + self.context.minimum_version = ssl.TLSVersion.TLSv1_2 self.bearer = "" def request(self, method, path, body=None, form=None): diff --git a/roles/runai/filter_plugins/runai.py b/roles/runai/filter_plugins/runai.py index 56ba74c8a..ea3484753 100644 --- a/roles/runai/filter_plugins/runai.py +++ b/roles/runai/filter_plugins/runai.py @@ -37,7 +37,7 @@ def runai_gpu_count(nodes_doc): try: total += max(int(value), 0) except (TypeError, ValueError): - pass + continue # non-numeric allocatable counts as zero return total diff --git a/scripts/validation/tests/test_runai.py b/scripts/validation/tests/test_runai.py index 84c74b452..2f84cd2fe 100644 --- a/scripts/validation/tests/test_runai.py +++ b/scripts/validation/tests/test_runai.py @@ -712,8 +712,6 @@ def test_unknown_existence_is_treated_as_existing(self): def test_concurrent_creation_is_refused_and_not_owned(self): kube, cmds = self._kube(set()) - orig = vr.run - def racing(cmd, timeout=60, input_text=None): if cmd[:3] == ["kubectl", "get", "namespace"]: return 0, "", "" # looked free ... diff --git a/tests/runai/bin/helm b/tests/runai/bin/helm index f3bbdb607..1dc7c3d02 100755 --- a/tests/runai/bin/helm +++ b/tests/runai/bin/helm @@ -4,14 +4,18 @@ import json, os, sys state = os.environ["FAKE_STATE"] os.makedirs(state, exist_ok=True) db = os.path.join(state, "releases.json") -rels = json.load(open(db)) if os.path.exists(db) else {} +rels = {} +if os.path.exists(db): + with open(db) as fh: + rels = json.load(fh) with open(os.path.join(state, "calls.log"), "a") as log: log.write(" ".join(a for a in sys.argv[1:] if a != "-") + "\n") a = sys.argv[1:] def opt(name, default=""): return a[a.index(name) + 1] if name in a else default def save(): - json.dump(rels, open(db, "w")) + with open(db, "w") as fh: + json.dump(rels, fh) if a[0] == "list": ns = opt("--namespace") out = [dict(name=k.split("/", 1)[1], namespace=v["ns"], chart=v["chart"], status="deployed") From 0c4377506f4c78721cc5343453753e0188b2cf37 Mon Sep 17 00:00:00 2001 From: Doug Holt Date: Sat, 17 Oct 2026 15:59:11 -0600 Subject: [PATCH 28/28] runai: set TLS minimum on a local context so CodeQL tracks it --- roles/runai/files/runai_api.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/roles/runai/files/runai_api.py b/roles/runai/files/runai_api.py index d52df149c..4c0033678 100644 --- a/roles/runai/files/runai_api.py +++ b/roles/runai/files/runai_api.py @@ -64,8 +64,9 @@ def nest(pairs): class Client: def __init__(self, domain, ip, ca_file, timeout=30): self.domain, self.ip, self.timeout = domain, ip or domain, timeout - self.context = ssl.create_default_context(cafile=ca_file or None) - self.context.minimum_version = ssl.TLSVersion.TLSv1_2 + ctx = ssl.create_default_context(cafile=ca_file or None) + ctx.minimum_version = ssl.TLSVersion.TLSv1_2 + self.context = ctx self.bearer = "" def request(self, method, path, body=None, form=None):