From 5dd986b230dae6334cef4c8eecbbba844d333f3c Mon Sep 17 00:00:00 2001 From: Yoo HoJun Date: Tue, 19 May 2026 17:01:01 +0900 Subject: [PATCH 01/29] feat: support flag parameters without valuesAllow flags to work with only key names, omitting values. --- .gitignore | 1 + src/workflow/CommandExecutor.py | 28 +++++++++++-- src/workflow/ParameterManager.py | 70 ++++++++++++++++++++++++++++++++ 3 files changed, 96 insertions(+), 3 deletions(-) diff --git a/.gitignore b/.gitignore index 227f773a..15251294 100644 --- a/.gitignore +++ b/.gitignore @@ -13,3 +13,4 @@ gdpr_consent/node_modules/ *~ .streamlit/secrets.toml docs/superpowers/ +.venv/ \ No newline at end of file diff --git a/src/workflow/CommandExecutor.py b/src/workflow/CommandExecutor.py index 86f265b2..b15a193e 100644 --- a/src/workflow/CommandExecutor.py +++ b/src/workflow/CommandExecutor.py @@ -5,7 +5,7 @@ import threading from pathlib import Path from .Logger import Logger -from .ParameterManager import ParameterManager +from .ParameterManager import ParameterManager, bool_param_paths_from_param_xml_ini import sys import importlib.util import json @@ -216,7 +216,7 @@ def read_stderr(): stdout_thread.join() stderr_thread.join() - def run_topp(self, tool: str, input_output: dict, custom_params: dict = {}) -> bool: + def run_topp(self, tool: str, input_output: dict, custom_params: dict = {}, tool_instance_name: str = None) -> bool: """ Constructs and executes commands for the specified tool OpenMS TOPP tool based on the given input and output configurations. Ensures that all input/output file lists @@ -234,6 +234,10 @@ def run_topp(self, tool: str, input_output: dict, custom_params: dict = {}) -> b tool (str): The executable name or path of the tool. input_output (dict): A dictionary specifying the input/output parameter names (as key) and their corresponding file paths (as value). custom_params (dict): A dictionary of custom parameters to pass to the tool. + tool_instance_name (str, optional): Key for ``params.json`` when it differs + from ``tool`` (e.g. multiple instances). Defaults to ``tool``. + Custom parameters whose keys appear in the tool's ParamXML ``type="bool"`` + entries are passed as valueless CLI flags (``-name`` only when enabled). Returns: bool: True if all commands succeeded, False if any failed. @@ -261,8 +265,15 @@ def run_topp(self, tool: str, input_output: dict, custom_params: dict = {}) -> b commands = [] - # Load parameters for non-defaults params = self.parameter_manager.get_parameters_from_json() + + topp_tool_ini_path = Path(self.parameter_manager.ini_dir, f"{tool}.ini") + # Keys of type="bool" in the .ini: TOPP treats these as on/off flags (omit value when off) + topp_bool_flag_param_keys = ( + bool_param_paths_from_param_xml_ini(topp_tool_ini_path, tool) + if topp_tool_ini_path.exists() + else set() + ) # Construct commands for each process for i in range(n_processes): command = [tool] @@ -284,6 +295,16 @@ def run_topp(self, tool: str, input_output: dict, custom_params: dict = {}) -> b # Add non-default TOPP tool parameters if tool in params.keys(): for k, v in params[tool].items(): + + if k in topp_bool_flag_param_keys and v != "": + # CLI flag: include "-k" only when enabled + if isinstance(v, str): + is_enabled = v.lower() == "true" + else: + is_enabled = bool(v) + if is_enabled: + command += [f"-{k}"] + continue command += [f"-{k}"] # Skip only empty strings (pass flag with no value) # Note: 0 and 0.0 are valid values, so use explicit check @@ -295,6 +316,7 @@ def run_topp(self, tool: str, input_output: dict, custom_params: dict = {}) -> b # Add custom parameters for k, v in custom_params.items(): command += [f"-{k}"] + # Skip only empty strings (pass flag with no value) # Note: 0 and 0.0 are valid values, so use explicit check if v != "" and v is not None: diff --git a/src/workflow/ParameterManager.py b/src/workflow/ParameterManager.py index b0c36263..b8f490ee 100644 --- a/src/workflow/ParameterManager.py +++ b/src/workflow/ParameterManager.py @@ -3,8 +3,52 @@ import shutil import subprocess import streamlit as st +import xml.etree.ElementTree as ET from pathlib import Path + +def bool_param_paths_from_param_xml_ini(ini_path: Path, tool_stem: str) -> set[str]: + """ + Return short parameter paths for every ```` in a ParamXML .ini file. + + Paths match the suffix after ``Tool:1:`` in pyOpenMS (e.g. ``algorithm:epd:masstrace_snr_filtering``). + """ + try: + root = ET.parse(ini_path).getroot() + except (ET.ParseError, OSError): + return set() + + def local_tag(el: ET.Element) -> str: + t = el.tag + return t.rsplit("}", 1)[-1] if isinstance(t, str) and "}" in t else str(t) + + out: set[str] = set() + + def walk(el: ET.Element, parts: tuple[str, ...]) -> None: + for ch in el: + lt = local_tag(ch) + if lt == "NODE": + nm = ch.get("name") or "" + walk(ch, parts + (nm,)) + elif lt == "ITEM" and (ch.get("type") or "").lower() == "bool": + nm = ch.get("name") or "" + segs = [p for p in parts if p] + if nm: + segs.append(nm) + if not segs: + continue + # Strip tool root NODE name and instance NODE "1" (not part of pyOpenMS short keys) + while segs and segs[0] in (tool_stem, "1"): + segs.pop(0) + if segs: + out.add(":".join(segs)) + + for ch in root: + if local_tag(ch) == "NODE": + walk(ch, ()) + return out + + class ParameterManager: """ Manages the parameters for a workflow, including saving parameters to a JSON file, @@ -29,6 +73,29 @@ def __init__(self, workflow_dir: Path, workflow_name: str = None): # Store workflow name for preset loading; default to directory stem if not provided self.workflow_name = workflow_name or workflow_dir.stem + def bool_pairs_session_key(self) -> str: + """Session state key holding a set of (tool name, param path) for bool TOPP params.""" + return f"{self.ini_dir.parent.stem}-topp-bool-pairs" + + def get_bool_param_pairs(self) -> set: + """Return the cached set of (tool, param path) bool params; empty set if none.""" + return st.session_state.get(self.bool_pairs_session_key(), set()) + + def _merge_bool_params_from_ini(self, tool: str) -> None: + """Load tool.ini (XML) and merge type=bool parameter paths into session_state.""" + ini_path = Path(self.ini_dir, f"{tool}.ini") + if not ini_path.exists(): + return + try: + sk = self.bool_pairs_session_key() + if sk not in st.session_state: + st.session_state[sk] = set() + for short in bool_param_paths_from_param_xml_ini(ini_path, tool): + st.session_state[sk].add((tool, short)) + except RuntimeError: + # No Streamlit session (e.g. plain `python` import) + pass + def create_ini(self, tool: str) -> bool: """ Create an ini file for a TOPP tool if it doesn't exist. @@ -41,11 +108,14 @@ def create_ini(self, tool: str) -> bool: """ ini_path = Path(self.ini_dir, tool + ".ini") if ini_path.exists(): + self._merge_bool_params_from_ini(tool) return True try: subprocess.call([tool, "-write_ini", str(ini_path)]) except FileNotFoundError: return False + if ini_path.exists(): + self._merge_bool_params_from_ini(tool) return ini_path.exists() def save_parameters(self) -> None: From b0e682efefe9f36b38012496096a699bffa2a984 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 25 May 2026 15:26:32 +0000 Subject: [PATCH 02/29] ci(docker): publish multi-arch (amd64 + arm64) images to GHCR MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Mirror FLASHApp's split-build / manifest-merge approach so both linux/amd64 and linux/arm64 are published for the full and simple variants. Existing `-full` / `-simple` / `latest` tags become multi-arch manifests — k8s overlays, docker-compose users, and direct `docker pull` callers transparently get the right arch. Dockerfile.arm (delta from Dockerfile): - aarch64 miniforge installer - conditional THIRDPARTY/Linux/aarch64 copy (some OpenMS releases ship an empty/missing aarch64 dir) - pruned thirdparty PATH to tools that actually have ARM builds: LuciPHOr2, MSGFPlus, ThermoRawFileParser, Comet, Percolator, Sage Dockerfile_simple.arm (delta from Dockerfile_simple): - aarch64 miniforge installer only — pyOpenMS ships aarch64 wheels on PyPI, so `pip install -r requirements.txt` works as-is The shared docker/entrypoint.sh is reused as-is on ARM: its apptainer/read-only-root handling is arch-neutral and worth keeping. Base stays ubuntu:22.04 (Redis 6.0 predates the ARM64-COW-BUG warning, so no `--ignore-warnings` flag needed). Workflow changes (build-and-test.yml): - `build` renamed `build-amd64`; per-arch tags carry `-amd64`. - New `build-arm64` job runs on `ubuntu-24.04-arm`, builds the `.arm` Dockerfiles for both variants, ends with a pull-back + /_stcore/health probe on push events. - New `create-manifest` job stitches `--amd64` + `--arm64` into multi-arch `-` and `latest` manifests. - test-apptainer / test-nginx / test-traefik / publish-apptainer keep consuming the amd64 artifact only. SIF publishing stays amd64-only this iteration. - PRs build both arches (registry cache keeps warm runs cheap) but don't push; manifest creation also skipped on PRs. Branch-protection note: the `build` required check is renamed to `build-amd64`. Admins should update protected-branch rules and add `build-arm64` / `create-manifest` if those should also be required. --- .github/workflows/build-and-test.yml | 185 +++++++++++++++++++++-- Dockerfile.arm | 211 +++++++++++++++++++++++++++ Dockerfile_simple.arm | 127 ++++++++++++++++ 3 files changed, 513 insertions(+), 10 deletions(-) create mode 100644 Dockerfile.arm create mode 100644 Dockerfile_simple.arm diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml index 6ed24064..5ca863a2 100644 --- a/.github/workflows/build-and-test.yml +++ b/.github/workflows/build-and-test.yml @@ -38,7 +38,10 @@ jobs: kubectl kustomize k8s/overlays/prod/ | \ kubeconform -summary -strict -kubernetes-version 1.28.0 -skip IngressRoute - build: + build-amd64: + # amd64 path. Produces per-arch tags `--amd64`; the + # multi-arch manifest under `-` (and `latest`) is stitched + # together in `create-manifest` once the sibling `build-arm64` succeeds. needs: lint-manifests runs-on: ubuntu-latest permissions: @@ -75,22 +78,23 @@ jobs: with: images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} tags: | - type=ref,event=branch,suffix=-${{ matrix.variant }} - type=ref,event=tag,suffix=-${{ matrix.variant }} - type=sha,prefix=,suffix=-${{ matrix.variant }} - type=raw,value=latest,enable=${{ matrix.variant == 'full' && github.event_name == 'push' && github.ref == 'refs/heads/main' }} + type=ref,event=branch,suffix=-${{ matrix.variant }}-amd64 + type=ref,event=tag,suffix=-${{ matrix.variant }}-amd64 + type=sha,prefix=,suffix=-${{ matrix.variant }}-amd64 + type=raw,value=latest-amd64,enable=${{ matrix.variant == 'full' && github.event_name == 'push' && github.ref == 'refs/heads/main' }} - name: Build and conditionally push uses: docker/build-push-action@v5 with: context: . file: ${{ matrix.dockerfile }} + platforms: linux/amd64 load: true push: ${{ github.event_name != 'pull_request' }} tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} - cache-from: type=registry,ref=${{ env.REGISTRY }}/${{ env.IMAGE_NAME_LC }}/cache:${{ matrix.variant }} - cache-to: ${{ github.event_name != 'pull_request' && format('type=registry,ref={0}/{1}/cache:{2},mode=max', env.REGISTRY, env.IMAGE_NAME_LC, matrix.variant) || '' }} + cache-from: type=registry,ref=${{ env.REGISTRY }}/${{ env.IMAGE_NAME_LC }}/cache:${{ matrix.variant }}-amd64 + cache-to: ${{ github.event_name != 'pull_request' && format('type=registry,ref={0}/{1}/cache:{2}-amd64,mode=max', env.REGISTRY, env.IMAGE_NAME_LC, matrix.variant) || '' }} build-args: | GITHUB_TOKEN=${{ secrets.GITHUB_TOKEN }} @@ -111,13 +115,174 @@ jobs: path: /tmp/image.tar retention-days: 1 + build-arm64: + # arm64 path. Runs on a native ARM64 runner (no QEMU). Produces per-arch + # tags `--arm64`; gets merged into the multi-arch manifest + # under `-` by the `create-manifest` job below. The build + # uses a separate `Dockerfile.arm` / `Dockerfile_simple.arm` that swaps + # the miniforge installer to aarch64 and (for the full variant) guards + # the THIRDPARTY/Linux/aarch64 copy. Apptainer/nginx/traefik integration + # tests still run only on the amd64 artifact — those gates do not need + # arch duplication right now (HPC consumers of the SIF are amd64). + needs: lint-manifests + runs-on: ubuntu-24.04-arm + permissions: + contents: read + packages: write + strategy: + fail-fast: false + matrix: + include: + - variant: full + dockerfile: Dockerfile.arm + - variant: simple + dockerfile: Dockerfile_simple.arm + steps: + - name: Free disk space + # OpenMS source build needs ~25 GB of scratch space; the ARM runner + # image is tighter than the AMD one out of the box. Mirrors what + # FLASHApp's publish-docker-images.yml does at the top of its ARM job. + run: | + sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc /opt/hostedtoolcache || true + sudo apt-get clean + df -h + + - uses: actions/checkout@v4 + + - name: Compute lowercase image name (OCI refs must be lowercase) + run: echo "IMAGE_NAME_LC=${IMAGE_NAME,,}" >> "$GITHUB_ENV" + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Log in to GHCR + if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository + uses: docker/login-action@v3 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Extract metadata (tags, labels) + id: meta + uses: docker/metadata-action@v5 + with: + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + tags: | + type=ref,event=branch,suffix=-${{ matrix.variant }}-arm64 + type=ref,event=tag,suffix=-${{ matrix.variant }}-arm64 + type=sha,prefix=,suffix=-${{ matrix.variant }}-arm64 + type=raw,value=latest-arm64,enable=${{ matrix.variant == 'full' && github.event_name == 'push' && github.ref == 'refs/heads/main' }} + + - name: Build and conditionally push + uses: docker/build-push-action@v5 + with: + context: . + file: ${{ matrix.dockerfile }} + platforms: linux/arm64 + push: ${{ github.event_name != 'pull_request' }} + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + cache-from: type=registry,ref=${{ env.REGISTRY }}/${{ env.IMAGE_NAME_LC }}/cache:${{ matrix.variant }}-arm64 + cache-to: ${{ github.event_name != 'pull_request' && format('type=registry,ref={0}/{1}/cache:{2}-arm64,mode=max', env.REGISTRY, env.IMAGE_NAME_LC, matrix.variant) || '' }} + provenance: false + build-args: | + GITHUB_TOKEN=${{ secrets.GITHUB_TOKEN }} + + - name: Smoke test the just-pushed arm64 image + # PRs build (validates Dockerfile.arm parses + compiles) but don't + # push, so there's nothing to pull back on PR events. On push/tag, + # pull the just-published image and verify /_stcore/health to catch + # entrypoint regressions that wouldn't surface in the build itself. + if: github.event_name != 'pull_request' + run: | + set -euo pipefail + IMAGE_REF="${{ env.REGISTRY }}/${{ env.IMAGE_NAME_LC }}:${{ github.sha }}-${{ matrix.variant }}-arm64" + echo "Smoke-testing $IMAGE_REF" + docker pull "$IMAGE_REF" + docker run -d --rm --name smoketest -p 8501:8501 "$IMAGE_REF" + for i in $(seq 1 90); do + if curl -fsSo /dev/null --max-time 2 http://127.0.0.1:8501/_stcore/health; then + echo "Streamlit healthy after ${i} attempts" + docker stop smoketest + exit 0 + fi + sleep 2 + done + echo "ERROR: /_stcore/health never returned 200" + docker logs smoketest || true + docker stop smoketest || true + exit 1 + + create-manifest: + # Stitch the per-arch tags into multi-arch manifest lists. The manifest + # tags reuse the OLD scheme (`-`, `latest`) so existing + # consumers (k8s overlays, docker-compose users, `docker pull` callers) + # keep working transparently — docker now auto-selects the right arch + # on pull. PRs don't push per-arch tags, so there's nothing to merge. + needs: [build-amd64, build-arm64] + if: github.event_name != 'pull_request' + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + strategy: + fail-fast: false + matrix: + variant: [full, simple] + steps: + - name: Compute lowercase image name + run: echo "IMAGE_NAME_LC=${IMAGE_NAME,,}" >> "$GITHUB_ENV" + + - name: Log in to GHCR + uses: docker/login-action@v3 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Compute manifest tags + id: meta + uses: docker/metadata-action@v5 + with: + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + # NB: no -amd64/-arm64 suffix here. These are the multi-arch + # manifest names; they must match the pre-arm64 tag scheme so + # `:main-full`, `:v1.0.0-full`, `:latest` continue to resolve. + tags: | + type=ref,event=branch,suffix=-${{ matrix.variant }} + type=ref,event=tag,suffix=-${{ matrix.variant }} + type=sha,prefix=,suffix=-${{ matrix.variant }} + type=raw,value=latest,enable=${{ matrix.variant == 'full' && github.event_name == 'push' && github.ref == 'refs/heads/main' }} + + - name: Create and push multi-arch manifests + # Iterate over manifest tags (newline-separated from metadata-action) + # and merge the matching `-amd64` / `-arm64` per-arch tags into each. + # `--amend` makes the step idempotent across workflow_dispatch reruns. + # `docker manifest push` accepts only one ref per invocation, hence + # the loop. + run: | + set -euo pipefail + while IFS= read -r manifest_tag; do + [ -z "$manifest_tag" ] && continue + amd_tag="${manifest_tag}-amd64" + arm_tag="${manifest_tag}-arm64" + echo "Creating manifest ${manifest_tag} from:" + echo " amd: ${amd_tag}" + echo " arm: ${arm_tag}" + docker manifest create "$manifest_tag" \ + --amend "$amd_tag" \ + --amend "$arm_tag" + docker manifest push "$manifest_tag" + done <<< "${{ steps.meta.outputs.tags }}" + test-apptainer: # Apptainer/Singularity is the dominant container runtime on HPC clusters. # It mounts the root filesystem read-only and runs as the host user's UID # (not root inside the image). The entrypoint must tolerate both: this job # exercises that contract by running the built image under apptainer and # waiting for the streamlit /_stcore/health endpoint to come up. - needs: build + needs: build-amd64 runs-on: ubuntu-latest strategy: fail-fast: false @@ -335,7 +500,7 @@ jobs: done <<< "${{ steps.meta.outputs.tags }}" test-nginx: - needs: build + needs: build-amd64 runs-on: ubuntu-latest strategy: fail-fast: false @@ -422,7 +587,7 @@ jobs: done test-traefik: - needs: build + needs: build-amd64 runs-on: ubuntu-latest strategy: fail-fast: false diff --git a/Dockerfile.arm b/Dockerfile.arm new file mode 100644 index 00000000..8571bc57 --- /dev/null +++ b/Dockerfile.arm @@ -0,0 +1,211 @@ +# This Dockerfile builds OpenMS, the TOPP tools, pyOpenMS and thidparty tools. +# It also adds a basic streamlit server that serves a pyOpenMS-based app. +# hints: +# build image and give it a name (here: streamlitapp) with: docker build -f Dockerfile.arm --no-cache -t streamlitapp:latest-arm64 --build-arg GITHUB_TOKEN= . 2>&1 | tee build.log +# check if image was build: docker image ls +# run container: docker run -p 8501:8501 streamlitappsimple:latest +# debug container after build (comment out ENTRYPOINT) and run container with interactive /bin/bash shell +# prune unused images/etc. to free disc space (e.g. might be needed on gitpod). Use with care.: docker system prune --all --force + +FROM ubuntu:22.04 AS setup-build-system +ARG OPENMS_REPO=https://github.com/OpenMS/OpenMS.git +ARG OPENMS_BRANCH=release/3.5.0 +ARG PORT=8501 +# Streamlit app GitHub user name (to download artifact from). +ARG GITHUB_USER=OpenMS +# Streamlit app GitHub repository name (to download artifact from). +ARG GITHUB_REPO=streamlit-template + +USER root + +# Install required Ubuntu packages. +RUN apt-get -y update +RUN apt-get install -y --no-install-recommends --no-install-suggests g++ autoconf automake patch libtool make git gpg wget ca-certificates curl jq libgtk2.0-dev openjdk-8-jdk cron +RUN update-ca-certificates +RUN apt-get install -y --no-install-recommends --no-install-suggests libsvm-dev libeigen3-dev coinor-libcbc-dev libglpk-dev libzip-dev zlib1g-dev libxerces-c-dev libbz2-dev libomp-dev libhdf5-dev +RUN apt-get install -y --no-install-recommends --no-install-suggests libboost-date-time1.74-dev \ + libboost-iostreams1.74-dev \ + libboost-regex1.74-dev \ + libboost-math1.74-dev \ + libboost-random1.74-dev +RUN apt-get install -y --no-install-recommends --no-install-suggests qt6-base-dev libqt6svg6-dev libqt6opengl6-dev libqt6openglwidgets6 libgl-dev + +# Install Github CLI +RUN (type -p wget >/dev/null || (apt-get update && apt-get install wget -y)) \ + && mkdir -p -m 755 /etc/apt/keyrings \ + && wget -qO- https://cli.github.com/packages/githubcli-archive-keyring.gpg | tee /etc/apt/keyrings/githubcli-archive-keyring.gpg > /dev/null \ + && chmod go+r /etc/apt/keyrings/githubcli-archive-keyring.gpg \ + && echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/githubcli-archive-keyring.gpg] https://cli.github.com/packages stable main" | tee /etc/apt/sources.list.d/github-cli.list > /dev/null \ + && apt-get update \ + && apt-get install gh -y + +# Download and install miniforge. +ENV PATH="/root/miniforge3/bin:${PATH}" +RUN wget -q \ + https://github.com/conda-forge/miniforge/releases/latest/download/Miniforge3-Linux-aarch64.sh \ + && bash Miniforge3-Linux-aarch64.sh -b \ + && rm -f Miniforge3-Linux-aarch64.sh +RUN mamba --version + +# Make /root traversable so the entrypoint can `source +# /root/miniforge3/bin/activate ...` when the container runs as a non-root +# user (apptainer/singularity maps the host UID into the container; the +# default ubuntu /root is 0700 which would block path traversal). +x only, +# not +r, so the directory listing remains private. +RUN chmod o+x /root + +# Setup mamba environment. +RUN mamba create -n streamlit-env python=3.10 +RUN echo "mamba activate streamlit-env" >> ~/.bashrc +SHELL ["/bin/bash", "--rcfile", "~/.bashrc"] +SHELL ["mamba", "run", "-n", "streamlit-env", "/bin/bash", "-c"] + +# Install up-to-date cmake via mamba and packages for pyOpenMS build. +RUN mamba install cmake +RUN pip install --upgrade pip && python -m pip install -U setuptools nose cython "autowrap<=0.24" pandas numpy pytest + +# Clone OpenMS branch and the associcated contrib+thirdparties+pyOpenMS-doc submodules. +RUN git clone --recursive --depth=1 -b ${OPENMS_BRANCH} --single-branch ${OPENMS_REPO} && cd /OpenMS + +# Pull Linux compatible third-party dependencies and store them in directory thirdparty. +WORKDIR /OpenMS +RUN mkdir /thirdparty && \ + git submodule update --init THIRDPARTY && \ + cp -r THIRDPARTY/All/* /thirdparty && \ + if [ -d "THIRDPARTY/Linux/aarch64" ]; then \ + cp -r THIRDPARTY/Linux/aarch64/* /thirdparty; \ + fi && \ + chmod -R +x /thirdparty +ENV PATH="/thirdparty/LuciPHOr2:/thirdparty/MSGFPlus:/thirdparty/ThermoRawFileParser:/thirdparty/Comet:/thirdparty/Percolator:/thirdparty/Sage:${PATH}" + +# Build OpenMS and pyOpenMS. +FROM setup-build-system AS compile-openms +WORKDIR / + +# Set up build directory. +RUN mkdir /openms-build +WORKDIR /openms-build + +# Configure. +RUN /bin/bash -c "cmake -DCMAKE_BUILD_TYPE='Release' -DCMAKE_PREFIX_PATH='/OpenMS/contrib-build/;/usr/;/usr/local' -DHAS_XSERVER=OFF -DBOOST_USE_STATIC=OFF -DPYOPENMS=ON ../OpenMS -DPY_MEMLEAK_DISABLE=On" + +# Build TOPP tools and clean up. +RUN make -j4 TOPP +RUN rm -rf src doc CMakeFiles + +# Build pyOpenMS wheels and install via pip. +RUN make -j4 pyopenms +WORKDIR /openms-build/pyOpenMS +RUN pip install dist/*.whl + +# Install other dependencies (excluding pyopenms) +COPY requirements.txt ./requirements.txt +RUN grep -Ev '^pyopenms([=<>!~].*)?$' requirements.txt > requirements_cleaned.txt && mv requirements_cleaned.txt requirements.txt +RUN pip install -r requirements.txt + +WORKDIR / +RUN mkdir openms + +# Copy TOPP tools bin directory, add to PATH. +RUN cp -r openms-build/bin /openms/bin +ENV PATH="/openms/bin/:${PATH}" + +# Copy TOPP tools bin directory, add to PATH. +RUN cp -r openms-build/lib /openms/lib +ENV LD_LIBRARY_PATH="/openms/lib/:${LD_LIBRARY_PATH}" + +# Copy share folder, add to PATH, remove source directory. +RUN cp -r OpenMS/share/OpenMS /openms/share +RUN rm -rf OpenMS +ENV OPENMS_DATA_PATH="/openms/share/" + +# Remove build directory. +RUN rm -rf openms-build + +# Prepare and run streamlit app. +FROM compile-openms AS run-app + +# Install Redis server for job queue and nginx for load balancing. +# Redis data lives under $RUNTIME_DIR at runtime (see entrypoint.sh) so no +# /var/lib/redis setup is needed - that path is not writable under Apptainer. +RUN apt-get update && apt-get install -y --no-install-recommends redis-server nginx \ + && rm -rf /var/lib/apt/lists/* + +# Create Redis data directory. Default 0755 root-owned is enough: the docker +# entrypoint runs as root (can write regardless of mode), and the apptainer +# entrypoint relocates Redis state to /tmp/openms-runtime-* so this dir is +# never written under apptainer. +RUN mkdir -p /var/lib/redis + +# Pre-create bind-mount targets so apptainer/singularity has a real attach +# point. Docker auto-creates missing `-v` targets, but singularity uses a +# read-only underlay and silently ignores `:rw` when the target isn't a +# real directory in the SIF — writes then fail with EROFS even though the +# host bind path is writable. Pre-creating these directories costs one +# inode each and changes nothing in docker mode (the user's volume mount +# shadows them). +RUN mkdir -p /workspaces-streamlit-template /mounted-data + +# Create workdir and copy over all streamlit related files/folders. + +# note: specifying folder with slash as suffix and repeating the folder name seems important to preserve directory structure +WORKDIR /app +COPY assets/ /app/assets +COPY content/ /app/content +COPY docs/ /app/docs +COPY example-data/ /app/example-data +COPY gdpr_consent/ /app/gdpr_consent +COPY hooks/ /app/hooks +COPY src/ /app/src +COPY utils/ /app/utils +COPY app.py /app/app.py +COPY settings.json /app/settings.json +COPY default-parameters.json /app/default-parameters.json +COPY presets.json /app/presets.json + +# For streamlit configuration +COPY .streamlit/ /app/.streamlit/ +COPY clean-up-workspaces.py /app/clean-up-workspaces.py + +# add cron job to the crontab +RUN echo "0 3 * * * /root/miniforge3/envs/streamlit-env/bin/python /app/clean-up-workspaces.py >> /app/clean-up-workspaces.log 2>&1" | crontab - + +# Set default worker count (can be overridden via environment variable) +ENV RQ_WORKER_COUNT=1 +ENV REDIS_URL=redis://localhost:6379/0 + +# Number of Streamlit server instances for load balancing (default: 1 = no load balancer) +# Set to >1 to enable nginx load balancer with multiple Streamlit instances +ENV STREAMLIT_SERVER_COUNT=1 + +# Install the apptainer-compatible entrypoint that starts cron (when the root +# FS is writable), Redis, RQ workers, optional nginx load balancer, and the +# Streamlit server. The script falls back to /tmp paths under apptainer. +COPY docker/entrypoint.sh /app/entrypoint.sh +RUN chmod +x /app/entrypoint.sh + +# Patch Analytics +RUN mamba run -n streamlit-env python hooks/hook-analytics.py + +# Set Online Deployment +RUN jq '.online_deployment = true' settings.json > tmp.json && mv tmp.json settings.json + +# Point the in-app mounted-drive browser at the conventional bind-mount path. +# The browser only renders when this directory exists at runtime, i.e. when +# the user starts the container with `-v /host/path:/mounted-data`. +RUN jq '.local_data_dir = "/mounted-data"' settings.json > tmp.json && mv tmp.json settings.json + +# Download latest OpenMS App executable as a ZIP file. +# ARG declared here (not at the top) — otherwise the per-run token busts the cache. +ARG GITHUB_TOKEN +RUN if [ -n "$GITHUB_TOKEN" ]; then \ + echo "GITHUB_TOKEN is set, proceeding to download the release asset..."; \ + gh release download -R ${GITHUB_USER}/${GITHUB_REPO} -p "OpenMS-App.zip" -D /app; \ + else \ + echo "GITHUB_TOKEN is not set, skipping the release asset download."; \ + fi + + +# Run app as container entrypoint. +EXPOSE $PORT +ENTRYPOINT ["/app/entrypoint.sh"] diff --git a/Dockerfile_simple.arm b/Dockerfile_simple.arm new file mode 100644 index 00000000..be57317d --- /dev/null +++ b/Dockerfile_simple.arm @@ -0,0 +1,127 @@ +# This Dockerfile creates a container with pyOpenMS +# It also adds a basic streamlit server that serves a pyOpenMS-based app. +# hints: +# build image with: docker build -f Dockerfile_simple.arm --no-cache -t streamlitapp:latest-arm64 --build-arg GITHUB_TOKEN= . 2>&1 | tee build.log +# check if image was build: docker image ls +# run container: docker run -p 8501:8501 streamlitapp:latest +# debug container after build (comment out ENTRYPOINT) and run container with interactive /bin/bash shell +# prune unused images/etc. to free disc space (e.g. might be needed on gitpod). Use with care.: docker system prune --all --force + +FROM ubuntu:22.04 AS stage1 +ARG OPENMS_REPO=https://github.com/OpenMS/OpenMS.git +ARG OPENMS_BRANCH=develop +ARG PORT=8501 +# Streamlit app GitHub user name (to download artifact from). +ARG GITHUB_USER=OpenMS +# Streamlit app GitHub repository name (to download artifact from). +ARG GITHUB_REPO=streamlit-template + + +# Step 1: set up a sane build system +USER root + +RUN apt-get -y update +# note: streamlit in docker needs libgtk2.0-dev (see https://yugdamor.medium.com/importerror-libgthread-2-0-so-0-cannot-open-shared-object-file-no-such-file-or-directory-895b94a7827b) +RUN apt-get install -y --no-install-recommends --no-install-suggests wget ca-certificates libgtk2.0-dev curl jq cron nginx +RUN update-ca-certificates + +# Install Github CLI +RUN (type -p wget >/dev/null || (apt-get update && apt-get install wget -y)) \ + && mkdir -p -m 755 /etc/apt/keyrings \ + && wget -qO- https://cli.github.com/packages/githubcli-archive-keyring.gpg | tee /etc/apt/keyrings/githubcli-archive-keyring.gpg > /dev/null \ + && chmod go+r /etc/apt/keyrings/githubcli-archive-keyring.gpg \ + && echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/githubcli-archive-keyring.gpg] https://cli.github.com/packages stable main" | tee /etc/apt/sources.list.d/github-cli.list > /dev/null \ + && apt-get update \ + && apt-get install gh -y + +# Download and install miniforge. +ENV PATH="/root/miniforge3/bin:${PATH}" +RUN wget -q \ + https://github.com/conda-forge/miniforge/releases/latest/download/Miniforge3-Linux-aarch64.sh \ + && bash Miniforge3-Linux-aarch64.sh -b \ + && rm -f Miniforge3-Linux-aarch64.sh +RUN mamba --version + +# Make /root traversable so the entrypoint can `source +# /root/miniforge3/bin/activate ...` when the container runs as a non-root +# user (apptainer/singularity maps the host UID into the container; the +# default ubuntu /root is 0700 which would block path traversal). +x only, +# not +r, so the directory listing remains private. +RUN chmod o+x /root + +# Setup mamba environment. +RUN mamba create -n streamlit-env python=3.10 +RUN echo "mamba activate streamlit-env" >> ~/.bashrc +SHELL ["/bin/bash", "--rcfile", "~/.bashrc"] +SHELL ["mamba", "run", "-n", "streamlit-env", "/bin/bash", "-c"] + +#################################### install streamlit +# install packages +COPY requirements.txt requirements.txt +RUN mamba install pip +RUN python -m pip install --upgrade pip +RUN python -m pip install -r requirements.txt + +# Pre-create bind-mount targets so apptainer/singularity has a real attach +# point. Docker auto-creates missing `-v` targets, but singularity uses a +# read-only underlay and silently ignores `:rw` when the target isn't a +# real directory in the SIF — writes then fail with EROFS even though the +# host bind path is writable. +RUN mkdir -p /workspaces-streamlit-template /mounted-data + +# create workdir and copy over all streamlit related files/folders +WORKDIR /app +# note: specifying folder with slash as suffix and repeating the folder name seems important to preserve directory structure +WORKDIR /app +COPY assets/ /app/assets +COPY content/ /app/content +COPY docs/ /app/docs +COPY example-data/ /app/example-data +COPY gdpr_consent/ /app/gdpr_consent +COPY hooks/ /app/hooks +COPY src/ /app/src +COPY utils/ /app/utils +COPY app.py /app/app.py +COPY settings.json /app/settings.json +COPY default-parameters.json /app/default-parameters.json +COPY presets.json /app/presets.json + +# For streamlit configuration +COPY .streamlit/ /app/.streamlit/ + +COPY clean-up-workspaces.py /app/clean-up-workspaces.py + +# add cron job to the crontab +RUN echo "0 3 * * * /root/miniforge3/envs/streamlit-env/bin/python /app/clean-up-workspaces.py >> /app/clean-up-workspaces.log 2>&1" | crontab - + +# Number of Streamlit server instances for load balancing (default: 1 = no load balancer) +# Set to >1 to enable nginx load balancer with multiple Streamlit instances +ENV STREAMLIT_SERVER_COUNT=1 + +# Install the apptainer-compatible entrypoint (shared with the full image). +# The script auto-skips the Redis/RQ section when redis-server is not +# installed, so it works equally well in the simple variant. +COPY docker/entrypoint.sh /app/entrypoint.sh +RUN chmod +x /app/entrypoint.sh + +# Patch Analytics +RUN mamba run -n streamlit-env python hooks/hook-analytics.py + +# Set Online Deployment +RUN jq '.online_deployment = true' settings.json > tmp.json && mv tmp.json settings.json + +# Download latest OpenMS App executable as a ZIP file. +# ARG declared here (not at the top) — otherwise the per-run token busts the cache. +ARG GITHUB_TOKEN +RUN if [ -n "$GITHUB_TOKEN" ]; then \ + echo "GITHUB_TOKEN is set, proceeding to download the release asset..."; \ + gh release download -R ${GITHUB_USER}/${GITHUB_REPO} -p "OpenMS-App.zip" -D /app; \ + else \ + echo "GITHUB_TOKEN is not set, skipping the release asset download."; \ + fi + +# make sure that mamba environment is used +SHELL ["mamba", "run", "-n", "streamlit-env", "/bin/bash", "-c"] + +EXPOSE $PORT +ENTRYPOINT ["/app/entrypoint.sh"] From d32c1b9cbc97c4dbf5e2111273416bb6b0e6b6d1 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 25 May 2026 19:43:35 +0000 Subject: [PATCH 03/29] ci(docker): extend apptainer/nginx/traefik tests to cover arm64 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Previously the apptainer/nginx/traefik integration tests only ran against the amd64 artifact, so the arm64 image was validated solely by its build succeeding plus a post-push /_stcore/health probe. Now all three integration matrices fan out over arch=[amd64, arm64] with a matrix-driven runs-on, exercising the read-only-root apptainer contract and both kind-based ingress paths on a native ARM runner too. Changes: - `build-amd64` artifact renamed from `openms-streamlit--image` to `openms-streamlit--amd64-image` for symmetry. - `build-arm64` now also `load: true`'s the built image, retags to the kind-friendly `openms-streamlit:test`, saves it as a tar, and uploads it as `openms-streamlit--arm64-image`. The post-push pull-back smoke test is removed — the new apptainer/ nginx/traefik runs subsume it and avoid the slow GHCR pull. - `test-apptainer`, `test-nginx`, `test-traefik` matrices switched from `variant: [full, simple]` to an `include:` list with {variant, arch, runner} tuples; `runs-on: ${{ matrix.runner }}` selects `ubuntu-latest` for amd64 and `ubuntu-24.04-arm` for arm64. Artifact download names get `${{ matrix.arch }}` interpolated. - SIF upload at the tail of `test-apptainer` gated on `matrix.arch == 'amd64'`: arm64 still runs the full apptainer contract end-to-end, but only amd64 produces the SIF that `publish-apptainer` ships to GHCR (HPC SIF consumers are amd64). Note on `publish-apptainer`: it stays on `needs: test-apptainer`, which now waits for the arm64 matrix entries too — meaning an arm64 apptainer regression will block amd64 SIF publishing. Conservative on purpose; happy to decouple via separate jobs if that turns out to be too strict in practice. --- .github/workflows/build-and-test.yml | 115 +++++++++++++++++---------- 1 file changed, 74 insertions(+), 41 deletions(-) diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml index 5ca863a2..012291fd 100644 --- a/.github/workflows/build-and-test.yml +++ b/.github/workflows/build-and-test.yml @@ -111,7 +111,7 @@ jobs: - name: Upload image artifact uses: actions/upload-artifact@v4 with: - name: openms-streamlit-${{ matrix.variant }}-image + name: openms-streamlit-${{ matrix.variant }}-amd64-image path: /tmp/image.tar retention-days: 1 @@ -121,9 +121,9 @@ jobs: # under `-` by the `create-manifest` job below. The build # uses a separate `Dockerfile.arm` / `Dockerfile_simple.arm` that swaps # the miniforge installer to aarch64 and (for the full variant) guards - # the THIRDPARTY/Linux/aarch64 copy. Apptainer/nginx/traefik integration - # tests still run only on the amd64 artifact — those gates do not need - # arch duplication right now (HPC consumers of the SIF are amd64). + # the THIRDPARTY/Linux/aarch64 copy. The built image is also uploaded as + # an artifact so the apptainer / nginx / traefik integration jobs can + # exercise the ARM image on a native ARM runner (matrix arch=arm64). needs: lint-manifests runs-on: ubuntu-24.04-arm permissions: @@ -180,6 +180,7 @@ jobs: context: . file: ${{ matrix.dockerfile }} platforms: linux/arm64 + load: true push: ${{ github.event_name != 'pull_request' }} tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} @@ -189,30 +190,22 @@ jobs: build-args: | GITHUB_TOKEN=${{ secrets.GITHUB_TOKEN }} - - name: Smoke test the just-pushed arm64 image - # PRs build (validates Dockerfile.arm parses + compiles) but don't - # push, so there's nothing to pull back on PR events. On push/tag, - # pull the just-published image and verify /_stcore/health to catch - # entrypoint regressions that wouldn't surface in the build itself. - if: github.event_name != 'pull_request' + - name: Retag for kind (stable local tag) run: | - set -euo pipefail - IMAGE_REF="${{ env.REGISTRY }}/${{ env.IMAGE_NAME_LC }}:${{ github.sha }}-${{ matrix.variant }}-arm64" - echo "Smoke-testing $IMAGE_REF" - docker pull "$IMAGE_REF" - docker run -d --rm --name smoketest -p 8501:8501 "$IMAGE_REF" - for i in $(seq 1 90); do - if curl -fsSo /dev/null --max-time 2 http://127.0.0.1:8501/_stcore/health; then - echo "Streamlit healthy after ${i} attempts" - docker stop smoketest - exit 0 - fi - sleep 2 - done - echo "ERROR: /_stcore/health never returned 200" - docker logs smoketest || true - docker stop smoketest || true - exit 1 + # load:true above loaded all meta-action tags into local docker. + # Retag the first one to the stable name the kustomize overlay expects. + FIRST_TAG=$(printf '%s\n' "${{ steps.meta.outputs.tags }}" | head -n 1) + docker tag "$FIRST_TAG" openms-streamlit:test + + - name: Save image as tar + run: docker save openms-streamlit:test -o /tmp/image.tar + + - name: Upload image artifact + uses: actions/upload-artifact@v4 + with: + name: openms-streamlit-${{ matrix.variant }}-arm64-image + path: /tmp/image.tar + retention-days: 1 create-manifest: # Stitch the per-arch tags into multi-arch manifest lists. The manifest @@ -282,19 +275,31 @@ jobs: # (not root inside the image). The entrypoint must tolerate both: this job # exercises that contract by running the built image under apptainer and # waiting for the streamlit /_stcore/health endpoint to come up. - needs: build-amd64 - runs-on: ubuntu-latest + needs: [build-amd64, build-arm64] + runs-on: ${{ matrix.runner }} strategy: fail-fast: false matrix: - variant: [full, simple] + include: + - variant: full + arch: amd64 + runner: ubuntu-latest + - variant: full + arch: arm64 + runner: ubuntu-24.04-arm + - variant: simple + arch: amd64 + runner: ubuntu-latest + - variant: simple + arch: arm64 + runner: ubuntu-24.04-arm steps: - uses: actions/checkout@v4 - name: Download image artifact uses: actions/download-artifact@v4 with: - name: openms-streamlit-${{ matrix.variant }}-image + name: openms-streamlit-${{ matrix.variant }}-${{ matrix.arch }}-image path: /tmp - name: Install apptainer @@ -424,8 +429,12 @@ jobs: if: always() run: apptainer instance stop openms-test || true - - name: Upload validated SIF artifact (push events only) - if: success() && github.event_name != 'pull_request' + - name: Upload validated SIF artifact (amd64 push events only) + # SIF publishing stays amd64-only this iteration (HPC consumers of + # the SIF are amd64). The arm64 matrix entry still exercises the + # full apptainer contract end-to-end; it just doesn't upload the + # resulting SIF for downstream publishing. + if: success() && github.event_name != 'pull_request' && matrix.arch == 'amd64' uses: actions/upload-artifact@v4 with: name: openms-streamlit-${{ matrix.variant }}-sif @@ -500,19 +509,31 @@ jobs: done <<< "${{ steps.meta.outputs.tags }}" test-nginx: - needs: build-amd64 - runs-on: ubuntu-latest + needs: [build-amd64, build-arm64] + runs-on: ${{ matrix.runner }} strategy: fail-fast: false matrix: - variant: [full, simple] + include: + - variant: full + arch: amd64 + runner: ubuntu-latest + - variant: full + arch: arm64 + runner: ubuntu-24.04-arm + - variant: simple + arch: amd64 + runner: ubuntu-latest + - variant: simple + arch: arm64 + runner: ubuntu-24.04-arm steps: - uses: actions/checkout@v4 - name: Download image artifact uses: actions/download-artifact@v4 with: - name: openms-streamlit-${{ matrix.variant }}-image + name: openms-streamlit-${{ matrix.variant }}-${{ matrix.arch }}-image path: /tmp - name: Load image into local docker @@ -587,19 +608,31 @@ jobs: done test-traefik: - needs: build-amd64 - runs-on: ubuntu-latest + needs: [build-amd64, build-arm64] + runs-on: ${{ matrix.runner }} strategy: fail-fast: false matrix: - variant: [full, simple] + include: + - variant: full + arch: amd64 + runner: ubuntu-latest + - variant: full + arch: arm64 + runner: ubuntu-24.04-arm + - variant: simple + arch: amd64 + runner: ubuntu-latest + - variant: simple + arch: arm64 + runner: ubuntu-24.04-arm steps: - uses: actions/checkout@v4 - name: Download image artifact uses: actions/download-artifact@v4 with: - name: openms-streamlit-${{ matrix.variant }}-image + name: openms-streamlit-${{ matrix.variant }}-${{ matrix.arch }}-image path: /tmp - name: Load image into local docker From 1d73b6726e8286c27cc3771bfaf63fb691352458 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 25 May 2026 20:07:21 +0000 Subject: [PATCH 04/29] fix(arm): use two-pass cmake so TOPP links against system libstdc++ MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The ARM build of `make -j4 TOPP` failed at the link step with /usr/bin/ld: /root/miniforge3/lib/libyaml-cpp.so.0.8: undefined reference to `std::ios_base_library_init()@GLIBCXX_3.4.32' The conda-forge libyaml-cpp wheel for aarch64 is built against GLIBCXX_3.4.32 (gcc 13+), but Ubuntu 22.04's system g++ ships with an older libstdc++. Running cmake inside the mamba shell lets it discover /root/miniforge3/lib first, so the conda-forge yaml-cpp gets linked into every TOPP binary and breaks. amd64 happens to work because the conda-forge amd64 yaml-cpp build is older. Fix mirrors FLASHApp's Dockerfile.arm: configure OpenMS in two cmake passes — pass 1 under plain `/bin/bash` with `-DCMAKE_IGNORE_PREFIX_PATH=/root/miniforge3` so cmake resolves C++ deps from the system tree (libyaml-cpp from contrib, boost from apt, etc.); pass 2 under `mamba run` with `-DPYOPENMS=ON` so the Python bindings still find conda-forge Python / Cython / NumPy. The IGNORE_PREFIX_PATH flag is repeated on pass 2 to keep the cached C++ link command unchanged. Only Dockerfile.arm changes; Dockerfile (amd64) keeps its single-pass cmake to avoid disturbing the working x86 path. --- Dockerfile.arm | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/Dockerfile.arm b/Dockerfile.arm index 8571bc57..2e9a6aba 100644 --- a/Dockerfile.arm +++ b/Dockerfile.arm @@ -86,8 +86,21 @@ WORKDIR / RUN mkdir /openms-build WORKDIR /openms-build -# Configure. -RUN /bin/bash -c "cmake -DCMAKE_BUILD_TYPE='Release' -DCMAKE_PREFIX_PATH='/OpenMS/contrib-build/;/usr/;/usr/local' -DHAS_XSERVER=OFF -DBOOST_USE_STATIC=OFF -DPYOPENMS=ON ../OpenMS -DPY_MEMLEAK_DISABLE=On" +# Configure (two-pass — mirrors FLASHApp.arm). +# Pass 1 runs under plain bash so cmake does NOT search /root/miniforge3 +# when resolving C++ system dependencies. On ARM the conda-forge build of +# libyaml-cpp.so.0.8 is linked against a newer libstdc++ (GLIBCXX_3.4.32, +# i.e. gcc 13+) than ubuntu:22.04's system g++ ships, so letting cmake +# pick the miniforge yaml-cpp makes TOPP linking fail with +# undefined reference to `std::ios_base_library_init()@GLIBCXX_3.4.32` +# amd64 happens to work because its conda-forge yaml-cpp build is older. +# Pass 2 re-runs cmake inside the mamba env with PYOPENMS=ON so the Python +# bindings can find the conda-forge Python/Cython/NumPy; CMAKE_IGNORE_PREFIX_PATH +# keeps the C++ link command unchanged from pass 1. +SHELL ["/bin/bash", "-c"] +RUN cmake -DCMAKE_BUILD_TYPE='Release' -DCMAKE_PREFIX_PATH='/OpenMS/contrib-build/;/usr/;/usr/local' -DCMAKE_IGNORE_PREFIX_PATH=/root/miniforge3 -DHAS_XSERVER=OFF -DBOOST_USE_STATIC=OFF ../OpenMS +SHELL ["mamba", "run", "-n", "streamlit-env", "/bin/bash", "-c"] +RUN cmake -DPYOPENMS=ON -DPY_MEMLEAK_DISABLE=On -DCMAKE_IGNORE_PREFIX_PATH=/root/miniforge3 . # Build TOPP tools and clean up. RUN make -j4 TOPP From f11bc99fd559bc79ed54e3849a2a7f0be4487f0c Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 25 May 2026 20:13:37 +0000 Subject: [PATCH 05/29] fix(arm): install cmake via apt so pass-1 cmake is on plain bash PATH MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The two-pass cmake split from 1d73b67 runs pass 1 under `SHELL ["/bin/bash", "-c"]`, but the only cmake on the image is the one from `mamba install cmake` at /root/miniforge3/envs/streamlit-env/bin/cmake — not on plain bash's PATH. Result: exit 127 (command not found) the moment pass 1 invokes cmake. FLASHApp.arm sidesteps this by installing cmake via apt; do the same here (just append `cmake` to the existing apt-get install line). The mamba cmake install stays, so pass 2 under the mamba shell continues to use the conda-forge cmake exactly as it did before. Ubuntu 22.04 ships cmake 3.22, comfortably above OpenMS 3.5's 3.15 floor. --- Dockerfile.arm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile.arm b/Dockerfile.arm index 2e9a6aba..e0bbd0f9 100644 --- a/Dockerfile.arm +++ b/Dockerfile.arm @@ -20,7 +20,7 @@ USER root # Install required Ubuntu packages. RUN apt-get -y update -RUN apt-get install -y --no-install-recommends --no-install-suggests g++ autoconf automake patch libtool make git gpg wget ca-certificates curl jq libgtk2.0-dev openjdk-8-jdk cron +RUN apt-get install -y --no-install-recommends --no-install-suggests g++ autoconf automake patch libtool make git gpg wget ca-certificates curl jq libgtk2.0-dev openjdk-8-jdk cron cmake RUN update-ca-certificates RUN apt-get install -y --no-install-recommends --no-install-suggests libsvm-dev libeigen3-dev coinor-libcbc-dev libglpk-dev libzip-dev zlib1g-dev libxerces-c-dev libbz2-dev libomp-dev libhdf5-dev RUN apt-get install -y --no-install-recommends --no-install-suggests libboost-date-time1.74-dev \ From 5185c3e0ddd8ade74fd6264ea26d0a07aa077508 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 25 May 2026 20:24:52 +0000 Subject: [PATCH 06/29] fix(arm): call mamba cmake by full path in pass 1 (apt 3.22 is too old) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The previous fix (install cmake via apt) didn't actually help: OpenMS 3.5's CMakeLists.txt requires cmake >= 3.24, and Ubuntu 22.04's apt cmake is 3.22.1, which fails configure with CMake Error at src/openms/extern/CMakeLists.txt:11 (cmake_minimum_required): CMake 3.24 or higher is required. You are running version 3.22.1 That's exactly why the existing x86 Dockerfile installs cmake via mamba (the conda-forge build is 3.30+). FLASHApp.arm escapes this by using ubuntu:24.04 (apt cmake 3.28); we stay on 22.04 to minimize churn vs. the working x86 Dockerfile. Fix: in pass 1, call the mamba-env cmake by its full path `/root/miniforge3/envs/streamlit-env/bin/cmake`. The plain-bash SHELL is still in effect, so cmake doesn't pick up any conda-forge environment side effects, and CMAKE_IGNORE_PREFIX_PATH keeps it from auto-discovering miniforge libraries during find_package. The cmake binary itself runs against miniforge's libstdc++, but that's a runtime detail of cmake — it doesn't leak into the configured project's link command. The apt cmake addition from f11bc99 is now redundant but harmless; leaving it in place to keep this diff focused. --- Dockerfile.arm | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/Dockerfile.arm b/Dockerfile.arm index e0bbd0f9..53d99d93 100644 --- a/Dockerfile.arm +++ b/Dockerfile.arm @@ -94,11 +94,15 @@ WORKDIR /openms-build # pick the miniforge yaml-cpp makes TOPP linking fail with # undefined reference to `std::ios_base_library_init()@GLIBCXX_3.4.32` # amd64 happens to work because its conda-forge yaml-cpp build is older. +# We call the mamba-env cmake by full path so we get a version >= 3.24 +# (OpenMS 3.5's floor); ubuntu:22.04's apt cmake is 3.22 which is too old. +# CMAKE_IGNORE_PREFIX_PATH keeps cmake from auto-discovering miniforge libs +# even though the binary itself lives there. # Pass 2 re-runs cmake inside the mamba env with PYOPENMS=ON so the Python # bindings can find the conda-forge Python/Cython/NumPy; CMAKE_IGNORE_PREFIX_PATH # keeps the C++ link command unchanged from pass 1. SHELL ["/bin/bash", "-c"] -RUN cmake -DCMAKE_BUILD_TYPE='Release' -DCMAKE_PREFIX_PATH='/OpenMS/contrib-build/;/usr/;/usr/local' -DCMAKE_IGNORE_PREFIX_PATH=/root/miniforge3 -DHAS_XSERVER=OFF -DBOOST_USE_STATIC=OFF ../OpenMS +RUN /root/miniforge3/envs/streamlit-env/bin/cmake -DCMAKE_BUILD_TYPE='Release' -DCMAKE_PREFIX_PATH='/OpenMS/contrib-build/;/usr/;/usr/local' -DCMAKE_IGNORE_PREFIX_PATH=/root/miniforge3 -DHAS_XSERVER=OFF -DBOOST_USE_STATIC=OFF ../OpenMS SHELL ["mamba", "run", "-n", "streamlit-env", "/bin/bash", "-c"] RUN cmake -DPYOPENMS=ON -DPY_MEMLEAK_DISABLE=On -DCMAKE_IGNORE_PREFIX_PATH=/root/miniforge3 . From 0bab3ae16d64dbc5bfa5b7248c77f711b2348263 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 25 May 2026 20:31:53 +0000 Subject: [PATCH 07/29] ci: free disk space at the start of each integration test job MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two failures in the previous run (test-traefik full, test-nginx simple) ended with the runner reporting "No space left on device" while flushing its diagnostic log. ubuntu-latest starts with ~14 GB free; downloading the full image artifact (5-8 GB), loading it into docker (decompressed, larger), pulling kind's node image, then loading the OCI tar into the kind cluster easily exceeds that budget. Mirror the cleanup already used by `build-arm64`: drop the runner's preinstalled dotnet / android SDK / ghc / hostedtoolcache to recover ~30 GB. Same step now runs at the top of test-apptainer, test-nginx, and test-traefik on both amd64 (ubuntu-latest) and arm64 (ubuntu-24.04-arm) matrix entries — the arm runner is at least as tight as amd64. --- .github/workflows/build-and-test.yml | 30 ++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml index 012291fd..ae256dcd 100644 --- a/.github/workflows/build-and-test.yml +++ b/.github/workflows/build-and-test.yml @@ -296,6 +296,16 @@ jobs: steps: - uses: actions/checkout@v4 + - name: Free disk space + # ubuntu-latest has ~14 GB free; the full image (5-8 GB) plus kind + # node image plus loading the OCI tar into both docker and kind can + # exhaust it. The arm runner is even tighter. Same incantation as + # `build-arm64`'s "Free disk space" step. + run: | + sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc /opt/hostedtoolcache || true + sudo apt-get clean + df -h + - name: Download image artifact uses: actions/download-artifact@v4 with: @@ -530,6 +540,16 @@ jobs: steps: - uses: actions/checkout@v4 + - name: Free disk space + # ubuntu-latest has ~14 GB free; the full image (5-8 GB) plus kind + # node image plus loading the OCI tar into both docker and kind can + # exhaust it. The arm runner is even tighter. Same incantation as + # `build-arm64`'s "Free disk space" step. + run: | + sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc /opt/hostedtoolcache || true + sudo apt-get clean + df -h + - name: Download image artifact uses: actions/download-artifact@v4 with: @@ -629,6 +649,16 @@ jobs: steps: - uses: actions/checkout@v4 + - name: Free disk space + # ubuntu-latest has ~14 GB free; the full image (5-8 GB) plus kind + # node image plus loading the OCI tar into both docker and kind can + # exhaust it. The arm runner is even tighter. Same incantation as + # `build-arm64`'s "Free disk space" step. + run: | + sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc /opt/hostedtoolcache || true + sudo apt-get clean + df -h + - name: Download image artifact uses: actions/download-artifact@v4 with: From 4790d46f8a835788ea5104fa1a49e61187020455 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 25 May 2026 20:59:34 +0000 Subject: [PATCH 08/29] fix(arm): keep CMakeFiles/ between make TOPP and make pyopenms After the two-pass cmake configure landed in 5185c3e, the next attempt got past `make -j4 TOPP` (the link error is fixed) but failed fast in `make -j4 pyopenms` with: CMake Error: Not a file: /openms-build/CMakeFiles/VerifyGlobs.cmake CMake Error: Error processing file: /openms-build/CMakeFiles/VerifyGlobs.cmake make: *** [Makefile:11553: cmake_check_build_system] Error 1 `VerifyGlobs.cmake` is generated by cmake for `file(GLOB CONFIGURE_DEPENDS ...)` targets and is consulted by `cmake_check_build_system` at the top of every subsequent `make` invocation. The intermediate cleanup line RUN rm -rf src doc CMakeFiles deleted it, which is fine on the x86 single-pass build (different cmake codepath when PYOPENMS=ON is set in the initial configure, no VerifyGlobs.cmake generated) but breaks the ARM two-pass build. Stop deleting CMakeFiles/ between `make TOPP` and `make pyopenms`. We still drop `src/` and `doc/` for disk savings; keeping CMakeFiles costs only a few hundred MB on the intermediate layer. --- Dockerfile.arm | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/Dockerfile.arm b/Dockerfile.arm index 53d99d93..17659800 100644 --- a/Dockerfile.arm +++ b/Dockerfile.arm @@ -108,7 +108,16 @@ RUN cmake -DPYOPENMS=ON -DPY_MEMLEAK_DISABLE=On -DCMAKE_IGNORE_PREFIX_PATH=/root # Build TOPP tools and clean up. RUN make -j4 TOPP -RUN rm -rf src doc CMakeFiles +# NOTE: do NOT delete CMakeFiles/ here. The two-pass cmake configure used +# above generates CMakeFiles/VerifyGlobs.cmake for the pyOpenMS targets' +# CONFIGURE_DEPENDS globs; the next `make -j4 pyopenms` runs +# `cmake_check_build_system` which fails fast if VerifyGlobs.cmake is gone: +# CMake Error: Not a file: /openms-build/CMakeFiles/VerifyGlobs.cmake +# The x86 single-pass build seems to avoid generating that file (different +# cmake codepath when PYOPENMS is set during the initial configure), which +# is why it can still `rm -rf CMakeFiles` here. CMakeFiles/ adds ~a few +# hundred MB to the intermediate layer — acceptable. +RUN rm -rf src doc # Build pyOpenMS wheels and install via pip. RUN make -j4 pyopenms From c7fdf00070e0d3613c3f12c69cf130f84b299a69 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 25 May 2026 22:12:35 +0000 Subject: [PATCH 09/29] =?UTF-8?q?ci(apptainer):=20drop=20arm64=20=E2=80=94?= =?UTF-8?q?=20no=20upstream=20aarch64=20.deb?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit eWaterCycle/setup-apptainer@v2 installs apptainer from the upstream .deb asset on the GitHub release. Upstream apptainer only publishes amd64 .debs (verified: every v1.3.x release lists only `apptainer__amd64.deb`, no _arm64 / _aarch64 variant). On the ubuntu-24.04-arm runner the action's `apt-get install ./apptainer_*.deb` fails with sudo exit code 100 because the package can't be resolved. Building apptainer from source on the ARM runner would add ~15 minutes and a maintenance surface (Go toolchain, suid configuration) for limited value — HPC SIF consumers remain amd64. Revert test-apptainer to amd64-only and document why. test-nginx and test-traefik still exercise the ARM image via kind, which gives us functional ARM coverage at the docker-runtime level even without apptainer. Side cleanups now that arm64 is gone from this matrix: - artifact name back to a literal `*-amd64-image` (no matrix.arch) - SIF upload gate drops the `matrix.arch == 'amd64'` check --- .github/workflows/build-and-test.yml | 37 +++++++++++----------------- 1 file changed, 15 insertions(+), 22 deletions(-) diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml index ae256dcd..d9155839 100644 --- a/.github/workflows/build-and-test.yml +++ b/.github/workflows/build-and-test.yml @@ -275,24 +275,21 @@ jobs: # (not root inside the image). The entrypoint must tolerate both: this job # exercises that contract by running the built image under apptainer and # waiting for the streamlit /_stcore/health endpoint to come up. - needs: [build-amd64, build-arm64] - runs-on: ${{ matrix.runner }} + # + # amd64 only: upstream apptainer does NOT publish arm64 .deb assets + # (https://github.com/apptainer/apptainer/releases — every release lists + # only `apptainer__amd64.deb`), so eWaterCycle/setup-apptainer fails + # on ubuntu-24.04-arm with "sudo exit code 100" when its + # `apt-get install ./apptainer_*.deb` resolves a non-existent package. + # Building apptainer from source on the arm runner would add ~15 min and + # significant maintenance surface for limited value (HPC SIF consumers + # remain amd64). Re-evaluate if upstream starts publishing arm64 builds. + needs: build-amd64 + runs-on: ubuntu-latest strategy: fail-fast: false matrix: - include: - - variant: full - arch: amd64 - runner: ubuntu-latest - - variant: full - arch: arm64 - runner: ubuntu-24.04-arm - - variant: simple - arch: amd64 - runner: ubuntu-latest - - variant: simple - arch: arm64 - runner: ubuntu-24.04-arm + variant: [full, simple] steps: - uses: actions/checkout@v4 @@ -309,7 +306,7 @@ jobs: - name: Download image artifact uses: actions/download-artifact@v4 with: - name: openms-streamlit-${{ matrix.variant }}-${{ matrix.arch }}-image + name: openms-streamlit-${{ matrix.variant }}-amd64-image path: /tmp - name: Install apptainer @@ -439,12 +436,8 @@ jobs: if: always() run: apptainer instance stop openms-test || true - - name: Upload validated SIF artifact (amd64 push events only) - # SIF publishing stays amd64-only this iteration (HPC consumers of - # the SIF are amd64). The arm64 matrix entry still exercises the - # full apptainer contract end-to-end; it just doesn't upload the - # resulting SIF for downstream publishing. - if: success() && github.event_name != 'pull_request' && matrix.arch == 'amd64' + - name: Upload validated SIF artifact (push events only) + if: success() && github.event_name != 'pull_request' uses: actions/upload-artifact@v4 with: name: openms-streamlit-${{ matrix.variant }}-sif From f466229e60e41209f50a5d7841e8ab721f08893c Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 26 May 2026 07:03:16 +0000 Subject: [PATCH 10/29] fix(ci): keep /opt/hostedtoolcache when freeing disk space kind/kubectl/helm setup actions fail with "Cache directory '/opt/hostedtoolcache' does not exist". Drop just dotnet/android/ghc (~34 GB) and leave the tool cache in place. --- .github/workflows/build-and-test.yml | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml index d9155839..b2c3b317 100644 --- a/.github/workflows/build-and-test.yml +++ b/.github/workflows/build-and-test.yml @@ -143,7 +143,9 @@ jobs: # image is tighter than the AMD one out of the box. Mirrors what # FLASHApp's publish-docker-images.yml does at the top of its ARM job. run: | - sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc /opt/hostedtoolcache || true + # Keep /opt/hostedtoolcache: helm/kind-action and setup-kubectl + # cache binaries there and fail if the directory is missing. + sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc || true sudo apt-get clean df -h @@ -299,7 +301,9 @@ jobs: # exhaust it. The arm runner is even tighter. Same incantation as # `build-arm64`'s "Free disk space" step. run: | - sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc /opt/hostedtoolcache || true + # Keep /opt/hostedtoolcache: helm/kind-action and setup-kubectl + # cache binaries there and fail if the directory is missing. + sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc || true sudo apt-get clean df -h @@ -539,7 +543,9 @@ jobs: # exhaust it. The arm runner is even tighter. Same incantation as # `build-arm64`'s "Free disk space" step. run: | - sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc /opt/hostedtoolcache || true + # Keep /opt/hostedtoolcache: helm/kind-action and setup-kubectl + # cache binaries there and fail if the directory is missing. + sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc || true sudo apt-get clean df -h @@ -648,7 +654,9 @@ jobs: # exhaust it. The arm runner is even tighter. Same incantation as # `build-arm64`'s "Free disk space" step. run: | - sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc /opt/hostedtoolcache || true + # Keep /opt/hostedtoolcache: helm/kind-action and setup-kubectl + # cache binaries there and fail if the directory is missing. + sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc || true sudo apt-get clean df -h From f0d1db16c36d71c987f1e2f11160402f63e43803 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 26 May 2026 08:20:45 +0000 Subject: [PATCH 11/29] ci: dump cluster state on test-nginx/test-traefik failure curl exit-22 doesn't tell us whether the pod, service, or ingress is the broken link. Dump pods/logs/ingress/controller logs on failure so the next run surfaces the actual cause. --- .github/workflows/build-and-test.yml | 41 ++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml index b2c3b317..8fb54503 100644 --- a/.github/workflows/build-and-test.yml +++ b/.github/workflows/build-and-test.yml @@ -626,6 +626,27 @@ jobs: echo "$host -> 200 OK" done + - name: Dump cluster state on failure + if: failure() + run: | + echo "=== nodes ===" + kubectl get nodes -o wide || true + echo "=== pods (all namespaces) ===" + kubectl get pods -A -o wide || true + echo "=== app pods describe ===" + kubectl describe pod -n openms -l app=${SLUG} || true + echo "=== app pod logs ===" + kubectl logs -n openms -l app=${SLUG} --tail=200 --all-containers --prefix || true + echo "=== app pod previous logs (if crashed) ===" + kubectl logs -n openms -l app=${SLUG} --tail=200 --all-containers --prefix --previous || true + echo "=== ingress ===" + kubectl get ingress -A -o wide || true + kubectl describe ingress -n openms || true + echo "=== services + endpoints ===" + kubectl get svc,endpoints -n openms || true + echo "=== ingress-nginx controller logs ===" + kubectl logs -n ingress-nginx -l app.kubernetes.io/component=controller --tail=200 || true + test-traefik: needs: [build-amd64, build-arm64] runs-on: ${{ matrix.runner }} @@ -741,3 +762,23 @@ jobs: echo "" echo "$host -> 200 OK" done + + - name: Dump cluster state on failure + if: failure() + run: | + echo "=== nodes ===" + kubectl get nodes -o wide || true + echo "=== pods (all namespaces) ===" + kubectl get pods -A -o wide || true + echo "=== app pods describe ===" + kubectl describe pod -n openms -l app=${SLUG} || true + echo "=== app pod logs ===" + kubectl logs -n openms -l app=${SLUG} --tail=200 --all-containers --prefix || true + echo "=== app pod previous logs (if crashed) ===" + kubectl logs -n openms -l app=${SLUG} --tail=200 --all-containers --prefix --previous || true + echo "=== traefik ingressroute ===" + kubectl get ingressroute -A -o yaml || true + echo "=== services + endpoints ===" + kubectl get svc,endpoints -n openms || true + echo "=== traefik controller logs ===" + kubectl logs -n traefik -l app.kubernetes.io/name=traefik --tail=200 || true From 7aadb0656d89e0611bd80fa287f5c3dae681829b Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 26 May 2026 11:48:30 +0000 Subject: [PATCH 12/29] fix(ci): load images into kind via image-archive, drop docker load \`docker load\` + \`kind load docker-image\` keeps the image in both host docker AND each kind node's containerd. With a 5-8 GB image and two kind nodes that's ~25 GB of duplicated storage, which trips the "no space left on device" error in kind's ctr import. Switch to \`kind load image-archive\` so the tar streams directly into each node, and rm the tar after to reclaim /tmp. --- .github/workflows/build-and-test.yml | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml index 8fb54503..7890072c 100644 --- a/.github/workflows/build-and-test.yml +++ b/.github/workflows/build-and-test.yml @@ -555,9 +555,6 @@ jobs: name: openms-streamlit-${{ matrix.variant }}-${{ matrix.arch }}-image path: /tmp - - name: Load image into local docker - run: docker load -i /tmp/image.tar - - name: Create kind cluster uses: helm/kind-action@v1 with: @@ -565,7 +562,13 @@ jobs: config: .github/kind-config.yaml - name: Load image into kind cluster - run: kind load docker-image openms-streamlit:test --name test-cluster + # Use `kind load image-archive` (not docker-image) so we never store + # the image in host docker. Saves ~5-8 GB on /var/lib/docker. Delete + # the tar afterwards to free the same again on /tmp — the image is + # now in both kind nodes' containerd, which is enough. + run: | + kind load image-archive /tmp/image.tar --name test-cluster + rm -f /tmp/image.tar - name: Install nginx ingress controller run: | @@ -687,9 +690,6 @@ jobs: name: openms-streamlit-${{ matrix.variant }}-${{ matrix.arch }}-image path: /tmp - - name: Load image into local docker - run: docker load -i /tmp/image.tar - - name: Create kind cluster uses: helm/kind-action@v1 with: @@ -697,7 +697,13 @@ jobs: config: .github/kind-config.yaml - name: Load image into kind cluster - run: kind load docker-image openms-streamlit:test --name traefik-test + # Use `kind load image-archive` (not docker-image) so we never store + # the image in host docker. Saves ~5-8 GB on /var/lib/docker. Delete + # the tar afterwards to free the same again on /tmp — the image is + # now in both kind nodes' containerd, which is enough. + run: | + kind load image-archive /tmp/image.tar --name traefik-test + rm -f /tmp/image.tar - name: Set up Helm uses: azure/setup-helm@v4 From 611902101e1eba9e41080b055ec668711e009f24 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 26 May 2026 12:04:23 +0000 Subject: [PATCH 13/29] ci: re-trigger workflow run From 88dcdb03a40dccf0e1c6b23ddc4cc994118b6fc2 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 26 May 2026 13:18:35 +0000 Subject: [PATCH 14/29] ci: re-trigger workflow run after outage From 39d6d25935fe8e193ba96b2c0d15aa0bbee286b6 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 26 May 2026 17:57:30 +0000 Subject: [PATCH 15/29] fix(ci): match kind image to what the kustomize overlay references 503s in test-nginx/test-traefik traced to two issues: 1. The prod overlay maps openms-streamlit -> ghcr.io/openms/streamlit-template:main-full, but the build job was re-tagging the local image as openms-streamlit:test. Rendered manifests pointed at the registry name; kind only had :test loaded; pods stayed ErrImagePull. Retag as :main-full so kind has exactly the ref the manifests use. 2. Three of the four pod specs declare imagePullPolicy: Always; the existing sed only rewrote IfNotPresent. With Always and no registry creds in kind, pods loop on ImagePullBackOff. Extend the sed to catch both. --- .github/workflows/build-and-test.yml | 28 ++++++++++++++++------------ 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml index 7890072c..91b24739 100644 --- a/.github/workflows/build-and-test.yml +++ b/.github/workflows/build-and-test.yml @@ -98,15 +98,17 @@ jobs: build-args: | GITHUB_TOKEN=${{ secrets.GITHUB_TOKEN }} - - name: Retag for kind (stable local tag) + - name: Retag for kind (image name the kustomize overlay points at) run: | - # load:true above loaded all meta-action tags into local docker. - # Retag the first one to the stable name the kustomize overlay expects. + # The prod overlay sets `newName: ghcr.io/openms/streamlit-template`, + # `newTag: main-full`. The rendered manifests reference that exact + # ref, so we need it loaded into kind under that name. Tag invariant + # across branches/variants so the test always works. FIRST_TAG=$(printf '%s\n' "${{ steps.meta.outputs.tags }}" | head -n 1) - docker tag "$FIRST_TAG" openms-streamlit:test + docker tag "$FIRST_TAG" ghcr.io/openms/streamlit-template:main-full - name: Save image as tar - run: docker save openms-streamlit:test -o /tmp/image.tar + run: docker save ghcr.io/openms/streamlit-template:main-full -o /tmp/image.tar - name: Upload image artifact uses: actions/upload-artifact@v4 @@ -192,15 +194,17 @@ jobs: build-args: | GITHUB_TOKEN=${{ secrets.GITHUB_TOKEN }} - - name: Retag for kind (stable local tag) + - name: Retag for kind (image name the kustomize overlay points at) run: | - # load:true above loaded all meta-action tags into local docker. - # Retag the first one to the stable name the kustomize overlay expects. + # The prod overlay sets `newName: ghcr.io/openms/streamlit-template`, + # `newTag: main-full`. The rendered manifests reference that exact + # ref, so we need it loaded into kind under that name. Tag invariant + # across branches/variants so the test always works. FIRST_TAG=$(printf '%s\n' "${{ steps.meta.outputs.tags }}" | head -n 1) - docker tag "$FIRST_TAG" openms-streamlit:test + docker tag "$FIRST_TAG" ghcr.io/openms/streamlit-template:main-full - name: Save image as tar - run: docker save openms-streamlit:test -o /tmp/image.tar + run: docker save ghcr.io/openms/streamlit-template:main-full -o /tmp/image.tar - name: Upload image artifact uses: actions/upload-artifact@v4 @@ -580,7 +584,7 @@ jobs: # Filter out Traefik IngressRoute (kind cluster uses nginx) and force imagePullPolicy=Never kubectl kustomize k8s/overlays/prod/ | \ yq 'select(.kind != "IngressRoute")' | \ - sed 's|imagePullPolicy: IfNotPresent|imagePullPolicy: Never|g' | \ + sed -E 's|imagePullPolicy: (IfNotPresent\|Always)|imagePullPolicy: Never|g' | \ sed 's|storageClassName: cinder-csi|storageClassName: standard|g' > /tmp/manifests.yaml for i in 1 2 3 4 5; do if kubectl apply -f /tmp/manifests.yaml; then @@ -720,7 +724,7 @@ jobs: - name: Deploy with Kustomize (full manifests, no filter) run: | kubectl kustomize k8s/overlays/prod/ | \ - sed 's|imagePullPolicy: IfNotPresent|imagePullPolicy: Never|g' | \ + sed -E 's|imagePullPolicy: (IfNotPresent\|Always)|imagePullPolicy: Never|g' | \ sed 's|storageClassName: cinder-csi|storageClassName: standard|g' > /tmp/manifests.yaml for i in 1 2 3 4 5; do if kubectl apply -f /tmp/manifests.yaml; then From 0ad89fa41b2f134665b487a68c725ce6c29f67e6 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 27 May 2026 16:06:51 +0000 Subject: [PATCH 16/29] fix(ci): disable provenance on build-amd64 to keep pushes single-manifest create-manifest fails with "ghcr.io/openms/streamlit-template:main-full-amd64 is a manifest list" because docker/build-push-action v5 adds a provenance attestation by default, which buildx packs as a manifest list (image + attestation entries). docker manifest create rejects manifest lists as components. build-arm64 already sets provenance: false for the same reason; mirror that on the amd64 path so both per-arch tags are flat image manifests that can be merged into the multi-arch manifest. --- .github/workflows/build-and-test.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml index 91b24739..3b4055bd 100644 --- a/.github/workflows/build-and-test.yml +++ b/.github/workflows/build-and-test.yml @@ -93,6 +93,11 @@ jobs: push: ${{ github.event_name != 'pull_request' }} tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} + # provenance/attestations turn the pushed tag into a manifest list, + # which the create-manifest job's `docker manifest create` then + # refuses ("is a manifest list"). Keep the push as a single-platform + # image manifest — same as the build-arm64 job. + provenance: false cache-from: type=registry,ref=${{ env.REGISTRY }}/${{ env.IMAGE_NAME_LC }}/cache:${{ matrix.variant }}-amd64 cache-to: ${{ github.event_name != 'pull_request' && format('type=registry,ref={0}/{1}/cache:{2}-amd64,mode=max', env.REGISTRY, env.IMAGE_NAME_LC, matrix.variant) || '' }} build-args: | From a703e052c46746fdaf6a03dcbf95a1c49db14956 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=20David=20M=C3=BCller?= <57191390+t0mdavid-m@users.noreply.github.com> Date: Wed, 17 Jun 2026 14:54:02 +0200 Subject: [PATCH 17/29] Merge pull request #396 from OpenMS/claude/laughing-fermat-ppiw0s Add configurable legal links (Impressum, Privacy, Terms) --- README.md | 21 +++++ gdpr_consent/dist/bundle.js | 2 +- gdpr_consent/src/main.ts | 13 +++ settings.json | 5 ++ src/common/captcha_.py | 11 ++- src/common/common.py | 48 ++++++++++- tests/test_legal_links.py | 160 ++++++++++++++++++++++++++++++++++++ 7 files changed, 255 insertions(+), 5 deletions(-) create mode 100644 tests/test_legal_links.py diff --git a/README.md b/README.md index 09cec892..5a1db4b2 100644 --- a/README.md +++ b/README.md @@ -164,6 +164,27 @@ nginx config, PID files — to `/tmp/openms-runtime-$$`, which is always writable inside an apptainer container. The workspace cleanup cron job is skipped in this mode; rerun `clean-up-workspaces.py` manually if needed. +## ⚖️ Legal pages (Impressum, Privacy Policy, Terms of Use) + +Every page shows **Impressum**, **Privacy Policy** and **Terms of Use** links at +the bottom of the sidebar, and the GDPR consent banner links to the privacy +policy. By default these point to the centrally maintained official OpenMS pages +(`https://openms.de/impressum`, `/privacy`, `/terms`). + +If you self-host a fork, override them in `settings.json` — an Impressum must +name the **actual operator**, not OpenMS: + +```json +"legal_links": { + "impressum": "https://your-domain.example/impressum", + "privacy": "https://your-domain.example/privacy", + "terms": "https://your-domain.example/terms" +} +``` + +Any link you omit falls back to its OpenMS default. The `privacy` URL is reused +for the consent banner's privacy-policy link, so consent and policy stay in sync. + ## Documentation Documentation for **users** and **developers** is included as pages in [this template app](https://abi-services.cs.uni-tuebingen.de/streamlit-template/), indicated by the 📖 icon. diff --git a/gdpr_consent/dist/bundle.js b/gdpr_consent/dist/bundle.js index 86144573..0a48bfdf 100644 --- a/gdpr_consent/dist/bundle.js +++ b/gdpr_consent/dist/bundle.js @@ -235,7 +235,7 @@ eval("__webpack_require__.r(__webpack_exports__);\n/* harmony export */ __webpac /***/ ((__unused_webpack_module, __webpack_exports__, __webpack_require__) => { "use strict"; -eval("__webpack_require__.r(__webpack_exports__);\n/* harmony import */ var streamlit_component_lib__WEBPACK_IMPORTED_MODULE_0__ = __webpack_require__(/*! streamlit-component-lib */ \"./node_modules/streamlit-component-lib/dist/index.js\");\nvar __awaiter = (undefined && undefined.__awaiter) || function (thisArg, _arguments, P, generator) {\n function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }\n return new (P || (P = Promise))(function (resolve, reject) {\n function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }\n function rejected(value) { try { step(generator[\"throw\"](value)); } catch (e) { reject(e); } }\n function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }\n step((generator = generator.apply(thisArg, _arguments || [])).next());\n });\n};\nvar __generator = (undefined && undefined.__generator) || function (thisArg, body) {\n var _ = { label: 0, sent: function() { if (t[0] & 1) throw t[1]; return t[1]; }, trys: [], ops: [] }, f, y, t, g;\n return g = { next: verb(0), \"throw\": verb(1), \"return\": verb(2) }, typeof Symbol === \"function\" && (g[Symbol.iterator] = function() { return this; }), g;\n function verb(n) { return function (v) { return step([n, v]); }; }\n function step(op) {\n if (f) throw new TypeError(\"Generator is already executing.\");\n while (g && (g = 0, op[0] && (_ = 0)), _) try {\n if (f = 1, y && (t = op[0] & 2 ? y[\"return\"] : op[0] ? y[\"throw\"] || ((t = y[\"return\"]) && t.call(y), 0) : y.next) && !(t = t.call(y, op[1])).done) return t;\n if (y = 0, t) op = [op[0] & 2, t.value];\n switch (op[0]) {\n case 0: case 1: t = op; break;\n case 4: _.label++; return { value: op[1], done: false };\n case 5: _.label++; y = op[1]; op = [0]; continue;\n case 7: op = _.ops.pop(); _.trys.pop(); continue;\n default:\n if (!(t = _.trys, t = t.length > 0 && t[t.length - 1]) && (op[0] === 6 || op[0] === 2)) { _ = 0; continue; }\n if (op[0] === 3 && (!t || (op[1] > t[0] && op[1] < t[3]))) { _.label = op[1]; break; }\n if (op[0] === 6 && _.label < t[1]) { _.label = t[1]; t = op; break; }\n if (t && _.label < t[2]) { _.label = t[2]; _.ops.push(op); break; }\n if (t[2]) _.ops.pop();\n _.trys.pop(); continue;\n }\n op = body.call(thisArg, _);\n } catch (e) { op = [6, e]; y = 0; } finally { f = t = 0; }\n if (op[0] & 5) throw op[1]; return { value: op[0] ? op[1] : void 0, done: true };\n }\n};\n\n// Defines the configuration for Klaro\nvar klaroConfig = {\n mustConsent: true,\n acceptAll: true,\n services: []\n};\n// This will make klaroConfig globally accessible\nwindow.klaroConfig = klaroConfig;\n// Function to safely access the Klaro manager\nfunction getKlaroManager() {\n var _a;\n return ((_a = window.klaro) === null || _a === void 0 ? void 0 : _a.getManager) ? window.klaro.getManager() : null;\n}\n// Waits until Klaro Manager is available\nfunction waitForKlaroManager() {\n return __awaiter(this, arguments, void 0, function (maxWaitTime, interval) {\n var startTime, klaroManager;\n if (maxWaitTime === void 0) { maxWaitTime = 5000; }\n if (interval === void 0) { interval = 100; }\n return __generator(this, function (_a) {\n switch (_a.label) {\n case 0:\n startTime = Date.now();\n _a.label = 1;\n case 1:\n if (!(Date.now() - startTime < maxWaitTime)) return [3 /*break*/, 3];\n klaroManager = getKlaroManager();\n if (klaroManager) {\n return [2 /*return*/, klaroManager];\n }\n return [4 /*yield*/, new Promise(function (resolve) { return setTimeout(resolve, interval); })];\n case 2:\n _a.sent();\n return [3 /*break*/, 1];\n case 3: throw new Error(\"Klaro manager did not become available within the allowed time.\");\n }\n });\n });\n}\n// Helper function to handle unknown errors\nfunction handleError(error) {\n if (error instanceof Error) {\n console.error(\"Error:\", error.message);\n }\n else {\n console.error(\"Unknown error:\", error);\n }\n}\n// Tracking was accepted\nfunction callback() {\n return __awaiter(this, void 0, void 0, function () {\n var manager, return_vals, _i, _a, service, error_1;\n return __generator(this, function (_b) {\n switch (_b.label) {\n case 0:\n _b.trys.push([0, 2, , 3]);\n return [4 /*yield*/, waitForKlaroManager()];\n case 1:\n manager = _b.sent();\n if (manager.confirmed) {\n return_vals = {};\n for (_i = 0, _a = klaroConfig.services; _i < _a.length; _i++) {\n service = _a[_i];\n return_vals[service.name] = manager.getConsent(service.name);\n }\n streamlit_component_lib__WEBPACK_IMPORTED_MODULE_0__.Streamlit.setComponentValue(return_vals);\n }\n return [3 /*break*/, 3];\n case 2:\n error_1 = _b.sent();\n handleError(error_1);\n return [3 /*break*/, 3];\n case 3: return [2 /*return*/];\n }\n });\n });\n}\n// Stores if the component has been rendered before\nvar rendered = false;\nfunction onRender(event) {\n // Klaro does not work if embedded multiple times\n if (rendered) {\n return;\n }\n rendered = true;\n var data = event.detail;\n if (data.args['google_analytics']) {\n klaroConfig.services.push({\n name: 'google-analytics',\n cookies: [\n /^_ga(_.*)?/ // we delete the Google Analytics cookies if the user declines its use\n ],\n purposes: ['analytics'],\n onAccept: callback,\n onDecline: callback,\n });\n }\n if (data.args['piwik_pro']) {\n klaroConfig.services.push({\n name: 'piwik-pro',\n purposes: ['analytics'],\n onAccept: callback,\n onDecline: callback,\n });\n }\n if (data.args['matomo']) {\n klaroConfig.services.push({\n name: 'matomo',\n purposes: ['analytics'],\n onAccept: callback,\n onDecline: callback,\n });\n }\n // Create a new script element\n var script = document.createElement('script');\n // Set the necessary attributes\n script.defer = true;\n script.type = 'application/javascript';\n script.src = 'https://cdn.kiprotect.com/klaro/v0.7/klaro.js';\n // Set the klaro config\n script.setAttribute('data-config', 'klaroConfig');\n // Append the script to the head or body\n document.head.appendChild(script);\n}\n// Attach our `onRender` handler to Streamlit's render event.\nstreamlit_component_lib__WEBPACK_IMPORTED_MODULE_0__.Streamlit.events.addEventListener(streamlit_component_lib__WEBPACK_IMPORTED_MODULE_0__.Streamlit.RENDER_EVENT, onRender);\n// Tell Streamlit we're ready to start receiving data. We won't get our\n// first RENDER_EVENT until we call this function.\nstreamlit_component_lib__WEBPACK_IMPORTED_MODULE_0__.Streamlit.setComponentReady();\n// Finally, tell Streamlit to update the initial height.\nstreamlit_component_lib__WEBPACK_IMPORTED_MODULE_0__.Streamlit.setFrameHeight(1000);\n\n\n//# sourceURL=webpack://gdpr_consent/./src/main.ts?"); +eval("__webpack_require__.r(__webpack_exports__);\n/* harmony import */ var streamlit_component_lib__WEBPACK_IMPORTED_MODULE_0__ = __webpack_require__(/*! streamlit-component-lib */ \"./node_modules/streamlit-component-lib/dist/index.js\");\nvar __awaiter = (undefined && undefined.__awaiter) || function (thisArg, _arguments, P, generator) {\n function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }\n return new (P || (P = Promise))(function (resolve, reject) {\n function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }\n function rejected(value) { try { step(generator[\"throw\"](value)); } catch (e) { reject(e); } }\n function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }\n step((generator = generator.apply(thisArg, _arguments || [])).next());\n });\n};\nvar __generator = (undefined && undefined.__generator) || function (thisArg, body) {\n var _ = { label: 0, sent: function() { if (t[0] & 1) throw t[1]; return t[1]; }, trys: [], ops: [] }, f, y, t, g;\n return g = { next: verb(0), \"throw\": verb(1), \"return\": verb(2) }, typeof Symbol === \"function\" && (g[Symbol.iterator] = function() { return this; }), g;\n function verb(n) { return function (v) { return step([n, v]); }; }\n function step(op) {\n if (f) throw new TypeError(\"Generator is already executing.\");\n while (g && (g = 0, op[0] && (_ = 0)), _) try {\n if (f = 1, y && (t = op[0] & 2 ? y[\"return\"] : op[0] ? y[\"throw\"] || ((t = y[\"return\"]) && t.call(y), 0) : y.next) && !(t = t.call(y, op[1])).done) return t;\n if (y = 0, t) op = [op[0] & 2, t.value];\n switch (op[0]) {\n case 0: case 1: t = op; break;\n case 4: _.label++; return { value: op[1], done: false };\n case 5: _.label++; y = op[1]; op = [0]; continue;\n case 7: op = _.ops.pop(); _.trys.pop(); continue;\n default:\n if (!(t = _.trys, t = t.length > 0 && t[t.length - 1]) && (op[0] === 6 || op[0] === 2)) { _ = 0; continue; }\n if (op[0] === 3 && (!t || (op[1] > t[0] && op[1] < t[3]))) { _.label = op[1]; break; }\n if (op[0] === 6 && _.label < t[1]) { _.label = t[1]; t = op; break; }\n if (t && _.label < t[2]) { _.label = t[2]; _.ops.push(op); break; }\n if (t[2]) _.ops.pop();\n _.trys.pop(); continue;\n }\n op = body.call(thisArg, _);\n } catch (e) { op = [6, e]; y = 0; } finally { f = t = 0; }\n if (op[0] & 5) throw op[1]; return { value: op[0] ? op[1] : void 0, done: true };\n }\n};\n\n// Defines the configuration for Klaro\nvar klaroConfig = {\n mustConsent: true,\n acceptAll: true,\n services: []\n};\n// This will make klaroConfig globally accessible\nwindow.klaroConfig = klaroConfig;\n// Function to safely access the Klaro manager\nfunction getKlaroManager() {\n var _a;\n return ((_a = window.klaro) === null || _a === void 0 ? void 0 : _a.getManager) ? window.klaro.getManager() : null;\n}\n// Waits until Klaro Manager is available\nfunction waitForKlaroManager() {\n return __awaiter(this, arguments, void 0, function (maxWaitTime, interval) {\n var startTime, klaroManager;\n if (maxWaitTime === void 0) { maxWaitTime = 5000; }\n if (interval === void 0) { interval = 100; }\n return __generator(this, function (_a) {\n switch (_a.label) {\n case 0:\n startTime = Date.now();\n _a.label = 1;\n case 1:\n if (!(Date.now() - startTime < maxWaitTime)) return [3 /*break*/, 3];\n klaroManager = getKlaroManager();\n if (klaroManager) {\n return [2 /*return*/, klaroManager];\n }\n return [4 /*yield*/, new Promise(function (resolve) { return setTimeout(resolve, interval); })];\n case 2:\n _a.sent();\n return [3 /*break*/, 1];\n case 3: throw new Error(\"Klaro manager did not become available within the allowed time.\");\n }\n });\n });\n}\n// Helper function to handle unknown errors\nfunction handleError(error) {\n if (error instanceof Error) {\n console.error(\"Error:\", error.message);\n }\n else {\n console.error(\"Unknown error:\", error);\n }\n}\n// Tracking was accepted\nfunction callback() {\n return __awaiter(this, void 0, void 0, function () {\n var manager, return_vals, _i, _a, service, error_1;\n return __generator(this, function (_b) {\n switch (_b.label) {\n case 0:\n _b.trys.push([0, 2, , 3]);\n return [4 /*yield*/, waitForKlaroManager()];\n case 1:\n manager = _b.sent();\n if (manager.confirmed) {\n return_vals = {};\n for (_i = 0, _a = klaroConfig.services; _i < _a.length; _i++) {\n service = _a[_i];\n return_vals[service.name] = manager.getConsent(service.name);\n }\n streamlit_component_lib__WEBPACK_IMPORTED_MODULE_0__.Streamlit.setComponentValue(return_vals);\n }\n return [3 /*break*/, 3];\n case 2:\n error_1 = _b.sent();\n handleError(error_1);\n return [3 /*break*/, 3];\n case 3: return [2 /*return*/];\n }\n });\n });\n}\n// Stores if the component has been rendered before\nvar rendered = false;\nfunction onRender(event) {\n // Klaro does not work if embedded multiple times\n if (rendered) {\n return;\n }\n rendered = true;\n var data = event.detail;\n if (data.args['google_analytics']) {\n klaroConfig.services.push({\n name: 'google-analytics',\n cookies: [\n /^_ga(_.*)?/ // we delete the Google Analytics cookies if the user declines its use\n ],\n purposes: ['analytics'],\n onAccept: callback,\n onDecline: callback,\n });\n }\n if (data.args['piwik_pro']) {\n klaroConfig.services.push({\n name: 'piwik-pro',\n purposes: ['analytics'],\n onAccept: callback,\n onDecline: callback,\n });\n }\n if (data.args['matomo']) {\n klaroConfig.services.push({\n name: 'matomo',\n purposes: ['analytics'],\n onAccept: callback,\n onDecline: callback,\n });\n }\n // Link the consent banner to the privacy policy. Setting privacyPolicyUrl\n // on the 'zz' fallback language makes Klaro render its default\n // \"To learn more, please read our privacy policy.\" text with the URL,\n // regardless of the browser locale.\n if (data.args['privacy_policy']) {\n klaroConfig.translations = {\n zz: {\n privacyPolicyUrl: data.args['privacy_policy']\n }\n };\n }\n // Create a new script element\n var script = document.createElement('script');\n // Set the necessary attributes\n script.defer = true;\n script.type = 'application/javascript';\n script.src = 'https://cdn.kiprotect.com/klaro/v0.7/klaro.js';\n // Set the klaro config\n script.setAttribute('data-config', 'klaroConfig');\n // Append the script to the head or body\n document.head.appendChild(script);\n}\n// Attach our `onRender` handler to Streamlit's render event.\nstreamlit_component_lib__WEBPACK_IMPORTED_MODULE_0__.Streamlit.events.addEventListener(streamlit_component_lib__WEBPACK_IMPORTED_MODULE_0__.Streamlit.RENDER_EVENT, onRender);\n// Tell Streamlit we're ready to start receiving data. We won't get our\n// first RENDER_EVENT until we call this function.\nstreamlit_component_lib__WEBPACK_IMPORTED_MODULE_0__.Streamlit.setComponentReady();\n// Finally, tell Streamlit to update the initial height.\nstreamlit_component_lib__WEBPACK_IMPORTED_MODULE_0__.Streamlit.setFrameHeight(1000);\n\n\n//# sourceURL=webpack://gdpr_consent/./src/main.ts?"); /***/ }), diff --git a/gdpr_consent/src/main.ts b/gdpr_consent/src/main.ts index 059fef89..408f4a4b 100644 --- a/gdpr_consent/src/main.ts +++ b/gdpr_consent/src/main.ts @@ -14,6 +14,7 @@ let klaroConfig: { mustConsent: boolean; acceptAll: boolean; services: Service[]; + translations?: Record; } = { mustConsent: true, acceptAll: true, @@ -125,6 +126,18 @@ function onRender(event: Event): void { ) } + // Link the consent banner to the privacy policy. Setting privacyPolicyUrl + // on the 'zz' fallback language makes Klaro render its default + // "To learn more, please read our privacy policy." text with the URL, + // regardless of the browser locale. + if (data.args['privacy_policy']) { + klaroConfig.translations = { + zz: { + privacyPolicyUrl: data.args['privacy_policy'] + } + } + } + // Create a new script element var script = document.createElement('script') diff --git a/settings.json b/settings.json index 0adf0b6e..60424cb3 100644 --- a/settings.json +++ b/settings.json @@ -3,6 +3,11 @@ "github-user": "OpenMS", "version": "1.1.1", "repository-name": "streamlit-template", + "legal_links": { + "impressum": "https://openms.de/impressum", + "privacy": "https://openms.de/privacy", + "terms": "https://openms.de/terms" + }, "analytics": { "google-analytics": { "enabled": false, diff --git a/src/common/captcha_.py b/src/common/captcha_.py index 498b1336..282e1246 100644 --- a/src/common/captcha_.py +++ b/src/common/captcha_.py @@ -186,7 +186,7 @@ def add_page(main_script_path_str: str, page_name: str) -> None: # define the function for the captcha control -def captcha_control(): +def captcha_control(privacy_policy_url: str = ""): """ Control and verification of a CAPTCHA to ensure the user is not a robot. @@ -199,6 +199,10 @@ def captcha_control(): The CAPTCHA text is generated as a session state and should not change during refreshes. + Args: + privacy_policy_url (str, optional): URL shown as the privacy policy link + in the GDPR consent banner. Defaults to "". + Returns: None """ @@ -214,7 +218,10 @@ def captcha_control(): with st.spinner(): # Ask for consent st.session_state.tracking_consent = consent_component( - google_analytics=ga, piwik_pro=pp, matomo=mt + google_analytics=ga, + piwik_pro=pp, + matomo=mt, + privacy_policy=privacy_policy_url, ) if st.session_state.tracking_consent is None: # No response by user yet diff --git a/src/common/common.py b/src/common/common.py index 643a2247..c048064f 100644 --- a/src/common/common.py +++ b/src/common/common.py @@ -31,6 +31,37 @@ # Detect system platform OS_PLATFORM = sys.platform +# Default legal/GDPR page links. These point to the centrally maintained +# official OpenMS pages. Forks that self-host should override them via the +# "legal_links" key in settings.json (an Impressum must name the actual +# operator). The defaults live here too — not only in settings.json — so that +# downstream apps built from an older settings.json without a "legal_links" +# key still inherit working legal links by default. +DEFAULT_LEGAL_LINKS = { + "impressum": "https://openms.de/impressum", + "privacy": "https://openms.de/privacy", + "terms": "https://openms.de/terms", +} + + +def get_legal_links() -> dict[str, str]: + """ + Return the legal page URLs (Impressum, Privacy Policy, Terms of Use). + + Values from the "legal_links" object in settings.json override the + built-in OpenMS defaults. Empty override values are ignored so a blank + entry can't erase a default. + + Returns: + dict[str, str]: Mapping of "impressum", "privacy" and "terms" to URLs. + """ + overrides = ( + st.session_state.settings.get("legal_links", {}) + if "settings" in st.session_state + else {} + ) + return {**DEFAULT_LEGAL_LINKS, **{k: v for k, v in overrides.items() if v}} + def is_safe_workspace_name(name: str) -> bool: """ @@ -519,7 +550,7 @@ def page_setup(page: str = "") -> dict[str, Any]: # Render the sidebar params = render_sidebar(page) - captcha_control() + captcha_control(privacy_policy_url=get_legal_links()["privacy"]) # If run in hosted mode, show captcha as long as it has not been solved # if not "local" in sys.argv: @@ -532,7 +563,7 @@ def page_setup(page: str = "") -> dict[str, Any]: "controllo" in params.keys() and params["controllo"] == False ): # Apply captcha by calling the captcha_control function - captcha_control() + captcha_control(privacy_policy_url=get_legal_links()["privacy"]) return params @@ -764,6 +795,19 @@ def change_workspace(): f'
{app_name}
Version: {version_info}
', unsafe_allow_html=True, ) + + # Legal links (Impressum, Privacy Policy, Terms of Use), shown on every + # page. URLs are configurable via "legal_links" in settings.json. + links = get_legal_links() + st.markdown( + '
' + f'Impressum · ' + f'Privacy Policy · ' + f'Terms of Use' + "
", + unsafe_allow_html=True, + ) return params diff --git a/tests/test_legal_links.py b/tests/test_legal_links.py new file mode 100644 index 00000000..a38e2017 --- /dev/null +++ b/tests/test_legal_links.py @@ -0,0 +1,160 @@ +""" +Tests for get_legal_links() in src/common/common.py. + +get_legal_links() resolves the Impressum / Privacy Policy / Terms of Use URLs +shown in the sidebar footer (on every page) and the privacy-policy link wired +into the GDPR consent banner. It merges the optional "legal_links" object from +settings.json over the built-in official-OpenMS defaults so that: + + * apps built from a settings.json without a "legal_links" key still inherit + working legal links by default, + * a self-hosting fork can override any or all of the three URLs, + * an empty/blank override value never erases a default. + +Streamlit (and the other heavy runtime deps pulled in by common.py) are mocked +before import so the helper can be unit-tested without a running Streamlit app, +mirroring tests/test_parameter_presets.py. +""" +import os +import sys +from unittest.mock import MagicMock + +# Add project root to path for imports +PROJECT_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +sys.path.append(PROJECT_ROOT) + + +class FakeSessionState(dict): + """Minimal stand-in for Streamlit's SessionState. + + Supports both attribute access (``state.settings``) and item/membership + access (``"settings" in state``), exactly like the real SessionState that + common.py relies on. + """ + + def __getattr__(self, name): + try: + return self[name] + except KeyError as exc: + raise AttributeError(name) from exc + + def __setattr__(self, name, value): + self[name] = value + + +# Mock streamlit (with a SessionState-like session_state) and the other heavy +# imports pulled in by src/common/common.py, so importing get_legal_links here +# doesn't require a running Streamlit app context. +# +# IMPORTANT: these mocks are installed into sys.modules only for the duration of +# the import below and then restored, so they don't leak into other test modules +# (e.g. the AppTest-based tests that need the real `streamlit` package). This +# mirrors the pattern in tests/test_parameter_presets.py. +mock_streamlit = MagicMock() +mock_streamlit.session_state = FakeSessionState() + +_MOCKED_MODULES = { + "streamlit": mock_streamlit, + "streamlit.components": MagicMock(), + "streamlit.components.v1": MagicMock(), + "streamlit.source_util": MagicMock(), + "pandas": MagicMock(), + "psutil": MagicMock(), + # Local submodules with their own heavy deps (e.g. the captcha image library). + "src.common.captcha_": MagicMock(), + "src.common.admin": MagicMock(), +} +_saved_modules = {name: sys.modules.get(name) for name in _MOCKED_MODULES} +sys.modules.update(_MOCKED_MODULES) + +# Force a FRESH import of src.common.common under the streamlit mock, even if an +# earlier test module (e.g. test_gui.py) already imported the real-streamlit-bound +# version. Save whatever was cached first so we can restore it afterwards. +_saved_common = sys.modules.pop("src.common.common", None) + +from src.common.common import get_legal_links, DEFAULT_LEGAL_LINKS # noqa: E402 + +# Restore the real modules (or remove ones that weren't present) so that other +# test modules get the genuine packages. +for _name, _orig in _saved_modules.items(): + if _orig is None: + sys.modules.pop(_name, None) + else: + sys.modules[_name] = _orig +# Restore the original cached common module (the real-streamlit-bound one, if +# any) so AppTest-based test modules keep getting the genuine package. +# get_legal_links keeps working: it holds a reference to the freshly-imported +# mock-bound module's globals (and the same `mock_streamlit` object the tests +# mutate). +if _saved_common is None: + sys.modules.pop("src.common.common", None) +else: + sys.modules["src.common.common"] = _saved_common + + +def setup_function(_): + """Reset session_state before each test for isolation.""" + mock_streamlit.session_state = FakeSessionState() + + +def test_defaults_point_to_openms(): + """The built-in defaults are the official OpenMS pages.""" + assert DEFAULT_LEGAL_LINKS == { + "impressum": "https://openms.de/impressum", + "privacy": "https://openms.de/privacy", + "terms": "https://openms.de/terms", + } + + +def test_defaults_when_settings_not_loaded(): + """No settings loaded at all -> defaults, no crash.""" + mock_streamlit.session_state = FakeSessionState() + assert get_legal_links() == DEFAULT_LEGAL_LINKS + + +def test_defaults_when_no_legal_links_key(): + """settings present but without 'legal_links' -> all OpenMS defaults.""" + mock_streamlit.session_state = FakeSessionState({"settings": {}}) + assert get_legal_links() == DEFAULT_LEGAL_LINKS + + +def test_overrides_replace_defaults(): + """A fork's custom legal_links replace every default.""" + mock_streamlit.session_state = FakeSessionState( + { + "settings": { + "legal_links": { + "impressum": "https://acme.example/impressum", + "privacy": "https://acme.example/privacy", + "terms": "https://acme.example/terms", + } + } + } + ) + assert get_legal_links() == { + "impressum": "https://acme.example/impressum", + "privacy": "https://acme.example/privacy", + "terms": "https://acme.example/terms", + } + + +def test_partial_override_keeps_other_defaults(): + """Overriding only one link leaves the others at their OpenMS default.""" + mock_streamlit.session_state = FakeSessionState( + {"settings": {"legal_links": {"impressum": "https://acme.example/impressum"}}} + ) + links = get_legal_links() + assert links["impressum"] == "https://acme.example/impressum" + assert links["privacy"] == DEFAULT_LEGAL_LINKS["privacy"] + assert links["terms"] == DEFAULT_LEGAL_LINKS["terms"] + + +def test_empty_or_none_override_falls_back_to_default(): + """A blank/None override must not erase the default for that key.""" + mock_streamlit.session_state = FakeSessionState( + {"settings": {"legal_links": {"privacy": "", "impressum": None}}} + ) + links = get_legal_links() + assert links["privacy"] == DEFAULT_LEGAL_LINKS["privacy"] + assert links["impressum"] == DEFAULT_LEGAL_LINKS["impressum"] + assert links["terms"] == DEFAULT_LEGAL_LINKS["terms"] From 76a793e6ff71e273b3d74e4944eabe1b9f4c3892 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=20David=20M=C3=BCller?= <57191390+t0mdavid-m@users.noreply.github.com> Date: Wed, 17 Jun 2026 17:39:44 +0200 Subject: [PATCH 18/29] fix treafik ingress --- k8s/base/traefik-ingressroute.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/k8s/base/traefik-ingressroute.yaml b/k8s/base/traefik-ingressroute.yaml index b202891f..857314fd 100644 --- a/k8s/base/traefik-ingressroute.yaml +++ b/k8s/base/traefik-ingressroute.yaml @@ -4,7 +4,8 @@ metadata: name: streamlit-traefik spec: entryPoints: - - web + - web # 301-redirects to websecure + - websecure routes: - match: PathPrefix(`/`) kind: Rule @@ -16,3 +17,4 @@ spec: name: stroute httpOnly: true sameSite: lax + secure: true # only send the affinity cookie over HTTPS \ No newline at end of file From 2b8cda25c5033bc289cad2bf45a892c5c47cab38 Mon Sep 17 00:00:00 2001 From: Copilot <198982749+Copilot@users.noreply.github.com> Date: Tue, 14 Jul 2026 10:03:14 +0200 Subject: [PATCH 19/29] Fix tool_instance_name resolution in save_parameters and support multiple TOPP tool instances (#354) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Fix tool instance name handling and move custom defaults to params.json Resolves the issue where multiple instances of the same TOPP tool could not have independent custom defaults. The .ini file (keyed by tool name) only supported one set of defaults — the first instance's defaults were baked in and subsequent instances were silently ignored. Changes: - Add get_merged_params() to ParameterManager for centralized three-layer parameter resolution (ini defaults < _defaults < user overrides) - Replace .ini file mutation with _defaults seeding in params.json, keyed by tool instance name - Update save_parameters() to compare against _defaults when determining which values are non-default - Update run_topp() and non_default_params_summary() to use merged params - Resolve instance names to real tool names across save_parameters, input_TOPP, run_topp, and get_topp_parameters * Remove redundant -ini flag from run_topp The .ini file is no longer mutated with custom defaults — it stays pristine. All parameters (_defaults + user overrides) are already passed as CLI flags via get_merged_params(), making -ini redundant. * dont disable cross origin --------- Co-authored-by: Tom David Mueller Co-authored-by: Tom David Müller <57191390+t0mdavid-m@users.noreply.github.com> --- src/workflow/CommandExecutor.py | 56 ++--- src/workflow/ParameterManager.py | 72 ++++-- src/workflow/StreamlitUI.py | 78 ++++--- tests/test_parameter_defaults.py | 363 +++++++++++++++++++++++++++++++ tests/test_tool_instance_name.py | 268 +++++++++++++++++++++++ 5 files changed, 753 insertions(+), 84 deletions(-) create mode 100644 tests/test_parameter_defaults.py create mode 100644 tests/test_tool_instance_name.py diff --git a/src/workflow/CommandExecutor.py b/src/workflow/CommandExecutor.py index b15a193e..11bb1486 100644 --- a/src/workflow/CommandExecutor.py +++ b/src/workflow/CommandExecutor.py @@ -234,10 +234,9 @@ def run_topp(self, tool: str, input_output: dict, custom_params: dict = {}, tool tool (str): The executable name or path of the tool. input_output (dict): A dictionary specifying the input/output parameter names (as key) and their corresponding file paths (as value). custom_params (dict): A dictionary of custom parameters to pass to the tool. - tool_instance_name (str, optional): Key for ``params.json`` when it differs - from ``tool`` (e.g. multiple instances). Defaults to ``tool``. - Custom parameters whose keys appear in the tool's ParamXML ``type="bool"`` - entries are passed as valueless CLI flags (``-name`` only when enabled). + tool_instance_name (str, optional): A unique instance name for this tool + invocation, used for parameter lookup when multiple instances of the + same tool exist. If not provided, defaults to the tool name. Returns: bool: True if all commands succeeded, False if any failed. @@ -246,6 +245,8 @@ def run_topp(self, tool: str, input_output: dict, custom_params: dict = {}, tool ValueError: If the lengths of input/output file lists are inconsistent, except for single string inputs. """ + # Use tool_instance_name for parameter lookup, fall back to tool name + params_key = tool_instance_name if tool_instance_name else tool # check input: any input lists must be same length, other items can be a single string # e.g. input_mzML : [list of n mzML files], output_featureXML : [list of n featureXML files], input_database : database.tsv io_lengths = [len(v) for v in input_output.values() if len(v) > 1] @@ -265,15 +266,8 @@ def run_topp(self, tool: str, input_output: dict, custom_params: dict = {}, tool commands = [] - params = self.parameter_manager.get_parameters_from_json() - - topp_tool_ini_path = Path(self.parameter_manager.ini_dir, f"{tool}.ini") - # Keys of type="bool" in the .ini: TOPP treats these as on/off flags (omit value when off) - topp_bool_flag_param_keys = ( - bool_param_paths_from_param_xml_ini(topp_tool_ini_path, tool) - if topp_tool_ini_path.exists() - else set() - ) + # Load merged parameters (_defaults + user overrides) for this tool instance + merged_params = self.parameter_manager.get_merged_params(params_key) # Construct commands for each process for i in range(n_processes): command = [tool] @@ -292,27 +286,16 @@ def run_topp(self, tool: str, input_output: dict, custom_params: dict = {}, tool # standard case, files was a list of strings, take the file name at index else: command += [value[i]] - # Add non-default TOPP tool parameters - if tool in params.keys(): - for k, v in params[tool].items(): - - if k in topp_bool_flag_param_keys and v != "": - # CLI flag: include "-k" only when enabled - if isinstance(v, str): - is_enabled = v.lower() == "true" - else: - is_enabled = bool(v) - if is_enabled: - command += [f"-{k}"] - continue - command += [f"-{k}"] - # Skip only empty strings (pass flag with no value) - # Note: 0 and 0.0 are valid values, so use explicit check - if v != "" and v is not None: - if isinstance(v, str) and "\n" in v: - command += v.split("\n") - else: - command += [str(v)] + # Add merged TOPP tool parameters (_defaults + user overrides) + for k, v in merged_params.items(): + command += [f"-{k}"] + # Skip only empty strings (pass flag with no value) + # Note: 0 and 0.0 are valid values, so use explicit check + if v != "" and v is not None: + if isinstance(v, str) and "\n" in v: + command += v.split("\n") + else: + command += [str(v)] # Add custom parameters for k, v in custom_params.items(): command += [f"-{k}"] @@ -328,11 +311,6 @@ def run_topp(self, tool: str, input_output: dict, custom_params: dict = {}, tool command += ["-threads", str(threads_per_command)] commands.append(command) - # check if a ini file has been written, if yes use it (contains custom defaults) - ini_path = Path(self.parameter_manager.ini_dir, tool + ".ini") - if ini_path.exists(): - command += ["-ini", str(ini_path)] - # Run command(s) if len(commands) == 1: return self.run_command(commands[0]) diff --git a/src/workflow/ParameterManager.py b/src/workflow/ParameterManager.py index b8f490ee..2838c1b9 100644 --- a/src/workflow/ParameterManager.py +++ b/src/workflow/ParameterManager.py @@ -135,7 +135,7 @@ def save_parameters(self) -> None: # Advanced parameters are only in session state if the view is active json_params = self.get_parameters_from_json() | json_params - # get a list of TOPP tools which are in session state + # get a list of TOPP tools (or tool instance names) which are in session state current_topp_tools = list( set( [ @@ -145,12 +145,16 @@ def save_parameters(self) -> None: ] ) ) - # for each TOPP tool, open the ini file + # Retrieve the instance-name → real-tool-name mapping (set by input_TOPP) + tool_instance_map = st.session_state.get("_topp_tool_instance_map", {}) + # for each TOPP tool (or instance name), open the ini file for tool in current_topp_tools: - if not self.create_ini(tool): + # Resolve instance name to real tool name for create_ini / ini loading + real_tool = tool_instance_map.get(tool, tool) + if not self.create_ini(real_tool): # Could not create ini file - skip this tool continue - ini_path = Path(self.ini_dir, f"{tool}.ini") + ini_path = Path(self.ini_dir, f"{real_tool}.ini") if tool not in json_params: json_params[tool] = {} # load the param object @@ -162,19 +166,26 @@ def save_parameters(self) -> None: # Skip display keys used by multiselect widgets if key.endswith("_display"): continue - # get ini_key - ini_key = key.replace(self.topp_param_prefix, "").encode() + # get ini_key – map instance name back to real tool name + ini_key = key.replace(self.topp_param_prefix, "") + if tool != real_tool: + ini_key = ini_key.replace(f"{tool}:1:", f"{real_tool}:1:", 1) + ini_key = ini_key.encode() # get ini (default) value by ini_key ini_value = param.getValue(ini_key) is_list_param = isinstance(ini_value, list) - # check if value is different from default OR is an empty list parameter + # Effective default: _defaults value if present, else ini value + short_key = key.split(":1:")[1] + defaults = json_params.get("_defaults", {}).get(tool, {}) + default_value = defaults.get(short_key, ini_value) + # check if value is different from effective default OR is an empty list parameter if ( - (ini_value != value) - or (key.split(":1:")[1] in json_params[tool]) + (default_value != value) + or (short_key in json_params[tool]) or (is_list_param and not value) # Always save empty list params ): # store non-default value - json_params[tool][key.split(":1:")[1]] = value + json_params[tool][short_key] = value # Save to json file with open(self.params_file, "w", encoding="utf-8") as f: json.dump(json_params, f, indent=4) @@ -200,17 +211,44 @@ def get_parameters_from_json(self) -> dict: st.error("**ERROR**: Attempting to load an invalid JSON parameter file. Reset to defaults.") return {} - def get_topp_parameters(self, tool: str) -> dict: + def get_merged_params(self, tool_instance_name: str, ini_params: dict = None) -> dict: + """ + Three-layer parameter merge: ini defaults < _defaults < user overrides. + + Args: + tool_instance_name: Instance name (or tool name) to look up in params.json. + ini_params: Base parameters from the .ini file. Optional — callers that + don't need the ini layer (e.g., run_topp, which passes -ini separately) + can omit this. + + Returns: + Merged dict with the effective value for each parameter. + """ + params = self.get_parameters_from_json() + defaults = params.get("_defaults", {}).get(tool_instance_name, {}) + user = params.get(tool_instance_name, {}) + + merged = {} + if ini_params: + merged.update(ini_params) + merged.update(defaults) + merged.update(user) + return merged + + def get_topp_parameters(self, tool: str, tool_instance_name: str = None) -> dict: """ Get all parameters for a TOPP tool, merging defaults with user values. Args: - tool: Name of the TOPP tool (e.g., "CometAdapter") + tool: Name of the TOPP tool executable (e.g., "CometAdapter") + tool_instance_name: Optional instance name used for parameter storage + (e.g., "IDFilter_step1"). If not provided, defaults to tool name. Returns: Dict with parameter names as keys (without tool prefix) and their values. Returns empty dict if ini file doesn't exist. """ + instance_name = tool_instance_name or tool ini_path = Path(self.ini_dir, f"{tool}.ini") if not ini_path.exists(): return {} @@ -221,18 +259,14 @@ def get_topp_parameters(self, tool: str) -> dict: # Build dict from ini (extract short key names) prefix = f"{tool}:1:" - full_params = {} + ini_params = {} for key in param.keys(): key_str = key.decode() if isinstance(key, bytes) else str(key) if prefix in key_str: short_key = key_str.split(prefix, 1)[1] - full_params[short_key] = param.getValue(key) - - # Override with user-modified values from JSON - user_params = self.get_parameters_from_json().get(tool, {}) - full_params.update(user_params) + ini_params[short_key] = param.getValue(key) - return full_params + return self.get_merged_params(instance_name, ini_params=ini_params) def reset_to_default_parameters(self) -> None: """ diff --git a/src/workflow/StreamlitUI.py b/src/workflow/StreamlitUI.py index 3112f9f2..9c24dc2c 100644 --- a/src/workflow/StreamlitUI.py +++ b/src/workflow/StreamlitUI.py @@ -830,6 +830,7 @@ def input_TOPP( display_subsections: bool = True, display_subsection_tabs: bool = False, custom_defaults: dict = {}, + tool_instance_name: str = None, ) -> None: """ Generates input widgets for TOPP tool parameters dynamically based on the tool's @@ -845,29 +846,43 @@ def input_TOPP( display_subsections (bool, optional): Whether to split parameters into subsections based on the prefix. Defaults to True. display_subsection_tabs (bool, optional): Whether to display main subsections in separate tabs (if more than one main section). Defaults to False. custom_defaults (dict, optional): Dictionary of custom defaults to use. Defaults to an empty dict. + tool_instance_name (str, optional): A unique instance name for this tool + invocation. Allows multiple instances of the same TOPP tool with + independent parameters (e.g., two IDFilter calls). If not provided, + defaults to topp_tool_name. The instance name is used for session + state keys and parameter storage, while topp_tool_name is used for + the actual tool executable and ini file creation. """ + # Default instance name to the tool name when not provided + if tool_instance_name is None: + tool_instance_name = topp_tool_name + + # Register instance-name → real-tool-name mapping in session state + if "_topp_tool_instance_map" not in st.session_state: + st.session_state["_topp_tool_instance_map"] = {} + st.session_state["_topp_tool_instance_map"][tool_instance_name] = topp_tool_name if not display_subsections: display_subsection_tabs = False if display_subsection_tabs: display_subsections = True - # write defaults ini files + # Create pristine ini file (never mutated with custom defaults) ini_file_path = Path(self.parameter_manager.ini_dir, f"{topp_tool_name}.ini") - ini_existed = ini_file_path.exists() if not self.parameter_manager.create_ini(topp_tool_name): st.error(f"TOPP tool **'{topp_tool_name}'** not found.") return - if not ini_existed: - # update custom defaults if necessary - if custom_defaults: - param = poms.Param() - poms.ParamXMLFile().load(str(ini_file_path), param) - for key, value in custom_defaults.items(): - encoded_key = f"{topp_tool_name}:1:{key}".encode() - if encoded_key in param.keys(): - param.setValue(encoded_key, value) - poms.ParamXMLFile().store(str(ini_file_path), param) + + # Seed custom defaults into params.json under _defaults key + if custom_defaults: + params = self.parameter_manager.get_parameters_from_json() + if "_defaults" not in params: + params["_defaults"] = {} + params["_defaults"][tool_instance_name] = custom_defaults + with open(self.parameter_manager.params_file, "w", encoding="utf-8") as f: + json.dump(params, f, indent=4) + # Refresh self.params so widget resolution sees _defaults + self.params = self.parameter_manager.get_parameters_from_json() # read into Param object param = poms.Param() @@ -944,18 +959,18 @@ def _matches_parameter(pattern: str, key: bytes) -> bool: ) params.append(p) - # for each parameter in params_decoded - # if a parameter with custom default value exists, use that value - # else check if the parameter is already in self.params, if yes take the value from self.params + # Build ini_params dict for three-layer merge + ini_params = {} + for p in params: + name = p["key"].decode().split(":1:")[1] + ini_params[name] = p["value"] + + # Resolve effective values: ini < _defaults < user overrides + merged = self.parameter_manager.get_merged_params(tool_instance_name, ini_params=ini_params) for p in params: name = p["key"].decode().split(":1:")[1] - if topp_tool_name in self.params: - if name in self.params[topp_tool_name]: - p["value"] = self.params[topp_tool_name][name] - elif name in custom_defaults: - p["value"] = custom_defaults[name] - elif name in custom_defaults: - p["value"] = custom_defaults[name] + if name in merged: + p["value"] = merged[name] # Ensure list parameters stay as lists after loading from JSON # (JSON may store single-item lists as strings) if p["original_is_list"] and isinstance(p["value"], str): @@ -989,7 +1004,7 @@ def _matches_parameter(pattern: str, key: bytes) -> bool: # Display tool name if required if display_tool_name: - st.markdown(f"**{topp_tool_name}**") + st.markdown(f"**{tool_instance_name}**") tab_names = [k for k in param_sections.keys() if ":" not in k] tabs = None @@ -1017,8 +1032,11 @@ def display_TOPP_params(params: dict, num_cols): cols = st.columns(num_cols) i = 0 for p in params: - # get key and name - key = f"{self.parameter_manager.topp_param_prefix}{p['key'].decode()}" + # get key and name – use tool_instance_name in session state key + key_str = p['key'].decode() + if tool_instance_name != topp_tool_name: + key_str = key_str.replace(f"{topp_tool_name}:1:", f"{tool_instance_name}:1:", 1) + key = f"{self.parameter_manager.topp_param_prefix}{key_str}" name = p["name"] try: # sometimes strings with newline, handle as list @@ -1590,7 +1608,8 @@ def remove_full_paths(d: dict) -> dict: general = {} for k, v in params.items(): - # skip if v is a file path + if k == "_defaults": + continue if isinstance(v, dict): topp[k] = v elif ".py" in k: @@ -1601,6 +1620,13 @@ def remove_full_paths(d: dict) -> dict: else: general[k] = v + # Merge _defaults into topp so summary shows custom defaults + user overrides + defaults = params.get("_defaults", {}) + for tool_name, default_vals in defaults.items(): + if tool_name not in topp: + topp[tool_name] = {} + topp[tool_name] = {**default_vals, **topp.get(tool_name, {})} + markdown = [] def dict_to_markdown(d: dict): diff --git a/tests/test_parameter_defaults.py b/tests/test_parameter_defaults.py new file mode 100644 index 00000000..9ccd679e --- /dev/null +++ b/tests/test_parameter_defaults.py @@ -0,0 +1,363 @@ +""" +Tests for get_merged_params() and the refactored get_topp_parameters(). + +This module verifies the three-layer parameter merge: + ini defaults < _defaults < user overrides +""" +import os +import sys +import json +import pytest +import tempfile +from pathlib import Path +from unittest.mock import MagicMock + +# Add project root to path for imports +PROJECT_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +sys.path.append(PROJECT_ROOT) + +# Mock streamlit before importing ParameterManager so that the imported module +# uses a controllable `st.session_state` (a plain dict) instead of the real one, +# which requires a running Streamlit app context. +mock_streamlit = MagicMock() +mock_streamlit.session_state = {} + +# Temporarily replace streamlit in sys.modules so that ParameterManager's +# `import streamlit as st` picks up the mock. Restore immediately after import +# so other test files (e.g., test_gui.py AppTest) get the real streamlit. +_original_streamlit = sys.modules.get('streamlit') +sys.modules['streamlit'] = mock_streamlit + +from src.workflow.ParameterManager import ParameterManager + +if _original_streamlit is not None: + sys.modules['streamlit'] = _original_streamlit +else: + sys.modules.pop('streamlit', None) + +# Remove cached src.workflow modules that were imported with mocked streamlit so +# that AppTest (in test_gui.py) re-imports them fresh with the real package. +for _key in list(sys.modules.keys()): + if _key.startswith('src.workflow'): + sys.modules.pop(_key, None) + + +@pytest.fixture(autouse=True) +def reset_streamlit_state(): + """Reset mock streamlit session state before each test.""" + mock_streamlit.session_state.clear() + yield + + +@pytest.fixture +def temp_workflow_dir(): + """Create a temporary workflow directory for testing.""" + with tempfile.TemporaryDirectory() as tmpdir: + workflow_dir = Path(tmpdir) / "test-workflow" + workflow_dir.mkdir() + ini_dir = workflow_dir / "ini" + ini_dir.mkdir() + yield workflow_dir + + +class TestGetMergedParams: + """Tests for ParameterManager.get_merged_params().""" + + def test_returns_ini_params_when_no_json(self, temp_workflow_dir): + """ini params returned when params.json doesn't exist.""" + pm = ParameterManager(temp_workflow_dir) + ini_params = {"algorithm:param": 1.0, "algorithm:other": "hello"} + + result = pm.get_merged_params("SomeTool", ini_params=ini_params) + + assert result == {"algorithm:param": 1.0, "algorithm:other": "hello"} + + def test_defaults_override_ini(self, temp_workflow_dir): + """_defaults layer overrides ini values.""" + pm = ParameterManager(temp_workflow_dir) + params_json = { + "_defaults": {"SomeTool": {"algorithm:param": 42.0}} + } + with open(pm.params_file, "w") as f: + json.dump(params_json, f) + + result = pm.get_merged_params("SomeTool", ini_params={"algorithm:param": 1.0}) + + assert result["algorithm:param"] == 42.0 + + def test_user_overrides_defaults(self, temp_workflow_dir): + """User overrides take priority over _defaults.""" + pm = ParameterManager(temp_workflow_dir) + params_json = { + "_defaults": {"SomeTool": {"algorithm:param": 42.0}}, + "SomeTool": {"algorithm:param": 99.0} + } + with open(pm.params_file, "w") as f: + json.dump(params_json, f) + + result = pm.get_merged_params("SomeTool", ini_params={"algorithm:param": 1.0}) + + assert result["algorithm:param"] == 99.0 + + def test_full_three_layer_merge(self, temp_workflow_dir): + """All three layers merge correctly: ini < _defaults < user.""" + pm = ParameterManager(temp_workflow_dir) + params_json = { + "_defaults": { + "SomeTool": { + "algorithm:param_a": 10.0, # overrides ini + "algorithm:param_b": 20.0, # overrides ini, NOT overridden by user + } + }, + "SomeTool": { + "algorithm:param_a": 99.0, # overrides _defaults + "algorithm:param_c": 55.0, # only in user + } + } + with open(pm.params_file, "w") as f: + json.dump(params_json, f) + + ini_params = { + "algorithm:param_a": 1.0, + "algorithm:param_b": 2.0, + "algorithm:param_d": 3.0, # only in ini + } + + result = pm.get_merged_params("SomeTool", ini_params=ini_params) + + assert result["algorithm:param_a"] == 99.0 # user wins + assert result["algorithm:param_b"] == 20.0 # _defaults wins over ini + assert result["algorithm:param_c"] == 55.0 # user-only key present + assert result["algorithm:param_d"] == 3.0 # ini-only key present + + def test_no_ini_params(self, temp_workflow_dir): + """Works when ini_params is None.""" + pm = ParameterManager(temp_workflow_dir) + params_json = { + "_defaults": {"SomeTool": {"algorithm:param": 42.0}}, + "SomeTool": {"algorithm:other": 7.0} + } + with open(pm.params_file, "w") as f: + json.dump(params_json, f) + + result = pm.get_merged_params("SomeTool") + + assert result["algorithm:param"] == 42.0 + assert result["algorithm:other"] == 7.0 + + def test_different_instances_same_tool(self, temp_workflow_dir): + """Different instance names get independent _defaults and user overrides.""" + pm = ParameterManager(temp_workflow_dir) + params_json = { + "_defaults": { + "IDFilter_step1": {"score:min": 0.05}, + "IDFilter_step2": {"score:min": 0.01}, + }, + "IDFilter_step1": {"score:min": 0.001}, + } + with open(pm.params_file, "w") as f: + json.dump(params_json, f) + + result1 = pm.get_merged_params("IDFilter_step1", ini_params={"score:min": 0.5}) + result2 = pm.get_merged_params("IDFilter_step2", ini_params={"score:min": 0.5}) + + assert result1["score:min"] == 0.001 # user override for step1 + assert result2["score:min"] == 0.01 # _defaults for step2 (no user override) + + def test_empty_params_json(self, temp_workflow_dir): + """Returns empty dict when params.json is empty and no ini_params.""" + pm = ParameterManager(temp_workflow_dir) + with open(pm.params_file, "w") as f: + json.dump({}, f) + + result = pm.get_merged_params("SomeTool") + + assert result == {} + + +class TestGetToppParametersWithDefaults: + + def test_get_topp_parameters_includes_defaults(self, temp_workflow_dir): + """get_topp_parameters merges _defaults between ini and user values.""" + pm = ParameterManager(temp_workflow_dir) + params_json = { + "_defaults": {"SomeTool": {"algorithm:param": 42.0}}, + "SomeTool": {"algorithm:other": 99.0} + } + with open(pm.params_file, "w") as f: + json.dump(params_json, f) + + result = pm.get_merged_params("SomeTool", ini_params={"algorithm:param": 1.0}) + assert result["algorithm:param"] == 42.0 + assert result["algorithm:other"] == 99.0 + + +class TestDefaultsSeeding: + + def test_seed_writes_defaults_to_params_json(self, temp_workflow_dir): + """Seeding creates _defaults entry in params.json.""" + pm = ParameterManager(temp_workflow_dir) + custom_defaults = {"param_a": 10.0, "param_b": "fast"} + + # Simulate what input_TOPP seeding does + params = pm.get_parameters_from_json() + if "_defaults" not in params: + params["_defaults"] = {} + params["_defaults"]["MyTool"] = custom_defaults + with open(pm.params_file, "w") as f: + json.dump(params, f) + + # Verify + loaded = pm.get_parameters_from_json() + assert loaded["_defaults"]["MyTool"] == {"param_a": 10.0, "param_b": "fast"} + + def test_seed_is_idempotent(self, temp_workflow_dir): + """Seeding the same tool twice overwrites cleanly.""" + pm = ParameterManager(temp_workflow_dir) + + # First seed + params = {"_defaults": {"Tool": {"p1": 1.0}}, "other_key": "keep"} + with open(pm.params_file, "w") as f: + json.dump(params, f) + + # Second seed with updated defaults + params = pm.get_parameters_from_json() + params["_defaults"]["Tool"] = {"p1": 2.0} + with open(pm.params_file, "w") as f: + json.dump(params, f) + + loaded = pm.get_parameters_from_json() + assert loaded["_defaults"]["Tool"]["p1"] == 2.0 + assert loaded["other_key"] == "keep" + + def test_seed_multiple_instances(self, temp_workflow_dir): + """Different instances of the same tool get independent _defaults.""" + pm = ParameterManager(temp_workflow_dir) + params = { + "_defaults": { + "IDFilter_strict": {"score:pep": 0.01}, + "IDFilter_lenient": {"score:pep": 0.05}, + } + } + with open(pm.params_file, "w") as f: + json.dump(params, f) + + loaded = pm.get_parameters_from_json() + assert loaded["_defaults"]["IDFilter_strict"]["score:pep"] == 0.01 + assert loaded["_defaults"]["IDFilter_lenient"]["score:pep"] == 0.05 + + +try: + import pyopenms as poms + HAS_PYOPENMS = True +except ImportError: + HAS_PYOPENMS = False + + +@pytest.mark.skipif(not HAS_PYOPENMS, reason="pyopenms not available") +class TestSaveParametersWithDefaults: + + def _create_fake_ini(self, pm, tool_name, params_dict): + """Create a fake .ini file with given parameters.""" + param = poms.Param() + for key, value in params_dict.items(): + param.setValue(f"{tool_name}:1:{key}".encode(), value) + poms.ParamXMLFile().store(str(Path(pm.ini_dir, f"{tool_name}.ini")), param) + + def test_value_matching_custom_default_not_saved(self, temp_workflow_dir): + """A value equal to the _defaults entry should not be saved as a user override.""" + pm = ParameterManager(temp_workflow_dir) + + # Create a fake ini with a default value + self._create_fake_ini(pm, "Tool", {"param_a": 10.0}) + + # Pre-seed _defaults with a different value than ini + params = {"_defaults": {"Tool": {"param_a": 42.0}}} + with open(pm.params_file, "w") as f: + json.dump(params, f) + + # Session state has value matching the custom default (42.0), not the ini default (10.0) + mock_streamlit.session_state[f"{pm.topp_param_prefix}Tool:1:param_a"] = 42.0 + mock_streamlit.session_state["_topp_tool_instance_map"] = {"Tool": "Tool"} + + pm.save_parameters() + + with open(pm.params_file, "r") as f: + saved = json.load(f) + + # param_a should NOT appear under Tool (it matches the _defaults value) + assert "param_a" not in saved.get("Tool", {}) + # _defaults should still be present + assert saved["_defaults"]["Tool"]["param_a"] == 42.0 + + def test_value_different_from_custom_default_saved(self, temp_workflow_dir): + """A value different from _defaults entry should be saved as user override.""" + pm = ParameterManager(temp_workflow_dir) + + # Create a fake ini with a default value + self._create_fake_ini(pm, "Tool", {"param_a": 10.0}) + + params = {"_defaults": {"Tool": {"param_a": 42.0}}} + with open(pm.params_file, "w") as f: + json.dump(params, f) + + mock_streamlit.session_state[f"{pm.topp_param_prefix}Tool:1:param_a"] = 99.0 + mock_streamlit.session_state["_topp_tool_instance_map"] = {"Tool": "Tool"} + + pm.save_parameters() + + with open(pm.params_file, "r") as f: + saved = json.load(f) + + assert saved["Tool"]["param_a"] == 99.0 + + +class TestNonDefaultParamsSummaryDefaults: + + def test_defaults_key_excluded_from_classification(self): + """_defaults dict should not appear as a TOPP tool in the summary.""" + params = { + "_defaults": {"Tool": {"p1": 10}}, + "Tool": {"p1": 20}, + "general_param": "value" + } + # Simulate the classification logic + topp = {} + general = {} + for k, v in params.items(): + if k == "_defaults": + continue + if isinstance(v, dict): + topp[k] = v + else: + general[k] = v + + assert "_defaults" not in topp + assert "Tool" in topp + assert "general_param" in general + + def test_defaults_merged_into_summary(self): + """_defaults values should appear in summary merged with user overrides.""" + params = { + "_defaults": { + "ToolA": {"p1": 10, "p2": 20}, + "ToolB": {"p3": 30} + }, + "ToolA": {"p1": 99} + } + # Simulate the merge logic for summary + topp = {} + for k, v in params.items(): + if k == "_defaults": + continue + if isinstance(v, dict): + topp[k] = v + + defaults = params.get("_defaults", {}) + for tool_name, default_vals in defaults.items(): + if tool_name not in topp: + topp[tool_name] = {} + topp[tool_name] = {**default_vals, **topp.get(tool_name, {})} + + assert topp["ToolA"] == {"p1": 99, "p2": 20} # user override wins for p1 + assert topp["ToolB"] == {"p3": 30} # defaults-only tool appears diff --git a/tests/test_tool_instance_name.py b/tests/test_tool_instance_name.py new file mode 100644 index 00000000..cd060cac --- /dev/null +++ b/tests/test_tool_instance_name.py @@ -0,0 +1,268 @@ +""" +Tests for the tool_instance_name functionality. + +This module verifies that save_parameters correctly resolves tool instance names +to real tool names when calling create_ini, and that parameters are stored and +retrieved using the instance name as the key. +""" +import os +import sys +import json +import pytest +import tempfile +from pathlib import Path +from unittest.mock import patch, MagicMock, call + +# Add project root to path for imports +PROJECT_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +sys.path.append(PROJECT_ROOT) + +# Mock streamlit before importing ParameterManager +mock_streamlit = MagicMock() +mock_streamlit.session_state = {} + +_original_streamlit = sys.modules.get('streamlit') +sys.modules['streamlit'] = mock_streamlit + +from src.workflow.ParameterManager import ParameterManager + +if _original_streamlit is not None: + sys.modules['streamlit'] = _original_streamlit +else: + sys.modules.pop('streamlit', None) + +# Remove cached src.workflow modules +for _key in list(sys.modules.keys()): + if _key.startswith('src.workflow'): + sys.modules.pop(_key, None) + + +@pytest.fixture +def temp_workflow_dir(): + """Create a temporary workflow directory for testing.""" + with tempfile.TemporaryDirectory() as tmpdir: + workflow_dir = Path(tmpdir) / "test-workflow" + workflow_dir.mkdir() + ini_dir = workflow_dir / "ini" + ini_dir.mkdir() + yield workflow_dir + + +@pytest.fixture(autouse=True) +def reset_streamlit_state(): + """Reset mock streamlit session state before each test.""" + mock_streamlit.session_state.clear() + yield + + +class TestSaveParametersWithInstanceName: + """Tests for save_parameters correctly resolving tool instance names.""" + + def test_save_parameters_uses_real_tool_name_for_create_ini(self, temp_workflow_dir): + """Test that save_parameters resolves instance name to real tool name + before calling create_ini.""" + pm = ParameterManager(temp_workflow_dir) + + # Simulate session state with an instance name (IDFilter_step1) + # that differs from the real tool name (IDFilter) + mock_streamlit.session_state[f"{pm.topp_param_prefix}IDFilter_step1:1:score:pep"] = 0.05 + # Register instance mapping (as input_TOPP would) + mock_streamlit.session_state["_topp_tool_instance_map"] = { + "IDFilter_step1": "IDFilter" + } + + # Mock create_ini to track calls - return False (tool not found) + # to prevent further processing that requires actual ini files + with patch.object(pm, 'create_ini', return_value=False) as mock_create_ini: + pm.save_parameters() + + # Verify create_ini was called with the REAL tool name, not the instance name + mock_create_ini.assert_called_once_with("IDFilter") + + def test_save_parameters_without_instance_map_uses_tool_name_directly(self, temp_workflow_dir): + """Test that save_parameters works normally when no instance map exists + (backward compatibility).""" + pm = ParameterManager(temp_workflow_dir) + + # Simulate session state with a normal tool name (no instance mapping) + mock_streamlit.session_state[f"{pm.topp_param_prefix}IDFilter:1:score:pep"] = 0.05 + + with patch.object(pm, 'create_ini', return_value=False) as mock_create_ini: + pm.save_parameters() + + # Should use the tool name directly + mock_create_ini.assert_called_once_with("IDFilter") + + def test_save_parameters_stores_under_instance_name(self, temp_workflow_dir): + """Test that parameters are stored in JSON under the instance name, not + the real tool name.""" + pm = ParameterManager(temp_workflow_dir) + + # Create a mock ini file for IDFilter + ini_path = temp_workflow_dir / "ini" / "IDFilter.ini" + ini_path.touch() + + # Set up instance mapping + mock_streamlit.session_state["_topp_tool_instance_map"] = { + "IDFilter_step1": "IDFilter" + } + mock_streamlit.session_state[f"{pm.topp_param_prefix}IDFilter_step1:1:score:pep"] = 0.05 + + # Mock pyopenms Param and ParamXMLFile to avoid needing real ini files + mock_param = MagicMock() + mock_param.getValue.return_value = 0.01 # Different from session state value + + with patch.object(pm, 'create_ini', return_value=True), \ + patch('pyopenms.Param', return_value=mock_param), \ + patch('pyopenms.ParamXMLFile') as mock_xml: + pm.save_parameters() + + # Load saved parameters + with open(pm.params_file, "r") as f: + saved = json.load(f) + + # Parameters should be stored under the instance name + assert "IDFilter_step1" in saved + assert saved["IDFilter_step1"]["score:pep"] == 0.05 + + def test_save_parameters_multiple_instances_same_tool(self, temp_workflow_dir): + """Test that two instances of the same tool get separate parameter entries.""" + pm = ParameterManager(temp_workflow_dir) + + ini_path = temp_workflow_dir / "ini" / "IDFilter.ini" + ini_path.touch() + + # Set up two instances with different parameter values + mock_streamlit.session_state["_topp_tool_instance_map"] = { + "IDFilter_step1": "IDFilter", + "IDFilter_step2": "IDFilter", + } + mock_streamlit.session_state[f"{pm.topp_param_prefix}IDFilter_step1:1:score:pep"] = 0.01 + mock_streamlit.session_state[f"{pm.topp_param_prefix}IDFilter_step2:1:score:pep"] = 0.05 + + mock_param = MagicMock() + mock_param.getValue.return_value = 0.0 # Default differs from both + + with patch.object(pm, 'create_ini', return_value=True), \ + patch('pyopenms.Param', return_value=mock_param), \ + patch('pyopenms.ParamXMLFile'): + pm.save_parameters() + + with open(pm.params_file, "r") as f: + saved = json.load(f) + + # Both instances should have separate entries + assert "IDFilter_step1" in saved + assert "IDFilter_step2" in saved + assert saved["IDFilter_step1"]["score:pep"] == 0.01 + assert saved["IDFilter_step2"]["score:pep"] == 0.05 + + def test_save_parameters_ini_key_maps_instance_to_real_tool(self, temp_workflow_dir): + """Test that ini_key correctly maps instance name back to real tool name + for param.getValue lookup.""" + pm = ParameterManager(temp_workflow_dir) + + ini_path = temp_workflow_dir / "ini" / "IDFilter.ini" + ini_path.touch() + + mock_streamlit.session_state["_topp_tool_instance_map"] = { + "IDFilter_step1": "IDFilter" + } + mock_streamlit.session_state[f"{pm.topp_param_prefix}IDFilter_step1:1:score:pep"] = 0.05 + + mock_param = MagicMock() + mock_param.getValue.return_value = 0.01 + + with patch.object(pm, 'create_ini', return_value=True), \ + patch('pyopenms.Param', return_value=mock_param), \ + patch('pyopenms.ParamXMLFile'): + pm.save_parameters() + + # Verify that param.getValue was called with the REAL tool name key + # (IDFilter:1:score:pep), not the instance name key (IDFilter_step1:1:score:pep) + mock_param.getValue.assert_called_with(b"IDFilter:1:score:pep") + + def test_save_parameters_display_keys_skipped_with_instance_name(self, temp_workflow_dir): + """Test that _display keys are still skipped when using instance names.""" + pm = ParameterManager(temp_workflow_dir) + + ini_path = temp_workflow_dir / "ini" / "IDFilter.ini" + ini_path.touch() + + mock_streamlit.session_state["_topp_tool_instance_map"] = { + "IDFilter_step1": "IDFilter" + } + mock_streamlit.session_state[f"{pm.topp_param_prefix}IDFilter_step1:1:score:pep"] = 0.05 + mock_streamlit.session_state[f"{pm.topp_param_prefix}IDFilter_step1:1:score:pep_display"] = ["0.05"] + + mock_param = MagicMock() + mock_param.getValue.return_value = 0.01 + + with patch.object(pm, 'create_ini', return_value=True), \ + patch('pyopenms.Param', return_value=mock_param), \ + patch('pyopenms.ParamXMLFile'): + pm.save_parameters() + + with open(pm.params_file, "r") as f: + saved = json.load(f) + + # _display key should not be stored + assert "score:pep_display" not in saved.get("IDFilter_step1", {}) + assert "score:pep" in saved.get("IDFilter_step1", {}) + + +class TestGetToppParametersWithInstanceName: + """Tests for get_topp_parameters with tool_instance_name.""" + + def test_get_topp_parameters_with_instance_name(self, temp_workflow_dir): + """Test that get_topp_parameters uses instance name for JSON lookup.""" + pm = ParameterManager(temp_workflow_dir) + + # Create params.json with instance-keyed parameters + params = { + "IDFilter_step1": { + "score:pep": 0.05 + } + } + with open(pm.params_file, "w") as f: + json.dump(params, f) + + # Create a mock ini file + ini_path = temp_workflow_dir / "ini" / "IDFilter.ini" + ini_path.touch() + + mock_param = MagicMock() + mock_param.keys.return_value = [b"IDFilter:1:score:pep"] + mock_param.getValue.return_value = 0.01 # default + + with patch('pyopenms.Param', return_value=mock_param), \ + patch('pyopenms.ParamXMLFile'): + result = pm.get_topp_parameters("IDFilter", tool_instance_name="IDFilter_step1") + + # Should return the instance-specific value + assert result["score:pep"] == 0.05 + + def test_get_topp_parameters_without_instance_name_backward_compat(self, temp_workflow_dir): + """Test that get_topp_parameters works without instance name (backward compat).""" + pm = ParameterManager(temp_workflow_dir) + + params = { + "IDFilter": { + "score:pep": 0.05 + } + } + with open(pm.params_file, "w") as f: + json.dump(params, f) + + ini_path = temp_workflow_dir / "ini" / "IDFilter.ini" + ini_path.touch() + + mock_param = MagicMock() + mock_param.keys.return_value = [b"IDFilter:1:score:pep"] + mock_param.getValue.return_value = 0.01 + + with patch('pyopenms.Param', return_value=mock_param), \ + patch('pyopenms.ParamXMLFile'): + result = pm.get_topp_parameters("IDFilter") + + assert result["score:pep"] == 0.05 From 0f02ecf317da2b1d94b7a9fd9f0b44cceb27871c Mon Sep 17 00:00:00 2001 From: JKVISION1101 <120628409+JKVISION1101@users.noreply.github.com> Date: Thu, 16 Jul 2026 20:40:49 +0900 Subject: [PATCH 20/29] InputTOPP: add reactive option and split impl/fragment for conditional UI (#398) * InputTOPP reactive parameter * docs: document reactive/conditional-UI pattern for workflow widgets Add developer + agent-facing documentation for the reactive=True option on input_widget, select_input_file and input_TOPP, which un-isolates a parameter widget from its st.fragment so a change reruns configure() and enables conditional UI. - docs/toppframework.py: "Reactive parameters (conditional UI)" block with a combined input_widget + input_TOPP example in the Parameter Input section - .claude/skills/create-workflow.md: UI-widget note, Conditional UI section, checklist item - CLAUDE.md: reactive note in the TOPP Workflows section - docs/build_app.md: pointer to the framework doc --------- Co-authored-by: jkvision1101 Co-authored-by: Tom David Mueller --- .claude/skills/create-workflow.md | 22 +++++++++++++ CLAUDE.md | 2 ++ docs/build_app.md | 2 ++ docs/toppframework.py | 32 ++++++++++++++++++ src/workflow/StreamlitUI.py | 55 ++++++++++++++++++++++++++++++- 5 files changed, 112 insertions(+), 1 deletion(-) diff --git a/.claude/skills/create-workflow.md b/.claude/skills/create-workflow.md index 07856db2..38e981dc 100644 --- a/.claude/skills/create-workflow.md +++ b/.claude/skills/create-workflow.md @@ -120,10 +120,31 @@ The 4 pages call these methods respectively: - `self.ui.input_TOPP("ToolName", custom_defaults={})` — auto-generated TOPP parameter UI - `self.ui.input_python("script_name")` — auto-generated Python tool parameter UI - `self.ui.input_widget(key, default, label)` — single custom widget +- `select_input_file`, `input_TOPP` and `input_widget` accept `reactive=True` to rerun `configure()` when the widget changes (for conditional UI — see below) ### Logging - `self.logger.log("message")` — log progress during execution +## Conditional UI (reactive) + +Parameter widgets are isolated in an `st.fragment` by default, so a change reruns only that +widget and can't show/hide other widgets. Pass `reactive=True` to render the widget in the +parent scope instead — a change then reruns `configure()`. Read the live value from +`st.session_state` (not `self.params`, which is stale within the rerun) using +`self.parameter_manager.param_prefix` for custom-widget keys or `topp_param_prefix` for TOPP +keys of the form `":1:"`. + +```python +@st.fragment +def configure(self) -> None: + pm = self.parameter_manager + # changing the tool's `type` selectbox reruns configure() + self.ui.input_TOPP("IsobaricAnalyzer", reactive=True) + iso_type = st.session_state.get(f"{pm.topp_param_prefix}IsobaricAnalyzer:1:type", "") + if iso_type.startswith("tmt"): + self.ui.input_widget("tmt-channels", 10, "TMT channels", widget_type="number") +``` + ## Reference Files - Example workflow: `src/Workflow.py` @@ -144,6 +165,7 @@ The 4 pages call these methods respectively: - [ ] `__init__` calls `super().__init__("Name", st.session_state["workspace"])` - [ ] `upload()`, `configure()`, `execution()`, `results()` implemented - [ ] `@st.fragment` on `configure()` and `results()` +- [ ] `reactive=True` on any widget whose value controls other widgets' visibility - [ ] 4 content pages created in `content/` - [ ] All 4 pages registered as a group in `app.py` - [ ] Default parameters added to `default-parameters.json` if needed diff --git a/CLAUDE.md b/CLAUDE.md index 3eeb1927..e0e0a222 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -89,6 +89,8 @@ Each workflow gets 4 content pages (upload, configure, run, results) that call ` Decorate `configure()` and `results()` with `@st.fragment` for partial reruns. +For conditional UI (a widget that shows/hides other widgets), pass `reactive=True` to `input_widget`, `select_input_file`, or `input_TOPP` so a change reruns the parent `configure()` instead of only its isolated fragment. Read the changed value from `st.session_state` (not `self.params`, which is stale within the rerun) via `parameter_manager.param_prefix` for custom-widget keys or `topp_param_prefix` for TOPP keys (`":1:"`). + ### Python Tools Custom scripts in `src/python-tools/` define a `DEFAULTS` list for auto-generated UI: diff --git a/docs/build_app.md b/docs/build_app.md index 9a345745..08b7d245 100644 --- a/docs/build_app.md +++ b/docs/build_app.md @@ -133,3 +133,5 @@ Take a look at the example pages `Simple Workflow` or `Workflow with mzML files` ### Complex workflow using TOPP tools This template app features a module in `src/workflow` that allows for complex and long workflows to be built very efficiently. Check out the `TOPP Workflow Framework` page for more information (on the *sidebar*). + +For building **conditional parameter UI** (widgets that appear or disappear based on another parameter's value), see the *Reactive parameters* subsection of the `TOPP Workflow Framework` page's *Parameter Input* section. diff --git a/docs/toppframework.py b/docs/toppframework.py index 592fc5f7..a91b2be7 100644 --- a/docs/toppframework.py +++ b/docs/toppframework.py @@ -132,6 +132,38 @@ def content(): "💡 Access parameter widget values by their **key** in the `self.params` object, e.g. `self.params['mzML-files']` will give all selected mzML files." ) + st.markdown( + """ +### Reactive parameters (conditional UI) + +By default every parameter widget is rendered inside an isolated `st.fragment`: changing it reruns only that widget, which keeps large parameter sections fast but also means a widget **cannot** show or hide other widgets. Pass **`reactive=True`** to `self.ui.input_widget`, `self.ui.select_input_file` or `self.ui.input_TOPP` to render the widget directly in the parent `configure()` scope instead — a change then reruns `configure()`, so you can conditionally render downstream widgets based on the current value. + +Read the current value from `st.session_state` (**not** `self.params`, which is only loaded once per run and is stale within the same rerun) using the parameter-manager prefixes: `param_prefix` for `input_widget` / `select_input_file` keys, and `topp_param_prefix` for TOPP keys of the form `":1:"` (the `` matches the tool's ini, as shown for each widget in the panel). +""" + ) + st.code( + '''@st.fragment +def configure(self) -> None: + pm = self.parameter_manager + + # A custom widget that reveals more widgets when checked + self.ui.input_widget( + "advanced-mode", False, "Advanced mode", + widget_type="checkbox", reactive=True, + ) + if st.session_state.get(f"{pm.param_prefix}advanced-mode", False): + self.ui.input_widget("threads", 4, "Threads", widget_type="number") + + # A TOPP parameter driving conditional UI (e.g. TMT type -> channel count) + self.ui.input_TOPP("IsobaricAnalyzer", reactive=True) + iso_type = st.session_state.get(f"{pm.topp_param_prefix}IsobaricAnalyzer:1:type", "") + if iso_type.startswith("tmt"): + self.ui.input_widget("tmt-channels", 10, "TMT channels", widget_type="number")''' + ) + st.info( + "💡 `reactive=True` is opt-in per widget and defaults to `False`. On `input_TOPP` it un-isolates the whole tool panel, so **any** parameter change reruns `configure()` — leave it off unless another widget's visibility depends on a value in that panel." + ) + with st.expander("**Code documentation**", expanded=True): st.help(StreamlitUI.input_widget) st.help(StreamlitUI.select_input_file) diff --git a/src/workflow/StreamlitUI.py b/src/workflow/StreamlitUI.py index 9c24dc2c..12601343 100644 --- a/src/workflow/StreamlitUI.py +++ b/src/workflow/StreamlitUI.py @@ -819,7 +819,6 @@ def format_files(input: Any) -> List[str]: self.parameter_manager.save_parameters() - @st.fragment def input_TOPP( self, topp_tool_name: str, @@ -831,6 +830,7 @@ def input_TOPP( display_subsection_tabs: bool = False, custom_defaults: dict = {}, tool_instance_name: str = None, + reactive: bool = False, ) -> None: """ Generates input widgets for TOPP tool parameters dynamically based on the tool's @@ -842,6 +842,8 @@ def input_TOPP( num_cols (int, optional): Number of columns to use for the layout. Defaults to 3. exclude_parameters (List[str], optional): List of parameter names to exclude from the widget. Defaults to an empty list. include_parameters (List[str], optional): List of parameter names to include in the widget. Defaults to an empty list. + flag_parameters (List[str], optional): List of parameter names that should + be treated as no-value CLI flags during command construction. display_tool_name (bool, optional): Whether to display the TOPP tool name. Defaults to True. display_subsections (bool, optional): Whether to split parameters into subsections based on the prefix. Defaults to True. display_subsection_tabs (bool, optional): Whether to display main subsections in separate tabs (if more than one main section). Defaults to False. @@ -852,7 +854,58 @@ def input_TOPP( defaults to topp_tool_name. The instance name is used for session state keys and parameter storage, while topp_tool_name is used for the actual tool executable and ini file creation. + reactive (bool, optional): If True, widget changes trigger the parent + section to re-render, enabling conditional UI based on this widget's + value. Use when downstream UI depends on a parameter value (e.g., + TMT type driving channel count). Default is False. """ + if reactive: + self._input_TOPP_impl( + topp_tool_name, num_cols, exclude_parameters, include_parameters, + flag_parameters, display_tool_name, display_subsections, + display_subsection_tabs, custom_defaults, tool_instance_name, + ) + else: + self._input_TOPP_fragmented( + topp_tool_name, num_cols, exclude_parameters, include_parameters, + flag_parameters, display_tool_name, display_subsections, + display_subsection_tabs, custom_defaults, tool_instance_name, + ) + + @st.fragment + def _input_TOPP_fragmented( + self, + topp_tool_name: str, + num_cols: int = 4, + exclude_parameters: List[str] = [], + include_parameters: List[str] = [], + flag_parameters: List[str] = [], + display_tool_name: bool = True, + display_subsections: bool = True, + display_subsection_tabs: bool = False, + custom_defaults: dict = {}, + tool_instance_name: str = None, + ) -> None: + self._input_TOPP_impl( + topp_tool_name, num_cols, exclude_parameters, include_parameters, + flag_parameters, display_tool_name, display_subsections, + display_subsection_tabs, custom_defaults, tool_instance_name, + ) + + def _input_TOPP_impl( + self, + topp_tool_name: str, + num_cols: int = 4, + exclude_parameters: List[str] = [], + include_parameters: List[str] = [], + flag_parameters: List[str] = [], + display_tool_name: bool = True, + display_subsections: bool = True, + display_subsection_tabs: bool = False, + custom_defaults: dict = {}, + tool_instance_name: str = None, + ) -> None: + """Internal implementation of input_TOPP - contains all the widget logic.""" # Default instance name to the tool name when not provided if tool_instance_name is None: tool_instance_name = topp_tool_name From ad9bdb2e2a8dceac4fa969dc6a7c416a6720feac Mon Sep 17 00:00:00 2001 From: hjn0415 Date: Thu, 16 Jul 2026 21:34:18 +0900 Subject: [PATCH 21/29] Add flag parameter support for TOPP tools (#397) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Add support for boolean CLI flag parameters in TOPP tools Allows input_TOPP() to designate parameters as flags (present/absent) instead of key-value pairs, persisted to params.json and session_state so run_topp() can correctly build the command line. * Add unit tests for TOPP flag parameter support Exercise run_topp()'s flag-aware command construction and the flag-source contract written by input_TOPP(). Working-behaviour tests cover: bare flag emission for truthy bool/"true" values, omission for falsy/"false", empty and None regular params skipped, 0/0.0 preserved, multiline splitting, custom_params scalar/list/flag handling, and params.json-vs-session_state precedence. A TestKnownCodeRabbitBugs group asserts the correct behaviour for the two CodeRabbit findings (per-tool flag fallback; list expansion / empty-list skipping). Those tests fail against the current code, documenting the defects until they are fixed. * ci: run continuous-integration on pull_request The continuous-integration workflow (which runs `pytest test_gui.py tests/`) was triggered on push only, so pull requests from forks never executed the tests/ suite as a check. Add the pull_request trigger so the test job gates PRs as well. * Fix flag-parameter command construction and green the test suite Address both CodeRabbit findings on run_topp() (CommandExecutor.py): - Per-tool flag fallback: consult session_state per params_key instead of only when the whole _flag_params map is empty, so one tool's saved flags no longer shadow another tool's session flags. - List handling in the non-flag branches: skip empty lists in both the merged and custom loops, and expand non-empty merged lists into separate args instead of stringifying them as "['a', 'b']". Move the four tests that pinned these findings out of TestKnownCodeRabbitBugs into the behaviour classes they now guard (they pass with the fixes applied). Also fix a pre-existing, unrelated CI failure in test_queue_manager_cancel.py: test_stopped_status_is_mapped_in_get_job_info compared JobStatus enum members by identity via a re-import, which mismatches once other test modules pop src.workflow.* from sys.modules. Compare .name instead — reload-proof and semantically equivalent. --------- Co-authored-by: Tom David Mueller --- .github/workflows/ci.yml | 2 +- src/workflow/CommandExecutor.py | 56 +++-- src/workflow/StreamlitUI.py | 13 ++ tests/test_queue_manager_cancel.py | 4 +- tests/test_topp_flag_parameters.py | 346 +++++++++++++++++++++++++++++ 5 files changed, 402 insertions(+), 19 deletions(-) create mode 100644 tests/test_topp_flag_parameters.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5ee0902b..34b5359d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,6 +1,6 @@ name: continuous-integration -on: [push] +on: [push, pull_request] jobs: test: diff --git a/src/workflow/CommandExecutor.py b/src/workflow/CommandExecutor.py index 11bb1486..6479cb02 100644 --- a/src/workflow/CommandExecutor.py +++ b/src/workflow/CommandExecutor.py @@ -268,6 +268,15 @@ def run_topp(self, tool: str, input_output: dict, custom_params: dict = {}, tool # Load merged parameters (_defaults + user overrides) for this tool instance merged_params = self.parameter_manager.get_merged_params(params_key) + + # Load flag parameter names: params.json takes priority (survives session restart), + # session_state is the live fallback during the current session. + flag_map = self.parameter_manager.get_parameters_from_json().get("_flag_params", {}) + flag_list = flag_map.get(params_key) + if flag_list is None: + flag_list = st.session_state.get("_topp_flag_params", {}).get(params_key, []) + flag_params: set = set(flag_list) + # Construct commands for each process for i in range(n_processes): command = [tool] @@ -288,25 +297,42 @@ def run_topp(self, tool: str, input_output: dict, custom_params: dict = {}, tool command += [value[i]] # Add merged TOPP tool parameters (_defaults + user overrides) for k, v in merged_params.items(): - command += [f"-{k}"] - # Skip only empty strings (pass flag with no value) - # Note: 0 and 0.0 are valid values, so use explicit check - if v != "" and v is not None: - if isinstance(v, str) and "\n" in v: - command += v.split("\n") + if k in flag_params: + # CLI flag: include "-k" only when truthy, omit when false + if isinstance(v, str): + is_enabled = v.lower() == "true" else: - command += [str(v)] + is_enabled = bool(v) + if is_enabled: + command += [f"-{k}"] + continue + # Regular parameter: skip empty/None/empty-list, append value otherwise + if v == "" or v is None or (isinstance(v, list) and not v): + continue + command += [f"-{k}"] + if isinstance(v, str) and "\n" in v: + command += v.split("\n") + elif isinstance(v, list): + command += [str(x) for x in v] + else: + command += [str(v)] # Add custom parameters for k, v in custom_params.items(): - command += [f"-{k}"] - - # Skip only empty strings (pass flag with no value) - # Note: 0 and 0.0 are valid values, so use explicit check - if v != "" and v is not None: - if isinstance(v, list): - command += [str(x) for x in v] + if k in flag_params: + if isinstance(v, str): + is_enabled = v.lower() == "true" else: - command += [str(v)] + is_enabled = bool(v) + if is_enabled: + command += [f"-{k}"] + continue + if v == "" or v is None or (isinstance(v, list) and not v): + continue + command += [f"-{k}"] + if isinstance(v, list): + command += [str(x) for x in v] + else: + command += [str(v)] # Add threads parameter for TOPP tools command += ["-threads", str(threads_per_command)] commands.append(command) diff --git a/src/workflow/StreamlitUI.py b/src/workflow/StreamlitUI.py index 12601343..d426e3a9 100644 --- a/src/workflow/StreamlitUI.py +++ b/src/workflow/StreamlitUI.py @@ -825,6 +825,7 @@ def input_TOPP( num_cols: int = 4, exclude_parameters: List[str] = [], include_parameters: List[str] = [], + flag_parameters: List[str] = [], display_tool_name: bool = True, display_subsections: bool = True, display_subsection_tabs: bool = False, @@ -915,6 +916,18 @@ def _input_TOPP_impl( st.session_state["_topp_tool_instance_map"] = {} st.session_state["_topp_tool_instance_map"][tool_instance_name] = topp_tool_name + # Persist flag_parameters to session_state and params.json so run_topp + # can skip appending a value for these boolean CLI flags. + if "_topp_flag_params" not in st.session_state: + st.session_state["_topp_flag_params"] = {} + st.session_state["_topp_flag_params"][tool_instance_name] = list(flag_parameters) + _fp = self.parameter_manager.get_parameters_from_json() + if "_flag_params" not in _fp: + _fp["_flag_params"] = {} + _fp["_flag_params"][tool_instance_name] = list(flag_parameters) + with open(self.parameter_manager.params_file, "w", encoding="utf-8") as _f: + json.dump(_fp, _f, indent=4) + if not display_subsections: display_subsection_tabs = False if display_subsection_tabs: diff --git a/tests/test_queue_manager_cancel.py b/tests/test_queue_manager_cancel.py index 0f877082..c8ef44aa 100644 --- a/tests/test_queue_manager_cancel.py +++ b/tests/test_queue_manager_cancel.py @@ -153,9 +153,7 @@ def test_stopped_status_is_mapped_in_get_job_info(monkeypatch): info = qm.get_job_info("stopped-job") assert info is not None - assert info.status == __import__( - "src.workflow.QueueManager", fromlist=["JobStatus"] - ).JobStatus.CANCELED, ( + assert info.status.name == "CANCELED", ( "RQ 'stopped' status should be reported as CANCELED to the UI; " "otherwise stopped jobs appear stuck in 'queued'." ) diff --git a/tests/test_topp_flag_parameters.py b/tests/test_topp_flag_parameters.py new file mode 100644 index 00000000..627f50ee --- /dev/null +++ b/tests/test_topp_flag_parameters.py @@ -0,0 +1,346 @@ +""" +Unit tests for PR #397 — flag parameter support for TOPP tools. + +PR #397 lets a caller mark certain TOPP parameters as CLI *flags*: parameters +passed by presence only (e.g. ``-force``), without a trailing value. The flag +names are persisted per tool instance by ``input_TOPP()`` into both +``st.session_state["_topp_flag_params"]`` and ``params.json["_flag_params"]``, +and consumed by ``run_topp()`` in ``src/workflow/CommandExecutor.py`` when it +builds the command line. + +These tests exercise ``run_topp()`` — the consumer that turns the persisted flag +definitions and merged parameters into an actual command. Driving ``run_topp()`` +also validates the persistence *contract* (the exact ``_flag_params`` / +``_topp_flag_params`` shapes that ``input_TOPP()`` writes), which is where the two +halves of the feature meet. + +The suite covers the working behaviour of the feature and also guards the two +issues CodeRabbit flagged during review, which are now fixed in ``run_topp()``: + - Finding 1 (per-tool flag fallback): + https://github.com/OpenMS/streamlit-template/pull/397#discussion_r3585023551 + - Finding 2 (list expansion / empty-list skipping): + https://github.com/OpenMS/streamlit-template/pull/397#discussion_r3585023558 +The tests that pin those findings carry a "CodeRabbit finding N" note in their +docstrings and live alongside the related behaviour they protect. +""" +import os +import sys +import json +import tempfile +from pathlib import Path +from unittest.mock import MagicMock + +import pytest + +# Add project root to path for imports +PROJECT_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +sys.path.append(PROJECT_ROOT) + +# --------------------------------------------------------------------------- +# Import the modules under test with `streamlit` (and `pyopenms`) mocked at the +# sys.modules level. Both CommandExecutor and ParameterManager do +# `import streamlit as st` at module top (ParameterManager also +# `import pyopenms as poms`). run_topp() only needs `st.session_state` to behave +# like a plain dict and never touches pyopenms, so lightweight mocks keep the +# test runnable without those heavy deps installed while still exercising the +# real command-construction logic. Mirrors tests/test_tool_instance_name.py. +# --------------------------------------------------------------------------- +mock_streamlit = MagicMock() +mock_streamlit.session_state = {} + +_original_streamlit = sys.modules.get("streamlit") +_original_pyopenms = sys.modules.get("pyopenms") +sys.modules["streamlit"] = mock_streamlit +if _original_pyopenms is None: + sys.modules["pyopenms"] = MagicMock() + +from src.workflow.ParameterManager import ParameterManager +from src.workflow.CommandExecutor import CommandExecutor + +# Restore the original modules so other test files import the real ones. The +# classes imported above keep their module-level `st`/`poms` bound to the mocks. +if _original_streamlit is not None: + sys.modules["streamlit"] = _original_streamlit +else: + sys.modules.pop("streamlit", None) +if _original_pyopenms is None: + sys.modules.pop("pyopenms", None) + +for _key in list(sys.modules.keys()): + if _key.startswith("src.workflow"): + sys.modules.pop(_key, None) + + +TOOL = "FeatureFinderMetabo" + + +@pytest.fixture(autouse=True) +def reset_session_state(): + """Give each test a fresh, empty mocked session_state.""" + mock_streamlit.session_state = {} + yield + mock_streamlit.session_state = {} + + +def build_command( + params_json=None, + session_state=None, + *, + tool=TOOL, + input_output=None, + custom_params=None, + tool_instance_name=None, +): + """ + Invoke ``run_topp()`` with the supplied ``params.json`` content and + ``session_state``, and return the single command list it builds. + + ``run_command`` / ``run_multiple_commands`` are stubbed so nothing is + executed; the built command is captured from the ``run_command`` mock. + ``max_threads`` is pinned to 1 so the trailing ``-threads`` argument is + deterministic. + """ + if input_output is None: + input_output = {"in": ["input.mzML"], "out": ["output.featureXML"]} + + params_json = dict(params_json or {}) + params_json.setdefault("max_threads", 1) + + with tempfile.TemporaryDirectory() as tmpdir: + workflow_dir = Path(tmpdir) + pm = ParameterManager(workflow_dir) + with open(pm.params_file, "w", encoding="utf-8") as f: + json.dump(params_json, f) + + mock_streamlit.session_state = dict(session_state or {}) + + executor = CommandExecutor(workflow_dir, MagicMock(), pm) + executor.run_command = MagicMock(return_value=True) + executor.run_multiple_commands = MagicMock(return_value=True) + + executor.run_topp( + tool, + input_output, + custom_params=custom_params or {}, + tool_instance_name=tool_instance_name or tool, + ) + + assert executor.run_command.call_count == 1, ( + "expected exactly one single-process command, got " + f"{executor.run_command.call_count}" + ) + return executor.run_command.call_args.args[0] + + +# --------------------------- assertion helpers ----------------------------- + +def has_flag(cmd, name): + """True if ``-name`` appears anywhere in the command.""" + return f"-{name}" in cmd + + +def token_after(cmd, name): + """The single token immediately following ``-name`` (or None if it is last).""" + idx = cmd.index(f"-{name}") + return cmd[idx + 1] if idx + 1 < len(cmd) else None + + +def values_after(cmd, name): + """All value tokens following ``-name`` up to the next ``-flag`` token.""" + idx = cmd.index(f"-{name}") + vals = [] + for tok in cmd[idx + 1:]: + if tok.startswith("-"): + break + vals.append(tok) + return vals + + +def is_bare_flag(cmd, name): + """True if ``-name`` is present with no value (next token is another flag).""" + if not has_flag(cmd, name): + return False + nxt = token_after(cmd, name) + return nxt is None or nxt.startswith("-") + + +# ============================ working behaviour ============================ + + +class TestCommandSkeleton: + def test_input_output_files_prefixed(self): + cmd = build_command() + assert cmd[0] == TOOL + assert cmd[1:5] == ["-in", "input.mzML", "-out", "output.featureXML"] + # threads pinned to 1 and always appended last + assert cmd[-2:] == ["-threads", "1"] + + def test_collected_files_passed_as_single_list(self): + # A [["a", "b"]] entry is expanded in place after its -key. + cmd = build_command(input_output={"in": [["a.mzML", "b.mzML"]], "out": ["c.featureXML"]}) + assert cmd[1:4] == ["-in", "a.mzML", "b.mzML"] + + +class TestFlagParameters: + """Flags emit a bare ``-key`` when enabled and nothing when disabled.""" + + def test_flag_true_bool_emits_bare_flag(self): + cmd = build_command( + {"_flag_params": {TOOL: ["force"]}, TOOL: {"force": True}} + ) + assert is_bare_flag(cmd, "force") + assert "True" not in cmd + + def test_flag_string_true_emits_bare_flag(self): + cmd = build_command( + {"_flag_params": {TOOL: ["force"]}, TOOL: {"force": "true"}} + ) + assert is_bare_flag(cmd, "force") + assert "true" not in cmd + + def test_flag_string_true_is_case_insensitive(self): + cmd = build_command( + {"_flag_params": {TOOL: ["force"]}, TOOL: {"force": "True"}} + ) + assert is_bare_flag(cmd, "force") + + def test_flag_false_bool_omitted(self): + cmd = build_command( + {"_flag_params": {TOOL: ["force"]}, TOOL: {"force": False}} + ) + assert not has_flag(cmd, "force") + + def test_flag_string_false_omitted(self): + cmd = build_command( + {"_flag_params": {TOOL: ["force"]}, TOOL: {"force": "false"}} + ) + assert not has_flag(cmd, "force") + + +class TestRegularParameters: + """Non-flag merged parameters keep the existing value-appending behaviour.""" + + def test_empty_string_skipped(self): + cmd = build_command({TOOL: {"opt": ""}}) + assert not has_flag(cmd, "opt") + + def test_none_skipped(self): + cmd = build_command({TOOL: {"opt": None}}) + assert not has_flag(cmd, "opt") + + def test_zero_is_preserved(self): + # 0 and 0.0 are valid values, not "empty" — they must be passed through. + cmd = build_command({TOOL: {"min_int": 0, "min_float": 0.0}}) + assert values_after(cmd, "min_int") == ["0"] + assert values_after(cmd, "min_float") == ["0.0"] + + def test_scalar_value_appended(self): + cmd = build_command({TOOL: {"mz_tolerance": 10.5}}) + assert values_after(cmd, "mz_tolerance") == ["10.5"] + + def test_multiline_string_split_into_args(self): + cmd = build_command({TOOL: {"seq": "ALPHA\nBETA\nGAMMA"}}) + assert values_after(cmd, "seq") == ["ALPHA", "BETA", "GAMMA"] + + def test_merged_list_param_expanded(self): + """ + CodeRabbit finding 2 (fixed): a list-valued merged parameter expands into + separate CLI args, not its Python ``str()`` (e.g. ``"['a', 'b']"``). + https://github.com/OpenMS/streamlit-template/pull/397#discussion_r3585023558 + """ + cmd = build_command({TOOL: {"ids": ["a", "b"]}}) + assert values_after(cmd, "ids") == ["a", "b"] + + def test_merged_empty_list_param_skipped(self): + """ + CodeRabbit finding 2 (fixed): an empty-list merged parameter is omitted + entirely rather than emitting a ``-key`` with no usable value. + https://github.com/OpenMS/streamlit-template/pull/397#discussion_r3585023558 + """ + cmd = build_command({TOOL: {"ids": []}}) + assert not has_flag(cmd, "ids") + + +class TestCustomParameters: + """custom_params share the flag set and expand non-empty lists.""" + + def test_custom_flag_truthy_bare(self): + cmd = build_command( + {"_flag_params": {TOOL: ["force"]}}, + custom_params={"force": True}, + ) + assert is_bare_flag(cmd, "force") + + def test_custom_flag_false_omitted(self): + cmd = build_command( + {"_flag_params": {TOOL: ["force"]}}, + custom_params={"force": False}, + ) + assert not has_flag(cmd, "force") + + def test_custom_scalar_value(self): + cmd = build_command(custom_params={"extra": 5}) + assert values_after(cmd, "extra") == ["5"] + + def test_custom_nonempty_list_expanded(self): + cmd = build_command(custom_params={"ids": ["a", "b", "c"]}) + assert values_after(cmd, "ids") == ["a", "b", "c"] + + def test_custom_empty_string_skipped(self): + cmd = build_command(custom_params={"opt": ""}) + assert not has_flag(cmd, "opt") + + def test_custom_empty_list_param_skipped(self): + """ + CodeRabbit finding 2 (fixed): an empty-list custom parameter is omitted + rather than emitting a bare ``-key`` with no value. + https://github.com/OpenMS/streamlit-template/pull/397#discussion_r3585023558 + """ + cmd = build_command(custom_params={"ids": []}) + assert not has_flag(cmd, "ids") + + +class TestFlagSourceContract: + """Where run_topp() reads the flag definitions from.""" + + def test_flag_params_loaded_from_params_json(self): + # Survives a session restart: only params.json carries the flag list. + cmd = build_command( + {"_flag_params": {TOOL: ["force"]}, TOOL: {"force": True}}, + session_state={}, + ) + assert is_bare_flag(cmd, "force") + + def test_fallback_to_session_state_when_json_has_no_flags(self): + # params.json has no _flag_params at all -> live session_state is used. + cmd = build_command( + {TOOL: {"force": True}}, + session_state={"_topp_flag_params": {TOOL: ["force"]}}, + ) + assert is_bare_flag(cmd, "force") + + def test_params_json_takes_priority_over_session_state(self): + # params.json says "force" is a flag; session_state disagrees (empty). + # params.json wins, so force is treated as a flag (bare, no value). + cmd = build_command( + {"_flag_params": {TOOL: ["force"]}, TOOL: {"force": True}}, + session_state={"_topp_flag_params": {TOOL: []}}, + ) + assert is_bare_flag(cmd, "force") + assert "True" not in cmd + + def test_flag_fallback_uses_current_tool_when_other_tool_has_flags(self): + """ + CodeRabbit finding 1 (fixed): when params.json._flag_params holds an entry + for a DIFFERENT tool, the current tool's flags must still be read from the + session_state fallback. Previously the global ``if not flag_map`` check + skipped the fallback whenever any tool had flags, so the current tool's + flag was treated as a regular parameter and emitted as ``-force True``. + https://github.com/OpenMS/streamlit-template/pull/397#discussion_r3585023551 + """ + cmd = build_command( + {"_flag_params": {"OtherTool": ["some_flag"]}, TOOL: {"force": True}}, + session_state={"_topp_flag_params": {TOOL: ["force"]}}, + ) + assert is_bare_flag(cmd, "force") + assert "True" not in cmd From 3e42f00550f522cd6bc6b408db309371d08f2112 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=20David=20M=C3=BCller?= <57191390+t0mdavid-m@users.noreply.github.com> Date: Fri, 28 Aug 2026 13:57:05 +0200 Subject: [PATCH 22/29] Pin cython below 3.3 so the full image builds again (#400) Both full-variant image builds have failed since 2026-08-25: AttributeError: module 'Cython.Compiler.Nodes' has no attribute 'CConstTypeNode'. Did you mean: 'CBaseTypeNode'? ... buildx failed: process "mamba run -n streamlit-env /bin/bash -c make -j4 pyopenms" did not complete successfully: exit code: 2 Line 65 of both Dockerfiles pins autowrap but not cython. cython 3.3.0 was released on 2026-08-22 and removed Cython.Compiler.Nodes.CConstTypeNode and CConstOrVolatileTypeNode, which autowrap 0.24's PXDParser._check_type_constness calls while generating the pyOpenMS bindings. The build resolved cython-3.2.9 on 2026-08-21 and succeeded; it resolved cython-3.3.0 on 2026-08-25 and did not. 3.2.9 is the last 3.2.x on PyPI, so `<3.3` restores exactly the resolution that was green. The lower bound matches what pyOpenMS asserts for itself during the build (`Cython>=3.1.0`). Bumping autowrap instead does not work: the newest release is 0.27.0, uploaded 2026-01-14, seven months before cython 3.3.0, and it declares only `Cython >= 3.0` - an open upper bound with no knowledge of the removed AST nodes. No published autowrap can be assumed to handle them, and a build that happened to succeed would silently change the generated bindings. Claude-Session: https://claude.ai/code/session_018tuhTmRVKxCXo5gJSJU8e8 Co-authored-by: Claude Opus 5 (1M context) --- Dockerfile | 2 +- Dockerfile.arm | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 2d1b5daf..d8cb0766 100644 --- a/Dockerfile +++ b/Dockerfile @@ -62,7 +62,7 @@ SHELL ["mamba", "run", "-n", "streamlit-env", "/bin/bash", "-c"] # Install up-to-date cmake via mamba and packages for pyOpenMS build. RUN mamba install cmake -RUN pip install --upgrade pip && python -m pip install -U setuptools nose cython "autowrap<=0.24" pandas numpy pytest +RUN pip install --upgrade pip && python -m pip install -U setuptools nose "cython>=3.1.0,<3.3" "autowrap<=0.24" pandas numpy pytest # Clone OpenMS branch and the associcated contrib+thirdparties+pyOpenMS-doc submodules. RUN git clone --recursive --depth=1 -b ${OPENMS_BRANCH} --single-branch ${OPENMS_REPO} && cd /OpenMS diff --git a/Dockerfile.arm b/Dockerfile.arm index 17659800..2e2fea24 100644 --- a/Dockerfile.arm +++ b/Dockerfile.arm @@ -62,7 +62,7 @@ SHELL ["mamba", "run", "-n", "streamlit-env", "/bin/bash", "-c"] # Install up-to-date cmake via mamba and packages for pyOpenMS build. RUN mamba install cmake -RUN pip install --upgrade pip && python -m pip install -U setuptools nose cython "autowrap<=0.24" pandas numpy pytest +RUN pip install --upgrade pip && python -m pip install -U setuptools nose "cython>=3.1.0,<3.3" "autowrap<=0.24" pandas numpy pytest # Clone OpenMS branch and the associcated contrib+thirdparties+pyOpenMS-doc submodules. RUN git clone --recursive --depth=1 -b ${OPENMS_BRANCH} --single-branch ${OPENMS_REPO} && cd /OpenMS From e2c2a834e5d41c7e0147edb414801ba6048300b0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=20David=20M=C3=BCller?= <57191390+t0mdavid-m@users.noreply.github.com> Date: Fri, 28 Aug 2026 15:33:05 +0200 Subject: [PATCH 23/29] Let the app run on more than one node (#399) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Add failing tests for multi-node distribution Tests first, committed red. Each one fails against the current code for a reason worth naming: - test_parameter_integrity: a failed params.json read currently returns {}, which save_parameters merges and writes back, erasing _defaults, _flag_params and every other tool's values while telling the user it was a deliberate reset. - test_tasks: tasks.py logs WORKFLOW FINISHED unconditionally and returns a dict from its exception handler, so RQ records every job successful and FailedJobRegistry is structurally empty. This module had no coverage at all. - test_seed_demos: the seed-demos initContainer uses `cp -rn`, which under replicas: 2 either skips a file another pod is still writing, permanently, or exits 1 on EEXIST into Init:CrashLoopBackOff. - test_storage_health / test_sidebar_monitors: for a storage indicator that must read Redis only. A stat on a hard NFS mount blocks in uninterruptible sleep, so a widget that touches the filesystem becomes the hang it reports. docker/seed-demos.sh is extracted so the manifest and the test share one copy rather than drifting. .gitattributes pins *.sh to LF: a CRLF checkout breaks both sh in the container and the test. test_topp_flag_parameters keeps its argv assertions unchanged; only the fixture is hardened, because thread resolution is about to depend on a settings.json read relative to the working directory. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01XFgL1SSeMCM3J1ZuAVTXv7 * Make workflow failures observable Nothing downstream can be validated while every failure reports as success, so this lands first. The success signal was inverted in both directions at once. Locally, Workflow.execution() was annotated -> None and returned nothing, so the WORKFLOW FINISHED marker was never logged and a successful run rendered "Errors occurred". In queue mode tasks.py logged that marker unconditionally and returned a dict from its exception handler, so RQ recorded every crash as a success and health.py's failed_jobs was structurally always zero. Both are fixed together: honouring the bool while execution() still returned None would have made every workflow report failed. execution() now also gates on each run_topp/run_python result. Those already returned False on a non-zero exit and nobody looked, which is why a tool that died mid-run still reported success. A subclass still declaring -> None keeps working, with a deprecation warning in the workflow log. params.json reads are split rather than flagged. get_parameters_from_json stays tolerant for the two constructors and five read-only callers, but no longer launders a read failure into {} silently, and its st.error is now conditional on a real session — tasks.py builds a ParameterManager in the RQ worker where it was a silent no-op. The read-modify-write-back sites use a strict read that raises and aborts the write instead of persisting {}. StreamlitUI's input_TOPP path gets more than a raise: it runs on every configure() render, so it preserves the file and offers a reset rather than dumping a traceback on the parameter page. max_threads.online has been dead code since PR #333: the worker has no ScriptRunContext, so st.session_state.get("settings", {}) is {}, the online branch was unreachable, and it silently used the hardcoded local default of 4. A Streamlit-free settings loader fixes that, and the result is memoised because run_topp resolves it twice and both reads must agree. create-workflow.md documented the broken -> None signature, which would have regenerated the bug in every new workflow. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01XFgL1SSeMCM3J1ZuAVTXv7 * Tolerate a shared, remote, restartable filesystem Every change here is a no-op or an improvement on the current single-node volume, so it ships before the storage cutover rather than with it. That way the cutover has working instrumentation to be watched with. The seed-demos initContainer no longer uses `cp -rn`. It copies to a private temp directory and renames: rename(2) on a directory is atomic, so the winner renames and the loser cleans up and exits 0. A lock file would be the wrong tool. It also gets a bounded timeout and exits 0 on failure, because it blocks pod start on the mount and would otherwise defeat the decision that the UI survives a storage outage. The storage indicator reads Redis and never the filesystem. A worker with the mount refreshes a per-node key with a TTL, so the TTL is the liveness mechanism and there is no timeout logic to get wrong. Per-node keying matters because a single healthy node would otherwise mask a wedged one. It reports three states, not two: connected, unreachable, and unknown when Redis itself is down — a red indicator caused by a dead Redis sends an operator debugging the wrong layer. monitor_hardware is now hidden in online mode. It reads psutil on the Streamlit pod, which does no work once execution is distributed, and would show an idle web pod while a worker saturates. The cleanup CronJob gets activeDeadlineSeconds. It is concurrencyPolicy: Forbid, so one run hung on a storage restart would block every subsequent night, silently and permanently. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01XFgL1SSeMCM3J1ZuAVTXv7 * Serve the workspace volume over NFS The app could not use more than one node: a ReadWriteOnce Cinder volume attaches to exactly one, and four workloads mount it. This runs NFS-Ganesha over that volume and hands out a ReadWriteMany claim instead. k8s/storage/ is a separate kustomize root rather than a component, because k8s/base sets `namespace: openms` and the namespace transformer runs after patches — it would clobber any per-object namespace, and components inherit the parent's transformers. It is deliberately prefix-free so namePrefix never has to be duplicated into a Helm values file. workspaces-nfs-pvc is a NEW claim, not an edit of workspaces-pvc. A bound PVC's spec is immutable apart from resources.requests, so apply would be rejected outright, and the only way past that is deleting a claim whose cinder-csi class reclaims with Delete — destroying the volume the rollback depends on. The mountPath is unchanged, and every consumer already reaches the volume through claimName plus /workspaces-streamlit-template, so nothing in src/ changes and absolute symlinks and external_files.txt survive untouched. Four Ganesha settings are load-bearing and commented as non-tunable. existingClaim and reclaimPolicy: Retain keep the data across pod and PV lifecycles. strategy: Recreate avoids the deadlock where a new pod cannot mount the claim the old one still holds. device-based-fsids: false is the subtle one: the default derives NFS file handles from the backing device's major/minor, and a Cinder /dev/vdX minor is not stable across re-attach, so every client would ESTALE after the first restart — weeks later, with nothing to link it to the cause. An explicit memory limit is set because N concurrent write streams put N sets of buffers in one userspace process, and OOM there is the one genuine collapse mode in this design. The NetworkPolicy needs two allow rules, not one. Ganesha exports no_root_squash, and the namespace is shared with another application, so a pod-label rule alone is the isolation that matters — but the provisioner emits in-tree nfs: PVs which the kubelet mounts from the node's own address in the host network namespace, matching no podSelector. Without the second rule every mount hangs. It ships as a placeholder that must be narrowed to the real node addresses. CI applied only the prod overlay, so a storage root it never applied would have stranded every pod; it now applies both. The `|| true` on the deployment availability wait is removed — a Pending, CrashLooping or unschedulable Deployment passed silently, leaving the ingress curl as the only real gate. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01XFgL1SSeMCM3J1ZuAVTXv7 * Let the scheduler place worker pods Removes per-node memory tiering entirely. Nothing in k8s/ now pins a pod to a node — no nodeSelector, no nodeName, no nodeAffinity, no openms.de/memory-tier labels. A tier survives only as a pod size, which is a request the scheduler can act on rather than a label an operator has to maintain. The memory-tier components were already the right shape; they just carried a nodeselector.yaml patch targeting `kind: Deployment` with no name, which landed on redis and streamlit as well as the worker while missing the CronJob entirely. Deleting the patch files and their kustomization entries is atomic: a kustomization referencing a deleted path fails to build. requests now equal limits in both tiers. That is architectural rather than tidy: the pod size is the tier, so a worker requesting 1Gi against a 16Gi limit would be schedulable anywhere and then OOM. Guaranteed QoS also moves oom_score_adj from roughly 937 to -997, so the worker stops being near the top of the node's kill list. rq-worker moves to a fixed replica count with topologySpreadConstraints over kubernetes.io/hostname. Those constraints were inert before: maxSkew is measured over eligible domains, and the nodeSelector left exactly one node eligible, so any spread rule written earlier would have passed while changing nothing. k8s/overlays/ci exists because a production-sized worker cannot be scheduled on a CI runner; it is prod with the worker shrunk, so the kind jobs exercise the real manifests rather than a hand-maintained copy. docs/kubernetes-deployment.md claimed the VolumeBinding plugin pins pods to the node holding the attached volume. It does not — the scheduler checks PV node affinity only, and the result is a Multi-Attach hang at attach time rather than a scheduling refusal. That belief is why the old topology looked intentional. configure-k8s-deployment.md had no question about worker size or replica count at all, and told operators co-location was enforced by the RWO mount. Both are now wrong and both are fixed. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01XFgL1SSeMCM3J1ZuAVTXv7 * Pin the CI toolchain and make its gates fail honestly Every job that renders k8s/storage/ was failing on one thing: azure/setup-helm was unpinned at four sites and `latest` now resolves to Helm v4.2.4, which removed the `-c` shorthand that the kustomize inside kubectl uses to probe it. `kubectl kustomize --enable-helm` refused outright, so assert-invariants died in its first assertion and all eight kind jobs died before applying a single manifest - with none of the eleven new assertions ever executing. Pin helm to v3.21.4 at all four sites, and pin kubectl and kind-action too: setup-kubectl resolved `latest` from the frozen legacy GCS bucket (v1.31.0, kustomize 5.4.2) while the kind jobs ran v1.35.0 / 5.7.1, so the static gates validated output the clusters never saw. Not fixed by bumping kubectl past 1.36 instead: kustomize 5.8.1 stops applying the namespace transformer to Helm-inflated objects, which would deploy Ganesha outside its NetworkPolicy for anyone using the documented apply pipe. Unmasked by that, four gates that could not fail: - lint-manifests piped a failing kustomize into kubeconform under a shell with no pipefail, printed `Valid: 0`, and passed in 11s. The chart it exists to validate had never been validated. Render to a file and require it non-empty - kubeconform exits 0 on empty input, so pipefail alone would not catch a render that succeeds and emits nothing. - The storage apply loop used `if ...; then break; fi`, which never trips `set -e`, so total failure fell through to a command substitution that killed the step before the authored error message could run. - The Traefik host list is derived through `grep -oP | tr`; a miss yields an empty string with exit 0, and the only curl of the app through Traefik then iterated zero times, green. Also `--set service.type=ClusterIP` is a dead values path in that chart - the key is service.spec.type - so the Service had been LoadBalancer all along. - assert_workers_spread_across_nodes walked only the selector keys that were present, so a constraint that lost its app scoping matched the pod template and passed. That is precisely the regression the base manifest's own comment calls load-bearing. Adds two assertions. assert_netpol_admits_every_node checks from the cluster that every node InternalIP is admitted on 2049, because the node CIDR is rewritten by CI rather than shipped and a sed whose target moves fails silently forty minutes later as an unrelated timeout. assert_storage_identity_values covers the three quarters of invariant 2 that nothing checked - Retain, the fixed backing claim, the single replica - each of which renders, deploys and passes every runtime assertion while costing deleted data. Also rewrites the kind node CIDR (kindnetd enforces NetworkPolicy, and the kubelet mounts in-tree nfs: PVs from the node address, so the shipped RFC 5737 placeholder hangs every cross-node mount), sizes the CI claims to what the runner can actually provide, and drops the `-l app=${SLUG}` selectors from the failure dump - SLUG is exported after the storage steps, so every job that died early dumped an empty namespace and looked like an empty cluster. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01XFgL1SSeMCM3J1ZuAVTXv7 * Size the worker from its cgroup, and harden two RWX-sensitive spots Three corrections that the move to a genuinely shared volume makes matter, kept deliberately narrow - the rest of the review backlog is a follow-up. max_threads now comes from the container's own cgroup CPU quota in online mode, with max_threads.online left as the fallback for deployments that expose none. Reading settings.json instead of session state fixed the worker reading a hardcoded 4, but replaced it with a single static number that cannot describe both a 4 cpu low-tier worker and a 20 cpu high-tier one - and described neither. The worker runs at requests.cpu == limits.cpu, so its pod spec is its allocation. Both cgroup generations are handled, quotas round up (1500m is two useful threads, not one), and "no quota" reports None rather than 1 so a laptop does not silently serialise. The trade-off is deliberate and user-visible: an operator can no longer under-subscribe a container that declares more CPU than it wants used. _write_parameter_file keyed its scratch file on os.getpid() alone. Streamlit serves every session from threads of one process, and that process is pid 1 in every replica, so two sessions sharing a ?workspace= link - or two pods, now that the volume really is ReadWriteMany - resolved to the same path and interleaved into it, publishing exactly the torn file the function exists to prevent. Adds a uuid, plus flush+fsync before the rename, since os.replace orders the directory entry and not the data behind it. allowVolumeExpansion was the one storageClass key still inherited in a values file that restates its defaults on purpose. The chart defaults it true and an in-tree nfs: PV has no expander, so a resize is accepted and then sits in FileSystemResizePending forever. The existing max_threads tests pin detect_cpu_quota to None explicitly rather than trusting the runner to be unconstrained, and reach it through the function's own globals - the module-globals idiom this file already uses, because src.workflow.* is popped from sys.modules and a dotted-name patch would target a second copy of the module. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01XFgL1SSeMCM3J1ZuAVTXv7 * Release the NFS mounts before kind tears its nodes down All eight kind jobs are red, and every one of them is red only in the post step. Their 25 normal steps all concluded success - including every runtime assertion, up to and including "4 pods mount template-app-workspaces-nfs-pvc across 2 nodes", which is the acceptance criterion for this whole PR. `docker rm -f` on a kind node whose kubelet still holds a `hard` NFSv4.1 mount cannot complete. The Ganesha server is a pod inside the cluster being destroyed, so it dies alongside its own clients; the pending RPCs block in uninterruptible sleep, SIGKILL is ignored, containerd never sees an exit event, and dockerd gives up after ~12s. This is not a regression. Round 1 teared down cleanly because it died before a single NFS mount existed - the wedge is the first evidence the mounts are real, and it is a live demonstration of the D-state mechanism the rq-worker probe comments reason about. Production never destroys an NFS server and its clients in one `docker rm -f`, so this is a CI-harness gap rather than a product defect. Release the mounts in a last normal step. Normal steps all run before any post step, which is also why "Dump cluster state on failure" is `skipped` in these jobs - no `if:` on a normal step can observe a post step failing, so the only red jobs in the pipeline currently ship zero diagnostics. The order cannot be rearranged: diagnostics first, because after the deletes there is nothing left to name; then the clients, while the server is still up and their unmounts can complete; then a lazy detach for whatever the kubelet did not release, which returns without waiting on the server and so cannot itself hang; then the server, because killing it while a client mount is still attached is the deadlock verbatim. Deliberately NOT `ignore_failed_clean: true` on kind-action. That input is real and would green all eight jobs immediately, but it works by discarding the error - permanently deleting the only signal that teardown wedged, including if this step later regresses through a cluster-name typo. The residual-mount echo at the end is the check that replaces it. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01XFgL1SSeMCM3J1ZuAVTXv7 * Close three silent-failure paths in the kind jobs, and pin ingress-nginx Four small fixes to the CI steps, none of which fired in the last run - which is the reason to make them now rather than while debugging something else. The two app-overlay apply loops still had the shape the previous commit claimed to have fixed. It fixed the two STORAGE apply loops and left these: `kubectl apply` inside an `if` condition is exempt from `set -e`, so after five failures the last command run is `sleep`, the loop exits 0, and the step passes having created nothing. The symptom would surface three steps later as "Wait for Redis to be ready" timing out on a pod that never existed. All four loops now track an `applied` flag and exit explicitly. ingress-nginx was the last unpinned upstream of the class that took this whole pipeline down through Helm 4, and here the usual moving-ref trap has inverted into rot: the repository is archived, so `main` will not move again, but the kind node image will. Pinned to controller-v1.15.1, whose support table lists k8s 1.35 - not v1.14.0, which stops at 1.34 while these jobs create kindest/node:v1.35.0. The Traefik host discovery read `.spec.routes[0].match` only. Prod renders a single route carrying both hostnames so this changes nothing today, but a second route would have dropped silently out of the only gate that exercises the app through Traefik. The NFS module probe reported "absent" about a working host. `grep nfs /proc/filesystems` printed nothing on every node of every job in a run where those same nodes went on to mount type=nfs4 vers=4.1 two minutes later - the module autoloads on first mount, and this step runs before any mount exists. It now attempts a modprobe and, if the grep still finds nothing, says why the answer is not conclusive. This is the line a triager reads to decide whether a hung mount is a policy problem or a missing module, so it needs to not lie. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01XFgL1SSeMCM3J1ZuAVTXv7 * Stop three assertions passing on conditions they exist to catch All thirteen assertions ran green last round. Two of them could not have gone red for the thing they claim to check, and a third never looked at the object whose placement mattered most. assert_netpol_admits_every_node never checked the port. It collected `.from[].ipBlock.cidr` without descending into the sibling `.ports`, and flattened across every policy in the namespace, while its PASS line asserted "admitted on 2049". Accidentally correct on today's single-rule policy; change the port to 111, or add any unrelated policy carrying a broad ipBlock, and it still reported success while every cross-node mount hung - the forty-minute Deployment-availability timeout its own docstring exists to pre-empt. Rules are now filtered by port before their CIDRs are collected, keeping a ports-less rule as all-ports. Verified against synthetic payloads: the happy path is unchanged, port 111 now yields empty and trips the guard, an unrelated policy on 9100 is excluded, and UDP/2049 is excluded. assert_storage_identity_values tested the backing claim with `-z` where its two siblings use equality, so `persistence.existingClaim` and the PVC name in nfs-backing-pvc.yaml - hand-synced, as the values file itself concedes - were verified by nothing. Rename the PVC without touching the values and the server would sit Pending on a claim that does not exist, with all four static assertions green. Now compared against the rendered PVC name, in an `else` so a dropped claim reports once instead of twice. Both workload selects are also scoped to the server's own app label, derived from `nameOverride` rather than hardcoded, so a chart that grows a second workload produces a real answer instead of two lines and a message naming neither. assert_no_node_pinning_anywhere never saw the Ganesha StatefulSet. Its rendered half rendered only the prod overlay, and its text half asks git, which prunes the vendored chart under k8s/storage/charts/ through .gitignore - so the pod this entire PR is about placing was checked by neither half, and neither was k8s/overlays/ci/, which is what the kind jobs actually apply. It now scans three rendered roots through a shared helper. The PASS string is rewritten accordingly: it previously ended by stating that the vendored chart is not scanned, which would have become false. PersistentVolume is skipped by that scan. `spec.nodeAffinity` on a PV is volume topology, not pod pinning, and is mandatory for a local volume. Nothing in the three roots renders a PV today, but the storage root is now among them and a chart that ever emitted one would have turned invariant 1 permanently red with no way to green it except weakening the check. The scan requires helm outright rather than `if command -v helm`. A check that silently does less work when a tool is missing is the shape this round exists to remove. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01XFgL1SSeMCM3J1ZuAVTXv7 * Ship the storage runbook, decisions and defect register Eleven tracked files cited A16-RUNBOOK.md, A16-DECISIONS.md or DEFECTS.md as the authority for why a value is what it is - and none of the three was in the repository. They sat in an untracked scratch directory that only ever existed on the machine the multi-node work was done on. That includes the only documented rollback for the storage cutover. As it stood, merging would have shipped a storage tier whose escape hatch pointed at a file nobody else has, with k8s/base/workspace-pvc.yaml telling the next operator to consult section 5 of something they cannot open. All three are now under docs/, with every reference repointed: A16-RUNBOOK.md -> docs/a16-storage-runbook.md A16-DECISIONS.md -> docs/a16-storage-decisions.md DEFECTS.md -> docs/storage-defect-register.md Each gains a header saying what it is and how far to trust it. They are records of the multi-node work rather than living documents: cluster-specific values in the runbook are examples, the decision record is a snapshot whose reasoning the code supersedes where they disagree, and the defect register lists what was found rather than what was fixed - several entries are still live, which is worth stating at the top rather than leaving a reader to infer. content/documentation.py opens named files rather than globbing docs/, so these three add no chapter to the in-app documentation page and test_gui.py::test_documentation is unaffected. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01XFgL1SSeMCM3J1ZuAVTXv7 * Point the storage docs at each other by the names they now have 51a2b6c moved the three research documents under docs/ and repointed every reference from k8s/, .github/, src/ and tests/. It missed the eight references the documents make to each other, which kept their pre-move names: A16-RUNBOOK.md, A16-DECISIONS.md and DEFECTS.md, none of which exists at any path in the tree. One of those was the rollback procedure for the storage cutover. k8s/base/ workspace-pvc.yaml cites docs/a16-storage-runbook.md as authoritative for it, so the outer hop resolved and the inner one led nowhere - the same dead end R4 was written to close, one level further down. Also replaces the hardcoded node name in the runbook's pre-flight check. That command decides whether clients can mount via the in-tree nfs: type or whether the cluster needs csi-driver-nfs, and it named one Kubermatic-generated node that has almost certainly been rotated away since. The line immediately below it already says to run it against both nodes, so it now shows how to list them. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_018tuhTmRVKxCXo5gJSJU8e8 * Stop four assertions claiming more than they check assert_netpol_string_matches_overlay failed only when the ipBlock set was EMPTY, so the most dangerous possible value passed: 0.0.0.0/0 kept it green AND removed the TEST-NET-1 placeholder NOTE, which matches only ^192\.0\.2\. - the worst setting produced a quieter log than the safe one, over a no_root_squash export. It now hard-fails on 0.0.0.0/0 and on any prefix shorter than /12, and its PASS names the node rule instead of claiming the policy admits exactly one app. A prefix threshold rather than a "warn on non-RFC1918" rule, which inverts the hazard: pod networks ARE RFC1918 - kind defaults to 10.244.0.0/16 - so that rule would stay silent on 10.0.0.0/8 while firing on the harmless placeholder. It bounds width, not overlap; deciding overlap needs the cluster's own pod CIDR. assert_no_node_pinning_anywhere skipped an empty CI_ASSERT_* root with a bare `continue` while its PASS went on naming all three, degrading the sentence to "prod, and k8s/storage" with a double space as the only trace. That is the silent-skip shape the hard `_ci_require kubectl helm` eight lines above was added to remove. Empty is now a failure, the PASS is built from the roots actually scanned, and each root logs its document count - the step previously emitted its PASS sentence and nothing else. The pinning text scan piped through `grep -v nodeSelectorTerms`, which looked like it suppressed a false positive and did not: `nodeSelectorTerms:` on its own line does not match _ci_pin_re at all, because the pattern requires nodeSelector to be followed by `:`, end-of-line or `/`. Its only effect was dropping lines that DID match and contained the substring - a flow-style nodeAffinity, which is genuine pinning. Verified a no-op against the current tree. assert_storage_identity_values derives the server's app label from nameOverride but never named it in the PASS, so a green log did not record which workload was inspected. And assert_netpol_admits_every_node discarded jq's stderr, so a syntax error in its filter surfaced as "the NetworkPolicies admit no ipBlock" - pointing at the manifests when the fault is in this file. Neither bash -n nor shellcheck parses an embedded jq program. Adds assert_doc_links_resolve: every backticked *.md path in tracked Markdown has to name a tracked file. Deliberately general rather than a grep for the three filenames the previous commit fixed - that grep would itself be a hand-synced literal, correct until the fourth rename and silent afterwards. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_018tuhTmRVKxCXo5gJSJU8e8 * Make four CI steps say what they actually did The teardown step added last round released NFS mounts before kind destroys its nodes, but every command in it is `|| true` and it discarded the one diagnostic that would show whether it did anything. A green teardown was therefore ambiguous between "the mounts were released" and "there were none to release" - and if `kind get nodes` had returned nothing, all three loops would have iterated zero times and the step would have exited 0 having done nothing at all. It now derives the node list once and reuses it (the cluster can disappear between loops during teardown), warns when that list is empty, counts the NFS mounts present before teardown and warns when there are none, and warns when any survive. It stays non-gating: it runs `if: always()`, including on jobs that died before creating a cluster, so a gating version would manufacture red on unrelated failures. Exercised against all four outcomes with stubbed kind/docker/kubectl. Also drops the `2>/dev/null` from the D-state probe in the same step. Suppressed, a failing ps prints nothing - byte-identical to "no processes in D state", a false negative in the diagnostic that identified the deadlock in the first place. ps stays rather than /proc//status because wchan is the column that named it (nfs_wait_bit_killable); kind v0.31.0's base image installs procps. The NFS module probe threw away modprobe's exit status, so a SUCCESSFUL modprobe followed by no /proc/filesystems match - conclusive evidence that a node cannot mount NFS - printed the same "only conclusive once a mount has been attempted" hedge as a modprobe that could not run. It now distinguishes them. shellcheck was behind `if command -v`, so a missing shellcheck produced a green step with no line saying so; since shellcheck prints nothing when it finds nothing, the log could not tell "ran clean" from "never ran". Now unconditional, matching the hard `_ci_require kubectl helm` the same round introduced. /tmp/prod-rendered.yaml had no emptiness fence, unlike its two siblings. `bash -e` catches a render that fails, not one that succeeds emitting zero documents - and that case leaves PROD_RES empty while CI_RES holds a real value, so the ci-vs-prod worker comparison is FALSE, passes, and prints a real worker size having compared it against nothing. Finally, the ingress-nginx pin comment claimed the cluster's k8s version "floats upward" because node_image is unpinned. It does not: `version: v0.31.0` fixes the default node image at kindest/node:v1.35.0, and the same comment states that value three lines earlier. Reworded, with the verified support range. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_018tuhTmRVKxCXo5gJSJU8e8 * Give each consumer a build dependency at the granularity it needs `needs:` in GitHub Actions is job-level, not matrix-leaf-level. With one `build-amd64` job carrying a {full, simple} matrix, a failure in EITHER leg marked the whole job failed and skipped every consumer - including test-nginx and test-traefik, which are the only place the multi-node storage tier is exercised on a cluster at all. On 2026-08-25 an upstream cython 3.3.0 release removed CConstTypeNode, which autowrap 0.24 calls, and broke the `full` build on both arches. All 8 kind leaves were skipped without their matrices ever expanding, even though both `simple` images had been built, uploaded and were sitting unexpired in the same run. A break in one image variant cost 100% of the storage-tier runtime coverage instead of 50%, and R1, R2, R3, R7, R8 and R9 all went from failing to untested. build-amd64 and build-arm64 become four jobs - build-{full,simple}-{amd64,arm64} - routed through a new reusable workflow so that is not four copies of the same eighty lines. The two jobs were already near duplicates of each other; the only behavioural difference between them was which ran the disk-space step, and that is preserved exactly through a `free_disk_space` input rather than quietly normalised, so the split introduces no delta to compare against when the kind suite finally runs. The kind jobs keep `needs:` on all four builds - that is what provides ordering - and add `if: always()` to override only the other thing `needs:` does, which is skip on a failed dependency. The condition is the `simple` pair specifically: those legs install a prebuilt pyopenms wheel and are the robust ones, while the `full` legs compile OpenMS from source and are where upstream breakage lands. When `full` breaks, the simple leaves now run and assert everything and the full leaves go red at the artifact download - honest, since their image genuinely does not exist, and a new step names the cause rather than leaving "unable to find any artifacts". When the simple builds are themselves broken there is nothing to test against and the whole job skips as before. No staleness risk: download-artifact@v4 is scoped to the current run. create-manifest now also depends on both kind jobs. It publishes `latest` and the multi-arch tags the cluster pulls, and nothing in the graph named test-nginx or test-traefik - so merging published an image whose storage tier had never been exercised on a cluster. `main` carries no branch protection, so the graph is the only place this can be enforced. Accepted cost: every push to main now waits on a ~40 minute kind matrix. test-apptainer is deliberately NOT given the same override. Apptainer is a side deployment - the cluster runs the docker image - so the coverage worth rescuing from a one-variant break is the kind suite, not that. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_018tuhTmRVKxCXo5gJSJU8e8 * Put the missing space back into the NetworkPolicy PASS The reworded PASS from the previous commit interpolated the admitted CIDRs with `tr '\n' ' '` and relied on a trailing space that is never there: command substitution strips the trailing newline, so a single CIDR produced "plus ipBlock 192.0.2.0/24for kubelet mounts". Cosmetic, but it is a message about what the check verified, in the round whose whole subject is messages claiming exactly what their code did. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_018tuhTmRVKxCXo5gJSJU8e8 --------- Co-authored-by: Claude Opus 5 (1M context) --- .claude/skills/configure-k8s-deployment.md | 84 +- .claude/skills/create-workflow.md | 28 +- .gitattributes | 5 + .github/kind-config.yaml | 32 +- .github/scripts/ci-assertions.sh | 2014 +++++++++++++++++ .github/workflows/build-and-test.yml | 1648 ++++++++++++-- .github/workflows/build-image.yml | 151 ++ .gitignore | 11 +- CLAUDE.md | 294 ++- Dockerfile | 6 + Dockerfile.arm | 6 + Dockerfile_simple | 6 + Dockerfile_simple.arm | 6 + docker/seed-demos.sh | 270 +++ docs/a16-storage-decisions.md | 309 +++ docs/a16-storage-runbook.md | 297 +++ docs/kubernetes-deployment.md | 148 +- docs/storage-defect-register.md | 335 +++ k8s/base/cleanup-cronjob.yaml | 23 +- k8s/base/namespace.yaml | 30 + k8s/base/rq-worker-deployment.yaml | 188 +- k8s/base/streamlit-deployment.yaml | 24 +- k8s/base/workspace-pvc.yaml | 93 +- .../memory-tier-high/kustomization.yaml | 12 +- .../memory-tier-high/nodeselector.yaml | 4 - .../memory-tier-high/worker-resources.yaml | 52 +- .../memory-tier-low/kustomization.yaml | 12 +- .../memory-tier-low/nodeselector.yaml | 4 - .../memory-tier-low/worker-resources.yaml | 28 +- k8s/overlays/ci/kustomization.yaml | 38 + k8s/overlays/ci/worker-resources.yaml | 27 + k8s/storage/ganesha-values.yaml | 266 +++ k8s/storage/kustomization.yaml | 86 + k8s/storage/namespace.yaml | 32 + k8s/storage/networkpolicy.yaml | 162 ++ k8s/storage/nfs-backing-pvc.yaml | 43 + src/Workflow.py | 40 +- src/common/common.py | 120 +- src/workflow/CommandExecutor.py | 90 +- src/workflow/ParameterManager.py | 336 ++- src/workflow/QueueManager.py | 23 +- src/workflow/StreamlitUI.py | 100 +- src/workflow/health.py | 517 ++++- src/workflow/settings_io.py | 151 ++ src/workflow/tasks.py | 73 +- tests/test_command_executor_run_python.py | 184 ++ tests/test_parameter_integrity.py | 986 ++++++++ tests/test_seed_demos.py | 869 +++++++ tests/test_sidebar_monitors.py | 322 +++ tests/test_storage_health.py | 848 +++++++ tests/test_streamlit_ui_param_read.py | 240 ++ tests/test_tasks.py | 484 ++++ tests/test_topp_flag_parameters.py | 33 +- 53 files changed, 11689 insertions(+), 501 deletions(-) create mode 100644 .gitattributes create mode 100644 .github/scripts/ci-assertions.sh create mode 100644 .github/workflows/build-image.yml create mode 100644 docker/seed-demos.sh create mode 100644 docs/a16-storage-decisions.md create mode 100644 docs/a16-storage-runbook.md create mode 100644 docs/storage-defect-register.md delete mode 100644 k8s/components/memory-tier-high/nodeselector.yaml delete mode 100644 k8s/components/memory-tier-low/nodeselector.yaml create mode 100644 k8s/overlays/ci/kustomization.yaml create mode 100644 k8s/overlays/ci/worker-resources.yaml create mode 100644 k8s/storage/ganesha-values.yaml create mode 100644 k8s/storage/kustomization.yaml create mode 100644 k8s/storage/namespace.yaml create mode 100644 k8s/storage/networkpolicy.yaml create mode 100644 k8s/storage/nfs-backing-pvc.yaml create mode 100644 src/workflow/settings_io.py create mode 100644 tests/test_command_executor_run_python.py create mode 100644 tests/test_parameter_integrity.py create mode 100644 tests/test_seed_demos.py create mode 100644 tests/test_sidebar_monitors.py create mode 100644 tests/test_storage_health.py create mode 100644 tests/test_streamlit_ui_param_read.py create mode 100644 tests/test_tasks.py diff --git a/.claude/skills/configure-k8s-deployment.md b/.claude/skills/configure-k8s-deployment.md index 556c54af..63d30c82 100644 --- a/.claude/skills/configure-k8s-deployment.md +++ b/.claude/skills/configure-k8s-deployment.md @@ -16,8 +16,10 @@ These are facts about the target cluster, not steps for you to execute. Mention - Traefik ingress controller installed in the cluster (handles `IngressRoute` CRDs). - Cluster has read access to GHCR for pulling the app image. -- A storage class compatible with `cinder-csi` (or equivalent `ReadWriteOnce` provider). For live PVC grows, the storage class needs `allowVolumeExpansion: true`. -- Worker nodes labelled `openms.de/memory-tier=low` and `openms.de/memory-tier=high`. The memory-tier components add the matching `nodeSelector`. +- A storage class that provisions `ReadWriteOnce` volumes (de.NBI: `cinder-csi`). That is the **backing** volume for the storage tier, not the claim the app mounts. For live grows of it, the class needs `allowVolumeExpansion: true`. +- The fork's storage tier (`k8s/storage/`) already applied, publishing the `ReadWriteMany` StorageClass the workspaces PVC names, and `mount.nfs` present on every node. Applied in the other order, every pod sits `Pending` on a class that does not exist yet. +- **No node labels, of any kind.** Nothing under `k8s/` selects a node any more: the scheduler places pods and the manifests only declare how big a worker is. The `openms.de/memory-tier` labels the older components matched are gone, and CI fails any manifest that reintroduces a `nodeSelector`, `nodeName` or `nodeAffinity` (`assert_no_node_pinning_anywhere`). +- Room for ` x ` (Q5 and Q6) across the cluster's nodes. A worker is Guaranteed QoS, so its request is reserved for as long as the pod runs, queue empty or not. - DNS for `*.webapps.openms.de` and `*.webapps.openms.org` pointing at the cluster's Traefik load balancer. ## Step 1 — Recon the fork @@ -27,16 +29,17 @@ Before asking the user anything, read a small known set of files directly (do no 1. `git remote get-url origin` and the repo name → seeds the slug and GHCR ref defaults. 2. `k8s/overlays/prod/kustomization.yaml` — if anything has already been edited away from the template stub, treat those values as the user's prior choices to confirm rather than overwrite blindly. 3. `k8s/base/kustomization.yaml`, `k8s/base/streamlit-deployment.yaml`, `k8s/base/rq-worker-deployment.yaml`, `k8s/base/workspace-pvc.yaml` — confirm the layout still matches the template: - - PVC `metadata.name` is `workspaces-pvc`. + - PVC `metadata.name` is `workspaces-nfs-pvc`, `ReadWriteMany`, on a `storageClassName` this fork's `k8s/storage/` publishes. - Deployments reference `image: openms-streamlit` (the placeholder Kustomize swaps). - - `streamlit-deployment.yaml` has `claimName: workspaces-pvc`. (Co-location of the workspace-using pods is enforced by the shared RWO PVC mount, not by a pod-affinity rule.) + - `streamlit-deployment.yaml` and `rq-worker-deployment.yaml` both carry `claimName: workspaces-nfs-pvc`. Being RWX, that claim constrains placement not at all: the workspace-using pods are placed independently by the scheduler, not co-located, and there is no pod-affinity rule pulling them together either. + - `rq-worker-deployment.yaml` has a fixed `replicas` and a `topologySpreadConstraints` block over `kubernetes.io/hostname` with `maxSkew: 1`. Read the replica count from here — it is the default you propose in Q6. 4. `.github/workflows/build-and-test.yml` — confirm which tags CI publishes (the OpenMS template publishes `-full`, `-simple`, `-full`, `-simple`, plus `latest` on `main`-full pushes). If any of those files are missing, renamed, or significantly restructured, stop and ask the user how to proceed. Do not pattern-match the standard answers onto an unknown layout. ## Step 2 — Interview the user -Ask the user the questions below in 2–3 batched `AskUserQuestion` turns rather than one long sequential interrogation. Group related questions together (slug + subdomain + GHCR ref make a coherent first batch). +Ask the user the questions below in 3 batched `AskUserQuestion` turns rather than one long sequential interrogation. Group related questions together: slug + subdomain + GHCR ref + tag make a coherent identity batch, and worker size + replica count + storage size make a coherent capacity batch — they trade against each other and against the same cluster. Each question must include: @@ -67,17 +70,26 @@ The user confirms or overrides each one. Do not omit the "what this controls" li - *What this controls:* the public URL users type into a browser. The IngressRoute always wires up **both** `.webapps.openms.de` and `.webapps.openms.org` (one IngressRoute, two `Host(...)` matchers OR-ed together), so users land on the same app regardless of which TLD they remember. - *Default:* the slug. But ask — `OpenDIAKiosk` chose subdomain `opendia` (different from its slug `opendiakiosk`) for a shorter URL, so this is not always identical to the slug. -### Q5. Memory tier +### Q5. Worker size -- *What this controls:* which set of cluster nodes the app's pods are eligible to schedule on. `memory-tier-low` targets the standard worker nodes (correct for ~90% of template forks). `memory-tier-high` targets the high-memory nodes reserved for heavy DIA / OpenSwath workloads. Picking the wrong tier means either pods stuck `Pending` (overflowed low tier) or starving other heavy apps off the high tier. -- *Default:* `memory-tier-low`. -- *Override prompt:* "Does this app run DIA spectral-library construction, OpenSwath peak picking, DIA-LFQ, or comparable heavy OpenMS workloads?" If yes → `memory-tier-high`. +- *What this controls:* how much memory and CPU **one RQ worker** reserves. This is a pod size, not a node selection — the component sets `requests` equal to `limits`, so the number is simultaneously what the scheduler reserves and the hard ceiling the kernel enforces, and Kubernetes then places the pod wherever it fits. Too small and TOPP tools are OOMKilled mid-workflow; too large and the worker either sits `Pending` or reserves most of a node it never uses. +- *Default:* `memory-tier-low` — 16Gi / 4 cpu per worker, correct for ~90% of template forks. +- *Override prompt:* "Does this app run DIA spectral-library construction, OpenSwath peak picking, DIA-LFQ, or comparable heavy OpenMS workloads?" If yes → `memory-tier-high`, 180Gi / 20 cpu, which is most of a high-memory node held for as long as the worker lives. +- *If neither number fits:* edit `k8s/components/memory-tier-/worker-resources.yaml` and keep `requests` **equal to** `limits` on both memory and cpu. Splitting them drops the pod to Burstable QoS and back near the top of the node's OOM-kill list; CI fails that (`assert_worker_qos_guaranteed`). The cluster-side ceiling is the `LimitRange` in `k8s/base/limitrange.yaml`. +- *What it no longer controls:* placement. Nothing schedules by node label any more — a tier is a size. A fork that needs one specific machine has to take that up with the cluster operator, not encode it here. -### Q6. Workspace storage size +### Q6. Worker replica count + +- *What this controls:* how many workflows the deployment runs at once — RQ takes one job per worker process, so N workers is N concurrent workflows and nothing else. It is also what actually spends a second node: the workers carry `topologySpreadConstraints` with `maxSkew: 1` over `kubernetes.io/hostname`, so replicas are placed one per node before any node takes a second. +- *Default:* whatever `k8s/base/rq-worker-deployment.yaml` already says (2 — one per node on the two-node OpenMS cluster). Propose a different number only when the answer to "how many nodes can hold a worker of the Q5 size" is not 2. +- *Cost of raising it:* `replicas x Q5 size` is reserved cluster-wide for as long as the workers run, queued jobs or not. A replica with nowhere to fit stays `Pending` rather than doubling up on a node — `whenUnsatisfiable: DoNotSchedule` — so over-provisioning is visible in `kubectl get pods` rather than silent. +- *Do not answer 1* unless serialized workflows are genuinely wanted: a single worker leaves every other node with nothing to run, which is the state this deployment was in before. + +### Q7. Workspace storage size - *What this controls:* the persistent disk allocated for Streamlit session workspaces, uploaded files, intermediate analysis outputs, and any reference data the app seeds at startup. Too small → users hit "no space left" mid-analysis; too large → wasted cluster storage budget. -- *Default:* **500 Gi** (matches the stock base, so the default needs zero file edits). -- *Note on naming (do not ask the user):* the PVC base name stays `workspaces-pvc` for every fork. Kustomize's `namePrefix: -` automatically scopes it to `-workspaces-pvc` in the cluster, so cross-fork name collisions are impossible. Override the size only — never rename the PVC. +- *Default:* **400 Gi** (matches the stock base, so the default needs zero file edits). +- *Note on naming (do not ask the user):* the PVC base name stays `workspaces-nfs-pvc` for every fork. Kustomize's `namePrefix: -` automatically scopes it to `-workspaces-nfs-pvc` in the cluster, so cross-fork name collisions are impossible. Override the size only — never rename the PVC. ## Step 3 — Apply the answers to `k8s/overlays/prod/kustomization.yaml` @@ -91,7 +103,7 @@ resources: - ../../base components: - - ../../components/memory-tier- # Q5: low | high + - ../../components/memory-tier- # Q5: low | high — worker SIZE, not node selection namePrefix: - # Q1 @@ -138,26 +150,49 @@ Substitution map: - `` → Q4 answer (note: `` is independent of ``). - `` → `low` or `high` from Q5. +Q6 needs a file edit **only if the answer differs from the base default**. If it does, add one more patch to the same `patches:` list. Patch the overlay, never `k8s/base/rq-worker-deployment.yaml`, so the fork's capacity choice stays in the one file this skill owns and a template rebase cannot quietly revert it: + +```yaml + - target: + kind: Deployment + name: rq-worker + patch: | + - op: replace + path: /spec/replicas + value: # Q6 +``` + +Leave the `topologySpreadConstraints` in the base alone either way. It is what puts those replicas on different nodes, and it is scoped to this fork only because kustomize copies `commonLabels.app` into the constraint's `labelSelector` — dropping `commonLabels` from the overlay silently widens it to every fork's workers in the namespace. + About `images[0].name: openms-streamlit` — **this is the match key, not a value the user picks.** Kustomize's `images:` transformer is find-and-replace: `name` is the literal image string Kustomize searches for in the rendered manifests, and `newName`/`newTag` are what it substitutes. Both base Deployments (`streamlit-deployment.yaml`, `rq-worker-deployment.yaml`) reference `image: openms-streamlit`; the overlay's `name: openms-streamlit` matches that literal and rewrites it to `:`. If you change this field, no rewrite happens and the cluster pulls a non-existent `openms-streamlit:latest` and gets `ImagePullBackOff`. Leave it alone. About the `||` in the IngressRoute match — both TLDs always go in. The OpenMS infra publishes apps on both `.de` and `.org` so users land on the same app no matter which they remember. One IngressRoute, two `Host(...)` matchers OR-ed together is the right shape; do not split into two IngressRoute objects. ## Step 4 — Optional storage resize -Skip this step if the user accepted the 500 Gi default. +Skip this step if the user accepted the 400 Gi default. -Otherwise, edit `k8s/base/workspace-pvc.yaml` and change a single line: +Otherwise **two** claims move together, because the app no longer mounts a volume directly — it claims space out of the one the NFS server exports: ```yaml +# 1. k8s/storage/nfs-backing-pvc.yaml — the Cinder volume Ganesha exports +spec: + resources: + requests: + storage: + +# 2. k8s/base/workspace-pvc.yaml — what the app claims out of it spec: resources: requests: - storage: # Q6: e.g. 100Gi, 1Ti, 3Ti + storage: # Q7: e.g. 100Gi, 1Ti, 3Ti ``` -Do **not** rename the PVC, the base `kustomization.yaml` resource list, or the `claimName` in `streamlit-deployment.yaml`. Kustomize's `namePrefix` already gives the in-cluster PVC a unique per-fork name; renaming the base creates a 3-file cascade for no benefit. +The app's claim must stay **materially below** the backing volume, and the backing volume grows first. nfs-provisioner `statfs()`es the export on every provision and refuses outright any claim larger than the *free* space it finds there — unconditionally, not gated on quotas — and a fresh filesystem never reports its full nominal size. The stock pair is 400Gi against 500Gi; keep a comparable margin, or the claim never binds and every pod stays `Pending` behind it. + +Do **not** rename either PVC, the base `kustomization.yaml` resource list, or the `claimName` in `streamlit-deployment.yaml` / `rq-worker-deployment.yaml`. Kustomize's `namePrefix` already gives the in-cluster workspaces PVC a unique per-fork name; renaming the base creates a multi-file cascade for no benefit. -Operator caveat (mention in handoff, not your job to verify): in-place expansion of an *already-deployed* PVC requires the StorageClass to have `allowVolumeExpansion: true`. If the operator's `cinder-csi` class does not allow expansion, growing a live PVC requires recreation, not a manifest edit. Resizing on first deploy is unaffected. +Operator caveat (mention in handoff, not your job to verify): in-place expansion of an *already-deployed* PVC requires the StorageClass to have `allowVolumeExpansion: true`, and that applies to the backing `cinder-csi` claim. If the class does not allow expansion, growing a live volume requires recreation, not a manifest edit. Sizing on first deploy is unaffected. ## Step 5 — Handoff @@ -165,14 +200,15 @@ After committing the edits, tell the user the next steps belong to a human opera 1. Open a PR with the overlay edits and have it reviewed. 2. Merge to `main`. CI (`build-and-test.yml`) rebuilds and pushes the image to GHCR with the tag from Q3. The kind integration jobs (`test-nginx`, `test-traefik`) auto-discover slug and Traefik hostnames from the overlay output, so no workflow edits are needed for fork-specific values. -3. Cluster operator runs `kubectl apply -k k8s/overlays/prod/` against the OpenMS cluster. -4. Operator verifies with `kubectl -n openms rollout status deployment/-streamlit` and a browser check on `https://.webapps.openms.de`. +3. Cluster operator applies the fork's storage tier first (`kubectl kustomize --enable-helm k8s/storage/ | kubectl apply -f -`), then `kubectl apply -k k8s/overlays/prod/`. That order matters — the overlay's PVC names a StorageClass the storage tier publishes. +4. Operator verifies with `kubectl -n openms rollout status deployment/-streamlit` and a browser check on `https://.webapps.openms.de`, then `kubectl -n openms get pods -o wide -l component=rq-worker` to confirm the workers came up on **different** nodes, each with `.status.qosClass` of `Guaranteed`. ## Reference files - Overlay: `k8s/overlays/prod/kustomization.yaml` -- Memory-tier components: `k8s/components/memory-tier-{low,high}/` +- Worker/Streamlit sizing components: `k8s/components/memory-tier-{low,high}/` — `worker-resources.yaml` + `streamlit-resources.yaml`, pod sizes only, no node selection - Base manifests: `k8s/base/*.yaml` +- Storage tier: `k8s/storage/` — NFS-Ganesha serving the RWX workspace volume - CI workflow: `.github/workflows/build-and-test.yml` (build + lint + kind integration) - In-app reference: the "Developers Guide: Kubernetes Deployment" Documentation page in the running Streamlit app. @@ -183,7 +219,9 @@ After committing the edits, tell the user the next steps belong to a human opera - [ ] `namePrefix`, `commonLabels.app`, `images[0].newName`, `images[0].newTag` written in the overlay - [ ] IngressRoute patch written: both `.de` and `.org` hostnames, plus the `-streamlit` service reference - [ ] Redis URL written in both Deployment patches (`streamlit` and `rq-worker`) -- [ ] Memory-tier component selected -- [ ] Storage size in `k8s/base/workspace-pvc.yaml` updated only if the user picked a non-default size; PVC name and `claimName` untouched +- [ ] Worker size component selected (Q5) +- [ ] Worker replica count (Q6) patched into the overlay only if it differs from the base default; the base Deployment left untouched +- [ ] No `nodeSelector`, `nodeName`, `nodeAffinity` or `openms.de/memory-tier` anywhere in the overlay or the components — `assert_no_node_pinning_anywhere` fails the PR on any of them +- [ ] Storage sizes in `k8s/storage/nfs-backing-pvc.yaml` and `k8s/base/workspace-pvc.yaml` changed together, backing volume materially larger, and only if the user picked a non-default size; PVC names and `claimName`s untouched - [ ] `.github/workflows/build-and-test.yml` uses dynamic overlay discovery (no `template-app` / `template.webapps.openms.*` literals); patched in if the fork's workflow was on the old hardcoded shape - [ ] Changes committed on a feature branch (no PR opened unless the user asked for one) diff --git a/.claude/skills/create-workflow.md b/.claude/skills/create-workflow.md index 38e981dc..01400c8f 100644 --- a/.claude/skills/create-workflow.md +++ b/.claude/skills/create-workflow.md @@ -44,22 +44,31 @@ class MyWorkflow(WorkflowManager): with t[1]: self.ui.input_TOPP("ToolName2") - def execution(self) -> None: + def execution(self) -> bool: + # Must return True on success, False on any failure - see "Execution" below. if not self.params["mzML-files"]: self.logger.log("ERROR: No input files selected.") - return + return False in_files = self.file_manager.get_files(self.params["mzML-files"]) self.logger.log(f"Processing {len(in_files)} files...") # Step 1 out_step1 = self.file_manager.get_files(in_files, "featureXML", "step1") - self.executor.run_topp("ToolName1", input_output={"in": in_files, "out": out_step1}) + if not self.executor.run_topp( + "ToolName1", input_output={"in": in_files, "out": out_step1} + ): + return False # Step 2 in_step2 = self.file_manager.get_files(out_step1, collect=True) out_step2 = self.file_manager.get_files("result.consensusXML", set_results_dir="step2") - self.executor.run_topp("ToolName2", input_output={"in": in_step2, "out": out_step2}) + if not self.executor.run_topp( + "ToolName2", input_output={"in": in_step2, "out": out_step2} + ): + return False + + return True @st.fragment def results(self) -> None: @@ -111,8 +120,14 @@ The 4 pages call these methods respectively: - `self.file_manager.get_files("name.ext", set_results_dir="dir")` — single result file ### Execution -- `self.executor.run_topp("ToolName", input_output={"in": [...], "out": [...]})` — run a TOPP tool -- `self.executor.run_python("script_name", {"in": [...]})` — run a Python tool from `src/python-tools/` +- `self.executor.run_topp("ToolName", input_output={"in": [...], "out": [...]})` — run a TOPP tool; returns `False` if any command failed +- `self.executor.run_python("script_name", {"in": [...]})` — run a Python tool from `src/python-tools/`; returns `False` if the script is missing or exited non-zero +- **`execution()` must be annotated `-> bool` and return `True` only when every step + succeeded**, `False` on bad input or a failed tool. `workflow_process()` logs the + `WORKFLOW FINISHED` marker only for a truthy return, and a missing marker is classified + as an error — so an `execution()` that returns nothing renders "Errors occurred, check + log file." after a perfectly good run. Gate every `run_topp` / `run_python` call on its + return value too, or a tool that fails halfway is still reported as a success. ### UI widgets - `self.ui.upload_widget(key, name, file_types, fallback)` — file upload with example data fallback @@ -164,6 +179,7 @@ def configure(self) -> None: - [ ] Workflow class in `src/` subclassing `WorkflowManager` - [ ] `__init__` calls `super().__init__("Name", st.session_state["workspace"])` - [ ] `upload()`, `configure()`, `execution()`, `results()` implemented +- [ ] `execution()` annotated `-> bool`, returning `True` on success and `False` on any failure - [ ] `@st.fragment` on `configure()` and `results()` - [ ] `reactive=True` on any widget whose value controls other widgets' visibility - [ ] 4 content pages created in `content/` diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 00000000..6e46e983 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,5 @@ +# Shell scripts are executed by /bin/sh inside Linux containers and by POSIX +# shells in tests/test_seed_demos.py. A CRLF checkout on Windows breaks both: +# sh reports "not found" on every carriage return, and a shebang with a +# trailing CR fails to exec. Pin them to LF regardless of core.autocrlf. +*.sh text eol=lf diff --git a/.github/kind-config.yaml b/.github/kind-config.yaml index 4e729b5b..2ebfcbfa 100644 --- a/.github/kind-config.yaml +++ b/.github/kind-config.yaml @@ -1,27 +1,35 @@ kind: Cluster apiVersion: kind.x-k8s.io/v1alpha4 -# Two-node cluster that mirrors the production memory-tier topology, so -# the Build-and-Test job passes regardless of which tier a fork selects -# in its overlay (memory-tier-low or memory-tier-high). Without both -# labels present, flipping the overlay would leave pods Pending on the -# single kind node. +# Two schedulable nodes, which is the smallest cluster that can observe the +# property the deployment now rests on: workers spread across hosts, and one +# workspace volume reachable from all of them. On a single node every spread +# constraint is satisfied vacuously - maxSkew is measured over eligible +# domains and one domain is always perfectly balanced - so a one-node cluster +# would report success for exactly the arrangement this suite exists to rule +# out. +# +# No node labels. Nothing under k8s/ selects a node any more: the scheduler +# places pods, and the manifests only declare how big a worker is. The nodes +# carried openms.de/memory-tier=low and =high while the memory-tier components +# patched a matching nodeSelector onto every Deployment; those patches are +# gone, and leaving the labels behind would imply the pinning had survived +# somewhere. assert_no_node_pinning_anywhere text-scans this file for exactly +# that. nodes: - role: control-plane # Multi-node kind clusters taint the control-plane with - # node-role.kubernetes.io/control-plane:NoSchedule. Clear it so - # app pods with nodeSelector memory-tier=low can actually land - # here (single-node kind had no such taint, hence the original - # workflow passed without this patch). + # node-role.kubernetes.io/control-plane:NoSchedule. Clear it so app pods + # can land here as well: with the taint in place there is one schedulable + # node, and every cross-node assertion - two pods on two nodes, the + # cross-node write, the cross-node flock, the worker spread - collapses + # into a tautology it cannot fail. kubeadmConfigPatches: - | kind: InitConfiguration nodeRegistration: taints: [] labels: - openms.de/memory-tier: low # ingress-ready is required by the kind variant of the # ingress-nginx deploy manifest applied in CI. ingress-ready: "true" - role: worker - labels: - openms.de/memory-tier: high diff --git a/.github/scripts/ci-assertions.sh b/.github/scripts/ci-assertions.sh new file mode 100644 index 00000000..aeb9d15a --- /dev/null +++ b/.github/scripts/ci-assertions.sh @@ -0,0 +1,2014 @@ +#!/usr/bin/env bash +# Cluster and manifest assertions for the multi-node deployment work. +# +# Source this file inside a workflow step, then call the assertions: +# +# - name: Assert cross-node write visibility +# run: | +# source .github/scripts/ci-assertions.sh +# assert_cross_node_write_visible +# +# Every assertion prints exactly one `PASS: ...` or `FAIL: ...` line and +# returns non-zero on failure, so it fails the step under the default `bash -e` +# shell. None of them can pass vacuously: an assertion that finds nothing to +# inspect fails rather than reporting success. +# +# assert_two_pods_two_nodes two pods, one workspace volume, two nodes +# assert_cross_node_write_visible each pod reads the other pod's writes +# assert_posix_contract the mount really is NFS; absolute symlink, +# atomic rename, and flock that excludes a +# holder on the *other* node +# assert_survives_nfs_restart a workflow running throughout a Ganesha +# restart completes, with no lost output +# assert_stable_identity_across_restart +# file handles survive a Ganesha restart +# assert_fsids_pinned static; the storage root pins +# device-based-fsids off +# assert_netpol_string_matches_overlay +# assert_netpol_admits_every_node +# assert_storage_identity_values +# static; the storage NetworkPolicy admits +# exactly the overlay's commonLabels.app, in +# the overlay's namespace, on 2049 alone +# assert_no_node_pinning_anywhere static; no nodeSelector, nodeName, +# nodeAffinity or memory tier +# assert_doc_links_resolve static; every backticked *.md path in +# tracked Markdown names a tracked file +# assert_workers_spread_across_nodes +# every worker replica is Running, on more +# than one node, within the declared maxSkew +# assert_worker_qos_guaranteed worker pods are Running and Guaranteed QoS +# +# Inputs, all optional; the defaults are discovered from the cluster: +# CI_ASSERT_NS namespace of the deployment (default: openms) +# CI_ASSERT_APP value of the `app` label (default: $SLUG, else +# .commonLabels.app of the overlay) +# CI_ASSERT_OVERLAY kustomize root to render (default: k8s/overlays/prod) +# CI_ASSERT_BASE kustomize root the overlay (default: k8s/base) +# builds on, read for its namespace +# CI_ASSERT_CLAIM workspace PVC name (default: discovered) +# CI_ASSERT_MOUNT workspace mount path (default: the WORKSPACES_DIR +# the deployments use) +# CI_ASSERT_IMAGE image for throwaway pods (default: the app image, +# which is already on every kind node so nothing pulls) +# CI_ASSERT_TIMEOUT seconds to wait for a pod (default: 180) +# CI_ASSERT_STORAGE_NS namespace of the NFS server +# (default: template-app-storage) +# CI_ASSERT_STORAGE_ROOT kustomize root serving it (default: k8s/storage) + +CI_ASSERT_NS="${CI_ASSERT_NS:-openms}" +CI_ASSERT_OVERLAY="${CI_ASSERT_OVERLAY:-k8s/overlays/prod}" +# The overlay the kind jobs actually apply. Scanned for pinning too: +# prod is the one under test, but a nodeSelector reintroduced here would +# pin every pod CI ever schedules while prod stayed clean. +CI_ASSERT_CI_OVERLAY="${CI_ASSERT_CI_OVERLAY:-k8s/overlays/ci}" +CI_ASSERT_MOUNT="${CI_ASSERT_MOUNT:-/workspaces-streamlit-template}" +CI_ASSERT_TIMEOUT="${CI_ASSERT_TIMEOUT:-180}" +CI_ASSERT_BASE="${CI_ASSERT_BASE:-k8s/base}" +CI_ASSERT_STORAGE_NS="${CI_ASSERT_STORAGE_NS:-template-app-storage}" +CI_ASSERT_STORAGE_ROOT="${CI_ASSERT_STORAGE_ROOT:-k8s/storage}" +# Seconds to wait for the NFS server to come back, and for a client operation to +# get through afterwards. NFSv4.1's grace period is ~90s by default, during +# which state-mutating operations are refused and a `hard` mount simply retries. +CI_ASSERT_RESTART_TIMEOUT="${CI_ASSERT_RESTART_TIMEOUT:-300}" +# assert_survives_nfs_restart: how many one-per-second records the stand-in +# workflow writes, and how long it is then given to finish. The workflow has to +# outlast the restart *plus* the ~90s grace period with time to spare, or the +# assertion measures the timeout instead of the filesystem. +CI_ASSERT_WORKFLOW_RECORDS="${CI_ASSERT_WORKFLOW_RECORDS:-60}" +CI_ASSERT_WORKFLOW_TIMEOUT="${CI_ASSERT_WORKFLOW_TIMEOUT:-420}" +# Trees scanned by assert_no_node_pinning_anywhere. Space separated. +CI_ASSERT_PINNING_PATHS="${CI_ASSERT_PINNING_PATHS:-k8s .github/kind-config.yaml}" + +# --- reporting ------------------------------------------------------------- +# +# _ci_fail only reports; the caller decides when to return, so that cleanup +# still runs and every failure of a multi-part assertion is printed rather than +# just the first one. + +_ci_pass() { + printf 'PASS: %s\n' "$*" +} + +_ci_fail() { + printf 'FAIL: %s\n' "$*" >&2 + if [ -n "${GITHUB_ACTIONS:-}" ]; then + printf '::error title=CI assertion failed::%s\n' "$*" + fi +} + +# --- discovery ------------------------------------------------------------- + +_ci_have() { + command -v "$1" >/dev/null 2>&1 +} + +_ci_require() { + # Fail loudly on a missing tool rather than silently skipping the check. + local missing="" _ci_tool + for _ci_tool in "$@"; do + if ! _ci_have "$_ci_tool"; then + missing="$missing $_ci_tool" + fi + done + if [ -n "$missing" ]; then + _ci_fail "required tool(s) not on PATH:$missing" + return 1 + fi + return 0 +} + +_ci_app() { + # The `app` label every object carries, from commonLabels in the overlay. + if [ -n "${CI_ASSERT_APP:-}" ]; then + printf '%s\n' "$CI_ASSERT_APP" + return 0 + fi + if [ -n "${SLUG:-}" ]; then + printf '%s\n' "$SLUG" + return 0 + fi + local app="" + if _ci_have yq && [ -f "$CI_ASSERT_OVERLAY/kustomization.yaml" ]; then + app="$(yq '.commonLabels.app' "$CI_ASSERT_OVERLAY/kustomization.yaml" 2>/dev/null || true)" + fi + if [ "$app" = "null" ]; then + app="" + fi + printf '%s\n' "$app" +} + +_ci_claim() { + # Name of the workspace PVC, read off whichever Deployment mounts it, so + # the assertions keep working when namePrefix changes. + if [ -n "${CI_ASSERT_CLAIM:-}" ]; then + printf '%s\n' "$CI_ASSERT_CLAIM" + return 0 + fi + kubectl get deployment -n "$CI_ASSERT_NS" -l "app=$1" \ + -o jsonpath='{range .items[*]}{range .spec.template.spec.volumes[?(@.name=="workspaces")]}{.persistentVolumeClaim.claimName}{"\n"}{end}{end}' \ + 2>/dev/null | sed '/^$/d' | head -n 1 || true +} + +_ci_image() { + # The app image is already present on every kind node (`kind load + # image-archive`), so a helper pod built from it never pulls, and its + # userland is the one the app actually runs on. + if [ -n "${CI_ASSERT_IMAGE:-}" ]; then + printf '%s\n' "$CI_ASSERT_IMAGE" + return 0 + fi + local image + image="$(kubectl get deployment -n "$CI_ASSERT_NS" -l "app=$1,component=streamlit" \ + -o jsonpath='{.items[0].spec.template.spec.containers[0].image}' 2>/dev/null || true)" + if [ -z "$image" ]; then + image="ubuntu:22.04" + fi + printf '%s\n' "$image" +} + +_ci_nodes_for_selector() { + # Distinct nodes hosting Running pods that match a label selector, used by + # assert_workers_spread_across_nodes. Running is filtered on server-side: + # a Pending pod already carries every label and would otherwise be counted + # as though it had landed somewhere. + kubectl get pods -n "$CI_ASSERT_NS" -l "$1" \ + --field-selector=status.phase=Running \ + -o jsonpath='{range .items[*]}{.spec.nodeName}{"\n"}{end}' 2>/dev/null \ + | sed '/^$/d' | sort -u || true +} + +_ci_ready_node_names() { + # Every Ready node, one per line. The denominator for a spread check: a + # skew computed only over the nodes that already hold a worker cannot see + # the node that holds none, which is the interesting case. + kubectl get nodes --no-headers 2>/dev/null | awk '$2 == "Ready" { print $1 }' || true +} + +_ci_ready_pods() { + # `` for every Running *and* Ready pod in a namespace. + # + # The uid, not the name, is what identifies a pod across a restart here: + # the NFS server is a StatefulSet, so its replacement comes back as the + # same `nfs-server-0`, and a name comparison would wait forever for a name + # that is never going to change. + # + # `kubectl wait --all` cannot stand in for any of this either: it reports + # success against zero pods, which is precisely the state a restart + # assertion has to catch. + kubectl get pods -n "$1" -o json 2>/dev/null | jq -r ' + .items[] + | select(.status.phase == "Running") + | select([.status.conditions[]? | select(.type == "Ready") | .status] | index("True")) + | "\(.metadata.name)\t\(.metadata.uid)"' 2>/dev/null || true +} + +_ci_kustomize_storage() { + # Render the storage root. --enable-helm because that root inflates the + # Ganesha chart; the flag is inert on a root with no helmCharts field, so it + # costs nothing if the chart is ever vendored as plain manifests instead. + kubectl kustomize --enable-helm "$CI_ASSERT_STORAGE_ROOT" +} + +_ci_two_ready_nodes() { + # The names of two distinct Ready nodes, one per line, or nothing at all. + kubectl get nodes --no-headers 2>/dev/null | awk '$2 == "Ready" { print $1 }' | head -n 2 || true +} + +_ci_pinning_files() { + # The files assert_no_node_pinning_anywhere text-scans, one per line. + # + # git-tracked files PLUS untracked ones that are not gitignored. The + # ignored half is the part that matters: `kubectl kustomize --enable-helm + # k8s/storage/` makes kustomize vendor the Ganesha chart into + # k8s/storage/charts/, and that chart carries its own `nodeSelector: {}` + # default plus the template and README lines that go with it. Scanning the + # raw working tree therefore turns this assertion permanently red on any + # checkout where the storage root has been rendered once - which, in the + # assert-invariants job, is guaranteed, because the NetworkPolicy step + # renders it first. k8s/storage/charts/ is .gitignored, so + # `--exclude-standard` prunes it by construction. + # + # Tracked-only would have been simpler and was wrong: a new manifest is + # untracked until `git add`, so a nodeSelector added in a new file under + # k8s/ would have reported PASS right up to the moment it was committed - + # and k8s/overlays/ci/ and k8s/storage/*.yaml were themselves untracked + # while this branch was being built, i.e. unscanned exactly when they were + # being written. The one thing still skipped is deliberately-ignored + # files, and the pass message says so. + local p + if git rev-parse --is-inside-work-tree >/dev/null 2>&1; then + for p in $CI_ASSERT_PINNING_PATHS; do + git ls-files --cached --others --exclude-standard -- "$p" 2>/dev/null || true + done + return 0 + fi + # No git (a tarball checkout): fall back to the filesystem, pruning any + # vendored chart directory by name. + for p in $CI_ASSERT_PINNING_PATHS; do + [ -e "$p" ] || continue + find "$p" -type d -name charts -prune -o -type f -print 2>/dev/null || true + done +} + +# --- throwaway pods -------------------------------------------------------- + +_ci_helper_pod_manifest() { + # _ci_helper_pod_manifest + # + # nodeName, not nodeSelector: bypassing the scheduler keeps a failure + # unambiguously about the volume rather than about taints or affinity, and + # this repo is in the business of removing nodeSelectors. + local nodeline="" + if [ -n "$2" ]; then + nodeline=" nodeName: $2" + fi + cat < + if ! _ci_helper_pod_manifest "$@" | kubectl apply -f - >/dev/null; then + return 1 + fi + if ! kubectl wait -n "$CI_ASSERT_NS" --for=condition=Ready "pod/$1" \ + --timeout="${CI_ASSERT_TIMEOUT}s" >/dev/null; then + printf 'helper pod %s never became Ready:\n' "$1" >&2 + kubectl describe pod -n "$CI_ASSERT_NS" "$1" >&2 || true + return 1 + fi + return 0 +} + +_ci_delete_pods() { + if [ "$#" -gt 0 ]; then + kubectl delete pod -n "$CI_ASSERT_NS" "$@" \ + --ignore-not-found --wait=false >/dev/null 2>&1 || true + fi +} + +_ci_exec_script() { + # _ci_exec_script [arg...] < script + # + # The script arrives on stdin, so nothing has to survive a round of shell + # quoting. /bin/sh in the app image is dash: keep those scripts POSIX. + local pod="$1" + shift + kubectl exec -i -n "$CI_ASSERT_NS" "$pod" -- /bin/sh -s "$@" +} + +# --- assertions ------------------------------------------------------------ + +assert_two_pods_two_nodes() { + # The acceptance criterion for the whole project: two pods that mount the + # *same* workspace claim, Running on two different nodes. Filtering by the + # claim matters - redis mounts nothing, so counting app pods alone would go + # green while the shared workspace was still stuck on one node. + _ci_require kubectl jq || return 1 + _ci_min_nodes="${1:-2}" + _ci_rc=0 + + _ci_a="$(_ci_app)" + if [ -z "$_ci_a" ]; then + _ci_fail "cannot determine the app label (set SLUG or CI_ASSERT_APP)" + return 1 + fi + _ci_c="$(_ci_claim "$_ci_a")" + if [ -z "$_ci_c" ]; then + _ci_fail "no Deployment in $CI_ASSERT_NS mounts a 'workspaces' volume - nothing to assert" + return 1 + fi + + _ci_rows="$(kubectl get pods -n "$CI_ASSERT_NS" -l "app=$_ci_a" -o json 2>/dev/null \ + | jq -r --arg claim "$_ci_c" ' + .items[] + | select(.status.phase == "Running") + | select([.spec.volumes[]? | .persistentVolumeClaim.claimName? // empty] | index($claim)) + | "\(.metadata.name)\t\(.spec.nodeName)"' 2>/dev/null || true)" + _ci_rows="$(printf '%s\n' "$_ci_rows" | sed '/^$/d')" + printf 'pods mounting %s:\n%s\n' "$_ci_c" "${_ci_rows:- }" + + _ci_pods="$(printf '%s\n' "$_ci_rows" | sed '/^$/d' | wc -l | tr -d ' ')" + if [ "$_ci_pods" -lt 2 ]; then + _ci_fail "only $_ci_pods Running pod(s) mount $_ci_c - two are needed to prove the volume is shared" + _ci_rc=1 + fi + + _ci_nodes="$(printf '%s\n' "$_ci_rows" | awk -F'\t' 'NF > 1 && $2 != "" { print $2 }' | sort -u)" + _ci_n="$(printf '%s\n' "$_ci_nodes" | sed '/^$/d' | wc -l | tr -d ' ')" + if [ "$_ci_n" -lt "$_ci_min_nodes" ]; then + _ci_fail "pods mounting $_ci_c occupy $_ci_n node(s), expected at least $_ci_min_nodes: $(printf '%s' "$_ci_nodes" | tr '\n' ' ')" + _ci_rc=1 + fi + + if [ "$_ci_rc" -eq 0 ]; then + _ci_pass "$_ci_pods pods mount $_ci_c across $_ci_n nodes: $(printf '%s' "$_ci_nodes" | tr '\n' ' ')" + fi + return "$_ci_rc" +} + +assert_cross_node_write_visible() { + # A pod on node A and a pod on node B, both holding the workspace volume at + # the same time, each reading what the other wrote. Concurrency is the + # point: a sequential write-then-read passes on an RWO volume that simply + # detached and re-attached, which proves nothing. + _ci_require kubectl || return 1 + _ci_rc=0 + + _ci_a="$(_ci_app)" + if [ -z "$_ci_a" ]; then + _ci_fail "cannot determine the app label (set SLUG or CI_ASSERT_APP)" + return 1 + fi + _ci_c="$(_ci_claim "$_ci_a")" + if [ -z "$_ci_c" ]; then + _ci_fail "no Deployment in $CI_ASSERT_NS mounts a 'workspaces' volume - nothing to assert" + return 1 + fi + _ci_img="$(_ci_image "$_ci_a")" + + _ci_ready_nodes="$(kubectl get nodes --no-headers 2>/dev/null | awk '$2 == "Ready" { print $1 }' || true)" + _ci_node_a="$(printf '%s\n' "$_ci_ready_nodes" | sed -n '1p')" + _ci_node_b="$(printf '%s\n' "$_ci_ready_nodes" | sed -n '2p')" + if [ -z "$_ci_node_b" ]; then + _ci_fail "cluster has fewer than two Ready nodes - cross-node visibility cannot be asserted" + return 1 + fi + + _ci_stamp="$(date +%s)" + _ci_writer="ci-xnode-writer-$_ci_stamp" + _ci_reader="ci-xnode-reader-$_ci_stamp" + _ci_dir="$CI_ASSERT_MOUNT/.ci-cross-node-$_ci_stamp" + _ci_token_w="written-on-$_ci_node_a" + _ci_token_r="written-on-$_ci_node_b" + + if ! _ci_start_helper_pod "$_ci_writer" "$_ci_node_a" "$_ci_img" "$_ci_c"; then + _ci_fail "writer pod could not start on $_ci_node_a" + _ci_delete_pods "$_ci_writer" + return 1 + fi + # The reader has to come up while the writer still holds the mount. + if ! _ci_start_helper_pod "$_ci_reader" "$_ci_node_b" "$_ci_img" "$_ci_c"; then + _ci_fail "reader pod could not start on $_ci_node_b while the writer holds $_ci_c on $_ci_node_a" + _ci_delete_pods "$_ci_writer" "$_ci_reader" + return 1 + fi + + _ci_exec_script "$_ci_writer" "$_ci_dir" "$_ci_token_w" <<'WRITE' || _ci_rc=1 +set -e +mkdir -p "$1" +printf '%s' "$2" > "$1/from-writer" +WRITE + if [ "$_ci_rc" -ne 0 ]; then + _ci_fail "writer pod on $_ci_node_a could not write to $_ci_dir" + fi + + if [ "$_ci_rc" -eq 0 ]; then + _ci_got="$(_ci_exec_script "$_ci_reader" "$_ci_dir/from-writer" "$_ci_dir" "$_ci_token_r" <<'READ' || true +i=0 +while [ "$i" -lt 30 ]; do + if [ -f "$1" ]; then + cat "$1" + mkdir -p "$2" + printf '%s' "$3" > "$2/from-reader" + exit 0 + fi + i=$((i + 1)) + sleep 2 +done +exit 1 +READ +)" + if [ "$_ci_got" != "$_ci_token_w" ]; then + _ci_fail "pod on $_ci_node_b read '$_ci_got' from $_ci_dir/from-writer, expected '$_ci_token_w'" + _ci_rc=1 + fi + fi + + if [ "$_ci_rc" -eq 0 ]; then + _ci_got="$(_ci_exec_script "$_ci_writer" "$_ci_dir/from-reader" <<'READBACK' || true +i=0 +while [ "$i" -lt 30 ]; do + if [ -f "$1" ]; then + cat "$1" + exit 0 + fi + i=$((i + 1)) + sleep 2 +done +exit 1 +READBACK +)" + if [ "$_ci_got" != "$_ci_token_r" ]; then + _ci_fail "pod on $_ci_node_a read '$_ci_got' from $_ci_dir/from-reader, expected '$_ci_token_r'" + _ci_rc=1 + fi + fi + + _ci_exec_script "$_ci_writer" "$_ci_dir" <<'CLEAN' >/dev/null 2>&1 || true +rm -rf "$1" +CLEAN + _ci_delete_pods "$_ci_writer" "$_ci_reader" + + if [ "$_ci_rc" -eq 0 ]; then + _ci_pass "$_ci_node_a and $_ci_node_b hold $_ci_c at the same time and each reads the other's writes" + fi + return "$_ci_rc" +} + +assert_posix_contract() { + # The filesystem guarantees src/ depends on, checked on the real mount: + # the mount is NFS without this the whole assertion is vacuous. On the + # pre-change cluster the helper lands on the node + # holding the RWO volume and every check below passes + # against overlayfs, proving nothing about the export + # absolute symlinks common.py:179, :678 and fileupload.py:91 all call + # symlink_to(x.resolve()), so demo seeding breaks if + # readlink does not return the absolute target + # atomic rename the seed-demos initContainer replaces `cp -rn` with + # copy-to-temp plus `mv -T` + # flock ACROSS NODES the property the locking design actually needs. A + # single-pod flock test proves only local semantics: + # an NFS mount with local_lock=all, or a client-side + # emulation, passes it and still leaves locking a + # no-op between the two workers. So the second holder + # is a pod on the other node. If this half fails, the + # lock backend moves from flock to Redis. + _ci_require kubectl || return 1 + _ci_rc=0 + + _ci_a="$(_ci_app)" + if [ -z "$_ci_a" ]; then + _ci_fail "cannot determine the app label (set SLUG or CI_ASSERT_APP)" + return 1 + fi + _ci_c="$(_ci_claim "$_ci_a")" + if [ -z "$_ci_c" ]; then + _ci_fail "no Deployment in $CI_ASSERT_NS mounts a 'workspaces' volume - nothing to assert" + return 1 + fi + _ci_img="$(_ci_image "$_ci_a")" + + _ci_ready_nodes="$(_ci_two_ready_nodes)" + _ci_node_a="$(printf '%s\n' "$_ci_ready_nodes" | sed -n '1p')" + _ci_node_b="$(printf '%s\n' "$_ci_ready_nodes" | sed -n '2p')" + if [ -z "$_ci_node_b" ]; then + _ci_fail "cluster has fewer than two Ready nodes - cross-client locking cannot be asserted" + return 1 + fi + + _ci_stamp="$(date +%s)" + _ci_holder="ci-posix-holder-$_ci_stamp" + _ci_rival="ci-posix-rival-$_ci_stamp" + _ci_dir="$CI_ASSERT_MOUNT/.ci-posix-contract-$_ci_stamp" + + if ! _ci_start_helper_pod "$_ci_holder" "$_ci_node_a" "$_ci_img" "$_ci_c"; then + _ci_fail "helper pod could not start on $_ci_node_a with $_ci_c mounted at $CI_ASSERT_MOUNT" + _ci_delete_pods "$_ci_holder" + return 1 + fi + if ! _ci_start_helper_pod "$_ci_rival" "$_ci_node_b" "$_ci_img" "$_ci_c"; then + _ci_fail "helper pod could not start on $_ci_node_b while $_ci_holder holds $_ci_c on $_ci_node_a" + _ci_delete_pods "$_ci_holder" "$_ci_rival" + return 1 + fi + + # Pod A: prove the mount is NFS, then symlink, rename, and take the lock. + # The holder runs detached with its output closed so `kubectl exec` returns + # while it still holds the lock; it releases when pod B drops the release + # file, so nothing here depends on a fixed sleep. + _ci_exec_script "$_ci_holder" "$CI_ASSERT_MOUNT" "$_ci_dir" <<'POSIXA' || _ci_rc=1 +set -e +m="$1" +d="$2" + +fstype="$(awk -v mp="$m" '$2 == mp { t = $3 } END { print t }' /proc/mounts)" +opts="$(awk -v mp="$m" '$2 == mp { o = $4 } END { print o }' /proc/mounts)" +echo "mount: $m type=${fstype:-} opts=${opts:-}" +case "$fstype" in + nfs|nfs4) ;; + "") + echo "mount: $m is not a mount point in this pod at all" + exit 1 + ;; + *) + echo "mount: $m is '$fstype', not NFS - this assertion would be measuring the node's local filesystem" + exit 1 + ;; +esac + +rm -rf "$d" +mkdir -p "$d" + +# 1. an absolute symlink survives readlink and can be read through +printf 'payload' > "$d/target" +ln -s "$d/target" "$d/link" +got="$(readlink "$d/link")" +if [ "$got" != "$d/target" ]; then + echo "symlink: readlink returned '$got', expected '$d/target'" + exit 1 +fi +if [ "$(cat "$d/link")" != "payload" ]; then + echo "symlink: reading through the link did not return the target's content" + exit 1 +fi + +# 2. rename over an existing file, and rename of a directory (mv -T) +printf 'old' > "$d/dst" +printf 'new' > "$d/src" +mv "$d/src" "$d/dst" +if [ "$(cat "$d/dst")" != "new" ]; then + echo "rename: destination was not replaced" + exit 1 +fi +if [ -e "$d/src" ]; then + echo "rename: source still exists after mv" + exit 1 +fi +mkdir "$d/dsrc" +: > "$d/dsrc/f" +mv -T "$d/dsrc" "$d/ddst" +if [ ! -f "$d/ddst/f" ]; then + echo "rename: directory rename lost its contents" + exit 1 +fi + +# 3. take an exclusive lock and hold it until the other node says otherwise +: > "$d/lock" +( + exec 9> "$d/lock" + flock -x 9 + : > "$d/held" + i=0 + while [ ! -f "$d/release" ] && [ "$i" -lt 180 ]; do + i=$((i + 1)) + sleep 1 + done +) > /dev/null 2>&1 & + +i=0 +while [ ! -f "$d/held" ] && [ "$i" -lt 30 ]; do + i=$((i + 1)) + sleep 1 +done +if [ ! -f "$d/held" ]; then + echo "flock: the holder on this node never acquired the lock" + exit 1 +fi +echo "absolute symlink and atomic rename behave; exclusive lock held on this node" +POSIXA + + if [ "$_ci_rc" -ne 0 ]; then + _ci_fail "the POSIX contract does not hold on $_ci_c at $CI_ASSERT_MOUNT (pod on $_ci_node_a)" + _ci_exec_script "$_ci_holder" "$_ci_dir" <<'CLEAN' > /dev/null 2>&1 || true +: > "$1/release" +rm -rf "$1" +CLEAN + _ci_delete_pods "$_ci_holder" "$_ci_rival" + return 1 + fi + + # Pod B, on the other node: the same mount has to be NFS here too, the + # holder's lock has to exclude it, and once released it has to be takeable - + # otherwise "cannot acquire" would prove nothing but a broken lock file. + _ci_exec_script "$_ci_rival" "$CI_ASSERT_MOUNT" "$_ci_dir" <<'POSIXB' || _ci_rc=1 +m="$1" +d="$2" +rc=0 + +fstype="$(awk -v mp="$m" '$2 == mp { t = $3 } END { print t }' /proc/mounts)" +echo "mount: $m type=${fstype:-}" +case "$fstype" in + nfs|nfs4) ;; + *) + echo "mount: $m is '${fstype:-}', not NFS, on this node" + exit 1 + ;; +esac + +i=0 +while [ ! -f "$d/held" ] && [ "$i" -lt 60 ]; do + i=$((i + 1)) + sleep 1 +done +if [ ! -f "$d/held" ]; then + echo "flock: the other node's lock never became visible here" + exit 1 +fi + +if flock -n -x "$d/lock" true; then + echo "flock: this node took a lock the other node already holds, so locking is a no-op between clients" + rc=1 +fi + +: > "$d/release" + +# Generous, because the holder learns about the release file through the +# directory attribute cache: with default actimeo a negative lookup can be +# cached for up to acdirmax, 60s, before it revalidates and sees the file. +if ! flock -w 180 -x "$d/lock" true; then + echo "flock: could not take the lock even after the other node released it" + rc=1 +fi + +if [ "$(readlink "$d/link")" != "$d/target" ]; then + echo "symlink: the absolute symlink does not resolve on this node" + rc=1 +fi + +exit "$rc" +POSIXB + + _ci_exec_script "$_ci_holder" "$_ci_dir" <<'CLEAN' > /dev/null 2>&1 || true +: > "$1/release" +rm -rf "$1" +CLEAN + _ci_delete_pods "$_ci_holder" "$_ci_rival" + + if [ "$_ci_rc" -ne 0 ]; then + _ci_fail "the POSIX contract does not hold across nodes on $_ci_c at $CI_ASSERT_MOUNT" + return 1 + fi + _ci_pass "POSIX contract holds on $_ci_c: NFS mount, absolute symlink, atomic rename, and flock that excludes a holder on $_ci_node_a from $_ci_node_b" + return 0 +} + +_ci_scan_rendered_for_pinning() { + # Report every object in a RENDERED manifest stream that pins pods to a + # node. Factored out of assert_no_node_pinning_anywhere because three + # separate roots have to be scanned and they are not rendered the same way. + # + # PersistentVolume is skipped whole. `spec.nodeAffinity` on a PV is volume + # TOPOLOGY - where the bytes are - not pod pinning, and it is mandatory for + # a local volume. Nothing in the roots scanned today renders a PV, since the + # provisioner creates them at runtime, but this function now also reads the + # storage root, and a chart that ever emitted a static PV would otherwise + # turn invariant 1 permanently red with no way to make it green except + # weakening the check. + # + # $1 rendered file, $2 label printed against each hit. + awk -v src="$2" ' + /^---[[:space:]]*$/ { kind = ""; name = ""; inmeta = 0; next } + /^kind:/ && kind == "" { kind = $2; next } + /^metadata:/ { inmeta = 1; next } + /^[^[:space:]]/ { inmeta = 0 } + inmeta && /^ name:/ && name == "" { name = $2 } + kind == "PersistentVolume" { next } + /^[[:space:]]*(nodeSelector|nodeName|nodeAffinity)[[:space:]]*:/ { + key = $1 + sub(/:.*$/, "", key) + print " " src ": " kind "/" name " pins pods with " key " (rendered line " NR ")" + } + ' "$1" || true +} + +assert_no_node_pinning_anywhere() { + # Invariant 1 as a test: Kubernetes decides placement, the config must not. + # Static, so it needs no cluster: + # 1. no object rendered from the overlay pins a pod, by nodeSelector, + # nodeName or nodeAffinity + # 2. no manifest in the tree mentions any of those, or the + # openms.de/memory-tier label (see _ci_pinning_files for what "in the + # tree" excludes, and why) + # It also stops the pinning being reintroduced by a later fork rebase, + # which is exactly how it would come back. + # helm as well as kubectl, because the storage root is one of the three + # roots scanned below and inflating it shells out to helm. A hard + # requirement, deliberately not `if command -v helm`: a check that silently + # does less work when a tool is missing is the shape this round exists to + # remove. + _ci_require kubectl helm || return 1 + _ci_rc=0 + + # THREE rendered roots, not one. The Ganesha StatefulSet is the pod whose + # placement mattered most in this work, and until now nothing inspected it: + # the rendered half only ever rendered the prod overlay, and the text half + # below asks git, which prunes the vendored chart under k8s/storage/charts/ + # through .gitignore. k8s/overlays/ci/ was equally unseen, and it is what + # the kind jobs actually apply - a nodeSelector reintroduced there would pin + # every pod CI ever schedules while prod stayed clean. + _ci_hits="" + _ci_roots_scanned="" + for _ci_root in "$CI_ASSERT_OVERLAY" "$CI_ASSERT_CI_OVERLAY" "$CI_ASSERT_STORAGE_ROOT"; do + # An empty root is a hard failure, not a `continue`. All three are + # ${VAR:-default} overridable, so emptying one silently removed a third + # of this check while the PASS below went on naming all three - the + # exact silent-skip shape the hard `_ci_require kubectl helm` above was + # added to remove. Do NOT relax this into an opt-out "to unblock CI": + # that would make an empty root the documented way to switch the check + # off while its message keeps claiming full coverage. + if [ -z "$_ci_root" ]; then + _ci_fail "one of CI_ASSERT_OVERLAY / CI_ASSERT_CI_OVERLAY / CI_ASSERT_STORAGE_ROOT is empty, so a third of this check would be skipped in silence" + return 1 + fi + _ci_rendered="$(mktemp)" + _ci_err="$(mktemp)" + _ci_ok=0 + if [ "$_ci_root" = "$CI_ASSERT_STORAGE_ROOT" ]; then + _ci_kustomize_storage > "$_ci_rendered" 2>"$_ci_err" && _ci_ok=1 + else + kubectl kustomize "$_ci_root" > "$_ci_rendered" 2>"$_ci_err" && _ci_ok=1 + fi + if [ "$_ci_ok" -ne 1 ]; then + _ci_fail "kubectl kustomize $_ci_root failed" + cat "$_ci_err" >&2 || true + rm -f "$_ci_rendered" "$_ci_err" + return 1 + fi + rm -f "$_ci_err" + if [ ! -s "$_ci_rendered" ]; then + _ci_fail "kubectl kustomize $_ci_root rendered nothing - the check would pass vacuously" + rm -f "$_ci_rendered" + return 1 + fi + # One line per root as it is scanned. Without this the step emitted only + # its PASS sentence, so nothing in the log substantiated the claim that + # three roots had been rendered at all. + # `^kind:` rather than `^---`: kustomize writes a separator BETWEEN + # documents, so counting those is always one short. Verified against + # the prod overlay - 13 objects, 13 `^kind:` lines, 12 separators. + # Counted with grep rather than yq so this assertion keeps requiring + # only kubectl and helm. + printf 'rendered %s: %s documents\n' "$_ci_root" \ + "$(grep -c '^kind:' "$_ci_rendered" 2>/dev/null || echo '?')" + _ci_roots_scanned="$_ci_roots_scanned $_ci_root" + _ci_found="$(_ci_scan_rendered_for_pinning "$_ci_rendered" "$_ci_root")" + rm -f "$_ci_rendered" + if [ -n "$_ci_found" ]; then + if [ -z "$_ci_hits" ]; then + _ci_hits="$_ci_found" + else + _ci_hits="$(printf '%s +%s' "$_ci_hits" "$_ci_found")" + fi + fi + done + if [ -n "$_ci_hits" ]; then + _ci_fail "a rendered root still contains objects that pin pods to nodes" + printf '%s +' "$_ci_hits" >&2 + _ci_rc=1 + fi + + # The text half runs over COMMENT-STRIPPED lines. `.github/kind-config.yaml` + # explains its control-plane taint patch in prose that contains the word + # nodeSelector, and a raw grep matches that comment for ever - including + # after the labels it describes have been deleted, which would leave this + # assertion red with no way left to make it green except weakening it. + # + # Keys are matched at the start of a mapping entry, or as a JSON-patch + # `path:` segment - which is how k8s/components/memory-tier-*/ spell it - + # so the Downward API reference `fieldPath: spec.nodeName` in the worker + # Deployment, which pins nothing, is not mistaken for pinning. + _ci_pin_re='(^|[[:space:]/])(nodeSelector|nodeName|nodeAffinity)([[:space:]]*:|[[:space:]]*$|/)|openms\.de/memory-tier' + _ci_scanned=0 + _ci_hits="" + for _ci_file in $(_ci_pinning_files); do + [ -f "$_ci_file" ] || continue + _ci_scanned=$((_ci_scanned + 1)) + _ci_tmp="$(mktemp)" + sed 's/#.*$//' "$_ci_file" > "$_ci_tmp" + # No `| grep -v nodeSelectorTerms` here. That filter looked like it was + # suppressing a false positive, and was not: `nodeSelectorTerms:` on its + # own line does not match _ci_pin_re at all, because the pattern requires + # `nodeSelector` to be followed by `:`, end-of-line or `/`, and there it + # is followed by `Terms:`. Its only effect was to drop lines that DID + # match and happened to contain the substring - i.e. a flow-style + # `nodeAffinity: {required: {nodeSelectorTerms: [...]}}`, which is + # genuine pod pinning. PersistentVolume topology is not a concern for + # this half: the only `kind: PersistentVolume` in the tree is inside the + # vendored chart's README, which .gitignore prunes from _ci_pinning_files. + _ci_found="$(grep -nE "$_ci_pin_re" "$_ci_tmp" || true)" + rm -f "$_ci_tmp" + if [ -n "$_ci_found" ]; then + _ci_found="$(printf '%s' "$_ci_found" | sed "s|^| $_ci_file:|")" + if [ -z "$_ci_hits" ]; then + _ci_hits="$_ci_found" + else + _ci_hits="$(printf '%s\n%s' "$_ci_hits" "$_ci_found")" + fi + fi + done + if [ "$_ci_scanned" -eq 0 ]; then + _ci_fail "no files under '$CI_ASSERT_PINNING_PATHS' were scanned - the check would pass vacuously" + _ci_rc=1 + elif [ -n "$_ci_hits" ]; then + _ci_fail "manifests under '$CI_ASSERT_PINNING_PATHS' still pin pods to nodes (nodeSelector / nodeName / nodeAffinity / openms.de/memory-tier)" + printf '%s\n' "$_ci_hits" >&2 + _ci_rc=1 + fi + + if [ "$_ci_rc" -eq 0 ]; then + # Built from the roots actually scanned, not from the three variables. + # Interpolating the variables meant an emptied one degraded the sentence + # to "prod, and k8s/storage", a double space being the only trace that + # a third of the check had not run. + _ci_pass "no node pinning anywhere:${_ci_roots_scanned} all render none - the storage root is what covers the vendored Ganesha chart, which the text scan cannot see - and none of the $_ci_scanned files scanned under '$CI_ASSERT_PINNING_PATHS' references any (tracked + untracked; gitignored files are not scanned)" + fi + return "$_ci_rc" +} + +assert_doc_links_resolve() { + # Every `some/file.md` written in backticks inside a tracked Markdown file + # has to name a tracked path. + # + # This exists because the commit that moved three research documents under + # docs/ repointed every reference to them from k8s/, .github/, src/ and + # tests/ - and missed the eight cross-references the three documents make to + # each other, which kept their pre-move names. One of those was the rollback + # procedure for the storage cutover, cited by k8s/base/workspace-pvc.yaml as + # authoritative: the outer hop resolved, the inner one did not. + # + # Deliberately general rather than a grep for the three old filenames. That + # grep would itself be a hand-synced literal, correct until the fourth + # rename and silent afterwards - the same class of defect it would be + # guarding against. + # + # Backticks only: no Markdown-link form (`[text](path.md)`) appears in this + # tree, and matching one would also have to cope with anchors and relative + # traversal. If that form is ever introduced, extend this rather than + # assuming it is covered. + _ci_require git || return 1 + _ci_rc=0 + + _ci_docs="$(git ls-files '*.md' 2>/dev/null || true)" + if [ -z "$_ci_docs" ]; then + _ci_fail "no tracked Markdown files found; the doc link check would pass vacuously" + return 1 + fi + + # `-o` so one line per token, not per line of prose; `sort -u` because the + # same target is cited many times and a dangling one should be reported once. + _ci_refs="$(printf '%s\n' "$_ci_docs" \ + | xargs grep -ohE '`[A-Za-z0-9_./-]+\.md`' 2>/dev/null \ + | tr -d '`' | sort -u || true)" + if [ -z "$_ci_refs" ]; then + _ci_fail "no backticked *.md references found in $(printf '%s\n' "$_ci_docs" | wc -l) tracked Markdown files; the extraction has stopped matching" + return 1 + fi + + # Resolved against the repository root first, then against docs/, because + # both forms are in use: k8s/base/workspace-pvc.yaml writes the full + # `docs/a16-storage-runbook.md`, while a sibling reference inside docs/ + # may reasonably write just `build_app.md`. + _ci_dangling="" + _ci_checked=0 + for _ci_ref in $_ci_refs; do + _ci_checked=$((_ci_checked + 1)) + if git ls-files --error-unmatch "$_ci_ref" >/dev/null 2>&1; then + continue + fi + if git ls-files --error-unmatch "docs/$_ci_ref" >/dev/null 2>&1; then + continue + fi + _ci_dangling="$_ci_dangling $_ci_ref" + done + + if [ -n "$_ci_dangling" ]; then + _ci_fail "tracked Markdown cites .md paths that do not exist:$_ci_dangling" + printf '%s\n' "$_ci_dangling" | tr ' ' '\n' | grep -v '^$' | while read -r _ci_d; do + git ls-files '*.md' -z 2>/dev/null \ + | xargs -0 grep -n -- "\`$_ci_d\`" 2>/dev/null | sed 's|^| |' >&2 || true + done + _ci_rc=1 + fi + + if [ "$_ci_rc" -eq 0 ]; then + _ci_pass "all $_ci_checked backticked .md references in tracked Markdown resolve to tracked paths" + fi + return "$_ci_rc" +} + +_ci_worker_deployment() { + # Name of the rq-worker Deployment, discovered by label so namePrefix stays + # irrelevant. + kubectl get deployment -n "$CI_ASSERT_NS" -l "app=$1,component=rq-worker" \ + -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || true +} + +assert_workers_spread_across_nodes() { + # Step 4's acceptance criterion for the workers themselves: every replica + # Running, on more than one node, inside the skew the Deployment declares. + # + # This is the assertion that does NOT go green from deleting the + # nodeSelector alone. Placement is only half of it: + # + # 1. every replica is Running. `.spec.replicas` is the denominator, so a + # worker that cannot be scheduled - the exact state an oversized + # request produces - fails here rather than being quietly left out of + # the node count. assert_two_pods_two_nodes cannot stand in: it starts + # its own helper pods with an explicit nodeName, which proves the + # volume is reachable from two nodes and says nothing about where the + # scheduler put the workers. + # 2. they occupy at least two nodes. + # 3. the observed per-node counts, over ALL Ready nodes rather than only + # those already holding a worker, are within the constraint's own + # maxSkew. + # 4. the constraint's labelSelector actually matches the worker pods' + # labels. kustomize copies the overlay's `commonLabels.app` into + # topologySpreadConstraints/labelSelector/matchLabels; if that ever + # stops happening the constraint starts counting every rq-worker in + # the namespace, including other forks', and nothing about the + # placement in a single-fork test cluster would look different. + _ci_require kubectl jq || return 1 + _ci_rc=0 + + _ci_a="$(_ci_app)" + if [ -z "$_ci_a" ]; then + _ci_fail "cannot determine the app label (set SLUG or CI_ASSERT_APP)" + return 1 + fi + _ci_sel="${1:-app=$_ci_a,component=rq-worker}" + + _ci_ready_nodes="$(_ci_ready_node_names)" + _ci_node_count="$(printf '%s\n' "$_ci_ready_nodes" | sed '/^$/d' | wc -l | tr -d ' ')" + if [ "$_ci_node_count" -lt 2 ]; then + _ci_fail "cluster has $_ci_node_count Ready node(s) - spreading cannot be observed on fewer than two, and every constraint would pass vacuously" + return 1 + fi + + _ci_deploy="$(_ci_worker_deployment "$_ci_a")" + if [ -z "$_ci_deploy" ]; then + _ci_fail "no Deployment matching app=$_ci_a,component=rq-worker in $CI_ASSERT_NS - there are no workers to spread" + return 1 + fi + _ci_want="$(kubectl get deployment -n "$CI_ASSERT_NS" "$_ci_deploy" \ + -o jsonpath='{.spec.replicas}' 2>/dev/null || true)" + case "$_ci_want" in + ''|*[!0-9]*) + _ci_fail "could not read .spec.replicas from deployment/$_ci_deploy" + return 1 + ;; + esac + if [ "$_ci_want" -lt 2 ]; then + _ci_fail "deployment/$_ci_deploy declares replicas: $_ci_want - one worker is trivially 'spread' over one node, so this assertion would pass without observing anything" + return 1 + fi + + # `` per worker pod, Pending ones INCLUDED: + # they are the failure this assertion exists to catch, so they have to be + # counted rather than filtered out server-side the way the other + # assertions do it. + _ci_rows="$(kubectl get pods -n "$CI_ASSERT_NS" -l "$_ci_sel" -o json 2>/dev/null \ + | jq -r '.items[] | "\(.metadata.name)\t\(.status.phase)\t\(.spec.nodeName // "")"' 2>/dev/null || true)" + _ci_rows="$(printf '%s\n' "$_ci_rows" | sed '/^$/d')" + if [ -z "$_ci_rows" ]; then + _ci_fail "no pods match $_ci_sel in $CI_ASSERT_NS - spreading cannot be asserted" + return 1 + fi + printf 'worker pods (name / phase / node):\n%s\n' "$_ci_rows" + + # One node name per RUNNING pod, duplicates kept - two workers on one node + # is a skew of 2 over a two-node cluster, and a `sort -u` here would report + # it as a tidy one-each. `$_ci_placements` is the multiset, `$_ci_running` + # the distinct nodes. Both come out of the single snapshot above rather + # than a second query, so they cannot disagree with each other or with the + # listing just printed. + _ci_placements="$(printf '%s\n' "$_ci_rows" | awk -F'\t' '$2 == "Running" && $3 != "" { print $3 }')" + _ci_running="$(printf '%s\n' "$_ci_placements" | sed '/^$/d' | sort -u)" + _ci_running_count="$(printf '%s\n' "$_ci_rows" | awk -F'\t' '$2 == "Running"' | sed '/^$/d' | wc -l | tr -d ' ')" + if [ "$_ci_running_count" -lt "$_ci_want" ]; then + _ci_fail "deployment/$_ci_deploy wants $_ci_want replicas but only $_ci_running_count are Running - a replica the scheduler cannot place is the failure mode an oversized request produces" + kubectl get pods -n "$CI_ASSERT_NS" -l "$_ci_sel" -o wide >&2 || true + kubectl describe pods -n "$CI_ASSERT_NS" -l "$_ci_sel" 2>/dev/null \ + | grep -A5 '^Events:' >&2 || true + _ci_rc=1 + fi + + _ci_n="$(printf '%s\n' "$_ci_running" | sed '/^$/d' | wc -l | tr -d ' ')" + if [ "$_ci_n" -lt 2 ]; then + _ci_fail "$_ci_running_count Running worker(s) occupy $_ci_n node(s) across a $_ci_node_count-node cluster: $(printf '%s' "$_ci_running" | tr '\n' ' ')" + _ci_rc=1 + fi + + # maxSkew as the Deployment declares it, so retuning the constraint retunes + # the assertion with it rather than leaving a hardcoded 1 behind. + _ci_skew_max="$(kubectl get deployment -n "$CI_ASSERT_NS" "$_ci_deploy" -o json 2>/dev/null \ + | jq -r '[.spec.template.spec.topologySpreadConstraints[]? + | select(.topologyKey == "kubernetes.io/hostname") + | .maxSkew] | first // empty' 2>/dev/null || true)" + if [ -z "$_ci_skew_max" ]; then + _ci_fail "deployment/$_ci_deploy declares no topologySpreadConstraint over kubernetes.io/hostname - two nodes here would be the scheduler's default scoring, which it is free to stop doing" + _ci_rc=1 + else + # Counts over every Ready node, zeros included: a node holding no + # worker is a domain the constraint was measured over and is the whole + # point of the check. + _ci_skew="$( { printf '%s\n' "$_ci_ready_nodes" | sed '/^$/d' | sed 's/$/\t0/' + printf '%s\n' "$_ci_placements" | sed '/^$/d' | sed 's/$/\t1/'; } \ + | awk -F'\t' '{ c[$1] += $2 } + END { for (n in c) { if (!seen++ || c[n] > hi) hi = c[n]; + if (lo == "" || c[n] < lo) lo = c[n] } + print hi - lo }')" + if [ -n "$_ci_skew" ] && [ "$_ci_skew" -gt "$_ci_skew_max" ]; then + _ci_fail "worker pods are skewed $_ci_skew across the $_ci_node_count Ready nodes, above the declared maxSkew of $_ci_skew_max" + _ci_rc=1 + fi + fi + + # The labelSelector has to select the workers themselves. + _ci_bad="$(kubectl get deployment -n "$CI_ASSERT_NS" "$_ci_deploy" -o json 2>/dev/null \ + | jq -r --arg app "$_ci_a" '.spec.template as $t + | [.spec.template.spec.topologySpreadConstraints[]? + | select(.topologyKey == "kubernetes.io/hostname") + | (.labelSelector.matchLabels // {}) as $m + | if ($m | length) == 0 then "labelSelector.matchLabels is empty, so the constraint counts every pod in the namespace" + elif [$m | to_entries[] | select($t.metadata.labels[.key] != .value)] | length > 0 + then "labelSelector.matchLabels \($m) does not match the pod template labels \($t.metadata.labels)" + # NOTE: no apostrophes in this jq program - it is inside a + # single-quoted shell string, where one would end it. + # to_entries only walks keys that are PRESENT, so a bare + # {component: rq-worker} selector matched the pod template + # and passed, while counting every other fork rq-worker in + # this shared namespace against our skew. That is the + # regression k8s/base/rq-worker-deployment.yaml calls + # load-bearing in the comment above its own constraint, + # and the check above cannot see it, because the app label + # is injected by commonLabels at overlay time rather than + # declared in the base. + elif ($m.app // "") != $app + then "labelSelector.matchLabels \($m) carries no app=\($app) key, so the constraint counts every rq-worker in the namespace rather than only these workers" + else empty end] | .[]' 2>/dev/null || true)" + if [ -n "$_ci_bad" ]; then + _ci_fail "deployment/$_ci_deploy has a spread constraint that does not select its own workers: $_ci_bad" + _ci_rc=1 + fi + + if [ "$_ci_rc" -eq 0 ]; then + _ci_pass "$_ci_running_count/$_ci_want worker replicas Running on $_ci_n of $_ci_node_count nodes within maxSkew $_ci_skew_max: $(printf '%s' "$_ci_running" | tr '\n' ' ')" + fi + return "$_ci_rc" +} + +assert_worker_qos_guaranteed() { + # Guaranteed QoS gives the worker oom_score_adj -997 instead of burstable's + # ~937, which is near the top of the node's kill list. Catches a + # requests != limits regression, which otherwise only shows up as an + # OOMKill under load. + # + # Running pods only, and that filter is load-bearing rather than tidiness: + # .status.qosClass is computed at ADMISSION and is populated on a Pending + # pod, so a selector without it reports "every worker is Guaranteed" for + # two replicas that were never scheduled - which is exactly the state an + # oversized request produces, and exactly the state this suite must not + # call a pass. The count of Running replicas against .spec.replicas belongs + # to assert_workers_spread_across_nodes; here it is enough that at least + # one worker is actually running to have a QoS class worth reading. + _ci_require kubectl || return 1 + _ci_rc=0 + + _ci_a="$(_ci_app)" + if [ -z "$_ci_a" ]; then + _ci_fail "cannot determine the app label (set SLUG or CI_ASSERT_APP)" + return 1 + fi + _ci_sel="${1:-app=$_ci_a,component=rq-worker}" + + _ci_rows="$(kubectl get pods -n "$CI_ASSERT_NS" -l "$_ci_sel" \ + --field-selector=status.phase=Running \ + -o jsonpath='{range .items[*]}{.metadata.name}{"\t"}{.status.qosClass}{"\n"}{end}' 2>/dev/null || true)" + _ci_rows="$(printf '%s\n' "$_ci_rows" | sed '/^$/d')" + if [ -z "$_ci_rows" ]; then + _ci_fail "no Running pods match $_ci_sel in $CI_ASSERT_NS - QoS cannot be asserted, and reading it off a Pending pod would report a pass for a worker that never started" + kubectl get pods -n "$CI_ASSERT_NS" -l "$_ci_sel" -o wide >&2 || true + return 1 + fi + printf '%s\n' "$_ci_rows" + + _ci_bad="$(printf '%s\n' "$_ci_rows" \ + | awk -F'\t' '$2 != "Guaranteed" { printf "%s=%s ", $1, ($2 == "" ? "" : $2) }')" + if [ -n "$_ci_bad" ]; then + _ci_fail "worker pods are not Guaranteed QoS (requests must equal limits): $_ci_bad" + kubectl get pods -n "$CI_ASSERT_NS" -l "$_ci_sel" \ + -o jsonpath='{range .items[*]}{.metadata.name}{" requests="}{.spec.containers[*].resources.requests}{" limits="}{.spec.containers[*].resources.limits}{"\n"}{end}' >&2 || true + _ci_rc=1 + fi + + if [ "$_ci_rc" -eq 0 ]; then + _ci_pass "every Running pod matching $_ci_sel is Guaranteed QoS" + fi + return "$_ci_rc" +} + +assert_fsids_pinned() { + # The static half of invariant 2, split out so it can also run in the cheap + # manifest-only job on every push rather than only after a full kind deploy. + # + # Ganesha derives every NFS file handle it hands to every client from the + # backing device's major/minor by default, and a Cinder /dev/vdX minor is + # not stable across re-attach: left alone, every client ESTALEs the first + # time this pod moves - typically weeks later, with nothing pointing back at + # the change that caused it. + # + # Read off the RENDERED root rather than the values file, so it holds + # whichever key the chart happens to expose, and matched exhaustively rather + # than on the first hit - `head -n 1` would accept a `false` followed by a + # later `true`. kustomize strips YAML comments, so no amount of prose about + # fsids in the values file can satisfy this. + # + # helm is required explicitly: `kubectl kustomize --enable-helm` shells out + # to it, and "required tool(s) not on PATH: helm" is a better answer than a + # kustomize error about a chart it could not inflate. + _ci_require kubectl helm || return 1 + _ci_rc=0 + + _ci_rendered="$(mktemp)" + _ci_err="$(mktemp)" + if ! _ci_kustomize_storage > "$_ci_rendered" 2>"$_ci_err"; then + _ci_fail "kubectl kustomize --enable-helm $CI_ASSERT_STORAGE_ROOT failed" + cat "$_ci_err" >&2 || true + rm -f "$_ci_rendered" "$_ci_err" + return 1 + fi + rm -f "$_ci_err" + if [ ! -s "$_ci_rendered" ]; then + _ci_fail "$CI_ASSERT_STORAGE_ROOT rendered nothing - the fsid check would pass vacuously" + rm -f "$_ci_rendered" + return 1 + fi + + _ci_fsid="$(grep -oiE 'device-based-fsids[^A-Za-z]*(true|false)' "$_ci_rendered" || true)" + rm -f "$_ci_rendered" + if [ -z "$_ci_fsid" ]; then + _ci_fail "$CI_ASSERT_STORAGE_ROOT never sets device-based-fsids; Ganesha defaults it to true and derives file handles from a device minor number that a Cinder re-attach renumbers" + return 1 + fi + _ci_bad="$(printf '%s\n' "$_ci_fsid" | grep -i 'true' || true)" + if [ -n "$_ci_bad" ]; then + _ci_fail "$CI_ASSERT_STORAGE_ROOT renders device-based fsids ON, which makes every client ESTALE the first time the volume is re-attached" + printf '%s\n' "$_ci_fsid" >&2 + _ci_rc=1 + fi + + if [ "$_ci_rc" -eq 0 ]; then + _ci_pass "$CI_ASSERT_STORAGE_ROOT pins device-based fsids off: $(printf '%s' "$_ci_fsid" | tr '\n' ' ')" + fi + return "$_ci_rc" +} + +assert_storage_identity_values() { + # The other three quarters of invariant 2. `device-based-fsids` above is + # only one of the four things that make the NFS server come back serving + # the same bytes under the same identity; the remaining three were asserted + # nowhere, so a values edit dropping any of them rendered clean, + # kubeconformed clean, deployed clean, and passed every runtime assertion + # in this file. What it costs is not a red build - it is deleted data on a + # PVC delete, or a second Ganesha Pending forever on an RWO claim. + # + # Read off the RENDERED root for the same reason as assert_fsids_pinned: + # it holds whichever key the chart exposes, and kustomize strips the + # comments that would otherwise look like compliance. + _ci_require kubectl helm yq || return 1 + _ci_rc=0 + + _ci_rendered="$(mktemp)" + _ci_err="$(mktemp)" + if ! _ci_kustomize_storage > "$_ci_rendered" 2>"$_ci_err"; then + _ci_fail "kubectl kustomize --enable-helm $CI_ASSERT_STORAGE_ROOT failed" + cat "$_ci_err" >&2 || true + rm -f "$_ci_rendered" "$_ci_err" + return 1 + fi + rm -f "$_ci_err" + if [ ! -s "$_ci_rendered" ]; then + _ci_fail "$CI_ASSERT_STORAGE_ROOT rendered nothing - these checks would pass vacuously" + rm -f "$_ci_rendered" + return 1 + fi + + # 1. reclaimPolicy: Retain on the class this root publishes. The cluster + # default is Delete, which destroys the volume with the claim. + _ci_reclaim="$(yq 'select(.kind == "StorageClass") | .reclaimPolicy' "$_ci_rendered" \ + | grep -v '^---$' | grep -v '^$' || true)" + if [ "$_ci_reclaim" != "Retain" ]; then + _ci_fail "the StorageClass reclaimPolicy is '${_ci_reclaim:-unset}', not Retain: deleting a PV or PVC would destroy the workspace data rather than orphan it" + _ci_rc=1 + fi + + # The workload selects below are scoped to the server's own app label, not + # to "any StatefulSet or Deployment". The chart is free to grow a second + # workload - a metrics exporter, a sidecar controller - and an unscoped + # select would then yield two lines, turning both checks into a spurious + # failure whose message names no workload at all. + # + # Derived from the values file rather than hardcoded: replacing one + # hand-synced literal with another is not an improvement. `nameOverride` + # is what the chart renders into `app:`; `fullnameOverride` names the + # objects and is a different string on purpose. + _ci_srv="$(yq '.nameOverride' "$CI_ASSERT_STORAGE_ROOT/ganesha-values.yaml" 2>/dev/null \ + | grep -v '^null$' || true)" + if [ -z "$_ci_srv" ]; then + _ci_fail "could not read nameOverride from $CI_ASSERT_STORAGE_ROOT/ganesha-values.yaml; cannot identify the NFS server workload" + rm -f "$_ci_rendered" + return 1 + fi + + # 2. A fixed, pre-existing backing claim, whose name matches a PVC this + # root actually renders. Dynamic provisioning for the BACKING volume + # means a re-provisioned claim is a different volume and the server + # comes back empty; a claim name that matches nothing means the pod + # sits Pending forever. `persistence.existingClaim` in the values and + # `metadata.name` in nfs-backing-pvc.yaml are hand-synced, and until + # now nothing checked that they still agreed. + _ci_claim="$(yq "select((.kind == \"StatefulSet\" or .kind == \"Deployment\") + and .metadata.labels.app == \"$_ci_srv\") + | .spec.template.spec.volumes[]? + | select(.persistentVolumeClaim) + | .persistentVolumeClaim.claimName" "$_ci_rendered" \ + | grep -v '^---$' | grep -v '^$' | grep -v '^null$' || true)" + if [ -z "$_ci_claim" ]; then + _ci_fail "the NFS server mounts no persistentVolumeClaim, so persistence.existingClaim was dropped and its backing store is not a fixed volume" + _ci_rc=1 + else + # `else`, not a second unguarded `if`: on a dropped claim both branches + # would fire and the second message would misdirect. + _ci_pvc="$(yq 'select(.kind == "PersistentVolumeClaim") | .metadata.name' "$_ci_rendered" \ + | grep -v '^---$' | grep -v '^$' | grep -v '^null$' || true)" + if [ "$_ci_claim" != "$_ci_pvc" ]; then + _ci_fail "the NFS server mounts claim '$_ci_claim' but $CI_ASSERT_STORAGE_ROOT renders PVC '${_ci_pvc:-none}': the pod would sit Pending on a claim that does not exist" + _ci_rc=1 + fi + fi + + # 3. Exactly one replica. Two Ganesha pods on one RWO claim is worse than + # the RollingUpdate deadlock `strategy: Recreate` exists to avoid. + _ci_replicas="$(yq "select((.kind == \"StatefulSet\" or .kind == \"Deployment\") + and .metadata.labels.app == \"$_ci_srv\") + | .spec.replicas" "$_ci_rendered" \ + | grep -v '^---$' | grep -v '^$' | grep -v '^null$' || true)" + if [ "$_ci_replicas" != "1" ]; then + _ci_fail "the NFS server renders replicas='${_ci_replicas:-unset}', not 1: a second pod cannot mount the RWO backing claim and would sit Pending forever" + _ci_rc=1 + fi + + rm -f "$_ci_rendered" + if [ "$_ci_rc" -eq 0 ]; then + # The app label is named as well as the three values: it is the scope + # every check above was read through, and a PASS that omits it does not + # say which workload was inspected. + _ci_pass "storage identity pinned for app='$_ci_srv': reclaimPolicy=Retain, backing claim=$_ci_claim, replicas=$_ci_replicas" + fi + return "$_ci_rc" +} + +assert_survives_nfs_restart() { + # The single most valuable assertion in this file: delete the Ganesha pod + # while a workflow is running, and assert the workflow COMPLETES. + # + # A Ganesha restart is routine - upgrades, evictions, node drains, OOM - and + # this is the exact case a `soft` mount would have corrupted, by returning a + # short read or an EIO into the middle of a featureXML instead of blocking. + # Nothing else here covers it: assert_stable_identity_across_restart runs a + # handful of file operations after the server is back, so it cannot observe + # a job that died during the outage or output that went missing while the + # ~90s NFSv4.1 grace period was refusing state-mutating operations. + # + # The stand-in workflow is deliberately shaped like the real one rather than + # being a single `cp`: + # - one long-lived file descriptor, opened BEFORE the restart and written + # through afterwards, the way a TOPP tool holds its output file + # - one open-append-close per line, the way src/workflow/Logger.py writes + # - a `WORKFLOW FINISHED` marker as the last line, which is the same + # signal src/workflow/_log_status.py classifies a run by + # Its exit status is recorded in /tmp, NOT on the mount, so a wedged export + # cannot hide the result it is being judged on. + _ci_require kubectl jq || return 1 + _ci_rc=0 + _ci_run_rc=0 + + _ci_a="$(_ci_app)" + if [ -z "$_ci_a" ]; then + _ci_fail "cannot determine the app label (set SLUG or CI_ASSERT_APP)" + return 1 + fi + _ci_c="$(_ci_claim "$_ci_a")" + if [ -z "$_ci_c" ]; then + _ci_fail "no Deployment in $CI_ASSERT_NS mounts a 'workspaces' volume - nothing to assert" + return 1 + fi + _ci_img="$(_ci_image "$_ci_a")" + _ci_records="$CI_ASSERT_WORKFLOW_RECORDS" + + _ci_row="$(_ci_ready_pods "$CI_ASSERT_STORAGE_NS" | head -n 1)" + _ci_before="$(printf '%s' "$_ci_row" | cut -f1)" + _ci_before_uid="$(printf '%s' "$_ci_row" | cut -f2)" + if [ -z "$_ci_before" ] || [ -z "$_ci_before_uid" ]; then + _ci_fail "no Ready pod in $CI_ASSERT_STORAGE_NS - the NFS server is not running, so restarting it proves nothing" + kubectl get pods -n "$CI_ASSERT_STORAGE_NS" -o wide >&2 || true + return 1 + fi + + _ci_stamp="$(date +%s)" + _ci_pod="ci-workflow-$_ci_stamp" + _ci_dir="$CI_ASSERT_MOUNT/.ci-survives-restart-$_ci_stamp" + + # Unpinned: the scheduler places it, and the mount it ends up holding is the + # one that has to survive. + if ! _ci_start_helper_pod "$_ci_pod" "" "$_ci_img" "$_ci_c"; then + _ci_fail "client pod could not start with $_ci_c mounted at $CI_ASSERT_MOUNT" + _ci_delete_pods "$_ci_pod" + return 1 + fi + + _ci_exec_script "$_ci_pod" "$_ci_dir" "$_ci_records" <<'START' || _ci_run_rc=1 +set -e +d="$1" +n="$2" +rm -rf "$d" +mkdir -p "$d" +rm -f /tmp/ci-workflow-status /tmp/ci-workflow-stderr +: > /tmp/ci-workflow-stderr +( + set -e + trap 'rc=$?; echo "$rc" > /tmp/ci-workflow-status' EXIT + # Opened once, up front. Every write after the restart therefore has to be + # served against a file handle the previous server handed out. + exec 8> "$d/stream.log" + i=1 + while [ "$i" -le "$n" ]; do + echo "record $i" >&8 + echo "record $i" >> "$d/perline.log" + i=$((i + 1)) + sleep 1 + done + echo "WORKFLOW FINISHED" >&8 + echo "WORKFLOW FINISHED" >> "$d/perline.log" +) > /dev/null 2>> /tmp/ci-workflow-stderr & +echo "workflow started, writing $n records to $d" +START + if [ "$_ci_run_rc" -ne 0 ]; then + _ci_fail "the stand-in workflow could not be started in $_ci_dir" + _ci_delete_pods "$_ci_pod" + return 1 + fi + + # Let it get properly under way before the server is pulled out from under + # it. A restart that lands before the first write would test the mount, not + # a workflow in progress. + _ci_got="$(_ci_exec_script "$_ci_pod" "$_ci_dir" 5 <<'PROGRESS' || true +d="$1" +want="$2" +i=0 +while [ "$i" -lt 60 ]; do + if [ -f "$d/perline.log" ] && [ "$(wc -l < "$d/perline.log")" -ge "$want" ]; then + wc -l < "$d/perline.log" + exit 0 + fi + i=$((i + 1)) + sleep 1 +done +exit 1 +PROGRESS +)" + if [ -z "$_ci_got" ]; then + _ci_fail "the stand-in workflow wrote nothing in 60s, so there is no run in progress to interrupt" + _ci_delete_pods "$_ci_pod" + return 1 + fi + printf 'workflow is %s records in; deleting NFS server pod %s in %s\n' "$(printf '%s' "$_ci_got" | tr -d ' ')" "$_ci_before" "$CI_ASSERT_STORAGE_NS" + + if ! kubectl delete pod -n "$CI_ASSERT_STORAGE_NS" "$_ci_before" --wait=true --timeout="${CI_ASSERT_RESTART_TIMEOUT}s" > /dev/null; then + _ci_fail "could not delete the NFS server pod $_ci_before" + _ci_delete_pods "$_ci_pod" + return 1 + fi + + # A pod with a NEW uid has to come back Ready. Matching on the name would + # never fire - a StatefulSet reuses it. + _ci_deadline=$(( $(date +%s) + CI_ASSERT_RESTART_TIMEOUT )) + _ci_after_uid="" + while [ "$(date +%s)" -lt "$_ci_deadline" ]; do + _ci_row="$(_ci_ready_pods "$CI_ASSERT_STORAGE_NS" | awk -F'\t' -v old="$_ci_before_uid" '$2 != "" && $2 != old { print; exit }')" + _ci_after="$(printf '%s' "$_ci_row" | cut -f1)" + _ci_after_uid="$(printf '%s' "$_ci_row" | cut -f2)" + if [ -n "$_ci_after_uid" ]; then + break + fi + sleep 5 + done + if [ -z "$_ci_after_uid" ]; then + _ci_fail "no replacement NFS server pod became Ready within ${CI_ASSERT_RESTART_TIMEOUT}s of deleting $_ci_before" + kubectl get pods -n "$CI_ASSERT_STORAGE_NS" -o wide >&2 || true + _ci_delete_pods "$_ci_pod" + return 1 + fi + printf 'NFS server came back as %s (uid %s -> %s)\n' "$_ci_after" "$_ci_before_uid" "$_ci_after_uid" + + # Now wait it out. The workflow has to survive the outage AND the grace + # period, during which a `hard` mount simply blocks and retries. The status + # file lives on the pod's own /tmp, so polling it never touches the export. + _ci_deadline=$(( $(date +%s) + CI_ASSERT_WORKFLOW_TIMEOUT )) + _ci_status="" + while [ "$(date +%s)" -lt "$_ci_deadline" ]; do + _ci_status="$(_ci_exec_script "$_ci_pod" <<'STATUS' || true +if [ -f /tmp/ci-workflow-status ]; then + cat /tmp/ci-workflow-status +fi +STATUS +)" + _ci_status="$(printf '%s' "$_ci_status" | tr -d '[:space:]')" + if [ -n "$_ci_status" ]; then + break + fi + sleep 5 + done + + if [ -z "$_ci_status" ]; then + _ci_fail "the workflow had not finished ${CI_ASSERT_WORKFLOW_TIMEOUT}s after the restart - it is still blocked, or it died without recording a status" + _ci_rc=1 + elif [ "$_ci_status" != "0" ]; then + _ci_fail "the workflow exited $_ci_status across the restart of $_ci_before; a hard mount is supposed to block and resume, not fail" + _ci_rc=1 + fi + + # Completeness, not just exit status: a workflow that "finished" having + # silently dropped half its output is the failure this is guarding against. + _ci_exec_script "$_ci_pod" "$_ci_dir" "$_ci_records" <<'VERIFY' || _ci_rc=1 +d="$1" +n="$2" +rc=0 + +check_log() { + if [ ! -f "$1" ]; then + echo "$1 does not exist - the workflow never created it" + return 1 + fi + awk -v n="$2" -v f="$1" ' + NR <= n { + if ($0 != "record " NR) { + printf "%s line %d is [%s], expected [record %d]\n", f, NR, $0, NR + bad = 1 + exit 1 + } + next + } + NR == n + 1 { + if ($0 != "WORKFLOW FINISHED") { + printf "%s line %d is [%s], expected the WORKFLOW FINISHED marker\n", f, NR, $0 + bad = 1 + exit 1 + } + next + } + { + printf "%s has an unexpected trailing line %d: [%s]\n", f, NR, $0 + bad = 1 + exit 1 + } + END { + if (bad) { + exit 1 + } + if (NR < n + 1) { + printf "%s holds %d lines, expected %d records plus the marker\n", f, NR, n + exit 1 + } + } + ' "$1" +} + +# The long-lived descriptor and the open-append-close writer have to agree. +check_log "$d/stream.log" "$n" || rc=1 +check_log "$d/perline.log" "$n" || rc=1 + +if [ -s /tmp/ci-workflow-stderr ]; then + echo "the workflow wrote to stderr:" + cat /tmp/ci-workflow-stderr + rc=1 +fi +exit "$rc" +VERIFY + + _ci_exec_script "$_ci_pod" "$_ci_dir" <<'CLEAN' > /dev/null 2>&1 || true +rm -rf "$1" +CLEAN + _ci_delete_pods "$_ci_pod" + + if [ "$_ci_rc" -ne 0 ]; then + _ci_fail "a workflow running across the restart of $_ci_before did not complete intact" + return 1 + fi + _ci_pass "a workflow writing $_ci_records records through the mount survived the NFS server restart ($_ci_before, uid $_ci_before_uid -> $_ci_after_uid) and finished complete and in order" + return 0 +} + +assert_stable_identity_across_restart() { + # Invariant 2: the filesystem pod comes back pointing at the same bytes, + # under the same identity, on every restart. Two halves, both required. + # + # static assert_fsids_pinned, above: the rendered storage root has to + # pin device-based-fsids off. + # + # runtime delete the NFS server pod underneath a live client, then make + # that client *create* a file through the mount it opened before + # the restart. A create cannot be served out of the page cache + # the way a re-read can, so it has to present a pre-restart file + # handle to the new server and ESTALEs if identity moved. + # + # The static half is what keeps this from passing vacuously: kind's backing + # device is not renumbered when the pod restarts, so the runtime half alone + # would go green with device-based fsids left switched on. + _ci_require kubectl jq || return 1 + _ci_rc=0 + _ci_run_rc=0 + + if ! assert_fsids_pinned; then + _ci_rc=1 + fi + + _ci_a="$(_ci_app)" + if [ -z "$_ci_a" ]; then + _ci_fail "cannot determine the app label (set SLUG or CI_ASSERT_APP)" + return 1 + fi + _ci_c="$(_ci_claim "$_ci_a")" + if [ -z "$_ci_c" ]; then + _ci_fail "no Deployment in $CI_ASSERT_NS mounts a 'workspaces' volume - nothing to assert" + return 1 + fi + _ci_img="$(_ci_image "$_ci_a")" + + _ci_row="$(_ci_ready_pods "$CI_ASSERT_STORAGE_NS" | head -n 1)" + _ci_before="$(printf '%s' "$_ci_row" | cut -f1)" + _ci_before_uid="$(printf '%s' "$_ci_row" | cut -f2)" + if [ -z "$_ci_before" ] || [ -z "$_ci_before_uid" ]; then + _ci_fail "no Ready pod in $CI_ASSERT_STORAGE_NS - the NFS server is not running, so restarting it proves nothing" + kubectl get pods -n "$CI_ASSERT_STORAGE_NS" -o wide >&2 || true + return 1 + fi + + _ci_stamp="$(date +%s)" + _ci_pod="ci-estale-$_ci_stamp" + _ci_dir="$CI_ASSERT_MOUNT/.ci-stable-identity-$_ci_stamp" + + # Unpinned: the scheduler places it, and the mount it ends up holding is the + # one that has to survive. + if ! _ci_start_helper_pod "$_ci_pod" "" "$_ci_img" "$_ci_c"; then + _ci_fail "client pod could not start with $_ci_c mounted at $CI_ASSERT_MOUNT" + _ci_delete_pods "$_ci_pod" + return 1 + fi + + _ci_exec_script "$_ci_pod" "$_ci_dir" <<'PRE' || _ci_run_rc=1 +set -e +mkdir -p "$1" +printf 'before-restart' > "$1/before" +ls -1 "$1" >/dev/null +PRE + if [ "$_ci_run_rc" -ne 0 ]; then + _ci_fail "the client could not write to $_ci_dir before the restart, so there is nothing left to prove" + _ci_delete_pods "$_ci_pod" + return 1 + fi + + printf 'deleting NFS server pod %s in %s\n' "$_ci_before" "$CI_ASSERT_STORAGE_NS" + if ! kubectl delete pod -n "$CI_ASSERT_STORAGE_NS" "$_ci_before" --wait=true --timeout="${CI_ASSERT_RESTART_TIMEOUT}s" >/dev/null; then + _ci_fail "could not delete the NFS server pod $_ci_before" + _ci_delete_pods "$_ci_pod" + return 1 + fi + + # A pod with a *new uid* has to come back Ready. Matching on the name + # would never fire - a StatefulSet reuses it - and `kubectl wait --all` + # would accept an empty namespace as success. + _ci_deadline=$(( $(date +%s) + CI_ASSERT_RESTART_TIMEOUT )) + _ci_after_uid="" + while [ "$(date +%s)" -lt "$_ci_deadline" ]; do + _ci_row="$(_ci_ready_pods "$CI_ASSERT_STORAGE_NS" | awk -F'\t' -v old="$_ci_before_uid" '$2 != "" && $2 != old { print; exit }')" + _ci_after="$(printf '%s' "$_ci_row" | cut -f1)" + _ci_after_uid="$(printf '%s' "$_ci_row" | cut -f2)" + if [ -n "$_ci_after_uid" ]; then + break + fi + sleep 5 + done + if [ -z "$_ci_after_uid" ]; then + _ci_fail "no replacement NFS server pod became Ready within ${CI_ASSERT_RESTART_TIMEOUT}s of deleting $_ci_before" + kubectl get pods -n "$CI_ASSERT_STORAGE_NS" -o wide >&2 || true + kubectl describe pods -n "$CI_ASSERT_STORAGE_NS" >&2 || true + _ci_delete_pods "$_ci_pod" + return 1 + fi + printf 'NFS server came back as %s (uid %s -> %s)\n' "$_ci_after" "$_ci_before_uid" "$_ci_after_uid" + + _ci_exec_script "$_ci_pod" "$_ci_dir" <<'POST' || _ci_run_rc=1 +d="$1" +rc=0 +err=/tmp/ci-stale-err +: > "$err" + +# Bound every operation: a wedged `hard` mount blocks in uninterruptible sleep +# and `kubectl exec` has no timeout of its own. Generous, because NFSv4.1's +# ~90s grace period refuses state-mutating operations while it runs and a hard +# mount simply retries through it. +TMO="" +if command -v timeout >/dev/null 2>&1; then + TMO="timeout 240" +fi + +# The create is the load-bearing one: unlike a re-read it cannot be answered out +# of this client's cache, so it has to reach the server carrying a file handle +# obtained before the restart. +if ! $TMO sh -c 'printf after-restart > "$1/after"' sh "$d" 2>>"$err"; then + echo "creating a file through the pre-restart mount failed" + rc=1 +fi + +if ! got="$($TMO cat "$d/before" 2>>"$err")"; then + echo "reading back the pre-restart file failed" + rc=1 +fi +if [ "$got" != "before-restart" ]; then + echo "read '$got' from $d/before, expected 'before-restart'" + rc=1 +fi + +if ! $TMO ls -1 "$d" >/dev/null 2>>"$err"; then + echo "listing $d failed" + rc=1 +fi + +if grep -qi 'stale' "$err"; then + echo "ESTALE on the mount this client held across the restart:" + rc=1 +fi +if [ "$rc" -ne 0 ] && [ -s "$err" ]; then + cat "$err" +fi +exit "$rc" +POST + + _ci_exec_script "$_ci_pod" "$_ci_dir" <<'CLEAN' >/dev/null 2>&1 || true +rm -rf "$1" +CLEAN + _ci_delete_pods "$_ci_pod" + + if [ "$_ci_run_rc" -ne 0 ]; then + _ci_fail "the client did not come through the restart of $_ci_before with its file handles intact" + _ci_rc=1 + fi + if [ "$_ci_rc" -ne 0 ]; then + return 1 + fi + _ci_pass "file handles survive an NFS server restart ($_ci_before, uid $_ci_before_uid -> $_ci_after_uid)" + return 0 +} + +assert_netpol_string_matches_overlay() { + # The storage root is deliberately prefix-free and cannot see the prod + # overlay's commonLabels, so the NetworkPolicy that opens 2049 has to + # hardcode both the `app` value and the namespace it admits. Drift shows up + # as a mount that hangs at 3am rather than as a lint error, so it is checked + # here, where it costs nothing. + # + # Comparing the `app` literal alone is not enough, and every gap below has + # been mutation-tested against this tree: + # + # - splitting namespaceSelector and podSelector into two `from` elements + # turns the AND into an OR, which admits every pod in `openms` (NuXL + # included) and every pod anywhere carrying the label. Both selectors + # therefore have to sit in ONE element. + # - pointing namespaceSelector at another namespace breaks every real + # mount while leaving the `app` string correct, so it is compared with + # `namespace:` in k8s/base/kustomization.yaml. + # - moving the port off 2049 silently closes the export, so the ports are + # compared too - and a rule with no `ports` at all opens every port on + # the server, which is worse than the drift being guarded against. + _ci_require kubectl helm jq yq || return 1 + _ci_rc=0 + + _ci_expected="$(yq '.commonLabels.app' "$CI_ASSERT_OVERLAY/kustomization.yaml" 2>/dev/null || true)" + if [ -z "$_ci_expected" ] || [ "$_ci_expected" = "null" ]; then + _ci_fail "$CI_ASSERT_OVERLAY/kustomization.yaml has no commonLabels.app - there is nothing for the NetworkPolicy to match" + return 1 + fi + _ci_expected_ns="$(yq '.namespace' "$CI_ASSERT_BASE/kustomization.yaml" 2>/dev/null || true)" + if [ -z "$_ci_expected_ns" ] || [ "$_ci_expected_ns" = "null" ]; then + _ci_fail "$CI_ASSERT_BASE/kustomization.yaml sets no namespace - there is nothing for the NetworkPolicy's namespaceSelector to match" + return 1 + fi + + # stdout and stderr to separate files: helm and kustomize both warn on + # stderr, and a warning folded into the rendered stream would be parsed as + # part of the manifests. + _ci_rendered="$(mktemp)" + _ci_err="$(mktemp)" + if ! _ci_kustomize_storage > "$_ci_rendered" 2>"$_ci_err"; then + _ci_fail "kubectl kustomize --enable-helm $CI_ASSERT_STORAGE_ROOT failed" + cat "$_ci_err" >&2 || true + rm -f "$_ci_rendered" "$_ci_err" + return 1 + fi + rm -f "$_ci_err" + if [ ! -s "$_ci_rendered" ]; then + _ci_fail "$CI_ASSERT_STORAGE_ROOT rendered nothing - the label check would pass vacuously" + rm -f "$_ci_rendered" + return 1 + fi + + if ! grep -q '^kind: NetworkPolicy' "$_ci_rendered"; then + _ci_fail "$CI_ASSERT_STORAGE_ROOT renders no NetworkPolicy at all, so the export is open to every pod in the cluster" + rm -f "$_ci_rendered" + return 1 + fi + + # One JSON document per line, then jq into tagged rows: RULE per ingress + # rule, FROM per peer, PORT per port. A peer is flattened to + # , which is what makes the + # AND/OR distinction visible - two peers show up as two FROM rows, each with + # half of the pair missing. + _ci_rows="$(yq -o=json -I=0 '.' "$_ci_rendered" 2>/dev/null | jq -r ' + select(.kind == "NetworkPolicy") + | .metadata.name as $n + | (.spec.ingress // []) as $rules + | $rules[] + | . as $r + | "RULE\t\($n)\t\(($r.from // []) | length)\t\(($r.ports // []) | length)", + (($r.from // [])[] + | "FROM\t\($n)\t\(if has("namespaceSelector") then (.namespaceSelector.matchLabels["kubernetes.io/metadata.name"] // "") else "-" end)\t\(if has("podSelector") then (.podSelector.matchLabels.app // "") else "-" end)\t\(if has("ipBlock") then (.ipBlock.cidr // "") else "-" end)"), + (($r.ports // [])[] + | "PORT\t\($n)\t\(.protocol // "TCP")\t\(.port)") + ' 2>/dev/null || true)" + rm -f "$_ci_rendered" + + if [ -z "$_ci_rows" ]; then + _ci_fail "$CI_ASSERT_STORAGE_ROOT renders NetworkPolicies but not one ingress rule, so nothing can reach 2049 and every mount hangs" + return 1 + fi + printf '%s\n' "$_ci_rows" + + # Every rule has to name its ports. A rule with `from` but no `ports` opens + # the whole pod, not just NFS. + _ci_bad="$(printf '%s\n' "$_ci_rows" | awk -F'\t' '$1 == "RULE" && $3 > 0 && $4 == 0 { print $2 }')" + if [ -n "$_ci_bad" ]; then + _ci_fail "these NetworkPolicy rules admit traffic on every port, not just NFS: $(printf '%s' "$_ci_bad" | tr '\n' ' ')" + _ci_rc=1 + fi + + # Ports: TCP 2049 and nothing else. + _ci_bad="$(printf '%s\n' "$_ci_rows" | awk -F'\t' '$1 == "PORT" && !($3 == "TCP" && $4 == "2049") { printf "%s/%s ", $3, $4 }')" + if [ -n "$_ci_bad" ]; then + _ci_fail "the storage NetworkPolicy opens $_ci_bad; NFSv4.1 needs TCP 2049 and only 2049, and anything else is either a hole or a mount that cannot connect" + _ci_rc=1 + fi + + # Peers that carry a podSelector must carry a namespaceSelector in the SAME + # element, and vice versa. An element with only one of the two is an OR in + # disguise. + _ci_bad="$(printf '%s\n' "$_ci_rows" | awk -F'\t' '$1 == "FROM" && $5 == "-" && $3 == "-" && $4 != "-" { printf "app=%s ", $4 }')" + if [ -n "$_ci_bad" ]; then + _ci_fail "the storage NetworkPolicy admits $_ci_bad from ANY namespace - the namespaceSelector and podSelector have to share one 'from' element, or they are ORed" + _ci_rc=1 + fi + _ci_bad="$(printf '%s\n' "$_ci_rows" | awk -F'\t' '$1 == "FROM" && $5 == "-" && $3 != "-" && $4 == "-" { printf "%s ", $3 }')" + if [ -n "$_ci_bad" ]; then + _ci_fail "the storage NetworkPolicy admits EVERY pod in namespace $_ci_bad, which on the de.NBI cluster includes app=nuxl-app and hands it root over every workspace" + _ci_rc=1 + fi + + # Values, for the peers that pair the two selectors up. + _ci_apps="$(printf '%s\n' "$_ci_rows" | awk -F'\t' '$1 == "FROM" && $4 != "-" { print $4 }' | sort -u)" + if [ -z "$_ci_apps" ]; then + _ci_fail "$CI_ASSERT_STORAGE_ROOT renders NetworkPolicies, but none of them admits clients by their 'app' label - nothing can be compared with $CI_ASSERT_OVERLAY" + _ci_rc=1 + else + _ci_bad="$(printf '%s\n' "$_ci_apps" | grep -Fxv -- "$_ci_expected" || true)" + if [ -n "$_ci_bad" ]; then + _ci_fail "the storage NetworkPolicy admits app='$(printf '%s' "$_ci_bad" | tr '\n' ' ')' but $CI_ASSERT_OVERLAY labels its pods app='$_ci_expected', so those pods cannot reach 2049" + _ci_rc=1 + fi + fi + + _ci_found_ns="$(printf '%s\n' "$_ci_rows" | awk -F'\t' '$1 == "FROM" && $3 != "-" { print $3 }' | sort -u)" + if [ -n "$_ci_found_ns" ]; then + _ci_bad="$(printf '%s\n' "$_ci_found_ns" | grep -Fxv -- "$_ci_expected_ns" || true)" + if [ -n "$_ci_bad" ]; then + _ci_fail "the storage NetworkPolicy admits clients from namespace '$(printf '%s' "$_ci_bad" | tr '\n' ' ')' but $CI_ASSERT_BASE/kustomization.yaml deploys the app into '$_ci_expected_ns', so no real client matches" + _ci_rc=1 + fi + fi + + # The node rule is informational here: kubelet-side mounts come from a node + # address, so its CIDR is cluster-specific and cannot be derived from the + # manifests. Print it, and say so when it is still the shipped placeholder. + _ci_cidrs="$(printf '%s\n' "$_ci_rows" | awk -F'\t' '$1 == "FROM" && $5 != "-" { print $5 }' | sort -u)" + if [ -z "$_ci_cidrs" ]; then + _ci_fail "$CI_ASSERT_STORAGE_ROOT admits no ipBlock on 2049. The provisioner emits in-tree 'nfs:' PVs, which the kubelet mounts from the node's own address - that matches no podSelector, so on a policy-enforcing CNI every mount hangs" + _ci_rc=1 + else + printf 'storage NetworkPolicy admits nodes in: %s\n' "$(printf '%s' "$_ci_cidrs" | tr '\n' ' ')" + if printf '%s\n' "$_ci_cidrs" | grep -q '^192\.0\.2\.'; then + printf 'NOTE: that is RFC 5737 TEST-NET-1, the shipped placeholder. Set it to the cluster node CIDR before deploying, or every mount will hang (k8s/storage/networkpolicy.yaml).\n' + fi + + # Breadth. Until now the only failure here was an EMPTY set, so the most + # dangerous possible value passed: 0.0.0.0/0 kept this green AND removed + # the placeholder NOTE above, which only matches ^192\.0\.2\. - the worst + # setting produced a quieter log than the safe one. + # + # The export is no_root_squash, so a range wide enough to contain the pod + # network hands every pod in the cluster root over every workspace. That + # hazard is stated in k8s/storage/networkpolicy.yaml and was asserted by + # nothing. + # + # A prefix-length threshold, deliberately NOT an "warn on non-RFC1918" + # rule: pod networks ARE RFC1918 - kind defaults to 10.244.0.0/16 - so + # that rule would stay silent on 10.0.0.0/8, the most dangerous realistic + # value, while firing on the harmless TEST-NET-1 placeholder. It inverts + # the hazard. /12 is a judgement call; real node ranges are /16 or + # tighter and networkpolicy.yaml recommends one /32 per node. + # + # This bounds WIDTH, not overlap. A node rule set to exactly the pod + # CIDR (kind: 10.244.0.0/16) is /16 and passes here. Deciding overlap + # needs the cluster's actual pod and service networks, so it belongs in + # a kind-job assertion rather than this manifest-only one; what this + # catches is the realistic error, someone widening the range "to cover + # everything" - and 10.0.0.0/8 does contain the pod network. + _ci_broad="" + for _ci_cidr in $_ci_cidrs; do + case "$_ci_cidr" in + *:*) continue ;; # IPv6: a threshold calibrated for IPv4 says + # nothing useful about a /48 or a /64. + */*) : ;; + *) _ci_fail "the storage NetworkPolicy admits ipBlock '$_ci_cidr', which is not CIDR notation" + _ci_rc=1 + continue ;; + esac + _ci_prefix="${_ci_cidr##*/}" + case "$_ci_prefix" in + ''|*[!0-9]*) + _ci_fail "the storage NetworkPolicy admits ipBlock '$_ci_cidr', whose prefix length is not a number" + _ci_rc=1 + continue ;; + esac + if [ "$_ci_prefix" -lt 12 ]; then + _ci_broad="$_ci_broad $_ci_cidr" + fi + done + if [ -n "$_ci_broad" ]; then + _ci_fail "the storage NetworkPolicy admits$_ci_broad on 2049, which is wide enough to contain the pod network. The export is no_root_squash, so that gives every pod in the cluster root over every workspace - name the nodes' own addresses instead (one /32 per node is fine)" + _ci_rc=1 + fi + fi + + if [ "$_ci_rc" -eq 0 ]; then + # The node rule is named rather than left out. "admits exactly + # app=" was an over-claim: the same policy set also admits a raw + # ipBlock, which this function bounds but cannot verify against the real + # cluster - only assert_netpol_admits_every_node can do that. + _ci_pass "the storage NetworkPolicy admits app='$_ci_expected' in namespace '$_ci_expected_ns', ANDed in one peer, on TCP 2049 alone, plus ipBlock $(printf '%s' "$_ci_cidrs" | tr '\n' ' ') for kubelet mounts" + fi + return "$_ci_rc" +} + +assert_netpol_admits_every_node() { + # The runtime companion to assert_netpol_string_matches_overlay, which can + # only note that the shipped node CIDR is a placeholder - it reads + # manifests, and a node address cannot be derived from a manifest. + # + # This one has a cluster, so it can answer the question that actually + # decides whether the storage tier works: is every node's kubelet allowed + # to reach 2049? The provisioner emits in-tree `nfs:` PVs, which the + # KUBELET mounts from the node's own address in the host network namespace. + # That matches no podSelector, so on a policy-enforcing CNI the only thing + # standing between a worker and an indefinitely hanging mount is an ipBlock + # covering its node - and the value shipped in networkpolicy.yaml is RFC + # 5737 TEST-NET-1, which covers nothing. + # + # Without this, the CI rewrite that patches that CIDR is an unverified sed: + # if its target string ever moves, the rewrite no-ops and the suite fails + # forty minutes later as a Deployment availability timeout that names + # nothing. + _ci_require kubectl jq python3 || return 1 + _ci_rc=0 + + # Rules are filtered by PORT before their CIDRs are collected. Reading + # `.from[].ipBlock.cidr` without descending into the sibling `.ports` - + # and flattening across every policy in the namespace while doing it - + # produces a check whose PASS string claims 2049 without ever having + # looked at a port. It is accidentally right on today's single-rule + # policy; change `port: 2049` to 111, or add any unrelated policy with a + # broad ipBlock, and it still reports "admitted on 2049" while every + # cross-node mount hangs. That is exactly the forty-minute Deployment + # timeout this assertion exists to pre-empt. + # + # A rule with no `ports` at all means all ports, so it is kept. `endPort` + # ranges and named ports are dropped by the equality: fail-safe, since the + # result is a spurious red rather than a false green. + _ci_cidrs="$(kubectl get networkpolicy -n "$CI_ASSERT_STORAGE_NS" -o json 2>/dev/null \ + | jq -r '[.items[].spec.ingress[]? + | select((.ports // []) | length == 0 + or any(.[]?; (.port == 2049) and ((.protocol // "TCP") == "TCP"))) + | .from[]?.ipBlock.cidr // empty] | .[]' || true)" + # jq's stderr is deliberately NOT redirected to /dev/null. A syntax error in + # the filter above produces empty output, which lands in the branch below + # and blames the manifests - sending whoever reads it to networkpolicy.yaml + # when the defect is in this file. Nothing catches that earlier either: + # `bash -n` and `shellcheck` do not parse an embedded jq program. + if [ -z "$_ci_cidrs" ]; then + _ci_fail "the NetworkPolicies in $CI_ASSERT_STORAGE_NS admit no ipBlock on TCP 2049, so no kubelet can mount the share (if jq printed a parse error above, the fault is this filter, not the manifests)" + return 1 + fi + + _ci_nodes="$(kubectl get nodes \ + -o jsonpath='{range .items[*]}{.status.addresses[?(@.type=="InternalIP")].address}{"\n"}{end}' \ + 2>/dev/null || true)" + if [ -z "$_ci_nodes" ]; then + _ci_fail "could not read any node InternalIP; cannot tell whether the storage NetworkPolicy admits them" + return 1 + fi + + # python3 rather than shell arithmetic: these are CIDRs, and a prefix-length + # comparison written in awk is exactly the kind of thing that silently + # passes on /16 and fails on /12. + _ci_missing="" + for _ci_n in $_ci_nodes; do + if ! CI_NODE_IP="$_ci_n" CI_CIDRS="$_ci_cidrs" python3 -c ' +import ipaddress, os, sys +ip = ipaddress.ip_address(os.environ["CI_NODE_IP"]) +for c in os.environ["CI_CIDRS"].split(): + try: + if ip in ipaddress.ip_network(c, strict=False): + sys.exit(0) + except ValueError: + continue +sys.exit(1) +' 2>/dev/null; then + _ci_missing="$_ci_missing $_ci_n" + fi + done + + if [ -n "$_ci_missing" ]; then + _ci_fail "node InternalIP(s) not admitted on 2049 by the storage NetworkPolicy, so their kubelets cannot mount the workspace share:$_ci_missing (policy admits: $(printf '%s' "$_ci_cidrs" | tr '\n' ' '))" + _ci_rc=1 + fi + + if [ "$_ci_rc" -eq 0 ]; then + _ci_pass "every node InternalIP is admitted on 2049 by the storage NetworkPolicy: $(printf '%s' "$_ci_cidrs" | tr '\n' ' ')" + fi + return "$_ci_rc" +} diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml index 3b4055bd..07880038 100644 --- a/.github/workflows/build-and-test.yml +++ b/.github/workflows/build-and-test.yml @@ -24,7 +24,37 @@ jobs: sudo mv kubeconform /usr/local/bin/ - name: Install kubectl + # Pinned to the version helm/kind-action installs in the kind jobs, so + # every job in this file renders through the same kustomize (v5.7.1). + # Unpinned, setup-kubectl@v3 resolves `latest` from the frozen legacy + # GCS bucket - kubectl v1.31.0, kustomize 5.4.2 - and the static gates + # then validate output the kind clusters never see. uses: azure/setup-kubectl@v3 + with: + version: v1.35.0 + + - name: Set up Helm + # kubectl kustomize --enable-helm shells out to `helm` to inflate the + # Ganesha chart in k8s/storage/. + # + # PINNED TO THE 3.x LINE ON PURPOSE, and pinned identically at all four + # `Set up Helm` steps in this file. The kustomize built into kubectl + # <= 1.35 (v5.7.1) probes the binary with `helm version -c --short` - a + # shorthand Helm 4 removed - and then refuses any major that is not 3. + # An unpinned `latest` resolved to Helm v4.2.4 and took every job that + # renders k8s/storage/ down with + # error: unknown shorthand flag: 'c' in -c + # Do NOT "fix" this by bumping kubectl past 1.36 instead: kustomize + # v5.8.1 stopped applying the namespace transformer to Helm-inflated + # objects, so the chart's ServiceAccount, Service and StatefulSet would + # render with no metadata.namespace - contradicting the contract stated + # at the top of k8s/storage/kustomization.yaml, and silently deploying + # Ganesha outside its NetworkPolicy for anyone using the documented + # `kustomize | kubectl apply -f -` pipe. + uses: azure/setup-helm@v4 + with: + # renovate: datasource=github-releases depName=helm/helm + version: v3.21.4 - name: Validate base manifests run: | @@ -34,189 +64,285 @@ jobs: k8s/base/*.yaml - name: Validate kustomized overlay output + # Same shape as "Validate the storage root" below: rendered to a file + # and checked non-empty, because this shell has no pipefail and + # kubeconform reports success on empty input. + run: | + kubectl kustomize k8s/overlays/prod/ > /tmp/prod-lint.yaml + if [ ! -s /tmp/prod-lint.yaml ]; then + echo "::error::k8s/overlays/prod/ rendered nothing" + exit 1 + fi + kubeconform -summary -strict -kubernetes-version 1.28.0 -skip IngressRoute \ + < /tmp/prod-lint.yaml + + - name: Validate the CI overlay + # k8s/overlays/ci/ is what the two kind jobs actually apply, so it is + # the overlay whose breakage takes them down - and it is the one + # nothing else renders: every static assertion reads prod on purpose, + # because production sizing is the thing under test. + # + # kubeconform is only half of it. A `patches:` entry whose target + # matches nothing is NOT an error in kustomize 5 - it renders happily + # and silently applies nothing - so the day `rq-worker` is renamed in + # the base, this overlay would keep validating while quietly handing + # the kind jobs a full-sized 16Gi worker that no runner can schedule. + # That failure is worth ten seconds here rather than an hour into a + # kind job. Both properties are checked against the render: + # 1. the worker really is smaller than prod's, i.e. the patch landed; + # 2. requests still equal limits, so the shrink cannot introduce a + # Burstable worker that makes assert_worker_qos_guaranteed report + # a QoS regression existing nowhere but in CI's own sizing. + run: | + kubectl kustomize k8s/overlays/ci/ > /tmp/ci-rendered.yaml + kubeconform -summary -strict -kubernetes-version 1.28.0 -skip IngressRoute \ + < /tmp/ci-rendered.yaml + kubectl kustomize k8s/overlays/prod/ > /tmp/prod-rendered.yaml + # Fenced like its two siblings at "Validate the prod overlay" and + # "Validate the storage root". `bash -e` catches a render that FAILS; + # it does not catch one that succeeds and emits zero documents. That + # case matters more here than for ci-rendered.yaml above, which the + # CI_RES null probe below already covers: an empty prod render leaves + # PROD_RES empty while CI_RES holds a real value, so the + # "CI_RES = PROD_RES" comparison is FALSE, passes, and prints a real + # worker size having compared it against nothing. + if [ ! -s /tmp/prod-rendered.yaml ]; then + echo "::error::k8s/overlays/prod/ rendered nothing; the ci-vs-prod worker comparison below would compare against an empty file and pass" + exit 1 + fi + WORKER_RESOURCES='select(.kind == "Deployment" and (.metadata.name | test("rq-worker$"))) + | .spec.template.spec.containers[] | select(.name == "rq-worker") | .resources' + CI_RES=$(yq "$WORKER_RESOURCES" /tmp/ci-rendered.yaml) + PROD_RES=$(yq "$WORKER_RESOURCES" /tmp/prod-rendered.yaml) + if [ -z "$CI_RES" ] || [ "$CI_RES" = "null" ]; then + echo "::error::k8s/overlays/ci/ renders no rq-worker container resources - the patch target has drifted" + exit 1 + fi + if [ "$CI_RES" = "$PROD_RES" ]; then + echo "::error::k8s/overlays/ci/ renders the same worker size as prod, so its resource patch matched nothing; the kind jobs would deploy a worker no runner can schedule" + echo "$CI_RES" + exit 1 + fi + if [ "$(yq "$WORKER_RESOURCES | .requests" /tmp/ci-rendered.yaml)" != \ + "$(yq "$WORKER_RESOURCES | .limits" /tmp/ci-rendered.yaml)" ]; then + echo "::error::the CI worker has requests != limits, so it is Burstable; assert_worker_qos_guaranteed would fail on a regression that exists only in CI's sizing" + echo "$CI_RES" + exit 1 + fi + echo "CI worker sized down to:" + echo "$CI_RES" + + - name: Validate the storage root + # Rendered, not file-by-file: k8s/storage/ carries a Helm values file, + # which is not a Kubernetes manifest and would fail kubeconform on + # sight, and rendering is the only way the chart's own output gets + # validated at all. --enable-helm is inert on a root with no helmCharts + # field, so this keeps working if the chart is ever vendored. + # + # Rendered to a file rather than piped. `bash -e {0}` does not set + # pipefail, so a failing kustomize on the left of a pipe was invisible: + # this step printed the Helm 4 error, then + # Summary: 0 resource found parsing stdin - Valid: 0 + # and the job concluded success. The `[ -s ]` test is the load-bearing + # half of the fix, not the redirect - kubeconform exits 0 on empty + # input, so a render that SUCCEEDS and emits zero documents would still + # pass a pipefail-corrected pipe. run: | - kubectl kustomize k8s/overlays/prod/ | \ - kubeconform -summary -strict -kubernetes-version 1.28.0 -skip IngressRoute + kubectl kustomize --enable-helm k8s/storage/ > /tmp/storage-rendered.yaml + if [ ! -s /tmp/storage-rendered.yaml ]; then + echo "::error::k8s/storage/ rendered nothing; the chart was never validated" + exit 1 + fi + kubeconform -summary -strict -kubernetes-version 1.28.0 \ + < /tmp/storage-rendered.yaml - build-amd64: - # amd64 path. Produces per-arch tags `--amd64`; the - # multi-arch manifest under `-` (and `latest`) is stitched - # together in `create-manifest` once the sibling `build-arm64` succeeds. - needs: lint-manifests + assert-invariants: + # Static assertions over the manifests themselves. This job deliberately + # gates nothing - it has no `needs:` and nothing needs it - so that a red + # invariant does not also hide the build and kind suites, which are the + # only place cluster behaviour is observable. + # + # `assert_no_node_pinning_anywhere` is green as of the commit that deleted + # k8s/components/memory-tier-*/nodeselector.yaml, and from here on it is a + # ratchet: it exists to stop the pinning being reintroduced by a fork + # rebase, which is exactly how it would come back. Its staying green is + # not automatic - the NetworkPolicy step below renders the storage root, + # and that render makes kustomize vendor the Ganesha chart, whose own + # `nodeSelector` keys would otherwise be found by the pinning scan for + # ever. k8s/storage/charts/ is .gitignored and the scan asks git for its + # file list with `--exclude-standard`, so the two steps do not collide. runs-on: ubuntu-latest - permissions: - contents: read - packages: write - strategy: - fail-fast: false - matrix: - include: - - variant: full - dockerfile: Dockerfile - - variant: simple - dockerfile: Dockerfile_simple steps: - uses: actions/checkout@v4 - - name: Compute lowercase image name (OCI refs must be lowercase) - run: echo "IMAGE_NAME_LC=${IMAGE_NAME,,}" >> "$GITHUB_ENV" - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - - name: Log in to GHCR - if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository - uses: docker/login-action@v3 - with: - registry: ${{ env.REGISTRY }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Extract metadata (tags, labels) - id: meta - uses: docker/metadata-action@v5 + - name: Install kubectl + # Pinned to the version helm/kind-action installs in the kind jobs, so + # every job in this file renders through the same kustomize (v5.7.1). + # Unpinned, setup-kubectl@v3 resolves `latest` from the frozen legacy + # GCS bucket - kubectl v1.31.0, kustomize 5.4.2 - and the static gates + # then validate output the kind clusters never see. + uses: azure/setup-kubectl@v3 with: - images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} - tags: | - type=ref,event=branch,suffix=-${{ matrix.variant }}-amd64 - type=ref,event=tag,suffix=-${{ matrix.variant }}-amd64 - type=sha,prefix=,suffix=-${{ matrix.variant }}-amd64 - type=raw,value=latest-amd64,enable=${{ matrix.variant == 'full' && github.event_name == 'push' && github.ref == 'refs/heads/main' }} + version: v1.35.0 - - name: Build and conditionally push - uses: docker/build-push-action@v5 - with: - context: . - file: ${{ matrix.dockerfile }} - platforms: linux/amd64 - load: true - push: ${{ github.event_name != 'pull_request' }} - tags: ${{ steps.meta.outputs.tags }} - labels: ${{ steps.meta.outputs.labels }} - # provenance/attestations turn the pushed tag into a manifest list, - # which the create-manifest job's `docker manifest create` then - # refuses ("is a manifest list"). Keep the push as a single-platform - # image manifest — same as the build-arm64 job. - provenance: false - cache-from: type=registry,ref=${{ env.REGISTRY }}/${{ env.IMAGE_NAME_LC }}/cache:${{ matrix.variant }}-amd64 - cache-to: ${{ github.event_name != 'pull_request' && format('type=registry,ref={0}/{1}/cache:{2}-amd64,mode=max', env.REGISTRY, env.IMAGE_NAME_LC, matrix.variant) || '' }} - build-args: | - GITHUB_TOKEN=${{ secrets.GITHUB_TOKEN }} - - - name: Retag for kind (image name the kustomize overlay points at) - run: | - # The prod overlay sets `newName: ghcr.io/openms/streamlit-template`, - # `newTag: main-full`. The rendered manifests reference that exact - # ref, so we need it loaded into kind under that name. Tag invariant - # across branches/variants so the test always works. - FIRST_TAG=$(printf '%s\n' "${{ steps.meta.outputs.tags }}" | head -n 1) - docker tag "$FIRST_TAG" ghcr.io/openms/streamlit-template:main-full - - - name: Save image as tar - run: docker save ghcr.io/openms/streamlit-template:main-full -o /tmp/image.tar - - - name: Upload image artifact - uses: actions/upload-artifact@v4 + - name: Set up Helm + # kubectl kustomize --enable-helm shells out to `helm` to inflate the + # Ganesha chart in k8s/storage/. + # + # PINNED TO THE 3.x LINE ON PURPOSE, and pinned identically at all four + # `Set up Helm` steps in this file. The kustomize built into kubectl + # <= 1.35 (v5.7.1) probes the binary with `helm version -c --short` - a + # shorthand Helm 4 removed - and then refuses any major that is not 3. + # An unpinned `latest` resolved to Helm v4.2.4 and took every job that + # renders k8s/storage/ down with + # error: unknown shorthand flag: 'c' in -c + # Do NOT "fix" this by bumping kubectl past 1.36 instead: kustomize + # v5.8.1 stopped applying the namespace transformer to Helm-inflated + # objects, so the chart's ServiceAccount, Service and StatefulSet would + # render with no metadata.namespace - contradicting the contract stated + # at the top of k8s/storage/kustomization.yaml, and silently deploying + # Ganesha outside its NetworkPolicy for anyone using the documented + # `kustomize | kubectl apply -f -` pipe. + uses: azure/setup-helm@v4 with: - name: openms-streamlit-${{ matrix.variant }}-amd64-image - path: /tmp/image.tar - retention-days: 1 + # renovate: datasource=github-releases depName=helm/helm + version: v3.21.4 + + - name: Check the assertion library + # shellcheck is run unconditionally, not behind `if command -v`. Guarded, + # a missing shellcheck produced a green step with no line saying so - and + # since shellcheck prints nothing when it finds nothing, the log could + # not distinguish "ran clean" from "never ran". That is the same + # pass-on-zero-work shape as `if command -v helm`, which + # assert_no_node_pinning_anywhere replaced with a hard `_ci_require`. + # ubuntu-latest ships shellcheck; if a future image drops it, this step + # going red with "command not found" is the honest answer. + # + # -S error, not -S warning: raising it may well be free, but the only + # evidence for that is a single unreproduced run, and sizing the batch + # belongs on a runner rather than in this commit. + run: | + bash -n .github/scripts/ci-assertions.sh + shellcheck --version | sed -n '2p' + shellcheck -S error .github/scripts/ci-assertions.sh + + - name: Assert the storage NetworkPolicy matches the overlay + # Runs before the pinning check because it is the step that renders the + # storage root, and the pinning scan's exclusion of the vendored chart + # is the thing most likely to rot; having the render happen first means + # the scan always runs against the state a real job leaves behind. + run: | + source .github/scripts/ci-assertions.sh + assert_netpol_string_matches_overlay + + - name: Assert device-based fsids are pinned off + # The static half of invariant 2. It lives here as well as inside + # assert_stable_identity_across_restart so that it runs on every push, + # in seconds, rather than only after a full kind deploy - the failure + # it catches surfaces weeks later as ESTALE, so cheap and early is + # worth more than thorough and late. + run: | + source .github/scripts/ci-assertions.sh + assert_fsids_pinned + + - name: Assert the storage identity values are pinned + # The other three quarters of invariant 2. device-based-fsids above is + # only one of the four values that make the NFS server come back + # serving the same bytes under the same identity; a values edit + # dropping Retain, the fixed backing claim or the single replica + # rendered clean, deployed clean and passed every runtime assertion, + # and cost deleted data rather than a red build. + run: | + source .github/scripts/ci-assertions.sh + assert_storage_identity_values - build-arm64: - # arm64 path. Runs on a native ARM64 runner (no QEMU). Produces per-arch - # tags `--arm64`; gets merged into the multi-arch manifest - # under `-` by the `create-manifest` job below. The build - # uses a separate `Dockerfile.arm` / `Dockerfile_simple.arm` that swaps - # the miniforge installer to aarch64 and (for the full variant) guards - # the THIRDPARTY/Linux/aarch64 copy. The built image is also uploaded as - # an artifact so the apptainer / nginx / traefik integration jobs can - # exercise the ARM image on a native ARM runner (matrix arch=arm64). + - name: Assert no node pinning anywhere + # Invariant 1: Kubernetes decides placement, the config must not. + run: | + source .github/scripts/ci-assertions.sh + assert_no_node_pinning_anywhere + + - name: Assert every documented .md reference resolves + # The commit that moved the three storage research documents under docs/ + # repointed every reference from k8s/, .github/, src/ and tests/, and + # missed the eight the documents make to each other. One of those was + # the storage rollback procedure, which k8s/base/workspace-pvc.yaml + # cites as authoritative - so the outer hop resolved and the inner one + # led nowhere. Cheap enough to run on every push; nothing else in the + # pipeline reads a doc path at all. + run: | + source .github/scripts/ci-assertions.sh + assert_doc_links_resolve + + # Four per-variant, per-arch build jobs, all routed through the reusable + # .github/workflows/build-image.yml. Until 2026-08-25 these were two jobs, + # `build-amd64` and `build-arm64`, each carrying a {full, simple} matrix. + # + # Separate JOBS rather than matrix legs because `needs:` is job-level, not + # leaf-level: a failure in one leg marks the whole job failed and skips every + # consumer, so a break in one image variant took the entire kind integration + # suite with it. build-image.yml's header records what that cost. Consumers + # below now name exactly the variants they actually download. + + build-full-amd64: needs: lint-manifests - runs-on: ubuntu-24.04-arm permissions: contents: read packages: write - strategy: - fail-fast: false - matrix: - include: - - variant: full - dockerfile: Dockerfile.arm - - variant: simple - dockerfile: Dockerfile_simple.arm - steps: - - name: Free disk space - # OpenMS source build needs ~25 GB of scratch space; the ARM runner - # image is tighter than the AMD one out of the box. Mirrors what - # FLASHApp's publish-docker-images.yml does at the top of its ARM job. - run: | - # Keep /opt/hostedtoolcache: helm/kind-action and setup-kubectl - # cache binaries there and fail if the directory is missing. - sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc || true - sudo apt-get clean - df -h - - - uses: actions/checkout@v4 - - - name: Compute lowercase image name (OCI refs must be lowercase) - run: echo "IMAGE_NAME_LC=${IMAGE_NAME,,}" >> "$GITHUB_ENV" - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - - name: Log in to GHCR - if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository - uses: docker/login-action@v3 - with: - registry: ${{ env.REGISTRY }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Extract metadata (tags, labels) - id: meta - uses: docker/metadata-action@v5 - with: - images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} - tags: | - type=ref,event=branch,suffix=-${{ matrix.variant }}-arm64 - type=ref,event=tag,suffix=-${{ matrix.variant }}-arm64 - type=sha,prefix=,suffix=-${{ matrix.variant }}-arm64 - type=raw,value=latest-arm64,enable=${{ matrix.variant == 'full' && github.event_name == 'push' && github.ref == 'refs/heads/main' }} - - - name: Build and conditionally push - uses: docker/build-push-action@v5 - with: - context: . - file: ${{ matrix.dockerfile }} - platforms: linux/arm64 - load: true - push: ${{ github.event_name != 'pull_request' }} - tags: ${{ steps.meta.outputs.tags }} - labels: ${{ steps.meta.outputs.labels }} - cache-from: type=registry,ref=${{ env.REGISTRY }}/${{ env.IMAGE_NAME_LC }}/cache:${{ matrix.variant }}-arm64 - cache-to: ${{ github.event_name != 'pull_request' && format('type=registry,ref={0}/{1}/cache:{2}-arm64,mode=max', env.REGISTRY, env.IMAGE_NAME_LC, matrix.variant) || '' }} - provenance: false - build-args: | - GITHUB_TOKEN=${{ secrets.GITHUB_TOKEN }} - - - name: Retag for kind (image name the kustomize overlay points at) - run: | - # The prod overlay sets `newName: ghcr.io/openms/streamlit-template`, - # `newTag: main-full`. The rendered manifests reference that exact - # ref, so we need it loaded into kind under that name. Tag invariant - # across branches/variants so the test always works. - FIRST_TAG=$(printf '%s\n' "${{ steps.meta.outputs.tags }}" | head -n 1) - docker tag "$FIRST_TAG" ghcr.io/openms/streamlit-template:main-full - - - name: Save image as tar - run: docker save ghcr.io/openms/streamlit-template:main-full -o /tmp/image.tar - - - name: Upload image artifact - uses: actions/upload-artifact@v4 - with: - name: openms-streamlit-${{ matrix.variant }}-arm64-image - path: /tmp/image.tar - retention-days: 1 + uses: ./.github/workflows/build-image.yml + with: + variant: full + dockerfile: Dockerfile + arch: amd64 + runner: ubuntu-latest + platform: linux/amd64 + secrets: inherit + + build-simple-amd64: + needs: lint-manifests + permissions: + contents: read + packages: write + uses: ./.github/workflows/build-image.yml + with: + variant: simple + dockerfile: Dockerfile_simple + arch: amd64 + runner: ubuntu-latest + platform: linux/amd64 + secrets: inherit + + build-full-arm64: + # Native ARM64 runner, no QEMU. `Dockerfile.arm` swaps the miniforge + # installer to aarch64 and guards the THIRDPARTY/Linux/aarch64 copy. + needs: lint-manifests + permissions: + contents: read + packages: write + uses: ./.github/workflows/build-image.yml + with: + variant: full + dockerfile: Dockerfile.arm + arch: arm64 + runner: ubuntu-24.04-arm + platform: linux/arm64 + free_disk_space: true + secrets: inherit + + build-simple-arm64: + needs: lint-manifests + permissions: + contents: read + packages: write + uses: ./.github/workflows/build-image.yml + with: + variant: simple + dockerfile: Dockerfile_simple.arm + arch: arm64 + runner: ubuntu-24.04-arm + platform: linux/arm64 + free_disk_space: true + secrets: inherit create-manifest: # Stitch the per-arch tags into multi-arch manifest lists. The manifest @@ -224,7 +350,18 @@ jobs: # consumers (k8s overlays, docker-compose users, `docker pull` callers) # keep working transparently — docker now auto-selects the right arch # on pull. PRs don't push per-arch tags, so there's nothing to merge. - needs: [build-amd64, build-arm64] + # + # The kind jobs are dependencies, not just the builds. This is what + # publishes `latest` and the multi-arch tags the de.NBI cluster pulls, and + # until now nothing in the graph named test-nginx or test-traefik at all - + # so a merge published an image whose multi-node storage tier had never been + # exercised on a cluster. `main` carries no branch protection either, so the + # graph is the only place this can be enforced. + # + # Accepted cost: every push to main now waits on a ~40 minute kind matrix, + # and a teardown failure blocks publishing for a reason unrelated to the + # image. That is the trade; the alternative was publishing untested. + needs: [build-full-amd64, build-simple-amd64, build-full-arm64, build-simple-arm64, test-nginx, test-traefik] if: github.event_name != 'pull_request' runs-on: ubuntu-latest permissions: @@ -295,7 +432,17 @@ jobs: # Building apptainer from source on the arm runner would add ~15 min and # significant maintenance surface for limited value (HPC SIF consumers # remain amd64). Re-evaluate if upstream starts publishing arm64 builds. - needs: build-amd64 + # + # Both amd64 variants, because this job has its own {full, simple} matrix + # and each leg downloads the matching artifact. + # + # Deliberately NOT given the `if: always() && ...` override the kind jobs + # below carry: `needs:` skips this whole job if either build fails, so a + # broken `full` image still costs the `simple` apptainer leg. That is + # accepted. Apptainer is a side deployment - the cluster runs the docker + # image - so the coverage worth rescuing from a one-variant break is the + # kind suite, not this. Add the same override here if that ever changes. + needs: [build-full-amd64, build-simple-amd64] runs-on: ubuntu-latest strategy: fail-fast: false @@ -308,7 +455,8 @@ jobs: # ubuntu-latest has ~14 GB free; the full image (5-8 GB) plus kind # node image plus loading the OCI tar into both docker and kind can # exhaust it. The arm runner is even tighter. Same incantation as - # `build-arm64`'s "Free disk space" step. + # the "Free disk space" step in .github/workflows/build-image.yml, which + # the arm64 callers enable via free_disk_space. run: | # Keep /opt/hostedtoolcache: helm/kind-action and setup-kubectl # cache binaries there and fail if the directory is missing. @@ -525,7 +673,30 @@ jobs: done <<< "${{ steps.meta.outputs.tags }}" test-nginx: - needs: [build-amd64, build-arm64] + # All four build jobs, so this job waits for every image it might download - + # `needs:` is what provides the ordering, and it is kept. + # + # `if: always()` overrides only the OTHER thing `needs:` does, which is skip + # this job when a dependency fails. Without it a broken `full` build skipped + # all four leaves here, including the two `simple` ones whose images had + # been built and uploaded successfully - 100% of the storage-tier runtime + # coverage lost to a break in one image variant. + # + # The condition is the `simple` pair specifically. Those legs install a + # prebuilt pyopenms wheel and are the robust ones; the `full` legs compile + # OpenMS from source and are where upstream breakage lands. When `full` is + # broken, the `simple` leaves run and assert everything, and the `full` + # leaves go red at "Download image artifact" - honest, since their image + # genuinely does not exist. When the `simple` builds are themselves broken + # there is nothing to test against, and the whole job skips as before. + # + # No staleness risk: download-artifact@v4 is scoped to the current run, so a + # leaf either gets THIS run's image or fails. It cannot silently test the + # previous one. + needs: [build-full-amd64, build-simple-amd64, build-full-arm64, build-simple-arm64] + if: always() && !cancelled() + && needs.build-simple-amd64.result == 'success' + && needs.build-simple-arm64.result == 'success' runs-on: ${{ matrix.runner }} strategy: fail-fast: false @@ -550,7 +721,8 @@ jobs: # ubuntu-latest has ~14 GB free; the full image (5-8 GB) plus kind # node image plus loading the OCI tar into both docker and kind can # exhaust it. The arm runner is even tighter. Same incantation as - # `build-arm64`'s "Free disk space" step. + # the "Free disk space" step in .github/workflows/build-image.yml, which + # the arm64 callers enable via free_disk_space. run: | # Keep /opt/hostedtoolcache: helm/kind-action and setup-kubectl # cache binaries there and fail if the directory is missing. @@ -559,14 +731,34 @@ jobs: df -h - name: Download image artifact + id: download-image uses: actions/download-artifact@v4 with: name: openms-streamlit-${{ matrix.variant }}-${{ matrix.arch }}-image path: /tmp + - name: Explain a missing image + # This job runs under `if: always() && `, + # so when the `full` build is broken its two leaves reach here and the + # download above fails with "unable to find any artifacts" - true, but + # it names neither the variant nor the cause. Say both, then stay red: + # the leaf genuinely tested nothing and must not report success. + if: always() && steps.download-image.outcome == 'failure' + run: | + echo "::error::openms-streamlit-${{ matrix.variant }}-${{ matrix.arch }}-image was not produced by this run, so the ${{ matrix.variant }} ${{ matrix.arch }} build failed earlier. This leaf tested nothing; the other variant's leaves are unaffected and their results stand." + exit 1 + - name: Create kind cluster uses: helm/kind-action@v1 with: + # Pinned to what the action picks today. kubectl_version must stay + # equal to the `Install kubectl` pin above, or the static jobs and + # the kind jobs render k8s/storage/ through different kustomize + # versions and disagree about namespaces on Helm-inflated objects. + # node_image is deliberately NOT pinned: a digest that goes stale + # fails cluster creation outright, with no fallback. + version: v0.31.0 + kubectl_version: v1.35.0 cluster_name: test-cluster config: .github/kind-config.yaml @@ -578,33 +770,336 @@ jobs: run: | kind load image-archive /tmp/image.tar --name test-cluster rm -f /tmp/image.tar + # Diagnostic only. The image has just been written into both kind + # nodes' containerd, and nfs-provisioner refuses claims larger than + # the free space under /export - so when a claim fails to bind, this + # line is the difference between "out of disk" and a lost afternoon. + df -h / || true - name: Install nginx ingress controller run: | - kubectl apply -f https://raw.githubusercontent.com/kubernetes/ingress-nginx/main/deploy/static/provider/kind/deploy.yaml + # Pinned. `main` was the last unpinned upstream of the class that took + # this whole pipeline down through Helm 4 - and here the usual + # moving-ref trap has inverted into rot, because the repository is + # archived: the ref will not move again, but the kind node image will. + # + # controller-v1.15.1, not v1.14.0: upstream's support table at this + # tag lists v1.15.1 as covering k8s 1.31-1.35 and v1.14.0 as covering + # only 1.30-1.34, and the cluster created above is + # kindest/node:v1.35.0. + # + # That cluster version is PINNED, indirectly: `version: v0.31.0` on + # kind-action fixes the default node image, because kind v0.31.0 + # ships kindest/node:v1.35.0. It does not drift on its own. The + # separate decision not to pin `node_image` is made at that step for + # an unrelated reason - a digest that goes stale fails cluster + # creation outright, with no fallback. + # + # So the trigger for re-checking this tag is the `version:` pin + # moving, not the passage of time: bump kind, then read the support + # table again. + kubectl apply -f https://raw.githubusercontent.com/kubernetes/ingress-nginx/controller-v1.15.1/deploy/static/provider/kind/deploy.yaml kubectl wait --namespace ingress-nginx --for=condition=ready pod --selector=app.kubernetes.io/component=controller --timeout=90s + - name: Set up Helm + # kubectl kustomize --enable-helm shells out to `helm` to inflate the + # Ganesha chart in k8s/storage/. + # + # PINNED TO THE 3.x LINE ON PURPOSE, and pinned identically at all four + # `Set up Helm` steps in this file. The kustomize built into kubectl + # <= 1.35 (v5.7.1) probes the binary with `helm version -c --short` - a + # shorthand Helm 4 removed - and then refuses any major that is not 3. + # An unpinned `latest` resolved to Helm v4.2.4 and took every job that + # renders k8s/storage/ down with + # error: unknown shorthand flag: 'c' in -c + # Do NOT "fix" this by bumping kubectl past 1.36 instead: kustomize + # v5.8.1 stopped applying the namespace transformer to Helm-inflated + # objects, so the chart's ServiceAccount, Service and StatefulSet would + # render with no metadata.namespace - contradicting the contract stated + # at the top of k8s/storage/kustomization.yaml, and silently deploying + # Ganesha outside its NetworkPolicy for anyone using the documented + # `kustomize | kubectl apply -f -` pipe. + uses: azure/setup-helm@v4 + with: + # renovate: datasource=github-releases depName=helm/helm + version: v3.21.4 + + - name: Report the NFS client helper on the kind nodes + # Diagnostic, not a gate - the gate is whether the claim binds and the + # pods come up. An in-tree `nfs:` PV is mounted by the kubelet on the + # node, which needs /sbin/mount.nfs there; without this line a missing + # helper surfaces several steps later as "wrong fs type, bad option, + # bad superblock" in a pod event. Same probe as the storage runbook's + # section 0 runs against the real nodes. + # + # The /proc/filesystems half needs the modprobe and the explainer: the + # nfs module autoloads on FIRST MOUNT, and this step runs before any + # mount exists, so the grep printed nothing on every node of every job + # in a run where the same nodes went on to mount `type=nfs4 vers=4.1` + # two minutes later. This is the line a triager reads to decide whether + # the module is present, and unqualified it answered "absent" about a + # perfectly working host. + run: | + for n in $(kind get nodes --name test-cluster); do + echo "--- $n" + docker exec "$n" sh -c 'ls -l /sbin/mount.nfs* /usr/sbin/mount.nfs* 2>&1; echo ---; if modprobe nfs 2>/dev/null; then MP="modprobe loaded it"; else MP="modprobe could not (absent, or no such module)"; fi; grep nfs /proc/filesystems || echo "(nfs absent from /proc/filesystems; $MP. Loaded-and-still-absent is CONCLUSIVE: this node cannot mount NFS. Otherwise the module may still autoload on first mount, and this says nothing.)"' || true + done + + - name: Deploy the storage root (NFS-Ganesha) + # Before the overlay, because workspaces-pvc is ReadWriteMany on the + # StorageClass this root publishes: with no provisioner running the + # claim never binds and every app pod stays Pending. Ganesha is just a + # pod on a PVC and kind's `standard` class is RWO local-path, so the + # whole NFS re-export genuinely runs here. + run: | + # Nothing vendors the chart - .gitignore excludes k8s/storage/charts/ + # on purpose - so every job re-fetches index.yaml and the tarball from + # GitHub Pages. A transient 5xx there presents as exactly the same + # "kubectl kustomize --enable-helm failed" this pipeline has already + # been taken down by once, so retry the render rather than the apply + # alone. + rendered=0 + for i in 1 2 3; do + if kubectl kustomize --enable-helm k8s/storage/ > /tmp/storage-rendered.yaml; then + rendered=1 + break + fi + echo "Render attempt $i failed, retrying in ${i}0s..." + sleep "${i}0" + done + if [ "$rendered" -ne 1 ] || [ ! -s /tmp/storage-rendered.yaml ]; then + echo "::error::could not render k8s/storage/ (needs Helm 3.x on PATH)" + exit 1 + fi + # Ganesha's own backing claim is now the only Cinder-classed object in + # the tree. kind has no cinder-csi, so rewrite it to `standard`, and + # fail loudly if the string is gone rather than letting the rewrite + # no-op and quietly take the storage path out of CI with it. + if ! grep -q 'storageClassName: cinder-csi' /tmp/storage-rendered.yaml; then + echo "::error::k8s/storage/ no longer renders 'storageClassName: cinder-csi'; the rewrite below would silently no-op" + exit 1 + fi + # 500Gi is right for de.NBI and absurd on a runner. local-path never + # checks capacity so it binds anyway, but nfs-provisioner statfs()es + # /export and refuses any workspace claim larger than the free space + # it reports - and a 500Gi Bound claim in the failure dump sends the + # next reader looking in the wrong place entirely. + if ! grep -q 'storage: 500Gi' /tmp/storage-rendered.yaml; then + echo "::error::k8s/storage/ no longer renders 'storage: 500Gi'; the size rewrite below would silently no-op" + exit 1 + fi + sed -e 's|storageClassName: cinder-csi|storageClassName: standard|g' \ + -e 's|storage: 500Gi|storage: 8Gi|' \ + /tmp/storage-rendered.yaml > /tmp/storage.yaml + # The node CIDR in networkpolicy.yaml ships as RFC 5737 TEST-NET-1 and + # is documented there as a placeholder to be set before the first + # deploy. It is not cosmetic here: kindnetd runs kube-network-policies + # unconditionally, the namespace default-deny therefore applies, and + # the in-tree `nfs:` PVs this chart emits are mounted by the KUBELET + # from the node's own address in the host netns - matching no + # podSelector and no ipBlock. Left alone, every mount on a node not + # running Ganesha hangs, which the worker's DoNotSchedule spread + # guarantees will happen to exactly one replica. + if ! grep -q 'cidr: 192.0.2.0/24' /tmp/storage.yaml; then + echo "::error::k8s/storage/networkpolicy.yaml no longer carries the 192.0.2.0/24 placeholder; this rewrite would silently no-op and every cross-node mount would hang" + exit 1 + fi + # Derived, not hardcoded: kind pins only its IPv6 subnet, so the IPv4 + # one comes from Docker's default address pool and is not 172.18/16 + # by contract. + NODE_CIDR=$(docker network inspect kind \ + -f '{{range .IPAM.Config}}{{.Subnet}} {{end}}' \ + | tr ' ' '\n' | grep -v ':' | grep -E '^[0-9.]+/[0-9]+$' | head -n1) + if [ -z "$NODE_CIDR" ]; then + echo "::error::could not read an IPv4 subnet from the kind docker network" + docker network inspect kind -f '{{json .IPAM.Config}}' + exit 1 + fi + echo "admitting kind nodes on 2049 from $NODE_CIDR" + sed -i "s|cidr: 192.0.2.0/24|cidr: ${NODE_CIDR}|" /tmp/storage.yaml + # kubeconform validated /tmp/storage-rendered.yaml, not the file that + # is actually applied. Re-check the one value the seds above compute + # rather than copy, so a malformed CIDR fails here and names itself + # instead of surfacing as an API-server rejection mid-retry-loop. + if ! grep -Eq '^[[:space:]]*cidr: [0-9]+\.[0-9]+\.[0-9]+\.[0-9]+/[0-9]+$' /tmp/storage.yaml; then + echo "::error::the rewritten NetworkPolicy CIDR is not a bare IPv4 block" + grep -n 'cidr:' /tmp/storage.yaml + exit 1 + fi + applied=0 + for i in 1 2 3 4 5; do + if kubectl apply -f /tmp/storage.yaml; then + applied=1 + echo "Storage apply succeeded on attempt $i" + break + fi + echo "Attempt $i failed, retrying in ${i}0s..." + sleep "${i}0" + done + # `if ... then break; fi` never trips `set -e`, so without this the + # loop fell through on total failure and the step died on the bare + # NotFound from the substitution below - taking the authored error + # message and its diagnostic dump with it, in precisely the case the + # retry loop exists to report. + if [ "$applied" -ne 1 ]; then + echo "::error::could not apply k8s/storage/ after 5 attempts" + kubectl get all -n template-app-storage || true + exit 1 + fi + # `kubectl wait --all` reports success against zero objects, so the + # server has to be found before it is waited on. `|| true` because an + # empty result must reach the explicit check below rather than kill + # the step through `bash -e`. + WORKLOADS=$(kubectl get deploy,statefulset -n template-app-storage -o name 2>/dev/null || true) + if [ -z "$WORKLOADS" ]; then + echo "::error::k8s/storage/ created no Deployment or StatefulSet in template-app-storage" + kubectl get all -n template-app-storage + exit 1 + fi + for w in $WORKLOADS; do + kubectl rollout status -n template-app-storage "$w" --timeout=300s + done + kubectl get pods,pvc -n template-app-storage -o wide + kubectl get storageclass + # Diagnostic, never a gate: this is the filesystem nfs-provisioner + # measures when it decides whether a workspace claim fits. + kubectl exec -n template-app-storage \ + "$(kubectl get pod -n template-app-storage -l app=nfs-server -o name | head -n1)" \ + -- df -h /export || true + - name: Deploy with Kustomize run: | + # k8s/overlays/ci/, not prod: prod's rq-worker is 16Gi / 4 cpu with + # requests == limits, which is right on the de.NBI nodes and + # impossible here. This runner has 4 vCPU and ~15.6Gi in total, every + # kind node is a container on that one host advertising the whole of + # it, and kube-system already holds ~1 cpu in requests - so a single + # replica does not fit, let alone two, and both would sit Pending + # until "Verify all deployments are available" failed at 180s and + # took every assertion below it with it. The CI overlay is prod with + # exactly one thing changed: the worker's size. Same replica count, + # same spread constraint, same labels, same images, so everything + # this suite observes is still the production arrangement. The lint + # job checks that the shrink actually applied and stayed Guaranteed. + # # Filter out Traefik IngressRoute (kind cluster uses nginx) and force imagePullPolicy=Never - kubectl kustomize k8s/overlays/prod/ | \ + # No cinder-csi rewrite here any more. The only Cinder-classed + # object left in the tree is Ganesha's own backing claim in + # k8s/storage/, and the storage step above rewrites that one - loudly + # if the string ever disappears. A sed kept here would match nothing + # and quietly imply the storage path was still being exercised. + kubectl kustomize k8s/overlays/ci/ | \ yq 'select(.kind != "IngressRoute")' | \ - sed -E 's|imagePullPolicy: (IfNotPresent\|Always)|imagePullPolicy: Never|g' | \ - sed 's|storageClassName: cinder-csi|storageClassName: standard|g' > /tmp/manifests.yaml + sed -E 's|imagePullPolicy: (IfNotPresent\|Always)|imagePullPolicy: Never|g' > /tmp/manifests.yaml + # Size the workspaces claim for the runner's disk. nfs-provisioner + # statfs()es /export on every provision and refuses outright any + # claim larger than the free space it finds there - the check is + # unconditional, not gated on quotas. In kind the export is a + # local-path volume on the GitHub runner's disk, tens of GiB, so the + # production figure could never bind and every pod would sit Pending + # behind it. Fail loudly if the request has gone, rather than letting + # the rewrite no-op and take the storage path out of CI with it. + if ! grep -qE 'storage: [0-9]+Gi' /tmp/manifests.yaml; then + echo "::error::the rendered overlay has no 'storage: Gi' request; the rewrite below would silently no-op" + exit 1 + fi + # 1Gi, not 4Gi: nfs-provisioner statfs()es /export and refuses any + # claim larger than the free space it reports, and /export here is a + # local-path directory on a runner disk that has just absorbed the + # app image into both kind nodes' containerd. No assertion in this + # suite writes anywhere near a gigabyte - the largest is ~60 short + # log lines - so the headroom buys nothing and costs a 300s Bound + # timeout whose message is about bytes. + sed -i -E 's|storage: [0-9]+Gi|storage: 1Gi|g' /tmp/manifests.yaml + applied=0 for i in 1 2 3 4 5; do if kubectl apply -f /tmp/manifests.yaml; then + applied=1 echo "Deploy succeeded on attempt $i" break fi echo "Attempt $i failed, retrying in ${i}0s..." sleep "${i}0" done + # Same fix as the storage apply loop above. `kubectl apply` inside an + # `if` condition is exempt from `set -e`, so on five failures the last + # command run is `sleep`, the loop exits 0, and the step passes having + # created nothing - surfacing three steps later as "Wait for Redis to + # be ready" timing out on a pod that never existed. + if [ "$applied" -ne 1 ]; then + echo "::error::could not apply the overlay after 5 attempts" + kubectl get all -n openms || true + exit 1 + fi - name: Discover overlay identity + # Still read off prod, even though ci/ is what was applied: the CI + # overlay has no commonLabels of its own, it inherits prod's whole + # identity and only repatches the worker's resources. Reading the + # source of that identity keeps this working if the CI overlay ever + # grows a second patch. run: | SLUG=$(yq '.commonLabels.app' k8s/overlays/prod/kustomization.yaml) echo "SLUG=$SLUG" >> "$GITHUB_ENV" + - name: Verify the workspace StorageClass contract + # k8s/base/workspace-pvc.yaml names a class k8s/storage/ has to publish, + # and the mount options ride on that class because a PVC has nowhere to + # carry them. Checking all of it here turns drift into one failed step + # with the actual values printed, instead of a Pending pod at 3am. + run: | + # Read the claim off the Deployment that mounts it, so namePrefix and + # any second PVC in the namespace stay irrelevant. + CLAIM=$(kubectl get deployment -n openms -l app=${SLUG},component=streamlit \ + -o jsonpath='{.items[0].spec.template.spec.volumes[?(@.name=="workspaces")].persistentVolumeClaim.claimName}') + if [ -z "${CLAIM}" ]; then + echo "::error::no streamlit Deployment in openms mounts a 'workspaces' volume - there is no claim to check" + kubectl get deployment -n openms -o wide + exit 1 + fi + kubectl wait -n openms --for=jsonpath='{.status.phase}'=Bound \ + "pvc/${CLAIM}" --timeout=300s + MODES=$(kubectl get pvc -n openms "${CLAIM}" -o jsonpath='{.status.accessModes[*]}') + case " ${MODES} " in + *" ReadWriteMany "*) ;; + *) + echo "::error::${CLAIM} bound as '${MODES}', not ReadWriteMany; every pod that mounts it is back to sharing one node" + exit 1 + ;; + esac + CLASS=$(kubectl get pvc -n openms "${CLAIM}" -o jsonpath='{.spec.storageClassName}') + echo "${CLAIM} is Bound ReadWriteMany on StorageClass '${CLASS}'" + if ! kubectl get storageclass "${CLASS}" >/dev/null 2>&1; then + echo "::error::the workspaces PVC names StorageClass '${CLASS}', which k8s/storage/ does not publish" + kubectl get storageclass + exit 1 + fi + OPTS=$(kubectl get storageclass "${CLASS}" -o jsonpath='{.mountOptions[*]}') + echo "mount options: ${OPTS:-}" + case " ${OPTS} " in + *" soft "*) + echo "::error::'${CLASS}' mounts soft; a routine Ganesha restart would then truncate in-flight writes instead of blocking" + exit 1 + ;; + *" hard "*) ;; + *) + echo "::error::'${CLASS}' does not pin 'hard'. It is also the kernel default, but the design states it explicitly so that adding 'soft' is a visible diff" + exit 1 + ;; + esac + echo "${OPTS}" | grep -qE '(^| )(nfs)?vers=4\.1( |$)' || { + echo "::error::'${CLASS}' does not pin NFSv4.1, which is what supplies integrated locking on one port" + exit 1 + } + echo "${OPTS}" | grep -qE '(^| )nconnect=4( |$)' || { + echo "::error::'${CLASS}' does not set nconnect=4" + exit 1 + } + if echo "${OPTS}" | grep -qE '(^| )(actimeo|acregmin|acregmax|acdirmin|acdirmax)='; then + echo "::error::'${CLASS}' overrides the attribute cache timeouts; close-to-open consistency already covers this app, so actimeo stays at its default" + exit 1 + fi + - name: Wait for Redis to be ready run: | kubectl wait -n openms --for=condition=ready pod -l app=${SLUG},component=redis --timeout=60s @@ -614,9 +1109,12 @@ jobs: kubectl run redis-test -n openms --image=redis:7-alpine --rm -i --restart=Never -- redis-cli -h ${SLUG}-redis.openms.svc.cluster.local ping - name: Verify all deployments are available + # No `|| true`: a Pending, CrashLooping or unschedulable Deployment has to + # fail the job, otherwise the ingress curl below is the only real gate. + # Diagnostics come from "Dump cluster state on failure". run: | - kubectl wait -n openms --for=condition=available deployment -l app=${SLUG} --timeout=180s || true - kubectl get pods -n openms -l app=${SLUG} + kubectl wait -n openms --for=condition=available deployment -l app=${SLUG} --timeout=180s + kubectl get pods -n openms -l app=${SLUG} -o wide kubectl get services -n openms -l app=${SLUG} - name: Curl both hostnames via nginx ingress @@ -638,6 +1136,117 @@ jobs: echo "$host -> 200 OK" done + # Everything below is `.github/scripts/ci-assertions.sh`. Each assertion + # discovers the namespace, the app label, the workspace claim and the + # helper image from the cluster, so SLUG above is the only input any of + # them needs. + # + # The NetworkPolicy assertion is first, and then the two placement ones, + # because all three are seconds long where the storage ones are up to 45 + # minutes of deliberate timeouts. They read the state that "Verify all + # deployments are available" has just established, and if the workers are + # not where they should be then everything after this is noise measured + # against a broken deployment. + - name: Assert every node is admitted to the NFS share + # First, because it is the failure that disguises itself as all the + # others. The share is behind a namespace-wide default-deny, the + # in-tree `nfs:` PVs are mounted by the KUBELET from the node address, + # and the CIDR that admits them is rewritten by the storage deploy step + # rather than shipped - so if that rewrite ever no-ops, every mount on + # every node except Ganesha's own hangs, and the first thing anyone + # sees is an unrelated assertion timing out forty minutes later. + timeout-minutes: 2 + run: | + source .github/scripts/ci-assertions.sh + assert_netpol_admits_every_node + + - name: Assert the worker replicas are spread across nodes + # Step 4's acceptance criterion for the workers. Every replica Running + # - so an unschedulable one fails here rather than being left out of + # the node count - on more than one node, inside the maxSkew the + # Deployment itself declares, under a spread constraint whose + # labelSelector really selects these workers. + # + # assert_two_pods_two_nodes below does NOT cover this: it starts its + # own helper pods with an explicit nodeName, which proves the volume is + # reachable from two nodes and says nothing about where the scheduler + # put the rq-workers. Deleting the nodeSelector is necessary for this + # to pass and is not sufficient - a spread constraint scoped to the + # wrong labels, or dropped entirely, fails right here. + timeout-minutes: 5 + run: | + source .github/scripts/ci-assertions.sh + assert_workers_spread_across_nodes + + - name: Assert the workers are Guaranteed QoS + # requests == limits, read off `.status.qosClass` on the RUNNING pods. + # A Burstable worker scores oom_score_adj ~969 and sits near the top of + # the node's kill list holding hours of work; the regression that + # produces it is a one-line edit to a memory-tier component and is + # otherwise invisible until something OOMKills under load. + timeout-minutes: 5 + run: | + source .github/scripts/ci-assertions.sh + assert_worker_qos_guaranteed + + - name: Assert cross-node write visibility + # Two pods holding the same claim on two different nodes at the same + # time, each reading what the other wrote. + timeout-minutes: 10 + run: | + source .github/scripts/ci-assertions.sh + assert_cross_node_write_visible + + - name: Assert the POSIX contract holds on the shared volume + # Absolute symlinks, atomic rename and flock - what src/ already + # depends on and what NFS is allowed to take away. It first checks the + # mount really is NFS, because every one of those passes against the + # node's local filesystem too, and the flock half runs across two pods + # on two nodes, because a lock that only excludes a second holder on + # the same node is a lock the two workers do not have. + timeout-minutes: 15 + run: | + source .github/scripts/ci-assertions.sh + assert_posix_contract + + - name: Assert a workflow survives an NFS server restart + # The single most valuable assertion in the set: delete the Ganesha pod + # while a workflow is running and assert the workflow COMPLETES, with + # every line of its output present and in order. A Ganesha restart is + # routine, and this is exactly the case a `soft` mount would have + # corrupted - into a truncated featureXML rather than a failed job. + # Slow by construction: the restart and the ~90s NFSv4.1 grace period + # both pass underneath a running job, which is the point. + timeout-minutes: 20 + run: | + source .github/scripts/ci-assertions.sh + assert_survives_nfs_restart + + - name: Assert stable identity across an NFS server restart + # Invariant 2. Deletes the Ganesha pod under a live client, so it is + # slow by construction: NFSv4.1 spends ~90s in its grace period. + timeout-minutes: 15 + run: | + source .github/scripts/ci-assertions.sh + assert_stable_identity_across_restart + + - name: Assert two pods on two nodes share the workspace volume + # The acceptance criterion for the whole exercise, and the last of these + # to go green: it stayed red until the memory-tier nodeSelector patches + # were deleted. Everything above it is about the export itself and + # passes as soon as ReadWriteMany works, which is what kept "RWX proven + # while still single-node" observable. + # + # It carried `continue-on-error: true` while it was red by design - a + # step that always fails makes every genuine regression around it + # indistinguishable from the expected red, because nobody looks at an + # already-failing job. That commit has landed, so the criterion gates + # like everything else here. + timeout-minutes: 10 + run: | + source .github/scripts/ci-assertions.sh + assert_two_pods_two_nodes + - name: Dump cluster state on failure if: failure() run: | @@ -645,12 +1254,31 @@ jobs: kubectl get nodes -o wide || true echo "=== pods (all namespaces) ===" kubectl get pods -A -o wide || true + echo "=== storage tier ===" + kubectl get all,pvc -n template-app-storage -o wide || true + kubectl describe pods -n template-app-storage || true + for p in $(kubectl get pods -n template-app-storage -o name 2>/dev/null); do + kubectl logs -n template-app-storage "$p" --all-containers --prefix --tail=200 || true + done + echo "=== storage classes and volumes ===" + kubectl get storageclass || true + kubectl get pv,pvc -A || true + # Unfiltered on purpose. SLUG is exported by "Discover overlay + # identity", which runs AFTER the storage steps - so any failure + # before it left ${SLUG} empty here, every `-l app=` matched nothing, + # and this dump reported an empty namespace on a cluster that simply + # had not been deployed to yet. In CI the openms namespace holds only + # this app, so dropping the selector cannot lose anything. echo "=== app pods describe ===" - kubectl describe pod -n openms -l app=${SLUG} || true + kubectl describe pod -n openms || true echo "=== app pod logs ===" - kubectl logs -n openms -l app=${SLUG} --tail=200 --all-containers --prefix || true + for p in $(kubectl get pods -n openms -o name 2>/dev/null); do + kubectl logs -n openms "$p" --all-containers --prefix --tail=200 || true + done echo "=== app pod previous logs (if crashed) ===" - kubectl logs -n openms -l app=${SLUG} --tail=200 --all-containers --prefix --previous || true + for p in $(kubectl get pods -n openms -o name 2>/dev/null); do + kubectl logs -n openms "$p" --all-containers --prefix --tail=200 --previous || true + done echo "=== ingress ===" kubectl get ingress -A -o wide || true kubectl describe ingress -n openms || true @@ -659,8 +1287,121 @@ jobs: echo "=== ingress-nginx controller logs ===" kubectl logs -n ingress-nginx -l app.kubernetes.io/component=controller --tail=200 || true + - name: Release the NFS mounts before kind tears the nodes down + # `docker rm -f` on a kind node whose kubelet still holds a `hard` + # NFSv4.1 mount cannot complete: the Ganesha server is a pod INSIDE the + # cluster being destroyed, so it dies alongside its own clients, the + # pending RPCs block in uninterruptible sleep, SIGKILL is ignored and + # containerd never sees an exit event. dockerd gives up after ~12s and + # `Post Create kind cluster` fails - in all 8 kind jobs, on both arches. + # + # This has to be the LAST NORMAL STEP: every normal step runs before any + # post step, and no `if:` on a normal step can observe a post step's + # failure (which is also why "Dump cluster state on failure" above is + # `skipped` in a job that is red only at teardown). + # + # Order is load-bearing. Diagnostics FIRST, because after the deletes + # there is nothing left to name. Then clients, while the server is still + # up and their unmounts can complete. Then a lazy detach for whatever the + # kubelet has not released - `-l` returns without waiting on the server, + # so it cannot itself hang. The server LAST: killing it while a client + # mount is still attached is the deadlock verbatim. + # + # This step gates nothing; every command is `|| true`. + if: always() + run: | + CL=test-cluster + # Derived ONCE and reused by all three loops below. They run during + # teardown, when the cluster can disappear between them, and + # re-deriving would silently shrink the list mid-step. An empty list + # is announced rather than tolerated: every loop would otherwise + # iterate zero times and the step would exit 0 having done nothing - + # indistinguishable from having worked. That is the failure a wrong + # cluster name produces, and it is why $CL is checked against + # `cluster_name` on the kind-action step above. + NODES="$(kind get nodes --name "$CL" 2>/dev/null || true)" + if [ -z "$NODES" ]; then + echo "::warning::kind get nodes --name $CL returned nothing - this step released no mounts, and a green teardown says nothing about it" + fi + + echo "=== NFS mounts and D-state processes, before teardown ===" + mounts_before=0 + for n in $NODES; do + echo "--- $n" + # Counted as well as printed. Whether there was anything to release + # is the one thing a green teardown cannot tell you on its own, and + # it is the difference between "this step fixed the deadlock" and + # "something else did". + m="$(docker exec "$n" sh -c 'grep -E " nfs4? " /proc/self/mounts' || true)" + if [ -n "$m" ]; then + printf '%s\n' "$m" + mounts_before=$((mounts_before + $(printf '%s\n' "$m" | grep -c .))) + fi + # No `2>/dev/null` on ps. Suppressed, a failing ps prints nothing, + # which is byte-identical to "no processes in D state" - a false + # negative in the diagnostic that identified round 2's deadlock. + # Keep ps rather than reading /proc//status: wchan is the + # column that named it (nfs_wait_bit_killable). kind v0.31.0's base + # image installs procps, so ps failing would itself be news. + docker exec "$n" sh -c 'ps -eo pid,stat,wchan:24,comm | awk "\$2 ~ /D/"' || true + done + if [ -n "$NODES" ]; then + if [ "$mounts_before" -eq 0 ]; then + echo "::warning::this step released no NFS mounts - there were none to release, so a green teardown proves nothing about the deadlock it exists to prevent" + else + echo "$mounts_before NFS mount(s) held before teardown" + fi + fi + + # Order is load-bearing: clients first, while the server is still up + # and their unmounts can complete; then a lazy detach for whatever the + # kubelet has not released; the server namespace LAST. + kubectl delete ns openms --wait=true --timeout=120s || true + for n in $NODES; do + docker exec "$n" sh -c 'umount -f -l -a -t nfs,nfs4' || true + done + kubectl delete ns template-app-storage --wait=false || true + + echo "=== residual NFS mounts (expected: none) ===" + residual="" + for n in $NODES; do + echo "--- $n" + r="$(docker exec "$n" sh -c 'grep -E " nfs4? " /proc/self/mounts' || true)" + if [ -n "$r" ]; then + printf '%s\n' "$r" + residual="$residual $n" + fi + done + if [ -n "$residual" ]; then + # Collapsed to one line: GitHub's ::warning:: is line-oriented. + echo "::warning::NFS mounts still held after this step on:$residual - if teardown nevertheless passes, something other than this step is responsible" + fi + test-traefik: - needs: [build-amd64, build-arm64] + # All four build jobs, so this job waits for every image it might download - + # `needs:` is what provides the ordering, and it is kept. + # + # `if: always()` overrides only the OTHER thing `needs:` does, which is skip + # this job when a dependency fails. Without it a broken `full` build skipped + # all four leaves here, including the two `simple` ones whose images had + # been built and uploaded successfully - 100% of the storage-tier runtime + # coverage lost to a break in one image variant. + # + # The condition is the `simple` pair specifically. Those legs install a + # prebuilt pyopenms wheel and are the robust ones; the `full` legs compile + # OpenMS from source and are where upstream breakage lands. When `full` is + # broken, the `simple` leaves run and assert everything, and the `full` + # leaves go red at "Download image artifact" - honest, since their image + # genuinely does not exist. When the `simple` builds are themselves broken + # there is nothing to test against, and the whole job skips as before. + # + # No staleness risk: download-artifact@v4 is scoped to the current run, so a + # leaf either gets THIS run's image or fails. It cannot silently test the + # previous one. + needs: [build-full-amd64, build-simple-amd64, build-full-arm64, build-simple-arm64] + if: always() && !cancelled() + && needs.build-simple-amd64.result == 'success' + && needs.build-simple-arm64.result == 'success' runs-on: ${{ matrix.runner }} strategy: fail-fast: false @@ -685,7 +1426,8 @@ jobs: # ubuntu-latest has ~14 GB free; the full image (5-8 GB) plus kind # node image plus loading the OCI tar into both docker and kind can # exhaust it. The arm runner is even tighter. Same incantation as - # `build-arm64`'s "Free disk space" step. + # the "Free disk space" step in .github/workflows/build-image.yml, which + # the arm64 callers enable via free_disk_space. run: | # Keep /opt/hostedtoolcache: helm/kind-action and setup-kubectl # cache binaries there and fail if the directory is missing. @@ -694,14 +1436,29 @@ jobs: df -h - name: Download image artifact + id: download-image uses: actions/download-artifact@v4 with: name: openms-streamlit-${{ matrix.variant }}-${{ matrix.arch }}-image path: /tmp + - name: Explain a missing image + # This job runs under `if: always() && `, + # so when the `full` build is broken its two leaves reach here and the + # download above fails with "unable to find any artifacts" - true, but + # it names neither the variant nor the cause. Say both, then stay red: + # the leaf genuinely tested nothing and must not report success. + if: always() && steps.download-image.outcome == 'failure' + run: | + echo "::error::openms-streamlit-${{ matrix.variant }}-${{ matrix.arch }}-image was not produced by this run, so the ${{ matrix.variant }} ${{ matrix.arch }} build failed earlier. This leaf tested nothing; the other variant's leaves are unaffected and their results stand." + exit 1 + - name: Create kind cluster uses: helm/kind-action@v1 with: + # Keep in lockstep with the nginx job and the `Install kubectl` pins. + version: v0.31.0 + kubectl_version: v1.35.0 cluster_name: traefik-test config: .github/kind-config.yaml @@ -713,50 +1470,340 @@ jobs: run: | kind load image-archive /tmp/image.tar --name traefik-test rm -f /tmp/image.tar + # Diagnostic only. The image has just been written into both kind + # nodes' containerd, and nfs-provisioner refuses claims larger than + # the free space under /export - so when a claim fails to bind, this + # line is the difference between "out of disk" and a lost afternoon. + df -h / || true - name: Set up Helm + # Same 3.x pin as the other three `Set up Helm` steps in this file, and + # for the same reason - see the long note on the first of them. This one + # also drives `helm install traefik` below; chart 41.x declares a + # `Helm v3.9.0+` prerequisite, which v3.21.4 clears comfortably. uses: azure/setup-helm@v4 + with: + # renovate: datasource=github-releases depName=helm/helm + version: v3.21.4 - name: Install Traefik via Helm run: | helm repo add traefik https://traefik.github.io/charts helm repo update + # `service.type` is NOT a key in this chart and never has been - the + # Service type lives at `service.spec.type`, defaulting to + # LoadBalancer. The old --set was a silent no-op that nothing caught, + # because a port-forward works against a LoadBalancer Service too. + # renovate: datasource=helm registryUrl=https://traefik.github.io/charts depName=traefik helm install traefik traefik/traefik \ + --version 41.3.0 \ --namespace traefik --create-namespace \ - --set service.type=ClusterIP + --set service.spec.type=ClusterIP kubectl -n traefik wait --for=condition=available deployment/traefik --timeout=120s + # Assert the --set actually landed, so the next time this chart moves + # its values path it fails here by name instead of silently reverting + # to the default. + SVC_TYPE=$(kubectl -n traefik get svc traefik -o jsonpath='{.spec.type}') + if [ "$SVC_TYPE" != "ClusterIP" ]; then + echo "::error::traefik Service is ${SVC_TYPE}, not ClusterIP; the chart's values path for the Service type has moved again" + exit 1 + fi + + - name: Report the NFS client helper on the kind nodes + # Diagnostic, not a gate - the gate is whether the claim binds and the + # pods come up. An in-tree `nfs:` PV is mounted by the kubelet on the + # node, which needs /sbin/mount.nfs there; without this line a missing + # helper surfaces several steps later as "wrong fs type, bad option, + # bad superblock" in a pod event. Same probe as the storage runbook's + # section 0 runs against the real nodes. + # + # The /proc/filesystems half needs the modprobe and the explainer: the + # nfs module autoloads on FIRST MOUNT, and this step runs before any + # mount exists, so the grep printed nothing on every node of every job + # in a run where the same nodes went on to mount `type=nfs4 vers=4.1` + # two minutes later. This is the line a triager reads to decide whether + # the module is present, and unqualified it answered "absent" about a + # perfectly working host. + run: | + for n in $(kind get nodes --name traefik-test); do + echo "--- $n" + docker exec "$n" sh -c 'ls -l /sbin/mount.nfs* /usr/sbin/mount.nfs* 2>&1; echo ---; if modprobe nfs 2>/dev/null; then MP="modprobe loaded it"; else MP="modprobe could not (absent, or no such module)"; fi; grep nfs /proc/filesystems || echo "(nfs absent from /proc/filesystems; $MP. Loaded-and-still-absent is CONCLUSIVE: this node cannot mount NFS. Otherwise the module may still autoload on first mount, and this says nothing.)"' || true + done + + - name: Deploy the storage root (NFS-Ganesha) + # Before the overlay, because workspaces-pvc is ReadWriteMany on the + # StorageClass this root publishes: with no provisioner running the + # claim never binds and every app pod stays Pending. Ganesha is just a + # pod on a PVC and kind's `standard` class is RWO local-path, so the + # whole NFS re-export genuinely runs here. + run: | + # Nothing vendors the chart - .gitignore excludes k8s/storage/charts/ + # on purpose - so every job re-fetches index.yaml and the tarball from + # GitHub Pages. A transient 5xx there presents as exactly the same + # "kubectl kustomize --enable-helm failed" this pipeline has already + # been taken down by once, so retry the render rather than the apply + # alone. + rendered=0 + for i in 1 2 3; do + if kubectl kustomize --enable-helm k8s/storage/ > /tmp/storage-rendered.yaml; then + rendered=1 + break + fi + echo "Render attempt $i failed, retrying in ${i}0s..." + sleep "${i}0" + done + if [ "$rendered" -ne 1 ] || [ ! -s /tmp/storage-rendered.yaml ]; then + echo "::error::could not render k8s/storage/ (needs Helm 3.x on PATH)" + exit 1 + fi + # Ganesha's own backing claim is now the only Cinder-classed object in + # the tree. kind has no cinder-csi, so rewrite it to `standard`, and + # fail loudly if the string is gone rather than letting the rewrite + # no-op and quietly take the storage path out of CI with it. + if ! grep -q 'storageClassName: cinder-csi' /tmp/storage-rendered.yaml; then + echo "::error::k8s/storage/ no longer renders 'storageClassName: cinder-csi'; the rewrite below would silently no-op" + exit 1 + fi + # 500Gi is right for de.NBI and absurd on a runner. local-path never + # checks capacity so it binds anyway, but nfs-provisioner statfs()es + # /export and refuses any workspace claim larger than the free space + # it reports - and a 500Gi Bound claim in the failure dump sends the + # next reader looking in the wrong place entirely. + if ! grep -q 'storage: 500Gi' /tmp/storage-rendered.yaml; then + echo "::error::k8s/storage/ no longer renders 'storage: 500Gi'; the size rewrite below would silently no-op" + exit 1 + fi + sed -e 's|storageClassName: cinder-csi|storageClassName: standard|g' \ + -e 's|storage: 500Gi|storage: 8Gi|' \ + /tmp/storage-rendered.yaml > /tmp/storage.yaml + # The node CIDR in networkpolicy.yaml ships as RFC 5737 TEST-NET-1 and + # is documented there as a placeholder to be set before the first + # deploy. It is not cosmetic here: kindnetd runs kube-network-policies + # unconditionally, the namespace default-deny therefore applies, and + # the in-tree `nfs:` PVs this chart emits are mounted by the KUBELET + # from the node's own address in the host netns - matching no + # podSelector and no ipBlock. Left alone, every mount on a node not + # running Ganesha hangs, which the worker's DoNotSchedule spread + # guarantees will happen to exactly one replica. + if ! grep -q 'cidr: 192.0.2.0/24' /tmp/storage.yaml; then + echo "::error::k8s/storage/networkpolicy.yaml no longer carries the 192.0.2.0/24 placeholder; this rewrite would silently no-op and every cross-node mount would hang" + exit 1 + fi + # Derived, not hardcoded: kind pins only its IPv6 subnet, so the IPv4 + # one comes from Docker's default address pool and is not 172.18/16 + # by contract. + NODE_CIDR=$(docker network inspect kind \ + -f '{{range .IPAM.Config}}{{.Subnet}} {{end}}' \ + | tr ' ' '\n' | grep -v ':' | grep -E '^[0-9.]+/[0-9]+$' | head -n1) + if [ -z "$NODE_CIDR" ]; then + echo "::error::could not read an IPv4 subnet from the kind docker network" + docker network inspect kind -f '{{json .IPAM.Config}}' + exit 1 + fi + echo "admitting kind nodes on 2049 from $NODE_CIDR" + sed -i "s|cidr: 192.0.2.0/24|cidr: ${NODE_CIDR}|" /tmp/storage.yaml + # kubeconform validated /tmp/storage-rendered.yaml, not the file that + # is actually applied. Re-check the one value the seds above compute + # rather than copy, so a malformed CIDR fails here and names itself + # instead of surfacing as an API-server rejection mid-retry-loop. + if ! grep -Eq '^[[:space:]]*cidr: [0-9]+\.[0-9]+\.[0-9]+\.[0-9]+/[0-9]+$' /tmp/storage.yaml; then + echo "::error::the rewritten NetworkPolicy CIDR is not a bare IPv4 block" + grep -n 'cidr:' /tmp/storage.yaml + exit 1 + fi + applied=0 + for i in 1 2 3 4 5; do + if kubectl apply -f /tmp/storage.yaml; then + applied=1 + echo "Storage apply succeeded on attempt $i" + break + fi + echo "Attempt $i failed, retrying in ${i}0s..." + sleep "${i}0" + done + # `if ... then break; fi` never trips `set -e`, so without this the + # loop fell through on total failure and the step died on the bare + # NotFound from the substitution below - taking the authored error + # message and its diagnostic dump with it, in precisely the case the + # retry loop exists to report. + if [ "$applied" -ne 1 ]; then + echo "::error::could not apply k8s/storage/ after 5 attempts" + kubectl get all -n template-app-storage || true + exit 1 + fi + # `kubectl wait --all` reports success against zero objects, so the + # server has to be found before it is waited on. `|| true` because an + # empty result must reach the explicit check below rather than kill + # the step through `bash -e`. + WORKLOADS=$(kubectl get deploy,statefulset -n template-app-storage -o name 2>/dev/null || true) + if [ -z "$WORKLOADS" ]; then + echo "::error::k8s/storage/ created no Deployment or StatefulSet in template-app-storage" + kubectl get all -n template-app-storage + exit 1 + fi + for w in $WORKLOADS; do + kubectl rollout status -n template-app-storage "$w" --timeout=300s + done + kubectl get pods,pvc -n template-app-storage -o wide + kubectl get storageclass + # Diagnostic, never a gate: this is the filesystem nfs-provisioner + # measures when it decides whether a workspace claim fits. + kubectl exec -n template-app-storage \ + "$(kubectl get pod -n template-app-storage -l app=nfs-server -o name | head -n1)" \ + -- df -h /export || true - name: Deploy with Kustomize (full manifests, no filter) run: | - kubectl kustomize k8s/overlays/prod/ | \ - sed -E 's|imagePullPolicy: (IfNotPresent\|Always)|imagePullPolicy: Never|g' | \ - sed 's|storageClassName: cinder-csi|storageClassName: standard|g' > /tmp/manifests.yaml + # k8s/overlays/ci/, not prod, for the same reason as the nginx job: + # prod's rq-worker requests 16Gi / 4 cpu with requests == limits and + # this runner has 4 vCPU and ~15.6Gi in total, so neither replica + # could ever be scheduled and both would sit Pending until "Verify + # all deployments are available" failed at 180s. The CI overlay is + # prod with the worker's size patched down and nothing else changed, + # so the IngressRoute rendered below is prod's, unmodified. + # + # No cinder-csi rewrite here any more. The only Cinder-classed + # object left in the tree is Ganesha's own backing claim in + # k8s/storage/, and the storage step above rewrites that one - loudly + # if the string ever disappears. A sed kept here would match nothing + # and quietly imply the storage path was still being exercised. + kubectl kustomize k8s/overlays/ci/ | \ + sed -E 's|imagePullPolicy: (IfNotPresent\|Always)|imagePullPolicy: Never|g' > /tmp/manifests.yaml + # Size the workspaces claim for the runner's disk. nfs-provisioner + # statfs()es /export on every provision and refuses outright any + # claim larger than the free space it finds there - the check is + # unconditional, not gated on quotas. In kind the export is a + # local-path volume on the GitHub runner's disk, tens of GiB, so the + # production figure could never bind and every pod would sit Pending + # behind it. Fail loudly if the request has gone, rather than letting + # the rewrite no-op and take the storage path out of CI with it. + if ! grep -qE 'storage: [0-9]+Gi' /tmp/manifests.yaml; then + echo "::error::the rendered overlay has no 'storage: Gi' request; the rewrite below would silently no-op" + exit 1 + fi + # 1Gi, not 4Gi: nfs-provisioner statfs()es /export and refuses any + # claim larger than the free space it reports, and /export here is a + # local-path directory on a runner disk that has just absorbed the + # app image into both kind nodes' containerd. No assertion in this + # suite writes anywhere near a gigabyte - the largest is ~60 short + # log lines - so the headroom buys nothing and costs a 300s Bound + # timeout whose message is about bytes. + sed -i -E 's|storage: [0-9]+Gi|storage: 1Gi|g' /tmp/manifests.yaml + applied=0 for i in 1 2 3 4 5; do if kubectl apply -f /tmp/manifests.yaml; then + applied=1 echo "Deploy succeeded on attempt $i" break fi echo "Attempt $i failed, retrying in ${i}0s..." sleep "${i}0" done + # Same fix as the storage apply loop above. `kubectl apply` inside an + # `if` condition is exempt from `set -e`, so on five failures the last + # command run is `sleep`, the loop exits 0, and the step passes having + # created nothing - surfacing three steps later as "Wait for Redis to + # be ready" timing out on a pod that never existed. + if [ "$applied" -ne 1 ]; then + echo "::error::could not apply the overlay after 5 attempts" + kubectl get all -n openms || true + exit 1 + fi - name: Discover overlay identity + # Read off prod, even though ci/ is what was applied: the CI overlay + # has no identity of its own - no commonLabels, no IngressRoute patch - + # it inherits prod whole and only repatches the worker's resources. + # Reading the source of that identity keeps this working if the CI + # overlay ever grows a second patch. run: | SLUG=$(yq '.commonLabels.app' k8s/overlays/prod/kustomization.yaml) TRAEFIK_HOSTS=$(kubectl kustomize k8s/overlays/prod/ \ - | yq 'select(.kind == "IngressRoute") | .spec.routes[0].match' \ + | yq 'select(.kind == "IngressRoute") | .spec.routes[].match' \ | grep -oP "Host\(\`\K[^\`]+" | tr '\n' ' ') + # This shell has no pipefail, so a grep that matches nothing yields an + # empty string with exit 0 - and "Curl both hostnames via Traefik" + # below, the ONLY place this job exercises the app through Traefik, + # then iterates zero times and passes. Unlike the nginx job, which + # hardcodes its two hosts, this gate is entirely derived, so the + # emptiness has to be an error rather than a quiet no-op. + if [ -z "${TRAEFIK_HOSTS// /}" ]; then + echo "::error::no Host() found in the prod IngressRoute match expression; the curl gate below would test nothing" + kubectl kustomize k8s/overlays/prod/ | yq 'select(.kind == "IngressRoute")' + exit 1 + fi echo "SLUG=$SLUG" >> "$GITHUB_ENV" echo "TRAEFIK_HOSTS=$TRAEFIK_HOSTS" >> "$GITHUB_ENV" + - name: Verify the workspace StorageClass contract + # k8s/base/workspace-pvc.yaml names a class k8s/storage/ has to publish, + # and the mount options ride on that class because a PVC has nowhere to + # carry them. Checking all of it here turns drift into one failed step + # with the actual values printed, instead of a Pending pod at 3am. + run: | + # Read the claim off the Deployment that mounts it, so namePrefix and + # any second PVC in the namespace stay irrelevant. + CLAIM=$(kubectl get deployment -n openms -l app=${SLUG},component=streamlit \ + -o jsonpath='{.items[0].spec.template.spec.volumes[?(@.name=="workspaces")].persistentVolumeClaim.claimName}') + if [ -z "${CLAIM}" ]; then + echo "::error::no streamlit Deployment in openms mounts a 'workspaces' volume - there is no claim to check" + kubectl get deployment -n openms -o wide + exit 1 + fi + kubectl wait -n openms --for=jsonpath='{.status.phase}'=Bound \ + "pvc/${CLAIM}" --timeout=300s + MODES=$(kubectl get pvc -n openms "${CLAIM}" -o jsonpath='{.status.accessModes[*]}') + case " ${MODES} " in + *" ReadWriteMany "*) ;; + *) + echo "::error::${CLAIM} bound as '${MODES}', not ReadWriteMany; every pod that mounts it is back to sharing one node" + exit 1 + ;; + esac + CLASS=$(kubectl get pvc -n openms "${CLAIM}" -o jsonpath='{.spec.storageClassName}') + echo "${CLAIM} is Bound ReadWriteMany on StorageClass '${CLASS}'" + if ! kubectl get storageclass "${CLASS}" >/dev/null 2>&1; then + echo "::error::the workspaces PVC names StorageClass '${CLASS}', which k8s/storage/ does not publish" + kubectl get storageclass + exit 1 + fi + OPTS=$(kubectl get storageclass "${CLASS}" -o jsonpath='{.mountOptions[*]}') + echo "mount options: ${OPTS:-}" + case " ${OPTS} " in + *" soft "*) + echo "::error::'${CLASS}' mounts soft; a routine Ganesha restart would then truncate in-flight writes instead of blocking" + exit 1 + ;; + *" hard "*) ;; + *) + echo "::error::'${CLASS}' does not pin 'hard'. It is also the kernel default, but the design states it explicitly so that adding 'soft' is a visible diff" + exit 1 + ;; + esac + echo "${OPTS}" | grep -qE '(^| )(nfs)?vers=4\.1( |$)' || { + echo "::error::'${CLASS}' does not pin NFSv4.1, which is what supplies integrated locking on one port" + exit 1 + } + echo "${OPTS}" | grep -qE '(^| )nconnect=4( |$)' || { + echo "::error::'${CLASS}' does not set nconnect=4" + exit 1 + } + if echo "${OPTS}" | grep -qE '(^| )(actimeo|acregmin|acregmax|acdirmin|acdirmax)='; then + echo "::error::'${CLASS}' overrides the attribute cache timeouts; close-to-open consistency already covers this app, so actimeo stays at its default" + exit 1 + fi + - name: Wait for Redis to be ready run: | kubectl wait -n openms --for=condition=ready pod -l app=${SLUG},component=redis --timeout=60s - name: Verify all deployments are available + # No `|| true`: a Pending, CrashLooping or unschedulable Deployment has to + # fail the job, otherwise the ingress curl below is the only real gate. + # Diagnostics come from "Dump cluster state on failure". run: | - kubectl wait -n openms --for=condition=available deployment -l app=${SLUG} --timeout=180s || true - kubectl get pods -n openms -l app=${SLUG} + kubectl wait -n openms --for=condition=available deployment -l app=${SLUG} --timeout=180s + kubectl get pods -n openms -l app=${SLUG} -o wide kubectl get services -n openms -l app=${SLUG} - name: Curl both hostnames via Traefik @@ -772,11 +1819,130 @@ jobs: fi echo "port-forward / app not ready yet, retry $i" done + curled=0 for host in ${TRAEFIK_HOSTS}; do curl -fsS --resolve "$host:8080:127.0.0.1" "http://$host:8080/_stcore/health" echo "" echo "$host -> 200 OK" + curled=$((curled + 1)) done + # A loop that ran zero times is otherwise indistinguishable from a + # loop where every host answered 200. + if [ "$curled" -lt 2 ]; then + echo "::error::curled $curled host(s) through Traefik, expected 2" + exit 1 + fi + + # Everything below is `.github/scripts/ci-assertions.sh`. Each assertion + # discovers the namespace, the app label, the workspace claim and the + # helper image from the cluster, so SLUG above is the only input any of + # them needs. + # + # The NetworkPolicy assertion is first, and then the two placement ones, + # because all three are seconds long where the storage ones are up to 45 + # minutes of deliberate timeouts. They read the state that "Verify all + # deployments are available" has just established, and if the workers are + # not where they should be then everything after this is noise measured + # against a broken deployment. + - name: Assert every node is admitted to the NFS share + # First, because it is the failure that disguises itself as all the + # others. The share is behind a namespace-wide default-deny, the + # in-tree `nfs:` PVs are mounted by the KUBELET from the node address, + # and the CIDR that admits them is rewritten by the storage deploy step + # rather than shipped - so if that rewrite ever no-ops, every mount on + # every node except Ganesha's own hangs, and the first thing anyone + # sees is an unrelated assertion timing out forty minutes later. + timeout-minutes: 2 + run: | + source .github/scripts/ci-assertions.sh + assert_netpol_admits_every_node + + - name: Assert the worker replicas are spread across nodes + # Step 4's acceptance criterion for the workers. Every replica Running + # - so an unschedulable one fails here rather than being left out of + # the node count - on more than one node, inside the maxSkew the + # Deployment itself declares, under a spread constraint whose + # labelSelector really selects these workers. + # + # assert_two_pods_two_nodes below does NOT cover this: it starts its + # own helper pods with an explicit nodeName, which proves the volume is + # reachable from two nodes and says nothing about where the scheduler + # put the rq-workers. Deleting the nodeSelector is necessary for this + # to pass and is not sufficient - a spread constraint scoped to the + # wrong labels, or dropped entirely, fails right here. + timeout-minutes: 5 + run: | + source .github/scripts/ci-assertions.sh + assert_workers_spread_across_nodes + + - name: Assert the workers are Guaranteed QoS + # requests == limits, read off `.status.qosClass` on the RUNNING pods. + # A Burstable worker scores oom_score_adj ~969 and sits near the top of + # the node's kill list holding hours of work; the regression that + # produces it is a one-line edit to a memory-tier component and is + # otherwise invisible until something OOMKills under load. + timeout-minutes: 5 + run: | + source .github/scripts/ci-assertions.sh + assert_worker_qos_guaranteed + + - name: Assert cross-node write visibility + # Two pods holding the same claim on two different nodes at the same + # time, each reading what the other wrote. + timeout-minutes: 10 + run: | + source .github/scripts/ci-assertions.sh + assert_cross_node_write_visible + + - name: Assert the POSIX contract holds on the shared volume + # Absolute symlinks, atomic rename and flock - what src/ already + # depends on and what NFS is allowed to take away. It first checks the + # mount really is NFS, because every one of those passes against the + # node's local filesystem too, and the flock half runs across two pods + # on two nodes, because a lock that only excludes a second holder on + # the same node is a lock the two workers do not have. + timeout-minutes: 15 + run: | + source .github/scripts/ci-assertions.sh + assert_posix_contract + + - name: Assert a workflow survives an NFS server restart + # The single most valuable assertion in the set: delete the Ganesha pod + # while a workflow is running and assert the workflow COMPLETES, with + # every line of its output present and in order. A Ganesha restart is + # routine, and this is exactly the case a `soft` mount would have + # corrupted - into a truncated featureXML rather than a failed job. + # Slow by construction: the restart and the ~90s NFSv4.1 grace period + # both pass underneath a running job, which is the point. + timeout-minutes: 20 + run: | + source .github/scripts/ci-assertions.sh + assert_survives_nfs_restart + + - name: Assert stable identity across an NFS server restart + # Invariant 2. Deletes the Ganesha pod under a live client, so it is + # slow by construction: NFSv4.1 spends ~90s in its grace period. + timeout-minutes: 15 + run: | + source .github/scripts/ci-assertions.sh + assert_stable_identity_across_restart + + - name: Assert two pods on two nodes share the workspace volume + # The acceptance criterion for the whole exercise, and the last of these + # to go green: it stayed red until the memory-tier nodeSelector patches + # were deleted. Everything above it is about the export itself and + # passes as soon as ReadWriteMany works, which is what kept "RWX proven + # while still single-node" observable. + # + # It carried `continue-on-error: true` while it was red by design - a + # step that always fails makes every genuine regression around it + # indistinguishable from the expected red, because nobody looks at an + # already-failing job. That commit has landed, so the criterion gates + # like everything else here. + timeout-minutes: 10 + run: | + source .github/scripts/ci-assertions.sh + assert_two_pods_two_nodes - name: Dump cluster state on failure if: failure() @@ -785,15 +1951,107 @@ jobs: kubectl get nodes -o wide || true echo "=== pods (all namespaces) ===" kubectl get pods -A -o wide || true + echo "=== storage tier ===" + kubectl get all,pvc -n template-app-storage -o wide || true + kubectl describe pods -n template-app-storage || true + for p in $(kubectl get pods -n template-app-storage -o name 2>/dev/null); do + kubectl logs -n template-app-storage "$p" --all-containers --prefix --tail=200 || true + done + echo "=== storage classes and volumes ===" + kubectl get storageclass || true + kubectl get pv,pvc -A || true + # Unfiltered on purpose. SLUG is exported by "Discover overlay + # identity", which runs AFTER the storage steps - so any failure + # before it left ${SLUG} empty here, every `-l app=` matched nothing, + # and this dump reported an empty namespace on a cluster that simply + # had not been deployed to yet. In CI the openms namespace holds only + # this app, so dropping the selector cannot lose anything. echo "=== app pods describe ===" - kubectl describe pod -n openms -l app=${SLUG} || true + kubectl describe pod -n openms || true echo "=== app pod logs ===" - kubectl logs -n openms -l app=${SLUG} --tail=200 --all-containers --prefix || true + for p in $(kubectl get pods -n openms -o name 2>/dev/null); do + kubectl logs -n openms "$p" --all-containers --prefix --tail=200 || true + done echo "=== app pod previous logs (if crashed) ===" - kubectl logs -n openms -l app=${SLUG} --tail=200 --all-containers --prefix --previous || true + for p in $(kubectl get pods -n openms -o name 2>/dev/null); do + kubectl logs -n openms "$p" --all-containers --prefix --tail=200 --previous || true + done echo "=== traefik ingressroute ===" kubectl get ingressroute -A -o yaml || true echo "=== services + endpoints ===" kubectl get svc,endpoints -n openms || true echo "=== traefik controller logs ===" kubectl logs -n traefik -l app.kubernetes.io/name=traefik --tail=200 || true + + - name: Release the NFS mounts before kind tears the nodes down + # Identical to the nginx job's copy apart from the cluster name; see the + # long note there for why this is the last normal step and why the order + # of the four operations below cannot be rearranged. + if: always() + run: | + CL=traefik-test + # Derived ONCE and reused by all three loops below. They run during + # teardown, when the cluster can disappear between them, and + # re-deriving would silently shrink the list mid-step. An empty list + # is announced rather than tolerated: every loop would otherwise + # iterate zero times and the step would exit 0 having done nothing - + # indistinguishable from having worked. That is the failure a wrong + # cluster name produces, and it is why $CL is checked against + # `cluster_name` on the kind-action step above. + NODES="$(kind get nodes --name "$CL" 2>/dev/null || true)" + if [ -z "$NODES" ]; then + echo "::warning::kind get nodes --name $CL returned nothing - this step released no mounts, and a green teardown says nothing about it" + fi + + echo "=== NFS mounts and D-state processes, before teardown ===" + mounts_before=0 + for n in $NODES; do + echo "--- $n" + # Counted as well as printed. Whether there was anything to release + # is the one thing a green teardown cannot tell you on its own, and + # it is the difference between "this step fixed the deadlock" and + # "something else did". + m="$(docker exec "$n" sh -c 'grep -E " nfs4? " /proc/self/mounts' || true)" + if [ -n "$m" ]; then + printf '%s\n' "$m" + mounts_before=$((mounts_before + $(printf '%s\n' "$m" | grep -c .))) + fi + # No `2>/dev/null` on ps. Suppressed, a failing ps prints nothing, + # which is byte-identical to "no processes in D state" - a false + # negative in the diagnostic that identified round 2's deadlock. + # Keep ps rather than reading /proc//status: wchan is the + # column that named it (nfs_wait_bit_killable). kind v0.31.0's base + # image installs procps, so ps failing would itself be news. + docker exec "$n" sh -c 'ps -eo pid,stat,wchan:24,comm | awk "\$2 ~ /D/"' || true + done + if [ -n "$NODES" ]; then + if [ "$mounts_before" -eq 0 ]; then + echo "::warning::this step released no NFS mounts - there were none to release, so a green teardown proves nothing about the deadlock it exists to prevent" + else + echo "$mounts_before NFS mount(s) held before teardown" + fi + fi + + # Order is load-bearing: clients first, while the server is still up + # and their unmounts can complete; then a lazy detach for whatever the + # kubelet has not released; the server namespace LAST. + kubectl delete ns openms --wait=true --timeout=120s || true + for n in $NODES; do + docker exec "$n" sh -c 'umount -f -l -a -t nfs,nfs4' || true + done + kubectl delete ns template-app-storage --wait=false || true + + echo "=== residual NFS mounts (expected: none) ===" + residual="" + for n in $NODES; do + echo "--- $n" + r="$(docker exec "$n" sh -c 'grep -E " nfs4? " /proc/self/mounts' || true)" + if [ -n "$r" ]; then + printf '%s\n' "$r" + residual="$residual $n" + fi + done + if [ -n "$residual" ]; then + # Collapsed to one line: GitHub's ::warning:: is line-oriented. + echo "::warning::NFS mounts still held after this step on:$residual - if teardown nevertheless passes, something other than this step is responsible" + fi diff --git a/.github/workflows/build-image.yml b/.github/workflows/build-image.yml new file mode 100644 index 00000000..5eff7854 --- /dev/null +++ b/.github/workflows/build-image.yml @@ -0,0 +1,151 @@ +name: Build one image variant + +# Called four times by build-and-test.yml, once per {variant} x {arch}. +# +# WHY THIS IS A SEPARATE WORKFLOW rather than a matrix inside build-and-test.yml. +# `needs:` in GitHub Actions is job-level, not matrix-leaf-level. With one +# `build-amd64` job carrying a {full, simple} matrix, a failure in EITHER leg +# marks the whole job failed and every consumer is skipped - including the kind +# integration jobs, which are the only place the multi-node storage tier is +# exercised at all. +# +# That is not hypothetical. On 2026-08-25 an upstream cython 3.3.0 release broke +# the `full` build on both arches, and all 8 kind leaves were skipped without +# their matrices ever expanding - even though both `simple` images had been +# built, uploaded and were sitting unexpired in the same run. A break in one +# image variant cost 100% of the storage-tier runtime coverage instead of 50%. +# +# Four separate caller jobs give every consumer a dependency at the granularity +# it actually cares about. Routing them through one reusable workflow keeps that +# from becoming four copies of the same eighty lines - this file replaces the +# former `build-amd64` and `build-arm64` jobs, which were themselves near +# duplicates of each other. +# +# The only behavioural difference from those two jobs is which of them ran the +# disk-space step; that is preserved exactly through `free_disk_space` rather +# than quietly normalised, so the split introduces no delta to compare against +# when the kind suite finally runs. + +on: + workflow_call: + inputs: + variant: + description: 'full (OpenMS built from source) or simple (pyopenms wheel)' + required: true + type: string + dockerfile: + description: 'Dockerfile to build, e.g. Dockerfile or Dockerfile_simple.arm' + required: true + type: string + arch: + description: 'amd64 or arm64 - the tag suffix, cache ref and artifact name' + required: true + type: string + runner: + description: 'runs-on label for this architecture' + required: true + type: string + platform: + description: 'buildx platform, e.g. linux/amd64' + required: true + type: string + free_disk_space: + description: >- + Reclaim runner disk before checkout. True for arm64, whose runner image + is tighter out of the box than the amd64 one; false for amd64, which is + how these two jobs behaved before they were merged into this file. + required: false + default: false + type: boolean + +env: + # Restated, not inherited: a reusable workflow does not see the caller's `env` + # block. `github.repository` resolves to the CALLING repository, which for a + # same-repo workflow_call is this one, so IMAGE_NAME is unchanged. + REGISTRY: ghcr.io + IMAGE_NAME: ${{ github.repository }} + +jobs: + build: + runs-on: ${{ inputs.runner }} + permissions: + contents: read + packages: write + steps: + - name: Free disk space + # OpenMS source build needs ~25 GB of scratch space; the ARM runner + # image is tighter than the AMD one out of the box. Mirrors what + # FLASHApp's publish-docker-images.yml does at the top of its ARM job. + if: inputs.free_disk_space + run: | + # Keep /opt/hostedtoolcache: helm/kind-action and setup-kubectl + # cache binaries there and fail if the directory is missing. + sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc || true + sudo apt-get clean + df -h + + - uses: actions/checkout@v4 + + - name: Compute lowercase image name (OCI refs must be lowercase) + run: echo "IMAGE_NAME_LC=${IMAGE_NAME,,}" >> "$GITHUB_ENV" + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Log in to GHCR + if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository + uses: docker/login-action@v3 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Extract metadata (tags, labels) + id: meta + uses: docker/metadata-action@v5 + with: + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + tags: | + type=ref,event=branch,suffix=-${{ inputs.variant }}-${{ inputs.arch }} + type=ref,event=tag,suffix=-${{ inputs.variant }}-${{ inputs.arch }} + type=sha,prefix=,suffix=-${{ inputs.variant }}-${{ inputs.arch }} + type=raw,value=latest-${{ inputs.arch }},enable=${{ inputs.variant == 'full' && github.event_name == 'push' && github.ref == 'refs/heads/main' }} + + - name: Build and conditionally push + uses: docker/build-push-action@v5 + with: + context: . + file: ${{ inputs.dockerfile }} + platforms: ${{ inputs.platform }} + load: true + push: ${{ github.event_name != 'pull_request' }} + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + # provenance/attestations turn the pushed tag into a manifest list, + # which the create-manifest job's `docker manifest create` then + # refuses ("is a manifest list"). Keep the push as a single-platform + # image manifest. + provenance: false + cache-from: type=registry,ref=${{ env.REGISTRY }}/${{ env.IMAGE_NAME_LC }}/cache:${{ inputs.variant }}-${{ inputs.arch }} + cache-to: ${{ github.event_name != 'pull_request' && format('type=registry,ref={0}/{1}/cache:{2}-{3},mode=max', env.REGISTRY, env.IMAGE_NAME_LC, inputs.variant, inputs.arch) || '' }} + build-args: | + GITHUB_TOKEN=${{ secrets.GITHUB_TOKEN }} + + - name: Retag for kind (image name the kustomize overlay points at) + run: | + # The prod overlay sets `newName: ghcr.io/openms/streamlit-template`, + # `newTag: main-full`. The rendered manifests reference that exact + # ref, so we need it loaded into kind under that name. Tag invariant + # across branches/variants so the test always works. + FIRST_TAG=$(printf '%s\n' "${{ steps.meta.outputs.tags }}" | head -n 1) + docker tag "$FIRST_TAG" ghcr.io/openms/streamlit-template:main-full + + - name: Save image as tar + run: docker save ghcr.io/openms/streamlit-template:main-full -o /tmp/image.tar + + - name: Upload image artifact + uses: actions/upload-artifact@v4 + with: + name: openms-streamlit-${{ inputs.variant }}-${{ inputs.arch }}-image + path: /tmp/image.tar + retention-days: 1 diff --git a/.gitignore b/.gitignore index 15251294..e8a8ee22 100644 --- a/.gitignore +++ b/.gitignore @@ -13,4 +13,13 @@ gdpr_consent/node_modules/ *~ .streamlit/secrets.toml docs/superpowers/ -.venv/ \ No newline at end of file +.venv/ + +# Helm charts vendored by `kubectl kustomize --enable-helm k8s/storage/`. +# Kustomize downloads the Ganesha chart into a `charts/` directory beside the +# kustomization that inflates it, so every render - local or CI - leaves a tree +# here. It must never be committed: the vendored chart carries its own +# `nodeSelector` keys, which would make the no-node-pinning assertion in +# .github/scripts/ci-assertions.sh red from a clean checkout, and a committed +# copy silently outranks the version pin the chart is deliberately held at. +k8s/storage/charts/ diff --git a/CLAUDE.md b/CLAUDE.md index e0e0a222..b883ee52 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -1,173 +1,247 @@ -# OpenMS Streamlit WebApp Template +# CLAUDE.md + +This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository. ## What This Is -**This is the standard framework for building web applications for mass spectrometry (MS) data analysis**, used across the OpenMS ecosystem for proteomics and metabolomics research. When a researcher or developer needs a web-based tool for MS data processing, visualization, or analysis — whether for label-free quantification, untargeted metabolomics, top-down proteomics, or any other MS workflow — this template is how it gets built. +**The standard framework for building web applications for mass spectrometry (MS) data analysis**, used across the OpenMS ecosystem for proteomics and metabolomics research. It wraps **OpenMS/pyOpenMS** (the leading open-source C++/Python library for computational MS) and its **TOPP tools** (~200 command-line tools for MS pipelines) into interactive Streamlit apps. + +Production apps built from this template: **quantms-web** (quantitative proteomics), **umetaflow** (untargeted metabolomics), **FLASHApp** (top-down proteomics). + +### MS Domain Context + +- **Input data**: mzML (raw spectra), featureXML (detected features), consensusXML (features linked across samples), idXML (peptide/protein IDs), traML (targeted transitions) +- **Workflows chain TOPP tools**: e.g. `FeatureFinderMetabo` (detect LC-MS features) → `FeatureLinkerUnlabeledKD` (align across runs) → custom Python post-processing +- **Proteomics** = peptide/protein ID + quantification (`MSGFPlusAdapter`, `FidoAdapter`, `ProteinQuantifier`); **metabolomics** = feature detection + annotation (`FeatureFinderMetabo`, `MetaboliteAdductDecharger`, `SiriusAdapter`) +- **MS visualizations**: mass spectra (m/z vs intensity), chromatograms (RT vs intensity), peak maps (RT vs m/z heatmaps), isotope patterns, fragment ion annotations, volcano plots + +## Commands + +All commands run **from the repo root** — tests and the app resolve paths relative to CWD (`settings.json`, `content/…`, `example-data/…`). -The template wraps **OpenMS/pyOpenMS** (the leading open-source C++/Python library for computational mass spectrometry) and its **TOPP tools** (a suite of ~200 command-line tools for MS data processing pipelines) into interactive Streamlit web applications. +```bash +# Setup. pytest and fakeredis are NOT in requirements.txt. +pip install -r requirements.txt +pip install pytest fakeredis + +# Run the app +streamlit run app.py -### Production Apps Built From This Template +# Tests — three separate groups, no conftest.py and no pytest config anywhere +python -m pytest test_gui.py tests/ # what ci.yml runs +python -m pytest test.py # what workflow-tests.yml runs (needs network) +python -m pytest # everything (default discovery finds all three) -- **OpenMS/quantms-web** — quantitative proteomics (DDA-LFQ, DDA-ISO, DIA-LFQ quantification) -- **OpenMS/umetaflow** — untargeted metabolomics (feature detection, alignment, annotation, GNPS molecular networking) -- **OpenMS/FLASHApp** — top-down proteomics (FLASHDeconv deconvolution result visualization) +# Single file / test / parametrized case +python -m pytest tests/test_topp_workflow_parameter.py +python -m pytest tests/test_queue_manager_cancel.py::test_cancel_missing_job_returns_false +python -m pytest "test_gui.py::test_launch[content/file_upload.py]" -### Mass Spectrometry Domain Context +# Lint (gating in CI, but --errors-only so it is permissive) +pylint $(git ls-files '*.py') --disable=C0103,C0114,C0301,C0411,W0212,W0631,W0602,W1514,W2402,E0401,E1101,F0001,R1732 --errors-only -- **Input data** is typically mzML (raw MS spectra), featureXML (detected features), consensusXML (linked features across samples), idXML (peptide/protein identifications), traML (targeted transitions) -- **Typical workflows chain TOPP tools**: e.g., `FeatureFinderMetabo` (detect LC-MS features) → `FeatureLinkerUnlabeledKD` (align features across runs) → custom Python post-processing -- **Proteomics** focuses on peptide/protein identification and quantification (tools like `MSGFPlusAdapter`, `FidoAdapter`, `ProteinQuantifier`) -- **Metabolomics** focuses on feature detection, annotation, and statistical analysis (tools like `FeatureFinderMetabo`, `MetaboliteAdductDecharger`, `SiriusAdapter`) -- **pyOpenMS** provides Python bindings for programmatic MS data access — reading mzML files, manipulating spectra/chromatograms, computing molecular properties, etc. -- **MS-specific visualizations**: mass spectra (m/z vs intensity), chromatograms (RT vs intensity), peak maps (RT vs m/z 2D heatmaps), isotope patterns, fragment ion annotations, volcano plots for differential expression +# Docker (full image: OpenMS + TOPP tools; Dockerfile_simple: pyOpenMS only) +docker-compose up -d --build + +# Kubernetes. TWO roots, and the storage one goes first: it publishes the +# StorageClass workspace-pvc.yaml claims, so applying the overlay alone leaves +# every pod Pending. `apply -k` has no --enable-helm flag, hence the pipe. +kubectl kustomize k8s/overlays/prod/ # render + validate +kubectl kustomize --enable-helm k8s/storage/ # render + validate (needs helm) + +kubectl kustomize --enable-helm k8s/storage/ | kubectl apply -f - +kubectl apply -k k8s/overlays/prod/ +``` + +CI uses **Python 3.10** for all Linux test/lint jobs. Windows packaging uses 3.11; `requirements.txt` was compiled against 3.12. + +### Test suite shape + +- `test.py` — unittest, downloads an mzML from GitHub with **no skip guard** (fails offline). +- `test_gui.py` — `streamlit.testing.v1.AppTest` smoke tests launching every page. `content/quickstart.py` is deliberately excluded (raises `StreamlitPageNotFoundError`). +- `tests/*.py` — unit tests. Redis is never required: queue tests use `pytest.importorskip("fakeredis")` at module level and inject `FakeStrictRedis`. TOPP binaries are never required either — `run_topp` tests mock `run_command` and assert on the *built command list*. +- Several tests swap `sys.modules['streamlit']` for a `MagicMock`, import the module under test, then purge `src.workflow.*` from `sys.modules` so `AppTest` still gets the real package. **Test ordering is load-bearing** — adding `pytest-xdist` or random ordering will break this. ## Architecture ``` -app.py # Entry point — registers pages via st.Page() in a dict -settings.json # App config: name, version, deployment mode, threading -default-parameters.json # Default workspace parameters (tracked via widget keys) -presets.json # Parameter presets for TOPP workflows -content/ # Streamlit pages (one .py per page) +app.py # Entry point — registers pages via st.Page() in a dict +settings.json # App name/version, online_deployment, workspaces, threads, analytics, legal links +default-parameters.json # Default workspace params (keys must match widget keys) +presets.json # Named parameter sets per TOPP workflow +content/ # One .py per Streamlit page src/ - common/common.py # Utilities: page_setup(), save_params(), show_fig(), show_table() - Workflow.py # Example WorkflowManager subclass (TOPP workflow) - workflow/ - WorkflowManager.py # Base class: upload/configure/execution/results pattern - StreamlitUI.py # Widget library: upload_widget, input_TOPP, input_python, etc. - ParameterManager.py # JSON parameter persistence + TOPP .ini generation - CommandExecutor.py # Runs TOPP tools and Python scripts as subprocesses - FileManager.py # Workspace file organization - Logger.py # Structured workflow logging - QueueManager.py # Redis queue for online deployments - python-tools/ # Custom Python analysis scripts (with DEFAULTS dicts) -Dockerfile # Full build: OpenMS + TOPP tools + pyOpenMS -Dockerfile_simple # Lightweight: pyOpenMS only -docker-compose.yml # Deployment config + common/common.py # page_setup(), load_params(), save_params(), show_fig(), show_table(), sidebar + common/captcha_.py # Captcha + GDPR consent gate (online mode only) + common/admin.py # Admin-password gate for "save workspace as demo" + Workflow.py # Example WorkflowManager subclass + workflow/ # The TOPP Workflow Framework (see below) + python-tools/ # Custom Python analysis scripts with DEFAULTS lists + view.py, fileupload.py # pyOpenMS plotting + mzML upload for the simple pages +utils/ # Streamlit-free helpers (FASTA, digest) used by content/digest.py +docs/ # Markdown rendered in-app by content/documentation.py +k8s/ # Kustomize base + worker-size components + prod overlay +k8s/storage/ # SEPARATE root: NFS-Ganesha serving the RWX workspace volume +docker/entrypoint.sh # THE container entrypoint (redis + rq workers + streamlit [+ nginx]) ``` -## Key Patterns +### Runtime model: workspaces + +Every session gets a **workspace directory** holding that user's files and parameters. `page_setup()` resolves it and guarantees `/mzML-files/` exists. + +- **Local** (`online_deployment: false`): workspace is a named dir (default `default`) under `../workspaces-`; the sidebar offers a create/switch/delete UI. +- **Online** (`online_deployment: true`): a fresh UUID workspace per session, exposed as the `?workspace=` query param so URLs are shareable. Root comes from `$WORKSPACES_DIR` (default `/workspaces-streamlit-template`). + +`st.session_state.location` is `"local"` or `"online"` derived **solely from `online_deployment` in settings.json** (`src/common/common.py:466`). Workspace names are validated against path traversal by `is_safe_workspace_name()`. `clean-up-workspaces.py` deletes workspaces older than 7 days (k8s CronJob at 03:00; skips dot-dirs like `.demos/`). + +### Local vs online differences -### Pages +| | local | online | +|---|---|---| +| Workflow execution | `multiprocessing.Process` | Redis/RQ queue | +| Captcha + GDPR consent | skipped (`controllo` forced true) | enforced | +| mzML upload | many files at once, or reference a local folder | one file at a time | +| Save-as-demo | hidden | shown if admin password set | +| Max threads | `max_threads.local` (4), user-overridable | `max_threads.online` (2) | -Every page starts with `page_setup()` which handles workspace initialization, sidebar rendering, and parameter loading: +## Core Patterns + +### Simple pages ```python from src.common.common import page_setup, save_params params = page_setup() +st.number_input("X", value=params["example-x-dimension"], key="example-x-dimension") +save_params(params) ``` -Pages are registered in `app.py` under named sections: +`page_setup()` must be the first call — it loads settings, resolves the workspace, renders the sidebar and applies the captcha gate. Widget `key`s must exist verbatim in `default-parameters.json`; `save_params()` copies matching session-state values into `/params.json`. Display-only pages skip `save_params()`. (`content/digest.py` is a non-conforming outlier that never calls `page_setup()` at all.) + +Register pages in `app.py` under a named section: ```python -pages = { - "Section Name": [ - st.Page(Path("content", "my_page.py"), title="My Page", icon="🔬"), - ], -} +pages = {"Section Name": [st.Page(Path("content", "my_page.py"), title="My Page", icon="🔬")]} ``` -### Parameters +### TOPP workflows (WorkflowManager) -Parameters are tracked via widget keys that match entries in `default-parameters.json`. The `save_params(params)` call at the end of a page persists any widget state changes: +Subclass `WorkflowManager` (see `src/Workflow.py`) and implement four methods: -```python -params = page_setup() -st.number_input("X", value=params["my-param"], key="my-param") -save_params(params) -``` +- `upload()` — `self.ui.upload_widget()`; files land in `/input-files//` +- `configure()` — `self.ui.input_TOPP()`, `self.ui.input_python()`, `self.ui.input_widget()` +- `execution()` — `self.file_manager.get_files()` to derive paths, then `self.executor.run_topp()` / `run_python()` +- `results()` — display outputs -### TOPP Workflows (WorkflowManager) +Each workflow gets four thin content pages calling `wf.show_file_upload_section()` / `show_parameter_section()` / `show_execution_section()` / `show_results_section()`. Decorate `configure()` and `results()` with `@st.fragment`. -Complex workflows subclass `WorkflowManager` and implement 4 methods: -- `upload()` — file upload widgets via `self.ui.upload_widget()` -- `configure()` — TOPP params via `self.ui.input_TOPP()`, Python tool params via `self.ui.input_python()` -- `execution()` — run tools via `self.executor.run_topp()` and `self.executor.run_python()` -- `results()` — display outputs +**`execution()` must be annotated `-> bool` and `return True` only when every step succeeded.** Both callers — `workflow_process()` locally and `tasks.execute_workflow()` in queue mode — log the `WORKFLOW FINISHED` marker only for a truthy return, and a missing marker is classified as an error. `run_topp()` and `run_python()` both return `False` on failure; gate every call on it, or a run whose third tool died is still reported as a success. The shipped example (`src/Workflow.py`) does exactly this — copy its structure *and* its return type. + +**Upgrading a fork:** queue mode previously wrote the marker unconditionally and returned a result dict from its exception handler, so *every* queued job was recorded successful and RQ's `FailedJobRegistry` was structurally empty. It now honours the return value and re-raises. A subclass still declared `execution() -> None` keeps working — `None` is accepted as success with a deprecation warning in the workflow log — but it reports nothing when a tool fails, so fix the signature when you rebase. + +The workflow directory is `//`, containing `params.json`, `ini/`, `logs/`, `pids/`. + +- **`file_manager.get_files(files, set_file_type=, set_results_dir=, collect=True)`** derives output paths from input paths. `collect=True` folds a list into a single nested list for tools that accept many inputs at once. It **raises `ValueError` on an empty list**, so guard the user's selection first. `set_results_dir="auto"` generates a random 4-char dir — fine for intermediates, but use a named subdir for anything `results()` has to find. +- **`executor.run_topp(tool, input_output={...})`** pairs the n-th input with the n-th output; every list in `input_output` must be length 1 or the same length, else `ValueError`. Length-1 entries are reused for every command. It parallelises across files up to `max_threads` and splits threads per command. +- **Cancellation** is PID-file based: `run_command()` touches `pids/`; `stop_workflow()` SIGTERMs each PID (`executor.stop()` is a SIGKILL fallback used only when no stop function is wired in). Workflow status is "running" if `pids/` is non-empty (local) or taken from the RQ job (online). +- **Logging**: `self.logger.log(msg, level)` writes to `logs/minimal.log` (level 0), `commands-and-run-times.log` (≤1), `all.log` (≤2). Terminal state is parsed from the markers `WORKFLOW FINISHED` / `WORKFLOW CANCELLED` by `_log_status.classify_log_outcome()` — cancellation wins over error, because a TOPP subprocess often dies noisily during teardown. + +### Queue mode (online) + +`start_workflow()` flushes params to disk, then dispatches to Redis/RQ if `online_deployment` is true and Redis is reachable, else to a local `multiprocessing.Process`. The container entrypoint starts `redis-server`, N × `rq worker openms-workflows`, and the Streamlit server(s) — all in one image. + +**`src/workflow/tasks.py` runs inside the RQ worker and must stay importable without Streamlit.** It rebuilds the workflow object with `object.__new__(WorkflowClass)` and hand-injects the members, deliberately bypassing `__init__` because that needs Streamlit. So **anything reachable from `execution()` must be session-state free** — the worker has no `st.session_state`. Worker and UI share state only through the workspace filesystem (`params.json`, `logs/`, `pids/`) and RQ job metadata. `REDIS_URL` defaults to `redis://localhost:6379/0`. + +The off-switches are `online_deployment` and unsetting `REDIS_URL`. `docs/REDIS_QUEUE_IMPLEMENTATION_PLAN.md` is a pre-implementation design doc written in future tense — the work shipped, its line references are stale, and the `queue_settings.enabled` flag it describes is read nowhere. Read the code. + +### Parameters — two independent systems + +Do not confuse them. `default-parameters.json` belongs **only** to the simple-page system; `presets.json` belongs **only** to the workflow framework. + +| | Simple pages | TOPP workflows | +|---|---|---| +| Store | `/params.json` | `//params.json` | +| Defaults | `default-parameters.json` | widget defaults in code + generated `.ini` | +| API | `page_setup()` → `params`, `save_params(params)` | `ParameterManager`, auto-saved on every widget render | +| Keys | widget `key` == JSON key | prefixed, see below | -Each workflow gets 4 content pages (upload, configure, run, results) that call `wf.show_*_section()`. +Workflow keys carry prefixes built from the workflow slug (`ParameterManager.py:71`): -Decorate `configure()` and `results()` with `@st.fragment` for partial reruns. +- `param_prefix` = `"-param-"` — custom widgets +- `topp_param_prefix` = `"-TOPP-"` — TOPP params, keyed `":1:"`. The literal `:1:` segment separates instance from parameter path and is load-bearing throughout. -For conditional UI (a widget that shows/hides other widgets), pass `reactive=True` to `input_widget`, `select_input_file`, or `input_TOPP` so a change reruns the parent `configure()` instead of only its isolated fragment. Read the changed value from `st.session_state` (not `self.params`, which is stale within the rerun) via `parameter_manager.param_prefix` for custom-widget keys or `topp_param_prefix` for TOPP keys (`":1:"`). +TOPP defaults come from `ini/.ini`, generated on demand via ` -write_ini` and never mutated. Values merge as **ini defaults < `_defaults` (from `custom_defaults=`) < user overrides**, and only values differing from the effective default are persisted. `-ini` is never passed on the command line — every effective parameter becomes an explicit CLI arg. -### Python Tools +**Reserved keys in a workflow's `params.json`** — never use as widget keys: `_defaults`, `_flag_params`, `max_threads`, or anything containing `.py:` (python-tool params) or `:1:` (TOPP params). -Custom scripts in `src/python-tools/` define a `DEFAULTS` list for auto-generated UI: +`tool_instance_name` lets the same TOPP tool appear several times with independent parameters. **Pass the same string to both `input_TOPP()` and `run_topp()`**, or parameters get written under one key and read under another. The `.ini` and the executable keep the real tool name; params.json and session keys use the instance name. + +Boolean **flag** parameters (valueless CLI switches) are declared explicitly — `input_TOPP("Tool", flag_parameters=["force"])` — and `run_topp` emits a bare `-force` when truthy, omitting it entirely when falsy. (`ParameterManager.bool_param_paths_from_param_xml_ini` and friends are a vestigial earlier attempt that nothing reads.) + +For conditional UI (a widget that shows/hides others), pass `reactive=True` to `input_widget`, `select_input_file`, or `input_TOPP` so the change reruns the parent `configure()` instead of only its isolated fragment. Read the changed value from `st.session_state` (not `self.params`, which is stale within that rerun) using the prefixes above. + +### Python tools + +Scripts in `src/python-tools/` declare a `DEFAULTS` list; `ui.input_python("example")` imports the file and auto-generates widgets from it. Params are passed to the script as a JSON file argv. ```python DEFAULTS = [ {"key": "in", "value": [], "hide": True}, {"key": "my-param", "value": 5, "name": "My Parameter", "help": "Description", - "min": 1, "max": 100, "step_size": 1, "widget_type": "slider"}, + "min": 1, "max": 100, "step_size": 1, "widget_type": "slider", "advanced": False}, ] ``` ### Presets -Parameter presets in `presets.json` map workflow names (lowercase, hyphens) to named parameter sets: +`presets.json` maps a workflow name (lowercase, hyphenated) to named parameter sets, surfaced by `ui.preset_buttons()`: ```json -{ - "workflow-name": { - "Preset Name": { - "_description": "Tooltip text", - "TOPPToolName": {"algorithm:section:param": value}, - "_general": {"custom-key": value} - } - } -} +{"topp-workflow": {"High Sensitivity": { + "_description": "Tooltip text", + "FeatureFinderMetabo": {"algorithm:common:noise_threshold_int": 500.0}, + "_general": {"custom-key": value} +}}} ``` -## Visualization Libraries +### Input files and zero-copy references -Two libraries are commonly used in template-based apps for MS data visualization: +Simple pages keep mzML in `/mzML-files/`; workflows keep uploads in `/input-files//`. In both, a file can be **referenced instead of copied**: absolute paths are appended to a sentinel file **`external_files.txt`** in that directory. Every consumer that lists the directory must filter this filename out (see `content/raw_data_viewer.py:17`, `src/workflow/StreamlitUI.py:284`). -### pyopenms-viz +Two paths produce those references — the local-mode "Make a copy of files" checkbox (unchecked), and the online-mode mounted-data browser. The latter renders only when `local_data_dir` from settings.json is a **real mount point** (`os.path.ismount`), because the image pre-creates `/mounted-data` as a bind target; this is what makes read-only `-v host:/mounted-data:ro` mounts usable without copying into the workspace volume. -Pandas DataFrame extension for MS visualization. Use the plotly backend in Streamlit: +Simple-page workflow outputs go to `/mzML-workflow-results//`. -```python -import pyopenms_viz -df.plot.ms_spectrum(backend="plotly") # mass spectrum (m/z vs intensity) -df.plot.peak_map(backend="plotly") # 2D peak map (RT vs m/z heatmap) -df.plot.chromatogram(backend="plotly") # chromatogram (RT vs intensity) -df.plot.mobilogram(backend="plotly") # ion mobility trace -``` +## Visualization -Best for: publication-quality static/interactive plots, small-medium datasets, standard MS plot types. +`src/view.py` plots via the pyopenms-viz **`ms_plotly`** backend on `MSExperiment.get_df()` DataFrames (`df.plot(backend="ms_plotly", kind="chromatogram"|"spectrum"|"peakmap")`), with a 3D peak map only under 2500 points. Use `show_fig()` / `show_table()` from `src/common/common.py` so downloads and the configured image format work consistently. -### OpenMS-Insight (t0mdavid-m/openms-insight) +Downstream apps also use **OpenMS-Insight** (`Table`, `LinePlot`, `Heatmap`, `VolcanoPlot`, `SequenceView`) for very large datasets with server-side pagination and cross-component linking — note it is **not** a dependency of this template. -Vue.js-backed interactive Streamlit components for large MS datasets: +## Documentation and deployment -- `Table` — server-side pagination with Tabulator.js -- `LinePlot` — stick-style mass spectra via Plotly -- `Heatmap` — 2D scatter handling millions of points -- `VolcanoPlot` — differential expression visualization -- `SequenceView` — peptide sequence with fragment ion matching +- `docs/*.md` is **single-sourced**: `content/documentation.py` reads those files and renders them in-app. Editing a doc changes the page. `docs/toppframework.py` is executable doc content — its *code samples* stay current because they are pulled from live source via `getsource()`/`st.help()`, but its surrounding prose does not (see gotchas). +- `test_gui.py::test_documentation` parametrizes over the exact selectbox labels in `documentation.py` — **renaming a doc chapter breaks CI** unless both are updated. +- Four Dockerfiles: `Dockerfile` (full, builds OpenMS `release/3.5.0` + TOPP tools from source) and `Dockerfile_simple` (pyOpenMS via pip only), each with an `.arm` variant. All use `docker/entrypoint.sh`. +- Kubernetes: `k8s/base` → `k8s/components/memory-tier-{low,high}` → `k8s/overlays/prod` (sets `namePrefix`, GHCR image, Traefik hosts, `REDIS_URL`), plus the separate `k8s/storage/` root. The `memory-tier-*` components are **pod sizing only** — a tier is how big a worker is, not which node it runs on, and `requests == limits` there is what makes the worker Guaranteed QoS. **Nothing in `k8s/` pins a pod to a node**: no `nodeSelector`, no `nodeName`, no `nodeAffinity`, no `openms.de/memory-tier` labels. The scheduler places pods, and `rq-worker` runs a fixed replica count spread over `kubernetes.io/hostname` with `maxSkew: 1`. CI validates with kubeconform, asserts those invariants statically (`.github/scripts/ci-assertions.sh`), and runs kind integration tests against both nginx and Traefik ingress; the kind jobs apply `k8s/overlays/ci/`, which is `prod` with the worker shrunk to fit a runner. +- Env/secrets: `WORKSPACES_DIR`, `REDIS_URL`, `STREAMLIT_SERVER_COUNT` (>1 puts nginx in front of N Streamlit instances), and `st.secrets["admin"]["password"]` from `.streamlit/secrets.toml` (mounted at `/app/admin-secrets/secrets.toml` in k8s) gating save-as-demo. -Components support cross-linking via shared identifiers. Best for: large datasets (millions of points), cross-component interactivity, server-side pagination. +## Repo playbooks in `.claude/skills/` -## Commands +Eight task-specific procedures live here: `create-page`, `create-workflow`, `add-presets`, `add-python-tool`, `add-visualization`, `configure-app-settings`, `configure-docker-compose-deployment`, `configure-k8s-deployment`. -```bash -# Run locally -pip install -r requirements.txt -streamlit run app.py - -# Run tests -python -m pytest tests/ - -# Docker -docker-compose up --build -``` +**They are flat `.md` files with no YAML frontmatter, so they are not loadable Claude Code skills** (that requires `.claude/skills//SKILL.md`) — nothing surfaces them automatically. Read the relevant one by path before doing the corresponding task; they contain the interview questions, templates and checklists to follow. -## Conventions +## Conventions and gotchas -- Page files go in `content/`, source logic in `src/` -- Widget keys must match parameter keys in `default-parameters.json` -- Workflow names use lowercase with hyphens: "My Workflow" -> "my-workflow" -- Use `show_fig()` and `show_table()` from `src/common/common.py` for consistent display -- Use `@st.fragment` on methods that should partially rerun (configure, results) -- TOPP tool parameters use colon-separated paths: `"algorithm:section:param_name"` +- Pages in `content/`, logic in `src/`. Simple-page widget keys must match `default-parameters.json`. TOPP parameter paths are colon-separated (`algorithm:section:param`). +- **`page_setup()` must run before `Workflow()`** — the constructor dereferences `st.session_state["workspace"]`, and the upload widgets need `location` / `previous_dir` / `local_dir`, all set by `page_setup()`. +- **`input_TOPP()` and `input_python()` require `st.session_state["advanced"]`**, which only `parameter_section()` creates. Calling them from a page that does not go through `show_parameter_section()` raises `KeyError`. +- **The workflow slug is load-bearing in three places at once**: the workflow directory, the `presets.json` top-level key, and every session-state prefix. Renaming a workflow silently orphans its presets *and* abandons existing user parameters. +- Presets apply by writing `params.json` and *deleting* the matching session keys so widgets re-initialize. This misses widgets left at `widget_type="auto"` for numeric/selectbox/multiselect types (an auto-widget double-prefixes its session key), whose stale value then overwrites the preset — pass an explicit `widget_type` for any parameter you intend to drive from a preset. +- Keep new `src/workflow/` logic importable with `streamlit` and `pyopenms` mocked at the `sys.modules` level — that is how most of `tests/` exercises it. +- `docs/toppframework.py` is the most current prose on the framework but has drifted: it still names `src/TOPPWorkflow.py` and `content/6_TOPP-Workflow.py`, and its `input_python` example passes an `input_output=` argument that method does not accept. +- **There is no `pyproject.toml`** despite the pip-compile header in `requirements.txt` — it was never committed, so the lockfile cannot be regenerated. Edit `requirements.txt` directly (the redis/rq/xlsxwriter lines at the bottom are already hand-maintained). +- **The root `entrypoint.sh` is dead code** — superseded by `docker/entrypoint.sh` (all four Dockerfiles copy that one). Its `RUNTIME_DIR` default even names a different project. +- `hooks/hook-analytics.py` is *not* a PyInstaller hook: it rewrites Streamlit's installed `static/index.html` in site-packages to inject analytics. Only run it inside a container image build. +- `.streamlit/secrets.toml` is gitignored; only `.example` is tracked. `example-data/workspaces/` *is* tracked, so save-as-demo writes into version control. +- `docs/build_app.md` is stale where it refers to `APP_NAME`/`REPOSITORY_NAME` in `src/common.py`; those now live in `settings.json` and the module is `src/common/common.py`. diff --git a/Dockerfile b/Dockerfile index d8cb0766..2758925d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -182,6 +182,12 @@ ENV STREAMLIT_SERVER_COUNT=1 COPY docker/entrypoint.sh /app/entrypoint.sh RUN chmod +x /app/entrypoint.sh +# Install the demo-seeding script run by the seed-demos initContainer +# (k8s/base/streamlit-deployment.yaml). One copy of the logic, so what the +# image ships is what tests/test_seed_demos.py executes. +COPY docker/seed-demos.sh /app/docker/seed-demos.sh +RUN chmod +x /app/docker/seed-demos.sh + # Patch Analytics RUN mamba run -n streamlit-env python hooks/hook-analytics.py diff --git a/Dockerfile.arm b/Dockerfile.arm index 2e2fea24..ce17c682 100644 --- a/Dockerfile.arm +++ b/Dockerfile.arm @@ -210,6 +210,12 @@ ENV STREAMLIT_SERVER_COUNT=1 COPY docker/entrypoint.sh /app/entrypoint.sh RUN chmod +x /app/entrypoint.sh +# Install the demo-seeding script run by the seed-demos initContainer +# (k8s/base/streamlit-deployment.yaml). One copy of the logic, so what the +# image ships is what tests/test_seed_demos.py executes. +COPY docker/seed-demos.sh /app/docker/seed-demos.sh +RUN chmod +x /app/docker/seed-demos.sh + # Patch Analytics RUN mamba run -n streamlit-env python hooks/hook-analytics.py diff --git a/Dockerfile_simple b/Dockerfile_simple index 163bcfe6..77aafedd 100644 --- a/Dockerfile_simple +++ b/Dockerfile_simple @@ -104,6 +104,12 @@ ENV STREAMLIT_SERVER_COUNT=1 COPY docker/entrypoint.sh /app/entrypoint.sh RUN chmod +x /app/entrypoint.sh +# Install the demo-seeding script run by the seed-demos initContainer +# (k8s/base/streamlit-deployment.yaml). One copy of the logic, so what the +# image ships is what tests/test_seed_demos.py executes. +COPY docker/seed-demos.sh /app/docker/seed-demos.sh +RUN chmod +x /app/docker/seed-demos.sh + # Patch Analytics RUN mamba run -n streamlit-env python hooks/hook-analytics.py diff --git a/Dockerfile_simple.arm b/Dockerfile_simple.arm index be57317d..b2bdcbc3 100644 --- a/Dockerfile_simple.arm +++ b/Dockerfile_simple.arm @@ -104,6 +104,12 @@ ENV STREAMLIT_SERVER_COUNT=1 COPY docker/entrypoint.sh /app/entrypoint.sh RUN chmod +x /app/entrypoint.sh +# Install the demo-seeding script run by the seed-demos initContainer +# (k8s/base/streamlit-deployment.yaml). One copy of the logic, so what the +# image ships is what tests/test_seed_demos.py executes. +COPY docker/seed-demos.sh /app/docker/seed-demos.sh +RUN chmod +x /app/docker/seed-demos.sh + # Patch Analytics RUN mamba run -n streamlit-env python hooks/hook-analytics.py diff --git a/docker/seed-demos.sh b/docker/seed-demos.sh new file mode 100644 index 00000000..2e3f6d90 --- /dev/null +++ b/docker/seed-demos.sh @@ -0,0 +1,270 @@ +#!/bin/sh +# +# Seed the demo workspaces onto the shared workspaces volume. +# +# Run by the `seed-demos` initContainer in k8s/base/streamlit-deployment.yaml +# and executed directly by tests/test_seed_demos.py, so the shipped logic and +# the tested logic are the same file. Do not inline a second copy in the +# manifest. +# +# Usage: sh seed-demos.sh [SOURCE_DIR] [DEST_DIR] +# SOURCE_DIR defaults to $SEED_SOURCE_DIR, then /app/example-data/workspaces +# DEST_DIR defaults to $SEED_DEST_DIR, then ${WORKSPACES_DIR}/.demos +# +# This replaces `cp -rn /app/example-data/workspaces/. /`, which had two +# failure branches under `replicas: 2` and gets sharper once the volume is +# ReadWriteMany and the replicas can genuinely run at the same instant: +# +# silent cp writes in place with no temp-and-rename, so a replica can see a +# file another one is still writing, decide it exists, and skip it. +# A replica killed mid-copy leaves a truncated file that `-n` makes +# every later run skip forever, so the demos never repair themselves. +# fatal `-n` does not apply to directories, and EEXIST is a hard exit 1, so +# the losing pod ends in Init:CrashLoopBackOff and never resolves. +# +# The fix is to copy into a private staging directory and rename it into place. +# rename(2) on a directory is atomic and fails on a non-empty target, which is +# exactly the arbitration wanted: the winner renames, the loser cleans up and +# exits 0. A lock file would be the wrong tool. +# +# An existing destination is *merged into*, not skipped: entries the volume is +# missing are copied to a staging name and renamed in one at a time, and +# entries that already exist are never touched. That keeps the two documented +# guarantees (docs/kubernetes-deployment.md, "Demo workspaces") - demos shipped +# in a new image appear after a redeploy, and admin-saved demos and edits on +# the volume survive - while repairing a half-seeded `.demos/` that the old +# `mkdir -p` + `cp -rn` could leave behind forever. +# +# Exit status is always 0. This container gates pod start on a shared network +# filesystem, so a seeding failure must degrade the demos, never the app. +# +# stdout carries one machine-readable line, `seed-demos: outcome=`, +# alongside the human-readable ones. tests/test_seed_demos.py reads that line +# rather than the prose, so re-wording a message cannot turn a correct +# implementation red, and the outcomes distinguish the branches that matter: +# +# seeded this run copied the tree and renamed it into place +# backfilled the destination existed and was missing entries, now restored +# present the destination existed and was complete +# lost-race another replica renamed its copy in first +# skipped nothing was done (no source, copy failed, or ran out of time) +# +set -eu + +SOURCE_DIR="${1:-${SEED_SOURCE_DIR:-/app/example-data/workspaces}}" +DEST_DIR="${2:-${SEED_DEST_DIR:-${WORKSPACES_DIR:-/workspaces-streamlit-template}/.demos}}" +SEED_TIMEOUT="${SEED_TIMEOUT:-300}" +SEED_KILL_AFTER="${SEED_KILL_AFTER:-30}" +# Minutes of inactivity after which a staging directory is assumed abandoned. +# Far longer than SEED_TIMEOUT, so a run still in progress is never reaped. +SEED_STAGE_REAP_MINUTES="${SEED_STAGE_REAP_MINUTES:-120}" + +# Trailing slashes are stripped before anything derives a path from DEST_DIR. +# With `/.demos/` the staging directory would be created *inside* the +# destination, `mkdir -p` would bring the destination into existence as a side +# effect, `mv -T` would then fail EINVAL (the target is a path prefix of the +# source), and the empty `.demos/` left behind would short-circuit every later, +# healthy run. SEED_DEST_DIR and argv are both documented inputs, so this is +# reachable without touching the manifest. +while [ "$DEST_DIR" != "/" ] && [ "${DEST_DIR%/}" != "$DEST_DIR" ]; do + DEST_DIR="${DEST_DIR%/}" +done +while [ "$SOURCE_DIR" != "/" ] && [ "${SOURCE_DIR%/}" != "$SOURCE_DIR" ]; do + SOURCE_DIR="${SOURCE_DIR%/}" +done + +# The staging directory is a sibling of the destination so the rename stays +# within one filesystem. +# +# It is named for the pod *and* the pid. The pid alone is not unique here: each +# pod has its own pid namespace, so two replicas on two nodes writing to one +# ReadWriteMany volume are routinely both pid 1, and would then share a staging +# directory and clobber each other's copy. $HOSTNAME is the pod name, which +# carries the ReplicaSet's random suffix. A restarted pod reuses its name, which +# is why the staging directory is removed before it is created: the leftover can +# only belong to a previous incarnation of this same pod. +seed_host() { + if [ -n "${HOSTNAME:-}" ]; then + printf '%s' "$HOSTNAME" + elif command -v hostname >/dev/null 2>&1; then + hostname + else + printf 'unknown-host' + fi +} +SEED_STAGE_DIR="${SEED_STAGE_DIR:-${DEST_DIR}.tmp.$(seed_host).$$}" + +outcome() { + echo "seed-demos: outcome=$1" +} + +# Bound the whole run rather than just the copy: every syscall below touches +# the shared mount, the `stat` behind `[ -d ]` included, and any of them can +# block. Re-exec under `timeout` so one bound covers all of them, then swallow +# the result. (A process wedged in uninterruptible sleep on a `hard` mount +# survives even SIGKILL; this bounds the reachable-but-slow cases, which are +# the ones a restarting NFS server actually produces.) +# +# The re-exec goes through an interpreter, never `"$0"` on its own: the script +# is installed by `COPY` in four Dockerfiles and run as `sh /app/.../seed-demos.sh` +# from the manifest, so nothing else in the chain needs it to carry the +# executable bit, and a checkout where it does not (git records 100644 for a +# new file when core.fileMode is false, which is every Windows clone) would +# otherwise fail the exec with EACCES - reported by `timeout` as exit 126, and +# indistinguishable from a wedged mount unless the status is inspected. +if [ -z "${SEED_DEMOS_BOUNDED:-}" ] && command -v timeout >/dev/null 2>&1; then + SEED_DEMOS_BOUNDED=1 + export SEED_DEMOS_BOUNDED SEED_STAGE_DIR + SEED_SHELL="${SEED_SHELL:-/bin/sh}" + if [ ! -x "$SEED_SHELL" ]; then + SEED_SHELL=sh + fi + seed_status=0 + timeout -k "$SEED_KILL_AFTER" "$SEED_TIMEOUT" \ + "$SEED_SHELL" "$0" "$SOURCE_DIR" "$DEST_DIR" || seed_status=$? + if [ "$seed_status" -eq 0 ]; then + exit 0 + fi + case "$seed_status" in + 124|137) + echo "seed-demos: seeding did not finish within ${SEED_TIMEOUT}s, skipping demos" + echo "seed-demos: pod start must not wait on a wedged mount" >&2 + ;; + 126|127) + echo "seed-demos: could not run the seeding script, skipping demos" + echo "seed-demos: $SEED_SHELL $0 exited $seed_status (not executable, or not found)" >&2 + ;; + *) + echo "seed-demos: seeding failed, skipping demos" + echo "seed-demos: $SEED_SHELL $0 exited $seed_status" >&2 + ;; + esac + outcome skipped + # Best-effort, and bounded too: the staging directory is ours alone, and + # leaking one per pod start would fill the shared volume. + timeout -k 5 30 rm -rf "$SEED_STAGE_DIR" >/dev/null 2>&1 || true + exit 0 +fi + +cleanup() { + if [ -n "$SEED_STAGE_DIR" ]; then + rm -rf "$SEED_STAGE_DIR" >/dev/null 2>&1 || true + fi +} +trap cleanup EXIT INT TERM + +# Reclaim staging directories abandoned by a SIGKILL - an eviction, a drain, an +# OOM - which the trap above cannot cover. They are dot-named siblings of +# `.demos`, and clean-up-workspaces.py skips every top-level entry starting with +# a dot, so nothing else on the volume would ever remove them: a pod replaced +# under a new ReplicaSet suffix comes back with a different name and leaks +# another full copy of example-data/workspaces. Best effort and bounded by the +# same `timeout` as the rest of the run. +sweep_abandoned_stages() { + stage_parent=$(dirname "$DEST_DIR") + stage_base=$(basename "$DEST_DIR") + [ -d "$stage_parent" ] || return 0 + find "$stage_parent" -maxdepth 1 -type d -name "${stage_base}.tmp.*" \ + -mmin "+${SEED_STAGE_REAP_MINUTES}" -exec rm -rf {} + >/dev/null 2>&1 || true + return 0 +} +sweep_abandoned_stages + +# Copy the entries the destination is missing, one atomic rename each. +# +# Directories are created empty and filled; files and symlinks are staged +# outside the destination and renamed in, so no reader ever sees a half-written +# demo file. Anything that already exists is left exactly as it is - that is +# what preserves admin-saved demos and hand edits, and it is why this can run on +# every pod start. +backfill_missing() { + restored=0 + rm -rf "$SEED_STAGE_DIR" + mkdir -p "$SEED_STAGE_DIR" + # `find` walks pre-order, so a directory is always created before the + # entries that go inside it. The list is materialised first: piping into + # `while` would run the loop in a subshell and lose the counter. + if ! (cd "$SOURCE_DIR" && find . -mindepth 1) > "$SEED_STAGE_DIR/manifest" 2>/dev/null; then + return 0 + fi + while IFS= read -r relative; do + relative="${relative#./}" + [ -n "$relative" ] || continue + target="$DEST_DIR/$relative" + # -e is false for a broken symlink, so -L is asked as well: a dangling + # link is still an entry somebody put there deliberately. + if [ -e "$target" ] || [ -L "$target" ]; then + continue + fi + source_entry="$SOURCE_DIR/$relative" + if [ -d "$source_entry" ] && [ ! -L "$source_entry" ]; then + mkdir -p "$target" >/dev/null 2>&1 || true + continue + fi + staged="$SEED_STAGE_DIR/entry.$restored" + # `mv -T`, not plain `mv`: if a racing replica created the target as a + # directory between the check above and here, plain `mv` would move the + # staged copy *inside* it and leave a stray `entry.N` in the tree. + if cp -R "$source_entry" "$staged" >/dev/null 2>&1 && + mv -T "$staged" "$target" >/dev/null 2>&1; then + restored=$((restored + 1)) + else + rm -rf "$staged" >/dev/null 2>&1 || true + fi + done < "$SEED_STAGE_DIR/manifest" + rm -rf "$SEED_STAGE_DIR" + return 0 +} + +if [ -d "$DEST_DIR" ]; then + if [ ! -d "$SOURCE_DIR" ]; then + echo "seed-demos: demos already present, no source tree to merge" + outcome present + exit 0 + fi + backfill_missing + if [ "$restored" -gt 0 ]; then + echo "seed-demos: demos already present, restored $restored missing entries" + outcome backfilled + else + echo "seed-demos: demos already present, nothing to do" + outcome present + fi + exit 0 +fi + +if [ ! -d "$SOURCE_DIR" ]; then + # No destination directory is created here on purpose: one left behind + # would short-circuit every later, healthy run. + echo "seed-demos: no demo source tree, skipping demos" + echo "seed-demos: $SOURCE_DIR is not a directory" >&2 + outcome skipped + exit 0 +fi + +rm -rf "$SEED_STAGE_DIR" +mkdir -p "$SEED_STAGE_DIR" + +if ! cp -r "$SOURCE_DIR/." "$SEED_STAGE_DIR/"; then + echo "seed-demos: copy failed, skipping demos" + echo "seed-demos: could not copy $SOURCE_DIR into $SEED_STAGE_DIR" >&2 + outcome skipped + exit 0 +fi + +if mv -T "$SEED_STAGE_DIR" "$DEST_DIR" 2>/dev/null; then + SEED_STAGE_DIR="" + echo "seed-demos: demo workspaces seeded" + outcome seeded +elif [ -d "$DEST_DIR" ]; then + # `mv -T` onto an existing non-empty directory fails, which is the whole + # arbitration mechanism. Another replica got there first. + echo "seed-demos: another replica seeded the demos first" + outcome lost-race +else + echo "seed-demos: could not install the demos, skipping them" + echo "seed-demos: mv -T $SEED_STAGE_DIR $DEST_DIR failed" >&2 + outcome skipped +fi + +exit 0 diff --git a/docs/a16-storage-decisions.md b/docs/a16-storage-decisions.md new file mode 100644 index 00000000..76539980 --- /dev/null +++ b/docs/a16-storage-decisions.md @@ -0,0 +1,309 @@ +> **What this is.** The decision record behind the NFS-Ganesha storage tier — +> what was chosen, what was rejected, and why. Manifest comments in `k8s/storage/` +> cite it by decision number when a value is deliberate rather than arbitrary. +> +> It is a snapshot of the reasoning at the time of the multi-node work, not a +> living document. Where it disagrees with the code, the code is current. + +# A16 implementation — decision record + +Outcome of a design interview, 2026-08-19. Subject: running a Ganesha NFS-server +pod on a Cinder ReadWriteOnce volume to synthesise RWX, so the OpenMS +streamlit-template can use both de.NBI Berlin nodes. + +Ten questions, all answered, plus a concurrency assessment added afterwards. +Each entry records the decision, not the debate. + +--- + +## Locked decisions + +### 1. A16 is the destination, not a bridge +Consequence: the NFS server must be hardened rather than tolerated, and the +server image is a multi-year commitment. + +### 2. Availability posture — outage accepted, recovery is destructive +A storage-node failure is a **total outage for all users**, not 1/N. Accepted +deliberately. + +Recovery does **not** require de.NBI support, because the data is disposable: +provision a fresh Cinder volume, repoint the claim, restart. Losing in-flight +workspaces is acceptable — the app already deletes them on a timer. + +Backups are handled with OpenStack tooling, outside Kubernetes, which also +defuses the `reclaimPolicy: Delete` hazard recorded in `docs/storage-defect-register.md`. + +> Open: confirm those are Cinder *backups* rather than only *snapshots*. If +> snapshots live in the same Ceph pool as the volume, a backend failure takes +> both — acceptable for a bridge, weak for a destination. + +### 3. Server: sig-storage Ganesha, pinned by digest +`kubernetes-sigs/nfs-ganesha-server-and-external-provisioner`, chart configured +with `existingClaim` so the zero-migration property is preserved. + +Rejected: `itsthenetwork/nfs-server-alpine` (last pushed 7+ years ago, needs +`privileged` plus host kernel modules) and `openebs/dynamic-nfs-provisioner` +(archived). Userspace Ganesha avoids kernel-module dependencies on node images +that de.NBI controls, not us. + +**Mandatory settings, not optional:** + +- `device-based-fsids: false`, with a fixed fsid. It defaults to `true` and + derives NFS file handles from the backing device's major/minor, which is not + stable across Cinder re-attach. Left alone, every client ESTALEs after the + first NFS pod restart — weeks later, with no obvious cause. +- `strategy: Recreate`. The default `RollingUpdate` deadlocks: the new pod + cannot mount the RWO claim while the old one still holds it. +- 1 replica. The chart supports no more, which is consistent with decision 2. +- **Explicit memory request and limit on the Ganesha pod.** The chart default is + modest, and N concurrent write streams mean N sets of buffers and dirty page + cache in one userspace process. Left at defaults, heavy load OOMKills it — + every client then hangs on `hard` mounts and eats the ~90s NFSv4 grace period + on restart, with the same load waiting when it returns. This is the one + genuine collapse mode in the design, and it is a one-line fix. + +### 4. Namespace layout — Option B, split +- **New `openms-storage`**, labelled `pod-security.kubernetes.io/enforce=privileged`. + Holds Ganesha and the Cinder PVC. +- **`openms`** labelled `enforce=baseline` explicitly, rather than relying on the + absence of a label. +- **No volume migration.** Provision a fresh Cinder PVC in the new namespace, + seed `.demos`, cut over. The disposable-data property makes the usual + patch-PV / clear-claimRef / rebind surgery unnecessary, and the old volume + stays intact as an instant rollback. + +Rationale: `openms` is shared with another application (`app=nuxl-app`), and +Ganesha exports `no_root_squash`. Keeping the storage tier in its own namespace +means the exemption covers only the component that needs it. + +### 5. Ordering — fixes, then storage, then spread +Three PRs, in order: + +1. **`params.json` Fix 1** + `tasks.py` re-raise + the `src/Workflow.py` return + value. +2. `openms-storage` + Ganesha + the NFS-backed PV. +3. Delete the memory-tier `nodeselector.yaml` patches, keep the components as + resource patches, set requests == limits, and add the second worker. See + decision 10 — the tier is now the pod size, not the node. + +PR 1 is not optional. Until `tasks.py:143-148` re-raises instead of returning a +dict, queue mode reports every failure as success — so the cutover in PR 2 could +not be validated against anything. + +**Fix 1** (required before A16): stop `get_parameters_from_json` laundering read +failures into `{}`. Ten lines, zero runtime I/O. Separates *file absent* — +legitimately `{}` — from *read failed*, which must never become `{}`, because +the caller merges the result and writes it back. On NFS this is the case that +matters: a single user alone on the system loses every stored parameter when the +NFS server restarts and the read returns ESTALE. + +**Fix 2** (atomic writes) deferred. When it lands: use +`tempfile.mkstemp(dir=path.parent)`, **not** a pid-based temp name — +Streamlit runs every session as a thread inside one process, so all concurrent +sessions in a pod share a pid and would interleave into one temp file. Also note +Fix 2 adds ~4 metadata operations per widget render on a metadata-bound +filesystem; measure it after cutover and consider debouncing the save. + +**Fix 3** (locking) deferred until the second worker exists. When it lands, +prefer `flock` on the params file over a Redis lock: NFSv4.1 has integrated +locking, so the lock and the data live on the same filesystem and cannot +diverge — whereas the current Redis (1 replica, 256Mi, `--appendonly no`) could +only support an advisory lock needing a fencing token. + +### 6. Mount options — `hard`, NFSv4.1 +- **`hard`**, not `soft`. The data is disposable but the *jobs* are not: a soft + mount converts a brief server restart into a truncated featureXML, which is + the one failure class the 7-day retention does not cover. Clients block and + resume instead. +- **NFSv4.1** — integrated locking (needed for Fix 3), a single port 2049 with + no separate NLM/statd (which makes the NetworkPolicy tractable), and stateful + reclaim. +- **`nconnect=4`** — nodes run kernel 6.8, and it helps the metadata-heavy + access pattern specifically. +- **Default `actimeo`** — close-to-open consistency already guarantees a fresh + read on every `open()`, and `Logger` closes per message while the UI reopens + per second. Lowering it would add GETATTR traffic for no correctness gain. +- **Default grace period.** Accept ~90s of degraded service after any Ganesha + restart, during which locks are reclaimed but new ones are refused. + +### 7. Detection — signal only, never self-heal +With `hard` mounts the failure mode is a hang, not an error, and nothing in the +system currently detects one: `health.py` inspects only Redis and RQ, +`/_stcore/health` is filesystem-blind, `rq-worker` has no probes, and RQ's parent +process keeps heartbeating while its work horse is blocked in an NFS syscall. + +- **Readiness probe only on `rq-worker`**: `timeout 5 stat …/.nfs-probe`. + Readiness has no traffic effect on a worker, so it is purely an alertable + signal. +- **Never a liveness probe on `rq-worker`.** It would kill in-flight TOPP jobs, + and restarting cannot fix NFS — a crash loop that destroys work while the + fault persists. +- **NOT in Streamlit's readiness probe.** An earlier draft put the mount check + there so a wedged mount produced an honest 503. Reversed: with `hard` mounts a + saturated share makes `stat` exceed the probe timeout, so the probe would pull + every Streamlit pod out of Traefik and the app would go *down* under exactly + the load it should survive. The probe timeout would silently become a + load-shedding threshold. Degradation should be visible, not fatal — the + sidebar indicator carries that signal instead. +- **TCP liveness on Ganesha's 2049.** The one place a liveness probe is correct. +- The `.nfs-probe` sentinel is exempt from the 7-day GC twice over: + `clean-up-workspaces.py:27` skips dot-names, and it only considers directories. + +### 8. Sidebar storage indicator +Follows the existing `monitor_queue()` pattern at `common.py:231` — +`@st.fragment(run_every=5)` backed by a function in `health.py`, rendering +nothing in local mode when `REDIS_URL` is unset. + +**The sidebar must never touch the filesystem.** A `stat` on a `hard`-mounted +wedged path blocks in uninterruptible sleep and cannot be killed, so a fragment +re-running every 5 seconds would accumulate unkillable threads — the indicator +would become the hang. + +Inverted design: + +``` +canary pod sidebar + stat .nfs-probe ──ok──▶ SETEX storage:ok: 30 ──▶ reads Redis only + └── blocks on wedged NFS ──▶ key expires ──▶ indicator goes red +``` + +- Redis TTL *is* the liveness mechanism; no timeout logic to get wrong. +- Key per node via the Downward API, or a healthy node masks a wedged one. +- **Three states**: connected / unreachable / **unknown when Redis is down**. + A red tick caused by a dead Redis sends you debugging the wrong layer. +- Prefer a separate small canary Deployment over a thread in `rq-worker`. + +### 9. NetworkPolicy — pod-label-scoped +Ingress rule in `openms-storage`, admitting only pods labelled +`app: template-app` — which the prod overlay already sets via `commonLabels`. +Plus a default-deny ingress in that namespace. No egress rules. + +Namespace-scoping was rejected: it would grant NuXL's pods root read/write to +every workspace, undoing the reason for the namespace split. Add +`openms-storage` to the CI kustomize render check so label drift appears as a +diff rather than as a 3am mount failure. + +### 10. Worker topology — tiers become pod sizes, not node labels + +**Fixed worker count. No autoscaling, no per-job pods, no dispatcher.** KEDA, +`ScaledJob` and the RQ-worker-as-dispatcher options are all out. + +**The tier moves from the node to the pod.** Instead of +`openms.de/memory-tier: high|low` node labels plus a `nodeSelector`, each worker +Deployment is simply *sized*: N small workers with small requests/limits, M large +workers with large ones. The cluster then places them wherever they fit. + +Why this is better than what the interview had landed on: + +- **It deletes a defect instead of working around it.** `D2` in `docs/storage-defect-register.md` — the + unscoped `nodeSelector` patching `kind: Deployment` with no name — stops needing + to be scoped, because the `nodeselector.yaml` patches are removed outright. The + `memory-tier-*` components survive as *resource* patches, which is exactly what + is now wanted. +- **It scales without redesign.** At 100 nodes there are no labels to maintain and + no per-tier node pools to balance. A large worker asks for its memory and lands + wherever there is room. +- **`topologySpreadConstraints` start working.** A13 found they produce a false + pass today, because `maxSkew` is measured over *eligible* domains and the + nodeSelector left exactly one eligible node. With the selector gone every node is + eligible, so `maxSkew: 1` genuinely enforces the spread. +- **The cleanup CronJob problem disappears.** `D1` — the CronJob hanging on + Multi-Attach because it carries no placement constraint — is moot once the volume + is RWX. + +**Consequence that is now architectural, not advisory: requests must equal limits.** +The pod size *is* the tier, so a large worker requesting 2Gi against a 180Gi limit +would be schedulable anywhere and would then OOM. Setting requests == limits gives +Guaranteed QoS and `oom_score_adj = -997` instead of today's burstable ~937. +Previously a recommendation; under this design the architecture does not work +without it. + +**Queue strategy: one queue per size class.** A job is routed by choosing the queue +matching the smallest size class that fits. Each worker still watches exactly one +queue name, so RQ's crash-safe `BLMOVE` dequeue stays engaged. The routing function +itself is trivial; the interesting part is how a workflow declares its ceiling — +see the open item below. + +**Revisit trigger.** Fixed workers statically partition capacity. That is correct +at the current size and stops being correct as node count grows, because the split +becomes a guess that is wrong most of the time and fragmentation appears. Revisit +when node count grows materially or when the large-tier workers are observed idle +while the small-tier queue backs up. Instrumentation for that already exists: +`health.py` reads per-queue depth and worker state for the sidebar. + +### 11. Concurrent write behaviour under DIA load + +Assessed for ~30 concurrent users each producing ~100 GB of output over a run. + +**Accepted: the design degrades before it collapses.** NFS has no per-client QoS, +so all writers share one queue at the server, and `hard` mounts mean nobody errors +— they wait. Saturation therefore presents as everything slowing together rather +than as corruption. That is an acceptable failure mode here. + +Two caveats recorded rather than solved: + +- **There are no bulkheads.** One pathological workflow degrades every user, + including the UI, because everything mounts the same share. Accepted. +- **`.osw` and `.sqMass` are SQLite** — small random writes plus an `fsync` per + commit, each of which is a synchronous round trip over NFS. Synchronous op + latency, not bandwidth, is what saturates first; you can be far below the NIC's + capacity and still be fully queued. NFSv4.1's integrated locking makes this + *supported*, not *fast*. + +**Escape hatch if writes do saturate**, not adopted now: write results to +node-local scratch during the run and promote to the shared tier on completion. +That converts N hours-long streams of small synchronous writes into N bulk +sequential copies, and keeps SQLite on local disk where it belongs. It composes +with everything above — no change to Ganesha, the namespace layout or the mount +options. Cost: results are not visible until the workflow finishes, and a crashed +pod loses them. + +**Unmeasured.** de.NBI Berlin's per-volume Cinder throughput, whether QoS caps +apply, and node NIC speed are all unknown, and they set the real ceiling. A load +test of N pods writing concurrently plus a representative SQLite commit workload +would find the knee where latency goes non-linear — that number is the true +concurrent-user limit, and it will be smaller than the bandwidth arithmetic +suggests. + +--- + +## Open items + +| # | Item | Blocks | +|---|---|---| +| 1 | Does the node have `mount.nfs`? Decides in-tree `nfs:` PV vs csi-driver-nfs in `kube-system`. Probe: `kubectl debug node/ -n openms …` | PR 2 | +| 2 | How a workflow **declares its memory ceiling** — constant, class attribute, or a callable over (params, inputs, threads). The routing function is trivial; the declaration is the design. Research outline at `../job-tier-routing` | PR 3 | +| ~~3~~ | ~~Fixed fsid value~~ — **closed**: `Export_Id: 1`, `deviceBasedFsids: false`, never changed. See `docs/a16-storage-runbook.md` §1 | — | +| ~~4~~ | ~~`.demos` seeding race~~ — **closed**: copy-to-temp plus `mv -T`, script in `docs/a16-storage-runbook.md` §2 | — | +| ~~5~~ | ~~Cutover verification~~ — **closed**: six ordered pass/fail assertions in `docs/a16-storage-runbook.md` §4 | — | +| ~~6~~ | ~~Rollback procedure~~ — **closed**: `docs/a16-storage-runbook.md` §5. Cheap by construction, since the cutover never touches the original volume | — | + +--- + +## Facts established during the interview + +Cluster, verified live: + +- `openms` carries **no** `pod-security.kubernetes.io/*` labels, so it inherits + the built-in default of `privileged`. Confirmed empirically — a pod requesting + `DAC_READ_SEARCH` admits. +- `default` and `kubernetes-dashboard` carry `enforce=baseline`; `kube-system`, + `kube-public`, `kube-node-lease` and `cloud-init-settings` carry `privileged`. + So PSA is applied per namespace, not cluster-wide. +- `kubectl auth can-i create daemonsets -n kube-system` → **yes**; + `update namespaces` → **yes**. csi-driver-nfs is installable. +- **`openms` is shared with another application**, labelled `app=nuxl-app`. +- A `traefik` namespace exists and is unlabelled. + +Codebase, verified by reading: + +- Retention is **7 days**, not two weeks — `clean-up-workspaces.py:15`, + hardcoded, with no env var or settings key. +- Nothing enumerates the workflow directory itself; every `iterdir()` in + `src/workflow/` targets a subdirectory. A `params.json.*.tmp` sibling is inert. +- `health.py` inspects only Redis and RQ, and is wired to `common.py:235` for + sidebar display — not as a probe endpoint. +- `os.getpid()` is not unique per session: Streamlit runs sessions as threads + within one process. + +New defect found during the interview — see `docs/storage-defect-register.md` entry D7. diff --git a/docs/a16-storage-runbook.md b/docs/a16-storage-runbook.md new file mode 100644 index 00000000..aa9099cf --- /dev/null +++ b/docs/a16-storage-runbook.md @@ -0,0 +1,297 @@ +> **What this is.** The operational runbook for the NFS-Ganesha storage tier +> that makes the workspace volume ReadWriteMany, so the app can run on more than +> one Kubernetes node. It covers the cutover, the verification matrix and the +> rollback, and it is the authority several manifests in `k8s/` point at by name. +> +> It records the design as of the multi-node work on the de.NBI Berlin cluster. +> Cluster-specific values (node CIDRs, volume sizes, node names) are examples, +> not settings to copy. See `docs/kubernetes-deployment.md` for the general +> deployment guide. + +# A16 cutover runbook — PR 2 + +Companion to `docs/a16-storage-decisions.md`. That file records *what* was decided; this one +records *how to do it and how to undo it*. + +Closes open items 3, 4, 5 and 6. Items 1 (`mount.nfs` probe) and 2 (memory-ceiling +declaration) remain open and are noted where they bite. + +--- + +## 0. Pre-flight + +**Blocking — item 1.** Decides whether clients mount via the in-tree `nfs:` type +or need csi-driver-nfs in `kube-system`: + +```bash +kubectl get nodes -o name # then, for EACH node returned: +kubectl debug node/ -n openms -it --image=busybox -- \ + sh -c 'ls -l /host/sbin/mount.nfs* /host/usr/sbin/mount.nfs* 2>&1; + echo ---; grep nfs /host/proc/filesystems' +``` + +Run against both nodes — a node pool can drift. Missing helper means +csi-driver-nfs, **not** installing `nfs-common` via a DaemonSet: Kubermatic +replaces nodes on upgrade, so package installation becomes a permanent obligation +with a race window on every new node. + +**Non-blocking but do it first.** PR 1 must be merged and deployed — the +`params.json` Fix 1, the `tasks.py` re-raise, and the `src/Workflow.py` return +value. Without the re-raise, queue mode reports every failure as success and none +of section 4's verification below means anything. + +--- + +## 1. Fixed fsid — open item 3, decided + +**Set `Export_Id: 1` and `device-based-fsids: false`. Never change either.** + +The value itself is arbitrary; its *stability* is the whole point. With +`device-based-fsids: true` (the chart default) Ganesha derives NFS file handles +from the backing device's major/minor number, and a Cinder volume's `/dev/vdX` +minor is not stable across re-attach. Every client then gets `ESTALE` after the +first NFS pod restart — typically weeks later, with no obvious cause and no +correlation to the change that caused it. + +Record it in the values file with a comment stating that changing it invalidates +every client's file handles, because the next person to read this will otherwise +treat it as a tunable: + +```yaml +# Export_Id is part of every NFS file handle handed to every client. +# Changing it, or enabling device-based-fsids, invalidates all of them: +# clients get ESTALE until they remount. It is not a tunable. Do not touch. +storageClass: + ... +nfs: + exportId: 1 + deviceBasedFsids: false +``` + +Verify after deploy — the export should show a stable fsid independent of the +device: + +```bash +kubectl -n template-app-storage exec "$(kubectl -n template-app-storage get pod -l app=nfs-server -o name | head -n 1)" -- \ + sh -c 'grep -i "Export_Id\|Fsid" /export/vfs.conf 2>/dev/null || cat /etc/ganesha/ganesha.conf' +``` + +--- + +## 2. `.demos` seeding race — open item 4, fix + +`streamlit-deployment.yaml:25-26` currently runs, in an initContainer, under +`replicas: 2`: + +```sh +mkdir -p /workspaces-streamlit-template/.demos +cp -rn /app/example-data/workspaces/. /workspaces-streamlit-template/.demos/ +``` + +Two failure branches, one silent and one fatal: + +- **Silent**: `cp` writes in place with no temp-and-rename, so pod B can see a + file pod A is still writing, decide it exists, and skip it — permanently. A + truncated demo workspace that never repairs itself. +- **Fatal**: `-n` does not apply to directories, and `mkdirat`/`openat(O_EXCL)` + returning `EEXIST` is a hard `exit 1` — so the losing pod ends in + `Init:CrashLoopBackOff` and never self-resolves. + +Both get worse with RWX, because the two replicas can now genuinely run at the +same moment on different nodes. + +**Fix — copy to a private temp directory, then rename. A lock file is the wrong +tool; `rename(2)` on a directory is already atomic.** + +```sh +set -eu +DEST=/workspaces-streamlit-template/.demos +if [ -d "$DEST" ]; then + echo "demos already seeded" + exit 0 +fi +TMP="${DEST}.tmp.$$" +rm -rf "$TMP" +mkdir -p "$TMP" +cp -r /app/example-data/workspaces/. "$TMP"/ +# mv -T onto an existing non-empty directory fails, which is exactly what we +# want: the winner renames, the loser cleans up and exits 0. +if mv -T "$TMP" "$DEST" 2>/dev/null; then + echo "demos seeded" +else + echo "lost seeding race, another replica won" + rm -rf "$TMP" +fi +``` + +Note the `if [ -d ]; then … fi` rather than `[ -d ] && exit 0` — under `set -e` +the latter exits non-zero when the directory is absent, which is the common path. + +--- + +## 3. Cutover + +Decision 4 provisions a **fresh** Cinder PVC in `template-app-storage` rather than +migrating the existing one, so the old volume stays intact and untouched +throughout. That is what makes section 5 cheap. + +1. Create `template-app-storage`, labelled `pod-security.kubernetes.io/enforce=privileged`. +2. Label `openms` explicitly `enforce=baseline` — do not rely on the absent label. +3. Provision the new Cinder PVC in `template-app-storage`. +4. Deploy Ganesha with `existingClaim`, `Export_Id: 1`, `deviceBasedFsids: false`, + `strategy: Recreate`, 1 replica, and **explicit memory request and limit**. +5. Apply the default-deny ingress plus the two allow rules on 2049: the + pod-label-scoped one admitting `app: template-app` in `openms`, and an + `ipBlock` for the cluster's node addresses. **The second is not optional + and ships as a placeholder.** The provisioner emits in-tree `nfs:` PVs, + which the kubelet mounts from the node's own address in the host network + namespace — that matches no `podSelector`, so with the placeholder left in + place every mount hangs. Read the addresses from `kubectl get nodes -o wide` + and narrow the range as far as it will go; check it does not contain the pod + CIDR, which would hand every pod in the cluster root over every workspace. +6. Create the PVC in `openms` on the new StorageClass, mounted at the unchanged + path `/workspaces-streamlit-template`. It is a NEW claim, `workspaces-nfs-pvc`, + not an edit of `workspaces-pvc`: a bound PVC's spec is immutable apart from + `resources.requests`, so `kubectl apply` would be rejected outright, and the + only way past that is deleting a claim whose `cinder-csi` class reclaims with + `Delete` — destroying the volume section 5 rolls back to. +7. Seed `.demos` via the fixed initContainer above, and create the `.nfs-probe` + sentinel. +8. Repoint the streamlit and rq-worker Deployments at the new claim. Delete the + `nodeselector.yaml` patches; keep the `memory-tier-*` components as resource + patches; set requests == limits. +9. Deploy the storage canary and the sidebar indicator. + +--- + +## 4. Verification — open item 5 + +Run in order. Each is a pass/fail assertion, not an observation. **Stop at the +first failure**; later steps assume earlier ones passed. + +### 4.1 The thing the whole project is for + +```bash +kubectl -n openms get pods -o wide -l app=template-app +``` + +**Assert:** at least two pods, on **two different nodes**, all `Running`. +This is the acceptance criterion for the entire exercise — if every pod is still +on one node, nothing was achieved regardless of what else passes. + +### 4.2 Shared visibility, cross-node + +From a pod on node A, then a pod on node B: + +```bash +# node A +kubectl -n openms exec -- sh -c 'echo hello-from-a > /workspaces-streamlit-template/.crosscheck' +# node B +kubectl -n openms exec -- cat /workspaces-streamlit-template/.crosscheck +``` + +**Assert:** `hello-from-a`. Close-to-open consistency guarantees this on a fresh +`open()`; if it fails, the mount options are wrong. + +### 4.3 POSIX features the codebase actually requires + +```bash +kubectl -n openms exec -- sh -c ' + cd /workspaces-streamlit-template + ln -sf /app/example-data/mzML/Control.mzML .symcheck && readlink .symcheck # absolute symlink + echo x > .rncheck.tmp && mv .rncheck.tmp .rncheck && echo rename-ok # atomic rename + flock .rncheck -c "echo flock-ok" # advisory locking + rm -f .symcheck .rncheck' +``` + +**Assert:** an absolute path from `readlink`, `rename-ok`, and `flock-ok`. + +Absolute symlinks are non-negotiable — `common.py:179`, `:678` and +`fileupload.py:91` all call `symlink_to(x.resolve())`, so demo seeding breaks +without them. `flock` is not needed today but is the intended lock backend for +Fix 3, so failing here changes that decision to a Redis lock. + +### 4.4 End-to-end workflow + +Run a real workflow through the UI, small enough to finish quickly. + +**Assert:** it completes, results appear in `results()`, and the log renders. Then +confirm the failure signal actually works — this is what PR 1 bought: + +```bash +kubectl -n openms exec deploy/rq-worker -- \ + python -c "from redis import Redis; from rq import Queue; import os; + q=Queue('openms-workflows', connection=Redis.from_url(os.environ['REDIS_URL'])); + print('failed:', len(q.failed_job_registry))" +``` + +**Assert:** a deliberately broken workflow lands in the failed registry. Before +PR 1 this count was structurally always zero, so a non-zero value here is the +proof that PR 1 works. + +### 4.5 Ganesha restart — the routine case + +```bash +kubectl -n template-app-storage delete pod -l app=nfs-server +``` + +**Assert:** clients block rather than erroring (this is `hard` mounts working as +intended); a workflow running throughout **completes**; the sidebar indicator goes +red then green within ~30s of the pod becoming ready plus the grace period. + +This is the single most valuable test in the list, because a Ganesha restart is a +routine event and this is exactly the case a `soft` mount would have corrupted. + +### 4.6 Detection + +```bash +kubectl -n template-app-storage scale statefulset -l app=nfs-server --replicas=0 +``` + +**Assert:** within ~30s the `rq-worker` pod reports `0/1 Ready`; the sidebar +indicator shows unreachable; **Streamlit stays `Ready` and keeps serving** — that +last one verifies decision 7's reversal, that a wedged mount degrades the app +rather than removing it from service. Scale back to 1 and confirm recovery. + +--- + +## 5. Rollback — open item 6 + +**Rollback is cheap by construction**, because the cutover never touched the +original volume. + +**Trigger it if:** 4.1 shows pods still co-located after the nodeSelector removal; +4.3 fails on symlinks or atomic rename; 4.5 shows workflows dying rather than +blocking through a restart; or sustained `ESTALE` appears in worker logs. + +**Procedure** — one apply, roughly a minute: + +1. `kubectl -n openms scale deploy/streamlit deploy/rq-worker --replicas=0` +2. Re-apply the previous manifests: Deployments mounting the **original** + `workspaces-pvc` in `openms`, with the `memory-tier-low` nodeSelector restored. + That claim is still Bound to its Cinder volume — the cutover created + `workspaces-nfs-pvc` alongside it and never touched it — so there is no + restore step, only an apply. +3. Scale back up. Everything lands on one node again, as before. +4. `kubectl -n template-app-storage scale statefulset -l app=nfs-server --replicas=0` — leave the + namespace and its volume in place for diagnosis rather than deleting them. + +**What is lost:** anything written since cutover. Acceptable under decision 2, and +the same loss the 7-day GC inflicts on schedule anyway. + +**What is not lost:** the original Cinder volume, untouched throughout. + +**Do not** delete the `template-app-storage` PVC while diagnosing — `reclaimPolicy` is +`Delete`, so removing the claim destroys the volume and the evidence with it. + +--- + +## 6. After cutover — first week + +- Watch the sidebar indicator for unexplained red flaps: that is `ESTALE` from an + fsid problem, and it appears late rather than immediately. +- Watch worker memory against the new requests == limits. Guaranteed QoS means the + pod is killed at its limit rather than being allowed to borrow, so a limit that + was previously generous headroom is now a hard wall. +- Measure before assuming: per-volume Cinder throughput and node NIC speed are both + unknown, and they set the real concurrent-user ceiling described in decision 11. diff --git a/docs/kubernetes-deployment.md b/docs/kubernetes-deployment.md index d1264c58..25e1aa66 100644 --- a/docs/kubernetes-deployment.md +++ b/docs/kubernetes-deployment.md @@ -10,7 +10,8 @@ The template ships a full Kubernetes deployment stack designed for the de.NBI cl - A Redis Deployment as the job-queue backing store - An RQ worker Deployment running background workflows - A nightly cleanup CronJob for stale workspaces -- A shared PersistentVolumeClaim holding per-user workspace data +- A shared `ReadWriteMany` PersistentVolumeClaim holding per-user workspace data +- A separate storage tier (`k8s/storage/`) running one NFS-Ganesha server that re-exports a single Cinder volume as `ReadWriteMany`, which is what lets the pods above be placed on any node - A Traefik IngressRoute routing external traffic to the Streamlit service (with session affinity) Every production OpenMS webapp (quantms-web, umetaflow, FLASHApp) deploys via this stack. @@ -35,8 +36,8 @@ Every production OpenMS webapp (quantms-web, umetaflow, FLASHApp) deploys via th │ Streamlit Deployment │ │ (N replicas, default 2) │ │ │ - │ [co-located with rq-worker + │ - │ cleanup pods via shared RWO PVC] │ + │ [placed by the scheduler; the RWX │ + │ volume constrains nothing] │ └────────┬────────────────────────┬───────┘ │ REDIS_URL │ │ │ /workspaces-... @@ -46,15 +47,19 @@ Every production OpenMS webapp (quantms-web, umetaflow, FLASHApp) deploys via th │ (1 replica) │ ▼ └────────────────────────┘ ┌────────────────────────┐ ▲ │ Workspace PVC │ - │ │ ReadWriteOnce 500Gi │ - │ REDIS_URL │ (cinder-csi) │ + │ │ ReadWriteMany 400Gi │ + │ REDIS_URL │ (-nfs, served │ + │ │ by k8s/storage/) │ │ └────────────────────────┘ │ ▲ │ │ ┌────────┴───────────────────────┴────────┐ │ RQ Worker Deployment │ - │ (1 replica) │ + │ (2 replicas) │ │ rq worker openms-workflows │ + │ │ + │ [spread one per node, maxSkew 1 │ + │ over kubernetes.io/hostname] │ └─────────────────────────────────────────┘ ┌─────────────────────────────────────────┐ @@ -70,19 +75,30 @@ Every production OpenMS webapp (quantms-web, umetaflow, FLASHApp) deploys via th |-----------|---------|----------|-------------| | Streamlit Deployment | Serves the web UI | N (default 2) | Yes | | Redis Deployment | Job-queue backing store | 1 | No | -| RQ Worker Deployment | Runs background workflows from the Redis queue | 1 | Yes | +| RQ Worker Deployment | Runs background workflows from the Redis queue | N (default 2) | Yes | | Cleanup CronJob | Removes stale workspaces nightly at 03:00 UTC | — | Yes | | Workspace PVC | Shared `/workspaces-*` directory for session data | — | — | | Traefik IngressRoute | External HTTP entrypoint with sticky sessions | — | — | | nginx Ingress | Alternative HTTP entrypoint used by the CI kind cluster | — | — | -### Pod co-location via the RWO PVC +### Pod placement and the shared workspace volume + +All workspace-using pods (Streamlit, RQ worker, Cleanup) of a given fork mount the same `-workspaces-nfs-pvc`, which is `ReadWriteMany` on the `-nfs` StorageClass published by the storage tier in `k8s/storage/`. Because it is RWX it imposes **no** placement constraint: any number of pods on any number of nodes can hold it at once, and the scheduler alone decides where they land. + +That is what the storage tier is for. The previous claim was `ReadWriteOnce` on `cinder-csi`, and a Cinder volume attaches to exactly one node, so every workspace-using pod ended up on that node whether it fitted there or not. + +Nothing else constrains placement either. The `memory-tier-*` components are pure resource patches: they set the Streamlit and worker `requests`/`limits`, and a tier is now a **pod size**, not a set of nodes. No manifest in `k8s/` carries a `nodeSelector`, a `nodeName`, a `nodeAffinity` or an `openms.de/memory-tier` label, and CI fails the build if one reappears (`assert_no_node_pinning_anywhere`). Placement is the scheduler's decision alone; the manifests only tell it how big a worker is. See Step 4b below. + +The workers go one step further and are actively spread. `rq-worker` runs a fixed replica count (2 by default) with a `topologySpreadConstraints` of `maxSkew: 1` over `kubernetes.io/hostname`, so the replicas land on different nodes before any node takes a second one. Two details make that work rather than merely look right: -All workspace-using pods (Streamlit, RQ worker, Cleanup) of a given fork mount the same `-workspaces-pvc` (`ReadWriteOnce`, `cinder-csi`). Once the first pod schedules, the volume is attached to that node and the kube-scheduler's `VolumeBinding` plugin pins every subsequent pod that mounts the same PVC to the same node. NodeSelector (`openms.de/memory-tier`) picks which set of nodes the fork is eligible for; the RWO mount picks the specific node within that set. +- `maxSkew` is measured over **eligible** domains. While the `nodeSelector` was there, exactly one node was eligible, every distribution had skew 0, and a spread constraint written then would have passed while changing nothing. Deleting the selector is what gave the constraint something to measure. +- The constraint's `labelSelector` names `component: rq-worker` in the base, and kustomize copies the overlay's `commonLabels.app` into it. Topology spread counts pods per namespace and every fork deploys into `openms`, so without that label one fork's workers would be counted against another's skew. -There is no pod-affinity rule. Forks are isolated from each other — co-location applies within a fork (because they share a PVC), not across forks (each fork has its own PVC). +`whenUnsatisfiable: DoNotSchedule`, so a replica with nowhere to go stays `Pending` instead of quietly doubling up — including during a node drain, where the cordoned node still counts as a domain. That is visible in `kubectl get pods` and recoverable; a soft constraint that silently stops spreading is neither. -Co-location is a placement constraint, not a replica cap. The Streamlit deployment can scale to N replicas — they all land on the same node alongside the worker. +Recoverable is not free, though, and it is `strategy` that buys it. A Deployment is `Available` only when `replicas - maxUnavailable` pods are ready, and the RollingUpdate default of 25% rounds *down* to `maxUnavailable: 0` at two replicas — so one `Pending` worker during a drain, or one worker failing its storage readiness probe during a Ganesha restart, would take the whole Deployment out of `Available` and hang every subsequent `kubectl rollout status`. `rq-worker` therefore sets `maxUnavailable: 1` explicitly (and `maxSurge: 0`, because a surge replica needs a whole extra worker's worth of memory free somewhere, which a cluster sized for the steady-state count does not have). The pair is what makes a lost worker a queue running at half rate rather than a blocked rollout. That the *other* replica is genuinely up is asserted directly against `.spec.replicas`, by `assert_workers_spread_across_nodes` in CI, rather than being inferred from the `Available` condition. + +Forks are isolated from each other: each has its own claim, its own export and its own storage namespace. ### Ingress @@ -104,15 +120,19 @@ Creates the `openms` namespace. All resources deploy into it. Redis 7 Deployment (1 replica) + ClusterIP Service on port 6379. Backs the RQ job queue. Low resource requests (64Mi / 50m CPU). ### `workspace-pvc.yaml` -PersistentVolumeClaim `workspaces-pvc`: -- `accessModes: [ReadWriteOnce]` -- `storageClassName: cinder-csi` -- `resources.requests.storage: 500Gi` +PersistentVolumeClaim `workspaces-nfs-pvc`: +- `accessModes: [ReadWriteMany]` +- `storageClassName: template-app-nfs` — published by `k8s/storage/`, and cluster-scoped, so a fork renames it (see Step 4c) +- `resources.requests.storage: 400Gi` + +The size has to stay materially below the backing volume in `k8s/storage/nfs-backing-pvc.yaml` (500Gi). nfs-provisioner `statfs()`es its export on every provision and refuses any claim larger than the free space it finds there, so a claim sized equal to the backing volume never binds at all. + +This is a **different object** from the `workspaces-pvc` earlier releases used. A bound PVC's spec is immutable apart from `resources.requests`, so the `ReadWriteOnce` claim could not be converted in place — and deleting it would have destroyed its Cinder volume, because `cinder-csi` reclaims with `Delete`. The old claim is therefore left alone, which is also what makes rollback a single re-apply of the previous manifests. Delete it by hand once the cutover has been stable for a week. Demo workspaces live under a hidden `.demos/` subdirectory of this PVC (see [Demo workspaces](#demo-workspaces) below). User workspaces live at the PVC root, one directory per session UUID. ### Demo workspaces -Demo workspaces are seeded onto the `workspaces-pvc` at `/workspaces-streamlit-template/.demos/` by the `seed-demos` initContainer on the Streamlit Deployment. The init runs `cp -rn /app/example-data/workspaces/. /workspaces-streamlit-template/.demos/` — new demos shipped in an image appear after redeploy, but existing entries on the PV (including admin-saved demos and edits) are preserved. +Demo workspaces are seeded onto the workspace PVC at `/workspaces-streamlit-template/.demos/` by the `seed-demos` initContainer on the Streamlit Deployment. The init runs `docker/seed-demos.sh` — new demos shipped in an image appear after redeploy, but existing entries on the PV (including admin-saved demos and edits) are preserved. On an empty volume the whole tree is staged in a private directory and renamed into place, so `replicas: 2` cannot corrupt it and a pod killed mid-copy leaves nothing half-written; on a volume that already has `.demos/`, only the entries it is missing are copied in, one atomic rename each. The script bounds itself with `timeout` and always exits 0, so a slow or restarting NFS server degrades the demos instead of holding the pod in `Init`. The ConfigMap override points `demo_workspaces.source_dirs` at `/workspaces-streamlit-template/.demos`, so both Streamlit pods and RQ workers read demos from the PV. The "Save as Demo" admin flow writes to the same path. @@ -132,14 +152,20 @@ Main Streamlit Deployment. Key fields: - Mounts the workspace PVC at `/workspaces-streamlit-template` - Mounts `settings-overrides.json` from the ConfigMap as a `subPath` - Readiness and liveness probes hit `/_stcore/health` -- Co-located with the RQ worker (and any cleanup Job) on the node the RWO `workspaces-pvc` is attached to +- Mounts the RWX workspace PVC, which places no constraint on which node the pod lands on - `seed-demos` initContainer merges image-shipped demos into `.demos/` on the PVC (see [Demo workspaces](#demo-workspaces)) ### `streamlit-service.yaml` ClusterIP Service exposing Streamlit on port 8501. ### `rq-worker-deployment.yaml` -RQ worker Deployment (1 replica). Runs `rq worker openms-workflows --url $REDIS_URL`. Shares the workspace PVC, so it co-locates onto the same node as the Streamlit pods via the RWO mount. +RQ worker Deployment, 2 replicas by default. Runs `rq worker openms-workflows --url $REDIS_URL`. Shares the workspace PVC over NFS, so it can run on any node, including one no Streamlit pod is on. + +- `replicas: 2` — RQ takes one job per worker process, so this is exactly how many workflows the deployment can run at once. It was 1, which serialised every workflow in the deployment and was also the only thing keeping the queue path's concurrent-access bugs latent. `memory-tier-high` overrides it to 1, because a 180Gi worker fits on a high-memory node and nowhere else (see [Step 4b](#step-4b--choose-the-worker-size-and-how-many-workers)). +- `topologySpreadConstraints`, `maxSkew: 1` over `kubernetes.io/hostname`, `DoNotSchedule` — one worker per node before any node takes a second (see [Pod placement and the shared workspace volume](#pod-placement-and-the-shared-workspace-volume)). +- `strategy: RollingUpdate` with `maxSurge: 0`, `maxUnavailable: 1`, both explicit. The defaults resolve to `maxUnavailable: 0` at this replica count, which would make one `Pending` or `NotReady` worker enough to take the Deployment out of `Available`. +- Sized by the memory-tier component with `requests` equal to `limits`, which is what makes the pod Guaranteed QoS: `oom_score_adj` of −997 rather than the ~969 a 1Gi request against a 16Gi limit used to score. CI reads `.status.qosClass` off the **Running** worker pods (`assert_worker_qos_guaranteed`) — the class is assigned at admission and is populated on a `Pending` pod too, so a check that did not filter on phase would report a pass for workers that were never scheduled. +- Readiness probe only, never liveness: a liveness failure would SIGKILL the container mid-TOPP-job, and a restart cannot fix a wedged NFS mount. ### `cleanup-cronjob.yaml` CronJob that runs `python clean-up-workspaces.py` nightly at 03:00 UTC. Uses `concurrencyPolicy: Forbid`, retains 3 successful and 3 failed jobs. Shares the workspace PVC. @@ -162,12 +188,34 @@ Lists all base resources under the `openms` namespace. ### `streamlit-secrets.yaml` Ships with an empty admin password by default and is included in `k8s/base/kustomization.yaml`, so `kubectl apply -k` always creates the `streamlit-secrets` Secret. The Streamlit Deployment mounts it at `/app/admin-secrets/`, and `.streamlit/config.toml` registers that path under `[secrets].files` so `st.secrets` picks it up. The admin password gates the "Save as Demo" feature — when empty (default), that UI is hidden entirely; set a password to enable it. The volume mount keeps `optional: true` so forks that inject the Secret out-of-band (Vault, External Secrets Operator) or rename it still boot. See "Configuring the admin password" below. +## 3b. The storage tier (`k8s/storage/`) + +A **separate kustomize root**, applied on its own — it is not referenced from `k8s/overlays/prod/`, because `k8s/base` sets `namespace: openms` and kustomize's namespace transformer runs after patches, so objects pulled in from there would be dragged back into `openms`. + +| File | Purpose | +|------|---------| +| `namespace.yaml` | `template-app-storage`, at `pod-security.kubernetes.io/enforce=privileged`. Ganesha needs `DAC_READ_SEARCH` and `SYS_RESOURCE`; keeping it out of the shared `openms` namespace is what stops that exemption applying to every co-tenant. | +| `nfs-backing-pvc.yaml` | `nfs-server-data`, 500Gi `ReadWriteOnce` on `cinder-csi`. The single volume every workspace lives inside. A fresh volume, not a migration. | +| `ganesha-values.yaml` | Helm values for `nfs-server-provisioner` 1.8.0, pinned. Four settings there are not tunables — `persistence.existingClaim`, `storageClass.reclaimPolicy: Retain`, `replicaCount: 1` and `extraArgs.device-based-fsids` — and the file says why at length. | +| `networkpolicy.yaml` | Default-deny ingress, plus two holes on TCP 2049: one for pods labelled `app: template-app` in `openms`, one `ipBlock` for the cluster's nodes. **The node CIDR ships as a placeholder and must be set before the first deploy** — see Step 6. | +| `kustomization.yaml` | Ties the above together and inflates the chart. Carries the fork checklist for the hand-synced names. | + +Rendering this root needs Helm on `PATH`, because of the `helmCharts` block, and `kubectl apply -k` accepts no `--enable-helm` flag: + +```bash +kubectl kustomize --enable-helm k8s/storage/ # inspect +kubectl kustomize --enable-helm k8s/storage/ | kubectl apply -f - +``` + +Rendering also writes the vendored chart into `k8s/storage/charts/`. That directory is gitignored and must never be committed. + ## 4. Fork-and-deploy guide ### Prerequisites - `kubectl` configured for the target cluster -- A storage class that supports `ReadWriteOnce` volumes (de.NBI uses `cinder-csi`) +- A storage class that supports `ReadWriteOnce` volumes (de.NBI uses `cinder-csi`) — the storage tier turns one such volume into the `ReadWriteMany` class the app claims +- `mount.nfs` present on every node, since the provisioner emits in-tree `nfs:` PersistentVolumes that the kubelet mounts (`ls -l /sbin/mount.nfs*` on each node) - An ingress controller (Traefik, or nginx if you patch the nginx Ingress instead) - Read access to GHCR for pulling the app image - A DNS record pointing to the cluster's ingress load balancer @@ -200,18 +248,50 @@ Open `k8s/overlays/prod/kustomization.yaml` and change the following fields: The overlay leaves the nginx `Ingress` unpatched because Traefik is the production ingress. If you are deploying to an nginx-only cluster, add an overlay patch for both `rules[].host` entries in the base `Ingress` (same `.de` / `.org` pattern) instead of the IngressRoute patch. -### Step 4b — Select a memory tier +### Step 4b — Choose the worker size, and how many workers The overlay pulls in one of two Kustomize components under `components:`: ```yaml components: - - ../../components/memory-tier-low # default: light app on low-mem node + - ../../components/memory-tier-low # default: 16Gi / 4 cpu per worker # OR - - ../../components/memory-tier-high # memory-intensive app on high-mem node + - ../../components/memory-tier-high # 180Gi / 20 cpu per worker ``` -`memory-tier-low` is the right choice for most apps. Switch to `memory-tier-high` only if the workload genuinely needs tens of GB of RAM (DIA spectral-library + OpenSwath peak picking, DIA-LFQ). The tier component adds the matching `nodeSelector: openms.de/memory-tier=` plus `requests`/`limits` sized for that node, so cluster nodes must already be labelled `openms.de/memory-tier=low` / `...=high`. +A tier is a **pod size**, not a set of nodes. Each component patches the Streamlit and `rq-worker` `resources` and nothing else — no node label is involved, and none has to exist on the cluster. `memory-tier-low` is right for most apps; switch to `memory-tier-high` only if the workload genuinely needs tens of GB of RAM (DIA spectral-library + OpenSwath peak picking, DIA-LFQ). + +Both components set `requests` **equal to** `limits` on memory and cpu, which is the exact condition for Guaranteed QoS. Keep it that way when retuning. Only the request enters the Burstable `oom_score_adj` formula, so a worker asking for 1Gi against a 16Gi ceiling is schedulable on every node in the cluster and sits near the top of that node's kill list once it fills up — which is what the old sizing did. The cluster-side maximum is the `LimitRange` in `k8s/base/limitrange.yaml`, at 200Gi / 20 cpu per container — note the high tier's `cpu: 20` sits exactly *on* that ceiling, so raising it without raising the `LimitRange` first is rejected at admission as a quota violation rather than showing up as a `Pending` pod. + +How many workers is a separate decision, and the one that actually spends a second node. The base ships `replicas: 2`, one per node on the two-node OpenMS cluster, and `memory-tier-high` is the one place in `k8s/` that overrides it — down to 1, because a 180Gi Guaranteed request fits on a high-memory node and nowhere else. With `DoNotSchedule` forbidding the second replica from doubling up on the node that could hold it, an inherited `replicas: 2` there is a worker that is `Pending` for ever. A fork with N high-memory nodes raises it to N. Otherwise change the count in the overlay rather than the base, so a template rebase cannot revert it: + +```yaml +patches: + - target: + kind: Deployment + name: rq-worker + patch: | + - op: replace + path: /spec/replicas + value: 4 +``` + +`replicas x worker size` is reserved cluster-wide for as long as the workers run, queue empty or not, so raise it only when there is somewhere for the extra workers to fit. One that does not fit stays `Pending`. + +### Step 4c — Rename the storage tier for your fork + +Several of the storage tier's names are **cluster-scoped**, or would otherwise collide when two forks of this template share a cluster, and `namePrefix` cannot reach them (`persistence.existingClaim` is an opaque string kustomize does not rewrite). Change `template-app-` to your own slug in all of these, together: + +| Where | Field | +|-------|-------| +| `k8s/storage/kustomization.yaml` | `namespace:`, twice | +| `k8s/storage/namespace.yaml` | `metadata.name` | +| `k8s/storage/nfs-backing-pvc.yaml` | `metadata.namespace` | +| `k8s/storage/ganesha-values.yaml` | `fullnameOverride`, `storageClass.name`, `storageClass.provisionerName` | +| `k8s/base/workspace-pvc.yaml` | `storageClassName`, which must equal `storageClass.name` above | +| `k8s/storage/networkpolicy.yaml` | the `app: template-app` literal, which must equal `commonLabels.app` in your overlay | + +The last two are checked by CI: `assert_netpol_string_matches_overlay` compares the label, and the kind jobs fail the deploy if the workspaces PVC names a StorageClass the storage root does not publish. ### Step 5 — Configure the admin password (optional) @@ -242,10 +322,23 @@ kubectl -n openms rollout restart deployment/-streamlit ### Step 6 — Deploy +**Set the node CIDR first.** `k8s/storage/networkpolicy.yaml` ships `192.0.2.0/24` (RFC 5737 TEST-NET-1) as a placeholder, which admits nothing. The provisioner emits in-tree `nfs:` volumes, and the kubelet mounts those from the node's own address rather than from a pod IP, so without this every mount hangs on a CNI that enforces NetworkPolicy. Read the node addresses off the cluster and narrow the range as far as it will go: + +```bash +kubectl get nodes -o wide # the INTERNAL-IP column +``` + +Then apply the storage tier **before** the overlay. It publishes the StorageClass the workspaces PVC claims; applied in the other order, every pod sits `Pending` on a class that does not exist yet: + ```bash +kubectl kustomize --enable-helm k8s/storage/ | kubectl apply -f - +kubectl -n template-app-storage rollout status statefulset -l app=nfs-server --timeout=300s + kubectl apply -k k8s/overlays/prod/ ``` +The first of those needs Helm on `PATH`. Both applies are idempotent, and on an upgrade the storage one is usually a no-op. + ### Step 7 — Verify ```bash @@ -272,13 +365,18 @@ One unified workflow owns manifest lint, Docker build, push, and kind integratio - **Job 1 — `lint-manifests`:** - `kubeconform` runs against `k8s/base/*.yaml` with strict mode and Kubernetes 1.28 schemas (excluding `kustomization.yaml` and the Traefik CRD `traefik-ingressroute.yaml`). - `kubectl kustomize k8s/overlays/prod/` must succeed; the kustomized output is re-validated through `kubeconform` (with `IngressRoute` skipped). + - `kubectl kustomize k8s/overlays/ci/` — the overlay the kind jobs apply — must succeed and validate the same way. Because a `patches:` entry whose target matches nothing is not an error in kustomize, the step additionally checks that the rendered CI worker is actually *smaller* than prod's and still has `requests == limits`; otherwise a renamed patch target would hand the kind jobs a full-sized worker no runner can schedule, an hour before anything noticed. + - `kubectl kustomize --enable-helm k8s/storage/` must succeed and its output is validated the same way, so the Ganesha chart's own rendered objects are covered too. +- **Job 1b — `assert-invariants`:** static assertions from `.github/scripts/ci-assertions.sh` over the manifests — that the storage NetworkPolicy still admits exactly the overlay's `commonLabels.app`, that `device-based-fsids` is pinned off, and that nothing anywhere pins a pod to a node. It deliberately has no `needs:` and nothing needs it, so a red invariant cannot hide the build. - Takes ~30s. Fails fast so manifest typos never trigger the hours-long full Docker build. - **Job 2 — `build`** (`needs: lint-manifests`, matrix over `[full, simple]`): - Builds `Dockerfile` (full, includes TOPP tools) or `Dockerfile_simple` (pyOpenMS only) depending on the matrix leg. - **Buildx registry cache** (`type=registry,…,mode=max`) stored at `ghcr.io//cache:full` and `:simple`. A `cache-from` read is attempted on every event; `cache-to` write only on push/tag/workflow_dispatch (fork PRs can't write). Repeat builds with an unchanged Dockerfile finish in minutes. - **Push** on push/tag/workflow_dispatch events (not on PRs). Tags: `-full` / `-simple`, `v-full` / `v-simple`, `-full` / `-simple`. `latest` is emitted only for the full variant on push to `main`. - - **Kind integration** runs per variant: creates a kind cluster, loads the just-built image, installs the nginx ingress controller, applies the kustomized `prod` overlay (filtering Traefik `IngressRoute`, forcing `imagePullPolicy: Never` and `storageClassName: standard`), asserts Redis + deployments become ready, and curls both `.de` and `.org` hostnames through the nginx ingress to verify dual-host routing. -- **Job 3 — `traefik-integration`** (`needs: lint-manifests`, runs once on `Dockerfile_simple`): builds the simple image, brings up a second kind cluster, installs Traefik via Helm (`service.type=ClusterIP`), applies the full kustomized overlay without filtering the `IngressRoute` (still patching `imagePullPolicy: Never` and `storageClassName: standard` for kind compatibility), and curls both hostnames through Traefik. Catches IngressRoute-syntax regressions that the nginx-side test cannot. + - **Kind integration** runs per variant: creates a two-node kind cluster, loads the just-built image, installs the nginx ingress controller, applies the storage root (rewriting the backing claim's `cinder-csi` to kind's `standard`), then applies the kustomized **`ci`** overlay (filtering Traefik `IngressRoute`, forcing `imagePullPolicy: Never`, and shrinking the workspaces claim to fit the runner's disk). It asserts Redis and every Deployment become ready, curls both `.de` and `.org` hostnames through the nginx ingress, and then runs the assertions from `.github/scripts/ci-assertions.sh`: that every worker replica is Running on more than one node within the declared `maxSkew` (`assert_workers_spread_across_nodes`), that the Running workers are `Guaranteed` QoS (`assert_worker_qos_guaranteed`), cross-node write visibility, the POSIX contract, a workflow surviving a Ganesha restart, stable file-handle identity across that restart, and two pods on two nodes sharing the volume. The two placement assertions run first because they take seconds while the storage ones budget up to 45 minutes of deliberate timeouts. + +The kind jobs apply `k8s/overlays/ci/`, which is `k8s/overlays/prod/` with the worker `resources` patched down and nothing else changed. Production sizes a worker at 16Gi / 4 cpu with requests equal to limits; a GitHub runner has 4 vCPU and ~15.6Gi in total, every kind node is a container on that one host advertising the whole of it, and both replicas would sit `Pending` for ever. Same replica count, same spread constraint, same labels, same images — and every static assertion still renders `k8s/overlays/prod/`, so the sizing is the only thing CI does not observe directly. +- **Job 3 — `traefik-integration`** (`needs: lint-manifests`, runs once on `Dockerfile_simple`): builds the simple image, brings up a second kind cluster, installs Traefik via Helm (`service.type=ClusterIP`), applies the storage root and then the full kustomized `ci` overlay without filtering the `IngressRoute` (still patching `imagePullPolicy: Never` and the claim size for kind compatibility), and curls both hostnames through Traefik. Catches IngressRoute-syntax regressions that the nginx-side test cannot. It runs the same placement and storage assertions as the nginx job. - **Auth:** uses the workflow's `GITHUB_TOKEN` for GHCR login and as a build argument for in-image private-resource access. Fork PRs skip login (their `GITHUB_TOKEN` is read-only) but can still read the public cache. - **PR behavior:** all three jobs run on pull requests. No tags are pushed and no cache is written. The kind integration still runs, exercising manifests end-to-end. If branch protection requires these checks, a failure blocks merge. diff --git a/docs/storage-defect-register.md b/docs/storage-defect-register.md new file mode 100644 index 00000000..176a4810 --- /dev/null +++ b/docs/storage-defect-register.md @@ -0,0 +1,335 @@ +> **What this is.** The defect register compiled while making this template run +> across multiple nodes: verified defects in the workflow framework, the +> Kubernetes manifests and the CI, each with the evidence that established it. +> Source comments and tests cite these IDs (`A2`, `G1`, `D2`, …) to explain what +> a guard is defending against. +> +> **Not all of these are fixed.** It is a register of what was found, not a list +> of what was resolved, and it records the state at the time of that work. Check +> the code before assuming an entry is still live. + +# Verified defect register + +Found while researching node-distributed deployment, 2026-08-17/18. +**Every item here is independent of the distribution question** — these are bugs +in `main` today. Several are made dramatically worse by adding a second node, +which is noted where it applies. + +Provenance is stated per entry: + +- **[repo]** — verified by reading this repository directly during the session. +- **[agent]** — verified by a research agent against external source + (CPython, Kubernetes, RQ, Streamlit) and not independently re-checked here. +- **[agent+repo]** — agent finding, confirmed against this repository. + +--- + +## A. Silent data loss + +### A1. `@st.cache_data` memoises a side-effecting write **[agent+repo]** +`src/fileupload.py:9` + +`save_uploaded_mzML` is decorated with `@st.cache_data`, but its whole purpose +is the side effect of writing files. The destination is read from +`st.session_state.workspace` *inside* the body, so it is **not part of the cache +key**; Streamlit keys `UploadedFile` on name + content. + +Upload a file to workspace A, then upload the byte-identical, same-named file to +workspace B: cache hit, body never executes, **the file is never written to +workspace B**, and the cached `st.success("Successfully added uploaded files!")` +is replayed so the user is told it worked. + +Live today. `replicas: 2` makes it non-deterministic (per-replica cache), not +better. Fix: drop the decorator, or key on the workspace explicitly. + +### A2. A torn `params.json` read permanently deletes parameters **[agent+repo]** +`ParameterManager.py:136`, `:207-212`, `:190-191`, `:377-378` + +`save_parameters` is a read-modify-write: + +- `:136` — `json_params = self.get_parameters_from_json() | json_params` +- `:207-212` — `get_parameters_from_json` wraps the read in a bare `except:` + that returns `{}` after showing *"Attempting to load an invalid JSON parameter + file. Reset to defaults."* +- `:190-191` / `:377-378` — the write is `open(path, "w")` + `json.dump`, + truncate-then-rewrite with no temp-file + `os.replace`. + +So a single transient torn read makes the merge `{} | this_session_params`, and +that is written back — **permanently erasing `_defaults`, `_flag_params`, and +every other tool's saved values**, while telling the user it was a deliberate +reset. + +`os.replace` alone does **not** fix this. It prevents third parties observing a +torn file; it does nothing about lost update or the empty-dict laundering. The +lock is the load-bearing half. Reported reproducible on current `main` in ~15 +minutes with two processes. + +--- + +## B. Failures reported as successes + +### B1. Queue mode can never report a failed workflow **[repo]** +`src/workflow/tasks.py:108`, `:143-148` + +`:108` logs the `WORKFLOW FINISHED` marker unconditionally, without inspecting +what `execution()` returned. `:143-148` **returns a dict** from the exception +handler instead of re-raising. + +RQ therefore records every job as finished successfully, including crashed ones. +Consequences: `health.py`'s `failed_jobs` metric is structurally always zero; +`Retry` would be inert for application failures; no queue-mode UI can show a +failure. + +### B2. The success signal is inverted in the other direction too **[repo]** +`src/Workflow.py:54` + +Annotated `-> None` and returns nothing, so `workflow_process()` never logs the +marker and a **successful** local run renders "Errors occurred, check log file." +Documented in CLAUDE.md; combined with B1 the signal is wrong in both modes, in +opposite directions. + +### B3. An ImportError renders "Workflow completed successfully" **[agent+repo]** +`src/workflow/tasks.py:46-55` vs `:60` + +The module imports and `importlib.import_module` run **before** the `logs/` +rmtree at `:60`. An ImportError therefore appends `ERROR:` to the *previous* +run's log, and `classify_log_outcome` substring-searches the whole file — so +`StreamlitUI.py:1569-1571` reports success for a workflow that never started. +The `st_ctime` displayed at `:1564` is inode change time, so the stale log even +looks fresh. + +Fix: slice the log from the last `STARTING WORKFLOW`. ~5 lines; +`tests/test_log_status.py` already exists. + +### B4. A killed pod shows "running" forever **[repo]** +`StreamlitUI.py:1509-1513` + +```python +pid_exists = self.executor.pid_dir.exists() and list(self.executor.pid_dir.iterdir()) +if not is_running and pid_exists: + is_running = True +``` + +This does not merely read `pids/` as a hint — it **overrides an authoritative +"not running" from the queue**. A SIGKILLed pod never reaches +`CommandExecutor.py:150` to unlink its PID file, so the UI shows +"Workflow running..." on a 1s rerun loop indefinitely, after RQ has already +finished or failed the job. + +--- + +## C. Log loss + +### C1. Tool output is truncated by a race **[agent+repo]** +`CommandExecutor.py:185-186`, `:201-202` + +Both reader threads `break` as soon as `process.poll() is not None`, discarding +whatever is still buffered in the pipe. + +Reproduced with the repository's exact reader code and latency injected in place +of `Logger.log`: + +| per-line reader latency | buggy | fixed | +|---|---|---| +| 0 ms | 1000/1000 | 1000/1000 | +| 0.5 ms | **3**/1000 | 1000/1000 | +| 2.0 ms | **1**/1000 | 1000/1000 | + +Two consequences. It is a **race**, so a naive regression test passes against the +buggy code — latency injection is load-bearing in any test. And because +`Logger.log` does open/write/close per line, a network filesystem puts it +squarely in the 0.5–2 ms band: **distribution converts a latent race into ~99.9% +silent log loss.** + +Correction to an earlier belief: this does **not** discard the +`WORKFLOW FINISHED` / `WORKFLOW CANCELLED` markers. Those are written directly +by `Logger` from the Python process and never traverse the subprocess pipe. +What is lost is the tool's own output — the diagnostic you need *after* the run +has already been marked failed. + +The correct idiom already exists in this repo at `src/run_subprocess.py:29` +(`output == "" and process.poll() is not None`), which makes the +`CommandExecutor` version look like an editing accident. + +### C2. `bufsize=1 # Line buffered` is a no-op **[agent]** +`CommandExecutor.py:123` + +`Popen` passes `line_buffering` only to the **stdin** wrapper; stdout/stderr get +`line_buffering=False` and `bufsize` normalises to `-1`. + +### C3. `src/run_subprocess.py` can deadlock **[agent]** +Drains stdout fully, then stderr — so a child filling the 64 KiB stderr pipe +buffer blocks forever. + +--- + +## D. Kubernetes manifests + +### D1. Nightly workspace cleanup can die permanently and silently **[repo]** +`k8s/base/cleanup-cronjob.yaml` + +The CronJob mounts the RWO PVC but carries **no placement constraint** — the +memory-tier component patches `kind: Deployment`, and a CronJob is not one. It +can therefore be scheduled on the node where the Cinder volume is not attached +and hang on Multi-Attach. With `concurrencyPolicy: Forbid`, one hung job blocks +**every** subsequent cleanup, with no error surface. + +Check: `kubectl get cronjob,jobs -A | grep -i cleanup` + +### D2. The memory-tier nodeSelector is unscoped **[repo]** +`k8s/components/memory-tier-{low,high}/nodeselector.yaml` + +Patches `target: {kind: Deployment}` with **no name**, so +`openms.de/memory-tier` lands on `streamlit`, `rq-worker` *and* `redis`. + +This is why the second node is unusable — and it means an RWX storage migration +would **silently do nothing**: everything would stay pinned, and the migration +would look like it succeeded. + +### D3. `rq-worker` has no probes, no grace period, no PDB **[repo]** +`k8s/base/rq-worker-deployment.yaml` + +Zero probes (streamlit and redis have two each), so a worker wedged on a dead +mount is invisible indefinitely. No `terminationGracePeriodSeconds`, so the 30s +default kills an hour-long job 30s into a drain. No PodDisruptionBudget. + +### D4. `k8s/base/ingress.yaml` is a dead dependency **[agent]** +kubernetes/ingress-nginx was archived read-only on 2026-03-24 — no further +security patches; the mooted successor was abandoned. + +### D5. Nothing reaps zombies **[agent]** +RQ's `wait_for_horse` calls `os.wait4(self.horse_pid, 0)` — a specific pid, +never `waitpid(-1)`. The manifest runs `exec rq worker`, so PID 1 is the worker. +`kill_horse` uses SIGKILL, so this fires on every cancellation. + +Manifest-only fix: `shareProcessNamespace: true` makes the pause container PID 1, +which reaps with `waitpid(-1, NULL, WNOHANG)`. No image rebuild. + +### D6. `docs/kubernetes-deployment.md:81` is wrong **[agent]** +Asserts the VolumeBinding plugin pins pods to the node holding the attached +volume. It does not — `volume_binding.go` checks PV *node affinity* only. The +scheduler will place a pod on the wrong node; you get a Multi-Attach hang at +attach time, not a scheduling refusal. + +### D7. The 7-day GC can delete workspaces that are actively in use **[repo]** +`clean-up-workspaces.py:15`, `:32` + +Two things, found during the A16 design interview. + +First, retention is **7 days**, not the two weeks it is often described as: +`threshold = current_time - (86400 * 7)`, hardcoded, with no env var or +settings key. + +Second, and worse: line 32 uses `os.path.getmtime(directory)` on the *top-level +workspace directory*. A directory's mtime changes only when an entry directly +inside it changes. Once `mzML-files/` and the workflow directory exist, nothing +a user does refreshes it — writing `topp-workflow/params.json` touches the +workflow directory's mtime, not the workspace's, and `page_setup()`'s +`mkdir(parents=True, exist_ok=True)` does not update mtime on an existing +directory. + +So a user working in the same workspace for eight days has it deleted underneath +them on day seven, mid-session, while actively using it. + +Fix: walk for the newest mtime beneath the workspace rather than reading the top +directory's own, or touch the workspace directory in `page_setup()`. The walk is +more correct; the touch is one line. + +--- + +## E. Concurrency, smaller + +All **[agent+repo]** unless noted. + +- `WorkflowManager.py:94` — `pid_dir.mkdir()` without `exist_ok`, and it runs + *after* the child is spawned at `:91`. +- `Logger.py:26-28` — exists-then-mkdir TOCTOU. +- `CommandExecutor.py:413` — writes python-tool params to a **fixed** path in the + workflow dir, unlinked at `:419`. Two concurrent runs collide. +- `tasks.py:96-97` — the `results/` rmtree has **no** `ignore_errors` (unlike + `logs/` at `:60`) and raises into the generic handler; `:97` mkdirs without + `exist_ok`. +- `WorkflowManager.py:64` — `job_id` is built from the workflow **slug** plus + `int(time.time())`, not the workspace. Two users, same workflow, same second + collide — and RQ without `unique=True` does no collision detection, silently + overwriting existing job data for a reused id. +- seed-demos `cp -rn` under `replicas: 2` — has a silent branch (cp writes in + place with no temp+rename, so a pod can skip a file another is still writing, + permanently) and a fatal one (`EEXIST` is a hard exit 1, and `-n` never applies + to directories) → `Init:CrashLoopBackOff` that never self-resolves. Fix is + copy-to-temp plus `mv -T`, not a lock file. + +--- + +## G. Configuration silently ignored, and unvalidated input + +### G1. `max_threads.online` has been dead code for 6.5 months **[agent]** +`CommandExecutor.py:39`, regression from `eb9c205` (PR #333, 2026-01-31) + +At the streamlit 1.49.1 tag, `get_session_state()` returns a lazily-created +**empty global mock** when there is no `ScriptRunContext`, rather than raising. +Inside the RQ work horse there is no context, so +`st.session_state.get("settings", {})` is `{}`, `online_deployment` reads +`False`, and the **local** branch runs. The effective budget is +`params.json["max_threads"]` or the hardcoded literal `4` — never +`max_threads.online: 2`. + +PR #333's final step moved the helper out of `common.py`, where session state +did exist. Live since 2026-01-31; quantms-web is affected identically. + +### G2. Any anonymous session can set the shared worker's thread budget **[agent]** +The params-import uploader writes uploaded JSON verbatim — **no reserved-key +filter and no clamp**. `max_threads` is a reserved key per CLAUDE.md, and +nothing enforces that. On a shared online deployment an anonymous visitor can +therefore set the worker's thread budget to an arbitrary value. + +### G3. The 16Gi memory limit buys no OOM protection **[agent]** +`k8s/components/memory-tier-low/worker-resources.yaml` + +Burstable `oom_score_adj = 1000 - 1000 * memRequest / capacity`, so the 1Gi +request scores ~969 against BestEffort's 1000. **Only the request affects OOM +ranking; the limit does not.** Compounding it, `MSGFPlusAdapter` defaults +`java_memory` to 3500 MB *per process* divided by nothing — four concurrent +adapters ask 14 GB against that 16Gi limit. + +Note the standard advice, "just delete the CPU limit", is a trap here: every +auto-detecting runtime reads the *limit* and falls back to the **node**, never +the request. + +--- + +## F. Configuration and CI + +- `requirements.txt:142-143` — `rq>=1.16.0` and `redis>=5.0.0` are **unbounded**, + so rebuilds install RQ 2.11 today. Features are present but not guaranteed; + the unbounded floor is itself a supply-chain risk. **[repo]** +- `.github/workflows/build-and-test.yml:593`, `:733` — the + `sed 's|storageClassName: cinder-csi|standard|g'` silently no-ops the moment + the PVC stops naming `cinder-csi`, so kind CI would quietly stop exercising the + storage path. **[repo]** +- `.streamlit/config.toml:14` — `maxUploadSize = 200` MB, against routinely + larger mzML. **[repo]** +- CI runs `kubeconform -kubernetes-version 1.28.0`, which will reject + `matchLabelKeys` (added in 1.29) if anti-affinity work adopts it. **[agent]** + +--- + +## Suggested order + +The dependency that matters most: **`replicas: 1` currently serialises the queue, +and that is the only reason A2 and the E-group races are not firing.** A second +worker makes them reachable for the first time. + +1. **B1, B2, B3** — make failures visible. Everything else is unverifiable until + a failed run reports as failed. Small, no dependencies. +2. **A1, A2** — the two silent data-loss bugs. A2 needs the lock, not just + atomic writes. +3. **C1** — delete the `poll()` break. Note it increases log volume, which makes + the per-second whole-file re-read at `StreamlitUI.py:1546` worse, so pair it + with a decision about log transport. +4. **D1, D2, D3, G1, G3** — manifest and config; no algorithmic risk. + Note the CPU oversubscription originally suspected here is largely absent: + `TOPPBase` registers `-threads` with default 1 and calls + `omp_set_num_threads()` before `main_()`, so it is a real hard cap. Memory, + not CPU, is the binding resource. +5. Only then anything from the distribution work itself. diff --git a/k8s/base/cleanup-cronjob.yaml b/k8s/base/cleanup-cronjob.yaml index 86481876..ba65f325 100644 --- a/k8s/base/cleanup-cronjob.yaml +++ b/k8s/base/cleanup-cronjob.yaml @@ -11,6 +11,22 @@ spec: failedJobsHistoryLimit: 3 jobTemplate: spec: + # concurrencyPolicy above is Forbid, so an Active Job suppresses every + # later schedule. Without a deadline, one run wedged on the shared + # workspaces mount - an NFS server restart is routine - silently blocks + # cleanup forever, with no error surface. The deadline forces the Job to + # Failed, which is terminal, so the next night runs. + # Generous: the cleanup walks every workspace over the network. + # + # Belt and braces with the in-container `timeout` below, because this + # deadline alone does not close the hole it names. It is a control-plane + # bound applied to a syscall-level hang: a pod stuck in D-state on a + # `hard` mount cannot terminate, and on recent Kubernetes the Job's + # terminal Failed condition is only added once every pod has actually + # terminated (FailureTarget first), so Forbid can still suppress the next + # schedule. The `timeout` inside the container covers the + # reachable-but-slow cases without depending on the cluster version. + activeDeadlineSeconds: 3600 template: metadata: labels: @@ -24,8 +40,11 @@ spec: command: ["/bin/bash", "-c"] args: - | + set -euo pipefail source /root/miniforge3/bin/activate streamlit-env - exec python clean-up-workspaces.py + # Inside activeDeadlineSeconds (3600), so the Job still fails + # if this somehow outlives its own bound. + exec timeout -k 60 3000 python clean-up-workspaces.py env: - name: WORKSPACES_DIR value: "/workspaces-streamlit-template" @@ -42,4 +61,4 @@ spec: volumes: - name: workspaces persistentVolumeClaim: - claimName: workspaces-pvc + claimName: workspaces-nfs-pvc diff --git a/k8s/base/namespace.yaml b/k8s/base/namespace.yaml index 20842f63..d7a0d6d8 100644 --- a/k8s/base/namespace.yaml +++ b/k8s/base/namespace.yaml @@ -4,3 +4,33 @@ metadata: name: openms labels: app.kubernetes.io/part-of: openms-streamlit + # Pod Security: `warn` and `audit`, deliberately NOT `enforce`. + # + # This namespace is shared. On the de.NBI cluster `openms` also holds + # `app=nuxl-app`, and k8s/storage/namespace.yaml uses exactly that fact to + # justify not granting the Ganesha exemption here. The argument runs in + # reverse too: an `enforce` label is a namespace-wide admission rule, so + # stamping one from this repo's overlay would police every pod of every + # co-tenant. A NuXL pod using hostPath, a privileged container or an added + # capability would start failing admission on its next rollout, with + # nothing in NuXL's own manifests having changed - and the failure would + # look like a NuXL problem. + # + # `warn` returns the same verdict to whoever runs `kubectl apply`, and + # `audit` records it in the API server audit log, without blocking anyone. + # That is the dry run this app can impose unilaterally. Turning it into + # `enforce` is a cluster-owner decision, taken once every co-tenant's pods + # are known clean - the audit log is the evidence for it. + # + # This app's own pods have nothing to lose either way: no securityContext, + # no hostPath, no host namespaces and no added capabilities anywhere in + # k8s/base or the overlay. The storage tier, which does need more, lives in + # `template-app-storage` (k8s/storage/namespace.yaml) for that reason. + # + # The `-version` pins matter as much as the levels. Left off, the policy + # tracks `latest` and silently tightens under a cluster upgrade; pinned, a + # newer baseline is adopted by editing this file. + pod-security.kubernetes.io/warn: baseline + pod-security.kubernetes.io/warn-version: v1.28 + pod-security.kubernetes.io/audit: baseline + pod-security.kubernetes.io/audit-version: v1.28 diff --git a/k8s/base/rq-worker-deployment.yaml b/k8s/base/rq-worker-deployment.yaml index 021add38..7687e9b1 100644 --- a/k8s/base/rq-worker-deployment.yaml +++ b/k8s/base/rq-worker-deployment.yaml @@ -5,7 +5,60 @@ metadata: labels: component: rq-worker spec: - replicas: 1 + # Two workers, not one. RQ runs one job per worker process, so `replicas: 1` + # serialised every workflow in a deployment however much hardware sat + # underneath it - and on the de.NBI pair the second node could not have been + # used anyway, because the memory-tier component pinned every Deployment to + # one labelled node and the workspace claim was ReadWriteOnce on a Cinder + # volume that attaches to exactly one node. Both of those are gone. This is + # the line that actually spends the second node. + # + # Fixed rather than autoscaled: each worker is Guaranteed QoS at whichever + # size k8s/components/memory-tier-*/worker-resources.yaml gives it, so N + # replicas reserve N times that size for as long as they run. The count is a + # capacity decision against real nodes - see the worker-size and + # worker-replica questions in .claude/skills/configure-k8s-deployment.md. + # + # 2 is one per node on the production pair, which is the point at which the + # spread constraint below has anything to do. A higher count is legal + # (maxSkew 1 permits 2/1 across two nodes) but only pays off where a node can + # hold two workers of this size at once. It is also the first configuration + # in which two workflow runs are genuinely concurrent: everything they share + # goes through the workspace volume, so a fork that adds shared mutable state + # outside it has no serialisation left to hide behind. + replicas: 2 + # Explicit, because the default is wrong here in a way that only shows up + # once replicas > 1. RollingUpdate defaults to 25%/25%, which against 2 + # replicas rounds to maxSurge 1 and maxUnavailable **0** (maxUnavailable + # rounds down, and the 0/0 fallback does not apply because surge is 1). A + # Deployment's Available condition needs `replicas - maxUnavailable` ready, + # so with 0 the whole Deployment goes non-Available the moment either worker + # is Pending or NotReady - and both of those are routine here rather than + # exceptional: + # - whenUnsatisfiable: DoNotSchedule below parks a replica in Pending + # during any node drain, since a cordoned node still counts as a domain + # under the default nodeTaintsPolicy; + # - the readinessProbe below fails every worker whose mount is blocked + # during a routine Ganesha restart. + # Either would then hang `kubectl rollout status deployment/-rq-worker` + # and fail CI's `kubectl wait --for=condition=available`, turning a + # degradation that the design calls survivable into a blocked rollout. + # + # maxUnavailable 1: one worker down is a queue running at half rate, which is + # what "visible and recoverable" has to mean if it is to mean anything. That + # the OTHER replica is up is not weakened by this - it is asserted directly, + # against .spec.replicas, by assert_workers_spread_across_nodes. + # + # maxSurge 0: a surge replica needs a whole extra worker's worth of memory + # free somewhere, and these are Guaranteed pods sized to most of a node + # (180Gi in the high tier). Surging first would leave the new pod Pending on + # a cluster that has exactly enough room for the steady-state count, which is + # the normal case. Terminate-then-replace instead. + strategy: + type: RollingUpdate + rollingUpdate: + maxSurge: 0 + maxUnavailable: 1 selector: matchLabels: component: rq-worker @@ -14,6 +67,50 @@ spec: labels: component: rq-worker spec: + # Spread the workers over nodes: one each until every node has one. + # + # maxSkew is measured over ELIGIBLE domains, which is why this had to + # wait for the nodeSelector deletion rather than shipping alongside it. + # With a selector matching exactly one node there is one eligible domain, + # every distribution has skew 0, and a constraint written here would have + # passed while changing nothing at all. + # + # DoNotSchedule, not ScheduleAnyway: soft spreading is a scoring + # preference the scheduler can trade away against other priorities, and a + # worker that usually spreads is a second node that is used right up + # until the day it matters. + # + # The cost is a Pending replica during node maintenance: with the default + # nodeTaintsPolicy a cordoned node still counts as a domain, so a drain + # parks the evicted worker in Pending rather than doubling it up on the + # survivor. That is visible in `kubectl get pods` and it clears when the + # node comes back - but only because `strategy.maxUnavailable` above is + # 1. At the default of 0 for this replica count the same Pending replica + # would take the Deployment out of Available and hang every subsequent + # `kubectl rollout status`, which is a blocked rollout, not a + # degradation. The two settings are a pair; changing one without the + # other is what makes this bite. Add `nodeTaintsPolicy: Honor` here + # (Kubernetes >= 1.27) if node maintenance becomes routine enough that + # even the Pending replica is unwelcome. + # + # The selector names `component` alone. The overlay's + # `commonLabels: {app: }` is copied into it by kustomize, whose + # commonLabels field specs reach + # topologySpreadConstraints/labelSelector/matchLabels with create: false + # (verified against the kustomize 5.7.1 built into kubectl). That scoping + # is load-bearing: every fork of this template deploys into the same + # `openms` namespace and topology spread counts pods per namespace, so + # without the app label one fork's workers would be counted against + # another's skew. A fork that drops commonLabels from its overlay loses + # that scoping, and the constraint starts balancing every rq-worker in + # the namespace as though they were all its own. + topologySpreadConstraints: + - maxSkew: 1 + topologyKey: kubernetes.io/hostname + whenUnsatisfiable: DoNotSchedule + labelSelector: + matchLabels: + component: rq-worker containers: - name: rq-worker image: openms-streamlit @@ -30,6 +127,93 @@ spec: value: "redis://redis:6379/0" - name: WORKSPACES_DIR value: "/workspaces-streamlit-template" + # Downward API. The storage heartbeat is keyed per node + # (storage:ok:) and the node registration alongside it + # (storage:node:) is what makes a *missing* heartbeat + # readable, so a healthy node cannot mask a node whose mount has + # wedged - with one shared key the indicator would stay green + # straight through a real outage. Without this env var + # health.current_node_name() falls back to the pod hostname, which + # changes on every restart and keys the heartbeat per pod instead + # of per node. + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + # Readiness only, and deliberately no liveness counterpart: a + # liveness failure would SIGKILL the container mid-TOPP-job, and a + # restart cannot fix a wedged NFS mount - it would destroy hours of + # work while the fault persists. rq-worker sits behind no Service, so + # readiness has no traffic effect either way; it is purely an + # alertable signal, and it doubles as the writer of the sidebar's + # storage heartbeat (src/workflow/health.py). + # + # One process, not a shell pipeline, because the ordering inside it + # is the contract and belongs somewhere a test can drive it + # (tests/test_storage_health.py). `probe_storage()` registers the + # node, *then* writes to the volume, and publishes the heartbeat only + # if that write succeeded. Nothing here is wrapped in `|| true`: a + # `|| true` on the storage check would report Ready on a wedged mount + # and publish a green heartbeat with it, which is the whole failure + # this probe exists to catch. + # + # A write, not a `stat`: with default `actimeo` a stat can be answered + # from the client's attribute cache while the server is gone, which + # is a false green. A write always reaches the server, and it creates + # the sentinel on a fresh volume instead of failing forever on a file + # nothing else makes. The file is dot-named and not a directory, so + # clean-up-workspaces.py skips it twice over. + # + # The interpreter is named explicitly rather than activated: `conda + # activate` costs a few hundred ms of shell hooks on every probe, and + # this is the same env the container command activates one screen up. + # + # Timing, against the ~90s NFSv4 grace period a Ganesha restart + # imposes - all I/O blocks until it ends, so the numbers have to + # outlast it or every routine restart flaps the worker: + # inner timeout 15s - a cold interpreter plus `import redis` is + # ~200ms idle, but this pod is by design + # CPU-saturated running TOPP tools under a CFS + # quota, and the two Redis calls are bounded at + # health.REDIS_SOCKET_TIMEOUT (2s) each. The + # budget has to cover all of that *and* leave + # room for a slow-but-alive mount, or the probe + # reports a storage fault that is really a busy + # node. + # timeoutSeconds 20 - kubelet's own bound, above the inner one so + # that a reachable-but-slow mount produces a + # clean exit 1 from `timeout`. On a genuinely + # wedged `hard` mount it is kubelet that ends + # the attempt, not `timeout`: the blocked write + # sits in uninterruptible sleep, TERM and KILL + # are both ignored, and `timeout` then blocks in + # wait() for a child that cannot die. The probe + # still fails, and the process stays until the + # mount recovers. + # periodSeconds 30 - one attempt per 30s. Those unkillable + # processes accumulate one per period during an + # outage, so the cadence bounds the pile. + # failureThreshold 5 - 4 x 30s + up to 20s = ~140s of continuous + # failure before NotReady, ~50s clear of the + # grace period. + # health.STORAGE_HEARTBEAT_TTL (120s) is 4 x periodSeconds, so the + # sidebar indicator goes red after three missed refreshes - still + # before the worker is marked NotReady. Degradation should be visible, + # not fatal. + readinessProbe: + exec: + command: + - /bin/bash + - -c + - | + set -u + py=/root/miniforge3/envs/streamlit-env/bin/python + exec timeout -k 3 15 "$py" -m src.workflow.health --probe + initialDelaySeconds: 15 + periodSeconds: 30 + timeoutSeconds: 20 + failureThreshold: 5 + successThreshold: 1 volumeMounts: - name: workspaces mountPath: /workspaces-streamlit-template @@ -40,7 +224,7 @@ spec: volumes: - name: workspaces persistentVolumeClaim: - claimName: workspaces-pvc + claimName: workspaces-nfs-pvc - name: config configMap: name: streamlit-config diff --git a/k8s/base/streamlit-deployment.yaml b/k8s/base/streamlit-deployment.yaml index 121d438e..5ef690cd 100644 --- a/k8s/base/streamlit-deployment.yaml +++ b/k8s/base/streamlit-deployment.yaml @@ -15,15 +15,27 @@ spec: component: streamlit spec: initContainers: + # The seeding logic lives in docker/seed-demos.sh, not inline here, so + # that tests/test_seed_demos.py executes the script the image ships. + # It stages the copy in a private directory and renames it into place - + # rename(2) on a directory is atomic and fails on a non-empty target, + # so with replicas: 2 the winner renames and the loser stands down. + # + # The script bounds itself with `timeout` and always exits 0: this + # container gates pod start on the shared workspaces mount, and a pod + # held in Init by a slow or restarting NFS server would defeat the + # point of Streamlit being able to serve through a storage outage. - name: seed-demos image: openms-streamlit imagePullPolicy: Always - command: ["/bin/sh", "-c"] + command: ["/bin/sh", "/app/docker/seed-demos.sh"] args: - - | - set -eu - mkdir -p /workspaces-streamlit-template/.demos - cp -rn /app/example-data/workspaces/. /workspaces-streamlit-template/.demos/ + - /app/example-data/workspaces + - /workspaces-streamlit-template/.demos + env: + # Wall clock for the whole seeding run, copy and stat calls alike. + - name: SEED_TIMEOUT + value: "300" volumeMounts: - name: workspaces mountPath: /workspaces-streamlit-template @@ -70,7 +82,7 @@ spec: volumes: - name: workspaces persistentVolumeClaim: - claimName: workspaces-pvc + claimName: workspaces-nfs-pvc - name: config configMap: name: streamlit-config diff --git a/k8s/base/workspace-pvc.yaml b/k8s/base/workspace-pvc.yaml index b3613beb..a9d6bac2 100644 --- a/k8s/base/workspace-pvc.yaml +++ b/k8s/base/workspace-pvc.yaml @@ -1,11 +1,96 @@ +# The shared workspace volume, served over NFS by the Ganesha server in +# k8s/storage/. +# +# --------------------------------------------------------------------------- +# This is a NEW claim, not an edit of the old one. That is deliberate. +# --------------------------------------------------------------------------- +# The previous claim was named `workspaces-pvc` and is `ReadWriteOnce` on +# `cinder-csi`. A PVC's spec is immutable once it is Bound - everything except +# `resources.requests` - so changing `accessModes` and `storageClassName` on +# that object is rejected by the API server: +# +# spec: Forbidden: spec is immutable after creation except +# resources.requests for bound claims +# +# `kubectl apply -k k8s/overlays/prod/` would fail half way through, and the +# only way past it is deleting the claim - which, on a `cinder-csi` class that +# reclaims with `Delete`, destroys the original volume. That volume is the +# rollback: docs/a16-storage-runbook.md section 5 re-applies the previous manifests and +# expects `-workspaces-pvc` still to be there, Bound, holding the data. +# +# So the RWX claim gets a new name and the workload manifests are repointed at +# it (docs/a16-storage-runbook.md section 3, steps 6 and 8). The old claim is +# left alone: it +# stays Bound to its Cinder volume, costs nothing but disk, and rollback is one +# `kubectl apply` of the previous manifests with no restore step. Delete it by +# hand once the cutover has been stable for a week - never as part of a deploy. +# +# The mount path does not move. Every consumer reaches this volume through +# `claimName` plus `/workspaces-streamlit-template`, so nothing in `src/` +# changes and absolute symlinks and `external_files.txt` survive byte for byte. apiVersion: v1 kind: PersistentVolumeClaim metadata: - name: workspaces-pvc + name: workspaces-nfs-pvc spec: + # ReadWriteMany, so the scheduler is free to place streamlit, rq-worker and + # the cleanup CronJob wherever they fit. accessModes: - - ReadWriteOnce - storageClassName: cinder-csi + - ReadWriteMany + # Served by the NFS-Ganesha server in k8s/storage/, which re-exports a + # single Cinder volume over NFSv4.1. This name must equal storageClass.name + # in the chart values there; the "Verify the workspace StorageClass contract" + # step in .github/workflows/build-and-test.yml fails the deploy - and prints + # the classes that do exist - if the two ever drift. + # + # It carries the app prefix because a StorageClass is cluster scoped. Two + # forks of this template on one cluster (the de.NBI cluster already hosts + # more than one OpenMS webapp) would otherwise fight over a class called + # plain `nfs`, and whichever installed second would silently bind its + # workspaces to the other app's export. A fork changes this string and + # `storageClass.name` in k8s/storage/ganesha-values.yaml together. + # + # The mount options ride on that StorageClass, because a PVC has nowhere to + # put them and the provisioner copies them onto every PV it emits. The same + # CI step asserts they are really there: + # + # hard a Ganesha restart is routine; `soft` would convert one into a + # truncated featureXML, the single failure class the 7-day + # retention does not cover. Clients block and resume instead. + # vers=4.1 integrated locking, and one port (2049) with no side-band + # NLM/statd, which is what keeps the NetworkPolicy tractable. + # nconnect=4 the nodes run kernel 6.8 and this access pattern is + # metadata-heavy rather than bandwidth-bound. + # default actimeo - close-to-open consistency already guarantees a fresh + # read on every open(), and Logger closes per message while the + # UI reopens per second. Lowering it buys GETATTR traffic and + # no correctness. + storageClassName: template-app-nfs resources: requests: - storage: 500Gi + # MUST stay materially below the backing volume in + # k8s/storage/nfs-backing-pvc.yaml, which is 500Gi. + # + # nfs-provisioner does not carve this out of thin air: `validateOptions` + # statfs()es /export on every provision and refuses the claim outright + # with "insufficient available space N bytes to satisfy claim for M + # bytes" whenever the request exceeds the export's *free* space. That + # check is unconditional - it is not gated on quotas being enabled. A + # freshly made filesystem on a 500 GiB volume never reports 500 GiB free + # (ext4 alone reserves 5%), so a claim equal to the backing volume can + # never bind, on any cluster, and every pod stays Pending on it. + # + # 400Gi leaves ~75 GiB of headroom over the ~475 GiB an ext4 filesystem + # on a 500 GiB volume actually offers. Raising it means growing the + # backing volume first, by more than the same amount. + # + # There is no quota behind the number either - the provisioner hands out + # a directory on the shared export unless `enableXfsQuota` is set, which + # it is not. This is what `kubectl get pvc` reports and what has to clear + # the statfs check, not a per-workspace limit. + # + # CI shrinks this to a few GiB before applying: a kind node's /export + # lives on the runner's disk, which has tens of GiB free, so the real + # figure would fail the same check there. See the "Size the workspaces + # claim for the runner disk" step in .github/workflows/build-and-test.yml. + storage: 400Gi diff --git a/k8s/components/memory-tier-high/kustomization.yaml b/k8s/components/memory-tier-high/kustomization.yaml index 46d4ac77..d280983f 100644 --- a/k8s/components/memory-tier-high/kustomization.yaml +++ b/k8s/components/memory-tier-high/kustomization.yaml @@ -1,10 +1,16 @@ apiVersion: kustomize.config.k8s.io/v1alpha1 kind: Component +# Resource patches only. A tier is a POD SIZE, not a place: nothing in this +# component - and nothing anywhere else in k8s/ - decides which node a pod +# lands on. That is the scheduler's job, and the pinning patch that used to +# live here took it away, unscoped, from every Deployment in the base +# (streamlit and redis included), which is what made the second node +# unusable. It also made topologySpreadConstraints inert, since maxSkew is +# measured over eligible domains and exactly one node was ever eligible. +# `assert_no_node_pinning_anywhere` in .github/scripts/ci-assertions.sh keeps +# it from coming back on a fork rebase. patches: - - path: nodeselector.yaml - target: - kind: Deployment - path: streamlit-resources.yaml target: kind: Deployment diff --git a/k8s/components/memory-tier-high/nodeselector.yaml b/k8s/components/memory-tier-high/nodeselector.yaml deleted file mode 100644 index e848c024..00000000 --- a/k8s/components/memory-tier-high/nodeselector.yaml +++ /dev/null @@ -1,4 +0,0 @@ -- op: add - path: /spec/template/spec/nodeSelector - value: - openms.de/memory-tier: high diff --git a/k8s/components/memory-tier-high/worker-resources.yaml b/k8s/components/memory-tier-high/worker-resources.yaml index 0b7a2517..85c56999 100644 --- a/k8s/components/memory-tier-high/worker-resources.yaml +++ b/k8s/components/memory-tier-high/worker-resources.yaml @@ -3,14 +3,62 @@ kind: Deployment metadata: name: rq-worker spec: + # The replica count travels with the size, and only this tier overrides it. + # k8s/base/rq-worker-deployment.yaml says 2, which is one per node on a pair + # of equally-sized nodes - true of the low tier, false here. A 180Gi + # Guaranteed request fits on a high-memory node and nowhere else, and the + # de.NBI pair has one of those, so an inherited `replicas: 2` puts the second + # worker in Pending permanently: `whenUnsatisfiable: DoNotSchedule` forbids + # doubling it up on the node that could hold it, and no other node can. + # + # Nothing in kustomize couples a resource patch to a replica count, so the + # coupling has to be written down somewhere; here is the only place that sees + # both numbers. A fork with N high-memory nodes raises this to N. Note that + # at 1 replica the spread constraint in the base has nothing to do and + # assert_workers_spread_across_nodes will say so - that assertion is written + # for the shipped (low-tier) sizing that CI deploys, and a high-tier fork + # running it needs N >= 2 for it to mean anything. + replicas: 1 template: spec: containers: - name: rq-worker + # requests == limits for Guaranteed QoS; see + # ../memory-tier-low/worker-resources.yaml for why that is + # architectural rather than advisory once the tier is a pod size + # instead of a node label. + # + # Specific to this tier is what the number now means. 180Gi was + # written as a burst ceiling - uniform across heavy workers so that + # whichever app was active could borrow the shared pool - and as a + # request it stops being a borrow and becomes a reservation the + # scheduler holds whether a workflow is running or not, i.e. most of + # a high-memory node. Fixed workers statically partition capacity; + # that is the accepted trade at this cluster size (decision 10 in + # docs/a16-storage-decisions.md), but a fork sharing its + # node pool with another heavy app should size the worker to its own + # slice and lower BOTH blocks together rather than reopening a gap + # between them. + # + # cpu 20 sits exactly ON the LimitRange maximum in + # k8s/base/limitrange.yaml (200Gi / 20 cpu), so it is a ceiling as + # well as a reservation: this number can only move down unless the + # LimitRange moves first. Raising it alone does not produce a + # Pending pod that says "no node has this much CPU" - the API server + # rejects the Deployment's pods outright as a LimitRange violation, + # which reads as a quota error rather than a capacity one. + # + # This cpu value IS the worker's thread budget, not merely a ceiling + # on it. CommandExecutor._get_max_threads() reads the container's + # cgroup CPU quota in online mode, so run_topp parallelises across + # files up to 20 here and up to 4 on the low tier, with + # max_threads.online in settings.json left as the fallback for + # deployments that expose no quota. Changing cpu therefore changes + # throughput directly - keep requests == limits when you do. resources: requests: - memory: "2Gi" - cpu: "2" + memory: "180Gi" + cpu: "20" limits: memory: "180Gi" cpu: "20" diff --git a/k8s/components/memory-tier-low/kustomization.yaml b/k8s/components/memory-tier-low/kustomization.yaml index 46d4ac77..d280983f 100644 --- a/k8s/components/memory-tier-low/kustomization.yaml +++ b/k8s/components/memory-tier-low/kustomization.yaml @@ -1,10 +1,16 @@ apiVersion: kustomize.config.k8s.io/v1alpha1 kind: Component +# Resource patches only. A tier is a POD SIZE, not a place: nothing in this +# component - and nothing anywhere else in k8s/ - decides which node a pod +# lands on. That is the scheduler's job, and the pinning patch that used to +# live here took it away, unscoped, from every Deployment in the base +# (streamlit and redis included), which is what made the second node +# unusable. It also made topologySpreadConstraints inert, since maxSkew is +# measured over eligible domains and exactly one node was ever eligible. +# `assert_no_node_pinning_anywhere` in .github/scripts/ci-assertions.sh keeps +# it from coming back on a fork rebase. patches: - - path: nodeselector.yaml - target: - kind: Deployment - path: streamlit-resources.yaml target: kind: Deployment diff --git a/k8s/components/memory-tier-low/nodeselector.yaml b/k8s/components/memory-tier-low/nodeselector.yaml deleted file mode 100644 index 49a98d2f..00000000 --- a/k8s/components/memory-tier-low/nodeselector.yaml +++ /dev/null @@ -1,4 +0,0 @@ -- op: add - path: /spec/template/spec/nodeSelector - value: - openms.de/memory-tier: low diff --git a/k8s/components/memory-tier-low/worker-resources.yaml b/k8s/components/memory-tier-low/worker-resources.yaml index b205195b..fb3f79e1 100644 --- a/k8s/components/memory-tier-low/worker-resources.yaml +++ b/k8s/components/memory-tier-low/worker-resources.yaml @@ -7,10 +7,34 @@ spec: spec: containers: - name: rq-worker + # requests == limits, on cpu as well as memory, because that is the + # exact condition for Guaranteed QoS - and Guaranteed is the point. + # A Burstable pod gets oom_score_adj = 1000 - 1000 * request / + # nodeCapacity, so the old 1Gi request scored ~969 out of 1000: the + # worker sat near the top of the node's kill list while holding + # hours of work, and the 16Gi limit bought nothing against it, since + # only the request enters that formula. Guaranteed pins the worker + # at -997 instead, below everything else on the node. + # + # That is architectural now rather than a tuning preference. The + # tier no longer selects a node - the tier IS the pod size - so this + # request is the only thing the scheduler is told about how big a + # worker is. A worker asking for 1Gi against a 16Gi ceiling is + # schedulable on every node in the cluster and OOMKilled on most of + # them. + # + # Converged upwards deliberately: under Guaranteed the limit is a + # wall the kernel enforces at exactly this number, so pulling it + # down to the old request would start killing workflows that run + # today, whereas raising the request only changes where the pod is + # placed. Retune both blocks together - k8s/base/limitrange.yaml + # caps a container at 200Gi / 20 cpu, and CI fails the deploy if + # .status.qosClass stops being Guaranteed + # (assert_worker_qos_guaranteed). resources: requests: - memory: "1Gi" - cpu: "500m" + memory: "16Gi" + cpu: "4" limits: memory: "16Gi" cpu: "4" diff --git a/k8s/overlays/ci/kustomization.yaml b/k8s/overlays/ci/kustomization.yaml new file mode 100644 index 00000000..2c5c653f --- /dev/null +++ b/k8s/overlays/ci/kustomization.yaml @@ -0,0 +1,38 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +# The production overlay with exactly one thing changed: the rq-worker is +# shrunk to a size a GitHub-hosted runner can schedule. APPLIED ONLY BY THE +# KIND JOBS in .github/workflows/build-and-test.yml - operators and every +# static assertion still use k8s/overlays/prod. +# +# k8s/components/memory-tier-low sizes a worker at 16Gi / 4 cpu with +# requests == limits, which is right on the de.NBI nodes and impossible in +# kind. An ubuntu-latest runner has 4 vCPU and ~15.6Gi of RAM in total; every +# kind node is a container on that one host advertising the whole of it, and +# kube-system already holds ~1 cpu of that in requests. Both worker replicas +# would sit Pending for ever and take the whole kind suite down with them - +# including the two assertions that exist to watch the workers +# (assert_workers_spread_across_nodes, assert_worker_qos_guaranteed) and the +# storage assertions that need a running deployment to look at. +# +# CI bends, not k8s/components/: production sizing is the thing under test +# everywhere else. Nothing else differs from prod, so the spread constraint, +# the replica count, the labels, the namePrefix and the images are all the +# production ones - shrinking the worker is not allowed to change what the +# suite is observing. +# +# The `lint-manifests` job renders this root and checks two things about the +# result, because kustomize does NOT error on a `patches:` entry whose target +# matches nothing - it renders happily and applies nothing. So: that the +# worker really came out smaller than prod's (the patch below landed), and +# that requests still equal limits (the shrink did not make it Burstable and +# so turn assert_worker_qos_guaranteed red over CI's own sizing). +resources: + - ../prod + +patches: + - path: worker-resources.yaml + target: + kind: Deployment + name: rq-worker diff --git a/k8s/overlays/ci/worker-resources.yaml b/k8s/overlays/ci/worker-resources.yaml new file mode 100644 index 00000000..f7339e7b --- /dev/null +++ b/k8s/overlays/ci/worker-resources.yaml @@ -0,0 +1,27 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: rq-worker +spec: + template: + spec: + containers: + - name: rq-worker + # requests == limits, exactly as in k8s/components/memory-tier-*/ - + # only the numbers change. assert_worker_qos_guaranteed reads + # .status.qosClass off the running pod, so a CI-shrunk worker that + # went Burstable would report a QoS regression that exists nowhere + # but here. + # + # 1Gi / 500m is what the low tier used to REQUEST before requests + # and limits converged, so it is a size a worker has always been + # started at; two replicas of it leave the runner room for the + # Streamlit pods, Ganesha, the ingress controller and kube-system on + # one physical 4-cpu host. + resources: + requests: + memory: "1Gi" + cpu: "500m" + limits: + memory: "1Gi" + cpu: "500m" diff --git a/k8s/storage/ganesha-values.yaml b/k8s/storage/ganesha-values.yaml new file mode 100644 index 00000000..b3e58853 --- /dev/null +++ b/k8s/storage/ganesha-values.yaml @@ -0,0 +1,266 @@ +# Helm values for the Ganesha NFS server. +# +# Chart: nfs-server-provisioner 1.8.0 (appVersion 4.0.8), from +# kubernetes-sigs/nfs-ganesha-server-and-external-provisioner. +# Consumed by the `helmCharts` block in kustomization.yaml, which is +# why rendering this root needs `kubectl kustomize --enable-helm`. +# +# One userspace Ganesha pod re-exports the single ReadWriteOnce Cinder volume +# in `template-app-storage` as ReadWriteMany, which is what lets streamlit, +# the rq-workers and the nightly cleanup CronJob be placed on any node instead +# of all being stuck on whichever node the Cinder volume is attached to. +# +# =========================================================================== +# THE FOUR SETTINGS BELOW MARKED "NOT A TUNABLE" ARE NOT TUNABLES. +# =========================================================================== +# They are not performance knobs, and they are not chart defaults nobody got +# round to reviewing. Together they are the only reason this server comes back +# serving the same bytes under the same identity after a restart - and it will +# be restarted, by upgrades, evictions, node drains and OOM. They are: +# +# persistence.existingClaim the backing volume is a fixed, pre-existing +# claim, never dynamically provisioned +# storageClass.reclaimPolicy Retain, so deleting a PV or PVC does not +# destroy the data with it +# replicaCount one pod, and never two on one RWO claim +# extraArgs.device-based-fsids NFS file handles that survive a re-attach +# +# Each one fails the same way: silently, long after the change, as clients +# getting ESTALE or as workspaces that appear to have vanished. If you think +# one of them is wrong, read `docs/a16-storage-decisions.md` +# decision 3 and `docs/a16-storage-runbook.md` section 1 first. +# +# --------------------------------------------------------------------------- +# Names shared with the rest of k8s/storage/ - all four are hand-synced +# --------------------------------------------------------------------------- +# nfs-server-data the Cinder PVC in nfs-backing-pvc.yaml, consumed +# below as `persistence.existingClaim` +# template-app-nfs the StorageClass name the workspaces PVC in `openms` +# claims. Cluster scoped, hence the app prefix +# template-app-nfs-server every object this chart renders, and the +# provisioner name. The ClusterRole and +# ClusterRoleBinding among them are cluster scoped, +# hence the app prefix +# nfs-server the `app` label on the server pod, which the +# storage runbook addresses it by. Namespaced, so +# it needs no prefix +# template-app-storage the namespace, which must enforce the `privileged` +# Pod Security Standard - see `securityContext` below +# +# The prefixed three exist because several forks of this template share the +# de.NBI cluster and each brings its own export. kustomize cannot apply a +# namePrefix here (see kustomization.yaml), so they are written out by hand; +# that file carries the full fork checklist. +# +# The chart version is pinned in kustomization.yaml and must stay pinned. The +# chart's `master` branch carries an unreleased appVersion 6.5 whose image tag +# was never published to registry.k8s.io, so an unpinned render can resolve to +# an image that cannot be pulled at all. + +# The chart derives every object name from the release name, giving +# `nfs-server-nfs-server-provisioner` here. Both overrides are set so the +# names are fixed by this file rather than by whatever the release is called: +# `fullnameOverride` names the StatefulSet, Service, ServiceAccount, +# ClusterRole and ClusterRoleBinding, and `nameOverride` sets the `app` label +# on the pod. +# +# They differ on purpose. The objects `fullnameOverride` names include the +# ClusterRole and the ClusterRoleBinding, which are CLUSTER scoped: two forks +# of this template on one cluster would otherwise overwrite each other's RBAC, +# so that one carries the app prefix. The `app` label is namespaced and the +# namespace is already prefixed, so `nameOverride` stays short - which matters +# because docs/a16-storage-runbook.md section 4.5, "the single most valuable +# test in the list", restarts the server with `delete pod -l app=nfs-server`. +# Addressing it +# by label rather than by object name is also what keeps the runbook working +# for a fork that changes the prefix. +# +# Note for whoever runs section 4.6: this chart renders a StatefulSet, not a +# Deployment, so it is `scale statefulset -l app=nfs-server`. +nameOverride: nfs-server +fullnameOverride: template-app-nfs-server + +# NOT A TUNABLE. +# The chart's own template says it: "TODO: Investigate how/if nfs-provisioner +# can be scaled out beyond 1 replica". Two Ganesha pods over one volume is +# worse than an outage - they would hand out conflicting state and neither +# would know - and the second would not schedule anyway, because the backing +# claim is ReadWriteOnce. +# +# The other half of the one-pod rule is how the pod gets *replaced*, and it +# needs no setting here. The chart renders a StatefulSet, not a Deployment: +# updating a StatefulSet deletes pod 0, waits for it to terminate completely, +# and only then creates the replacement. There is no surge, so the RWO claim +# is never held by two pods and the rolling-update deadlock that a Deployment +# would hit here cannot occur. That is exactly the `strategy: Recreate` +# guarantee, obtained structurally. Converting this workload to a Deployment, +# or raising the replica count, reintroduces the deadlock. +replicaCount: 1 + +image: + repository: registry.k8s.io/sig-storage/nfs-provisioner + # Digest-pinned per docs/a16-storage-decisions.md decision 3: A16 is the destination rather + # than a bridge, so the server image is a multi-year commitment and should + # not move underneath us. The chart uses the digest and ignores the tag when + # both are set; the tag is kept only to record which release the digest is. + # Verified against registry.k8s.io as the multi-arch index for v4.0.8 + # (linux/amd64 and linux/arm64 both present). + tag: v4.0.8 + digest: sha256:c825f3d5e28bde099bd7a3daace28772d412c9157ad47fa752a9ad0baafc118d + pullPolicy: IfNotPresent + +# NOT A TUNABLE. +# Ganesha defaults to deriving every NFS file handle it hands to every client +# from the major/minor device number of the backing volume at `/export`. A +# Cinder volume's `/dev/vdX` minor is not stable across detach and re-attach, +# so with the default left in place every client gets ESTALE the first time +# this pod moves to another node - typically weeks later, with no obvious link +# to the change that caused it. The upstream flag documentation says the same +# thing in fewer words: set it false on any cloud-backed volume. +# +# With it disabled the export ids come from `vfs.conf`, which lives on the +# backing volume and therefore outlives the pod. That is the specific reason +# this design uses the chart's StorageClass rather than a hand-rolled export. +# +# `grace-period` is deliberately absent. The default 90s is what lets clients +# reclaim their locks after a restart; the chart's commented-out +# `grace-period: 0` example applies only to servers whose export is *not* +# persisted, which is the opposite of this deployment. +extraArgs: + device-based-fsids: "false" + +persistence: + enabled: true + # NOT A TUNABLE. + # A fixed, pre-existing claim. Leave this unset and the chart falls back to + # a volumeClaimTemplate instead - and a re-provisioned claim is a different, + # empty volume, so the server would come back serving nothing and every + # workspace would appear to have vanished. + # + # Hand-synced with `metadata.name` in nfs-backing-pvc.yaml. Nothing checks + # it: kustomize treats this as an opaque string and does not rewrite or + # validate it, so a typo shows up as a pod stuck Pending on a claim that + # does not exist. + # + # `accessMode`, `size` and `storageClass` are unused on this path and are + # deliberately not repeated here, so there is only one place that describes + # the backing volume. + existingClaim: nfs-server-data + +storageClass: + create: true + + # Cluster-scoped, so it must not collide with an existing class on the + # cluster - and a plain `nfs` would, the moment a second fork of this + # template deploys here. Whichever installed second would lose the create, + # and every workspaces PVC in the cluster would then bind to one app's + # export. Hence the app prefix. The workspaces PVC in `openms` claims exactly + # this name; CI's "Verify the workspace StorageClass contract" step fails the + # deploy if the two ever drift. + name: template-app-nfs + defaultClass: false + + # Pinned rather than derived. Left unset the chart generates + # `cluster.local/`, so a renamed release would silently change the + # StorageClass's provisioner and orphan every PV already bound through it. + # This is the value the default produces from the `fullnameOverride` above, + # so writing it out changes nothing now and prevents that later. It is a + # cluster-wide identifier, which is the second reason for the app prefix: + # two forks sharing `cluster.local/nfs-server` would each answer the other's + # provisioning requests. + provisionerName: cluster.local/template-app-nfs-server + + # NOT A TUNABLE. + # The chart defaults to `Delete`, as does the cluster's `cinder-csi` class - + # that is the hazard recorded in docs/storage-defect-register.md. With `Retain`, deleting a PV or + # a PVC leaves the data on the backing volume instead of destroying it, and + # in particular deleting a claim while diagnosing an outage no longer + # destroys the evidence along with it. + reclaimPolicy: Retain + + # The only storageClass key that was still inherited, in a file that restates + # its defaults on purpose. The chart defaults it to `true`, and an in-tree + # `nfs:` PV has no volume expander behind it - so a `kubectl edit pvc` that + # raises the request is ACCEPTED, and then sits in FileSystemResizePending + # forever with no controller that will ever act on it. Rejecting the resize + # outright is the honest answer: growing this tier means growing Ganesha's + # backing Cinder volume, which is a separate operation on a different claim. + # `null` is the chart's documented off-switch and removes the field entirely. + allowVolumeExpansion: null + + # The released 1.8.0 chart defaults these to `vers=3`, which is wrong here + # twice over: NFSv3 has no integrated locking, and it needs rpcbind (111), + # mountd (20048) and a pinned nlockmgr port, so it could not work behind a + # NetworkPolicy that admits 2049 alone. Per docs/a16-storage-decisions.md decision 6: + # + # hard a restart of this server must block the client and resume, + # never return a short read. A `soft` mount turns a routine + # Ganesha restart into a truncated featureXML - the one + # failure class the 7-day workspace retention does not cover. + # vers=4.1 integrated locking, stateful reclaim, and a single port, + # which is what makes the NetworkPolicy tractable. + # nconnect=4 the nodes run kernel 6.8, and the access pattern here is + # metadata-heavy rather than bandwidth-bound. + # + # `retrans` and `timeo` are the chart's own defaults, carried over unchanged + # because they have to be restated to keep the two above. There is + # deliberately no `actimeo` or `noac`: close-to-open consistency already + # guarantees a fresh read on every open(), and Logger closes per message + # while the UI reopens per second, so lowering it would only add GETATTR + # traffic for no correctness gain. + mountOptions: + - hard + - vers=4.1 + - nconnect=4 + - retrans=2 + - timeo=30 + +# The provisioner creates and deletes PersistentVolumes cluster-wide, so it +# needs its own ServiceAccount and ClusterRole rather than running as +# `default`. +rbac: + create: true + +# Restated at the chart default on purpose, because it is load-bearing rather +# than incidental: Ganesha needs DAC_READ_SEARCH and SYS_RESOURCE to serve an +# export it does not own, and it exports `no_root_squash`. Those capabilities +# are the reason this workload lives in `template-app-storage` at +# `pod-security.kubernetes.io/enforce=privileged` instead of in `openms`, +# which is shared with another application. Removing them does not harden +# anything; it stops the server working. +securityContext: + capabilities: + add: + - DAC_READ_SEARCH + - SYS_RESOURCE + +# The chart ships no resource block at all, which leaves this pod BestEffort: +# nothing reserved, and first on the kernel's kill list. N concurrent write +# streams mean N sets of buffers and dirty pages inside one userspace process, +# so under load it gets OOMKilled - and then every client hangs on its `hard` +# mount, the restart burns the ~90s NFSv4 grace period, and the same load is +# still waiting when it returns. docs/a16-storage-decisions.md decision 3 calls this the one +# genuine collapse mode in the design. +# +# Having explicit requests, and having them equal to the limits, is not +# optional. Equality makes the pod Guaranteed QoS (`oom_score_adj -997`). +# Without it the rq-workers become Guaranteed in the worker-spreading step +# while this pod stays burstable at roughly 938, so under node memory pressure +# the kernel would kill the storage server - an outage for every user - in +# preference to the single worker that caused the pressure. +# +# The *numbers*, unlike the four settings above, are a tunable. They are +# reasoned starting points rather than measurements, and docs/a16-storage-decisions.md +# decision 11 asks for a load test to find the real knee. Raise `memory` if +# the pod is ever OOMKilled; raise `cpu` if +# `container_cpu_cfs_throttled_seconds_total` climbs under load. Move requests +# and limits together, or the QoS class silently drops to burstable. Note also +# that a kind node in CI is far smaller than a de.NBI node, so raising these +# may need a CI-only resource patch, the same way the worker sizes do. +resources: + requests: + cpu: "1" + memory: 4Gi + limits: + cpu: "1" + memory: 4Gi diff --git a/k8s/storage/kustomization.yaml b/k8s/storage/kustomization.yaml new file mode 100644 index 00000000..98c1103f --- /dev/null +++ b/k8s/storage/kustomization.yaml @@ -0,0 +1,86 @@ +# Storage tier: one Ganesha NFS server re-exporting a single ReadWriteOnce +# Cinder volume as ReadWriteMany, so streamlit, the rq-workers and the nightly +# cleanup CronJob can be placed on any node by the scheduler instead of all +# being pinned to whichever node the Cinder volume happens to be attached to. +# +# This is a SEPARATE kustomize root, applied on its own. It is deliberately not +# a component of k8s/base and is not referenced from k8s/overlays/prod: +# +# - k8s/base sets `namespace: openms`. Components inherit their parent's +# transformers, and the namespace transformer runs *after* patches, so a +# per-object namespace set from inside the base tree gets clobbered - these +# objects would end up back in `openms`, which is exactly what the storage +# split exists to prevent (see k8s/storage/namespace.yaml). +# - No `namePrefix` here. `persistence.existingClaim` in the Ganesha values +# is an opaque string that kustomize does not rewrite, so a prefix +# transformer would rename the backing claim out from under the server and +# leave the pod Pending on a claim that does not exist. +# +# That does NOT mean the names are generic. Several forks of this template +# share the de.NBI cluster, so the names below carry the app prefix by hand +# instead of by transformer. FORKING THIS TEMPLATE: change `template-app-` to +# your own slug in all of the following, together. docs/kubernetes-deployment.md +# step 4c has the same list as a table. +# +# 1. `namespace:` below, twice (the transformer and the helmCharts entry) +# 2. `metadata.name` in namespace.yaml, and `metadata.namespace` in +# nfs-backing-pvc.yaml +# 3. `fullnameOverride` and `storageClass.provisionerName` in +# ganesha-values.yaml - these name the ClusterRole, the ClusterRoleBinding +# and the provisioner, all of which are cluster scoped +# 4. `storageClass.name` in ganesha-values.yaml, which is cluster scoped too, +# and must keep matching `storageClassName` in k8s/base/workspace-pvc.yaml +# +# Also hand-synced, but from the prod overlay rather than from here: the +# `app: template-app` literal in networkpolicy.yaml, which CI asserts against +# `commonLabels.app`. +# +# Rendering this root needs Helm on PATH, because of the `helmCharts` block: +# +# kubectl kustomize --enable-helm k8s/storage/ +# kubectl kustomize --enable-helm k8s/storage/ | kubectl apply -f - +# +# `kubectl apply -k` accepts no --enable-helm flag, so the pipe above is the +# way to apply this root - and it is what CI has to run, since a storage root +# CI never applies strands every pod in `openms` on a claim nothing serves. +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +# Applies to every object below, the Helm-inflated ones included. The Namespace +# object itself is cluster scoped; its name already matches, so the transformer +# is a no-op there rather than a rename. +namespace: template-app-storage + +resources: + - namespace.yaml + - nfs-backing-pvc.yaml + - networkpolicy.yaml + +helmCharts: + # kubernetes-sigs/nfs-ganesha-server-and-external-provisioner. Chosen over a + # hand-rolled Deployment plus export ConfigMap for one specific reason: the + # chart's provisioner keeps its export ids in `vfs.conf` on the *backing + # volume*, so they survive the pod. That, with `device-based-fsids: false` + # in the values, is what keeps NFS file handles stable across a restart - + # without it clients ESTALE the first time the pod moves, typically weeks + # later and with no obvious link to the change that caused it. + # + # `version` is pinned: an unpinned chart re-resolves on every render, and a + # chart bump that changed the export layout would look like a redeploy. + # + # `releaseName` feeds the chart's object names and its `release:` label. The + # chart's default naming is `-nfs-server-provisioner` unless the + # values set `fullnameOverride`, which they do - nothing in this file can + # supply it. The storage runbook addresses the server by its + # `app: nfs-server` + # label rather than by object name, so it is unaffected by that override. + - name: nfs-server-provisioner + repo: https://kubernetes-sigs.github.io/nfs-ganesha-server-and-external-provisioner/ + version: 1.8.0 + releaseName: nfs-server + # Also set here, not only in the transformer above: the chart renders + # `.Release.Namespace` into its RBAC subjects, and a chart that resolved it + # to `default` would be repaired by the transformer only where kustomize + # knows to look. + namespace: template-app-storage + valuesFile: ganesha-values.yaml diff --git a/k8s/storage/namespace.yaml b/k8s/storage/namespace.yaml new file mode 100644 index 00000000..f8715de1 --- /dev/null +++ b/k8s/storage/namespace.yaml @@ -0,0 +1,32 @@ +# The storage tier lives in its own namespace, not in `openms`. +# +# Ganesha re-exports the workspace volume with `no_root_squash` and needs +# elevated privileges to do it, so this namespace has to sit at the +# `privileged` Pod Security level. `openms` is shared with other applications +# (`app=nuxl-app` on the de.NBI cluster), so granting that exemption there +# would hand it to every pod of every app in the namespace. Splitting the tier +# keeps the exemption scoped to the one component that needs it. +# +# The name carries the app prefix for the same reason `namePrefix` exists in +# k8s/overlays/prod/kustomization.yaml: several forks of this template +# (quantms-web, UmetaFlow, FLASHApp) share the de.NBI cluster, and each one +# brings its own Ganesha server, its own backing volume and its own export. +# A namespace called plain `openms-storage` would collide on every object +# inside it - the StatefulSet, the Service, the backing PVC - and the second +# fork to deploy would take over the first one's export. Renaming this is one +# of the four strings a fork changes; the list is in kustomization.yaml. +apiVersion: v1 +kind: Namespace +metadata: + name: template-app-storage + labels: + app.kubernetes.io/part-of: openms-streamlit + # Chosen, not inherited. An unlabelled namespace already falls back to the + # built-in default of `privileged`; writing it out makes the exemption + # visible and reviewable rather than implied by an absent label. + # + # Pinned to a policy version so a cluster upgrade cannot tighten what + # `privileged` means underneath a workload that has to keep + # DAC_READ_SEARCH and SYS_RESOURCE. + pod-security.kubernetes.io/enforce: privileged + pod-security.kubernetes.io/enforce-version: v1.28 diff --git a/k8s/storage/networkpolicy.yaml b/k8s/storage/networkpolicy.yaml new file mode 100644 index 00000000..a4087f14 --- /dev/null +++ b/k8s/storage/networkpolicy.yaml @@ -0,0 +1,162 @@ +# Ingress policy for the storage tier. +# +# Ganesha exports the workspace volume with `no_root_squash`, so anything that +# can open TCP 2049 in this namespace holds *root* read and write over every +# user's workspace: it mounts the export, acts as uid 0 inside it, and reads, +# rewrites or deletes any file belonging to anyone. Reachability is the whole +# access-control story here - there is no second check behind it. +# +# That is not theoretical. The client pods live in `openms`, which on the +# de.NBI cluster is shared with a second application labelled `app: nuxl-app`. +# Admitting the whole namespace would therefore hand NuXL's pods root over +# every workspace and undo the reason the storage tier was split out at all, so +# the allow rules below are scoped as narrowly as the traffic allows. +# +# =========================================================================== +# WHO THE CLIENT ACTUALLY IS: the node, not the pod. +# =========================================================================== +# The chart's provisioner emits in-tree `nfs:` PersistentVolumes. An in-tree +# NFS volume is mounted by the KUBELET, on the node, in the host network +# namespace - the source address on the wire is the node's IP, not the pod's. +# It therefore matches no `podSelector` at all, and the default-deny below +# drops it. Symptom: every mount hangs, on any CNI that enforces policy for +# node-origin traffic. `ipBlock` is the only selector NetworkPolicy offers +# that can name a node, which is why the second allow rule exists and why it +# has to be filled in per cluster. +# +# The pod-scoped rule is kept alongside it, not replaced by it: it is what +# admits a client that mounts from inside a pod (a csi-driver-nfs node plugin, +# or a debug pod running `mount -t nfs4` by hand), and it documents the +# intended blast radius even where the node rule is what carries the traffic. +# +# Kubelet health probes originate from the node in the same way. Most CNIs +# exempt them, but that exemption is CNI-dependent; the node rule below covers +# them too. + +# Default deny. Selects every pod in the namespace and permits no ingress at +# all. The allow rules below are the only thing that adds any back. +# +# Not redundant, even though those rules also select every pod: +# NetworkPolicies are additive, so the moment an allow rule is narrowed to a +# subset of pods (a metrics sidecar, a second export) everything it stops +# naming falls back to deny rather than to open. Deleting this makes that +# future edit silently permissive. +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: default-deny-ingress + labels: + app.kubernetes.io/part-of: openms-streamlit +spec: + podSelector: {} + policyTypes: + - Ingress +--- +# Hole 1: NFS from the app's own pods. +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: allow-nfs-from-app + labels: + app.kubernetes.io/part-of: openms-streamlit + app.kubernetes.io/component: nfs-server +spec: + # Every pod in this namespace, rather than the chart's own labels + # (`app: nfs-server`, `release: `): the namespace holds nothing + # but the storage tier, so the two are equivalent today, and this way a + # renamed Helm release or a changed chart label scheme cannot quietly detach + # the rule from the server pod. Such a detachment would fail closed - the + # default-deny above would be all that still matched it - but the symptom is + # every client mount hanging, which is an expensive way to find out about a + # label change. + podSelector: {} + policyTypes: + - Ingress + ingress: + - from: + # Both selectors are ONE list element, so they are ANDed: a pod must + # carry the app label *and* live in `openms`. Written as two elements + # they would be ORed, admitting every pod in that namespace (NuXL + # included) and every pod anywhere else that happens to carry the + # label. Either alone defeats the point of this file, so CI asserts + # they share one element as well as checking their values. + - namespaceSelector: + matchLabels: + # Set automatically by the API server on every namespace since + # 1.21, so it needs no cooperation from k8s/base/namespace.yaml. + # Hand-synced with `namespace: openms` in + # k8s/base/kustomization.yaml - a fork that renames the + # application namespace has to change this line too, and CI + # compares the two strings. + kubernetes.io/metadata.name: openms + podSelector: + matchLabels: + # HAND-SYNCED with `commonLabels.app` in + # k8s/overlays/prod/kustomization.yaml, and CI asserts the two + # strings are equal. + # + # Nothing propagates the value here. That transformer belongs to + # the prod overlay, this is a separate root applied on its own, + # and the label therefore has to be repeated as a literal. Every + # workspace-mounting workload in `openms` - streamlit, rq-worker + # and the nightly cleanup CronJob - receives the label from that + # one transformer, so if the two strings drift, all of them lose + # the mount at once and the symptom is a hung mount at deploy + # time rather than a lint error. Hence the CI assertion. + app: template-app + ports: + # 2049 only, and only TCP. The clients mount NFSv4.1, which carries + # mount, lock and status over this single port; NFSv3 would also need + # 111 (rpcbind), 20048 (mountd) and a statically pinned nlockmgr port. + # Leaving those closed is deliberate: a client that silently falls back + # to v3 fails to mount instead of quietly working through a wider hole. + - protocol: TCP + port: 2049 +--- +# Hole 2: NFS from the nodes. This is the rule the real mounts come through. +# +# =========================================================================== +# THE CIDR BELOW IS A PLACEHOLDER AND MUST BE SET BEFORE THE FIRST DEPLOY. +# =========================================================================== +# 192.0.2.0/24 is RFC 5737 TEST-NET-1: reserved for documentation and routed +# nowhere, so as shipped this rule admits nothing. That is the safe direction +# to be wrong in - an unset CIDR shows up immediately, at cutover, as pods +# stuck in ContainerCreating with `mount.nfs: Connection timed out`, which is +# loud and reversible. The opposite mistake is not: a CIDR wide enough to +# contain the POD network hands every pod in the cluster, NuXL included, root +# over every workspace, and nothing about that is visible from the outside. +# +# Set it to the range covering the INTERNAL IPs of the cluster's nodes. The +# addresses are in `kubectl get nodes -o wide` under INTERNAL-IP. Then check +# the range against the pod and service networks before applying - if it +# overlaps either, narrow it. One /32 per node is perfectly acceptable, and is +# what a small fixed cluster should use. Note that a hostNetwork pod shares +# its node's address and is admitted by this rule whatever its labels say; +# that is inherent to naming nodes by address, and is the reason the range +# should be as tight as the cluster allows. +# +# docs/a16-storage-runbook.md section 3 step 5 carries this as an explicit cutover step. +# +# Cilium, which is what the de.NBI user clusters run: from 1.14 remote nodes +# carry the `remote-node` identity, and CIDR rules do not select node +# identities unless the agent runs with `policy-cidr-match-mode=nodes`. If +# mounts still hang with the correct CIDR in place, that flag is the next +# thing to check - not a wider selector here. +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: allow-nfs-from-nodes + labels: + app.kubernetes.io/part-of: openms-streamlit + app.kubernetes.io/component: nfs-server +spec: + podSelector: {} + policyTypes: + - Ingress + ingress: + - from: + - ipBlock: + cidr: 192.0.2.0/24 + ports: + - protocol: TCP + port: 2049 diff --git a/k8s/storage/nfs-backing-pvc.yaml b/k8s/storage/nfs-backing-pvc.yaml new file mode 100644 index 00000000..7855ad5f --- /dev/null +++ b/k8s/storage/nfs-backing-pvc.yaml @@ -0,0 +1,43 @@ +# Backing volume for the NFS server. Every workspace in the cluster lives +# inside this one Cinder volume, which Ganesha re-exports as ReadWriteMany so +# that pods on more than one node can share it. +# +# The name is fixed and load bearing: the Ganesha chart consumes this claim +# through `persistence.existingClaim`, never through dynamic provisioning. A +# dynamically provisioned claim is re-provisioned as an *empty* volume whenever +# it is recreated, so the server would come back serving different bytes and +# every workspace would appear to vanish. +# +# This is a fresh volume, not a migration of k8s/base/workspace-pvc.yaml. +# Workspaces are deleted after 7 days anyway, so there is nothing worth moving, +# and leaving the original volume untouched keeps rollback to a single +# re-apply of the previous manifests. +# +# Do not delete this claim while diagnosing a storage problem: `cinder-csi` +# reclaims with `Delete`, so removing the claim destroys the volume and the +# evidence together. Scale the server to zero instead. +# +# The size is not just bookkeeping. nfs-provisioner statfs()es /export on every +# provision and refuses any claim larger than the export's *free* space, so +# this volume has to be materially bigger than the workspaces claim in +# k8s/base/workspace-pvc.yaml (400Gi against the ~475 GiB an ext4 filesystem on +# a 500 GiB volume actually offers). Sizing the two equally - which is what the +# first draft did - means the workspaces claim can never bind, on any cluster. +# Grow this one first if the workspaces claim ever needs to grow. +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: nfs-server-data + namespace: template-app-storage + labels: + app.kubernetes.io/part-of: openms-streamlit + app.kubernetes.io/component: nfs-server +spec: + # Cinder attaches to exactly one node. That is the whole reason this volume + # is behind an NFS server instead of being mounted by the app directly. + accessModes: + - ReadWriteOnce + storageClassName: cinder-csi + resources: + requests: + storage: 500Gi diff --git a/src/Workflow.py b/src/Workflow.py index 7523a0cb..02245c25 100644 --- a/src/Workflow.py +++ b/src/Workflow.py @@ -51,11 +51,18 @@ def configure(self) -> None: # Parameters are specified within the file in the DEFAULTS dictionary. self.ui.input_python("example") - def execution(self) -> None: + def execution(self) -> bool: + # Return True only if every step succeeded. The caller + # (WorkflowManager.workflow_process in local mode, tasks.execute_workflow + # in queue mode) writes the "WORKFLOW FINISHED" marker based on this, + # and that marker is the only thing the run page reads to decide whether + # the workflow succeeded. Returning None marks a successful run failed; + # ignoring an executor result marks a failed run successful. + # Any parameter checks, here simply checking if mzML files are selected if not self.params["mzML-files"]: self.logger.log("ERROR: No mzML files selected.") - return + return False # Get mzML files with FileManager in_mzML = self.file_manager.get_files(self.params["mzML-files"]) @@ -69,10 +76,13 @@ def execution(self) -> None: ) # Run FeatureFinderMetabo tool with input and output files. + # run_topp returns False if any of the commands it ran failed. self.logger.log("Detecting features...") - self.executor.run_topp( + if not self.executor.run_topp( "FeatureFinderMetabo", input_output={"in": in_mzML, "out": out_ffm} - ) + ): + self.logger.log("ERROR: Feature detection failed.") + return False # Prepare input and output files for feature linking in_fl = self.file_manager.get_files(out_ffm, collect=True) @@ -82,17 +92,29 @@ def execution(self) -> None: # Run FeatureLinkerUnlabaeledKD with all feature maps passed at once self.logger.log("Linking features...") - self.executor.run_topp( + if not self.executor.run_topp( "FeatureLinkerUnlabeledKD", input_output={"in": in_fl, "out": out_fl} - ) + ): + self.logger.log("ERROR: Feature linking failed.") + return False + + # run_python has the same contract as run_topp: False if the script was + # not found or exited non-zero. self.logger.log("Exporting consensus features to pandas DataFrame...") - self.executor.run_python( + if not self.executor.run_python( "export_consensus_feature_df", input_output={"in": out_fl[0]} - ) + ): + self.logger.log("ERROR: Exporting consensus features failed.") + return False + # Check if adduct detection should be run. if self.params["run-python-script"]: # Example for a custom Python tool, which is located in src/python-tools. - self.executor.run_python("example", {"in": in_mzML}) + if not self.executor.run_python("example", {"in": in_mzML}): + self.logger.log("ERROR: Custom Python script failed.") + return False + + return True @st.fragment def results(self) -> None: diff --git a/src/common/common.py b/src/common/common.py index c048064f..b7f79837 100644 --- a/src/common/common.py +++ b/src/common/common.py @@ -5,6 +5,7 @@ import time import psutil import shutil +import logging import pandas as pd import streamlit as st @@ -28,6 +29,8 @@ save_workspace_as_demo, ) +logger = logging.getLogger(__name__) + # Detect system platform OS_PLATFORM = sys.platform @@ -216,6 +219,17 @@ def copy_demo_workspace(demo_name: str, target_path: Path) -> bool: @st.fragment(run_every=5) def monitor_hardware(): + """ + Display CPU and RAM of the machine running this Streamlit process. + + Gated by the caller on where the workflows actually run, not on + `online_deployment`: in Kubernetes psutil here reports a Streamlit pod that + does no work, and an idle 2Gi container shown while a 64Gi worker saturates + is worse than showing nothing. In the single-container images - the full + one under docker-compose, and `Dockerfile_simple`, which has no queue at + all - this process and the workflows share a machine and these are the only + numbers that expander has. See `health.queue_workers_are_remote()`. + """ cpu_progress = psutil.cpu_percent(interval=None) / 100 ram_progress = 1 - psutil.virtual_memory().available / psutil.virtual_memory().total @@ -272,6 +286,106 @@ def monitor_queue(): pass # Silently fail if queue not available +@st.fragment(run_every=5) +def monitor_storage(): + """ + Display shared storage status in sidebar. + + Reads Redis and nothing else. The workspace volume is a shared NFS mount + online, and its failure mode is a hang rather than an error: a `stat` on a + `hard` mount that has wedged blocks in uninterruptible sleep and cannot be + killed, so a fragment re-running every 5 seconds would accumulate unkillable + threads - this indicator would become the very hang it exists to report. + The mount is touched by the rq-worker readiness probe instead, which + publishes a per node heartbeat whose Redis TTL is the liveness mechanism. + + Renders nothing where no node publishes a heartbeat at all - local mode, + docker-compose, apptainer - because a red tick for a mechanism that is not + deployed is a false alarm pointing at the wrong layer. + """ + try: + from src.workflow.health import get_storage_status + + status = get_storage_status() + if not status: + return + + state = status.get("state", "unknown") + nodes = status.get("nodes") or [] + stale = status.get("stale_nodes") or [] + + st.markdown("---") + st.markdown("**Storage Status**") + + if state == "connected": + st.markdown(":green[Connected]") + st.caption( + f"{len(nodes)} node{'' if len(nodes) == 1 else 's'} reporting a " + "healthy shared volume." + ) + elif state == "degraded": + # The case per node heartbeats exist for. Collapsing them would + # show green here, because a healthy node is still reporting. + st.markdown(":orange[Degraded]") + st.caption( + f"{', '.join(stale)} stopped confirming the shared volume. " + "Workflows running there may be stalled; other nodes are fine." + ) + elif state == "unreachable": + st.markdown(":red[Unreachable]") + st.caption( + "No node has confirmed the shared volume recently. Running " + "workflows may be stalled and new results may not appear." + ) + else: + # Redis is down, which says nothing about the mount. Showing this as + # unreachable storage would send an operator to the wrong layer. + st.markdown(":gray[Unknown]") + st.caption("Cannot reach the queue, so storage status is unknown.") + + st.caption(f"Last fetched at: {time.strftime('%H:%M:%S')}") + + except Exception: + # Never break the sidebar over an indicator, but never swallow it in + # silence either: a broken indicator that renders nothing is + # indistinguishable from a deployment that has no shared storage. + logger.exception("Could not render the shared storage indicator") + + +def render_resource_utilization(): + """ + Render the sidebar's "Resource Utilization" expander. + + Extracted from `render_sidebar()` so the choice of panels is reachable + without standing up a whole session (tests/test_sidebar_monitors.py). + + Which panels apply is a question about the deployment, and the three + answers do not line up with `online_deployment`: + + - Kubernetes: the queue and the shared volume are what a user's job waits + on, and psutil here would describe a Streamlit pod that runs nothing. + - Full image under docker/docker-compose: one container runs Redis, the RQ + workers and Streamlit, so all three panels describe the same machine. + - `Dockerfile_simple`: `online_deployment` is baked true but there is no + Redis and no queue, so the psutil panel is the only content there is. + Gating it on `online_deployment` left the expander empty. + """ + from src.workflow.health import get_queue_metrics, queue_workers_are_remote + + with st.expander("📊 **Resource Utilization**"): + # `{}` means "no queue in this deployment" - the same distinction + # get_queue_metrics() already draws for the metrics themselves. + metrics = get_queue_metrics() + if metrics: + monitor_queue() + monitor_storage() + # Short-circuited on `available`: an unreachable queue is not evidence + # that the workers are elsewhere, and this saves the second round trip + # in exactly the case where Redis is slow to answer. + if not (metrics.get("available") and queue_workers_are_remote()): + monitor_hardware() + + def load_params(default: bool = False) -> dict[str, Any]: """ Load parameters from a JSON file and return a dictionary containing them. @@ -765,11 +879,7 @@ def change_workspace(): else: st.session_state["spectrum_num_bins"] = 50 - with st.expander("📊 **Resource Utilization**"): - monitor_hardware() - # Show queue metrics in online mode - if st.session_state.settings.get("online_deployment", False): - monitor_queue() + render_resource_utilization() # Display OpenMS WebApp Template Version from settings.json with st.container(): diff --git a/src/workflow/CommandExecutor.py b/src/workflow/CommandExecutor.py index 6479cb02..141ae5c4 100644 --- a/src/workflow/CommandExecutor.py +++ b/src/workflow/CommandExecutor.py @@ -6,6 +6,7 @@ from pathlib import Path from .Logger import Logger from .ParameterManager import ParameterManager, bool_param_paths_from_param_xml_ini +from .settings_io import load_settings, detect_cpu_quota import sys import importlib.util import json @@ -25,28 +26,73 @@ def __init__(self, workflow_dir: Path, logger: Logger, parameter_manager: Parame self.pid_dir = Path(workflow_dir, "pids") self.logger = logger self.parameter_manager = parameter_manager + # Memoised thread budget, resolved lazily by _get_max_threads(). + self._max_threads = None def _get_max_threads(self) -> int: """ Get max threads for current deployment mode. - In local mode, reads from parameter manager (persisted params.json). - In online mode, uses the configured value directly from settings. + Settings are read from settings.json on disk, not from + st.session_state: inside the RQ work horse there is no + ScriptRunContext, so st.session_state is an empty mock, the online + branch was unreachable and the hardcoded local default silently won + instead of max_threads.online. + + In local mode the workspace params.json may override the configured + value - that is what the "Threads" widget writes, and it is only + rendered in local mode. Online the worker is shared and sized by its + pod spec, so user supplied params.json is ignored. + + Online the container's own cgroup CPU quota wins over + max_threads.online where one exists. The rq-worker runs at + requests.cpu == limits.cpu, so its pod spec IS its allocation, and the + memory-tier components size that per deployment - a 4 cpu worker and a + 20 cpu worker cannot both be described by one number in settings.json, + and the number that was there described neither. max_threads.online + remains the fallback for every deployment where no quota is readable: + bare metal, docker-compose without a cpu limit, a dev box. + + The trade-off, deliberately taken: an operator can no longer + under-subscribe a container that declares more CPU than it wants used. + For a Guaranteed pod that is the right call, because the declaration + and the allocation are the same thing. + + The result is memoised for the lifetime of this executor: run_topp() + consults it twice, once for its own thread split and once more inside + run_multiple_commands(), and the two must agree. Returns: int: Maximum number of threads to use for parallel processing (minimum 1). """ - settings = st.session_state.get("settings", {}) + if self._max_threads is not None: + return self._max_threads + + settings = load_settings() max_threads_config = settings.get("max_threads", {"local": 4, "online": 2}) if settings.get("online_deployment", False): - value = max_threads_config.get("online", 2) + quota = detect_cpu_quota() + if quota is not None: + value = quota + else: + value = max_threads_config.get("online", 2) else: default = max_threads_config.get("local", 4) params = self.parameter_manager.get_parameters_from_json() value = params.get("max_threads", default) - return max(1, int(value)) + try: + value = int(value) + except (TypeError, ValueError): + # params.json is user supplied - the parameter import uploader + # writes it verbatim - so a non numeric thread budget degrades to + # serial execution instead of taking the whole run down. + self.logger.log(f"Invalid max_threads value {value!r}, falling back to 1.", 1) + value = 1 + + self._max_threads = max(1, value) + return self._max_threads def run_multiple_commands( self, commands: list[str] @@ -361,7 +407,7 @@ def stop(self) -> None: shutil.rmtree(self.pid_dir, ignore_errors=True) self.logger.log("Workflow stopped.") - def run_python(self, script_file: str, input_output: dict = {}) -> None: + def run_python(self, script_file: str, input_output: dict = {}) -> bool: """ Executes a specified Python script with dynamic input and output parameters, optionally logging the execution process. The method identifies and loads @@ -378,6 +424,13 @@ def run_python(self, script_file: str, input_output: dict = {}) -> None: If the path is omitted, the method looks for the script in 'src/python-tools/'. The '.py' extension is appended if not present. input_output (dict, optional): A dictionary specifying the input/output parameter names (as key) and their corresponding file paths (as value). Defaults to {}. + + Returns: + bool: True if the script ran and exited 0, False otherwise. Same + contract as run_topp(), so execution() can gate on it. This used + to discard run_command()'s result and return None, which made + every `if not self.executor.run_python(...)` guard dead code and + left a failed Python tool indistinguishable from a successful one. """ # Check if script file exists (can be specified without path and extension) # default location: src/python-tools/script_file @@ -387,8 +440,12 @@ def run_python(self, script_file: str, input_output: dict = {}) -> None: if not path.exists(): path = Path("src", "python-tools", script_file) if not path.exists(): - self.logger.log(f"Script file not found: {script_file}") - + # Bail out here: falling through to spec_from_file_location on a + # path that does not exist raises out of the workflow instead of + # reporting a failed step. + self.logger.log(f"ERROR: Script file not found: {script_file}", 0) + return False + # load DEFAULTS if path.parent not in sys.path: sys.path.append(str(path.parent)) @@ -399,7 +456,7 @@ def run_python(self, script_file: str, input_output: dict = {}) -> None: if defaults is None: self.logger.log(f"WARNING: No DEFAULTS found in {path.name}") # run command without params - self.run_command(["python", str(path)]) + return self.run_command(["python", str(path)]) elif isinstance(defaults, list): defaults = {entry["key"]: entry["value"] for entry in defaults} # load paramters from JSON file @@ -414,6 +471,17 @@ def run_python(self, script_file: str, input_output: dict = {}) -> None: with open(tmp_params_file, "w", encoding="utf-8") as f: json.dump(defaults, f, indent=4) # run command - self.run_command(["python", str(path), str(tmp_params_file)]) + success = self.run_command(["python", str(path), str(tmp_params_file)]) # remove tmp params file - tmp_params_file.unlink() \ No newline at end of file + tmp_params_file.unlink() + return success + + # DEFAULTS exists but is not the documented list of parameter dicts, so + # nothing ran. Reporting that as success would put the workflow's + # WORKFLOW FINISHED marker on a step which never executed. + self.logger.log( + f"ERROR: DEFAULTS in {path.name} is a {type(defaults).__name__}, " + "expected a list of parameter dicts. Script not run.", + 0, + ) + return False \ No newline at end of file diff --git a/src/workflow/ParameterManager.py b/src/workflow/ParameterManager.py index 2838c1b9..cb0473e4 100644 --- a/src/workflow/ParameterManager.py +++ b/src/workflow/ParameterManager.py @@ -1,11 +1,18 @@ import pyopenms as poms +import errno import json +import logging +import os import shutil import subprocess import streamlit as st +import time +import uuid import xml.etree.ElementTree as ET from pathlib import Path +logger = logging.getLogger(__name__) + def bool_param_paths_from_param_xml_ini(ini_path: Path, tool_stem: str) -> set[str]: """ @@ -49,6 +56,191 @@ def walk(el: ET.Element, parts: tuple[str, ...]) -> None: return out +class InvalidParameterFileError(ValueError): + """ + Raised when a params.json exists but cannot be read as a parameter dict. + + Deliberately distinct from an absent file, which simply means a first run + with no stored parameters. Conflating the two is the erasure bug: a failed + read reported as an empty dict gets merged into a read-modify-write and + written back as the current session's subset, permanently deleting + ``_defaults``, ``_flag_params`` and every other tool's values. + """ + + +class TransientParameterFileError(InvalidParameterFileError): + """ + Raised when the read failed for a reason expected to clear by itself. + + A shared NFS export is restarted by upgrades, evictions and node drains, + and clients see ESTALE/EIO until it is back. The stored bytes are almost + certainly intact, so callers must still refuse to write - the current + content is unknown - but must not offer to delete the file. Resetting + during a self-healing blip is what would turn a momentary outage into the + permanent data loss this split exists to prevent. + + A subclass, so every ``except InvalidParameterFileError`` guard keeps + catching it; only code that treats the two differently has to know. + """ + + +# errno values meaning "the storage is not answering right now", as opposed to +# "this file is broken". Looked up defensively: not all exist on every platform. +_TRANSIENT_READ_ERRNOS = frozenset( + value + for value in ( + getattr(errno, name, None) + for name in ( + "ESTALE", + "EIO", + "ENOTCONN", + "ETIMEDOUT", + "EAGAIN", + "EINTR", + "ENOLINK", + "EREMOTEIO", + ) + ) + if value is not None +) + +# Two retries, at 0.2s and 0.4s. Long enough to ride out a re-established NFS +# connection, short enough that a Streamlit rerun does not visibly stall. Only +# transient errnos retry, so a corrupt file still fails on the first attempt. +_READ_ATTEMPTS = 3 +_READ_RETRY_DELAY_SECONDS = 0.2 + + +def _read_parameter_file(params_file: Path) -> dict: + """ + Read a parameter JSON file. Pure I/O, no Streamlit and no session state. + + Args: + params_file: Path of the JSON file to read. + + Returns: + dict: The stored parameters, or an empty dict if the file does not + exist at all - a first run has no parameters and that is not a + failure. + + Raises: + TransientParameterFileError: The storage did not answer (ESTALE, EIO, + ...), and still did not after retrying. The file is probably fine. + InvalidParameterFileError: The file exists but could not be read as a + JSON object: malformed or torn content, a bad encoding, a directory + in its place, permissions. Callers which merge the result and write + it back must abort on this rather than treat it as "no parameters". + """ + last_error = None + for attempt in range(_READ_ATTEMPTS): + try: + with open(params_file, "r", encoding="utf-8") as f: + params = json.load(f) + break + except FileNotFoundError: + # Ordered before OSError on purpose: an absent file is a first run, + # not a failure, and conflating the two is the erasure bug. + return {} + except OSError as e: + if e.errno not in _TRANSIENT_READ_ERRNOS: + raise InvalidParameterFileError( + f"Could not read parameter file {params_file}: {e}" + ) from e + last_error = e + if attempt + 1 < _READ_ATTEMPTS: + time.sleep(_READ_RETRY_DELAY_SECONDS * (attempt + 1)) + except ValueError as e: + # json.JSONDecodeError *and* UnicodeDecodeError - the latter is a + # ValueError, not an OSError, so catching OSError alone let a + # params.json written in the platform codepage escape both readers + # uncaught and take the workflow page down with a raw traceback. + raise InvalidParameterFileError( + f"Could not read parameter file {params_file}: {e}" + ) from e + else: + raise TransientParameterFileError( + f"Could not read parameter file {params_file} after " + f"{_READ_ATTEMPTS} attempts: {last_error}" + ) from last_error + + if not isinstance(params, dict): + raise InvalidParameterFileError( + f"Parameter file {params_file} does not hold a JSON object " + f"(found {type(params).__name__})." + ) + return params + + +def _write_parameter_file(params_file: Path, params: dict) -> None: + """ + Write a parameter JSON file atomically: temp file, then ``os.replace()``. + + A truncate-then-rewrite that dies half way - an OOM kill, ENOSPC - leaves a + file which is neither the old parameters nor the new ones. Every later + strict read rejects it, so ``save_parameters()`` becomes a permanent silent + no-op and the parameter page stops on every render, with no way out but the + reset button. ``os.replace()`` is atomic on POSIX and on Windows, so a + reader sees either the whole old file or the whole new one. + + This does not make concurrent writers safe - the last writer still wins the + whole file, docs/storage-defect-register.md A2 - it only removes the torn file. + + The scratch name carries a uuid as well as the pid. The pid alone is not + unique per writer: Streamlit serves every session from threads of ONE + process, and in a container that process is pid 1 in every replica - so two + sessions sharing a `?workspace=` link, or two pods now that the volume is + genuinely ReadWriteMany, resolved to the same scratch path and interleaved + their writes into it. Whichever one reached os.replace() second published a + file the other was still writing, which is precisely the torn file this + function exists to prevent. + + flush + fsync before the rename because os.replace() orders the directory + entry, not the data behind it: on NFS a crash between the two can otherwise + publish a name pointing at unwritten blocks. + + Deliberately not tempfile.mkstemp(): it creates at mode 0600, so params.json + would quietly change permissions the first time it was rewritten. + + Args: + params_file: Destination path. + params: Parameters to store. + """ + tmp_file = params_file.with_name( + f".{params_file.name}.{os.getpid()}.{uuid.uuid4().hex}.tmp" + ) + try: + with open(tmp_file, "w", encoding="utf-8") as f: + json.dump(params, f, indent=4) + f.flush() + os.fsync(f.fileno()) + os.replace(tmp_file, params_file) + except BaseException: + # Never leave the scratch file behind next to params.json. + try: + tmp_file.unlink() + except OSError: + pass + raise + + +def _streamlit_session_active() -> bool: + """ + True only while running inside a real Streamlit script run. + + tasks.py builds a ParameterManager inside the RQ work horse, where there is + no ScriptRunContext and st.error() is a silent no-op - so the user facing + half of a message is skipped there and the log carries it instead. + """ + try: + from streamlit.runtime.scriptrunner import get_script_run_ctx + + # suppress_warning, or every probe from the worker logs a "missing + # ScriptRunContext!" warning next to the message it is deciding about. + return get_script_run_ctx(suppress_warning=True) is not None + except Exception: # streamlit absent, mocked out, or internals moved + return False + + class ParameterManager: """ Manages the parameters for a workflow, including saving parameters to a JSON file, @@ -132,8 +324,18 @@ def save_parameters(self) -> None: } # Merge with parameters from json - # Advanced parameters are only in session state if the view is active - json_params = self.get_parameters_from_json() | json_params + # Advanced parameters are only in session state if the view is active. + # The read has to be strict: merging a tolerant empty dict from a failed + # read and writing that back is what erases every stored value, so a + # failed read aborts the write and leaves the file on disk untouched. + try: + stored_params = self.read_parameters_strict() + except InvalidParameterFileError as e: + self._report_unreadable_parameter_file( + e, "Parameters were not saved, the existing file is left unchanged." + ) + return + json_params = stored_params | json_params # get a list of TOPP tools (or tool instance names) which are in session state current_topp_tools = list( @@ -187,29 +389,79 @@ def save_parameters(self) -> None: # store non-default value json_params[tool][short_key] = value # Save to json file - with open(self.params_file, "w", encoding="utf-8") as f: - json.dump(json_params, f, indent=4) + _write_parameter_file(self.params_file, json_params) def get_parameters_from_json(self) -> dict: """ Loads parameters from the JSON file if it exists and returns them as a dictionary. - If the file does not exist, it returns an empty dictionary. + If the file does not exist or cannot be read, it returns an empty dictionary. + + Tolerant on purpose, for the constructors and the read-only callers which + have no way to handle a raise. Never use it as the read half of a + read-modify-write: an empty dict from a failed read, merged over the + stored parameters and written back, erases them. Use + read_parameters_strict() there. Returns: dict: A dictionary containing the loaded parameters. Keys are parameter names, and values are parameter values. """ - # Check if parameter file exists - if not Path(self.params_file).exists(): + try: + return _read_parameter_file(self.params_file) + except InvalidParameterFileError as e: + self._report_unreadable_parameter_file( + e, "Falling back to default parameters." + ) return {} - else: - # Load parameters from json file - try: - with open(self.params_file, "r", encoding="utf-8") as f: - return json.load(f) - except: - st.error("**ERROR**: Attempting to load an invalid JSON parameter file. Reset to defaults.") - return {} + + def read_parameters_strict(self) -> dict: + """ + Loads parameters from the JSON file, raising instead of hiding a failed read. + + This is the reader for every read-modify-write-back site. + get_parameters_from_json() cannot be used there: its empty dict for an + unreadable file gets merged with the current session's parameters and + written back, permanently erasing everything the session does not hold - + while the user is told it was a deliberate reset to defaults. + + Returns: + dict: The stored parameters. Empty only if params.json does not exist. + + Raises: + InvalidParameterFileError: params.json exists but is unreadable. + """ + return _read_parameter_file(self.params_file) + + def write_parameters(self, params: dict) -> None: + """ + Write params.json, atomically, for the write-back sites outside this class. + + StreamlitUI._input_TOPP_impl() seeds ``_flag_params`` and ``_defaults`` + by reading with read_parameters_strict(), merging, and writing straight + back, so it needs the same atomic write save_parameters() uses - a torn + file written there wedges the parameter page on every later render. + + Args: + params: The complete parameter dict to store. + """ + _write_parameter_file(self.params_file, params) + + def _report_unreadable_parameter_file(self, error: Exception, consequence: str) -> None: + """ + Report an unreadable params.json, to the log always and to the user if there is one. + + logging is the sink which always works: tasks.py builds a + ParameterManager inside the RQ work horse, where st.error() reaches + nobody. + + Args: + error: The read failure being reported. + consequence: What the caller did about it, in the user's terms. + """ + message = f"{error} {consequence}" + logger.error(message) + if _streamlit_session_active(): + st.error(f"**ERROR**: {message}") def get_merged_params(self, tool_instance_name: str, ini_params: dict = None) -> dict: """ @@ -268,13 +520,42 @@ def get_topp_parameters(self, tool: str, tool_instance_name: str = None) -> dict return self.get_merged_params(instance_name, ini_params=ini_params) - def reset_to_default_parameters(self) -> None: + def reset_to_default_parameters(self) -> Path | None: """ - Resets the parameters to their default values by deleting the custom parameters - JSON file. + Reset to defaults by moving the stored parameter file out of the way. + + Moved, not deleted. This is also the recovery offered when params.json + cannot be read, and an unreadable file is very often a healthy file + behind a storage blip, so deleting it there would turn a self-healing + outage into permanent loss. The bytes are kept under a timestamped name + beside the original and the caller is told where they went. A rename + also copes with a directory in place of params.json, which + ``unlink()`` cannot - that used to make the reset button itself raise, + leaving the parameter page stopped on every render with no way out. + + Returns: + Path: Where the previous parameters were moved, or None when there + was nothing to reset. + + Raises: + OSError: The file exists but could not be moved. Callers rendering + a reset affordance must surface this rather than let it escape. """ - # Delete custom params json file - self.params_file.unlink(missing_ok=True) + if not self.params_file.exists(): + return None + + stamp = time.strftime("%Y%m%d-%H%M%S") + backup = self.params_file.with_name(f"{self.params_file.name}.{stamp}.bak") + collision = 1 + while backup.exists(): + backup = self.params_file.with_name( + f"{self.params_file.name}.{stamp}-{collision}.bak" + ) + collision += 1 + + self.params_file.rename(backup) + logger.info("Reset parameters, previous file kept at %s", backup) + return backup def load_presets(self) -> dict: """ @@ -341,8 +622,16 @@ def apply_preset(self, preset_name: str) -> bool: if not preset: return False - # Load existing parameters - current_params = self.get_parameters_from_json() + # Load existing parameters. Strict, because this is a read-modify-write + # back too: a tolerant empty dict here would delete every parameter the + # preset does not mention and still report success. + try: + current_params = self.read_parameters_strict() + except InvalidParameterFileError as e: + self._report_unreadable_parameter_file( + e, "The preset was not applied, the existing file is left unchanged." + ) + return False # Collect keys to delete from session_state keys_to_delete = [] @@ -374,8 +663,7 @@ def apply_preset(self, preset_name: str) -> bool: del st.session_state[session_key] # Save updated parameters to file - with open(self.params_file, "w", encoding="utf-8") as f: - json.dump(current_params, f, indent=4) + _write_parameter_file(self.params_file, current_params) return True diff --git a/src/workflow/QueueManager.py b/src/workflow/QueueManager.py index 21db9a8b..1dd35147 100644 --- a/src/workflow/QueueManager.py +++ b/src/workflow/QueueManager.py @@ -7,12 +7,13 @@ """ import os -import json from typing import Optional, Callable, Any from dataclasses import dataclass from enum import Enum from pathlib import Path +from .settings_io import load_settings + class JobStatus(Enum): """Job status enumeration matching RQ states""" @@ -63,6 +64,13 @@ def __init__(self): queue_settings = settings.get("queue_settings", {}) self._default_timeout = queue_settings.get("default_timeout", 7200) self._default_result_ttl = queue_settings.get("result_ttl", 86400) + # RQ's own default failure_ttl is one year. Failed jobs only started + # existing once execute_workflow began re-raising, and each one keeps + # its full traceback in Redis - which runs with a 256Mi limit, no + # persistence and no maxmemory-policy, so it is OOMKilled rather than + # evicting. "No mzML files selected" is a routine user error; it must + # not leave a year-lived key behind. + self._default_failure_ttl = queue_settings.get("failure_ttl", 86400) if self._is_online: self._init_redis() @@ -70,11 +78,7 @@ def __init__(self): @staticmethod def _load_settings() -> dict: """Load settings.json once; return empty dict on failure.""" - try: - with open("settings.json", "r") as f: - return json.load(f) - except Exception: - return {} + return load_settings() def _check_online_mode(self, settings: dict) -> bool: """Check if running in online mode""" @@ -119,6 +123,7 @@ def submit_job( job_id: Optional[str] = None, timeout: Optional[int] = None, result_ttl: Optional[int] = None, + failure_ttl: Optional[int] = None, description: str = "" ) -> Optional[str]: """ @@ -131,6 +136,9 @@ def submit_job( job_id: Optional custom job ID (defaults to UUID) timeout: Job timeout in seconds (defaults to settings.json queue_settings.default_timeout) result_ttl: How long to keep results (defaults to settings.json queue_settings.result_ttl) + failure_ttl: How long to keep failed jobs, with their traceback + (defaults to settings.json queue_settings.failure_ttl). Without + it RQ keeps them for a year. description: Human-readable job description Returns: @@ -144,6 +152,8 @@ def submit_job( timeout = self._default_timeout if result_ttl is None: result_ttl = self._default_result_ttl + if failure_ttl is None: + failure_ttl = self._default_failure_ttl try: job = self._queue.enqueue( @@ -153,6 +163,7 @@ def submit_job( job_id=job_id, job_timeout=timeout, result_ttl=result_ttl, + failure_ttl=failure_ttl, description=description, meta={"description": description, "progress": 0.0, "current_step": ""} ) diff --git a/src/workflow/StreamlitUI.py b/src/workflow/StreamlitUI.py index d426e3a9..f9adf5ec 100644 --- a/src/workflow/StreamlitUI.py +++ b/src/workflow/StreamlitUI.py @@ -21,6 +21,10 @@ tk_file_dialog, ) from src.workflow._log_status import classify_log_outcome +from src.workflow.ParameterManager import ( + InvalidParameterFileError, + TransientParameterFileError, +) def _mounted_data_root() -> Union[Path, None]: @@ -893,6 +897,78 @@ def _input_TOPP_fragmented( display_subsection_tabs, custom_defaults, tool_instance_name, ) + def _read_params_for_update(self, tool_instance_name: str) -> dict: + """ + Strict read of params.json for the read-modify-write-back sites below. + + Both callers merge into what they read and write the result straight + back, so a tolerant empty dict from an unreadable file would be written + over everything stored - erasing `_defaults`, `_flag_params` and every + other tool's values while looking like a successful save. + read_parameters_strict() raises instead, and here that raise becomes a + halt: the write is skipped entirely, so the file on disk is preserved + for inspection and can still be repaired by hand, or reset from the + button rendered below. + + Args: + tool_instance_name (str): Instance currently being rendered, used to + keep the reset button key unique across tool fragments. + + Returns: + dict: Stored parameters, or an empty dict for an absent file. + + Does not return if the file exists but cannot be read. + """ + try: + return self.parameter_manager.read_parameters_strict() + except TransientParameterFileError as e: + # Ordered before InvalidParameterFileError, of which this is a + # subclass. The storage is not answering; the file itself is most + # likely intact, so deliberately no reset affordance here - a reset + # during a Ganesha restart is what would actually destroy it. + st.error( + f"**ERROR**: {e} Nothing was written, so no stored parameters " + "were lost. This usually clears by itself, reload the page in " + "a moment." + ) + st.stop() + # st.stop() raises, so this is only reached when Streamlit is + # stubbed out; re-raise rather than fall through to the write. + raise + except InvalidParameterFileError as e: + st.error( + f"**ERROR**: {e} Nothing was written, so no stored parameters " + "were lost. Repair the file, or reset to defaults to continue." + ) + if st.button( + "⚠️ Reset parameters to defaults", + help="Move the unreadable parameter file aside and start over from the default parameters.", + key=f"reset_unreadable_params_{tool_instance_name}", + ): + try: + backup = self.parameter_manager.reset_to_default_parameters() + except OSError as reset_error: + # Without this the button itself raises and the page stops + # again on the next render, with no remaining way out. + st.error( + f"**ERROR**: Could not move " + f"{self.parameter_manager.params_file} aside: " + f"{reset_error}" + ) + else: + self.parameter_manager.clear_parameter_session_state() + st.toast( + f"Parameters reset to defaults, previous file kept as {backup.name}" + if backup is not None + else "Parameters reset to defaults" + ) + # Every tool section is affected, not just this fragment. + st.rerun(scope="app") + st.stop() + # st.stop() raises, so this is only reached when Streamlit is + # stubbed out; re-raise rather than fall through to the write. + raise + def _input_TOPP_impl( self, topp_tool_name: str, @@ -921,12 +997,14 @@ def _input_TOPP_impl( if "_topp_flag_params" not in st.session_state: st.session_state["_topp_flag_params"] = {} st.session_state["_topp_flag_params"][tool_instance_name] = list(flag_parameters) - _fp = self.parameter_manager.get_parameters_from_json() + _fp = self._read_params_for_update(tool_instance_name) if "_flag_params" not in _fp: _fp["_flag_params"] = {} _fp["_flag_params"][tool_instance_name] = list(flag_parameters) - with open(self.parameter_manager.params_file, "w", encoding="utf-8") as _f: - json.dump(_fp, _f, indent=4) + # Atomic: a truncate-then-rewrite interrupted here leaves a torn + # params.json which every later strict read rejects, wedging this very + # method on each render. + self.parameter_manager.write_parameters(_fp) if not display_subsections: display_subsection_tabs = False @@ -941,14 +1019,15 @@ def _input_TOPP_impl( # Seed custom defaults into params.json under _defaults key if custom_defaults: - params = self.parameter_manager.get_parameters_from_json() + params = self._read_params_for_update(tool_instance_name) if "_defaults" not in params: params["_defaults"] = {} params["_defaults"][tool_instance_name] = custom_defaults - with open(self.parameter_manager.params_file, "w", encoding="utf-8") as f: - json.dump(params, f, indent=4) - # Refresh self.params so widget resolution sees _defaults - self.params = self.parameter_manager.get_parameters_from_json() + self.parameter_manager.write_parameters(params) + # Refresh self.params so widget resolution sees _defaults - reuse + # what was just written instead of reading the file back, which + # could be torn by a concurrent writer. + self.params = params # read into Param object param = poms.Param() @@ -1465,7 +1544,10 @@ def parameter_section(self, custom_parameter_function) -> None: key="param_import_uploader" ) if up is not None: - with open(self.parameter_manager.params_file, "w") as f: + # encoding="utf-8" explicitly: the payload is decoded as utf-8 + # just above, and writing it back in the platform codepage + # produced a params.json that no later utf-8 read could decode. + with open(self.parameter_manager.params_file, "w", encoding="utf-8") as f: f.write(up.read().decode("utf-8")) self.parameter_manager.clear_parameter_session_state() st.toast("Parameters imported") diff --git a/src/workflow/health.py b/src/workflow/health.py index 6473f9e7..9e5ba99a 100644 --- a/src/workflow/health.py +++ b/src/workflow/health.py @@ -1,11 +1,100 @@ """ -Health check utilities for Redis queue monitoring. +Health check utilities for Redis queue and shared storage monitoring. -Provides functions to check Redis and worker health status -for display in the sidebar metrics. +Provides functions to check Redis and worker health status for display in the +sidebar metrics, plus the shared storage heartbeat: `probe_storage()` runs on +the node that holds the mount (from the rq-worker readiness probe), +`get_storage_status()` is what the sidebar fragment renders. + +Three constraints hold for everything in this module: + +* **No Streamlit.** The heartbeat writer runs next to the RQ worker, which has + no ScriptRunContext - the same constraint `tasks.py` carries. +* **The readers never touch the filesystem.** A `stat` on a `hard` NFS mount + that has wedged blocks in uninterruptible sleep and cannot be killed, so a + sidebar fragment re-running every 5 seconds would accumulate unkillable + threads: the indicator would become the very hang it exists to report. The + mount is touched only by `probe_storage()`, which runs from the readiness + probe under a shell `timeout`, in a process nobody waits on. +* **Every Redis call is bounded.** All of them go through `_redis_client()`, + which sets a connect and a socket timeout. An unbounded call is reachable + from the sidebar - `get_queue_metrics()` drives a fragment on the session's + ScriptRunner thread - so a Redis that is black-holing packets would otherwise + freeze the session rather than show it as unavailable. """ +import logging import os +import socket +import sys +import time + +logger = logging.getLogger(__name__) + +# Shared storage heartbeat: an inverted health check. +# +# rq-worker readiness probe sidebar +# SET storage:node: (a worker lives here) --> reads Redis only +# touch .nfs-probe --ok--> SET storage:ok: --> +# \-- blocks on a wedged mount --> storage:ok expires --> goes red +# +# Redis' TTL *is* the liveness mechanism, so there is no timeout logic here to +# get wrong, and nothing in the Streamlit process ever calls into the mount. +# +# Two keys, not one, and the order between them is the whole design: +# +# storage:node: published *before* the mount is touched, so it keeps +# being refreshed by a node whose mount has wedged. This +# is the expected-node baseline. Without it "no key" and +# "no such node" are indistinguishable, and a healthy +# node silently covers for a wedged one. +# storage:ok: published *after* the mount answered. Its absence +# against a present storage:node: is exactly the +# signal the indicator exists to show. +# +# STORAGE_SENTINEL_NAME is named here and nowhere else. The readiness probe in +# k8s/base/rq-worker-deployment.yaml runs `--probe` rather than spelling the +# path out in YAML, precisely so there is no second copy to drift from this +# one - drift would show up as a permanently red indicator, not as a lint +# error. tests/test_storage_health.py runs the probe's command to keep that +# delegation honest. +STORAGE_SENTINEL_NAME = ".nfs-probe" +STORAGE_HEARTBEAT_PREFIX = "storage:ok:" +STORAGE_NODE_PREFIX = "storage:node:" +# Four times the readiness probe's periodSeconds (30s): three refreshes may be +# missed before the indicator goes red, which keeps a slow probe - a cold +# interpreter on a CPU-saturated worker, a DNS hiccup on the way to Redis - +# from turning it red while the mount is healthy. It still goes red before the +# probe's failureThreshold marks the worker NotReady (4 x 30s plus a 20s +# timeout, ~140s): degradation should be visible before it is fatal. Keep the +# two in step - a TTL below periodSeconds makes the indicator flicker red +# between healthy probes. +STORAGE_HEARTBEAT_TTL = 120 +# Ten probe periods. A node stops being *expected* five minutes after its +# worker stopped probing at all - a drain, a scale-down, a deleted node - so a +# node that legitimately went away does not leave the indicator permanently +# degraded. A wedged mount does not clear it: the registration is written +# before the mount is touched, so a wedged node keeps re-registering and stays +# visible for as long as its worker is alive. +STORAGE_NODE_TTL = 300 + +DEFAULT_REDIS_URL = "redis://localhost:6379/0" +DEFAULT_WORKSPACES_DIR = "/workspaces-streamlit-template" +# Every Redis call made from the Streamlit process is bounded: the sidebar +# fragment re-runs every 5 seconds and must not sit in a connect() to a Redis +# that is black-holing packets. +REDIS_SOCKET_TIMEOUT = 2 + + +def _redis_client(redis_url: str): + """Connect to Redis with every socket operation bounded.""" + from redis import Redis + + return Redis.from_url( + redis_url, + socket_connect_timeout=REDIS_SOCKET_TIMEOUT, + socket_timeout=REDIS_SOCKET_TIMEOUT, + ) def check_redis_health() -> dict: @@ -15,12 +104,10 @@ def check_redis_health() -> dict: Returns: Dictionary with health status and metrics """ - redis_url = os.environ.get("REDIS_URL", "redis://localhost:6379/0") + redis_url = os.environ.get("REDIS_URL", DEFAULT_REDIS_URL) try: - from redis import Redis - - redis = Redis.from_url(redis_url) + redis = _redis_client(redis_url) redis.ping() info = redis.info() @@ -49,13 +136,12 @@ def check_worker_health() -> dict: Returns: Dictionary with worker status and metrics """ - redis_url = os.environ.get("REDIS_URL", "redis://localhost:6379/0") + redis_url = os.environ.get("REDIS_URL", DEFAULT_REDIS_URL) try: - from redis import Redis from rq import Worker, Queue - redis = Redis.from_url(redis_url) + redis = _redis_client(redis_url) queue = Queue("openms-workflows", connection=redis) workers = Worker.all(connection=redis) @@ -101,11 +187,10 @@ def get_queue_metrics() -> dict: return {} try: - from redis import Redis from rq import Worker, Queue - redis_url = os.environ.get("REDIS_URL", "redis://localhost:6379/0") - redis = Redis.from_url(redis_url) + redis_url = os.environ.get("REDIS_URL", DEFAULT_REDIS_URL) + redis = _redis_client(redis_url) # Test connection redis.ping() @@ -127,3 +212,409 @@ def get_queue_metrics() -> dict: } except Exception: return {"available": False} + + +def _as_text(value) -> str: + """Redis hands back bytes or str depending on the client's decoding.""" + if isinstance(value, bytes): + return value.decode(errors="replace") + return "" if value is None else str(value) + + +def queue_workers_are_remote() -> bool: + """ + Whether the workflows run on a machine other than this one. + + The sidebar's psutil panel measures *this* process. Whether that is the + right machine depends on the deployment, not on `online_deployment`: + + - Kubernetes: Streamlit and rq-worker are separate pods, usually on + separate nodes. psutil here would show an idle 2Gi pod while a 64Gi + worker saturates, so the panel is hidden. + - The full Docker image (and docker-compose): `docker/entrypoint.sh` starts + Redis, the RQ workers *and* Streamlit in one container, so psutil is + measuring exactly the machine that runs the workflows. The panel stays. + - `Dockerfile_simple`: no Redis at all, workflows run in-process. The panel + stays - it is the only thing that expander has to show. + + RQ records each worker's hostname at birth, so the question answers itself + from data the queue already holds. Uncertainty resolves to False: a + possibly-wrong number beats an empty panel, and every failure here (no + Redis, no RQ, an unreachable queue, no workers registered) means nothing + has *proved* the workers are elsewhere. + + Returns: + bool: True only when the queue is reachable, workers are registered, + and none of them runs on this host. + """ + redis_url = os.environ.get("REDIS_URL") + if not redis_url: + return False + + try: + from rq import Worker + + client = _redis_client(redis_url) + client.ping() + hostnames = { + _as_text(getattr(worker, "hostname", "")) for worker in Worker.all(connection=client) + } + hostnames.discard("") + except Exception: + return False + + return bool(hostnames) and socket.gethostname() not in hostnames + + +def storage_heartbeat_key(node: str) -> str: + """ + Redis key carrying one node's storage heartbeat: its mount answered. + + Keyed per node, never a single shared key: with one key a healthy node + would keep refreshing it while another node's mount is wedged, and the + indicator would stay green straight through a real outage. The per node + keys only deliver that if something also says which nodes are *expected* - + see `storage_node_key()`. + + Args: + node: Node name, as the Downward API reports `spec.nodeName`. + + Returns: + str: The Redis key for that node. + """ + return f"{STORAGE_HEARTBEAT_PREFIX}{node}" + + +def storage_node_key(node: str) -> str: + """ + Redis key registering that a worker lives on this node. + + The expected-node baseline. It is written before the mount is touched, so + it survives a wedged mount and makes that node's missing heartbeat visible + instead of merely absent. It expires (`STORAGE_NODE_TTL`) so a node that + was drained stops being expected. + + Args: + node: Node name, as the Downward API reports `spec.nodeName`. + + Returns: + str: The Redis key for that node. + """ + return f"{STORAGE_NODE_PREFIX}{node}" + + +def current_node_name() -> str: + """ + Name of the node this process runs on. + + `NODE_NAME` is injected from the Downward API (`spec.nodeName`) in + k8s/base/rq-worker-deployment.yaml. Outside Kubernetes - docker-compose, a + local run - the container hostname is the closest analogue and keeps the + key per writer rather than shared. + + Returns: + str: The node name. + """ + return os.environ.get("NODE_NAME") or socket.gethostname() + + +def register_storage_node(node: str = "", ttl: int = STORAGE_NODE_TTL) -> bool: + """ + Register that a worker is running on this node. + + Deliberately says nothing about the mount, and is therefore called *before* + the mount is touched: a node whose mount has wedged must keep answering + "I exist" so that its missing heartbeat reads as a fault rather than as an + absent node. + + Never raises: a Redis that is down says nothing about the mount, and the + caller's exit code is a statement about storage. + + Args: + node: Node name to register. Defaults to `current_node_name()`. + ttl: Seconds the registration stays valid. + + Returns: + bool: True if the registration was published. + """ + node = node or current_node_name() + redis_url = os.environ.get("REDIS_URL", DEFAULT_REDIS_URL) + + try: + client = _redis_client(redis_url) + client.set(storage_node_key(node), str(int(time.time())), ex=int(ttl)) + return True + except Exception as e: + logger.warning("Could not register the storage node %s: %s", node, e) + return False + + +def touch_storage_sentinel(workspaces_dir: str = "") -> bool: + """ + Write to the shared volume, the way the readiness probe means it. + + A write, not a `stat`: with the default `actimeo` a stat can be answered + from the client's attribute cache while the server is gone, which is a + false green. `utime` is a metadata write, so it always reaches the server, + and creating the file first means a fresh volume passes instead of failing + forever on a file nothing else makes. The sentinel is dot-named and is not + a directory, so clean-up-workspaces.py skips it twice over. + + This is the one function in this module that touches the filesystem, and it + is only ever called from `probe_storage()` in the worker's readiness probe: + on a `hard` mount this call can block in uninterruptible sleep, which is + survivable only in a process nobody waits on and a shell `timeout` is + holding the stopwatch for. + + Args: + workspaces_dir: Directory holding the sentinel. Defaults to + `$WORKSPACES_DIR`, then `DEFAULT_WORKSPACES_DIR`. + + Returns: + bool: True if the volume accepted the write. + """ + directory = ( + workspaces_dir + or os.environ.get("WORKSPACES_DIR") + or DEFAULT_WORKSPACES_DIR + ) + sentinel = os.path.join(directory, STORAGE_SENTINEL_NAME) + + try: + # Append mode: create if missing, never truncate. The open alone can be + # served from the client's cache, which is why utime follows. + with open(sentinel, "ab"): + pass + os.utime(sentinel, None) + return True + except Exception as e: + logger.warning("The shared volume did not accept a write at %s: %s", sentinel, e) + return False + + +def write_storage_heartbeat(node: str = "", ttl: int = STORAGE_HEARTBEAT_TTL) -> bool: + """ + Publish this node's storage heartbeat: its mount answered just now. + + **Verifies nothing.** Call `probe_storage()` unless the mount has already + been confirmed in this same process - this function on its own publishes a + green light for a volume it never touched. It is separate so that the + blocking half lives in `touch_storage_sentinel()`, in a process that may + block forever and be abandoned, which the Redis write must not be. + + Expiry is the whole mechanism - a heartbeat is a claim with a deadline, not + a flag somebody has to remember to clear. The node registration is + refreshed alongside it, in one round trip, so a heartbeat always implies an + expected node. + + Never raises: a Redis that is down says nothing about the mount, and the + caller's exit code is a statement about storage. + + Args: + node: Node name to key the heartbeat under. Defaults to + `current_node_name()`. + ttl: Seconds the heartbeat stays valid. + + Returns: + bool: True if the heartbeat was published. + """ + node = node or current_node_name() + redis_url = os.environ.get("REDIS_URL", DEFAULT_REDIS_URL) + + try: + client = _redis_client(redis_url) + # SET ... EX rather than SETEX: same single atomic write-with-deadline, + # without the command Redis deprecated in 2.6.12. Pipelined so the two + # keys cost one round trip, not two. + stamp = str(int(time.time())) + pipeline = client.pipeline() + pipeline.set(storage_heartbeat_key(node), stamp, ex=int(ttl)) + pipeline.set(storage_node_key(node), stamp, ex=int(STORAGE_NODE_TTL)) + pipeline.execute() + return True + except Exception as e: + # Not an error worth failing a probe over, but an operator chasing a red + # indicator against a healthy mount needs to find this line. + logger.warning("Could not publish the storage heartbeat for %s: %s", node, e) + return False + + +def probe_storage(workspaces_dir: str = "", node: str = "") -> bool: + """ + The rq-worker readiness probe, in one process. + + Ordering is the contract, and it is load-bearing in both directions: + + 1. `register_storage_node()` - unconditional, before anything can block, so + a node with a wedged mount still counts as expected. + 2. `touch_storage_sentinel()` - the only call that can block. If it fails + or never returns, the probe fails and no heartbeat is published. + 3. `write_storage_heartbeat()` - reached only from a mount that answered. + + Publishing is best effort in both directions: a Redis that is down says + nothing about the mount, so it must not fail the probe, and a mount that + did not answer must not publish regardless of how healthy Redis is. + + Args: + workspaces_dir: Directory holding the sentinel. Defaults to + `$WORKSPACES_DIR`. + node: Node name. Defaults to `current_node_name()`. + + Returns: + bool: True if the shared volume accepted a write. + """ + node = node or current_node_name() + + register_storage_node(node) + if not touch_storage_sentinel(workspaces_dir): + return False + write_storage_heartbeat(node) + return True + + +def get_storage_status() -> dict: + """ + Read the shared storage heartbeats for the sidebar indicator. + + Reads Redis and nothing else. Never stat the mount from here: on a `hard` + mount that call blocks in uninterruptible sleep and the indicator becomes + the hang it is supposed to report. + + Follows `get_queue_metrics()`'s distinction between "not applicable" and + "cannot tell": + + - `{}` - no `REDIS_URL`, or a Redis nothing has ever published a heartbeat + to. Both mean there is no shared storage being reported here and the + sidebar renders nothing. The second case is what keeps every non + Kubernetes deployment of the image quiet: `Dockerfile` bakes in both + `online_deployment` and a `REDIS_URL`, but the readiness probe that + writes the heartbeat exists only in `k8s/`, and a permanent red tick on + docker-compose would be exactly the wrong-layer alarm this indicator was + built to avoid. + - `{"available": False, "state": "unknown", ...}` - Redis is down, which + says nothing about the mount. Reporting that as unreachable storage sends + an operator to debug the wrong layer. + - `{"available": True, "state": "connected"}` - every expected node's mount + answered within its TTL. + - `{"available": True, "state": "degraded"}` - some did and some did not. + This is the case per node keys exist for: with one shared key, or by + collapsing the keys with `any()`, a healthy node would cover for a node + whose mount is wedged and the indicator would stay green through a real + outage. + - `{"available": True, "state": "unreachable"}` - no expected node's mount + answered. + + Returns: + dict: Empty when not applicable, otherwise `available`, `state`, + `nodes` (reporting a healthy mount), `stale_nodes` (expected but + silent) and, when Redis is down, `error`. + """ + redis_url = os.environ.get("REDIS_URL") + if not redis_url: + return {} + + try: + client = _redis_client(redis_url) + + # Test connection + client.ping() + + # SCAN, not KEYS: KEYS blocks the whole server for the length of the + # keyspace, and this runs from a sidebar fragment every 5 seconds. + healthy = _scan_suffixes(client, STORAGE_HEARTBEAT_PREFIX) + expected = _scan_suffixes(client, STORAGE_NODE_PREFIX) + except Exception as e: + return { + "available": False, + "state": "unknown", + "nodes": [], + "stale_nodes": [], + "error": str(e), + } + + # A heartbeat implies an expected node even if the registration expired + # first (a much longer TTL, so only reachable if Redis lost the key). + expected |= healthy + if not expected: + return {} + + stale = expected - healthy + if not healthy: + state = "unreachable" + elif stale: + state = "degraded" + else: + state = "connected" + + return { + "available": True, + "state": state, + "nodes": sorted(healthy), + "stale_nodes": sorted(stale), + } + + +def _scan_suffixes(client, prefix: str) -> set: + """Every key under `prefix`, with the prefix stripped off.""" + found = set() + for key in client.scan_iter(match=f"{prefix}*", count=100): + found.add(_as_text(key)[len(prefix):]) + found.discard("") + return found + + +def _cli(argv=None) -> int: + """ + Command line entry point, used by the rq-worker readiness probe. + + `--probe` is the probe itself: register the node, write to the shared + volume, and publish the heartbeat only if that write succeeded. `--status` + is for a human with a shell (`kubectl exec ... -- python -m + src.workflow.health --status`). + + There is deliberately no flag that publishes a heartbeat without checking + the mount. It would look like the obvious remedy for a red indicator and + would turn it green while the volume stayed wedged. + """ + import argparse + import json + + parser = argparse.ArgumentParser( + prog="python -m src.workflow.health", + description="Shared storage heartbeat for the OpenMS web app.", + ) + parser.add_argument( + "--probe", + action="store_true", + help=( + "confirm the shared volume accepts a write and publish this node's " + "storage heartbeat; exit 1 if the volume did not answer" + ), + ) + parser.add_argument( + "--status", + action="store_true", + help="print the storage status the sidebar renders, as JSON", + ) + args = parser.parse_args(argv) + + if args.status: + print(json.dumps(get_storage_status(), indent=2)) + return 0 + if args.probe: + if probe_storage(): + return 0 + directory = os.environ.get("WORKSPACES_DIR") or DEFAULT_WORKSPACES_DIR + print( + f"storage probe failed: {os.path.join(directory, STORAGE_SENTINEL_NAME)} " + "did not accept a write", + file=sys.stderr, + ) + return 1 + + parser.print_help() + return 2 + + +if __name__ == "__main__": + sys.exit(_cli()) diff --git a/src/workflow/settings_io.py b/src/workflow/settings_io.py new file mode 100644 index 00000000..f688b07b --- /dev/null +++ b/src/workflow/settings_io.py @@ -0,0 +1,151 @@ +""" +Streamlit free access to settings.json. + +Inside the RQ work horse there is no ScriptRunContext, so `st.session_state` is +an empty global mock and every setting read through session state silently +falls back to its hardcoded default there (docs/storage-defect-register.md G1). Code which has to +behave the same in the worker and in a Streamlit session reads the settings +from disk through this module instead. + +Kept dependency free on purpose: `src/workflow/tasks.py` runs inside the worker +and must stay importable without Streamlit. +""" + +import json +import logging +import math +from pathlib import Path + +logger = logging.getLogger(__name__) + +SETTINGS_FILE_NAME = "settings.json" + +CGROUP_V2_CPU_MAX = Path("/sys/fs/cgroup/cpu.max") +CGROUP_V1_CPU_QUOTA = Path("/sys/fs/cgroup/cpu/cpu.cfs_quota_us") +CGROUP_V1_CPU_PERIOD = Path("/sys/fs/cgroup/cpu/cpu.cfs_period_us") + + +def load_settings(path: Path | str = SETTINGS_FILE_NAME) -> dict: + """ + Read the app settings from disk. + + The path is resolved against the process working directory by default, + which is how every other reader in this repository resolves it (app.py:8, + src/common/common.py:362, test_gui.py:14). Deliberately not memoised: the + caller decides how long a value stays valid. + + Settings are only ever read here, never written back, so tolerating an + unreadable file cannot destroy it - unlike a workflow's params.json, see + ParameterManager.read_parameters_strict(). + + Total by construction: it never raises, whatever is on disk. The two other + readers in this repository (app.py, src/common/common.py) open settings.json + with the platform default encoding, so a file hand-edited on a cp1252 box + loads there; decoding with errors="replace" here keeps this loader from + being the one place that hard-fails on it - inside the RQ work horse, where + nothing catches it and the job dies. + + Args: + path: Location of the settings file. + + Returns: + dict: The parsed settings, or an empty dict if the file is absent, + unreadable or does not hold a JSON object. Callers are expected to + supply their own defaults per key. + """ + settings_file = Path(path) + try: + with open(settings_file, "r", encoding="utf-8", errors="replace") as f: + settings = json.load(f) + except FileNotFoundError: + logger.warning( + "No %s in %s, falling back to default settings.", settings_file, Path.cwd() + ) + return {} + except (ValueError, OSError) as e: + # ValueError covers json.JSONDecodeError and UnicodeDecodeError, which + # is a ValueError rather than an OSError - the gap that let an + # undecodable settings.json escape a handler documented as total. + logger.warning( + "Could not read %s (%s), falling back to default settings.", settings_file, e + ) + return {} + + if not isinstance(settings, dict): + logger.warning( + "%s does not hold a JSON object (found %s), falling back to default settings.", + settings_file, + type(settings).__name__, + ) + return {} + return settings + + +def _read_int(path: Path) -> int | None: + """Read a single integer out of a cgroup pseudo-file, or None.""" + try: + return int(path.read_text(encoding="utf-8").strip()) + except (OSError, ValueError): + return None + + +def detect_cpu_quota() -> int | None: + """ + Number of CPUs this container is actually allowed to use, from its cgroup. + + Returns None whenever no CPU ceiling can be established - not on Linux, not + in a container, or in a container with no quota set - and the caller then + falls back to its configured value. None is "do not know", never "one". + + Why this exists: the rq-worker runs at ``requests.cpu == limits.cpu`` + (Guaranteed QoS), so its pod spec IS its CPU allocation, and the two are + sized per deployment by the memory-tier components. A single + ``max_threads.online`` in settings.json cannot be right for both a 4 cpu + worker and a 20 cpu one, and the one that was there described neither. + + Both cgroup generations are handled: + + * v2 (``/sys/fs/cgroup/cpu.max``) holds ``" "``, or + ``"max "`` when unconstrained. + * v1 (``cpu.cfs_quota_us`` / ``cpu.cfs_period_us``) uses a quota of ``-1`` + to mean unconstrained. + + Rounded UP: Kubernetes turns ``cpu: 1500m`` into a quota of 1.5 cores, and + two runnable threads throttled to 1.5 cores finish sooner than one thread + using 1.0 and leaving 0.5 idle. + + Returns: + int | None: CPU count, at least 1, or None if no quota applies. + """ + quota_us: float | None = None + period_us: int | None = None + + try: + raw = CGROUP_V2_CPU_MAX.read_text(encoding="utf-8").strip() + except OSError: + raw = "" + if raw: + parts = raw.split() + # "max " is the unconstrained case and must read as None, not + # as a fallthrough to v1 - a host with both hierarchies mounted would + # otherwise answer from the wrong one. + if parts[0] == "max": + return None + try: + quota_us = float(parts[0]) + period_us = int(parts[1]) if len(parts) > 1 else 100000 + except (IndexError, ValueError): + logger.warning("Unparsable %s: %r", CGROUP_V2_CPU_MAX, raw) + return None + else: + v1_quota = _read_int(CGROUP_V1_CPU_QUOTA) + if v1_quota is None or v1_quota <= 0: + # -1 is the documented "no limit"; 0 is not a legal quota and is + # treated the same rather than propagated as a zero CPU budget. + return None + quota_us = float(v1_quota) + period_us = _read_int(CGROUP_V1_CPU_PERIOD) or 100000 + + if not quota_us or not period_us or period_us <= 0: + return None + return max(1, math.ceil(quota_us / period_us)) diff --git a/src/workflow/tasks.py b/src/workflow/tasks.py index 7c283bc6..52a34ec2 100644 --- a/src/workflow/tasks.py +++ b/src/workflow/tasks.py @@ -12,6 +12,17 @@ from pathlib import Path +class WorkflowExecutionError(RuntimeError): + """ + Raised when a workflow's execution() reports failure by returning False. + + RQ records a job as failed only if the job function raises, so an + application level failure has to be turned into an exception here. + Returning a result dict instead leaves the FailedJobRegistry empty and + health.py's failed_jobs metric structurally zero. + """ + + def execute_workflow( workflow_dir: str, workflow_class: str, @@ -30,6 +41,14 @@ def execute_workflow( Returns: Dictionary with execution results + + Raises: + Exception: Anything raised while setting up or running the workflow is + re-raised after the failure has been written to the workflow logs, + and a WorkflowExecutionError if execution() returned False. RQ + needs the job function to raise to record the job as failed. A + legacy execution() returning None is accepted as success with a + deprecation warning in the workflow log. """ try: from rq import get_current_job @@ -62,7 +81,10 @@ def execute_workflow( # Load parameters from saved params.json params_file = workflow_path / "params.json" if params_file.exists(): - with open(params_file, "r") as f: + # encoding="utf-8" explicitly, matching every other reader and + # writer of params.json: the platform codepage here would decode a + # file written elsewhere differently, or fail outright. + with open(params_file, "r", encoding="utf-8") as f: params = json.load(f) else: params = {} @@ -101,10 +123,35 @@ def execute_workflow( _update_progress(job, 0.15, "Executing workflow steps...") - # Execute the workflow - workflow.execution() - - # Log workflow completion + # Execute the workflow. It reports success with a bool + # (WorkflowManager.execution is declared -> bool), same as the local + # mode in WorkflowManager.workflow_process. + success = workflow.execution() + + if success is None: + # Migration shim for workflows written against the older example, + # which was annotated -> None and returned nothing. Those + # subclasses live in downstream repositories (quantms-web, + # umetaflow, FLASHApp) and cannot be fixed in the same commit as + # this file, so treating their None as a failure would report every + # successful queued run as failed the moment they rebase. Accepted + # as success, loudly, so they have a runway; explicit False is + # still a failure. + logger.log( + "WARNING: execution() returned None and was treated as success. " + "Annotate it '-> bool' and return True on success / False on " + "failure - see .claude/skills/create-workflow.md. A future " + "release will treat None as a failure.", + 0, + ) + elif not success: + raise WorkflowExecutionError( + f"Workflow execution reported failure (returned {success!r})" + ) + + # Log workflow completion. Only write the marker for a run which + # actually succeeded, it is what classify_log_outcome reads to tell a + # finished run from a failed one. logger.log("WORKFLOW FINISHED") _update_progress(job, 1.0, "Workflow completed") @@ -119,8 +166,6 @@ def execute_workflow( } except Exception as e: - error_msg = f"Workflow failed: {str(e)}\n{traceback.format_exc()}" - # Log error to workflow logs try: log_dir = workflow_path / "logs" @@ -130,7 +175,10 @@ def execute_workflow( log_file = log_dir / log_name with open(log_file, "a") as f: f.write(f"\n\nERROR: {str(e)}\n") - f.write(traceback.format_exc()) + # A workflow reporting failure by returning False is not a + # crash, the reason is in the tool output above. + if not isinstance(e, WorkflowExecutionError): + f.write(traceback.format_exc()) except Exception: pass @@ -141,11 +189,10 @@ def execute_workflow( except Exception: pass - return { - "success": False, - "workflow_dir": str(workflow_path), - "error": error_msg - } + # Re-raise so RQ moves the job into the FailedJobRegistry. Returning a + # {"success": False} dict here looked like a normal return to RQ, which + # recorded every crashed workflow as finished successfully. + raise def _update_progress(job, progress: float, step: str) -> None: diff --git a/tests/test_command_executor_run_python.py b/tests/test_command_executor_run_python.py new file mode 100644 index 00000000..f2c9dade --- /dev/null +++ b/tests/test_command_executor_run_python.py @@ -0,0 +1,184 @@ +""" +The ``run_python()`` result contract. + +``CommandExecutor.run_topp()`` returns a bool and ``execution()`` gates on it. +``run_python()`` looked like it did the same — the shipped workflow guards both +of its Python steps with ``if not self.executor.run_python(...)`` — but it was +annotated ``-> None`` and simply discarded the bool ``run_command()`` handed +back, at every one of its branches. So those guards were dead code: a Python +tool that exited non-zero still produced ``execution() == True``, +``tasks.execute_workflow`` still wrote the ``WORKFLOW FINISHED`` marker, and +``classify_log_outcome`` still reported "finished" next to the +``ERROR: Command failed with exit code`` line already in the log. + +That is invisible from ``tests/test_tasks.py``, which drives a ``MagicMock`` +executor: ``run_python.return_value = False`` is a value the real method could +not produce, so the parametrized "python-export-fails" case certified a code +path that could never run. These tests pin the real thing. +""" +import json +import os +import sys +from pathlib import Path +from unittest.mock import MagicMock + +import pytest + +# Add project root to path for imports +PROJECT_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +sys.path.append(PROJECT_ROOT) + +# --------------------------------------------------------------------------- +# Import with `streamlit` and `pyopenms` mocked at the sys.modules level, +# mirroring tests/test_topp_flag_parameters.py. run_python() needs neither: it +# reads params.json through ParameterManager and shells out through +# run_command(), which is stubbed below. +# --------------------------------------------------------------------------- +mock_streamlit = MagicMock() +mock_streamlit.session_state = {} + +_original_streamlit = sys.modules.get("streamlit") +_original_pyopenms = sys.modules.get("pyopenms") +sys.modules["streamlit"] = mock_streamlit +if _original_pyopenms is None: + sys.modules["pyopenms"] = MagicMock() + +from src.workflow.ParameterManager import ParameterManager +from src.workflow.CommandExecutor import CommandExecutor + +# Restore the original modules so other test files import the real ones. The +# classes imported above keep their module-level `st`/`poms` bound to the mocks. +if _original_streamlit is not None: + sys.modules["streamlit"] = _original_streamlit +else: + sys.modules.pop("streamlit", None) +if _original_pyopenms is None: + sys.modules.pop("pyopenms", None) + +for _key in list(sys.modules.keys()): + if _key.startswith("src.workflow"): + sys.modules.pop(_key, None) + + +SCRIPT_WITH_DEFAULTS = '''DEFAULTS = [ + {"key": "in", "value": [], "hide": True}, + {"key": "my-param", "value": 5, "name": "My Parameter"}, +] +''' + +SCRIPT_WITHOUT_DEFAULTS = "PLACEHOLDER = 1\n" + + +def _executor(tmp_path, run_command_result): + """A CommandExecutor whose run_command() reports the given exit status.""" + executor = CommandExecutor(tmp_path, MagicMock(), ParameterManager(tmp_path)) + executor.run_command = MagicMock(return_value=run_command_result) + return executor + + +def _write_tool(tmp_path, source=SCRIPT_WITH_DEFAULTS, name="example_tool.py"): + script = tmp_path / name + script.write_text(source, encoding="utf-8") + return script + + +@pytest.mark.parametrize("exit_status", [True, False], ids=["exit-0", "exit-non-zero"]) +def test_run_python_reports_the_scripts_exit_status(tmp_path, exit_status): + """ + The bool from run_command() has to reach the caller. + + Without it every `if not self.executor.run_python(...)` in execution() is + dead code and a failed Python tool is reported as a completed workflow. + """ + executor = _executor(tmp_path, exit_status) + script = _write_tool(tmp_path) + + result = executor.run_python(str(script), {"in": ["a.mzML"]}) + + assert result is exit_status, ( + f"run_python() returned {result!r} for a script that exited " + f"{'0' if exit_status else 'non-zero'}; it must return the same bool " + "run_topp() does, or execution() cannot tell the two apart" + ) + assert executor.run_command.call_count == 1 + + +def test_run_python_without_defaults_reports_the_exit_status(tmp_path): + """The no-DEFAULTS branch runs the script too, so it reports on it too.""" + executor = _executor(tmp_path, False) + script = _write_tool(tmp_path, SCRIPT_WITHOUT_DEFAULTS, "bare_tool.py") + + assert executor.run_python(str(script)) is False + assert executor.run_command.call_args.args[0] == ["python", str(script)] + + +def test_run_python_passes_the_parameters_file_and_cleans_it_up(tmp_path): + """ + The DEFAULTS branch writes a temporary params file, passes it as argv and + removes it. Pinned alongside the return value so returning early on failure + cannot leak it into the workflow directory. + """ + executor = _executor(tmp_path, False) + script = _write_tool(tmp_path) + + executor.run_python(str(script), {"in": ["a.mzML"]}) + + command = executor.run_command.call_args.args[0] + assert command[:2] == ["python", str(script)] + assert not Path(command[2]).exists(), ( + "the temporary parameter file was left behind in the workflow directory" + ) + + +def test_run_python_passes_input_output_and_stored_parameters(tmp_path): + """ + Guards the argv the exit status is reported *about*: DEFAULTS, overridden by + the stored params.json entries for this script, overridden by input_output. + """ + pm = ParameterManager(tmp_path) + script = _write_tool(tmp_path) + pm.params_file.write_text( + json.dumps({f"{script.name}:my-param": 42, "unrelated": 1}), encoding="utf-8" + ) + + executor = CommandExecutor(tmp_path, MagicMock(), pm) + captured = {} + + def capture(command): + captured["params"] = json.loads( + Path(command[2]).read_text(encoding="utf-8") + ) + return True + + executor.run_command = MagicMock(side_effect=capture) + + assert executor.run_python(str(script), {"in": ["a.mzML"]}) is True + assert captured["params"]["my-param"] == 42 + assert captured["params"]["in"] == ["a.mzML"] + assert "unrelated" not in captured["params"] + + +def test_missing_script_returns_false_instead_of_raising(tmp_path): + """ + A script that is nowhere to be found used to log and fall through into + importlib, raising out of execution() rather than reporting a failed step - + and in queue mode that traceback is all the user gets. + """ + executor = _executor(tmp_path, True) + + assert executor.run_python("definitely-not-a-real-tool") is False + executor.run_command.assert_not_called() + + +def test_defaults_of_the_wrong_shape_is_not_reported_as_success(tmp_path): + """ + DEFAULTS must be the documented list of parameter dicts. Anything else runs + nothing at all, which is a failed step, not a successful one - reporting it + as success would put the WORKFLOW FINISHED marker on a run that skipped a + step silently. + """ + executor = _executor(tmp_path, True) + script = _write_tool(tmp_path, 'DEFAULTS = {"my-param": 5}\n', "dict_tool.py") + + assert executor.run_python(str(script)) is False + executor.run_command.assert_not_called() diff --git a/tests/test_parameter_integrity.py b/tests/test_parameter_integrity.py new file mode 100644 index 00000000..b1a04111 --- /dev/null +++ b/tests/test_parameter_integrity.py @@ -0,0 +1,986 @@ +""" +Parameter-file integrity tests. + +These pin the contract that a *failed read* of ``params.json`` must never turn +into a *write* that erases it — the live data-loss bug recorded as A2 in +``docs/storage-defect-register.md`` and the first item of Step 1 in the +multi-node distribution plan. + +The bug, end to end: + +* ``ParameterManager.save_parameters()`` is a read-modify-write: + ``ParameterManager.py:136`` does ``self.get_parameters_from_json() | json_params``. +* ``get_parameters_from_json()`` (``:193-212``) wraps the read in a bare + ``except:`` that returns ``{}`` for *any* failure — a torn read from a + concurrent truncate-then-rewrite, an unreadable file, anything. +* So one transient bad read makes the merge ``{} | this_session_params``, and + that subset is written back, permanently deleting ``_defaults``, + ``_flag_params`` and every other tool's stored values — while telling the user + it was a deliberate "Reset to defaults". + +``apply_preset()`` (``:345``) has the identical shape and is covered here too. +The plan's fix is to split the read: a tolerant ``get_parameters_from_json()`` +that keeps returning ``{}`` for its read-only callers, and a strict +``read_parameters_strict()`` that **raises** on a malformed file and returns +``{}`` only for a genuinely absent one, used by the read-modify-write-back +sites. Conflating "absent" with "unreadable" *is* the bug, so both branches get +their own test. + +The next group pins the shapes an unreadable file actually takes on a shared, +restartable filesystem — a bad encoding, JSON that is not an object, a directory +in its place, and a storage blip that clears by itself — plus the two halves of +the recovery: the write must be atomic, and the reset offered on failure must +not be the thing that destroys the data. + +The last two groups cover G1: ``CommandExecutor._get_max_threads()`` reads +settings out of ``st.session_state``, which is empty inside the RQ work horse, +so the configured ``max_threads`` has been silently replaced by a hardcoded +``4``; and ``src/workflow/settings_io.load_settings()``, the Streamlit-free +loader it now reads them through. + +Every test here is expected to fail before Step 1 lands. +""" +import builtins +import contextlib +import errno +import json +import os +import sys +import time +from pathlib import Path +from unittest.mock import MagicMock + +import pytest + +# Add project root to path for imports +PROJECT_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +sys.path.append(PROJECT_ROOT) + +# --------------------------------------------------------------------------- +# Import the modules under test with `streamlit` (and `pyopenms`) mocked at the +# sys.modules level, mirroring tests/test_topp_flag_parameters.py. Both modules +# do `import streamlit as st` at the top (ParameterManager also +# `import pyopenms as poms`); nothing exercised here needs more than a dict-like +# `st.session_state`, so the mocks keep the suite runnable without the heavy +# deps while still driving the real read/merge/write logic. +# --------------------------------------------------------------------------- +mock_streamlit = MagicMock() +mock_streamlit.session_state = {} + +_original_streamlit = sys.modules.get("streamlit") +_original_pyopenms = sys.modules.get("pyopenms") +sys.modules["streamlit"] = mock_streamlit +if _original_pyopenms is None: + sys.modules["pyopenms"] = MagicMock() + +from src.workflow.ParameterManager import ParameterManager +from src.workflow.CommandExecutor import CommandExecutor +from src.workflow.settings_io import load_settings + +# Restore the original modules so other test files import the real ones. The +# classes imported above keep their module-level `st`/`poms` bound to the mocks. +if _original_streamlit is not None: + sys.modules["streamlit"] = _original_streamlit +else: + sys.modules.pop("streamlit", None) +if _original_pyopenms is None: + sys.modules.pop("pyopenms", None) + +for _key in list(sys.modules.keys()): + if _key.startswith("src.workflow"): + sys.modules.pop(_key, None) + + +# Name of the strict, raising reader introduced by Step 1 of the plan. +STRICT_READER_NAME = "read_parameters_strict" + +# Reached through the class rather than imported, because the sys.modules +# entries for src.workflow.* are popped after the import block above - and +# because the name does not exist before Step 1, so importing it at module +# scope would turn this file into a collection error instead of a failure. +_MODULE_GLOBALS = ParameterManager.__init__.__globals__ + + +def _error_class(name: str) -> type: + """The named exception class from ParameterManager, or fail loudly.""" + cls = _MODULE_GLOBALS.get(name) + if cls is None: + pytest.fail( + f"ParameterManager.{name} does not exist. Step 1 of the plan " + "requires a read failure to be reported as its own exception type, " + "so that every read-modify-write-back site can abort on it." + ) + return cls + + +def invalid_parameter_file_error() -> type: + """The exception a failed strict read raises.""" + return _error_class("InvalidParameterFileError") + + +def transient_parameter_file_error() -> type: + """ + The subclass used for a read failure expected to clear by itself. + + A restarted NFS export makes clients ESTALE for a moment. Reporting that + the same way as a corrupt file sends the user to a reset affordance which + would delete a perfectly intact params.json. + """ + return _error_class("TransientParameterFileError") + + +# The one general parameter whose widget is on screen in the tests below, i.e. +# the only key the saving session legitimately owns. +CHANGED_KEY = "example-general-param" + +# A realistic, fully populated params.json: the reserved keys, two TOPP tools' +# stored values, and two general parameters. A session that is not currently +# rendering the TOPP widgets (a different page, a collapsed fragment, advanced +# view off) holds *none* of this in session state — which is exactly why +# save_parameters() merges the file back in, and exactly what a laundered `{}` +# destroys. +FULL_PARAMS = { + "_defaults": { + "FeatureFinderMetabo": {"algorithm:common:noise_threshold_int": 1000.0}, + "MetaboliteAdductDecharger": { + "algorithm:MetaboliteFeatureDeconvolution:charge_max": 3 + }, + }, + "_flag_params": {"FeatureFinderMetabo": ["force"]}, + "max_threads": 3, + "FeatureFinderMetabo": { + "algorithm:common:noise_threshold_int": 500.0, + "algorithm:mtd:mass_error_ppm": 10.0, + }, + "MetaboliteAdductDecharger": { + "algorithm:MetaboliteFeatureDeconvolution:charge_max": 2 + }, + CHANGED_KEY: "keep-me", + "run-adduct-detection": True, +} + + +@pytest.fixture(autouse=True) +def reset_session_state(): + """Give each test a fresh, empty mocked session_state.""" + mock_streamlit.session_state = {} + yield + mock_streamlit.session_state = {} + + +def _assert_unparseable(text: str) -> None: + """Guard the fixtures themselves: the corruption must really break json.""" + with pytest.raises(json.JSONDecodeError): + json.loads(text) + + +def _write_torn_params_file(pm: ParameterManager) -> str: + """ + Leave params.json holding FULL_PARAMS plus a fragment of a second write. + + This is what a reader sees when a concurrent writer's truncate-then-rewrite + interleaves with it: unparseable, but with every original byte still on + disk and recoverable by hand — until something rewrites the file. + + Returns the full text that was written. + """ + text = json.dumps(FULL_PARAMS, indent=4) + '\n{\n "_defaults": {\n' + _assert_unparseable(text) + pm.params_file.write_text(text, encoding="utf-8") + return text + + +def _require_strict_reader(pm: ParameterManager): + """ + Return the strict reader from Step 1 of the plan, or fail loudly. + + Accepts either a ``ParameterManager`` method or a module-level function + taking the params file path, since only the name and the behaviour are + pinned by the plan. Uses ``pytest.fail`` (a BaseException) rather than + letting an AttributeError escape, so that a ``pytest.raises(Exception)`` + around the call cannot mistake "not implemented yet" for "raised properly". + """ + reader = getattr(pm, STRICT_READER_NAME, None) + if reader is not None: + return reader + + # sys.modules entries for src.workflow.* are popped after import above, so + # reach the module namespace through the class instead. + module_globals = ParameterManager.__init__.__globals__ + fn = module_globals.get(STRICT_READER_NAME) + if fn is not None: + return lambda: fn(pm.params_file) + + pytest.fail( + f"ParameterManager.{STRICT_READER_NAME}() does not exist. Step 1 of the " + "plan requires a strict reader that raises on a malformed params.json " + "and returns {} only for an absent one, used by every " + "read-modify-write-back site." + ) + + +# --------------------------------------------------------------------------- +# The erasure bug (docs/storage-defect-register.md A2) +# --------------------------------------------------------------------------- + + +def test_corrupt_read_does_not_erase(tmp_path): + """ + A session holding a subset of the keys must not be able to delete the rest. + + Sequence: params.json holds two tools' values plus `_defaults` and + `_flag_params`; a second process's write leaves the file momentarily + unparseable; this session — which only ever rendered one general widget — + saves. Whether save_parameters() refuses loudly or quietly is not pinned + here; what is pinned is that nothing on disk is lost. + """ + pm = ParameterManager(tmp_path) + torn_text = _write_torn_params_file(pm) + + # Only one key is in session state: the widget this page actually rendered. + mock_streamlit.session_state = {f"{pm.param_prefix}{CHANGED_KEY}": "changed"} + + # Narrow on purpose: aborting the write is the point, but it must abort on + # the read failure and nothing else. A blanket suppress would let any crash + # satisfy the assertions below. + with contextlib.suppress(invalid_parameter_file_error()): + pm.save_parameters() + + surviving_text = pm.params_file.read_text(encoding="utf-8") + + try: + surviving = json.loads(surviving_text) + except json.JSONDecodeError: + # The file was left alone: still corrupt, but every byte is there and + # it can be repaired. Nothing was laundered away. + assert surviving_text == torn_text, ( + "params.json was partially rewritten after a failed read; it must " + "be left intact so the stored parameters remain recoverable" + ) + return + + # The file was rewritten, so it has to be at least as complete as before. + for key, value in FULL_PARAMS.items(): + assert key in surviving, ( + f"'{key}' was erased from params.json by a save that could not read " + "it first (docs/storage-defect-register.md A2): the failed read became an empty dict and " + f"the merge wrote back only this session's subset {sorted(surviving)}" + ) + if key != CHANGED_KEY: + assert surviving[key] == value, ( + f"'{key}' lost its stored value after a failed read: " + f"{surviving[key]!r} != {value!r}" + ) + + +def test_truncated_read_does_not_rewrite_params_file(tmp_path): + """ + Same bug, second corruption shape: a half-written file. + + Here the tail of the original content is genuinely gone from disk, so the + only safe behaviour is to touch nothing — a rewrite would replace a + repairable file with a valid-JSON file that has silently lost everything. + """ + pm = ParameterManager(tmp_path) + full_text = json.dumps(FULL_PARAMS, indent=4) + half_written = full_text[: len(full_text) // 2] + _assert_unparseable(half_written) + pm.params_file.write_text(half_written, encoding="utf-8") + before = pm.params_file.read_bytes() + + mock_streamlit.session_state = {f"{pm.param_prefix}{CHANGED_KEY}": "changed"} + + with contextlib.suppress(invalid_parameter_file_error()): + pm.save_parameters() + + assert pm.params_file.read_bytes() == before, ( + "params.json was rewritten from a session subset after a torn read; " + "the unreadable file must be preserved, not replaced" + ) + + +def test_apply_preset_does_not_erase_on_corrupt_read(tmp_path, monkeypatch): + """ + apply_preset() is the second read-modify-write-back site (:345). + + It has the same shape as save_parameters() — read, merge, write — so an + unreadable file makes applying a preset delete every parameter the preset + does not mention, and report success while doing it. + """ + workflow_dir = tmp_path / "topp-workflow" + workflow_dir.mkdir() + pm = ParameterManager(workflow_dir, workflow_name="TOPP Workflow") + torn_text = _write_torn_params_file(pm) + + # presets.json is read relative to the process CWD (ParameterManager.py:287). + cwd = tmp_path / "app-root" + cwd.mkdir() + (cwd / "presets.json").write_text( + json.dumps( + { + "topp-workflow": { + "High Sensitivity": { + "_description": "Tooltip text", + "FeatureFinderMetabo": { + "algorithm:common:noise_threshold_int": 50.0 + }, + } + } + } + ), + encoding="utf-8", + ) + monkeypatch.chdir(cwd) + + result = None + with contextlib.suppress(invalid_parameter_file_error()): + result = pm.apply_preset("High Sensitivity") + + assert result is not True, ( + "apply_preset() reported success against a params.json it could not read" + ) + + surviving_text = pm.params_file.read_text(encoding="utf-8") + try: + surviving = json.loads(surviving_text) + except json.JSONDecodeError: + assert surviving_text == torn_text, ( + "params.json was partially rewritten while applying a preset over " + "an unreadable file" + ) + return + + for key in FULL_PARAMS: + assert key in surviving, ( + f"'{key}' was erased by apply_preset() after a failed read; the " + f"file now holds only {sorted(surviving)}" + ) + + +# --------------------------------------------------------------------------- +# Absent vs unreadable — the distinction whose absence causes the erasure +# --------------------------------------------------------------------------- + + +def test_absent_parameter_file_reads_as_empty(tmp_path): + """An absent params.json is a first run, not a failure: {} is correct.""" + pm = ParameterManager(tmp_path) + assert not pm.params_file.exists() + + assert pm.get_parameters_from_json() == {} + assert _require_strict_reader(pm)() == {}, ( + "the strict reader must still return {} for a genuinely absent file — " + "a first run has no parameters and that is not an error" + ) + + +def test_malformed_parameter_file_does_not_read_as_empty(tmp_path): + """ + A malformed params.json must be distinguishable from an absent one. + + Returning {} for both is what lets save_parameters() merge an empty dict + over real data. The tolerant reader keeps its {} contract for its read-only + callers; the strict reader used by the write-back sites must raise. + """ + pm = ParameterManager(tmp_path) + _write_torn_params_file(pm) + + reader = _require_strict_reader(pm) + with pytest.raises(invalid_parameter_file_error()): + result = reader() + pytest.fail( + f"the strict reader returned {result!r} for an unreadable " + "params.json instead of raising; an unreadable file must never be " + "reported as 'no parameters'" + ) + + # Deliberate asymmetry: the tolerant reader keeps its signature and its {} + # for the constructors and read-only callers that cannot handle a raise. + assert pm.get_parameters_from_json() == {} + + +# --------------------------------------------------------------------------- +# max_threads outside a Streamlit session (docs/storage-defect-register.md G1) +# --------------------------------------------------------------------------- + + +def _executor_in_worker( + tmp_path, monkeypatch, settings, params_json=None, cpu_quota=None +): + """ + Build a CommandExecutor under the RQ work horse's real conditions. + + `cpu_quota` pins what detect_cpu_quota() reports. It defaults to None — "no + readable CPU ceiling" — so the settings-driven tests below assert the + setting rather than whatever cgroup the test host happens to have. Left + unpinned they would pass on an unconstrained runner and fail the moment CI + ran inside a container with a quota, which is a fixture reading the + environment rather than a test. + + No ScriptRunContext means `st.session_state` is an empty global mock, so + `st.session_state.get("settings", {})` yields {} and every configured value + has to come from settings.json, which this repo resolves against the process + CWD everywhere it is read (app.py:8, common.py:362, QueueManager.py:74, + test_gui.py:14). The chdir happens before the executor is built so that + memoising the lookup — which the plan requires, since run_topp() calls it + twice and the two calls must agree — cannot serve a value read from some + other directory. + """ + cwd = tmp_path / "app-root" + cwd.mkdir() + (cwd / "settings.json").write_text(json.dumps(settings), encoding="utf-8") + + workflow_dir = tmp_path / "workflow" + workflow_dir.mkdir() + + monkeypatch.chdir(cwd) + monkeypatch.delenv("REDIS_URL", raising=False) + # Reached through the function's own globals, not by dotted name: the + # import block above pops src.workflow.* from sys.modules, so a string + # target would re-import a *different* CommandExecutor module - one bound + # to the real streamlit - and patch a copy this executor never calls. + monkeypatch.setitem( + CommandExecutor._get_max_threads.__globals__, + "detect_cpu_quota", + lambda: cpu_quota, + ) + + pm = ParameterManager(workflow_dir) + if params_json is not None: + pm.params_file.write_text(json.dumps(params_json), encoding="utf-8") + + mock_streamlit.session_state = {} + return CommandExecutor(workflow_dir, MagicMock(), pm) + + +def test_max_threads_without_session_state(tmp_path, monkeypatch): + """ + The worker must honour max_threads.online, not the hardcoded literal 4. + + `st.session_state.get("settings", {})` is {} inside the work horse, so the + online branch of _get_max_threads() (CommandExecutor.py:41) is unreachable + there and the local branch's hardcoded 4 wins — dead config since #333 + (docs/storage-defect-register.md G1), and a correctness bug once a worker has a hard memory + limit sized for a known thread count. + """ + executor = _executor_in_worker( + tmp_path, + monkeypatch, + {"online_deployment": True, "max_threads": {"local": 9, "online": 6}}, + ) + + assert executor._get_max_threads() == 6, ( + "max_threads.online was ignored outside a Streamlit session; a value of " + "4 means the hardcoded literal was used instead of settings.json" + ) + + +def test_max_threads_local_without_session_state_uses_configured_local( + tmp_path, monkeypatch +): + """Same failure in local mode: the configured local value, not the literal 4.""" + executor = _executor_in_worker( + tmp_path, + monkeypatch, + {"online_deployment": False, "max_threads": {"local": 9, "online": 6}}, + ) + + assert executor._get_max_threads() == 9, ( + "max_threads.local from settings.json was ignored; 4 is the hardcoded " + "fallback that only applies when the setting is missing entirely" + ) + + +def test_max_threads_online_ignores_workspace_params_json(tmp_path, monkeypatch): + """ + In online mode the workspace's params.json must not set the thread budget. + + Workers are shared and params.json is user-supplied — the import uploader + writes it verbatim, with no reserved-key filter (docs/storage-defect-register.md G2) — so the + per-workspace value may only apply to the local, single-user deployment. + """ + executor = _executor_in_worker( + tmp_path, + monkeypatch, + {"online_deployment": True, "max_threads": {"local": 9, "online": 6}}, + params_json={"max_threads": 3, "example-general-param": "x"}, + ) + + assert executor._get_max_threads() == 6, ( + "a workspace params.json overrode max_threads.online on a shared worker" + ) + + +# --------------------------------------------------------------------------- +# The shapes an unreadable params.json actually takes +# --------------------------------------------------------------------------- + + +def test_undecodable_parameter_file_does_not_read_as_empty(tmp_path): + """ + A params.json that is not valid UTF-8 is unreadable, not empty. + + UnicodeDecodeError subclasses ValueError, not OSError, so a handler + catching (JSONDecodeError, OSError) misses it entirely: it escapes the + tolerant reader - which both constructors call, taking the whole page down + - and escapes the strict reader as a type no `except + InvalidParameterFileError` guard catches, so save_parameters() loses its + "left unchanged" protection. + + Reachable in this repo: the parameter import uploader decodes the upload as + utf-8 and writes it back out, and a torn write can also cut a multi-byte + sequence in half. + """ + pm = ParameterManager(tmp_path) + # cp1252 bytes for {"note": "\u00b5m tolerance"} - 0xb5 is not valid UTF-8. + pm.params_file.write_bytes(b'{"note": "\xb5m tolerance"}') + before = pm.params_file.read_bytes() + + with pytest.raises(invalid_parameter_file_error()): + _require_strict_reader(pm)() + + assert pm.get_parameters_from_json() == {}, ( + "the tolerant reader must keep degrading to {} - its two callers are " + "constructors with no way to handle a raise" + ) + + mock_streamlit.session_state = {f"{pm.param_prefix}{CHANGED_KEY}": "changed"} + with contextlib.suppress(invalid_parameter_file_error()): + pm.save_parameters() + + assert pm.params_file.read_bytes() == before, ( + "params.json was rewritten from a session subset after an undecodable " + "read" + ) + + +@pytest.mark.parametrize( + "content", ["[1, 2, 3]", '"just a string"', "42", "null"], ids=["array", "string", "number", "null"] +) +def test_parameter_file_that_is_not_an_object_is_rejected(tmp_path, content): + """ + Valid JSON that is not an object is still not a parameter dict. + + ``{} | ["a"]`` raises and ``{} | "x"`` raises, but a JSON ``null`` merges as + nothing at all - so without this guard the strict reader would hand a + non-dict to the merge and either crash the run or quietly erase the file. + """ + pm = ParameterManager(tmp_path) + pm.params_file.write_text(content, encoding="utf-8") + + with pytest.raises(invalid_parameter_file_error()): + _require_strict_reader(pm)() + + assert pm.get_parameters_from_json() == {} + + +def test_directory_in_place_of_parameter_file_is_rejected(tmp_path): + """ + A directory where params.json should be is unreadable, not absent. + + Raises IsADirectoryError on Linux and PermissionError on Windows; both are + OSError, and neither may be reported as "no parameters stored". + """ + pm = ParameterManager(tmp_path) + pm.params_file.mkdir() + + with pytest.raises(invalid_parameter_file_error()): + _require_strict_reader(pm)() + + assert pm.get_parameters_from_json() == {} + assert pm.params_file.is_dir(), "the directory must be left in place" + + +def _flaky_open(monkeypatch, target: Path, failures: int, errno_value: int) -> None: + """Make the first ``failures`` opens of ``target`` fail with ``errno_value``.""" + real_open = builtins.open + remaining = {"n": failures} + + def fake_open(file, *args, **kwargs): + if ( + isinstance(file, (str, os.PathLike)) + and Path(file) == target + and remaining["n"] > 0 + ): + remaining["n"] -= 1 + raise OSError(errno_value, os.strerror(errno_value)) + return real_open(file, *args, **kwargs) + + monkeypatch.setattr(builtins, "open", fake_open) + # The retry delay is real time; nothing here depends on its duration. + monkeypatch.setattr(time, "sleep", lambda _seconds: None) + + +def test_transient_read_failure_is_retried(tmp_path, monkeypatch): + """ + ESTALE from a restarted NFS export must not be reported as corruption. + + Invariant 2 of the plan expects the Ganesha pod to be restarted by + upgrades, evictions and node drains. Every client ESTALEs while it comes + back. With no retry, a routine restart puts the user in front of a "reset + to defaults" button whose only effect would be to delete an intact file. + """ + pm = ParameterManager(tmp_path) + pm.params_file.write_text(json.dumps(FULL_PARAMS, indent=4), encoding="utf-8") + _flaky_open(monkeypatch, pm.params_file, failures=1, errno_value=errno.ESTALE) + + assert _require_strict_reader(pm)() == FULL_PARAMS, ( + "a single ESTALE must be retried, not turned into a permanent failure" + ) + + +def test_persistent_transient_failure_is_reported_as_transient(tmp_path, monkeypatch): + """ + A storage outage that outlasts the retries is still not corruption. + + It stays an InvalidParameterFileError - every write-back site must still + abort, the content is unknown - but as a distinguishable subclass, so the + UI can withhold the destructive reset affordance. + """ + pm = ParameterManager(tmp_path) + pm.params_file.write_text(json.dumps(FULL_PARAMS, indent=4), encoding="utf-8") + _flaky_open(monkeypatch, pm.params_file, failures=99, errno_value=errno.ESTALE) + + with pytest.raises(transient_parameter_file_error()): + _require_strict_reader(pm)() + + assert issubclass( + transient_parameter_file_error(), invalid_parameter_file_error() + ), ( + "it must stay a subclass, or every existing `except " + "InvalidParameterFileError` write-back guard stops catching it" + ) + + +def test_permanent_read_failure_is_not_reported_as_transient(tmp_path): + """The control: a corrupt file must not be misreported as a passing blip.""" + pm = ParameterManager(tmp_path) + _write_torn_params_file(pm) + + with pytest.raises(invalid_parameter_file_error()) as excinfo: + _require_strict_reader(pm)() + + assert not isinstance(excinfo.value, transient_parameter_file_error()), ( + "unparseable content will never clear by itself; offering 'try again' " + "for it leaves the user with no way forward" + ) + + +def test_interrupted_write_leaves_the_previous_parameters_intact(tmp_path, monkeypatch): + """ + A write that dies half way must not leave a file that is neither version. + + Truncate-then-rewrite plus an OOM kill or ENOSPC - the failure modes this + deployment is being sized against - leaves a torn params.json which every + later strict read rejects. save_parameters() then becomes a permanent + silent no-op and the parameter page stops on every render, with only the + destructive reset left as an escape. + """ + pm = ParameterManager(tmp_path) + pm.params_file.write_text(json.dumps(FULL_PARAMS, indent=4), encoding="utf-8") + before = pm.params_file.read_bytes() + + def half_written_dump(obj, fp, **kwargs): + fp.write('{\n "_defaults": {') + raise OSError(errno.ENOSPC, os.strerror(errno.ENOSPC)) + + monkeypatch.setattr(json, "dump", half_written_dump) + mock_streamlit.session_state = {f"{pm.param_prefix}{CHANGED_KEY}": "changed"} + + with pytest.raises(OSError): + pm.save_parameters() + + monkeypatch.undo() + + assert pm.params_file.read_bytes() == before, ( + "params.json holds a half-written document; the write must go to a " + "temporary file and be moved into place with os.replace()" + ) + assert json.loads(pm.params_file.read_text(encoding="utf-8")) == FULL_PARAMS + leftovers = [q.name for q in tmp_path.iterdir() if q.name.endswith(".tmp")] + assert leftovers == [], f"scratch files left behind: {leftovers}" + + +def test_reset_keeps_the_previous_parameters(tmp_path): + """ + The recovery offered for an unreadable file must not be the data loss. + + reset_to_default_parameters() backs the reading UI's "reset to defaults" + button, which is offered exactly when the file could not be read - and an + unreadable file is very often a healthy file behind a storage blip. + Deleting it there converts a self-healing outage into permanent loss. + """ + pm = ParameterManager(tmp_path) + torn_text = _write_torn_params_file(pm) + + backup = pm.reset_to_default_parameters() + + assert not pm.params_file.exists(), "the workflow must start from defaults" + assert backup is not None and backup.exists(), ( + "the previous parameter file must be kept, not unlinked" + ) + assert backup.read_text(encoding="utf-8") == torn_text + assert pm.get_parameters_from_json() == {} + + +def test_reset_handles_a_directory_in_place_of_the_parameter_file(tmp_path): + """ + unlink() raises on a directory, so the reset button used to raise too - + leaving the parameter page stopped on every render with no way out. + """ + pm = ParameterManager(tmp_path) + pm.params_file.mkdir() + (pm.params_file / "unexpected.txt").write_text("x", encoding="utf-8") + + backup = pm.reset_to_default_parameters() + + assert not pm.params_file.exists() + assert backup is not None and backup.is_dir() + + +def test_reset_with_no_stored_parameters_is_a_no_op(tmp_path): + """Nothing stored, nothing to preserve - and no crash.""" + pm = ParameterManager(tmp_path) + assert pm.reset_to_default_parameters() is None + assert not pm.params_file.exists() + + +# --------------------------------------------------------------------------- +# settings_io.load_settings - the Streamlit-free loader _get_max_threads uses +# --------------------------------------------------------------------------- + + +def test_load_settings_reads_the_file_in_the_working_directory(tmp_path, monkeypatch): + """The default path resolves against the CWD, as every other reader does.""" + monkeypatch.chdir(tmp_path) + (tmp_path / "settings.json").write_text( + json.dumps({"online_deployment": True}), encoding="utf-8" + ) + + assert load_settings() == {"online_deployment": True} + + +def test_load_settings_accepts_an_explicit_path(tmp_path): + elsewhere = tmp_path / "config" / "settings.json" + elsewhere.parent.mkdir() + elsewhere.write_text(json.dumps({"max_threads": {"online": 6}}), encoding="utf-8") + + assert load_settings(elsewhere) == {"max_threads": {"online": 6}} + + +def test_load_settings_absent_file_is_empty(tmp_path): + assert load_settings(tmp_path / "nothing-here.json") == {} + + +@pytest.mark.parametrize( + "content", + ["{not json", "[1, 2, 3]", '"a string"', ""], + ids=["malformed", "array", "string", "empty"], +) +def test_load_settings_unusable_content_is_empty(tmp_path, content): + """ + Total by construction, as its docstring promises and QueueManager relies on. + + QueueManager._load_settings() delegates here and is documented to return an + empty dict on failure; it is called from QueueManager.__init__, which + WorkflowManager.__init__ calls behind an `except ImportError` that would not + catch anything else. CommandExecutor._get_max_threads() calls it inside the + RQ work horse, where a raise kills the job. + """ + settings_file = tmp_path / "settings.json" + settings_file.write_text(content, encoding="utf-8") + + assert load_settings(settings_file) == {} + + +def test_load_settings_survives_a_non_utf8_file(tmp_path): + """ + A settings.json in the platform codepage must not be fatal here alone. + + app.py and src/common/common.py open it with the platform default encoding, + so a file hand-edited on a Windows box loads in the UI. If this loader is + the only strict reader, the same file starts the app and then kills the RQ + worker - the exact split it was written to remove. + """ + settings_file = tmp_path / "settings.json" + # cp1252 for {"app-name": "M\u00fcller Lab", "online_deployment": true} + settings_file.write_bytes( + b'{"app-name": "M\xfcller Lab", "online_deployment": true}' + ) + + settings = load_settings(settings_file) + + assert settings.get("online_deployment") is True, ( + "an undecodable byte in one value must not lose the whole file" + ) + + +def test_load_settings_unreadable_file_is_empty(tmp_path): + """A directory in place of settings.json is an OSError, not a crash.""" + settings_file = tmp_path / "settings.json" + settings_file.mkdir() + + assert load_settings(settings_file) == {} + + +def test_max_threads_is_memoised(tmp_path, monkeypatch): + """ + run_topp() consults _get_max_threads() twice - once for its own thread + split, once inside run_multiple_commands() - and the two must agree, or the + per-command budget and the parallelism are computed from different numbers. + """ + executor = _executor_in_worker( + tmp_path, + monkeypatch, + {"online_deployment": True, "max_threads": {"local": 9, "online": 6}}, + ) + + first = executor._get_max_threads() + Path("settings.json").write_text( + json.dumps({"online_deployment": True, "max_threads": {"online": 1}}), + encoding="utf-8", + ) + + assert executor._get_max_threads() == first, ( + "the thread budget was re-read mid-run; the two call sites in run_topp() " + "would then disagree" + ) + + +# --------------------------------------------------------------------------- +# max_threads from the container's cgroup CPU quota +# +# The rq-worker runs at requests.cpu == limits.cpu (Guaranteed QoS), and the +# memory-tier components size that per deployment - 4 cpu on the low tier, 20 +# on the high one. A single max_threads.online in settings.json cannot describe +# both, so where a real quota is readable it wins, and the setting stays as the +# fallback for deployments that have none. +# --------------------------------------------------------------------------- + + +def _settings_io_ns(): + """ + The real detect_cpu_quota and the module globals it reads its paths from. + + Same reason as the monkeypatch in _executor_in_worker: src.workflow.* is + popped from sys.modules by the import block at the top of this file, so + `from src.workflow import settings_io` would build a SECOND module object + and patching its constants would leave the function under test reading the + real /sys/fs/cgroup. Going through __globals__ reaches the namespace the + live function actually closes over. + """ + detect = CommandExecutor._get_max_threads.__globals__["detect_cpu_quota"] + return detect, detect.__globals__ + + +def test_cpu_quota_beats_configured_online_value(tmp_path, monkeypatch): + """A readable quota is the allocation; max_threads.online is only a guess.""" + executor = _executor_in_worker( + tmp_path, + monkeypatch, + {"online_deployment": True, "max_threads": {"local": 9, "online": 2}}, + cpu_quota=20, + ) + + assert executor._get_max_threads() == 20, ( + "the container's own CPU quota was ignored in favour of a static " + "max_threads.online, which is wrong on every tier it was not written for" + ) + + +def test_cpu_quota_ignored_in_local_mode(tmp_path, monkeypatch): + """ + Local mode keeps its params.json override. + + The Threads widget is rendered only locally and writing it must keep + working, so quota detection has to stay confined to the online branch. + """ + executor = _executor_in_worker( + tmp_path, + monkeypatch, + {"online_deployment": False, "max_threads": {"local": 9, "online": 2}}, + params_json={"max_threads": 3}, + cpu_quota=20, + ) + + assert executor._get_max_threads() == 3, ( + "cgroup detection leaked into local mode and overrode the user's " + "Threads setting" + ) + + +def test_configured_online_value_used_when_no_quota(tmp_path, monkeypatch): + """No quota - bare metal, an unlimited container - falls back to settings.""" + executor = _executor_in_worker( + tmp_path, + monkeypatch, + {"online_deployment": True, "max_threads": {"local": 9, "online": 6}}, + cpu_quota=None, + ) + + assert executor._get_max_threads() == 6 + + +@pytest.mark.parametrize( + "cpu_max, expected", + [ + ("400000 100000", 4), # cpu: 4 + ("2000000 100000", 20), # cpu: 20, the high tier + ("150000 100000", 2), # cpu: 1500m rounds UP, not down + ("50000 100000", 1), # cpu: 500m floors at 1, never 0 + ("max 100000", None), # unconstrained + ("garbage", None), + ("", None), + ], +) +def test_detect_cpu_quota_cgroup_v2(tmp_path, monkeypatch, cpu_max, expected): + """cgroup v2 writes ' ', or 'max ' when unlimited.""" + detect, g = _settings_io_ns() + + cpu_file = tmp_path / "cpu.max" + cpu_file.write_text(cpu_max, encoding="utf-8") + monkeypatch.setitem(g, "CGROUP_V2_CPU_MAX", cpu_file) + # Point v1 at somewhere absent, so a fallthrough cannot be mistaken for a + # v2 read succeeding. + monkeypatch.setitem(g, "CGROUP_V1_CPU_QUOTA", tmp_path / "nope" / "cfs_quota_us") + + assert detect() == expected + + +@pytest.mark.parametrize( + "quota, period, expected", + [ + ("400000", "100000", 4), + ("-1", "100000", None), # v1's documented "no limit" + ("0", "100000", None), # not a legal quota; never a 0 thread budget + ], +) +def test_detect_cpu_quota_cgroup_v1(tmp_path, monkeypatch, quota, period, expected): + """cgroup v1 splits the same information across two files, -1 meaning none.""" + detect, g = _settings_io_ns() + + quota_file = tmp_path / "cpu.cfs_quota_us" + period_file = tmp_path / "cpu.cfs_period_us" + quota_file.write_text(quota, encoding="utf-8") + period_file.write_text(period, encoding="utf-8") + monkeypatch.setitem(g, "CGROUP_V2_CPU_MAX", tmp_path / "absent-cpu.max") + monkeypatch.setitem(g, "CGROUP_V1_CPU_QUOTA", quota_file) + monkeypatch.setitem(g, "CGROUP_V1_CPU_PERIOD", period_file) + + assert detect() == expected + + +def test_detect_cpu_quota_absent_everywhere(tmp_path, monkeypatch): + """ + Off Linux, or outside a container, the answer is None - never 1. + + A 1 here would silently serialise every workflow on a developer's laptop + and look like a performance problem rather than a configuration one. + """ + detect, g = _settings_io_ns() + + monkeypatch.setitem(g, "CGROUP_V2_CPU_MAX", tmp_path / "no-cpu.max") + monkeypatch.setitem(g, "CGROUP_V1_CPU_QUOTA", tmp_path / "no-quota") + monkeypatch.setitem(g, "CGROUP_V1_CPU_PERIOD", tmp_path / "no-period") + + assert detect() is None diff --git a/tests/test_seed_demos.py b/tests/test_seed_demos.py new file mode 100644 index 00000000..49bf698b --- /dev/null +++ b/tests/test_seed_demos.py @@ -0,0 +1,869 @@ +""" +Red tests for the ``.demos`` seeding race (Step 2 of the multi-node plan). + +``k8s/base/streamlit-deployment.yaml`` seeds the demo workspaces from an +initContainer that runs, under ``replicas: 2``:: + + mkdir -p /workspaces-streamlit-template/.demos + cp -rn /app/example-data/workspaces/. /workspaces-streamlit-template/.demos/ + +``cp -rn`` has two failure branches, one silent and one fatal (docs/a16-storage-runbook.md +section 2): + +* **Silent** - ``cp`` writes in place with no temp-and-rename, so replica B can + see a file replica A is still writing, decide it exists, and skip it. Worse, a + replica killed mid-copy (eviction, node drain, OOM) leaves a truncated file + that every later run skips, because ``-n`` only looks at existence. The demo + workspace is then permanently broken and never repairs itself. +* **Fatal** - ``-n`` does not apply to directories; ``mkdirat`` / + ``openat(O_EXCL)`` returning ``EEXIST`` is a hard ``exit 1``, so the losing pod + ends in ``Init:CrashLoopBackOff`` and never self-resolves. + +Both get sharper once the workspace volume is ``ReadWriteMany``, because the two +replicas can then genuinely run at the same instant on different nodes. + +The fix is copy-to-private-temp plus ``mv -T``: ``rename(2)`` on a directory is +already atomic, so the winner renames and the loser cleans up and exits 0. A +lock file is the wrong tool. + +------------------------------------------------------------------------------ +Contract these tests pin +------------------------------------------------------------------------------ + +``docker/seed-demos.sh`` - one copy of the logic, referenced by both the +Kubernetes manifest and this test, so the shipped script is the tested script. + +* Invocation: ``sh docker/seed-demos.sh ``. Both are also + accepted from the environment as ``SEED_SOURCE_DIR`` / ``SEED_DEST_DIR``, and + the destination may default to ``${WORKSPACES_DIR}/.demos``; the tests below + pass all three, so any of those wirings satisfies them. +* Exit status is **always 0**. The initContainer blocks pod start on a shared + mount, so a seeding failure must degrade the demos, never the app. +* stdout carries one machine-readable line, ``seed-demos: outcome=``, + naming the branch taken: ``seeded``, ``backfilled``, ``present``, + ``lost-race`` or ``skipped``. The tests read that line and not the prose + around it, so re-wording a message cannot turn a correct implementation red - + and the outcomes are distinct enough to tell the ``mv -T`` loss apart from a + run that simply found the destination already there. +* The run is bounded by ``timeout`` - an unbounded ``cp`` against a wedged NFS + mount holds the pod in ``Init`` forever - and the bound is asserted by wedging + a ``cp`` for ten minutes, not by grepping the source for the word. +* A run that does not complete leaves **no** destination directory behind, so + the next pod re-seeds from scratch. +* An existing destination is **merged into**, never skipped: + docs/kubernetes-deployment.md promises that demos shipped in a new image + appear after a redeploy while entries already on the volume survive, and a + half-seeded ``.demos/`` - what ``mkdir -p`` + ``cp -rn`` leaves behind when a + pod is killed mid-copy - must repair itself rather than short-circuit every + later run forever. +""" + +import hashlib +import os +import random +import shutil +import signal +import subprocess +import time +from pathlib import Path + +import pytest + +REPO_ROOT = Path(__file__).resolve().parents[1] +SEED_SCRIPT = REPO_ROOT / "docker" / "seed-demos.sh" +STREAMLIT_DEPLOYMENT = REPO_ROOT / "k8s" / "base" / "streamlit-deployment.yaml" +DOCKERFILES = ( + REPO_ROOT / "Dockerfile", + REPO_ROOT / "Dockerfile.arm", + REPO_ROOT / "Dockerfile_simple", + REPO_ROOT / "Dockerfile_simple.arm", +) + +DEST_NAME = ".demos" + +# Big enough that a concurrent run is still copying when the second one starts. +_BULK_FILES = 48 +_BULK_BYTES = 384 * 1024 + +# One large file, so an interrupted run is killed while that file is being +# written rather than after the copy already finished. Killing a process tree +# costs ~150ms on Windows, so the copy has to outlast that comfortably. +_HUGE_BYTES = 64 * 1024 * 1024 + + +# --------------------------------------------------------------------------- +# POSIX shell discovery +# +# CI runs on ubuntu-latest, where `sh` is on PATH. On Windows the only usable +# `sh` is the one Git for Windows ships; `bash` in system32 is the WSL launcher, +# which rewrites paths into a different filesystem namespace. +# --------------------------------------------------------------------------- +def _find_posix_shell(): + found = shutil.which("sh") + if found: + return found + candidates = [] + git = shutil.which("git") + if git: + git_root = Path(git).resolve().parents[1] + candidates += [git_root / "bin" / "sh.exe", git_root / "usr" / "bin" / "sh.exe"] + candidates += [ + Path("C:/Program Files/Git/bin/sh.exe"), + Path("C:/Program Files/Git/usr/bin/sh.exe"), + ] + for candidate in candidates: + if candidate.exists(): + return str(candidate) + return None + + +SHELL = _find_posix_shell() + +pytestmark = pytest.mark.skipif( + SHELL is None, reason="no POSIX shell available to run docker/seed-demos.sh" +) + + +def _sh_path(path) -> str: + """ + Render a path the way the POSIX shell under test expects it. + + A trailing separator is preserved: `Path.as_posix()` drops it, and one test + passes `/.demos/` deliberately, because a script that derives its + staging directory from `${DEST_DIR}.tmp...` behaves very differently when + the destination ends in a slash. + """ + text = Path(path).as_posix() + if os.name == "nt" and len(text) > 1 and text[1] == ":": + text = "/" + text[0].lower() + text[2:] + if str(path).endswith(("/", "\\")) and not text.endswith("/"): + text += "/" + return text + + +def _require_seed_script(): + if not SEED_SCRIPT.exists(): + pytest.fail( + f"{SEED_SCRIPT} does not exist.\n" + "The seeding logic is still inlined in " + "k8s/base/streamlit-deployment.yaml, where it cannot be tested. " + "Extract it to docker/seed-demos.sh and have the manifest run that " + "one copy (docs/a16-storage-runbook.md section 2)." + ) + + +# --------------------------------------------------------------------------- +# Fixtures and helpers +# --------------------------------------------------------------------------- +def _write_source_tree(root: Path) -> None: + """A stand-in for example-data/workspaces: nested dirs and bulky files.""" + rng = random.Random(20260817) + + (root / "demo-workspace").mkdir(parents=True) + (root / "demo-workspace" / "params.json").write_text( + '{"example-x-dimension": 100}\n', encoding="utf-8" + ) + (root / "demo-workspace" / "nested" / "deep").mkdir(parents=True) + (root / "demo-workspace" / "nested" / "deep" / "note.txt").write_text( + "kept\n", encoding="utf-8" + ) + + mzml = root / "demo-workspace" / "mzML-files" + mzml.mkdir() + for index in range(_BULK_FILES): + (mzml / f"Sample_{index:02d}.mzML").write_bytes(rng.randbytes(_BULK_BYTES)) + + +@pytest.fixture(scope="session") +def source_tree(tmp_path_factory) -> Path: + root = tmp_path_factory.mktemp("example-data-workspaces") + _write_source_tree(root) + return root + + +@pytest.fixture(scope="session") +def source_snapshot(source_tree) -> dict: + return _snapshot(source_tree) + + +@pytest.fixture(scope="session") +def huge_source_tree(tmp_path_factory) -> Path: + """A tree dominated by one large file, so the copy is slow to finish.""" + root = tmp_path_factory.mktemp("example-data-huge") + mzml = root / "demo-workspace" / "mzML-files" + mzml.mkdir(parents=True) + (root / "demo-workspace" / "params.json").write_text("{}\n", encoding="utf-8") + block = random.Random(20260818).randbytes(1024 * 1024) + with open(mzml / "Huge.mzML", "wb") as handle: + for _ in range(_HUGE_BYTES // len(block)): + handle.write(block) + return root + + +@pytest.fixture +def workspaces_root(tmp_path) -> Path: + root = tmp_path / "workspaces-streamlit-template" + root.mkdir() + return root + + +# --------------------------------------------------------------------------- +# Shims +# +# The script calls `cp` by name, so a directory in front of PATH can replace it +# with one that hangs or dawdles. That is the only portable way to reproduce +# what a wedged NFS server does to a copy, and it is what lets the bounding and +# the rename arbitration be asserted as behaviour rather than as source text. +# --------------------------------------------------------------------------- +@pytest.fixture +def shim_dir(tmp_path) -> Path: + directory = tmp_path / "shim-bin" + directory.mkdir() + return directory + + +def _install_shim(directory: Path, name: str, body: str) -> Path: + shim = directory / name + shim.write_text(f"#!/bin/sh\n{body}", encoding="utf-8", newline="\n") + shim.chmod(0o755) + return shim + + +def _shim_env(directory: Path, **extra) -> dict: + # Native form, not the shell's: PATH is handed to the shell by the OS, and + # the MSYS runtime converts a well-formed Windows PATH into POSIX form for + # the child. A POSIX entry spliced into a Windows PATH survives neither. + separator = ";" if os.name == "nt" else ":" + entry = str(directory) if os.name == "nt" else _sh_path(directory) + env = {"PATH": entry + separator + os.environ.get("PATH", "")} + env.update(extra) + return env + + +def _shell_has(command: str) -> bool: + """Whether the POSIX shell under test can find `command` on its PATH.""" + if SHELL is None: + return False + probe = subprocess.run( + [SHELL, "-c", f"command -v {command}"], + capture_output=True, + text=True, + check=False, + ) + return probe.returncode == 0 and probe.stdout.strip() != "" + + +def _require_shell_timeout() -> None: + if not _shell_has("timeout"): + pytest.skip("no `timeout` for the shell to bound the run with") + + +def _real_cp() -> str: + """The `cp` the shim delegates to, as an absolute path the shell resolves.""" + probe = subprocess.run( + [SHELL, "-c", "command -v cp"], capture_output=True, text=True, check=False + ) + path = probe.stdout.strip().splitlines()[0] if probe.stdout.strip() else "" + if not path.startswith("/"): + pytest.skip("cannot locate a real `cp` for the slow-copy shim") + return path + + +def _snapshot(root: Path) -> dict: + """Map every entry under ``root`` to its type and content digest.""" + snapshot = {} + for path in sorted(root.rglob("*")): + relative = path.relative_to(root).as_posix() + if path.is_symlink(): + snapshot[relative] = ("symlink", os.readlink(str(path))) + elif path.is_dir(): + snapshot[relative] = ("dir", None) + else: + snapshot[relative] = ("file", hashlib.sha256(path.read_bytes()).hexdigest()) + return snapshot + + +def _total_bytes(root: Path) -> int: + """Bytes currently under ``root``, tolerating a tree being written to.""" + total = 0 + for path in root.rglob("*"): + try: + if path.is_file() and not path.is_symlink(): + total += path.stat().st_size + except OSError: + continue + return total + + +def _describe(actual: dict, expected: dict) -> str: + missing = sorted(set(expected) - set(actual)) + extra = sorted(set(actual) - set(expected)) + differing = sorted( + name for name in set(actual) & set(expected) if actual[name] != expected[name] + ) + return f"missing={missing} extra={extra} differing={differing}" + + +def _assert_complete(dest: Path, expected: dict) -> None: + assert dest.is_dir(), f"{dest} was not created" + actual = _snapshot(dest) + assert actual == expected, ( + "seeded tree is not a byte-complete copy of the source: " + + _describe(actual, expected) + ) + + +def _popen_seed(source, dest, workspaces_root, script=None, extra_env=None): + env = dict(os.environ) + env["WORKSPACES_DIR"] = _sh_path(workspaces_root) + env["SEED_SOURCE_DIR"] = _sh_path(source) + env["SEED_DEST_DIR"] = _sh_path(dest) + env.update(extra_env or {}) + kwargs = {} + if os.name != "nt": + # Own process group, so an interrupted run takes `cp` down with the + # shell instead of orphaning it to finish the copy behind our back. + kwargs["start_new_session"] = True + return subprocess.Popen( + [SHELL, _sh_path(script or SEED_SCRIPT), _sh_path(source), _sh_path(dest)], + env=env, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + errors="replace", + **kwargs, + ) + + +def _kill_tree(proc) -> None: + if proc.poll() is not None: + return + if os.name == "nt": + subprocess.run( + ["taskkill", "/F", "/T", "/PID", str(proc.pid)], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + check=False, + ) + else: + try: + os.killpg(os.getpgid(proc.pid), signal.SIGKILL) + except (ProcessLookupError, PermissionError): + proc.kill() + try: + proc.wait(timeout=60) + except subprocess.TimeoutExpired: + proc.kill() + + +def _run_seed(source, dest, workspaces_root, timeout: int = 300, **kwargs): + proc = _popen_seed(source, dest, workspaces_root, **kwargs) + try: + stdout, stderr = proc.communicate(timeout=timeout) + except subprocess.TimeoutExpired: + _kill_tree(proc) + pytest.fail( + f"docker/seed-demos.sh did not finish within {timeout}s. The copy " + "must be bounded so a wedged mount cannot hold the pod in Init." + ) + return proc.returncode, stdout, stderr + + +# The documented outcomes, and whether each one means the run installed +# something. Reading a declared token rather than the prose keeps a correct +# implementation from going red over a re-worded message - `_NOOP_MARKERS` +# containing "race" would have failed a winner that announced "won the race". +_OUTCOMES = { + "seeded": "seeded", + "backfilled": "seeded", + "present": "noop", + "lost-race": "noop", + "skipped": "noop", +} +_OUTCOME_PREFIX = "seed-demos: outcome=" + + +def _outcome(stdout: str) -> str: + """The outcome token the run declared, as its own line on stdout.""" + declared = [ + line.strip()[len(_OUTCOME_PREFIX):].strip() + for line in stdout.splitlines() + if line.strip().startswith(_OUTCOME_PREFIX) + ] + assert declared, ( + "the run declared no outcome. docker/seed-demos.sh must print exactly " + f"one `{_OUTCOME_PREFIX}` line so a test reads a declared " + f"branch rather than guessing from prose.\nstdout: {stdout!r}" + ) + # The last one wins: a run killed by its own `timeout` prints the child's + # verdict and then the parent's, and the parent's is the authoritative one. + outcome = declared[-1] + assert outcome in _OUTCOMES, ( + f"unknown outcome {outcome!r}; the documented set is " + f"{sorted(_OUTCOMES)}\nstdout: {stdout!r}" + ) + return outcome + + +def _classify(stdout: str) -> str: + """Whether a run installed anything, from the outcome it declared.""" + return _OUTCOMES[_outcome(stdout)] + + +# --------------------------------------------------------------------------- +# Tests +# --------------------------------------------------------------------------- +def test_seed_demos_copies_the_whole_tree(source_tree, source_snapshot, workspaces_root): + _require_seed_script() + dest = workspaces_root / DEST_NAME + + code, stdout, stderr = _run_seed(source_tree, dest, workspaces_root) + + assert code == 0, f"exit {code}\nstdout: {stdout}\nstderr: {stderr}" + assert _classify(stdout) == "seeded", f"unexpected stdout: {stdout!r}" + _assert_complete(dest, source_snapshot) + assert sorted(p.name for p in workspaces_root.iterdir()) == [DEST_NAME], ( + "the staging directory must not survive a successful seed" + ) + + +def test_second_run_is_a_noop_and_exits_zero( + source_tree, source_snapshot, workspaces_root +): + """The already-seeded path. `cp -rn` fails here with EEXIST -> exit 1.""" + _require_seed_script() + dest = workspaces_root / DEST_NAME + + first_code, _, first_err = _run_seed(source_tree, dest, workspaces_root) + assert first_code == 0, first_err + + code, stdout, stderr = _run_seed(source_tree, dest, workspaces_root) + + assert code == 0, ( + "a pod restarting against an already-seeded volume must exit 0, not " + f"land in Init:CrashLoopBackOff.\nexit {code}\nstdout: {stdout}\n" + f"stderr: {stderr}" + ) + assert _classify(stdout) == "noop", f"unexpected stdout: {stdout!r}" + _assert_complete(dest, source_snapshot) + assert sorted(p.name for p in workspaces_root.iterdir()) == [DEST_NAME] + + +def test_concurrent_seed_no_corruption(source_tree, source_snapshot, workspaces_root): + """ + Two replicas seeding one volume at the same moment. + + Exactly one may report seeding; the other must exit 0 having done nothing; + and the result must be a byte-complete copy with no staging leftovers. + """ + _require_seed_script() + dest = workspaces_root / DEST_NAME + + first = _popen_seed(source_tree, dest, workspaces_root) + second = _popen_seed(source_tree, dest, workspaces_root) + results = [] + for proc in (first, second): + try: + stdout, stderr = proc.communicate(timeout=300) + except subprocess.TimeoutExpired: + _kill_tree(proc) + pytest.fail("a concurrent seeding run never finished") + results.append((proc.returncode, stdout, stderr)) + + for index, (code, stdout, stderr) in enumerate(results): + assert code == 0, ( + f"replica {index} exited {code}; the loser of the race must exit 0 " + f"or its pod never leaves Init.\nstdout: {stdout}\nstderr: {stderr}" + ) + + outcomes = [_classify(stdout) for _, stdout, _ in results] + assert outcomes.count("seeded") == 1, ( + f"expected exactly one replica to seed, got {outcomes} from " + f"{[stdout for _, stdout, _ in results]!r}" + ) + assert outcomes.count("noop") == 1, ( + f"expected exactly one replica to stand down, got {outcomes}" + ) + + _assert_complete(dest, source_snapshot) + assert sorted(p.name for p in workspaces_root.iterdir()) == [DEST_NAME], ( + "the loser must clean up its staging directory; leaking one per pod " + "start fills the shared volume with duplicate demo trees" + ) + + +def test_destination_is_never_observable_in_a_partial_state( + huge_source_tree, workspaces_root +): + """ + `.demos` must go from absent to complete in one step, because the step is a + `rename(2)`. Nothing else makes an interrupted run safe: a replica killed + mid-copy (eviction, drain, OOM) leaves behind exactly whatever the + destination looked like at that instant. + + This is the silent branch of the `cp -rn` bug, asserted directly instead of + by killing a process - `cp -rn` creates `.demos` and copies into it in + place, so a kill leaves a truncated file that `-n` makes every later run + skip forever, and the demo workspace never repairs itself. + + Sampling can only miss the partial state, never invent one: for a correct + implementation the observation below is impossible at any instant. + """ + _require_seed_script() + expected = _snapshot(huge_source_tree) + total = _total_bytes(huge_source_tree) + dest = workspaces_root / DEST_NAME + + proc = _popen_seed(huge_source_tree, dest, workspaces_root) + partial = None + deadline = time.monotonic() + 300 + while proc.poll() is None and time.monotonic() < deadline: + try: + if dest.exists(): + seen = _total_bytes(dest) + if seen < total: + partial = seen + break + except OSError: + continue + try: + stdout, stderr = proc.communicate(timeout=300) + except subprocess.TimeoutExpired: + _kill_tree(proc) + pytest.fail("the seeding run never finished") + + assert partial is None, ( + f"{DEST_NAME} was observable holding {partial} of {total} bytes while " + "the copy was still running; a replica killed at that instant leaves " + "a truncated demo workspace that no later run repairs. Stage the copy " + "elsewhere and rename it into place." + ) + assert proc.returncode == 0, f"stdout: {stdout}\nstderr: {stderr}" + _assert_complete(dest, expected) + + +def test_unreachable_source_exits_zero_without_seeding(workspaces_root): + """ + A failed copy must not block pod start, and must not leave a destination + directory behind that a later, healthy run would treat as already seeded. + """ + _require_seed_script() + missing = workspaces_root.parent / "no-such-source" + dest = workspaces_root / DEST_NAME + + code, stdout, stderr = _run_seed(missing, dest, workspaces_root) + + assert code == 0, ( + "seeding failure must degrade the demos, not the pod.\n" + f"exit {code}\nstdout: {stdout}\nstderr: {stderr}" + ) + assert not dest.exists(), ( + "a failed run left .demos behind; every later run would then short-" + "circuit on it and the demos would stay empty forever" + ) + assert list(workspaces_root.iterdir()) == [], "staging leftovers" + + +def test_a_wedged_copy_cannot_hold_the_pod_in_init(source_tree, workspaces_root, shim_dir): + """ + The initContainer blocks pod start on the shared mount. An unbounded `cp` + against a wedged NFS server holds the pod in Init indefinitely, which is + exactly the outcome the storage design is meant to avoid. + + Asserted by wedging the copy for ten minutes and requiring the run back + inside its own `SEED_TIMEOUT`. Grepping the source for the word "timeout" + would pass on any script carrying it in a comment, including one that still + runs a plain `cp -rn`. + """ + _require_seed_script() + _require_shell_timeout() + _install_shim(shim_dir, "cp", "sleep 600\n") + dest = workspaces_root / DEST_NAME + + started = time.monotonic() + code, stdout, stderr = _run_seed( + source_tree, + dest, + workspaces_root, + timeout=180, + extra_env=_shim_env(shim_dir, SEED_TIMEOUT="3", SEED_KILL_AFTER="1"), + ) + elapsed = time.monotonic() - started + + assert elapsed < 120, ( + f"the run took {elapsed:.0f}s against a `cp` wedged for 600s; the whole " + "run must be bounded, not just hoped to finish" + ) + assert code == 0, ( + "a run that ran out of time must still let the pod start.\n" + f"exit {code}\nstdout: {stdout}\nstderr: {stderr}" + ) + assert _classify(stdout) == "noop", f"unexpected stdout: {stdout!r}" + assert not dest.exists(), ( + "a run that gave up left .demos behind; every later run would then " + "short-circuit on it and the demos would stay empty forever" + ) + assert list(workspaces_root.iterdir()) == [], ( + "the staging directory outlived the run that gave up on it" + ) + + +def test_runs_without_the_executable_bit(source_tree, source_snapshot, tmp_path): + """ + The script is invoked as `sh /app/docker/seed-demos.sh` from the manifest, + so nothing requires it to carry the executable bit - and git records a new + file as 100644 wherever `core.fileMode` is false, which is every Windows + clone. A run that re-execs itself as `"$0"` under `timeout` needs the bit + anyway: the exec fails EACCES, `timeout` reports 126, and the script + diagnoses an instant exec failure as a wedged mount and seeds nothing. + """ + _require_seed_script() + copied = tmp_path / "seed-demos.sh" + copied.write_bytes(SEED_SCRIPT.read_bytes()) + os.chmod(copied, 0o644) + workspaces_root = tmp_path / "workspaces-streamlit-template" + workspaces_root.mkdir() + dest = workspaces_root / DEST_NAME + + code, stdout, stderr = _run_seed(source_tree, dest, workspaces_root, script=copied) + + assert code == 0, f"exit {code}\nstdout: {stdout}\nstderr: {stderr}" + assert _classify(stdout) == "seeded", ( + "a non-executable copy of the script seeded nothing, so the run must " + f"be re-execing itself rather than an interpreter.\nstdout: {stdout!r}" + ) + _assert_complete(dest, source_snapshot) + + +def test_the_rename_is_what_arbitrates( + source_tree, source_snapshot, workspaces_root, shim_dir +): + """ + Two replicas that both start with the destination absent: the loser must + lose *at the rename*, not by noticing a finished `.demos` before it began. + + Without the delay the loser can legitimately short-circuit on the + already-present check, and the concurrency test cannot tell the two apart - + so an implementation whose `mv -T` arbitration was broken could still pass + it. A `cp` that takes two seconds puts both replicas past that check before + either can rename. + """ + _require_seed_script() + _install_shim(shim_dir, "cp", 'sleep 2\nexec %s "$@"\n' % _real_cp()) + dest = workspaces_root / DEST_NAME + env = _shim_env(shim_dir) + + first = _popen_seed(source_tree, dest, workspaces_root, extra_env=env) + second = _popen_seed(source_tree, dest, workspaces_root, extra_env=env) + results = [] + for proc in (first, second): + try: + stdout, stderr = proc.communicate(timeout=300) + except subprocess.TimeoutExpired: + _kill_tree(proc) + pytest.fail("a concurrent seeding run never finished") + results.append((proc.returncode, stdout, stderr)) + + for index, (code, stdout, stderr) in enumerate(results): + assert code == 0, ( + f"replica {index} exited {code}\nstdout: {stdout}\nstderr: {stderr}" + ) + + outcomes = sorted(_outcome(stdout) for _, stdout, _ in results) + assert outcomes == ["lost-race", "seeded"], ( + "with both replicas past the already-present check, one must seed and " + "the other must lose the rename; got " + f"{outcomes} from {[stdout for _, stdout, _ in results]!r}" + ) + _assert_complete(dest, source_snapshot) + assert sorted(p.name for p in workspaces_root.iterdir()) == [DEST_NAME] + + +def test_a_half_seeded_destination_repairs_itself( + source_tree, source_snapshot, workspaces_root +): + """ + A pod killed mid-copy under the old `mkdir -p` + `cp -rn` leaves `.demos/` + existing and incomplete - in the worst case existing and *empty*. Skipping + on the mere existence of the directory makes that permanent: nothing else + repairs it either, because clean-up-workspaces.py skips top-level dot + entries. + + So an existing destination is merged into, not stepped around. + """ + _require_seed_script() + dest = workspaces_root / DEST_NAME + dest.mkdir() + + code, stdout, stderr = _run_seed(source_tree, dest, workspaces_root) + + assert code == 0, f"exit {code}\nstdout: {stdout}\nstderr: {stderr}" + assert _outcome(stdout) == "backfilled", f"unexpected stdout: {stdout!r}" + _assert_complete(dest, source_snapshot) + assert sorted(p.name for p in workspaces_root.iterdir()) == [DEST_NAME] + + +def test_new_image_demos_appear_and_existing_entries_survive( + source_tree, source_snapshot, workspaces_root +): + """ + The documented contract (docs/kubernetes-deployment.md, "Demo workspaces"): + demos shipped in a newer image appear after a redeploy, and everything + already on the volume - admin-saved demos, hand edits - is preserved + byte-for-byte. + """ + _require_seed_script() + dest = workspaces_root / DEST_NAME + + code, _, stderr = _run_seed(source_tree, dest, workspaces_root) + assert code == 0, stderr + + # Two things the volume has that the image does not: a demo an admin saved, + # and an edit to a demo the image ships. + admin_demo = dest / "admin-saved" / "params.json" + admin_demo.parent.mkdir(parents=True) + admin_demo.write_text('{"saved": true}\n', encoding="utf-8") + edited = dest / "demo-workspace" / "params.json" + edited.write_text('{"edited": true}\n', encoding="utf-8") + + # ... and one the newer image ships that the volume has never seen. + added = source_tree / "demo-workspace" / "nested" / "added-by-new-image.txt" + added.write_text("new\n", encoding="utf-8") + try: + code, stdout, stderr = _run_seed(source_tree, dest, workspaces_root) + finally: + added.unlink() + + assert code == 0, f"exit {code}\nstdout: {stdout}\nstderr: {stderr}" + assert _outcome(stdout) == "backfilled", f"unexpected stdout: {stdout!r}" + assert (dest / "demo-workspace" / "nested" / "added-by-new-image.txt").read_text( + encoding="utf-8" + ) == "new\n", "a demo added in a newer image never reached the volume" + assert admin_demo.read_text(encoding="utf-8") == '{"saved": true}\n', ( + "an admin-saved demo was destroyed by re-seeding" + ) + assert edited.read_text(encoding="utf-8") == '{"edited": true}\n', ( + "an edit to a shipped demo was overwritten by re-seeding" + ) + assert sorted(p.name for p in workspaces_root.iterdir()) == [DEST_NAME] + + +def test_a_trailing_slash_on_the_destination_is_harmless( + source_tree, source_snapshot, workspaces_root +): + """ + `SEED_DEST_DIR` and argv are documented inputs, so `/.demos/` is + reachable without touching the manifest. Deriving the staging directory + from it naively puts the staging copy *inside* the destination, brings the + destination into existence as a side effect of `mkdir -p`, fails the + `mv -T` with EINVAL, and leaves an empty `.demos/` that short-circuits + every later run forever. + """ + _require_seed_script() + dest = workspaces_root / DEST_NAME + + code, stdout, stderr = _run_seed( + source_tree, f"{_sh_path(dest)}/", workspaces_root + ) + + assert code == 0, f"exit {code}\nstdout: {stdout}\nstderr: {stderr}" + assert _classify(stdout) == "seeded", f"unexpected stdout: {stdout!r}" + _assert_complete(dest, source_snapshot) + assert sorted(p.name for p in workspaces_root.iterdir()) == [DEST_NAME] + + +def test_abandoned_staging_directories_are_reclaimed( + source_tree, source_snapshot, workspaces_root +): + """ + The EXIT trap does not run on SIGKILL - an eviction, a drain, an OOM - so a + staging directory can outlive the pod that made it. It is dot-named, and + clean-up-workspaces.py skips every top-level entry starting with a dot, so + nothing else on the volume would ever reclaim it: each replacement pod + arrives under a new ReplicaSet suffix and leaks another full copy of + example-data/workspaces. + """ + _require_seed_script() + dest = workspaces_root / DEST_NAME + abandoned = workspaces_root / f"{DEST_NAME}.tmp.evicted-pod.1" + (abandoned / "demo-workspace").mkdir(parents=True) + (abandoned / "demo-workspace" / "params.json").write_text("{}\n", encoding="utf-8") + stale = time.time() - 6 * 3600 + os.utime(abandoned, (stale, stale)) + + # An in-flight run's staging directory, from a replica still copying. + in_flight = workspaces_root / f"{DEST_NAME}.tmp.busy-pod.1" + in_flight.mkdir() + + code, stdout, stderr = _run_seed(source_tree, dest, workspaces_root) + + assert code == 0, f"exit {code}\nstdout: {stdout}\nstderr: {stderr}" + assert not abandoned.exists(), ( + "an abandoned staging directory survived; on a shared volume it is a " + "full duplicate of the demo tree that nothing will ever remove" + ) + assert in_flight.exists(), ( + "a staging directory that was just created was reaped; that would " + "delete a concurrent replica's copy out from under it mid-run" + ) + _assert_complete(dest, source_snapshot) + + +def test_manifest_invokes_the_shared_seed_script(): + manifest = STREAMLIT_DEPLOYMENT.read_text(encoding="utf-8") + + assert "cp -rn" not in manifest, ( + "k8s/base/streamlit-deployment.yaml still seeds with `cp -rn`, which " + "either skips a partially written file or exits 1 on EEXIST" + ) + assert SEED_SCRIPT.name in manifest, ( + "the seed-demos initContainer must run docker/seed-demos.sh so the " + "shipped logic is the tested logic, not a second inline copy" + ) + + +def _in_image_paths(dockerfile_text: str) -> set: + """In-image destinations a COPY line could give docker/seed-demos.sh.""" + paths = set() + for line in dockerfile_text.splitlines(): + parts = line.strip().split() + if len(parts) < 3 or parts[0].upper() != "COPY": + continue + sources, destination = parts[1:-1], parts[-1] + for source in sources: + if source.startswith("--"): + continue + normalized = source.replace("\\", "/") + if normalized.endswith(SEED_SCRIPT.name): + paths.add( + destination + SEED_SCRIPT.name + if destination.endswith("/") + else destination + ) + elif normalized.rstrip("/") in ("docker", "./docker"): + paths.add(destination.rstrip("/") + "/" + SEED_SCRIPT.name) + return paths + + +@pytest.mark.parametrize("dockerfile", DOCKERFILES, ids=lambda p: p.name) +def test_every_image_installs_the_seed_script(dockerfile): + """ + The manifest names a path inside the image. `docker/` is not copied + wholesale - each Dockerfile installs `docker/entrypoint.sh` by name - so a + seed script that is never COPYed lands the initContainer in + Init:CrashLoopBackOff with `not found`, on every image variant. + """ + manifest = STREAMLIT_DEPLOYMENT.read_text(encoding="utf-8") + referenced = { + token + for token in manifest.replace('"', " ").replace("'", " ").split() + if token.endswith(SEED_SCRIPT.name) + } + assert referenced, "the manifest does not reference docker/seed-demos.sh at all" + + installed = _in_image_paths(dockerfile.read_text(encoding="utf-8")) + assert referenced & installed, ( + f"{dockerfile.name} does not COPY {SEED_SCRIPT.name} to any path the " + f"manifest runs ({sorted(referenced)}); it installs it at " + f"{sorted(installed) or 'nowhere'}" + ) diff --git a/tests/test_sidebar_monitors.py b/tests/test_sidebar_monitors.py new file mode 100644 index 00000000..737ac1e3 --- /dev/null +++ b/tests/test_sidebar_monitors.py @@ -0,0 +1,322 @@ +""" +Tests for the sidebar's "Resource Utilization" expander (Step 2 of the +multi-node plan). + +Two decisions live there, and both are about *which machine* the numbers +describe: + +* ``monitor_hardware()`` reads psutil in the Streamlit process. Once the + workers are spread across nodes that process runs no workflow at all, so the + panel would show an idle 2Gi pod while a 64Gi worker saturates. +* ``monitor_storage()`` reads the shared volume's heartbeat out of Redis, and + must distinguish "no shared storage here" from "the shared storage is gone" + from "I cannot tell, because Redis is down". Each of the three sends an + operator somewhere different. + +The obvious gate for the first one - ``online_deployment`` - is wrong, and +wrong in a way that is invisible from Kubernetes: ``Dockerfile_simple`` bakes +``online_deployment: true`` and ships no Redis, so the workflows run in this +very process, and gating psutil on that flag empties the expander completely. +``Dockerfile`` bakes it too, and ``docker/entrypoint.sh`` starts Redis, the RQ +workers *and* Streamlit in one container, where psutil is once again measuring +exactly the right machine. So the question is answered from the queue - are any +workers registered, and are they all on some other host - not from a flag. + +``src/common/common.py`` imports Streamlit at module scope, so it is imported +here with the heavy dependencies mocked at the ``sys.modules`` level and the +originals restored immediately afterwards, mirroring +``tests/test_legal_links.py`` (and, for ``src.workflow``, +``tests/test_topp_flag_parameters.py``). Every other test module - and +``test_gui.py``'s AppTest smoke tests in particular - still gets the real +packages. +""" +import importlib +import os +import sys +import types +from unittest.mock import MagicMock + +import pytest + +# Add project root to path for imports +PROJECT_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +sys.path.append(PROJECT_ROOT) + + +def _identity_decorator(*args, **kwargs): + """Stand in for `st.fragment(run_every=5)`, which wraps the monitors.""" + + def _wrap(function): + return function + + return _wrap + + +mock_streamlit = MagicMock() +mock_streamlit.session_state = {} +# A real function, not a MagicMock attribute: `@st.fragment(...)` applied to a +# MagicMock replaces the decorated function with another MagicMock, and the +# module's monitors would then be uncallable stand-ins rather than the code +# under test. +mock_streamlit.fragment = _identity_decorator + +_MOCKED_MODULES = { + "streamlit": mock_streamlit, + "streamlit.components": MagicMock(), + "streamlit.components.v1": MagicMock(), + "streamlit.source_util": MagicMock(), + "pandas": MagicMock(), + "psutil": MagicMock(), + # Local submodules with their own heavy deps (e.g. the captcha image library). + "src.common.captcha_": MagicMock(), + "src.common.admin": MagicMock(), +} +_saved_modules = {name: sys.modules.get(name) for name in _MOCKED_MODULES} +sys.modules.update(_MOCKED_MODULES) + +# Force a FRESH import under the mocks, even if an earlier test module (e.g. +# test_gui.py, or tests/test_legal_links.py, which mocks Streamlit differently) +# already imported a version bound to something else. +# +# `import_module`, not `from src.common import common`: the second form is +# satisfied by the *attribute* the import system left on the `src.common` +# package object, which survives popping `sys.modules["src.common.common"]`. +# It would hand back the module another test file imported under its own mock, +# where `st.fragment` is a plain MagicMock - so every monitor here would be a +# MagicMock stand-in, calling them would do nothing, and these tests would pass +# or fail depending on which files ran first. +_package = sys.modules.get("src.common") +_saved_attribute = getattr(_package, "common", None) if _package is not None else None +_saved_common = sys.modules.pop("src.common.common", None) + +common = importlib.import_module("src.common.common") # noqa: E402 + +for _name, _original in _saved_modules.items(): + if _original is None: + sys.modules.pop(_name, None) + else: + sys.modules[_name] = _original +if _saved_common is None: + sys.modules.pop("src.common.common", None) +else: + sys.modules["src.common.common"] = _saved_common +if _package is not None: + if _saved_attribute is None: + delattr(_package, "common") + else: + setattr(_package, "common", _saved_attribute) + + +# --------------------------------------------------------------------------- +# Stubbing health +# +# The monitors import `src.workflow.health` inside their own bodies, so the +# stub is installed in `sys.modules` rather than patched onto a module object. +# Several test files purge `src.workflow.*` from `sys.modules` after importing +# under mocks, and pytest imports every test module before running any test - +# so by the time these run, the `health` this file imported at collection time +# may no longer be the one the function-level import resolves to. Patching +# `sys.modules` is immune to that. +# --------------------------------------------------------------------------- +def _stub_health(monkeypatch, **functions): + stub = types.ModuleType("src.workflow.health") + for name, value in functions.items(): + setattr(stub, name, value) + monkeypatch.setitem(sys.modules, "src.workflow.health", stub) + return stub + + +@pytest.fixture +def rendered(monkeypatch): + """Record which panels the expander rendered, without running them.""" + mock_streamlit.reset_mock() + mock_streamlit.fragment = _identity_decorator + panels = [] + for name in ("monitor_queue", "monitor_storage", "monitor_hardware"): + monkeypatch.setattr( + common, name, (lambda captured: lambda: panels.append(captured))(name) + ) + return panels + + +def _deployment(monkeypatch, *, queue: dict, workers_remote: bool): + _stub_health( + monkeypatch, + get_queue_metrics=lambda: queue, + queue_workers_are_remote=lambda: workers_remote, + ) + + +# --------------------------------------------------------------------------- +# Which panels apply +# --------------------------------------------------------------------------- +def test_monitor_hardware_hidden_online(monkeypatch, rendered): + """ + Kubernetes: separate Streamlit and rq-worker pods. psutil here describes a + process that runs no workflow, so the panel is hidden and the queue and the + shared volume - what the user's job actually waits on - take its place. + """ + _deployment( + monkeypatch, + queue={"available": True, "total_workers": 2}, + workers_remote=True, + ) + + common.render_resource_utilization() + + assert "monitor_hardware" not in rendered, ( + "the psutil panel describes the Streamlit pod, which does no work; an " + "idle 2Gi container shown while a 64Gi worker saturates is worse than " + "showing nothing" + ) + assert rendered == ["monitor_queue", "monitor_storage"], rendered + + +def test_monitor_hardware_shown_when_the_workers_share_this_container( + monkeypatch, rendered +): + """ + The full image under docker/docker-compose: `docker/entrypoint.sh` runs + Redis, the RQ workers and Streamlit in one container, so psutil is + measuring exactly the machine the workflows run on. + """ + _deployment( + monkeypatch, + queue={"available": True, "total_workers": 2}, + workers_remote=False, + ) + + common.render_resource_utilization() + + assert rendered == ["monitor_queue", "monitor_storage", "monitor_hardware"], rendered + + +def test_the_expander_is_never_empty_without_a_queue(monkeypatch, rendered): + """ + `Dockerfile_simple` bakes `online_deployment: true`, installs no + redis-server, and runs the workflows in this process. Gating the psutil + panel on that flag left the expander rendering an empty body - the queue + and storage panels draw nothing without a queue, and the one panel that had + real numbers was the one being suppressed. + """ + _deployment(monkeypatch, queue={}, workers_remote=False) + + common.render_resource_utilization() + + assert rendered == ["monitor_hardware"], rendered + + +# --------------------------------------------------------------------------- +# What the storage indicator says +# --------------------------------------------------------------------------- +def _render_storage(monkeypatch, status): + mock_streamlit.reset_mock() + mock_streamlit.fragment = _identity_decorator + _stub_health( + monkeypatch, + get_storage_status=status if callable(status) else (lambda: status), + ) + common.monitor_storage() + + +def _markdown() -> str: + return " ".join( + str(call.args[0]) for call in mock_streamlit.markdown.call_args_list if call.args + ) + + +def _captions() -> str: + return " ".join( + str(call.args[0]) for call in mock_streamlit.caption.call_args_list if call.args + ) + + +def test_storage_indicator_renders_each_state(monkeypatch): + """ + Colour is the whole payload here, so it is what gets asserted: + + - green while every node's mount answers, + - amber naming the node whose mount stopped answering, because a healthy + node must not cover for a wedged one, + - red when no node answers at all, + - grey, never red, when Redis is down: that says nothing about the mount, + and reporting it as a storage fault sends the operator to the wrong layer. + """ + _render_storage( + monkeypatch, {"available": True, "state": "connected", "nodes": ["node-a"]} + ) + assert ":green[" in _markdown(), _markdown() + + _render_storage( + monkeypatch, + { + "available": True, + "state": "degraded", + "nodes": ["node-a"], + "stale_nodes": ["node-b"], + }, + ) + degraded = _markdown() + assert ":green[" not in degraded, ( + "one node's mount is wedged; a green tick is the outage the per node " + f"heartbeats exist to make visible: {degraded}" + ) + assert ":orange[" in degraded, degraded + assert "node-b" in _captions(), ( + "an operator needs to know which node to look at, not just that one " + f"broke: {_captions()}" + ) + + _render_storage( + monkeypatch, + { + "available": True, + "state": "unreachable", + "nodes": [], + "stale_nodes": ["node-a"], + }, + ) + assert ":red[" in _markdown(), _markdown() + + _render_storage(monkeypatch, {"available": False, "state": "unknown", "nodes": []}) + unknown = _markdown() + assert ":red[" not in unknown, ( + f"a dead Redis must not be reported as failed storage: {unknown}" + ) + assert ":gray[" in unknown, unknown + + +def test_storage_indicator_renders_nothing_where_nothing_publishes(monkeypatch): + """ + `{}` is "no shared storage is being reported here", which is every local + run, every docker-compose run and every apptainer run of the image. A red + tick there would be a permanent false alarm about a mechanism that is not + deployed. + """ + _render_storage(monkeypatch, {}) + + assert _markdown() == "" + assert _captions() == "" + + +def test_a_broken_storage_indicator_is_not_silent(monkeypatch, caplog): + """ + The indicator must never break the sidebar, but swallowing the failure + leaves a broken indicator indistinguishable from a deployment that has no + shared storage - undetectable in production and in CI alike. + """ + + def _explode(): + raise RuntimeError("redis client blew up") + + with caplog.at_level("ERROR"): + _render_storage(monkeypatch, _explode) # must not raise + + assert caplog.records, ( + "the indicator swallowed an exception without logging it, so a broken " + "indicator is silent in production and invisible in CI" + ) + assert any( + "redis client blew up" in str(record.exc_info) or "redis client blew up" in record.getMessage() + for record in caplog.records + ), [record.getMessage() for record in caplog.records] diff --git a/tests/test_storage_health.py b/tests/test_storage_health.py new file mode 100644 index 00000000..0a9d6869 --- /dev/null +++ b/tests/test_storage_health.py @@ -0,0 +1,848 @@ +""" +Tests for the sidebar storage indicator (Step 2 of the multi-node plan). + +Once the workspace volume is a shared NFS mount, its failure mode is a *hang*, +not an error, and nothing in the app detected one: ``health.py`` inspected only +Redis and RQ, ``/_stcore/health`` is filesystem-blind, and RQ's parent process +keeps heartbeating while its work horse is blocked in an NFS syscall. + +The design (docs/a16-storage-decisions.md section 8) inverts the check rather than adding one +to the UI:: + + rq-worker readiness probe sidebar + SET storage:node: (a worker lives here) --> reads Redis only + touch .nfs-probe --ok--> SET storage:ok: --> + \\-- blocks on wedged NFS --> storage:ok expires --> goes red + +Redis' TTL *is* the liveness mechanism, so there is no timeout logic to get +wrong, and the sidebar never touches the filesystem: a ``stat`` on a ``hard`` +mount blocks in uninterruptible sleep, so a fragment re-running every 5 seconds +would accumulate unkillable threads - the indicator would *become* the hang. + +------------------------------------------------------------------------------ +Contract these tests pin +------------------------------------------------------------------------------ + +In ``src/workflow/health.py``, which must stay importable in the RQ worker with +no Streamlit: + +* ``STORAGE_HEARTBEAT_PREFIX`` / ``STORAGE_NODE_PREFIX`` / ``STORAGE_SENTINEL_NAME`` + / ``STORAGE_HEARTBEAT_TTL`` - shared constants. +* ``storage_heartbeat_key(node)``, ``storage_node_key(node)`` - per node keys, + because one healthy node must not mask a wedged one. +* ``probe_storage()`` - what the readiness probe runs, and where the ordering + lives: register the node, *then* write to the volume, and publish the + heartbeat only if that write succeeded. Wrapping the volume check in + ``|| true`` - or publishing before it - reports Ready on a wedged mount, and + that mutation must not survive. +* ``get_storage_status()`` - the reader, called by the sidebar fragment. + Extends ``get_queue_metrics()``'s existing ``{}`` vs ``{"available": False}`` + distinction: + + =============================== ========================================= + ``{}`` no ``REDIS_URL``, or nothing has ever + published a heartbeat: render nothing + ``{"available": False, ...}`` Redis is down: state ``"unknown"`` + ``{"available": True, ...}`` state ``"connected"`` / ``"degraded"`` / + ``"unreachable"``, plus ``nodes`` and + ``stale_nodes`` + =============================== ========================================= + + Neither of the extra states is cosmetic. A red tick caused by a dead Redis + sends an operator to debug storage when the fault is in the queue; a red tick + in a deployment that has no heartbeat writer at all - every docker-compose + and apptainer run of the image, which bakes in both ``online_deployment`` and + a ``REDIS_URL`` - is a permanent false alarm pointing at the wrong layer; and + a *green* tick while one node's mount is wedged is the outage the per node + keys exist to make visible. +""" + +import os +import re +import shutil +import subprocess +import sys +import textwrap +import time +from contextlib import ExitStack +from pathlib import Path +from unittest import mock + +import pytest + +fakeredis = pytest.importorskip("fakeredis") +import redis as redis_module # noqa: E402 (fakeredis guarantees this import) + +REPO_ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(REPO_ROOT)) + +from src.workflow import health # noqa: E402 + +RQ_WORKER_DEPLOYMENT = REPO_ROOT / "k8s" / "base" / "rq-worker-deployment.yaml" +CLEANUP_CRONJOB = REPO_ROOT / "k8s" / "base" / "cleanup-cronjob.yaml" + + +class _DeadRedis: + """A client whose every call fails the way an unreachable Redis does.""" + + def __getattr__(self, name): + def _refused(*args, **kwargs): + raise redis_module.exceptions.ConnectionError("Connection refused") + + return _refused + + +def _patch_from_url(monkeypatch, client): + """Point every `Redis.from_url(...)` in health.py at `client`.""" + monkeypatch.setattr( + redis_module.Redis, + "from_url", + classmethod(lambda cls, *args, **kwargs: client), + raising=False, + ) + monkeypatch.setattr( + redis_module, "from_url", lambda *args, **kwargs: client, raising=False + ) + + +@pytest.fixture +def fake_redis(monkeypatch): + client = fakeredis.FakeStrictRedis() + monkeypatch.setenv("REDIS_URL", "redis://localhost:6379/0") + _patch_from_url(monkeypatch, client) + return client + + +def _nodes(status: dict) -> set: + """The reporting nodes, however the implementation chooses to carry them.""" + return set(status.get("nodes") or []) + + +def _decode(value): + return value.decode() if isinstance(value, bytes) else value + + +def _heartbeat_keys(client) -> set: + return { + _decode(key) + for key in client.keys("*") + if _decode(key).startswith(health.STORAGE_HEARTBEAT_PREFIX) + } + + +def _probe_node(node: str, workspaces_dir) -> bool: + """Run the readiness probe's own entry point for one node.""" + return health.probe_storage(workspaces_dir=str(workspaces_dir), node=node) + + +# --------------------------------------------------------------------------- +# The reader +# --------------------------------------------------------------------------- +def test_indicator_three_states(fake_redis, monkeypatch, tmp_path): + """fresh key -> connected; expired key -> unreachable; no Redis -> unknown.""" + monkeypatch.setenv("NODE_NAME", "node-a") + assert _probe_node("node-a", tmp_path) is True + + status = health.get_storage_status() + assert status.get("available") is True, status + assert status.get("state") == "connected", status + assert _nodes(status) == {"node-a"}, status + + # The key expires because nothing refreshed it - the worker is blocked in + # an NFS syscall and never got to its SETEX. + key = health.storage_heartbeat_key("node-a") + fake_redis.psetex(key, 20, b"1") + time.sleep(0.1) + assert fake_redis.exists(key) == 0, "precondition: the heartbeat has expired" + + status = health.get_storage_status() + assert status.get("available") is True, status + assert status.get("state") == "unreachable", status + assert _nodes(status) == set(), status + + # Redis itself is gone. This must NOT read as unreachable storage, or the + # operator debugs the wrong layer. + _patch_from_url(monkeypatch, _DeadRedis()) + + status = health.get_storage_status() + assert status.get("available") is False, status + assert status.get("state") == "unknown", ( + "a dead Redis says nothing about the mount; reporting it as " + f"'unreachable' sends the operator to the wrong layer: {status}" + ) + + +def test_a_wedged_node_is_not_masked_by_a_healthy_one(fake_redis, monkeypatch, tmp_path): + """ + Two nodes, one of them wedged. This is the whole reason the heartbeat is + keyed per node, and collapsing the keys with `any()` gives it away: the + indicator would report "connected - 1 node reporting a healthy shared + volume" and stay green straight through the outage, which is exactly what + a single shared key would have done. + """ + assert _probe_node("node-a", tmp_path) is True + assert _probe_node("node-b", tmp_path) is True + assert health.get_storage_status().get("state") == "connected" + + # node-b's mount wedges: its readiness probe blocks in the write and never + # reaches its heartbeat. Its registration keeps being refreshed, because + # that happens before anything can block. + fake_redis.delete(health.storage_heartbeat_key("node-b")) + health.register_storage_node("node-b") + + status = health.get_storage_status() + assert status.get("state") != "connected", ( + "one node's mount is wedged and the indicator is still green: the per " + f"node keys are being collapsed rather than compared: {status}" + ) + assert status.get("state") == "degraded", status + assert _nodes(status) == {"node-a"}, status + assert set(status.get("stale_nodes") or []) == {"node-b"}, ( + "the wedged node has to be named, or an operator cannot tell which " + f"node to look at: {status}" + ) + + +def test_local_mode_reports_nothing(monkeypatch): + """No REDIS_URL means local mode; the sidebar renders no indicator at all.""" + monkeypatch.delenv("REDIS_URL", raising=False) + _patch_from_url(monkeypatch, _DeadRedis()) + + assert health.get_storage_status() == {}, ( + "with no REDIS_URL the indicator is not applicable, which is the empty " + "dict get_queue_metrics() already returns - not an error state" + ) + + +def test_a_deployment_with_no_heartbeat_writer_reports_nothing(fake_redis): + """ + A reachable Redis that nobody publishes a heartbeat to is not a storage + outage - it is a deployment with no shared storage to report on. + + This is not hypothetical: `Dockerfile` bakes in `online_deployment: true` + *and* `ENV REDIS_URL`, `docker/entrypoint.sh` starts Redis in the same + container, and the only heartbeat writer in the repo is the Kubernetes + readiness probe. Reading "no keys" as "unreachable" puts a permanent red + "running workflows may be stalled" in every docker-compose and apptainer + deployment of the image, pointing at a layer that does not exist there. + """ + assert health.get_storage_status() == {}, ( + "nothing has ever published a heartbeat here, so there is nothing to " + "report; a red tick would be a false alarm about a mechanism that is " + "not deployed" + ) + + +def test_indicator_never_touches_filesystem(fake_redis, monkeypatch, tmp_path): + """ + The sidebar fragment re-runs every 5 seconds. A `stat` on a `hard`-mounted + wedged path blocks in uninterruptible sleep and cannot be killed, so the + indicator would become the very hang it exists to report. + + The tripwires below record *and* raise: recording alone would miss a call + swallowed by a broad `except`, and raising alone would let one be reported + as an ordinary unavailable status. + """ + assert _probe_node("node-a", tmp_path) is True + # Warm up any lazy imports before the filesystem is booby-trapped. + health.get_storage_status() + + touched = [] + + def _tripwire(name): + def _blocked(*args, **kwargs): + touched.append((name, args[:1])) + raise AssertionError(f"the storage indicator called {name}{args[:1]}") + + return _blocked + + targets = ( + "os.stat", + "os.lstat", + "os.open", + "os.listdir", + "os.scandir", + "os.access", + "os.utime", + "builtins.open", + ) + with ExitStack() as stack: + for target in targets: + stack.enter_context(mock.patch(target, new=_tripwire(target))) + status = health.get_storage_status() + + assert touched == [], ( + f"the sidebar must reach the mount only through Redis: {touched}" + ) + assert status.get("state") in {"connected", "degraded", "unreachable", "unknown"}, status + + +# --------------------------------------------------------------------------- +# The writer +# --------------------------------------------------------------------------- +def test_heartbeat_key_is_per_node(fake_redis, monkeypatch, tmp_path): + """ + Two writers on two nodes produce two keys, each with an expiry. A single + shared key would let a healthy node keep refreshing it while another node's + mount is wedged, and expiry is the entire liveness mechanism - a heartbeat + written without one never goes stale. + """ + monkeypatch.setenv("NODE_NAME", "node-a") + health.write_storage_heartbeat() + monkeypatch.setenv("NODE_NAME", "node-b") + health.write_storage_heartbeat() + + assert health.storage_heartbeat_key("node-a") == ( + health.STORAGE_HEARTBEAT_PREFIX + "node-a" + ) + assert _heartbeat_keys(fake_redis) == { + health.storage_heartbeat_key("node-a"), + health.storage_heartbeat_key("node-b"), + } + + for key in _heartbeat_keys(fake_redis): + ttl = fake_redis.ttl(key) + assert 0 < ttl <= health.STORAGE_HEARTBEAT_TTL, ( + f"{key} has ttl {ttl}; expiry is the whole liveness mechanism, so a " + "heartbeat written without one never goes stale" + ) + + assert _nodes(health.get_storage_status()) == {"node-a", "node-b"} + + +def test_the_probe_publishes_only_after_the_volume_answers( + fake_redis, monkeypatch, tmp_path +): + """ + The ordering inside the probe, asserted where a mutation to it cannot hide. + + A probe that publishes the heartbeat regardless - the `|| true` the shell + version wrapped it in - reports a healthy shared volume while the volume is + gone, and the indicator goes green through the outage it exists to show. + """ + monkeypatch.setenv("NODE_NAME", "node-a") + missing = tmp_path / "not" / "a" / "mount" + + assert health.probe_storage(workspaces_dir=str(missing)) is False, ( + "a volume that will not accept a write must fail the probe; the " + "readiness verdict is a statement about storage" + ) + assert fake_redis.exists(health.storage_heartbeat_key("node-a")) == 0, ( + "the probe published a healthy-storage heartbeat for a volume that " + "rejected the write" + ) + assert fake_redis.exists(health.storage_node_key("node-a")) == 1, ( + "a node whose mount is wedged must still register, or its missing " + "heartbeat is indistinguishable from a node that does not exist and " + "the indicator quietly goes green" + ) + assert health.get_storage_status().get("state") == "unreachable" + + # The volume comes back. + assert health.probe_storage(workspaces_dir=str(tmp_path)) is True + assert (tmp_path / health.STORAGE_SENTINEL_NAME).exists(), ( + "the probe must *write*: with the default actimeo a stat can be " + "answered from the client's attribute cache while the server is gone" + ) + assert health.get_storage_status().get("state") == "connected" + + +def test_the_probe_survives_a_dead_redis(monkeypatch, tmp_path): + """ + A Redis that is down says nothing about the mount, so it must not fail the + readiness probe: the worker would be marked NotReady for a fault in another + component entirely. + """ + monkeypatch.setenv("REDIS_URL", "redis://localhost:6379/0") + monkeypatch.setenv("NODE_NAME", "node-a") + _patch_from_url(monkeypatch, _DeadRedis()) + + assert health.probe_storage(workspaces_dir=str(tmp_path)) is True + assert (tmp_path / health.STORAGE_SENTINEL_NAME).exists() + + +def test_heartbeat_writer_needs_no_streamlit(tmp_path): + """ + The writer runs in the RQ worker, which has no Streamlit session and must + stay importable without the package - the same constraint tasks.py carries. + + The functions are *called*, not merely looked up: an `import streamlit` + inside a function body would sail past a `hasattr` check and only fail in + the worker, at the one moment nobody is watching. + """ + code = textwrap.dedent( + """ + import json + import sys + + sys.modules["streamlit"] = None # any `import streamlit` now raises + + from src.workflow import health + + for name in ( + "STORAGE_HEARTBEAT_PREFIX", + "STORAGE_NODE_PREFIX", + "STORAGE_HEARTBEAT_TTL", + "STORAGE_SENTINEL_NAME", + "storage_heartbeat_key", + "storage_node_key", + "write_storage_heartbeat", + "register_storage_node", + "probe_storage", + "get_storage_status", + ): + assert hasattr(health, name), "health." + name + " is missing" + + # Redis is deliberately unreachable: publishing must degrade to False, + # never raise, and never fail the probe on the mount's behalf. + assert health.probe_storage(workspaces_dir=sys.argv[1]) is True + assert health.write_storage_heartbeat(node="node-a") is False + assert health.register_storage_node(node="node-a") is False + assert health.storage_heartbeat_key("n") == health.STORAGE_HEARTBEAT_PREFIX + "n" + assert json.dumps(health.get_storage_status()) is not None + print("ok") + """ + ) + result = subprocess.run( + [sys.executable, "-c", code, str(tmp_path)], + cwd=str(REPO_ROOT), + capture_output=True, + text=True, + env=dict( + os.environ, + PYTHONPATH=str(REPO_ROOT), + # Port 1 is reserved and never listening: connection refused, + # immediately, with no dependency on the network. + REDIS_URL="redis://127.0.0.1:1/0", + ), + ) + assert result.returncode == 0, ( + "src/workflow/health.py must expose *and run* the storage heartbeat API " + f"without importing Streamlit.\nstdout: {result.stdout}\n" + f"stderr: {result.stderr}" + ) + assert (tmp_path / health.STORAGE_SENTINEL_NAME).exists() + + +# --------------------------------------------------------------------------- +# Bounding +# +# The sidebar re-runs its fragments every 5 seconds on the session's own +# ScriptRunner thread, so an unbounded Redis call does not degrade an +# indicator - it freezes the session. Bounding only the storage reader closes +# half the hang: `get_queue_metrics()` drives the fragment rendered directly +# above it, on the same cadence and the same thread. +# --------------------------------------------------------------------------- +REDIS_READERS = ( + "check_redis_health", + "check_worker_health", + "get_queue_metrics", + "get_storage_status", + "queue_workers_are_remote", +) + + +@pytest.mark.parametrize("call", REDIS_READERS) +def test_every_redis_call_is_bounded(monkeypatch, call): + """ + Every client this module builds carries both timeouts. + + Asserted on the construction rather than on the clock, because redis-py's + own defaults have moved over the versions this repo floats across + (`redis>=5.0.0`, unpinned): on one of them an "unbounded" call happens to + fail after five seconds and a stopwatch cannot tell the two apart. + """ + built = [] + + def _record(*args, **kwargs): + built.append(kwargs) + return _DeadRedis() + + monkeypatch.setenv("REDIS_URL", "redis://localhost:6379/0") + monkeypatch.setattr( + redis_module.Redis, + "from_url", + classmethod(lambda cls, *args, **kwargs: _record(*args, **kwargs)), + raising=False, + ) + + getattr(health, call)() + + assert built, f"health.{call}() never built a Redis client" + for kwargs in built: + assert kwargs.get("socket_connect_timeout") == health.REDIS_SOCKET_TIMEOUT, ( + f"health.{call}() connects without a bound: {kwargs}" + ) + assert kwargs.get("socket_timeout") == health.REDIS_SOCKET_TIMEOUT, ( + f"health.{call}() reads without a bound, so a Redis that accepts " + f"the connection and then never answers hangs the caller: {kwargs}" + ) + + +def test_a_redis_that_never_answers_does_not_hang_the_sidebar(monkeypatch): + """ + The same property end to end, against a socket that completes the TCP + handshake and then says nothing - a Redis behind a partition, a wedged + host, a dropped conntrack entry. A connect timeout alone does not cover it: + the connect succeeds and the read blocks. Nothing accepts the connection + here; the kernel's backlog completes the handshake on its own. + """ + import socket as socket_module + import threading + + server = socket_module.socket() + server.bind(("127.0.0.1", 0)) + server.listen(1) + monkeypatch.setenv("REDIS_URL", f"redis://127.0.0.1:{server.getsockname()[1]}/0") + + finished = threading.Event() + + def _call(): + try: + health.get_queue_metrics() + finally: + finished.set() + + worker = threading.Thread(target=_call, daemon=True) + started = time.monotonic() + worker.start() + try: + completed = finished.wait(timeout=30) + finally: + server.close() + elapsed = time.monotonic() - started + + assert completed, ( + "get_queue_metrics() never returned against a Redis that accepts the " + "connection and then never answers; called from a sidebar fragment " + f"that is the whole session frozen (waited {elapsed:.0f}s)" + ) + + +# --------------------------------------------------------------------------- +# Where the psutil panel applies +# --------------------------------------------------------------------------- +def _register_worker(client, hostname: str, name: str) -> None: + from rq import Worker + + worker = Worker(["openms-workflows"], connection=client, name=name) + worker.hostname = hostname + worker.register_birth() + + +def test_workers_in_this_container_are_not_remote(fake_redis): + """ + The full image runs Redis, the RQ workers and Streamlit in one container + (`docker/entrypoint.sh`), so psutil in the Streamlit process measures + exactly the machine the workflows run on. Hiding the panel there - which + gating it on `online_deployment` did, because the image bakes that flag + true - leaves the "Resource Utilization" expander with nothing in it. + """ + import socket + + _register_worker(fake_redis, socket.gethostname(), "worker-1") + + assert health.queue_workers_are_remote() is False + + +def test_workers_in_other_pods_are_remote(fake_redis): + """ + In Kubernetes the workers are separate pods, usually on separate nodes, so + psutil here describes an idle 2Gi Streamlit pod while a 64Gi worker + saturates. + """ + _register_worker(fake_redis, "template-app-rq-worker-7c9f-abcde", "worker-1") + _register_worker(fake_redis, "template-app-rq-worker-7c9f-fghij", "worker-2") + + assert health.queue_workers_are_remote() is True + + +def test_an_unreachable_queue_is_not_proof_of_remote_workers(monkeypatch): + """ + `Dockerfile_simple` has no Redis at all, and the apptainer branch of the + entrypoint exports `REDIS_URL` before checking whether redis-server exists. + Neither says the workflows run elsewhere - they run in this process - so + the panel stays. + """ + monkeypatch.setenv("REDIS_URL", "redis://localhost:6379/0") + _patch_from_url(monkeypatch, _DeadRedis()) + assert health.queue_workers_are_remote() is False + + monkeypatch.delenv("REDIS_URL", raising=False) + assert health.queue_workers_are_remote() is False + + +# --------------------------------------------------------------------------- +# The manifests +# +# The probe is a shell command in YAML, so it is extracted and *run* rather +# than grepped: a substring check for "readinessProbe" survives replacing the +# storage check's `|| exit 1` with `|| true`, which reports Ready on a wedged +# mount and is the entire failure the probe exists to catch. +# --------------------------------------------------------------------------- +def _readiness_probe_script() -> str: + """The shell body of rq-worker's readiness probe, dedented.""" + manifest = RQ_WORKER_DEPLOYMENT.read_text(encoding="utf-8") + assert "readinessProbe:" in manifest, ( + "rq-worker has no readiness probe, so a wedged mount is invisible: the " + "RQ parent keeps heartbeating while its work horse is blocked" + ) + block = manifest[manifest.index("readinessProbe:"):] + marker = "- |\n" + assert marker in block, "the probe's command is not a literal block scalar" + lines = block[block.index(marker) + len(marker):].splitlines() + indent = len(lines[0]) - len(lines[0].lstrip()) + body = [] + for line in lines: + if line.strip() and (len(line) - len(line.lstrip())) < indent: + break + body.append(line[indent:]) + return "\n".join(body).rstrip() + "\n" + + +def _require_probe_prerequisites() -> str: + """The bash that can run the probe, or a skip.""" + bash = _bash() + if bash is None: + pytest.skip("no bash to run the readiness probe with") + probe = subprocess.run( + [bash, "-c", "command -v timeout"], capture_output=True, text=True, check=False + ) + if probe.returncode != 0 or not probe.stdout.strip(): + pytest.skip("no `timeout` for the probe to bound itself with") + return bash + + +def _bash(): + found = shutil.which("bash") + if found: + return found + for candidate in ( + Path("C:/Program Files/Git/bin/bash.exe"), + Path("C:/Program Files/Git/usr/bin/bash.exe"), + ): + if candidate.exists(): + return str(candidate) + return None + + +def _posix(path) -> str: + text = Path(path).as_posix() + if os.name == "nt" and len(text) > 1 and text[1] == ":": + text = "/" + text[0].lower() + text[2:] + return text + + +def _run_readiness_probe(workspaces_dir) -> subprocess.CompletedProcess: + """ + Run the manifest's probe command, with only the container's interpreter + path swapped for this one. Everything else - the `timeout`, the module it + invokes, the exit status handling - is the shipped text. + """ + bash = _require_probe_prerequisites() + script = _readiness_probe_script() + script, count = re.subn( + r"(?m)^py=.*$", "py=" + _posix(sys.executable), script, count=1 + ) + assert count == 1, ( + "the probe no longer names its interpreter on a `py=` line; update " + f"this substitution:\n{script}" + ) + return subprocess.run( + [bash, "-c", script], + cwd=str(REPO_ROOT), + capture_output=True, + text=True, + env=dict( + os.environ, + PYTHONPATH=str(REPO_ROOT), + WORKSPACES_DIR=str(workspaces_dir), + # Unreachable on purpose: the probe's verdict is about storage, and + # a dead Redis must not change it in either direction. + REDIS_URL="redis://127.0.0.1:1/0", + NODE_NAME="node-a", + ), + timeout=120, + ) + + +def test_readiness_probe_passes_on_a_volume_that_accepts_a_write(tmp_path): + result = _run_readiness_probe(tmp_path) + + assert result.returncode == 0, ( + f"the probe failed on a healthy volume.\nstdout: {result.stdout}\n" + f"stderr: {result.stderr}" + ) + assert (tmp_path / health.STORAGE_SENTINEL_NAME).exists(), ( + "the probe did not write the sentinel; `stat` alone can be answered " + "from the client's attribute cache while the server is gone" + ) + + +def test_readiness_probe_fails_when_the_volume_rejects_a_write(tmp_path): + """ + The mutation that matters: replacing the storage check's `|| exit 1` with + `|| true` leaves every assertion about the probe's *shape* green while the + probe reports Ready on a wedged mount and publishes a heartbeat with it. + """ + result = _run_readiness_probe(tmp_path / "not" / "a" / "mount") + + assert result.returncode != 0, ( + "the probe reported Ready for a volume that would not accept a write.\n" + f"stdout: {result.stdout}\nstderr: {result.stderr}" + ) + + +def test_readiness_probe_has_no_escape_hatch(): + """ + Nothing in the probe may swallow a failure. Kept alongside the behavioural + tests above because `|| true` is how this gets broken - it looks like + defensive shell and it silently inverts the verdict. + """ + script = _readiness_probe_script() + + assert "src.workflow.health" in script, ( + "the probe must run the module that owns the sentinel name and the " + f"ordering, not a second copy of the logic in YAML:\n{script}" + ) + assert "--probe" in script, f"the probe must run `--probe`:\n{script}" + assert "|| true" not in script, ( + "a `|| true` in the readiness probe reports Ready on a wedged mount, " + f"which is the whole failure it exists to catch:\n{script}" + ) + assert "exit 0" not in script, ( + "an unconditional `exit 0` has the same effect as `|| true`:\n" + script + ) + + +def test_readiness_probe_uses_the_containers_own_interpreter(): + """ + The probe names an interpreter path outright instead of activating the + environment, because `conda activate` costs a few hundred milliseconds of + shell hooks on every probe. That buys a hardcoded path, which drifts + silently: rename the env or move the conda root and every probe fails with + "No such file or directory", the workers go NotReady, and the sidebar goes + red - all for a rename nothing pointed at this file. + """ + manifest = RQ_WORKER_DEPLOYMENT.read_text(encoding="utf-8") + + activate = re.search(r"source\s+(\S+)/bin/activate\s+(\S+)", manifest) + assert activate, ( + "the rq-worker container no longer activates a conda environment; the " + "probe's interpreter path has nothing left to agree with" + ) + conda_root, environment = activate.group(1), activate.group(2) + + interpreter = re.search(r"(?m)^py=(\S+)$", _readiness_probe_script()) + assert interpreter, "the probe does not name its interpreter on a `py=` line" + assert interpreter.group(1) == f"{conda_root}/envs/{environment}/bin/python", ( + f"the probe runs {interpreter.group(1)}, but the container itself runs " + f"in {environment} under {conda_root}" + ) + + +def test_worker_gets_its_node_name_from_the_downward_api(): + """ + Without `NODE_NAME`, `current_node_name()` falls back to the hostname - + the *pod* name in Kubernetes, which changes on every restart. The + heartbeat would then be keyed per pod, the registry would fill with dead + pod names, and the per node visibility the design turns on would be gone. + """ + manifest = RQ_WORKER_DEPLOYMENT.read_text(encoding="utf-8") + downward = re.search( + r"-\s+name:\s+NODE_NAME\s+valueFrom:\s+fieldRef:\s+fieldPath:\s+(\S+)", + manifest, + ) + assert downward, ( + "rq-worker does not take NODE_NAME from the Downward API:\n" + manifest + ) + assert downward.group(1) == "spec.nodeName", ( + f"NODE_NAME reads {downward.group(1)}, not the node's name" + ) + + +def test_worker_has_no_liveness_probe(): + manifest = RQ_WORKER_DEPLOYMENT.read_text(encoding="utf-8") + + assert "livenessProbe" not in manifest, ( + "never a liveness probe on rq-worker: it would kill in-flight TOPP jobs, " + "and restarting cannot fix NFS" + ) + + +def test_probe_timing_outlasts_the_nfs_grace_period(): + """ + A Ganesha restart imposes a ~90s NFSv4 grace period during which all I/O + blocks. Probe timing has to outlast it, or every routine restart flaps the + worker - and the sidebar must go red *before* the worker is marked + NotReady, so degradation is visible before it is fatal. + """ + manifest = RQ_WORKER_DEPLOYMENT.read_text(encoding="utf-8") + + def _value(field: str) -> int: + match = re.search(rf"(?m)^\s+{field}:\s+(\d+)\s*$", manifest) + assert match, f"the readiness probe declares no {field}" + return int(match.group(1)) + + period = _value("periodSeconds") + threshold = _value("failureThreshold") + kubelet_timeout = _value("timeoutSeconds") + + to_not_ready = (threshold - 1) * period + kubelet_timeout + assert to_not_ready > 90, ( + f"{to_not_ready}s of failure before NotReady does not outlast the ~90s " + "NFSv4 grace period; every Ganesha restart would flap the worker" + ) + assert health.STORAGE_HEARTBEAT_TTL >= 2 * period, ( + f"a heartbeat TTL of {health.STORAGE_HEARTBEAT_TTL}s against a " + f"{period}s probe period turns the indicator red on a single slow " + "probe, while the mount is healthy" + ) + assert health.STORAGE_HEARTBEAT_TTL < to_not_ready, ( + "the indicator must go red before the worker is marked NotReady: " + "degradation should be visible before it is fatal" + ) + + inner = re.search(r"timeout\s+-k\s+\d+\s+(\d+)", _readiness_probe_script()) + assert inner, "the probe does not bound itself with `timeout`" + assert int(inner.group(1)) < kubelet_timeout, ( + f"the probe's own bound ({inner.group(1)}s) is not inside kubelet's " + f"({kubelet_timeout}s), so a slow-but-alive mount never produces a " + "clean exit status" + ) + + +def test_cleanup_cronjob_cannot_wedge_forever(): + """ + `concurrencyPolicy: Forbid` means one Active Job suppresses every later + schedule, so a run hung on the shared mount silently ends nightly cleanup + forever. + + Both bounds are required. `activeDeadlineSeconds` is a control-plane bound + applied to a syscall-level hang: a pod in D-state cannot terminate, and on + recent Kubernetes the Job's terminal Failed condition is only added once + every pod has terminated, so Forbid can still suppress the next schedule. + The in-container `timeout` closes the reachable-but-slow cases without + depending on the cluster version. + """ + manifest = CLEANUP_CRONJOB.read_text(encoding="utf-8") + + assert "concurrencyPolicy: Forbid" in manifest, ( + "this test is about what Forbid does to a hung run; if the policy " + "changed, revisit both bounds below" + ) + deadline = re.search(r"(?m)^\s+activeDeadlineSeconds:\s+(\d+)\s*$", manifest) + assert deadline, ( + "the cleanup CronJob has no activeDeadlineSeconds, so one run wedged " + "on the workspaces mount blocks every subsequent night" + ) + + inner = re.search(r"timeout\s+-k\s+\d+\s+(\d+)\s", manifest) + assert inner, ( + "the cleanup command is not wrapped in `timeout`; activeDeadlineSeconds " + "alone leaves the container itself unbounded" + ) + assert int(inner.group(1)) < int(deadline.group(1)), ( + f"the in-container bound ({inner.group(1)}s) must fire before the Job " + f"deadline ({deadline.group(1)}s), or it never fires at all" + ) diff --git a/tests/test_streamlit_ui_param_read.py b/tests/test_streamlit_ui_param_read.py new file mode 100644 index 00000000..3b5ed483 --- /dev/null +++ b/tests/test_streamlit_ui_param_read.py @@ -0,0 +1,240 @@ +""" +``StreamlitUI._read_params_for_update()`` — the read half of the two +read-modify-write-back sites inside ``_input_TOPP_impl()``. + +Both of them merge into what they read and write the result straight back, and +they run on *every* ``input_TOPP()`` of *every* ``configure()`` render. A +tolerant empty dict there is the erasure bug at its highest frequency: it would +be written over ``_defaults``, ``_flag_params`` and every other tool's values +while looking like a successful save. + +Raising alone is not enough at this site, which is why it has its own method and +its own tests. The contract is: + +* an unreadable file skips the write entirely, so the bytes on disk stay + recoverable, and surfaces an error with a way forward; +* the way forward preserves the previous file rather than deleting it — the + most common cause of an unreadable params.json on a shared filesystem is a + healthy params.json behind a storage blip; +* a failure that is expected to clear by itself does *not* offer that reset at + all, because taking it during a Ganesha restart is what would destroy an + intact file; +* an absent file still reads as ``{}``: a first run has no parameters. +""" +import json +import os +import sys +from unittest.mock import MagicMock + +import pytest + +# Add project root to path for imports +PROJECT_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +sys.path.append(PROJECT_ROOT) + +# --------------------------------------------------------------------------- +# Import with `streamlit`, `pyopenms` and `src.common.common` mocked at the +# sys.modules level, mirroring tests/test_topp_flag_parameters.py. The method +# under test only calls st.error / st.button / st.toast / st.stop, and with a +# mocked st.stop() the documented fallback applies: it re-raises rather than +# falling through to the write, which is exactly the property to assert. +# --------------------------------------------------------------------------- +mock_streamlit = MagicMock() +mock_streamlit.session_state = {} + +_originals = { + name: sys.modules.get(name) + for name in ("streamlit", "pyopenms", "src.common.common") +} +sys.modules["streamlit"] = mock_streamlit +if _originals["pyopenms"] is None: + sys.modules["pyopenms"] = MagicMock() +if _originals["src.common.common"] is None: + # Pulls in captcha / psutil / pandas and a lot of Streamlit surface; none + # of it is reachable from the method under test. + sys.modules["src.common.common"] = MagicMock() + +from src.workflow.ParameterManager import ( + InvalidParameterFileError, + ParameterManager, + TransientParameterFileError, +) +from src.workflow.StreamlitUI import StreamlitUI + +for name, module in _originals.items(): + if module is not None: + sys.modules[name] = module + else: + sys.modules.pop(name, None) + +for _key in list(sys.modules.keys()): + if _key.startswith("src.workflow"): + sys.modules.pop(_key, None) + + +TOOL_INSTANCE = "FeatureFinderMetabo" + +STORED_PARAMS = { + "_defaults": {TOOL_INSTANCE: {"algorithm:common:noise_threshold_int": 1000.0}}, + "_flag_params": {TOOL_INSTANCE: ["force"]}, + "MetaboliteAdductDecharger": { + "algorithm:MetaboliteFeatureDeconvolution:charge_max": 2 + }, +} + + +@pytest.fixture(autouse=True) +def reset_streamlit_mock(): + """Fresh session state and a *falsy* st.button between tests.""" + mock_streamlit.reset_mock() + mock_streamlit.session_state = {} + # A bare MagicMock() return value is truthy, which would press every button + # on the page. + mock_streamlit.button.return_value = False + yield + mock_streamlit.reset_mock() + mock_streamlit.session_state = {} + + +def _ui(tmp_path): + """ + A StreamlitUI carrying only the member the method touches. + + __init__ is bypassed: it calls get_parameters_from_json(), which would + already have absorbed the failure under test. + """ + ui = object.__new__(StreamlitUI) + ui.parameter_manager = ParameterManager(tmp_path) + return ui + + +def _write_torn_params_file(ui) -> bytes: + """Leave params.json unparseable but with every original byte on disk.""" + text = json.dumps(STORED_PARAMS, indent=4) + '\n{\n "_defaults": {\n' + ui.parameter_manager.params_file.write_text(text, encoding="utf-8") + return ui.parameter_manager.params_file.read_bytes() + + +def test_readable_file_is_returned_unchanged(tmp_path): + ui = _ui(tmp_path) + ui.parameter_manager.params_file.write_text( + json.dumps(STORED_PARAMS), encoding="utf-8" + ) + + assert ui._read_params_for_update(TOOL_INSTANCE) == STORED_PARAMS + mock_streamlit.error.assert_not_called() + mock_streamlit.stop.assert_not_called() + + +def test_absent_file_reads_as_empty(tmp_path): + """A first run has no stored parameters; that is not a failure.""" + ui = _ui(tmp_path) + assert not ui.parameter_manager.params_file.exists() + + assert ui._read_params_for_update(TOOL_INSTANCE) == {} + mock_streamlit.error.assert_not_called() + mock_streamlit.stop.assert_not_called() + + +def test_unreadable_file_stops_instead_of_returning_an_empty_dict(tmp_path): + """ + The one thing that must never happen here: returning {} to the caller, + which merges it and writes it straight back. + """ + ui = _ui(tmp_path) + before = _write_torn_params_file(ui) + + with pytest.raises(InvalidParameterFileError): + ui._read_params_for_update(TOOL_INSTANCE) + + mock_streamlit.stop.assert_called_once() + assert mock_streamlit.error.called, "the user has to be told why nothing saved" + assert ui.parameter_manager.params_file.read_bytes() == before, ( + "the unreadable file must be preserved so it can still be repaired" + ) + + +def test_unreadable_file_offers_a_reset_keyed_per_tool_instance(tmp_path): + """ + Without an affordance the page stops on every render with no way forward. + The key has to carry the instance name: this runs once per tool fragment, + and duplicate widget keys raise. + """ + ui = _ui(tmp_path) + _write_torn_params_file(ui) + + with pytest.raises(InvalidParameterFileError): + ui._read_params_for_update(TOOL_INSTANCE) + + assert mock_streamlit.button.called + assert TOOL_INSTANCE in mock_streamlit.button.call_args.kwargs["key"] + + +def test_reset_preserves_the_unreadable_file(tmp_path): + """ + Pressing reset must not be the data loss. An unreadable params.json is very + often an intact params.json behind a storage blip, so the previous file is + moved aside rather than unlinked. + """ + ui = _ui(tmp_path) + before = _write_torn_params_file(ui) + mock_streamlit.button.return_value = True + + with pytest.raises(InvalidParameterFileError): + ui._read_params_for_update(TOOL_INSTANCE) + + assert not ui.parameter_manager.params_file.exists() + backups = [p for p in tmp_path.iterdir() if p.name.endswith(".bak")] + assert len(backups) == 1, f"expected the previous file to be kept, found {backups}" + assert backups[0].read_bytes() == before + + +def test_transient_failure_does_not_offer_the_destructive_reset(tmp_path, monkeypatch): + """ + ESTALE from a restarting NFS export is not corruption. Offering "reset to + defaults" there invites the user to delete a file that is about to become + readable again on its own. + """ + ui = _ui(tmp_path) + ui.parameter_manager.params_file.write_text( + json.dumps(STORED_PARAMS), encoding="utf-8" + ) + + def unreachable(): + raise TransientParameterFileError("storage did not answer") + + monkeypatch.setattr( + ui.parameter_manager, "read_parameters_strict", unreachable + ) + + with pytest.raises(TransientParameterFileError): + ui._read_params_for_update(TOOL_INSTANCE) + + mock_streamlit.stop.assert_called_once() + assert mock_streamlit.error.called + mock_streamlit.button.assert_not_called() + assert ui.parameter_manager.params_file.exists() + + +def test_a_failed_reset_is_reported_rather_than_raised(tmp_path, monkeypatch): + """ + If the reset itself cannot run, the button used to raise, the file stayed, + and the next render stopped again — an unrecoverable halt of the parameter + page. The failure has to be surfaced instead. + """ + ui = _ui(tmp_path) + _write_torn_params_file(ui) + mock_streamlit.button.return_value = True + + def refuse(): + raise PermissionError("read-only filesystem") + + monkeypatch.setattr( + ui.parameter_manager, "reset_to_default_parameters", refuse + ) + + with pytest.raises(InvalidParameterFileError): + ui._read_params_for_update(TOOL_INSTANCE) + + messages = " ".join(str(call) for call in mock_streamlit.error.call_args_list) + assert "read-only filesystem" in messages diff --git a/tests/test_tasks.py b/tests/test_tasks.py new file mode 100644 index 00000000..32a0ecf1 --- /dev/null +++ b/tests/test_tasks.py @@ -0,0 +1,484 @@ +""" +Tests for the failure signal of a workflow run. + +``src/workflow/tasks.py`` is the RQ worker entry point for online (queue) mode +and has no test coverage at all. Two defects there make a failed workflow +indistinguishable from a successful one: + + - ``tasks.py:105-108`` calls ``workflow.execution()``, discards its return + value and logs the ``WORKFLOW FINISHED`` marker unconditionally. A workflow + that reported failure by returning ``False`` still ends up with the success + marker in its log, which is the only thing + ``src/workflow/_log_status.classify_log_outcome`` looks at. + - ``tasks.py:144-148`` catches every exception and *returns a dict* instead of + re-raising. RQ only records a job as failed when the job function raises, so + ``FailedJobRegistry`` is structurally always empty and ``health.py``'s + ``failed_jobs`` metric can never be anything but zero. + +The other half of the same signal lives in ``src/Workflow.py``: ``execution()`` +is annotated ``-> None``, returns nothing, and throws away the ``bool`` that +every ``run_topp`` / ``run_python`` call returns. ``WorkflowManager.execution`` +is declared ``-> bool`` and ``WorkflowManager.workflow_process`` already gates +the marker on it, so the shipped example workflow renders +"Errors occurred, check log file." for a *successful* local run and says nothing +at all when a tool failed. + +The two halves have to be fixed together: honouring the bool in ``tasks.py`` +while ``execution()`` still returns ``None`` would mark every workflow failed. +Each test below therefore pins both directions of the signal, so neither defect +can be "fixed" by inverting it. +""" + +import os +import sys +import json +import time +import uuid +from pathlib import Path +from unittest.mock import MagicMock + +import pytest + +fakeredis = pytest.importorskip("fakeredis") +rq = pytest.importorskip("rq") +# src/Workflow.py pulls in the whole Streamlit UI stack (StreamlitUI -> +# src.common.common -> captcha / psutil / pandas), so the real packages are +# needed here rather than the sys.modules MagicMock used by the tests that only +# touch ParameterManager / CommandExecutor. +pytest.importorskip("streamlit") +pytest.importorskip("pyopenms") + +from rq import Queue, SimpleWorker + +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +from src.workflow import tasks +from src.workflow.FileManager import FileManager +from src.workflow.Logger import Logger +from src.workflow.QueueManager import QueueManager + +# Importing src.Workflow drags in src.workflow.{StreamlitUI, ParameterManager, +# CommandExecutor, ...} bound to the *real* streamlit. Later test modules +# (test_tool_instance_name.py, test_topp_flag_parameters.py) swap +# sys.modules['streamlit'] for a MagicMock before importing those same modules, +# and would silently get these real ones back from the module cache instead. +# Restore the pre-import view of sys.modules so they still import cleanly; the +# Workflow class keeps its own module globals either way. +_src_modules_before = { + k: v for k, v in sys.modules.items() if k == "src" or k.startswith("src.") +} + +from src.Workflow import Workflow + +for _key in [k for k in sys.modules if k == "src" or k.startswith("src.")]: + sys.modules.pop(_key, None) +sys.modules.update(_src_modules_before) + + +QUEUE_NAME = "openms-workflows-test" + + +class _FakeRedisSimpleWorker(SimpleWorker): + """ + SimpleWorker that skips RQ's start-up CLIENT LIST lookup. + + rq's Worker.__init__ reads client["addr"] out of CLIENT LIST to record the + worker's ip address; fakeredis answers CLIENT LIST without that field, so + constructing any worker against it raises KeyError before a single job + runs. ip_address is telemetry only and nothing under test reads it - the + job execution and success/failure bookkeeping below are untouched. + """ + + def _set_ip_address(self, connection) -> None: + self.ip_address = "unknown" + + +# --------------------------------------------------------------------------- +# Stand-in workflows. They live at module scope because execute_workflow +# resolves the class with importlib.import_module(workflow_module), and this +# test module is itself importable under its own __name__. +# --------------------------------------------------------------------------- + + +class RaisingWorkflow: + """A workflow that crashes, e.g. on a TOPP tool that is not installed.""" + + def execution(self) -> bool: + raise RuntimeError("deliberate workflow failure") + + +class FalseReturningWorkflow: + """A workflow reporting failure the documented way: by returning False.""" + + def execution(self) -> bool: + return False + + +class TrueReturningWorkflow: + """A workflow that succeeds.""" + + def execution(self) -> bool: + return True + + +class NoneReturningWorkflow: + """ + A workflow written against the older example: annotated -> None, returns + nothing. + + This is the shape src/Workflow.py shipped until Step 1, and the shape the + downstream subclasses (quantms-web, umetaflow, FLASHApp) still carry - they + live in their own repositories and cannot be fixed in the same commit as + tasks.py. + """ + + def execution(self) -> None: + return None + + +# ------------------------------- helpers ----------------------------------- + + +def _prepare_workflow_dir(tmp_path: Path, name: str) -> Path: + """Create a workflow directory with the params.json the worker expects.""" + workflow_dir = tmp_path / name + workflow_dir.mkdir(parents=True, exist_ok=True) + with open(Path(workflow_dir, "params.json"), "w", encoding="utf-8") as f: + json.dump({"max_threads": 1}, f) + return workflow_dir + + +def _run_in_worker(workflow_dir: Path, workflow_class: str) -> tuple: + """ + Enqueue tasks.execute_workflow and drain the queue with an in-process RQ + worker backed by fakeredis. Returns (queue, job_id). + + SimpleWorker performs the job in the current process (no fork), so the + stand-in workflow classes above are importable from the worker while RQ's + own success/failure bookkeeping - the thing under test - still runs for + real. + + Queue name and job id are unique per call: older fakeredis releases share + one server between bare FakeStrictRedis() instances, which would otherwise + let one run's registries leak into the next one's assertions. + """ + run_id = uuid.uuid4().hex[:8] + job_id = f"job-{workflow_class}-{run_id}" + connection = fakeredis.FakeStrictRedis() + queue = Queue(f"{QUEUE_NAME}-{run_id}", connection=connection) + queue.enqueue( + tasks.execute_workflow, + kwargs={ + "workflow_dir": str(workflow_dir), + "workflow_class": workflow_class, + "workflow_module": __name__, + }, + job_id=job_id, + ) + _FakeRedisSimpleWorker([queue], connection=connection).work( + burst=True, logging_level="CRITICAL" + ) + return queue, job_id + + +def _read_logs(workflow_dir: Path) -> dict: + """Return {log file name: content} for the three log tiers Logger writes.""" + log_dir = Path(workflow_dir, "logs") + logs = {} + for name in ("minimal.log", "commands-and-run-times.log", "all.log"): + log_file = Path(log_dir, name) + logs[name] = log_file.read_text(encoding="utf-8") if log_file.exists() else "" + return logs + + +def _make_workflow(workflow_dir: Path, params: dict, executor) -> Workflow: + """ + Build a src.Workflow.Workflow the way the RQ worker does: bypass __init__ + (it dereferences st.session_state) and inject the members execution() uses. + + Mirrors tasks.execute_workflow:81-88, so anything breaking here breaks the + worker too. + """ + workflow = object.__new__(Workflow) + workflow.name = workflow_dir.name + workflow.workflow_dir = workflow_dir + workflow.file_manager = FileManager(workflow_dir) + workflow.logger = Logger(workflow_dir) + workflow.parameter_manager = MagicMock() + workflow.executor = executor + workflow.params = params + # workflow_process() and execute_workflow() both create results/ before + # calling execution(); FileManager._create_results_sub_dir mkdirs without + # parents=True and would fail otherwise. + Path(workflow_dir, "results").mkdir(parents=True, exist_ok=True) + return workflow + + +def _make_executor(topp_results=(True, True), python_result=True) -> MagicMock: + """ + A stand-in CommandExecutor whose run_topp / run_python return bools. + + Both really do: run_python() used to be annotated -> None and discard + run_command()'s result, which made every `if not run_python(...)` guard in + execution() dead code and this mock's False a value production could never + produce. tests/test_command_executor_run_python.py pins the real contract. + """ + executor = MagicMock() + executor.run_topp.side_effect = list(topp_results) + executor.run_python.return_value = python_result + return executor + + +# =========================== tasks.execute_workflow ========================== + + +def test_failed_workflow_reaches_failed_registry(tmp_path): + """ + A workflow whose execution() raises must be recorded as failed by RQ. + + execute_workflow swallows the exception and returns + {"success": False, ...}, so from RQ's point of view the job returned + normally: it lands in the finished registry and FailedJobRegistry stays + empty. That makes health.py's failed_jobs metric structurally zero and + leaves Retry inert for application failures. + + The second half is the control for the fix: re-raising must stay confined + to actual failures, or "every job failed" would satisfy the first half. + """ + failing_dir = _prepare_workflow_dir(tmp_path, "failing-workflow") + queue, job_id = _run_in_worker(failing_dir, "RaisingWorkflow") + + failed_ids = queue.failed_job_registry.get_job_ids() + finished_ids = queue.finished_job_registry.get_job_ids() + + assert failed_ids == [job_id], ( + "A crashed workflow must reach RQ's FailedJobRegistry " + f"(failed={failed_ids}, finished={finished_ids}). execute_workflow has " + "to re-raise instead of returning a result dict." + ) + assert finished_ids == [], ( + "A crashed workflow must not be recorded as finished successfully." + ) + + ok_dir = _prepare_workflow_dir(tmp_path, "succeeding-workflow") + ok_queue, ok_job_id = _run_in_worker(ok_dir, "TrueReturningWorkflow") + + assert ok_queue.failed_job_registry.get_job_ids() == [], ( + "A successful workflow must not be reported as failed." + ) + assert ok_queue.finished_job_registry.get_job_ids() == [ok_job_id] + + +def test_execution_returning_false_is_a_failure(tmp_path): + """ + Returning False from execution() is the documented failure signal + (WorkflowManager.execution is declared -> bool and workflow_process gates + the marker on it). The worker must honour it and *not* write the + WORKFLOW FINISHED marker, which is what classify_log_outcome reads when it + decides a run succeeded. + + The second half pins the marker string and the log tiers that receive it + (as tests/test_workflow_manager_stop.py does for WORKFLOW CANCELLED), so + the first half cannot be satisfied by never writing the marker at all. + """ + failing_dir = _prepare_workflow_dir(tmp_path, "failing-workflow") + queue, job_id = _run_in_worker(failing_dir, "FalseReturningWorkflow") + + # Returning False is the *documented* failure signal and by far the most + # common one - "No mzML files selected" reaches it. Asserting only on the + # log markers would let execute_workflow swallow it back into a + # {"success": False} return, leaving the FailedJobRegistry empty and + # health.py's failed_jobs metric structurally zero all over again. + assert queue.failed_job_registry.get_job_ids() == [job_id], ( + "a workflow reporting failure by returning False must reach RQ's " + "FailedJobRegistry, not just be missing a log marker" + ) + assert queue.finished_job_registry.get_job_ids() == [] + + for name, content in _read_logs(failing_dir).items(): + assert "STARTING WORKFLOW" in content, ( + f"{name} should carry the start marker - the run did happen." + ) + assert "WORKFLOW FINISHED" not in content, ( + f"{name} must not claim the workflow finished: execution() returned " + "False. execute_workflow logs the marker unconditionally, so a " + "failed queued run is reported to the user as a success." + ) + + ok_dir = _prepare_workflow_dir(tmp_path, "succeeding-workflow") + _run_in_worker(ok_dir, "TrueReturningWorkflow") + + for name, content in _read_logs(ok_dir).items(): + assert "WORKFLOW FINISHED" in content, ( + f"{name} must still carry the finished marker when execution() " + "returned True." + ) + + +def test_legacy_execution_returning_none_is_accepted_as_success(tmp_path): + """ + A subclass still annotated -> None must not have every run reported failed. + + Queue mode used to log WORKFLOW FINISHED unconditionally, so for the forks + carrying the old example's signature every queued run was recorded + successful. Honouring the bool strictly would flip all of them to failed on + the first rebase, for runs in which every step succeeded - and the plan's + "ship it in the same commit as Workflow.py" rule cannot reach a subclass + that lives in another repository. None is therefore accepted as success, + loudly; an explicit False stays a failure, which is what + test_execution_returning_false_is_a_failure pins. + """ + legacy_dir = _prepare_workflow_dir(tmp_path, "legacy-workflow") + queue, job_id = _run_in_worker(legacy_dir, "NoneReturningWorkflow") + + assert queue.failed_job_registry.get_job_ids() == [], ( + "a legacy execution() returning None must not be reported as a failure" + ) + assert queue.finished_job_registry.get_job_ids() == [job_id] + + logs = _read_logs(legacy_dir) + assert "WORKFLOW FINISHED" in logs["minimal.log"] + assert "execution() returned None" in logs["minimal.log"], ( + "accepting None silently would leave the fork with no signal to " + "migrate on; the deprecation has to reach the log the user reads" + ) + + +def test_failed_jobs_are_evicted_rather_than_kept_for_a_year(tmp_path, monkeypatch): + """ + Failed jobs are brand-new state, and they must expire. + + Before execute_workflow started re-raising, the FailedJobRegistry was + structurally always empty - that was the defect. Now every failure keeps a + job hash with its full traceback in Redis, and RQ's default failure_ttl is + one year. That Redis has memory request == limit == 256Mi, no persistence + and no maxmemory-policy, so it is OOMKilled rather than evicting; "No mzML + files selected" is a routine user error and must not leave a year-lived key + behind. + """ + monkeypatch.chdir(tmp_path) + monkeypatch.delenv("REDIS_URL", raising=False) + Path(tmp_path, "settings.json").write_text( + json.dumps({"online_deployment": False}), encoding="utf-8" + ) + + connection = fakeredis.FakeStrictRedis() + queue = Queue(f"{QUEUE_NAME}-{uuid.uuid4().hex[:8]}", connection=connection) + + # Built through __init__ so the retention actually comes from the + # configured defaults, then pointed at the fake queue - local mode means no + # Redis connection was attempted during construction. + manager = QueueManager() + manager._is_online = True + manager._queue = queue + + failing_dir = _prepare_workflow_dir(tmp_path, "failing-workflow") + job_id = manager.submit_job( + tasks.execute_workflow, + kwargs={ + "workflow_dir": str(failing_dir), + "workflow_class": "RaisingWorkflow", + "workflow_module": __name__, + }, + job_id=f"ttl-job-{uuid.uuid4().hex[:8]}", + ) + assert job_id is not None, "the job was not enqueued at all" + + _FakeRedisSimpleWorker([queue], connection=connection).work( + burst=True, logging_level="CRITICAL" + ) + + registry = queue.failed_job_registry + assert registry.get_job_ids() == [job_id] + + expires_in = connection.zscore(registry.key, job_id) - time.time() + assert 0 < expires_in <= 7 * 24 * 3600, ( + "the failed job is retained for " + f"{expires_in / 86400:.1f} days; enqueue has to pass failure_ttl " + "alongside result_ttl, or RQ keeps it for a year" + ) + + +# ============================ src.Workflow.Workflow ========================== + + +def test_execution_returns_true_on_success(tmp_path): + """ + The shipped example workflow must report success by returning True. + + It is annotated -> None and returns nothing, so workflow_process() never + logs WORKFLOW FINISHED and a successful local run renders + "Errors occurred, check log file." + """ + workflow_dir = _prepare_workflow_dir(tmp_path, "topp-workflow") + executor = _make_executor() + workflow = _make_workflow( + workflow_dir, + {"mzML-files": ["a.mzML", "b.mzML"], "run-python-script": False}, + executor, + ) + + result = workflow.execution() + + assert result is True, ( + f"execution() must return True when every step succeeded; got {result!r}." + ) + # Proves the True did not come from an early exit: both TOPP steps and the + # consensus export actually ran. + assert [c.args[0] for c in executor.run_topp.call_args_list] == [ + "FeatureFinderMetabo", + "FeatureLinkerUnlabeledKD", + ] + assert executor.run_python.call_count == 1 + + +def test_execution_returns_false_on_missing_input(tmp_path): + """ + "No mzML files selected" is the workflow's own parameter check. It logs an + ERROR and bails, but returns None, so the caller cannot tell it apart from + a completed run. + """ + workflow_dir = _prepare_workflow_dir(tmp_path, "topp-workflow") + executor = _make_executor() + workflow = _make_workflow( + workflow_dir, {"mzML-files": [], "run-python-script": False}, executor + ) + + result = workflow.execution() + + assert result is False, ( + f"execution() must return False when the input check fails; got {result!r}." + ) + executor.run_topp.assert_not_called() + assert "ERROR: No mzML files selected." in _read_logs(workflow_dir)["minimal.log"] + + +@pytest.mark.parametrize( + "topp_results, python_result", + [ + ((False, True), True), + ((True, False), True), + ((True, True), False), + ], + ids=["feature-detection-fails", "feature-linking-fails", "python-export-fails"], +) +def test_tool_failure_propagates(tmp_path, topp_results, python_result): + """ + run_topp / run_python already return a bool. execution() discards every one + of them, so a tool that failed - a missing binary, a bad parameter, a + SIGKILLed process - is reported to the user as a completed workflow. + """ + workflow_dir = _prepare_workflow_dir(tmp_path, "topp-workflow") + executor = _make_executor(topp_results=topp_results, python_result=python_result) + workflow = _make_workflow( + workflow_dir, + {"mzML-files": ["a.mzML", "b.mzML"], "run-python-script": False}, + executor, + ) + + result = workflow.execution() + + assert result is False, ( + "execution() must return False as soon as an executor call fails; " + f"got {result!r}." + ) diff --git a/tests/test_topp_flag_parameters.py b/tests/test_topp_flag_parameters.py index 627f50ee..3dcd10ef 100644 --- a/tests/test_topp_flag_parameters.py +++ b/tests/test_topp_flag_parameters.py @@ -97,8 +97,16 @@ def build_command( ``run_command`` / ``run_multiple_commands`` are stubbed so nothing is executed; the built command is captured from the ``run_command`` mock. + ``max_threads`` is pinned to 1 so the trailing ``-threads`` argument is - deterministic. + deterministic, and pinning it takes *both* halves: ``_get_max_threads()`` + resolves ``online_deployment`` from a ``settings.json`` read relative to the + process working directory, and only consults the workspace ``params.json`` + on the local branch. So the temporary directory gets its own local-mode + ``settings.json`` and becomes the CWD for the call — otherwise these argv + assertions would silently depend on the repository-root config and on where + pytest happens to be run from, and a fork shipping + ``online_deployment: true`` would see ``-threads 2`` here. """ if input_output is None: input_output = {"in": ["input.mzML"], "out": ["output.featureXML"]} @@ -108,6 +116,12 @@ def build_command( with tempfile.TemporaryDirectory() as tmpdir: workflow_dir = Path(tmpdir) + with open(Path(tmpdir, "settings.json"), "w", encoding="utf-8") as f: + json.dump( + {"online_deployment": False, "max_threads": {"local": 1, "online": 1}}, + f, + ) + pm = ParameterManager(workflow_dir) with open(pm.params_file, "w", encoding="utf-8") as f: json.dump(params_json, f) @@ -118,12 +132,17 @@ def build_command( executor.run_command = MagicMock(return_value=True) executor.run_multiple_commands = MagicMock(return_value=True) - executor.run_topp( - tool, - input_output, - custom_params=custom_params or {}, - tool_instance_name=tool_instance_name or tool, - ) + previous_cwd = os.getcwd() + os.chdir(tmpdir) + try: + executor.run_topp( + tool, + input_output, + custom_params=custom_params or {}, + tool_instance_name=tool_instance_name or tool, + ) + finally: + os.chdir(previous_cwd) assert executor.run_command.call_count == 1, ( "expected exactly one single-process command, got " From d4088e52eec038b53eb5fa0dc28e56c0a81da593 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=20David=20M=C3=BCller?= <57191390+t0mdavid-m@users.noreply.github.com> Date: Mon, 31 Aug 2026 20:15:58 +0200 Subject: [PATCH 24/29] fix(workflow): stop losing widget input on Streamlit < 1.50 (#402) * fix(workflow): stop losing widget input on Streamlit < 1.50 `_input_widget_impl` fed the persisted parameter back into every widget as its initial-value argument (`value=` / `default=` / `index=`). On Streamlit < 1.50 that argument is hashed into the widget's element id, so the id changed on every interaction; the following interaction then arrived under the now-stale id and was silently discarded. The visible effect was in the mzML file selector: selecting six files kept only three, because every second click was dropped. Verified against the real `StreamlitUI._input_widget_impl` and `ParameterManager` on streamlit 1.49.1, the pinned version: before: 6 clicked -> 3 stick ['A.mzML', 'C.mzML', 'E.mzML'] after: 6 clicked -> 6 stick Streamlit ignores the initial-value argument once a keyed widget already has an entry in session state, so seeding on first render only is behaviour-preserving: `apply_preset()` and `clear_parameter_session_state()` already delete the session keys when they want params.json to take effect again. Regression-tested for text, number (int and float), checkbox, selectbox, slider and multiselect - first-render seeding, reload from params.json and value type are all unchanged, on both 1.49.1 and 1.53.1. Also sorts the `select_input_file` options: `path.iterdir()` is unordered, so both the displayed order and (on < 1.50) the element id varied between restarts. Streamlit >= 1.50 stopped hashing `default` into the element id and was already unaffected; this fix is version-independent. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_016Qn3mLqr7zBr7rgCokx6ku * fix(ci): bump pinned cmake to 3.31.12, the pin no longer resolves `build-openms` started failing on every branch with: cmake - cmake not installed. The package was not found with the source(s) listed. Version was specified as '3.31.1'. Chocolatey delisted that exact patch: its package listing feed now returns 3.31.6, 3.31.10, 3.31.11 and 3.31.12 for the 3.31 series, but not 3.31.1. Nothing in this repo changed - the last green run was 2026-08-29. Moves to 3.31.12, the newest patch in the same minor series, deliberately staying off cmake 4.x: 4.0 dropped compatibility with `cmake_minimum_required(VERSION < 3.5)`, which the OpenMS build still relies on. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_016Qn3mLqr7zBr7rgCokx6ku * fix(tests): make the streamlit mock in test_tool_instance_name actually bind The module swapped `sys.modules['streamlit']` for a MagicMock, imported ParameterManager, restored the real streamlit, and only then dropped the cached `src.workflow` modules. That ordering is wrong: a module binds `st` once, at import time, so if any earlier-collected test had already imported `src.workflow.ParameterManager` against the real streamlit, the import here was just a cache hit and the mock never took effect. The result was a test that passed alone and failed in a full run - the six `TestSaveParametersWithInstanceName` cases asserted against `mock_streamlit.session_state` while ParameterManager was reading the real one. Drops the cached modules *before* the import so the mock binds, and again afterwards so later test files re-import against the real streamlit. pytest tests/test_results_helpers.py tests/test_tool_instance_name.py before: 6 failed, 6 passed after: 12 passed pytest tests/test_workflow_manager_stop.py tests/test_tool_instance_name.py before: 6 failed, 5 passed after: 11 passed Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_016Qn3mLqr7zBr7rgCokx6ku --------- Co-authored-by: Claude Opus 5 (1M context) --- .../build-windows-executable-app.yaml | 2 +- src/workflow/StreamlitUI.py | 32 +++++++++++++------ tests/test_tool_instance_name.py | 24 +++++++++++--- 3 files changed, 43 insertions(+), 15 deletions(-) diff --git a/.github/workflows/build-windows-executable-app.yaml b/.github/workflows/build-windows-executable-app.yaml index 99b41773..03748875 100644 --- a/.github/workflows/build-windows-executable-app.yaml +++ b/.github/workflows/build-windows-executable-app.yaml @@ -55,7 +55,7 @@ jobs: shell: bash run: | choco install ccache ninja -y --no-progress - choco install cmake --version=3.31.1 -y --no-progress --force + choco install cmake --version=3.31.12 -y --no-progress --force ## GH CLI "SHOULD BE" installed. Sometimes I had to manually install nonetheless. Super weird. # https://github.com/actions/runner-images/blob/main/images/win/scripts/Installers/Install-GitHub-CLI.ps1 echo "C:/Program Files (x86)/GitHub CLI" >> $GITHUB_PATH diff --git a/src/workflow/StreamlitUI.py b/src/workflow/StreamlitUI.py index f9adf5ec..69cf3264 100644 --- a/src/workflow/StreamlitUI.py +++ b/src/workflow/StreamlitUI.py @@ -559,7 +559,9 @@ def _select_input_file_impl(self, key, name, multiple, display_file_path, reacti if not path.exists(): st.warning(f"No **{name}** files!") return - options = [str(f) for f in path.iterdir() if "external_files.txt" not in str(f)] + options = sorted( + str(f) for f in path.iterdir() if "external_files.txt" not in str(f) + ) # Check if local files are available external_files = Path( @@ -675,11 +677,21 @@ def format_files(input: Any) -> List[str]: key = f"{self.parameter_manager.param_prefix}{key}" + # Streamlit ignores a widget's initial-value argument (value=/default=/index=) + # once that key already exists in session state -- but on Streamlit < 1.50 the + # argument is still hashed into the widget's element id. Since this method feeds + # the persisted parameter straight back in as that argument, the id changed on + # every interaction, the following interaction arrived under the now-stale id and + # was silently dropped: selecting six mzML files kept only three. Seed the widget + # on first render only; from then on session state owns the value. + def seed(**kwargs: Any) -> dict: + return {} if key in st.session_state else kwargs + if widget_type == "text": - st.text_input(name, value=value, key=key, help=help, on_change=on_change) + st.text_input(name, key=key, help=help, on_change=on_change, **seed(value=value)) elif widget_type == "textarea": - st.text_area(name, value=value, key=key, help=help, on_change=on_change) + st.text_area(name, key=key, help=help, on_change=on_change, **seed(value=value)) elif widget_type == "number": number_type = float if isinstance(value, float) else int @@ -693,27 +705,27 @@ def format_files(input: Any) -> List[str]: name, min_value=min_value, max_value=max_value, - value=value, step=step_size, format=None, key=key, help=help, on_change=on_change, + **seed(value=value), ) elif widget_type == "checkbox": - st.checkbox(name, value=value, key=key, help=help, on_change=on_change) + st.checkbox(name, key=key, help=help, on_change=on_change, **seed(value=value)) elif widget_type == "selectbox": if options is not None: st.selectbox( name, options=options, - index=options.index(value) if value in options else 0, key=key, format_func=format_files, help=help, on_change=on_change, + **seed(index=options.index(value) if value in options else 0), ) else: st.warning(f"Select widget '{name}' requires options parameter") @@ -723,11 +735,11 @@ def format_files(input: Any) -> List[str]: st.multiselect( name, options=options, - default=value, key=key, format_func=format_files, help=help, on_change=on_change, + **seed(default=value), ) else: st.warning(f"Select widget '{name}' requires options parameter") @@ -744,12 +756,12 @@ def format_files(input: Any) -> List[str]: name, min_value=min_value, max_value=max_value, - value=value, step=step_size, key=key, format=None, help=help, on_change=on_change, + **seed(value=value), ) else: st.warning( @@ -757,12 +769,12 @@ def format_files(input: Any) -> List[str]: ) elif widget_type == "password": - st.text_input(name, value=value, type="password", key=key, help=help, on_change=on_change) + st.text_input(name, type="password", key=key, help=help, on_change=on_change, **seed(value=value)) elif widget_type == "auto": # Auto-determine widget type based on value if isinstance(value, bool): - st.checkbox(name, value=value, key=key, help=help, on_change=on_change) + st.checkbox(name, key=key, help=help, on_change=on_change, **seed(value=value)) elif isinstance(value, (int, float)): self._input_widget_impl( key, diff --git a/tests/test_tool_instance_name.py b/tests/test_tool_instance_name.py index cd060cac..d1eef17a 100644 --- a/tests/test_tool_instance_name.py +++ b/tests/test_tool_instance_name.py @@ -24,6 +24,23 @@ _original_streamlit = sys.modules.get('streamlit') sys.modules['streamlit'] = mock_streamlit + +def _drop_cached_workflow_modules() -> None: + """Forget any cached src.workflow modules. + + A module binds `st` once, at import time. If an earlier test file has already + imported src.workflow.ParameterManager against the real streamlit, the import + below is just a cache hit and the mock never takes effect - which is why these + tests passed when run alone but failed in a full-suite run. + """ + for _key in list(sys.modules.keys()): + if _key.startswith('src.workflow'): + sys.modules.pop(_key, None) + + +# Drop first, so the import below really binds the mock. +_drop_cached_workflow_modules() + from src.workflow.ParameterManager import ParameterManager if _original_streamlit is not None: @@ -31,10 +48,9 @@ else: sys.modules.pop('streamlit', None) -# Remove cached src.workflow modules -for _key in list(sys.modules.keys()): - if _key.startswith('src.workflow'): - sys.modules.pop(_key, None) +# Drop again, so later test files re-import against the real streamlit instead of +# the mock-bound modules this file just created. +_drop_cached_workflow_modules() @pytest.fixture From a9a3c7aeed67003ab002f7faa51849727c144c99 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=20David=20M=C3=BCller?= <57191390+t0mdavid-m@users.noreply.github.com> Date: Mon, 7 Sep 2026 22:45:48 +0200 Subject: [PATCH 25/29] Add the notebook-to-webapp skill framework (#403) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Add the notebook-to-webapp skill framework Six skills that turn a Jupyter notebook into an app built on this template, plus the docs and glossary they refer to. `notebook-to-webapp` orchestrates; the five stage skills also work standalone against an existing app: capture-notebook-workflow classify cells, extract a Streamlit-free python tool, golden-value test interview-parameters one batch review of demo shortcuts and config parameters, ranked by provenance scaffold-workflow-app WorkflowManager subclass, four pages, registration, template cleanup, smoke run build-insight-dashboard link graph, wireframe, panels, style verify-webapp-usability headless-browser gate for pages AppTest cannot see Users enter through the Quickstart page, which holds a prompt they paste into an agentic terminal; it names the orchestrator by file path, because a repo cloned mid-session has its skills on disk but not in the running session's registry. The eight existing task playbooks move from `.md` to `/SKILL.md` with frontmatter, so they surface as loadable skills alongside the new ones rather than needing to be found by hand. Documentation gains a "From Notebook to Web App" chapter, rendered from docs/notebook_to_webapp.md like the others. Design rationale is in docs/notebook-to-webapp-design.md and docs/adr/0001..0005; the glossary is CONTEXT.md. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_017KoGDZbCevZMRYSJmDbwDG * The template's own source is machinery in a user-facing turn A build explained a preset that would not refresh by naming `StreamlitUI.preset_buttons`, `ParameterManager.apply_preset:661` and "a Python tool" to a mass spectrometrist. Its decision was right — it moved the button into the workflow rather than patching shared template code — but the sentence handed over two class names and a line number. The rule against slipping into instrument language when something goes wrong already covered this in spirit; every example it named was one of the framework's own checks, so the template's source read as outside it. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_017KoGDZbCevZMRYSJmDbwDG * The ending never asks an open question A build closed with "The Results page reads well as it stands — shall we call it done, or is there something on it you'd change?" — a decision put to the user with nothing to select, which is the one shape this framework refuses everywhere else, and which re-opens the round they had just closed. Found by eval/judge.py: shown two transcripts of the same notebook without being told which was which, a judge picked the older one and gave this ending as a reason. The register check could not see it — it looks for drawn controls and framework vocabulary, and an open question is neither. The "at most one offer" line now says what an offer is: it names the thing you would do next. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_017KoGDZbCevZMRYSJmDbwDG * Preflight: control absent is not control disconnected A real run on Linux ended preflight saying "Everything's set up ... and I can open pages in your Chrome and check them myself", retracted the second half later — "Correct — I can't" — and only then raised the extension: "I'd have to ask you to set that up". Three gaps behind that: - The ladder assumed the probe runs and answers. When ToolSearch returns no schema there is no probe, and launching a browser cannot create one. That is now its own rung, distinguished from a schema that loads and answers "not connected", which launching and re-probing can still fix. - The offer to install was written inside a code block as somewhere the ladder ends up. It is now an instruction with its consequence: raise it in preflight or not at all, because raising it later lands the request in the middle of a decision — the thing this step exists to prevent. - Nothing forbade a summary sentence that joined opening a page to checking one. They are the two rows of the table above and have different requirements, so until a page has been driven the summary says what was installed and nothing about checking. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_017KoGDZbCevZMRYSJmDbwDG * Never resolve a fork yourself and report it as applied A run opened with "Two calls before I build the config page", laid out the evidence for both, offered nothing to select, and followed with "Applied: relative precursor tolerance now defaults to 0, fragment tolerance to 20 ppm" — then spent a later turn explaining that at 20 ppm the best match explains 5 peaks rather than the notebook's 33. Every word of the evidence was right and the user decided nothing. Naming something a decision and then making it reads as consultation and works as an announcement. Found by eval/judge.py on a blind pair; register.py scored the same build 0 findings, because the shape has no tick-boxes, no framework vocabulary and sits under the word budget. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_017KoGDZbCevZMRYSJmDbwDG * Take the user to the extension install, don't defer it A run ended preflight with "Happy to point you at the setup for next time if you want it", turning a one-click fix available right then into homework. Claude cannot finish the install — the extension comes from the store, in the user's browser, under their claude.ai account, so the click and the sign-in are theirs — but everything up to that point is not. Preflight now opens the install page and says the one thing to click. Where the tools load but report nothing answering, the run carries on with capture while they install and re-probes before the first design round. Where the tools have no schema at all, nothing done now can help this run, so it waits and hands them the restart: "one click to add it, then start me again — nothing's been decided yet". Those two failures looked alike and are not, so CONTEXT.md now distinguishes control absent from control disconnected, and defines browser control against merely opening a page. The prohibition on naming the extension is narrowed rather than dropped: never to explain a limitation, which tells a mass spectrometrist about this framework's wiring and gives them nothing to do; only inside a request, where it comes with an action and a reason. After a decline it is not named again. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_017KoGDZbCevZMRYSJmDbwDG * A browser that answers is not necessarily your browser A run drove a browser successfully for a whole build, pointed it at localhost:8577, and read a different Streamlit app on that port — reporting its pages back as the user's. Chrome was on Windows, the shell was Linux, and the extension reported the browser as not local. Nothing in the tab context distinguishes the two cases: it reads "New Tab" chrome://newtab/ either way. So confirming control now has a second half. Serve a token from this machine, drive the browser to it, require the token back. A browser that fails that is never driven — a screenshot of the wrong app is worse than no screenshot, because it is a finding about someone else's software delivered as theirs. The remedy splits by what the extension says. Not local: the browser is on another machine and there is no second one to switch to, but the app binds every interface, so the question is which address that host can route to — try one and re-marker. Local but still failing: the extension is answering for a browser not signed in under the same account, which the probe's own error names. That run ended "Nothing to fix from here", which is the third failure. Falling back to the headless browser is fine; declaring a dead end is not. The cost is stated once in the user's terms, with the one thing that would change it. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_017KoGDZbCevZMRYSJmDbwDG * Factor browser setup out into connect-browser-control Preflight step 4 had grown to roughly 150 lines — find a browser, launch it, probe, confirm by driving, prove it is this machine's browser, split the remedy by locality, offer the install, state the fallback. It was the largest single block in the orchestrator and none of it is specific to notebook ports: anything that needs a driveable browser needs all of it. It is now its own skill, and the spine points at it. Two things the split let us state that were not stated before. Chromium only, and no Firefox. The native messaging host Claude Code registers declares a `chrome-extension://` origin, which Firefox cannot load, and it is registered for Chrome, Edge and Chromium alone. So the answer to "is there a Firefox extension" is no, and not one this framework can supply. Open the app in the browser you drive. A user whose default is Firefox was shown the app there while the framework checked a Chrome rendering — two pictures, and a header truncated in one sits fine in the other. With control confirmed, the app opens where the checks happen; without it, in their default, with no implication that anything can see it. That also fixes a rule this session introduced: "opening a page starts a browser, so re-probe" assumed what opened was Chromium. `xdg-open` opens their default, so the re-probe now happens only when what opened was the browser that was found. Rules and probe mappings moved with the text, since rules.json is keyed by skill directory. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_017KoGDZbCevZMRYSJmDbwDG * Name the browser to open, the cache to clear, the offer to make Three defects, two of them reported from a real run. A user who had just installed the extension in Chrome was shown their app in Firefox. The rule said to open the app in the browser being driven but gave no command, and every obvious command — start, open, xdg-open — opens the default. It now names the executable, using the two forms that were already verified for launching it. Insight keys its cache by config hash, so a panel can render unchanged after a real edit. That was one row of a table headed "component rules that are not obvious"; it is now at the moment the symptom appears, in the panel round, as the first suspect rather than a footnote. And this skill was supplying the very sentence handover.md now forbids: "the page reads well as it is — shall we call it done?" A blind judge picked an older build over a newer one partly for that ending, and the rule against it went into handover.md without noticing where the wording came from. The offer now names what would happen next, and points at handover.md for when the round is also the last turn. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_017KoGDZbCevZMRYSJmDbwDG * Consider only the attached browser; never explain it to the user A real run told a user "the browser reachable from this session lives on another machine, so I can't look at your finished pages myself... running this on the machine your browser is on is what would change that." It obeyed every rule the skill had -- cost stated once, in their terms, the extension unnamed -- and still explained the deployment topology of a coding assistant to a mass spectrometrist, then offered relocation as the fix. The prohibition named a noun (the extension) when the rule was about explaining a limitation at all, so the shape survived by swapping in another noun. Claude Code is routinely a shell on one machine and a browser on another, and the user picks neither. So: - Where the attached browser runs may route the next command. It is never a finding and never reaches the user. - The marker check asks whether the browser can reach this host, walking candidate addresses, not whether it is "ours". - With control, the app opens by navigating the browser already held -- the user's own window. No executable, no default-browser lookup, which makes the Firefox failure structurally impossible rather than forbidden. - The URL the user is given is always localhost:. - No preamble. Rounds ask them to look at one named thing when it matters, which is what a design round sounds like anyway. Also corrected two claims of my own. The native messaging host is registered for seven browsers (Chrome, Edge, Chromium, Brave, Arc, Opera, Vivaldi), not three -- the registry disagreed, and the old claim would have skipped a working Brave or Vivaldi. And the browser that rendered a different Streamlit app was most likely an earlier build of the user's own, not "someone else's software"; the cause never mattered, only that the pixels were different. coherence.py found five contradictions across five passes, all introduced by adding a rule beside an existing one without re-reading it -- including an exemption clause that leaked past its trigger. Clean on the fifth. 199/199 rule placements present, 0 retired claims live, 25/25 browser rules probe-reached, 0 corpus leaks. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_017KoGDZbCevZMRYSJmDbwDG * Never narrate preflight Measured in 10 builds of 19, across fifteen wordings: "Preflight - probing environment in parallel", "Now the browser side of the preflight", "Preflight: Python, browser, dependencies". Each hands a mass spectrometrist a list of this framework's own steps, and the word means nothing to them. register.py could not see it -- `preflight` was absent from its internal vocabulary, so every one of those builds scored clean. Adding it (and `browser control|setup|side`, keeping bare "browser" legal) turns four sampled builds from 0 findings into 1-3. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_017KoGDZbCevZMRYSJmDbwDG * Fix a stale count and reconcile two definitions of "stage" Both found by coherence.py, both pre-existing. "Five habits:" heads six bullets -- a09c6f7 inserted one and left the count. "A stage is something the user would name -- pulling the analysis out, checking it against their numbers, building the app" gave capture two nameable stages, while the Stages table gives it one and the announcement rule says "each stage except stage 5". Anchored the definition to the table so the two agree. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_017KoGDZbCevZMRYSJmDbwDG * Preflight's silence must not swallow the install request A live run on the user's own notebook read: "Setting up - a minute, then I'll ask about your notebook." then "Ready. What's the path to your notebook?" and never offered the browser extension at all. That is my regression. The narration finding got answered with prohibitions -- "neither state is ever explained", "no preamble", "never narrate this stage" -- stacked in two skills, and they reached past narration into the one turn preflight is supposed to produce. writing-skills says exactly this: a wrong-shape failure answered with prohibitions gets over-applied, and the fix is a positive recipe with a required slot. So both are now recipes rather than bans: connect-browser-control gains a four-row table that IS the whole user-facing output of browser setup. Three rows are silent. Row two -- no schema, or "not connected" -- is the install request, marked required, with the silence rules explicitly not reaching it: a run that stays quiet there has silently decided on the user's behalf that they would rather not have been asked. notebook-to-webapp states preflight's output as at most two turns: an optional "setting up" line, and the install request whenever step 3 earns one. No build could have caught this. Every corpus build runs on a Windows box with Edge and the extension present, so the missing-extension branch is unreachable from the harness -- 25 ticks of green on a path the corpus cannot walk. The followability case now probes it directly. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_017KoGDZbCevZMRYSJmDbwDG * Never settle for the headless browser without being told to A live run drove a browser for an entire preflight that the user could not see, said nothing about it either way, and carried on -- while a Chromium browser sat installed and extension-free at /usr/bin/google-chrome, one click from working. The user asked twice what was happening before anything surfaced. The premise that broke: attached is not theirs. The extension answering proves a browser exists somewhere reachable, not that anyone is sitting at it, and only the marker separates the two. CONTEXT.md said otherwise and is corrected. connect-browser-control now ends in exactly two states -- control confirmed, or no control after an explicit decline. There is no third ending and a run never reaches the gate's headless browser on its own judgement: - silence is earned by confirmed control or by a decline, nothing else - the browser search is ungated: it runs whenever control is not confirmed, which is the case an earlier gate excluded and the one that most needs it - a candidate you are blocked from testing is a candidate that failed; the run that reported "I can't yet trust it" and promised itself a retest was inventing a state this skill does not have - the ask is scaled to the road -- the click, the click and a restart, or a browser to install - preflight stops there until they answer, and only an explicit no is a decline; whatever they reply, the probe is read before their words - on a decline, one line, and it is about them: "I'll ask you to look at a page or two as we go" - asked directly, answer fully -- the silence rules govern what you volunteer, not what you answer notebook-to-webapp's preflight goes from two user-facing turns to three, and the notebook question waits behind the ask. Retired: the search runs only when nothing is attached; the silent rows are silent because nothing in them is theirs to do; preflight produces at most two user-facing turns. RULES 210/210 . RETIRED_CLAIMS 0 live . GENERALITY 0 leaks . CONFORMANCE 3/3 FOLLOWABILITY 14/14 unreachable, 10/10 install, 11/12 preflight COHERENCE found two contradictions in the rewrite, both closed: the table collapsed "not connected" and no-schema into one row when they take different asks, and no-schema skipped the search that its own last row depends on. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_017KoGDZbCevZMRYSJmDbwDG * Presence cannot be measured, and control can go stale silently A fellow agent's debug report on a preflight that took 25 calls and two wrong user-facing questions where 8 calls and 0 questions would have done. Its findings were audited against the record, most held, several did not, and probes run on the reporting host settled what neither of us could read. The standing constraint that shaped the rest: the only browser this framework drives is the local one. WHAT WAS MEASURED, not reasoned switch_browser EXCLUDES the browser you are already attached to. One browser connected returns "No other browsers available to switch to" in under six seconds; two browsers with the user sitting at the attached one timed out at ~124s three times running while that user watched their own Chrome and reported "Nothing appears..". Its own description claims it prompts "every Chrome browser with the extension installed". It does not. So nothing available proves a human is in front of a window, and the skill no longer implies otherwise. Confirmed control goes stale, and it fails in the SUCCESS direction. A user closed their browser; the device left the account list on its own, the session was silently re-attached to a browser on another machine, and navigate went on returning "Navigated to " on every call while rendering where nobody was. pgrep counted three live Chrome processes after that device had gone. Only serving a page and failing to read it back revealed any of it. get_page_text refuses a body under ten characters. Bisected on a live server, same tab, same Content-type: abcdefghij passes, abcdefghi fails, and padding the nine with newlines or spaces still fails -- the threshold is on trimmed text. A run lost its marker to a nine-character token and diagnosed the extractor. Display names are positional. The same deviceId was "Browser 2" one day and "Browser 1" the next; the list renumbers on membership change. THE MARKER, rewritten - loopback is the only candidate; the routable walk is deleted. A routable candidate has never returned a token in any recorded run, and the classifier's treatment of named routable addresses differs by host - --bind 127.0.0.1 written out, because http.server DEFAULTS to 0.0.0.0 and dropping the flag widens the bind rather than narrowing it - curl from this shell before driving, or a dead server and an unreachable browser are the same error -- one run wasted four calls navigating at a corpse after a port collision - the token is at least ten characters, and the port is genuinely random; three recorded builds all used 8791 - get_page_text is named and on the ToolSearch line, matching out of the page BODY: tab context echoes back the URL you requested, token and all, after a navigation that totally failed ENUMERATION, read but never shown list_connected_browsers is read to route and never rendered. Its description mandates an AskUserQuestion listing every browser with its deviceId, which is the topology retired at t187; the mandate lives in that description alone and select_browser carries none of it. Select by deviceId, never by name, and note that nothing in the list says which browser you are attached to -- a run that reported "attached to the wrong one" had inferred it by elimination from a list holding exactly one non-local device. RECOVERY, which is new Control is confirmed once and then used; there is no re-check between rounds. When a page comes back wrong, re-enumerate rather than re-probe -- tabs_context answers healthily for whatever you are attached to now. The repair is one action and it is theirs: reopen the browser. Measured, the device returns on its own with no reinstall and no sign-in. Then select by deviceId and marker again. Also: check the port is held by the process this run launched before pointing a browser at it. WHAT THE REPORT GOT WRONG Its headline recommendation, enumerate before any user-facing ask, is refuted by its own connectedAt values: the browser it wanted appeared 19m33s into the preflight, so the extension question was correct and enumeration then would have changed nothing. The real defect was five calls later, re-probing where it should have re-enumerated. "Compound commands are denied wholesale" is refuted by its own timeline -- four were allowed and two of five denials were plain single commands. Its 5.6 and 5.7 are rejected outright; 5.7 attacks a placement that does not exist and would resurrect a claim retired at t190. Retired: the routable candidate walk; --bind 0.0.0.0; the not-local branch that blessed driving a browser the extension reports as elsewhere; "a browser that loads what this host serves is on the user's side of the wire" (sshd listens here, so loopback proves nothing about location); "read the page" naming no tool. Coherence found four contradictions across three rounds and three were pre-existing: the marker was still said to separate presence twenty lines above the correction saying nothing can; "no sentence in preflight saying you will be asking them to look" forbade the mandated decline line; and the same retired behaviour was numbered third in one place and fourth in another. notebook-to-webapp-preflight had been stuck at 11/12 since t190 and the probe was right: on Windows the default browser IS the Edge the run just launched, one window not two, and the re-probe rule never said so. RULES 225/225 . RETIRED_CLAIMS 0 live . GENERALITY 0 leaks . CONFORMANCE 3/3 COHERENCE 0/3 runs . FOLLOWABILITY 17/17 unreachable, 10/10 install, 7/7 lost (new case), 12/12 preflight Known hole, accepted deliberately: nothing loads a page between preflight and the user's app, and the port check only sees this host, so a browser elsewhere resolving localhost to its own port still shows another app's pixels. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_017KoGDZbCevZMRYSJmDbwDG * Name the browser, then watch for the device A live preflight on the previous commit's text got further and still failed twice: it opened the extension store page in Firefox, and then waited to be told the extension had arrived rather than looking. Its user had to type "continue". The session was interviewed, and both failures turned out to be omissions rather than violations -- it did what the text said, and the text was wrong. THE STORE PAGE It had Chrome. It searched the family with command -v, found /usr/bin/google-chrome, launched it, probed -- and then opened the store page with xdg-open, which resolved to Firefox, where a chrome-extension:// origin cannot be installed at all. It then told its user "I've opened a page in your Chrome" on no evidence. Two sentences licensed that, and neither was wrong on its own: this skill's "open it the same way you open any page", and the orchestrator's open/drive table offering start/open/xdg-open with no caveat at the point of use. The sentence that should have stopped it -- the one whose worked example is this exact failure -- opens "With control," and was read as scoping the paragraph to opening the app later. So the store page now opens by naming the executable, in both shell forms, and the orchestrator's table gains the row that decides it: the default is right when any browser will do, wrong when one specific browser must receive the page. Measured, naming the executable hands the URL to an already-running instance rather than starting a second one -- 94ms return, a /favicon.ico line in the served page's log that curl never produces, and a process count rising by exactly one renderer. THE WATCH Between opening the store page and the user's "continue" the run made one call. It never read list_connected_browsers. That device's connectedAt proved it had connected before the user typed anything, so the entire wait was invented. Nothing had forbidden a loop. The wording simply described enumeration as a response to their reply -- "whatever they reply, re-enumerate" -- which pinned it to a turn boundary instead of to the clock. Now: the ask stops the notebook question and anything that moves the run past them, but not setup that has to happen regardless. The gate's headless browser install was already sequenced after the ask, so it runs during the click and nobody had taken that overlap. Then one look at the end of preflight, then every fifteen seconds until the device appears or they speak. Scoped, because the watch as first written was not: on the road where ToolSearch returns no schema there is no list to poll, which is exactly why that row of the ask asks for a restart. WHAT WAS FALSIFIED "Edge ships as the default on Windows, so start there opens the browser you already launched" -- retired. This Windows machine, with Edge installed, defaults to Firefox. Worse, that paragraph caused the failure above: the run on Linux read it as licence to expect xdg-open to resolve to the Chrome it had just launched. What replaces it carries the lookup commands and sits on the decline road, where a run standing at the ask never reads it. tabs_context_mcp answers for the MCP tab group, not for the browser. Measured byte-identical after a launch whose page had demonstrably been fetched -- so it is a false negative for confirming anything opened by other means, and it is what the next run would have reached for. Nothing confirms the launch now; the watch confirms the outcome. The native-messaging-host paths list registrations, not installations: a Windows box registering Chromium and Vivaldi had neither, only Edge; a Linux box carried manifests in five browser directories with one browser on PATH. The confirmation is one navigation again. The screenshot goes on the marker page -- computer returns "Successfully captured screenshot (1470x751, jpeg)" on the served text/plain page with the token legible -- so the separate ordinary-page step a live run skipped as redundant is deleted rather than restored. Preflight's three-turn budget had no home for acknowledging a click, so a run invented a fourth turn, "Setup's done." It was true; the marker had run and passed. The acknowledgement now rides on the next question -- "Thanks -- what's the path to your notebook?" -- and is not a turn of its own. COHERENCE, and the failure mode it exposed Six findings over four rounds, and four of them were one pattern: amending a rule leaves every sentence that SUMMARISED that rule asserting the old one. Worth naming, because this tick created three of them and every other check stayed green throughout. - the opening still said "preflight does not move past it" after the ask was scoped to hold only the notebook question - "Both launch forms above" silently changed meaning when the store-page commands became a second pair of launch forms above the same sentence - the orchestrator still called it "the one that stops this stage" beside the new rule saying it stops the question and not step 4 - the orchestrator named two things that reach the screen while its own table mandated three -- a reader would have delivered the Upload, Configure and Results rounds as interviews Two were older. The skill disclaimed presence in three places and then called the confirmed browser "the user's own window"; loopback reach is now stated as what it is -- the closest thing to their window anything here can establish, treated as such on purpose, never a claim that anyone is sitting at it. And the setup-summary rule read as licence for a summary that the turn budget does not allow; it now says it governs the wording of one licensed elsewhere. Two of the instruments were wrong as well, and both were found by using them. eval/coherence.py died on an arrow in the judge's prose, because Windows stdout is cp1252 -- and it died AFTER the judging was paid for, losing the finding rather than the run. eval/generality.py matched corpus artefacts case-sensitively, so a corpus stem survived being lowercased into a filename. EUBIC_Task3_Quant had been sitting in notebook-to-webapp as `eubic_task3_quant.py` behind a green check: an agent handed that notebook could read its own expected output filename out of the skill it was being measured against, which is the exact overfitting the check exists to prevent. The match is now case-insensitive, which flags that line and nothing else across all seven skills, and the example carries placeholders. RULES 236/236 . RETIRED_CLAIMS 0 live . GENERALITY 0 leaks . CONFORMANCE ok FOLLOWABILITY 13/13 install, 20/20 unreachable COHERENCE connect-browser-control 0/3 runs, clean SKILL_DEBT 24441 -> 26432: ten rules added, five claims retired. Left open deliberately: the watch has no clock ceiling, so a user who walks away after the ask leaves a run looking every fifteen seconds. And the executable launch is measured in one configuration -- same user, same profile, already running -- not in the class. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_017KoGDZbCevZMRYSJmDbwDG * Insight is the first solution, and a missing one has to say so A live session built a results page with no OpenMS-Insight components in it. Interviewed, it turned out to have done nothing wrong: it attempted the import, got ModuleNotFoundError, and fell back exactly as this skill told it to. The framework had no way for it to do better. WHAT WAS MISSING There was no install step anywhere. openms-insight is not in the template's requirements.txt -- correctly, per the design doc, which says it belongs to generated apps and not to the template -- but no skill in the chain ever put it in the app's either. It worked only on machines whose venv already carried it, which is every machine the corpus has ever run on. So build-insight-dashboard now establishes it before mapping anything: the line goes into the app's requirements.txt, the venv is synced, and the import is confirmed. Verified by importing, not assumed. INSIGHT FIRST MEANS A LOOKUP, NOT A JUDGEMENT ADR 0002 pinned half the output contract -- files, columns, identifiers -- and left `role` an undocumented free-form string, while the dashboard stage was told to "map each role to a component". Both ends were unpinned, so "no Insight component fits" was a judgement reachable in one step. The vocabulary is now fixed in capture-notebook-workflow (table, mirror, peakmap, chromatogram -- the four in use, each with an exact component), and build-insight-dashboard carries the table that maps them. A role in it gets that component; there is no choice to make. A step may coin a role outside it, and must say it has. The run that prompted this had roles `table` and `mirror`. Both had components. Nothing failed to fit. THE FALLBACK HAS TWO REASONS AND NAMES WHICH Retired: "Where no Insight component fits, fall back to pyopenms-viz and say so in the wireframe." It could not tell a design judgement from a missing package, and what it produced was worse than a miscategorisation. The wireframe that run showed its user read: "The mirror plot is drawn with pyopenms-viz, the same backend your notebook used." True, and it attributes the choice to continuity with their notebook. The environment fact appears nowhere in the transcript. Its own later self would have read that as a considered design decision. Its diagnosis of the cause is the part worth keeping: the register rules leave this finding no channel. notebook-to-webapp forbids naming framework plumbing; verify-webapp-usability makes the gate internal. "The headline component is not installed" is exactly what every one of those rules pushes out of a turn, and none of them says where to route it instead. So the reason now rides on the wireframe as a required clause -- one turn, not two, and the slot cannot be filled without naming which of the two reasons it was. THE GATE WAS NEGOTIATED, NOT PASSED NATIVE_PANEL_SELECTORS counted Plotly, Vega and Pyplot but not stDataFrame. That bites only the table role: a plot falling back renders Plotly and counted, a table has no plot to fall back to. The run read a real 9/10 on a page missing its table, then changed --expect-components from 2 to 1 and recorded a green 10/10. stDataFrame now counts, the panel count comes from the link graph, and lowering it to reach PASS is named as the one thing a run may never do. AND THE WIREFRAME MAKES SECTION 1 UNSKIPPABLE That run read lines 1-120 of a 195-line skill and produced no link graph at all -- "not one line reached the user or my own reasoning" -- going straight to the wireframe. Every fix above lives in section 1, so every panel in the wireframe now names the role it came from and the component it uses. You cannot fill that in without having done the mapping. APPS DECLARE WHAT THEY IMPORT The two most recent builds ship apps whose dashboards import openms_insight while their requirements.txt never mentions it; a clean install of either dies on import. Three earlier apps carry the line, so this stopped happening rather than never happened. Stated generally in scaffold-workflow-app, because Insight is the instance that got caught rather than the whole problem. THREE DOCUMENTS DISAGREED add-visualization claimed requirements.txt contains openms-insight -- false of this repo, and checking requirements.txt is the obvious verification, so that line would have actively misled a reader. The design doc pins openms-insight 0.2.0 in an environment table. CLAUDE.md was the only accurate statement and is now the complete one. COHERENCE build-insight-dashboard and capture-notebook-workflow both come back coherent. scaffold-workflow-app reported a different contradiction on each of three runs, and two were real and pre-existing: one section mandated editing the root test_gui.py page list while another said the template's tests are not yours to change (the seam is now named -- tests/ is a directory, test_gui.py is a root file), and an anecdote named three files then called them "the three files and the fourth". Both fixed. A document that yields a new finding per run has a backlog, and clearing it belongs in its own pass, not this one. RULES 244/244 . RETIRED_CLAIMS 0 live (32) . GENERALITY 0 leaks . CONFORMANCE ok FOLLOWABILITY 7/7 build-insight-dashboard-unavailable (new case), 22/23 main SKILL_DEBT 26432 -> 27917: eight rules added, two claims retired. Left open, deliberately. The main insight case sits at 22/23 missing "three axes named", consistently over two runs; there is no before-measurement, and the likeliest cause is that widening that case from 19 to 23 probes thinned the answer rather than the skill getting worse. And counting stDataFrame could count an incidental show_table() as a panel and so mask a missing one -- the 50x50 size filter is what stands between those, and it is unmeasured. The new followability case exists because nothing in the corpus could have caught this: the failure only occurs on a machine without the package, and the corpus runs on a machine with it. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_017KoGDZbCevZMRYSJmDbwDG * Five corrections the first live debriefs found The debrief harness ran for the first time and returned 40 items across two builds. Four of its confirmed findings indicted edits made the same morning, and each is a correction: a statement that was false, or a rule that made a legitimate build impossible. chromatogram no longer maps to LinePlot. Verified against the component's own docstring, which reads "Interactive stick plot component" and "Stick-style peak visualization (vertical lines from baseline)". The class name was the trap: a build read it, mapped an elution profile onto it, and drew 156 points as 156 vertical bars. Insight ships eight components and none draws a continuous trace, so chromatogram is the role that legitimately has none. It takes pyopenms-viz with the reason "no component fits". capture-notebook-workflow's vocabulary block still promised that a role in the list had a component waiting for it, and now does not. Hidden and missing are different findings. A two-tab layout puts a panel behind an inactive tab whose iframe reports 0x0, so the gate's total fell 5 to 4 -- against a rule forbidding exactly that number. The count is now scoped to one screen and each tab gated separately; what is still forbidden is lowering it for a panel that failed to render. The screenshot half of the marker check had no verdict. The recipe takes a screenshot on the marker page, but the table under it read on the token alone, so a session that passed the marker, drove pages, read their text and had every computer call fail landed on "it can reach you, say nothing". Reading a page and looking at one fail apart. A background command's description is user-facing text. The harness prints it back on completion, so a run can keep every silence rule in its prose and still put `Install headless browser for the gate` on screen. Name the command for what the user is waiting on. A failing background command is not narrated. One build printed `Background command "Restart the app" failed with exit code 1` on three separate screens and the reader flagged every one. Found by pointing register.py's vocabulary rules at real build screens for the first time -- they had only ever graded hand-written transcripts, so rules that existed for dozens of ticks had never been asked. RULES 246/246 . RETIRED 0 live (33) . CONFORMANCE ok . GENERALITY 0 leaks COHERENCE build-insight-dashboard and connect-browser-control both coherent Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_017KoGDZbCevZMRYSJmDbwDG * run_python spawns the interpreter running the app, and its tests say so Two builds on two different notebooks were put in an impossible position by the repository, not by any skill. scaffold-workflow-app sanctions exactly one edit inside src/workflow/ -- run_python() must spawn sys.executable rather than the literal "python", because a venv's interpreter is not on PATH under that name. Making that edit breaks two assertions in tests/test_command_executor_run_python.py, which pin the literal. And the same skill forbids editing the template's tests to match a change you made. So a build that follows the guidance correctly ends with a red suite it is not allowed to fix. Both builds found it independently and reported it in their debriefs; it is the first implementer-side pitfall to reach the recurrence threshold, at two builds across two notebooks. Both halves of the fix had been sitting uncommitted in the working tree since this morning, so every clone carried the unfixed executor together with the tests that assume it. Committing them is the whole repair -- the guidance was right and the clone was inconsistent with it. src/workflow/CommandExecutor.py "python" -> sys.executable, 2 sites tests/test_command_executor_run_python.py the two assertions that pin it 7 passed. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_017KoGDZbCevZMRYSJmDbwDG * An anecdote about a mistake is not a rule against it A build cloning the current skills opened with "Preflight's done. Let me read your notebook." This skill already contained that mistake, written up: a run that wrote "Setup's done." and explained honestly why -- two visible turns of trouble had just happened and silence read as the request having gone nowhere. The write-up stayed an anecdote. The rule beside it says what is never said is anything *about control*, and a stage of setup finishing is not about control, so nothing forbade the turn and the next run made it one word worse by reaching for the internal name. Setup finishing is not news -- the user did not know it had started. preflight, marker, the gate and browser name machinery they were never shown. Measured: this is the first build to clone t194 and leak anyway, and it leaked through the gap that rule did not cover rather than through the one it did. The same build's background command reads "Install dependencies" and the whole run is otherwise clean, which is t194 holding. RULES 247/247 Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_017KoGDZbCevZMRYSJmDbwDG * The opening line is a slot, and three builds each found a way to fill it Three builds on three different notebooks opened with a status line accounting for setup, each running with the rules the previous one had earned: t2i "Now preflight -- Python, dependencies, and browser setup, before I ask you anything." t1p "Preflight's done. Let me read your notebook." t3q "Control confirmed -- nothing to ask. Now the gate's browser and a look at the notebook." The second ran under a rule against narrating control. The third ran under that plus a rule against the word preflight, and said "Control confirmed" instead. Each rule closed a phrasing and the next run reached past it, which is what a vocabulary rule does to a pressure it does not name. The pressure is that setup took real time and finished, and saying nothing feels like saying nothing at all. The user did not know it had started, so its ending is not news. Naming the slot rather than the words: there is no turn before the first question. Measured against t194, which named a channel -- a background command's description is user-facing text -- and held without anyone touching its wording: the same build renamed its command from "Install headless browser for the gate" to "Install browser for page checks" unprompted. Structure held where vocabulary was evaded. The leak is confined to screen-000 in every case; nothing after it narrates. RULES 248/248 Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_017KoGDZbCevZMRYSJmDbwDG * A stage gets an outcome line, and the old count was false of its own example Two builds on two notebooks followed "one progress line per stage" exactly and reported the two longest silences in their runs. Capture covered the extracted script, the golden test and a real transcription bug on a single line; scaffold covered the workflow class, four pages, registration, the template cleanup, three test lists and the dashboard on another. One of them stated the gap directly: "nothing in it says what to do when a single stage runs long." The count was already wrong against this skill's own worked example. That example gives capture two lines -- it opens, and then "Verified it reproduces your notebook", a result the user can check -- while the sentence above it said one. Both runs followed the sentence and dropped the line the example shows, which is the line that would have broken the silence. The outcome line is not narration. It is the answer to the question the silence makes them ask. Where a stage genuinely produces nothing checkable until it ends, it keeps its one line and the next question closes the gap; a stage running long is not a reason to say more about the stage. This is the counterweight to today's other five edits, which all removed narration. It was found by asking the two agents that built the apps, and no output check could have produced it: both builds scored clean and the silence is invisible in an artifact. RULES 248/248 . RETIRED 0 live (34) Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_017KoGDZbCevZMRYSJmDbwDG * t198 forbade a line the framework is supposed to write Written two hours ago as "there is no turn before the first question. Not a short one, not a framing one." That contradicts notebook-to-webapp, which sanctions exactly one: "Optionally one line saying you are setting up -- 'Setting up -- a minute, then I'll ask about your notebook.'" The next build settled it. Cloning t198 and that sanctioned line together, it opened with the sanctioned one and produced the first clean opening measured: no preflight, no control, no gate, no browser, and register clean across every screen. The rule I wrote would have deleted the line that fixed the problem. So the rule is about direction, not existence. All three failures reported work already done -- "Now preflight -- Python, dependencies and browser setup", "Preflight's done", "Control confirmed -- nothing to ask". The line that works looks forward: it says what is happening and what comes next, and names no machinery. Nothing reports that setup finished. Measured openings, in order: "Now preflight -- Python, dependencies, and browser setup..." backward "Preflight's done. Let me read your notebook." backward "Control confirmed -- nothing to ask. Now the gate's browser." backward "Setting up -- a minute, then I'll ask about your notebook." forward, clean RULES 248/248 . RETIRED 0 live (34) Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_017KoGDZbCevZMRYSJmDbwDG * The outcome line needed a bound; without one it doubled the count t199 permitted a second progress line per stage -- the outcome line this skill's own example already showed -- to fix the two longest silences two builds had reported. The first build to run with it wrote eleven progress lines where the same notebook had written six the build before. 0904-2141-t1p t199 absent 6 progress lines 0905-0032-t1p t199 present 11 progress lines Same notebook, same persona, one variable. Among the eleven was "Now test_gui.py -- all three of its lists.", which is verbatim the line the bad column of this skill's own example is annotated for: a step that only exists because this framework has a rule about it is not a step the user is waiting on. The permission was read as licence, and it brought back the failure the whole rule exists to prevent. The bound is what makes it a permission. Five stage openings plus an outcome line for each stage producing a result before it ends -- in practice four to six for a whole build, against four in the worked example. A seventh means steps are being narrated. This is the second time today a rule of mine was evaded or over-read, and both times the fix was a countable structure rather than a description. t194 named a channel and held. t197 named a word and was walked around in one build. t199 described a permission and was stretched in one build; t200 gives it a number. RULES 248/248 . RETIRED 0 live (34) Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_017KoGDZbCevZMRYSJmDbwDG * Trouble spends a budgeted line rather than earning a new one Two builds on two notebooks reported the same inversion. One stated it exactly: "the stretch where I had least to report was the stretch where I was busiest, and the user saw nothing for longest precisely when something was wrong." The longest run in one of them was a held port, an unsupported widget_type, and a server that exited three times with no traceback. Every rule here correctly forbids narrating any of that, and "a fault with no user-visible symptom is a reason to say less" then makes the busiest stretch the quietest by construction. Saying you are still going is not describing the fault; it is the one fact the silence makes them guess at. The form matters more than the content here. Two edits today added a permission and both inflated what they permitted -- t199 doubled the progress-line count in one build, and t197's forbidden word was walked around in one. This one spends a line from the existing four-to-six budget instead of adding to it, so there is nothing new to stretch. Measured trade, three builds, same instruments: no permission, no bound 4 silence items, 6 progress lines permission, no bound 2 silence items, 11 progress lines permission, bounded 4 silence items, 5 progress lines The bound restored the count and the silence came back with it. This spends one of those five rather than reaching for a twelfth. RULES 249/249 . RETIRED 0 live (34) . CONFORMANCE ok Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_017KoGDZbCevZMRYSJmDbwDG * The same inversion lives in the design round, and t201 only fixed one skill t201 gave a stage in trouble a budgeted line to say it is still going, in notebook-to-webapp. The next build reported the identical inversion from build-insight-dashboard instead: "each iteration was edit -> clear the Insight cache -> restart -> render -> read the screenshot, and each one ended in a fault I am specifically told not to describe -- phantom-wrapped rows, truncated headers, a column-not-found on the sort field, a failed restart -- so the rule that keeps instrument language off the screen also guarantees that the stretch with the most going wrong is the one with nothing said." Four builds have now reported it, in three skills. This is the standing obligation in eval/README.md: apply a correction everywhere the claim lives, which six earlier ticks failed to do. Same form as t201 and for the same reason -- one line, spent from the four-to-six budget rather than added to it. Two edits tonight that added a permission had it stretched within a single build. RULES 250/250 . RETIRED 0 live (34) . CONFORMANCE ok Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_017KoGDZbCevZMRYSJmDbwDG * Setup is not a stage, said out loud this time Two consecutive builds on two notebooks, both carrying every rule in this repository, wrote a progress line for setup: Now the browser control probe. Control confirmed -- token and screenshot both came back. Now the gate's headless browser. Now step 3 of setup -- browser control. "Step 3 of setup" is the tell. It numbers something the user never knew had steps, which means the run had modelled setup as a stage with progress in it. The rule already forbade this. The stage table at the end of the file has five rows -- capture, interview, scaffold, dashboard, verify -- and the sentence above it says the stages are the five and nothing else is one. Setup is not there. Both runs read past the implicit form, which is what an implicit form does when the thing it excludes genuinely takes time and every other time-taking thing gets a line. So it is explicit, with the check attached: before writing a progress line, name which of the five it belongs to; if the answer is "setup", you have your answer. This is a violation rather than an omission -- the third distinct phrasing of the same turn tonight, after "Preflight's done" and "Control confirmed -- nothing to ask". The first two were answered by banning words and were walked around within one build each. This one names the category instead. RULES 251/251 . RETIRED 0 live (34) Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_017KoGDZbCevZMRYSJmDbwDG * Two concrete defects the debriefs promoted, both crashing or misreading preset_buttons is the one ui.* method the framework already calls ------------------------------------------------------------------ Two builds on two notebooks added self.ui.preset_buttons() to the end of configure(). One rendered the presets twice; the other crashed the Configure page on duplicate widget keys. The skill stated the fact -- they auto-appear via parameter_section() -- and never said not to call it, and every other ui.* method IS theirs to call, so the inference is the natural one. Named now, with where to look when presets do not appear: presets.json or the workflow name in it, never a missing call. Port ownership cannot be read from the PID ------------------------------------------ A venv's python re-execs as the shared base interpreter, so another build's Streamlit server has a byte-identical executable path. One build confirmed a listening PID and moved on. Another, after a failed restart, took what it thought was a free port and got a 200 back from a different build's app still running on this machine. Both were about to judge someone else's page and call it their own. The rule to verify the port already existed; what was missing is that the obvious way to verify it does not work. Match on the command line, which carries the app path and the --server.port you passed. Both promoted at two builds across two notebooks. These are the first promoted findings of the night that are engineering defects rather than what reaches the screen. RULES 253/253 . RETIRED 0 live (34) . CONFORMANCE ok Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_017KoGDZbCevZMRYSJmDbwDG * eval/rules.json was committed by accident and rode into nine clones A `git add eval/rules.json` in the t198 commit staged the harness's own rule file. Every build cloned since 67261c9 carries it -- nine of them, including both model probes -- so a driven session could see the rules it is being measured against, and their baselines, which describe the exact failures the loop is watching for. Nothing read or modified it: the clone's copy is byte-identical to HEAD and the Haiku session's file panel merely listed it. But the surface should not exist, and a probe run against a repo containing its own answer key is not a clean measurement. Untracked, and eval/ is now ignored so it cannot happen again. The nine builds in the window are recorded in eval/README.md rather than discarded -- their artifacts are still valid, and no evidence of contamination was found. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_017KoGDZbCevZMRYSJmDbwDG * The marker recipe walked into the trap the framework warns about twice capture-notebook-workflow: "`python` on PATH is often a Microsoft Store stub that prints help and exits." notebook-to-webapp names "the Windows Store-stub trap" twice. And this skill's own marker recipe read: serve python -m http.server --bind 127.0.0.1 --directory A sonnet probe followed that line literally, hit the stub, and lost the loopback check to it -- reported in its blind debrief as "used the bare `python -m http.server` command shown in the skill's own marker-check example". On a default Windows box the server never starts and the marker fails as though control were broken, which is the one failure this check exists to distinguish from. Now: the interpreter you already resolved -- sys.executable, the venv's python, or whatever capture's probe returned. Two things worth noting about how this was found. It is a contradiction between two skills, and nothing in the harness checks for that: rules_present verifies a sentence exists, coherence.py reads one document at a time, and neither can see one skill contradicting its neighbour. And it took a different model to surface it -- the default builds resolve an interpreter out of habit and never execute the line as written, so the defect sat in a recipe that had been read dozens of times. RULES 254/254 . RETIRED 0 live (34) . CONFORMANCE ok Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_017KoGDZbCevZMRYSJmDbwDG * A width taken from one layout was stated as a property of all of them The rule gave a fixed content width for a 1280px window and told runs to budget their column widths against it. Two builds on two notebooks did exactly that and wrapped onto the phantom second row anyway: t3q measured the real content width at ~830px t1p measured 864px, with the wrap at a 865px sum Each then re-derived the budget from a screenshot and repeated gate runs. One of them named the fault precisely: "the number in the rule is a measurement from one layout rather than a property to compute." So the figure is gone and the instruction is to measure. This is the opposite call from t200, which fixed a failure by giving a count -- and the difference is what the number describes. A build's progress lines are a property of the framework's own behaviour and can be bounded in advance. A rendered page's content width is a property of the page, and any constant is one app's measurement wearing a rule's clothes. RULES 254/254 . RETIRED 0 live (35) . CONFORMANCE ok Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_017KoGDZbCevZMRYSJmDbwDG * The trouble line was rationed by the budget it was meant to relieve t201 gave a stuck stage one line to say it was still going, taken from the four to six rather than added to them. I chose that deliberately: t199 had added a permission and the count doubled in a single build, so spending from the budget looked like the cautious form. A build spent it and reported what that cost: "spending it made the next silence longer, not shorter -- I said 'Still on the results page' once during the peak-map rebuild and then held the rest of that stage silent through seventeen shell commands because a second line would have broken the count." This is the first observed use of the line, and it produced the opposite of its purpose. Rationing relief with the budget it relieves is worse than either alternative. One per stage, uncounted. That is still a bound -- at most five across a build, and only for a stage actually stuck -- so it does not reopen what t199 did, which was a permission with no number attached at all. The same build also stacked three machinery lines above one question and named why: "I wrote them because each fix felt like news, which is exactly the failure the register rule names." That is the plateau, already recorded, and no rule here addresses it. RULES 254/254 . RETIRED 0 live (35) Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ini/code/session_017KoGDZbCevZMRYSJmDbwDG * The anti-example was the most copyable sentence on the page This skill demonstrated the wrong way to write progress lines by printing them in a left-hand column beside the right ones, eight against four. Three builds on two notebooks then produced the left column's test_gui.py line almost verbatim: skill, bad column Now test_gui.py -- all three of its lists. 0904-2141-t1p Now test_gui... 0905-0032-t1p Now test_gui... 0905-1207-t3q Now updating test_gui.py -- all three of its hand-written lists. The commentary under the block explained precisely why that line is wrong -- it reads a phrase out of cleanup.md, and a step that exists only because the framework has a rule about it is not a step the user is waiting on. Reading the explanation did not stop three runs from copying the sentence it explained. A well-formed sentence in a skill can be copied whichever column it sits in, and this one was left-aligned, first on the line, and grammatical. The bad version is now described in prose and never written out as a usable line. This is the counterpart to t207. There, a number that described the artifact had to go because it varied; here, a sentence that described bad output had to go because it could be lifted. Both were teaching by example and both examples were being used rather than read. RULES 255/255 . RETIRED 0 live (35) . CONFORMANCE ok Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_017KoGDZbCevZMRYSJmDbwDG * The skills were supplying the sentences the builds kept producing t209 removed one anti-example -- notebook-to-webapp's test_gui.py line, printed in a left-hand column as the wrong way to write a progress line. Three builds on two notebooks had reproduced it almost verbatim; the first build after removal did the same scaffold work, edited test_gui.py, and wrote no line about it. So the mechanism was checked against every forbidden term register.py knows, in every skill. Four more written-out anti-examples turned up across two files, and every one is a string some build has already emitted: connect-browser-control the browser-half heading, and a background command description naming the gate notebook-to-webapp the browser-probe heading, the control-confirmed sentence, the numbered setup step, and the preflight line Most of those I wrote overnight, as evidence in the tables documenting exactly these failures. Four generations of setup rules were aimed at a behaviour the file beside them was demonstrating in clean, copyable prose. All of them are described now instead of quoted. The one line that stays written out is the sanctioned one, because that is the sentence a run should be able to lift. This does not close the narration plateau. DEFAULTS/OUTPUTS, WorkflowManager, and verify-webapp-usability's self-quoted violation have no anti-example behind them and still stand. But the plateau's headline evidence -- the same setup sentence recurring under four rule generations -- has a cause that is not the rules failing to bind. RULES 256/256 . RETIRED 0 live (35) . CONFORMANCE ok Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_017KoGDZbCevZMRYSJmDbwDG * Re-navigation does not recover a wedged tab, and three builds proved it The rule said a timed-out screenshot is a page mid-rerun and to re-navigate. That is true and incomplete: it recovers a busy page, not a tab that has stopped answering, and the timeout message is identical for both. Three builds on three notebooks: t1p re-navigated the same tab three times over ~40 seconds, every screenshot still timing out, while the server answered curl in 1.5ms t2i wedged twice, both times just after the File Upload page re-copied a 37 MB fallback mzML t3q the tab stopped answering after a long stretch of driving the app A fresh tab fixed it instantly in every case, and the skill did not mention opening one. The expensive part is the misdiagnosis. One run read three dead Start Workflow clicks as the button being broken and went through the log, the pid directory, the process table and the multiprocessing spawn path before the tab turned out to be at fault. So the rule now says: if the server answers curl and the page does not answer you, suspect the tab before the app. One caveat carried with it -- opening a new tab moves the MCP tab group, so a tabId recorded before the switch is stale and a batch of calls lands on nothing. That was reported by the build that hit it mid-batch. RULES 257/257 . RETIRED 0 live (35) . CONFORMANCE ok Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_017KoGDZbCevZMRYSJmDbwDG * Measure where it is judged, and name the preset key builds keep deriving t212 -- the width rule said measure and not where ------------------------------------------------ t207 removed a fixed content-width figure because it was one layout's measurement. It told runs to measure and did not say in which browser. One build measured 704px of table in the 1568px window it was driving, sized five columns to 695px, and folded onto the phantom second row in the gate -- where that panel is about 400px. The gate is what judges the page and it runs at 1280px; the driving browser is whatever size the user's window happens to be. Another build spent four edit-restart-gate cycles, a minute each, bisecting to 810px from nothing. Observed fits in the gate are now 810, 830 and 864, so the rule names them as a starting point: one gate run to confirm rather than four to search. t213 -- the preset key three builds read the source to find ---------------------------------------------------------- A python tool's parameters live under _general keyed