From de6ac5aba1bcf224827cd8579480ba4174c92dc1 Mon Sep 17 00:00:00 2001 From: Aaron Trowbridge Date: Thu, 29 Oct 2026 08:36:39 -0400 Subject: [PATCH 1/2] =?UTF-8?q?test(notturno):=20RED=20=E2=80=94=20gates'?= =?UTF-8?q?=20vitest=20steps=20run=20under=20CI's=20env=20(CI=3Dtrue,=20GI?= =?UTF-8?q?THUB=5FACTIONS=3Dtrue,=20scratch=20empty=20HOME,=20ambient=20se?= =?UTF-8?q?crets=20scrubbed)=20and=20a=20red=20suite=20names=20its=20faili?= =?UTF-8?q?ng=20files=20(first=205,=20+N=20more)=20in=20the=20RESULT=20lin?= =?UTF-8?q?e=20(#2007)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../amico-run/test/notturno_gates.test.ts | 79 ++++++++++++++++++- 1 file changed, 78 insertions(+), 1 deletion(-) diff --git a/packages/amico-run/test/notturno_gates.test.ts b/packages/amico-run/test/notturno_gates.test.ts index 8afd27a77..044d05796 100644 --- a/packages/amico-run/test/notturno_gates.test.ts +++ b/packages/amico-run/test/notturno_gates.test.ts @@ -15,7 +15,7 @@ // origin/lane/* (or origin/main), and maps the diff to vitest packages. import { describe, test, expect, beforeEach, afterEach } from "vitest"; import { spawnSync } from "node:child_process"; -import { copyFileSync, existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, statSync, writeFileSync, appendFileSync } from "node:fs"; +import { copyFileSync, existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, statSync, symlinkSync, writeFileSync, appendFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { dirname, join } from "node:path"; @@ -174,6 +174,83 @@ describe(".notturno/gates — the night runner's pre-push gate (#1980)", () => { }); }); +// #2007: in the Modal sandbox the extension/amico-run vitest steps went red +// whatever the diff, and the summary hid which tests. The vitest steps must run +// the way CI's `fast` job does (CI=true, GITHUB_ACTIONS=true, an empty runner +// HOME, no ambient secrets/dev-shell env), and a red suite must name its +// failing files in the RESULT line (the runner surfaces only the last lines). +describe(".notturno/gates — vitest steps mirror CI and name failing files (#2007)", () => { + let fx: string; + const VITEST_PKG = join(REPO, "packages", "amico-run", "node_modules", "vitest"); + + /** give a fixture package a real vitest (symlinked from this checkout) */ + function linkVitest(pkg: string) { + const nm = join(fx, "packages", pkg, "node_modules"); + mkdirSync(join(nm, ".bin"), { recursive: true }); + symlinkSync(VITEST_PKG, join(nm, "vitest")); + symlinkSync(join(VITEST_PKG, "vitest.mjs"), join(nm, ".bin", "vitest")); + } + + beforeEach(() => { + fx = makeFixture(); + linkVitest("foo"); + linkVitest("bar"); + }); + afterEach(() => { + rmSync(fx, { recursive: true, force: true }); + rmSync(upstreamOf(fx), { recursive: true, force: true }); + }); + + test("the vitest step runs under CI's env: CI=true, GITHUB_ACTIONS=true, a scratch empty HOME, ambient secrets scrubbed", () => { + const outerHome = process.env.HOME ?? ""; + put(fx, "packages/foo/test/env.test.ts", [ + `import { test, expect } from "vitest";`, + `import { existsSync, readdirSync } from "node:fs";`, + `import { join } from "node:path";`, + `test("ci env", () => {`, + ` expect(process.env.CI).toBe("true");`, + ` expect(process.env.GITHUB_ACTIONS).toBe("true");`, + ` const home = process.env.HOME ?? "";`, + ` expect(home).not.toBe(${JSON.stringify(outerHome)});`, + ` expect(home).not.toBe("");`, + ` expect(existsSync(join(home, ".amico"))).toBe(false);`, + ` expect(existsSync(join(home, ".config", "opencode"))).toBe(false);`, + ` expect(readdirSync(home)).toEqual([]);`, + ` expect(process.env.AWS_SECRET_ACCESS_KEY).toBeUndefined();`, + ` expect(process.env.GH_TOKEN).toBeUndefined();`, + ` expect(process.env.AMICO_LEAK).toBeUndefined();`, + ` expect(process.env.NOTTURNO_ISSUE).toBeUndefined();`, + `});`, + ``, + ].join("\n")); + const r = run("./.notturno/gates", [], fx, { + NOTTURNO_GATES_STEPS: "vitest", + AWS_SECRET_ACCESS_KEY: "leak", + GH_TOKEN: "ghp_leak", + AMICO_LEAK: "1", + NOTTURNO_ISSUE: "2007", + }); + expect(r.status, r.out).toBe(0); + expect(r.out).toMatch(/1 passed/); + expect(lastLine(r.stdout)).toMatch(/GREEN/); + }); + + test("a red suite names its failing test files (first 5) in the RESULT line", () => { + for (const n of ["a", "b", "c", "d", "e", "f", "g"]) { + put(fx, `packages/foo/test/${n}.test.ts`, `import { test, expect } from "vitest";\ntest("${n}", () => { expect(1).toBe(2); });\n`); + } + put(fx, "packages/foo/test/ok.test.ts", `import { test, expect } from "vitest";\ntest("ok", () => { expect(1).toBe(1); });\n`); + put(fx, "packages/bar/test/bad.test.ts", `import { test, expect } from "vitest";\ntest("bad", () => { expect(1).toBe(2); });\n`); + const r = run("./.notturno/gates", [], fx, { NOTTURNO_GATES_STEPS: "vitest" }); + expect(r.status, r.out).not.toBe(0); + const summary = lastLine(r.stdout); + expect(summary).toMatch(/RED/); + expect(summary).toContain("vitest:foo [test/a.test.ts, test/b.test.ts, test/c.test.ts, test/d.test.ts, test/e.test.ts, +2 more]"); + expect(summary).toContain("vitest:bar [test/bad.test.ts]"); + expect(summary).not.toContain("ok.test.ts"); + }); +}); + describe("this repo's committed tree", () => { test("AGENTS.md's verification-gates section teaches the overlay rule: refresh_manifest, then drift_gate, before commit", () => { const agents = readFileSync(join(REPO, "AGENTS.md"), "utf8"); From f444f5691ac5f62e90ae6897f85bd46ad555617a Mon Sep 17 00:00:00 2001 From: Aaron Trowbridge Date: Thu, 29 Oct 2026 08:36:39 -0400 Subject: [PATCH 2/2] =?UTF-8?q?fix(notturno):=20gates=20run=20vitest=20sui?= =?UTF-8?q?tes=20the=20way=20CI's=20fast=20job=20does=20=E2=80=94=20env=20?= =?UTF-8?q?-i=20+=20CI=3Dtrue=20GITHUB=5FACTIONS=3Dtrue=20TZ=3DUTC=20RUNNE?= =?UTF-8?q?R=5FTEMP=20GITHUB=5FBASE=5FREF,=20fresh=20empty=20scratch=20HOM?= =?UTF-8?q?E=20per=20suite,=20package-local=20vitest=20bin;=20JSON=20repor?= =?UTF-8?q?ter=20on=20every=20suite=20=E2=86=92=20RESULT=20names=20failing?= =?UTF-8?q?=20files=20(quarantine-gate=20violations=20for=20amico-run);=20?= =?UTF-8?q?build=20is=20its=20own=20step;=20warn=20on=20node=E2=89=A020=20?= =?UTF-8?q?/=20root=20(#2007)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .notturno/gates | 139 +++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 120 insertions(+), 19 deletions(-) diff --git a/.notturno/gates b/.notturno/gates index fdfedf49c..ae15ead5b 100755 --- a/.notturno/gates +++ b/.notturno/gates @@ -18,8 +18,20 @@ # loopback suites below (fleet-dependent; they self-skip without # an sshd and mutate ~/.ssh/authorized_keys when one exists). # Vault-dependent tests self-skip on their own existsSync/env -# guards, exactly as in CI. A `pnpm -r run build` precedes the -# suites, as in CI (several suites read built bundles). +# guards, exactly as in CI. A `pnpm -r run build` (step `build`) +# precedes the suites, as in CI (several suites read built bundles). +# The suites run in CI's environment (#2007), not the sandbox's: +# a scrubbed env (`env -i` + PATH/TMPDIR/locale/USER) with CI=true, +# GITHUB_ACTIONS=true, RUNNER_TEMP, GITHUB_BASE_REF= (a +# pull_request run), TZ=UTC, and a fresh EMPTY scratch HOME — the +# GitHub runner's HOME has no ~/.amico, no ~/.config/opencode, no +# fleet; the sandbox's HOME has the implementer's opencode config +# and the env carries AWS/GH tokens + NOTTURNO_*, which tests read. +# A red suite names its failing test files (first 5) in RESULT: +# failed: vitest:extension [test/a.test.ts, test/b.test.ts] +# Divergences the gate cannot erase are printed as `warn:` lines: +# node major != CI's 20, running as root (CI's runner is not — +# chmod-000 tests cannot fail for root). # NOT here (need the vendored engine binary / bun materialization / Julia / # PyPI): build-binary, engine-tests, vsix-gate, boot-smoke, schema-roundtrip. # @@ -30,7 +42,9 @@ # Unset/unresolvable → the nearest of origin/lane/* and # origin/main (fewest commits from merge-base to HEAD). # No base at all → every package's suite runs. -# NOTTURNO_GATES_STEPS comma list subset of drift,typecheck,vitest (default all) +# NOTTURNO_GATES_STEPS comma list subset of drift,typecheck,build,vitest (default all; +# build only runs when vitest has packages to test) +# NOTTURNO_GATES_HERMETIC=0 run the suites in the ambient env/HOME (debugging only) set -uo pipefail ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" @@ -45,16 +59,16 @@ PLAN=0 for a in "$@"; do case "$a" in --plan) PLAN=1 ;; - -h|--help) sed -n '2,36p' "$0"; exit 0 ;; + -h|--help) sed -n '2,52p' "$0"; exit 0 ;; *) echo "[notturno-gates] RESULT: RED — unknown argument: $a"; exit 2 ;; esac done -STEPS="${NOTTURNO_GATES_STEPS:-drift,typecheck,vitest}" +STEPS="${NOTTURNO_GATES_STEPS:-drift,typecheck,build,vitest}" for s in ${STEPS//,/ }; do case "$s" in - drift|typecheck|vitest) ;; - *) echo "[notturno-gates] RESULT: RED — unknown step '$s' in NOTTURNO_GATES_STEPS (known: drift,typecheck,vitest)"; exit 2 ;; + drift|typecheck|build|vitest) ;; + *) echo "[notturno-gates] RESULT: RED — unknown step '$s' in NOTTURNO_GATES_STEPS (known: drift,typecheck,build,vitest)"; exit 2 ;; esac done want() { case ",$STEPS," in *",$1,"*) return 0 ;; *) return 1 ;; esac; } @@ -146,9 +160,11 @@ PASSED="" FAILED="" HINTS="" T0=$SECONDS +STEP_DETAIL="" # a step may set this to a short " [..]" suffix for the RESULT line step() { # step local name="$1" hint="$2" t=$SECONDS rc shift 2 + STEP_DETAIL="" say "── $name: $*" "$@" rc=$? @@ -156,28 +172,113 @@ step() { # step PASSED="$PASSED $name" say "── $name: ok ($((SECONDS - t))s)" else - FAILED="$FAILED $name" + FAILED="$FAILED $name$STEP_DETAIL" [ -n "$hint" ] && HINTS="$HINTS $hint;" - say "── $name: FAIL exit $rc ($((SECONDS - t))s)" + say "── $name: FAIL exit $rc ($((SECONDS - t))s)${STEP_DETAIL}" + fi +} + +# ── CI's environment for the vitest suites (#2007) ─────────────────────────── +SCRATCH="$(mktemp -d "${TMPDIR:-/tmp}/notturno-gates.XXXXXX")" || { echo "[notturno-gates] RESULT: RED — mktemp failed"; exit 2; } +trap 'chmod -R u+rwx "$SCRATCH" 2>/dev/null; rm -rf "$SCRATCH"' EXIT +CI_TMP="$SCRATCH/tmp" +mkdir -p "$CI_TMP" "$SCRATCH/runner-temp" + +# run a command the way CI's `fast` job runs a test step +ci_env() { + if [ "${NOTTURNO_GATES_HERMETIC:-1}" = "0" ]; then "$@"; return; fi + local home + home="$(mktemp -d "$SCRATCH/home.XXXXXX")" || return 2 # fresh + empty per suite + local keep=() v + for v in PATH LANG LC_ALL USER LOGNAME SHELL; do + [ -n "${!v:-}" ] && keep+=("$v=${!v}") + done + env -i "${keep[@]}" \ + HOME="$home" TMPDIR="$CI_TMP" RUNNER_TEMP="$SCRATCH/runner-temp" TZ=UTC \ + CI=true GITHUB_ACTIONS=true ${BASE:+GITHUB_BASE_REF="${BASE#origin/}"} \ + "$@" +} + +vitest_bin() { # the package's own vitest bin (no pnpm/corepack under the scratch HOME) + if [ -x "packages/$1/node_modules/.bin/vitest" ]; then echo "$ROOT/packages/$1/node_modules/.bin/vitest"; else echo ""; fi +} + +# failed_files [quarantine-gate-log] → "test/a.test.ts, test/b.test.ts, +N more" (first 5) +failed_files() { + node - "$@" <<'NODE' 2>/dev/null +const fs = require("fs"), path = require("path"); +const [json, gateLog] = process.argv.slice(2); +const files = []; +const add = (f) => { if (f && !files.includes(f)) files.push(f); }; +if (gateLog && fs.existsSync(gateLog)) { + // quarantine-gate violations: "... FAIL ::::: ..." + for (const m of fs.readFileSync(gateLog, "utf8").matchAll(/ FAIL [\w-]+ [^:\s]+::([^:]+)::/g)) add(m[1]); +} +if (!files.length && json && fs.existsSync(json)) { + let r; try { r = JSON.parse(fs.readFileSync(json, "utf8")); } catch { r = null; } + for (const t of (r && r.testResults) || []) { + const bad = t.status === "failed" || (t.assertionResults || []).some((a) => a.status === "failed"); + if (bad) add(path.relative(process.env.PKG_DIR || process.cwd(), t.name)); + } +} +if (files.length) { + const head = files.slice(0, 5); + if (files.length > 5) head.push(`+${files.length - 5} more`); + process.stdout.write(head.join(", ")); +} +NODE +} + +set_detail() { # set_detail [gate-log] + local f + f="$(PKG_DIR="$ROOT/packages/$1" failed_files "$2" "${3:-}")" + [ -n "$f" ] && STEP_DETAIL=" [$f]" + return 0 +} + +vitest_args() { # common CI flags + exclusions + a JSON report + local g + VARGS=(run --passWithNoTests --exclude '**/slow/**') + for g in $ENV_DEPENDENT_EXCLUDES; do VARGS+=(--exclude "$g"); done + VARGS+=(--reporter=default --reporter=json "--outputFile.json=$1") +} + +run_vitest() { # run_vitest + local bin + bin="$(vitest_bin "$1")" + vitest_args "$2" + if [ -n "$bin" ]; then + (cd "packages/$1" && ci_env "$bin" "${VARGS[@]}") + else + (cd "packages/$1" && ci_env pnpm exec vitest "${VARGS[@]}") fi } run_amico_run_suite() { - local out vexit excl=() g - out="${TMPDIR:-/tmp}/notturno-amico-run-$$.json" - for g in $ENV_DEPENDENT_EXCLUDES; do excl+=(--exclude "$g"); done - (cd packages/amico-run && pnpm exec vitest run --passWithNoTests --exclude '**/slow/**' "${excl[@]}" \ - --reporter=default --reporter=json --outputFile.json="$out") + local out="$SCRATCH/vitest-amico-run.json" glog="$SCRATCH/quarantine-amico-run.log" vexit qexit + run_vitest amico-run "$out" vexit=$? - (cd packages/amico-run && node ../../scripts/test_quarantine_gate.mjs --runner-exit "$vexit" "$out") + (cd packages/amico-run && node ../../scripts/test_quarantine_gate.mjs --runner-exit "$vexit" "$out") 2>&1 | tee "$glog" + qexit=${PIPESTATUS[0]} + [ "$qexit" -ne 0 ] && set_detail amico-run "$out" "$glog" + return "$qexit" } run_pkg_suite() { - local p="$1" excl=() g - for g in $ENV_DEPENDENT_EXCLUDES; do excl+=(--exclude "$g"); done - (cd "packages/$p" && pnpm exec vitest run --passWithNoTests --exclude '**/slow/**' "${excl[@]}") + local p="$1" out="$SCRATCH/vitest-$1.json" rc + run_vitest "$p" "$out" + rc=$? + [ "$rc" -ne 0 ] && set_detail "$p" "$out" + return "$rc" } +if want vitest && [ -n "$SELECTED" ]; then + NODE_MAJOR="$(node -p 'process.versions.node.split(".")[0]' 2>/dev/null || echo '?')" + say "vitest env: $([ "${NOTTURNO_GATES_HERMETIC:-1}" = "0" ] && echo 'AMBIENT (NOTTURNO_GATES_HERMETIC=0)' || echo "CI mirror — env -i, CI=true GITHUB_ACTIONS=true TZ=UTC, scratch HOME under $SCRATCH${BASE:+, GITHUB_BASE_REF=${BASE#origin/}}"); node $(node --version 2>/dev/null || echo '?')" + [ "$NODE_MAJOR" != "20" ] && say "warn: node major $NODE_MAJOR != CI's 20 (actions/setup-node node-version: 20) — a red that CI would not show may be a node-version difference" + [ "$(id -u 2>/dev/null)" = "0" ] && say "warn: running as root — CI's runner is unprivileged; permission tests (chmod 000/0500 dirs) cannot fail for root" +fi + if want drift; then step drift "overlay edit without a manifest refresh? run 'node packages/app-bundle/scripts/refresh_manifest.mjs' then drift_gate, and commit manifest.json" \ node packages/app-bundle/scripts/drift_gate.mjs @@ -186,7 +287,7 @@ if want typecheck; then step typecheck "" pnpm -r run typecheck fi if want vitest && [ -n "$SELECTED" ]; then - step build "" pnpm -r run build + want build && step build "" pnpm -r run build for p in $SELECTED; do if [ "$p" = "amico-run" ]; then step "vitest:$p" "" run_amico_run_suite