feat(validate): catch broken references and show findings on the PR

AGENTS.md

@@ -135,7 +135,8 @@ precisely.
    run it locally for the org it names and fix the errors. Without that org's
    `.env.<org>`, run `VAPI_PRIVATE_API_KEY=validate-only npm run validate -- <org>`;
    never ask for a real key just to validate. Don't weaken the check or the
-   workflow to get past it.
+   workflow to get past it, and never edit the state file or `.vapi-ignore`
+   to make a reference resolve: fix the name, or pull.
 4. **Build PR checks offline** if `vapi-checks.yml` exists:
    `npm run check -- --all --dry-run`. Fix anything it reports.
 5. **Deploy only with a yes** (safety rule 1): `npm run apply -- <org>`, or

docs/guides/commands.md

@@ -10,7 +10,7 @@ The other commands are direct only.
 | Command | Usage | What it does |
 | --- | --- | --- |
 | `npm run setup` | `npm run setup [-- <org>]` | Connect an org: creates `.env.<org>` and `resources/<org>/`. |
-| `npm run validate` | `npm run validate -- <org>` | Check resource files offline. Run it before every `apply`. |
+| `npm run validate` | `npm run validate -- <org>` | Check resource files offline: API shape rules, and that every reference names a file or a state entry. `apply` runs it first. On GitHub Actions, findings are also shown on the files in the pull request. |
 | `npm run apply` | `npm run apply -- <org> [types or paths]` | **The default deploy:** pull, merge, then push. See [workflows](workflows.md). |
 | `npm run pull` | `npm run pull -- <org> [--force] [--bootstrap]` | Sync platform changes down; never overwrites local edits unless `--force`. |
 | `npm run push` | `npm run push -- <org> [--dry-run] [--strict]` | Push without pulling first. Prefer `apply`. `--strict` aborts before any API call if validation finds an error. |

docs/guides/troubleshooting.md

@@ -2,7 +2,8 @@
 
 ## "Reference not found" warnings
 
-The referenced resource doesn't exist. Check:
+The referenced resource doesn't exist. `npm run validate` reports these as
+`dangling-reference` errors before you deploy. Check:
 
 1. File exists in correct folder
 2. Filename matches exactly (case-sensitive)
@@ -30,7 +31,9 @@ bypassed).
 
 ## "Credential with ID not found" errors
 
-The credential UUID doesn't exist in the target org. Fix:
+The credential UUID doesn't exist in the target org. `npm run validate`
+warns about a credential name that isn't in the state file
+(`unresolved-credential`). Fix:
 
 1. Run `npm run pull -- <org>` to fetch credentials into the state file
 2. If the credential doesn't exist, create it in the Vapi dashboard with the same name
@@ -96,11 +99,25 @@ VAPI_PRIVATE_API_KEY=validate-only npm run validate -- <org>
 a placeholder is enough. You don't need that org's real key or its
 `.env.<org>`.
 
-Each error names the resource (`assistants/<id>`), field and rule. Plain `push` only warns about
-these errors, so a repository that has been deploying with `push` can carry
-some from before the check existed; they show up on the next pull request,
-whatever it changes. Fix them in that PR or a separate one first. `apply`
-refuses to deploy until they're fixed anyway.
+Each finding names the resource (`assistants/<id>`), the rule and, where it
+can, the field. On GitHub it's also shown on the file in the pull request.
+Plain `push` only warns about these errors (unless `--strict`), so a
+repository that has been deploying with `push` can carry some from before
+the check existed; they show up on the next pull request, whatever it
+changes. Fix them in that PR or a separate one first. `apply` refuses to
+deploy until they're fixed anyway.
+
+| Rule | Severity | What to do |
+| --- | --- | --- |
+| `dangling-reference` | error | A reference names no local file and no state entry. Fix the name (it's the file name without extension, including any folder), or run `npm run pull -- <org>` if the resource was created in the dashboard. Don't add a state entry by hand. |
+| `malformed-reference` | error | A reference list holds an empty entry or something that isn't a name, often a `- ` left while editing. Remove it or write the name. |
+| `override-tool-by-name` | error | References inside `assistantOverrides`, `membersOverrides` and `targetOverrides` aren't resolved. Put the tool inline under the override's `tools:append`. `model.tools` there replaces the member's whole tool set; see [squads](../learnings/squads.md). |
+| `reference-to-ignored` | error | The referenced resource matches `.vapi-ignore`, so this repo never deploys it. Remove the reference, or reference it by UUID if it must stay dashboard-owned. Don't edit `.vapi-ignore` to get past this without the resource owner's sign-off: un-ignoring gives the resource to gitops (see [YAML conventions](../learnings/yaml-conventions.md)). |
+| `name-length` | error | Shorten the name to 40 characters or fewer. |
+| `voice-provider-schema` | error | Move the setting to where that voice provider expects it; the message says where. |
+| `unresolved-credential` | warning | The credential name isn't in the state file. Run `npm run pull -- <org> --bootstrap` and commit the state file, or create the credential in the dashboard first. |
+| `reference-by-uuid` | warning | A UUID references a resource this repo tracks, which only works in one org and breaks promotion. Use the name the warning gives. UUIDs of resources the repo doesn't track (dashboard-owned or ignored ones) aren't reported. |
+| `so-assistant-lockstep`, `prompt-duplicate-*`, `max-tokens-floor` | warning | Follow the message; see [structured outputs](../learnings/structured-outputs.md) and [writing prompts](writing-prompts.md). |
 
 A `Failed to import TypeScript resource … is not set` error from this check
 means a `.ts` resource reads a variable from `.env.<org>`, which CI doesn't

improvements.md

@@ -82,13 +82,14 @@ you which stack PR closes the row.**
 | 28  | Handoff tools 400 on first push into an empty org          | Push aborts before the assistant-linking pass runs  | None       | RESOLVED 2026-08-01                             |
 | 29  | SO linking sent filtered `assistantIds` arrays              | Silent unlink of live-but-untracked assistants      | None       | RESOLVED 2026-08-03 (#51)                       |
 | 30  | Tool-linking pass could PATCH a raw assistant slug          | Mid-push 400 naming the wrong resource              | None       | RESOLVED 2026-08-03 (#51)                       |
-| 31  | Unresolved references handled 3 inconsistent ways, no dangling-ref check | Same authoring mistake, three different failure modes | None | Open                                            |
+| 31  | Unresolved references handled 3 inconsistent ways, no dangling-ref check | Same authoring mistake, three different failure modes | None | Mitigated 2026-10-03 (#77): validate/apply/CI block it; plain push only warns |
 | 32  | Test suite never ran in CI; 20 tests rotted after the hash store | Regression guards for #22/#23 silently stopped running | None | RESOLVED 2026-09-30 (#56)                      |
 | 33  | `npm run sim` reported every run as passed                | A failing suite exited 0 — false green             | None       | RESOLVED 2026-10-01                             |
 | 34  | No pre-merge simulation signal; simulations only tested what was deployed | A PR that breaks an agent merges green           | #33        | RESOLVED 2026-10-01                             |
 | 35  | A failed promotion pushed nothing, not even state       | git lost track of resources already on the platform | None       | RESOLVED 2026-10-01                             |
 | 36  | `cleanup` deletes resources excluded by `.vapi-ignore`   | A destructive cleanup can delete resources another team owns | None       | RESOLVED 2026-10-03                             |
 | 37  | Resource validation ran only at deploy time, after merge | A config `apply` refuses could merge and block deploys and promotion | #32        | RESOLVED 2026-10-03 (#76)                       |
+| 38  | Promotion can't carry a simulation that uses a stock personality by UUID | Gated orgs, and any promoted tests, need local personality files | None       | Open                                            |
 
 **Active backlog after cleanup:** `#2`, `#6`, `#8`, `#12`, `#20`, `#24–#26`, `#31`, and the open remainder of `#27` (wiring the listing-completeness verdict into push/delete/audit, and moving `cleanup.ts` onto the shared pager). Resolved entries stay in this file as historical incident notes per the maintenance directive; stale superseded backlog rows are not duplicated.
 
@@ -1568,6 +1569,8 @@ the repo and a subsequent push runs.
 
 ## 31. Unresolved references are handled three different ways depending on the field, and `validate.ts` has no dangling-reference check
 
+**[Mitigated 2026-10-03] (#77)**: validate, apply and CI block it; plain `push` only warns, and the three runtime behaviours remain.
+
 **Discovered:** while fixing #29 and #30 — those two entries close the
 loudest and quietest failure modes for their specific fields, but the
 underlying question ("what happens when a reference resolves to nothing")
@@ -1646,9 +1649,42 @@ conditions; it doesn't require picking one runtime behavior (filter vs.
 defer vs. 400) for every field, since it stops the push before any of those
 three behaviors gets a chance to run.
 
+### Possible fix (landed)
+
+`src/validate-refs.ts` uses `referencesCollect` (`src/resolver.ts`): the
+`extractReferencedIds` walk plus scenario judges'
+`evaluations[].structuredOutputId`, the same collector `reference-to-ignored`
+uses. It reports an error for any
+name that matches no local file and no state entry (`dangling-reference`).
+Names matched by `.vapi-ignore` are left to `reference-to-ignored`, which
+`npm run validate` now runs too. Alongside it: `override-tool-by-name`
+(an error: push never resolves `toolIds` inside overrides),
+`malformed-reference` (an error: an empty or non-name list entry, which used
+to crash `validate`), `unresolved-credential` and `reference-by-uuid`
+(warnings; only for a UUID this repo tracks, naming the file to use).
+`validate` skips `.vapi-ignore`d files, as push does. `validate`
+reads the committed state file, so the check runs offline and in CI, and
+`apply` stops on it before its pull. `push` runs the same checks with its
+other validators: warnings by default, blocking under `--strict`.
+
 ### Status
 
-**Open.**
+**Mitigated 2026-10-03 (#77).** `validate`, `apply` and the Validate
+resources check stop an unresolved reference before any of that runs. Still
+open:
+
+- Plain `push` only warns (unless `--strict`), then filters, defers or sends
+  raw names exactly as before.
+- A reference whose file was deleted, while its state entry remains, passes
+  validation. Under `push --force` the orphan pass deletes the target and its
+  state entry before the apply pass, and the reference then hits the
+  resolver's silent drop.
+- Names push never resolves still go unchecked: `hooks[].do[].toolId` and
+  `artifactPlan.structuredOutputIds` inside overrides, `model.toolRefs` at any
+  depth, and `toolIds` in a squad's inline `members[].assistant`. Only
+  `toolIds` inside overrides are checked.
+- Annotations carry a file but no line, so GitHub shows them at the top of
+  the file, and it keeps at most 10 per type per step.
 
 ---
 
@@ -1986,6 +2022,45 @@ against fixture orgs.
 
 ---
 
+## 38. Promotion can't carry a simulation that uses a stock personality by UUID
+
+**Discovered:** 2026-10-03, while testing the promotion check gate (#66).
+
+### Problem
+
+Every org has Vapi's stock simulation personalities, with fixed UUIDs
+(`a0000000-0000-4000-8000-00000000000<n>`). A simulation can reference one by
+that UUID, and `npm run check` accepts it. Promotion's dependency check
+doesn't: it treats the UUID as a managed dependency that must exist as a file
+in the source org.
+
+### Current behavior (Verified)
+
+Promoting a simulation with `personalityId: a0000000-…` fails with
+"Referenced managed dependency is missing from source:
+personalities/a0000000-…", and nothing is promoted.
+
+### Risk
+
+Teams that use stock personalities can't promote their tests, and can't gate
+an org on a check whose tests use them.
+
+### Current mitigation
+
+Use personality files under `simulations/personalities/` in any org you
+promote out of. The promotion guide says so.
+
+### Possible fix
+
+Treat stock personality UUIDs as present in every org in promotion's
+dependency check, as `check-payload.ts` already does.
+
+### Status
+
+**Open.**
+
+---
+
 ## Out of scope (intentionally not improvements)
 
 - **State file is identity-only and not git-ignored.** It's intentionally

src/push.ts

@@ -39,6 +39,7 @@ import {
   validateNoIgnoredReferences,
   validateResources,
 } from "./validate.ts";
+import { validateReferences } from "./validate-refs.ts";
 
 // Map a resource label to its state-file key. Used for snapshotting —
 // snapshot directories are keyed by the same names the state file uses.
@@ -1714,6 +1715,12 @@ async function main(): Promise<void> {
       // a config that references an ignored resource is a contradiction the
       // operator should see.
       ...validateNoIgnoredReferences(loadedResources, loadIgnorePatterns()),
+      ...validateReferences({
+        loaded: loadedResources,
+        org: VAPI_ENV,
+        state,
+        ignorePatterns: loadIgnorePatterns(),
+      }),
     ];
     if (findings.length > 0) {
       console.log(summarizeFindings(findings));

src/resolver.ts

@@ -1,4 +1,4 @@
-import type { ResourceState, StateFile } from "./types.ts";
+import type { ResourceState, ResourceType, StateFile } from "./types.ts";
 
 // ─────────────────────────────────────────────────────────────────────────────
 // ID Resolution - Convert resource IDs to Vapi UUIDs
@@ -358,8 +358,10 @@ export function extractReferencedIds(
   const scenarios: string[] = [];
   const simulations: string[] = [];
 
-  // Helper to clean IDs (remove comments)
-  const cleanId = (id: string) => id.split("##")[0]?.trim() ?? "";
+  // Helper to clean IDs (remove comments). A non-string entry (an empty
+  // `- ` list item, an object) comes back as "" instead of throwing.
+  const cleanId = (id: unknown) =>
+    typeof id === "string" ? (id.split("##")[0]?.trim() ?? "") : "";
 
   // Check root level toolIds
   if (Array.isArray(data.toolIds)) {
@@ -455,3 +457,54 @@ export function extractReferencedIds(
     simulations,
   };
 }
+
+// The reference fields push resolves by name, by the type they name, and the
+// resource types that can carry them.
+export const REFERENCE_TYPES: Array<{
+  refKey: keyof ExtractedReferences;
+  refType: ResourceType;
+}> = [
+  { refKey: "tools", refType: "tools" },
+  { refKey: "structuredOutputs", refType: "structuredOutputs" },
+  { refKey: "assistants", refType: "assistants" },
+  { refKey: "personalities", refType: "personalities" },
+  { refKey: "scenarios", refType: "scenarios" },
+  { refKey: "simulations", refType: "simulations" },
+];
+
+export const RESOURCE_TYPES_WITH_REFS: ResourceType[] = [
+  "tools",
+  "structuredOutputs",
+  "assistants",
+  "squads",
+  "personalities",
+  "scenarios",
+  "simulations",
+  "simulationSuites",
+  "evals",
+];
+
+// Every reference push resolves, by type: `extractReferencedIds` plus scenario
+// judges' `evaluations[].structuredOutputId`, which resolveReferences handles
+// separately. Names are cleaned of `##` comments; "" marks an entry that is
+// empty or isn't a name. One collector for every reference rule, so a field
+// can't be checked by one rule and missed by another.
+export function referencesCollect(
+  data: Record<string, unknown>,
+): Map<ResourceType, string[]> {
+  const extracted = extractReferencedIds(data);
+  const refs = new Map<ResourceType, string[]>();
+  for (const { refKey, refType } of REFERENCE_TYPES)
+    refs.set(refType, [...extracted[refKey]]);
+  if (Array.isArray(data.evaluations))
+    for (const evaluation of data.evaluations) {
+      if (!evaluation || typeof evaluation !== "object") continue;
+      if (!("structuredOutputId" in evaluation)) continue;
+      const id = (evaluation as { structuredOutputId?: unknown })
+        .structuredOutputId;
+      refs
+        .get("structuredOutputs")!
+        .push(typeof id === "string" ? (id.split("##")[0]?.trim() ?? "") : "");
+    }
+  return refs;
+}

src/state.ts

@@ -48,7 +48,7 @@ function migrateSection(
 // State Management
 // ─────────────────────────────────────────────────────────────────────────────
 
-function createEmptyState(): StateFile {
+export function createEmptyState(): StateFile {
   return {
     credentials: {},
     assistants: {},