From d4162d7555e8e14650aaa9395efa86955d991605 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 19 Oct 2026 05:28:32 +0000 Subject: [PATCH] feat(bench): add questions that name no Effort, and filler that looks real Two new ways to test `proof find` the way agents will use it: - `questions/ambiguous.json` holds 24 questions rewritten from tier 0-2 questions. None of them names an Effort, an Effort title, or a platform that points at one Effort. Each lists `rivals`, near-miss records in other Efforts. `check.mjs` proves each answer is unique across the whole graph and rejects a platform word that one Effort dominates. The new category is `ambiguous-effort`. - `build.mjs --scale N --filler-overlap ` makes that share of filler records reuse real subsystem names while linking to no real record and stating no answer value. Default builds are unchanged, byte for byte, at 1x and at 10x. Every script still asks the original 246 questions unless given `--only ambiguous`, so earlier runs stay comparable. The ambiguous set was frozen before `find` ran on it. On the 10x graph, search puts the evidence in the top five for 23 of 24 and the answer on a card for 87%. With 30% look-alike filler, the gate on all 246 questions still passes: evidence 80%, answer on a card 62%. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01DVVnhYL1Bz7mCfEKDoyLMW --- benchmarks/proof-recall/README.md | 117 +- .../proof-recall/questions/ambiguous.json | 1073 +++++++++++++++++ benchmarks/proof-recall/scripts/build.mjs | 158 ++- benchmarks/proof-recall/scripts/check.mjs | 174 ++- benchmarks/proof-recall/scripts/compare.mjs | 11 +- .../proof-recall/scripts/find-recall.mjs | 35 +- benchmarks/proof-recall/scripts/grade.mjs | 15 +- benchmarks/proof-recall/scripts/graph.mjs | 20 +- .../proof-recall/scripts/prepare-run.mjs | 12 +- benchmarks/proof-recall/scripts/run.mjs | 5 +- benchmarks/proof-recall/spec/_filler.mjs | 294 +++-- 11 files changed, 1779 insertions(+), 135 deletions(-) create mode 100644 benchmarks/proof-recall/questions/ambiguous.json diff --git a/benchmarks/proof-recall/README.md b/benchmarks/proof-recall/README.md index f784bea2..b693cd58 100644 --- a/benchmarks/proof-recall/README.md +++ b/benchmarks/proof-recall/README.md @@ -11,22 +11,23 @@ Each question has a known answer and a stated path through the graph. ## Layout -| Path | What it holds | -| ------------------------- | -------------------------------------------------------- | -| `WORLD.md` | The shared facts every Effort must agree on | -| `spec/.mjs` | One Effort's history, as an ordered list of mutations | -| `questions/.json` | That Effort's questions and answers | -| `questions/cross.json` | Questions that need two or more Efforts | -| `scripts/build.mjs` | Replays every spec through the Proof writer | -| `scripts/check.mjs` | Proves each answer against the built graph | -| `scripts/grade.mjs` | Scores one run | -| `scripts/run.mjs` | Runs one headless agent session per question | -| `scripts/task.mjs` | The task text each session reads | -| `scripts/compare.mjs` | Compares two runs question by question | -| `scripts/find-recall.mjs` | Checks offline whether `proof find` reaches the evidence | -| `runs/` | Answers and costs from each pilot and full run | -| `.flatbread-proof/` | The built graph, committed so runs need no build | -| `flatbread.config.js` | Lets `flatbread proof …` read the graph from this folder | +| Path | What it holds | +| -------------------------- | -------------------------------------------------------- | +| `WORLD.md` | The shared facts every Effort must agree on | +| `spec/.mjs` | One Effort's history, as an ordered list of mutations | +| `questions/.json` | That Effort's questions and answers | +| `questions/cross.json` | Questions that need two or more Efforts | +| `questions/ambiguous.json` | Questions that name no Effort, asked only on request | +| `scripts/build.mjs` | Replays every spec through the Proof writer | +| `scripts/check.mjs` | Proves each answer against the built graph | +| `scripts/grade.mjs` | Scores one run | +| `scripts/run.mjs` | Runs one headless agent session per question | +| `scripts/task.mjs` | The task text each session reads | +| `scripts/compare.mjs` | Compares two runs question by question | +| `scripts/find-recall.mjs` | Checks offline whether `proof find` reaches the evidence | +| `runs/` | Answers and costs from each pilot and full run | +| `.flatbread-proof/` | The built graph, committed so runs need no build | +| `flatbread.config.js` | Lets `flatbread proof …` read the graph from this folder | ## Build and check @@ -160,6 +161,8 @@ Each question file holds one Effort's questions: | `checks` | Machine checks that compute or confirm the answer | | `traps` | The wrong answers a careless reader gives, and why | | `scoped` | `true` when wording other than the title pins the Effort down | +| `from` | In `ambiguous.json` only: the question it was rewritten from | +| `rivals` | In `ambiguous.json` only: near-miss records in other Efforts | ### Answer kinds @@ -210,17 +213,21 @@ A check without `expect` must compute the answer itself. A `select` scans the qu `expect` must return exactly that value. `record`, `records`, and `count` answers need at least one computing check. -| `type` | Fields | Returns | -| ------------ | -------------------------------------------------------------------------------------------------- | -------------------------------------------------- | -| `field` | `record`, `field` (or `standing`) | that frontmatter value | -| `head` | `start` | the newest version's key | -| `chain` | `start` | every version's key, oldest first | -| `asof` | `start`, `at` | the version current at `at` | -| `select` | `kinds`, `where`, `live`, `retracted`, `derivesFrom`, `restsOn`, `bodyIncludes`, `count`, `effort` | matching keys, or their count | -| `efforts` | `where`, `count` | matching Efforts' keys, or their count | -| `compaction` | `field`: `head`, `heads`, `stale`, `changes`, `changed`, `staleTargets` | the Effort's compaction state | -| `body` | `record`, `includes`, `afterExcerpt` | whether the text is present (and past the excerpt) | -| `edge` | `edge` | whether the edge exists | +| `type` | Fields | Returns | +| ------------ | ---------------------------------------------------------------------------------------------------------------------------------- | -------------------------------------------------- | +| `field` | `record`, `field` (or `standing`) | that frontmatter value | +| `head` | `start` | the newest version's key | +| `chain` | `start` | every version's key, oldest first | +| `asof` | `start`, `at` | the version current at `at` | +| `select` | `kinds`, `where`, `live`, `retracted`, `derivesFrom`, `restsOn`, `invalidates`, `bodyIncludes`, `titleIncludes`, `count`, `effort` | matching keys, or their count | +| `efforts` | `where`, `count` | matching Efforts' keys, or their count | +| `compaction` | `field`: `head`, `heads`, `stale`, `changes`, `changed`, `staleTargets` | the Effort's compaction state | +| `body` | `record`, `includes`, `afterExcerpt` | whether the text is present (and past the excerpt) | +| `edge` | `edge` | whether the edge exists | + +`bodyIncludes` and `titleIncludes` take one phrase or a list, and a record +must contain every phrase. `invalidates` keeps records that invalidate the +named record. A record's **standing** is its state or status, or `live`, overridden by `retracted`, `superseded`, or `invalidated`. `select` with `live: true` keeps @@ -330,6 +337,41 @@ A benchmark agent must never see `spec/`, `questions/`, or `keymap.json`. with more right answers over the runs, and is the stricter reading. The script warns when the run files record different `find_hash` values. +**Ambiguous-Effort questions.** `questions/ambiguous.json` holds 24 +questions rewritten from tier 0-2 questions so that they name no Effort and +use words several Efforts share, such as "shader cache", "the +retrospective", "refraction", or "Skiff". Each names its source question in +`from`. Each lists in `rivals` at least one record from another Effort that +the wording also fits, and a trap says why that record is wrong. Checks over +every Effort (`effort: "*"`) prove that only one record or value fits. +`check.mjs` also refuses a question that names an Effort or one of its code +names, or uses a platform word that mostly one Effort's records use, such +as "handheld". It cannot prove that every other word in a question is +shared, so a reader who knows the subject may still guess the Effort. + +No script asks these questions by default, so every default stays on the +original 246: pass `--only ambiguous` to `run.mjs`, `grade.mjs`, +`compare.mjs`, or `find-recall.mjs`, or `--only default,ambiguous` for both +sets in one run, or name their ids with `--ids`. `check.mjs` always checks +them. + +**Adversarial filler.** This build makes 30% of the filler reuse real +subsystem names from the specs, such as Tern, VSM, and saltglass refraction: + +```bash +node scripts/build.mjs --scale 10 --filler-overlap 0.3 --out +``` + +These records still link to no real record and measure only housekeeping, +such as warning counts. The builder redraws any that would match a +question's check or state one of its answers, so `check.mjs --graph ` +gives the same result as on the plain 10x graph. Filler outside that 30% is +the same as in a plain build. The overlap records are the same on every +build with the same fraction and the same question files. The redraws read +every file in `questions/`, so editing any question can change them. The +build prints a hash of those files; rebuild an adversarial graph after any +question edit before you compare runs on it. + Three rules shaped this protocol, each learned from a pilot below: - **One session per question.** A session that answers many questions @@ -704,3 +746,24 @@ The largest reply grew by about 700 characters, to 7,360. That is close to 7,500 characters, the most this step allowed for the mean reply. A later change that adds passage lines should watch the largest reply, not only the mean. + +#### Questions that name no Effort, and filler that looks real + +The 24 ambiguous-Effort questions were frozen before `find` was run on +them. With the cards above (same `find_hash`), the stripped set on the 10x +graph gives: + +| Graph | Questions | Evidence in top 5 | Answer on a card | Right Effort first | Mean reply chars | +| --------------------------- | --------- | ----------------- | ---------------- | ------------------ | ---------------- | +| 10x | 24 | 96% | 87% | 88% | 6,448 | +| 10x, `--filler-overlap 0.3` | 24 | 96% | 87% | 92% | 6,433 | +| 10x, `--filler-overlap 0.3` | 246 | 80% | 62% (gate PASS) | 80% | 6,326 | +| 10x (from the table above) | 246 | 81% | 62% | 81% | 6,368 | + +Two things follow. First, search finds the right record even when the +question names no Effort. These questions were rewritten from tier 0-2 +lookups, though, so they are easier than the full set, and they test +`find` more than they test an agent. Second, filler that reuses real +subsystem names cost the gate one point of evidence and no answers, so the +10x result is not only a sign that filler is easy to tell apart. The paid +runs (Runs A to C in the plan) still have to show that agents gain from it. diff --git a/benchmarks/proof-recall/questions/ambiguous.json b/benchmarks/proof-recall/questions/ambiguous.json new file mode 100644 index 00000000..a545d92e --- /dev/null +++ b/benchmarks/proof-recall/questions/ambiguous.json @@ -0,0 +1,1073 @@ +{ + "effort": "ambiguous", + "questions": [ + { + "id": "ambiguous-001", + "from": "ports-002", + "tier": 0, + "hops": 0, + "categories": ["ambiguous-effort", "truncation", "lookup"], + "question": "Before I shrink the shader cache on Skiff, how large is it allowed to grow on the device's storage right now?", + "answer": { + "kind": "value", + "value": "384 MB", + "accept": ["384", "384 megabytes"] + }, + "evidence": ["ports.dec-pso-ondevice"], + "checks": [ + { + "type": "select", + "effort": "*", + "live": true, + "bodyIncludes": ["cache", "capped at"], + "expect": ["ports.dec-pso-ondevice"] + }, + { + "type": "body", + "record": "ports.dec-pso-ondevice", + "includes": "capped at 384 MB", + "afterExcerpt": true, + "expect": true + } + ], + "traps": [ + "212 MB is how much the cache held after a full playthrough, not its cap.", + "140 MB is the size of the prebuilt blobs the cache replaced.", + "\"Shader cache\" also names the build farm's compile cache. Its storage Risk is about a 4 TB farm volume, and that cache never ships to players." + ], + "rivals": ["tools.rsk-storage"] + }, + { + "id": "ambiguous-002", + "from": "ports-011", + "tier": 1, + "hops": 1, + "categories": ["ambiguous-effort", "distractor", "retraction"], + "question": "When a player comes back to an area they have already visited, what share of lookups hit the shader cache?", + "answer": { + "kind": "value", + "value": "99.6 percent", + "accept": ["99.6%", "99.6"] + }, + "evidence": ["ports.dec-pso-ondevice", "ports.fnd-cache-warm"], + "path": [["ports.dec-pso-ondevice", "supports", "ports.fnd-cache-warm"]], + "checks": [ + { + "type": "select", + "effort": "*", + "live": true, + "bodyIncludes": "second visit", + "expect": ["ports.fnd-cache-warm"] + }, + { + "type": "body", + "record": "ports.fnd-cache-warm", + "includes": "99.6 percent of pipeline lookups hit the cache", + "expect": true + } + ], + "traps": [ + "97 percent is the build farm's shader compile cache hit rate, in a retracted Finding posted here by mistake.", + "The build farm's shader cache reports hit rates too (94% steady state, 92.4% in week four), but they count shader permutations in farm builds, not lookups on a player's device." + ], + "rivals": ["tools.fnd-hitrate"] + }, + { + "id": "ambiguous-003", + "from": "tools-009", + "tier": 2, + "hops": 2, + "categories": ["ambiguous-effort", "blob"], + "question": "The shader cache warmed up week by week after the key change. What hit rate did it reach in week four?", + "answer": { + "kind": "value", + "value": "92.4%", + "accept": ["92.4", "92.4 percent"] + }, + "evidence": [ + "tools.fnd-hitrate", + "tools.cit-hitrate", + "tools.blb-hitrate" + ], + "path": [ + ["tools.fnd-hitrate", "cites", "tools.cit-hitrate"], + ["tools.cit-hitrate", "blob", "tools.blb-hitrate"] + ], + "checks": [ + { + "type": "select", + "effort": "*", + "kinds": ["blob"], + "bodyIncludes": ["hit_rate", "week"], + "expect": ["tools.blb-hitrate"] + }, + { + "type": "body", + "record": "tools.blb-hitrate", + "includes": "\"hit_rate\": 92.4", + "expect": true + } + ], + "traps": [ + "The Finding body gives only week 1 (71%).", + "The retrospective's 94% is the later steady-state rate, and the February summary says only \"low 90s\".", + "Blob bodies are left out of list and records digests; only `proof get` on the Blob shows the series.", + "Two SDF GI capture payloads also carry a hit-rate field, for the GPU L2 cache, with no weeks.", + "The on-device pipeline cache is also called the shader cache, and its key includes the driver version, but it reports one second-visit figure (99.6 percent), not weekly ones." + ], + "rivals": ["ports.fnd-cache-warm"] + }, + { + "id": "ambiguous-004", + "from": "tools-003", + "tier": 0, + "hops": 0, + "categories": ["ambiguous-effort", "truncation", "lookup"], + "question": "Going by the retrospective, how long does a clean rebuild take now that the cache is warm?", + "answer": { + "kind": "value", + "value": "6 min 40 s", + "accept": ["6 minutes 40 seconds", "6:40", "400 seconds", "6m40s"] + }, + "evidence": ["tools.fnd-retro"], + "checks": [ + { + "type": "select", + "effort": "*", + "kinds": ["finding"], + "live": true, + "titleIncludes": "retrospective", + "expect": ["ports.fnd-retro", "tools.fnd-retro"] + }, + { + "type": "select", + "effort": "*", + "kinds": ["finding"], + "live": true, + "titleIncludes": "retrospective", + "bodyIncludes": "clean rebuild", + "expect": ["tools.fnd-retro"] + }, + { + "type": "body", + "record": "tools.fnd-retro", + "includes": "6 min 40 s", + "afterExcerpt": true, + "expect": true + } + ], + "traps": [ + "The figure sits at the end of a long body, past the 600-character excerpt that list and records digests show.", + "The hit-rate report's 7.2-minute median build time is a different measure.", + "There are two retrospectives. The Skiff beta one says nothing about rebuild times." + ], + "rivals": ["ports.fnd-retro"] + }, + { + "id": "ambiguous-005", + "from": "ports-034", + "tier": 0, + "hops": 0, + "categories": ["ambiguous-effort", "truncation", "lookup"], + "question": "The retrospective puts a schedule cost on the wrong pipeline choice. How many weeks did it cost?", + "answer": { + "kind": "value", + "value": "6 weeks", + "accept": ["6", "six weeks"] + }, + "evidence": ["ports.fnd-retro"], + "checks": [ + { + "type": "select", + "effort": "*", + "kinds": ["finding"], + "live": true, + "titleIncludes": "retrospective", + "expect": ["ports.fnd-retro", "tools.fnd-retro"] + }, + { + "type": "select", + "effort": "*", + "kinds": ["finding"], + "live": true, + "titleIncludes": "retrospective", + "bodyIncludes": "pipeline choice", + "expect": ["ports.fnd-retro"] + }, + { + "type": "body", + "record": "ports.fnd-retro", + "includes": "cost 6 weeks of schedule", + "afterExcerpt": true, + "expect": true + } + ], + "traps": [ + "The Kelp switch took one week; that figure sits earlier in the same record.", + "The shader compile cache retrospective says the team lost about three weeks blaming the network for flaky nightly builds. That was a wrong diagnosis, not a pipeline choice." + ], + "rivals": ["tools.fnd-retro"] + }, + { + "id": "ambiguous-006", + "from": "render-013", + "tier": 0, + "hops": 0, + "categories": ["ambiguous-effort", "truncation", "lookup"], + "question": "I want to add a channel to the history the upscaler keeps between frames. How precisely is each channel stored today?", + "answer": { + "kind": "value", + "value": "16-bit float", + "accept": ["FP16", "16-bit float per channel", "half float"] + }, + "evidence": ["render.dec-tern"], + "checks": [ + { + "type": "select", + "effort": "*", + "live": true, + "bodyIncludes": "history buffer", + "expect": ["render.dec-tern"] + }, + { + "type": "body", + "record": "render.dec-tern", + "includes": "stored as 16-bit float per channel", + "afterExcerpt": true, + "expect": true + } + ], + "traps": [ + "FP16 packing of reprojection inputs is a different change, in a Tern cost Finding.", + "The Skiff upscaler Decision picks a Tern preset but says nothing about the history buffer; every preset shares one format." + ], + "rivals": ["ports.dec-tern-skiff"] + }, + { + "id": "ambiguous-007", + "from": "render-009", + "tier": 0, + "hops": 0, + "categories": ["ambiguous-effort", "truncation", "current-state"], + "question": "Someone wants the refraction fallback to take more steps. What step cap is in force now?", + "answer": { + "kind": "value", + "value": "32", + "accept": ["32 steps"] + }, + "evidence": ["render.dec-refr-hybrid"], + "checks": [ + { + "type": "select", + "effort": "*", + "kinds": ["decision"], + "live": true, + "bodyIncludes": ["refraction", "capped at"], + "expect": ["render.dec-refr-hybrid"] + }, + { + "type": "select", + "effort": "*", + "kinds": ["decision"], + "live": true, + "bodyIncludes": "capped at", + "expect": ["ports.dec-pso-ondevice", "render.dec-refr-hybrid"] + }, + { + "type": "body", + "record": "render.dec-refr-hybrid", + "includes": "capped at 32 steps", + "afterExcerpt": true, + "expect": true + } + ], + "traps": [ + "64 steps is what the October cost Finding and the rejected full ray-march used.", + "The only other live Decision that caps something is the Skiff pipeline cache, at 384 MB.", + "A rejected Skiff Decision would have dropped saltglass refraction to half resolution there. That limits pixels, not steps." + ], + "rivals": ["ports.dec-pso-ondevice", "ports.dec-skiff-native"] + }, + { + "id": "ambiguous-008", + "from": "netcode-006", + "tier": 1, + "hops": 1, + "categories": ["ambiguous-effort", "rejected-alternative"], + "question": "Someone wants co-op to run deterministically, with each peer waiting for every other peer's input before it simulates a frame. Which Decision turned that approach down?", + "answer": { + "kind": "record", + "value": "netcode.dec-rollback" + }, + "evidence": ["netcode.dec-lockstep", "netcode.dec-rollback"], + "path": [["netcode.dec-lockstep", "rejected_by", "netcode.dec-rollback"]], + "checks": [ + { + "type": "select", + "effort": "*", + "kinds": ["decision"], + "titleIncludes": "lockstep", + "expect": ["netcode.dec-lockstep"] + }, + { + "type": "select", + "effort": "*", + "kinds": ["decision"], + "where": { + "standing": "superseded" + }, + "derivesFrom": "netcode.iss-sync" + }, + { + "type": "edge", + "edge": [ + "netcode.dec-lockstep", + "rejected_by", + "netcode.dec-rollback" + ], + "expect": true + }, + { + "type": "select", + "effort": "*", + "kinds": ["decision"], + "bodyIncludes": "exchange only inputs", + "expect": ["world.dec-tide-fixed"] + } + ], + "traps": [ + "The host-authoritative Decision is today's answer to the sync question, but it rejected nothing; lockstep was already rejected.", + "The lockstep playtest Finding explains the loss but is not a Decision.", + "The fixed-point tide Decision keeps tides deterministic and says co-op peers exchange only inputs. But it is accepted, and what it rejected was the host streaming tide heights, not this approach." + ], + "rivals": ["world.dec-tide-fixed"] + }, + { + "id": "ambiguous-009", + "from": "world-015", + "tier": 0, + "hops": 0, + "categories": ["ambiguous-effort", "lookup"], + "question": "Gameplay scripts keep running long on Skiff. How many milliseconds per frame do all scripts together get there?", + "answer": { + "kind": "value", + "value": "3.0 ms", + "accept": ["3 ms", "3.0", "3"] + }, + "evidence": ["world.con-gull-tick"], + "checks": [ + { + "type": "select", + "effort": "*", + "kinds": ["constraint"], + "live": true, + "bodyIncludes": "script", + "expect": ["world.con-gull-tick"] + }, + { + "type": "body", + "record": "world.con-gull-tick", + "includes": "3.0 ms per frame on Skiff", + "expect": true + } + ], + "traps": [ + "2.0 ms is Meridian's budget.", + "25 ms is the whole Skiff 40 Hz frame, not the script share." + ], + "rivals": ["ports.con-frame-2"] + }, + { + "id": "ambiguous-010", + "from": "world-027", + "tier": 1, + "hops": 1, + "categories": ["ambiguous-effort", "current-state", "distractor"], + "question": "We want to stream more data per second on Skiff. What sustained read rate does the current streaming budget allow?", + "answer": { + "kind": "value", + "value": "160 MB/s", + "accept": ["160"] + }, + "evidence": ["world.con-io-budget", "world.con-io-budget-2"], + "path": [ + ["world.con-io-budget", "superseded_by", "world.con-io-budget-2"] + ], + "checks": [ + { + "type": "select", + "effort": "*", + "kinds": ["constraint"], + "live": true, + "bodyIncludes": "MB/s", + "expect": ["world.con-io-budget-2"] + }, + { + "type": "body", + "record": "world.con-io-budget-2", + "includes": "under 160 MB/s", + "expect": true + } + ], + "traps": [ + "220 MB/s is the superseded December budget.", + "The streaming pool budget has a near-identical title but limits memory, not reads.", + "61 MB/s is the worst sustained read measured on budget microSD cards, a Finding, not a budget." + ], + "rivals": ["ports.fnd-sd-speed-2"] + }, + { + "id": "ambiguous-011", + "from": "world-007", + "tier": 0, + "hops": 0, + "categories": ["ambiguous-effort", "truncation", "lookup"], + "question": "When the 400 MB memory cut hit the pool, how much of it came back from texture mips?", + "answer": { + "kind": "value", + "value": "260 MB", + "accept": ["260"] + }, + "evidence": ["world.con-pool-2"], + "checks": [ + { + "type": "select", + "effort": "*", + "bodyIncludes": ["400 MB", "texture mip"], + "expect": ["world.con-pool-2"] + }, + { + "type": "body", + "record": "world.con-pool-2", + "includes": "texture mips give back 260 MB", + "afterExcerpt": true, + "expect": true + } + ], + "traps": [ + "The split is in the last paragraph of a superseded constraint; the current constraint does not repeat it.", + "320 MB is the tide-band reservation, not part of the cut.", + "About 900 MB is what Kelp's high-ratio texture mode saved on the Skiff harbor route, a different cut." + ], + "rivals": ["ports.dec-kelp-skiff"] + }, + { + "id": "ambiguous-012", + "from": "world-005", + "tier": 0, + "hops": 0, + "categories": ["ambiguous-effort", "truncation", "lookup"], + "question": "Players say that something they dug up at low tide moves after they save and load the game again. In the worst case, how far inland did it end up?", + "answer": { + "kind": "value", + "value": "11 m", + "accept": ["11 metres", "11 meters", "11"] + }, + "evidence": ["world.fnd-save-tide-mismatch"], + "checks": [ + { + "type": "select", + "effort": "*", + "bodyIncludes": ["low tide", "inland"], + "expect": ["world.fnd-save-tide-mismatch"] + }, + { + "type": "body", + "record": "world.fnd-save-tide-mismatch", + "includes": "11 m inland", + "afterExcerpt": true, + "expect": true + } + ], + "traps": [ + "The distance is in the final paragraph, past the browse excerpt; 31 of 50 is the repro count.", + "The Skiff delta-save Decision is about how much autosave writes, not where chests end up." + ], + "rivals": ["ports.dec-delta-save"] + }, + { + "id": "ambiguous-013", + "from": "world-041", + "tier": 2, + "hops": 2, + "categories": ["ambiguous-effort", "blob"], + "question": "We're tuning what streams first when the tide falls. In the Skiff trace with 256 m cells, what share of the reads were textures?", + "answer": { + "kind": "value", + "value": "61%", + "accept": ["0.61", "61 percent"] + }, + "evidence": [ + "world.fnd-tide-burst", + "world.cit-trace-tide-turn", + "world.blob-trace-tide-turn" + ], + "path": [ + ["world.fnd-tide-burst", "cites", "world.cit-trace-tide-turn"], + ["world.cit-trace-tide-turn", "blob", "world.blob-trace-tide-turn"] + ], + "checks": [ + { + "type": "select", + "effort": "*", + "kinds": ["blob"], + "bodyIncludes": "\"cell_size_m\": 256", + "expect": ["world.blob-trace-tide-turn"] + }, + { + "type": "body", + "record": "world.blob-trace-tide-turn", + "includes": "\"textures\": 0.61", + "expect": true + } + ], + "traps": [ + "30% is the share of a cell that changes when the tide falls, not a read share.", + "The Skiff harbor memory capture also puts textures first, but in megabytes of memory, not as a share of reads." + ], + "rivals": ["ports.blb-mem-capture"] + }, + { + "id": "ambiguous-014", + "from": "ports-015", + "tier": 2, + "hops": 2, + "categories": ["ambiguous-effort", "blob", "distractor"], + "question": "We need to cut memory in the harbor on Skiff. Which category was largest in the capture that showed its peak going over the memory budget?", + "answer": { + "kind": "value", + "value": "textures", + "accept": ["texture", "textures (3184 MB)"] + }, + "evidence": [ + "ports.fnd-mem-capture", + "ports.cit-mem-capture", + "ports.blb-mem-capture" + ], + "path": [ + ["ports.fnd-mem-capture", "cites", "ports.cit-mem-capture"], + ["ports.cit-mem-capture", "blob", "ports.blb-mem-capture"] + ], + "checks": [ + { + "type": "select", + "effort": "*", + "kinds": ["blob"], + "bodyIncludes": "categories_mb", + "expect": ["ports.blb-mem-capture"] + }, + { + "type": "body", + "record": "ports.blb-mem-capture", + "includes": "\"textures\": 3184", + "expect": true + } + ], + "traps": [ + "The Finding says audio banks grew the most; growth is not size.", + "The streaming pool is the second largest at 2048 MB.", + "The streaming pool occupancy capture also records a memory peak, but only for the pool, and it does not split memory into categories." + ], + "rivals": ["world.blob-pool-occupancy"] + }, + { + "id": "ambiguous-015", + "from": "netcode-003", + "tier": 0, + "hops": 0, + "categories": ["ambiguous-effort", "lookup", "retraction", "distractor"], + "question": "I'm adding another component to the snapshot. How big is one player's snapshot for a single frame right now?", + "answer": { + "kind": "value", + "value": "38 KB", + "accept": ["38", "38 kilobytes"] + }, + "evidence": ["netcode.fnd-snapshot", "netcode.fnd-noise"], + "checks": [ + { + "type": "select", + "effort": "*", + "live": true, + "bodyIncludes": ["snapshot", "per frame"], + "expect": ["netcode.fnd-snapshot"] + }, + { + "type": "body", + "record": "netcode.fnd-snapshot", + "includes": "38 KB per frame", + "expect": true + }, + { + "type": "field", + "record": "netcode.fnd-noise", + "field": "standing", + "expect": "retracted" + } + ], + "traps": [ + "A retracted Finding gives 14 KB per player; it measured save-game snapshots and was filed here by mistake.", + "About 600 KB is the whole 4-frame ring for four players, not one player's frame.", + "A full world snapshot save is 310 MB. That is a save file, not a per-frame snapshot." + ], + "rivals": ["world.fnd-save-snapshot-size"] + }, + { + "id": "ambiguous-016", + "from": "tools-007", + "tier": 2, + "hops": 2, + "categories": ["ambiguous-effort", "dead-evidence", "invalidation"], + "question": "Before I touch the nightly build settings: which accepted Decision still rests on a diagnosis of the flaky nightlies that turned out to be wrong?", + "answer": { + "kind": "record", + "value": "tools.dec-timeout" + }, + "evidence": [ + "tools.dec-timeout", + "tools.fnd-flaky-net", + "tools.fnd-flaky-oom" + ], + "path": [ + ["tools.dec-timeout", "derives_from", "tools.fnd-flaky-net"], + ["tools.fnd-flaky-net", "invalidated_by", "tools.fnd-flaky-oom"] + ], + "checks": [ + { + "type": "select", + "effort": "*", + "kinds": ["finding"], + "where": { + "standing": "invalidated" + }, + "titleIncludes": "nightly", + "expect": ["tools.fnd-flaky-net"] + }, + { + "type": "select", + "effort": "*", + "kinds": ["decision"], + "where": { + "standing": "accepted" + }, + "derivesFrom": "tools.fnd-flaky-net" + } + ], + "traps": [ + "The job cap is the real fix for the flaky builds, but it rests on the correct memory diagnosis.", + "The superseded cache keys rest on superseded or live records, not on a disproved Finding.", + "Other Efforts hold accepted Decisions that rest on dead evidence too, but none rests on a nightly-build diagnosis." + ], + "rivals": ["ports.dec-stream-block"] + }, + { + "id": "ambiguous-017", + "from": "ports-017", + "tier": 2, + "hops": 2, + "categories": ["ambiguous-effort", "dead-evidence", "invalidation"], + "question": "We chose the size of our tile reads on Skiff from a storage speed measurement. Which Finding showed that measurement was wrong?", + "answer": { + "kind": "record", + "value": "ports.fnd-sd-speed-2" + }, + "evidence": [ + "ports.dec-stream-block", + "ports.fnd-sd-speed", + "ports.fnd-sd-speed-2" + ], + "path": [ + ["ports.dec-stream-block", "derives_from", "ports.fnd-sd-speed"], + ["ports.fnd-sd-speed", "invalidated_by", "ports.fnd-sd-speed-2"] + ], + "checks": [ + { + "type": "select", + "effort": "*", + "kinds": ["decision"], + "live": true, + "bodyIncludes": "in 2 MB blocks", + "expect": ["ports.dec-stream-block"] + }, + { + "type": "select", + "effort": "*", + "kinds": ["decision"], + "live": true, + "restsOn": "invalidated", + "bodyIncludes": "read", + "expect": ["ports.dec-stream-block", "world.dec-save-version"] + }, + { + "type": "edge", + "edge": [ + "ports.dec-stream-block", + "derives_from", + "ports.fnd-sd-speed" + ], + "expect": true + }, + { + "type": "select", + "effort": "*", + "kinds": ["finding"], + "invalidates": "ports.fnd-sd-speed" + } + ], + "traps": [ + "The 2 MB read Decision is still accepted; nothing invalidated it directly, so reading only its state hides the problem.", + "A retail-unit trace also showed that a dev-kit streaming read figure was wrong, but the 2 MB read Decision does not rest on that figure.", + "The save version Decision also rests on a disproved Finding, but that Finding timed save upgrades, not storage speed." + ], + "rivals": ["world.fnd-io-trace-2", "world.dec-save-version"] + }, + { + "id": "ambiguous-018", + "from": "world-026", + "tier": 1, + "hops": 1, + "categories": ["ambiguous-effort", "invalidation", "distractor"], + "question": "Our December measurement of sustained streaming reads on Skiff turned out to be wrong. Which Finding showed it?", + "answer": { + "kind": "record", + "value": "world.fnd-io-trace-2" + }, + "evidence": ["world.fnd-io-trace-1", "world.fnd-io-trace-2"], + "path": [ + ["world.fnd-io-trace-1", "invalidated_by", "world.fnd-io-trace-2"] + ], + "checks": [ + { + "type": "select", + "effort": "*", + "kinds": ["finding"], + "where": { + "standing": "invalidated" + }, + "bodyIncludes": "MB/s", + "expect": ["ports.fnd-sd-speed", "world.fnd-io-trace-1"] + }, + { + "type": "field", + "record": "world.fnd-io-trace-1", + "field": "created_at", + "expect": "2025-12-08T10:00:00Z" + }, + { + "type": "field", + "record": "ports.fnd-sd-speed", + "field": "created_at", + "expect": "2026-03-04T10:30:00Z" + }, + { + "type": "select", + "effort": "*", + "kinds": ["finding"], + "invalidates": "world.fnd-io-trace-1" + } + ], + "traps": [ + "The I/O budget constraint was superseded, not invalidated, and it is not a finding.", + "The microSD read measurement was wrong too, but it was taken in March, and the full-card test disproved it.", + "The Skiff beta retrospective mentions a December pipeline blob test; that tested pipeline blobs, not streaming reads." + ], + "rivals": ["ports.fnd-sd-speed-2", "ports.fnd-retro"] + }, + { + "id": "ambiguous-019", + "from": "render-001", + "tier": 0, + "hops": 0, + "categories": ["ambiguous-effort", "lookup", "current-state"], + "question": "We're adding a pass to the 120 Hz performance mode on Meridian. How many milliseconds of each frame may GPU work take?", + "answer": { + "kind": "value", + "value": "7.4", + "accept": ["7.4 ms"] + }, + "evidence": ["render.con-120-2"], + "checks": [ + { + "type": "select", + "effort": "*", + "kinds": ["constraint"], + "live": true, + "titleIncludes": "120 Hz", + "expect": ["render.con-120-2"] + }, + { + "type": "select", + "effort": "*", + "kinds": ["constraint"], + "live": true, + "bodyIncludes": "120 Hz", + "expect": [ + "ports.con-frame-2", + "render.con-120-2", + "render.con-budget-3" + ] + }, + { + "type": "body", + "record": "render.con-120-2", + "includes": "Lantern GPU passes get 7.4 ms", + "expect": true + } + ], + "traps": [ + "8.3 ms is the whole 120 Hz frame, not the GPU share.", + "15.0 ms is the GPU share in Meridian's 60 Hz quality mode, in a Frame budget Constraint.", + "0.9 ms is the headroom left after the GPU share.", + "The ports Frame budget Constraint mentions the 120 Hz mode but gives it no budget; it says that budget belongs elsewhere." + ], + "rivals": ["ports.con-frame-2"] + }, + { + "id": "ambiguous-020", + "from": "render-035", + "tier": 2, + "hops": 3, + "categories": ["ambiguous-effort", "blob", "lifecycle"], + "question": "We're resizing the pool that overflowed at low tide. At its peak, how many pages did the trace behind that risk's evidence ask for?", + "answer": { + "kind": "value", + "value": "5912", + "accept": ["5,912"] + }, + "evidence": [ + "render.rsk-vsm-overflow", + "render.fnd-vsm-overflow", + "render.cit-vsm-pages", + "render.blb-vsm-pages" + ], + "path": [ + ["render.rsk-vsm-overflow", "evidence", "render.fnd-vsm-overflow"], + ["render.fnd-vsm-overflow", "cites", "render.cit-vsm-pages"], + ["render.cit-vsm-pages", "blob", "render.blb-vsm-pages"] + ], + "checks": [ + { + "type": "select", + "effort": "*", + "kinds": ["risk"], + "titleIncludes": ["pool", "overflow"], + "expect": ["render.rsk-vsm-overflow"] + }, + { + "type": "select", + "effort": "*", + "kinds": ["blob"], + "bodyIncludes": "peak_pages_requested", + "expect": ["render.blb-vsm-pages"] + }, + { + "type": "body", + "record": "render.blb-vsm-pages", + "includes": "\"peak_pages_requested\": 5912", + "expect": true + } + ], + "traps": [ + "4,096 is the pool size, not the peak request.", + "The streaming pool Risk also came true, but it was a budget cut, not an overflow, and it has no page trace." + ], + "rivals": ["world.rsk-pool-cut"] + }, + { + "id": "ambiguous-021", + "from": "ports-008", + "tier": 1, + "hops": 1, + "categories": ["ambiguous-effort", "rejected-alternative"], + "question": "When we chose the upscaler for Skiff, which Decision's acceptance rejected the spatial upscaler?", + "answer": { + "kind": "record", + "value": "ports.dec-tern-skiff" + }, + "evidence": ["ports.dec-skiff-spatial", "ports.dec-tern-skiff"], + "path": [ + ["ports.dec-skiff-spatial", "rejected_by", "ports.dec-tern-skiff"] + ], + "checks": [ + { + "type": "select", + "effort": "*", + "kinds": ["decision"], + "where": { + "standing": "rejected" + }, + "titleIncludes": "spatial", + "expect": ["ports.dec-skiff-spatial"] + }, + { + "type": "select", + "effort": "*", + "kinds": ["decision"], + "derivesFrom": "ports.iss-upscale-q", + "where": { + "state": "accepted" + } + }, + { + "type": "edge", + "edge": [ + "ports.dec-skiff-spatial", + "rejected_by", + "ports.dec-tern-skiff" + ], + "expect": true + } + ], + "traps": [ + "The native 720p option was rejected by the same acceptance; it is a fellow loser, not the rejecter.", + "The answer names the render Decision \"Ship Tern as Lantern's only upscaler\" as its reason, but that render Decision did not reject the spatial option." + ], + "rivals": ["render.dec-tern"] + }, + { + "id": "ambiguous-022", + "from": "render-048", + "tier": 2, + "hops": 2, + "categories": ["ambiguous-effort", "blocker"], + "question": "I'm taking over the open blocker about the caustics. A test found that one of the proposed fixes leaves the caustics ghosting. Which Decision is that?", + "answer": { + "kind": "record", + "value": "render.dec-caustic-clamp" + }, + "evidence": [ + "render.iss-caustic-flicker", + "render.dec-caustic-clamp", + "render.fnd-caustic-clamp-test" + ], + "path": [ + ["render.iss-caustic-flicker", "supports", "render.dec-caustic-clamp"], + [ + "render.dec-caustic-clamp", + "supports", + "render.fnd-caustic-clamp-test" + ] + ], + "checks": [ + { + "type": "select", + "effort": "*", + "kinds": ["issue"], + "live": true, + "titleIncludes": "flicker", + "expect": ["render.iss-caustic-flicker"] + }, + { + "type": "select", + "effort": "*", + "kinds": ["decision"], + "where": { + "standing": "proposed" + }, + "derivesFrom": "render.iss-caustic-flicker", + "expect": ["render.dec-caustic-clamp", "render.dec-caustic-seed"] + }, + { + "type": "select", + "effort": "*", + "kinds": ["finding"], + "titleIncludes": "ghost", + "derivesFrom": "render.dec-caustic-clamp", + "expect": ["render.fnd-caustic-clamp-test"] + }, + { + "type": "select", + "effort": "*", + "kinds": ["decision"], + "derivesFrom": "render.iss-caustic-flicker", + "bodyIncludes": "clamp" + }, + { + "type": "select", + "effort": "*", + "kinds": ["decision"], + "where": { + "standing": "proposed" + }, + "derivesFrom": "ports.iss-blk-storm", + "expect": ["ports.dec-storm-30", "ports.dec-storm-cap"] + } + ], + "traps": [ + "The tide-locked seed test found frozen caustics, not ghosting.", + "The open Skiff cert blocker about tide storms also has two proposed fixes and a test, but the test found a CPU saving, not ghosting.", + "A rejected Skiff Decision would have stepped caustics down a quality tier under GPU load. It was never a fix for the caustics blocker." + ], + "rivals": ["ports.dec-storm-cap", "ports.dec-dynres-tiers"] + }, + { + "id": "ambiguous-023", + "from": "ports-006", + "tier": 1, + "hops": 1, + "categories": ["ambiguous-effort", "supersession", "current-state"], + "question": "Which Constraint sets how much memory the game may use on Skiff today?", + "answer": { + "kind": "record", + "value": "ports.con-skiff-mem-8" + }, + "evidence": ["ports.con-skiff-mem-9", "ports.con-skiff-mem-8"], + "path": [ + ["ports.con-skiff-mem-9", "superseded_by", "ports.con-skiff-mem-8"] + ], + "checks": [ + { + "type": "select", + "effort": "*", + "kinds": ["constraint"], + "live": true, + "titleIncludes": "memory", + "bodyIncludes": "Skiff" + }, + { + "type": "head", + "start": "ports.con-skiff-mem-9", + "expect": "ports.con-skiff-mem-8" + } + ], + "traps": [ + "Both versions share the title Skiff memory budget; the 9 GB version was superseded.", + "The streaming pool budget (1.6 GB) is one system's share of the Skiff total, not the total." + ], + "rivals": ["world.con-pool-3"] + }, + { + "id": "ambiguous-024", + "from": "world-028", + "tier": 1, + "hops": 1, + "categories": ["ambiguous-effort", "rejected-alternative"], + "question": "We're thinking again about letting one machine run the tide and send the result to everyone else. Which Finding measured the cost that ruled that out last time?", + "answer": { + "kind": "record", + "value": "world.fnd-tide-host-bw" + }, + "evidence": ["world.dec-tide-host", "world.fnd-tide-host-bw"], + "path": [["world.dec-tide-host", "supports", "world.fnd-tide-host-bw"]], + "checks": [ + { + "type": "select", + "effort": "*", + "kinds": ["decision"], + "titleIncludes": "tide height", + "expect": ["world.dec-tide-host"] + }, + { + "type": "select", + "effort": "*", + "kinds": ["finding"], + "derivesFrom": "world.dec-tide-host" + } + ], + "traps": [ + "The strict-float finding ruled out a different alternative.", + "The netcode Effort's host-authoritative spike Finding measures the client CPU saved by not running tides, not the bandwidth of streaming tide heights." + ], + "rivals": ["netcode.fnd-host-spike"] + } + ] +} diff --git a/benchmarks/proof-recall/scripts/build.mjs b/benchmarks/proof-recall/scripts/build.mjs index dd96663e..3e1ef4d2 100644 --- a/benchmarks/proof-recall/scripts/build.mjs +++ b/benchmarks/proof-recall/scripts/build.mjs @@ -10,6 +10,11 @@ // add filler until each Effort is // about 10 times its size; every // question keeps its answer +// node scripts/build.mjs --scale 10 --filler-overlap 0.3 --out /tmp/x10adv +// make 30% of the filler reuse real +// subsystem names (see +// spec/_filler.mjs); answers still +// hold import { createHash } from 'node:crypto'; import { existsSync } from 'node:fs'; import { cp, mkdir, readdir, readFile, rm, writeFile } from 'node:fs/promises'; @@ -20,7 +25,12 @@ import { generateArtifactId, ProofMutationSchema, } from '@flatbread/proof'; -import { fillerEvents, fillerTerms, mergeEvents } from '../spec/_filler.mjs'; +import { + fillerEvents, + fillerTerms, + mergeEvents, + overlapTerms, +} from '../spec/_filler.mjs'; const BENCH = resolve(dirname(fileURLToPath(import.meta.url)), '..'); const SPEC_DIR = join(BENCH, 'spec'); @@ -69,14 +79,24 @@ const GRAPH_DIRS = [ ]; function args(argv) { - const out = { only: [], out: join(BENCH, '.flatbread-proof'), scale: 1 }; + const out = { + only: [], + out: join(BENCH, '.flatbread-proof'), + scale: 1, + overlap: 0, + }; for (let i = 0; i < argv.length; i++) { if (argv[i] === '--only') out.only.push(...argv[++i].split(',')); else if (argv[i] === '--out') out.out = resolve(argv[++i]); else if (argv[i] === '--scale') out.scale = Number(argv[++i]); + else if (argv[i] === '--filler-overlap') out.overlap = Number(argv[++i]); else throw new Error(`Unknown argument ${argv[i]}`); } if (!(out.scale >= 1)) throw new Error('--scale must be 1 or more'); + if (!(out.overlap >= 0 && out.overlap <= 1)) + throw new Error('--filler-overlap must be a fraction from 0 to 1'); + if (out.overlap > 0 && !(out.scale > 1)) + throw new Error('--filler-overlap needs --scale above 1'); // The committed graph is the unscaled one. if (out.scale > 1 && out.out === join(BENCH, '.flatbread-proof')) throw new Error('A scaled build needs --out'); @@ -100,6 +120,121 @@ async function assertFillerIsDisjoint(specs) { throw new Error(`Filler subsystem "${term}" appears in real content`); } +/** The question sets, and each spec as text, for the overlap checks. */ +async function readContent(specs) { + const dir = join(BENCH, 'questions'); + const files = (await readdir(dir)).filter((name) => name.endsWith('.json')); + const sets = await Promise.all( + files.map(async (name) => + JSON.parse(await readFile(join(dir, name), 'utf8')) + ) + ); + // The overlap redraws depend on these files, so a hash of them tells + // whether two adversarial graphs were built from the same questions. + const hash = createHash('sha256'); + for (const name of [...files].sort()) + hash.update(`${name}\0`).update(await readFile(join(dir, name))); + return { + sets, + specText: specs.map((spec) => JSON.stringify(spec)), + questionsHash: hash.digest('hex'), + }; +} + +/** + * Overlap filler must use real names and nothing a question asks about: + * each real name appears in its Effort's spec, and no housekeeping metric + * or question phrase appears in any spec or question. + */ +function assertOverlapVocabulary(specs, { sets, specText }) { + const all = [...specText, ...sets.map((set) => JSON.stringify(set))] + .join('\n') + .toLowerCase(); + for (const [i, spec] of specs.entries()) { + const { real, housekeeping } = overlapTerms(spec.key); + for (const name of real) + if (!specText[i].toLowerCase().includes(name.toLowerCase())) + throw new Error( + `Overlap name "${name}" is not in the ${spec.key} spec` + ); + for (const term of housekeeping) + if (all.includes(term.toLowerCase())) + throw new Error( + `Overlap filler term "${term}" appears in real content` + ); + } +} + +/** + * For one Effort, a test that is true when an overlap filler event could + * change a question's answer. It is true when the event's title and body + * hold: + * + * - every phrase of a `select` check's `bodyIncludes` and `titleIncludes` + * together, for a check that scans this Effort and the event's kind; + * - a `body` check's phrase, or a value answer, that is a measurement: a + * number with a unit or word, such as "384 MB" or "6 min 40 s", matched as + * whole words so "4 minutes" does not match "14 minutes". + * + * Bare numbers ("2", "87") and names ("Dara Whitlock") are left out: filler + * prints both everywhere, and neither states a fact about a real subsystem. + */ +function overlapCollision(effortKey, { sets }) { + const selects = []; // { kinds, phrases }: all phrases must appear + const measures = new Set(); // each must appear as whole words + const scans = (set, check) => + check.effort === '*' || + (Array.isArray(check.effort) + ? check.effort.includes(effortKey) + : (check.effort ?? set.effort) === effortKey); + const list = (value) => (Array.isArray(value) ? value : [value]); + const isMeasure = (text) => /\d/.test(text) && /[a-z%]/i.test(text); + for (const set of sets) + for (const q of set.questions) { + for (const check of q.checks ?? []) { + if (check.type === 'select' && scans(set, check)) { + const phrases = ['bodyIncludes', 'titleIncludes'].flatMap((field) => + check[field] ? list(check[field]) : [] + ); + if (phrases.length) + selects.push({ + kinds: check.kinds ?? null, + phrases: phrases.map((p) => p.toLowerCase()), + }); + } + if (check.type === 'body') + for (const p of list(check.includes)) + if (isMeasure(p)) measures.add(p.toLowerCase()); + } + if (q.answer.kind === 'value') + for (const gold of [q.answer.value, ...(q.answer.accept ?? [])]) + if (isMeasure(String(gold))) measures.add(String(gold).toLowerCase()); + } + const wholeWord = (text, phrase) => { + for ( + let i = text.indexOf(phrase); + i >= 0; + i = text.indexOf(phrase, i + 1) + ) { + const before = text[i - 1] ?? ' '; + const after = text[i + phrase.length] ?? ' '; + if (!/[\w.]/.test(before) && !/\w/.test(after)) return true; + } + return false; + }; + return (event) => { + const kind = KIND_BY_TYPE[event.type]; + const text = `${event.title}\n${event.body}`.toLowerCase(); + return ( + selects.some( + ({ kinds, phrases }) => + (!kinds || kinds.includes(kind)) && + phrases.every((p) => text.includes(p)) + ) || [...measures].some((m) => wholeWord(text, m)) + ); + }; +} + /** Fixed id: the writer's own format, with a suffix hashed from the key. */ export function benchId(kind, title, qualifiedKey) { const bytes = createHash('sha256').update(qualifiedKey).digest(); @@ -193,8 +328,25 @@ async function main() { if (!specs.length) throw new Error('No specs matched'); if (options.scale > 1) { await assertFillerIsDisjoint(specs); + const content = options.overlap > 0 ? await readContent(specs) : null; + if (content) { + assertOverlapVocabulary(specs, content); + console.log(`filler overlap: questions sha256 ${content.questionsHash}`); + } for (const spec of specs) - spec.events = mergeEvents(spec.events, fillerEvents(spec, options.scale)); + spec.events = mergeEvents( + spec.events, + fillerEvents( + spec, + options.scale, + content + ? { + overlap: options.overlap, + collides: overlapCollision(spec.key, content), + } + : undefined + ) + ); } await mkdir(options.out, { recursive: true }); for (const dir of GRAPH_DIRS) diff --git a/benchmarks/proof-recall/scripts/check.mjs b/benchmarks/proof-recall/scripts/check.mjs index 247842d5..2813a0a6 100644 --- a/benchmarks/proof-recall/scripts/check.mjs +++ b/benchmarks/proof-recall/scripts/check.mjs @@ -3,8 +3,11 @@ // Exits 1 on any error, so authors can loop on it. // // node scripts/check.mjs all question files -// node scripts/check.mjs --only render one Effort's questions +// node scripts/check.mjs --only render one question set // node scripts/check.mjs --graph /tmp/x a graph built elsewhere +// +// This script checks every file, including the opt-in sets in OPT_IN_SETS. +// Other scripts load those only when asked for them by name. import { readdir, readFile } from 'node:fs/promises'; import { dirname, join, resolve } from 'node:path'; import { fileURLToPath, pathToFileURL } from 'node:url'; @@ -41,7 +44,19 @@ export const CATEGORIES = [ 'distractor', // a similar record in another Effort or thread 'cross-effort', // needs records from two Efforts 'rationale', // explain why, scored by rubric + // Rewritten so it names no Effort and uses words several Efforts share; + // the reader must find the right Effort first. Only in ambiguous.json. + 'ambiguous-effort', ]; +// Question files whose `effort` names no Effort: their questions span +// several, so every key is qualified as `.`. +export const MULTI_EFFORT_SETS = ['cross', 'ambiguous']; +// Sets that load only when a script names them (`--only ambiguous`) or asks +// for one of their ids, so every default keeps the original 246 questions. +export const OPT_IN_SETS = ['ambiguous']; +// Names that point at one Effort without its title: the engine and renderer +// code names that Effort titles use. +const EFFORT_NAMES = ['Kestrel', 'Lantern']; const ANSWER_KINDS = ['record', 'records', 'value', 'boolean', 'count', 'text']; const NEEDS_CHECK = new Set(['record', 'records', 'count']); @@ -55,21 +70,136 @@ function args(argv) { return out; } -export async function loadQuestions(only = []) { +/** + * Question files, sorted by name. `only` lists the sets to load by their + * `effort` field; the word `default` stands for every set outside + * OPT_IN_SETS. With no `only`, the default sets load, plus the opt-in ones + * when `optIn` is true. + */ +export async function loadQuestions(only = [], { optIn = false } = {}) { const dir = join(BENCH, 'questions'); const files = (await readdir(dir)) .filter((name) => name.endsWith('.json')) .sort(); + const wanted = (name) => + only.length + ? only.includes(name) || + (only.includes('default') && !OPT_IN_SETS.includes(name)) + : optIn || !OPT_IN_SETS.includes(name); const sets = []; for (const file of files) { const set = JSON.parse(await readFile(join(dir, file), 'utf8')); - if (only.length && !only.includes(set.effort)) continue; - sets.push({ file, ...set }); + if (wanted(set.effort)) sets.push({ file, ...set }); } return sets; } -function validate(graph, set, q, errors, seen, warnings = []) { +/** + * Questions picked by set and id, as run.mjs and prepare-run.mjs take them. + * Ids are looked up in every set, so `--ids` reaches the opt-in sets too. + */ +export async function pickQuestions({ only = [], ids = [] } = {}) { + const wanted = new Set(ids); + return (await loadQuestions(only, { optIn: wanted.size > 0 })) + .map((set) => ({ + ...set, + questions: set.questions.filter((q) => !wanted.size || wanted.has(q.id)), + })) + .filter((set) => set.questions.length); +} + +/** Problems with an ambiguous-Effort question, as messages. */ +function ambiguityErrors(graph, q, sources) { + const out = []; + const source = sources.get(q.from); + if (!source) return [`"from" must name a question in an Effort file`]; + if (source.q.tier > 2) + out.push(`its source ${q.from} is tier ${source.q.tier}; use tier 0-2`); + for (const c of ['ambiguous-effort', ...source.q.categories]) + if (!q.categories?.includes(c)) out.push(`needs category ${c}`); + if (JSON.stringify(q.answer) !== JSON.stringify(qualifiedAnswer(source))) + out.push(`its answer differs from ${q.from}'s`); + const text = q.question ?? ''; + const efforts = Object.keys(graph.keymap) + .filter((key) => key.endsWith('.effort')) + .map((key) => key.split('.')[0]); + const titles = efforts.map( + (name) => + graph.records.get(graph.keymap[`${name}.effort`]).frontmatter.title + ); + for (const title of titles) + if (text.toLowerCase().includes(title.toLowerCase())) + out.push(`names the Effort "${title}"`); + if (/\bEfforts?\b/i.test(text)) out.push('says "Effort"'); + for (const name of EFFORT_NAMES) + if (new RegExp(`\\b${name}\\b`, 'i').test(text)) + out.push(`names ${name}, which singles out one Effort`); + // "the ports team", "the render work": an Effort key used as a name. + const keyed = new RegExp( + `\\b(${efforts.join('|')})\\s+(team|work|side|Effort)\\b`, + 'i' + ).exec(text); + if (keyed) out.push(`"${keyed[0]}" names an Effort`); + // A platform word is fair only if records in two or more Efforts use it + // and no one Effort holds more than two thirds of those records. + for (const word of PLATFORM_WORDS) { + const pattern = new RegExp( + `\\b${word}\\b`, + word === word.toLowerCase() ? 'i' : '' + ); + if (!pattern.test(text)) continue; + const perEffort = new Map(); + for (const rec of graph.records.values()) + if ( + rec.kind !== 'effort' && + pattern.test(`${rec.frontmatter.title ?? ''}\n${rec.body}`) + ) + perEffort.set( + rec.frontmatter.effort, + (perEffort.get(rec.frontmatter.effort) ?? 0) + 1 + ); + const total = [...perEffort.values()].reduce((a, b) => a + b, 0); + if (perEffort.size < 2) + out.push(`says "${word}", which fewer than two Efforts use`); + else if (Math.max(...perEffort.values()) > (total * 2) / 3) + out.push(`says "${word}", which mostly one Effort uses`); + } + // Rivals are records in other Efforts that the wording also fits: the + // near misses that make the question ambiguous. A trap says why each one + // is wrong. + if (!q.rivals?.length) + out.push('needs `rivals`: near-miss records in other Efforts'); + const ownEffort = graph.keymap[`${source.effort}.effort`]; + for (const key of q.rivals ?? []) { + const rec = graph.records.get(graph.keymap[key]); + if (rec?.frontmatter.effort === ownEffort) + out.push(`rival ${key} is in ${source.effort}, the answer's own Effort`); + } + return out; +} + +// Words for a platform. "handheld" and "home console" stand for Skiff and +// Meridian, but may lean on the ports Effort, which owns both ports. +const PLATFORM_WORDS = ['PC', 'Meridian', 'Skiff', 'handheld', 'home console']; + +/** A source question's answer with its keys qualified by its Effort. */ +function qualifiedAnswer({ effort, q }) { + const { kind, value } = q.answer; + if (kind === 'record') return { ...q.answer, value: qualify(effort, value) }; + if (kind === 'records') + return { ...q.answer, value: value.map((key) => qualify(effort, key)) }; + return q.answer; +} + +function validate( + graph, + set, + q, + errors, + seen, + warnings = [], + sources = new Map() +) { const err = (message) => errors.push(`${set.file} ${q.id ?? '?'}: ${message}`); if (!q.id || seen.has(q.id)) return err('missing or duplicate id'); @@ -94,8 +224,20 @@ function validate(graph, set, q, errors, seen, warnings = []) { err('text answers need a rubric'); if (answer.kind !== 'text' && answer.value === undefined) err('answer needs a value'); + const multi = MULTI_EFFORT_SETS.includes(set.effort); + if ( + q.categories?.includes('ambiguous-effort') !== + (set.effort === 'ambiguous') + ) + err( + 'the ambiguous-effort category belongs to ambiguous.json, and every question there needs it' + ); + if (set.effort === 'ambiguous') + for (const message of ambiguityErrors(graph, q, sources)) err(message); const keysOk = (keys, label) => { for (const key of keys ?? []) { + if (multi && !String(key).includes('.')) + err(`${label}: key "${key}" needs its Effort, as .`); try { idFor(graph, set.effort, key); } catch (error) { @@ -105,6 +247,7 @@ function validate(graph, set, q, errors, seen, warnings = []) { }; if (!q.evidence?.length) err('needs evidence keys'); keysOk(q.evidence, 'evidence'); + keysOk(q.rivals, 'rivals'); if (answer.kind === 'record') keysOk([answer.value], 'answer'); if (answer.kind === 'records') keysOk(answer.value, 'answer'); const path = q.path ?? []; @@ -117,7 +260,7 @@ function validate(graph, set, q, errors, seen, warnings = []) { // Runs show a question alone. If an Effort-wide check would also find // answers in another Effort, the question must name its own, or carry // `"scoped": true` once an author confirms the wording pins it down. - if (set.effort !== 'cross' && !q.scoped) { + if (!multi && !q.scoped) { const title = graph.records.get(graph.keymap[`${set.effort}.effort`]) ?.frontmatter.title; const named = @@ -297,14 +440,21 @@ export function graphStats(graph, effortKey) { async function main() { const options = args(process.argv.slice(2)); const graph = await loadGraph(options.graph); - const sets = await loadQuestions(options.only); + const sets = await loadQuestions(options.only, { optIn: true }); + // Ambiguous questions are checked against their sources, so load every + // Effort file even under --only. + const sources = new Map(); + for (const set of await loadQuestions()) + if (!MULTI_EFFORT_SETS.includes(set.effort)) + for (const q of set.questions) + sources.set(q.id, { effort: set.effort, q }); const errors = []; const warnings = []; const seen = new Set(); const all = []; for (const set of sets) for (const q of set.questions ?? []) { - validate(graph, set, q, errors, seen, warnings); + validate(graph, set, q, errors, seen, warnings, sources); all.push({ ...q, effort: set.effort }); } const efforts = [ @@ -324,9 +474,11 @@ async function main() { }) ); } - const other = all.filter((q) => !efforts.includes(q.effort)); - if (other.length) - console.log(JSON.stringify({ effort: 'cross', questions: other.length })); + for (const name of MULTI_EFFORT_SETS) { + const qs = all.filter((q) => q.effort === name); + if (qs.length) + console.log(JSON.stringify({ effort: name, questions: qs.length })); + } if (warnings.length) { console.warn(warnings.join('\n')); console.warn(`${warnings.length} warning(s)`); diff --git a/benchmarks/proof-recall/scripts/compare.mjs b/benchmarks/proof-recall/scripts/compare.mjs index ca7f8e26..d103a2bc 100644 --- a/benchmarks/proof-recall/scripts/compare.mjs +++ b/benchmarks/proof-recall/scripts/compare.mjs @@ -2,9 +2,14 @@ // Compares two runs over the questions both answered, pair by pair. // // node scripts/compare.mjs runs/.json runs/.json [--skip id1,id2] +// [--only ambiguous] // node scripts/compare.mjs runs/a-r1.json,runs/a-r2.json \ // runs/b-r1.json,runs/b-r2.json // +// It compares the original 246 questions. `--only` names the sets to compare +// instead, as run.mjs takes them, so a run that also asked the +// ambiguous-Effort set reports that set apart: `--only ambiguous`. +// // With several files on each side, the runs are pooled: run i of side A is // paired with run i of side B, and each question in each pair of runs is one // row, named `#` in the lists of questions only one side got right. @@ -52,10 +57,12 @@ async function main() { const argv = process.argv.slice(2); const at = argv.indexOf('--skip'); const skip = new Set(at >= 0 ? argv.splice(at, 2)[1].split(',') : []); + const atOnly = argv.indexOf('--only'); + const only = atOnly >= 0 ? argv.splice(atOnly, 2)[1].split(',') : []; const [pathA, pathB] = argv; if (!pathB) throw new Error( - 'usage: compare.mjs [,more] [,more] [--skip ids]' + 'usage: compare.mjs [,more] [,more] [--skip ids] [--only sets]' ); const load = (paths) => Promise.all( @@ -86,7 +93,7 @@ async function main() { ? runs.map((run, i) => run.run?.name ?? paths.split(',')[i]).join(',') : runs[0].run?.name ?? paths; const graph = await loadGraph(join(BENCH, '.flatbread-proof')); - const sets = await loadQuestions(); + const sets = await loadQuestions(only); const rows = []; for (const [i, a] of runsA.entries()) for (const set of sets) diff --git a/benchmarks/proof-recall/scripts/find-recall.mjs b/benchmarks/proof-recall/scripts/find-recall.mjs index 9c3869b7..f070b856 100644 --- a/benchmarks/proof-recall/scripts/find-recall.mjs +++ b/benchmarks/proof-recall/scripts/find-recall.mjs @@ -3,7 +3,14 @@ // whether its top hits hold the evidence, before any paid agent run. // // node scripts/find-recall.mjs [--graph ] [--ids a,b] -// [--split tune|holdout|all] [--out ] [--jobs 8] +// [--only ambiguous] [--split tune|holdout|all] [--out ] +// [--jobs 8] +// +// The gate asks the original 246 questions. `--only` names question sets +// by their `effort` field instead, as run.mjs takes them: `--only ambiguous` +// asks the ambiguous-Effort set, whose questions name no Effort. `--ids` +// picks questions from any set. Run the ambiguous set only after it is +// written and frozen, since its point is that nobody tuned it to `find`. // // The gate verdict that counts is the one on the 10x graph, given with // `--graph `. The default, the committed 1x graph, is a quick @@ -16,7 +23,7 @@ // names, or platform names, with --all. Topic words that an // Effort title also uses, such as "build farm", stay elsewhere. // scoped the question text, with --effort set to the question's Effort -// (cross-Effort questions are skipped) +// (cross-Effort and ambiguous-Effort questions are skipped) // The plan asked for a third set of queries taken from pilot transcripts. // No pilot ran `proof find`, so none exist; `scoped` stands in until a find // pilot gives real agent queries. @@ -42,7 +49,9 @@ // split's rows are the same as its part of an `all` // run. // right_effort_first the top Effort in `efforts by match` is the -// question's own (single-Effort questions, --all sets) +// question's own (single-Effort questions, --all sets; +// for an ambiguous-Effort question, the Effort of the +// question it was rewritten `from`) // reply_chars, cards_chars // the tables print the mean reply size and the // largest one @@ -71,7 +80,7 @@ import { request } from 'node:http'; import { tmpdir } from 'node:os'; import { dirname, join, resolve } from 'node:path'; import { fileURLToPath } from 'node:url'; -import { loadQuestions } from './check.mjs'; +import { MULTI_EFFORT_SETS, pickQuestions } from './check.mjs'; import { findHash } from './collect.mjs'; import { loadGraph, qualify } from './graph.mjs'; import { prepareFolder } from './prepare-run.mjs'; @@ -101,6 +110,7 @@ function args(argv) { const out = { graph: join(BENCH, '.flatbread-proof'), ids: [], + only: [], out: join(BENCH, 'runs', 'find-recall.json'), jobs: 8, split: 'all', @@ -109,6 +119,7 @@ function args(argv) { const flag = argv[i]; if (flag === '--graph') out.graph = resolve(argv[++i]); else if (flag === '--ids') out.ids = argv[++i].split(','); + else if (flag === '--only') out.only = argv[++i].split(','); else if (flag === '--out') out.out = resolve(argv[++i]); else if (flag === '--jobs') out.jobs = Number(argv[++i]); else if (flag === '--split') out.split = argv[++i]; @@ -263,9 +274,9 @@ function scoreReply(graph, effortIds, row, stdout) { ), answer_on_card: answerOnCard(row.q, answers, reply.cards), right_effort_first: - row.set === 'scoped' || row.effort === 'cross' + row.set === 'scoped' || !effortIds[row.home] ? null - : reply.find.efforts?.[0]?.id === effortIds[row.effort], + : reply.find.efforts?.[0]?.id === effortIds[row.home], reply_chars: stdout.length, cards_chars: reply.cards.length, hits: top5.map((hit) => hit.matched_id ?? hit.id), @@ -341,12 +352,14 @@ async function main() { 'that counts is the one on the 10x graph (--graph ).' ); const find_hash = await findHash(); - const wanted = new Set(options.ids); const jobs = []; - for (const set of await loadQuestions()) + for (const set of await pickQuestions(options)) for (const q of set.questions) { - if (wanted.size && !wanted.has(q.id)) continue; - const row = { q, effort: set.effort }; + // The Effort a reader should land in: the file's own, or for an + // ambiguous question, that of its source (ids are `-NNN`). + const home = + set.effort === 'ambiguous' ? q.from.split('-')[0] : set.effort; + const row = { q, effort: set.effort, home }; jobs.push({ ...row, set: 'raw', query: q.question, scope: ['--all'] }); jobs.push({ ...row, @@ -354,7 +367,7 @@ async function main() { query: stripNames(q.question, titles), scope: ['--all'], }); - if (set.effort !== 'cross') + if (!MULTI_EFFORT_SETS.includes(set.effort)) jobs.push({ ...row, set: 'scoped', diff --git a/benchmarks/proof-recall/scripts/grade.mjs b/benchmarks/proof-recall/scripts/grade.mjs index b929907c..e84bf7f9 100644 --- a/benchmarks/proof-recall/scripts/grade.mjs +++ b/benchmarks/proof-recall/scripts/grade.mjs @@ -1,7 +1,11 @@ #!/usr/bin/env node // Scores one benchmark run. // -// node scripts/grade.mjs runs/.json +// node scripts/grade.mjs runs/.json [--only ambiguous] +// +// It scores the original 246 questions that the run answered. `--only` +// names the sets to score instead, as run.mjs takes them: `ambiguous` for +// the ambiguous-Effort set alone, `default,ambiguous` for both. // // A run file holds one entry per question id: // { "run": { "strategy": "brief", "model": "..." }, @@ -115,12 +119,15 @@ function summarize(rows, by) { } async function main() { - const runPath = process.argv[2]; - if (!runPath) throw new Error('usage: grade.mjs '); + const argv = process.argv.slice(2); + const at = argv.indexOf('--only'); + const only = at >= 0 ? argv.splice(at, 2)[1].split(',') : []; + const [runPath] = argv; + if (!runPath) throw new Error('usage: grade.mjs [--only sets]'); const run = JSON.parse(await readFile(resolve(runPath), 'utf8')); const graph = await loadGraph(join(BENCH, '.flatbread-proof')); const rows = []; - for (const set of await loadQuestions()) + for (const set of await loadQuestions(only)) for (const q of set.questions) { const entry = run.answers?.[q.id]; if (!entry) continue; diff --git a/benchmarks/proof-recall/scripts/graph.mjs b/benchmarks/proof-recall/scripts/graph.mjs index 4d27b758..7938417f 100644 --- a/benchmarks/proof-recall/scripts/graph.mjs +++ b/benchmarks/proof-recall/scripts/graph.mjs @@ -230,7 +230,9 @@ function compactionState(graph, effortId) { */ export function runCheck(graph, effort, check) { const keyOf = (rec) => graph.keyById.get(rec.id); - // Only Effort-wide checks need it; cross-Effort files name `check.effort`. + // Only Effort-wide checks need it. Files that span Efforts (`cross`, + // `ambiguous`) have no Effort of their own, so their checks name + // `check.effort` and qualify every key as `.`. const effortId = () => idFor(graph, check.effort ?? effort, 'effort'); switch (check.type) { case 'field': { @@ -292,9 +294,23 @@ export function runCheck(graph, effort, check) { return support && list(check.restsOn).includes(standing(support)); }) ); + if (check.invalidates) { + const target = idFor(graph, effort, check.invalidates); + recs = recs.filter((rec) => + list(rec.frontmatter.invalidates).includes(target) + ); + } + // Each text filter takes one phrase or a list; a record must contain + // every phrase, ignoring case. + const contains = (text, phrases) => + list(phrases).every((p) => + text.toLowerCase().includes(p.toLowerCase()) + ); if (check.bodyIncludes) + recs = recs.filter((rec) => contains(rec.body, check.bodyIncludes)); + if (check.titleIncludes) recs = recs.filter((rec) => - rec.body.toLowerCase().includes(check.bodyIncludes.toLowerCase()) + contains(String(rec.frontmatter.title ?? ''), check.titleIncludes) ); const keys = recs.map(keyOf).sort(); return check.count ? keys.length : keys; diff --git a/benchmarks/proof-recall/scripts/prepare-run.mjs b/benchmarks/proof-recall/scripts/prepare-run.mjs index 40a68886..c49f1830 100644 --- a/benchmarks/proof-recall/scripts/prepare-run.mjs +++ b/benchmarks/proof-recall/scripts/prepare-run.mjs @@ -10,7 +10,7 @@ import { cp, mkdir, rm, symlink, writeFile } from 'node:fs/promises'; import { dirname, join, resolve } from 'node:path'; import { fileURLToPath, pathToFileURL } from 'node:url'; -import { loadQuestions } from './check.mjs'; +import { pickQuestions } from './check.mjs'; const BENCH = resolve(dirname(fileURLToPath(import.meta.url)), '..'); // The form an answer takes, never its content. @@ -24,12 +24,14 @@ export const FORMATS = { }; const REPO = resolve(BENCH, '..', '..'); -/** Questions as an agent may see them: text and answer form only. */ +/** + * Questions as an agent may see them: text and answer form only. The + * default is the original 246; `only` names sets (`ambiguous`, or + * `default,ambiguous` for both), and `ids` finds questions in any set. + */ export async function publicQuestions({ only = [], ids = [] } = {}) { - const wanted = new Set(ids); - return (await loadQuestions(only)) + return (await pickQuestions({ only, ids })) .flatMap((set) => set.questions) - .filter((q) => !wanted.size || wanted.has(q.id)) .map((q) => ({ id: q.id, question: q.question, diff --git a/benchmarks/proof-recall/scripts/run.mjs b/benchmarks/proof-recall/scripts/run.mjs index 7afbbed1..e4157eba 100644 --- a/benchmarks/proof-recall/scripts/run.mjs +++ b/benchmarks/proof-recall/scripts/run.mjs @@ -9,7 +9,10 @@ // node scripts/run.mjs --stop-server [--server ] // // --strategies names the task texts in task.mjs to run; the default is -// brief,nobrief. Each session works in ///, which +// brief,nobrief. The default questions are the original 246. --only names +// question sets by their `effort` field: `--only ambiguous` runs the +// ambiguous-Effort set alone, and `--only default,ambiguous` runs both in +// one run. --ids picks questions from any set. Each session works in ///, which // holds a link to the shared copy of the graph, TASK.md, the agent's // answer.json, and transcript.jsonl. A rerun skips sessions that already // have both an answer and a transcript, so an interrupted run picks up where diff --git a/benchmarks/proof-recall/spec/_filler.mjs b/benchmarks/proof-recall/spec/_filler.mjs index 96e76ec3..65cb74aa 100644 --- a/benchmarks/proof-recall/spec/_filler.mjs +++ b/benchmarks/proof-recall/spec/_filler.mjs @@ -8,6 +8,16 @@ // the Effort's head compaction, which therefore covers it. A seeded random // generator makes the output identical on every build. // +// `build.mjs --filler-overlap ` makes filler adversarial. That +// share of filler threads and records take a real subsystem name from the +// Effort's own spec (Tern, VSM, saltglass refraction) in place of a made-up +// one. They still link to no real record, and they measure only routine +// things no question asks about, such as warning counts. Their numbers come +// from a second seeded generator, so the other filler records stay exactly +// as in a plain build. build.mjs redraws any overlap draft that collides +// with a question, so the output is fixed for a given fraction and a given +// set of question files; editing any question can change it. +// // The leading underscore keeps `loadSpecs` from reading this file as a spec. const PEOPLE = { @@ -108,6 +118,74 @@ const SUBSYSTEMS = { ], }; +// Real subsystem names that overlap filler reuses. Each must appear in its +// Effort's spec; `build.mjs` checks that. +const OVERLAP_SUBSYSTEMS = { + render: [ + 'Tern', + 'VSM', + 'saltglass refraction', + 'caustics', + 'SDF GI', + 'Brine curve', + 'visibility buffer', + 'meshlets', + ], + world: [ + 'tide simulation', + 'streaming pool', + 'Gull VM', + 'fibers', + 'Kelp', + 'delta saves', + 'archetype tables', + 'hot reload', + ], + ports: [ + 'pipeline cache', + 'dynamic resolution', + 'microSD', + 'fast travel', + 'autosave', + 'Kelp', + 'Tern', + ], + tools: [ + 'shader compile cache', + 'cache key', + 'remote execution', + 'nightly builds', + 'fetch timeout', + 'build farm', + ], + netcode: [ + 'rollback window', + 'host-authoritative sync', + 'lockstep', + 'desync', + 'input delay', + 'tide simulation', + ], +}; +// What overlap filler measures and asks: housekeeping that no question asks +// about. `build.mjs` refuses any of these that a spec or question uses. +const OVERLAP_METRICS = [ + ['warning count', '', 1, 60], + ['log lines per minute', '', 5, 900], + ['unit test time', 's', 2, 120], + ['config parse time', 'ms', 1, 90], + ['symbol upload time', 's', 3, 200], + ['console command count', '', 2, 80], +]; +const OVERLAP_QUESTIONS = [ + 'should log less', + 'needs a clearer error message', + 'should move its settings to the new config format', + 'should drop its old console commands', + 'needs its own unit test target', + 'should print fewer warnings', +]; + const PLATFORMS = ['PC', 'Meridian', 'Skiff']; const METRICS = [ ['load time', 'ms', 20, 900], @@ -174,11 +252,36 @@ export function fillerTerms(effortKey) { return SUBSYSTEMS[effortKey] ?? []; } +/** + * The words overlap filler uses, for build.mjs to check: `real` names must + * appear in the Effort's spec; `housekeeping` phrases must appear in no spec + * or question. + */ +export function overlapTerms(effortKey) { + return { + real: OVERLAP_SUBSYSTEMS[effortKey] ?? [], + housekeeping: [ + ...OVERLAP_METRICS.map(([metric]) => metric), + ...OVERLAP_QUESTIONS, + ], + }; +} + /** * Filler events for one spec, so the Effort ends with about `scale` times * its own records. Returns events with `at` and `key` like a spec's own. + * + * With `overlap` above 0, that share of filler threads and records reuse a + * real subsystem name (see the header). `collides(event)` is true when an + * overlap event would match a question's check or state one of its + * answers; such an item draws new words, and after five tries falls back to + * plain filler, so overlap filler never changes an answer. */ -export function fillerEvents(spec, scale) { +export function fillerEvents( + spec, + scale, + { overlap = 0, collides = () => false } = {} +) { if (scale <= 1) return []; const subsystems = SUBSYSTEMS[spec.key]; const people = PEOPLE[spec.key]; @@ -193,6 +296,41 @@ export function fillerEvents(spec, scale) { 6 * 3600e3; const rand = random(`filler:${spec.key}`); const pick = (list) => list[Math.floor(rand() * list.length)]; + // Overlap draws come from their own stream, so they never shift `rand`. + const overlapRand = random(`overlap:${spec.key}`); + const overlapPick = (list) => list[Math.floor(overlapRand() * list.length)]; + const realNames = OVERLAP_SUBSYSTEMS[spec.key] ?? []; + if (overlap > 0 && !realNames.length) + throw new Error(`No overlap vocabulary for ${spec.key}`); + const measure = (lo, hi, draw) => + +(lo + draw() * (hi - lo)).toFixed(lo < 1 ? 1 : 0); + /** + * Candidate words for one filler item (subsystem, metric, value, and + * question), best first. An item outside the overlap share gets only its + * plain words; one inside gets five drafts with real names and + * housekeeping metrics, then its plain words as the last resort. + */ + const overlapWords = (plain) => { + if (!(overlap > 0) || overlapRand() >= overlap) return [plain]; + const tries = []; + for (let i = 0; i < 5; i++) { + const [metric, unit, lo, hi] = overlapPick(OVERLAP_METRICS); + tries.push({ + sub: overlapPick(realNames), + metric, + unit, + value: measure(lo, hi, overlapRand), + question: overlapPick(OVERLAP_QUESTIONS), + }); + } + return [...tries, plain]; + }; + const firstClean = (options, render) => + options.find( + (words, i) => + i === options.length - 1 || + !render(words).some((event) => collides(event)) + ); const at = (ms) => new Date(Math.floor(ms / 60e3) * 60e3).toISOString(); const events = []; let n = 0; @@ -209,52 +347,59 @@ export function fillerEvents(spec, scale) { // A small thread: a question, the measurement that answers it, and the // accepted Decision that closes it. Its links stay inside the thread. const question = pick(QUESTIONS); + const [metric, unit, lo, hi] = pick(METRICS); + const value = measure(lo, hi, rand); + const verdict = pick(VERDICTS); + const doIt = rand() < 0.5; + const cheap = rand() < 0.5; + const followup = pick(FOLLOWUPS); const issue = `fill-iss-${id}`; const finding = `fill-fnd-${id}`; const decision = `fill-dec-${id}`; - const [metric, unit, lo, hi] = pick(METRICS); - const value = +(lo + rand() * (hi - lo)).toFixed(lo < 1 ? 1 : 0); + const render = (w) => { + const amount = `${w.value}${w.unit ? ` ${w.unit}` : ''}`; + return [ + { + at: at(time), + key: issue, + type: 'WriteIssue', + kind: 'question', + title: `Whether the ${w.sub} ${w.question} (${id})`, + body: `${person} asked whether the ${w.sub} ${w.question}. It came up while looking at build ${build} on ${platform}.`, + created_by: person, + }, + { + at: at(time + 3600e3), + key: finding, + type: 'WriteFinding', + kind: 'measurement', + title: `${capitalize(w.sub)} ${ + w.metric + } is ${amount} on ${platform} (${id})`, + body: `${person} measured the ${w.sub} ${w.metric} on ${platform}, build ${build}: ${amount}. That is ${verdict}.`, + derives_from: [issue], + created_by: person, + }, + { + at: at(time + 2 * 3600e3), + key: decision, + type: 'WriteDecision', + title: `${capitalize(w.sub)}: ${ + doIt ? 'do it' : 'leave it as is' + } (${id})`, + body: `The ${w.sub} ${ + w.metric + } is ${amount} on ${platform}, so the change is ${ + cheap ? 'cheap enough to make now' : 'not worth making this year' + }. ${followup}`, + derives_from: [issue, finding], + created_by: person, + }, + ]; + }; + const plain = { sub, question, metric, unit, value }; events.push( - { - at: at(time), - key: issue, - type: 'WriteIssue', - kind: 'question', - title: `Whether the ${sub} ${question} (${id})`, - body: `${person} asked whether the ${sub} ${question}. It came up while looking at build ${build} on ${platform}.`, - created_by: person, - }, - { - at: at(time + 3600e3), - key: finding, - type: 'WriteFinding', - kind: 'measurement', - title: `${capitalize(sub)} ${metric} is ${value}${ - unit ? ` ${unit}` : '' - } on ${platform} (${id})`, - body: `${person} measured the ${sub} ${metric} on ${platform}, build ${build}: ${value}${ - unit ? ` ${unit}` : '' - }. That is ${pick(VERDICTS)}.`, - derives_from: [issue], - created_by: person, - }, - { - at: at(time + 2 * 3600e3), - key: decision, - type: 'WriteDecision', - title: `${capitalize(sub)}: ${ - rand() < 0.5 ? 'do it' : 'leave it as is' - } (${id})`, - body: `The ${sub} ${metric} is ${value}${ - unit ? ` ${unit}` : '' - } on ${platform}, so the change is ${ - rand() < 0.5 - ? 'cheap enough to make now' - : 'not worth making this year' - }. ${pick(FOLLOWUPS)}`, - derives_from: [issue, finding], - created_by: person, - }, + ...render(firstClean(overlapWords(plain), render)), { at: at(time + 3 * 3600e3), type: 'AcceptDecision', @@ -270,36 +415,47 @@ export function fillerEvents(spec, scale) { ); budget -= 3; } else if (roll < 0.4) { - events.push({ - at: at(time), - key: `fill-cit-${id}`, - type: 'WriteCitation', - title: `${capitalize(sub)} capture, build ${build} (${id})`, - body: `https://perf.tidewater.example/${spec.key}/${sub.replace( - /\W+/g, - '-' - )}/${build}`, - role: 'evidence', - created_by: person, - }); + const render = (w) => [ + { + at: at(time), + key: `fill-cit-${id}`, + type: 'WriteCitation', + title: `${capitalize(w.sub)} capture, build ${build} (${id})`, + body: `https://perf.tidewater.example/${spec.key}/${w.sub.replace( + /\W+/g, + '-' + )}/${build}`, + role: 'evidence', + created_by: person, + }, + ]; + events.push(...render(firstClean(overlapWords({ sub }), render))); budget -= 1; } else { const [metric, unit, lo, hi] = pick(METRICS); - const value = +(lo + rand() * (hi - lo)).toFixed(lo < 1 ? 1 : 0); + const value = measure(lo, hi, rand); const runs = 3 + Math.floor(rand() * 8); - events.push({ - at: at(time), - key: `fill-fnd-${id}`, - type: 'WriteFinding', - kind: 'measurement', - title: `${capitalize( - sub - )} ${metric} on ${platform}, build ${build} (${id})`, - body: `${person} measured the ${sub} ${metric} on ${platform}, build ${build}, over ${runs} runs: ${value}${ - unit ? ` ${unit}` : '' - }. That is ${pick(VERDICTS)}. ${pick(FOLLOWUPS)}`, - created_by: person, - }); + const verdict = pick(VERDICTS); + const followup = pick(FOLLOWUPS); + const render = (w) => [ + { + at: at(time), + key: `fill-fnd-${id}`, + type: 'WriteFinding', + kind: 'measurement', + title: `${capitalize(w.sub)} ${ + w.metric + } on ${platform}, build ${build} (${id})`, + body: `${person} measured the ${w.sub} ${ + w.metric + } on ${platform}, build ${build}, over ${runs} runs: ${w.value}${ + w.unit ? ` ${w.unit}` : '' + }. That is ${verdict}. ${followup}`, + created_by: person, + }, + ]; + const plain = { sub, metric, unit, value }; + events.push(...render(firstClean(overlapWords(plain), render))); budget -= 1; } }