diff --git a/benchmarks/proof-recall/README.md b/benchmarks/proof-recall/README.md index f784bea2..b693cd58 100644 --- a/benchmarks/proof-recall/README.md +++ b/benchmarks/proof-recall/README.md @@ -11,22 +11,23 @@ Each question has a known answer and a stated path through the graph. ## Layout -| Path | What it holds | -| ------------------------- | -------------------------------------------------------- | -| `WORLD.md` | The shared facts every Effort must agree on | -| `spec/.mjs` | One Effort's history, as an ordered list of mutations | -| `questions/.json` | That Effort's questions and answers | -| `questions/cross.json` | Questions that need two or more Efforts | -| `scripts/build.mjs` | Replays every spec through the Proof writer | -| `scripts/check.mjs` | Proves each answer against the built graph | -| `scripts/grade.mjs` | Scores one run | -| `scripts/run.mjs` | Runs one headless agent session per question | -| `scripts/task.mjs` | The task text each session reads | -| `scripts/compare.mjs` | Compares two runs question by question | -| `scripts/find-recall.mjs` | Checks offline whether `proof find` reaches the evidence | -| `runs/` | Answers and costs from each pilot and full run | -| `.flatbread-proof/` | The built graph, committed so runs need no build | -| `flatbread.config.js` | Lets `flatbread proof …` read the graph from this folder | +| Path | What it holds | +| -------------------------- | -------------------------------------------------------- | +| `WORLD.md` | The shared facts every Effort must agree on | +| `spec/.mjs` | One Effort's history, as an ordered list of mutations | +| `questions/.json` | That Effort's questions and answers | +| `questions/cross.json` | Questions that need two or more Efforts | +| `questions/ambiguous.json` | Questions that name no Effort, asked only on request | +| `scripts/build.mjs` | Replays every spec through the Proof writer | +| `scripts/check.mjs` | Proves each answer against the built graph | +| `scripts/grade.mjs` | Scores one run | +| `scripts/run.mjs` | Runs one headless agent session per question | +| `scripts/task.mjs` | The task text each session reads | +| `scripts/compare.mjs` | Compares two runs question by question | +| `scripts/find-recall.mjs` | Checks offline whether `proof find` reaches the evidence | +| `runs/` | Answers and costs from each pilot and full run | +| `.flatbread-proof/` | The built graph, committed so runs need no build | +| `flatbread.config.js` | Lets `flatbread proof …` read the graph from this folder | ## Build and check @@ -160,6 +161,8 @@ Each question file holds one Effort's questions: | `checks` | Machine checks that compute or confirm the answer | | `traps` | The wrong answers a careless reader gives, and why | | `scoped` | `true` when wording other than the title pins the Effort down | +| `from` | In `ambiguous.json` only: the question it was rewritten from | +| `rivals` | In `ambiguous.json` only: near-miss records in other Efforts | ### Answer kinds @@ -210,17 +213,21 @@ A check without `expect` must compute the answer itself. A `select` scans the qu `expect` must return exactly that value. `record`, `records`, and `count` answers need at least one computing check. -| `type` | Fields | Returns | -| ------------ | -------------------------------------------------------------------------------------------------- | -------------------------------------------------- | -| `field` | `record`, `field` (or `standing`) | that frontmatter value | -| `head` | `start` | the newest version's key | -| `chain` | `start` | every version's key, oldest first | -| `asof` | `start`, `at` | the version current at `at` | -| `select` | `kinds`, `where`, `live`, `retracted`, `derivesFrom`, `restsOn`, `bodyIncludes`, `count`, `effort` | matching keys, or their count | -| `efforts` | `where`, `count` | matching Efforts' keys, or their count | -| `compaction` | `field`: `head`, `heads`, `stale`, `changes`, `changed`, `staleTargets` | the Effort's compaction state | -| `body` | `record`, `includes`, `afterExcerpt` | whether the text is present (and past the excerpt) | -| `edge` | `edge` | whether the edge exists | +| `type` | Fields | Returns | +| ------------ | ---------------------------------------------------------------------------------------------------------------------------------- | -------------------------------------------------- | +| `field` | `record`, `field` (or `standing`) | that frontmatter value | +| `head` | `start` | the newest version's key | +| `chain` | `start` | every version's key, oldest first | +| `asof` | `start`, `at` | the version current at `at` | +| `select` | `kinds`, `where`, `live`, `retracted`, `derivesFrom`, `restsOn`, `invalidates`, `bodyIncludes`, `titleIncludes`, `count`, `effort` | matching keys, or their count | +| `efforts` | `where`, `count` | matching Efforts' keys, or their count | +| `compaction` | `field`: `head`, `heads`, `stale`, `changes`, `changed`, `staleTargets` | the Effort's compaction state | +| `body` | `record`, `includes`, `afterExcerpt` | whether the text is present (and past the excerpt) | +| `edge` | `edge` | whether the edge exists | + +`bodyIncludes` and `titleIncludes` take one phrase or a list, and a record +must contain every phrase. `invalidates` keeps records that invalidate the +named record. A record's **standing** is its state or status, or `live`, overridden by `retracted`, `superseded`, or `invalidated`. `select` with `live: true` keeps @@ -330,6 +337,41 @@ A benchmark agent must never see `spec/`, `questions/`, or `keymap.json`. with more right answers over the runs, and is the stricter reading. The script warns when the run files record different `find_hash` values. +**Ambiguous-Effort questions.** `questions/ambiguous.json` holds 24 +questions rewritten from tier 0-2 questions so that they name no Effort and +use words several Efforts share, such as "shader cache", "the +retrospective", "refraction", or "Skiff". Each names its source question in +`from`. Each lists in `rivals` at least one record from another Effort that +the wording also fits, and a trap says why that record is wrong. Checks over +every Effort (`effort: "*"`) prove that only one record or value fits. +`check.mjs` also refuses a question that names an Effort or one of its code +names, or uses a platform word that mostly one Effort's records use, such +as "handheld". It cannot prove that every other word in a question is +shared, so a reader who knows the subject may still guess the Effort. + +No script asks these questions by default, so every default stays on the +original 246: pass `--only ambiguous` to `run.mjs`, `grade.mjs`, +`compare.mjs`, or `find-recall.mjs`, or `--only default,ambiguous` for both +sets in one run, or name their ids with `--ids`. `check.mjs` always checks +them. + +**Adversarial filler.** This build makes 30% of the filler reuse real +subsystem names from the specs, such as Tern, VSM, and saltglass refraction: + +```bash +node scripts/build.mjs --scale 10 --filler-overlap 0.3 --out +``` + +These records still link to no real record and measure only housekeeping, +such as warning counts. The builder redraws any that would match a +question's check or state one of its answers, so `check.mjs --graph ` +gives the same result as on the plain 10x graph. Filler outside that 30% is +the same as in a plain build. The overlap records are the same on every +build with the same fraction and the same question files. The redraws read +every file in `questions/`, so editing any question can change them. The +build prints a hash of those files; rebuild an adversarial graph after any +question edit before you compare runs on it. + Three rules shaped this protocol, each learned from a pilot below: - **One session per question.** A session that answers many questions @@ -704,3 +746,24 @@ The largest reply grew by about 700 characters, to 7,360. That is close to 7,500 characters, the most this step allowed for the mean reply. A later change that adds passage lines should watch the largest reply, not only the mean. + +#### Questions that name no Effort, and filler that looks real + +The 24 ambiguous-Effort questions were frozen before `find` was run on +them. With the cards above (same `find_hash`), the stripped set on the 10x +graph gives: + +| Graph | Questions | Evidence in top 5 | Answer on a card | Right Effort first | Mean reply chars | +| --------------------------- | --------- | ----------------- | ---------------- | ------------------ | ---------------- | +| 10x | 24 | 96% | 87% | 88% | 6,448 | +| 10x, `--filler-overlap 0.3` | 24 | 96% | 87% | 92% | 6,433 | +| 10x, `--filler-overlap 0.3` | 246 | 80% | 62% (gate PASS) | 80% | 6,326 | +| 10x (from the table above) | 246 | 81% | 62% | 81% | 6,368 | + +Two things follow. First, search finds the right record even when the +question names no Effort. These questions were rewritten from tier 0-2 +lookups, though, so they are easier than the full set, and they test +`find` more than they test an agent. Second, filler that reuses real +subsystem names cost the gate one point of evidence and no answers, so the +10x result is not only a sign that filler is easy to tell apart. The paid +runs (Runs A to C in the plan) still have to show that agents gain from it. diff --git a/benchmarks/proof-recall/questions/ambiguous.json b/benchmarks/proof-recall/questions/ambiguous.json new file mode 100644 index 00000000..a545d92e --- /dev/null +++ b/benchmarks/proof-recall/questions/ambiguous.json @@ -0,0 +1,1073 @@ +{ + "effort": "ambiguous", + "questions": [ + { + "id": "ambiguous-001", + "from": "ports-002", + "tier": 0, + "hops": 0, + "categories": ["ambiguous-effort", "truncation", "lookup"], + "question": "Before I shrink the shader cache on Skiff, how large is it allowed to grow on the device's storage right now?", + "answer": { + "kind": "value", + "value": "384 MB", + "accept": ["384", "384 megabytes"] + }, + "evidence": ["ports.dec-pso-ondevice"], + "checks": [ + { + "type": "select", + "effort": "*", + "live": true, + "bodyIncludes": ["cache", "capped at"], + "expect": ["ports.dec-pso-ondevice"] + }, + { + "type": "body", + "record": "ports.dec-pso-ondevice", + "includes": "capped at 384 MB", + "afterExcerpt": true, + "expect": true + } + ], + "traps": [ + "212 MB is how much the cache held after a full playthrough, not its cap.", + "140 MB is the size of the prebuilt blobs the cache replaced.", + "\"Shader cache\" also names the build farm's compile cache. Its storage Risk is about a 4 TB farm volume, and that cache never ships to players." + ], + "rivals": ["tools.rsk-storage"] + }, + { + "id": "ambiguous-002", + "from": "ports-011", + "tier": 1, + "hops": 1, + "categories": ["ambiguous-effort", "distractor", "retraction"], + "question": "When a player comes back to an area they have already visited, what share of lookups hit the shader cache?", + "answer": { + "kind": "value", + "value": "99.6 percent", + "accept": ["99.6%", "99.6"] + }, + "evidence": ["ports.dec-pso-ondevice", "ports.fnd-cache-warm"], + "path": [["ports.dec-pso-ondevice", "supports", "ports.fnd-cache-warm"]], + "checks": [ + { + "type": "select", + "effort": "*", + "live": true, + "bodyIncludes": "second visit", + "expect": ["ports.fnd-cache-warm"] + }, + { + "type": "body", + "record": "ports.fnd-cache-warm", + "includes": "99.6 percent of pipeline lookups hit the cache", + "expect": true + } + ], + "traps": [ + "97 percent is the build farm's shader compile cache hit rate, in a retracted Finding posted here by mistake.", + "The build farm's shader cache reports hit rates too (94% steady state, 92.4% in week four), but they count shader permutations in farm builds, not lookups on a player's device." + ], + "rivals": ["tools.fnd-hitrate"] + }, + { + "id": "ambiguous-003", + "from": "tools-009", + "tier": 2, + "hops": 2, + "categories": ["ambiguous-effort", "blob"], + "question": "The shader cache warmed up week by week after the key change. What hit rate did it reach in week four?", + "answer": { + "kind": "value", + "value": "92.4%", + "accept": ["92.4", "92.4 percent"] + }, + "evidence": [ + "tools.fnd-hitrate", + "tools.cit-hitrate", + "tools.blb-hitrate" + ], + "path": [ + ["tools.fnd-hitrate", "cites", "tools.cit-hitrate"], + ["tools.cit-hitrate", "blob", "tools.blb-hitrate"] + ], + "checks": [ + { + "type": "select", + "effort": "*", + "kinds": ["blob"], + "bodyIncludes": ["hit_rate", "week"], + "expect": ["tools.blb-hitrate"] + }, + { + "type": "body", + "record": "tools.blb-hitrate", + "includes": "\"hit_rate\": 92.4", + "expect": true + } + ], + "traps": [ + "The Finding body gives only week 1 (71%).", + "The retrospective's 94% is the later steady-state rate, and the February summary says only \"low 90s\".", + "Blob bodies are left out of list and records digests; only `proof get` on the Blob shows the series.", + "Two SDF GI capture payloads also carry a hit-rate field, for the GPU L2 cache, with no weeks.", + "The on-device pipeline cache is also called the shader cache, and its key includes the driver version, but it reports one second-visit figure (99.6 percent), not weekly ones." + ], + "rivals": ["ports.fnd-cache-warm"] + }, + { + "id": "ambiguous-004", + "from": "tools-003", + "tier": 0, + "hops": 0, + "categories": ["ambiguous-effort", "truncation", "lookup"], + "question": "Going by the retrospective, how long does a clean rebuild take now that the cache is warm?", + "answer": { + "kind": "value", + "value": "6 min 40 s", + "accept": ["6 minutes 40 seconds", "6:40", "400 seconds", "6m40s"] + }, + "evidence": ["tools.fnd-retro"], + "checks": [ + { + "type": "select", + "effort": "*", + "kinds": ["finding"], + "live": true, + "titleIncludes": "retrospective", + "expect": ["ports.fnd-retro", "tools.fnd-retro"] + }, + { + "type": "select", + "effort": "*", + "kinds": ["finding"], + "live": true, + "titleIncludes": "retrospective", + "bodyIncludes": "clean rebuild", + "expect": ["tools.fnd-retro"] + }, + { + "type": "body", + "record": "tools.fnd-retro", + "includes": "6 min 40 s", + "afterExcerpt": true, + "expect": true + } + ], + "traps": [ + "The figure sits at the end of a long body, past the 600-character excerpt that list and records digests show.", + "The hit-rate report's 7.2-minute median build time is a different measure.", + "There are two retrospectives. The Skiff beta one says nothing about rebuild times." + ], + "rivals": ["ports.fnd-retro"] + }, + { + "id": "ambiguous-005", + "from": "ports-034", + "tier": 0, + "hops": 0, + "categories": ["ambiguous-effort", "truncation", "lookup"], + "question": "The retrospective puts a schedule cost on the wrong pipeline choice. How many weeks did it cost?", + "answer": { + "kind": "value", + "value": "6 weeks", + "accept": ["6", "six weeks"] + }, + "evidence": ["ports.fnd-retro"], + "checks": [ + { + "type": "select", + "effort": "*", + "kinds": ["finding"], + "live": true, + "titleIncludes": "retrospective", + "expect": ["ports.fnd-retro", "tools.fnd-retro"] + }, + { + "type": "select", + "effort": "*", + "kinds": ["finding"], + "live": true, + "titleIncludes": "retrospective", + "bodyIncludes": "pipeline choice", + "expect": ["ports.fnd-retro"] + }, + { + "type": "body", + "record": "ports.fnd-retro", + "includes": "cost 6 weeks of schedule", + "afterExcerpt": true, + "expect": true + } + ], + "traps": [ + "The Kelp switch took one week; that figure sits earlier in the same record.", + "The shader compile cache retrospective says the team lost about three weeks blaming the network for flaky nightly builds. That was a wrong diagnosis, not a pipeline choice." + ], + "rivals": ["tools.fnd-retro"] + }, + { + "id": "ambiguous-006", + "from": "render-013", + "tier": 0, + "hops": 0, + "categories": ["ambiguous-effort", "truncation", "lookup"], + "question": "I want to add a channel to the history the upscaler keeps between frames. How precisely is each channel stored today?", + "answer": { + "kind": "value", + "value": "16-bit float", + "accept": ["FP16", "16-bit float per channel", "half float"] + }, + "evidence": ["render.dec-tern"], + "checks": [ + { + "type": "select", + "effort": "*", + "live": true, + "bodyIncludes": "history buffer", + "expect": ["render.dec-tern"] + }, + { + "type": "body", + "record": "render.dec-tern", + "includes": "stored as 16-bit float per channel", + "afterExcerpt": true, + "expect": true + } + ], + "traps": [ + "FP16 packing of reprojection inputs is a different change, in a Tern cost Finding.", + "The Skiff upscaler Decision picks a Tern preset but says nothing about the history buffer; every preset shares one format." + ], + "rivals": ["ports.dec-tern-skiff"] + }, + { + "id": "ambiguous-007", + "from": "render-009", + "tier": 0, + "hops": 0, + "categories": ["ambiguous-effort", "truncation", "current-state"], + "question": "Someone wants the refraction fallback to take more steps. What step cap is in force now?", + "answer": { + "kind": "value", + "value": "32", + "accept": ["32 steps"] + }, + "evidence": ["render.dec-refr-hybrid"], + "checks": [ + { + "type": "select", + "effort": "*", + "kinds": ["decision"], + "live": true, + "bodyIncludes": ["refraction", "capped at"], + "expect": ["render.dec-refr-hybrid"] + }, + { + "type": "select", + "effort": "*", + "kinds": ["decision"], + "live": true, + "bodyIncludes": "capped at", + "expect": ["ports.dec-pso-ondevice", "render.dec-refr-hybrid"] + }, + { + "type": "body", + "record": "render.dec-refr-hybrid", + "includes": "capped at 32 steps", + "afterExcerpt": true, + "expect": true + } + ], + "traps": [ + "64 steps is what the October cost Finding and the rejected full ray-march used.", + "The only other live Decision that caps something is the Skiff pipeline cache, at 384 MB.", + "A rejected Skiff Decision would have dropped saltglass refraction to half resolution there. That limits pixels, not steps." + ], + "rivals": ["ports.dec-pso-ondevice", "ports.dec-skiff-native"] + }, + { + "id": "ambiguous-008", + "from": "netcode-006", + "tier": 1, + "hops": 1, + "categories": ["ambiguous-effort", "rejected-alternative"], + "question": "Someone wants co-op to run deterministically, with each peer waiting for every other peer's input before it simulates a frame. Which Decision turned that approach down?", + "answer": { + "kind": "record", + "value": "netcode.dec-rollback" + }, + "evidence": ["netcode.dec-lockstep", "netcode.dec-rollback"], + "path": [["netcode.dec-lockstep", "rejected_by", "netcode.dec-rollback"]], + "checks": [ + { + "type": "select", + "effort": "*", + "kinds": ["decision"], + "titleIncludes": "lockstep", + "expect": ["netcode.dec-lockstep"] + }, + { + "type": "select", + "effort": "*", + "kinds": ["decision"], + "where": { + "standing": "superseded" + }, + "derivesFrom": "netcode.iss-sync" + }, + { + "type": "edge", + "edge": [ + "netcode.dec-lockstep", + "rejected_by", + "netcode.dec-rollback" + ], + "expect": true + }, + { + "type": "select", + "effort": "*", + "kinds": ["decision"], + "bodyIncludes": "exchange only inputs", + "expect": ["world.dec-tide-fixed"] + } + ], + "traps": [ + "The host-authoritative Decision is today's answer to the sync question, but it rejected nothing; lockstep was already rejected.", + "The lockstep playtest Finding explains the loss but is not a Decision.", + "The fixed-point tide Decision keeps tides deterministic and says co-op peers exchange only inputs. But it is accepted, and what it rejected was the host streaming tide heights, not this approach." + ], + "rivals": ["world.dec-tide-fixed"] + }, + { + "id": "ambiguous-009", + "from": "world-015", + "tier": 0, + "hops": 0, + "categories": ["ambiguous-effort", "lookup"], + "question": "Gameplay scripts keep running long on Skiff. How many milliseconds per frame do all scripts together get there?", + "answer": { + "kind": "value", + "value": "3.0 ms", + "accept": ["3 ms", "3.0", "3"] + }, + "evidence": ["world.con-gull-tick"], + "checks": [ + { + "type": "select", + "effort": "*", + "kinds": ["constraint"], + "live": true, + "bodyIncludes": "script", + "expect": ["world.con-gull-tick"] + }, + { + "type": "body", + "record": "world.con-gull-tick", + "includes": "3.0 ms per frame on Skiff", + "expect": true + } + ], + "traps": [ + "2.0 ms is Meridian's budget.", + "25 ms is the whole Skiff 40 Hz frame, not the script share." + ], + "rivals": ["ports.con-frame-2"] + }, + { + "id": "ambiguous-010", + "from": "world-027", + "tier": 1, + "hops": 1, + "categories": ["ambiguous-effort", "current-state", "distractor"], + "question": "We want to stream more data per second on Skiff. What sustained read rate does the current streaming budget allow?", + "answer": { + "kind": "value", + "value": "160 MB/s", + "accept": ["160"] + }, + "evidence": ["world.con-io-budget", "world.con-io-budget-2"], + "path": [ + ["world.con-io-budget", "superseded_by", "world.con-io-budget-2"] + ], + "checks": [ + { + "type": "select", + "effort": "*", + "kinds": ["constraint"], + "live": true, + "bodyIncludes": "MB/s", + "expect": ["world.con-io-budget-2"] + }, + { + "type": "body", + "record": "world.con-io-budget-2", + "includes": "under 160 MB/s", + "expect": true + } + ], + "traps": [ + "220 MB/s is the superseded December budget.", + "The streaming pool budget has a near-identical title but limits memory, not reads.", + "61 MB/s is the worst sustained read measured on budget microSD cards, a Finding, not a budget." + ], + "rivals": ["ports.fnd-sd-speed-2"] + }, + { + "id": "ambiguous-011", + "from": "world-007", + "tier": 0, + "hops": 0, + "categories": ["ambiguous-effort", "truncation", "lookup"], + "question": "When the 400 MB memory cut hit the pool, how much of it came back from texture mips?", + "answer": { + "kind": "value", + "value": "260 MB", + "accept": ["260"] + }, + "evidence": ["world.con-pool-2"], + "checks": [ + { + "type": "select", + "effort": "*", + "bodyIncludes": ["400 MB", "texture mip"], + "expect": ["world.con-pool-2"] + }, + { + "type": "body", + "record": "world.con-pool-2", + "includes": "texture mips give back 260 MB", + "afterExcerpt": true, + "expect": true + } + ], + "traps": [ + "The split is in the last paragraph of a superseded constraint; the current constraint does not repeat it.", + "320 MB is the tide-band reservation, not part of the cut.", + "About 900 MB is what Kelp's high-ratio texture mode saved on the Skiff harbor route, a different cut." + ], + "rivals": ["ports.dec-kelp-skiff"] + }, + { + "id": "ambiguous-012", + "from": "world-005", + "tier": 0, + "hops": 0, + "categories": ["ambiguous-effort", "truncation", "lookup"], + "question": "Players say that something they dug up at low tide moves after they save and load the game again. In the worst case, how far inland did it end up?", + "answer": { + "kind": "value", + "value": "11 m", + "accept": ["11 metres", "11 meters", "11"] + }, + "evidence": ["world.fnd-save-tide-mismatch"], + "checks": [ + { + "type": "select", + "effort": "*", + "bodyIncludes": ["low tide", "inland"], + "expect": ["world.fnd-save-tide-mismatch"] + }, + { + "type": "body", + "record": "world.fnd-save-tide-mismatch", + "includes": "11 m inland", + "afterExcerpt": true, + "expect": true + } + ], + "traps": [ + "The distance is in the final paragraph, past the browse excerpt; 31 of 50 is the repro count.", + "The Skiff delta-save Decision is about how much autosave writes, not where chests end up." + ], + "rivals": ["ports.dec-delta-save"] + }, + { + "id": "ambiguous-013", + "from": "world-041", + "tier": 2, + "hops": 2, + "categories": ["ambiguous-effort", "blob"], + "question": "We're tuning what streams first when the tide falls. In the Skiff trace with 256 m cells, what share of the reads were textures?", + "answer": { + "kind": "value", + "value": "61%", + "accept": ["0.61", "61 percent"] + }, + "evidence": [ + "world.fnd-tide-burst", + "world.cit-trace-tide-turn", + "world.blob-trace-tide-turn" + ], + "path": [ + ["world.fnd-tide-burst", "cites", "world.cit-trace-tide-turn"], + ["world.cit-trace-tide-turn", "blob", "world.blob-trace-tide-turn"] + ], + "checks": [ + { + "type": "select", + "effort": "*", + "kinds": ["blob"], + "bodyIncludes": "\"cell_size_m\": 256", + "expect": ["world.blob-trace-tide-turn"] + }, + { + "type": "body", + "record": "world.blob-trace-tide-turn", + "includes": "\"textures\": 0.61", + "expect": true + } + ], + "traps": [ + "30% is the share of a cell that changes when the tide falls, not a read share.", + "The Skiff harbor memory capture also puts textures first, but in megabytes of memory, not as a share of reads." + ], + "rivals": ["ports.blb-mem-capture"] + }, + { + "id": "ambiguous-014", + "from": "ports-015", + "tier": 2, + "hops": 2, + "categories": ["ambiguous-effort", "blob", "distractor"], + "question": "We need to cut memory in the harbor on Skiff. Which category was largest in the capture that showed its peak going over the memory budget?", + "answer": { + "kind": "value", + "value": "textures", + "accept": ["texture", "textures (3184 MB)"] + }, + "evidence": [ + "ports.fnd-mem-capture", + "ports.cit-mem-capture", + "ports.blb-mem-capture" + ], + "path": [ + ["ports.fnd-mem-capture", "cites", "ports.cit-mem-capture"], + ["ports.cit-mem-capture", "blob", "ports.blb-mem-capture"] + ], + "checks": [ + { + "type": "select", + "effort": "*", + "kinds": ["blob"], + "bodyIncludes": "categories_mb", + "expect": ["ports.blb-mem-capture"] + }, + { + "type": "body", + "record": "ports.blb-mem-capture", + "includes": "\"textures\": 3184", + "expect": true + } + ], + "traps": [ + "The Finding says audio banks grew the most; growth is not size.", + "The streaming pool is the second largest at 2048 MB.", + "The streaming pool occupancy capture also records a memory peak, but only for the pool, and it does not split memory into categories." + ], + "rivals": ["world.blob-pool-occupancy"] + }, + { + "id": "ambiguous-015", + "from": "netcode-003", + "tier": 0, + "hops": 0, + "categories": ["ambiguous-effort", "lookup", "retraction", "distractor"], + "question": "I'm adding another component to the snapshot. How big is one player's snapshot for a single frame right now?", + "answer": { + "kind": "value", + "value": "38 KB", + "accept": ["38", "38 kilobytes"] + }, + "evidence": ["netcode.fnd-snapshot", "netcode.fnd-noise"], + "checks": [ + { + "type": "select", + "effort": "*", + "live": true, + "bodyIncludes": ["snapshot", "per frame"], + "expect": ["netcode.fnd-snapshot"] + }, + { + "type": "body", + "record": "netcode.fnd-snapshot", + "includes": "38 KB per frame", + "expect": true + }, + { + "type": "field", + "record": "netcode.fnd-noise", + "field": "standing", + "expect": "retracted" + } + ], + "traps": [ + "A retracted Finding gives 14 KB per player; it measured save-game snapshots and was filed here by mistake.", + "About 600 KB is the whole 4-frame ring for four players, not one player's frame.", + "A full world snapshot save is 310 MB. That is a save file, not a per-frame snapshot." + ], + "rivals": ["world.fnd-save-snapshot-size"] + }, + { + "id": "ambiguous-016", + "from": "tools-007", + "tier": 2, + "hops": 2, + "categories": ["ambiguous-effort", "dead-evidence", "invalidation"], + "question": "Before I touch the nightly build settings: which accepted Decision still rests on a diagnosis of the flaky nightlies that turned out to be wrong?", + "answer": { + "kind": "record", + "value": "tools.dec-timeout" + }, + "evidence": [ + "tools.dec-timeout", + "tools.fnd-flaky-net", + "tools.fnd-flaky-oom" + ], + "path": [ + ["tools.dec-timeout", "derives_from", "tools.fnd-flaky-net"], + ["tools.fnd-flaky-net", "invalidated_by", "tools.fnd-flaky-oom"] + ], + "checks": [ + { + "type": "select", + "effort": "*", + "kinds": ["finding"], + "where": { + "standing": "invalidated" + }, + "titleIncludes": "nightly", + "expect": ["tools.fnd-flaky-net"] + }, + { + "type": "select", + "effort": "*", + "kinds": ["decision"], + "where": { + "standing": "accepted" + }, + "derivesFrom": "tools.fnd-flaky-net" + } + ], + "traps": [ + "The job cap is the real fix for the flaky builds, but it rests on the correct memory diagnosis.", + "The superseded cache keys rest on superseded or live records, not on a disproved Finding.", + "Other Efforts hold accepted Decisions that rest on dead evidence too, but none rests on a nightly-build diagnosis." + ], + "rivals": ["ports.dec-stream-block"] + }, + { + "id": "ambiguous-017", + "from": "ports-017", + "tier": 2, + "hops": 2, + "categories": ["ambiguous-effort", "dead-evidence", "invalidation"], + "question": "We chose the size of our tile reads on Skiff from a storage speed measurement. Which Finding showed that measurement was wrong?", + "answer": { + "kind": "record", + "value": "ports.fnd-sd-speed-2" + }, + "evidence": [ + "ports.dec-stream-block", + "ports.fnd-sd-speed", + "ports.fnd-sd-speed-2" + ], + "path": [ + ["ports.dec-stream-block", "derives_from", "ports.fnd-sd-speed"], + ["ports.fnd-sd-speed", "invalidated_by", "ports.fnd-sd-speed-2"] + ], + "checks": [ + { + "type": "select", + "effort": "*", + "kinds": ["decision"], + "live": true, + "bodyIncludes": "in 2 MB blocks", + "expect": ["ports.dec-stream-block"] + }, + { + "type": "select", + "effort": "*", + "kinds": ["decision"], + "live": true, + "restsOn": "invalidated", + "bodyIncludes": "read", + "expect": ["ports.dec-stream-block", "world.dec-save-version"] + }, + { + "type": "edge", + "edge": [ + "ports.dec-stream-block", + "derives_from", + "ports.fnd-sd-speed" + ], + "expect": true + }, + { + "type": "select", + "effort": "*", + "kinds": ["finding"], + "invalidates": "ports.fnd-sd-speed" + } + ], + "traps": [ + "The 2 MB read Decision is still accepted; nothing invalidated it directly, so reading only its state hides the problem.", + "A retail-unit trace also showed that a dev-kit streaming read figure was wrong, but the 2 MB read Decision does not rest on that figure.", + "The save version Decision also rests on a disproved Finding, but that Finding timed save upgrades, not storage speed." + ], + "rivals": ["world.fnd-io-trace-2", "world.dec-save-version"] + }, + { + "id": "ambiguous-018", + "from": "world-026", + "tier": 1, + "hops": 1, + "categories": ["ambiguous-effort", "invalidation", "distractor"], + "question": "Our December measurement of sustained streaming reads on Skiff turned out to be wrong. Which Finding showed it?", + "answer": { + "kind": "record", + "value": "world.fnd-io-trace-2" + }, + "evidence": ["world.fnd-io-trace-1", "world.fnd-io-trace-2"], + "path": [ + ["world.fnd-io-trace-1", "invalidated_by", "world.fnd-io-trace-2"] + ], + "checks": [ + { + "type": "select", + "effort": "*", + "kinds": ["finding"], + "where": { + "standing": "invalidated" + }, + "bodyIncludes": "MB/s", + "expect": ["ports.fnd-sd-speed", "world.fnd-io-trace-1"] + }, + { + "type": "field", + "record": "world.fnd-io-trace-1", + "field": "created_at", + "expect": "2025-12-08T10:00:00Z" + }, + { + "type": "field", + "record": "ports.fnd-sd-speed", + "field": "created_at", + "expect": "2026-03-04T10:30:00Z" + }, + { + "type": "select", + "effort": "*", + "kinds": ["finding"], + "invalidates": "world.fnd-io-trace-1" + } + ], + "traps": [ + "The I/O budget constraint was superseded, not invalidated, and it is not a finding.", + "The microSD read measurement was wrong too, but it was taken in March, and the full-card test disproved it.", + "The Skiff beta retrospective mentions a December pipeline blob test; that tested pipeline blobs, not streaming reads." + ], + "rivals": ["ports.fnd-sd-speed-2", "ports.fnd-retro"] + }, + { + "id": "ambiguous-019", + "from": "render-001", + "tier": 0, + "hops": 0, + "categories": ["ambiguous-effort", "lookup", "current-state"], + "question": "We're adding a pass to the 120 Hz performance mode on Meridian. How many milliseconds of each frame may GPU work take?", + "answer": { + "kind": "value", + "value": "7.4", + "accept": ["7.4 ms"] + }, + "evidence": ["render.con-120-2"], + "checks": [ + { + "type": "select", + "effort": "*", + "kinds": ["constraint"], + "live": true, + "titleIncludes": "120 Hz", + "expect": ["render.con-120-2"] + }, + { + "type": "select", + "effort": "*", + "kinds": ["constraint"], + "live": true, + "bodyIncludes": "120 Hz", + "expect": [ + "ports.con-frame-2", + "render.con-120-2", + "render.con-budget-3" + ] + }, + { + "type": "body", + "record": "render.con-120-2", + "includes": "Lantern GPU passes get 7.4 ms", + "expect": true + } + ], + "traps": [ + "8.3 ms is the whole 120 Hz frame, not the GPU share.", + "15.0 ms is the GPU share in Meridian's 60 Hz quality mode, in a Frame budget Constraint.", + "0.9 ms is the headroom left after the GPU share.", + "The ports Frame budget Constraint mentions the 120 Hz mode but gives it no budget; it says that budget belongs elsewhere." + ], + "rivals": ["ports.con-frame-2"] + }, + { + "id": "ambiguous-020", + "from": "render-035", + "tier": 2, + "hops": 3, + "categories": ["ambiguous-effort", "blob", "lifecycle"], + "question": "We're resizing the pool that overflowed at low tide. At its peak, how many pages did the trace behind that risk's evidence ask for?", + "answer": { + "kind": "value", + "value": "5912", + "accept": ["5,912"] + }, + "evidence": [ + "render.rsk-vsm-overflow", + "render.fnd-vsm-overflow", + "render.cit-vsm-pages", + "render.blb-vsm-pages" + ], + "path": [ + ["render.rsk-vsm-overflow", "evidence", "render.fnd-vsm-overflow"], + ["render.fnd-vsm-overflow", "cites", "render.cit-vsm-pages"], + ["render.cit-vsm-pages", "blob", "render.blb-vsm-pages"] + ], + "checks": [ + { + "type": "select", + "effort": "*", + "kinds": ["risk"], + "titleIncludes": ["pool", "overflow"], + "expect": ["render.rsk-vsm-overflow"] + }, + { + "type": "select", + "effort": "*", + "kinds": ["blob"], + "bodyIncludes": "peak_pages_requested", + "expect": ["render.blb-vsm-pages"] + }, + { + "type": "body", + "record": "render.blb-vsm-pages", + "includes": "\"peak_pages_requested\": 5912", + "expect": true + } + ], + "traps": [ + "4,096 is the pool size, not the peak request.", + "The streaming pool Risk also came true, but it was a budget cut, not an overflow, and it has no page trace." + ], + "rivals": ["world.rsk-pool-cut"] + }, + { + "id": "ambiguous-021", + "from": "ports-008", + "tier": 1, + "hops": 1, + "categories": ["ambiguous-effort", "rejected-alternative"], + "question": "When we chose the upscaler for Skiff, which Decision's acceptance rejected the spatial upscaler?", + "answer": { + "kind": "record", + "value": "ports.dec-tern-skiff" + }, + "evidence": ["ports.dec-skiff-spatial", "ports.dec-tern-skiff"], + "path": [ + ["ports.dec-skiff-spatial", "rejected_by", "ports.dec-tern-skiff"] + ], + "checks": [ + { + "type": "select", + "effort": "*", + "kinds": ["decision"], + "where": { + "standing": "rejected" + }, + "titleIncludes": "spatial", + "expect": ["ports.dec-skiff-spatial"] + }, + { + "type": "select", + "effort": "*", + "kinds": ["decision"], + "derivesFrom": "ports.iss-upscale-q", + "where": { + "state": "accepted" + } + }, + { + "type": "edge", + "edge": [ + "ports.dec-skiff-spatial", + "rejected_by", + "ports.dec-tern-skiff" + ], + "expect": true + } + ], + "traps": [ + "The native 720p option was rejected by the same acceptance; it is a fellow loser, not the rejecter.", + "The answer names the render Decision \"Ship Tern as Lantern's only upscaler\" as its reason, but that render Decision did not reject the spatial option." + ], + "rivals": ["render.dec-tern"] + }, + { + "id": "ambiguous-022", + "from": "render-048", + "tier": 2, + "hops": 2, + "categories": ["ambiguous-effort", "blocker"], + "question": "I'm taking over the open blocker about the caustics. A test found that one of the proposed fixes leaves the caustics ghosting. Which Decision is that?", + "answer": { + "kind": "record", + "value": "render.dec-caustic-clamp" + }, + "evidence": [ + "render.iss-caustic-flicker", + "render.dec-caustic-clamp", + "render.fnd-caustic-clamp-test" + ], + "path": [ + ["render.iss-caustic-flicker", "supports", "render.dec-caustic-clamp"], + [ + "render.dec-caustic-clamp", + "supports", + "render.fnd-caustic-clamp-test" + ] + ], + "checks": [ + { + "type": "select", + "effort": "*", + "kinds": ["issue"], + "live": true, + "titleIncludes": "flicker", + "expect": ["render.iss-caustic-flicker"] + }, + { + "type": "select", + "effort": "*", + "kinds": ["decision"], + "where": { + "standing": "proposed" + }, + "derivesFrom": "render.iss-caustic-flicker", + "expect": ["render.dec-caustic-clamp", "render.dec-caustic-seed"] + }, + { + "type": "select", + "effort": "*", + "kinds": ["finding"], + "titleIncludes": "ghost", + "derivesFrom": "render.dec-caustic-clamp", + "expect": ["render.fnd-caustic-clamp-test"] + }, + { + "type": "select", + "effort": "*", + "kinds": ["decision"], + "derivesFrom": "render.iss-caustic-flicker", + "bodyIncludes": "clamp" + }, + { + "type": "select", + "effort": "*", + "kinds": ["decision"], + "where": { + "standing": "proposed" + }, + "derivesFrom": "ports.iss-blk-storm", + "expect": ["ports.dec-storm-30", "ports.dec-storm-cap"] + } + ], + "traps": [ + "The tide-locked seed test found frozen caustics, not ghosting.", + "The open Skiff cert blocker about tide storms also has two proposed fixes and a test, but the test found a CPU saving, not ghosting.", + "A rejected Skiff Decision would have stepped caustics down a quality tier under GPU load. It was never a fix for the caustics blocker." + ], + "rivals": ["ports.dec-storm-cap", "ports.dec-dynres-tiers"] + }, + { + "id": "ambiguous-023", + "from": "ports-006", + "tier": 1, + "hops": 1, + "categories": ["ambiguous-effort", "supersession", "current-state"], + "question": "Which Constraint sets how much memory the game may use on Skiff today?", + "answer": { + "kind": "record", + "value": "ports.con-skiff-mem-8" + }, + "evidence": ["ports.con-skiff-mem-9", "ports.con-skiff-mem-8"], + "path": [ + ["ports.con-skiff-mem-9", "superseded_by", "ports.con-skiff-mem-8"] + ], + "checks": [ + { + "type": "select", + "effort": "*", + "kinds": ["constraint"], + "live": true, + "titleIncludes": "memory", + "bodyIncludes": "Skiff" + }, + { + "type": "head", + "start": "ports.con-skiff-mem-9", + "expect": "ports.con-skiff-mem-8" + } + ], + "traps": [ + "Both versions share the title Skiff memory budget; the 9 GB version was superseded.", + "The streaming pool budget (1.6 GB) is one system's share of the Skiff total, not the total." + ], + "rivals": ["world.con-pool-3"] + }, + { + "id": "ambiguous-024", + "from": "world-028", + "tier": 1, + "hops": 1, + "categories": ["ambiguous-effort", "rejected-alternative"], + "question": "We're thinking again about letting one machine run the tide and send the result to everyone else. Which Finding measured the cost that ruled that out last time?", + "answer": { + "kind": "record", + "value": "world.fnd-tide-host-bw" + }, + "evidence": ["world.dec-tide-host", "world.fnd-tide-host-bw"], + "path": [["world.dec-tide-host", "supports", "world.fnd-tide-host-bw"]], + "checks": [ + { + "type": "select", + "effort": "*", + "kinds": ["decision"], + "titleIncludes": "tide height", + "expect": ["world.dec-tide-host"] + }, + { + "type": "select", + "effort": "*", + "kinds": ["finding"], + "derivesFrom": "world.dec-tide-host" + } + ], + "traps": [ + "The strict-float finding ruled out a different alternative.", + "The netcode Effort's host-authoritative spike Finding measures the client CPU saved by not running tides, not the bandwidth of streaming tide heights." + ], + "rivals": ["netcode.fnd-host-spike"] + } + ] +} diff --git a/benchmarks/proof-recall/scripts/build.mjs b/benchmarks/proof-recall/scripts/build.mjs index dd96663e..3e1ef4d2 100644 --- a/benchmarks/proof-recall/scripts/build.mjs +++ b/benchmarks/proof-recall/scripts/build.mjs @@ -10,6 +10,11 @@ // add filler until each Effort is // about 10 times its size; every // question keeps its answer +// node scripts/build.mjs --scale 10 --filler-overlap 0.3 --out /tmp/x10adv +// make 30% of the filler reuse real +// subsystem names (see +// spec/_filler.mjs); answers still +// hold import { createHash } from 'node:crypto'; import { existsSync } from 'node:fs'; import { cp, mkdir, readdir, readFile, rm, writeFile } from 'node:fs/promises'; @@ -20,7 +25,12 @@ import { generateArtifactId, ProofMutationSchema, } from '@flatbread/proof'; -import { fillerEvents, fillerTerms, mergeEvents } from '../spec/_filler.mjs'; +import { + fillerEvents, + fillerTerms, + mergeEvents, + overlapTerms, +} from '../spec/_filler.mjs'; const BENCH = resolve(dirname(fileURLToPath(import.meta.url)), '..'); const SPEC_DIR = join(BENCH, 'spec'); @@ -69,14 +79,24 @@ const GRAPH_DIRS = [ ]; function args(argv) { - const out = { only: [], out: join(BENCH, '.flatbread-proof'), scale: 1 }; + const out = { + only: [], + out: join(BENCH, '.flatbread-proof'), + scale: 1, + overlap: 0, + }; for (let i = 0; i < argv.length; i++) { if (argv[i] === '--only') out.only.push(...argv[++i].split(',')); else if (argv[i] === '--out') out.out = resolve(argv[++i]); else if (argv[i] === '--scale') out.scale = Number(argv[++i]); + else if (argv[i] === '--filler-overlap') out.overlap = Number(argv[++i]); else throw new Error(`Unknown argument ${argv[i]}`); } if (!(out.scale >= 1)) throw new Error('--scale must be 1 or more'); + if (!(out.overlap >= 0 && out.overlap <= 1)) + throw new Error('--filler-overlap must be a fraction from 0 to 1'); + if (out.overlap > 0 && !(out.scale > 1)) + throw new Error('--filler-overlap needs --scale above 1'); // The committed graph is the unscaled one. if (out.scale > 1 && out.out === join(BENCH, '.flatbread-proof')) throw new Error('A scaled build needs --out'); @@ -100,6 +120,121 @@ async function assertFillerIsDisjoint(specs) { throw new Error(`Filler subsystem "${term}" appears in real content`); } +/** The question sets, and each spec as text, for the overlap checks. */ +async function readContent(specs) { + const dir = join(BENCH, 'questions'); + const files = (await readdir(dir)).filter((name) => name.endsWith('.json')); + const sets = await Promise.all( + files.map(async (name) => + JSON.parse(await readFile(join(dir, name), 'utf8')) + ) + ); + // The overlap redraws depend on these files, so a hash of them tells + // whether two adversarial graphs were built from the same questions. + const hash = createHash('sha256'); + for (const name of [...files].sort()) + hash.update(`${name}\0`).update(await readFile(join(dir, name))); + return { + sets, + specText: specs.map((spec) => JSON.stringify(spec)), + questionsHash: hash.digest('hex'), + }; +} + +/** + * Overlap filler must use real names and nothing a question asks about: + * each real name appears in its Effort's spec, and no housekeeping metric + * or question phrase appears in any spec or question. + */ +function assertOverlapVocabulary(specs, { sets, specText }) { + const all = [...specText, ...sets.map((set) => JSON.stringify(set))] + .join('\n') + .toLowerCase(); + for (const [i, spec] of specs.entries()) { + const { real, housekeeping } = overlapTerms(spec.key); + for (const name of real) + if (!specText[i].toLowerCase().includes(name.toLowerCase())) + throw new Error( + `Overlap name "${name}" is not in the ${spec.key} spec` + ); + for (const term of housekeeping) + if (all.includes(term.toLowerCase())) + throw new Error( + `Overlap filler term "${term}" appears in real content` + ); + } +} + +/** + * For one Effort, a test that is true when an overlap filler event could + * change a question's answer. It is true when the event's title and body + * hold: + * + * - every phrase of a `select` check's `bodyIncludes` and `titleIncludes` + * together, for a check that scans this Effort and the event's kind; + * - a `body` check's phrase, or a value answer, that is a measurement: a + * number with a unit or word, such as "384 MB" or "6 min 40 s", matched as + * whole words so "4 minutes" does not match "14 minutes". + * + * Bare numbers ("2", "87") and names ("Dara Whitlock") are left out: filler + * prints both everywhere, and neither states a fact about a real subsystem. + */ +function overlapCollision(effortKey, { sets }) { + const selects = []; // { kinds, phrases }: all phrases must appear + const measures = new Set(); // each must appear as whole words + const scans = (set, check) => + check.effort === '*' || + (Array.isArray(check.effort) + ? check.effort.includes(effortKey) + : (check.effort ?? set.effort) === effortKey); + const list = (value) => (Array.isArray(value) ? value : [value]); + const isMeasure = (text) => /\d/.test(text) && /[a-z%]/i.test(text); + for (const set of sets) + for (const q of set.questions) { + for (const check of q.checks ?? []) { + if (check.type === 'select' && scans(set, check)) { + const phrases = ['bodyIncludes', 'titleIncludes'].flatMap((field) => + check[field] ? list(check[field]) : [] + ); + if (phrases.length) + selects.push({ + kinds: check.kinds ?? null, + phrases: phrases.map((p) => p.toLowerCase()), + }); + } + if (check.type === 'body') + for (const p of list(check.includes)) + if (isMeasure(p)) measures.add(p.toLowerCase()); + } + if (q.answer.kind === 'value') + for (const gold of [q.answer.value, ...(q.answer.accept ?? [])]) + if (isMeasure(String(gold))) measures.add(String(gold).toLowerCase()); + } + const wholeWord = (text, phrase) => { + for ( + let i = text.indexOf(phrase); + i >= 0; + i = text.indexOf(phrase, i + 1) + ) { + const before = text[i - 1] ?? ' '; + const after = text[i + phrase.length] ?? ' '; + if (!/[\w.]/.test(before) && !/\w/.test(after)) return true; + } + return false; + }; + return (event) => { + const kind = KIND_BY_TYPE[event.type]; + const text = `${event.title}\n${event.body}`.toLowerCase(); + return ( + selects.some( + ({ kinds, phrases }) => + (!kinds || kinds.includes(kind)) && + phrases.every((p) => text.includes(p)) + ) || [...measures].some((m) => wholeWord(text, m)) + ); + }; +} + /** Fixed id: the writer's own format, with a suffix hashed from the key. */ export function benchId(kind, title, qualifiedKey) { const bytes = createHash('sha256').update(qualifiedKey).digest(); @@ -193,8 +328,25 @@ async function main() { if (!specs.length) throw new Error('No specs matched'); if (options.scale > 1) { await assertFillerIsDisjoint(specs); + const content = options.overlap > 0 ? await readContent(specs) : null; + if (content) { + assertOverlapVocabulary(specs, content); + console.log(`filler overlap: questions sha256 ${content.questionsHash}`); + } for (const spec of specs) - spec.events = mergeEvents(spec.events, fillerEvents(spec, options.scale)); + spec.events = mergeEvents( + spec.events, + fillerEvents( + spec, + options.scale, + content + ? { + overlap: options.overlap, + collides: overlapCollision(spec.key, content), + } + : undefined + ) + ); } await mkdir(options.out, { recursive: true }); for (const dir of GRAPH_DIRS) diff --git a/benchmarks/proof-recall/scripts/check.mjs b/benchmarks/proof-recall/scripts/check.mjs index 247842d5..2813a0a6 100644 --- a/benchmarks/proof-recall/scripts/check.mjs +++ b/benchmarks/proof-recall/scripts/check.mjs @@ -3,8 +3,11 @@ // Exits 1 on any error, so authors can loop on it. // // node scripts/check.mjs all question files -// node scripts/check.mjs --only render one Effort's questions +// node scripts/check.mjs --only render one question set // node scripts/check.mjs --graph /tmp/x a graph built elsewhere +// +// This script checks every file, including the opt-in sets in OPT_IN_SETS. +// Other scripts load those only when asked for them by name. import { readdir, readFile } from 'node:fs/promises'; import { dirname, join, resolve } from 'node:path'; import { fileURLToPath, pathToFileURL } from 'node:url'; @@ -41,7 +44,19 @@ export const CATEGORIES = [ 'distractor', // a similar record in another Effort or thread 'cross-effort', // needs records from two Efforts 'rationale', // explain why, scored by rubric + // Rewritten so it names no Effort and uses words several Efforts share; + // the reader must find the right Effort first. Only in ambiguous.json. + 'ambiguous-effort', ]; +// Question files whose `effort` names no Effort: their questions span +// several, so every key is qualified as `.`. +export const MULTI_EFFORT_SETS = ['cross', 'ambiguous']; +// Sets that load only when a script names them (`--only ambiguous`) or asks +// for one of their ids, so every default keeps the original 246 questions. +export const OPT_IN_SETS = ['ambiguous']; +// Names that point at one Effort without its title: the engine and renderer +// code names that Effort titles use. +const EFFORT_NAMES = ['Kestrel', 'Lantern']; const ANSWER_KINDS = ['record', 'records', 'value', 'boolean', 'count', 'text']; const NEEDS_CHECK = new Set(['record', 'records', 'count']); @@ -55,21 +70,136 @@ function args(argv) { return out; } -export async function loadQuestions(only = []) { +/** + * Question files, sorted by name. `only` lists the sets to load by their + * `effort` field; the word `default` stands for every set outside + * OPT_IN_SETS. With no `only`, the default sets load, plus the opt-in ones + * when `optIn` is true. + */ +export async function loadQuestions(only = [], { optIn = false } = {}) { const dir = join(BENCH, 'questions'); const files = (await readdir(dir)) .filter((name) => name.endsWith('.json')) .sort(); + const wanted = (name) => + only.length + ? only.includes(name) || + (only.includes('default') && !OPT_IN_SETS.includes(name)) + : optIn || !OPT_IN_SETS.includes(name); const sets = []; for (const file of files) { const set = JSON.parse(await readFile(join(dir, file), 'utf8')); - if (only.length && !only.includes(set.effort)) continue; - sets.push({ file, ...set }); + if (wanted(set.effort)) sets.push({ file, ...set }); } return sets; } -function validate(graph, set, q, errors, seen, warnings = []) { +/** + * Questions picked by set and id, as run.mjs and prepare-run.mjs take them. + * Ids are looked up in every set, so `--ids` reaches the opt-in sets too. + */ +export async function pickQuestions({ only = [], ids = [] } = {}) { + const wanted = new Set(ids); + return (await loadQuestions(only, { optIn: wanted.size > 0 })) + .map((set) => ({ + ...set, + questions: set.questions.filter((q) => !wanted.size || wanted.has(q.id)), + })) + .filter((set) => set.questions.length); +} + +/** Problems with an ambiguous-Effort question, as messages. */ +function ambiguityErrors(graph, q, sources) { + const out = []; + const source = sources.get(q.from); + if (!source) return [`"from" must name a question in an Effort file`]; + if (source.q.tier > 2) + out.push(`its source ${q.from} is tier ${source.q.tier}; use tier 0-2`); + for (const c of ['ambiguous-effort', ...source.q.categories]) + if (!q.categories?.includes(c)) out.push(`needs category ${c}`); + if (JSON.stringify(q.answer) !== JSON.stringify(qualifiedAnswer(source))) + out.push(`its answer differs from ${q.from}'s`); + const text = q.question ?? ''; + const efforts = Object.keys(graph.keymap) + .filter((key) => key.endsWith('.effort')) + .map((key) => key.split('.')[0]); + const titles = efforts.map( + (name) => + graph.records.get(graph.keymap[`${name}.effort`]).frontmatter.title + ); + for (const title of titles) + if (text.toLowerCase().includes(title.toLowerCase())) + out.push(`names the Effort "${title}"`); + if (/\bEfforts?\b/i.test(text)) out.push('says "Effort"'); + for (const name of EFFORT_NAMES) + if (new RegExp(`\\b${name}\\b`, 'i').test(text)) + out.push(`names ${name}, which singles out one Effort`); + // "the ports team", "the render work": an Effort key used as a name. + const keyed = new RegExp( + `\\b(${efforts.join('|')})\\s+(team|work|side|Effort)\\b`, + 'i' + ).exec(text); + if (keyed) out.push(`"${keyed[0]}" names an Effort`); + // A platform word is fair only if records in two or more Efforts use it + // and no one Effort holds more than two thirds of those records. + for (const word of PLATFORM_WORDS) { + const pattern = new RegExp( + `\\b${word}\\b`, + word === word.toLowerCase() ? 'i' : '' + ); + if (!pattern.test(text)) continue; + const perEffort = new Map(); + for (const rec of graph.records.values()) + if ( + rec.kind !== 'effort' && + pattern.test(`${rec.frontmatter.title ?? ''}\n${rec.body}`) + ) + perEffort.set( + rec.frontmatter.effort, + (perEffort.get(rec.frontmatter.effort) ?? 0) + 1 + ); + const total = [...perEffort.values()].reduce((a, b) => a + b, 0); + if (perEffort.size < 2) + out.push(`says "${word}", which fewer than two Efforts use`); + else if (Math.max(...perEffort.values()) > (total * 2) / 3) + out.push(`says "${word}", which mostly one Effort uses`); + } + // Rivals are records in other Efforts that the wording also fits: the + // near misses that make the question ambiguous. A trap says why each one + // is wrong. + if (!q.rivals?.length) + out.push('needs `rivals`: near-miss records in other Efforts'); + const ownEffort = graph.keymap[`${source.effort}.effort`]; + for (const key of q.rivals ?? []) { + const rec = graph.records.get(graph.keymap[key]); + if (rec?.frontmatter.effort === ownEffort) + out.push(`rival ${key} is in ${source.effort}, the answer's own Effort`); + } + return out; +} + +// Words for a platform. "handheld" and "home console" stand for Skiff and +// Meridian, but may lean on the ports Effort, which owns both ports. +const PLATFORM_WORDS = ['PC', 'Meridian', 'Skiff', 'handheld', 'home console']; + +/** A source question's answer with its keys qualified by its Effort. */ +function qualifiedAnswer({ effort, q }) { + const { kind, value } = q.answer; + if (kind === 'record') return { ...q.answer, value: qualify(effort, value) }; + if (kind === 'records') + return { ...q.answer, value: value.map((key) => qualify(effort, key)) }; + return q.answer; +} + +function validate( + graph, + set, + q, + errors, + seen, + warnings = [], + sources = new Map() +) { const err = (message) => errors.push(`${set.file} ${q.id ?? '?'}: ${message}`); if (!q.id || seen.has(q.id)) return err('missing or duplicate id'); @@ -94,8 +224,20 @@ function validate(graph, set, q, errors, seen, warnings = []) { err('text answers need a rubric'); if (answer.kind !== 'text' && answer.value === undefined) err('answer needs a value'); + const multi = MULTI_EFFORT_SETS.includes(set.effort); + if ( + q.categories?.includes('ambiguous-effort') !== + (set.effort === 'ambiguous') + ) + err( + 'the ambiguous-effort category belongs to ambiguous.json, and every question there needs it' + ); + if (set.effort === 'ambiguous') + for (const message of ambiguityErrors(graph, q, sources)) err(message); const keysOk = (keys, label) => { for (const key of keys ?? []) { + if (multi && !String(key).includes('.')) + err(`${label}: key "${key}" needs its Effort, as .`); try { idFor(graph, set.effort, key); } catch (error) { @@ -105,6 +247,7 @@ function validate(graph, set, q, errors, seen, warnings = []) { }; if (!q.evidence?.length) err('needs evidence keys'); keysOk(q.evidence, 'evidence'); + keysOk(q.rivals, 'rivals'); if (answer.kind === 'record') keysOk([answer.value], 'answer'); if (answer.kind === 'records') keysOk(answer.value, 'answer'); const path = q.path ?? []; @@ -117,7 +260,7 @@ function validate(graph, set, q, errors, seen, warnings = []) { // Runs show a question alone. If an Effort-wide check would also find // answers in another Effort, the question must name its own, or carry // `"scoped": true` once an author confirms the wording pins it down. - if (set.effort !== 'cross' && !q.scoped) { + if (!multi && !q.scoped) { const title = graph.records.get(graph.keymap[`${set.effort}.effort`]) ?.frontmatter.title; const named = @@ -297,14 +440,21 @@ export function graphStats(graph, effortKey) { async function main() { const options = args(process.argv.slice(2)); const graph = await loadGraph(options.graph); - const sets = await loadQuestions(options.only); + const sets = await loadQuestions(options.only, { optIn: true }); + // Ambiguous questions are checked against their sources, so load every + // Effort file even under --only. + const sources = new Map(); + for (const set of await loadQuestions()) + if (!MULTI_EFFORT_SETS.includes(set.effort)) + for (const q of set.questions) + sources.set(q.id, { effort: set.effort, q }); const errors = []; const warnings = []; const seen = new Set(); const all = []; for (const set of sets) for (const q of set.questions ?? []) { - validate(graph, set, q, errors, seen, warnings); + validate(graph, set, q, errors, seen, warnings, sources); all.push({ ...q, effort: set.effort }); } const efforts = [ @@ -324,9 +474,11 @@ async function main() { }) ); } - const other = all.filter((q) => !efforts.includes(q.effort)); - if (other.length) - console.log(JSON.stringify({ effort: 'cross', questions: other.length })); + for (const name of MULTI_EFFORT_SETS) { + const qs = all.filter((q) => q.effort === name); + if (qs.length) + console.log(JSON.stringify({ effort: name, questions: qs.length })); + } if (warnings.length) { console.warn(warnings.join('\n')); console.warn(`${warnings.length} warning(s)`); diff --git a/benchmarks/proof-recall/scripts/compare.mjs b/benchmarks/proof-recall/scripts/compare.mjs index ca7f8e26..d103a2bc 100644 --- a/benchmarks/proof-recall/scripts/compare.mjs +++ b/benchmarks/proof-recall/scripts/compare.mjs @@ -2,9 +2,14 @@ // Compares two runs over the questions both answered, pair by pair. // // node scripts/compare.mjs runs/.json runs/.json [--skip id1,id2] +// [--only ambiguous] // node scripts/compare.mjs runs/a-r1.json,runs/a-r2.json \ // runs/b-r1.json,runs/b-r2.json // +// It compares the original 246 questions. `--only` names the sets to compare +// instead, as run.mjs takes them, so a run that also asked the +// ambiguous-Effort set reports that set apart: `--only ambiguous`. +// // With several files on each side, the runs are pooled: run i of side A is // paired with run i of side B, and each question in each pair of runs is one // row, named `#` in the lists of questions only one side got right. @@ -52,10 +57,12 @@ async function main() { const argv = process.argv.slice(2); const at = argv.indexOf('--skip'); const skip = new Set(at >= 0 ? argv.splice(at, 2)[1].split(',') : []); + const atOnly = argv.indexOf('--only'); + const only = atOnly >= 0 ? argv.splice(atOnly, 2)[1].split(',') : []; const [pathA, pathB] = argv; if (!pathB) throw new Error( - 'usage: compare.mjs [,more] [,more] [--skip ids]' + 'usage: compare.mjs [,more] [,more] [--skip ids] [--only sets]' ); const load = (paths) => Promise.all( @@ -86,7 +93,7 @@ async function main() { ? runs.map((run, i) => run.run?.name ?? paths.split(',')[i]).join(',') : runs[0].run?.name ?? paths; const graph = await loadGraph(join(BENCH, '.flatbread-proof')); - const sets = await loadQuestions(); + const sets = await loadQuestions(only); const rows = []; for (const [i, a] of runsA.entries()) for (const set of sets) diff --git a/benchmarks/proof-recall/scripts/find-recall.mjs b/benchmarks/proof-recall/scripts/find-recall.mjs index 9c3869b7..f070b856 100644 --- a/benchmarks/proof-recall/scripts/find-recall.mjs +++ b/benchmarks/proof-recall/scripts/find-recall.mjs @@ -3,7 +3,14 @@ // whether its top hits hold the evidence, before any paid agent run. // // node scripts/find-recall.mjs [--graph ] [--ids a,b] -// [--split tune|holdout|all] [--out ] [--jobs 8] +// [--only ambiguous] [--split tune|holdout|all] [--out ] +// [--jobs 8] +// +// The gate asks the original 246 questions. `--only` names question sets +// by their `effort` field instead, as run.mjs takes them: `--only ambiguous` +// asks the ambiguous-Effort set, whose questions name no Effort. `--ids` +// picks questions from any set. Run the ambiguous set only after it is +// written and frozen, since its point is that nobody tuned it to `find`. // // The gate verdict that counts is the one on the 10x graph, given with // `--graph `. The default, the committed 1x graph, is a quick @@ -16,7 +23,7 @@ // names, or platform names, with --all. Topic words that an // Effort title also uses, such as "build farm", stay elsewhere. // scoped the question text, with --effort set to the question's Effort -// (cross-Effort questions are skipped) +// (cross-Effort and ambiguous-Effort questions are skipped) // The plan asked for a third set of queries taken from pilot transcripts. // No pilot ran `proof find`, so none exist; `scoped` stands in until a find // pilot gives real agent queries. @@ -42,7 +49,9 @@ // split's rows are the same as its part of an `all` // run. // right_effort_first the top Effort in `efforts by match` is the -// question's own (single-Effort questions, --all sets) +// question's own (single-Effort questions, --all sets; +// for an ambiguous-Effort question, the Effort of the +// question it was rewritten `from`) // reply_chars, cards_chars // the tables print the mean reply size and the // largest one @@ -71,7 +80,7 @@ import { request } from 'node:http'; import { tmpdir } from 'node:os'; import { dirname, join, resolve } from 'node:path'; import { fileURLToPath } from 'node:url'; -import { loadQuestions } from './check.mjs'; +import { MULTI_EFFORT_SETS, pickQuestions } from './check.mjs'; import { findHash } from './collect.mjs'; import { loadGraph, qualify } from './graph.mjs'; import { prepareFolder } from './prepare-run.mjs'; @@ -101,6 +110,7 @@ function args(argv) { const out = { graph: join(BENCH, '.flatbread-proof'), ids: [], + only: [], out: join(BENCH, 'runs', 'find-recall.json'), jobs: 8, split: 'all', @@ -109,6 +119,7 @@ function args(argv) { const flag = argv[i]; if (flag === '--graph') out.graph = resolve(argv[++i]); else if (flag === '--ids') out.ids = argv[++i].split(','); + else if (flag === '--only') out.only = argv[++i].split(','); else if (flag === '--out') out.out = resolve(argv[++i]); else if (flag === '--jobs') out.jobs = Number(argv[++i]); else if (flag === '--split') out.split = argv[++i]; @@ -263,9 +274,9 @@ function scoreReply(graph, effortIds, row, stdout) { ), answer_on_card: answerOnCard(row.q, answers, reply.cards), right_effort_first: - row.set === 'scoped' || row.effort === 'cross' + row.set === 'scoped' || !effortIds[row.home] ? null - : reply.find.efforts?.[0]?.id === effortIds[row.effort], + : reply.find.efforts?.[0]?.id === effortIds[row.home], reply_chars: stdout.length, cards_chars: reply.cards.length, hits: top5.map((hit) => hit.matched_id ?? hit.id), @@ -341,12 +352,14 @@ async function main() { 'that counts is the one on the 10x graph (--graph ).' ); const find_hash = await findHash(); - const wanted = new Set(options.ids); const jobs = []; - for (const set of await loadQuestions()) + for (const set of await pickQuestions(options)) for (const q of set.questions) { - if (wanted.size && !wanted.has(q.id)) continue; - const row = { q, effort: set.effort }; + // The Effort a reader should land in: the file's own, or for an + // ambiguous question, that of its source (ids are `-NNN`). + const home = + set.effort === 'ambiguous' ? q.from.split('-')[0] : set.effort; + const row = { q, effort: set.effort, home }; jobs.push({ ...row, set: 'raw', query: q.question, scope: ['--all'] }); jobs.push({ ...row, @@ -354,7 +367,7 @@ async function main() { query: stripNames(q.question, titles), scope: ['--all'], }); - if (set.effort !== 'cross') + if (!MULTI_EFFORT_SETS.includes(set.effort)) jobs.push({ ...row, set: 'scoped', diff --git a/benchmarks/proof-recall/scripts/grade.mjs b/benchmarks/proof-recall/scripts/grade.mjs index b929907c..e84bf7f9 100644 --- a/benchmarks/proof-recall/scripts/grade.mjs +++ b/benchmarks/proof-recall/scripts/grade.mjs @@ -1,7 +1,11 @@ #!/usr/bin/env node // Scores one benchmark run. // -// node scripts/grade.mjs runs/.json +// node scripts/grade.mjs runs/.json [--only ambiguous] +// +// It scores the original 246 questions that the run answered. `--only` +// names the sets to score instead, as run.mjs takes them: `ambiguous` for +// the ambiguous-Effort set alone, `default,ambiguous` for both. // // A run file holds one entry per question id: // { "run": { "strategy": "brief", "model": "..." }, @@ -115,12 +119,15 @@ function summarize(rows, by) { } async function main() { - const runPath = process.argv[2]; - if (!runPath) throw new Error('usage: grade.mjs '); + const argv = process.argv.slice(2); + const at = argv.indexOf('--only'); + const only = at >= 0 ? argv.splice(at, 2)[1].split(',') : []; + const [runPath] = argv; + if (!runPath) throw new Error('usage: grade.mjs [--only sets]'); const run = JSON.parse(await readFile(resolve(runPath), 'utf8')); const graph = await loadGraph(join(BENCH, '.flatbread-proof')); const rows = []; - for (const set of await loadQuestions()) + for (const set of await loadQuestions(only)) for (const q of set.questions) { const entry = run.answers?.[q.id]; if (!entry) continue; diff --git a/benchmarks/proof-recall/scripts/graph.mjs b/benchmarks/proof-recall/scripts/graph.mjs index 4d27b758..7938417f 100644 --- a/benchmarks/proof-recall/scripts/graph.mjs +++ b/benchmarks/proof-recall/scripts/graph.mjs @@ -230,7 +230,9 @@ function compactionState(graph, effortId) { */ export function runCheck(graph, effort, check) { const keyOf = (rec) => graph.keyById.get(rec.id); - // Only Effort-wide checks need it; cross-Effort files name `check.effort`. + // Only Effort-wide checks need it. Files that span Efforts (`cross`, + // `ambiguous`) have no Effort of their own, so their checks name + // `check.effort` and qualify every key as `.`. const effortId = () => idFor(graph, check.effort ?? effort, 'effort'); switch (check.type) { case 'field': { @@ -292,9 +294,23 @@ export function runCheck(graph, effort, check) { return support && list(check.restsOn).includes(standing(support)); }) ); + if (check.invalidates) { + const target = idFor(graph, effort, check.invalidates); + recs = recs.filter((rec) => + list(rec.frontmatter.invalidates).includes(target) + ); + } + // Each text filter takes one phrase or a list; a record must contain + // every phrase, ignoring case. + const contains = (text, phrases) => + list(phrases).every((p) => + text.toLowerCase().includes(p.toLowerCase()) + ); if (check.bodyIncludes) + recs = recs.filter((rec) => contains(rec.body, check.bodyIncludes)); + if (check.titleIncludes) recs = recs.filter((rec) => - rec.body.toLowerCase().includes(check.bodyIncludes.toLowerCase()) + contains(String(rec.frontmatter.title ?? ''), check.titleIncludes) ); const keys = recs.map(keyOf).sort(); return check.count ? keys.length : keys; diff --git a/benchmarks/proof-recall/scripts/prepare-run.mjs b/benchmarks/proof-recall/scripts/prepare-run.mjs index 40a68886..c49f1830 100644 --- a/benchmarks/proof-recall/scripts/prepare-run.mjs +++ b/benchmarks/proof-recall/scripts/prepare-run.mjs @@ -10,7 +10,7 @@ import { cp, mkdir, rm, symlink, writeFile } from 'node:fs/promises'; import { dirname, join, resolve } from 'node:path'; import { fileURLToPath, pathToFileURL } from 'node:url'; -import { loadQuestions } from './check.mjs'; +import { pickQuestions } from './check.mjs'; const BENCH = resolve(dirname(fileURLToPath(import.meta.url)), '..'); // The form an answer takes, never its content. @@ -24,12 +24,14 @@ export const FORMATS = { }; const REPO = resolve(BENCH, '..', '..'); -/** Questions as an agent may see them: text and answer form only. */ +/** + * Questions as an agent may see them: text and answer form only. The + * default is the original 246; `only` names sets (`ambiguous`, or + * `default,ambiguous` for both), and `ids` finds questions in any set. + */ export async function publicQuestions({ only = [], ids = [] } = {}) { - const wanted = new Set(ids); - return (await loadQuestions(only)) + return (await pickQuestions({ only, ids })) .flatMap((set) => set.questions) - .filter((q) => !wanted.size || wanted.has(q.id)) .map((q) => ({ id: q.id, question: q.question, diff --git a/benchmarks/proof-recall/scripts/run.mjs b/benchmarks/proof-recall/scripts/run.mjs index 7afbbed1..e4157eba 100644 --- a/benchmarks/proof-recall/scripts/run.mjs +++ b/benchmarks/proof-recall/scripts/run.mjs @@ -9,7 +9,10 @@ // node scripts/run.mjs --stop-server [--server ] // // --strategies names the task texts in task.mjs to run; the default is -// brief,nobrief. Each session works in ///, which +// brief,nobrief. The default questions are the original 246. --only names +// question sets by their `effort` field: `--only ambiguous` runs the +// ambiguous-Effort set alone, and `--only default,ambiguous` runs both in +// one run. --ids picks questions from any set. Each session works in ///, which // holds a link to the shared copy of the graph, TASK.md, the agent's // answer.json, and transcript.jsonl. A rerun skips sessions that already // have both an answer and a transcript, so an interrupted run picks up where diff --git a/benchmarks/proof-recall/spec/_filler.mjs b/benchmarks/proof-recall/spec/_filler.mjs index 96e76ec3..65cb74aa 100644 --- a/benchmarks/proof-recall/spec/_filler.mjs +++ b/benchmarks/proof-recall/spec/_filler.mjs @@ -8,6 +8,16 @@ // the Effort's head compaction, which therefore covers it. A seeded random // generator makes the output identical on every build. // +// `build.mjs --filler-overlap ` makes filler adversarial. That +// share of filler threads and records take a real subsystem name from the +// Effort's own spec (Tern, VSM, saltglass refraction) in place of a made-up +// one. They still link to no real record, and they measure only routine +// things no question asks about, such as warning counts. Their numbers come +// from a second seeded generator, so the other filler records stay exactly +// as in a plain build. build.mjs redraws any overlap draft that collides +// with a question, so the output is fixed for a given fraction and a given +// set of question files; editing any question can change it. +// // The leading underscore keeps `loadSpecs` from reading this file as a spec. const PEOPLE = { @@ -108,6 +118,74 @@ const SUBSYSTEMS = { ], }; +// Real subsystem names that overlap filler reuses. Each must appear in its +// Effort's spec; `build.mjs` checks that. +const OVERLAP_SUBSYSTEMS = { + render: [ + 'Tern', + 'VSM', + 'saltglass refraction', + 'caustics', + 'SDF GI', + 'Brine curve', + 'visibility buffer', + 'meshlets', + ], + world: [ + 'tide simulation', + 'streaming pool', + 'Gull VM', + 'fibers', + 'Kelp', + 'delta saves', + 'archetype tables', + 'hot reload', + ], + ports: [ + 'pipeline cache', + 'dynamic resolution', + 'microSD', + 'fast travel', + 'autosave', + 'Kelp', + 'Tern', + ], + tools: [ + 'shader compile cache', + 'cache key', + 'remote execution', + 'nightly builds', + 'fetch timeout', + 'build farm', + ], + netcode: [ + 'rollback window', + 'host-authoritative sync', + 'lockstep', + 'desync', + 'input delay', + 'tide simulation', + ], +}; +// What overlap filler measures and asks: housekeeping that no question asks +// about. `build.mjs` refuses any of these that a spec or question uses. +const OVERLAP_METRICS = [ + ['warning count', '', 1, 60], + ['log lines per minute', '', 5, 900], + ['unit test time', 's', 2, 120], + ['config parse time', 'ms', 1, 90], + ['symbol upload time', 's', 3, 200], + ['console command count', '', 2, 80], +]; +const OVERLAP_QUESTIONS = [ + 'should log less', + 'needs a clearer error message', + 'should move its settings to the new config format', + 'should drop its old console commands', + 'needs its own unit test target', + 'should print fewer warnings', +]; + const PLATFORMS = ['PC', 'Meridian', 'Skiff']; const METRICS = [ ['load time', 'ms', 20, 900], @@ -174,11 +252,36 @@ export function fillerTerms(effortKey) { return SUBSYSTEMS[effortKey] ?? []; } +/** + * The words overlap filler uses, for build.mjs to check: `real` names must + * appear in the Effort's spec; `housekeeping` phrases must appear in no spec + * or question. + */ +export function overlapTerms(effortKey) { + return { + real: OVERLAP_SUBSYSTEMS[effortKey] ?? [], + housekeeping: [ + ...OVERLAP_METRICS.map(([metric]) => metric), + ...OVERLAP_QUESTIONS, + ], + }; +} + /** * Filler events for one spec, so the Effort ends with about `scale` times * its own records. Returns events with `at` and `key` like a spec's own. + * + * With `overlap` above 0, that share of filler threads and records reuse a + * real subsystem name (see the header). `collides(event)` is true when an + * overlap event would match a question's check or state one of its + * answers; such an item draws new words, and after five tries falls back to + * plain filler, so overlap filler never changes an answer. */ -export function fillerEvents(spec, scale) { +export function fillerEvents( + spec, + scale, + { overlap = 0, collides = () => false } = {} +) { if (scale <= 1) return []; const subsystems = SUBSYSTEMS[spec.key]; const people = PEOPLE[spec.key]; @@ -193,6 +296,41 @@ export function fillerEvents(spec, scale) { 6 * 3600e3; const rand = random(`filler:${spec.key}`); const pick = (list) => list[Math.floor(rand() * list.length)]; + // Overlap draws come from their own stream, so they never shift `rand`. + const overlapRand = random(`overlap:${spec.key}`); + const overlapPick = (list) => list[Math.floor(overlapRand() * list.length)]; + const realNames = OVERLAP_SUBSYSTEMS[spec.key] ?? []; + if (overlap > 0 && !realNames.length) + throw new Error(`No overlap vocabulary for ${spec.key}`); + const measure = (lo, hi, draw) => + +(lo + draw() * (hi - lo)).toFixed(lo < 1 ? 1 : 0); + /** + * Candidate words for one filler item (subsystem, metric, value, and + * question), best first. An item outside the overlap share gets only its + * plain words; one inside gets five drafts with real names and + * housekeeping metrics, then its plain words as the last resort. + */ + const overlapWords = (plain) => { + if (!(overlap > 0) || overlapRand() >= overlap) return [plain]; + const tries = []; + for (let i = 0; i < 5; i++) { + const [metric, unit, lo, hi] = overlapPick(OVERLAP_METRICS); + tries.push({ + sub: overlapPick(realNames), + metric, + unit, + value: measure(lo, hi, overlapRand), + question: overlapPick(OVERLAP_QUESTIONS), + }); + } + return [...tries, plain]; + }; + const firstClean = (options, render) => + options.find( + (words, i) => + i === options.length - 1 || + !render(words).some((event) => collides(event)) + ); const at = (ms) => new Date(Math.floor(ms / 60e3) * 60e3).toISOString(); const events = []; let n = 0; @@ -209,52 +347,59 @@ export function fillerEvents(spec, scale) { // A small thread: a question, the measurement that answers it, and the // accepted Decision that closes it. Its links stay inside the thread. const question = pick(QUESTIONS); + const [metric, unit, lo, hi] = pick(METRICS); + const value = measure(lo, hi, rand); + const verdict = pick(VERDICTS); + const doIt = rand() < 0.5; + const cheap = rand() < 0.5; + const followup = pick(FOLLOWUPS); const issue = `fill-iss-${id}`; const finding = `fill-fnd-${id}`; const decision = `fill-dec-${id}`; - const [metric, unit, lo, hi] = pick(METRICS); - const value = +(lo + rand() * (hi - lo)).toFixed(lo < 1 ? 1 : 0); + const render = (w) => { + const amount = `${w.value}${w.unit ? ` ${w.unit}` : ''}`; + return [ + { + at: at(time), + key: issue, + type: 'WriteIssue', + kind: 'question', + title: `Whether the ${w.sub} ${w.question} (${id})`, + body: `${person} asked whether the ${w.sub} ${w.question}. It came up while looking at build ${build} on ${platform}.`, + created_by: person, + }, + { + at: at(time + 3600e3), + key: finding, + type: 'WriteFinding', + kind: 'measurement', + title: `${capitalize(w.sub)} ${ + w.metric + } is ${amount} on ${platform} (${id})`, + body: `${person} measured the ${w.sub} ${w.metric} on ${platform}, build ${build}: ${amount}. That is ${verdict}.`, + derives_from: [issue], + created_by: person, + }, + { + at: at(time + 2 * 3600e3), + key: decision, + type: 'WriteDecision', + title: `${capitalize(w.sub)}: ${ + doIt ? 'do it' : 'leave it as is' + } (${id})`, + body: `The ${w.sub} ${ + w.metric + } is ${amount} on ${platform}, so the change is ${ + cheap ? 'cheap enough to make now' : 'not worth making this year' + }. ${followup}`, + derives_from: [issue, finding], + created_by: person, + }, + ]; + }; + const plain = { sub, question, metric, unit, value }; events.push( - { - at: at(time), - key: issue, - type: 'WriteIssue', - kind: 'question', - title: `Whether the ${sub} ${question} (${id})`, - body: `${person} asked whether the ${sub} ${question}. It came up while looking at build ${build} on ${platform}.`, - created_by: person, - }, - { - at: at(time + 3600e3), - key: finding, - type: 'WriteFinding', - kind: 'measurement', - title: `${capitalize(sub)} ${metric} is ${value}${ - unit ? ` ${unit}` : '' - } on ${platform} (${id})`, - body: `${person} measured the ${sub} ${metric} on ${platform}, build ${build}: ${value}${ - unit ? ` ${unit}` : '' - }. That is ${pick(VERDICTS)}.`, - derives_from: [issue], - created_by: person, - }, - { - at: at(time + 2 * 3600e3), - key: decision, - type: 'WriteDecision', - title: `${capitalize(sub)}: ${ - rand() < 0.5 ? 'do it' : 'leave it as is' - } (${id})`, - body: `The ${sub} ${metric} is ${value}${ - unit ? ` ${unit}` : '' - } on ${platform}, so the change is ${ - rand() < 0.5 - ? 'cheap enough to make now' - : 'not worth making this year' - }. ${pick(FOLLOWUPS)}`, - derives_from: [issue, finding], - created_by: person, - }, + ...render(firstClean(overlapWords(plain), render)), { at: at(time + 3 * 3600e3), type: 'AcceptDecision', @@ -270,36 +415,47 @@ export function fillerEvents(spec, scale) { ); budget -= 3; } else if (roll < 0.4) { - events.push({ - at: at(time), - key: `fill-cit-${id}`, - type: 'WriteCitation', - title: `${capitalize(sub)} capture, build ${build} (${id})`, - body: `https://perf.tidewater.example/${spec.key}/${sub.replace( - /\W+/g, - '-' - )}/${build}`, - role: 'evidence', - created_by: person, - }); + const render = (w) => [ + { + at: at(time), + key: `fill-cit-${id}`, + type: 'WriteCitation', + title: `${capitalize(w.sub)} capture, build ${build} (${id})`, + body: `https://perf.tidewater.example/${spec.key}/${w.sub.replace( + /\W+/g, + '-' + )}/${build}`, + role: 'evidence', + created_by: person, + }, + ]; + events.push(...render(firstClean(overlapWords({ sub }), render))); budget -= 1; } else { const [metric, unit, lo, hi] = pick(METRICS); - const value = +(lo + rand() * (hi - lo)).toFixed(lo < 1 ? 1 : 0); + const value = measure(lo, hi, rand); const runs = 3 + Math.floor(rand() * 8); - events.push({ - at: at(time), - key: `fill-fnd-${id}`, - type: 'WriteFinding', - kind: 'measurement', - title: `${capitalize( - sub - )} ${metric} on ${platform}, build ${build} (${id})`, - body: `${person} measured the ${sub} ${metric} on ${platform}, build ${build}, over ${runs} runs: ${value}${ - unit ? ` ${unit}` : '' - }. That is ${pick(VERDICTS)}. ${pick(FOLLOWUPS)}`, - created_by: person, - }); + const verdict = pick(VERDICTS); + const followup = pick(FOLLOWUPS); + const render = (w) => [ + { + at: at(time), + key: `fill-fnd-${id}`, + type: 'WriteFinding', + kind: 'measurement', + title: `${capitalize(w.sub)} ${ + w.metric + } on ${platform}, build ${build} (${id})`, + body: `${person} measured the ${w.sub} ${ + w.metric + } on ${platform}, build ${build}, over ${runs} runs: ${w.value}${ + w.unit ? ` ${w.unit}` : '' + }. That is ${verdict}. ${followup}`, + created_by: person, + }, + ]; + const plain = { sub, metric, unit, value }; + events.push(...render(firstClean(overlapWords(plain), render))); budget -= 1; } }