From 67c209b71e2a78686c02a54e9283375eb0d0b619 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 20 Oct 2026 05:23:06 +0000 Subject: [PATCH] docs(bench): record the Phase 2 trial of code and branch reads Haiku answered the 22 code-context and 22 branch questions twice on the git test bed, with `findcode` (proof find --path and --refs all) or with read-only git alone: - correct: 85 against 84 of 88 question-runs (2 to 1); - read tokens: 11,642 against 26,283 (56% fewer); tool calls 7 against 13. Both arms are near the ceiling on a test bed this small, so the trial shows the saving, not an accuracy gain. The README and a measurement Finding say so, and name the larger test bed needed next. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01DVVnhYL1Bz7mCfEKDoyLMW --- ...-the-reading-on-a-sma--4ffp9s93s9swpctv.md | 16 + benchmarks/proof-recall/README.md | 30 ++ .../proof-recall/runs/p2-r1-findcode.json | 373 +++++++++++++++++ .../proof-recall/runs/p2-r1-nobrief.json | 375 ++++++++++++++++++ .../proof-recall/runs/p2-r2-findcode.json | 373 +++++++++++++++++ .../proof-recall/runs/p2-r2-nobrief.json | 373 +++++++++++++++++ 6 files changed, 1540 insertions(+) create mode 100644 .flatbread-proof/findings/fnd-code-and-branch-reads-halve-the-reading-on-a-sma--4ffp9s93s9swpctv.md create mode 100644 benchmarks/proof-recall/runs/p2-r1-findcode.json create mode 100644 benchmarks/proof-recall/runs/p2-r1-nobrief.json create mode 100644 benchmarks/proof-recall/runs/p2-r2-findcode.json create mode 100644 benchmarks/proof-recall/runs/p2-r2-nobrief.json diff --git a/.flatbread-proof/findings/fnd-code-and-branch-reads-halve-the-reading-on-a-sma--4ffp9s93s9swpctv.md b/.flatbread-proof/findings/fnd-code-and-branch-reads-halve-the-reading-on-a-sma--4ffp9s93s9swpctv.md new file mode 100644 index 00000000..67e5173a --- /dev/null +++ b/.flatbread-proof/findings/fnd-code-and-branch-reads-halve-the-reading-on-a-sma--4ffp9s93s9swpctv.md @@ -0,0 +1,16 @@ +--- +id: fnd-code-and-branch-reads-halve-the-reading-on-a-sma--4ffp9s93s9swpctv +effort: eff-effort-graph-memory-and-agent-wedge--szeqvmgqjqnhd002 +title: Code and branch reads halve the reading on a small test bed +kind: measurement +created_at: '2026-10-20T05:22:58.158Z' +derives_from: + - fnd-search-cards-raise-accuracy-and-cut-reading-on-t--ydqm2ybyqjedzw9y +--- + +Phase 2 added two reads for teams with many branches: `proof find --path` lists the records that govern a piece of code, and `proof find --refs all` overlays what other branches propose, with a CONFLICTS block. Haiku answered 22 code-context and 22 branch questions twice on a git test bed, each session in its own clone, with the new reads or with read-only git alone. + +- Accuracy matched: 85 against 84 of 88 question-runs (2 to 1). +- The new reads cut read tokens by 56 percent (11,642 against 26,283) and tool calls from 13 to 7, on both question sets. + +The test bed is small (524 records, about 40 source files), so git and grep reach every answer and both arms sit near the ceiling. The saving is real; an accuracy gain needs a test bed large enough that crawling stops working, as the 10x graph showed for search. diff --git a/benchmarks/proof-recall/README.md b/benchmarks/proof-recall/README.md index 8c3cef29..f2788b60 100644 --- a/benchmarks/proof-recall/README.md +++ b/benchmarks/proof-recall/README.md @@ -1334,3 +1334,33 @@ The accuracy gain is all on the tune questions this round was chosen from before and had little room. The saving in reading carries over: 15% fewer tokens on holdout. So this round is kept for clearer, cheaper reads; it has not shown that it raises accuracy on questions it was not built from. + +### Phase 2 trial: code context and branches + +Haiku, twice, on the 22 code-context and 22 branch questions of the git +test bed (`build-repo.mjs`), each session in its own clone: +`run.mjs --repo --strategies findcode,nobrief --only code,branch`. +In this mode every arm may use read-only git, so no-brief can run +`git log`, `git show :` and `git diff main...origin/`; +`findcode` adds `proof find --path` and `--refs all`. Run files: +`runs/p2-r{1,2}-*.json`. + +Pooled over both runs (88 question-runs per arm): + +| Measure | `findcode` | no-brief | +| ------------------------------- | ----------- | ----------- | +| Correct | 85 | 84 | +| Only this arm right | 2 | 1 | +| Mean read tokens | 11,642 | 26,283 | +| Mean tool calls | 7 | 13 | +| Code set: correct / read tok. | 42 / 10,186 | 41 / 29,912 | +| Branch set: correct / read tok. | 43 / 13,097 | 43 / 22,654 | + +Both arms are near the ceiling, so this trial cannot show an accuracy +gain: with git on a 524-record graph and about 40 source files, an agent +finds these answers by grepping and reading branches directly. What the +new reads change is cost: 56% fewer read tokens and half the tool calls, +on both sets. The 10x test showed that accuracy separates only once +crawling stops scaling; the same test is needed here, on a test bed with +the 10x graph, a larger codebase and more branches, before claiming more +than the saving. diff --git a/benchmarks/proof-recall/runs/p2-r1-findcode.json b/benchmarks/proof-recall/runs/p2-r1-findcode.json new file mode 100644 index 00000000..10cf0e2c --- /dev/null +++ b/benchmarks/proof-recall/runs/p2-r1-findcode.json @@ -0,0 +1,373 @@ +{ + "run": { + "name": "p2-r1-findcode", + "protocol": "one fresh session per question", + "models": ["claude-haiku-5-5"], + "find_hash": "c80612938d90f14747e3be3341c6434f3d78efcbd611bf5085facc8653eee5c2", + "state_hash": "be19faeb18e0059bde6482cc2b7e2f1df8368d691e4a903ce1b63cc9aa5a177b", + "read_hash": "6e94ef68632e522abca8eaf873b3238cf4ae45394a4810e66f1595c3fe2ee839" + }, + "answers": { + "branch-001": { + "answer": 41, + "tokens": 12417, + "fixed_tokens": 2730, + "processed_tokens": 83595, + "tool_calls": 6, + "turns": 7 + }, + "branch-002": { + "answer": "dec-use-virtual-shadow-maps-for-the-sun--x8nqkn9q7eaf1ap4", + "tokens": 17155, + "fixed_tokens": 2736, + "processed_tokens": 147605, + "tool_calls": 9, + "turns": 10 + }, + "branch-003": { + "answer": "proposed", + "tokens": 12402, + "fixed_tokens": 2731, + "processed_tokens": 98949, + "tool_calls": 7, + "turns": 8 + }, + "branch-004": { + "answer": "ines/vsm-pool", + "tokens": 11569, + "fixed_tokens": 1107, + "processed_tokens": 103615, + "tool_calls": 8, + "turns": 9 + }, + "branch-005": { + "answer": "iss-how-should-the-tide-simulation-save-time-in-cell--72k2mwf4nb3g978p", + "tokens": 18214, + "fixed_tokens": 1112, + "processed_tokens": 86232, + "tool_calls": 6, + "turns": 6 + }, + "branch-006": { + "answer": "dec-step-tide-cells-beyond-600-m-at-a-quarter-of-the--1zccsxxw2014sh20", + "tokens": 13243, + "fixed_tokens": 1103, + "processed_tokens": 118836, + "tool_calls": 10, + "turns": 9 + }, + "branch-007": { + "answer": 48, + "tokens": 11219, + "fixed_tokens": 1105, + "processed_tokens": 55240, + "tool_calls": 4, + "turns": 5 + }, + "branch-008": { + "answer": "fnd-tide-storms-push-skiff-40-hz-frames-to-28-4-ms-o--28gemvtznv4zztdc", + "tokens": 10229, + "fixed_tokens": 1103, + "processed_tokens": 77173, + "tool_calls": 6, + "turns": 7 + }, + "branch-009": { + "answer": "27.0 ms", + "tokens": 15945, + "fixed_tokens": 1106, + "processed_tokens": 118937, + "tool_calls": 11, + "turns": 9 + }, + "branch-010": { + "answer": [ + "fnd-world-core-state-with-tide-level-of-detail--x947x5kj16hcngef", + "fnd-lantern-after-the-spring-tide-page-pool-change--9zbcsqh53mgv7h4p" + ], + "tokens": 20542, + "fixed_tokens": 1104, + "processed_tokens": 187207, + "tool_calls": 15, + "turns": 11 + }, + "branch-011": { + "answer": false, + "tokens": 11326, + "fixed_tokens": 1105, + "processed_tokens": 70380, + "tool_calls": 5, + "turns": 6 + }, + "branch-012": { + "answer": "superseded", + "tokens": 16603, + "fixed_tokens": 1107, + "processed_tokens": 117576, + "tool_calls": 8, + "turns": 8 + }, + "branch-013": { + "answer": "kenji/tide-lod", + "tokens": 12013, + "fixed_tokens": 1107, + "processed_tokens": 56957, + "tool_calls": 4, + "turns": 5 + }, + "branch-014": { + "answer": "Priya Natarajan", + "tokens": 9130, + "fixed_tokens": 1105, + "processed_tokens": 53014, + "tool_calls": 4, + "turns": 5 + }, + "branch-015": { + "answer": 3, + "tokens": 17935, + "fixed_tokens": 1103, + "processed_tokens": 146488, + "tool_calls": 10, + "turns": 10 + }, + "branch-016": { + "answer": "proposed", + "tokens": 14322, + "fixed_tokens": 1105, + "processed_tokens": 81055, + "tool_calls": 5, + "turns": 6 + }, + "branch-017": { + "answer": 46, + "tokens": 7610, + "fixed_tokens": 1104, + "processed_tokens": 48341, + "tool_calls": 5, + "turns": 5 + }, + "branch-018": { + "answer": "dec-use-virtual-shadow-maps-for-the-sun-with-a-6-144--gnxz9wk2q9xsxc96", + "tokens": 19648, + "fixed_tokens": 1106, + "processed_tokens": 127915, + "tool_calls": 7, + "turns": 8 + }, + "branch-019": { + "answer": false, + "tokens": 11853, + "fixed_tokens": 1108, + "processed_tokens": 64181, + "tool_calls": 9, + "turns": 6 + }, + "branch-020": { + "answer": "origin/ines/vsm-pool and origin/tomasz/vsm-evict would replace main's accepted sun-shadow Decision dec-use-virtual-shadow-maps-for-the-sun--x8nqkn9q7eaf1ap4.", + "tokens": 11042, + "fixed_tokens": 1103, + "processed_tokens": 68511, + "tool_calls": 5, + "turns": 6 + }, + "branch-021": { + "answer": "dec-cap-tide-storm-particles-at-6-000-on-skiff--resze52afjxyxys1", + "tokens": 14154, + "fixed_tokens": 1100, + "processed_tokens": 95002, + "tool_calls": 6, + "turns": 7 + }, + "branch-022": { + "answer": "priya/skiff-cpu", + "tokens": 29585, + "fixed_tokens": 1108, + "processed_tokens": 155971, + "tool_calls": 11, + "turns": 9 + }, + "code-001": { + "answer": "dec-use-virtual-shadow-maps-for-the-sun--x8nqkn9q7eaf1ap4", + "tokens": 10633, + "fixed_tokens": 1099, + "processed_tokens": 78878, + "tool_calls": 6, + "turns": 7 + }, + "code-002": { + "answer": "dec-key-the-shader-cache-on-preprocessed-source-defi--3a05ff0r954vxh30", + "tokens": 11541, + "fixed_tokens": 1100, + "processed_tokens": 55852, + "tool_calls": 4, + "turns": 5 + }, + "code-003": { + "answer": [ + "rsk-eviction-scoring-asks-the-tide-forecast-about-ev--zfwghmgds54h8y46" + ], + "tokens": 19059, + "fixed_tokens": 1102, + "processed_tokens": 160501, + "tool_calls": 10, + "turns": 11 + }, + "code-004": { + "answer": "fnd-pool-eviction-keeps-cells-the-tide-exposes-withi--tzseexbv0m0qw2jk", + "tokens": 15547, + "fixed_tokens": 1101, + "processed_tokens": 124477, + "tool_calls": 9, + "turns": 10 + }, + "code-005": { + "answer": false, + "tokens": 16487, + "fixed_tokens": 1100, + "processed_tokens": 120990, + "tool_calls": 10, + "turns": 9 + }, + "code-006": { + "answer": "fnd-tide-priority-reads-starve-player-proximity-load--by0d7b0bk39xz3rk", + "tokens": 8293, + "fixed_tokens": 1105, + "processed_tokens": 60761, + "tool_calls": 5, + "turns": 6 + }, + "code-007": { + "answer": "dec-cap-each-farm-agent-at-6-parallel-shader-compile--q5w4sfjm73tsj8x8", + "tokens": 13180, + "fixed_tokens": 1103, + "processed_tokens": 92205, + "tool_calls": 6, + "turns": 7 + }, + "code-008": { + "answer": "iss-a-timed-out-cache-fetch-is-retried-once-and-then--sn5thmgf02v8rfe4", + "tokens": 12751, + "fixed_tokens": 1102, + "processed_tokens": 99584, + "tool_calls": 7, + "turns": 8 + }, + "code-009": { + "answer": "600p", + "tokens": 13254, + "fixed_tokens": 1098, + "processed_tokens": 83004, + "tool_calls": 6, + "turns": 7 + }, + "code-010": { + "answer": "iss-skiff-stays-at-the-lowest-resolution-for-two-sec--erke2h7xf7vepdv2", + "tokens": 11672, + "fixed_tokens": 1104, + "processed_tokens": 71187, + "tool_calls": 6, + "turns": 6 + }, + "code-011": { + "answer": "dec-visibility-buffer-on-all-platforms-skiff-resolve--k1ackk3c3275r5y1", + "tokens": 19582, + "fixed_tokens": 1103, + "processed_tokens": 111356, + "tool_calls": 8, + "turns": 8 + }, + "code-012": { + "answer": "dec-ship-four-tern-quality-presets-adding-ultra-perf--3jg1wyq3bshv839c", + "tokens": 10744, + "fixed_tokens": 1099, + "processed_tokens": 61613, + "tool_calls": 6, + "turns": 6 + }, + "code-013": { + "answer": [ + "dec-add-a-second-tide-harmonic-for-spring-and-neap-t--1c3s7dtyxvj70gvw", + "dec-wrap-the-tide-phase-accumulator-at-each-40-minut--hx88m4hbg27n0xwy", + "dec-run-the-tide-simulation-in-32-32-fixed-point--c5rjfcnmhne4h05e" + ], + "tokens": 8455, + "fixed_tokens": 1102, + "processed_tokens": 56327, + "tool_calls": 9, + "turns": 6 + }, + "code-014": { + "answer": "con-the-tide-step-allocates-no-memory--bgce2y8wndwrwe0r", + "tokens": 10473, + "fixed_tokens": 1098, + "processed_tokens": 82387, + "tool_calls": 6, + "turns": 7 + }, + "code-015": { + "answer": "con-saves-from-beta-onward-must-load-in-every-later--v6z856sxtgc5sb09", + "tokens": 7743, + "fixed_tokens": 1101, + "processed_tokens": 59385, + "tool_calls": 5, + "turns": 6 + }, + "code-016": { + "answer": "false, fnd-tern-zeroes-history-weight-wherever-saltglass-se--hhgcrcq2n03rwr53", + "tokens": 13111, + "fixed_tokens": 1108, + "processed_tokens": 90747, + "tool_calls": 6, + "turns": 7 + }, + "code-017": { + "answer": "A checksum mismatch names the state block (Tide, physics, or Gull state) that diverged and the frame window in which it happened. Record: dec-hash-tide-physics-and-gull-state-every-30-frames--yqf2m6hqbjhwvkzq", + "tokens": 6962, + "fixed_tokens": 1101, + "processed_tokens": 55361, + "tool_calls": 5, + "turns": 6 + }, + "code-018": { + "answer": "unknown", + "tokens": 10562, + "fixed_tokens": 1102, + "processed_tokens": 61306, + "tool_calls": 5, + "turns": 6 + }, + "code-019": { + "answer": "800 nits", + "tokens": 9499, + "fixed_tokens": 1110, + "processed_tokens": 49980, + "tool_calls": 4, + "turns": 5 + }, + "code-020": { + "answer": "dec-ship-prebuilt-pipeline-blobs-in-the-skiff-packag--3vjysjgy9qkvzebn", + "tokens": 6577, + "fixed_tokens": 1103, + "processed_tokens": 54358, + "tool_calls": 5, + "turns": 6 + }, + "code-021": { + "answer": "iss-shadow-pages-run-out-at-spring-low-tide--f4fgy6b44jbkrsmh", + "tokens": 11949, + "fixed_tokens": 1100, + "processed_tokens": 94976, + "tool_calls": 7, + "turns": 8 + }, + "code-022": { + "answer": "fnd-sun-shadow-pages-are-evicted-oldest-sampled-firs--ndrr54bv4nvr1mbx", + "tokens": 12232, + "fixed_tokens": 1104, + "processed_tokens": 80999, + "tool_calls": 6, + "turns": 7 + } + } +} diff --git a/benchmarks/proof-recall/runs/p2-r1-nobrief.json b/benchmarks/proof-recall/runs/p2-r1-nobrief.json new file mode 100644 index 00000000..012ba9d3 --- /dev/null +++ b/benchmarks/proof-recall/runs/p2-r1-nobrief.json @@ -0,0 +1,375 @@ +{ + "run": { + "name": "p2-r1-nobrief", + "protocol": "one fresh session per question", + "models": ["claude-haiku-5-5"], + "find_hash": "c80612938d90f14747e3be3341c6434f3d78efcbd611bf5085facc8653eee5c2", + "state_hash": "be19faeb18e0059bde6482cc2b7e2f1df8368d691e4a903ce1b63cc9aa5a177b", + "read_hash": "6e94ef68632e522abca8eaf873b3238cf4ae45394a4810e66f1595c3fe2ee839" + }, + "answers": { + "branch-001": { + "answer": 41, + "tokens": 32855, + "fixed_tokens": 2735, + "processed_tokens": 203011, + "tool_calls": 11, + "turns": 10 + }, + "branch-002": { + "answer": "dec-use-virtual-shadow-maps-for-the-sun--x8nqkn9q7eaf1ap4", + "tokens": 16037, + "fixed_tokens": 2735, + "processed_tokens": 91672, + "tool_calls": 7, + "turns": 8 + }, + "branch-003": { + "answer": "proposed", + "tokens": 8526, + "fixed_tokens": 1106, + "processed_tokens": 87525, + "tool_calls": 11, + "turns": 9 + }, + "branch-004": { + "answer": "ines/vsm-pool", + "tokens": 39917, + "fixed_tokens": 1109, + "processed_tokens": 360603, + "tool_calls": 13, + "turns": 14 + }, + "branch-005": { + "answer": "unknown", + "tokens": 35721, + "fixed_tokens": 1110, + "processed_tokens": 254153, + "tool_calls": 11, + "turns": 12 + }, + "branch-006": { + "answer": "dec-step-tide-cells-beyond-600-m-at-a-quarter-of-the--1zccsxxw2014sh20", + "tokens": 35010, + "fixed_tokens": 1108, + "processed_tokens": 309767, + "tool_calls": 20, + "turns": 13 + }, + "branch-007": { + "answer": 48, + "tokens": 100499, + "fixed_tokens": 1104, + "processed_tokens": 1175236, + "tool_calls": 21, + "turns": 21 + }, + "branch-008": { + "answer": "fnd-tide-storms-push-skiff-40-hz-frames-to-28-4-ms-o--28gemvtznv4zztdc", + "tokens": 13722, + "fixed_tokens": 1108, + "processed_tokens": 153723, + "tool_calls": 15, + "turns": 13 + }, + "branch-009": { + "answer": "27.0 ms", + "tokens": 8682, + "fixed_tokens": 1113, + "processed_tokens": 67808, + "tool_calls": 10, + "turns": 7 + }, + "branch-010": { + "answer": [ + "fnd-lantern-after-the-spring-tide-page-pool-change--9zbcsqh53mgv7h4p", + "fnd-world-core-state-with-tide-level-of-detail--x947x5kj16hcngef" + ], + "tokens": 31264, + "fixed_tokens": 1110, + "processed_tokens": 148245, + "tool_calls": 14, + "turns": 8 + }, + "branch-011": { + "answer": false, + "tokens": 29613, + "fixed_tokens": 1111, + "processed_tokens": 246530, + "tool_calls": 13, + "turns": 12 + }, + "branch-012": { + "answer": "superseded", + "tokens": 34105, + "fixed_tokens": 1105, + "processed_tokens": 386139, + "tool_calls": 16, + "turns": 17 + }, + "branch-013": { + "answer": "kenji/tide-lod", + "tokens": 21324, + "fixed_tokens": 1110, + "processed_tokens": 224411, + "tool_calls": 14, + "turns": 15 + }, + "branch-014": { + "answer": "Priya Natarajan", + "tokens": 7407, + "fixed_tokens": 1104, + "processed_tokens": 53907, + "tool_calls": 9, + "turns": 6 + }, + "branch-015": { + "answer": 3, + "tokens": 14467, + "fixed_tokens": 1107, + "processed_tokens": 102188, + "tool_calls": 10, + "turns": 9 + }, + "branch-016": { + "answer": "proposed", + "tokens": 11923, + "fixed_tokens": 1106, + "processed_tokens": 115174, + "tool_calls": 10, + "turns": 10 + }, + "branch-017": { + "answer": 46, + "tokens": 6668, + "fixed_tokens": 1108, + "processed_tokens": 52452, + "tool_calls": 5, + "turns": 6 + }, + "branch-018": { + "answer": "dec-use-virtual-shadow-maps-for-the-sun-with-a-6-144--gnxz9wk2q9xsxc96", + "tokens": 27799, + "fixed_tokens": 1107, + "processed_tokens": 275741, + "tool_calls": 16, + "turns": 15 + }, + "branch-019": { + "answer": false, + "tokens": 6389, + "fixed_tokens": 1110, + "processed_tokens": 53040, + "tool_calls": 9, + "turns": 6 + }, + "branch-020": { + "answer": "Two open branches would replace it: `ines/vsm-pool` (its accepted `dec-use-virtual-shadow-maps-for-the-sun-with-a-6-144--gnxz9wk2q9xsxc96` supersedes `dec-use-virtual-shadow-maps-for-the-sun--x8nqkn9q7eaf1ap4`) and `tomasz/vsm-evict` (its proposed `dec-use-virtual-shadow-maps-for-the-sun-with-coarser--jtcbmrm4c48ma8z8` supersedes it).", + "tokens": 28766, + "fixed_tokens": 1107, + "processed_tokens": 247302, + "tool_calls": 17, + "turns": 12 + }, + "branch-021": { + "answer": "dec-cap-tide-storm-particles-at-6-000-on-skiff--resze52afjxyxys1", + "tokens": 12947, + "fixed_tokens": 1110, + "processed_tokens": 126480, + "tool_calls": 10, + "turns": 11 + }, + "branch-022": { + "answer": "priya/skiff-cpu", + "tokens": 31251, + "fixed_tokens": 1113, + "processed_tokens": 202891, + "tool_calls": 11, + "turns": 11 + }, + "code-001": { + "answer": "dec-use-virtual-shadow-maps-for-the-sun--x8nqkn9q7eaf1ap4", + "tokens": 42075, + "fixed_tokens": 1099, + "processed_tokens": 444596, + "tool_calls": 14, + "turns": 15 + }, + "code-002": { + "answer": "dec-key-the-shader-cache-on-preprocessed-source-defi--3a05ff0r954vxh30", + "tokens": 14980, + "fixed_tokens": 1103, + "processed_tokens": 137327, + "tool_calls": 10, + "turns": 10 + }, + "code-003": { + "answer": [ + "fnd-field-captures-show-forecast-eviction-cuts-reloa--ee6m7fwv4798jdmv", + "fnd-pool-eviction-keeps-cells-the-tide-exposes-withi--23m2232ez16e3m6w", + "rsk-eviction-scoring-asks-the-tide-forecast-about-ev--zfwghmgds54h8y46" + ], + "tokens": 74170, + "fixed_tokens": 1102, + "processed_tokens": 972033, + "tool_calls": 25, + "turns": 25 + }, + "code-004": { + "answer": "fnd-pool-eviction-keeps-cells-the-tide-exposes-withi--tzseexbv0m0qw2jk", + "tokens": 123314, + "fixed_tokens": 1111, + "processed_tokens": 1543792, + "tool_calls": 28, + "turns": 29 + }, + "code-005": { + "answer": "false dec-give-reads-for-tide-exposed-cells-top-priority--8ftkvqhawbdf5djh", + "tokens": 39484, + "fixed_tokens": 1107, + "processed_tokens": 388080, + "tool_calls": 14, + "turns": 14 + }, + "code-006": { + "answer": "fnd-tide-priority-reads-starve-player-proximity-load--by0d7b0bk39xz3rk", + "tokens": 35294, + "fixed_tokens": 1102, + "processed_tokens": 473338, + "tool_calls": 18, + "turns": 18 + }, + "code-007": { + "answer": "dec-cap-each-farm-agent-at-6-parallel-shader-compile--q5w4sfjm73tsj8x8", + "tokens": 15848, + "fixed_tokens": 1102, + "processed_tokens": 137385, + "tool_calls": 9, + "turns": 10 + }, + "code-008": { + "answer": "iss-a-timed-out-cache-fetch-is-retried-once-and-then--sn5thmgf02v8rfe4", + "tokens": 28329, + "fixed_tokens": 1110, + "processed_tokens": 204704, + "tool_calls": 9, + "turns": 10 + }, + "code-009": { + "answer": "600p", + "tokens": 20560, + "fixed_tokens": 1099, + "processed_tokens": 136210, + "tool_calls": 8, + "turns": 9 + }, + "code-010": { + "answer": "iss-skiff-stays-at-the-lowest-resolution-for-two-sec--erke2h7xf7vepdv2", + "tokens": 15614, + "fixed_tokens": 1111, + "processed_tokens": 133079, + "tool_calls": 9, + "turns": 10 + }, + "code-011": { + "answer": "dec-visibility-buffer-on-all-platforms-skiff-resolve--k1ackk3c3275r5y1", + "tokens": 18928, + "fixed_tokens": 1108, + "processed_tokens": 192052, + "tool_calls": 14, + "turns": 14 + }, + "code-012": { + "answer": "dec-ship-four-tern-quality-presets-adding-ultra-perf--3jg1wyq3bshv839c", + "tokens": 57534, + "fixed_tokens": 1106, + "processed_tokens": 516458, + "tool_calls": 15, + "turns": 16 + }, + "code-013": { + "answer": [ + "dec-add-a-second-tide-harmonic-for-spring-and-neap-t--1c3s7dtyxvj70gvw", + "dec-wrap-the-tide-phase-accumulator-at-each-40-minut--hx88m4hbg27n0xwy", + "dec-run-the-tide-simulation-in-32-32-fixed-point--c5rjfcnmhne4h05e" + ], + "tokens": 9942, + "fixed_tokens": 1106, + "processed_tokens": 77662, + "tool_calls": 11, + "turns": 8 + }, + "code-014": { + "answer": "con-the-tide-step-allocates-no-memory--bgce2y8wndwrwe0r", + "tokens": 17409, + "fixed_tokens": 1104, + "processed_tokens": 119662, + "tool_calls": 8, + "turns": 9 + }, + "code-015": { + "answer": "con-saves-from-beta-onward-must-load-in-every-later--v6z856sxtgc5sb09", + "tokens": 31572, + "fixed_tokens": 1108, + "processed_tokens": 299322, + "tool_calls": 13, + "turns": 14 + }, + "code-016": { + "answer": "false, dec-feed-a-saltglass-reactive-mask-into-tern--xezjs7v3vq3r8e6k", + "tokens": 34593, + "fixed_tokens": 1102, + "processed_tokens": 408933, + "tool_calls": 17, + "turns": 17 + }, + "code-017": { + "answer": "A checksum mismatch names the state block (tide, physics, or Gull state) that diverged and the frame window in which it happened, per dec-hash-tide-physics-and-gull-state-every-30-frames--yqf2m6hqbjhwvkzq.", + "tokens": 11618, + "fixed_tokens": 1106, + "processed_tokens": 86347, + "tool_calls": 7, + "turns": 8 + }, + "code-018": { + "answer": "unknown", + "tokens": 49113, + "fixed_tokens": 1103, + "processed_tokens": 560435, + "tool_calls": 16, + "turns": 17 + }, + "code-019": { + "answer": "800 nits", + "tokens": 38952, + "fixed_tokens": 1105, + "processed_tokens": 437447, + "tool_calls": 14, + "turns": 15 + }, + "code-020": { + "answer": "dec-ship-prebuilt-pipeline-blobs-in-the-skiff-packag--3vjysjgy9qkvzebn", + "tokens": 23032, + "fixed_tokens": 1104, + "processed_tokens": 196175, + "tool_calls": 10, + "turns": 11 + }, + "code-021": { + "answer": "iss-shadow-pages-run-out-at-spring-low-tide--f4fgy6b44jbkrsmh", + "tokens": 62606, + "fixed_tokens": 1111, + "processed_tokens": 448965, + "tool_calls": 18, + "turns": 15 + }, + "code-022": { + "answer": "fnd-sun-shadow-pages-are-evicted-oldest-sampled-firs--ndrr54bv4nvr1mbx", + "tokens": 51591, + "fixed_tokens": 1101, + "processed_tokens": 578619, + "tool_calls": 18, + "turns": 19 + } + } +} diff --git a/benchmarks/proof-recall/runs/p2-r2-findcode.json b/benchmarks/proof-recall/runs/p2-r2-findcode.json new file mode 100644 index 00000000..37aa9441 --- /dev/null +++ b/benchmarks/proof-recall/runs/p2-r2-findcode.json @@ -0,0 +1,373 @@ +{ + "run": { + "name": "p2-r2-findcode", + "protocol": "one fresh session per question", + "models": ["claude-haiku-5-5"], + "find_hash": "c80612938d90f14747e3be3341c6434f3d78efcbd611bf5085facc8653eee5c2", + "state_hash": "be19faeb18e0059bde6482cc2b7e2f1df8368d691e4a903ce1b63cc9aa5a177b", + "read_hash": "6e94ef68632e522abca8eaf873b3238cf4ae45394a4810e66f1595c3fe2ee839" + }, + "answers": { + "branch-001": { + "answer": "#41", + "tokens": 13188, + "fixed_tokens": 1106, + "processed_tokens": 89507, + "tool_calls": 6, + "turns": 7 + }, + "branch-002": { + "answer": "dec-use-virtual-shadow-maps-for-the-sun--x8nqkn9q7eaf1ap4", + "tokens": 14881, + "fixed_tokens": 1103, + "processed_tokens": 157312, + "tool_calls": 10, + "turns": 11 + }, + "branch-003": { + "answer": "proposed", + "tokens": 9830, + "fixed_tokens": 1110, + "processed_tokens": 62619, + "tool_calls": 5, + "turns": 6 + }, + "branch-004": { + "answer": "origin/ines/vsm-pool", + "tokens": 9591, + "fixed_tokens": 1103, + "processed_tokens": 67086, + "tool_calls": 5, + "turns": 6 + }, + "branch-005": { + "answer": "iss-how-should-the-tide-simulation-save-time-in-cell--72k2mwf4nb3g978p", + "tokens": 14696, + "fixed_tokens": 1106, + "processed_tokens": 108578, + "tool_calls": 7, + "turns": 8 + }, + "branch-006": { + "answer": "dec-step-tide-cells-beyond-600-m-at-a-quarter-of-the--1zccsxxw2014sh20", + "tokens": 9455, + "fixed_tokens": 1110, + "processed_tokens": 53667, + "tool_calls": 4, + "turns": 5 + }, + "branch-007": { + "answer": "#48", + "tokens": 12769, + "fixed_tokens": 1105, + "processed_tokens": 58374, + "tool_calls": 6, + "turns": 5 + }, + "branch-008": { + "answer": "fnd-tide-storms-push-skiff-40-hz-frames-to-28-4-ms-o--28gemvtznv4zztdc", + "tokens": 14592, + "fixed_tokens": 1109, + "processed_tokens": 109110, + "tool_calls": 8, + "turns": 8 + }, + "branch-009": { + "answer": "27.0 ms", + "tokens": 12554, + "fixed_tokens": 1108, + "processed_tokens": 102006, + "tool_calls": 7, + "turns": 8 + }, + "branch-010": { + "answer": [ + "fnd-world-core-state-with-tide-level-of-detail--x947x5kj16hcngef", + "fnd-lantern-after-the-spring-tide-page-pool-change--9zbcsqh53mgv7h4p" + ], + "tokens": 15147, + "fixed_tokens": 1100, + "processed_tokens": 79375, + "tool_calls": 12, + "turns": 6 + }, + "branch-011": { + "answer": false, + "tokens": 18279, + "fixed_tokens": 1103, + "processed_tokens": 132039, + "tool_calls": 8, + "turns": 9 + }, + "branch-012": { + "answer": "superseded", + "tokens": 16585, + "fixed_tokens": 1104, + "processed_tokens": 168746, + "tool_calls": 11, + "turns": 12 + }, + "branch-013": { + "answer": "kenji/tide-lod", + "tokens": 11879, + "fixed_tokens": 1106, + "processed_tokens": 56638, + "tool_calls": 4, + "turns": 5 + }, + "branch-014": { + "answer": "Priya Natarajan", + "tokens": 12894, + "fixed_tokens": 1101, + "processed_tokens": 70320, + "tool_calls": 5, + "turns": 6 + }, + "branch-015": { + "answer": 3, + "tokens": 15251, + "fixed_tokens": 1104, + "processed_tokens": 103611, + "tool_calls": 10, + "turns": 9 + }, + "branch-016": { + "answer": "proposed", + "tokens": 11084, + "fixed_tokens": 1103, + "processed_tokens": 70169, + "tool_calls": 5, + "turns": 6 + }, + "branch-017": { + "answer": 46, + "tokens": 8817, + "fixed_tokens": 1104, + "processed_tokens": 74365, + "tool_calls": 6, + "turns": 7 + }, + "branch-018": { + "answer": "dec-use-virtual-shadow-maps-for-the-sun-with-a-6-144--gnxz9wk2q9xsxc96", + "tokens": 21354, + "fixed_tokens": 1106, + "processed_tokens": 174280, + "tool_calls": 9, + "turns": 10 + }, + "branch-019": { + "answer": false, + "tokens": 6091, + "fixed_tokens": 1103, + "processed_tokens": 42260, + "tool_calls": 7, + "turns": 5 + }, + "branch-020": { + "answer": "Two open branches would replace it: ines/vsm-pool and tomasz/vsm-evict. Main's accepted sun-shadow Decision is dec-use-virtual-shadow-maps-for-the-sun--x8nqkn9q7eaf1ap4, which ines/vsm-pool supersedes via dec-use-virtual-shadow-maps-for-the-sun-with-a-6-144--gnxz9wk2q9xsxc96 and tomasz/vsm-evict supersedes via dec-use-virtual-shadow-maps-for-the-sun-with-coarser--jtcbmrm4c48ma8z8.", + "tokens": 15423, + "fixed_tokens": 1102, + "processed_tokens": 107942, + "tool_calls": 9, + "turns": 8 + }, + "branch-021": { + "answer": "dec-cap-tide-storm-particles-at-6-000-on-skiff--resze52afjxyxys1", + "tokens": 17584, + "fixed_tokens": 1102, + "processed_tokens": 131260, + "tool_calls": 9, + "turns": 10 + }, + "branch-022": { + "answer": "priya/skiff-cpu", + "tokens": 29687, + "fixed_tokens": 1106, + "processed_tokens": 237440, + "tool_calls": 13, + "turns": 14 + }, + "code-001": { + "answer": "dec-use-virtual-shadow-maps-for-the-sun--x8nqkn9q7eaf1ap4", + "tokens": 10487, + "fixed_tokens": 1108, + "processed_tokens": 78188, + "tool_calls": 6, + "turns": 7 + }, + "code-002": { + "answer": "dec-key-the-shader-cache-on-preprocessed-source-defi--3a05ff0r954vxh30", + "tokens": 13164, + "fixed_tokens": 1104, + "processed_tokens": 82700, + "tool_calls": 6, + "turns": 7 + }, + "code-003": { + "answer": [ + "rsk-eviction-scoring-asks-the-tide-forecast-about-ev--zfwghmgds54h8y46" + ], + "tokens": 12476, + "fixed_tokens": 1100, + "processed_tokens": 68831, + "tool_calls": 5, + "turns": 6 + }, + "code-004": { + "answer": "fnd-pool-eviction-keeps-cells-the-tide-exposes-withi--tzseexbv0m0qw2jk", + "tokens": 11520, + "fixed_tokens": 1099, + "processed_tokens": 133772, + "tool_calls": 11, + "turns": 12 + }, + "code-005": { + "answer": "false", + "tokens": 12166, + "fixed_tokens": 1102, + "processed_tokens": 98772, + "tool_calls": 7, + "turns": 8 + }, + "code-006": { + "answer": "fnd-tide-priority-reads-starve-player-proximity-load--by0d7b0bk39xz3rk", + "tokens": 8514, + "fixed_tokens": 1098, + "processed_tokens": 61127, + "tool_calls": 5, + "turns": 6 + }, + "code-007": { + "answer": "dec-cap-each-farm-agent-at-6-parallel-shader-compile--q5w4sfjm73tsj8x8", + "tokens": 11727, + "fixed_tokens": 1104, + "processed_tokens": 86086, + "tool_calls": 6, + "turns": 7 + }, + "code-008": { + "answer": "iss-a-timed-out-cache-fetch-is-retried-once-and-then--sn5thmgf02v8rfe4", + "tokens": 9528, + "fixed_tokens": 1100, + "processed_tokens": 75896, + "tool_calls": 6, + "turns": 7 + }, + "code-009": { + "answer": "600p", + "tokens": 13283, + "fixed_tokens": 1104, + "processed_tokens": 83528, + "tool_calls": 6, + "turns": 7 + }, + "code-010": { + "answer": "iss-skiff-stays-at-the-lowest-resolution-for-two-sec--erke2h7xf7vepdv2", + "tokens": 11008, + "fixed_tokens": 1100, + "processed_tokens": 66167, + "tool_calls": 6, + "turns": 6 + }, + "code-011": { + "answer": "dec-visibility-buffer-on-all-platforms-skiff-resolve--k1ackk3c3275r5y1", + "tokens": 11459, + "fixed_tokens": 1103, + "processed_tokens": 79114, + "tool_calls": 7, + "turns": 8 + }, + "code-012": { + "answer": "dec-ship-four-tern-quality-presets-adding-ultra-perf--3jg1wyq3bshv839c", + "tokens": 12049, + "fixed_tokens": 1107, + "processed_tokens": 78826, + "tool_calls": 6, + "turns": 7 + }, + "code-013": { + "answer": [ + "dec-add-a-second-tide-harmonic-for-spring-and-neap-t--1c3s7dtyxvj70gvw", + "dec-wrap-the-tide-phase-accumulator-at-each-40-minut--hx88m4hbg27n0xwy", + "dec-run-the-tide-simulation-in-32-32-fixed-point--c5rjfcnmhne4h05e" + ], + "tokens": 8872, + "fixed_tokens": 1104, + "processed_tokens": 56902, + "tool_calls": 9, + "turns": 6 + }, + "code-014": { + "answer": "con-the-tide-step-allocates-no-memory--bgce2y8wndwrwe0r", + "tokens": 9725, + "fixed_tokens": 1100, + "processed_tokens": 52227, + "tool_calls": 4, + "turns": 5 + }, + "code-015": { + "answer": "con-saves-from-beta-onward-must-load-in-every-later--v6z856sxtgc5sb09", + "tokens": 11088, + "fixed_tokens": 1100, + "processed_tokens": 73565, + "tool_calls": 6, + "turns": 7 + }, + "code-016": { + "answer": "false (fnd-tern-zeroes-history-weight-wherever-saltglass-se--hhgcrcq2n03rwr53)", + "tokens": 8610, + "fixed_tokens": 1101, + "processed_tokens": 61095, + "tool_calls": 5, + "turns": 6 + }, + "code-017": { + "answer": "A checksum mismatch tells you which of the three state blocks (tide, physics, or Gull state) diverged between peers and the frame window in which it happened. Record: dec-hash-tide-physics-and-gull-state-every-30-frames--yqf2m6hqbjhwvkzq", + "tokens": 8341, + "fixed_tokens": 1100, + "processed_tokens": 61165, + "tool_calls": 5, + "turns": 6 + }, + "code-018": { + "answer": "unknown", + "tokens": 10512, + "fixed_tokens": 1102, + "processed_tokens": 61244, + "tool_calls": 5, + "turns": 6 + }, + "code-019": { + "answer": "800 nits", + "tokens": 10935, + "fixed_tokens": 1102, + "processed_tokens": 81052, + "tool_calls": 6, + "turns": 7 + }, + "code-020": { + "answer": "dec-ship-prebuilt-pipeline-blobs-in-the-skiff-packag--3vjysjgy9qkvzebn", + "tokens": 8706, + "fixed_tokens": 1108, + "processed_tokens": 61540, + "tool_calls": 5, + "turns": 6 + }, + "code-021": { + "answer": "iss-shadow-pages-run-out-at-spring-low-tide--f4fgy6b44jbkrsmh", + "tokens": 9826, + "fixed_tokens": 1105, + "processed_tokens": 76918, + "tool_calls": 6, + "turns": 7 + }, + "code-022": { + "answer": "fnd-sun-shadow-pages-are-evicted-oldest-sampled-firs--ndrr54bv4nvr1mbx", + "tokens": 12377, + "fixed_tokens": 1102, + "processed_tokens": 72324, + "tool_calls": 6, + "turns": 6 + } + } +} diff --git a/benchmarks/proof-recall/runs/p2-r2-nobrief.json b/benchmarks/proof-recall/runs/p2-r2-nobrief.json new file mode 100644 index 00000000..daf863fb --- /dev/null +++ b/benchmarks/proof-recall/runs/p2-r2-nobrief.json @@ -0,0 +1,373 @@ +{ + "run": { + "name": "p2-r2-nobrief", + "protocol": "one fresh session per question", + "models": ["claude-haiku-5-5"], + "find_hash": "c80612938d90f14747e3be3341c6434f3d78efcbd611bf5085facc8653eee5c2", + "state_hash": "be19faeb18e0059bde6482cc2b7e2f1df8368d691e4a903ce1b63cc9aa5a177b", + "read_hash": "6e94ef68632e522abca8eaf873b3238cf4ae45394a4810e66f1595c3fe2ee839" + }, + "answers": { + "branch-001": { + "answer": 41, + "tokens": 27246, + "fixed_tokens": 1111, + "processed_tokens": 190801, + "tool_calls": 11, + "turns": 10 + }, + "branch-002": { + "answer": "dec-use-virtual-shadow-maps-for-the-sun--x8nqkn9q7eaf1ap4", + "tokens": 28207, + "fixed_tokens": 1111, + "processed_tokens": 232921, + "tool_calls": 11, + "turns": 12 + }, + "branch-003": { + "answer": "proposed", + "tokens": 6491, + "fixed_tokens": 1106, + "processed_tokens": 50926, + "tool_calls": 5, + "turns": 6 + }, + "branch-004": { + "answer": "ines/vsm-pool", + "tokens": 34288, + "fixed_tokens": 1102, + "processed_tokens": 258373, + "tool_calls": 17, + "turns": 13 + }, + "branch-005": { + "answer": "iss-how-should-the-tide-simulation-save-time-in-cell--72k2mwf4nb3g978p", + "tokens": 32641, + "fixed_tokens": 1108, + "processed_tokens": 264939, + "tool_calls": 15, + "turns": 12 + }, + "branch-006": { + "answer": "dec-step-tide-cells-beyond-600-m-at-a-quarter-of-the--1zccsxxw2014sh20", + "tokens": 9817, + "fixed_tokens": 1105, + "processed_tokens": 80644, + "tool_calls": 10, + "turns": 8 + }, + "branch-007": { + "answer": 48, + "tokens": 52928, + "fixed_tokens": 1106, + "processed_tokens": 747194, + "tool_calls": 20, + "turns": 21 + }, + "branch-008": { + "answer": "fnd-tide-storms-push-skiff-40-hz-frames-to-28-4-ms-o--28gemvtznv4zztdc", + "tokens": 28652, + "fixed_tokens": 1102, + "processed_tokens": 384133, + "tool_calls": 16, + "turns": 17 + }, + "branch-009": { + "answer": "27.0 ms", + "tokens": 10428, + "fixed_tokens": 1108, + "processed_tokens": 113292, + "tool_calls": 12, + "turns": 11 + }, + "branch-010": { + "answer": [ + "fnd-lantern-after-the-spring-tide-page-pool-change--9zbcsqh53mgv7h4p", + "fnd-world-core-state-with-tide-level-of-detail--x947x5kj16hcngef" + ], + "tokens": 17393, + "fixed_tokens": 1109, + "processed_tokens": 104011, + "tool_calls": 10, + "turns": 8 + }, + "branch-011": { + "answer": false, + "tokens": 42165, + "fixed_tokens": 1105, + "processed_tokens": 535163, + "tool_calls": 20, + "turns": 20 + }, + "branch-012": { + "answer": "superseded", + "tokens": 32944, + "fixed_tokens": 1108, + "processed_tokens": 400990, + "tool_calls": 16, + "turns": 17 + }, + "branch-013": { + "answer": "kenji/tide-lod", + "tokens": 33410, + "fixed_tokens": 1105, + "processed_tokens": 335290, + "tool_calls": 13, + "turns": 14 + }, + "branch-014": { + "answer": "Priya Natarajan", + "tokens": 7350, + "fixed_tokens": 1116, + "processed_tokens": 64753, + "tool_calls": 9, + "turns": 7 + }, + "branch-015": { + "answer": 3, + "tokens": 11832, + "fixed_tokens": 1105, + "processed_tokens": 131527, + "tool_calls": 12, + "turns": 12 + }, + "branch-016": { + "answer": "proposed", + "tokens": 27408, + "fixed_tokens": 1107, + "processed_tokens": 236070, + "tool_calls": 16, + "turns": 13 + }, + "branch-017": { + "answer": 46, + "tokens": 6943, + "fixed_tokens": 1115, + "processed_tokens": 53169, + "tool_calls": 7, + "turns": 6 + }, + "branch-018": { + "answer": "dec-use-virtual-shadow-maps-for-the-sun-with-a-6-144--gnxz9wk2q9xsxc96", + "tokens": 13856, + "fixed_tokens": 1105, + "processed_tokens": 104966, + "tool_calls": 10, + "turns": 9 + }, + "branch-019": { + "answer": false, + "tokens": 5976, + "fixed_tokens": 1106, + "processed_tokens": 50368, + "tool_calls": 8, + "turns": 6 + }, + "branch-020": { + "answer": "Two open branches would replace it: `ines/vsm-pool`, whose accepted Decision `dec-use-virtual-shadow-maps-for-the-sun-with-a-6-144--gnxz9wk2q9xsxc96` supersedes it, and `tomasz/vsm-evict`, whose proposed Decision `dec-use-virtual-shadow-maps-for-the-sun-with-coarser--jtcbmrm4c48ma8z8` supersedes it.", + "tokens": 25351, + "fixed_tokens": 1108, + "processed_tokens": 180968, + "tool_calls": 14, + "turns": 11 + }, + "branch-021": { + "answer": "dec-cap-tide-storm-particles-at-6-000-on-skiff--resze52afjxyxys1", + "tokens": 12205, + "fixed_tokens": 1112, + "processed_tokens": 126458, + "tool_calls": 12, + "turns": 11 + }, + "branch-022": { + "answer": "priya/skiff-cpu", + "tokens": 26353, + "fixed_tokens": 1107, + "processed_tokens": 132094, + "tool_calls": 17, + "turns": 9 + }, + "code-001": { + "answer": "dec-use-virtual-shadow-maps-for-the-sun--x8nqkn9q7eaf1ap4", + "tokens": 42125, + "fixed_tokens": 1103, + "processed_tokens": 400228, + "tool_calls": 13, + "turns": 14 + }, + "code-002": { + "answer": "dec-key-the-shader-cache-on-preprocessed-source-defi--3a05ff0r954vxh30", + "tokens": 12419, + "fixed_tokens": 1111, + "processed_tokens": 78726, + "tool_calls": 6, + "turns": 7 + }, + "code-003": { + "answer": [ + "rsk-eviction-scoring-asks-the-tide-forecast-about-ev--zfwghmgds54h8y46" + ], + "tokens": 52174, + "fixed_tokens": 1103, + "processed_tokens": 802419, + "tool_calls": 24, + "turns": 24 + }, + "code-004": { + "answer": "fnd-pool-eviction-keeps-cells-the-tide-exposes-withi--tzseexbv0m0qw2jk", + "tokens": 43539, + "fixed_tokens": 1102, + "processed_tokens": 661136, + "tool_calls": 22, + "turns": 23 + }, + "code-005": { + "answer": "false: dec-give-reads-for-tide-exposed-cells-top-priority--8ftkvqhawbdf5djh", + "tokens": 39229, + "fixed_tokens": 1109, + "processed_tokens": 298815, + "tool_calls": 12, + "turns": 12 + }, + "code-006": { + "answer": "fnd-tide-priority-reads-starve-player-proximity-load--by0d7b0bk39xz3rk", + "tokens": 24128, + "fixed_tokens": 1104, + "processed_tokens": 358818, + "tool_calls": 19, + "turns": 20 + }, + "code-007": { + "answer": "dec-cap-each-farm-agent-at-6-parallel-shader-compile--q5w4sfjm73tsj8x8", + "tokens": 12564, + "fixed_tokens": 1105, + "processed_tokens": 79327, + "tool_calls": 6, + "turns": 7 + }, + "code-008": { + "answer": "iss-a-timed-out-cache-fetch-is-retried-once-and-then--sn5thmgf02v8rfe4", + "tokens": 28222, + "fixed_tokens": 1104, + "processed_tokens": 171482, + "tool_calls": 9, + "turns": 9 + }, + "code-009": { + "answer": "600p", + "tokens": 19898, + "fixed_tokens": 1109, + "processed_tokens": 89825, + "tool_calls": 6, + "turns": 7 + }, + "code-010": { + "answer": "iss-skiff-stays-at-the-lowest-resolution-for-two-sec--erke2h7xf7vepdv2", + "tokens": 7544, + "fixed_tokens": 1106, + "processed_tokens": 85150, + "tool_calls": 8, + "turns": 9 + }, + "code-011": { + "answer": "dec-visibility-buffer-on-all-platforms-skiff-resolve--k1ackk3c3275r5y1", + "tokens": 11750, + "fixed_tokens": 1106, + "processed_tokens": 159571, + "tool_calls": 14, + "turns": 15 + }, + "code-012": { + "answer": "dec-ship-four-tern-quality-presets-adding-ultra-perf--3jg1wyq3bshv839c", + "tokens": 35006, + "fixed_tokens": 1105, + "processed_tokens": 316200, + "tool_calls": 14, + "turns": 13 + }, + "code-013": { + "answer": [ + "dec-run-the-tide-simulation-in-32-32-fixed-point--c5rjfcnmhne4h05e", + "dec-wrap-the-tide-phase-accumulator-at-each-40-minut--hx88m4hbg27n0xwy", + "dec-add-a-second-tide-harmonic-for-spring-and-neap-t--1c3s7dtyxvj70gvw" + ], + "tokens": 8456, + "fixed_tokens": 1105, + "processed_tokens": 54972, + "tool_calls": 9, + "turns": 6 + }, + "code-014": { + "answer": "con-the-tide-step-allocates-no-memory--bgce2y8wndwrwe0r", + "tokens": 11108, + "fixed_tokens": 1110, + "processed_tokens": 60000, + "tool_calls": 5, + "turns": 6 + }, + "code-015": { + "answer": "con-saves-from-beta-onward-must-load-in-every-later--v6z856sxtgc5sb09", + "tokens": 20431, + "fixed_tokens": 1108, + "processed_tokens": 256058, + "tool_calls": 15, + "turns": 16 + }, + "code-016": { + "answer": "false (dec-feed-a-saltglass-reactive-mask-into-tern--xezjs7v3vq3r8e6k)", + "tokens": 23194, + "fixed_tokens": 1106, + "processed_tokens": 430828, + "tool_calls": 23, + "turns": 24 + }, + "code-017": { + "answer": "A checksum mismatch tells you which state block diverged between peers (tide, physics, or Gull state) and the frame window in which it happened. Decision: dec-hash-tide-physics-and-gull-state-every-30-frames--yqf2m6hqbjhwvkzq", + "tokens": 14968, + "fixed_tokens": 1106, + "processed_tokens": 111443, + "tool_calls": 9, + "turns": 9 + }, + "code-018": { + "answer": "unknown", + "tokens": 38373, + "fixed_tokens": 1108, + "processed_tokens": 364860, + "tool_calls": 13, + "turns": 14 + }, + "code-019": { + "answer": "800 nits", + "tokens": 22827, + "fixed_tokens": 1106, + "processed_tokens": 166371, + "tool_calls": 10, + "turns": 10 + }, + "code-020": { + "answer": "dec-ship-prebuilt-pipeline-blobs-in-the-skiff-packag--3vjysjgy9qkvzebn", + "tokens": 21619, + "fixed_tokens": 1113, + "processed_tokens": 177140, + "tool_calls": 9, + "turns": 10 + }, + "code-021": { + "answer": "iss-shadow-pages-run-out-at-spring-low-tide--f4fgy6b44jbkrsmh", + "tokens": 12634, + "fixed_tokens": 1106, + "processed_tokens": 114560, + "tool_calls": 11, + "turns": 10 + }, + "code-022": { + "answer": "unknown", + "tokens": 46009, + "fixed_tokens": 1111, + "processed_tokens": 667538, + "tool_calls": 19, + "turns": 20 + } + } +}