From 6751ab55fdf6ff4c687b635355efcc19b3a053f4 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 20 Oct 2026 08:19:37 +0000 Subject: [PATCH] docs(bench): record Phase 2 on the scaled test bed Haiku answered the 44 code and branch questions twice on the 10x test bed, with `findcode` (proof find --path and --refs all) or with read-only git alone: - correct: 85 against 85 of 88 question-runs (1 to 1), as on the small bed (85 against 84); - read tokens: 13,422 against 22,829 (41% fewer); tool calls 7 against 15. Scale does not separate the arms, because git log and git diff stay precise as the graph grows. The README and a measurement Finding say the reads earn their place by cost, and name the test that could show more. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01DVVnhYL1Bz7mCfEKDoyLMW --- ...ads-still-match-git-o--e2ws7dke8rr7qbej.md | 16 + benchmarks/proof-recall/README.md | 25 + .../proof-recall/runs/p2x10-r1-findcode.json | 461 ++++++++++++++++++ .../proof-recall/runs/p2x10-r1-nobrief.json | 461 ++++++++++++++++++ .../proof-recall/runs/p2x10-r2-findcode.json | 461 ++++++++++++++++++ .../proof-recall/runs/p2x10-r2-nobrief.json | 461 ++++++++++++++++++ 6 files changed, 1885 insertions(+) create mode 100644 .flatbread-proof/findings/fnd-at-scale-code-and-branch-reads-still-match-git-o--e2ws7dke8rr7qbej.md create mode 100644 benchmarks/proof-recall/runs/p2x10-r1-findcode.json create mode 100644 benchmarks/proof-recall/runs/p2x10-r1-nobrief.json create mode 100644 benchmarks/proof-recall/runs/p2x10-r2-findcode.json create mode 100644 benchmarks/proof-recall/runs/p2x10-r2-nobrief.json diff --git a/.flatbread-proof/findings/fnd-at-scale-code-and-branch-reads-still-match-git-o--e2ws7dke8rr7qbej.md b/.flatbread-proof/findings/fnd-at-scale-code-and-branch-reads-still-match-git-o--e2ws7dke8rr7qbej.md new file mode 100644 index 00000000..fcffd4fe --- /dev/null +++ b/.flatbread-proof/findings/fnd-at-scale-code-and-branch-reads-still-match-git-o--e2ws7dke8rr7qbej.md @@ -0,0 +1,16 @@ +--- +id: fnd-at-scale-code-and-branch-reads-still-match-git-o--e2ws7dke8rr7qbej +effort: eff-effort-graph-memory-and-agent-wedge--szeqvmgqjqnhd002 +title: 'At scale, code and branch reads still match git on accuracy' +kind: measurement +created_at: '2026-10-20T08:19:29.482Z' +derives_from: + - fnd-code-and-branch-reads-halve-the-reading-on-a-sma--4ffp9s93s9swpctv +--- + +The Phase 2 trial was rerun on a test bed ten times larger: the 10x graph (5,131 records), about 470 code files with filler that grep cannot tell apart, and ten branches with filler conflicts. Haiku answered the same 44 code and branch questions twice, with `proof find --path` and `--refs all` or with read-only git alone. + +- Accuracy matched again: 85 against 85 of 88 question-runs (1 to 1), as on the small bed (85 against 84). +- The new reads read 41 percent fewer tokens (13,422 against 22,829) and made 7 tool calls instead of 15. + +Unlike search over record text, these reads do not gain accuracy at scale, because git is already a precise index for code: `git log -- ` and `git diff main...origin/` stay small as the graph grows. `--path` and `--refs` are worth keeping for cost. A test that could show more would ask about code no single path names, or about many branches at once. diff --git a/benchmarks/proof-recall/README.md b/benchmarks/proof-recall/README.md index 5a924f7c..e18e9dbd 100644 --- a/benchmarks/proof-recall/README.md +++ b/benchmarks/proof-recall/README.md @@ -1428,3 +1428,28 @@ on both sets. The 10x test showed that accuracy separates only once crawling stops scaling; the same test is needed here, on a test bed with the 10x graph, a larger codebase and more branches, before claiming more than the saving. + +### Phase 2 at scale + +The same trial on the scaled test bed (`build-repo.mjs --scale 10`: the 10x +graph, about 470 code files, ten branches), Haiku, twice, with +`--timeout 1200`. Run files: `runs/p2x10-r{1,2}-*.json`. + +| Test bed | `findcode` correct | no-brief correct | Read tokens, `findcode` / no-brief | Tool calls | +| ------------------- | ------------------ | ---------------- | ---------------------------------- | ---------- | +| small (524 records) | 85 of 88 | 84 of 88 | 11,642 / 26,283 | 7 / 13 | +| scaled (5,131) | 85 of 88 | 85 of 88 | 13,422 / 22,829 | 7 / 15 | + +On the scaled bed the arms disagree on two question-runs, one each way. +`findcode` read 41% fewer tokens (code set 11,384 against 24,642; branch +set 15,460 against 21,016). No session timed out; one no-brief session +was flagged for reading outside its folder, and it was reading its own +background task's output, not anything from the benchmark. + +So scale does not separate the arms here, unlike search on the 10x graph. +Git is already a precise index for code: `git log -- ` and +`git diff main...origin/` stay small however large the graph and +codebase grow, and the filler never touches a real file. `--path` and +`--refs` earn their place by cost, about half the reading and half the +tool calls, not by accuracy. A test that could show more would ask about +code no single path names, or about many branches at once. diff --git a/benchmarks/proof-recall/runs/p2x10-r1-findcode.json b/benchmarks/proof-recall/runs/p2x10-r1-findcode.json new file mode 100644 index 00000000..1d109d19 --- /dev/null +++ b/benchmarks/proof-recall/runs/p2x10-r1-findcode.json @@ -0,0 +1,461 @@ +{ + "run": { + "name": "p2x10-r1-findcode", + "protocol": "one fresh session per question", + "models": ["claude-haiku-5-5"], + "find_hash": "c80612938d90f14747e3be3341c6434f3d78efcbd611bf5085facc8653eee5c2", + "state_hash": "be19faeb18e0059bde6482cc2b7e2f1df8368d691e4a903ce1b63cc9aa5a177b", + "read_hash": "6e94ef68632e522abca8eaf873b3238cf4ae45394a4810e66f1595c3fe2ee839" + }, + "answers": { + "branch-001": { + "answer": 41, + "tokens": 15874, + "fixed_tokens": 2741, + "processed_tokens": 97858, + "tool_calls": 7, + "turns": 7, + "timed_out": false, + "left_folder": false + }, + "branch-002": { + "answer": "dec-use-virtual-shadow-maps-for-the-sun--x8nqkn9q7eaf1ap4", + "tokens": 18928, + "fixed_tokens": 2740, + "processed_tokens": 151640, + "tool_calls": 11, + "turns": 10, + "timed_out": false, + "left_folder": false + }, + "branch-003": { + "answer": "proposed", + "tokens": 11837, + "fixed_tokens": 1109, + "processed_tokens": 89994, + "tool_calls": 6, + "turns": 7, + "timed_out": false, + "left_folder": false + }, + "branch-004": { + "answer": "ines/vsm-pool", + "tokens": 14194, + "fixed_tokens": 1113, + "processed_tokens": 112897, + "tool_calls": 7, + "turns": 8, + "timed_out": false, + "left_folder": false + }, + "branch-005": { + "answer": "iss-how-should-the-tide-simulation-save-time-in-cell--72k2mwf4nb3g978p", + "tokens": 18494, + "fixed_tokens": 1108, + "processed_tokens": 112958, + "tool_calls": 6, + "turns": 7, + "timed_out": false, + "left_folder": false + }, + "branch-006": { + "answer": "dec-step-tide-cells-beyond-600-m-at-a-quarter-of-the--1zccsxxw2014sh20", + "tokens": 14542, + "fixed_tokens": 1109, + "processed_tokens": 73726, + "tool_calls": 5, + "turns": 6, + "timed_out": false, + "left_folder": false + }, + "branch-007": { + "answer": 48, + "tokens": 12111, + "fixed_tokens": 1111, + "processed_tokens": 57160, + "tool_calls": 4, + "turns": 5, + "timed_out": false, + "left_folder": false + }, + "branch-008": { + "answer": "fnd-tide-storms-push-skiff-40-hz-frames-to-28-4-ms-o--28gemvtznv4zztdc", + "tokens": 13177, + "fixed_tokens": 1109, + "processed_tokens": 85028, + "tool_calls": 6, + "turns": 7, + "timed_out": false, + "left_folder": false + }, + "branch-009": { + "answer": "27.0 ms", + "tokens": 13251, + "fixed_tokens": 1115, + "processed_tokens": 93383, + "tool_calls": 6, + "turns": 7, + "timed_out": false, + "left_folder": false + }, + "branch-010": { + "answer": [ + "fnd-world-core-state-with-tide-level-of-detail--x947x5kj16hcngef", + "fnd-lantern-after-the-spring-tide-page-pool-change--9zbcsqh53mgv7h4p" + ], + "tokens": 19458, + "fixed_tokens": 1108, + "processed_tokens": 74881, + "tool_calls": 4, + "turns": 5, + "timed_out": false, + "left_folder": false + }, + "branch-011": { + "answer": false, + "tokens": 23458, + "fixed_tokens": 1111, + "processed_tokens": 142707, + "tool_calls": 12, + "turns": 8, + "timed_out": false, + "left_folder": false + }, + "branch-012": { + "answer": "superseded", + "tokens": 14069, + "fixed_tokens": 1109, + "processed_tokens": 111912, + "tool_calls": 7, + "turns": 8, + "timed_out": false, + "left_folder": false + }, + "branch-013": { + "answer": "origin/kenji/tide-lod", + "tokens": 13670, + "fixed_tokens": 1115, + "processed_tokens": 60301, + "tool_calls": 5, + "turns": 5, + "timed_out": false, + "left_folder": false + }, + "branch-014": { + "answer": "Priya Natarajan", + "tokens": 11635, + "fixed_tokens": 1118, + "processed_tokens": 54414, + "tool_calls": 4, + "turns": 5, + "timed_out": false, + "left_folder": false + }, + "branch-015": { + "answer": 3, + "tokens": 25283, + "fixed_tokens": 1111, + "processed_tokens": 225012, + "tool_calls": 12, + "turns": 12, + "timed_out": false, + "left_folder": false + }, + "branch-016": { + "answer": "proposed", + "tokens": 17338, + "fixed_tokens": 1118, + "processed_tokens": 129360, + "tool_calls": 8, + "turns": 9, + "timed_out": false, + "left_folder": false + }, + "branch-017": { + "answer": 46, + "tokens": 10282, + "fixed_tokens": 1111, + "processed_tokens": 81717, + "tool_calls": 6, + "turns": 7, + "timed_out": false, + "left_folder": false + }, + "branch-018": { + "answer": "dec-use-virtual-shadow-maps-for-the-sun-with-a-6-144--gnxz9wk2q9xsxc96", + "tokens": 23662, + "fixed_tokens": 1117, + "processed_tokens": 190245, + "tool_calls": 9, + "turns": 10, + "timed_out": false, + "left_folder": false + }, + "branch-019": { + "answer": false, + "tokens": 15285, + "fixed_tokens": 1112, + "processed_tokens": 83837, + "tool_calls": 6, + "turns": 7, + "timed_out": false, + "left_folder": false + }, + "branch-020": { + "answer": "Two open branches would replace it: origin/ines/vsm-pool (dec-use-virtual-shadow-maps-for-the-sun-with-a-6-144--gnxz9wk2q9xsxc96) and origin/tomasz/vsm-evict (dec-use-virtual-shadow-maps-for-the-sun-with-coarser--jtcbmrm4c48ma8z8).", + "tokens": 12271, + "fixed_tokens": 1109, + "processed_tokens": 74433, + "tool_calls": 5, + "turns": 6, + "timed_out": false, + "left_folder": false + }, + "branch-021": { + "answer": "dec-cap-tide-storm-particles-at-6-000-on-skiff--resze52afjxyxys1", + "tokens": 15358, + "fixed_tokens": 1108, + "processed_tokens": 151782, + "tool_calls": 10, + "turns": 11, + "timed_out": false, + "left_folder": false + }, + "branch-022": { + "answer": "priya/skiff-cpu", + "tokens": 28751, + "fixed_tokens": 1108, + "processed_tokens": 147570, + "tool_calls": 17, + "turns": 8, + "timed_out": false, + "left_folder": false + }, + "code-001": { + "answer": "dec-use-virtual-shadow-maps-for-the-sun--x8nqkn9q7eaf1ap4", + "tokens": 15887, + "fixed_tokens": 1111, + "processed_tokens": 98784, + "tool_calls": 7, + "turns": 8, + "timed_out": false, + "left_folder": false + }, + "code-002": { + "answer": "dec-key-the-shader-cache-on-preprocessed-source-defi--3a05ff0r954vxh30", + "tokens": 11836, + "fixed_tokens": 1112, + "processed_tokens": 70682, + "tool_calls": 5, + "turns": 6, + "timed_out": false, + "left_folder": false + }, + "code-003": { + "answer": [ + "rsk-eviction-scoring-asks-the-tide-forecast-about-ev--zfwghmgds54h8y46" + ], + "tokens": 11405, + "fixed_tokens": 1111, + "processed_tokens": 118357, + "tool_calls": 9, + "turns": 10, + "timed_out": false, + "left_folder": false + }, + "code-004": { + "answer": "fnd-pool-eviction-keeps-cells-the-tide-exposes-withi--tzseexbv0m0qw2jk", + "tokens": 18036, + "fixed_tokens": 1111, + "processed_tokens": 181039, + "tool_calls": 12, + "turns": 13, + "timed_out": false, + "left_folder": false + }, + "code-005": { + "answer": "false dec-give-reads-for-tide-exposed-cells-top-priority--8ftkvqhawbdf5djh", + "tokens": 16401, + "fixed_tokens": 1106, + "processed_tokens": 115996, + "tool_calls": 8, + "turns": 9, + "timed_out": false, + "left_folder": false + }, + "code-006": { + "answer": "fnd-tide-priority-reads-starve-player-proximity-load--by0d7b0bk39xz3rk", + "tokens": 8322, + "fixed_tokens": 1110, + "processed_tokens": 61314, + "tool_calls": 5, + "turns": 6, + "timed_out": false, + "left_folder": false + }, + "code-007": { + "answer": "dec-cap-each-farm-agent-at-6-parallel-shader-compile--q5w4sfjm73tsj8x8", + "tokens": 13327, + "fixed_tokens": 1111, + "processed_tokens": 75039, + "tool_calls": 6, + "turns": 6, + "timed_out": false, + "left_folder": false + }, + "code-008": { + "answer": "iss-a-timed-out-cache-fetch-is-retried-once-and-then--sn5thmgf02v8rfe4", + "tokens": 12416, + "fixed_tokens": 1106, + "processed_tokens": 88030, + "tool_calls": 6, + "turns": 7, + "timed_out": false, + "left_folder": false + }, + "code-009": { + "answer": "600p", + "tokens": 8362, + "fixed_tokens": 1108, + "processed_tokens": 61460, + "tool_calls": 5, + "turns": 6, + "timed_out": false, + "left_folder": false + }, + "code-010": { + "answer": "iss-skiff-stays-at-the-lowest-resolution-for-two-sec--erke2h7xf7vepdv2", + "tokens": 13149, + "fixed_tokens": 1108, + "processed_tokens": 83834, + "tool_calls": 6, + "turns": 7, + "timed_out": false, + "left_folder": false + }, + "code-011": { + "answer": "dec-visibility-buffer-on-all-platforms-skiff-resolve--k1ackk3c3275r5y1", + "tokens": 12915, + "fixed_tokens": 1109, + "processed_tokens": 73841, + "tool_calls": 6, + "turns": 7, + "timed_out": false, + "left_folder": false + }, + "code-012": { + "answer": "dec-ship-four-tern-quality-presets-adding-ultra-perf--3jg1wyq3bshv839c", + "tokens": 12400, + "fixed_tokens": 1112, + "processed_tokens": 71098, + "tool_calls": 6, + "turns": 6, + "timed_out": false, + "left_folder": false + }, + "code-013": { + "answer": [ + "dec-add-a-second-tide-harmonic-for-spring-and-neap-t--1c3s7dtyxvj70gvw", + "dec-wrap-the-tide-phase-accumulator-at-each-40-minut--hx88m4hbg27n0xwy", + "dec-run-the-tide-simulation-in-32-32-fixed-point--c5rjfcnmhne4h05e" + ], + "tokens": 8609, + "fixed_tokens": 1109, + "processed_tokens": 56519, + "tool_calls": 9, + "turns": 6, + "timed_out": false, + "left_folder": false + }, + "code-014": { + "answer": "con-the-tide-step-allocates-no-memory--bgce2y8wndwrwe0r", + "tokens": 8302, + "fixed_tokens": 1108, + "processed_tokens": 47233, + "tool_calls": 4, + "turns": 5, + "timed_out": false, + "left_folder": false + }, + "code-015": { + "answer": "con-saves-from-beta-onward-must-load-in-every-later--v6z856sxtgc5sb09", + "tokens": 12428, + "fixed_tokens": 1108, + "processed_tokens": 99773, + "tool_calls": 7, + "turns": 8, + "timed_out": false, + "left_folder": false + }, + "code-016": { + "answer": "false fnd-tern-zeroes-history-weight-wherever-saltglass-se--hhgcrcq2n03rwr53", + "tokens": 9778, + "fixed_tokens": 1107, + "processed_tokens": 66612, + "tool_calls": 5, + "turns": 6, + "timed_out": false, + "left_folder": false + }, + "code-017": { + "answer": "According to dec-hash-tide-physics-and-gull-state-every-30-frames--yqf2m6hqbjhwvkzq, a checksum mismatch tells you which of the three state blocks (tide, physics, or Gull state) diverged and the frame window in which it happened.", + "tokens": 6882, + "fixed_tokens": 1108, + "processed_tokens": 55424, + "tool_calls": 5, + "turns": 6, + "timed_out": false, + "left_folder": false + }, + "code-018": { + "answer": "unknown", + "tokens": 11835, + "fixed_tokens": 1111, + "processed_tokens": 65386, + "tool_calls": 5, + "turns": 6, + "timed_out": false, + "left_folder": false + }, + "code-019": { + "answer": "800 nits", + "tokens": 16782, + "fixed_tokens": 1111, + "processed_tokens": 66080, + "tool_calls": 5, + "turns": 5, + "timed_out": false, + "left_folder": false + }, + "code-020": { + "answer": "dec-ship-prebuilt-pipeline-blobs-in-the-skiff-packag--3vjysjgy9qkvzebn", + "tokens": 12904, + "fixed_tokens": 1108, + "processed_tokens": 90429, + "tool_calls": 6, + "turns": 7, + "timed_out": false, + "left_folder": false + }, + "code-021": { + "answer": "iss-shadow-pages-run-out-at-spring-low-tide--f4fgy6b44jbkrsmh", + "tokens": 14291, + "fixed_tokens": 1107, + "processed_tokens": 124995, + "tool_calls": 8, + "turns": 9, + "timed_out": false, + "left_folder": false + }, + "code-022": { + "answer": "fnd-sun-shadow-pages-are-evicted-oldest-sampled-firs--ndrr54bv4nvr1mbx", + "tokens": 13717, + "fixed_tokens": 1106, + "processed_tokens": 83721, + "tool_calls": 6, + "turns": 7, + "timed_out": false, + "left_folder": false + } + } +} diff --git a/benchmarks/proof-recall/runs/p2x10-r1-nobrief.json b/benchmarks/proof-recall/runs/p2x10-r1-nobrief.json new file mode 100644 index 00000000..556c2659 --- /dev/null +++ b/benchmarks/proof-recall/runs/p2x10-r1-nobrief.json @@ -0,0 +1,461 @@ +{ + "run": { + "name": "p2x10-r1-nobrief", + "protocol": "one fresh session per question", + "models": ["claude-haiku-5-5"], + "find_hash": "c80612938d90f14747e3be3341c6434f3d78efcbd611bf5085facc8653eee5c2", + "state_hash": "be19faeb18e0059bde6482cc2b7e2f1df8368d691e4a903ce1b63cc9aa5a177b", + "read_hash": "6e94ef68632e522abca8eaf873b3238cf4ae45394a4810e66f1595c3fe2ee839" + }, + "answers": { + "branch-001": { + "answer": 41, + "tokens": 17423, + "fixed_tokens": 2743, + "processed_tokens": 176934, + "tool_calls": 14, + "turns": 14, + "timed_out": false, + "left_folder": false + }, + "branch-002": { + "answer": "dec-use-virtual-shadow-maps-for-the-sun--x8nqkn9q7eaf1ap4", + "tokens": 15313, + "fixed_tokens": 2739, + "processed_tokens": 119107, + "tool_calls": 15, + "turns": 10, + "timed_out": false, + "left_folder": false + }, + "branch-003": { + "answer": "proposed", + "tokens": 8444, + "fixed_tokens": 1116, + "processed_tokens": 57437, + "tool_calls": 7, + "turns": 6, + "timed_out": false, + "left_folder": false + }, + "branch-004": { + "answer": "ines/vsm-pool", + "tokens": 41498, + "fixed_tokens": 1112, + "processed_tokens": 549355, + "tool_calls": 22, + "turns": 23, + "timed_out": false, + "left_folder": false + }, + "branch-005": { + "answer": "iss-how-should-the-tide-simulation-save-time-in-cell--72k2mwf4nb3g978p", + "tokens": 28649, + "fixed_tokens": 1116, + "processed_tokens": 319256, + "tool_calls": 20, + "turns": 15, + "timed_out": false, + "left_folder": false + }, + "branch-006": { + "answer": "dec-step-tide-cells-beyond-600-m-at-a-quarter-of-the--1zccsxxw2014sh20", + "tokens": 34111, + "fixed_tokens": 1111, + "processed_tokens": 298931, + "tool_calls": 15, + "turns": 13, + "timed_out": false, + "left_folder": false + }, + "branch-007": { + "answer": 48, + "tokens": 45244, + "fixed_tokens": 1119, + "processed_tokens": 800488, + "tool_calls": 27, + "turns": 28, + "timed_out": false, + "left_folder": false + }, + "branch-008": { + "answer": "fnd-tide-storms-push-skiff-40-hz-frames-to-28-4-ms-o--28gemvtznv4zztdc", + "tokens": 15314, + "fixed_tokens": 1118, + "processed_tokens": 173906, + "tool_calls": 14, + "turns": 13, + "timed_out": false, + "left_folder": false + }, + "branch-009": { + "answer": "27.0 ms", + "tokens": 23297, + "fixed_tokens": 1114, + "processed_tokens": 199026, + "tool_calls": 14, + "turns": 12, + "timed_out": false, + "left_folder": false + }, + "branch-010": { + "answer": [ + "fnd-world-core-state-with-tide-level-of-detail--x947x5kj16hcngef", + "fnd-lantern-after-the-spring-tide-page-pool-change--9zbcsqh53mgv7h4p" + ], + "tokens": 25330, + "fixed_tokens": 1118, + "processed_tokens": 336394, + "tool_calls": 16, + "turns": 18, + "timed_out": false, + "left_folder": true + }, + "branch-011": { + "answer": false, + "tokens": 33941, + "fixed_tokens": 1114, + "processed_tokens": 345476, + "tool_calls": 19, + "turns": 15, + "timed_out": false, + "left_folder": false + }, + "branch-012": { + "answer": "superseded", + "tokens": 24779, + "fixed_tokens": 1117, + "processed_tokens": 198683, + "tool_calls": 10, + "turns": 11, + "timed_out": false, + "left_folder": false + }, + "branch-013": { + "answer": "kenji/tide-lod", + "tokens": 25712, + "fixed_tokens": 1120, + "processed_tokens": 317972, + "tool_calls": 14, + "turns": 15, + "timed_out": false, + "left_folder": false + }, + "branch-014": { + "answer": "Priya Natarajan", + "tokens": 11702, + "fixed_tokens": 1115, + "processed_tokens": 139789, + "tool_calls": 11, + "turns": 12, + "timed_out": false, + "left_folder": false + }, + "branch-015": { + "answer": 3, + "tokens": 16295, + "fixed_tokens": 1120, + "processed_tokens": 142598, + "tool_calls": 12, + "turns": 11, + "timed_out": false, + "left_folder": false + }, + "branch-016": { + "answer": "proposed", + "tokens": 12608, + "fixed_tokens": 1111, + "processed_tokens": 132081, + "tool_calls": 10, + "turns": 11, + "timed_out": false, + "left_folder": false + }, + "branch-017": { + "answer": 46, + "tokens": 7474, + "fixed_tokens": 1112, + "processed_tokens": 56865, + "tool_calls": 5, + "turns": 6, + "timed_out": false, + "left_folder": false + }, + "branch-018": { + "answer": "dec-use-virtual-shadow-maps-for-the-sun-with-a-6-144--gnxz9wk2q9xsxc96", + "tokens": 15529, + "fixed_tokens": 1110, + "processed_tokens": 218724, + "tool_calls": 16, + "turns": 16, + "timed_out": false, + "left_folder": false + }, + "branch-019": { + "answer": false, + "tokens": 8065, + "fixed_tokens": 1114, + "processed_tokens": 56392, + "tool_calls": 6, + "turns": 6, + "timed_out": false, + "left_folder": false + }, + "branch-020": { + "answer": "ines/vsm-pool and tomasz/vsm-evict. Both branches add a record that supersedes `dec-use-virtual-shadow-maps-for-the-sun--x8nqkn9q7eaf1ap4`: ines/vsm-pool's replacement is accepted, while tomasz/vsm-evict's is only proposed.", + "tokens": 21654, + "fixed_tokens": 1118, + "processed_tokens": 234648, + "tool_calls": 15, + "turns": 16, + "timed_out": false, + "left_folder": false + }, + "branch-021": { + "answer": "dec-cap-tide-storm-particles-at-6-000-on-skiff--resze52afjxyxys1", + "tokens": 38240, + "fixed_tokens": 1110, + "processed_tokens": 486987, + "tool_calls": 23, + "turns": 19, + "timed_out": false, + "left_folder": false + }, + "branch-022": { + "answer": "priya/skiff-cpu", + "tokens": 27038, + "fixed_tokens": 1111, + "processed_tokens": 165446, + "tool_calls": 14, + "turns": 10, + "timed_out": false, + "left_folder": false + }, + "code-001": { + "answer": "dec-use-virtual-shadow-maps-for-the-sun--x8nqkn9q7eaf1ap4", + "tokens": 29745, + "fixed_tokens": 1113, + "processed_tokens": 605866, + "tool_calls": 27, + "turns": 28, + "timed_out": false, + "left_folder": false + }, + "code-002": { + "answer": "dec-key-the-shader-cache-on-preprocessed-source-defi--3a05ff0r954vxh30", + "tokens": 23990, + "fixed_tokens": 1112, + "processed_tokens": 300230, + "tool_calls": 14, + "turns": 15, + "timed_out": false, + "left_folder": false + }, + "code-003": { + "answer": [ + "rsk-eviction-scoring-asks-the-tide-forecast-about-ev--zfwghmgds54h8y46" + ], + "tokens": 47354, + "fixed_tokens": 1115, + "processed_tokens": 797432, + "tool_calls": 31, + "turns": 32, + "timed_out": false, + "left_folder": false + }, + "code-004": { + "answer": "fnd-pool-eviction-keeps-cells-the-tide-exposes-withi--tzseexbv0m0qw2jk", + "tokens": 42262, + "fixed_tokens": 1108, + "processed_tokens": 638248, + "tool_calls": 24, + "turns": 25, + "timed_out": false, + "left_folder": false + }, + "code-005": { + "answer": "false dec-give-reads-for-tide-exposed-cells-top-priority--8ftkvqhawbdf5djh", + "tokens": 22419, + "fixed_tokens": 1110, + "processed_tokens": 448089, + "tool_calls": 27, + "turns": 28, + "timed_out": false, + "left_folder": false + }, + "code-006": { + "answer": "fnd-tide-priority-reads-starve-player-proximity-load--by0d7b0bk39xz3rk", + "tokens": 24609, + "fixed_tokens": 1113, + "processed_tokens": 284051, + "tool_calls": 14, + "turns": 15, + "timed_out": false, + "left_folder": false + }, + "code-007": { + "answer": "dec-cap-each-farm-agent-at-6-parallel-shader-compile--q5w4sfjm73tsj8x8", + "tokens": 24125, + "fixed_tokens": 1113, + "processed_tokens": 399232, + "tool_calls": 21, + "turns": 22, + "timed_out": false, + "left_folder": false + }, + "code-008": { + "answer": "iss-a-timed-out-cache-fetch-is-retried-once-and-then--sn5thmgf02v8rfe4", + "tokens": 36694, + "fixed_tokens": 1111, + "processed_tokens": 874441, + "tool_calls": 34, + "turns": 35, + "timed_out": false, + "left_folder": false + }, + "code-009": { + "answer": "600p", + "tokens": 22712, + "fixed_tokens": 1109, + "processed_tokens": 145755, + "tool_calls": 10, + "turns": 9, + "timed_out": false, + "left_folder": false + }, + "code-010": { + "answer": "iss-skiff-stays-at-the-lowest-resolution-for-two-sec--erke2h7xf7vepdv2", + "tokens": 11064, + "fixed_tokens": 1107, + "processed_tokens": 99963, + "tool_calls": 8, + "turns": 9, + "timed_out": false, + "left_folder": false + }, + "code-011": { + "answer": "dec-visibility-buffer-on-all-platforms-skiff-resolve--k1ackk3c3275r5y1", + "tokens": 10157, + "fixed_tokens": 1108, + "processed_tokens": 118873, + "tool_calls": 11, + "turns": 12, + "timed_out": false, + "left_folder": false + }, + "code-012": { + "answer": "dec-ship-four-tern-quality-presets-adding-ultra-perf--3jg1wyq3bshv839c", + "tokens": 34621, + "fixed_tokens": 1117, + "processed_tokens": 298362, + "tool_calls": 12, + "turns": 12, + "timed_out": false, + "left_folder": false + }, + "code-013": { + "answer": [ + "dec-add-a-second-tide-harmonic-for-spring-and-neap-t--1c3s7dtyxvj70gvw", + "dec-wrap-the-tide-phase-accumulator-at-each-40-minut--hx88m4hbg27n0xwy", + "dec-run-the-tide-simulation-in-32-32-fixed-point--c5rjfcnmhne4h05e" + ], + "tokens": 9803, + "fixed_tokens": 1109, + "processed_tokens": 88185, + "tool_calls": 11, + "turns": 9, + "timed_out": false, + "left_folder": false + }, + "code-014": { + "answer": "con-the-tide-step-allocates-no-memory--bgce2y8wndwrwe0r", + "tokens": 14024, + "fixed_tokens": 1112, + "processed_tokens": 146736, + "tool_calls": 10, + "turns": 11, + "timed_out": false, + "left_folder": false + }, + "code-015": { + "answer": "con-saves-from-beta-onward-must-load-in-every-later--v6z856sxtgc5sb09", + "tokens": 19190, + "fixed_tokens": 1111, + "processed_tokens": 166142, + "tool_calls": 10, + "turns": 11, + "timed_out": false, + "left_folder": false + }, + "code-016": { + "answer": false, + "tokens": 31494, + "fixed_tokens": 1113, + "processed_tokens": 359315, + "tool_calls": 16, + "turns": 16, + "timed_out": false, + "left_folder": false + }, + "code-017": { + "answer": "A checksum mismatch tells you which state block diverged (tide, physics, or Gull state) and the frame window it happened in. Record: dec-hash-tide-physics-and-gull-state-every-30-frames--yqf2m6hqbjhwvkzq", + "tokens": 23779, + "fixed_tokens": 1108, + "processed_tokens": 196067, + "tool_calls": 12, + "turns": 11, + "timed_out": false, + "left_folder": false + }, + "code-018": { + "answer": "unknown", + "tokens": 64708, + "fixed_tokens": 1112, + "processed_tokens": 1579704, + "tool_calls": 37, + "turns": 38, + "timed_out": false, + "left_folder": false + }, + "code-019": { + "answer": "800 nits", + "tokens": 30411, + "fixed_tokens": 1107, + "processed_tokens": 393745, + "tool_calls": 16, + "turns": 17, + "timed_out": false, + "left_folder": false + }, + "code-020": { + "answer": "dec-ship-prebuilt-pipeline-blobs-in-the-skiff-packag--3vjysjgy9qkvzebn", + "tokens": 34925, + "fixed_tokens": 1109, + "processed_tokens": 398864, + "tool_calls": 14, + "turns": 15, + "timed_out": false, + "left_folder": false + }, + "code-021": { + "answer": "iss-shadow-pages-run-out-at-spring-low-tide--f4fgy6b44jbkrsmh", + "tokens": 19815, + "fixed_tokens": 1111, + "processed_tokens": 185239, + "tool_calls": 13, + "turns": 12, + "timed_out": false, + "left_folder": false + }, + "code-022": { + "answer": "fnd-sun-shadow-pages-are-evicted-oldest-sampled-firs--ndrr54bv4nvr1mbx", + "tokens": 33279, + "fixed_tokens": 1111, + "processed_tokens": 366349, + "tool_calls": 15, + "turns": 16, + "timed_out": false, + "left_folder": false + } + } +} diff --git a/benchmarks/proof-recall/runs/p2x10-r2-findcode.json b/benchmarks/proof-recall/runs/p2x10-r2-findcode.json new file mode 100644 index 00000000..b52ded1f --- /dev/null +++ b/benchmarks/proof-recall/runs/p2x10-r2-findcode.json @@ -0,0 +1,461 @@ +{ + "run": { + "name": "p2x10-r2-findcode", + "protocol": "one fresh session per question", + "models": ["claude-haiku-5-5"], + "find_hash": "c80612938d90f14747e3be3341c6434f3d78efcbd611bf5085facc8653eee5c2", + "state_hash": "be19faeb18e0059bde6482cc2b7e2f1df8368d691e4a903ce1b63cc9aa5a177b", + "read_hash": "6e94ef68632e522abca8eaf873b3238cf4ae45394a4810e66f1595c3fe2ee839" + }, + "answers": { + "branch-001": { + "answer": 41, + "tokens": 12514, + "fixed_tokens": 1115, + "processed_tokens": 91667, + "tool_calls": 6, + "turns": 7, + "timed_out": false, + "left_folder": false + }, + "branch-002": { + "answer": "dec-use-virtual-shadow-maps-for-the-sun--x8nqkn9q7eaf1ap4", + "tokens": 15435, + "fixed_tokens": 1106, + "processed_tokens": 131514, + "tool_calls": 10, + "turns": 9, + "timed_out": false, + "left_folder": false + }, + "branch-003": { + "answer": "proposed", + "tokens": 8573, + "fixed_tokens": 1115, + "processed_tokens": 58233, + "tool_calls": 6, + "turns": 6, + "timed_out": false, + "left_folder": false + }, + "branch-004": { + "answer": "ines/vsm-pool", + "tokens": 20152, + "fixed_tokens": 1112, + "processed_tokens": 143527, + "tool_calls": 9, + "turns": 9, + "timed_out": false, + "left_folder": false + }, + "branch-005": { + "answer": "iss-how-should-the-tide-simulation-save-time-in-cell--72k2mwf4nb3g978p", + "tokens": 15629, + "fixed_tokens": 1110, + "processed_tokens": 77126, + "tool_calls": 5, + "turns": 6, + "timed_out": false, + "left_folder": false + }, + "branch-006": { + "answer": "dec-step-tide-cells-beyond-600-m-at-a-quarter-of-the--1zccsxxw2014sh20", + "tokens": 18345, + "fixed_tokens": 1110, + "processed_tokens": 69441, + "tool_calls": 5, + "turns": 5, + "timed_out": false, + "left_folder": false + }, + "branch-007": { + "answer": "48", + "tokens": 15449, + "fixed_tokens": 1115, + "processed_tokens": 76205, + "tool_calls": 6, + "turns": 6, + "timed_out": false, + "left_folder": false + }, + "branch-008": { + "answer": "fnd-tide-storms-push-skiff-40-hz-frames-to-28-4-ms-o--28gemvtznv4zztdc", + "tokens": 16957, + "fixed_tokens": 1110, + "processed_tokens": 130962, + "tool_calls": 8, + "turns": 9, + "timed_out": false, + "left_folder": false + }, + "branch-009": { + "answer": "27.0 ms", + "tokens": 17919, + "fixed_tokens": 1114, + "processed_tokens": 157679, + "tool_calls": 10, + "turns": 10, + "timed_out": false, + "left_folder": false + }, + "branch-010": { + "answer": [ + "fnd-world-core-state-with-tide-level-of-detail--x947x5kj16hcngef", + "fnd-lantern-after-the-spring-tide-page-pool-change--9zbcsqh53mgv7h4p" + ], + "tokens": 18069, + "fixed_tokens": 1118, + "processed_tokens": 86413, + "tool_calls": 5, + "turns": 6, + "timed_out": false, + "left_folder": false + }, + "branch-011": { + "answer": false, + "tokens": 19625, + "fixed_tokens": 1110, + "processed_tokens": 133418, + "tool_calls": 10, + "turns": 8, + "timed_out": false, + "left_folder": false + }, + "branch-012": { + "answer": "superseded", + "tokens": 17476, + "fixed_tokens": 1110, + "processed_tokens": 106841, + "tool_calls": 7, + "turns": 7, + "timed_out": false, + "left_folder": false + }, + "branch-013": { + "answer": "kenji/tide-lod", + "tokens": 15131, + "fixed_tokens": 1114, + "processed_tokens": 98371, + "tool_calls": 6, + "turns": 7, + "timed_out": false, + "left_folder": false + }, + "branch-014": { + "answer": "Priya Natarajan", + "tokens": 11644, + "fixed_tokens": 1109, + "processed_tokens": 54256, + "tool_calls": 4, + "turns": 5, + "timed_out": false, + "left_folder": false + }, + "branch-015": { + "answer": 3, + "tokens": 16898, + "fixed_tokens": 1109, + "processed_tokens": 97881, + "tool_calls": 7, + "turns": 8, + "timed_out": false, + "left_folder": false + }, + "branch-016": { + "answer": "proposed", + "tokens": 17655, + "fixed_tokens": 1107, + "processed_tokens": 114175, + "tool_calls": 6, + "turns": 7, + "timed_out": false, + "left_folder": false + }, + "branch-017": { + "answer": 46, + "tokens": 8999, + "fixed_tokens": 1118, + "processed_tokens": 76409, + "tool_calls": 7, + "turns": 7, + "timed_out": false, + "left_folder": false + }, + "branch-018": { + "answer": "dec-use-virtual-shadow-maps-for-the-sun-with-a-6-144--gnxz9wk2q9xsxc96", + "tokens": 27496, + "fixed_tokens": 1113, + "processed_tokens": 248493, + "tool_calls": 10, + "turns": 11, + "timed_out": false, + "left_folder": false + }, + "branch-019": { + "answer": false, + "tokens": 17068, + "fixed_tokens": 1121, + "processed_tokens": 128144, + "tool_calls": 19, + "turns": 9, + "timed_out": false, + "left_folder": false + }, + "branch-020": { + "answer": "Two open branches would replace main's accepted sun-shadow Decision dec-use-virtual-shadow-maps-for-the-sun--x8nqkn9q7eaf1ap4: ines/vsm-pool (draft #41, superseded by dec-use-virtual-shadow-maps-for-the-sun-with-a-6-144--gnxz9wk2q9xsxc96) and tomasz/vsm-evict (draft #44, superseded by dec-use-virtual-shadow-maps-for-the-sun-with-coarser--jtcbmrm4c48ma8z8).", + "tokens": 21950, + "fixed_tokens": 1108, + "processed_tokens": 175785, + "tool_calls": 9, + "turns": 10, + "timed_out": false, + "left_folder": false + }, + "branch-021": { + "answer": "dec-cap-tide-storm-particles-at-6-000-on-skiff--resze52afjxyxys1", + "tokens": 15488, + "fixed_tokens": 1110, + "processed_tokens": 89080, + "tool_calls": 6, + "turns": 7, + "timed_out": false, + "left_folder": false + }, + "branch-022": { + "answer": "priya/skiff-cpu", + "tokens": 21037, + "fixed_tokens": 1112, + "processed_tokens": 118364, + "tool_calls": 8, + "turns": 8, + "timed_out": false, + "left_folder": false + }, + "code-001": { + "answer": "dec-use-virtual-shadow-maps-for-the-sun--x8nqkn9q7eaf1ap4", + "tokens": 13637, + "fixed_tokens": 1106, + "processed_tokens": 74939, + "tool_calls": 6, + "turns": 6, + "timed_out": false, + "left_folder": false + }, + "code-002": { + "answer": "dec-key-the-shader-cache-on-preprocessed-source-defi--3a05ff0r954vxh30", + "tokens": 11563, + "fixed_tokens": 1110, + "processed_tokens": 54888, + "tool_calls": 4, + "turns": 5, + "timed_out": false, + "left_folder": false + }, + "code-003": { + "answer": [ + "rsk-eviction-scoring-asks-the-tide-forecast-about-ev--zfwghmgds54h8y46" + ], + "tokens": 14004, + "fixed_tokens": 1108, + "processed_tokens": 126639, + "tool_calls": 9, + "turns": 10, + "timed_out": false, + "left_folder": false + }, + "code-004": { + "answer": "fnd-pool-eviction-keeps-cells-the-tide-exposes-withi--tzseexbv0m0qw2jk", + "tokens": 9211, + "fixed_tokens": 1106, + "processed_tokens": 86511, + "tool_calls": 7, + "turns": 8, + "timed_out": false, + "left_folder": false + }, + "code-005": { + "answer": "false dec-give-reads-for-tide-exposed-cells-top-priority--8ftkvqhawbdf5djh", + "tokens": 16400, + "fixed_tokens": 1107, + "processed_tokens": 115944, + "tool_calls": 8, + "turns": 9, + "timed_out": false, + "left_folder": false + }, + "code-006": { + "answer": "fnd-tide-priority-reads-starve-player-proximity-load--by0d7b0bk39xz3rk", + "tokens": 8539, + "fixed_tokens": 1112, + "processed_tokens": 61709, + "tool_calls": 5, + "turns": 6, + "timed_out": false, + "left_folder": false + }, + "code-007": { + "answer": "dec-cap-each-farm-agent-at-6-parallel-shader-compile--q5w4sfjm73tsj8x8", + "tokens": 10167, + "fixed_tokens": 1111, + "processed_tokens": 68587, + "tool_calls": 5, + "turns": 6, + "timed_out": false, + "left_folder": false + }, + "code-008": { + "answer": "iss-a-timed-out-cache-fetch-is-retried-once-and-then--sn5thmgf02v8rfe4", + "tokens": 14037, + "fixed_tokens": 1112, + "processed_tokens": 63081, + "tool_calls": 4, + "turns": 5, + "timed_out": false, + "left_folder": false + }, + "code-009": { + "answer": "600p", + "tokens": 15243, + "fixed_tokens": 1117, + "processed_tokens": 87773, + "tool_calls": 6, + "turns": 7, + "timed_out": false, + "left_folder": false + }, + "code-010": { + "answer": "iss-skiff-stays-at-the-lowest-resolution-for-two-sec--erke2h7xf7vepdv2", + "tokens": 14579, + "fixed_tokens": 1106, + "processed_tokens": 98495, + "tool_calls": 7, + "turns": 8, + "timed_out": false, + "left_folder": false + }, + "code-011": { + "answer": "dec-visibility-buffer-on-all-platforms-skiff-resolve--k1ackk3c3275r5y1", + "tokens": 16389, + "fixed_tokens": 1108, + "processed_tokens": 99480, + "tool_calls": 7, + "turns": 8, + "timed_out": false, + "left_folder": false + }, + "code-012": { + "answer": "dec-ship-four-tern-quality-presets-adding-ultra-perf--3jg1wyq3bshv839c", + "tokens": 12359, + "fixed_tokens": 1110, + "processed_tokens": 86210, + "tool_calls": 6, + "turns": 7, + "timed_out": false, + "left_folder": false + }, + "code-013": { + "answer": [ + "dec-add-a-second-tide-harmonic-for-spring-and-neap-t--1c3s7dtyxvj70gvw", + "dec-wrap-the-tide-phase-accumulator-at-each-40-minut--hx88m4hbg27n0xwy", + "dec-run-the-tide-simulation-in-32-32-fixed-point--c5rjfcnmhne4h05e" + ], + "tokens": 8526, + "fixed_tokens": 1107, + "processed_tokens": 56341, + "tool_calls": 9, + "turns": 6, + "timed_out": false, + "left_folder": false + }, + "code-014": { + "answer": "con-the-tide-step-allocates-no-memory--bgce2y8wndwrwe0r", + "tokens": 8746, + "fixed_tokens": 1107, + "processed_tokens": 63302, + "tool_calls": 5, + "turns": 6, + "timed_out": false, + "left_folder": false + }, + "code-015": { + "answer": "con-saves-from-beta-onward-must-load-in-every-later--v6z856sxtgc5sb09", + "tokens": 12468, + "fixed_tokens": 1105, + "processed_tokens": 76539, + "tool_calls": 6, + "turns": 7, + "timed_out": false, + "left_folder": false + }, + "code-016": { + "answer": "false, fnd-tern-zeroes-history-weight-wherever-saltglass-se--hhgcrcq2n03rwr53", + "tokens": 16710, + "fixed_tokens": 1108, + "processed_tokens": 105871, + "tool_calls": 6, + "turns": 7, + "timed_out": false, + "left_folder": false + }, + "code-017": { + "answer": "A checksum mismatch tells you which state block diverged (tide, physics, or Gull state) and the frame window in which it happened. Decision dec-hash-tide-physics-and-gull-state-every-30-frames--yqf2m6hqbjhwvkzq says so.", + "tokens": 13088, + "fixed_tokens": 1108, + "processed_tokens": 78836, + "tool_calls": 6, + "turns": 7, + "timed_out": false, + "left_folder": false + }, + "code-018": { + "answer": "unknown", + "tokens": 13247, + "fixed_tokens": 1110, + "processed_tokens": 68264, + "tool_calls": 5, + "turns": 6, + "timed_out": false, + "left_folder": false + }, + "code-019": { + "answer": "800 nits", + "tokens": 16377, + "fixed_tokens": 1103, + "processed_tokens": 82109, + "tool_calls": 5, + "turns": 6, + "timed_out": false, + "left_folder": false + }, + "code-020": { + "answer": "dec-ship-prebuilt-pipeline-blobs-in-the-skiff-packag--3vjysjgy9qkvzebn", + "tokens": 6655, + "fixed_tokens": 1102, + "processed_tokens": 54732, + "tool_calls": 5, + "turns": 6, + "timed_out": false, + "left_folder": false + }, + "code-021": { + "answer": "iss-shadow-pages-run-out-at-spring-low-tide--f4fgy6b44jbkrsmh", + "tokens": 17641, + "fixed_tokens": 1111, + "processed_tokens": 153455, + "tool_calls": 10, + "turns": 11, + "timed_out": false, + "left_folder": false + }, + "code-022": { + "answer": "fnd-sun-shadow-pages-are-evicted-oldest-sampled-firs--ndrr54bv4nvr1mbx", + "tokens": 10115, + "fixed_tokens": 1111, + "processed_tokens": 76822, + "tool_calls": 6, + "turns": 7, + "timed_out": false, + "left_folder": false + } + } +} diff --git a/benchmarks/proof-recall/runs/p2x10-r2-nobrief.json b/benchmarks/proof-recall/runs/p2x10-r2-nobrief.json new file mode 100644 index 00000000..ae3a74bd --- /dev/null +++ b/benchmarks/proof-recall/runs/p2x10-r2-nobrief.json @@ -0,0 +1,461 @@ +{ + "run": { + "name": "p2x10-r2-nobrief", + "protocol": "one fresh session per question", + "models": ["claude-haiku-5-5"], + "find_hash": "c80612938d90f14747e3be3341c6434f3d78efcbd611bf5085facc8653eee5c2", + "state_hash": "be19faeb18e0059bde6482cc2b7e2f1df8368d691e4a903ce1b63cc9aa5a177b", + "read_hash": "6e94ef68632e522abca8eaf873b3238cf4ae45394a4810e66f1595c3fe2ee839" + }, + "answers": { + "branch-001": { + "answer": "#41", + "tokens": 14377, + "fixed_tokens": 1114, + "processed_tokens": 147120, + "tool_calls": 13, + "turns": 11, + "timed_out": false, + "left_folder": false + }, + "branch-002": { + "answer": "dec-use-virtual-shadow-maps-for-the-sun--x8nqkn9q7eaf1ap4", + "tokens": 8327, + "fixed_tokens": 1108, + "processed_tokens": 57595, + "tool_calls": 6, + "turns": 6, + "timed_out": false, + "left_folder": false + }, + "branch-003": { + "answer": "proposed", + "tokens": 7966, + "fixed_tokens": 1114, + "processed_tokens": 67921, + "tool_calls": 7, + "turns": 7, + "timed_out": false, + "left_folder": false + }, + "branch-004": { + "answer": "ines/vsm-pool", + "tokens": 35980, + "fixed_tokens": 1113, + "processed_tokens": 504283, + "tool_calls": 20, + "turns": 20, + "timed_out": false, + "left_folder": false + }, + "branch-005": { + "answer": "unknown", + "tokens": 38357, + "fixed_tokens": 1115, + "processed_tokens": 371573, + "tool_calls": 19, + "turns": 15, + "timed_out": false, + "left_folder": false + }, + "branch-006": { + "answer": "dec-step-tide-cells-beyond-600-m-at-a-quarter-of-the--1zccsxxw2014sh20", + "tokens": 17163, + "fixed_tokens": 1113, + "processed_tokens": 161223, + "tool_calls": 12, + "turns": 11, + "timed_out": false, + "left_folder": false + }, + "branch-007": { + "answer": 48, + "tokens": 35233, + "fixed_tokens": 1112, + "processed_tokens": 470084, + "tool_calls": 18, + "turns": 19, + "timed_out": false, + "left_folder": false + }, + "branch-008": { + "answer": "fnd-tide-storms-push-skiff-40-hz-frames-to-28-4-ms-o--28gemvtznv4zztdc", + "tokens": 17843, + "fixed_tokens": 1110, + "processed_tokens": 229520, + "tool_calls": 16, + "turns": 16, + "timed_out": false, + "left_folder": false + }, + "branch-009": { + "answer": "27.0 ms", + "tokens": 8671, + "fixed_tokens": 1108, + "processed_tokens": 67998, + "tool_calls": 6, + "turns": 7, + "timed_out": false, + "left_folder": false + }, + "branch-010": { + "answer": [ + "fnd-world-core-state-with-tide-level-of-detail--x947x5kj16hcngef", + "fnd-lantern-after-the-spring-tide-page-pool-change--9zbcsqh53mgv7h4p" + ], + "tokens": 39106, + "fixed_tokens": 1110, + "processed_tokens": 290405, + "tool_calls": 14, + "turns": 12, + "timed_out": false, + "left_folder": false + }, + "branch-011": { + "answer": false, + "tokens": 36741, + "fixed_tokens": 1109, + "processed_tokens": 421201, + "tool_calls": 18, + "turns": 18, + "timed_out": false, + "left_folder": false + }, + "branch-012": { + "answer": "superseded", + "tokens": 16010, + "fixed_tokens": 1108, + "processed_tokens": 188178, + "tool_calls": 14, + "turns": 14, + "timed_out": false, + "left_folder": false + }, + "branch-013": { + "answer": "kenji/tide-lod", + "tokens": 33976, + "fixed_tokens": 1119, + "processed_tokens": 595511, + "tool_calls": 26, + "turns": 27, + "timed_out": false, + "left_folder": false + }, + "branch-014": { + "answer": "Priya Natarajan", + "tokens": 23377, + "fixed_tokens": 1114, + "processed_tokens": 146659, + "tool_calls": 9, + "turns": 9, + "timed_out": false, + "left_folder": false + }, + "branch-015": { + "answer": 3, + "tokens": 18641, + "fixed_tokens": 1111, + "processed_tokens": 178113, + "tool_calls": 15, + "turns": 13, + "timed_out": false, + "left_folder": false + }, + "branch-016": { + "answer": "proposed", + "tokens": 16422, + "fixed_tokens": 1118, + "processed_tokens": 150338, + "tool_calls": 11, + "turns": 10, + "timed_out": false, + "left_folder": false + }, + "branch-017": { + "answer": 46, + "tokens": 8356, + "fixed_tokens": 1112, + "processed_tokens": 58781, + "tool_calls": 5, + "turns": 6, + "timed_out": false, + "left_folder": false + }, + "branch-018": { + "answer": "dec-use-virtual-shadow-maps-for-the-sun-with-a-6-144--gnxz9wk2q9xsxc96", + "tokens": 15609, + "fixed_tokens": 1121, + "processed_tokens": 157642, + "tool_calls": 13, + "turns": 12, + "timed_out": false, + "left_folder": false + }, + "branch-019": { + "answer": false, + "tokens": 5675, + "fixed_tokens": 1112, + "processed_tokens": 51423, + "tool_calls": 5, + "turns": 6, + "timed_out": false, + "left_folder": false + }, + "branch-020": { + "answer": "ines/vsm-pool and tomasz/vsm-evict. Ines's branch supersedes the sun-shadow Decision with dec-use-virtual-shadow-maps-for-the-sun-with-a-6-144--gnxz9wk2q9xsxc96 (accepted), and Tomasz's supersedes it with dec-use-virtual-shadow-maps-for-the-sun-with-coarser--jtcbmrm4c48ma8z8 (proposed).", + "tokens": 28332, + "fixed_tokens": 1119, + "processed_tokens": 410414, + "tool_calls": 18, + "turns": 18, + "timed_out": false, + "left_folder": false + }, + "branch-021": { + "answer": "dec-cap-tide-storm-particles-at-6-000-on-skiff--resze52afjxyxys1", + "tokens": 20326, + "fixed_tokens": 1114, + "processed_tokens": 227293, + "tool_calls": 15, + "turns": 16, + "timed_out": false, + "left_folder": false + }, + "branch-022": { + "answer": "priya/skiff-cpu", + "tokens": 32830, + "fixed_tokens": 1112, + "processed_tokens": 352425, + "tool_calls": 18, + "turns": 17, + "timed_out": false, + "left_folder": false + }, + "code-001": { + "answer": "dec-use-virtual-shadow-maps-for-the-sun--x8nqkn9q7eaf1ap4", + "tokens": 13085, + "fixed_tokens": 1111, + "processed_tokens": 195293, + "tool_calls": 15, + "turns": 16, + "timed_out": false, + "left_folder": false + }, + "code-002": { + "answer": "dec-key-the-shader-cache-on-preprocessed-source-defi--3a05ff0r954vxh30", + "tokens": 11527, + "fixed_tokens": 1110, + "processed_tokens": 151749, + "tool_calls": 12, + "turns": 13, + "timed_out": false, + "left_folder": false + }, + "code-003": { + "answer": [ + "rsk-eviction-scoring-asks-the-tide-forecast-about-ev--zfwghmgds54h8y46" + ], + "tokens": 42553, + "fixed_tokens": 1107, + "processed_tokens": 1186556, + "tool_calls": 48, + "turns": 48, + "timed_out": false, + "left_folder": false + }, + "code-004": { + "answer": "fnd-pool-eviction-keeps-cells-the-tide-exposes-withi--tzseexbv0m0qw2jk", + "tokens": 31466, + "fixed_tokens": 1108, + "processed_tokens": 396287, + "tool_calls": 18, + "turns": 18, + "timed_out": false, + "left_folder": false + }, + "code-005": { + "answer": "false", + "tokens": 25491, + "fixed_tokens": 1115, + "processed_tokens": 280150, + "tool_calls": 13, + "turns": 14, + "timed_out": false, + "left_folder": false + }, + "code-006": { + "answer": "fnd-tide-priority-reads-starve-player-proximity-load--by0d7b0bk39xz3rk", + "tokens": 28761, + "fixed_tokens": 1109, + "processed_tokens": 318286, + "tool_calls": 14, + "turns": 15, + "timed_out": false, + "left_folder": false + }, + "code-007": { + "answer": "dec-cap-each-farm-agent-at-6-parallel-shader-compile--q5w4sfjm73tsj8x8", + "tokens": 20838, + "fixed_tokens": 1113, + "processed_tokens": 316802, + "tool_calls": 19, + "turns": 20, + "timed_out": false, + "left_folder": false + }, + "code-008": { + "answer": "iss-a-timed-out-cache-fetch-is-retried-once-and-then--sn5thmgf02v8rfe4", + "tokens": 15493, + "fixed_tokens": 1114, + "processed_tokens": 157682, + "tool_calls": 11, + "turns": 12, + "timed_out": false, + "left_folder": false + }, + "code-009": { + "answer": "600p", + "tokens": 20318, + "fixed_tokens": 1109, + "processed_tokens": 133682, + "tool_calls": 8, + "turns": 9, + "timed_out": false, + "left_folder": false + }, + "code-010": { + "answer": "iss-skiff-stays-at-the-lowest-resolution-for-two-sec--erke2h7xf7vepdv2", + "tokens": 26096, + "fixed_tokens": 1116, + "processed_tokens": 344773, + "tool_calls": 15, + "turns": 16, + "timed_out": false, + "left_folder": false + }, + "code-011": { + "answer": "dec-visibility-buffer-on-all-platforms-skiff-resolve--k1ackk3c3275r5y1", + "tokens": 12829, + "fixed_tokens": 1108, + "processed_tokens": 129697, + "tool_calls": 11, + "turns": 12, + "timed_out": false, + "left_folder": false + }, + "code-012": { + "answer": "dec-ship-four-tern-quality-presets-adding-ultra-perf--3jg1wyq3bshv839c", + "tokens": 37863, + "fixed_tokens": 1113, + "processed_tokens": 598739, + "tool_calls": 23, + "turns": 21, + "timed_out": false, + "left_folder": false + }, + "code-013": { + "answer": [ + "dec-add-a-second-tide-harmonic-for-spring-and-neap-t--1c3s7dtyxvj70gvw", + "dec-wrap-the-tide-phase-accumulator-at-each-40-minut--hx88m4hbg27n0xwy", + "dec-run-the-tide-simulation-in-32-32-fixed-point--c5rjfcnmhne4h05e" + ], + "tokens": 8655, + "fixed_tokens": 1115, + "processed_tokens": 55711, + "tool_calls": 9, + "turns": 6, + "timed_out": false, + "left_folder": false + }, + "code-014": { + "answer": "con-the-tide-step-allocates-no-memory--bgce2y8wndwrwe0r", + "tokens": 10772, + "fixed_tokens": 1112, + "processed_tokens": 59677, + "tool_calls": 5, + "turns": 6, + "timed_out": false, + "left_folder": false + }, + "code-015": { + "answer": "con-saves-from-beta-onward-must-load-in-every-later--v6z856sxtgc5sb09", + "tokens": 18657, + "fixed_tokens": 1110, + "processed_tokens": 165185, + "tool_calls": 10, + "turns": 11, + "timed_out": false, + "left_folder": false + }, + "code-016": { + "answer": "false, dec-feed-a-saltglass-reactive-mask-into-tern--xezjs7v3vq3r8e6k", + "tokens": 27615, + "fixed_tokens": 1110, + "processed_tokens": 243103, + "tool_calls": 12, + "turns": 13, + "timed_out": false, + "left_folder": false + }, + "code-017": { + "answer": "A checksum mismatch names the state block that diverged between peers and the frame window in which it happened; each peer hashes the three state blocks separately every 30 frames. Record: dec-hash-tide-physics-and-gull-state-every-30-frames--yqf2m6hqbjhwvkzq", + "tokens": 22161, + "fixed_tokens": 1111, + "processed_tokens": 233074, + "tool_calls": 13, + "turns": 14, + "timed_out": false, + "left_folder": false + }, + "code-018": { + "answer": "unknown", + "tokens": 50137, + "fixed_tokens": 1116, + "processed_tokens": 1288163, + "tool_calls": 36, + "turns": 36, + "timed_out": false, + "left_folder": false + }, + "code-019": { + "answer": "800 nits", + "tokens": 26827, + "fixed_tokens": 1109, + "processed_tokens": 302996, + "tool_calls": 18, + "turns": 15, + "timed_out": false, + "left_folder": false + }, + "code-020": { + "answer": "dec-ship-prebuilt-pipeline-blobs-in-the-skiff-packag--3vjysjgy9qkvzebn", + "tokens": 24370, + "fixed_tokens": 1118, + "processed_tokens": 282765, + "tool_calls": 14, + "turns": 14, + "timed_out": false, + "left_folder": false + }, + "code-021": { + "answer": "iss-shadow-pages-run-out-at-spring-low-tide--f4fgy6b44jbkrsmh", + "tokens": 14049, + "fixed_tokens": 1111, + "processed_tokens": 112378, + "tool_calls": 8, + "turns": 9, + "timed_out": false, + "left_folder": false + }, + "code-022": { + "answer": "fnd-sun-shadow-pages-are-evicted-oldest-sampled-firs--ndrr54bv4nvr1mbx", + "tokens": 32397, + "fixed_tokens": 1108, + "processed_tokens": 386275, + "tool_calls": 16, + "turns": 17, + "timed_out": false, + "left_folder": false + } + } +}