feat(bench): keep private question sets outside the repository

benchmarks/proof-recall/README.md

@@ -337,6 +337,14 @@ A benchmark agent must never see `spec/`, `questions/`, or `keymap.json`.
    with more right answers over the runs, and is the stricter reading. The
    script warns when the run files record different `find_hash` values.
 
+**Question sets kept outside the repository.** A set about a private
+graph must never enter Git. Set `PROOF_RECALL_QUESTIONS` to a folder of
+question files and `PROOF_RECALL_RUNS` to a folder for run files; every
+script then reads questions and writes runs there, and `--graph` points at
+the graph, whose `keymap.json` maps `<effort>.<key>` to record ids. With
+neither set, nothing changes. Use such a set only as a held-out check:
+never read its cards or outputs to choose a change.
+
 **Ambiguous-Effort questions.** `questions/ambiguous.json` holds 24
 questions rewritten from tier 0-2 questions so that they name no Effort and
 use words several Efforts share, such as "shader cache", "the

benchmarks/proof-recall/scripts/check.mjs

@@ -24,6 +24,19 @@ import {
 } from './graph.mjs';
 
 const BENCH = resolve(dirname(fileURLToPath(import.meta.url)), '..');
+
+/**
+ * Where question files and run files live. `PROOF_RECALL_QUESTIONS` and
+ * `PROOF_RECALL_RUNS` point them outside this folder, so a question set
+ * about a private graph can be checked and run without ever entering the
+ * repository.
+ */
+export const QUESTIONS_DIR = process.env.PROOF_RECALL_QUESTIONS
+  ? resolve(process.env.PROOF_RECALL_QUESTIONS)
+  : join(BENCH, 'questions');
+export const RUNS_DIR = process.env.PROOF_RECALL_RUNS
+  ? resolve(process.env.PROOF_RECALL_RUNS)
+  : join(BENCH, 'runs');
 export const CATEGORIES = [
   'current-state', // what holds now
   'lookup', // a fact stated in one record
@@ -77,7 +90,7 @@ function args(argv) {
  * when `optIn` is true.
  */
 export async function loadQuestions(only = [], { optIn = false } = {}) {
-  const dir = join(BENCH, 'questions');
+  const dir = QUESTIONS_DIR;
   const files = (await readdir(dir))
     .filter((name) => name.endsWith('.json'))
     .sort();

benchmarks/proof-recall/scripts/find-recall.mjs

@@ -80,7 +80,7 @@ import { request } from 'node:http';
 import { tmpdir } from 'node:os';
 import { dirname, join, resolve } from 'node:path';
 import { fileURLToPath } from 'node:url';
-import { MULTI_EFFORT_SETS, pickQuestions } from './check.mjs';
+import { MULTI_EFFORT_SETS, RUNS_DIR, pickQuestions } from './check.mjs';
 import { findHash } from './collect.mjs';
 import { loadGraph, qualify } from './graph.mjs';
 import { prepareFolder } from './prepare-run.mjs';
@@ -111,7 +111,7 @@ function args(argv) {
     graph: join(BENCH, '.flatbread-proof'),
     ids: [],
     only: [],
-    out: join(BENCH, 'runs', 'find-recall.json'),
+    out: join(RUNS_DIR, 'find-recall.json'),
     jobs: 8,
     split: 'all',
   };

benchmarks/proof-recall/scripts/run.mjs

@@ -57,6 +57,7 @@ import {
   prepareFolder,
   publicQuestions,
 } from './prepare-run.mjs';
+import { RUNS_DIR } from './check.mjs';
 import { STRATEGIES, taskText } from './task.mjs';
 
 const BENCH = resolve(dirname(fileURLToPath(import.meta.url)), '..');
@@ -146,9 +147,11 @@ function serverAnswers(socket) {
       }
     );
     req.on('error', () => answer(false));
+    // A busy server accepts the connection but may not reply in time; a dead
+    // socket refuses it at once. So a timeout means a server is there.
     req.setTimeout(2000, () => {
       req.destroy();
-      answer(false);
+      answer(true);
     });
     req.end();
   });
@@ -185,9 +188,20 @@ async function ensureServer(socket) {
       { cwd: dir, detached: true, stdio: ['ignore', log, log] }
     );
     child.unref();
-    await writeFile(`${socket}.pid`, `${child.pid}\n`);
+    // A server busy building a large graph can miss the probe above, so this
+    // child may find the socket taken and exit. Record its pid only when it
+    // is the one that answers, or --stop-server would miss the live server.
+    let exited = false;
+    child.once('exit', () => (exited = true));
     for (let i = 0; i < 300; i++) {
-      if (await serverAnswers(socket)) return 'started';
+      if (await serverAnswers(socket)) {
+        // A child that found the socket taken exits within its own probe's
+        // two-second timeout; wait that long before calling it the server.
+        for (let j = 0; j < 15 && !exited; j++) await sleep(200);
+        if (exited) return 'reused';
+        await writeFile(`${socket}.pid`, `${child.pid}\n`);
+        return 'started';
+      }
       await sleep(200);
     }
     throw new Error(`proof serve did not start; see ${join(dir, 'serve.log')}`);
@@ -370,7 +384,7 @@ async function main() {
   // Root ignores file modes, so check that no session changed the graph.
   if ((await treeHash(shared)) !== hash)
     console.warn(`The shared graph in ${shared} changed during the run`);
-  await mkdir(join(BENCH, 'runs'), { recursive: true });
+  await mkdir(RUNS_DIR, { recursive: true });
   for (const strategy of options.strategies)
     await collect(
       sessions