diff --git a/benchmarks/proof-recall/README.md b/benchmarks/proof-recall/README.md index f13f4e00..42c42791 100644 --- a/benchmarks/proof-recall/README.md +++ b/benchmarks/proof-recall/README.md @@ -337,6 +337,14 @@ A benchmark agent must never see `spec/`, `questions/`, or `keymap.json`. with more right answers over the runs, and is the stricter reading. The script warns when the run files record different `find_hash` values. +**Question sets kept outside the repository.** A set about a private +graph must never enter Git. Set `PROOF_RECALL_QUESTIONS` to a folder of +question files and `PROOF_RECALL_RUNS` to a folder for run files; every +script then reads questions and writes runs there, and `--graph` points at +the graph, whose `keymap.json` maps `.` to record ids. With +neither set, nothing changes. Use such a set only as a held-out check: +never read its cards or outputs to choose a change. + **Ambiguous-Effort questions.** `questions/ambiguous.json` holds 24 questions rewritten from tier 0-2 questions so that they name no Effort and use words several Efforts share, such as "shader cache", "the diff --git a/benchmarks/proof-recall/scripts/check.mjs b/benchmarks/proof-recall/scripts/check.mjs index 2813a0a6..eb820135 100644 --- a/benchmarks/proof-recall/scripts/check.mjs +++ b/benchmarks/proof-recall/scripts/check.mjs @@ -24,6 +24,19 @@ import { } from './graph.mjs'; const BENCH = resolve(dirname(fileURLToPath(import.meta.url)), '..'); + +/** + * Where question files and run files live. `PROOF_RECALL_QUESTIONS` and + * `PROOF_RECALL_RUNS` point them outside this folder, so a question set + * about a private graph can be checked and run without ever entering the + * repository. + */ +export const QUESTIONS_DIR = process.env.PROOF_RECALL_QUESTIONS + ? resolve(process.env.PROOF_RECALL_QUESTIONS) + : join(BENCH, 'questions'); +export const RUNS_DIR = process.env.PROOF_RECALL_RUNS + ? resolve(process.env.PROOF_RECALL_RUNS) + : join(BENCH, 'runs'); export const CATEGORIES = [ 'current-state', // what holds now 'lookup', // a fact stated in one record @@ -77,7 +90,7 @@ function args(argv) { * when `optIn` is true. */ export async function loadQuestions(only = [], { optIn = false } = {}) { - const dir = join(BENCH, 'questions'); + const dir = QUESTIONS_DIR; const files = (await readdir(dir)) .filter((name) => name.endsWith('.json')) .sort(); diff --git a/benchmarks/proof-recall/scripts/find-recall.mjs b/benchmarks/proof-recall/scripts/find-recall.mjs index f070b856..45265703 100644 --- a/benchmarks/proof-recall/scripts/find-recall.mjs +++ b/benchmarks/proof-recall/scripts/find-recall.mjs @@ -80,7 +80,7 @@ import { request } from 'node:http'; import { tmpdir } from 'node:os'; import { dirname, join, resolve } from 'node:path'; import { fileURLToPath } from 'node:url'; -import { MULTI_EFFORT_SETS, pickQuestions } from './check.mjs'; +import { MULTI_EFFORT_SETS, RUNS_DIR, pickQuestions } from './check.mjs'; import { findHash } from './collect.mjs'; import { loadGraph, qualify } from './graph.mjs'; import { prepareFolder } from './prepare-run.mjs'; @@ -111,7 +111,7 @@ function args(argv) { graph: join(BENCH, '.flatbread-proof'), ids: [], only: [], - out: join(BENCH, 'runs', 'find-recall.json'), + out: join(RUNS_DIR, 'find-recall.json'), jobs: 8, split: 'all', }; diff --git a/benchmarks/proof-recall/scripts/run.mjs b/benchmarks/proof-recall/scripts/run.mjs index e4157eba..2603e378 100644 --- a/benchmarks/proof-recall/scripts/run.mjs +++ b/benchmarks/proof-recall/scripts/run.mjs @@ -57,6 +57,7 @@ import { prepareFolder, publicQuestions, } from './prepare-run.mjs'; +import { RUNS_DIR } from './check.mjs'; import { STRATEGIES, taskText } from './task.mjs'; const BENCH = resolve(dirname(fileURLToPath(import.meta.url)), '..'); @@ -146,9 +147,11 @@ function serverAnswers(socket) { } ); req.on('error', () => answer(false)); + // A busy server accepts the connection but may not reply in time; a dead + // socket refuses it at once. So a timeout means a server is there. req.setTimeout(2000, () => { req.destroy(); - answer(false); + answer(true); }); req.end(); }); @@ -185,9 +188,20 @@ async function ensureServer(socket) { { cwd: dir, detached: true, stdio: ['ignore', log, log] } ); child.unref(); - await writeFile(`${socket}.pid`, `${child.pid}\n`); + // A server busy building a large graph can miss the probe above, so this + // child may find the socket taken and exit. Record its pid only when it + // is the one that answers, or --stop-server would miss the live server. + let exited = false; + child.once('exit', () => (exited = true)); for (let i = 0; i < 300; i++) { - if (await serverAnswers(socket)) return 'started'; + if (await serverAnswers(socket)) { + // A child that found the socket taken exits within its own probe's + // two-second timeout; wait that long before calling it the server. + for (let j = 0; j < 15 && !exited; j++) await sleep(200); + if (exited) return 'reused'; + await writeFile(`${socket}.pid`, `${child.pid}\n`); + return 'started'; + } await sleep(200); } throw new Error(`proof serve did not start; see ${join(dir, 'serve.log')}`); @@ -370,7 +384,7 @@ async function main() { // Root ignores file modes, so check that no session changed the graph. if ((await treeHash(shared)) !== hash) console.warn(`The shared graph in ${shared} changed during the run`); - await mkdir(join(BENCH, 'runs'), { recursive: true }); + await mkdir(RUNS_DIR, { recursive: true }); for (const strategy of options.strategies) await collect( sessions