fix(evals): isolate daemon lifecycle and validate fact responses

crates/tracedecay-cli/tests/refresh_memory_journey.rs

@@ -17,6 +17,7 @@ use std::time::{Duration, Instant};
 
 use serde_json::{Value, json};
 use tempfile::TempDir;
+use tracedecay_runtime_core::logging::log_daemon_event;
 
 use common::fixture::{git_capture, git_run};
 use common::{
@@ -32,6 +33,17 @@ const FUNCTIONS_PER_FILE: usize = 160;
 const INDEX_TIMEOUT: Duration = Duration::from_secs(900);
 const STATUS_POLL: Duration = Duration::from_secs(5);
 
+fn report_phase(started: Instant, phase: &str, edit: usize) {
+    log_daemon_event(
+        "refresh_memory_journey_phase",
+        &[
+            ("phase", phase.to_owned()),
+            ("edit", edit.to_string()),
+            ("elapsed_ms", started.elapsed().as_millis().to_string()),
+        ],
+    );
+}
+
 /// Every function of every file changes with `edit`, so each refresh
 /// reparses the whole corpus.
 fn write_corpus(project: &Path, edit: usize) {
@@ -53,12 +65,25 @@ fn write_corpus(project: &Path, edit: usize) {
 }
 
 fn run_cli(home: &Path, project: &Path, args: &[&str]) -> Vec<u8> {
+    let started = Instant::now();
+    log_daemon_event(
+        "refresh_memory_journey_command_started",
+        &[("args", format!("{args:?}"))],
+    );
     let output = tracedecay_command_with_home(home)
         .args(args)
         .current_dir(project)
         .stdin(Stdio::null())
         .output()
         .unwrap_or_else(|error| panic!("tracedecay {args:?}: {error}"));
+    log_daemon_event(
+        "refresh_memory_journey_command_finished",
+        &[
+            ("args", format!("{args:?}")),
+            ("elapsed_ms", started.elapsed().as_millis().to_string()),
+            ("status", output.status.to_string()),
+        ],
+    );
     assert!(
         output.status.success(),
         "tracedecay {args:?} failed\nstdout:\n{}\nstderr:\n{}",
@@ -104,9 +129,19 @@ fn status(home: &Path, project: &Path) -> Value {
 /// Waits until the index serves a complete generation of `revision` and
 /// returns that generation's id.
 fn wait_for_current(home: &Path, project: &Path, revision: &str) -> String {
-    let deadline = Instant::now() + INDEX_TIMEOUT;
+    let started = Instant::now();
+    let deadline = started + INDEX_TIMEOUT;
     loop {
         let status = status(home, project);
+        log_daemon_event(
+            "refresh_memory_journey_status",
+            &[
+                ("expected_revision", revision.to_owned()),
+                ("elapsed_ms", started.elapsed().as_millis().to_string()),
+                ("freshness", status["code_index_freshness"].to_string()),
+                ("memory", status["memory"].to_string()),
+            ],
+        );
         let worktree = &status["code_index_freshness"]["worktree"];
         if status["code_index_freshness"]["status"] == "current"
             && worktree["source_revision"] == revision
@@ -156,6 +191,8 @@ fn assert_within_limit(home: &Path, project: &Path, pid: u32) {
 
 #[test]
 fn repeated_heavy_refreshes_under_the_limit_are_admitted() {
+    let started = Instant::now();
+    report_phase(started, "prepare_corpus", 0);
     let home = TempDir::new().expect("isolated home");
     let home = canonical_existing_path(home.path());
     let workspace = TempDir::new().expect("project root");
@@ -168,30 +205,39 @@ fn repeated_heavy_refreshes_under_the_limit_are_admitted() {
     let cold_revision = git_capture(&project, &["rev-parse", "HEAD"]);
 
     let log = home.join("daemon.log");
+    report_phase(started, "start_daemon", 0);
     let daemon = spawn_tracedecay_daemon_logged(&home, &log, |command| {
         command
             .env(
                 "TRACEDECAY_RESIDENT_MEMORY_LIMIT_BYTES",
                 LIMIT_BYTES.to_string(),
             )
+            .env("TRACEDECAY_TEST_DAEMON_LOG_STDERR", "1")
             .env("RUST_LOG", "info");
     });
+    report_phase(started, "initialize_project", 0);
     initialize_tracedecay_cli_project(&home, &project);
+    report_phase(started, "wait_for_current", 0);
     let mut generation = wait_for_current(&home, &project, &cold_revision);
     assert_within_limit(&home, &project, daemon.id());
+    report_phase(started, "generation_current", 0);
 
     for edit in 1..=REFRESHES {
+        report_phase(started, "prepare_corpus", edit);
         write_corpus(&project, edit);
         git_run(&project, &["commit", "--quiet", "-am", "edit every file"]);
         let revision = git_capture(&project, &["rev-parse", "HEAD"]);
+        report_phase(started, "sync", edit);
         run_cli(&home, &project, &["sync"]);
+        report_phase(started, "wait_for_current", edit);
         let refreshed = wait_for_current(&home, &project, &revision);
         assert_ne!(
             refreshed, generation,
             "refresh {edit} published a generation"
         );
         generation = refreshed;
         assert_within_limit(&home, &project, daemon.id());
+        report_phase(started, "generation_current", edit);
     }
 
     let log = fs::read_to_string(&log).expect("daemon log");
@@ -206,4 +252,5 @@ fn repeated_heavy_refreshes_under_the_limit_are_admitted() {
         "the daemon refused work it had room for:\n{log}"
     );
     drop(daemon);
+    report_phase(started, "complete", REFRESHES);
 }

crates/tracedecay/tests/common/mod.rs

@@ -693,6 +693,7 @@ fn drain_daemon_stderr(
     startup: String,
     mut stderr: BufReader<std::process::ChildStderr>,
     log: Option<PathBuf>,
+    mirror_stderr: bool,
 ) {
     std::thread::spawn(move || {
         if let Some(path) =
@@ -703,6 +704,18 @@ fn drain_daemon_stderr(
                 .open(path)
         {
             let _ = file.write_all(startup.as_bytes());
+            if mirror_stderr {
+                let mut line = Vec::new();
+                while stderr
+                    .read_until(b'\n', &mut line)
+                    .is_ok_and(|read| read != 0)
+                {
+                    let _ = file.write_all(&line);
+                    let _ = std::io::stderr().write_all(&line);
+                    line.clear();
+                }
+                return;
+            }
             let _ = std::io::copy(&mut stderr, &mut file);
             return;
         }
@@ -1165,6 +1178,8 @@ pub fn spawn_tracedecay_daemon_with(
 
 /// [`spawn_tracedecay_daemon_with`], streaming the daemon's stderr to `log`
 /// for assertions over its diagnostic events.
+/// Set `TRACEDECAY_TEST_DAEMON_LOG_STDERR=1` in `configure` to also stream the
+/// diagnostics to test stderr, preserving evidence when a runner kills a test.
 pub fn spawn_tracedecay_daemon_logged(
     home: &Path,
     log: &Path,
@@ -1272,6 +1287,9 @@ fn spawn_tracedecay_daemon_process(
         .stdout(Stdio::null())
         .stderr(Stdio::piped());
     configure(&mut command);
+    let mirror_stderr = command.get_envs().any(|(key, value)| {
+        key == "TRACEDECAY_TEST_DAEMON_LOG_STDERR" && value == Some(OsStr::new("1"))
+    });
     bind_to_test_process(&mut command);
     let child = command.spawn().expect("tracedecay daemon should start");
     let mut daemon = DaemonProcess::new(child);
@@ -1312,6 +1330,9 @@ fn spawn_tracedecay_daemon_process(
         let read = stderr
             .read_line(&mut startup)
             .expect("daemon stderr should be readable");
+        if mirror_stderr {
+            let _ = std::io::stderr().write_all(&startup.as_bytes()[line_start..]);
+        }
         if read == 0 {
             let status = daemon
                 .child
@@ -1326,7 +1347,7 @@ fn spawn_tracedecay_daemon_process(
             break;
         }
     }
-    drain_daemon_stderr(startup, stderr, log);
+    drain_daemon_stderr(startup, stderr, log, mirror_stderr);
     daemon
 }
 

crates/tracedecay/tests/runtime_acceptance_suite/host_event_fixture_test.rs

@@ -1997,18 +1997,21 @@ fn assert_legal_host_response(provider: &str, state: &str, expected: &Value, out
             *project_root = Value::String("<PROJECT_ROOT>".to_string());
         }
         assert_eq!(document, expected, "{provider}/{state} stdout");
-    } else if provider == "pi" && !stdout.is_empty() {
-        // The real daemon can finish its advisory cycle before this hook.
-        // Pi carries that notice as plain text rather than a JSON context field.
-        // This fixture seeds no findings; reject arbitrary guidance or payloads.
-        assert_eq!(expected["stdout"].as_str().unwrap(), "");
+    } else if (provider == "pi" && !stdout.is_empty())
+        || (provider == "kiro" && stdout != expected["stdout"].as_str().unwrap())
+    {
+        // Advisory completion can race with admission. These hosts carry its
+        // zero-finding notice as plain text.
+        assert_eq!(
+            expected["stdout"].as_str().unwrap(),
+            if provider == "kiro" { "{}" } else { "" }
+        );
         let encoded = stdout
             .strip_prefix("TraceDecay feedback ready for authorized lookup: ")
             .unwrap_or_else(|| panic!("{provider}/{state} emitted unexpected context: {stdout}"));
         let notice: tracedecay_application::advisory::AdvisoryHookLookupNoticeV1 =
-            serde_json::from_str(encoded)
-                .expect("Pi feedback notice must match its typed contract");
-        notice.validate().expect("Pi feedback notice must be valid");
+            serde_json::from_str(encoded).expect("feedback notice must match its typed contract");
+        notice.validate().expect("feedback notice must be valid");
         assert_eq!(notice.returned_findings, 0);
         assert_eq!(notice.omitted_findings, 0);
     } else {

evals/memory/BUILD.bazel

@@ -7,6 +7,7 @@ py_test(
         "//benchmark_data:run_benchmarks.py",
         "//evals/hermetic:score.py",
         "//scripts:build-eval-binary.sh",
+        "//scripts:lib/portable_process.py",
     ],
     imports = ["."],
     legacy_create_init = 0,

evals/memory/README.md

@@ -30,6 +30,12 @@ python3 evals/memory/run_real_model.py --scenario memory-no-pollution \
 The real-model runner creates a unique temporary user home for each run and
 installs the normal user-level Hermes integration there. It never selects a
 named Hermes profile or uses `HERMES_HOME` as TraceDecay storage.
+Each scenario also starts an isolated `tracedecay daemon run` with a private
+socket that must be ready before
+`init`. Setup, daemon, agent, and cancellation failures are written to the
+report as error entries. Cleanup is attempted on every exit path. If owned
+work cannot be stopped or deletion fails, retained paths and cleanup errors
+are recorded in the report.
 
 Run reports land under `evals/memory/runs/<timestamp>/` (gitignored).