diff --git a/crates/tracedecay-agent-hosts/src/agents/claude/tests.rs b/crates/tracedecay-agent-hosts/src/agents/claude/tests.rs index 07725b1ec1..db426a6db3 100644 --- a/crates/tracedecay-agent-hosts/src/agents/claude/tests.rs +++ b/crates/tracedecay-agent-hosts/src/agents/claude/tests.rs @@ -1,4 +1,7 @@ -use super::super::{load_json_file_strict, safe_write_json_file, safe_write_text_file}; +use super::super::{ + load_json_file_strict, safe_write_json_file, safe_write_text_file, + unexpected_bundle_entrypoints, +}; use super::*; use serde_json::json; #[cfg(unix)] @@ -148,46 +151,46 @@ fn native_cache_content_drift_and_binary_relocation_require_refresh() { .unwrap() ); copy_rendered_bundle_to_native_cache(home.path(), new_bin); + let source = plugin_deploy_dir(home.path()); + let cache = claude_current_cached_plugin_root(home.path()); let rendered = rendered_plugin_files(new_bin, &ServeArgs::default()).unwrap(); let relatives = rendered .iter() - .map(|(relative, _)| (*relative).to_string()) + .map(|(relative, _)| (*relative).to_owned()) .collect::>(); - for root in [ - plugin_deploy_dir(home.path()), - claude_current_cached_plugin_root(home.path()), - ] { - for (relative, expected) in &rendered { - let path = root.join(relative); - let actual = std::fs::read(&path).unwrap(); - assert!( - actual == expected.as_bytes(), - "relocated bundle differs at {}: actual bytes={}, expected bytes={}, first differing byte={:?}", - path.display(), - actual.len(), - expected.len(), - actual - .iter() - .zip(expected.as_bytes()) - .position(|(actual, expected)| actual != expected) - ); - } + for (relative, expected) in rendered { assert_eq!( - super::super::unexpected_bundle_entrypoints( - &root, - &relatives, - &[".claude-plugin", "agents", "commands", "hooks", "skills"], - ) - .unwrap(), - Vec::::new(), - "relocation left auto-discovered entries in {}", + std::fs::read_to_string(source.join(relative)).unwrap(), + expected, + "relocated marketplace bytes differ at {relative}" + ); + assert_eq!( + std::fs::read_to_string(cache.join(relative)).unwrap(), + expected, + "refreshed native cache bytes differ at {relative}" + ); + } + for root in [&source, &cache] { + let unexpected = unexpected_bundle_entrypoints( + root, + &relatives, + &[".claude-plugin", "agents", "commands", "hooks", "skills"], + ) + .unwrap(); + assert!( + unexpected.is_empty(), + "refresh left auto-discovered entrypoints outside the catalog at {}: {unexpected:?}", root.display() ); } - assert!(claude_plugin_registration_is_active(home.path()).unwrap()); + assert!( + claude_plugin_registration_is_active(home.path()).unwrap(), + "refresh must preserve the exact native plugin registration" + ); assert!( claude_plugin_is_natively_active(home.path(), Some((new_bin, &ServeArgs::default()))) - .unwrap() + .unwrap(), + "matching refreshed bytes must retain the exact discovery inventory" ); } diff --git a/crates/tracedecay-cli/tests/core_cli_suite/cli_non_interactive_test.rs b/crates/tracedecay-cli/tests/core_cli_suite/cli_non_interactive_test.rs index 25be1f0fc4..3aa1fcd802 100644 --- a/crates/tracedecay-cli/tests/core_cli_suite/cli_non_interactive_test.rs +++ b/crates/tracedecay-cli/tests/core_cli_suite/cli_non_interactive_test.rs @@ -1200,14 +1200,21 @@ fn init_wait_holds_until_first_generation_is_ready() { git(project.path(), &["init", "-b", "main"]); commit_all(project.path(), "fixture repository"); + let daemon_log = home.path().join("init-wait-daemon.log"); + let _daemon = + crate::common::spawn_tracedecay_daemon_logged(home.path(), &daemon_log, |command| { + command.env("RUST_LOG", "info"); + }); let mut command = tracedecay_command(home.path(), project.path()); command.args(["init", "--wait"]); let output = run_with_timeout(command, cli_timeout()); assert!( output.status.success(), - "init --wait should succeed once the first generation is ready\nstdout:\n{}\nstderr:\n{}", + "init --wait should succeed once the first generation is ready\nstdout:\n{}\nstderr:\n{}\ndaemon:\n{}", String::from_utf8_lossy(&output.stdout), - String::from_utf8_lossy(&output.stderr) + String::from_utf8_lossy(&output.stderr), + std::fs::read_to_string(&daemon_log) + .unwrap_or_else(|error| format!("daemon log unavailable: {error}")) ); let stderr = String::from_utf8_lossy(&output.stderr); assert!( @@ -3328,7 +3335,11 @@ fn branch_search_serves_a_committed_generation_behind_dirty_worktree_state() { .unwrap(); init_project_fixture(home.path(), &project_root); - let _daemon = crate::common::spawn_tracedecay_daemon(home.path()); + let daemon_log = home.path().join("dirty-branch-daemon.log"); + let _daemon = + crate::common::spawn_tracedecay_daemon_logged(home.path(), &daemon_log, |command| { + command.env("RUST_LOG", "info"); + }); let ready_deadline = Instant::now() + Duration::from_secs(30); loop { let mut ready = tracedecay_command_without_daemon(home.path(), &project_root); @@ -3352,9 +3363,11 @@ fn branch_search_serves_a_committed_generation_behind_dirty_worktree_state() { } assert!( Instant::now() < ready_deadline, - "dirty worktree generation did not become queryable\nstdout:\n{}\nstderr:\n{}", + "dirty worktree generation did not become queryable\nstdout:\n{}\nstderr:\n{}\ndaemon:\n{}", String::from_utf8_lossy(&ready.stdout), - String::from_utf8_lossy(&ready.stderr) + String::from_utf8_lossy(&ready.stderr), + std::fs::read_to_string(&daemon_log) + .unwrap_or_else(|error| format!("daemon log unavailable: {error}")) ); std::thread::sleep(Duration::from_millis(100)); } diff --git a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/memory_tests.rs b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/memory_tests.rs index 0ebe34ada8..62139a246d 100644 --- a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/memory_tests.rs +++ b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/memory_tests.rs @@ -73,6 +73,17 @@ fn expected_worker_reservation_on(runtime: &CodeIndexWorkerRuntimeV1, remaining_ ) } +/// Ledger scenarios own their measured input independently of sibling tests. +pub(super) fn ledger_memory(limit: NonZeroU64) -> Arc { + Arc::new(ProcessResidentMemoryV1::with_pressure( + limit, + Arc::new(ResidentMemoryPressureV1::with_sampler( + limit, + Arc::new(|| None), + )), + )) +} + #[test] fn captured_source_bytes_are_charged_until_the_snapshot_drops() { let project = fixture(); @@ -84,9 +95,7 @@ fn captured_source_bytes_are_charged_until_the_snapshot_drops() { Arc::new(SharedCodeIndexBytePoolV1::default()), ) .expect("open scheduler"); - let authority = Arc::new(ProcessResidentMemoryV1::new( - NonZeroU64::new(1024 * 1024).expect("source memory limit"), - )); + let authority = ledger_memory(NonZeroU64::new(1024 * 1024).expect("source memory limit")); scheduler.bind_resident_memory(Arc::clone(&authority)); let captured = scheduler @@ -119,9 +128,7 @@ fn reconcile_leaves_no_captured_source_charged_after_the_build() { Arc::new(SharedCodeIndexBytePoolV1::default()), ) .expect("open scheduler"); - let authority = Arc::new(ProcessResidentMemoryV1::new( - DEFAULT_PROCESS_RESIDENT_MEMORY_LIMIT_V1, - )); + let authority = ledger_memory(DEFAULT_PROCESS_RESIDENT_MEMORY_LIMIT_V1); scheduler.bind_resident_memory(Arc::clone(&authority)); let captured = scheduler .capture_authoritative_snapshot(None) @@ -171,7 +178,7 @@ fn source_capture_refuses_before_build_when_retention_cannot_be_charged() { Arc::new(SharedCodeIndexBytePoolV1::default()), ) .expect("open scheduler"); - let authority = Arc::new(ProcessResidentMemoryV1::new(NonZeroU64::MIN)); + let authority = ledger_memory(NonZeroU64::MIN); scheduler.bind_resident_memory(Arc::clone(&authority)); assert!(matches!( @@ -291,9 +298,7 @@ fn worker_memory_reservation_is_charged_and_released_by_raii() { 2, DEFAULT_PROCESS_RESIDENT_MEMORY_LIMIT_V1.get(), ); - let authority = Arc::new(ProcessResidentMemoryV1::new( - DEFAULT_PROCESS_RESIDENT_MEMORY_LIMIT_V1, - )); + let authority = ledger_memory(DEFAULT_PROCESS_RESIDENT_MEMORY_LIMIT_V1); scheduler.bind_resident_memory(Arc::clone(&authority)); let reservation = scheduler .reserve_worker_memory() @@ -325,14 +330,7 @@ fn default_authority_worker_reserve_leaves_typed_snapshot_headroom() { let runtime = bind_automatic_worker_runtime(&scheduler, 128, 128 * 1024 * 1024 * 1024); // This case isolates the ledger's worker/snapshot split. Live process // pressure can narrow the slab and is exercised with explicit samples below. - let pressure = Arc::new(ResidentMemoryPressureV1::with_sampler( - DEFAULT_PROCESS_RESIDENT_MEMORY_LIMIT_V1, - Arc::new(|| None), - )); - let authority = Arc::new(ProcessResidentMemoryV1::with_pressure( - DEFAULT_PROCESS_RESIDENT_MEMORY_LIMIT_V1, - pressure, - )); + let authority = ledger_memory(DEFAULT_PROCESS_RESIDENT_MEMORY_LIMIT_V1); scheduler.bind_resident_memory(Arc::clone(&authority)); let _worker = scheduler @@ -376,12 +374,12 @@ fn worker_memory_reservation_refusal_is_typed() { 2, DEFAULT_PROCESS_RESIDENT_MEMORY_LIMIT_V1.get(), ); - let authority = Arc::new(ProcessResidentMemoryV1::new( + let authority = ledger_memory( NonZeroU64::new( tracedecay_code_index::parallelism::INDEX_WORKER_RESIDENT_BUDGET_BYTES_V1 - 1, ) .expect("positive test limit"), - )); + ); scheduler.bind_resident_memory(authority); assert!(matches!( scheduler.reserve_worker_memory(), diff --git a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/reconcile.rs b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/reconcile.rs index 14e34cf58d..d8bbfe3f5e 100644 --- a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/reconcile.rs +++ b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/reconcile.rs @@ -37,6 +37,8 @@ use tracedecay_domain::{ }; use tracedecay_graph_db::GraphConflictContextV1; use tracedecay_privacy::CODE_SOURCE_SANITIZER_VERSION_V1; +#[cfg(any(test, feature = "test-helpers"))] +use tracedecay_runtime_core::resident_memory::ResidentMemoryPressureV1; use tracedecay_runtime_core::resident_memory::{ ProcessResidentMemoryV1, RESIDENT_MEMORY_PRESSURE_ADMISSION_FLOOR_BYTES_V1, RESIDENT_OWNER_IDLE_WINDOW_V1, ResidentMemoryAdmissionFailureV1, ResidentMemoryComponentIdV1, @@ -1250,7 +1252,9 @@ impl HistoricalCodeIndexGenerationOwnerV1 { } impl CodeIndexWorktreeSchedulerV1 { - /// A standalone owner under the path policy a fresh profile resolves. + /// A standalone fixture owner under the path policy a fresh profile resolves. + /// Concurrent cases do not contribute measured pressure to this owner; + /// pressure scenarios bind their own explicit readings after opening it. #[cfg(any(test, feature = "test-helpers"))] pub fn open( project_id: ProjectId, @@ -1258,14 +1262,23 @@ impl CodeIndexWorktreeSchedulerV1 { store_root: PathBuf, byte_pool: Arc, ) -> Result { - Self::open_with_policy( + let mut scheduler = Self::open_with_policy( project_id, project_root, store_root, byte_pool, CodeIndexHintPolicyV1::default(), crate::config::registry_default_index_path_policy(), - ) + )?; + let limit = detected_process_resident_memory_limit_v1(); + scheduler.bind_resident_memory(Arc::new(ProcessResidentMemoryV1::with_pressure( + limit, + Arc::new(ResidentMemoryPressureV1::with_sampler( + limit, + Arc::new(|| None), + )), + ))); + Ok(scheduler) } pub fn open_with_policy( diff --git a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/registry/resident_memory.rs b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/registry/resident_memory.rs index b971b0dc1a..486d42d8fb 100644 --- a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/registry/resident_memory.rs +++ b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/registry/resident_memory.rs @@ -2,11 +2,12 @@ use std::sync::Arc; -#[cfg(any(test, feature = "test-helpers"))] -use tracedecay_runtime_core::resident_memory::DEFAULT_PROCESS_RESIDENT_MEMORY_LIMIT_V1; pub use tracedecay_runtime_core::resident_memory::ProcessResidentMemoryV1; #[cfg(any(test, feature = "test-helpers"))] -use tracedecay_runtime_core::resident_memory::RESIDENT_OWNER_IDLE_WINDOW_V1; +use tracedecay_runtime_core::resident_memory::{ + DEFAULT_PROCESS_RESIDENT_MEMORY_LIMIT_V1, RESIDENT_OWNER_IDLE_WINDOW_V1, + ResidentMemoryPressureV1, +}; use tracedecay_runtime_core::resident_memory::{ResidentOwnersV1, process_resident_owners_v1}; use super::CodeIndexSchedulerRegistryV1; @@ -15,10 +16,17 @@ impl CodeIndexSchedulerRegistryV1 { #[cfg(any(test, feature = "test-helpers"))] #[cfg_attr(not(test), allow(dead_code))] pub fn new(max_worktrees: usize) -> Self { + // A fixture owns its ledger and pressure readings. Concurrent tests' + // RSS is not an input; pressure journeys pass an explicit authority. + let pressure = Arc::new(ResidentMemoryPressureV1::with_sampler( + DEFAULT_PROCESS_RESIDENT_MEMORY_LIMIT_V1, + Arc::new(|| None), + )); Self::with_resident_memory_and_progress_producer_incarnation( max_worktrees, - Arc::new(ProcessResidentMemoryV1::new( + Arc::new(ProcessResidentMemoryV1::with_pressure( DEFAULT_PROCESS_RESIDENT_MEMORY_LIMIT_V1, + pressure, )), 1, ) diff --git a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/tests/residency.rs b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/tests/residency.rs index 4cd725b7cb..875d53625d 100644 --- a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/tests/residency.rs +++ b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/tests/residency.rs @@ -1476,14 +1476,6 @@ fn ledger_leaving_one_mib( .expect("the held memory fits the ledger") } -fn sixteen_gib_authority() -> Arc { - let limit = NonZeroU64::new(16 * 1024 * 1024 * 1024).unwrap(); - Arc::new(ProcessResidentMemoryV1::with_pressure( - limit, - Arc::new(ResidentMemoryPressureV1::new(limit)), - )) -} - /// A 16 GiB authority reading a fixed process size, so a held ledger refuses /// a refresh however much memory the tests beside it hold. fn fixed_reading_authority() -> Arc { @@ -1510,7 +1502,7 @@ async fn a_refresh_refused_for_memory_publishes_when_an_idle_owner_releases_it() let fixture = GitFixture::new(&[("src/main.rs", "fn main() {}\n")]); let store = TempDir::new().expect("store root"); let owners = Arc::new(ResidentOwnersV1::new(IDLE_WINDOW)); - let resident_memory = sixteen_gib_authority(); + let resident_memory = fixed_reading_authority(); let (registry, _) = mounted_core_query_worktree_in( CodeIndexSchedulerRegistryV1::with_resident_memory(1, Arc::clone(&resident_memory)) .with_resident_owners(Arc::clone(&owners)), @@ -1576,7 +1568,7 @@ async fn a_refresh_refused_by_the_ledger_publishes_when_the_blocking_reservation let fixture = GitFixture::new(&[("src/main.rs", "fn main() {}\n")]); let store = TempDir::new().expect("store root"); let owners = Arc::new(ResidentOwnersV1::new(IDLE_WINDOW)); - let resident_memory = sixteen_gib_authority(); + let resident_memory = fixed_reading_authority(); let (registry, _) = mounted_core_query_worktree_in( CodeIndexSchedulerRegistryV1::with_resident_memory(1, Arc::clone(&resident_memory)) .with_resident_owners(Arc::clone(&owners)), diff --git a/crates/tracedecay-code-index-runtime/tests/code_index_scheduler/reconcile.rs b/crates/tracedecay-code-index-runtime/tests/code_index_scheduler/reconcile.rs index 30f6f177a3..182c11a2ba 100644 --- a/crates/tracedecay-code-index-runtime/tests/code_index_scheduler/reconcile.rs +++ b/crates/tracedecay-code-index-runtime/tests/code_index_scheduler/reconcile.rs @@ -30,9 +30,11 @@ use tracedecay_domain::{ use tracedecay_runtime_core::resident_memory::{ DEFAULT_PROCESS_RESIDENT_MEMORY_LIMIT_V1, ProcessResidentMemoryV1, ResidentHoldingV1, ResidentOwnerBytesV1, ResidentOwnerKindV1, ResidentOwnerReleaseV1, ResidentOwnerSampleV1, - ResidentOwnerScopeV1, ResidentOwnerV1, ResidentOwnersV1, sampled_process_resident_bytes_v1, + ResidentOwnerScopeV1, ResidentOwnerV1, ResidentOwnersV1, + detected_process_resident_memory_limit_v1, sampled_process_resident_bytes_v1, }; +use super::super::memory_tests::ledger_memory; #[cfg(unix)] use super::stage_raw_git_path; use super::{ @@ -1437,12 +1439,17 @@ fn retained_decode_incremental_seal_and_text_publish_stay_below_the_high_waterma let fixture = GitFixture::new(&sources); let store = TempDir::new().expect("store root"); + // This scale journey measures real process pressure through admission + // as well as its RSS assertions, so retain the production pressure cell. + let memory_limit = detected_process_resident_memory_limit_v1(); + { let mut seed = scheduler( &fixture, store.path().to_path_buf(), Arc::new(SharedCodeIndexBytePoolV1::default()), ); + seed.bind_resident_memory(Arc::new(ProcessResidentMemoryV1::new(memory_limit))); published(seed.reconcile_now().expect("seed retained generation")); } @@ -1455,6 +1462,7 @@ fn retained_decode_incremental_seal_and_text_publish_stay_below_the_high_waterma store.path().to_path_buf(), Arc::new(SharedCodeIndexBytePoolV1::default()), ); + scheduler.bind_resident_memory(Arc::new(ProcessResidentMemoryV1::new(memory_limit))); assert_eq!(scheduler.publication.sealed_decode_count(), 0); assert!(matches!( scheduler @@ -11410,7 +11418,7 @@ fn graph_off_changed_source_worker_memory_denial_retries_without_decode() { ) > tight_limit.get(), "the fixture limit must deny the installed worker plan" ); - let tight = Arc::new(ProcessResidentMemoryV1::new(tight_limit)); + let tight = ledger_memory(tight_limit); scheduler.bind_resident_memory(Arc::clone(&tight)); let denied = scheduler.reconcile_retained_text_generation_with(&metadata_a, true); assert!( @@ -11435,9 +11443,7 @@ fn graph_off_changed_source_worker_memory_denial_retries_without_decode() { "a denied worker reservation must not leak a charge" ); - let adequate = Arc::new(ProcessResidentMemoryV1::new( - DEFAULT_PROCESS_RESIDENT_MEMORY_LIMIT_V1, - )); + let adequate = ledger_memory(DEFAULT_PROCESS_RESIDENT_MEMORY_LIMIT_V1); scheduler.bind_resident_memory(Arc::clone(&adequate)); let outcome = scheduler .reconcile_retained_text_generation_with(&metadata_a, true) diff --git a/crates/tracedecay-code-index-runtime/tests/code_index_scheduler/serving.rs b/crates/tracedecay-code-index-runtime/tests/code_index_scheduler/serving.rs index e5c882effc..00751d63f6 100644 --- a/crates/tracedecay-code-index-runtime/tests/code_index_scheduler/serving.rs +++ b/crates/tracedecay-code-index-runtime/tests/code_index_scheduler/serving.rs @@ -37,7 +37,7 @@ use tracedecay_domain::{ PrivacyDomainId, ProjectId, ProviderEvaluationStateV1, PublicRetrieverStatus, QueryNormalizationRevision, RelationEdgeKindV1, RetrievalBudget, RetrievalRequest, RetrievalScope, RetrievalSnapshot, RetrieverKind, RetrieverOutcome, SanitizerRevision, - ScoreDomainId, SingleRootScopeV1, TemporalModeV1, UtcMicros, VectorWatermark, + ScoreDomainId, SingleRootScopeV1, TemporalModeV1, UtcMicros, VectorWatermark, WorktreeId, encode_lowercase_hex, sha256_hex_suffix, }; use tracedecay_query::retrieval::{ @@ -54,12 +54,14 @@ use tracedecay_query::retrieval::{ }; use tracedecay_runtime_core::resident_memory::{ DEFAULT_PROCESS_RESIDENT_MEMORY_LIMIT_V1, ProcessResidentMemoryV1, ProcessResidentSampleV1, - RESIDENT_MEMORY_PRESSURE_HIGH_WATERMARK_PERMILLE_V1, ResidentMemoryComponentIdV1, - ResidentMemoryPressureV1, resident_memory_watermark_bytes_v1, + RESIDENT_MEMORY_PRESSURE_HIGH_WATERMARK_PERMILLE_V1, ResidentMemoryAdmissionFailureV1, + ResidentMemoryComponentIdV1, ResidentMemoryKeyV1, ResidentMemoryPressureV1, + resident_memory_watermark_bytes_v1, }; use tracedecay_session_temporal_store::SessionTemporalAccess; +use super::super::memory_tests::ledger_memory; use super::{ ALPHA_LIB_V1, CALLER_PAGE, CALLER_STAR, GitFixture, OwnerSignals, ReadyRetrievalControlV1, SERVING_SEAT_FAILURE_CEILING, active_text_artifact_path, application_context, @@ -2072,9 +2074,9 @@ fn reader_reservation_refusal_precedes_missing_artifact_access() { store.path().to_path_buf(), Arc::new(SharedCodeIndexBytePoolV1::default()), ); - scheduler.bind_resident_memory(Arc::new(ProcessResidentMemoryV1::new( + scheduler.bind_resident_memory(ledger_memory( std::num::NonZeroU64::new(1024 * 1024).expect("tight memory limit"), - ))); + )); let latest = scheduler.latest_complete().expect("restored generation"); assert!( matches!( @@ -2090,6 +2092,55 @@ fn reader_reservation_refusal_precedes_missing_artifact_access() { assert!(latest.text_projection_needs_work()); } +#[test] +fn overlapping_text_builds_share_one_admission_watermark_headroom() { + let limit = NonZeroU64::new(1_000).expect("nonzero process ceiling"); + let authority = Arc::new(ProcessResidentMemoryV1::with_pressure( + limit, + Arc::new(ResidentMemoryPressureV1::with_sampler( + limit, + Arc::new(|| None), + )), + )); + let key = ResidentMemoryKeyV1 { + project_id: ProjectId::new("project.overlapping-text-builds").expect("project"), + worktree_id: WorktreeId::new("worktree.overlapping-text-builds").expect("worktree"), + generation_id: CodeGenerationId::new("generation.overlapping-text-builds") + .expect("generation"), + component: ResidentMemoryComponentIdV1::new("code-text-artifact-build").expect("component"), + }; + let requested = NonZeroU64::new(300).expect("nonzero build request"); + let builds = [(); 3].map(|()| { + authority + .reserve_up_to( + key.clone(), + requested, + authority.admission_watermark_bytes(), + ) + .expect("each build fits beneath the shared 900-byte watermark") + }); + + assert_eq!( + authority.snapshot().used_bytes, + 900, + "the retained ledger owns the three build ceilings, not three margins" + ); + assert!(matches!( + authority.reserve_up_to( + key, + NonZeroU64::new(100).expect("remaining hard-limit room"), + authority.admission_watermark_bytes(), + ), + Err(ResidentMemoryAdmissionFailureV1::ReservationCeiling { + used_bytes: 900, + requested_bytes: 100, + limit_bytes: 900, + }) + )); + drop(builds); + assert_eq!(authority.snapshot().used_bytes, 0); +} + #[test] fn retained_text_admission_reports_its_block_and_recovers_after_headroom_release() { let fixture = GitFixture::new(&[("src/lib.rs", "pub fn reserved_until_release() {}\n")]); @@ -2391,9 +2442,7 @@ fn text_artifact_ceilings_reserve_through_process_resident_memory() { Arc::new(SharedCodeIndexBytePoolV1::default()), ); published(scheduler.reconcile_now().expect("publish")); - let tight = Arc::new(ProcessResidentMemoryV1::new( - std::num::NonZeroU64::new(1024 * 1024).expect("tight limit"), - )); + let tight = ledger_memory(std::num::NonZeroU64::new(1024 * 1024).expect("tight limit")); scheduler.bind_resident_memory(Arc::clone(&tight)); let latest = scheduler.latest_complete().expect("latest generation"); let denied = latest.advance_text_serving(usize::MAX); @@ -2422,7 +2471,14 @@ fn text_artifact_ceilings_reserve_through_process_resident_memory() { RESIDENT_MEMORY_PRESSURE_HIGH_WATERMARK_PERMILLE_V1, ); let fitting_live = watermark - build_floor; - for (live_bytes, admitted) in [(fitting_live, true), (fitting_live + 1, false)] { + for (live_bytes, cgroup_headroom, admitted) in [ + (fitting_live, None, true), + (fitting_live + 1, None, false), + // Shared cgroup room must cover the actual build, not a second copy + // of the band above the process admission watermark. + (0, Some(build_floor), true), + (0, Some(build_floor - 1), false), + ] { let pressure = Arc::new(ResidentMemoryPressureV1::with_sampler( measured_limit, Arc::new(move || { @@ -2430,7 +2486,7 @@ fn text_artifact_ceilings_reserve_through_process_resident_memory() { resident_bytes: live_bytes, unreclaimable_bytes: live_bytes, swapped_bytes: 0, - cgroup_headroom_bytes: None, + cgroup_headroom_bytes: cgroup_headroom, }) }), )); @@ -2447,11 +2503,13 @@ fn text_artifact_ceilings_reserve_through_process_resident_memory() { let latest = scheduler .latest_complete() .expect("measured latest generation"); - let advanced = latest.advance_text_serving(1); + // Open the build and its reservation without consuming source pages, + // so every case tests admission of the same unpublished artifact. + let advanced = latest.advance_text_serving(0); if admitted { assert!( advanced.is_ok(), - "{live_bytes} B live leaves the build floor below the watermark: {advanced:?}" + "{live_bytes} B live with cgroup room {cgroup_headroom:?} admits the build floor: {advanced:?}" ); } else { assert!( @@ -2459,7 +2517,7 @@ fn text_artifact_ceilings_reserve_through_process_resident_memory() { advanced, Err(tracedecay_query::retrieval::RetrievalPortError::ResidentMemoryRefused(_)) ), - "{live_bytes} B live leaves less than the build floor: {advanced:?}" + "{live_bytes} B live with cgroup room {cgroup_headroom:?} leaves less than the build floor: {advanced:?}" ); assert_eq!( measured.snapshot().used_bytes, @@ -2471,9 +2529,7 @@ fn text_artifact_ceilings_reserve_through_process_resident_memory() { // An adequate authority admits the build and holds the reader charge // for as long as the artifact owners serve. - let adequate = Arc::new(ProcessResidentMemoryV1::new( - DEFAULT_PROCESS_RESIDENT_MEMORY_LIMIT_V1, - )); + let adequate = ledger_memory(DEFAULT_PROCESS_RESIDENT_MEMORY_LIMIT_V1); { // A fresh scheduler restores the durable generation; the unchanged // fixture needs no republish. diff --git a/crates/tracedecay-global-db/src/observation_adapter.rs b/crates/tracedecay-global-db/src/observation_adapter.rs index 5bc6062563..648f47944a 100644 --- a/crates/tracedecay-global-db/src/observation_adapter.rs +++ b/crates/tracedecay-global-db/src/observation_adapter.rs @@ -984,7 +984,8 @@ async fn load_observation_preflight( } } let (retrieval_aliases, alias_owners) = - read_retrieval_aliases_from_snapshot(&snapshot, writes, OPERATION).await?; + read_retrieval_aliases_from_snapshot(&snapshot, writes, &admission_refusals, OPERATION) + .await?; let mut sibling_candidate_anchors = HashSet::new(); for write in writes { let observation = write.observation(); @@ -1255,6 +1256,7 @@ fn retrieval_alias_key( async fn read_retrieval_aliases_from_snapshot( snapshot: &DatabaseEngineReadSnapshot, writes: &[AnchoredObservationWrite], + admission_refusals: &HashMap<(String, String), PayloadDigestV1>, operation: &'static str, ) -> ObservationStoreResult<( HashMap<(String, String, String), RetrievalAnchorId>, @@ -1262,6 +1264,13 @@ async fn read_retrieval_aliases_from_snapshot( )> { let mut requested = HashSet::new(); for write in writes { + let observation = write.observation(); + if admission_refusals.contains_key(&( + observation.observation_id().as_str().to_owned(), + observation.payload_reference().digest().as_str().to_owned(), + )) { + continue; + } for alias in write.retrieval_anchor().aliases() { requested.insert(retrieval_alias_key(write.observation().scope(), alias)?); } @@ -1285,10 +1294,13 @@ async fn read_retrieval_aliases_from_snapshot( let mut rows = snapshot .query( "SELECT alias.owner_json, alias.alias_kind, alias.locator_digest, - alias.anchor_id, binding.observation_id + alias.anchor_id, owner.observation_id FROM retrieval_anchor_aliases AS alias LEFT JOIN observation_retrieval_anchors AS binding ON binding.anchor_id = alias.anchor_id + LEFT JOIN observations AS owner + ON owner.observation_id = binding.observation_id + AND json_extract(owner.observation_json, '$.__retention_released') IS NULL JOIN json_each(?1) AS requested ON alias.owner_json = json_extract(requested.value, '$.scope_json') diff --git a/crates/tracedecay-global-db/src/observation_batch_tests.rs b/crates/tracedecay-global-db/src/observation_batch_tests.rs index 5898e9ff89..3a6aae3899 100644 --- a/crates/tracedecay-global-db/src/observation_batch_tests.rs +++ b/crates/tracedecay-global-db/src/observation_batch_tests.rs @@ -9,7 +9,7 @@ use std::sync::atomic::{AtomicU64, Ordering}; use tempfile::TempDir; use tracedecay_domain::{ CanonicalMessageRoleV1, CanonicalObservationEnvelopeV1, CanonicalObservationEvidenceV1, - CanonicalObservationFactV1, CanonicalObservationRelationsV1, DurableObservationV1, + CanonicalObservationFactV1, CanonicalObservationRelationsV1, DurableObservationV1, FactOwnerV1, ObservationCollisionOutcomeV1, ObservationId, ObservationIdentityMaterialV1, ObservationOrderingDomainV1, ObservationScopeV1, ObservationSourceCursorV1, ObservationSourceGenerationV1, ObservationSourceIdentityV1, ObservationSourceRangeV1, @@ -18,8 +18,10 @@ use tracedecay_domain::{ SanitizationReceiptV1, SanitizerDispositionV1, SensitivityV1, SessionId, UtcMicros, }; use tracedecay_store::{ - AnchoredObservationWrite, FOREGROUND_BATCH_MAX_OPERATIONS, ObservationBatchPersistOutcome, - ObservationPersistOutcome, ObservationStore, ObservationStoreError, ObservationWrite, + AnchorDispositionReasonClassV1, AnchorDispositionStateV1, AnchoredObservationWrite, + FOREGROUND_BATCH_MAX_OPERATIONS, ObservationBatchPersistOutcome, ObservationPersistOutcome, + ObservationStore, ObservationStoreError, ObservationWrite, RetrievalAnchorDispositionRecordV1, + RetrievalAnchorDispositionStore, }; use tracing::field::{Field, Visit}; use tracing::span::{Attributes, Id, Record}; @@ -837,6 +839,101 @@ async fn non_codex_alias_collision_stays_typed_over_a_corrupt_alias_owner() { ); } +#[asupersync::test] +async fn codex_sibling_over_a_retention_released_owner_is_a_typed_alias_collision() { + const DAY_MICROS: i64 = 24 * 60 * 60 * 1_000_000; + + let (_tmp, runtime, store) = open_profile_store().await; + let thread = THREAD_ROLLOUT.write(); + assert!(matches!( + store.persist_observation(thread.clone()).await.unwrap(), + ObservationPersistOutcome::Committed(_) + )); + + let database = runtime + .registered_database(HostAdmissionScope::Profile) + .unwrap(); + let now = tracedecay_contracts::clock::now_micros(); + let anchor = thread.retrieval_anchor(); + let record = RetrievalAnchorDispositionRecordV1::new( + "observation-batch:retention-released-owner", + anchor.anchor_id().clone(), + FactOwnerV1::from(anchor.owner().clone()), + AnchorDispositionStateV1::Deleted, + None, + AnchorDispositionReasonClassV1::Retention, + UtcMicros(now.0 - 40 * DAY_MICROS), + ) + .unwrap(); + database + .runtime_database() + .append_disposition(record) + .await + .unwrap(); + let report = database + .run_observation_retention( + None, + &crate::observation::retention::ObservationRetentionConfig::default(), + crate::observation::retention::RetentionMode::Apply, + now, + ) + .await + .unwrap(); + assert_eq!( + ( + report.anchors_released.acted, + report.observations_released.acted + ), + (1, 1) + ); + + let unrelated = sequential_writes( + &SessionId::new("session.observation-batch.unrelated").unwrap(), + 1, + ) + .into_iter() + .next() + .unwrap(); + let revert = REVERT_ROLLOUT.write(); + let batch = store + .persist_observations(vec![unrelated.clone(), revert.clone()]) + .await; + + let error = batch.unwrap_err(); + assert!( + matches!( + &error, + ObservationStoreError::RetrievalAnchorAliasCollision { + existing_anchor_id, + .. + } if **existing_anchor_id == *thread.retrieval_anchor_id() + ), + "expected the released owner's alias collision, got {error:?}" + ); + assert!( + store + .get_observation(unrelated.observation().observation_id()) + .await + .unwrap() + .is_none() + ); + assert!( + store + .get_observation(revert.observation().observation_id()) + .await + .unwrap() + .is_none() + ); + assert!(matches!( + store.persist_observation(revert.clone()).await, + Err(ObservationStoreError::RetrievalAnchorAliasCollision { .. }) + )); + assert!(matches!( + store.persist_observation(unrelated.clone()).await, + Ok(ObservationPersistOutcome::Committed(_)) + )); +} + #[asupersync::test] async fn persist_observations_dispatches_one_runtime_command_independent_of_batch_size() { let mut expected_snapshot_probes = None; diff --git a/crates/tracedecay-session-runtime/src/session_temporal_refresh_scheduler/registry_tests.rs b/crates/tracedecay-session-runtime/src/session_temporal_refresh_scheduler/registry_tests.rs index ac150a4d45..d52c9c9eb7 100644 --- a/crates/tracedecay-session-runtime/src/session_temporal_refresh_scheduler/registry_tests.rs +++ b/crates/tracedecay-session-runtime/src/session_temporal_refresh_scheduler/registry_tests.rs @@ -4,17 +4,23 @@ use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; use std::time::Duration; use futures_util::FutureExt; +use tracedecay_domain::SessionId; use tracedecay_global_db::tests::harness::RegisteredGlobalDbHarness; use tracedecay_runtime_core::native_runtime; +use tracedecay_session_temporal_store::SessionTemporalStore; use tracedecay_sessions::serving::{ SessionProjectionServingState, SessionProjectionServingStatusPort, SessionProjectionStaleReason, }; +use tracedecay_store::{ + SessionRefreshBeginOrJoinRequestV1, SessionRefreshDispositionV1, SessionRefreshFrontierV1, + SessionRefreshReceiptRequestV1, SessionRefreshStore, SessionRefreshTerminalStateV1, +}; use super::super::history::{ SessionHistoricalCapacityRelease, SessionHistoricalIngestOutcome, SessionHistoricalIngestPass, SessionHistoricalIngestor, SharedSessionHistoricalIngestor, }; -use super::super::wake::SessionTemporalRefreshWakeState; +use super::super::wake::{SessionTemporalRefreshWake, SessionTemporalRefreshWakeState}; use super::SessionTemporalRefreshSchedulerRegistry; use crate::StoreOwnerKey; @@ -137,6 +143,46 @@ async fn mount_held_project( history } +/// A replacement worker may consume a handed-off request before rekey returns. +/// Join its real pass and require the durable completion, rather than racing +/// the consumer for a transient queue length or dirty flag. These empty sessions +/// have no observation history for discovery to reconstruct a lost request from. +async fn assert_rekey_completed_requests( + harness: &RegisteredGlobalDbHarness, + wake: &SessionTemporalRefreshWake, + sessions: &[&str], +) { + assert!( + wake.wake_and_wait_until_idle(Duration::from_secs(5)).await, + "the retained route must join the replacement worker's pass" + ); + let store = SessionTemporalStore::new(harness.registered.as_ref()); + for name in sessions { + let session_id = SessionId::new(*name).expect("session id"); + let joined = store + .begin_or_join_session_refresh(SessionRefreshBeginOrJoinRequestV1::new( + session_id.clone(), + SessionRefreshFrontierV1::new(0, 0).expect("empty frontier"), + )) + .await + .expect("join the handed-off refresh"); + assert_eq!( + joined.disposition(), + SessionRefreshDispositionV1::Joined, + "{name}: the replacement must have begun the handed-off request" + ); + let receipt = store + .session_refresh_receipt(SessionRefreshReceiptRequestV1::new( + joined.operation_id().clone(), + session_id, + )) + .await + .expect("read handed-off refresh receipt") + .expect("the replacement must complete the handed-off request"); + assert_eq!(receipt.state(), SessionRefreshTerminalStateV1::Complete); + } +} + /// Mounts a project whose worker holds its history pass and a profile entry. async fn mount_drain_fixture( harness: &RegisteredGlobalDbHarness, @@ -410,20 +456,11 @@ async fn a_retried_rekey_recovers_the_dropped_attempts_route_and_requests() { rekey.await; assert!(registry.project_state(&old_owner).await.is_none()); - let new_state = registry + registry .project_state(&new_owner) .await .expect("new owner entry"); - assert_eq!( - new_state.queued_request_count(), - 1, - "the retry must land the retired owner's queued request on the new owner" - ); - assert!( - old_wake.wake(), - "the old route must reach the new owner once the rekey completes" - ); - assert!(new_state.take_dirty()); + assert_rekey_completed_requests(&harness, &old_wake, &["session.dropped-rekey"]).await; registry.shutdown().await; } @@ -787,20 +824,11 @@ async fn a_mapped_rekey_lands_the_workers_queued_requests() { rekey.await; assert!(registry.project_state(&old_owner).await.is_none()); - let new_state = registry + registry .project_state(&new_owner) .await .expect("new owner entry"); - assert_eq!( - new_state.queued_request_count(), - 1, - "the new owner must inherit the mapped worker's queued request" - ); - assert!( - old_wake.wake(), - "the old route must reach the new owner once the rekey completes" - ); - assert!(new_state.take_dirty()); + assert_rekey_completed_requests(&harness, &old_wake, &["session.mapped-rekey"]).await; registry.shutdown().await; } diff --git a/crates/tracedecay/tests/dashboard_api_test/explorer.rs b/crates/tracedecay/tests/dashboard_api_test/explorer.rs index 28d218f51a..a6b363ce0f 100644 --- a/crates/tracedecay/tests/dashboard_api_test/explorer.rs +++ b/crates/tracedecay/tests/dashboard_api_test/explorer.rs @@ -1,21 +1,27 @@ -use std::time::Duration; +use std::time::{Duration, Instant}; use crate::dashboard_api_support::*; use serde_json::{Value, json}; fn wait_for_query(agent: &ureq::Agent, base_url: &str, run_id: &str) -> Value { + let started = Instant::now(); + let mut last_response = None; for _ in 0..100 { let (status, body) = get_json(agent, &format!("{base_url}/api/explorer/queries/{run_id}")); assert_eq!(status, 200, "query status should resolve: {body}"); if matches!( body["payload"]["state"].as_str(), - Some("completed" | "partial" | "cancelled" | "error") + Some("completed" | "partial" | "cancelled" | "timed_out" | "error") ) { return body; } + last_response = Some(body); std::thread::sleep(Duration::from_millis(10)); } - panic!("explorer query {run_id} did not reach a terminal state"); + panic!( + "explorer query {run_id} did not reach a terminal state after {:?}; last response: {last_response:?}", + started.elapsed() + ); } #[test] diff --git a/crates/tracedecay/tests/storage_suite/multi_connection_test/ownership.rs b/crates/tracedecay/tests/storage_suite/multi_connection_test/ownership.rs index 08de819f7a..1ac4f4fbea 100644 --- a/crates/tracedecay/tests/storage_suite/multi_connection_test/ownership.rs +++ b/crates/tracedecay/tests/storage_suite/multi_connection_test/ownership.rs @@ -207,19 +207,40 @@ fn twelve_mcp_cli_and_hook_clients_share_one_daemon_profile_store_owner() { let project_id = tracedecay_runtime_core::storage::default_profile_project_id(&project_path); let data_root = tracedecay_runtime_core::storage::profile_sharded_data_root(&profile_root, &project_id); - let host = tracedecay_domain::NativeHostIdentityV1::CursorDesktop; - let (_, capture_report) = tracedecay_hooks::HookSpoolV1::open( - data_root.join("hook-v2-spool").join(host.hook_key()), - tracedecay_hooks::HookSpoolConfigV1::stock(host), - tracedecay_contracts::now_micros(), - ) - .expect("open native capture receipt"); - // Sequence survives daemon replay/acknowledgement, so this proves each - // callback was captured even if the owner already drained its envelope. + // A commit-lock timeout can stage an envelope whose first spool frame + // the daemon already acknowledged. Importing it appends a duplicate + // frame, so the sequence counts attempts rather than distinct callbacks. + // Each callback's completed receipt instead proves that its own capture + // landed, including one safely staged for replay, rather than returning + // the same transport response for an unbound hook. + let analytics = std::fs::read_to_string(data_root.join("hook_analytics.jsonl")) + .expect("read native capture outcomes"); + let mut captured_sessions = analytics + .lines() + .map(|line| serde_json::from_str::(line).expect("hook analytics row")) + .filter(|row| { + row["event"] == "hook_completed" + && row["agent"] == "cursor" + && row["hook_name"] == "afterFileEdit" + }) + .map(|row| { + assert_eq!( + row["disposition"]["status"], "accepted_for_replay", + "native callback must be durably captured: {row}" + ); + row["session_id"] + .as_str() + .expect("capture names the callback session") + .to_owned() + }) + .collect::>(); + captured_sessions.sort(); assert_eq!( - capture_report.next_sequence, - 1 + CONCURRENT_CLIENTS_PER_PATH as u64, - "every native callback must be captured, not merely accepted as unbound" + captured_sessions, + (0..CONCURRENT_CLIENTS_PER_PATH) + .map(|ordinal| format!("session-{ordinal}")) + .collect::>(), + "every native callback must have exactly one successful capture outcome" ); let daemon_stderr = std::fs::read_to_string(&daemon_stderr_path).expect("read daemon stderr"); assert!(