Skip to content

A self-improving RLM agent for coding workflows and long-running autonomous tasks.

Rust

persist child usage attribution rows to the parent transcript (#3386)

Sebastian Müllerd592d5e77464
diff --git a/.changes/child-usage-attribution-rows.md b/.changes/child-usage-attribution-rows.md
new file mode 100644
index 0000000000..1ae5f3f82e
--- /dev/null
+++ b/.changes/child-usage-attribution-rows.md
@@ -0,0 +1,2 @@
+- A parent session's usage and cost totals include its subagents' usage again: child usage attribution rows are written to the parent's transcript file, so live totals count child cost and a resumed parent keeps it, matching the TypeScript version.
+- Attribution retries are idempotent: each row keeps the stable identity of its frozen report, and a re-delivered report appends nothing once that identity is already in the transcript, so a partial save or a lost cursor cannot double-count child usage.
diff --git a/crates/pa-cli/src/print_boundary/tests.rs b/crates/pa-cli/src/print_boundary/tests.rs
index 5f38e602bc..0560be0b70 100644
--- a/crates/pa-cli/src/print_boundary/tests.rs
+++ b/crates/pa-cli/src/print_boundary/tests.rs
@@ -129,6 +129,7 @@ async fn faux_engine_with_telemetry(
additional_prompt_paths: Vec::new(),
extra_builtin_skill_overrides: Vec::new(),
rlm_subagent_host: None,
+ rlm_usage_store: None,
progress_note_emit: None,
rlm_depth: None,
daemonless: true,
diff --git a/crates/pa-cli/src/print_goal.rs b/crates/pa-cli/src/print_goal.rs
index 51f04589dd..0bf9b12cbf 100644
--- a/crates/pa-cli/src/print_goal.rs
+++ b/crates/pa-cli/src/print_goal.rs
@@ -853,6 +853,7 @@ mod tests {
additional_prompt_paths: Vec::new(),
extra_builtin_skill_overrides: Vec::new(),
rlm_subagent_host: None,
+ rlm_usage_store: None,
progress_note_emit: None,
daemonless: true,
rlm_depth: None,
@@ -1468,6 +1469,7 @@ mod tests {
additional_prompt_paths: Vec::new(),
extra_builtin_skill_overrides: Vec::new(),
rlm_subagent_host: None,
+ rlm_usage_store: None,
progress_note_emit: None,
daemonless: true,
rlm_depth: None,
diff --git a/crates/pa-cli/src/print_runtime.rs b/crates/pa-cli/src/print_runtime.rs
index 7393f33074..4640cd3fb1 100644
--- a/crates/pa-cli/src/print_runtime.rs
+++ b/crates/pa-cli/src/print_runtime.rs
@@ -755,6 +755,7 @@ pub(crate) async fn build_headless_engine_with(
.collect(),
extra_builtin_skill_overrides: vec![],
rlm_subagent_host,
+ rlm_usage_store: None,
progress_note_emit: None,
rlm_depth,
daemonless: true,
diff --git a/crates/pa-cli/src/print_runtime/faux.rs b/crates/pa-cli/src/print_runtime/faux.rs
index 07e3cb3d20..b65659ddb5 100644
--- a/crates/pa-cli/src/print_runtime/faux.rs
+++ b/crates/pa-cli/src/print_runtime/faux.rs
@@ -162,6 +162,7 @@ pub(crate) async fn build_faux_engine_with(
additional_prompt_paths: vec![],
extra_builtin_skill_overrides: vec![],
rlm_subagent_host,
+ rlm_usage_store: None,
progress_note_emit: None,
rlm_depth,
daemonless: true,
diff --git a/crates/pa-cli/src/print_session_command.rs b/crates/pa-cli/src/print_session_command.rs
index 29b076fa04..a71ab1a2e5 100644
--- a/crates/pa-cli/src/print_session_command.rs
+++ b/crates/pa-cli/src/print_session_command.rs
@@ -345,6 +345,7 @@ mod tests {
additional_prompt_paths: Vec::new(),
extra_builtin_skill_overrides: Vec::new(),
rlm_subagent_host: None,
+ rlm_usage_store: None,
progress_note_emit: None,
daemonless: true,
rlm_depth: None,
diff --git a/crates/pa-core/src/session/manager/append.rs b/crates/pa-core/src/session/manager/append.rs
index 901f9d8c37..ea19fa6a88 100644
--- a/crates/pa-core/src/session/manager/append.rs
+++ b/crates/pa-core/src/session/manager/append.rs
@@ -274,7 +274,8 @@ impl SessionManager {
(id, write_error)
}
- /// Fold child usage into the target assistant and record the attribution.
+ /// Fold child usage into the target assistant and record the attribution
+ /// under the caller's row id (the idempotent retry's stable identity).
///
/// # Errors
///
@@ -282,6 +283,7 @@ impl SessionManager {
/// durable append's I/O error.
pub fn append_child_usage_attribution(
&mut self,
+ row_id: &str,
target_id: &str,
child_usage: pa_types::ai::Usage,
aggregate_usage: pa_types::ai::Usage,
@@ -304,8 +306,9 @@ impl SessionManager {
format!("Assistant message entry {target_id} not found"),
)
})?;
- let base = self.next_base();
- let id = base.id.clone().unwrap_or_default();
+ let mut base = self.next_base();
+ base.id = Some(row_id.to_owned());
+ let id = row_id.to_owned();
self.append_entry(FileEntry::ChildUsageAttributed {
payload: pa_types::session::ChildUsageAttributionEntry {
target_id: target_id.to_string(),
diff --git a/crates/pa-core/src/session/window_tests.rs b/crates/pa-core/src/session/window_tests.rs
index 93e334e76a..06a61497b4 100644
--- a/crates/pa-core/src/session/window_tests.rs
+++ b/crates/pa-core/src/session/window_tests.rs
@@ -805,6 +805,7 @@ async fn adopted_manager_live_appends_match_full_reopen() {
// the manager folds its own copy (the detached window has none).
manager
.append_child_usage_attribution(
+ "attr-1",
"a1",
pa_types::ai::Usage {
input: 5,
diff --git a/crates/pa-core/src/session_engine/engine.rs b/crates/pa-core/src/session_engine/engine.rs
index 790bb819f7..8f2411ca86 100644
--- a/crates/pa-core/src/session_engine/engine.rs
+++ b/crates/pa-core/src/session_engine/engine.rs
@@ -46,6 +46,7 @@ pub struct SessionEngineConfig {
pub additional_prompt_paths: Vec<String>,
pub extra_builtin_skill_overrides: Vec<String>,
pub rlm_subagent_host: Option<Arc<dyn super::rlm_host::RlmSubagentHost>>,
+ pub rlm_usage_store: Option<Arc<dyn super::rlm_usage::RlmChildUsageStore>>,
/// Announces every accepted `rlm.progress.note`; `None` keeps the store
/// the only surface.
pub progress_note_emit: Option<super::rlm_host::RlmProgressNoteEmit>,
@@ -224,6 +225,7 @@ pub async fn create_session(mut config: SessionEngineConfig) -> anyhow::Result<S
super::runtime_wiring::RlmWiring {
model_registry: None,
subagent_host: config.rlm_subagent_host.clone(),
+ usage_store: config.rlm_usage_store.clone(),
progress_note_emit: config.progress_note_emit.clone(),
},
config.queued_goal_context_purge.clone(),
diff --git a/crates/pa-core/src/session_engine/engine/tests.rs b/crates/pa-core/src/session_engine/engine/tests.rs
index bb1d6578db..a3dbf37e7c 100644
--- a/crates/pa-core/src/session_engine/engine/tests.rs
+++ b/crates/pa-core/src/session_engine/engine/tests.rs
@@ -80,6 +80,7 @@ async fn engine_runs_tool_loop_and_persists() {
additional_prompt_paths: vec![],
extra_builtin_skill_overrides: vec![],
rlm_subagent_host: None,
+ rlm_usage_store: None,
progress_note_emit: None,
rlm_depth: None,
daemonless: true,
@@ -181,6 +182,7 @@ async fn spawned_child_prompt_stamps_its_depth() {
additional_prompt_paths: vec![],
extra_builtin_skill_overrides: vec![],
rlm_subagent_host: None,
+ rlm_usage_store: None,
progress_note_emit: None,
rlm_depth: Some(2),
daemonless: true,
@@ -250,6 +252,7 @@ async fn oauth_creds_unlock_generic_mcp_gating_in_new_sessions() {
additional_prompt_paths: vec![],
extra_builtin_skill_overrides: vec![],
rlm_subagent_host: None,
+ rlm_usage_store: None,
progress_note_emit: None,
rlm_depth: None,
daemonless: true,
@@ -436,6 +439,7 @@ async fn create_session_appends_missing_resume_settings_rows() {
additional_prompt_paths: vec![],
extra_builtin_skill_overrides: vec![],
rlm_subagent_host: None,
+ rlm_usage_store: None,
progress_note_emit: None,
rlm_depth: None,
daemonless: true,
diff --git a/crates/pa-core/src/session_engine/rlm_host.rs b/crates/pa-core/src/session_engine/rlm_host.rs
index 16a2f32de8..c573737d8a 100644
--- a/crates/pa-core/src/session_engine/rlm_host.rs
+++ b/crates/pa-core/src/session_engine/rlm_host.rs
@@ -1086,6 +1086,7 @@ mod tests {
crate::session_engine::runtime_wiring::RlmWiring {
model_registry: Some(Arc::clone(&registry)),
subagent_host: host,
+ usage_store: None,
progress_note_emit: None,
},
None,
@@ -1261,6 +1262,7 @@ mod tests {
crate::session_engine::runtime_wiring::RlmWiring {
model_registry: None,
subagent_host: None,
+ usage_store: None,
progress_note_emit: Some(Arc::new(move |message, timestamp| {
sink.lock().unwrap().push((message.to_string(), timestamp));
})),
diff --git a/crates/pa-core/src/session_engine/rlm_usage.rs b/crates/pa-core/src/session_engine/rlm_usage.rs
index 3dce65e1f1..9679dfb190 100644
--- a/crates/pa-core/src/session_engine/rlm_usage.rs
+++ b/crates/pa-core/src/session_engine/rlm_usage.rs
@@ -31,15 +31,157 @@ fn add_cost(total: pa_types::JsNumber, usage: pa_types::JsNumber) -> pa_types::J
/// Child work affects session-level billable totals, not the parent's
/// model-facing context size, so the context tokens are restored after the fold.
-pub(crate) fn attribute_child_usage(parent_usage: &mut Usage, child_usage: &Usage) {
+pub fn attribute_child_usage(parent_usage: &mut Usage, child_usage: &Usage) {
let parent_context_tokens = super::compaction::calculate_context_tokens(parent_usage);
add_assistant_usage(parent_usage, child_usage);
parent_usage.total_tokens = parent_context_tokens;
}
+/// Compare the immutable attribution identity using the transcript's JSON number decoding.
+/// The aggregate is deliberately excluded: later children can advance it.
+///
+/// # Errors
+///
+/// Returns a serialization error if the supplied usage cannot use the wire representation.
+pub fn child_usage_identity_matches(
+ payload: &pa_types::session::ChildUsageAttributionEntry,
+ target_id: &str,
+ child_usage: Usage,
+ origin: Option<ChildUsageOrigin>,
+) -> std::io::Result<bool> {
+ let wire_usage: Usage = serde_json::from_str(&serde_json::to_string(&child_usage)?)?;
+ Ok(payload.target_id == target_id
+ && (payload.child_usage == child_usage || payload.child_usage == wire_usage)
+ && payload.origin == origin)
+}
+
+pub type RlmChildUsageFuture<'a, T> =
+ std::pin::Pin<std::boxed::Box<dyn std::future::Future<Output = T> + Send + 'a>>;
+
+/// A confirmed store result, carrying the current authoritative aggregate.
+#[derive(Debug, Clone, Copy)]
+pub enum ChildUsageAppendResult {
+ /// First confirmed durable result for this owned attribution intent.
+ Created(Usage),
+ /// This ID was already confirmed; later children may have advanced the aggregate.
+ Existing(Usage),
+}
+
+/// The attribution producer's durable transcript boundary. Implementations
+/// serialize mutation under the session owner, compute the aggregate from
+/// their own authoritative state, and return [`ChildUsageAppendResult::Existing`]
+/// for a row id already in the store (the identity must match), appending one
+/// row that carries the caller's id otherwise. Callers must preserve row ID,
+/// target, child usage and origin across retries.
+pub trait RlmChildUsageStore: Send + Sync {
+ fn last_assistant(&self) -> RlmChildUsageFuture<'_, Option<(String, Usage)>>;
+
+ fn append_attribution(
+ &self,
+ row_id: &str,
+ target_id: &str,
+ child_usage: Usage,
+ origin: Option<ChildUsageOrigin>,
+ ) -> RlmChildUsageFuture<'_, std::io::Result<ChildUsageAppendResult>>;
+}
+
+pub(crate) struct SessionUsageStore(pub(crate) std::sync::Arc<tokio::sync::Mutex<SessionManager>>);
+
+impl RlmChildUsageStore for SessionUsageStore {
+ fn last_assistant(&self) -> RlmChildUsageFuture<'_, Option<(String, Usage)>> {
+ let session = std::sync::Arc::clone(&self.0);
+ Box::pin(async move {
+ let session = session.lock().await;
+ session
+ .retained_entries()
+ .iter()
+ .rev()
+ .find_map(last_assistant_row)
+ })
+ }
+
+ fn append_attribution(
+ &self,
+ row_id: &str,
+ target_id: &str,
+ child_usage: Usage,
+ origin: Option<ChildUsageOrigin>,
+ ) -> RlmChildUsageFuture<'_, std::io::Result<ChildUsageAppendResult>> {
+ let session = std::sync::Arc::clone(&self.0);
+ let target_id = target_id.to_string();
+ let row_id = row_id.to_string();
+ Box::pin(async move {
+ let mut session = session.lock().await;
+ // The thin idempotency check: a row this id already wrote is
+ // confirmed, whatever the caller's last attempt returned. The
+ // retained scan is the same view `last_assistant` reads, so a
+ // windowed manager answers instead of asserting full history.
+ let existing = session
+ .retained_entries()
+ .iter()
+ .rev()
+ .find(|entry| entry.id() == Some(row_id.as_str()));
+ if let Some(entry) = existing {
+ let pa_types::session::FileEntry::ChildUsageAttributed { payload, .. } = entry
+ else {
+ return Err(attribution_id_collision());
+ };
+ if !child_usage_identity_matches(payload, &target_id, child_usage, origin)? {
+ return Err(attribution_id_collision());
+ }
+ return Ok(ChildUsageAppendResult::Existing(target_assistant_usage(
+ &session, &target_id,
+ )?));
+ }
+ let mut aggregate = target_assistant_usage(&session, &target_id)?;
+ attribute_child_usage(&mut aggregate, &child_usage);
+ session.append_child_usage_attribution(
+ &row_id,
+ &target_id,
+ child_usage,
+ aggregate,
+ origin,
+ )?;
+ Ok(ChildUsageAppendResult::Created(aggregate))
+ })
+ }
+}
+
+/// The duplicate retry of an id that landed under another identity is a
+/// transcript bug, not a retry: refuse it instead of folding twice.
+fn attribution_id_collision() -> std::io::Error {
+ std::io::Error::new(std::io::ErrorKind::InvalidData, "attribution ID collision")
+}
+
+/// The target assistant row's current usage: the running aggregate, since
+/// every attribution append folds its aggregate back into the row. The
+/// retained scan answers on a windowed manager too, where the by-id
+/// lookup's full-history assertion would panic.
+fn target_assistant_usage(session: &SessionManager, target_id: &str) -> std::io::Result<Usage> {
+ session
+ .retained_entries()
+ .iter()
+ .rev()
+ .find_map(|entry| match entry {
+ pa_types::session::FileEntry::Message {
+ message: pa_types::session::AgentMessage::Assistant(assistant),
+ ..
+ } if entry.id() == Some(target_id) => Some(assistant.usage),
+ _ => None,
+ })
+ .ok_or_else(|| {
+ std::io::Error::new(
+ std::io::ErrorKind::InvalidInput,
+ format!("Assistant message entry {target_id} not found"),
+ )
+ })
+}
+
/// Per-origin batches in first-seen order.
#[derive(Debug, Clone)]
pub struct RlmChildUsageReport {
+ /// Frozen observation identity; clones and retries must preserve this ID.
+ pub report_id: String,
pub rlm_child_id: String,
pub batches: Vec<(ChildUsageOrigin, Usage)>,
}
@@ -47,9 +189,9 @@ pub struct RlmChildUsageReport {
/// The producer the daemon's child observation feeds: spawn registration
/// plus the durable flush. One instance per session engine.
pub struct RlmChildUsageAttributions {
- session: std::sync::Arc<tokio::sync::Mutex<SessionManager>>,
- /// The aggregate base per parent assistant row, shared by all its children;
- /// held across the durable append so batches serialize in observation order.
+ store: std::sync::Arc<dyn RlmChildUsageStore>,
+ /// Last confirmed aggregate per parent row; held to serialize report handoffs.
+ /// The store owns the authoritative aggregate calculation, including retries.
bases: tokio::sync::Mutex<HashMap<String, Usage>>,
/// The parent assistant row each child attributes to, captured at spawn;
/// dropped in [`Self::forget_child`].
@@ -66,9 +208,9 @@ pub struct RlmChildUsageAttributions {
}
impl RlmChildUsageAttributions {
- pub fn new(session: std::sync::Arc<tokio::sync::Mutex<SessionManager>>) -> Self {
+ pub fn new(store: std::sync::Arc<dyn RlmChildUsageStore>) -> Self {
Self {
- session,
+ store,
bases: tokio::sync::Mutex::new(HashMap::new()),
children: std::sync::Mutex::new(HashMap::new()),
telemetry: std::sync::Mutex::new(None),
@@ -102,14 +244,7 @@ impl RlmChildUsageAttributions {
Box::pin(async move { forward.register_spawn(rlm_child_id).await }).await;
return;
}
- let target = {
- let session = self.session.lock().await;
- session
- .retained_entries()
- .iter()
- .rev()
- .find_map(last_assistant_row)
- };
+ let target = self.store.last_assistant().await;
if let Some((target_id, usage)) = target {
// Seed the base BEFORE the registration publishes: a report finding the entry
// must not compute from the default (or_insert preserves a broken first aggregate).
@@ -124,17 +259,18 @@ impl RlmChildUsageAttributions {
}
/// Flush one observed report: batches fold into the target row's cumulative aggregate,
- /// one durable `child_usage_attributed` row per batch; a failed append is logged and
- /// dropped, so attribution bookkeeping never breaks the observing path.
+ /// one durable `child_usage_attributed` row per batch. Returns whether every batch
+ /// persisted (nothing owed — an unregistered or forgotten child has no durable target,
+ /// and retrying cannot help — counts as persisted); a failed append is logged and stops
+ /// the flush, leaving the rest for the next observation's retry.
///
/// # Panics
///
/// Panics if the forward, children, or telemetry state mutexes are poisoned.
- pub async fn record_child_usage(&self, report: RlmChildUsageReport) {
+ pub async fn record_child_usage(&self, report: RlmChildUsageReport) -> bool {
let forward = self.forward.lock().expect("rlm usage forward lock").clone();
if let Some(forward) = forward {
- Box::pin(async move { forward.record_child_usage(report).await }).await;
- return;
+ return Box::pin(async move { forward.record_child_usage(report).await }).await;
}
// The guard drops at its own statement: a scrutinee temp held
// across the fallback await is not Send.
@@ -151,46 +287,41 @@ impl RlmChildUsageAttributions {
None => {
// Never registered here (raced a rebuild, or the
// child outlived its engine): no durable target.
- return;
+ return true;
}
},
};
- // A report racing the handoff can find the registration copied before the bases:
- // read the retired side's base BEFORE our bases lock.
- let fallback_base = self.fallback_base(&target_id).await;
let mut bases = self.bases.lock().await;
// The forward re-check runs WITH the bases lock held: a handoff that armed
// mid-report blocks its bases copy on this lock, so the adoption carries it.
let forward = self.forward.lock().expect("rlm usage forward lock").clone();
if let Some(forward) = forward {
- // Release the bases BEFORE forwarding: the successor's fallback_base re-locks
- // THIS producer's bases (a tokio Mutex is not reentrant); holding it across
- // the await would deadlock the handoff path and the adoption.
+ // Release the bases BEFORE forwarding: the successor's fallback
+ // consult re-locks THIS producer's bases (a tokio Mutex is not
+ // reentrant); holding it across the await would deadlock the
+ // handoff path and the adoption.
drop(bases);
- Box::pin(async move { forward.record_child_usage(report).await }).await;
- return;
+ return Box::pin(async move { forward.record_child_usage(report).await }).await;
}
- for (origin, usage) in report.batches {
- let base = bases
- .get(&target_id)
- .copied()
- .or(fallback_base)
- .unwrap_or_default();
- let mut aggregate = base;
- attribute_child_usage(&mut aggregate, &usage);
- match self.session.lock().await.append_child_usage_attribution(
- &target_id,
- usage,
- aggregate,
- Some(origin),
- ) {
- Ok(_) => {
+ for (batch_index, (origin, usage)) in report.batches.into_iter().enumerate() {
+ let row_id = format!("{}-{batch_index}", report.report_id);
+ match self
+ .store
+ .append_attribution(&row_id, &target_id, usage, Some(origin))
+ .await
+ {
+ Ok(result) => {
+ let (aggregate, created) = match result {
+ ChildUsageAppendResult::Created(aggregate) => (aggregate, true),
+ ChildUsageAppendResult::Existing(aggregate) => (aggregate, false),
+ };
bases.insert(target_id.clone(), aggregate);
if let Some(telemetry) = self
.telemetry
.lock()
.expect("rlm usage telemetry lock")
.as_ref()
+ .filter(|_| created)
{
telemetry.note_child_usage_attributed(
usage.input,
@@ -203,9 +334,11 @@ impl RlmChildUsageAttributions {
}
Err(error) => {
eprintln!("pa-core: RLM child usage attribution not persisted: {error}");
+ return false;
}
}
}
+ true
}
/// A rebuild keeps the session's live children: adopt the retired
@@ -289,18 +422,6 @@ impl RlmChildUsageAttributions {
Some(target_id)
}
- /// The retired side's frozen cumulative base: read BEFORE our own
- /// bases lock (lock order: the fallback's bases first, ours second).
- async fn fallback_base(&self, target_id: &str) -> Option<Usage> {
- for producer in self.fallback_chain() {
- let retired_bases = producer.bases.lock().await;
- if let Some(base) = retired_bases.get(target_id) {
- return Some(*base);
- }
- }
- None
- }
-
/// Drop one child's registration once its final observation lands; the
/// aggregate base stays. The retired side's copy is pruned too, so a
/// straggler cannot resurrect it through the fallback.
@@ -349,7 +470,7 @@ pub trait RlmChildUsageSink: Send + Sync {
fn record(
&self,
report: RlmChildUsageReport,
- ) -> std::pin::Pin<std::boxed::Box<dyn std::future::Future<Output = ()> + Send + '_>>;
+ ) -> std::pin::Pin<std::boxed::Box<dyn std::future::Future<Output = bool> + Send + '_>>;
/// Drop the registration so sequential children do not accumulate.
fn forget(
@@ -441,6 +562,12 @@ mod tests {
.unwrap_or_else(|| row[key]["cost"]["total"].as_i64().expect("cost number") as f64)
}
+ fn producer(
+ manager: std::sync::Arc<tokio::sync::Mutex<SessionManager>>,
+ ) -> RlmChildUsageAttributions {
+ RlmChildUsageAttributions::new(std::sync::Arc::new(SessionUsageStore(manager)))
+ }
+
/// Captured TS fixture (assistant row 4f61089a, archive 01a0a7d4-cdd9):
/// the folded aggregate carries input 52,898, parts summing to 77,321,
/// and totalTokens FROZEN at the parent's 23,032; the raw parent's
@@ -450,10 +577,11 @@ mod tests {
let raw_parent = usage_block(2_690, 1_577, 19_917, 0, 23_032, 0.0);
let child = usage_block(50_208, 2_929, 0, 0, 53_137, 0.008_995_7);
let (_tmp, manager) = manager_with_assistant(raw_parent);
- let producer = RlmChildUsageAttributions::new(manager.clone());
+ let producer = producer(manager.clone());
producer.register_spawn("sub-abc12345").await;
producer
.record_child_usage(RlmChildUsageReport {
+ report_id: uuid::Uuid::new_v4().to_string(),
rlm_child_id: "sub-abc12345".to_string(),
batches: vec![(ChildUsageOrigin::SpawnTask, child)],
})
@@ -500,11 +628,12 @@ mod tests {
async fn rebuild_adoption_continues_the_aggregate_chain() {
let raw_parent = usage_block(1_000, 0, 0, 0, 4_096, 0.0);
let (_tmp, manager) = manager_with_assistant(raw_parent);
- let retired = std::sync::Arc::new(RlmChildUsageAttributions::new(manager.clone()));
- let fresh = std::sync::Arc::new(RlmChildUsageAttributions::new(manager.clone()));
+ let retired = std::sync::Arc::new(producer(manager.clone()));
+ let fresh = std::sync::Arc::new(producer(manager.clone()));
retired.register_spawn("sub-rebuild1").await;
retired
.record_child_usage(RlmChildUsageReport {
+ report_id: uuid::Uuid::new_v4().to_string(),
rlm_child_id: "sub-rebuild1".to_string(),
batches: vec![(
ChildUsageOrigin::SpawnTask,
@@ -515,6 +644,7 @@ mod tests {
fresh.adopt_registrations(&retired).await;
fresh
.record_child_usage(RlmChildUsageReport {
+ report_id: uuid::Uuid::new_v4().to_string(),
rlm_child_id: "sub-rebuild1".to_string(),
batches: vec![(
ChildUsageOrigin::AgentMessage,
@@ -534,6 +664,7 @@ mod tests {
// The retired producer now FORWARDS: a late emission still lands on the adopted chain.
retired
.record_child_usage(RlmChildUsageReport {
+ report_id: uuid::Uuid::new_v4().to_string(),
rlm_child_id: "sub-rebuild1".to_string(),
batches: vec![(
ChildUsageOrigin::DirectUser,
@@ -551,6 +682,7 @@ mod tests {
fresh.forget_child("sub-rebuild1").await;
fresh
.record_child_usage(RlmChildUsageReport {
+ report_id: uuid::Uuid::new_v4().to_string(),
rlm_child_id: "sub-rebuild1".to_string(),
batches: vec![(
ChildUsageOrigin::DirectUser,
@@ -579,12 +711,13 @@ mod tests {
);
fresh
.record_child_usage(RlmChildUsageReport {
+ report_id: uuid::Uuid::new_v4().to_string(),
rlm_child_id: "sub-rebuild2".to_string(),
batches: vec![(ChildUsageOrigin::SpawnTask, usage_block(2, 2, 0, 0, 4, 0.0))],
})
.await;
- // A report racing the adoption's bases copy (base dropped to simulate):
- // the fold starts from the retired side's frozen handoff base.
+ // A report racing the adoption's bases copy (base dropped to simulate)
+ // must use the store's current aggregate, preserving every earlier report.
let target_of_second_for_base = fresh
.children
.lock()
@@ -595,6 +728,7 @@ mod tests {
fresh.bases.lock().await.remove(&target_of_second_for_base);
fresh
.record_child_usage(RlmChildUsageReport {
+ report_id: uuid::Uuid::new_v4().to_string(),
rlm_child_id: "sub-rebuild2".to_string(),
batches: vec![(
ChildUsageOrigin::AgentMessage,
@@ -609,8 +743,8 @@ mod tests {
.collect();
assert_eq!(
base_race.last().unwrap()["aggregateUsage"]["input"],
- 1_015,
- "the raced report folds onto the frozen retired base, not the default"
+ 1_025,
+ "the raced report preserves all usage already confirmed by the store"
);
// A registration the adoption copy raced (inserted on the
// retired side directly): the fallback consult adopts it.
@@ -627,6 +761,7 @@ mod tests {
}
fresh
.record_child_usage(RlmChildUsageReport {
+ report_id: uuid::Uuid::new_v4().to_string(),
rlm_child_id: "sub-raced".to_string(),
batches: vec![(ChildUsageOrigin::SpawnTask, usage_block(3, 3, 0, 0, 6, 0.0))],
})
@@ -641,19 +776,26 @@ mod tests {
6,
"the forwarded spawn, the base race, and the raced registration all attribute"
);
- // Continues the same cumulative chain: the raced 1,015 + 3 input.
- assert_eq!(raced[5]["aggregateUsage"]["input"], 1_018);
+ assert_eq!(
+ raced
+ .iter()
+ .map(|row| row["aggregateUsage"]["input"].as_u64().unwrap())
+ .collect::<Vec<_>>(),
+ vec![1_010, 1_017, 1_018, 1_020, 1_025, 1_028],
+ "handoff and stale producer caches cannot discard confirmed usage"
+ );
}
#[tokio::test]
async fn multiple_children_and_origins_share_the_cumulative_base() {
let raw_parent = usage_block(1_000, 100, 0, 0, 1_100, 0.01);
let (_tmp, manager) = manager_with_assistant(raw_parent);
- let producer = RlmChildUsageAttributions::new(manager.clone());
+ let producer = producer(manager.clone());
producer.register_spawn("sub-one").await;
producer.register_spawn("sub-two").await;
producer
.record_child_usage(RlmChildUsageReport {
+ report_id: uuid::Uuid::new_v4().to_string(),
rlm_child_id: "sub-one".to_string(),
batches: vec![(
ChildUsageOrigin::SpawnTask,
@@ -663,6 +805,7 @@ mod tests {
.await;
producer
.record_child_usage(RlmChildUsageReport {
+ report_id: uuid::Uuid::new_v4().to_string(),
rlm_child_id: "sub-two".to_string(),
batches: vec![
(
@@ -698,9 +841,10 @@ mod tests {
#[tokio::test]
async fn unregistered_child_report_attributes_nothing() {
let (_tmp, manager) = manager_with_assistant(usage_block(1, 1, 0, 0, 2, 0.0));
- let producer = RlmChildUsageAttributions::new(manager.clone());
+ let producer = producer(manager.clone());
producer
.record_child_usage(RlmChildUsageReport {
+ report_id: uuid::Uuid::new_v4().to_string(),
rlm_child_id: "sub-unknown".to_string(),
batches: vec![(
ChildUsageOrigin::SpawnTask,
@@ -713,4 +857,201 @@ mod tests {
.iter()
.all(|row| row["type"] != "child_usage_attributed"));
}
+
+ /// A store whose appends fail while the toggle is set, so a report's
+ /// first delivery fails and its retry persists.
+ struct FailingAppendStore {
+ store: SessionUsageStore,
+ fail: std::sync::atomic::AtomicBool,
+ }
+
+ impl RlmChildUsageStore for FailingAppendStore {
+ fn last_assistant(&self) -> RlmChildUsageFuture<'_, Option<(String, Usage)>> {
+ self.store.last_assistant()
+ }
+
+ fn append_attribution(
+ &self,
+ row_id: &str,
+ target_id: &str,
+ child_usage: Usage,
+ origin: Option<ChildUsageOrigin>,
+ ) -> RlmChildUsageFuture<'_, std::io::Result<ChildUsageAppendResult>> {
+ if self.fail.load(std::sync::atomic::Ordering::Relaxed) {
+ return Box::pin(std::future::ready(Err(std::io::Error::other(
+ "append disabled",
+ ))));
+ }
+ self.store
+ .append_attribution(row_id, target_id, child_usage, origin)
+ }
+ }
+
+ /// A failed append returns false and writes no row; the retry persists
+ /// the batch once, folding from the unchanged base (no double fold).
+ #[tokio::test]
+ async fn a_failed_append_leaves_the_base_for_the_retry() {
+ let raw_parent = usage_block(1_000, 0, 0, 0, 4_096, 0.0);
+ let (_tmp, manager) = manager_with_assistant(raw_parent);
+ let store = std::sync::Arc::new(FailingAppendStore {
+ store: SessionUsageStore(manager.clone()),
+ fail: std::sync::atomic::AtomicBool::new(true),
+ });
+ let producer = RlmChildUsageAttributions::new(store.clone());
+ producer.register_spawn("sub-flaky").await;
+ let report = || RlmChildUsageReport {
+ report_id: uuid::Uuid::new_v4().to_string(),
+ rlm_child_id: "sub-flaky".to_string(),
+ batches: vec![(
+ ChildUsageOrigin::SpawnTask,
+ usage_block(10, 5, 0, 0, 15, 0.01),
+ )],
+ };
+ assert!(!producer.record_child_usage(report()).await);
+ let rows = file_rows(&manager).await;
+ assert!(rows
+ .iter()
+ .all(|row| row["type"] != "child_usage_attributed"));
+
+ store
+ .fail
+ .store(false, std::sync::atomic::Ordering::Relaxed);
+ assert!(producer.record_child_usage(report()).await);
+ let rows = file_rows(&manager).await;
+ let attributed: Vec<&serde_json::Value> = rows
+ .iter()
+ .filter(|row| row["type"] == "child_usage_attributed")
+ .collect();
+ assert_eq!(attributed.len(), 1, "the retry persisted the batch once");
+ assert_eq!(attributed[0]["aggregateUsage"]["input"], 1_010);
+ }
+
+ enum FailurePoint {
+ BeforeSecondAppend,
+ AfterFirstAppend,
+ }
+
+ /// Fail one chosen store acknowledgment while retaining real transcript writes.
+ struct InterruptedAppendStore {
+ store: SessionUsageStore,
+ calls: std::sync::atomic::AtomicUsize,
+ point: FailurePoint,
+ }
+
+ impl RlmChildUsageStore for InterruptedAppendStore {
+ fn last_assistant(&self) -> RlmChildUsageFuture<'_, Option<(String, Usage)>> {
+ self.store.last_assistant()
+ }
+
+ fn append_attribution(
+ &self,
+ row_id: &str,
+ target_id: &str,
+ child_usage: Usage,
+ origin: Option<ChildUsageOrigin>,
+ ) -> RlmChildUsageFuture<'_, std::io::Result<ChildUsageAppendResult>> {
+ let target_id = target_id.to_string();
+ let row_id = row_id.to_string();
+ Box::pin(async move {
+ let call = self.calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
+ if matches!(self.point, FailurePoint::BeforeSecondAppend) && call == 1 {
+ return Err(std::io::Error::other("injected second append failure"));
+ }
+ let result = self
+ .store
+ .append_attribution(&row_id, &target_id, child_usage, origin)
+ .await?;
+ if matches!(self.point, FailurePoint::AfterFirstAppend) && call == 0 {
+ return Err(std::io::Error::other("injected lost append acknowledgment"));
+ }
+ Ok(result)
+ })
+ }
+ }
+
+ /// Acknowledged origins must not be folded again when a later origin fails.
+ #[tokio::test]
+ async fn a_partially_persisted_report_is_counted_once_on_retry() {
+ let (_tmp, manager) = manager_with_assistant(usage_block(1_000, 0, 0, 0, 4_096, 0.0));
+ let producer =
+ RlmChildUsageAttributions::new(std::sync::Arc::new(InterruptedAppendStore {
+ store: SessionUsageStore(manager.clone()),
+ calls: std::sync::atomic::AtomicUsize::new(0),
+ point: FailurePoint::BeforeSecondAppend,
+ }));
+ producer.register_spawn("sub-partial").await;
+ let report = RlmChildUsageReport {
+ report_id: uuid::Uuid::new_v4().to_string(),
+ rlm_child_id: "sub-partial".to_string(),
+ batches: vec![
+ (
+ ChildUsageOrigin::SpawnTask,
+ usage_block(10, 0, 0, 0, 10, 0.01),
+ ),
+ (
+ ChildUsageOrigin::AgentMessage,
+ usage_block(20, 0, 0, 0, 20, 0.02),
+ ),
+ ],
+ };
+ assert!(!producer.record_child_usage(report.clone()).await);
+ assert!(producer.record_child_usage(report).await);
+ let rows = file_rows(&manager).await;
+ let attributions: Vec<_> = rows
+ .iter()
+ .filter(|row| row["type"] == "child_usage_attributed")
+ .map(|row| {
+ serde_json::json!({
+ "origin": row["origin"],
+ "childInput": row["childUsage"]["input"],
+ "aggregateInput": row["aggregateUsage"]["input"],
+ })
+ })
+ .collect();
+ assert_eq!(
+ attributions,
+ vec![
+ serde_json::json!({"origin": "spawn_task", "childInput": 10, "aggregateInput": 1010}),
+ serde_json::json!({"origin": "agent_message", "childInput": 20, "aggregateInput": 1030}),
+ ]
+ );
+ }
+
+ /// An error after a complete real append cannot safely mean nothing persisted.
+ #[tokio::test]
+ async fn a_persisted_report_with_lost_acknowledgment_is_not_appended_twice() {
+ let (_tmp, manager) = manager_with_assistant(usage_block(1_000, 0, 0, 0, 4_096, 0.0));
+ let producer =
+ RlmChildUsageAttributions::new(std::sync::Arc::new(InterruptedAppendStore {
+ store: SessionUsageStore(manager.clone()),
+ calls: std::sync::atomic::AtomicUsize::new(0),
+ point: FailurePoint::AfterFirstAppend,
+ }));
+ producer.register_spawn("sub-uncertain").await;
+ let report = RlmChildUsageReport {
+ report_id: uuid::Uuid::new_v4().to_string(),
+ rlm_child_id: "sub-uncertain".to_string(),
+ batches: vec![(
+ ChildUsageOrigin::SpawnTask,
+ usage_block(10, 0, 0, 0, 10, 0.01),
+ )],
+ };
+ assert!(!producer.record_child_usage(report.clone()).await);
+ let rows = file_rows(&manager).await;
+ assert_eq!(
+ rows.iter()
+ .filter(|row| row["type"] == "child_usage_attributed")
+ .count(),
+ 1
+ );
+ assert!(producer.record_child_usage(report).await);
+ let rows = file_rows(&manager).await;
+ assert_eq!(
+ rows.iter()
+ .filter(|row| row["type"] == "child_usage_attributed")
+ .count(),
+ 1,
+ "the complete disk row survives the failed acknowledgment and must be reused"
+ );
+ }
}
diff --git a/crates/pa-core/src/session_engine/runtime_wiring.rs b/crates/pa-core/src/session_engine/runtime_wiring.rs
index 1effbb0aef..75eedd9cf6 100644
--- a/crates/pa-core/src/session_engine/runtime_wiring.rs
+++ b/crates/pa-core/src/session_engine/runtime_wiring.rs
@@ -35,6 +35,7 @@ pub struct RlmWiring {
/// Child-session machinery backing `rlm.spawn`/`rlm.create_session` and
/// the roster/collect/delete surface.
pub subagent_host: Option<Arc<dyn RlmSubagentHost>>,
+ pub usage_store: Option<Arc<dyn super::rlm_usage::RlmChildUsageStore>>,
/// Announces every accepted `rlm.progress.note`; `None` keeps the store
/// the only surface.
pub progress_note_emit: Option<super::rlm_host::RlmProgressNoteEmit>,
@@ -166,7 +167,8 @@ pub fn wire_session_runtime(
Arc::new(registry)
});
let rlm_usage = Arc::new(super::rlm_usage::RlmChildUsageAttributions::new(
- session.clone(),
+ rlm.usage_store
+ .unwrap_or_else(|| Arc::new(super::rlm_usage::SessionUsageStore(session.clone()))),
));
// The daemon supplies the child-session host; an embedding without
// one keeps the no-children behavior, whose self-rename appends the
diff --git a/crates/pa-core/tests/abort_kernel_cell.rs b/crates/pa-core/tests/abort_kernel_cell.rs
index 0cf8ae91cf..85ef1a298b 100644
--- a/crates/pa-core/tests/abort_kernel_cell.rs
+++ b/crates/pa-core/tests/abort_kernel_cell.rs
@@ -164,6 +164,7 @@ async fn abort_during_a_kernel_cell_settles_the_turn_immediately() {
additional_prompt_paths: Vec::new(),
extra_builtin_skill_overrides: Vec::new(),
rlm_subagent_host: None,
+ rlm_usage_store: None,
progress_note_emit: None,
rlm_depth: None,
daemonless: true,
diff --git a/crates/pa-core/tests/turn_boundary_kernel.rs b/crates/pa-core/tests/turn_boundary_kernel.rs
index 6e80977c6f..744990834a 100644
--- a/crates/pa-core/tests/turn_boundary_kernel.rs
+++ b/crates/pa-core/tests/turn_boundary_kernel.rs
@@ -199,6 +199,7 @@ async fn turn_boundary_host_requests_round_trip_through_a_real_kernel() {
additional_prompt_paths: Vec::new(),
extra_builtin_skill_overrides: Vec::new(),
rlm_subagent_host: None,
+ rlm_usage_store: None,
progress_note_emit: None,
rlm_depth: None,
daemonless: true,
diff --git a/crates/pa-daemon/src/agent_engine.rs b/crates/pa-daemon/src/agent_engine.rs
index df5671af8a..5bec2032d0 100644
--- a/crates/pa-daemon/src/agent_engine.rs
+++ b/crates/pa-daemon/src/agent_engine.rs
@@ -230,6 +230,9 @@ pub struct AgentSessionEngine {
usage_producer: std::sync::Mutex<
Option<std::sync::Arc<pa_core::session_engine::rlm_usage::RlmChildUsageAttributions>>,
>,
+ rlm_usage_store: std::sync::Mutex<
+ Option<std::sync::Arc<dyn pa_core::session_engine::rlm_usage::RlmChildUsageStore>>,
+ >,
own_summary: std::sync::Arc<std::sync::Mutex<Option<Value>>>,
pub(crate) create_resources: std::sync::RwLock<CreateSessionResources>,
pub(crate) autonomous:
diff --git a/crates/pa-daemon/src/agent_engine/config.rs b/crates/pa-daemon/src/agent_engine/config.rs
index 1d2d35aadd..7e7232bba2 100644
--- a/crates/pa-daemon/src/agent_engine/config.rs
+++ b/crates/pa-daemon/src/agent_engine/config.rs
@@ -112,11 +112,9 @@ impl pa_core::session_engine::rlm_usage::RlmChildUsageSink for ProducerUsageSink
fn record(
&self,
report: pa_core::session_engine::rlm_usage::RlmChildUsageReport,
- ) -> std::pin::Pin<std::boxed::Box<dyn std::future::Future<Output = ()> + Send + '_>> {
+ ) -> std::pin::Pin<std::boxed::Box<dyn std::future::Future<Output = bool> + Send + '_>> {
let producer = std::sync::Arc::clone(&self.0);
- Box::pin(async move {
- producer.record_child_usage(report).await;
- })
+ Box::pin(async move { producer.record_child_usage(report).await })
}
fn forget(
diff --git a/crates/pa-daemon/src/agent_engine/lifecycle.rs b/crates/pa-daemon/src/agent_engine/lifecycle.rs
index 1abed1189d..df21317da4 100644
--- a/crates/pa-daemon/src/agent_engine/lifecycle.rs
+++ b/crates/pa-daemon/src/agent_engine/lifecycle.rs
@@ -166,6 +166,7 @@ impl AgentSessionEngine {
link,
children,
usage_producer: std::sync::Mutex::new(None),
+ rlm_usage_store: std::sync::Mutex::new(None),
quota_park: std::sync::Arc::new(std::sync::Mutex::new(None)),
quota_parked_this_run: std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false)),
autonomous_driver,
@@ -273,6 +274,13 @@ impl AgentSessionEngine {
.expect("compaction summary sink lock") = Some(sink);
}
+ pub(crate) fn set_rlm_usage_store(
+ &self,
+ store: std::sync::Arc<dyn pa_core::session_engine::rlm_usage::RlmChildUsageStore>,
+ ) {
+ *self.rlm_usage_store.lock().expect("rlm usage store lock") = Some(store);
+ }
+
/// Installs the worker's accepted-note announcer, threaded into every
/// session build.
///
@@ -927,6 +935,11 @@ impl AgentSessionEngine {
.lock()
.expect("goal queue purge lock")
.clone();
+ let rlm_usage_store = self
+ .rlm_usage_store
+ .lock()
+ .expect("rlm usage store lock")
+ .clone();
let progress_note_emit = self
.progress_note_emit
.lock()
@@ -989,6 +1002,7 @@ impl AgentSessionEngine {
rlm_subagent_host: self.children.clone().map(|children| {
children as Arc<dyn pa_core::session_engine::rlm_host::RlmSubagentHost>
}),
+ rlm_usage_store,
progress_note_emit,
rlm_depth: Some(self.rlm_depth.load(std::sync::atomic::Ordering::Relaxed)),
// The worker session runs under the daemon scheduler and its wake-ups.
diff --git a/crates/pa-daemon/src/lease.rs b/crates/pa-daemon/src/lease.rs
index cd019243df..e6a3684b1c 100644
--- a/crates/pa-daemon/src/lease.rs
+++ b/crates/pa-daemon/src/lease.rs
@@ -375,6 +375,16 @@ impl Drop for SessionLease {
}
}
+/// Counts an admitted mutation through errors and unwinding. This participates
+/// in the existing bounded release drain; it does not extend its deadline.
+struct OwnedMutationFlight<'a>(&'a std::sync::atomic::AtomicUsize);
+
+impl Drop for OwnedMutationFlight<'_> {
+ fn drop(&mut self) {
+ self.0.fetch_sub(1, std::sync::atomic::Ordering::SeqCst);
+ }
+}
+
impl SessionLease {
pub(crate) fn acquire_target(&self, path: &Path) -> Result<std::sync::Arc<Self>> {
let agent_dir = self
@@ -385,25 +395,34 @@ impl SessionLease {
acquire_runtime_session_lease(path, agent_dir).map(std::sync::Arc::new)
}
- pub(crate) fn append(&self, path: &Path, bytes: &[u8]) -> Result<()> {
+ /// Hold the existing append admission fence across proof, mutation and sync.
+ /// Release retains its existing ten-second drain policy: a wedged IO call
+ /// or suspended process beyond that bound is not absolute handoff safety.
+ pub(crate) fn with_owned_mutation<T>(
+ &self,
+ path: &Path,
+ action: impl FnOnce() -> Result<T>,
+ ) -> Result<T> {
self.append_in_flight
.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
- let result = (|| {
- anyhow::ensure!(
- !self.released.load(std::sync::atomic::Ordering::SeqCst)
- && canonical_session_path(path) == self.session_path,
- "session lease does not own append target"
- );
+ let _flight = OwnedMutationFlight(&self.append_in_flight);
+ anyhow::ensure!(
+ !self.released.load(std::sync::atomic::Ordering::SeqCst)
+ && canonical_session_path(path) == self.session_path,
+ "session lease does not own append target"
+ );
+ action()
+ }
+
+ pub(crate) fn append(&self, path: &Path, bytes: &[u8]) -> Result<()> {
+ self.with_owned_mutation(path, || {
pa_core::session::window::append_cached(
path,
bytes,
pa_core::session::window::AppendOwnership::SessionLeaseHeld,
)?;
Ok(())
- })();
- self.append_in_flight
- .fetch_sub(1, std::sync::atomic::Ordering::SeqCst);
- result
+ })
}
}
@@ -889,6 +908,86 @@ mod tests {
assert_eq!(reclaim_retry_delay_ms(true, &unrelated, 1), None);
}
+ #[test]
+ fn owned_mutation_release_waits_for_the_admitted_callback() {
+ use std::io::Write;
+ let dir = tempfile::tempdir().unwrap();
+ let path = dir.path().join("session.jsonl");
+ fs::write(&path, b"{}\n").unwrap();
+ let lease = std::sync::Arc::new(acquire_runtime_session_lease(&path, dir.path()).unwrap());
+ let directory = lease.directory.clone();
+ let (started_tx, started_rx) = std::sync::mpsc::sync_channel(1);
+ let (finish_tx, finish_rx) = std::sync::mpsc::sync_channel(1);
+ std::thread::scope(|scope| {
+ let writer_lease = std::sync::Arc::clone(&lease);
+ let writer_path = path.clone();
+ let mutation = scope.spawn(move || {
+ writer_lease.with_owned_mutation(&writer_path, || {
+ started_tx.send(()).unwrap();
+ finish_rx.recv_timeout(Duration::from_secs(3)).unwrap();
+ let mut file = fs::OpenOptions::new().append(true).open(&writer_path)?;
+ file.write_all(b"continued\n")?;
+ file.sync_all()?;
+ Ok(())
+ })
+ });
+ started_rx.recv_timeout(Duration::from_secs(3)).unwrap();
+ let releaser_lease = std::sync::Arc::clone(&lease);
+ let release = scope.spawn(move || releaser_lease.release());
+ let deadline = Instant::now() + Duration::from_secs(2);
+ while !lease.released.load(std::sync::atomic::Ordering::SeqCst)
+ && Instant::now() < deadline
+ {
+ std::thread::yield_now();
+ }
+ assert!(lease.released.load(std::sync::atomic::Ordering::SeqCst));
+ assert_eq!(
+ lease
+ .append_in_flight
+ .load(std::sync::atomic::Ordering::SeqCst),
+ 1
+ );
+ assert!(
+ directory.exists(),
+ "the admitted callback still owns the lease inside its drain bound"
+ );
+ finish_tx.send(()).unwrap();
+ mutation.join().unwrap().unwrap();
+ release.join().unwrap();
+ });
+ assert_eq!(fs::read(&path).unwrap(), b"{}\ncontinued\n");
+ assert!(!directory.exists());
+ }
+
+ #[test]
+ fn owned_mutation_errors_and_unwind_do_not_leak_a_flight() {
+ let dir = tempfile::tempdir().unwrap();
+ let path = dir.path().join("session.jsonl");
+ fs::write(&path, b"{}\n").unwrap();
+ let lease = acquire_runtime_session_lease(&path, dir.path()).unwrap();
+ assert!(lease
+ .with_owned_mutation::<()>(&path, || Err(anyhow!("injected callback error")))
+ .is_err());
+ assert_eq!(
+ lease
+ .append_in_flight
+ .load(std::sync::atomic::Ordering::SeqCst),
+ 0
+ );
+ let panicked = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
+ let _ = lease.with_owned_mutation::<()>(&path, || panic!("injected callback unwind"));
+ }));
+ assert!(panicked.is_err());
+ assert_eq!(
+ lease
+ .append_in_flight
+ .load(std::sync::atomic::Ordering::SeqCst),
+ 0
+ );
+ lease.release();
+ assert!(!lease.directory.exists());
+ }
+
#[test]
fn release_drains_an_append_still_in_flight() {
let dir = tempfile::tempdir().unwrap();
diff --git a/crates/pa-daemon/src/rlm_children.rs b/crates/pa-daemon/src/rlm_children.rs
index b2f3687e27..ba28bd4797 100644
--- a/crates/pa-daemon/src/rlm_children.rs
+++ b/crates/pa-daemon/src/rlm_children.rs
@@ -169,6 +169,14 @@ struct ChildRecord {
/// The parent session closed while this child ran: the settle watcher
/// exits without a notice — none is owed to a session being torn down.
closed_by_parent: bool,
+ /// Confirmed retirement completed its roster, notice and settle bookkeeping.
+ retirement_finished: bool,
+ /// Claim the synchronous delete notifier once across interrupted retirement retries.
+ retirement_notified: bool,
+ /// Claim the retirement roster event once even if a later await is canceled.
+ retirement_roster_emitted: bool,
+ /// The final unregister acknowledged completion; retries must not repeat it.
+ usage_registration_forgotten: bool,
/// The child's durable session file: the usage walk's source.
session_file: Option<String>,
/// Rows of [`ChildRecord::session_file`] already folded into the
@@ -177,6 +185,8 @@ struct ChildRecord {
/// nothing has been observed since the reseed, and the first delivery
/// primes the cursor at the file's tail.
attributed_rows: Option<usize>,
+ /// Frozen before delivery: cancellation and retries keep the same row identities.
+ pending_usage_report: Option<(RlmChildUsageReport, usize)>,
/// A follow-up usage watcher is live for this retained child
/// (delayed agent messaging after the task run settled).
usage_watch_live: bool,
@@ -312,6 +322,18 @@ pub type DeleteNotifier = std::sync::Arc<dyn Fn(&str) + Send + Sync>;
pub(crate) type ChildUpdateSink = std::sync::Arc<dyn Fn(Value) + Send + Sync>;
+#[derive(Debug, Clone, Copy)]
+enum RetirementIntent {
+ Delete { was_running: bool },
+ Close,
+}
+
+#[derive(Clone)]
+struct RetiredChild {
+ record: Arc<Mutex<ChildRecord>>,
+ intent: RetirementIntent,
+}
+
struct SupervisorChildSessionsInner {
link: Arc<SupervisorLink>,
agent_dir: PathBuf,
@@ -324,6 +346,8 @@ struct SupervisorChildSessionsInner {
// an await, so sync engine paths can set it without a runtime `block_on`.
identity: std::sync::Mutex<ParentIdentity>,
children: Mutex<Vec<Arc<Mutex<ChildRecord>>>>,
+ /// Confirmed-dead children whose final usage remains owed. Not roster or slot owners.
+ retired_usage: std::sync::Mutex<Vec<RetiredChild>>,
/// Spawn-name reservations held until admission is durable, so
/// parallel same-name spawns cannot both admit (default names never reserve).
pending_spawn_names: std::sync::Mutex<std::collections::HashSet<String>>,
@@ -389,6 +413,7 @@ impl SupervisorChildSessions {
parent_session_name: std::sync::Mutex::new(None),
identity: std::sync::Mutex::new(ParentIdentity::with_default_depth()),
children: Mutex::new(Vec::new()),
+ retired_usage: std::sync::Mutex::new(Vec::new()),
pending_spawn_names: std::sync::Mutex::new(std::collections::HashSet::new()),
deleted_children: std::sync::Mutex::new(std::collections::HashMap::new()),
turn_done: tokio::sync::watch::Sender::new(0),
@@ -672,8 +697,13 @@ impl SupervisorChildSessions {
prompt_admitted: true,
error: None,
closed_by_parent: false,
+ retirement_finished: false,
+ retirement_notified: false,
+ retirement_roster_emitted: false,
+ usage_registration_forgotten: false,
session_file: None,
attributed_rows: Some(0),
+ pending_usage_report: None,
usage_watch_live: false,
usage_rearm: false,
emit_lock: std::sync::Arc::new(tokio::sync::Mutex::new(())),
@@ -844,30 +874,6 @@ impl SupervisorChildSessionsInner {
sink(row);
}
- pub(crate) async fn emit_child_removal(&self, record: &Arc<Mutex<ChildRecord>>) {
- let sink = self
- .child_update_sink
- .lock()
- .unwrap_or_else(std::sync::PoisonError::into_inner)
- .clone();
- let Some(sink) = sink else {
- return;
- };
- let row = {
- let record = record.lock().await;
- json!({
- "id": record.rlm_child_id,
- "activeSessionId": record.active_session_id,
- "sessionName": record.session_name,
- "label": record.session_name,
- "status": "cancelled",
- "sessionDir": record.session_dir,
- "error": "Deleted by parent orchestrator",
- })
- };
- sink(row);
- }
-
/// Wait for the parent turn that spawned a task to complete (generation
/// strictly greater than the one captured at spawn admission). Bounded:
/// a turn that never settles releases the child anyway.
diff --git a/crates/pa-daemon/src/rlm_children/host.rs b/crates/pa-daemon/src/rlm_children/host.rs
index 2f19a42815..08bf098a3c 100644
--- a/crates/pa-daemon/src/rlm_children/host.rs
+++ b/crates/pa-daemon/src/rlm_children/host.rs
@@ -3,11 +3,10 @@
//! `collect`, `rename`) over the supervisor's child-sessions registry,
//! with the spawn-admission helpers only this surface uses.
use super::{
- assert_thinking_supported, bail, create_default_rlm_subagent_session_name,
- create_rlm_child_terminal_notice, json, now_ms, resolve_child_model, rlm_child_label,
- rlm_depth_limit_error, spawn_name_unavailable_error, Arc, ChildCloseReason, ChildRecord,
- Context, DaemonCommand, Duration, Instant, Mutex, Path, PathBuf, Result, RlmChildResult,
- RlmChildTerminalNotice, RlmCreateSessionHandle, RlmCreateSessionRequest,
+ assert_thinking_supported, bail, create_default_rlm_subagent_session_name, json, now_ms,
+ resolve_child_model, rlm_child_label, rlm_depth_limit_error, spawn_name_unavailable_error, Arc,
+ ChildCloseReason, ChildRecord, Context, DaemonCommand, Duration, Instant, Mutex, Path, PathBuf,
+ Result, RlmChildResult, RlmCreateSessionHandle, RlmCreateSessionRequest,
RlmDeleteSubagentResult, RlmHostFuture, RlmSpawnHandle, RlmSpawnRequest, RlmSubagentEntry,
RlmSubagentHost, SpawnNameReservationGuard, SupervisorChildSessions,
SupervisorChildSessionsInner, Value, KILL_TIMEOUT_MS, RENAME_TIMEOUT_MS,
@@ -137,8 +136,13 @@ impl RlmSubagentHost for SupervisorChildSessions {
prompt_admitted: false,
error: None,
closed_by_parent: false,
+ retirement_finished: false,
+ retirement_notified: false,
+ retirement_roster_emitted: false,
+ usage_registration_forgotten: false,
session_file: created.session_file.clone(),
attributed_rows: Some(0),
+ pending_usage_report: None,
usage_watch_live: false,
usage_rearm: false,
emit_lock: std::sync::Arc::new(tokio::sync::Mutex::new(())),
@@ -347,6 +351,9 @@ impl RlmSubagentHost for SupervisorChildSessions {
fn delete_subagent(&self, target: String) -> RlmHostFuture<RlmDeleteSubagentResult> {
let this = Arc::clone(&self.inner);
Box::pin(async move {
+ if !this.replay_retired_usage(Some(&target)).await {
+ bail!("RLM child usage remains owed after retirement; retry deletion");
+ }
// Selector errors surface unwrapped (the TS message is the
// product surface); only the kill below gets a delete context.
let record = match this.resolve_record(&target, "subagent").await {
@@ -400,9 +407,10 @@ impl RlmSubagentHost for SupervisorChildSessions {
};
let rename_lock = record.lock().await.rename_lock.clone();
let _rename_guard = rename_lock.lock().await;
+ let already_terminated = this.has_retired_usage(&record);
let no_longer_matches = {
let record = record.lock().await;
- record.closed_by_parent || !record.matches(&target)
+ (record.closed_by_parent && !already_terminated) || !record.matches(&target)
};
if no_longer_matches {
bail!("No direct RLM subagent matches \"{target}\" in the current parent session");
@@ -424,83 +432,28 @@ impl RlmSubagentHost for SupervisorChildSessions {
let was_running = record.lock().await.settled_status.is_none();
// Capture before the kill: a deleted running child's durable
// rows are its last observable spend on the parent side.
- this.emit_child_usage(&record).await;
- this.command(&command, KILL_TIMEOUT_MS)
- .await
- .with_context(|| format!("kill RLM child \"{target}\""))?;
- // Rows can land between the pre-kill capture and the kill
- // reaching the worker: the post-kill walk is the LAST observation
- // (the cursor keeps it free of double-billing).
- this.emit_child_usage(&record).await;
- // The final observation landed: the registration drops.
- this.forget_child_usage(&record).await;
- // The watcher owns an Arc to this record; deleting the roster
- // row alone cannot stop its polling loop.
- {
- let mut record = record.lock().await;
- record.closed_by_parent = true;
- if was_running {
- record.settled_status = Some("cancelled");
- record.error = Some("Deleted by parent orchestrator".to_string());
- }
+ if !already_terminated {
+ this.emit_child_usage(&record).await;
+ this.command(&command, KILL_TIMEOUT_MS)
+ .await
+ .with_context(|| format!("kill RLM child \"{target}\""))?;
+ this.retain_retired_usage(&record, super::RetirementIntent::Delete { was_running });
+ // Observe any rows that landed before the acknowledged kill.
+ this.emit_child_usage(&record).await;
}
+ // The final observation landed: the registration drops.
+ let usage_persisted = this.forget_child_usage(&record).await;
+ this.finish_retirement(&record).await;
let entry = {
let record = record.lock().await;
SupervisorChildSessions::entry(&record)
};
- // The tombstone keeps the identity behind the registry so a
- // just-deleted selector still answers `collect`.
- {
- let record = record.lock().await;
- this.remember_deleted_child(&record);
- }
- if was_running {
- this.emit_child_update(&record).await;
- } else {
- this.emit_child_removal(&record).await;
+ if usage_persisted {
+ this.remove_retired_usage(&record);
}
- // The deletion commits BEFORE the best-effort terminal
- // notice: the notice's supervisor delivery can ride its full
- // timeout, and a deleted child must leave the registry and the
- // cached context tree immediately, not after it.
- this.children
- .lock()
- .await
- .retain(|candidate| !Arc::ptr_eq(candidate, &record));
- // A deleted subagent leaves `/context` immediately (the
- // background refresh would otherwise resurrect it).
- if let Some(notify) = this
- .delete_notifier
- .lock()
- .expect("delete notifier lock")
- .clone()
- {
- notify(&entry.rlm_child_id);
+ if !usage_persisted {
+ bail!("RLM child usage remains owed after retirement; retry deletion");
}
- // A still-running child was cut short by the delete: the
- // parent receives the cancelled terminal notice.
- if was_running {
- let notice = {
- let mut record = record.lock().await;
- let claimed = !record.notice_delivered;
- record.notice_delivered = true;
- claimed.then(|| RlmChildTerminalNotice::Cancelled {
- child_id: record.rlm_child_id.clone(),
- session_name: record.session_name.clone(),
- reason: Some("Deleted by parent orchestrator".to_string()),
- })
- };
- if let Some(notice) = notice {
- this.deliver_terminal_notice(create_rlm_child_terminal_notice(
- &notice,
- now_ms(),
- ))
- .await;
- }
- }
- // The deletion settles the run: a parked barrier re-reads a removed
- // record as settled.
- this.fire_settle_hook(&record).await;
Ok(RlmDeleteSubagentResult {
subagent: entry,
outcome: Some("deleted"),
@@ -536,11 +489,8 @@ impl RlmSubagentHost for SupervisorChildSessions {
.collect::<Vec<_>>();
match matches.len() {
0 => return Err(miss),
- 1 => deleted_results.push(
- SupervisorChildSessions::deleted_collect_result(
- &matches[0],
- ),
- ),
+ 1 => deleted_results
+ .push(SupervisorChildSessions::deleted_collect_result(&matches[0])),
_ => bail!(
"RLM child selector \"{target}\" is ambiguous in the current parent session"
),
@@ -646,7 +596,9 @@ impl RlmSubagentHost for SupervisorChildSessions {
};
if let Some(record) = &record {
if record.lock().await.closed_by_parent {
- bail!("rlm.rename can only rename the current session or one of its direct children");
+ bail!(
+ "rlm.rename can only rename the current session or one of its direct children"
+ );
}
}
// The rename itself is daemon-owned: the supervisor's live
diff --git a/crates/pa-daemon/src/rlm_children/lifecycle.rs b/crates/pa-daemon/src/rlm_children/lifecycle.rs
index 503972426b..356f0782f3 100644
--- a/crates/pa-daemon/src/rlm_children/lifecycle.rs
+++ b/crates/pa-daemon/src/rlm_children/lifecycle.rs
@@ -485,7 +485,7 @@ impl SupervisorChildSessionsInner {
self.refresh_record(record).await;
// The run's final rows are attributed before the terminal
// notice rides the parent's follow-up route.
- self.emit_child_usage(record).await;
+ self.emit_final_child_usage(record).await;
// TS records the return before delivering the notice: the
// pending edge must be on the parent's ledger before the
// notice's follow-up turn mints its own request.
@@ -533,7 +533,7 @@ impl SupervisorChildSessionsInner {
if unreachable_polls >= WATCH_MAX_UNREACHABLE_POLLS {
// A dead child keeps whatever rows its file already
// holds; capture them before the terminal notice.
- self.emit_child_usage(record).await;
+ self.emit_final_child_usage(record).await;
if self
.settle_failed(
record,
diff --git a/crates/pa-daemon/src/rlm_children/registry.rs b/crates/pa-daemon/src/rlm_children/registry.rs
index dfd2e74f16..00945ed9e7 100644
--- a/crates/pa-daemon/src/rlm_children/registry.rs
+++ b/crates/pa-daemon/src/rlm_children/registry.rs
@@ -98,15 +98,32 @@ impl SupervisorChildSessionsInner {
/// Delete one inactive child by id: refresh the registry row first, refuse one still in flight,
/// and tear a settled one down with its ledger tombstone (same as `rlm.delete_subagent`).
pub(super) async fn delete_inactive_subagent(&self, child_id: &str) -> Result<&'static str> {
+ if !self.replay_retired_usage(Some(child_id)).await {
+ bail!("RLM child usage remains owed after retirement; retry deletion");
+ }
let children = self.children.lock().await.clone();
for record in &children {
let matched = record.lock().await.rlm_child_id == child_id;
if !matched {
continue;
}
+ let rename_lock = record.lock().await.rename_lock.clone();
+ let _guard = rename_lock.lock().await;
+ let already_terminated = self.has_retired_usage(record);
+ if !self
+ .children
+ .lock()
+ .await
+ .iter()
+ .any(|held| Arc::ptr_eq(held, record))
+ {
+ continue;
+ }
// Freshness pass: a child that just went idle settles here
// and stays deletable.
- self.refresh_record(record).await;
+ if !already_terminated {
+ self.refresh_record(record).await;
+ }
let (running, active_session_id) = {
let record = record.lock().await;
(
@@ -114,90 +131,100 @@ impl SupervisorChildSessionsInner {
record.active_session_id.clone(),
)
};
- if running {
+ if running && !already_terminated {
return Ok("running");
}
// Capture before the unlink: a deleted child's already-durable
// rows are its only remaining spend record on the parent side.
- self.emit_child_usage(record).await;
- let command = DaemonCommand::Kill {
- id: None,
- active_session_id: active_session_id.clone(),
- rest: serde_json::Map::from_iter([
- ("rlmLedgerDelete".to_string(), json!("user")),
- ("rlmChildId".to_string(), json!(child_id)),
- ]),
- };
- self.command(&command, KILL_TIMEOUT_MS)
- .await
- .with_context(|| format!("kill RLM child \"{child_id}\""))?;
- // Rows can land between the pre-kill capture and the kill
- // (a settle racing the kill): the post-kill walk is the last
- // observation; the cursor keeps it free of double-billing.
- self.emit_child_usage(record).await;
- self.forget_child_usage(record).await;
- // Any live usage watcher retires with the record: a
- // follow-up watch must not keep polling the killed worker.
- record.lock().await.closed_by_parent = true;
- // The inactive delete leaves the same tombstone as the live
- // delete, so `collect` answers a just-deleted selector.
- {
- let record = record.lock().await;
- self.remember_deleted_child(&record);
+ if !already_terminated {
+ self.emit_child_usage(record).await;
+ let command = DaemonCommand::Kill {
+ id: None,
+ active_session_id: active_session_id.clone(),
+ rest: serde_json::Map::from_iter([
+ ("rlmLedgerDelete".to_string(), json!("user")),
+ ("rlmChildId".to_string(), json!(child_id)),
+ ]),
+ };
+ self.command(&command, KILL_TIMEOUT_MS)
+ .await
+ .with_context(|| format!("kill RLM child \"{child_id}\""))?;
+ // Rows can land between the pre-kill capture and the kill
+ // (a settle racing the kill): the post-kill walk is the last
+ // observation; the cursor keeps it free of double-billing.
+ self.retain_retired_usage(
+ record,
+ super::RetirementIntent::Delete { was_running: false },
+ );
+ self.emit_child_usage(record).await;
+ }
+ let usage_persisted = self.forget_child_usage(record).await;
+ self.finish_retirement(record).await;
+ if usage_persisted {
+ self.remove_retired_usage(record);
+ }
+ if !usage_persisted {
+ bail!("RLM child usage remains owed after retirement; retry deletion");
}
- self.emit_child_removal(record).await;
- self.children
- .lock()
- .await
- .retain(|candidate| !Arc::ptr_eq(candidate, record));
- // The deleted child is a resume site for the owed goal continuation.
- self.fire_settle_hook(record).await;
return Ok("deleted");
}
- Ok("not_found")
+ if self
+ .deleted_children
+ .lock()
+ .unwrap_or_else(std::sync::PoisonError::into_inner)
+ .contains_key(child_id)
+ {
+ Ok("deleted")
+ } else {
+ Ok("not_found")
+ }
}
/// A child whose session is already gone is a completed no-op; every
/// other close failure is kept (walk all children, rethrow the first error).
pub(super) async fn close_children_inner(&self, reason: ChildCloseReason) -> Result<()> {
+ let usage_persisted = self.replay_retired_usage(None).await;
let children = self.children.lock().await.clone();
- let mut close_error: Option<anyhow::Error> = None;
+ let mut close_error = (!usage_persisted)
+ .then(|| anyhow::anyhow!("RLM child usage remains owed after retirement; retry close"));
for record in &children {
+ let rename_lock = record.lock().await.rename_lock.clone();
+ let _guard = rename_lock.lock().await;
+ if !self
+ .children
+ .lock()
+ .await
+ .iter()
+ .any(|held| Arc::ptr_eq(held, record))
{
- let mut record = record.lock().await;
- record.closed_by_parent = true;
- record.notice_delivered = true;
+ continue;
}
- // Capture before the close: the teardown settles each run;
- // nothing observes the child after the kill.
- self.emit_child_usage(record).await;
- let active_session_id = record.lock().await.active_session_id.clone();
- if let Err(error) = self.kill_child(&active_session_id, reason).await {
- if unknown_session(&error).is_some() {
- // Already gone: the missing child is a no-op success.
- // The dead worker's file is frozen — the pre-kill walk
- // covered its rows; the registration drops with it.
- self.forget_child_usage(record).await;
- self.children
- .lock()
- .await
- .retain(|candidate| !Arc::ptr_eq(candidate, record));
- continue;
+ if !self.has_retired_usage(record) {
+ {
+ let mut record = record.lock().await;
+ record.closed_by_parent = true;
+ record.notice_delivered = true;
}
- // A failed close keeps the child tracked so the
- // caller can retry.
- close_error.get_or_insert(error);
- continue;
+ self.emit_child_usage(record).await;
+ let active_session_id = record.lock().await.active_session_id.clone();
+ if let Err(error) = self.kill_child(&active_session_id, reason).await {
+ if unknown_session(&error).is_none() {
+ close_error.get_or_insert(error);
+ continue;
+ }
+ }
+ self.retain_retired_usage(record, super::RetirementIntent::Close);
+ self.emit_child_usage(record).await;
+ }
+ let usage_persisted = self.forget_child_usage(record).await;
+ self.finish_retirement(record).await;
+ if usage_persisted {
+ self.remove_retired_usage(record);
+ } else {
+ close_error.get_or_insert_with(|| {
+ anyhow::anyhow!("RLM child usage remains owed after retirement; retry close")
+ });
}
- // Rows can land between the pre-kill capture and the kill:
- // the post-kill walk is the last observation; the registration
- // drops with the child.
- self.emit_child_usage(record).await;
- self.forget_child_usage(record).await;
- self.children
- .lock()
- .await
- .retain(|candidate| !Arc::ptr_eq(candidate, record));
}
// The walk changed the registry: wake a parked barrier (a closed
// child is settled work, settled here by its removal).
@@ -209,6 +236,122 @@ impl SupervisorChildSessionsInner {
}
}
+ /// Complete a confirmed retirement once, including an interrupted caller's bookkeeping.
+ pub(super) async fn finish_retirement(&self, record: &Arc<Mutex<ChildRecord>>) {
+ if record.lock().await.retirement_finished {
+ return;
+ }
+ let intent = self
+ .retirement_intent(record)
+ .expect("confirmed retirement receipt");
+ let deleted = matches!(intent, super::RetirementIntent::Delete { .. });
+ let cancelled_by_delete = matches!(
+ intent,
+ super::RetirementIntent::Delete { was_running: true }
+ );
+ let child_id = {
+ let mut row = record.lock().await;
+ row.closed_by_parent = true;
+ match intent {
+ super::RetirementIntent::Delete { was_running } => {
+ if was_running {
+ row.settled_status = Some("cancelled");
+ row.error = Some("Deleted by parent orchestrator".into());
+ }
+ self.remember_deleted_child(&row);
+ }
+ super::RetirementIntent::Close => {
+ row.notice_delivered = true;
+ }
+ }
+ row.rlm_child_id.clone()
+ };
+ if deleted {
+ let sink = self
+ .child_update_sink
+ .lock()
+ .unwrap_or_else(std::sync::PoisonError::into_inner)
+ .clone();
+ let row = {
+ let mut row = record.lock().await;
+ if row.retirement_roster_emitted {
+ None
+ } else {
+ row.retirement_roster_emitted = true;
+ Some(if cancelled_by_delete {
+ row.snapshot_value()
+ } else {
+ json!({
+ "id": row.rlm_child_id, "activeSessionId": row.active_session_id,
+ "sessionName": row.session_name, "label": row.session_name,
+ "status": "cancelled", "sessionDir": row.session_dir,
+ "error": "Deleted by parent orchestrator"
+ })
+ })
+ }
+ };
+ if let (Some(sink), Some(row)) = (sink, row) {
+ sink(row);
+ }
+ }
+ self.children
+ .lock()
+ .await
+ .retain(|held| !Arc::ptr_eq(held, record));
+ if deleted {
+ let notify = self
+ .delete_notifier
+ .lock()
+ .expect("delete notifier lock")
+ .clone();
+ let claimed = {
+ let mut row = record.lock().await;
+ let claimed = !row.retirement_notified;
+ row.retirement_notified = true;
+ claimed
+ };
+ if let Some(notify) = notify.filter(|_| claimed) {
+ notify(&child_id);
+ }
+ let notice = {
+ let mut row = record.lock().await;
+ let claimed = cancelled_by_delete && !row.notice_delivered;
+ row.notice_delivered = true;
+ claimed.then(|| super::RlmChildTerminalNotice::Cancelled {
+ child_id: row.rlm_child_id.clone(),
+ session_name: row.session_name.clone(),
+ reason: Some("Deleted by parent orchestrator".into()),
+ })
+ };
+ if let Some(notice) = notice {
+ self.deliver_terminal_notice(super::create_rlm_child_terminal_notice(
+ &notice,
+ super::now_ms(),
+ ))
+ .await;
+ }
+ }
+ self.refresh_running().await;
+ let notify_hook = {
+ let mut row = record.lock().await;
+ row.settled = true;
+ row.retirement_finished = true;
+ deleted
+ };
+ // No await follows the completion claim: cancellation cannot strand its notifications.
+ self.settle_notify.notify_waiters();
+ if notify_hook {
+ let hook = self
+ .settle_hook
+ .lock()
+ .unwrap_or_else(std::sync::PoisonError::into_inner)
+ .clone();
+ if let Some(hook) = hook {
+ std::thread::spawn(move || hook());
+ }
+ }
+ }
+
/// The one record matching a selector, or `None` (the send hook's
/// silent miss: a delivered message may target a non-child family member).
pub(super) async fn find_record(&self, target: &str) -> Option<Arc<Mutex<ChildRecord>>> {
@@ -381,8 +524,13 @@ fn ledger_child_records(
prompt_admitted: true,
error: None,
closed_by_parent: false,
+ retirement_finished: false,
+ retirement_notified: false,
+ retirement_roster_emitted: false,
+ usage_registration_forgotten: false,
session_file: Some(edge.child),
attributed_rows: None,
+ pending_usage_report: None,
usage_watch_live: false,
usage_rearm: false,
emit_lock: std::sync::Arc::new(tokio::sync::Mutex::new(())),
diff --git a/crates/pa-daemon/src/rlm_children/usage.rs b/crates/pa-daemon/src/rlm_children/usage.rs
index fe766e5203..3567919f4f 100644
--- a/crates/pa-daemon/src/rlm_children/usage.rs
+++ b/crates/pa-daemon/src/rlm_children/usage.rs
@@ -9,13 +9,14 @@ use super::{
impl SupervisorChildSessionsInner {
/// Deliver the child's unattributed usage rows as one per-origin
- /// report; the cursor advances past every parsed row (no
- /// double-billing); without a sink nothing is read, a torn trailing
- /// line lands on the next read.
- pub(super) async fn emit_child_usage(&self, record: &Arc<Mutex<ChildRecord>>) {
+ /// report; the cursor advances only past the rows the sink persisted
+ /// (no double-billing; a failed delivery re-reads the same rows);
+ /// without a sink nothing is read, a torn trailing line lands on the
+ /// next read. Returns false if the source could not be observed or its report is still owed.
+ pub(super) async fn emit_child_usage(&self, record: &Arc<Mutex<ChildRecord>>) -> bool {
let sink = self.usage_sink.lock().expect("usage sink lock").clone();
let Some(sink) = sink else {
- return;
+ return true;
};
// One emission at a time per child — an interleaved re-read
// would double-bill, and the emission lock spans the whole flow
@@ -25,63 +26,206 @@ impl SupervisorChildSessionsInner {
// would deadlock the two.
let emit_lock = record.lock().await.emit_lock.clone();
let emit_guard = emit_lock.lock().await;
- let (rlm_child_id, session_file, from) = {
- let record_guard = record.lock().await;
- (
- record_guard.rlm_child_id.clone(),
- record_guard
- .session_file
- .clone()
- .filter(|path| !path.is_empty()),
- record_guard.attributed_rows,
- )
- };
- let Some(session_file) = session_file else {
- return;
- };
- // A reseeded row nothing has observed yet: nothing to bill until a delivery primes it.
- let Some(from) = from else {
- return;
- };
- // The open+parse is a blocking read of a file that can reach tens
- // of megabytes: run it on the blocking pool, never the async
- // worker (a slow file read must not stall unrelated tasks on the
- // runtime).
- let path = PathBuf::from(session_file);
- let joined =
- tokio::task::spawn_blocking(move || crate::session_store::SessionFile::open(&path))
+ if record.lock().await.usage_registration_forgotten {
+ return true;
+ }
+ let mut drain_fresh_interval = record.lock().await.pending_usage_report.is_some();
+ let persisted = loop {
+ let pending = record.lock().await.pending_usage_report.clone();
+ let (report, next) = if let Some(pending) = pending {
+ pending
+ } else {
+ let (rlm_child_id, session_file, from) = {
+ let record_guard = record.lock().await;
+ (
+ record_guard.rlm_child_id.clone(),
+ record_guard
+ .session_file
+ .clone()
+ .filter(|path| !path.is_empty()),
+ record_guard.attributed_rows,
+ )
+ };
+ let Some(session_file) = session_file else {
+ return true;
+ };
+ let Some(from) = from else {
+ return true;
+ };
+ let path = PathBuf::from(session_file);
+ let joined = tokio::task::spawn_blocking(move || {
+ crate::session_store::SessionFile::open(&path)
+ })
.await
.ok();
- let Some(Ok(store)) = joined else {
- // Same failure contract as before: a torn or unreadable file
- // leaves the cursor untouched — the next observation retries.
- return;
+ let Some(Ok(store)) = joined else {
+ return false;
+ };
+ let (batches, next) =
+ crate::rlm_child_usage::child_usage_batches(store.entries(), from);
+ if batches.is_empty() {
+ record.lock().await.attributed_rows = Some(next);
+ return true;
+ }
+ let report = RlmChildUsageReport {
+ report_id: uuid::Uuid::new_v4().to_string(),
+ rlm_child_id,
+ batches,
+ };
+ // Install before awaiting the sink. A canceled future leaves the immutable
+ // interval available; newer child rows wait behind this exact report.
+ record.lock().await.pending_usage_report = Some((report.clone(), next));
+ (report, next)
+ };
+ let persisted = sink.record(report).await;
+ if persisted {
+ let mut record = record.lock().await;
+ record.attributed_rows = Some(next);
+ record.pending_usage_report = None;
+ drop(record);
+ if drain_fresh_interval {
+ drain_fresh_interval = false;
+ continue;
+ }
+ }
+ break persisted;
};
- let (batches, next) = crate::rlm_child_usage::child_usage_batches(store.entries(), from);
- {
- let mut record_guard = record.lock().await;
- record_guard.attributed_rows = Some(next);
+ drop(emit_guard);
+ persisted
+ }
+
+ /// Replay a failed terminal observation once before its watcher retires.
+ /// This caps observations, not elapsed time; a second failure stays owed to the existing owner.
+ pub(super) async fn emit_final_child_usage(&self, record: &Arc<Mutex<ChildRecord>>) {
+ if self.emit_child_usage(record).await {
+ return;
}
- if batches.is_empty() {
+ if record.lock().await.closed_by_parent {
return;
}
- sink.record(RlmChildUsageReport {
- rlm_child_id,
- batches,
- })
- .await;
- drop(emit_guard);
+ self.emit_child_usage(record).await;
}
- /// Drop one child's attribution registration after its final
- /// observation, so sequential children do not accumulate registrations in the producer.
- pub(super) async fn forget_child_usage(&self, record: &Arc<Mutex<ChildRecord>>) {
+ /// Drop registration only after the final report persisted; return whether it is cleared.
+ /// A failed bounded replay leaves both the report and its registration owed.
+ pub(super) async fn forget_child_usage(&self, record: &Arc<Mutex<ChildRecord>>) -> bool {
+ if record.lock().await.usage_registration_forgotten {
+ return true;
+ }
let sink = self.usage_sink.lock().expect("usage sink lock").clone();
let Some(sink) = sink else {
- return;
+ return true;
};
- let rlm_child_id = record.lock().await.rlm_child_id.clone();
- sink.forget(&rlm_child_id).await;
+ // A failed final delivery keeps its report owed, and no observation
+ // follows this call: replay it once before the registration drops,
+ // or the forget orphans the retained report.
+ // Also observe the frozen child's tail if a prior caller was canceled before capturing it.
+ if !self.emit_child_usage(record).await {
+ return false;
+ }
+ let emit_lock = record.lock().await.emit_lock.clone();
+ let _emit_guard = emit_lock.lock().await;
+ let mut row = record.lock().await;
+ if row.usage_registration_forgotten {
+ return true;
+ }
+ if row.pending_usage_report.is_some() {
+ return false;
+ }
+ // Forget touches only the producer registry and does not re-enter the child record.
+ // Serialize its acknowledgment with final emission and record the receipt without an await.
+ sink.forget(&row.rlm_child_id).await;
+ row.usage_registration_forgotten = true;
+ true
+ }
+
+ /// Keep a confirmed-dead child's final accounting reachable after roster retirement.
+ pub(super) fn retain_retired_usage(
+ &self,
+ record: &Arc<Mutex<ChildRecord>>,
+ intent: super::RetirementIntent,
+ ) {
+ let mut retired = self
+ .retired_usage
+ .lock()
+ .unwrap_or_else(std::sync::PoisonError::into_inner);
+ if !retired.iter().any(|held| Arc::ptr_eq(&held.record, record)) {
+ retired.push(super::RetiredChild {
+ record: Arc::clone(record),
+ intent,
+ });
+ }
+ }
+
+ pub(super) fn retirement_intent(
+ &self,
+ record: &Arc<Mutex<ChildRecord>>,
+ ) -> Option<super::RetirementIntent> {
+ self.retired_usage
+ .lock()
+ .unwrap_or_else(std::sync::PoisonError::into_inner)
+ .iter()
+ .find(|held| Arc::ptr_eq(&held.record, record))
+ .map(|held| held.intent)
+ }
+
+ pub(super) fn has_retired_usage(&self, record: &Arc<Mutex<ChildRecord>>) -> bool {
+ self.retirement_intent(record).is_some()
+ }
+
+ pub(super) fn remove_retired_usage(&self, record: &Arc<Mutex<ChildRecord>>) {
+ self.retired_usage
+ .lock()
+ .unwrap_or_else(std::sync::PoisonError::into_inner)
+ .retain(|held| !Arc::ptr_eq(&held.record, record));
+ }
+
+ /// One bounded replay per matching receipt; never repeat a confirmed worker kill.
+ pub(super) async fn replay_retired_usage(&self, target: Option<&str>) -> bool {
+ let retired = self
+ .retired_usage
+ .lock()
+ .unwrap_or_else(std::sync::PoisonError::into_inner)
+ .clone();
+ let mut persisted = true;
+ for retirement in retired {
+ let record = retirement.record;
+ let matches = {
+ let row = record.lock().await;
+ target.is_none_or(|target| row.matches(target))
+ };
+ if !matches {
+ continue;
+ }
+ let rename_lock = record.lock().await.rename_lock.clone();
+ let _guard = rename_lock.lock().await;
+ if !self
+ .retired_usage
+ .lock()
+ .unwrap_or_else(std::sync::PoisonError::into_inner)
+ .iter()
+ .any(|held| Arc::ptr_eq(&held.record, &record))
+ {
+ continue;
+ }
+ // An interrupted caller still owns lifecycle completion through the roster.
+ if self
+ .children
+ .lock()
+ .await
+ .iter()
+ .any(|held| Arc::ptr_eq(held, &record))
+ {
+ continue;
+ }
+ let usage_persisted = self.forget_child_usage(&record).await;
+ self.finish_retirement(&record).await;
+ if usage_persisted {
+ self.remove_retired_usage(&record);
+ }
+ persisted &= usage_persisted;
+ }
+ persisted
}
/// Start the follow-up usage watcher for a settled child that is
@@ -154,7 +298,7 @@ impl SupervisorChildSessionsInner {
if unreachable_polls >= WATCH_MAX_UNREACHABLE_POLLS {
// A dead child keeps whatever rows its file
// already holds; capture them, then stop.
- self.emit_child_usage(&record).await;
+ self.emit_final_child_usage(&record).await;
record.lock().await.usage_watch_live = false;
return;
}
@@ -203,7 +347,7 @@ impl SupervisorChildSessionsInner {
// side of a socket).
tokio::time::sleep(Duration::from_millis(WATCH_SETTLE_GRACE_MS)).await;
if matches!(self.child_busy(&active_session_id).await, Ok(false)) {
- self.emit_child_usage(&record).await;
+ self.emit_final_child_usage(&record).await;
break;
}
}
@@ -218,7 +362,7 @@ impl SupervisorChildSessionsInner {
// A dead child keeps whatever rows its
// file already holds; capture them, then
// stop.
- self.emit_child_usage(&record).await;
+ self.emit_final_child_usage(&record).await;
break;
}
}
@@ -229,7 +373,7 @@ impl SupervisorChildSessionsInner {
// The turn never showed busy: it completed between two
// polls (bill its rows) or never started (the cursor walk
// is a no-op).
- self.emit_child_usage(&record).await;
+ self.emit_final_child_usage(&record).await;
}
// The tail: a delivery that arrived while this watcher was live
// re-arms it; otherwise it retires. The flag read and clear
diff --git a/crates/pa-daemon/src/rlm_children/usage_emit_tests.rs b/crates/pa-daemon/src/rlm_children/usage_emit_tests.rs
index d5bd9659a8..a98985b8b1 100644
--- a/crates/pa-daemon/src/rlm_children/usage_emit_tests.rs
+++ b/crates/pa-daemon/src/rlm_children/usage_emit_tests.rs
@@ -2,26 +2,41 @@ use super::*;
use pa_core::session_engine::rlm_usage::RlmChildUsageReport;
use std::sync::Arc;
-/// A capturing sink: reports land in a shared vector for assertions.
+/// A capturing sink: reports land in a shared vector for assertions;
+/// `fail` flips `record` into a failing no-op.
#[derive(Default)]
-struct CapturingSink(std::sync::Mutex<Vec<RlmChildUsageReport>>);
+struct CapturingSink {
+ reports: std::sync::Mutex<Vec<RlmChildUsageReport>>,
+ fail: std::sync::atomic::AtomicBool,
+ forgotten: std::sync::Mutex<Vec<String>>,
+ attempts: std::sync::atomic::AtomicUsize,
+}
impl pa_core::session_engine::rlm_usage::RlmChildUsageSink for CapturingSink {
fn record(
&self,
report: RlmChildUsageReport,
- ) -> std::pin::Pin<Box<dyn std::future::Future<Output = ()> + Send + '_>> {
- let reports = &self.0;
+ ) -> std::pin::Pin<Box<dyn std::future::Future<Output = bool> + Send + '_>> {
+ let (reports, fail) = (&self.reports, &self.fail);
Box::pin(async move {
+ self.attempts
+ .fetch_add(1, std::sync::atomic::Ordering::Relaxed);
+ if fail.load(std::sync::atomic::Ordering::Relaxed) {
+ return false;
+ }
reports.lock().expect("reports lock").push(report);
+ true
})
}
fn forget(
&self,
- _rlm_child_id: &str,
+ rlm_child_id: &str,
) -> std::pin::Pin<Box<dyn std::future::Future<Output = ()> + Send + '_>> {
- Box::pin(async move {})
+ let child_id = rlm_child_id.to_owned();
+ Box::pin(async move {
+ self.forgotten.lock().unwrap().push(child_id);
+ })
}
}
@@ -46,8 +61,13 @@ fn record_with_file(child_id: &str, session_file: &Path) -> Arc<Mutex<ChildRecor
prompt_admitted: true,
error: None,
closed_by_parent: false,
+ retirement_finished: false,
+ retirement_notified: false,
+ retirement_roster_emitted: false,
+ usage_registration_forgotten: false,
session_file: Some(session_file.display().to_string()),
attributed_rows: Some(0),
+ pending_usage_report: None,
usage_watch_live: false,
usage_rearm: false,
emit_lock: std::sync::Arc::new(tokio::sync::Mutex::new(())),
@@ -97,7 +117,7 @@ async fn emit_reads_once_and_advances_the_cursor() {
let record = record_with_file("sub-emit1", &file);
sessions.inner.emit_child_usage(&record).await;
- let reports = sink.0.lock().expect("reports lock").clone();
+ let reports = sink.reports.lock().expect("reports lock").clone();
assert_eq!(reports.len(), 1);
assert_eq!(reports[0].rlm_child_id, "sub-emit1");
let [(origin, usage)] = reports[0].batches[..] else {
@@ -111,11 +131,68 @@ async fn emit_reads_once_and_advances_the_cursor() {
assert!(consumed.is_some_and(|rows| rows > 0));
sessions.inner.emit_child_usage(&record).await;
- let reports = sink.0.lock().expect("reports lock").clone();
+ let reports = sink.reports.lock().expect("reports lock").clone();
assert_eq!(reports.len(), 1);
assert_eq!(record.lock().await.attributed_rows, consumed);
}
+/// A failed delivery keeps the cursor so the next emission re-delivers the
+/// same rows; once the sink persists, the report lands exactly once.
+#[tokio::test]
+async fn emit_keeps_the_cursor_until_the_sink_persists() {
+ let tmp = tempfile::tempdir().unwrap();
+ let file = child_file(tmp.path());
+ let sessions = registry(Path::new("/agent"));
+ let sink = Arc::new(CapturingSink::default());
+ sessions.set_usage_sink(sink.clone());
+ let record = record_with_file("sub-emit5", &file);
+
+ sink.fail.store(true, std::sync::atomic::Ordering::Relaxed);
+ sessions.inner.emit_child_usage(&record).await;
+ assert_eq!(record.lock().await.attributed_rows, Some(0));
+ assert_eq!(sink.reports.lock().expect("reports lock").len(), 0);
+
+ sink.fail.store(false, std::sync::atomic::Ordering::Relaxed);
+ sessions.inner.emit_child_usage(&record).await;
+ let reports = sink.reports.lock().expect("reports lock").clone();
+ assert_eq!(reports.len(), 1);
+ assert_eq!(reports[0].rlm_child_id, "sub-emit5");
+ assert!(record
+ .lock()
+ .await
+ .attributed_rows
+ .is_some_and(|rows| rows > 0));
+}
+
+/// A failed final observation keeps its report owed; the follow-up forget
+/// replays it before the registration drops instead of orphaning it.
+#[tokio::test]
+async fn forget_replays_the_retained_report_before_unregistering() {
+ let tmp = tempfile::tempdir().unwrap();
+ let file = child_file(tmp.path());
+ let sessions = registry(Path::new("/agent"));
+ let sink = Arc::new(CapturingSink::default());
+ sessions.set_usage_sink(sink.clone());
+ let record = record_with_file("sub-forget1", &file);
+
+ sink.fail.store(true, std::sync::atomic::Ordering::Relaxed);
+ sessions.inner.emit_child_usage(&record).await;
+ assert!(record.lock().await.pending_usage_report.is_some());
+ assert_eq!(record.lock().await.attributed_rows, Some(0));
+
+ sink.fail.store(false, std::sync::atomic::Ordering::Relaxed);
+ sessions.inner.forget_child_usage(&record).await;
+ let reports = sink.reports.lock().expect("reports lock").clone();
+ assert_eq!(reports.len(), 1);
+ assert_eq!(reports[0].rlm_child_id, "sub-forget1");
+ assert!(record.lock().await.pending_usage_report.is_none());
+ assert!(record
+ .lock()
+ .await
+ .attributed_rows
+ .is_some_and(|rows| rows > 0));
+}
+
#[tokio::test]
async fn emit_without_a_sink_consumes_nothing() {
let tmp = tempfile::tempdir().unwrap();
@@ -135,12 +212,12 @@ async fn emit_tolerates_missing_and_absent_files() {
sessions.set_usage_sink(sink.clone());
let missing = record_with_file("sub-emit3", &tmp.path().join("absent.jsonl"));
sessions.inner.emit_child_usage(&missing).await;
- assert_eq!(sink.0.lock().expect("reports lock").len(), 0);
+ assert_eq!(sink.reports.lock().expect("reports lock").len(), 0);
let no_file = record_with_file("sub-emit4", &tmp.path().join("x.jsonl"));
no_file.lock().await.session_file = None;
sessions.inner.emit_child_usage(&no_file).await;
- assert_eq!(sink.0.lock().expect("reports lock").len(), 0);
+ assert_eq!(sink.reports.lock().expect("reports lock").len(), 0);
}
/// The reseed lists the parent's live ledger children as settled rows
@@ -274,7 +351,7 @@ async fn reseed_lists_live_ledger_children_settled_and_bills_only_after_a_delive
.expect("the reseeded row");
// Nothing is owed: the pre-restart history never re-bills.
sessions.inner.emit_child_usage(&record).await;
- assert_eq!(sink.0.lock().expect("reports lock").len(), 0);
+ assert_eq!(sink.reports.lock().expect("reports lock").len(), 0);
// A delivery primes the lazy cursor at the tail; the delivered turn
// appends its row; the next emit bills only that row.
SupervisorChildSessionsInner::arm_usage_watch(&sessions.inner, &record).await;
@@ -286,10 +363,967 @@ async fn reseed_lists_live_ledger_children_settled_and_bills_only_after_a_delive
content.push('\n');
std::fs::write(Path::new(&child_one), content).expect("append the row");
sessions.inner.emit_child_usage(&record).await;
- let reports = sink.0.lock().expect("reports lock").clone();
+ let reports = sink.reports.lock().expect("reports lock").clone();
assert_eq!(reports.len(), 1);
let [(_, usage)] = reports[0].batches[..] else {
panic!("one batch: {:?}", reports[0].batches);
};
assert_eq!((usage.input, usage.output), (10, 5));
}
+
+/// Cancellation after delivery starts retains the exact interval; rows arriving
+/// behind it are drained once before the caller's final forget.
+#[tokio::test]
+async fn canceled_interval_keeps_identity_and_drains_new_rows_before_forget() {
+ use std::io::Write;
+ struct CancelFirstSink {
+ attempts: std::sync::Mutex<Vec<RlmChildUsageReport>>,
+ started: tokio::sync::Notify,
+ forgotten: std::sync::atomic::AtomicBool,
+ }
+ impl pa_core::session_engine::rlm_usage::RlmChildUsageSink for CancelFirstSink {
+ fn record(
+ &self,
+ report: RlmChildUsageReport,
+ ) -> std::pin::Pin<Box<dyn std::future::Future<Output = bool> + Send + '_>> {
+ Box::pin(async move {
+ let first = {
+ let mut attempts = self.attempts.lock().unwrap();
+ let first = attempts.is_empty();
+ attempts.push(report);
+ first
+ };
+ if first {
+ self.started.notify_one();
+ std::future::pending::<()>().await;
+ }
+ true
+ })
+ }
+ fn forget(
+ &self,
+ _id: &str,
+ ) -> std::pin::Pin<Box<dyn std::future::Future<Output = ()> + Send + '_>> {
+ Box::pin(async move {
+ self.forgotten
+ .store(true, std::sync::atomic::Ordering::SeqCst);
+ })
+ }
+ }
+ let tmp = tempfile::tempdir().unwrap();
+ let file = child_file(tmp.path());
+ let sessions = registry(Path::new("/agent"));
+ let sink = Arc::new(CancelFirstSink {
+ attempts: std::sync::Mutex::new(vec![]),
+ started: tokio::sync::Notify::new(),
+ forgotten: std::sync::atomic::AtomicBool::new(false),
+ });
+ sessions.set_usage_sink(sink.clone());
+ let record = record_with_file("sub-canceled", &file);
+ let inner = sessions.inner.clone();
+ let emitted_record = record.clone();
+ let emit = tokio::spawn(async move {
+ inner.emit_child_usage(&emitted_record).await;
+ });
+ tokio::time::timeout(Duration::from_secs(5), sink.started.notified())
+ .await
+ .unwrap();
+ emit.abort();
+ assert!(emit.await.unwrap_err().is_cancelled());
+ let pending = record.lock().await.pending_usage_report.clone().unwrap();
+ let mut later: serde_json::Value = std::fs::read_to_string(&file)
+ .unwrap()
+ .lines()
+ .filter_map(|line| serde_json::from_str::<serde_json::Value>(line).ok())
+ .find(|row| row["message"]["role"] == "assistant")
+ .unwrap();
+ later["id"] = serde_json::json!("new-completion");
+ later["parentId"] = serde_json::json!("a0");
+ later["message"]["usage"]["input"] = serde_json::json!(70);
+ let mut append = std::fs::OpenOptions::new()
+ .append(true)
+ .open(&file)
+ .unwrap();
+ writeln!(append, "{}", serde_json::to_string(&later).unwrap()).unwrap();
+ sessions.inner.emit_child_usage(&record).await;
+ sessions.inner.forget_child_usage(&record).await;
+ {
+ let attempts = sink.attempts.lock().unwrap();
+ assert_eq!(
+ attempts.len(),
+ 3,
+ "canceled attempt, same-ID retry, then fresh interval"
+ );
+ assert_eq!(attempts[0].report_id, pending.0.report_id);
+ assert_eq!(attempts[1].report_id, pending.0.report_id);
+ assert_eq!(attempts[0].batches, attempts[1].batches);
+ assert_ne!(attempts[2].report_id, pending.0.report_id);
+ assert_eq!(attempts[2].batches.len(), 1);
+ assert_eq!(attempts[2].batches[0].1.input, 70);
+ assert!(sink.forgotten.load(std::sync::atomic::Ordering::SeqCst));
+ }
+ assert!(record.lock().await.pending_usage_report.is_none());
+ assert_eq!(
+ record.lock().await.attributed_rows,
+ Some(
+ crate::session_store::SessionFile::open(&file)
+ .unwrap()
+ .entries()
+ .len()
+ )
+ );
+}
+
+#[tokio::test]
+async fn failed_final_replay_preserves_report_and_registration_for_a_bounded_retry() {
+ let tmp = tempfile::tempdir().unwrap();
+ let file = child_file(tmp.path());
+ let sessions = registry(tmp.path());
+ let sink = Arc::new(CapturingSink::default());
+ sessions.set_usage_sink(sink.clone());
+ let record = record_with_file("sub-owed-final", &file);
+ sink.fail.store(true, std::sync::atomic::Ordering::Relaxed);
+ sessions.inner.emit_child_usage(&record).await;
+ let frozen = record.lock().await.pending_usage_report.clone().unwrap();
+ sessions.inner.forget_child_usage(&record).await;
+ assert!(
+ sink.forgotten.lock().unwrap().is_empty(),
+ "an owed report must remain registered"
+ );
+ assert_eq!(
+ sink.attempts.load(std::sync::atomic::Ordering::Relaxed),
+ 2,
+ "one initial attempt plus one bounded final replay"
+ );
+ {
+ let held = record.lock().await.pending_usage_report.clone().unwrap();
+ assert_eq!(held.0.report_id, frozen.0.report_id);
+ assert_eq!(held.0.batches, frozen.0.batches);
+ assert_eq!(held.1, frozen.1);
+ }
+ sink.fail.store(false, std::sync::atomic::Ordering::Relaxed);
+ sessions.inner.forget_child_usage(&record).await;
+ {
+ let reports = sink.reports.lock().unwrap();
+ let observed: Vec<_> = reports
+ .iter()
+ .map(|report| {
+ (
+ report.report_id.clone(),
+ report.rlm_child_id.clone(),
+ report.batches.clone(),
+ )
+ })
+ .collect();
+ assert_eq!(
+ observed,
+ vec![(frozen.0.report_id, frozen.0.rlm_child_id, frozen.0.batches)]
+ );
+ }
+ assert_eq!(
+ sink.forgotten.lock().unwrap().as_slice(),
+ &["sub-owed-final"]
+ );
+ assert!(record.lock().await.pending_usage_report.is_none());
+ assert_eq!(sink.attempts.load(std::sync::atomic::Ordering::Relaxed), 3);
+}
+
+/// A real supervisor link whose second kill fails: retirement replay must not resend it.
+async fn retirement_registry(
+ dir: &Path,
+) -> (
+ SupervisorChildSessions,
+ Arc<std::sync::atomic::AtomicUsize>,
+ Arc<std::sync::Mutex<Vec<Value>>>,
+ tokio::task::JoinHandle<()>,
+) {
+ use tokio::io::{AsyncBufReadExt, AsyncWriteExt, BufReader};
+ let socket = dir.join("retirement.sock");
+ let listener = pa_types::platform::transport::bind_transport(&socket)
+ .await
+ .unwrap();
+ let kills = Arc::new(std::sync::atomic::AtomicUsize::new(0));
+ let observed = Arc::clone(&kills);
+ let commands = Arc::new(std::sync::Mutex::new(Vec::new()));
+ let observed_commands = Arc::clone(&commands);
+ let task = tokio::spawn(async move {
+ loop {
+ let Ok(stream) = listener.accept().await else {
+ return;
+ };
+ let (reader, mut writer) = stream.split();
+ let mut reader = BufReader::new(reader);
+ let hello =
+ json!({"type":"daemon_hello", "protocol":crate::protocol::current_protocol_info()});
+ writer
+ .write_all(format!("{hello}\n").as_bytes())
+ .await
+ .unwrap();
+ let mut line = String::new();
+ if reader.read_line(&mut line).await.unwrap() == 0 {
+ continue;
+ }
+ let envelope: Value = serde_json::from_str(&line).unwrap();
+ observed_commands
+ .lock()
+ .unwrap()
+ .push(envelope["command"].clone());
+ let command = envelope["command"]["type"].as_str().unwrap();
+ let id = envelope["id"].as_str().unwrap();
+ let response = if command == "kill"
+ && observed.fetch_add(1, std::sync::atomic::Ordering::Relaxed) > 0
+ {
+ crate::protocol::response_failure(
+ Some(id),
+ command,
+ "Unknown active session: child-live",
+ None,
+ )
+ } else {
+ crate::protocol::response_success(Some(id), command, None)
+ };
+ writer
+ .write_all(format!("{}\n", crate::protocol::response_line(&response)).as_bytes())
+ .await
+ .unwrap();
+ }
+ });
+ let registry = SupervisorChildSessions::new(
+ Arc::new(SupervisorLink::new(socket)),
+ dir.to_path_buf(),
+ "parent-live".into(),
+ Arc::new(crate::model_allowlist::ModelRefusalTelemetry::new(
+ dir.to_path_buf(),
+ /*telemetry_disabled*/ true,
+ )),
+ );
+ (registry, kills, commands, task)
+}
+
+#[tokio::test]
+async fn repeated_delete_replays_retired_usage_without_repeating_lifecycle_work() {
+ let tmp = tempfile::tempdir().unwrap();
+ let file = child_file(tmp.path());
+ let (sessions, kills, _commands, server) = retirement_registry(tmp.path()).await;
+ let sink = Arc::new(CapturingSink::default());
+ sink.fail.store(true, std::sync::atomic::Ordering::Relaxed);
+ sessions.set_usage_sink(sink.clone());
+ let record = record_with_file("sub-retire-delete", &file);
+ record.lock().await.settled_status = Some("done");
+ sessions.inner.children.lock().await.push(record.clone());
+ let settled = Arc::new(std::sync::atomic::AtomicUsize::new(0));
+ let notified = Arc::new(std::sync::atomic::AtomicUsize::new(0));
+ let (settle_tx, mut settle_rx) = tokio::sync::mpsc::unbounded_channel();
+ let observed = settled.clone();
+ sessions.set_settle_hook(Arc::new(move || {
+ observed.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
+ let _ = settle_tx.send(());
+ }));
+ let observed = notified.clone();
+ sessions.set_delete_notifier(Arc::new(move |_| {
+ observed.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
+ }));
+ let first = sessions
+ .delete_subagent("sub-retire-delete".into())
+ .await
+ .unwrap_err();
+ assert!(first.to_string().contains("usage remains owed"));
+ tokio::time::timeout(Duration::from_secs(5), settle_rx.recv())
+ .await
+ .unwrap()
+ .unwrap();
+ assert!(sessions.list_subagents().await.unwrap().is_empty());
+ assert!(!sessions.any_running().await);
+ assert!(sink.forgotten.lock().unwrap().is_empty());
+ let frozen = record.lock().await.pending_usage_report.clone().unwrap().0;
+ let attempts = sink.attempts.load(std::sync::atomic::Ordering::Relaxed);
+ assert!(sessions
+ .delete_subagent("sub-retire-delete".into())
+ .await
+ .unwrap_err()
+ .to_string()
+ .contains("usage remains owed"));
+ assert_eq!(
+ sink.attempts.load(std::sync::atomic::Ordering::Relaxed),
+ attempts + 1
+ );
+ assert!(sink.forgotten.lock().unwrap().is_empty());
+ sink.fail.store(false, std::sync::atomic::Ordering::Relaxed);
+ let attempts = sink.attempts.load(std::sync::atomic::Ordering::Relaxed);
+ assert_eq!(
+ sessions
+ .delete_subagent("sub-retire-delete".into())
+ .await
+ .unwrap()
+ .outcome,
+ Some("deleted")
+ );
+ assert_eq!(
+ sink.attempts.load(std::sync::atomic::Ordering::Relaxed),
+ attempts + 1
+ );
+ {
+ let reports = sink.reports.lock().unwrap();
+ assert_eq!(reports.len(), 1);
+ assert_eq!(
+ (reports[0].report_id.clone(), reports[0].batches.clone()),
+ (frozen.report_id, frozen.batches)
+ );
+ }
+ assert_eq!(
+ sink.forgotten.lock().unwrap().as_slice(),
+ &["sub-retire-delete"]
+ );
+ sessions
+ .delete_subagent("sub-retire-delete".into())
+ .await
+ .unwrap();
+ assert_eq!(
+ (
+ kills.load(std::sync::atomic::Ordering::Relaxed),
+ settled.load(std::sync::atomic::Ordering::Relaxed),
+ notified.load(std::sync::atomic::Ordering::Relaxed)
+ ),
+ (1, 1, 1)
+ );
+ server.abort();
+ let _ = server.await;
+}
+
+#[tokio::test]
+async fn repeated_close_replays_retired_usage_without_a_second_kill() {
+ let tmp = tempfile::tempdir().unwrap();
+ let file = child_file(tmp.path());
+ let (sessions, kills, _commands, server) = retirement_registry(tmp.path()).await;
+ let sink = Arc::new(CapturingSink::default());
+ sink.fail.store(true, std::sync::atomic::Ordering::Relaxed);
+ sessions.set_usage_sink(sink.clone());
+ let record = record_with_file("sub-retire-close", &file);
+ sessions.inner.children.lock().await.push(record.clone());
+ let first = sessions
+ .close_children(ChildCloseReason::Shutdown)
+ .await
+ .unwrap_err();
+ assert!(first.to_string().contains("usage remains owed"));
+ assert!(sessions.list_subagents().await.unwrap().is_empty());
+ assert!(!sessions.any_running().await);
+ assert!(sink.forgotten.lock().unwrap().is_empty());
+ let frozen = record.lock().await.pending_usage_report.clone().unwrap().0;
+ let attempts = sink.attempts.load(std::sync::atomic::Ordering::Relaxed);
+ assert!(sessions
+ .close_children(ChildCloseReason::Shutdown)
+ .await
+ .unwrap_err()
+ .to_string()
+ .contains("usage remains owed"));
+ assert_eq!(
+ sink.attempts.load(std::sync::atomic::Ordering::Relaxed),
+ attempts + 1
+ );
+ assert!(sink.forgotten.lock().unwrap().is_empty());
+ sink.fail.store(false, std::sync::atomic::Ordering::Relaxed);
+ let attempts = sink.attempts.load(std::sync::atomic::Ordering::Relaxed);
+ sessions
+ .close_children(ChildCloseReason::Shutdown)
+ .await
+ .unwrap();
+ assert_eq!(
+ sink.attempts.load(std::sync::atomic::Ordering::Relaxed),
+ attempts + 1
+ );
+ {
+ let reports = sink.reports.lock().unwrap();
+ assert_eq!(reports.len(), 1);
+ assert_eq!(
+ (reports[0].report_id.clone(), reports[0].batches.clone()),
+ (frozen.report_id, frozen.batches)
+ );
+ }
+ assert_eq!(
+ sink.forgotten.lock().unwrap().as_slice(),
+ &["sub-retire-close"]
+ );
+ sessions
+ .close_children(ChildCloseReason::Shutdown)
+ .await
+ .unwrap();
+ assert_eq!(kills.load(std::sync::atomic::Ordering::Relaxed), 1);
+ server.abort();
+ let _ = server.await;
+}
+
+struct AckPausedSink {
+ capture: CapturingSink,
+ kills: Arc<std::sync::atomic::AtomicUsize>,
+ after_ack: tokio::sync::Notify,
+ release: tokio::sync::Notify,
+ recovered: std::sync::atomic::AtomicBool,
+}
+
+impl RlmChildUsageSink for AckPausedSink {
+ fn record(
+ &self,
+ report: RlmChildUsageReport,
+ ) -> std::pin::Pin<Box<dyn std::future::Future<Output = bool> + Send + '_>> {
+ Box::pin(async move {
+ if self.kills.load(std::sync::atomic::Ordering::Relaxed) > 0
+ && !self.recovered.load(std::sync::atomic::Ordering::Relaxed)
+ {
+ self.after_ack.notify_one();
+ self.release.notified().await;
+ }
+ self.capture.record(report).await
+ })
+ }
+ fn forget(
+ &self,
+ id: &str,
+ ) -> std::pin::Pin<Box<dyn std::future::Future<Output = ()> + Send + '_>> {
+ self.capture.forget(id)
+ }
+}
+
+#[derive(Clone, Copy)]
+enum RetirementCaller {
+ Delete,
+ Close,
+ InactiveAfterDelete,
+}
+
+async fn cancellation_after_ack_preserves_retirement(caller: RetirementCaller) {
+ let tmp = tempfile::tempdir().unwrap();
+ let file = child_file(tmp.path());
+ let (sessions, kills, commands, server) = retirement_registry(tmp.path()).await;
+ let sink = Arc::new(AckPausedSink {
+ capture: CapturingSink::default(),
+ kills: kills.clone(),
+ after_ack: tokio::sync::Notify::new(),
+ release: tokio::sync::Notify::new(),
+ recovered: std::sync::atomic::AtomicBool::new(false),
+ });
+ sink.capture
+ .fail
+ .store(true, std::sync::atomic::Ordering::Relaxed);
+ sessions.set_usage_sink(sink.clone());
+ let record = record_with_file("sub-cancel-after-ack", &file);
+ let notifications = Arc::new(std::sync::Mutex::new(Vec::new()));
+ let observed = notifications.clone();
+ sessions.set_delete_notifier(Arc::new(move |id| {
+ observed.lock().unwrap().push(id.to_string());
+ }));
+ let roster_events = Arc::new(std::sync::Mutex::new(Vec::new()));
+ let observed = roster_events.clone();
+ sessions.set_child_update_sink(Arc::new(move |row| observed.lock().unwrap().push(row)));
+ let (settle_tx, mut settle_rx) = tokio::sync::mpsc::unbounded_channel();
+ sessions.set_settle_hook(Arc::new(move || {
+ let _ = settle_tx.send(());
+ }));
+ sessions.inner.children.lock().await.push(record.clone());
+ let owned = sessions.clone();
+ let retirement = tokio::spawn(async move {
+ match caller {
+ RetirementCaller::Delete | RetirementCaller::InactiveAfterDelete => owned
+ .delete_subagent("sub-cancel-after-ack".into())
+ .await
+ .map(|_| ()),
+ RetirementCaller::Close => owned.close_children(ChildCloseReason::Shutdown).await,
+ }
+ });
+ tokio::time::timeout(Duration::from_secs(5), sink.after_ack.notified())
+ .await
+ .unwrap();
+ retirement.abort();
+ assert!(retirement.await.unwrap_err().is_cancelled());
+ assert_eq!(kills.load(std::sync::atomic::Ordering::Relaxed), 1);
+ let frozen = record.lock().await.pending_usage_report.clone().unwrap().0;
+ assert!(sink.capture.forgotten.lock().unwrap().is_empty());
+ sink.capture
+ .fail
+ .store(false, std::sync::atomic::Ordering::Relaxed);
+ sink.recovered
+ .store(true, std::sync::atomic::Ordering::Relaxed);
+ match caller {
+ RetirementCaller::Delete => {
+ sessions
+ .delete_subagent("sub-cancel-after-ack".into())
+ .await
+ .unwrap();
+ }
+ RetirementCaller::InactiveAfterDelete => {
+ assert_eq!(
+ sessions
+ .delete_inactive_subagent("sub-cancel-after-ack")
+ .await
+ .unwrap(),
+ "deleted"
+ );
+ }
+ RetirementCaller::Close => {
+ sessions
+ .close_children(ChildCloseReason::Shutdown)
+ .await
+ .unwrap();
+ }
+ }
+ assert!(sessions.list_subagents().await.unwrap().is_empty());
+ assert!(!sessions.any_running().await);
+ assert_eq!(kills.load(std::sync::atomic::Ordering::Relaxed), 1);
+ match caller {
+ RetirementCaller::Delete | RetirementCaller::InactiveAfterDelete => {
+ tokio::time::timeout(Duration::from_secs(5), settle_rx.recv())
+ .await
+ .unwrap()
+ .unwrap();
+ assert_eq!(
+ notifications.lock().unwrap().as_slice(),
+ &["sub-cancel-after-ack"]
+ );
+ let collected = sessions
+ .collect(vec!["sub-cancel-after-ack".into()], 0)
+ .await
+ .unwrap();
+ assert_eq!(
+ collected
+ .iter()
+ .map(|row| (
+ row.rlm_child_id.clone(),
+ row.status,
+ row.settled,
+ row.error.clone()
+ ))
+ .collect::<Vec<_>>(),
+ vec![(
+ "sub-cancel-after-ack".into(),
+ "cancelled",
+ true,
+ Some("Deleted by parent orchestrator".into())
+ )]
+ );
+ {
+ let events = roster_events.lock().unwrap();
+ assert_eq!(events.len(), 1);
+ let mut observed = events[0].clone();
+ observed.as_object_mut().unwrap().remove("durationMs");
+ assert_eq!(
+ observed,
+ json!({"id":"sub-cancel-after-ack", "activeSessionId":"child-live", "sessionName":"child", "label":"child", "status":"cancelled", "sessionDir":"", "error":"Deleted by parent orchestrator"})
+ );
+ }
+ {
+ let traffic = commands.lock().unwrap();
+ assert_eq!(
+ traffic
+ .iter()
+ .filter(|row| row["type"] == "follow_up")
+ .count(),
+ 1
+ );
+ }
+ sessions
+ .delete_subagent("sub-cancel-after-ack".into())
+ .await
+ .unwrap();
+ assert!(settle_rx.try_recv().is_err());
+ assert_eq!(notifications.lock().unwrap().len(), 1);
+ }
+ RetirementCaller::Close => {
+ assert!(notifications.lock().unwrap().is_empty());
+ assert!(roster_events.lock().unwrap().is_empty());
+ assert!(settle_rx.try_recv().is_err());
+ assert_eq!(
+ commands
+ .lock()
+ .unwrap()
+ .iter()
+ .filter(|row| row["type"] == "follow_up")
+ .count(),
+ 0
+ );
+ }
+ }
+ {
+ let reports = sink.capture.reports.lock().unwrap();
+ assert_eq!(reports.len(), 1);
+ assert_eq!(
+ (reports[0].report_id.clone(), reports[0].batches.clone()),
+ (frozen.report_id, frozen.batches)
+ );
+ }
+ assert_eq!(
+ sink.capture.forgotten.lock().unwrap().as_slice(),
+ &["sub-cancel-after-ack"]
+ );
+ server.abort();
+ let _ = server.await;
+}
+
+#[tokio::test]
+async fn delete_canceled_after_kill_ack_retries_without_a_second_kill() {
+ cancellation_after_ack_preserves_retirement(RetirementCaller::Delete).await;
+}
+
+#[tokio::test]
+async fn close_canceled_after_kill_ack_retries_without_a_second_kill() {
+ cancellation_after_ack_preserves_retirement(RetirementCaller::Close).await;
+}
+
+#[tokio::test]
+async fn concurrent_close_replays_debt_after_the_first_caller_retires_the_roster() {
+ let tmp = tempfile::tempdir().unwrap();
+ let file = child_file(tmp.path());
+ let (sessions, kills, _commands, server) = retirement_registry(tmp.path()).await;
+ let sink = Arc::new(AckPausedSink {
+ capture: CapturingSink::default(),
+ kills: kills.clone(),
+ after_ack: tokio::sync::Notify::new(),
+ release: tokio::sync::Notify::new(),
+ recovered: std::sync::atomic::AtomicBool::new(false),
+ });
+ sink.capture
+ .fail
+ .store(true, std::sync::atomic::Ordering::Relaxed);
+ sessions.set_usage_sink(sink.clone());
+ let record = record_with_file("sub-concurrent-close", &file);
+ sessions.inner.children.lock().await.push(record.clone());
+ let owned = sessions.clone();
+ let first = tokio::spawn(async move { owned.close_children(ChildCloseReason::Shutdown).await });
+ tokio::time::timeout(Duration::from_secs(5), sink.after_ack.notified())
+ .await
+ .unwrap();
+ let frozen = record.lock().await.pending_usage_report.clone().unwrap().0;
+ let mut retry = Box::pin(sessions.close_children(ChildCloseReason::Shutdown));
+ // Drive the actual second caller to its first wait while the first owns rename_lock.
+ std::future::poll_fn(|cx| {
+ assert!(std::future::Future::poll(retry.as_mut(), cx).is_pending());
+ std::task::Poll::Ready(())
+ })
+ .await;
+ sink.recovered
+ .store(true, std::sync::atomic::Ordering::Relaxed);
+ sink.release.notify_one();
+ assert!(first
+ .await
+ .unwrap()
+ .unwrap_err()
+ .to_string()
+ .contains("usage remains owed"));
+ let attempts = sink
+ .capture
+ .attempts
+ .load(std::sync::atomic::Ordering::Relaxed);
+ assert!(retry
+ .await
+ .unwrap_err()
+ .to_string()
+ .contains("usage remains owed"));
+ assert_eq!(
+ sink.capture
+ .attempts
+ .load(std::sync::atomic::Ordering::Relaxed),
+ attempts + 1
+ );
+ assert!(sessions.list_subagents().await.unwrap().is_empty());
+ assert!(sink.capture.forgotten.lock().unwrap().is_empty());
+ sink.capture
+ .fail
+ .store(false, std::sync::atomic::Ordering::Relaxed);
+ sessions
+ .close_children(ChildCloseReason::Shutdown)
+ .await
+ .unwrap();
+ {
+ let reports = sink.capture.reports.lock().unwrap();
+ assert_eq!(reports.len(), 1);
+ assert_eq!(
+ (reports[0].report_id.clone(), reports[0].batches.clone()),
+ (frozen.report_id, frozen.batches)
+ );
+ }
+ assert_eq!(
+ sink.capture.forgotten.lock().unwrap().as_slice(),
+ &["sub-concurrent-close"]
+ );
+ assert_eq!(kills.load(std::sync::atomic::Ordering::Relaxed), 1);
+ server.abort();
+ let _ = server.await;
+}
+
+#[tokio::test]
+async fn inactive_delete_finishes_a_running_record_whose_kill_already_acknowledged() {
+ cancellation_after_ack_preserves_retirement(RetirementCaller::InactiveAfterDelete).await;
+}
+
+#[tokio::test]
+async fn close_preserves_final_usage_across_a_transient_child_file_read_failure() {
+ let tmp = tempfile::tempdir().unwrap();
+ let file = child_file(tmp.path());
+ let original = std::fs::read(&file).unwrap();
+ let parked = tmp.path().join("intact-child.jsonl");
+ std::fs::rename(&file, &parked).unwrap();
+ // A directory at the requested path fails open/read on all supported platforms.
+ // The original transcript remains intact, without relying on root-sensitive permissions.
+ std::fs::create_dir(&file).unwrap();
+ let (sessions, kills, _commands, server) = retirement_registry(tmp.path()).await;
+ let sink = Arc::new(CapturingSink::default());
+ sessions.set_usage_sink(sink.clone());
+ let record = record_with_file("sub-transient-read", &file);
+ sessions.inner.children.lock().await.push(record.clone());
+ let first = sessions
+ .close_children(ChildCloseReason::Shutdown)
+ .await
+ .unwrap_err();
+ assert!(first.to_string().contains("usage remains owed"));
+ assert!(sessions.list_subagents().await.unwrap().is_empty());
+ assert!(!sessions.any_running().await);
+ assert!(record.lock().await.pending_usage_report.is_none());
+ assert_eq!(record.lock().await.attributed_rows, Some(0));
+ assert!(sink.forgotten.lock().unwrap().is_empty());
+ assert_eq!(sink.attempts.load(std::sync::atomic::Ordering::Relaxed), 0);
+ assert_eq!(std::fs::read(&parked).unwrap(), original);
+ assert!(sessions
+ .close_children(ChildCloseReason::Shutdown)
+ .await
+ .unwrap_err()
+ .to_string()
+ .contains("usage remains owed"));
+ assert!(sink.forgotten.lock().unwrap().is_empty());
+ assert_eq!(kills.load(std::sync::atomic::Ordering::Relaxed), 1);
+ std::fs::remove_dir(&file).unwrap();
+ std::fs::rename(&parked, &file).unwrap();
+ sessions
+ .close_children(ChildCloseReason::Shutdown)
+ .await
+ .unwrap();
+ {
+ let reports = sink.reports.lock().unwrap();
+ // Decode the independently pinned wire numbers once, as the transcript reader does.
+ let expected: pa_types::ai::Usage = serde_json::from_str(
+ r#"{"input":50208,"output":2929,"cacheRead":0,"cacheWrite":0,"totalTokens":53137,"cost":{"input":0.0075312,"output":0.0014645,"cacheRead":0,"cacheWrite":0,"total":0.0089957}}"#,
+ ).unwrap();
+ assert_eq!(
+ reports
+ .iter()
+ .map(|report| (report.rlm_child_id.clone(), report.batches.clone()))
+ .collect::<Vec<_>>(),
+ vec![(
+ "sub-transient-read".into(),
+ vec![(pa_types::session::ChildUsageOrigin::SpawnTask, expected)]
+ )]
+ );
+ }
+ assert_eq!(
+ sink.forgotten.lock().unwrap().as_slice(),
+ &["sub-transient-read"]
+ );
+ assert_eq!(kills.load(std::sync::atomic::Ordering::Relaxed), 1);
+ assert_eq!(std::fs::read(&file).unwrap(), original);
+ sessions
+ .close_children(ChildCloseReason::Shutdown)
+ .await
+ .unwrap();
+ assert_eq!(sink.attempts.load(std::sync::atomic::Ordering::Relaxed), 1);
+ server.abort();
+ let _ = server.await;
+}
+
+struct FinalWatcherSink {
+ capture: CapturingSink,
+ child_file: PathBuf,
+ attempts: std::sync::Mutex<Vec<RlmChildUsageReport>>,
+ commands: Arc<std::sync::Mutex<Vec<Value>>>,
+ recovered: std::sync::atomic::AtomicBool,
+}
+
+impl RlmChildUsageSink for FinalWatcherSink {
+ fn record(
+ &self,
+ report: RlmChildUsageReport,
+ ) -> std::pin::Pin<Box<dyn std::future::Future<Output = bool> + Send + '_>> {
+ Box::pin(async move {
+ let attempt = {
+ let mut attempts = self.attempts.lock().unwrap();
+ attempts.push(report.clone());
+ attempts.len()
+ };
+ if attempt == 3 {
+ assert!(
+ self.commands
+ .lock()
+ .unwrap()
+ .iter()
+ .all(|command| command["type"] != "follow_up"),
+ "the bounded final replay must precede the parent's terminal notice"
+ );
+ }
+ if attempt == 1 {
+ // The slice-top flush succeeds; the child's final turn lands afterward.
+ let row = json!({"type":"message","id":"a2","parentId":"a1",
+ "timestamp":"2026-09-23T00:00:03.000Z", "message": {
+ "role":"assistant","content":[],"stopReason":"toolUse",
+ "usage":{"input":7,"output":3,"cacheRead":0,"cacheWrite":0,"totalTokens":10,
+ "cost":{"input":0,"output":0,"cacheRead":0,"cacheWrite":0,"total":0}}
+ }});
+ let mut file = std::fs::OpenOptions::new()
+ .append(true)
+ .open(&self.child_file)
+ .unwrap();
+ std::io::Write::write_all(&mut file, format!("{row}\n").as_bytes()).unwrap();
+ self.capture.record(report).await
+ } else if attempt == 2 || !self.recovered.load(std::sync::atomic::Ordering::Relaxed) {
+ false
+ } else {
+ self.capture.record(report).await
+ }
+ })
+ }
+ fn forget(
+ &self,
+ id: &str,
+ ) -> std::pin::Pin<Box<dyn std::future::Future<Output = ()> + Send + '_>> {
+ self.capture.forget(id)
+ }
+}
+
+#[derive(Clone, Copy)]
+enum FinalWatcherFailure {
+ Transient,
+ Persistent,
+}
+
+async fn settled_watcher_replays_only_one_failed_final_interval(failure: FinalWatcherFailure) {
+ let tmp = tempfile::tempdir().unwrap();
+ let file = child_file(tmp.path());
+ let (sessions, kills, commands, server) = retirement_registry(tmp.path()).await;
+ let sink = Arc::new(FinalWatcherSink {
+ capture: CapturingSink::default(),
+ child_file: file.clone(),
+ attempts: std::sync::Mutex::new(Vec::new()),
+ commands: commands.clone(),
+ recovered: std::sync::atomic::AtomicBool::new(matches!(
+ failure,
+ FinalWatcherFailure::Transient
+ )),
+ });
+ sessions.set_usage_sink(sink.clone());
+ let record = record_with_file("sub-final-watch", &file);
+ record.lock().await.session_dir = tmp.path().join("child-display").display().to_string();
+ sessions.inner.children.lock().await.push(record.clone());
+ let (settle_tx, mut settle_rx) = tokio::sync::mpsc::unbounded_channel();
+ sessions.set_settle_hook(Arc::new(move || {
+ let _ = settle_tx.send(());
+ }));
+ tokio::time::timeout(
+ Duration::from_secs(5),
+ sessions.inner.watch_child_settle(&record),
+ )
+ .await
+ .unwrap();
+ tokio::time::timeout(Duration::from_secs(5), settle_rx.recv())
+ .await
+ .unwrap()
+ .unwrap();
+ assert!(!sessions.any_running().await);
+ assert_eq!(
+ sessions.list_subagents().await.unwrap()[0].status,
+ "completed"
+ );
+ let complete_report = |report: &RlmChildUsageReport| {
+ (
+ report.report_id.clone(),
+ report.rlm_child_id.clone(),
+ report.batches.clone(),
+ )
+ };
+ let final_report = {
+ let attempts = sink.attempts.lock().unwrap();
+ assert_eq!(
+ attempts.len(),
+ 3,
+ "one slice-top attempt and exactly two final attempts"
+ );
+ assert_eq!(complete_report(&attempts[1]), complete_report(&attempts[2]));
+ assert_ne!(
+ attempts[0].report_id, attempts[1].report_id,
+ "a new interval gets its own report identity"
+ );
+ attempts[1].clone()
+ };
+ assert_eq!(
+ final_report.batches,
+ vec![(
+ pa_types::session::ChildUsageOrigin::SpawnTask,
+ pa_types::ai::Usage {
+ input: 7,
+ output: 3,
+ total_tokens: 10,
+ ..Default::default()
+ }
+ )]
+ );
+ assert!(sink.capture.forgotten.lock().unwrap().is_empty());
+ assert_eq!(
+ commands
+ .lock()
+ .unwrap()
+ .iter()
+ .filter(|command| command["type"] == "follow_up")
+ .count(),
+ 1
+ );
+ assert_eq!(kills.load(std::sync::atomic::Ordering::Relaxed), 0);
+ assert!(settle_rx.try_recv().is_err());
+ match failure {
+ FinalWatcherFailure::Transient => {
+ {
+ let reports = sink.capture.reports.lock().unwrap();
+ assert_eq!(reports.len(), 2);
+ assert_eq!(complete_report(&reports[1]), complete_report(&final_report));
+ }
+ assert!(record.lock().await.pending_usage_report.is_none());
+ assert_eq!(record.lock().await.attributed_rows, Some(3));
+ }
+ FinalWatcherFailure::Persistent => {
+ {
+ let held = record.lock().await;
+ let pending = held.pending_usage_report.as_ref().unwrap();
+ assert_eq!(
+ (complete_report(&pending.0), pending.1),
+ (complete_report(&final_report), 3)
+ );
+ assert_eq!(held.attributed_rows, Some(2));
+ }
+ sink.recovered
+ .store(true, std::sync::atomic::Ordering::Relaxed);
+ sessions
+ .close_children(ChildCloseReason::Shutdown)
+ .await
+ .unwrap();
+ assert_eq!(
+ sink.attempts.lock().unwrap().len(),
+ 4,
+ "later explicit close still owns the frozen debt"
+ );
+ let reports = sink.capture.reports.lock().unwrap();
+ assert_eq!(reports.len(), 2);
+ assert_eq!(complete_report(&reports[1]), complete_report(&final_report));
+ assert_eq!(
+ sink.capture.forgotten.lock().unwrap().as_slice(),
+ &["sub-final-watch"]
+ );
+ assert_eq!(kills.load(std::sync::atomic::Ordering::Relaxed), 1);
+ assert!(settle_rx.try_recv().is_err());
+ }
+ }
+ server.abort();
+ let _ = server.await;
+}
+
+#[tokio::test]
+async fn settled_watcher_replays_a_failed_final_emit_before_its_notice_and_settle() {
+ settled_watcher_replays_only_one_failed_final_interval(FinalWatcherFailure::Transient).await;
+}
+
+#[tokio::test]
+async fn settled_watcher_caps_a_persistent_final_failure_and_preserves_the_debt() {
+ settled_watcher_replays_only_one_failed_final_interval(FinalWatcherFailure::Persistent).await;
+}
diff --git a/crates/pa-daemon/src/rlm_children/watch_tests.rs b/crates/pa-daemon/src/rlm_children/watch_tests.rs
index 0967f90232..5c9cb49c7c 100644
--- a/crates/pa-daemon/src/rlm_children/watch_tests.rs
+++ b/crates/pa-daemon/src/rlm_children/watch_tests.rs
@@ -1257,8 +1257,13 @@ fn an_already_settled_child_never_re_scores_as_an_unreachable_error() {
prompt_admitted: true,
error: None,
closed_by_parent: false,
+ retirement_finished: false,
+ retirement_notified: false,
+ retirement_roster_emitted: false,
+ usage_registration_forgotten: false,
session_file: None,
attributed_rows: Some(0),
+ pending_usage_report: None,
usage_watch_live: false,
usage_rearm: false,
emit_lock: std::sync::Arc::new(tokio::sync::Mutex::new(())),
@@ -1320,6 +1325,10 @@ async fn the_unreachable_give_up_skips_the_exit_capture_when_a_capture_stands()
prompt_admitted: true,
error: None,
closed_by_parent: false,
+ retirement_finished: false,
+ retirement_notified: false,
+ retirement_roster_emitted: false,
+ usage_registration_forgotten: false,
session_file: Some(
std::env::temp_dir()
.join(format!(
@@ -1330,6 +1339,7 @@ async fn the_unreachable_give_up_skips_the_exit_capture_when_a_capture_stands()
.to_string(),
),
attributed_rows: Some(0),
+ pending_usage_report: None,
usage_watch_live: false,
usage_rearm: false,
emit_lock: std::sync::Arc::new(tokio::sync::Mutex::new(())),
diff --git a/crates/pa-daemon/src/session_store.rs b/crates/pa-daemon/src/session_store.rs
index 8f5c14d56b..6d7a410ae2 100644
--- a/crates/pa-daemon/src/session_store.rs
+++ b/crates/pa-daemon/src/session_store.rs
@@ -28,6 +28,10 @@ mod stream_tests;
mod window_tests;
// The index concern lives in session_store::index; the facade re-imports keep the callers in scope.
+mod child_usage;
+#[cfg(test)]
+#[path = "session_store/child_usage_tests.rs"]
+mod child_usage_tests;
mod index;
use index::fold_child_usage_attributions;
@@ -131,6 +135,7 @@ pub struct SessionFile {
pub(crate) window: Option<SessionWindow>,
pub(crate) trace_upload: Option<std::sync::Arc<pa_core::agent_traces::ContinuousTraceUpload>>,
pub(crate) lease: Option<std::sync::Arc<crate::lease::SessionLease>>,
+ child_usage_recovery: child_usage::ChildUsageRecovery,
/// Rows the full reader skipped as unparsable (the TS loader's
/// lenient skip): zero on a healthy file. A reopen that found damage
/// logs this count (silent skips are how a torn session degraded
diff --git a/crates/pa-daemon/src/session_store/child_usage.rs b/crates/pa-daemon/src/session_store/child_usage.rs
new file mode 100644
index 0000000000..b7c0f1dfeb
--- /dev/null
+++ b/crates/pa-daemon/src/session_store/child_usage.rs
@@ -0,0 +1,461 @@
+//! The idempotent child-usage attribution append: one durable row per stable
+//! row identity, written through the same persist core as every other row.
+use super::{SessionEntry, SessionFile};
+use anyhow::{ensure, Context, Result};
+use pa_core::session_engine::rlm_usage::{
+ attribute_child_usage, child_usage_identity_matches, ChildUsageAppendResult,
+};
+use pa_types::{
+ ai::Usage,
+ session::{ChildUsageAttributionEntry, ChildUsageOrigin},
+};
+use std::collections::HashSet;
+use std::io::{Read, Seek, SeekFrom, Write};
+use std::path::PathBuf;
+
+/// Attribution-local recovery state. One write can remain unresolved; recovered
+/// creation acknowledgments are bounded by the attribution rows retained by this
+/// store and are removed when their own frozen report retries successfully.
+/// There is no fixed global cap: the normal emitter owns one frozen failed report
+/// per child and stops at its first failed batch. Dropping that owner prematurely
+/// can retain its ID until this store is dropped; terminal ownership is separate.
+#[derive(Debug, Clone, Default)]
+pub(super) struct ChildUsageRecovery {
+ pub(super) pending: Option<UnconfirmedChildRow>,
+ pub(super) created: HashSet<String>,
+}
+
+#[derive(Debug, Clone)]
+pub(super) struct UnconfirmedChildRow {
+ path: PathBuf,
+ header: serde_json::Value,
+ header_wire: serde_json::Value,
+ identity: Option<(u64, u64)>,
+ start: Option<u64>,
+ prefix_end: u64,
+ prefix_tail: Vec<u8>,
+ entry: SessionEntry,
+ bytes: Vec<u8>,
+}
+
+fn file_identity(metadata: &std::fs::Metadata) -> (u64, u64) {
+ #[cfg(unix)]
+ {
+ use std::os::unix::fs::MetadataExt;
+ (metadata.dev(), metadata.ino())
+ }
+ #[cfg(windows)]
+ {
+ use std::os::windows::fs::MetadataExt;
+ // Creation time is a limited replacement anchor, not a unique Windows
+ // file ID. Owned path/header/prefix/row checks and the existing managed
+ // single-writer lease remain necessary; noncooperative writers are out
+ // of this attribution-local recovery contract.
+ (metadata.creation_time(), 0)
+ }
+ #[cfg(not(any(unix, windows)))]
+ {
+ let _ = metadata;
+ (0, 0)
+ }
+}
+
+/// Compare complete headers with the actual wire tag included. The frozen
+/// original and its one wire decode cover reader float normalization without
+/// decoding an already loaded header a second time or ignoring extension fields.
+fn header_matches_receipt(header: &super::SessionHeader, receipt: &UnconfirmedChildRow) -> bool {
+ let actual = super::session_header_line(header);
+ actual == receipt.header || actual == receipt.header_wire
+}
+
+/// Recovery observation updates the index; confirmation also owns the final
+/// durability proof and acknowledgment bookkeeping within the same lease flight.
+#[derive(Clone, Copy, PartialEq, Eq)]
+enum ChildUsageRecoveryMode {
+ Observe,
+ Confirm,
+}
+
+impl SessionFile {
+ /// Append one attribution row once: the row id is the retry identity, so
+ /// a re-delivered report finds its own row instead of billing the child
+ /// twice. A fresh id persists through [`Self::persist_entry_at`]'s shared
+ /// core, the same write-then-index contract every other row gets.
+ ///
+ /// # Errors
+ ///
+ /// Returns an error when the row id already names another row, the
+ /// immutable identity changed, the target assistant is missing, or the
+ /// durable append or its recovery confirmation fails. A complete owned
+ /// row may be indexed after an error so later aggregates include it;
+ /// acknowledgment still requires a successful durability confirmation.
+ pub(crate) fn append_child_usage_once(
+ &mut self,
+ row_id: &str,
+ target_id: &str,
+ child_usage: Usage,
+ origin: Option<ChildUsageOrigin>,
+ ) -> Result<ChildUsageAppendResult> {
+ ensure!(!row_id.is_empty(), "attribution ID must not be empty");
+ self.confirm_child_usage_write()?;
+ if let Some(entry) = self.entry(row_id) {
+ ensure!(
+ entry.type_ == "child_usage_attributed",
+ "attribution ID collision"
+ );
+ let payload: ChildUsageAttributionEntry = serde_json::from_value(entry.fields.clone())?;
+ ensure!(
+ child_usage_identity_matches(&payload, target_id, child_usage, origin)?,
+ "attribution ID collision"
+ );
+ let current = self.attribution_target_usage(target_id)?;
+ return Ok(if self.child_usage_recovery.created.remove(row_id) {
+ ChildUsageAppendResult::Created(current)
+ } else {
+ ChildUsageAppendResult::Existing(current)
+ });
+ }
+ let mut aggregate = self.attribution_target_usage(target_id)?;
+ attribute_child_usage(&mut aggregate, &child_usage);
+ let entry = SessionEntry {
+ type_: "child_usage_attributed".to_owned(),
+ id: row_id.to_owned(),
+ parent_id: self.leaf_id.clone(),
+ timestamp: crate::util::now_iso(),
+ fields: serde_json::to_value(ChildUsageAttributionEntry {
+ target_id: target_id.to_owned(),
+ child_usage,
+ aggregate_usage: aggregate,
+ origin,
+ })?,
+ };
+ // Capture only this owned row, not the transcript. The EOF anchor and
+ // existing file identity distinguish an absent attempt from a partial
+ // or foreign change. Bootstrap rewrite has no existing append offset.
+ let receipt = if self.path.as_os_str().is_empty() {
+ None
+ } else {
+ let header = super::session_header_line(&self.header);
+ let header_wire = serde_json::from_slice(&serde_json::to_vec(&header)?)?;
+ let mut receipt = UnconfirmedChildRow {
+ path: self.path.clone(),
+ header,
+ header_wire,
+ identity: None,
+ start: None,
+ prefix_end: 0,
+ prefix_tail: Vec::new(),
+ bytes: serde_json::to_vec(&entry)?,
+ entry: entry.clone(),
+ };
+ receipt.bytes.push(b'\n');
+ match std::fs::File::open(&self.path) {
+ Ok(mut file) => {
+ let metadata = file.metadata()?;
+ receipt.identity = Some(file_identity(&metadata));
+ receipt.prefix_end = metadata.len();
+ let tail_len = metadata.len().min(16) as usize;
+ file.seek(SeekFrom::End(-(tail_len as i64)))?;
+ receipt.prefix_tail.resize(tail_len, 0);
+ file.read_exact(&mut receipt.prefix_tail)?;
+ receipt.start = Some(
+ metadata.len()
+ + u64::from(
+ metadata.len() > 0 && receipt.prefix_tail.last() != Some(&b'\n'),
+ ),
+ );
+ }
+ Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
+ Err(error) => return Err(error.into()),
+ }
+ Some(receipt)
+ };
+ if let Err(error) = self.persist_built_entry(entry) {
+ self.child_usage_recovery.pending = receipt;
+ // A completed own row must affect the next aggregate before this
+ // store lock is released, even though its original caller sees Err.
+ if let Err(recovery) = self.recover_child_usage_write(ChildUsageRecoveryMode::Observe) {
+ return Err(error.context(format!("attribution remains unresolved: {recovery}")));
+ }
+ return Err(error);
+ }
+ Ok(ChildUsageAppendResult::Created(aggregate))
+ }
+
+ /// Confirm an unresolved own row before another durable mutation or ACK.
+ pub(super) fn confirm_child_usage_write(&mut self) -> Result<()> {
+ self.recover_child_usage_write(ChildUsageRecoveryMode::Confirm)
+ }
+
+ /// Observe only the frozen owned row; no lenient whole-file scan or truncation.
+ fn recover_child_usage_write(&mut self, mode: ChildUsageRecoveryMode) -> Result<()> {
+ let Some(receipt) = self.child_usage_recovery.pending.clone() else {
+ return Ok(());
+ };
+ let lease = self.lease.clone();
+ let path = self.path.clone();
+ let observe = || {
+ ensure!(
+ self.path == receipt.path && header_matches_receipt(&self.header, &receipt),
+ "attribution recovery session changed"
+ );
+ let mut file = match std::fs::OpenOptions::new()
+ .read(true)
+ .append(true)
+ .open(&receipt.path)
+ {
+ Ok(file) => file,
+ Err(error)
+ if error.kind() == std::io::ErrorKind::NotFound && receipt.start.is_none() =>
+ {
+ self.child_usage_recovery.pending = None;
+ return Ok(());
+ }
+ Err(error) => return Err(error.into()),
+ };
+ let metadata = file.metadata()?;
+ if let Some(identity) = receipt.identity {
+ ensure!(
+ file_identity(&metadata) == identity,
+ "attribution recovery file replaced"
+ );
+ }
+ let header_line = super::read_first_line_bounded_from(
+ &mut file,
+ super::SESSION_LIST_HEADER_READ_MAX_BYTES,
+ )
+ .context("attribution recovery header unreadable")?;
+ let header = super::parse_session_header_line(std::str::from_utf8(&header_line)?)
+ .context("attribution recovery header invalid")?;
+ ensure!(
+ header_matches_receipt(&header, &receipt),
+ "attribution recovery header changed"
+ );
+ let start = if let Some(start) = receipt.start {
+ ensure!(
+ metadata.len() >= receipt.prefix_end,
+ "attribution recovery prefix shrank"
+ );
+ file.seek(SeekFrom::Start(
+ receipt.prefix_end - receipt.prefix_tail.len() as u64,
+ ))?;
+ let mut tail = vec![0; receipt.prefix_tail.len()];
+ file.read_exact(&mut tail)?;
+ ensure!(
+ tail == receipt.prefix_tail,
+ "attribution recovery prefix changed"
+ );
+ if start > receipt.prefix_end && metadata.len() >= start {
+ file.seek(SeekFrom::Start(receipt.prefix_end))?;
+ let mut separator = [0];
+ file.read_exact(&mut separator)?;
+ ensure!(
+ separator == *b"\n",
+ "attribution recovery separator changed"
+ );
+ }
+ if metadata.len() == receipt.prefix_end || metadata.len() == start {
+ self.child_usage_recovery.pending = None;
+ return Ok(());
+ }
+ start
+ } else {
+ // The atomic bootstrap's own row must be its exact last line.
+ ensure!(
+ metadata.len() >= receipt.bytes.len() as u64,
+ "attribution bootstrap is incomplete"
+ );
+ metadata.len() - receipt.bytes.len() as u64
+ };
+ let expected_end = start + receipt.bytes.len() as u64;
+ let mut completed_tail = false;
+ if metadata.len() < expected_end {
+ // Only an append receipt has a proven row start. Bootstrap remains
+ // fail-closed rather than guessing which rewritten bytes are missing.
+ ensure!(
+ receipt.start.is_some() && metadata.len() > start,
+ "attribution recovery row is incomplete"
+ );
+ let present = usize::try_from(metadata.len() - start)?;
+ file.seek(SeekFrom::Start(start))?;
+ let mut partial = vec![0; present];
+ file.read_exact(&mut partial)?;
+ ensure!(
+ partial.as_slice() == &receipt.bytes[..present],
+ "attribution recovery owned prefix conflicts"
+ );
+ ensure!(
+ file.metadata()?.len() == metadata.len()
+ && file_identity(&std::fs::metadata(&receipt.path)?)
+ == file_identity(&file.metadata()?),
+ "attribution recovery tail changed before completion"
+ );
+ #[cfg(test)]
+ if super::child_usage_tests::FAIL_CHILD_TAIL_COMPLETION
+ .with(|fault| fault.replace(false))
+ {
+ anyhow::bail!("injected owned tail completion IO error");
+ }
+ // The append-mode descriptor writes only the missing owned bytes;
+ // append_cached would insert a newline into this incomplete row.
+ file.write_all(&receipt.bytes[present..])?;
+ file.flush()?;
+ pa_core::platform::fsync(&file)?;
+ ensure!(
+ file.metadata()?.len() == expected_end,
+ "attribution recovery tail changed during completion"
+ );
+ completed_tail = true;
+ }
+ file.seek(SeekFrom::Start(start))?;
+ let mut actual = vec![0; receipt.bytes.len()];
+ file.read_exact(&mut actual)?;
+ ensure!(
+ actual == receipt.bytes,
+ "attribution recovery row conflicts"
+ );
+ file.seek(SeekFrom::End(-1))?;
+ let mut tail = [0];
+ file.read_exact(&mut tail)?;
+ ensure!(
+ tail == *b"\n",
+ "attribution recovery file has an incomplete tail"
+ );
+ if completed_tail {
+ ensure!(
+ file_identity(&std::fs::metadata(&receipt.path)?)
+ == file_identity(&file.metadata()?),
+ "attribution recovery file changed after completion"
+ );
+ // The row is now newline-closed. Empty append invalidates the old
+ // window certificate without certifying a suffix as a fresh row.
+ // Stay within the same lease fence rather than recursively admitting
+ // a second mutation while release may already be draining this one.
+ let ownership = if self.lease.is_some() {
+ pa_core::session::window::AppendOwnership::SessionLeaseHeld
+ } else {
+ pa_core::session::window::AppendOwnership::Unleased
+ };
+ pa_core::session::window::append_cached(&receipt.path, &[], ownership)?;
+ ensure!(
+ file_identity(&std::fs::metadata(&receipt.path)?)
+ == file_identity(&file.metadata()?),
+ "attribution recovery file changed after completion sync"
+ );
+ }
+ // Loading the exact owned JSON can normalize a cost float. Admit the
+ // complete frozen object or its complete wire-decoded object, never a
+ // fieldwise mixture or an aggregate-only exception.
+ let original = serde_json::to_value(&receipt.entry)?;
+ let decoded: SessionEntry = serde_json::from_slice(&receipt.bytes)?;
+ let decoded = serde_json::to_value(decoded)?;
+ if let Some(existing) = self.entry(&receipt.entry.id) {
+ let indexed = serde_json::to_value(existing)?;
+ ensure!(
+ indexed == original || indexed == decoded,
+ "attribution recovery indexed row conflicts"
+ );
+ } else {
+ // Normally immediate, before any later row can branch from stale state.
+ // Hydration may already have installed it; do not steal its selected leaf.
+ if self.leaf_id == receipt.entry.parent_id {
+ self.push_index(receipt.entry.clone());
+ } else {
+ // A generic in-memory append may have arrived while recovery
+ // could not read the file. Install the physical ordering first,
+ // then merge its unsaved rows without changing the selected leaf.
+ let mut full = Self::open(&receipt.path)?;
+ ensure!(
+ header_matches_receipt(&full.header, &receipt),
+ "attribution recovery snapshot session changed"
+ );
+ let owned = full
+ .entry(&receipt.entry.id)
+ .context("attribution recovery snapshot lost row")?;
+ let indexed = serde_json::to_value(owned)?;
+ ensure!(
+ indexed == original || indexed == decoded,
+ "attribution recovery snapshot row conflicts"
+ );
+ for entry in &self.entries {
+ if !full.by_id.contains_key(&entry.id) {
+ full.push_index(entry.clone());
+ }
+ }
+ full.leaf_id.clone_from(&self.leaf_id);
+ full.lease.clone_from(&self.lease);
+ full.trace_upload.clone_from(&self.trace_upload);
+ full.child_usage_recovery
+ .clone_from(&self.child_usage_recovery);
+ full.anthropic_warning_shown = self.anthropic_warning_shown;
+ *self = full;
+ }
+ }
+ self.child_usage_recovery.created.insert(receipt.entry.id);
+ #[cfg(test)]
+ if let Some(hook) =
+ super::child_usage_tests::AFTER_CHILD_OBSERVE.with(|hook| hook.borrow_mut().take())
+ {
+ hook();
+ }
+ if mode == ChildUsageRecoveryMode::Confirm {
+ // This proof and sync belong to the already admitted recovery.
+ // Release may be draining it; do not start a second admission.
+ #[cfg(test)]
+ if super::child_usage_tests::FAIL_CHILD_CONFIRM_SYNC
+ .with(|fault| fault.replace(false))
+ {
+ anyhow::bail!("injected attribution confirmation sync failure");
+ }
+ ensure!(
+ file_identity(&std::fs::metadata(&self.path)?)
+ == file_identity(&file.metadata()?),
+ "attribution confirmation file changed before sync"
+ );
+ if self.lease.is_some() {
+ pa_core::session::window::append_cached(
+ &self.path,
+ &[],
+ pa_core::session::window::AppendOwnership::SessionLeaseHeld,
+ )?;
+ } else {
+ pa_core::platform::fsync(&file)?;
+ }
+ ensure!(
+ file_identity(&std::fs::metadata(&self.path)?)
+ == file_identity(&file.metadata()?),
+ "attribution confirmation file changed"
+ );
+ self.child_usage_recovery.pending = None;
+ if let Some(traces) = &self.trace_upload {
+ traces.persisted(&self.path);
+ }
+ // Retain Created until its own report receives acknowledgment.
+ }
+ Ok(())
+ };
+ match lease {
+ Some(lease) => lease.with_owned_mutation(&path, observe),
+ None => observe(),
+ }
+ }
+
+ /// The target assistant row's current usage: the running aggregate, since
+ /// each indexed attribution folds its aggregate back into the row.
+ fn attribution_target_usage(&self, target_id: &str) -> Result<Usage> {
+ let message = self
+ .entry(target_id)
+ .and_then(|entry| entry.fields.get("message"))
+ .context("child attribution target is missing")?;
+ ensure!(
+ message.get("role").and_then(serde_json::Value::as_str) == Some("assistant"),
+ "child attribution target is not an assistant"
+ );
+ Ok(message
+ .get("usage")
+ .cloned()
+ .map(serde_json::from_value)
+ .transpose()?
+ .unwrap_or_default())
+ }
+}
diff --git a/crates/pa-daemon/src/session_store/child_usage_tests.rs b/crates/pa-daemon/src/session_store/child_usage_tests.rs
new file mode 100644
index 0000000000..18dfd6322e
--- /dev/null
+++ b/crates/pa-daemon/src/session_store/child_usage_tests.rs
@@ -0,0 +1,686 @@
+use super::SessionFile;
+use pa_core::session_engine::rlm_usage::ChildUsageAppendResult;
+use pa_types::ai::Usage;
+use serde_json::json;
+
+fn fixture() -> (tempfile::TempDir, SessionFile, String) {
+ let dir = tempfile::tempdir().unwrap();
+ let mut store = SessionFile::create("/synthetic", None, 0);
+ let id = store.append_message(&json!({
+ "role": "assistant", "content": [], "api": "openai-completions", "provider": "synthetic",
+ "model": "synthetic", "stopReason": "stop", "timestamp": 1,
+ "usage": Usage::default()
+ }));
+ store.set_path(dir.path().join("session.jsonl"));
+ store.rewrite().unwrap();
+ (dir, store, id)
+}
+
+fn child() -> Usage {
+ Usage {
+ input: 10,
+ ..Usage::default()
+ }
+}
+
+fn attribution_rows(store: &SessionFile) -> Vec<serde_json::Value> {
+ std::fs::read_to_string(&store.path)
+ .unwrap()
+ .lines()
+ .map(|line| serde_json::from_str::<serde_json::Value>(line).unwrap())
+ .filter(|row| row["type"] == "child_usage_attributed")
+ .collect()
+}
+
+/// A retried row id appends nothing: the retry answers `Existing` with the
+/// target's current aggregate (a later child may have advanced it), and the
+/// transcript keeps exactly the rows that first landed — after a reopen too.
+#[test]
+fn existing_retry_returns_current_aggregate_after_other_child() {
+ let (_dir, mut store, target) = fixture();
+ store
+ .append_child_usage_once("a", &target, child(), None)
+ .unwrap();
+ store
+ .append_child_usage_once("b", &target, child(), None)
+ .unwrap();
+ let result = store
+ .append_child_usage_once("a", &target, child(), None)
+ .unwrap();
+ let expected = Usage {
+ input: 20,
+ total_tokens: 10,
+ ..Usage::default()
+ };
+ assert!(
+ matches!(result, ChildUsageAppendResult::Existing(usage) if usage == expected),
+ "the retry reuses the current aggregate: {result:?}"
+ );
+ assert_eq!(attribution_rows(&store).len(), 2);
+ let mut reopened = SessionFile::open(&store.path).unwrap();
+ let result = reopened
+ .append_child_usage_once("a", &target, child(), None)
+ .unwrap();
+ assert!(
+ matches!(result, ChildUsageAppendResult::Existing(usage) if usage == expected),
+ "the reopened store answers from its durable index: {result:?}"
+ );
+ assert_eq!(attribution_rows(&reopened).len(), 2);
+}
+
+// Test-only exact postcondition injection: the real append has written/synced
+// its complete JSONL row, but the caller sees an error before indexing it.
+// This models complete-write/sync-error recovery; it does not induce OS fsync failure.
+std::thread_local! {
+ pub(super) static FAIL_AFTER_CHILD_APPEND: std::cell::Cell<bool> = const { std::cell::Cell::new(false) };
+ pub(super) static AFTER_CHILD_OBSERVE: std::cell::RefCell<Option<Box<dyn FnOnce()>>> = const { std::cell::RefCell::new(None) };
+ pub(super) static FAIL_CHILD_CONFIRM_SYNC: std::cell::Cell<bool> = const { std::cell::Cell::new(false) };
+ pub(super) static PARTIAL_CHILD_APPEND: std::cell::Cell<Option<usize>> = const { std::cell::Cell::new(None) };
+ pub(super) static FAIL_CHILD_TAIL_COMPLETION: std::cell::Cell<bool> = const { std::cell::Cell::new(false) };
+}
+
+#[test]
+fn completed_append_error_retry_keeps_one_physical_attribution() {
+ let (_dir, mut store, target) = fixture();
+ FAIL_AFTER_CHILD_APPEND.with(|fault| fault.set(true));
+ assert!(store
+ .append_child_usage_once("report-a", &target, child(), None)
+ .is_err());
+ assert_eq!(
+ attribution_rows(&store).len(),
+ 1,
+ "the fault is after physical append"
+ );
+ let result = store
+ .append_child_usage_once("report-a", &target, child(), None)
+ .unwrap();
+ let mut expected = Usage::default();
+ pa_core::session_engine::rlm_usage::attribute_child_usage(&mut expected, &child());
+ let confirmed = match result {
+ ChildUsageAppendResult::Created(usage) | ChildUsageAppendResult::Existing(usage) => usage,
+ };
+ assert_eq!(confirmed, expected);
+ assert_eq!(
+ attribution_rows(&store).len(),
+ 1,
+ "same stable report must not append twice"
+ );
+}
+
+fn windowed_fixture() -> (tempfile::TempDir, SessionFile, String, Usage) {
+ let dir = tempfile::tempdir().unwrap();
+ let path = dir.path().join("session.jsonl");
+ let mut full = SessionFile::create("/synthetic", None, 0);
+ full.set_path(path.clone());
+ let mut first_kept = String::new();
+ for i in 0..220 {
+ let id = full.append_message(&json!({
+ "role": "user", "content": format!("fixture {i}"), "timestamp": i
+ }));
+ if i == 210 {
+ first_kept = id;
+ }
+ }
+ let raw = Usage {
+ input: 1000,
+ total_tokens: 4096,
+ ..Usage::default()
+ };
+ let target = full.append_message(&json!({
+ "role":"assistant", "content":[], "api":"openai-completions",
+ "provider":"synthetic", "model":"synthetic", "stopReason":"stop",
+ "timestamp":221, "usage":raw
+ }));
+ full.append_entry(
+ "compaction",
+ json!({
+ "summary":"fixture", "firstKeptEntryId":first_kept, "tokensBefore":10000
+ }),
+ );
+ full.rewrite().unwrap();
+ let live = SessionFile::open_windowed(&path).unwrap();
+ assert!(live.window.is_some(), "exercise production hydration path");
+
+ (dir, live, target, raw)
+}
+
+#[test]
+fn hydration_after_an_uncertain_append_does_not_lose_child_usage() {
+ use pa_core::session_engine::rlm_usage::attribute_child_usage;
+ use pa_types::session::ChildUsageOrigin;
+
+ let (_dir, mut live, target, raw) = windowed_fixture();
+
+ let a = Usage {
+ input: 10,
+ ..Usage::default()
+ };
+ let b = Usage {
+ input: 20,
+ ..Usage::default()
+ };
+ FAIL_AFTER_CHILD_APPEND.with(|fault| fault.set(true));
+ let result =
+ live.append_child_usage_once("report-a-0", &target, a, Some(ChildUsageOrigin::SpawnTask));
+ assert!(result.is_err());
+ assert_eq!(
+ attribution_rows(&live).len(),
+ 1,
+ "the fault is after physical append"
+ );
+ live.append_child_usage_once(
+ "report-b-0",
+ &target,
+ b,
+ Some(ChildUsageOrigin::AgentMessage),
+ )
+ .unwrap();
+
+ // dispatch(get_session_tree/export/navigate) opens and installs a full
+ // snapshot while the same worker's frozen report A remains retryable.
+ live.install_full_history(SessionFile::open(&live.path).unwrap());
+ let result = live
+ .append_child_usage_once("report-a-0", &target, a, Some(ChildUsageOrigin::SpawnTask))
+ .unwrap();
+ let confirmed = match result {
+ ChildUsageAppendResult::Created(usage) | ChildUsageAppendResult::Existing(usage) => usage,
+ };
+ let mut expected = raw;
+ attribute_child_usage(&mut expected, &a);
+ attribute_child_usage(&mut expected, &b);
+ assert_eq!(
+ confirmed, expected,
+ "both distinct child batches must survive hydration"
+ );
+ assert_eq!(attribution_rows(&live).len(), 2);
+}
+
+#[test]
+fn recovered_created_receipts_are_acknowledged_once_and_do_not_accumulate() {
+ let (_dir, mut store, target) = fixture();
+ for cycle in 0..4 {
+ let a = format!("recovered-{cycle}");
+ let b = format!("later-{cycle}");
+ FAIL_AFTER_CHILD_APPEND.with(|fault| fault.set(true));
+ assert!(store
+ .append_child_usage_once(&a, &target, child(), None)
+ .is_err());
+ assert_eq!(store.child_usage_recovery.created.len(), 1);
+ assert!(store.child_usage_recovery.pending.is_some());
+ assert!(matches!(
+ store
+ .append_child_usage_once(&b, &target, child(), None)
+ .unwrap(),
+ ChildUsageAppendResult::Created(_)
+ ));
+ assert!(store.child_usage_recovery.pending.is_none());
+ assert_eq!(store.child_usage_recovery.created.len(), 1);
+ assert!(matches!(
+ store
+ .append_child_usage_once(&a, &target, child(), None)
+ .unwrap(),
+ ChildUsageAppendResult::Created(_)
+ ));
+ assert!(store.child_usage_recovery.created.is_empty());
+ assert!(matches!(
+ store
+ .append_child_usage_once(&a, &target, child(), None)
+ .unwrap(),
+ ChildUsageAppendResult::Existing(_)
+ ));
+ }
+ assert_eq!(attribution_rows(&store).len(), 8);
+}
+
+#[test]
+fn generic_append_after_uncertain_attribution_preserves_bytes_and_branch_order() {
+ let (_dir, mut store, target) = fixture();
+ FAIL_AFTER_CHILD_APPEND.with(|fault| fault.set(true));
+ assert!(store
+ .append_child_usage_once("a", &target, child(), None)
+ .is_err());
+ let before = std::fs::read(&store.path).unwrap();
+ let c = store
+ .persist_entry(
+ "custom",
+ json!({"customType":"synthetic", "data":{"marker":1}}),
+ )
+ .unwrap();
+ assert_eq!(store.entry(&c).unwrap().parent_id.as_deref(), Some("a"));
+ store
+ .append_child_usage_once("b", &target, child(), None)
+ .unwrap();
+ let after = std::fs::read(&store.path).unwrap();
+ assert!(after.starts_with(&before));
+ let observed: Vec<_> = store.branch().iter().map(|row| row.id.clone()).collect();
+ assert_eq!(
+ observed,
+ vec![target.clone(), "a".to_owned(), c, "b".to_owned()]
+ );
+ let mut reopened = SessionFile::open(&store.path).unwrap();
+ let reopened_branch: Vec<_> = reopened.branch().iter().map(|row| row.id.clone()).collect();
+ assert_eq!(reopened_branch, observed);
+ assert!(matches!(
+ store
+ .append_child_usage_once("a", &target, child(), None)
+ .unwrap(),
+ ChildUsageAppendResult::Created(_)
+ ));
+ assert!(matches!(
+ reopened
+ .append_child_usage_once("a", &target, child(), None)
+ .unwrap(),
+ ChildUsageAppendResult::Existing(_)
+ ));
+}
+
+#[test]
+fn failed_confirmation_keeps_pending_and_blocks_a_new_attribution() {
+ let (_dir, mut store, target) = fixture();
+ FAIL_AFTER_CHILD_APPEND.with(|fault| fault.set(true));
+ assert!(store
+ .append_child_usage_once("a", &target, child(), None)
+ .is_err());
+ let before = std::fs::read(&store.path).unwrap();
+ FAIL_CHILD_CONFIRM_SYNC.with(|fault| fault.set(true));
+ assert!(store
+ .append_child_usage_once("b", &target, child(), None)
+ .is_err());
+ assert_eq!(std::fs::read(&store.path).unwrap(), before);
+ assert!(store.child_usage_recovery.pending.is_some());
+ assert_eq!(store.child_usage_recovery.created.len(), 1);
+ FAIL_CHILD_CONFIRM_SYNC.with(|fault| fault.set(true));
+ assert!(store
+ .persist_entry("custom", json!({"customType":"blocked"}))
+ .is_err());
+ assert_eq!(std::fs::read(&store.path).unwrap(), before);
+ assert!(matches!(
+ store
+ .append_child_usage_once("a", &target, child(), None)
+ .unwrap(),
+ ChildUsageAppendResult::Created(_)
+ ));
+ assert!(store.child_usage_recovery.pending.is_none());
+ assert!(store.child_usage_recovery.created.is_empty());
+}
+
+#[test]
+fn bootstrap_complete_error_can_confirm_one_owned_row() {
+ let dir = tempfile::tempdir().unwrap();
+ let mut store = SessionFile::create("/synthetic", None, 0);
+ let target = store.append_message(&json!({"role":"assistant", "content":[],
+ "timestamp":1, "usage":Usage::default()}));
+ store.set_path(dir.path().join("new.jsonl"));
+ FAIL_AFTER_CHILD_APPEND.with(|fault| fault.set(true));
+ assert!(store
+ .append_child_usage_once("a", &target, child(), None)
+ .is_err());
+ assert_eq!(attribution_rows(&store).len(), 1);
+ assert!(matches!(
+ store
+ .append_child_usage_once("a", &target, child(), None)
+ .unwrap(),
+ ChildUsageAppendResult::Created(_)
+ ));
+ assert_eq!(attribution_rows(&store).len(), 1);
+ assert!(store.child_usage_recovery.pending.is_none());
+}
+
+#[test]
+fn conflicting_owned_bytes_and_incomplete_tail_fail_closed_without_truncation() {
+ use std::io::Write;
+ let (_dir, mut store, target) = fixture();
+ FAIL_AFTER_CHILD_APPEND.with(|fault| fault.set(true));
+ assert!(store
+ .append_child_usage_once("a", &target, child(), None)
+ .is_err());
+ {
+ let mut file = std::fs::OpenOptions::new()
+ .append(true)
+ .open(&store.path)
+ .unwrap();
+ file.write_all(b"{\"unrelated\":").unwrap();
+ }
+ let before = std::fs::read(&store.path).unwrap();
+ assert!(store
+ .append_child_usage_once("b", &target, child(), None)
+ .is_err());
+ assert_eq!(std::fs::read(&store.path).unwrap(), before);
+ assert!(store.child_usage_recovery.pending.is_some());
+ let mut conflict = before;
+ let owned_start = conflict
+ .windows(11)
+ .position(|bytes| bytes == b"\"input\":10,")
+ .unwrap();
+ conflict[owned_start + 8] = b'2';
+ std::fs::write(&store.path, &conflict).unwrap();
+ assert!(store
+ .append_child_usage_once("a", &target, child(), None)
+ .is_err());
+ assert_eq!(std::fs::read(&store.path).unwrap(), conflict);
+}
+
+#[test]
+fn hydration_before_confirmation_retains_pending_and_created_receipts() {
+ let (_dir, mut live, target, _raw) = windowed_fixture();
+ FAIL_AFTER_CHILD_APPEND.with(|fault| fault.set(true));
+ assert!(live
+ .append_child_usage_once("a", &target, child(), None)
+ .is_err());
+ live.install_full_history(SessionFile::open(&live.path).unwrap());
+ assert!(live.window.is_none());
+ assert!(live.child_usage_recovery.pending.is_some());
+ assert_eq!(live.child_usage_recovery.created.len(), 1);
+ FAIL_CHILD_CONFIRM_SYNC.with(|fault| fault.set(true));
+ assert!(live
+ .append_child_usage_once("a", &target, child(), None)
+ .is_err());
+ assert!(live.child_usage_recovery.pending.is_some());
+ assert!(matches!(
+ live.append_child_usage_once("a", &target, child(), None)
+ .unwrap(),
+ ChildUsageAppendResult::Created(_)
+ ));
+ assert!(live.child_usage_recovery.pending.is_none());
+ assert!(live.child_usage_recovery.created.is_empty());
+ assert_eq!(attribution_rows(&live).len(), 1);
+}
+
+#[test]
+fn changed_session_identity_cannot_confirm_an_owned_attribution() {
+ let (_dir, mut store, target) = fixture();
+ FAIL_AFTER_CHILD_APPEND.with(|fault| fault.set(true));
+ assert!(store
+ .append_child_usage_once("a", &target, child(), None)
+ .is_err());
+ let mut rows: Vec<serde_json::Value> = std::fs::read_to_string(&store.path)
+ .unwrap()
+ .lines()
+ .map(|row| serde_json::from_str(row).unwrap())
+ .collect();
+ rows[0]["id"] = json!("foreign-session");
+ let changed = rows
+ .iter()
+ .map(serde_json::Value::to_string)
+ .collect::<Vec<_>>()
+ .join("\n")
+ + "\n";
+ std::fs::write(&store.path, &changed).unwrap();
+ let error = store
+ .append_child_usage_once("a", &target, child(), None)
+ .unwrap_err();
+ assert!(error.to_string().contains("header changed"), "{error}");
+ assert_eq!(std::fs::read(&store.path).unwrap(), changed.as_bytes());
+ assert!(store.child_usage_recovery.pending.is_some());
+}
+
+#[test]
+fn nonzero_cost_hydration_confirms_the_complete_wire_decoded_owned_row() {
+ use pa_types::ai::UsageCost;
+ let (_dir, mut live, target, _raw) = windowed_fixture();
+ let spend = Usage {
+ input: 10,
+ output: 7,
+ total_tokens: 17,
+ cost: UsageCost {
+ input: pa_types::JsNumber(0.007_531_199_999_999_999),
+ output: pa_types::JsNumber(0.001_464_499_999_999_999_9),
+ total: pa_types::JsNumber(0.008_995_7),
+ ..UsageCost::default()
+ },
+ ..Usage::default()
+ };
+ FAIL_AFTER_CHILD_APPEND.with(|fault| fault.set(true));
+ assert!(live
+ .append_child_usage_once("cost-a", &target, spend, None)
+ .is_err());
+ let before = std::fs::read(&live.path).unwrap();
+ let full = SessionFile::open(&live.path).unwrap();
+ let wire_expected = serde_json::from_value::<pa_types::session::ChildUsageAttributionEntry>(
+ full.entry("cost-a").unwrap().fields.clone(),
+ )
+ .unwrap()
+ .aggregate_usage;
+ live.install_full_history(full);
+ let confirmed = live
+ .append_child_usage_once("cost-a", &target, spend, None)
+ .unwrap();
+ assert!(matches!(confirmed, ChildUsageAppendResult::Created(usage) if usage == wire_expected));
+ assert_eq!(std::fs::read(&live.path).unwrap(), before);
+ assert!(live.child_usage_recovery.created.is_empty());
+}
+
+#[test]
+fn recovery_preserves_complete_wire_header_extensions_across_plain_reopen() {
+ let (_dir, mut store, target) = fixture();
+ store.header.rest.insert(
+ "extension".to_owned(),
+ json!({
+ "retained": true, "cost": 0.007_531_199_999_999_999,
+ }),
+ );
+ store.rewrite().unwrap();
+ FAIL_AFTER_CHILD_APPEND.with(|fault| fault.set(true));
+ assert!(store
+ .append_child_usage_once("a", &target, child(), None)
+ .is_err());
+ let before = std::fs::read(&store.path).unwrap();
+ let reopened = SessionFile::open(&store.path).unwrap();
+ assert_eq!(reopened.header.rest.get("type"), Some(&json!("session")));
+ let before_header: serde_json::Value =
+ serde_json::from_slice(before.split(|byte| *byte == b'\n').next().unwrap()).unwrap();
+ let reopened_header = super::session_header_line(&reopened.header);
+ assert_eq!(before_header, reopened_header);
+ assert!(matches!(
+ store
+ .append_child_usage_once("a", &target, child(), None)
+ .unwrap(),
+ ChildUsageAppendResult::Created(_)
+ ));
+ assert_eq!(std::fs::read(&store.path).unwrap(), before);
+}
+
+#[test]
+fn conflicting_partial_owned_write_remains_owed_without_truncating_its_healthy_prefix() {
+ let (_dir, mut store, target) = fixture();
+ let prefix = std::fs::read(&store.path).unwrap();
+ PARTIAL_CHILD_APPEND.with(|fault| fault.set(Some(12)));
+ FAIL_CHILD_TAIL_COMPLETION.with(|fault| fault.set(true));
+ assert!(store
+ .append_child_usage_once("partial-a", &target, child(), None)
+ .is_err());
+ FAIL_CHILD_TAIL_COMPLETION.with(|fault| fault.set(false));
+ let mut partial = std::fs::read(&store.path).unwrap();
+ // A foreign change is not an owned prefix: retain every byte and fail closed.
+ partial[prefix.len() + 9] = b'X';
+ std::fs::write(&store.path, &partial).unwrap();
+ assert!(partial.starts_with(&prefix));
+ assert_eq!(partial.len(), prefix.len() + 12);
+ assert!(store.child_usage_recovery.pending.is_some());
+ assert!(store.child_usage_recovery.created.is_empty());
+ assert!(store
+ .append_child_usage_once("b", &target, child(), None)
+ .is_err());
+ assert!(store
+ .persist_entry("custom", json!({"customType":"blocked-after-partial"}))
+ .is_err());
+ assert!(store
+ .append_child_usage_once("partial-a", &target, child(), None)
+ .is_err());
+ assert_eq!(std::fs::read(&store.path).unwrap(), partial);
+}
+
+#[test]
+fn recovery_preserves_valid_unterminated_eof_and_rejects_a_changed_separator() {
+ let (_dir, mut store, target) = fixture();
+ let mut prefix = std::fs::read(&store.path).unwrap();
+ assert_eq!(prefix.pop(), Some(b'\n'));
+ std::fs::write(&store.path, &prefix).unwrap();
+ FAIL_AFTER_CHILD_APPEND.with(|fault| fault.set(true));
+ assert!(store
+ .append_child_usage_once("a", &target, child(), None)
+ .is_err());
+ let healthy = std::fs::read(&store.path).unwrap();
+ assert!(healthy.starts_with(&prefix));
+ assert_eq!(healthy[prefix.len()], b'\n');
+ let mut damaged = healthy.clone();
+ damaged[prefix.len()] = b'X';
+ std::fs::write(&store.path, &damaged).unwrap();
+ let error = store
+ .append_child_usage_once("a", &target, child(), None)
+ .unwrap_err();
+ assert!(error.to_string().contains("separator changed"), "{error}");
+ assert_eq!(std::fs::read(&store.path).unwrap(), damaged);
+ std::fs::write(&store.path, &healthy).unwrap();
+ assert!(matches!(
+ store
+ .append_child_usage_once("a", &target, child(), None)
+ .unwrap(),
+ ChildUsageAppendResult::Created(_)
+ ));
+ assert_eq!(std::fs::read(&store.path).unwrap(), healthy);
+ assert_eq!(attribution_rows(&store).len(), 1);
+}
+
+#[test]
+fn exact_owned_partial_tail_completes_without_truncation_and_unblocks_generic_writes() {
+ let (_dir, mut store, target) = fixture();
+ PARTIAL_CHILD_APPEND.with(|fault| fault.set(Some(12)));
+ FAIL_CHILD_TAIL_COMPLETION.with(|fault| fault.set(true));
+ assert!(store
+ .append_child_usage_once("partial-a", &target, child(), None)
+ .is_err());
+ FAIL_CHILD_TAIL_COMPLETION.with(|fault| fault.set(false));
+ let partial = std::fs::read(&store.path).unwrap();
+ let mut expected = Usage::default();
+ pa_core::session_engine::rlm_usage::attribute_child_usage(&mut expected, &child());
+ let confirmed = store
+ .append_child_usage_once("partial-a", &target, child(), None)
+ .unwrap();
+ assert!(matches!(confirmed, ChildUsageAppendResult::Created(usage) if usage == expected));
+ let completed = std::fs::read(&store.path).unwrap();
+ assert!(completed.starts_with(&partial));
+ assert!(completed.len() > partial.len());
+ assert_eq!(attribution_rows(&store).len(), 1);
+ assert!(store.child_usage_recovery.pending.is_none());
+ assert!(store.child_usage_recovery.created.is_empty());
+ let c = store
+ .persist_entry(
+ "custom",
+ json!({"customType":"after-completion", "data":{}}),
+ )
+ .unwrap();
+ assert_eq!(
+ store.entry(&c).unwrap().parent_id.as_deref(),
+ Some("partial-a")
+ );
+ assert!(matches!(
+ store
+ .append_child_usage_once("partial-a", &target, child(), None)
+ .unwrap(),
+ ChildUsageAppendResult::Existing(_)
+ ));
+ assert_eq!(attribution_rows(&store).len(), 1);
+ assert!(std::fs::read(&store.path).unwrap().starts_with(&completed));
+}
+
+#[test]
+fn owned_tail_completion_obeys_a_managed_lease_and_refuses_a_released_one() {
+ let (dir, mut store, target) = fixture();
+ PARTIAL_CHILD_APPEND.with(|fault| fault.set(Some(12)));
+ FAIL_CHILD_TAIL_COMPLETION.with(|fault| fault.set(true));
+ assert!(store
+ .append_child_usage_once("leased-a", &target, child(), None)
+ .is_err());
+ FAIL_CHILD_TAIL_COMPLETION.with(|fault| fault.set(false));
+ let partial = std::fs::read(&store.path).unwrap();
+ let stale = std::sync::Arc::new(
+ crate::lease::acquire_runtime_session_lease(&store.path, dir.path()).unwrap(),
+ );
+ stale.release();
+ store.lease = Some(stale);
+ assert!(store
+ .append_child_usage_once("leased-a", &target, child(), None)
+ .is_err());
+ assert_eq!(std::fs::read(&store.path).unwrap(), partial);
+ store.lease = Some(std::sync::Arc::new(
+ crate::lease::acquire_runtime_session_lease(&store.path, dir.path()).unwrap(),
+ ));
+ assert!(matches!(
+ store
+ .append_child_usage_once("leased-a", &target, child(), None)
+ .unwrap(),
+ ChildUsageAppendResult::Created(_)
+ ));
+ assert!(std::fs::read(&store.path).unwrap().starts_with(&partial));
+ assert_eq!(attribution_rows(&store).len(), 1);
+ store.lease.take().unwrap().release();
+}
+
+#[test]
+fn admitted_confirmation_finishes_while_release_drains_and_later_writes_are_refused() {
+ let (dir, mut store, target) = fixture();
+ let lease = std::sync::Arc::new(
+ crate::lease::acquire_runtime_session_lease(&store.path, dir.path()).unwrap(),
+ );
+ store.lease = Some(std::sync::Arc::clone(&lease));
+ FAIL_AFTER_CHILD_APPEND.with(|fault| fault.set(true));
+ assert!(store
+ .append_child_usage_once("release-a", &target, child(), None)
+ .is_err());
+ assert_eq!(attribution_rows(&store).len(), 1);
+ let original_bytes = std::fs::read(&store.path).unwrap();
+ let (start_tx, start_rx) = std::sync::mpsc::sync_channel(1);
+ std::thread::scope(|scope| {
+ let releaser = std::sync::Arc::clone(&lease);
+ let release = scope.spawn(move || {
+ start_rx
+ .recv_timeout(std::time::Duration::from_secs(3))
+ .unwrap();
+ releaser.release();
+ });
+ let probe_lease = std::sync::Arc::clone(&lease);
+ let path = store.path.clone();
+ AFTER_CHILD_OBSERVE.with(|hook| {
+ *hook.borrow_mut() = Some(Box::new(move || {
+ start_tx.send(()).unwrap();
+ let deadline = std::time::Instant::now() + std::time::Duration::from_secs(2);
+ loop {
+ if probe_lease.with_owned_mutation(&path, || Ok(())).is_err() {
+ break;
+ }
+ assert!(
+ std::time::Instant::now() < deadline,
+ "release did not refuse new admission"
+ );
+ std::thread::yield_now();
+ }
+ }));
+ });
+ let result = store.append_child_usage_once("release-a", &target, child(), None);
+ release.join().unwrap();
+ let mut expected = Usage::default();
+ pa_core::session_engine::rlm_usage::attribute_child_usage(&mut expected, &child());
+ assert!(
+ matches!(result.unwrap(), ChildUsageAppendResult::Created(usage) if usage == expected)
+ );
+ });
+ assert!(store.child_usage_recovery.pending.is_none());
+ assert!(store.child_usage_recovery.created.is_empty());
+ assert_eq!(std::fs::read(&store.path).unwrap(), original_bytes);
+ assert!(store
+ .persist_entry("custom", json!({"customType":"after-release"}))
+ .is_err());
+ assert!(store
+ .append_child_usage_once("release-b", &target, child(), None)
+ .is_err());
+ assert_eq!(std::fs::read(&store.path).unwrap(), original_bytes);
+ let mut reopened = SessionFile::open(&store.path).unwrap();
+ assert!(matches!(
+ reopened
+ .append_child_usage_once("release-a", &target, child(), None)
+ .unwrap(),
+ ChildUsageAppendResult::Existing(_)
+ ));
+ assert_eq!(attribution_rows(&reopened).len(), 1);
+}
diff --git a/crates/pa-daemon/src/session_store/info.rs b/crates/pa-daemon/src/session_store/info.rs
index a833313af4..a18fda2394 100644
--- a/crates/pa-daemon/src/session_store/info.rs
+++ b/crates/pa-daemon/src/session_store/info.rs
@@ -104,8 +104,8 @@ impl SessionInfoGeneration {
}
}
-/// The cross-file memory budget on retained per-assistant-message usage
-/// records: keeps large families (~2k sessions, 150k entries) resident.
+/// The cross-file memory budget on retained assistant usage records and
+/// stable attribution IDs: keeps large families resident without unbounded retry history.
const SESSION_SCAN_MAX_RETAINED_USAGE_ENTRIES: usize = 400_000;
/// The cached-state count ceiling: the cap must sit well ABOVE a catalog
@@ -671,6 +671,7 @@ pub(super) fn fold_scan_entry(acc: &mut SessionScanAccumulator, raw: &str) -> Op
}
"child_usage_attributed" => {
acc.usage_scan.fold_child_attribution(
+ &entry.id,
entry.target_id.as_deref(),
entry.child_usage.map(Usage::from),
entry.aggregate_usage.map(Usage::from),
@@ -741,3 +742,104 @@ pub fn find_most_recent_session_for_cwd(session_dir: &Path, cwd: &str) -> Option
})
.map(|info| info.path)
}
+
+#[cfg(test)]
+mod cache_tests {
+ use super::*;
+
+ #[test]
+ fn cached_prefix_resume_subtracts_a_duplicate_attribution_once() {
+ use std::io::{BufReader, Seek, SeekFrom, Write};
+
+ let dir = tempfile::tempdir().unwrap();
+ let path = dir.path().join("isolated-prefix.jsonl");
+ let assistant = r#"{"type":"message","id":"a","timestamp":"2026-10-11T00:00:00.000Z","message":{"role":"assistant","timestamp":1790110000000,"usage":{"input":100,"output":10,"totalTokens":110}}}"#;
+ let attribution = r#"{"type":"child_usage_attributed","id":"report","timestamp":"2026-10-11T00:00:00.000Z","targetId":"a","childUsage":{"input":20,"output":2,"totalTokens":22},"aggregateUsage":{"input":120,"output":12,"totalTokens":132}}"#;
+ for row in [assistant, attribution] {
+ serde_json::from_str::<SessionInfoEntry<'_>>(row).unwrap();
+ }
+ let prefix = format!("{assistant}\n{attribution}\n");
+ fs::write(&path, &prefix).unwrap();
+ let mut file = fs::File::open(&path).unwrap();
+ let generation = SessionInfoGeneration::from_metadata(&file.metadata().unwrap());
+ let mut state = SessionScanState::fresh(generation);
+ state
+ .scan_from_cursor(&mut BufReader::new(&mut file), generation.len)
+ .unwrap();
+ assert_eq!(state.offset, generation.len);
+ let expected = state.acc.usage_scan.totals();
+ let mut cache = SessionInfoScanCache::default();
+ cache.store_state(&path, state);
+ let cached = cache.states.get(&path).unwrap().clone_for_resume();
+ let mut resumed: SessionScanState =
+ serde_json::from_slice(&serde_json::to_vec(&cached).unwrap()).unwrap();
+ {
+ let mut append = fs::OpenOptions::new().append(true).open(&path).unwrap();
+ writeln!(append, "{attribution}").unwrap();
+ }
+ let grown_len = file.metadata().unwrap().len();
+ assert!(grown_len > cached.offset);
+ file.seek(SeekFrom::Start(resumed.offset)).unwrap();
+ resumed
+ .scan_from_cursor(&mut BufReader::new(&mut file), grown_len)
+ .unwrap();
+ assert_eq!(resumed.offset, grown_len);
+ assert_eq!(resumed.acc.usage_scan.totals(), expected);
+ assert_eq!(resumed.acc.usage_scan.retained_entries(), 2);
+ drop(file);
+ dir.close().unwrap();
+ }
+
+ #[test]
+ fn attribution_ids_share_the_retained_usage_budget_and_trigger_lru_eviction() {
+ let generation = SessionInfoGeneration {
+ len: 0,
+ dev: 0,
+ ino: 0,
+ mtime: 0,
+ mtime_ns: 0,
+ ctime: 0,
+ ctime_ns: 0,
+ };
+ let mut cache = SessionInfoScanCache::default();
+ let old_path = Path::new("old.jsonl");
+ let new_path = Path::new("new.jsonl");
+ let mut old = SessionScanState::fresh(generation);
+ old.acc
+ .usage_scan
+ .fold_message("assistant", Some("assistant"), Some(Usage::default()));
+ for index in 1..SESSION_SCAN_MAX_RETAINED_USAGE_ENTRIES {
+ let row = format!(
+ r#"{{"type":"child_usage_attributed","id":"report-{index}","timestamp":"2026-10-11T00:00:00.000Z","targetId":"assistant","childUsage":{{}},"aggregateUsage":{{}}}}"#
+ );
+ serde_json::from_str::<SessionInfoEntry<'_>>(&row).unwrap();
+ fold_scan_entry(&mut old.acc, &row).unwrap();
+ }
+ cache.store_state(old_path, old);
+ assert_eq!(
+ cache.retained_usage_entries,
+ SESSION_SCAN_MAX_RETAINED_USAGE_ENTRIES
+ );
+ let mut new = SessionScanState::fresh(generation);
+ new.acc
+ .usage_scan
+ .fold_message("assistant", Some("assistant"), Some(Usage::default()));
+ fold_scan_entry(&mut new.acc, r#"{"type":"child_usage_attributed","id":"new-report","timestamp":"2026-10-11T00:00:00.000Z","targetId":"assistant","childUsage":{},"aggregateUsage":{}}"#).unwrap();
+ cache.store_state(new_path, new);
+ assert_eq!(
+ cache.states.keys().cloned().collect::<Vec<_>>(),
+ vec![new_path.to_path_buf()]
+ );
+ assert_eq!(cache.retained_usage_entries, 2);
+ assert_eq!(
+ cache.order.values().cloned().collect::<Vec<_>>(),
+ vec![new_path.to_path_buf()]
+ );
+ assert_eq!(cache.ordinal_by_path.len(), 1);
+ cache.drop_state(new_path);
+ assert!(cache.states.is_empty());
+ assert_eq!(cache.retained_usage_entries, 0);
+ assert!(cache.order.is_empty());
+ assert!(cache.ordinal_by_path.is_empty());
+ }
+}
diff --git a/crates/pa-daemon/src/session_store/info_sidecar.rs b/crates/pa-daemon/src/session_store/info_sidecar.rs
index 2138a9ecd0..1d43264558 100644
--- a/crates/pa-daemon/src/session_store/info_sidecar.rs
+++ b/crates/pa-daemon/src/session_store/info_sidecar.rs
@@ -20,7 +20,7 @@ use super::info::{session_info_cache, SessionScanState};
/// [`SessionScanAccumulator`](super::info::SessionScanAccumulator) or
/// [`UsageScan`](crate::session_usage::UsageScan) - the persisted fold's
/// fields or semantics.
-const INFO_SIDECAR_VERSION: u32 = 1;
+const INFO_SIDECAR_VERSION: u32 = 2;
/// The on-disk envelope: the version gate plus the scan state.
#[derive(Serialize, Deserialize)]
diff --git a/crates/pa-daemon/src/session_store/read.rs b/crates/pa-daemon/src/session_store/read.rs
index 2b659798df..272932fc37 100644
--- a/crates/pa-daemon/src/session_store/read.rs
+++ b/crates/pa-daemon/src/session_store/read.rs
@@ -156,6 +156,7 @@ impl SessionFile {
leaf_id: None,
window: None,
lease: None,
+ child_usage_recovery: super::child_usage::ChildUsageRecovery::default(),
trace_upload: None,
anthropic_warning_shown: false,
skipped_lines: 0,
@@ -215,6 +216,7 @@ impl SessionFile {
leaf_id: None,
window: None,
lease: None,
+ child_usage_recovery: super::child_usage::ChildUsageRecovery::default(),
trace_upload: None,
anthropic_warning_shown,
skipped_lines: 0,
@@ -284,6 +286,8 @@ impl SessionFile {
full.leaf_id.clone_from(&self.leaf_id);
full.lease.clone_from(&self.lease);
full.trace_upload.clone_from(&self.trace_upload);
+ full.child_usage_recovery
+ .clone_from(&self.child_usage_recovery);
// The merged store's leaf is the window's leaf: re-derive the gate
// from the merged ACTIVE branch — the full open's own-leaf answer
// can disagree, a post-snapshot marker must hydrate, and an
@@ -312,6 +316,7 @@ impl SessionFile {
leaf_id: None,
window: None,
lease: None,
+ child_usage_recovery: super::child_usage::ChildUsageRecovery::default(),
trace_upload: None,
anthropic_warning_shown: false,
skipped_lines: 0,
diff --git a/crates/pa-daemon/src/session_store/tests.rs b/crates/pa-daemon/src/session_store/tests.rs
index 5a971e88e4..e748855833 100644
--- a/crates/pa-daemon/src/session_store/tests.rs
+++ b/crates/pa-daemon/src/session_store/tests.rs
@@ -1286,3 +1286,55 @@ fn hydrating_full_history_preserves_the_trace_controller() {
&controller
));
}
+
+#[test]
+fn listing_and_frozen_usage_agree_on_duplicate_attribution_ids() {
+ let dir = temp_dir();
+ let mut session = SessionFile::create("/tmp", None, 0);
+ let path = dir.join(session_file_name(session.session_id()));
+ session.set_path(path.clone());
+ let usage = |input, output, cost| {
+ json!({
+ "input": input, "output": output, "cacheRead": 0, "cacheWrite": 0,
+ "totalTokens": input + output,
+ "cost": {"input": 0.0, "output": cost, "cacheRead": 0.0, "cacheWrite": 0.0, "total": cost}
+ })
+ };
+ let target = session.append_message(&json!({
+ "role": "assistant", "content": [], "timestamp": 1_790_110_000_000_u64,
+ "usage": usage(100u64, 10u64, 8.0)
+ }));
+ session.rewrite().unwrap();
+ let attributed = json!({
+ "type": "child_usage_attributed", "id": "report-1", "targetId": target,
+ "timestamp": "2026-10-11T00:00:00.000Z",
+ "childUsage": usage(20, 2, 2.0), "aggregateUsage": usage(120, 12, 10.0)
+ });
+ serde_json::from_str::<super::info::SessionInfoEntry<'_>>(&attributed.to_string()).unwrap();
+ let append = || {
+ use std::io::Write;
+ let mut file = fs::OpenOptions::new().append(true).open(&path).unwrap();
+ writeln!(file, "{attributed}").unwrap();
+ };
+ append();
+ let expected = Some(crate::session_usage::SessionUsageSummary {
+ input_tokens: 100,
+ output_tokens: 10,
+ cost: 8.0,
+ });
+ assert_eq!(read_session_info(&path).unwrap().usage, expected);
+ let prefix_len = fs::metadata(&path).unwrap().len();
+ // The appended duplicate must not subtract again in either scanner.
+ append();
+ let grown_len = fs::metadata(&path).unwrap().len();
+ assert!(grown_len > prefix_len);
+ assert_eq!(read_session_info(&path).unwrap().usage, expected);
+ assert_eq!(
+ crate::session_usage::read_session_usage(&path),
+ Some(crate::session_usage::SessionUsageTotals {
+ own: serde_json::from_value(usage(100, 10, 8.0)).unwrap(),
+ total: serde_json::from_value(usage(120, 12, 10.0)).unwrap(),
+ })
+ );
+ fs::remove_dir_all(dir).unwrap();
+}
diff --git a/crates/pa-daemon/src/session_store/write.rs b/crates/pa-daemon/src/session_store/write.rs
index 9692197a3a..7cb17d43fc 100644
--- a/crates/pa-daemon/src/session_store/write.rs
+++ b/crates/pa-daemon/src/session_store/write.rs
@@ -28,6 +28,10 @@ impl SessionEntry {
impl SessionFile {
/// Persist only the newly appended creation records on a resumed file.
pub(crate) fn persist_appended(&self, start: usize) -> Result<()> {
+ anyhow::ensure!(
+ self.child_usage_recovery.pending.is_none(),
+ "confirm unresolved attribution before persisting appended rows"
+ );
let mut bytes = Vec::new();
for entry in &self.entries[start..] {
write_line(&mut bytes, entry)?;
@@ -101,6 +105,10 @@ impl SessionFile {
/// Returns an error when the store only holds a window, or the write,
/// flush, sync, or rename fails; an empty path answers `Ok(())`.
pub fn rewrite(&self) -> Result<()> {
+ anyhow::ensure!(
+ self.child_usage_recovery.pending.is_none(),
+ "confirm unresolved attribution before rewriting session"
+ );
anyhow::ensure!(
self.window.is_none(),
"full history required before rewriting session"
@@ -190,10 +198,6 @@ impl SessionFile {
fields: Value,
timestamp: &str,
) -> Result<String> {
- anyhow::ensure!(
- self.window.is_none() || self.path.exists(),
- "window-backed session file is missing"
- );
let mut entry = SessionEntry::new(
entry_type,
self.leaf_id.clone(),
@@ -206,10 +210,48 @@ impl SessionFile {
if self.window.is_some() {
entry.id = uuid::Uuid::new_v4().to_string();
}
+ self.persist_built_entry(entry)
+ }
+
+ /// The durable-append core [`Self::persist_entry_at`] and the attribution
+ /// append-once share: the entry arrives fully built (its id included),
+ /// its line reaches the file first, and the entry joins the index only
+ /// after, so a failed append leaves no indexed row a reload would
+ /// resurface as the leaf.
+ ///
+ /// # Errors
+ ///
+ /// Returns an error when the store only holds a window without a file,
+ /// or the append, flush, sync, or the bootstrap rewrite fails.
+ pub(super) fn persist_built_entry(&mut self, mut entry: SessionEntry) -> Result<String> {
+ self.confirm_child_usage_write()?;
+ // A preflight recovery can advance the leaf after the caller built its
+ // generic row. Preserve that recovered parent instead of branching it away.
+ entry.parent_id.clone_from(&self.leaf_id);
+ anyhow::ensure!(
+ self.window.is_none() || self.path.exists(),
+ "window-backed session file is missing"
+ );
let id = entry.id.clone();
if !self.path.as_os_str().is_empty() && self.path.exists() {
let mut bytes = Vec::new();
write_line(&mut bytes, &entry)?;
+ #[cfg(test)]
+ if let Some(cut) = (entry.type_ == "child_usage_attributed")
+ .then(|| super::child_usage_tests::PARTIAL_CHILD_APPEND.with(std::cell::Cell::take))
+ .flatten()
+ {
+ // Isolated unleased fixture only: simulate write_all's partial
+ // prefix + Err postcondition, not a real injected OS failure.
+ anyhow::ensure!(
+ self.lease.is_none() && cut < bytes.len(),
+ "invalid partial append fixture"
+ );
+ let mut file = fs::OpenOptions::new().append(true).open(&self.path)?;
+ file.write_all(&bytes[..cut])?;
+ file.flush()?;
+ anyhow::bail!("injected partial attribution append error");
+ }
match &self.lease {
Some(lease) => lease.append(&self.path, &bytes)?,
None => pa_core::session::window::append_cached(
@@ -219,6 +261,13 @@ impl SessionFile {
)
.with_context(|| format!("append to {}", self.path.display()))?,
}
+ #[cfg(test)]
+ if entry.type_ == "child_usage_attributed"
+ && super::child_usage_tests::FAIL_AFTER_CHILD_APPEND
+ .with(|fault| fault.replace(false))
+ {
+ anyhow::bail!("injected post-append pre-index error");
+ }
// The line is in the file now: index it so the in-memory leaf matches what a
// reload sees.
self.push_index(entry);
@@ -230,8 +279,21 @@ impl SessionFile {
// indexed first; a failed rewrite rolls the index back. The live
// attribution fold is DEFERRED until the rewrite succeeds.
let previous_leaf = self.leaf_id.clone();
+ #[cfg(test)]
+ let entry_is_child = entry.type_ == "child_usage_attributed";
self.push_index_inner(entry, false);
- if let Err(error) = self.rewrite() {
+ let rewritten = self.rewrite();
+ #[cfg(test)]
+ let rewritten = rewritten.and_then(|()| {
+ if entry_is_child
+ && super::child_usage_tests::FAIL_AFTER_CHILD_APPEND
+ .with(|fault| fault.replace(false))
+ {
+ anyhow::bail!("injected post-bootstrap pre-index error");
+ }
+ Ok(())
+ });
+ if let Err(error) = rewritten {
self.by_id.remove(&id);
self.entries.pop();
self.leaf_id = previous_leaf;
diff --git a/crates/pa-daemon/src/session_store_info_tests.rs b/crates/pa-daemon/src/session_store_info_tests.rs
index 4feece8971..4bbf6b9079 100644
--- a/crates/pa-daemon/src/session_store_info_tests.rs
+++ b/crates/pa-daemon/src/session_store_info_tests.rs
@@ -72,6 +72,7 @@ fn legacy_read_session_info(path: &Path) -> Option<SessionInfo> {
})
};
usage_scan.fold_child_attribution(
+ &entry.id,
entry.fields.get("targetId").and_then(Value::as_str),
usage_field("childUsage"),
usage_field("aggregateUsage"),
diff --git a/crates/pa-daemon/src/session_usage.rs b/crates/pa-daemon/src/session_usage.rs
index 8f5979100e..32bbff1a77 100644
--- a/crates/pa-daemon/src/session_usage.rs
+++ b/crates/pa-daemon/src/session_usage.rs
@@ -259,14 +259,15 @@ pub struct UsageScan {
assistant_usage_by_id: AssistantUsageById,
#[serde(with = "usage_bits")]
attributed_child_usage: Usage,
+ attributed_child_ids: std::collections::HashSet<String>,
#[serde(with = "usage_bits")]
summarization_usage: Usage,
}
impl UsageScan {
- /// The per-assistant-message records the scan state keeps resident.
+ /// The assistant usage records and stable attribution IDs kept resident.
pub(crate) fn retained_entries(&self) -> usize {
- self.assistant_usage_by_id.len()
+ self.assistant_usage_by_id.len() + self.attributed_child_ids.len()
}
/// TS `foldSessionScanLine`: the raw assistant usage keyed by entry id (only an assistant
@@ -281,9 +282,12 @@ impl UsageScan {
}
/// A `child_usage_attributed` entry folds only when its target is already
- /// in the map; a malformed attribution contributes nothing.
+ /// in the map; a malformed attribution contributes nothing. Stable nonempty
+ /// row IDs subtract child spend once; legacy rows without an ID each fold.
+ /// Every valid row still replaces the target aggregate in file order.
pub(crate) fn fold_child_attribution(
&mut self,
+ row_id: &str,
target_id: Option<&str>,
child_usage: Option<Usage>,
aggregate_usage: Option<Usage>,
@@ -295,7 +299,9 @@ impl UsageScan {
};
if self.assistant_usage_by_id.contains(target_id) {
self.assistant_usage_by_id.set(target_id, aggregate_usage);
- add_assistant_usage(&mut self.attributed_child_usage, &child_usage);
+ if row_id.is_empty() || self.attributed_child_ids.insert(row_id.to_owned()) {
+ add_assistant_usage(&mut self.attributed_child_usage, &child_usage);
+ }
}
}
@@ -368,6 +374,7 @@ impl ScanEntry {
scan.fold_message(&self.id, role, usage);
}
"child_usage_attributed" => scan.fold_child_attribution(
+ &self.id,
self.target_id.as_deref(),
self.child_usage.map(Usage::from),
self.aggregate_usage.map(Usage::from),
@@ -475,6 +482,78 @@ mod tests {
})
}
+ #[test]
+ fn duplicate_attribution_ids_subtract_once_and_preserve_latest_aggregate() {
+ let assistant = message(
+ "a",
+ "assistant",
+ &serde_json::to_value(usage(100, 10, 8.0)).unwrap(),
+ );
+ let mut first = attribution("a", usage(20, 2, 2.0), usage(120, 12, 10.0));
+ first["id"] = json!("report-1");
+ let mut second = attribution("a", usage(30, 3, 4.0), usage(150, 15, 14.0));
+ second["id"] = json!("report-2");
+ let mut repeated = first.clone();
+ // Aggregate replacement still follows file order, even for a repeated ID.
+ repeated["aggregateUsage"] = serde_json::to_value(usage(160, 16, 16.0)).unwrap();
+ let mut full = UsageScan::default();
+ for row in [&assistant, &first] {
+ serde_json::from_value::<ScanEntry>(row.clone())
+ .unwrap()
+ .fold_into(&mut full);
+ }
+ let mut resumed: UsageScan =
+ serde_json::from_slice(&serde_json::to_vec(&full).unwrap()).unwrap();
+ for scan in [&mut full, &mut resumed] {
+ for row in [&second, &repeated] {
+ serde_json::from_value::<ScanEntry>(row.clone())
+ .unwrap()
+ .fold_into(scan);
+ }
+ assert_eq!(
+ scan.totals(),
+ SessionUsageTotals {
+ own: usage(110, 11, 10.0),
+ total: usage(160, 16, 16.0),
+ }
+ );
+ }
+ }
+
+ #[test]
+ fn orphan_ids_are_not_consumed_and_legacy_attributions_remain_distinct() {
+ let mut orphan = attribution("a", usage(20, 2, 2.0), usage(120, 12, 10.0));
+ orphan["id"] = json!("report-1");
+ let assistant = message(
+ "a",
+ "assistant",
+ &serde_json::to_value(usage(100, 10, 8.0)).unwrap(),
+ );
+ let missing_id = attribution("a", usage(30, 3, 4.0), usage(150, 15, 14.0));
+ let mut empty_id = attribution("a", usage(40, 4, 8.0), usage(190, 19, 22.0));
+ empty_id["id"] = json!("");
+ let mut scan = UsageScan::default();
+ for row in [
+ orphan.clone(),
+ assistant,
+ orphan.clone(),
+ orphan,
+ missing_id,
+ empty_id,
+ ] {
+ serde_json::from_value::<ScanEntry>(row)
+ .unwrap()
+ .fold_into(&mut scan);
+ }
+ assert_eq!(
+ scan.totals(),
+ SessionUsageTotals {
+ own: usage(100, 10, 8.0),
+ total: usage(190, 19, 22.0),
+ }
+ );
+ }
+
/// The latest aggregate replaces the raw block, every child block
/// accumulates, and the summary subtracts the child spend (no double count).
#[test]
diff --git a/crates/pa-daemon/src/state_getters/usage.rs b/crates/pa-daemon/src/state_getters/usage.rs
index 0da399abd7..ddd008521e 100644
--- a/crates/pa-daemon/src/state_getters/usage.rs
+++ b/crates/pa-daemon/src/state_getters/usage.rs
@@ -82,6 +82,20 @@ fn subtract_usage(total: &mut Value, usage: &Value) {
}
}
+/// The attribution rows own usage counts, each id once: a retried durable
+/// append can land the same id twice (bytes written, fsync errored), and the
+/// second row must not subtract its batch again. First occurrence wins; a row
+/// without an id carries no identity to dedupe by.
+fn unique_attribution_rows(
+ all_entries: &[crate::session_store::SessionEntry],
+) -> impl Iterator<Item = &crate::session_store::SessionEntry> + '_ {
+ let mut seen = std::collections::HashSet::new();
+ all_entries.iter().filter(move |entry| {
+ entry.type_ == "child_usage_attributed"
+ && (entry.id.is_empty() || seen.insert(entry.id.clone()))
+ })
+}
+
/// The branch's cumulative assistant usage (`totalUsage`) minus the child
/// usage attributions targeting those assistants (`ownUsage`). Totals stay
/// cumulative across compactions — compaction shrinks the context, not the spend.
@@ -109,10 +123,7 @@ pub(crate) fn compute_own_and_total_usage(
}
}
let mut own = total.clone();
- for entry in all_entries {
- if entry.type_ != "child_usage_attributed" {
- continue;
- }
+ for entry in unique_attribution_rows(all_entries) {
let Some(target_id) = entry.fields.get("targetId").and_then(Value::as_str) else {
continue;
};
@@ -233,10 +244,7 @@ pub(crate) fn compute_own_usage_by_model(
if unresolved {
return None;
}
- for entry in all_entries {
- if entry.type_ != "child_usage_attributed" {
- continue;
- }
+ for entry in unique_attribution_rows(all_entries) {
let Some(target_id) = entry.fields.get("targetId").and_then(Value::as_str) else {
continue;
};
@@ -317,11 +325,10 @@ mod usage_tests {
.collect(),
};
let assistant_entry = entry(&assistant, "m1");
- let attribution_entry = entry(&attribution, "a1");
let entries = vec![
assistant_entry.clone(),
- attribution_entry.clone(),
- attribution_entry.clone(),
+ entry(&attribution, "a1"),
+ entry(&attribution, "a2"),
];
let branch_refs: Vec<&crate::session_store::SessionEntry> = entries.iter().collect();
let (own, total) = compute_own_and_total_usage(&branch_refs, &entries);
@@ -331,11 +338,11 @@ mod usage_tests {
// Subtraction clamps at zero instead of going negative (TS attribution-drift guard).
let entries_more = vec![
assistant_entry,
- attribution_entry.clone(),
- attribution_entry.clone(),
- attribution_entry.clone(),
- attribution_entry.clone(),
- attribution_entry,
+ entry(&attribution, "a1"),
+ entry(&attribution, "a2"),
+ entry(&attribution, "a3"),
+ entry(&attribution, "a4"),
+ entry(&attribution, "a5"),
];
let branch_refs_more: Vec<&crate::session_store::SessionEntry> =
entries_more.iter().collect();
@@ -344,6 +351,56 @@ mod usage_tests {
assert_eq!(own["cost"]["total"].as_f64(), Some(0.0));
}
+ /// A duplicated attribution id — a retried durable append whose bytes
+ /// landed but whose fsync errored — is subtracted once: own usage keeps
+ /// one batch, the total keeps its last-wins aggregate.
+ #[test]
+ fn a_duplicated_attribution_id_is_subtracted_once() {
+ let assistant = json!({
+ "type": "message", "id": "m1",
+ "message": {
+ "role": "assistant", "content": "text",
+ "usage": {
+ "input": 100, "output": 10, "cacheRead": 0, "cacheWrite": 0,
+ "totalTokens": 110,
+ "cost": { "input": 1, "output": 2, "cacheRead": 0, "cacheWrite": 0, "total": 3 },
+ },
+ },
+ });
+ let attribution = json!({
+ "type": "child_usage_attributed", "id": "a1",
+ "targetId": "m1",
+ "childUsage": {
+ "input": 40, "output": 5, "cacheRead": 0, "cacheWrite": 0,
+ "totalTokens": 45,
+ "cost": { "input": 0.5, "output": 1, "cacheRead": 0, "cacheWrite": 0, "total": 1.5 },
+ },
+ });
+ let entry = |value: &Value, id: &str| crate::session_store::SessionEntry {
+ type_: value["type"].as_str().expect("type").to_string(),
+ id: id.to_string(),
+ parent_id: None,
+ timestamp: "2024-01-01T00:00:00.000Z".to_string(),
+ fields: value
+ .as_object()
+ .expect("object")
+ .clone()
+ .into_iter()
+ .collect(),
+ };
+ // The retry's second row carries the same id and the same bytes.
+ let entries = vec![
+ entry(&assistant, "m1"),
+ entry(&attribution, "a1"),
+ entry(&attribution, "a1"),
+ ];
+ let branch: Vec<&crate::session_store::SessionEntry> = entries.iter().collect();
+ let (own, total) = compute_own_and_total_usage(&branch, &entries);
+ assert_eq!(total["input"], json!(100));
+ assert_eq!(own["input"], json!(60), "the duplicated id subtracts once");
+ assert_eq!(own["cost"]["total"].as_f64(), Some(1.5));
+ }
+
/// A `branch_summary` row served by an auxiliary model (TS #2411) bills on
/// the model its entry records, not the timeline's current model.
#[test]
diff --git a/crates/pa-daemon/src/worker.rs b/crates/pa-daemon/src/worker.rs
index 6dc4255bf1..5aaa081b64 100644
--- a/crates/pa-daemon/src/worker.rs
+++ b/crates/pa-daemon/src/worker.rs
@@ -448,6 +448,8 @@ impl Worker {
// The in-run autonomous continuation seam: the hook holds
// itself weakly through the registered arc.
concrete.register_arc();
+ concrete
+ .set_rlm_usage_store(std::sync::Arc::new(CoreUsageStore(Arc::clone(&core))));
let sink_core = Arc::clone(&core);
let sink_notify = Arc::clone(&work_notify);
let autonomous_sink: crate::agent_engine::AutonomousAdmission = {
@@ -950,6 +952,73 @@ fn emit_refinement_row(
true
}
+struct CoreUsageStore(Arc<Mutex<SessionCore>>);
+
+fn missing_usage_target(target_id: &str) -> std::io::Error {
+ std::io::Error::new(
+ std::io::ErrorKind::InvalidInput,
+ format!("Assistant message entry {target_id} not found"),
+ )
+}
+
+fn assistant_message(entry: &crate::session_store::SessionEntry) -> Option<&Value> {
+ if entry.type_ != "message" {
+ return None;
+ }
+ let message = entry.fields.get("message")?;
+ (crate::types::message_role(message) == Some("assistant")).then_some(message)
+}
+
+impl pa_core::session_engine::rlm_usage::RlmChildUsageStore for CoreUsageStore {
+ fn last_assistant(
+ &self,
+ ) -> pa_core::session_engine::rlm_usage::RlmChildUsageFuture<
+ '_,
+ Option<(String, pa_types::ai::Usage)>,
+ > {
+ let core = Arc::clone(&self.0);
+ Box::pin(async move {
+ let core = core.lock().unwrap();
+ let store = core.store.as_ref()?;
+ store.entries().iter().rev().find_map(|entry| {
+ let message = assistant_message(entry)?;
+ Some((
+ entry.id.clone(),
+ message
+ .get("usage")
+ .cloned()
+ .and_then(|usage| serde_json::from_value(usage).ok())
+ .unwrap_or_default(),
+ ))
+ })
+ })
+ }
+
+ fn append_attribution(
+ &self,
+ row_id: &str,
+ target_id: &str,
+ child_usage: pa_types::ai::Usage,
+ origin: Option<pa_types::session::ChildUsageOrigin>,
+ ) -> pa_core::session_engine::rlm_usage::RlmChildUsageFuture<
+ '_,
+ std::io::Result<pa_core::session_engine::rlm_usage::ChildUsageAppendResult>,
+ > {
+ let core = Arc::clone(&self.0);
+ let row_id = row_id.to_owned();
+ let target_id = target_id.to_owned();
+ Box::pin(async move {
+ let mut core = core.lock().unwrap();
+ let Some(store) = core.store.as_mut() else {
+ return Err(missing_usage_target(&target_id));
+ };
+ store
+ .append_child_usage_once(&row_id, &target_id, child_usage, origin)
+ .map_err(|error| std::io::Error::other(format!("{error:#}")))
+ })
+ }
+}
+
/// Whether one queued item is a minted goal-context turn (TS's
/// `_clearQueuedGoalContexts` predicate on the injected custom row).
fn is_goal_context_item(item: &QueuedItem) -> bool {
diff --git a/crates/pa-daemon/tests/agent_family_e2e.rs b/crates/pa-daemon/tests/agent_family_e2e.rs
index 0d736cb382..90f80871b3 100644
--- a/crates/pa-daemon/tests/agent_family_e2e.rs
+++ b/crates/pa-daemon/tests/agent_family_e2e.rs
@@ -1242,6 +1242,7 @@ async fn parent_renames_a_child_end_to_end() {
pa_core::session_engine::runtime_wiring::RlmWiring {
model_registry: None,
subagent_host: Some(Arc::clone(&children) as Arc<dyn RlmSubagentHost>),
+ usage_store: None,
progress_note_emit: None,
},
None,
diff --git a/crates/pa-daemon/tests/protocol_breadth_b10_b11.rs b/crates/pa-daemon/tests/protocol_breadth_b10_b11.rs
index 0b03061f2e..4a0477deb3 100644
--- a/crates/pa-daemon/tests/protocol_breadth_b10_b11.rs
+++ b/crates/pa-daemon/tests/protocol_breadth_b10_b11.rs
@@ -694,7 +694,7 @@ fn wave_b10_heartbeat_snapshot_fallback() {
// Freeze the worker: the supervisor's 5s forward cannot answer, and the last-good
// snapshot keeps the row in the merged catalog.
- let pid = worker_pid(&agent_dir);
+ let pid = worker_pid(&agent_dir, &session_id);
stop(pid);
client.send_command("s3", &json!({ "type": "heartbeats_list" }));
let response = client.read_response("s3");
@@ -759,7 +759,7 @@ fn wave_b10_single_worker_slice_is_byte_identical() {
// Freeze the worker: the served slices answer both catalogs fast, byte-identical
// to the fresh-forward responses.
- let pid = worker_pid(&agent_dir);
+ let pid = worker_pid(&agent_dir, &session_id);
stop(pid);
let started = Instant::now();
client.send_command("fresh2", &json!({ "type": "heartbeats_list" }));
@@ -828,8 +828,9 @@ fn wave_b10_catalog_slices_served_without_reforwarding() {
// Both slices serve without re-consulting: both workers frozen, the second list still
// answers fast.
- let pid_a = worker_pid(&agent_dir);
- let pid_b = second_worker_pid(&agent_dir, pid_a);
+ let pid_a = worker_pid(&agent_dir, &session_a);
+ let pid_b = worker_pid(&agent_dir, &session_b);
+ assert_ne!(pid_a, pid_b, "sessions must own distinct workers");
stop(pid_a);
stop(pid_b);
let started = Instant::now();
@@ -844,6 +845,20 @@ fn wave_b10_catalog_slices_served_without_reforwarding() {
"a re-forward to a frozen worker would hold the 5s timeout: {served:?}"
);
+ // A fresh observer has no earlier heartbeat broadcasts buffered and issues no
+ // catalog read that could refresh a slice. The pause broadcast below proves the
+ // supervisor has processed that mutation's generation invalidation.
+ let (mut observer, _hello) = Client::connect(&socket);
+ // Hello is written before this connection subscribes to broadcasts. A local,
+ // read-only RPC response proves the subscribed command loop is now running;
+ // it neither consults a worker nor refreshes or invalidates a catalog slice.
+ observer.send_command(
+ "observer-ready",
+ &json!({ "type": "update_restore_status" }),
+ );
+ let ready = observer.read_response("observer-ready");
+ assert_eq!(ready["success"], true, "{ready}");
+
// A worker-side mutation bumps that worker's generation: its slice is stale, so the
// next list consults it again (the frozen mutated worker fails, TS `failed`).
cont(pid_a);
@@ -857,6 +872,9 @@ fn wave_b10_catalog_slices_served_without_reforwarding() {
);
let response = client_b.read_response("pause");
assert_eq!(response["success"], true, "{response}");
+ observer.next_line_of_type(&mut std::collections::VecDeque::new(), "heartbeats_changed");
+ assert_eq!(worker_pid(&agent_dir, &session_a), pid_a);
+ assert_eq!(worker_pid(&agent_dir, &session_b), pid_b);
stop(pid_a);
stop(pid_b);
client_a.send_command("list3", &json!({ "type": "heartbeats_list" }));
@@ -876,41 +894,6 @@ fn heartbeat_prompts(response: &Value) -> Vec<String> {
.collect()
}
-/// A second resident worker's pid (distinct from `first`).
-fn second_worker_pid(agent_dir: &std::path::Path, first: u32) -> u32 {
- let deadline = Instant::now() + Duration::from_secs(10);
- loop {
- let mut pids = Vec::new();
- for entry in std::fs::read_dir(agent_dir.join("daemon-workers")).expect("dir") {
- let Ok(entry) = entry else { continue };
- for descriptor in std::fs::read_dir(entry.path()).expect("worker dir") {
- let Ok(descriptor) = descriptor else { continue };
- let path = descriptor.path();
- if path.extension().and_then(|ext| ext.to_str()) != Some("json") {
- continue;
- }
- let Ok(text) = std::fs::read_to_string(&path) else {
- continue;
- };
- let Ok(value): Result<Value, _> = serde_json::from_str(&text) else {
- continue;
- };
- if let Some(pid) = value["pid"].as_u64().filter(|pid| *pid as u32 != first) {
- pids.push(pid as u32);
- }
- }
- }
- if let Some(pid) = pids.first() {
- return *pid;
- }
- assert!(
- Instant::now() < deadline,
- "second worker descriptor never appeared"
- );
- std::thread::sleep(Duration::from_millis(20));
- }
-}
-
/// Create one scripted live session on `client` (the `scripted_session` helper's shape,
/// for a second session on a shared daemon).
fn create_scripted_session(
@@ -946,7 +929,7 @@ fn create_scripted_session(
}
/// The resident worker's pid from its persisted descriptor.
-fn worker_pid(agent_dir: &std::path::Path) -> u32 {
+fn worker_pid(agent_dir: &std::path::Path, active_session_id: &str) -> u32 {
let deadline = Instant::now() + Duration::from_secs(10);
loop {
let mut paths: Vec<PathBuf> = std::fs::read_dir(agent_dir.join("daemon-workers"))
@@ -959,17 +942,20 @@ fn worker_pid(agent_dir: &std::path::Path) -> u32 {
.filter(|path| path.extension().and_then(|ext| ext.to_str()) == Some("json"))
.collect();
paths.sort();
- if let Some(path) = paths.last() {
+ for path in paths {
let descriptor: Value =
serde_json::from_str(&std::fs::read_to_string(path).expect("descriptor read"))
.expect("descriptor parse");
- if let Some(pid) = descriptor["pid"].as_u64() {
+ if descriptor["rootActiveSessionId"] != active_session_id {
+ continue;
+ }
+ if let Some(pid) = descriptor["pid"].as_u64().filter(|pid| *pid > 0) {
return pid as u32;
}
}
assert!(
Instant::now() < deadline,
- "worker descriptor never appeared"
+ "matching worker descriptor never appeared"
);
std::thread::sleep(Duration::from_millis(50));
}
@@ -984,6 +970,25 @@ fn stop(pid: u32) {
.success(),
"SIGSTOP {pid} failed"
);
+ // kill's exit status acknowledges sending the signal, not completion of the
+ // worker's group stop. Observe the state before issuing any catalog command.
+ let deadline = Instant::now() + Duration::from_secs(10);
+ loop {
+ let output = Command::new("ps")
+ .args(["-p", &pid.to_string(), "-o", "stat="])
+ .output()
+ .expect("read worker process state");
+ assert!(output.status.success(), "cannot inspect worker {pid}");
+ let state = String::from_utf8_lossy(&output.stdout);
+ if state.trim_start().starts_with('T') {
+ break;
+ }
+ assert!(
+ Instant::now() < deadline,
+ "worker {pid} did not stop: {state}"
+ );
+ std::thread::sleep(Duration::from_millis(10));
+ }
}
fn cont(pid: u32) {
diff --git a/crates/pa-daemon/tests/rlm_child_usage_attribution_e2e.rs b/crates/pa-daemon/tests/rlm_child_usage_attribution_e2e.rs
new file mode 100644
index 0000000000..9d3e7851ca
--- /dev/null
+++ b/crates/pa-daemon/tests/rlm_child_usage_attribution_e2e.rs
@@ -0,0 +1,469 @@
+//! RLM child-usage attribution end to end (S2): a spawned child settles, and the
+//! `child_usage_attributed` row reaches the PARENT's durable transcript in the TS row
+//! shape, so live and resumed usage views fold the child's billable usage. Tests skip
+//! without a live kernel install.
+// Stack-resident futures by design; boxing for a lint tick is a perf regression.
+#![allow(clippy::large_futures)]
+// Fn length is a style gate, not correctness.
+#![allow(clippy::too_many_lines)]
+#![cfg(unix)]
+
+use std::io::{BufRead, BufReader, Write};
+use std::os::unix::net::UnixStream;
+use std::path::{Path, PathBuf};
+use std::process::{Child, Command, Stdio};
+use std::time::{Duration, Instant};
+
+use serde_json::{json, Value};
+
+struct Daemon {
+ child: Child,
+}
+
+impl Drop for Daemon {
+ fn drop(&mut self) {
+ let _ = self.child.kill();
+ let _ = self.child.wait();
+ }
+}
+
+// The timeout panic path cannot wait on the child; the test exits and reaps it.
+#[allow(clippy::zombie_processes)]
+fn spawn_supervisor(socket: &Path, agent_dir: &Path, kernel_python: &Path) -> Daemon {
+ let binary = env!("CARGO_BIN_EXE_pa-daemon");
+ let child = Command::new(binary)
+ .arg("supervisor")
+ .arg("--socket")
+ .arg(socket)
+ .arg("--agent-dir")
+ .arg(agent_dir)
+ .env("PRIME_AGENT_KERNEL_PYTHON", kernel_python)
+ // Hermetic agent dir: the ambient environment exports a real
+ // agent dir; point every fallback at the test sandbox instead.
+ .env("PRIME_AGENT_CODING_AGENT_DIR", agent_dir)
+ .env_remove("PRIME_API_KEY")
+ .stdout(Stdio::null())
+ .stderr(Stdio::null())
+ .env(
+ pa_daemon::worker::WORKER_SUPERVISOR_LOST_EXIT_MS_ENV,
+ "15000",
+ )
+ .spawn()
+ .expect("spawn pa-daemon supervisor");
+ Daemon { child }
+}
+
+/// The kernel Python; `PA_E2E_KERNEL_PYTHON` points at an explicit interpreter instead.
+fn kernel_python() -> Option<PathBuf> {
+ if let Some(explicit) = std::env::var_os("PA_E2E_KERNEL_PYTHON") {
+ let explicit = PathBuf::from(explicit);
+ assert!(
+ explicit.exists(),
+ "PA_E2E_KERNEL_PYTHON {} not found",
+ explicit.display()
+ );
+ return Some(explicit);
+ }
+ let candidate = PathBuf::from(std::env::var("HOME").map_or_else(
+ |_| "/home/ubuntu/.prime/agent/kernel-venv/bin/python".to_string(),
+ |home| format!("{home}/.prime/agent/kernel-venv/bin/python"),
+ ));
+ if candidate.exists() {
+ return Some(candidate);
+ }
+ eprintln!(
+ "kernel python {} not found; skipping live RLM child-usage e2e",
+ candidate.display()
+ );
+ None
+}
+
+fn wait_socket_ready(socket: &Path) {
+ let deadline = Instant::now() + Duration::from_secs(15);
+ loop {
+ if UnixStream::connect(socket).is_ok() {
+ return;
+ }
+ assert!(Instant::now() < deadline, "supervisor socket never came up");
+ std::thread::sleep(Duration::from_millis(20));
+ }
+}
+
+/// JSONL supervisor client (command envelopes, id-matched responses).
+struct Client {
+ reader: BufReader<UnixStream>,
+ writer: UnixStream,
+}
+
+impl Client {
+ fn connect(socket: &Path) -> (Self, Value) {
+ let stream = UnixStream::connect(socket).expect("connect supervisor");
+ let write_stream = stream.try_clone().expect("clone socket");
+ let mut client = Client {
+ reader: BufReader::new(stream),
+ writer: write_stream,
+ };
+ let hello = client.read_line();
+ (client, hello)
+ }
+
+ fn send_command(&mut self, id: &str, command: &Value) {
+ let envelope = json!({
+ "type": "command",
+ "id": id,
+ "protocol": { "name": "prime-agent.daemon", "version": 7 },
+ "command": command,
+ });
+ let mut line = serde_json::to_string(&envelope).expect("serialize command");
+ line.push('\n');
+ self.writer.write_all(line.as_bytes()).expect("send");
+ self.writer.flush().expect("flush");
+ }
+
+ fn read_line(&mut self) -> Value {
+ let mut line = String::new();
+ let deadline = Instant::now() + Duration::from_mins(2);
+ self.reader
+ .get_mut()
+ .set_read_timeout(Some(Duration::from_millis(100)))
+ .expect("set timeout");
+ loop {
+ match self.reader.read_line(&mut line) {
+ Ok(0) => panic!("supervisor closed the connection"),
+ Ok(_) if line.trim().is_empty() => line.clear(),
+ Ok(_) => return serde_json::from_str(line.trim()).expect("parse response line"),
+ Err(error) => {
+ assert!(
+ Instant::now() < deadline,
+ "timed out waiting for supervisor line: {error}"
+ );
+ }
+ }
+ }
+ }
+
+ fn read_response(&mut self, id: &str) -> Value {
+ let deadline = Instant::now() + Duration::from_mins(4);
+ loop {
+ assert!(Instant::now() < deadline, "no response for id {id}");
+ let line = self.read_line();
+ if line.get("id").and_then(Value::as_str) == Some(id) {
+ return line;
+ }
+ }
+ }
+}
+
+/// Run one turn (prompt + idle wait) on the session.
+fn run_turn(client: &mut Client, session_id: &str, message: &str, id: &str) {
+ client.send_command(
+ id,
+ &json!({ "type": "prompt", "activeSessionId": session_id, "message": message }),
+ );
+ let prompted = client.read_response(id);
+ assert_eq!(prompted["success"], true, "prompt failed: {prompted}");
+ let idle_id = format!("{id}-idle");
+ client.send_command(
+ &idle_id,
+ &json!({ "type": "wait_for_idle", "activeSessionId": session_id }),
+ );
+ let idle = client.read_response(&idle_id);
+ assert_eq!(idle["success"], true, "wait_for_idle failed: {idle}");
+}
+
+/// Poll for a kernel cell's receipt content (the cell writes its verdict;
+/// an empty read is the create-before-write window, not a verdict yet).
+fn await_receipt(receipt: &Path) -> String {
+ let deadline = Instant::now() + Duration::from_mins(1);
+ loop {
+ if let Ok(content) = std::fs::read_to_string(receipt) {
+ if !content.is_empty() {
+ return content;
+ }
+ }
+ assert!(
+ Instant::now() < deadline,
+ "kernel cell receipt never appeared at {}",
+ receipt.display()
+ );
+ std::thread::sleep(Duration::from_millis(50));
+ }
+}
+
+/// The kernel cell of the spawn turn: spawn one RLM child through the
+/// product `rlm.spawn` surface and record its child id + session dir.
+fn spawn_cell(receipt: &Path, error_receipt: &Path) -> String {
+ format!(
+ "import json, traceback\ntry:\n handle = await rlm.spawn(\"run the lane task\", name=\"kid\")\n open({receipt:?}, \"w\").write(json.dumps({{\"rlm_child_id\": handle.rlm_child_id, \"session_dir\": str(handle.session_dir)}}))\n print(handle.rlm_child_id)\nexcept Exception:\n open({error_receipt:?}, \"w\").write(traceback.format_exc())\n raise",
+ receipt = receipt.display().to_string(),
+ error_receipt = error_receipt.display().to_string(),
+ )
+}
+
+/// The parent's faux script whose turns run the spawn cell.
+fn write_parent_script(dir: &Path, first_cell: &str) -> PathBuf {
+ let script = dir.join("parent.json");
+ std::fs::write(
+ &script,
+ json!({
+ "engine": "faux",
+ "responses": [
+ { "content": [
+ { "type": "toolCall", "name": "ipython", "arguments": { "code": first_cell } },
+ ] },
+ { "text": "spawn turn done" },
+ ],
+ })
+ .to_string(),
+ )
+ .expect("write parent script");
+ script
+}
+
+/// Create a scripted parent session through the supervisor. The create's `childScript`
+/// (the harness seam for the TS inherited `sessionConfig`) makes every child scripted.
+fn create_parent(
+ client: &mut Client,
+ dir: &Path,
+ parent_script: &Path,
+ child_script: &Path,
+ id: &str,
+) -> Value {
+ let sessions_dir = dir.join("sessions");
+ std::fs::create_dir_all(&sessions_dir).expect("sessions dir");
+ client.send_command(
+ id,
+ &json!({
+ "type": "create",
+ "name": "parent",
+ "config": {
+ "cwd": dir.to_string_lossy(),
+ "sessionDir": sessions_dir.to_string_lossy(),
+ "script": parent_script.to_string_lossy(),
+ "childScript": child_script.to_string_lossy(),
+ },
+ }),
+ );
+ let created = client.read_response(id);
+ assert_eq!(created["success"], true, "create parent failed: {created}");
+ created["data"].clone()
+}
+
+/// The transcript rows of one session file, in file order.
+fn transcript_rows(path: &Path) -> Vec<Value> {
+ std::fs::read_to_string(path)
+ .unwrap_or_else(|_| panic!("read session file {}", path.display()))
+ .lines()
+ .filter_map(|line| serde_json::from_str::<Value>(line).ok())
+ .collect()
+}
+
+/// Every JSON number as a float, so a whole-value integer serializes equal
+/// to the same number computed as an f64 sum.
+fn numbers_as_f64(value: Value) -> Value {
+ match value {
+ Value::Number(number) => json!(number.as_f64().unwrap_or_default()),
+ Value::Array(items) => Value::Array(items.into_iter().map(numbers_as_f64).collect()),
+ Value::Object(fields) => Value::Object(
+ fields
+ .into_iter()
+ .map(|(key, value)| (key, numbers_as_f64(value)))
+ .collect(),
+ ),
+ other => other,
+ }
+}
+
+/// One usage fold: token fields add, cost fields add as floats.
+fn add_usage(base: &Value, add: &Value) -> Value {
+ let tokens = |usage: &Value, field: &str| usage.get(field).and_then(Value::as_u64).unwrap_or(0);
+ let cost = |usage: &Value, field: &str| {
+ usage
+ .get("cost")
+ .and_then(|cost| cost.get(field))
+ .and_then(Value::as_f64)
+ .unwrap_or(0.0)
+ };
+ json!({
+ "input": tokens(base, "input") + tokens(add, "input"),
+ "output": tokens(base, "output") + tokens(add, "output"),
+ "cacheRead": tokens(base, "cacheRead") + tokens(add, "cacheRead"),
+ "cacheWrite": tokens(base, "cacheWrite") + tokens(add, "cacheWrite"),
+ "totalTokens": tokens(base, "totalTokens"),
+ "cost": {
+ "input": cost(base, "input") + cost(add, "input"),
+ "output": cost(base, "output") + cost(add, "output"),
+ "cacheRead": cost(base, "cacheRead") + cost(add, "cacheRead"),
+ "cacheWrite": cost(base, "cacheWrite") + cost(add, "cacheWrite"),
+ "total": cost(base, "total") + cost(add, "total"),
+ },
+ })
+}
+
+/// The child's settled assistant usage: one foldable assistant row per
+/// one-reply scripted child (the spawn-task completion).
+fn settled_child_usage(child_file: &Path) -> Value {
+ let rows = transcript_rows(child_file);
+ let assistants: Vec<&Value> = rows
+ .iter()
+ .filter(|row| {
+ row["type"] == "message"
+ && row["message"]["role"] == "assistant"
+ && !matches!(
+ row["message"]["stopReason"].as_str(),
+ Some("error" | "aborted")
+ )
+ })
+ .collect();
+ let [assistant] = assistants[..] else {
+ panic!("the child holds one foldable assistant row: {assistants:?}");
+ };
+ assistant["message"]["usage"].clone()
+}
+
+/// A spawned child's billable usage reaches the parent's durable transcript as one
+/// `child_usage_attributed` row in the TS shape, targeting the durable assistant row
+/// that issued the spawn; a fresh reopen of the parent file folds the aggregate (the
+/// resume semantics).
+#[test]
+fn child_usage_attributed_row_reaches_the_parent_transcript() {
+ let Some(kernel_python) = kernel_python() else {
+ return;
+ };
+ let dir = tempfile::TempDir::new().expect("temp dir");
+ let agent_dir = dir.path().join("agent");
+ std::fs::create_dir_all(&agent_dir).expect("agent dir");
+ let socket = dir.path().join("supervisor.sock");
+ let receipts = dir.path().join("receipts");
+ std::fs::create_dir_all(&receipts).expect("receipts dir");
+ let spawn_receipt = receipts.join("spawn.json");
+ let spawn_error = receipts.join("spawn.error");
+
+ // A real-engine child over the faux provider: its assistant row carries
+ // estimated usage the parent attributes.
+ let child_script = dir.path().join("child.json");
+ std::fs::write(
+ &child_script,
+ json!({
+ "engine": "faux",
+ "responses": [ { "text": "child reply" } ],
+ })
+ .to_string(),
+ )
+ .expect("write child script");
+ let parent_script = write_parent_script(dir.path(), &spawn_cell(&spawn_receipt, &spawn_error));
+ let _daemon = spawn_supervisor(&socket, &agent_dir, &kernel_python);
+ wait_socket_ready(&socket);
+ let (mut client, hello) = Client::connect(&socket);
+ assert_eq!(hello["type"], "daemon_hello");
+ let parent = create_parent(&mut client, dir.path(), &parent_script, &child_script, "c1");
+ let parent_id = parent["activeSessionId"]
+ .as_str()
+ .or_else(|| parent["id"].as_str())
+ .expect("parent active session id")
+ .to_string();
+ let parent_session_file = parent["sessionFile"]
+ .as_str()
+ .expect("parent session file")
+ .to_string();
+
+ // Turn 1: the kernel cell spawns the child mid-turn through the
+ // product `rlm.spawn` surface.
+ run_turn(&mut client, &parent_id, "spawn the kid", "t1");
+ let spawned: Value =
+ serde_json::from_str(&await_receipt(&spawn_receipt)).expect("spawn receipt json");
+ assert!(
+ !spawn_error.exists(),
+ "the spawn cell failed: {}",
+ std::fs::read_to_string(&spawn_error).unwrap_or_default()
+ );
+ let child_session_dir = spawned["session_dir"]
+ .as_str()
+ .expect("child session dir")
+ .to_string();
+ let child_session_file = std::fs::read_dir(&child_session_dir)
+ .expect("child session dir")
+ .filter_map(std::result::Result::ok)
+ .map(|entry| entry.path())
+ .find(|path| {
+ path.extension().and_then(|extension| extension.to_str()) == Some("jsonl")
+ && pa_daemon::session_store::is_valid_session_file(path)
+ })
+ .expect("one child session file");
+
+ // Observable readiness: the attribution row lands in the parent's
+ // durable transcript once the child settles and its usage is observed.
+ let deadline = Instant::now() + Duration::from_mins(2);
+ let row = loop {
+ let row = transcript_rows(Path::new(&parent_session_file))
+ .into_iter()
+ .find(|row| row["type"] == "child_usage_attributed");
+ if let Some(row) = row {
+ break row;
+ }
+ assert!(
+ Instant::now() < deadline,
+ "the child_usage_attributed row never reached the parent transcript; parent rows: {:?}; child file: {}",
+ transcript_rows(Path::new(&parent_session_file)),
+ std::fs::read_to_string(&child_session_file).unwrap_or_default(),
+ );
+ std::thread::sleep(Duration::from_millis(200));
+ };
+
+ // The target names an assistant message row in the same file, and that
+ // row is the spawn's issuer (the ipython tool call that ran the cell).
+ let rows = transcript_rows(Path::new(&parent_session_file));
+ let target_id = row["targetId"].as_str().expect("targetId");
+ let target = rows
+ .iter()
+ .find(|entry| entry["id"] == json!(target_id))
+ .expect("the targetId names a row in the same file");
+ assert_eq!(target["type"], "message");
+ assert_eq!(target["message"]["role"], "assistant");
+ assert!(
+ target["message"]["content"]
+ .as_array()
+ .expect("content blocks")
+ .iter()
+ .any(|block| block["type"] == "toolCall" && block["name"] == "ipython"),
+ "the target is the spawning assistant row: {target}"
+ );
+
+ // The whole row (minus id/parentId/timestamp) matches the TS shape: the
+ // child's settled usage and the folded aggregate over the parent row.
+ let child_usage = settled_child_usage(&child_session_file);
+ let aggregate = add_usage(&target["message"]["usage"], &child_usage);
+ let expected = json!({
+ "type": "child_usage_attributed",
+ "targetId": target_id,
+ "childUsage": child_usage,
+ "aggregateUsage": aggregate,
+ "origin": "spawn_task",
+ });
+ let stripped = {
+ let mut object = row.as_object().expect("row object").clone();
+ for key in ["id", "parentId", "timestamp"] {
+ object.remove(key);
+ }
+ Value::Object(object)
+ };
+ assert_eq!(
+ numbers_as_f64(stripped),
+ numbers_as_f64(expected),
+ "the attribution row shape"
+ );
+
+ // A fresh reopen folds the aggregate into the target assistant row —
+ // the view a resumed parent replays from the file.
+ let store = pa_daemon::session_store::SessionFile::open(Path::new(&parent_session_file))
+ .expect("reopen parent file");
+ let folded = store
+ .entries()
+ .iter()
+ .find(|entry| entry.id == target_id)
+ .expect("the target row on reopen");
+ assert_eq!(
+ folded.fields["message"]["usage"], row["aggregateUsage"],
+ "the reopened store folds the aggregate into the target row"
+ );
+}