Skip to content

A self-improving RLM agent for coding workflows and long-running autonomous tasks.

Rust

[RSI, feature] Record child cancellation as a parent->child semantic edge (#3680)

Seth Karten71beca9478f5
diff --git a/crates/pa-core/src/session_engine/semantic_edges.rs b/crates/pa-core/src/session_engine/semantic_edges.rs
index 67a374ae1c..fffb9c9972 100644
--- a/crates/pa-core/src/session_engine/semantic_edges.rs
+++ b/crates/pa-core/src/session_engine/semantic_edges.rs
@@ -50,6 +50,27 @@ pub enum CompactionStatus {
Cancelled,
}
+/// Which parent-side action cancelled a child (the `ChildCancelled`
+/// line's actor): a user-facing tool delete, a user cancel, or the
+/// session-kill close walk. Daemon lifecycle closes (Shutdown/Replaced)
+/// are NOT parent actions and never record the line.
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
+pub enum ChildCancelledReason {
+ /// `rlm.delete_subagent` tore a live child down.
+ #[serde(rename = "parent_delete")]
+ ParentDelete,
+ /// The user cancelled the child's run (`cancel_rlm_child`).
+ #[serde(rename = "user_cancel")]
+ UserCancel,
+ /// The close walk killed the live child (a parent session kill).
+ #[serde(rename = "close:killed")]
+ CloseKilled,
+ /// A reason value this build does not know. The catch-all keeps a future
+ /// reason from failing the whole ledger read and disabling the recorder.
+ #[serde(other, rename = "unknown")]
+ Unknown,
+}
+
/// One ledger line. Field order matches the TS object literals so the
/// serialized bytes are identical (`JSON.stringify` insertion order).
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
@@ -87,6 +108,22 @@ pub enum SemanticEdgeLedgerEvent {
child_session_id: String,
request_id: String,
},
+ /// The parent cancelled a child mid-run (nano-rlm #202's
+ /// `subagent_cancel`): an action the PARENT took, so like a spawn
+ /// it is parent -> child, anchored to the cancelling request (the
+ /// parent's last turn); the child's cancelled state is derived from
+ /// this line, not carried on it. `reason` names which parent-side
+ /// action produced the line — one line covers three actors (`rlm.
+ /// delete_subagent`, the TUI user cancel, and the session-kill close
+ /// walk), and consumers filter on it; a daemon lifecycle close
+ /// (Shutdown/Replaced) records no line at all (it is not a parent
+ /// action, so an anchored cancel would be a training-data mislabel).
+ ChildCancelled {
+ session_id: String,
+ child_session_id: String,
+ request_id: String,
+ reason: ChildCancelledReason,
+ },
/// An event a newer writer appended (TS `JSON.parse` accepts any line
/// and the replay ignores unknown types): forward-compatible resume.
#[serde(other)]
@@ -156,6 +193,10 @@ struct RecorderState {
last_turn: Option<Turn>,
parked_retry: Option<Turn>,
open_compactions: HashSet<String>,
+ /// Children this parent already cancelled (the durable half of
+ /// nano-rlm #202's at-most-once `cancel_recorded` flag): a replay
+ /// rebuilds it from the ledger's `child_cancelled` lines.
+ cancelled_children: HashSet<String>,
}
/// Append-only semantic-edge recorder for one session (TS
@@ -336,6 +377,41 @@ impl SemanticEdgeRecorder {
},
);
}
+
+ /// The parent cancelled a child mid-run (nano-rlm #202's
+ /// `recordSubagentCancel`, the cancel twin of a spawn: an action the
+ /// PARENT took): a parent -> child cancel line anchored to the
+ /// parent's last request (the cancelling turn) and naming the
+ /// parent-side action in `reason` — a delete, a user cancel, or the
+ /// session-kill close walk; a daemon lifecycle close (Shutdown/
+ /// Replaced) never reaches here (it is not a parent action, so its
+ /// line would mislabel the run). Recorded at most once per child
+ /// across resumes. `None` anchors to nothing — a recorder with no
+ /// live request (no turn ever started, or the ledger disabled)
+ /// records nothing: an absent edge beats a wrong one.
+ pub fn record_child_cancelled(&self, child_session_id: &str, reason: ChildCancelledReason) {
+ let mut state = self.state();
+ let Some(request_id) = state.last_turn.as_ref().map(|turn| turn.request_id.clone()) else {
+ return;
+ };
+ if state.cancelled_children.contains(child_session_id) {
+ return;
+ }
+ let recorded = self.append(
+ &mut state,
+ &SemanticEdgeLedgerEvent::ChildCancelled {
+ session_id: self.session_id.clone(),
+ child_session_id: child_session_id.to_string(),
+ request_id,
+ reason,
+ },
+ );
+ if recorded {
+ state
+ .cancelled_children
+ .insert(child_session_id.to_string());
+ }
+ }
}
/// The last `request_finished` in a ledger: the parent's read of a child's
@@ -400,6 +476,17 @@ fn replay_event(state: &mut RecorderState, session_id: &str, event: &SemanticEdg
} if event_session_id == session_id => {
state.open_compactions.insert(compaction_id.clone());
}
+ // Restores the at-most-once cancel claim after a resume: a
+ // reopen must not append a second cancel line for a child the
+ // pre-restart session already cancelled.
+ SemanticEdgeLedgerEvent::ChildCancelled {
+ session_id: event_session_id,
+ child_session_id,
+ request_id: _,
+ reason: _,
+ } if event_session_id == session_id => {
+ state.cancelled_children.insert(child_session_id.clone());
+ }
SemanticEdgeLedgerEvent::CompactionFinished {
compaction_id,
status,
@@ -416,6 +503,7 @@ fn replay_event(state: &mut RecorderState, session_id: &str, event: &SemanticEdg
| SemanticEdgeLedgerEvent::SessionRegistered { .. }
| SemanticEdgeLedgerEvent::CompactionBegun { .. }
| SemanticEdgeLedgerEvent::ChildReturned { .. }
+ | SemanticEdgeLedgerEvent::ChildCancelled { .. }
| SemanticEdgeLedgerEvent::Unknown => {}
}
}
diff --git a/crates/pa-core/src/session_engine/semantic_edges/tests.rs b/crates/pa-core/src/session_engine/semantic_edges/tests.rs
index d27fe35af2..0c6969e604 100644
--- a/crates/pa-core/src/session_engine/semantic_edges/tests.rs
+++ b/crates/pa-core/src/session_engine/semantic_edges/tests.rs
@@ -154,6 +154,87 @@ fn a_parked_retry_reuses_its_id_only_for_an_identical_body_in_the_same_epoch() {
);
}
+/// The parent cancels a child mid-run: one parent -> child
+/// `child_cancelled` line anchored to the cancelling request, recorded at
+/// most once per child — including across a reopen, whose replay rebuilds
+/// the at-most-once claim (nano-rlm #202's `recordSubagentCancel`).
+#[test]
+fn a_child_cancel_is_ledgered_once_anchored_to_the_cancelling_request() {
+ let (identity, _dir) = identity("session-h");
+ let recorder = SemanticEdgeRecorder::open(identity.clone());
+ let request_id = recorder
+ .start_turn_request(FINGERPRINT_A)
+ .expect("the recorder mints ids");
+ let cancel = SemanticEdgeLedgerEvent::ChildCancelled {
+ session_id: "session-h".to_string(),
+ child_session_id: "child-session".to_string(),
+ request_id: request_id.clone(),
+ reason: super::ChildCancelledReason::UserCancel,
+ };
+ // The same child cancelled twice records one line.
+ recorder.record_child_cancelled("child-session", super::ChildCancelledReason::UserCancel);
+ recorder.record_child_cancelled("child-session", super::ChildCancelledReason::UserCancel);
+ assert_eq!(
+ read_ledger(&identity),
+ vec![
+ session_registered("session-h"),
+ request_started("session-h", &request_id),
+ cancel.clone(),
+ ]
+ );
+ // A reopen replays the cancel line, so a later cancel of the same
+ // child records nothing.
+ drop(recorder);
+ let reopened = SemanticEdgeRecorder::open(identity.clone());
+ reopened.record_child_cancelled("child-session", super::ChildCancelledReason::UserCancel);
+ // A different child still records, anchored to the replayed last turn.
+ reopened.record_child_cancelled("child-other", super::ChildCancelledReason::ParentDelete);
+ assert_eq!(
+ read_ledger(&identity),
+ vec![
+ session_registered("session-h"),
+ request_started("session-h", &request_id),
+ cancel,
+ SemanticEdgeLedgerEvent::ChildCancelled {
+ session_id: "session-h".to_string(),
+ child_session_id: "child-other".to_string(),
+ request_id,
+ reason: super::ChildCancelledReason::ParentDelete,
+ },
+ ]
+ );
+}
+
+/// A cancel with no live request anchors to nothing: a recorder that
+/// never started a turn, or one whose ledger is disabled, records no
+/// cancel (an absent edge beats a wrong one).
+#[test]
+fn a_child_cancel_without_a_live_request_records_nothing() {
+ let (identity, _dir) = identity("session-i");
+ let recorder = SemanticEdgeRecorder::open(identity.clone());
+ // No turn ever started: nothing anchors the cancel.
+ recorder.record_child_cancelled("child-session", super::ChildCancelledReason::UserCancel);
+ assert_eq!(
+ read_ledger(&identity),
+ vec![session_registered("session-i")]
+ );
+ // A live turn first, then a directory at the ledger path fails the
+ // cancel's append and permanently disables the recorder (works as
+ // root too).
+ recorder
+ .start_turn_request(FINGERPRINT_A)
+ .expect("the recorder mints ids");
+ let path = identity.ledger_path.as_ref().unwrap();
+ std::fs::remove_file(path).unwrap();
+ std::fs::create_dir(path).unwrap();
+ recorder.record_child_cancelled("child-session", super::ChildCancelledReason::UserCancel);
+ std::fs::remove_dir(path).unwrap();
+ // The disabled recorder has no last turn: even a fresh child records
+ // no cancel.
+ recorder.record_child_cancelled("child-other", super::ChildCancelledReason::UserCancel);
+ assert_eq!(read_ledger(&identity), Vec::new());
+}
+
#[tokio::test]
async fn split_summary_slices_commit_in_resolve_order() {
let (identity, _dir) = identity("session-g");
diff --git a/crates/pa-daemon/src/agent_engine/tests/goal.rs b/crates/pa-daemon/src/agent_engine/tests/goal.rs
index 79f2a8e52a..5890270b2c 100644
--- a/crates/pa-daemon/src/agent_engine/tests/goal.rs
+++ b/crates/pa-daemon/src/agent_engine/tests/goal.rs
@@ -776,9 +776,14 @@ fn running_children_owe_the_continuation_and_settle_delivers_it() {
assert!(engine
.runtime
.block_on(async { handles.driver.lock().await.owes_continuation() }));
+ // Settle the child through the settle funnel: a user cancel only settles
+ // a child whose abort is confirmed, and this synthetic child has no live
+ // supervisor, so the cancel could never confirm (PR #3680 review; the
+ // seam change that fires the funnel lives in #3715). The continuation is
+ // owed until the child SETTLES, which is what this drives.
engine
.runtime
- .block_on(async { children.cancel_child_run("child-1").await });
+ .block_on(async { children.settle_test_child("child-session").await });
for _ in 0..200 {
if !goal_work.lock().unwrap().is_empty() {
break;
diff --git a/crates/pa-daemon/src/rlm_children.rs b/crates/pa-daemon/src/rlm_children.rs
index ba35bc9533..02d9cf72cb 100644
--- a/crates/pa-daemon/src/rlm_children.rs
+++ b/crates/pa-daemon/src/rlm_children.rs
@@ -198,6 +198,13 @@ struct ChildRecord {
/// The parent session closed while this child ran: the settle watcher
/// exits without a notice — none is owed to a session being torn down.
closed_by_parent: bool,
+ /// A parent delete is in flight for this child: a settle watcher that
+ /// wakes inside the kill window must not record a clean
+ /// `child_returned` (the delete records a cancel edge instead). Kept
+ /// apart from `settled_status` so the live run is never faked settled:
+ /// the watcher stays alive and the row stays correctly reported, which
+ /// also lets a failed kill simply clear the mark.
+ retiring: bool,
/// Confirmed retirement completed its roster, notice and settle bookkeeping.
retirement_finished: bool,
/// Claim the synchronous delete notifier once across interrupted retirement retries.
@@ -699,14 +706,23 @@ impl SupervisorChildSessions {
/// registry).
#[cfg(test)]
pub(crate) async fn settle_test_child(&self, child_active_session_id: &str) {
- let children = self.inner.children.lock().await;
- for record in children.iter() {
- let mut record = record.lock().await;
- if record.active_session_id == child_active_session_id {
- record.settled_status = Some("done");
- // The settle funnel's flag: the quiescence predicate reads it, not
- // the terminal status.
- record.settled = true;
+ let children = self.inner.children.lock().await.clone();
+ for record in children {
+ let matched = {
+ let mut record = record.lock().await;
+ let matched = record.active_session_id == child_active_session_id;
+ if matched {
+ record.settled_status = Some("done");
+ // The settle funnel's flag: the quiescence predicate reads it, not
+ // the terminal status.
+ record.settled = true;
+ }
+ matched
+ };
+ if matched {
+ // The settle FUNNEL, not just the flags: the goal
+ // continuation is released by the hook a real settle fires.
+ self.inner.fire_settle_hook(&record).await;
}
}
}
@@ -742,6 +758,7 @@ impl SupervisorChildSessions {
prompt_admitted: true,
error: None,
closed_by_parent: false,
+ retiring: false,
retirement_finished: false,
retirement_notified: false,
retirement_roster_emitted: false,
diff --git a/crates/pa-daemon/src/rlm_children/host.rs b/crates/pa-daemon/src/rlm_children/host.rs
index 25f526984b..e868016246 100644
--- a/crates/pa-daemon/src/rlm_children/host.rs
+++ b/crates/pa-daemon/src/rlm_children/host.rs
@@ -173,6 +173,7 @@ impl RlmSubagentHost for SupervisorChildSessions {
prompt_admitted: false,
error: None,
closed_by_parent: false,
+ retiring: false,
retirement_finished: false,
retirement_notified: false,
retirement_roster_emitted: false,
@@ -506,14 +507,37 @@ impl RlmSubagentHost for SupervisorChildSessions {
]),
};
drop(record_guard);
- let was_running = record.lock().await.settled_status.is_none();
+ let was_running = {
+ // A delete of a LIVE run must not let a settle watcher that
+ // wakes inside the kill window append a clean
+ // `child_returned` beside the cancel line. The gate is this
+ // mark, not a fake settle: a fake settle would make the
+ // watcher treat the live run as finished and exit, which a
+ // failed kill would then strand (a live, unwatched child).
+ // A settled child's delete records no cancel line, so it
+ // leaves its return alone.
+ let mut row = record.lock().await;
+ let was_running = row.settled_status.is_none();
+ if was_running {
+ row.retiring = true;
+ }
+ was_running
+ };
// Capture before the kill: a deleted running child's durable
// rows are its last observable spend on the parent side.
if !already_terminated {
this.emit_child_usage(&record).await;
- this.command(&command, KILL_TIMEOUT_MS)
+ if let Err(error) = this
+ .command(&command, KILL_TIMEOUT_MS)
.await
- .with_context(|| format!("kill RLM child \"{target}\""))?;
+ .with_context(|| format!("kill RLM child \"{target}\""))
+ {
+ // The kill never landed: clear the mark so the
+ // still-live child stays watchable and a later cancel
+ // can still record the parent -> child edge.
+ record.lock().await.retiring = false;
+ return Err(error);
+ }
this.retain_retired_usage(&record, super::RetirementIntent::Delete { was_running });
// Observe any rows that landed before the acknowledged kill.
this.emit_child_usage(&record).await;
diff --git a/crates/pa-daemon/src/rlm_children/lifecycle.rs b/crates/pa-daemon/src/rlm_children/lifecycle.rs
index 7e4a953f0a..62d4047557 100644
--- a/crates/pa-daemon/src/rlm_children/lifecycle.rs
+++ b/crates/pa-daemon/src/rlm_children/lifecycle.rs
@@ -1,6 +1,8 @@
//! The child-lifecycle concern: session creation (create/launch), the
//! prompt/kill/close routing, settle watching with its notices, and the
//! spawn-admission outbox types (`CreatedSessionIds`, `CreatedChild`).
+use pa_core::session_engine::semantic_edges::ChildCancelledReason;
+
use super::{
anyhow, cap_text, compact_rlm_text, create_rlm_child_failure_message,
create_rlm_child_terminal_notice, json, now_ms, Arc, ChildCloseReason, ChildObservationSource,
@@ -500,6 +502,22 @@ impl SupervisorChildSessionsInner {
self.refresh_record(record).await;
let settled = record.lock().await.settled_status.is_some();
if settled {
+ // A delete in flight (`retiring`) owns this run's terminal
+ // edge: the watcher must not run the settle tail into the
+ // delete's kill window. The tail would record nothing
+ // (`record_child_return` holds the mark), deliver the
+ // terminal notice and exit, so a failed kill would strand
+ // a settled row with no watcher and no ledger line, and a
+ // confirmed kill would race its cancel line with the
+ // notice, the goal hook and the display flip. Park
+ // instead: a failed kill clears the mark and the loop
+ // settles the run normally, and a confirmed kill closes
+ // the record (`closed_by_parent` ends the watch at the
+ // loop top).
+ if record.lock().await.retiring {
+ tokio::time::sleep(Duration::from_millis(WATCH_POLL_INTERVAL_MS)).await;
+ continue;
+ }
// Stability re-check: a prompt admitted to an idle worker
// can read idle once between the admission and the turn
// pop (the queue snapshot and the busy flag change under
@@ -508,9 +526,24 @@ impl SupervisorChildSessionsInner {
// (a queued continuation) keeps watching. An unreachable
// worker (an idle passivation, a crash) leaves the settled
// verdict standing, as in `refresh_record`.
+ //
+ // The re-check wipes only a WATCHER-set settle (the
+ // premature `done` of the admission race): a
+ // registry-set `cancelled` (a user abort or a parent kill
+ // in flight, `cancel_child_run`/`close_children_inner`)
+ // stands — wiping it would let this watcher refresh the
+ // still-live run back to `done` and record a clean
+ // `child_returned` beside the cancel line, so the row
+ // keeps its registry settle and this loop keeps watching
+ // until the abort lands (the cancel path owns the
+ // terminal handling).
tokio::time::sleep(Duration::from_millis(WATCH_SETTLE_GRACE_MS)).await;
if matches!(self.child_busy(&active_session_id).await, Ok(true)) {
- record.lock().await.settled_status = None;
+ let mut record = record.lock().await;
+ if matches!(record.settled_status, Some("done")) {
+ record.settled_status = None;
+ }
+ drop(record);
continue;
}
self.refresh_record(record).await;
@@ -593,10 +626,12 @@ impl SupervisorChildSessionsInner {
/// committed request, read from the child's own ledger file (the
/// cross-process form of TS's in-process
/// `child.semanticEdges.lastCommittedRequestId`). Only a `done` or
- /// `error` settle returns — TS never records a cancelled run (its done
- /// arm throws on `run.error` and the record gates on `status ===
- /// "error"`). A child without a durable session id, or a parent
- /// without a recorder, records nothing.
+ /// `error` settle returns — a cancelled run never returns (it is the
+ /// parent's action, so it records through
+ /// [`Self::record_child_cancelled`] instead, nano-rlm #202) — and a
+ /// delete in flight ([`ChildRecord::retiring`]) returns nothing either.
+ /// A child without a durable session id, or a parent without a
+ /// recorder, records nothing.
async fn record_child_return(&self, record: &Arc<Mutex<ChildRecord>>) {
let recorder = self
.semantic_edges
@@ -608,7 +643,10 @@ impl SupervisorChildSessionsInner {
};
let (child_session_id, child_dir) = {
let record = record.lock().await;
- if !matches!(record.settled_status, Some("done" | "error")) {
+ // A delete in flight owns this child's terminal edge: the
+ // delete records a cancel line, so a watcher that wakes inside
+ // the kill window must not append a clean return beside it.
+ if record.retiring || !matches!(record.settled_status, Some("done" | "error")) {
return;
}
(record.session_id.clone(), record.session_dir.clone())
@@ -633,6 +671,34 @@ impl SupervisorChildSessionsInner {
recorder.record_child_returned(&child_session_id, last_committed);
}
+ /// Record a parent-cancelled child in the parent's semantic-edge
+ /// ledger (nano-rlm #202's `recordSubagentCancel`): the cancel twin
+ /// of a spawn — an action the PARENT took, anchored to the parent's
+ /// in-flight (cancelling) request, never a passive child return.
+ /// `reason` names the parent-side action (a delete, a user cancel,
+ /// the session-kill close walk) on the line; a daemon lifecycle
+ /// close never reaches here. A child without a durable session id,
+ /// or a parent without a recorder, records nothing.
+ pub(super) async fn record_child_cancelled(
+ &self,
+ record: &Arc<Mutex<ChildRecord>>,
+ reason: ChildCancelledReason,
+ ) {
+ let recorder = self
+ .semantic_edges
+ .lock()
+ .expect("semantic edges lock")
+ .clone();
+ let Some(recorder) = recorder else {
+ return;
+ };
+ let child_session_id = record.lock().await.session_id.clone();
+ let Some(child_session_id) = child_session_id else {
+ return;
+ };
+ recorder.record_child_cancelled(&child_session_id, reason);
+ }
+
/// Deliver the no-reply terminal notice for a settled child that never
/// sent an agent message to the parent. Exactly-once: the record's
/// notice claim collapses the races between the watcher, a natural
diff --git a/crates/pa-daemon/src/rlm_children/registry.rs b/crates/pa-daemon/src/rlm_children/registry.rs
index f95161a9cb..f2c0b938b5 100644
--- a/crates/pa-daemon/src/rlm_children/registry.rs
+++ b/crates/pa-daemon/src/rlm_children/registry.rs
@@ -9,6 +9,8 @@ use super::{
DeletedChild, Map, Mutex, Result, SupervisorChildSessionsInner, KILL_TIMEOUT_MS,
};
+use pa_core::session_engine::semantic_edges::ChildCancelledReason;
+
/// The already-gone marker inside a close failure: a close walking a
/// child that died earlier must not fail (a missing child session is a completed no-op).
fn unknown_session(error: &anyhow::Error) -> Option<()> {
@@ -46,8 +48,16 @@ impl SupervisorChildSessionsInner {
/// Abort one child's live run (see
/// [`SupervisorChildSessions::cancel_child_run`]): claim the notice,
- /// settle the row as `cancelled`, then abort the worker's in-flight
- /// turn (best-effort; the registry is the user-visible state).
+ /// settle the row as `cancelled` (so a parked watcher cannot append a
+ /// clean `child_returned` beside the cancel line), then abort the
+ /// worker's in-flight turn (best-effort; the registry is the
+ /// user-visible state). A refused abort is undone wholesale: the child
+ /// never got the command, so the row stays live and un-settled, the
+ /// terminal notice claim restores its pre-cancel value, a later
+ /// confirmed cancel can still record the edge — and the answer is
+ /// `false` (the `cancel_rlm_child` wire contract reports the
+ /// confirmed effect, so a user whose cancel never reached the child
+ /// can retry).
pub(super) async fn cancel_child_run(&self, child_id: &str) -> bool {
let children = self.children.lock().await.clone();
for record in &children {
@@ -65,15 +75,17 @@ impl SupervisorChildSessionsInner {
if !matched || !running {
continue;
}
- {
+ let notice_was_delivered = {
let mut record = record.lock().await;
// The no-reply terminal notice is suppressed for a
// cancelled run; a settle watcher that already claimed it
// keeps its claim.
+ let notice_was_delivered = record.notice_delivered;
record.notice_delivered = true;
record.settled_status = Some("cancelled");
record.error = Some("Cancelled by user".to_string());
- }
+ notice_was_delivered
+ };
self.emit_child_update(record).await;
// Capture before the abort: the completed turns' usage (the
// aborted turn's partial row folds nowhere) must not die with
@@ -84,10 +96,50 @@ impl SupervisorChildSessionsInner {
active_session_id: active_session_id.clone(),
rest: Map::default(),
};
- let _ = self
+ let abort_result = self
.command(&abort, KILL_TIMEOUT_MS)
.await
.with_context(|| format!("abort RLM child session {active_session_id}"));
+ // The cancel line records the confirmed effect only (nano-rlm
+ // #202 records after its `terminate`): an abort that reached
+ // the child, or one whose session is already gone (a dead
+ // child is cancelled), leaves the line; a real abort failure
+ // keeps the edge absent, so a later successful cancel of the
+ // still-running child can still record it (the dedup never saw
+ // the failed attempt).
+ let cancelled_confirmed = abort_result.is_ok()
+ || abort_result
+ .as_ref()
+ .err()
+ .is_some_and(|error| unknown_session(error).is_some());
+ if cancelled_confirmed {
+ self.record_child_cancelled(record, ChildCancelledReason::UserCancel)
+ .await;
+ } else {
+ // The abort was refused for a real reason: the child never
+ // got the command, so undo the whole pre-abort claim — the
+ // row stays live, the terminal notice stays owed, and no
+ // settle hook fires (a still-running child must not open
+ // the quiescence barrier or release an owed continuation).
+ // A later confirmed cancel (a retry, a close, a delete) can
+ // then still record the parent -> child edge. The notice
+ // claim restores its PRE-cancel value: a watcher that had
+ // already claimed the terminal notice keeps the claim, and
+ // an unclaimed one stays owed exactly once.
+ let mut row = record.lock().await;
+ row.settled_status = None;
+ row.error = None;
+ row.notice_delivered = notice_was_delivered;
+ drop(row);
+ // The pre-abort update published `cancelled`; publish the
+ // restored live row so subscribers do not keep showing a
+ // child that is still running as cancelled.
+ self.emit_child_update(record).await;
+ // The wire contract answers the confirmed effect: the
+ // child never got the command, so the cancel did not
+ // happen — the user sees `cancelled: false` and can retry.
+ return false;
+ }
// The settled/cancelled child releases an owed goal continuation.
self.fire_settle_hook(record).await;
return true;
@@ -206,13 +258,46 @@ impl SupervisorChildSessionsInner {
record.notice_delivered = true;
}
self.emit_child_usage(record).await;
- let active_session_id = record.lock().await.active_session_id.clone();
+ let (active_session_id, was_running) = {
+ let mut record = record.lock().await;
+ let was_running = record.settled_status.is_none();
+ // A close that kills a live run settles the row
+ // `cancelled` (the same registry state
+ // `cancel_child_run` leaves): a watcher parked inside
+ // its wait slice can then never refresh the killed run
+ // back to `done` and record a clean `child_returned`
+ // beside the cancel line.
+ if was_running {
+ record.settled_status = Some("cancelled");
+ }
+ (record.active_session_id.clone(), was_running)
+ };
if let Err(error) = self.kill_child(&active_session_id, reason).await {
if unknown_session(&error).is_none() {
+ // The kill never landed: undo the pre-kill settle so
+ // the still-live child is not reported cancelled, and
+ // a later successful stop can still record its cancel
+ // edge (the recorder's dedup never saw this attempt).
+ if was_running {
+ record.lock().await.settled_status = None;
+ }
close_error.get_or_insert(error);
continue;
}
}
+ // A session-kill close of a live run is the parent's
+ // cancel (nano-rlm #202's killed-child gap): a killed
+ // child leaves a cancel line, not a clean disappearance.
+ // Only the kill close records it — a daemon lifecycle
+ // close (Shutdown: the resume entry survives; Replaced:
+ // cron jobs survive) is NOT a parent action, so its line
+ // would mislabel every restart/upgrade with running
+ // children as a parent cancel anchored to a stale
+ // request (and the dedup would make it permanent).
+ if was_running && matches!(reason, ChildCloseReason::Killed) {
+ self.record_child_cancelled(record, ChildCancelledReason::CloseKilled)
+ .await;
+ }
self.retain_retired_usage(record, super::RetirementIntent::Close);
self.emit_child_usage(record).await;
}
@@ -266,6 +351,14 @@ impl SupervisorChildSessionsInner {
}
row.rlm_child_id.clone()
};
+ // A delete that killed a live run is the parent's cancel (nano-rlm
+ // #202's delete gap): the child's cancel line lands before the
+ // registry row is gone, so a deleted child never vanishes from the
+ // semantic graph.
+ if cancelled_by_delete {
+ self.record_child_cancelled(record, ChildCancelledReason::ParentDelete)
+ .await;
+ }
if deleted {
let sink = self
.child_update_sink
@@ -528,6 +621,7 @@ fn ledger_child_records(
prompt_admitted: true,
error: None,
closed_by_parent: false,
+ retiring: false,
retirement_finished: false,
retirement_notified: false,
retirement_roster_emitted: false,
diff --git a/crates/pa-daemon/src/rlm_children/usage_emit_tests.rs b/crates/pa-daemon/src/rlm_children/usage_emit_tests.rs
index 82bd6c2a0d..9a250ba4af 100644
--- a/crates/pa-daemon/src/rlm_children/usage_emit_tests.rs
+++ b/crates/pa-daemon/src/rlm_children/usage_emit_tests.rs
@@ -65,6 +65,7 @@ fn record_with_file(child_id: &str, session_file: &Path) -> Arc<Mutex<ChildRecor
prompt_admitted: true,
error: None,
closed_by_parent: false,
+ retiring: false,
retirement_finished: false,
retirement_notified: false,
retirement_roster_emitted: false,
diff --git a/crates/pa-daemon/src/rlm_children/watch_tests.rs b/crates/pa-daemon/src/rlm_children/watch_tests.rs
index f1275e8530..caa3649d30 100644
--- a/crates/pa-daemon/src/rlm_children/watch_tests.rs
+++ b/crates/pa-daemon/src/rlm_children/watch_tests.rs
@@ -1,5 +1,8 @@
use super::*;
use crate::protocol::{response_failure, response_success};
+use pa_core::session_engine::semantic_edges::{
+ ChildCancelledReason, SemanticEdgeIdentity, SemanticEdgeLedgerEvent, SemanticEdgeRecorder,
+};
use pa_types::platform::transport::bind_transport;
use pa_types::session::AgentMessage;
use serde_json::{json, Value};
@@ -11,10 +14,17 @@ use tokio::sync::mpsc;
/// How the fake supervisor answers a child `kill`.
enum FakeKill {
Success,
+ /// The kill succeeds, but only after `ms` (the delete race window: the
+ /// watcher wakes and tries to settle while the kill is in flight).
+ SuccessAfterMs(u64),
/// The child session is gone (the route failure a supervisor answers for a non-resident child).
UnknownSession,
/// The kill fails for a real reason (a stuck worker).
Failure,
+ /// The kill fails for a real reason, but only after `ms` (the delete
+ /// race window with a refused kill: the watcher wakes and settles while
+ /// the kill is still pending).
+ FailureAfterMs(u64),
}
/// How the fake answers the child link: healthy, a failing `prompt`,
@@ -238,24 +248,47 @@ async fn spawn_fake_supervisor(
)
}
}
+ "abort" => {
+ if child_subagents.abort_fails.load(Ordering::SeqCst) {
+ response_failure(
+ Some(&id),
+ command_type,
+ "abort refused by the fake supervisor",
+ None,
+ )
+ } else {
+ response_success(Some(&id), command_type, None)
+ }
+ }
"kill" => {
let _ = kill_tx.send(command.clone());
match *kill_behavior {
FakeKill::Success => {
response_success(Some(&id), command_type, None)
}
+ FakeKill::SuccessAfterMs(ms) => {
+ tokio::time::sleep(std::time::Duration::from_millis(ms))
+ .await;
+ response_success(Some(&id), command_type, None)
+ }
FakeKill::UnknownSession => response_failure(
Some(&id),
command_type,
"Unknown active session: child-live",
None,
),
- FakeKill::Failure => response_failure(
- Some(&id),
- command_type,
- "kill refused by the fake supervisor",
- None,
- ),
+ FakeKill::Failure | FakeKill::FailureAfterMs(_) => {
+ if let FakeKill::FailureAfterMs(ms) = *kill_behavior {
+ tokio::time::sleep(std::time::Duration::from_millis(ms))
+ .await;
+ }
+ response_failure(
+ Some(&id),
+ command_type,
+ "kill refused by the fake supervisor",
+ None,
+ )
+ }
}
}
"follow_up" => {
@@ -305,6 +338,9 @@ struct FakeChildSubagents {
running: AtomicBool,
quiescent_waits: std::sync::atomic::AtomicUsize,
answer_reads: AtomicU32,
+ /// The fake answers `abort` with a real refusal (the cancel path's
+ /// failed-abort lane).
+ abort_fails: AtomicBool,
live_state: std::sync::Mutex<Option<Value>>,
peer_gate: Option<Arc<FakePeerGate>>,
/// Holds the task `prompt` in flight: the worker has not accepted it yet.
@@ -1263,6 +1299,369 @@ async fn close_children_keeps_a_failed_child_tracked() {
assert_eq!(entries.len(), 1, "the failed child stays tracked for retry");
}
+/// A failed close must not poison the cancel record (PR #3680 review
+/// finding): a real kill failure undoes the pre-kill settle, so the
+/// still-live child stays `running` on the roster and a later successful
+/// stop still records the parent -> child cancel edge (the failed attempt
+/// never reached the recorder's dedup).
+#[tokio::test]
+async fn a_failed_close_leaves_the_child_running_and_can_still_record_its_cancel() {
+ let (follow_up_tx, _follow_up_rx) = mpsc::unbounded_channel();
+ let (sessions, _kill_rx) =
+ sessions_with_fake_supervisor(follow_up_tx, 10_000, FakeKill::Failure, FakeChild::Healthy)
+ .await;
+ let (recorder, ledger_path, _dir) = seeded_parent_recorder();
+ sessions.set_semantic_edges(Some(recorder));
+ let handle = spawn_child(&sessions).await;
+ sessions.notify_turn_done();
+ one_running_child(&sessions).await;
+
+ let error = sessions
+ .close_children(ChildCloseReason::Killed)
+ .await
+ .expect_err("a real close failure must propagate");
+ assert!(
+ format!("{error:#}").contains("kill refused"),
+ "the close error must surface the kill failure: {error:#}"
+ );
+
+ // The kill never landed, so the pre-kill settle is undone: the
+ // still-live child is not misreported cancelled.
+ let entries = sessions.list_subagents().await.expect("child roster");
+ assert_eq!(entries.len(), 1, "the failed child stays tracked for retry");
+ assert_eq!(
+ entries[0].status, "running",
+ "a failed close must not mark the still-live child cancelled: {entries:?}"
+ );
+ assert!(
+ cancelled_lines(&ledger_path).is_empty(),
+ "a failed close records no cancel edge: {:?}",
+ cancelled_lines(&ledger_path)
+ );
+
+ // A later successful stop still records the edge: the failed attempt
+ // did not burn the at-most-once claim.
+ assert!(sessions.cancel_child_run(&handle.rlm_child_id).await);
+ assert_eq!(
+ cancelled_lines(&ledger_path),
+ vec![(
+ "parent-lead".to_string(),
+ "child-file".to_string(),
+ "parent-request-1".to_string(),
+ ChildCancelledReason::UserCancel
+ )],
+ "the retry records the cancel edge the failed close could not"
+ );
+}
+
+/// A delete-killed live run never records a clean return beside its cancel
+/// line (PR #3680 review finding, the delete twin of the close case): the
+/// delete settles the row `cancelled` before its kill, so a settle watcher
+/// that wakes inside the delete's in-flight window cannot refresh the
+/// killed run back to `done` and append a `child_returned` after the cancel
+/// line.
+#[tokio::test]
+async fn a_deleted_live_child_never_records_a_clean_return() {
+ let (follow_up_tx, _follow_up_rx) = mpsc::unbounded_channel();
+ let child_subagents = Arc::new(FakeChildSubagents::default());
+ // A live grandchild holds the watcher's wait slice open until the test
+ // releases it inside the delete's in-flight window.
+ child_subagents.running.store(true, Ordering::SeqCst);
+ let (sessions, mut kill_rx) = sessions_with_fake_child_subagents(
+ follow_up_tx,
+ 100,
+ FakeKill::SuccessAfterMs(2_000),
+ FakeChild::Healthy,
+ Arc::clone(&child_subagents),
+ )
+ .await;
+ let (recorder, ledger_path, _dir) = seeded_parent_recorder();
+ sessions.set_semantic_edges(Some(recorder));
+ let (hook_tx, mut hook_rx) = mpsc::unbounded_channel();
+ sessions.set_settle_hook(Arc::new(move || {
+ let _ = hook_tx.send(());
+ }));
+ let handle = spawn_child(&sessions).await;
+ // The child committed one request before the delete cut it short (its
+ // own ledger carries the line the parent's return recording reads).
+ std::fs::create_dir_all(&handle.session_dir).expect("child session dir");
+ let mut child_ledger = String::new();
+ child_ledger.push_str("{\"type\":\"session_registered\",\"session_id\":\"child-file\"}");
+ child_ledger.push('\n');
+ child_ledger.push_str("{\"type\":\"request_finished\",\"request_id\":\"child-request-1\"}");
+ child_ledger.push('\n');
+ std::fs::write(
+ Path::new(&handle.session_dir).join("semantic-edges.jsonl"),
+ child_ledger,
+ )
+ .expect("write the child ledger");
+ sessions.notify_turn_done();
+ // Wait until the watcher is provably parked inside its held quiescence
+ // wait (the fake counts them), never on one still starting.
+ let parked = Instant::now() + Duration::from_secs(10);
+ while child_subagents.quiescent_waits.load(Ordering::SeqCst) == 0 {
+ assert!(
+ Instant::now() < parked,
+ "the watcher never parked in its wait slice"
+ );
+ tokio::time::sleep(Duration::from_millis(20)).await;
+ }
+
+ // The delete reads the live row (and settles it `cancelled`) before it
+ // sends the kill; the kill answers only after its delay. Waiting for
+ // the kill therefore proves the pre-kill settle landed, and releasing
+ // the parked watcher now puts its refresh inside the delete's window.
+ let delete_sessions = sessions.clone();
+ let child_id = handle.rlm_child_id.clone();
+ let delete = tokio::spawn(async move { delete_sessions.delete_subagent(child_id).await });
+ let kill = kill_rx.recv().await.expect("the delete reaches the kill");
+ assert_eq!(kill["type"], "kill");
+ child_subagents.running.store(false, Ordering::SeqCst);
+ // The watcher parks while the delete owns the terminal edge (the kill
+ // window holds the `retiring` mark), so the hook fires exactly once:
+ // the delete's own `finish_retirement`. A watcher that ran its settle
+ // tail into the window would fire here too and deliver the terminal
+ // notice before the delete's cancel overwrites the verdict.
+ tokio::time::timeout(Duration::from_secs(15), hook_rx.recv())
+ .await
+ .expect("the delete fires the settle hook")
+ .expect("hook channel open");
+ assert!(
+ hook_rx.try_recv().is_err(),
+ "the watcher must not fire the settle hook inside the delete's kill window"
+ );
+ delete
+ .await
+ .expect("the delete task joins")
+ .expect("delete the running child");
+
+ assert_eq!(
+ cancelled_lines(&ledger_path),
+ vec![(
+ "parent-lead".to_string(),
+ "child-file".to_string(),
+ "parent-request-1".to_string(),
+ ChildCancelledReason::ParentDelete
+ )],
+ "the delete's kill records the cancel line"
+ );
+ assert!(
+ returned_children(&ledger_path).is_empty(),
+ "a deleted live child never records a child_returned line: {:?}",
+ returned_children(&ledger_path)
+ );
+}
+
+/// A failed delete must not poison the cancel record (PR #3680 review
+/// finding): the pre-kill settle is undone on a real kill failure, so the
+/// still-live child stays `running` and a later confirmed cancel still
+/// records the parent -> child edge.
+#[tokio::test]
+async fn a_failed_delete_leaves_the_child_running_and_can_still_record_its_cancel() {
+ let (follow_up_tx, _follow_up_rx) = mpsc::unbounded_channel();
+ let (sessions, _kill_rx) =
+ sessions_with_fake_supervisor(follow_up_tx, 10_000, FakeKill::Failure, FakeChild::Healthy)
+ .await;
+ let (recorder, ledger_path, _dir) = seeded_parent_recorder();
+ sessions.set_semantic_edges(Some(recorder));
+ let handle = spawn_child(&sessions).await;
+
+ let error = sessions
+ .delete_subagent(handle.rlm_child_id.clone())
+ .await
+ .expect_err("a real delete failure must propagate");
+ assert!(
+ format!("{error:#}").contains("kill refused"),
+ "the delete error must surface the kill failure: {error:#}"
+ );
+
+ // The kill never landed, so the pre-kill settle is undone: the
+ // still-live child is not misreported cancelled.
+ let entries = sessions.list_subagents().await.expect("child roster");
+ assert_eq!(entries.len(), 1, "the failed child stays tracked for retry");
+ assert_eq!(
+ entries[0].status, "running",
+ "a failed delete must not mark the still-live child cancelled: {entries:?}"
+ );
+ assert!(
+ cancelled_lines(&ledger_path).is_empty(),
+ "a failed delete records no cancel edge: {:?}",
+ cancelled_lines(&ledger_path)
+ );
+
+ // A later confirmed stop still records the edge: the failed attempt
+ // did not burn the at-most-once claim.
+ assert!(sessions.cancel_child_run(&handle.rlm_child_id).await);
+ assert_eq!(
+ cancelled_lines(&ledger_path),
+ vec![(
+ "parent-lead".to_string(),
+ "child-file".to_string(),
+ "parent-request-1".to_string(),
+ ChildCancelledReason::UserCancel
+ )],
+ "the retry records the cancel edge the failed delete could not"
+ );
+}
+
+/// A failed delete leaves the child watched (PR #3680 review finding): the
+/// delete mark is cleared, so the watcher survives and the still-live run's
+/// later natural settle still lands its return edge.
+#[tokio::test]
+async fn a_failed_delete_leaves_the_child_watched_and_its_later_return_able_to_land() {
+ let (follow_up_tx, _follow_up_rx) = mpsc::unbounded_channel();
+ let child_subagents = Arc::new(FakeChildSubagents::default());
+ // A live grandchild keeps the run unsettled (and so its watcher alive)
+ // until the test releases it.
+ child_subagents.running.store(true, Ordering::SeqCst);
+ let (sessions, _kill_rx) = sessions_with_fake_child_subagents(
+ follow_up_tx,
+ 100,
+ FakeKill::Failure,
+ FakeChild::Healthy,
+ Arc::clone(&child_subagents),
+ )
+ .await;
+ let (recorder, ledger_path, _dir) = seeded_parent_recorder();
+ sessions.set_semantic_edges(Some(recorder));
+ let handle = spawn_child(&sessions).await;
+ // The child committed one request: the return recording reads it from
+ // the child's own ledger.
+ std::fs::create_dir_all(&handle.session_dir).expect("child session dir");
+ let mut child_ledger = String::new();
+ child_ledger.push_str("{\"type\":\"session_registered\",\"session_id\":\"child-file\"}");
+ child_ledger.push('\n');
+ child_ledger.push_str("{\"type\":\"request_finished\",\"request_id\":\"child-request-1\"}");
+ child_ledger.push('\n');
+ std::fs::write(
+ Path::new(&handle.session_dir).join("semantic-edges.jsonl"),
+ child_ledger,
+ )
+ .expect("write the child ledger");
+ sessions.notify_turn_done();
+ one_running_child(&sessions).await;
+
+ let error = sessions
+ .delete_subagent(handle.rlm_child_id.clone())
+ .await
+ .expect_err("a real delete failure must propagate");
+ assert!(
+ format!("{error:#}").contains("kill refused"),
+ "the delete error must surface the kill failure: {error:#}"
+ );
+
+ // The child finishes naturally afterwards: its watcher is still live,
+ // so the return edge lands instead of being dropped by a stale mark.
+ child_subagents.running.store(false, Ordering::SeqCst);
+ let deadline = Instant::now() + Duration::from_secs(10);
+ while returned_children(&ledger_path).is_empty() {
+ assert!(
+ Instant::now() < deadline,
+ "the settled child never recorded its return: {:?}",
+ returned_children(&ledger_path)
+ );
+ tokio::time::sleep(Duration::from_millis(50)).await;
+ }
+ assert_eq!(
+ returned_children(&ledger_path),
+ vec!["child-file".to_string()],
+ "the failed delete must not drop the child's terminal edge"
+ );
+}
+
+/// A run that settles inside a failed delete's kill window must not lose
+/// its return edge (PR #3680 review finding): the watcher parks while the
+/// kill is pending, so the refused kill leaves it alive to settle the run
+/// normally once the mark clears. A watcher that ran its settle tail into
+/// the window would exit before the failure lands, stranding a settled row
+/// with no watcher and no ledger line.
+#[tokio::test]
+async fn a_child_settling_inside_a_failed_deletes_kill_window_still_lands_its_return() {
+ let (follow_up_tx, _follow_up_rx) = mpsc::unbounded_channel();
+ let child_subagents = Arc::new(FakeChildSubagents::default());
+ // A live grandchild holds the watcher's wait slice open until the test
+ // releases it inside the delete's in-flight kill window.
+ child_subagents.running.store(true, Ordering::SeqCst);
+ let (sessions, mut kill_rx) = sessions_with_fake_child_subagents(
+ follow_up_tx,
+ 100,
+ FakeKill::FailureAfterMs(1_500),
+ FakeChild::Healthy,
+ Arc::clone(&child_subagents),
+ )
+ .await;
+ let (recorder, ledger_path, _dir) = seeded_parent_recorder();
+ sessions.set_semantic_edges(Some(recorder));
+ let handle = spawn_child(&sessions).await;
+ // The child committed one request: the return recording reads it from
+ // the child's own ledger.
+ std::fs::create_dir_all(&handle.session_dir).expect("child session dir");
+ let mut child_ledger = String::new();
+ child_ledger.push_str("{\"type\":\"session_registered\",\"session_id\":\"child-file\"}");
+ child_ledger.push('\n');
+ child_ledger.push_str("{\"type\":\"request_finished\",\"request_id\":\"child-request-1\"}");
+ child_ledger.push('\n');
+ std::fs::write(
+ Path::new(&handle.session_dir).join("semantic-edges.jsonl"),
+ child_ledger,
+ )
+ .expect("write the child ledger");
+ sessions.notify_turn_done();
+ // Wait until the watcher is provably parked inside its held quiescence
+ // wait (the fake counts them), never on one still starting.
+ let parked = Instant::now() + Duration::from_secs(10);
+ while child_subagents.quiescent_waits.load(Ordering::SeqCst) == 0 {
+ assert!(
+ Instant::now() < parked,
+ "the watcher never parked in its wait slice"
+ );
+ tokio::time::sleep(Duration::from_millis(20)).await;
+ }
+
+ // The delete sets `retiring` and sends the kill; the fake holds its
+ // refusal, so releasing the watcher now puts its refresh inside the
+ // delete's kill window: the run settles while the kill is pending.
+ let delete_sessions = sessions.clone();
+ let child_id = handle.rlm_child_id.clone();
+ let delete = tokio::spawn(async move { delete_sessions.delete_subagent(child_id).await });
+ let kill = kill_rx.recv().await.expect("the delete reaches the kill");
+ assert_eq!(kill["type"], "kill");
+ child_subagents.running.store(false, Ordering::SeqCst);
+
+ let error = delete
+ .await
+ .expect("the delete task joins")
+ .expect_err("the refused kill must fail the delete");
+ assert!(
+ format!("{error:#}").contains("kill refused"),
+ "the delete error must surface the kill failure: {error:#}"
+ );
+ assert!(
+ cancelled_lines(&ledger_path).is_empty(),
+ "a failed delete records no cancel edge: {:?}",
+ cancelled_lines(&ledger_path)
+ );
+
+ // The watcher survived the window (it parked instead of running its
+ // tail into it), so the run's natural settle still lands its return
+ // edge. A watcher that exited inside the window would leave this line
+ // forever absent.
+ let deadline = Instant::now() + Duration::from_secs(10);
+ while returned_children(&ledger_path).is_empty() {
+ assert!(
+ Instant::now() < deadline,
+ "a run settling inside a failed delete's kill window never recorded its return: {:?}",
+ returned_children(&ledger_path)
+ );
+ tokio::time::sleep(Duration::from_millis(50)).await;
+ }
+ assert_eq!(
+ returned_children(&ledger_path),
+ vec!["child-file".to_string()],
+ "the parked watcher settles the run once the delete fails"
+ );
+}
+
#[tokio::test]
async fn a_replied_child_gets_no_terminal_notice() {
let (follow_up_tx, mut follow_up_rx) = mpsc::unbounded_channel();
@@ -1280,6 +1679,595 @@ async fn a_replied_child_gets_no_terminal_notice() {
assert!(extra.is_err(), "a replied child must not deliver a notice");
}
+/// A parent recorder over a fresh ledger seeded with one started request:
+/// the replay restores the in-flight turn, so the registry's cancel sites
+/// anchor the `child_cancelled` line to it (the same wire a live parent
+/// session would carry mid-turn).
+fn seeded_parent_recorder() -> (
+ std::sync::Arc<SemanticEdgeRecorder>,
+ PathBuf,
+ tempfile::TempDir,
+) {
+ let dir = tempfile::TempDir::new().expect("temp dir");
+ let ledger_path = dir.path().join("semantic-edges.jsonl");
+ let registered = SemanticEdgeLedgerEvent::SessionRegistered {
+ session_id: "parent-lead".to_string(),
+ parent_session_id: None,
+ spawned_by_request_id: None,
+ };
+ let started = SemanticEdgeLedgerEvent::RequestStarted {
+ request_id: "parent-request-1".to_string(),
+ session_id: "parent-lead".to_string(),
+ compaction_id: None,
+ };
+ std::fs::write(
+ &ledger_path,
+ format!(
+ "{}\n{}\n",
+ serde_json::to_string(&registered).expect("serialize the registered line"),
+ serde_json::to_string(&started).expect("serialize the started line")
+ ),
+ )
+ .expect("write the seeded ledger");
+ let recorder = std::sync::Arc::new(SemanticEdgeRecorder::open(SemanticEdgeIdentity {
+ session_id: "parent-lead".to_string(),
+ ledger_path: Some(ledger_path.clone()),
+ parent_session_id: None,
+ spawned_by_request_id: None,
+ }));
+ (recorder, ledger_path, dir)
+}
+
+/// The `child_cancelled` lines of one parent ledger: (parent, child,
+/// cancelling request, which parent-side action cancelled the run).
+fn cancelled_lines(path: &Path) -> Vec<(String, String, String, ChildCancelledReason)> {
+ std::fs::read_to_string(path)
+ .unwrap_or_default()
+ .lines()
+ .filter_map(|line| serde_json::from_str::<SemanticEdgeLedgerEvent>(line.trim()).ok())
+ .filter_map(|event| match event {
+ SemanticEdgeLedgerEvent::ChildCancelled {
+ session_id,
+ child_session_id,
+ request_id,
+ reason,
+ } => Some((session_id, child_session_id, request_id, reason)),
+ _ => None,
+ })
+ .collect()
+}
+
+/// A user-cancelled run records the cancel edge (nano-rlm #202's
+/// `agent.cancel` gap): one parent -> child line anchored to the
+/// parent's in-flight request, never a clean disappearance.
+#[tokio::test]
+async fn a_cancelled_run_records_the_cancel_edge_on_the_parent_ledger() {
+ let (follow_up_tx, _follow_up_rx) = mpsc::unbounded_channel();
+ let (sessions, _kill_rx) =
+ sessions_with_fake_supervisor(follow_up_tx, 10_000, FakeKill::Success, FakeChild::Healthy)
+ .await;
+ let (recorder, ledger_path, _dir) = seeded_parent_recorder();
+ sessions.set_semantic_edges(Some(recorder));
+ let handle = spawn_child(&sessions).await;
+
+ assert!(sessions.cancel_child_run(&handle.rlm_child_id).await);
+
+ assert_eq!(
+ cancelled_lines(&ledger_path),
+ vec![(
+ "parent-lead".to_string(),
+ "child-file".to_string(),
+ "parent-request-1".to_string(),
+ ChildCancelledReason::UserCancel
+ )],
+ "the cancel lands as a parent -> child edge anchored to the cancelling request"
+ );
+}
+
+/// A delete that kills a live run records the cancel edge (nano-rlm
+/// #202's delete gap): a deleted child leaves a cancel line, not a clean
+/// disappearance.
+#[tokio::test]
+async fn a_deleted_running_child_records_the_cancel_edge_on_the_parent_ledger() {
+ let (follow_up_tx, _follow_up_rx) = mpsc::unbounded_channel();
+ let (sessions, _kill_rx) =
+ sessions_with_fake_supervisor(follow_up_tx, 10_000, FakeKill::Success, FakeChild::Healthy)
+ .await;
+ let (recorder, ledger_path, _dir) = seeded_parent_recorder();
+ sessions.set_semantic_edges(Some(recorder));
+ let handle = spawn_child(&sessions).await;
+
+ let deleted = sessions
+ .delete_subagent(handle.rlm_child_id.clone())
+ .await
+ .expect("delete the running child");
+ assert_eq!(deleted.outcome, Some("deleted"));
+
+ assert_eq!(
+ cancelled_lines(&ledger_path),
+ vec![(
+ "parent-lead".to_string(),
+ "child-file".to_string(),
+ "parent-request-1".to_string(),
+ ChildCancelledReason::ParentDelete
+ )],
+ "the running child's delete lands as a parent -> child cancel edge"
+ );
+}
+
+/// A settled child's delete leaves its return alone (PR #3680 review
+/// finding): the delete marks only a LIVE run, so a `done` child torn down
+/// inside its settle grace still records its `child_returned` edge, and no
+/// `child_cancelled` line is written for it.
+#[tokio::test]
+async fn a_deleted_settled_child_still_records_its_return() {
+ let (follow_up_tx, _follow_up_rx) = mpsc::unbounded_channel();
+ let (sessions, _kill_rx) =
+ sessions_with_fake_supervisor(follow_up_tx, 0, FakeKill::Success, FakeChild::Healthy).await;
+ let (recorder, ledger_path, _dir) = seeded_parent_recorder();
+ sessions.set_semantic_edges(Some(recorder));
+ let handle = spawn_child(&sessions).await;
+ // The child committed one request: the return recording reads it.
+ std::fs::create_dir_all(&handle.session_dir).expect("child session dir");
+ let mut child_ledger = String::new();
+ child_ledger.push_str("{\"type\":\"session_registered\",\"session_id\":\"child-file\"}");
+ child_ledger.push('\n');
+ child_ledger.push_str("{\"type\":\"request_finished\",\"request_id\":\"child-request-1\"}");
+ child_ledger.push('\n');
+ std::fs::write(
+ Path::new(&handle.session_dir).join("semantic-edges.jsonl"),
+ child_ledger,
+ )
+ .expect("write the child ledger");
+ sessions.notify_turn_done();
+ // The watcher's refresh settles the row `done`; the tail that records
+ // the return follows the settle grace.
+ let settled = Instant::now() + Duration::from_secs(10);
+ loop {
+ let entries = sessions.list_subagents().await.expect("child roster");
+ if entries.iter().any(|entry| entry.status == "completed") {
+ break;
+ }
+ assert!(Instant::now() < settled, "child never settled: {entries:?}");
+ tokio::time::sleep(Duration::from_millis(20)).await;
+ }
+
+ // Delete inside the grace window: the run already settled, so no cancel
+ // is owed and the return must still land.
+ sessions
+ .delete_subagent(handle.rlm_child_id.clone())
+ .await
+ .expect("delete the settled child");
+
+ let deadline = Instant::now() + Duration::from_secs(10);
+ while returned_children(&ledger_path).is_empty() {
+ assert!(
+ Instant::now() < deadline,
+ "the deleted settled child never recorded its return"
+ );
+ tokio::time::sleep(Duration::from_millis(50)).await;
+ }
+ assert_eq!(
+ returned_children(&ledger_path),
+ vec!["child-file".to_string()],
+ "a settled child's delete leaves its return edge alone"
+ );
+ assert!(
+ cancelled_lines(&ledger_path).is_empty(),
+ "a settled child's delete records no cancel line: {:?}",
+ cancelled_lines(&ledger_path)
+ );
+}
+
+/// Only a cut-short run cancels: a delete of an already-settled child
+/// records no cancel line (its return line already stands).
+#[tokio::test]
+async fn a_deleted_settled_child_records_no_cancel_edge() {
+ let (follow_up_tx, _follow_up_rx) = mpsc::unbounded_channel();
+ let (sessions, _kill_rx) =
+ sessions_with_fake_supervisor(follow_up_tx, 0, FakeKill::Success, FakeChild::Healthy).await;
+ let (recorder, ledger_path, _dir) = seeded_parent_recorder();
+ sessions.set_semantic_edges(Some(recorder));
+ let handle = spawn_child(&sessions).await;
+ sessions.notify_turn_done();
+ let deadline = Instant::now() + Duration::from_secs(10);
+ loop {
+ let entries = sessions.list_subagents().await.expect("child roster");
+ if entries.iter().any(|entry| entry.status == "completed") {
+ break;
+ }
+ assert!(
+ Instant::now() < deadline,
+ "child never settled: {entries:?}"
+ );
+ tokio::time::sleep(Duration::from_millis(50)).await;
+ }
+
+ sessions
+ .delete_subagent(handle.rlm_child_id.clone())
+ .await
+ .expect("delete the settled child");
+
+ assert_eq!(
+ cancelled_lines(&ledger_path),
+ Vec::new(),
+ "a settled child's delete records no cancel edge"
+ );
+}
+
+/// A close that kills a live run records the cancel edge (nano-rlm
+/// #202's killed-child gap): a killed child leaves a cancel line on the
+/// parent's ledger, not a clean disappearance.
+#[tokio::test]
+async fn a_closed_running_child_records_the_cancel_edge_on_the_parent_ledger() {
+ let (follow_up_tx, _follow_up_rx) = mpsc::unbounded_channel();
+ let (sessions, _kill_rx) =
+ sessions_with_fake_supervisor(follow_up_tx, 10_000, FakeKill::Success, FakeChild::Healthy)
+ .await;
+ let (recorder, ledger_path, _dir) = seeded_parent_recorder();
+ sessions.set_semantic_edges(Some(recorder));
+ let _handle = spawn_child(&sessions).await;
+
+ sessions
+ .close_children(ChildCloseReason::Killed)
+ .await
+ .expect("close the running child");
+
+ assert_eq!(
+ cancelled_lines(&ledger_path),
+ vec![(
+ "parent-lead".to_string(),
+ "child-file".to_string(),
+ "parent-request-1".to_string(),
+ ChildCancelledReason::CloseKilled
+ )],
+ "the close's kill lands as a parent -> child cancel edge"
+ );
+}
+
+/// A daemon lifecycle close (Shutdown: the daemon stops; Replaced: an
+/// upgrade) kills running children but records NO `child_cancelled` line
+/// (PR #3680 review finding): the close walk is not a parent action, so a
+/// permanent cancel line anchored to the parent's stale last request would
+/// mislabel every restart or upgrade as a parent cancel — and the
+/// at-most-once dedup would make it permanent. Only the session-kill
+/// close (`Killed`) is the parent's cancel.
+#[tokio::test]
+async fn a_lifecycle_close_records_no_cancel_line() {
+ for reason in [ChildCloseReason::Shutdown, ChildCloseReason::Replaced] {
+ let (follow_up_tx, _follow_up_rx) = mpsc::unbounded_channel();
+ let (sessions, _kill_rx) = sessions_with_fake_supervisor(
+ follow_up_tx,
+ 10_000,
+ FakeKill::Success,
+ FakeChild::Healthy,
+ )
+ .await;
+ let (recorder, ledger_path, _dir) = seeded_parent_recorder();
+ sessions.set_semantic_edges(Some(recorder));
+ let _handle = spawn_child(&sessions).await;
+
+ sessions
+ .close_children(reason)
+ .await
+ .expect("close the running child");
+
+ assert!(
+ cancelled_lines(&ledger_path).is_empty(),
+ "a lifecycle close records no cancel line: {:?}",
+ cancelled_lines(&ledger_path)
+ );
+ assert!(
+ returned_children(&ledger_path).is_empty(),
+ "a lifecycle close records no return line either: {:?}",
+ returned_children(&ledger_path)
+ );
+ // The close still tore the live child down.
+ let entries = sessions.list_subagents().await.expect("child roster");
+ assert!(
+ entries.is_empty() || entries.iter().all(|entry| entry.status == "cancelled"),
+ "the lifecycle close tore the running child down: {entries:?}"
+ );
+ }
+}
+
+/// The `child_returned` lines of one parent ledger: the child session ids.
+fn returned_children(path: &Path) -> Vec<String> {
+ std::fs::read_to_string(path)
+ .unwrap_or_default()
+ .lines()
+ .filter_map(|line| serde_json::from_str::<SemanticEdgeLedgerEvent>(line.trim()).ok())
+ .filter_map(|event| match event {
+ SemanticEdgeLedgerEvent::ChildReturned {
+ child_session_id, ..
+ } => Some(child_session_id),
+ _ => None,
+ })
+ .collect()
+}
+
+/// A refused abort leaves the child live and records no cancel edge (PR
+/// #3680 review finding): the ledger mirrors the confirmed effect only, and
+/// the row is left live so a later confirmed cancel can still record the
+/// parent -> child edge.
+#[tokio::test]
+async fn a_refused_abort_leaves_the_child_running_and_a_later_cancel_records_the_edge() {
+ let (follow_up_tx, _follow_up_rx) = mpsc::unbounded_channel();
+ let child_subagents = Arc::new(FakeChildSubagents::default());
+ child_subagents.abort_fails.store(true, Ordering::SeqCst);
+ let (sessions, _kill_rx) = sessions_with_fake_child_subagents(
+ follow_up_tx,
+ 10_000,
+ FakeKill::Success,
+ FakeChild::Healthy,
+ Arc::clone(&child_subagents),
+ )
+ .await;
+ let (recorder, ledger_path, _dir) = seeded_parent_recorder();
+ sessions.set_semantic_edges(Some(recorder));
+ let (hook_tx, mut hook_rx) = mpsc::unbounded_channel();
+ sessions.set_settle_hook(Arc::new(move || {
+ let _ = hook_tx.send(());
+ }));
+ let rows: Arc<std::sync::Mutex<Vec<Value>>> = Arc::new(std::sync::Mutex::new(Vec::new()));
+ let sink_rows = Arc::clone(&rows);
+ sessions.set_child_update_sink(Arc::new(move |child| {
+ sink_rows.lock().unwrap().push(child);
+ }));
+ let handle = spawn_child(&sessions).await;
+
+ assert!(
+ !sessions.cancel_child_run(&handle.rlm_child_id).await,
+ "a refused abort answers the confirmed effect: the cancel did not happen"
+ );
+
+ // The refused abort never reached the child: the row stays live, so a
+ // still-running child is not misreported cancelled.
+ let entries = sessions.list_subagents().await.expect("child roster");
+ assert_eq!(
+ entries[0].status, "running",
+ "a refused abort must leave the live child running: {entries:?}"
+ );
+ assert_eq!(
+ cancelled_lines(&ledger_path),
+ Vec::new(),
+ "a refused abort records no cancel line"
+ );
+ // And nothing is released: the quiescence barrier still sees the live
+ // child, so no settle hook may fire.
+ assert!(
+ sessions.any_running().await,
+ "a refused abort must leave the run unsettled"
+ );
+ // The restored row is republished: subscribers must not keep showing a
+ // live child as cancelled.
+ {
+ let published = rows.lock().unwrap();
+ assert_eq!(
+ published.last().and_then(|row| row["status"].as_str()),
+ Some("running"),
+ "the refused abort must republish the live row: {published:?}"
+ );
+ }
+ assert!(
+ tokio::time::timeout(Duration::from_millis(300), hook_rx.recv())
+ .await
+ .is_err(),
+ "a refused abort must not fire the settle hook"
+ );
+
+ // A later confirmed cancel records the edge the refused attempt could
+ // not: the settle undo left the row live for the retry.
+ child_subagents.abort_fails.store(false, Ordering::SeqCst);
+ assert!(
+ sessions.cancel_child_run(&handle.rlm_child_id).await,
+ "the retry claims the still-live row"
+ );
+ assert_eq!(
+ cancelled_lines(&ledger_path),
+ vec![(
+ "parent-lead".to_string(),
+ "child-file".to_string(),
+ "parent-request-1".to_string(),
+ ChildCancelledReason::UserCancel
+ )],
+ "the confirmed retry records the cancel edge"
+ );
+ tokio::time::timeout(Duration::from_secs(5), hook_rx.recv())
+ .await
+ .expect("the confirmed cancel fires the settle hook")
+ .expect("hook channel open");
+}
+
+/// A close-killed live run never records a clean return beside its cancel
+/// line (PR #3680 review finding): the settle watcher is parked inside
+/// its wait slice when the close kills the child, and its post-kill
+/// refresh used to re-mark the killed run `done` — appending a
+/// `child_returned` line after the cancel line. The close now settles
+/// the row `cancelled`, so the parked watcher's tail records nothing.
+/// A registry-set `cancelled` survives the watcher's grace re-check (PR
+/// #3680 review finding): the watcher parks mid-slice while a user cancel
+/// settles the row with its abort in flight; a busy child at the grace
+/// re-check used to wipe the registry's settle, refresh the still-live run
+/// back to `done`, and record a clean `child_returned` beside the cancel
+/// line. The re-check now wipes only a watcher-set `done` (the admission
+/// race it exists for), so the registry's cancelled stands: the cut-short
+/// run never records a return, and the roster keeps reporting it
+/// cancelled.
+#[tokio::test]
+async fn a_cancelled_rows_settle_survives_the_watcher_grace_recheck() {
+ let (follow_up_tx, _follow_up_rx) = mpsc::unbounded_channel();
+ let child_subagents = Arc::new(FakeChildSubagents::default());
+ // The child reads busy for the whole cancel window: the parked watcher
+ // provably hits its grace re-check with the row already settled
+ // `cancelled` by the registry and the child still busy — the exact
+ // wipe window.
+ child_subagents.live_state.lock().unwrap().replace(json!({
+ "isStreaming": true,
+ "hasRunningSubagents": false,
+ "sessionActions": { "queuedCount": 0 },
+ }));
+ let (sessions, _kill_rx) = sessions_with_fake_child_subagents(
+ follow_up_tx,
+ 100,
+ FakeKill::Success,
+ FakeChild::Healthy,
+ Arc::clone(&child_subagents),
+ )
+ .await;
+ let (recorder, ledger_path, _dir) = seeded_parent_recorder();
+ sessions.set_semantic_edges(Some(recorder));
+ let (hook_tx, mut hook_rx) = mpsc::unbounded_channel();
+ sessions.set_settle_hook(Arc::new(move || {
+ let _ = hook_tx.send(());
+ }));
+ let handle = spawn_child(&sessions).await;
+ // The child committed one request before the cancel cut it short (the
+ // wrongly-recorded return would read this line).
+ std::fs::create_dir_all(&handle.session_dir).expect("child session dir");
+ let mut child_ledger = String::new();
+ child_ledger.push_str("{\"type\":\"session_registered\",\"session_id\":\"child-file\"}");
+ child_ledger.push('\n');
+ child_ledger.push_str("{\"type\":\"request_finished\",\"request_id\":\"child-request-1\"}");
+ child_ledger.push('\n');
+ std::fs::write(
+ Path::new(&handle.session_dir).join("semantic-edges.jsonl"),
+ child_ledger,
+ )
+ .expect("write the child ledger");
+ sessions.notify_turn_done();
+
+ assert!(
+ sessions.cancel_child_run(&handle.rlm_child_id).await,
+ "the confirmed cancel claims the running child"
+ );
+
+ // Hold the busy state through at least two post-cancel watcher slices
+ // (each wait_for_idle is one slice): every post-cancel slice observes
+ // the registry-set `cancelled` while the child still reads busy — the
+ // exact grace re-check window — so the buggy unconditional wipe fires
+ // here deterministically. Only then let the run read idle.
+ let slices_before_cancel = child_subagents.quiescent_waits.load(Ordering::SeqCst);
+ let cancelled_and_busy = Instant::now() + Duration::from_secs(15);
+ while child_subagents.quiescent_waits.load(Ordering::SeqCst) < slices_before_cancel + 2 {
+ assert!(
+ Instant::now() < cancelled_and_busy,
+ "the watcher never re-checked the cancelled row while busy"
+ );
+ tokio::time::sleep(Duration::from_millis(20)).await;
+ }
+ child_subagents.live_state.lock().unwrap().take();
+
+ // The cancel path fires the hook, the watcher's settle tail fires it
+ // again once the child reads idle; the second fire bounds the tail.
+ let mut hook_fires = 0;
+ while hook_fires < 2 {
+ tokio::time::timeout(Duration::from_secs(15), hook_rx.recv())
+ .await
+ .expect("the cancel and the watcher's tail each fire the hook")
+ .expect("hook channel open");
+ hook_fires += 1;
+ }
+
+ assert_eq!(
+ cancelled_lines(&ledger_path),
+ vec![(
+ "parent-lead".to_string(),
+ "child-file".to_string(),
+ "parent-request-1".to_string(),
+ ChildCancelledReason::UserCancel
+ )],
+ "the confirmed cancel records exactly one cancel line"
+ );
+ assert!(
+ returned_children(&ledger_path).is_empty(),
+ "a cancel-cut run never records a child_returned line: {:?}",
+ returned_children(&ledger_path)
+ );
+ let entries = sessions.list_subagents().await.expect("child roster");
+ assert_eq!(
+ entries[0].status, "cancelled",
+ "the cancelled row stays cancelled after the watcher's pass: {entries:?}"
+ );
+}
+
+#[tokio::test]
+async fn a_close_killed_child_never_records_a_clean_return() {
+ let (follow_up_tx, _follow_up_rx) = mpsc::unbounded_channel();
+ let child_subagents = Arc::new(FakeChildSubagents::default());
+ // A live grandchild holds the watcher's wait slice open indefinitely,
+ // so the close provably lands on a watcher parked mid-wait.
+ child_subagents.running.store(true, Ordering::SeqCst);
+ let (sessions, _kill_rx) = sessions_with_fake_child_subagents(
+ follow_up_tx,
+ 100,
+ FakeKill::Success,
+ FakeChild::Healthy,
+ Arc::clone(&child_subagents),
+ )
+ .await;
+ let (recorder, ledger_path, _dir) = seeded_parent_recorder();
+ sessions.set_semantic_edges(Some(recorder));
+ let (hook_tx, mut hook_rx) = mpsc::unbounded_channel();
+ sessions.set_settle_hook(Arc::new(move || {
+ let _ = hook_tx.send(());
+ }));
+ let handle = spawn_child(&sessions).await;
+ // The child committed one request before the close cut it short (its
+ // own ledger carries the line the parent's return recording reads).
+ std::fs::create_dir_all(&handle.session_dir).expect("child session dir");
+ let mut child_ledger = String::new();
+ child_ledger.push_str("{\"type\":\"session_registered\",\"session_id\":\"child-file\"}");
+ child_ledger.push('\n');
+ child_ledger.push_str("{\"type\":\"request_finished\",\"request_id\":\"child-request-1\"}");
+ child_ledger.push('\n');
+ std::fs::write(
+ Path::new(&handle.session_dir).join("semantic-edges.jsonl"),
+ child_ledger,
+ )
+ .expect("write the child ledger");
+ sessions.notify_turn_done();
+ // Wait until the watcher is provably parked inside its held quiescence
+ // wait (the fake counts them), never on one still starting.
+ let parked = Instant::now() + Duration::from_secs(10);
+ while child_subagents.quiescent_waits.load(Ordering::SeqCst) == 0 {
+ assert!(
+ Instant::now() < parked,
+ "the watcher never parked in its wait slice"
+ );
+ tokio::time::sleep(Duration::from_millis(20)).await;
+ }
+
+ sessions
+ .close_children(ChildCloseReason::Killed)
+ .await
+ .expect("close the running child");
+ // Release the parked watcher: its wait slice answers, its refresh runs,
+ // and its settle tail fires the hook — the return recording precedes
+ // the hook in that tail, so the hook bounds the assertion.
+ child_subagents.running.store(false, Ordering::SeqCst);
+ tokio::time::timeout(Duration::from_secs(15), hook_rx.recv())
+ .await
+ .expect("the parked watcher runs its settle tail")
+ .expect("hook channel open");
+
+ assert_eq!(
+ cancelled_lines(&ledger_path),
+ vec![(
+ "parent-lead".to_string(),
+ "child-file".to_string(),
+ "parent-request-1".to_string(),
+ ChildCancelledReason::CloseKilled
+ )],
+ "the close's kill records the cancel line"
+ );
+ assert!(
+ returned_children(&ledger_path).is_empty(),
+ "a close-killed run never records a child_returned line: {:?}",
+ returned_children(&ledger_path)
+ );
+}
+
/// A target whose delete receipt already returned resolves immediately to the settled cancelled
/// envelope without spending the timeout budget; unknown selectors keep erroring.
#[tokio::test]
@@ -1620,6 +2608,7 @@ fn an_already_settled_child_never_re_scores_as_an_unreachable_error() {
prompt_admitted: true,
error: None,
closed_by_parent: false,
+ retiring: false,
retirement_finished: false,
retirement_notified: false,
retirement_roster_emitted: false,
@@ -1692,6 +2681,7 @@ async fn the_unreachable_give_up_skips_the_exit_capture_when_a_capture_stands()
prompt_admitted: true,
error: None,
closed_by_parent: false,
+ retiring: false,
retirement_finished: false,
retirement_notified: false,
retirement_roster_emitted: false,
diff --git a/crates/pa-daemon/tests/rlm_children_delete_e2e.rs b/crates/pa-daemon/tests/rlm_children_delete_e2e.rs
new file mode 100644
index 0000000000..ed23871f41
--- /dev/null
+++ b/crates/pa-daemon/tests/rlm_children_delete_e2e.rs
@@ -0,0 +1,495 @@
+//! The delete-turn e2e: a live child deleted through the product
+//! `rlm.delete_subagent` surface mid-turn records its `child_cancelled`
+//! line (with the `parent_delete` reason) on the parent's ledger — a
+//! deleted running child leaves a cancel line, not a clean disappearance.
+//! The harness is the self-contained supervisor/client scaffolding this
+//! suite's e2e files each carry. Tests skip without a live kernel install.
+// Fn length is a style gate, not correctness.
+#![allow(clippy::too_many_lines)]
+#![cfg(unix)]
+
+use std::io::{BufRead, BufReader, Write};
+use std::os::unix::net::UnixStream;
+use std::path::{Path, PathBuf};
+use std::process::{Child, Command, Stdio};
+use std::time::{Duration, Instant};
+
+use pa_core::session_engine::semantic_edges::{
+ ChildCancelledReason, SemanticEdgeLedgerEvent as LedgerEvent,
+};
+
+use serde_json::{json, Value};
+
+struct Daemon {
+ child: Child,
+ #[allow(dead_code)]
+ socket: PathBuf,
+}
+
+impl Drop for Daemon {
+ fn drop(&mut self) {
+ let _ = self.child.kill();
+ let _ = self.child.wait();
+ }
+}
+
+// The timeout panic path cannot wait on the child; the test exits and reaps it.
+#[allow(clippy::zombie_processes)]
+fn spawn_supervisor(socket: &Path, agent_dir: &Path, kernel_python: &Path) -> Daemon {
+ let binary = env!("CARGO_BIN_EXE_pa-daemon");
+ let child = Command::new(binary)
+ .arg("supervisor")
+ .arg("--socket")
+ .arg(socket)
+ .arg("--agent-dir")
+ .arg(agent_dir)
+ .env("PRIME_AGENT_KERNEL_PYTHON", kernel_python)
+ // Hermetic agent dir: the ambient environment exports a real
+ // agent dir; point every fallback at the test sandbox instead.
+ .env("PRIME_AGENT_CODING_AGENT_DIR", agent_dir)
+ .env_remove("PRIME_API_KEY")
+ .stdout(Stdio::null())
+ .stderr(Stdio::null())
+ // A supervisor killed at teardown must not leak its session workers into later
+ // test binaries: the supervisor-lost exit runs here.
+ .env(
+ pa_daemon::worker::WORKER_SUPERVISOR_LOST_EXIT_MS_ENV,
+ "15000",
+ )
+ .spawn()
+ .expect("spawn pa-daemon supervisor");
+ Daemon {
+ child,
+ socket: socket.to_path_buf(),
+ }
+}
+
+/// The kernel Python; `PA_E2E_KERNEL_PYTHON` points at an explicit interpreter instead.
+fn kernel_python() -> Option<PathBuf> {
+ if let Some(explicit) = std::env::var_os("PA_E2E_KERNEL_PYTHON") {
+ let explicit = PathBuf::from(explicit);
+ assert!(
+ explicit.exists(),
+ "PA_E2E_KERNEL_PYTHON {} not found",
+ explicit.display()
+ );
+ return Some(explicit);
+ }
+ let candidate = PathBuf::from(std::env::var("HOME").map_or_else(
+ |_| "/home/ubuntu/.prime/agent/kernel-venv/bin/python".to_string(),
+ |home| format!("{home}/.prime/agent/kernel-venv/bin/python"),
+ ));
+ if candidate.exists() {
+ return Some(candidate);
+ }
+ eprintln!(
+ "kernel python {} not found; skipping live RLM delete e2e",
+ candidate.display()
+ );
+ None
+}
+
+fn wait_socket_ready(socket: &Path) {
+ let deadline = Instant::now() + Duration::from_secs(15);
+ loop {
+ if UnixStream::connect(socket).is_ok() {
+ return;
+ }
+ assert!(Instant::now() < deadline, "supervisor socket never came up");
+ std::thread::sleep(Duration::from_millis(20));
+ }
+}
+
+/// JSONL supervisor client (command envelopes, id-matched responses).
+struct Client {
+ reader: BufReader<UnixStream>,
+ writer: UnixStream,
+}
+
+impl Client {
+ fn connect(socket: &Path) -> (Self, Value) {
+ let stream = UnixStream::connect(socket).expect("connect supervisor");
+ let write_stream = stream.try_clone().expect("clone socket");
+ let mut client = Client {
+ reader: BufReader::new(stream),
+ writer: write_stream,
+ };
+ let hello = client.read_line();
+ (client, hello)
+ }
+
+ fn send_command(&mut self, id: &str, command: &Value) {
+ let envelope = json!({
+ "type": "command",
+ "id": id,
+ "protocol": { "name": "prime-agent.daemon", "version": 7 },
+ "command": command,
+ });
+ let mut line = serde_json::to_string(&envelope).expect("serialize command");
+ line.push('\n');
+ self.writer.write_all(line.as_bytes()).expect("send");
+ self.writer.flush().expect("flush");
+ }
+
+ fn read_line(&mut self) -> Value {
+ let mut line = String::new();
+ let deadline = Instant::now() + Duration::from_mins(2);
+ self.reader
+ .get_mut()
+ .set_read_timeout(Some(Duration::from_millis(100)))
+ .expect("set timeout");
+ loop {
+ match self.reader.read_line(&mut line) {
+ Ok(0) => panic!("supervisor closed the connection"),
+ Ok(_) if line.trim().is_empty() => line.clear(),
+ Ok(_) => return serde_json::from_str(line.trim()).expect("parse response line"),
+ Err(error) => {
+ assert!(
+ Instant::now() < deadline,
+ "timed out waiting for supervisor line: {error}"
+ );
+ }
+ }
+ }
+ }
+
+ fn read_response(&mut self, id: &str) -> Value {
+ let deadline = Instant::now() + Duration::from_mins(4);
+ loop {
+ assert!(Instant::now() < deadline, "no response for id {id}");
+ let line = self.read_line();
+ if line.get("id").and_then(Value::as_str) == Some(id) {
+ return line;
+ }
+ }
+ }
+}
+
+/// Run one turn (prompt + idle wait) on the session.
+fn run_turn(client: &mut Client, session_id: &str, message: &str, id: &str) {
+ client.send_command(
+ id,
+ &json!({ "type": "prompt", "activeSessionId": session_id, "message": message }),
+ );
+ let prompted = client.read_response(id);
+ assert_eq!(prompted["success"], true, "prompt failed: {prompted}");
+ let idle_id = format!("{id}-idle");
+ client.send_command(
+ &idle_id,
+ &json!({ "type": "wait_for_idle", "activeSessionId": session_id }),
+ );
+ let idle = client.read_response(&idle_id);
+ assert_eq!(idle["success"], true, "wait_for_idle failed: {idle}");
+}
+
+/// Poll for a kernel cell's receipt content (the cell writes its verdict;
+/// an empty read is the create-before-write window, not a verdict yet).
+fn await_receipt(receipt: &Path) -> String {
+ let deadline = Instant::now() + Duration::from_mins(1);
+ loop {
+ if let Ok(content) = std::fs::read_to_string(receipt) {
+ if !content.is_empty() {
+ return content;
+ }
+ }
+ assert!(
+ Instant::now() < deadline,
+ "kernel cell receipt never appeared at {}",
+ receipt.display()
+ );
+ std::thread::sleep(Duration::from_millis(50));
+ }
+}
+
+fn create_parent(
+ client: &mut Client,
+ dir: &Path,
+ parent_script: &Path,
+ child_script: &Path,
+ id: &str,
+) -> Value {
+ let sessions_dir = dir.join("sessions");
+ std::fs::create_dir_all(&sessions_dir).expect("sessions dir");
+ client.send_command(
+ id,
+ &json!({
+ "type": "create",
+ "name": "parent",
+ "config": {
+ "cwd": dir.to_string_lossy(),
+ "sessionDir": sessions_dir.to_string_lossy(),
+ "script": parent_script.to_string_lossy(),
+ "childScript": child_script.to_string_lossy(),
+ },
+ }),
+ );
+ let created = client.read_response(id);
+ assert_eq!(created["success"], true, "create parent failed: {created}");
+ created["data"].clone()
+}
+
+fn semantic_spawn_cell(receipt: &Path, error_receipt: &Path) -> String {
+ format!(
+ "import json, traceback\ntry:\n handle = await rlm.spawn(\"child task text\", name=\"kid\")\n open({receipt:?}, \"w\").write(json.dumps({{\"rlm_child_id\": handle.rlm_child_id, \"session_dir\": str(handle.session_dir)}}))\n print(handle.rlm_child_id)\nexcept Exception:\n open({error_receipt:?}, \"w\").write(traceback.format_exc())\n raise",
+ receipt = receipt.display().to_string(),
+ error_receipt = error_receipt.display().to_string(),
+ )
+}
+
+/// Parse one semantic-edge ledger file into its events. A file that does
+/// not exist yet reads as empty: the settle poll runs ahead of the
+/// parent's return recording.
+fn semantic_ledger(path: &Path) -> Vec<LedgerEvent> {
+ let Ok(contents) = std::fs::read_to_string(path) else {
+ return Vec::new();
+ };
+ contents
+ .lines()
+ .filter(|line| !line.is_empty())
+ .map(|line| serde_json::from_str(line).expect("ledger line"))
+ .collect()
+}
+
+/// The durable session id the child's `session_registered` names (the
+/// parent-side `child_returned` assertions pin the same id across both
+/// ledgers).
+fn child_ledger_session_id(ledger: &[LedgerEvent]) -> String {
+ ledger
+ .iter()
+ .find_map(|event| match event {
+ LedgerEvent::SessionRegistered { session_id, .. } => Some(session_id.clone()),
+ _ => None,
+ })
+ .expect("the child ledger registered its session")
+}
+
+/// The kernel cell of the delete turn: delete the spawned child through
+/// the product `rlm.delete_subagent` surface and record its verdict.
+fn semantic_delete_cell(
+ spawn_receipt: &Path,
+ delete_receipt: &Path,
+ delete_error: &Path,
+) -> String {
+ format!(
+ "import json, traceback\ntry:\n child = json.load(open({spawn_receipt:?}))\n sub = await rlm.delete_subagent(child[\"rlm_child_id\"])\n open({delete_receipt:?}, \"w\").write(json.dumps({{\"status\": sub.status, \"rlm_child_id\": sub.rlm_child_id}}))\n print(sub.status)\nexcept Exception:\n open({delete_error:?}, \"w\").write(traceback.format_exc())\n raise",
+ spawn_receipt = spawn_receipt.display().to_string(),
+ delete_receipt = delete_receipt.display().to_string(),
+ delete_error = delete_error.display().to_string(),
+ )
+}
+
+/// The parent's faux script whose two turns run the spawn cell, then the
+/// delete cell (each toolCall turn ends with its own text response).
+fn write_two_turn_parent_script(dir: &Path, first_cell: &str, second_cell: &str) -> PathBuf {
+ let script = dir.join("parent.json");
+ std::fs::write(
+ &script,
+ json!({
+ "engine": "faux",
+ "responses": [
+ { "content": [
+ { "type": "toolCall", "name": "python_repl", "arguments": { "code": first_cell } },
+ ] },
+ { "text": "spawn turn done" },
+ { "content": [
+ { "type": "toolCall", "name": "python_repl", "arguments": { "code": second_cell } },
+ ] },
+ { "text": "delete turn done" },
+ ],
+ })
+ .to_string(),
+ )
+ .expect("write parent script");
+ script
+}
+
+/// A delete that kills a live run records the cancel edge on the parent's
+/// ledger (nano-rlm #202's delete gap): the `rlm.delete_subagent` tool runs
+/// mid-turn, so the `child_cancelled` line anchors to that deleting request
+/// and lands before the deleting turn commits — a deleted child leaves a
+/// cancel line, not a clean disappearance, and a cut-short run never
+/// records a `child_returned` one.
+#[test]
+fn a_deleted_running_child_records_the_cancel_edge() {
+ let Some(kernel_python) = kernel_python() else {
+ return;
+ };
+ let dir = tempfile::TempDir::new().expect("temp dir");
+ let agent_dir = dir.path().join("agent");
+ std::fs::create_dir_all(&agent_dir).expect("agent dir");
+ let socket = dir.path().join("supervisor.sock");
+ let receipts = dir.path().join("receipts");
+ std::fs::create_dir_all(&receipts).expect("receipts dir");
+ let spawn_receipt = receipts.join("spawn.json");
+ let spawn_error = receipts.join("spawn.error");
+ let delete_receipt = receipts.join("delete.json");
+ let delete_error = receipts.join("delete.error");
+
+ // The child's delayed reply keeps its task turn running while the
+ // delete lands (a live run the parent cuts short).
+ let child_script = dir.path().join("child.json");
+ std::fs::write(
+ &child_script,
+ json!({
+ "engine": "faux",
+ "responses": [ { "text": "child reply", "delayMs": 30_000 } ],
+ })
+ .to_string(),
+ )
+ .expect("write child script");
+ let parent_script = write_two_turn_parent_script(
+ dir.path(),
+ &semantic_spawn_cell(&spawn_receipt, &spawn_error),
+ &semantic_delete_cell(&spawn_receipt, &delete_receipt, &delete_error),
+ );
+ let _daemon = spawn_supervisor(&socket, &agent_dir, &kernel_python);
+ wait_socket_ready(&socket);
+ let (mut client, hello) = Client::connect(&socket);
+ assert_eq!(hello["type"], "daemon_hello");
+ let parent = create_parent(&mut client, dir.path(), &parent_script, &child_script, "c1");
+ let parent_id = parent["activeSessionId"]
+ .as_str()
+ .or_else(|| parent["id"].as_str())
+ .expect("parent active session id")
+ .to_string();
+ let parent_session_id = parent["sessionId"]
+ .as_str()
+ .expect("parent session id")
+ .to_string();
+ let parent_session_file = parent["sessionFile"]
+ .as_str()
+ .expect("parent session file")
+ .to_string();
+
+ // Turn 1: the kernel cell spawns the child mid-turn.
+ run_turn(&mut client, &parent_id, "spawn the kid", "t1");
+ let spawned: Value =
+ serde_json::from_str(&await_receipt(&spawn_receipt)).expect("spawn receipt json");
+ assert!(
+ !spawn_error.exists(),
+ "the spawn cell failed: {}",
+ std::fs::read_to_string(&spawn_error).unwrap_or_default()
+ );
+ let child_id = spawned["rlm_child_id"]
+ .as_str()
+ .expect("child id")
+ .to_string();
+ let child_session_dir = spawned["session_dir"]
+ .as_str()
+ .expect("child session dir")
+ .to_string();
+
+ // Turn 2: the kernel cell deletes the running child mid-turn.
+ run_turn(&mut client, &parent_id, "delete the kid", "t2");
+ let deleted: Value =
+ serde_json::from_str(&await_receipt(&delete_receipt)).expect("delete receipt json");
+ assert!(
+ !delete_error.exists(),
+ "the delete cell failed: {}",
+ std::fs::read_to_string(&delete_error).unwrap_or_default()
+ );
+ assert_eq!(deleted["rlm_child_id"], json!(child_id));
+ assert_eq!(
+ deleted["status"], "cancelled",
+ "a deleted running child reports the cancelled row: {deleted}"
+ );
+
+ // The ledger paths are spelled independently of the product's
+ // derivation on purpose: byte-compatible LOCATIONS are part of the TS
+ // parity contract this e2e pins.
+ let parent_ledger_path = std::path::Path::new(&parent_session_file)
+ .parent()
+ .and_then(Path::parent)
+ .map(|sessions| sessions.join("session-artifacts").join(&parent_session_id))
+ .map(|artifacts| artifacts.join("semantic-edges.jsonl"))
+ .expect("parent ledger path");
+ let child_ledger_path = Path::new(&child_session_dir).join("semantic-edges.jsonl");
+ // The cancel line lands while the deleting turn is still open; poll
+ // until it exists (the same deadline; the terminal shape, not a fixed
+ // delay).
+ let settle_deadline = Instant::now() + Duration::from_mins(2);
+ let (parent_ledger, child_ledger) = loop {
+ let parent_ledger = semantic_ledger(&parent_ledger_path);
+ let child_ledger = semantic_ledger(&child_ledger_path);
+ // The cancel line names its parent-side actor: the delete, not a
+ // user cancel or a close walk.
+ let cancelled = parent_ledger.iter().any(|event| {
+ matches!(
+ event,
+ LedgerEvent::ChildCancelled { child_session_id, reason: ChildCancelledReason::ParentDelete, .. }
+ if child_session_id == &child_ledger_session_id(&child_ledger)
+ )
+ });
+ if cancelled {
+ break (parent_ledger, child_ledger);
+ }
+ assert!(
+ Instant::now() < settle_deadline,
+ "the parent never recorded the cancel: parent {parent_ledger:?}; child {child_ledger:?}"
+ );
+ std::thread::sleep(Duration::from_millis(200));
+ };
+ // The cancel line names the parent, the child, and the IN-FLIGHT
+ // deleting request: the last request the parent started before the
+ // cancel line, still open at the line (its commit follows), and never
+ // the spawn turn's request (that one committed long before).
+ let cancelled_at = parent_ledger
+ .iter()
+ .position(|event| {
+ matches!(
+ event,
+ LedgerEvent::ChildCancelled {
+ session_id,
+ child_session_id,
+ ..
+ } if session_id == &parent_session_id
+ && child_session_id == &child_ledger_session_id(&child_ledger)
+ )
+ })
+ .expect("the parent recorded the child's cancel");
+ let (anchored_request, spawning_request) = {
+ let anchored_request = match &parent_ledger[cancelled_at] {
+ LedgerEvent::ChildCancelled { request_id, .. } => request_id.clone(),
+ _ => unreachable!("the position above matched a cancel line"),
+ };
+ let spawning_request = parent_ledger
+ .iter()
+ .find_map(|event| match event {
+ LedgerEvent::RequestStarted { request_id, .. } => Some(request_id.clone()),
+ _ => None,
+ })
+ .expect("the parent's first request_started");
+ (anchored_request, spawning_request)
+ };
+ assert_ne!(
+ anchored_request, spawning_request,
+ "the cancel anchors to the deleting turn, not the spawn turn: {parent_ledger:?}"
+ );
+ // The anchored request is the LAST request the parent started before
+ // the cancel line: the deleting turn's request (the one whose tool
+ // call ran `rlm.delete_subagent`). A later request — the cancelled
+ // child's follow-up notice turn — only starts after the line lands.
+ assert!(
+ parent_ledger[..cancelled_at]
+ .iter()
+ .rev()
+ .find_map(|event| match event {
+ LedgerEvent::RequestStarted { request_id, .. } => Some(request_id.clone()),
+ _ => None,
+ })
+ .is_some_and(|last_started| last_started == anchored_request),
+ "the cancel line anchors to the deleting turn's last-started request: {parent_ledger:?}"
+ );
+ // A cut-short run never returns: no `child_returned` line may name the
+ // deleted child.
+ assert!(
+ parent_ledger.iter().all(|event| {
+ !matches!(
+ event,
+ LedgerEvent::ChildReturned { child_session_id, .. }
+ if child_session_id == &child_ledger_session_id(&child_ledger)
+ )
+ }),
+ "a deleted running child records no return line: {parent_ledger:?}"
+ );
+}