A self-improving RLM agent for coding workflows and long-running autonomous tasks.
Rust
send a failure notice when rlm children fail (#3139)
Sebastian Müller2adb5b57ffdf
diff --git a/.changes/fix-rlm-child-failure-notice.md b/.changes/fix-rlm-child-failure-notice.mdnew file mode 100644index 0000000000..295fd54bdb--- /dev/null+++ b/.changes/fix-rlm-child-failure-notice.md@@ -0,0 +1 @@+- A subagent that fails because its task cannot be delivered, or whose worker becomes unreachable, now sends a failure notice with the error to its parent, and a goal waiting on it resumes.diff --git a/crates/pa-daemon/src/rlm_children.rs b/crates/pa-daemon/src/rlm_children.rsindex e97641474e..378baefef3 100644--- a/crates/pa-daemon/src/rlm_children.rs+++ b/crates/pa-daemon/src/rlm_children.rs@@ -24,10 +24,11 @@ use pa_core::session_engine::rlm_host::{RlmSubagentHost,};use pa_core::session_engine::rlm_notices::{- create_rlm_child_terminal_notice, RlmChildTerminalNotice,+ create_rlm_child_failure_message, create_rlm_child_terminal_notice, RlmChildTerminalNotice,};use pa_core::session_engine::rlm_usage::{RlmChildUsageReport, RlmChildUsageSink};use pa_types::daemon::{DaemonCommand, DaemonSessionLifecycle, PromptInput};+use pa_types::session::CustomMessage;use serde_json::{json, Value};use tokio::sync::Mutex;diff --git a/crates/pa-daemon/src/rlm_children/host.rs b/crates/pa-daemon/src/rlm_children/host.rsindex 0e27fe62fb..783686ed7e 100644--- a/crates/pa-daemon/src/rlm_children/host.rs+++ b/crates/pa-daemon/src/rlm_children/host.rs@@ -3,13 +3,13 @@//! `collect`) over the supervisor's child-sessions registry, with the//! spawn-admission helpers only this surface uses.use super::{- assert_thinking_supported, bail, create_default_rlm_subagent_session_name, json, now_ms,- resolve_child_model, rlm_child_label, spawn_name_unavailable, Arc, ChildCloseReason,- ChildRecord, Context, DaemonCommand, Duration, Instant, Mutex, Path, PathBuf, Result,- RlmChildResult, RlmChildTerminalNotice, RlmCreateSessionHandle, RlmCreateSessionRequest,- RlmDeleteSubagentResult, RlmHostFuture, RlmSpawnHandle, RlmSpawnRequest, RlmSubagentEntry,- RlmSubagentHost, SpawnNameReservationGuard, SupervisorChildSessions,- SupervisorChildSessionsInner, Value, KILL_TIMEOUT_MS,+ assert_thinking_supported, bail, create_default_rlm_subagent_session_name,+ create_rlm_child_terminal_notice, json, now_ms, resolve_child_model, rlm_child_label,+ spawn_name_unavailable, Arc, ChildCloseReason, ChildRecord, Context, DaemonCommand, Duration,+ Instant, Mutex, Path, PathBuf, Result, RlmChildResult, RlmChildTerminalNotice,+ RlmCreateSessionHandle, RlmCreateSessionRequest, RlmDeleteSubagentResult, RlmHostFuture,+ RlmSpawnHandle, RlmSpawnRequest, RlmSubagentEntry, RlmSubagentHost, SpawnNameReservationGuard,+ SupervisorChildSessions, SupervisorChildSessionsInner, Value, KILL_TIMEOUT_MS,};/// Resolve the child model with the daemon `allowedModels` allowlist@@ -216,8 +216,13 @@ impl RlmSubagentHost for SupervisorChildSessions {let _ = watcher_this.kill_child(&child_active_session_id, ChildCloseReason::Killed).await;- watcher_record.lock().await.settled_status = Some("error");- watcher_this.fire_settle_hook(&watcher_record).await;+ watcher_this+ .settle_failed(+ &watcher_record,+ format!("{retry_error:#}"),+ super::lifecycle::FailedArm::Prompt,+ )+ .await;return;}}@@ -405,7 +410,11 @@ impl RlmSubagentHost for SupervisorChildSessions {})};if let Some(notice) = notice {- this.deliver_terminal_notice(¬ice).await;+ this.deliver_terminal_notice(create_rlm_child_terminal_notice(+ ¬ice,+ now_ms(),+ ))+ .await;}}// The deletion settles the run (TS `_finishRlmRunDeletion`,diff --git a/crates/pa-daemon/src/rlm_children/lifecycle.rs b/crates/pa-daemon/src/rlm_children/lifecycle.rsindex 9d0eac12c8..08642ab470 100644--- a/crates/pa-daemon/src/rlm_children/lifecycle.rs+++ b/crates/pa-daemon/src/rlm_children/lifecycle.rs@@ -2,13 +2,13 @@//! prompt/kill/close routing, settle watching with its notices, and the//! spawn-admission outbox types (`CreatedSessionIds`, `CreatedChild`).use super::{- anyhow, compact_rlm_text, create_rlm_child_terminal_notice, json, now_ms, Arc,- ChildCloseReason, ChildRecord, Context, DaemonCommand, DaemonSessionLifecycle, Duration, Map,- Mutex, ParentIdentity, Path, PromptInput, Result, RlmChildTerminalNotice,- SupervisorChildSessionsInner, Value, CREATE_TIMEOUT_MS, IDLE_WAIT_GRACE_MS, KILL_TIMEOUT_MS,- NOTICE_DELIVERY_TIMEOUT_MS, PROMPT_TIMEOUT_MS, RUNTIME_METADATA_PROMPT_MAX, STATE_TIMEOUT_MS,- WATCH_MAX_UNREACHABLE_POLLS, WATCH_POLL_INTERVAL_MS, WATCH_SETTLE_GRACE_MS,- WATCH_WAIT_SLICE_MS,+ anyhow, compact_rlm_text, create_rlm_child_failure_message, create_rlm_child_terminal_notice,+ json, now_ms, Arc, ChildCloseReason, ChildRecord, Context, CustomMessage, DaemonCommand,+ DaemonSessionLifecycle, Duration, Map, Mutex, ParentIdentity, Path, PromptInput, Result,+ RlmChildTerminalNotice, SupervisorChildSessionsInner, Value, CREATE_TIMEOUT_MS,+ IDLE_WAIT_GRACE_MS, KILL_TIMEOUT_MS, NOTICE_DELIVERY_TIMEOUT_MS, PROMPT_TIMEOUT_MS,+ RUNTIME_METADATA_PROMPT_MAX, STATE_TIMEOUT_MS, WATCH_MAX_UNREACHABLE_POLLS,+ WATCH_POLL_INTERVAL_MS, WATCH_SETTLE_GRACE_MS, WATCH_WAIT_SLICE_MS,};/// Parsed ids of one created child session.@@ -44,7 +44,7 @@ impl CreatedChild {/// The plain text of a custom row's content (the notice turn's model/// prompt); `None` for non-text content shapes.-fn custom_message_text(message: &pa_types::session::CustomMessage) -> Option<String> {+fn custom_message_text(message: &CustomMessage) -> Option<String> {match &message.content {pa_types::ai::UserContent::Text(text) => Some(text.clone()),pa_types::ai::UserContent::Blocks(_) => None,@@ -451,20 +451,30 @@ impl SupervisorChildSessionsInner {} else {unreachable_polls += 1;if unreachable_polls >= WATCH_MAX_UNREACHABLE_POLLS {- {- let mut state = record.lock().await;- if !should_mark_unreachable_error(&state) {- return;- }- state.settled_status = Some("error");- state.error = Some("Child worker unreachable".to_string());- }// A dead child keeps whatever rows its file already// holds; capture them before the terminal notice.self.emit_child_usage(record).await;- self.deliver_settle_notice(record).await;- self.fire_settle_hook(record).await;- return;+ if self+ .settle_failed(+ record,+ "Child worker unreachable".to_string(),+ FailedArm::Unreachable,+ )+ .await+ {+ return;+ }+ // The claim lost to a verdict a reader's refresh landed+ // after this pass's settle read (the worker answered+ // idle, so the dead streak is broken): the settled+ // branch owes that verdict its notice and funnel. With+ // no verdict left, a cancel or a parent close owns the+ // tail.+ if record.lock().await.settled_status.is_none() {+ return;+ }+ unreachable_polls = 0;+ continue;}}tokio::time::sleep(Duration::from_millis(WATCH_POLL_INTERVAL_MS)).await;@@ -476,19 +486,72 @@ impl SupervisorChildSessionsInner {/// notice claim collapses the races between the watcher, a natural/// settle during `delete_subagent`, and the delete path itself.async fn deliver_settle_notice(&self, record: &Arc<Mutex<ChildRecord>>) {- let notice = {+ let message = {let mut record = record.lock().await;if record.notice_delivered || record.replied_since_task {return;}record.notice_delivered = true;- RlmChildTerminalNotice::CompletedWithoutReply {+ let notice = RlmChildTerminalNotice::CompletedWithoutReply {child_id: record.rlm_child_id.clone(),session_name: record.session_name.clone(),last_assistant_text_preview: record.answer_preview.clone(),+ };+ create_rlm_child_terminal_notice(¬ice, now_ms())+ };+ self.deliver_terminal_notice(message).await;+ }++ /// Settle one child as failed (TS's thrown-run arm): the record+ /// settles `error` with its failure text and the parent receives the+ /// `rlm_child_failure` row — whether or not the child replied — then+ /// the settle funnel fires like TS's `finally` resume. Exactly-once:+ /// the claim is split by arm. The unreachable arm keeps+ /// `should_mark_unreachable_error` — an already-settled child keeps+ /// its verdict (#3104's passivation semantics: a verdict a reader's+ /// refresh lands inside the give-up keeps standing, and the claim+ /// reports the loss so the watcher runs that verdict's settle tail).+ /// The prompt arm bails only on a REAL terminal verdict:+ /// a task turn that provably never started cannot have genuinely+ /// completed, so a `done` visible inside the prompt-failure window is+ /// `refresh_record`'s admission-window misread of an alive-but-idle+ /// worker, not a settle verdict — the failure row must still land+ /// (and, since #3171, the run must still mark settled, or the+ /// quiescence barrier parks forever). Returns whether this call+ /// claimed the settle.+ pub(super) async fn settle_failed(+ &self,+ record: &Arc<Mutex<ChildRecord>>,+ error: String,+ arm: FailedArm,+ ) -> bool {+ let message = {+ let mut record = record.lock().await;+ let keep_verdict = match arm {+ FailedArm::Unreachable => !should_mark_unreachable_error(&record),+ FailedArm::Prompt => {+ record.closed_by_parent+ || record.notice_delivered+ || matches!(record.settled_status, Some("error" | "cancelled"))+ }+ };+ if keep_verdict {+ return false;}+ record.settled_status = Some("error");+ record.notice_delivered = true;+ let message = create_rlm_child_failure_message(+ &record.rlm_child_id,+ &record.session_name,+ &error,+ now_ms(),+ );+ record.error = Some(error);+ message};- self.deliver_terminal_notice(¬ice).await;+ self.deliver_terminal_notice(message).await;+ self.fire_settle_hook(record).await;+ true}/// Deliver one terminal notice into the parent session: the notice rides@@ -501,8 +564,7 @@ impl SupervisorChildSessionsInner {/// reserved-kind row exclusively with a live mint, and answers/// anything a caller sends — with or without a guessed nonce —/// loudly instead.- pub(super) async fn deliver_terminal_notice(&self, notice: &RlmChildTerminalNotice) {- let message = create_rlm_child_terminal_notice(notice, now_ms());+ pub(super) async fn deliver_terminal_notice(&self, message: CustomMessage) {let Some(content) = custom_message_text(&message) else {eprintln!("pa-daemon: RLM child notice carried no text content");return;@@ -538,12 +600,23 @@ impl SupervisorChildSessionsInner {}}-/// Whether the unreachable-poller may re-score a child as an error: a-/// parent-closed child, a noticed child, or an ALREADY-SETTLED child-/// keeps its POSITIVE verdict — its worker leaving afterward (the idle-/// passivation's graceful stop, a crash after settle, or a give-up) is-/// residency churn, not a settle verdict change. The settle state is-/// durable and the passive roster row stays the representation.+/// Which failure arm is claiming the settle: the two arms bail on+/// different evidence (see `settle_failed`).+pub(super) enum FailedArm {+ /// The task-prompt admission failed and its retry failed.+ Prompt,+ /// The unreachable-poller's give-up (`WATCH_MAX_UNREACHABLE_POLLS`+ /// consecutive dead polls).+ Unreachable,+}++/// Whether a child may still settle as an error (the unreachable+/// give-up's claim): a parent-closed child, a noticed child, or an+/// ALREADY-SETTLED child keeps its POSITIVE verdict — its worker leaving+/// afterward (the idle passivation's graceful stop, a crash after+/// settle, or a give-up) is residency churn, not a settle verdict+/// change. The settle state is durable and the passive roster row+/// stays the representation.pub(super) fn should_mark_unreachable_error(state: &ChildRecord) -> bool {!(state.closed_by_parent || state.notice_delivered || state.settled_status.is_some())}diff --git a/crates/pa-daemon/src/rlm_children/watch_tests.rs b/crates/pa-daemon/src/rlm_children/watch_tests.rsindex cc637d375f..cfa56936e9 100644--- a/crates/pa-daemon/src/rlm_children/watch_tests.rs+++ b/crates/pa-daemon/src/rlm_children/watch_tests.rs@@ -1,8 +1,9 @@use super::*;use crate::protocol::{response_failure, response_success};use pa_types::platform::transport::bind_transport;+use pa_types::session::AgentMessage;use serde_json::{json, Value};-use std::sync::atomic::{AtomicBool, Ordering};+use std::sync::atomic::{AtomicBool, AtomicU32, Ordering};use std::sync::Arc;use tokio::io::{AsyncBufReadExt, AsyncWriteExt, BufReader};use tokio::sync::mpsc;@@ -17,27 +18,58 @@ enum FakeKill {Failure,}+/// How the fake answers the child link: healthy, a failing `prompt`,+/// or a failing `get_state` (unreachable).+#[derive(Clone, Copy)]+enum FakeChild {+ Healthy,+ PromptFails,+ Unreachable,+ /// The worker leaves right after the settle answer is captured:+ /// every later child read fails.+ LeavesAfterSettle,+ /// Unreachable; the watcher's give-up poll parks until the test+ /// lands a reader's verdict.+ UnreachableUntilVerdict,+}+/// A scripted JSONL supervisor for the watcher tests: creates one child/// session, reports it idle with a final answer, and captures the/// `follow_up` commands routed to the parent (the terminal-notice/// deliveries). `idle_delay_ms` paces `wait_for_idle` so a test can act-/// while the child is still "running". `worker_leaves_after_settle`-/// fails every child read after the settle answer is captured.+/// while the child is still "running". `child` scripts the child link+/// (`FakeChild`).async fn spawn_fake_supervisor(socket: std::path::PathBuf,follow_up_tx: mpsc::UnboundedSender<Value>,idle_delay_ms: u64,kill_tx: mpsc::UnboundedSender<Value>,kill_behavior: FakeKill,- worker_leaves_after_settle: bool,+ child: FakeChild,child_subagents: Arc<FakeChildSubagents>,) {let kill_behavior = std::sync::Arc::new(kill_behavior);+ // Per-fake child session file: a fixed path would let a leftover file+ // from another test (or run) carry the prompt text, flip the+ // prompt-retry arbitration to `landed`, and turn an expected failure+ // row into a no-reply notice.+ let child_session_file = std::sync::Arc::new(+ std::env::temp_dir()+ .join(format!(+ "pa-rlm-watch-child-{}.jsonl",+ uuid::Uuid::new_v4().simple()+ ))+ .to_string_lossy()+ .to_string(),+ );let listener = bind_transport(&socket).await.unwrap();tokio::spawn(async move {// Shared across link connections: a left worker fails every child// read on whichever connection carries it.let gone = Arc::new(AtomicBool::new(false));+ // Shared across link connections: the give-up gate counts the+ // child's state reads on whichever connection carries them.+ let state_reads = Arc::new(AtomicU32::new(0));loop {let Ok(stream) = listener.accept().await else {return;@@ -46,6 +78,8 @@ async fn spawn_fake_supervisor(let kill_tx = kill_tx.clone();let kill_behavior = std::sync::Arc::clone(&kill_behavior);let gone = Arc::clone(&gone);+ let state_reads = Arc::clone(&state_reads);+ let child_session_file = std::sync::Arc::clone(&child_session_file);let child_subagents = Arc::clone(&child_subagents);tokio::spawn(async move {let (reader, mut writer) = stream.split();@@ -66,6 +100,34 @@ async fn spawn_fake_supervisor(let command = value["command"].clone();let command_type: &str = command["type"].as_str().unwrap_or_default();let response = match command_type {+ _ if matches!(+ (child, command_type),+ (FakeChild::PromptFails, "prompt")+ | (+ FakeChild::Unreachable | FakeChild::UnreachableUntilVerdict,+ "get_state",+ )+ ) =>+ {+ // Two get_state reads per watcher pass for the+ // parked variant (the refresh read, then the+ // liveness poll): the last one is the give-up+ // poll. Only that variant counts, so a plain+ // unreachable child keeps its instant refusal.+ if matches!(child, FakeChild::UnreachableUntilVerdict)+ && state_reads.fetch_add(1, Ordering::SeqCst) + 1+ == 2 * WATCH_MAX_UNREACHABLE_POLLS+ {+ GIVE_UP_POLL.notify_one();+ VERDICT_LANDED.notified().await;+ }+ response_failure(+ Some(&id),+ command_type,+ "refused by the fake supervisor",+ None,+ )+ }"get_state" | "wait_for_idle" if gone.load(Ordering::SeqCst) => {response_failure(Some(&id),@@ -80,7 +142,7 @@ async fn spawn_fake_supervisor(Some(json!({"activeSessionId": "child-live","sessionId": "child-file",- "sessionFile": "/tmp/child.jsonl",+ "sessionFile": *child_session_file,"sessionName": "f20-worker",})),),@@ -112,7 +174,7 @@ async fn spawn_fake_supervisor("get_last_assistant_text" => {// The settle capture: with the knob set, the// worker leaves right after its final answer.- if worker_leaves_after_settle {+ if matches!(child, FakeChild::LeavesAfterSettle) {gone.store(true, Ordering::SeqCst);}response_success(@@ -166,13 +228,13 @@ async fn sessions_with_fake_supervisor(follow_up_tx: mpsc::UnboundedSender<Value>,idle_delay_ms: u64,kill_behavior: FakeKill,- worker_leaves_after_settle: bool,+ child: FakeChild,) -> (SupervisorChildSessions, mpsc::UnboundedReceiver<Value>) {sessions_with_fake_child_subagents(follow_up_tx,idle_delay_ms,kill_behavior,- worker_leaves_after_settle,+ child,Arc::new(FakeChildSubagents::default()),).await@@ -193,7 +255,7 @@ async fn sessions_with_fake_child_subagents(follow_up_tx: mpsc::UnboundedSender<Value>,idle_delay_ms: u64,kill_behavior: FakeKill,- worker_leaves_after_settle: bool,+ child: FakeChild,child_subagents: Arc<FakeChildSubagents>,) -> (SupervisorChildSessions, mpsc::UnboundedReceiver<Value>) {let socket = std::env::temp_dir().join(format!(@@ -207,7 +269,7 @@ async fn sessions_with_fake_child_subagents(idle_delay_ms,kill_tx,kill_behavior,- worker_leaves_after_settle,+ child,child_subagents,).await;@@ -248,7 +310,8 @@ async fn spawn_child(sessions: &SupervisorChildSessions) -> RlmSpawnHandle {async fn roster_snapshot_does_not_wait_for_a_slow_child_worker() {let (follow_up_tx, _follow_up_rx) = mpsc::unbounded_channel();let (sessions, _kill_rx) =- sessions_with_fake_supervisor(follow_up_tx, 1_000, FakeKill::Success, false).await;+ sessions_with_fake_supervisor(follow_up_tx, 1_000, FakeKill::Success, FakeChild::Healthy)+ .await;sessions.push_test_child(RlmChildIdentity {rlm_child_id: "child-id".to_string(),@@ -271,7 +334,7 @@ async fn roster_snapshot_does_not_wait_for_a_slow_child_worker() {async fn a_settled_child_without_a_reply_delivers_the_terminal_notice() {let (follow_up_tx, mut follow_up_rx) = mpsc::unbounded_channel();let (sessions, _kill_rx) =- sessions_with_fake_supervisor(follow_up_tx, 0, FakeKill::Success, false).await;+ sessions_with_fake_supervisor(follow_up_tx, 0, FakeKill::Success, FakeChild::Healthy).await;let handle = spawn_child(&sessions).await;// The worker releases the detached prompt at its turn boundary.sessions.notify_turn_done();@@ -307,8 +370,13 @@ async fn a_settled_child_without_a_reply_delivers_the_terminal_notice() {#[tokio::test]async fn a_worker_leaving_inside_the_settle_grace_keeps_the_verdict() {let (follow_up_tx, mut follow_up_rx) = mpsc::unbounded_channel();- let (sessions, _kill_rx) =- sessions_with_fake_supervisor(follow_up_tx, 0, FakeKill::Success, true).await;+ let (sessions, _kill_rx) = sessions_with_fake_supervisor(+ follow_up_tx,+ 0,+ FakeKill::Success,+ FakeChild::LeavesAfterSettle,+ )+ .await;let settled = sessions.settle_notified();spawn_child(&sessions).await;sessions.notify_turn_done();@@ -327,6 +395,223 @@ async fn a_worker_leaving_inside_the_settle_grace_keeps_the_verdict() {.is_some_and(|content| content.contains("the child final answer")));}+/// The `follow_up` carries exactly the TS failure row (`createRlmChildFailureMessage`).+fn assert_failure_row(follow_up: &Value, child_id: &str, error: &str) {+ let custom = &follow_up["customMessage"];+ let timestamp = custom["timestamp"].as_u64().expect("failure row timestamp");+ let expected = create_rlm_child_failure_message(child_id, "f20-worker", error, timestamp);+ assert_eq!(+ *custom,+ serde_json::to_value(AgentMessage::Custom(expected)).unwrap()+ );+}++/// A child whose task prompt cannot be routed settles `error`, delivers+/// the TS failure row and releases the owed continuation.+#[tokio::test]+async fn a_child_whose_task_prompt_cannot_be_routed_delivers_the_failure_notice() {+ let (follow_up_tx, mut follow_up_rx) = mpsc::unbounded_channel();+ let (sessions, _kill_rx) =+ sessions_with_fake_supervisor(follow_up_tx, 0, FakeKill::Success, FakeChild::PromptFails)+ .await;+ let (hook_tx, mut hook_rx) = mpsc::unbounded_channel();+ sessions.set_settle_hook(Arc::new(move || {+ let _ = hook_tx.send(());+ }));+ let handle = spawn_child(&sessions).await;+ // The worker releases the detached prompt at its turn boundary.+ sessions.notify_turn_done();++ let follow_up = tokio::time::timeout(Duration::from_secs(10), follow_up_rx.recv())+ .await+ .expect("the failed child must deliver its failure notice")+ .expect("the follow_up channel stays open");+ assert_failure_row(+ &follow_up,+ &handle.rlm_child_id,+ "prompt RLM child session child-live: refused by the fake supervisor",+ );+ let entries = sessions.list_subagents().await.expect("child roster");+ assert_eq!(entries[0].status, "error");+ tokio::time::timeout(Duration::from_secs(10), hook_rx.recv())+ .await+ .expect("the failed child releases the owed continuation")+ .expect("hook channel open");+ // Exactly one failure row lands: the claim admits one writer.+ let extra = tokio::time::timeout(Duration::from_millis(300), follow_up_rx.recv()).await;+ assert!(extra.is_err(), "no second notice may arrive");+}++/// An unreachable child delivers the failure row, not the no-reply notice.+/// A short tick keeps the paused clock's auto-advance from firing link+/// deadlines while the real-socket round trips are in flight.+#[tokio::test(start_paused = true)]+async fn an_unreachable_child_delivers_the_failure_notice_instead_of_no_reply() {+ tokio::spawn(async {+ let mut tick = tokio::time::interval(Duration::from_secs(1));+ loop {+ tick.tick().await;+ }+ });+ let (follow_up_tx, mut follow_up_rx) = mpsc::unbounded_channel();+ let (sessions, _kill_rx) =+ sessions_with_fake_supervisor(follow_up_tx, 0, FakeKill::Success, FakeChild::Unreachable)+ .await;+ let handle = spawn_child(&sessions).await;+ sessions.notify_turn_done();++ let follow_up = tokio::time::timeout(Duration::from_secs(3_600), follow_up_rx.recv())+ .await+ .expect("the unreachable child must deliver its failure notice")+ .expect("follow_up channel open");+ assert_failure_row(&follow_up, &handle.rlm_child_id, "Child worker unreachable");+ // Exactly one failure row lands: the give-up claims once.+ let extra = tokio::time::timeout(Duration::from_millis(300), follow_up_rx.recv()).await;+ assert!(extra.is_err(), "no second notice may arrive");+}++/// Hand-off into the unreachable give-up's poll: the fake parks the+/// give-up poll, the test lands the reader's verdict, the fake releases+/// the poll. Only the lost-claim test uses these.+static GIVE_UP_POLL: tokio::sync::Notify = tokio::sync::Notify::const_new();+static VERDICT_LANDED: tokio::sync::Notify = tokio::sync::Notify::const_new();++/// A reader's refresh can settle the child between the watcher's settle+/// read and its unreachable give-up claim: the claim keeps that verdict,+/// and the watcher still runs its settle tail.+#[tokio::test(start_paused = true)]+async fn a_verdict_landing_inside_the_unreachable_give_up_still_runs_the_settle_tail() {+ tokio::spawn(async {+ let mut tick = tokio::time::interval(Duration::from_secs(1));+ loop {+ tick.tick().await;+ }+ });+ let (follow_up_tx, mut follow_up_rx) = mpsc::unbounded_channel();+ let (sessions, _kill_rx) = sessions_with_fake_supervisor(+ follow_up_tx,+ 0,+ FakeKill::Success,+ FakeChild::UnreachableUntilVerdict,+ )+ .await;+ let settled = sessions.settle_notified();+ spawn_child(&sessions).await;+ sessions.notify_turn_done();++ GIVE_UP_POLL.notified().await;+ {+ // What a `collect`'s `refresh_record` writes when the worker+ // answers idle.+ let record = Arc::clone(&sessions.inner.children.lock().await[0]);+ let mut record = record.lock().await;+ record.settled_status = Some("done");+ }+ VERDICT_LANDED.notify_one();++ tokio::time::timeout(Duration::from_secs(3_600), settled)+ .await+ .expect("the settle funnel fires for the reader's verdict");+ assert!(!sessions.any_running().await, "the run must mark settled");+ let notice = follow_up_rx+ .try_recv()+ .expect("the verdict's no-reply notice is still owed");+ assert_eq!(+ notice["customMessage"]["customType"],+ "rlm_child_terminal_notice"+ );+}++/// The review's C1 interleaving: a `collect` that lands inside the+/// prompt-failure window (after the detached task flips+/// `prompt_admitted`, before the retry verdict) reads the alive-but-idle+/// worker as `done` — `refresh_record`'s admission-window misread, not a+/// settle verdict. The prompt arm's failure claim must ignore it: the+/// failure row still lands, the roster reports `error`, the hook fires,+/// and the run marks settled (post-#3171 an unsettled failed run parks+/// `waitForRlmQuiescence` forever).+#[tokio::test]+async fn a_collect_inside_the_prompt_failure_window_does_not_swallow_the_failure_row() {+ let (follow_up_tx, mut follow_up_rx) = mpsc::unbounded_channel();+ let (sessions, _kill_rx) =+ sessions_with_fake_supervisor(follow_up_tx, 0, FakeKill::Success, FakeChild::PromptFails)+ .await;+ let (hook_tx, mut hook_rx) = mpsc::unbounded_channel();+ sessions.set_settle_hook(Arc::new(move || {+ let _ = hook_tx.send(());+ }));+ let handle = spawn_child(&sessions).await;+ // The window's entry condition, set deterministically: the detached+ // prompt task flips `prompt_admitted` BEFORE its first+ // `prompt_child` (host.rs), and the collect below races that task in+ // production. With the flag set and the worker alive-but-idle, the+ // real collect path scores the misread.+ sessions.inner.children.lock().await[0]+ .lock()+ .await+ .prompt_admitted = true;+ let misread = sessions+ .collect(vec![handle.rlm_child_id.clone()], 0)+ .await+ .expect("collect inside the window");+ assert_eq!(+ misread[0].status, "done",+ "the premise: the collect misreads the pre-prompt idle worker as done"+ );++ // The task prompt now runs and fails (prompt + retry) inside the+ // window the collect already scored.+ sessions.notify_turn_done();++ let follow_up = tokio::time::timeout(Duration::from_secs(10), follow_up_rx.recv())+ .await+ .expect("the failure row must land despite the collect's misread")+ .expect("follow_up channel open");+ assert_failure_row(+ &follow_up,+ &handle.rlm_child_id,+ "prompt RLM child session child-live: refused by the fake supervisor",+ );+ let entries = sessions.list_subagents().await.expect("child roster");+ assert_eq!(entries[0].status, "error");+ tokio::time::timeout(Duration::from_secs(10), hook_rx.recv())+ .await+ .expect("the settle funnel fires despite the misread")+ .expect("hook channel open");+ assert!(+ !sessions.any_running().await,+ "the failed run must mark settled: an unsettled run parks the quiescence barrier"+ );+ let extra = tokio::time::timeout(Duration::from_millis(300), follow_up_rx.recv()).await;+ assert!(extra.is_err(), "no second notice may arrive");+}++/// A child that already replied still delivers the failure row when its+/// task prompt fails: TS sends the catch-arm row regardless of reply+/// count (`agent-session.ts:13026-13045`).+#[tokio::test]+async fn a_replied_child_whose_prompt_fails_still_delivers_the_failure_row() {+ let (follow_up_tx, mut follow_up_rx) = mpsc::unbounded_channel();+ let (sessions, _kill_rx) =+ sessions_with_fake_supervisor(follow_up_tx, 0, FakeKill::Success, FakeChild::PromptFails)+ .await;+ let handle = spawn_child(&sessions).await;+ sessions.mark_replied("child-live").await;+ sessions.notify_turn_done();++ let follow_up = tokio::time::timeout(Duration::from_secs(10), follow_up_rx.recv())+ .await+ .expect("the replied child must still deliver the failure row")+ .expect("follow_up channel open");+ assert_failure_row(+ &follow_up,+ &handle.rlm_child_id,+ "prompt RLM child session child-live: refused by the fake supervisor",+ );+ let extra = tokio::time::timeout(Duration::from_millis(300), follow_up_rx.recv()).await;+ assert!(extra.is_err(), "no second notice may arrive");+}+/// A parent counts as running while any descendant runs: the child's own/// turn is done, but while it reports a running grandchild the parent's/// row stays `running` and the registry keeps the parent's summary busy;@@ -343,7 +628,7 @@ async fn a_child_with_a_running_grandchild_keeps_the_parent_running() {follow_up_tx,0,FakeKill::Success,- false,+ FakeChild::Healthy,Arc::clone(&grandchild),).await;@@ -385,7 +670,7 @@ async fn collect_with_a_timeout_waits_for_a_running_grandchild() {follow_up_tx,0,FakeKill::Success,- false,+ FakeChild::Healthy,Arc::clone(&grandchild),).await;@@ -443,7 +728,8 @@ async fn close_children_stops_the_child_and_clears_the_roster() {// A long idle keeps the child mid-run while the close fires, so the// settle watcher is parked instead of raced.let (sessions, mut kill_rx) =- sessions_with_fake_supervisor(follow_up_tx, 10_000, FakeKill::Success, false).await;+ sessions_with_fake_supervisor(follow_up_tx, 10_000, FakeKill::Success, FakeChild::Healthy)+ .await;spawn_child(&sessions).await;sessions.notify_turn_done();one_running_child(&sessions).await;@@ -482,8 +768,13 @@ async fn close_children_stops_the_child_and_clears_the_roster() {#[tokio::test]async fn close_children_treats_an_already_gone_child_as_a_no_op() {let (follow_up_tx, _follow_up_rx) = mpsc::unbounded_channel();- let (sessions, _kill_rx) =- sessions_with_fake_supervisor(follow_up_tx, 10_000, FakeKill::UnknownSession, false).await;+ let (sessions, _kill_rx) = sessions_with_fake_supervisor(+ follow_up_tx,+ 10_000,+ FakeKill::UnknownSession,+ FakeChild::Healthy,+ )+ .await;spawn_child(&sessions).await;sessions.notify_turn_done();one_running_child(&sessions).await;@@ -507,7 +798,8 @@ async fn close_children_treats_an_already_gone_child_as_a_no_op() {async fn close_children_keeps_a_failed_child_tracked() {let (follow_up_tx, _follow_up_rx) = mpsc::unbounded_channel();let (sessions, _kill_rx) =- sessions_with_fake_supervisor(follow_up_tx, 10_000, FakeKill::Failure, false).await;+ sessions_with_fake_supervisor(follow_up_tx, 10_000, FakeKill::Failure, FakeChild::Healthy)+ .await;spawn_child(&sessions).await;sessions.notify_turn_done();one_running_child(&sessions).await;@@ -533,7 +825,8 @@ async fn a_replied_child_gets_no_terminal_notice() {// A slow idle wait keeps the child "running" while the test marks// the reply.let (sessions, _kill_rx) =- sessions_with_fake_supervisor(follow_up_tx, 250, FakeKill::Success, false).await;+ sessions_with_fake_supervisor(follow_up_tx, 250, FakeKill::Success, FakeChild::Healthy)+ .await;let handle = spawn_child(&sessions).await;assert!(!handle.rlm_child_id.is_empty());sessions.mark_replied("child-live").await;@@ -553,7 +846,7 @@ async fn a_replied_child_gets_no_terminal_notice() {async fn collect_answers_a_just_deleted_target_with_the_cancelled_envelope() {let (follow_up_tx, _follow_up_rx) = mpsc::unbounded_channel();let (sessions, _kill_rx) =- sessions_with_fake_supervisor(follow_up_tx, 0, FakeKill::Success, false).await;+ sessions_with_fake_supervisor(follow_up_tx, 0, FakeKill::Success, FakeChild::Healthy).await;let handle = spawn_child(&sessions).await;sessions.notify_turn_done();// The child settles with its final answer before the delete.@@ -626,7 +919,7 @@ async fn collect_answers_a_just_deleted_target_with_the_cancelled_envelope() {async fn collect_answers_the_cancelled_envelope_after_an_inactive_delete() {let (follow_up_tx, _follow_up_rx) = mpsc::unbounded_channel();let (sessions, _kill_rx) =- sessions_with_fake_supervisor(follow_up_tx, 0, FakeKill::Success, false).await;+ sessions_with_fake_supervisor(follow_up_tx, 0, FakeKill::Success, FakeChild::Healthy).await;let handle = spawn_child(&sessions).await;sessions.notify_turn_done();// The inactive delete requires a settled child (a running child