Skip to content

A self-improving RLM agent for coding workflows and long-running autonomous tasks.

Rust

send a failure notice when rlm children fail (#3139)

Sebastian Müller2adb5b57ffdf
diff --git a/.changes/fix-rlm-child-failure-notice.md b/.changes/fix-rlm-child-failure-notice.md
new file mode 100644
index 0000000000..295fd54bdb
--- /dev/null
+++ b/.changes/fix-rlm-child-failure-notice.md
@@ -0,0 +1 @@
+- A subagent that fails because its task cannot be delivered, or whose worker becomes unreachable, now sends a failure notice with the error to its parent, and a goal waiting on it resumes.
diff --git a/crates/pa-daemon/src/rlm_children.rs b/crates/pa-daemon/src/rlm_children.rs
index e97641474e..378baefef3 100644
--- a/crates/pa-daemon/src/rlm_children.rs
+++ b/crates/pa-daemon/src/rlm_children.rs
@@ -24,10 +24,11 @@ use pa_core::session_engine::rlm_host::{
RlmSubagentHost,
};
use pa_core::session_engine::rlm_notices::{
- create_rlm_child_terminal_notice, RlmChildTerminalNotice,
+ create_rlm_child_failure_message, create_rlm_child_terminal_notice, RlmChildTerminalNotice,
};
use pa_core::session_engine::rlm_usage::{RlmChildUsageReport, RlmChildUsageSink};
use pa_types::daemon::{DaemonCommand, DaemonSessionLifecycle, PromptInput};
+use pa_types::session::CustomMessage;
use serde_json::{json, Value};
use tokio::sync::Mutex;
diff --git a/crates/pa-daemon/src/rlm_children/host.rs b/crates/pa-daemon/src/rlm_children/host.rs
index 0e27fe62fb..783686ed7e 100644
--- a/crates/pa-daemon/src/rlm_children/host.rs
+++ b/crates/pa-daemon/src/rlm_children/host.rs
@@ -3,13 +3,13 @@
//! `collect`) over the supervisor's child-sessions registry, with the
//! spawn-admission helpers only this surface uses.
use super::{
- assert_thinking_supported, bail, create_default_rlm_subagent_session_name, json, now_ms,
- resolve_child_model, rlm_child_label, spawn_name_unavailable, Arc, ChildCloseReason,
- ChildRecord, Context, DaemonCommand, Duration, Instant, Mutex, Path, PathBuf, Result,
- RlmChildResult, RlmChildTerminalNotice, RlmCreateSessionHandle, RlmCreateSessionRequest,
- RlmDeleteSubagentResult, RlmHostFuture, RlmSpawnHandle, RlmSpawnRequest, RlmSubagentEntry,
- RlmSubagentHost, SpawnNameReservationGuard, SupervisorChildSessions,
- SupervisorChildSessionsInner, Value, KILL_TIMEOUT_MS,
+ assert_thinking_supported, bail, create_default_rlm_subagent_session_name,
+ create_rlm_child_terminal_notice, json, now_ms, resolve_child_model, rlm_child_label,
+ spawn_name_unavailable, Arc, ChildCloseReason, ChildRecord, Context, DaemonCommand, Duration,
+ Instant, Mutex, Path, PathBuf, Result, RlmChildResult, RlmChildTerminalNotice,
+ RlmCreateSessionHandle, RlmCreateSessionRequest, RlmDeleteSubagentResult, RlmHostFuture,
+ RlmSpawnHandle, RlmSpawnRequest, RlmSubagentEntry, RlmSubagentHost, SpawnNameReservationGuard,
+ SupervisorChildSessions, SupervisorChildSessionsInner, Value, KILL_TIMEOUT_MS,
};
/// Resolve the child model with the daemon `allowedModels` allowlist
@@ -216,8 +216,13 @@ impl RlmSubagentHost for SupervisorChildSessions {
let _ = watcher_this
.kill_child(&child_active_session_id, ChildCloseReason::Killed)
.await;
- watcher_record.lock().await.settled_status = Some("error");
- watcher_this.fire_settle_hook(&watcher_record).await;
+ watcher_this
+ .settle_failed(
+ &watcher_record,
+ format!("{retry_error:#}"),
+ super::lifecycle::FailedArm::Prompt,
+ )
+ .await;
return;
}
}
@@ -405,7 +410,11 @@ impl RlmSubagentHost for SupervisorChildSessions {
})
};
if let Some(notice) = notice {
- this.deliver_terminal_notice(&notice).await;
+ this.deliver_terminal_notice(create_rlm_child_terminal_notice(
+ &notice,
+ now_ms(),
+ ))
+ .await;
}
}
// The deletion settles the run (TS `_finishRlmRunDeletion`,
diff --git a/crates/pa-daemon/src/rlm_children/lifecycle.rs b/crates/pa-daemon/src/rlm_children/lifecycle.rs
index 9d0eac12c8..08642ab470 100644
--- a/crates/pa-daemon/src/rlm_children/lifecycle.rs
+++ b/crates/pa-daemon/src/rlm_children/lifecycle.rs
@@ -2,13 +2,13 @@
//! prompt/kill/close routing, settle watching with its notices, and the
//! spawn-admission outbox types (`CreatedSessionIds`, `CreatedChild`).
use super::{
- anyhow, compact_rlm_text, create_rlm_child_terminal_notice, json, now_ms, Arc,
- ChildCloseReason, ChildRecord, Context, DaemonCommand, DaemonSessionLifecycle, Duration, Map,
- Mutex, ParentIdentity, Path, PromptInput, Result, RlmChildTerminalNotice,
- SupervisorChildSessionsInner, Value, CREATE_TIMEOUT_MS, IDLE_WAIT_GRACE_MS, KILL_TIMEOUT_MS,
- NOTICE_DELIVERY_TIMEOUT_MS, PROMPT_TIMEOUT_MS, RUNTIME_METADATA_PROMPT_MAX, STATE_TIMEOUT_MS,
- WATCH_MAX_UNREACHABLE_POLLS, WATCH_POLL_INTERVAL_MS, WATCH_SETTLE_GRACE_MS,
- WATCH_WAIT_SLICE_MS,
+ anyhow, compact_rlm_text, create_rlm_child_failure_message, create_rlm_child_terminal_notice,
+ json, now_ms, Arc, ChildCloseReason, ChildRecord, Context, CustomMessage, DaemonCommand,
+ DaemonSessionLifecycle, Duration, Map, Mutex, ParentIdentity, Path, PromptInput, Result,
+ RlmChildTerminalNotice, SupervisorChildSessionsInner, Value, CREATE_TIMEOUT_MS,
+ IDLE_WAIT_GRACE_MS, KILL_TIMEOUT_MS, NOTICE_DELIVERY_TIMEOUT_MS, PROMPT_TIMEOUT_MS,
+ RUNTIME_METADATA_PROMPT_MAX, STATE_TIMEOUT_MS, WATCH_MAX_UNREACHABLE_POLLS,
+ WATCH_POLL_INTERVAL_MS, WATCH_SETTLE_GRACE_MS, WATCH_WAIT_SLICE_MS,
};
/// Parsed ids of one created child session.
@@ -44,7 +44,7 @@ impl CreatedChild {
/// The plain text of a custom row's content (the notice turn's model
/// prompt); `None` for non-text content shapes.
-fn custom_message_text(message: &pa_types::session::CustomMessage) -> Option<String> {
+fn custom_message_text(message: &CustomMessage) -> Option<String> {
match &message.content {
pa_types::ai::UserContent::Text(text) => Some(text.clone()),
pa_types::ai::UserContent::Blocks(_) => None,
@@ -451,20 +451,30 @@ impl SupervisorChildSessionsInner {
} else {
unreachable_polls += 1;
if unreachable_polls >= WATCH_MAX_UNREACHABLE_POLLS {
- {
- let mut state = record.lock().await;
- if !should_mark_unreachable_error(&state) {
- return;
- }
- state.settled_status = Some("error");
- state.error = Some("Child worker unreachable".to_string());
- }
// A dead child keeps whatever rows its file already
// holds; capture them before the terminal notice.
self.emit_child_usage(record).await;
- self.deliver_settle_notice(record).await;
- self.fire_settle_hook(record).await;
- return;
+ if self
+ .settle_failed(
+ record,
+ "Child worker unreachable".to_string(),
+ FailedArm::Unreachable,
+ )
+ .await
+ {
+ return;
+ }
+ // The claim lost to a verdict a reader's refresh landed
+ // after this pass's settle read (the worker answered
+ // idle, so the dead streak is broken): the settled
+ // branch owes that verdict its notice and funnel. With
+ // no verdict left, a cancel or a parent close owns the
+ // tail.
+ if record.lock().await.settled_status.is_none() {
+ return;
+ }
+ unreachable_polls = 0;
+ continue;
}
}
tokio::time::sleep(Duration::from_millis(WATCH_POLL_INTERVAL_MS)).await;
@@ -476,19 +486,72 @@ impl SupervisorChildSessionsInner {
/// notice claim collapses the races between the watcher, a natural
/// settle during `delete_subagent`, and the delete path itself.
async fn deliver_settle_notice(&self, record: &Arc<Mutex<ChildRecord>>) {
- let notice = {
+ let message = {
let mut record = record.lock().await;
if record.notice_delivered || record.replied_since_task {
return;
}
record.notice_delivered = true;
- RlmChildTerminalNotice::CompletedWithoutReply {
+ let notice = RlmChildTerminalNotice::CompletedWithoutReply {
child_id: record.rlm_child_id.clone(),
session_name: record.session_name.clone(),
last_assistant_text_preview: record.answer_preview.clone(),
+ };
+ create_rlm_child_terminal_notice(&notice, now_ms())
+ };
+ self.deliver_terminal_notice(message).await;
+ }
+
+ /// Settle one child as failed (TS's thrown-run arm): the record
+ /// settles `error` with its failure text and the parent receives the
+ /// `rlm_child_failure` row — whether or not the child replied — then
+ /// the settle funnel fires like TS's `finally` resume. Exactly-once:
+ /// the claim is split by arm. The unreachable arm keeps
+ /// `should_mark_unreachable_error` — an already-settled child keeps
+ /// its verdict (#3104's passivation semantics: a verdict a reader's
+ /// refresh lands inside the give-up keeps standing, and the claim
+ /// reports the loss so the watcher runs that verdict's settle tail).
+ /// The prompt arm bails only on a REAL terminal verdict:
+ /// a task turn that provably never started cannot have genuinely
+ /// completed, so a `done` visible inside the prompt-failure window is
+ /// `refresh_record`'s admission-window misread of an alive-but-idle
+ /// worker, not a settle verdict — the failure row must still land
+ /// (and, since #3171, the run must still mark settled, or the
+ /// quiescence barrier parks forever). Returns whether this call
+ /// claimed the settle.
+ pub(super) async fn settle_failed(
+ &self,
+ record: &Arc<Mutex<ChildRecord>>,
+ error: String,
+ arm: FailedArm,
+ ) -> bool {
+ let message = {
+ let mut record = record.lock().await;
+ let keep_verdict = match arm {
+ FailedArm::Unreachable => !should_mark_unreachable_error(&record),
+ FailedArm::Prompt => {
+ record.closed_by_parent
+ || record.notice_delivered
+ || matches!(record.settled_status, Some("error" | "cancelled"))
+ }
+ };
+ if keep_verdict {
+ return false;
}
+ record.settled_status = Some("error");
+ record.notice_delivered = true;
+ let message = create_rlm_child_failure_message(
+ &record.rlm_child_id,
+ &record.session_name,
+ &error,
+ now_ms(),
+ );
+ record.error = Some(error);
+ message
};
- self.deliver_terminal_notice(&notice).await;
+ self.deliver_terminal_notice(message).await;
+ self.fire_settle_hook(record).await;
+ true
}
/// Deliver one terminal notice into the parent session: the notice rides
@@ -501,8 +564,7 @@ impl SupervisorChildSessionsInner {
/// reserved-kind row exclusively with a live mint, and answers
/// anything a caller sends — with or without a guessed nonce —
/// loudly instead.
- pub(super) async fn deliver_terminal_notice(&self, notice: &RlmChildTerminalNotice) {
- let message = create_rlm_child_terminal_notice(notice, now_ms());
+ pub(super) async fn deliver_terminal_notice(&self, message: CustomMessage) {
let Some(content) = custom_message_text(&message) else {
eprintln!("pa-daemon: RLM child notice carried no text content");
return;
@@ -538,12 +600,23 @@ impl SupervisorChildSessionsInner {
}
}
-/// Whether the unreachable-poller may re-score a child as an error: a
-/// parent-closed child, a noticed child, or an ALREADY-SETTLED child
-/// keeps its POSITIVE verdict — its worker leaving afterward (the idle
-/// passivation's graceful stop, a crash after settle, or a give-up) is
-/// residency churn, not a settle verdict change. The settle state is
-/// durable and the passive roster row stays the representation.
+/// Which failure arm is claiming the settle: the two arms bail on
+/// different evidence (see `settle_failed`).
+pub(super) enum FailedArm {
+ /// The task-prompt admission failed and its retry failed.
+ Prompt,
+ /// The unreachable-poller's give-up (`WATCH_MAX_UNREACHABLE_POLLS`
+ /// consecutive dead polls).
+ Unreachable,
+}
+
+/// Whether a child may still settle as an error (the unreachable
+/// give-up's claim): a parent-closed child, a noticed child, or an
+/// ALREADY-SETTLED child keeps its POSITIVE verdict — its worker leaving
+/// afterward (the idle passivation's graceful stop, a crash after
+/// settle, or a give-up) is residency churn, not a settle verdict
+/// change. The settle state is durable and the passive roster row
+/// stays the representation.
pub(super) fn should_mark_unreachable_error(state: &ChildRecord) -> bool {
!(state.closed_by_parent || state.notice_delivered || state.settled_status.is_some())
}
diff --git a/crates/pa-daemon/src/rlm_children/watch_tests.rs b/crates/pa-daemon/src/rlm_children/watch_tests.rs
index cc637d375f..cfa56936e9 100644
--- a/crates/pa-daemon/src/rlm_children/watch_tests.rs
+++ b/crates/pa-daemon/src/rlm_children/watch_tests.rs
@@ -1,8 +1,9 @@
use super::*;
use crate::protocol::{response_failure, response_success};
use pa_types::platform::transport::bind_transport;
+use pa_types::session::AgentMessage;
use serde_json::{json, Value};
-use std::sync::atomic::{AtomicBool, Ordering};
+use std::sync::atomic::{AtomicBool, AtomicU32, Ordering};
use std::sync::Arc;
use tokio::io::{AsyncBufReadExt, AsyncWriteExt, BufReader};
use tokio::sync::mpsc;
@@ -17,27 +18,58 @@ enum FakeKill {
Failure,
}
+/// How the fake answers the child link: healthy, a failing `prompt`,
+/// or a failing `get_state` (unreachable).
+#[derive(Clone, Copy)]
+enum FakeChild {
+ Healthy,
+ PromptFails,
+ Unreachable,
+ /// The worker leaves right after the settle answer is captured:
+ /// every later child read fails.
+ LeavesAfterSettle,
+ /// Unreachable; the watcher's give-up poll parks until the test
+ /// lands a reader's verdict.
+ UnreachableUntilVerdict,
+}
+
/// A scripted JSONL supervisor for the watcher tests: creates one child
/// session, reports it idle with a final answer, and captures the
/// `follow_up` commands routed to the parent (the terminal-notice
/// deliveries). `idle_delay_ms` paces `wait_for_idle` so a test can act
-/// while the child is still "running". `worker_leaves_after_settle`
-/// fails every child read after the settle answer is captured.
+/// while the child is still "running". `child` scripts the child link
+/// (`FakeChild`).
async fn spawn_fake_supervisor(
socket: std::path::PathBuf,
follow_up_tx: mpsc::UnboundedSender<Value>,
idle_delay_ms: u64,
kill_tx: mpsc::UnboundedSender<Value>,
kill_behavior: FakeKill,
- worker_leaves_after_settle: bool,
+ child: FakeChild,
child_subagents: Arc<FakeChildSubagents>,
) {
let kill_behavior = std::sync::Arc::new(kill_behavior);
+ // Per-fake child session file: a fixed path would let a leftover file
+ // from another test (or run) carry the prompt text, flip the
+ // prompt-retry arbitration to `landed`, and turn an expected failure
+ // row into a no-reply notice.
+ let child_session_file = std::sync::Arc::new(
+ std::env::temp_dir()
+ .join(format!(
+ "pa-rlm-watch-child-{}.jsonl",
+ uuid::Uuid::new_v4().simple()
+ ))
+ .to_string_lossy()
+ .to_string(),
+ );
let listener = bind_transport(&socket).await.unwrap();
tokio::spawn(async move {
// Shared across link connections: a left worker fails every child
// read on whichever connection carries it.
let gone = Arc::new(AtomicBool::new(false));
+ // Shared across link connections: the give-up gate counts the
+ // child's state reads on whichever connection carries them.
+ let state_reads = Arc::new(AtomicU32::new(0));
loop {
let Ok(stream) = listener.accept().await else {
return;
@@ -46,6 +78,8 @@ async fn spawn_fake_supervisor(
let kill_tx = kill_tx.clone();
let kill_behavior = std::sync::Arc::clone(&kill_behavior);
let gone = Arc::clone(&gone);
+ let state_reads = Arc::clone(&state_reads);
+ let child_session_file = std::sync::Arc::clone(&child_session_file);
let child_subagents = Arc::clone(&child_subagents);
tokio::spawn(async move {
let (reader, mut writer) = stream.split();
@@ -66,6 +100,34 @@ async fn spawn_fake_supervisor(
let command = value["command"].clone();
let command_type: &str = command["type"].as_str().unwrap_or_default();
let response = match command_type {
+ _ if matches!(
+ (child, command_type),
+ (FakeChild::PromptFails, "prompt")
+ | (
+ FakeChild::Unreachable | FakeChild::UnreachableUntilVerdict,
+ "get_state",
+ )
+ ) =>
+ {
+ // Two get_state reads per watcher pass for the
+ // parked variant (the refresh read, then the
+ // liveness poll): the last one is the give-up
+ // poll. Only that variant counts, so a plain
+ // unreachable child keeps its instant refusal.
+ if matches!(child, FakeChild::UnreachableUntilVerdict)
+ && state_reads.fetch_add(1, Ordering::SeqCst) + 1
+ == 2 * WATCH_MAX_UNREACHABLE_POLLS
+ {
+ GIVE_UP_POLL.notify_one();
+ VERDICT_LANDED.notified().await;
+ }
+ response_failure(
+ Some(&id),
+ command_type,
+ "refused by the fake supervisor",
+ None,
+ )
+ }
"get_state" | "wait_for_idle" if gone.load(Ordering::SeqCst) => {
response_failure(
Some(&id),
@@ -80,7 +142,7 @@ async fn spawn_fake_supervisor(
Some(json!({
"activeSessionId": "child-live",
"sessionId": "child-file",
- "sessionFile": "/tmp/child.jsonl",
+ "sessionFile": *child_session_file,
"sessionName": "f20-worker",
})),
),
@@ -112,7 +174,7 @@ async fn spawn_fake_supervisor(
"get_last_assistant_text" => {
// The settle capture: with the knob set, the
// worker leaves right after its final answer.
- if worker_leaves_after_settle {
+ if matches!(child, FakeChild::LeavesAfterSettle) {
gone.store(true, Ordering::SeqCst);
}
response_success(
@@ -166,13 +228,13 @@ async fn sessions_with_fake_supervisor(
follow_up_tx: mpsc::UnboundedSender<Value>,
idle_delay_ms: u64,
kill_behavior: FakeKill,
- worker_leaves_after_settle: bool,
+ child: FakeChild,
) -> (SupervisorChildSessions, mpsc::UnboundedReceiver<Value>) {
sessions_with_fake_child_subagents(
follow_up_tx,
idle_delay_ms,
kill_behavior,
- worker_leaves_after_settle,
+ child,
Arc::new(FakeChildSubagents::default()),
)
.await
@@ -193,7 +255,7 @@ async fn sessions_with_fake_child_subagents(
follow_up_tx: mpsc::UnboundedSender<Value>,
idle_delay_ms: u64,
kill_behavior: FakeKill,
- worker_leaves_after_settle: bool,
+ child: FakeChild,
child_subagents: Arc<FakeChildSubagents>,
) -> (SupervisorChildSessions, mpsc::UnboundedReceiver<Value>) {
let socket = std::env::temp_dir().join(format!(
@@ -207,7 +269,7 @@ async fn sessions_with_fake_child_subagents(
idle_delay_ms,
kill_tx,
kill_behavior,
- worker_leaves_after_settle,
+ child,
child_subagents,
)
.await;
@@ -248,7 +310,8 @@ async fn spawn_child(sessions: &SupervisorChildSessions) -> RlmSpawnHandle {
async fn roster_snapshot_does_not_wait_for_a_slow_child_worker() {
let (follow_up_tx, _follow_up_rx) = mpsc::unbounded_channel();
let (sessions, _kill_rx) =
- sessions_with_fake_supervisor(follow_up_tx, 1_000, FakeKill::Success, false).await;
+ sessions_with_fake_supervisor(follow_up_tx, 1_000, FakeKill::Success, FakeChild::Healthy)
+ .await;
sessions
.push_test_child(RlmChildIdentity {
rlm_child_id: "child-id".to_string(),
@@ -271,7 +334,7 @@ async fn roster_snapshot_does_not_wait_for_a_slow_child_worker() {
async fn a_settled_child_without_a_reply_delivers_the_terminal_notice() {
let (follow_up_tx, mut follow_up_rx) = mpsc::unbounded_channel();
let (sessions, _kill_rx) =
- sessions_with_fake_supervisor(follow_up_tx, 0, FakeKill::Success, false).await;
+ sessions_with_fake_supervisor(follow_up_tx, 0, FakeKill::Success, FakeChild::Healthy).await;
let handle = spawn_child(&sessions).await;
// The worker releases the detached prompt at its turn boundary.
sessions.notify_turn_done();
@@ -307,8 +370,13 @@ async fn a_settled_child_without_a_reply_delivers_the_terminal_notice() {
#[tokio::test]
async fn a_worker_leaving_inside_the_settle_grace_keeps_the_verdict() {
let (follow_up_tx, mut follow_up_rx) = mpsc::unbounded_channel();
- let (sessions, _kill_rx) =
- sessions_with_fake_supervisor(follow_up_tx, 0, FakeKill::Success, true).await;
+ let (sessions, _kill_rx) = sessions_with_fake_supervisor(
+ follow_up_tx,
+ 0,
+ FakeKill::Success,
+ FakeChild::LeavesAfterSettle,
+ )
+ .await;
let settled = sessions.settle_notified();
spawn_child(&sessions).await;
sessions.notify_turn_done();
@@ -327,6 +395,223 @@ async fn a_worker_leaving_inside_the_settle_grace_keeps_the_verdict() {
.is_some_and(|content| content.contains("the child final answer")));
}
+/// The `follow_up` carries exactly the TS failure row (`createRlmChildFailureMessage`).
+fn assert_failure_row(follow_up: &Value, child_id: &str, error: &str) {
+ let custom = &follow_up["customMessage"];
+ let timestamp = custom["timestamp"].as_u64().expect("failure row timestamp");
+ let expected = create_rlm_child_failure_message(child_id, "f20-worker", error, timestamp);
+ assert_eq!(
+ *custom,
+ serde_json::to_value(AgentMessage::Custom(expected)).unwrap()
+ );
+}
+
+/// A child whose task prompt cannot be routed settles `error`, delivers
+/// the TS failure row and releases the owed continuation.
+#[tokio::test]
+async fn a_child_whose_task_prompt_cannot_be_routed_delivers_the_failure_notice() {
+ let (follow_up_tx, mut follow_up_rx) = mpsc::unbounded_channel();
+ let (sessions, _kill_rx) =
+ sessions_with_fake_supervisor(follow_up_tx, 0, FakeKill::Success, FakeChild::PromptFails)
+ .await;
+ let (hook_tx, mut hook_rx) = mpsc::unbounded_channel();
+ sessions.set_settle_hook(Arc::new(move || {
+ let _ = hook_tx.send(());
+ }));
+ let handle = spawn_child(&sessions).await;
+ // The worker releases the detached prompt at its turn boundary.
+ sessions.notify_turn_done();
+
+ let follow_up = tokio::time::timeout(Duration::from_secs(10), follow_up_rx.recv())
+ .await
+ .expect("the failed child must deliver its failure notice")
+ .expect("the follow_up channel stays open");
+ assert_failure_row(
+ &follow_up,
+ &handle.rlm_child_id,
+ "prompt RLM child session child-live: refused by the fake supervisor",
+ );
+ let entries = sessions.list_subagents().await.expect("child roster");
+ assert_eq!(entries[0].status, "error");
+ tokio::time::timeout(Duration::from_secs(10), hook_rx.recv())
+ .await
+ .expect("the failed child releases the owed continuation")
+ .expect("hook channel open");
+ // Exactly one failure row lands: the claim admits one writer.
+ let extra = tokio::time::timeout(Duration::from_millis(300), follow_up_rx.recv()).await;
+ assert!(extra.is_err(), "no second notice may arrive");
+}
+
+/// An unreachable child delivers the failure row, not the no-reply notice.
+/// A short tick keeps the paused clock's auto-advance from firing link
+/// deadlines while the real-socket round trips are in flight.
+#[tokio::test(start_paused = true)]
+async fn an_unreachable_child_delivers_the_failure_notice_instead_of_no_reply() {
+ tokio::spawn(async {
+ let mut tick = tokio::time::interval(Duration::from_secs(1));
+ loop {
+ tick.tick().await;
+ }
+ });
+ let (follow_up_tx, mut follow_up_rx) = mpsc::unbounded_channel();
+ let (sessions, _kill_rx) =
+ sessions_with_fake_supervisor(follow_up_tx, 0, FakeKill::Success, FakeChild::Unreachable)
+ .await;
+ let handle = spawn_child(&sessions).await;
+ sessions.notify_turn_done();
+
+ let follow_up = tokio::time::timeout(Duration::from_secs(3_600), follow_up_rx.recv())
+ .await
+ .expect("the unreachable child must deliver its failure notice")
+ .expect("follow_up channel open");
+ assert_failure_row(&follow_up, &handle.rlm_child_id, "Child worker unreachable");
+ // Exactly one failure row lands: the give-up claims once.
+ let extra = tokio::time::timeout(Duration::from_millis(300), follow_up_rx.recv()).await;
+ assert!(extra.is_err(), "no second notice may arrive");
+}
+
+/// Hand-off into the unreachable give-up's poll: the fake parks the
+/// give-up poll, the test lands the reader's verdict, the fake releases
+/// the poll. Only the lost-claim test uses these.
+static GIVE_UP_POLL: tokio::sync::Notify = tokio::sync::Notify::const_new();
+static VERDICT_LANDED: tokio::sync::Notify = tokio::sync::Notify::const_new();
+
+/// A reader's refresh can settle the child between the watcher's settle
+/// read and its unreachable give-up claim: the claim keeps that verdict,
+/// and the watcher still runs its settle tail.
+#[tokio::test(start_paused = true)]
+async fn a_verdict_landing_inside_the_unreachable_give_up_still_runs_the_settle_tail() {
+ tokio::spawn(async {
+ let mut tick = tokio::time::interval(Duration::from_secs(1));
+ loop {
+ tick.tick().await;
+ }
+ });
+ let (follow_up_tx, mut follow_up_rx) = mpsc::unbounded_channel();
+ let (sessions, _kill_rx) = sessions_with_fake_supervisor(
+ follow_up_tx,
+ 0,
+ FakeKill::Success,
+ FakeChild::UnreachableUntilVerdict,
+ )
+ .await;
+ let settled = sessions.settle_notified();
+ spawn_child(&sessions).await;
+ sessions.notify_turn_done();
+
+ GIVE_UP_POLL.notified().await;
+ {
+ // What a `collect`'s `refresh_record` writes when the worker
+ // answers idle.
+ let record = Arc::clone(&sessions.inner.children.lock().await[0]);
+ let mut record = record.lock().await;
+ record.settled_status = Some("done");
+ }
+ VERDICT_LANDED.notify_one();
+
+ tokio::time::timeout(Duration::from_secs(3_600), settled)
+ .await
+ .expect("the settle funnel fires for the reader's verdict");
+ assert!(!sessions.any_running().await, "the run must mark settled");
+ let notice = follow_up_rx
+ .try_recv()
+ .expect("the verdict's no-reply notice is still owed");
+ assert_eq!(
+ notice["customMessage"]["customType"],
+ "rlm_child_terminal_notice"
+ );
+}
+
+/// The review's C1 interleaving: a `collect` that lands inside the
+/// prompt-failure window (after the detached task flips
+/// `prompt_admitted`, before the retry verdict) reads the alive-but-idle
+/// worker as `done` — `refresh_record`'s admission-window misread, not a
+/// settle verdict. The prompt arm's failure claim must ignore it: the
+/// failure row still lands, the roster reports `error`, the hook fires,
+/// and the run marks settled (post-#3171 an unsettled failed run parks
+/// `waitForRlmQuiescence` forever).
+#[tokio::test]
+async fn a_collect_inside_the_prompt_failure_window_does_not_swallow_the_failure_row() {
+ let (follow_up_tx, mut follow_up_rx) = mpsc::unbounded_channel();
+ let (sessions, _kill_rx) =
+ sessions_with_fake_supervisor(follow_up_tx, 0, FakeKill::Success, FakeChild::PromptFails)
+ .await;
+ let (hook_tx, mut hook_rx) = mpsc::unbounded_channel();
+ sessions.set_settle_hook(Arc::new(move || {
+ let _ = hook_tx.send(());
+ }));
+ let handle = spawn_child(&sessions).await;
+ // The window's entry condition, set deterministically: the detached
+ // prompt task flips `prompt_admitted` BEFORE its first
+ // `prompt_child` (host.rs), and the collect below races that task in
+ // production. With the flag set and the worker alive-but-idle, the
+ // real collect path scores the misread.
+ sessions.inner.children.lock().await[0]
+ .lock()
+ .await
+ .prompt_admitted = true;
+ let misread = sessions
+ .collect(vec![handle.rlm_child_id.clone()], 0)
+ .await
+ .expect("collect inside the window");
+ assert_eq!(
+ misread[0].status, "done",
+ "the premise: the collect misreads the pre-prompt idle worker as done"
+ );
+
+ // The task prompt now runs and fails (prompt + retry) inside the
+ // window the collect already scored.
+ sessions.notify_turn_done();
+
+ let follow_up = tokio::time::timeout(Duration::from_secs(10), follow_up_rx.recv())
+ .await
+ .expect("the failure row must land despite the collect's misread")
+ .expect("follow_up channel open");
+ assert_failure_row(
+ &follow_up,
+ &handle.rlm_child_id,
+ "prompt RLM child session child-live: refused by the fake supervisor",
+ );
+ let entries = sessions.list_subagents().await.expect("child roster");
+ assert_eq!(entries[0].status, "error");
+ tokio::time::timeout(Duration::from_secs(10), hook_rx.recv())
+ .await
+ .expect("the settle funnel fires despite the misread")
+ .expect("hook channel open");
+ assert!(
+ !sessions.any_running().await,
+ "the failed run must mark settled: an unsettled run parks the quiescence barrier"
+ );
+ let extra = tokio::time::timeout(Duration::from_millis(300), follow_up_rx.recv()).await;
+ assert!(extra.is_err(), "no second notice may arrive");
+}
+
+/// A child that already replied still delivers the failure row when its
+/// task prompt fails: TS sends the catch-arm row regardless of reply
+/// count (`agent-session.ts:13026-13045`).
+#[tokio::test]
+async fn a_replied_child_whose_prompt_fails_still_delivers_the_failure_row() {
+ let (follow_up_tx, mut follow_up_rx) = mpsc::unbounded_channel();
+ let (sessions, _kill_rx) =
+ sessions_with_fake_supervisor(follow_up_tx, 0, FakeKill::Success, FakeChild::PromptFails)
+ .await;
+ let handle = spawn_child(&sessions).await;
+ sessions.mark_replied("child-live").await;
+ sessions.notify_turn_done();
+
+ let follow_up = tokio::time::timeout(Duration::from_secs(10), follow_up_rx.recv())
+ .await
+ .expect("the replied child must still deliver the failure row")
+ .expect("follow_up channel open");
+ assert_failure_row(
+ &follow_up,
+ &handle.rlm_child_id,
+ "prompt RLM child session child-live: refused by the fake supervisor",
+ );
+ let extra = tokio::time::timeout(Duration::from_millis(300), follow_up_rx.recv()).await;
+ assert!(extra.is_err(), "no second notice may arrive");
+}
+
/// A parent counts as running while any descendant runs: the child's own
/// turn is done, but while it reports a running grandchild the parent's
/// row stays `running` and the registry keeps the parent's summary busy;
@@ -343,7 +628,7 @@ async fn a_child_with_a_running_grandchild_keeps_the_parent_running() {
follow_up_tx,
0,
FakeKill::Success,
- false,
+ FakeChild::Healthy,
Arc::clone(&grandchild),
)
.await;
@@ -385,7 +670,7 @@ async fn collect_with_a_timeout_waits_for_a_running_grandchild() {
follow_up_tx,
0,
FakeKill::Success,
- false,
+ FakeChild::Healthy,
Arc::clone(&grandchild),
)
.await;
@@ -443,7 +728,8 @@ async fn close_children_stops_the_child_and_clears_the_roster() {
// A long idle keeps the child mid-run while the close fires, so the
// settle watcher is parked instead of raced.
let (sessions, mut kill_rx) =
- sessions_with_fake_supervisor(follow_up_tx, 10_000, FakeKill::Success, false).await;
+ sessions_with_fake_supervisor(follow_up_tx, 10_000, FakeKill::Success, FakeChild::Healthy)
+ .await;
spawn_child(&sessions).await;
sessions.notify_turn_done();
one_running_child(&sessions).await;
@@ -482,8 +768,13 @@ async fn close_children_stops_the_child_and_clears_the_roster() {
#[tokio::test]
async fn close_children_treats_an_already_gone_child_as_a_no_op() {
let (follow_up_tx, _follow_up_rx) = mpsc::unbounded_channel();
- let (sessions, _kill_rx) =
- sessions_with_fake_supervisor(follow_up_tx, 10_000, FakeKill::UnknownSession, false).await;
+ let (sessions, _kill_rx) = sessions_with_fake_supervisor(
+ follow_up_tx,
+ 10_000,
+ FakeKill::UnknownSession,
+ FakeChild::Healthy,
+ )
+ .await;
spawn_child(&sessions).await;
sessions.notify_turn_done();
one_running_child(&sessions).await;
@@ -507,7 +798,8 @@ async fn close_children_treats_an_already_gone_child_as_a_no_op() {
async fn close_children_keeps_a_failed_child_tracked() {
let (follow_up_tx, _follow_up_rx) = mpsc::unbounded_channel();
let (sessions, _kill_rx) =
- sessions_with_fake_supervisor(follow_up_tx, 10_000, FakeKill::Failure, false).await;
+ sessions_with_fake_supervisor(follow_up_tx, 10_000, FakeKill::Failure, FakeChild::Healthy)
+ .await;
spawn_child(&sessions).await;
sessions.notify_turn_done();
one_running_child(&sessions).await;
@@ -533,7 +825,8 @@ async fn a_replied_child_gets_no_terminal_notice() {
// A slow idle wait keeps the child "running" while the test marks
// the reply.
let (sessions, _kill_rx) =
- sessions_with_fake_supervisor(follow_up_tx, 250, FakeKill::Success, false).await;
+ sessions_with_fake_supervisor(follow_up_tx, 250, FakeKill::Success, FakeChild::Healthy)
+ .await;
let handle = spawn_child(&sessions).await;
assert!(!handle.rlm_child_id.is_empty());
sessions.mark_replied("child-live").await;
@@ -553,7 +846,7 @@ async fn a_replied_child_gets_no_terminal_notice() {
async fn collect_answers_a_just_deleted_target_with_the_cancelled_envelope() {
let (follow_up_tx, _follow_up_rx) = mpsc::unbounded_channel();
let (sessions, _kill_rx) =
- sessions_with_fake_supervisor(follow_up_tx, 0, FakeKill::Success, false).await;
+ sessions_with_fake_supervisor(follow_up_tx, 0, FakeKill::Success, FakeChild::Healthy).await;
let handle = spawn_child(&sessions).await;
sessions.notify_turn_done();
// The child settles with its final answer before the delete.
@@ -626,7 +919,7 @@ async fn collect_answers_a_just_deleted_target_with_the_cancelled_envelope() {
async fn collect_answers_the_cancelled_envelope_after_an_inactive_delete() {
let (follow_up_tx, _follow_up_rx) = mpsc::unbounded_channel();
let (sessions, _kill_rx) =
- sessions_with_fake_supervisor(follow_up_tx, 0, FakeKill::Success, false).await;
+ sessions_with_fake_supervisor(follow_up_tx, 0, FakeKill::Success, FakeChild::Healthy).await;
let handle = spawn_child(&sessions).await;
sessions.notify_turn_done();
// The inactive delete requires a settled child (a running child