Skip to content

A self-improving RLM agent for coding workflows and long-running autonomous tasks.

Rust

rlm: settle a child whose final model call failed as error, not done (#3754)

ktfe62c04bc1d7
diff --git a/crates/pa-daemon/src/rlm_children/lifecycle.rs b/crates/pa-daemon/src/rlm_children/lifecycle.rs
index 62d4047557..9f28538917 100644
--- a/crates/pa-daemon/src/rlm_children/lifecycle.rs
+++ b/crates/pa-daemon/src/rlm_children/lifecycle.rs
@@ -417,11 +417,17 @@ impl SupervisorChildSessionsInner {
return;
}
}
- let (active_session_id, direct_observation) = {
+ let (active_session_id, direct_observation, unsettled_session_file) = {
let record = record.lock().await;
(
record.active_session_id.clone(),
Arc::clone(&record.direct_observation),
+ record
+ .settled_status
+ .is_none()
+ .then(|| record.session_file.clone())
+ .flatten()
+ .filter(|path| !path.is_empty()),
)
};
let state = {
@@ -454,6 +460,22 @@ impl SupervisorChildSessionsInner {
.await
.ok()
.flatten();
+ // How the run's final turn ended (#3366): a run whose last model
+ // call failed (a provider error, after its retries if it had any)
+ // settles `error` with that text instead of a `done` with no
+ // answer. The worker persists every row before it reads idle,
+ // so its durable file holds the final turn; the read stays off
+ // the async workers.
+ let final_turn_error = match unsettled_session_file {
+ Some(path) => tokio::task::spawn_blocking(move || final_turn_error_from_file(&path))
+ .await
+ .ok()
+ .flatten(),
+ None => None,
+ }
+ // A provider error can carry a whole raw response body (a gateway's
+ // HTML error page); the text reaches the parent's context.
+ .map(|error| compact_rlm_text(&error));
let answer = raw_answer.as_deref().map(compact_rlm_text);
let answer_text = raw_answer
.as_deref()
@@ -469,7 +491,13 @@ impl SupervisorChildSessionsInner {
// no answer yet, so a settled preview is still never overwritten
// with a later miss.
if record.settled_status.is_none() {
- record.settled_status = Some("done");
+ match final_turn_error {
+ Some(error) => {
+ record.settled_status = Some("error");
+ record.error = Some(error);
+ }
+ None => record.settled_status = Some("done"),
+ }
}
if (!record.answer_captured || record.answer_preview.is_none()) && answer.is_some() {
record.answer_preview = answer;
@@ -479,7 +507,8 @@ impl SupervisorChildSessionsInner {
}
/// Watch one admitted child until its run settles, then deliver the parent's terminal notice
- /// when the child never replied; the watcher stops as soon as the record is removed.
+ /// (the failure row or the no-reply notice) when the child has not replied since its task;
+ /// the watcher stops as soon as the record is removed.
pub(super) async fn watch_child_settle(&self, record: &Arc<Mutex<ChildRecord>>) {
let mut unreachable_polls: u32 = 0;
loop {
@@ -528,7 +557,8 @@ impl SupervisorChildSessionsInner {
// verdict standing, as in `refresh_record`.
//
// The re-check wipes only a WATCHER-set settle (the
- // premature `done` of the admission race): a
+ // premature `done` of the admission race, or a refresh's
+ // final-turn `error`, which claimed no notice yet): a
// registry-set `cancelled` (a user abort or a parent kill
// in flight, `cancel_child_run`/`close_children_inner`)
// stands — wiping it would let this watcher refresh the
@@ -540,8 +570,14 @@ impl SupervisorChildSessionsInner {
tokio::time::sleep(Duration::from_millis(WATCH_SETTLE_GRACE_MS)).await;
if matches!(self.child_busy(&active_session_id).await, Ok(true)) {
let mut record = record.lock().await;
- if matches!(record.settled_status, Some("done")) {
+ let refresh_verdict = match record.settled_status {
+ Some("done") => true,
+ Some("error") => !record.notice_delivered,
+ _ => false,
+ };
+ if refresh_verdict {
record.settled_status = None;
+ record.error = None;
}
drop(record);
continue;
@@ -699,10 +735,13 @@ impl SupervisorChildSessionsInner {
recorder.record_child_cancelled(&child_session_id, reason);
}
- /// Deliver the no-reply terminal notice for a settled child that never
- /// sent an agent message to the parent. Exactly-once: the record's
- /// notice claim collapses the races between the watcher, a natural
- /// settle during `delete_subagent`, and the delete path itself.
+ /// Deliver the settled child's terminal notice when it has not replied
+ /// since its task: the failure row for a run whose final turn ended in
+ /// an error, else the no-reply notice. A child that already replied
+ /// gets neither (its answer is the parent's notice; the roster and
+ /// collect still report the error). Exactly-once: the record's notice claim collapses the races between
+ /// the watcher, a natural settle during `delete_subagent`, and the
+ /// delete path itself.
async fn deliver_settle_notice(&self, record: &Arc<Mutex<ChildRecord>>) {
let message = {
let mut record = record.lock().await;
@@ -710,12 +749,25 @@ impl SupervisorChildSessionsInner {
return;
}
record.notice_delivered = true;
- let notice = RlmChildTerminalNotice::CompletedWithoutReply {
- child_id: record.rlm_child_id.clone(),
- session_name: record.session_name.clone(),
- last_assistant_text_preview: record.answer_preview.clone(),
+ let failure = match (record.settled_status, &record.error) {
+ (Some("error"), Some(error)) => Some(error.clone()),
+ _ => None,
};
- create_rlm_child_terminal_notice(&notice, now_ms())
+ if let Some(error) = failure {
+ create_rlm_child_failure_message(
+ &record.rlm_child_id,
+ &record.session_name,
+ &error,
+ now_ms(),
+ )
+ } else {
+ let notice = RlmChildTerminalNotice::CompletedWithoutReply {
+ child_id: record.rlm_child_id.clone(),
+ session_name: record.session_name.clone(),
+ last_assistant_text_preview: record.answer_preview.clone(),
+ };
+ create_rlm_child_terminal_notice(&notice, now_ms())
+ }
};
self.deliver_terminal_notice(message).await;
}
@@ -898,6 +950,83 @@ pub(super) fn last_assistant_text_from_file(session_file: &str) -> Option<String
None
}
+/// The error a child's final turn ended in, read from its durable session
+/// file (#3366), scanning back from the newest row. A failed
+/// `provider_retry_outcome` row (the retry episode's one durable outcome)
+/// answers its text, and an assistant row that stopped on `error` answers
+/// its error message. The scan answers `None` at the first other row that
+/// bounds the final turn: any other message row (a successful or aborted
+/// assistant reply, a tool result, a user prompt) or an `agent_message`
+/// row (a task delivered after an older failure). A missing file answers
+/// `None`. Non-message entries and other custom rows (notices, a recovered
+/// retry's outcome) are skipped.
+pub(super) fn final_turn_error_from_file(session_file: &str) -> Option<String> {
+ use pa_types::session::{AgentMessage, FileEntry};
+ let content = std::fs::read_to_string(session_file).ok()?;
+ for line in content.lines().rev() {
+ let Ok(entry) = serde_json::from_str::<FileEntry>(line.trim()) else {
+ continue;
+ };
+ match entry {
+ FileEntry::CustomMessage { payload, .. } => {
+ if payload.custom_type
+ == pa_core::session_engine::agent_messaging::AGENT_MESSAGE_CUSTOM_TYPE
+ {
+ return None;
+ }
+ if payload.custom_type
+ != pa_core::session_engine::messages::PROVIDER_RETRY_OUTCOME_CUSTOM_TYPE
+ {
+ continue;
+ }
+ let failed = payload
+ .details
+ .as_ref()
+ .and_then(|details| details.get("success"))
+ .and_then(Value::as_bool)
+ == Some(false);
+ if !failed {
+ continue;
+ }
+ // The outcome row always carries the exhausted-retry text.
+ if let pa_types::ai::UserContent::Text(text) = payload.content {
+ return Some(text);
+ }
+ }
+ FileEntry::Message {
+ message: AgentMessage::Assistant(assistant),
+ ..
+ } => {
+ if assistant.stop_reason != pa_types::ai::StopReason::Error {
+ return None;
+ }
+ let error = assistant
+ .error_message
+ .filter(|message| !message.trim().is_empty())
+ .unwrap_or_else(|| "The final model call failed".to_string());
+ return Some(format!("Error: {error}"));
+ }
+ FileEntry::Message { .. } => return None,
+ FileEntry::Header { .. }
+ | FileEntry::ToolStarted { .. }
+ | FileEntry::ToolProgress { .. }
+ | FileEntry::ThinkingLevelChange { .. }
+ | FileEntry::ServiceTierChange { .. }
+ | FileEntry::ModelChange { .. }
+ | FileEntry::Compaction { .. }
+ | FileEntry::BranchSummary { .. }
+ | FileEntry::Custom { .. }
+ | FileEntry::ChildUsageAttributed { .. }
+ | FileEntry::Label { .. }
+ | FileEntry::SessionInfo { .. }
+ | FileEntry::SessionState { .. }
+ | FileEntry::GitState { .. }
+ | FileEntry::Unknown { .. } => {}
+ }
+ }
+ None
+}
+
/// The busy flags of one child `get_state` summary (streaming, its own
/// descendants, or queued work in flight).
fn child_busy_flags(state: &Value) -> bool {
@@ -913,11 +1042,21 @@ fn child_busy_flags(state: &Value) -> bool {
}
#[cfg(test)]
-mod exit_capture_tests {
+pub(super) mod exit_capture_tests {
use super::*;
/// One serialized assistant row with the given text.
fn assistant_row(text: &str) -> String {
+ assistant_row_ending(text, pa_types::ai::StopReason::Stop, None)
+ }
+
+ /// One serialized assistant row with the given text, stop reason and
+ /// error message (a failed model call's row).
+ pub(crate) fn assistant_row_ending(
+ text: &str,
+ stop_reason: pa_types::ai::StopReason,
+ error_message: Option<&str>,
+ ) -> String {
let message = pa_types::session::AgentMessage::Assistant(pa_types::ai::AssistantMessage {
content: vec![pa_types::ai::AssistantContentBlock::Text(
pa_types::ai::TextContent {
@@ -933,9 +1072,9 @@ mod exit_capture_tests {
response_id: None,
diagnostics: None,
usage: pa_types::ai::Usage::default(),
- stop_reason: pa_types::ai::StopReason::Stop,
+ stop_reason,
stop_reason_raw: None,
- error_message: None,
+ error_message: error_message.map(str::to_string),
timestamp: 2,
rest: Map::default(),
});
@@ -983,4 +1122,122 @@ mod exit_capture_tests {
let _ = std::fs::remove_file(&file);
let _ = std::fs::remove_file(&bare);
}
+
+ /// The retry episode's durable outcome row, as the worker persists it.
+ pub(crate) fn retry_outcome_row(success: bool) -> String {
+ let outcome = pa_core::session_engine::messages::create_provider_retry_outcome_message(
+ success,
+ 3,
+ "429 Provider returned error",
+ );
+ let pa_types::ai::UserContent::Text(content) = outcome.content else {
+ unreachable!("the outcome row carries text content");
+ };
+ json!({
+ "type": "custom_message",
+ "id": "row-outcome",
+ "parentId": null,
+ "customType": outcome.custom_type,
+ "content": content,
+ "display": true,
+ "details": outcome.details,
+ })
+ .to_string()
+ }
+
+ /// A stored row of `kind` (`message` or `custom_message`) with `fields`,
+ /// in the shape the session file holds.
+ fn stored_row(kind: &str, fields: &Value) -> String {
+ let mut row = json!({"type": kind, "id": "row-0", "parentId": null});
+ row.as_object_mut()
+ .expect("row object")
+ .extend(fields.as_object().expect("fields object").clone());
+ row.to_string()
+ }
+
+ /// A user prompt row.
+ fn user_row(text: &str) -> String {
+ stored_row(
+ "message",
+ &json!({"message": {"role": "user", "content": text, "timestamp": 1}}),
+ )
+ }
+
+ /// An `rlm.spawn` task row: an `agent_message` custom row.
+ fn task_row() -> String {
+ stored_row(
+ "custom_message",
+ &json!({
+ "customType": pa_core::session_engine::agent_messaging::AGENT_MESSAGE_CUSTOM_TYPE,
+ "content": "the task",
+ "display": true,
+ "details": {},
+ }),
+ )
+ }
+
+ fn final_turn_error_of(rows: &[String]) -> Option<String> {
+ let file = std::env::temp_dir().join(format!(
+ "pa-rlm-final-turn-{}.jsonl",
+ uuid::Uuid::new_v4().simple()
+ ));
+ std::fs::write(&file, format!("{}\n{}\n", task_row(), rows.join("\n")))
+ .expect("write the session file");
+ let error = final_turn_error_from_file(&file.to_string_lossy());
+ let _ = std::fs::remove_file(&file);
+ error
+ }
+
+ /// #3366: a run whose model calls all failed reads as failed, with the
+ /// provider error text; a run whose last turn answered reads clean, and
+ /// a failure older than a later turn-bounding row does not stand.
+ #[test]
+ fn the_final_turn_reader_reports_how_the_run_ended() {
+ use pa_types::ai::StopReason;
+ let failed_call =
+ assistant_row_ending("", StopReason::Error, Some("429 Provider returned error"));
+ let failed_run = [failed_call.clone(), retry_outcome_row(false)];
+ let with_later = |later: String| {
+ let mut rows = failed_run.to_vec();
+ rows.push(later);
+ final_turn_error_of(&rows)
+ };
+ assert_eq!(
+ [
+ final_turn_error_of(&failed_run),
+ final_turn_error_of(std::slice::from_ref(&failed_call)),
+ final_turn_error_of(&[assistant_row_ending("", StopReason::Aborted, None)]),
+ // An earlier failure the run recovered from does not stand.
+ final_turn_error_of(&[
+ failed_call.clone(),
+ retry_outcome_row(true),
+ assistant_row("the final say"),
+ ]),
+ final_turn_error_of(&[assistant_row("the final say"), retry_outcome_row(true)]),
+ final_turn_error_of(&[]),
+ // An older failure, then a later row that bounds the turn.
+ with_later(user_row("try again")),
+ with_later(task_row()),
+ with_later(assistant_row("the final say")),
+ ],
+ [
+ Some(
+ "\u{26a0} Error: Retry failed after 3 attempts: 429 Provider returned error"
+ .to_string()
+ ),
+ Some("Error: 429 Provider returned error".to_string()),
+ None,
+ None,
+ None,
+ None,
+ None,
+ None,
+ None,
+ ]
+ );
+ assert_eq!(
+ final_turn_error_from_file("/nonexistent/pa-rlm-final-turn.jsonl"),
+ None
+ );
+ }
}
diff --git a/crates/pa-daemon/src/rlm_children/watch_tests.rs b/crates/pa-daemon/src/rlm_children/watch_tests.rs
index caa3649d30..78fd103784 100644
--- a/crates/pa-daemon/src/rlm_children/watch_tests.rs
+++ b/crates/pa-daemon/src/rlm_children/watch_tests.rs
@@ -43,6 +43,10 @@ enum FakeChild {
/// Unreachable; the watcher's give-up poll parks until the test
/// lands a reader's verdict.
UnreachableUntilVerdict,
+ /// Every model call of the task run fails (#3366): the admitted
+ /// prompt's run leaves a failed assistant row and the retry
+ /// episode's failed outcome row in the child's session file.
+ FinalTurnFails,
}
/// A scripted JSONL supervisor for the watcher tests: creates one child
@@ -173,6 +177,21 @@ async fn spawn_fake_supervisor(
})),
),
"prompt" => {
+ if matches!(child, FakeChild::FinalTurnFails) {
+ use super::lifecycle::exit_capture_tests::{
+ assistant_row_ending, retry_outcome_row,
+ };
+ let failed_call = assistant_row_ending(
+ "",
+ pa_types::ai::StopReason::Error,
+ Some("429 Provider returned error"),
+ );
+ std::fs::write(
+ &*child_session_file,
+ format!("{failed_call}\n{}\n", retry_outcome_row(false)),
+ )
+ .unwrap();
+ }
if let Some(gate) = child_subagents
.prompt_gate
.as_ref()
@@ -999,6 +1018,67 @@ async fn a_child_whose_task_prompt_cannot_be_routed_delivers_the_failure_notice(
assert!(extra.is_err(), "no second notice may arrive");
}
+/// #3366: a child whose model calls all failed settles `error`, not
+/// `done`: the parent receives the failure row carrying the provider
+/// error (not the no-reply notice), and the roster and the collect
+/// envelope report the error with its text.
+#[tokio::test]
+async fn a_child_whose_final_turn_failed_settles_as_an_error() {
+ let (follow_up_tx, mut follow_up_rx) = mpsc::unbounded_channel();
+ let (sessions, _kill_rx) = sessions_with_fake_supervisor(
+ follow_up_tx,
+ 0,
+ FakeKill::Success,
+ FakeChild::FinalTurnFails,
+ )
+ .await;
+ let handle = spawn_child(&sessions).await;
+ sessions.notify_turn_done();
+
+ let follow_up = tokio::time::timeout(Duration::from_secs(10), follow_up_rx.recv())
+ .await
+ .expect("the failed child must deliver its failure notice")
+ .expect("the follow_up channel stays open");
+ let error = "\u{26a0} Error: Retry failed after 3 attempts: 429 Provider returned error";
+ assert_failure_row(&follow_up, &handle.rlm_child_id, error);
+ let entries = sessions.list_subagents().await.expect("child roster");
+ assert_eq!(entries[0].status, "error");
+ let results = sessions
+ .collect(vec![handle.rlm_child_id.clone()], 0)
+ .await
+ .expect("collect the failed child");
+ assert_eq!(
+ (results[0].status, results[0].error.as_deref()),
+ ("error", Some(error))
+ );
+ // Exactly one notice lands: the watcher delivers once.
+ let extra = tokio::time::timeout(Duration::from_millis(300), follow_up_rx.recv()).await;
+ assert!(extra.is_err(), "no second notice may arrive");
+}
+
+/// A child that replied before its final model call failed (the cleanup
+/// call after the reply, #3366) sends the parent no failure row on top of
+/// its answer; the roster still reports the error.
+#[tokio::test]
+async fn a_replied_child_whose_final_turn_failed_sends_no_failure_row() {
+ let (follow_up_tx, mut follow_up_rx) = mpsc::unbounded_channel();
+ let (sessions, _kill_rx) = sessions_with_fake_supervisor(
+ follow_up_tx,
+ 250,
+ FakeKill::Success,
+ FakeChild::FinalTurnFails,
+ )
+ .await;
+ spawn_child(&sessions).await;
+ sessions.mark_replied("child-live").await;
+ sessions.notify_turn_done();
+
+ let extra = tokio::time::timeout(Duration::from_secs(2), follow_up_rx.recv()).await;
+ assert!(extra.is_err(), "a replied child must not deliver a notice");
+ let entries = sessions.list_subagents().await.expect("child roster");
+ assert_eq!(entries[0].status, "error");
+}
+
/// An unreachable child delivers the failure row, not the no-reply notice.
/// A short tick keeps the paused clock's auto-advance from firing link
/// deadlines while the real-socket round trips are in flight.