A self-improving RLM agent for coding workflows and long-running autonomous tasks.
Rust
fix(pa-daemon): a child's settled verdict survives its worker leaving inside the settle grace (#3225)
Sebastian Müller84b40b591e1c
diff --git a/crates/pa-daemon/src/rlm_children/lifecycle.rs b/crates/pa-daemon/src/rlm_children/lifecycle.rsindex 25f9f85516..1928df4cdc 100644--- a/crates/pa-daemon/src/rlm_children/lifecycle.rs+++ b/crates/pa-daemon/src/rlm_children/lifecycle.rs@@ -418,9 +418,11 @@ impl SupervisorChildSessionsInner {// pop (the queue snapshot and the busy flag change under// different locks on the far side of a socket). A short// grace closes that window; a child that went busy again- // (a queued continuation) keeps watching.+ // (a queued continuation) keeps watching. An unreachable+ // worker (an idle passivation, a crash) leaves the settled+ // verdict standing, as in `refresh_record`.tokio::time::sleep(Duration::from_millis(WATCH_SETTLE_GRACE_MS)).await;- if !matches!(self.child_busy(&active_session_id).await, Ok(false)) {+ if matches!(self.child_busy(&active_session_id).await, Ok(true)) {record.lock().await.settled_status = None;continue;}diff --git a/crates/pa-daemon/src/rlm_children/watch_tests.rs b/crates/pa-daemon/src/rlm_children/watch_tests.rsindex 5b46604289..ec71822a70 100644--- a/crates/pa-daemon/src/rlm_children/watch_tests.rs+++ b/crates/pa-daemon/src/rlm_children/watch_tests.rs@@ -2,15 +2,11 @@ use super::*;use crate::protocol::{response_failure, response_success};use pa_types::platform::transport::bind_transport;use serde_json::{json, Value};+use std::sync::atomic::{AtomicBool, Ordering};use std::sync::Arc;use tokio::io::{AsyncBufReadExt, AsyncWriteExt, BufReader};use tokio::sync::mpsc;-/// A scripted JSONL supervisor for the watcher tests: creates one child-/// session, reports it idle with a final answer, and captures the-/// `follow_up` commands routed to the parent (the terminal-notice-/// deliveries). `idle_delay_ms` paces `wait_for_idle` so a test can act-/// while the child is still "running"./// How the fake supervisor answers a child `kill`.enum FakeKill {Success,@@ -21,16 +17,26 @@ enum FakeKill {Failure,}+/// A scripted JSONL supervisor for the watcher tests: creates one child+/// session, reports it idle with a final answer, and captures the+/// `follow_up` commands routed to the parent (the terminal-notice+/// deliveries). `idle_delay_ms` paces `wait_for_idle` so a test can act+/// while the child is still "running". `worker_leaves_after_settle`+/// fails every child read after the settle answer is captured.async fn spawn_fake_supervisor(socket: std::path::PathBuf,follow_up_tx: mpsc::UnboundedSender<Value>,idle_delay_ms: u64,kill_tx: mpsc::UnboundedSender<Value>,kill_behavior: FakeKill,+ worker_leaves_after_settle: bool,) {let kill_behavior = std::sync::Arc::new(kill_behavior);let listener = bind_transport(&socket).await.unwrap();tokio::spawn(async move {+ // Shared across link connections: a left worker fails every child+ // read on whichever connection carries it.+ let gone = Arc::new(AtomicBool::new(false));loop {let Ok(stream) = listener.accept().await else {return;@@ -38,6 +44,7 @@ async fn spawn_fake_supervisor(let follow_up_tx = follow_up_tx.clone();let kill_tx = kill_tx.clone();let kill_behavior = std::sync::Arc::clone(&kill_behavior);+ let gone = Arc::clone(&gone);tokio::spawn(async move {let (reader, mut writer) = stream.split();let mut reader = BufReader::new(reader);@@ -57,6 +64,14 @@ async fn spawn_fake_supervisor(let command = value["command"].clone();let command_type: &str = command["type"].as_str().unwrap_or_default();let response = match command_type {+ "get_state" | "wait_for_idle" if gone.load(Ordering::SeqCst) => {+ response_failure(+ Some(&id),+ command_type,+ "Unknown active session: child-live",+ None,+ )+ }"create" => response_success(Some(&id),command_type,@@ -81,11 +96,18 @@ async fn spawn_fake_supervisor("sessionActions": { "queuedCount": 0 },})),),- "get_last_assistant_text" => response_success(- Some(&id),- command_type,- Some(json!({ "text": "the child final answer" })),- ),+ "get_last_assistant_text" => {+ // The settle capture: with the knob set, the+ // worker leaves right after its final answer.+ if worker_leaves_after_settle {+ gone.store(true, Ordering::SeqCst);+ }+ response_success(+ Some(&id),+ command_type,+ Some(json!({ "text": "the child final answer" })),+ )+ }"kill" => {let _ = kill_tx.send(command.clone());match *kill_behavior {@@ -131,6 +153,7 @@ async fn sessions_with_fake_supervisor(follow_up_tx: mpsc::UnboundedSender<Value>,idle_delay_ms: u64,kill_behavior: FakeKill,+ worker_leaves_after_settle: bool,) -> (SupervisorChildSessions, mpsc::UnboundedReceiver<Value>) {let socket = std::env::temp_dir().join(format!("pa-rlm-watch-{}.sock",@@ -143,6 +166,7 @@ async fn sessions_with_fake_supervisor(idle_delay_ms,kill_tx,kill_behavior,+ worker_leaves_after_settle,).await;let link = Arc::new(crate::supervisor_link::SupervisorLink::new(socket));@@ -182,7 +206,7 @@ async fn spawn_child(sessions: &SupervisorChildSessions) -> RlmSpawnHandle {async fn roster_snapshot_does_not_wait_for_a_slow_child_worker() {let (follow_up_tx, _follow_up_rx) = mpsc::unbounded_channel();let (sessions, _kill_rx) =- sessions_with_fake_supervisor(follow_up_tx, 1_000, FakeKill::Success).await;+ sessions_with_fake_supervisor(follow_up_tx, 1_000, FakeKill::Success, false).await;sessions.push_test_child(RlmChildIdentity {rlm_child_id: "child-id".to_string(),@@ -205,7 +229,7 @@ async fn roster_snapshot_does_not_wait_for_a_slow_child_worker() {async fn a_settled_child_without_a_reply_delivers_the_terminal_notice() {let (follow_up_tx, mut follow_up_rx) = mpsc::unbounded_channel();let (sessions, _kill_rx) =- sessions_with_fake_supervisor(follow_up_tx, 0, FakeKill::Success).await;+ sessions_with_fake_supervisor(follow_up_tx, 0, FakeKill::Success, false).await;let handle = spawn_child(&sessions).await;// The worker releases the detached prompt at its turn boundary.sessions.notify_turn_done();@@ -235,6 +259,32 @@ async fn a_settled_child_without_a_reply_delivers_the_terminal_notice() {assert!(extra.is_err(), "no second notice may arrive");}+/// The settle grace re-marks only a BUSY child as running: a worker that+/// leaves inside the grace (the idle passivation's stop, a crash) keeps+/// the settled verdict, and the settle tail (notice, funnel) still runs.+#[tokio::test]+async fn a_worker_leaving_inside_the_settle_grace_keeps_the_verdict() {+ let (follow_up_tx, mut follow_up_rx) = mpsc::unbounded_channel();+ let (sessions, _kill_rx) =+ sessions_with_fake_supervisor(follow_up_tx, 0, FakeKill::Success, true).await;+ let settled = sessions.settle_notified();+ spawn_child(&sessions).await;+ sessions.notify_turn_done();++ tokio::time::timeout(Duration::from_secs(10), settled)+ .await+ .expect("the settle funnel fires although the worker left");+ assert!(!sessions.any_running().await);+ let roster = sessions.list_subagents().await.expect("child roster");+ assert_eq!(roster[0].status, "completed");+ let notice = follow_up_rx+ .try_recv()+ .expect("the no-reply notice is still owed");+ assert!(notice["customMessage"]["content"]+ .as_str()+ .is_some_and(|content| content.contains("the child final answer")));+}+/// One child row exists and is running before the close tests run.async fn one_running_child(sessions: &SupervisorChildSessions) {let deadline = Instant::now() + Duration::from_secs(5);@@ -260,7 +310,7 @@ async fn close_children_stops_the_child_and_clears_the_roster() {// A long idle keeps the child mid-run while the close fires, so the// settle watcher is parked instead of raced.let (sessions, mut kill_rx) =- sessions_with_fake_supervisor(follow_up_tx, 10_000, FakeKill::Success).await;+ sessions_with_fake_supervisor(follow_up_tx, 10_000, FakeKill::Success, false).await;spawn_child(&sessions).await;sessions.notify_turn_done();one_running_child(&sessions).await;@@ -300,7 +350,7 @@ async fn close_children_stops_the_child_and_clears_the_roster() {async fn close_children_treats_an_already_gone_child_as_a_no_op() {let (follow_up_tx, _follow_up_rx) = mpsc::unbounded_channel();let (sessions, _kill_rx) =- sessions_with_fake_supervisor(follow_up_tx, 10_000, FakeKill::UnknownSession).await;+ sessions_with_fake_supervisor(follow_up_tx, 10_000, FakeKill::UnknownSession, false).await;spawn_child(&sessions).await;sessions.notify_turn_done();one_running_child(&sessions).await;@@ -324,7 +374,7 @@ async fn close_children_treats_an_already_gone_child_as_a_no_op() {async fn close_children_keeps_a_failed_child_tracked() {let (follow_up_tx, _follow_up_rx) = mpsc::unbounded_channel();let (sessions, _kill_rx) =- sessions_with_fake_supervisor(follow_up_tx, 10_000, FakeKill::Failure).await;+ sessions_with_fake_supervisor(follow_up_tx, 10_000, FakeKill::Failure, false).await;spawn_child(&sessions).await;sessions.notify_turn_done();one_running_child(&sessions).await;@@ -350,7 +400,7 @@ async fn a_replied_child_gets_no_terminal_notice() {// A slow idle wait keeps the child "running" while the test marks// the reply.let (sessions, _kill_rx) =- sessions_with_fake_supervisor(follow_up_tx, 250, FakeKill::Success).await;+ sessions_with_fake_supervisor(follow_up_tx, 250, FakeKill::Success, false).await;let handle = spawn_child(&sessions).await;assert!(!handle.rlm_child_id.is_empty());sessions.mark_replied("child-live").await;@@ -370,7 +420,7 @@ async fn a_replied_child_gets_no_terminal_notice() {async fn collect_answers_a_just_deleted_target_with_the_cancelled_envelope() {let (follow_up_tx, _follow_up_rx) = mpsc::unbounded_channel();let (sessions, _kill_rx) =- sessions_with_fake_supervisor(follow_up_tx, 0, FakeKill::Success).await;+ sessions_with_fake_supervisor(follow_up_tx, 0, FakeKill::Success, false).await;let handle = spawn_child(&sessions).await;sessions.notify_turn_done();// The child settles with its final answer before the delete.@@ -443,7 +493,7 @@ async fn collect_answers_a_just_deleted_target_with_the_cancelled_envelope() {async fn collect_answers_the_cancelled_envelope_after_an_inactive_delete() {let (follow_up_tx, _follow_up_rx) = mpsc::unbounded_channel();let (sessions, _kill_rx) =- sessions_with_fake_supervisor(follow_up_tx, 0, FakeKill::Success).await;+ sessions_with_fake_supervisor(follow_up_tx, 0, FakeKill::Success, false).await;let handle = spawn_child(&sessions).await;sessions.notify_turn_done();// The inactive delete requires a settled child (a running childdiff --git a/crates/pa-daemon/tests/idle_passivation_e2e.rs b/crates/pa-daemon/tests/idle_passivation_e2e.rsindex 19515757c8..7b30c99ae6 100644--- a/crates/pa-daemon/tests/idle_passivation_e2e.rs+++ b/crates/pa-daemon/tests/idle_passivation_e2e.rs@@ -405,10 +405,11 @@ async fn a_settled_child_passivates_stays_listable_and_revives_by_prompt() {// wakes a fresh worker over the saved file (the route's wake arm// resolves the saved session and launches). The faux engine's script// is spawn-time config (not session-file state), so the replayed- // worker's turn runs the default provider — the WAKE oracle here is- // the respawn + the prompt's delivery into the child's session file;- // the model-answer revival is the VM census's leg (the real binary- // against the offline mock).+ // worker's turn runs the default provider: the response's outcome+ // depends on the host's credentials and is not asserted — the WAKE+ // oracle is the revival prompt's row in the child's session file+ // (only a woken worker writes it); the model-answer revival is the+ // VM census's leg (the real binary against the offline mock).let child_file_rows_before = std::fs::read_to_string({let roster = children.list_subagents()@@ -424,23 +425,16 @@ async fn a_settled_child_passivates_stays_listable_and_revives_by_prompt() {))}).map_or(0, |content| content.lines().count());+ let revive_prompt = "revive: answer again";client.send_command("revive",&json!({"type": "prompt_and_wait","activeSessionId": child_session_id,- "message": "revive: answer again",+ "message": revive_prompt,}),);let revived = client.read_response("revive");- assert!(- revived["success"] == true- || revived- .get("error")- .and_then(Value::as_str)- .is_some_and(|error| error.contains("Insufficient balance")),- "the wake must relaunch the child (the faux replay's model turn may fail auth; the wake itself must not): {revived}"- );// The revival respawned a worker for the child's session (a fresh// pid serves the replayed file) and the prompt's row landed in the// child's session file (the delivery half of the revival).@@ -470,8 +464,13 @@ async fn a_settled_child_passivates_stays_listable_and_revives_by_prompt() {}assert!(Instant::now() < deadline,- "the revival never delivered the prompt into the child's session file ({child_file:?}, rows {rows_after} <= {child_file_rows_before})"+ "the revival never delivered the prompt into the child's session file ({child_file:?}, rows {rows_after} <= {child_file_rows_before}, response: {revived})");tokio::time::sleep(Duration::from_millis(100)).await;}+ let grown = std::fs::read_to_string(&child_file).expect("the child session file");+ assert!(+ grown.contains(revive_prompt),+ "the revival prompt never landed in the child's session file ({child_file:?}, response: {revived})"+ );}