A self-improving RLM agent for coding workflows and long-running autonomous tasks.
Rust
carry childscript to spawned children and wait for rlm quiescence (#3171)
Sebastian Müller45e8f4970fc7
diff --git a/crates/pa-daemon/src/engine/scripted.rs b/crates/pa-daemon/src/engine/scripted.rsindex 1cf15a8586..b6c0af4f79 100644--- a/crates/pa-daemon/src/engine/scripted.rs+++ b/crates/pa-daemon/src/engine/scripted.rs@@ -273,16 +273,20 @@ impl SessionEngine for ScriptedEngine {&self,prompt_index: usize,request: PromptRequest,- _aborted: &dyn Fn() -> bool,+ aborted: &dyn Fn() -> bool,emit: &mut dyn FnMut(EngineEvent) -> bool,) {let cancelled = || EngineEvent::Done(Err("prompt cancelled".to_string()));let scripted = self.responses.get(prompt_index).cloned();let text = match &scripted {Some(response) => {- let delay = Self::response_delay_ms(response);- if delay > 0 {- std::thread::sleep(std::time::Duration::from_millis(delay));+ // The scripted hold honors the worker's cancel probe: a+ // close on a held scripted child settles at the abort,+ // not at the hold's end.+ let delay = std::time::Duration::from_millis(Self::response_delay_ms(response));+ if !abortable_sleep(delay, aborted) {+ emit(cancelled());+ return;}Self::response_text(response)}@@ -516,7 +520,7 @@ impl SessionEngine for ScriptedEngine {Some(&signal),move |delay| {let wait_signal = wait_signal.clone();- async move { abortable_sleep(delay, &wait_signal) }+ async move { abortable_sleep(delay, &|| wait_signal.is_aborted()) }},move || {let responses = StdArc::clone(&responses);@@ -609,7 +613,11 @@ impl SessionEngine for ScriptedEngine {};}let delay_ms = Self::response_delay_ms(entry);- if delay_ms > 0 && !abortable_sleep(std::time::Duration::from_millis(delay_ms), signal) {+ if delay_ms > 0+ && !abortable_sleep(std::time::Duration::from_millis(delay_ms), &|| {+ signal.is_aborted()+ })+ {return CompactionOutcome::Aborted;}if signal.is_aborted() {@@ -674,7 +682,11 @@ impl SessionEngine for ScriptedEngine {};}let delay_ms = Self::response_delay_ms(entry);- if delay_ms > 0 && !abortable_sleep(std::time::Duration::from_millis(delay_ms), signal) {+ if delay_ms > 0+ && !abortable_sleep(std::time::Duration::from_millis(delay_ms), &|| {+ signal.is_aborted()+ })+ {return BranchSummaryOutcome::Aborted;}if signal.is_aborted() {@@ -770,7 +782,11 @@ fn scripted_side_question_turn(return message;}let delay_ms = ScriptedEngine::response_delay_ms(entry);- if delay_ms > 0 && !abortable_sleep(std::time::Duration::from_millis(delay_ms), signal) {+ if delay_ms > 0+ && !abortable_sleep(std::time::Duration::from_millis(delay_ms), &|| {+ signal.is_aborted()+ })+ {set_text(&mut message, &text);message.stop_reason = StopReason::Aborted;return message;@@ -784,17 +800,17 @@ fn scripted_side_question_turn(message}-/// Sleep `delay` in slices, stopping early when `signal` aborts.+/// Sleep `delay` in slices, stopping early when `aborted` turns true./// Returns `false` when the wait ended aborted.-fn abortable_sleep(delay: std::time::Duration, signal: &AbortSignal) -> bool {+fn abortable_sleep(delay: std::time::Duration, aborted: &dyn Fn() -> bool) -> bool {let mut remaining = delay;while !remaining.is_zero() {- if signal.is_aborted() {+ if aborted() {return false;}let slice = remaining.min(std::time::Duration::from_millis(25));std::thread::sleep(slice);remaining = remaining.saturating_sub(slice);}- !signal.is_aborted()+ !aborted()}diff --git a/crates/pa-daemon/src/rlm_children.rs b/crates/pa-daemon/src/rlm_children.rsindex 73b6e6de66..deef81431e 100644--- a/crates/pa-daemon/src/rlm_children.rs+++ b/crates/pa-daemon/src/rlm_children.rs@@ -163,6 +163,11 @@ struct ChildRecord {/// Terminal state (`done` | `error` | `cancelled`); running while/// absent.settled_status: Option<&'static str>,+ /// TS `run.settled`: set only by the settle funnel, after the+ /// terminal notice is delivered - the terminal status above flips+ /// earlier (TS's `run.status`/`run.settled` pair), so the quiescence+ /// predicate waits out the notice window.+ settled: bool,answer_preview: Option<String>,answer_captured: bool,/// An agent message from this child reached the parent since its task@@ -328,6 +333,12 @@ struct SupervisorChildSessionsInner {/// The parent engine's child-settle hook (goal continuation resume);/// `None` until the engine wires it.settle_hook: std::sync::Mutex<Option<Arc<dyn Fn() + Send + Sync>>>,+ /// The quiescence barrier's wake (TS `waitForRlmQuiescence`,+ /// agent-session.ts): `notify_waiters` fires once per settled child+ /// run - every settle site funnels through the settle hook below -+ /// and once per close walk, so a barrier parked behind descendant+ /// work re-reads the registry when the descendants settle.+ settle_notify: tokio::sync::Notify,/// The worker's model-allowlist refusal telemetry (`model refused`):/// `spawn/create_session` refusals emit through the engine's shared/// lazily-built client.@@ -371,6 +382,7 @@ impl SupervisorChildSessions {deleted_children: std::sync::Mutex::new(std::collections::HashMap::new()),turn_done: tokio::sync::watch::Sender::new(0),settle_hook: std::sync::Mutex::new(None),+ settle_notify: tokio::sync::Notify::new(),model_refusal_telemetry,usage_sink: std::sync::Mutex::new(None),delete_notifier: std::sync::Mutex::new(None),@@ -440,13 +452,20 @@ impl SupervisorChildSessions {.contains(name)}+ /// The barrier's wake permit (the `settle_notify` field owns the+ /// semantics).+ pub(crate) fn settle_notified(&self) -> tokio::sync::futures::Notified<'_> {+ self.inner.settle_notify.notified()+ }+/// Whether any tracked child run is still unsettled (TS- /// `_hasUnsettledRlmQuiescenceWork`'s child-run arm: a record without- /// a terminal state).+ /// `_hasUnsettledRlmQuiescenceWork`'s child-run arm: a run whose+ /// settle funnel has not fired - the terminal status flips before+ /// the terminal notice is delivered, `run.settled` after).pub async fn any_running(&self) -> bool {let children = self.inner.children.lock().await;for record in children.iter() {- if record.lock().await.settled_status.is_none() {+ if !record.lock().await.settled {return true;}}@@ -637,6 +656,7 @@ impl SupervisorChildSessions {label: String::new(),started_at_ms: 0,settled_status: None,+ settled: false,answer_preview: None,answer_captured: false,replied_since_task: false,@@ -767,8 +787,13 @@ impl SupervisorChildSessions {impl SupervisorChildSessionsInner {/// Fire the settle hook off-thread (the settle sites run inside- /// watcher tasks; the hook owns its own scheduling).- pub(crate) fn fire_settle_hook(&self) {+ /// watcher tasks; the hook owns its own scheduling). The funnel is+ /// the one definition of a settled run (TS's run task `finally`): it+ /// marks the record settled and wakes the quiescence barrier, only+ /// after the terminal notice is delivered.+ pub(crate) async fn fire_settle_hook(&self, record: &Arc<Mutex<ChildRecord>>) {+ record.lock().await.settled = true;+ self.settle_notify.notify_waiters();let hook = self.settle_hook.lock()diff --git a/crates/pa-daemon/src/rlm_children/host.rs b/crates/pa-daemon/src/rlm_children/host.rsindex 6ff5cd0014..73dc4b4056 100644--- a/crates/pa-daemon/src/rlm_children/host.rs+++ b/crates/pa-daemon/src/rlm_children/host.rs@@ -136,6 +136,7 @@ impl RlmSubagentHost for SupervisorChildSessions {label: rlm_child_label(&request.prompt),started_at_ms: now_ms(),settled_status: None,+ settled: false,answer_preview: None,answer_captured: false,replied_since_task: false,@@ -215,6 +216,7 @@ impl RlmSubagentHost for SupervisorChildSessions {.kill_child(&child_active_session_id, ChildCloseReason::Killed).await;watcher_record.lock().await.settled_status = Some("error");+ watcher_this.fire_settle_hook(&watcher_record).await;return;}}@@ -405,6 +407,10 @@ impl RlmSubagentHost for SupervisorChildSessions {this.deliver_terminal_notice(¬ice).await;}}+ // The deletion settles the run (TS `_finishRlmRunDeletion`,+ // the same resume site the inactive delete funnels through):+ // a parked barrier re-reads a removed record as settled.+ this.fire_settle_hook(&record).await;Ok(RlmDeleteSubagentResult {subagent: entry,outcome: Some("deleted"),diff --git a/crates/pa-daemon/src/rlm_children/lifecycle.rs b/crates/pa-daemon/src/rlm_children/lifecycle.rsindex 6409906100..25f9f85516 100644--- a/crates/pa-daemon/src/rlm_children/lifecycle.rs+++ b/crates/pa-daemon/src/rlm_children/lifecycle.rs@@ -433,7 +433,7 @@ impl SupervisorChildSessionsInner {// A settled child releases an owed goal continuation (TS// `_maybeResumeGoalContinuationAfterRlmWork` at the child// settle sites).- self.fire_settle_hook();+ self.fire_settle_hook(record).await;return;}// Still running (a timed-out slice or a re-queued continuation):@@ -456,7 +456,7 @@ impl SupervisorChildSessionsInner {// holds; capture them before the terminal notice.self.emit_child_usage(record).await;self.deliver_settle_notice(record).await;- self.fire_settle_hook();+ self.fire_settle_hook(record).await;return;}}diff --git a/crates/pa-daemon/src/rlm_children/registry.rs b/crates/pa-daemon/src/rlm_children/registry.rsindex fb427b27c4..f3826ae763 100644--- a/crates/pa-daemon/src/rlm_children/registry.rs+++ b/crates/pa-daemon/src/rlm_children/registry.rs@@ -97,7 +97,7 @@ impl SupervisorChildSessionsInner {.with_context(|| format!("abort RLM child session {active_session_id}"));// The settled/cancelled child releases an owed goal// continuation.- self.fire_settle_hook();+ self.fire_settle_hook(record).await;return true;}false@@ -168,7 +168,7 @@ impl SupervisorChildSessionsInner {.retain(|candidate| !Arc::ptr_eq(candidate, record));// The deleted child is a TS resume site for the owed goal// continuation (`_finishRlmRunDeletion`).- self.fire_settle_hook();+ self.fire_settle_hook(record).await;return Ok("deleted");}Ok("not_found")@@ -227,6 +227,9 @@ impl SupervisorChildSessionsInner {.await.retain(|candidate| !Arc::ptr_eq(candidate, record));}+ // The walk changed the registry: wake a parked barrier (a closed+ // child is settled work, settled here by its removal).+ self.settle_notify.notify_waiters();match close_error {Some(error) => Err(error),None => Ok(()),diff --git a/crates/pa-daemon/src/rlm_children/usage_emit_tests.rs b/crates/pa-daemon/src/rlm_children/usage_emit_tests.rsindex b9241e6ec5..9572f0cd44 100644--- a/crates/pa-daemon/src/rlm_children/usage_emit_tests.rs+++ b/crates/pa-daemon/src/rlm_children/usage_emit_tests.rs@@ -36,6 +36,7 @@ fn record_with_file(child_id: &str, session_file: &Path) -> Arc<Mutex<ChildRecorlabel: "child".to_string(),started_at_ms: 0,settled_status: None,+ settled: false,answer_preview: None,answer_captured: false,replied_since_task: false,diff --git a/crates/pa-daemon/src/rlm_children/watch_tests.rs b/crates/pa-daemon/src/rlm_children/watch_tests.rsindex 2194f9fc62..5b46604289 100644--- a/crates/pa-daemon/src/rlm_children/watch_tests.rs+++ b/crates/pa-daemon/src/rlm_children/watch_tests.rs@@ -495,6 +495,7 @@ fn an_already_settled_child_never_re_scores_as_an_unreachable_error() {label: "task".to_string(),started_at_ms: 0,settled_status: None,+ settled: false,answer_preview: None,answer_captured: false,replied_since_task: false,diff --git a/crates/pa-daemon/src/supervisor/worker_lifecycle.rs b/crates/pa-daemon/src/supervisor/worker_lifecycle.rsindex 8526610961..e9c62c5030 100644--- a/crates/pa-daemon/src/supervisor/worker_lifecycle.rs+++ b/crates/pa-daemon/src/supervisor/worker_lifecycle.rs@@ -239,6 +239,9 @@ impl Supervisor {"rlmMaxDepth","parentSessionPath","models",+ // The scripted-parent verification seam: a dropped key leaves+ // spawned children scriptless.+ "childScript","systemPrompt","appendSystemPrompt","skills",diff --git a/crates/pa-daemon/src/worker/commands.rs b/crates/pa-daemon/src/worker/commands.rsindex e466dd0418..2b33edb461 100644--- a/crates/pa-daemon/src/worker/commands.rs+++ b/crates/pa-daemon/src/worker/commands.rs@@ -90,7 +90,9 @@ impl Worker {}"set_auto_compaction" => self.handle_set_auto_compaction(payload),"wait_for_idle" => self.handle_wait_for_idle().await,- "wait_for_headless_completion" => self.handle_wait_for_headless_completion().await,+ "wait_for_headless_completion" => {+ self.handle_wait_for_headless_completion(payload).await+ }"get_state" => self.handle_get_state(),"get_messages" => self.handle_get_messages(),"get_session_header" => self.handle_get_session_header(),diff --git a/crates/pa-daemon/src/worker/lifecycle.rs b/crates/pa-daemon/src/worker/lifecycle.rsindex 3e71cad9e6..05a3d78a62 100644--- a/crates/pa-daemon/src/worker/lifecycle.rs+++ b/crates/pa-daemon/src/worker/lifecycle.rs@@ -631,33 +631,79 @@ impl Worker {}}- pub(crate) async fn handle_wait_for_idle(&self) -> DaemonResponse {+ /// The idle park shared by `wait_for_idle` and the headless barrier:+ /// register the permit before the flag check, or a turn that settles+ /// between the check and the await loses its wake+ /// (`notify_waiters` only reaches registered futures).+ async fn wait_until_idle(&self) {loop {+ let idle = self.idle_notify.notified();{let core = self.core.lock().unwrap();if !core.busy && core.steering.is_empty() && core.follow_up.is_empty() {- return response_success(None, "wait_for_idle", None);+ return;}}- self.idle_notify.notified().await;+ idle.await;}}+ pub(crate) async fn handle_wait_for_idle(&self) -> DaemonResponse {+ self.wait_until_idle().await;+ response_success(None, "wait_for_idle", None)+ }+/// `wait_for_headless_completion` (TS daemon command): settle the/// headless run first (same idle wait as `wait_for_idle`), then answer/// the autonomous-run accounting snapshot (`DaemonAutonomousStatus`).- pub(crate) async fn handle_wait_for_headless_completion(&self) -> DaemonResponse {+ /// `waitForRlmQuiescence` (TS `waitForHeadlessCompletion`'s strong arm+ /// over `agent-session.ts` `waitForRlmQuiescence`): the barrier also+ /// owns descendant work - it holds past the parent's idle until every+ /// tracked child run's settle funnel fires (after the terminal notice+ /// is delivered), and the settle loop re-runs the idle wait, so a+ /// settled child's terminal notice (a queued parent turn) drains+ /// inside the barrier exactly like TS's "work may start at the+ /// child-settlement boundary" re-check.+ pub(crate) async fn handle_wait_for_headless_completion(+ &self,+ payload: &Value,+ ) -> DaemonResponse {if let Err(response) = self.require_created("wait_for_headless_completion") {return response;}+ let children = if payload.get("waitForRlmQuiescence").and_then(Value::as_bool) == Some(true)+ {+ self.agent_engine+ .as_ref()+ .and_then(|engine| engine.children.clone())+ } else {+ None+ };loop {- {- let core = self.core.lock().unwrap();- if !core.busy && core.steering.is_empty() && core.follow_up.is_empty() {+ self.wait_until_idle().await;+ let Some(children) = children.as_ref() else {+ break;+ };+ // Register the permit before the unsettled-work read: a run+ // that settles between the read and the await still wakes+ // this waiter.+ let settled = children.settle_notified();+ if !children.any_running().await {+ // A settle funnel queues its terminal-notice follow-up+ // BEFORE it marks the run settled, so every notice owed by+ // the runs settled at this read is already queued: one+ // more idle wait drains them before the barrier answers.+ self.wait_until_idle().await;+ // A notice can start new child work during that drain (a+ // child-settle hook spawning a descendant): re-read the+ // runs before answering, so the barrier holds for the+ // new work too instead of completing at the boundary.+ if !children.any_running().await {break;}+ continue;}- self.idle_notify.notified().await;+ settled.await;}// The idle wait finished, so no turn holds the accounting state;// the snapshot read cannot interleave with a running turn.diff --git a/crates/pa-daemon/tests/rlm_quiescence_barrier_e2e.rs b/crates/pa-daemon/tests/rlm_quiescence_barrier_e2e.rsnew file mode 100644index 0000000000..6e32d3a474--- /dev/null+++ b/crates/pa-daemon/tests/rlm_quiescence_barrier_e2e.rs@@ -0,0 +1,489 @@+//! The worker's RLM quiescence barrier e2e (`wait_for_headless_completion`+//! with `waitForRlmQuiescence`, the `rlm_quiescence_barrier` capability the+//! worker advertises).+//!+//! TS ground truth - `packages/coding-agent/src/modes/headless-completion.ts`+//! settles `waitForHeadlessCompletion({ waitForRlmQuiescence: true })` through+//! `core/agent-session.ts` `waitForRlmQuiescence()`, so the barrier owns the+//! parent's own idle FIRST, then every admitted descendant run's settlement,+//! and it loops back through the idle wait because work may start at the+//! child-settlement boundary (a settled child's terminal notice queues a+//! parent turn). A barrier without the flag - and the pre-fix worker, which+//! ignored the flag - answers at the parent's idle while a child run is+//! still in flight.+//!+//! Verified end to end against a real supervisor, a real parent worker+//! session whose kernel cell spawns the child through the product+//! `rlm.spawn` surface, and a scripted child worker held mid-run.+//!+//! 1. `wait_for_headless_completion` with `waitForRlmQuiescence: true`+//! answers only after the held child turn settles and the terminal+//! notice it queues on the parent drains: the parent's+//! `get_rlm_children` reports the child terminal and its messages+//! carry the notice turn at answer time. The pre-fix worker answered+//! at the parent's idle (elapsed milliseconds) with the child still+//! running.+//! 2. The barrier reads the run's settlement, not the roster status:+//! the roster flips terminal inside the watcher's settle grace,+//! ~250ms before the run's settle funnel, so a barrier sent once the+//! roster says `done` still holds for the terminal notice's parent+//! turn. A status-reading barrier answers inside that window.+//!+//! The parent's kernel Python is ambient product state; like the other+//! live-kernel verifiers these tests skip (with a note) on machines+//! without a live install.+// Pedantic-gate dispositions (fleet-uniform ruling; see this lane's PR for+// the full rationale): the timeout panic path cannot wait on the supervisor+// child; the test process exits immediately afterwards, reaping it.+#![allow(clippy::zombie_processes)]+#![cfg(unix)]++use std::io::{BufRead, BufReader, Write};+use std::os::unix::net::UnixStream;+use std::path::{Path, PathBuf};+use std::process::{Child, Command, Stdio};+use std::time::{Duration, Instant};++use serde_json::{json, Value};++struct Daemon {+ child: Child,+ #[allow(dead_code)]+ socket: PathBuf,+}++impl Drop for Daemon {+ fn drop(&mut self) {+ let _ = self.child.kill();+ let _ = self.child.wait();+ }+}++fn spawn_supervisor(socket: &Path, agent_dir: &Path, kernel_python: &Path) -> Daemon {+ let binary = env!("CARGO_BIN_EXE_pa-daemon");+ let child = Command::new(binary)+ .arg("supervisor")+ .arg("--socket")+ .arg(socket)+ .arg("--agent-dir")+ .arg(agent_dir)+ .env("PRIME_AGENT_KERNEL_PYTHON", kernel_python)+ // Hermetic agent dir: the ambient environment exports a real+ // agent dir; point every fallback at the test sandbox instead.+ .env("PRIME_AGENT_CODING_AGENT_DIR", agent_dir)+ .env_remove("PRIME_API_KEY")+ .stdout(Stdio::null())+ .stderr(Stdio::null())+ // A supervisor killed at teardown must not leak its session workers+ // into later test binaries: the worker's supervisor-lost exit (TS+ // `exitIfSupervisorOrphanedForTooLong`) runs on this short window+ // instead of the 5-minute default.+ .env(+ pa_daemon::worker::WORKER_SUPERVISOR_LOST_EXIT_MS_ENV,+ "15000",+ )+ .spawn()+ .expect("spawn pa-daemon supervisor");+ Daemon {+ child,+ socket: socket.to_path_buf(),+ }+}++/// The kernel Python with prime-agent-runtime installed; set+/// `PA_E2E_KERNEL_PYTHON` to point at an explicit interpreter instead.+fn kernel_python() -> Option<PathBuf> {+ if let Some(explicit) = std::env::var_os("PA_E2E_KERNEL_PYTHON") {+ let explicit = PathBuf::from(explicit);+ assert!(+ explicit.exists(),+ "PA_E2E_KERNEL_PYTHON {} not found",+ explicit.display()+ );+ return Some(explicit);+ }+ let candidate = PathBuf::from(std::env::var("HOME").map_or_else(+ |_| "/home/ubuntu/.prime/agent/kernel-venv/bin/python".to_string(),+ |home| format!("{home}/.prime/agent/kernel-venv/bin/python"),+ ));+ if candidate.exists() {+ return Some(candidate);+ }+ eprintln!(+ "kernel python {} not found; skipping live RLM quiescence e2e",+ candidate.display()+ );+ None+}++fn wait_socket_ready(socket: &Path) {+ let deadline = Instant::now() + Duration::from_secs(15);+ loop {+ if UnixStream::connect(socket).is_ok() {+ return;+ }+ assert!(Instant::now() < deadline, "supervisor socket never came up");+ std::thread::sleep(Duration::from_millis(20));+ }+}++/// JSONL supervisor client (command envelopes, id-matched responses).+struct Client {+ reader: BufReader<UnixStream>,+ writer: UnixStream,+}++impl Client {+ fn connect(socket: &Path) -> (Self, Value) {+ let stream = UnixStream::connect(socket).expect("connect supervisor");+ let write_stream = stream.try_clone().expect("clone socket");+ let mut client = Client {+ reader: BufReader::new(stream),+ writer: write_stream,+ };+ let hello = client.read_line();+ (client, hello)+ }++ fn send_command(&mut self, id: &str, command: &Value) {+ let envelope = json!({+ "type": "command",+ "id": id,+ "protocol": { "name": "prime-agent.daemon", "version": 7 },+ "command": command,+ });+ let mut line = serde_json::to_string(&envelope).expect("serialize command");+ line.push('\n');+ self.writer.write_all(line.as_bytes()).expect("send");+ self.writer.flush().expect("flush");+ }++ fn read_line(&mut self) -> Value {+ let mut line = String::new();+ let deadline = Instant::now() + Duration::from_secs(120);+ self.reader+ .get_mut()+ .set_read_timeout(Some(Duration::from_millis(100)))+ .expect("set timeout");+ loop {+ line.clear();+ match self.reader.read_line(&mut line) {+ Ok(0) => panic!("supervisor closed the connection"),+ Ok(_) if line.trim().is_empty() => {}+ Ok(_) => return serde_json::from_str(line.trim()).expect("parse response line"),+ Err(error) => {+ assert!(+ Instant::now() < deadline,+ "timed out waiting for supervisor line: {error}"+ );+ }+ }+ }+ }++ fn read_response(&mut self, id: &str) -> Value {+ let deadline = Instant::now() + Duration::from_secs(240);+ loop {+ assert!(Instant::now() < deadline, "no response for id {id}");+ let line = self.read_line();+ if line.get("id").and_then(Value::as_str) == Some(id) {+ return line;+ }+ }+ }+}++/// The parent's tracked-children wire surface (`get_rlm_children`).+fn rlm_children_rows(client: &mut Client, id: &str, parent: &str) -> Vec<Value> {+ client.send_command(+ id,+ &json!({ "type": "get_rlm_children", "activeSessionId": parent }),+ );+ let response = client.read_response(id);+ assert_eq!(+ response["success"], true,+ "get_rlm_children failed: {response}"+ );+ response["data"]["children"]+ .as_array()+ .cloned()+ .expect("children array")+}++/// Run one turn (prompt + idle wait) on the session.+fn run_turn(client: &mut Client, session_id: &str, message: &str, id: &str) {+ client.send_command(+ id,+ &json!({ "type": "prompt", "activeSessionId": session_id, "message": message }),+ );+ let prompted = client.read_response(id);+ assert_eq!(prompted["success"], true, "prompt failed: {prompted}");+ let idle_id = format!("{id}-idle");+ client.send_command(+ &idle_id,+ &json!({ "type": "wait_for_idle", "activeSessionId": session_id }),+ );+ let idle = client.read_response(&idle_id);+ assert_eq!(idle["success"], true, "wait_for_idle failed: {idle}");+}++/// Poll until a kernel cell's receipt content appears (the cell writes its+/// verdict).+fn await_receipt(receipt: &Path) -> String {+ let deadline = Instant::now() + Duration::from_secs(60);+ loop {+ if let Ok(content) = std::fs::read_to_string(receipt) {+ return content;+ }+ assert!(+ Instant::now() < deadline,+ "kernel cell receipt never appeared at {}",+ receipt.display()+ );+ std::thread::sleep(Duration::from_millis(50));+ }+}++/// The kernel cell of the spawn turn: spawn one RLM child through the+/// product `rlm.spawn` surface and record its child id.+fn spawn_cell(receipt: &Path, error_receipt: &Path) -> String {+ format!(+ "import json, traceback\ntry:\n handle = await rlm.spawn(\"run the lane task\", name=\"kid\")\n open({receipt:?}, \"w\").write(json.dumps({{\"rlm_child_id\": handle.rlm_child_id}}))\n print(handle.rlm_child_id)\nexcept Exception:\n open({error_receipt:?}, \"w\").write(traceback.format_exc())\n raise",+ receipt = receipt.display().to_string(),+ error_receipt = error_receipt.display().to_string(),+ )+}++/// The child's scripted engine: one held response keeps its task turn+/// running past the parent's idle, so the barrier must own it.+fn write_child_script(dir: &Path) -> PathBuf {+ let script = dir.join("child.json");+ std::fs::write(+ &script,+ json!({ "responses": [ { "text": "kid done", "delayMs": 5_000 } ] }).to_string(),+ )+ .expect("write child script");+ script+}++/// The parent's faux script whose turns run the spawn cell; the third+/// response answers the settled child's terminal-notice turn (the no-reply+/// notice the watcher queues on the parent).+fn write_parent_script(dir: &Path, first_cell: &str) -> PathBuf {+ let script = dir.join("parent.json");+ std::fs::write(+ &script,+ json!({+ "engine": "faux",+ "responses": [+ { "content": [+ { "type": "toolCall", "name": "ipython", "arguments": { "code": first_cell } },+ ] },+ { "text": "spawn turn done" },+ { "text": "notice seen" },+ ],+ })+ .to_string(),+ )+ .expect("write parent script");+ script+}++/// Create a scripted parent session through the supervisor. The create's+/// `childScript` (the harness seam mirroring the TS child runtime's+/// inherited `sessionConfig`) makes every `rlm.spawn` child a scripted+/// worker.+fn create_parent(+ client: &mut Client,+ dir: &Path,+ parent_script: &Path,+ child_script: &Path,+ id: &str,+) -> Value {+ let sessions_dir = dir.join("sessions");+ std::fs::create_dir_all(&sessions_dir).expect("sessions dir");+ client.send_command(+ id,+ &json!({+ "type": "create",+ "name": "parent",+ "config": {+ "cwd": dir.to_string_lossy(),+ "sessionDir": sessions_dir.to_string_lossy(),+ "script": parent_script.to_string_lossy(),+ "childScript": child_script.to_string_lossy(),+ },+ }),+ );+ let created = client.read_response(id);+ assert_eq!(created["success"], true, "create parent failed: {created}");+ created["data"].clone()+}++/// The parent's messages as one JSON string (the barrier's answer-time+/// transcript read).+fn messages_text(client: &mut Client, id: &str, parent_id: &str) -> String {+ client.send_command(+ id,+ &json!({ "type": "get_messages", "activeSessionId": parent_id }),+ );+ client.read_response(id)["data"].to_string()+}++/// One scripted lane with a held scripted child (the shared setup of the+/// barrier tests): the parent went idle with the child's 5s held turn in+/// flight, ready for the barrier.+struct HeldChildLane {+ _daemon: Daemon,+ _dir: tempfile::TempDir,+ client: Client,+ parent_id: String,+ child_id: String,+}++fn spawn_held_child_lane(kernel_python: &Path) -> HeldChildLane {+ let dir = tempfile::TempDir::new().expect("temp dir");+ let agent_dir = dir.path().join("agent");+ std::fs::create_dir_all(&agent_dir).expect("agent dir");+ let socket = dir.path().join("supervisor.sock");+ let receipts = dir.path().join("receipts");+ std::fs::create_dir_all(&receipts).expect("receipts dir");+ let spawn_receipt = receipts.join("spawn.json");+ let spawn_error = receipts.join("spawn.error");++ let child_script = write_child_script(dir.path());+ let parent_script = write_parent_script(dir.path(), &spawn_cell(&spawn_receipt, &spawn_error));+ let daemon = spawn_supervisor(&socket, &agent_dir, kernel_python);+ wait_socket_ready(&socket);+ let (mut client, hello) = Client::connect(&socket);+ assert_eq!(hello["type"], "daemon_hello");+ let parent = create_parent(&mut client, dir.path(), &parent_script, &child_script, "c1");+ let parent_id = parent["activeSessionId"]+ .as_str()+ .expect("parent active session id")+ .to_string();++ // Turn 1: the kernel cell spawns the child through the parent's own+ // registry; the parent settles while the child's held turn runs on.+ run_turn(&mut client, &parent_id, "spawn the kid", "t1");+ let spawned: Value =+ serde_json::from_str(&await_receipt(&spawn_receipt)).expect("spawn receipt json");+ let child_id = spawned["rlm_child_id"]+ .as_str()+ .expect("child id")+ .to_string();+ assert!(+ !spawn_error.exists(),+ "the spawn cell failed: {}",+ std::fs::read_to_string(&spawn_error).unwrap_or_default()+ );+ HeldChildLane {+ _daemon: daemon,+ _dir: dir,+ client,+ parent_id,+ child_id,+ }+}++/// `wait_for_headless_completion` with `waitForRlmQuiescence: true` owns+/// descendant work (TS `waitForRlmQuiescence`): the parent goes idle while+/// the spawned child's 5s held turn still runs, and the barrier must+/// answer only after that turn settles and the terminal notice it queues+/// on the parent drains - the registry reports the child terminal, and+/// the parent's messages carry the notice turn at answer time. The+/// pre-fix worker ignored the flag and answered at the parent's idle,+/// with the child still running.+#[test]+fn headless_completion_waits_for_rlm_quiescence() {+ let Some(kernel_python) = kernel_python() else {+ return;+ };+ let mut lane = spawn_held_child_lane(&kernel_python);++ // The barrier request on the idle parent: the child's 5s hold is+ // unsettled descendant work, so the answer must hold past it.+ lane.client.send_command(+ "w1",+ &json!({+ "type": "wait_for_headless_completion",+ "activeSessionId": lane.parent_id,+ "waitForRlmQuiescence": true,+ }),+ );+ let completion = lane.client.read_response("w1");+ assert_eq!(+ completion["success"], true,+ "wait_for_headless_completion failed: {completion}"+ );++ // The answer lands only after the child's held turn settled and the+ // terminal notice's parent turn drained.+ let messages = messages_text(&mut lane.client, "m1", &lane.parent_id);+ assert!(+ messages.contains("notice seen"),+ "the barrier answered before the terminal-notice turn drained"+ );+ let rows = rlm_children_rows(&mut lane.client, "g1", &lane.parent_id);+ let child_row = rows+ .iter()+ .find(|row| row["id"] == json!(lane.child_id))+ .expect("the spawned child's roster row");+ assert_eq!(+ child_row["status"], "done",+ "the barrier answered while the child was still running"+ );+}++/// The barrier reads the run's settlement, not the roster status: the+/// roster flips terminal inside the watcher's settle grace, and the+/// record only settles at the settle funnel - after the grace re-check,+/// the usage emit, and the terminal-notice delivery (TS keeps+/// `run.status` and `run.settled`, set in the run's `finally`, apart). A+/// barrier sent once the roster says `done` must still hold for the+/// terminal notice's parent turn; a status-reading barrier answers+/// inside the settle window.+#[test]+fn headless_completion_holds_through_the_settle_window() {+ let Some(kernel_python) = kernel_python() else {+ return;+ };+ let mut lane = spawn_held_child_lane(&kernel_python);++ // The roster's terminal status is the settle window's start: the+ // roster says `done` here while the child's run has not settled yet.+ let deadline = Instant::now() + Duration::from_secs(30);+ loop {+ let rows = rlm_children_rows(&mut lane.client, "g1", &lane.parent_id);+ let row = rows+ .iter()+ .find(|row| row["id"] == json!(lane.child_id))+ .expect("the spawned child's roster row");+ if row["status"] != "running" {+ break;+ }+ assert!(+ Instant::now() < deadline,+ "the child never reached a terminal roster status"+ );+ std::thread::sleep(Duration::from_millis(5));+ }+ lane.client.send_command(+ "w1",+ &json!({+ "type": "wait_for_headless_completion",+ "activeSessionId": lane.parent_id,+ "waitForRlmQuiescence": true,+ }),+ );+ let completion = lane.client.read_response("w1");+ assert_eq!(+ completion["success"], true,+ "wait_for_headless_completion failed: {completion}"+ );+ let messages = messages_text(&mut lane.client, "m1", &lane.parent_id);+ assert!(+ messages.contains("notice seen"),+ "the barrier answered inside the settle window, before the terminal-notice turn drained"+ );+}