Skip to content

A self-improving RLM agent for coding workflows and long-running autonomous tasks.

Rust

carry childscript to spawned children and wait for rlm quiescence (#3171)

Sebastian Müller45e8f4970fc7
diff --git a/crates/pa-daemon/src/engine/scripted.rs b/crates/pa-daemon/src/engine/scripted.rs
index 1cf15a8586..b6c0af4f79 100644
--- a/crates/pa-daemon/src/engine/scripted.rs
+++ b/crates/pa-daemon/src/engine/scripted.rs
@@ -273,16 +273,20 @@ impl SessionEngine for ScriptedEngine {
&self,
prompt_index: usize,
request: PromptRequest,
- _aborted: &dyn Fn() -> bool,
+ aborted: &dyn Fn() -> bool,
emit: &mut dyn FnMut(EngineEvent) -> bool,
) {
let cancelled = || EngineEvent::Done(Err("prompt cancelled".to_string()));
let scripted = self.responses.get(prompt_index).cloned();
let text = match &scripted {
Some(response) => {
- let delay = Self::response_delay_ms(response);
- if delay > 0 {
- std::thread::sleep(std::time::Duration::from_millis(delay));
+ // The scripted hold honors the worker's cancel probe: a
+ // close on a held scripted child settles at the abort,
+ // not at the hold's end.
+ let delay = std::time::Duration::from_millis(Self::response_delay_ms(response));
+ if !abortable_sleep(delay, aborted) {
+ emit(cancelled());
+ return;
}
Self::response_text(response)
}
@@ -516,7 +520,7 @@ impl SessionEngine for ScriptedEngine {
Some(&signal),
move |delay| {
let wait_signal = wait_signal.clone();
- async move { abortable_sleep(delay, &wait_signal) }
+ async move { abortable_sleep(delay, &|| wait_signal.is_aborted()) }
},
move || {
let responses = StdArc::clone(&responses);
@@ -609,7 +613,11 @@ impl SessionEngine for ScriptedEngine {
};
}
let delay_ms = Self::response_delay_ms(entry);
- if delay_ms > 0 && !abortable_sleep(std::time::Duration::from_millis(delay_ms), signal) {
+ if delay_ms > 0
+ && !abortable_sleep(std::time::Duration::from_millis(delay_ms), &|| {
+ signal.is_aborted()
+ })
+ {
return CompactionOutcome::Aborted;
}
if signal.is_aborted() {
@@ -674,7 +682,11 @@ impl SessionEngine for ScriptedEngine {
};
}
let delay_ms = Self::response_delay_ms(entry);
- if delay_ms > 0 && !abortable_sleep(std::time::Duration::from_millis(delay_ms), signal) {
+ if delay_ms > 0
+ && !abortable_sleep(std::time::Duration::from_millis(delay_ms), &|| {
+ signal.is_aborted()
+ })
+ {
return BranchSummaryOutcome::Aborted;
}
if signal.is_aborted() {
@@ -770,7 +782,11 @@ fn scripted_side_question_turn(
return message;
}
let delay_ms = ScriptedEngine::response_delay_ms(entry);
- if delay_ms > 0 && !abortable_sleep(std::time::Duration::from_millis(delay_ms), signal) {
+ if delay_ms > 0
+ && !abortable_sleep(std::time::Duration::from_millis(delay_ms), &|| {
+ signal.is_aborted()
+ })
+ {
set_text(&mut message, &text);
message.stop_reason = StopReason::Aborted;
return message;
@@ -784,17 +800,17 @@ fn scripted_side_question_turn(
message
}
-/// Sleep `delay` in slices, stopping early when `signal` aborts.
+/// Sleep `delay` in slices, stopping early when `aborted` turns true.
/// Returns `false` when the wait ended aborted.
-fn abortable_sleep(delay: std::time::Duration, signal: &AbortSignal) -> bool {
+fn abortable_sleep(delay: std::time::Duration, aborted: &dyn Fn() -> bool) -> bool {
let mut remaining = delay;
while !remaining.is_zero() {
- if signal.is_aborted() {
+ if aborted() {
return false;
}
let slice = remaining.min(std::time::Duration::from_millis(25));
std::thread::sleep(slice);
remaining = remaining.saturating_sub(slice);
}
- !signal.is_aborted()
+ !aborted()
}
diff --git a/crates/pa-daemon/src/rlm_children.rs b/crates/pa-daemon/src/rlm_children.rs
index 73b6e6de66..deef81431e 100644
--- a/crates/pa-daemon/src/rlm_children.rs
+++ b/crates/pa-daemon/src/rlm_children.rs
@@ -163,6 +163,11 @@ struct ChildRecord {
/// Terminal state (`done` | `error` | `cancelled`); running while
/// absent.
settled_status: Option<&'static str>,
+ /// TS `run.settled`: set only by the settle funnel, after the
+ /// terminal notice is delivered - the terminal status above flips
+ /// earlier (TS's `run.status`/`run.settled` pair), so the quiescence
+ /// predicate waits out the notice window.
+ settled: bool,
answer_preview: Option<String>,
answer_captured: bool,
/// An agent message from this child reached the parent since its task
@@ -328,6 +333,12 @@ struct SupervisorChildSessionsInner {
/// The parent engine's child-settle hook (goal continuation resume);
/// `None` until the engine wires it.
settle_hook: std::sync::Mutex<Option<Arc<dyn Fn() + Send + Sync>>>,
+ /// The quiescence barrier's wake (TS `waitForRlmQuiescence`,
+ /// agent-session.ts): `notify_waiters` fires once per settled child
+ /// run - every settle site funnels through the settle hook below -
+ /// and once per close walk, so a barrier parked behind descendant
+ /// work re-reads the registry when the descendants settle.
+ settle_notify: tokio::sync::Notify,
/// The worker's model-allowlist refusal telemetry (`model refused`):
/// `spawn/create_session` refusals emit through the engine's shared
/// lazily-built client.
@@ -371,6 +382,7 @@ impl SupervisorChildSessions {
deleted_children: std::sync::Mutex::new(std::collections::HashMap::new()),
turn_done: tokio::sync::watch::Sender::new(0),
settle_hook: std::sync::Mutex::new(None),
+ settle_notify: tokio::sync::Notify::new(),
model_refusal_telemetry,
usage_sink: std::sync::Mutex::new(None),
delete_notifier: std::sync::Mutex::new(None),
@@ -440,13 +452,20 @@ impl SupervisorChildSessions {
.contains(name)
}
+ /// The barrier's wake permit (the `settle_notify` field owns the
+ /// semantics).
+ pub(crate) fn settle_notified(&self) -> tokio::sync::futures::Notified<'_> {
+ self.inner.settle_notify.notified()
+ }
+
/// Whether any tracked child run is still unsettled (TS
- /// `_hasUnsettledRlmQuiescenceWork`'s child-run arm: a record without
- /// a terminal state).
+ /// `_hasUnsettledRlmQuiescenceWork`'s child-run arm: a run whose
+ /// settle funnel has not fired - the terminal status flips before
+ /// the terminal notice is delivered, `run.settled` after).
pub async fn any_running(&self) -> bool {
let children = self.inner.children.lock().await;
for record in children.iter() {
- if record.lock().await.settled_status.is_none() {
+ if !record.lock().await.settled {
return true;
}
}
@@ -637,6 +656,7 @@ impl SupervisorChildSessions {
label: String::new(),
started_at_ms: 0,
settled_status: None,
+ settled: false,
answer_preview: None,
answer_captured: false,
replied_since_task: false,
@@ -767,8 +787,13 @@ impl SupervisorChildSessions {
impl SupervisorChildSessionsInner {
/// Fire the settle hook off-thread (the settle sites run inside
- /// watcher tasks; the hook owns its own scheduling).
- pub(crate) fn fire_settle_hook(&self) {
+ /// watcher tasks; the hook owns its own scheduling). The funnel is
+ /// the one definition of a settled run (TS's run task `finally`): it
+ /// marks the record settled and wakes the quiescence barrier, only
+ /// after the terminal notice is delivered.
+ pub(crate) async fn fire_settle_hook(&self, record: &Arc<Mutex<ChildRecord>>) {
+ record.lock().await.settled = true;
+ self.settle_notify.notify_waiters();
let hook = self
.settle_hook
.lock()
diff --git a/crates/pa-daemon/src/rlm_children/host.rs b/crates/pa-daemon/src/rlm_children/host.rs
index 6ff5cd0014..73dc4b4056 100644
--- a/crates/pa-daemon/src/rlm_children/host.rs
+++ b/crates/pa-daemon/src/rlm_children/host.rs
@@ -136,6 +136,7 @@ impl RlmSubagentHost for SupervisorChildSessions {
label: rlm_child_label(&request.prompt),
started_at_ms: now_ms(),
settled_status: None,
+ settled: false,
answer_preview: None,
answer_captured: false,
replied_since_task: false,
@@ -215,6 +216,7 @@ impl RlmSubagentHost for SupervisorChildSessions {
.kill_child(&child_active_session_id, ChildCloseReason::Killed)
.await;
watcher_record.lock().await.settled_status = Some("error");
+ watcher_this.fire_settle_hook(&watcher_record).await;
return;
}
}
@@ -405,6 +407,10 @@ impl RlmSubagentHost for SupervisorChildSessions {
this.deliver_terminal_notice(&notice).await;
}
}
+ // The deletion settles the run (TS `_finishRlmRunDeletion`,
+ // the same resume site the inactive delete funnels through):
+ // a parked barrier re-reads a removed record as settled.
+ this.fire_settle_hook(&record).await;
Ok(RlmDeleteSubagentResult {
subagent: entry,
outcome: Some("deleted"),
diff --git a/crates/pa-daemon/src/rlm_children/lifecycle.rs b/crates/pa-daemon/src/rlm_children/lifecycle.rs
index 6409906100..25f9f85516 100644
--- a/crates/pa-daemon/src/rlm_children/lifecycle.rs
+++ b/crates/pa-daemon/src/rlm_children/lifecycle.rs
@@ -433,7 +433,7 @@ impl SupervisorChildSessionsInner {
// A settled child releases an owed goal continuation (TS
// `_maybeResumeGoalContinuationAfterRlmWork` at the child
// settle sites).
- self.fire_settle_hook();
+ self.fire_settle_hook(record).await;
return;
}
// Still running (a timed-out slice or a re-queued continuation):
@@ -456,7 +456,7 @@ impl SupervisorChildSessionsInner {
// holds; capture them before the terminal notice.
self.emit_child_usage(record).await;
self.deliver_settle_notice(record).await;
- self.fire_settle_hook();
+ self.fire_settle_hook(record).await;
return;
}
}
diff --git a/crates/pa-daemon/src/rlm_children/registry.rs b/crates/pa-daemon/src/rlm_children/registry.rs
index fb427b27c4..f3826ae763 100644
--- a/crates/pa-daemon/src/rlm_children/registry.rs
+++ b/crates/pa-daemon/src/rlm_children/registry.rs
@@ -97,7 +97,7 @@ impl SupervisorChildSessionsInner {
.with_context(|| format!("abort RLM child session {active_session_id}"));
// The settled/cancelled child releases an owed goal
// continuation.
- self.fire_settle_hook();
+ self.fire_settle_hook(record).await;
return true;
}
false
@@ -168,7 +168,7 @@ impl SupervisorChildSessionsInner {
.retain(|candidate| !Arc::ptr_eq(candidate, record));
// The deleted child is a TS resume site for the owed goal
// continuation (`_finishRlmRunDeletion`).
- self.fire_settle_hook();
+ self.fire_settle_hook(record).await;
return Ok("deleted");
}
Ok("not_found")
@@ -227,6 +227,9 @@ impl SupervisorChildSessionsInner {
.await
.retain(|candidate| !Arc::ptr_eq(candidate, record));
}
+ // The walk changed the registry: wake a parked barrier (a closed
+ // child is settled work, settled here by its removal).
+ self.settle_notify.notify_waiters();
match close_error {
Some(error) => Err(error),
None => Ok(()),
diff --git a/crates/pa-daemon/src/rlm_children/usage_emit_tests.rs b/crates/pa-daemon/src/rlm_children/usage_emit_tests.rs
index b9241e6ec5..9572f0cd44 100644
--- a/crates/pa-daemon/src/rlm_children/usage_emit_tests.rs
+++ b/crates/pa-daemon/src/rlm_children/usage_emit_tests.rs
@@ -36,6 +36,7 @@ fn record_with_file(child_id: &str, session_file: &Path) -> Arc<Mutex<ChildRecor
label: "child".to_string(),
started_at_ms: 0,
settled_status: None,
+ settled: false,
answer_preview: None,
answer_captured: false,
replied_since_task: false,
diff --git a/crates/pa-daemon/src/rlm_children/watch_tests.rs b/crates/pa-daemon/src/rlm_children/watch_tests.rs
index 2194f9fc62..5b46604289 100644
--- a/crates/pa-daemon/src/rlm_children/watch_tests.rs
+++ b/crates/pa-daemon/src/rlm_children/watch_tests.rs
@@ -495,6 +495,7 @@ fn an_already_settled_child_never_re_scores_as_an_unreachable_error() {
label: "task".to_string(),
started_at_ms: 0,
settled_status: None,
+ settled: false,
answer_preview: None,
answer_captured: false,
replied_since_task: false,
diff --git a/crates/pa-daemon/src/supervisor/worker_lifecycle.rs b/crates/pa-daemon/src/supervisor/worker_lifecycle.rs
index 8526610961..e9c62c5030 100644
--- a/crates/pa-daemon/src/supervisor/worker_lifecycle.rs
+++ b/crates/pa-daemon/src/supervisor/worker_lifecycle.rs
@@ -239,6 +239,9 @@ impl Supervisor {
"rlmMaxDepth",
"parentSessionPath",
"models",
+ // The scripted-parent verification seam: a dropped key leaves
+ // spawned children scriptless.
+ "childScript",
"systemPrompt",
"appendSystemPrompt",
"skills",
diff --git a/crates/pa-daemon/src/worker/commands.rs b/crates/pa-daemon/src/worker/commands.rs
index e466dd0418..2b33edb461 100644
--- a/crates/pa-daemon/src/worker/commands.rs
+++ b/crates/pa-daemon/src/worker/commands.rs
@@ -90,7 +90,9 @@ impl Worker {
}
"set_auto_compaction" => self.handle_set_auto_compaction(payload),
"wait_for_idle" => self.handle_wait_for_idle().await,
- "wait_for_headless_completion" => self.handle_wait_for_headless_completion().await,
+ "wait_for_headless_completion" => {
+ self.handle_wait_for_headless_completion(payload).await
+ }
"get_state" => self.handle_get_state(),
"get_messages" => self.handle_get_messages(),
"get_session_header" => self.handle_get_session_header(),
diff --git a/crates/pa-daemon/src/worker/lifecycle.rs b/crates/pa-daemon/src/worker/lifecycle.rs
index 3e71cad9e6..05a3d78a62 100644
--- a/crates/pa-daemon/src/worker/lifecycle.rs
+++ b/crates/pa-daemon/src/worker/lifecycle.rs
@@ -631,33 +631,79 @@ impl Worker {
}
}
- pub(crate) async fn handle_wait_for_idle(&self) -> DaemonResponse {
+ /// The idle park shared by `wait_for_idle` and the headless barrier:
+ /// register the permit before the flag check, or a turn that settles
+ /// between the check and the await loses its wake
+ /// (`notify_waiters` only reaches registered futures).
+ async fn wait_until_idle(&self) {
loop {
+ let idle = self.idle_notify.notified();
{
let core = self.core.lock().unwrap();
if !core.busy && core.steering.is_empty() && core.follow_up.is_empty() {
- return response_success(None, "wait_for_idle", None);
+ return;
}
}
- self.idle_notify.notified().await;
+ idle.await;
}
}
+ pub(crate) async fn handle_wait_for_idle(&self) -> DaemonResponse {
+ self.wait_until_idle().await;
+ response_success(None, "wait_for_idle", None)
+ }
+
/// `wait_for_headless_completion` (TS daemon command): settle the
/// headless run first (same idle wait as `wait_for_idle`), then answer
/// the autonomous-run accounting snapshot (`DaemonAutonomousStatus`).
- pub(crate) async fn handle_wait_for_headless_completion(&self) -> DaemonResponse {
+ /// `waitForRlmQuiescence` (TS `waitForHeadlessCompletion`'s strong arm
+ /// over `agent-session.ts` `waitForRlmQuiescence`): the barrier also
+ /// owns descendant work - it holds past the parent's idle until every
+ /// tracked child run's settle funnel fires (after the terminal notice
+ /// is delivered), and the settle loop re-runs the idle wait, so a
+ /// settled child's terminal notice (a queued parent turn) drains
+ /// inside the barrier exactly like TS's "work may start at the
+ /// child-settlement boundary" re-check.
+ pub(crate) async fn handle_wait_for_headless_completion(
+ &self,
+ payload: &Value,
+ ) -> DaemonResponse {
if let Err(response) = self.require_created("wait_for_headless_completion") {
return response;
}
+ let children = if payload.get("waitForRlmQuiescence").and_then(Value::as_bool) == Some(true)
+ {
+ self.agent_engine
+ .as_ref()
+ .and_then(|engine| engine.children.clone())
+ } else {
+ None
+ };
loop {
- {
- let core = self.core.lock().unwrap();
- if !core.busy && core.steering.is_empty() && core.follow_up.is_empty() {
+ self.wait_until_idle().await;
+ let Some(children) = children.as_ref() else {
+ break;
+ };
+ // Register the permit before the unsettled-work read: a run
+ // that settles between the read and the await still wakes
+ // this waiter.
+ let settled = children.settle_notified();
+ if !children.any_running().await {
+ // A settle funnel queues its terminal-notice follow-up
+ // BEFORE it marks the run settled, so every notice owed by
+ // the runs settled at this read is already queued: one
+ // more idle wait drains them before the barrier answers.
+ self.wait_until_idle().await;
+ // A notice can start new child work during that drain (a
+ // child-settle hook spawning a descendant): re-read the
+ // runs before answering, so the barrier holds for the
+ // new work too instead of completing at the boundary.
+ if !children.any_running().await {
break;
}
+ continue;
}
- self.idle_notify.notified().await;
+ settled.await;
}
// The idle wait finished, so no turn holds the accounting state;
// the snapshot read cannot interleave with a running turn.
diff --git a/crates/pa-daemon/tests/rlm_quiescence_barrier_e2e.rs b/crates/pa-daemon/tests/rlm_quiescence_barrier_e2e.rs
new file mode 100644
index 0000000000..6e32d3a474
--- /dev/null
+++ b/crates/pa-daemon/tests/rlm_quiescence_barrier_e2e.rs
@@ -0,0 +1,489 @@
+//! The worker's RLM quiescence barrier e2e (`wait_for_headless_completion`
+//! with `waitForRlmQuiescence`, the `rlm_quiescence_barrier` capability the
+//! worker advertises).
+//!
+//! TS ground truth - `packages/coding-agent/src/modes/headless-completion.ts`
+//! settles `waitForHeadlessCompletion({ waitForRlmQuiescence: true })` through
+//! `core/agent-session.ts` `waitForRlmQuiescence()`, so the barrier owns the
+//! parent's own idle FIRST, then every admitted descendant run's settlement,
+//! and it loops back through the idle wait because work may start at the
+//! child-settlement boundary (a settled child's terminal notice queues a
+//! parent turn). A barrier without the flag - and the pre-fix worker, which
+//! ignored the flag - answers at the parent's idle while a child run is
+//! still in flight.
+//!
+//! Verified end to end against a real supervisor, a real parent worker
+//! session whose kernel cell spawns the child through the product
+//! `rlm.spawn` surface, and a scripted child worker held mid-run.
+//!
+//! 1. `wait_for_headless_completion` with `waitForRlmQuiescence: true`
+//! answers only after the held child turn settles and the terminal
+//! notice it queues on the parent drains: the parent's
+//! `get_rlm_children` reports the child terminal and its messages
+//! carry the notice turn at answer time. The pre-fix worker answered
+//! at the parent's idle (elapsed milliseconds) with the child still
+//! running.
+//! 2. The barrier reads the run's settlement, not the roster status:
+//! the roster flips terminal inside the watcher's settle grace,
+//! ~250ms before the run's settle funnel, so a barrier sent once the
+//! roster says `done` still holds for the terminal notice's parent
+//! turn. A status-reading barrier answers inside that window.
+//!
+//! The parent's kernel Python is ambient product state; like the other
+//! live-kernel verifiers these tests skip (with a note) on machines
+//! without a live install.
+// Pedantic-gate dispositions (fleet-uniform ruling; see this lane's PR for
+// the full rationale): the timeout panic path cannot wait on the supervisor
+// child; the test process exits immediately afterwards, reaping it.
+#![allow(clippy::zombie_processes)]
+#![cfg(unix)]
+
+use std::io::{BufRead, BufReader, Write};
+use std::os::unix::net::UnixStream;
+use std::path::{Path, PathBuf};
+use std::process::{Child, Command, Stdio};
+use std::time::{Duration, Instant};
+
+use serde_json::{json, Value};
+
+struct Daemon {
+ child: Child,
+ #[allow(dead_code)]
+ socket: PathBuf,
+}
+
+impl Drop for Daemon {
+ fn drop(&mut self) {
+ let _ = self.child.kill();
+ let _ = self.child.wait();
+ }
+}
+
+fn spawn_supervisor(socket: &Path, agent_dir: &Path, kernel_python: &Path) -> Daemon {
+ let binary = env!("CARGO_BIN_EXE_pa-daemon");
+ let child = Command::new(binary)
+ .arg("supervisor")
+ .arg("--socket")
+ .arg(socket)
+ .arg("--agent-dir")
+ .arg(agent_dir)
+ .env("PRIME_AGENT_KERNEL_PYTHON", kernel_python)
+ // Hermetic agent dir: the ambient environment exports a real
+ // agent dir; point every fallback at the test sandbox instead.
+ .env("PRIME_AGENT_CODING_AGENT_DIR", agent_dir)
+ .env_remove("PRIME_API_KEY")
+ .stdout(Stdio::null())
+ .stderr(Stdio::null())
+ // A supervisor killed at teardown must not leak its session workers
+ // into later test binaries: the worker's supervisor-lost exit (TS
+ // `exitIfSupervisorOrphanedForTooLong`) runs on this short window
+ // instead of the 5-minute default.
+ .env(
+ pa_daemon::worker::WORKER_SUPERVISOR_LOST_EXIT_MS_ENV,
+ "15000",
+ )
+ .spawn()
+ .expect("spawn pa-daemon supervisor");
+ Daemon {
+ child,
+ socket: socket.to_path_buf(),
+ }
+}
+
+/// The kernel Python with prime-agent-runtime installed; set
+/// `PA_E2E_KERNEL_PYTHON` to point at an explicit interpreter instead.
+fn kernel_python() -> Option<PathBuf> {
+ if let Some(explicit) = std::env::var_os("PA_E2E_KERNEL_PYTHON") {
+ let explicit = PathBuf::from(explicit);
+ assert!(
+ explicit.exists(),
+ "PA_E2E_KERNEL_PYTHON {} not found",
+ explicit.display()
+ );
+ return Some(explicit);
+ }
+ let candidate = PathBuf::from(std::env::var("HOME").map_or_else(
+ |_| "/home/ubuntu/.prime/agent/kernel-venv/bin/python".to_string(),
+ |home| format!("{home}/.prime/agent/kernel-venv/bin/python"),
+ ));
+ if candidate.exists() {
+ return Some(candidate);
+ }
+ eprintln!(
+ "kernel python {} not found; skipping live RLM quiescence e2e",
+ candidate.display()
+ );
+ None
+}
+
+fn wait_socket_ready(socket: &Path) {
+ let deadline = Instant::now() + Duration::from_secs(15);
+ loop {
+ if UnixStream::connect(socket).is_ok() {
+ return;
+ }
+ assert!(Instant::now() < deadline, "supervisor socket never came up");
+ std::thread::sleep(Duration::from_millis(20));
+ }
+}
+
+/// JSONL supervisor client (command envelopes, id-matched responses).
+struct Client {
+ reader: BufReader<UnixStream>,
+ writer: UnixStream,
+}
+
+impl Client {
+ fn connect(socket: &Path) -> (Self, Value) {
+ let stream = UnixStream::connect(socket).expect("connect supervisor");
+ let write_stream = stream.try_clone().expect("clone socket");
+ let mut client = Client {
+ reader: BufReader::new(stream),
+ writer: write_stream,
+ };
+ let hello = client.read_line();
+ (client, hello)
+ }
+
+ fn send_command(&mut self, id: &str, command: &Value) {
+ let envelope = json!({
+ "type": "command",
+ "id": id,
+ "protocol": { "name": "prime-agent.daemon", "version": 7 },
+ "command": command,
+ });
+ let mut line = serde_json::to_string(&envelope).expect("serialize command");
+ line.push('\n');
+ self.writer.write_all(line.as_bytes()).expect("send");
+ self.writer.flush().expect("flush");
+ }
+
+ fn read_line(&mut self) -> Value {
+ let mut line = String::new();
+ let deadline = Instant::now() + Duration::from_secs(120);
+ self.reader
+ .get_mut()
+ .set_read_timeout(Some(Duration::from_millis(100)))
+ .expect("set timeout");
+ loop {
+ line.clear();
+ match self.reader.read_line(&mut line) {
+ Ok(0) => panic!("supervisor closed the connection"),
+ Ok(_) if line.trim().is_empty() => {}
+ Ok(_) => return serde_json::from_str(line.trim()).expect("parse response line"),
+ Err(error) => {
+ assert!(
+ Instant::now() < deadline,
+ "timed out waiting for supervisor line: {error}"
+ );
+ }
+ }
+ }
+ }
+
+ fn read_response(&mut self, id: &str) -> Value {
+ let deadline = Instant::now() + Duration::from_secs(240);
+ loop {
+ assert!(Instant::now() < deadline, "no response for id {id}");
+ let line = self.read_line();
+ if line.get("id").and_then(Value::as_str) == Some(id) {
+ return line;
+ }
+ }
+ }
+}
+
+/// The parent's tracked-children wire surface (`get_rlm_children`).
+fn rlm_children_rows(client: &mut Client, id: &str, parent: &str) -> Vec<Value> {
+ client.send_command(
+ id,
+ &json!({ "type": "get_rlm_children", "activeSessionId": parent }),
+ );
+ let response = client.read_response(id);
+ assert_eq!(
+ response["success"], true,
+ "get_rlm_children failed: {response}"
+ );
+ response["data"]["children"]
+ .as_array()
+ .cloned()
+ .expect("children array")
+}
+
+/// Run one turn (prompt + idle wait) on the session.
+fn run_turn(client: &mut Client, session_id: &str, message: &str, id: &str) {
+ client.send_command(
+ id,
+ &json!({ "type": "prompt", "activeSessionId": session_id, "message": message }),
+ );
+ let prompted = client.read_response(id);
+ assert_eq!(prompted["success"], true, "prompt failed: {prompted}");
+ let idle_id = format!("{id}-idle");
+ client.send_command(
+ &idle_id,
+ &json!({ "type": "wait_for_idle", "activeSessionId": session_id }),
+ );
+ let idle = client.read_response(&idle_id);
+ assert_eq!(idle["success"], true, "wait_for_idle failed: {idle}");
+}
+
+/// Poll until a kernel cell's receipt content appears (the cell writes its
+/// verdict).
+fn await_receipt(receipt: &Path) -> String {
+ let deadline = Instant::now() + Duration::from_secs(60);
+ loop {
+ if let Ok(content) = std::fs::read_to_string(receipt) {
+ return content;
+ }
+ assert!(
+ Instant::now() < deadline,
+ "kernel cell receipt never appeared at {}",
+ receipt.display()
+ );
+ std::thread::sleep(Duration::from_millis(50));
+ }
+}
+
+/// The kernel cell of the spawn turn: spawn one RLM child through the
+/// product `rlm.spawn` surface and record its child id.
+fn spawn_cell(receipt: &Path, error_receipt: &Path) -> String {
+ format!(
+ "import json, traceback\ntry:\n handle = await rlm.spawn(\"run the lane task\", name=\"kid\")\n open({receipt:?}, \"w\").write(json.dumps({{\"rlm_child_id\": handle.rlm_child_id}}))\n print(handle.rlm_child_id)\nexcept Exception:\n open({error_receipt:?}, \"w\").write(traceback.format_exc())\n raise",
+ receipt = receipt.display().to_string(),
+ error_receipt = error_receipt.display().to_string(),
+ )
+}
+
+/// The child's scripted engine: one held response keeps its task turn
+/// running past the parent's idle, so the barrier must own it.
+fn write_child_script(dir: &Path) -> PathBuf {
+ let script = dir.join("child.json");
+ std::fs::write(
+ &script,
+ json!({ "responses": [ { "text": "kid done", "delayMs": 5_000 } ] }).to_string(),
+ )
+ .expect("write child script");
+ script
+}
+
+/// The parent's faux script whose turns run the spawn cell; the third
+/// response answers the settled child's terminal-notice turn (the no-reply
+/// notice the watcher queues on the parent).
+fn write_parent_script(dir: &Path, first_cell: &str) -> PathBuf {
+ let script = dir.join("parent.json");
+ std::fs::write(
+ &script,
+ json!({
+ "engine": "faux",
+ "responses": [
+ { "content": [
+ { "type": "toolCall", "name": "ipython", "arguments": { "code": first_cell } },
+ ] },
+ { "text": "spawn turn done" },
+ { "text": "notice seen" },
+ ],
+ })
+ .to_string(),
+ )
+ .expect("write parent script");
+ script
+}
+
+/// Create a scripted parent session through the supervisor. The create's
+/// `childScript` (the harness seam mirroring the TS child runtime's
+/// inherited `sessionConfig`) makes every `rlm.spawn` child a scripted
+/// worker.
+fn create_parent(
+ client: &mut Client,
+ dir: &Path,
+ parent_script: &Path,
+ child_script: &Path,
+ id: &str,
+) -> Value {
+ let sessions_dir = dir.join("sessions");
+ std::fs::create_dir_all(&sessions_dir).expect("sessions dir");
+ client.send_command(
+ id,
+ &json!({
+ "type": "create",
+ "name": "parent",
+ "config": {
+ "cwd": dir.to_string_lossy(),
+ "sessionDir": sessions_dir.to_string_lossy(),
+ "script": parent_script.to_string_lossy(),
+ "childScript": child_script.to_string_lossy(),
+ },
+ }),
+ );
+ let created = client.read_response(id);
+ assert_eq!(created["success"], true, "create parent failed: {created}");
+ created["data"].clone()
+}
+
+/// The parent's messages as one JSON string (the barrier's answer-time
+/// transcript read).
+fn messages_text(client: &mut Client, id: &str, parent_id: &str) -> String {
+ client.send_command(
+ id,
+ &json!({ "type": "get_messages", "activeSessionId": parent_id }),
+ );
+ client.read_response(id)["data"].to_string()
+}
+
+/// One scripted lane with a held scripted child (the shared setup of the
+/// barrier tests): the parent went idle with the child's 5s held turn in
+/// flight, ready for the barrier.
+struct HeldChildLane {
+ _daemon: Daemon,
+ _dir: tempfile::TempDir,
+ client: Client,
+ parent_id: String,
+ child_id: String,
+}
+
+fn spawn_held_child_lane(kernel_python: &Path) -> HeldChildLane {
+ let dir = tempfile::TempDir::new().expect("temp dir");
+ let agent_dir = dir.path().join("agent");
+ std::fs::create_dir_all(&agent_dir).expect("agent dir");
+ let socket = dir.path().join("supervisor.sock");
+ let receipts = dir.path().join("receipts");
+ std::fs::create_dir_all(&receipts).expect("receipts dir");
+ let spawn_receipt = receipts.join("spawn.json");
+ let spawn_error = receipts.join("spawn.error");
+
+ let child_script = write_child_script(dir.path());
+ let parent_script = write_parent_script(dir.path(), &spawn_cell(&spawn_receipt, &spawn_error));
+ let daemon = spawn_supervisor(&socket, &agent_dir, kernel_python);
+ wait_socket_ready(&socket);
+ let (mut client, hello) = Client::connect(&socket);
+ assert_eq!(hello["type"], "daemon_hello");
+ let parent = create_parent(&mut client, dir.path(), &parent_script, &child_script, "c1");
+ let parent_id = parent["activeSessionId"]
+ .as_str()
+ .expect("parent active session id")
+ .to_string();
+
+ // Turn 1: the kernel cell spawns the child through the parent's own
+ // registry; the parent settles while the child's held turn runs on.
+ run_turn(&mut client, &parent_id, "spawn the kid", "t1");
+ let spawned: Value =
+ serde_json::from_str(&await_receipt(&spawn_receipt)).expect("spawn receipt json");
+ let child_id = spawned["rlm_child_id"]
+ .as_str()
+ .expect("child id")
+ .to_string();
+ assert!(
+ !spawn_error.exists(),
+ "the spawn cell failed: {}",
+ std::fs::read_to_string(&spawn_error).unwrap_or_default()
+ );
+ HeldChildLane {
+ _daemon: daemon,
+ _dir: dir,
+ client,
+ parent_id,
+ child_id,
+ }
+}
+
+/// `wait_for_headless_completion` with `waitForRlmQuiescence: true` owns
+/// descendant work (TS `waitForRlmQuiescence`): the parent goes idle while
+/// the spawned child's 5s held turn still runs, and the barrier must
+/// answer only after that turn settles and the terminal notice it queues
+/// on the parent drains - the registry reports the child terminal, and
+/// the parent's messages carry the notice turn at answer time. The
+/// pre-fix worker ignored the flag and answered at the parent's idle,
+/// with the child still running.
+#[test]
+fn headless_completion_waits_for_rlm_quiescence() {
+ let Some(kernel_python) = kernel_python() else {
+ return;
+ };
+ let mut lane = spawn_held_child_lane(&kernel_python);
+
+ // The barrier request on the idle parent: the child's 5s hold is
+ // unsettled descendant work, so the answer must hold past it.
+ lane.client.send_command(
+ "w1",
+ &json!({
+ "type": "wait_for_headless_completion",
+ "activeSessionId": lane.parent_id,
+ "waitForRlmQuiescence": true,
+ }),
+ );
+ let completion = lane.client.read_response("w1");
+ assert_eq!(
+ completion["success"], true,
+ "wait_for_headless_completion failed: {completion}"
+ );
+
+ // The answer lands only after the child's held turn settled and the
+ // terminal notice's parent turn drained.
+ let messages = messages_text(&mut lane.client, "m1", &lane.parent_id);
+ assert!(
+ messages.contains("notice seen"),
+ "the barrier answered before the terminal-notice turn drained"
+ );
+ let rows = rlm_children_rows(&mut lane.client, "g1", &lane.parent_id);
+ let child_row = rows
+ .iter()
+ .find(|row| row["id"] == json!(lane.child_id))
+ .expect("the spawned child's roster row");
+ assert_eq!(
+ child_row["status"], "done",
+ "the barrier answered while the child was still running"
+ );
+}
+
+/// The barrier reads the run's settlement, not the roster status: the
+/// roster flips terminal inside the watcher's settle grace, and the
+/// record only settles at the settle funnel - after the grace re-check,
+/// the usage emit, and the terminal-notice delivery (TS keeps
+/// `run.status` and `run.settled`, set in the run's `finally`, apart). A
+/// barrier sent once the roster says `done` must still hold for the
+/// terminal notice's parent turn; a status-reading barrier answers
+/// inside the settle window.
+#[test]
+fn headless_completion_holds_through_the_settle_window() {
+ let Some(kernel_python) = kernel_python() else {
+ return;
+ };
+ let mut lane = spawn_held_child_lane(&kernel_python);
+
+ // The roster's terminal status is the settle window's start: the
+ // roster says `done` here while the child's run has not settled yet.
+ let deadline = Instant::now() + Duration::from_secs(30);
+ loop {
+ let rows = rlm_children_rows(&mut lane.client, "g1", &lane.parent_id);
+ let row = rows
+ .iter()
+ .find(|row| row["id"] == json!(lane.child_id))
+ .expect("the spawned child's roster row");
+ if row["status"] != "running" {
+ break;
+ }
+ assert!(
+ Instant::now() < deadline,
+ "the child never reached a terminal roster status"
+ );
+ std::thread::sleep(Duration::from_millis(5));
+ }
+ lane.client.send_command(
+ "w1",
+ &json!({
+ "type": "wait_for_headless_completion",
+ "activeSessionId": lane.parent_id,
+ "waitForRlmQuiescence": true,
+ }),
+ );
+ let completion = lane.client.read_response("w1");
+ assert_eq!(
+ completion["success"], true,
+ "wait_for_headless_completion failed: {completion}"
+ );
+ let messages = messages_text(&mut lane.client, "m1", &lane.parent_id);
+ assert!(
+ messages.contains("notice seen"),
+ "the barrier answered inside the settle window, before the terminal-notice turn drained"
+ );
+}