Skip to content

A self-improving RLM agent for coding workflows and long-running autonomous tasks.

Rust

factory: machine-recovery fixes for the M1-M6 triage classes (run-scoped names, binding fidelity, needs-verify, pause payloads) (#3462)

Seth Kartenb8451e352bce
diff --git a/crates/pa-core/src/session_engine/rlm_host.rs b/crates/pa-core/src/session_engine/rlm_host.rs
index 3b94f2d421..2fc3582d1a 100644
--- a/crates/pa-core/src/session_engine/rlm_host.rs
+++ b/crates/pa-core/src/session_engine/rlm_host.rs
@@ -61,7 +61,8 @@ pub struct RlmSubagentEntry {
pub session_id: Option<String>,
pub session_name: String,
pub session_dir: String,
- /// `running` | `completed` | `error`.
+ /// `running` | `completed` | `error` | `cancelled` (a cancelled run
+ /// keeps its status verbatim in the registry row).
pub status: &'static str,
#[serde(skip_serializing_if = "Option::is_none")]
pub activity: Option<RlmSubagentActivity>,
@@ -97,6 +98,9 @@ pub struct RlmSubagentActivity {
pub struct RlmDeleteSubagentResult {
pub subagent: RlmSubagentEntry,
/// `deleted` | `skipped_running`; absent when the host reports neither.
+ /// A re-delete of an already-deleted selector is idempotent and reports
+ /// `deleted` with the tombstone row (retirement never errors on the
+ /// second call).
#[serde(skip_serializing_if = "Option::is_none")]
pub outcome: Option<&'static str>,
}
@@ -114,6 +118,10 @@ pub struct RlmChildResult {
pub settled: bool,
#[serde(skip_serializing_if = "Option::is_none")]
pub answer_preview: Option<String>,
+ /// The full final-answer text (the settle-binding lane; the roster
+ /// preview stays compact), bounded by the host.
+ #[serde(skip_serializing_if = "Option::is_none")]
+ pub answer_text: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub error: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
@@ -861,6 +869,7 @@ mod tests {
status: "done",
settled: true,
answer_preview: Some("all done".into()),
+ answer_text: Some("all done".into()),
error: None,
duration_ms: Some(2_000),
tool_use_count: Some(3),
diff --git a/crates/pa-daemon/src/rlm_child_model.rs b/crates/pa-daemon/src/rlm_child_model.rs
index 2de9f06627..99f3aea9ff 100644
--- a/crates/pa-daemon/src/rlm_child_model.rs
+++ b/crates/pa-daemon/src/rlm_child_model.rs
@@ -13,6 +13,13 @@ const MODEL_ERROR_SUGGESTION_LIMIT: usize = 3;
pub const ANSWER_PREVIEW_MAX_CHARS: usize = 160;
/// Cap on the one-line task label shown in kernel rosters.
pub const LABEL_MAX_CHARS: usize = 200;
+/// Cap on the FULL final-answer text the collect envelope carries (the
+/// settle-binding data plane): the roster preview stays
+/// [`ANSWER_PREVIEW_MAX_CHARS`], while a consumer that binds outputs from
+/// the answer (the factory's output ports) needs the whole fenced JSON,
+/// so the lane is bounded instead of unbounded, never ballooning the
+/// collect reply for a runaway answer.
+pub const ANSWER_TEXT_MAX_CHARS: usize = 65_536;
const ELLIPSIS: &str = "...";
/// The model catalog the RLM surface resolves against: the same
@@ -191,7 +198,8 @@ pub fn rlm_child_label(prompt: &str) -> String {
}
/// Whitespace-collapsed text capped at `max` chars with an ellipsis.
-fn cap_text(text: &str, max: usize) -> String {
+#[must_use]
+pub fn cap_text(text: &str, max: usize) -> String {
if text.chars().count() <= max {
return text.to_string();
}
@@ -589,4 +597,23 @@ mod tests {
assert_eq!(preview.chars().count(), ANSWER_PREVIEW_MAX_CHARS);
assert!(preview.ends_with("..."));
}
+
+ #[test]
+ fn the_binding_lane_keeps_far_more_than_the_preview_cap() {
+ // The collect envelope's full-answer lane: a fenced JSON output
+ // longer than the 160-character roster preview binds whole from
+ // it, so the bound stays far above the preview cap and only a
+ // genuinely oversized payload (over ANSWER_TEXT_MAX_CHARS) sees
+ // the ellipsis.
+ let long = "y".repeat(ANSWER_TEXT_MAX_CHARS + 50);
+ let capped = cap_text(&long, ANSWER_TEXT_MAX_CHARS);
+ assert_eq!(capped.chars().count(), ANSWER_TEXT_MAX_CHARS);
+ assert!(capped.ends_with("..."));
+ let wide_json = format!(
+ "{{\"blob\": \"{}\"}}",
+ "z".repeat(ANSWER_PREVIEW_MAX_CHARS * 4)
+ );
+ let lane = cap_text(&wide_json, ANSWER_TEXT_MAX_CHARS);
+ assert_eq!(lane, wide_json, "a wide-but-bounded JSON keeps its tail");
+ }
}
diff --git a/crates/pa-daemon/src/rlm_children.rs b/crates/pa-daemon/src/rlm_children.rs
index 8bb9d11b26..89c99f8f70 100644
--- a/crates/pa-daemon/src/rlm_children.rs
+++ b/crates/pa-daemon/src/rlm_children.rs
@@ -25,7 +25,8 @@ use serde_json::{json, Value};
use tokio::sync::Mutex;
use crate::rlm_child_model::{
- assert_thinking_supported, compact_rlm_text, resolve_child_model, rlm_child_label,
+ assert_thinking_supported, cap_text, compact_rlm_text, resolve_child_model, rlm_child_label,
+ ANSWER_TEXT_MAX_CHARS,
};
use crate::supervisor_link::SupervisorLink;
use crate::util::now_ms;
@@ -143,6 +144,11 @@ struct ChildRecord {
/// notice window.
settled: bool,
answer_preview: Option<String>,
+ /// The full final-answer text (bounded by
+ /// [`ANSWER_TEXT_MAX_CHARS`]), the collect envelope's binding lane:
+ /// a consumer that extracts outputs (the factory's ports) needs the
+ /// whole fenced JSON, while the roster rows keep the compact preview.
+ answer_text: Option<String>,
answer_captured: bool,
/// A child agent message arrived since its task was admitted; the
/// no-reply terminal notice is withheld once set.
@@ -256,6 +262,10 @@ struct DeletedChild {
session_name: String,
session_dir: String,
started_at_ms: u64,
+ /// The roster status the row carried at its delete (the idempotent
+ /// re-delete receipt reports it verbatim, so a repeated retire reads
+ /// the same verdict as the first).
+ status: &'static str,
answer_preview: Option<String>,
/// The envelope's error: the child's own terminal error when one was
/// recorded, else the delete reason.
@@ -651,6 +661,7 @@ impl SupervisorChildSessions {
settled_status: None,
settled: false,
answer_preview: None,
+ answer_text: None,
answer_captured: false,
replied_since_task: false,
notice_delivered: false,
@@ -741,6 +752,7 @@ impl SupervisorChildSessions {
status: record.status(),
settled: record.settled_status.is_some(),
answer_preview: record.answer_preview.clone(),
+ answer_text: record.answer_text.clone(),
error: record.error.clone(),
duration_ms: Some(now_ms().saturating_sub(record.started_at_ms)),
tool_use_count: None,
@@ -758,6 +770,7 @@ impl SupervisorChildSessions {
status: "cancelled",
settled: true,
answer_preview: deleted.answer_preview.clone(),
+ answer_text: None,
error: Some(deleted.error.clone()),
duration_ms: Some(now_ms().saturating_sub(deleted.started_at_ms)),
tool_use_count: None,
diff --git a/crates/pa-daemon/src/rlm_children/host.rs b/crates/pa-daemon/src/rlm_children/host.rs
index 5618be236a..54679db076 100644
--- a/crates/pa-daemon/src/rlm_children/host.rs
+++ b/crates/pa-daemon/src/rlm_children/host.rs
@@ -157,6 +157,7 @@ impl RlmSubagentHost for SupervisorChildSessions {
settled_status: None,
settled: false,
answer_preview: None,
+ answer_text: None,
answer_captured: false,
replied_since_task: false,
notice_delivered: false,
@@ -375,7 +376,55 @@ impl RlmSubagentHost for SupervisorChildSessions {
Box::pin(async move {
// Selector errors surface unwrapped (the TS message is the
// product surface); only the kill below gets a delete context.
- let record = this.resolve_record(&target, "subagent").await?;
+ let record = match this.resolve_record(&target, "subagent").await {
+ Ok(record) => record,
+ Err(miss) => {
+ // Retirement is idempotent (the M5 class): a selector
+ // whose delete receipt already returned answers from
+ // the tombstone instead of erroring, so a re-delete of
+ // a settled child (an operator retire, the factory's
+ // cancel pass) never reports the slot as still held.
+ let matches = this
+ .deleted_children
+ .lock()
+ .unwrap_or_else(std::sync::PoisonError::into_inner)
+ .values()
+ .filter(|deleted| deleted.matches(&target))
+ .cloned()
+ .collect::<Vec<_>>();
+ match matches.len() {
+ 0 => return Err(miss),
+ 1 => {
+ let deleted = &matches[0];
+ return Ok(RlmDeleteSubagentResult {
+ subagent: RlmSubagentEntry {
+ rlm_child_id: deleted.rlm_child_id.clone(),
+ active_session_id: Some(deleted.active_session_id.clone()),
+ session_id: deleted.session_id.clone(),
+ session_name: deleted.session_name.clone(),
+ session_dir: deleted.session_dir.clone(),
+ status: deleted.status,
+ activity: None,
+ tool_use_count: None,
+ duration_ms: Some(
+ now_ms().saturating_sub(deleted.started_at_ms),
+ ),
+ answer_preview: deleted.answer_preview.clone(),
+ replied_since_task: None,
+ progress_note: None,
+ label: None,
+ last_activity_at: Some(deleted.started_at_ms),
+ activity_stale_ms: None,
+ },
+ outcome: Some("deleted"),
+ });
+ }
+ _ => bail!(
+ "RLM subagent selector \"{target}\" is ambiguous in the current parent session"
+ ),
+ }
+ }
+ };
let rename_lock = record.lock().await.rename_lock.clone();
let _rename_guard = rename_lock.lock().await;
let no_longer_matches = {
diff --git a/crates/pa-daemon/src/rlm_children/lifecycle.rs b/crates/pa-daemon/src/rlm_children/lifecycle.rs
index 531705922b..356371d647 100644
--- a/crates/pa-daemon/src/rlm_children/lifecycle.rs
+++ b/crates/pa-daemon/src/rlm_children/lifecycle.rs
@@ -2,13 +2,14 @@
//! prompt/kill/close routing, settle watching with its notices, and the
//! spawn-admission outbox types (`CreatedSessionIds`, `CreatedChild`).
use super::{
- anyhow, compact_rlm_text, create_rlm_child_failure_message, create_rlm_child_terminal_notice,
- json, now_ms, Arc, ChildCloseReason, ChildRecord, Context, CustomMessage, DaemonCommand,
- DaemonSessionLifecycle, Duration, Map, Mutex, ParentIdentity, Path, PromptInput, Result,
- RlmChildTerminalNotice, SupervisorChildSessionsInner, Value, CREATE_TIMEOUT_MS,
- IDLE_WAIT_GRACE_MS, KILL_TIMEOUT_MS, NOTICE_DELIVERY_TIMEOUT_MS, PROMPT_TIMEOUT_MS,
- RUNTIME_METADATA_PROMPT_MAX, STATE_TIMEOUT_MS, WATCH_MAX_UNREACHABLE_POLLS,
- WATCH_POLL_INTERVAL_MS, WATCH_SETTLE_GRACE_MS, WATCH_WAIT_SLICE_MS,
+ anyhow, cap_text, compact_rlm_text, create_rlm_child_failure_message,
+ create_rlm_child_terminal_notice, json, now_ms, Arc, ChildCloseReason, ChildRecord, Context,
+ CustomMessage, DaemonCommand, DaemonSessionLifecycle, Duration, Map, Mutex, ParentIdentity,
+ Path, PromptInput, Result, RlmChildTerminalNotice, SupervisorChildSessionsInner, Value,
+ ANSWER_TEXT_MAX_CHARS, CREATE_TIMEOUT_MS, IDLE_WAIT_GRACE_MS, KILL_TIMEOUT_MS,
+ NOTICE_DELIVERY_TIMEOUT_MS, PROMPT_TIMEOUT_MS, RUNTIME_METADATA_PROMPT_MAX, STATE_TIMEOUT_MS,
+ WATCH_MAX_UNREACHABLE_POLLS, WATCH_POLL_INTERVAL_MS, WATCH_SETTLE_GRACE_MS,
+ WATCH_WAIT_SLICE_MS,
};
/// Parsed ids of one created child session.
@@ -367,8 +368,12 @@ impl SupervisorChildSessionsInner {
> 0)
}
- /// The child's final answer text, compacted for the roster preview.
- async fn child_answer(&self, active_session_id: &str) -> Result<Option<String>> {
+ /// One `GetLastAssistantText` round trip: the raw text the roster
+ /// preview and the collect envelope's binding lane both derive from
+ /// (the preview compacts it; the binding lane caps it at
+ /// [`ANSWER_TEXT_MAX_CHARS`] instead, so the whole fenced JSON a
+ /// settle binds from survives).
+ async fn child_answer_raw(&self, active_session_id: &str) -> Result<Option<String>> {
let command = DaemonCommand::GetLastAssistantText {
id: None,
active_session_id: active_session_id.to_string(),
@@ -379,7 +384,7 @@ impl SupervisorChildSessionsInner {
.get("text")
.and_then(Value::as_str)
.filter(|text| !text.is_empty())
- .map(compact_rlm_text))
+ .map(str::to_string))
}
/// Best-effort bounded wait for one child and its descendants to go
@@ -419,18 +424,35 @@ impl SupervisorChildSessionsInner {
return;
}
// Capture the answer before taking the record lock (the capture is
- // a link round trip).
- let answer = self.child_answer(&active_session_id).await.ok().flatten();
+ // a link round trip): one fetch yields the raw text both lanes
+ // derive from — the compact preview for the roster rows and the
+ // full text for the collect envelope's binding lane.
+ let raw_answer = self
+ .child_answer_raw(&active_session_id)
+ .await
+ .ok()
+ .flatten();
+ let answer = raw_answer.as_deref().map(compact_rlm_text);
+ let answer_text = raw_answer
+ .as_deref()
+ .map(|text| cap_text(text, ANSWER_TEXT_MAX_CHARS));
let mut record = record.lock().await;
+ // The settle verdict is one-shot, the answer capture is not: a
+ // `None` first capture (the settle raced the worker's answer
+ // hand-off) froze the row empty before — the capture fill only
+ // ran while `settled_status` was unset, so a later refresh could
+ // never deliver the text and a re-collect answered the same empty
+ // envelope forever. The fill now runs on every refresh: it only
+ // writes when the fresh round trip produced a text AND the row has
+ // no answer yet, so a settled preview is still never overwritten
+ // with a later miss.
if record.settled_status.is_none() {
record.settled_status = Some("done");
- // A settled preview is never overwritten with a later miss; a
- // `None` capture (the settle raced the admission-to-run
- // hand-off) recovers on a later refresh.
- if !record.answer_captured || record.answer_preview.is_none() {
- record.answer_preview = answer;
- record.answer_captured = true;
- }
+ }
+ if (!record.answer_captured || record.answer_preview.is_none()) && answer.is_some() {
+ record.answer_preview = answer;
+ record.answer_text = answer_text;
+ record.answer_captured = true;
}
}
@@ -639,6 +661,48 @@ impl SupervisorChildSessionsInner {
error: String,
arm: FailedArm,
) -> bool {
+ // The exit capture (the M4 seam): a child that dies without
+ // reporting still said something — the live worker's last
+ // assistant text when it is still reachable, else the durable
+ // session file's last assistant row. It lands on the record only
+ // when no answer was captured, so a positive verdict's capture
+ // stands and the round trip never runs when one does; the
+ // parent-side reader (the factory's provisional answer) sees
+ // what the child last said instead of a bare failure row.
+ let live_or_file = {
+ let record = record.lock().await;
+ if record.answer_captured && record.answer_preview.is_some() {
+ None
+ } else {
+ Some((
+ record.active_session_id.clone(),
+ record.session_file.clone(),
+ ))
+ }
+ };
+ let exit_text = match live_or_file {
+ Some((active_session_id, session_file)) => {
+ let live = self
+ .child_answer_raw(&active_session_id)
+ .await
+ .ok()
+ .flatten();
+ if live.is_some() {
+ live
+ } else {
+ match session_file.filter(|path| !path.is_empty()) {
+ Some(path) => tokio::task::spawn_blocking(move || {
+ last_assistant_text_from_file(&path)
+ })
+ .await
+ .ok()
+ .flatten(),
+ None => None,
+ }
+ }
+ }
+ None => None,
+ };
let message = {
let mut record = record.lock().await;
let keep_verdict = match arm {
@@ -654,6 +718,13 @@ impl SupervisorChildSessionsInner {
}
record.settled_status = Some("error");
record.notice_delivered = true;
+ if let Some(text) = exit_text {
+ if !record.answer_captured || record.answer_preview.is_none() {
+ record.answer_preview = Some(compact_rlm_text(&text));
+ record.answer_text = Some(cap_text(&text, ANSWER_TEXT_MAX_CHARS));
+ record.answer_captured = true;
+ }
+ }
let message = create_rlm_child_failure_message(
&record.rlm_child_id,
&record.session_name,
@@ -739,3 +810,107 @@ pub(super) enum FailedArm {
pub(super) fn should_mark_unreachable_error(state: &ChildRecord) -> bool {
!(state.closed_by_parent || state.notice_delivered || state.settled_status.is_some())
}
+
+/// The child's last assistant text from its durable session file (the exit
+/// capture's fallback when the worker is already gone): the file is the
+/// record a worker replacement replays from, so its last assistant row is
+/// the child's final say even after a crash. The newest non-empty text row
+/// wins; a file with no assistant rows (the prompt arm's never-started
+/// turn) answers nothing.
+pub(super) fn last_assistant_text_from_file(session_file: &str) -> Option<String> {
+ let content = std::fs::read_to_string(session_file).ok()?;
+ for line in content.lines().rev() {
+ let Ok(pa_types::session::FileEntry::Message {
+ message: pa_types::session::AgentMessage::Assistant(assistant),
+ ..
+ }) = serde_json::from_str::<pa_types::session::FileEntry>(line.trim())
+ else {
+ continue;
+ };
+ let text = assistant
+ .content
+ .iter()
+ .filter_map(|block| match block {
+ pa_types::ai::AssistantContentBlock::Text(text) => Some(text.text.clone()),
+ _ => None,
+ })
+ .collect::<String>();
+ if !text.is_empty() {
+ return Some(text);
+ }
+ }
+ None
+}
+
+#[cfg(test)]
+mod exit_capture_tests {
+ use super::*;
+
+ /// One serialized assistant row with the given text.
+ fn assistant_row(text: &str) -> String {
+ let message = pa_types::session::AgentMessage::Assistant(pa_types::ai::AssistantMessage {
+ content: vec![pa_types::ai::AssistantContentBlock::Text(
+ pa_types::ai::TextContent {
+ text: text.to_string(),
+ text_signature: None,
+ rest: Map::default(),
+ },
+ )],
+ api: "faux".to_string(),
+ provider: "faux".to_string(),
+ model: "faux-1".to_string(),
+ response_model: None,
+ response_id: None,
+ diagnostics: None,
+ usage: pa_types::ai::Usage::default(),
+ stop_reason: pa_types::ai::StopReason::Stop,
+ stop_reason_raw: None,
+ error_message: None,
+ timestamp: 2,
+ rest: Map::default(),
+ });
+ let entry = pa_types::session::FileEntry::Message {
+ message,
+ base: pa_types::session::EntryBase {
+ id: Some("row-1".to_string()),
+ parent_id: None,
+ timestamp: None,
+ rest: Map::default(),
+ },
+ };
+ serde_json::to_string(&entry).expect("serialize the assistant row")
+ }
+
+ /// The exit capture reads the child's last say from its durable file
+ /// after the worker is gone: the newest non-empty assistant text wins.
+ #[test]
+ fn the_exit_capture_reader_takes_the_last_assistant_row() {
+ let file = std::env::temp_dir().join(format!(
+ "pa-rlm-exit-capture-{}.jsonl",
+ uuid::Uuid::new_v4().simple()
+ ));
+ let user_row = r#"{"type":"message","role":"user","content":"the task"}"#;
+ std::fs::write(
+ &file,
+ format!(
+ "{user_row}\n{}\n{}\n",
+ assistant_row("an earlier turn's say"),
+ assistant_row("the final say")
+ ),
+ )
+ .expect("write the session file");
+ let text = last_assistant_text_from_file(&file.to_string_lossy())
+ .expect("the last assistant row is readable");
+ assert_eq!(text, "the final say");
+ // A file with no assistant rows (the prompt arm's never-started
+ // turn) answers nothing.
+ let bare = std::env::temp_dir().join(format!(
+ "pa-rlm-exit-capture-bare-{}.jsonl",
+ uuid::Uuid::new_v4().simple()
+ ));
+ std::fs::write(&bare, format!("{user_row}\n")).expect("write the bare file");
+ assert_eq!(last_assistant_text_from_file(&bare.to_string_lossy()), None);
+ let _ = std::fs::remove_file(&file);
+ let _ = std::fs::remove_file(&bare);
+ }
+}
diff --git a/crates/pa-daemon/src/rlm_children/registry.rs b/crates/pa-daemon/src/rlm_children/registry.rs
index 7cdcb405b3..466b318ba7 100644
--- a/crates/pa-daemon/src/rlm_children/registry.rs
+++ b/crates/pa-daemon/src/rlm_children/registry.rs
@@ -32,6 +32,7 @@ impl SupervisorChildSessionsInner {
session_name: record.session_name.clone(),
session_dir: record.session_dir.clone(),
started_at_ms: record.started_at_ms,
+ status: record.roster_status(),
answer_preview: record.answer_preview.clone(),
error: record
.error
@@ -376,6 +377,7 @@ fn ledger_child_records(
),
settled: true,
answer_preview: None,
+ answer_text: None,
answer_captured: false,
replied_since_task: false,
notice_delivered: true,
diff --git a/crates/pa-daemon/src/rlm_children/usage_emit_tests.rs b/crates/pa-daemon/src/rlm_children/usage_emit_tests.rs
index b718fb5db9..d5bd9659a8 100644
--- a/crates/pa-daemon/src/rlm_children/usage_emit_tests.rs
+++ b/crates/pa-daemon/src/rlm_children/usage_emit_tests.rs
@@ -39,6 +39,7 @@ fn record_with_file(child_id: &str, session_file: &Path) -> Arc<Mutex<ChildRecor
settled_status: None,
settled: false,
answer_preview: None,
+ answer_text: None,
answer_captured: false,
replied_since_task: false,
notice_delivered: false,
diff --git a/crates/pa-daemon/src/rlm_children/watch_tests.rs b/crates/pa-daemon/src/rlm_children/watch_tests.rs
index d53abc6df2..ae604ebfc1 100644
--- a/crates/pa-daemon/src/rlm_children/watch_tests.rs
+++ b/crates/pa-daemon/src/rlm_children/watch_tests.rs
@@ -24,6 +24,9 @@ enum FakeChild {
Healthy,
PromptFails,
Unreachable,
+ /// The first `get_last_assistant_text` reads no text (the settle raced
+ /// the worker's answer hand-off); later reads answer it.
+ TextLate,
/// The worker leaves right after the settle answer is captured:
/// every later child read fails.
LeavesAfterSettle,
@@ -176,11 +179,20 @@ async fn spawn_fake_supervisor(
if matches!(child, FakeChild::LeavesAfterSettle) {
gone.store(true, Ordering::SeqCst);
}
- response_success(
- Some(&id),
- command_type,
- Some(json!({ "text": "the child final answer" })),
- )
+ // Every answer read counts; the late-text
+ // variant's FIRST read races the worker's
+ // answer hand-off and reads no text (a None
+ // capture), later reads answer.
+ let reads = child_subagents.answer_reads.fetch_add(1, Ordering::SeqCst);
+ if matches!(child, FakeChild::TextLate) && reads == 0 {
+ response_success(Some(&id), command_type, Some(json!({})))
+ } else {
+ response_success(
+ Some(&id),
+ command_type,
+ Some(json!({ "text": "the child final answer" })),
+ )
+ }
}
"kill" => {
let _ = kill_tx.send(command.clone());
@@ -241,11 +253,14 @@ async fn sessions_with_fake_supervisor(
/// The fake child's own subagents: whether one still runs (the child
/// reports `hasRunningSubagents`, and a `waitForRlmQuiescence` idle wait
-/// holds until it finishes), and how many such waits started.
+/// holds until it finishes), how many such waits started, and how many
+/// `get_last_assistant_text` reads the child link answered (the
+/// late-text variant scripts its first read empty off the count).
#[derive(Default)]
struct FakeChildSubagents {
running: AtomicBool,
quiescent_waits: std::sync::atomic::AtomicUsize,
+ answer_reads: AtomicU32,
}
/// [`sessions_with_fake_supervisor`] whose child reports its own running
@@ -963,15 +978,217 @@ async fn collect_answers_a_just_deleted_target_with_the_cancelled_envelope() {
missing.to_string(),
"No direct RLM child matches \"ghost\" in the current parent session"
);
- // The delete selector itself keeps its TS miss: the tombstone
- // answers collect only.
- let gone = sessions
+ // Retirement is idempotent (the M5 class): a re-delete of the same
+ // selector answers from the tombstone with the row's status at its
+ // delete, never the old "no longer resolves" miss.
+ let redeleted = sessions
.delete_subagent("f20-worker".to_string())
.await
- .expect_err("the deleted child no longer resolves for a delete");
+ .expect("the re-delete of a retired child is idempotent");
+ assert_eq!(redeleted.outcome, Some("deleted"));
+ assert_eq!(redeleted.subagent.status, "completed");
+ assert_eq!(redeleted.subagent.session_name, "f20-worker");
+ // A genuinely unknown selector keeps its miss.
+ let gone = sessions
+ .delete_subagent("ghost".to_string())
+ .await
+ .expect_err("an unknown selector still errors");
assert_eq!(
gone.to_string(),
- "No direct RLM subagent matches \"f20-worker\" in the current parent session"
+ "No direct RLM subagent matches \"ghost\" in the current parent session"
+ );
+}
+
+/// The M5 pin: a settled (terminal) child's delete releases its name slot —
+/// a same-name spawn admits again — and the delete of a settled child
+/// returns the terminal row instead of erroring.
+#[tokio::test]
+async fn a_deleted_settled_child_releases_its_name_slot_and_deletes_idempotently() {
+ let (follow_up_tx, _follow_up_rx) = mpsc::unbounded_channel();
+ let (sessions, _kill_rx) =
+ sessions_with_fake_supervisor(follow_up_tx, 0, FakeKill::Success, FakeChild::Healthy).await;
+ let handle = spawn_child(&sessions).await;
+ sessions.notify_turn_done();
+ // The child settles before the delete (a terminal-status row).
+ let deadline = Instant::now() + Duration::from_secs(10);
+ loop {
+ let entries = sessions.list_subagents().await.expect("child roster");
+ if entries.iter().any(|entry| entry.status == "completed") {
+ break;
+ }
+ assert!(
+ Instant::now() < deadline,
+ "child never settled: {entries:?}"
+ );
+ tokio::time::sleep(Duration::from_millis(50)).await;
+ }
+ let deleted = sessions
+ .delete_subagent(handle.rlm_child_id.clone())
+ .await
+ .expect("delete the settled child");
+ assert_eq!(deleted.subagent.status, "completed");
+
+ // The name slot is released: the supervisor admits a same-name spawn.
+ let respawned = spawn_child(&sessions).await;
+ assert_eq!(respawned.name, "f20-worker");
+
+ // Deleting a child whose run was cancelled (the factory's stop pass)
+ // reports the cancelled row, and a second delete of it is idempotent.
+ // The cancel lands before the turn-done notification, so the respawned
+ // child is provably still running when it is cut short.
+ assert!(
+ sessions.cancel_child_run(&respawned.rlm_child_id).await,
+ "cancel the respawned child"
+ );
+ let cancelled_delete = sessions
+ .delete_subagent(respawned.rlm_child_id.clone())
+ .await
+ .expect("delete the cancelled child");
+ assert_eq!(cancelled_delete.subagent.status, "cancelled");
+ let again = sessions
+ .delete_subagent(respawned.rlm_child_id.clone())
+ .await
+ .expect("the second delete of the cancelled child is idempotent");
+ assert_eq!(again.subagent.status, "cancelled");
+}
+
+/// The exit capture (the M4 seam): an unreachable child settles `error`
+/// through the give-up, but its last assistant text still lands on the row —
+/// the live read first, the durable session file second — so the parent-side
+/// reader (the factory's provisional answer) sees the child's final say, not
+/// a bare failure row.
+#[tokio::test(start_paused = true)]
+async fn a_failed_settle_captures_the_childs_last_assistant_text() {
+ tokio::spawn(async {
+ let mut tick = tokio::time::interval(Duration::from_secs(1));
+ loop {
+ tick.tick().await;
+ }
+ });
+ let (follow_up_tx, _follow_up_rx) = mpsc::unbounded_channel();
+ let (sessions, _kill_rx) =
+ sessions_with_fake_supervisor(follow_up_tx, 0, FakeKill::Success, FakeChild::Unreachable)
+ .await;
+ let handle = spawn_child(&sessions).await;
+ let settled = sessions.settle_notified();
+ sessions.notify_turn_done();
+ tokio::time::timeout(Duration::from_secs(3_600), settled)
+ .await
+ .expect("the unreachable give-up settles the child as failed");
+ let results = sessions
+ .collect(vec![handle.rlm_child_id.clone()], 0)
+ .await
+ .expect("collect the failed child");
+ assert_eq!(results[0].status, "error");
+ assert_eq!(
+ results[0].answer_preview.as_deref(),
+ Some("the child final answer"),
+ "the exit capture preserved the last assistant text on the failed row"
+ );
+ assert_eq!(
+ results[0].answer_text.as_deref(),
+ Some("the child final answer"),
+ "the binding lane carries the exit capture"
+ );
+}
+
+/// Review finding (PR #3462): a `None` first answer capture (the settle
+/// raced the worker's answer hand-off) FROZE the row empty — the capture fill
+/// only ran while the settle verdict was unset, so no later refresh could
+/// deliver the text and every re-collect answered the same empty envelope. The
+/// fill now runs on every refresh: it writes only when the fresh round trip
+/// produced a text AND the row has no answer yet.
+#[tokio::test]
+async fn a_none_first_answer_capture_recovers_on_a_later_collect() {
+ let (follow_up_tx, _follow_up_rx) = mpsc::unbounded_channel();
+ let (sessions, _kill_rx) =
+ sessions_with_fake_supervisor(follow_up_tx, 0, FakeKill::Success, FakeChild::TextLate)
+ .await;
+ let handle = spawn_child(&sessions).await;
+ sessions.notify_turn_done();
+ // The watcher's settle refresh consumed the empty first answer read and
+ // settled the child with no captured answer.
+ let deadline = Instant::now() + Duration::from_secs(10);
+ loop {
+ let entries = sessions.list_subagents().await.expect("child roster");
+ if entries.iter().any(|entry| entry.status == "completed") {
+ break;
+ }
+ assert!(
+ Instant::now() < deadline,
+ "child never settled: {entries:?}"
+ );
+ tokio::time::sleep(Duration::from_millis(50)).await;
+ }
+ // A later collect refreshes the row and the answer lands: the settle
+ // verdict is one-shot, the capture is not.
+ let results = sessions
+ .collect(vec![handle.rlm_child_id.clone()], 0)
+ .await
+ .expect("collect the settled child");
+ assert_eq!(results[0].status, "done");
+ assert_eq!(
+ results[0].answer_text.as_deref(),
+ Some("the child final answer"),
+ "the late answer must land after the empty first capture"
+ );
+ assert_eq!(
+ results[0].answer_preview.as_deref(),
+ Some("the child final answer")
+ );
+}
+
+/// The collect envelope carries the settled child's full final answer as its
+/// binding lane (the factory's output capture binds the whole fenced JSON from
+/// it), while the roster preview stays the compact form; the deleted envelope's
+/// binding lane is empty (the tombstone only keeps the preview).
+#[tokio::test]
+async fn collect_carries_the_full_answer_text_of_a_settled_child() {
+ let (follow_up_tx, _follow_up_rx) = mpsc::unbounded_channel();
+ let (sessions, _kill_rx) =
+ sessions_with_fake_supervisor(follow_up_tx, 0, FakeKill::Success, FakeChild::Healthy).await;
+ let handle = spawn_child(&sessions).await;
+ sessions.notify_turn_done();
+ let deadline = Instant::now() + Duration::from_secs(10);
+ loop {
+ let entries = sessions.list_subagents().await.expect("child roster");
+ if entries.iter().any(|entry| entry.status == "completed") {
+ break;
+ }
+ assert!(
+ Instant::now() < deadline,
+ "child never settled: {entries:?}"
+ );
+ tokio::time::sleep(Duration::from_millis(50)).await;
+ }
+ let results = sessions
+ .collect(vec![handle.rlm_child_id.clone()], 0)
+ .await
+ .expect("collect the settled child");
+ assert_eq!(
+ results[0].answer_preview.as_deref(),
+ Some("the child final answer")
+ );
+ assert_eq!(
+ results[0].answer_text.as_deref(),
+ Some("the child final answer"),
+ "the binding lane carries the full final answer"
+ );
+ sessions
+ .delete_subagent(handle.rlm_child_id.clone())
+ .await
+ .expect("delete the settled child");
+ let deleted = sessions
+ .collect(vec![handle.rlm_child_id.clone()], 0)
+ .await
+ .expect("collect the deleted child");
+ assert_eq!(
+ deleted[0].answer_preview.as_deref(),
+ Some("the child final answer")
+ );
+ assert_eq!(
+ deleted[0].answer_text, None,
+ "the tombstone carries no binding lane"
);
}
@@ -1034,6 +1251,7 @@ fn an_already_settled_child_never_re_scores_as_an_unreachable_error() {
settled_status: None,
settled: false,
answer_preview: None,
+ answer_text: None,
answer_captured: false,
replied_since_task: false,
notice_delivered: false,
@@ -1065,3 +1283,77 @@ fn an_already_settled_child_never_re_scores_as_an_unreachable_error() {
noticed.notice_delivered = true;
assert!(!super::lifecycle::should_mark_unreachable_error(&noticed));
}
+
+/// The unreachable give-up's exit capture is a wasted round trip when a
+/// captured answer already stands (a child that settled, went busy again
+/// on a queued continuation, then lost its worker): the record keeps its
+/// capture, so the fetched text could never land. The give-up must skip
+/// the capture entirely — zero `get_last_assistant_text` reads, the
+/// standing capture unchanged.
+#[tokio::test]
+async fn the_unreachable_give_up_skips_the_exit_capture_when_a_capture_stands() {
+ let (follow_up_tx, _follow_up_rx) = mpsc::unbounded_channel();
+ let child_subagents = Arc::new(FakeChildSubagents::default());
+ let (sessions, _kill_rx) = sessions_with_fake_child_subagents(
+ follow_up_tx,
+ 0,
+ FakeKill::Success,
+ FakeChild::Unreachable,
+ Arc::clone(&child_subagents),
+ )
+ .await;
+ let record = Arc::new(tokio::sync::Mutex::new(ChildRecord {
+ rlm_child_id: "child-id".to_string(),
+ session_name: "lane".to_string(),
+ active_session_id: "child-live".to_string(),
+ session_id: Some("child-file".to_string()),
+ session_dir: "/tmp".to_string(),
+ model: String::new(),
+ label: "task".to_string(),
+ started_at_ms: 0,
+ settled_status: None,
+ settled: false,
+ answer_preview: Some("the captured say".to_string()),
+ answer_text: Some("the captured say".to_string()),
+ answer_captured: true,
+ replied_since_task: false,
+ notice_delivered: false,
+ prompt_admitted: true,
+ error: None,
+ closed_by_parent: false,
+ session_file: Some(
+ std::env::temp_dir()
+ .join(format!(
+ "pa-rlm-watch-skip-{}.jsonl",
+ uuid::Uuid::new_v4().simple()
+ ))
+ .to_string_lossy()
+ .to_string(),
+ ),
+ attributed_rows: Some(0),
+ usage_watch_live: false,
+ usage_rearm: false,
+ emit_lock: std::sync::Arc::new(tokio::sync::Mutex::new(())),
+ last_emitted_status: None,
+ rename_lock: std::sync::Arc::new(tokio::sync::Mutex::new(())),
+ }));
+ let claimed = sessions
+ .inner
+ .settle_failed(
+ &record,
+ "Child worker unreachable".to_string(),
+ super::lifecycle::FailedArm::Unreachable,
+ )
+ .await;
+ assert!(claimed, "the give-up claims the settle");
+ assert_eq!(
+ child_subagents.answer_reads.load(Ordering::SeqCst),
+ 0,
+ "no exit-capture round trip may run while a captured answer stands"
+ );
+ let record = record.lock().await;
+ assert_eq!(record.settled_status, Some("error"));
+ assert_eq!(record.answer_preview.as_deref(), Some("the captured say"));
+ assert_eq!(record.answer_text.as_deref(), Some("the captured say"));
+ assert!(record.answer_captured);
+}
diff --git a/crates/pa-daemon/tests/rlm_children_e2e.rs b/crates/pa-daemon/tests/rlm_children_e2e.rs
index 1c85f1482f..0bf4eb8a49 100644
--- a/crates/pa-daemon/tests/rlm_children_e2e.rs
+++ b/crates/pa-daemon/tests/rlm_children_e2e.rs
@@ -436,14 +436,24 @@ async fn rlm_children_spawn_roster_collect_delete_end_to_end() {
.map(|_| ())
});
- // An unknown delete target is the TS selector miss.
- let gone = children
+ // Retirement is idempotent (the M5 class): a re-delete of the deleted
+ // child's name answers from the tombstone with the row's status at its
+ // delete instead of the selector miss, so a repeated retire never
+ // reports the name slot as still held.
+ let redeleted = children
.delete_subagent("worker-a".to_string())
.await
- .expect_err("deleted child no longer resolves");
+ .expect("the re-delete of a retired child is idempotent");
+ assert_eq!(redeleted.outcome, Some("deleted"));
+ assert_eq!(redeleted.subagent.rlm_child_id, handle.rlm_child_id);
+ // An unknown delete target keeps the TS selector miss.
+ let gone = children
+ .delete_subagent("ghost-worker".to_string())
+ .await
+ .expect_err("an unknown selector still errors");
assert_eq!(
gone.to_string(),
- "No direct RLM subagent matches \"worker-a\" in the current parent session"
+ "No direct RLM subagent matches \"ghost-worker\" in the current parent session"
);
// TS #2388 F4: a just-deleted target resolves immediately to the settled cancelled
diff --git a/prime-agent-runtime/src/rlm/__init__.py b/prime-agent-runtime/src/rlm/__init__.py
index 94169486d0..cecca69dc5 100644
--- a/prime-agent-runtime/src/rlm/__init__.py
+++ b/prime-agent-runtime/src/rlm/__init__.py
@@ -57,6 +57,10 @@ class RLMSubagentActivity:
@dataclass(frozen=True)
class RLMSubagent:
+ """One direct child row: `status` is `running` | `completed` | `error` |
+ `cancelled` (a cancelled child keeps its status verbatim in the
+ registry row, the TS-era semantics)."""
+
rlm_child_id: str
active_session_id: str | None
session_id: str | None
@@ -82,7 +86,11 @@ class RLMProgressNoteResult:
@dataclass(frozen=True)
class RLMChildResult:
- """Terminal or in-progress state of one direct child, from `collect()`."""
+ """Terminal or in-progress state of one direct child, from `collect()`.
+
+ `answer_text` is the child's full final answer (the settle-binding
+ lane, host-bounded); `answer_preview` is the compact roster preview.
+ """
rlm_child_id: str
session_name: str | None
@@ -90,10 +98,11 @@ class RLMChildResult:
status: str
settled: bool
answer_preview: str | None
- error: str | None
- duration_ms: int | None
- tool_use_count: int | None
- replied_since_task: bool | None
+ answer_text: str | None = None
+ error: str | None = None
+ duration_ms: int | None = None
+ tool_use_count: int | None = None
+ replied_since_task: bool | None = None
def _spawn_handle_from_payload(payload: Any) -> RLMSpawnHandle:
@@ -313,7 +322,7 @@ def _subagent_from_payload(payload: Any, operation: str = "rlm.list_subagents")
raise RuntimeError(f"{operation} entry is missing session_name")
if not isinstance(session_dir, str) or not session_dir:
raise RuntimeError(f"{operation} entry is missing session_dir")
- if status not in {"running", "completed", "error"}:
+ if status not in {"running", "completed", "error", "cancelled"}:
raise RuntimeError(f"{operation} entry has invalid status")
return RLMSubagent(
rlm_child_id=child_id,
@@ -396,6 +405,7 @@ def _child_result_from_payload(payload: Any) -> RLMChildResult:
status=status,
settled=settled,
answer_preview=_optional_str("answer_preview"),
+ answer_text=_optional_str("answer_text"),
error=_optional_str("error"),
duration_ms=_optional_int("duration_ms"),
tool_use_count=_optional_int("tool_use_count"),
diff --git a/prime-agent-runtime/src/rlm/factory.py b/prime-agent-runtime/src/rlm/factory.py
index 411e756640..743e9efbb9 100644
--- a/prime-agent-runtime/src/rlm/factory.py
+++ b/prime-agent-runtime/src/rlm/factory.py
@@ -533,10 +533,13 @@ def validate_factory_machine(machine: Any) -> list[str]:
seen_ids.add(state_id)
states_by_id[state_id] = state
- # Configured inline subagent names label the spawned children verbatim,
- # so two states sharing one name would collide on the supervisor's
- # unique sibling-name requirement at spawn time; reject the duplicate at
- # write time instead (the same reason duplicate state ids are rejected).
+ # Configured inline subagent names label the spawned children
+ # run-scoped (the spawn label prefixes the configured name with the run
+ # id), so two states sharing one name would still collide within a run
+ # on the supervisor's unique sibling-name requirement at spawn time
+ # (one common per-run prefix preserves the relative collision shape);
+ # reject the duplicate at write time instead (the same reason
+ # duplicate state ids are rejected).
seen_subagent_names: dict[str, str] = {}
for state_id, state in states_by_id.items():
_validate_state_fields(
@@ -1005,12 +1008,24 @@ __all__ = [
# ---------------------------------------------------------------------------
ANSWER_CAPTURE_CAP = 200
-"""Local safety cap for captured answers.
+"""Local cap for captured-answer PREVIEWS: the ``answer_captured`` ledger
+event and the status node's ``answer_preview`` stay compact (the host's
+own roster previews cap at 160 characters, ``compactRlmText``).
+"""
-``rlm.collect`` already returns previews: the host caps them at 160
-characters (``compactRlmText``). Input binding and every rendered prompt
-therefore work on capped preview text; full child outputs stay in the
-child's own session and are never seen by the executor.
+ANSWER_BINDING_CAP = 8192
+"""Local cap for the settle-capture binding lane.
+
+``rlm.collect`` carries the child's full final answer (``answer_text``,
+host-bounded at 64 KiB): input binding and output capture work on that
+text, not the roster preview — a fenced JSON output longer than the old
+200-character preview cap was truncated before the executor ever saw it,
+and the downstream bind failed with "no JSON object containing output"
+although the child settled with exactly the fenced JSON (the M2 class).
+The factory caps its own storage here so a run of many long answers
+cannot balloon kernel memory; a bind failure names this cap when the
+capture was cut at it. Full child transcripts stay in the child's own
+session regardless.
"""
EVENT_WINDOW = 200
@@ -1103,20 +1118,34 @@ def _child_name(run_id: str, state_id: str, instance_index: int, attempt: int) -
return "-".join(parts)
+def _run_prefix(run_id: str) -> str:
+ """The run-scoped label prefix (the generated label's third part)."""
+ return run_id[:6]
+
+
def _spawn_label(
configured: str | None, run_id: str, state_id: str, instance_index: int, attempt: int
) -> str:
"""Sibling label for one spawned instance: the state's configured inline
- subagent ``name`` when it has one, else the generated label.
-
- The configured name is used verbatim for the state's first instance on
- its first attempt (agents message the child by exactly this label); the
- SAME disambiguation suffixes as the generated label -- ``i<n>`` for
- later instances (re-entry, foreach fan-out), ``a<n>`` for retries --
- keep every admission unique: the supervisor rejects duplicate sibling
- names, and one state's settled children stay registered for the
- run's life, so a re-entering state (``max_entries`` > 1) would collide
- with its own earlier child on a verbatim name.
+ subagent ``name`` prefixed with the run id, else the generated label.
+
+ Every machine child label is run-scoped: the supervisor's sibling
+ names are per-parent-session, and a parent session outlives its runs,
+ so a verbatim configured name would collide with a prior run's
+ settled children (they stay registered for the run's life) or with a
+ live agent of the same name -- the M1 unresolvable-pause class.
+ The ``<run6>-`` prefix (the same run slug the generated label
+ carries) keeps cross-run admissions unique; agents message the child
+ by the label the run reports (the ``spawned`` event's ``name`` and
+ the status node's instance rows), not by the configured name alone.
+
+ The SAME disambiguation suffixes as the generated label -- ``i<n>``
+ for later instances (re-entry, foreach fan-out), ``a<n>`` for retries
+ -- keep every admission within a run unique: the supervisor rejects
+ duplicate sibling names, and one state's settled children stay
+ registered for the run's life, so a re-entering state
+ (``max_entries`` > 1) would collide with its own earlier child on an
+ unprefixed repeat.
A suffixed label never exceeds the host's 64-character spawn-name cap:
an overflowing base shrinks to a digest-suffixed token of the full
@@ -1128,7 +1157,7 @@ def _spawn_label(
if attempt > 1:
suffix_parts.append(f"a{attempt}")
suffix = "".join(f"-{part}" for part in suffix_parts)
- base = configured
+ base = f"{_run_prefix(run_id)}-{configured}"
if len(base) + len(suffix) > SUBAGENT_NAME_MAX_LENGTH:
# The host caps spawn names at 64 characters, so a suffixed label
# that would exceed it shrinks its base first -- and a bare
@@ -1163,14 +1192,19 @@ def _suffixed_spawn_form(base: str, candidate: str) -> bool:
return True
-def _parse_json_output(answer: str, output_name: str) -> tuple[Any, str | None]:
+def _parse_json_output(
+ answer: str, output_name: str, *, truncated_at: int | None = None
+) -> tuple[Any, str | None]:
"""Extract one named JSON output from an upstream answer.
Prefers the fenced `````json`` block whose object contains the output
name, scanning trailing blocks first (an answer with several fences,
e.g. a verdict block followed by a summary block, binds from whichever
block carries the port), then falls back to parsing the whole answer.
- Returns ``(value, None)`` or ``(None, error_sentence)``.
+ Returns ``(value, None)`` or ``(None, error_sentence)``. A capture cut
+ at the binding cap (``truncated_at``) names the size and the cap in the
+ error, so a too-large fenced JSON reads as a size problem, not a
+ missing output.
"""
candidates: list[str] = list(reversed(_FENCED_JSON_RE.findall(answer)))
candidates.append(answer.strip())
@@ -1181,7 +1215,13 @@ def _parse_json_output(answer: str, output_name: str) -> tuple[Any, str | None]:
continue
if isinstance(parsed, dict) and output_name in parsed:
return parsed[output_name], None
- return None, f"no JSON object containing output {output_name!r} in the upstream answer"
+ error = f"no JSON object containing output {output_name!r} in the upstream answer"
+ if truncated_at is not None:
+ error += (
+ f"; the captured answer is truncated at {truncated_at} characters"
+ " - keep the fenced JSON block compact"
+ )
+ return None, error
def _render_prompt(template: str, values: dict[str, str]) -> str:
@@ -1310,7 +1350,10 @@ class _NodeInstance:
child_id: str | None = None
spawned_at: float | None = None
duration_ms: int | None = None
- answer: str | None = None # capped collect preview (ANSWER_CAPTURE_CAP)
+ answer: str | None = None # the binding lane's captured answer (ANSWER_BINDING_CAP)
+ provisional: bool = False # the answer came from a child EXIT, not a
+ # settle: the state reads needs-verify instead of silently counting the
+ # exit's last assistant text as delivered work (the M4 class).
error: str | None = None
tool_uses: int = 0
@@ -1334,6 +1377,10 @@ class _StateEntry:
output_errors: dict[str, str] | None = None # ports whose json capture failed
is_settle: bool = False # True once the entry settled (done or error)
consumed: bool = False # True once the settle's transitions were evaluated
+ needs_verify: bool = False # a child exit carried a provisional answer
+ # (the M4 oversight mark): the entry failed, but the child's last
+ # assistant text is preserved as a provisional answer and the run
+ # surfaces the state in status()["needs_verify"].
@dataclass
@@ -1396,6 +1443,10 @@ class FactoryRun:
run_budget_ms: int | None = None
budget_reported: bool = False
pause_reason: str | None = None
+ # The last entry-failure reason (admission or binding): a paused run's
+ # status payload carries it with a one-line remedy (the M6 class), so
+ # an operator reads WHAT failed from status() instead of archaeology.
+ last_error: str | None = None
states: dict[str, _StateRun] = field(default_factory=dict)
order: list[str] = field(default_factory=list)
transitions_from: dict[str, list[dict[str, Any]]] = field(default_factory=dict)
@@ -1572,7 +1623,7 @@ class FactoryExecutor:
for event in run.events:
if event["stage"] in ("recorded", "arrived"):
event["stage"] = "delivered"
- return {
+ payload: dict[str, Any] = {
"run_id": run.run_id,
"spec_id": run.spec_id,
"name": run.name,
@@ -1582,6 +1633,29 @@ class FactoryExecutor:
"elapsed_ms": int((self._now_fn() - run.started_at) * 1000),
"usage": self._usage_report(run),
}
+ # The M4 oversight signal: every state whose latest terminal entry
+ # carried a provisional answer (a child exit) needs verification —
+ # root-visible without ledger archaeology.
+ needs_verify = [
+ state_id
+ for state_id in run.order
+ if any(entry.needs_verify for entry in run.states[state_id].entries)
+ ]
+ if needs_verify:
+ payload["needs_verify"] = needs_verify
+ # A paused run carries the LAST failed admission/bind error and a
+ # one-line remedy (the M6 class), so the operator reads WHAT failed
+ # and what to do next from status() alone.
+ if run.state == "paused":
+ payload["pause_reason"] = run.pause_reason
+ if run.last_error is not None:
+ payload["last_error"] = run.last_error
+ payload["remedy"] = (
+ f"resume with await rlm.factory.resume('{run.run_id}') to continue the "
+ "remaining states; the failed state stays error - fix its subagent and "
+ "start a fresh run to redo it"
+ )
+ return payload
async def stop(self, run_id: str) -> dict[str, Any]:
"""Cancel every running child of the run and mark it stopped.
@@ -1671,8 +1745,7 @@ class FactoryExecutor:
"entries_used": state.entries_used,
"max_entries": state.max_entries,
"entries": [
- {"index": entry.index, "status": entry.status, "error": entry.error}
- for entry in state.entries
+ self._entry_report(entry) for entry in state.entries
],
"instances": [
{
@@ -1703,11 +1776,33 @@ class FactoryExecutor:
if include_answer:
latest = state.latest_settle()
if latest is not None and latest.answer:
- report["answer_preview"] = latest.answer
+ # The report previews the settle answer compactly; the
+ # full binding text stays on the entry.
+ report["answer_preview"] = latest.answer[:ANSWER_CAPTURE_CAP]
if state.error is not None:
report["error"] = state.error
+ if any(entry.needs_verify for entry in state.entries):
+ report["needs_verify"] = True
return report
+ def _entry_report(self, entry: _StateEntry) -> dict[str, Any]:
+ """One entry's report row: its status/error plus, for a needs-verify
+ entry, the provisional answer the child exit captured (compacted)."""
+ row: dict[str, Any] = {"index": entry.index, "status": entry.status, "error": entry.error}
+ if entry.needs_verify:
+ row["needs_verify"] = True
+ provisional = next(
+ (
+ instance.answer
+ for instance in reversed(entry.instances)
+ if instance.provisional and instance.answer
+ ),
+ None,
+ )
+ if provisional is not None:
+ row["provisional_answer"] = provisional[:ANSWER_CAPTURE_CAP]
+ return row
+
def _usage_report(self, run: FactoryRun) -> dict[str, Any]:
"""The usage block ``status()`` returns; the graph snapshot reuses it."""
return {
@@ -2504,6 +2599,21 @@ class FactoryExecutor:
value = outputs[src_output]
if latest is None or failure is not None:
if inp.get("optional"):
+ # An optional input over a DIFFERENT state that has a
+ # live entry (pending or running) waits: the source is
+ # about to settle, and a sentinel here would spawn the
+ # dependent while its upstream is still running (the
+ # M3 premature-spawn class -- the children exited
+ # "waiting" and the bookkeeping counted them). The
+ # sentinel still binds for the self-input loop form and
+ # for a source with no live entry, so loop states keep
+ # re-entering before their upstream partner has RUN.
+ if latest is None and src_id != state.state_id:
+ source_state_live = source_state is not None and any(
+ other.status in ("pending", "running") for other in source_state.entries
+ )
+ if source_state_live:
+ return None, None # wait for the in-flight source's settle
# Optional inputs bind a null sentinel whenever their
# source offers no value -- never settled, errored
# settle, or a settled source that captured nothing
@@ -2689,6 +2799,18 @@ class FactoryExecutor:
elif result.status != "done":
child_reason = f"child settled with unexpected status {result.status!r}"
if child_reason is not None:
+ # The exit capture (the M4 class): an exited child's last
+ # assistant text — whatever the exit envelope carried — is a
+ # PROVISIONAL answer. The child may have finished its work and
+ # died without reporting; the answer is preserved on the
+ # instance and the state reads needs-verify instead of silently
+ # counting the exit as settled work.
+ exit_answer = (
+ (result.answer_text or result.answer_preview or "")[:ANSWER_BINDING_CAP] or None
+ )
+ if exit_answer:
+ instance.answer = exit_answer
+ instance.provisional = True
# Child failures retry (same rendered prompt, attempts+1) while
# attempts remain; then the entry failure_policy applies.
await self._apply_instance_failure(run, state, entry, instance, child_reason, retry=True)
@@ -2709,7 +2831,14 @@ class FactoryExecutor:
)
return
instance.status = "done"
- instance.answer = (result.answer_preview or "")[:ANSWER_CAPTURE_CAP] or None
+ # The binding lane first: the collect envelope's full final answer
+ # keeps the whole fenced JSON; the 160-character roster preview is
+ # the fallback (older hosts, a capture that raced a worker
+ # teardown). Both are capped locally at ANSWER_BINDING_CAP.
+ instance.answer = (
+ (result.answer_text or result.answer_preview or "")[:ANSWER_BINDING_CAP] or None
+ )
+ instance.provisional = False # a real settle supersedes any exit capture
self._event(
run,
"settled",
@@ -2726,13 +2855,13 @@ class FactoryExecutor:
node=state.state_id,
entry=entry.index,
instance=instance.index,
- answer=instance.answer,
+ answer=instance.answer[:ANSWER_CAPTURE_CAP],
stage="arrived",
)
if entry.status == "running" and entry.instances and all(i.status == "done" for i in entry.instances):
entry.status = "done"
entry.answer = self._entry_answer(entry)
- self._capture_outputs(state, entry)
+ await self._capture_outputs(run, state, entry)
self._queue_settle(run, state, entry)
def _entry_answer(self, entry: _StateEntry) -> str | None:
@@ -2742,16 +2871,20 @@ class FactoryExecutor:
return None
return "\n\n".join(answers)
- def _capture_outputs(self, state: _StateRun, entry: _StateEntry) -> None:
+ def _capture_ports(self, state: _StateRun, entry: _StateEntry) -> None:
"""Capture the state's declared output ports from the entry's answer.
Text ports keep the captured string; json ports parse as in V1
binding, with the parse error recorded on the settle so a reader
(guard or input binding) fails deterministically instead of
- re-parsing.
+ re-parsing. A capture cut at the binding cap names the size and the
+ cap in the parse error.
"""
outputs: dict[str, Any] = {}
errors: dict[str, str] = {}
+ truncated_at = (
+ len(entry.answer) if entry.answer is not None and len(entry.answer) >= ANSWER_BINDING_CAP else None
+ )
for out in state.spec.get("outputs") or []:
name, port_type = out.get("name"), out.get("type")
if port_type == "text":
@@ -2760,7 +2893,7 @@ class FactoryExecutor:
continue
if entry.answer is None:
continue
- parsed, error = _parse_json_output(entry.answer, name)
+ parsed, error = _parse_json_output(entry.answer, name, truncated_at=truncated_at)
if error is None:
outputs[name] = parsed
else:
@@ -2768,6 +2901,86 @@ class FactoryExecutor:
entry.outputs = outputs
entry.output_errors = errors
+ async def _capture_outputs(self, run: FactoryRun, state: _StateRun, entry: _StateEntry) -> None:
+ """Capture the settle's outputs, retrying the capture once.
+
+ A settle can race the child's answer capture on the host side (a
+ ``None`` answer recovers on a later refresh), and a declared port
+ that binds no value from the first capture is exactly the M2
+ silent-failure class: the downstream would run on nothing. When a
+ declared output binds no value, the settled children are
+ re-collected ONCE (a settled child keeps its result until deleted)
+ and the capture re-runs over the refreshed answers; a port that
+ still binds nothing records a deterministic capture failure in the
+ ledger (``output_capture_failed``) instead of passing silently.
+ """
+ self._capture_ports(state, entry)
+ declared = state.spec.get("outputs") or []
+ # Presence, not truthiness: a port that parsed as JSON null is a
+ # captured value (a downstream binding reads null from it), so the
+ # retry condition and the failure report check the KEY, never the
+ # value — a null output never reads as unbound.
+ if not declared or not any(
+ out.get("name") not in (entry.outputs or {}) for out in declared
+ ):
+ return # every declared port bound a value
+ refreshed = await self._refresh_entry_answers(run, state, entry)
+ if refreshed:
+ self._capture_ports(state, entry)
+ for out in declared:
+ name = out.get("name")
+ error = (entry.output_errors or {}).get(name)
+ if error is None and name not in (entry.outputs or {}):
+ error = f"output {name!r} captured no value from the upstream answer"
+ if error is not None:
+ self._event(
+ run,
+ "output_capture_failed",
+ node=state.state_id,
+ entry=entry.index,
+ port=name,
+ error=error,
+ detail=f"output {name!r} did not bind after one capture retry",
+ )
+
+ async def _refresh_entry_answers(self, run: FactoryRun, state: _StateRun, entry: _StateEntry) -> bool:
+ """Re-collect the entry's settled children once; refresh any longer
+ captured answer. Returns whether any answer changed."""
+ from . import collect
+
+ child_ids = [
+ instance.child_id
+ for instance in entry.instances
+ if instance.child_id is not None and instance.status == "done"
+ ]
+ if not child_ids:
+ return False
+ try:
+ results = await collect(child_ids, timeout_ms=0)
+ except Exception as exc:
+ self._event(
+ run,
+ "output_capture_failed",
+ node=state.state_id,
+ entry=entry.index,
+ error=f"capture retry could not re-collect the settled children: {exc}",
+ detail="the first capture stands",
+ )
+ return False
+ by_child = {result.rlm_child_id: result for result in results}
+ changed = False
+ for instance in entry.instances:
+ result = by_child.get(instance.child_id or "")
+ if result is None:
+ continue
+ answer = (result.answer_text or result.answer_preview or "")[:ANSWER_BINDING_CAP] or None
+ if answer and (instance.answer is None or len(answer) > len(instance.answer)):
+ instance.answer = answer
+ changed = True
+ if changed:
+ entry.answer = self._entry_answer(entry)
+ return changed
+
async def _apply_instance_failure(
self, run: FactoryRun, state: _StateRun, entry: _StateEntry, instance: _NodeInstance, reason: str, *, retry: bool
) -> None:
@@ -2816,12 +3029,36 @@ class FactoryExecutor:
async def _apply_entry_failure_policy(
self, run: FactoryRun, state: _StateRun, entry: _StateEntry, reason: str
) -> None:
+ # The verify mark is oversight, not failure-policy bookkeeping: a
+ # later foreach sibling can exit with a provisional answer AFTER
+ # the entry already went terminal (a sibling failed it first), so
+ # the check runs BEFORE the terminal guard -- the guard would
+ # otherwise swallow the late exit's signal, leaving the captured
+ # answer invisible in every needs_verify surface.
+ # entry.needs_verify keeps the mark and its event one-shot per entry.
+ if not entry.needs_verify and any(
+ instance.provisional and instance.answer for instance in entry.instances
+ ):
+ # An exit with a provisional answer is distinguishable from
+ # completed work only through the verify mark: the entry keeps
+ # its error verdict, but the preserved answer and the
+ # status-level needs_verify signal hand the operator the
+ # revival-race decision instead of a silent drop.
+ entry.needs_verify = True
+ self._event(
+ run,
+ "needs_verify",
+ node=state.state_id,
+ entry=entry.index,
+ detail="child exit captured a provisional answer - verify the remote state",
+ )
if entry.status in TERMINAL_ENTRY_STATUSES:
return # the policy already ran for this entry
policy = state.spec.get("failure_policy", RUN_FAILURE_POLICY_DEFAULT)
entry.status = "error"
entry.error = reason
state.error = reason
+ run.last_error = f"state {state.state_id!r} failed: {reason}"
self._event(run, "node_error", node=state.state_id, entry=entry.index, error=reason, detail=f"failure_policy {policy}")
# The entry is terminal: its prepared-but-never-admitted instances
# (a foreach queue behind max_parallel, or a rate-limit deferral)
@@ -4484,9 +4721,9 @@ fixer ever runs and its re-entry re-binds the real report; `max_entries`
bounds the loop; `monitoring` is a `resident` that stays alive under the
parent session after the run ends. Both worked examples bound their
emitted payloads in the prompt — a capped findings list here, a capped
-file list in the review-sweep example — because captured answers are
-capped previews: an unbounded payload truncates at the cap and fails to
-bind.
+file list in the review-sweep example — because the executor's capture
+cap still bounds very large payloads: an unbounded payload truncates at
+the cap and fails to bind.
## Authoring reference
@@ -4496,14 +4733,17 @@ bind.
content is the prompt template; `metadata.model`/`metadata.thinking` are
spawn settings) or an inline `{"prompt": ...}` object with optional
`name`/`model`/`thinking`. The optional `name` labels the spawned
- children (at most 64 characters, unique across the machine's states —
- a name another state's name can suffix onto, `foo` vs `foo-i1`, is
- rejected at write time): the first instance is named exactly `name` —
- the label to message the child by — and re-entries, foreach fan-out,
- and retries disambiguate with the same `-i<n>`/`-a<n>` suffixes the
- generated labels use; a suffixed label that would pass the host's
- 64-character cap shrinks its base with a digest of the full name, like
- the generated labels do.
+ children run-scoped (at most 64 characters, unique across the
+ machine's states — a name another state's name can suffix onto, `foo`
+ vs `foo-i1`, is rejected at write time): the spawn label is
+ `<run6>-name` — the supervisor's sibling names are per-parent-session,
+ and a parent session outlives its runs, so the run prefix keeps two
+ runs of one machine from colliding; read the exact label from the
+ `spawned` event's `name` or the status node's instance rows — and
+ re-entries, foreach fan-out, and retries disambiguate with the same
+ `-i<n>`/`-a<n>` suffixes the generated labels use; a suffixed label
+ that would pass the host's 64-character cap shrinks its base with a
+ digest of the full name, like the generated labels do.
- **Ports**: inputs and outputs of type `text` or `json`. An input binds
`"from": "<state_id>.<output_name>"`; types must match, duplicates are
rejected, and nothing can read from a resident. Bound values render into
@@ -4638,6 +4878,11 @@ status["events"] # trailing ledger: spawned, settled, answer_captured,
# transition_fired, node_error, milestone, ...
status["usage"] # spawns, settled, tool_uses, max_parallel, max_children, running,
# transitions_fired
+status["needs_verify"] # (when set) states whose child EXITED carrying a
+ # provisional answer - verify the remote state; the
+ # entry row carries the provisional_answer preview
+status["pause_reason"] # (paused) why the run paused; with "last_error" (the
+status["last_error"] # last admission/bind failure) and a one-line "remedy"
```
`graph()` and `watch()` are the live monitoring views this namespace
@@ -4714,10 +4959,11 @@ watched = await rlm.factory.watch(result["run_id"], 30)
`max_entries`, `max_transitions`, and `run.max_children` (total
admissions); the default `escalate` policy pauses
instead of failing, so read `status` (or the notice) before resuming.
-- Captured answers are capped previews (about 160 characters) and outputs
- bind from them: keep declared outputs compact — a small fenced json
- block or one short line — and let the full answer live in the child's
- session.
+- Outputs bind from the child's full final answer (the collect envelope
+ carries it; the executor caps its own capture at 8,192 characters): a
+ fenced json output block binds whole as long as it fits that cap, so
+ keep declared outputs compact — the rosters and ledger previews stay
+ ~160 characters — and let the full answer live in the child's session.
- Run registries live in kernel memory: a kernel restart loses `status`
for old runs, but the children keep running under the supervisor
(`rlm.list_subagents` sees them). Stop runs before restarting, or
diff --git a/prime-agent-runtime/test/test_factory.py b/prime-agent-runtime/test/test_factory.py
index 6a8dde9be1..d4eec47db7 100644
--- a/prime-agent-runtime/test/test_factory.py
+++ b/prime-agent-runtime/test/test_factory.py
@@ -32,8 +32,11 @@ races) plus the accepted review findings from both:
- long state ids disambiguate their spawn names with a digest, so two
states sharing a 20-character prefix never collide on the supervisor's
unique sibling-name requirement;
-- a configured inline subagent name labels the spawned children verbatim
- (the first instance), with the generated label's -i<n>/-a<n> suffixes on
+- a configured inline subagent name labels the spawned children
+ run-scoped: the spawn label prefixes the configured name with the run
+ id (the supervisor's sibling names are per-parent-session and a parent
+ outlives its runs, so a verbatim name would collide with a prior run's
+ children), with the generated label's -i<n>/-a<n> suffixes on
re-entry, foreach fan-out, and retries; over-length names and names
duplicated across states are rejected at write time (Macroscope review
finding: the name was dropped, so children always got the generated
@@ -62,6 +65,7 @@ from unittest.mock import patch
import rlm as rlm_module
from rlm import factory as factory_module
from rlm.factory import (
+ ANSWER_BINDING_CAP,
ANSWER_CAPTURE_CAP,
BACKOFF_MAX_ATTEMPTS,
EVENT_WINDOW,
@@ -105,6 +109,11 @@ from rlm.harness import HarnessState
# Spec fixtures
# ---------------------------------------------------------------------------
+#: The host's roster-preview cap (the daemon's ANSWER_PREVIEW_MAX_CHARS):
+#: the fake host mirrors it so the binding lane's full text is what
+#: distinguishes a binding that survives truncation from one that does not.
+HOST_ANSWER_PREVIEW_CHARS = 160
+
def state(state_id: str, **overrides: Any) -> dict[str, Any]:
base: dict[str, Any] = {"id": state_id, "subagent": "worker"}
@@ -2330,23 +2339,38 @@ class ChildNameTest(unittest.TestCase):
class SpawnLabelTest(unittest.TestCase):
- """The configured inline subagent name labels spawned children.
-
- The first instance of a state spawns with the configured name verbatim
- (the label agents message the child by); later instances and retries
- keep the generated label's -i<n>/-a<n> suffixes, because one state's
- settled children stay registered for the run's life and the supervisor
- rejects duplicate sibling names. (Macroscope review finding: the name
- was dropped, so every child got the generated label.)
+ """The configured inline subagent name labels spawned children run-scoped.
+
+ Every machine child label carries the run id prefix: the supervisor's
+ sibling names are per-parent-session, and a parent session outlives
+ its runs, so a verbatim configured name would collide with a prior
+ run's settled children (they stay registered for the run's life) —
+ the M1 unresolvable-pause class. Later instances and retries keep the
+ generated label's -i<n>/-a<n> suffixes, because the supervisor rejects
+ duplicate sibling names. (Macroscope review finding: the name was
+ dropped, so every child got the generated label.)
"""
- def test_configured_name_labels_the_first_instance_verbatim(self) -> None:
- self.assertEqual(_spawn_label("reviewer", "0123456789abcdef", "reviewing", 0, 1), "reviewer")
+ def test_configured_name_labels_the_first_instance_run_scoped(self) -> None:
+ self.assertEqual(
+ _spawn_label("reviewer", "0123456789abcdef", "reviewing", 0, 1),
+ "012345-reviewer",
+ )
+
+ def test_two_runs_of_one_machine_never_share_a_label(self) -> None:
+ # M1: machine child names are supervisor-global across runs; the
+ # run prefix keeps two runs of the same machine from colliding on
+ # the supervisor's unique sibling-name requirement.
+ one = _spawn_label("impl-build", "0123456789abcdef", "implement", 0, 1)
+ two = _spawn_label("impl-build", "fedcba9876543210", "implement", 0, 1)
+ self.assertEqual(one, "012345-impl-build")
+ self.assertEqual(two, "fedcba-impl-build")
+ self.assertNotEqual(one, two)
def test_configured_name_keeps_the_generated_suffixes_when_disambiguating(self) -> None:
- self.assertEqual(_spawn_label("reviewer", "r", "reviewing", 1, 1), "reviewer-i1")
- self.assertEqual(_spawn_label("reviewer", "r", "reviewing", 0, 2), "reviewer-a2")
- self.assertEqual(_spawn_label("reviewer", "r", "reviewing", 2, 3), "reviewer-i2-a3")
+ self.assertEqual(_spawn_label("reviewer", "r", "reviewing", 1, 1), "r-reviewer-i1")
+ self.assertEqual(_spawn_label("reviewer", "r", "reviewing", 0, 2), "r-reviewer-a2")
+ self.assertEqual(_spawn_label("reviewer", "r", "reviewing", 2, 3), "r-reviewer-i2-a3")
self.assertNotEqual(
_spawn_label("reviewer", "r", "reviewing", 0, 1), _spawn_label("reviewer", "r", "reviewing", 1, 1)
)
@@ -2358,17 +2382,21 @@ class SpawnLabelTest(unittest.TestCase):
)
def test_suffixed_labels_stay_within_the_host_cap(self) -> None:
- # A configured name at the 64-character cap still fits its first
- # instance; a suffixed admission that would pass the cap shrinks
- # its base with a digest of the full name, like the generated
- # labels do (Macroscope and Cursor review findings: the overflow
- # would fail every later spawn admission).
+ # The run prefix costs 7 characters of the 64-character cap, so a
+ # long configured name never appears verbatim: a label that would
+ # pass the cap shrinks its base with a digest of the full prefixed
+ # name, like the generated labels do (Macroscope and Cursor review
+ # findings: the overflow would fail every later spawn admission).
long_name = "x" * SUBAGENT_NAME_MAX_LENGTH
- self.assertEqual(_spawn_label(long_name, "r", "a", 0, 1), long_name)
+ prefixed_base = f"r-{long_name}"
+ digest = hashlib.sha256(prefixed_base.encode("utf-8")).hexdigest()[:16]
+ first = _spawn_label(long_name, "r", "a", 0, 1)
+ self.assertLessEqual(len(first), SUBAGENT_NAME_MAX_LENGTH)
+ self.assertIn(digest, first)
second = _spawn_label(long_name, "r", "a", 1, 1)
self.assertLessEqual(len(second), SUBAGENT_NAME_MAX_LENGTH)
self.assertTrue(second.endswith("-i1"))
- self.assertIn(hashlib.sha256(long_name.encode("utf-8")).hexdigest()[:16], second)
+ self.assertIn(digest, second)
# Truncation alone could collide (two long names sharing the
# prefix): the digest keeps them distinct.
sharing_prefix = "x" * (SUBAGENT_NAME_MAX_LENGTH - 1) + "y"
@@ -2489,7 +2517,7 @@ class FactoryHelpTest(unittest.TestCase):
# The configured inline subagent name contract.
self.assertIn("The optional `name` labels the spawned children", flat)
- self.assertIn("the first instance is named exactly `name`", flat)
+ self.assertIn("the spawn label is `<run6>-name`", flat)
self.assertIn("unique across the machine's states", flat)
self.assertIn("a name another state's name can suffix onto, `foo` vs `foo-i1`, is rejected at write time", flat)
@@ -2675,11 +2703,21 @@ def _node_of_name(name: str) -> str:
"""The factory node a spawn name belongs to.
Generated labels are "sw-<node id>-<run>-..."; a state with a
- configured inline subagent name spawns children named by it verbatim,
- so such a name keys the node by the whole string.
+ configured inline subagent name spawns children labeled
+ "<run6>-<configured>" (run-scoped), so a leading 6-hex run prefix keys
+ the node by the configured base with any trailing -i<n>/-a<n>
+ disambiguation parts stripped (the test machines' configured names
+ never end in those shapes).
"""
parts = name.split("-")
- return parts[1] if parts[0] == "sw" and len(parts) > 2 else name
+ if parts[0] == "sw" and len(parts) > 2:
+ return parts[1]
+ if len(parts[0]) == 6 and all(char in "0123456789abcdef" for char in parts[0]):
+ base = parts[1:]
+ while len(base) > 1 and re.fullmatch(r"i[1-9][0-9]*|a[2-9][0-9]*", base[-1]):
+ base = base[:-1]
+ return "-".join(base)
+ return name
class FakeHost:
@@ -2756,7 +2794,18 @@ class FakeHost:
"duration_ms": 5,
}
if answer is not None:
- entry["answer_preview"] = answer
+ # The real host hands the kernel a compact preview (about 160
+ # characters, whitespace-collapsed, ellipsis tail) plus the
+ # FULL final answer as the collect envelope's binding lane;
+ # the fake mirrors both, so a long fenced JSON binds from the
+ # full lane exactly like production.
+ compact = " ".join(answer.split())
+ entry["answer_preview"] = (
+ compact[:HOST_ANSWER_PREVIEW_CHARS - 3] + "..."
+ if len(compact) > HOST_ANSWER_PREVIEW_CHARS
+ else compact
+ )
+ entry["answer_text"] = answer
if error is not None:
entry["error"] = error
return entry
@@ -2779,6 +2828,14 @@ class FakeHost:
# current one, so rate_limit_first<n> fails exactly the first n.
name = payload["kwargs"]["name"]
node_id = _node_of_name(name)
+ # The supervisor rejects duplicate sibling names: a prior run's
+ # settled children stay registered, so a same-name admission
+ # fails exactly like the real host (the M1 class).
+ if any(child["name"] == name for child in self.children.values()):
+ raise RuntimeError(
+ f'Agent name "{name}" is unavailable: an agent of that '
+ "name already exists at depth 1 under this parent"
+ )
attempted = len(
[
p
@@ -2835,12 +2892,15 @@ class FakeHost:
if request_type == "rlm.delete_subagent":
target = payload["target"]
child = self.children.pop(target, None)
+ # The real host's delete receipt reports the row as cancelled
+ # (a live child's delete settles it cancelled), so the kernel's
+ # payload validation sees the same shape it sees in production.
return {
"subagent": {
"rlm_child_id": target,
"session_name": child["name"] if child else "unknown",
"session_dir": f"/tmp/{target}",
- "status": "running",
+ "status": "cancelled",
},
"outcome": "deleted",
}
@@ -2860,6 +2920,41 @@ class DeleteFailsHost(FakeHost):
return await super().__call__(request_type, payload)
+class LateAnswerHost(FakeHost):
+ """FakeHost whose FIRST collect of a late child settles it with no
+ captured answer (the host-side capture race); later collects return
+ the real answer."""
+
+ def __init__(self, clock: "FakeClock | None" = None, late_children: "set[str] | frozenset[str]" = frozenset()):
+ super().__init__(clock=clock)
+ self.late_children = set(late_children)
+ self.collects_of: dict[str, int] = {}
+
+ async def __call__(self, request_type: str, payload: dict[str, Any] | None = None) -> dict[str, Any]:
+ payload = payload or {}
+ if request_type == "rlm.collect":
+ late = [t for t in payload["targets"] if t in self.late_children and self.collects_of.get(t, 0) == 0]
+ if late:
+ for target in late:
+ self.collects_of[target] = self.collects_of.get(target, 0) + 1
+ self.calls.append((request_type, payload))
+ self.collects += 1
+ if self.clock is not None:
+ self.clock.advance(self.clock.advance_per_collect)
+ return {
+ "results": [
+ self._entry(
+ child_id=target,
+ name=self.children[target]["name"],
+ status="done",
+ settled=True,
+ )
+ for target in late
+ ]
+ }
+ return await super().__call__(request_type, payload)
+
+
class DeadNoticeHost(FakeHost):
"""FakeHost whose factory.progress bridge is dead (every call raises)."""
@@ -3219,9 +3314,10 @@ class FactoryExecutorTest(_ExecutorTestCase):
async def test_inline_subagent_name_labels_the_spawned_child(self) -> None:
# Macroscope review finding: the inline subagent name was dropped by
# _resolve_subagents, so the child spawned with the generated label
- # instead of the configured name agents message the child by. The
- # name rides through run creation to the spawn call, and the spawned
- # event's ledger entry carries the same label.
+ # instead of the configured name. The name rides through run creation
+ # to the spawn call (run-scoped: prefixed with the run id), and the
+ # spawned event's ledger entry carries the same label — the label
+ # agents message the child by.
self.store_factory(
{
"run": {"failure_policy": "continue"},
@@ -3233,15 +3329,68 @@ class FactoryExecutorTest(_ExecutorTestCase):
result = await self.start()
status = await self.settle(result)
self.assertEqual(status["state"], "done")
+ label = f"{result['run_id'][:6]}-reviewer"
self.assertEqual(
[p["kwargs"]["name"] for p in self.host.calls_of("rlm.run")],
- ["reviewer"],
+ [label],
)
self.assertEqual(
[event["name"] for event in self.all_events_of(result, "spawned")],
- ["reviewer"],
+ [label],
)
+ @async_test
+ async def test_two_runs_of_one_machine_do_not_collide_on_child_names(self) -> None:
+ # M1: machine child names are supervisor-global across runs — a
+ # prior run's settled children stay registered for the run's life
+ # (and forever when their delete failed), so a verbatim configured
+ # name made a fresh run's spawn admission fail with "Agent name ...
+ # is unavailable" and the machine paused at spawn-zero. The
+ # run-scoped label prefix keeps run 2's admissions unique; the
+ # FakeHost rejects duplicate sibling names exactly like the real
+ # supervisor, so this test fails without the prefix.
+ self.host.outcomes["impl"] = {"status": "done", "answer": "IMPL"}
+ self.host.outcomes["check"] = {"status": "done", "answer": "CHECK"}
+ self.store_machine(
+ {
+ "run": {"failure_policy": "escalate", "max_parallel": 4},
+ "states": [
+ {
+ "id": "impl",
+ "entry": True,
+ "subagent": {"prompt": "Build.", "name": "impl-build"},
+ "outputs": [{"name": "built", "type": "text"}],
+ },
+ {
+ "id": "check",
+ "subagent": "worker",
+ "inputs": [{"name": "built", "type": "text", "from": "impl.built"}],
+ },
+ ],
+ "transitions": [{"from": "impl", "to": "check"}],
+ }
+ )
+ first = await self.start()
+ first_status = await self.settle(first)
+ self.assertEqual(first_status["state"], "done")
+ second = await self.start()
+ second_status = await self.settle(second)
+ self.assertEqual(second_status["state"], "done")
+ # Run 2's children spawn with distinct, run-scoped labels; neither
+ # admission collided with run 1's still-registered children.
+ first_label = f"{first['run_id'][:6]}-impl-build"
+ second_label = f"{second['run_id'][:6]}-impl-build"
+ self.assertNotEqual(first_label, second_label)
+ self.assertEqual(
+ [event["name"] for event in self.all_events_of(first, "spawned") if event["node"] == "impl"],
+ [first_label],
+ )
+ self.assertEqual(
+ [event["name"] for event in self.all_events_of(second, "spawned") if event["node"] == "impl"],
+ [second_label],
+ )
+ self.assertEqual(self.all_events_of(second, "node_error"), [])
+
@async_test
async def test_inline_subagent_name_disambiguates_reentry_and_foreach(self) -> None:
# One state's settled children stay registered for the run's life,
@@ -3265,9 +3414,10 @@ class FactoryExecutorTest(_ExecutorTestCase):
result = await self.start()
status = await self.settle(result)
self.assertEqual(status["state"], "done")
+ run_prefix = result["run_id"][:6]
self.assertEqual(
[p["kwargs"]["name"] for p in self.host.calls_of("rlm.run")],
- ["worker", "worker-i1"],
+ [f"{run_prefix}-worker", f"{run_prefix}-worker-i1"],
)
self.host.outcomes["src"] = {
@@ -3293,18 +3443,19 @@ class FactoryExecutorTest(_ExecutorTestCase):
},
spec_id="fan",
)
- result = await self.start("fan")
- status = await self.settle(result)
+ fan_result = await self.start("fan")
+ status = await self.settle(fan_result)
self.assertEqual(status["state"], "done")
# The first foreach instance keeps the configured name; the second
# disambiguates with the instance suffix.
+ fan_prefix = fan_result["run_id"][:6]
self.assertEqual(
[
p["kwargs"]["name"]
for p in self.host.calls_of("rlm.run")
- if p["kwargs"]["name"].startswith("expander")
+ if f"-expander" in p["kwargs"]["name"]
],
- ["expander", "expander-i1"],
+ [f"{fan_prefix}-expander", f"{fan_prefix}-expander-i1"],
)
@async_test
@@ -3570,6 +3721,124 @@ class FactoryExecutorTest(_ExecutorTestCase):
self.assertEqual(len(captured), ANSWER_CAPTURE_CAP)
self.assertEqual(captured, long_answer[:ANSWER_CAPTURE_CAP])
+ @async_test
+ async def test_json_output_longer_than_the_preview_cap_binds_from_the_full_answer(self) -> None:
+ # M2: the settle's declared fenced JSON was longer than the host's
+ # ~160-character roster preview, so the old preview-only capture
+ # truncated it mid-object and the downstream bind failed with "no
+ # JSON object containing output" although the child settled with
+ # exactly the fenced JSON. The collect envelope's full answer is
+ # the binding lane; the preview stays a preview.
+ worktree = "/Users/x/Research/pa-worktrees/rp-3354-the-very-long-worktree-name"
+ payload = {"worktree": worktree, "notes": "y" * 150}
+ long_json = f"Intro prose.\n\n```json\n{json.dumps(payload)}\n```\n\nOutro."
+ assert len(long_json) > HOST_ANSWER_PREVIEW_CHARS
+ self.host.outcomes["setup"] = {"status": "done", "answer": long_json}
+ self.host.outcomes["impl"] = {"status": "done", "answer": "DONE"}
+ self.store_machine(
+ {
+ "run": {"failure_policy": "escalate", "max_parallel": 4},
+ "states": [
+ {
+ "id": "setup",
+ "entry": True,
+ "subagent": "worker",
+ "outputs": [{"name": "worktree", "type": "json"}],
+ },
+ {
+ "id": "impl",
+ "subagent": "worker",
+ "inputs": [{"name": "worktree", "type": "json", "from": "setup.worktree"}],
+ },
+ ],
+ "transitions": [{"from": "setup", "to": "impl"}],
+ }
+ )
+ result = await self.start()
+ status = await self.settle(result)
+ self.assertEqual(status["state"], "done")
+ impl_prompt = self.host.spawn_calls("impl")[0]["prompt"]
+ self.assertIn(f"- worktree: {json.dumps(worktree)}", impl_prompt)
+ self.assertEqual(self.all_events_of(result, "output_capture_failed"), [])
+ # The event ledger previews compactly; the binding lane stays full:
+ # the captured answer is the raw final text (newlines intact), not
+ # the host's whitespace-collapsed roster preview.
+ captured_event = self.all_events_of(result, "answer_captured")[0]
+ self.assertLessEqual(len(captured_event["answer"]), ANSWER_CAPTURE_CAP)
+ self.assertIn("```json\n", captured_event["answer"])
+ self.assertNotIn("...", captured_event["answer"])
+
+ @async_test
+ async def test_a_json_null_output_is_a_captured_value_not_a_failure(self) -> None:
+ # Review finding (PR #3462): the capture-complete check conflated a
+ # port that parsed as JSON null with a missing port — a legit null
+ # output triggered the capture retry and recorded
+ # output_capture_failed although the bind succeeded. Presence of
+ # the key is the captured test, never the value.
+ self.host.outcomes["src"] = {"status": "done", "answer": '```json\n{"data": null}\n```'}
+ self.store_machine(
+ {
+ "run": {"failure_policy": "continue", "max_parallel": 4},
+ "states": [
+ {
+ "id": "src",
+ "entry": True,
+ "subagent": "worker",
+ "outputs": [{"name": "data", "type": "json"}],
+ },
+ {
+ "id": "dep",
+ "subagent": {"prompt": "Proceed."},
+ "inputs": [{"name": "data", "type": "json", "from": "src.data", "optional": True}],
+ },
+ ],
+ "transitions": [{"from": "src", "to": "dep"}],
+ }
+ )
+ result = await self.start()
+ status = await self.settle(result)
+ self.assertEqual(status["state"], "done")
+ self.assertEqual(self.all_events_of(result, "output_capture_failed"), [])
+ # The null was captured and bound: the dependent renders the JSON
+ # null, not the no-value sentinel path.
+ self.assertIn("- data: null", self.host.spawn_calls("dep")[0]["prompt"])
+
+ @async_test
+ async def test_capture_failure_names_the_binding_cap_and_records_the_event(self) -> None:
+ # A declared json output bigger than the executor's binding cap
+ # fails to parse with the SIZE named in the error (not a bare
+ # missing-output sentence), and the settle records the failure in
+ # the ledger instead of passing silently.
+ huge = f'{{"blob": "{"z" * (ANSWER_BINDING_CAP + 50)}"}}'
+ self.host.outcomes["src"] = {"status": "done", "answer": f"```json\n{huge}\n```"}
+ self.store_machine(
+ {
+ "run": {"failure_policy": "continue", "max_parallel": 4},
+ "states": [
+ {
+ "id": "src",
+ "entry": True,
+ "subagent": "worker",
+ "outputs": [{"name": "blob", "type": "json"}],
+ },
+ {
+ "id": "dep",
+ "subagent": {"prompt": "Use {blob}."},
+ "inputs": [{"name": "blob", "type": "json", "from": "src.blob", "optional": True}],
+ },
+ ],
+ "transitions": [{"from": "src", "to": "dep"}],
+ }
+ )
+ result = await self.start()
+ status = await self.settle(result)
+ # The optional dependent still proceeds on the sentinel (the
+ # authored contract), but the source's capture failure is recorded.
+ capture_failures = self.all_events_of(result, "output_capture_failed")
+ self.assertEqual(len(capture_failures), 1)
+ self.assertIn(str(ANSWER_BINDING_CAP), capture_failures[0]["error"])
+ self.assertIn("keep the fenced JSON block compact", capture_failures[0]["error"])
+
# -- foreach ----------------------------------------------------------------
@@ -4206,6 +4475,42 @@ class FactoryExecutorTest(_ExecutorTestCase):
run_stopped_events = [e for e in self.executor._runs[result["run_id"]].events if e["kind"] == "run_stopped"]
self.assertEqual(len(run_stopped_events), 1)
+ @async_test
+ async def test_stop_delete_receipts_with_cancelled_status_complete_cleanly(self) -> None:
+ # M5: the host's delete receipt reports a cancelled row (a live
+ # child's delete settles it cancelled; a previously-cancelled
+ # settled child keeps the status verbatim), and the kernel's
+ # payload validation must accept it — the receipt raising
+ # "entry has invalid status" used to turn every factory stop into
+ # cancel_failed bookkeeping and make retirement look failed.
+ self.host.outcomes["a"] = {"status": "running"}
+ self.host.outcomes["b"] = {"status": "running"}
+ self.store_factory(
+ {
+ "nodes": [
+ {"id": "a", "subagent": "worker"},
+ {"id": "b", "subagent": "worker"},
+ ],
+ }
+ )
+ result = await self.start()
+ await self.wait_until(lambda: self.host.collects >= 1)
+ stopped = await rlm_module.rlm.factory.stop(result["run_id"])
+ self.assertEqual(stopped["state"], "stopped")
+ self.assertEqual(sorted(stopped["cancelled"]), ["a", "b"])
+ # The delete receipts were accepted: no cancel_failed ledger rows,
+ # and both children released their name slots on the fake host.
+ self.assertEqual(self.all_events_of(result, "cancel_failed"), [])
+ self.assertEqual(
+ [event.get("detail") for event in self.all_events_of(result, "cancelled")],
+ [None, None],
+ )
+ # A retire of the already-cancelled children (the operator's
+ # delete_subagent on a settled/cancelled row) does not raise.
+ for child_id in ("child-1", "child-2"):
+ deleted = await rlm_module.rlm.delete_subagent(child_id)
+ self.assertEqual(deleted.status, "cancelled")
+
@async_test
async def test_concurrent_stop_runs_one_cancellation_pass(self) -> None:
# Review finding: two concurrent stop() calls would both pass a bare
@@ -5393,6 +5698,243 @@ class FactoryExecutorTest(_ExecutorTestCase):
["foreach expanded to zero items; nothing to run"] * 2,
)
+ @async_test
+ async def test_optional_input_waits_for_a_live_upstream_before_spawning(self) -> None:
+ # M3: an optional input over ANOTHER state that is still running
+ # must not bind its null sentinel yet — the sentinel spawned the
+ # dependent while its upstream was in flight (the children exited
+ # "waiting" and the state bookkeeping counted them). The entry
+ # waits for the in-flight source's settle, then binds the real
+ # value; a source with no live entry keeps the loop-form sentinel.
+ self.host.outcomes["impl"] = {"status": "running"}
+ self.host.outcomes["validate"] = {"status": "done", "answer": "VALIDATED"}
+ self.store_machine(
+ {
+ "run": {"failure_policy": "continue", "max_parallel": 4},
+ "states": [
+ {"id": "seed", "entry": True, "subagent": "worker"},
+ {
+ "id": "impl",
+ "subagent": "worker",
+ "outputs": [{"name": "result", "type": "text"}],
+ },
+ {
+ "id": "validate",
+ "subagent": {"prompt": "Validate the build at {result}."},
+ "inputs": [
+ {"name": "result", "type": "text", "from": "impl.result", "optional": True}
+ ],
+ },
+ ],
+ "transitions": [
+ {"from": "seed", "to": "impl"},
+ {"from": "seed", "to": "validate"},
+ ],
+ }
+ )
+ result = await self.start()
+ # impl is admitted and still running; the loop must prepare nothing.
+ await self.wait_until(lambda: len(self.host.spawn_calls("impl")) == 1)
+ await self.wait_until(lambda: self.host.collects >= 3)
+ self.assertEqual(self.host.spawn_calls("validate"), [])
+ # The upstream settles with a real value; validate spawns with it
+ # bound, never the null sentinel.
+ self.host.outcomes["impl"] = {"status": "done", "answer": "WTREE-A1"}
+ status = await self.settle(result)
+ self.assertEqual(status["state"], "done")
+ validate_calls = self.host.spawn_calls("validate")
+ self.assertEqual(len(validate_calls), 1)
+ self.assertIn("WTREE-A1", validate_calls[0]["prompt"])
+ self.assertNotIn("None", validate_calls[0]["prompt"])
+
+ @async_test
+ async def test_capture_retry_rebinds_a_settle_that_raced_the_answer_capture(self) -> None:
+ # M2: the host can settle a child with no captured answer yet (the
+ # capture recovers on a later refresh). The settle-capture retry
+ # re-collects the settled child once and re-runs the capture, so
+ # the declared output binds instead of leaving the settle empty
+ # and failing the downstream bind.
+ self.host = LateAnswerHost(clock=self.clock, late_children={"child-2"})
+ repatch = patch.object(rlm_module, "host_request", self.host)
+ repatch.start()
+ self.addCleanup(repatch.stop)
+ self.host.outcomes["build"] = {
+ "status": "done",
+ "answer": '```json\n{"built": "BIN-A1"}\n```',
+ }
+ self.host.outcomes["use"] = {"status": "done", "answer": "USED"}
+ self.store_machine(
+ {
+ "run": {"failure_policy": "escalate", "max_parallel": 4},
+ "states": [
+ {"id": "seed", "entry": True, "subagent": "worker"},
+ {
+ "id": "build",
+ "subagent": "worker",
+ "outputs": [{"name": "built", "type": "json"}],
+ },
+ {
+ "id": "use",
+ "subagent": {"prompt": "Use {built}."},
+ "inputs": [{"name": "built", "type": "json", "from": "build.built"}],
+ },
+ ],
+ "transitions": [
+ {"from": "seed", "to": "build"},
+ {"from": "build", "to": "use"},
+ ],
+ }
+ )
+ result = await self.start()
+ status = await self.settle(result)
+ self.assertEqual(status["state"], "done")
+ # The first collect of the build child settled it with no answer;
+ # the capture retry re-collected and the port bound.
+ self.assertEqual(self.host.collects_of.get("child-2"), 1)
+ self.assertEqual(
+ [call["prompt"] for call in self.host.spawn_calls("use")],
+ ['Use "BIN-A1".'],
+ )
+ self.assertEqual(self.all_events_of(result, "output_capture_failed"), [])
+
+ @async_test
+ async def test_child_exit_captures_a_provisional_answer_and_marks_needs_verify(self) -> None:
+ # M4: a child that dies without reporting is distinguishable from
+ # completed work. The exit envelope's last assistant text is
+ # captured as a PROVISIONAL answer, the state's entry row reports
+ # needs_verify with that answer, and status() carries the
+ # root-visible needs_verify signal — the exit never silently
+ # counts as settled work.
+ self.host.child_outcomes["child-1"] = {
+ "status": "error",
+ "error": "worker exited mid-flight",
+ "answer": "The CI gate ran clean on lane A; the report is at /tmp/report.md.",
+ }
+ self.store_machine(
+ {
+ "run": {"failure_policy": "continue", "max_parallel": 4},
+ "states": [
+ {"id": "work", "entry": True, "subagent": "worker"},
+ ],
+ "transitions": [],
+ }
+ )
+ result = await self.start()
+ status = await self.settle(result)
+ self.assertEqual(status["state"], "failed")
+ self.assertEqual(status["needs_verify"], ["work"])
+ work = self.state_report(status, "work")
+ self.assertEqual(work["needs_verify"], True)
+ entry_row = work["entries"][0]
+ self.assertEqual(entry_row["status"], "error")
+ self.assertEqual(entry_row["needs_verify"], True)
+ self.assertIn("report is at /tmp/report.md", entry_row["provisional_answer"])
+ self.assertEqual(
+ [event["node"] for event in self.all_events_of(result, "needs_verify")],
+ ["work"],
+ )
+ # The exit is not silently counted as a settled answer: the usage
+ # stays a plain error settle and no answer_captured row lands.
+ self.assertEqual(self.all_events_of(result, "answer_captured"), [])
+ self.assertEqual(status["usage"]["settled"], 1)
+
+ @async_test
+ async def test_late_sibling_exit_after_terminal_entry_still_marks_needs_verify(self) -> None:
+ # Review finding (Macroscope): a foreach sibling exiting with a
+ # provisional answer AFTER its entry already went terminal (a
+ # sibling failed it first) hit _apply_entry_failure_policy's
+ # terminal guard before the needs_verify mark: the exit capture
+ # kept the answer on the instance, but entry.needs_verify stayed
+ # unset and no needs_verify event fired -- the late exit's work
+ # vanished from every verify surface. The mark now runs before the
+ # terminal guard, independently of the entry's failure policy.
+ self.host.outcomes["src"] = {"status": "done", "answer": '{"items": ["a", "b"]}'}
+ self.store_factory(
+ {
+ "run": {"failure_policy": "continue", "max_parallel": 8},
+ "nodes": [
+ {"id": "src", "subagent": "worker", "outputs": [{"name": "items", "type": "json"}]},
+ {
+ "id": "fan",
+ "subagent": "worker",
+ "depends_on": ["src"],
+ "inputs": [{"name": "items", "type": "json", "from": "src.items"}],
+ "foreach": {"over": "items", "max": 2},
+ },
+ ],
+ }
+ )
+ result = await self.start()
+ run = self.executor._runs[result["run_id"]]
+ # src is child-1; the fan instances are child-2 (i0) and child-3
+ # (i1). i0 fails permanently with no answer: the entry goes error
+ # while i1 is still running.
+ self.host.child_outcomes["child-2"] = {"status": "error", "error": "boom-1"}
+ self.host.child_outcomes["child-3"] = {"status": "running"}
+ await self.wait_until(
+ lambda: any(entry.status == "error" for entry in run.states["fan"].entries)
+ )
+ # i1 exits NOW with a provisional answer; the entry is already
+ # terminal, so only a pre-guard verify mark can surface it.
+ self.host.child_outcomes["child-3"] = {
+ "status": "error",
+ "error": "worker exited mid-flight",
+ "answer": "Lane B finished; results staged at /tmp/lane-b.md.",
+ }
+ status = await self.settle(result)
+ self.assertEqual(status["state"], "failed")
+ self.assertEqual(status["needs_verify"], ["fan"])
+ fan = self.state_report(status, "fan")
+ self.assertEqual(fan["needs_verify"], True)
+ entry_row = fan["entries"][0]
+ self.assertEqual(entry_row["status"], "error")
+ self.assertEqual(entry_row["needs_verify"], True)
+ self.assertIn("results staged at /tmp/lane-b.md", entry_row["provisional_answer"])
+ self.assertEqual(
+ [event["node"] for event in self.all_events_of(result, "needs_verify")],
+ ["fan"],
+ )
+ # i0 failed with no answer; the entry failed exactly once, so the
+ # needs_verify event stays one-shot and no retry was queued.
+ self.assertEqual(len(self.host.spawn_calls("fan")), 2)
+ self.assertEqual(self.all_events_of(result, "retry"), [])
+
+ @async_test
+ async def test_paused_run_status_carries_the_last_error_and_remedy(self) -> None:
+ # M6: a paused run carries the LAST failed admission/bind error and
+ # a one-line remedy in its status payload — no root-side
+ # archaeology over pending-state lists to learn what failed.
+ self.host.outcomes["src"] = {"status": "done", "answer": "no json here"}
+ self.store_machine(
+ {
+ "run": {"failure_policy": "escalate", "max_parallel": 4},
+ "states": [
+ {
+ "id": "src",
+ "entry": True,
+ "subagent": "worker",
+ "outputs": [{"name": "data", "type": "json"}],
+ },
+ {
+ "id": "dep",
+ "subagent": "worker",
+ "inputs": [{"name": "data", "type": "json", "from": "src.data"}],
+ },
+ ],
+ "transitions": [{"from": "src", "to": "dep"}],
+ }
+ )
+ result = await self.start()
+ status = await self.settle(result)
+ self.assertEqual(status["state"], "paused")
+ self.assertEqual(
+ status["pause_reason"],
+ "input 'data': no JSON object containing output 'data' in the upstream answer",
+ )
+ self.assertIn("state 'dep' failed", status["last_error"])
+ self.assertIn("no JSON object containing output 'data'", status["last_error"])
+ self.assertIn(f"rlm.factory.resume('{result['run_id']}')", status["remedy"])
+
@async_test
async def test_machine_required_input_over_an_errored_source_fails_the_dependent_entry(self) -> None:
# The boundary the authoring reference pins: "their required input
@@ -8005,11 +8547,13 @@ class ScriptedHost:
@staticmethod
def answer_for(name):
- if name.startswith("files-source"):
+ # The spawn labels are run-prefixed (run-scoped names), so the
+ # configured name matches anywhere in the label.
+ if "files-source" in name:
return "```json\n{\"files\": [\"sample.ts\"]}\n```"
- if name.startswith("file-reviewer"):
+ if "file-reviewer" in name:
return "sample.ts: clean"
- if name.startswith("review-aggregator"):
+ if "review-aggregator" in name:
return "```json\n{\"issues\": [], \"clean\": 1}\n```"
return "done"