Skip to content

A self-improving RLM agent for coding workflows and long-running autonomous tasks.

Rust

[factory] escalate resume re-admits the failed entry; worker deaths respawn once (#3631)

Sebastian Müller8d58e5b6f9ec
diff --git a/prime-agent-runtime/src/rlm/factory.py b/prime-agent-runtime/src/rlm/factory.py
index 673a784176..aeed2dd4b5 100644
--- a/prime-agent-runtime/src/rlm/factory.py
+++ b/prime-agent-runtime/src/rlm/factory.py
@@ -1086,6 +1086,17 @@ _RATE_LIMIT_MARKERS = (
"usage limit",
)
+INFRA_VERIFY_QUIET_SECONDS = POLL_TIMEOUT_MS / 1000
+"""Quiet period before the executor re-reads a worker that settled with an
+infra error (one collect-poll slice): the re-check must not race the very
+probe streak that reported the death. The sleep is injected, so tests see
+no wall-clock wait."""
+
+_INFRA_ERROR_MARKERS = (
+ "child worker unreachable",
+ "supervisor link",
+)
+
_FENCED_JSON_RE = re.compile(r"```json\s*(.*?)\s*```", re.DOTALL)
TERMINAL_ENTRY_STATUSES = ("done", "error", "cancelled")
@@ -1096,6 +1107,15 @@ def _is_rate_limit_error(message: str) -> bool:
return any(marker in lowered for marker in _RATE_LIMIT_MARKERS)
+def _is_infra_error(message: str) -> bool:
+ """Heuristic: the supervisor reports worker deaths and lost supervisor
+ links as one-shot error settles (``Child worker unreachable`` after the
+ daemon's unreachable-probe streak, or a ``supervisor link ...`` error
+ from a failed kickoff). Infra, not the task's failure."""
+ lowered = message.lower()
+ return any(marker in lowered for marker in _INFRA_ERROR_MARKERS)
+
+
def _child_name(run_id: str, state_id: str, instance_index: int, attempt: int) -> str:
"""Unique, readable sibling name for one spawned instance (host caps names at 64).
@@ -1342,8 +1362,18 @@ class _NodeInstance:
prompt: str # fully rendered; re-spawns reuse it verbatim
status: str = "pending" # pending | running | done | error | cancelled
attempt: int = 0 # spawn admissions tried for this instance; a rate-limit
- # deferral counts here too, and ``retries`` compares against this same
- # counter, so one transient 429 admission consumes one declared retry.
+ # deferral counts here too, and charged retries compare against
+ # ``attempt - retry_base``, so one transient 429 admission consumes one
+ # declared retry.
+ retry_base: int = 0 # the attempt count at the last retry-budget reset:
+ # an infra respawn or a resume's re-admission grants fresh ``retries``
+ # by moving this base instead of zeroing ``attempt`` (spawn labels
+ # count on it staying monotonic).
+ infra_respawned: bool = False # the instance's one free worker-death
+ # respawn is spent; a resume's re-admission grants a fresh one.
+ capture_failed: bool = False # this settle completed its entry but a
+ # declared output never bound; the preserved answer stays provisional
+ # until the capture is redone or the entry fails permanently.
rate_limit_streak: int = 0 # consecutive rate-limited admissions in the
# control loop's backoff path; a successful spawn resets it, and
# BACKOFF_MAX_ATTEMPTS in a row fails the node. An admission-phase
@@ -1636,8 +1666,9 @@ class FactoryExecutor:
"usage": self._usage_report(run),
}
# The M4 oversight signal: every state whose latest terminal entry
- # carried a provisional answer (a child exit) needs verification —
- # root-visible without ledger archaeology.
+ # carried a provisional answer (a child exit, or a settle whose
+ # declared output never bound) needs verification — root-visible
+ # without ledger archaeology.
needs_verify = [
state_id
for state_id in run.order
@@ -1653,9 +1684,10 @@ class FactoryExecutor:
if run.last_error is not None:
payload["last_error"] = run.last_error
payload["remedy"] = (
- f"resume with await rlm.factory.resume('{run.run_id}') to continue the "
- "remaining states; the failed state stays error - fix its subagent and "
- "start a fresh run to redo it"
+ f"resume with await rlm.factory.resume('{run.run_id}') to re-admit the "
+ "failed state's entry as a fresh attempt (same rendered prompt, no extra "
+ "max_entries); author the state with failure_policy 'continue' to run its "
+ "dependents past a failure instead"
)
return payload
@@ -1685,9 +1717,14 @@ class FactoryExecutor:
async def resume(self, run_id: str) -> dict[str, Any]:
"""Resume a paused run (escalate, budget, or max_transitions pause).
- A budget or max_transitions pause is reported once per run:
- resuming after it is an explicit operator decision and no further
- budget pauses fire. Raises ``ValueError`` when the run is not paused.
+ An escalate pause holds its failed entry unsettled, so resume
+ re-admits that entry first: a fresh attempt of the same rendered
+ prompt in the same entry slot -- no extra ``max_entries`` spend,
+ fresh ``retries``. The failure only ever propagates to dependents
+ under ``failure_policy: "continue"``. A budget or max_transitions
+ pause is reported once per run: resuming after it is an explicit
+ operator decision and no further budget pauses fire. Raises
+ ``ValueError`` when the run is not paused.
"""
run = self._require_run(run_id)
if run.state != "paused":
@@ -1702,8 +1739,12 @@ class FactoryExecutor:
run.pause_reason = None
self._touch(run)
self._event(run, "resumed", detail="resumed by caller")
- # Evaluate settles first: paused runs may still carry transitions to
- # fire (escalate) before anything can be admitted.
+ # An escalate pause holds its failed entry unsettled: re-admit it
+ # BEFORE anything evaluates or admits, so the retry -- not the
+ # failure's propagation -- is what resume runs.
+ self._readmit_held_entries(run)
+ # Evaluate settles next: a max_transitions pause left transitions
+ # mid-settle, and resume continues after the ones that fired.
await self._evaluate_settles(run)
# allow_backoff=False: like run(), resume() must never sleep inside
# the calling model turn; rate-limited admissions defer to the loop.
@@ -1719,6 +1760,49 @@ class FactoryExecutor:
"pending": self._pending_state_ids(run),
}
+ def _readmit_held_entries(self, run: FactoryRun) -> None:
+ """Re-admit every entry an escalate pause holds unsettled.
+
+ An escalate failure never queues its settle (the pause lands before
+ the failure propagates), so a held entry is exactly ``error`` with
+ ``is_settle`` unset -- that predicate is the one held-entry
+ definition. Resume turns each held entry back into a fresh attempt
+ of the SAME entry slot: the errored and cancelled instances re-run
+ their rendered prompt (``attempt`` keeps counting, so spawn labels
+ stay unique), ``max_entries`` is not spent again, and the
+ ``retries`` budget restarts (``retry_base`` moves to the current
+ attempt count). A binding-failure entry has no instances: it
+ re-binds through the readiness pass, and a still-broken upstream
+ settles it into the same pause again.
+ """
+ for state_id in run.order:
+ state = run.states[state_id]
+ for entry in state.entries:
+ if entry.status != "error" or entry.is_settle:
+ continue
+ entry.status = "running" if entry.instances else "pending"
+ entry.error = None
+ entry.needs_verify = False
+ state.error = None
+ for instance in entry.instances:
+ if instance.status not in ("error", "cancelled"):
+ continue # running and done siblings keep their state
+ instance.status = "pending"
+ instance.error = None
+ instance.answer = None
+ instance.provisional = False
+ instance.capture_failed = False
+ instance.rate_limit_streak = 0
+ instance.retry_base = instance.attempt
+ instance.infra_respawned = False
+ self._event(
+ run,
+ "readmitted",
+ node=state_id,
+ entry=entry.index,
+ detail="resume re-admits the failed entry (same rendered prompt)",
+ )
+
# -- graph, watch, host activity ------------------------------------------
def _state_report(
@@ -2789,6 +2873,26 @@ class FactoryExecutor:
) -> None:
if instance.status != "running":
return # cancelled (stop/fail_fast) while the collect was in flight
+ if result.status == "error" and _is_infra_error(result.error or ""):
+ # A worker death or a lost supervisor link is infra, not the
+ # task's failure: the executor verifies it itself (a short
+ # quiet period, then one re-read through the supervisor)
+ # instead of failing the entry on the first error row and
+ # flagging the parent to do the verifying.
+ fresh = await self._recheck_infra_death(instance, result)
+ if fresh is None:
+ return # a stop/fail_fast pass claimed it, or the worker answered
+ result = fresh
+ if (
+ entry.status == "running"
+ and not instance.infra_respawned
+ and result.status == "error"
+ and _is_infra_error(result.error or "")
+ ):
+ await self._respawn_infra_dead(run, state, entry, instance, result)
+ return
+ # otherwise: a second verified death (or an entry a sibling
+ # failed first) takes the normal retries/policy path below
instance.duration_ms = result.duration_ms
instance.tool_uses = result.tool_use_count or 0
run.settle_count += 1
@@ -2813,6 +2917,7 @@ class FactoryExecutor:
if exit_answer:
instance.answer = exit_answer
instance.provisional = True
+ instance.capture_failed = False # the preserved text is an exit now
# Child failures retry (same rendered prompt, attempts+1) while
# attempts remain; then the entry failure_policy applies.
await self._apply_instance_failure(run, state, entry, instance, child_reason, retry=True)
@@ -2841,6 +2946,7 @@ class FactoryExecutor:
(result.answer_text or result.answer_preview or "")[:ANSWER_BINDING_CAP] or None
)
instance.provisional = False # a real settle supersedes any exit capture
+ instance.capture_failed = False
self._event(
run,
"settled",
@@ -2861,11 +2967,122 @@ class FactoryExecutor:
stage="arrived",
)
if entry.status == "running" and entry.instances and all(i.status == "done" for i in entry.instances):
- entry.status = "done"
entry.answer = self._entry_answer(entry)
- await self._capture_outputs(run, state, entry)
+ capture_failure = await self._capture_outputs(run, state, entry)
+ if entry.status != "running":
+ return # a stop()/fail_fast pass claimed the entry mid-capture
+ if capture_failure is not None:
+ # A declared output that never bound after the capture
+ # retry is the producing entry's task failure, not a silent
+ # valueless settle: no settle is queued (a dependent never
+ # enters on an unbound required input), retries then
+ # failure_policy apply, and the answer stays provisional
+ # for needs_verify if the entry fails permanently.
+ instance.provisional = True
+ instance.capture_failed = True
+ await self._apply_instance_failure(
+ run, state, entry, instance, capture_failure, retry=True
+ )
+ return
+ entry.status = "done"
self._queue_settle(run, state, entry)
+ async def _recheck_infra_death(self, instance: _NodeInstance, result: Any) -> Any:
+ """Liveness-verify one infra-flavoured error settle before it is
+ applied.
+
+ The supervisor reports a dead worker as a one-shot error settle, so
+ the executor re-reads the worker after a short quiet period instead
+ of trusting the first row. Returns the fresh result to apply -- a
+ settle that landed meanwhile supersedes the error, and the same
+ error read back means the death is verified -- or ``None`` when a
+ concurrent pass (stop(), fail_fast) claimed the instance or the
+ worker answered alive (the loop keeps collecting it).
+ """
+ from . import collect
+
+ child_id = instance.child_id
+ await self._sleep_fn(INFRA_VERIFY_QUIET_SECONDS)
+ if instance.status != "running" or instance.child_id != child_id:
+ return None # a stop, fail_fast, or resumed lane claimed the instance
+ results = await collect([child_id], timeout_ms=0)
+ if instance.status != "running" or instance.child_id != child_id:
+ return None
+ row = next((r for r in results if r.rlm_child_id == child_id), None)
+ if row is not None and not row.settled:
+ return None # the worker answered: alive, keep collecting it
+ # No fresh row at all means the child vanished from the supervisor
+ # (a hand delete): as dead as the error said, so the original
+ # settle applies through the normal failure path.
+ return row or result
+
+ async def _respawn_infra_dead(
+ self, run: FactoryRun, state: _StateRun, entry: _StateEntry, instance: _NodeInstance, result: Any
+ ) -> None:
+ """Re-spawn one worker the executor verified dead, free of charge.
+
+ Infra is not the task's failure: the dead attempt counts as a
+ settle (its ledger row keeps the error), but the instance is
+ re-queued with the same rendered prompt -- spending neither a
+ ``retries`` slot nor a fresh ``max_entries`` entry -- and the dead
+ attempt's exit text is dropped, so no needs-verify mark is raised
+ for work the respawn redoes. ``infra_respawned`` keeps this to ONE
+ free respawn; a second verified death falls through to the normal
+ retries/failure_policy path.
+ """
+ from . import delete_subagent
+
+ old_child = instance.child_id or ""
+ # The worker may be alive-but-unreachable rather than dead: retire
+ # it so the respawn cannot run beside its zombie in the same
+ # worktree. Best-effort and bounded — the host's delete kills under
+ # the daemon's own timeout, the rest is registry bookkeeping — so
+ # an already-gone child or a dead bridge must not wedge the
+ # respawn; the supervisor-side zombie sweep this pairs with lands
+ # in PR 2 of the [factory] group.
+ try:
+ await delete_subagent(old_child)
+ except Exception:
+ pass
+ if instance.status != "running" or run.state not in ("running", "paused"):
+ return # a stop()/fail_fast pass claimed the instance mid-delete
+ # A sibling may have paused the run in this same collect batch.
+ # Preserve the verified death as pending; resume admits it for free
+ # instead of collecting the deleted child again.
+ instance.duration_ms = result.duration_ms
+ instance.tool_uses = result.tool_use_count or 0
+ run.settle_count += 1
+ run.tool_use_total += instance.tool_uses
+ instance.infra_respawned = True
+ instance.retry_base += 1
+ instance.status = "pending"
+ instance.error = None
+ instance.answer = None
+ instance.provisional = False
+ instance.capture_failed = False
+ reason = result.error or "child worker unreachable"
+ self._event(
+ run,
+ "settled",
+ node=state.state_id,
+ entry=entry.index,
+ instance=instance.index,
+ status="error",
+ error=reason,
+ duration_ms=instance.duration_ms,
+ )
+ self._event(
+ run,
+ "infra_respawn",
+ node=state.state_id,
+ entry=entry.index,
+ instance=instance.index,
+ detail=(
+ f"worker verified dead ({reason}); re-spawning the same rendered prompt "
+ "once (no retry, no max_entries)"
+ ),
+ )
+
def _entry_answer(self, entry: _StateEntry) -> str | None:
"""Captured answer for binding: one preview, or all instances joined."""
answers = [instance.answer for instance in entry.instances if instance.status == "done" and instance.answer]
@@ -2903,7 +3120,7 @@ class FactoryExecutor:
entry.outputs = outputs
entry.output_errors = errors
- async def _capture_outputs(self, run: FactoryRun, state: _StateRun, entry: _StateEntry) -> None:
+ async def _capture_outputs(self, run: FactoryRun, state: _StateRun, entry: _StateEntry) -> str | None:
"""Capture the settle's outputs, retrying the capture once.
A settle can race the child's answer capture on the host side (a
@@ -2911,10 +3128,13 @@ class FactoryExecutor:
that binds no value from the first capture is exactly the M2
silent-failure class: the downstream would run on nothing. When a
declared output binds no value, the settled children are
- re-collected ONCE (a settled child keeps its result until deleted)
+ refreshed in one bounded capture window (a settled child keeps its
+ result until deleted; missing text waits for the worker hand-off),
and the capture re-runs over the refreshed answers; a port that
still binds nothing records a deterministic capture failure in the
- ledger (``output_capture_failed``) instead of passing silently.
+ ledger (``output_capture_failed``) and fails the entry -- its
+ joined port errors return as the task-failure reason, ``None``
+ when every declared port bound a value.
"""
self._capture_ports(state, entry)
declared = state.spec.get("outputs") or []
@@ -2925,10 +3145,11 @@ class FactoryExecutor:
if not declared or not any(
out.get("name") not in (entry.outputs or {}) for out in declared
):
- return # every declared port bound a value
+ return None # every declared port bound a value
refreshed = await self._refresh_entry_answers(run, state, entry)
if refreshed:
self._capture_ports(state, entry)
+ failures = []
for out in declared:
name = out.get("name")
error = (entry.output_errors or {}).get(name)
@@ -2944,10 +3165,12 @@ class FactoryExecutor:
error=error,
detail=f"output {name!r} did not bind after one capture retry",
)
+ failures.append(error)
+ return "; ".join(failures) or None
async def _refresh_entry_answers(self, run: FactoryRun, state: _StateRun, entry: _StateEntry) -> bool:
- """Re-collect the entry's settled children once; refresh any longer
- captured answer. Returns whether any answer changed."""
+ """Refresh settled answers, waiting one bounded poll window for a
+ missing hand-off. Returns whether any answer changed."""
from . import collect
child_ids = [
@@ -2957,30 +3180,45 @@ class FactoryExecutor:
]
if not child_ids:
return False
- try:
- results = await collect(child_ids, timeout_ms=0)
- except Exception as exc:
- self._event(
- run,
- "output_capture_failed",
- node=state.state_id,
- entry=entry.index,
- error=f"capture retry could not re-collect the settled children: {exc}",
- detail="the first capture stands",
- )
- return False
- by_child = {result.rlm_child_id: result for result in results}
+ # An idle worker may expose its settle before handing off its final
+ # assistant text. Give that missing capture one bounded poll window;
+ # existing nonempty (but invalid) output still gets only one refresh.
+ deadline = self._now_fn() + POLL_TIMEOUT_MS / 1000
changed = False
- for instance in entry.instances:
- result = by_child.get(instance.child_id or "")
- if result is None:
- continue
- answer = (result.answer_text or result.answer_preview or "")[:ANSWER_BINDING_CAP] or None
- if answer and (instance.answer is None or len(answer) > len(instance.answer)):
- instance.answer = answer
- changed = True
- if changed:
- entry.answer = self._entry_answer(entry)
+ while entry.status == "running":
+ try:
+ results = await collect(child_ids, timeout_ms=0)
+ except Exception as exc:
+ self._event(
+ run,
+ "output_capture_failed",
+ node=state.state_id,
+ entry=entry.index,
+ error=f"capture retry could not re-collect the settled children: {exc}",
+ detail="the first capture stands",
+ )
+ return changed
+ if entry.status != "running":
+ return changed # stop/fail_fast claimed the entry during collect
+ by_child = {result.rlm_child_id: result for result in results}
+ for instance in entry.instances:
+ result = by_child.get(instance.child_id or "")
+ if result is None:
+ continue
+ answer = (result.answer_text or result.answer_preview or "")[:ANSWER_BINDING_CAP] or None
+ if answer and (instance.answer is None or len(answer) > len(instance.answer)):
+ instance.answer = answer
+ changed = True
+ if changed:
+ entry.answer = self._entry_answer(entry)
+ missing = any(
+ instance.status == "done" and instance.child_id is not None and not instance.answer
+ for instance in entry.instances
+ )
+ remaining = deadline - self._now_fn()
+ if not missing or remaining <= 0:
+ return changed
+ await self._sleep_fn(min(0.05, remaining))
return changed
async def _apply_instance_failure(
@@ -3007,8 +3245,15 @@ class FactoryExecutor:
# count the pending instance as in-flight, so the control loop
# would never finish the run. The failed instance settles as an
# error instead; the policy call below is a no-op on a terminal
- # entry.
- if retry and instance.attempt <= retries and entry.status == "running":
+ # entry. ``attempt - retry_base`` charges only the admissions
+ # since the last budget reset: an infra respawn and a resume's
+ # re-admission restart the declared retries without zeroing the
+ # spawn counter the labels count on.
+ if (
+ retry
+ and instance.attempt - instance.retry_base <= retries
+ and entry.status == "running"
+ ):
instance.status = "pending"
instance.error = None
self._event(
@@ -3047,13 +3292,27 @@ class FactoryExecutor:
# status-level needs_verify signal hand the operator the
# revival-race decision instead of a silent drop.
entry.needs_verify = True
- self._event(
- run,
- "needs_verify",
- node=state.state_id,
- entry=entry.index,
- detail="child exit captured a provisional answer - verify the remote state",
- )
+ # The detail names what the mark preserves: a settle whose
+ # declared output never bound, a respawned worker's last words
+ # after its own death, or a plain child exit. The capture mark
+ # is checked first: ``infra_respawned`` outlives its respawn,
+ # so a respawned attempt that later settles and fails capture
+ # preserves the capture's provenance, not the old death's.
+ respawned = any(instance.infra_respawned for instance in entry.instances)
+ capture_failed = any(instance.capture_failed for instance in entry.instances)
+ if capture_failed:
+ detail = (
+ "output capture failed; the settled answer is preserved as a "
+ "provisional answer - fix the declared output or verify by hand"
+ )
+ elif respawned:
+ detail = (
+ "worker verified dead by the executor after its automatic respawn; "
+ "last assistant text preserved as a provisional answer"
+ )
+ else:
+ detail = "child exit captured a provisional answer - verify the remote state"
+ self._event(run, "needs_verify", node=state.state_id, entry=entry.index, detail=detail)
if entry.status in TERMINAL_ENTRY_STATUSES:
return # the policy already ran for this entry
policy = state.spec.get("failure_policy", RUN_FAILURE_POLICY_DEFAULT)
@@ -3077,9 +3336,14 @@ class FactoryExecutor:
instance=instance.index,
detail="entry failed before admission",
)
- # The failed entry settles too: guard-less transitions (the compiled
- # dag's depends_on edges) fire from error settles so dependents run.
- self._queue_settle(run, state, entry)
+ if policy != "escalate":
+ # The failed entry settles: guard-less transitions (the compiled
+ # dag's depends_on edges) fire from error settles so dependents
+ # run. Escalate instead HOLDS the failed entry unsettled -- the
+ # pause lands before the failure propagates, no downstream entry
+ # budget is spent on it, and resume() re-admits the entry as a
+ # fresh attempt.
+ self._queue_settle(run, state, entry)
if run.state != "running":
# stop() (or another transition) owns the run state now; keep the
# entry's error but do not overwrite the final state.
@@ -3110,7 +3374,8 @@ class FactoryExecutor:
await self._milestone(
run,
"paused",
- f"state {state.state_id} failed: {reason}; resume with await rlm.factory.resume('{run.run_id}')",
+ f"state {state.state_id} failed: {reason}; resume with await rlm.factory.resume('{run.run_id}') "
+ "to re-admit the failed entry as a fresh attempt",
node=state.state_id,
)
@@ -3597,7 +3862,11 @@ async def stop_factory(run_id: str) -> dict[str, Any]:
async def resume_factory(run_id: str) -> dict[str, Any]:
- """Resume a paused run (escalate, budget, or max_transitions pause)."""
+ """Resume a paused run (escalate, budget, or max_transitions pause).
+
+ An escalate pause re-admits the failed entry as a fresh attempt of the
+ same rendered prompt first; see ``FactoryExecutor.resume``.
+ """
require_factory_enabled()
return await default_factory_executor().resume(run_id)
@@ -4726,7 +4995,8 @@ parent session after the run ends. Both worked examples bound their
emitted payloads in the prompt — a capped findings list here, a capped
file list in the review-sweep example — because the executor's capture
cap still bounds very large payloads: an unbounded payload truncates at
-the cap and fails to bind.
+the cap, fails to capture, and fails the producing state through its
+`retries` and `failure_policy` instead of settling valueless.
## Authoring reference
@@ -4753,11 +5023,17 @@ the cap and fails to bind.
`{input_name}` placeholders (one pass; inputs without a placeholder are
appended in a trailing `## Inputs` section). A required input whose source
has not settled yet keeps the entry pending; over a settled source that
- offers no value (an errored settle, a port the settle captured no value
- for, or a JSON capture failure) it fails the dependent entry, while
+ offers no value (an errored settle, or a settle that captured nothing for
+ the port) it fails the dependent entry, while
`"optional": true` binds a null sentinel in every no-value case (a
source that offers no value is not a value, so the dependent that
- declared the input optional proceeds). A required self-input is
+ declared the input optional proceeds). A state's own declared outputs
+ capture from its settle's full answer: a declared port that binds no
+ value — no parseable fenced JSON for it, or a payload past the
+ executor's capture cap — is retried once and then FAILS the producing
+ entry as a task failure (`retries`, then `failure_policy`, apply, and
+ the answer stays preserved under `needs_verify`), so a valueless
+ settle never reaches a dependent's bind. A required self-input is
rejected at validation —
`state X input 'name' cannot require itself: mark the self-input optional
- a required one can never bind on the state's first entry` — while an
@@ -4774,11 +5050,12 @@ the cap and fails to bind.
dotted path into a `json` output; `eq`/`ne` compare JSON-strictly (a
boolean never equals a number), comparison ops need a numeric value,
`contains` a non-empty list, `exists` no value, and a missing or
- unparseable port fails every op except `exists`. A failed settle still
- fires guard-less transitions, so dependents under `continue` run; their
- required input over the failed source then fails the dependent entry,
- while an optional input over the failed source binds the null sentinel
- and the dependent proceeds.
+ unparseable port fails every op except `exists`. Under `continue` a
+ failed settle still fires guard-less transitions, so dependents run;
+ their required input over the failed source then fails the dependent
+ entry, while an optional input over the failed source binds the null
+ sentinel and the dependent proceeds. Under the default `escalate` the
+ failure never propagates that far (see bounds and policies).
- **Cycles are legal**: there is no acyclicity requirement — self-loops and
back edges validate. The one rule is an entry state somewhere; a dag
whose every node depends on another compiles to no entry states and is
@@ -4806,9 +5083,19 @@ the cap and fails to bind.
(0..10, same rendered prompt), `budget_ms` (admission to settlement;
exceeding it fails the attempt without a retry), and `failure_policy` —
`fail_fast` (cancel every child, run failed), `continue` (entry stays
- errored; the run finishes and reports failed if any state errored), or
- `escalate` (the default: pause the run; resuming is the operator's
- decision).
+ errored; the run finishes and reports failed if any state errored; its
+ failed settle fires guard-less transitions so dependents run), or
+ `escalate` (the default: pause the run BEFORE the failed settle
+ propagates — no transition fires and no downstream entry is spent — and
+ `resume` re-admits the failed entry as a fresh attempt: same rendered
+ prompt, same entry slot, fresh `retries`; a binding failure over a
+ broken upstream re-binds and pauses again, so stop and fix the upstream
+ or author the dependent's state with `continue`). A child whose worker
+ died or whose supervisor link was lost is infra, not the task's
+ failure: the executor re-checks it after a short quiet period, then
+ re-spawns the same prompt once without spending `retries` or
+ `max_entries`; a second death fails through `retries`/`failure_policy`
+ like any task failure.
- **Retry re-renders the same prompt into a changed world**: a retry runs
the same prompt after an earlier attempt already changed the external
state it works on, so a prompt that mutates external state — a fixing
@@ -4890,9 +5177,12 @@ status["events"] # trailing ledger: spawned, settled, answer_captured,
# transition_fired, node_error, milestone, ...
status["usage"] # spawns, settled, tool_uses, max_parallel, max_children, running,
# transitions_fired
-status["needs_verify"] # (when set) states whose child EXITED carrying a
- # provisional answer - verify the remote state; the
- # entry row carries the provisional_answer preview
+status["needs_verify"] # (when set) states whose entry keeps a
+ # provisional answer: a child exit with text, or a
+ # settled answer whose declared output failed to
+ # capture (worker deaths are liveness-checked and
+ # re-spawned by the executor first) - the entry row
+ # carries the provisional_answer preview
status["pause_reason"] # (paused) why the run paused; with "last_error" (the
status["last_error"] # last admission/bind failure) and a one-line "remedy"
```
@@ -4935,7 +5225,8 @@ watched = await rlm.factory.watch(result["run_id"], 30)
conversation once per kind per run, the pause notices with the resume
call spelled out — a paused run does not need polling to be noticed.
- `stop(run_id)` cancels every running child of the run (idempotent);
- `resume(run_id)` continues a paused run and raises on a non-paused one.
+ `resume(run_id)` continues a paused run and raises on a non-paused one;
+ after an escalate pause it first re-admits the failed entry.
- The activity lane the daemon and TUI speak is camelCase on the wire
(`runId`, `specId`, `timeoutMs`); the kernel API here
(`rlm.factory.*`) is snake_case.
diff --git a/prime-agent-runtime/test/test_factory.py b/prime-agent-runtime/test/test_factory.py
index 54a6e347ff..1c973bfd59 100644
--- a/prime-agent-runtime/test/test_factory.py
+++ b/prime-agent-runtime/test/test_factory.py
@@ -2493,10 +2493,18 @@ class FactoryHelpTest(unittest.TestCase):
# The worked examples bound their emitted payloads — captured answers
# are capped previews (~160-200 chars), so an unbounded json payload
- # would truncate at the cap and fail to bind.
+ # would truncate at the cap, fail to capture, and fail its state.
self.assertIn('findings": ["at most three one-line findings"]', doc)
self.assertIn("capped at the eight most relevant", doc)
- self.assertIn("an unbounded payload truncates at the cap and fails to bind", flat)
+ self.assertIn(
+ "an unbounded payload truncates at the cap, fails to capture, "
+ "and fails the producing state",
+ flat,
+ )
+ # The output-capture contract: a declared port that never binds
+ # fails the producing entry as a task failure.
+ self.assertIn("is retried once and then FAILS the producing", flat)
+ self.assertIn("the answer stays preserved under `needs_verify`", flat)
# The Discovery section ships the machine library as present (this
# branch merges it): the bundled location, the seed names, the CLI
@@ -3578,7 +3586,12 @@ class FactoryExecutorTest(_ExecutorTestCase):
self.assertEqual([call["prompt"] for call in self.host.spawn_calls("b")], ["Process 7."])
@async_test
- async def test_bad_json_input_fails_node_without_spawning(self) -> None:
+ async def test_unparseable_output_fails_the_producer_and_never_spawns_the_dependent(self) -> None:
+ # a's declared json output never parses: the capture failure fails a
+ # itself (defect C) instead of settling it valueless, so b's required
+ # input over the errored source fails at binding without ever
+ # spawning; c still ran and finished, so the run completes but
+ # reports failed.
self.host.outcomes["a"] = {"status": "done", "answer": "not json at all"}
self.store_factory(
{
@@ -3597,12 +3610,13 @@ class FactoryExecutorTest(_ExecutorTestCase):
)
result = await self.start()
status = await self.settle(result)
- # b failed at binding (continue policy): never spawned, marked error;
- # c still ran and finished, so the run completes but reports failed.
self.assertEqual(self.host.spawn_calls("b"), [])
+ a = self.node_status(status, "a")
+ self.assertEqual(a["status"], "error")
+ self.assertIn("no JSON object containing output 'result'", a["error"])
b = self.node_status(status, "b")
self.assertEqual(b["status"], "error")
- self.assertIn("no JSON object containing output", b["error"])
+ self.assertIn("is unavailable (latest settle status 'error')", b["error"])
self.assertEqual(self.node_status(status, "c")["status"], "done")
self.assertEqual(status["state"], "failed")
@@ -3819,6 +3833,61 @@ class FactoryExecutorTest(_ExecutorTestCase):
# null, not the no-value sentinel path.
self.assertIn("- data: null", self.host.spawn_calls("dep")[0]["prompt"])
+ @async_test
+ async def test_uncaptured_output_fails_the_producing_entry_instead_of_the_dependent(self) -> None:
+ # Defect C: a done settle whose declared output never bound (no
+ # parseable value after the one capture retry) used to settle the
+ # entry done and queue the settle -- the dependent's required input
+ # then failed over the valueless source, spending its entry budget
+ # on the producer's failure. The capture failure now fails the
+ # PRODUCING entry as a task failure at the origin: retries, then
+ # failure_policy, apply; no settle is queued, so escalate pauses at
+ # the producer and the dependent never enters.
+ self.host.outcomes["setup"] = {"status": "done", "answer": "no json here"}
+ self.store_machine(
+ {
+ "states": [
+ {
+ "id": "setup",
+ "entry": True,
+ "subagent": "worker",
+ "outputs": [{"name": "config", "type": "json"}],
+ "retries": 1,
+ },
+ {
+ "id": "build",
+ "subagent": "worker",
+ "inputs": [{"name": "config", "type": "json", "from": "setup.config"}],
+ "max_entries": 4,
+ },
+ ],
+ "transitions": [{"from": "setup", "to": "build"}],
+ }
+ )
+ result = await self.start()
+ paused = await self.settle(result)
+ self.assertEqual(paused["state"], "paused")
+ setup = self.state_report(paused, "setup")
+ self.assertEqual(setup["status"], "error")
+ self.assertIn("config", setup["error"])
+ # the failed capture spent the one retry re-spawning the same
+ # prompt; the second capture failure is permanent
+ self.assertEqual(len(self.host.spawn_calls("setup")), 2)
+ self.assertEqual(len(self.all_events_of(result, "retry")), 1)
+ self.assertEqual(len(self.all_events_of(result, "output_capture_failed")), 2)
+ # the dependent never entered on the unbound required input
+ self.assertEqual(self.state_report(paused, "build")["entries_used"], 0)
+ self.assertEqual(self.host.spawn_calls("build"), [])
+ self.assertEqual(self.all_events_of(result, "transition_fired"), [])
+ # the settled answer is preserved as provisional, so the permanent
+ # failure surfaces needs_verify with the answer for review
+ self.assertEqual(paused["needs_verify"], ["setup"])
+ self.assertIn("no json here", setup["entries"][0]["provisional_answer"])
+ # the paused milestone tells the operator resume re-admits the entry
+ paused_notice = next(n for n in self.host.notices if n["kind"] == "paused")
+ self.assertIn("state setup failed", paused_notice["detail"])
+ self.assertIn("re-admit", paused_notice["detail"])
+
@async_test
async def test_capture_failure_names_the_binding_cap_and_records_the_event(self) -> None:
# A declared json output bigger than the executor's binding cap
@@ -4147,38 +4216,103 @@ class FactoryExecutorTest(_ExecutorTestCase):
self.assertEqual(self.host.notice_kinds(), ["failed"])
@async_test
- async def test_escalate_pauses_notifies_and_resume_continues(self) -> None:
- self.host.outcomes["a"] = {"status": "error", "error": "boom"}
- self.store_factory(
+ async def test_escalate_resume_readmits_the_failed_entry(self) -> None:
+ # Defect A (live run fa1a8f1c): resume after an escalate pause fired
+ # the ERRORED settle forward -- the guard-less transition entered the
+ # dependent, its required input failed to bind over the errored
+ # source, and it spent one of its max_entries entries on nothing.
+ # The escalate pause now lands BEFORE the failure propagates, and
+ # resume re-admits the failed entry as a fresh attempt of the same
+ # rendered prompt in the same entry slot.
+ self.host.child_outcomes["child-1"] = {"status": "error", "error": "boom"}
+ self.host.child_outcomes["child-2"] = {"status": "done", "answer": "impl-done"}
+ self.store_machine(
{
- "nodes": [
- {"id": "a", "subagent": "worker"},
- {"id": "b", "subagent": "worker", "depends_on": ["a"]},
+ "states": [
+ {
+ "id": "implement",
+ "entry": True,
+ "subagent": "worker",
+ "outputs": [{"name": "result", "type": "text"}],
+ },
+ {
+ "id": "review",
+ "subagent": "worker",
+ "inputs": [{"name": "result", "type": "text", "from": "implement.result"}],
+ "max_entries": 4,
+ },
],
+ "transitions": [{"from": "implement", "to": "review"}],
}
)
result = await self.start()
paused = await self.settle(result)
self.assertEqual(paused["state"], "paused")
- self.assertEqual(self.node_status(paused, "a")["status"], "error")
- self.assertEqual(self.node_status(paused, "b")["status"], "pending")
- self.assertEqual(self.host.spawn_calls("b"), [])
+ self.assertEqual(self.node_status(paused, "implement")["status"], "error")
+ self.assertEqual(self.node_status(paused, "review")["entries_used"], 0)
self.assertIn("paused", self.host.notice_kinds())
- # resume() restarts the run: b is ready (a is terminal) and finishes.
+ # resume re-admits the failed entry: a fresh attempt of the same
+ # rendered prompt, the same entry slot -- no extra max_entries.
resumed = await rlm_module.rlm.factory.resume(result["run_id"])
self.assertEqual(resumed["state"], "running")
final = await self.settle(result)
- self.assertEqual(final["state"], "failed") # a errored, b done
- self.assertEqual(self.node_status(final, "b")["status"], "done")
- self.assertEqual(self.host.notice_kinds(), ["paused", "failed"])
+ self.assertEqual(final["state"], "done")
+ implement = self.node_status(final, "implement")
+ self.assertEqual(implement["status"], "done")
+ self.assertEqual(implement["entries_used"], 1)
+ self.assertEqual([entry["status"] for entry in implement["entries"]], ["done"])
+ self.assertEqual(len(self.host.spawn_calls("implement")), 2)
+ review = self.node_status(final, "review")
+ self.assertEqual(review["status"], "done")
+ self.assertEqual(review["entries_used"], 1)
+ self.assertIn("impl-done", self.host.spawn_calls("review")[0]["prompt"])
+ self.assertEqual(self.host.notice_kinds(), ["paused", "finished"])
# a second resume on the finished run must refuse
with self.assertRaisesRegex(ValueError, "not paused"):
await rlm_module.rlm.factory.resume(result["run_id"])
+ @async_test
+ async def test_escalate_pause_spends_no_downstream_entry(self) -> None:
+ # The pre-propagation pause: the errored settle never fires the
+ # guard-less transition, so the dependent's entry budget is never
+ # spent on the failure. Resume re-admits the failed entry, and its
+ # own failure re-pauses the run with the same reason.
+ self.host.outcomes["implement"] = {"status": "error", "error": "boom"}
+ self.store_machine(
+ {
+ "states": [
+ {
+ "id": "implement",
+ "entry": True,
+ "subagent": "worker",
+ "outputs": [{"name": "result", "type": "text"}],
+ },
+ {
+ "id": "review",
+ "subagent": "worker",
+ "inputs": [{"name": "result", "type": "text", "from": "implement.result"}],
+ "max_entries": 4,
+ },
+ ],
+ "transitions": [{"from": "implement", "to": "review"}],
+ }
+ )
+ result = await self.start()
+ paused = await self.settle(result)
+ self.assertEqual(paused["state"], "paused")
+ await rlm_module.rlm.factory.resume(result["run_id"])
+ second = await self.settle(result)
+ self.assertEqual(second["state"], "paused")
+ self.assertEqual(second["pause_reason"], "boom")
+ self.assertEqual(self.node_status(second, "review")["entries_used"], 0)
+ self.assertEqual(self.all_events_of(result, "transition_fired"), [])
+ implement = self.node_status(second, "implement")
+ self.assertEqual(implement["entries_used"], 1)
+ self.assertEqual(len(self.host.spawn_calls("implement")), 2)
+
@async_test
async def test_resume_defers_rate_limited_admissions(self) -> None:
- self.host.outcomes["a"] = {"status": "error", "error": "boom"}
- self.host.rate_limit_first["b"] = 2
+ self.host.child_outcomes["child-1"] = {"status": "error", "error": "boom"}
self.store_factory(
{
"nodes": [
@@ -4190,6 +4324,9 @@ class FactoryExecutorTest(_ExecutorTestCase):
result = await self.start()
paused = await self.settle(result)
self.assertEqual(paused["state"], "paused")
+ # The rate limit hits a's re-admission, not b: escalate holds the
+ # failed entry unsettled and resume() re-admits it first.
+ self.host.rate_limit_first["a"] = 3
sleep_mark = len(self.sleeps.sleeps)
resumed = await rlm_module.rlm.factory.resume(result["run_id"])
# resume() must not sleep in the calling turn: the rate-limited
@@ -4197,11 +4334,14 @@ class FactoryExecutorTest(_ExecutorTestCase):
self.assertEqual(resumed["started"], [])
self.assertEqual(self.sleeps.sleeps[sleep_mark:], [])
final = await self.settle(result)
- self.assertEqual(final["state"], "failed") # a errored, b done
+ self.assertEqual(final["state"], "done") # a redone, b done
+ self.assertEqual(self.node_status(final, "a")["status"], "done")
self.assertEqual(self.node_status(final, "b")["status"], "done")
- # b needed 3 admissions total: one deferred at resume, then one
- # rate-limited retry inside the loop's backoff, then the success.
- self.assertEqual(len(self.host.spawn_calls("b")), 3)
+ # a needed 4 admissions total: the failed first, one deferred at
+ # resume, one rate-limited retry inside the loop's backoff, then
+ # the success; b then runs once.
+ self.assertEqual(len(self.host.spawn_calls("a")), 4)
+ self.assertEqual(len(self.host.spawn_calls("b")), 1)
self.assertEqual(self.sleeps.sleeps[sleep_mark:], [1.0])
@async_test
@@ -4209,7 +4349,9 @@ class FactoryExecutorTest(_ExecutorTestCase):
# A run that pauses, resumes, and pauses again must record the second
# pause in the ledger even though the parent notice fires once per
# kind: status() is the only reader of the repeat.
- self.host.outcomes["a"] = {"status": "error", "error": "a boom"}
+ # a fails once and its re-admitted retry succeeds, so the second
+ # pause is b's own failure -- the milestone the repeat must record.
+ self.host.child_outcomes["child-1"] = {"status": "error", "error": "a boom"}
self.host.outcomes["b"] = {"status": "error", "error": "b boom"}
self.store_machine(
{
@@ -4259,6 +4401,168 @@ class FactoryExecutorTest(_ExecutorTestCase):
self.assertEqual(self.node_status(status, "a")["status"], "error")
self.assertEqual(status["state"], "failed")
+ # -- worker deaths (infra) --------------------------------------------------------
+
+ @async_test
+ async def test_worker_death_respawns_once_without_spending_retries_or_max_entries(self) -> None:
+ # Defect B (live run fa1a8f1c): "Child worker unreachable" consumed
+ # the entry and, with default retries=0, escalated into a pause --
+ # twenty minutes of child work discarded on an infra hiccup. A
+ # verified worker death is infra, not the task's failure: the
+ # executor liveness-verifies it itself (a short quiet period, then
+ # one re-collect) and re-spawns the same rendered prompt once,
+ # spending neither retries nor a fresh max_entries slot.
+ self.host.child_outcomes["child-1"] = {
+ "status": "error",
+ "error": "Child worker unreachable",
+ "answer": "partial work",
+ }
+ self.store_factory({"nodes": [{"id": "work", "subagent": "worker"}]})
+ result = await self.start()
+ status = await self.settle(result)
+ self.assertEqual(status["state"], "done")
+ self.assertEqual(self.host.notice_kinds(), ["finished"])
+ work = self.node_status(status, "work")
+ self.assertEqual(work["status"], "done")
+ self.assertEqual(work["entries_used"], 1)
+ self.assertEqual(len(self.host.spawn_calls("work")), 2)
+ # the dead attempt's exit text was dropped, not marked for verify:
+ # the respawn redid the work and the entry settled for real
+ self.assertNotIn("needs_verify", status)
+ self.assertNotIn("needs_verify", work)
+ # the executor verified the death itself before re-spawning
+ self.assertIn(factory_module.INFRA_VERIFY_QUIET_SECONDS, self.sleeps.sleeps)
+
+ @async_test
+ async def test_second_worker_death_falls_through_retries_and_policy(self) -> None:
+ # Only ONE free respawn: a second verified death charges the normal
+ # retries budget (the free respawn consumed none) and then the
+ # failure_policy applies like any task failure.
+ self.host.outcomes["work"] = {"status": "error", "error": "Child worker unreachable"}
+ self.store_factory(
+ {
+ "run": {"failure_policy": "continue"},
+ "nodes": [{"id": "work", "subagent": "worker", "retries": 1}],
+ }
+ )
+ result = await self.start()
+ status = await self.settle(result)
+ self.assertEqual(status["state"], "failed")
+ work = self.node_status(status, "work")
+ self.assertEqual(work["status"], "error")
+ # initial spawn, free respawn, one charged retry: three admissions
+ self.assertEqual(len(self.host.spawn_calls("work")), 3)
+ self.assertEqual(work["attempts"], 3)
+ self.assertEqual(len(self.all_events_of(result, "infra_respawn")), 1)
+ self.assertEqual(len(self.all_events_of(result, "retry")), 1)
+
+ @async_test
+ async def test_paused_collect_batch_preserves_the_free_infra_respawn(self) -> None:
+ self.host.child_outcomes["child-1"] = {"status": "error", "error": "task failed"}
+ self.host.child_outcomes["child-2"] = {
+ "status": "error", "error": "Child worker unreachable"
+ }
+ self.store_factory({"nodes": [
+ {"id": "a", "subagent": "worker"},
+ {"id": "b", "subagent": "worker"},
+ ]})
+ result = await self.start()
+ run = self.executor._runs[result["run_id"]]
+ await run.task # finish the whole batch, including b after a pauses it
+ self.assertEqual(run.state, "paused")
+ instance = run.states["b"].entries[0].instances[0]
+ self.assertEqual(instance.status, "pending")
+ self.assertTrue(instance.infra_respawned)
+ self.assertEqual(self.host.deleted_targets(), ["child-2"])
+ await rlm_module.rlm.factory.resume(result["run_id"])
+ status = await self.settle(result, max_polls=1000)
+ self.assertEqual(status["state"], "done")
+ self.assertEqual(len(self.host.spawn_calls("b")), 2)
+ self.assertEqual(self.node_status(status, "b")["entries_used"], 1)
+ self.assertEqual(self.all_events_of(result, "retry"), [])
+ self.assertEqual(len(self.all_events_of(result, "infra_respawn")), 1)
+
+ @async_test
+ async def test_stop_during_output_capture_preserves_cancelled_entry(self) -> None:
+ self.host.outcomes["work"] = {"status": "done", "answer": "no JSON"}
+ capture_gate = self.host.gate("rlm.collect", 2)
+ capture_entered = self.host.gate_entered("rlm.collect", 2)
+ self.store_factory({"nodes": [{
+ "id": "work", "subagent": "worker",
+ "outputs": [{"name": "data", "type": "json"}],
+ }]})
+ result = await self.start()
+ run = self.executor._runs[result["run_id"]]
+ await capture_entered.wait()
+ await rlm_module.rlm.factory.stop(result["run_id"])
+ entry = run.states["work"].entries[0]
+ self.assertEqual(entry.status, "cancelled")
+ capture_gate.set()
+ await run.task
+ self.assertEqual(run.state, "stopped")
+ self.assertEqual(entry.status, "cancelled")
+ self.assertFalse(entry.instances[0].provisional)
+ self.assertEqual(self.all_events_of(result, "retry"), [])
+
+ @async_test
+ async def test_worker_death_recheck_adopts_a_live_settle(self) -> None:
+ # The liveness re-check is not a rubber stamp: when the worker
+ # answers again during the quiet period, its fresh settle supersedes
+ # the error row and nothing is re-spawned.
+ self.host.child_outcomes["child-1"] = {
+ "status": "error",
+ "error": "Child worker unreachable",
+ }
+ host = self.host
+
+ class ReviveSleep(ClockSleep):
+ """The executor's quiet-period sleep revives child-1: the
+ worker was never dead, and the re-read must see it settled."""
+
+ async def __call__(self, seconds: float) -> None:
+ await super().__call__(seconds)
+ host.child_outcomes["child-1"] = {"status": "done", "answer": "late done"}
+
+ self.executor._sleep_fn = ReviveSleep(self.clock)
+ self.store_factory({"nodes": [{"id": "work", "subagent": "worker"}]})
+ result = await self.start()
+ status = await self.settle(result)
+ self.assertEqual(status["state"], "done")
+ self.assertEqual(len(self.host.spawn_calls("work")), 1)
+ self.assertEqual(self.all_events_of(result, "infra_respawn"), [])
+ self.assertEqual(self.node_status(status, "work")["answer_preview"], "late done")
+
+ @async_test
+ async def test_resume_grants_fresh_retries_and_a_fresh_respawn(self) -> None:
+ # The approved resume semantics: a re-admitted entry restarts both
+ # budgets. The charged retry after the free respawn must be granted
+ # again, and an infra death of the re-admitted attempt must get its
+ # own free respawn instead of failing through a spent one.
+ self.host.child_outcomes["child-1"] = {"status": "error", "error": "Child worker unreachable"}
+ self.host.child_outcomes["child-2"] = {"status": "error", "error": "boom"}
+ self.host.child_outcomes["child-3"] = {"status": "error", "error": "boom"}
+ self.host.child_outcomes["child-4"] = {"status": "error", "error": "Child worker unreachable"}
+ self.store_factory({"nodes": [{"id": "work", "subagent": "worker", "retries": 1}]})
+ result = await self.start()
+ paused = await self.settle(result)
+ # infra death -> free respawn; the respawn's own task failure spent
+ # the one charged retry; the retry failed too -> escalate pause.
+ self.assertEqual(paused["state"], "paused")
+ self.assertEqual(len(self.host.spawn_calls("work")), 3)
+ self.assertEqual(len(self.all_events_of(result, "infra_respawn")), 1)
+ self.assertEqual(len(self.all_events_of(result, "retry")), 1)
+ # resume re-admits the entry with fresh budgets: the re-admitted
+ # attempt's infra death respawns free again, and this time it
+ # settles done.
+ await rlm_module.rlm.factory.resume(result["run_id"])
+ final = await self.settle(result)
+ self.assertEqual(final["state"], "done")
+ self.assertEqual(self.node_status(final, "work")["entries_used"], 1)
+ self.assertEqual(len(self.host.spawn_calls("work")), 5)
+ self.assertEqual(len(self.all_events_of(result, "infra_respawn")), 2)
+ self.assertEqual(len(self.all_events_of(result, "retry")), 1)
+ self.assertEqual(self.host.notice_kinds(), ["paused", "finished"])
+
# -- budgets ----------------------------------------------------------------------
@async_test
@@ -4919,7 +5223,7 @@ class FactoryExecutorTest(_ExecutorTestCase):
# contract it validated is pinned here: run-level milestones send
# {run_id, kind, detail} (no node), node-level ones add "node", and
# every notice goes through one "factory.progress" host request.
- self.host.outcomes["a"] = {"status": "error", "error": "boom"}
+ self.host.child_outcomes["child-1"] = {"status": "error", "error": "boom"}
self.store_factory(
{
"nodes": [
@@ -4933,20 +5237,20 @@ class FactoryExecutorTest(_ExecutorTestCase):
self.assertEqual(paused["state"], "paused")
resume = await rlm_module.rlm.factory.resume(result["run_id"])
final = await self.settle(result)
- self.assertEqual(final["state"], "failed")
+ self.assertEqual(final["state"], "done")
kinds = self.host.notice_kinds()
- self.assertEqual(kinds, ["paused", "failed"])
- paused_notice, failed_notice = self.host.notices
+ self.assertEqual(kinds, ["paused", "finished"])
+ paused_notice, finished_notice = self.host.notices
self.assertEqual(paused_notice["kind"], "paused")
self.assertEqual(paused_notice["run_id"], result["run_id"])
self.assertIn("state a failed", paused_notice["detail"])
self.assertIn("resume", paused_notice["detail"])
self.assertEqual(paused_notice.get("node"), "a") # node-level milestone
- self.assertEqual(failed_notice["kind"], "failed")
- self.assertEqual(failed_notice["run_id"], result["run_id"])
- self.assertIsNone(failed_notice.get("node")) # run-level milestone
+ self.assertEqual(finished_notice["kind"], "finished")
+ self.assertEqual(finished_notice["run_id"], result["run_id"])
+ self.assertIsNone(finished_notice.get("node")) # run-level milestone
self.assertEqual(set(paused_notice.keys()), {"run_id", "kind", "detail", "node"})
- self.assertEqual(set(failed_notice.keys()), {"run_id", "kind", "detail"})
+ self.assertEqual(set(finished_notice.keys()), {"run_id", "kind", "detail"})
# one host request per notice, typed factory.progress
self.assertEqual(len(self.host.calls_of("factory.progress")), 2)
@@ -4958,7 +5262,7 @@ class FactoryExecutorTest(_ExecutorTestCase):
patcher = patch.object(rlm_module, "host_request", failing)
patcher.start()
self.addCleanup(patcher.stop)
- failing.outcomes["a"] = {"status": "error", "error": "boom"}
+ failing.child_outcomes["child-1"] = {"status": "error", "error": "boom"}
self.store_factory(
{
"nodes": [
@@ -4989,7 +5293,7 @@ class FactoryExecutorTest(_ExecutorTestCase):
# the run is not wedged: resume still works end to end.
await rlm_module.rlm.factory.resume(result["run_id"])
final = await self.settle(result)
- self.assertEqual(final["state"], "failed")
+ self.assertEqual(final["state"], "done")
self.assertEqual(self.node_status(final, "b")["status"], "done")
@@ -5813,6 +6117,88 @@ class FactoryExecutorTest(_ExecutorTestCase):
)
self.assertEqual(self.all_events_of(result, "output_capture_failed"), [])
+ @async_test
+ async def test_missing_answer_capture_has_a_bounded_poll_window(self) -> None:
+ self.host.outcomes["work"] = {"status": "done"}
+ self.store_factory({"nodes": [{"id": "work", "subagent": "worker",
+ "outputs": [{"name": "data", "type": "json"}]}]})
+ before = self.clock()
+ result = await self.start()
+ status = await self.settle(result)
+ elapsed = self.clock() - before
+ self.assertEqual(status["state"], "paused")
+ self.assertGreater(elapsed, 0)
+ self.assertLessEqual(elapsed, factory_module.POLL_TIMEOUT_MS / 1000 + 1e-9)
+ self.assertEqual(len(self.host.spawn_calls("work")), 1)
+ self.assertIn("captured no value", self.node_status(status, "work")["error"])
+
+ @async_test
+ async def test_stop_during_missing_answer_poll_preserves_cancellation(self) -> None:
+ self.host.outcomes["work"] = {"status": "done"}
+ sleeps = GatedSleep()
+ self.executor._sleep_fn = sleeps
+ self.store_factory({"nodes": [{"id": "work", "subagent": "worker",
+ "outputs": [{"name": "data", "type": "json"}]}]})
+ result = await self.start()
+ run = self.executor._runs[result["run_id"]]
+ await self.wait_until(sleeps.entered.is_set)
+ await rlm_module.rlm.factory.stop(result["run_id"])
+ sleeps.release.set()
+ await run.task
+ entry = run.states["work"].entries[0]
+ self.assertEqual(run.state, "stopped")
+ self.assertEqual(entry.status, "cancelled")
+ self.assertFalse(entry.instances[0].provisional)
+ self.assertEqual(self.all_events_of(result, "retry"), [])
+ self.assertEqual(len(self.host.spawn_calls("work")), 1)
+
+ @async_test
+ async def test_capture_waits_for_the_settled_workers_answer_handoff(self) -> None:
+ handoff_requested = asyncio.Event()
+ answer_ready = asyncio.Event()
+
+ class HandoffHost(FakeHost):
+ empty_reads = 0
+
+ async def __call__(self, request_type, payload=None):
+ response = await super().__call__(request_type, payload)
+ if request_type == "rlm.collect" and not answer_ready.is_set():
+ for row in response["results"]:
+ if row["rlm_child_id"] == "child-1":
+ row.pop("answer_text", None)
+ row.pop("answer_preview", None)
+ self.empty_reads += 1
+ if self.empty_reads >= 2:
+ handoff_requested.set()
+ return response
+
+ class YieldingSleep(ClockSleep):
+ async def __call__(self, seconds):
+ await super().__call__(seconds)
+ await yield_loop_turn()
+
+ self.host = HandoffHost(clock=self.clock)
+ self.host.outcomes["work"] = {"status": "done", "answer": '{"data": {"ok": true}}'}
+ repatch = patch.object(rlm_module, "host_request", self.host)
+ repatch.start()
+ self.addCleanup(repatch.stop)
+ self.executor._sleep_fn = YieldingSleep(self.clock)
+
+ async def finish_handoff():
+ await handoff_requested.wait()
+ answer_ready.set()
+
+ handoff = asyncio.create_task(finish_handoff())
+ self.store_factory({"nodes": [{"id": "work", "subagent": "worker",
+ "outputs": [{"name": "data", "type": "json"}]}]})
+ result = await self.start()
+ status = await self.settle(result)
+ await handoff
+ self.assertEqual(status["state"], "done")
+ self.assertEqual(self.node_status(status, "work")["entries_used"], 1)
+ self.assertEqual(len(self.host.spawn_calls("work")), 1)
+ self.assertEqual(self.all_events_of(result, "output_capture_failed"), [])
+
@async_test
async def test_child_exit_captures_a_provisional_answer_and_marks_needs_verify(self) -> None:
# M4: a child that dies without reporting is distinguishable from
@@ -5919,7 +6305,9 @@ class FactoryExecutorTest(_ExecutorTestCase):
async def test_paused_run_status_carries_the_last_error_and_remedy(self) -> None:
# M6: a paused run carries the LAST failed admission/bind error and
# a one-line remedy in its status payload — no root-side
- # archaeology over pending-state lists to learn what failed.
+ # archaeology over pending-state lists to learn what failed. The
+ # pause here is src's own failed output capture (defect C): the
+ # failure lands at the producer, before the dependent's bind.
self.host.outcomes["src"] = {"status": "done", "answer": "no json here"}
self.store_machine(
{
@@ -5945,11 +6333,12 @@ class FactoryExecutorTest(_ExecutorTestCase):
self.assertEqual(status["state"], "paused")
self.assertEqual(
status["pause_reason"],
- "input 'data': no JSON object containing output 'data' in the upstream answer",
+ "no JSON object containing output 'data' in the upstream answer",
)
- self.assertIn("state 'dep' failed", status["last_error"])
+ self.assertIn("state 'src' failed", status["last_error"])
self.assertIn("no JSON object containing output 'data'", status["last_error"])
self.assertIn(f"rlm.factory.resume('{result['run_id']}')", status["remedy"])
+ self.assertEqual(status["needs_verify"], ["src"])
@async_test
async def test_machine_required_input_over_an_errored_source_fails_the_dependent_entry(self) -> None:
@@ -5998,10 +6387,10 @@ class FactoryExecutorTest(_ExecutorTestCase):
@async_test
async def test_machine_optional_input_over_a_settled_source_with_no_captured_value_binds_the_null_sentinel(self) -> None:
# Review finding (PR #3199), second no-value condition: the source
- # settled done but captured no value for the declared port (an
- # empty answer). The optional dependent proceeds on the null
- # sentinel; only the required dependent over the same valueless
- # settle fails its entry.
+ # settled but captured no value for the declared port (an empty
+ # answer). The optional dependent proceeds on the null sentinel;
+ # only the required dependent over the same valueless settle fails
+ # its entry.
self.host.outcomes["src"] = {"status": "done", "answer": ""}
self.store_machine(
{
@@ -6037,6 +6426,12 @@ class FactoryExecutorTest(_ExecutorTestCase):
result = await self.start()
status = await self.settle(result)
self.assertEqual(status["state"], "failed")
+ # the empty answer never captured the text port, so the capture
+ # failure failed the source entry itself (defect C) instead of
+ # settling it valueless
+ src = self.state_report(status, "src")
+ self.assertEqual([entry["status"] for entry in src["entries"]], ["error"])
+ self.assertIn("captured no value", src["entries"][0]["error"])
opt = self.state_report(status, "opt")
self.assertEqual(opt["entries_used"], 1)
self.assertEqual([entry["status"] for entry in opt["entries"]], ["done"])
@@ -6045,7 +6440,7 @@ class FactoryExecutorTest(_ExecutorTestCase):
self.assertEqual(req["entries_used"], 1)
self.assertEqual([entry["status"] for entry in req["entries"]], ["error"])
self.assertIn(
- "input 'go' from state 'src' has no captured output 'go'",
+ "input 'go' from state 'src' is unavailable (latest settle status 'error')",
req["entries"][0]["error"],
)
self.assertEqual(self.host.spawn_calls("req"), [])
@@ -6053,10 +6448,10 @@ class FactoryExecutorTest(_ExecutorTestCase):
@async_test
async def test_machine_optional_input_over_a_failed_json_capture_binds_the_null_sentinel(self) -> None:
# Review finding (PR #3199), third no-value condition: the source
- # settled done, declared a json port, but its answer never parsed
- # for that port (a JSON capture failure recorded on the settle).
- # The optional dependent proceeds on the null sentinel; only the
- # required dependent over the same failed capture fails its entry.
+ # settled, declared a json port, but its answer never parsed for
+ # that port (a JSON capture failure). The optional dependent
+ # proceeds on the null sentinel; only the required dependent over
+ # the same failed capture fails its entry.
self.host.outcomes["src"] = {"status": "done", "answer": "no json here"}
self.store_machine(
{
@@ -6092,6 +6487,13 @@ class FactoryExecutorTest(_ExecutorTestCase):
result = await self.start()
status = await self.settle(result)
self.assertEqual(status["state"], "failed")
+ # the failed json capture failed the source entry itself (defect C),
+ # and its settled answer stays preserved for review
+ src = self.state_report(status, "src")
+ self.assertEqual([entry["status"] for entry in src["entries"]], ["error"])
+ self.assertIn("no JSON object containing output 'data'", src["entries"][0]["error"])
+ self.assertEqual(status["needs_verify"], ["src"])
+ self.assertIn("no json here", src["entries"][0]["provisional_answer"])
opt = self.state_report(status, "opt")
self.assertEqual(opt["entries_used"], 1)
self.assertEqual([entry["status"] for entry in opt["entries"]], ["done"])
@@ -6100,7 +6502,7 @@ class FactoryExecutorTest(_ExecutorTestCase):
self.assertEqual(req["entries_used"], 1)
self.assertEqual([entry["status"] for entry in req["entries"]], ["error"])
self.assertIn(
- "input 'data': no JSON object containing output 'data' in the upstream answer",
+ "input 'data' from state 'src' is unavailable (latest settle status 'error')",
req["entries"][0]["error"],
)
self.assertEqual(self.host.spawn_calls("req"), [])
@@ -6263,14 +6665,9 @@ class FactoryExecutorTest(_ExecutorTestCase):
# The admission below is gated mid-spawn so a stale loop that wrongly
# continued would re-admit the SAME pending instance (a duplicate
# child) while the new loop's admission is still in flight.
- self.host.outcomes["a"] = {"status": "error", "error": "boom"}
- self.host.rate_limit_first["b"] = 1 # resume()'s admission defers once
+ self.host.child_outcomes["child-1"] = {"status": "error", "error": "boom"}
progress_gate = self.host.gate("factory.progress", 1)
progress_entered = self.host.gate_entered("factory.progress", 1)
- # rlm.run call #1 is a's admission at run(); #2 is b's first admission
- # at resume() (rate limited, deferred nonblockingly); the gate holds
- # call #3 -- the NEW loop's retry admission -- mid-spawn.
- spawn_gate = self.host.gate("rlm.run", 3)
self.store_factory(
{
"nodes": [
@@ -6288,26 +6685,35 @@ class FactoryExecutorTest(_ExecutorTestCase):
old_task = run.task
self.assertIsNotNone(old_task)
self.assertFalse(old_task.done())
- # resume() bumps the generation, fires a->b, defers b's first
- # admission (rate limited, nonblocking), and starts the new loop.
+ # The rate limit hits a's re-admission: resume()'s admission defers
+ # once, and the new loop's retry is the gated mid-spawn.
+ self.host.rate_limit_first["a"] = 2
+ # rlm.run call #1 is a's admission at run(); #2 is a's re-admission
+ # at resume() (rate limited, deferred nonblockingly); the gate holds
+ # call #3 -- the NEW loop's retry admission -- mid-spawn.
+ spawn_gate = self.host.gate("rlm.run", 3)
+ # resume() bumps the generation, re-admits a's failed entry (the
+ # admission defers, rate limited, nonblocking), and starts the new
+ # loop.
resumed = await rlm_module.rlm.factory.resume(result["run_id"])
self.assertEqual(resumed["state"], "running")
self.assertEqual(resumed["started"], [])
self.assertIsNot(run.task, old_task)
# Releasing the milestone lets the old loop continue: with the
# generation bump it must exit without admitting anything (the new
- # loop owns b, suspended mid-admission at the spawn gate).
+ # loop owns a's retry, suspended mid-admission at the spawn gate).
progress_gate.set()
await old_task
self.assertTrue(old_task.done())
spawn_gate.set()
final = await self.settle(result)
- self.assertEqual(final["state"], "failed")
+ self.assertEqual(final["state"], "done")
self.assertEqual(self.node_status(final, "b")["status"], "done")
- # b has exactly two admissions: the deferred one at resume() and
- # the new loop's successful retry. A stale second loop would admit
- # it once more while the gated spawn was in flight.
- self.assertEqual(len(self.host.spawn_calls("b")), 2)
+ # a has exactly three admissions: the failed first, the deferred
+ # one at resume() and the new loop's successful retry. A stale
+ # second loop would admit it once more while the gated spawn was
+ # in flight.
+ self.assertEqual(len(self.host.spawn_calls("a")), 3)
self.assertEqual(self.all_events_of(result, "executor_error"), [])
self.assertEqual(
len([e for e in self.all_events_of(result, "state_entry") if e.get("node") == "b"]), 1
diff --git a/scripts/review_factory_resumption_parity.py b/scripts/review_factory_resumption_parity.py
new file mode 100644
index 0000000000..090dc8211e
--- /dev/null
+++ b/scripts/review_factory_resumption_parity.py
@@ -0,0 +1,218 @@
+#!/usr/bin/env python3
+"""Hosted real-daemon factory recovery receipt; TS has no advertised factory lane.
+
+The Rust kernel runs a real factory child, fails its declared JSON output,
+resumes the same entry, and binds the second child's valid output. Historical
+TS absence is scoped to its actual capability advertisement, not full parity.
+"""
+import argparse
+import http.server
+import json
+import os
+from pathlib import Path
+import subprocess
+import sys
+import tempfile
+import threading
+import time
+
+from review_kernel_env_parity import KernelClient as Client
+from review_draft_disconnect_parity import digest, env_for, TS_ARCHIVE_SHA256
+from review_goal_fixture_lifecycle import OwnedProcesses, TAG, finish
+from review_restore_notice_parity import texts
+
+MARKER = 'FACTORY_RECOVERY_RESULT='
+CELL = '''import asyncio, json, time
+factory_harness = rlm.get_harness_state()
+factory_harness.create_factory('Recovery fixture', 'Actual factory output recovery', id='recovery-fixture', machine={
+ 'states': [{'id': 'work', 'entry': True, 'max_entries': 1,
+ 'subagent': {'prompt': 'FACTORY_RECOVERY_CHILD'},
+ 'outputs': [{'name': 'data', 'type': 'json'}]},
+ {'id': 'verified', 'max_entries': 1,
+ 'subagent': {'prompt': 'FACTORY_RECOVERY_CHILD_VERIFY'}}],
+ 'transitions': [{'from': 'work', 'to': 'verified',
+ 'when': {'output': 'data', 'path': 'ok', 'op': 'eq', 'value': True}}]})
+factory_started = await rlm.factory.run('recovery-fixture')
+print('FACTORY_RECOVERY_RESULT=' + json.dumps(factory_started))'''
+
+
+class Provider(http.server.ThreadingHTTPServer):
+ daemon_threads = True
+
+ def __init__(self):
+ super().__init__(('127.0.0.1', 0), Handler)
+ self.requests = []
+ self.parent_requests = []
+ self.child_requests = []
+
+
+class Handler(http.server.BaseHTTPRequestHandler):
+ def log_message(self, *_args):
+ pass
+
+ def do_POST(self):
+ if self.path != '/v1/chat/completions':
+ self.send_error(404)
+ return
+ body = json.loads(self.rfile.read(int(self.headers['Content-Length'])))
+ self.server.requests.append(body)
+ parent = any('FACTORY_PARENT_PROBE' in ''.join(texts(m.get('content')))
+ for m in body.get('messages', []) if m.get('role') == 'user')
+ child = not parent and any('FACTORY_RECOVERY_CHILD' in ''.join(texts(m.get('content')))
+ for m in body.get('messages', []) if m.get('role') == 'user')
+ if child:
+ self.server.child_requests.append(body)
+ delta = {'content': 'first attempt has no JSON' if len(self.server.child_requests) == 1
+ else '```json\n{"data": {"ok": true}}\n```'}
+ reason = 'stop'
+ else:
+ self.server.parent_requests.append(body)
+ if len(self.server.parent_requests) == 1:
+ offered = [t['function']['name'] for t in body.get('tools', [])]
+ assert 'python_repl' in offered, offered
+ delta = {'tool_calls': [{'index': 0, 'id': 'factory_probe', 'type': 'function',
+ 'function': {'name': 'python_repl', 'arguments': json.dumps({'code': CELL})}}]}
+ reason = 'tool_calls'
+ else:
+ delta, reason = {'content': 'Factory probe complete.'}, 'stop'
+ def chunk(delta, reason=None):
+ return {'id': 'factory-fixture', 'object': 'chat.completion.chunk', 'created': 1,
+ 'model': 'factory-fixture', 'choices': [{'index': 0, 'delta': delta, 'finish_reason': reason}]}
+ wire = ''.join('data: ' + json.dumps(row) + '\n\n' for row in (
+ chunk({'role': 'assistant', **delta}), chunk({}, reason))) + 'data: [DONE]\n\n'
+ self.send_response(200)
+ self.send_header('Content-Type', 'text/event-stream')
+ self.send_header('Content-Length', str(len(wire.encode())))
+ self.end_headers()
+ self.wfile.write(wire.encode())
+
+
+def extract(requests):
+ assert len(requests) == 2, 'unexpected parent provider sequence'
+ results = [m for m in requests[-1]['messages'] if m.get('role') == 'tool'
+ and m.get('tool_call_id') == 'factory_probe']
+ assert len(results) == 1, 'missing/duplicate current tool result'
+ values = [json.loads(line[len(MARKER):]) for content in texts(results[0].get('content'))
+ for line in content.splitlines() if line.startswith(MARKER)]
+ assert len(values) == 1, 'missing/duplicate actual factory receipt'
+ return values[0]
+
+
+def count(row, snake, camel):
+ assert not (snake in row and camel in row), 'ambiguous duplicate wire keys'
+ return row[snake] if snake in row else row[camel]
+
+
+def validate(probe):
+ assert probe['paused']['state'] == 'paused', probe
+ assert probe['done']['state'] == 'done', probe
+ nodes = probe['done']['nodes']
+ assert sorted(n['id'] for n in nodes) == ['verified', 'work'], nodes
+ verified = next(n for n in nodes if n['id'] == 'verified')
+ assert verified['status'] == 'done' and count(verified, 'entries_used', 'entriesUsed') == 1 and verified['attempts'] == 1, verified
+ node = next(n for n in nodes if n['id'] == 'work')
+ assert count(node, 'entries_used', 'entriesUsed') == 1 and node['attempts'] == 2, node
+ assert node['status'] == 'done', node
+
+
+def observe(binary, kind, root, observed):
+ env = env_for(root, kind)
+ env[TAG] = str(root)
+ server = Provider()
+ threading.Thread(target=server.serve_forever, daemon=True).start()
+ (root / 'agent/settings.json').write_text(json.dumps({'factory': {'enabled': True},
+ 'prewarmPythonKernel': False, 'defaultProvider': 'parity-local', 'defaultModel': 'parity-1',
+ 'onboardingCompleted': True, 'onboardingShown': True, 'telemetry': {'noticeShown': True}}))
+ (root / 'agent/models.json').write_text(json.dumps({'providers': {'parity-local': {
+ 'api': 'openai-completions', 'apiKey': 'fixture-only',
+ 'baseUrl': f'http://127.0.0.1:{server.server_port}/v1', 'models': [{
+ 'id': 'parity-1', 'name': 'Factory fixture', 'reasoning': False,
+ 'contextWindow': 128000, 'maxTokens': 4096}]}}}))
+ path = root / 'd.sock'
+ scope = OwnedProcesses(binary, root, path)
+ daemon = control = None
+ log = open(root / 'daemon.log', 'wb')
+ try:
+ command = [str(binary), '--mode', 'daemon', '--daemon-socket', str(path)]
+ daemon = subprocess.Popen(command, cwd=root, env=env, stdout=log,
+ stderr=subprocess.STDOUT, start_new_session=True)
+ scope.register(daemon, command)
+ control = Client(path, daemon)
+ observed['hello'] = control.hello
+ caps = control.hello.get('serverCapabilities', [])
+ if kind == 'ts':
+ assert 'factory_activity' not in caps, 'TS unexpectedly advertises factory'
+ observed['factory_lane_advertised'] = False
+ else:
+ created = control.call('create', config={'cwd': str(root), 'agentDir': str(root / 'agent'),
+ 'provider': 'parity-local', 'model': 'parity-1', 'telemetryDisabled': True,
+ 'noExtensions': True, 'noContextFiles': True})
+ active = created.get('activeSessionId') or created.get('id')
+ assert active, created
+ control.call('attach', activeSessionId=active)
+ control.call('prompt', activeSessionId=active, message='Run FACTORY_PARENT_PROBE')
+ control.call('wait_for_idle', timeout_seconds=120, activeSessionId=active)
+ started = extract(server.parent_requests[:2])
+ observed['started'] = started
+ run_id = started['run_id']
+ def settled():
+ snapshots = []
+ deadline = time.monotonic() + 60
+ while time.monotonic() < deadline:
+ status = control.call('factory_activity', activeSessionId=active,
+ action='status', runId=run_id)
+ snapshots.append(status)
+ if status['state'] != 'running':
+ return status
+ threading.Event().wait(0.025)
+ observed.setdefault('timed_out_snapshots', []).extend(snapshots)
+ raise AssertionError('factory did not leave running after parent turn completed')
+ paused = settled()
+ assert paused['state'] == 'paused', paused
+ observed['resume_response'] = control.call('factory_activity', activeSessionId=active,
+ action='resume', runId=run_id)
+ observed['probe'] = {'paused': paused, 'done': settled()}
+ validate(observed['probe'])
+ assert len(server.child_requests) == 3, 'not two producer attempts plus one guarded verifier'
+ observed['behavior_verified'] = True
+ finally:
+ observed['provider_requests'] = server.requests
+ observed['wire'] = list(control.wire) if control else []
+ finish(scope, control, daemon, observed)
+ log.close()
+ observed['daemon_log'] = (root / 'daemon.log').read_text(errors='replace')[-10000:]
+ server.shutdown()
+ server.server_close()
+
+
+def main():
+ parser = argparse.ArgumentParser(description=__doc__)
+ parser.add_argument('--receipt', type=Path, required=True)
+ args = parser.parse_args()
+ result = {'verified': False, 'scope': __doc__, 'runs': {}, 'not_verified': [
+ 'TS factory behavior (no advertised historical counterpart)', 'TUI',
+ 'worker-death respawn (separate executor regression)', 'pause/stop races (separate regressions)']}
+ try:
+ assert sys.platform == 'linux', 'hosted Linux only'
+ assert digest(Path(os.environ['PA_TS_ARCHIVE'])) == TS_ARCHIVE_SHA256
+ result['ts_archive_sha256'] = TS_ARCHIVE_SHA256
+ with tempfile.TemporaryDirectory(prefix='pa-factory-', dir='/tmp') as folder:
+ for kind in ('ts', 'rust'):
+ binary = Path(os.environ[f'PA_{kind.upper()}_BINARY']).resolve()
+ observed = result['runs'][kind] = {'binary_sha256': digest(binary)}
+ try:
+ observe(binary, kind, Path(folder) / kind, observed)
+ except Exception as error:
+ observed['error'] = str(error)
+ assert all(r.get('behavior_verified') and r.get('cleanup_verified') and 'error' not in r
+ for r in result['runs'].values()), 'actual factory observation failed'
+ result['verified'] = True
+ except Exception as error:
+ result['error'] = str(error)
+ args.receipt.parent.mkdir(parents=True, exist_ok=True)
+ args.receipt.write_text(json.dumps(result, ensure_ascii=False, indent=2))
+ return 0 if result['verified'] else 1
+
+
+if __name__ == '__main__':
+ sys.exit(main())
diff --git a/scripts/test_review_factory_resumption_parity.py b/scripts/test_review_factory_resumption_parity.py
new file mode 100644
index 0000000000..74543fee91
--- /dev/null
+++ b/scripts/test_review_factory_resumption_parity.py
@@ -0,0 +1,44 @@
+import copy
+import json
+import unittest
+from review_factory_resumption_parity import MARKER, extract, validate
+
+
+class FactoryReceiptGuards(unittest.TestCase):
+ def probe(self):
+ return {'paused': {'state': 'paused'}, 'done': {'state': 'done', 'nodes': [
+ {'id': 'work', 'status': 'done', 'entries_used': 1, 'attempts': 2},
+ {'id': 'verified', 'status': 'done', 'entries_used': 1, 'attempts': 1}]}}
+
+ def test_rejects_missing_resume_extra_entry_or_extra_attempt(self):
+ validate(self.probe())
+ for key, value in [('entries_used', 2), ('attempts', 1), ('attempts', 3), ('status', 'error')]:
+ probe = self.probe()
+ probe['done']['nodes'][0][key] = value
+ with self.assertRaises(AssertionError):
+ validate(probe)
+
+ def test_accepts_explicit_wire_camel_case_without_ignoring_duplicates(self):
+ probe = self.probe()
+ for row in probe['done']['nodes']:
+ row['entriesUsed'] = row.pop('entries_used')
+ validate(probe)
+ probe['done']['nodes'][0]['entries_used'] = 1
+ with self.assertRaises(AssertionError):
+ validate(probe)
+
+ def test_rejects_missing_or_historical_tool_receipt(self):
+ probe = self.probe()
+ tool = {'role': 'tool', 'tool_call_id': 'factory_probe',
+ 'content': MARKER + json.dumps(probe)}
+ rows = [{}, {'messages': [tool]}]
+ self.assertEqual(extract(rows), probe)
+ for changed in [[{'messages': [tool]}, {'messages': []}],
+ [{}, {'messages': [tool, copy.deepcopy(tool)]}],
+ [{}, {'messages': [{**tool, 'tool_call_id': 'historical'}]}]]:
+ with self.assertRaises(AssertionError):
+ extract(changed)
+
+
+if __name__ == '__main__':
+ unittest.main()