A self-improving RLM agent for coding workflows and long-running autonomous tasks.
Rust
[factory] escalate resume re-admits the failed entry; worker deaths respawn once (#3631)
Sebastian Müller8d58e5b6f9ec
diff --git a/prime-agent-runtime/src/rlm/factory.py b/prime-agent-runtime/src/rlm/factory.pyindex 673a784176..aeed2dd4b5 100644--- a/prime-agent-runtime/src/rlm/factory.py+++ b/prime-agent-runtime/src/rlm/factory.py@@ -1086,6 +1086,17 @@ _RATE_LIMIT_MARKERS = ("usage limit",)+INFRA_VERIFY_QUIET_SECONDS = POLL_TIMEOUT_MS / 1000+"""Quiet period before the executor re-reads a worker that settled with an+infra error (one collect-poll slice): the re-check must not race the very+probe streak that reported the death. The sleep is injected, so tests see+no wall-clock wait."""++_INFRA_ERROR_MARKERS = (+ "child worker unreachable",+ "supervisor link",+)+_FENCED_JSON_RE = re.compile(r"```json\s*(.*?)\s*```", re.DOTALL)TERMINAL_ENTRY_STATUSES = ("done", "error", "cancelled")@@ -1096,6 +1107,15 @@ def _is_rate_limit_error(message: str) -> bool:return any(marker in lowered for marker in _RATE_LIMIT_MARKERS)+def _is_infra_error(message: str) -> bool:+ """Heuristic: the supervisor reports worker deaths and lost supervisor+ links as one-shot error settles (``Child worker unreachable`` after the+ daemon's unreachable-probe streak, or a ``supervisor link ...`` error+ from a failed kickoff). Infra, not the task's failure."""+ lowered = message.lower()+ return any(marker in lowered for marker in _INFRA_ERROR_MARKERS)++def _child_name(run_id: str, state_id: str, instance_index: int, attempt: int) -> str:"""Unique, readable sibling name for one spawned instance (host caps names at 64).@@ -1342,8 +1362,18 @@ class _NodeInstance:prompt: str # fully rendered; re-spawns reuse it verbatimstatus: str = "pending" # pending | running | done | error | cancelledattempt: int = 0 # spawn admissions tried for this instance; a rate-limit- # deferral counts here too, and ``retries`` compares against this same- # counter, so one transient 429 admission consumes one declared retry.+ # deferral counts here too, and charged retries compare against+ # ``attempt - retry_base``, so one transient 429 admission consumes one+ # declared retry.+ retry_base: int = 0 # the attempt count at the last retry-budget reset:+ # an infra respawn or a resume's re-admission grants fresh ``retries``+ # by moving this base instead of zeroing ``attempt`` (spawn labels+ # count on it staying monotonic).+ infra_respawned: bool = False # the instance's one free worker-death+ # respawn is spent; a resume's re-admission grants a fresh one.+ capture_failed: bool = False # this settle completed its entry but a+ # declared output never bound; the preserved answer stays provisional+ # until the capture is redone or the entry fails permanently.rate_limit_streak: int = 0 # consecutive rate-limited admissions in the# control loop's backoff path; a successful spawn resets it, and# BACKOFF_MAX_ATTEMPTS in a row fails the node. An admission-phase@@ -1636,8 +1666,9 @@ class FactoryExecutor:"usage": self._usage_report(run),}# The M4 oversight signal: every state whose latest terminal entry- # carried a provisional answer (a child exit) needs verification —- # root-visible without ledger archaeology.+ # carried a provisional answer (a child exit, or a settle whose+ # declared output never bound) needs verification — root-visible+ # without ledger archaeology.needs_verify = [state_idfor state_id in run.order@@ -1653,9 +1684,10 @@ class FactoryExecutor:if run.last_error is not None:payload["last_error"] = run.last_errorpayload["remedy"] = (- f"resume with await rlm.factory.resume('{run.run_id}') to continue the "- "remaining states; the failed state stays error - fix its subagent and "- "start a fresh run to redo it"+ f"resume with await rlm.factory.resume('{run.run_id}') to re-admit the "+ "failed state's entry as a fresh attempt (same rendered prompt, no extra "+ "max_entries); author the state with failure_policy 'continue' to run its "+ "dependents past a failure instead")return payload@@ -1685,9 +1717,14 @@ class FactoryExecutor:async def resume(self, run_id: str) -> dict[str, Any]:"""Resume a paused run (escalate, budget, or max_transitions pause).- A budget or max_transitions pause is reported once per run:- resuming after it is an explicit operator decision and no further- budget pauses fire. Raises ``ValueError`` when the run is not paused.+ An escalate pause holds its failed entry unsettled, so resume+ re-admits that entry first: a fresh attempt of the same rendered+ prompt in the same entry slot -- no extra ``max_entries`` spend,+ fresh ``retries``. The failure only ever propagates to dependents+ under ``failure_policy: "continue"``. A budget or max_transitions+ pause is reported once per run: resuming after it is an explicit+ operator decision and no further budget pauses fire. Raises+ ``ValueError`` when the run is not paused."""run = self._require_run(run_id)if run.state != "paused":@@ -1702,8 +1739,12 @@ class FactoryExecutor:run.pause_reason = Noneself._touch(run)self._event(run, "resumed", detail="resumed by caller")- # Evaluate settles first: paused runs may still carry transitions to- # fire (escalate) before anything can be admitted.+ # An escalate pause holds its failed entry unsettled: re-admit it+ # BEFORE anything evaluates or admits, so the retry -- not the+ # failure's propagation -- is what resume runs.+ self._readmit_held_entries(run)+ # Evaluate settles next: a max_transitions pause left transitions+ # mid-settle, and resume continues after the ones that fired.await self._evaluate_settles(run)# allow_backoff=False: like run(), resume() must never sleep inside# the calling model turn; rate-limited admissions defer to the loop.@@ -1719,6 +1760,49 @@ class FactoryExecutor:"pending": self._pending_state_ids(run),}+ def _readmit_held_entries(self, run: FactoryRun) -> None:+ """Re-admit every entry an escalate pause holds unsettled.++ An escalate failure never queues its settle (the pause lands before+ the failure propagates), so a held entry is exactly ``error`` with+ ``is_settle`` unset -- that predicate is the one held-entry+ definition. Resume turns each held entry back into a fresh attempt+ of the SAME entry slot: the errored and cancelled instances re-run+ their rendered prompt (``attempt`` keeps counting, so spawn labels+ stay unique), ``max_entries`` is not spent again, and the+ ``retries`` budget restarts (``retry_base`` moves to the current+ attempt count). A binding-failure entry has no instances: it+ re-binds through the readiness pass, and a still-broken upstream+ settles it into the same pause again.+ """+ for state_id in run.order:+ state = run.states[state_id]+ for entry in state.entries:+ if entry.status != "error" or entry.is_settle:+ continue+ entry.status = "running" if entry.instances else "pending"+ entry.error = None+ entry.needs_verify = False+ state.error = None+ for instance in entry.instances:+ if instance.status not in ("error", "cancelled"):+ continue # running and done siblings keep their state+ instance.status = "pending"+ instance.error = None+ instance.answer = None+ instance.provisional = False+ instance.capture_failed = False+ instance.rate_limit_streak = 0+ instance.retry_base = instance.attempt+ instance.infra_respawned = False+ self._event(+ run,+ "readmitted",+ node=state_id,+ entry=entry.index,+ detail="resume re-admits the failed entry (same rendered prompt)",+ )+# -- graph, watch, host activity ------------------------------------------def _state_report(@@ -2789,6 +2873,26 @@ class FactoryExecutor:) -> None:if instance.status != "running":return # cancelled (stop/fail_fast) while the collect was in flight+ if result.status == "error" and _is_infra_error(result.error or ""):+ # A worker death or a lost supervisor link is infra, not the+ # task's failure: the executor verifies it itself (a short+ # quiet period, then one re-read through the supervisor)+ # instead of failing the entry on the first error row and+ # flagging the parent to do the verifying.+ fresh = await self._recheck_infra_death(instance, result)+ if fresh is None:+ return # a stop/fail_fast pass claimed it, or the worker answered+ result = fresh+ if (+ entry.status == "running"+ and not instance.infra_respawned+ and result.status == "error"+ and _is_infra_error(result.error or "")+ ):+ await self._respawn_infra_dead(run, state, entry, instance, result)+ return+ # otherwise: a second verified death (or an entry a sibling+ # failed first) takes the normal retries/policy path belowinstance.duration_ms = result.duration_msinstance.tool_uses = result.tool_use_count or 0run.settle_count += 1@@ -2813,6 +2917,7 @@ class FactoryExecutor:if exit_answer:instance.answer = exit_answerinstance.provisional = True+ instance.capture_failed = False # the preserved text is an exit now# Child failures retry (same rendered prompt, attempts+1) while# attempts remain; then the entry failure_policy applies.await self._apply_instance_failure(run, state, entry, instance, child_reason, retry=True)@@ -2841,6 +2946,7 @@ class FactoryExecutor:(result.answer_text or result.answer_preview or "")[:ANSWER_BINDING_CAP] or None)instance.provisional = False # a real settle supersedes any exit capture+ instance.capture_failed = Falseself._event(run,"settled",@@ -2861,11 +2967,122 @@ class FactoryExecutor:stage="arrived",)if entry.status == "running" and entry.instances and all(i.status == "done" for i in entry.instances):- entry.status = "done"entry.answer = self._entry_answer(entry)- await self._capture_outputs(run, state, entry)+ capture_failure = await self._capture_outputs(run, state, entry)+ if entry.status != "running":+ return # a stop()/fail_fast pass claimed the entry mid-capture+ if capture_failure is not None:+ # A declared output that never bound after the capture+ # retry is the producing entry's task failure, not a silent+ # valueless settle: no settle is queued (a dependent never+ # enters on an unbound required input), retries then+ # failure_policy apply, and the answer stays provisional+ # for needs_verify if the entry fails permanently.+ instance.provisional = True+ instance.capture_failed = True+ await self._apply_instance_failure(+ run, state, entry, instance, capture_failure, retry=True+ )+ return+ entry.status = "done"self._queue_settle(run, state, entry)+ async def _recheck_infra_death(self, instance: _NodeInstance, result: Any) -> Any:+ """Liveness-verify one infra-flavoured error settle before it is+ applied.++ The supervisor reports a dead worker as a one-shot error settle, so+ the executor re-reads the worker after a short quiet period instead+ of trusting the first row. Returns the fresh result to apply -- a+ settle that landed meanwhile supersedes the error, and the same+ error read back means the death is verified -- or ``None`` when a+ concurrent pass (stop(), fail_fast) claimed the instance or the+ worker answered alive (the loop keeps collecting it).+ """+ from . import collect++ child_id = instance.child_id+ await self._sleep_fn(INFRA_VERIFY_QUIET_SECONDS)+ if instance.status != "running" or instance.child_id != child_id:+ return None # a stop, fail_fast, or resumed lane claimed the instance+ results = await collect([child_id], timeout_ms=0)+ if instance.status != "running" or instance.child_id != child_id:+ return None+ row = next((r for r in results if r.rlm_child_id == child_id), None)+ if row is not None and not row.settled:+ return None # the worker answered: alive, keep collecting it+ # No fresh row at all means the child vanished from the supervisor+ # (a hand delete): as dead as the error said, so the original+ # settle applies through the normal failure path.+ return row or result++ async def _respawn_infra_dead(+ self, run: FactoryRun, state: _StateRun, entry: _StateEntry, instance: _NodeInstance, result: Any+ ) -> None:+ """Re-spawn one worker the executor verified dead, free of charge.++ Infra is not the task's failure: the dead attempt counts as a+ settle (its ledger row keeps the error), but the instance is+ re-queued with the same rendered prompt -- spending neither a+ ``retries`` slot nor a fresh ``max_entries`` entry -- and the dead+ attempt's exit text is dropped, so no needs-verify mark is raised+ for work the respawn redoes. ``infra_respawned`` keeps this to ONE+ free respawn; a second verified death falls through to the normal+ retries/failure_policy path.+ """+ from . import delete_subagent++ old_child = instance.child_id or ""+ # The worker may be alive-but-unreachable rather than dead: retire+ # it so the respawn cannot run beside its zombie in the same+ # worktree. Best-effort and bounded — the host's delete kills under+ # the daemon's own timeout, the rest is registry bookkeeping — so+ # an already-gone child or a dead bridge must not wedge the+ # respawn; the supervisor-side zombie sweep this pairs with lands+ # in PR 2 of the [factory] group.+ try:+ await delete_subagent(old_child)+ except Exception:+ pass+ if instance.status != "running" or run.state not in ("running", "paused"):+ return # a stop()/fail_fast pass claimed the instance mid-delete+ # A sibling may have paused the run in this same collect batch.+ # Preserve the verified death as pending; resume admits it for free+ # instead of collecting the deleted child again.+ instance.duration_ms = result.duration_ms+ instance.tool_uses = result.tool_use_count or 0+ run.settle_count += 1+ run.tool_use_total += instance.tool_uses+ instance.infra_respawned = True+ instance.retry_base += 1+ instance.status = "pending"+ instance.error = None+ instance.answer = None+ instance.provisional = False+ instance.capture_failed = False+ reason = result.error or "child worker unreachable"+ self._event(+ run,+ "settled",+ node=state.state_id,+ entry=entry.index,+ instance=instance.index,+ status="error",+ error=reason,+ duration_ms=instance.duration_ms,+ )+ self._event(+ run,+ "infra_respawn",+ node=state.state_id,+ entry=entry.index,+ instance=instance.index,+ detail=(+ f"worker verified dead ({reason}); re-spawning the same rendered prompt "+ "once (no retry, no max_entries)"+ ),+ )+def _entry_answer(self, entry: _StateEntry) -> str | None:"""Captured answer for binding: one preview, or all instances joined."""answers = [instance.answer for instance in entry.instances if instance.status == "done" and instance.answer]@@ -2903,7 +3120,7 @@ class FactoryExecutor:entry.outputs = outputsentry.output_errors = errors- async def _capture_outputs(self, run: FactoryRun, state: _StateRun, entry: _StateEntry) -> None:+ async def _capture_outputs(self, run: FactoryRun, state: _StateRun, entry: _StateEntry) -> str | None:"""Capture the settle's outputs, retrying the capture once.A settle can race the child's answer capture on the host side (a@@ -2911,10 +3128,13 @@ class FactoryExecutor:that binds no value from the first capture is exactly the M2silent-failure class: the downstream would run on nothing. When adeclared output binds no value, the settled children are- re-collected ONCE (a settled child keeps its result until deleted)+ refreshed in one bounded capture window (a settled child keeps its+ result until deleted; missing text waits for the worker hand-off),and the capture re-runs over the refreshed answers; a port thatstill binds nothing records a deterministic capture failure in the- ledger (``output_capture_failed``) instead of passing silently.+ ledger (``output_capture_failed``) and fails the entry -- its+ joined port errors return as the task-failure reason, ``None``+ when every declared port bound a value."""self._capture_ports(state, entry)declared = state.spec.get("outputs") or []@@ -2925,10 +3145,11 @@ class FactoryExecutor:if not declared or not any(out.get("name") not in (entry.outputs or {}) for out in declared):- return # every declared port bound a value+ return None # every declared port bound a valuerefreshed = await self._refresh_entry_answers(run, state, entry)if refreshed:self._capture_ports(state, entry)+ failures = []for out in declared:name = out.get("name")error = (entry.output_errors or {}).get(name)@@ -2944,10 +3165,12 @@ class FactoryExecutor:error=error,detail=f"output {name!r} did not bind after one capture retry",)+ failures.append(error)+ return "; ".join(failures) or Noneasync def _refresh_entry_answers(self, run: FactoryRun, state: _StateRun, entry: _StateEntry) -> bool:- """Re-collect the entry's settled children once; refresh any longer- captured answer. Returns whether any answer changed."""+ """Refresh settled answers, waiting one bounded poll window for a+ missing hand-off. Returns whether any answer changed."""from . import collectchild_ids = [@@ -2957,30 +3180,45 @@ class FactoryExecutor:]if not child_ids:return False- try:- results = await collect(child_ids, timeout_ms=0)- except Exception as exc:- self._event(- run,- "output_capture_failed",- node=state.state_id,- entry=entry.index,- error=f"capture retry could not re-collect the settled children: {exc}",- detail="the first capture stands",- )- return False- by_child = {result.rlm_child_id: result for result in results}+ # An idle worker may expose its settle before handing off its final+ # assistant text. Give that missing capture one bounded poll window;+ # existing nonempty (but invalid) output still gets only one refresh.+ deadline = self._now_fn() + POLL_TIMEOUT_MS / 1000changed = False- for instance in entry.instances:- result = by_child.get(instance.child_id or "")- if result is None:- continue- answer = (result.answer_text or result.answer_preview or "")[:ANSWER_BINDING_CAP] or None- if answer and (instance.answer is None or len(answer) > len(instance.answer)):- instance.answer = answer- changed = True- if changed:- entry.answer = self._entry_answer(entry)+ while entry.status == "running":+ try:+ results = await collect(child_ids, timeout_ms=0)+ except Exception as exc:+ self._event(+ run,+ "output_capture_failed",+ node=state.state_id,+ entry=entry.index,+ error=f"capture retry could not re-collect the settled children: {exc}",+ detail="the first capture stands",+ )+ return changed+ if entry.status != "running":+ return changed # stop/fail_fast claimed the entry during collect+ by_child = {result.rlm_child_id: result for result in results}+ for instance in entry.instances:+ result = by_child.get(instance.child_id or "")+ if result is None:+ continue+ answer = (result.answer_text or result.answer_preview or "")[:ANSWER_BINDING_CAP] or None+ if answer and (instance.answer is None or len(answer) > len(instance.answer)):+ instance.answer = answer+ changed = True+ if changed:+ entry.answer = self._entry_answer(entry)+ missing = any(+ instance.status == "done" and instance.child_id is not None and not instance.answer+ for instance in entry.instances+ )+ remaining = deadline - self._now_fn()+ if not missing or remaining <= 0:+ return changed+ await self._sleep_fn(min(0.05, remaining))return changedasync def _apply_instance_failure(@@ -3007,8 +3245,15 @@ class FactoryExecutor:# count the pending instance as in-flight, so the control loop# would never finish the run. The failed instance settles as an# error instead; the policy call below is a no-op on a terminal- # entry.- if retry and instance.attempt <= retries and entry.status == "running":+ # entry. ``attempt - retry_base`` charges only the admissions+ # since the last budget reset: an infra respawn and a resume's+ # re-admission restart the declared retries without zeroing the+ # spawn counter the labels count on.+ if (+ retry+ and instance.attempt - instance.retry_base <= retries+ and entry.status == "running"+ ):instance.status = "pending"instance.error = Noneself._event(@@ -3047,13 +3292,27 @@ class FactoryExecutor:# status-level needs_verify signal hand the operator the# revival-race decision instead of a silent drop.entry.needs_verify = True- self._event(- run,- "needs_verify",- node=state.state_id,- entry=entry.index,- detail="child exit captured a provisional answer - verify the remote state",- )+ # The detail names what the mark preserves: a settle whose+ # declared output never bound, a respawned worker's last words+ # after its own death, or a plain child exit. The capture mark+ # is checked first: ``infra_respawned`` outlives its respawn,+ # so a respawned attempt that later settles and fails capture+ # preserves the capture's provenance, not the old death's.+ respawned = any(instance.infra_respawned for instance in entry.instances)+ capture_failed = any(instance.capture_failed for instance in entry.instances)+ if capture_failed:+ detail = (+ "output capture failed; the settled answer is preserved as a "+ "provisional answer - fix the declared output or verify by hand"+ )+ elif respawned:+ detail = (+ "worker verified dead by the executor after its automatic respawn; "+ "last assistant text preserved as a provisional answer"+ )+ else:+ detail = "child exit captured a provisional answer - verify the remote state"+ self._event(run, "needs_verify", node=state.state_id, entry=entry.index, detail=detail)if entry.status in TERMINAL_ENTRY_STATUSES:return # the policy already ran for this entrypolicy = state.spec.get("failure_policy", RUN_FAILURE_POLICY_DEFAULT)@@ -3077,9 +3336,14 @@ class FactoryExecutor:instance=instance.index,detail="entry failed before admission",)- # The failed entry settles too: guard-less transitions (the compiled- # dag's depends_on edges) fire from error settles so dependents run.- self._queue_settle(run, state, entry)+ if policy != "escalate":+ # The failed entry settles: guard-less transitions (the compiled+ # dag's depends_on edges) fire from error settles so dependents+ # run. Escalate instead HOLDS the failed entry unsettled -- the+ # pause lands before the failure propagates, no downstream entry+ # budget is spent on it, and resume() re-admits the entry as a+ # fresh attempt.+ self._queue_settle(run, state, entry)if run.state != "running":# stop() (or another transition) owns the run state now; keep the# entry's error but do not overwrite the final state.@@ -3110,7 +3374,8 @@ class FactoryExecutor:await self._milestone(run,"paused",- f"state {state.state_id} failed: {reason}; resume with await rlm.factory.resume('{run.run_id}')",+ f"state {state.state_id} failed: {reason}; resume with await rlm.factory.resume('{run.run_id}') "+ "to re-admit the failed entry as a fresh attempt",node=state.state_id,)@@ -3597,7 +3862,11 @@ async def stop_factory(run_id: str) -> dict[str, Any]:async def resume_factory(run_id: str) -> dict[str, Any]:- """Resume a paused run (escalate, budget, or max_transitions pause)."""+ """Resume a paused run (escalate, budget, or max_transitions pause).++ An escalate pause re-admits the failed entry as a fresh attempt of the+ same rendered prompt first; see ``FactoryExecutor.resume``.+ """require_factory_enabled()return await default_factory_executor().resume(run_id)@@ -4726,7 +4995,8 @@ parent session after the run ends. Both worked examples bound theiremitted payloads in the prompt — a capped findings list here, a cappedfile list in the review-sweep example — because the executor's capturecap still bounds very large payloads: an unbounded payload truncates at-the cap and fails to bind.+the cap, fails to capture, and fails the producing state through its+`retries` and `failure_policy` instead of settling valueless.## Authoring reference@@ -4753,11 +5023,17 @@ the cap and fails to bind.`{input_name}` placeholders (one pass; inputs without a placeholder areappended in a trailing `## Inputs` section). A required input whose sourcehas not settled yet keeps the entry pending; over a settled source that- offers no value (an errored settle, a port the settle captured no value- for, or a JSON capture failure) it fails the dependent entry, while+ offers no value (an errored settle, or a settle that captured nothing for+ the port) it fails the dependent entry, while`"optional": true` binds a null sentinel in every no-value case (asource that offers no value is not a value, so the dependent that- declared the input optional proceeds). A required self-input is+ declared the input optional proceeds). A state's own declared outputs+ capture from its settle's full answer: a declared port that binds no+ value — no parseable fenced JSON for it, or a payload past the+ executor's capture cap — is retried once and then FAILS the producing+ entry as a task failure (`retries`, then `failure_policy`, apply, and+ the answer stays preserved under `needs_verify`), so a valueless+ settle never reaches a dependent's bind. A required self-input isrejected at validation —`state X input 'name' cannot require itself: mark the self-input optional- a required one can never bind on the state's first entry` — while an@@ -4774,11 +5050,12 @@ the cap and fails to bind.dotted path into a `json` output; `eq`/`ne` compare JSON-strictly (aboolean never equals a number), comparison ops need a numeric value,`contains` a non-empty list, `exists` no value, and a missing or- unparseable port fails every op except `exists`. A failed settle still- fires guard-less transitions, so dependents under `continue` run; their- required input over the failed source then fails the dependent entry,- while an optional input over the failed source binds the null sentinel- and the dependent proceeds.+ unparseable port fails every op except `exists`. Under `continue` a+ failed settle still fires guard-less transitions, so dependents run;+ their required input over the failed source then fails the dependent+ entry, while an optional input over the failed source binds the null+ sentinel and the dependent proceeds. Under the default `escalate` the+ failure never propagates that far (see bounds and policies).- **Cycles are legal**: there is no acyclicity requirement — self-loops andback edges validate. The one rule is an entry state somewhere; a dagwhose every node depends on another compiles to no entry states and is@@ -4806,9 +5083,19 @@ the cap and fails to bind.(0..10, same rendered prompt), `budget_ms` (admission to settlement;exceeding it fails the attempt without a retry), and `failure_policy` —`fail_fast` (cancel every child, run failed), `continue` (entry stays- errored; the run finishes and reports failed if any state errored), or- `escalate` (the default: pause the run; resuming is the operator's- decision).+ errored; the run finishes and reports failed if any state errored; its+ failed settle fires guard-less transitions so dependents run), or+ `escalate` (the default: pause the run BEFORE the failed settle+ propagates — no transition fires and no downstream entry is spent — and+ `resume` re-admits the failed entry as a fresh attempt: same rendered+ prompt, same entry slot, fresh `retries`; a binding failure over a+ broken upstream re-binds and pauses again, so stop and fix the upstream+ or author the dependent's state with `continue`). A child whose worker+ died or whose supervisor link was lost is infra, not the task's+ failure: the executor re-checks it after a short quiet period, then+ re-spawns the same prompt once without spending `retries` or+ `max_entries`; a second death fails through `retries`/`failure_policy`+ like any task failure.- **Retry re-renders the same prompt into a changed world**: a retry runsthe same prompt after an earlier attempt already changed the externalstate it works on, so a prompt that mutates external state — a fixing@@ -4890,9 +5177,12 @@ status["events"] # trailing ledger: spawned, settled, answer_captured,# transition_fired, node_error, milestone, ...status["usage"] # spawns, settled, tool_uses, max_parallel, max_children, running,# transitions_fired-status["needs_verify"] # (when set) states whose child EXITED carrying a- # provisional answer - verify the remote state; the- # entry row carries the provisional_answer preview+status["needs_verify"] # (when set) states whose entry keeps a+ # provisional answer: a child exit with text, or a+ # settled answer whose declared output failed to+ # capture (worker deaths are liveness-checked and+ # re-spawned by the executor first) - the entry row+ # carries the provisional_answer previewstatus["pause_reason"] # (paused) why the run paused; with "last_error" (thestatus["last_error"] # last admission/bind failure) and a one-line "remedy"```@@ -4935,7 +5225,8 @@ watched = await rlm.factory.watch(result["run_id"], 30)conversation once per kind per run, the pause notices with the resumecall spelled out — a paused run does not need polling to be noticed.- `stop(run_id)` cancels every running child of the run (idempotent);- `resume(run_id)` continues a paused run and raises on a non-paused one.+ `resume(run_id)` continues a paused run and raises on a non-paused one;+ after an escalate pause it first re-admits the failed entry.- The activity lane the daemon and TUI speak is camelCase on the wire(`runId`, `specId`, `timeoutMs`); the kernel API here(`rlm.factory.*`) is snake_case.diff --git a/prime-agent-runtime/test/test_factory.py b/prime-agent-runtime/test/test_factory.pyindex 54a6e347ff..1c973bfd59 100644--- a/prime-agent-runtime/test/test_factory.py+++ b/prime-agent-runtime/test/test_factory.py@@ -2493,10 +2493,18 @@ class FactoryHelpTest(unittest.TestCase):# The worked examples bound their emitted payloads — captured answers# are capped previews (~160-200 chars), so an unbounded json payload- # would truncate at the cap and fail to bind.+ # would truncate at the cap, fail to capture, and fail its state.self.assertIn('findings": ["at most three one-line findings"]', doc)self.assertIn("capped at the eight most relevant", doc)- self.assertIn("an unbounded payload truncates at the cap and fails to bind", flat)+ self.assertIn(+ "an unbounded payload truncates at the cap, fails to capture, "+ "and fails the producing state",+ flat,+ )+ # The output-capture contract: a declared port that never binds+ # fails the producing entry as a task failure.+ self.assertIn("is retried once and then FAILS the producing", flat)+ self.assertIn("the answer stays preserved under `needs_verify`", flat)# The Discovery section ships the machine library as present (this# branch merges it): the bundled location, the seed names, the CLI@@ -3578,7 +3586,12 @@ class FactoryExecutorTest(_ExecutorTestCase):self.assertEqual([call["prompt"] for call in self.host.spawn_calls("b")], ["Process 7."])@async_test- async def test_bad_json_input_fails_node_without_spawning(self) -> None:+ async def test_unparseable_output_fails_the_producer_and_never_spawns_the_dependent(self) -> None:+ # a's declared json output never parses: the capture failure fails a+ # itself (defect C) instead of settling it valueless, so b's required+ # input over the errored source fails at binding without ever+ # spawning; c still ran and finished, so the run completes but+ # reports failed.self.host.outcomes["a"] = {"status": "done", "answer": "not json at all"}self.store_factory({@@ -3597,12 +3610,13 @@ class FactoryExecutorTest(_ExecutorTestCase):)result = await self.start()status = await self.settle(result)- # b failed at binding (continue policy): never spawned, marked error;- # c still ran and finished, so the run completes but reports failed.self.assertEqual(self.host.spawn_calls("b"), [])+ a = self.node_status(status, "a")+ self.assertEqual(a["status"], "error")+ self.assertIn("no JSON object containing output 'result'", a["error"])b = self.node_status(status, "b")self.assertEqual(b["status"], "error")- self.assertIn("no JSON object containing output", b["error"])+ self.assertIn("is unavailable (latest settle status 'error')", b["error"])self.assertEqual(self.node_status(status, "c")["status"], "done")self.assertEqual(status["state"], "failed")@@ -3819,6 +3833,61 @@ class FactoryExecutorTest(_ExecutorTestCase):# null, not the no-value sentinel path.self.assertIn("- data: null", self.host.spawn_calls("dep")[0]["prompt"])+ @async_test+ async def test_uncaptured_output_fails_the_producing_entry_instead_of_the_dependent(self) -> None:+ # Defect C: a done settle whose declared output never bound (no+ # parseable value after the one capture retry) used to settle the+ # entry done and queue the settle -- the dependent's required input+ # then failed over the valueless source, spending its entry budget+ # on the producer's failure. The capture failure now fails the+ # PRODUCING entry as a task failure at the origin: retries, then+ # failure_policy, apply; no settle is queued, so escalate pauses at+ # the producer and the dependent never enters.+ self.host.outcomes["setup"] = {"status": "done", "answer": "no json here"}+ self.store_machine(+ {+ "states": [+ {+ "id": "setup",+ "entry": True,+ "subagent": "worker",+ "outputs": [{"name": "config", "type": "json"}],+ "retries": 1,+ },+ {+ "id": "build",+ "subagent": "worker",+ "inputs": [{"name": "config", "type": "json", "from": "setup.config"}],+ "max_entries": 4,+ },+ ],+ "transitions": [{"from": "setup", "to": "build"}],+ }+ )+ result = await self.start()+ paused = await self.settle(result)+ self.assertEqual(paused["state"], "paused")+ setup = self.state_report(paused, "setup")+ self.assertEqual(setup["status"], "error")+ self.assertIn("config", setup["error"])+ # the failed capture spent the one retry re-spawning the same+ # prompt; the second capture failure is permanent+ self.assertEqual(len(self.host.spawn_calls("setup")), 2)+ self.assertEqual(len(self.all_events_of(result, "retry")), 1)+ self.assertEqual(len(self.all_events_of(result, "output_capture_failed")), 2)+ # the dependent never entered on the unbound required input+ self.assertEqual(self.state_report(paused, "build")["entries_used"], 0)+ self.assertEqual(self.host.spawn_calls("build"), [])+ self.assertEqual(self.all_events_of(result, "transition_fired"), [])+ # the settled answer is preserved as provisional, so the permanent+ # failure surfaces needs_verify with the answer for review+ self.assertEqual(paused["needs_verify"], ["setup"])+ self.assertIn("no json here", setup["entries"][0]["provisional_answer"])+ # the paused milestone tells the operator resume re-admits the entry+ paused_notice = next(n for n in self.host.notices if n["kind"] == "paused")+ self.assertIn("state setup failed", paused_notice["detail"])+ self.assertIn("re-admit", paused_notice["detail"])+@async_testasync def test_capture_failure_names_the_binding_cap_and_records_the_event(self) -> None:# A declared json output bigger than the executor's binding cap@@ -4147,38 +4216,103 @@ class FactoryExecutorTest(_ExecutorTestCase):self.assertEqual(self.host.notice_kinds(), ["failed"])@async_test- async def test_escalate_pauses_notifies_and_resume_continues(self) -> None:- self.host.outcomes["a"] = {"status": "error", "error": "boom"}- self.store_factory(+ async def test_escalate_resume_readmits_the_failed_entry(self) -> None:+ # Defect A (live run fa1a8f1c): resume after an escalate pause fired+ # the ERRORED settle forward -- the guard-less transition entered the+ # dependent, its required input failed to bind over the errored+ # source, and it spent one of its max_entries entries on nothing.+ # The escalate pause now lands BEFORE the failure propagates, and+ # resume re-admits the failed entry as a fresh attempt of the same+ # rendered prompt in the same entry slot.+ self.host.child_outcomes["child-1"] = {"status": "error", "error": "boom"}+ self.host.child_outcomes["child-2"] = {"status": "done", "answer": "impl-done"}+ self.store_machine({- "nodes": [- {"id": "a", "subagent": "worker"},- {"id": "b", "subagent": "worker", "depends_on": ["a"]},+ "states": [+ {+ "id": "implement",+ "entry": True,+ "subagent": "worker",+ "outputs": [{"name": "result", "type": "text"}],+ },+ {+ "id": "review",+ "subagent": "worker",+ "inputs": [{"name": "result", "type": "text", "from": "implement.result"}],+ "max_entries": 4,+ },],+ "transitions": [{"from": "implement", "to": "review"}],})result = await self.start()paused = await self.settle(result)self.assertEqual(paused["state"], "paused")- self.assertEqual(self.node_status(paused, "a")["status"], "error")- self.assertEqual(self.node_status(paused, "b")["status"], "pending")- self.assertEqual(self.host.spawn_calls("b"), [])+ self.assertEqual(self.node_status(paused, "implement")["status"], "error")+ self.assertEqual(self.node_status(paused, "review")["entries_used"], 0)self.assertIn("paused", self.host.notice_kinds())- # resume() restarts the run: b is ready (a is terminal) and finishes.+ # resume re-admits the failed entry: a fresh attempt of the same+ # rendered prompt, the same entry slot -- no extra max_entries.resumed = await rlm_module.rlm.factory.resume(result["run_id"])self.assertEqual(resumed["state"], "running")final = await self.settle(result)- self.assertEqual(final["state"], "failed") # a errored, b done- self.assertEqual(self.node_status(final, "b")["status"], "done")- self.assertEqual(self.host.notice_kinds(), ["paused", "failed"])+ self.assertEqual(final["state"], "done")+ implement = self.node_status(final, "implement")+ self.assertEqual(implement["status"], "done")+ self.assertEqual(implement["entries_used"], 1)+ self.assertEqual([entry["status"] for entry in implement["entries"]], ["done"])+ self.assertEqual(len(self.host.spawn_calls("implement")), 2)+ review = self.node_status(final, "review")+ self.assertEqual(review["status"], "done")+ self.assertEqual(review["entries_used"], 1)+ self.assertIn("impl-done", self.host.spawn_calls("review")[0]["prompt"])+ self.assertEqual(self.host.notice_kinds(), ["paused", "finished"])# a second resume on the finished run must refusewith self.assertRaisesRegex(ValueError, "not paused"):await rlm_module.rlm.factory.resume(result["run_id"])+ @async_test+ async def test_escalate_pause_spends_no_downstream_entry(self) -> None:+ # The pre-propagation pause: the errored settle never fires the+ # guard-less transition, so the dependent's entry budget is never+ # spent on the failure. Resume re-admits the failed entry, and its+ # own failure re-pauses the run with the same reason.+ self.host.outcomes["implement"] = {"status": "error", "error": "boom"}+ self.store_machine(+ {+ "states": [+ {+ "id": "implement",+ "entry": True,+ "subagent": "worker",+ "outputs": [{"name": "result", "type": "text"}],+ },+ {+ "id": "review",+ "subagent": "worker",+ "inputs": [{"name": "result", "type": "text", "from": "implement.result"}],+ "max_entries": 4,+ },+ ],+ "transitions": [{"from": "implement", "to": "review"}],+ }+ )+ result = await self.start()+ paused = await self.settle(result)+ self.assertEqual(paused["state"], "paused")+ await rlm_module.rlm.factory.resume(result["run_id"])+ second = await self.settle(result)+ self.assertEqual(second["state"], "paused")+ self.assertEqual(second["pause_reason"], "boom")+ self.assertEqual(self.node_status(second, "review")["entries_used"], 0)+ self.assertEqual(self.all_events_of(result, "transition_fired"), [])+ implement = self.node_status(second, "implement")+ self.assertEqual(implement["entries_used"], 1)+ self.assertEqual(len(self.host.spawn_calls("implement")), 2)+@async_testasync def test_resume_defers_rate_limited_admissions(self) -> None:- self.host.outcomes["a"] = {"status": "error", "error": "boom"}- self.host.rate_limit_first["b"] = 2+ self.host.child_outcomes["child-1"] = {"status": "error", "error": "boom"}self.store_factory({"nodes": [@@ -4190,6 +4324,9 @@ class FactoryExecutorTest(_ExecutorTestCase):result = await self.start()paused = await self.settle(result)self.assertEqual(paused["state"], "paused")+ # The rate limit hits a's re-admission, not b: escalate holds the+ # failed entry unsettled and resume() re-admits it first.+ self.host.rate_limit_first["a"] = 3sleep_mark = len(self.sleeps.sleeps)resumed = await rlm_module.rlm.factory.resume(result["run_id"])# resume() must not sleep in the calling turn: the rate-limited@@ -4197,11 +4334,14 @@ class FactoryExecutorTest(_ExecutorTestCase):self.assertEqual(resumed["started"], [])self.assertEqual(self.sleeps.sleeps[sleep_mark:], [])final = await self.settle(result)- self.assertEqual(final["state"], "failed") # a errored, b done+ self.assertEqual(final["state"], "done") # a redone, b done+ self.assertEqual(self.node_status(final, "a")["status"], "done")self.assertEqual(self.node_status(final, "b")["status"], "done")- # b needed 3 admissions total: one deferred at resume, then one- # rate-limited retry inside the loop's backoff, then the success.- self.assertEqual(len(self.host.spawn_calls("b")), 3)+ # a needed 4 admissions total: the failed first, one deferred at+ # resume, one rate-limited retry inside the loop's backoff, then+ # the success; b then runs once.+ self.assertEqual(len(self.host.spawn_calls("a")), 4)+ self.assertEqual(len(self.host.spawn_calls("b")), 1)self.assertEqual(self.sleeps.sleeps[sleep_mark:], [1.0])@async_test@@ -4209,7 +4349,9 @@ class FactoryExecutorTest(_ExecutorTestCase):# A run that pauses, resumes, and pauses again must record the second# pause in the ledger even though the parent notice fires once per# kind: status() is the only reader of the repeat.- self.host.outcomes["a"] = {"status": "error", "error": "a boom"}+ # a fails once and its re-admitted retry succeeds, so the second+ # pause is b's own failure -- the milestone the repeat must record.+ self.host.child_outcomes["child-1"] = {"status": "error", "error": "a boom"}self.host.outcomes["b"] = {"status": "error", "error": "b boom"}self.store_machine({@@ -4259,6 +4401,168 @@ class FactoryExecutorTest(_ExecutorTestCase):self.assertEqual(self.node_status(status, "a")["status"], "error")self.assertEqual(status["state"], "failed")+ # -- worker deaths (infra) --------------------------------------------------------++ @async_test+ async def test_worker_death_respawns_once_without_spending_retries_or_max_entries(self) -> None:+ # Defect B (live run fa1a8f1c): "Child worker unreachable" consumed+ # the entry and, with default retries=0, escalated into a pause --+ # twenty minutes of child work discarded on an infra hiccup. A+ # verified worker death is infra, not the task's failure: the+ # executor liveness-verifies it itself (a short quiet period, then+ # one re-collect) and re-spawns the same rendered prompt once,+ # spending neither retries nor a fresh max_entries slot.+ self.host.child_outcomes["child-1"] = {+ "status": "error",+ "error": "Child worker unreachable",+ "answer": "partial work",+ }+ self.store_factory({"nodes": [{"id": "work", "subagent": "worker"}]})+ result = await self.start()+ status = await self.settle(result)+ self.assertEqual(status["state"], "done")+ self.assertEqual(self.host.notice_kinds(), ["finished"])+ work = self.node_status(status, "work")+ self.assertEqual(work["status"], "done")+ self.assertEqual(work["entries_used"], 1)+ self.assertEqual(len(self.host.spawn_calls("work")), 2)+ # the dead attempt's exit text was dropped, not marked for verify:+ # the respawn redid the work and the entry settled for real+ self.assertNotIn("needs_verify", status)+ self.assertNotIn("needs_verify", work)+ # the executor verified the death itself before re-spawning+ self.assertIn(factory_module.INFRA_VERIFY_QUIET_SECONDS, self.sleeps.sleeps)++ @async_test+ async def test_second_worker_death_falls_through_retries_and_policy(self) -> None:+ # Only ONE free respawn: a second verified death charges the normal+ # retries budget (the free respawn consumed none) and then the+ # failure_policy applies like any task failure.+ self.host.outcomes["work"] = {"status": "error", "error": "Child worker unreachable"}+ self.store_factory(+ {+ "run": {"failure_policy": "continue"},+ "nodes": [{"id": "work", "subagent": "worker", "retries": 1}],+ }+ )+ result = await self.start()+ status = await self.settle(result)+ self.assertEqual(status["state"], "failed")+ work = self.node_status(status, "work")+ self.assertEqual(work["status"], "error")+ # initial spawn, free respawn, one charged retry: three admissions+ self.assertEqual(len(self.host.spawn_calls("work")), 3)+ self.assertEqual(work["attempts"], 3)+ self.assertEqual(len(self.all_events_of(result, "infra_respawn")), 1)+ self.assertEqual(len(self.all_events_of(result, "retry")), 1)++ @async_test+ async def test_paused_collect_batch_preserves_the_free_infra_respawn(self) -> None:+ self.host.child_outcomes["child-1"] = {"status": "error", "error": "task failed"}+ self.host.child_outcomes["child-2"] = {+ "status": "error", "error": "Child worker unreachable"+ }+ self.store_factory({"nodes": [+ {"id": "a", "subagent": "worker"},+ {"id": "b", "subagent": "worker"},+ ]})+ result = await self.start()+ run = self.executor._runs[result["run_id"]]+ await run.task # finish the whole batch, including b after a pauses it+ self.assertEqual(run.state, "paused")+ instance = run.states["b"].entries[0].instances[0]+ self.assertEqual(instance.status, "pending")+ self.assertTrue(instance.infra_respawned)+ self.assertEqual(self.host.deleted_targets(), ["child-2"])+ await rlm_module.rlm.factory.resume(result["run_id"])+ status = await self.settle(result, max_polls=1000)+ self.assertEqual(status["state"], "done")+ self.assertEqual(len(self.host.spawn_calls("b")), 2)+ self.assertEqual(self.node_status(status, "b")["entries_used"], 1)+ self.assertEqual(self.all_events_of(result, "retry"), [])+ self.assertEqual(len(self.all_events_of(result, "infra_respawn")), 1)++ @async_test+ async def test_stop_during_output_capture_preserves_cancelled_entry(self) -> None:+ self.host.outcomes["work"] = {"status": "done", "answer": "no JSON"}+ capture_gate = self.host.gate("rlm.collect", 2)+ capture_entered = self.host.gate_entered("rlm.collect", 2)+ self.store_factory({"nodes": [{+ "id": "work", "subagent": "worker",+ "outputs": [{"name": "data", "type": "json"}],+ }]})+ result = await self.start()+ run = self.executor._runs[result["run_id"]]+ await capture_entered.wait()+ await rlm_module.rlm.factory.stop(result["run_id"])+ entry = run.states["work"].entries[0]+ self.assertEqual(entry.status, "cancelled")+ capture_gate.set()+ await run.task+ self.assertEqual(run.state, "stopped")+ self.assertEqual(entry.status, "cancelled")+ self.assertFalse(entry.instances[0].provisional)+ self.assertEqual(self.all_events_of(result, "retry"), [])++ @async_test+ async def test_worker_death_recheck_adopts_a_live_settle(self) -> None:+ # The liveness re-check is not a rubber stamp: when the worker+ # answers again during the quiet period, its fresh settle supersedes+ # the error row and nothing is re-spawned.+ self.host.child_outcomes["child-1"] = {+ "status": "error",+ "error": "Child worker unreachable",+ }+ host = self.host++ class ReviveSleep(ClockSleep):+ """The executor's quiet-period sleep revives child-1: the+ worker was never dead, and the re-read must see it settled."""++ async def __call__(self, seconds: float) -> None:+ await super().__call__(seconds)+ host.child_outcomes["child-1"] = {"status": "done", "answer": "late done"}++ self.executor._sleep_fn = ReviveSleep(self.clock)+ self.store_factory({"nodes": [{"id": "work", "subagent": "worker"}]})+ result = await self.start()+ status = await self.settle(result)+ self.assertEqual(status["state"], "done")+ self.assertEqual(len(self.host.spawn_calls("work")), 1)+ self.assertEqual(self.all_events_of(result, "infra_respawn"), [])+ self.assertEqual(self.node_status(status, "work")["answer_preview"], "late done")++ @async_test+ async def test_resume_grants_fresh_retries_and_a_fresh_respawn(self) -> None:+ # The approved resume semantics: a re-admitted entry restarts both+ # budgets. The charged retry after the free respawn must be granted+ # again, and an infra death of the re-admitted attempt must get its+ # own free respawn instead of failing through a spent one.+ self.host.child_outcomes["child-1"] = {"status": "error", "error": "Child worker unreachable"}+ self.host.child_outcomes["child-2"] = {"status": "error", "error": "boom"}+ self.host.child_outcomes["child-3"] = {"status": "error", "error": "boom"}+ self.host.child_outcomes["child-4"] = {"status": "error", "error": "Child worker unreachable"}+ self.store_factory({"nodes": [{"id": "work", "subagent": "worker", "retries": 1}]})+ result = await self.start()+ paused = await self.settle(result)+ # infra death -> free respawn; the respawn's own task failure spent+ # the one charged retry; the retry failed too -> escalate pause.+ self.assertEqual(paused["state"], "paused")+ self.assertEqual(len(self.host.spawn_calls("work")), 3)+ self.assertEqual(len(self.all_events_of(result, "infra_respawn")), 1)+ self.assertEqual(len(self.all_events_of(result, "retry")), 1)+ # resume re-admits the entry with fresh budgets: the re-admitted+ # attempt's infra death respawns free again, and this time it+ # settles done.+ await rlm_module.rlm.factory.resume(result["run_id"])+ final = await self.settle(result)+ self.assertEqual(final["state"], "done")+ self.assertEqual(self.node_status(final, "work")["entries_used"], 1)+ self.assertEqual(len(self.host.spawn_calls("work")), 5)+ self.assertEqual(len(self.all_events_of(result, "infra_respawn")), 2)+ self.assertEqual(len(self.all_events_of(result, "retry")), 1)+ self.assertEqual(self.host.notice_kinds(), ["paused", "finished"])+# -- budgets ----------------------------------------------------------------------@async_test@@ -4919,7 +5223,7 @@ class FactoryExecutorTest(_ExecutorTestCase):# contract it validated is pinned here: run-level milestones send# {run_id, kind, detail} (no node), node-level ones add "node", and# every notice goes through one "factory.progress" host request.- self.host.outcomes["a"] = {"status": "error", "error": "boom"}+ self.host.child_outcomes["child-1"] = {"status": "error", "error": "boom"}self.store_factory({"nodes": [@@ -4933,20 +5237,20 @@ class FactoryExecutorTest(_ExecutorTestCase):self.assertEqual(paused["state"], "paused")resume = await rlm_module.rlm.factory.resume(result["run_id"])final = await self.settle(result)- self.assertEqual(final["state"], "failed")+ self.assertEqual(final["state"], "done")kinds = self.host.notice_kinds()- self.assertEqual(kinds, ["paused", "failed"])- paused_notice, failed_notice = self.host.notices+ self.assertEqual(kinds, ["paused", "finished"])+ paused_notice, finished_notice = self.host.noticesself.assertEqual(paused_notice["kind"], "paused")self.assertEqual(paused_notice["run_id"], result["run_id"])self.assertIn("state a failed", paused_notice["detail"])self.assertIn("resume", paused_notice["detail"])self.assertEqual(paused_notice.get("node"), "a") # node-level milestone- self.assertEqual(failed_notice["kind"], "failed")- self.assertEqual(failed_notice["run_id"], result["run_id"])- self.assertIsNone(failed_notice.get("node")) # run-level milestone+ self.assertEqual(finished_notice["kind"], "finished")+ self.assertEqual(finished_notice["run_id"], result["run_id"])+ self.assertIsNone(finished_notice.get("node")) # run-level milestoneself.assertEqual(set(paused_notice.keys()), {"run_id", "kind", "detail", "node"})- self.assertEqual(set(failed_notice.keys()), {"run_id", "kind", "detail"})+ self.assertEqual(set(finished_notice.keys()), {"run_id", "kind", "detail"})# one host request per notice, typed factory.progressself.assertEqual(len(self.host.calls_of("factory.progress")), 2)@@ -4958,7 +5262,7 @@ class FactoryExecutorTest(_ExecutorTestCase):patcher = patch.object(rlm_module, "host_request", failing)patcher.start()self.addCleanup(patcher.stop)- failing.outcomes["a"] = {"status": "error", "error": "boom"}+ failing.child_outcomes["child-1"] = {"status": "error", "error": "boom"}self.store_factory({"nodes": [@@ -4989,7 +5293,7 @@ class FactoryExecutorTest(_ExecutorTestCase):# the run is not wedged: resume still works end to end.await rlm_module.rlm.factory.resume(result["run_id"])final = await self.settle(result)- self.assertEqual(final["state"], "failed")+ self.assertEqual(final["state"], "done")self.assertEqual(self.node_status(final, "b")["status"], "done")@@ -5813,6 +6117,88 @@ class FactoryExecutorTest(_ExecutorTestCase):)self.assertEqual(self.all_events_of(result, "output_capture_failed"), [])+ @async_test+ async def test_missing_answer_capture_has_a_bounded_poll_window(self) -> None:+ self.host.outcomes["work"] = {"status": "done"}+ self.store_factory({"nodes": [{"id": "work", "subagent": "worker",+ "outputs": [{"name": "data", "type": "json"}]}]})+ before = self.clock()+ result = await self.start()+ status = await self.settle(result)+ elapsed = self.clock() - before+ self.assertEqual(status["state"], "paused")+ self.assertGreater(elapsed, 0)+ self.assertLessEqual(elapsed, factory_module.POLL_TIMEOUT_MS / 1000 + 1e-9)+ self.assertEqual(len(self.host.spawn_calls("work")), 1)+ self.assertIn("captured no value", self.node_status(status, "work")["error"])++ @async_test+ async def test_stop_during_missing_answer_poll_preserves_cancellation(self) -> None:+ self.host.outcomes["work"] = {"status": "done"}+ sleeps = GatedSleep()+ self.executor._sleep_fn = sleeps+ self.store_factory({"nodes": [{"id": "work", "subagent": "worker",+ "outputs": [{"name": "data", "type": "json"}]}]})+ result = await self.start()+ run = self.executor._runs[result["run_id"]]+ await self.wait_until(sleeps.entered.is_set)+ await rlm_module.rlm.factory.stop(result["run_id"])+ sleeps.release.set()+ await run.task+ entry = run.states["work"].entries[0]+ self.assertEqual(run.state, "stopped")+ self.assertEqual(entry.status, "cancelled")+ self.assertFalse(entry.instances[0].provisional)+ self.assertEqual(self.all_events_of(result, "retry"), [])+ self.assertEqual(len(self.host.spawn_calls("work")), 1)++ @async_test+ async def test_capture_waits_for_the_settled_workers_answer_handoff(self) -> None:+ handoff_requested = asyncio.Event()+ answer_ready = asyncio.Event()++ class HandoffHost(FakeHost):+ empty_reads = 0++ async def __call__(self, request_type, payload=None):+ response = await super().__call__(request_type, payload)+ if request_type == "rlm.collect" and not answer_ready.is_set():+ for row in response["results"]:+ if row["rlm_child_id"] == "child-1":+ row.pop("answer_text", None)+ row.pop("answer_preview", None)+ self.empty_reads += 1+ if self.empty_reads >= 2:+ handoff_requested.set()+ return response++ class YieldingSleep(ClockSleep):+ async def __call__(self, seconds):+ await super().__call__(seconds)+ await yield_loop_turn()++ self.host = HandoffHost(clock=self.clock)+ self.host.outcomes["work"] = {"status": "done", "answer": '{"data": {"ok": true}}'}+ repatch = patch.object(rlm_module, "host_request", self.host)+ repatch.start()+ self.addCleanup(repatch.stop)+ self.executor._sleep_fn = YieldingSleep(self.clock)++ async def finish_handoff():+ await handoff_requested.wait()+ answer_ready.set()++ handoff = asyncio.create_task(finish_handoff())+ self.store_factory({"nodes": [{"id": "work", "subagent": "worker",+ "outputs": [{"name": "data", "type": "json"}]}]})+ result = await self.start()+ status = await self.settle(result)+ await handoff+ self.assertEqual(status["state"], "done")+ self.assertEqual(self.node_status(status, "work")["entries_used"], 1)+ self.assertEqual(len(self.host.spawn_calls("work")), 1)+ self.assertEqual(self.all_events_of(result, "output_capture_failed"), [])+@async_testasync def test_child_exit_captures_a_provisional_answer_and_marks_needs_verify(self) -> None:# M4: a child that dies without reporting is distinguishable from@@ -5919,7 +6305,9 @@ class FactoryExecutorTest(_ExecutorTestCase):async def test_paused_run_status_carries_the_last_error_and_remedy(self) -> None:# M6: a paused run carries the LAST failed admission/bind error and# a one-line remedy in its status payload — no root-side- # archaeology over pending-state lists to learn what failed.+ # archaeology over pending-state lists to learn what failed. The+ # pause here is src's own failed output capture (defect C): the+ # failure lands at the producer, before the dependent's bind.self.host.outcomes["src"] = {"status": "done", "answer": "no json here"}self.store_machine({@@ -5945,11 +6333,12 @@ class FactoryExecutorTest(_ExecutorTestCase):self.assertEqual(status["state"], "paused")self.assertEqual(status["pause_reason"],- "input 'data': no JSON object containing output 'data' in the upstream answer",+ "no JSON object containing output 'data' in the upstream answer",)- self.assertIn("state 'dep' failed", status["last_error"])+ self.assertIn("state 'src' failed", status["last_error"])self.assertIn("no JSON object containing output 'data'", status["last_error"])self.assertIn(f"rlm.factory.resume('{result['run_id']}')", status["remedy"])+ self.assertEqual(status["needs_verify"], ["src"])@async_testasync def test_machine_required_input_over_an_errored_source_fails_the_dependent_entry(self) -> None:@@ -5998,10 +6387,10 @@ class FactoryExecutorTest(_ExecutorTestCase):@async_testasync def test_machine_optional_input_over_a_settled_source_with_no_captured_value_binds_the_null_sentinel(self) -> None:# Review finding (PR #3199), second no-value condition: the source- # settled done but captured no value for the declared port (an- # empty answer). The optional dependent proceeds on the null- # sentinel; only the required dependent over the same valueless- # settle fails its entry.+ # settled but captured no value for the declared port (an empty+ # answer). The optional dependent proceeds on the null sentinel;+ # only the required dependent over the same valueless settle fails+ # its entry.self.host.outcomes["src"] = {"status": "done", "answer": ""}self.store_machine({@@ -6037,6 +6426,12 @@ class FactoryExecutorTest(_ExecutorTestCase):result = await self.start()status = await self.settle(result)self.assertEqual(status["state"], "failed")+ # the empty answer never captured the text port, so the capture+ # failure failed the source entry itself (defect C) instead of+ # settling it valueless+ src = self.state_report(status, "src")+ self.assertEqual([entry["status"] for entry in src["entries"]], ["error"])+ self.assertIn("captured no value", src["entries"][0]["error"])opt = self.state_report(status, "opt")self.assertEqual(opt["entries_used"], 1)self.assertEqual([entry["status"] for entry in opt["entries"]], ["done"])@@ -6045,7 +6440,7 @@ class FactoryExecutorTest(_ExecutorTestCase):self.assertEqual(req["entries_used"], 1)self.assertEqual([entry["status"] for entry in req["entries"]], ["error"])self.assertIn(- "input 'go' from state 'src' has no captured output 'go'",+ "input 'go' from state 'src' is unavailable (latest settle status 'error')",req["entries"][0]["error"],)self.assertEqual(self.host.spawn_calls("req"), [])@@ -6053,10 +6448,10 @@ class FactoryExecutorTest(_ExecutorTestCase):@async_testasync def test_machine_optional_input_over_a_failed_json_capture_binds_the_null_sentinel(self) -> None:# Review finding (PR #3199), third no-value condition: the source- # settled done, declared a json port, but its answer never parsed- # for that port (a JSON capture failure recorded on the settle).- # The optional dependent proceeds on the null sentinel; only the- # required dependent over the same failed capture fails its entry.+ # settled, declared a json port, but its answer never parsed for+ # that port (a JSON capture failure). The optional dependent+ # proceeds on the null sentinel; only the required dependent over+ # the same failed capture fails its entry.self.host.outcomes["src"] = {"status": "done", "answer": "no json here"}self.store_machine({@@ -6092,6 +6487,13 @@ class FactoryExecutorTest(_ExecutorTestCase):result = await self.start()status = await self.settle(result)self.assertEqual(status["state"], "failed")+ # the failed json capture failed the source entry itself (defect C),+ # and its settled answer stays preserved for review+ src = self.state_report(status, "src")+ self.assertEqual([entry["status"] for entry in src["entries"]], ["error"])+ self.assertIn("no JSON object containing output 'data'", src["entries"][0]["error"])+ self.assertEqual(status["needs_verify"], ["src"])+ self.assertIn("no json here", src["entries"][0]["provisional_answer"])opt = self.state_report(status, "opt")self.assertEqual(opt["entries_used"], 1)self.assertEqual([entry["status"] for entry in opt["entries"]], ["done"])@@ -6100,7 +6502,7 @@ class FactoryExecutorTest(_ExecutorTestCase):self.assertEqual(req["entries_used"], 1)self.assertEqual([entry["status"] for entry in req["entries"]], ["error"])self.assertIn(- "input 'data': no JSON object containing output 'data' in the upstream answer",+ "input 'data' from state 'src' is unavailable (latest settle status 'error')",req["entries"][0]["error"],)self.assertEqual(self.host.spawn_calls("req"), [])@@ -6263,14 +6665,9 @@ class FactoryExecutorTest(_ExecutorTestCase):# The admission below is gated mid-spawn so a stale loop that wrongly# continued would re-admit the SAME pending instance (a duplicate# child) while the new loop's admission is still in flight.- self.host.outcomes["a"] = {"status": "error", "error": "boom"}- self.host.rate_limit_first["b"] = 1 # resume()'s admission defers once+ self.host.child_outcomes["child-1"] = {"status": "error", "error": "boom"}progress_gate = self.host.gate("factory.progress", 1)progress_entered = self.host.gate_entered("factory.progress", 1)- # rlm.run call #1 is a's admission at run(); #2 is b's first admission- # at resume() (rate limited, deferred nonblockingly); the gate holds- # call #3 -- the NEW loop's retry admission -- mid-spawn.- spawn_gate = self.host.gate("rlm.run", 3)self.store_factory({"nodes": [@@ -6288,26 +6685,35 @@ class FactoryExecutorTest(_ExecutorTestCase):old_task = run.taskself.assertIsNotNone(old_task)self.assertFalse(old_task.done())- # resume() bumps the generation, fires a->b, defers b's first- # admission (rate limited, nonblocking), and starts the new loop.+ # The rate limit hits a's re-admission: resume()'s admission defers+ # once, and the new loop's retry is the gated mid-spawn.+ self.host.rate_limit_first["a"] = 2+ # rlm.run call #1 is a's admission at run(); #2 is a's re-admission+ # at resume() (rate limited, deferred nonblockingly); the gate holds+ # call #3 -- the NEW loop's retry admission -- mid-spawn.+ spawn_gate = self.host.gate("rlm.run", 3)+ # resume() bumps the generation, re-admits a's failed entry (the+ # admission defers, rate limited, nonblocking), and starts the new+ # loop.resumed = await rlm_module.rlm.factory.resume(result["run_id"])self.assertEqual(resumed["state"], "running")self.assertEqual(resumed["started"], [])self.assertIsNot(run.task, old_task)# Releasing the milestone lets the old loop continue: with the# generation bump it must exit without admitting anything (the new- # loop owns b, suspended mid-admission at the spawn gate).+ # loop owns a's retry, suspended mid-admission at the spawn gate).progress_gate.set()await old_taskself.assertTrue(old_task.done())spawn_gate.set()final = await self.settle(result)- self.assertEqual(final["state"], "failed")+ self.assertEqual(final["state"], "done")self.assertEqual(self.node_status(final, "b")["status"], "done")- # b has exactly two admissions: the deferred one at resume() and- # the new loop's successful retry. A stale second loop would admit- # it once more while the gated spawn was in flight.- self.assertEqual(len(self.host.spawn_calls("b")), 2)+ # a has exactly three admissions: the failed first, the deferred+ # one at resume() and the new loop's successful retry. A stale+ # second loop would admit it once more while the gated spawn was+ # in flight.+ self.assertEqual(len(self.host.spawn_calls("a")), 3)self.assertEqual(self.all_events_of(result, "executor_error"), [])self.assertEqual(len([e for e in self.all_events_of(result, "state_entry") if e.get("node") == "b"]), 1diff --git a/scripts/review_factory_resumption_parity.py b/scripts/review_factory_resumption_parity.pynew file mode 100644index 0000000000..090dc8211e--- /dev/null+++ b/scripts/review_factory_resumption_parity.py@@ -0,0 +1,218 @@+#!/usr/bin/env python3+"""Hosted real-daemon factory recovery receipt; TS has no advertised factory lane.++The Rust kernel runs a real factory child, fails its declared JSON output,+resumes the same entry, and binds the second child's valid output. Historical+TS absence is scoped to its actual capability advertisement, not full parity.+"""+import argparse+import http.server+import json+import os+from pathlib import Path+import subprocess+import sys+import tempfile+import threading+import time++from review_kernel_env_parity import KernelClient as Client+from review_draft_disconnect_parity import digest, env_for, TS_ARCHIVE_SHA256+from review_goal_fixture_lifecycle import OwnedProcesses, TAG, finish+from review_restore_notice_parity import texts++MARKER = 'FACTORY_RECOVERY_RESULT='+CELL = '''import asyncio, json, time+factory_harness = rlm.get_harness_state()+factory_harness.create_factory('Recovery fixture', 'Actual factory output recovery', id='recovery-fixture', machine={+ 'states': [{'id': 'work', 'entry': True, 'max_entries': 1,+ 'subagent': {'prompt': 'FACTORY_RECOVERY_CHILD'},+ 'outputs': [{'name': 'data', 'type': 'json'}]},+ {'id': 'verified', 'max_entries': 1,+ 'subagent': {'prompt': 'FACTORY_RECOVERY_CHILD_VERIFY'}}],+ 'transitions': [{'from': 'work', 'to': 'verified',+ 'when': {'output': 'data', 'path': 'ok', 'op': 'eq', 'value': True}}]})+factory_started = await rlm.factory.run('recovery-fixture')+print('FACTORY_RECOVERY_RESULT=' + json.dumps(factory_started))'''+++class Provider(http.server.ThreadingHTTPServer):+ daemon_threads = True++ def __init__(self):+ super().__init__(('127.0.0.1', 0), Handler)+ self.requests = []+ self.parent_requests = []+ self.child_requests = []+++class Handler(http.server.BaseHTTPRequestHandler):+ def log_message(self, *_args):+ pass++ def do_POST(self):+ if self.path != '/v1/chat/completions':+ self.send_error(404)+ return+ body = json.loads(self.rfile.read(int(self.headers['Content-Length'])))+ self.server.requests.append(body)+ parent = any('FACTORY_PARENT_PROBE' in ''.join(texts(m.get('content')))+ for m in body.get('messages', []) if m.get('role') == 'user')+ child = not parent and any('FACTORY_RECOVERY_CHILD' in ''.join(texts(m.get('content')))+ for m in body.get('messages', []) if m.get('role') == 'user')+ if child:+ self.server.child_requests.append(body)+ delta = {'content': 'first attempt has no JSON' if len(self.server.child_requests) == 1+ else '```json\n{"data": {"ok": true}}\n```'}+ reason = 'stop'+ else:+ self.server.parent_requests.append(body)+ if len(self.server.parent_requests) == 1:+ offered = [t['function']['name'] for t in body.get('tools', [])]+ assert 'python_repl' in offered, offered+ delta = {'tool_calls': [{'index': 0, 'id': 'factory_probe', 'type': 'function',+ 'function': {'name': 'python_repl', 'arguments': json.dumps({'code': CELL})}}]}+ reason = 'tool_calls'+ else:+ delta, reason = {'content': 'Factory probe complete.'}, 'stop'+ def chunk(delta, reason=None):+ return {'id': 'factory-fixture', 'object': 'chat.completion.chunk', 'created': 1,+ 'model': 'factory-fixture', 'choices': [{'index': 0, 'delta': delta, 'finish_reason': reason}]}+ wire = ''.join('data: ' + json.dumps(row) + '\n\n' for row in (+ chunk({'role': 'assistant', **delta}), chunk({}, reason))) + 'data: [DONE]\n\n'+ self.send_response(200)+ self.send_header('Content-Type', 'text/event-stream')+ self.send_header('Content-Length', str(len(wire.encode())))+ self.end_headers()+ self.wfile.write(wire.encode())+++def extract(requests):+ assert len(requests) == 2, 'unexpected parent provider sequence'+ results = [m for m in requests[-1]['messages'] if m.get('role') == 'tool'+ and m.get('tool_call_id') == 'factory_probe']+ assert len(results) == 1, 'missing/duplicate current tool result'+ values = [json.loads(line[len(MARKER):]) for content in texts(results[0].get('content'))+ for line in content.splitlines() if line.startswith(MARKER)]+ assert len(values) == 1, 'missing/duplicate actual factory receipt'+ return values[0]+++def count(row, snake, camel):+ assert not (snake in row and camel in row), 'ambiguous duplicate wire keys'+ return row[snake] if snake in row else row[camel]+++def validate(probe):+ assert probe['paused']['state'] == 'paused', probe+ assert probe['done']['state'] == 'done', probe+ nodes = probe['done']['nodes']+ assert sorted(n['id'] for n in nodes) == ['verified', 'work'], nodes+ verified = next(n for n in nodes if n['id'] == 'verified')+ assert verified['status'] == 'done' and count(verified, 'entries_used', 'entriesUsed') == 1 and verified['attempts'] == 1, verified+ node = next(n for n in nodes if n['id'] == 'work')+ assert count(node, 'entries_used', 'entriesUsed') == 1 and node['attempts'] == 2, node+ assert node['status'] == 'done', node+++def observe(binary, kind, root, observed):+ env = env_for(root, kind)+ env[TAG] = str(root)+ server = Provider()+ threading.Thread(target=server.serve_forever, daemon=True).start()+ (root / 'agent/settings.json').write_text(json.dumps({'factory': {'enabled': True},+ 'prewarmPythonKernel': False, 'defaultProvider': 'parity-local', 'defaultModel': 'parity-1',+ 'onboardingCompleted': True, 'onboardingShown': True, 'telemetry': {'noticeShown': True}}))+ (root / 'agent/models.json').write_text(json.dumps({'providers': {'parity-local': {+ 'api': 'openai-completions', 'apiKey': 'fixture-only',+ 'baseUrl': f'http://127.0.0.1:{server.server_port}/v1', 'models': [{+ 'id': 'parity-1', 'name': 'Factory fixture', 'reasoning': False,+ 'contextWindow': 128000, 'maxTokens': 4096}]}}}))+ path = root / 'd.sock'+ scope = OwnedProcesses(binary, root, path)+ daemon = control = None+ log = open(root / 'daemon.log', 'wb')+ try:+ command = [str(binary), '--mode', 'daemon', '--daemon-socket', str(path)]+ daemon = subprocess.Popen(command, cwd=root, env=env, stdout=log,+ stderr=subprocess.STDOUT, start_new_session=True)+ scope.register(daemon, command)+ control = Client(path, daemon)+ observed['hello'] = control.hello+ caps = control.hello.get('serverCapabilities', [])+ if kind == 'ts':+ assert 'factory_activity' not in caps, 'TS unexpectedly advertises factory'+ observed['factory_lane_advertised'] = False+ else:+ created = control.call('create', config={'cwd': str(root), 'agentDir': str(root / 'agent'),+ 'provider': 'parity-local', 'model': 'parity-1', 'telemetryDisabled': True,+ 'noExtensions': True, 'noContextFiles': True})+ active = created.get('activeSessionId') or created.get('id')+ assert active, created+ control.call('attach', activeSessionId=active)+ control.call('prompt', activeSessionId=active, message='Run FACTORY_PARENT_PROBE')+ control.call('wait_for_idle', timeout_seconds=120, activeSessionId=active)+ started = extract(server.parent_requests[:2])+ observed['started'] = started+ run_id = started['run_id']+ def settled():+ snapshots = []+ deadline = time.monotonic() + 60+ while time.monotonic() < deadline:+ status = control.call('factory_activity', activeSessionId=active,+ action='status', runId=run_id)+ snapshots.append(status)+ if status['state'] != 'running':+ return status+ threading.Event().wait(0.025)+ observed.setdefault('timed_out_snapshots', []).extend(snapshots)+ raise AssertionError('factory did not leave running after parent turn completed')+ paused = settled()+ assert paused['state'] == 'paused', paused+ observed['resume_response'] = control.call('factory_activity', activeSessionId=active,+ action='resume', runId=run_id)+ observed['probe'] = {'paused': paused, 'done': settled()}+ validate(observed['probe'])+ assert len(server.child_requests) == 3, 'not two producer attempts plus one guarded verifier'+ observed['behavior_verified'] = True+ finally:+ observed['provider_requests'] = server.requests+ observed['wire'] = list(control.wire) if control else []+ finish(scope, control, daemon, observed)+ log.close()+ observed['daemon_log'] = (root / 'daemon.log').read_text(errors='replace')[-10000:]+ server.shutdown()+ server.server_close()+++def main():+ parser = argparse.ArgumentParser(description=__doc__)+ parser.add_argument('--receipt', type=Path, required=True)+ args = parser.parse_args()+ result = {'verified': False, 'scope': __doc__, 'runs': {}, 'not_verified': [+ 'TS factory behavior (no advertised historical counterpart)', 'TUI',+ 'worker-death respawn (separate executor regression)', 'pause/stop races (separate regressions)']}+ try:+ assert sys.platform == 'linux', 'hosted Linux only'+ assert digest(Path(os.environ['PA_TS_ARCHIVE'])) == TS_ARCHIVE_SHA256+ result['ts_archive_sha256'] = TS_ARCHIVE_SHA256+ with tempfile.TemporaryDirectory(prefix='pa-factory-', dir='/tmp') as folder:+ for kind in ('ts', 'rust'):+ binary = Path(os.environ[f'PA_{kind.upper()}_BINARY']).resolve()+ observed = result['runs'][kind] = {'binary_sha256': digest(binary)}+ try:+ observe(binary, kind, Path(folder) / kind, observed)+ except Exception as error:+ observed['error'] = str(error)+ assert all(r.get('behavior_verified') and r.get('cleanup_verified') and 'error' not in r+ for r in result['runs'].values()), 'actual factory observation failed'+ result['verified'] = True+ except Exception as error:+ result['error'] = str(error)+ args.receipt.parent.mkdir(parents=True, exist_ok=True)+ args.receipt.write_text(json.dumps(result, ensure_ascii=False, indent=2))+ return 0 if result['verified'] else 1+++if __name__ == '__main__':+ sys.exit(main())diff --git a/scripts/test_review_factory_resumption_parity.py b/scripts/test_review_factory_resumption_parity.pynew file mode 100644index 0000000000..74543fee91--- /dev/null+++ b/scripts/test_review_factory_resumption_parity.py@@ -0,0 +1,44 @@+import copy+import json+import unittest+from review_factory_resumption_parity import MARKER, extract, validate+++class FactoryReceiptGuards(unittest.TestCase):+ def probe(self):+ return {'paused': {'state': 'paused'}, 'done': {'state': 'done', 'nodes': [+ {'id': 'work', 'status': 'done', 'entries_used': 1, 'attempts': 2},+ {'id': 'verified', 'status': 'done', 'entries_used': 1, 'attempts': 1}]}}++ def test_rejects_missing_resume_extra_entry_or_extra_attempt(self):+ validate(self.probe())+ for key, value in [('entries_used', 2), ('attempts', 1), ('attempts', 3), ('status', 'error')]:+ probe = self.probe()+ probe['done']['nodes'][0][key] = value+ with self.assertRaises(AssertionError):+ validate(probe)++ def test_accepts_explicit_wire_camel_case_without_ignoring_duplicates(self):+ probe = self.probe()+ for row in probe['done']['nodes']:+ row['entriesUsed'] = row.pop('entries_used')+ validate(probe)+ probe['done']['nodes'][0]['entries_used'] = 1+ with self.assertRaises(AssertionError):+ validate(probe)++ def test_rejects_missing_or_historical_tool_receipt(self):+ probe = self.probe()+ tool = {'role': 'tool', 'tool_call_id': 'factory_probe',+ 'content': MARKER + json.dumps(probe)}+ rows = [{}, {'messages': [tool]}]+ self.assertEqual(extract(rows), probe)+ for changed in [[{'messages': [tool]}, {'messages': []}],+ [{}, {'messages': [tool, copy.deepcopy(tool)]}],+ [{}, {'messages': [{**tool, 'tool_call_id': 'historical'}]}]]:+ with self.assertRaises(AssertionError):+ extract(changed)+++if __name__ == '__main__':+ unittest.main()