Skip to content

A self-improving RLM agent for coding workflows and long-running autonomous tasks.

Rust

factory: authoring notes and resident-retries validation (#3629)

Sebastian Müllerf2425dc1b537
diff --git a/.changes/factory-authoring-notes.md b/.changes/factory-authoring-notes.md
new file mode 100644
index 0000000000..9f9c86db48
--- /dev/null
+++ b/.changes/factory-authoring-notes.md
@@ -0,0 +1 @@
+- Factory resident states now reject `retries`, and the factory help plus the pr-manager seed's fixing prompt spell out the recovery posture a re-run needs when an earlier attempt already changed the working tree.
diff --git a/prime-agent-runtime/src/rlm/factory.py b/prime-agent-runtime/src/rlm/factory.py
index 743e9efbb9..673a784176 100644
--- a/prime-agent-runtime/src/rlm/factory.py
+++ b/prime-agent-runtime/src/rlm/factory.py
@@ -317,6 +317,8 @@ def _validate_state_fields(
if "retries" in state and not (_is_int(state.get("retries")) and 0 <= state.get("retries") <= MAX_RETRIES):
errors.append(f"{noun} {ref} retries must be an integer between 0 and {MAX_RETRIES}")
+ elif is_resident and state.get("retries", NODE_RETRIES_DEFAULT) > 0:
+ errors.append(f"resident {noun} {ref} cannot declare retries")
if "failure_policy" in state and state.get("failure_policy") not in FAILURE_POLICIES:
errors.append(
@@ -491,7 +493,7 @@ def validate_factory_machine(machine: Any) -> list[str]:
the machine is valid. Rules: states are 1..1024 with unique slug ids and
at least one entry state; every state requires a subagent and entry
states declare no inputs; resident states declare no outputs, foreach,
- or outgoing transitions; wait blocks are rejected (the watch host
+ retries, or outgoing transitions; wait blocks are rejected (the watch host
handlers arrive with the communication series); transitions reference
existing states (self-loops are legal re-entry) and may carry one guard
over the from-state's latest settle output; a transition with a LIST of
@@ -4683,9 +4685,10 @@ rlm.harness.create_factory(
{
"id": "fixing",
"subagent": {"prompt": (
- "Address the review findings in the verdict below. Make the "
- "smallest targeted fixes, run the relevant tests, and return a "
- "fenced json block of the form "
+ "Address the review findings in the verdict below. First "
+ "reconcile any changes an earlier attempt left in the working "
+ "tree. Make the smallest targeted fixes, run the relevant "
+ "tests, and return a fenced json block of the form "
'{"fix_report": {"fixed": ["finding that was addressed"], '
'"skipped": ["finding left alone and why"]}}. '
"Output only the json block.\n\n{verdict}")},
@@ -4785,9 +4788,9 @@ the cap and fails to bind.
child rendered with its item bound as that input; an empty list settles
the entry with no children.
- **Residents**: `"lifecycle": "resident"` states declare no outputs, no
- foreach, and no outgoing transitions, nothing reads from them, and their
- instance stays alive under the parent session after the run completes
- (stop the run to retire it).
+ foreach, no retries, and no outgoing transitions, nothing reads from
+ them, and their instance stays alive under the parent session after the
+ run completes (stop the run to retire it).
- **Bounds and policies**: `run.max_parallel` (1..64, default 8) is the
run's global budget of simultaneously running instances — not a
per-node limit. `run.max_children` (default 10,000, capped at
@@ -4806,6 +4809,15 @@ the cap and fails to bind.
errored; the run finishes and reports failed if any state errored), or
`escalate` (the default: pause the run; resuming is the operator's
decision).
+- **Retry re-renders the same prompt into a changed world**: a retry runs
+ the same prompt after an earlier attempt already changed the external
+ state it works on, so a prompt that mutates external state — a fixing
+ state's worktree, a tracker, a remote API — states its recovery posture:
+ reconcile (start from what the earlier attempt left with `git status`
+ and `git diff`, keep the changes that already serve the task, revert
+ half-finished edits) or reset to a clean baseline, never fail on the
+ old state or duplicate finished work. Retries with no recovery note
+ still validate: validation cannot judge prose.
- **Dead configurations fail loudly, never wedge**: a pending entry whose
input source never settled, a `max_parallel` cap held entirely by
never-settling residents with work queued, or nothing in flight and
diff --git a/prime-agent-runtime/src/rlm/machines/pr-manager/MACHINE.md b/prime-agent-runtime/src/rlm/machines/pr-manager/MACHINE.md
index 25f9e672b6..bca77468b5 100644
--- a/prime-agent-runtime/src/rlm/machines/pr-manager/MACHINE.md
+++ b/prime-agent-runtime/src/rlm/machines/pr-manager/MACHINE.md
@@ -85,7 +85,7 @@ state's `max_entries`; the run pauses (escalates) instead of spinning.
{
"id": "fixing",
"subagent": {
- "prompt": "Address the review findings in the verdict below. Make the smallest targeted fixes in the working tree, run the relevant tests, and reply with a fenced json block of the form {\\\"fix_report\\\": {\\\"fixed\\\": [\\\"finding that was addressed\\\"], \\\"skipped\\\": [\\\"finding left alone and why\\\"]}}. Output only the json block.\\n\\n{verdict}",
+ "prompt": "Address the review findings in the verdict below. This state can run again (a retry or a later review round) over a working tree an earlier attempt already changed: start from `git status` and `git diff`, keep the changes that already address a finding, and revert half-finished edits before making new ones. Make the smallest targeted fixes in the working tree, run the relevant tests, and reply with a fenced json block of the form {\\\"fix_report\\\": {\\\"fixed\\\": [\\\"finding that was addressed\\\"], \\\"skipped\\\": [\\\"finding left alone and why\\\"]}}. Output only the json block.\\n\\n{verdict}",
"name": "pr-fixing"
},
"inputs": [
diff --git a/prime-agent-runtime/test/test_factory.py b/prime-agent-runtime/test/test_factory.py
index d4eec47db7..54a6e347ff 100644
--- a/prime-agent-runtime/test/test_factory.py
+++ b/prime-agent-runtime/test/test_factory.py
@@ -644,6 +644,14 @@ class ValidateFactorySpecTest(unittest.TestCase):
errors = validate_factory_spec(uses_foreach)
self.assertEqual(errors, ["resident node watcher cannot use foreach"])
+ declares_retries = {"nodes": [node("watcher", lifecycle="resident", retries=2)]}
+ errors = validate_factory_spec(declares_retries)
+ self.assertEqual(errors, ["resident node watcher cannot declare retries"])
+
+ # Retries 0 is the canonical default, so nothing is declared.
+ zero_retries = {"nodes": [node("watcher", lifecycle="resident", retries=0)]}
+ self.assertEqual(validate_factory_spec(zero_retries), [])
+
# Empty outputs list on a resident node is fine: nothing is declared.
empty_outputs = {"nodes": [node("watcher", lifecycle="resident", outputs=[])]}
self.assertEqual(validate_factory_spec(empty_outputs), [])
@@ -1400,6 +1408,14 @@ class ValidateFactoryMachineTest(unittest.TestCase):
"transitions": [{"from": "src", "to": "watcher"}],
}
self.assertEqual(validate_factory_machine(uses_foreach), ["resident state watcher cannot use foreach"])
+ declares_retries = {
+ "states": [state("watcher", entry=True, lifecycle="resident", retries=1)],
+ "transitions": [],
+ }
+ self.assertEqual(
+ validate_factory_machine(declares_retries),
+ ["resident state watcher cannot declare retries"],
+ )
leaves_resident = {
"states": [
{"id": "entry", "entry": True, "subagent": "w"},
diff --git a/scripts/review_factory_validation_parity.py b/scripts/review_factory_validation_parity.py
new file mode 100644
index 0000000000..b123a05f61
--- /dev/null
+++ b/scripts/review_factory_validation_parity.py
@@ -0,0 +1,186 @@
+#!/usr/bin/env python3
+"""Hosted actual factory harness validator API receipt.
+
+Rust executes real harness writes through its kernel. Historical TS absence
+is scoped to its actual capability advertisement, not full behavior parity.
+"""
+import argparse
+import http.server
+import json
+import os
+from pathlib import Path
+import subprocess
+import sys
+import tempfile
+import threading
+
+from review_kernel_env_parity import KernelClient as Client
+from review_draft_disconnect_parity import digest, env_for, TS_ARCHIVE_SHA256
+from review_goal_fixture_lifecycle import OwnedProcesses, TAG, finish
+from review_restore_notice_parity import texts
+
+MARKER = 'FACTORY_VALIDATION_RESULT='
+CELL = '''import json
+fixture_harness = rlm.get_harness_state()
+fixture_results = []
+for label, retries in [('positive', 1), ('zero', 0), ('malformed', 'bad')]:
+ try:
+ value = fixture_harness.create_factory('Validation fixture', 'Production harness write',
+ id='fixture-' + label, dag={'nodes': [{'id': 'watcher', 'lifecycle': 'resident',
+ 'subagent': {'prompt': 'Watch fixture'}, 'retries': retries}]})
+ fixture_results.append({'case': label, 'stored': fixture_harness.get('factory', 'fixture-' + label) is not None})
+ except ValueError as error:
+ fixture_results.append({'case': label, 'error': str(error),
+ 'stored': fixture_harness.get('factory', 'fixture-' + label) is not None})
+print('FACTORY_VALIDATION_RESULT=' + json.dumps(fixture_results))'''
+
+
+class Provider(http.server.ThreadingHTTPServer):
+ daemon_threads = True
+
+ def __init__(self):
+ super().__init__(('127.0.0.1', 0), Handler)
+ self.requests = []
+ self.parent_requests = []
+ self.errors = []
+
+
+class Handler(http.server.BaseHTTPRequestHandler):
+ def log_message(self, *_args):
+ pass
+
+ def do_POST(self):
+ if self.path != '/v1/chat/completions':
+ self.send_error(404)
+ return
+ body = json.loads(self.rfile.read(int(self.headers['Content-Length'])))
+ self.server.requests.append(body)
+ parent = any('FACTORY_VALIDATION_PARENT' in ''.join(texts(m.get('content')))
+ for m in body.get('messages', []) if m.get('role') == 'user')
+ if not parent:
+ self.server.errors.append('unexpected child or unrelated provider request')
+ self.send_error(422)
+ return
+ self.server.parent_requests.append(body)
+ if len(self.server.parent_requests) == 1:
+ offered = [t['function']['name'] for t in body.get('tools', [])]
+ repl = next((name for name in ('python_repl', 'ipython') if name in offered), None)
+ assert repl is not None, offered
+ delta = {'tool_calls': [{'index': 0, 'id': 'factory_probe', 'type': 'function',
+ 'function': {'name': repl, 'arguments': json.dumps({'code': CELL})}}]}
+ reason = 'tool_calls'
+ else:
+ delta, reason = {'content': 'Factory validation probe complete.'}, 'stop'
+ def chunk(delta, reason=None):
+ return {'id': 'factory-fixture', 'object': 'chat.completion.chunk', 'created': 1,
+ 'model': 'factory-fixture', 'choices': [{'index': 0, 'delta': delta, 'finish_reason': reason}]}
+ wire = ''.join('data: ' + json.dumps(row) + '\n\n' for row in (
+ chunk({'role': 'assistant', **delta}), chunk({}, reason))) + 'data: [DONE]\n\n'
+ self.send_response(200)
+ self.send_header('Content-Type', 'text/event-stream')
+ self.send_header('Content-Length', str(len(wire.encode())))
+ self.end_headers()
+ self.wfile.write(wire.encode())
+
+
+def extract(requests):
+ assert len(requests) == 2, 'unexpected parent provider sequence'
+ results = [m for m in requests[-1]['messages'] if m.get('role') == 'tool'
+ and m.get('tool_call_id') == 'factory_probe']
+ assert len(results) == 1, 'missing/duplicate current tool result'
+ values = [json.loads(line[len(MARKER):]) for content in texts(results[0].get('content'))
+ for line in content.splitlines() if line.startswith(MARKER)]
+ assert len(values) == 1, 'missing/duplicate actual factory receipt'
+ return values[0]
+
+
+def validate(probe):
+ assert len(probe) == 3, probe
+ positive, zero, malformed = probe
+ assert positive['case'] == 'positive' and positive['stored'] is False, positive
+ assert 'resident node watcher cannot declare retries' in positive['error'], positive
+ assert zero == {'case': 'zero', 'stored': True}, zero
+ assert malformed['case'] == 'malformed' and malformed['stored'] is False, malformed
+ assert 'retries must be an integer between 0 and 10' in malformed['error'], malformed
+
+
+def observe(binary, kind, root, observed):
+ env = env_for(root, kind)
+ env[TAG] = str(root)
+ server = Provider()
+ threading.Thread(target=server.serve_forever, daemon=True).start()
+ (root / 'agent/settings.json').write_text(json.dumps({'factory': {'enabled': True},
+ 'prewarmPythonKernel': False, 'defaultProvider': 'parity-local', 'defaultModel': 'parity-1'}))
+ (root / 'agent/models.json').write_text(json.dumps({'providers': {'parity-local': {
+ 'api': 'openai-completions', 'apiKey': 'fixture-only',
+ 'baseUrl': f'http://127.0.0.1:{server.server_port}/v1', 'models': [{
+ 'id': 'parity-1', 'name': 'Factory fixture', 'reasoning': False,
+ 'contextWindow': 128000, 'maxTokens': 4096}]}}}))
+ path = root / 'd.sock'
+ scope = OwnedProcesses(binary, root, path)
+ daemon = control = None
+ log = open(root / 'daemon.log', 'wb')
+ try:
+ command = [str(binary), '--mode', 'daemon', '--daemon-socket', str(path)]
+ daemon = subprocess.Popen(command, cwd=root, env=env, stdout=log,
+ stderr=subprocess.STDOUT, start_new_session=True)
+ scope.register(daemon, command)
+ control = Client(path, daemon)
+ observed['hello'] = control.hello
+ caps = control.hello.get('serverCapabilities', [])
+ if kind == 'ts':
+ assert 'factory_activity' not in caps, 'TS unexpectedly advertises factory'
+ observed['factory_lane_advertised'] = False
+ else:
+ created = control.call('create', config={'cwd': str(root), 'agentDir': str(root / 'agent'),
+ 'provider': 'parity-local', 'model': 'parity-1', 'telemetryDisabled': True})
+ active = created.get('activeSessionId') or created.get('id')
+ assert active, created
+ control.call('attach', activeSessionId=active)
+ control.call('prompt', activeSessionId=active, message='Run FACTORY_VALIDATION_PARENT')
+ control.call('wait_for_idle', timeout_seconds=120, activeSessionId=active)
+ observed['probe'] = extract(server.parent_requests)
+ validate(observed['probe'])
+ assert not server.errors, server.errors
+ observed['behavior_verified'] = True
+ finally:
+ observed['provider_requests'] = server.requests
+ observed['provider_errors'] = server.errors
+ observed['wire'] = list(control.wire) if control else []
+ finish(scope, control, daemon, observed)
+ log.close()
+ observed['daemon_log'] = (root / 'daemon.log').read_text(errors='replace')[-10000:]
+ server.shutdown()
+ server.server_close()
+
+
+def main():
+ parser = argparse.ArgumentParser(description=__doc__)
+ parser.add_argument('--receipt', type=Path, required=True)
+ args = parser.parse_args()
+ result = {'verified': False, 'scope': __doc__, 'runs': {}, 'not_verified': [
+ 'TS factory behavior (no advertised historical counterpart)', 'TUI', 'factory execution and recovery', 'TS validator equality']}
+ try:
+ assert sys.platform == 'linux', 'hosted Linux only'
+ assert digest(Path(os.environ['PA_TS_ARCHIVE'])) == TS_ARCHIVE_SHA256
+ result['ts_archive_sha256'] = TS_ARCHIVE_SHA256
+ with tempfile.TemporaryDirectory(prefix='pa-factory-', dir='/tmp') as folder:
+ for kind in ('ts', 'rust'):
+ binary = Path(os.environ[f'PA_{kind.upper()}_BINARY']).resolve()
+ observed = result['runs'][kind] = {'binary_sha256': digest(binary)}
+ try:
+ observe(binary, kind, Path(folder) / kind, observed)
+ except Exception as error:
+ observed['error'] = str(error)
+ assert all(r.get('behavior_verified') and r.get('cleanup_verified') and 'error' not in r
+ for r in result['runs'].values()), 'actual factory observation failed'
+ result['verified'] = True
+ except Exception as error:
+ result['error'] = str(error)
+ args.receipt.parent.mkdir(parents=True, exist_ok=True)
+ args.receipt.write_text(json.dumps(result, ensure_ascii=False, indent=2))
+ return 0 if result['verified'] else 1
+
+
+if __name__ == '__main__':
+ sys.exit(main())
diff --git a/scripts/test_review_factory_validation_parity.py b/scripts/test_review_factory_validation_parity.py
new file mode 100644
index 0000000000..da18476e98
--- /dev/null
+++ b/scripts/test_review_factory_validation_parity.py
@@ -0,0 +1,52 @@
+import io
+import json
+from types import SimpleNamespace
+import unittest
+from review_factory_validation_parity import validate, Handler
+
+
+class FactoryValidationGuards(unittest.TestCase):
+ def value(self):
+ return [
+ {'case': 'positive', 'stored': False, 'error': 'resident node watcher cannot declare retries'},
+ {'case': 'zero', 'stored': True},
+ {'case': 'malformed', 'stored': False, 'error': 'node watcher retries must be an integer between 0 and 10'}]
+
+ def test_rejects_acceptance_or_storage_of_invalid_specs(self):
+ validate(self.value())
+ for index in (0, 2):
+ value = self.value()
+ value[index]['stored'] = True
+ with self.assertRaises(AssertionError):
+ validate(value)
+ value = self.value()
+ value[1]['stored'] = False
+ with self.assertRaises(AssertionError):
+ validate(value)
+
+ def test_unexpected_child_request_is_rejected_before_a_model_response(self):
+ body = json.dumps({'messages': [{'role': 'user', 'content': 'unexpected child'}]}).encode()
+ handler = Handler.__new__(Handler)
+ handler.path = '/v1/chat/completions'
+ handler.headers = {'Content-Length': str(len(body))}
+ handler.rfile = io.BytesIO(body)
+ handler.server = SimpleNamespace(requests=[], parent_requests=[], errors=[])
+ statuses = []
+ handler.send_error = statuses.append
+ handler.do_POST()
+ self.assertEqual(statuses, [422])
+ self.assertEqual(handler.server.parent_requests, [])
+ self.assertEqual(handler.server.errors, ['unexpected child or unrelated provider request'])
+
+ def test_requires_correct_diagnostic_and_all_cases(self):
+ for index in (0, 2):
+ value = self.value()
+ value[index]['error'] = 'unrelated failure'
+ with self.assertRaises(AssertionError):
+ validate(value)
+ with self.assertRaises(AssertionError):
+ validate(self.value()[:-1])
+
+
+if __name__ == '__main__':
+ unittest.main()