A self-improving RLM agent for coding workflows and long-running autonomous tasks.
Rust
factory: authoring notes and resident-retries validation (#3629)
Sebastian Müllerf2425dc1b537
diff --git a/.changes/factory-authoring-notes.md b/.changes/factory-authoring-notes.mdnew file mode 100644index 0000000000..9f9c86db48--- /dev/null+++ b/.changes/factory-authoring-notes.md@@ -0,0 +1 @@+- Factory resident states now reject `retries`, and the factory help plus the pr-manager seed's fixing prompt spell out the recovery posture a re-run needs when an earlier attempt already changed the working tree.diff --git a/prime-agent-runtime/src/rlm/factory.py b/prime-agent-runtime/src/rlm/factory.pyindex 743e9efbb9..673a784176 100644--- a/prime-agent-runtime/src/rlm/factory.py+++ b/prime-agent-runtime/src/rlm/factory.py@@ -317,6 +317,8 @@ def _validate_state_fields(if "retries" in state and not (_is_int(state.get("retries")) and 0 <= state.get("retries") <= MAX_RETRIES):errors.append(f"{noun} {ref} retries must be an integer between 0 and {MAX_RETRIES}")+ elif is_resident and state.get("retries", NODE_RETRIES_DEFAULT) > 0:+ errors.append(f"resident {noun} {ref} cannot declare retries")if "failure_policy" in state and state.get("failure_policy") not in FAILURE_POLICIES:errors.append(@@ -491,7 +493,7 @@ def validate_factory_machine(machine: Any) -> list[str]:the machine is valid. Rules: states are 1..1024 with unique slug ids andat least one entry state; every state requires a subagent and entrystates declare no inputs; resident states declare no outputs, foreach,- or outgoing transitions; wait blocks are rejected (the watch host+ retries, or outgoing transitions; wait blocks are rejected (the watch hosthandlers arrive with the communication series); transitions referenceexisting states (self-loops are legal re-entry) and may carry one guardover the from-state's latest settle output; a transition with a LIST of@@ -4683,9 +4685,10 @@ rlm.harness.create_factory({"id": "fixing","subagent": {"prompt": (- "Address the review findings in the verdict below. Make the "- "smallest targeted fixes, run the relevant tests, and return a "- "fenced json block of the form "+ "Address the review findings in the verdict below. First "+ "reconcile any changes an earlier attempt left in the working "+ "tree. Make the smallest targeted fixes, run the relevant "+ "tests, and return a fenced json block of the form "'{"fix_report": {"fixed": ["finding that was addressed"], ''"skipped": ["finding left alone and why"]}}. '"Output only the json block.\n\n{verdict}")},@@ -4785,9 +4788,9 @@ the cap and fails to bind.child rendered with its item bound as that input; an empty list settlesthe entry with no children.- **Residents**: `"lifecycle": "resident"` states declare no outputs, no- foreach, and no outgoing transitions, nothing reads from them, and their- instance stays alive under the parent session after the run completes- (stop the run to retire it).+ foreach, no retries, and no outgoing transitions, nothing reads from+ them, and their instance stays alive under the parent session after the+ run completes (stop the run to retire it).- **Bounds and policies**: `run.max_parallel` (1..64, default 8) is therun's global budget of simultaneously running instances — not aper-node limit. `run.max_children` (default 10,000, capped at@@ -4806,6 +4809,15 @@ the cap and fails to bind.errored; the run finishes and reports failed if any state errored), or`escalate` (the default: pause the run; resuming is the operator'sdecision).+- **Retry re-renders the same prompt into a changed world**: a retry runs+ the same prompt after an earlier attempt already changed the external+ state it works on, so a prompt that mutates external state — a fixing+ state's worktree, a tracker, a remote API — states its recovery posture:+ reconcile (start from what the earlier attempt left with `git status`+ and `git diff`, keep the changes that already serve the task, revert+ half-finished edits) or reset to a clean baseline, never fail on the+ old state or duplicate finished work. Retries with no recovery note+ still validate: validation cannot judge prose.- **Dead configurations fail loudly, never wedge**: a pending entry whoseinput source never settled, a `max_parallel` cap held entirely bynever-settling residents with work queued, or nothing in flight anddiff --git a/prime-agent-runtime/src/rlm/machines/pr-manager/MACHINE.md b/prime-agent-runtime/src/rlm/machines/pr-manager/MACHINE.mdindex 25f9e672b6..bca77468b5 100644--- a/prime-agent-runtime/src/rlm/machines/pr-manager/MACHINE.md+++ b/prime-agent-runtime/src/rlm/machines/pr-manager/MACHINE.md@@ -85,7 +85,7 @@ state's `max_entries`; the run pauses (escalates) instead of spinning.{"id": "fixing","subagent": {- "prompt": "Address the review findings in the verdict below. Make the smallest targeted fixes in the working tree, run the relevant tests, and reply with a fenced json block of the form {\\\"fix_report\\\": {\\\"fixed\\\": [\\\"finding that was addressed\\\"], \\\"skipped\\\": [\\\"finding left alone and why\\\"]}}. Output only the json block.\\n\\n{verdict}",+ "prompt": "Address the review findings in the verdict below. This state can run again (a retry or a later review round) over a working tree an earlier attempt already changed: start from `git status` and `git diff`, keep the changes that already address a finding, and revert half-finished edits before making new ones. Make the smallest targeted fixes in the working tree, run the relevant tests, and reply with a fenced json block of the form {\\\"fix_report\\\": {\\\"fixed\\\": [\\\"finding that was addressed\\\"], \\\"skipped\\\": [\\\"finding left alone and why\\\"]}}. Output only the json block.\\n\\n{verdict}","name": "pr-fixing"},"inputs": [diff --git a/prime-agent-runtime/test/test_factory.py b/prime-agent-runtime/test/test_factory.pyindex d4eec47db7..54a6e347ff 100644--- a/prime-agent-runtime/test/test_factory.py+++ b/prime-agent-runtime/test/test_factory.py@@ -644,6 +644,14 @@ class ValidateFactorySpecTest(unittest.TestCase):errors = validate_factory_spec(uses_foreach)self.assertEqual(errors, ["resident node watcher cannot use foreach"])+ declares_retries = {"nodes": [node("watcher", lifecycle="resident", retries=2)]}+ errors = validate_factory_spec(declares_retries)+ self.assertEqual(errors, ["resident node watcher cannot declare retries"])++ # Retries 0 is the canonical default, so nothing is declared.+ zero_retries = {"nodes": [node("watcher", lifecycle="resident", retries=0)]}+ self.assertEqual(validate_factory_spec(zero_retries), [])+# Empty outputs list on a resident node is fine: nothing is declared.empty_outputs = {"nodes": [node("watcher", lifecycle="resident", outputs=[])]}self.assertEqual(validate_factory_spec(empty_outputs), [])@@ -1400,6 +1408,14 @@ class ValidateFactoryMachineTest(unittest.TestCase):"transitions": [{"from": "src", "to": "watcher"}],}self.assertEqual(validate_factory_machine(uses_foreach), ["resident state watcher cannot use foreach"])+ declares_retries = {+ "states": [state("watcher", entry=True, lifecycle="resident", retries=1)],+ "transitions": [],+ }+ self.assertEqual(+ validate_factory_machine(declares_retries),+ ["resident state watcher cannot declare retries"],+ )leaves_resident = {"states": [{"id": "entry", "entry": True, "subagent": "w"},diff --git a/scripts/review_factory_validation_parity.py b/scripts/review_factory_validation_parity.pynew file mode 100644index 0000000000..b123a05f61--- /dev/null+++ b/scripts/review_factory_validation_parity.py@@ -0,0 +1,186 @@+#!/usr/bin/env python3+"""Hosted actual factory harness validator API receipt.++Rust executes real harness writes through its kernel. Historical TS absence+is scoped to its actual capability advertisement, not full behavior parity.+"""+import argparse+import http.server+import json+import os+from pathlib import Path+import subprocess+import sys+import tempfile+import threading++from review_kernel_env_parity import KernelClient as Client+from review_draft_disconnect_parity import digest, env_for, TS_ARCHIVE_SHA256+from review_goal_fixture_lifecycle import OwnedProcesses, TAG, finish+from review_restore_notice_parity import texts++MARKER = 'FACTORY_VALIDATION_RESULT='+CELL = '''import json+fixture_harness = rlm.get_harness_state()+fixture_results = []+for label, retries in [('positive', 1), ('zero', 0), ('malformed', 'bad')]:+ try:+ value = fixture_harness.create_factory('Validation fixture', 'Production harness write',+ id='fixture-' + label, dag={'nodes': [{'id': 'watcher', 'lifecycle': 'resident',+ 'subagent': {'prompt': 'Watch fixture'}, 'retries': retries}]})+ fixture_results.append({'case': label, 'stored': fixture_harness.get('factory', 'fixture-' + label) is not None})+ except ValueError as error:+ fixture_results.append({'case': label, 'error': str(error),+ 'stored': fixture_harness.get('factory', 'fixture-' + label) is not None})+print('FACTORY_VALIDATION_RESULT=' + json.dumps(fixture_results))'''+++class Provider(http.server.ThreadingHTTPServer):+ daemon_threads = True++ def __init__(self):+ super().__init__(('127.0.0.1', 0), Handler)+ self.requests = []+ self.parent_requests = []+ self.errors = []+++class Handler(http.server.BaseHTTPRequestHandler):+ def log_message(self, *_args):+ pass++ def do_POST(self):+ if self.path != '/v1/chat/completions':+ self.send_error(404)+ return+ body = json.loads(self.rfile.read(int(self.headers['Content-Length'])))+ self.server.requests.append(body)+ parent = any('FACTORY_VALIDATION_PARENT' in ''.join(texts(m.get('content')))+ for m in body.get('messages', []) if m.get('role') == 'user')+ if not parent:+ self.server.errors.append('unexpected child or unrelated provider request')+ self.send_error(422)+ return+ self.server.parent_requests.append(body)+ if len(self.server.parent_requests) == 1:+ offered = [t['function']['name'] for t in body.get('tools', [])]+ repl = next((name for name in ('python_repl', 'ipython') if name in offered), None)+ assert repl is not None, offered+ delta = {'tool_calls': [{'index': 0, 'id': 'factory_probe', 'type': 'function',+ 'function': {'name': repl, 'arguments': json.dumps({'code': CELL})}}]}+ reason = 'tool_calls'+ else:+ delta, reason = {'content': 'Factory validation probe complete.'}, 'stop'+ def chunk(delta, reason=None):+ return {'id': 'factory-fixture', 'object': 'chat.completion.chunk', 'created': 1,+ 'model': 'factory-fixture', 'choices': [{'index': 0, 'delta': delta, 'finish_reason': reason}]}+ wire = ''.join('data: ' + json.dumps(row) + '\n\n' for row in (+ chunk({'role': 'assistant', **delta}), chunk({}, reason))) + 'data: [DONE]\n\n'+ self.send_response(200)+ self.send_header('Content-Type', 'text/event-stream')+ self.send_header('Content-Length', str(len(wire.encode())))+ self.end_headers()+ self.wfile.write(wire.encode())+++def extract(requests):+ assert len(requests) == 2, 'unexpected parent provider sequence'+ results = [m for m in requests[-1]['messages'] if m.get('role') == 'tool'+ and m.get('tool_call_id') == 'factory_probe']+ assert len(results) == 1, 'missing/duplicate current tool result'+ values = [json.loads(line[len(MARKER):]) for content in texts(results[0].get('content'))+ for line in content.splitlines() if line.startswith(MARKER)]+ assert len(values) == 1, 'missing/duplicate actual factory receipt'+ return values[0]+++def validate(probe):+ assert len(probe) == 3, probe+ positive, zero, malformed = probe+ assert positive['case'] == 'positive' and positive['stored'] is False, positive+ assert 'resident node watcher cannot declare retries' in positive['error'], positive+ assert zero == {'case': 'zero', 'stored': True}, zero+ assert malformed['case'] == 'malformed' and malformed['stored'] is False, malformed+ assert 'retries must be an integer between 0 and 10' in malformed['error'], malformed+++def observe(binary, kind, root, observed):+ env = env_for(root, kind)+ env[TAG] = str(root)+ server = Provider()+ threading.Thread(target=server.serve_forever, daemon=True).start()+ (root / 'agent/settings.json').write_text(json.dumps({'factory': {'enabled': True},+ 'prewarmPythonKernel': False, 'defaultProvider': 'parity-local', 'defaultModel': 'parity-1'}))+ (root / 'agent/models.json').write_text(json.dumps({'providers': {'parity-local': {+ 'api': 'openai-completions', 'apiKey': 'fixture-only',+ 'baseUrl': f'http://127.0.0.1:{server.server_port}/v1', 'models': [{+ 'id': 'parity-1', 'name': 'Factory fixture', 'reasoning': False,+ 'contextWindow': 128000, 'maxTokens': 4096}]}}}))+ path = root / 'd.sock'+ scope = OwnedProcesses(binary, root, path)+ daemon = control = None+ log = open(root / 'daemon.log', 'wb')+ try:+ command = [str(binary), '--mode', 'daemon', '--daemon-socket', str(path)]+ daemon = subprocess.Popen(command, cwd=root, env=env, stdout=log,+ stderr=subprocess.STDOUT, start_new_session=True)+ scope.register(daemon, command)+ control = Client(path, daemon)+ observed['hello'] = control.hello+ caps = control.hello.get('serverCapabilities', [])+ if kind == 'ts':+ assert 'factory_activity' not in caps, 'TS unexpectedly advertises factory'+ observed['factory_lane_advertised'] = False+ else:+ created = control.call('create', config={'cwd': str(root), 'agentDir': str(root / 'agent'),+ 'provider': 'parity-local', 'model': 'parity-1', 'telemetryDisabled': True})+ active = created.get('activeSessionId') or created.get('id')+ assert active, created+ control.call('attach', activeSessionId=active)+ control.call('prompt', activeSessionId=active, message='Run FACTORY_VALIDATION_PARENT')+ control.call('wait_for_idle', timeout_seconds=120, activeSessionId=active)+ observed['probe'] = extract(server.parent_requests)+ validate(observed['probe'])+ assert not server.errors, server.errors+ observed['behavior_verified'] = True+ finally:+ observed['provider_requests'] = server.requests+ observed['provider_errors'] = server.errors+ observed['wire'] = list(control.wire) if control else []+ finish(scope, control, daemon, observed)+ log.close()+ observed['daemon_log'] = (root / 'daemon.log').read_text(errors='replace')[-10000:]+ server.shutdown()+ server.server_close()+++def main():+ parser = argparse.ArgumentParser(description=__doc__)+ parser.add_argument('--receipt', type=Path, required=True)+ args = parser.parse_args()+ result = {'verified': False, 'scope': __doc__, 'runs': {}, 'not_verified': [+ 'TS factory behavior (no advertised historical counterpart)', 'TUI', 'factory execution and recovery', 'TS validator equality']}+ try:+ assert sys.platform == 'linux', 'hosted Linux only'+ assert digest(Path(os.environ['PA_TS_ARCHIVE'])) == TS_ARCHIVE_SHA256+ result['ts_archive_sha256'] = TS_ARCHIVE_SHA256+ with tempfile.TemporaryDirectory(prefix='pa-factory-', dir='/tmp') as folder:+ for kind in ('ts', 'rust'):+ binary = Path(os.environ[f'PA_{kind.upper()}_BINARY']).resolve()+ observed = result['runs'][kind] = {'binary_sha256': digest(binary)}+ try:+ observe(binary, kind, Path(folder) / kind, observed)+ except Exception as error:+ observed['error'] = str(error)+ assert all(r.get('behavior_verified') and r.get('cleanup_verified') and 'error' not in r+ for r in result['runs'].values()), 'actual factory observation failed'+ result['verified'] = True+ except Exception as error:+ result['error'] = str(error)+ args.receipt.parent.mkdir(parents=True, exist_ok=True)+ args.receipt.write_text(json.dumps(result, ensure_ascii=False, indent=2))+ return 0 if result['verified'] else 1+++if __name__ == '__main__':+ sys.exit(main())diff --git a/scripts/test_review_factory_validation_parity.py b/scripts/test_review_factory_validation_parity.pynew file mode 100644index 0000000000..da18476e98--- /dev/null+++ b/scripts/test_review_factory_validation_parity.py@@ -0,0 +1,52 @@+import io+import json+from types import SimpleNamespace+import unittest+from review_factory_validation_parity import validate, Handler+++class FactoryValidationGuards(unittest.TestCase):+ def value(self):+ return [+ {'case': 'positive', 'stored': False, 'error': 'resident node watcher cannot declare retries'},+ {'case': 'zero', 'stored': True},+ {'case': 'malformed', 'stored': False, 'error': 'node watcher retries must be an integer between 0 and 10'}]++ def test_rejects_acceptance_or_storage_of_invalid_specs(self):+ validate(self.value())+ for index in (0, 2):+ value = self.value()+ value[index]['stored'] = True+ with self.assertRaises(AssertionError):+ validate(value)+ value = self.value()+ value[1]['stored'] = False+ with self.assertRaises(AssertionError):+ validate(value)++ def test_unexpected_child_request_is_rejected_before_a_model_response(self):+ body = json.dumps({'messages': [{'role': 'user', 'content': 'unexpected child'}]}).encode()+ handler = Handler.__new__(Handler)+ handler.path = '/v1/chat/completions'+ handler.headers = {'Content-Length': str(len(body))}+ handler.rfile = io.BytesIO(body)+ handler.server = SimpleNamespace(requests=[], parent_requests=[], errors=[])+ statuses = []+ handler.send_error = statuses.append+ handler.do_POST()+ self.assertEqual(statuses, [422])+ self.assertEqual(handler.server.parent_requests, [])+ self.assertEqual(handler.server.errors, ['unexpected child or unrelated provider request'])++ def test_requires_correct_diagnostic_and_all_cases(self):+ for index in (0, 2):+ value = self.value()+ value[index]['error'] = 'unrelated failure'+ with self.assertRaises(AssertionError):+ validate(value)+ with self.assertRaises(AssertionError):+ validate(self.value()[:-1])+++if __name__ == '__main__':+ unittest.main()