A self-improving RLM agent for coding workflows and long-running autonomous tasks.
Rust
#!/usr/bin/env python3
"""Observe actual refinement errors and compare captured pinned TS outputs.
The Rust output-limit message intentionally differs. Fixture verification does
not waive parity: parity_equal and every error string are recorded separately.
"""
import argparse
import http.server
import json
import os
from pathlib import Path
import subprocess
import tempfile
import threading
from check_saved_catalog import CatalogClient
from lifecycle_lib import OwnedProcesses, TAG, digest, env_for, finish
class Server(http.server.ThreadingHTTPServer):
daemon_threads = True
def __init__(self, scenario):
super().__init__(('127.0.0.1', 0), ErrorHandler)
self.scenario, self.requests, self.received_headers = scenario, [], []
class ErrorHandler(http.server.BaseHTTPRequestHandler):
def log_message(self, *_args):
pass
def do_POST(self):
body = json.loads(self.rfile.read(int(self.headers['Content-Length'])))
self.server.requests.append({'path': self.path, 'body': body})
self.server.received_headers.append(dict(self.headers))
if self.path != '/v1/chat/completions':
self.send_error(404)
return
if self.server.scenario == 'provider_error':
data = json.dumps({'error': {'message': 'fixture token budget invalid',
'type': 'invalid_request_error', 'code': 'invalid_request_error'}}).encode()
self.send_response(400)
self.send_header('Content-Type', 'application/json')
else:
content = '{"summary":"cut' if self.server.scenario == 'length' else 'not-json'
reason = 'length' if self.server.scenario == 'length' else 'stop'
def chunk(delta, finish=None):
return {'id': 'fixture', 'object': 'chat.completion.chunk', 'created': 1,
'model': 'parity-1', 'choices': [{'index': 0, 'delta': delta,
'finish_reason': finish}]}
data = (''.join('data: ' + json.dumps(c) + '\n\n' for c in (
chunk({'role': 'assistant', 'content': content}), chunk({}, reason)))
+ 'data: [DONE]\n\n').encode()
self.send_response(200)
self.send_header('Content-Type', 'text/event-stream')
self.send_header('Content-Length', str(len(data)))
self.end_headers()
self.wfile.write(data)
TS_OBSERVED = {'length': 'Refinement failed: the model stopped before completing its JSON object. This usually means the output budget was exhausted; retry with a smaller request.', 'provider_error': 'Refinement failed: 400 fixture token budget invalid', 'normal_invalid': 'Refiner did not return a JSON object'}
def observe(binary, root, case, receipt):
env = env_for(root)
env[TAG] = str(root)
server = Server(case)
thread = threading.Thread(target=server.serve_forever, daemon=True)
thread.start()
(root / 'agent/auth.json').write_text(json.dumps({'prime-inference': {
'type': 'api_key', 'key': 'fixture-only',
'primeTeam': {'teamId': 'team-1', 'name': 'Test Team'}}}))
(root / 'agent/models.json').write_text(json.dumps({'providers': {
'prime-inference': {'api': 'openai-completions',
'baseUrl': f'http://127.0.0.1:{server.server_port}/v1',
'models': [{'id': 'parity-1', 'name': 'Parity', 'reasoning': False,
'input': ['text'], 'contextWindow': 128000, 'maxTokens': 8192}]}}}))
(root / 'agent/settings.json').write_text(json.dumps({
'defaultProvider': 'prime-inference', 'defaultModel': 'parity-1',
'auxiliaryModel': 'prime-inference/parity-1',
'prewarmPythonKernel': False, 'autoRefine': {'enabled': False},
'onboardingCompleted': True}))
path = root / 'd.sock'
scope = OwnedProcesses(binary, root, path)
process = client = None
with open(root / 'daemon.log', 'wb') as log:
try:
command = [str(binary), '--mode', 'daemon', '--daemon-socket', str(path)]
process = subprocess.Popen(command, cwd=root, env=env, stdout=log,
stderr=subprocess.STDOUT, start_new_session=True)
scope.register(process, command)
client = CatalogClient(path, process)
receipt['hello'] = client.hello
created = client.call('create', config={
'cwd': str(root), 'agentDir': str(root / 'agent'),
'sessionDir': str(root / 'agent/sessions'),
'provider': 'prime-inference', 'model': 'parity-1',
'telemetryDisabled': True, 'noExtensions': True, 'noContextFiles': True})
active = created.get('activeSessionId') or created.get('id')
assert active, created
try:
client.call('refine', activeSessionId=active,
instructions='Fixture no-op refinement')
except AssertionError as error:
response = error.args[0]
assert isinstance(response, dict) and response.get('command') == 'refine', response
assert response.get('success') is False and isinstance(response.get('error'), str), response
actual = response['error']
else:
raise AssertionError('refine must report the fixture failure')
assert len(server.requests) == 1, server.requests
receipt.update(actual_error=actual, ts_observed_error=TS_OBSERVED[case],
parity_equal=actual == TS_OBSERVED[case])
if case == 'length':
body = server.requests[0]['body']
budget = body.get('max_tokens', body.get('max_completion_tokens'))
assert isinstance(budget, int) and budget > 0, body
receipt['requested_output_tokens'] = budget
assert actual == ('Refinement failed: the model hit its output limit '
f'({budget} tokens, prime-inference/parity-1)'), actual
elif case == 'provider_error':
assert actual.startswith('Refinement failed: ') \
and 'fixture token budget invalid' in actual, actual
else:
assert actual == 'Refiner did not return a JSON object', actual
receipt['behavior_verified'] = True
finally:
receipt['http_requests'] = server.requests
receipt['http_headers'] = server.received_headers
receipt['raw_daemon_sent_hex'] = list(client.sent) if client else []
receipt['raw_daemon_received_hex'] = list(client.received) if client else []
finish(scope, client, process, receipt)
server.shutdown()
server.server_close()
thread.join(timeout=2)
receipt['provider_stopped'] = not thread.is_alive()
log.flush()
receipt['daemon_log'] = (root / 'daemon.log').read_text(errors='replace')[-3000:]
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('--receipt', required=True, type=Path)
args = parser.parse_args()
receipt = {'verified': False, 'scope': __doc__, 'cases': {},
'ts_reference': 'v0.9.8 / 7d442aafa985f9342134fac16c2ef41f03fb45c1',
'ts_binary_sha256': 'e50d4e4cf5735ea16cee30c3ca48fccb628ae2169fc867c33d461b8a90e87bc5',
'not_verified': ['auto-review HTTP execution', 'TUI frames',
'full daemon wire-byte equality', 'live Prime Inference']}
try:
binary = Path(os.environ['PA_RUST_BINARY']).resolve()
receipt['binary_sha256'] = digest(binary)
with tempfile.TemporaryDirectory(prefix='pa-refine-errors-', dir='/tmp') as folder:
for case in TS_OBSERVED:
observed = receipt['cases'][case] = {}
try:
observe(binary, Path(folder) / case, case, observed)
except Exception as error:
observed['error'] = str(error)
receipt['verified'] = all(c.get('behavior_verified') and c.get('cleanup_verified')
and c.get('provider_stopped') and not c.get('error')
for c in receipt['cases'].values())
receipt['parity_equal'] = all(c.get('parity_equal') is True
for c in receipt['cases'].values())
if not receipt['verified']:
receipt['error'] = 'one or more output controls failed; see cases'
except Exception as error:
receipt['error'] = str(error)
args.receipt.parent.mkdir(parents=True, exist_ok=True)
args.receipt.write_text(json.dumps(receipt, indent=2))
print(json.dumps({'verified': receipt['verified'], 'error': receipt.get('error')}))
return 0 if receipt['verified'] else 1
if __name__ == '__main__':
raise SystemExit(main())