A self-improving RLM agent for coding workflows and long-running autonomous tasks.
Rust
side calls: report the real error instead of "did not return a JSON object" (#3814)
Sebastian Müllerc0325bd6314c
diff --git a/.changes/side-call-errors.md b/.changes/side-call-errors.mdnew file mode 100644index 0000000000..c6842e4d3b--- /dev/null+++ b/.changes/side-call-errors.md@@ -0,0 +1 @@+- `/refine` and the automatic refine review now say why they failed: a reply cut off at the model's output limit reports the limit and the model, and a provider error reports the provider's message, instead of "Refiner did not return a JSON object".diff --git a/crates/pa-core/src/refinement/executor.rs b/crates/pa-core/src/refinement/executor.rsindex 7a22633de5..02b24c3fba 100644--- a/crates/pa-core/src/refinement/executor.rs+++ b/crates/pa-core/src/refinement/executor.rs@@ -10,7 +10,7 @@ use super::{infer_refinement_result_scope, merge_refinement_history, HarnessScope, HarnessState,RefinementKind, RefinementResult, REFINEMENT_KINDS,};-use pa_types::ai::AssistantMessage;+use pa_types::ai::{AssistantMessage, StopReason};use pa_types::session::AgentMessage;#[derive(Debug, Default, Clone)]@@ -257,8 +257,9 @@ pub async fn plan_refinement(let mut request_model = model.clone();request_model.max_tokens = request_max_tokens.min(REFINEMENT_MAX_OUTPUT_TOKENS);+ let max_tokens = request_model.max_tokens;let reply = refine_call(request_model, REFINEMENT_SYSTEM_PROMPT, user_prompt).await?;- let text = assistant_text(&reply);+ let text = reply_text(&reply, max_tokens, "Refinement failed")?;Ok(RefinementPlan {proposal: parse_proposal(&text).map_err(anyhow::Error::msg)?,id,@@ -267,8 +268,22 @@ pub async fn plan_refinement(})}-fn assistant_text(reply: &AssistantMessage) -> String {- reply+/// The text of a side call's reply. A reply cut off at its output limit or+/// failed by the provider is an error that says so, never parsed as JSON.+fn reply_text(reply: &AssistantMessage, max_tokens: u64, failure: &str) -> anyhow::Result<String> {+ match reply.stop_reason {+ StopReason::Length => anyhow::bail!(+ "{failure}: the model hit its output limit ({max_tokens} tokens, {}/{})",+ reply.provider,+ reply.model+ ),+ StopReason::Error => anyhow::bail!(+ "{failure}: {}",+ reply.error_message.as_deref().unwrap_or("Unknown error")+ ),+ StopReason::Stop | StopReason::ToolUse | StopReason::Aborted => {}+ }+ Ok(reply.content.iter().filter_map(|block| match block {@@ -276,7 +291,7 @@ fn assistant_text(reply: &AssistantMessage) -> String {_ => None,}).collect::<Vec<_>>()- .join("\n")+ .join("\n"))}/// Apply a plan to the (re-read) harness state. `factory_enabled` is the@@ -382,8 +397,13 @@ pub async fn review_auto_refine()?;let mut request_model = model.clone();request_model.max_tokens = request_max_tokens.min(AUTO_REFINE_REVIEW_MAX_OUTPUT_TOKENS);+ let max_tokens = request_model.max_tokens;let reply = review_call(request_model, AUTO_REFINE_REVIEW_SYSTEM_PROMPT, user_prompt).await?;- parse_auto_refine_review(&assistant_text(&reply))+ parse_auto_refine_review(&reply_text(+ &reply,+ max_tokens,+ "Auto-refine review failed",+ )?)}#[cfg(test)]@@ -480,6 +500,48 @@ mod tests {);}+ /// A cut-off or failed reply reports its stop instead of a JSON parse+ /// error on its partial text.+ #[tokio::test]+ async fn a_cut_off_or_failed_reply_reports_why() {+ let stopped = |stop_reason: StopReason| -> RefinerFn {+ Box::new(move |_model, _system, _prompt| {+ Box::pin(async move {+ Ok(AssistantMessage {+ stop_reason,+ error_message: Some("max_tokens is too large".to_string()),+ ..text_message(r#"{"summary": "cut"#)+ })+ })+ })+ };+ let (state, model) = (super::super::empty_harness_state(), test_model());+ let options = RefineOptions::default();+ let plan = plan_refinement(+ &[],+ &state,+ &[],+ &model,+ &options,+ stopped(StopReason::Length),+ );+ let ctx = AutoRefineReviewContext {+ reason: "checkpoint".to_string(),+ turns_since_last_review: 1,+ };+ let review = review_auto_refine(&[], &state, &[], &model, &ctx, stopped(StopReason::Error));+ assert_eq!(+ [+ plan.await.err().unwrap().to_string(),+ review.await.err().unwrap().to_string()+ ],+ [+ "Refinement failed: the model hit its output limit (8000 tokens, test/m)",+ "Auto-refine review failed: max_tokens is too large",+ ]+ );+ }+#[test]fn refinement_ids_are_canonical() {let id = generate_refinement_id();diff --git a/scripts/check_refine_errors.py b/scripts/check_refine_errors.pynew file mode 100644index 0000000000..f28e2a04b4--- /dev/null+++ b/scripts/check_refine_errors.py@@ -0,0 +1,176 @@+#!/usr/bin/env python3+"""Observe actual refinement errors and compare captured pinned TS outputs.++The Rust output-limit message intentionally differs. Fixture verification does+not waive parity: parity_equal and every error string are recorded separately.+"""+import argparse+import http.server+import json+import os+from pathlib import Path+import subprocess+import tempfile+import threading++from check_saved_catalog import CatalogClient+from lifecycle_lib import OwnedProcesses, TAG, digest, env_for, finish+++class Server(http.server.ThreadingHTTPServer):+ daemon_threads = True++ def __init__(self, scenario):+ super().__init__(('127.0.0.1', 0), ErrorHandler)+ self.scenario, self.requests, self.received_headers = scenario, [], []+++class ErrorHandler(http.server.BaseHTTPRequestHandler):+ def log_message(self, *_args):+ pass++ def do_POST(self):+ body = json.loads(self.rfile.read(int(self.headers['Content-Length'])))+ self.server.requests.append({'path': self.path, 'body': body})+ self.server.received_headers.append(dict(self.headers))+ if self.path != '/v1/chat/completions':+ self.send_error(404)+ return+ if self.server.scenario == 'provider_error':+ data = json.dumps({'error': {'message': 'fixture token budget invalid',+ 'type': 'invalid_request_error', 'code': 'invalid_request_error'}}).encode()+ self.send_response(400)+ self.send_header('Content-Type', 'application/json')+ else:+ content = '{"summary":"cut' if self.server.scenario == 'length' else 'not-json'+ reason = 'length' if self.server.scenario == 'length' else 'stop'+ def chunk(delta, finish=None):+ return {'id': 'fixture', 'object': 'chat.completion.chunk', 'created': 1,+ 'model': 'parity-1', 'choices': [{'index': 0, 'delta': delta,+ 'finish_reason': finish}]}+ data = (''.join('data: ' + json.dumps(c) + '\n\n' for c in (+ chunk({'role': 'assistant', 'content': content}), chunk({}, reason)))+ + 'data: [DONE]\n\n').encode()+ self.send_response(200)+ self.send_header('Content-Type', 'text/event-stream')+ self.send_header('Content-Length', str(len(data)))+ self.end_headers()+ self.wfile.write(data)+++TS_OBSERVED = {'length': 'Refinement failed: the model stopped before completing its JSON object. This usually means the output budget was exhausted; retry with a smaller request.', 'provider_error': 'Refinement failed: 400 fixture token budget invalid', 'normal_invalid': 'Refiner did not return a JSON object'}+++def observe(binary, root, case, receipt):+ env = env_for(root)+ env[TAG] = str(root)+ server = Server(case)+ thread = threading.Thread(target=server.serve_forever, daemon=True)+ thread.start()+ (root / 'agent/auth.json').write_text(json.dumps({'prime-inference': {+ 'type': 'api_key', 'key': 'fixture-only',+ 'primeTeam': {'teamId': 'team-1', 'name': 'Test Team'}}}))+ (root / 'agent/models.json').write_text(json.dumps({'providers': {+ 'prime-inference': {'api': 'openai-completions',+ 'baseUrl': f'http://127.0.0.1:{server.server_port}/v1',+ 'models': [{'id': 'parity-1', 'name': 'Parity', 'reasoning': False,+ 'input': ['text'], 'contextWindow': 128000, 'maxTokens': 8192}]}}}))+ (root / 'agent/settings.json').write_text(json.dumps({+ 'defaultProvider': 'prime-inference', 'defaultModel': 'parity-1',+ 'auxiliaryModel': 'prime-inference/parity-1',+ 'prewarmPythonKernel': False, 'autoRefine': {'enabled': False},+ 'onboardingCompleted': True}))+ path = root / 'd.sock'+ scope = OwnedProcesses(binary, root, path)+ process = client = None+ with open(root / 'daemon.log', 'wb') as log:+ try:+ command = [str(binary), '--mode', 'daemon', '--daemon-socket', str(path)]+ process = subprocess.Popen(command, cwd=root, env=env, stdout=log,+ stderr=subprocess.STDOUT, start_new_session=True)+ scope.register(process, command)+ client = CatalogClient(path, process)+ receipt['hello'] = client.hello+ created = client.call('create', config={+ 'cwd': str(root), 'agentDir': str(root / 'agent'),+ 'sessionDir': str(root / 'agent/sessions'),+ 'provider': 'prime-inference', 'model': 'parity-1',+ 'telemetryDisabled': True, 'noExtensions': True, 'noContextFiles': True})+ active = created.get('activeSessionId') or created.get('id')+ assert active, created+ try:+ client.call('refine', activeSessionId=active,+ instructions='Fixture no-op refinement')+ except AssertionError as error:+ response = error.args[0]+ assert isinstance(response, dict) and response.get('command') == 'refine', response+ assert response.get('success') is False and isinstance(response.get('error'), str), response+ actual = response['error']+ else:+ raise AssertionError('refine must report the fixture failure')+ assert len(server.requests) == 1, server.requests+ receipt.update(actual_error=actual, ts_observed_error=TS_OBSERVED[case],+ parity_equal=actual == TS_OBSERVED[case])+ if case == 'length':+ body = server.requests[0]['body']+ budget = body.get('max_tokens', body.get('max_completion_tokens'))+ assert isinstance(budget, int) and budget > 0, body+ receipt['requested_output_tokens'] = budget+ assert actual == ('Refinement failed: the model hit its output limit '+ f'({budget} tokens, prime-inference/parity-1)'), actual+ elif case == 'provider_error':+ assert actual.startswith('Refinement failed: ') \+ and 'fixture token budget invalid' in actual, actual+ else:+ assert actual == 'Refiner did not return a JSON object', actual+ receipt['behavior_verified'] = True+ finally:+ receipt['http_requests'] = server.requests+ receipt['http_headers'] = server.received_headers+ receipt['raw_daemon_sent_hex'] = list(client.sent) if client else []+ receipt['raw_daemon_received_hex'] = list(client.received) if client else []+ finish(scope, client, process, receipt)+ server.shutdown()+ server.server_close()+ thread.join(timeout=2)+ receipt['provider_stopped'] = not thread.is_alive()+ log.flush()+ receipt['daemon_log'] = (root / 'daemon.log').read_text(errors='replace')[-3000:]+++def main():+ parser = argparse.ArgumentParser(description=__doc__)+ parser.add_argument('--receipt', required=True, type=Path)+ args = parser.parse_args()+ receipt = {'verified': False, 'scope': __doc__, 'cases': {},+ 'ts_reference': 'v0.9.8 / 7d442aafa985f9342134fac16c2ef41f03fb45c1',+ 'ts_binary_sha256': 'e50d4e4cf5735ea16cee30c3ca48fccb628ae2169fc867c33d461b8a90e87bc5',+ 'not_verified': ['auto-review HTTP execution', 'TUI frames',+ 'full daemon wire-byte equality', 'live Prime Inference']}+ try:+ binary = Path(os.environ['PA_RUST_BINARY']).resolve()+ receipt['binary_sha256'] = digest(binary)+ with tempfile.TemporaryDirectory(prefix='pa-refine-errors-', dir='/tmp') as folder:+ for case in TS_OBSERVED:+ observed = receipt['cases'][case] = {}+ try:+ observe(binary, Path(folder) / case, case, observed)+ except Exception as error:+ observed['error'] = str(error)+ receipt['verified'] = all(c.get('behavior_verified') and c.get('cleanup_verified')+ and c.get('provider_stopped') and not c.get('error')+ for c in receipt['cases'].values())+ receipt['parity_equal'] = all(c.get('parity_equal') is True+ for c in receipt['cases'].values())+ if not receipt['verified']:+ receipt['error'] = 'one or more output controls failed; see cases'+ except Exception as error:+ receipt['error'] = str(error)+ args.receipt.parent.mkdir(parents=True, exist_ok=True)+ args.receipt.write_text(json.dumps(receipt, indent=2))+ print(json.dumps({'verified': receipt['verified'], 'error': receipt.get('error')}))+ return 0 if receipt['verified'] else 1+++if __name__ == '__main__':+ raise SystemExit(main())