Skip to content

A self-improving RLM agent for coding workflows and long-running autonomous tasks.

Rust

side calls: report the real error instead of "did not return a JSON object" (#3814)

Sebastian Müllerc0325bd6314c
diff --git a/.changes/side-call-errors.md b/.changes/side-call-errors.md
new file mode 100644
index 0000000000..c6842e4d3b
--- /dev/null
+++ b/.changes/side-call-errors.md
@@ -0,0 +1 @@
+- `/refine` and the automatic refine review now say why they failed: a reply cut off at the model's output limit reports the limit and the model, and a provider error reports the provider's message, instead of "Refiner did not return a JSON object".
diff --git a/crates/pa-core/src/refinement/executor.rs b/crates/pa-core/src/refinement/executor.rs
index 7a22633de5..02b24c3fba 100644
--- a/crates/pa-core/src/refinement/executor.rs
+++ b/crates/pa-core/src/refinement/executor.rs
@@ -10,7 +10,7 @@ use super::{
infer_refinement_result_scope, merge_refinement_history, HarnessScope, HarnessState,
RefinementKind, RefinementResult, REFINEMENT_KINDS,
};
-use pa_types::ai::AssistantMessage;
+use pa_types::ai::{AssistantMessage, StopReason};
use pa_types::session::AgentMessage;
#[derive(Debug, Default, Clone)]
@@ -257,8 +257,9 @@ pub async fn plan_refinement(
let mut request_model = model.clone();
request_model.max_tokens = request_max_tokens.min(REFINEMENT_MAX_OUTPUT_TOKENS);
+ let max_tokens = request_model.max_tokens;
let reply = refine_call(request_model, REFINEMENT_SYSTEM_PROMPT, user_prompt).await?;
- let text = assistant_text(&reply);
+ let text = reply_text(&reply, max_tokens, "Refinement failed")?;
Ok(RefinementPlan {
proposal: parse_proposal(&text).map_err(anyhow::Error::msg)?,
id,
@@ -267,8 +268,22 @@ pub async fn plan_refinement(
})
}
-fn assistant_text(reply: &AssistantMessage) -> String {
- reply
+/// The text of a side call's reply. A reply cut off at its output limit or
+/// failed by the provider is an error that says so, never parsed as JSON.
+fn reply_text(reply: &AssistantMessage, max_tokens: u64, failure: &str) -> anyhow::Result<String> {
+ match reply.stop_reason {
+ StopReason::Length => anyhow::bail!(
+ "{failure}: the model hit its output limit ({max_tokens} tokens, {}/{})",
+ reply.provider,
+ reply.model
+ ),
+ StopReason::Error => anyhow::bail!(
+ "{failure}: {}",
+ reply.error_message.as_deref().unwrap_or("Unknown error")
+ ),
+ StopReason::Stop | StopReason::ToolUse | StopReason::Aborted => {}
+ }
+ Ok(reply
.content
.iter()
.filter_map(|block| match block {
@@ -276,7 +291,7 @@ fn assistant_text(reply: &AssistantMessage) -> String {
_ => None,
})
.collect::<Vec<_>>()
- .join("\n")
+ .join("\n"))
}
/// Apply a plan to the (re-read) harness state. `factory_enabled` is the
@@ -382,8 +397,13 @@ pub async fn review_auto_refine(
)?;
let mut request_model = model.clone();
request_model.max_tokens = request_max_tokens.min(AUTO_REFINE_REVIEW_MAX_OUTPUT_TOKENS);
+ let max_tokens = request_model.max_tokens;
let reply = review_call(request_model, AUTO_REFINE_REVIEW_SYSTEM_PROMPT, user_prompt).await?;
- parse_auto_refine_review(&assistant_text(&reply))
+ parse_auto_refine_review(&reply_text(
+ &reply,
+ max_tokens,
+ "Auto-refine review failed",
+ )?)
}
#[cfg(test)]
@@ -480,6 +500,48 @@ mod tests {
);
}
+ /// A cut-off or failed reply reports its stop instead of a JSON parse
+ /// error on its partial text.
+ #[tokio::test]
+ async fn a_cut_off_or_failed_reply_reports_why() {
+ let stopped = |stop_reason: StopReason| -> RefinerFn {
+ Box::new(move |_model, _system, _prompt| {
+ Box::pin(async move {
+ Ok(AssistantMessage {
+ stop_reason,
+ error_message: Some("max_tokens is too large".to_string()),
+ ..text_message(r#"{"summary": "cut"#)
+ })
+ })
+ })
+ };
+ let (state, model) = (super::super::empty_harness_state(), test_model());
+ let options = RefineOptions::default();
+ let plan = plan_refinement(
+ &[],
+ &state,
+ &[],
+ &model,
+ &options,
+ stopped(StopReason::Length),
+ );
+ let ctx = AutoRefineReviewContext {
+ reason: "checkpoint".to_string(),
+ turns_since_last_review: 1,
+ };
+ let review = review_auto_refine(&[], &state, &[], &model, &ctx, stopped(StopReason::Error));
+ assert_eq!(
+ [
+ plan.await.err().unwrap().to_string(),
+ review.await.err().unwrap().to_string()
+ ],
+ [
+ "Refinement failed: the model hit its output limit (8000 tokens, test/m)",
+ "Auto-refine review failed: max_tokens is too large",
+ ]
+ );
+ }
+
#[test]
fn refinement_ids_are_canonical() {
let id = generate_refinement_id();
diff --git a/scripts/check_refine_errors.py b/scripts/check_refine_errors.py
new file mode 100644
index 0000000000..f28e2a04b4
--- /dev/null
+++ b/scripts/check_refine_errors.py
@@ -0,0 +1,176 @@
+#!/usr/bin/env python3
+"""Observe actual refinement errors and compare captured pinned TS outputs.
+
+The Rust output-limit message intentionally differs. Fixture verification does
+not waive parity: parity_equal and every error string are recorded separately.
+"""
+import argparse
+import http.server
+import json
+import os
+from pathlib import Path
+import subprocess
+import tempfile
+import threading
+
+from check_saved_catalog import CatalogClient
+from lifecycle_lib import OwnedProcesses, TAG, digest, env_for, finish
+
+
+class Server(http.server.ThreadingHTTPServer):
+ daemon_threads = True
+
+ def __init__(self, scenario):
+ super().__init__(('127.0.0.1', 0), ErrorHandler)
+ self.scenario, self.requests, self.received_headers = scenario, [], []
+
+
+class ErrorHandler(http.server.BaseHTTPRequestHandler):
+ def log_message(self, *_args):
+ pass
+
+ def do_POST(self):
+ body = json.loads(self.rfile.read(int(self.headers['Content-Length'])))
+ self.server.requests.append({'path': self.path, 'body': body})
+ self.server.received_headers.append(dict(self.headers))
+ if self.path != '/v1/chat/completions':
+ self.send_error(404)
+ return
+ if self.server.scenario == 'provider_error':
+ data = json.dumps({'error': {'message': 'fixture token budget invalid',
+ 'type': 'invalid_request_error', 'code': 'invalid_request_error'}}).encode()
+ self.send_response(400)
+ self.send_header('Content-Type', 'application/json')
+ else:
+ content = '{"summary":"cut' if self.server.scenario == 'length' else 'not-json'
+ reason = 'length' if self.server.scenario == 'length' else 'stop'
+ def chunk(delta, finish=None):
+ return {'id': 'fixture', 'object': 'chat.completion.chunk', 'created': 1,
+ 'model': 'parity-1', 'choices': [{'index': 0, 'delta': delta,
+ 'finish_reason': finish}]}
+ data = (''.join('data: ' + json.dumps(c) + '\n\n' for c in (
+ chunk({'role': 'assistant', 'content': content}), chunk({}, reason)))
+ + 'data: [DONE]\n\n').encode()
+ self.send_response(200)
+ self.send_header('Content-Type', 'text/event-stream')
+ self.send_header('Content-Length', str(len(data)))
+ self.end_headers()
+ self.wfile.write(data)
+
+
+TS_OBSERVED = {'length': 'Refinement failed: the model stopped before completing its JSON object. This usually means the output budget was exhausted; retry with a smaller request.', 'provider_error': 'Refinement failed: 400 fixture token budget invalid', 'normal_invalid': 'Refiner did not return a JSON object'}
+
+
+def observe(binary, root, case, receipt):
+ env = env_for(root)
+ env[TAG] = str(root)
+ server = Server(case)
+ thread = threading.Thread(target=server.serve_forever, daemon=True)
+ thread.start()
+ (root / 'agent/auth.json').write_text(json.dumps({'prime-inference': {
+ 'type': 'api_key', 'key': 'fixture-only',
+ 'primeTeam': {'teamId': 'team-1', 'name': 'Test Team'}}}))
+ (root / 'agent/models.json').write_text(json.dumps({'providers': {
+ 'prime-inference': {'api': 'openai-completions',
+ 'baseUrl': f'http://127.0.0.1:{server.server_port}/v1',
+ 'models': [{'id': 'parity-1', 'name': 'Parity', 'reasoning': False,
+ 'input': ['text'], 'contextWindow': 128000, 'maxTokens': 8192}]}}}))
+ (root / 'agent/settings.json').write_text(json.dumps({
+ 'defaultProvider': 'prime-inference', 'defaultModel': 'parity-1',
+ 'auxiliaryModel': 'prime-inference/parity-1',
+ 'prewarmPythonKernel': False, 'autoRefine': {'enabled': False},
+ 'onboardingCompleted': True}))
+ path = root / 'd.sock'
+ scope = OwnedProcesses(binary, root, path)
+ process = client = None
+ with open(root / 'daemon.log', 'wb') as log:
+ try:
+ command = [str(binary), '--mode', 'daemon', '--daemon-socket', str(path)]
+ process = subprocess.Popen(command, cwd=root, env=env, stdout=log,
+ stderr=subprocess.STDOUT, start_new_session=True)
+ scope.register(process, command)
+ client = CatalogClient(path, process)
+ receipt['hello'] = client.hello
+ created = client.call('create', config={
+ 'cwd': str(root), 'agentDir': str(root / 'agent'),
+ 'sessionDir': str(root / 'agent/sessions'),
+ 'provider': 'prime-inference', 'model': 'parity-1',
+ 'telemetryDisabled': True, 'noExtensions': True, 'noContextFiles': True})
+ active = created.get('activeSessionId') or created.get('id')
+ assert active, created
+ try:
+ client.call('refine', activeSessionId=active,
+ instructions='Fixture no-op refinement')
+ except AssertionError as error:
+ response = error.args[0]
+ assert isinstance(response, dict) and response.get('command') == 'refine', response
+ assert response.get('success') is False and isinstance(response.get('error'), str), response
+ actual = response['error']
+ else:
+ raise AssertionError('refine must report the fixture failure')
+ assert len(server.requests) == 1, server.requests
+ receipt.update(actual_error=actual, ts_observed_error=TS_OBSERVED[case],
+ parity_equal=actual == TS_OBSERVED[case])
+ if case == 'length':
+ body = server.requests[0]['body']
+ budget = body.get('max_tokens', body.get('max_completion_tokens'))
+ assert isinstance(budget, int) and budget > 0, body
+ receipt['requested_output_tokens'] = budget
+ assert actual == ('Refinement failed: the model hit its output limit '
+ f'({budget} tokens, prime-inference/parity-1)'), actual
+ elif case == 'provider_error':
+ assert actual.startswith('Refinement failed: ') \
+ and 'fixture token budget invalid' in actual, actual
+ else:
+ assert actual == 'Refiner did not return a JSON object', actual
+ receipt['behavior_verified'] = True
+ finally:
+ receipt['http_requests'] = server.requests
+ receipt['http_headers'] = server.received_headers
+ receipt['raw_daemon_sent_hex'] = list(client.sent) if client else []
+ receipt['raw_daemon_received_hex'] = list(client.received) if client else []
+ finish(scope, client, process, receipt)
+ server.shutdown()
+ server.server_close()
+ thread.join(timeout=2)
+ receipt['provider_stopped'] = not thread.is_alive()
+ log.flush()
+ receipt['daemon_log'] = (root / 'daemon.log').read_text(errors='replace')[-3000:]
+
+
+def main():
+ parser = argparse.ArgumentParser(description=__doc__)
+ parser.add_argument('--receipt', required=True, type=Path)
+ args = parser.parse_args()
+ receipt = {'verified': False, 'scope': __doc__, 'cases': {},
+ 'ts_reference': 'v0.9.8 / 7d442aafa985f9342134fac16c2ef41f03fb45c1',
+ 'ts_binary_sha256': 'e50d4e4cf5735ea16cee30c3ca48fccb628ae2169fc867c33d461b8a90e87bc5',
+ 'not_verified': ['auto-review HTTP execution', 'TUI frames',
+ 'full daemon wire-byte equality', 'live Prime Inference']}
+ try:
+ binary = Path(os.environ['PA_RUST_BINARY']).resolve()
+ receipt['binary_sha256'] = digest(binary)
+ with tempfile.TemporaryDirectory(prefix='pa-refine-errors-', dir='/tmp') as folder:
+ for case in TS_OBSERVED:
+ observed = receipt['cases'][case] = {}
+ try:
+ observe(binary, Path(folder) / case, case, observed)
+ except Exception as error:
+ observed['error'] = str(error)
+ receipt['verified'] = all(c.get('behavior_verified') and c.get('cleanup_verified')
+ and c.get('provider_stopped') and not c.get('error')
+ for c in receipt['cases'].values())
+ receipt['parity_equal'] = all(c.get('parity_equal') is True
+ for c in receipt['cases'].values())
+ if not receipt['verified']:
+ receipt['error'] = 'one or more output controls failed; see cases'
+ except Exception as error:
+ receipt['error'] = str(error)
+ args.receipt.parent.mkdir(parents=True, exist_ok=True)
+ args.receipt.write_text(json.dumps(receipt, indent=2))
+ print(json.dumps({'verified': receipt['verified'], 'error': receipt.get('error')}))
+ return 0 if receipt['verified'] else 1
+
+
+if __name__ == '__main__':
+ raise SystemExit(main())