Skip to content

Solution for long term memory for agent coding CLIs and to facilitate handoff between different agent vendors

Rust

fix(llm): stop retrying a Codex output-budget stop as a transient 502

Rudy Mizrahi Celekli77861b5b8f03
diff --git a/crates/ai-memory-llm/src/codex_responses.rs b/crates/ai-memory-llm/src/codex_responses.rs
index f5c07a5b..3b8504c0 100644
--- a/crates/ai-memory-llm/src/codex_responses.rs
+++ b/crates/ai-memory-llm/src/codex_responses.rs
@@ -57,7 +57,7 @@ pub(crate) async fn post_codex_responses(
});
}
if body.stream {
- parse_sse_response(&response_text_limited(response).await?)
+ parse_sse_response(&response_text_limited(response).await?, body.model)
} else {
response_json_limited::<CodexResponsesResponse>(response).await
}
diff --git a/crates/ai-memory-llm/src/openai_oauth.rs b/crates/ai-memory-llm/src/openai_oauth.rs
index a9951d9d..90bb61df 100644
--- a/crates/ai-memory-llm/src/openai_oauth.rs
+++ b/crates/ai-memory-llm/src/openai_oauth.rs
@@ -42,8 +42,9 @@ pub const CODEX_CLIENT_ID: &str = "app_EMoamEEZ73f0CkXaXp7hrann";
pub const OAUTH_SCOPES: &str = "openid profile email offline_access";
/// Status code used when the Codex SSE stream carries a terminal error
-/// event (`response.failed`/`response.incomplete`/`response.cancelled`
-/// or a top-level `error` payload). 502 ("Bad Gateway") models the
+/// event (`response.failed`/`response.cancelled`, a
+/// `response.incomplete` the output budget does not explain, or a
+/// top-level `error` payload). 502 ("Bad Gateway") models the
/// situation accurately: the upstream connected, but the response it
/// sent was not a usable completion — the Codex backend, not
/// ai-memory's HTTP layer, produced the failure.
@@ -333,7 +334,7 @@ pub(crate) fn build_request<'a>(
}
}
-pub(crate) fn parse_sse_response(body: &str) -> LlmResult<CodexResponsesResponse> {
+pub(crate) fn parse_sse_response(body: &str, model: &str) -> LlmResult<CodexResponsesResponse> {
let mut current_event: Option<String> = None;
let mut data_lines: Vec<&str> = Vec::new();
let mut output_text = String::new();
@@ -378,6 +379,27 @@ pub(crate) fn parse_sse_response(body: &str) -> LlmResult<CodexResponsesResponse
let response = value.get("response").cloned().unwrap_or(value);
completed = Some(serde_json::from_value(response)?);
}
+ // An `incomplete` stop for `max_output_tokens` is an output-budget
+ // stop, not a provider fault, and the 502 below is transient: the
+ // fallback chain and the SessionEnd queue would re-send the same
+ // expensive prompt only to have it truncated again at the same
+ // budget. The OpenAI and Copilot paths already report this stop as
+ // `TruncatedResponse`, which is terminal. Every other non-completed
+ // status keeps the 502 path.
+ "response.incomplete"
+ if value
+ .pointer("/response/incomplete_details/reason")
+ .and_then(serde_json::Value::as_str)
+ == Some("max_output_tokens") =>
+ {
+ return Err(LlmError::TruncatedResponse {
+ model: model.to_owned(),
+ completion_tokens: value
+ .pointer("/response/usage/output_tokens")
+ .and_then(serde_json::Value::as_u64)
+ .and_then(|tokens| u32::try_from(tokens).ok()),
+ });
+ }
"response.failed" | "response.incomplete" | "response.cancelled" | "error" => {
return Err(LlmError::Provider {
status: SSE_TERMINAL_ERROR_STATUS,
@@ -443,7 +465,7 @@ pub(crate) fn model_uses_default_temperature(model: &str) -> bool {
#[derive(Debug, Serialize)]
pub(crate) struct CodexResponsesRequest<'a> {
- model: &'a str,
+ pub(crate) model: &'a str,
#[serde(skip_serializing_if = "Option::is_none")]
instructions: Option<&'a str>,
input: Vec<CodexInputMessage<'a>>,
@@ -773,7 +795,7 @@ mod tests {
"data: [DONE]\n\n"
);
- let response = parse_sse_response(body).unwrap();
+ let response = parse_sse_response(body, "gpt-5.5").unwrap();
assert_eq!(response.model.as_deref(), Some("gpt-5.5"));
assert_eq!(response.output_text.as_deref(), Some("hello"));
let usage = response.usage.expect("usage from completed event");
@@ -783,18 +805,13 @@ mod tests {
#[test]
fn parse_sse_response_surfaces_terminal_failure_events() {
- for event in [
- "response.failed",
- "response.incomplete",
- "response.cancelled",
- "error",
- ] {
+ for event in ["response.failed", "response.cancelled", "error"] {
let body = format!(
"event: {event}\n\
data: {{\"type\":\"{event}\",\"error\":{{\"message\":\"stream stopped\"}}}}\n\n"
);
- let err = parse_sse_response(&body).expect_err("terminal event must fail");
+ let err = parse_sse_response(&body, "gpt-5.5").expect_err("terminal event must fail");
match err {
LlmError::Provider { status, body } => {
assert_eq!(status, 502);
@@ -809,6 +826,62 @@ mod tests {
}
}
+ /// A `response.incomplete` stop whose `incomplete_details.reason` is
+ /// `max_output_tokens` is an output-budget stop, so it must land in the
+ /// terminal `truncated-response` class rather than the transient 502 that
+ /// `is_transient()` tells the fallback chain and the SessionEnd queue to
+ /// re-send. `incomplete` for any other reason stays on the 502 path (the
+ /// control below).
+ #[test]
+ fn parse_sse_response_maps_incomplete_to_truncated_response() {
+ let body = concat!(
+ "event: response.incomplete\n",
+ "data: {\"type\":\"response.incomplete\",\"response\":{\"status\":\"incomplete\",",
+ "\"model\":\"gpt-5.5-wire\",\"usage\":{\"input_tokens\":12,\"output_tokens\":16000},",
+ "\"incomplete_details\":{\"reason\":\"max_output_tokens\"}}}\n\n"
+ );
+
+ let err = parse_sse_response(body, "gpt-5.5").expect_err("a budget stop must not parse");
+ match &err {
+ LlmError::TruncatedResponse {
+ model,
+ completion_tokens,
+ } => {
+ assert_eq!(
+ model, "gpt-5.5",
+ "the configured model label, not the wire one"
+ );
+ assert_eq!(*completion_tokens, Some(16_000));
+ }
+ other => panic!("expected TruncatedResponse, got {other:?}"),
+ }
+ assert_eq!(err.class(), "truncated-response");
+ assert!(!err.is_transient(), "a budget stop must not be retried");
+ }
+
+ /// Control for the arm above: a `response.incomplete` the output budget
+ /// does not explain keeps the terminal 502 path.
+ #[test]
+ fn parse_sse_response_keeps_non_budget_incomplete_on_provider_path() {
+ let body = concat!(
+ "event: response.incomplete\n",
+ "data: {\"type\":\"response.incomplete\",\"response\":{\"status\":\"incomplete\",",
+ "\"incomplete_details\":{\"reason\":\"content_filter\"}}}\n\n"
+ );
+
+ let err = parse_sse_response(body, "gpt-5.5").expect_err("terminal event must fail");
+ match err {
+ LlmError::Provider { status, body } => {
+ assert_eq!(status, SSE_TERMINAL_ERROR_STATUS);
+ assert!(
+ body.contains("content_filter"),
+ "body should carry the reason: {body}"
+ );
+ }
+ other => panic!("expected Provider error, got {other:?}"),
+ }
+ }
+
#[test]
fn parse_sse_response_requires_completed_event() {
let body = concat!(
@@ -817,7 +890,8 @@ mod tests {
"data: [DONE]\n\n",
);
- let err = parse_sse_response(body).expect_err("missing completed event must fail");
+ let err =
+ parse_sse_response(body, "gpt-5.5").expect_err("missing completed event must fail");
assert!(
err.to_string()
.contains("stream closed before response.completed"),
@@ -833,7 +907,7 @@ mod tests {
#[test]
fn parse_sse_response_promotes_top_level_error_without_event_header() {
let body = "data: {\"error\":{\"message\":\"context length exceeded\"}}\n\n";
- let err = parse_sse_response(body)
+ let err = parse_sse_response(body, "gpt-5.5")
.expect_err("top-level error payload must surface as a terminal error");
match err {
LlmError::Provider { status, body } => {
@@ -857,7 +931,7 @@ mod tests {
// `data:` line carrying an unterminated JSON object, no blank
// line after — the post-loop flush is what runs.
let body = "data: {\"type\":\"response.output_text.delta\",\"delta\":\"hel";
- let err = parse_sse_response(body).expect_err("truncated stream must fail");
+ let err = parse_sse_response(body, "gpt-5.5").expect_err("truncated stream must fail");
assert!(
err.to_string()
.contains("stream truncated before final event closed"),
@@ -870,7 +944,7 @@ mod tests {
/// "stream closed before response.completed" error.
#[test]
fn parse_sse_response_empty_body_reports_no_completed_event() {
- let err = parse_sse_response("").expect_err("empty body must fail");
+ let err = parse_sse_response("", "gpt-5.5").expect_err("empty body must fail");
assert!(
err.to_string()
.contains("stream closed before response.completed"),
@@ -893,7 +967,7 @@ mod tests {
"data: {\"type\":\"response.completed\",\"response\":{\"model\":\"gpt-5.5\",\"output_text\":\"line1\\nline2\",\"output\":[]}}\n\n",
"data: [DONE]\n\n",
);
- let resp = parse_sse_response(body).expect("multi-line content must parse");
+ let resp = parse_sse_response(body, "gpt-5.5").expect("multi-line content must parse");
assert_eq!(resp.output_text.as_deref(), Some("line1\nline2"));
}