Skip to content

Use Claude Code with your ChatGPT, Kimi, Cursor or Grok subscription via a local Anthropic-compatible proxy

Rust

retry empty buffered codex completions (#71)

Kunaiidd890622a38f
diff --git a/src/providers/codex/mod.rs b/src/providers/codex/mod.rs
index fdf1580..5a3289d 100644
--- a/src/providers/codex/mod.rs
+++ b/src/providers/codex/mod.rs
@@ -51,6 +51,8 @@ use self::translate::request::{
};
const MAX_RETRYABLE_LIVE_STREAM_RETRIES: u32 = 10;
+const MAX_EMPTY_COMPLETION_RETRIES: u32 = 10;
+const EMPTY_CODEX_COMPLETION_DETAIL: &str = "empty_codex_completion";
use self::translate::stream::translate_stream_bytes_with_traffic;
// ---------------------------------------------------------------------------
@@ -226,16 +228,44 @@ impl Provider for CodexProvider {
.await;
}
- let upstream = match client
- .post_codex(&translated, &ctx, Some(&continuation))
- .await
- {
- Ok(r) => r,
- Err(e) => {
+ let mut continuation = Some(continuation);
+ let mut attempt = 0_u32;
+ let upstream = loop {
+ let response = match client
+ .post_codex(&translated, &ctx, continuation.as_ref())
+ .await
+ {
+ Ok(r) => r,
+ Err(e) => {
+ abort_compaction_attempt(
+ ctx.session_id.as_deref(),
+ compact_boundary,
+ &translated,
+ );
+ abort_continuation(ctx.session_id.as_deref(), turn_id);
+ return map_codex_error_to_response(&e);
+ }
+ };
+ if !is_empty_codex_success_completion(&response.body) {
+ break response;
+ }
+ // A successful terminal event with no output would translate into
+ // an empty end_turn; retry with full context instead.
+ let error = empty_buffered_completion_error();
+ drop_live_continuation_for_retry(&mut continuation);
+ if attempt >= MAX_EMPTY_COMPLETION_RETRIES {
+ abort_compaction_attempt(ctx.session_id.as_deref(), compact_boundary, &translated);
+ abort_continuation(ctx.session_id.as_deref(), turn_id);
+ return map_codex_error_to_response(&error);
+ }
+ let delay = compute_backoff_delay(attempt, None);
+ if delay.exceeds_budget {
abort_compaction_attempt(ctx.session_id.as_deref(), compact_boundary, &translated);
abort_continuation(ctx.session_id.as_deref(), turn_id);
- return map_codex_error_to_response(&e);
+ return map_codex_error_to_response(&error);
}
+ attempt += 1;
+ sleep(delay.wait_ms).await;
};
if want_stream {
@@ -842,6 +872,45 @@ where
(headers, Body::from_stream(stream)).into_response()
}
+fn empty_buffered_completion_error() -> client::CodexError {
+ client::CodexError {
+ status: 503,
+ message: "Codex completed without producing output".to_string(),
+ detail: Some(EMPTY_CODEX_COMPLETION_DETAIL.to_string()),
+ retry_after: None,
+ origin: match config::codex_transport() {
+ config::CodexTransport::Http => client::CodexErrorOrigin::BufferedHttp,
+ _ => client::CodexErrorOrigin::BufferedWebSocket,
+ },
+ }
+}
+
+/// True when the buffered upstream body ended in a successful terminal event
+/// without ever producing semantic output (text, thinking, tool, web search).
+fn is_empty_codex_success_completion(upstream_sse: &[u8]) -> bool {
+ use self::translate::reducer::{ReducerEvent, TERM_COMPLETED, TERM_DONE};
+
+ let Ok(events) = self::translate::reducer::reduce_upstream_bytes(upstream_sse) else {
+ return false;
+ };
+ let mut saw_success_terminal = false;
+ for event in &events {
+ match event {
+ ReducerEvent::TextDelta { text, .. } if !text.is_empty() => return false,
+ ReducerEvent::ThinkingStart { .. }
+ | ReducerEvent::ToolStart { .. }
+ | ReducerEvent::WebSearch { .. } => return false,
+ ReducerEvent::Finish { terminal_type, .. }
+ if terminal_type == TERM_COMPLETED || terminal_type == TERM_DONE =>
+ {
+ saw_success_terminal = true;
+ }
+ _ => {}
+ }
+ }
+ saw_success_terminal
+}
+
fn is_codex_terminal_event(payload: &serde_json::Value) -> bool {
matches!(
payload.get("type").and_then(|v| v.as_str()),
@@ -956,6 +1025,9 @@ fn map_codex_error_to_response(err: &client::CodexError) -> Response {
if is_context_window_overflow(message) {
return map_codex_failure_to_response(message);
}
+ if err.detail.as_deref() == Some(EMPTY_CODEX_COMPLETION_DETAIL) {
+ return json_error(StatusCode::SERVICE_UNAVAILABLE, "api_error", &err.message);
+ }
match err.status {
401 => json_error(
@@ -1128,6 +1200,114 @@ fn format_auth_saved_output(auth_path: &str, account_id: Option<&str>) -> String
mod tests {
use super::*;
+ fn upstream_sse(events: &[serde_json::Value]) -> Vec<u8> {
+ let mut bytes = Vec::new();
+ for event in events {
+ bytes.extend_from_slice(format!("data: {event}\n\n").as_bytes());
+ }
+ bytes
+ }
+
+ #[test]
+ fn terminal_only_completed_upstream_is_empty_completion() {
+ let body = upstream_sse(&[serde_json::json!({
+ "type": "response.completed",
+ "response": {"id": "resp_1", "status": "completed", "incomplete_details": null, "usage": {"input_tokens": 5, "output_tokens": 0}}
+ })]);
+ assert!(is_empty_codex_success_completion(&body));
+ }
+
+ #[test]
+ fn terminal_only_done_upstream_is_empty_completion() {
+ let body = upstream_sse(&[serde_json::json!({
+ "type": "response.done",
+ "response": {"id": "resp_1", "usage": {}}
+ })]);
+ assert!(is_empty_codex_success_completion(&body));
+ }
+
+ #[test]
+ fn empty_message_item_is_empty_completion() {
+ let body = upstream_sse(&[
+ serde_json::json!({
+ "type": "response.output_item.added",
+ "output_index": 0,
+ "item": {"type": "message", "id": "msg_1"}
+ }),
+ serde_json::json!({
+ "type": "response.output_item.done",
+ "output_index": 0,
+ "item": {"type": "message"}
+ }),
+ serde_json::json!({
+ "type": "response.completed",
+ "response": {"id": "resp_1", "usage": {}}
+ }),
+ ]);
+ assert!(is_empty_codex_success_completion(&body));
+ }
+
+ #[test]
+ fn upstream_with_text_is_not_empty_completion() {
+ let body = upstream_sse(&[
+ serde_json::json!({
+ "type": "response.output_item.added",
+ "output_index": 0,
+ "item": {"type": "message", "id": "msg_1"}
+ }),
+ serde_json::json!({
+ "type": "response.output_text.delta",
+ "output_index": 0,
+ "delta": "hello"
+ }),
+ serde_json::json!({
+ "type": "response.output_item.done",
+ "output_index": 0,
+ "item": {"type": "message"}
+ }),
+ serde_json::json!({
+ "type": "response.completed",
+ "response": {"id": "resp_1", "usage": {}}
+ }),
+ ]);
+ assert!(!is_empty_codex_success_completion(&body));
+ }
+
+ #[test]
+ fn upstream_with_tool_call_is_not_empty_completion() {
+ let body = upstream_sse(&[
+ serde_json::json!({
+ "type": "response.output_item.added",
+ "output_index": 0,
+ "item": {"type": "function_call", "call_id": "call_1", "name": "Read", "arguments": ""}
+ }),
+ serde_json::json!({
+ "type": "response.output_item.done",
+ "output_index": 0,
+ "item": {"type": "function_call", "call_id": "call_1", "name": "Read", "arguments": "{}"}
+ }),
+ serde_json::json!({
+ "type": "response.completed",
+ "response": {"id": "resp_1", "usage": {}}
+ }),
+ ]);
+ assert!(!is_empty_codex_success_completion(&body));
+ }
+
+ #[test]
+ fn terminal_only_incomplete_upstream_is_not_empty_completion() {
+ let body = upstream_sse(&[serde_json::json!({
+ "type": "response.incomplete",
+ "response": {"id": "resp_1", "incomplete_details": {"reason": "max_output_tokens"}, "usage": {}}
+ })]);
+ assert!(!is_empty_codex_success_completion(&body));
+ }
+
+ #[test]
+ fn upstream_without_terminal_event_is_not_empty_completion() {
+ assert!(!is_empty_codex_success_completion(&upstream_sse(&[])));
+ }
+
fn request_with_tools(tools: serde_json::Value) -> MessagesRequest {
serde_json::from_value(serde_json::json!({
"model": "gpt-5.6-luna",
diff --git a/src/retry.rs b/src/retry.rs
index 48b6196..7060368 100644
--- a/src/retry.rs
+++ b/src/retry.rs
@@ -1,3 +1,4 @@
+use std::sync::atomic::{AtomicBool, Ordering};
use std::time::Duration;
pub const RETRY_INITIAL_DELAY_MS: u64 = 2000;
@@ -39,7 +40,18 @@ pub fn compute_backoff_delay(attempt: u32, retry_after: Option<&str>) -> Backoff
}
}
+static ZERO_RETRY_DELAY_FOR_TESTS: AtomicBool = AtomicBool::new(false);
+
+/// Make retry sleeps return immediately so exhaustion paths can be exercised
+/// in tests without waiting out the real backoff schedule.
+pub fn set_zero_retry_delay_for_tests(enabled: bool) {
+ ZERO_RETRY_DELAY_FOR_TESTS.store(enabled, Ordering::SeqCst);
+}
+
pub async fn sleep(ms: u64) {
+ if ZERO_RETRY_DELAY_FOR_TESTS.load(Ordering::SeqCst) {
+ return;
+ }
tokio::time::sleep(Duration::from_millis(ms)).await;
}
diff --git a/tests/smoke_cutover.rs b/tests/smoke_cutover.rs
index d0a6aeb..2055b75 100644
--- a/tests/smoke_cutover.rs
+++ b/tests/smoke_cutover.rs
@@ -693,6 +693,235 @@ async fn smoke_codex_http_messages_uses_mock_upstream() {
assert_eq!(sent["stream"], true);
}
+/// Resets the retry-delay override even when the test panics, so later tests
+/// in this process keep real backoff behavior.
+struct ZeroRetryDelayGuard;
+
+impl ZeroRetryDelayGuard {
+ fn enable() -> Self {
+ claude_code_proxy::retry::set_zero_retry_delay_for_tests(true);
+ ZeroRetryDelayGuard
+ }
+}
+
+impl Drop for ZeroRetryDelayGuard {
+ fn drop(&mut self) {
+ claude_code_proxy::retry::set_zero_retry_delay_for_tests(false);
+ }
+}
+
+fn empty_completion_sse() -> Vec<u8> {
+ concat!(
+ "data: {\"type\":\"response.completed\",\"response\":{\"id\":\"resp_empty\",",
+ "\"status\":\"completed\",\"incomplete_details\":null,",
+ "\"usage\":{\"input_tokens\":5,\"output_tokens\":0}}}\n\n"
+ )
+ .as_bytes()
+ .to_vec()
+}
+
+fn empty_message_completion_sse() -> Vec<u8> {
+ concat!(
+ "data: {\"type\":\"response.output_item.added\",\"output_index\":0,",
+ "\"item\":{\"type\":\"message\",\"id\":\"msg_empty\"}}\n\n",
+ "data: {\"type\":\"response.output_item.done\",\"output_index\":0,",
+ "\"item\":{\"type\":\"message\"}}\n\n",
+ "data: {\"type\":\"response.completed\",\"response\":{\"id\":\"resp_empty\",",
+ "\"status\":\"completed\",\"incomplete_details\":null,",
+ "\"usage\":{\"input_tokens\":5,\"output_tokens\":0}}}\n\n"
+ )
+ .as_bytes()
+ .to_vec()
+}
+
+fn buffered_success_sse(text: &str) -> Vec<u8> {
+ format!(
+ concat!(
+ "data: {{\"type\":\"response.output_item.added\",\"output_index\":0,\"item\":{{\"type\":\"message\",\"id\":\"msg_up\"}}}}\n\n",
+ "data: {{\"type\":\"response.output_text.delta\",\"output_index\":0,\"delta\":\"{text}\"}}\n\n",
+ "data: {{\"type\":\"response.output_item.done\",\"output_index\":0,\"item\":{{\"type\":\"message\"}}}}\n\n",
+ "data: {{\"type\":\"response.completed\",\"response\":{{\"id\":\"resp_1\",\"usage\":{{\"input_tokens\":5,\"output_tokens\":2}}}}}}\n\n"
+ ),
+ text = text
+ )
+ .into_bytes()
+}
+
+#[allow(clippy::await_holding_lock)]
+#[tokio::test]
+async fn smoke_codex_http_retries_empty_completion() {
+ let _guard = env_lock();
+ let _delay_guard = ZeroRetryDelayGuard::enable();
+ let config = TempDir::new().unwrap();
+ write_auth(config.path(), "codex");
+
+ let attempts = Arc::new(std::sync::atomic::AtomicUsize::new(0));
+ let upstream = spawn_http_upstream({
+ let attempts = attempts.clone();
+ move |_body: Value| {
+ let attempt = attempts.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
+ if attempt == 0 {
+ empty_completion_sse()
+ } else {
+ buffered_success_sse("buffered retry ok")
+ }
+ }
+ })
+ .await;
+
+ let _config_env = EnvGuard::set("CCP_CONFIG_DIR", config.path());
+ let _base_url_env = EnvGuard::set("CCP_CODEX_BASE_URL", &upstream);
+ let _transport_env = EnvGuard::set("CCP_CODEX_TRANSPORT", "http");
+
+ let response = call_messages("gpt-5.5").await;
+ let status = response.status();
+ let body = axum::body::to_bytes(response.into_body(), usize::MAX)
+ .await
+ .unwrap();
+ let body_text = String::from_utf8_lossy(&body);
+
+ assert_eq!(status, StatusCode::OK, "body: {body_text}");
+ let value: Value = serde_json::from_slice(&body).unwrap();
+ assert_eq!(value["content"][0]["text"], "buffered retry ok");
+ assert_eq!(
+ attempts.load(std::sync::atomic::Ordering::SeqCst),
+ 2,
+ "empty completion must trigger one retry"
+ );
+}
+
+#[allow(clippy::await_holding_lock)]
+#[tokio::test]
+async fn smoke_codex_http_retries_empty_message_completion() {
+ let _guard = env_lock();
+ let _delay_guard = ZeroRetryDelayGuard::enable();
+ let config = TempDir::new().unwrap();
+ write_auth(config.path(), "codex");
+
+ let attempts = Arc::new(std::sync::atomic::AtomicUsize::new(0));
+ let upstream = spawn_http_upstream({
+ let attempts = attempts.clone();
+ move |_body: Value| {
+ let attempt = attempts.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
+ if attempt == 0 {
+ empty_message_completion_sse()
+ } else {
+ buffered_success_sse("empty message retry ok")
+ }
+ }
+ })
+ .await;
+
+ let _config_env = EnvGuard::set("CCP_CONFIG_DIR", config.path());
+ let _base_url_env = EnvGuard::set("CCP_CODEX_BASE_URL", &upstream);
+ let _transport_env = EnvGuard::set("CCP_CODEX_TRANSPORT", "http");
+
+ let response = call_messages("gpt-5.5").await;
+ let status = response.status();
+ let body = axum::body::to_bytes(response.into_body(), usize::MAX)
+ .await
+ .unwrap();
+ let body_text = String::from_utf8_lossy(&body);
+
+ assert_eq!(status, StatusCode::OK, "body: {body_text}");
+ let value: Value = serde_json::from_slice(&body).unwrap();
+ assert_eq!(value["content"][0]["text"], "empty message retry ok");
+ assert_eq!(attempts.load(std::sync::atomic::Ordering::SeqCst), 2);
+}
+
+#[allow(clippy::await_holding_lock)]
+#[tokio::test]
+async fn smoke_codex_http_stream_retries_empty_completion() {
+ let _guard = env_lock();
+ let _delay_guard = ZeroRetryDelayGuard::enable();
+ let config = TempDir::new().unwrap();
+ write_auth(config.path(), "codex");
+
+ let attempts = Arc::new(std::sync::atomic::AtomicUsize::new(0));
+ let upstream = spawn_http_upstream({
+ let attempts = attempts.clone();
+ move |_body: Value| {
+ let attempt = attempts.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
+ if attempt == 0 {
+ empty_completion_sse()
+ } else {
+ buffered_success_sse("buffered stream retry ok")
+ }
+ }
+ })
+ .await;
+
+ let _config_env = EnvGuard::set("CCP_CONFIG_DIR", config.path());
+ let _base_url_env = EnvGuard::set("CCP_CODEX_BASE_URL", &upstream);
+ let _transport_env = EnvGuard::set("CCP_CODEX_TRANSPORT", "http");
+
+ let response = call_messages_body(json!({
+ "model": "gpt-5.5",
+ "max_tokens": 64,
+ "stream": true,
+ "messages": [{"role":"user","content":"one"}]
+ }))
+ .await;
+ let status = response.status();
+ let body = axum::body::to_bytes(response.into_body(), usize::MAX)
+ .await
+ .unwrap();
+ let body_text = String::from_utf8_lossy(&body);
+
+ assert_eq!(status, StatusCode::OK, "body: {body_text}");
+ assert!(
+ body_text.contains("buffered stream retry ok"),
+ "expected retried text in SSE body: {body_text}"
+ );
+ assert_eq!(attempts.load(std::sync::atomic::Ordering::SeqCst), 2);
+}
+
+#[allow(clippy::await_holding_lock)]
+#[tokio::test]
+async fn smoke_codex_http_empty_completions_exhaust_to_service_unavailable() {
+ let _guard = env_lock();
+ let _delay_guard = ZeroRetryDelayGuard::enable();
+ let config = TempDir::new().unwrap();
+ write_auth(config.path(), "codex");
+
+ let attempts = Arc::new(std::sync::atomic::AtomicUsize::new(0));
+ let upstream = spawn_http_upstream({
+ let attempts = attempts.clone();
+ move |_body: Value| {
+ attempts.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
+ empty_completion_sse()
+ }
+ })
+ .await;
+
+ let _config_env = EnvGuard::set("CCP_CONFIG_DIR", config.path());
+ let _base_url_env = EnvGuard::set("CCP_CODEX_BASE_URL", &upstream);
+ let _transport_env = EnvGuard::set("CCP_CODEX_TRANSPORT", "http");
+
+ let response = call_messages("gpt-5.5").await;
+ let status = response.status();
+ let body = axum::body::to_bytes(response.into_body(), usize::MAX)
+ .await
+ .unwrap();
+ let body_text = String::from_utf8_lossy(&body);
+
+ assert_eq!(
+ status,
+ StatusCode::SERVICE_UNAVAILABLE,
+ "exhausted empty completions must surface an explicit error: {body_text}"
+ );
+ assert!(
+ body_text.contains("Codex completed without producing output"),
+ "unexpected exhaustion body: {body_text}"
+ );
+ // Initial attempt plus MAX_EMPTY_COMPLETION_RETRIES retries.
+ assert_eq!(
+ attempts.load(std::sync::atomic::Ordering::SeqCst),
+ 11,
+ "retry loop must stay bounded"
+ );
+}
+
#[allow(clippy::await_holding_lock)]
#[tokio::test]
async fn smoke_codex_http_server_compaction_replays_native_history() {