Skip to content

Use Claude Code with your ChatGPT, Kimi, Cursor or Grok subscription via a local Anthropic-compatible proxy

Rust

retry codex overload stream errors

Raine Virtac97e25f5d042
diff --git a/src/providers/codex/index.ts b/src/providers/codex/index.ts
index 206fa73..6569e48 100644
--- a/src/providers/codex/index.ts
+++ b/src/providers/codex/index.ts
@@ -327,6 +327,14 @@ async function handleMessages(body: AnthropicRequest, ctx: RequestContext): Prom
upstreamHeaders: upstream.headers,
traffic: ctx.traffic,
requestSize,
+ retryUpstream: async () => {
+ const retry = await postCodex(translated, ctx, { continuation });
+ return {
+ body: retry.body,
+ headers: retry.headers,
+ requestSize,
+ };
+ },
onFinish: (finish) => {
if (finish.continuationEligible) {
recordContinuation(ctx.sessionId, translated, finish.responseId, finish.outputItems);
diff --git a/src/providers/codex/translate/reducer.ts b/src/providers/codex/translate/reducer.ts
index 62f86c8..773fc73 100644
--- a/src/providers/codex/translate/reducer.ts
+++ b/src/providers/codex/translate/reducer.ts
@@ -9,7 +9,7 @@ import { serverToolUseIdFromCodexWebSearchId } from "./web-search-compat.ts";
export class UpstreamStreamError extends Error {
constructor(
- public kind: "rate_limit" | "failed",
+ public kind: "rate_limit" | "overloaded" | "failed",
message: string,
public retryAfterSeconds?: number,
) {
@@ -389,7 +389,11 @@ export async function* reduceUpstream(
}
if (t === "response.failed" || t === "response.error" || t === "error") {
const message = p?.response?.error?.message || p?.error?.message || "Upstream error";
- throw new UpstreamStreamError("failed", message);
+ throw new UpstreamStreamError(
+ upstreamFailureKind(p, message),
+ message,
+ retryAfterSecondsFromPayload(p),
+ );
}
if (t === "response.output_item.added") {
@@ -653,6 +657,47 @@ function webSearchQuery(item: unknown): string {
return "";
}
+function upstreamFailureKind(
+ payload: {
+ status?: unknown;
+ status_code?: unknown;
+ response?: { error?: { code?: unknown; type?: unknown } };
+ error?: { code?: unknown; type?: unknown };
+ },
+ message: string,
+): "overloaded" | "failed" {
+ const status = payload.status ?? payload.status_code;
+ const code = payload.response?.error?.code ?? payload.error?.code;
+ const type = payload.response?.error?.type ?? payload.error?.type;
+ const lowerMessage = message.toLowerCase();
+ if (
+ status === 529 ||
+ status === "529" ||
+ code === "overloaded_error" ||
+ type === "overloaded_error" ||
+ lowerMessage.includes("overloaded")
+ ) {
+ return "overloaded";
+ }
+ return "failed";
+}
+
+function retryAfterSecondsFromPayload(payload: {
+ retry_after_seconds?: unknown;
+ headers?: Record<string, unknown>;
+ response?: { error?: { retry_after_seconds?: unknown } };
+ error?: { retry_after_seconds?: unknown };
+}): number | undefined {
+ const raw =
+ payload.response?.error?.retry_after_seconds ??
+ payload.error?.retry_after_seconds ??
+ payload.retry_after_seconds ??
+ payload.headers?.["retry-after"] ??
+ payload.headers?.["Retry-After"];
+ const value = typeof raw === "number" ? raw : typeof raw === "string" ? Number(raw) : NaN;
+ return Number.isFinite(value) && value >= 0 ? value : undefined;
+}
+
function isCodexWebSocketCloseError(err: unknown): boolean {
const message = err instanceof Error ? err.message : String(err);
return (
diff --git a/src/providers/codex/translate/stream.test.ts b/src/providers/codex/translate/stream.test.ts
index 8ca65de..5cd71d6 100644
--- a/src/providers/codex/translate/stream.test.ts
+++ b/src/providers/codex/translate/stream.test.ts
@@ -117,7 +117,90 @@ function webSearchChunks(): string[] {
];
}
+function textChunks(text: string): string[] {
+ return [
+ sse("response.output_item.added", {
+ output_index: 0,
+ item: { type: "message", id: "msg_upstream" },
+ }),
+ sse("response.output_text.delta", {
+ output_index: 0,
+ delta: text,
+ }),
+ sse("response.output_item.done", {
+ output_index: 0,
+ item: { type: "message", id: "msg_upstream" },
+ }),
+ sse("response.completed", { response: { usage: { input_tokens: 10, output_tokens: 5 } } }),
+ ];
+}
+
+function overloadChunk(): string {
+ return sse("error", {
+ status: 529,
+ error: {
+ type: "overloaded_error",
+ message: "Our servers are currently overloaded. Please try again later.",
+ },
+ });
+}
+
describe("translateStream", () => {
+ it("retries overloaded stream errors before downstream output starts", async () => {
+ let retryCalls = 0;
+ const logs: Array<{ msg: string; fields: unknown }> = [];
+ const log = {
+ ...silentLog,
+ warn: (msg: string, fields: unknown) => logs.push({ msg, fields }),
+ };
+
+ const output = await collect(
+ translateStream(upstreamFromChunks([overloadChunk()]), {
+ messageId: "msg_1",
+ model: "gpt-5.5",
+ log,
+ retryUpstream: async () => {
+ retryCalls++;
+ return { body: upstreamFromChunks(textChunks("Recovered")) };
+ },
+ computeRetryDelay: () => ({ waitMs: 0, exceedsBudget: false }),
+ }),
+ );
+
+ expect(retryCalls).toBe(1);
+ expect(output).toContain("Recovered");
+ expect(output).toContain("event: message_stop");
+ expect(output).not.toContain("event: error");
+ expect(
+ logs.some(
+ (entry) => entry.msg === "upstream stream error before downstream output, retrying",
+ ),
+ ).toBe(true);
+ });
+
+ it("does not retry overloaded stream errors after downstream output starts", async () => {
+ let retryCalls = 0;
+
+ const output = await collect(
+ translateStream(upstreamFromChunks([...textChunks("Partial").slice(0, 2), overloadChunk()]), {
+ messageId: "msg_1",
+ model: "gpt-5.5",
+ log: silentLog,
+ retryUpstream: async () => {
+ retryCalls++;
+ return { body: upstreamFromChunks(textChunks("Recovered")) };
+ },
+ computeRetryDelay: () => ({ waitMs: 0, exceedsBudget: false }),
+ }),
+ );
+
+ expect(retryCalls).toBe(0);
+ expect(output).toContain("Partial");
+ expect(output).toContain("event: content_block_stop");
+ expect(output).toContain("event: error");
+ expect(output).toContain('"type":"overloaded_error"');
+ });
+
it("emits keepalive pings while Read arguments are buffered", async () => {
const chunks = [
...readChunks(['{"file_path"', '":"/tmp/a"}']),
diff --git a/src/providers/codex/translate/stream.ts b/src/providers/codex/translate/stream.ts
index 83555bf..7ac6b4a 100644
--- a/src/providers/codex/translate/stream.ts
+++ b/src/providers/codex/translate/stream.ts
@@ -15,6 +15,7 @@ import {
} from "./reducer.ts";
import { emitMessageStart } from "../../shared/anthropic-sse.ts";
import { buildWebSearchCompatBlocks } from "./web-search-compat.ts";
+import { computeBackoffDelay, sleep, type BackoffOutcome } from "../../retry.ts";
/**
* Translate a Codex Responses SSE stream into Anthropic SSE events.
@@ -26,6 +27,13 @@ import { buildWebSearchCompatBlocks } from "./web-search-compat.ts";
*/
const KEEPALIVE_INTERVAL_MS = 15_000;
const STREAM_WATCHDOG_INTERVAL_MS = 30_000;
+const MAX_RETRYABLE_STREAM_RETRIES = 10;
+
+interface RetryableUpstream {
+ body: ReadableStream<Uint8Array>;
+ headers?: Headers;
+ requestSize?: CodexRequestSizeSummary;
+}
export function translateStream(
upstream: ReadableStream<Uint8Array>,
@@ -40,13 +48,16 @@ export function translateStream(
requestSize?: CodexRequestSizeSummary;
onFinish?: (finish: Extract<ReducerEvent, { kind: "finish" }>) => void;
onInvalidateContinuation?: () => void;
+ retryUpstream?: () => Promise<RetryableUpstream>;
+ maxRetryableStreamRetries?: number;
+ computeRetryDelay?: (attempt: number, retryAfter?: string) => BackoffOutcome;
},
): ReadableStream<Uint8Array> {
const encoder = new TextEncoder();
return new ReadableStream<Uint8Array>({
async start(controller) {
const openBlocks = new Map<number, { type: "text" | "tool"; id?: string; name?: string }>();
- const diagnostics = attachTrafficCapture(createUpstreamStreamDiagnostics(), opts.traffic);
+ let diagnostics = attachTrafficCapture(createUpstreamStreamDiagnostics(), opts.traffic);
let closed = false;
let messageStarted = false;
let lastEmitAt = 0;
@@ -95,6 +106,12 @@ export function translateStream(
}
openBlocks.clear();
};
+ const resetBeforeRetry = () => {
+ openBlocks.clear();
+ webSearchEvents.length = 0;
+ deferredContentEvents.length = 0;
+ lastReducerEvent = undefined;
+ };
const logWatchdog = () => {
const tools = activeToolCalls();
opts.log.info("codex stream watchdog", {
@@ -213,54 +230,77 @@ export function translateStream(
}
};
+ let currentUpstream = upstream;
+ let currentUpstreamHeaders = opts.upstreamHeaders;
+ let currentRequestSize = opts.requestSize;
+ let retryAttempt = 0;
+ let terminalError: unknown;
try {
- for await (const e of reduceUpstream(upstream, opts.log, diagnostics)) {
- lastReducerEvent = e.kind;
- if (e.kind === "web-search") {
- webSearchEvents.push(e);
- continue;
- }
- if (webSearchEvents.length && isContentEvent(e)) {
- deferredContentEvents.push(e);
- continue;
- }
- switch (e.kind) {
- case "text-start":
- case "text-delta":
- case "text-stop":
- case "tool-start":
- case "tool-delta":
- case "tool-stop":
- emitContentEvent(e);
- break;
- case "tool-progress":
- case "progress":
- emitPingIfStale();
- break;
- case "finish":
- if (openBlocks.size) {
- throw new UpstreamStreamError("failed", "Stream finished with open content blocks");
+ for (;;) {
+ diagnostics = attachTrafficCapture(createUpstreamStreamDiagnostics(), opts.traffic);
+ try {
+ for await (const e of reduceUpstream(currentUpstream, opts.log, diagnostics)) {
+ lastReducerEvent = e.kind;
+ if (e.kind === "web-search") {
+ webSearchEvents.push(e);
+ continue;
}
- emitWebSearchCompatBlocks();
- for (const event of deferredContentEvents) {
- emitContentEvent(event);
+ if (webSearchEvents.length && isContentEvent(e)) {
+ deferredContentEvents.push(e);
+ continue;
}
- if (openBlocks.size) {
- throw new UpstreamStreamError("failed", "Stream finished with open content blocks");
+ switch (e.kind) {
+ case "text-start":
+ case "text-delta":
+ case "text-stop":
+ case "tool-start":
+ case "tool-delta":
+ case "tool-stop":
+ emitContentEvent(e);
+ break;
+ case "tool-progress":
+ case "progress":
+ emitPingIfStale();
+ break;
+ case "finish":
+ if (openBlocks.size) {
+ throw new UpstreamStreamError(
+ "failed",
+ "Stream finished with open content blocks",
+ );
+ }
+ emitWebSearchCompatBlocks();
+ for (const event of deferredContentEvents) {
+ emitContentEvent(event);
+ }
+ if (openBlocks.size) {
+ throw new UpstreamStreamError(
+ "failed",
+ "Stream finished with open content blocks",
+ );
+ }
+ ensureMessageStart();
+ opts.onFinish?.(e);
+ emit("message_delta", {
+ type: "message_delta",
+ delta: { stop_reason: e.stopReason, stop_sequence: null },
+ usage: mapUsageToAnthropic(e.usage, {
+ webSearchRequests: e.webSearchRequests,
+ }),
+ });
+ emit("message_stop", { type: "message_stop" });
+ break;
}
- ensureMessageStart();
- opts.onFinish?.(e);
- emit("message_delta", {
- type: "message_delta",
- delta: { stop_reason: e.stopReason, stop_sequence: null },
- usage: mapUsageToAnthropic(e.usage, {
- webSearchRequests: e.webSearchRequests,
- }),
- });
- emit("message_stop", { type: "message_stop" });
+ }
+ break;
+ } catch (err) {
+ if (!(await retryPreMessageStreamError(err))) {
+ terminalError ??= err;
break;
+ }
}
}
+ if (terminalError) throw terminalError;
} catch (err) {
const activeToolCalls = Array.from(openBlocks.values()).filter(
(block): block is { type: "tool"; id: string; name: string } =>
@@ -291,8 +331,8 @@ export function translateStream(
openBlocks: openBlockDetails,
clientAborted: opts.signal?.aborted ?? false,
diagnostics: describeDiagnostics(diagnostics),
- upstreamHeaders: describeHeaders(opts.upstreamHeaders),
- requestSize: opts.requestSize,
+ upstreamHeaders: describeHeaders(currentUpstreamHeaders),
+ requestSize: currentRequestSize,
};
const diagnosticFile = await writeDiagnosticFile(
opts.reqId,
@@ -305,7 +345,7 @@ export function translateStream(
emit("error", {
type: "error",
error: {
- type: err.kind === "rate_limit" ? "rate_limit_error" : "api_error",
+ type: anthropicStreamErrorType(err),
message: err.message,
},
});
@@ -317,8 +357,8 @@ export function translateStream(
openBlocks: openBlockDetails,
clientAborted: opts.signal?.aborted ?? false,
diagnostics: describeDiagnostics(diagnostics),
- upstreamHeaders: describeHeaders(opts.upstreamHeaders),
- requestSize: opts.requestSize,
+ upstreamHeaders: describeHeaders(currentUpstreamHeaders),
+ requestSize: currentRequestSize,
};
const diagnosticFile = await writeDiagnosticFile(
opts.reqId,
@@ -337,10 +377,76 @@ export function translateStream(
clearInterval(watchdog);
safeClose();
}
+
+ async function retryPreMessageStreamError(err: unknown): Promise<boolean> {
+ const retryInfo = retryableStreamErrorInfo(err);
+ const maxRetries = opts.maxRetryableStreamRetries ?? MAX_RETRYABLE_STREAM_RETRIES;
+ if (
+ !retryInfo ||
+ !opts.retryUpstream ||
+ messageStarted ||
+ retryAttempt >= maxRetries ||
+ opts.signal?.aborted
+ ) {
+ return false;
+ }
+ const retryDelay = opts.computeRetryDelay ?? computeBackoffDelay;
+ const { waitMs, exceedsBudget } = retryDelay(retryAttempt, retryInfo.retryAfter);
+ if (exceedsBudget) {
+ opts.log.warn("upstream stream retry-after exceeds budget; giving up", {
+ kind: retryInfo.kind,
+ retryAfter: retryInfo.retryAfter,
+ maxDelayMs: waitMs,
+ });
+ return false;
+ }
+ const nextAttempt = retryAttempt + 1;
+ opts.log.warn("upstream stream error before downstream output, retrying", {
+ kind: retryInfo.kind,
+ attempt: nextAttempt,
+ maxRetries,
+ waitMs,
+ retryAfter: retryInfo.retryAfter,
+ message: retryInfo.message,
+ diagnostics: describeDiagnostics(diagnostics),
+ requestSize: currentRequestSize,
+ });
+ resetBeforeRetry();
+ retryAttempt = nextAttempt;
+ try {
+ await sleep(waitMs, opts.signal);
+ const retry = await opts.retryUpstream();
+ currentUpstream = retry.body;
+ currentUpstreamHeaders = retry.headers ?? currentUpstreamHeaders;
+ currentRequestSize = retry.requestSize ?? currentRequestSize;
+ return true;
+ } catch (retryErr) {
+ terminalError = retryErr;
+ return false;
+ }
+ }
},
});
}
+function retryableStreamErrorInfo(
+ err: unknown,
+): { kind: "rate_limit" | "overloaded"; retryAfter?: string; message: string } | undefined {
+ if (!(err instanceof UpstreamStreamError)) return undefined;
+ if (err.kind !== "rate_limit" && err.kind !== "overloaded") return undefined;
+ return {
+ kind: err.kind,
+ retryAfter: err.retryAfterSeconds !== undefined ? String(err.retryAfterSeconds) : undefined,
+ message: err.message,
+ };
+}
+
+function anthropicStreamErrorType(err: UpstreamStreamError): string {
+ if (err.kind === "rate_limit") return "rate_limit_error";
+ if (err.kind === "overloaded") return "overloaded_error";
+ return "api_error";
+}
+
function isClosedControllerError(err: unknown): boolean {
return err instanceof TypeError && err.message.includes("Controller is already closed");
}
diff --git a/src/providers/shared/upstream-errors.ts b/src/providers/shared/upstream-errors.ts
index d67b444..175dd41 100644
--- a/src/providers/shared/upstream-errors.ts
+++ b/src/providers/shared/upstream-errors.ts
@@ -38,5 +38,13 @@ export function mapUpstreamStreamErrorToResponse(err: UpstreamStreamErrorLike):
headers,
});
}
+ if (err.kind === "overloaded") {
+ const headers: Record<string, string> = {};
+ if (err.retryAfterSeconds) headers["retry-after"] = String(err.retryAfterSeconds);
+ return jsonResponse(anthropicErrorBody("overloaded_error", err.message), {
+ status: 529,
+ headers,
+ });
+ }
return jsonError(502, "api_error", err.message);
}