Use Claude Code with your ChatGPT, Kimi, Cursor or Grok subscription via a local Anthropic-compatible proxy
Rust
retry codex overload stream errors
Raine Virtac97e25f5d042
diff --git a/src/providers/codex/index.ts b/src/providers/codex/index.tsindex 206fa73..6569e48 100644--- a/src/providers/codex/index.ts+++ b/src/providers/codex/index.ts@@ -327,6 +327,14 @@ async function handleMessages(body: AnthropicRequest, ctx: RequestContext): PromupstreamHeaders: upstream.headers,traffic: ctx.traffic,requestSize,+ retryUpstream: async () => {+ const retry = await postCodex(translated, ctx, { continuation });+ return {+ body: retry.body,+ headers: retry.headers,+ requestSize,+ };+ },onFinish: (finish) => {if (finish.continuationEligible) {recordContinuation(ctx.sessionId, translated, finish.responseId, finish.outputItems);diff --git a/src/providers/codex/translate/reducer.ts b/src/providers/codex/translate/reducer.tsindex 62f86c8..773fc73 100644--- a/src/providers/codex/translate/reducer.ts+++ b/src/providers/codex/translate/reducer.ts@@ -9,7 +9,7 @@ import { serverToolUseIdFromCodexWebSearchId } from "./web-search-compat.ts";export class UpstreamStreamError extends Error {constructor(- public kind: "rate_limit" | "failed",+ public kind: "rate_limit" | "overloaded" | "failed",message: string,public retryAfterSeconds?: number,) {@@ -389,7 +389,11 @@ export async function* reduceUpstream(}if (t === "response.failed" || t === "response.error" || t === "error") {const message = p?.response?.error?.message || p?.error?.message || "Upstream error";- throw new UpstreamStreamError("failed", message);+ throw new UpstreamStreamError(+ upstreamFailureKind(p, message),+ message,+ retryAfterSecondsFromPayload(p),+ );}if (t === "response.output_item.added") {@@ -653,6 +657,47 @@ function webSearchQuery(item: unknown): string {return "";}+function upstreamFailureKind(+ payload: {+ status?: unknown;+ status_code?: unknown;+ response?: { error?: { code?: unknown; type?: unknown } };+ error?: { code?: unknown; type?: unknown };+ },+ message: string,+): "overloaded" | "failed" {+ const status = payload.status ?? payload.status_code;+ const code = payload.response?.error?.code ?? payload.error?.code;+ const type = payload.response?.error?.type ?? payload.error?.type;+ const lowerMessage = message.toLowerCase();+ if (+ status === 529 ||+ status === "529" ||+ code === "overloaded_error" ||+ type === "overloaded_error" ||+ lowerMessage.includes("overloaded")+ ) {+ return "overloaded";+ }+ return "failed";+}++function retryAfterSecondsFromPayload(payload: {+ retry_after_seconds?: unknown;+ headers?: Record<string, unknown>;+ response?: { error?: { retry_after_seconds?: unknown } };+ error?: { retry_after_seconds?: unknown };+}): number | undefined {+ const raw =+ payload.response?.error?.retry_after_seconds ??+ payload.error?.retry_after_seconds ??+ payload.retry_after_seconds ??+ payload.headers?.["retry-after"] ??+ payload.headers?.["Retry-After"];+ const value = typeof raw === "number" ? raw : typeof raw === "string" ? Number(raw) : NaN;+ return Number.isFinite(value) && value >= 0 ? value : undefined;+}+function isCodexWebSocketCloseError(err: unknown): boolean {const message = err instanceof Error ? err.message : String(err);return (diff --git a/src/providers/codex/translate/stream.test.ts b/src/providers/codex/translate/stream.test.tsindex 8ca65de..5cd71d6 100644--- a/src/providers/codex/translate/stream.test.ts+++ b/src/providers/codex/translate/stream.test.ts@@ -117,7 +117,90 @@ function webSearchChunks(): string[] {];}+function textChunks(text: string): string[] {+ return [+ sse("response.output_item.added", {+ output_index: 0,+ item: { type: "message", id: "msg_upstream" },+ }),+ sse("response.output_text.delta", {+ output_index: 0,+ delta: text,+ }),+ sse("response.output_item.done", {+ output_index: 0,+ item: { type: "message", id: "msg_upstream" },+ }),+ sse("response.completed", { response: { usage: { input_tokens: 10, output_tokens: 5 } } }),+ ];+}++function overloadChunk(): string {+ return sse("error", {+ status: 529,+ error: {+ type: "overloaded_error",+ message: "Our servers are currently overloaded. Please try again later.",+ },+ });+}+describe("translateStream", () => {+ it("retries overloaded stream errors before downstream output starts", async () => {+ let retryCalls = 0;+ const logs: Array<{ msg: string; fields: unknown }> = [];+ const log = {+ ...silentLog,+ warn: (msg: string, fields: unknown) => logs.push({ msg, fields }),+ };++ const output = await collect(+ translateStream(upstreamFromChunks([overloadChunk()]), {+ messageId: "msg_1",+ model: "gpt-5.5",+ log,+ retryUpstream: async () => {+ retryCalls++;+ return { body: upstreamFromChunks(textChunks("Recovered")) };+ },+ computeRetryDelay: () => ({ waitMs: 0, exceedsBudget: false }),+ }),+ );++ expect(retryCalls).toBe(1);+ expect(output).toContain("Recovered");+ expect(output).toContain("event: message_stop");+ expect(output).not.toContain("event: error");+ expect(+ logs.some(+ (entry) => entry.msg === "upstream stream error before downstream output, retrying",+ ),+ ).toBe(true);+ });++ it("does not retry overloaded stream errors after downstream output starts", async () => {+ let retryCalls = 0;++ const output = await collect(+ translateStream(upstreamFromChunks([...textChunks("Partial").slice(0, 2), overloadChunk()]), {+ messageId: "msg_1",+ model: "gpt-5.5",+ log: silentLog,+ retryUpstream: async () => {+ retryCalls++;+ return { body: upstreamFromChunks(textChunks("Recovered")) };+ },+ computeRetryDelay: () => ({ waitMs: 0, exceedsBudget: false }),+ }),+ );++ expect(retryCalls).toBe(0);+ expect(output).toContain("Partial");+ expect(output).toContain("event: content_block_stop");+ expect(output).toContain("event: error");+ expect(output).toContain('"type":"overloaded_error"');+ });+it("emits keepalive pings while Read arguments are buffered", async () => {const chunks = [...readChunks(['{"file_path"', '":"/tmp/a"}']),diff --git a/src/providers/codex/translate/stream.ts b/src/providers/codex/translate/stream.tsindex 83555bf..7ac6b4a 100644--- a/src/providers/codex/translate/stream.ts+++ b/src/providers/codex/translate/stream.ts@@ -15,6 +15,7 @@ import {} from "./reducer.ts";import { emitMessageStart } from "../../shared/anthropic-sse.ts";import { buildWebSearchCompatBlocks } from "./web-search-compat.ts";+import { computeBackoffDelay, sleep, type BackoffOutcome } from "../../retry.ts";/*** Translate a Codex Responses SSE stream into Anthropic SSE events.@@ -26,6 +27,13 @@ import { buildWebSearchCompatBlocks } from "./web-search-compat.ts";*/const KEEPALIVE_INTERVAL_MS = 15_000;const STREAM_WATCHDOG_INTERVAL_MS = 30_000;+const MAX_RETRYABLE_STREAM_RETRIES = 10;++interface RetryableUpstream {+ body: ReadableStream<Uint8Array>;+ headers?: Headers;+ requestSize?: CodexRequestSizeSummary;+}export function translateStream(upstream: ReadableStream<Uint8Array>,@@ -40,13 +48,16 @@ export function translateStream(requestSize?: CodexRequestSizeSummary;onFinish?: (finish: Extract<ReducerEvent, { kind: "finish" }>) => void;onInvalidateContinuation?: () => void;+ retryUpstream?: () => Promise<RetryableUpstream>;+ maxRetryableStreamRetries?: number;+ computeRetryDelay?: (attempt: number, retryAfter?: string) => BackoffOutcome;},): ReadableStream<Uint8Array> {const encoder = new TextEncoder();return new ReadableStream<Uint8Array>({async start(controller) {const openBlocks = new Map<number, { type: "text" | "tool"; id?: string; name?: string }>();- const diagnostics = attachTrafficCapture(createUpstreamStreamDiagnostics(), opts.traffic);+ let diagnostics = attachTrafficCapture(createUpstreamStreamDiagnostics(), opts.traffic);let closed = false;let messageStarted = false;let lastEmitAt = 0;@@ -95,6 +106,12 @@ export function translateStream(}openBlocks.clear();};+ const resetBeforeRetry = () => {+ openBlocks.clear();+ webSearchEvents.length = 0;+ deferredContentEvents.length = 0;+ lastReducerEvent = undefined;+ };const logWatchdog = () => {const tools = activeToolCalls();opts.log.info("codex stream watchdog", {@@ -213,54 +230,77 @@ export function translateStream(}};+ let currentUpstream = upstream;+ let currentUpstreamHeaders = opts.upstreamHeaders;+ let currentRequestSize = opts.requestSize;+ let retryAttempt = 0;+ let terminalError: unknown;try {- for await (const e of reduceUpstream(upstream, opts.log, diagnostics)) {- lastReducerEvent = e.kind;- if (e.kind === "web-search") {- webSearchEvents.push(e);- continue;- }- if (webSearchEvents.length && isContentEvent(e)) {- deferredContentEvents.push(e);- continue;- }- switch (e.kind) {- case "text-start":- case "text-delta":- case "text-stop":- case "tool-start":- case "tool-delta":- case "tool-stop":- emitContentEvent(e);- break;- case "tool-progress":- case "progress":- emitPingIfStale();- break;- case "finish":- if (openBlocks.size) {- throw new UpstreamStreamError("failed", "Stream finished with open content blocks");+ for (;;) {+ diagnostics = attachTrafficCapture(createUpstreamStreamDiagnostics(), opts.traffic);+ try {+ for await (const e of reduceUpstream(currentUpstream, opts.log, diagnostics)) {+ lastReducerEvent = e.kind;+ if (e.kind === "web-search") {+ webSearchEvents.push(e);+ continue;}- emitWebSearchCompatBlocks();- for (const event of deferredContentEvents) {- emitContentEvent(event);+ if (webSearchEvents.length && isContentEvent(e)) {+ deferredContentEvents.push(e);+ continue;}- if (openBlocks.size) {- throw new UpstreamStreamError("failed", "Stream finished with open content blocks");+ switch (e.kind) {+ case "text-start":+ case "text-delta":+ case "text-stop":+ case "tool-start":+ case "tool-delta":+ case "tool-stop":+ emitContentEvent(e);+ break;+ case "tool-progress":+ case "progress":+ emitPingIfStale();+ break;+ case "finish":+ if (openBlocks.size) {+ throw new UpstreamStreamError(+ "failed",+ "Stream finished with open content blocks",+ );+ }+ emitWebSearchCompatBlocks();+ for (const event of deferredContentEvents) {+ emitContentEvent(event);+ }+ if (openBlocks.size) {+ throw new UpstreamStreamError(+ "failed",+ "Stream finished with open content blocks",+ );+ }+ ensureMessageStart();+ opts.onFinish?.(e);+ emit("message_delta", {+ type: "message_delta",+ delta: { stop_reason: e.stopReason, stop_sequence: null },+ usage: mapUsageToAnthropic(e.usage, {+ webSearchRequests: e.webSearchRequests,+ }),+ });+ emit("message_stop", { type: "message_stop" });+ break;}- ensureMessageStart();- opts.onFinish?.(e);- emit("message_delta", {- type: "message_delta",- delta: { stop_reason: e.stopReason, stop_sequence: null },- usage: mapUsageToAnthropic(e.usage, {- webSearchRequests: e.webSearchRequests,- }),- });- emit("message_stop", { type: "message_stop" });+ }+ break;+ } catch (err) {+ if (!(await retryPreMessageStreamError(err))) {+ terminalError ??= err;break;+ }}}+ if (terminalError) throw terminalError;} catch (err) {const activeToolCalls = Array.from(openBlocks.values()).filter((block): block is { type: "tool"; id: string; name: string } =>@@ -291,8 +331,8 @@ export function translateStream(openBlocks: openBlockDetails,clientAborted: opts.signal?.aborted ?? false,diagnostics: describeDiagnostics(diagnostics),- upstreamHeaders: describeHeaders(opts.upstreamHeaders),- requestSize: opts.requestSize,+ upstreamHeaders: describeHeaders(currentUpstreamHeaders),+ requestSize: currentRequestSize,};const diagnosticFile = await writeDiagnosticFile(opts.reqId,@@ -305,7 +345,7 @@ export function translateStream(emit("error", {type: "error",error: {- type: err.kind === "rate_limit" ? "rate_limit_error" : "api_error",+ type: anthropicStreamErrorType(err),message: err.message,},});@@ -317,8 +357,8 @@ export function translateStream(openBlocks: openBlockDetails,clientAborted: opts.signal?.aborted ?? false,diagnostics: describeDiagnostics(diagnostics),- upstreamHeaders: describeHeaders(opts.upstreamHeaders),- requestSize: opts.requestSize,+ upstreamHeaders: describeHeaders(currentUpstreamHeaders),+ requestSize: currentRequestSize,};const diagnosticFile = await writeDiagnosticFile(opts.reqId,@@ -337,10 +377,76 @@ export function translateStream(clearInterval(watchdog);safeClose();}++ async function retryPreMessageStreamError(err: unknown): Promise<boolean> {+ const retryInfo = retryableStreamErrorInfo(err);+ const maxRetries = opts.maxRetryableStreamRetries ?? MAX_RETRYABLE_STREAM_RETRIES;+ if (+ !retryInfo ||+ !opts.retryUpstream ||+ messageStarted ||+ retryAttempt >= maxRetries ||+ opts.signal?.aborted+ ) {+ return false;+ }+ const retryDelay = opts.computeRetryDelay ?? computeBackoffDelay;+ const { waitMs, exceedsBudget } = retryDelay(retryAttempt, retryInfo.retryAfter);+ if (exceedsBudget) {+ opts.log.warn("upstream stream retry-after exceeds budget; giving up", {+ kind: retryInfo.kind,+ retryAfter: retryInfo.retryAfter,+ maxDelayMs: waitMs,+ });+ return false;+ }+ const nextAttempt = retryAttempt + 1;+ opts.log.warn("upstream stream error before downstream output, retrying", {+ kind: retryInfo.kind,+ attempt: nextAttempt,+ maxRetries,+ waitMs,+ retryAfter: retryInfo.retryAfter,+ message: retryInfo.message,+ diagnostics: describeDiagnostics(diagnostics),+ requestSize: currentRequestSize,+ });+ resetBeforeRetry();+ retryAttempt = nextAttempt;+ try {+ await sleep(waitMs, opts.signal);+ const retry = await opts.retryUpstream();+ currentUpstream = retry.body;+ currentUpstreamHeaders = retry.headers ?? currentUpstreamHeaders;+ currentRequestSize = retry.requestSize ?? currentRequestSize;+ return true;+ } catch (retryErr) {+ terminalError = retryErr;+ return false;+ }+ }},});}+function retryableStreamErrorInfo(+ err: unknown,+): { kind: "rate_limit" | "overloaded"; retryAfter?: string; message: string } | undefined {+ if (!(err instanceof UpstreamStreamError)) return undefined;+ if (err.kind !== "rate_limit" && err.kind !== "overloaded") return undefined;+ return {+ kind: err.kind,+ retryAfter: err.retryAfterSeconds !== undefined ? String(err.retryAfterSeconds) : undefined,+ message: err.message,+ };+}++function anthropicStreamErrorType(err: UpstreamStreamError): string {+ if (err.kind === "rate_limit") return "rate_limit_error";+ if (err.kind === "overloaded") return "overloaded_error";+ return "api_error";+}+function isClosedControllerError(err: unknown): boolean {return err instanceof TypeError && err.message.includes("Controller is already closed");}diff --git a/src/providers/shared/upstream-errors.ts b/src/providers/shared/upstream-errors.tsindex d67b444..175dd41 100644--- a/src/providers/shared/upstream-errors.ts+++ b/src/providers/shared/upstream-errors.ts@@ -38,5 +38,13 @@ export function mapUpstreamStreamErrorToResponse(err: UpstreamStreamErrorLike):headers,});}+ if (err.kind === "overloaded") {+ const headers: Record<string, string> = {};+ if (err.retryAfterSeconds) headers["retry-after"] = String(err.retryAfterSeconds);+ return jsonResponse(anthropicErrorBody("overloaded_error", err.message), {+ status: 529,+ headers,+ });+ }return jsonError(502, "api_error", err.message);}