Skip to content

Use Claude Code with your ChatGPT, Kimi, Cursor or Grok subscription via a local Anthropic-compatible proxy

Rust

retry Codex socket transport failures

Raine Virta3f9bd6be39cb
diff --git a/src/providers/codex/client.test.ts b/src/providers/codex/client.test.ts
index 19d00d2..6521ecb 100644
--- a/src/providers/codex/client.test.ts
+++ b/src/providers/codex/client.test.ts
@@ -1,5 +1,10 @@
import { afterEach, describe, expect, it } from "bun:test";
-import { CodexHeaderTimeoutError, setCodexHeaderTimeoutForTests } from "./client.ts";
+import {
+ CodexHeaderTimeoutError,
+ CodexTransportError,
+ isRetryableCodexTransportError,
+ setCodexHeaderTimeoutForTests,
+} from "./client.ts";
const originalFetch = globalThis.fetch;
@@ -16,3 +21,20 @@ describe("CodexHeaderTimeoutError", () => {
expect(err.message).toContain("123ms");
});
});
+
+describe("CodexTransportError", () => {
+ it("classifies Bun socket-close fetch failures as retryable transport errors", () => {
+ const err = new TypeError(
+ "The socket connection was closed unexpectedly. For more information, pass `verbose: true` in the second argument to fetch()",
+ );
+
+ expect(isRetryableCodexTransportError(err)).toBe(true);
+ expect(new CodexTransportError(err).message).toContain(
+ "socket connection was closed unexpectedly",
+ );
+ });
+
+ it("does not retry abort errors as transport failures", () => {
+ expect(isRetryableCodexTransportError(new DOMException("Aborted", "AbortError"))).toBe(false);
+ });
+});
diff --git a/src/providers/codex/client.ts b/src/providers/codex/client.ts
index 8a6d36d..9c7067d 100644
--- a/src/providers/codex/client.ts
+++ b/src/providers/codex/client.ts
@@ -12,7 +12,7 @@ import { forceRefresh, getAuth } from "./auth/manager.ts";
import type { Logger } from "../../log.ts";
import type { RequestContext } from "../types.ts";
import { toWebSocketRequest, type ResponsesRequest } from "./translate/request.ts";
-import { retryOn429, sleep } from "../retry.ts";
+import { computeBackoffDelay, retryOn429, sleep } from "../retry.ts";
import { headersToRecord } from "../../traffic.ts";
import { summarizeCodexRequestSize } from "./request-summary.ts";
import {
@@ -25,6 +25,7 @@ import {
import { clearContinuation, type ContinuationCandidate } from "./continuation.ts";
const FETCH_WATCHDOG_INTERVAL_MS = 30_000;
+const MAX_TRANSPORT_RETRIES = 10;
let fetchHeaderTimeoutMs = 60_000;
let fetchHeaderTimeoutRetries = 1;
@@ -50,7 +51,7 @@ export async function postCodex(
opts: PostCodexOptions = {},
): Promise<CodexResponse> {
const log = ctx.childLogger("codex.client");
- return retryHeaderTimeouts(
+ return retryTransientPostFailures(
() =>
retryOn429(() => attemptPostCodex(body, ctx, log, opts), {
log,
@@ -66,7 +67,7 @@ export async function postCodex(
);
}
-async function retryHeaderTimeouts(
+async function retryTransientPostFailures(
run: () => Promise<CodexResponse>,
log: Logger,
signal: AbortSignal | undefined,
@@ -76,14 +77,21 @@ async function retryHeaderTimeouts(
try {
return await run();
} catch (err) {
- if (!(err instanceof CodexHeaderTimeoutError) || attempt >= fetchHeaderTimeoutRetries)
- throw err;
- const waitMs = fetchHeaderTimeoutMs <= 10 ? 0 : 500 + Math.round(Math.random() * 1000);
- log.warn("codex response headers timed out, retrying", {
+ const retryInfo = codexPostRetryInfo(err);
+ if (!retryInfo || attempt >= retryInfo.maxRetries) throw err;
+ const waitMs =
+ err instanceof CodexHeaderTimeoutError
+ ? fetchHeaderTimeoutMs <= 10
+ ? 0
+ : 500 + Math.round(Math.random() * 1000)
+ : computeBackoffDelay(attempt).waitMs;
+ log.warn(retryInfo.message, {
+ reason: retryInfo.reason,
attempt: attempt + 1,
- maxRetries: fetchHeaderTimeoutRetries,
+ maxRetries: retryInfo.maxRetries,
waitMs,
- timeoutMs: fetchHeaderTimeoutMs,
+ err: describeError(err),
+ timeoutMs: err instanceof CodexHeaderTimeoutError ? fetchHeaderTimeoutMs : undefined,
model: body.model,
inputCount: body.input.length,
toolCount: body.tools?.length ?? 0,
@@ -94,6 +102,26 @@ async function retryHeaderTimeouts(
}
}
+function codexPostRetryInfo(
+ err: unknown,
+): { reason: "header_timeout" | "transport"; maxRetries: number; message: string } | undefined {
+ if (err instanceof CodexHeaderTimeoutError) {
+ return {
+ reason: "header_timeout",
+ maxRetries: fetchHeaderTimeoutRetries,
+ message: "codex response headers timed out, retrying",
+ };
+ }
+ if (err instanceof CodexTransportError) {
+ return {
+ reason: "transport",
+ maxRetries: MAX_TRANSPORT_RETRIES,
+ message: "codex transport error before response, retrying",
+ };
+ }
+ return undefined;
+}
+
async function attemptPostCodex(
body: ResponsesRequest,
ctx: RequestContext,
@@ -364,6 +392,9 @@ async function doFetchHttp(
if (headerTimeout.signal.reason instanceof CodexHeaderTimeoutError) {
throw headerTimeout.signal.reason;
}
+ if (!ctx.signal.aborted && isRetryableCodexTransportError(err)) {
+ throw new CodexTransportError(err);
+ }
throw err;
} finally {
ctx.signal.removeEventListener("abort", onAbort);
@@ -390,7 +421,9 @@ function isSafeFullWebSocketRetry(
function shouldResetWebSocketPool(continuation: ContinuationCandidate | undefined): boolean {
if (!continuation?.disabledReason) return false;
- return continuation.disabledReason !== "missing_state" && continuation.disabledReason !== "disabled";
+ return (
+ continuation.disabledReason !== "missing_state" && continuation.disabledReason !== "disabled"
+ );
}
export class CodexHeaderTimeoutError extends Error {
@@ -400,6 +433,13 @@ export class CodexHeaderTimeoutError extends Error {
}
}
+export class CodexTransportError extends Error {
+ constructor(public originalError: unknown) {
+ super(errorMessage(originalError));
+ this.name = "CodexTransportError";
+ }
+}
+
export class CodexError extends Error {
constructor(
public status: number,
@@ -411,3 +451,55 @@ export class CodexError extends Error {
this.name = "CodexError";
}
}
+
+export function isRetryableCodexTransportError(err: unknown): boolean {
+ if (isAbortError(err)) return false;
+ const text = errorSearchText(err);
+ return (
+ text.includes("socket connection was closed unexpectedly") ||
+ text.includes("connection closed unexpectedly") ||
+ text.includes("connection reset") ||
+ text.includes("econnreset") ||
+ text.includes("epipe") ||
+ text.includes("etimedout") ||
+ text.includes("und_err_socket") ||
+ text.includes("fetch failed")
+ );
+}
+
+function isAbortError(err: unknown): boolean {
+ return err instanceof Error && err.name === "AbortError";
+}
+
+function errorMessage(err: unknown): string {
+ return err instanceof Error ? err.message : String(err);
+}
+
+function errorSearchText(err: unknown): string {
+ const parts: string[] = [];
+ let current: unknown = err;
+ for (let depth = 0; current && depth < 4; depth++) {
+ if (current instanceof Error) {
+ parts.push(current.name, current.message);
+ const code = (current as Error & { code?: unknown }).code;
+ if (code !== undefined) parts.push(String(code));
+ current = (current as Error & { cause?: unknown }).cause;
+ } else {
+ parts.push(String(current));
+ break;
+ }
+ }
+ return parts.join(" ").toLowerCase();
+}
+
+function describeError(err: unknown): Record<string, unknown> {
+ if (!(err instanceof Error)) return { message: String(err) };
+ const code = (err as Error & { code?: unknown }).code;
+ return {
+ name: err.name,
+ message: err.message,
+ code,
+ cause:
+ err.cause instanceof Error ? { name: err.cause.name, message: err.cause.message } : err.cause,
+ };
+}
diff --git a/src/providers/codex/translate/reducer.ts b/src/providers/codex/translate/reducer.ts
index 54086d2..4a572bd 100644
--- a/src/providers/codex/translate/reducer.ts
+++ b/src/providers/codex/translate/reducer.ts
@@ -687,7 +687,7 @@ function upstreamFailureKind(
type === "server_error" ||
type === "internal_server_error" ||
type === "internal_error" ||
- lowerMessage.includes("you can retry your request")
+ isRetryableTransportOrServerMessage(lowerMessage)
) {
return "transient";
}
@@ -727,7 +727,24 @@ function isCodexWebSocketCloseError(err: unknown): boolean {
function upstreamReadError(err: unknown): Error {
if (err instanceof UpstreamStreamError) return err;
const message = err instanceof Error ? err.message : String(err);
- return new UpstreamStreamError("failed", `Upstream stream read failed: ${message}`);
+ return new UpstreamStreamError(
+ isRetryableTransportOrServerMessage(message.toLowerCase()) ? "transient" : "failed",
+ `Upstream stream read failed: ${message}`,
+ );
+}
+
+function isRetryableTransportOrServerMessage(lowerMessage: string): boolean {
+ return (
+ lowerMessage.includes("you can retry your request") ||
+ lowerMessage.includes("socket connection was closed unexpectedly") ||
+ lowerMessage.includes("connection closed unexpectedly") ||
+ lowerMessage.includes("connection reset") ||
+ lowerMessage.includes("econnreset") ||
+ lowerMessage.includes("epipe") ||
+ lowerMessage.includes("etimedout") ||
+ lowerMessage.includes("und_err_socket") ||
+ lowerMessage.includes("fetch failed")
+ );
}
export function mapUsageToAnthropic(
diff --git a/src/providers/codex/translate/stream.test.ts b/src/providers/codex/translate/stream.test.ts
index dbe9e9f..2e13241 100644
--- a/src/providers/codex/translate/stream.test.ts
+++ b/src/providers/codex/translate/stream.test.ts
@@ -210,6 +210,31 @@ describe("translateStream", () => {
expect(output).not.toContain("event: error");
});
+ it("retries upstream socket read failures before downstream output starts", async () => {
+ let retryCalls = 0;
+ const err = new TypeError(
+ "The socket connection was closed unexpectedly. For more information, pass `verbose: true` in the second argument to fetch()",
+ );
+
+ const output = await collect(
+ translateStream(upstreamThatErrorsAfterChunks([], err), {
+ messageId: "msg_1",
+ model: "gpt-5.5",
+ log: silentLog,
+ retryUpstream: async () => {
+ retryCalls++;
+ return { body: upstreamFromChunks(textChunks("Recovered")) };
+ },
+ computeRetryDelay: () => ({ waitMs: 0, exceedsBudget: false }),
+ }),
+ );
+
+ expect(retryCalls).toBe(1);
+ expect(output).toContain("Recovered");
+ expect(output).toContain("event: message_stop");
+ expect(output).not.toContain("event: error");
+ });
+
it("does not retry overloaded stream errors after downstream output starts", async () => {
let retryCalls = 0;