Skip to content

Claude Code. Any Model. The most powerful AI coding agent now speaks every language.

fix(openai-codex): abort a Codex leg after 150 s without an output item

Jack Rudenko8a4f1229d826
diff --git a/packages/cli/src/handlers/composed-handler.ts b/packages/cli/src/handlers/composed-handler.ts
index a19608eb..7156afdf 100644
--- a/packages/cli/src/handlers/composed-handler.ts
+++ b/packages/cli/src/handlers/composed-handler.ts
@@ -113,6 +113,7 @@ import { createOllamaJsonlStream } from "./shared/stream-parsers/ollama-jsonl.js
import {
type ResponsesRetryOptions,
createResponsesStreamHandler,
+ resolveStallPolicy,
} from "./shared/stream-parsers/openai-responses-sse.js";
import { createStreamingResponseHandler } from "./shared/stream-parsers/openai-sse.js";
import { TokenTracker, type UsageCacheDetail } from "./shared/token-tracker.js";
@@ -1402,6 +1403,9 @@ export class ComposedHandler implements ModelHandler {
// note there for why it cannot run at this point.)
const endpoint = this.provider.getEndpoint(this.targetModel);
+ // Read beside the endpoint, from the same artifact: the arm that signs THIS
+ // request. Decides the Responses stall guard's default scope below.
+ const authArm = this.provider.currentAuthArm?.();
// The ORIGINAL inbound body, not the normalized `claudeRequest` clone: a
// header carrying conversation identity must see what Claude Code sent.
//
@@ -2064,10 +2068,19 @@ export class ComposedHandler implements ModelHandler {
// when all of them fail hand back a 503 the client will retry itself. Without
// this, `server_is_overloaded` became an assistant text block with
// stop_reason end_turn — a retryable failure frozen into the transcript.
+ // The stall guard's zero: when the response the parser will read had its
+ // headers. Stamped here (a 200 reaches this line the moment its fetch
+ // resolved) and again by each 7b re-issue, so the sniff window counts
+ // against the guard rather than adding to it.
+ let responseHeadersAt = Date.now();
if (this.resolveStreamFormat() === "openai-responses-sse") {
- const settled = await this.settleResponsesStreamHead(response, () =>
- this.provider.enqueueRequest ? this.provider.enqueueRequest(doFetch) : doFetch()
- );
+ const settled = await this.settleResponsesStreamHead(response, async () => {
+ const next = await (this.provider.enqueueRequest
+ ? this.provider.enqueueRequest(doFetch)
+ : doFetch());
+ responseHeadersAt = Date.now();
+ return next;
+ });
if (settled.kind === "exhausted") {
const waited = STREAM_RETRY_DELAYS_MS.slice(0, settled.attempts).reduce(
(sum, ms) => sum + ms,
@@ -2262,7 +2275,18 @@ export class ComposedHandler implements ModelHandler {
legDeadlineAt: () => recoveryClock().now() + resolveTier1DeadlineMs(),
});
responsesTurn = {
- retry: { issueLeg },
+ retry: {
+ issueLeg,
+ // Default scope: `openai-codex` on the subscription arm, the only
+ // backend the stall data comes from; CLAUDISH_RESPONSES_STALL_MS
+ // overrides it everywhere.
+ stall: resolveStallPolicy({
+ env: process.env,
+ providerName: this.provider.name,
+ arm: authArm,
+ }),
+ firstHeadersAt: responseHeadersAt,
+ },
onTurnSummary: (s) => {
legRetries = s.legRetries;
},
diff --git a/packages/cli/src/handlers/shared/stream-parsers/openai-responses-sse.ts b/packages/cli/src/handlers/shared/stream-parsers/openai-responses-sse.ts
index bf45e1c2..a37a4aeb 100644
--- a/packages/cli/src/handlers/shared/stream-parsers/openai-responses-sse.ts
+++ b/packages/cli/src/handlers/shared/stream-parsers/openai-responses-sse.ts
@@ -17,6 +17,7 @@ import { ClaudeSseSink } from "./responses/claude-sse-sink.js";
import { type ResponsesRetryOptions, runResponsesTurn } from "./responses/responses-turn.js";
export type { ResponsesRetryOptions } from "./responses/responses-turn.js";
+export { resolveStallPolicy, type StallPolicy } from "./responses/stall-guard.js";
export function createResponsesStreamHandler(
c: Context,
diff --git a/packages/cli/src/handlers/shared/stream-parsers/responses/responses-leg.ts b/packages/cli/src/handlers/shared/stream-parsers/responses/responses-leg.ts
index 7bbe991b..db317e3c 100644
--- a/packages/cli/src/handlers/shared/stream-parsers/responses/responses-leg.ts
+++ b/packages/cli/src/handlers/shared/stream-parsers/responses/responses-leg.ts
@@ -38,6 +38,7 @@ import {
isParallelWrapperName,
parseParallelWrapper,
} from "./parallel-wrapper.js";
+import { type StallClock, StallGuard, isOutputBearingEvent } from "./stall-guard.js";
/** One leg's usage, read from `response.completed` / `.incomplete`. */
export interface Usage {
@@ -74,6 +75,12 @@ export interface ResponsesLegContext {
onToolCall?: (name: string, argsJson: string) => string | null | undefined;
shouldBufferTool?: (name: string) => boolean;
onAssistantText?: (text: string, kind?: "text" | "reasoning") => void;
+ /**
+ * This leg's stall guard (stall-guard.ts): fires `thresholdMs` after
+ * `headersAt` (this leg's response headers, on `clock`) unless an
+ * output-bearing event arrived first. Absent or 0 ⇒ unarmed.
+ */
+ stall?: { thresholdMs: number; headersAt: number; clock?: StallClock };
}
type FnCall = {
@@ -168,6 +175,29 @@ class ResponsesLeg {
if (signal.aborted) onAbort();
else signal.addEventListener("abort", onAbort, { once: true });
+ // A fired guard cancels the reader; the pending read then settles and the
+ // leg reports `stall` instead of how the cut-short stream looked.
+ let stalledAfterMs: number | undefined;
+ const stall = this.ctx.stall;
+ const guard =
+ stall && stall.thresholdMs > 0
+ ? new StallGuard(
+ stall.thresholdMs,
+ stall.headersAt,
+ (afterMs) => {
+ stalledAfterMs = afterMs;
+ log(`[ResponsesTurn] no output item ${Math.round(afterMs / 1000)}s after headers`);
+ reader.cancel().catch(() => {});
+ },
+ stall.clock
+ )
+ : undefined;
+ const stalled = (): LegOutcome | undefined => {
+ if (stalledAfterMs === undefined || signal.aborted) return undefined;
+ this.closeAll();
+ return { kind: "stall", afterMs: stalledAfterMs };
+ };
+
const decoder = new TextDecoder();
let buffer = "";
try {
@@ -192,7 +222,9 @@ class ResponsesLeg {
// event, costs that event only, never the turn.
let stop: Stop | undefined;
try {
- stop = await this.dispatch(JSON.parse(data));
+ const event = JSON.parse(data);
+ if (guard?.armed && isOutputBearingEvent(event?.type)) guard.disarm();
+ stop = await this.dispatch(event);
} catch (parseError) {
log(`[ResponsesSSE] Parse error: ${parseError}`);
continue;
@@ -207,6 +239,8 @@ class ResponsesLeg {
}
} catch (error) {
if (signal.aborted) return { kind: "cancelled", usage: this.usage };
+ const stalledOutcome = stalled();
+ if (stalledOutcome) return stalledOutcome;
log(`[ResponsesSSE] Stream error: ${error}`);
this.closeReasoning();
this.closeText();
@@ -218,10 +252,13 @@ class ResponsesLeg {
this.closeTools();
return { kind: "stream-error", error, openToolBlock, usage: this.usage };
} finally {
+ guard?.disarm();
signal.removeEventListener("abort", onAbort);
}
if (signal.aborted) return { kind: "cancelled", usage: this.usage };
+ const stalledOutcome = stalled();
+ if (stalledOutcome) return stalledOutcome;
// The stream ended while a wrapper was still held: its arguments were never
// completed, so there is nothing valid to expand.
diff --git a/packages/cli/src/handlers/shared/stream-parsers/responses/responses-turn.ts b/packages/cli/src/handlers/shared/stream-parsers/responses/responses-turn.ts
index bb6fa9d4..bc4cb139 100644
--- a/packages/cli/src/handlers/shared/stream-parsers/responses/responses-turn.ts
+++ b/packages/cli/src/handlers/shared/stream-parsers/responses/responses-turn.ts
@@ -10,7 +10,9 @@
* ── RETRY WITHIN THE TURN ────────────────────────────────────────────────────
*
* About 1% of ChatGPT Codex turns end with a degenerate `parallel` wrapper
- * (parallel-wrapper.ts). When the caller passes `retry`, such a leg is retried
+ * (parallel-wrapper.ts), usually after a 361-484 s keepalive-only stall. With
+ * `retry.stall` armed (stall-guard.ts), a leg with no output item 150 s after
+ * its headers is cut off too. When the caller passes `retry`, such a leg is retried
* by issuing the same body again while the downstream message stays open; the
* next leg continues at the next block index. A leg is issued only while no
* tool_use has been emitted in this message:
@@ -66,6 +68,7 @@ import {
type Usage,
parseResponsesLeg,
} from "./responses-leg.js";
+import { type StallClock, type StallPolicy, systemStallClock } from "./stall-guard.js";
export interface ResponsesTurnOptions extends Omit<ResponsesLegContext, "sink"> {
/**
@@ -98,6 +101,16 @@ export interface ResponsesRetryOptions {
issueLeg: (body: unknown | undefined, signal: AbortSignal) => Promise<LegResponse>;
/** Retries per leg position. Default {@link MAX_LEG_RETRIES}. */
maxLegRetries?: number;
+ /**
+ * Abort a leg with no output item for too long (stall-guard.ts): attempts
+ * 1..n-1 use `thresholdMs`, the final attempt `finalThresholdMs`. Absent or
+ * `thresholdMs: 0` ⇒ unarmed.
+ */
+ stall?: StallPolicy;
+ /** When the FIRST leg's response headers arrived (the stall clock's zero). */
+ firstHeadersAt?: number;
+ /** The stall guard's clock; tests inject a fake one. Default wall clock. */
+ stallClock?: StallClock;
}
/** The in-band text for an in-stream API error; context overflow gets an actionable message. */
@@ -210,8 +223,14 @@ class ResponsesTurn {
async run(firstResponse: Response): Promise<void> {
try {
let response = firstResponse;
+ let headersAt =
+ this.opts.retry?.firstHeadersAt ?? (this.opts.retry?.stallClock ?? systemStallClock).now();
while (true) {
- const outcome = await parseResponsesLeg(response, { ...this.opts, sink: this.sink });
+ const outcome = await parseResponsesLeg(response, {
+ ...this.opts,
+ sink: this.sink,
+ stall: this.stallFor(headersAt),
+ });
this.legUsages.push("usage" in outcome ? outcome.usage : undefined);
const decision = this.retryDecision(outcome);
if (decision.action === "finish") {
@@ -232,7 +251,8 @@ class ResponsesTurn {
}
const next = await this.issueRetry(outcome);
if (!next) return;
- response = next;
+ response = next.response;
+ headersAt = next.headersAt;
}
} catch (error) {
// Only a throwing callback can land here; the leg itself never throws.
@@ -264,7 +284,28 @@ class ResponsesTurn {
}
/** Issue the next leg; undefined when the turn has ended instead. */
- private async issueRetry(outcome: LegOutcome): Promise<Response | undefined> {
+ /**
+ * This leg's stall guard settings: the final attempt (no retry left) gets the
+ * longer threshold, so no attempt is unbounded and the last one still has
+ * room for a long reasoning phase.
+ */
+ private stallFor(headersAt: number): ResponsesLegContext["stall"] {
+ const retry = this.opts.retry;
+ const policy = retry?.stall;
+ if (!retry || !policy || !("finalThresholdMs" in policy) || policy.thresholdMs <= 0) {
+ return undefined;
+ }
+ const isFinal = this.legRetries >= (retry.maxLegRetries ?? MAX_LEG_RETRIES);
+ return {
+ thresholdMs: isFinal ? policy.finalThresholdMs : policy.thresholdMs,
+ headersAt,
+ clock: retry.stallClock,
+ };
+ }
+
+ private async issueRetry(
+ outcome: LegOutcome
+ ): Promise<{ response: Response; headersAt: number } | undefined> {
const retry = this.opts.retry;
if (!retry) return undefined;
this.legRetries++;
@@ -275,7 +316,7 @@ class ResponsesTurn {
return undefined;
}
const leg = await retry.issueLeg(undefined, this.sink.signal);
- if (leg.kind === "ok") return leg.response;
+ if (leg.kind === "ok") return { response: leg.response, headersAt: leg.headersAt };
if (leg.kind === "error") this.sink.endWithError(leg.message);
this.reportUsage();
return undefined;
diff --git a/packages/cli/src/handlers/shared/stream-parsers/responses/stall-guard.ts b/packages/cli/src/handlers/shared/stream-parsers/responses/stall-guard.ts
new file mode 100644
index 00000000..e6a2b944
--- /dev/null
+++ b/packages/cli/src/handlers/shared/stream-parsers/responses/stall-guard.ts
@@ -0,0 +1,143 @@
+/**
+ * Abort a Responses leg that produces no output item for too long.
+ *
+ * ── THE FAILURE ──────────────────────────────────────────────────────────────
+ *
+ * About 1% of ChatGPT Codex turns stall: the backend sends `response.created`
+ * and keepalives only, for 361-484 s, and then the degenerate `parallel`
+ * wrapper arrives in one burst (see parallel-wrapper.ts). claudish's own 1 s
+ * pings keep Claude Code waiting the whole time, and `API_TIMEOUT_MS` does not
+ * bound a flushed, pinging stream (the leak ran 472 s). The guard turns that
+ * wait into a retry of the same body (`responses-turn.ts`).
+ *
+ * ── THE DATA (all `openai-codex`, ChatGPT backend) ───────────────────────────
+ *
+ * | sample | n | headers → first output item |
+ * |----------------------------------------|-------|-------------------------------|
+ * | real sessions, 19 debug logs | 8,239 | p50 1.2 s, p90 3.6 s, |
+ * | 2026-07-15 .. 2026-10-16 | | p99 8.5 s, p99.9 28.3 s, |
+ * | | | healthy max 61.7 s |
+ * | B experiment, healthy legs | 297 | max 7.38 s (from created) |
+ * | leaks (B experiment + the leak log) | 4 | 361 s, 456 s, 484.5 s, >150 s |
+ *
+ * In the 61.7 s legs `response.created` arrived together with the first item
+ * (the backend holds the head), so the clock starts at the RESPONSE HEADERS,
+ * never at `response.created`. It is armed where the leg's fetch resolved, so
+ * the 7b sniff window (up to 12 s) counts against it rather than adding to it.
+ *
+ * ── THE THRESHOLDS ───────────────────────────────────────────────────────────
+ *
+ * Attempts 1..n-1: 150 s, 2.4x the healthy max and 2.4x below the fastest
+ * leak. Final attempt: 300 s (2x), 4.9x the healthy max and still below the
+ * fastest leak, so the last attempt keeps room for an unusually long reasoning
+ * phase without waiting out a whole leak. No attempt is unbounded. Not derived
+ * from `API_TIMEOUT_MS`, which does not bound a pinging stream.
+ *
+ * ── THE SCOPE ────────────────────────────────────────────────────────────────
+ *
+ * The data is from the ChatGPT Codex backend only, so the default scope is
+ * exactly that: claudish provider `openai-codex` on the OAuth (subscription)
+ * arm. Native `openai@`, the `openai-codex` api-key arm and custom `responses`
+ * endpoints are unarmed unless `CLAUDISH_RESPONSES_STALL_MS` is set: a value
+ * > 0 arms it everywhere with that threshold (final attempt 2x), `0` disarms it
+ * everywhere.
+ */
+
+import { log } from "../../../../logger.js";
+
+/** Default threshold for attempts 1..n-1. */
+export const DEFAULT_STALL_THRESHOLD_MS = 150_000;
+
+/** Armed: a threshold per attempt, the last attempt's longer. Unarmed: `thresholdMs: 0`. */
+export type StallPolicy = { thresholdMs: number; finalThresholdMs: number } | { thresholdMs: 0 };
+
+export const STALL_ENV_VAR = "CLAUDISH_RESPONSES_STALL_MS";
+
+/**
+ * The stall policy for one request's connection. `arm` is the credential arm
+ * of THIS request (`"oauth"` for the subscription).
+ */
+export function resolveStallPolicy(i: {
+ env?: Record<string, string | undefined>;
+ providerName: string;
+ arm?: string;
+}): StallPolicy {
+ const raw = i.env?.[STALL_ENV_VAR];
+ if (raw !== undefined && raw.trim() !== "") {
+ const ms = Number(raw);
+ if (Number.isFinite(ms) && ms >= 0) {
+ return ms > 0 ? { thresholdMs: ms, finalThresholdMs: 2 * ms } : { thresholdMs: 0 };
+ }
+ log(`[ResponsesTurn] ignoring ${STALL_ENV_VAR}=${raw}: not a non-negative number`);
+ }
+ if (i.providerName === "openai-codex" && i.arm === "oauth") {
+ return {
+ thresholdMs: DEFAULT_STALL_THRESHOLD_MS,
+ finalThresholdMs: 2 * DEFAULT_STALL_THRESHOLD_MS,
+ };
+ }
+ return { thresholdMs: 0 };
+}
+
+/** The clock a guard runs on. Tests inject a fake one. */
+export interface StallClock {
+ now(): number;
+ setTimeout(fn: () => void, ms: number): unknown;
+ clearTimeout(handle: unknown): void;
+}
+
+export const systemStallClock: StallClock = {
+ now: () => Date.now(),
+ setTimeout: (fn, ms) => setTimeout(fn, ms),
+ clearTimeout: (handle) => clearTimeout(handle as ReturnType<typeof setTimeout>),
+};
+
+/**
+ * Is this Responses event output-bearing? The first one disarms the guard:
+ * `output_item.added` of any type, or any `*.delta`. `response.created`,
+ * `in_progress`, keepalives and metadata do not count.
+ */
+export function isOutputBearingEvent(type: unknown): boolean {
+ return (
+ typeof type === "string" && (type === "response.output_item.added" || type.endsWith(".delta"))
+ );
+}
+
+/**
+ * One leg's guard: fires `onFire(afterMs)` once, `thresholdMs` after
+ * `headersAt`, unless `disarm()` ran first.
+ */
+export class StallGuard {
+ private handle: unknown;
+ private done = false;
+
+ constructor(
+ thresholdMs: number,
+ private readonly headersAt: number,
+ private readonly onFire: (afterMs: number) => void,
+ private readonly clock: StallClock = systemStallClock
+ ) {
+ if (thresholdMs <= 0) {
+ this.done = true;
+ return;
+ }
+ const remaining = Math.max(0, headersAt + thresholdMs - clock.now());
+ this.handle = clock.setTimeout(() => this.fire(), remaining);
+ }
+
+ get armed(): boolean {
+ return !this.done;
+ }
+
+ disarm(): void {
+ if (this.done) return;
+ this.done = true;
+ this.clock.clearTimeout(this.handle);
+ }
+
+ private fire(): void {
+ if (this.done) return;
+ this.done = true;
+ this.onFire(this.clock.now() - this.headersAt);
+ }
+}
diff --git a/packages/cli/src/providers/transport/openai-codex.ts b/packages/cli/src/providers/transport/openai-codex.ts
index 225af8b0..03874d5e 100644
--- a/packages/cli/src/providers/transport/openai-codex.ts
+++ b/packages/cli/src/providers/transport/openai-codex.ts
@@ -25,7 +25,7 @@ import { normalizeCodexModel } from "../../adapters/codex-api-format.js";
import { lookupModelForProvider } from "../../adapters/model-catalog.js";
import { credentials } from "../../auth/credentials/authority.js";
import { recordSignedArm } from "../../auth/credentials/billing-probe.js";
-import type { RequestAuth } from "../../auth/credentials/types.js";
+import type { CredentialArm, RequestAuth } from "../../auth/credentials/types.js";
import { classifyConnectionError } from "../../handlers/shared/connection-error.js";
import { conversationKey } from "./conversation-key.js";
import { OpenAIProviderTransport } from "./openai.js";
@@ -85,6 +85,15 @@ export class OpenAICodexTransport extends OpenAIProviderTransport {
recordSignedArm("openai-codex", signedWithOAuth ? "subscription" : "metered");
}
+ /**
+ * The arm of the artifact `getEndpoint`/`getHeaders` read now: `"oauth"` for
+ * the ChatGPT subscription, `"api-key"` (or unknown) otherwise. Read from the
+ * artifact itself, never from its truthiness (see refreshAuth above).
+ */
+ currentAuthArm(): CredentialArm | undefined {
+ return this.cachedAuth?.arm;
+ }
+
/**
* OAuth tokens only work with chatgpt.com/backend-api (endpoint comes from the
* cached auth artifact). API keys use the standard OpenAI endpoint (super).
diff --git a/packages/cli/src/providers/transport/types.ts b/packages/cli/src/providers/transport/types.ts
index 440a179e..0d49cf95 100644
--- a/packages/cli/src/providers/transport/types.ts
+++ b/packages/cli/src/providers/transport/types.ts
@@ -5,6 +5,8 @@
* Does NOT own: message conversion, tool format, pricing (those are ModelAdapter concerns).
*/
+import type { CredentialArm } from "../../auth/credentials/types.js";
+
/** The wire format used for streaming responses */
export type StreamFormat =
| "openai-sse"
@@ -111,6 +113,16 @@ export interface ProviderTransport {
*/
forceRefreshAuth?(): Promise<void>;
+ /**
+ * The credential arm of the auth artifact `getEndpoint`/`getHeaders` read
+ * right now (`RequestAuth.arm`). Read where the request's endpoint is
+ * computed, so it names the arm that signed this request. Absent or
+ * undefined ⇒ unknown, which every consumer treats as NOT the subscription.
+ * Only `openai-codex` implements it; the Responses stall guard is armed by
+ * default only on its `"oauth"` arm.
+ */
+ currentAuthArm?(): CredentialArm | undefined;
+
/**
* Optional per-provider verdict on whether an error response is TERMINAL
* (won't recover on retry) — the transport's own reading of its provider's