Claude Code. Any Model. The most powerful AI coding agent now speaks every language.
fix(openai-codex): abort a Codex leg after 150 s without an output item
Jack Rudenko8a4f1229d826
diff --git a/packages/cli/src/handlers/composed-handler.ts b/packages/cli/src/handlers/composed-handler.tsindex a19608eb..7156afdf 100644--- a/packages/cli/src/handlers/composed-handler.ts+++ b/packages/cli/src/handlers/composed-handler.ts@@ -113,6 +113,7 @@ import { createOllamaJsonlStream } from "./shared/stream-parsers/ollama-jsonl.jsimport {type ResponsesRetryOptions,createResponsesStreamHandler,+ resolveStallPolicy,} from "./shared/stream-parsers/openai-responses-sse.js";import { createStreamingResponseHandler } from "./shared/stream-parsers/openai-sse.js";import { TokenTracker, type UsageCacheDetail } from "./shared/token-tracker.js";@@ -1402,6 +1403,9 @@ export class ComposedHandler implements ModelHandler {// note there for why it cannot run at this point.)const endpoint = this.provider.getEndpoint(this.targetModel);+ // Read beside the endpoint, from the same artifact: the arm that signs THIS+ // request. Decides the Responses stall guard's default scope below.+ const authArm = this.provider.currentAuthArm?.();// The ORIGINAL inbound body, not the normalized `claudeRequest` clone: a// header carrying conversation identity must see what Claude Code sent.//@@ -2064,10 +2068,19 @@ export class ComposedHandler implements ModelHandler {// when all of them fail hand back a 503 the client will retry itself. Without// this, `server_is_overloaded` became an assistant text block with// stop_reason end_turn — a retryable failure frozen into the transcript.+ // The stall guard's zero: when the response the parser will read had its+ // headers. Stamped here (a 200 reaches this line the moment its fetch+ // resolved) and again by each 7b re-issue, so the sniff window counts+ // against the guard rather than adding to it.+ let responseHeadersAt = Date.now();if (this.resolveStreamFormat() === "openai-responses-sse") {- const settled = await this.settleResponsesStreamHead(response, () =>- this.provider.enqueueRequest ? this.provider.enqueueRequest(doFetch) : doFetch()- );+ const settled = await this.settleResponsesStreamHead(response, async () => {+ const next = await (this.provider.enqueueRequest+ ? this.provider.enqueueRequest(doFetch)+ : doFetch());+ responseHeadersAt = Date.now();+ return next;+ });if (settled.kind === "exhausted") {const waited = STREAM_RETRY_DELAYS_MS.slice(0, settled.attempts).reduce((sum, ms) => sum + ms,@@ -2262,7 +2275,18 @@ export class ComposedHandler implements ModelHandler {legDeadlineAt: () => recoveryClock().now() + resolveTier1DeadlineMs(),});responsesTurn = {- retry: { issueLeg },+ retry: {+ issueLeg,+ // Default scope: `openai-codex` on the subscription arm, the only+ // backend the stall data comes from; CLAUDISH_RESPONSES_STALL_MS+ // overrides it everywhere.+ stall: resolveStallPolicy({+ env: process.env,+ providerName: this.provider.name,+ arm: authArm,+ }),+ firstHeadersAt: responseHeadersAt,+ },onTurnSummary: (s) => {legRetries = s.legRetries;},diff --git a/packages/cli/src/handlers/shared/stream-parsers/openai-responses-sse.ts b/packages/cli/src/handlers/shared/stream-parsers/openai-responses-sse.tsindex bf45e1c2..a37a4aeb 100644--- a/packages/cli/src/handlers/shared/stream-parsers/openai-responses-sse.ts+++ b/packages/cli/src/handlers/shared/stream-parsers/openai-responses-sse.ts@@ -17,6 +17,7 @@ import { ClaudeSseSink } from "./responses/claude-sse-sink.js";import { type ResponsesRetryOptions, runResponsesTurn } from "./responses/responses-turn.js";export type { ResponsesRetryOptions } from "./responses/responses-turn.js";+export { resolveStallPolicy, type StallPolicy } from "./responses/stall-guard.js";export function createResponsesStreamHandler(c: Context,diff --git a/packages/cli/src/handlers/shared/stream-parsers/responses/responses-leg.ts b/packages/cli/src/handlers/shared/stream-parsers/responses/responses-leg.tsindex 7bbe991b..db317e3c 100644--- a/packages/cli/src/handlers/shared/stream-parsers/responses/responses-leg.ts+++ b/packages/cli/src/handlers/shared/stream-parsers/responses/responses-leg.ts@@ -38,6 +38,7 @@ import {isParallelWrapperName,parseParallelWrapper,} from "./parallel-wrapper.js";+import { type StallClock, StallGuard, isOutputBearingEvent } from "./stall-guard.js";/** One leg's usage, read from `response.completed` / `.incomplete`. */export interface Usage {@@ -74,6 +75,12 @@ export interface ResponsesLegContext {onToolCall?: (name: string, argsJson: string) => string | null | undefined;shouldBufferTool?: (name: string) => boolean;onAssistantText?: (text: string, kind?: "text" | "reasoning") => void;+ /**+ * This leg's stall guard (stall-guard.ts): fires `thresholdMs` after+ * `headersAt` (this leg's response headers, on `clock`) unless an+ * output-bearing event arrived first. Absent or 0 ⇒ unarmed.+ */+ stall?: { thresholdMs: number; headersAt: number; clock?: StallClock };}type FnCall = {@@ -168,6 +175,29 @@ class ResponsesLeg {if (signal.aborted) onAbort();else signal.addEventListener("abort", onAbort, { once: true });+ // A fired guard cancels the reader; the pending read then settles and the+ // leg reports `stall` instead of how the cut-short stream looked.+ let stalledAfterMs: number | undefined;+ const stall = this.ctx.stall;+ const guard =+ stall && stall.thresholdMs > 0+ ? new StallGuard(+ stall.thresholdMs,+ stall.headersAt,+ (afterMs) => {+ stalledAfterMs = afterMs;+ log(`[ResponsesTurn] no output item ${Math.round(afterMs / 1000)}s after headers`);+ reader.cancel().catch(() => {});+ },+ stall.clock+ )+ : undefined;+ const stalled = (): LegOutcome | undefined => {+ if (stalledAfterMs === undefined || signal.aborted) return undefined;+ this.closeAll();+ return { kind: "stall", afterMs: stalledAfterMs };+ };+const decoder = new TextDecoder();let buffer = "";try {@@ -192,7 +222,9 @@ class ResponsesLeg {// event, costs that event only, never the turn.let stop: Stop | undefined;try {- stop = await this.dispatch(JSON.parse(data));+ const event = JSON.parse(data);+ if (guard?.armed && isOutputBearingEvent(event?.type)) guard.disarm();+ stop = await this.dispatch(event);} catch (parseError) {log(`[ResponsesSSE] Parse error: ${parseError}`);continue;@@ -207,6 +239,8 @@ class ResponsesLeg {}} catch (error) {if (signal.aborted) return { kind: "cancelled", usage: this.usage };+ const stalledOutcome = stalled();+ if (stalledOutcome) return stalledOutcome;log(`[ResponsesSSE] Stream error: ${error}`);this.closeReasoning();this.closeText();@@ -218,10 +252,13 @@ class ResponsesLeg {this.closeTools();return { kind: "stream-error", error, openToolBlock, usage: this.usage };} finally {+ guard?.disarm();signal.removeEventListener("abort", onAbort);}if (signal.aborted) return { kind: "cancelled", usage: this.usage };+ const stalledOutcome = stalled();+ if (stalledOutcome) return stalledOutcome;// The stream ended while a wrapper was still held: its arguments were never// completed, so there is nothing valid to expand.diff --git a/packages/cli/src/handlers/shared/stream-parsers/responses/responses-turn.ts b/packages/cli/src/handlers/shared/stream-parsers/responses/responses-turn.tsindex bb6fa9d4..bc4cb139 100644--- a/packages/cli/src/handlers/shared/stream-parsers/responses/responses-turn.ts+++ b/packages/cli/src/handlers/shared/stream-parsers/responses/responses-turn.ts@@ -10,7 +10,9 @@* ── RETRY WITHIN THE TURN ────────────────────────────────────────────────────** About 1% of ChatGPT Codex turns end with a degenerate `parallel` wrapper- * (parallel-wrapper.ts). When the caller passes `retry`, such a leg is retried+ * (parallel-wrapper.ts), usually after a 361-484 s keepalive-only stall. With+ * `retry.stall` armed (stall-guard.ts), a leg with no output item 150 s after+ * its headers is cut off too. When the caller passes `retry`, such a leg is retried* by issuing the same body again while the downstream message stays open; the* next leg continues at the next block index. A leg is issued only while no* tool_use has been emitted in this message:@@ -66,6 +68,7 @@ import {type Usage,parseResponsesLeg,} from "./responses-leg.js";+import { type StallClock, type StallPolicy, systemStallClock } from "./stall-guard.js";export interface ResponsesTurnOptions extends Omit<ResponsesLegContext, "sink"> {/**@@ -98,6 +101,16 @@ export interface ResponsesRetryOptions {issueLeg: (body: unknown | undefined, signal: AbortSignal) => Promise<LegResponse>;/** Retries per leg position. Default {@link MAX_LEG_RETRIES}. */maxLegRetries?: number;+ /**+ * Abort a leg with no output item for too long (stall-guard.ts): attempts+ * 1..n-1 use `thresholdMs`, the final attempt `finalThresholdMs`. Absent or+ * `thresholdMs: 0` ⇒ unarmed.+ */+ stall?: StallPolicy;+ /** When the FIRST leg's response headers arrived (the stall clock's zero). */+ firstHeadersAt?: number;+ /** The stall guard's clock; tests inject a fake one. Default wall clock. */+ stallClock?: StallClock;}/** The in-band text for an in-stream API error; context overflow gets an actionable message. */@@ -210,8 +223,14 @@ class ResponsesTurn {async run(firstResponse: Response): Promise<void> {try {let response = firstResponse;+ let headersAt =+ this.opts.retry?.firstHeadersAt ?? (this.opts.retry?.stallClock ?? systemStallClock).now();while (true) {- const outcome = await parseResponsesLeg(response, { ...this.opts, sink: this.sink });+ const outcome = await parseResponsesLeg(response, {+ ...this.opts,+ sink: this.sink,+ stall: this.stallFor(headersAt),+ });this.legUsages.push("usage" in outcome ? outcome.usage : undefined);const decision = this.retryDecision(outcome);if (decision.action === "finish") {@@ -232,7 +251,8 @@ class ResponsesTurn {}const next = await this.issueRetry(outcome);if (!next) return;- response = next;+ response = next.response;+ headersAt = next.headersAt;}} catch (error) {// Only a throwing callback can land here; the leg itself never throws.@@ -264,7 +284,28 @@ class ResponsesTurn {}/** Issue the next leg; undefined when the turn has ended instead. */- private async issueRetry(outcome: LegOutcome): Promise<Response | undefined> {+ /**+ * This leg's stall guard settings: the final attempt (no retry left) gets the+ * longer threshold, so no attempt is unbounded and the last one still has+ * room for a long reasoning phase.+ */+ private stallFor(headersAt: number): ResponsesLegContext["stall"] {+ const retry = this.opts.retry;+ const policy = retry?.stall;+ if (!retry || !policy || !("finalThresholdMs" in policy) || policy.thresholdMs <= 0) {+ return undefined;+ }+ const isFinal = this.legRetries >= (retry.maxLegRetries ?? MAX_LEG_RETRIES);+ return {+ thresholdMs: isFinal ? policy.finalThresholdMs : policy.thresholdMs,+ headersAt,+ clock: retry.stallClock,+ };+ }++ private async issueRetry(+ outcome: LegOutcome+ ): Promise<{ response: Response; headersAt: number } | undefined> {const retry = this.opts.retry;if (!retry) return undefined;this.legRetries++;@@ -275,7 +316,7 @@ class ResponsesTurn {return undefined;}const leg = await retry.issueLeg(undefined, this.sink.signal);- if (leg.kind === "ok") return leg.response;+ if (leg.kind === "ok") return { response: leg.response, headersAt: leg.headersAt };if (leg.kind === "error") this.sink.endWithError(leg.message);this.reportUsage();return undefined;diff --git a/packages/cli/src/handlers/shared/stream-parsers/responses/stall-guard.ts b/packages/cli/src/handlers/shared/stream-parsers/responses/stall-guard.tsnew file mode 100644index 00000000..e6a2b944--- /dev/null+++ b/packages/cli/src/handlers/shared/stream-parsers/responses/stall-guard.ts@@ -0,0 +1,143 @@+/**+ * Abort a Responses leg that produces no output item for too long.+ *+ * ── THE FAILURE ──────────────────────────────────────────────────────────────+ *+ * About 1% of ChatGPT Codex turns stall: the backend sends `response.created`+ * and keepalives only, for 361-484 s, and then the degenerate `parallel`+ * wrapper arrives in one burst (see parallel-wrapper.ts). claudish's own 1 s+ * pings keep Claude Code waiting the whole time, and `API_TIMEOUT_MS` does not+ * bound a flushed, pinging stream (the leak ran 472 s). The guard turns that+ * wait into a retry of the same body (`responses-turn.ts`).+ *+ * ── THE DATA (all `openai-codex`, ChatGPT backend) ───────────────────────────+ *+ * | sample | n | headers → first output item |+ * |----------------------------------------|-------|-------------------------------|+ * | real sessions, 19 debug logs | 8,239 | p50 1.2 s, p90 3.6 s, |+ * | 2026-07-15 .. 2026-10-16 | | p99 8.5 s, p99.9 28.3 s, |+ * | | | healthy max 61.7 s |+ * | B experiment, healthy legs | 297 | max 7.38 s (from created) |+ * | leaks (B experiment + the leak log) | 4 | 361 s, 456 s, 484.5 s, >150 s |+ *+ * In the 61.7 s legs `response.created` arrived together with the first item+ * (the backend holds the head), so the clock starts at the RESPONSE HEADERS,+ * never at `response.created`. It is armed where the leg's fetch resolved, so+ * the 7b sniff window (up to 12 s) counts against it rather than adding to it.+ *+ * ── THE THRESHOLDS ───────────────────────────────────────────────────────────+ *+ * Attempts 1..n-1: 150 s, 2.4x the healthy max and 2.4x below the fastest+ * leak. Final attempt: 300 s (2x), 4.9x the healthy max and still below the+ * fastest leak, so the last attempt keeps room for an unusually long reasoning+ * phase without waiting out a whole leak. No attempt is unbounded. Not derived+ * from `API_TIMEOUT_MS`, which does not bound a pinging stream.+ *+ * ── THE SCOPE ────────────────────────────────────────────────────────────────+ *+ * The data is from the ChatGPT Codex backend only, so the default scope is+ * exactly that: claudish provider `openai-codex` on the OAuth (subscription)+ * arm. Native `openai@`, the `openai-codex` api-key arm and custom `responses`+ * endpoints are unarmed unless `CLAUDISH_RESPONSES_STALL_MS` is set: a value+ * > 0 arms it everywhere with that threshold (final attempt 2x), `0` disarms it+ * everywhere.+ */++import { log } from "../../../../logger.js";++/** Default threshold for attempts 1..n-1. */+export const DEFAULT_STALL_THRESHOLD_MS = 150_000;++/** Armed: a threshold per attempt, the last attempt's longer. Unarmed: `thresholdMs: 0`. */+export type StallPolicy = { thresholdMs: number; finalThresholdMs: number } | { thresholdMs: 0 };++export const STALL_ENV_VAR = "CLAUDISH_RESPONSES_STALL_MS";++/**+ * The stall policy for one request's connection. `arm` is the credential arm+ * of THIS request (`"oauth"` for the subscription).+ */+export function resolveStallPolicy(i: {+ env?: Record<string, string | undefined>;+ providerName: string;+ arm?: string;+}): StallPolicy {+ const raw = i.env?.[STALL_ENV_VAR];+ if (raw !== undefined && raw.trim() !== "") {+ const ms = Number(raw);+ if (Number.isFinite(ms) && ms >= 0) {+ return ms > 0 ? { thresholdMs: ms, finalThresholdMs: 2 * ms } : { thresholdMs: 0 };+ }+ log(`[ResponsesTurn] ignoring ${STALL_ENV_VAR}=${raw}: not a non-negative number`);+ }+ if (i.providerName === "openai-codex" && i.arm === "oauth") {+ return {+ thresholdMs: DEFAULT_STALL_THRESHOLD_MS,+ finalThresholdMs: 2 * DEFAULT_STALL_THRESHOLD_MS,+ };+ }+ return { thresholdMs: 0 };+}++/** The clock a guard runs on. Tests inject a fake one. */+export interface StallClock {+ now(): number;+ setTimeout(fn: () => void, ms: number): unknown;+ clearTimeout(handle: unknown): void;+}++export const systemStallClock: StallClock = {+ now: () => Date.now(),+ setTimeout: (fn, ms) => setTimeout(fn, ms),+ clearTimeout: (handle) => clearTimeout(handle as ReturnType<typeof setTimeout>),+};++/**+ * Is this Responses event output-bearing? The first one disarms the guard:+ * `output_item.added` of any type, or any `*.delta`. `response.created`,+ * `in_progress`, keepalives and metadata do not count.+ */+export function isOutputBearingEvent(type: unknown): boolean {+ return (+ typeof type === "string" && (type === "response.output_item.added" || type.endsWith(".delta"))+ );+}++/**+ * One leg's guard: fires `onFire(afterMs)` once, `thresholdMs` after+ * `headersAt`, unless `disarm()` ran first.+ */+export class StallGuard {+ private handle: unknown;+ private done = false;++ constructor(+ thresholdMs: number,+ private readonly headersAt: number,+ private readonly onFire: (afterMs: number) => void,+ private readonly clock: StallClock = systemStallClock+ ) {+ if (thresholdMs <= 0) {+ this.done = true;+ return;+ }+ const remaining = Math.max(0, headersAt + thresholdMs - clock.now());+ this.handle = clock.setTimeout(() => this.fire(), remaining);+ }++ get armed(): boolean {+ return !this.done;+ }++ disarm(): void {+ if (this.done) return;+ this.done = true;+ this.clock.clearTimeout(this.handle);+ }++ private fire(): void {+ if (this.done) return;+ this.done = true;+ this.onFire(this.clock.now() - this.headersAt);+ }+}diff --git a/packages/cli/src/providers/transport/openai-codex.ts b/packages/cli/src/providers/transport/openai-codex.tsindex 225af8b0..03874d5e 100644--- a/packages/cli/src/providers/transport/openai-codex.ts+++ b/packages/cli/src/providers/transport/openai-codex.ts@@ -25,7 +25,7 @@ import { normalizeCodexModel } from "../../adapters/codex-api-format.js";import { lookupModelForProvider } from "../../adapters/model-catalog.js";import { credentials } from "../../auth/credentials/authority.js";import { recordSignedArm } from "../../auth/credentials/billing-probe.js";-import type { RequestAuth } from "../../auth/credentials/types.js";+import type { CredentialArm, RequestAuth } from "../../auth/credentials/types.js";import { classifyConnectionError } from "../../handlers/shared/connection-error.js";import { conversationKey } from "./conversation-key.js";import { OpenAIProviderTransport } from "./openai.js";@@ -85,6 +85,15 @@ export class OpenAICodexTransport extends OpenAIProviderTransport {recordSignedArm("openai-codex", signedWithOAuth ? "subscription" : "metered");}+ /**+ * The arm of the artifact `getEndpoint`/`getHeaders` read now: `"oauth"` for+ * the ChatGPT subscription, `"api-key"` (or unknown) otherwise. Read from the+ * artifact itself, never from its truthiness (see refreshAuth above).+ */+ currentAuthArm(): CredentialArm | undefined {+ return this.cachedAuth?.arm;+ }+/*** OAuth tokens only work with chatgpt.com/backend-api (endpoint comes from the* cached auth artifact). API keys use the standard OpenAI endpoint (super).diff --git a/packages/cli/src/providers/transport/types.ts b/packages/cli/src/providers/transport/types.tsindex 440a179e..0d49cf95 100644--- a/packages/cli/src/providers/transport/types.ts+++ b/packages/cli/src/providers/transport/types.ts@@ -5,6 +5,8 @@* Does NOT own: message conversion, tool format, pricing (those are ModelAdapter concerns).*/+import type { CredentialArm } from "../../auth/credentials/types.js";+/** The wire format used for streaming responses */export type StreamFormat =| "openai-sse"@@ -111,6 +113,16 @@ export interface ProviderTransport {*/forceRefreshAuth?(): Promise<void>;+ /**+ * The credential arm of the auth artifact `getEndpoint`/`getHeaders` read+ * right now (`RequestAuth.arm`). Read where the request's endpoint is+ * computed, so it names the arm that signed this request. Absent or+ * undefined ⇒ unknown, which every consumer treats as NOT the subscription.+ * Only `openai-codex` implements it; the Responses stall guard is armed by+ * default only on its `"oauth"` arm.+ */+ currentAuthArm?(): CredentialArm | undefined;+/*** Optional per-provider verdict on whether an error response is TERMINAL* (won't recover on retry) — the transport's own reading of its provider's