Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
fix(llm-proxy): bound upstream streams by a typed idle timeout and total cap
Alfredo Lopez0ec093c74fc0
diff --git a/codex-llm-proxy/src/codexTransport.ts b/codex-llm-proxy/src/codexTransport.tsindex 8364d87ea..f3c9b0145 100644--- a/codex-llm-proxy/src/codexTransport.ts+++ b/codex-llm-proxy/src/codexTransport.ts@@ -15,7 +15,7 @@ import {assertAllowedUpstreamUrl,fetchFrozenOrigin,} from './originPolicy.js'-import { assertBoundedDeadline } from './requestLimits.js'+import { assertBoundedDeadline, assertBoundedIdleTimeout } from './requestLimits.js'import { ToolNameMap } from './toolNameMap.js'import { type SafeUsage, parseSafeUsage } from './usage.js'@@ -56,6 +56,8 @@ export type StreamCodexCompletionInput = {ticket: TransportTicketdeadlineMs?: numbermaxDeadlineMs?: number+ /** Lowers `STREAM_LIMITS.upstreamIdleTimeoutMs`; never raises it. */+ upstreamIdleTimeoutMs?: numbersignal?: AbortSignalredeem: (input: {executionTicket: string@@ -119,6 +121,7 @@ export async function streamCodexCompletion(assertBoundedDeadline(input.deadlineMs ?? request.deadlineMs, input.maxDeadlineMs ?? 300_000),redeemed.transport.maxStreamDurationMs)+ const idleTimeoutMs = assertBoundedIdleTimeout(input.upstreamIdleTimeoutMs)const accessToken = redeemed.accessTokenlet outcome: StreamCodexCompletionResult['outcome'] = 'unknown'@@ -130,6 +133,7 @@ export async function streamCodexCompletion(accessToken,chatgptAccountId: redeemed.chatgptAccountId,deadlineMs,+ idleTimeoutMs,signal: input.signal,fetchFn: input.fetchFn,lookup: input.lookup,@@ -199,14 +203,92 @@ async function readUpstreamStream(input: {accessToken: stringchatgptAccountId?: stringdeadlineMs: number+ idleTimeoutMs: numbersignal?: AbortSignalfetchFn: typeof fetchlookup?: OriginPolicyOptions['lookup']onFrame?: FrameSink}): Promise<StreamCodexCompletionResult> {+ const deadline = new UpstreamDeadline(input.deadlineMs, input.idleTimeoutMs)+ try {+ return await dispatchUpstreamStream(input, deadline)+ } finally {+ deadline.clear()+ }+}++/**+ * Two upstream bounds, each aborting with its own typed reason. The total+ * bound runs from dispatch to the end of the stream. The idle bound runs only+ * while the proxy waits on the upstream (response headers or the next chunk),+ * so a client that is slow to drain frames never counts as upstream silence.+ */+class UpstreamDeadline {+ private readonly controller = new AbortController()+ private readonly started = Date.now()+ private readonly total: ReturnType<typeof setTimeout>+ private idle: ReturnType<typeof setTimeout> | undefined++ constructor(+ private readonly totalMs: number,+ private readonly idleMs: number+ ) {+ this.total = setTimeout(() => {+ this.controller.abort(+ new CodexTransportError(+ 'stream_duration_exceeded',+ 'upstream stream exceeded maxStreamDurationMs',+ { limitMs: this.totalMs, elapsedMs: Date.now() - this.started }+ )+ )+ }, totalMs)+ }++ get signal(): AbortSignal {+ return this.controller.signal+ }++ /**+ * Wait on one upstream operation under the idle timeout. The wait also ends+ * as soon as `signal` aborts, rejecting with its reason, because the+ * operation itself may not observe the signal (a stalled body read).+ */+ async waitUpstream<T>(operation: Promise<T>, signal: AbortSignal): Promise<T> {+ signal.throwIfAborted()+ this.idle = setTimeout(() => {+ this.controller.abort(+ new CodexTransportError('provider_unavailable', 'upstream stream idle timeout', {+ idleTimeoutMs: this.idleMs,+ elapsedMs: Date.now() - this.started,+ })+ )+ }, this.idleMs)+ let onAbort: (() => void) | undefined+ const aborted = new Promise<never>((_, reject) => {+ onAbort = () => reject(signal.reason)+ signal.addEventListener('abort', onAbort, { once: true })+ })+ try {+ return await Promise.race([operation, aborted])+ } finally {+ clearTimeout(this.idle)+ this.idle = undefined+ if (onAbort) signal.removeEventListener('abort', onAbort)+ }+ }++ clear(): void {+ clearTimeout(this.total)+ if (this.idle !== undefined) clearTimeout(this.idle)+ }+}++async function dispatchUpstreamStream(+ input: Parameters<typeof readUpstreamStream>[0],+ deadline: UpstreamDeadline+): Promise<StreamCodexCompletionResult> {const url = assertAllowedUpstreamUrl(CODEX_COMPLETIONS_ORIGIN, 'completions')- const timeout = AbortSignal.timeout(input.deadlineMs)- const signal = input.signal ? AbortSignal.any([input.signal, timeout]) : timeout+ const signal = input.signal ? AbortSignal.any([input.signal, deadline.signal]) : deadline.signalconst headers = chatgptUpstreamHeaders(input.accessToken, {'content-type': 'application/json',accept: 'text/event-stream',@@ -225,17 +307,20 @@ async function readUpstreamStream(input: {message.role === 'assistant' ? (message.toolCalls ?? []).map(call => call.name) : []),])- const response = await fetchFrozenOrigin({- url,- fetchFn: input.fetchFn,- lookup: input.lookup,- init: {- method: 'POST',- signal,- headers,- body: JSON.stringify(toUpstreamPayload(input.request, names)),- },- })+ const response = await deadline.waitUpstream(+ fetchFrozenOrigin({+ url,+ fetchFn: input.fetchFn,+ lookup: input.lookup,+ init: {+ method: 'POST',+ signal,+ headers,+ body: JSON.stringify(toUpstreamPayload(input.request, names)),+ },+ }),+ signal+ )if (!response.ok || !response.body) {logger.warn({@@ -257,7 +342,7 @@ async function readUpstreamStream(input: {}throw new CodexTransportError('provider_unavailable', 'upstream completion failed')}- return consumeSse(response.body, input.onFrame, signal, names)+ return consumeSse(response.body, input.onFrame, signal, names, deadline)}function toUpstreamPayload(@@ -333,7 +418,8 @@ async function consumeSse(body: ReadableStream<Uint8Array>,onFrame: FrameSink | undefined,signal: AbortSignal,- names: ToolNameMap+ names: ToolNameMap,+ deadline: UpstreamDeadline): Promise<StreamCodexCompletionResult> {const reader = body.getReader()const decoder = new TextDecoder()@@ -373,8 +459,10 @@ async function consumeSse(let usage: SafeUsage | undefinedconst maxSseBufferBytes = 1_048_576try {- while (!signal.aborted) {- const { done, value } = await reader.read()+ // An abort ends the read with the signal's reason: the caller's own abort+ // becomes `canceled`, a deadline abort surfaces as its typed error.+ for (;;) {+ const { done, value } = await deadline.waitUpstream(reader.read(), signal)if (done) breakbuffer += decoder.decode(value, { stream: true }).replace(/\r\n/g, '\n').replace(/\r/g, '\n')if (buffer.length > maxSseBufferBytes) {@@ -389,7 +477,7 @@ async function consumeSse(if (mapped.completed) completed = trueif (mapped.failed) failed = true}- if (signal.aborted) break+ signal.throwIfAborted()}buffer += decoder.decode().replace(/\r\n/g, '\n').replace(/\r/g, '\n')if (buffer.trim()) {@@ -403,7 +491,11 @@ async function consumeSse(await reader.cancel().catch(() => undefined)throw error} finally {- reader.releaseLock()+ try {+ reader.releaseLock()+ } catch {+ // A read that lost the race to an abort may still hold the lock after cancel.+ }}for (const call of pending.values()) {if (call.emitted) continue@@ -411,7 +503,7 @@ async function consumeSse(await acceptFrame({ type: 'tool_call', id: call.id, name: call.name, arguments: args })call.emitted = true}- if (signal.aborted) return { outcome: 'canceled', usage }+ signal.throwIfAborted()if (failed) {throw new CodexTransportError('provider_unavailable', 'upstream response failed')}diff --git a/codex-llm-proxy/src/requestLimits.ts b/codex-llm-proxy/src/requestLimits.tsindex 550e48cde..c56762de5 100644--- a/codex-llm-proxy/src/requestLimits.ts+++ b/codex-llm-proxy/src/requestLimits.ts@@ -4,6 +4,9 @@ export const STREAM_LIMITS = {maxConcurrentStreams: 8,maxQueuedRequests: 16,maxStreamDurationMs: 300_000,+ // Longest silence tolerated while waiting on the upstream (response headers+ // or the next SSE chunk). Matches the Codex CLI stream idle timeout.+ upstreamIdleTimeoutMs: 300_000,} as constexport class RequestLimitError extends Error {@@ -22,6 +25,14 @@ export function assertBoundedDeadline(deadlineMs: number | undefined, maxDeadlinreturn Math.min(requested, maxDeadlineMs, STREAM_LIMITS.maxStreamDurationMs, CONTRACT_LIMITS.maxDeadlineMs)}+export function assertBoundedIdleTimeout(idleTimeoutMs: number | undefined): number {+ const requested = idleTimeoutMs ?? STREAM_LIMITS.upstreamIdleTimeoutMs+ if (!Number.isInteger(requested) || requested <= 0) {+ throw new RequestLimitError('upstream idle timeout is invalid')+ }+ return Math.min(requested, STREAM_LIMITS.upstreamIdleTimeoutMs)+}+export class StreamGate {private running = 0private queued = 0diff --git a/codex-llm-proxy/test/codexTransport.conformance.test.ts b/codex-llm-proxy/test/codexTransport.conformance.test.tsindex d8e8dc04a..2b064a39e 100644--- a/codex-llm-proxy/test/codexTransport.conformance.test.ts+++ b/codex-llm-proxy/test/codexTransport.conformance.test.ts@@ -745,13 +745,19 @@ describe('streamCodexCompletion', () => {.fn().mockRejectedValueOnce(new Error('finalize 500')).mockResolvedValueOnce({ providerAttemptId: 'att-1', outcome: 'canceled', duplicate: true })+ let upstreamCanceled = falseconst fetchFn = vi.fn(async () => {const stream = new ReadableStream({start(controller) {controller.enqueue(new TextEncoder().encode('data: {"type":"response.output_text.delta","delta":"x"}\n\n'))- setTimeout(() => controller.close(), 20)+ setTimeout(() => {+ if (!upstreamCanceled) controller.close()+ }, 20)+ },+ cancel() {+ upstreamCanceled = true},})return new Response(stream, { headers: { 'content-type': 'text/event-stream' } })@@ -781,6 +787,7 @@ describe('streamCodexCompletion', () => {})expect(frames[0]).toEqual({ type: 'text', text: 'x' })expect(result.outcome).toBe('canceled')+ expect(upstreamCanceled).toBe(true)expect(finalize).toHaveBeenCalledTimes(2)expect(finalize.mock.calls[0]?.[0]?.receipt.outcome).toBe('canceled')})@@ -1227,3 +1234,144 @@ describe('streamCodexCompletion', () => {])})})++const TEXT_DELTA = 'data: {"type":"response.output_text.delta","delta":"x"}\n\n'+const COMPLETED = 'data: {"type":"response.completed","response":{"usage":{}}}\n\n'++/**+ * An upstream body that ignores `init.signal`, like a socket that stays open+ * while sending nothing. Only a transport that races each read against its own+ * timers can end an attempt reading from it.+ */+function upstreamBody(chunks: Array<{ afterMs: number; text: string } | 'stall'>): {+ fetchFn: typeof fetch+ cancel: ReturnType<typeof vi.fn>+} {+ const encoder = new TextEncoder()+ const cancel = vi.fn()+ const fetchFn = vi.fn(async () => {+ let index = 0+ const body = new ReadableStream<Uint8Array>(+ {+ async pull(controller) {+ const next = chunks[index]+ index += 1+ if (next === undefined) {+ controller.close()+ return+ }+ if (next === 'stall') {+ await new Promise<never>(() => undefined)+ return+ }+ await new Promise(resolve => setTimeout(resolve, next.afterMs))+ controller.enqueue(encoder.encode(next.text))+ },+ cancel,+ },+ { highWaterMark: 0 }+ )+ return new Response(body, { status: 200, headers: { 'content-type': 'text/event-stream' } })+ }) as unknown as typeof fetch+ return { fetchFn, cancel }+}++function attemptInput(+ overrides: Partial<StreamCodexCompletionInput>+): StreamCodexCompletionInput & { finalize: ReturnType<typeof vi.fn> } {+ const finalize = vi.fn(async (_input: Parameters<StreamCodexCompletionInput['finalize']>[0]) => ({+ providerAttemptId: 'att-timeouts',+ outcome: 'error' as const,+ duplicate: false,+ }))+ return {+ executionTicket: 'ticket-timeouts',+ requestHash: REQUEST_HASH,+ request: REQUEST,+ ticket: {+ jti: 'jti-timeouts',+ hostRef: 'research-host',+ model: REQUEST.model,+ requestHash: REQUEST_HASH,+ providerAttemptId: 'att-timeouts',+ },+ maxDeadlineMs: 300_000,+ redeem: async () => redeemSuccess(),+ finalize,+ fetchFn: upstreamBody([]).fetchFn,+ lookup: async () => [{ address: '1.2.3.4', family: 4 }],+ ...overrides,+ } as StreamCodexCompletionInput & { finalize: ReturnType<typeof vi.fn> }+}++function finalizedOutcome(finalize: ReturnType<typeof vi.fn>): unknown {+ expect(finalize).toHaveBeenCalledTimes(1)+ return (finalize.mock.calls[0]?.[0] as { receipt: { outcome: string } }).receipt.outcome+}++describe('streamCodexCompletion upstream timeouts', () => {+ it('ends a stalled upstream stream at the idle timeout with a typed provider_unavailable', async () => {+ const upstream = upstreamBody([{ afterMs: 0, text: TEXT_DELTA }, 'stall'])+ const input = attemptInput({ fetchFn: upstream.fetchFn, upstreamIdleTimeoutMs: 50 })+ const frames: unknown[] = []+ const error = await streamCodexCompletion({ ...input, onFrame: frame => void frames.push(frame) }).then(+ () => undefined,+ (err: unknown) => err+ )+ expect(frames).toEqual([{ type: 'text', text: 'x' }])+ expect(error).toBeInstanceOf(CodexTransportError)+ expect(error).toMatchObject({+ code: 'provider_unavailable',+ message: 'upstream stream idle timeout',+ details: { idleTimeoutMs: 50 },+ })+ expect(finalizedOutcome(input.finalize)).toBe('error')+ expect(upstream.cancel).toHaveBeenCalledTimes(1)+ }, 2_000)++ it('keeps the attempt alive while upstream chunks arrive inside the idle window', async () => {+ const chunks = Array.from({ length: 6 }, () => ({ afterMs: 30, text: TEXT_DELTA }))+ const upstream = upstreamBody([...chunks, { afterMs: 30, text: COMPLETED }])+ const input = attemptInput({ fetchFn: upstream.fetchFn, upstreamIdleTimeoutMs: 50 })+ const result = await streamCodexCompletion(input)+ expect(result.outcome).toBe('success')+ expect(finalizedOutcome(input.finalize)).toBe('success')+ }, 2_000)++ it('ends a still-active upstream stream at the total cap with stream_duration_exceeded', async () => {+ const endless = Array.from({ length: 200 }, () => ({ afterMs: 10, text: TEXT_DELTA }))+ const upstream = upstreamBody(endless)+ const input = attemptInput({+ fetchFn: upstream.fetchFn,+ maxDeadlineMs: 150,+ upstreamIdleTimeoutMs: 1_000,+ })+ const error = await streamCodexCompletion(input).then(+ () => undefined,+ (err: unknown) => err+ )+ expect(error).toBeInstanceOf(CodexTransportError)+ expect(error).toMatchObject({+ code: 'stream_duration_exceeded',+ message: 'upstream stream exceeded maxStreamDurationMs',+ details: { limitMs: 150 },+ })+ expect(finalizedOutcome(input.finalize)).toBe('error')+ }, 2_000)++ it('reports a client abort during a stalled stream as canceled, not as a timeout', async () => {+ const upstream = upstreamBody([{ afterMs: 0, text: TEXT_DELTA }, 'stall'])+ const abort = new AbortController()+ const input = attemptInput({+ fetchFn: upstream.fetchFn,+ signal: abort.signal,+ maxDeadlineMs: 1_000,+ upstreamIdleTimeoutMs: 1_000,+ })+ setTimeout(() => abort.abort(), 50)+ const result = await streamCodexCompletion(input)+ expect(result.outcome).toBe('canceled')+ expect(finalizedOutcome(input.finalize)).toBe('canceled')+ expect(upstream.cancel).toHaveBeenCalledTimes(1)+ }, 2_000)+})diff --git a/codex-llm-proxy/test/streamLimitsFreeze.test.ts b/codex-llm-proxy/test/streamLimitsFreeze.test.tsindex 60762e52b..0821c473b 100644--- a/codex-llm-proxy/test/streamLimitsFreeze.test.ts+++ b/codex-llm-proxy/test/streamLimitsFreeze.test.ts@@ -9,7 +9,7 @@ const fixturePath = new URL(describe('codex-subscription stream limits freeze', () => {it('pins the proxy STREAM_LIMITS to the limits the fixture publishes', () => {- // StreamGate and the stream deadline enforce these three published bounds+ // StreamGate and the upstream deadlines enforce these published bounds// from the proxy's own constant, not from the contract package, so the// Codex freeze gate in tests/e2e cannot see them.const { limits } = JSON.parse(readFileSync(fixturePath, 'utf8')) as {@@ -20,6 +20,7 @@ describe('codex-subscription stream limits freeze', () => {'maxConcurrentStreams','maxQueuedRequests','maxStreamDurationMs',+ 'upstreamIdleTimeoutMs',])for (const name of Object.keys(streamLimits)) {expect({ [name]: limits[name] }).toEqual({ [name]: streamLimits[name] })diff --git a/docs/architecture/codex-subscription-transport-contract.md b/docs/architecture/codex-subscription-transport-contract.mdindex 4c4ab7a30..48e56e018 100644--- a/docs/architecture/codex-subscription-transport-contract.md+++ b/docs/architecture/codex-subscription-transport-contract.md@@ -49,17 +49,18 @@ OAuth scopes: `openid`, `profile`, `email`, `offline_access`.All values are finite and greater than zero. `maxRetriesPerAttempt` is `1`:one physical execution per ticket. A retry or fallback must mint a new attempt.-| Limit | Value |-| -------------------- | ------- |-| maxRequestBodyBytes | 1048576 |-| maxMessages | 1024 |-| maxToolCalls | 256 |-| maxOutputTokens | 16384 |-| maxStreamDurationMs | 300000 |-| maxDeadlineMs | 300000 |-| maxConcurrentStreams | 8 |-| maxQueuedRequests | 16 |-| maxRetriesPerAttempt | 1 |+| Limit | Value |+| --------------------- | ------- |+| maxRequestBodyBytes | 1048576 |+| maxMessages | 1024 |+| maxToolCalls | 256 |+| maxOutputTokens | 16384 |+| maxStreamDurationMs | 300000 |+| maxDeadlineMs | 300000 |+| maxConcurrentStreams | 8 |+| maxQueuedRequests | 16 |+| upstreamIdleTimeoutMs | 300000 |+| maxRetriesPerAttempt | 1 |Tool definitions have no independent count ceiling in the Evenfire requestcontract. The entire serialized request, including all definitions, remainsdiff --git a/grok-llm-proxy/src/grokTransport.ts b/grok-llm-proxy/src/grokTransport.tsindex 3e6983ca4..7314d2aec 100644--- a/grok-llm-proxy/src/grokTransport.ts+++ b/grok-llm-proxy/src/grokTransport.ts@@ -15,7 +15,7 @@ import {assertAllowedUpstreamUrl,fetchFrozenOrigin,} from './originPolicy.js'-import { assertBoundedDeadline } from './requestLimits.js'+import { assertBoundedDeadline, assertBoundedIdleTimeout } from './requestLimits.js'import { ToolNameMap } from './toolNameMap.js'import { type SafeUsage, parseSafeUsage } from './usage.js'@@ -83,6 +83,8 @@ export type StreamGrokCompletionInput = {ticket: TransportTicketdeadlineMs?: numbermaxDeadlineMs?: number+ /** Lowers `STREAM_LIMITS.upstreamIdleTimeoutMs`; never raises it. */+ upstreamIdleTimeoutMs?: numbersignal?: AbortSignalredeem: (input: {executionTicket: string@@ -146,6 +148,7 @@ export async function streamGrokCompletion(assertBoundedDeadline(input.deadlineMs ?? request.deadlineMs, input.maxDeadlineMs ?? 300_000),redeemed.transport.maxStreamDurationMs)+ const idleTimeoutMs = assertBoundedIdleTimeout(input.upstreamIdleTimeoutMs)const accessToken = redeemed.accessTokenlet outcome: StreamGrokCompletionResult['outcome'] = 'unknown'@@ -156,6 +159,7 @@ export async function streamGrokCompletion(request,accessToken,deadlineMs,+ idleTimeoutMs,signal: input.signal,fetchFn: input.fetchFn,lookup: input.lookup,@@ -251,14 +255,92 @@ async function readUpstreamStream(input: {request: GrokCompletionRequestV1accessToken: stringdeadlineMs: number+ idleTimeoutMs: numbersignal?: AbortSignalfetchFn: typeof fetchlookup?: OriginPolicyOptions['lookup']onFrame?: FrameSink}): Promise<StreamGrokCompletionResult> {+ const deadline = new UpstreamDeadline(input.deadlineMs, input.idleTimeoutMs)+ try {+ return await dispatchUpstreamStream(input, deadline)+ } finally {+ deadline.clear()+ }+}++/**+ * Two upstream bounds, each aborting with its own typed reason. The total+ * bound runs from dispatch to the end of the stream. The idle bound runs only+ * while the proxy waits on the upstream (response headers or the next chunk),+ * so a client that is slow to drain frames never counts as upstream silence.+ */+class UpstreamDeadline {+ private readonly controller = new AbortController()+ private readonly started = Date.now()+ private readonly total: ReturnType<typeof setTimeout>+ private idle: ReturnType<typeof setTimeout> | undefined++ constructor(+ private readonly totalMs: number,+ private readonly idleMs: number+ ) {+ this.total = setTimeout(() => {+ this.controller.abort(+ new GrokTransportError(+ 'stream_duration_exceeded',+ 'upstream stream exceeded maxStreamDurationMs',+ { limitMs: this.totalMs, elapsedMs: Date.now() - this.started }+ )+ )+ }, totalMs)+ }++ get signal(): AbortSignal {+ return this.controller.signal+ }++ /**+ * Wait on one upstream operation under the idle timeout. The wait also ends+ * as soon as `signal` aborts, rejecting with its reason, because the+ * operation itself may not observe the signal (a stalled body read).+ */+ async waitUpstream<T>(operation: Promise<T>, signal: AbortSignal): Promise<T> {+ signal.throwIfAborted()+ this.idle = setTimeout(() => {+ this.controller.abort(+ new GrokTransportError('provider_unavailable', 'upstream stream idle timeout', {+ idleTimeoutMs: this.idleMs,+ elapsedMs: Date.now() - this.started,+ })+ )+ }, this.idleMs)+ let onAbort: (() => void) | undefined+ const aborted = new Promise<never>((_, reject) => {+ onAbort = () => reject(signal.reason)+ signal.addEventListener('abort', onAbort, { once: true })+ })+ try {+ return await Promise.race([operation, aborted])+ } finally {+ clearTimeout(this.idle)+ this.idle = undefined+ if (onAbort) signal.removeEventListener('abort', onAbort)+ }+ }++ clear(): void {+ clearTimeout(this.total)+ if (this.idle !== undefined) clearTimeout(this.idle)+ }+}++async function dispatchUpstreamStream(+ input: Parameters<typeof readUpstreamStream>[0],+ deadline: UpstreamDeadline+): Promise<StreamGrokCompletionResult> {const url = assertAllowedUpstreamUrl(GROK_COMPLETIONS_ORIGIN, 'completions')- const timeout = AbortSignal.timeout(input.deadlineMs)- const signal = input.signal ? AbortSignal.any([input.signal, timeout]) : timeout+ const signal = input.signal ? AbortSignal.any([input.signal, deadline.signal]) : deadline.signalconst headers = grokUpstreamHeaders(input.accessToken, {'content-type': 'application/json',accept: 'text/event-stream',@@ -269,17 +351,20 @@ async function readUpstreamStream(input: {message.role === 'assistant' ? (message.toolCalls ?? []).map(call => call.name) : []),])- const response = await fetchFrozenOrigin({- url,- fetchFn: input.fetchFn,- lookup: input.lookup,- init: {- method: 'POST',- signal,- headers,- body: JSON.stringify(toUpstreamPayload(input.request, names)),- },- })+ const response = await deadline.waitUpstream(+ fetchFrozenOrigin({+ url,+ fetchFn: input.fetchFn,+ lookup: input.lookup,+ init: {+ method: 'POST',+ signal,+ headers,+ body: JSON.stringify(toUpstreamPayload(input.request, names)),+ },+ }),+ signal+ )if (!response.ok || !response.body) {logger.warn({@@ -316,7 +401,7 @@ async function readUpstreamStream(input: {}throw new GrokTransportError('provider_unavailable', 'upstream completion failed')}- return consumeSse(response.body, input.onFrame, signal, names)+ return consumeSse(response.body, input.onFrame, signal, names, deadline)}function toUpstreamPayload(@@ -392,7 +477,8 @@ async function consumeSse(body: ReadableStream<Uint8Array>,onFrame: FrameSink | undefined,signal: AbortSignal,- names: ToolNameMap+ names: ToolNameMap,+ deadline: UpstreamDeadline): Promise<StreamGrokCompletionResult> {const reader = body.getReader()const decoder = new TextDecoder()@@ -433,8 +519,10 @@ async function consumeSse(let usage: SafeUsage | undefinedconst maxSseBufferBytes = 1_048_576try {- while (!signal.aborted) {- const { done, value } = await reader.read()+ // An abort ends the read with the signal's reason: the caller's own abort+ // becomes `canceled`, a deadline abort surfaces as its typed error.+ for (;;) {+ const { done, value } = await deadline.waitUpstream(reader.read(), signal)if (done) breakbuffer += decoder.decode(value, { stream: true }).replace(/\r\n/g, '\n').replace(/\r/g, '\n')if (buffer.length > maxSseBufferBytes) {@@ -449,7 +537,7 @@ async function consumeSse(if (mapped.completed) completed = trueif (mapped.failed) failed = true}- if (signal.aborted) break+ signal.throwIfAborted()}buffer += decoder.decode().replace(/\r\n/g, '\n').replace(/\r/g, '\n')if (buffer.trim()) {@@ -463,7 +551,11 @@ async function consumeSse(await reader.cancel().catch(() => undefined)throw error} finally {- reader.releaseLock()+ try {+ reader.releaseLock()+ } catch {+ // A read that lost the race to an abort may still hold the lock after cancel.+ }}for (const call of pending.values()) {if (call.emitted) continue@@ -471,7 +563,7 @@ async function consumeSse(await acceptFrame({ type: 'tool_call', id: call.id, name: call.name, arguments: args })call.emitted = true}- if (signal.aborted) return { outcome: 'canceled', usage }+ signal.throwIfAborted()if (failed) {throw new GrokTransportError('provider_unavailable', 'upstream response failed')}diff --git a/grok-llm-proxy/src/requestLimits.ts b/grok-llm-proxy/src/requestLimits.tsindex ba22dd3b7..5da88d377 100644--- a/grok-llm-proxy/src/requestLimits.ts+++ b/grok-llm-proxy/src/requestLimits.ts@@ -4,6 +4,9 @@ export const STREAM_LIMITS = {maxConcurrentStreams: 8,maxQueuedRequests: 16,maxStreamDurationMs: 300_000,+ // Longest silence tolerated while waiting on the upstream (response headers+ // or the next SSE chunk). Matches the Grok Build inference idle timeout.+ upstreamIdleTimeoutMs: 600_000,} as constexport class RequestLimitError extends Error {@@ -30,6 +33,14 @@ export function assertBoundedDeadline()}+export function assertBoundedIdleTimeout(idleTimeoutMs: number | undefined): number {+ const requested = idleTimeoutMs ?? STREAM_LIMITS.upstreamIdleTimeoutMs+ if (!Number.isInteger(requested) || requested <= 0) {+ throw new RequestLimitError('upstream idle timeout is invalid')+ }+ return Math.min(requested, STREAM_LIMITS.upstreamIdleTimeoutMs)+}+export class StreamGate {private running = 0private queued = 0diff --git a/grok-llm-proxy/test/contractFreeze.test.ts b/grok-llm-proxy/test/contractFreeze.test.tsindex bfafd9930..92d92c507 100644--- a/grok-llm-proxy/test/contractFreeze.test.ts+++ b/grok-llm-proxy/test/contractFreeze.test.ts@@ -35,6 +35,7 @@ const FIXTURE_LIMIT_KEYS = ['maxDeadlineMs','maxConcurrentStreams','maxQueuedRequests',+ 'upstreamIdleTimeoutMs','maxRetriesPerAttempt',] as const@@ -151,7 +152,7 @@ describe('grok-subscription contract freeze', () => {})it('pins the proxy STREAM_LIMITS to the limits the fixture publishes', () => {- // StreamGate and the stream deadline enforce these three published bounds+ // StreamGate and the upstream deadlines enforce these published bounds// from the proxy's own constant, not from the contract package.const { limits } = readFixture()const streamLimits: Record<string, number> = { ...STREAM_LIMITS }@@ -159,6 +160,7 @@ describe('grok-subscription contract freeze', () => {'maxConcurrentStreams','maxQueuedRequests','maxStreamDurationMs',+ 'upstreamIdleTimeoutMs',])for (const name of Object.keys(streamLimits)) {expect({ [name]: limits[name] }).toEqual({ [name]: streamLimits[name] })diff --git a/grok-llm-proxy/test/grokTransport.conformance.test.ts b/grok-llm-proxy/test/grokTransport.conformance.test.tsindex 5b9213762..60fbcc063 100644--- a/grok-llm-proxy/test/grokTransport.conformance.test.ts+++ b/grok-llm-proxy/test/grokTransport.conformance.test.ts@@ -870,13 +870,19 @@ describe('streamGrokCompletion', () => {.fn().mockRejectedValueOnce(new Error('finalize 500')).mockResolvedValueOnce({ providerAttemptId: 'att-1', outcome: 'canceled', duplicate: true })+ let upstreamCanceled = falseconst fetchFn = vi.fn(async () => {const stream = new ReadableStream({start(controller) {controller.enqueue(new TextEncoder().encode('data: {"type":"response.output_text.delta","delta":"x"}\n\n'))- setTimeout(() => controller.close(), 20)+ setTimeout(() => {+ if (!upstreamCanceled) controller.close()+ }, 20)+ },+ cancel() {+ upstreamCanceled = true},})return new Response(stream, { headers: { 'content-type': 'text/event-stream' } })@@ -906,6 +912,7 @@ describe('streamGrokCompletion', () => {})expect(frames[0]).toEqual({ type: 'text', text: 'x' })expect(result.outcome).toBe('canceled')+ expect(upstreamCanceled).toBe(true)expect(finalize).toHaveBeenCalledTimes(2)expect(finalize.mock.calls[0]?.[0]?.receipt.outcome).toBe('canceled')})@@ -1358,3 +1365,144 @@ describe('streamGrokCompletion', () => {])})})++const TEXT_DELTA = 'data: {"type":"response.output_text.delta","delta":"x"}\n\n'+const COMPLETED = 'data: {"type":"response.completed","response":{"usage":{}}}\n\n'++/**+ * An upstream body that ignores `init.signal`, like a socket that stays open+ * while sending nothing. Only a transport that races each read against its own+ * timers can end an attempt reading from it.+ */+function upstreamBody(chunks: Array<{ afterMs: number; text: string } | 'stall'>): {+ fetchFn: typeof fetch+ cancel: ReturnType<typeof vi.fn>+} {+ const encoder = new TextEncoder()+ const cancel = vi.fn()+ const fetchFn = vi.fn(async () => {+ let index = 0+ const body = new ReadableStream<Uint8Array>(+ {+ async pull(controller) {+ const next = chunks[index]+ index += 1+ if (next === undefined) {+ controller.close()+ return+ }+ if (next === 'stall') {+ await new Promise<never>(() => undefined)+ return+ }+ await new Promise(resolve => setTimeout(resolve, next.afterMs))+ controller.enqueue(encoder.encode(next.text))+ },+ cancel,+ },+ { highWaterMark: 0 }+ )+ return new Response(body, { status: 200, headers: { 'content-type': 'text/event-stream' } })+ }) as unknown as typeof fetch+ return { fetchFn, cancel }+}++function attemptInput(+ overrides: Partial<StreamGrokCompletionInput>+): StreamGrokCompletionInput & { finalize: ReturnType<typeof vi.fn> } {+ const finalize = vi.fn(async (_input: Parameters<StreamGrokCompletionInput['finalize']>[0]) => ({+ providerAttemptId: 'att-timeouts',+ outcome: 'error' as const,+ duplicate: false,+ }))+ return {+ executionTicket: 'ticket-timeouts',+ requestHash: REQUEST_HASH,+ request: REQUEST,+ ticket: {+ jti: 'jti-timeouts',+ hostRef: 'research-host',+ model: REQUEST.model,+ requestHash: REQUEST_HASH,+ providerAttemptId: 'att-timeouts',+ },+ maxDeadlineMs: 300_000,+ redeem: async () => redeemSuccess(),+ finalize,+ fetchFn: upstreamBody([]).fetchFn,+ lookup: async () => [{ address: '1.2.3.4', family: 4 }],+ ...overrides,+ } as StreamGrokCompletionInput & { finalize: ReturnType<typeof vi.fn> }+}++function finalizedOutcome(finalize: ReturnType<typeof vi.fn>): unknown {+ expect(finalize).toHaveBeenCalledTimes(1)+ return (finalize.mock.calls[0]?.[0] as { receipt: { outcome: string } }).receipt.outcome+}++describe('streamGrokCompletion upstream timeouts', () => {+ it('ends a stalled upstream stream at the idle timeout with a typed provider_unavailable', async () => {+ const upstream = upstreamBody([{ afterMs: 0, text: TEXT_DELTA }, 'stall'])+ const input = attemptInput({ fetchFn: upstream.fetchFn, upstreamIdleTimeoutMs: 50 })+ const frames: unknown[] = []+ const error = await streamGrokCompletion({ ...input, onFrame: frame => void frames.push(frame) }).then(+ () => undefined,+ (err: unknown) => err+ )+ expect(frames).toEqual([{ type: 'text', text: 'x' }])+ expect(error).toBeInstanceOf(GrokTransportError)+ expect(error).toMatchObject({+ code: 'provider_unavailable',+ message: 'upstream stream idle timeout',+ details: { idleTimeoutMs: 50 },+ })+ expect(finalizedOutcome(input.finalize)).toBe('error')+ expect(upstream.cancel).toHaveBeenCalledTimes(1)+ }, 2_000)++ it('keeps the attempt alive while upstream chunks arrive inside the idle window', async () => {+ const chunks = Array.from({ length: 6 }, () => ({ afterMs: 30, text: TEXT_DELTA }))+ const upstream = upstreamBody([...chunks, { afterMs: 30, text: COMPLETED }])+ const input = attemptInput({ fetchFn: upstream.fetchFn, upstreamIdleTimeoutMs: 50 })+ const result = await streamGrokCompletion(input)+ expect(result.outcome).toBe('success')+ expect(finalizedOutcome(input.finalize)).toBe('success')+ }, 2_000)++ it('ends a still-active upstream stream at the total cap with stream_duration_exceeded', async () => {+ const endless = Array.from({ length: 200 }, () => ({ afterMs: 10, text: TEXT_DELTA }))+ const upstream = upstreamBody(endless)+ const input = attemptInput({+ fetchFn: upstream.fetchFn,+ maxDeadlineMs: 150,+ upstreamIdleTimeoutMs: 1_000,+ })+ const error = await streamGrokCompletion(input).then(+ () => undefined,+ (err: unknown) => err+ )+ expect(error).toBeInstanceOf(GrokTransportError)+ expect(error).toMatchObject({+ code: 'stream_duration_exceeded',+ message: 'upstream stream exceeded maxStreamDurationMs',+ details: { limitMs: 150 },+ })+ expect(finalizedOutcome(input.finalize)).toBe('error')+ }, 2_000)++ it('reports a client abort during a stalled stream as canceled, not as a timeout', async () => {+ const upstream = upstreamBody([{ afterMs: 0, text: TEXT_DELTA }, 'stall'])+ const abort = new AbortController()+ const input = attemptInput({+ fetchFn: upstream.fetchFn,+ signal: abort.signal,+ maxDeadlineMs: 1_000,+ upstreamIdleTimeoutMs: 1_000,+ })+ setTimeout(() => abort.abort(), 50)+ const result = await streamGrokCompletion(input)+ expect(result.outcome).toBe('canceled')+ expect(finalizedOutcome(input.finalize)).toBe('canceled')+ expect(upstream.cancel).toHaveBeenCalledTimes(1)+ }, 2_000)+})diff --git a/tests/e2e/fixtures/codex-subscription/sanitized-upstream-contract.json b/tests/e2e/fixtures/codex-subscription/sanitized-upstream-contract.jsonindex 2a0339ba4..456b4755e 100644--- a/tests/e2e/fixtures/codex-subscription/sanitized-upstream-contract.json+++ b/tests/e2e/fixtures/codex-subscription/sanitized-upstream-contract.json@@ -48,6 +48,7 @@"maxDeadlineMs": 300000,"maxConcurrentStreams": 8,"maxQueuedRequests": 16,+ "upstreamIdleTimeoutMs": 300000,"maxRetriesPerAttempt": 1},"errorTaxonomy": [diff --git a/tests/e2e/fixtures/grok-subscription/sanitized-upstream-contract.json b/tests/e2e/fixtures/grok-subscription/sanitized-upstream-contract.jsonindex b6b27622c..7bfa5aca8 100644--- a/tests/e2e/fixtures/grok-subscription/sanitized-upstream-contract.json+++ b/tests/e2e/fixtures/grok-subscription/sanitized-upstream-contract.json@@ -62,6 +62,7 @@"maxDeadlineMs": 300000,"maxConcurrentStreams": 8,"maxQueuedRequests": 16,+ "upstreamIdleTimeoutMs": 600000,"maxRetriesPerAttempt": 1},"errorTaxonomy": [diff --git a/tests/e2e/fixtures/package-lock.json b/tests/e2e/fixtures/package-lock.jsonindex 0710b38c1..85e91fde0 100644--- a/tests/e2e/fixtures/package-lock.json+++ b/tests/e2e/fixtures/package-lock.json@@ -1,12 +1,12 @@{"name": "@clerum/e2e-test-fixtures",- "version": "1.0.88",+ "version": "1.0.89","lockfileVersion": 3,"requires": true,"packages": {"": {"name": "@clerum/e2e-test-fixtures",- "version": "1.0.88",+ "version": "1.0.89","dependencies": {"@kubernetes/client-node": "^1.0.0"},diff --git a/tests/e2e/fixtures/package.json b/tests/e2e/fixtures/package.jsonindex 1e7319e6d..19d45b543 100644--- a/tests/e2e/fixtures/package.json+++ b/tests/e2e/fixtures/package.json@@ -1,6 +1,6 @@{"name": "@clerum/e2e-test-fixtures",- "version": "1.0.88",+ "version": "1.0.89","description": "E2E test fixtures and utilities for Clerum testing","type": "module","main": "index.ts",diff --git a/tests/e2e/integration/codex-subscription-contract-freeze.test.ts b/tests/e2e/integration/codex-subscription-contract-freeze.test.tsindex aa4e4844b..433a30bbe 100644--- a/tests/e2e/integration/codex-subscription-contract-freeze.test.ts+++ b/tests/e2e/integration/codex-subscription-contract-freeze.test.ts@@ -45,6 +45,7 @@ const REQUIRED_LIMIT_KEYS = ['maxDeadlineMs','maxConcurrentStreams','maxQueuedRequests',+ 'upstreamIdleTimeoutMs','maxRetriesPerAttempt',] as constdiff --git a/tests/e2e/package-lock.json b/tests/e2e/package-lock.jsonindex 26f0686c2..a7a7dfda3 100644--- a/tests/e2e/package-lock.json+++ b/tests/e2e/package-lock.json@@ -1,12 +1,12 @@{"name": "clerum-e2e-tests",- "version": "0.1.88",+ "version": "0.1.89","lockfileVersion": 3,"requires": true,"packages": {"": {"name": "clerum-e2e-tests",- "version": "0.1.88",+ "version": "0.1.89","devDependencies": {"@clerum/llm-provider-attempt-contract": "file:../../packages/llm-provider-attempt-contract","@types/jsonwebtoken": "^9.0.10",diff --git a/tests/e2e/package.json b/tests/e2e/package.jsonindex 6a3592a52..d004ff9cc 100644--- a/tests/e2e/package.json+++ b/tests/e2e/package.json@@ -1,6 +1,6 @@{"name": "clerum-e2e-tests",- "version": "0.1.88",+ "version": "0.1.89","private": true,"type": "module","scripts": {