Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript

fix(llm-proxy): bound upstream streams by a typed idle timeout and total cap

Alfredo Lopez0ec093c74fc0
diff --git a/codex-llm-proxy/src/codexTransport.ts b/codex-llm-proxy/src/codexTransport.ts
index 8364d87ea..f3c9b0145 100644
--- a/codex-llm-proxy/src/codexTransport.ts
+++ b/codex-llm-proxy/src/codexTransport.ts
@@ -15,7 +15,7 @@ import {
assertAllowedUpstreamUrl,
fetchFrozenOrigin,
} from './originPolicy.js'
-import { assertBoundedDeadline } from './requestLimits.js'
+import { assertBoundedDeadline, assertBoundedIdleTimeout } from './requestLimits.js'
import { ToolNameMap } from './toolNameMap.js'
import { type SafeUsage, parseSafeUsage } from './usage.js'
@@ -56,6 +56,8 @@ export type StreamCodexCompletionInput = {
ticket: TransportTicket
deadlineMs?: number
maxDeadlineMs?: number
+ /** Lowers `STREAM_LIMITS.upstreamIdleTimeoutMs`; never raises it. */
+ upstreamIdleTimeoutMs?: number
signal?: AbortSignal
redeem: (input: {
executionTicket: string
@@ -119,6 +121,7 @@ export async function streamCodexCompletion(
assertBoundedDeadline(input.deadlineMs ?? request.deadlineMs, input.maxDeadlineMs ?? 300_000),
redeemed.transport.maxStreamDurationMs
)
+ const idleTimeoutMs = assertBoundedIdleTimeout(input.upstreamIdleTimeoutMs)
const accessToken = redeemed.accessToken
let outcome: StreamCodexCompletionResult['outcome'] = 'unknown'
@@ -130,6 +133,7 @@ export async function streamCodexCompletion(
accessToken,
chatgptAccountId: redeemed.chatgptAccountId,
deadlineMs,
+ idleTimeoutMs,
signal: input.signal,
fetchFn: input.fetchFn,
lookup: input.lookup,
@@ -199,14 +203,92 @@ async function readUpstreamStream(input: {
accessToken: string
chatgptAccountId?: string
deadlineMs: number
+ idleTimeoutMs: number
signal?: AbortSignal
fetchFn: typeof fetch
lookup?: OriginPolicyOptions['lookup']
onFrame?: FrameSink
}): Promise<StreamCodexCompletionResult> {
+ const deadline = new UpstreamDeadline(input.deadlineMs, input.idleTimeoutMs)
+ try {
+ return await dispatchUpstreamStream(input, deadline)
+ } finally {
+ deadline.clear()
+ }
+}
+
+/**
+ * Two upstream bounds, each aborting with its own typed reason. The total
+ * bound runs from dispatch to the end of the stream. The idle bound runs only
+ * while the proxy waits on the upstream (response headers or the next chunk),
+ * so a client that is slow to drain frames never counts as upstream silence.
+ */
+class UpstreamDeadline {
+ private readonly controller = new AbortController()
+ private readonly started = Date.now()
+ private readonly total: ReturnType<typeof setTimeout>
+ private idle: ReturnType<typeof setTimeout> | undefined
+
+ constructor(
+ private readonly totalMs: number,
+ private readonly idleMs: number
+ ) {
+ this.total = setTimeout(() => {
+ this.controller.abort(
+ new CodexTransportError(
+ 'stream_duration_exceeded',
+ 'upstream stream exceeded maxStreamDurationMs',
+ { limitMs: this.totalMs, elapsedMs: Date.now() - this.started }
+ )
+ )
+ }, totalMs)
+ }
+
+ get signal(): AbortSignal {
+ return this.controller.signal
+ }
+
+ /**
+ * Wait on one upstream operation under the idle timeout. The wait also ends
+ * as soon as `signal` aborts, rejecting with its reason, because the
+ * operation itself may not observe the signal (a stalled body read).
+ */
+ async waitUpstream<T>(operation: Promise<T>, signal: AbortSignal): Promise<T> {
+ signal.throwIfAborted()
+ this.idle = setTimeout(() => {
+ this.controller.abort(
+ new CodexTransportError('provider_unavailable', 'upstream stream idle timeout', {
+ idleTimeoutMs: this.idleMs,
+ elapsedMs: Date.now() - this.started,
+ })
+ )
+ }, this.idleMs)
+ let onAbort: (() => void) | undefined
+ const aborted = new Promise<never>((_, reject) => {
+ onAbort = () => reject(signal.reason)
+ signal.addEventListener('abort', onAbort, { once: true })
+ })
+ try {
+ return await Promise.race([operation, aborted])
+ } finally {
+ clearTimeout(this.idle)
+ this.idle = undefined
+ if (onAbort) signal.removeEventListener('abort', onAbort)
+ }
+ }
+
+ clear(): void {
+ clearTimeout(this.total)
+ if (this.idle !== undefined) clearTimeout(this.idle)
+ }
+}
+
+async function dispatchUpstreamStream(
+ input: Parameters<typeof readUpstreamStream>[0],
+ deadline: UpstreamDeadline
+): Promise<StreamCodexCompletionResult> {
const url = assertAllowedUpstreamUrl(CODEX_COMPLETIONS_ORIGIN, 'completions')
- const timeout = AbortSignal.timeout(input.deadlineMs)
- const signal = input.signal ? AbortSignal.any([input.signal, timeout]) : timeout
+ const signal = input.signal ? AbortSignal.any([input.signal, deadline.signal]) : deadline.signal
const headers = chatgptUpstreamHeaders(input.accessToken, {
'content-type': 'application/json',
accept: 'text/event-stream',
@@ -225,17 +307,20 @@ async function readUpstreamStream(input: {
message.role === 'assistant' ? (message.toolCalls ?? []).map(call => call.name) : []
),
])
- const response = await fetchFrozenOrigin({
- url,
- fetchFn: input.fetchFn,
- lookup: input.lookup,
- init: {
- method: 'POST',
- signal,
- headers,
- body: JSON.stringify(toUpstreamPayload(input.request, names)),
- },
- })
+ const response = await deadline.waitUpstream(
+ fetchFrozenOrigin({
+ url,
+ fetchFn: input.fetchFn,
+ lookup: input.lookup,
+ init: {
+ method: 'POST',
+ signal,
+ headers,
+ body: JSON.stringify(toUpstreamPayload(input.request, names)),
+ },
+ }),
+ signal
+ )
if (!response.ok || !response.body) {
logger.warn(
{
@@ -257,7 +342,7 @@ async function readUpstreamStream(input: {
}
throw new CodexTransportError('provider_unavailable', 'upstream completion failed')
}
- return consumeSse(response.body, input.onFrame, signal, names)
+ return consumeSse(response.body, input.onFrame, signal, names, deadline)
}
function toUpstreamPayload(
@@ -333,7 +418,8 @@ async function consumeSse(
body: ReadableStream<Uint8Array>,
onFrame: FrameSink | undefined,
signal: AbortSignal,
- names: ToolNameMap
+ names: ToolNameMap,
+ deadline: UpstreamDeadline
): Promise<StreamCodexCompletionResult> {
const reader = body.getReader()
const decoder = new TextDecoder()
@@ -373,8 +459,10 @@ async function consumeSse(
let usage: SafeUsage | undefined
const maxSseBufferBytes = 1_048_576
try {
- while (!signal.aborted) {
- const { done, value } = await reader.read()
+ // An abort ends the read with the signal's reason: the caller's own abort
+ // becomes `canceled`, a deadline abort surfaces as its typed error.
+ for (;;) {
+ const { done, value } = await deadline.waitUpstream(reader.read(), signal)
if (done) break
buffer += decoder.decode(value, { stream: true }).replace(/\r\n/g, '\n').replace(/\r/g, '\n')
if (buffer.length > maxSseBufferBytes) {
@@ -389,7 +477,7 @@ async function consumeSse(
if (mapped.completed) completed = true
if (mapped.failed) failed = true
}
- if (signal.aborted) break
+ signal.throwIfAborted()
}
buffer += decoder.decode().replace(/\r\n/g, '\n').replace(/\r/g, '\n')
if (buffer.trim()) {
@@ -403,7 +491,11 @@ async function consumeSse(
await reader.cancel().catch(() => undefined)
throw error
} finally {
- reader.releaseLock()
+ try {
+ reader.releaseLock()
+ } catch {
+ // A read that lost the race to an abort may still hold the lock after cancel.
+ }
}
for (const call of pending.values()) {
if (call.emitted) continue
@@ -411,7 +503,7 @@ async function consumeSse(
await acceptFrame({ type: 'tool_call', id: call.id, name: call.name, arguments: args })
call.emitted = true
}
- if (signal.aborted) return { outcome: 'canceled', usage }
+ signal.throwIfAborted()
if (failed) {
throw new CodexTransportError('provider_unavailable', 'upstream response failed')
}
diff --git a/codex-llm-proxy/src/requestLimits.ts b/codex-llm-proxy/src/requestLimits.ts
index 550e48cde..c56762de5 100644
--- a/codex-llm-proxy/src/requestLimits.ts
+++ b/codex-llm-proxy/src/requestLimits.ts
@@ -4,6 +4,9 @@ export const STREAM_LIMITS = {
maxConcurrentStreams: 8,
maxQueuedRequests: 16,
maxStreamDurationMs: 300_000,
+ // Longest silence tolerated while waiting on the upstream (response headers
+ // or the next SSE chunk). Matches the Codex CLI stream idle timeout.
+ upstreamIdleTimeoutMs: 300_000,
} as const
export class RequestLimitError extends Error {
@@ -22,6 +25,14 @@ export function assertBoundedDeadline(deadlineMs: number | undefined, maxDeadlin
return Math.min(requested, maxDeadlineMs, STREAM_LIMITS.maxStreamDurationMs, CONTRACT_LIMITS.maxDeadlineMs)
}
+export function assertBoundedIdleTimeout(idleTimeoutMs: number | undefined): number {
+ const requested = idleTimeoutMs ?? STREAM_LIMITS.upstreamIdleTimeoutMs
+ if (!Number.isInteger(requested) || requested <= 0) {
+ throw new RequestLimitError('upstream idle timeout is invalid')
+ }
+ return Math.min(requested, STREAM_LIMITS.upstreamIdleTimeoutMs)
+}
+
export class StreamGate {
private running = 0
private queued = 0
diff --git a/codex-llm-proxy/test/codexTransport.conformance.test.ts b/codex-llm-proxy/test/codexTransport.conformance.test.ts
index d8e8dc04a..2b064a39e 100644
--- a/codex-llm-proxy/test/codexTransport.conformance.test.ts
+++ b/codex-llm-proxy/test/codexTransport.conformance.test.ts
@@ -745,13 +745,19 @@ describe('streamCodexCompletion', () => {
.fn()
.mockRejectedValueOnce(new Error('finalize 500'))
.mockResolvedValueOnce({ providerAttemptId: 'att-1', outcome: 'canceled', duplicate: true })
+ let upstreamCanceled = false
const fetchFn = vi.fn(async () => {
const stream = new ReadableStream({
start(controller) {
controller.enqueue(
new TextEncoder().encode('data: {"type":"response.output_text.delta","delta":"x"}\n\n')
)
- setTimeout(() => controller.close(), 20)
+ setTimeout(() => {
+ if (!upstreamCanceled) controller.close()
+ }, 20)
+ },
+ cancel() {
+ upstreamCanceled = true
},
})
return new Response(stream, { headers: { 'content-type': 'text/event-stream' } })
@@ -781,6 +787,7 @@ describe('streamCodexCompletion', () => {
})
expect(frames[0]).toEqual({ type: 'text', text: 'x' })
expect(result.outcome).toBe('canceled')
+ expect(upstreamCanceled).toBe(true)
expect(finalize).toHaveBeenCalledTimes(2)
expect(finalize.mock.calls[0]?.[0]?.receipt.outcome).toBe('canceled')
})
@@ -1227,3 +1234,144 @@ describe('streamCodexCompletion', () => {
])
})
})
+
+const TEXT_DELTA = 'data: {"type":"response.output_text.delta","delta":"x"}\n\n'
+const COMPLETED = 'data: {"type":"response.completed","response":{"usage":{}}}\n\n'
+
+/**
+ * An upstream body that ignores `init.signal`, like a socket that stays open
+ * while sending nothing. Only a transport that races each read against its own
+ * timers can end an attempt reading from it.
+ */
+function upstreamBody(chunks: Array<{ afterMs: number; text: string } | 'stall'>): {
+ fetchFn: typeof fetch
+ cancel: ReturnType<typeof vi.fn>
+} {
+ const encoder = new TextEncoder()
+ const cancel = vi.fn()
+ const fetchFn = vi.fn(async () => {
+ let index = 0
+ const body = new ReadableStream<Uint8Array>(
+ {
+ async pull(controller) {
+ const next = chunks[index]
+ index += 1
+ if (next === undefined) {
+ controller.close()
+ return
+ }
+ if (next === 'stall') {
+ await new Promise<never>(() => undefined)
+ return
+ }
+ await new Promise(resolve => setTimeout(resolve, next.afterMs))
+ controller.enqueue(encoder.encode(next.text))
+ },
+ cancel,
+ },
+ { highWaterMark: 0 }
+ )
+ return new Response(body, { status: 200, headers: { 'content-type': 'text/event-stream' } })
+ }) as unknown as typeof fetch
+ return { fetchFn, cancel }
+}
+
+function attemptInput(
+ overrides: Partial<StreamCodexCompletionInput>
+): StreamCodexCompletionInput & { finalize: ReturnType<typeof vi.fn> } {
+ const finalize = vi.fn(async (_input: Parameters<StreamCodexCompletionInput['finalize']>[0]) => ({
+ providerAttemptId: 'att-timeouts',
+ outcome: 'error' as const,
+ duplicate: false,
+ }))
+ return {
+ executionTicket: 'ticket-timeouts',
+ requestHash: REQUEST_HASH,
+ request: REQUEST,
+ ticket: {
+ jti: 'jti-timeouts',
+ hostRef: 'research-host',
+ model: REQUEST.model,
+ requestHash: REQUEST_HASH,
+ providerAttemptId: 'att-timeouts',
+ },
+ maxDeadlineMs: 300_000,
+ redeem: async () => redeemSuccess(),
+ finalize,
+ fetchFn: upstreamBody([]).fetchFn,
+ lookup: async () => [{ address: '1.2.3.4', family: 4 }],
+ ...overrides,
+ } as StreamCodexCompletionInput & { finalize: ReturnType<typeof vi.fn> }
+}
+
+function finalizedOutcome(finalize: ReturnType<typeof vi.fn>): unknown {
+ expect(finalize).toHaveBeenCalledTimes(1)
+ return (finalize.mock.calls[0]?.[0] as { receipt: { outcome: string } }).receipt.outcome
+}
+
+describe('streamCodexCompletion upstream timeouts', () => {
+ it('ends a stalled upstream stream at the idle timeout with a typed provider_unavailable', async () => {
+ const upstream = upstreamBody([{ afterMs: 0, text: TEXT_DELTA }, 'stall'])
+ const input = attemptInput({ fetchFn: upstream.fetchFn, upstreamIdleTimeoutMs: 50 })
+ const frames: unknown[] = []
+ const error = await streamCodexCompletion({ ...input, onFrame: frame => void frames.push(frame) }).then(
+ () => undefined,
+ (err: unknown) => err
+ )
+ expect(frames).toEqual([{ type: 'text', text: 'x' }])
+ expect(error).toBeInstanceOf(CodexTransportError)
+ expect(error).toMatchObject({
+ code: 'provider_unavailable',
+ message: 'upstream stream idle timeout',
+ details: { idleTimeoutMs: 50 },
+ })
+ expect(finalizedOutcome(input.finalize)).toBe('error')
+ expect(upstream.cancel).toHaveBeenCalledTimes(1)
+ }, 2_000)
+
+ it('keeps the attempt alive while upstream chunks arrive inside the idle window', async () => {
+ const chunks = Array.from({ length: 6 }, () => ({ afterMs: 30, text: TEXT_DELTA }))
+ const upstream = upstreamBody([...chunks, { afterMs: 30, text: COMPLETED }])
+ const input = attemptInput({ fetchFn: upstream.fetchFn, upstreamIdleTimeoutMs: 50 })
+ const result = await streamCodexCompletion(input)
+ expect(result.outcome).toBe('success')
+ expect(finalizedOutcome(input.finalize)).toBe('success')
+ }, 2_000)
+
+ it('ends a still-active upstream stream at the total cap with stream_duration_exceeded', async () => {
+ const endless = Array.from({ length: 200 }, () => ({ afterMs: 10, text: TEXT_DELTA }))
+ const upstream = upstreamBody(endless)
+ const input = attemptInput({
+ fetchFn: upstream.fetchFn,
+ maxDeadlineMs: 150,
+ upstreamIdleTimeoutMs: 1_000,
+ })
+ const error = await streamCodexCompletion(input).then(
+ () => undefined,
+ (err: unknown) => err
+ )
+ expect(error).toBeInstanceOf(CodexTransportError)
+ expect(error).toMatchObject({
+ code: 'stream_duration_exceeded',
+ message: 'upstream stream exceeded maxStreamDurationMs',
+ details: { limitMs: 150 },
+ })
+ expect(finalizedOutcome(input.finalize)).toBe('error')
+ }, 2_000)
+
+ it('reports a client abort during a stalled stream as canceled, not as a timeout', async () => {
+ const upstream = upstreamBody([{ afterMs: 0, text: TEXT_DELTA }, 'stall'])
+ const abort = new AbortController()
+ const input = attemptInput({
+ fetchFn: upstream.fetchFn,
+ signal: abort.signal,
+ maxDeadlineMs: 1_000,
+ upstreamIdleTimeoutMs: 1_000,
+ })
+ setTimeout(() => abort.abort(), 50)
+ const result = await streamCodexCompletion(input)
+ expect(result.outcome).toBe('canceled')
+ expect(finalizedOutcome(input.finalize)).toBe('canceled')
+ expect(upstream.cancel).toHaveBeenCalledTimes(1)
+ }, 2_000)
+})
diff --git a/codex-llm-proxy/test/streamLimitsFreeze.test.ts b/codex-llm-proxy/test/streamLimitsFreeze.test.ts
index 60762e52b..0821c473b 100644
--- a/codex-llm-proxy/test/streamLimitsFreeze.test.ts
+++ b/codex-llm-proxy/test/streamLimitsFreeze.test.ts
@@ -9,7 +9,7 @@ const fixturePath = new URL(
describe('codex-subscription stream limits freeze', () => {
it('pins the proxy STREAM_LIMITS to the limits the fixture publishes', () => {
- // StreamGate and the stream deadline enforce these three published bounds
+ // StreamGate and the upstream deadlines enforce these published bounds
// from the proxy's own constant, not from the contract package, so the
// Codex freeze gate in tests/e2e cannot see them.
const { limits } = JSON.parse(readFileSync(fixturePath, 'utf8')) as {
@@ -20,6 +20,7 @@ describe('codex-subscription stream limits freeze', () => {
'maxConcurrentStreams',
'maxQueuedRequests',
'maxStreamDurationMs',
+ 'upstreamIdleTimeoutMs',
])
for (const name of Object.keys(streamLimits)) {
expect({ [name]: limits[name] }).toEqual({ [name]: streamLimits[name] })
diff --git a/docs/architecture/codex-subscription-transport-contract.md b/docs/architecture/codex-subscription-transport-contract.md
index 4c4ab7a30..48e56e018 100644
--- a/docs/architecture/codex-subscription-transport-contract.md
+++ b/docs/architecture/codex-subscription-transport-contract.md
@@ -49,17 +49,18 @@ OAuth scopes: `openid`, `profile`, `email`, `offline_access`.
All values are finite and greater than zero. `maxRetriesPerAttempt` is `1`:
one physical execution per ticket. A retry or fallback must mint a new attempt.
-| Limit | Value |
-| -------------------- | ------- |
-| maxRequestBodyBytes | 1048576 |
-| maxMessages | 1024 |
-| maxToolCalls | 256 |
-| maxOutputTokens | 16384 |
-| maxStreamDurationMs | 300000 |
-| maxDeadlineMs | 300000 |
-| maxConcurrentStreams | 8 |
-| maxQueuedRequests | 16 |
-| maxRetriesPerAttempt | 1 |
+| Limit | Value |
+| --------------------- | ------- |
+| maxRequestBodyBytes | 1048576 |
+| maxMessages | 1024 |
+| maxToolCalls | 256 |
+| maxOutputTokens | 16384 |
+| maxStreamDurationMs | 300000 |
+| maxDeadlineMs | 300000 |
+| maxConcurrentStreams | 8 |
+| maxQueuedRequests | 16 |
+| upstreamIdleTimeoutMs | 300000 |
+| maxRetriesPerAttempt | 1 |
Tool definitions have no independent count ceiling in the Evenfire request
contract. The entire serialized request, including all definitions, remains
diff --git a/grok-llm-proxy/src/grokTransport.ts b/grok-llm-proxy/src/grokTransport.ts
index 3e6983ca4..7314d2aec 100644
--- a/grok-llm-proxy/src/grokTransport.ts
+++ b/grok-llm-proxy/src/grokTransport.ts
@@ -15,7 +15,7 @@ import {
assertAllowedUpstreamUrl,
fetchFrozenOrigin,
} from './originPolicy.js'
-import { assertBoundedDeadline } from './requestLimits.js'
+import { assertBoundedDeadline, assertBoundedIdleTimeout } from './requestLimits.js'
import { ToolNameMap } from './toolNameMap.js'
import { type SafeUsage, parseSafeUsage } from './usage.js'
@@ -83,6 +83,8 @@ export type StreamGrokCompletionInput = {
ticket: TransportTicket
deadlineMs?: number
maxDeadlineMs?: number
+ /** Lowers `STREAM_LIMITS.upstreamIdleTimeoutMs`; never raises it. */
+ upstreamIdleTimeoutMs?: number
signal?: AbortSignal
redeem: (input: {
executionTicket: string
@@ -146,6 +148,7 @@ export async function streamGrokCompletion(
assertBoundedDeadline(input.deadlineMs ?? request.deadlineMs, input.maxDeadlineMs ?? 300_000),
redeemed.transport.maxStreamDurationMs
)
+ const idleTimeoutMs = assertBoundedIdleTimeout(input.upstreamIdleTimeoutMs)
const accessToken = redeemed.accessToken
let outcome: StreamGrokCompletionResult['outcome'] = 'unknown'
@@ -156,6 +159,7 @@ export async function streamGrokCompletion(
request,
accessToken,
deadlineMs,
+ idleTimeoutMs,
signal: input.signal,
fetchFn: input.fetchFn,
lookup: input.lookup,
@@ -251,14 +255,92 @@ async function readUpstreamStream(input: {
request: GrokCompletionRequestV1
accessToken: string
deadlineMs: number
+ idleTimeoutMs: number
signal?: AbortSignal
fetchFn: typeof fetch
lookup?: OriginPolicyOptions['lookup']
onFrame?: FrameSink
}): Promise<StreamGrokCompletionResult> {
+ const deadline = new UpstreamDeadline(input.deadlineMs, input.idleTimeoutMs)
+ try {
+ return await dispatchUpstreamStream(input, deadline)
+ } finally {
+ deadline.clear()
+ }
+}
+
+/**
+ * Two upstream bounds, each aborting with its own typed reason. The total
+ * bound runs from dispatch to the end of the stream. The idle bound runs only
+ * while the proxy waits on the upstream (response headers or the next chunk),
+ * so a client that is slow to drain frames never counts as upstream silence.
+ */
+class UpstreamDeadline {
+ private readonly controller = new AbortController()
+ private readonly started = Date.now()
+ private readonly total: ReturnType<typeof setTimeout>
+ private idle: ReturnType<typeof setTimeout> | undefined
+
+ constructor(
+ private readonly totalMs: number,
+ private readonly idleMs: number
+ ) {
+ this.total = setTimeout(() => {
+ this.controller.abort(
+ new GrokTransportError(
+ 'stream_duration_exceeded',
+ 'upstream stream exceeded maxStreamDurationMs',
+ { limitMs: this.totalMs, elapsedMs: Date.now() - this.started }
+ )
+ )
+ }, totalMs)
+ }
+
+ get signal(): AbortSignal {
+ return this.controller.signal
+ }
+
+ /**
+ * Wait on one upstream operation under the idle timeout. The wait also ends
+ * as soon as `signal` aborts, rejecting with its reason, because the
+ * operation itself may not observe the signal (a stalled body read).
+ */
+ async waitUpstream<T>(operation: Promise<T>, signal: AbortSignal): Promise<T> {
+ signal.throwIfAborted()
+ this.idle = setTimeout(() => {
+ this.controller.abort(
+ new GrokTransportError('provider_unavailable', 'upstream stream idle timeout', {
+ idleTimeoutMs: this.idleMs,
+ elapsedMs: Date.now() - this.started,
+ })
+ )
+ }, this.idleMs)
+ let onAbort: (() => void) | undefined
+ const aborted = new Promise<never>((_, reject) => {
+ onAbort = () => reject(signal.reason)
+ signal.addEventListener('abort', onAbort, { once: true })
+ })
+ try {
+ return await Promise.race([operation, aborted])
+ } finally {
+ clearTimeout(this.idle)
+ this.idle = undefined
+ if (onAbort) signal.removeEventListener('abort', onAbort)
+ }
+ }
+
+ clear(): void {
+ clearTimeout(this.total)
+ if (this.idle !== undefined) clearTimeout(this.idle)
+ }
+}
+
+async function dispatchUpstreamStream(
+ input: Parameters<typeof readUpstreamStream>[0],
+ deadline: UpstreamDeadline
+): Promise<StreamGrokCompletionResult> {
const url = assertAllowedUpstreamUrl(GROK_COMPLETIONS_ORIGIN, 'completions')
- const timeout = AbortSignal.timeout(input.deadlineMs)
- const signal = input.signal ? AbortSignal.any([input.signal, timeout]) : timeout
+ const signal = input.signal ? AbortSignal.any([input.signal, deadline.signal]) : deadline.signal
const headers = grokUpstreamHeaders(input.accessToken, {
'content-type': 'application/json',
accept: 'text/event-stream',
@@ -269,17 +351,20 @@ async function readUpstreamStream(input: {
message.role === 'assistant' ? (message.toolCalls ?? []).map(call => call.name) : []
),
])
- const response = await fetchFrozenOrigin({
- url,
- fetchFn: input.fetchFn,
- lookup: input.lookup,
- init: {
- method: 'POST',
- signal,
- headers,
- body: JSON.stringify(toUpstreamPayload(input.request, names)),
- },
- })
+ const response = await deadline.waitUpstream(
+ fetchFrozenOrigin({
+ url,
+ fetchFn: input.fetchFn,
+ lookup: input.lookup,
+ init: {
+ method: 'POST',
+ signal,
+ headers,
+ body: JSON.stringify(toUpstreamPayload(input.request, names)),
+ },
+ }),
+ signal
+ )
if (!response.ok || !response.body) {
logger.warn(
{
@@ -316,7 +401,7 @@ async function readUpstreamStream(input: {
}
throw new GrokTransportError('provider_unavailable', 'upstream completion failed')
}
- return consumeSse(response.body, input.onFrame, signal, names)
+ return consumeSse(response.body, input.onFrame, signal, names, deadline)
}
function toUpstreamPayload(
@@ -392,7 +477,8 @@ async function consumeSse(
body: ReadableStream<Uint8Array>,
onFrame: FrameSink | undefined,
signal: AbortSignal,
- names: ToolNameMap
+ names: ToolNameMap,
+ deadline: UpstreamDeadline
): Promise<StreamGrokCompletionResult> {
const reader = body.getReader()
const decoder = new TextDecoder()
@@ -433,8 +519,10 @@ async function consumeSse(
let usage: SafeUsage | undefined
const maxSseBufferBytes = 1_048_576
try {
- while (!signal.aborted) {
- const { done, value } = await reader.read()
+ // An abort ends the read with the signal's reason: the caller's own abort
+ // becomes `canceled`, a deadline abort surfaces as its typed error.
+ for (;;) {
+ const { done, value } = await deadline.waitUpstream(reader.read(), signal)
if (done) break
buffer += decoder.decode(value, { stream: true }).replace(/\r\n/g, '\n').replace(/\r/g, '\n')
if (buffer.length > maxSseBufferBytes) {
@@ -449,7 +537,7 @@ async function consumeSse(
if (mapped.completed) completed = true
if (mapped.failed) failed = true
}
- if (signal.aborted) break
+ signal.throwIfAborted()
}
buffer += decoder.decode().replace(/\r\n/g, '\n').replace(/\r/g, '\n')
if (buffer.trim()) {
@@ -463,7 +551,11 @@ async function consumeSse(
await reader.cancel().catch(() => undefined)
throw error
} finally {
- reader.releaseLock()
+ try {
+ reader.releaseLock()
+ } catch {
+ // A read that lost the race to an abort may still hold the lock after cancel.
+ }
}
for (const call of pending.values()) {
if (call.emitted) continue
@@ -471,7 +563,7 @@ async function consumeSse(
await acceptFrame({ type: 'tool_call', id: call.id, name: call.name, arguments: args })
call.emitted = true
}
- if (signal.aborted) return { outcome: 'canceled', usage }
+ signal.throwIfAborted()
if (failed) {
throw new GrokTransportError('provider_unavailable', 'upstream response failed')
}
diff --git a/grok-llm-proxy/src/requestLimits.ts b/grok-llm-proxy/src/requestLimits.ts
index ba22dd3b7..5da88d377 100644
--- a/grok-llm-proxy/src/requestLimits.ts
+++ b/grok-llm-proxy/src/requestLimits.ts
@@ -4,6 +4,9 @@ export const STREAM_LIMITS = {
maxConcurrentStreams: 8,
maxQueuedRequests: 16,
maxStreamDurationMs: 300_000,
+ // Longest silence tolerated while waiting on the upstream (response headers
+ // or the next SSE chunk). Matches the Grok Build inference idle timeout.
+ upstreamIdleTimeoutMs: 600_000,
} as const
export class RequestLimitError extends Error {
@@ -30,6 +33,14 @@ export function assertBoundedDeadline(
)
}
+export function assertBoundedIdleTimeout(idleTimeoutMs: number | undefined): number {
+ const requested = idleTimeoutMs ?? STREAM_LIMITS.upstreamIdleTimeoutMs
+ if (!Number.isInteger(requested) || requested <= 0) {
+ throw new RequestLimitError('upstream idle timeout is invalid')
+ }
+ return Math.min(requested, STREAM_LIMITS.upstreamIdleTimeoutMs)
+}
+
export class StreamGate {
private running = 0
private queued = 0
diff --git a/grok-llm-proxy/test/contractFreeze.test.ts b/grok-llm-proxy/test/contractFreeze.test.ts
index bfafd9930..92d92c507 100644
--- a/grok-llm-proxy/test/contractFreeze.test.ts
+++ b/grok-llm-proxy/test/contractFreeze.test.ts
@@ -35,6 +35,7 @@ const FIXTURE_LIMIT_KEYS = [
'maxDeadlineMs',
'maxConcurrentStreams',
'maxQueuedRequests',
+ 'upstreamIdleTimeoutMs',
'maxRetriesPerAttempt',
] as const
@@ -151,7 +152,7 @@ describe('grok-subscription contract freeze', () => {
})
it('pins the proxy STREAM_LIMITS to the limits the fixture publishes', () => {
- // StreamGate and the stream deadline enforce these three published bounds
+ // StreamGate and the upstream deadlines enforce these published bounds
// from the proxy's own constant, not from the contract package.
const { limits } = readFixture()
const streamLimits: Record<string, number> = { ...STREAM_LIMITS }
@@ -159,6 +160,7 @@ describe('grok-subscription contract freeze', () => {
'maxConcurrentStreams',
'maxQueuedRequests',
'maxStreamDurationMs',
+ 'upstreamIdleTimeoutMs',
])
for (const name of Object.keys(streamLimits)) {
expect({ [name]: limits[name] }).toEqual({ [name]: streamLimits[name] })
diff --git a/grok-llm-proxy/test/grokTransport.conformance.test.ts b/grok-llm-proxy/test/grokTransport.conformance.test.ts
index 5b9213762..60fbcc063 100644
--- a/grok-llm-proxy/test/grokTransport.conformance.test.ts
+++ b/grok-llm-proxy/test/grokTransport.conformance.test.ts
@@ -870,13 +870,19 @@ describe('streamGrokCompletion', () => {
.fn()
.mockRejectedValueOnce(new Error('finalize 500'))
.mockResolvedValueOnce({ providerAttemptId: 'att-1', outcome: 'canceled', duplicate: true })
+ let upstreamCanceled = false
const fetchFn = vi.fn(async () => {
const stream = new ReadableStream({
start(controller) {
controller.enqueue(
new TextEncoder().encode('data: {"type":"response.output_text.delta","delta":"x"}\n\n')
)
- setTimeout(() => controller.close(), 20)
+ setTimeout(() => {
+ if (!upstreamCanceled) controller.close()
+ }, 20)
+ },
+ cancel() {
+ upstreamCanceled = true
},
})
return new Response(stream, { headers: { 'content-type': 'text/event-stream' } })
@@ -906,6 +912,7 @@ describe('streamGrokCompletion', () => {
})
expect(frames[0]).toEqual({ type: 'text', text: 'x' })
expect(result.outcome).toBe('canceled')
+ expect(upstreamCanceled).toBe(true)
expect(finalize).toHaveBeenCalledTimes(2)
expect(finalize.mock.calls[0]?.[0]?.receipt.outcome).toBe('canceled')
})
@@ -1358,3 +1365,144 @@ describe('streamGrokCompletion', () => {
])
})
})
+
+const TEXT_DELTA = 'data: {"type":"response.output_text.delta","delta":"x"}\n\n'
+const COMPLETED = 'data: {"type":"response.completed","response":{"usage":{}}}\n\n'
+
+/**
+ * An upstream body that ignores `init.signal`, like a socket that stays open
+ * while sending nothing. Only a transport that races each read against its own
+ * timers can end an attempt reading from it.
+ */
+function upstreamBody(chunks: Array<{ afterMs: number; text: string } | 'stall'>): {
+ fetchFn: typeof fetch
+ cancel: ReturnType<typeof vi.fn>
+} {
+ const encoder = new TextEncoder()
+ const cancel = vi.fn()
+ const fetchFn = vi.fn(async () => {
+ let index = 0
+ const body = new ReadableStream<Uint8Array>(
+ {
+ async pull(controller) {
+ const next = chunks[index]
+ index += 1
+ if (next === undefined) {
+ controller.close()
+ return
+ }
+ if (next === 'stall') {
+ await new Promise<never>(() => undefined)
+ return
+ }
+ await new Promise(resolve => setTimeout(resolve, next.afterMs))
+ controller.enqueue(encoder.encode(next.text))
+ },
+ cancel,
+ },
+ { highWaterMark: 0 }
+ )
+ return new Response(body, { status: 200, headers: { 'content-type': 'text/event-stream' } })
+ }) as unknown as typeof fetch
+ return { fetchFn, cancel }
+}
+
+function attemptInput(
+ overrides: Partial<StreamGrokCompletionInput>
+): StreamGrokCompletionInput & { finalize: ReturnType<typeof vi.fn> } {
+ const finalize = vi.fn(async (_input: Parameters<StreamGrokCompletionInput['finalize']>[0]) => ({
+ providerAttemptId: 'att-timeouts',
+ outcome: 'error' as const,
+ duplicate: false,
+ }))
+ return {
+ executionTicket: 'ticket-timeouts',
+ requestHash: REQUEST_HASH,
+ request: REQUEST,
+ ticket: {
+ jti: 'jti-timeouts',
+ hostRef: 'research-host',
+ model: REQUEST.model,
+ requestHash: REQUEST_HASH,
+ providerAttemptId: 'att-timeouts',
+ },
+ maxDeadlineMs: 300_000,
+ redeem: async () => redeemSuccess(),
+ finalize,
+ fetchFn: upstreamBody([]).fetchFn,
+ lookup: async () => [{ address: '1.2.3.4', family: 4 }],
+ ...overrides,
+ } as StreamGrokCompletionInput & { finalize: ReturnType<typeof vi.fn> }
+}
+
+function finalizedOutcome(finalize: ReturnType<typeof vi.fn>): unknown {
+ expect(finalize).toHaveBeenCalledTimes(1)
+ return (finalize.mock.calls[0]?.[0] as { receipt: { outcome: string } }).receipt.outcome
+}
+
+describe('streamGrokCompletion upstream timeouts', () => {
+ it('ends a stalled upstream stream at the idle timeout with a typed provider_unavailable', async () => {
+ const upstream = upstreamBody([{ afterMs: 0, text: TEXT_DELTA }, 'stall'])
+ const input = attemptInput({ fetchFn: upstream.fetchFn, upstreamIdleTimeoutMs: 50 })
+ const frames: unknown[] = []
+ const error = await streamGrokCompletion({ ...input, onFrame: frame => void frames.push(frame) }).then(
+ () => undefined,
+ (err: unknown) => err
+ )
+ expect(frames).toEqual([{ type: 'text', text: 'x' }])
+ expect(error).toBeInstanceOf(GrokTransportError)
+ expect(error).toMatchObject({
+ code: 'provider_unavailable',
+ message: 'upstream stream idle timeout',
+ details: { idleTimeoutMs: 50 },
+ })
+ expect(finalizedOutcome(input.finalize)).toBe('error')
+ expect(upstream.cancel).toHaveBeenCalledTimes(1)
+ }, 2_000)
+
+ it('keeps the attempt alive while upstream chunks arrive inside the idle window', async () => {
+ const chunks = Array.from({ length: 6 }, () => ({ afterMs: 30, text: TEXT_DELTA }))
+ const upstream = upstreamBody([...chunks, { afterMs: 30, text: COMPLETED }])
+ const input = attemptInput({ fetchFn: upstream.fetchFn, upstreamIdleTimeoutMs: 50 })
+ const result = await streamGrokCompletion(input)
+ expect(result.outcome).toBe('success')
+ expect(finalizedOutcome(input.finalize)).toBe('success')
+ }, 2_000)
+
+ it('ends a still-active upstream stream at the total cap with stream_duration_exceeded', async () => {
+ const endless = Array.from({ length: 200 }, () => ({ afterMs: 10, text: TEXT_DELTA }))
+ const upstream = upstreamBody(endless)
+ const input = attemptInput({
+ fetchFn: upstream.fetchFn,
+ maxDeadlineMs: 150,
+ upstreamIdleTimeoutMs: 1_000,
+ })
+ const error = await streamGrokCompletion(input).then(
+ () => undefined,
+ (err: unknown) => err
+ )
+ expect(error).toBeInstanceOf(GrokTransportError)
+ expect(error).toMatchObject({
+ code: 'stream_duration_exceeded',
+ message: 'upstream stream exceeded maxStreamDurationMs',
+ details: { limitMs: 150 },
+ })
+ expect(finalizedOutcome(input.finalize)).toBe('error')
+ }, 2_000)
+
+ it('reports a client abort during a stalled stream as canceled, not as a timeout', async () => {
+ const upstream = upstreamBody([{ afterMs: 0, text: TEXT_DELTA }, 'stall'])
+ const abort = new AbortController()
+ const input = attemptInput({
+ fetchFn: upstream.fetchFn,
+ signal: abort.signal,
+ maxDeadlineMs: 1_000,
+ upstreamIdleTimeoutMs: 1_000,
+ })
+ setTimeout(() => abort.abort(), 50)
+ const result = await streamGrokCompletion(input)
+ expect(result.outcome).toBe('canceled')
+ expect(finalizedOutcome(input.finalize)).toBe('canceled')
+ expect(upstream.cancel).toHaveBeenCalledTimes(1)
+ }, 2_000)
+})
diff --git a/tests/e2e/fixtures/codex-subscription/sanitized-upstream-contract.json b/tests/e2e/fixtures/codex-subscription/sanitized-upstream-contract.json
index 2a0339ba4..456b4755e 100644
--- a/tests/e2e/fixtures/codex-subscription/sanitized-upstream-contract.json
+++ b/tests/e2e/fixtures/codex-subscription/sanitized-upstream-contract.json
@@ -48,6 +48,7 @@
"maxDeadlineMs": 300000,
"maxConcurrentStreams": 8,
"maxQueuedRequests": 16,
+ "upstreamIdleTimeoutMs": 300000,
"maxRetriesPerAttempt": 1
},
"errorTaxonomy": [
diff --git a/tests/e2e/fixtures/grok-subscription/sanitized-upstream-contract.json b/tests/e2e/fixtures/grok-subscription/sanitized-upstream-contract.json
index b6b27622c..7bfa5aca8 100644
--- a/tests/e2e/fixtures/grok-subscription/sanitized-upstream-contract.json
+++ b/tests/e2e/fixtures/grok-subscription/sanitized-upstream-contract.json
@@ -62,6 +62,7 @@
"maxDeadlineMs": 300000,
"maxConcurrentStreams": 8,
"maxQueuedRequests": 16,
+ "upstreamIdleTimeoutMs": 600000,
"maxRetriesPerAttempt": 1
},
"errorTaxonomy": [
diff --git a/tests/e2e/fixtures/package-lock.json b/tests/e2e/fixtures/package-lock.json
index 0710b38c1..85e91fde0 100644
--- a/tests/e2e/fixtures/package-lock.json
+++ b/tests/e2e/fixtures/package-lock.json
@@ -1,12 +1,12 @@
{
"name": "@clerum/e2e-test-fixtures",
- "version": "1.0.88",
+ "version": "1.0.89",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "@clerum/e2e-test-fixtures",
- "version": "1.0.88",
+ "version": "1.0.89",
"dependencies": {
"@kubernetes/client-node": "^1.0.0"
},
diff --git a/tests/e2e/fixtures/package.json b/tests/e2e/fixtures/package.json
index 1e7319e6d..19d45b543 100644
--- a/tests/e2e/fixtures/package.json
+++ b/tests/e2e/fixtures/package.json
@@ -1,6 +1,6 @@
{
"name": "@clerum/e2e-test-fixtures",
- "version": "1.0.88",
+ "version": "1.0.89",
"description": "E2E test fixtures and utilities for Clerum testing",
"type": "module",
"main": "index.ts",
diff --git a/tests/e2e/integration/codex-subscription-contract-freeze.test.ts b/tests/e2e/integration/codex-subscription-contract-freeze.test.ts
index aa4e4844b..433a30bbe 100644
--- a/tests/e2e/integration/codex-subscription-contract-freeze.test.ts
+++ b/tests/e2e/integration/codex-subscription-contract-freeze.test.ts
@@ -45,6 +45,7 @@ const REQUIRED_LIMIT_KEYS = [
'maxDeadlineMs',
'maxConcurrentStreams',
'maxQueuedRequests',
+ 'upstreamIdleTimeoutMs',
'maxRetriesPerAttempt',
] as const
diff --git a/tests/e2e/package-lock.json b/tests/e2e/package-lock.json
index 26f0686c2..a7a7dfda3 100644
--- a/tests/e2e/package-lock.json
+++ b/tests/e2e/package-lock.json
@@ -1,12 +1,12 @@
{
"name": "clerum-e2e-tests",
- "version": "0.1.88",
+ "version": "0.1.89",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "clerum-e2e-tests",
- "version": "0.1.88",
+ "version": "0.1.89",
"devDependencies": {
"@clerum/llm-provider-attempt-contract": "file:../../packages/llm-provider-attempt-contract",
"@types/jsonwebtoken": "^9.0.10",
diff --git a/tests/e2e/package.json b/tests/e2e/package.json
index 6a3592a52..d004ff9cc 100644
--- a/tests/e2e/package.json
+++ b/tests/e2e/package.json
@@ -1,6 +1,6 @@
{
"name": "clerum-e2e-tests",
- "version": "0.1.88",
+ "version": "0.1.89",
"private": true,
"type": "module",
"scripts": {