Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript

feat(llm-proxy): raise the per-attempt stream cap to 30 min and bound queue wait

Alfredo Lopez6cd077f7ac9e
diff --git a/codex-llm-proxy/src/codexTransport.ts b/codex-llm-proxy/src/codexTransport.ts
index 5addba8dc..687f4c00c 100644
--- a/codex-llm-proxy/src/codexTransport.ts
+++ b/codex-llm-proxy/src/codexTransport.ts
@@ -72,7 +72,7 @@ export type StreamCodexCompletionInput = {
request: unknown
ticket: TransportTicket
deadlineMs?: number
- maxDeadlineMs?: number
+ maxDeadlineMs: number
/** Lowers `STREAM_LIMITS.upstreamIdleTimeoutMs`; never raises it. */
upstreamIdleTimeoutMs?: number
signal?: AbortSignal
@@ -141,7 +141,7 @@ export async function streamCodexCompletion(
throw new CodexTransportError('model_not_allowed', 'served model does not match the request')
}
const deadlineMs = Math.min(
- assertBoundedDeadline(input.deadlineMs ?? request.deadlineMs, input.maxDeadlineMs ?? 300_000),
+ assertBoundedDeadline(input.deadlineMs ?? request.deadlineMs, input.maxDeadlineMs),
redeemed.transport.maxStreamDurationMs
)
const idleTimeoutMs = assertBoundedIdleTimeout(input.upstreamIdleTimeoutMs)
diff --git a/codex-llm-proxy/src/config.ts b/codex-llm-proxy/src/config.ts
index 2084bd5f8..114ad903d 100644
--- a/codex-llm-proxy/src/config.ts
+++ b/codex-llm-proxy/src/config.ts
@@ -75,12 +75,12 @@ export function loadConfig(env: NodeJS.ProcessEnv = process.env): CodexLlmProxyC
maxStreamDurationMs: requiredPositiveInt(
'CODEX_LLM_PROXY_MAX_STREAM_DURATION_MS',
env.CODEX_LLM_PROXY_MAX_STREAM_DURATION_MS,
- 300_000
+ 1_800_000
),
maxDeadlineMs: requiredPositiveInt(
'CODEX_LLM_PROXY_MAX_DEADLINE_MS',
env.CODEX_LLM_PROXY_MAX_DEADLINE_MS,
- 300_000
+ 1_800_000
),
upstreamIdleTimeoutMs: requiredPositiveInt(
'CODEX_LLM_PROXY_UPSTREAM_IDLE_TIMEOUT_MS',
diff --git a/codex-llm-proxy/src/controlApiClient.ts b/codex-llm-proxy/src/controlApiClient.ts
index 148e445fa..a6150ada2 100644
--- a/codex-llm-proxy/src/controlApiClient.ts
+++ b/codex-llm-proxy/src/controlApiClient.ts
@@ -5,6 +5,10 @@ import {
CODEX_TRANSPORT_PROTOCOL,
} from './originPolicy.js'
+// Bound on one control-api call (redeem or finalize). Part of the time a
+// stream request may spend before its first byte; see STREAM_LIMITS.
+export const CONTROL_API_REQUEST_TIMEOUT_MS = 15_000
+
export type RedeemOperation = 'completion_stream' | 'completion_cancel' | 'connection_test'
export type RedeemAttemptSuccess = {
@@ -100,7 +104,7 @@ export class ControlApiClient {
'x-service-token': this.config.serviceName,
},
body: JSON.stringify(payload),
- signal: AbortSignal.timeout(15_000),
+ signal: AbortSignal.timeout(CONTROL_API_REQUEST_TIMEOUT_MS),
})
const raw = await response.text()
let parsed: unknown = null
@@ -126,11 +130,10 @@ function parseRedeem(body: unknown): RedeemAttemptSuccess {
throw new ControlApiClientError('provider_unavailable', 'redeem response is invalid')
}
const transport = body.transport
- // Absent keeps the historical 300s default. A present value must be a
- // finite positive number: zero/negative would reach AbortSignal.timeout as a
- // RangeError, and a non-number is a control-api contract violation.
- const maxStreamDurationMs =
- transport.maxStreamDurationMs === undefined ? 300_000 : transport.maxStreamDurationMs
+ // control-api sends this on every redeem, so an absent value is a contract
+ // violation like a non-number. Zero/negative would reach AbortSignal.timeout
+ // as a RangeError.
+ const maxStreamDurationMs = transport.maxStreamDurationMs
if (
typeof maxStreamDurationMs !== 'number' ||
!Number.isFinite(maxStreamDurationMs) ||
diff --git a/codex-llm-proxy/src/requestLimits.ts b/codex-llm-proxy/src/requestLimits.ts
index c56762de5..84e9c4242 100644
--- a/codex-llm-proxy/src/requestLimits.ts
+++ b/codex-llm-proxy/src/requestLimits.ts
@@ -3,7 +3,11 @@ import { LIMITS as CONTRACT_LIMITS } from '@clerum/llm-provider-attempt-contract
export const STREAM_LIMITS = {
maxConcurrentStreams: 8,
maxQueuedRequests: 16,
- maxStreamDurationMs: 300_000,
+ maxStreamDurationMs: 1_800_000,
+ // Longest time a request may wait for a stream slot. Bounded so that queue
+ // wait + redeem + the first keepalive stays below the Host HTTP client's
+ // 300 s header timeout.
+ maxQueueWaitMs: 60_000,
// Longest silence tolerated while waiting on the upstream (response headers
// or the next SSE chunk). Matches the Codex CLI stream idle timeout.
upstreamIdleTimeoutMs: 300_000,
@@ -39,13 +43,15 @@ export class StreamGate {
constructor(
private readonly maxConcurrent: number = STREAM_LIMITS.maxConcurrentStreams,
- private readonly maxQueued: number = STREAM_LIMITS.maxQueuedRequests
+ private readonly maxQueued: number = STREAM_LIMITS.maxQueuedRequests,
+ private readonly maxQueueWaitMs: number = STREAM_LIMITS.maxQueueWaitMs
) {}
/**
* Take a stream slot. When `signal` aborts (client disconnected) while the
* caller is still queued, the waiter is rejected and its queue slot freed, so
- * a dropped client never proceeds to redeem an attempt.
+ * a dropped client never proceeds to redeem an attempt. A waiter still
+ * queued after `maxQueueWaitMs` is rejected the same way.
*/
async acquire(signal?: AbortSignal): Promise<() => void> {
if (signal?.aborted) throw new RequestLimitError('stream request was aborted')
@@ -53,6 +59,7 @@ export class StreamGate {
if (this.queued >= this.maxQueued) throw new RequestLimitError('stream queue is full')
this.queued += 1
try {
+ const queuedAt = Date.now()
await new Promise<void>((resolve, reject) => {
let timer: ReturnType<typeof setTimeout> | undefined
const onAbort = () => {
@@ -65,6 +72,11 @@ export class StreamGate {
resolve()
return
}
+ if (Date.now() - queuedAt >= this.maxQueueWaitMs) {
+ signal?.removeEventListener('abort', onAbort)
+ reject(new RequestLimitError('stream queue wait exceeded'))
+ return
+ }
timer = setTimeout(wait, 10)
}
signal?.addEventListener('abort', onAbort, { once: true })
diff --git a/codex-llm-proxy/test/approvedToolsUpstream.test.ts b/codex-llm-proxy/test/approvedToolsUpstream.test.ts
index 099f787f6..beb481e0f 100644
--- a/codex-llm-proxy/test/approvedToolsUpstream.test.ts
+++ b/codex-llm-proxy/test/approvedToolsUpstream.test.ts
@@ -403,6 +403,7 @@ function transport(
duplicate: false,
}))
const pending = streamCodexCompletion({
+ maxDeadlineMs: 1_800_000,
executionTicket: `ticket-${id}`,
requestHash,
request: probeRequest,
@@ -424,7 +425,7 @@ function transport(
catalogOrigin: 'https://chatgpt.com/backend-api/codex/models?client_version=1.0.0',
operation: 'completion_stream',
servedModel: probeRequest.model,
- maxStreamDurationMs: 300_000,
+ maxStreamDurationMs: 1_800_000,
},
expiryClass: 'short_lived',
attemptReceipt: 'b'.repeat(64),
diff --git a/codex-llm-proxy/test/codexTransport.conformance.test.ts b/codex-llm-proxy/test/codexTransport.conformance.test.ts
index 3f0118758..044317e5b 100644
--- a/codex-llm-proxy/test/codexTransport.conformance.test.ts
+++ b/codex-llm-proxy/test/codexTransport.conformance.test.ts
@@ -49,7 +49,7 @@ function redeemSuccess(overrides: Partial<RedeemAttemptSuccess> = {}): RedeemAtt
catalogOrigin: 'https://chatgpt.com/backend-api/codex/models?client_version=1.0.0',
operation: 'completion_stream',
servedModel: 'gpt-5.1',
- maxStreamDurationMs: 300_000,
+ maxStreamDurationMs: 1_800_000,
},
expiryClass: 'short_lived',
attemptReceipt: 'a'.repeat(64),
@@ -97,6 +97,7 @@ describe('streamCodexCompletion', () => {
})
const frames: unknown[] = []
const pending = streamCodexCompletion({
+ maxDeadlineMs: 1_800_000,
executionTicket: 'ticket-drain',
requestHash: REQUEST_HASH,
request: REQUEST,
@@ -146,6 +147,7 @@ describe('streamCodexCompletion', () => {
const abort = new AbortController()
abort.abort()
const result = await streamCodexCompletion({
+ maxDeadlineMs: 1_800_000,
executionTicket: 'ticket-pre-abort',
requestHash: REQUEST_HASH,
request: REQUEST,
@@ -183,6 +185,7 @@ describe('streamCodexCompletion', () => {
sseResponse(['data: {"type":"response.completed"}\n\n'])
)
await streamCodexCompletion({
+ maxDeadlineMs: 1_800_000,
executionTicket: 'ticket-optionality',
requestHash,
request,
@@ -267,6 +270,7 @@ describe('streamCodexCompletion', () => {
const requestHash = hashCodexCompletionRequestV1(request)
const frames: unknown[] = []
await streamCodexCompletion({
+ maxDeadlineMs: 1_800_000,
executionTicket: 'ticket-arguments',
requestHash,
request,
@@ -309,6 +313,7 @@ describe('streamCodexCompletion', () => {
if (terminal !== 'unterminated')
frames.push(`data: ${JSON.stringify({ type: terminal })}\n\n`)
const pending = streamCodexCompletion({
+ maxDeadlineMs: 1_800_000,
executionTicket: 'ticket-partial',
requestHash: REQUEST_HASH,
request: REQUEST,
@@ -355,6 +360,7 @@ describe('streamCodexCompletion', () => {
)
frames.push('data: {"type":"response.completed"}\n\n')
const pending = streamCodexCompletion({
+ maxDeadlineMs: 1_800_000,
executionTicket: 'ticket-bound',
requestHash: REQUEST_HASH,
request: REQUEST,
@@ -408,6 +414,7 @@ describe('streamCodexCompletion', () => {
})
const result = await streamCodexCompletion({
+ maxDeadlineMs: 1_800_000,
executionTicket: 'ticket-1',
requestHash: REQUEST_HASH,
request: REQUEST,
@@ -460,6 +467,7 @@ describe('streamCodexCompletion', () => {
])
)
const result = await streamCodexCompletion({
+ maxDeadlineMs: 1_800_000,
executionTicket: 'ticket-1',
requestHash: REQUEST_HASH,
request: REQUEST,
@@ -490,6 +498,7 @@ describe('streamCodexCompletion', () => {
])
)
const result = await streamCodexCompletion({
+ maxDeadlineMs: 1_800_000,
executionTicket: 'ticket-1',
requestHash: REQUEST_HASH,
request: REQUEST,
@@ -518,6 +527,7 @@ describe('streamCodexCompletion', () => {
sseResponse(['data: {"type":"response.completed","response":{"usage":{}}}\n\n'])
)
await streamCodexCompletion({
+ maxDeadlineMs: 1_800_000,
executionTicket: 'ticket-1',
requestHash: REQUEST_HASH,
request: REQUEST,
@@ -546,6 +556,7 @@ describe('streamCodexCompletion', () => {
sseResponse(['data: {"type":"response.completed","response":{"usage":{}}}\n\n'])
)
await streamCodexCompletion({
+ maxDeadlineMs: 1_800_000,
executionTicket: 'ticket-1',
requestHash: REQUEST_HASH,
request: REQUEST,
@@ -576,6 +587,7 @@ describe('streamCodexCompletion', () => {
const fetchFn = vi.fn()
await expect(
streamCodexCompletion({
+ maxDeadlineMs: 1_800_000,
executionTicket: 'ticket-1',
requestHash: REQUEST_HASH,
request: REQUEST,
@@ -603,6 +615,7 @@ describe('streamCodexCompletion', () => {
const redeem = vi.fn()
await expect(
streamCodexCompletion({
+ maxDeadlineMs: 1_800_000,
executionTicket: 'ticket-1',
requestHash: 'f'.repeat(64),
request: REQUEST,
@@ -626,6 +639,7 @@ describe('streamCodexCompletion', () => {
it('rejects a served-model mismatch and loopback redirects', async () => {
await expect(
streamCodexCompletion({
+ maxDeadlineMs: 1_800_000,
executionTicket: 'ticket-1',
requestHash: REQUEST_HASH,
request: REQUEST,
@@ -656,6 +670,7 @@ describe('streamCodexCompletion', () => {
)
await expect(
streamCodexCompletion({
+ maxDeadlineMs: 1_800_000,
executionTicket: 'ticket-1',
requestHash: REQUEST_HASH,
request: REQUEST,
@@ -693,6 +708,7 @@ describe('streamCodexCompletion', () => {
}))
const order: string[] = []
await streamCodexCompletion({
+ maxDeadlineMs: 1_800_000,
executionTicket: 'ticket-redeemed',
requestHash: REQUEST_HASH,
request: REQUEST,
@@ -722,6 +738,7 @@ describe('streamCodexCompletion', () => {
for (const redeem of [denied, mismatched]) {
await expect(
streamCodexCompletion({
+ maxDeadlineMs: 1_800_000,
executionTicket: 'ticket-redeemed',
requestHash: REQUEST_HASH,
request: REQUEST,
@@ -751,6 +768,7 @@ describe('streamCodexCompletion', () => {
return sseResponse(['data: {"type":"response.completed","response":{"usage":{}}}\n\n'])
})
const result = await streamCodexCompletion({
+ maxDeadlineMs: 1_800_000,
executionTicket: 'ticket-1',
requestHash: REQUEST_HASH,
request: REQUEST,
@@ -777,6 +795,7 @@ describe('streamCodexCompletion', () => {
it('maps upstream 401 to connection_unavailable instead of origin_denied', async () => {
await expect(
streamCodexCompletion({
+ maxDeadlineMs: 1_800_000,
executionTicket: 'ticket-1',
requestHash: REQUEST_HASH,
request: REQUEST,
@@ -827,6 +846,7 @@ describe('streamCodexCompletion', () => {
const frames: unknown[] = []
const result = await streamCodexCompletion({
+ maxDeadlineMs: 1_800_000,
executionTicket: 'ticket-1',
requestHash: REQUEST_HASH,
request: REQUEST,
@@ -871,6 +891,7 @@ describe('streamCodexCompletion', () => {
providerAttemptId: 'att-1',
}
await streamCodexCompletion({
+ maxDeadlineMs: 1_800_000,
executionTicket: 't-a',
requestHash: REQUEST_HASH,
request: REQUEST,
@@ -885,6 +906,7 @@ describe('streamCodexCompletion', () => {
lookup: async () => [{ address: '1.2.3.4', family: 4 }],
})
await streamCodexCompletion({
+ maxDeadlineMs: 1_800_000,
executionTicket: 't-b',
requestHash: REQUEST_HASH,
request: REQUEST,
@@ -912,6 +934,7 @@ describe('streamCodexCompletion', () => {
sseResponse(['data: {"type":"response.output_text.delta","delta":"partial"}\n\n'])
)
const result = await streamCodexCompletion({
+ maxDeadlineMs: 1_800_000,
executionTicket: 'ticket-1',
requestHash: REQUEST_HASH,
request: REQUEST,
@@ -965,6 +988,7 @@ describe('streamCodexCompletion', () => {
sseResponse(['data: {"type":"response.completed","response":{"usage":{}}}\n\n'])
)
await streamCodexCompletion({
+ maxDeadlineMs: 1_800_000,
executionTicket: 'ticket-1',
requestHash,
request,
@@ -1062,6 +1086,7 @@ describe('streamCodexCompletion', () => {
])
})
const result = await streamCodexCompletion({
+ maxDeadlineMs: 1_800_000,
executionTicket: 'ticket-name-map',
requestHash,
request,
@@ -1117,6 +1142,7 @@ describe('streamCodexCompletion', () => {
})
await expect(
streamCodexCompletion({
+ maxDeadlineMs: 1_800_000,
executionTicket: 'ticket-unknown-alias',
requestHash,
request,
@@ -1153,6 +1179,7 @@ describe('streamCodexCompletion', () => {
sseResponse(['data: {"type":"response.completed","response":{"usage":{}}}\n\n'])
)
await streamCodexCompletion({
+ maxDeadlineMs: 1_800_000,
executionTicket: 'ticket-1',
requestHash: REQUEST_HASH,
request: REQUEST,
@@ -1194,6 +1221,7 @@ describe('streamCodexCompletion', () => {
sseResponse(['data: {"type":"response.completed","response":{"usage":{}}}\n\n'])
)
await streamCodexCompletion({
+ maxDeadlineMs: 1_800_000,
executionTicket: 'ticket-1',
requestHash,
request,
@@ -1230,6 +1258,7 @@ describe('streamCodexCompletion', () => {
)
await expect(
streamCodexCompletion({
+ maxDeadlineMs: 1_800_000,
executionTicket: 'ticket-1',
requestHash: REQUEST_HASH,
request: REQUEST,
@@ -1269,6 +1298,7 @@ describe('streamCodexCompletion', () => {
])
)
await streamCodexCompletion({
+ maxDeadlineMs: 1_800_000,
executionTicket: 'ticket-1',
requestHash: REQUEST_HASH,
request: REQUEST,
diff --git a/codex-llm-proxy/test/controlApiClient.test.ts b/codex-llm-proxy/test/controlApiClient.test.ts
index 9f47ad989..30f81abc9 100644
--- a/codex-llm-proxy/test/controlApiClient.test.ts
+++ b/codex-llm-proxy/test/controlApiClient.test.ts
@@ -31,7 +31,7 @@ function listen(handler: (req: IncomingMessage, body: unknown, res: Server) => v
catalogOrigin: 'https://chatgpt.com/backend-api/codex/models?client_version=1.0.0',
operation: 'completion_stream',
servedModel: 'gpt-5.1',
- maxStreamDurationMs: 300000,
+ maxStreamDurationMs: 1800000,
},
expiryClass: 'short_lived',
attemptReceipt: 'a'.repeat(64),
@@ -235,7 +235,9 @@ describe('ControlApiClient response hardening', () => {
}
)
- it('keeps a positive maxStreamDurationMs and defaults an absent one to 300000', async () => {
+ // control-api has sent maxStreamDurationMs on every redeem since the route
+ // existed, so an absent value is a contract violation, not an old server.
+ it('keeps a positive maxStreamDurationMs and rejects an absent one', async () => {
const positive = await jsonServer(redeemBody(120_000))
const absent = await jsonServer(redeemBody(undefined))
try {
@@ -247,7 +249,10 @@ describe('ControlApiClient response hardening', () => {
expect((await client(positive.url).redeem(input)).transport.maxStreamDurationMs).toBe(
120_000
)
- expect((await client(absent.url).redeem(input)).transport.maxStreamDurationMs).toBe(300_000)
+ await expect(client(absent.url).redeem(input)).rejects.toMatchObject({
+ code: 'provider_unavailable',
+ message: 'redeem maxStreamDurationMs is invalid',
+ })
} finally {
await positive.close()
await absent.close()
diff --git a/codex-llm-proxy/test/requestLimits.test.ts b/codex-llm-proxy/test/requestLimits.test.ts
index ed7ec59fa..deb988815 100644
--- a/codex-llm-proxy/test/requestLimits.test.ts
+++ b/codex-llm-proxy/test/requestLimits.test.ts
@@ -1,5 +1,23 @@
import { describe, expect, it } from 'vitest'
-import { RequestLimitError, StreamGate } from '../src/requestLimits.js'
+import { CONTROL_API_REQUEST_TIMEOUT_MS } from '../src/controlApiClient.js'
+import { RequestLimitError, STREAM_LIMITS, StreamGate } from '../src/requestLimits.js'
+import { DEFAULT_HEARTBEAT_INTERVAL_MS } from '../src/sseHeartbeat.js'
+
+// undici's default headersTimeout and bodyTimeout in the Node 24.16.0 image the
+// Host runs; the Host sets neither for its proxy calls.
+const HOST_UNDICI_TIMEOUT_MS = 300_000
+
+describe('stream timing invariant', () => {
+ it('reaches the first keepalive before the Host HTTP client times out', () => {
+ expect(STREAM_LIMITS.maxStreamDurationMs).toBe(1_800_000)
+ expect(STREAM_LIMITS.maxQueueWaitMs).toBe(60_000)
+ // Nothing is written while a request waits for a slot or for the redeem,
+ // so both must end, and one heartbeat interval pass, inside the timeout.
+ expect(
+ STREAM_LIMITS.maxQueueWaitMs + CONTROL_API_REQUEST_TIMEOUT_MS + DEFAULT_HEARTBEAT_INTERVAL_MS
+ ).toBeLessThan(HOST_UNDICI_TIMEOUT_MS)
+ })
+})
describe('StreamGate', () => {
it('admits a queued waiter once a running stream releases', async () => {
@@ -53,4 +71,21 @@ describe('StreamGate', () => {
const releaseNext = await next
releaseNext()
})
+
+ it('rejects a waiter queued longer than maxQueueWaitMs and frees its queue slot', async () => {
+ const gate = new StreamGate(1, 1, 50)
+ const release = await gate.acquire()
+ const started = Date.now()
+ await expect(gate.acquire()).rejects.toMatchObject({
+ name: 'RequestLimitError',
+ message: 'stream queue wait exceeded',
+ })
+ // Witness: the waiter was queued for the whole wait, not refused at once.
+ expect(Date.now() - started).toBeGreaterThanOrEqual(45)
+ // The queue slot is free again, so a new waiter queues instead of "queue is full".
+ const next = gate.acquire()
+ release()
+ const releaseNext = await next
+ releaseNext()
+ })
})
diff --git a/codex-llm-proxy/test/server.security.test.ts b/codex-llm-proxy/test/server.security.test.ts
index b0f930b8c..3934ba6e6 100644
--- a/codex-llm-proxy/test/server.security.test.ts
+++ b/codex-llm-proxy/test/server.security.test.ts
@@ -31,8 +31,8 @@ function config(overrides: Partial<CodexLlmProxyConfig> = {}): CodexLlmProxyConf
adminPort: 8081,
probePort: 9090,
maxBodyBytes: 1024,
- maxStreamDurationMs: 300_000,
- maxDeadlineMs: 300_000,
+ maxStreamDurationMs: 1_800_000,
+ maxDeadlineMs: 1_800_000,
upstreamIdleTimeoutMs: 300_000,
heartbeatIntervalMs: 15_000,
jwtIssuer: 'control-api',
@@ -137,7 +137,7 @@ describe('codex-llm-proxy security surface', () => {
executionTicket: ticket(),
requestHash: 'a'.repeat(64),
request: {},
- deadlineMs: 999_999,
+ deadlineMs: 1_800_001,
})
expect(deadline.body.error).toBe('invalid_request')
@@ -319,6 +319,16 @@ describe('codex-llm-proxy security surface', () => {
).toBe(45_000)
})
+ it('defaults the total stream cap and the deadline ceiling to 30 min', () => {
+ const loaded = loadConfig({
+ CODEX_LLM_PROXY_JWT_PUBLIC_KEY: publicKey,
+ CODEX_LLM_PROXY_CONTROL_API_URL: 'http://control-api:8080',
+ CODEX_LLM_PROXY_CONTROL_API_TOKEN: 'service-token',
+ })
+ expect(loaded.maxStreamDurationMs).toBe(1_800_000)
+ expect(loaded.maxDeadlineMs).toBe(1_800_000)
+ })
+
it('defaults the SSE heartbeat to 15 s and rejects a non-positive interval', () => {
const env = {
CODEX_LLM_PROXY_JWT_PUBLIC_KEY: publicKey,
@@ -531,7 +541,7 @@ describe('codex-llm-proxy attempt telemetry', () => {
return { executionTicket, requestHash, request: raw }
}
- function grantingClient(maxStreamDurationMs = 300_000): {
+ function grantingClient(maxStreamDurationMs = 1_800_000): {
client: ControlApiClient
receipts: unknown[]
} {
diff --git a/codex-llm-proxy/test/streamLimitsFreeze.test.ts b/codex-llm-proxy/test/streamLimitsFreeze.test.ts
index 21e141065..62c568c3b 100644
--- a/codex-llm-proxy/test/streamLimitsFreeze.test.ts
+++ b/codex-llm-proxy/test/streamLimitsFreeze.test.ts
@@ -45,6 +45,7 @@ describe('codex-subscription stream limits freeze', () => {
const streamLimits: Record<string, number> = { ...STREAM_LIMITS }
expect(Object.keys(streamLimits).sort()).toEqual([
'maxConcurrentStreams',
+ 'maxQueueWaitMs',
'maxQueuedRequests',
'maxStreamDurationMs',
'upstreamIdleTimeoutMs',
diff --git a/control-api/package-lock.json b/control-api/package-lock.json
index cc624b3db..f64de5231 100644
--- a/control-api/package-lock.json
+++ b/control-api/package-lock.json
@@ -1,12 +1,12 @@
{
"name": "clerum-control-api",
- "version": "0.1.599",
+ "version": "0.1.600",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "clerum-control-api",
- "version": "0.1.599",
+ "version": "0.1.600",
"dependencies": {
"@clerum/display-field": "file:../packages/display-field",
"@clerum/grok-provider-attempt-contract": "file:../packages/grok-provider-attempt-contract",
diff --git a/control-api/package.json b/control-api/package.json
index 944361935..20033d94b 100644
--- a/control-api/package.json
+++ b/control-api/package.json
@@ -1,6 +1,6 @@
{
"name": "clerum-control-api",
- "version": "0.1.599",
+ "version": "0.1.600",
"description": "Internal Clerum control plane REST API + UI for managing CRDs and secrets",
"main": "dist/main.js",
"engines": {
diff --git a/control-api/src/services/grokProviderAttemptRedemption.ts b/control-api/src/services/grokProviderAttemptRedemption.ts
index 7bc27b11e..8c759a278 100644
--- a/control-api/src/services/grokProviderAttemptRedemption.ts
+++ b/control-api/src/services/grokProviderAttemptRedemption.ts
@@ -26,7 +26,7 @@ const log = rootLogger.child({ module: 'grok-provider-attempt-redemption' })
export const GROK_COMPLETIONS_ORIGIN = COMPLETIONS_ORIGIN
export const GROK_CATALOG_ORIGIN = CATALOG_ORIGIN
export const GROK_TRANSPORT_PROTOCOL = TRANSPORT_PROTOCOL_VERSION
-export const GROK_MAX_STREAM_DURATION_MS = 300_000
+export const GROK_MAX_STREAM_DURATION_MS = 1_800_000
export type GrokProviderAttemptRedeemErrorCode =
| 'disabled'
diff --git a/control-api/src/services/llmProviderAttemptRedemption.ts b/control-api/src/services/llmProviderAttemptRedemption.ts
index c23b8fa21..f0b783e00 100644
--- a/control-api/src/services/llmProviderAttemptRedemption.ts
+++ b/control-api/src/services/llmProviderAttemptRedemption.ts
@@ -25,7 +25,7 @@ export const CODEX_COMPLETIONS_ORIGIN = 'https://chatgpt.com/backend-api/codex/r
export const CODEX_CATALOG_ORIGIN =
'https://chatgpt.com/backend-api/codex/models?client_version=1.0.0'
export const CODEX_TRANSPORT_PROTOCOL = 'codex-subscription-transport.v1'
-export const CODEX_MAX_STREAM_DURATION_MS = 300_000
+export const CODEX_MAX_STREAM_DURATION_MS = 1_800_000
export type LlmProviderAttemptRedeemErrorCode =
| 'disabled'
diff --git a/control-api/test/services.grokProviderAttemptRedemption.test.ts b/control-api/test/services.grokProviderAttemptRedemption.test.ts
index e21c48ce7..f0bf64e5f 100644
--- a/control-api/test/services.grokProviderAttemptRedemption.test.ts
+++ b/control-api/test/services.grokProviderAttemptRedemption.test.ts
@@ -294,6 +294,8 @@ describe('redeemGrokProviderAttempt error-path matrix', () => {
expect(result.accessToken).toBe('access-secret')
expect(result.expiryClass).toBe('upstream_managed')
expect(result.transport.servedModel).toBe(CLAIMS.model)
+ // The per-attempt total cap the proxy enforces (30 min).
+ expect(result.transport.maxStreamDurationMs).toBe(1_800_000)
expect(JSON.stringify(result)).not.toContain('refresh-secret')
expect(h.ensureFresh).toHaveBeenCalledWith('team-grok')
expect(h.markRedeemed).toHaveBeenCalledTimes(1)
diff --git a/control-api/test/services.llmProviderAttemptRedemption.test.ts b/control-api/test/services.llmProviderAttemptRedemption.test.ts
index a303adcdb..59c2ee35a 100644
--- a/control-api/test/services.llmProviderAttemptRedemption.test.ts
+++ b/control-api/test/services.llmProviderAttemptRedemption.test.ts
@@ -114,6 +114,8 @@ describe('redeemLlmProviderAttempt', () => {
expect(result.transport.completionsOrigin).toBe(
'https://chatgpt.com/backend-api/codex/responses'
)
+ // The per-attempt total cap the proxy enforces (30 min).
+ expect(result.transport.maxStreamDurationMs).toBe(1_800_000)
expect(JSON.stringify(result)).not.toContain('refresh-secret')
expect(JSON.stringify(result)).not.toMatch(/encrypted|Authorization/i)
})
diff --git a/deploy/base/control-plane/codex-llm-proxy.yaml b/deploy/base/control-plane/codex-llm-proxy.yaml
index 71e3abb29..41630ac67 100644
--- a/deploy/base/control-plane/codex-llm-proxy.yaml
+++ b/deploy/base/control-plane/codex-llm-proxy.yaml
@@ -19,8 +19,8 @@ data:
CODEX_LLM_PROXY_ADMIN_PORT: "8081"
CODEX_LLM_PROXY_PROBE_PORT: "9090"
CODEX_LLM_PROXY_MAX_BODY_BYTES: "1048576"
- CODEX_LLM_PROXY_MAX_STREAM_DURATION_MS: "300000"
- CODEX_LLM_PROXY_MAX_DEADLINE_MS: "300000"
+ CODEX_LLM_PROXY_MAX_STREAM_DURATION_MS: "1800000"
+ CODEX_LLM_PROXY_MAX_DEADLINE_MS: "1800000"
CODEX_LLM_PROXY_JWT_ISSUER: control-api
CODEX_LLM_PROXY_EXECUTION_ENABLED: "false"
CODEX_LLM_PROXY_CONTROL_API_URL: http://control-api-rpc-gateway.control-plane.svc.cluster.local:8090/api/v1
diff --git a/deploy/base/control-plane/grok-llm-proxy.yaml b/deploy/base/control-plane/grok-llm-proxy.yaml
index 0ef0e570c..96aec866f 100644
--- a/deploy/base/control-plane/grok-llm-proxy.yaml
+++ b/deploy/base/control-plane/grok-llm-proxy.yaml
@@ -19,8 +19,8 @@ data:
GROK_LLM_PROXY_ADMIN_PORT: "8081"
GROK_LLM_PROXY_PROBE_PORT: "9090"
GROK_LLM_PROXY_MAX_BODY_BYTES: "1048576"
- GROK_LLM_PROXY_MAX_STREAM_DURATION_MS: "300000"
- GROK_LLM_PROXY_MAX_DEADLINE_MS: "300000"
+ GROK_LLM_PROXY_MAX_STREAM_DURATION_MS: "1800000"
+ GROK_LLM_PROXY_MAX_DEADLINE_MS: "1800000"
GROK_LLM_PROXY_JWT_ISSUER: control-api
GROK_LLM_PROXY_EXECUTION_ENABLED: "false"
GROK_LLM_PROXY_CONTROL_API_URL: http://control-api-rpc-gateway.control-plane.svc.cluster.local:8090/api/v1
diff --git a/docs/architecture/codex-subscription-transport-contract.md b/docs/architecture/codex-subscription-transport-contract.md
index 71afe4c83..f396f1aed 100644
--- a/docs/architecture/codex-subscription-transport-contract.md
+++ b/docs/architecture/codex-subscription-transport-contract.md
@@ -55,10 +55,11 @@ one physical execution per ticket. A retry or fallback must mint a new attempt.
| maxMessages | 1024 |
| maxToolCalls | 256 |
| maxOutputTokens | 16384 |
-| maxStreamDurationMs | 300000 |
-| maxDeadlineMs | 300000 |
+| maxStreamDurationMs | 1800000 |
+| maxDeadlineMs | 1800000 |
| maxConcurrentStreams | 8 |
| maxQueuedRequests | 16 |
+| maxQueueWaitMs | 60000 |
| upstreamIdleTimeoutMs | 300000 |
| maxRetriesPerAttempt | 1 |
@@ -199,7 +200,12 @@ behavior changes:
- It respects SSE write backpressure.
- It drops queued stream-gate waiters on abort and checks the abort signal
before redeeming a ticket.
- - It requires `maxStreamDurationMs` greater than 0.
+ - It rejects a stream-gate waiter still queued after `maxQueueWaitMs` with
+ `provider_unavailable` (reason `stream queue wait exceeded`). Queue wait,
+ the 15 s control-api redeem timeout and the first keepalive together stay
+ below the Host HTTP client's 300 s header timeout.
+ - It requires the redeem response to carry `maxStreamDurationMs` greater
+ than 0. An absent value is a contract violation, not a default.
- It logs one `codex_proxy_attempt_finished` event per completion attempt,
with identifiers and counts only (never the body, ticket, frames, tool
names or arguments): `providerAttemptId`, `hostRef`, `model`,
diff --git a/docs/architecture/grok-subscription-transport-contract.md b/docs/architecture/grok-subscription-transport-contract.md
index 7822e0a83..0555dc920 100644
--- a/docs/architecture/grok-subscription-transport-contract.md
+++ b/docs/architecture/grok-subscription-transport-contract.md
@@ -166,10 +166,18 @@ Proxy robustness (both proxies):
stops waiting if the client closes.
- Queued stream-gate waiters stop when the request aborts, and the proxy
checks the abort signal before redeeming a ticket.
+- A stream-gate waiter still queued after `maxQueueWaitMs` (60 s) is rejected
+ with `provider_unavailable` (reason `stream queue wait exceeded`). Queue
+ wait, the 15 s control-api redeem timeout and the first keepalive together
+ stay below the Host HTTP client's 300 s header timeout.
+- A single attempt streams for at most `maxStreamDurationMs` (1 800 000 ms):
+ the minimum of the proxy configuration, `STREAM_LIMITS`, the contract
+ `maxDeadlineMs` and the value control-api returns on redeem.
- The proxy fails at startup when `GROK_LLM_PROXY_CONTROL_API_URL` or
`GROK_LLM_PROXY_CONTROL_API_TOKEN` is empty.
- An unrecognized finalize outcome maps to `unknown`, never `success`.
- `maxStreamDurationMs` must be greater than 0.
+ The redeem response must carry `maxStreamDurationMs` greater than 0; an
+ absent value is a contract violation, not a default.
## Identity headers
diff --git a/grok-llm-proxy/src/config.ts b/grok-llm-proxy/src/config.ts
index 5c309fdd3..333874e58 100644
--- a/grok-llm-proxy/src/config.ts
+++ b/grok-llm-proxy/src/config.ts
@@ -87,12 +87,12 @@ export function loadConfig(env: NodeJS.ProcessEnv = process.env): GrokLlmProxyCo
maxStreamDurationMs: requiredPositiveInt(
'GROK_LLM_PROXY_MAX_STREAM_DURATION_MS',
env.GROK_LLM_PROXY_MAX_STREAM_DURATION_MS,
- 300_000
+ 1_800_000
),
maxDeadlineMs: requiredPositiveInt(
'GROK_LLM_PROXY_MAX_DEADLINE_MS',
env.GROK_LLM_PROXY_MAX_DEADLINE_MS,
- 300_000
+ 1_800_000
),
upstreamIdleTimeoutMs: requiredPositiveInt(
'GROK_LLM_PROXY_UPSTREAM_IDLE_TIMEOUT_MS',
diff --git a/grok-llm-proxy/src/controlApiClient.ts b/grok-llm-proxy/src/controlApiClient.ts
index de023cce2..ae87b0b4b 100644
--- a/grok-llm-proxy/src/controlApiClient.ts
+++ b/grok-llm-proxy/src/controlApiClient.ts
@@ -5,6 +5,10 @@ import {
GROK_TRANSPORT_PROTOCOL,
} from './originPolicy.js'
+// Bound on one control-api call (redeem or finalize). Part of the time a
+// stream request may spend before its first byte; see STREAM_LIMITS.
+export const CONTROL_API_REQUEST_TIMEOUT_MS = 15_000
+
export type RedeemOperation = 'completion_stream' | 'completion_cancel' | 'connection_test'
export type RedeemAttemptSuccess = {
@@ -104,7 +108,7 @@ export class ControlApiClient {
'x-service-token': this.config.serviceName,
},
body: JSON.stringify(payload),
- signal: AbortSignal.timeout(15_000),
+ signal: AbortSignal.timeout(CONTROL_API_REQUEST_TIMEOUT_MS),
})
const raw = await response.text()
let parsed: unknown = null
@@ -139,11 +143,10 @@ function parseRedeem(body: unknown): RedeemAttemptSuccess {
throw new ControlApiClientError('provider_unavailable', 'redeem response is invalid')
}
const transport = body.transport
- // Absent keeps the historical 300s default. A present value must be a
- // finite positive number: zero/negative would reach AbortSignal.timeout as a
- // RangeError, and a non-number is a control-api contract violation.
- const maxStreamDurationMs =
- transport.maxStreamDurationMs === undefined ? 300_000 : transport.maxStreamDurationMs
+ // control-api sends this on every redeem, so an absent value is a contract
+ // violation like a non-number. Zero/negative would reach AbortSignal.timeout
+ // as a RangeError.
+ const maxStreamDurationMs = transport.maxStreamDurationMs
if (
typeof maxStreamDurationMs !== 'number' ||
!Number.isFinite(maxStreamDurationMs) ||
diff --git a/grok-llm-proxy/src/grokTransport.ts b/grok-llm-proxy/src/grokTransport.ts
index fc1878164..3a504c65a 100644
--- a/grok-llm-proxy/src/grokTransport.ts
+++ b/grok-llm-proxy/src/grokTransport.ts
@@ -99,7 +99,7 @@ export type StreamGrokCompletionInput = {
request: unknown
ticket: TransportTicket
deadlineMs?: number
- maxDeadlineMs?: number
+ maxDeadlineMs: number
/** Lowers `STREAM_LIMITS.upstreamIdleTimeoutMs`; never raises it. */
upstreamIdleTimeoutMs?: number
signal?: AbortSignal
@@ -168,7 +168,7 @@ export async function streamGrokCompletion(
throw new GrokTransportError('model_not_allowed', 'served model does not match the request')
}
const deadlineMs = Math.min(
- assertBoundedDeadline(input.deadlineMs ?? request.deadlineMs, input.maxDeadlineMs ?? 300_000),
+ assertBoundedDeadline(input.deadlineMs ?? request.deadlineMs, input.maxDeadlineMs),
redeemed.transport.maxStreamDurationMs
)
const idleTimeoutMs = assertBoundedIdleTimeout(input.upstreamIdleTimeoutMs)
diff --git a/grok-llm-proxy/src/requestLimits.ts b/grok-llm-proxy/src/requestLimits.ts
index 5da88d377..5b2f70a28 100644
--- a/grok-llm-proxy/src/requestLimits.ts
+++ b/grok-llm-proxy/src/requestLimits.ts
@@ -3,7 +3,11 @@ import { LIMITS as CONTRACT_LIMITS } from '@clerum/grok-provider-attempt-contrac
export const STREAM_LIMITS = {
maxConcurrentStreams: 8,
maxQueuedRequests: 16,
- maxStreamDurationMs: 300_000,
+ maxStreamDurationMs: 1_800_000,
+ // Longest time a request may wait for a stream slot. Bounded so that queue
+ // wait + redeem + the first keepalive stays below the Host HTTP client's
+ // 300 s header timeout.
+ maxQueueWaitMs: 60_000,
// Longest silence tolerated while waiting on the upstream (response headers
// or the next SSE chunk). Matches the Grok Build inference idle timeout.
upstreamIdleTimeoutMs: 600_000,
@@ -47,13 +51,15 @@ export class StreamGate {
constructor(
private readonly maxConcurrent: number = STREAM_LIMITS.maxConcurrentStreams,
- private readonly maxQueued: number = STREAM_LIMITS.maxQueuedRequests
+ private readonly maxQueued: number = STREAM_LIMITS.maxQueuedRequests,
+ private readonly maxQueueWaitMs: number = STREAM_LIMITS.maxQueueWaitMs
) {}
/**
* Take a stream slot. When `signal` aborts (client disconnected) while the
* caller is still queued, the waiter is rejected and its queue slot freed, so
- * a dropped client never proceeds to redeem an attempt.
+ * a dropped client never proceeds to redeem an attempt. A waiter still
+ * queued after `maxQueueWaitMs` is rejected the same way.
*/
async acquire(signal?: AbortSignal): Promise<() => void> {
if (signal?.aborted) throw new RequestLimitError('stream request was aborted')
@@ -61,6 +67,7 @@ export class StreamGate {
if (this.queued >= this.maxQueued) throw new RequestLimitError('stream queue is full')
this.queued += 1
try {
+ const queuedAt = Date.now()
await new Promise<void>((resolve, reject) => {
let timer: ReturnType<typeof setTimeout> | undefined
const onAbort = () => {
@@ -73,6 +80,11 @@ export class StreamGate {
resolve()
return
}
+ if (Date.now() - queuedAt >= this.maxQueueWaitMs) {
+ signal?.removeEventListener('abort', onAbort)
+ reject(new RequestLimitError('stream queue wait exceeded'))
+ return
+ }
timer = setTimeout(wait, 10)
}
signal?.addEventListener('abort', onAbort, { once: true })
diff --git a/grok-llm-proxy/test/contractFreeze.test.ts b/grok-llm-proxy/test/contractFreeze.test.ts
index c907016dd..9cd4d2825 100644
--- a/grok-llm-proxy/test/contractFreeze.test.ts
+++ b/grok-llm-proxy/test/contractFreeze.test.ts
@@ -35,6 +35,7 @@ const FIXTURE_LIMIT_KEYS = [
'maxDeadlineMs',
'maxConcurrentStreams',
'maxQueuedRequests',
+ 'maxQueueWaitMs',
'upstreamIdleTimeoutMs',
'maxRetriesPerAttempt',
] as const
@@ -187,6 +188,7 @@ describe('grok-subscription contract freeze', () => {
const streamLimits: Record<string, number> = { ...STREAM_LIMITS }
expect(Object.keys(streamLimits).sort()).toEqual([
'maxConcurrentStreams',
+ 'maxQueueWaitMs',
'maxQueuedRequests',
'maxStreamDurationMs',
'upstreamIdleTimeoutMs',
diff --git a/grok-llm-proxy/test/controlApiClient.test.ts b/grok-llm-proxy/test/controlApiClient.test.ts
index bc4726b9e..4b803a4ba 100644
--- a/grok-llm-proxy/test/controlApiClient.test.ts
+++ b/grok-llm-proxy/test/controlApiClient.test.ts
@@ -31,7 +31,7 @@ function listen(handler: (req: IncomingMessage, body: unknown, res: Server) => v
catalogOrigin: 'https://cli-chat-proxy.grok.com/v1/models',
operation: 'completion_stream',
servedModel: 'gpt-5.1',
- maxStreamDurationMs: 300000,
+ maxStreamDurationMs: 1800000,
},
expiryClass: 'short_lived',
attemptReceipt: 'a'.repeat(64),
@@ -244,7 +244,9 @@ describe('ControlApiClient response hardening', () => {
}
)
- it('keeps a positive maxStreamDurationMs and defaults an absent one to 300000', async () => {
+ // control-api has sent maxStreamDurationMs on every redeem since the route
+ // existed, so an absent value is a contract violation, not an old server.
+ it('keeps a positive maxStreamDurationMs and rejects an absent one', async () => {
const positive = await jsonServer(redeemBody(120_000))
const absent = await jsonServer(redeemBody(undefined))
try {
@@ -254,7 +256,10 @@ describe('ControlApiClient response hardening', () => {
operation: 'completion_stream' as const,
}
expect((await client(positive.url).redeem(input)).transport.maxStreamDurationMs).toBe(120_000)
- expect((await client(absent.url).redeem(input)).transport.maxStreamDurationMs).toBe(300_000)
+ await expect(client(absent.url).redeem(input)).rejects.toMatchObject({
+ code: 'provider_unavailable',
+ message: 'redeem maxStreamDurationMs is invalid',
+ })
} finally {
await positive.close()
await absent.close()
diff --git a/grok-llm-proxy/test/grokTransport.conformance.test.ts b/grok-llm-proxy/test/grokTransport.conformance.test.ts
index a682b1d0b..b66fd5da6 100644
--- a/grok-llm-proxy/test/grokTransport.conformance.test.ts
+++ b/grok-llm-proxy/test/grokTransport.conformance.test.ts
@@ -50,7 +50,7 @@ function redeemSuccess(overrides: Partial<RedeemAttemptSuccess> = {}): RedeemAtt
catalogOrigin: 'https://cli-chat-proxy.grok.com/v1/models',
operation: 'completion_stream',
servedModel: 'gpt-5.1',
- maxStreamDurationMs: 300_000,
+ maxStreamDurationMs: 1_800_000,
},
expiryClass: 'short_lived',
attemptReceipt: 'a'.repeat(64),
@@ -147,6 +147,7 @@ describe('streamGrokCompletion', () => {
})
const frames: unknown[] = []
const pending = streamGrokCompletion({
+ maxDeadlineMs: 1_800_000,
executionTicket: 'ticket-drain',
requestHash: REQUEST_HASH,
request: REQUEST,
@@ -203,6 +204,7 @@ describe('streamGrokCompletion', () => {
'{"error":"Your Grok CLI version (none) is outdated. Please update to version 0.1.202 or later."}',
})) as unknown as typeof fetch
const pending = streamGrokCompletion({
+ maxDeadlineMs: 1_800_000,
executionTicket: 'ticket-426',
requestHash: REQUEST_HASH,
request: REQUEST,
@@ -241,6 +243,7 @@ describe('streamGrokCompletion', () => {
const abort = new AbortController()
abort.abort()
const result = await streamGrokCompletion({
+ maxDeadlineMs: 1_800_000,
executionTicket: 'ticket-pre-abort',
requestHash: REQUEST_HASH,
request: REQUEST,
@@ -273,6 +276,7 @@ describe('streamGrokCompletion', () => {
if (terminal !== 'unterminated')
frames.push(`data: ${JSON.stringify({ type: terminal })}\n\n`)
const pending = streamGrokCompletion({
+ maxDeadlineMs: 1_800_000,
executionTicket: 'ticket-partial',
requestHash: REQUEST_HASH,
request: REQUEST,
@@ -319,6 +323,7 @@ describe('streamGrokCompletion', () => {
)
frames.push('data: {"type":"response.completed"}\n\n')
const pending = streamGrokCompletion({
+ maxDeadlineMs: 1_800_000,
executionTicket: 'ticket-bound',
requestHash: REQUEST_HASH,
request: REQUEST,
@@ -376,6 +381,7 @@ describe('streamGrokCompletion', () => {
)
frames.push('data: {"type":"response.completed"}\n\n')
const pending = streamGrokCompletion({
+ maxDeadlineMs: 1_800_000,
executionTicket: 'ticket-bound',
requestHash: REQUEST_HASH,
request: REQUEST,
@@ -428,6 +434,7 @@ describe('streamGrokCompletion', () => {
`data: ${JSON.stringify({ type: 'response.output_text.delta', delta: 'after' })}\n\n`,
]
const pending = streamGrokCompletion({
+ maxDeadlineMs: 1_800_000,
executionTicket: 'ticket-bound',
requestHash: REQUEST_HASH,
request: REQUEST,
@@ -479,6 +486,7 @@ describe('streamGrokCompletion', () => {
`data: ${JSON.stringify({ type: 'response.completed', response: { usage: {} } })}\n\n`,
]
const result = await streamGrokCompletion({
+ maxDeadlineMs: 1_800_000,
executionTicket: 'ticket-args-ok',
requestHash: REQUEST_HASH,
request: REQUEST,
@@ -528,6 +536,7 @@ describe('streamGrokCompletion', () => {
`data: ${JSON.stringify({ type: 'response.completed', response: { usage: {} } })}\n\n`,
]
const pending = streamGrokCompletion({
+ maxDeadlineMs: 1_800_000,
executionTicket: 'ticket-args-over',
requestHash: REQUEST_HASH,
request: REQUEST,
@@ -584,6 +593,7 @@ describe('streamGrokCompletion', () => {
})
const result = await streamGrokCompletion({
+ maxDeadlineMs: 1_800_000,
executionTicket: 'ticket-1',
requestHash: REQUEST_HASH,
request: REQUEST,
@@ -643,6 +653,7 @@ describe('streamGrokCompletion', () => {
])
)
const result = await streamGrokCompletion({
+ maxDeadlineMs: 1_800_000,
executionTicket: 'ticket-1',
requestHash: REQUEST_HASH,
request: REQUEST,
@@ -673,6 +684,7 @@ describe('streamGrokCompletion', () => {
])
)
const result = await streamGrokCompletion({
+ maxDeadlineMs: 1_800_000,
executionTicket: 'ticket-1',
requestHash: REQUEST_HASH,
request: REQUEST,
@@ -702,6 +714,7 @@ describe('streamGrokCompletion', () => {
)
await expect(
streamGrokCompletion({
+ maxDeadlineMs: 1_800_000,
executionTicket: 'ticket-1',
requestHash: REQUEST_HASH,
request: REQUEST,
@@ -728,6 +741,7 @@ describe('streamGrokCompletion', () => {
const redeem = vi.fn()
await expect(
streamGrokCompletion({
+ maxDeadlineMs: 1_800_000,
executionTicket: 'ticket-1',
requestHash: 'f'.repeat(64),
request: REQUEST,
@@ -751,6 +765,7 @@ describe('streamGrokCompletion', () => {
it('rejects a served-model mismatch and loopback redirects', async () => {
await expect(
streamGrokCompletion({
+ maxDeadlineMs: 1_800_000,
executionTicket: 'ticket-1',
requestHash: REQUEST_HASH,
request: REQUEST,
@@ -781,6 +796,7 @@ describe('streamGrokCompletion', () => {
)
await expect(
streamGrokCompletion({
+ maxDeadlineMs: 1_800_000,
executionTicket: 'ticket-1',
requestHash: REQUEST_HASH,
request: REQUEST,
@@ -818,6 +834,7 @@ describe('streamGrokCompletion', () => {
}))
const order: string[] = []
await streamGrokCompletion({
+ maxDeadlineMs: 1_800_000,
executionTicket: 'ticket-redeemed',
requestHash: REQUEST_HASH,
request: REQUEST,
@@ -847,6 +864,7 @@ describe('streamGrokCompletion', () => {
for (const redeem of [denied, mismatched]) {
await expect(
streamGrokCompletion({
+ maxDeadlineMs: 1_800_000,
executionTicket: 'ticket-redeemed',
requestHash: REQUEST_HASH,
request: REQUEST,
@@ -876,6 +894,7 @@ describe('streamGrokCompletion', () => {
return sseResponse(['data: {"type":"response.completed","response":{"usage":{}}}\n\n'])
})
const result = await streamGrokCompletion({
+ maxDeadlineMs: 1_800_000,
executionTicket: 'ticket-1',
requestHash: REQUEST_HASH,
request: REQUEST,
@@ -902,6 +921,7 @@ describe('streamGrokCompletion', () => {
it('maps upstream 401 to connection_unavailable instead of origin_denied', async () => {
await expect(
streamGrokCompletion({
+ maxDeadlineMs: 1_800_000,
executionTicket: 'ticket-1',
requestHash: REQUEST_HASH,
request: REQUEST,
@@ -952,6 +972,7 @@ describe('streamGrokCompletion', () => {
const frames: unknown[] = []
const result = await streamGrokCompletion({
+ maxDeadlineMs: 1_800_000,
executionTicket: 'ticket-1',
requestHash: REQUEST_HASH,
request: REQUEST,
@@ -996,6 +1017,7 @@ describe('streamGrokCompletion', () => {
providerAttemptId: 'att-1',
}
await streamGrokCompletion({
+ maxDeadlineMs: 1_800_000,
executionTicket: 't-a',
requestHash: REQUEST_HASH,
request: REQUEST,
@@ -1010,6 +1032,7 @@ describe('streamGrokCompletion', () => {
lookup: async () => [{ address: '1.2.3.4', family: 4 }],
})
await streamGrokCompletion({
+ maxDeadlineMs: 1_800_000,
executionTicket: 't-b',
requestHash: REQUEST_HASH,
request: REQUEST,
@@ -1037,6 +1060,7 @@ describe('streamGrokCompletion', () => {
sseResponse(['data: {"type":"response.output_text.delta","delta":"partial"}\n\n'])
)
const result = await streamGrokCompletion({
+ maxDeadlineMs: 1_800_000,
executionTicket: 'ticket-1',
requestHash: REQUEST_HASH,
request: REQUEST,
@@ -1090,6 +1114,7 @@ describe('streamGrokCompletion', () => {
sseResponse(['data: {"type":"response.completed","response":{"usage":{}}}\n\n'])
)
await streamGrokCompletion({
+ maxDeadlineMs: 1_800_000,
executionTicket: 'ticket-1',
requestHash,
request,
@@ -1187,6 +1212,7 @@ describe('streamGrokCompletion', () => {
])
})
const result = await streamGrokCompletion({
+ maxDeadlineMs: 1_800_000,
executionTicket: 'ticket-name-map',
requestHash,
request,
@@ -1242,6 +1268,7 @@ describe('streamGrokCompletion', () => {
})
await expect(
streamGrokCompletion({
+ maxDeadlineMs: 1_800_000,
executionTicket: 'ticket-unknown-alias',
requestHash,
request,
@@ -1278,6 +1305,7 @@ describe('streamGrokCompletion', () => {
sseResponse(['data: {"type":"response.completed","response":{"usage":{}}}\n\n'])
)
await streamGrokCompletion({
+ maxDeadlineMs: 1_800_000,
executionTicket: 'ticket-1',
requestHash: REQUEST_HASH,
request: REQUEST,
@@ -1319,6 +1347,7 @@ describe('streamGrokCompletion', () => {
sseResponse(['data: {"type":"response.completed","response":{"usage":{}}}\n\n'])
)
await streamGrokCompletion({
+ maxDeadlineMs: 1_800_000,
executionTicket: 'ticket-1',
requestHash,
request,
@@ -1361,6 +1390,7 @@ describe('streamGrokCompletion', () => {
)
await expect(
streamGrokCompletion({
+ maxDeadlineMs: 1_800_000,
executionTicket: 'ticket-1',
requestHash: REQUEST_HASH,
request: REQUEST,
@@ -1400,6 +1430,7 @@ describe('streamGrokCompletion', () => {
])
)
await streamGrokCompletion({
+ maxDeadlineMs: 1_800_000,
executionTicket: 'ticket-1',
requestHash: REQUEST_HASH,
request: REQUEST,
diff --git a/grok-llm-proxy/test/requestLimits.test.ts b/grok-llm-proxy/test/requestLimits.test.ts
index ed7ec59fa..deb988815 100644
--- a/grok-llm-proxy/test/requestLimits.test.ts
+++ b/grok-llm-proxy/test/requestLimits.test.ts
@@ -1,5 +1,23 @@
import { describe, expect, it } from 'vitest'
-import { RequestLimitError, StreamGate } from '../src/requestLimits.js'
+import { CONTROL_API_REQUEST_TIMEOUT_MS } from '../src/controlApiClient.js'
+import { RequestLimitError, STREAM_LIMITS, StreamGate } from '../src/requestLimits.js'
+import { DEFAULT_HEARTBEAT_INTERVAL_MS } from '../src/sseHeartbeat.js'
+
+// undici's default headersTimeout and bodyTimeout in the Node 24.16.0 image the
+// Host runs; the Host sets neither for its proxy calls.
+const HOST_UNDICI_TIMEOUT_MS = 300_000
+
+describe('stream timing invariant', () => {
+ it('reaches the first keepalive before the Host HTTP client times out', () => {
+ expect(STREAM_LIMITS.maxStreamDurationMs).toBe(1_800_000)
+ expect(STREAM_LIMITS.maxQueueWaitMs).toBe(60_000)
+ // Nothing is written while a request waits for a slot or for the redeem,
+ // so both must end, and one heartbeat interval pass, inside the timeout.
+ expect(
+ STREAM_LIMITS.maxQueueWaitMs + CONTROL_API_REQUEST_TIMEOUT_MS + DEFAULT_HEARTBEAT_INTERVAL_MS
+ ).toBeLessThan(HOST_UNDICI_TIMEOUT_MS)
+ })
+})
describe('StreamGate', () => {
it('admits a queued waiter once a running stream releases', async () => {
@@ -53,4 +71,21 @@ describe('StreamGate', () => {
const releaseNext = await next
releaseNext()
})
+
+ it('rejects a waiter queued longer than maxQueueWaitMs and frees its queue slot', async () => {
+ const gate = new StreamGate(1, 1, 50)
+ const release = await gate.acquire()
+ const started = Date.now()
+ await expect(gate.acquire()).rejects.toMatchObject({
+ name: 'RequestLimitError',
+ message: 'stream queue wait exceeded',
+ })
+ // Witness: the waiter was queued for the whole wait, not refused at once.
+ expect(Date.now() - started).toBeGreaterThanOrEqual(45)
+ // The queue slot is free again, so a new waiter queues instead of "queue is full".
+ const next = gate.acquire()
+ release()
+ const releaseNext = await next
+ releaseNext()
+ })
})
diff --git a/grok-llm-proxy/test/server.security.test.ts b/grok-llm-proxy/test/server.security.test.ts
index 2271f44c6..835186187 100644
--- a/grok-llm-proxy/test/server.security.test.ts
+++ b/grok-llm-proxy/test/server.security.test.ts
@@ -32,8 +32,8 @@ function config(overrides: Partial<GrokLlmProxyConfig> = {}): GrokLlmProxyConfig
adminPort: 8081,
probePort: 9090,
maxBodyBytes: 1024,
- maxStreamDurationMs: 300_000,
- maxDeadlineMs: 300_000,
+ maxStreamDurationMs: 1_800_000,
+ maxDeadlineMs: 1_800_000,
upstreamIdleTimeoutMs: 600_000,
heartbeatIntervalMs: 15_000,
jwtIssuer: 'control-api',
@@ -140,7 +140,7 @@ describe('grok-llm-proxy security surface', () => {
executionTicket: ticket(),
requestHash: 'a'.repeat(64),
request: {},
- deadlineMs: 999_999,
+ deadlineMs: 1_800_001,
})
expect(deadline.body.error).toBe('invalid_request')
@@ -322,6 +322,16 @@ describe('grok-llm-proxy security surface', () => {
).toBe(45_000)
})
+ it('defaults the total stream cap and the deadline ceiling to 30 min', () => {
+ const loaded = loadConfig({
+ GROK_LLM_PROXY_JWT_PUBLIC_KEY: publicKey,
+ GROK_LLM_PROXY_CONTROL_API_URL: 'http://control-api:8080',
+ GROK_LLM_PROXY_CONTROL_API_TOKEN: 'service-token',
+ })
+ expect(loaded.maxStreamDurationMs).toBe(1_800_000)
+ expect(loaded.maxDeadlineMs).toBe(1_800_000)
+ })
+
it('defaults the SSE heartbeat to 15 s and rejects a non-positive interval', () => {
const env = {
GROK_LLM_PROXY_JWT_PUBLIC_KEY: publicKey,
@@ -552,7 +562,7 @@ describe('grok-llm-proxy attempt telemetry', () => {
return { executionTicket, requestHash, request: raw }
}
- function grantingClient(maxStreamDurationMs = 300_000): {
+ function grantingClient(maxStreamDurationMs = 1_800_000): {
client: ControlApiClient
receipts: unknown[]
} {
diff --git a/packages/grok-provider-attempt-contract/index.cjs b/packages/grok-provider-attempt-contract/index.cjs
index 885a88318..0a6bd6c6c 100644
--- a/packages/grok-provider-attempt-contract/index.cjs
+++ b/packages/grok-provider-attempt-contract/index.cjs
@@ -34,7 +34,7 @@ const LIMITS = Object.freeze({
// actually returned; neither is a provider outage.
maxToolCalls: 256,
maxOutputTokens: 16384,
- maxDeadlineMs: 300000,
+ maxDeadlineMs: 1800000,
maxIdLength: 128,
// Free-form JSON trees (tool parameters, assistant tool-call arguments) may
// nest at most this many containers. Bounds recursion before hashing.
diff --git a/packages/grok-provider-attempt-contract/index.d.ts b/packages/grok-provider-attempt-contract/index.d.ts
index f50b4cee9..9a3cc7d7f 100644
--- a/packages/grok-provider-attempt-contract/index.d.ts
+++ b/packages/grok-provider-attempt-contract/index.d.ts
@@ -19,7 +19,7 @@ export declare const LIMITS: {
readonly maxMessages: 1024
readonly maxToolCalls: 256
readonly maxOutputTokens: 16384
- readonly maxDeadlineMs: 300000
+ readonly maxDeadlineMs: 1800000
readonly maxIdLength: 128
readonly maxNestingDepth: 64
}
diff --git a/packages/llm-provider-attempt-contract/index.cjs b/packages/llm-provider-attempt-contract/index.cjs
index 99ece4f0c..81fc89ecc 100644
--- a/packages/llm-provider-attempt-contract/index.cjs
+++ b/packages/llm-provider-attempt-contract/index.cjs
@@ -20,7 +20,7 @@ const LIMITS = Object.freeze({
// bound maxOutputTokens leaves 64 output tokens per call in one response.
maxToolCalls: 256,
maxOutputTokens: 16384,
- maxDeadlineMs: 300000,
+ maxDeadlineMs: 1800000,
maxIdLength: 128,
// Free-form JSON trees (tool parameters, assistant tool-call arguments) may
// nest at most this many containers. Bounds recursion before hashing.
diff --git a/packages/llm-provider-attempt-contract/index.d.ts b/packages/llm-provider-attempt-contract/index.d.ts
index 4e9958325..dc0594a57 100644
--- a/packages/llm-provider-attempt-contract/index.d.ts
+++ b/packages/llm-provider-attempt-contract/index.d.ts
@@ -18,7 +18,7 @@ export declare const LIMITS: {
readonly maxMessages: 1024
readonly maxToolCalls: 256
readonly maxOutputTokens: 16384
- readonly maxDeadlineMs: 300000
+ readonly maxDeadlineMs: 1800000
readonly maxIdLength: 128
readonly maxNestingDepth: 64
}
diff --git a/scripts/tests/test-codex-llm-proxy-deploy-contract.sh b/scripts/tests/test-codex-llm-proxy-deploy-contract.sh
index 61c49c048..9bd651039 100755
--- a/scripts/tests/test-codex-llm-proxy-deploy-contract.sh
+++ b/scripts/tests/test-codex-llm-proxy-deploy-contract.sh
@@ -52,6 +52,12 @@ for needle in (
if needle not in text:
errors.append(f"manifest missing {needle}")
+# The per-attempt cap is the minimum of every source, so a stale value here
+# silently holds it below the 30 min contract ceiling.
+for var in ("CODEX_LLM_PROXY_MAX_STREAM_DURATION_MS", "CODEX_LLM_PROXY_MAX_DEADLINE_MS"):
+ if f' {var}: "1800000"\n' not in text:
+ errors.append(f"manifest must set {var} to 1800000")
+
if "kind: Role" in text or "kind: RoleBinding" in text:
errors.append("proxy manifest must not declare Role/RoleBinding")
if "kind: ClusterRole" in text or "kind: ClusterRoleBinding" in text:
diff --git a/scripts/tests/test-grok-llm-proxy-deploy-contract.sh b/scripts/tests/test-grok-llm-proxy-deploy-contract.sh
index fb0985eb7..a19ea1bc6 100755
--- a/scripts/tests/test-grok-llm-proxy-deploy-contract.sh
+++ b/scripts/tests/test-grok-llm-proxy-deploy-contract.sh
@@ -60,6 +60,12 @@ for needle in (
if needle not in text:
errors.append(f"manifest missing {needle}")
+# The per-attempt cap is the minimum of every source, so a stale value here
+# silently holds it below the 30 min contract ceiling.
+for var in ("GROK_LLM_PROXY_MAX_STREAM_DURATION_MS", "GROK_LLM_PROXY_MAX_DEADLINE_MS"):
+ if f' {var}: "1800000"\n' not in text:
+ errors.append(f"manifest must set {var} to 1800000")
+
if "kind: Role" in text or "kind: RoleBinding" in text:
errors.append("proxy manifest must not declare Role/RoleBinding")
if "automountServiceAccountToken: true" in text:
diff --git a/tests/e2e/fixtures/codex-subscription/sanitized-upstream-contract.json b/tests/e2e/fixtures/codex-subscription/sanitized-upstream-contract.json
index f4351b95a..c061d25ab 100644
--- a/tests/e2e/fixtures/codex-subscription/sanitized-upstream-contract.json
+++ b/tests/e2e/fixtures/codex-subscription/sanitized-upstream-contract.json
@@ -44,10 +44,11 @@
"maxMessages": 1024,
"maxToolCalls": 256,
"maxOutputTokens": 16384,
- "maxStreamDurationMs": 300000,
- "maxDeadlineMs": 300000,
+ "maxStreamDurationMs": 1800000,
+ "maxDeadlineMs": 1800000,
"maxConcurrentStreams": 8,
"maxQueuedRequests": 16,
+ "maxQueueWaitMs": 60000,
"upstreamIdleTimeoutMs": 300000,
"maxRetriesPerAttempt": 1
},
diff --git a/tests/e2e/fixtures/grok-subscription/sanitized-upstream-contract.json b/tests/e2e/fixtures/grok-subscription/sanitized-upstream-contract.json
index f62446ac3..0a8a9a563 100644
--- a/tests/e2e/fixtures/grok-subscription/sanitized-upstream-contract.json
+++ b/tests/e2e/fixtures/grok-subscription/sanitized-upstream-contract.json
@@ -58,10 +58,11 @@
"maxMessages": 1024,
"maxToolCalls": 256,
"maxOutputTokens": 16384,
- "maxStreamDurationMs": 300000,
- "maxDeadlineMs": 300000,
+ "maxStreamDurationMs": 1800000,
+ "maxDeadlineMs": 1800000,
"maxConcurrentStreams": 8,
"maxQueuedRequests": 16,
+ "maxQueueWaitMs": 60000,
"upstreamIdleTimeoutMs": 600000,
"maxRetriesPerAttempt": 1
},
diff --git a/tests/e2e/fixtures/package-lock.json b/tests/e2e/fixtures/package-lock.json
index 3a9bac504..87f7ab185 100644
--- a/tests/e2e/fixtures/package-lock.json
+++ b/tests/e2e/fixtures/package-lock.json
@@ -1,12 +1,12 @@
{
"name": "@clerum/e2e-test-fixtures",
- "version": "1.0.90",
+ "version": "1.0.91",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "@clerum/e2e-test-fixtures",
- "version": "1.0.90",
+ "version": "1.0.91",
"dependencies": {
"@kubernetes/client-node": "^1.0.0"
},
diff --git a/tests/e2e/fixtures/package.json b/tests/e2e/fixtures/package.json
index 91a0be7f9..db0b424af 100644
--- a/tests/e2e/fixtures/package.json
+++ b/tests/e2e/fixtures/package.json
@@ -1,6 +1,6 @@
{
"name": "@clerum/e2e-test-fixtures",
- "version": "1.0.90",
+ "version": "1.0.91",
"description": "E2E test fixtures and utilities for Clerum testing",
"type": "module",
"main": "index.ts",
diff --git a/tests/e2e/integration/codex-subscription-contract-freeze.test.ts b/tests/e2e/integration/codex-subscription-contract-freeze.test.ts
index c7f6d4963..3d67eb793 100644
--- a/tests/e2e/integration/codex-subscription-contract-freeze.test.ts
+++ b/tests/e2e/integration/codex-subscription-contract-freeze.test.ts
@@ -45,6 +45,7 @@ const REQUIRED_LIMIT_KEYS = [
'maxDeadlineMs',
'maxConcurrentStreams',
'maxQueuedRequests',
+ 'maxQueueWaitMs',
'upstreamIdleTimeoutMs',
'maxRetriesPerAttempt',
] as const
diff --git a/tests/e2e/package-lock.json b/tests/e2e/package-lock.json
index 8c3e659d7..54012259b 100644
--- a/tests/e2e/package-lock.json
+++ b/tests/e2e/package-lock.json
@@ -1,12 +1,12 @@
{
"name": "clerum-e2e-tests",
- "version": "0.1.90",
+ "version": "0.1.91",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "clerum-e2e-tests",
- "version": "0.1.90",
+ "version": "0.1.91",
"devDependencies": {
"@clerum/llm-provider-attempt-contract": "file:../../packages/llm-provider-attempt-contract",
"@types/jsonwebtoken": "^9.0.10",
diff --git a/tests/e2e/package.json b/tests/e2e/package.json
index c0e1b212b..e1c88b7a1 100644
--- a/tests/e2e/package.json
+++ b/tests/e2e/package.json
@@ -1,6 +1,6 @@
{
"name": "clerum-e2e-tests",
- "version": "0.1.90",
+ "version": "0.1.91",
"private": true,
"type": "module",
"scripts": {