Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript

fix(llm-proxy): bound every admission wait by one clock and the ticket's life (#739)

Alfredo Lopezcad100ce6438
diff --git a/codex-llm-proxy/src/auth/executionTicketVerifier.ts b/codex-llm-proxy/src/auth/executionTicketVerifier.ts
index 0f1dada06..ae1ec5d8e 100644
--- a/codex-llm-proxy/src/auth/executionTicketVerifier.ts
+++ b/codex-llm-proxy/src/auth/executionTicketVerifier.ts
@@ -8,6 +8,8 @@ export type ExecutionTicketClaims = {
model: string
requestHash: string
providerAttemptId: string
+ /** The signed `exp` in epoch milliseconds (#739 D1-bis). */
+ expiresAtMs: number
}
export function verifyExecutionTicket(
@@ -34,6 +36,7 @@ export function verifyExecutionTicket(
model: claims.model,
requestHash: claims.requestHash,
providerAttemptId: claims.providerAttemptId,
+ expiresAtMs: claims.exp * 1000,
}
} catch {
return null
diff --git a/codex-llm-proxy/src/requestLimits.ts b/codex-llm-proxy/src/requestLimits.ts
index 72b9f752a..60beb47ef 100644
--- a/codex-llm-proxy/src/requestLimits.ts
+++ b/codex-llm-proxy/src/requestLimits.ts
@@ -37,6 +37,8 @@ export const IN_FLIGHT_BODY_BUDGET_BODIES = 3
* loopback, an 8 MiB body is read in 7.6-22.6 ms alone and in under 60 ms with
* three at once, so 10 s only cuts a body that has stalled. Without it a
* stalled body keeps its reservation until Node's 300 s `requestTimeout`.
+ * The wait before the grant is bounded separately, by the request's admission
+ * clock (`STREAM_LIMITS.maxQueueWaitMs` from arrival, #739 D1).
* Not env-tunable.
*/
export const BODY_READ_DEADLINE_MS = 10_000
@@ -45,9 +47,11 @@ export const STREAM_LIMITS = {
maxConcurrentStreams: 8,
maxQueuedRequests: 16,
maxStreamDurationMs: 1_800_000,
- // Longest time a request may wait for a stream slot. Bounded so that queue
- // wait + redeem + the first keepalive stays below the Host HTTP client's
- // 300 s header timeout.
+ // Longest total time a request may spend queued in this proxy: one
+ // admission clock from arrival bounds the body budget, the visual gate and
+ // the stream gate together (#739 D1). Bounded so that queue wait + redeem +
+ // the first keepalive stays below the Host HTTP client's 300 s header
+ // timeout.
maxQueueWaitMs: 60_000,
// Longest silence tolerated while waiting on the upstream (response headers
// or the next SSE chunk). Matches the Codex CLI stream idle timeout.
@@ -75,6 +79,20 @@ export class RequestLimitError extends Error {
}
}
+/**
+ * #739 D1-bis — the execution ticket expired while its request still waited
+ * for a stream slot. Redeeming it could only end in `ticket_expired`, so the
+ * request is refused as a capacity limit instead, before any redeem. It is a
+ * RequestLimitError on the wire (503 `provider_unavailable`); the attempt
+ * failure metric counts it as `provider_unavailable`, not `request_limit`.
+ */
+export class TicketLifeError extends RequestLimitError {
+ constructor() {
+ super('execution ticket expired while queued')
+ this.name = 'TicketLifeError'
+ }
+}
+
export function assertBoundedDeadline(deadlineMs: number | undefined, maxDeadlineMs: number): number {
// Without a valid maximum, Math.min below would return NaN for a valid
// request deadline instead of refusing it.
@@ -110,9 +128,14 @@ export class StreamGate {
* Take a stream slot. When `signal` aborts (client disconnected) while the
* caller is still queued, the waiter is rejected and its queue slot freed, so
* a dropped client never proceeds to redeem an attempt. A waiter still
- * queued after `maxQueueWaitMs` is rejected the same way.
+ * queued after `maxQueueWaitMs`, or at `deadlineAt` (epoch ms) when that
+ * comes first, is rejected the same way (#739 D1). A free slot is granted at
+ * once whatever the deadline; the caller checks a deadline already past.
*/
- async acquire(signal?: AbortSignal): Promise<() => void> {
+ async acquire(signal?: AbortSignal, deadlineAt?: number): Promise<() => void> {
+ if (deadlineAt !== undefined && !Number.isFinite(deadlineAt)) {
+ throw new RangeError(`a stream gate deadline must be a finite epoch time, got ${deadlineAt}`)
+ }
if (signal?.aborted) throw new RequestLimitError('stream request was aborted')
if (this.running >= this.maxConcurrent) {
if (this.queued >= this.maxQueued) throw new RequestLimitError('stream queue is full')
@@ -131,7 +154,11 @@ export class StreamGate {
resolve()
return
}
- if (Date.now() - queuedAt >= this.maxQueueWaitMs) {
+ const now = Date.now()
+ if (
+ now - queuedAt >= this.maxQueueWaitMs ||
+ (deadlineAt !== undefined && now >= deadlineAt)
+ ) {
signal?.removeEventListener('abort', onAbort)
reject(new RequestLimitError('stream queue wait exceeded'))
return
@@ -192,12 +219,18 @@ export class BodyBudget {
/**
* Reserve `bytes` of the budget. The returned release is idempotent. When
* `signal` aborts while the caller is queued, the waiter is rejected and
- * removed, and the waiters behind it are reconsidered.
+ * removed, and the waiters behind it are reconsidered. A waiter still queued
+ * at `deadlineAt` (epoch ms, the request's admission deadline, #739 D1) is
+ * rejected and removed the same way, so a large body at the head of the
+ * queue cannot hold back the smaller ones behind it past that instant.
*/
- async acquire(bytes: number, signal?: AbortSignal): Promise<() => void> {
+ async acquire(bytes: number, signal?: AbortSignal, deadlineAt?: number): Promise<() => void> {
if (!Number.isInteger(bytes) || bytes < 0 || bytes > this.capacityBytes) {
throw new RangeError(`a body of ${bytes} bytes cannot fit a budget of ${this.capacityBytes}`)
}
+ if (deadlineAt !== undefined && !Number.isFinite(deadlineAt)) {
+ throw new RangeError(`a body admission deadline must be a finite epoch time, got ${deadlineAt}`)
+ }
if (signal?.aborted) throw new RequestLimitError('body admission was aborted')
if (this.waiters.length === 0 && this.inFlight + bytes <= this.capacityBytes) {
return this.take(bytes)
@@ -206,22 +239,33 @@ export class BodyBudget {
throw new RequestLimitError('body admission queue is full')
}
return new Promise<() => void>((resolve, reject) => {
- const onAbort = () => {
+ let deadline: ReturnType<typeof setTimeout> | undefined
+ const leave = (reason: string) => {
const index = this.waiters.indexOf(waiter)
if (index === -1) return
this.waiters.splice(index, 1)
- reject(new RequestLimitError('body admission was aborted'))
+ clearTimeout(deadline)
+ signal?.removeEventListener('abort', onAbort)
+ reject(new RequestLimitError(reason))
this.drain()
}
+ const onAbort = () => leave('body admission was aborted')
const waiter: BodyWaiter = {
bytes,
grant: release => {
+ clearTimeout(deadline)
signal?.removeEventListener('abort', onAbort)
resolve(release)
},
}
signal?.addEventListener('abort', onAbort, { once: true })
this.waiters.push(waiter)
+ if (deadlineAt !== undefined) {
+ deadline = setTimeout(
+ () => leave('body admission wait exceeded'),
+ Math.max(0, deadlineAt - Date.now())
+ )
+ }
})
}
diff --git a/codex-llm-proxy/src/server.ts b/codex-llm-proxy/src/server.ts
index 2d5b17a20..d810be81a 100644
--- a/codex-llm-proxy/src/server.ts
+++ b/codex-llm-proxy/src/server.ts
@@ -40,12 +40,23 @@ import {
ENVELOPE_ALLOWANCE_BYTES,
IN_FLIGHT_BODY_BUDGET_BODIES,
RequestLimitError,
+ STREAM_LIMITS,
+ TicketLifeError,
streamGate,
visualStreamGate,
} from './requestLimits.js'
import { startSseHeartbeat } from './sseHeartbeat.js'
-type GatedRequest = Request & {
+type AdmittedRequest = Request & {
+ /**
+ * #739 D1 — the request's one admission clock, stamped by body admission:
+ * arrival + `STREAM_LIMITS.maxQueueWaitMs`, in epoch ms. Every wait the
+ * request performs (body budget, visual gate, stream gate) ends by then.
+ */
+ codexAdmissionDeadlineAt?: number
+}
+
+type GatedRequest = AdmittedRequest & {
codexStreamRelease?: () => void
/** Set by the platform gate before body admission (R9-M-B). */
codexPlatform?: PlatformJwtClaims
@@ -124,7 +135,9 @@ function boundedErrorHandler(err: unknown, _req: Request, res: Response, _next:
* its connection closed. Otherwise the reservation is held until the response
* closes, which for a completion is the whole stream (up to
* `maxStreamDurationMs`). A queued waiter is dropped if the client leaves
- * first; a full queue gets the stream gate's overload response.
+ * first; a full queue gets the stream gate's overload response. #739 D1: the
+ * request's admission clock is stamped here, at arrival, and a waiter still
+ * queued when it runs out gets the same overload response.
*/
function bodyAdmission(
budget: BodyBudget,
@@ -132,7 +145,9 @@ function bodyAdmission(
readDeadlineMs: number,
parse: RequestHandler
) {
- return (req: Request, res: Response, next: NextFunction): void => {
+ return (req: AdmittedRequest, res: Response, next: NextFunction): void => {
+ const admissionDeadlineAt = Date.now() + STREAM_LIMITS.maxQueueWaitMs
+ req.codexAdmissionDeadlineAt = admissionDeadlineAt
if (req.headers['transfer-encoding'] !== undefined) {
reject(res, 411, 'length_required')
return
@@ -159,7 +174,7 @@ function bodyAdmission(
if (release) release()
else abort.abort()
})
- budget.acquire(bytes, abort.signal).then(
+ budget.acquire(bytes, abort.signal, admissionDeadlineAt).then(
granted => {
if (abort.signal.aborted) {
granted()
@@ -301,7 +316,7 @@ export function createProxyApps(
const abortParse = (): void => parseAbort.abort()
req.once('aborted', abortParse)
try {
- release = await visualStreamGate.acquire(parseAbort.signal)
+ release = await visualStreamGate.acquire(parseAbort.signal, req.codexAdmissionDeadlineAt)
} catch (err) {
req.off('aborted', abortParse)
if (err instanceof RequestLimitError) {
@@ -342,6 +357,11 @@ export function createProxyApps(
releaseAdmission()
throw new Error('the completion route was reached without the platform gate')
}
+ const admissionDeadlineAt = gated.codexAdmissionDeadlineAt
+ if (admissionDeadlineAt === undefined) {
+ releaseAdmission()
+ throw new Error('the completion route was reached without body admission')
+ }
if (!req.is('application/json')) {
releaseAdmission()
reject(res, 415, 'unsupported_media_type')
@@ -439,7 +459,39 @@ export function createProxyApps(
releaseAdmission()
}
if (!release) {
- release = await streamGate.acquire(abort.signal)
+ // #739 D1-bis: the wait also ends when the ticket dies. A request
+ // still queued at `exp` could only be redeemed into ticket_expired,
+ // so it is refused here, before any redeem. The margin is zero: a
+ // ticket still alive when a slot frees is served as before.
+ const deadlineAt = Math.min(admissionDeadlineAt, ticket.expiresAtMs)
+ const refuseTicketLife = (): never => {
+ logger.warn(
+ {
+ event: 'codex_proxy_admission_refused',
+ reason: 'ticket_life',
+ providerAttemptId: ticket.providerAttemptId,
+ hostRef: ticket.hostRef,
+ },
+ 'admission refused'
+ )
+ throw new TicketLifeError()
+ }
+ if (deadlineAt <= Date.now()) {
+ if (ticket.expiresAtMs <= admissionDeadlineAt) refuseTicketLife()
+ throw new RequestLimitError('admission deadline exceeded')
+ }
+ try {
+ release = await streamGate.acquire(abort.signal, deadlineAt)
+ } catch (err) {
+ if (
+ err instanceof RequestLimitError &&
+ !abort.signal.aborted &&
+ Date.now() >= ticket.expiresAtMs
+ ) {
+ refuseTicketLife()
+ }
+ throw err
+ }
}
res.status(200)
res.setHeader('content-type', 'text/event-stream')
@@ -502,9 +554,12 @@ export function createProxyApps(
const mapped = mapError(err)
metrics.observeAttempt('error', 'completion_stream')
// A request limit answers provider_unavailable on the wire; its own
- // label keeps it apart from real upstream outages in the metric.
+ // label keeps it apart from real upstream outages in the metric. A
+ // ticket that died in the queue is counted as provider_unavailable.
metrics.observeAttemptFailure(
- err instanceof RequestLimitError ? 'request_limit' : failureLabel(mapped.code)
+ err instanceof RequestLimitError && !(err instanceof TicketLifeError)
+ ? 'request_limit'
+ : failureLabel(mapped.code)
)
if (err instanceof UpstreamTimeoutError) metrics.observeUpstreamTimeout(err.kind)
const deliveredAs = res.headersSent ? 'sse_error' : 'http_status'
diff --git a/codex-llm-proxy/src/sseHeartbeat.ts b/codex-llm-proxy/src/sseHeartbeat.ts
index 52b315b7c..39ee9c9be 100644
--- a/codex-llm-proxy/src/sseHeartbeat.ts
+++ b/codex-llm-proxy/src/sseHeartbeat.ts
@@ -9,16 +9,18 @@ const KEEPALIVE = ': keepalive\n\n'
/**
* Well under the Host's 300 s undici headers/body timeouts, so a silent
- * upstream never reaches them: queue wait (60 s) + redeem (15 s) + one
- * interval stays below 300 s.
+ * upstream never reaches them: the admission clock (60 s) + redeem (15 s) +
+ * one interval stays below 300 s.
*/
export const DEFAULT_HEARTBEAT_INTERVAL_MS = 15_000
/**
- * Largest interval config accepts. Two queue waits (visual gate, then stream
- * gate: 120 s), the 15 s redeem and one interval must still end before the
- * Host's 300 s headers timeout, and one interval must stay below its body
- * timeout.
+ * Largest interval config accepts. Every wait a request performs before the
+ * redeem (body budget, visual gate, stream gate) is bounded by one admission
+ * clock of `maxQueueWaitMs` (60 s) from arrival (#739 D1), so the queue time
+ * counts once: 60 s + the 15 s redeem + one 60 s interval = 135 s, which ends
+ * before the Host's 300 s headers timeout, and one interval stays below its
+ * body timeout.
*/
export const MAX_HEARTBEAT_INTERVAL_MS = 60_000
diff --git a/docs/architecture/codex-subscription-transport-contract.md b/docs/architecture/codex-subscription-transport-contract.md
index e1747447a..7ab3ef91a 100644
--- a/docs/architecture/codex-subscription-transport-contract.md
+++ b/docs/architecture/codex-subscription-transport-contract.md
@@ -340,10 +340,20 @@ behavior changes:
- It drops queued stream-gate waiters on abort and checks the abort signal
before redeeming a ticket. It also rejects an invalid or out-of-bounds
deadline before the redeem, so the single-use ticket is not consumed.
- - It rejects a stream-gate waiter still queued after `maxQueueWaitMs` with
- `provider_unavailable` (reason `stream queue wait exceeded`). Queue wait,
- the 15 s control-api redeem timeout and the first keepalive together stay
- below the Host HTTP client's 300 s header timeout.
+ - It gives each request one admission clock, stamped at arrival: arrival +
+ `maxQueueWaitMs`. The body budget, the visual gate and the stream gate
+ all wait against that same instant, so `maxQueueWaitMs` is the total
+ time a request may spend queued in the proxy. A waiter still queued when
+ it runs out is rejected with `provider_unavailable` (reason
+ `body admission wait exceeded` or `stream queue wait exceeded`). Queue
+ wait, the 15 s control-api redeem timeout and the first keepalive
+ together (60 + 15 + 60 = 135 s) stay below the Host HTTP client's 300 s
+ header timeout.
+ - The stream-gate wait also ends at the execution ticket's `exp`, with no
+ margin. A request still queued then is answered 503
+ `provider_unavailable` without a redeem, and the proxy logs
+ `codex_proxy_admission_refused` with `reason: ticket_life`,
+ `providerAttemptId` and `hostRef`.
- It requires the redeem response to carry `maxStreamDurationMs` greater
than 0. An absent value is a contract violation, not a default.
- It logs one `codex_proxy_attempt_finished` event per completion attempt,
diff --git a/docs/architecture/grok-subscription-transport-contract.md b/docs/architecture/grok-subscription-transport-contract.md
index 8c5c80394..22b26c969 100644
--- a/docs/architecture/grok-subscription-transport-contract.md
+++ b/docs/architecture/grok-subscription-transport-contract.md
@@ -204,10 +204,18 @@ Proxy robustness (both proxies):
checks the abort signal before redeeming a ticket. It also rejects an
invalid or out-of-bounds deadline before the redeem, so the single-use
ticket is not consumed.
-- A stream-gate waiter still queued after `maxQueueWaitMs` is rejected
- with `provider_unavailable` (reason `stream queue wait exceeded`). Queue
- wait, the 15 s control-api redeem timeout and the first keepalive together
- stay below the Host HTTP client's 300 s header timeout.
+- Each request gets one admission clock, stamped at arrival: arrival +
+ `maxQueueWaitMs`. The body budget and the stream gate both wait against
+ that same instant, so `maxQueueWaitMs` is the total time a request may
+ spend queued in the proxy. A waiter still queued when it runs out is
+ rejected with `provider_unavailable` (reason `body admission wait exceeded`
+ or `stream queue wait exceeded`). Queue wait, the 15 s control-api redeem
+ timeout and the first keepalive together (60 + 15 + 60 = 135 s) stay below
+ the Host HTTP client's 300 s header timeout.
+- The stream-gate wait also ends at the execution ticket's `exp`, with no
+ margin. A request still queued then is answered 503 `provider_unavailable`
+ without a redeem, and the proxy logs `grok_proxy_admission_refused` with
+ `reason: ticket_life`, `providerAttemptId` and `hostRef`.
- A single attempt streams for at most `maxStreamDurationMs`: the minimum of the proxy configuration, `STREAM_LIMITS`, the contract
`maxDeadlineMs` and the value control-api returns on redeem.
- The proxy fails at startup when `GROK_LLM_PROXY_CONTROL_API_URL` or
diff --git a/grok-llm-proxy/src/auth/executionTicketVerifier.ts b/grok-llm-proxy/src/auth/executionTicketVerifier.ts
index 404a9121c..0f30a4611 100644
--- a/grok-llm-proxy/src/auth/executionTicketVerifier.ts
+++ b/grok-llm-proxy/src/auth/executionTicketVerifier.ts
@@ -8,6 +8,8 @@ export type ExecutionTicketClaims = {
model: string
requestHash: string
providerAttemptId: string
+ /** The signed `exp` in epoch milliseconds (#739 D1-bis). */
+ expiresAtMs: number
}
export function verifyExecutionTicket(
@@ -34,6 +36,7 @@ export function verifyExecutionTicket(
model: claims.model,
requestHash: claims.requestHash,
providerAttemptId: claims.providerAttemptId,
+ expiresAtMs: claims.exp * 1000,
}
} catch {
return null
diff --git a/grok-llm-proxy/src/requestLimits.ts b/grok-llm-proxy/src/requestLimits.ts
index ac431e1b0..6c79b1391 100644
--- a/grok-llm-proxy/src/requestLimits.ts
+++ b/grok-llm-proxy/src/requestLimits.ts
@@ -35,7 +35,9 @@ export const IN_FLIGHT_BODY_BUDGET_BODIES = 3
* the Codex twin over loopback, an 8 MiB body is read in 7.6-22.6 ms alone and
* in under 60 ms with three at once, so 10 s only cuts a body that has
* stalled. Without it a stalled body keeps its reservation until Node's 300 s
- * `requestTimeout`. Not env-tunable.
+ * `requestTimeout`. The wait before the grant is bounded separately, by the
+ * request's admission clock (`STREAM_LIMITS.maxQueueWaitMs` from arrival,
+ * #739 D1). Not env-tunable.
*/
export const BODY_READ_DEADLINE_MS = 10_000
@@ -43,9 +45,10 @@ export const STREAM_LIMITS = {
maxConcurrentStreams: 8,
maxQueuedRequests: 16,
maxStreamDurationMs: 1_800_000,
- // Longest time a request may wait for a stream slot. Bounded so that queue
- // wait + redeem + the first keepalive stays below the Host HTTP client's
- // 300 s header timeout.
+ // Longest total time a request may spend queued in this proxy: one
+ // admission clock from arrival bounds the body budget and the stream gate
+ // together (#739 D1). Bounded so that queue wait + redeem + the first
+ // keepalive stays below the Host HTTP client's 300 s header timeout.
maxQueueWaitMs: 60_000,
// Longest silence tolerated while waiting on the upstream (response headers
// or the next SSE chunk). Matches the Grok Build inference idle timeout.
@@ -60,6 +63,20 @@ export class RequestLimitError extends Error {
}
}
+/**
+ * #739 D1-bis — the execution ticket expired while its request still waited
+ * for a stream slot. Redeeming it could only end in `ticket_expired`, so the
+ * request is refused as a capacity limit instead, before any redeem. It is a
+ * RequestLimitError on the wire (503 `provider_unavailable`); the attempt
+ * failure metric counts it as `provider_unavailable`, not `request_limit`.
+ */
+export class TicketLifeError extends RequestLimitError {
+ constructor() {
+ super('execution ticket expired while queued')
+ this.name = 'TicketLifeError'
+ }
+}
+
export function assertBoundedDeadline(
deadlineMs: number | undefined,
maxDeadlineMs: number
@@ -103,9 +120,14 @@ export class StreamGate {
* Take a stream slot. When `signal` aborts (client disconnected) while the
* caller is still queued, the waiter is rejected and its queue slot freed, so
* a dropped client never proceeds to redeem an attempt. A waiter still
- * queued after `maxQueueWaitMs` is rejected the same way.
+ * queued after `maxQueueWaitMs`, or at `deadlineAt` (epoch ms) when that
+ * comes first, is rejected the same way (#739 D1). A free slot is granted at
+ * once whatever the deadline; the caller checks a deadline already past.
*/
- async acquire(signal?: AbortSignal): Promise<() => void> {
+ async acquire(signal?: AbortSignal, deadlineAt?: number): Promise<() => void> {
+ if (deadlineAt !== undefined && !Number.isFinite(deadlineAt)) {
+ throw new RangeError(`a stream gate deadline must be a finite epoch time, got ${deadlineAt}`)
+ }
if (signal?.aborted) throw new RequestLimitError('stream request was aborted')
if (this.running >= this.maxConcurrent) {
if (this.queued >= this.maxQueued) throw new RequestLimitError('stream queue is full')
@@ -124,7 +146,11 @@ export class StreamGate {
resolve()
return
}
- if (Date.now() - queuedAt >= this.maxQueueWaitMs) {
+ const now = Date.now()
+ if (
+ now - queuedAt >= this.maxQueueWaitMs ||
+ (deadlineAt !== undefined && now >= deadlineAt)
+ ) {
signal?.removeEventListener('abort', onAbort)
reject(new RequestLimitError('stream queue wait exceeded'))
return
@@ -175,12 +201,18 @@ export class BodyBudget {
/**
* Reserve `bytes` of the budget. The returned release is idempotent. When
* `signal` aborts while the caller is queued, the waiter is rejected and
- * removed, and the waiters behind it are reconsidered.
+ * removed, and the waiters behind it are reconsidered. A waiter still queued
+ * at `deadlineAt` (epoch ms, the request's admission deadline, #739 D1) is
+ * rejected and removed the same way, so a large body at the head of the
+ * queue cannot hold back the smaller ones behind it past that instant.
*/
- async acquire(bytes: number, signal?: AbortSignal): Promise<() => void> {
+ async acquire(bytes: number, signal?: AbortSignal, deadlineAt?: number): Promise<() => void> {
if (!Number.isInteger(bytes) || bytes < 0 || bytes > this.capacityBytes) {
throw new RangeError(`a body of ${bytes} bytes cannot fit a budget of ${this.capacityBytes}`)
}
+ if (deadlineAt !== undefined && !Number.isFinite(deadlineAt)) {
+ throw new RangeError(`a body admission deadline must be a finite epoch time, got ${deadlineAt}`)
+ }
if (signal?.aborted) throw new RequestLimitError('body admission was aborted')
if (this.waiters.length === 0 && this.inFlight + bytes <= this.capacityBytes) {
return this.take(bytes)
@@ -189,22 +221,33 @@ export class BodyBudget {
throw new RequestLimitError('body admission queue is full')
}
return new Promise<() => void>((resolve, reject) => {
- const onAbort = () => {
+ let deadline: ReturnType<typeof setTimeout> | undefined
+ const leave = (reason: string) => {
const index = this.waiters.indexOf(waiter)
if (index === -1) return
this.waiters.splice(index, 1)
- reject(new RequestLimitError('body admission was aborted'))
+ clearTimeout(deadline)
+ signal?.removeEventListener('abort', onAbort)
+ reject(new RequestLimitError(reason))
this.drain()
}
+ const onAbort = () => leave('body admission was aborted')
const waiter: BodyWaiter = {
bytes,
grant: release => {
+ clearTimeout(deadline)
signal?.removeEventListener('abort', onAbort)
resolve(release)
},
}
signal?.addEventListener('abort', onAbort, { once: true })
this.waiters.push(waiter)
+ if (deadlineAt !== undefined) {
+ deadline = setTimeout(
+ () => leave('body admission wait exceeded'),
+ Math.max(0, deadlineAt - Date.now())
+ )
+ }
})
}
diff --git a/grok-llm-proxy/src/server.ts b/grok-llm-proxy/src/server.ts
index baa451342..7d28ca8fe 100644
--- a/grok-llm-proxy/src/server.ts
+++ b/grok-llm-proxy/src/server.ts
@@ -33,11 +33,22 @@ import {
BodyBudget,
IN_FLIGHT_BODY_BUDGET_BODIES,
RequestLimitError,
+ STREAM_LIMITS,
+ TicketLifeError,
streamGate,
} from './requestLimits.js'
import { startSseHeartbeat } from './sseHeartbeat.js'
-type GatedRequest = Request & {
+type AdmittedRequest = Request & {
+ /**
+ * #739 D1 — the request's one admission clock, stamped by body admission:
+ * arrival + `STREAM_LIMITS.maxQueueWaitMs`, in epoch ms. Every wait the
+ * request performs (body budget, stream gate) ends by then.
+ */
+ grokAdmissionDeadlineAt?: number
+}
+
+type GatedRequest = AdmittedRequest & {
/** Set by the platform gate before body admission (R9-M-B). */
grokPlatform?: PlatformJwtClaims
}
@@ -102,7 +113,9 @@ function boundedErrorHandler(err: unknown, _req: Request, res: Response, _next:
* reservation released and its connection closed. Otherwise the reservation is
* held until the response closes, which for a completion is the whole stream
* (up to `maxStreamDurationMs`). A queued waiter is dropped if the client
- * leaves first; a full queue gets the stream gate's overload response.
+ * leaves first; a full queue gets the stream gate's overload response. #739
+ * D1: the request's admission clock is stamped here, at arrival, and a waiter
+ * still queued when it runs out gets the same overload response.
*/
function bodyAdmission(
budget: BodyBudget,
@@ -110,7 +123,9 @@ function bodyAdmission(
readDeadlineMs: number,
parse: RequestHandler
) {
- return (req: Request, res: Response, next: NextFunction): void => {
+ return (req: AdmittedRequest, res: Response, next: NextFunction): void => {
+ const admissionDeadlineAt = Date.now() + STREAM_LIMITS.maxQueueWaitMs
+ req.grokAdmissionDeadlineAt = admissionDeadlineAt
if (req.headers['transfer-encoding'] !== undefined) {
reject(res, 411, 'length_required')
return
@@ -137,7 +152,7 @@ function bodyAdmission(
if (release) release()
else abort.abort()
})
- budget.acquire(bytes, abort.signal).then(
+ budget.acquire(bytes, abort.signal, admissionDeadlineAt).then(
granted => {
if (abort.signal.aborted) {
granted()
@@ -261,6 +276,10 @@ export function createProxyApps(
if (!platform) {
throw new Error('the completion route was reached without the platform gate')
}
+ const admissionDeadlineAt = (req as GatedRequest).grokAdmissionDeadlineAt
+ if (admissionDeadlineAt === undefined) {
+ throw new Error('the completion route was reached without body admission')
+ }
if (!req.is('application/json')) {
reject(res, 415, 'unsupported_media_type')
return
@@ -316,7 +335,39 @@ export function createProxyApps(
// it ends the response: a tick after `res.end()` would write after end.
let stopHeartbeat: (() => void) | undefined
try {
- release = await streamGate.acquire(abort.signal)
+ // #739 D1-bis: the wait also ends when the ticket dies. A request
+ // still queued at `exp` could only be redeemed into ticket_expired,
+ // so it is refused here, before any redeem. The margin is zero: a
+ // ticket still alive when a slot frees is served as before.
+ const deadlineAt = Math.min(admissionDeadlineAt, ticket.expiresAtMs)
+ const refuseTicketLife = (): never => {
+ logger.warn(
+ {
+ event: 'grok_proxy_admission_refused',
+ reason: 'ticket_life',
+ providerAttemptId: ticket.providerAttemptId,
+ hostRef: ticket.hostRef,
+ },
+ 'admission refused'
+ )
+ throw new TicketLifeError()
+ }
+ if (deadlineAt <= Date.now()) {
+ if (ticket.expiresAtMs <= admissionDeadlineAt) refuseTicketLife()
+ throw new RequestLimitError('admission deadline exceeded')
+ }
+ try {
+ release = await streamGate.acquire(abort.signal, deadlineAt)
+ } catch (err) {
+ if (
+ err instanceof RequestLimitError &&
+ !abort.signal.aborted &&
+ Date.now() >= ticket.expiresAtMs
+ ) {
+ refuseTicketLife()
+ }
+ throw err
+ }
res.status(200)
res.setHeader('content-type', 'text/event-stream')
res.setHeader('cache-control', 'no-cache')
@@ -377,9 +428,12 @@ export function createProxyApps(
const mapped = mapError(err)
metrics.observeAttempt('error', 'completion_stream')
// A request limit answers provider_unavailable on the wire; its own
- // label keeps it apart from real upstream outages in the metric.
+ // label keeps it apart from real upstream outages in the metric. A
+ // ticket that died in the queue is counted as provider_unavailable.
metrics.observeAttemptFailure(
- err instanceof RequestLimitError ? 'request_limit' : failureLabel(mapped.code)
+ err instanceof RequestLimitError && !(err instanceof TicketLifeError)
+ ? 'request_limit'
+ : failureLabel(mapped.code)
)
if (err instanceof UpstreamTimeoutError) metrics.observeUpstreamTimeout(err.kind)
const deliveredAs = res.headersSent ? 'sse_error' : 'http_status'
diff --git a/grok-llm-proxy/src/sseHeartbeat.ts b/grok-llm-proxy/src/sseHeartbeat.ts
index 1ea9ba011..8fb943871 100644
--- a/grok-llm-proxy/src/sseHeartbeat.ts
+++ b/grok-llm-proxy/src/sseHeartbeat.ts
@@ -9,15 +9,18 @@ const KEEPALIVE = ': keepalive\n\n'
/**
* Well under the Host's 300 s undici headers/body timeouts, so a silent
- * upstream never reaches them: queue wait (60 s) + redeem (15 s) + one
- * interval stays below 300 s.
+ * upstream never reaches them: the admission clock (60 s) + redeem (15 s) +
+ * one interval stays below 300 s.
*/
export const DEFAULT_HEARTBEAT_INTERVAL_MS = 15_000
/**
- * Largest interval config accepts. The queue wait (60 s), the 15 s redeem and
- * one interval must still end before the Host's 300 s headers timeout, and
- * one interval must stay below its body timeout.
+ * Largest interval config accepts. Every wait a request performs before the
+ * redeem (body budget, stream gate) is bounded by one admission clock of
+ * `maxQueueWaitMs` (60 s) from arrival (#739 D1), so the queue time counts
+ * once: 60 s + the 15 s redeem + one 60 s interval = 135 s, which ends before
+ * the Host's 300 s headers timeout, and one interval stays below its body
+ * timeout.
*/
export const MAX_HEARTBEAT_INTERVAL_MS = 60_000