Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
fix(llm-proxy): bound every admission wait by one clock and the ticket's life (#739)
Alfredo Lopezcad100ce6438
diff --git a/codex-llm-proxy/src/auth/executionTicketVerifier.ts b/codex-llm-proxy/src/auth/executionTicketVerifier.tsindex 0f1dada06..ae1ec5d8e 100644--- a/codex-llm-proxy/src/auth/executionTicketVerifier.ts+++ b/codex-llm-proxy/src/auth/executionTicketVerifier.ts@@ -8,6 +8,8 @@ export type ExecutionTicketClaims = {model: stringrequestHash: stringproviderAttemptId: string+ /** The signed `exp` in epoch milliseconds (#739 D1-bis). */+ expiresAtMs: number}export function verifyExecutionTicket(@@ -34,6 +36,7 @@ export function verifyExecutionTicket(model: claims.model,requestHash: claims.requestHash,providerAttemptId: claims.providerAttemptId,+ expiresAtMs: claims.exp * 1000,}} catch {return nulldiff --git a/codex-llm-proxy/src/requestLimits.ts b/codex-llm-proxy/src/requestLimits.tsindex 72b9f752a..60beb47ef 100644--- a/codex-llm-proxy/src/requestLimits.ts+++ b/codex-llm-proxy/src/requestLimits.ts@@ -37,6 +37,8 @@ export const IN_FLIGHT_BODY_BUDGET_BODIES = 3* loopback, an 8 MiB body is read in 7.6-22.6 ms alone and in under 60 ms with* three at once, so 10 s only cuts a body that has stalled. Without it a* stalled body keeps its reservation until Node's 300 s `requestTimeout`.+ * The wait before the grant is bounded separately, by the request's admission+ * clock (`STREAM_LIMITS.maxQueueWaitMs` from arrival, #739 D1).* Not env-tunable.*/export const BODY_READ_DEADLINE_MS = 10_000@@ -45,9 +47,11 @@ export const STREAM_LIMITS = {maxConcurrentStreams: 8,maxQueuedRequests: 16,maxStreamDurationMs: 1_800_000,- // Longest time a request may wait for a stream slot. Bounded so that queue- // wait + redeem + the first keepalive stays below the Host HTTP client's- // 300 s header timeout.+ // Longest total time a request may spend queued in this proxy: one+ // admission clock from arrival bounds the body budget, the visual gate and+ // the stream gate together (#739 D1). Bounded so that queue wait + redeem ++ // the first keepalive stays below the Host HTTP client's 300 s header+ // timeout.maxQueueWaitMs: 60_000,// Longest silence tolerated while waiting on the upstream (response headers// or the next SSE chunk). Matches the Codex CLI stream idle timeout.@@ -75,6 +79,20 @@ export class RequestLimitError extends Error {}}+/**+ * #739 D1-bis — the execution ticket expired while its request still waited+ * for a stream slot. Redeeming it could only end in `ticket_expired`, so the+ * request is refused as a capacity limit instead, before any redeem. It is a+ * RequestLimitError on the wire (503 `provider_unavailable`); the attempt+ * failure metric counts it as `provider_unavailable`, not `request_limit`.+ */+export class TicketLifeError extends RequestLimitError {+ constructor() {+ super('execution ticket expired while queued')+ this.name = 'TicketLifeError'+ }+}+export function assertBoundedDeadline(deadlineMs: number | undefined, maxDeadlineMs: number): number {// Without a valid maximum, Math.min below would return NaN for a valid// request deadline instead of refusing it.@@ -110,9 +128,14 @@ export class StreamGate {* Take a stream slot. When `signal` aborts (client disconnected) while the* caller is still queued, the waiter is rejected and its queue slot freed, so* a dropped client never proceeds to redeem an attempt. A waiter still- * queued after `maxQueueWaitMs` is rejected the same way.+ * queued after `maxQueueWaitMs`, or at `deadlineAt` (epoch ms) when that+ * comes first, is rejected the same way (#739 D1). A free slot is granted at+ * once whatever the deadline; the caller checks a deadline already past.*/- async acquire(signal?: AbortSignal): Promise<() => void> {+ async acquire(signal?: AbortSignal, deadlineAt?: number): Promise<() => void> {+ if (deadlineAt !== undefined && !Number.isFinite(deadlineAt)) {+ throw new RangeError(`a stream gate deadline must be a finite epoch time, got ${deadlineAt}`)+ }if (signal?.aborted) throw new RequestLimitError('stream request was aborted')if (this.running >= this.maxConcurrent) {if (this.queued >= this.maxQueued) throw new RequestLimitError('stream queue is full')@@ -131,7 +154,11 @@ export class StreamGate {resolve()return}- if (Date.now() - queuedAt >= this.maxQueueWaitMs) {+ const now = Date.now()+ if (+ now - queuedAt >= this.maxQueueWaitMs ||+ (deadlineAt !== undefined && now >= deadlineAt)+ ) {signal?.removeEventListener('abort', onAbort)reject(new RequestLimitError('stream queue wait exceeded'))return@@ -192,12 +219,18 @@ export class BodyBudget {/*** Reserve `bytes` of the budget. The returned release is idempotent. When* `signal` aborts while the caller is queued, the waiter is rejected and- * removed, and the waiters behind it are reconsidered.+ * removed, and the waiters behind it are reconsidered. A waiter still queued+ * at `deadlineAt` (epoch ms, the request's admission deadline, #739 D1) is+ * rejected and removed the same way, so a large body at the head of the+ * queue cannot hold back the smaller ones behind it past that instant.*/- async acquire(bytes: number, signal?: AbortSignal): Promise<() => void> {+ async acquire(bytes: number, signal?: AbortSignal, deadlineAt?: number): Promise<() => void> {if (!Number.isInteger(bytes) || bytes < 0 || bytes > this.capacityBytes) {throw new RangeError(`a body of ${bytes} bytes cannot fit a budget of ${this.capacityBytes}`)}+ if (deadlineAt !== undefined && !Number.isFinite(deadlineAt)) {+ throw new RangeError(`a body admission deadline must be a finite epoch time, got ${deadlineAt}`)+ }if (signal?.aborted) throw new RequestLimitError('body admission was aborted')if (this.waiters.length === 0 && this.inFlight + bytes <= this.capacityBytes) {return this.take(bytes)@@ -206,22 +239,33 @@ export class BodyBudget {throw new RequestLimitError('body admission queue is full')}return new Promise<() => void>((resolve, reject) => {- const onAbort = () => {+ let deadline: ReturnType<typeof setTimeout> | undefined+ const leave = (reason: string) => {const index = this.waiters.indexOf(waiter)if (index === -1) returnthis.waiters.splice(index, 1)- reject(new RequestLimitError('body admission was aborted'))+ clearTimeout(deadline)+ signal?.removeEventListener('abort', onAbort)+ reject(new RequestLimitError(reason))this.drain()}+ const onAbort = () => leave('body admission was aborted')const waiter: BodyWaiter = {bytes,grant: release => {+ clearTimeout(deadline)signal?.removeEventListener('abort', onAbort)resolve(release)},}signal?.addEventListener('abort', onAbort, { once: true })this.waiters.push(waiter)+ if (deadlineAt !== undefined) {+ deadline = setTimeout(+ () => leave('body admission wait exceeded'),+ Math.max(0, deadlineAt - Date.now())+ )+ }})}diff --git a/codex-llm-proxy/src/server.ts b/codex-llm-proxy/src/server.tsindex 2d5b17a20..d810be81a 100644--- a/codex-llm-proxy/src/server.ts+++ b/codex-llm-proxy/src/server.ts@@ -40,12 +40,23 @@ import {ENVELOPE_ALLOWANCE_BYTES,IN_FLIGHT_BODY_BUDGET_BODIES,RequestLimitError,+ STREAM_LIMITS,+ TicketLifeError,streamGate,visualStreamGate,} from './requestLimits.js'import { startSseHeartbeat } from './sseHeartbeat.js'-type GatedRequest = Request & {+type AdmittedRequest = Request & {+ /**+ * #739 D1 — the request's one admission clock, stamped by body admission:+ * arrival + `STREAM_LIMITS.maxQueueWaitMs`, in epoch ms. Every wait the+ * request performs (body budget, visual gate, stream gate) ends by then.+ */+ codexAdmissionDeadlineAt?: number+}++type GatedRequest = AdmittedRequest & {codexStreamRelease?: () => void/** Set by the platform gate before body admission (R9-M-B). */codexPlatform?: PlatformJwtClaims@@ -124,7 +135,9 @@ function boundedErrorHandler(err: unknown, _req: Request, res: Response, _next:* its connection closed. Otherwise the reservation is held until the response* closes, which for a completion is the whole stream (up to* `maxStreamDurationMs`). A queued waiter is dropped if the client leaves- * first; a full queue gets the stream gate's overload response.+ * first; a full queue gets the stream gate's overload response. #739 D1: the+ * request's admission clock is stamped here, at arrival, and a waiter still+ * queued when it runs out gets the same overload response.*/function bodyAdmission(budget: BodyBudget,@@ -132,7 +145,9 @@ function bodyAdmission(readDeadlineMs: number,parse: RequestHandler) {- return (req: Request, res: Response, next: NextFunction): void => {+ return (req: AdmittedRequest, res: Response, next: NextFunction): void => {+ const admissionDeadlineAt = Date.now() + STREAM_LIMITS.maxQueueWaitMs+ req.codexAdmissionDeadlineAt = admissionDeadlineAtif (req.headers['transfer-encoding'] !== undefined) {reject(res, 411, 'length_required')return@@ -159,7 +174,7 @@ function bodyAdmission(if (release) release()else abort.abort()})- budget.acquire(bytes, abort.signal).then(+ budget.acquire(bytes, abort.signal, admissionDeadlineAt).then(granted => {if (abort.signal.aborted) {granted()@@ -301,7 +316,7 @@ export function createProxyApps(const abortParse = (): void => parseAbort.abort()req.once('aborted', abortParse)try {- release = await visualStreamGate.acquire(parseAbort.signal)+ release = await visualStreamGate.acquire(parseAbort.signal, req.codexAdmissionDeadlineAt)} catch (err) {req.off('aborted', abortParse)if (err instanceof RequestLimitError) {@@ -342,6 +357,11 @@ export function createProxyApps(releaseAdmission()throw new Error('the completion route was reached without the platform gate')}+ const admissionDeadlineAt = gated.codexAdmissionDeadlineAt+ if (admissionDeadlineAt === undefined) {+ releaseAdmission()+ throw new Error('the completion route was reached without body admission')+ }if (!req.is('application/json')) {releaseAdmission()reject(res, 415, 'unsupported_media_type')@@ -439,7 +459,39 @@ export function createProxyApps(releaseAdmission()}if (!release) {- release = await streamGate.acquire(abort.signal)+ // #739 D1-bis: the wait also ends when the ticket dies. A request+ // still queued at `exp` could only be redeemed into ticket_expired,+ // so it is refused here, before any redeem. The margin is zero: a+ // ticket still alive when a slot frees is served as before.+ const deadlineAt = Math.min(admissionDeadlineAt, ticket.expiresAtMs)+ const refuseTicketLife = (): never => {+ logger.warn(+ {+ event: 'codex_proxy_admission_refused',+ reason: 'ticket_life',+ providerAttemptId: ticket.providerAttemptId,+ hostRef: ticket.hostRef,+ },+ 'admission refused'+ )+ throw new TicketLifeError()+ }+ if (deadlineAt <= Date.now()) {+ if (ticket.expiresAtMs <= admissionDeadlineAt) refuseTicketLife()+ throw new RequestLimitError('admission deadline exceeded')+ }+ try {+ release = await streamGate.acquire(abort.signal, deadlineAt)+ } catch (err) {+ if (+ err instanceof RequestLimitError &&+ !abort.signal.aborted &&+ Date.now() >= ticket.expiresAtMs+ ) {+ refuseTicketLife()+ }+ throw err+ }}res.status(200)res.setHeader('content-type', 'text/event-stream')@@ -502,9 +554,12 @@ export function createProxyApps(const mapped = mapError(err)metrics.observeAttempt('error', 'completion_stream')// A request limit answers provider_unavailable on the wire; its own- // label keeps it apart from real upstream outages in the metric.+ // label keeps it apart from real upstream outages in the metric. A+ // ticket that died in the queue is counted as provider_unavailable.metrics.observeAttemptFailure(- err instanceof RequestLimitError ? 'request_limit' : failureLabel(mapped.code)+ err instanceof RequestLimitError && !(err instanceof TicketLifeError)+ ? 'request_limit'+ : failureLabel(mapped.code))if (err instanceof UpstreamTimeoutError) metrics.observeUpstreamTimeout(err.kind)const deliveredAs = res.headersSent ? 'sse_error' : 'http_status'diff --git a/codex-llm-proxy/src/sseHeartbeat.ts b/codex-llm-proxy/src/sseHeartbeat.tsindex 52b315b7c..39ee9c9be 100644--- a/codex-llm-proxy/src/sseHeartbeat.ts+++ b/codex-llm-proxy/src/sseHeartbeat.ts@@ -9,16 +9,18 @@ const KEEPALIVE = ': keepalive\n\n'/*** Well under the Host's 300 s undici headers/body timeouts, so a silent- * upstream never reaches them: queue wait (60 s) + redeem (15 s) + one- * interval stays below 300 s.+ * upstream never reaches them: the admission clock (60 s) + redeem (15 s) ++ * one interval stays below 300 s.*/export const DEFAULT_HEARTBEAT_INTERVAL_MS = 15_000/**- * Largest interval config accepts. Two queue waits (visual gate, then stream- * gate: 120 s), the 15 s redeem and one interval must still end before the- * Host's 300 s headers timeout, and one interval must stay below its body- * timeout.+ * Largest interval config accepts. Every wait a request performs before the+ * redeem (body budget, visual gate, stream gate) is bounded by one admission+ * clock of `maxQueueWaitMs` (60 s) from arrival (#739 D1), so the queue time+ * counts once: 60 s + the 15 s redeem + one 60 s interval = 135 s, which ends+ * before the Host's 300 s headers timeout, and one interval stays below its+ * body timeout.*/export const MAX_HEARTBEAT_INTERVAL_MS = 60_000diff --git a/docs/architecture/codex-subscription-transport-contract.md b/docs/architecture/codex-subscription-transport-contract.mdindex e1747447a..7ab3ef91a 100644--- a/docs/architecture/codex-subscription-transport-contract.md+++ b/docs/architecture/codex-subscription-transport-contract.md@@ -340,10 +340,20 @@ behavior changes:- It drops queued stream-gate waiters on abort and checks the abort signalbefore redeeming a ticket. It also rejects an invalid or out-of-boundsdeadline before the redeem, so the single-use ticket is not consumed.- - It rejects a stream-gate waiter still queued after `maxQueueWaitMs` with- `provider_unavailable` (reason `stream queue wait exceeded`). Queue wait,- the 15 s control-api redeem timeout and the first keepalive together stay- below the Host HTTP client's 300 s header timeout.+ - It gives each request one admission clock, stamped at arrival: arrival ++ `maxQueueWaitMs`. The body budget, the visual gate and the stream gate+ all wait against that same instant, so `maxQueueWaitMs` is the total+ time a request may spend queued in the proxy. A waiter still queued when+ it runs out is rejected with `provider_unavailable` (reason+ `body admission wait exceeded` or `stream queue wait exceeded`). Queue+ wait, the 15 s control-api redeem timeout and the first keepalive+ together (60 + 15 + 60 = 135 s) stay below the Host HTTP client's 300 s+ header timeout.+ - The stream-gate wait also ends at the execution ticket's `exp`, with no+ margin. A request still queued then is answered 503+ `provider_unavailable` without a redeem, and the proxy logs+ `codex_proxy_admission_refused` with `reason: ticket_life`,+ `providerAttemptId` and `hostRef`.- It requires the redeem response to carry `maxStreamDurationMs` greaterthan 0. An absent value is a contract violation, not a default.- It logs one `codex_proxy_attempt_finished` event per completion attempt,diff --git a/docs/architecture/grok-subscription-transport-contract.md b/docs/architecture/grok-subscription-transport-contract.mdindex 8c5c80394..22b26c969 100644--- a/docs/architecture/grok-subscription-transport-contract.md+++ b/docs/architecture/grok-subscription-transport-contract.md@@ -204,10 +204,18 @@ Proxy robustness (both proxies):checks the abort signal before redeeming a ticket. It also rejects aninvalid or out-of-bounds deadline before the redeem, so the single-useticket is not consumed.-- A stream-gate waiter still queued after `maxQueueWaitMs` is rejected- with `provider_unavailable` (reason `stream queue wait exceeded`). Queue- wait, the 15 s control-api redeem timeout and the first keepalive together- stay below the Host HTTP client's 300 s header timeout.+- Each request gets one admission clock, stamped at arrival: arrival ++ `maxQueueWaitMs`. The body budget and the stream gate both wait against+ that same instant, so `maxQueueWaitMs` is the total time a request may+ spend queued in the proxy. A waiter still queued when it runs out is+ rejected with `provider_unavailable` (reason `body admission wait exceeded`+ or `stream queue wait exceeded`). Queue wait, the 15 s control-api redeem+ timeout and the first keepalive together (60 + 15 + 60 = 135 s) stay below+ the Host HTTP client's 300 s header timeout.+- The stream-gate wait also ends at the execution ticket's `exp`, with no+ margin. A request still queued then is answered 503 `provider_unavailable`+ without a redeem, and the proxy logs `grok_proxy_admission_refused` with+ `reason: ticket_life`, `providerAttemptId` and `hostRef`.- A single attempt streams for at most `maxStreamDurationMs`: the minimum of the proxy configuration, `STREAM_LIMITS`, the contract`maxDeadlineMs` and the value control-api returns on redeem.- The proxy fails at startup when `GROK_LLM_PROXY_CONTROL_API_URL` ordiff --git a/grok-llm-proxy/src/auth/executionTicketVerifier.ts b/grok-llm-proxy/src/auth/executionTicketVerifier.tsindex 404a9121c..0f30a4611 100644--- a/grok-llm-proxy/src/auth/executionTicketVerifier.ts+++ b/grok-llm-proxy/src/auth/executionTicketVerifier.ts@@ -8,6 +8,8 @@ export type ExecutionTicketClaims = {model: stringrequestHash: stringproviderAttemptId: string+ /** The signed `exp` in epoch milliseconds (#739 D1-bis). */+ expiresAtMs: number}export function verifyExecutionTicket(@@ -34,6 +36,7 @@ export function verifyExecutionTicket(model: claims.model,requestHash: claims.requestHash,providerAttemptId: claims.providerAttemptId,+ expiresAtMs: claims.exp * 1000,}} catch {return nulldiff --git a/grok-llm-proxy/src/requestLimits.ts b/grok-llm-proxy/src/requestLimits.tsindex ac431e1b0..6c79b1391 100644--- a/grok-llm-proxy/src/requestLimits.ts+++ b/grok-llm-proxy/src/requestLimits.ts@@ -35,7 +35,9 @@ export const IN_FLIGHT_BODY_BUDGET_BODIES = 3* the Codex twin over loopback, an 8 MiB body is read in 7.6-22.6 ms alone and* in under 60 ms with three at once, so 10 s only cuts a body that has* stalled. Without it a stalled body keeps its reservation until Node's 300 s- * `requestTimeout`. Not env-tunable.+ * `requestTimeout`. The wait before the grant is bounded separately, by the+ * request's admission clock (`STREAM_LIMITS.maxQueueWaitMs` from arrival,+ * #739 D1). Not env-tunable.*/export const BODY_READ_DEADLINE_MS = 10_000@@ -43,9 +45,10 @@ export const STREAM_LIMITS = {maxConcurrentStreams: 8,maxQueuedRequests: 16,maxStreamDurationMs: 1_800_000,- // Longest time a request may wait for a stream slot. Bounded so that queue- // wait + redeem + the first keepalive stays below the Host HTTP client's- // 300 s header timeout.+ // Longest total time a request may spend queued in this proxy: one+ // admission clock from arrival bounds the body budget and the stream gate+ // together (#739 D1). Bounded so that queue wait + redeem + the first+ // keepalive stays below the Host HTTP client's 300 s header timeout.maxQueueWaitMs: 60_000,// Longest silence tolerated while waiting on the upstream (response headers// or the next SSE chunk). Matches the Grok Build inference idle timeout.@@ -60,6 +63,20 @@ export class RequestLimitError extends Error {}}+/**+ * #739 D1-bis — the execution ticket expired while its request still waited+ * for a stream slot. Redeeming it could only end in `ticket_expired`, so the+ * request is refused as a capacity limit instead, before any redeem. It is a+ * RequestLimitError on the wire (503 `provider_unavailable`); the attempt+ * failure metric counts it as `provider_unavailable`, not `request_limit`.+ */+export class TicketLifeError extends RequestLimitError {+ constructor() {+ super('execution ticket expired while queued')+ this.name = 'TicketLifeError'+ }+}+export function assertBoundedDeadline(deadlineMs: number | undefined,maxDeadlineMs: number@@ -103,9 +120,14 @@ export class StreamGate {* Take a stream slot. When `signal` aborts (client disconnected) while the* caller is still queued, the waiter is rejected and its queue slot freed, so* a dropped client never proceeds to redeem an attempt. A waiter still- * queued after `maxQueueWaitMs` is rejected the same way.+ * queued after `maxQueueWaitMs`, or at `deadlineAt` (epoch ms) when that+ * comes first, is rejected the same way (#739 D1). A free slot is granted at+ * once whatever the deadline; the caller checks a deadline already past.*/- async acquire(signal?: AbortSignal): Promise<() => void> {+ async acquire(signal?: AbortSignal, deadlineAt?: number): Promise<() => void> {+ if (deadlineAt !== undefined && !Number.isFinite(deadlineAt)) {+ throw new RangeError(`a stream gate deadline must be a finite epoch time, got ${deadlineAt}`)+ }if (signal?.aborted) throw new RequestLimitError('stream request was aborted')if (this.running >= this.maxConcurrent) {if (this.queued >= this.maxQueued) throw new RequestLimitError('stream queue is full')@@ -124,7 +146,11 @@ export class StreamGate {resolve()return}- if (Date.now() - queuedAt >= this.maxQueueWaitMs) {+ const now = Date.now()+ if (+ now - queuedAt >= this.maxQueueWaitMs ||+ (deadlineAt !== undefined && now >= deadlineAt)+ ) {signal?.removeEventListener('abort', onAbort)reject(new RequestLimitError('stream queue wait exceeded'))return@@ -175,12 +201,18 @@ export class BodyBudget {/*** Reserve `bytes` of the budget. The returned release is idempotent. When* `signal` aborts while the caller is queued, the waiter is rejected and- * removed, and the waiters behind it are reconsidered.+ * removed, and the waiters behind it are reconsidered. A waiter still queued+ * at `deadlineAt` (epoch ms, the request's admission deadline, #739 D1) is+ * rejected and removed the same way, so a large body at the head of the+ * queue cannot hold back the smaller ones behind it past that instant.*/- async acquire(bytes: number, signal?: AbortSignal): Promise<() => void> {+ async acquire(bytes: number, signal?: AbortSignal, deadlineAt?: number): Promise<() => void> {if (!Number.isInteger(bytes) || bytes < 0 || bytes > this.capacityBytes) {throw new RangeError(`a body of ${bytes} bytes cannot fit a budget of ${this.capacityBytes}`)}+ if (deadlineAt !== undefined && !Number.isFinite(deadlineAt)) {+ throw new RangeError(`a body admission deadline must be a finite epoch time, got ${deadlineAt}`)+ }if (signal?.aborted) throw new RequestLimitError('body admission was aborted')if (this.waiters.length === 0 && this.inFlight + bytes <= this.capacityBytes) {return this.take(bytes)@@ -189,22 +221,33 @@ export class BodyBudget {throw new RequestLimitError('body admission queue is full')}return new Promise<() => void>((resolve, reject) => {- const onAbort = () => {+ let deadline: ReturnType<typeof setTimeout> | undefined+ const leave = (reason: string) => {const index = this.waiters.indexOf(waiter)if (index === -1) returnthis.waiters.splice(index, 1)- reject(new RequestLimitError('body admission was aborted'))+ clearTimeout(deadline)+ signal?.removeEventListener('abort', onAbort)+ reject(new RequestLimitError(reason))this.drain()}+ const onAbort = () => leave('body admission was aborted')const waiter: BodyWaiter = {bytes,grant: release => {+ clearTimeout(deadline)signal?.removeEventListener('abort', onAbort)resolve(release)},}signal?.addEventListener('abort', onAbort, { once: true })this.waiters.push(waiter)+ if (deadlineAt !== undefined) {+ deadline = setTimeout(+ () => leave('body admission wait exceeded'),+ Math.max(0, deadlineAt - Date.now())+ )+ }})}diff --git a/grok-llm-proxy/src/server.ts b/grok-llm-proxy/src/server.tsindex baa451342..7d28ca8fe 100644--- a/grok-llm-proxy/src/server.ts+++ b/grok-llm-proxy/src/server.ts@@ -33,11 +33,22 @@ import {BodyBudget,IN_FLIGHT_BODY_BUDGET_BODIES,RequestLimitError,+ STREAM_LIMITS,+ TicketLifeError,streamGate,} from './requestLimits.js'import { startSseHeartbeat } from './sseHeartbeat.js'-type GatedRequest = Request & {+type AdmittedRequest = Request & {+ /**+ * #739 D1 — the request's one admission clock, stamped by body admission:+ * arrival + `STREAM_LIMITS.maxQueueWaitMs`, in epoch ms. Every wait the+ * request performs (body budget, stream gate) ends by then.+ */+ grokAdmissionDeadlineAt?: number+}++type GatedRequest = AdmittedRequest & {/** Set by the platform gate before body admission (R9-M-B). */grokPlatform?: PlatformJwtClaims}@@ -102,7 +113,9 @@ function boundedErrorHandler(err: unknown, _req: Request, res: Response, _next:* reservation released and its connection closed. Otherwise the reservation is* held until the response closes, which for a completion is the whole stream* (up to `maxStreamDurationMs`). A queued waiter is dropped if the client- * leaves first; a full queue gets the stream gate's overload response.+ * leaves first; a full queue gets the stream gate's overload response. #739+ * D1: the request's admission clock is stamped here, at arrival, and a waiter+ * still queued when it runs out gets the same overload response.*/function bodyAdmission(budget: BodyBudget,@@ -110,7 +123,9 @@ function bodyAdmission(readDeadlineMs: number,parse: RequestHandler) {- return (req: Request, res: Response, next: NextFunction): void => {+ return (req: AdmittedRequest, res: Response, next: NextFunction): void => {+ const admissionDeadlineAt = Date.now() + STREAM_LIMITS.maxQueueWaitMs+ req.grokAdmissionDeadlineAt = admissionDeadlineAtif (req.headers['transfer-encoding'] !== undefined) {reject(res, 411, 'length_required')return@@ -137,7 +152,7 @@ function bodyAdmission(if (release) release()else abort.abort()})- budget.acquire(bytes, abort.signal).then(+ budget.acquire(bytes, abort.signal, admissionDeadlineAt).then(granted => {if (abort.signal.aborted) {granted()@@ -261,6 +276,10 @@ export function createProxyApps(if (!platform) {throw new Error('the completion route was reached without the platform gate')}+ const admissionDeadlineAt = (req as GatedRequest).grokAdmissionDeadlineAt+ if (admissionDeadlineAt === undefined) {+ throw new Error('the completion route was reached without body admission')+ }if (!req.is('application/json')) {reject(res, 415, 'unsupported_media_type')return@@ -316,7 +335,39 @@ export function createProxyApps(// it ends the response: a tick after `res.end()` would write after end.let stopHeartbeat: (() => void) | undefinedtry {- release = await streamGate.acquire(abort.signal)+ // #739 D1-bis: the wait also ends when the ticket dies. A request+ // still queued at `exp` could only be redeemed into ticket_expired,+ // so it is refused here, before any redeem. The margin is zero: a+ // ticket still alive when a slot frees is served as before.+ const deadlineAt = Math.min(admissionDeadlineAt, ticket.expiresAtMs)+ const refuseTicketLife = (): never => {+ logger.warn(+ {+ event: 'grok_proxy_admission_refused',+ reason: 'ticket_life',+ providerAttemptId: ticket.providerAttemptId,+ hostRef: ticket.hostRef,+ },+ 'admission refused'+ )+ throw new TicketLifeError()+ }+ if (deadlineAt <= Date.now()) {+ if (ticket.expiresAtMs <= admissionDeadlineAt) refuseTicketLife()+ throw new RequestLimitError('admission deadline exceeded')+ }+ try {+ release = await streamGate.acquire(abort.signal, deadlineAt)+ } catch (err) {+ if (+ err instanceof RequestLimitError &&+ !abort.signal.aborted &&+ Date.now() >= ticket.expiresAtMs+ ) {+ refuseTicketLife()+ }+ throw err+ }res.status(200)res.setHeader('content-type', 'text/event-stream')res.setHeader('cache-control', 'no-cache')@@ -377,9 +428,12 @@ export function createProxyApps(const mapped = mapError(err)metrics.observeAttempt('error', 'completion_stream')// A request limit answers provider_unavailable on the wire; its own- // label keeps it apart from real upstream outages in the metric.+ // label keeps it apart from real upstream outages in the metric. A+ // ticket that died in the queue is counted as provider_unavailable.metrics.observeAttemptFailure(- err instanceof RequestLimitError ? 'request_limit' : failureLabel(mapped.code)+ err instanceof RequestLimitError && !(err instanceof TicketLifeError)+ ? 'request_limit'+ : failureLabel(mapped.code))if (err instanceof UpstreamTimeoutError) metrics.observeUpstreamTimeout(err.kind)const deliveredAs = res.headersSent ? 'sse_error' : 'http_status'diff --git a/grok-llm-proxy/src/sseHeartbeat.ts b/grok-llm-proxy/src/sseHeartbeat.tsindex 1ea9ba011..8fb943871 100644--- a/grok-llm-proxy/src/sseHeartbeat.ts+++ b/grok-llm-proxy/src/sseHeartbeat.ts@@ -9,15 +9,18 @@ const KEEPALIVE = ': keepalive\n\n'/*** Well under the Host's 300 s undici headers/body timeouts, so a silent- * upstream never reaches them: queue wait (60 s) + redeem (15 s) + one- * interval stays below 300 s.+ * upstream never reaches them: the admission clock (60 s) + redeem (15 s) ++ * one interval stays below 300 s.*/export const DEFAULT_HEARTBEAT_INTERVAL_MS = 15_000/**- * Largest interval config accepts. The queue wait (60 s), the 15 s redeem and- * one interval must still end before the Host's 300 s headers timeout, and- * one interval must stay below its body timeout.+ * Largest interval config accepts. Every wait a request performs before the+ * redeem (body budget, stream gate) is bounded by one admission clock of+ * `maxQueueWaitMs` (60 s) from arrival (#739 D1), so the queue time counts+ * once: 60 s + the 15 s redeem + one 60 s interval = 135 s, which ends before+ * the Host's 300 s headers timeout, and one interval stays below its body+ * timeout.*/export const MAX_HEARTBEAT_INTERVAL_MS = 60_000