Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
fix(llm-proxies): scan Codex raw bodies before parse and read ticket life from the wait kind
Alfredo Lopez11d0f639ce43
diff --git a/codex-llm-proxy/src/requestLimits.ts b/codex-llm-proxy/src/requestLimits.tsindex 4c209f4c6..792e73c93 100644--- a/codex-llm-proxy/src/requestLimits.ts+++ b/codex-llm-proxy/src/requestLimits.ts@@ -28,6 +28,19 @@ export const DEFAULT_MAX_BODY_BYTES = CONTRACT_LIMITS.maxRequestBodyBytes + ENVE* and three queued 8 MiB bodies, #739 D5), the process peaked at 790 MiB of* RSS with `--max-old-space-size=384` and at 886-1009 MiB with an uncapped* heap, which is why the deployment sets that cap and a 1Gi memory limit.+ *+ * A8 — bytes alone do not bound the parse: JSON.parse allocates one heap+ * object per container, so the contracts cap containers at+ * `maxRequestContainers` (262 144) and every parser scans the raw body for it+ * before JSON.parse. Measured in this proxy (tsc build,+ * `--max-old-space-size=384`, upstream held open) with three ordinary bodies+ * at the 8 404 992-byte cap and two visual bodies in flight, each exactly+ * 262 144 containers: every run exited 0 and admitted all five, the heap kept+ * 154.4-163.4 MiB after a full GC and peaked at 205.6-238.4 MiB sampled. At+ * 524 288 containers one run peaked at 319.6 MiB, and at 1 048 576 the+ * process aborted. The bound for a new value: every run exits 0 and admits+ * five of five, at most 200 MiB after GC and 280 MiB sampled peak. These are+ * lower bounds: the serializations made while forwarding are not included.*/export const IN_FLIGHT_BODY_BUDGET_BODIES = 3@@ -79,9 +92,26 @@ export const VISUAL_STREAM_LIMITS = {// Largest delay setTimeout honors; Node fires anything above it after 1 ms.const MAX_TIMER_DELAY_MS = 2_147_483_647+/**+ * Why a wait was refused. `deadline` means the caller's own deadline ended+ * the wait, so a caller that passed the ticket's expiry as that deadline can+ * tell a dead ticket from a full queue without reading the clock again: the+ * deadline timer can fire a millisecond before `Date.now()` reaches it.+ */+export type RequestLimitKind =+ | 'aborted'+ | 'deadline'+ | 'invalid'+ | 'queue_full'+ | 'queue_wait'+ | 'ticket_life'+export class RequestLimitError extends Error {readonly code = 'provider_unavailable'- constructor(message: string) {+ constructor(+ message: string,+ readonly kind: RequestLimitKind+ ) {super(message)this.name = 'RequestLimitError'}@@ -96,7 +126,7 @@ export class RequestLimitError extends Error {*/export class TicketLifeError extends RequestLimitError {constructor() {- super('execution ticket expired while queued')+ super('execution ticket expired while queued', 'ticket_life')this.name = 'TicketLifeError'}}@@ -105,11 +135,11 @@ export function assertBoundedDeadline(deadlineMs: number | undefined, maxDeadlin// Without a valid maximum, Math.min below would return NaN for a valid// request deadline instead of refusing it.if (!Number.isInteger(maxDeadlineMs) || maxDeadlineMs <= 0) {- throw new RequestLimitError('deadline is invalid')+ throw new RequestLimitError('deadline is invalid', 'invalid')}const requested = deadlineMs ?? Math.min(maxDeadlineMs, STREAM_LIMITS.maxStreamDurationMs)if (!Number.isFinite(requested) || !Number.isInteger(requested) || requested <= 0) {- throw new RequestLimitError('deadline is invalid')+ throw new RequestLimitError('deadline is invalid', 'invalid')}return Math.min(requested, maxDeadlineMs, STREAM_LIMITS.maxStreamDurationMs, CONTRACT_LIMITS.maxDeadlineMs)}@@ -117,7 +147,7 @@ export function assertBoundedDeadline(deadlineMs: number | undefined, maxDeadlinexport function assertBoundedIdleTimeout(idleTimeoutMs: number | undefined): number {const requested = idleTimeoutMs ?? STREAM_LIMITS.upstreamIdleTimeoutMsif (!Number.isInteger(requested) || requested <= 0) {- throw new RequestLimitError('upstream idle timeout is invalid')+ throw new RequestLimitError('upstream idle timeout is invalid', 'invalid')}return Math.min(requested, STREAM_LIMITS.upstreamIdleTimeoutMs)}@@ -165,9 +195,9 @@ export class StreamGate {if (deadlineAt !== undefined && !Number.isFinite(deadlineAt)) {throw new RangeError(`a stream gate deadline must be a finite epoch time, got ${deadlineAt}`)}- if (signal?.aborted) throw new RequestLimitError('stream request was aborted')+ if (signal?.aborted) throw new RequestLimitError('stream request was aborted', 'aborted')if (this.running >= this.maxConcurrent) {- if (this.queued >= this.maxQueued) throw new RequestLimitError('stream queue is full')+ if (this.queued >= this.maxQueued) throw new RequestLimitError('stream queue is full', 'queue_full')this.queued += 1try {await new Promise<void>((resolve, reject) => {@@ -179,17 +209,24 @@ export class StreamGate {}const onAbort = () => {settle()- reject(new RequestLimitError('stream request was aborted'))- }- const expire = () => {- settle()- reject(new RequestLimitError('stream queue wait exceeded'))+ reject(new RequestLimitError('stream request was aborted', 'aborted'))}const queuedAt = performance.now()const waitBoundMs =deadlineAt === undefined? this.maxQueueWaitMs: Math.min(this.maxQueueWaitMs, deadlineAt - Date.now())+ // The caller's deadline governs only when it is the nearer bound.+ const deadlineGoverns = deadlineAt !== undefined && waitBoundMs < this.maxQueueWaitMs+ const expire = () => {+ settle()+ reject(+ new RequestLimitError(+ 'stream queue wait exceeded',+ deadlineGoverns ? 'deadline' : 'queue_wait'+ )+ )+ }const deadline = setTimeout(expire, Math.max(0, waitBoundMs))const wait = () => {// After the event loop stalls, a poll and the deadline can both be@@ -280,25 +317,25 @@ export class BodyBudget {if (deadlineAt !== undefined && !Number.isFinite(deadlineAt)) {throw new RangeError(`a body admission deadline must be a finite epoch time, got ${deadlineAt}`)}- if (signal?.aborted) throw new RequestLimitError('body admission was aborted')+ if (signal?.aborted) throw new RequestLimitError('body admission was aborted', 'aborted')if (this.waiters.length === 0 && this.inFlight + bytes <= this.capacityBytes) {return this.take(bytes)}if (this.waiters.length >= this.maxQueued) {- throw new RequestLimitError('body admission queue is full')+ throw new RequestLimitError('body admission queue is full', 'queue_full')}return new Promise<() => void>((resolve, reject) => {let deadline: ReturnType<typeof setTimeout> | undefined- const leave = (reason: string) => {+ const leave = (reason: string, kind: RequestLimitKind) => {const index = this.waiters.indexOf(waiter)if (index === -1) returnthis.waiters.splice(index, 1)clearTimeout(deadline)signal?.removeEventListener('abort', onAbort)- reject(new RequestLimitError(reason))+ reject(new RequestLimitError(reason, kind))this.drain()}- const onAbort = () => leave('body admission was aborted')+ const onAbort = () => leave('body admission was aborted', 'aborted')const waiter: BodyWaiter = {bytes,grant: release => {@@ -311,7 +348,7 @@ export class BodyBudget {this.waiters.push(waiter)if (deadlineAt !== undefined) {deadline = setTimeout(- () => leave('body admission wait exceeded'),+ () => leave('body admission wait exceeded', 'deadline'),Math.max(0, deadlineAt - Date.now()))}diff --git a/codex-llm-proxy/src/server.ts b/codex-llm-proxy/src/server.tsindex 226acca78..181c8873c 100644--- a/codex-llm-proxy/src/server.ts+++ b/codex-llm-proxy/src/server.ts@@ -32,8 +32,10 @@ import {defaultAddressLookup,} from './originPolicy.js'import {+ BODY_STRUCTURE_LIMITS,LIMITS,SCHEMA_VERSION_V2,+ createBodyStructureVerify,measureNonImageCompletionBytes,requestBodyLimitBytes,} from '@clerum/llm-provider-attempt-contract'@@ -108,23 +110,40 @@ function reject(res: Response, status: number, code: string): void {res.status(status).json({ error: code })}+// A8: every JSON parser scans the raw body before JSON.parse. JSON.parse+// allocates one heap object per container, so a body within the byte limit+// can still exhaust the heap; the scan refuses a body denser, more nested or+// with more containers than any request the contract accepts. Its depth bound+// (LIMITS.maxNestingDepth + 6) also keeps JSON.stringify and the contract's+// byte measurement from throwing RangeError on a deep body after parse.+const verifyBodyStructure = createBodyStructureVerify(BODY_STRUCTURE_LIMITS)+function boundedErrorHandler(err: unknown, _req: Request, res: Response, _next: () => void): void {- const typed = err as { type?: string; status?: number }+ const typed = err as { type?: string; status?: number; body?: unknown }if (typed?.type === 'entity.too.large' || typed?.status === 413) {reject(res, 413, 'payload_too_large')return}// R9-1: the parsers run with `inflate: false`, so body-parser refuses an// encoded body before reading it.- if (typed?.type === 'encoding.unsupported') {+ if (typed?.type === 'encoding.unsupported' || typed?.type === 'charset.unsupported') {reject(res, 415, 'unsupported_media_type')return}- if (err instanceof SyntaxError) {+ if (err instanceof SyntaxError || typed?.type === 'body.structure.too.deep') {reject(res, 400, 'invalid_request')return}- logger.error({ event: 'codex_proxy_error', err }, 'unhandled request error')+ // body-parser attaches the raw body to an error thrown by `verify` or by+ // JSON.parse. Such an error is logged by its type and status only.+ if (typed?.body !== undefined) {+ logger.error(+ { event: 'codex_proxy_error', type: typed.type, status: typed.status },+ 'unhandled request error'+ )+ } else {+ logger.error({ event: 'codex_proxy_error', err }, 'unhandled request error')+ }reject(res, 500, 'internal_error')}@@ -286,8 +305,16 @@ export function createProxyApps(const runtimeApp = express()// R9-1: `inflate: false` on every parser. The budgets count the declared wire// length, so an encoded body is refused (415) instead of inflated past it.- const ordinaryJson = express.json({ limit: config.maxBodyBytes, inflate: false })- const visualJson = express.json({ limit: config.maxVisualBodyBytes, inflate: false })+ const ordinaryJson = express.json({+ limit: config.maxBodyBytes,+ inflate: false,+ verify: verifyBodyStructure,+ })+ const visualJson = express.json({+ limit: config.maxVisualBodyBytes,+ inflate: false,+ verify: verifyBodyStructure,+ })// R9-M-B: the token is checked from the header before any budget is taken or// any body byte is read, so an anonymous caller cannot hold a reservation.// It runs after the rate limiter, so it cannot be forced ahead of the limit.@@ -339,7 +366,7 @@ export function createProxyApps(reject(res, 503, err.code)return}- next()+ next(err)return}req.off('aborted', abortParse)@@ -475,47 +502,70 @@ export function createProxyApps(// cap. Keep it for the ChatGPT stream only when the parsed V2 body still// exceeds that cap (image bytes stay resident). Padding, V1, and small// V2 release it and take the 8-wide gate.- if (visualRequest && wholeBodyBytes > config.maxBodyBytes) {- release = gated.codexStreamRelease- gated.codexStreamRelease = undefined- } else {- releaseAdmission()+ // #739 D1-bis: every admission wait also ends when the ticket dies. A+ // request still queued at `exp` could only be redeemed into+ // ticket_expired, so it is refused here, before any redeem. The margin+ // is zero: a ticket still alive when a slot frees is served as before.+ const deadlineAt = Math.min(admissionDeadlineAt, ticket.expiresAtMs)+ const refuseTicketLife = (): never => {+ logger.warn(+ {+ event: 'codex_proxy_admission_refused',+ reason: 'ticket_life',+ providerAttemptId: ticket.providerAttemptId,+ hostRef: ticket.hostRef,+ },+ 'admission refused'+ )+ throw new TicketLifeError()}- if (!release) {- // #739 D1-bis: the wait also ends when the ticket dies. A request- // still queued at `exp` could only be redeemed into ticket_expired,- // so it is refused here, before any redeem. The margin is zero: a- // ticket still alive when a slot frees is served as before.- const deadlineAt = Math.min(admissionDeadlineAt, ticket.expiresAtMs)- const refuseTicketLife = (): never => {- logger.warn(- {- event: 'codex_proxy_admission_refused',- reason: 'ticket_life',- providerAttemptId: ticket.providerAttemptId,- hostRef: ticket.hostRef,- },- 'admission refused'- )- throw new TicketLifeError()- }+ const waitWithinTicketLife = async (+ acquire: () => Promise<() => void>+ ): Promise<() => void> => {if (deadlineAt <= Date.now()) {if (ticket.expiresAtMs <= admissionDeadlineAt) refuseTicketLife()- throw new RequestLimitError('admission deadline exceeded')+ throw new RequestLimitError('admission deadline exceeded', 'deadline')}try {- release = await streamGate.acquire(abort.signal, deadlineAt)+ return await acquire()} catch (err) {+ // The wait ended on its own deadline, and that deadline was the+ // ticket's expiry: comparing clocks here misses a timer that fired+ // a millisecond early.if (err instanceof RequestLimitError &&+ err.kind === 'deadline' &&!abort.signal.aborted &&- Date.now() >= ticket.expiresAtMs+ ticket.expiresAtMs <= admissionDeadlineAt) {refuseTicketLife()}throw err}}+ if (visualRequest && wholeBodyBytes > config.maxBodyBytes) {+ release = gated.codexStreamRelease+ gated.codexStreamRelease = undefined+ } else if (gated.codexStreamRelease) {+ // A8 D3: a demoted body was read through the visual gate, so it holds+ // no body-budget reservation. It takes one for its compact size+ // before it gives up the visual slot, so the 8-wide stream gate+ // never holds more resident bodies than the budget admits. Every+ // request takes its locks in the same order (visual, budget,+ // stream), so no two requests wait on each other.+ try {+ gated.codexBodyRelease = await waitWithinTicketLife(() =>+ bodyBudget.acquire(wholeBodyBytes, abort.signal, deadlineAt)+ )+ } finally {+ releaseAdmission()+ }+ } else {+ releaseAdmission()+ }+ if (!release) {+ release = await waitWithinTicketLife(() => streamGate.acquire(abort.signal, deadlineAt))+ }res.status(200)res.setHeader('content-type', 'text/event-stream')res.setHeader('cache-control', 'no-cache')@@ -629,6 +679,10 @@ export function createProxyApps(} finally {stopHeartbeat?.()release?.()+ // A demoted body's reservation, when the stream ended before the body+ // was written upstream. Budgeted bodies are also released on close.+ gated.codexBodyRelease?.()+ gated.codexBodyRelease = undefined}})()})@@ -656,7 +710,7 @@ export function createProxyApps(bodyBudget,config.maxBodyBytes,bodyReadDeadlineMs,- express.json({ limit: config.maxBodyBytes, inflate: false })+ express.json({ limit: config.maxBodyBytes, inflate: false, verify: verifyBodyStructure }))const adminHandler = (kind: 'models' | 'test') => (req: Request, res: Response) => {if (!req.is('application/json')) {diff --git a/grok-llm-proxy/src/requestLimits.ts b/grok-llm-proxy/src/requestLimits.tsindex 5bae29166..aa98f8fb0 100644--- a/grok-llm-proxy/src/requestLimits.ts+++ b/grok-llm-proxy/src/requestLimits.ts@@ -84,9 +84,26 @@ export const VISUAL_STREAM_LIMITS = {// Largest delay setTimeout honors; Node fires anything above it after 1 ms.const MAX_TIMER_DELAY_MS = 2_147_483_647+/**+ * Why a wait was refused. `deadline` means the caller's own deadline ended+ * the wait, so a caller that passed the ticket's expiry as that deadline can+ * tell a dead ticket from a full queue without reading the clock again: the+ * deadline timer can fire a millisecond before `Date.now()` reaches it.+ */+export type RequestLimitKind =+ | 'aborted'+ | 'deadline'+ | 'invalid'+ | 'queue_full'+ | 'queue_wait'+ | 'ticket_life'+export class RequestLimitError extends Error {readonly code = 'provider_unavailable'- constructor(message: string) {+ constructor(+ message: string,+ readonly kind: RequestLimitKind+ ) {super(message)this.name = 'RequestLimitError'}@@ -101,7 +118,7 @@ export class RequestLimitError extends Error {*/export class TicketLifeError extends RequestLimitError {constructor() {- super('execution ticket expired while queued')+ super('execution ticket expired while queued', 'ticket_life')this.name = 'TicketLifeError'}}@@ -113,11 +130,11 @@ export function assertBoundedDeadline(// Without a valid maximum, Math.min below would return NaN for a valid// request deadline instead of refusing it.if (!Number.isInteger(maxDeadlineMs) || maxDeadlineMs <= 0) {- throw new RequestLimitError('deadline is invalid')+ throw new RequestLimitError('deadline is invalid', 'invalid')}const requested = deadlineMs ?? Math.min(maxDeadlineMs, STREAM_LIMITS.maxStreamDurationMs)if (!Number.isFinite(requested) || !Number.isInteger(requested) || requested <= 0) {- throw new RequestLimitError('deadline is invalid')+ throw new RequestLimitError('deadline is invalid', 'invalid')}return Math.min(requested,@@ -130,7 +147,7 @@ export function assertBoundedDeadline(export function assertBoundedIdleTimeout(idleTimeoutMs: number | undefined): number {const requested = idleTimeoutMs ?? STREAM_LIMITS.upstreamIdleTimeoutMsif (!Number.isInteger(requested) || requested <= 0) {- throw new RequestLimitError('upstream idle timeout is invalid')+ throw new RequestLimitError('upstream idle timeout is invalid', 'invalid')}return Math.min(requested, STREAM_LIMITS.upstreamIdleTimeoutMs)}@@ -178,9 +195,9 @@ export class StreamGate {if (deadlineAt !== undefined && !Number.isFinite(deadlineAt)) {throw new RangeError(`a stream gate deadline must be a finite epoch time, got ${deadlineAt}`)}- if (signal?.aborted) throw new RequestLimitError('stream request was aborted')+ if (signal?.aborted) throw new RequestLimitError('stream request was aborted', 'aborted')if (this.running >= this.maxConcurrent) {- if (this.queued >= this.maxQueued) throw new RequestLimitError('stream queue is full')+ if (this.queued >= this.maxQueued) throw new RequestLimitError('stream queue is full', 'queue_full')this.queued += 1try {await new Promise<void>((resolve, reject) => {@@ -192,17 +209,24 @@ export class StreamGate {}const onAbort = () => {settle()- reject(new RequestLimitError('stream request was aborted'))- }- const expire = () => {- settle()- reject(new RequestLimitError('stream queue wait exceeded'))+ reject(new RequestLimitError('stream request was aborted', 'aborted'))}const queuedAt = performance.now()const waitBoundMs =deadlineAt === undefined? this.maxQueueWaitMs: Math.min(this.maxQueueWaitMs, deadlineAt - Date.now())+ // The caller's deadline governs only when it is the nearer bound.+ const deadlineGoverns = deadlineAt !== undefined && waitBoundMs < this.maxQueueWaitMs+ const expire = () => {+ settle()+ reject(+ new RequestLimitError(+ 'stream queue wait exceeded',+ deadlineGoverns ? 'deadline' : 'queue_wait'+ )+ )+ }const deadline = setTimeout(expire, Math.max(0, waitBoundMs))const wait = () => {// After the event loop stalls, a poll and the deadline can both be@@ -293,25 +317,25 @@ export class BodyBudget {if (deadlineAt !== undefined && !Number.isFinite(deadlineAt)) {throw new RangeError(`a body admission deadline must be a finite epoch time, got ${deadlineAt}`)}- if (signal?.aborted) throw new RequestLimitError('body admission was aborted')+ if (signal?.aborted) throw new RequestLimitError('body admission was aborted', 'aborted')if (this.waiters.length === 0 && this.inFlight + bytes <= this.capacityBytes) {return this.take(bytes)}if (this.waiters.length >= this.maxQueued) {- throw new RequestLimitError('body admission queue is full')+ throw new RequestLimitError('body admission queue is full', 'queue_full')}return new Promise<() => void>((resolve, reject) => {let deadline: ReturnType<typeof setTimeout> | undefined- const leave = (reason: string) => {+ const leave = (reason: string, kind: RequestLimitKind) => {const index = this.waiters.indexOf(waiter)if (index === -1) returnthis.waiters.splice(index, 1)clearTimeout(deadline)signal?.removeEventListener('abort', onAbort)- reject(new RequestLimitError(reason))+ reject(new RequestLimitError(reason, kind))this.drain()}- const onAbort = () => leave('body admission was aborted')+ const onAbort = () => leave('body admission was aborted', 'aborted')const waiter: BodyWaiter = {bytes,grant: release => {@@ -324,7 +348,7 @@ export class BodyBudget {this.waiters.push(waiter)if (deadlineAt !== undefined) {deadline = setTimeout(- () => leave('body admission wait exceeded'),+ () => leave('body admission wait exceeded', 'deadline'),Math.max(0, deadlineAt - Date.now()))}diff --git a/grok-llm-proxy/src/server.ts b/grok-llm-proxy/src/server.tsindex bf64f8ac4..5e1af9fc8 100644--- a/grok-llm-proxy/src/server.ts+++ b/grok-llm-proxy/src/server.ts@@ -595,15 +595,19 @@ export function createProxyApps(): Promise<() => void> => {if (deadlineAt <= Date.now()) {if (ticket.expiresAtMs <= admissionDeadlineAt) refuseTicketLife()- throw new RequestLimitError('admission deadline exceeded')+ throw new RequestLimitError('admission deadline exceeded', 'deadline')}try {return await acquire()} catch (err) {+ // The wait ended on its own deadline, and that deadline was the+ // ticket's expiry: comparing clocks here misses a timer that fired+ // a millisecond early.if (err instanceof RequestLimitError &&+ err.kind === 'deadline' &&!abort.signal.aborted &&- Date.now() >= ticket.expiresAtMs+ ticket.expiresAtMs <= admissionDeadlineAt) {refuseTicketLife()}diff --git a/packages/llm-provider-attempt-contract/index.cjs b/packages/llm-provider-attempt-contract/index.cjsindex 530d5d27f..1e702539a 100644--- a/packages/llm-provider-attempt-contract/index.cjs+++ b/packages/llm-provider-attempt-contract/index.cjs@@ -53,7 +53,7 @@ const LIMITS = Object.freeze({// maxRequestBodyBytes with about four million of them, and three such// bodies at once exhaust a proxy capped at --max-old-space-size=384. A// conversation needs a few thousand containers; tool results are strings- // and never count. The A8 measurement of both proxies at this bound is in+ // and never count. The A8 measurement at this bound is in// codex-llm-proxy/src/requestLimits.ts. Must equal the Grok contract's// value.maxRequestContainers: 262144,