Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript

fix(llm-proxies): scan Codex raw bodies before parse and read ticket life from the wait kind

Alfredo Lopez11d0f639ce43
diff --git a/codex-llm-proxy/src/requestLimits.ts b/codex-llm-proxy/src/requestLimits.ts
index 4c209f4c6..792e73c93 100644
--- a/codex-llm-proxy/src/requestLimits.ts
+++ b/codex-llm-proxy/src/requestLimits.ts
@@ -28,6 +28,19 @@ export const DEFAULT_MAX_BODY_BYTES = CONTRACT_LIMITS.maxRequestBodyBytes + ENVE
* and three queued 8 MiB bodies, #739 D5), the process peaked at 790 MiB of
* RSS with `--max-old-space-size=384` and at 886-1009 MiB with an uncapped
* heap, which is why the deployment sets that cap and a 1Gi memory limit.
+ *
+ * A8 — bytes alone do not bound the parse: JSON.parse allocates one heap
+ * object per container, so the contracts cap containers at
+ * `maxRequestContainers` (262 144) and every parser scans the raw body for it
+ * before JSON.parse. Measured in this proxy (tsc build,
+ * `--max-old-space-size=384`, upstream held open) with three ordinary bodies
+ * at the 8 404 992-byte cap and two visual bodies in flight, each exactly
+ * 262 144 containers: every run exited 0 and admitted all five, the heap kept
+ * 154.4-163.4 MiB after a full GC and peaked at 205.6-238.4 MiB sampled. At
+ * 524 288 containers one run peaked at 319.6 MiB, and at 1 048 576 the
+ * process aborted. The bound for a new value: every run exits 0 and admits
+ * five of five, at most 200 MiB after GC and 280 MiB sampled peak. These are
+ * lower bounds: the serializations made while forwarding are not included.
*/
export const IN_FLIGHT_BODY_BUDGET_BODIES = 3
@@ -79,9 +92,26 @@ export const VISUAL_STREAM_LIMITS = {
// Largest delay setTimeout honors; Node fires anything above it after 1 ms.
const MAX_TIMER_DELAY_MS = 2_147_483_647
+/**
+ * Why a wait was refused. `deadline` means the caller's own deadline ended
+ * the wait, so a caller that passed the ticket's expiry as that deadline can
+ * tell a dead ticket from a full queue without reading the clock again: the
+ * deadline timer can fire a millisecond before `Date.now()` reaches it.
+ */
+export type RequestLimitKind =
+ | 'aborted'
+ | 'deadline'
+ | 'invalid'
+ | 'queue_full'
+ | 'queue_wait'
+ | 'ticket_life'
+
export class RequestLimitError extends Error {
readonly code = 'provider_unavailable'
- constructor(message: string) {
+ constructor(
+ message: string,
+ readonly kind: RequestLimitKind
+ ) {
super(message)
this.name = 'RequestLimitError'
}
@@ -96,7 +126,7 @@ export class RequestLimitError extends Error {
*/
export class TicketLifeError extends RequestLimitError {
constructor() {
- super('execution ticket expired while queued')
+ super('execution ticket expired while queued', 'ticket_life')
this.name = 'TicketLifeError'
}
}
@@ -105,11 +135,11 @@ export function assertBoundedDeadline(deadlineMs: number | undefined, maxDeadlin
// Without a valid maximum, Math.min below would return NaN for a valid
// request deadline instead of refusing it.
if (!Number.isInteger(maxDeadlineMs) || maxDeadlineMs <= 0) {
- throw new RequestLimitError('deadline is invalid')
+ throw new RequestLimitError('deadline is invalid', 'invalid')
}
const requested = deadlineMs ?? Math.min(maxDeadlineMs, STREAM_LIMITS.maxStreamDurationMs)
if (!Number.isFinite(requested) || !Number.isInteger(requested) || requested <= 0) {
- throw new RequestLimitError('deadline is invalid')
+ throw new RequestLimitError('deadline is invalid', 'invalid')
}
return Math.min(requested, maxDeadlineMs, STREAM_LIMITS.maxStreamDurationMs, CONTRACT_LIMITS.maxDeadlineMs)
}
@@ -117,7 +147,7 @@ export function assertBoundedDeadline(deadlineMs: number | undefined, maxDeadlin
export function assertBoundedIdleTimeout(idleTimeoutMs: number | undefined): number {
const requested = idleTimeoutMs ?? STREAM_LIMITS.upstreamIdleTimeoutMs
if (!Number.isInteger(requested) || requested <= 0) {
- throw new RequestLimitError('upstream idle timeout is invalid')
+ throw new RequestLimitError('upstream idle timeout is invalid', 'invalid')
}
return Math.min(requested, STREAM_LIMITS.upstreamIdleTimeoutMs)
}
@@ -165,9 +195,9 @@ export class StreamGate {
if (deadlineAt !== undefined && !Number.isFinite(deadlineAt)) {
throw new RangeError(`a stream gate deadline must be a finite epoch time, got ${deadlineAt}`)
}
- if (signal?.aborted) throw new RequestLimitError('stream request was aborted')
+ if (signal?.aborted) throw new RequestLimitError('stream request was aborted', 'aborted')
if (this.running >= this.maxConcurrent) {
- if (this.queued >= this.maxQueued) throw new RequestLimitError('stream queue is full')
+ if (this.queued >= this.maxQueued) throw new RequestLimitError('stream queue is full', 'queue_full')
this.queued += 1
try {
await new Promise<void>((resolve, reject) => {
@@ -179,17 +209,24 @@ export class StreamGate {
}
const onAbort = () => {
settle()
- reject(new RequestLimitError('stream request was aborted'))
- }
- const expire = () => {
- settle()
- reject(new RequestLimitError('stream queue wait exceeded'))
+ reject(new RequestLimitError('stream request was aborted', 'aborted'))
}
const queuedAt = performance.now()
const waitBoundMs =
deadlineAt === undefined
? this.maxQueueWaitMs
: Math.min(this.maxQueueWaitMs, deadlineAt - Date.now())
+ // The caller's deadline governs only when it is the nearer bound.
+ const deadlineGoverns = deadlineAt !== undefined && waitBoundMs < this.maxQueueWaitMs
+ const expire = () => {
+ settle()
+ reject(
+ new RequestLimitError(
+ 'stream queue wait exceeded',
+ deadlineGoverns ? 'deadline' : 'queue_wait'
+ )
+ )
+ }
const deadline = setTimeout(expire, Math.max(0, waitBoundMs))
const wait = () => {
// After the event loop stalls, a poll and the deadline can both be
@@ -280,25 +317,25 @@ export class BodyBudget {
if (deadlineAt !== undefined && !Number.isFinite(deadlineAt)) {
throw new RangeError(`a body admission deadline must be a finite epoch time, got ${deadlineAt}`)
}
- if (signal?.aborted) throw new RequestLimitError('body admission was aborted')
+ if (signal?.aborted) throw new RequestLimitError('body admission was aborted', 'aborted')
if (this.waiters.length === 0 && this.inFlight + bytes <= this.capacityBytes) {
return this.take(bytes)
}
if (this.waiters.length >= this.maxQueued) {
- throw new RequestLimitError('body admission queue is full')
+ throw new RequestLimitError('body admission queue is full', 'queue_full')
}
return new Promise<() => void>((resolve, reject) => {
let deadline: ReturnType<typeof setTimeout> | undefined
- const leave = (reason: string) => {
+ const leave = (reason: string, kind: RequestLimitKind) => {
const index = this.waiters.indexOf(waiter)
if (index === -1) return
this.waiters.splice(index, 1)
clearTimeout(deadline)
signal?.removeEventListener('abort', onAbort)
- reject(new RequestLimitError(reason))
+ reject(new RequestLimitError(reason, kind))
this.drain()
}
- const onAbort = () => leave('body admission was aborted')
+ const onAbort = () => leave('body admission was aborted', 'aborted')
const waiter: BodyWaiter = {
bytes,
grant: release => {
@@ -311,7 +348,7 @@ export class BodyBudget {
this.waiters.push(waiter)
if (deadlineAt !== undefined) {
deadline = setTimeout(
- () => leave('body admission wait exceeded'),
+ () => leave('body admission wait exceeded', 'deadline'),
Math.max(0, deadlineAt - Date.now())
)
}
diff --git a/codex-llm-proxy/src/server.ts b/codex-llm-proxy/src/server.ts
index 226acca78..181c8873c 100644
--- a/codex-llm-proxy/src/server.ts
+++ b/codex-llm-proxy/src/server.ts
@@ -32,8 +32,10 @@ import {
defaultAddressLookup,
} from './originPolicy.js'
import {
+ BODY_STRUCTURE_LIMITS,
LIMITS,
SCHEMA_VERSION_V2,
+ createBodyStructureVerify,
measureNonImageCompletionBytes,
requestBodyLimitBytes,
} from '@clerum/llm-provider-attempt-contract'
@@ -108,23 +110,40 @@ function reject(res: Response, status: number, code: string): void {
res.status(status).json({ error: code })
}
+// A8: every JSON parser scans the raw body before JSON.parse. JSON.parse
+// allocates one heap object per container, so a body within the byte limit
+// can still exhaust the heap; the scan refuses a body denser, more nested or
+// with more containers than any request the contract accepts. Its depth bound
+// (LIMITS.maxNestingDepth + 6) also keeps JSON.stringify and the contract's
+// byte measurement from throwing RangeError on a deep body after parse.
+const verifyBodyStructure = createBodyStructureVerify(BODY_STRUCTURE_LIMITS)
+
function boundedErrorHandler(err: unknown, _req: Request, res: Response, _next: () => void): void {
- const typed = err as { type?: string; status?: number }
+ const typed = err as { type?: string; status?: number; body?: unknown }
if (typed?.type === 'entity.too.large' || typed?.status === 413) {
reject(res, 413, 'payload_too_large')
return
}
// R9-1: the parsers run with `inflate: false`, so body-parser refuses an
// encoded body before reading it.
- if (typed?.type === 'encoding.unsupported') {
+ if (typed?.type === 'encoding.unsupported' || typed?.type === 'charset.unsupported') {
reject(res, 415, 'unsupported_media_type')
return
}
- if (err instanceof SyntaxError) {
+ if (err instanceof SyntaxError || typed?.type === 'body.structure.too.deep') {
reject(res, 400, 'invalid_request')
return
}
- logger.error({ event: 'codex_proxy_error', err }, 'unhandled request error')
+ // body-parser attaches the raw body to an error thrown by `verify` or by
+ // JSON.parse. Such an error is logged by its type and status only.
+ if (typed?.body !== undefined) {
+ logger.error(
+ { event: 'codex_proxy_error', type: typed.type, status: typed.status },
+ 'unhandled request error'
+ )
+ } else {
+ logger.error({ event: 'codex_proxy_error', err }, 'unhandled request error')
+ }
reject(res, 500, 'internal_error')
}
@@ -286,8 +305,16 @@ export function createProxyApps(
const runtimeApp = express()
// R9-1: `inflate: false` on every parser. The budgets count the declared wire
// length, so an encoded body is refused (415) instead of inflated past it.
- const ordinaryJson = express.json({ limit: config.maxBodyBytes, inflate: false })
- const visualJson = express.json({ limit: config.maxVisualBodyBytes, inflate: false })
+ const ordinaryJson = express.json({
+ limit: config.maxBodyBytes,
+ inflate: false,
+ verify: verifyBodyStructure,
+ })
+ const visualJson = express.json({
+ limit: config.maxVisualBodyBytes,
+ inflate: false,
+ verify: verifyBodyStructure,
+ })
// R9-M-B: the token is checked from the header before any budget is taken or
// any body byte is read, so an anonymous caller cannot hold a reservation.
// It runs after the rate limiter, so it cannot be forced ahead of the limit.
@@ -339,7 +366,7 @@ export function createProxyApps(
reject(res, 503, err.code)
return
}
- next()
+ next(err)
return
}
req.off('aborted', abortParse)
@@ -475,47 +502,70 @@ export function createProxyApps(
// cap. Keep it for the ChatGPT stream only when the parsed V2 body still
// exceeds that cap (image bytes stay resident). Padding, V1, and small
// V2 release it and take the 8-wide gate.
- if (visualRequest && wholeBodyBytes > config.maxBodyBytes) {
- release = gated.codexStreamRelease
- gated.codexStreamRelease = undefined
- } else {
- releaseAdmission()
+ // #739 D1-bis: every admission wait also ends when the ticket dies. A
+ // request still queued at `exp` could only be redeemed into
+ // ticket_expired, so it is refused here, before any redeem. The margin
+ // is zero: a ticket still alive when a slot frees is served as before.
+ const deadlineAt = Math.min(admissionDeadlineAt, ticket.expiresAtMs)
+ const refuseTicketLife = (): never => {
+ logger.warn(
+ {
+ event: 'codex_proxy_admission_refused',
+ reason: 'ticket_life',
+ providerAttemptId: ticket.providerAttemptId,
+ hostRef: ticket.hostRef,
+ },
+ 'admission refused'
+ )
+ throw new TicketLifeError()
}
- if (!release) {
- // #739 D1-bis: the wait also ends when the ticket dies. A request
- // still queued at `exp` could only be redeemed into ticket_expired,
- // so it is refused here, before any redeem. The margin is zero: a
- // ticket still alive when a slot frees is served as before.
- const deadlineAt = Math.min(admissionDeadlineAt, ticket.expiresAtMs)
- const refuseTicketLife = (): never => {
- logger.warn(
- {
- event: 'codex_proxy_admission_refused',
- reason: 'ticket_life',
- providerAttemptId: ticket.providerAttemptId,
- hostRef: ticket.hostRef,
- },
- 'admission refused'
- )
- throw new TicketLifeError()
- }
+ const waitWithinTicketLife = async (
+ acquire: () => Promise<() => void>
+ ): Promise<() => void> => {
if (deadlineAt <= Date.now()) {
if (ticket.expiresAtMs <= admissionDeadlineAt) refuseTicketLife()
- throw new RequestLimitError('admission deadline exceeded')
+ throw new RequestLimitError('admission deadline exceeded', 'deadline')
}
try {
- release = await streamGate.acquire(abort.signal, deadlineAt)
+ return await acquire()
} catch (err) {
+ // The wait ended on its own deadline, and that deadline was the
+ // ticket's expiry: comparing clocks here misses a timer that fired
+ // a millisecond early.
if (
err instanceof RequestLimitError &&
+ err.kind === 'deadline' &&
!abort.signal.aborted &&
- Date.now() >= ticket.expiresAtMs
+ ticket.expiresAtMs <= admissionDeadlineAt
) {
refuseTicketLife()
}
throw err
}
}
+ if (visualRequest && wholeBodyBytes > config.maxBodyBytes) {
+ release = gated.codexStreamRelease
+ gated.codexStreamRelease = undefined
+ } else if (gated.codexStreamRelease) {
+ // A8 D3: a demoted body was read through the visual gate, so it holds
+ // no body-budget reservation. It takes one for its compact size
+ // before it gives up the visual slot, so the 8-wide stream gate
+ // never holds more resident bodies than the budget admits. Every
+ // request takes its locks in the same order (visual, budget,
+ // stream), so no two requests wait on each other.
+ try {
+ gated.codexBodyRelease = await waitWithinTicketLife(() =>
+ bodyBudget.acquire(wholeBodyBytes, abort.signal, deadlineAt)
+ )
+ } finally {
+ releaseAdmission()
+ }
+ } else {
+ releaseAdmission()
+ }
+ if (!release) {
+ release = await waitWithinTicketLife(() => streamGate.acquire(abort.signal, deadlineAt))
+ }
res.status(200)
res.setHeader('content-type', 'text/event-stream')
res.setHeader('cache-control', 'no-cache')
@@ -629,6 +679,10 @@ export function createProxyApps(
} finally {
stopHeartbeat?.()
release?.()
+ // A demoted body's reservation, when the stream ended before the body
+ // was written upstream. Budgeted bodies are also released on close.
+ gated.codexBodyRelease?.()
+ gated.codexBodyRelease = undefined
}
})()
})
@@ -656,7 +710,7 @@ export function createProxyApps(
bodyBudget,
config.maxBodyBytes,
bodyReadDeadlineMs,
- express.json({ limit: config.maxBodyBytes, inflate: false })
+ express.json({ limit: config.maxBodyBytes, inflate: false, verify: verifyBodyStructure })
)
const adminHandler = (kind: 'models' | 'test') => (req: Request, res: Response) => {
if (!req.is('application/json')) {
diff --git a/grok-llm-proxy/src/requestLimits.ts b/grok-llm-proxy/src/requestLimits.ts
index 5bae29166..aa98f8fb0 100644
--- a/grok-llm-proxy/src/requestLimits.ts
+++ b/grok-llm-proxy/src/requestLimits.ts
@@ -84,9 +84,26 @@ export const VISUAL_STREAM_LIMITS = {
// Largest delay setTimeout honors; Node fires anything above it after 1 ms.
const MAX_TIMER_DELAY_MS = 2_147_483_647
+/**
+ * Why a wait was refused. `deadline` means the caller's own deadline ended
+ * the wait, so a caller that passed the ticket's expiry as that deadline can
+ * tell a dead ticket from a full queue without reading the clock again: the
+ * deadline timer can fire a millisecond before `Date.now()` reaches it.
+ */
+export type RequestLimitKind =
+ | 'aborted'
+ | 'deadline'
+ | 'invalid'
+ | 'queue_full'
+ | 'queue_wait'
+ | 'ticket_life'
+
export class RequestLimitError extends Error {
readonly code = 'provider_unavailable'
- constructor(message: string) {
+ constructor(
+ message: string,
+ readonly kind: RequestLimitKind
+ ) {
super(message)
this.name = 'RequestLimitError'
}
@@ -101,7 +118,7 @@ export class RequestLimitError extends Error {
*/
export class TicketLifeError extends RequestLimitError {
constructor() {
- super('execution ticket expired while queued')
+ super('execution ticket expired while queued', 'ticket_life')
this.name = 'TicketLifeError'
}
}
@@ -113,11 +130,11 @@ export function assertBoundedDeadline(
// Without a valid maximum, Math.min below would return NaN for a valid
// request deadline instead of refusing it.
if (!Number.isInteger(maxDeadlineMs) || maxDeadlineMs <= 0) {
- throw new RequestLimitError('deadline is invalid')
+ throw new RequestLimitError('deadline is invalid', 'invalid')
}
const requested = deadlineMs ?? Math.min(maxDeadlineMs, STREAM_LIMITS.maxStreamDurationMs)
if (!Number.isFinite(requested) || !Number.isInteger(requested) || requested <= 0) {
- throw new RequestLimitError('deadline is invalid')
+ throw new RequestLimitError('deadline is invalid', 'invalid')
}
return Math.min(
requested,
@@ -130,7 +147,7 @@ export function assertBoundedDeadline(
export function assertBoundedIdleTimeout(idleTimeoutMs: number | undefined): number {
const requested = idleTimeoutMs ?? STREAM_LIMITS.upstreamIdleTimeoutMs
if (!Number.isInteger(requested) || requested <= 0) {
- throw new RequestLimitError('upstream idle timeout is invalid')
+ throw new RequestLimitError('upstream idle timeout is invalid', 'invalid')
}
return Math.min(requested, STREAM_LIMITS.upstreamIdleTimeoutMs)
}
@@ -178,9 +195,9 @@ export class StreamGate {
if (deadlineAt !== undefined && !Number.isFinite(deadlineAt)) {
throw new RangeError(`a stream gate deadline must be a finite epoch time, got ${deadlineAt}`)
}
- if (signal?.aborted) throw new RequestLimitError('stream request was aborted')
+ if (signal?.aborted) throw new RequestLimitError('stream request was aborted', 'aborted')
if (this.running >= this.maxConcurrent) {
- if (this.queued >= this.maxQueued) throw new RequestLimitError('stream queue is full')
+ if (this.queued >= this.maxQueued) throw new RequestLimitError('stream queue is full', 'queue_full')
this.queued += 1
try {
await new Promise<void>((resolve, reject) => {
@@ -192,17 +209,24 @@ export class StreamGate {
}
const onAbort = () => {
settle()
- reject(new RequestLimitError('stream request was aborted'))
- }
- const expire = () => {
- settle()
- reject(new RequestLimitError('stream queue wait exceeded'))
+ reject(new RequestLimitError('stream request was aborted', 'aborted'))
}
const queuedAt = performance.now()
const waitBoundMs =
deadlineAt === undefined
? this.maxQueueWaitMs
: Math.min(this.maxQueueWaitMs, deadlineAt - Date.now())
+ // The caller's deadline governs only when it is the nearer bound.
+ const deadlineGoverns = deadlineAt !== undefined && waitBoundMs < this.maxQueueWaitMs
+ const expire = () => {
+ settle()
+ reject(
+ new RequestLimitError(
+ 'stream queue wait exceeded',
+ deadlineGoverns ? 'deadline' : 'queue_wait'
+ )
+ )
+ }
const deadline = setTimeout(expire, Math.max(0, waitBoundMs))
const wait = () => {
// After the event loop stalls, a poll and the deadline can both be
@@ -293,25 +317,25 @@ export class BodyBudget {
if (deadlineAt !== undefined && !Number.isFinite(deadlineAt)) {
throw new RangeError(`a body admission deadline must be a finite epoch time, got ${deadlineAt}`)
}
- if (signal?.aborted) throw new RequestLimitError('body admission was aborted')
+ if (signal?.aborted) throw new RequestLimitError('body admission was aborted', 'aborted')
if (this.waiters.length === 0 && this.inFlight + bytes <= this.capacityBytes) {
return this.take(bytes)
}
if (this.waiters.length >= this.maxQueued) {
- throw new RequestLimitError('body admission queue is full')
+ throw new RequestLimitError('body admission queue is full', 'queue_full')
}
return new Promise<() => void>((resolve, reject) => {
let deadline: ReturnType<typeof setTimeout> | undefined
- const leave = (reason: string) => {
+ const leave = (reason: string, kind: RequestLimitKind) => {
const index = this.waiters.indexOf(waiter)
if (index === -1) return
this.waiters.splice(index, 1)
clearTimeout(deadline)
signal?.removeEventListener('abort', onAbort)
- reject(new RequestLimitError(reason))
+ reject(new RequestLimitError(reason, kind))
this.drain()
}
- const onAbort = () => leave('body admission was aborted')
+ const onAbort = () => leave('body admission was aborted', 'aborted')
const waiter: BodyWaiter = {
bytes,
grant: release => {
@@ -324,7 +348,7 @@ export class BodyBudget {
this.waiters.push(waiter)
if (deadlineAt !== undefined) {
deadline = setTimeout(
- () => leave('body admission wait exceeded'),
+ () => leave('body admission wait exceeded', 'deadline'),
Math.max(0, deadlineAt - Date.now())
)
}
diff --git a/grok-llm-proxy/src/server.ts b/grok-llm-proxy/src/server.ts
index bf64f8ac4..5e1af9fc8 100644
--- a/grok-llm-proxy/src/server.ts
+++ b/grok-llm-proxy/src/server.ts
@@ -595,15 +595,19 @@ export function createProxyApps(
): Promise<() => void> => {
if (deadlineAt <= Date.now()) {
if (ticket.expiresAtMs <= admissionDeadlineAt) refuseTicketLife()
- throw new RequestLimitError('admission deadline exceeded')
+ throw new RequestLimitError('admission deadline exceeded', 'deadline')
}
try {
return await acquire()
} catch (err) {
+ // The wait ended on its own deadline, and that deadline was the
+ // ticket's expiry: comparing clocks here misses a timer that fired
+ // a millisecond early.
if (
err instanceof RequestLimitError &&
+ err.kind === 'deadline' &&
!abort.signal.aborted &&
- Date.now() >= ticket.expiresAtMs
+ ticket.expiresAtMs <= admissionDeadlineAt
) {
refuseTicketLife()
}
diff --git a/packages/llm-provider-attempt-contract/index.cjs b/packages/llm-provider-attempt-contract/index.cjs
index 530d5d27f..1e702539a 100644
--- a/packages/llm-provider-attempt-contract/index.cjs
+++ b/packages/llm-provider-attempt-contract/index.cjs
@@ -53,7 +53,7 @@ const LIMITS = Object.freeze({
// maxRequestBodyBytes with about four million of them, and three such
// bodies at once exhaust a proxy capped at --max-old-space-size=384. A
// conversation needs a few thousand containers; tool results are strings
- // and never count. The A8 measurement of both proxies at this bound is in
+ // and never count. The A8 measurement at this bound is in
// codex-llm-proxy/src/requestLimits.ts. Must equal the Grok contract's
// value.
maxRequestContainers: 262144,