Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript

fix: bound request-body memory in both LLM proxies before parsing (#731)

Alfredo Lopez774e0ae92961
diff --git a/codex-llm-proxy/src/requestLimits.ts b/codex-llm-proxy/src/requestLimits.ts
index 4c06401f8..0f87e8f28 100644
--- a/codex-llm-proxy/src/requestLimits.ts
+++ b/codex-llm-proxy/src/requestLimits.ts
@@ -13,6 +13,18 @@ export const ENVELOPE_ALLOWANCE_BYTES = 16 * 1024
*/
export const DEFAULT_MAX_BODY_BYTES = CONTRACT_LIMITS.maxRequestBodyBytes + ENVELOPE_ALLOWANCE_BYTES
+/**
+ * #731 R3-2 — how many maximum-size bodies may be read and parsed at once.
+ * About five copies of a body are alive while it is parsed and hashed (the raw
+ * buffer, the decoded string, the parsed object, the contract copy and the
+ * canonical serialization), so three 8 MiB bodies hold about 120 MiB of the
+ * pod's 256Mi. Without this bound the stream gate would let 24 bodies in.
+ */
+export const IN_FLIGHT_BODY_BUDGET_BODIES = 3
+
+/** The deployed admission budget in bytes: about 24 MiB of declared bodies. */
+export const IN_FLIGHT_BODY_BUDGET_BYTES = IN_FLIGHT_BODY_BUDGET_BODIES * DEFAULT_MAX_BODY_BYTES
+
export const STREAM_LIMITS = {
maxConcurrentStreams: 8,
maxQueuedRequests: 16,
@@ -84,3 +96,84 @@ export class StreamGate {
}
export const streamGate = new StreamGate()
+
+type BodyWaiter = { bytes: number; grant: (release: () => void) => void }
+
+/**
+ * #731 R3-2 — a byte budget for request bodies, taken from the declared
+ * `Content-Length` before the body is read. Bodies that do not fit wait in a
+ * bounded FIFO queue; a full queue is refused with the same RequestLimitError
+ * the stream gate raises, so the client sees the existing overload response.
+ */
+export class BodyBudget {
+ private inFlight = 0
+ private readonly waiters: BodyWaiter[] = []
+
+ constructor(
+ private readonly capacityBytes: number,
+ private readonly maxQueued: number = STREAM_LIMITS.maxQueuedRequests
+ ) {}
+
+ get inFlightBytes(): number {
+ return this.inFlight
+ }
+
+ get queued(): number {
+ return this.waiters.length
+ }
+
+ /**
+ * Reserve `bytes` of the budget. The returned release is idempotent. When
+ * `signal` aborts while the caller is queued, the waiter is rejected and
+ * removed, and the waiters behind it are reconsidered.
+ */
+ async acquire(bytes: number, signal?: AbortSignal): Promise<() => void> {
+ if (!Number.isInteger(bytes) || bytes < 0 || bytes > this.capacityBytes) {
+ throw new RangeError(`a body of ${bytes} bytes cannot fit a budget of ${this.capacityBytes}`)
+ }
+ if (signal?.aborted) throw new RequestLimitError('body admission was aborted')
+ if (this.waiters.length === 0 && this.inFlight + bytes <= this.capacityBytes) {
+ return this.take(bytes)
+ }
+ if (this.waiters.length >= this.maxQueued) {
+ throw new RequestLimitError('body admission queue is full')
+ }
+ return new Promise<() => void>((resolve, reject) => {
+ const onAbort = () => {
+ const index = this.waiters.indexOf(waiter)
+ if (index === -1) return
+ this.waiters.splice(index, 1)
+ reject(new RequestLimitError('body admission was aborted'))
+ this.drain()
+ }
+ const waiter: BodyWaiter = {
+ bytes,
+ grant: release => {
+ signal?.removeEventListener('abort', onAbort)
+ resolve(release)
+ },
+ }
+ signal?.addEventListener('abort', onAbort, { once: true })
+ this.waiters.push(waiter)
+ })
+ }
+
+ private take(bytes: number): () => void {
+ this.inFlight += bytes
+ let released = false
+ return () => {
+ if (released) return
+ released = true
+ this.inFlight -= bytes
+ this.drain()
+ }
+ }
+
+ /** Grant queued bodies in arrival order while the head of the queue fits. */
+ private drain(): void {
+ while (this.waiters.length > 0 && this.inFlight + this.waiters[0]!.bytes <= this.capacityBytes) {
+ const waiter = this.waiters.shift()!
+ waiter.grant(this.take(waiter.bytes))
+ }
+ }
+}
diff --git a/codex-llm-proxy/src/server.ts b/codex-llm-proxy/src/server.ts
index 7b48aa2ac..a49630b56 100644
--- a/codex-llm-proxy/src/server.ts
+++ b/codex-llm-proxy/src/server.ts
@@ -1,4 +1,4 @@
-import express, { type Express, type Request, type Response } from 'express'
+import express, { type Express, type NextFunction, type Request, type Response } from 'express'
import { rateLimit } from 'express-rate-limit'
import { createServer, type Server } from 'node:http'
import { Registry, collectDefaultMetrics } from 'prom-client'
@@ -21,7 +21,12 @@ import {
OriginDeniedError,
type OriginPolicyOptions,
} from './originPolicy.js'
-import { RequestLimitError, streamGate } from './requestLimits.js'
+import {
+ BodyBudget,
+ IN_FLIGHT_BODY_BUDGET_BODIES,
+ RequestLimitError,
+ streamGate,
+} from './requestLimits.js'
const COMPLETION_KEYS = new Set(['executionTicket', 'requestHash', 'request', 'deadlineMs'])
const ADMIN_KEYS = new Set(['accessToken'])
@@ -62,6 +67,62 @@ function boundedErrorHandler(err: unknown, _req: Request, res: Response, _next:
reject(res, 500, 'internal_error')
}
+/**
+ * #731 R3-2 — memory-bounded admission, run before express.json. A body is
+ * read only once its declared `Content-Length` fits the shared byte budget, so
+ * the bodies in memory are bounded by bytes rather than by the stream gate's
+ * request count. A body of undeclared length is refused (411) instead of being
+ * read unbounded. Bodies over the limit pass through so express.json answers
+ * 413 from the header without buffering them. The reservation is released when
+ * the response closes, or the queued waiter is dropped if the client leaves
+ * first; a full queue gets the stream gate's overload response.
+ */
+function bodyAdmission(budget: BodyBudget, maxBodyBytes: number) {
+ return (req: Request, res: Response, next: NextFunction): void => {
+ if (req.headers['transfer-encoding'] !== undefined) {
+ reject(res, 411, 'length_required')
+ return
+ }
+ const declared = req.headers['content-length']
+ if (declared === undefined) {
+ next()
+ return
+ }
+ if (!/^\d+$/.test(declared)) {
+ reject(res, 400, 'invalid_request')
+ return
+ }
+ const bytes = Number(declared)
+ if (bytes === 0 || bytes > maxBodyBytes) {
+ next()
+ return
+ }
+ const abort = new AbortController()
+ let release: (() => void) | undefined
+ res.once('close', () => {
+ if (release) release()
+ else abort.abort()
+ })
+ budget.acquire(bytes, abort.signal).then(
+ granted => {
+ if (abort.signal.aborted) {
+ granted()
+ return
+ }
+ release = granted
+ next()
+ },
+ (err: unknown) => {
+ if (!(err instanceof RequestLimitError)) {
+ next(err)
+ return
+ }
+ if (!abort.signal.aborted) reject(res, 503, 'provider_unavailable')
+ }
+ )
+ }
+}
+
export type ProxyRuntimeDeps = {
controlApiClient?: ControlApiClient
fetchFn?: typeof fetch
@@ -110,7 +171,11 @@ export function createProxyApps(config: CodexLlmProxyConfig, deps: ProxyRuntimeD
legacyHeaders: false,
})
+ // One budget for both apps: every body this process reads counts against it.
+ const bodyBudget = new BodyBudget(IN_FLIGHT_BODY_BUDGET_BODIES * config.maxBodyBytes)
+
const runtimeApp = express()
+ runtimeApp.use(bodyAdmission(bodyBudget, config.maxBodyBytes))
runtimeApp.use(express.json({ limit: config.maxBodyBytes }))
runtimeApp.post('/internal/runtime/v1/codex/completions', runtimeRateLimit, (req, res) => {
if (!req.is('application/json')) {
@@ -263,6 +328,7 @@ export function createProxyApps(config: CodexLlmProxyConfig, deps: ProxyRuntimeD
runtimeApp.use(boundedErrorHandler)
const adminApp = express()
+ adminApp.use(bodyAdmission(bodyBudget, config.maxBodyBytes))
adminApp.use(express.json({ limit: config.maxBodyBytes }))
const adminHandler = (kind: 'models' | 'test') => (req: Request, res: Response) => {
if (!req.is('application/json')) {
diff --git a/grok-llm-proxy/src/requestLimits.ts b/grok-llm-proxy/src/requestLimits.ts
index f0cc6d4bd..d76a0d975 100644
--- a/grok-llm-proxy/src/requestLimits.ts
+++ b/grok-llm-proxy/src/requestLimits.ts
@@ -13,6 +13,18 @@ export const ENVELOPE_ALLOWANCE_BYTES = 16 * 1024
*/
export const DEFAULT_MAX_BODY_BYTES = CONTRACT_LIMITS.maxRequestBodyBytes + ENVELOPE_ALLOWANCE_BYTES
+/**
+ * #731 R3-2 — how many maximum-size bodies may be read and parsed at once.
+ * About five copies of a body are alive while it is parsed and hashed (the raw
+ * buffer, the decoded string, the parsed object, the contract copy and the
+ * canonical serialization), so three 8 MiB bodies hold about 120 MiB of the
+ * pod's 256Mi. Without this bound the stream gate would let 24 bodies in.
+ */
+export const IN_FLIGHT_BODY_BUDGET_BODIES = 3
+
+/** The deployed admission budget in bytes: about 24 MiB of declared bodies. */
+export const IN_FLIGHT_BODY_BUDGET_BYTES = IN_FLIGHT_BODY_BUDGET_BODIES * DEFAULT_MAX_BODY_BYTES
+
export const STREAM_LIMITS = {
maxConcurrentStreams: 8,
maxQueuedRequests: 16,
@@ -92,3 +104,84 @@ export class StreamGate {
}
export const streamGate = new StreamGate()
+
+type BodyWaiter = { bytes: number; grant: (release: () => void) => void }
+
+/**
+ * #731 R3-2 — a byte budget for request bodies, taken from the declared
+ * `Content-Length` before the body is read. Bodies that do not fit wait in a
+ * bounded FIFO queue; a full queue is refused with the same RequestLimitError
+ * the stream gate raises, so the client sees the existing overload response.
+ */
+export class BodyBudget {
+ private inFlight = 0
+ private readonly waiters: BodyWaiter[] = []
+
+ constructor(
+ private readonly capacityBytes: number,
+ private readonly maxQueued: number = STREAM_LIMITS.maxQueuedRequests
+ ) {}
+
+ get inFlightBytes(): number {
+ return this.inFlight
+ }
+
+ get queued(): number {
+ return this.waiters.length
+ }
+
+ /**
+ * Reserve `bytes` of the budget. The returned release is idempotent. When
+ * `signal` aborts while the caller is queued, the waiter is rejected and
+ * removed, and the waiters behind it are reconsidered.
+ */
+ async acquire(bytes: number, signal?: AbortSignal): Promise<() => void> {
+ if (!Number.isInteger(bytes) || bytes < 0 || bytes > this.capacityBytes) {
+ throw new RangeError(`a body of ${bytes} bytes cannot fit a budget of ${this.capacityBytes}`)
+ }
+ if (signal?.aborted) throw new RequestLimitError('body admission was aborted')
+ if (this.waiters.length === 0 && this.inFlight + bytes <= this.capacityBytes) {
+ return this.take(bytes)
+ }
+ if (this.waiters.length >= this.maxQueued) {
+ throw new RequestLimitError('body admission queue is full')
+ }
+ return new Promise<() => void>((resolve, reject) => {
+ const onAbort = () => {
+ const index = this.waiters.indexOf(waiter)
+ if (index === -1) return
+ this.waiters.splice(index, 1)
+ reject(new RequestLimitError('body admission was aborted'))
+ this.drain()
+ }
+ const waiter: BodyWaiter = {
+ bytes,
+ grant: release => {
+ signal?.removeEventListener('abort', onAbort)
+ resolve(release)
+ },
+ }
+ signal?.addEventListener('abort', onAbort, { once: true })
+ this.waiters.push(waiter)
+ })
+ }
+
+ private take(bytes: number): () => void {
+ this.inFlight += bytes
+ let released = false
+ return () => {
+ if (released) return
+ released = true
+ this.inFlight -= bytes
+ this.drain()
+ }
+ }
+
+ /** Grant queued bodies in arrival order while the head of the queue fits. */
+ private drain(): void {
+ while (this.waiters.length > 0 && this.inFlight + this.waiters[0]!.bytes <= this.capacityBytes) {
+ const waiter = this.waiters.shift()!
+ waiter.grant(this.take(waiter.bytes))
+ }
+ }
+}
diff --git a/grok-llm-proxy/src/server.ts b/grok-llm-proxy/src/server.ts
index c3f0d53df..f0b837777 100644
--- a/grok-llm-proxy/src/server.ts
+++ b/grok-llm-proxy/src/server.ts
@@ -1,4 +1,4 @@
-import express, { type Express, type Request, type Response } from 'express'
+import express, { type Express, type NextFunction, type Request, type Response } from 'express'
import { rateLimit } from 'express-rate-limit'
import { type Server, createServer } from 'node:http'
import { Registry, collectDefaultMetrics } from 'prom-client'
@@ -21,7 +21,12 @@ import {
type OriginPolicyOptions,
defaultAddressLookup,
} from './originPolicy.js'
-import { RequestLimitError, streamGate } from './requestLimits.js'
+import {
+ BodyBudget,
+ IN_FLIGHT_BODY_BUDGET_BODIES,
+ RequestLimitError,
+ streamGate,
+} from './requestLimits.js'
const COMPLETION_KEYS = new Set(['executionTicket', 'requestHash', 'request', 'deadlineMs'])
const ADMIN_KEYS = new Set(['accessToken'])
@@ -62,6 +67,62 @@ function boundedErrorHandler(err: unknown, _req: Request, res: Response, _next:
reject(res, 500, 'internal_error')
}
+/**
+ * #731 R3-2 — memory-bounded admission, run before express.json. A body is
+ * read only once its declared `Content-Length` fits the shared byte budget, so
+ * the bodies in memory are bounded by bytes rather than by the stream gate's
+ * request count. A body of undeclared length is refused (411) instead of being
+ * read unbounded. Bodies over the limit pass through so express.json answers
+ * 413 from the header without buffering them. The reservation is released when
+ * the response closes, or the queued waiter is dropped if the client leaves
+ * first; a full queue gets the stream gate's overload response.
+ */
+function bodyAdmission(budget: BodyBudget, maxBodyBytes: number) {
+ return (req: Request, res: Response, next: NextFunction): void => {
+ if (req.headers['transfer-encoding'] !== undefined) {
+ reject(res, 411, 'length_required')
+ return
+ }
+ const declared = req.headers['content-length']
+ if (declared === undefined) {
+ next()
+ return
+ }
+ if (!/^\d+$/.test(declared)) {
+ reject(res, 400, 'invalid_request')
+ return
+ }
+ const bytes = Number(declared)
+ if (bytes === 0 || bytes > maxBodyBytes) {
+ next()
+ return
+ }
+ const abort = new AbortController()
+ let release: (() => void) | undefined
+ res.once('close', () => {
+ if (release) release()
+ else abort.abort()
+ })
+ budget.acquire(bytes, abort.signal).then(
+ granted => {
+ if (abort.signal.aborted) {
+ granted()
+ return
+ }
+ release = granted
+ next()
+ },
+ (err: unknown) => {
+ if (!(err instanceof RequestLimitError)) {
+ next(err)
+ return
+ }
+ if (!abort.signal.aborted) reject(res, 503, 'provider_unavailable')
+ }
+ )
+ }
+}
+
export type ProxyRuntimeDeps = {
controlApiClient?: ControlApiClient
fetchFn?: typeof fetch
@@ -113,7 +174,11 @@ export function createProxyApps(
legacyHeaders: false,
})
+ // One budget for both apps: every body this process reads counts against it.
+ const bodyBudget = new BodyBudget(IN_FLIGHT_BODY_BUDGET_BODIES * config.maxBodyBytes)
+
const runtimeApp = express()
+ runtimeApp.use(bodyAdmission(bodyBudget, config.maxBodyBytes))
runtimeApp.use(express.json({ limit: config.maxBodyBytes }))
runtimeApp.post('/internal/runtime/v1/grok/completions', runtimeRateLimit, (req, res) => {
if (!req.is('application/json')) {
@@ -266,6 +331,7 @@ export function createProxyApps(
runtimeApp.use(boundedErrorHandler)
const adminApp = express()
+ adminApp.use(bodyAdmission(bodyBudget, config.maxBodyBytes))
adminApp.use(express.json({ limit: config.maxBodyBytes }))
const adminHandler = (kind: 'models' | 'test') => (req: Request, res: Response) => {
if (!req.is('application/json')) {