Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
fix: bound request-body memory in both LLM proxies before parsing (#731)
Alfredo Lopez774e0ae92961
diff --git a/codex-llm-proxy/src/requestLimits.ts b/codex-llm-proxy/src/requestLimits.tsindex 4c06401f8..0f87e8f28 100644--- a/codex-llm-proxy/src/requestLimits.ts+++ b/codex-llm-proxy/src/requestLimits.ts@@ -13,6 +13,18 @@ export const ENVELOPE_ALLOWANCE_BYTES = 16 * 1024*/export const DEFAULT_MAX_BODY_BYTES = CONTRACT_LIMITS.maxRequestBodyBytes + ENVELOPE_ALLOWANCE_BYTES+/**+ * #731 R3-2 — how many maximum-size bodies may be read and parsed at once.+ * About five copies of a body are alive while it is parsed and hashed (the raw+ * buffer, the decoded string, the parsed object, the contract copy and the+ * canonical serialization), so three 8 MiB bodies hold about 120 MiB of the+ * pod's 256Mi. Without this bound the stream gate would let 24 bodies in.+ */+export const IN_FLIGHT_BODY_BUDGET_BODIES = 3++/** The deployed admission budget in bytes: about 24 MiB of declared bodies. */+export const IN_FLIGHT_BODY_BUDGET_BYTES = IN_FLIGHT_BODY_BUDGET_BODIES * DEFAULT_MAX_BODY_BYTES+export const STREAM_LIMITS = {maxConcurrentStreams: 8,maxQueuedRequests: 16,@@ -84,3 +96,84 @@ export class StreamGate {}export const streamGate = new StreamGate()++type BodyWaiter = { bytes: number; grant: (release: () => void) => void }++/**+ * #731 R3-2 — a byte budget for request bodies, taken from the declared+ * `Content-Length` before the body is read. Bodies that do not fit wait in a+ * bounded FIFO queue; a full queue is refused with the same RequestLimitError+ * the stream gate raises, so the client sees the existing overload response.+ */+export class BodyBudget {+ private inFlight = 0+ private readonly waiters: BodyWaiter[] = []++ constructor(+ private readonly capacityBytes: number,+ private readonly maxQueued: number = STREAM_LIMITS.maxQueuedRequests+ ) {}++ get inFlightBytes(): number {+ return this.inFlight+ }++ get queued(): number {+ return this.waiters.length+ }++ /**+ * Reserve `bytes` of the budget. The returned release is idempotent. When+ * `signal` aborts while the caller is queued, the waiter is rejected and+ * removed, and the waiters behind it are reconsidered.+ */+ async acquire(bytes: number, signal?: AbortSignal): Promise<() => void> {+ if (!Number.isInteger(bytes) || bytes < 0 || bytes > this.capacityBytes) {+ throw new RangeError(`a body of ${bytes} bytes cannot fit a budget of ${this.capacityBytes}`)+ }+ if (signal?.aborted) throw new RequestLimitError('body admission was aborted')+ if (this.waiters.length === 0 && this.inFlight + bytes <= this.capacityBytes) {+ return this.take(bytes)+ }+ if (this.waiters.length >= this.maxQueued) {+ throw new RequestLimitError('body admission queue is full')+ }+ return new Promise<() => void>((resolve, reject) => {+ const onAbort = () => {+ const index = this.waiters.indexOf(waiter)+ if (index === -1) return+ this.waiters.splice(index, 1)+ reject(new RequestLimitError('body admission was aborted'))+ this.drain()+ }+ const waiter: BodyWaiter = {+ bytes,+ grant: release => {+ signal?.removeEventListener('abort', onAbort)+ resolve(release)+ },+ }+ signal?.addEventListener('abort', onAbort, { once: true })+ this.waiters.push(waiter)+ })+ }++ private take(bytes: number): () => void {+ this.inFlight += bytes+ let released = false+ return () => {+ if (released) return+ released = true+ this.inFlight -= bytes+ this.drain()+ }+ }++ /** Grant queued bodies in arrival order while the head of the queue fits. */+ private drain(): void {+ while (this.waiters.length > 0 && this.inFlight + this.waiters[0]!.bytes <= this.capacityBytes) {+ const waiter = this.waiters.shift()!+ waiter.grant(this.take(waiter.bytes))+ }+ }+}diff --git a/codex-llm-proxy/src/server.ts b/codex-llm-proxy/src/server.tsindex 7b48aa2ac..a49630b56 100644--- a/codex-llm-proxy/src/server.ts+++ b/codex-llm-proxy/src/server.ts@@ -1,4 +1,4 @@-import express, { type Express, type Request, type Response } from 'express'+import express, { type Express, type NextFunction, type Request, type Response } from 'express'import { rateLimit } from 'express-rate-limit'import { createServer, type Server } from 'node:http'import { Registry, collectDefaultMetrics } from 'prom-client'@@ -21,7 +21,12 @@ import {OriginDeniedError,type OriginPolicyOptions,} from './originPolicy.js'-import { RequestLimitError, streamGate } from './requestLimits.js'+import {+ BodyBudget,+ IN_FLIGHT_BODY_BUDGET_BODIES,+ RequestLimitError,+ streamGate,+} from './requestLimits.js'const COMPLETION_KEYS = new Set(['executionTicket', 'requestHash', 'request', 'deadlineMs'])const ADMIN_KEYS = new Set(['accessToken'])@@ -62,6 +67,62 @@ function boundedErrorHandler(err: unknown, _req: Request, res: Response, _next:reject(res, 500, 'internal_error')}+/**+ * #731 R3-2 — memory-bounded admission, run before express.json. A body is+ * read only once its declared `Content-Length` fits the shared byte budget, so+ * the bodies in memory are bounded by bytes rather than by the stream gate's+ * request count. A body of undeclared length is refused (411) instead of being+ * read unbounded. Bodies over the limit pass through so express.json answers+ * 413 from the header without buffering them. The reservation is released when+ * the response closes, or the queued waiter is dropped if the client leaves+ * first; a full queue gets the stream gate's overload response.+ */+function bodyAdmission(budget: BodyBudget, maxBodyBytes: number) {+ return (req: Request, res: Response, next: NextFunction): void => {+ if (req.headers['transfer-encoding'] !== undefined) {+ reject(res, 411, 'length_required')+ return+ }+ const declared = req.headers['content-length']+ if (declared === undefined) {+ next()+ return+ }+ if (!/^\d+$/.test(declared)) {+ reject(res, 400, 'invalid_request')+ return+ }+ const bytes = Number(declared)+ if (bytes === 0 || bytes > maxBodyBytes) {+ next()+ return+ }+ const abort = new AbortController()+ let release: (() => void) | undefined+ res.once('close', () => {+ if (release) release()+ else abort.abort()+ })+ budget.acquire(bytes, abort.signal).then(+ granted => {+ if (abort.signal.aborted) {+ granted()+ return+ }+ release = granted+ next()+ },+ (err: unknown) => {+ if (!(err instanceof RequestLimitError)) {+ next(err)+ return+ }+ if (!abort.signal.aborted) reject(res, 503, 'provider_unavailable')+ }+ )+ }+}+export type ProxyRuntimeDeps = {controlApiClient?: ControlApiClientfetchFn?: typeof fetch@@ -110,7 +171,11 @@ export function createProxyApps(config: CodexLlmProxyConfig, deps: ProxyRuntimeDlegacyHeaders: false,})+ // One budget for both apps: every body this process reads counts against it.+ const bodyBudget = new BodyBudget(IN_FLIGHT_BODY_BUDGET_BODIES * config.maxBodyBytes)+const runtimeApp = express()+ runtimeApp.use(bodyAdmission(bodyBudget, config.maxBodyBytes))runtimeApp.use(express.json({ limit: config.maxBodyBytes }))runtimeApp.post('/internal/runtime/v1/codex/completions', runtimeRateLimit, (req, res) => {if (!req.is('application/json')) {@@ -263,6 +328,7 @@ export function createProxyApps(config: CodexLlmProxyConfig, deps: ProxyRuntimeDruntimeApp.use(boundedErrorHandler)const adminApp = express()+ adminApp.use(bodyAdmission(bodyBudget, config.maxBodyBytes))adminApp.use(express.json({ limit: config.maxBodyBytes }))const adminHandler = (kind: 'models' | 'test') => (req: Request, res: Response) => {if (!req.is('application/json')) {diff --git a/grok-llm-proxy/src/requestLimits.ts b/grok-llm-proxy/src/requestLimits.tsindex f0cc6d4bd..d76a0d975 100644--- a/grok-llm-proxy/src/requestLimits.ts+++ b/grok-llm-proxy/src/requestLimits.ts@@ -13,6 +13,18 @@ export const ENVELOPE_ALLOWANCE_BYTES = 16 * 1024*/export const DEFAULT_MAX_BODY_BYTES = CONTRACT_LIMITS.maxRequestBodyBytes + ENVELOPE_ALLOWANCE_BYTES+/**+ * #731 R3-2 — how many maximum-size bodies may be read and parsed at once.+ * About five copies of a body are alive while it is parsed and hashed (the raw+ * buffer, the decoded string, the parsed object, the contract copy and the+ * canonical serialization), so three 8 MiB bodies hold about 120 MiB of the+ * pod's 256Mi. Without this bound the stream gate would let 24 bodies in.+ */+export const IN_FLIGHT_BODY_BUDGET_BODIES = 3++/** The deployed admission budget in bytes: about 24 MiB of declared bodies. */+export const IN_FLIGHT_BODY_BUDGET_BYTES = IN_FLIGHT_BODY_BUDGET_BODIES * DEFAULT_MAX_BODY_BYTES+export const STREAM_LIMITS = {maxConcurrentStreams: 8,maxQueuedRequests: 16,@@ -92,3 +104,84 @@ export class StreamGate {}export const streamGate = new StreamGate()++type BodyWaiter = { bytes: number; grant: (release: () => void) => void }++/**+ * #731 R3-2 — a byte budget for request bodies, taken from the declared+ * `Content-Length` before the body is read. Bodies that do not fit wait in a+ * bounded FIFO queue; a full queue is refused with the same RequestLimitError+ * the stream gate raises, so the client sees the existing overload response.+ */+export class BodyBudget {+ private inFlight = 0+ private readonly waiters: BodyWaiter[] = []++ constructor(+ private readonly capacityBytes: number,+ private readonly maxQueued: number = STREAM_LIMITS.maxQueuedRequests+ ) {}++ get inFlightBytes(): number {+ return this.inFlight+ }++ get queued(): number {+ return this.waiters.length+ }++ /**+ * Reserve `bytes` of the budget. The returned release is idempotent. When+ * `signal` aborts while the caller is queued, the waiter is rejected and+ * removed, and the waiters behind it are reconsidered.+ */+ async acquire(bytes: number, signal?: AbortSignal): Promise<() => void> {+ if (!Number.isInteger(bytes) || bytes < 0 || bytes > this.capacityBytes) {+ throw new RangeError(`a body of ${bytes} bytes cannot fit a budget of ${this.capacityBytes}`)+ }+ if (signal?.aborted) throw new RequestLimitError('body admission was aborted')+ if (this.waiters.length === 0 && this.inFlight + bytes <= this.capacityBytes) {+ return this.take(bytes)+ }+ if (this.waiters.length >= this.maxQueued) {+ throw new RequestLimitError('body admission queue is full')+ }+ return new Promise<() => void>((resolve, reject) => {+ const onAbort = () => {+ const index = this.waiters.indexOf(waiter)+ if (index === -1) return+ this.waiters.splice(index, 1)+ reject(new RequestLimitError('body admission was aborted'))+ this.drain()+ }+ const waiter: BodyWaiter = {+ bytes,+ grant: release => {+ signal?.removeEventListener('abort', onAbort)+ resolve(release)+ },+ }+ signal?.addEventListener('abort', onAbort, { once: true })+ this.waiters.push(waiter)+ })+ }++ private take(bytes: number): () => void {+ this.inFlight += bytes+ let released = false+ return () => {+ if (released) return+ released = true+ this.inFlight -= bytes+ this.drain()+ }+ }++ /** Grant queued bodies in arrival order while the head of the queue fits. */+ private drain(): void {+ while (this.waiters.length > 0 && this.inFlight + this.waiters[0]!.bytes <= this.capacityBytes) {+ const waiter = this.waiters.shift()!+ waiter.grant(this.take(waiter.bytes))+ }+ }+}diff --git a/grok-llm-proxy/src/server.ts b/grok-llm-proxy/src/server.tsindex c3f0d53df..f0b837777 100644--- a/grok-llm-proxy/src/server.ts+++ b/grok-llm-proxy/src/server.ts@@ -1,4 +1,4 @@-import express, { type Express, type Request, type Response } from 'express'+import express, { type Express, type NextFunction, type Request, type Response } from 'express'import { rateLimit } from 'express-rate-limit'import { type Server, createServer } from 'node:http'import { Registry, collectDefaultMetrics } from 'prom-client'@@ -21,7 +21,12 @@ import {type OriginPolicyOptions,defaultAddressLookup,} from './originPolicy.js'-import { RequestLimitError, streamGate } from './requestLimits.js'+import {+ BodyBudget,+ IN_FLIGHT_BODY_BUDGET_BODIES,+ RequestLimitError,+ streamGate,+} from './requestLimits.js'const COMPLETION_KEYS = new Set(['executionTicket', 'requestHash', 'request', 'deadlineMs'])const ADMIN_KEYS = new Set(['accessToken'])@@ -62,6 +67,62 @@ function boundedErrorHandler(err: unknown, _req: Request, res: Response, _next:reject(res, 500, 'internal_error')}+/**+ * #731 R3-2 — memory-bounded admission, run before express.json. A body is+ * read only once its declared `Content-Length` fits the shared byte budget, so+ * the bodies in memory are bounded by bytes rather than by the stream gate's+ * request count. A body of undeclared length is refused (411) instead of being+ * read unbounded. Bodies over the limit pass through so express.json answers+ * 413 from the header without buffering them. The reservation is released when+ * the response closes, or the queued waiter is dropped if the client leaves+ * first; a full queue gets the stream gate's overload response.+ */+function bodyAdmission(budget: BodyBudget, maxBodyBytes: number) {+ return (req: Request, res: Response, next: NextFunction): void => {+ if (req.headers['transfer-encoding'] !== undefined) {+ reject(res, 411, 'length_required')+ return+ }+ const declared = req.headers['content-length']+ if (declared === undefined) {+ next()+ return+ }+ if (!/^\d+$/.test(declared)) {+ reject(res, 400, 'invalid_request')+ return+ }+ const bytes = Number(declared)+ if (bytes === 0 || bytes > maxBodyBytes) {+ next()+ return+ }+ const abort = new AbortController()+ let release: (() => void) | undefined+ res.once('close', () => {+ if (release) release()+ else abort.abort()+ })+ budget.acquire(bytes, abort.signal).then(+ granted => {+ if (abort.signal.aborted) {+ granted()+ return+ }+ release = granted+ next()+ },+ (err: unknown) => {+ if (!(err instanceof RequestLimitError)) {+ next(err)+ return+ }+ if (!abort.signal.aborted) reject(res, 503, 'provider_unavailable')+ }+ )+ }+}+export type ProxyRuntimeDeps = {controlApiClient?: ControlApiClientfetchFn?: typeof fetch@@ -113,7 +174,11 @@ export function createProxyApps(legacyHeaders: false,})+ // One budget for both apps: every body this process reads counts against it.+ const bodyBudget = new BodyBudget(IN_FLIGHT_BODY_BUDGET_BODIES * config.maxBodyBytes)+const runtimeApp = express()+ runtimeApp.use(bodyAdmission(bodyBudget, config.maxBodyBytes))runtimeApp.use(express.json({ limit: config.maxBodyBytes }))runtimeApp.post('/internal/runtime/v1/grok/completions', runtimeRateLimit, (req, res) => {if (!req.is('application/json')) {@@ -266,6 +331,7 @@ export function createProxyApps(runtimeApp.use(boundedErrorHandler)const adminApp = express()+ adminApp.use(bodyAdmission(bodyBudget, config.maxBodyBytes))adminApp.use(express.json({ limit: config.maxBodyBytes }))const adminHandler = (kind: 'models' | 'test') => (req: Request, res: Response) => {if (!req.is('application/json')) {