Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript

fix(grok-llm-proxy): scan raw bodies before parse and budget demoted bodies

Alfredo Lopeza548dc053469
diff --git a/grok-llm-proxy/src/server.ts b/grok-llm-proxy/src/server.ts
index dd4b94302..bf64f8ac4 100644
--- a/grok-llm-proxy/src/server.ts
+++ b/grok-llm-proxy/src/server.ts
@@ -8,8 +8,10 @@ import express, {
import { rateLimit } from 'express-rate-limit'
import { type Server, createServer } from 'node:http'
import {
+ BODY_STRUCTURE_LIMITS,
LIMITS,
SCHEMA_VERSION_V2,
+ createBodyStructureVerify,
measureNonImageCompletionBytes,
requestBodyLimitBytes,
} from '@clerum/grok-provider-attempt-contract'
@@ -139,23 +141,38 @@ function reject(res: Response, status: number, code: string): void {
res.status(status).json({ error: code })
}
+// A8: every JSON parser scans the raw body before JSON.parse. JSON.parse
+// allocates one heap object per container, so a body within the byte limit
+// can still exhaust the heap; the scan refuses a body denser, more nested or
+// with more containers than any request the contract accepts.
+const verifyBodyStructure = createBodyStructureVerify(BODY_STRUCTURE_LIMITS)
+
function boundedErrorHandler(err: unknown, _req: Request, res: Response, _next: () => void): void {
- const typed = err as { type?: string; status?: number }
+ const typed = err as { type?: string; status?: number; body?: unknown }
if (typed?.type === 'entity.too.large' || typed?.status === 413) {
reject(res, 413, 'payload_too_large')
return
}
// R9-1: the parsers run with `inflate: false`, so body-parser refuses an
// encoded body before reading it.
- if (typed?.type === 'encoding.unsupported') {
+ if (typed?.type === 'encoding.unsupported' || typed?.type === 'charset.unsupported') {
reject(res, 415, 'unsupported_media_type')
return
}
- if (err instanceof SyntaxError) {
+ if (err instanceof SyntaxError || typed?.type === 'body.structure.too.deep') {
reject(res, 400, 'invalid_request')
return
}
- logger.error({ event: 'grok_proxy_error', err }, 'unhandled request error')
+ // body-parser attaches the raw body to an error thrown by `verify` or by
+ // JSON.parse. Such an error is logged by its type and status only.
+ if (typed?.body !== undefined) {
+ logger.error(
+ { event: 'grok_proxy_error', type: typed.type, status: typed.status },
+ 'unhandled request error'
+ )
+ } else {
+ logger.error({ event: 'grok_proxy_error', err }, 'unhandled request error')
+ }
reject(res, 500, 'internal_error')
}
@@ -333,8 +350,16 @@ export function createProxyApps(
}
// R9-1: `inflate: false` on every parser. The budgets count the declared wire
// length, so an encoded body is refused (415) instead of inflated past it.
- const ordinaryJson = express.json({ limit: config.maxBodyBytes, inflate: false })
- const visualJson = express.json({ limit: config.maxVisualBodyBytes, inflate: false })
+ const ordinaryJson = express.json({
+ limit: config.maxBodyBytes,
+ inflate: false,
+ verify: verifyBodyStructure,
+ })
+ const visualJson = express.json({
+ limit: config.maxVisualBodyBytes,
+ inflate: false,
+ verify: verifyBodyStructure,
+ })
// The parser behind body admission. Every request here carries a verified
// platform JWT.
//
@@ -548,36 +573,32 @@ export function createProxyApps(
// cap. Keep it for the Grok stream only when the parsed V2 body still
// exceeds that cap (image bytes stay resident). Padding, V1, and small
// V2 release it and take the 8-wide gate.
- if (visualRequest && wholeBodyBytes > config.maxBodyBytes) {
- release = gated.grokStreamRelease
- gated.grokStreamRelease = undefined
- } else {
- releaseAdmission()
+ // #739 D1-bis: every admission wait also ends when the ticket dies. A
+ // request still queued at `exp` could only be redeemed into
+ // ticket_expired, so it is refused here, before any redeem. The margin
+ // is zero: a ticket still alive when a slot frees is served as before.
+ const deadlineAt = Math.min(admissionDeadlineAt, ticket.expiresAtMs)
+ const refuseTicketLife = (): never => {
+ logger.warn(
+ {
+ event: 'grok_proxy_admission_refused',
+ reason: 'ticket_life',
+ providerAttemptId: ticket.providerAttemptId,
+ hostRef: ticket.hostRef,
+ },
+ 'admission refused'
+ )
+ throw new TicketLifeError()
}
- if (!release) {
- // #739 D1-bis: the wait also ends when the ticket dies. A request
- // still queued at `exp` could only be redeemed into ticket_expired,
- // so it is refused here, before any redeem. The margin is zero: a
- // ticket still alive when a slot frees is served as before.
- const deadlineAt = Math.min(admissionDeadlineAt, ticket.expiresAtMs)
- const refuseTicketLife = (): never => {
- logger.warn(
- {
- event: 'grok_proxy_admission_refused',
- reason: 'ticket_life',
- providerAttemptId: ticket.providerAttemptId,
- hostRef: ticket.hostRef,
- },
- 'admission refused'
- )
- throw new TicketLifeError()
- }
+ const waitWithinTicketLife = async (
+ acquire: () => Promise<() => void>
+ ): Promise<() => void> => {
if (deadlineAt <= Date.now()) {
if (ticket.expiresAtMs <= admissionDeadlineAt) refuseTicketLife()
throw new RequestLimitError('admission deadline exceeded')
}
try {
- release = await streamGate.acquire(abort.signal, deadlineAt)
+ return await acquire()
} catch (err) {
if (
err instanceof RequestLimitError &&
@@ -589,6 +610,29 @@ export function createProxyApps(
throw err
}
}
+ if (visualRequest && wholeBodyBytes > config.maxBodyBytes) {
+ release = gated.grokStreamRelease
+ gated.grokStreamRelease = undefined
+ } else if (gated.grokStreamRelease) {
+ // A8 D3: a demoted body was read through the visual gate, so it holds
+ // no body-budget reservation. It takes one for its compact size
+ // before it gives up the visual slot, so the 8-wide stream gate
+ // never holds more resident bodies than the budget admits. Every
+ // request takes its locks in the same order (visual, budget,
+ // stream), so no two requests wait on each other.
+ try {
+ gated.grokBodyRelease = await waitWithinTicketLife(() =>
+ bodyBudget.acquire(wholeBodyBytes, abort.signal, deadlineAt)
+ )
+ } finally {
+ releaseAdmission()
+ }
+ } else {
+ releaseAdmission()
+ }
+ if (!release) {
+ release = await waitWithinTicketLife(() => streamGate.acquire(abort.signal, deadlineAt))
+ }
res.status(200)
res.setHeader('content-type', 'text/event-stream')
res.setHeader('cache-control', 'no-cache')
@@ -702,6 +746,10 @@ export function createProxyApps(
} finally {
stopHeartbeat?.()
release?.()
+ // A demoted body's reservation, when the stream ended before the body
+ // was written upstream. Budgeted bodies are also released on close.
+ gated.grokBodyRelease?.()
+ gated.grokBodyRelease = undefined
}
})()
})
@@ -729,7 +777,7 @@ export function createProxyApps(
bodyBudget,
config.maxBodyBytes,
bodyReadDeadlineMs,
- express.json({ limit: config.maxBodyBytes, inflate: false })
+ express.json({ limit: config.maxBodyBytes, inflate: false, verify: verifyBodyStructure })
)
const adminHandler = (kind: 'models' | 'test') => (req: Request, res: Response) => {
if (!req.is('application/json')) {