Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript

feat(grok-llm-proxy): admit V2 visual bodies through a 1-wide gate

Alfredo Lopezaf7a8f0a6b23
diff --git a/grok-llm-proxy/src/config.ts b/grok-llm-proxy/src/config.ts
index 8ca765fa6..75ff45088 100644
--- a/grok-llm-proxy/src/config.ts
+++ b/grok-llm-proxy/src/config.ts
@@ -1,3 +1,4 @@
+import { LIMITS } from '@clerum/grok-provider-attempt-contract'
import { DEFAULT_MAX_BODY_BYTES, STREAM_LIMITS } from './requestLimits.js'
import { DEFAULT_HEARTBEAT_INTERVAL_MS, MAX_HEARTBEAT_INTERVAL_MS } from './sseHeartbeat.js'
@@ -6,6 +7,7 @@ export type GrokLlmProxyConfig = {
adminPort: number
probePort: number
maxBodyBytes: number
+ maxVisualBodyBytes: number
maxStreamDurationMs: number
maxDeadlineMs: number
/** Lowers STREAM_LIMITS.upstreamIdleTimeoutMs; the transport never raises it. */
@@ -79,6 +81,16 @@ export function loadConfig(env: NodeJS.ProcessEnv = process.env): GrokLlmProxyCo
'GROK_LLM_PROXY_MAX_BODY_BYTES must be at least the contract request cap plus the envelope allowance'
)
}
+ // The visual ceiling already covers the envelope, so it takes no allowance
+ // on top; a lower limit would answer 413 to V2 envelopes the contract accepts.
+ const maxVisualBodyBytes = requiredPositiveInt(
+ 'GROK_LLM_PROXY_MAX_VISUAL_BODY_BYTES',
+ env.GROK_LLM_PROXY_MAX_VISUAL_BODY_BYTES,
+ LIMITS.maxVisualRequestBodyBytes
+ )
+ if (maxVisualBodyBytes < LIMITS.maxVisualRequestBodyBytes) {
+ throw new Error('Visual Grok requests require the full shared envelope byte budget')
+ }
return {
runtimePort: requiredPositiveInt(
'GROK_LLM_PROXY_RUNTIME_PORT',
@@ -96,6 +108,7 @@ export function loadConfig(env: NodeJS.ProcessEnv = process.env): GrokLlmProxyCo
9090
),
maxBodyBytes,
+ maxVisualBodyBytes,
maxStreamDurationMs: requiredPositiveInt(
'GROK_LLM_PROXY_MAX_STREAM_DURATION_MS',
env.GROK_LLM_PROXY_MAX_STREAM_DURATION_MS,
diff --git a/grok-llm-proxy/src/requestLimits.ts b/grok-llm-proxy/src/requestLimits.ts
index da39e4209..344fbb18c 100644
--- a/grok-llm-proxy/src/requestLimits.ts
+++ b/grok-llm-proxy/src/requestLimits.ts
@@ -20,7 +20,9 @@ export const DEFAULT_MAX_BODY_BYTES = CONTRACT_LIMITS.maxRequestBodyBytes + ENVE
* Several copies of a body are alive while it is parsed and hashed (the raw
* buffer, the decoded string, the parsed object, the contract copy and the
* canonical serialization). Without this bound the stream gate would let 24
- * bodies in. Measured with the full load the gates admit (eight 8 MiB streams
+ * bodies in.
+ * Bodies over the ordinary cap never take this budget: they are V2 visual
+ * envelopes, bounded by `visualStreamGate` instead. Measured with the full load the gates admit (eight 8 MiB streams
* and three queued 8 MiB bodies, #739 D5), the process peaked at 510 MiB of
* RSS with `--max-old-space-size=384` and at 480-511 MiB with an uncapped
* heap. That is past the former 256Mi limit, which is why the deployment sets
@@ -46,9 +48,11 @@ export const STREAM_LIMITS = {
maxQueuedRequests: 16,
maxStreamDurationMs: 1_800_000,
// Longest total time a request may spend queued in this proxy: one
- // admission clock from arrival bounds the body budget and the stream gate
- // together (#739 D1). Bounded so that queue wait + redeem + the first
- // keepalive stays below the Host HTTP client's 300 s header timeout.
+ // admission clock from arrival bounds the body budget, the visual gate and
+ // the stream gate together (#739 D1), so a visual request that waits at both
+ // gates still waits at most this long in total. Bounded so that queue wait +
+ // redeem + the first keepalive stays below the Host HTTP client's 300 s
+ // header timeout.
maxQueueWaitMs: 60_000,
// Longest silence tolerated while waiting on the upstream (response headers
// or the next SSE chunk). Same value as the Grok Build CLI default,
@@ -58,6 +62,20 @@ export const STREAM_LIMITS = {
upstreamIdleTimeoutMs: 600_000,
} as const
+/**
+ * Admission for a body whose Content-Length exceeds the ordinary cap. A V2
+ * request keeps its image bytes resident until the upstream stream ends, and
+ * a 35 MiB envelope is more than four ordinary bodies, so those requests take
+ * a 1-wide sibling of the 8-wide stream gate and keep the slot for the
+ * stream. Small bodies, including every valid V1, must not enter this gate.
+ * The 768Mi limit is sized for this slot plus the ordinary body budget;
+ * widening either one needs a new memory measurement first.
+ */
+export const VISUAL_STREAM_LIMITS = {
+ maxConcurrentStreams: 1,
+ maxQueuedRequests: 4,
+} as const
+
// Largest delay setTimeout honors; Node fires anything above it after 1 ms.
const MAX_TIMER_DELAY_MS = 2_147_483_647
@@ -207,10 +225,20 @@ export class StreamGate {
this.running = Math.max(0, this.running - 1)
}
}
+
+ /** Observable occupancy for tests. Production callers must not branch on this. */
+ snapshot(): { running: number; queued: number } {
+ return { running: this.running, queued: this.queued }
+ }
}
export const streamGate = new StreamGate()
+export const visualStreamGate = new StreamGate(
+ VISUAL_STREAM_LIMITS.maxConcurrentStreams,
+ VISUAL_STREAM_LIMITS.maxQueuedRequests
+)
+
type BodyWaiter = { bytes: number; grant: (release: () => void) => void }
/**
diff --git a/grok-llm-proxy/src/server.ts b/grok-llm-proxy/src/server.ts
index 162541b28..faf288c61 100644
--- a/grok-llm-proxy/src/server.ts
+++ b/grok-llm-proxy/src/server.ts
@@ -7,6 +7,12 @@ import express, {
} from 'express'
import { rateLimit } from 'express-rate-limit'
import { type Server, createServer } from 'node:http'
+import {
+ LIMITS,
+ SCHEMA_VERSION_V2,
+ measureNonImageCompletionBytes,
+ requestBodyLimitBytes,
+} from '@clerum/grok-provider-attempt-contract'
import { Registry, collectDefaultMetrics } from 'prom-client'
import { z } from 'zod'
import { verifyAdminPermit } from './auth/adminPermitVerifier.js'
@@ -34,11 +40,13 @@ import {
import {
BODY_READ_DEADLINE_MS,
BodyBudget,
+ ENVELOPE_ALLOWANCE_BYTES,
IN_FLIGHT_BODY_BUDGET_BODIES,
RequestLimitError,
STREAM_LIMITS,
TicketLifeError,
streamGate,
+ visualStreamGate,
} from './requestLimits.js'
import { startSseHeartbeat } from './sseHeartbeat.js'
@@ -46,7 +54,7 @@ type AdmittedRequest = Request & {
/**
* #739 D1 — the request's one admission clock, stamped by body admission:
* arrival + `STREAM_LIMITS.maxQueueWaitMs`, in epoch ms. Every wait the
- * request performs (body budget, stream gate) ends by then.
+ * request performs (body budget, visual gate, stream gate) ends by then.
*/
grokAdmissionDeadlineAt?: number
/**
@@ -60,6 +68,11 @@ type AdmittedRequest = Request & {
type GatedRequest = AdmittedRequest & {
/** Set by the platform gate before body admission (R9-M-B). */
grokPlatform?: PlatformJwtClaims
+ /**
+ * The visual gate slot a body above the ordinary cap was read under. The
+ * handler keeps it for the stream or releases it; every refusal releases it.
+ */
+ grokStreamRelease?: () => void
}
const COMPLETION_PATH = '/internal/runtime/v1/grok/completions'
@@ -82,6 +95,17 @@ function bearer(req: Request): string {
return raw.replace(/^bearer\s+/i, '').trim()
}
+function contentLengthBytes(req: Request): number | null {
+ const header = req.headers['content-length']
+ if (typeof header !== 'string' || !/^[0-9]+$/.test(header)) return null
+ const value = Number(header)
+ return Number.isSafeInteger(value) ? value : null
+}
+
+function isRecord(value: unknown): value is Record<string, unknown> {
+ return value !== null && typeof value === 'object' && !Array.isArray(value)
+}
+
function reject(res: Response, status: number, code: string): void {
if (res.headersSent) return
logger.warn({ event: 'grok_proxy_denied', code }, 'request denied')
@@ -118,8 +142,10 @@ function boundedErrorHandler(err: unknown, _req: Request, res: Response, _next:
* `Content-Length` is refused with 400. A request with neither header has no
* body under HTTP/1.1 (RFC 9112 §6.3): `parse` reads nothing, and bytes sent
* after its headers are parsed as the next request, which Node answers 400.
- * Bodies over the limit go to `parse` without a reservation, so express.json answers 413 from the header without
- * buffering them. A granted body must be read and parsed within
+ * Bodies over the limit go to `parse` without a reservation: on the admin app
+ * express.json answers 413 from the header without buffering them, and on the
+ * runtime app `selectTransportBudget` either answers 413 or admits them
+ * through the visual gate. A granted body must be read and parsed within
* `readDeadlineMs` of the grant, or it is answered 408 `request_timeout`, its
* reservation released and its connection closed. Otherwise the reservation is
* held until the upstream accepted the completion (#739 D2: the handler
@@ -213,6 +239,8 @@ export type ProxyRuntimeDeps = {
lookup?: OriginPolicyOptions['lookup']
/** Test seam for the body-read deadline. Production uses `BODY_READ_DEADLINE_MS`. */
bodyReadDeadlineMs?: number
+ /** Test seam: hang or observe a stream without contacting Grok. */
+ streamCompletion?: typeof streamGrokCompletion
}
export type ProxyServers = {
@@ -276,44 +304,131 @@ export function createProxyApps(
req.grokPlatform = platform
next()
}
- // Order: rate limit, token, body admission around the parser, handler.
- // R9-1: `inflate: false` on every parser. The budget counts the declared wire
+ // R9-1: `inflate: false` on every parser. The budgets count the declared wire
// length, so an encoded body is refused (415) instead of inflated past it.
+ const ordinaryJson = express.json({ limit: config.maxBodyBytes, inflate: false })
+ const visualJson = express.json({ limit: config.maxVisualBodyBytes, inflate: false })
+ // The parser behind body admission. Every request here carries a verified
+ // platform JWT.
+ //
+ // schemaVersion is not known until the body is parsed, so the visual gate
+ // must not be taken for every platform JWT. Only a declared Content-Length
+ // above the ordinary cap can be a visual envelope. Those bodies take the
+ // 1-wide gate (the image bytes stay resident for the Grok stream). Every
+ // smaller body, including every valid V1, stays on the ordinary parser and
+ // later the 8-wide stream gate. A missing length cannot be upgraded: body
+ // admission already refused a chunked body with 411, and a request with
+ // neither header has no body, so only a declared length takes a visual slot.
+ const selectTransportBudget = (req: GatedRequest, res: Response, next: NextFunction): void => {
+ const declared = contentLengthBytes(req)
+ if (declared !== null && declared > config.maxVisualBodyBytes) {
+ reject(res, 413, 'payload_too_large')
+ return
+ }
+ if (declared === null || declared <= config.maxBodyBytes) {
+ ordinaryJson(req, res, next)
+ return
+ }
+ void (async () => {
+ let release: (() => void) | undefined
+ const parseAbort = new AbortController()
+ const abortParse = (): void => parseAbort.abort()
+ req.once('aborted', abortParse)
+ try {
+ release = await visualStreamGate.acquire(parseAbort.signal, req.grokAdmissionDeadlineAt)
+ } catch (err) {
+ req.off('aborted', abortParse)
+ if (err instanceof RequestLimitError) {
+ reject(res, 503, 'provider_unavailable')
+ return
+ }
+ next(err)
+ return
+ }
+ req.off('aborted', abortParse)
+ req.grokStreamRelease = release
+ visualJson(req, res, err => {
+ if (err) {
+ req.grokStreamRelease?.()
+ req.grokStreamRelease = undefined
+ next(err)
+ return
+ }
+ next()
+ })
+ })()
+ }
+ // Order: rate limit, token, body admission around the transport budget,
+ // handler.
const runtimeAdmission = bodyAdmission(
bodyBudget,
config.maxBodyBytes,
bodyReadDeadlineMs,
- express.json({ limit: config.maxBodyBytes, inflate: false })
+ selectTransportBudget
)
runtimeApp.post(COMPLETION_PATH, runtimeRateLimit, platformGate, runtimeAdmission, (req, res) => {
- const platform = (req as GatedRequest).grokPlatform
+ const gated = req as GatedRequest
+ const releaseAdmission = (): void => {
+ gated.grokStreamRelease?.()
+ gated.grokStreamRelease = undefined
+ }
+ const platform = gated.grokPlatform
if (!platform) {
+ releaseAdmission()
throw new Error('the completion route was reached without the platform gate')
}
- const admissionDeadlineAt = (req as GatedRequest).grokAdmissionDeadlineAt
+ const admissionDeadlineAt = gated.grokAdmissionDeadlineAt
if (admissionDeadlineAt === undefined) {
+ releaseAdmission()
throw new Error('the completion route was reached without body admission')
}
if (!req.is('application/json')) {
+ releaseAdmission()
reject(res, 415, 'unsupported_media_type')
return
}
+ const request = isRecord(req.body) ? req.body.request : undefined
+ const visualDeclared = isRecord(request) && request.schemaVersion === SCHEMA_VERSION_V2
+ const configuredLimit = visualDeclared ? config.maxVisualBodyBytes : config.maxBodyBytes
+ const wholeBodyBytes = Buffer.byteLength(JSON.stringify(req.body ?? {}), 'utf8')
+ // A V1 envelope carries the ticket, the hash and the deadline beside a
+ // request that may itself sit at the contract cap (#731), so its limit adds
+ // the envelope allowance. A V2 envelope stays on the contract's visual
+ // ceiling, the same one buildGrokProxyEnvelope enforces.
+ const envelopeLimit = visualDeclared
+ ? requestBodyLimitBytes(request)
+ : requestBodyLimitBytes(request) + ENVELOPE_ALLOWANCE_BYTES
+ // Declaring V2 raises only the image budget. Text, tools and wrapper
+ // fields stay on the contract's maxRequestBodyBytes non-image ceiling, plus
+ // the same envelope allowance control-api's authorizer grants its wrapper.
+ if (
+ wholeBodyBytes > Math.min(configuredLimit, envelopeLimit) ||
+ measureNonImageCompletionBytes(req.body) > LIMITS.maxRequestBodyBytes + ENVELOPE_ALLOWANCE_BYTES
+ ) {
+ releaseAdmission()
+ reject(res, 413, 'payload_too_large')
+ return
+ }
const extra = Object.keys(req.body ?? {}).find(key => !COMPLETION_KEYS.has(key))
if (extra) {
+ releaseAdmission()
reject(res, 400, 'unknown_field')
return
}
const parsed = completionBodySchema.safeParse(req.body)
if (!parsed.success) {
+ releaseAdmission()
reject(res, 400, 'invalid_request')
return
}
if (parsed.data.deadlineMs !== undefined && parsed.data.deadlineMs > config.maxDeadlineMs) {
+ releaseAdmission()
reject(res, 400, 'invalid_request')
return
}
const ticket = verifyExecutionTicket(parsed.data.executionTicket, config)
if (!ticket) {
+ releaseAdmission()
// R17-2: body admission can wait up to maxQueueWaitMs, the ticket TTL,
// before this read, so an honest ticket can arrive expired. The Host
// retries ticket_expired with a fresh authorization.
@@ -325,10 +440,12 @@ export function createProxyApps(
return
}
if (platform.hostRefs.includes('*') || !platform.hostRefs.includes(ticket.hostRef)) {
+ releaseAdmission()
reject(res, 403, 'host_binding_mismatch')
return
}
if (!config.executionEnabled) {
+ releaseAdmission()
reject(res, 404, 'disabled')
return
}
@@ -340,6 +457,8 @@ export function createProxyApps(
// to req 'close' aborts the Grok hop on every call (3–12ms canceled).
// Abort only when the client drops the response before we finish writing.
abortWhenClientDisconnects(req, res, abort)
+ const visualRequest =
+ (parsed.data.request as { schemaVersion?: unknown }).schemaVersion === SCHEMA_VERSION_V2
// One `grok_proxy_attempt_finished` line per attempt. Identifiers and
// counts only: never the body, ticket, frames, tool names or arguments.
const attempt = {
@@ -356,44 +475,57 @@ export function createProxyApps(
// it ends the response: a tick after `res.end()` would write after end.
let stopHeartbeat: (() => void) | undefined
try {
- // #739 D1-bis: the wait also ends when the ticket dies. A request
- // still queued at `exp` could only be redeemed into ticket_expired,
- // so it is refused here, before any redeem. The margin is zero: a
- // ticket still alive when a slot frees is served as before.
- const deadlineAt = Math.min(admissionDeadlineAt, ticket.expiresAtMs)
- const refuseTicketLife = (): never => {
- logger.warn(
- {
- event: 'grok_proxy_admission_refused',
- reason: 'ticket_life',
- providerAttemptId: ticket.providerAttemptId,
- hostRef: ticket.hostRef,
- },
- 'admission refused'
- )
- throw new TicketLifeError()
+ // A visual slot exists only when Content-Length exceeded the ordinary
+ // cap. Keep it for the Grok stream only when the parsed V2 body still
+ // exceeds that cap (image bytes stay resident). Padding, V1, and small
+ // V2 release it and take the 8-wide gate.
+ if (visualRequest && wholeBodyBytes > config.maxBodyBytes) {
+ release = gated.grokStreamRelease
+ gated.grokStreamRelease = undefined
+ } else {
+ releaseAdmission()
}
- if (deadlineAt <= Date.now()) {
- if (ticket.expiresAtMs <= admissionDeadlineAt) refuseTicketLife()
- throw new RequestLimitError('admission deadline exceeded')
- }
- try {
- release = await streamGate.acquire(abort.signal, deadlineAt)
- } catch (err) {
- if (
- err instanceof RequestLimitError &&
- !abort.signal.aborted &&
- Date.now() >= ticket.expiresAtMs
- ) {
- refuseTicketLife()
+ if (!release) {
+ // #739 D1-bis: the wait also ends when the ticket dies. A request
+ // still queued at `exp` could only be redeemed into ticket_expired,
+ // so it is refused here, before any redeem. The margin is zero: a
+ // ticket still alive when a slot frees is served as before.
+ const deadlineAt = Math.min(admissionDeadlineAt, ticket.expiresAtMs)
+ const refuseTicketLife = (): never => {
+ logger.warn(
+ {
+ event: 'grok_proxy_admission_refused',
+ reason: 'ticket_life',
+ providerAttemptId: ticket.providerAttemptId,
+ hostRef: ticket.hostRef,
+ },
+ 'admission refused'
+ )
+ throw new TicketLifeError()
+ }
+ if (deadlineAt <= Date.now()) {
+ if (ticket.expiresAtMs <= admissionDeadlineAt) refuseTicketLife()
+ throw new RequestLimitError('admission deadline exceeded')
+ }
+ try {
+ release = await streamGate.acquire(abort.signal, deadlineAt)
+ } catch (err) {
+ if (
+ err instanceof RequestLimitError &&
+ !abort.signal.aborted &&
+ Date.now() >= ticket.expiresAtMs
+ ) {
+ refuseTicketLife()
+ }
+ throw err
}
- throw err
}
res.status(200)
res.setHeader('content-type', 'text/event-stream')
res.setHeader('cache-control', 'no-cache')
const started = Date.now()
- const result = await streamGrokCompletion({
+ const stream = deps.streamCompletion ?? streamGrokCompletion
+ const result = await stream({
executionTicket: parsed.data.executionTicket,
requestHash: parsed.data.requestHash,
request: parsed.data.request,
@@ -419,9 +551,8 @@ export function createProxyApps(
// reservation ends here rather than with the stream. The parsed
// copy the transport streams from stays; the raw one is dropped.
onUpstreamAccepted: () => {
- const admitted = req as GatedRequest
- admitted.grokBodyRelease?.()
- admitted.grokBodyRelease = undefined
+ gated.grokBodyRelease?.()
+ gated.grokBodyRelease = undefined
req.body = undefined
},
finalize: input => client.finalize(input),
@@ -641,6 +772,10 @@ const ATTEMPT_ERROR_STATUS: Record<string, number> = {
// The upstream's refusal of a prompt over the model's context window, at the
// upstream's own status class (R10).
context_length_exceeded: 400,
+ // A V2 request over one of its byte budgets (image, total or non-image
+ // share), refused by the contract before redeem, or a body over the proxy's
+ // own caps.
+ payload_too_large: 413,
Unauthorized: 401,
origin_denied: 403,
request_hash_mismatch: 403,