Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript

fix(llm): preserve local admission and visual ownership

Alfredo Lopez6c63f329aedd
diff --git a/codex-llm-proxy/src/requestLimits.ts b/codex-llm-proxy/src/requestLimits.ts
index 5bbf93aef..1093ab0be 100644
--- a/codex-llm-proxy/src/requestLimits.ts
+++ b/codex-llm-proxy/src/requestLimits.ts
@@ -66,6 +66,13 @@ export const IN_FLIGHT_BODY_BUDGET_BODIES = 3
*/
export const BODY_READ_DEADLINE_MS = 10_000
+/**
+ * Bounds the interval between a visual read timeout/client response close and
+ * forced request destruction. The parser's callback—not this timer—releases the
+ * visual slot and principal share.
+ */
+export const VISUAL_READ_CLOSE_GRACE_MS = 100
+
export const STREAM_LIMITS = {
maxConcurrentStreams: 8,
maxQueuedRequests: 16,
@@ -102,9 +109,10 @@ export const VISUAL_STREAM_LIMITS = {
* Fair share of the visual gate's entries — running plus queued — that one
* platform principal (`sub` plus sorted `hostRefs`) may hold at once. Above the
* share, further visual requests from that principal are refused 503
- * `provider_unavailable` with log reason `visual_host_share`, so one principal
+ * `visual_host_share`, so one principal
* cannot fill a gate that every other host still needs. The gate widths above
* are unchanged; the share only bounds how much of them one principal occupies.
+ * Its refusal is a local capacity outcome, not an upstream outage.
*/
export const VISUAL_PER_HOST_MAX_ADMITTED = 4
@@ -125,14 +133,18 @@ export type RequestLimitKind =
| 'queue_wait'
| 'ticket_life'
+export type RequestLimitCode = 'provider_unavailable' | 'proxy_capacity_exceeded' | 'visual_gate'
+
export class RequestLimitError extends Error {
- readonly code = 'provider_unavailable'
+ readonly code: RequestLimitCode
constructor(
message: string,
- readonly kind: RequestLimitKind
+ readonly kind: RequestLimitKind,
+ code: RequestLimitCode = 'provider_unavailable'
) {
super(message)
this.name = 'RequestLimitError'
+ this.code = code
}
}
@@ -186,7 +198,8 @@ export class StreamGate {
constructor(
private readonly maxConcurrent: number = STREAM_LIMITS.maxConcurrentStreams,
private readonly maxQueued: number = STREAM_LIMITS.maxQueuedRequests,
- private readonly maxQueueWaitMs: number = STREAM_LIMITS.maxQueueWaitMs
+ private readonly maxQueueWaitMs: number = STREAM_LIMITS.maxQueueWaitMs,
+ private readonly queueCapacityCode: RequestLimitCode = 'proxy_capacity_exceeded'
) {
// A NaN or fractional size would let every caller through or queue
// without bound, because the comparisons below would never hold.
@@ -229,7 +242,7 @@ export class StreamGate {
if (signal?.aborted) throw new RequestLimitError('stream request was aborted', 'aborted')
if (this.running >= this.maxConcurrent) {
if (this.queued >= this.maxQueued)
- throw new RequestLimitError('stream queue is full', 'queue_full')
+ throw new RequestLimitError('stream queue is full', 'queue_full', this.queueCapacityCode)
this.queued += 1
try {
await new Promise<void>((resolve, reject) => {
@@ -255,7 +268,8 @@ export class StreamGate {
reject(
new RequestLimitError(
'stream queue wait exceeded',
- deadlineGoverns ? 'deadline' : 'queue_wait'
+ deadlineGoverns ? 'deadline' : 'queue_wait',
+ deadlineGoverns ? 'provider_unavailable' : this.queueCapacityCode
)
)
}
@@ -297,7 +311,9 @@ export const streamGate = new StreamGate()
export const visualStreamGate = new StreamGate(
VISUAL_STREAM_LIMITS.maxConcurrentStreams,
- VISUAL_STREAM_LIMITS.maxQueuedRequests
+ VISUAL_STREAM_LIMITS.maxQueuedRequests,
+ STREAM_LIMITS.maxQueueWaitMs,
+ 'visual_gate'
)
type BodyWaiter = { bytes: number; grant: (release: () => void) => void }
@@ -356,7 +372,11 @@ export class BodyBudget {
return this.take(bytes)
}
if (this.waiters.length >= this.maxQueued) {
- throw new RequestLimitError('body admission queue is full', 'queue_full')
+ throw new RequestLimitError(
+ 'body admission queue is full',
+ 'queue_full',
+ 'proxy_capacity_exceeded'
+ )
}
return new Promise<() => void>((resolve, reject) => {
let deadline: ReturnType<typeof setTimeout> | undefined
diff --git a/codex-llm-proxy/src/server.ts b/codex-llm-proxy/src/server.ts
index 3e226698c..5dc5ec2d3 100644
--- a/codex-llm-proxy/src/server.ts
+++ b/codex-llm-proxy/src/server.ts
@@ -46,6 +46,7 @@ import {
STREAM_LIMITS,
TicketLifeError,
VISUAL_PER_HOST_MAX_ADMITTED,
+ VISUAL_READ_CLOSE_GRACE_MS,
streamGate,
visualStreamGate,
} from './requestLimits.js'
@@ -112,7 +113,9 @@ function isRecord(value: unknown): value is Record<string, unknown> {
function reject(res: Response, status: number, code: string, type?: string): void {
if (res.headersSent) return
logger.warn(
- type === undefined ? { event: 'codex_proxy_denied', code } : { event: 'codex_proxy_denied', code, type },
+ type === undefined
+ ? { event: 'codex_proxy_denied', code }
+ : { event: 'codex_proxy_denied', code, type },
'request denied'
)
res.status(status).json({ error: code })
@@ -267,7 +270,7 @@ function bodyAdmission(
},
'admission refused'
)
- reject(res, 503, 'provider_unavailable')
+ reject(res, 503, err.code)
}
)
}
@@ -287,6 +290,8 @@ export type ProxyRuntimeDeps = {
streamCompletion?: typeof streamCodexCompletion
/** Test seam for the body-read deadline. Production uses `BODY_READ_DEADLINE_MS`. */
bodyReadDeadlineMs?: number
+ /** Test seam for the bounded visual-parser close backstop. */
+ visualReadCloseGraceMs?: number
}
export type ProxyServers = {
@@ -334,9 +339,16 @@ export function createProxyApps(
// One budget for both apps: every body this process reads counts against it.
const bodyBudget = new BodyBudget(IN_FLIGHT_BODY_BUDGET_BODIES * config.maxBodyBytes)
const bodyReadDeadlineMs = deps.bodyReadDeadlineMs ?? BODY_READ_DEADLINE_MS
+ const visualReadCloseGraceMs = deps.visualReadCloseGraceMs ?? VISUAL_READ_CLOSE_GRACE_MS
+ for (const [name, value] of Object.entries({ bodyReadDeadlineMs, visualReadCloseGraceMs })) {
+ if (!Number.isSafeInteger(value) || value <= 0 || value > 2_147_483_647) {
+ throw new RangeError(`${name} must be a positive signed-32-bit integer`)
+ }
+ }
// Per-principal occupancy of the visual gate (running plus queued entries).
// Keyed on the verified platform principal: sub plus sorted hostRefs.
const visualPrincipalAdmissions = new Map<string, number>()
+ const visualRequestOwners = new WeakSet<GatedRequest>()
const runtimeApp = express()
// R9-1: `inflate: false` on every parser. The budgets count the declared wire
@@ -371,6 +383,9 @@ export function createProxyApps(
// The parser behind body admission. Every request here carries a verified
// platform JWT.
//
+ // A granted visual slot is released only after parser completion (or by the
+ // route's stream finally); a response close starts a bounded destruction
+ // backstop instead of freeing a live parser owner.
// schemaVersion is not known until the body is parsed, so the visual gate
// must not be taken for every platform JWT. Only a declared Content-Length
// above the ordinary cap can be a visual envelope. Those bodies take the
@@ -412,7 +427,7 @@ export function createProxyApps(
},
'admission refused'
)
- reject(res, 503, 'provider_unavailable')
+ reject(res, 503, 'visual_host_share')
return
}
visualPrincipalAdmissions.set(visualPrincipal, admittedByPrincipal + 1)
@@ -474,38 +489,55 @@ export function createProxyApps(
req.codexStreamRelease = undefined
}
req.codexStreamRelease = releaseVisual
- let expired = false
+ let cancelled = false
+ let parserSettled = false
+ let closeGrace: ReturnType<typeof setTimeout> | undefined
+ const cancelParser = (): void => {
+ if (parserSettled || closeGrace !== undefined) return
+ closeGrace = setTimeout(() => {
+ if (!parserSettled) req.destroy()
+ }, visualReadCloseGraceMs)
+ }
const readDeadline = setTimeout(() => {
- expired = true
- releaseVisual()
+ cancelled = true
+ cancelParser()
if (res.headersSent) return
// The rest of the body is never read, so the connection cannot be reused.
res.setHeader('connection', 'close')
reject(res, 408, 'request_timeout')
}, bodyReadDeadlineMs)
- res.once('close', () => {
+ const onResponseClose = (): void => {
+ cancelled = true
clearTimeout(readDeadline)
- // The close handler releases only while the slot is still held here. A kept
- // visual body transfers `req.codexStreamRelease` into the handler,
- // which clears the field and releases in its outer `finally`. A
- // demoted body releases through `releaseAdmission` before it joins
- // the ordinary stream gate; this hook may still fire after demotion
- // but finds the field empty and no-ops. Releasing here after hand-off
- // would free the visual slot and the principal share while the
- // handler still holds the parsed body during its (up to ~30 s)
- // unwind, admitting another large body over the gate width.
- if (req.codexStreamRelease) releaseVisual()
- })
+ cancelParser()
+ }
+ res.once('close', onResponseClose)
visualJson(req, res, err => {
+ parserSettled = true
clearTimeout(readDeadline)
- // The 408 already answered this request; the parser's late error
- // (the body aborted by the closed connection) has no one to reach.
- if (expired) return
+ if (closeGrace !== undefined) clearTimeout(closeGrace)
+ res.off('close', onResponseClose)
+ if (cancelled || err) {
+ req.body = undefined
+ if (err !== null && typeof err === 'object' && 'body' in err) {
+ delete (err as { body?: unknown }).body
+ }
+ }
+ // Capacity remains charged until body-parser has actually unwound. On
+ // success, `req.codexStreamRelease` transfers ownership to the handler,
+ // which releases it in its outer `finally`; refusals release here.
+ if (cancelled || err) releaseVisual()
+ if (cancelled) return
if (err) {
- releaseVisual()
next(err)
return
}
+ res.once('close', () => {
+ // A synchronous handler throw can happen before its async owner
+ // installs cancellation and a finally block. Once that owner has
+ // taken over, response close must not release its live work.
+ if (!visualRequestOwners.has(req)) releaseVisual()
+ })
next()
})
})()
@@ -611,6 +643,7 @@ export function createProxyApps(
// to req 'close' aborts the ChatGPT hop on every call (3–12ms canceled).
// Abort only when the client drops the response before we finish writing.
abortWhenClientDisconnects(req, res, abort)
+ visualRequestOwners.add(gated)
// One `codex_proxy_attempt_finished` line per attempt. Identifiers and
// counts only: never the body, ticket, frames, tool names or arguments.
const attempt = {
@@ -770,9 +803,9 @@ export function createProxyApps(
stopHeartbeat?.()
const mapped = mapError(err)
metrics.observeAttempt('error', 'completion_stream')
- // A request limit answers provider_unavailable on the wire; its own
- // label keeps it apart from real upstream outages in the metric. A
- // ticket that died in the queue is counted as provider_unavailable.
+ // Queue capacity has a terminal local wire code; the request_limit
+ // metric also includes abort/deadline kinds that keep their existing
+ // wire semantics. Ticket expiry remains provider_unavailable.
metrics.observeAttemptFailure(
err instanceof RequestLimitError && !(err instanceof TicketLifeError)
? 'request_limit'
@@ -780,8 +813,7 @@ export function createProxyApps(
)
if (err instanceof UpstreamTimeoutError) metrics.observeUpstreamTimeout(err.kind)
const deliveredAs = res.headersSent ? 'sse_error' : 'http_status'
- const causeCode =
- err instanceof ControlApiClientError ? err.causeCode : fetchCauseCode(err)
+ const causeCode = err instanceof ControlApiClientError ? err.causeCode : fetchCauseCode(err)
logger.warn(
{
event: 'codex_proxy_attempt_finished',
@@ -817,7 +849,9 @@ export function createProxyApps(
: undefined
const rejectedBy = typeof upstreamStatus === 'number' ? { upstreamStatus } : {}
if (deliveredAs === 'sse_error') {
- res.write(`data: ${JSON.stringify({ type: 'error', code: mapped.code, ...rejectedBy })}\n\n`)
+ res.write(
+ `data: ${JSON.stringify({ type: 'error', code: mapped.code, ...rejectedBy })}\n\n`
+ )
res.end()
return
}
@@ -835,6 +869,10 @@ export function createProxyApps(
} finally {
stopHeartbeat?.()
release?.()
+ // Defence in depth for an early throw after this async owner took over
+ // but before the visual release was moved into `release`.
+ gated.codexStreamRelease?.()
+ gated.codexStreamRelease = undefined
// A demoted body's reservation, when the stream ended before the body
// was written upstream. Budgeted bodies are also released on close.
gated.codexBodyRelease?.()
@@ -1005,6 +1043,9 @@ const ATTEMPT_ERROR_STATUS: Record<string, number> = {
// would spend the same budget again, so it is a gateway timeout, not 503.
stream_duration_exceeded: 504,
connection_unavailable: 503,
+ visual_host_share: 503,
+ visual_gate: 503,
+ proxy_capacity_exceeded: 503,
provider_unavailable: 503,
sse_buffer_exceeded: 503,
invalid_receipt: 503,
@@ -1045,7 +1086,6 @@ function isMappedError(err: unknown): err is MappedError {
function mapError(err: unknown): { status: number; code: string } {
if (!isMappedError(err)) return { status: 503, code: 'provider_unavailable' }
if (err instanceof OriginDeniedError) return { status: 403, code: 'origin_denied' }
- if (err instanceof RequestLimitError) return { status: 503, code: 'provider_unavailable' }
return { status: attemptErrorStatus(err.code), code: err.code }
}
diff --git a/codex-llm-proxy/test/bodyAdmission.test.ts b/codex-llm-proxy/test/bodyAdmission.test.ts
index cb1bf9675..fd0fa06b2 100644
--- a/codex-llm-proxy/test/bodyAdmission.test.ts
+++ b/codex-llm-proxy/test/bodyAdmission.test.ts
@@ -355,7 +355,7 @@ describe('codex-llm-proxy body admission (#731 R3-2)', () => {
}
}, 30_000)
- it('T-R3-2c answers provider_unavailable once the admission queue is full', async () => {
+ it('T-R3-2c answers proxy_capacity_exceeded once the admission queue is full', async () => {
// A small body limit keeps this cheap; the budget scales with it.
const maxBodyBytes = 16 * 1024
const proxy = await heldProxy(maxBodyBytes)
@@ -375,16 +375,16 @@ describe('codex-llm-proxy body admission (#731 R3-2)', () => {
const overflow = await post(proxy.port, payload('overflow'))
// Witnesses: the refusal was answered, and the admission logged why.
expect(overflow.status).toBe(503)
- expect(JSON.parse(overflow.body)).toEqual({ error: 'provider_unavailable' })
+ expect(JSON.parse(overflow.body)).toEqual({ error: 'proxy_capacity_exceeded' })
const logged = warn.mock.calls.map(call => call[0] as unknown as Record<string, unknown>)
expect(logged.filter(entry => entry?.event === 'codex_proxy_denied')).toEqual([
- { event: 'codex_proxy_denied', code: 'provider_unavailable' },
+ { event: 'codex_proxy_denied', code: 'proxy_capacity_exceeded' },
])
expect(logged.filter(entry => entry?.event === 'codex_proxy_admission_refused')).toEqual([
{
event: 'codex_proxy_admission_refused',
reason: 'body_budget',
- code: 'provider_unavailable',
+ code: 'proxy_capacity_exceeded',
kind: 'queue_full',
detail: expect.any(String),
},
diff --git a/codex-llm-proxy/test/server.security.test.ts b/codex-llm-proxy/test/server.security.test.ts
index ba6407aa2..bbcf213c6 100644
--- a/codex-llm-proxy/test/server.security.test.ts
+++ b/codex-llm-proxy/test/server.security.test.ts
@@ -1,9 +1,9 @@
+import { describe, expect, it, vi } from 'vitest'
+import jwt from 'jsonwebtoken'
import { generateKeyPairSync } from 'node:crypto'
import { createServer, request as httpRequest } from 'node:http'
import type { AddressInfo } from 'node:net'
-import jwt from 'jsonwebtoken'
import request from 'supertest'
-import { describe, expect, it, vi } from 'vitest'
import {
ENVELOPE_ALLOWANCE_BYTES as CONTRACT_ENVELOPE_ALLOWANCE_BYTES,
LIMITS,
@@ -15,7 +15,7 @@ import {
import { verifyAdminPermit } from '../src/auth/adminPermitVerifier.js'
import { verifyExecutionTicket } from '../src/auth/executionTicketVerifier.js'
import { CodexTransportError } from '../src/codexTransport.js'
-import { loadConfig, type CodexLlmProxyConfig } from '../src/config.js'
+import { type CodexLlmProxyConfig, loadConfig } from '../src/config.js'
import {
ControlApiClient,
ControlApiClientError,
@@ -184,9 +184,7 @@ describe('codex-llm-proxy security surface', () => {
})
it('reserves the larger transport budget for authenticated visual requests', async () => {
- const { runtimeApp, adminApp } = createProxyApps(
- config({ maxBodyBytes: 1_048_576 })
- )
+ const { runtimeApp, adminApp } = createProxyApps(config({ maxBodyBytes: 1_048_576 }))
// Deliberately invalid ticket: this test checks parser admission and the
// unchanged ticket gate, without redeeming or contacting any model.
const payload = {
@@ -266,7 +264,9 @@ describe('codex-llm-proxy security surface', () => {
try {
const visual = await send('x'.repeat(8_000_000), {
schemaVersion: 'codex-completion-request.v2',
- messages: [{ role: 'user', contentParts: [{ type: 'image', data: 'A'.repeat(1024 * 1024) }] }],
+ messages: [
+ { role: 'user', contentParts: [{ type: 'image', data: 'A'.repeat(1024 * 1024) }] },
+ ],
})
expect(visual.status).toBe(400)
expect(visual.body.error).toBe('invalid_request')
@@ -304,7 +304,10 @@ describe('codex-llm-proxy security surface', () => {
const { runtimeApp, probeApp } = createProxyApps(config({ maxBodyBytes: 65_536 }), {
streamCompletion: async () => {
streamCalls += 1
- throw new CodexTransportError('payload_too_large', 'request exceeds maxVisualRequestBodyBytes')
+ throw new CodexTransportError(
+ 'payload_too_large',
+ 'request exceeds maxVisualRequestBodyBytes'
+ )
},
})
const res = await request(runtimeApp)
@@ -316,7 +319,9 @@ describe('codex-llm-proxy security surface', () => {
expect(res.status).toBe(413)
expect(res.body).toEqual({ error: 'payload_too_large' })
const metricsText = (await request(probeApp).get('/metrics')).text
- expect(metricsText).toMatch(/^codex_proxy_attempt_failures_total\{code="payload_too_large"\} 1$/m)
+ expect(metricsText).toMatch(
+ /^codex_proxy_attempt_failures_total\{code="payload_too_large"\} 1$/m
+ )
})
it('does not let a V2 declaration raise the non-image budget to 24 MiB', async () => {
@@ -392,9 +397,7 @@ describe('codex-llm-proxy security surface', () => {
}, 60_000)
it('rate limits the completion endpoint before body parsing and authorization', async () => {
- const { runtimeApp } = createProxyApps(
- config({ maxBodyBytes: 1024 })
- )
+ const { runtimeApp } = createProxyApps(config({ maxBodyBytes: 1024 }))
const completion = () => request(runtimeApp).post('/internal/runtime/v1/codex/completions')
const oversized = {
executionTicket: 'invalid-ticket',
@@ -918,17 +921,21 @@ describe('codex-llm-proxy startup config', () => {
// accepts, so it is refused at startup, as the visual override is.
it('T-R9-3 refuses a body limit below the contract request cap plus the envelope allowance', () => {
const floor = LIMITS.maxRequestBodyBytes + ENVELOPE_ALLOWANCE_BYTES
- expect(() => loadConfig({ ...base, CODEX_LLM_PROXY_MAX_BODY_BYTES: String(floor - 1) })).toThrow(
+ expect(() =>
+ loadConfig({ ...base, CODEX_LLM_PROXY_MAX_BODY_BYTES: String(floor - 1) })
+ ).toThrow(
'CODEX_LLM_PROXY_MAX_BODY_BYTES must be at least the contract request cap plus the envelope allowance'
)
expect(() =>
loadConfig({ ...base, CODEX_LLM_PROXY_MAX_BODY_BYTES: String(LIMITS.maxRequestBodyBytes) })
).toThrow(/CODEX_LLM_PROXY_MAX_BODY_BYTES must be at least/)
// Witness: the floor itself and any larger value load.
- expect(loadConfig({ ...base, CODEX_LLM_PROXY_MAX_BODY_BYTES: String(floor) }).maxBodyBytes).toBe(floor)
- expect(loadConfig({ ...base, CODEX_LLM_PROXY_MAX_BODY_BYTES: String(floor + 1) }).maxBodyBytes).toBe(
- floor + 1
- )
+ expect(
+ loadConfig({ ...base, CODEX_LLM_PROXY_MAX_BODY_BYTES: String(floor) }).maxBodyBytes
+ ).toBe(floor)
+ expect(
+ loadConfig({ ...base, CODEX_LLM_PROXY_MAX_BODY_BYTES: String(floor + 1) }).maxBodyBytes
+ ).toBe(floor + 1)
})
it('T-R2-6c does not refuse a request at the contract cap with a real ticket as payload_too_large', async () => {
@@ -945,14 +952,11 @@ describe('codex-llm-proxy startup config', () => {
expect(typeof res.body.error).toBe('string')
})
- it.each([undefined, '', ' '])(
- 'fails at startup when the control-api URL is %j',
- value => {
- expect(() => loadConfig({ ...base, CODEX_LLM_PROXY_CONTROL_API_URL: value })).toThrow(
- /CODEX_LLM_PROXY_CONTROL_API_URL/
- )
- }
- )
+ it.each([undefined, '', ' '])('fails at startup when the control-api URL is %j', value => {
+ expect(() => loadConfig({ ...base, CODEX_LLM_PROXY_CONTROL_API_URL: value })).toThrow(
+ /CODEX_LLM_PROXY_CONTROL_API_URL/
+ )
+ })
it.each(['not a url', 'ftp://control-api/api/v1', 'file:///etc/passwd'])(
'fails at startup when the control-api URL %j is not http(s)',
@@ -1178,7 +1182,10 @@ describe('codex-llm-proxy attempt telemetry', () => {
// Denies the redeem after `delayMs`, which is longer than the heartbeat
// interval the caller configures.
- function slowDenyingClient(code: string, delayMs: number): {
+ function slowDenyingClient(
+ code: string,
+ delayMs: number
+ ): {
client: ControlApiClient
redeemCalls: () => number
} {
@@ -1215,7 +1222,8 @@ describe('codex-llm-proxy attempt telemetry', () => {
const warn = vi.spyOn(logger, 'warn')
const { client, receipts } = grantingClient(options.maxStreamDurationMs)
const apps = createProxyApps(config({ maxBodyBytes: 65_536, ...options.configOverrides }), {
- controlApiClient: options.controlApiClient ?? (deniedCode ? denyingClient(deniedCode) : client),
+ controlApiClient:
+ options.controlApiClient ?? (deniedCode ? denyingClient(deniedCode) : client),
fetchFn: options.fetchFn ?? upstream(textDeltas, calls, options.terminal),
lookup: options.lookup ?? lookup,
})
@@ -1699,6 +1707,37 @@ describe('codex-llm-proxy attempt telemetry', () => {
expect('err' in (lines[0] ?? {})).toBe(false)
})
+ it.each(['visual_host_share', 'visual_gate', 'proxy_capacity_exceeded'])(
+ 'normalizes real transport upstream 503 claiming %s to provider_unavailable',
+ async code => {
+ const fetchFn = vi
+ .fn<typeof fetch>()
+ .mockResolvedValue(
+ Response.json({ error: code, reason: 'local capacity claim' }, { status: 503 })
+ )
+ const { res, receipts, lines, metricsText } = await run(
+ 'att-spoofed-local',
+ 0,
+ 0,
+ undefined,
+ undefined,
+ { fetchFn }
+ )
+ expect(fetchFn).toHaveBeenCalledTimes(1)
+ expect(res.status).toBe(503)
+ expect(res.body).toEqual({ error: 'provider_unavailable' })
+ expect(receipts).toEqual([expect.objectContaining({ outcome: 'error' })])
+ expect(lines).toHaveLength(1)
+ expect(lines[0]).toMatchObject({
+ code: 'provider_unavailable',
+ deliveredAs: 'http_status',
+ httpStatus: 503,
+ })
+ expect(failureCount(metricsText, 'provider_unavailable')).toBe(1)
+ expect(failureCount(metricsText, 'request_limit')).toBe(0)
+ }
+ )
+
it('(g1-1a) answers 429 rate_limited and forwards a valid upstream Retry-After', async () => {
const { res, receipts, lines, metricsText } = await run(
'att-rate-http',
@@ -1776,7 +1815,9 @@ describe('codex-llm-proxy attempt telemetry', () => {
// Witness: a keepalive went out first, so only the frame can carry it.
expect(keepaliveCount(res.text)).toBeGreaterThanOrEqual(1)
expect(
- res.text.endsWith('data: {"type":"error","code":"upstream_rejected","upstreamStatus":402}\n\n')
+ res.text.endsWith(
+ 'data: {"type":"error","code":"upstream_rejected","upstreamStatus":402}\n\n'
+ )
).toBe(true)
expect(lines).toHaveLength(1)
expect(lines[0]).toMatchObject({
@@ -1885,14 +1926,9 @@ describe('codex-llm-proxy attempt telemetry', () => {
})
it('(p) T-R7-2c sends an SSE context_length_exceeded frame when text was already streamed', async () => {
- const { res, lines } = await run(
- 'att-context-sse',
- 1,
- 0,
- undefined,
- undefined,
- { terminal: UPSTREAM_CONTEXT_OVERFLOW_FRAMES }
- )
+ const { res, lines } = await run('att-context-sse', 1, 0, undefined, undefined, {
+ terminal: UPSTREAM_CONTEXT_OVERFLOW_FRAMES,
+ })
expect(res.status).toBe(200)
expect(res.text).toContain('data: {"type":"text","text":"t0"}')
expect(res.text).toContain('data: {"type":"error","code":"context_length_exceeded"}')
@@ -1998,13 +2034,7 @@ describe('codex-llm-proxy attempt telemetry', () => {
it.each(['constructor', '__proto__', 'toString', 'hasOwnProperty'])(
'(g) answers 503 for the inherited control-api code %j instead of crashing',
async code => {
- const { res, lines, metricsText } = await run(
- `att-proto-${code}`,
- 0,
- 0,
- undefined,
- code
- )
+ const { res, lines, metricsText } = await run(`att-proto-${code}`, 0, 0, undefined, code)
expect(res.status).toBe(503)
expect(res.body).toEqual({ error: code })
// Witness: the attempt was reached and logged, so the status above is the
@@ -2019,20 +2049,22 @@ describe('codex-llm-proxy attempt telemetry', () => {
it('(l) logs the request-limit reason and labels its failure metric request_limit', async () => {
const acquire = vi
.spyOn(streamGate, 'acquire')
- .mockRejectedValueOnce(new RequestLimitError('stream queue is full', 'queue_full'))
+ .mockRejectedValueOnce(
+ new RequestLimitError('stream queue is full', 'queue_full', 'proxy_capacity_exceeded')
+ )
try {
const { res, receipts, lines, metricsText } = await run('att-queue-full', 0, 0)
// Witness: the refusal came from the stream gate this test replaced.
expect(acquire).toHaveBeenCalledTimes(1)
expect(res.status).toBe(503)
- expect(res.body).toEqual({ error: 'provider_unavailable' })
+ expect(res.body).toEqual({ error: 'proxy_capacity_exceeded' })
// The gate refused before the redeem, so there is no receipt to finalize.
expect(receipts).toEqual([])
expect(lines).toHaveLength(1)
expect(lines[0]).toMatchObject({
providerAttemptId: 'att-queue-full',
outcome: 'failed',
- code: 'provider_unavailable',
+ code: 'proxy_capacity_exceeded',
reason: 'stream queue is full',
deliveredAs: 'http_status',
httpStatus: 503,
@@ -2094,9 +2126,7 @@ describe('codex-llm-proxy attempt telemetry', () => {
})
expect(res.status).toBe(200)
expect(keepaliveCount(res.text)).toBeGreaterThanOrEqual(1)
- expect(res.text.endsWith('data: {"type":"error","code":"provider_unavailable"}\n\n')).toBe(
- true
- )
+ expect(res.text.endsWith('data: {"type":"error","code":"provider_unavailable"}\n\n')).toBe(true)
expect(lines).toHaveLength(1)
expect(lines[0]).toMatchObject({
outcome: 'failed',
@@ -2306,7 +2336,10 @@ describe('codex-llm-proxy ticket-aware stream-gate wait (#739 D1-bis)', () => {
.set('Authorization', `Bearer ${platformToken()}`)
.send(envelope('att-ticket-life', ticketLifeMs))
.then(res => res)
- await until(() => streamGate.snapshot().queued === 1, 'the request queueing at the stream gate')
+ await until(
+ () => streamGate.snapshot().queued === 1,
+ 'the request queueing at the stream gate'
+ )
await vi.advanceTimersByTimeAsync(ticketLifeMs - 1)
// Witness: the request waited for the ticket's whole life, it was not
@@ -2359,7 +2392,10 @@ describe('codex-llm-proxy ticket-aware stream-gate wait (#739 D1-bis)', () => {
.set('Authorization', `Bearer ${platformToken()}`)
.send(envelope('att-ticket-alive', 60_000))
.then(res => res)
- await until(() => streamGate.snapshot().queued === 1, 'the request queueing at the stream gate')
+ await until(
+ () => streamGate.snapshot().queued === 1,
+ 'the request queueing at the stream gate'
+ )
await vi.advanceTimersByTimeAsync(50_000)
expect(await withinReal(reply, 100)).toBeUndefined()
@@ -2377,7 +2413,10 @@ describe('codex-llm-proxy ticket-aware stream-gate wait (#739 D1-bis)', () => {
}, 30_000)
/** A V2 envelope past `maxBodyBytes`, so it takes the visual gate. */
- function visualEnvelope(providerAttemptId: string, ticketLifeMs: number): Record<string, unknown> {
+ function visualEnvelope(
+ providerAttemptId: string,
+ ticketLifeMs: number
+ ): Record<string, unknown> {
const raw: Record<string, unknown> = {
schemaVersion: 'codex-completion-request.v2',
requestId: `req-${providerAttemptId}`,
@@ -2431,7 +2470,10 @@ describe('codex-llm-proxy ticket-aware stream-gate wait (#739 D1-bis)', () => {
.set('Authorization', `Bearer ${platformToken()}`)
.send(body)
.then(res => res)
- await until(() => visualStreamGate.snapshot().queued === 1, 'the body queueing at the visual gate')
+ await until(
+ () => visualStreamGate.snapshot().queued === 1,
+ 'the body queueing at the visual gate'
+ )
await vi.advanceTimersByTimeAsync(ticketLifeMs)
// Witness: the body was still waiting for a slot at the ticket's expiry.
@@ -2463,7 +2505,7 @@ describe('codex-llm-proxy ticket-aware stream-gate wait (#739 D1-bis)', () => {
// overflow witness naming gate saturation. Grok already pins this twin; the
// queue is filled directly so the refusal under test is the global visual
// gate, not the per-principal share.
- it('visual admission overflow answers 503 provider_unavailable and logs visual_gate', async () => {
+ it('visual admission overflow answers 503 visual_gate and logs visual_gate', async () => {
const warn = vi.spyOn(logger, 'warn')
const held: Array<() => void> = []
const waiting: Array<Promise<() => void>> = []
@@ -2487,12 +2529,12 @@ describe('codex-llm-proxy ticket-aware stream-gate wait (#739 D1-bis)', () => {
.set('Authorization', `Bearer ${platformToken({ sub: 'default/visual-gate-full' })}`)
.send(visualEnvelope('att-visual-gate-full', 60_000))
expect(res.status).toBe(503)
- expect(res.body).toEqual({ error: 'provider_unavailable' })
+ expect(res.body).toEqual({ error: 'visual_gate' })
expect(warn).toHaveBeenCalledWith(
expect.objectContaining({
event: 'codex_proxy_admission_refused',
reason: 'visual_gate',
- code: 'provider_unavailable',
+ code: 'visual_gate',
}),
'admission refused'
)
@@ -2837,7 +2879,7 @@ describe('codex-llm-proxy body budget release on upstream acceptance (#739 D2)',
const client = open(proxy.port, payload)
await until(() => client.ended(), 'the refusal')
expect(client.status()).toBe(503)
- expect(JSON.parse(client.received())).toEqual({ error: 'provider_unavailable' })
+ expect(JSON.parse(client.received())).toEqual({ error: 'proxy_capacity_exceeded' })
await until(() => proxy.closes() === 1, "the response's close event")
expect(client.errors()).toEqual([])
expect(proxy.reservations()).toEqual([Buffer.byteLength(payload)])
@@ -3004,7 +3046,10 @@ describe('codex-llm-proxy graceful drain on shutdown (#739 D6)', () => {
req.end(payload)
let closing: Promise<void> | undefined
try {
- await until(() => received.includes('data: {"type":"text","text":"t0"}'), 'the first SSE frame')
+ await until(
+ () => received.includes('data: {"type":"text","text":"t0"}'),
+ 'the first SSE frame'
+ )
closing = apps.close().then(() => {
order.push('closed')
})
diff --git a/codex-llm-proxy/test/streamGate.handoff.test.ts b/codex-llm-proxy/test/streamGate.handoff.test.ts
index 0d03ff10e..730cd3b12 100644
--- a/codex-llm-proxy/test/streamGate.handoff.test.ts
+++ b/codex-llm-proxy/test/streamGate.handoff.test.ts
@@ -360,7 +360,7 @@ describe('visual stream-gate handoff', () => {
)
const overflow = await postCompletion(port, 'codex-completion-request.v2', largeContent)
expect(overflow.status).toBe(503)
- expect(await overflow.json()).toEqual({ error: 'provider_unavailable' })
+ expect(await overflow.json()).toEqual({ error: 'visual_gate' })
const textWhileSaturated = postCompletion(port, 'codex-completion-request.v1')
await waitFor(
@@ -481,7 +481,7 @@ describe('visual stream-gate handoff', () => {
)
const fifth = await postCompletion(port, 'codex-completion-request.v2', largeContent)
expect(fifth.status).toBe(503)
- expect(await fifth.json()).toEqual({ error: 'provider_unavailable' })
+ expect(await fifth.json()).toEqual({ error: 'visual_host_share' })
expect(warn).toHaveBeenCalledWith(
{
event: 'codex_proxy_admission_refused',
@@ -619,7 +619,7 @@ describe('visual stream-gate handoff', () => {
const fifth = await postBody(port, platformToken(), invalidTicketBody(maxBodyBytes))
expect(fifth.status).toBe(503)
- expect(await fifth.json()).toEqual({ error: 'provider_unavailable' })
+ expect(await fifth.json()).toEqual({ error: 'visual_host_share' })
// The refusal names the principal it applied to, so an operator can tell
// which host was throttled.
expect(warn).toHaveBeenCalledWith(
@@ -704,7 +704,7 @@ describe('visual stream-gate handoff', () => {
for (let i = 0; i < targetCount; i += 1) {
const res = await postBody(port, platformToken(), invalidTicketBody(maxBodyBytes))
expect(res.status).toBe(503)
- expect(await res.json()).toEqual({ error: 'provider_unavailable' })
+ expect(await res.json()).toEqual({ error: 'visual_gate' })
}
expect(acquire).toHaveBeenCalledTimes(
VISUAL_STREAM_LIMITS.maxConcurrentStreams +
@@ -789,7 +789,7 @@ describe('visual stream-gate handoff', () => {
).toBe(2)
const fifth = await refusedRequest
expect(fifth.status).toBe(503)
- expect(await fifth.json()).toEqual({ error: 'provider_unavailable' })
+ expect(await fifth.json()).toEqual({ error: 'visual_host_share' })
expect(warn).toHaveBeenCalledWith(
{
event: 'codex_proxy_admission_refused',
diff --git a/codex-llm-proxy/test/streamLimitsFreeze.test.ts b/codex-llm-proxy/test/streamLimitsFreeze.test.ts
index 4e43e78a1..7ee0a3d09 100644
--- a/codex-llm-proxy/test/streamLimitsFreeze.test.ts
+++ b/codex-llm-proxy/test/streamLimitsFreeze.test.ts
@@ -1,5 +1,6 @@
import { describe, expect, it } from 'vitest'
-import { readdirSync, readFileSync } from 'node:fs'
+import { readFileSync, readdirSync } from 'node:fs'
+import ts from 'typescript'
import { STREAM_LIMITS } from '../src/requestLimits.js'
const fixturePath = new URL(
@@ -45,8 +46,10 @@ function emittedTransportCodes(): { codes: Set<string>; sites: number; construct
for (const name of readdirSync(srcDir).filter(file => file.endsWith('.ts'))) {
const source = readFileSync(new URL(name, srcDir), 'utf8')
constructions +=
- source.split('new CodexTransportError(').length - 1 +
- source.split('new UpstreamTimeoutError(').length - 1
+ source.split('new CodexTransportError(').length -
+ 1 +
+ source.split('new UpstreamTimeoutError(').length -
+ 1
for (const match of source.matchAll(TRANSPORT_ERROR_SITE)) {
const args = callArguments(source, match.index! + match[0].length - 1)
const code = args[match[1] === 'UpstreamTimeoutError' ? 1 : 0] ?? ''
@@ -70,19 +73,79 @@ const REJECT_SITE = /reject\(res,/g
// Refusal codes are wire strings; `Unauthorized` is the one spelled in capitals.
const REJECT_CODE_LITERAL = /^'([A-Za-z][A-Za-z0-9_]+)'$/
// The only computed codes a refusal may carry, as their exact source text.
-// `err.code` is a RequestLimitError (always provider_unavailable) and
+// `err.code` uses the closed, readonly RequestLimitCode union checked below;
// `mapped.code` is a transport or control-api code passed through mapError,
// whose transport half the scanner above already gates. Any other computed
// form cannot be checked against the taxonomy, so it fails here.
const COMPUTED_REJECT_CODES = ['err.code', 'mapped.code'] as const
+/** Computed admission codes are allowed only through this closed, readonly type. */
+function requestLimitCodes(
+ sourceText = readFileSync(new URL('requestLimits.ts', srcDir), 'utf8')
+): Set<string> {
+ const source = ts.createSourceFile('requestLimits.ts', sourceText, ts.ScriptTarget.Latest, true)
+ const alias = source.statements.find(
+ (node): node is ts.TypeAliasDeclaration =>
+ ts.isTypeAliasDeclaration(node) && node.name.text === 'RequestLimitCode'
+ )
+ if (!alias || !ts.isUnionTypeNode(alias.type)) {
+ throw new Error('RequestLimitCode must remain a closed literal union')
+ }
+ const codes = new Set(
+ alias.type.types.map(node => {
+ if (!ts.isLiteralTypeNode(node) || !ts.isStringLiteral(node.literal)) {
+ throw new Error('RequestLimitCode must not accept computed or open string types')
+ }
+ return node.literal.text
+ })
+ )
+ const owner = source.statements.find(
+ (node): node is ts.ClassDeclaration =>
+ ts.isClassDeclaration(node) && node.name?.text === 'RequestLimitError'
+ )
+ const property = owner?.members.find(
+ (node): node is ts.PropertyDeclaration =>
+ ts.isPropertyDeclaration(node) && node.name.getText(source) === 'code'
+ )
+ expect(property?.type?.getText(source)).toBe('RequestLimitCode')
+ expect(property?.modifiers?.some(node => node.kind === ts.SyntaxKind.ReadonlyKeyword)).toBe(true)
+ const constructor = owner?.members.find(ts.isConstructorDeclaration)
+ const input = constructor?.parameters.find(node => node.name.getText(source) === 'code')
+ expect(input?.type?.getText(source)).toBe('RequestLimitCode')
+ expect(input?.initializer?.getText(source)).toBe("'provider_unavailable'")
+ const assignments: string[] = []
+ const collect = (node: ts.Node): void => {
+ if (
+ ts.isBinaryExpression(node) &&
+ node.operatorToken.kind === ts.SyntaxKind.EqualsToken &&
+ node.left.getText(source) === 'this.code'
+ ) {
+ assignments.push(node.right.getText(source))
+ }
+ ts.forEachChild(node, collect)
+ }
+ if (owner) collect(owner)
+ expect(assignments).toEqual(['code'])
+ return codes
+}
+
+function rejectCodeLiteral(code: string, name: string): RegExpExecArray | null {
+ const literal = REJECT_CODE_LITERAL.exec(code)
+ const allowed = (COMPUTED_REJECT_CODES as readonly string[]).includes(code)
+ expect(
+ literal !== null || allowed,
+ `${name}: reject code must be a string literal or one of ${COMPUTED_REJECT_CODES.join(', ')}, got ${code}`
+ ).toBe(true)
+ return literal
+}
+
function emittedRejectCodes(): {
codes: Set<string>
computed: Set<string>
sites: number
occurrences: number
} {
- const codes = new Set<string>()
+ const codes = requestLimitCodes()
const computed = new Set<string>()
let sites = 0
let occurrences = 0
@@ -92,12 +155,7 @@ function emittedRejectCodes(): {
for (const match of source.matchAll(REJECT_SITE)) {
const args = callArguments(source, match.index! + 'reject'.length)
const code = args[2] ?? ''
- const literal = REJECT_CODE_LITERAL.exec(code)
- const allowed = (COMPUTED_REJECT_CODES as readonly string[]).includes(code)
- expect(
- literal !== null || allowed,
- `${name}: reject code must be a string literal or one of ${COMPUTED_REJECT_CODES.join(', ')}, got ${code}`
- ).toBe(true)
+ const literal = rejectCodeLiteral(code, name)
if (literal) codes.add(literal[1]!)
else computed.add(code)
sites += 1
@@ -107,6 +165,42 @@ function emittedRejectCodes(): {
}
describe('codex-subscription stream limits freeze', () => {
+ it('rejects open or mutable admission code producers while reading the closed production union', () => {
+ const sourceText = readFileSync(new URL('requestLimits.ts', srcDir), 'utf8')
+ const tree = ts.createSourceFile('requestLimits.ts', sourceText, ts.ScriptTarget.Latest, true)
+ const alias = tree.statements.find(
+ (node): node is ts.TypeAliasDeclaration =>
+ ts.isTypeAliasDeclaration(node) && node.name.text === 'RequestLimitCode'
+ )!
+ const withType = (type: string) =>
+ sourceText.slice(0, alias.type.getStart(tree)) + type + sourceText.slice(alias.type.end)
+ expect([...requestLimitCodes(sourceText)]).toEqual(
+ expect.arrayContaining(['provider_unavailable', 'proxy_capacity_exceeded', 'visual_gate'])
+ )
+ expect(() => requestLimitCodes(withType('string'))).toThrow('closed literal union')
+ expect(() => requestLimitCodes(withType("'provider_unavailable' | string"))).toThrow(
+ 'open string types'
+ )
+ expect(() =>
+ requestLimitCodes(
+ sourceText.replace('readonly code: RequestLimitCode', 'code: RequestLimitCode')
+ )
+ ).toThrow()
+ expect(() =>
+ requestLimitCodes(sourceText.replace('this.code = code', "this.code = 'arbitrary_code'"))
+ ).toThrow()
+ })
+
+ it.each(['caller.code', "err.code || 'provider_unavailable'", 'String(err.code)'])(
+ 'rejects unbounded computed refusal expression %s',
+ expression => {
+ expect(rejectCodeLiteral("'request_timeout'", 'fixture')).not.toBeNull()
+ expect(rejectCodeLiteral('err.code', 'fixture')).toBeNull()
+ expect(rejectCodeLiteral('mapped.code', 'fixture')).toBeNull()
+ expect(() => rejectCodeLiteral(expression, 'fixture')).toThrow('reject code must be')
+ }
+ )
+
it('pins the proxy STREAM_LIMITS to the limits the fixture publishes', () => {
// StreamGate and the upstream deadlines enforce these published bounds
// from the proxy's own constant, not from the contract package, so the
diff --git a/codex-llm-proxy/test/visualBodyReadOwnership.test.ts b/codex-llm-proxy/test/visualBodyReadOwnership.test.ts
new file mode 100644
index 000000000..d283efb0b
--- /dev/null
+++ b/codex-llm-proxy/test/visualBodyReadOwnership.test.ts
@@ -0,0 +1,532 @@
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
+import express, { type NextFunction, type Request, type Response } from 'express'
+import jwt from 'jsonwebtoken'
+import { generateKeyPairSync } from 'node:crypto'
+import { createServer } from 'node:http'
+import { connect as connectTcp } from 'node:net'
+import type { CodexLlmProxyConfig } from '../src/config.js'
+import {
+ VISUAL_PER_HOST_MAX_ADMITTED,
+ VISUAL_STREAM_LIMITS,
+ visualStreamGate,
+} from '../src/requestLimits.js'
+import { createProxyApps } from '../src/server.js'
+
+type ObservedRead = {
+ req: Request
+ res: Response
+ nativeSettled: boolean
+ returned: boolean
+ error: unknown
+ finishes: number
+ endCalls: number
+}
+
+const parserBarrier = vi.hoisted(() => {
+ let release!: () => void
+ let completion: Promise<void>
+ const reads: ObservedRead[] = []
+ const reset = () => {
+ reads.splice(0)
+ completion = new Promise<void>(resolve => {
+ release = resolve
+ })
+ }
+ reset()
+ return {
+ reads,
+ reset,
+ get completion() {
+ return completion
+ },
+ get settledCount() {
+ return reads.filter(read => read.nativeSettled).length
+ },
+ release() {
+ release()
+ },
+ }
+})
+
+vi.mock('express', async () => {
+ const actual = await vi.importActual<typeof import('express')>('express')
+ const actualDefault = (actual as unknown as { default: typeof express }).default
+ const mockedDefault = Object.assign(
+ (...args: Parameters<typeof express>) => actualDefault(...args),
+ {
+ json: (options?: Parameters<typeof actual.json>[0]) => {
+ const parser = actualDefault.json(options)
+ return (req: Request, res: Response, next: NextFunction) => {
+ // Observe the native visual reader. Ordinary JSON parsing stays native
+ // too, but must not count as starting the next visual reader.
+ const read: ObservedRead | undefined =
+ options?.limit === 8192
+ ? {
+ req,
+ res,
+ nativeSettled: false,
+ returned: false,
+ error: undefined,
+ finishes: 0,
+ endCalls: 0,
+ }
+ : undefined
+ if (read) {
+ parserBarrier.reads.push(read)
+ res.once('finish', () => {
+ read.finishes += 1
+ })
+ if (req.headers['x-visual-never-finish'] === '1') {
+ // Fault injection: no finish/close is produced by writing 408.
+ // The production close backstop must independently stop raw-body.
+ res.end = (() => {
+ read.endCalls += 1
+ return res
+ }) as Response['end']
+ }
+ }
+ parser(req, res, (err: unknown) => {
+ if (read) {
+ read.nativeSettled = true
+ read.error = err
+ }
+ const finish = () => {
+ next(err)
+ if (read) read.returned = true
+ }
+ if (req.headers['x-visual-parser-hold'] === '1') {
+ void parserBarrier.completion.then(finish)
+ } else {
+ finish()
+ }
+ })
+ }
+ },
+ }
+ )
+ return { ...actual, default: mockedDefault, json: mockedDefault.json }
+})
+
+const COMPLETIONS_PATH = '/internal/runtime/v1/codex/completions'
+const { privateKey, publicKey } = generateKeyPairSync('rsa', {
+ modulusLength: 2048,
+ publicKeyEncoding: { type: 'spki', format: 'pem' },
+ privateKeyEncoding: { type: 'pkcs8', format: 'pem' },
+})
+
+function config(): CodexLlmProxyConfig {
+ return {
+ runtimePort: 0,
+ adminPort: 0,
+ probePort: 0,
+ maxBodyBytes: 4096,
+ maxVisualBodyBytes: 8192,
+ maxStreamDurationMs: 1_800_000,
+ maxDeadlineMs: 1_800_000,
+ upstreamIdleTimeoutMs: 600_000,
+ heartbeatIntervalMs: 15_000,
+ jwtIssuer: 'control-api',
+ jwtPublicKey: publicKey,
+ executionEnabled: true,
+ controlApiBaseUrl: '',
+ controlApiServiceName: 'codex-llm-proxy',
+ controlApiServiceToken: '',
+ }
+}
+
+function platformToken(): string {
+ return jwt.sign(
+ {
+ sub: 'default/research-host',
+ hostRefs: ['research-host'],
+ workflowControlScopes: ['llm:codex:execute'],
+ scope: 'workflow:approval:request',
+ },
+ privateKey,
+ { algorithm: 'RS256', issuer: 'control-api', audience: 'workflow-approvals', expiresIn: 60 }
+ )
+}
+
+function platformTokenForOtherPrincipal(): string {
+ return jwt.sign(
+ {
+ sub: 'default/other-host',
+ hostRefs: ['research-host'],
+ workflowControlScopes: ['llm:codex:execute'],
+ scope: 'workflow:approval:request',
+ },
+ privateKey,
+ { algorithm: 'RS256', issuer: 'control-api', audience: 'workflow-approvals', expiresIn: 60 }
+ )
+}
+
+async function waitFor(predicate: () => boolean, label: string): Promise<void> {
+ const started = Date.now()
+ while (!predicate()) {
+ if (Date.now() - started > 5_000) throw new Error(label)
+ await new Promise(resolve => setTimeout(resolve, 5))
+ }
+}
+
+const sockets = new Set<ReturnType<typeof connectTcp>>()
+
+function stalledVisualBody(
+ port: number,
+ declared: number,
+ holdParser = false,
+ extraHeaders = '',
+ prefix = '{"executionTicket":'
+): { socket: ReturnType<typeof connectTcp>; response: Promise<number> } {
+ let owner: ReturnType<typeof connectTcp> | undefined
+ const response = new Promise<number>((resolve, reject) => {
+ owner = connectTcp(port, '127.0.0.1', () => {
+ owner?.write(
+ `POST ${COMPLETIONS_PATH} HTTP/1.1\r\n` +
+ `Host: 127.0.0.1:${port}\r\n` +
+ `Authorization: Bearer ${platformToken()}\r\n` +
+ `Content-Type: application/json\r\n` +
+ (holdParser ? 'X-Visual-Parser-Hold: 1\r\n' : '') +
+ extraHeaders +
+ `Content-Length: ${declared}\r\n` +
+ `\r\n` +
+ prefix
+ )
+ })
+ sockets.add(owner)
+ owner.once('close', () => sockets.delete(owner!))
+ owner.setTimeout(5_000, () => {
+ owner?.destroy()
+ reject(new Error('the stalled visual body produced no response'))
+ })
+ const chunks: Buffer[] = []
+ owner.on('data', data => {
+ chunks.push(data)
+ const match = /^HTTP\/1\.1 (\d+)/.exec(Buffer.concat(chunks).toString('utf8'))
+ if (match) resolve(Number(match[1]))
+ })
+ owner.on('error', err => {
+ if ((err as NodeJS.ErrnoException).code !== 'ECONNRESET') reject(err)
+ })
+ })
+ return { socket: owner!, response }
+}
+
+function invalidTicketBody(): string {
+ const content = 'x'.repeat(5000)
+ return JSON.stringify({
+ executionTicket: 'invalid-ticket',
+ requestHash: 'a'.repeat(64),
+ request: {
+ schemaVersion: 'codex-completion-request.v2',
+ requestId: 'req-visual-ownership',
+ idempotencyKey: 'idem-visual-ownership',
+ provider: 'codex-subscription',
+ model: 'gpt-5.1',
+ messages: [{ role: 'user', content }],
+ },
+ })
+}
+
+function postVisualRaw(
+ port: number,
+ body: string,
+ holdParser = false
+): { socket: ReturnType<typeof connectTcp>; response: Promise<number> } {
+ let owner: ReturnType<typeof connectTcp> | undefined
+ const response = new Promise<number>((resolve, reject) => {
+ owner = connectTcp(port, '127.0.0.1', () => {
+ owner?.write(
+ `POST ${COMPLETIONS_PATH} HTTP/1.1\r\n` +
+ `Host: 127.0.0.1:${port}\r\n` +
+ `Authorization: Bearer ${platformToken()}\r\n` +
+ `Content-Type: application/json\r\n` +
+ (holdParser ? 'X-Visual-Parser-Hold: 1\r\n' : '') +
+ `Content-Length: ${Buffer.byteLength(body)}\r\n` +
+ `\r\n` +
+ body
+ )
+ })
+ sockets.add(owner)
+ owner.once('close', () => sockets.delete(owner!))
+ owner.setTimeout(5_000, () => {
+ owner?.destroy()
+ reject(new Error('the visual request produced no response'))
+ })
+ const chunks: Buffer[] = []
+ owner.on('data', data => {
+ chunks.push(data)
+ const match = /^HTTP\/1\.1 (\d+)/.exec(Buffer.concat(chunks).toString('utf8'))
+ if (match) resolve(Number(match[1]))
+ })
+ owner.on('error', err => {
+ if ((err as NodeJS.ErrnoException).code !== 'ECONNRESET') reject(err)
+ })
+ })
+ return { socket: owner!, response }
+}
+
+async function postVisualStatus(port: number, body: string): Promise<number> {
+ const request = postVisualRaw(port, body)
+ const status = await request.response
+ request.socket.destroy()
+ return status
+}
+
+describe('codex visual body read ownership', () => {
+ const servers: Array<ReturnType<typeof createProxyApps>> = []
+ const listeners: Array<ReturnType<typeof createServer>> = []
+
+ beforeEach(() => parserBarrier.reset())
+
+ afterEach(async () => {
+ parserBarrier.release()
+ vi.restoreAllMocks()
+ for (const socket of sockets) socket.destroy()
+ await Promise.all(
+ listeners
+ .splice(0)
+ .map(
+ listener =>
+ new Promise<void>((resolve, reject) =>
+ listener.close(err => (err ? reject(err) : resolve()))
+ )
+ )
+ )
+ await Promise.all(servers.splice(0).map(server => server.close()))
+ await waitFor(
+ () => visualStreamGate.snapshot().running === 0 && visualStreamGate.snapshot().queued === 0,
+ 'the visual gate did not drain after the ownership test'
+ )
+ })
+
+ it('keeps the visual slot and principal share until the timed-out parser terminates', async () => {
+ const apps = createProxyApps(config(), {
+ bodyReadDeadlineMs: 200,
+ visualReadCloseGraceMs: 10,
+ })
+ servers.push(apps)
+ const listener = createServer(apps.runtimeApp).listen(0)
+ listeners.push(listener)
+ const address = listener.address()
+ if (!address || typeof address === 'string') throw new Error('listener has no port')
+
+ const readers = Array.from({ length: VISUAL_PER_HOST_MAX_ADMITTED }, () =>
+ stalledVisualBody(address.port, 5000, true)
+ )
+ await waitFor(
+ () =>
+ visualStreamGate.snapshot().running + visualStreamGate.snapshot().queued ===
+ VISUAL_PER_HOST_MAX_ADMITTED,
+ 'the stalled visual bodies did not fill the principal share'
+ )
+ const statuses = await Promise.all(
+ readers.slice(0, VISUAL_STREAM_LIMITS.maxConcurrentStreams).map(reader => reader.response)
+ )
+ expect(statuses).toEqual(
+ Array.from({ length: VISUAL_STREAM_LIMITS.maxConcurrentStreams }, () => 408)
+ )
+ expect(visualStreamGate.snapshot().running + visualStreamGate.snapshot().queued).toBe(
+ VISUAL_PER_HOST_MAX_ADMITTED
+ )
+
+ const shareRefusal = await postVisualStatus(address.port, invalidTicketBody())
+ expect(shareRefusal).toBe(503)
+ // A different principal can queue, but the reader must not start until the
+ // existing parser callbacks finish. This observes reader entry, not merely
+ // the absence of an HTTP response from the next request.
+ const nextSocket = connectTcp(address.port, '127.0.0.1')
+ sockets.add(nextSocket)
+ nextSocket.once('close', () => sockets.delete(nextSocket))
+ const nextStatus = new Promise<number>((resolve, reject) => {
+ nextSocket.once('connect', () => {
+ const body = invalidTicketBody()
+ nextSocket.write(
+ `POST ${COMPLETIONS_PATH} HTTP/1.1\r\n` +
+ `Host: 127.0.0.1:${address.port}\r\n` +
+ `Authorization: Bearer ${platformTokenForOtherPrincipal()}\r\n` +
+ `Content-Type: application/json\r\n` +
+ `Content-Length: ${Buffer.byteLength(body)}\r\n\r\n${body}`
+ )
+ })
+ nextSocket.on('data', data => {
+ const match = /^HTTP\/1\.1 (\d+)/.exec(data.toString('utf8'))
+ if (match) resolve(Number(match[1]))
+ })
+ nextSocket.once('error', reject)
+ })
+ await waitFor(
+ () =>
+ visualStreamGate.snapshot().queued ===
+ VISUAL_PER_HOST_MAX_ADMITTED - VISUAL_STREAM_LIMITS.maxConcurrentStreams + 1,
+ 'the next principal did not queue behind the retained parsers'
+ )
+ expect(parserBarrier.reads).toHaveLength(VISUAL_STREAM_LIMITS.maxConcurrentStreams)
+
+ for (const reader of readers) {
+ reader.response.catch(() => undefined)
+ reader.socket.destroy()
+ }
+ await waitFor(
+ () => parserBarrier.settledCount === VISUAL_STREAM_LIMITS.maxConcurrentStreams,
+ 'the held parsers did not terminate'
+ )
+ expect(visualStreamGate.snapshot().running).toBe(VISUAL_STREAM_LIMITS.maxConcurrentStreams)
+ expect(visualStreamGate.snapshot().queued).toBeGreaterThanOrEqual(1)
+ expect(parserBarrier.reads).toHaveLength(VISUAL_STREAM_LIMITS.maxConcurrentStreams)
+ parserBarrier.release()
+ expect(await nextStatus).toBe(403)
+ nextSocket.destroy()
+ await waitFor(
+ () => visualStreamGate.snapshot().running === 0 && visualStreamGate.snapshot().queued === 0,
+ 'the timed-out visual reader did not terminate and release its admission'
+ )
+ expect(
+ parserBarrier.reads
+ .slice(0, VISUAL_STREAM_LIMITS.maxConcurrentStreams)
+ .every(read => read.returned && read.req.body === undefined)
+ ).toBe(true)
+ })
+
+ it('uses the close backstop when the visual client disappears before the deadline', async () => {
+ const apps = createProxyApps(config(), {
+ bodyReadDeadlineMs: 5_000,
+ visualReadCloseGraceMs: 10,
+ })
+ servers.push(apps)
+ const listener = createServer(apps.runtimeApp).listen(0)
+ listeners.push(listener)
+ const address = listener.address()
+ if (!address || typeof address === 'string') throw new Error('listener has no port')
+
+ const stalled = stalledVisualBody(address.port, 5000)
+ await waitFor(
+ () => visualStreamGate.snapshot().running === 1,
+ 'the visual body did not take the slot'
+ )
+ stalled.socket.destroy()
+ stalled.response.catch(() => undefined)
+ await waitFor(
+ () => visualStreamGate.snapshot().running === 0 && visualStreamGate.snapshot().queued === 0,
+ 'a disconnected visual reader retained its admission after the close backstop'
+ )
+ })
+
+ it('releases the visual admission after a parser error', async () => {
+ const apps = createProxyApps(config())
+ servers.push(apps)
+ const listener = createServer(apps.runtimeApp).listen(0)
+ listeners.push(listener)
+ const address = listener.address()
+ if (!address || typeof address === 'string') throw new Error('listener has no port')
+
+ const status = await postVisualStatus(address.port, `{"executionTicket":${' '.repeat(5000)}`)
+ expect(status).toBe(400)
+ await waitFor(
+ () => visualStreamGate.snapshot().running === 0 && visualStreamGate.snapshot().queued === 0,
+ 'a malformed visual body did not release its admission'
+ )
+ })
+
+ it('transfers a successfully parsed visual body to the route and releases it after the response', async () => {
+ const apps = createProxyApps(config())
+ servers.push(apps)
+ const listener = createServer(apps.runtimeApp).listen(0)
+ listeners.push(listener)
+ const address = listener.address()
+ if (!address || typeof address === 'string') throw new Error('listener has no port')
+
+ const status = await postVisualStatus(address.port, invalidTicketBody())
+ expect(status).toBe(403)
+ await waitFor(
+ () => visualStreamGate.snapshot().running === 0 && visualStreamGate.snapshot().queued === 0,
+ 'a parsed visual body did not release its admission after the route response'
+ )
+ })
+ it('stops the native reader independently when a non-reading peer never finishes the response', async () => {
+ const apps = createProxyApps(config(), { bodyReadDeadlineMs: 40, visualReadCloseGraceMs: 15 })
+ servers.push(apps)
+ const listener = createServer(apps.runtimeApp).listen(0)
+ listeners.push(listener)
+ const address = listener.address()
+ if (!address || typeof address === 'string') throw new Error('listener has no port')
+
+ const stalled = stalledVisualBody(address.port, 5000, false, 'X-Visual-Never-Finish: 1\r\n')
+ stalled.response.catch(() => undefined)
+ stalled.socket.pause()
+ await waitFor(() => parserBarrier.reads.length === 1, 'the native visual reader did not start')
+ const read = parserBarrier.reads[0]!
+ await waitFor(() => read.returned, 'the independent close backstop did not terminate raw-body')
+ expect(read.endCalls).toBe(1)
+ expect(read.finishes).toBe(0)
+ expect(read.req.destroyed).toBe(true)
+ expect(read.req.body).toBeUndefined()
+ expect(visualStreamGate.snapshot()).toEqual({ running: 0, queued: 0 })
+ expect(await postVisualStatus(address.port, invalidTicketBody())).toBe(403)
+ })
+
+ it('clears a late successful native parse after its read deadline without dispatching it', async () => {
+ const apps = createProxyApps(config(), { bodyReadDeadlineMs: 40, visualReadCloseGraceMs: 1000 })
+ servers.push(apps)
+ const listener = createServer(apps.runtimeApp).listen(0)
+ listeners.push(listener)
+ const address = listener.address()
+ if (!address || typeof address === 'string') throw new Error('listener has no port')
+
+ const body = invalidTicketBody()
+ const split = 128
+ const stalled = stalledVisualBody(
+ address.port,
+ Buffer.byteLength(body),
+ false,
+ 'X-Visual-Never-Finish: 1\r\n',
+ body.slice(0, split)
+ )
+ stalled.response.catch(() => undefined)
+ await waitFor(
+ () => parserBarrier.reads[0]?.endCalls === 1,
+ 'the visual reader never reached its deadline'
+ )
+ const read = parserBarrier.reads[0]!
+ expect(read.nativeSettled).toBe(false)
+ expect(visualStreamGate.snapshot().running).toBe(1)
+ stalled.socket.write(body.slice(split))
+ await waitFor(() => read.returned, 'the late native parser callback did not run')
+ expect(read.error).toBeUndefined()
+ expect(read.req.body).toBeUndefined()
+ expect(visualStreamGate.snapshot()).toEqual({ running: 0, queued: 0 })
+ stalled.socket.destroy()
+ expect(await postVisualStatus(address.port, invalidTicketBody())).toBe(403)
+ })
+
+ it('clears the retained parsed body after response close before its callback returns', async () => {
+ const apps = createProxyApps(config(), { bodyReadDeadlineMs: 5000, visualReadCloseGraceMs: 15 })
+ servers.push(apps)
+ const listener = createServer(apps.runtimeApp).listen(0)
+ listeners.push(listener)
+ const address = listener.address()
+ if (!address || typeof address === 'string') throw new Error('listener has no port')
+
+ const request = postVisualRaw(address.port, invalidTicketBody(), true)
+ request.response.catch(() => undefined)
+ await waitFor(() => parserBarrier.settledCount === 1, 'the complete body was never parsed')
+ const read = parserBarrier.reads[0]!
+ expect(read.req.body).toBeDefined()
+ expect(read.returned).toBe(false)
+ request.socket.destroy()
+ await waitFor(() => read.res.destroyed, 'the response did not close')
+ expect(visualStreamGate.snapshot().running).toBe(1)
+ parserBarrier.release()
+ await waitFor(() => read.returned, 'the disconnected parser callback did not run')
+ expect(read.req.body).toBeUndefined()
+ expect(visualStreamGate.snapshot()).toEqual({ running: 0, queued: 0 })
+ expect(await postVisualStatus(address.port, invalidTicketBody())).toBe(403)
+ })
+
+ it.each([0, -1, 1.5, Number.NaN, Number.POSITIVE_INFINITY, 2147483648])(
+ 'rejects invalid native reader clocks (%s)',
+ value => {
+ expect(() => createProxyApps(config(), { bodyReadDeadlineMs: value })).toThrow(RangeError)
+ expect(() => createProxyApps(config(), { visualReadCloseGraceMs: value })).toThrow(RangeError)
+ }
+ )
+})
diff --git a/docs/architecture/codex-subscription-transport-contract.md b/docs/architecture/codex-subscription-transport-contract.md
index 9ebb10729..49e2d2284 100644
--- a/docs/architecture/codex-subscription-transport-contract.md
+++ b/docs/architecture/codex-subscription-transport-contract.md
@@ -107,12 +107,23 @@ existing kill switches (`MCP_HOST_CODEX_SUBSCRIPTION_ENABLED` and
`CODEX_LLM_PROXY_EXECUTION_ENABLED`). Missing image provenance returns
`image_source_invalid`. Limit errors, including HTTP 413 without a JSON body,
remain non-retryable and do not fail over. Images are never silently stripped.
-Two 503s the proxy gives an image request before redeem are different:
-`visual_gate` (the visual gate is full) and `visual_host_share` (the
-principal's share of it is full) answer `provider_unavailable`, which the Host
-classifies as `LLM_MODEL_OVERLOADED`, retryable, and a Host with a fallback
-provider configured fails over on it. Whether that should stay retryable is
-tracked in #868.
+Three local-capacity outcomes the proxy can give before redeem are closed wire
+identities: `visual_host_share` (the principal's share is full), `visual_gate`
+(the visual gate's queue is full or waited out), and
+`proxy_capacity_exceeded` (ordinary stream/body capacity). They classify as
+non-retryable `LLM_API_CALL_FAILED`, do not construct fallback or install its
+cooldown, and leave the primary usable for the next text turn. Only the
+proxy-owned machine-code field supplies that identity; upstream prose or a
+`reason` field never does. A timed-out visual reader keeps its slot and
+principal share until parser termination, with a bounded close/destruction
+backstop. Real upstream outage, connect-failure and 429 semantics remain
+unchanged.
+
+The critical Codex test title migrated from "visual admission overflow answers
+503 provider_unavailable and logs visual_gate" to "visual admission overflow
+answers 503 visual_gate and logs visual_gate". Its physical file and the original
+503, refusal-log, no-redeem and capacity-recovery protections remain pinned; the
+wire literal changed to reflect the local-capacity contract.
Residual risk. Tool screenshots reach the upstream in a user-role message,
after the tool messages, behind the fixed text "These images are output of the
@@ -504,6 +515,7 @@ behavior changes:
and logged as `codex_proxy_admission_refused` with `reason: visual_gate`.
A client that leaves while queued is not gate saturation: its place is freed
silently, with no log and no response.
+
- It requires the redeem response to carry `maxStreamDurationMs` greater
than 0. An absent value is a contract violation, not a default.
- It logs one `codex_proxy_attempt_finished` event per completion attempt,
@@ -542,6 +554,7 @@ behavior changes:
`provider_unavailable`. The metric labels it `request_limit` to keep it
apart from upstream outages, and the log line carries the limit's fixed
`reason`.
+
- **Live-target attestation.** Codex authorize attests the live Host or recipe
target. The allowed providers come only from the spec's model, allowed
models and fallbacks (Hosts) or agent providers (recipes). A target that
@@ -712,7 +725,7 @@ code the proxy constructs, and every code it refuses a request with
| -------------------------------------------------------- | --------------------------------- |
| `request exceeds maxRequestBodyBytes` | serialized UTF-8 byte cap |
| `request exceeds maxRequestBodyBytes outside image data` | non-image share of a V2 request |
- | `request exceeds maxRequestElements` | element count in `checkStructure` |
+ | `request exceeds maxRequestElements` | element count in `checkStructure` |
| `messages exceed <maxMessages>` | message count |
| `messages[i].toolCalls exceed <maxToolCalls>` | tool calls on one message |
diff --git a/docs/architecture/grok-subscription-transport-contract.md b/docs/architecture/grok-subscription-transport-contract.md
index 8aa3289dc..a49ed75e6 100644
--- a/docs/architecture/grok-subscription-transport-contract.md
+++ b/docs/architecture/grok-subscription-transport-contract.md
@@ -143,10 +143,11 @@ ordinary limit.
identity (`sub` plus sorted `hostRefs`) may hold at most 2 visual entries
(running or queued) of the gate's 5. A token whose `sub` is missing, empty or
not a string is refused 401 before any gate or share is taken. When the share
-is full, the proxy answers 503 `provider_unavailable` and logs
+is full, the proxy answers 503 `visual_host_share` and logs
`grok_proxy_admission_refused` with `reason: visual_host_share`. The share is
-released together with the visual slot: on read-deadline expiry, on client
-close before hand-off, and when the handler's finally block unwinds.
+released with the visual slot only after parser termination (or by the route's
+stream finally); a timeout or response close first bounds and destroys the
+still-owned request so body-parser can unwind.
The visual wait uses the request's single admission clock (arrival +
`maxQueueWaitMs`, 60 s). Unlike the ordinary stream gate, it does not end at
@@ -154,14 +155,16 @@ the ticket's `exp`. A ticket that expired while its visual body waited is
refused `ticket_expired` when the proxy verifies it after the read, before any
redeem. Once a slot is granted, the body must be read and parsed within
`BODY_READ_DEADLINE_MS` (10 s); otherwise the proxy answers 408
-`request_timeout` with `connection: close` and frees the slot. After the grant,
-the slot is also freed when the response closes for any other reason, which
-covers every refusal before the stream starts. The raw-body scan
+`request_timeout` with `connection: close`, starts a bounded destruction
+backstop, and releases the slot and principal share only in body-parser's
+completion callback. A response close before hand-off follows the same
+backstop. The raw-body scan
refuses a body nested deeper than 70 (`BODY_STRUCTURE_LIMITS.maxDepth`) with
400 `invalid_request` before `JSON.parse`, so `JSON.stringify` and the byte
-measurement never see a deeper tree. A visual-gate refusal (queue full, wait exceeded) is
-answered 503 `provider_unavailable` and logged as
-`grok_proxy_admission_refused` with `reason: visual_gate`.
+measurement never see a deeper tree. A visual-gate capacity refusal (queue
+full or queue wait) is answered 503 `visual_gate`. Ordinary stream/body queue
+full and queue-wait capacity refusals answer 503 `proxy_capacity_exceeded`;
+deadline, abort and ticket-life outcomes keep their existing identities.
A client that disconnects while its request is queued frees its queue place
only if Node sees the disconnect. Node keeps reading a queued request's socket
@@ -230,11 +233,12 @@ Errors:
`payload_too_large`. The Host maps `payload_too_large` to
`LLM_CONTEXT_LENGTH_EXCEEDED`, not retryable.
- Images are never stripped silently, and a size refusal never triggers a
- provider fallback. A 503 from the visual gate or from the per-principal
- share (`visual_gate`, `visual_host_share`) is different: the proxy answers
- `provider_unavailable`, the Host classifies it as `LLM_MODEL_OVERLOADED`,
- retryable, and a Host with a fallback provider configured fails over on it.
- Whether that should stay retryable is tracked in #868.
+ provider fallback. Local proxy capacity outcomes are also terminal:
+ `visual_host_share`, `visual_gate`, and ordinary
+ `proxy_capacity_exceeded` decode through the proxy-owned machine-code field,
+ classify as non-retryable `LLM_API_CALL_FAILED`, and do not install fallback
+ cooldown. An upstream `reason` string is never trusted as provenance. Real
+ upstream 503, connect failure and 429 behavior remains unchanged.
Desktop. The composer budget for `grok-subscription` is 16 MiB per image and
16 MiB decoded in total, with no dimension bound. That is the shared rpc-proxy
diff --git a/grok-llm-proxy/src/requestLimits.ts b/grok-llm-proxy/src/requestLimits.ts
index b579a3e0b..ecded6a9b 100644
--- a/grok-llm-proxy/src/requestLimits.ts
+++ b/grok-llm-proxy/src/requestLimits.ts
@@ -51,6 +51,13 @@ export const IN_FLIGHT_BODY_BUDGET_BODIES = 3
*/
export const BODY_READ_DEADLINE_MS = 10_000
+/**
+ * Bounds the interval between a visual read timeout/client response close and
+ * forced request destruction. The parser's callback—not this timer—releases the
+ * visual slot and principal share.
+ */
+export const VISUAL_READ_CLOSE_GRACE_MS = 100
+
export const STREAM_LIMITS = {
maxConcurrentStreams: 8,
maxQueuedRequests: 16,
@@ -94,9 +101,10 @@ export const VISUAL_STREAM_LIMITS = {
* Fair share of the visual gate's entries — running plus queued — that one
* platform principal (`sub` plus sorted `hostRefs`) may hold at once. Above the
* share, further visual requests from that principal are refused 503
- * `provider_unavailable` with log reason `visual_host_share`, so one principal
+ * `visual_host_share`, so one principal
* cannot fill a gate that every other host still needs. The gate widths above
* are unchanged; the share only bounds how much of them one principal occupies.
+ * Its refusal is a local capacity outcome, not an upstream outage.
*/
export const VISUAL_PER_HOST_MAX_ADMITTED = 2
@@ -117,14 +125,18 @@ export type RequestLimitKind =
| 'queue_wait'
| 'ticket_life'
+export type RequestLimitCode = 'provider_unavailable' | 'proxy_capacity_exceeded' | 'visual_gate'
+
export class RequestLimitError extends Error {
- readonly code = 'provider_unavailable'
+ readonly code: RequestLimitCode
constructor(
message: string,
- readonly kind: RequestLimitKind
+ readonly kind: RequestLimitKind,
+ code: RequestLimitCode = 'provider_unavailable'
) {
super(message)
this.name = 'RequestLimitError'
+ this.code = code
}
}
@@ -178,7 +190,8 @@ export class StreamGate {
constructor(
private readonly maxConcurrent: number = STREAM_LIMITS.maxConcurrentStreams,
private readonly maxQueued: number = STREAM_LIMITS.maxQueuedRequests,
- private readonly maxQueueWaitMs: number = STREAM_LIMITS.maxQueueWaitMs
+ private readonly maxQueueWaitMs: number = STREAM_LIMITS.maxQueueWaitMs,
+ private readonly queueCapacityCode: RequestLimitCode = 'proxy_capacity_exceeded'
) {
// A NaN or fractional size would let every caller through or queue
// without bound, because the comparisons below would never hold.
@@ -221,7 +234,7 @@ export class StreamGate {
if (signal?.aborted) throw new RequestLimitError('stream request was aborted', 'aborted')
if (this.running >= this.maxConcurrent) {
if (this.queued >= this.maxQueued)
- throw new RequestLimitError('stream queue is full', 'queue_full')
+ throw new RequestLimitError('stream queue is full', 'queue_full', this.queueCapacityCode)
this.queued += 1
try {
await new Promise<void>((resolve, reject) => {
@@ -247,7 +260,8 @@ export class StreamGate {
reject(
new RequestLimitError(
'stream queue wait exceeded',
- deadlineGoverns ? 'deadline' : 'queue_wait'
+ deadlineGoverns ? 'deadline' : 'queue_wait',
+ deadlineGoverns ? 'provider_unavailable' : this.queueCapacityCode
)
)
}
@@ -289,7 +303,9 @@ export const streamGate = new StreamGate()
export const visualStreamGate = new StreamGate(
VISUAL_STREAM_LIMITS.maxConcurrentStreams,
- VISUAL_STREAM_LIMITS.maxQueuedRequests
+ VISUAL_STREAM_LIMITS.maxQueuedRequests,
+ STREAM_LIMITS.maxQueueWaitMs,
+ 'visual_gate'
)
type BodyWaiter = { bytes: number; grant: (release: () => void) => void }
@@ -348,7 +364,11 @@ export class BodyBudget {
return this.take(bytes)
}
if (this.waiters.length >= this.maxQueued) {
- throw new RequestLimitError('body admission queue is full', 'queue_full')
+ throw new RequestLimitError(
+ 'body admission queue is full',
+ 'queue_full',
+ 'proxy_capacity_exceeded'
+ )
}
return new Promise<() => void>((resolve, reject) => {
let deadline: ReturnType<typeof setTimeout> | undefined
diff --git a/grok-llm-proxy/src/server.ts b/grok-llm-proxy/src/server.ts
index ff60f4a51..e560f8161 100644
--- a/grok-llm-proxy/src/server.ts
+++ b/grok-llm-proxy/src/server.ts
@@ -46,6 +46,7 @@ import {
STREAM_LIMITS,
TicketLifeError,
VISUAL_PER_HOST_MAX_ADMITTED,
+ VISUAL_READ_CLOSE_GRACE_MS,
streamGate,
visualStreamGate,
} from './requestLimits.js'
@@ -116,7 +117,9 @@ function isRecord(value: unknown): value is Record<string, unknown> {
function reject(res: Response, status: number, code: string, type?: string): void {
if (res.headersSent) return
logger.warn(
- type === undefined ? { event: 'grok_proxy_denied', code } : { event: 'grok_proxy_denied', code, type },
+ type === undefined
+ ? { event: 'grok_proxy_denied', code }
+ : { event: 'grok_proxy_denied', code, type },
'request denied'
)
res.status(status).json({ error: code })
@@ -270,7 +273,7 @@ function bodyAdmission(
},
'admission refused'
)
- reject(res, 503, 'provider_unavailable')
+ reject(res, 503, err.code)
}
)
}
@@ -288,6 +291,8 @@ export type ProxyRuntimeDeps = {
lookup?: OriginPolicyOptions['lookup']
/** Test seam for the body-read deadline. Production uses `BODY_READ_DEADLINE_MS`. */
bodyReadDeadlineMs?: number
+ /** Test seam for the bounded visual-parser close backstop. */
+ visualReadCloseGraceMs?: number
/** Test seam: hang or observe a stream without contacting Grok. */
streamCompletion?: typeof streamGrokCompletion
}
@@ -337,9 +342,16 @@ export function createProxyApps(
// One budget for both apps: every body this process reads counts against it.
const bodyBudget = new BodyBudget(IN_FLIGHT_BODY_BUDGET_BODIES * config.maxBodyBytes)
const bodyReadDeadlineMs = deps.bodyReadDeadlineMs ?? BODY_READ_DEADLINE_MS
+ const visualReadCloseGraceMs = deps.visualReadCloseGraceMs ?? VISUAL_READ_CLOSE_GRACE_MS
+ for (const [name, value] of Object.entries({ bodyReadDeadlineMs, visualReadCloseGraceMs })) {
+ if (!Number.isSafeInteger(value) || value <= 0 || value > 2_147_483_647) {
+ throw new RangeError(`${name} must be a positive signed-32-bit integer`)
+ }
+ }
// Per-principal occupancy of the visual gate (running plus queued entries).
// Keyed on the verified platform principal: sub plus sorted hostRefs.
const visualPrincipalAdmissions = new Map<string, number>()
+ const visualRequestOwners = new WeakSet<GatedRequest>()
const runtimeApp = express()
// R9-M-B: the token is checked from the header before any budget is taken or
@@ -384,12 +396,12 @@ export function createProxyApps(
// neither header has no body, so only a declared length takes a visual slot.
//
// A granted slot is released exactly once. The handler hands it to the
- // stream or releases it; every other exit (a parse error, a throw that
- // reaches the error handler, a dropped client) ends with the response's
- // `close` event, which releases whatever is still held. Like a budgeted
- // body, a visual body must be read and parsed within `bodyReadDeadlineMs` of
- // the grant, or it is answered 408 `request_timeout`, its slot released and
- // its connection closed.
+ // stream; parser errors and timeouts release it only after body-parser's
+ // callback proves that work has unwound. A response close starts a bounded
+ // request-destruction backstop rather than releasing live parser ownership.
+ // Like a budgeted body, a visual body must be read and parsed within
+ // `bodyReadDeadlineMs` of the grant, or it is answered 408 `request_timeout`
+ // with `connection: close`.
const selectTransportBudget = (req: GatedRequest, res: Response, next: NextFunction): void => {
const declared = contentLengthBytes(req)
if (declared !== null && declared > config.maxVisualBodyBytes) {
@@ -423,7 +435,7 @@ export function createProxyApps(
},
'admission refused'
)
- reject(res, 503, 'provider_unavailable')
+ reject(res, 503, 'visual_host_share')
return
}
visualPrincipalAdmissions.set(visualPrincipal, admittedByPrincipal + 1)
@@ -471,7 +483,7 @@ export function createProxyApps(
},
'admission refused'
)
- reject(res, 503, 'provider_unavailable')
+ reject(res, 503, err.code)
return
}
next(err)
@@ -485,38 +497,55 @@ export function createProxyApps(
req.grokStreamRelease = undefined
}
req.grokStreamRelease = releaseVisual
- let expired = false
+ let cancelled = false
+ let parserSettled = false
+ let closeGrace: ReturnType<typeof setTimeout> | undefined
+ const cancelParser = (): void => {
+ if (parserSettled || closeGrace !== undefined) return
+ closeGrace = setTimeout(() => {
+ if (!parserSettled) req.destroy()
+ }, visualReadCloseGraceMs)
+ }
const readDeadline = setTimeout(() => {
- expired = true
- releaseVisual()
+ cancelled = true
+ cancelParser()
if (res.headersSent) return
// The rest of the body is never read, so the connection cannot be reused.
res.setHeader('connection', 'close')
reject(res, 408, 'request_timeout')
}, bodyReadDeadlineMs)
- res.once('close', () => {
+ const onResponseClose = (): void => {
+ cancelled = true
clearTimeout(readDeadline)
- // The close handler releases only while the slot is still held here. A kept
- // visual body transfers `req.grokStreamRelease` into the handler,
- // which clears the field and releases in its outer `finally`. A
- // demoted body releases through `releaseAdmission` before it joins
- // the ordinary stream gate; this hook may still fire after demotion
- // but finds the field empty and no-ops. Releasing here after hand-off
- // would free the visual slot and the principal share while the
- // handler still holds the parsed body during its (up to ~30 s)
- // unwind, admitting another large body over the gate width.
- if (req.grokStreamRelease) releaseVisual()
- })
+ cancelParser()
+ }
+ res.once('close', onResponseClose)
visualJson(req, res, err => {
+ parserSettled = true
clearTimeout(readDeadline)
- // The 408 already answered this request; the parser's late error
- // (the body aborted by the closed connection) has no one to reach.
- if (expired) return
+ if (closeGrace !== undefined) clearTimeout(closeGrace)
+ res.off('close', onResponseClose)
+ if (cancelled || err) {
+ req.body = undefined
+ if (err !== null && typeof err === 'object' && 'body' in err) {
+ delete (err as { body?: unknown }).body
+ }
+ }
+ // Capacity remains charged until body-parser has actually unwound. On
+ // success, `req.grokStreamRelease` transfers ownership to the handler,
+ // which releases it in its outer `finally`; refusals release here.
+ if (cancelled || err) releaseVisual()
+ if (cancelled) return
if (err) {
- releaseVisual()
next(err)
return
}
+ res.once('close', () => {
+ // A synchronous handler throw can happen before its async owner
+ // installs cancellation and a finally block. Once that owner has
+ // taken over, response close must not release its live work.
+ if (!visualRequestOwners.has(req)) releaseVisual()
+ })
next()
})
})()
@@ -623,6 +652,7 @@ export function createProxyApps(
// to req 'close' aborts the Grok hop on every call (3–12ms canceled).
// Abort only when the client drops the response before we finish writing.
abortWhenClientDisconnects(req, res, abort)
+ visualRequestOwners.add(gated)
// One `grok_proxy_attempt_finished` line per attempt. Identifiers and
// counts only: never the body, ticket, frames, tool names or arguments.
const attempt = {
@@ -782,9 +812,9 @@ export function createProxyApps(
stopHeartbeat?.()
const mapped = mapError(err)
metrics.observeAttempt('error', 'completion_stream')
- // A request limit answers provider_unavailable on the wire; its own
- // label keeps it apart from real upstream outages in the metric. A
- // ticket that died in the queue is counted as provider_unavailable.
+ // Queue capacity has a terminal local wire code; the request_limit
+ // metric also includes abort/deadline kinds that keep their existing
+ // wire semantics. Ticket expiry remains provider_unavailable.
metrics.observeAttemptFailure(
err instanceof RequestLimitError && !(err instanceof TicketLifeError)
? 'request_limit'
@@ -792,8 +822,7 @@ export function createProxyApps(
)
if (err instanceof UpstreamTimeoutError) metrics.observeUpstreamTimeout(err.kind)
const deliveredAs = res.headersSent ? 'sse_error' : 'http_status'
- const causeCode =
- err instanceof ControlApiClientError ? err.causeCode : fetchCauseCode(err)
+ const causeCode = err instanceof ControlApiClientError ? err.causeCode : fetchCauseCode(err)
logger.warn(
{
event: 'grok_proxy_attempt_finished',
@@ -829,7 +858,9 @@ export function createProxyApps(
: undefined
const rejectedBy = typeof upstreamStatus === 'number' ? { upstreamStatus } : {}
if (deliveredAs === 'sse_error') {
- res.write(`data: ${JSON.stringify({ type: 'error', code: mapped.code, ...rejectedBy })}\n\n`)
+ res.write(
+ `data: ${JSON.stringify({ type: 'error', code: mapped.code, ...rejectedBy })}\n\n`
+ )
res.end()
return
}
@@ -847,6 +878,10 @@ export function createProxyApps(
} finally {
stopHeartbeat?.()
release?.()
+ // Defence in depth for an early throw after this async owner took over
+ // but before the visual release was moved into `release`.
+ gated.grokStreamRelease?.()
+ gated.grokStreamRelease = undefined
// A demoted body's reservation, when the stream ended before the body
// was written upstream. Budgeted bodies are also released on close.
gated.grokBodyRelease?.()
@@ -1022,6 +1057,9 @@ const ATTEMPT_ERROR_STATUS: Record<string, number> = {
// would spend the same budget again, so it is a gateway timeout, not 503.
stream_duration_exceeded: 504,
connection_unavailable: 503,
+ visual_host_share: 503,
+ visual_gate: 503,
+ proxy_capacity_exceeded: 503,
provider_unavailable: 503,
sse_buffer_exceeded: 503,
invalid_receipt: 503,
@@ -1062,7 +1100,6 @@ function isMappedError(err: unknown): err is MappedError {
function mapError(err: unknown): { status: number; code: string } {
if (!isMappedError(err)) return { status: 503, code: 'provider_unavailable' }
if (err instanceof OriginDeniedError) return { status: 403, code: 'origin_denied' }
- if (err instanceof RequestLimitError) return { status: 503, code: 'provider_unavailable' }
return { status: attemptErrorStatus(err.code), code: err.code }
}
diff --git a/grok-llm-proxy/test/bodyAdmission.test.ts b/grok-llm-proxy/test/bodyAdmission.test.ts
index e8f1deca2..1dad2a548 100644
--- a/grok-llm-proxy/test/bodyAdmission.test.ts
+++ b/grok-llm-proxy/test/bodyAdmission.test.ts
@@ -354,7 +354,7 @@ describe('grok-llm-proxy body admission (#731 R3-2)', () => {
}
}, 30_000)
- it('T-R3-2c-grok answers provider_unavailable once the admission queue is full', async () => {
+ it('T-R3-2c-grok answers proxy_capacity_exceeded once the admission queue is full', async () => {
// A small body limit keeps this cheap; the budget scales with it.
const maxBodyBytes = 16 * 1024
const proxy = await heldProxy(maxBodyBytes)
@@ -374,16 +374,16 @@ describe('grok-llm-proxy body admission (#731 R3-2)', () => {
const overflow = await post(proxy.port, payload('overflow'))
// Witnesses: the refusal was answered, and the admission logged why.
expect(overflow.status).toBe(503)
- expect(JSON.parse(overflow.body)).toEqual({ error: 'provider_unavailable' })
+ expect(JSON.parse(overflow.body)).toEqual({ error: 'proxy_capacity_exceeded' })
const logged = warn.mock.calls.map(call => call[0] as unknown as Record<string, unknown>)
expect(logged.filter(entry => entry?.event === 'grok_proxy_denied')).toEqual([
- { event: 'grok_proxy_denied', code: 'provider_unavailable' },
+ { event: 'grok_proxy_denied', code: 'proxy_capacity_exceeded' },
])
expect(logged.filter(entry => entry?.event === 'grok_proxy_admission_refused')).toEqual([
{
event: 'grok_proxy_admission_refused',
reason: 'body_budget',
- code: 'provider_unavailable',
+ code: 'proxy_capacity_exceeded',
kind: 'queue_full',
detail: expect.any(String),
},
diff --git a/grok-llm-proxy/test/contractFreeze.test.ts b/grok-llm-proxy/test/contractFreeze.test.ts
index 3e2d1d647..a2669b896 100644
--- a/grok-llm-proxy/test/contractFreeze.test.ts
+++ b/grok-llm-proxy/test/contractFreeze.test.ts
@@ -1,7 +1,8 @@
import { describe, expect, it } from 'vitest'
-import { readdirSync, readFileSync } from 'node:fs'
+import { readFileSync, readdirSync } from 'node:fs'
import { dirname, join } from 'node:path'
import { fileURLToPath } from 'node:url'
+import ts from 'typescript'
import {
CATALOG_ORIGIN,
COMPLETIONS_ORIGIN,
@@ -127,13 +128,17 @@ function emittedTransportCodes(): { codes: Set<string>; sites: number; construct
for (const name of readdirSync(srcDir).filter(file => file.endsWith('.ts'))) {
const source = readFileSync(join(srcDir, name), 'utf8')
constructions +=
- source.split('new GrokTransportError(').length - 1 +
- source.split('new UpstreamTimeoutError(').length - 1
+ source.split('new GrokTransportError(').length -
+ 1 +
+ source.split('new UpstreamTimeoutError(').length -
+ 1
for (const match of source.matchAll(TRANSPORT_ERROR_SITE)) {
const literal = /^'([a-z][a-z0-9_]+)'$/.exec(match[1]!)
// A computed code cannot be checked against the taxonomy, so it fails here.
- expect(literal, `${name}: transport error code must be a string literal, got ${match[1]}`)
- .not.toBeNull()
+ expect(
+ literal,
+ `${name}: transport error code must be a string literal, got ${match[1]}`
+ ).not.toBeNull()
codes.add(literal![1]!)
sites += 1
}
@@ -147,11 +152,71 @@ function emittedTransportCodes(): { codes: Set<string>; sites: number; construct
const REJECT_SITE = /reject\(res,\s*[^,()]+,\s*([^,()]+?)\s*(?:,\s*type\s*)?\)/g
// Refusal codes are wire strings; `Unauthorized` is the one spelled in capitals.
const REJECT_CODE_LITERAL = /^'([A-Za-z][A-Za-z0-9_]+)'$/
-// The only computed code a refusal may carry, as its exact source text:
+// The computed refusal forms are closed RequestLimitError codes and mapped errors.
// `mapped.code` is a transport or control-api code passed through mapError,
// whose transport half the scanner above already gates. Any other computed
// form cannot be checked against the taxonomy, so it fails here.
-const COMPUTED_REJECT_CODES = ['mapped.code'] as const
+const COMPUTED_REJECT_CODES = ['err.code', 'mapped.code'] as const
+
+/** Computed admission codes are allowed only through this closed, readonly type. */
+function requestLimitCodes(
+ sourceText = readFileSync(join(srcDir, 'requestLimits.ts'), 'utf8')
+): Set<string> {
+ const source = ts.createSourceFile('requestLimits.ts', sourceText, ts.ScriptTarget.Latest, true)
+ const alias = source.statements.find(
+ (node): node is ts.TypeAliasDeclaration =>
+ ts.isTypeAliasDeclaration(node) && node.name.text === 'RequestLimitCode'
+ )
+ if (!alias || !ts.isUnionTypeNode(alias.type)) {
+ throw new Error('RequestLimitCode must remain a closed literal union')
+ }
+ const codes = new Set(
+ alias.type.types.map(node => {
+ if (!ts.isLiteralTypeNode(node) || !ts.isStringLiteral(node.literal)) {
+ throw new Error('RequestLimitCode must not accept computed or open string types')
+ }
+ return node.literal.text
+ })
+ )
+ const owner = source.statements.find(
+ (node): node is ts.ClassDeclaration =>
+ ts.isClassDeclaration(node) && node.name?.text === 'RequestLimitError'
+ )
+ const property = owner?.members.find(
+ (node): node is ts.PropertyDeclaration =>
+ ts.isPropertyDeclaration(node) && node.name.getText(source) === 'code'
+ )
+ expect(property?.type?.getText(source)).toBe('RequestLimitCode')
+ expect(property?.modifiers?.some(node => node.kind === ts.SyntaxKind.ReadonlyKeyword)).toBe(true)
+ const constructor = owner?.members.find(ts.isConstructorDeclaration)
+ const input = constructor?.parameters.find(node => node.name.getText(source) === 'code')
+ expect(input?.type?.getText(source)).toBe('RequestLimitCode')
+ expect(input?.initializer?.getText(source)).toBe("'provider_unavailable'")
+ const assignments: string[] = []
+ const collect = (node: ts.Node): void => {
+ if (
+ ts.isBinaryExpression(node) &&
+ node.operatorToken.kind === ts.SyntaxKind.EqualsToken &&
+ node.left.getText(source) === 'this.code'
+ ) {
+ assignments.push(node.right.getText(source))
+ }
+ ts.forEachChild(node, collect)
+ }
+ if (owner) collect(owner)
+ expect(assignments).toEqual(['code'])
+ return codes
+}
+
+function rejectCodeLiteral(code: string, name: string): RegExpExecArray | null {
+ const literal = REJECT_CODE_LITERAL.exec(code)
+ const allowed = (COMPUTED_REJECT_CODES as readonly string[]).includes(code)
+ expect(
+ literal !== null || allowed,
+ `${name}: reject code must be a string literal or one of ${COMPUTED_REJECT_CODES.join(', ')}, got ${code}`
+ ).toBe(true)
+ return literal
+}
function emittedRejectCodes(): {
codes: Set<string>
@@ -159,7 +224,7 @@ function emittedRejectCodes(): {
sites: number
occurrences: number
} {
- const codes = new Set<string>()
+ const codes = requestLimitCodes()
const computed = new Set<string>()
let sites = 0
let occurrences = 0
@@ -168,12 +233,7 @@ function emittedRejectCodes(): {
occurrences += source.split('reject(res,').length - 1
for (const match of source.matchAll(REJECT_SITE)) {
const code = match[1]!
- const literal = REJECT_CODE_LITERAL.exec(code)
- const allowed = (COMPUTED_REJECT_CODES as readonly string[]).includes(code)
- expect(
- literal !== null || allowed,
- `${name}: reject code must be a string literal or one of ${COMPUTED_REJECT_CODES.join(', ')}, got ${code}`
- ).toBe(true)
+ const literal = rejectCodeLiteral(code, name)
if (literal) codes.add(literal[1]!)
else computed.add(code)
sites += 1
@@ -183,6 +243,42 @@ function emittedRejectCodes(): {
}
describe('grok-subscription contract freeze', () => {
+ it('rejects open or mutable admission code producers while reading the closed production union', () => {
+ const sourceText = readFileSync(join(srcDir, 'requestLimits.ts'), 'utf8')
+ const tree = ts.createSourceFile('requestLimits.ts', sourceText, ts.ScriptTarget.Latest, true)
+ const alias = tree.statements.find(
+ (node): node is ts.TypeAliasDeclaration =>
+ ts.isTypeAliasDeclaration(node) && node.name.text === 'RequestLimitCode'
+ )!
+ const withType = (type: string) =>
+ sourceText.slice(0, alias.type.getStart(tree)) + type + sourceText.slice(alias.type.end)
+ expect([...requestLimitCodes(sourceText)]).toEqual(
+ expect.arrayContaining(['provider_unavailable', 'proxy_capacity_exceeded', 'visual_gate'])
+ )
+ expect(() => requestLimitCodes(withType('string'))).toThrow('closed literal union')
+ expect(() => requestLimitCodes(withType("'provider_unavailable' | string"))).toThrow(
+ 'open string types'
+ )
+ expect(() =>
+ requestLimitCodes(
+ sourceText.replace('readonly code: RequestLimitCode', 'code: RequestLimitCode')
+ )
+ ).toThrow()
+ expect(() =>
+ requestLimitCodes(sourceText.replace('this.code = code', "this.code = 'arbitrary_code'"))
+ ).toThrow()
+ })
+
+ it.each(['caller.code', "err.code || 'provider_unavailable'", 'String(err.code)'])(
+ 'rejects unbounded computed refusal expression %s',
+ expression => {
+ expect(rejectCodeLiteral("'request_timeout'", 'fixture')).not.toBeNull()
+ expect(rejectCodeLiteral('err.code', 'fixture')).toBeNull()
+ expect(rejectCodeLiteral('mapped.code', 'fixture')).toBeNull()
+ expect(() => rejectCodeLiteral(expression, 'fixture')).toThrow('reject code must be')
+ }
+ )
+
it('imports the origin-policy constants the proxy actually enforces', () => {
expect(GROK_COMPLETIONS_ORIGIN).toBe(COMPLETIONS_ORIGIN)
expect(GROK_CATALOG_ORIGIN).toBe(CATALOG_ORIGIN)
diff --git a/grok-llm-proxy/test/server.security.test.ts b/grok-llm-proxy/test/server.security.test.ts
index 0800c266f..0c9be5a13 100644
--- a/grok-llm-proxy/test/server.security.test.ts
+++ b/grok-llm-proxy/test/server.security.test.ts
@@ -201,7 +201,9 @@ describe('grok-llm-proxy security surface', () => {
})
it('reserves the visual transport budget for an authenticated ~30 MiB V2 body', async () => {
- const { runtimeApp, adminApp } = createProxyApps(config({ maxBodyBytes: DEFAULT_MAX_BODY_BYTES }))
+ const { runtimeApp, adminApp } = createProxyApps(
+ config({ maxBodyBytes: DEFAULT_MAX_BODY_BYTES })
+ )
// Deliberately invalid ticket: this test checks parser admission and the
// unchanged ticket gate, without redeeming or contacting any model. 30 MiB
// is past the Codex 24 MiB visual ceiling and inside the Grok 35 MiB one.
@@ -287,7 +289,9 @@ describe('grok-llm-proxy security surface', () => {
const acquire = vi.spyOn(visualStreamGate, 'acquire')
const visual = await send('x'.repeat(8_000_000), {
schemaVersion: 'grok-completion-request.v2',
- messages: [{ role: 'user', contentParts: [{ type: 'image', data: 'A'.repeat(1024 * 1024) }] }],
+ messages: [
+ { role: 'user', contentParts: [{ type: 'image', data: 'A'.repeat(1024 * 1024) }] },
+ ],
})
expect(visual.status).toBe(400)
expect(visual.body.error).toBe('invalid_request')
@@ -340,7 +344,10 @@ describe('grok-llm-proxy security surface', () => {
const { runtimeApp, probeApp } = createProxyApps(config({ maxBodyBytes: 65_536 }), {
streamCompletion: async () => {
streamCalls += 1
- throw new GrokTransportError('payload_too_large', 'request exceeds maxVisualRequestBodyBytes')
+ throw new GrokTransportError(
+ 'payload_too_large',
+ 'request exceeds maxVisualRequestBodyBytes'
+ )
},
})
const res = await request(runtimeApp)
@@ -352,7 +359,9 @@ describe('grok-llm-proxy security surface', () => {
expect(res.status).toBe(413)
expect(res.body).toEqual({ error: 'payload_too_large' })
const metricsText = (await request(probeApp).get('/metrics')).text
- expect(metricsText).toMatch(/^grok_proxy_attempt_failures_total\{code="payload_too_large"\} 1$/m)
+ expect(metricsText).toMatch(
+ /^grok_proxy_attempt_failures_total\{code="payload_too_large"\} 1$/m
+ )
})
it('does not let a V2 declaration raise the non-image budget to 35 MiB', async () => {
@@ -854,10 +863,12 @@ describe('grok-llm-proxy startup config', () => {
loadConfig({ ...base, GROK_LLM_PROXY_MAX_BODY_BYTES: String(LIMITS.maxRequestBodyBytes) })
).toThrow(/GROK_LLM_PROXY_MAX_BODY_BYTES must be at least/)
// Witness: the floor itself and any larger value load.
- expect(loadConfig({ ...base, GROK_LLM_PROXY_MAX_BODY_BYTES: String(floor) }).maxBodyBytes).toBe(floor)
- expect(loadConfig({ ...base, GROK_LLM_PROXY_MAX_BODY_BYTES: String(floor + 1) }).maxBodyBytes).toBe(
- floor + 1
+ expect(loadConfig({ ...base, GROK_LLM_PROXY_MAX_BODY_BYTES: String(floor) }).maxBodyBytes).toBe(
+ floor
)
+ expect(
+ loadConfig({ ...base, GROK_LLM_PROXY_MAX_BODY_BYTES: String(floor + 1) }).maxBodyBytes
+ ).toBe(floor + 1)
})
it('defaults the visual body limit to the contract visual ceiling, with no allowance on top', () => {
@@ -1266,7 +1277,10 @@ describe('grok-llm-proxy attempt telemetry', () => {
// Denies the redeem after `delayMs`, which is longer than the heartbeat
// interval the caller configures.
- function slowDenyingClient(code: string, delayMs: number): {
+ function slowDenyingClient(
+ code: string,
+ delayMs: number
+ ): {
client: ControlApiClient
redeemCalls: () => number
} {
@@ -1661,6 +1675,33 @@ describe('grok-llm-proxy attempt telemetry', () => {
expect('err' in (lines[0] ?? {})).toBe(false)
})
+ it.each(['visual_host_share', 'visual_gate', 'proxy_capacity_exceeded'])(
+ 'normalizes real transport upstream 503 claiming %s to provider_unavailable',
+ async code => {
+ const fetchFn = vi
+ .fn<typeof fetch>()
+ .mockResolvedValue(
+ Response.json({ error: code, reason: 'local capacity claim' }, { status: 503 })
+ )
+ const { res, receipts, lines, metricsText } = await run({
+ providerAttemptId: 'att-spoofed-local',
+ fetchFn,
+ })
+ expect(fetchFn).toHaveBeenCalledTimes(1)
+ expect(res.status).toBe(503)
+ expect(res.body).toEqual({ error: 'provider_unavailable' })
+ expect(receipts).toEqual([expect.objectContaining({ outcome: 'error' })])
+ expect(lines).toHaveLength(1)
+ expect(lines[0]).toMatchObject({
+ code: 'provider_unavailable',
+ deliveredAs: 'http_status',
+ httpStatus: 503,
+ })
+ expect(failureCount(metricsText, 'provider_unavailable')).toBe(1)
+ expect(failureCount(metricsText, 'request_limit')).toBe(0)
+ }
+ )
+
it('(g1-1a) answers 429 rate_limited and forwards a valid upstream Retry-After', async () => {
const { res, receipts, lines, metricsText } = await run({
providerAttemptId: 'att-rate-http',
@@ -1732,7 +1773,9 @@ describe('grok-llm-proxy attempt telemetry', () => {
// Witness: a keepalive went out first, so only the frame can carry it.
expect(keepaliveCount(res.text)).toBeGreaterThanOrEqual(1)
expect(
- res.text.endsWith('data: {"type":"error","code":"upstream_rejected","upstreamStatus":402}\n\n')
+ res.text.endsWith(
+ 'data: {"type":"error","code":"upstream_rejected","upstreamStatus":402}\n\n'
+ )
).toBe(true)
expect(lines).toHaveLength(1)
expect(lines[0]).toMatchObject({
@@ -2143,20 +2186,24 @@ describe('grok-llm-proxy attempt telemetry', () => {
it('(rl1) logs the request-limit reason and labels its failure metric request_limit', async () => {
const acquire = vi
.spyOn(streamGate, 'acquire')
- .mockRejectedValueOnce(new RequestLimitError('stream queue is full', 'queue_full'))
+ .mockRejectedValueOnce(
+ new RequestLimitError('stream queue is full', 'queue_full', 'proxy_capacity_exceeded')
+ )
try {
- const { res, receipts, lines, metricsText } = await run({ providerAttemptId: 'att-queue-full' })
+ const { res, receipts, lines, metricsText } = await run({
+ providerAttemptId: 'att-queue-full',
+ })
// Witness: the refusal came from the stream gate this test replaced.
expect(acquire).toHaveBeenCalledTimes(1)
expect(res.status).toBe(503)
- expect(res.body).toEqual({ error: 'provider_unavailable' })
+ expect(res.body).toEqual({ error: 'proxy_capacity_exceeded' })
// The gate refused before the redeem, so there is no receipt to finalize.
expect(receipts).toEqual([])
expect(lines).toHaveLength(1)
expect(lines[0]).toMatchObject({
providerAttemptId: 'att-queue-full',
outcome: 'failed',
- code: 'provider_unavailable',
+ code: 'proxy_capacity_exceeded',
reason: 'stream queue is full',
deliveredAs: 'http_status',
httpStatus: 503,
@@ -2220,9 +2267,7 @@ describe('grok-llm-proxy attempt telemetry', () => {
})
expect(res.status).toBe(200)
expect(keepaliveCount(res.text)).toBeGreaterThanOrEqual(1)
- expect(res.text.endsWith('data: {"type":"error","code":"provider_unavailable"}\n\n')).toBe(
- true
- )
+ expect(res.text.endsWith('data: {"type":"error","code":"provider_unavailable"}\n\n')).toBe(true)
expect(lines).toHaveLength(1)
expect(lines[0]).toMatchObject({
outcome: 'failed',
@@ -2513,7 +2558,10 @@ describe('grok-llm-proxy ticket-aware stream-gate wait (#739 D1-bis)', () => {
}, 30_000)
/** A V2 envelope past `maxBodyBytes`, so it takes the visual gate. */
- function visualEnvelope(providerAttemptId: string, ticketLifeMs: number): Record<string, unknown> {
+ function visualEnvelope(
+ providerAttemptId: string,
+ ticketLifeMs: number
+ ): Record<string, unknown> {
const raw: Record<string, unknown> = {
schemaVersion: 'grok-completion-request.v2',
requestId: `req-${providerAttemptId}`,
@@ -2565,7 +2613,10 @@ describe('grok-llm-proxy ticket-aware stream-gate wait (#739 D1-bis)', () => {
.set('Authorization', `Bearer ${platformToken()}`)
.send(body)
.then(res => res)
- await until(() => visualStreamGate.snapshot().queued === 1, 'the body queueing at the visual gate')
+ await until(
+ () => visualStreamGate.snapshot().queued === 1,
+ 'the body queueing at the visual gate'
+ )
await vi.advanceTimersByTimeAsync(ticketLifeMs)
// Witness: the body was still waiting for the slot at the ticket's expiry.
@@ -2920,7 +2971,7 @@ describe('grok-llm-proxy body budget release on upstream acceptance (#739 D2)',
const client = open(proxy.port, payload)
await until(() => client.ended(), 'the refusal')
expect(client.status()).toBe(503)
- expect(JSON.parse(client.received())).toEqual({ error: 'provider_unavailable' })
+ expect(JSON.parse(client.received())).toEqual({ error: 'proxy_capacity_exceeded' })
await until(() => proxy.closes() === 1, "the response's close event")
expect(client.errors()).toEqual([])
expect(proxy.reservations()).toEqual([Buffer.byteLength(payload)])
@@ -3086,7 +3137,10 @@ describe('grok-llm-proxy graceful drain on shutdown (#739 D6)', () => {
req.end(payload)
let closing: Promise<void> | undefined
try {
- await until(() => received.includes('data: {"type":"text","text":"t0"}'), 'the first SSE frame')
+ await until(
+ () => received.includes('data: {"type":"text","text":"t0"}'),
+ 'the first SSE frame'
+ )
closing = apps.close().then(() => {
order.push('closed')
})
diff --git a/grok-llm-proxy/test/streamGate.handoff.test.ts b/grok-llm-proxy/test/streamGate.handoff.test.ts
index 194c8e2fb..2a9dec879 100644
--- a/grok-llm-proxy/test/streamGate.handoff.test.ts
+++ b/grok-llm-proxy/test/streamGate.handoff.test.ts
@@ -357,12 +357,12 @@ describe('grok visual stream-gate handoff', () => {
const warn = vi.spyOn(logger, 'warn')
const overflow = await postCompletion(port, 'grok-completion-request.v2', largeContent)
expect(overflow.status).toBe(503)
- expect(await overflow.json()).toEqual({ error: 'provider_unavailable' })
+ expect(await overflow.json()).toEqual({ error: 'visual_gate' })
expect(warn).toHaveBeenCalledWith(
expect.objectContaining({
event: 'grok_proxy_admission_refused',
reason: 'visual_gate',
- code: 'provider_unavailable',
+ code: 'visual_gate',
}),
'admission refused'
)
@@ -470,7 +470,7 @@ describe('grok visual stream-gate handoff', () => {
)
const third = await postCompletion(port, 'grok-completion-request.v2', largeContent)
expect(third.status).toBe(503)
- expect(await third.json()).toEqual({ error: 'provider_unavailable' })
+ expect(await third.json()).toEqual({ error: 'visual_host_share' })
expect(warn).toHaveBeenCalledWith(
{
event: 'grok_proxy_admission_refused',
@@ -605,7 +605,7 @@ describe('grok visual stream-gate handoff', () => {
const third = await postBody(port, platformToken(), invalidTicketBody(maxBodyBytes))
expect(third.status).toBe(503)
- expect(await third.json()).toEqual({ error: 'provider_unavailable' })
+ expect(await third.json()).toEqual({ error: 'visual_host_share' })
// The refusal names the principal it applied to, so an operator can tell
// which host was throttled.
expect(warn).toHaveBeenCalledWith(
@@ -684,7 +684,7 @@ describe('grok visual stream-gate handoff', () => {
for (let i = 0; i < targetCount; i += 1) {
const res = await postBody(port, platformToken(), invalidTicketBody(maxBodyBytes))
expect(res.status).toBe(503)
- expect(await res.json()).toEqual({ error: 'provider_unavailable' })
+ expect(await res.json()).toEqual({ error: 'visual_gate' })
}
expect(acquire).toHaveBeenCalledTimes(
1 + VISUAL_STREAM_LIMITS.maxQueuedRequests + targetCount
@@ -759,7 +759,7 @@ describe('grok visual stream-gate handoff', () => {
).toBe(1)
const third = await refusedRequest
expect(third.status).toBe(503)
- expect(await third.json()).toEqual({ error: 'provider_unavailable' })
+ expect(await third.json()).toEqual({ error: 'visual_host_share' })
expect(warn).toHaveBeenCalledWith(
{
event: 'grok_proxy_admission_refused',
diff --git a/grok-llm-proxy/test/visualBodyReadOwnership.test.ts b/grok-llm-proxy/test/visualBodyReadOwnership.test.ts
new file mode 100644
index 000000000..e32e5da43
--- /dev/null
+++ b/grok-llm-proxy/test/visualBodyReadOwnership.test.ts
@@ -0,0 +1,532 @@
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
+import express, { type NextFunction, type Request, type Response } from 'express'
+import jwt from 'jsonwebtoken'
+import { generateKeyPairSync } from 'node:crypto'
+import { createServer } from 'node:http'
+import { connect as connectTcp } from 'node:net'
+import type { GrokLlmProxyConfig } from '../src/config.js'
+import {
+ VISUAL_PER_HOST_MAX_ADMITTED,
+ VISUAL_STREAM_LIMITS,
+ visualStreamGate,
+} from '../src/requestLimits.js'
+import { createProxyApps } from '../src/server.js'
+
+type ObservedRead = {
+ req: Request
+ res: Response
+ nativeSettled: boolean
+ returned: boolean
+ error: unknown
+ finishes: number
+ endCalls: number
+}
+
+const parserBarrier = vi.hoisted(() => {
+ let release!: () => void
+ let completion: Promise<void>
+ const reads: ObservedRead[] = []
+ const reset = () => {
+ reads.splice(0)
+ completion = new Promise<void>(resolve => {
+ release = resolve
+ })
+ }
+ reset()
+ return {
+ reads,
+ reset,
+ get completion() {
+ return completion
+ },
+ get settledCount() {
+ return reads.filter(read => read.nativeSettled).length
+ },
+ release() {
+ release()
+ },
+ }
+})
+
+vi.mock('express', async () => {
+ const actual = await vi.importActual<typeof import('express')>('express')
+ const actualDefault = (actual as unknown as { default: typeof express }).default
+ const mockedDefault = Object.assign(
+ (...args: Parameters<typeof express>) => actualDefault(...args),
+ {
+ json: (options?: Parameters<typeof actual.json>[0]) => {
+ const parser = actualDefault.json(options)
+ return (req: Request, res: Response, next: NextFunction) => {
+ // Observe the native visual reader. Ordinary JSON parsing stays native
+ // too, but must not count as starting the next visual reader.
+ const read: ObservedRead | undefined =
+ options?.limit === 8192
+ ? {
+ req,
+ res,
+ nativeSettled: false,
+ returned: false,
+ error: undefined,
+ finishes: 0,
+ endCalls: 0,
+ }
+ : undefined
+ if (read) {
+ parserBarrier.reads.push(read)
+ res.once('finish', () => {
+ read.finishes += 1
+ })
+ if (req.headers['x-visual-never-finish'] === '1') {
+ // Fault injection: no finish/close is produced by writing 408.
+ // The production close backstop must independently stop raw-body.
+ res.end = (() => {
+ read.endCalls += 1
+ return res
+ }) as Response['end']
+ }
+ }
+ parser(req, res, (err: unknown) => {
+ if (read) {
+ read.nativeSettled = true
+ read.error = err
+ }
+ const finish = () => {
+ next(err)
+ if (read) read.returned = true
+ }
+ if (req.headers['x-visual-parser-hold'] === '1') {
+ void parserBarrier.completion.then(finish)
+ } else {
+ finish()
+ }
+ })
+ }
+ },
+ }
+ )
+ return { ...actual, default: mockedDefault, json: mockedDefault.json }
+})
+
+const COMPLETIONS_PATH = '/internal/runtime/v1/grok/completions'
+const { privateKey, publicKey } = generateKeyPairSync('rsa', {
+ modulusLength: 2048,
+ publicKeyEncoding: { type: 'spki', format: 'pem' },
+ privateKeyEncoding: { type: 'pkcs8', format: 'pem' },
+})
+
+function config(): GrokLlmProxyConfig {
+ return {
+ runtimePort: 0,
+ adminPort: 0,
+ probePort: 0,
+ maxBodyBytes: 4096,
+ maxVisualBodyBytes: 8192,
+ maxStreamDurationMs: 1_800_000,
+ maxDeadlineMs: 1_800_000,
+ upstreamIdleTimeoutMs: 600_000,
+ heartbeatIntervalMs: 15_000,
+ jwtIssuer: 'control-api',
+ jwtPublicKey: publicKey,
+ executionEnabled: true,
+ controlApiBaseUrl: '',
+ controlApiServiceName: 'grok-llm-proxy',
+ controlApiServiceToken: '',
+ }
+}
+
+function platformToken(): string {
+ return jwt.sign(
+ {
+ sub: 'default/research-host',
+ hostRefs: ['research-host'],
+ workflowControlScopes: ['llm:grok:execute'],
+ scope: 'workflow:approval:request',
+ },
+ privateKey,
+ { algorithm: 'RS256', issuer: 'control-api', audience: 'workflow-approvals', expiresIn: 60 }
+ )
+}
+
+function platformTokenForOtherPrincipal(): string {
+ return jwt.sign(
+ {
+ sub: 'default/other-host',
+ hostRefs: ['research-host'],
+ workflowControlScopes: ['llm:grok:execute'],
+ scope: 'workflow:approval:request',
+ },
+ privateKey,
+ { algorithm: 'RS256', issuer: 'control-api', audience: 'workflow-approvals', expiresIn: 60 }
+ )
+}
+
+async function waitFor(predicate: () => boolean, label: string): Promise<void> {
+ const started = Date.now()
+ while (!predicate()) {
+ if (Date.now() - started > 5_000) throw new Error(label)
+ await new Promise(resolve => setTimeout(resolve, 5))
+ }
+}
+
+const sockets = new Set<ReturnType<typeof connectTcp>>()
+
+function stalledVisualBody(
+ port: number,
+ declared: number,
+ holdParser = false,
+ extraHeaders = '',
+ prefix = '{"executionTicket":'
+): { socket: ReturnType<typeof connectTcp>; response: Promise<number> } {
+ let owner: ReturnType<typeof connectTcp> | undefined
+ const response = new Promise<number>((resolve, reject) => {
+ owner = connectTcp(port, '127.0.0.1', () => {
+ owner?.write(
+ `POST ${COMPLETIONS_PATH} HTTP/1.1\r\n` +
+ `Host: 127.0.0.1:${port}\r\n` +
+ `Authorization: Bearer ${platformToken()}\r\n` +
+ `Content-Type: application/json\r\n` +
+ (holdParser ? 'X-Visual-Parser-Hold: 1\r\n' : '') +
+ extraHeaders +
+ `Content-Length: ${declared}\r\n` +
+ `\r\n` +
+ prefix
+ )
+ })
+ sockets.add(owner)
+ owner.once('close', () => sockets.delete(owner!))
+ owner.setTimeout(5_000, () => {
+ owner?.destroy()
+ reject(new Error('the stalled visual body produced no response'))
+ })
+ const chunks: Buffer[] = []
+ owner.on('data', data => {
+ chunks.push(data)
+ const match = /^HTTP\/1\.1 (\d+)/.exec(Buffer.concat(chunks).toString('utf8'))
+ if (match) resolve(Number(match[1]))
+ })
+ owner.on('error', err => {
+ if ((err as NodeJS.ErrnoException).code !== 'ECONNRESET') reject(err)
+ })
+ })
+ return { socket: owner!, response }
+}
+
+function invalidTicketBody(): string {
+ const content = 'x'.repeat(5000)
+ return JSON.stringify({
+ executionTicket: 'invalid-ticket',
+ requestHash: 'a'.repeat(64),
+ request: {
+ schemaVersion: 'grok-completion-request.v2',
+ requestId: 'req-visual-ownership',
+ idempotencyKey: 'idem-visual-ownership',
+ provider: 'grok-subscription',
+ model: 'grok-4.6',
+ messages: [{ role: 'user', content }],
+ },
+ })
+}
+
+function postVisualRaw(
+ port: number,
+ body: string,
+ holdParser = false
+): { socket: ReturnType<typeof connectTcp>; response: Promise<number> } {
+ let owner: ReturnType<typeof connectTcp> | undefined
+ const response = new Promise<number>((resolve, reject) => {
+ owner = connectTcp(port, '127.0.0.1', () => {
+ owner?.write(
+ `POST ${COMPLETIONS_PATH} HTTP/1.1\r\n` +
+ `Host: 127.0.0.1:${port}\r\n` +
+ `Authorization: Bearer ${platformToken()}\r\n` +
+ `Content-Type: application/json\r\n` +
+ (holdParser ? 'X-Visual-Parser-Hold: 1\r\n' : '') +
+ `Content-Length: ${Buffer.byteLength(body)}\r\n` +
+ `\r\n` +
+ body
+ )
+ })
+ sockets.add(owner)
+ owner.once('close', () => sockets.delete(owner!))
+ owner.setTimeout(5_000, () => {
+ owner?.destroy()
+ reject(new Error('the visual request produced no response'))
+ })
+ const chunks: Buffer[] = []
+ owner.on('data', data => {
+ chunks.push(data)
+ const match = /^HTTP\/1\.1 (\d+)/.exec(Buffer.concat(chunks).toString('utf8'))
+ if (match) resolve(Number(match[1]))
+ })
+ owner.on('error', err => {
+ if ((err as NodeJS.ErrnoException).code !== 'ECONNRESET') reject(err)
+ })
+ })
+ return { socket: owner!, response }
+}
+
+async function postVisualStatus(port: number, body: string): Promise<number> {
+ const request = postVisualRaw(port, body)
+ const status = await request.response
+ request.socket.destroy()
+ return status
+}
+
+describe('grok visual body read ownership', () => {
+ const servers: Array<ReturnType<typeof createProxyApps>> = []
+ const listeners: Array<ReturnType<typeof createServer>> = []
+
+ beforeEach(() => parserBarrier.reset())
+
+ afterEach(async () => {
+ parserBarrier.release()
+ vi.restoreAllMocks()
+ for (const socket of sockets) socket.destroy()
+ await Promise.all(
+ listeners
+ .splice(0)
+ .map(
+ listener =>
+ new Promise<void>((resolve, reject) =>
+ listener.close(err => (err ? reject(err) : resolve()))
+ )
+ )
+ )
+ await Promise.all(servers.splice(0).map(server => server.close()))
+ await waitFor(
+ () => visualStreamGate.snapshot().running === 0 && visualStreamGate.snapshot().queued === 0,
+ 'the visual gate did not drain after the ownership test'
+ )
+ })
+
+ it('keeps the visual slot and principal share until the timed-out parser terminates', async () => {
+ const apps = createProxyApps(config(), {
+ bodyReadDeadlineMs: 200,
+ visualReadCloseGraceMs: 10,
+ })
+ servers.push(apps)
+ const listener = createServer(apps.runtimeApp).listen(0)
+ listeners.push(listener)
+ const address = listener.address()
+ if (!address || typeof address === 'string') throw new Error('listener has no port')
+
+ const readers = Array.from({ length: VISUAL_PER_HOST_MAX_ADMITTED }, () =>
+ stalledVisualBody(address.port, 5000, true)
+ )
+ await waitFor(
+ () =>
+ visualStreamGate.snapshot().running + visualStreamGate.snapshot().queued ===
+ VISUAL_PER_HOST_MAX_ADMITTED,
+ 'the stalled visual bodies did not fill the principal share'
+ )
+ const statuses = await Promise.all(
+ readers.slice(0, VISUAL_STREAM_LIMITS.maxConcurrentStreams).map(reader => reader.response)
+ )
+ expect(statuses).toEqual(
+ Array.from({ length: VISUAL_STREAM_LIMITS.maxConcurrentStreams }, () => 408)
+ )
+ expect(visualStreamGate.snapshot().running + visualStreamGate.snapshot().queued).toBe(
+ VISUAL_PER_HOST_MAX_ADMITTED
+ )
+
+ const shareRefusal = await postVisualStatus(address.port, invalidTicketBody())
+ expect(shareRefusal).toBe(503)
+ // A different principal can queue, but the reader must not start until the
+ // existing parser callbacks finish. This observes reader entry, not merely
+ // the absence of an HTTP response from the next request.
+ const nextSocket = connectTcp(address.port, '127.0.0.1')
+ sockets.add(nextSocket)
+ nextSocket.once('close', () => sockets.delete(nextSocket))
+ const nextStatus = new Promise<number>((resolve, reject) => {
+ nextSocket.once('connect', () => {
+ const body = invalidTicketBody()
+ nextSocket.write(
+ `POST ${COMPLETIONS_PATH} HTTP/1.1\r\n` +
+ `Host: 127.0.0.1:${address.port}\r\n` +
+ `Authorization: Bearer ${platformTokenForOtherPrincipal()}\r\n` +
+ `Content-Type: application/json\r\n` +
+ `Content-Length: ${Buffer.byteLength(body)}\r\n\r\n${body}`
+ )
+ })
+ nextSocket.on('data', data => {
+ const match = /^HTTP\/1\.1 (\d+)/.exec(data.toString('utf8'))
+ if (match) resolve(Number(match[1]))
+ })
+ nextSocket.once('error', reject)
+ })
+ await waitFor(
+ () =>
+ visualStreamGate.snapshot().queued ===
+ VISUAL_PER_HOST_MAX_ADMITTED - VISUAL_STREAM_LIMITS.maxConcurrentStreams + 1,
+ 'the next principal did not queue behind the retained parsers'
+ )
+ expect(parserBarrier.reads).toHaveLength(VISUAL_STREAM_LIMITS.maxConcurrentStreams)
+
+ for (const reader of readers) {
+ reader.response.catch(() => undefined)
+ reader.socket.destroy()
+ }
+ await waitFor(
+ () => parserBarrier.settledCount === VISUAL_STREAM_LIMITS.maxConcurrentStreams,
+ 'the held parsers did not terminate'
+ )
+ expect(visualStreamGate.snapshot().running).toBe(VISUAL_STREAM_LIMITS.maxConcurrentStreams)
+ expect(visualStreamGate.snapshot().queued).toBeGreaterThanOrEqual(1)
+ expect(parserBarrier.reads).toHaveLength(VISUAL_STREAM_LIMITS.maxConcurrentStreams)
+ parserBarrier.release()
+ expect(await nextStatus).toBe(403)
+ nextSocket.destroy()
+ await waitFor(
+ () => visualStreamGate.snapshot().running === 0 && visualStreamGate.snapshot().queued === 0,
+ 'the timed-out visual reader did not terminate and release its admission'
+ )
+ expect(
+ parserBarrier.reads
+ .slice(0, VISUAL_STREAM_LIMITS.maxConcurrentStreams)
+ .every(read => read.returned && read.req.body === undefined)
+ ).toBe(true)
+ })
+
+ it('uses the close backstop when the visual client disappears before the deadline', async () => {
+ const apps = createProxyApps(config(), {
+ bodyReadDeadlineMs: 5_000,
+ visualReadCloseGraceMs: 10,
+ })
+ servers.push(apps)
+ const listener = createServer(apps.runtimeApp).listen(0)
+ listeners.push(listener)
+ const address = listener.address()
+ if (!address || typeof address === 'string') throw new Error('listener has no port')
+
+ const stalled = stalledVisualBody(address.port, 5000)
+ await waitFor(
+ () => visualStreamGate.snapshot().running === 1,
+ 'the visual body did not take the slot'
+ )
+ stalled.socket.destroy()
+ stalled.response.catch(() => undefined)
+ await waitFor(
+ () => visualStreamGate.snapshot().running === 0 && visualStreamGate.snapshot().queued === 0,
+ 'a disconnected visual reader retained its admission after the close backstop'
+ )
+ })
+
+ it('releases the visual admission after a parser error', async () => {
+ const apps = createProxyApps(config())
+ servers.push(apps)
+ const listener = createServer(apps.runtimeApp).listen(0)
+ listeners.push(listener)
+ const address = listener.address()
+ if (!address || typeof address === 'string') throw new Error('listener has no port')
+
+ const status = await postVisualStatus(address.port, `{"executionTicket":${' '.repeat(5000)}`)
+ expect(status).toBe(400)
+ await waitFor(
+ () => visualStreamGate.snapshot().running === 0 && visualStreamGate.snapshot().queued === 0,
+ 'a malformed visual body did not release its admission'
+ )
+ })
+
+ it('transfers a successfully parsed visual body to the route and releases it after the response', async () => {
+ const apps = createProxyApps(config())
+ servers.push(apps)
+ const listener = createServer(apps.runtimeApp).listen(0)
+ listeners.push(listener)
+ const address = listener.address()
+ if (!address || typeof address === 'string') throw new Error('listener has no port')
+
+ const status = await postVisualStatus(address.port, invalidTicketBody())
+ expect(status).toBe(403)
+ await waitFor(
+ () => visualStreamGate.snapshot().running === 0 && visualStreamGate.snapshot().queued === 0,
+ 'a parsed visual body did not release its admission after the route response'
+ )
+ })
+ it('stops the native reader independently when a non-reading peer never finishes the response', async () => {
+ const apps = createProxyApps(config(), { bodyReadDeadlineMs: 40, visualReadCloseGraceMs: 15 })
+ servers.push(apps)
+ const listener = createServer(apps.runtimeApp).listen(0)
+ listeners.push(listener)
+ const address = listener.address()
+ if (!address || typeof address === 'string') throw new Error('listener has no port')
+
+ const stalled = stalledVisualBody(address.port, 5000, false, 'X-Visual-Never-Finish: 1\r\n')
+ stalled.response.catch(() => undefined)
+ stalled.socket.pause()
+ await waitFor(() => parserBarrier.reads.length === 1, 'the native visual reader did not start')
+ const read = parserBarrier.reads[0]!
+ await waitFor(() => read.returned, 'the independent close backstop did not terminate raw-body')
+ expect(read.endCalls).toBe(1)
+ expect(read.finishes).toBe(0)
+ expect(read.req.destroyed).toBe(true)
+ expect(read.req.body).toBeUndefined()
+ expect(visualStreamGate.snapshot()).toEqual({ running: 0, queued: 0 })
+ expect(await postVisualStatus(address.port, invalidTicketBody())).toBe(403)
+ })
+
+ it('clears a late successful native parse after its read deadline without dispatching it', async () => {
+ const apps = createProxyApps(config(), { bodyReadDeadlineMs: 40, visualReadCloseGraceMs: 1000 })
+ servers.push(apps)
+ const listener = createServer(apps.runtimeApp).listen(0)
+ listeners.push(listener)
+ const address = listener.address()
+ if (!address || typeof address === 'string') throw new Error('listener has no port')
+
+ const body = invalidTicketBody()
+ const split = 128
+ const stalled = stalledVisualBody(
+ address.port,
+ Buffer.byteLength(body),
+ false,
+ 'X-Visual-Never-Finish: 1\r\n',
+ body.slice(0, split)
+ )
+ stalled.response.catch(() => undefined)
+ await waitFor(
+ () => parserBarrier.reads[0]?.endCalls === 1,
+ 'the visual reader never reached its deadline'
+ )
+ const read = parserBarrier.reads[0]!
+ expect(read.nativeSettled).toBe(false)
+ expect(visualStreamGate.snapshot().running).toBe(1)
+ stalled.socket.write(body.slice(split))
+ await waitFor(() => read.returned, 'the late native parser callback did not run')
+ expect(read.error).toBeUndefined()
+ expect(read.req.body).toBeUndefined()
+ expect(visualStreamGate.snapshot()).toEqual({ running: 0, queued: 0 })
+ stalled.socket.destroy()
+ expect(await postVisualStatus(address.port, invalidTicketBody())).toBe(403)
+ })
+
+ it('clears the retained parsed body after response close before its callback returns', async () => {
+ const apps = createProxyApps(config(), { bodyReadDeadlineMs: 5000, visualReadCloseGraceMs: 15 })
+ servers.push(apps)
+ const listener = createServer(apps.runtimeApp).listen(0)
+ listeners.push(listener)
+ const address = listener.address()
+ if (!address || typeof address === 'string') throw new Error('listener has no port')
+
+ const request = postVisualRaw(address.port, invalidTicketBody(), true)
+ request.response.catch(() => undefined)
+ await waitFor(() => parserBarrier.settledCount === 1, 'the complete body was never parsed')
+ const read = parserBarrier.reads[0]!
+ expect(read.req.body).toBeDefined()
+ expect(read.returned).toBe(false)
+ request.socket.destroy()
+ await waitFor(() => read.res.destroyed, 'the response did not close')
+ expect(visualStreamGate.snapshot().running).toBe(1)
+ parserBarrier.release()
+ await waitFor(() => read.returned, 'the disconnected parser callback did not run')
+ expect(read.req.body).toBeUndefined()
+ expect(visualStreamGate.snapshot()).toEqual({ running: 0, queued: 0 })
+ expect(await postVisualStatus(address.port, invalidTicketBody())).toBe(403)
+ })
+
+ it.each([0, -1, 1.5, Number.NaN, Number.POSITIVE_INFINITY, 2147483648])(
+ 'rejects invalid native reader clocks (%s)',
+ value => {
+ expect(() => createProxyApps(config(), { bodyReadDeadlineMs: value })).toThrow(RangeError)
+ expect(() => createProxyApps(config(), { visualReadCloseGraceMs: value })).toThrow(RangeError)
+ }
+ )
+})
diff --git a/mcp-host/package-lock.json b/mcp-host/package-lock.json
index 7a702113a..3d9a15dd4 100644
--- a/mcp-host/package-lock.json
+++ b/mcp-host/package-lock.json
@@ -1,12 +1,12 @@
{
"name": "clerum-mcp-host",
- "version": "0.1.486",
+ "version": "0.1.487",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "clerum-mcp-host",
- "version": "0.1.486",
+ "version": "0.1.487",
"license": "SEE LICENSE IN LICENSE",
"dependencies": {
"@anthropic-ai/sdk": "^0.32.0",
diff --git a/mcp-host/package.json b/mcp-host/package.json
index 668c9959f..edc364693 100644
--- a/mcp-host/package.json
+++ b/mcp-host/package.json
@@ -1,6 +1,6 @@
{
"name": "clerum-mcp-host",
- "version": "0.1.486",
+ "version": "0.1.487",
"description": "MCP Host - reads Host CRD and provides LLM access via OpenAI/Claude",
"main": "dist/main.js",
"engines": {
diff --git a/mcp-host/src/llm/__tests__/codexLlmProxyClient.test.ts b/mcp-host/src/llm/__tests__/codexLlmProxyClient.test.ts
index 507a86543..a7bb1faab 100644
--- a/mcp-host/src/llm/__tests__/codexLlmProxyClient.test.ts
+++ b/mcp-host/src/llm/__tests__/codexLlmProxyClient.test.ts
@@ -10,6 +10,7 @@ import { LlmErrorCode } from '../../core/errors'
import {
CodexLlmProxyClient,
CodexProxyError,
+ codexProxyErrorMessage,
resolveCodexProxyRuntimeUrl,
} from '../codexLlmProxyClient'
import { CodexSubscriptionProvider } from '../codexSubscription'
@@ -435,6 +436,109 @@ describe('CodexLlmProxyClient', () => {
expect(classifyFailoverClass(classified.code, classified.retryable)).toBeNull()
})
+ it.each(['visual_host_share', 'visual_gate', 'proxy_capacity_exceeded'] as const)(
+ 'decodes the local %s admission code without failover and keeps the client usable',
+ async code => {
+ const input = {
+ executionTicket: 'ticket-123456',
+ requestHash: 'a'.repeat(64),
+ request: {},
+ }
+ const fetchFn = vi
+ .fn()
+ .mockResolvedValueOnce(
+ Response.json({ error: code, reason: 'untrusted upstream prose' }, { status: 503 })
+ )
+ .mockResolvedValueOnce(new Response(sse([{ type: 'done', outcome: 'success' }])))
+ const client = new CodexLlmProxyClient({
+ runtimeUrl: 'http://proxy/internal/runtime/v1/codex/completions',
+ readPlatformJwt: () => 'platform-jwt',
+ fetchFn,
+ })
+ const err = await client.stream(input).then(
+ () => undefined,
+ (e: unknown) => e
+ )
+ expect(fetchFn).toHaveBeenCalledTimes(1)
+ expect(err).toBeInstanceOf(CodexProxyError)
+ if (!(err instanceof CodexProxyError)) throw new Error('expected CodexProxyError')
+ expect(err).toMatchObject({
+ code,
+ message: codexProxyErrorMessage(code, 503),
+ dispatched: true,
+ })
+ expect(err.message).not.toContain('untrusted upstream prose')
+
+ const provider = new CodexSubscriptionProvider('gpt-5.3-codex', {} as never)
+ const classified = provider.classifyError(err)
+ expect(classified.code).toBe(LlmErrorCode.ApiCallFailed)
+ expect(classified.retryable).toBe(false)
+ expect(classified.providerCode).toBe(code)
+ expect(classifyFailoverClass(classified.code, classified.retryable)).toBeNull()
+
+ const next = await client.stream(input)
+ expect(next.outcome).toBe('success')
+ expect(fetchFn).toHaveBeenCalledTimes(2)
+
+ const outage = provider.classifyError(
+ new CodexProxyError('provider_unavailable', 'real upstream outage')
+ )
+ expect(outage.code).toBe(LlmErrorCode.ModelOverloaded)
+ expect(outage.retryable).toBe(true)
+ expect(classifyFailoverClass(outage.code, outage.retryable)).toBe('provider_unavailable')
+ }
+ )
+
+ it.each([
+ [
+ 'unknown_machine_code',
+ JSON.stringify({ error: 'future_proxy_error' }),
+ LlmErrorCode.ApiCallFailed,
+ false,
+ 'future_proxy_error',
+ ],
+ [
+ 'reason_only',
+ JSON.stringify({ reason: 'visual_gate' }),
+ LlmErrorCode.ModelOverloaded,
+ true,
+ 'provider_unavailable',
+ ],
+ ['malformed_json', '{', LlmErrorCode.ModelOverloaded, true, 'provider_unavailable'],
+ ] as const)(
+ 'keeps wire %s separate from trusted local refusal provenance',
+ async (_label, body, code, retryable, providerCode) => {
+ const fetchFn = vi
+ .fn()
+ .mockResolvedValue(
+ new Response(body, { status: 503, headers: { 'content-type': 'application/json' } })
+ )
+ const proxy = new CodexLlmProxyClient({
+ runtimeUrl: 'http://proxy/internal/runtime/v1/codex/completions',
+ readPlatformJwt: () => 'platform-jwt',
+ fetchFn,
+ })
+ const error = await proxy
+ .stream({
+ executionTicket: 'ticket-123456',
+ requestHash: 'a'.repeat(64),
+ request: {},
+ })
+ .then(
+ () => undefined,
+ (caught: unknown) => caught
+ )
+ expect(error).toBeInstanceOf(CodexProxyError)
+ expect(fetchFn).toHaveBeenCalledTimes(1)
+ const provider = new CodexSubscriptionProvider('gpt-5.3-codex', {} as never)
+ const classified = provider.classifyError(error)
+ expect(classified).toMatchObject({ code, retryable, providerCode })
+ expect(classifyFailoverClass(classified.code, classified.retryable)).toBe(
+ retryable ? 'provider_unavailable' : null
+ )
+ }
+ )
+
// T-TE-1 (D4) — the proxy passes control-api's redeem refusal through as
// `403 { error: 'ticket_expired' }` when the ticket died while the request
// was queued. Nothing reached the provider, so a re-authorized retry is the
diff --git a/mcp-host/src/llm/__tests__/codexSubscription.test.ts b/mcp-host/src/llm/__tests__/codexSubscription.test.ts
index 1ab985615..0bfb929ea 100644
--- a/mcp-host/src/llm/__tests__/codexSubscription.test.ts
+++ b/mcp-host/src/llm/__tests__/codexSubscription.test.ts
@@ -69,6 +69,37 @@ function deps(overrides?: {
}
describe('CodexSubscriptionProvider', () => {
+ it.each(['visual_host_share', 'visual_gate', 'proxy_capacity_exceeded'] as const)(
+ 'keeps the provider usable after a local %s admission refusal',
+ async code => {
+ const wired = deps({
+ stream: vi
+ .fn()
+ .mockRejectedValueOnce(
+ new CodexProxyError(code, 'proxy admission is full', { dispatched: true })
+ )
+ .mockResolvedValueOnce({ text: 'next text turn', toolCalls: [], outcome: 'success' }),
+ })
+ const provider = new CodexSubscriptionProvider('gpt-5.3-codex', wired as never)
+
+ await expect(
+ provider.completeSingleTurn([{ role: 'user', content: 'image' }])
+ ).rejects.toMatchObject({ code })
+ const result = await provider.completeSingleTurn([{ role: 'user', content: 'next text' }])
+ expect(result.content).toBe('next text turn')
+ expect(wired.authorize).toHaveBeenCalledTimes(2)
+ expect(wired.stream).toHaveBeenCalledTimes(2)
+
+ const classified = provider.classifyError(
+ new CodexProxyError(code, 'proxy admission is full', { dispatched: true })
+ )
+ expect(classified.code).toBe(LlmErrorCode.ApiCallFailed)
+ expect(classified.retryable).toBe(false)
+ expect(classified.providerCode).toBe(code)
+ expect(classifyFailoverClass(classified.code, classified.retryable)).toBeNull()
+ }
+ )
+
it.each(['unknown', 'canceled', 'error'])(
'never returns executable calls from a %s batch',
async outcome => {
diff --git a/mcp-host/src/llm/__tests__/grokLlmProxyClient.test.ts b/mcp-host/src/llm/__tests__/grokLlmProxyClient.test.ts
index fa3484335..c1cef922e 100644
--- a/mcp-host/src/llm/__tests__/grokLlmProxyClient.test.ts
+++ b/mcp-host/src/llm/__tests__/grokLlmProxyClient.test.ts
@@ -545,6 +545,90 @@ describe('GrokLlmProxyClient', () => {
expect(classifyFailoverClass(classified.code, classified.retryable)).toBeNull()
})
+ it.each(['visual_host_share', 'visual_gate', 'proxy_capacity_exceeded'] as const)(
+ 'decodes the local %s admission code without failover and keeps the client usable',
+ async code => {
+ const fetchFn = vi
+ .fn()
+ .mockResolvedValueOnce(
+ Response.json({ error: code, reason: 'untrusted upstream prose' }, { status: 503 })
+ )
+ .mockResolvedValueOnce(new Response(sse([{ type: 'done', outcome: 'success' }])))
+ const grok = client(fetchFn)
+ const err = await grok.stream(STREAM_INPUT).then(
+ () => undefined,
+ (e: unknown) => e
+ )
+ expect(fetchFn).toHaveBeenCalledTimes(1)
+ expect(err).toBeInstanceOf(GrokProxyError)
+ if (!(err instanceof GrokProxyError)) throw new Error('expected GrokProxyError')
+ expect(err).toMatchObject({
+ code,
+ message: grokProxyErrorMessage(code, 503),
+ dispatched: true,
+ })
+ expect(err.message).not.toContain('untrusted upstream prose')
+
+ const provider = new GrokSubscriptionProvider('grok-4.6', {} as never)
+ const classified = provider.classifyError(err)
+ expect(classified.code).toBe(LlmErrorCode.ApiCallFailed)
+ expect(classified.retryable).toBe(false)
+ expect(classified.providerCode).toBe(code)
+ expect(classifyFailoverClass(classified.code, classified.retryable)).toBeNull()
+
+ const next = await grok.stream(STREAM_INPUT)
+ expect(next.outcome).toBe('success')
+ expect(fetchFn).toHaveBeenCalledTimes(2)
+
+ const outage = provider.classifyError(
+ new GrokProxyError('provider_unavailable', 'real upstream outage')
+ )
+ expect(outage.code).toBe(LlmErrorCode.ModelOverloaded)
+ expect(outage.retryable).toBe(true)
+ expect(classifyFailoverClass(outage.code, outage.retryable)).toBe('provider_unavailable')
+ }
+ )
+
+ it.each([
+ [
+ 'unknown_machine_code',
+ JSON.stringify({ error: 'future_proxy_error' }),
+ LlmErrorCode.ApiCallFailed,
+ false,
+ 'future_proxy_error',
+ ],
+ [
+ 'reason_only',
+ JSON.stringify({ reason: 'visual_gate' }),
+ LlmErrorCode.ModelOverloaded,
+ true,
+ 'provider_unavailable',
+ ],
+ ['malformed_json', '{', LlmErrorCode.ModelOverloaded, true, 'provider_unavailable'],
+ ] as const)(
+ 'keeps wire %s separate from trusted local refusal provenance',
+ async (_label, body, code, retryable, providerCode) => {
+ const fetchFn = vi
+ .fn()
+ .mockResolvedValue(
+ new Response(body, { status: 503, headers: { 'content-type': 'application/json' } })
+ )
+ const proxy = client(fetchFn)
+ const error = await proxy.stream(STREAM_INPUT).then(
+ () => undefined,
+ (caught: unknown) => caught
+ )
+ expect(error).toBeInstanceOf(GrokProxyError)
+ expect(fetchFn).toHaveBeenCalledTimes(1)
+ const provider = new GrokSubscriptionProvider('grok-4.6', {} as never)
+ const classified = provider.classifyError(error)
+ expect(classified).toMatchObject({ code, retryable, providerCode })
+ expect(classifyFailoverClass(classified.code, classified.retryable)).toBe(
+ retryable ? 'provider_unavailable' : null
+ )
+ }
+ )
+
// T-TE-1 (D4) — the proxy passes control-api's redeem refusal through as
// `403 { error: 'ticket_expired' }` when the ticket died while the request
// was queued. Nothing reached the provider, so a re-authorized retry is the
diff --git a/mcp-host/src/llm/__tests__/grokSubscription.test.ts b/mcp-host/src/llm/__tests__/grokSubscription.test.ts
index ff97676d7..14d9581ec 100644
--- a/mcp-host/src/llm/__tests__/grokSubscription.test.ts
+++ b/mcp-host/src/llm/__tests__/grokSubscription.test.ts
@@ -47,6 +47,8 @@ function deps(overrides?: {
return {
authorizer: { authorize },
proxy: { stream },
+ authorize,
+ stream,
attemptContext: vi.fn(() => ({
policyRevision: 1,
policyHash: 'b'.repeat(64),
@@ -56,6 +58,37 @@ function deps(overrides?: {
}
describe('GrokSubscriptionProvider', () => {
+ it.each(['visual_host_share', 'visual_gate', 'proxy_capacity_exceeded'] as const)(
+ 'keeps the provider usable after a local %s admission refusal',
+ async code => {
+ const wired = deps({
+ stream: vi
+ .fn()
+ .mockRejectedValueOnce(
+ new GrokProxyError(code, 'proxy admission is full', { dispatched: true })
+ )
+ .mockResolvedValueOnce({ text: 'next text turn', toolCalls: [], outcome: 'success' }),
+ })
+ const provider = new GrokSubscriptionProvider('grok-4.6', wired as never)
+
+ await expect(
+ provider.completeSingleTurn([{ role: 'user', content: 'image' }])
+ ).rejects.toMatchObject({ code })
+ const result = await provider.completeSingleTurn([{ role: 'user', content: 'next text' }])
+ expect(result.content).toBe('next text turn')
+ expect(wired.authorize).toHaveBeenCalledTimes(2)
+ expect(wired.stream).toHaveBeenCalledTimes(2)
+
+ const classified = provider.classifyError(
+ new GrokProxyError(code, 'proxy admission is full', { dispatched: true })
+ )
+ expect(classified.code).toBe(LlmErrorCode.ApiCallFailed)
+ expect(classified.retryable).toBe(false)
+ expect(classified.providerCode).toBe(code)
+ expect(classifyFailoverClass(classified.code, classified.retryable)).toBeNull()
+ }
+ )
+
it.each(['unknown', 'canceled', 'error'])(
'never returns executable calls from a %s batch',
async outcome => {
diff --git a/mcp-host/src/llm/codexLlmProxyClient.ts b/mcp-host/src/llm/codexLlmProxyClient.ts
index a4b34519d..ecbc61f0d 100644
--- a/mcp-host/src/llm/codexLlmProxyClient.ts
+++ b/mcp-host/src/llm/codexLlmProxyClient.ts
@@ -9,6 +9,26 @@ import { upstreamRejectedStatus } from './upstreamRejected'
export const CODEX_PROXY_COMPLETIONS_PATH = '/internal/runtime/v1/codex/completions'
+/**
+ * Operator-facing text for a proxy denial. The three local-capacity codes say
+ * what the caller can do; they are not upstream outage diagnostics.
+ */
+export function codexProxyErrorMessage(code: string, status?: number): string {
+ if (
+ code === 'visual_host_share' ||
+ code === 'visual_gate' ||
+ code === 'proxy_capacity_exceeded'
+ ) {
+ return 'Codex proxy admission is full. Wait for an active request to finish or send fewer concurrent requests before retrying; this is not a Codex outage.'
+ }
+ if (code === 'payload_too_large') {
+ return 'Codex request is too large; use fewer or smaller images, or reduce context'
+ }
+ return status === undefined
+ ? `proxy stream failed with ${code}`
+ : `proxy stream failed with ${status} (${code})`
+}
+
export type CodexProxyErrorOptions = {
/**
* Whether a request had already left this process when the error was
@@ -166,7 +186,9 @@ export class CodexLlmProxyClient {
return this.streamOnce(input, false)
}
const payload = (await response.json().catch(() => ({}))) as Record<string, unknown>
- // A 429 is a rate limit, not a provider outage (G1-6, G1-11, #720): the
+ // Only the proxy-owned machine `error` field identifies the outcome; an
+ // upstream `reason` string is never promoted to trusted provenance. A
+ // 429 is a rate limit, not a provider outage (G1-6, G1-11, #720): the
// proxy answers `rate_limited` itself, and a 429 with no JSON code or a
// reason phrase is read the same way, so only a machine code a 429
// carries replaces `rate_limited`.
@@ -178,16 +200,10 @@ export class CodexLlmProxyClient {
: typeof payload.error === 'string'
? payload.error
: 'provider_unavailable'
- throw new CodexProxyError(
- code,
- code === 'payload_too_large'
- ? 'Codex request is too large; use fewer or smaller images, or reduce context'
- : `proxy stream failed with ${response.status} (${code})`,
- {
- retryAfterMs: response.status === 429 ? retryAfterMs(response) : undefined,
- upstreamStatus: upstreamRejectedStatus(code, payload.upstreamStatus),
- }
- )
+ throw new CodexProxyError(code, codexProxyErrorMessage(code, response.status), {
+ retryAfterMs: response.status === 429 ? retryAfterMs(response) : undefined,
+ upstreamStatus: upstreamRejectedStatus(code, payload.upstreamStatus),
+ })
}
if (!response.body) {
throw new CodexProxyError('provider_unavailable', 'proxy stream had no body')
@@ -221,7 +237,7 @@ async function readProxySse(body: ReadableStream<Uint8Array>): Promise<CodexProx
if (!line) continue
const frame = JSON.parse(line.slice(6)) as CodexProxyFrame
if (frame.type === 'error') {
- throw new CodexProxyError(frame.code, `proxy stream failed with ${frame.code}`, {
+ throw new CodexProxyError(frame.code, codexProxyErrorMessage(frame.code), {
upstreamStatus: upstreamRejectedStatus(frame.code, frame.upstreamStatus),
})
}
diff --git a/mcp-host/src/llm/failover/__tests__/engine.test.ts b/mcp-host/src/llm/failover/__tests__/engine.test.ts
index 193ff062f..d373ca322 100644
--- a/mcp-host/src/llm/failover/__tests__/engine.test.ts
+++ b/mcp-host/src/llm/failover/__tests__/engine.test.ts
@@ -102,6 +102,21 @@ describe('FailoverEngine', () => {
expect(metricInc).not.toHaveBeenCalled()
})
+ it('keeps the primary eligible after a local admission failure', async () => {
+ const e = engine()
+ const local = llmError(LlmErrorCode.ApiCallFailed, false)
+ const firstBuild = vi.fn(() => () => Promise.reject(local))
+ await expect(e.run(PRIMARY, firstBuild, classify)).rejects.toBe(local)
+ expect(firstBuild).toHaveBeenCalledTimes(1)
+ expect(onSwitch).not.toHaveBeenCalled()
+ expect(metricInc).not.toHaveBeenCalled()
+
+ const nextBuild = vi.fn(target => () => Promise.resolve(target.kind))
+ await expect(e.run(PRIMARY, nextBuild, classify)).resolves.toBe('primary')
+ expect(nextBuild).toHaveBeenCalledTimes(1)
+ expect(e.servedBy()).toEqual({ ...PRIMARY, fallback: false })
+ })
+
it('respects a restricted triggerOn: an out-of-set class propagates', async () => {
const e = engine(policy({ triggerOn: ['auth'] }))
const err = llmError(LlmErrorCode.RateLimited, true) // rate_limited not in triggerOn
diff --git a/mcp-host/src/llm/failover/__tests__/subscriptionAdmission.integration.test.ts b/mcp-host/src/llm/failover/__tests__/subscriptionAdmission.integration.test.ts
new file mode 100644
index 000000000..ce8fd6183
--- /dev/null
+++ b/mcp-host/src/llm/failover/__tests__/subscriptionAdmission.integration.test.ts
@@ -0,0 +1,358 @@
+import { afterEach, describe, expect, it, vi } from 'vitest'
+import jwt from 'jsonwebtoken'
+import { generateKeyPairSync, randomUUID } from 'node:crypto'
+import { type RequestListener, createServer } from 'node:http'
+import { type Socket, connect as connectTcp } from 'node:net'
+import { LlmErrorCode } from '../../../core/errors'
+import { CodexLlmProxyClient } from '../../codexLlmProxyClient'
+import { CodexSubscriptionProvider } from '../../codexSubscription'
+import { GrokLlmProxyClient } from '../../grokLlmProxyClient'
+import { GrokSubscriptionProvider } from '../../grokSubscription'
+import { type ClassifiedLike, FailoverEngine } from '../engine'
+import type { FailoverTarget, ModelPair } from '../types'
+
+type Gate = {
+ acquire(signal?: AbortSignal, deadlineAt?: number): Promise<() => void>
+ snapshot(): { running: number; queued: number }
+}
+type Budget = { readonly inFlightBytes: number; readonly queued: number }
+type ProxyLimitModule = {
+ streamGate: Gate
+ visualStreamGate: Gate
+ STREAM_LIMITS: { maxConcurrentStreams: number; maxQueuedRequests: number }
+ VISUAL_STREAM_LIMITS: { maxConcurrentStreams: number; maxQueuedRequests: number }
+ VISUAL_PER_HOST_MAX_ADMITTED: number
+ BodyBudget: {
+ prototype: {
+ acquire(bytes: number, signal?: AbortSignal, deadlineAt?: number): Promise<() => void>
+ }
+ }
+}
+type ProxyModule = {
+ createProxyApps(
+ config: Record<string, unknown>,
+ deps: Record<string, unknown>
+ ): { runtimeApp: RequestListener; close(): Promise<void> }
+}
+type StreamInput = {
+ executionTicket: string
+ requestHash: string
+ redeem(input: unknown): Promise<unknown>
+ onFrame(frame: { type: 'text'; text: string }): Promise<unknown>
+}
+type LocalProducer =
+ | 'visual_host_share'
+ | 'visual_gate'
+ | 'stream_queue_full'
+ | 'stream_queue_wait'
+ | 'body_queue_full'
+ | 'demotion_queue_full'
+const LOCAL_PRODUCERS: LocalProducer[] = [
+ 'visual_host_share',
+ 'visual_gate',
+ 'stream_queue_full',
+ 'stream_queue_wait',
+ 'body_queue_full',
+ 'demotion_queue_full',
+]
+const { privateKey, publicKey } = generateKeyPairSync('rsa', {
+ modulusLength: 2048,
+ publicKeyEncoding: { type: 'spki', format: 'pem' },
+ privateKeyEncoding: { type: 'pkcs8', format: 'pem' },
+})
+async function until(predicate: () => boolean, label: string): Promise<void> {
+ const deadline = Date.now() + 5000
+ while (!predicate()) {
+ if (Date.now() >= deadline) throw new Error(label)
+ await new Promise(resolve => setTimeout(resolve, 5))
+ }
+}
+const cleanup: Array<() => Promise<void>> = []
+afterEach(async () => {
+ for (const close of cleanup.splice(0).reverse()) await close()
+ vi.restoreAllMocks()
+})
+async function fixture(variant: 'grok' | 'codex') {
+ // Runtime imports preserve Host's production compiler boundary. This test
+ // exercises actual proxy HTTP producers, Host decoding and failover state.
+ const base = `../../../../../${variant}-llm-proxy/src/`
+ const proxyModule = await vi.importActual<ProxyModule>(`${base}server.ts`)
+ const limits = await vi.importActual<ProxyLimitModule>(`${base}requestLimits.ts`)
+ const model = variant === 'grok' ? 'grok-4.6' : 'gpt-5.3-codex'
+ const provider = `${variant}-subscription`
+ const platform = jwt.sign(
+ {
+ sub: 'default/admission-host',
+ hostRefs: ['admission-host'],
+ workflowControlScopes: [`llm:${variant}:execute`],
+ scope: 'workflow:approval:request',
+ },
+ privateKey,
+ { algorithm: 'RS256', issuer: 'control-api', audience: 'workflow-approvals', expiresIn: 60 }
+ )
+ const redeem = vi.fn(async () => undefined)
+ // Refusals must never reach this transport seam. A later accepted route uses
+ // it as a positive route/Host witness, not as real upstream compatibility.
+ const stream = vi.fn(async (input: StreamInput) => {
+ await input.redeem({ executionTicket: input.executionTicket, requestHash: input.requestHash })
+ await input.onFrame({ type: 'text', text: 'primary route recovered' })
+ return { outcome: 'success' as const }
+ })
+ const apps = proxyModule.createProxyApps(
+ {
+ runtimePort: 0,
+ adminPort: 0,
+ probePort: 0,
+ maxBodyBytes: 4096,
+ maxVisualBodyBytes: 8192,
+ maxStreamDurationMs: 1000,
+ maxDeadlineMs: 1000,
+ upstreamIdleTimeoutMs: 1000,
+ heartbeatIntervalMs: 15000,
+ jwtIssuer: 'control-api',
+ jwtPublicKey: publicKey,
+ executionEnabled: true,
+ controlApiBaseUrl: '',
+ controlApiServiceName: `${variant}-llm-proxy`,
+ controlApiServiceToken: '',
+ },
+ { controlApiClient: { redeem }, streamCompletion: stream, bodyReadDeadlineMs: 5000 }
+ )
+ const listener = createServer(apps.runtimeApp).listen(0, '127.0.0.1')
+ await new Promise<void>(resolve => listener.once('listening', resolve))
+ const address = listener.address()
+ if (!address || typeof address === 'string') throw new Error('the proxy has no local listener')
+ const port = address.port
+ const path = `/internal/runtime/v1/${variant}/completions`
+ const url = `http://127.0.0.1:${port}${path}`
+ const sockets: Socket[] = []
+ const held: Array<() => void> = []
+ const queued: Array<Promise<void>> = []
+ let restoreQueueClock: (() => void) | undefined
+ let padded = false
+ const hostFetch = vi.fn<typeof fetch>().mockImplementation((input, init) =>
+ fetch(input, {
+ ...init,
+ // A padded caller can have large wire bytes and a small logical V1 body.
+ ...(padded ? { body: `${String(init?.body)}${' '.repeat(5000)}` } : {}),
+ signal: init?.signal ?? AbortSignal.timeout(5000),
+ })
+ )
+ const clientOptions = { runtimeUrl: url, readPlatformJwt: () => platform, fetchFn: hostFetch }
+ const client =
+ variant === 'grok'
+ ? new GrokLlmProxyClient(clientOptions)
+ : new CodexLlmProxyClient(clientOptions)
+ const authorize = vi.fn(async (input: { requestHash: string }) => {
+ const id = randomUUID()
+ return {
+ providerAttemptId: id,
+ requestHash: input.requestHash,
+ executionTicket: jwt.sign(
+ {
+ jti: id,
+ typ: `${variant}-execution-ticket`,
+ hostRef: 'admission-host',
+ model,
+ requestHash: input.requestHash,
+ providerAttemptId: id,
+ },
+ privateKey,
+ {
+ algorithm: 'RS256',
+ issuer: 'control-api',
+ audience: `${variant}-llm-proxy`,
+ expiresIn: 60,
+ }
+ ),
+ expiresAt: new Date(Date.now() + 60000).toISOString(),
+ }
+ })
+ const deps = {
+ authorizer: { authorize },
+ proxy: client,
+ attemptContext: () => ({
+ policyRevision: 1,
+ policyHash: 'b'.repeat(64),
+ hostRef: 'admission-host',
+ }),
+ }
+ const primaryProvider =
+ variant === 'grok'
+ ? new GrokSubscriptionProvider(model, deps as never)
+ : new CodexSubscriptionProvider(model, deps as never)
+ const primary: ModelPair = { provider, model }
+ const onSwitch = vi.fn(),
+ metricInc = vi.fn()
+ const engine = new FailoverEngine(
+ {
+ cooldownSeconds: 300,
+ triggerOn: ['provider_unavailable', 'rate_limited'],
+ fallbacks: [{ provider: 'openai', model: 'gpt-5.4' }],
+ },
+ { onSwitch, metricInc }
+ )
+ const fallback = vi.fn(async () => ({ content: 'fallback' }))
+ const built: FailoverTarget[] = []
+ let classified: ClassifiedLike | undefined
+ const run = (content: string) =>
+ engine.run<{ content: string }>(
+ primary,
+ target => {
+ built.push(target)
+ return target.kind === 'primary'
+ ? () => primaryProvider.completeSingleTurn([{ role: 'user', content }])
+ : fallback
+ },
+ err => {
+ classified = primaryProvider.classifyError(err)
+ return classified
+ }
+ )
+ const stopCapacity = async () => {
+ padded = false
+ for (const socket of sockets.splice(0)) socket.destroy()
+ for (const release of held.splice(0)) release()
+ await Promise.all(queued.splice(0))
+ restoreQueueClock?.()
+ restoreQueueClock = undefined
+ await until(() => {
+ const ordinary = limits.streamGate.snapshot(),
+ visual = limits.visualStreamGate.snapshot()
+ return (
+ ordinary.running === 0 &&
+ ordinary.queued === 0 &&
+ visual.running === 0 &&
+ visual.queued === 0
+ )
+ }, 'the proxy admission fixture did not drain')
+ }
+ cleanup.push(async () => {
+ await stopCapacity()
+ listener.closeAllConnections()
+ await new Promise<void>((resolve, reject) =>
+ listener.close(err => (err ? reject(err) : resolve()))
+ )
+ await apps.close()
+ })
+ function park(count: number, declared: number): void {
+ for (let index = 0; index < count; index += 1) {
+ const socket = connectTcp(port, '127.0.0.1')
+ sockets.push(socket)
+ socket.once('connect', () =>
+ socket.write(
+ `POST ${path} HTTP/1.1\r\nHost: 127.0.0.1:${port}\r\n` +
+ `Authorization: Bearer ${platform}\r\nContent-Type: application/json\r\nContent-Length: ${declared}\r\n\r\n{`
+ )
+ )
+ socket.on('error', err => {
+ if ((err as NodeJS.ErrnoException).code !== 'ECONNRESET') throw err
+ })
+ socket.resume()
+ }
+ }
+ async function fill(gate: Gate, width: number, waiters: number): Promise<void> {
+ for (let index = 0; index < width; index += 1) held.push(await gate.acquire())
+ for (let index = 0; index < waiters; index += 1)
+ queued.push(gate.acquire().then(release => release()))
+ await until(() => gate.snapshot().queued === waiters, 'the actual stream gate was not filled')
+ }
+ async function block(kind: LocalProducer): Promise<void> {
+ if (kind === 'visual_host_share') {
+ park(limits.VISUAL_PER_HOST_MAX_ADMITTED, 6000)
+ await until(() => {
+ const s = limits.visualStreamGate.snapshot()
+ return s.running + s.queued === limits.VISUAL_PER_HOST_MAX_ADMITTED
+ }, 'the actual principal share was not filled')
+ } else if (kind === 'visual_gate') {
+ await fill(
+ limits.visualStreamGate,
+ limits.VISUAL_STREAM_LIMITS.maxConcurrentStreams,
+ limits.VISUAL_STREAM_LIMITS.maxQueuedRequests
+ )
+ } else if (kind === 'stream_queue_full') {
+ await fill(
+ limits.streamGate,
+ limits.STREAM_LIMITS.maxConcurrentStreams,
+ limits.STREAM_LIMITS.maxQueuedRequests
+ )
+ } else if (kind === 'stream_queue_wait') {
+ // Shorten only the actual gate clock; producer and HTTP path stay real.
+ const oldClock: unknown = Reflect.get(limits.streamGate, 'maxQueueWaitMs')
+ Reflect.set(limits.streamGate, 'maxQueueWaitMs', 35)
+ restoreQueueClock = () => {
+ Reflect.set(limits.streamGate, 'maxQueueWaitMs', oldClock)
+ }
+ await fill(limits.streamGate, limits.STREAM_LIMITS.maxConcurrentStreams, 0)
+ } else {
+ const acquisitions = vi.spyOn(limits.BodyBudget.prototype, 'acquire')
+ park(3, 4096)
+ await until(
+ () => (acquisitions.mock.contexts[0] as Budget | undefined)?.inFlightBytes === 3 * 4096,
+ 'the ordinary readers did not take the byte budget'
+ )
+ const budget = acquisitions.mock.contexts[0] as Budget
+ park(limits.STREAM_LIMITS.maxQueuedRequests, 4096)
+ await until(
+ () => budget.queued === limits.STREAM_LIMITS.maxQueuedRequests,
+ 'the actual body queue was not filled'
+ )
+ padded = kind === 'demotion_queue_full'
+ }
+ }
+ return {
+ block,
+ stopCapacity,
+ run,
+ engine,
+ primary,
+ authorize,
+ redeem,
+ stream,
+ hostFetch,
+ fallback,
+ built,
+ onSwitch,
+ metricInc,
+ classified: () => classified,
+ }
+}
+describe.each(['grok', 'codex'] as const)(
+ '%s subscription HTTP admission and failover',
+ variant => {
+ it.each(LOCAL_PRODUCERS)(
+ 'local %s keeps the next turn on primary without redeem or failover',
+ async kind => {
+ const f = await fixture(variant)
+ await f.block(kind)
+ const content = kind.startsWith('visual_') ? 'x'.repeat(5000) : 'bounded text'
+ const code = kind.startsWith('visual_') ? kind : 'proxy_capacity_exceeded'
+ await expect(f.run(content)).rejects.toMatchObject({ code, dispatched: true })
+ expect(f.classified()).toMatchObject({
+ code: LlmErrorCode.ApiCallFailed,
+ retryable: false,
+ providerCode: code,
+ providerDispatched: true,
+ })
+ expect(f.authorize).toHaveBeenCalledTimes(1)
+ expect(f.hostFetch).toHaveBeenCalledTimes(1)
+ expect(f.redeem).not.toHaveBeenCalled()
+ expect(f.stream).not.toHaveBeenCalled()
+ expect(f.built.map(target => target.kind)).toEqual(['primary'])
+ expect(f.fallback).not.toHaveBeenCalled()
+ expect(f.onSwitch).not.toHaveBeenCalled()
+ expect(f.metricInc).not.toHaveBeenCalled()
+ expect(f.engine.planTargets(f.primary)[0]?.kind).toBe('primary')
+ await f.stopCapacity()
+ expect((await f.run('next ordinary text')).content).toBe('primary route recovered')
+ expect(f.authorize).toHaveBeenCalledTimes(2)
+ expect(f.hostFetch).toHaveBeenCalledTimes(2)
+ expect(f.redeem).toHaveBeenCalledTimes(1)
+ expect(f.stream).toHaveBeenCalledTimes(1)
+ expect(f.built.map(target => target.kind)).toEqual(['primary', 'primary'])
+ expect(f.fallback).not.toHaveBeenCalled()
+ expect(f.engine.servedBy()).toEqual({ ...f.primary, fallback: false })
+ }
+ )
+ }
+)
diff --git a/mcp-host/src/llm/grokLlmProxyClient.ts b/mcp-host/src/llm/grokLlmProxyClient.ts
index 045a19333..8b92d3669 100644
--- a/mcp-host/src/llm/grokLlmProxyClient.ts
+++ b/mcp-host/src/llm/grokLlmProxyClient.ts
@@ -15,6 +15,13 @@ export const GROK_PROXY_COMPLETIONS_PATH = '/internal/runtime/v1/grok/completion
* sentence that says what to do; everything else keeps the diagnostic shape.
*/
export function grokProxyErrorMessage(code: string, status?: number): string {
+ if (
+ code === 'visual_host_share' ||
+ code === 'visual_gate' ||
+ code === 'proxy_capacity_exceeded'
+ ) {
+ return 'Grok proxy admission is full. Wait for an active request to finish or send fewer concurrent requests before retrying; this is not a Grok outage.'
+ }
if (code === 'client_upgrade_required') {
return 'Grok subscription inference is unavailable: xAI now requires a newer Grok client version than this deployment sends. An operator can set GROK_LLM_PROXY_CLIENT_VERSION to a current Grok Build release, or contact support — retrying will not help.'
}
@@ -194,7 +201,9 @@ export class GrokLlmProxyClient {
return this.streamOnce(input, false)
}
const payload = (await response.json().catch(() => ({}))) as Record<string, unknown>
- // A 413 with no JSON code comes from the gateway in front of the proxy
+ // Only the proxy-owned machine `error` field identifies the outcome; an
+ // upstream `reason` string is never promoted to trusted provenance. A
+ // 413 with no JSON code comes from the gateway in front of the proxy
// (nginx `client_max_body_size`): a size refusal of this request, never a
// provider outage (#739). A code the 413 carries still wins. A 429 is a
// rate limit (G1-6, G1-11, #720): the proxy answers `rate_limited`
diff --git a/scripts/ci/pinned-vitest-cases/codex-llm-proxy.json b/scripts/ci/pinned-vitest-cases/codex-llm-proxy.json
index d52dd3907..d890e99f2 100644
--- a/scripts/ci/pinned-vitest-cases/codex-llm-proxy.json
+++ b/scripts/ci/pinned-vitest-cases/codex-llm-proxy.json
@@ -49,16 +49,32 @@
},
{
"suite": "server.security.test.ts",
- "fullName": "codex-llm-proxy ticket-aware stream-gate wait (#739 D1-bis) visual admission overflow answers 503 provider_unavailable and logs visual_gate"
+ "fullName": "codex-llm-proxy ticket-aware stream-gate wait (#739 D1-bis) visual admission overflow answers 503 visual_gate and logs visual_gate"
+ },
+ {
+ "suite": "server.security.test.ts",
+ "fullName": "codex-llm-proxy attempt telemetry (l) logs the request-limit reason and labels its failure metric request_limit"
+ },
+ {
+ "suite": "bodyAdmission.test.ts",
+ "fullName": "codex-llm-proxy body admission (#731 R3-2) T-R3-2c answers proxy_capacity_exceeded once the admission queue is full"
},
{
"suite": "streamGate.handoff.test.ts",
"fullName": "visual stream-gate handoff releases the visual slot for a whitespace-padded small V2"
},
+ {
+ "suite": "streamGate.handoff.test.ts",
+ "fullName": "visual stream-gate handoff visual per-host share admits at most four visual entries per principal while another principal proceeds"
+ },
{
"suite": "streamGate.handoff.test.ts",
"fullName": "visual stream-gate handoff visual per-host share releases the principal share when the visual queue is full before grant"
},
+ {
+ "suite": "visualBodyReadOwnership.test.ts",
+ "fullName": "codex visual body read ownership keeps the visual slot and principal share until the timed-out parser terminates"
+ },
{
"suite": "streamGate.handoff.test.ts",
"fullName": "visual stream-gate handoff visual slot lifetime frees the queue place of a visual request aborted while it waited"
@@ -66,5 +82,25 @@
{
"suite": "streamGate.handoff.test.ts",
"fullName": "visual stream-gate handoff visual slot lifetime holds the queue place of a disconnected large-body waiter until its grant"
+ },
+ {
+ "suite": "visualBodyReadOwnership.test.ts",
+ "fullName": "codex visual body read ownership stops the native reader independently when a non-reading peer never finishes the response"
+ },
+ {
+ "suite": "visualBodyReadOwnership.test.ts",
+ "fullName": "codex visual body read ownership clears a late successful native parse after its read deadline without dispatching it"
+ },
+ {
+ "suite": "visualBodyReadOwnership.test.ts",
+ "fullName": "codex visual body read ownership clears the retained parsed body after response close before its callback returns"
+ },
+ {
+ "suite": "server.security.test.ts",
+ "fullName": "codex-llm-proxy attempt telemetry normalizes real transport upstream 503 claiming visual_gate to provider_unavailable"
+ },
+ {
+ "suite": "streamLimitsFreeze.test.ts",
+ "fullName": "codex-subscription stream limits freeze rejects open or mutable admission code producers while reading the closed production union"
}
]
diff --git a/scripts/ci/pinned-vitest-cases/grok-llm-proxy.json b/scripts/ci/pinned-vitest-cases/grok-llm-proxy.json
index 27b8b6296..06f231acb 100644
--- a/scripts/ci/pinned-vitest-cases/grok-llm-proxy.json
+++ b/scripts/ci/pinned-vitest-cases/grok-llm-proxy.json
@@ -51,14 +51,30 @@
"suite": "server.security.test.ts",
"fullName": "grok-llm-proxy attempt telemetry (r4-l1b) logs a catalog DNS lookup failure by cause code on both admin routes"
},
+ {
+ "suite": "server.security.test.ts",
+ "fullName": "grok-llm-proxy attempt telemetry (rl1) logs the request-limit reason and labels its failure metric request_limit"
+ },
+ {
+ "suite": "bodyAdmission.test.ts",
+ "fullName": "grok-llm-proxy body admission (#731 R3-2) T-R3-2c-grok answers proxy_capacity_exceeded once the admission queue is full"
+ },
{
"suite": "streamGate.handoff.test.ts",
"fullName": "grok visual stream-gate handoff releases the visual slot for a whitespace-padded small V2"
},
+ {
+ "suite": "streamGate.handoff.test.ts",
+ "fullName": "grok visual stream-gate handoff visual per-host share admits at most two visual entries per principal while another principal proceeds"
+ },
{
"suite": "streamGate.handoff.test.ts",
"fullName": "grok visual stream-gate handoff visual per-host share releases the principal share when the visual queue is full before grant"
},
+ {
+ "suite": "visualBodyReadOwnership.test.ts",
+ "fullName": "grok visual body read ownership keeps the visual slot and principal share until the timed-out parser terminates"
+ },
{
"suite": "streamGate.handoff.test.ts",
"fullName": "grok visual stream-gate handoff visual slot lifetime frees the queue place of a visual request aborted while it waited"
@@ -66,5 +82,25 @@
{
"suite": "streamGate.handoff.test.ts",
"fullName": "grok visual stream-gate handoff visual slot lifetime holds the queue place of a disconnected large-body waiter until its grant"
+ },
+ {
+ "suite": "visualBodyReadOwnership.test.ts",
+ "fullName": "grok visual body read ownership stops the native reader independently when a non-reading peer never finishes the response"
+ },
+ {
+ "suite": "visualBodyReadOwnership.test.ts",
+ "fullName": "grok visual body read ownership clears a late successful native parse after its read deadline without dispatching it"
+ },
+ {
+ "suite": "visualBodyReadOwnership.test.ts",
+ "fullName": "grok visual body read ownership clears the retained parsed body after response close before its callback returns"
+ },
+ {
+ "suite": "server.security.test.ts",
+ "fullName": "grok-llm-proxy attempt telemetry normalizes real transport upstream 503 claiming visual_gate to provider_unavailable"
+ },
+ {
+ "suite": "contractFreeze.test.ts",
+ "fullName": "grok-subscription contract freeze rejects open or mutable admission code producers while reading the closed production union"
}
]
diff --git a/scripts/ci/pinned-vitest-cases/mcp-host.json b/scripts/ci/pinned-vitest-cases/mcp-host.json
new file mode 100644
index 000000000..706f5489e
--- /dev/null
+++ b/scripts/ci/pinned-vitest-cases/mcp-host.json
@@ -0,0 +1,74 @@
+[
+ {
+ "suite": "subscriptionAdmission.integration.test.ts",
+ "fullName": "grok subscription HTTP admission and failover local visual_host_share keeps the next turn on primary without redeem or failover"
+ },
+ {
+ "suite": "subscriptionAdmission.integration.test.ts",
+ "fullName": "grok subscription HTTP admission and failover local visual_gate keeps the next turn on primary without redeem or failover"
+ },
+ {
+ "suite": "subscriptionAdmission.integration.test.ts",
+ "fullName": "grok subscription HTTP admission and failover local stream_queue_full keeps the next turn on primary without redeem or failover"
+ },
+ {
+ "suite": "subscriptionAdmission.integration.test.ts",
+ "fullName": "grok subscription HTTP admission and failover local stream_queue_wait keeps the next turn on primary without redeem or failover"
+ },
+ {
+ "suite": "subscriptionAdmission.integration.test.ts",
+ "fullName": "grok subscription HTTP admission and failover local body_queue_full keeps the next turn on primary without redeem or failover"
+ },
+ {
+ "suite": "subscriptionAdmission.integration.test.ts",
+ "fullName": "grok subscription HTTP admission and failover local demotion_queue_full keeps the next turn on primary without redeem or failover"
+ },
+ {
+ "suite": "subscriptionAdmission.integration.test.ts",
+ "fullName": "codex subscription HTTP admission and failover local visual_host_share keeps the next turn on primary without redeem or failover"
+ },
+ {
+ "suite": "subscriptionAdmission.integration.test.ts",
+ "fullName": "codex subscription HTTP admission and failover local visual_gate keeps the next turn on primary without redeem or failover"
+ },
+ {
+ "suite": "subscriptionAdmission.integration.test.ts",
+ "fullName": "codex subscription HTTP admission and failover local stream_queue_full keeps the next turn on primary without redeem or failover"
+ },
+ {
+ "suite": "subscriptionAdmission.integration.test.ts",
+ "fullName": "codex subscription HTTP admission and failover local stream_queue_wait keeps the next turn on primary without redeem or failover"
+ },
+ {
+ "suite": "subscriptionAdmission.integration.test.ts",
+ "fullName": "codex subscription HTTP admission and failover local body_queue_full keeps the next turn on primary without redeem or failover"
+ },
+ {
+ "suite": "subscriptionAdmission.integration.test.ts",
+ "fullName": "codex subscription HTTP admission and failover local demotion_queue_full keeps the next turn on primary without redeem or failover"
+ },
+ {
+ "suite": "grokLlmProxyClient.test.ts",
+ "fullName": "GrokLlmProxyClient keeps wire unknown_machine_code separate from trusted local refusal provenance"
+ },
+ {
+ "suite": "grokLlmProxyClient.test.ts",
+ "fullName": "GrokLlmProxyClient keeps wire reason_only separate from trusted local refusal provenance"
+ },
+ {
+ "suite": "grokLlmProxyClient.test.ts",
+ "fullName": "GrokLlmProxyClient keeps wire malformed_json separate from trusted local refusal provenance"
+ },
+ {
+ "suite": "codexLlmProxyClient.test.ts",
+ "fullName": "CodexLlmProxyClient keeps wire unknown_machine_code separate from trusted local refusal provenance"
+ },
+ {
+ "suite": "codexLlmProxyClient.test.ts",
+ "fullName": "CodexLlmProxyClient keeps wire reason_only separate from trusted local refusal provenance"
+ },
+ {
+ "suite": "codexLlmProxyClient.test.ts",
+ "fullName": "CodexLlmProxyClient keeps wire malformed_json separate from trusted local refusal provenance"
+ }
+]
diff --git a/scripts/tests/test-codex-subscription-t0.sh b/scripts/tests/test-codex-subscription-t0.sh
index 3eb98357a..641b4b6a6 100755
--- a/scripts/tests/test-codex-subscription-t0.sh
+++ b/scripts/tests/test-codex-subscription-t0.sh
@@ -242,6 +242,9 @@ run_group "control-api" "control-api" \
"test/routes.admin.codexSubscription.test.ts" \
"test/routes.admin.codexSubscription.hostWrite.test.ts" \
"test/routes.auth.codexSubscriptionCallback.test.ts" \
+ "test/routes.mcp-host.llmProviderAttempts.bodyAdmission.test.ts" \
+ "test/llmProviderAttemptBodyAdmission.test.ts" \
+ "test/llmProviderAttemptAuthorizer.cancellation.test.ts" \
"test/routes.mcp-host.llmProviderAttempts.test.ts" \
"test/routes.adminRecipes.test.ts" \
"test/routes.adminPluginWorkloadSdk.test.ts" \
@@ -300,6 +303,7 @@ run_group "codex-llm-proxy" "codex-llm-proxy" \
"test/sseBackpressure.test.ts" \
"test/sseHeartbeat.test.ts" \
"test/streamGate.handoff.test.ts" \
+ "test/visualBodyReadOwnership.test.ts" \
"test/streamLimitsFreeze.test.ts" \
"test/toolNameMap.test.ts"
@@ -325,6 +329,7 @@ run_group "mcp-host" "mcp-host" \
"src/llm/__tests__/codexSubscription.test.ts" \
"src/llm/__tests__/codexLlmProxyClient.test.ts" \
"src/llm/__tests__/subscriptionRequestHash.test.ts" \
+ "src/llm/failover/__tests__/subscriptionAdmission.integration.test.ts" \
"src/llm/__tests__/providerAttemptAuthorizer.test.ts" \
"src/llm/hostLlmBinding.test.ts" \
"src/config/configStore.test.ts" \
diff --git a/scripts/tests/test-grok-subscription-t0.sh b/scripts/tests/test-grok-subscription-t0.sh
index 08a51666d..a0536876d 100755
--- a/scripts/tests/test-grok-subscription-t0.sh
+++ b/scripts/tests/test-grok-subscription-t0.sh
@@ -252,6 +252,7 @@ run_group "grok-llm-proxy" "grok-llm-proxy" \
"test/sseBackpressure.test.ts" \
"test/sseHeartbeat.test.ts" \
"test/streamGate.handoff.test.ts" \
+ "test/visualBodyReadOwnership.test.ts" \
"test/toolNameMap.test.ts" \
"test/upstreamErrorHint.test.ts"
@@ -273,6 +274,9 @@ run_group "control-api grok" "control-api" \
"test/subscriptionGrantIdentity.test.ts" \
"test/llmProviders.test.ts" \
"test/routes.admin.grokSubscription.test.ts" \
+ "test/routes.mcp-host.llmProviderAttempts.bodyAdmission.test.ts" \
+ "test/llmProviderAttemptBodyAdmission.test.ts" \
+ "test/llmProviderAttemptAuthorizer.cancellation.test.ts" \
"test/routes.internal.llmProviderAttempts.grok.test.ts" \
"test/routes.adminPluginWorkloadSdk.test.ts" \
"test/routes.adminRecipes.test.ts" \
@@ -296,6 +300,7 @@ run_group "mcp-host grok" "mcp-host" \
"src/llm/__tests__/imageSource.test.ts" \
"src/llm/__tests__/grokLlmProxyClient.test.ts" \
"src/llm/__tests__/imageInput.test.ts" \
+ "src/llm/failover/__tests__/subscriptionAdmission.integration.test.ts" \
"src/llm/__tests__/providerAttemptAuthorizer.test.ts" \
"src/core/adapters/__tests__/llmImageCompatibility.test.ts" \
"src/agent/__tests__/taskExecutor.test.ts" \
diff --git a/tests/e2e/fixtures/codex-subscription/sanitized-upstream-contract.json b/tests/e2e/fixtures/codex-subscription/sanitized-upstream-contract.json
index 1bea6b6e4..af548bbd7 100644
--- a/tests/e2e/fixtures/codex-subscription/sanitized-upstream-contract.json
+++ b/tests/e2e/fixtures/codex-subscription/sanitized-upstream-contract.json
@@ -85,7 +85,10 @@
"disabled",
"Unauthorized",
"not_found",
- "internal_error"
+ "internal_error",
+ "visual_host_share",
+ "visual_gate",
+ "proxy_capacity_exceeded"
],
"termsAndTestAccount": {
"automationAuthorized": true,
diff --git a/tests/e2e/fixtures/grok-subscription/sanitized-upstream-contract.json b/tests/e2e/fixtures/grok-subscription/sanitized-upstream-contract.json
index bf373a3c7..1b8242b42 100644
--- a/tests/e2e/fixtures/grok-subscription/sanitized-upstream-contract.json
+++ b/tests/e2e/fixtures/grok-subscription/sanitized-upstream-contract.json
@@ -102,7 +102,10 @@
"host_binding_mismatch",
"Unauthorized",
"not_found",
- "internal_error"
+ "internal_error",
+ "visual_host_share",
+ "visual_gate",
+ "proxy_capacity_exceeded"
],
"termsAndTestAccount": {
"automationAuthorized": false,
diff --git a/tests/e2e/fixtures/package-lock.json b/tests/e2e/fixtures/package-lock.json
index 8c88caac1..2e21df64b 100644
--- a/tests/e2e/fixtures/package-lock.json
+++ b/tests/e2e/fixtures/package-lock.json
@@ -1,12 +1,12 @@
{
"name": "@clerum/e2e-test-fixtures",
- "version": "1.0.100",
+ "version": "1.0.101",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "@clerum/e2e-test-fixtures",
- "version": "1.0.100",
+ "version": "1.0.101",
"dependencies": {
"@kubernetes/client-node": "^1.0.0"
},
diff --git a/tests/e2e/fixtures/package.json b/tests/e2e/fixtures/package.json
index b80adc934..0523a0d75 100644
--- a/tests/e2e/fixtures/package.json
+++ b/tests/e2e/fixtures/package.json
@@ -1,6 +1,6 @@
{
"name": "@clerum/e2e-test-fixtures",
- "version": "1.0.100",
+ "version": "1.0.101",
"description": "E2E test fixtures and utilities for Clerum testing",
"type": "module",
"main": "index.ts",