Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
fix(llm): preserve local admission and visual ownership
Alfredo Lopez6c63f329aedd
diff --git a/codex-llm-proxy/src/requestLimits.ts b/codex-llm-proxy/src/requestLimits.tsindex 5bbf93aef..1093ab0be 100644--- a/codex-llm-proxy/src/requestLimits.ts+++ b/codex-llm-proxy/src/requestLimits.ts@@ -66,6 +66,13 @@ export const IN_FLIGHT_BODY_BUDGET_BODIES = 3*/export const BODY_READ_DEADLINE_MS = 10_000+/**+ * Bounds the interval between a visual read timeout/client response close and+ * forced request destruction. The parser's callback—not this timer—releases the+ * visual slot and principal share.+ */+export const VISUAL_READ_CLOSE_GRACE_MS = 100+export const STREAM_LIMITS = {maxConcurrentStreams: 8,maxQueuedRequests: 16,@@ -102,9 +109,10 @@ export const VISUAL_STREAM_LIMITS = {* Fair share of the visual gate's entries — running plus queued — that one* platform principal (`sub` plus sorted `hostRefs`) may hold at once. Above the* share, further visual requests from that principal are refused 503- * `provider_unavailable` with log reason `visual_host_share`, so one principal+ * `visual_host_share`, so one principal* cannot fill a gate that every other host still needs. The gate widths above* are unchanged; the share only bounds how much of them one principal occupies.+ * Its refusal is a local capacity outcome, not an upstream outage.*/export const VISUAL_PER_HOST_MAX_ADMITTED = 4@@ -125,14 +133,18 @@ export type RequestLimitKind =| 'queue_wait'| 'ticket_life'+export type RequestLimitCode = 'provider_unavailable' | 'proxy_capacity_exceeded' | 'visual_gate'+export class RequestLimitError extends Error {- readonly code = 'provider_unavailable'+ readonly code: RequestLimitCodeconstructor(message: string,- readonly kind: RequestLimitKind+ readonly kind: RequestLimitKind,+ code: RequestLimitCode = 'provider_unavailable') {super(message)this.name = 'RequestLimitError'+ this.code = code}}@@ -186,7 +198,8 @@ export class StreamGate {constructor(private readonly maxConcurrent: number = STREAM_LIMITS.maxConcurrentStreams,private readonly maxQueued: number = STREAM_LIMITS.maxQueuedRequests,- private readonly maxQueueWaitMs: number = STREAM_LIMITS.maxQueueWaitMs+ private readonly maxQueueWaitMs: number = STREAM_LIMITS.maxQueueWaitMs,+ private readonly queueCapacityCode: RequestLimitCode = 'proxy_capacity_exceeded') {// A NaN or fractional size would let every caller through or queue// without bound, because the comparisons below would never hold.@@ -229,7 +242,7 @@ export class StreamGate {if (signal?.aborted) throw new RequestLimitError('stream request was aborted', 'aborted')if (this.running >= this.maxConcurrent) {if (this.queued >= this.maxQueued)- throw new RequestLimitError('stream queue is full', 'queue_full')+ throw new RequestLimitError('stream queue is full', 'queue_full', this.queueCapacityCode)this.queued += 1try {await new Promise<void>((resolve, reject) => {@@ -255,7 +268,8 @@ export class StreamGate {reject(new RequestLimitError('stream queue wait exceeded',- deadlineGoverns ? 'deadline' : 'queue_wait'+ deadlineGoverns ? 'deadline' : 'queue_wait',+ deadlineGoverns ? 'provider_unavailable' : this.queueCapacityCode))}@@ -297,7 +311,9 @@ export const streamGate = new StreamGate()export const visualStreamGate = new StreamGate(VISUAL_STREAM_LIMITS.maxConcurrentStreams,- VISUAL_STREAM_LIMITS.maxQueuedRequests+ VISUAL_STREAM_LIMITS.maxQueuedRequests,+ STREAM_LIMITS.maxQueueWaitMs,+ 'visual_gate')type BodyWaiter = { bytes: number; grant: (release: () => void) => void }@@ -356,7 +372,11 @@ export class BodyBudget {return this.take(bytes)}if (this.waiters.length >= this.maxQueued) {- throw new RequestLimitError('body admission queue is full', 'queue_full')+ throw new RequestLimitError(+ 'body admission queue is full',+ 'queue_full',+ 'proxy_capacity_exceeded'+ )}return new Promise<() => void>((resolve, reject) => {let deadline: ReturnType<typeof setTimeout> | undefineddiff --git a/codex-llm-proxy/src/server.ts b/codex-llm-proxy/src/server.tsindex 3e226698c..5dc5ec2d3 100644--- a/codex-llm-proxy/src/server.ts+++ b/codex-llm-proxy/src/server.ts@@ -46,6 +46,7 @@ import {STREAM_LIMITS,TicketLifeError,VISUAL_PER_HOST_MAX_ADMITTED,+ VISUAL_READ_CLOSE_GRACE_MS,streamGate,visualStreamGate,} from './requestLimits.js'@@ -112,7 +113,9 @@ function isRecord(value: unknown): value is Record<string, unknown> {function reject(res: Response, status: number, code: string, type?: string): void {if (res.headersSent) returnlogger.warn(- type === undefined ? { event: 'codex_proxy_denied', code } : { event: 'codex_proxy_denied', code, type },+ type === undefined+ ? { event: 'codex_proxy_denied', code }+ : { event: 'codex_proxy_denied', code, type },'request denied')res.status(status).json({ error: code })@@ -267,7 +270,7 @@ function bodyAdmission(},'admission refused')- reject(res, 503, 'provider_unavailable')+ reject(res, 503, err.code)})}@@ -287,6 +290,8 @@ export type ProxyRuntimeDeps = {streamCompletion?: typeof streamCodexCompletion/** Test seam for the body-read deadline. Production uses `BODY_READ_DEADLINE_MS`. */bodyReadDeadlineMs?: number+ /** Test seam for the bounded visual-parser close backstop. */+ visualReadCloseGraceMs?: number}export type ProxyServers = {@@ -334,9 +339,16 @@ export function createProxyApps(// One budget for both apps: every body this process reads counts against it.const bodyBudget = new BodyBudget(IN_FLIGHT_BODY_BUDGET_BODIES * config.maxBodyBytes)const bodyReadDeadlineMs = deps.bodyReadDeadlineMs ?? BODY_READ_DEADLINE_MS+ const visualReadCloseGraceMs = deps.visualReadCloseGraceMs ?? VISUAL_READ_CLOSE_GRACE_MS+ for (const [name, value] of Object.entries({ bodyReadDeadlineMs, visualReadCloseGraceMs })) {+ if (!Number.isSafeInteger(value) || value <= 0 || value > 2_147_483_647) {+ throw new RangeError(`${name} must be a positive signed-32-bit integer`)+ }+ }// Per-principal occupancy of the visual gate (running plus queued entries).// Keyed on the verified platform principal: sub plus sorted hostRefs.const visualPrincipalAdmissions = new Map<string, number>()+ const visualRequestOwners = new WeakSet<GatedRequest>()const runtimeApp = express()// R9-1: `inflate: false` on every parser. The budgets count the declared wire@@ -371,6 +383,9 @@ export function createProxyApps(// The parser behind body admission. Every request here carries a verified// platform JWT.//+ // A granted visual slot is released only after parser completion (or by the+ // route's stream finally); a response close starts a bounded destruction+ // backstop instead of freeing a live parser owner.// schemaVersion is not known until the body is parsed, so the visual gate// must not be taken for every platform JWT. Only a declared Content-Length// above the ordinary cap can be a visual envelope. Those bodies take the@@ -412,7 +427,7 @@ export function createProxyApps(},'admission refused')- reject(res, 503, 'provider_unavailable')+ reject(res, 503, 'visual_host_share')return}visualPrincipalAdmissions.set(visualPrincipal, admittedByPrincipal + 1)@@ -474,38 +489,55 @@ export function createProxyApps(req.codexStreamRelease = undefined}req.codexStreamRelease = releaseVisual- let expired = false+ let cancelled = false+ let parserSettled = false+ let closeGrace: ReturnType<typeof setTimeout> | undefined+ const cancelParser = (): void => {+ if (parserSettled || closeGrace !== undefined) return+ closeGrace = setTimeout(() => {+ if (!parserSettled) req.destroy()+ }, visualReadCloseGraceMs)+ }const readDeadline = setTimeout(() => {- expired = true- releaseVisual()+ cancelled = true+ cancelParser()if (res.headersSent) return// The rest of the body is never read, so the connection cannot be reused.res.setHeader('connection', 'close')reject(res, 408, 'request_timeout')}, bodyReadDeadlineMs)- res.once('close', () => {+ const onResponseClose = (): void => {+ cancelled = trueclearTimeout(readDeadline)- // The close handler releases only while the slot is still held here. A kept- // visual body transfers `req.codexStreamRelease` into the handler,- // which clears the field and releases in its outer `finally`. A- // demoted body releases through `releaseAdmission` before it joins- // the ordinary stream gate; this hook may still fire after demotion- // but finds the field empty and no-ops. Releasing here after hand-off- // would free the visual slot and the principal share while the- // handler still holds the parsed body during its (up to ~30 s)- // unwind, admitting another large body over the gate width.- if (req.codexStreamRelease) releaseVisual()- })+ cancelParser()+ }+ res.once('close', onResponseClose)visualJson(req, res, err => {+ parserSettled = trueclearTimeout(readDeadline)- // The 408 already answered this request; the parser's late error- // (the body aborted by the closed connection) has no one to reach.- if (expired) return+ if (closeGrace !== undefined) clearTimeout(closeGrace)+ res.off('close', onResponseClose)+ if (cancelled || err) {+ req.body = undefined+ if (err !== null && typeof err === 'object' && 'body' in err) {+ delete (err as { body?: unknown }).body+ }+ }+ // Capacity remains charged until body-parser has actually unwound. On+ // success, `req.codexStreamRelease` transfers ownership to the handler,+ // which releases it in its outer `finally`; refusals release here.+ if (cancelled || err) releaseVisual()+ if (cancelled) returnif (err) {- releaseVisual()next(err)return}+ res.once('close', () => {+ // A synchronous handler throw can happen before its async owner+ // installs cancellation and a finally block. Once that owner has+ // taken over, response close must not release its live work.+ if (!visualRequestOwners.has(req)) releaseVisual()+ })next()})})()@@ -611,6 +643,7 @@ export function createProxyApps(// to req 'close' aborts the ChatGPT hop on every call (3–12ms canceled).// Abort only when the client drops the response before we finish writing.abortWhenClientDisconnects(req, res, abort)+ visualRequestOwners.add(gated)// One `codex_proxy_attempt_finished` line per attempt. Identifiers and// counts only: never the body, ticket, frames, tool names or arguments.const attempt = {@@ -770,9 +803,9 @@ export function createProxyApps(stopHeartbeat?.()const mapped = mapError(err)metrics.observeAttempt('error', 'completion_stream')- // A request limit answers provider_unavailable on the wire; its own- // label keeps it apart from real upstream outages in the metric. A- // ticket that died in the queue is counted as provider_unavailable.+ // Queue capacity has a terminal local wire code; the request_limit+ // metric also includes abort/deadline kinds that keep their existing+ // wire semantics. Ticket expiry remains provider_unavailable.metrics.observeAttemptFailure(err instanceof RequestLimitError && !(err instanceof TicketLifeError)? 'request_limit'@@ -780,8 +813,7 @@ export function createProxyApps()if (err instanceof UpstreamTimeoutError) metrics.observeUpstreamTimeout(err.kind)const deliveredAs = res.headersSent ? 'sse_error' : 'http_status'- const causeCode =- err instanceof ControlApiClientError ? err.causeCode : fetchCauseCode(err)+ const causeCode = err instanceof ControlApiClientError ? err.causeCode : fetchCauseCode(err)logger.warn({event: 'codex_proxy_attempt_finished',@@ -817,7 +849,9 @@ export function createProxyApps(: undefinedconst rejectedBy = typeof upstreamStatus === 'number' ? { upstreamStatus } : {}if (deliveredAs === 'sse_error') {- res.write(`data: ${JSON.stringify({ type: 'error', code: mapped.code, ...rejectedBy })}\n\n`)+ res.write(+ `data: ${JSON.stringify({ type: 'error', code: mapped.code, ...rejectedBy })}\n\n`+ )res.end()return}@@ -835,6 +869,10 @@ export function createProxyApps(} finally {stopHeartbeat?.()release?.()+ // Defence in depth for an early throw after this async owner took over+ // but before the visual release was moved into `release`.+ gated.codexStreamRelease?.()+ gated.codexStreamRelease = undefined// A demoted body's reservation, when the stream ended before the body// was written upstream. Budgeted bodies are also released on close.gated.codexBodyRelease?.()@@ -1005,6 +1043,9 @@ const ATTEMPT_ERROR_STATUS: Record<string, number> = {// would spend the same budget again, so it is a gateway timeout, not 503.stream_duration_exceeded: 504,connection_unavailable: 503,+ visual_host_share: 503,+ visual_gate: 503,+ proxy_capacity_exceeded: 503,provider_unavailable: 503,sse_buffer_exceeded: 503,invalid_receipt: 503,@@ -1045,7 +1086,6 @@ function isMappedError(err: unknown): err is MappedError {function mapError(err: unknown): { status: number; code: string } {if (!isMappedError(err)) return { status: 503, code: 'provider_unavailable' }if (err instanceof OriginDeniedError) return { status: 403, code: 'origin_denied' }- if (err instanceof RequestLimitError) return { status: 503, code: 'provider_unavailable' }return { status: attemptErrorStatus(err.code), code: err.code }}diff --git a/codex-llm-proxy/test/bodyAdmission.test.ts b/codex-llm-proxy/test/bodyAdmission.test.tsindex cb1bf9675..fd0fa06b2 100644--- a/codex-llm-proxy/test/bodyAdmission.test.ts+++ b/codex-llm-proxy/test/bodyAdmission.test.ts@@ -355,7 +355,7 @@ describe('codex-llm-proxy body admission (#731 R3-2)', () => {}}, 30_000)- it('T-R3-2c answers provider_unavailable once the admission queue is full', async () => {+ it('T-R3-2c answers proxy_capacity_exceeded once the admission queue is full', async () => {// A small body limit keeps this cheap; the budget scales with it.const maxBodyBytes = 16 * 1024const proxy = await heldProxy(maxBodyBytes)@@ -375,16 +375,16 @@ describe('codex-llm-proxy body admission (#731 R3-2)', () => {const overflow = await post(proxy.port, payload('overflow'))// Witnesses: the refusal was answered, and the admission logged why.expect(overflow.status).toBe(503)- expect(JSON.parse(overflow.body)).toEqual({ error: 'provider_unavailable' })+ expect(JSON.parse(overflow.body)).toEqual({ error: 'proxy_capacity_exceeded' })const logged = warn.mock.calls.map(call => call[0] as unknown as Record<string, unknown>)expect(logged.filter(entry => entry?.event === 'codex_proxy_denied')).toEqual([- { event: 'codex_proxy_denied', code: 'provider_unavailable' },+ { event: 'codex_proxy_denied', code: 'proxy_capacity_exceeded' },])expect(logged.filter(entry => entry?.event === 'codex_proxy_admission_refused')).toEqual([{event: 'codex_proxy_admission_refused',reason: 'body_budget',- code: 'provider_unavailable',+ code: 'proxy_capacity_exceeded',kind: 'queue_full',detail: expect.any(String),},diff --git a/codex-llm-proxy/test/server.security.test.ts b/codex-llm-proxy/test/server.security.test.tsindex ba6407aa2..bbcf213c6 100644--- a/codex-llm-proxy/test/server.security.test.ts+++ b/codex-llm-proxy/test/server.security.test.ts@@ -1,9 +1,9 @@+import { describe, expect, it, vi } from 'vitest'+import jwt from 'jsonwebtoken'import { generateKeyPairSync } from 'node:crypto'import { createServer, request as httpRequest } from 'node:http'import type { AddressInfo } from 'node:net'-import jwt from 'jsonwebtoken'import request from 'supertest'-import { describe, expect, it, vi } from 'vitest'import {ENVELOPE_ALLOWANCE_BYTES as CONTRACT_ENVELOPE_ALLOWANCE_BYTES,LIMITS,@@ -15,7 +15,7 @@ import {import { verifyAdminPermit } from '../src/auth/adminPermitVerifier.js'import { verifyExecutionTicket } from '../src/auth/executionTicketVerifier.js'import { CodexTransportError } from '../src/codexTransport.js'-import { loadConfig, type CodexLlmProxyConfig } from '../src/config.js'+import { type CodexLlmProxyConfig, loadConfig } from '../src/config.js'import {ControlApiClient,ControlApiClientError,@@ -184,9 +184,7 @@ describe('codex-llm-proxy security surface', () => {})it('reserves the larger transport budget for authenticated visual requests', async () => {- const { runtimeApp, adminApp } = createProxyApps(- config({ maxBodyBytes: 1_048_576 })- )+ const { runtimeApp, adminApp } = createProxyApps(config({ maxBodyBytes: 1_048_576 }))// Deliberately invalid ticket: this test checks parser admission and the// unchanged ticket gate, without redeeming or contacting any model.const payload = {@@ -266,7 +264,9 @@ describe('codex-llm-proxy security surface', () => {try {const visual = await send('x'.repeat(8_000_000), {schemaVersion: 'codex-completion-request.v2',- messages: [{ role: 'user', contentParts: [{ type: 'image', data: 'A'.repeat(1024 * 1024) }] }],+ messages: [+ { role: 'user', contentParts: [{ type: 'image', data: 'A'.repeat(1024 * 1024) }] },+ ],})expect(visual.status).toBe(400)expect(visual.body.error).toBe('invalid_request')@@ -304,7 +304,10 @@ describe('codex-llm-proxy security surface', () => {const { runtimeApp, probeApp } = createProxyApps(config({ maxBodyBytes: 65_536 }), {streamCompletion: async () => {streamCalls += 1- throw new CodexTransportError('payload_too_large', 'request exceeds maxVisualRequestBodyBytes')+ throw new CodexTransportError(+ 'payload_too_large',+ 'request exceeds maxVisualRequestBodyBytes'+ )},})const res = await request(runtimeApp)@@ -316,7 +319,9 @@ describe('codex-llm-proxy security surface', () => {expect(res.status).toBe(413)expect(res.body).toEqual({ error: 'payload_too_large' })const metricsText = (await request(probeApp).get('/metrics')).text- expect(metricsText).toMatch(/^codex_proxy_attempt_failures_total\{code="payload_too_large"\} 1$/m)+ expect(metricsText).toMatch(+ /^codex_proxy_attempt_failures_total\{code="payload_too_large"\} 1$/m+ )})it('does not let a V2 declaration raise the non-image budget to 24 MiB', async () => {@@ -392,9 +397,7 @@ describe('codex-llm-proxy security surface', () => {}, 60_000)it('rate limits the completion endpoint before body parsing and authorization', async () => {- const { runtimeApp } = createProxyApps(- config({ maxBodyBytes: 1024 })- )+ const { runtimeApp } = createProxyApps(config({ maxBodyBytes: 1024 }))const completion = () => request(runtimeApp).post('/internal/runtime/v1/codex/completions')const oversized = {executionTicket: 'invalid-ticket',@@ -918,17 +921,21 @@ describe('codex-llm-proxy startup config', () => {// accepts, so it is refused at startup, as the visual override is.it('T-R9-3 refuses a body limit below the contract request cap plus the envelope allowance', () => {const floor = LIMITS.maxRequestBodyBytes + ENVELOPE_ALLOWANCE_BYTES- expect(() => loadConfig({ ...base, CODEX_LLM_PROXY_MAX_BODY_BYTES: String(floor - 1) })).toThrow(+ expect(() =>+ loadConfig({ ...base, CODEX_LLM_PROXY_MAX_BODY_BYTES: String(floor - 1) })+ ).toThrow('CODEX_LLM_PROXY_MAX_BODY_BYTES must be at least the contract request cap plus the envelope allowance')expect(() =>loadConfig({ ...base, CODEX_LLM_PROXY_MAX_BODY_BYTES: String(LIMITS.maxRequestBodyBytes) })).toThrow(/CODEX_LLM_PROXY_MAX_BODY_BYTES must be at least/)// Witness: the floor itself and any larger value load.- expect(loadConfig({ ...base, CODEX_LLM_PROXY_MAX_BODY_BYTES: String(floor) }).maxBodyBytes).toBe(floor)- expect(loadConfig({ ...base, CODEX_LLM_PROXY_MAX_BODY_BYTES: String(floor + 1) }).maxBodyBytes).toBe(- floor + 1- )+ expect(+ loadConfig({ ...base, CODEX_LLM_PROXY_MAX_BODY_BYTES: String(floor) }).maxBodyBytes+ ).toBe(floor)+ expect(+ loadConfig({ ...base, CODEX_LLM_PROXY_MAX_BODY_BYTES: String(floor + 1) }).maxBodyBytes+ ).toBe(floor + 1)})it('T-R2-6c does not refuse a request at the contract cap with a real ticket as payload_too_large', async () => {@@ -945,14 +952,11 @@ describe('codex-llm-proxy startup config', () => {expect(typeof res.body.error).toBe('string')})- it.each([undefined, '', ' '])(- 'fails at startup when the control-api URL is %j',- value => {- expect(() => loadConfig({ ...base, CODEX_LLM_PROXY_CONTROL_API_URL: value })).toThrow(- /CODEX_LLM_PROXY_CONTROL_API_URL/- )- }- )+ it.each([undefined, '', ' '])('fails at startup when the control-api URL is %j', value => {+ expect(() => loadConfig({ ...base, CODEX_LLM_PROXY_CONTROL_API_URL: value })).toThrow(+ /CODEX_LLM_PROXY_CONTROL_API_URL/+ )+ })it.each(['not a url', 'ftp://control-api/api/v1', 'file:///etc/passwd'])('fails at startup when the control-api URL %j is not http(s)',@@ -1178,7 +1182,10 @@ describe('codex-llm-proxy attempt telemetry', () => {// Denies the redeem after `delayMs`, which is longer than the heartbeat// interval the caller configures.- function slowDenyingClient(code: string, delayMs: number): {+ function slowDenyingClient(+ code: string,+ delayMs: number+ ): {client: ControlApiClientredeemCalls: () => number} {@@ -1215,7 +1222,8 @@ describe('codex-llm-proxy attempt telemetry', () => {const warn = vi.spyOn(logger, 'warn')const { client, receipts } = grantingClient(options.maxStreamDurationMs)const apps = createProxyApps(config({ maxBodyBytes: 65_536, ...options.configOverrides }), {- controlApiClient: options.controlApiClient ?? (deniedCode ? denyingClient(deniedCode) : client),+ controlApiClient:+ options.controlApiClient ?? (deniedCode ? denyingClient(deniedCode) : client),fetchFn: options.fetchFn ?? upstream(textDeltas, calls, options.terminal),lookup: options.lookup ?? lookup,})@@ -1699,6 +1707,37 @@ describe('codex-llm-proxy attempt telemetry', () => {expect('err' in (lines[0] ?? {})).toBe(false)})+ it.each(['visual_host_share', 'visual_gate', 'proxy_capacity_exceeded'])(+ 'normalizes real transport upstream 503 claiming %s to provider_unavailable',+ async code => {+ const fetchFn = vi+ .fn<typeof fetch>()+ .mockResolvedValue(+ Response.json({ error: code, reason: 'local capacity claim' }, { status: 503 })+ )+ const { res, receipts, lines, metricsText } = await run(+ 'att-spoofed-local',+ 0,+ 0,+ undefined,+ undefined,+ { fetchFn }+ )+ expect(fetchFn).toHaveBeenCalledTimes(1)+ expect(res.status).toBe(503)+ expect(res.body).toEqual({ error: 'provider_unavailable' })+ expect(receipts).toEqual([expect.objectContaining({ outcome: 'error' })])+ expect(lines).toHaveLength(1)+ expect(lines[0]).toMatchObject({+ code: 'provider_unavailable',+ deliveredAs: 'http_status',+ httpStatus: 503,+ })+ expect(failureCount(metricsText, 'provider_unavailable')).toBe(1)+ expect(failureCount(metricsText, 'request_limit')).toBe(0)+ }+ )+it('(g1-1a) answers 429 rate_limited and forwards a valid upstream Retry-After', async () => {const { res, receipts, lines, metricsText } = await run('att-rate-http',@@ -1776,7 +1815,9 @@ describe('codex-llm-proxy attempt telemetry', () => {// Witness: a keepalive went out first, so only the frame can carry it.expect(keepaliveCount(res.text)).toBeGreaterThanOrEqual(1)expect(- res.text.endsWith('data: {"type":"error","code":"upstream_rejected","upstreamStatus":402}\n\n')+ res.text.endsWith(+ 'data: {"type":"error","code":"upstream_rejected","upstreamStatus":402}\n\n'+ )).toBe(true)expect(lines).toHaveLength(1)expect(lines[0]).toMatchObject({@@ -1885,14 +1926,9 @@ describe('codex-llm-proxy attempt telemetry', () => {})it('(p) T-R7-2c sends an SSE context_length_exceeded frame when text was already streamed', async () => {- const { res, lines } = await run(- 'att-context-sse',- 1,- 0,- undefined,- undefined,- { terminal: UPSTREAM_CONTEXT_OVERFLOW_FRAMES }- )+ const { res, lines } = await run('att-context-sse', 1, 0, undefined, undefined, {+ terminal: UPSTREAM_CONTEXT_OVERFLOW_FRAMES,+ })expect(res.status).toBe(200)expect(res.text).toContain('data: {"type":"text","text":"t0"}')expect(res.text).toContain('data: {"type":"error","code":"context_length_exceeded"}')@@ -1998,13 +2034,7 @@ describe('codex-llm-proxy attempt telemetry', () => {it.each(['constructor', '__proto__', 'toString', 'hasOwnProperty'])('(g) answers 503 for the inherited control-api code %j instead of crashing',async code => {- const { res, lines, metricsText } = await run(- `att-proto-${code}`,- 0,- 0,- undefined,- code- )+ const { res, lines, metricsText } = await run(`att-proto-${code}`, 0, 0, undefined, code)expect(res.status).toBe(503)expect(res.body).toEqual({ error: code })// Witness: the attempt was reached and logged, so the status above is the@@ -2019,20 +2049,22 @@ describe('codex-llm-proxy attempt telemetry', () => {it('(l) logs the request-limit reason and labels its failure metric request_limit', async () => {const acquire = vi.spyOn(streamGate, 'acquire')- .mockRejectedValueOnce(new RequestLimitError('stream queue is full', 'queue_full'))+ .mockRejectedValueOnce(+ new RequestLimitError('stream queue is full', 'queue_full', 'proxy_capacity_exceeded')+ )try {const { res, receipts, lines, metricsText } = await run('att-queue-full', 0, 0)// Witness: the refusal came from the stream gate this test replaced.expect(acquire).toHaveBeenCalledTimes(1)expect(res.status).toBe(503)- expect(res.body).toEqual({ error: 'provider_unavailable' })+ expect(res.body).toEqual({ error: 'proxy_capacity_exceeded' })// The gate refused before the redeem, so there is no receipt to finalize.expect(receipts).toEqual([])expect(lines).toHaveLength(1)expect(lines[0]).toMatchObject({providerAttemptId: 'att-queue-full',outcome: 'failed',- code: 'provider_unavailable',+ code: 'proxy_capacity_exceeded',reason: 'stream queue is full',deliveredAs: 'http_status',httpStatus: 503,@@ -2094,9 +2126,7 @@ describe('codex-llm-proxy attempt telemetry', () => {})expect(res.status).toBe(200)expect(keepaliveCount(res.text)).toBeGreaterThanOrEqual(1)- expect(res.text.endsWith('data: {"type":"error","code":"provider_unavailable"}\n\n')).toBe(- true- )+ expect(res.text.endsWith('data: {"type":"error","code":"provider_unavailable"}\n\n')).toBe(true)expect(lines).toHaveLength(1)expect(lines[0]).toMatchObject({outcome: 'failed',@@ -2306,7 +2336,10 @@ describe('codex-llm-proxy ticket-aware stream-gate wait (#739 D1-bis)', () => {.set('Authorization', `Bearer ${platformToken()}`).send(envelope('att-ticket-life', ticketLifeMs)).then(res => res)- await until(() => streamGate.snapshot().queued === 1, 'the request queueing at the stream gate')+ await until(+ () => streamGate.snapshot().queued === 1,+ 'the request queueing at the stream gate'+ )await vi.advanceTimersByTimeAsync(ticketLifeMs - 1)// Witness: the request waited for the ticket's whole life, it was not@@ -2359,7 +2392,10 @@ describe('codex-llm-proxy ticket-aware stream-gate wait (#739 D1-bis)', () => {.set('Authorization', `Bearer ${platformToken()}`).send(envelope('att-ticket-alive', 60_000)).then(res => res)- await until(() => streamGate.snapshot().queued === 1, 'the request queueing at the stream gate')+ await until(+ () => streamGate.snapshot().queued === 1,+ 'the request queueing at the stream gate'+ )await vi.advanceTimersByTimeAsync(50_000)expect(await withinReal(reply, 100)).toBeUndefined()@@ -2377,7 +2413,10 @@ describe('codex-llm-proxy ticket-aware stream-gate wait (#739 D1-bis)', () => {}, 30_000)/** A V2 envelope past `maxBodyBytes`, so it takes the visual gate. */- function visualEnvelope(providerAttemptId: string, ticketLifeMs: number): Record<string, unknown> {+ function visualEnvelope(+ providerAttemptId: string,+ ticketLifeMs: number+ ): Record<string, unknown> {const raw: Record<string, unknown> = {schemaVersion: 'codex-completion-request.v2',requestId: `req-${providerAttemptId}`,@@ -2431,7 +2470,10 @@ describe('codex-llm-proxy ticket-aware stream-gate wait (#739 D1-bis)', () => {.set('Authorization', `Bearer ${platformToken()}`).send(body).then(res => res)- await until(() => visualStreamGate.snapshot().queued === 1, 'the body queueing at the visual gate')+ await until(+ () => visualStreamGate.snapshot().queued === 1,+ 'the body queueing at the visual gate'+ )await vi.advanceTimersByTimeAsync(ticketLifeMs)// Witness: the body was still waiting for a slot at the ticket's expiry.@@ -2463,7 +2505,7 @@ describe('codex-llm-proxy ticket-aware stream-gate wait (#739 D1-bis)', () => {// overflow witness naming gate saturation. Grok already pins this twin; the// queue is filled directly so the refusal under test is the global visual// gate, not the per-principal share.- it('visual admission overflow answers 503 provider_unavailable and logs visual_gate', async () => {+ it('visual admission overflow answers 503 visual_gate and logs visual_gate', async () => {const warn = vi.spyOn(logger, 'warn')const held: Array<() => void> = []const waiting: Array<Promise<() => void>> = []@@ -2487,12 +2529,12 @@ describe('codex-llm-proxy ticket-aware stream-gate wait (#739 D1-bis)', () => {.set('Authorization', `Bearer ${platformToken({ sub: 'default/visual-gate-full' })}`).send(visualEnvelope('att-visual-gate-full', 60_000))expect(res.status).toBe(503)- expect(res.body).toEqual({ error: 'provider_unavailable' })+ expect(res.body).toEqual({ error: 'visual_gate' })expect(warn).toHaveBeenCalledWith(expect.objectContaining({event: 'codex_proxy_admission_refused',reason: 'visual_gate',- code: 'provider_unavailable',+ code: 'visual_gate',}),'admission refused')@@ -2837,7 +2879,7 @@ describe('codex-llm-proxy body budget release on upstream acceptance (#739 D2)',const client = open(proxy.port, payload)await until(() => client.ended(), 'the refusal')expect(client.status()).toBe(503)- expect(JSON.parse(client.received())).toEqual({ error: 'provider_unavailable' })+ expect(JSON.parse(client.received())).toEqual({ error: 'proxy_capacity_exceeded' })await until(() => proxy.closes() === 1, "the response's close event")expect(client.errors()).toEqual([])expect(proxy.reservations()).toEqual([Buffer.byteLength(payload)])@@ -3004,7 +3046,10 @@ describe('codex-llm-proxy graceful drain on shutdown (#739 D6)', () => {req.end(payload)let closing: Promise<void> | undefinedtry {- await until(() => received.includes('data: {"type":"text","text":"t0"}'), 'the first SSE frame')+ await until(+ () => received.includes('data: {"type":"text","text":"t0"}'),+ 'the first SSE frame'+ )closing = apps.close().then(() => {order.push('closed')})diff --git a/codex-llm-proxy/test/streamGate.handoff.test.ts b/codex-llm-proxy/test/streamGate.handoff.test.tsindex 0d03ff10e..730cd3b12 100644--- a/codex-llm-proxy/test/streamGate.handoff.test.ts+++ b/codex-llm-proxy/test/streamGate.handoff.test.ts@@ -360,7 +360,7 @@ describe('visual stream-gate handoff', () => {)const overflow = await postCompletion(port, 'codex-completion-request.v2', largeContent)expect(overflow.status).toBe(503)- expect(await overflow.json()).toEqual({ error: 'provider_unavailable' })+ expect(await overflow.json()).toEqual({ error: 'visual_gate' })const textWhileSaturated = postCompletion(port, 'codex-completion-request.v1')await waitFor(@@ -481,7 +481,7 @@ describe('visual stream-gate handoff', () => {)const fifth = await postCompletion(port, 'codex-completion-request.v2', largeContent)expect(fifth.status).toBe(503)- expect(await fifth.json()).toEqual({ error: 'provider_unavailable' })+ expect(await fifth.json()).toEqual({ error: 'visual_host_share' })expect(warn).toHaveBeenCalledWith({event: 'codex_proxy_admission_refused',@@ -619,7 +619,7 @@ describe('visual stream-gate handoff', () => {const fifth = await postBody(port, platformToken(), invalidTicketBody(maxBodyBytes))expect(fifth.status).toBe(503)- expect(await fifth.json()).toEqual({ error: 'provider_unavailable' })+ expect(await fifth.json()).toEqual({ error: 'visual_host_share' })// The refusal names the principal it applied to, so an operator can tell// which host was throttled.expect(warn).toHaveBeenCalledWith(@@ -704,7 +704,7 @@ describe('visual stream-gate handoff', () => {for (let i = 0; i < targetCount; i += 1) {const res = await postBody(port, platformToken(), invalidTicketBody(maxBodyBytes))expect(res.status).toBe(503)- expect(await res.json()).toEqual({ error: 'provider_unavailable' })+ expect(await res.json()).toEqual({ error: 'visual_gate' })}expect(acquire).toHaveBeenCalledTimes(VISUAL_STREAM_LIMITS.maxConcurrentStreams +@@ -789,7 +789,7 @@ describe('visual stream-gate handoff', () => {).toBe(2)const fifth = await refusedRequestexpect(fifth.status).toBe(503)- expect(await fifth.json()).toEqual({ error: 'provider_unavailable' })+ expect(await fifth.json()).toEqual({ error: 'visual_host_share' })expect(warn).toHaveBeenCalledWith({event: 'codex_proxy_admission_refused',diff --git a/codex-llm-proxy/test/streamLimitsFreeze.test.ts b/codex-llm-proxy/test/streamLimitsFreeze.test.tsindex 4e43e78a1..7ee0a3d09 100644--- a/codex-llm-proxy/test/streamLimitsFreeze.test.ts+++ b/codex-llm-proxy/test/streamLimitsFreeze.test.ts@@ -1,5 +1,6 @@import { describe, expect, it } from 'vitest'-import { readdirSync, readFileSync } from 'node:fs'+import { readFileSync, readdirSync } from 'node:fs'+import ts from 'typescript'import { STREAM_LIMITS } from '../src/requestLimits.js'const fixturePath = new URL(@@ -45,8 +46,10 @@ function emittedTransportCodes(): { codes: Set<string>; sites: number; constructfor (const name of readdirSync(srcDir).filter(file => file.endsWith('.ts'))) {const source = readFileSync(new URL(name, srcDir), 'utf8')constructions +=- source.split('new CodexTransportError(').length - 1 +- source.split('new UpstreamTimeoutError(').length - 1+ source.split('new CodexTransportError(').length -+ 1 ++ source.split('new UpstreamTimeoutError(').length -+ 1for (const match of source.matchAll(TRANSPORT_ERROR_SITE)) {const args = callArguments(source, match.index! + match[0].length - 1)const code = args[match[1] === 'UpstreamTimeoutError' ? 1 : 0] ?? ''@@ -70,19 +73,79 @@ const REJECT_SITE = /reject\(res,/g// Refusal codes are wire strings; `Unauthorized` is the one spelled in capitals.const REJECT_CODE_LITERAL = /^'([A-Za-z][A-Za-z0-9_]+)'$/// The only computed codes a refusal may carry, as their exact source text.-// `err.code` is a RequestLimitError (always provider_unavailable) and+// `err.code` uses the closed, readonly RequestLimitCode union checked below;// `mapped.code` is a transport or control-api code passed through mapError,// whose transport half the scanner above already gates. Any other computed// form cannot be checked against the taxonomy, so it fails here.const COMPUTED_REJECT_CODES = ['err.code', 'mapped.code'] as const+/** Computed admission codes are allowed only through this closed, readonly type. */+function requestLimitCodes(+ sourceText = readFileSync(new URL('requestLimits.ts', srcDir), 'utf8')+): Set<string> {+ const source = ts.createSourceFile('requestLimits.ts', sourceText, ts.ScriptTarget.Latest, true)+ const alias = source.statements.find(+ (node): node is ts.TypeAliasDeclaration =>+ ts.isTypeAliasDeclaration(node) && node.name.text === 'RequestLimitCode'+ )+ if (!alias || !ts.isUnionTypeNode(alias.type)) {+ throw new Error('RequestLimitCode must remain a closed literal union')+ }+ const codes = new Set(+ alias.type.types.map(node => {+ if (!ts.isLiteralTypeNode(node) || !ts.isStringLiteral(node.literal)) {+ throw new Error('RequestLimitCode must not accept computed or open string types')+ }+ return node.literal.text+ })+ )+ const owner = source.statements.find(+ (node): node is ts.ClassDeclaration =>+ ts.isClassDeclaration(node) && node.name?.text === 'RequestLimitError'+ )+ const property = owner?.members.find(+ (node): node is ts.PropertyDeclaration =>+ ts.isPropertyDeclaration(node) && node.name.getText(source) === 'code'+ )+ expect(property?.type?.getText(source)).toBe('RequestLimitCode')+ expect(property?.modifiers?.some(node => node.kind === ts.SyntaxKind.ReadonlyKeyword)).toBe(true)+ const constructor = owner?.members.find(ts.isConstructorDeclaration)+ const input = constructor?.parameters.find(node => node.name.getText(source) === 'code')+ expect(input?.type?.getText(source)).toBe('RequestLimitCode')+ expect(input?.initializer?.getText(source)).toBe("'provider_unavailable'")+ const assignments: string[] = []+ const collect = (node: ts.Node): void => {+ if (+ ts.isBinaryExpression(node) &&+ node.operatorToken.kind === ts.SyntaxKind.EqualsToken &&+ node.left.getText(source) === 'this.code'+ ) {+ assignments.push(node.right.getText(source))+ }+ ts.forEachChild(node, collect)+ }+ if (owner) collect(owner)+ expect(assignments).toEqual(['code'])+ return codes+}++function rejectCodeLiteral(code: string, name: string): RegExpExecArray | null {+ const literal = REJECT_CODE_LITERAL.exec(code)+ const allowed = (COMPUTED_REJECT_CODES as readonly string[]).includes(code)+ expect(+ literal !== null || allowed,+ `${name}: reject code must be a string literal or one of ${COMPUTED_REJECT_CODES.join(', ')}, got ${code}`+ ).toBe(true)+ return literal+}+function emittedRejectCodes(): {codes: Set<string>computed: Set<string>sites: numberoccurrences: number} {- const codes = new Set<string>()+ const codes = requestLimitCodes()const computed = new Set<string>()let sites = 0let occurrences = 0@@ -92,12 +155,7 @@ function emittedRejectCodes(): {for (const match of source.matchAll(REJECT_SITE)) {const args = callArguments(source, match.index! + 'reject'.length)const code = args[2] ?? ''- const literal = REJECT_CODE_LITERAL.exec(code)- const allowed = (COMPUTED_REJECT_CODES as readonly string[]).includes(code)- expect(- literal !== null || allowed,- `${name}: reject code must be a string literal or one of ${COMPUTED_REJECT_CODES.join(', ')}, got ${code}`- ).toBe(true)+ const literal = rejectCodeLiteral(code, name)if (literal) codes.add(literal[1]!)else computed.add(code)sites += 1@@ -107,6 +165,42 @@ function emittedRejectCodes(): {}describe('codex-subscription stream limits freeze', () => {+ it('rejects open or mutable admission code producers while reading the closed production union', () => {+ const sourceText = readFileSync(new URL('requestLimits.ts', srcDir), 'utf8')+ const tree = ts.createSourceFile('requestLimits.ts', sourceText, ts.ScriptTarget.Latest, true)+ const alias = tree.statements.find(+ (node): node is ts.TypeAliasDeclaration =>+ ts.isTypeAliasDeclaration(node) && node.name.text === 'RequestLimitCode'+ )!+ const withType = (type: string) =>+ sourceText.slice(0, alias.type.getStart(tree)) + type + sourceText.slice(alias.type.end)+ expect([...requestLimitCodes(sourceText)]).toEqual(+ expect.arrayContaining(['provider_unavailable', 'proxy_capacity_exceeded', 'visual_gate'])+ )+ expect(() => requestLimitCodes(withType('string'))).toThrow('closed literal union')+ expect(() => requestLimitCodes(withType("'provider_unavailable' | string"))).toThrow(+ 'open string types'+ )+ expect(() =>+ requestLimitCodes(+ sourceText.replace('readonly code: RequestLimitCode', 'code: RequestLimitCode')+ )+ ).toThrow()+ expect(() =>+ requestLimitCodes(sourceText.replace('this.code = code', "this.code = 'arbitrary_code'"))+ ).toThrow()+ })++ it.each(['caller.code', "err.code || 'provider_unavailable'", 'String(err.code)'])(+ 'rejects unbounded computed refusal expression %s',+ expression => {+ expect(rejectCodeLiteral("'request_timeout'", 'fixture')).not.toBeNull()+ expect(rejectCodeLiteral('err.code', 'fixture')).toBeNull()+ expect(rejectCodeLiteral('mapped.code', 'fixture')).toBeNull()+ expect(() => rejectCodeLiteral(expression, 'fixture')).toThrow('reject code must be')+ }+ )+it('pins the proxy STREAM_LIMITS to the limits the fixture publishes', () => {// StreamGate and the upstream deadlines enforce these published bounds// from the proxy's own constant, not from the contract package, so thediff --git a/codex-llm-proxy/test/visualBodyReadOwnership.test.ts b/codex-llm-proxy/test/visualBodyReadOwnership.test.tsnew file mode 100644index 000000000..d283efb0b--- /dev/null+++ b/codex-llm-proxy/test/visualBodyReadOwnership.test.ts@@ -0,0 +1,532 @@+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'+import express, { type NextFunction, type Request, type Response } from 'express'+import jwt from 'jsonwebtoken'+import { generateKeyPairSync } from 'node:crypto'+import { createServer } from 'node:http'+import { connect as connectTcp } from 'node:net'+import type { CodexLlmProxyConfig } from '../src/config.js'+import {+ VISUAL_PER_HOST_MAX_ADMITTED,+ VISUAL_STREAM_LIMITS,+ visualStreamGate,+} from '../src/requestLimits.js'+import { createProxyApps } from '../src/server.js'++type ObservedRead = {+ req: Request+ res: Response+ nativeSettled: boolean+ returned: boolean+ error: unknown+ finishes: number+ endCalls: number+}++const parserBarrier = vi.hoisted(() => {+ let release!: () => void+ let completion: Promise<void>+ const reads: ObservedRead[] = []+ const reset = () => {+ reads.splice(0)+ completion = new Promise<void>(resolve => {+ release = resolve+ })+ }+ reset()+ return {+ reads,+ reset,+ get completion() {+ return completion+ },+ get settledCount() {+ return reads.filter(read => read.nativeSettled).length+ },+ release() {+ release()+ },+ }+})++vi.mock('express', async () => {+ const actual = await vi.importActual<typeof import('express')>('express')+ const actualDefault = (actual as unknown as { default: typeof express }).default+ const mockedDefault = Object.assign(+ (...args: Parameters<typeof express>) => actualDefault(...args),+ {+ json: (options?: Parameters<typeof actual.json>[0]) => {+ const parser = actualDefault.json(options)+ return (req: Request, res: Response, next: NextFunction) => {+ // Observe the native visual reader. Ordinary JSON parsing stays native+ // too, but must not count as starting the next visual reader.+ const read: ObservedRead | undefined =+ options?.limit === 8192+ ? {+ req,+ res,+ nativeSettled: false,+ returned: false,+ error: undefined,+ finishes: 0,+ endCalls: 0,+ }+ : undefined+ if (read) {+ parserBarrier.reads.push(read)+ res.once('finish', () => {+ read.finishes += 1+ })+ if (req.headers['x-visual-never-finish'] === '1') {+ // Fault injection: no finish/close is produced by writing 408.+ // The production close backstop must independently stop raw-body.+ res.end = (() => {+ read.endCalls += 1+ return res+ }) as Response['end']+ }+ }+ parser(req, res, (err: unknown) => {+ if (read) {+ read.nativeSettled = true+ read.error = err+ }+ const finish = () => {+ next(err)+ if (read) read.returned = true+ }+ if (req.headers['x-visual-parser-hold'] === '1') {+ void parserBarrier.completion.then(finish)+ } else {+ finish()+ }+ })+ }+ },+ }+ )+ return { ...actual, default: mockedDefault, json: mockedDefault.json }+})++const COMPLETIONS_PATH = '/internal/runtime/v1/codex/completions'+const { privateKey, publicKey } = generateKeyPairSync('rsa', {+ modulusLength: 2048,+ publicKeyEncoding: { type: 'spki', format: 'pem' },+ privateKeyEncoding: { type: 'pkcs8', format: 'pem' },+})++function config(): CodexLlmProxyConfig {+ return {+ runtimePort: 0,+ adminPort: 0,+ probePort: 0,+ maxBodyBytes: 4096,+ maxVisualBodyBytes: 8192,+ maxStreamDurationMs: 1_800_000,+ maxDeadlineMs: 1_800_000,+ upstreamIdleTimeoutMs: 600_000,+ heartbeatIntervalMs: 15_000,+ jwtIssuer: 'control-api',+ jwtPublicKey: publicKey,+ executionEnabled: true,+ controlApiBaseUrl: '',+ controlApiServiceName: 'codex-llm-proxy',+ controlApiServiceToken: '',+ }+}++function platformToken(): string {+ return jwt.sign(+ {+ sub: 'default/research-host',+ hostRefs: ['research-host'],+ workflowControlScopes: ['llm:codex:execute'],+ scope: 'workflow:approval:request',+ },+ privateKey,+ { algorithm: 'RS256', issuer: 'control-api', audience: 'workflow-approvals', expiresIn: 60 }+ )+}++function platformTokenForOtherPrincipal(): string {+ return jwt.sign(+ {+ sub: 'default/other-host',+ hostRefs: ['research-host'],+ workflowControlScopes: ['llm:codex:execute'],+ scope: 'workflow:approval:request',+ },+ privateKey,+ { algorithm: 'RS256', issuer: 'control-api', audience: 'workflow-approvals', expiresIn: 60 }+ )+}++async function waitFor(predicate: () => boolean, label: string): Promise<void> {+ const started = Date.now()+ while (!predicate()) {+ if (Date.now() - started > 5_000) throw new Error(label)+ await new Promise(resolve => setTimeout(resolve, 5))+ }+}++const sockets = new Set<ReturnType<typeof connectTcp>>()++function stalledVisualBody(+ port: number,+ declared: number,+ holdParser = false,+ extraHeaders = '',+ prefix = '{"executionTicket":'+): { socket: ReturnType<typeof connectTcp>; response: Promise<number> } {+ let owner: ReturnType<typeof connectTcp> | undefined+ const response = new Promise<number>((resolve, reject) => {+ owner = connectTcp(port, '127.0.0.1', () => {+ owner?.write(+ `POST ${COMPLETIONS_PATH} HTTP/1.1\r\n` ++ `Host: 127.0.0.1:${port}\r\n` ++ `Authorization: Bearer ${platformToken()}\r\n` ++ `Content-Type: application/json\r\n` ++ (holdParser ? 'X-Visual-Parser-Hold: 1\r\n' : '') ++ extraHeaders ++ `Content-Length: ${declared}\r\n` ++ `\r\n` ++ prefix+ )+ })+ sockets.add(owner)+ owner.once('close', () => sockets.delete(owner!))+ owner.setTimeout(5_000, () => {+ owner?.destroy()+ reject(new Error('the stalled visual body produced no response'))+ })+ const chunks: Buffer[] = []+ owner.on('data', data => {+ chunks.push(data)+ const match = /^HTTP\/1\.1 (\d+)/.exec(Buffer.concat(chunks).toString('utf8'))+ if (match) resolve(Number(match[1]))+ })+ owner.on('error', err => {+ if ((err as NodeJS.ErrnoException).code !== 'ECONNRESET') reject(err)+ })+ })+ return { socket: owner!, response }+}++function invalidTicketBody(): string {+ const content = 'x'.repeat(5000)+ return JSON.stringify({+ executionTicket: 'invalid-ticket',+ requestHash: 'a'.repeat(64),+ request: {+ schemaVersion: 'codex-completion-request.v2',+ requestId: 'req-visual-ownership',+ idempotencyKey: 'idem-visual-ownership',+ provider: 'codex-subscription',+ model: 'gpt-5.1',+ messages: [{ role: 'user', content }],+ },+ })+}++function postVisualRaw(+ port: number,+ body: string,+ holdParser = false+): { socket: ReturnType<typeof connectTcp>; response: Promise<number> } {+ let owner: ReturnType<typeof connectTcp> | undefined+ const response = new Promise<number>((resolve, reject) => {+ owner = connectTcp(port, '127.0.0.1', () => {+ owner?.write(+ `POST ${COMPLETIONS_PATH} HTTP/1.1\r\n` ++ `Host: 127.0.0.1:${port}\r\n` ++ `Authorization: Bearer ${platformToken()}\r\n` ++ `Content-Type: application/json\r\n` ++ (holdParser ? 'X-Visual-Parser-Hold: 1\r\n' : '') ++ `Content-Length: ${Buffer.byteLength(body)}\r\n` ++ `\r\n` ++ body+ )+ })+ sockets.add(owner)+ owner.once('close', () => sockets.delete(owner!))+ owner.setTimeout(5_000, () => {+ owner?.destroy()+ reject(new Error('the visual request produced no response'))+ })+ const chunks: Buffer[] = []+ owner.on('data', data => {+ chunks.push(data)+ const match = /^HTTP\/1\.1 (\d+)/.exec(Buffer.concat(chunks).toString('utf8'))+ if (match) resolve(Number(match[1]))+ })+ owner.on('error', err => {+ if ((err as NodeJS.ErrnoException).code !== 'ECONNRESET') reject(err)+ })+ })+ return { socket: owner!, response }+}++async function postVisualStatus(port: number, body: string): Promise<number> {+ const request = postVisualRaw(port, body)+ const status = await request.response+ request.socket.destroy()+ return status+}++describe('codex visual body read ownership', () => {+ const servers: Array<ReturnType<typeof createProxyApps>> = []+ const listeners: Array<ReturnType<typeof createServer>> = []++ beforeEach(() => parserBarrier.reset())++ afterEach(async () => {+ parserBarrier.release()+ vi.restoreAllMocks()+ for (const socket of sockets) socket.destroy()+ await Promise.all(+ listeners+ .splice(0)+ .map(+ listener =>+ new Promise<void>((resolve, reject) =>+ listener.close(err => (err ? reject(err) : resolve()))+ )+ )+ )+ await Promise.all(servers.splice(0).map(server => server.close()))+ await waitFor(+ () => visualStreamGate.snapshot().running === 0 && visualStreamGate.snapshot().queued === 0,+ 'the visual gate did not drain after the ownership test'+ )+ })++ it('keeps the visual slot and principal share until the timed-out parser terminates', async () => {+ const apps = createProxyApps(config(), {+ bodyReadDeadlineMs: 200,+ visualReadCloseGraceMs: 10,+ })+ servers.push(apps)+ const listener = createServer(apps.runtimeApp).listen(0)+ listeners.push(listener)+ const address = listener.address()+ if (!address || typeof address === 'string') throw new Error('listener has no port')++ const readers = Array.from({ length: VISUAL_PER_HOST_MAX_ADMITTED }, () =>+ stalledVisualBody(address.port, 5000, true)+ )+ await waitFor(+ () =>+ visualStreamGate.snapshot().running + visualStreamGate.snapshot().queued ===+ VISUAL_PER_HOST_MAX_ADMITTED,+ 'the stalled visual bodies did not fill the principal share'+ )+ const statuses = await Promise.all(+ readers.slice(0, VISUAL_STREAM_LIMITS.maxConcurrentStreams).map(reader => reader.response)+ )+ expect(statuses).toEqual(+ Array.from({ length: VISUAL_STREAM_LIMITS.maxConcurrentStreams }, () => 408)+ )+ expect(visualStreamGate.snapshot().running + visualStreamGate.snapshot().queued).toBe(+ VISUAL_PER_HOST_MAX_ADMITTED+ )++ const shareRefusal = await postVisualStatus(address.port, invalidTicketBody())+ expect(shareRefusal).toBe(503)+ // A different principal can queue, but the reader must not start until the+ // existing parser callbacks finish. This observes reader entry, not merely+ // the absence of an HTTP response from the next request.+ const nextSocket = connectTcp(address.port, '127.0.0.1')+ sockets.add(nextSocket)+ nextSocket.once('close', () => sockets.delete(nextSocket))+ const nextStatus = new Promise<number>((resolve, reject) => {+ nextSocket.once('connect', () => {+ const body = invalidTicketBody()+ nextSocket.write(+ `POST ${COMPLETIONS_PATH} HTTP/1.1\r\n` ++ `Host: 127.0.0.1:${address.port}\r\n` ++ `Authorization: Bearer ${platformTokenForOtherPrincipal()}\r\n` ++ `Content-Type: application/json\r\n` ++ `Content-Length: ${Buffer.byteLength(body)}\r\n\r\n${body}`+ )+ })+ nextSocket.on('data', data => {+ const match = /^HTTP\/1\.1 (\d+)/.exec(data.toString('utf8'))+ if (match) resolve(Number(match[1]))+ })+ nextSocket.once('error', reject)+ })+ await waitFor(+ () =>+ visualStreamGate.snapshot().queued ===+ VISUAL_PER_HOST_MAX_ADMITTED - VISUAL_STREAM_LIMITS.maxConcurrentStreams + 1,+ 'the next principal did not queue behind the retained parsers'+ )+ expect(parserBarrier.reads).toHaveLength(VISUAL_STREAM_LIMITS.maxConcurrentStreams)++ for (const reader of readers) {+ reader.response.catch(() => undefined)+ reader.socket.destroy()+ }+ await waitFor(+ () => parserBarrier.settledCount === VISUAL_STREAM_LIMITS.maxConcurrentStreams,+ 'the held parsers did not terminate'+ )+ expect(visualStreamGate.snapshot().running).toBe(VISUAL_STREAM_LIMITS.maxConcurrentStreams)+ expect(visualStreamGate.snapshot().queued).toBeGreaterThanOrEqual(1)+ expect(parserBarrier.reads).toHaveLength(VISUAL_STREAM_LIMITS.maxConcurrentStreams)+ parserBarrier.release()+ expect(await nextStatus).toBe(403)+ nextSocket.destroy()+ await waitFor(+ () => visualStreamGate.snapshot().running === 0 && visualStreamGate.snapshot().queued === 0,+ 'the timed-out visual reader did not terminate and release its admission'+ )+ expect(+ parserBarrier.reads+ .slice(0, VISUAL_STREAM_LIMITS.maxConcurrentStreams)+ .every(read => read.returned && read.req.body === undefined)+ ).toBe(true)+ })++ it('uses the close backstop when the visual client disappears before the deadline', async () => {+ const apps = createProxyApps(config(), {+ bodyReadDeadlineMs: 5_000,+ visualReadCloseGraceMs: 10,+ })+ servers.push(apps)+ const listener = createServer(apps.runtimeApp).listen(0)+ listeners.push(listener)+ const address = listener.address()+ if (!address || typeof address === 'string') throw new Error('listener has no port')++ const stalled = stalledVisualBody(address.port, 5000)+ await waitFor(+ () => visualStreamGate.snapshot().running === 1,+ 'the visual body did not take the slot'+ )+ stalled.socket.destroy()+ stalled.response.catch(() => undefined)+ await waitFor(+ () => visualStreamGate.snapshot().running === 0 && visualStreamGate.snapshot().queued === 0,+ 'a disconnected visual reader retained its admission after the close backstop'+ )+ })++ it('releases the visual admission after a parser error', async () => {+ const apps = createProxyApps(config())+ servers.push(apps)+ const listener = createServer(apps.runtimeApp).listen(0)+ listeners.push(listener)+ const address = listener.address()+ if (!address || typeof address === 'string') throw new Error('listener has no port')++ const status = await postVisualStatus(address.port, `{"executionTicket":${' '.repeat(5000)}`)+ expect(status).toBe(400)+ await waitFor(+ () => visualStreamGate.snapshot().running === 0 && visualStreamGate.snapshot().queued === 0,+ 'a malformed visual body did not release its admission'+ )+ })++ it('transfers a successfully parsed visual body to the route and releases it after the response', async () => {+ const apps = createProxyApps(config())+ servers.push(apps)+ const listener = createServer(apps.runtimeApp).listen(0)+ listeners.push(listener)+ const address = listener.address()+ if (!address || typeof address === 'string') throw new Error('listener has no port')++ const status = await postVisualStatus(address.port, invalidTicketBody())+ expect(status).toBe(403)+ await waitFor(+ () => visualStreamGate.snapshot().running === 0 && visualStreamGate.snapshot().queued === 0,+ 'a parsed visual body did not release its admission after the route response'+ )+ })+ it('stops the native reader independently when a non-reading peer never finishes the response', async () => {+ const apps = createProxyApps(config(), { bodyReadDeadlineMs: 40, visualReadCloseGraceMs: 15 })+ servers.push(apps)+ const listener = createServer(apps.runtimeApp).listen(0)+ listeners.push(listener)+ const address = listener.address()+ if (!address || typeof address === 'string') throw new Error('listener has no port')++ const stalled = stalledVisualBody(address.port, 5000, false, 'X-Visual-Never-Finish: 1\r\n')+ stalled.response.catch(() => undefined)+ stalled.socket.pause()+ await waitFor(() => parserBarrier.reads.length === 1, 'the native visual reader did not start')+ const read = parserBarrier.reads[0]!+ await waitFor(() => read.returned, 'the independent close backstop did not terminate raw-body')+ expect(read.endCalls).toBe(1)+ expect(read.finishes).toBe(0)+ expect(read.req.destroyed).toBe(true)+ expect(read.req.body).toBeUndefined()+ expect(visualStreamGate.snapshot()).toEqual({ running: 0, queued: 0 })+ expect(await postVisualStatus(address.port, invalidTicketBody())).toBe(403)+ })++ it('clears a late successful native parse after its read deadline without dispatching it', async () => {+ const apps = createProxyApps(config(), { bodyReadDeadlineMs: 40, visualReadCloseGraceMs: 1000 })+ servers.push(apps)+ const listener = createServer(apps.runtimeApp).listen(0)+ listeners.push(listener)+ const address = listener.address()+ if (!address || typeof address === 'string') throw new Error('listener has no port')++ const body = invalidTicketBody()+ const split = 128+ const stalled = stalledVisualBody(+ address.port,+ Buffer.byteLength(body),+ false,+ 'X-Visual-Never-Finish: 1\r\n',+ body.slice(0, split)+ )+ stalled.response.catch(() => undefined)+ await waitFor(+ () => parserBarrier.reads[0]?.endCalls === 1,+ 'the visual reader never reached its deadline'+ )+ const read = parserBarrier.reads[0]!+ expect(read.nativeSettled).toBe(false)+ expect(visualStreamGate.snapshot().running).toBe(1)+ stalled.socket.write(body.slice(split))+ await waitFor(() => read.returned, 'the late native parser callback did not run')+ expect(read.error).toBeUndefined()+ expect(read.req.body).toBeUndefined()+ expect(visualStreamGate.snapshot()).toEqual({ running: 0, queued: 0 })+ stalled.socket.destroy()+ expect(await postVisualStatus(address.port, invalidTicketBody())).toBe(403)+ })++ it('clears the retained parsed body after response close before its callback returns', async () => {+ const apps = createProxyApps(config(), { bodyReadDeadlineMs: 5000, visualReadCloseGraceMs: 15 })+ servers.push(apps)+ const listener = createServer(apps.runtimeApp).listen(0)+ listeners.push(listener)+ const address = listener.address()+ if (!address || typeof address === 'string') throw new Error('listener has no port')++ const request = postVisualRaw(address.port, invalidTicketBody(), true)+ request.response.catch(() => undefined)+ await waitFor(() => parserBarrier.settledCount === 1, 'the complete body was never parsed')+ const read = parserBarrier.reads[0]!+ expect(read.req.body).toBeDefined()+ expect(read.returned).toBe(false)+ request.socket.destroy()+ await waitFor(() => read.res.destroyed, 'the response did not close')+ expect(visualStreamGate.snapshot().running).toBe(1)+ parserBarrier.release()+ await waitFor(() => read.returned, 'the disconnected parser callback did not run')+ expect(read.req.body).toBeUndefined()+ expect(visualStreamGate.snapshot()).toEqual({ running: 0, queued: 0 })+ expect(await postVisualStatus(address.port, invalidTicketBody())).toBe(403)+ })++ it.each([0, -1, 1.5, Number.NaN, Number.POSITIVE_INFINITY, 2147483648])(+ 'rejects invalid native reader clocks (%s)',+ value => {+ expect(() => createProxyApps(config(), { bodyReadDeadlineMs: value })).toThrow(RangeError)+ expect(() => createProxyApps(config(), { visualReadCloseGraceMs: value })).toThrow(RangeError)+ }+ )+})diff --git a/docs/architecture/codex-subscription-transport-contract.md b/docs/architecture/codex-subscription-transport-contract.mdindex 9ebb10729..49e2d2284 100644--- a/docs/architecture/codex-subscription-transport-contract.md+++ b/docs/architecture/codex-subscription-transport-contract.md@@ -107,12 +107,23 @@ existing kill switches (`MCP_HOST_CODEX_SUBSCRIPTION_ENABLED` and`CODEX_LLM_PROXY_EXECUTION_ENABLED`). Missing image provenance returns`image_source_invalid`. Limit errors, including HTTP 413 without a JSON body,remain non-retryable and do not fail over. Images are never silently stripped.-Two 503s the proxy gives an image request before redeem are different:-`visual_gate` (the visual gate is full) and `visual_host_share` (the-principal's share of it is full) answer `provider_unavailable`, which the Host-classifies as `LLM_MODEL_OVERLOADED`, retryable, and a Host with a fallback-provider configured fails over on it. Whether that should stay retryable is-tracked in #868.+Three local-capacity outcomes the proxy can give before redeem are closed wire+identities: `visual_host_share` (the principal's share is full), `visual_gate`+(the visual gate's queue is full or waited out), and+`proxy_capacity_exceeded` (ordinary stream/body capacity). They classify as+non-retryable `LLM_API_CALL_FAILED`, do not construct fallback or install its+cooldown, and leave the primary usable for the next text turn. Only the+proxy-owned machine-code field supplies that identity; upstream prose or a+`reason` field never does. A timed-out visual reader keeps its slot and+principal share until parser termination, with a bounded close/destruction+backstop. Real upstream outage, connect-failure and 429 semantics remain+unchanged.++The critical Codex test title migrated from "visual admission overflow answers+503 provider_unavailable and logs visual_gate" to "visual admission overflow+answers 503 visual_gate and logs visual_gate". Its physical file and the original+503, refusal-log, no-redeem and capacity-recovery protections remain pinned; the+wire literal changed to reflect the local-capacity contract.Residual risk. Tool screenshots reach the upstream in a user-role message,after the tool messages, behind the fixed text "These images are output of the@@ -504,6 +515,7 @@ behavior changes:and logged as `codex_proxy_admission_refused` with `reason: visual_gate`.A client that leaves while queued is not gate saturation: its place is freedsilently, with no log and no response.+- It requires the redeem response to carry `maxStreamDurationMs` greaterthan 0. An absent value is a contract violation, not a default.- It logs one `codex_proxy_attempt_finished` event per completion attempt,@@ -542,6 +554,7 @@ behavior changes:`provider_unavailable`. The metric labels it `request_limit` to keep itapart from upstream outages, and the log line carries the limit's fixed`reason`.+- **Live-target attestation.** Codex authorize attests the live Host or recipetarget. The allowed providers come only from the spec's model, allowedmodels and fallbacks (Hosts) or agent providers (recipes). A target that@@ -712,7 +725,7 @@ code the proxy constructs, and every code it refuses a request with| -------------------------------------------------------- | --------------------------------- || `request exceeds maxRequestBodyBytes` | serialized UTF-8 byte cap || `request exceeds maxRequestBodyBytes outside image data` | non-image share of a V2 request |- | `request exceeds maxRequestElements` | element count in `checkStructure` |+ | `request exceeds maxRequestElements` | element count in `checkStructure` || `messages exceed <maxMessages>` | message count || `messages[i].toolCalls exceed <maxToolCalls>` | tool calls on one message |diff --git a/docs/architecture/grok-subscription-transport-contract.md b/docs/architecture/grok-subscription-transport-contract.mdindex 8aa3289dc..a49ed75e6 100644--- a/docs/architecture/grok-subscription-transport-contract.md+++ b/docs/architecture/grok-subscription-transport-contract.md@@ -143,10 +143,11 @@ ordinary limit.identity (`sub` plus sorted `hostRefs`) may hold at most 2 visual entries(running or queued) of the gate's 5. A token whose `sub` is missing, empty ornot a string is refused 401 before any gate or share is taken. When the share-is full, the proxy answers 503 `provider_unavailable` and logs+is full, the proxy answers 503 `visual_host_share` and logs`grok_proxy_admission_refused` with `reason: visual_host_share`. The share is-released together with the visual slot: on read-deadline expiry, on client-close before hand-off, and when the handler's finally block unwinds.+released with the visual slot only after parser termination (or by the route's+stream finally); a timeout or response close first bounds and destroys the+still-owned request so body-parser can unwind.The visual wait uses the request's single admission clock (arrival +`maxQueueWaitMs`, 60 s). Unlike the ordinary stream gate, it does not end at@@ -154,14 +155,16 @@ the ticket's `exp`. A ticket that expired while its visual body waited isrefused `ticket_expired` when the proxy verifies it after the read, before anyredeem. Once a slot is granted, the body must be read and parsed within`BODY_READ_DEADLINE_MS` (10 s); otherwise the proxy answers 408-`request_timeout` with `connection: close` and frees the slot. After the grant,-the slot is also freed when the response closes for any other reason, which-covers every refusal before the stream starts. The raw-body scan+`request_timeout` with `connection: close`, starts a bounded destruction+backstop, and releases the slot and principal share only in body-parser's+completion callback. A response close before hand-off follows the same+backstop. The raw-body scanrefuses a body nested deeper than 70 (`BODY_STRUCTURE_LIMITS.maxDepth`) with400 `invalid_request` before `JSON.parse`, so `JSON.stringify` and the byte-measurement never see a deeper tree. A visual-gate refusal (queue full, wait exceeded) is-answered 503 `provider_unavailable` and logged as-`grok_proxy_admission_refused` with `reason: visual_gate`.+measurement never see a deeper tree. A visual-gate capacity refusal (queue+full or queue wait) is answered 503 `visual_gate`. Ordinary stream/body queue+full and queue-wait capacity refusals answer 503 `proxy_capacity_exceeded`;+deadline, abort and ticket-life outcomes keep their existing identities.A client that disconnects while its request is queued frees its queue placeonly if Node sees the disconnect. Node keeps reading a queued request's socket@@ -230,11 +233,12 @@ Errors:`payload_too_large`. The Host maps `payload_too_large` to`LLM_CONTEXT_LENGTH_EXCEEDED`, not retryable.- Images are never stripped silently, and a size refusal never triggers a- provider fallback. A 503 from the visual gate or from the per-principal- share (`visual_gate`, `visual_host_share`) is different: the proxy answers- `provider_unavailable`, the Host classifies it as `LLM_MODEL_OVERLOADED`,- retryable, and a Host with a fallback provider configured fails over on it.- Whether that should stay retryable is tracked in #868.+ provider fallback. Local proxy capacity outcomes are also terminal:+ `visual_host_share`, `visual_gate`, and ordinary+ `proxy_capacity_exceeded` decode through the proxy-owned machine-code field,+ classify as non-retryable `LLM_API_CALL_FAILED`, and do not install fallback+ cooldown. An upstream `reason` string is never trusted as provenance. Real+ upstream 503, connect failure and 429 behavior remains unchanged.Desktop. The composer budget for `grok-subscription` is 16 MiB per image and16 MiB decoded in total, with no dimension bound. That is the shared rpc-proxydiff --git a/grok-llm-proxy/src/requestLimits.ts b/grok-llm-proxy/src/requestLimits.tsindex b579a3e0b..ecded6a9b 100644--- a/grok-llm-proxy/src/requestLimits.ts+++ b/grok-llm-proxy/src/requestLimits.ts@@ -51,6 +51,13 @@ export const IN_FLIGHT_BODY_BUDGET_BODIES = 3*/export const BODY_READ_DEADLINE_MS = 10_000+/**+ * Bounds the interval between a visual read timeout/client response close and+ * forced request destruction. The parser's callback—not this timer—releases the+ * visual slot and principal share.+ */+export const VISUAL_READ_CLOSE_GRACE_MS = 100+export const STREAM_LIMITS = {maxConcurrentStreams: 8,maxQueuedRequests: 16,@@ -94,9 +101,10 @@ export const VISUAL_STREAM_LIMITS = {* Fair share of the visual gate's entries — running plus queued — that one* platform principal (`sub` plus sorted `hostRefs`) may hold at once. Above the* share, further visual requests from that principal are refused 503- * `provider_unavailable` with log reason `visual_host_share`, so one principal+ * `visual_host_share`, so one principal* cannot fill a gate that every other host still needs. The gate widths above* are unchanged; the share only bounds how much of them one principal occupies.+ * Its refusal is a local capacity outcome, not an upstream outage.*/export const VISUAL_PER_HOST_MAX_ADMITTED = 2@@ -117,14 +125,18 @@ export type RequestLimitKind =| 'queue_wait'| 'ticket_life'+export type RequestLimitCode = 'provider_unavailable' | 'proxy_capacity_exceeded' | 'visual_gate'+export class RequestLimitError extends Error {- readonly code = 'provider_unavailable'+ readonly code: RequestLimitCodeconstructor(message: string,- readonly kind: RequestLimitKind+ readonly kind: RequestLimitKind,+ code: RequestLimitCode = 'provider_unavailable') {super(message)this.name = 'RequestLimitError'+ this.code = code}}@@ -178,7 +190,8 @@ export class StreamGate {constructor(private readonly maxConcurrent: number = STREAM_LIMITS.maxConcurrentStreams,private readonly maxQueued: number = STREAM_LIMITS.maxQueuedRequests,- private readonly maxQueueWaitMs: number = STREAM_LIMITS.maxQueueWaitMs+ private readonly maxQueueWaitMs: number = STREAM_LIMITS.maxQueueWaitMs,+ private readonly queueCapacityCode: RequestLimitCode = 'proxy_capacity_exceeded') {// A NaN or fractional size would let every caller through or queue// without bound, because the comparisons below would never hold.@@ -221,7 +234,7 @@ export class StreamGate {if (signal?.aborted) throw new RequestLimitError('stream request was aborted', 'aborted')if (this.running >= this.maxConcurrent) {if (this.queued >= this.maxQueued)- throw new RequestLimitError('stream queue is full', 'queue_full')+ throw new RequestLimitError('stream queue is full', 'queue_full', this.queueCapacityCode)this.queued += 1try {await new Promise<void>((resolve, reject) => {@@ -247,7 +260,8 @@ export class StreamGate {reject(new RequestLimitError('stream queue wait exceeded',- deadlineGoverns ? 'deadline' : 'queue_wait'+ deadlineGoverns ? 'deadline' : 'queue_wait',+ deadlineGoverns ? 'provider_unavailable' : this.queueCapacityCode))}@@ -289,7 +303,9 @@ export const streamGate = new StreamGate()export const visualStreamGate = new StreamGate(VISUAL_STREAM_LIMITS.maxConcurrentStreams,- VISUAL_STREAM_LIMITS.maxQueuedRequests+ VISUAL_STREAM_LIMITS.maxQueuedRequests,+ STREAM_LIMITS.maxQueueWaitMs,+ 'visual_gate')type BodyWaiter = { bytes: number; grant: (release: () => void) => void }@@ -348,7 +364,11 @@ export class BodyBudget {return this.take(bytes)}if (this.waiters.length >= this.maxQueued) {- throw new RequestLimitError('body admission queue is full', 'queue_full')+ throw new RequestLimitError(+ 'body admission queue is full',+ 'queue_full',+ 'proxy_capacity_exceeded'+ )}return new Promise<() => void>((resolve, reject) => {let deadline: ReturnType<typeof setTimeout> | undefineddiff --git a/grok-llm-proxy/src/server.ts b/grok-llm-proxy/src/server.tsindex ff60f4a51..e560f8161 100644--- a/grok-llm-proxy/src/server.ts+++ b/grok-llm-proxy/src/server.ts@@ -46,6 +46,7 @@ import {STREAM_LIMITS,TicketLifeError,VISUAL_PER_HOST_MAX_ADMITTED,+ VISUAL_READ_CLOSE_GRACE_MS,streamGate,visualStreamGate,} from './requestLimits.js'@@ -116,7 +117,9 @@ function isRecord(value: unknown): value is Record<string, unknown> {function reject(res: Response, status: number, code: string, type?: string): void {if (res.headersSent) returnlogger.warn(- type === undefined ? { event: 'grok_proxy_denied', code } : { event: 'grok_proxy_denied', code, type },+ type === undefined+ ? { event: 'grok_proxy_denied', code }+ : { event: 'grok_proxy_denied', code, type },'request denied')res.status(status).json({ error: code })@@ -270,7 +273,7 @@ function bodyAdmission(},'admission refused')- reject(res, 503, 'provider_unavailable')+ reject(res, 503, err.code)})}@@ -288,6 +291,8 @@ export type ProxyRuntimeDeps = {lookup?: OriginPolicyOptions['lookup']/** Test seam for the body-read deadline. Production uses `BODY_READ_DEADLINE_MS`. */bodyReadDeadlineMs?: number+ /** Test seam for the bounded visual-parser close backstop. */+ visualReadCloseGraceMs?: number/** Test seam: hang or observe a stream without contacting Grok. */streamCompletion?: typeof streamGrokCompletion}@@ -337,9 +342,16 @@ export function createProxyApps(// One budget for both apps: every body this process reads counts against it.const bodyBudget = new BodyBudget(IN_FLIGHT_BODY_BUDGET_BODIES * config.maxBodyBytes)const bodyReadDeadlineMs = deps.bodyReadDeadlineMs ?? BODY_READ_DEADLINE_MS+ const visualReadCloseGraceMs = deps.visualReadCloseGraceMs ?? VISUAL_READ_CLOSE_GRACE_MS+ for (const [name, value] of Object.entries({ bodyReadDeadlineMs, visualReadCloseGraceMs })) {+ if (!Number.isSafeInteger(value) || value <= 0 || value > 2_147_483_647) {+ throw new RangeError(`${name} must be a positive signed-32-bit integer`)+ }+ }// Per-principal occupancy of the visual gate (running plus queued entries).// Keyed on the verified platform principal: sub plus sorted hostRefs.const visualPrincipalAdmissions = new Map<string, number>()+ const visualRequestOwners = new WeakSet<GatedRequest>()const runtimeApp = express()// R9-M-B: the token is checked from the header before any budget is taken or@@ -384,12 +396,12 @@ export function createProxyApps(// neither header has no body, so only a declared length takes a visual slot.//// A granted slot is released exactly once. The handler hands it to the- // stream or releases it; every other exit (a parse error, a throw that- // reaches the error handler, a dropped client) ends with the response's- // `close` event, which releases whatever is still held. Like a budgeted- // body, a visual body must be read and parsed within `bodyReadDeadlineMs` of- // the grant, or it is answered 408 `request_timeout`, its slot released and- // its connection closed.+ // stream; parser errors and timeouts release it only after body-parser's+ // callback proves that work has unwound. A response close starts a bounded+ // request-destruction backstop rather than releasing live parser ownership.+ // Like a budgeted body, a visual body must be read and parsed within+ // `bodyReadDeadlineMs` of the grant, or it is answered 408 `request_timeout`+ // with `connection: close`.const selectTransportBudget = (req: GatedRequest, res: Response, next: NextFunction): void => {const declared = contentLengthBytes(req)if (declared !== null && declared > config.maxVisualBodyBytes) {@@ -423,7 +435,7 @@ export function createProxyApps(},'admission refused')- reject(res, 503, 'provider_unavailable')+ reject(res, 503, 'visual_host_share')return}visualPrincipalAdmissions.set(visualPrincipal, admittedByPrincipal + 1)@@ -471,7 +483,7 @@ export function createProxyApps(},'admission refused')- reject(res, 503, 'provider_unavailable')+ reject(res, 503, err.code)return}next(err)@@ -485,38 +497,55 @@ export function createProxyApps(req.grokStreamRelease = undefined}req.grokStreamRelease = releaseVisual- let expired = false+ let cancelled = false+ let parserSettled = false+ let closeGrace: ReturnType<typeof setTimeout> | undefined+ const cancelParser = (): void => {+ if (parserSettled || closeGrace !== undefined) return+ closeGrace = setTimeout(() => {+ if (!parserSettled) req.destroy()+ }, visualReadCloseGraceMs)+ }const readDeadline = setTimeout(() => {- expired = true- releaseVisual()+ cancelled = true+ cancelParser()if (res.headersSent) return// The rest of the body is never read, so the connection cannot be reused.res.setHeader('connection', 'close')reject(res, 408, 'request_timeout')}, bodyReadDeadlineMs)- res.once('close', () => {+ const onResponseClose = (): void => {+ cancelled = trueclearTimeout(readDeadline)- // The close handler releases only while the slot is still held here. A kept- // visual body transfers `req.grokStreamRelease` into the handler,- // which clears the field and releases in its outer `finally`. A- // demoted body releases through `releaseAdmission` before it joins- // the ordinary stream gate; this hook may still fire after demotion- // but finds the field empty and no-ops. Releasing here after hand-off- // would free the visual slot and the principal share while the- // handler still holds the parsed body during its (up to ~30 s)- // unwind, admitting another large body over the gate width.- if (req.grokStreamRelease) releaseVisual()- })+ cancelParser()+ }+ res.once('close', onResponseClose)visualJson(req, res, err => {+ parserSettled = trueclearTimeout(readDeadline)- // The 408 already answered this request; the parser's late error- // (the body aborted by the closed connection) has no one to reach.- if (expired) return+ if (closeGrace !== undefined) clearTimeout(closeGrace)+ res.off('close', onResponseClose)+ if (cancelled || err) {+ req.body = undefined+ if (err !== null && typeof err === 'object' && 'body' in err) {+ delete (err as { body?: unknown }).body+ }+ }+ // Capacity remains charged until body-parser has actually unwound. On+ // success, `req.grokStreamRelease` transfers ownership to the handler,+ // which releases it in its outer `finally`; refusals release here.+ if (cancelled || err) releaseVisual()+ if (cancelled) returnif (err) {- releaseVisual()next(err)return}+ res.once('close', () => {+ // A synchronous handler throw can happen before its async owner+ // installs cancellation and a finally block. Once that owner has+ // taken over, response close must not release its live work.+ if (!visualRequestOwners.has(req)) releaseVisual()+ })next()})})()@@ -623,6 +652,7 @@ export function createProxyApps(// to req 'close' aborts the Grok hop on every call (3–12ms canceled).// Abort only when the client drops the response before we finish writing.abortWhenClientDisconnects(req, res, abort)+ visualRequestOwners.add(gated)// One `grok_proxy_attempt_finished` line per attempt. Identifiers and// counts only: never the body, ticket, frames, tool names or arguments.const attempt = {@@ -782,9 +812,9 @@ export function createProxyApps(stopHeartbeat?.()const mapped = mapError(err)metrics.observeAttempt('error', 'completion_stream')- // A request limit answers provider_unavailable on the wire; its own- // label keeps it apart from real upstream outages in the metric. A- // ticket that died in the queue is counted as provider_unavailable.+ // Queue capacity has a terminal local wire code; the request_limit+ // metric also includes abort/deadline kinds that keep their existing+ // wire semantics. Ticket expiry remains provider_unavailable.metrics.observeAttemptFailure(err instanceof RequestLimitError && !(err instanceof TicketLifeError)? 'request_limit'@@ -792,8 +822,7 @@ export function createProxyApps()if (err instanceof UpstreamTimeoutError) metrics.observeUpstreamTimeout(err.kind)const deliveredAs = res.headersSent ? 'sse_error' : 'http_status'- const causeCode =- err instanceof ControlApiClientError ? err.causeCode : fetchCauseCode(err)+ const causeCode = err instanceof ControlApiClientError ? err.causeCode : fetchCauseCode(err)logger.warn({event: 'grok_proxy_attempt_finished',@@ -829,7 +858,9 @@ export function createProxyApps(: undefinedconst rejectedBy = typeof upstreamStatus === 'number' ? { upstreamStatus } : {}if (deliveredAs === 'sse_error') {- res.write(`data: ${JSON.stringify({ type: 'error', code: mapped.code, ...rejectedBy })}\n\n`)+ res.write(+ `data: ${JSON.stringify({ type: 'error', code: mapped.code, ...rejectedBy })}\n\n`+ )res.end()return}@@ -847,6 +878,10 @@ export function createProxyApps(} finally {stopHeartbeat?.()release?.()+ // Defence in depth for an early throw after this async owner took over+ // but before the visual release was moved into `release`.+ gated.grokStreamRelease?.()+ gated.grokStreamRelease = undefined// A demoted body's reservation, when the stream ended before the body// was written upstream. Budgeted bodies are also released on close.gated.grokBodyRelease?.()@@ -1022,6 +1057,9 @@ const ATTEMPT_ERROR_STATUS: Record<string, number> = {// would spend the same budget again, so it is a gateway timeout, not 503.stream_duration_exceeded: 504,connection_unavailable: 503,+ visual_host_share: 503,+ visual_gate: 503,+ proxy_capacity_exceeded: 503,provider_unavailable: 503,sse_buffer_exceeded: 503,invalid_receipt: 503,@@ -1062,7 +1100,6 @@ function isMappedError(err: unknown): err is MappedError {function mapError(err: unknown): { status: number; code: string } {if (!isMappedError(err)) return { status: 503, code: 'provider_unavailable' }if (err instanceof OriginDeniedError) return { status: 403, code: 'origin_denied' }- if (err instanceof RequestLimitError) return { status: 503, code: 'provider_unavailable' }return { status: attemptErrorStatus(err.code), code: err.code }}diff --git a/grok-llm-proxy/test/bodyAdmission.test.ts b/grok-llm-proxy/test/bodyAdmission.test.tsindex e8f1deca2..1dad2a548 100644--- a/grok-llm-proxy/test/bodyAdmission.test.ts+++ b/grok-llm-proxy/test/bodyAdmission.test.ts@@ -354,7 +354,7 @@ describe('grok-llm-proxy body admission (#731 R3-2)', () => {}}, 30_000)- it('T-R3-2c-grok answers provider_unavailable once the admission queue is full', async () => {+ it('T-R3-2c-grok answers proxy_capacity_exceeded once the admission queue is full', async () => {// A small body limit keeps this cheap; the budget scales with it.const maxBodyBytes = 16 * 1024const proxy = await heldProxy(maxBodyBytes)@@ -374,16 +374,16 @@ describe('grok-llm-proxy body admission (#731 R3-2)', () => {const overflow = await post(proxy.port, payload('overflow'))// Witnesses: the refusal was answered, and the admission logged why.expect(overflow.status).toBe(503)- expect(JSON.parse(overflow.body)).toEqual({ error: 'provider_unavailable' })+ expect(JSON.parse(overflow.body)).toEqual({ error: 'proxy_capacity_exceeded' })const logged = warn.mock.calls.map(call => call[0] as unknown as Record<string, unknown>)expect(logged.filter(entry => entry?.event === 'grok_proxy_denied')).toEqual([- { event: 'grok_proxy_denied', code: 'provider_unavailable' },+ { event: 'grok_proxy_denied', code: 'proxy_capacity_exceeded' },])expect(logged.filter(entry => entry?.event === 'grok_proxy_admission_refused')).toEqual([{event: 'grok_proxy_admission_refused',reason: 'body_budget',- code: 'provider_unavailable',+ code: 'proxy_capacity_exceeded',kind: 'queue_full',detail: expect.any(String),},diff --git a/grok-llm-proxy/test/contractFreeze.test.ts b/grok-llm-proxy/test/contractFreeze.test.tsindex 3e2d1d647..a2669b896 100644--- a/grok-llm-proxy/test/contractFreeze.test.ts+++ b/grok-llm-proxy/test/contractFreeze.test.ts@@ -1,7 +1,8 @@import { describe, expect, it } from 'vitest'-import { readdirSync, readFileSync } from 'node:fs'+import { readFileSync, readdirSync } from 'node:fs'import { dirname, join } from 'node:path'import { fileURLToPath } from 'node:url'+import ts from 'typescript'import {CATALOG_ORIGIN,COMPLETIONS_ORIGIN,@@ -127,13 +128,17 @@ function emittedTransportCodes(): { codes: Set<string>; sites: number; constructfor (const name of readdirSync(srcDir).filter(file => file.endsWith('.ts'))) {const source = readFileSync(join(srcDir, name), 'utf8')constructions +=- source.split('new GrokTransportError(').length - 1 +- source.split('new UpstreamTimeoutError(').length - 1+ source.split('new GrokTransportError(').length -+ 1 ++ source.split('new UpstreamTimeoutError(').length -+ 1for (const match of source.matchAll(TRANSPORT_ERROR_SITE)) {const literal = /^'([a-z][a-z0-9_]+)'$/.exec(match[1]!)// A computed code cannot be checked against the taxonomy, so it fails here.- expect(literal, `${name}: transport error code must be a string literal, got ${match[1]}`)- .not.toBeNull()+ expect(+ literal,+ `${name}: transport error code must be a string literal, got ${match[1]}`+ ).not.toBeNull()codes.add(literal![1]!)sites += 1}@@ -147,11 +152,71 @@ function emittedTransportCodes(): { codes: Set<string>; sites: number; constructconst REJECT_SITE = /reject\(res,\s*[^,()]+,\s*([^,()]+?)\s*(?:,\s*type\s*)?\)/g// Refusal codes are wire strings; `Unauthorized` is the one spelled in capitals.const REJECT_CODE_LITERAL = /^'([A-Za-z][A-Za-z0-9_]+)'$/-// The only computed code a refusal may carry, as its exact source text:+// The computed refusal forms are closed RequestLimitError codes and mapped errors.// `mapped.code` is a transport or control-api code passed through mapError,// whose transport half the scanner above already gates. Any other computed// form cannot be checked against the taxonomy, so it fails here.-const COMPUTED_REJECT_CODES = ['mapped.code'] as const+const COMPUTED_REJECT_CODES = ['err.code', 'mapped.code'] as const++/** Computed admission codes are allowed only through this closed, readonly type. */+function requestLimitCodes(+ sourceText = readFileSync(join(srcDir, 'requestLimits.ts'), 'utf8')+): Set<string> {+ const source = ts.createSourceFile('requestLimits.ts', sourceText, ts.ScriptTarget.Latest, true)+ const alias = source.statements.find(+ (node): node is ts.TypeAliasDeclaration =>+ ts.isTypeAliasDeclaration(node) && node.name.text === 'RequestLimitCode'+ )+ if (!alias || !ts.isUnionTypeNode(alias.type)) {+ throw new Error('RequestLimitCode must remain a closed literal union')+ }+ const codes = new Set(+ alias.type.types.map(node => {+ if (!ts.isLiteralTypeNode(node) || !ts.isStringLiteral(node.literal)) {+ throw new Error('RequestLimitCode must not accept computed or open string types')+ }+ return node.literal.text+ })+ )+ const owner = source.statements.find(+ (node): node is ts.ClassDeclaration =>+ ts.isClassDeclaration(node) && node.name?.text === 'RequestLimitError'+ )+ const property = owner?.members.find(+ (node): node is ts.PropertyDeclaration =>+ ts.isPropertyDeclaration(node) && node.name.getText(source) === 'code'+ )+ expect(property?.type?.getText(source)).toBe('RequestLimitCode')+ expect(property?.modifiers?.some(node => node.kind === ts.SyntaxKind.ReadonlyKeyword)).toBe(true)+ const constructor = owner?.members.find(ts.isConstructorDeclaration)+ const input = constructor?.parameters.find(node => node.name.getText(source) === 'code')+ expect(input?.type?.getText(source)).toBe('RequestLimitCode')+ expect(input?.initializer?.getText(source)).toBe("'provider_unavailable'")+ const assignments: string[] = []+ const collect = (node: ts.Node): void => {+ if (+ ts.isBinaryExpression(node) &&+ node.operatorToken.kind === ts.SyntaxKind.EqualsToken &&+ node.left.getText(source) === 'this.code'+ ) {+ assignments.push(node.right.getText(source))+ }+ ts.forEachChild(node, collect)+ }+ if (owner) collect(owner)+ expect(assignments).toEqual(['code'])+ return codes+}++function rejectCodeLiteral(code: string, name: string): RegExpExecArray | null {+ const literal = REJECT_CODE_LITERAL.exec(code)+ const allowed = (COMPUTED_REJECT_CODES as readonly string[]).includes(code)+ expect(+ literal !== null || allowed,+ `${name}: reject code must be a string literal or one of ${COMPUTED_REJECT_CODES.join(', ')}, got ${code}`+ ).toBe(true)+ return literal+}function emittedRejectCodes(): {codes: Set<string>@@ -159,7 +224,7 @@ function emittedRejectCodes(): {sites: numberoccurrences: number} {- const codes = new Set<string>()+ const codes = requestLimitCodes()const computed = new Set<string>()let sites = 0let occurrences = 0@@ -168,12 +233,7 @@ function emittedRejectCodes(): {occurrences += source.split('reject(res,').length - 1for (const match of source.matchAll(REJECT_SITE)) {const code = match[1]!- const literal = REJECT_CODE_LITERAL.exec(code)- const allowed = (COMPUTED_REJECT_CODES as readonly string[]).includes(code)- expect(- literal !== null || allowed,- `${name}: reject code must be a string literal or one of ${COMPUTED_REJECT_CODES.join(', ')}, got ${code}`- ).toBe(true)+ const literal = rejectCodeLiteral(code, name)if (literal) codes.add(literal[1]!)else computed.add(code)sites += 1@@ -183,6 +243,42 @@ function emittedRejectCodes(): {}describe('grok-subscription contract freeze', () => {+ it('rejects open or mutable admission code producers while reading the closed production union', () => {+ const sourceText = readFileSync(join(srcDir, 'requestLimits.ts'), 'utf8')+ const tree = ts.createSourceFile('requestLimits.ts', sourceText, ts.ScriptTarget.Latest, true)+ const alias = tree.statements.find(+ (node): node is ts.TypeAliasDeclaration =>+ ts.isTypeAliasDeclaration(node) && node.name.text === 'RequestLimitCode'+ )!+ const withType = (type: string) =>+ sourceText.slice(0, alias.type.getStart(tree)) + type + sourceText.slice(alias.type.end)+ expect([...requestLimitCodes(sourceText)]).toEqual(+ expect.arrayContaining(['provider_unavailable', 'proxy_capacity_exceeded', 'visual_gate'])+ )+ expect(() => requestLimitCodes(withType('string'))).toThrow('closed literal union')+ expect(() => requestLimitCodes(withType("'provider_unavailable' | string"))).toThrow(+ 'open string types'+ )+ expect(() =>+ requestLimitCodes(+ sourceText.replace('readonly code: RequestLimitCode', 'code: RequestLimitCode')+ )+ ).toThrow()+ expect(() =>+ requestLimitCodes(sourceText.replace('this.code = code', "this.code = 'arbitrary_code'"))+ ).toThrow()+ })++ it.each(['caller.code', "err.code || 'provider_unavailable'", 'String(err.code)'])(+ 'rejects unbounded computed refusal expression %s',+ expression => {+ expect(rejectCodeLiteral("'request_timeout'", 'fixture')).not.toBeNull()+ expect(rejectCodeLiteral('err.code', 'fixture')).toBeNull()+ expect(rejectCodeLiteral('mapped.code', 'fixture')).toBeNull()+ expect(() => rejectCodeLiteral(expression, 'fixture')).toThrow('reject code must be')+ }+ )+it('imports the origin-policy constants the proxy actually enforces', () => {expect(GROK_COMPLETIONS_ORIGIN).toBe(COMPLETIONS_ORIGIN)expect(GROK_CATALOG_ORIGIN).toBe(CATALOG_ORIGIN)diff --git a/grok-llm-proxy/test/server.security.test.ts b/grok-llm-proxy/test/server.security.test.tsindex 0800c266f..0c9be5a13 100644--- a/grok-llm-proxy/test/server.security.test.ts+++ b/grok-llm-proxy/test/server.security.test.ts@@ -201,7 +201,9 @@ describe('grok-llm-proxy security surface', () => {})it('reserves the visual transport budget for an authenticated ~30 MiB V2 body', async () => {- const { runtimeApp, adminApp } = createProxyApps(config({ maxBodyBytes: DEFAULT_MAX_BODY_BYTES }))+ const { runtimeApp, adminApp } = createProxyApps(+ config({ maxBodyBytes: DEFAULT_MAX_BODY_BYTES })+ )// Deliberately invalid ticket: this test checks parser admission and the// unchanged ticket gate, without redeeming or contacting any model. 30 MiB// is past the Codex 24 MiB visual ceiling and inside the Grok 35 MiB one.@@ -287,7 +289,9 @@ describe('grok-llm-proxy security surface', () => {const acquire = vi.spyOn(visualStreamGate, 'acquire')const visual = await send('x'.repeat(8_000_000), {schemaVersion: 'grok-completion-request.v2',- messages: [{ role: 'user', contentParts: [{ type: 'image', data: 'A'.repeat(1024 * 1024) }] }],+ messages: [+ { role: 'user', contentParts: [{ type: 'image', data: 'A'.repeat(1024 * 1024) }] },+ ],})expect(visual.status).toBe(400)expect(visual.body.error).toBe('invalid_request')@@ -340,7 +344,10 @@ describe('grok-llm-proxy security surface', () => {const { runtimeApp, probeApp } = createProxyApps(config({ maxBodyBytes: 65_536 }), {streamCompletion: async () => {streamCalls += 1- throw new GrokTransportError('payload_too_large', 'request exceeds maxVisualRequestBodyBytes')+ throw new GrokTransportError(+ 'payload_too_large',+ 'request exceeds maxVisualRequestBodyBytes'+ )},})const res = await request(runtimeApp)@@ -352,7 +359,9 @@ describe('grok-llm-proxy security surface', () => {expect(res.status).toBe(413)expect(res.body).toEqual({ error: 'payload_too_large' })const metricsText = (await request(probeApp).get('/metrics')).text- expect(metricsText).toMatch(/^grok_proxy_attempt_failures_total\{code="payload_too_large"\} 1$/m)+ expect(metricsText).toMatch(+ /^grok_proxy_attempt_failures_total\{code="payload_too_large"\} 1$/m+ )})it('does not let a V2 declaration raise the non-image budget to 35 MiB', async () => {@@ -854,10 +863,12 @@ describe('grok-llm-proxy startup config', () => {loadConfig({ ...base, GROK_LLM_PROXY_MAX_BODY_BYTES: String(LIMITS.maxRequestBodyBytes) })).toThrow(/GROK_LLM_PROXY_MAX_BODY_BYTES must be at least/)// Witness: the floor itself and any larger value load.- expect(loadConfig({ ...base, GROK_LLM_PROXY_MAX_BODY_BYTES: String(floor) }).maxBodyBytes).toBe(floor)- expect(loadConfig({ ...base, GROK_LLM_PROXY_MAX_BODY_BYTES: String(floor + 1) }).maxBodyBytes).toBe(- floor + 1+ expect(loadConfig({ ...base, GROK_LLM_PROXY_MAX_BODY_BYTES: String(floor) }).maxBodyBytes).toBe(+ floor)+ expect(+ loadConfig({ ...base, GROK_LLM_PROXY_MAX_BODY_BYTES: String(floor + 1) }).maxBodyBytes+ ).toBe(floor + 1)})it('defaults the visual body limit to the contract visual ceiling, with no allowance on top', () => {@@ -1266,7 +1277,10 @@ describe('grok-llm-proxy attempt telemetry', () => {// Denies the redeem after `delayMs`, which is longer than the heartbeat// interval the caller configures.- function slowDenyingClient(code: string, delayMs: number): {+ function slowDenyingClient(+ code: string,+ delayMs: number+ ): {client: ControlApiClientredeemCalls: () => number} {@@ -1661,6 +1675,33 @@ describe('grok-llm-proxy attempt telemetry', () => {expect('err' in (lines[0] ?? {})).toBe(false)})+ it.each(['visual_host_share', 'visual_gate', 'proxy_capacity_exceeded'])(+ 'normalizes real transport upstream 503 claiming %s to provider_unavailable',+ async code => {+ const fetchFn = vi+ .fn<typeof fetch>()+ .mockResolvedValue(+ Response.json({ error: code, reason: 'local capacity claim' }, { status: 503 })+ )+ const { res, receipts, lines, metricsText } = await run({+ providerAttemptId: 'att-spoofed-local',+ fetchFn,+ })+ expect(fetchFn).toHaveBeenCalledTimes(1)+ expect(res.status).toBe(503)+ expect(res.body).toEqual({ error: 'provider_unavailable' })+ expect(receipts).toEqual([expect.objectContaining({ outcome: 'error' })])+ expect(lines).toHaveLength(1)+ expect(lines[0]).toMatchObject({+ code: 'provider_unavailable',+ deliveredAs: 'http_status',+ httpStatus: 503,+ })+ expect(failureCount(metricsText, 'provider_unavailable')).toBe(1)+ expect(failureCount(metricsText, 'request_limit')).toBe(0)+ }+ )+it('(g1-1a) answers 429 rate_limited and forwards a valid upstream Retry-After', async () => {const { res, receipts, lines, metricsText } = await run({providerAttemptId: 'att-rate-http',@@ -1732,7 +1773,9 @@ describe('grok-llm-proxy attempt telemetry', () => {// Witness: a keepalive went out first, so only the frame can carry it.expect(keepaliveCount(res.text)).toBeGreaterThanOrEqual(1)expect(- res.text.endsWith('data: {"type":"error","code":"upstream_rejected","upstreamStatus":402}\n\n')+ res.text.endsWith(+ 'data: {"type":"error","code":"upstream_rejected","upstreamStatus":402}\n\n'+ )).toBe(true)expect(lines).toHaveLength(1)expect(lines[0]).toMatchObject({@@ -2143,20 +2186,24 @@ describe('grok-llm-proxy attempt telemetry', () => {it('(rl1) logs the request-limit reason and labels its failure metric request_limit', async () => {const acquire = vi.spyOn(streamGate, 'acquire')- .mockRejectedValueOnce(new RequestLimitError('stream queue is full', 'queue_full'))+ .mockRejectedValueOnce(+ new RequestLimitError('stream queue is full', 'queue_full', 'proxy_capacity_exceeded')+ )try {- const { res, receipts, lines, metricsText } = await run({ providerAttemptId: 'att-queue-full' })+ const { res, receipts, lines, metricsText } = await run({+ providerAttemptId: 'att-queue-full',+ })// Witness: the refusal came from the stream gate this test replaced.expect(acquire).toHaveBeenCalledTimes(1)expect(res.status).toBe(503)- expect(res.body).toEqual({ error: 'provider_unavailable' })+ expect(res.body).toEqual({ error: 'proxy_capacity_exceeded' })// The gate refused before the redeem, so there is no receipt to finalize.expect(receipts).toEqual([])expect(lines).toHaveLength(1)expect(lines[0]).toMatchObject({providerAttemptId: 'att-queue-full',outcome: 'failed',- code: 'provider_unavailable',+ code: 'proxy_capacity_exceeded',reason: 'stream queue is full',deliveredAs: 'http_status',httpStatus: 503,@@ -2220,9 +2267,7 @@ describe('grok-llm-proxy attempt telemetry', () => {})expect(res.status).toBe(200)expect(keepaliveCount(res.text)).toBeGreaterThanOrEqual(1)- expect(res.text.endsWith('data: {"type":"error","code":"provider_unavailable"}\n\n')).toBe(- true- )+ expect(res.text.endsWith('data: {"type":"error","code":"provider_unavailable"}\n\n')).toBe(true)expect(lines).toHaveLength(1)expect(lines[0]).toMatchObject({outcome: 'failed',@@ -2513,7 +2558,10 @@ describe('grok-llm-proxy ticket-aware stream-gate wait (#739 D1-bis)', () => {}, 30_000)/** A V2 envelope past `maxBodyBytes`, so it takes the visual gate. */- function visualEnvelope(providerAttemptId: string, ticketLifeMs: number): Record<string, unknown> {+ function visualEnvelope(+ providerAttemptId: string,+ ticketLifeMs: number+ ): Record<string, unknown> {const raw: Record<string, unknown> = {schemaVersion: 'grok-completion-request.v2',requestId: `req-${providerAttemptId}`,@@ -2565,7 +2613,10 @@ describe('grok-llm-proxy ticket-aware stream-gate wait (#739 D1-bis)', () => {.set('Authorization', `Bearer ${platformToken()}`).send(body).then(res => res)- await until(() => visualStreamGate.snapshot().queued === 1, 'the body queueing at the visual gate')+ await until(+ () => visualStreamGate.snapshot().queued === 1,+ 'the body queueing at the visual gate'+ )await vi.advanceTimersByTimeAsync(ticketLifeMs)// Witness: the body was still waiting for the slot at the ticket's expiry.@@ -2920,7 +2971,7 @@ describe('grok-llm-proxy body budget release on upstream acceptance (#739 D2)',const client = open(proxy.port, payload)await until(() => client.ended(), 'the refusal')expect(client.status()).toBe(503)- expect(JSON.parse(client.received())).toEqual({ error: 'provider_unavailable' })+ expect(JSON.parse(client.received())).toEqual({ error: 'proxy_capacity_exceeded' })await until(() => proxy.closes() === 1, "the response's close event")expect(client.errors()).toEqual([])expect(proxy.reservations()).toEqual([Buffer.byteLength(payload)])@@ -3086,7 +3137,10 @@ describe('grok-llm-proxy graceful drain on shutdown (#739 D6)', () => {req.end(payload)let closing: Promise<void> | undefinedtry {- await until(() => received.includes('data: {"type":"text","text":"t0"}'), 'the first SSE frame')+ await until(+ () => received.includes('data: {"type":"text","text":"t0"}'),+ 'the first SSE frame'+ )closing = apps.close().then(() => {order.push('closed')})diff --git a/grok-llm-proxy/test/streamGate.handoff.test.ts b/grok-llm-proxy/test/streamGate.handoff.test.tsindex 194c8e2fb..2a9dec879 100644--- a/grok-llm-proxy/test/streamGate.handoff.test.ts+++ b/grok-llm-proxy/test/streamGate.handoff.test.ts@@ -357,12 +357,12 @@ describe('grok visual stream-gate handoff', () => {const warn = vi.spyOn(logger, 'warn')const overflow = await postCompletion(port, 'grok-completion-request.v2', largeContent)expect(overflow.status).toBe(503)- expect(await overflow.json()).toEqual({ error: 'provider_unavailable' })+ expect(await overflow.json()).toEqual({ error: 'visual_gate' })expect(warn).toHaveBeenCalledWith(expect.objectContaining({event: 'grok_proxy_admission_refused',reason: 'visual_gate',- code: 'provider_unavailable',+ code: 'visual_gate',}),'admission refused')@@ -470,7 +470,7 @@ describe('grok visual stream-gate handoff', () => {)const third = await postCompletion(port, 'grok-completion-request.v2', largeContent)expect(third.status).toBe(503)- expect(await third.json()).toEqual({ error: 'provider_unavailable' })+ expect(await third.json()).toEqual({ error: 'visual_host_share' })expect(warn).toHaveBeenCalledWith({event: 'grok_proxy_admission_refused',@@ -605,7 +605,7 @@ describe('grok visual stream-gate handoff', () => {const third = await postBody(port, platformToken(), invalidTicketBody(maxBodyBytes))expect(third.status).toBe(503)- expect(await third.json()).toEqual({ error: 'provider_unavailable' })+ expect(await third.json()).toEqual({ error: 'visual_host_share' })// The refusal names the principal it applied to, so an operator can tell// which host was throttled.expect(warn).toHaveBeenCalledWith(@@ -684,7 +684,7 @@ describe('grok visual stream-gate handoff', () => {for (let i = 0; i < targetCount; i += 1) {const res = await postBody(port, platformToken(), invalidTicketBody(maxBodyBytes))expect(res.status).toBe(503)- expect(await res.json()).toEqual({ error: 'provider_unavailable' })+ expect(await res.json()).toEqual({ error: 'visual_gate' })}expect(acquire).toHaveBeenCalledTimes(1 + VISUAL_STREAM_LIMITS.maxQueuedRequests + targetCount@@ -759,7 +759,7 @@ describe('grok visual stream-gate handoff', () => {).toBe(1)const third = await refusedRequestexpect(third.status).toBe(503)- expect(await third.json()).toEqual({ error: 'provider_unavailable' })+ expect(await third.json()).toEqual({ error: 'visual_host_share' })expect(warn).toHaveBeenCalledWith({event: 'grok_proxy_admission_refused',diff --git a/grok-llm-proxy/test/visualBodyReadOwnership.test.ts b/grok-llm-proxy/test/visualBodyReadOwnership.test.tsnew file mode 100644index 000000000..e32e5da43--- /dev/null+++ b/grok-llm-proxy/test/visualBodyReadOwnership.test.ts@@ -0,0 +1,532 @@+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'+import express, { type NextFunction, type Request, type Response } from 'express'+import jwt from 'jsonwebtoken'+import { generateKeyPairSync } from 'node:crypto'+import { createServer } from 'node:http'+import { connect as connectTcp } from 'node:net'+import type { GrokLlmProxyConfig } from '../src/config.js'+import {+ VISUAL_PER_HOST_MAX_ADMITTED,+ VISUAL_STREAM_LIMITS,+ visualStreamGate,+} from '../src/requestLimits.js'+import { createProxyApps } from '../src/server.js'++type ObservedRead = {+ req: Request+ res: Response+ nativeSettled: boolean+ returned: boolean+ error: unknown+ finishes: number+ endCalls: number+}++const parserBarrier = vi.hoisted(() => {+ let release!: () => void+ let completion: Promise<void>+ const reads: ObservedRead[] = []+ const reset = () => {+ reads.splice(0)+ completion = new Promise<void>(resolve => {+ release = resolve+ })+ }+ reset()+ return {+ reads,+ reset,+ get completion() {+ return completion+ },+ get settledCount() {+ return reads.filter(read => read.nativeSettled).length+ },+ release() {+ release()+ },+ }+})++vi.mock('express', async () => {+ const actual = await vi.importActual<typeof import('express')>('express')+ const actualDefault = (actual as unknown as { default: typeof express }).default+ const mockedDefault = Object.assign(+ (...args: Parameters<typeof express>) => actualDefault(...args),+ {+ json: (options?: Parameters<typeof actual.json>[0]) => {+ const parser = actualDefault.json(options)+ return (req: Request, res: Response, next: NextFunction) => {+ // Observe the native visual reader. Ordinary JSON parsing stays native+ // too, but must not count as starting the next visual reader.+ const read: ObservedRead | undefined =+ options?.limit === 8192+ ? {+ req,+ res,+ nativeSettled: false,+ returned: false,+ error: undefined,+ finishes: 0,+ endCalls: 0,+ }+ : undefined+ if (read) {+ parserBarrier.reads.push(read)+ res.once('finish', () => {+ read.finishes += 1+ })+ if (req.headers['x-visual-never-finish'] === '1') {+ // Fault injection: no finish/close is produced by writing 408.+ // The production close backstop must independently stop raw-body.+ res.end = (() => {+ read.endCalls += 1+ return res+ }) as Response['end']+ }+ }+ parser(req, res, (err: unknown) => {+ if (read) {+ read.nativeSettled = true+ read.error = err+ }+ const finish = () => {+ next(err)+ if (read) read.returned = true+ }+ if (req.headers['x-visual-parser-hold'] === '1') {+ void parserBarrier.completion.then(finish)+ } else {+ finish()+ }+ })+ }+ },+ }+ )+ return { ...actual, default: mockedDefault, json: mockedDefault.json }+})++const COMPLETIONS_PATH = '/internal/runtime/v1/grok/completions'+const { privateKey, publicKey } = generateKeyPairSync('rsa', {+ modulusLength: 2048,+ publicKeyEncoding: { type: 'spki', format: 'pem' },+ privateKeyEncoding: { type: 'pkcs8', format: 'pem' },+})++function config(): GrokLlmProxyConfig {+ return {+ runtimePort: 0,+ adminPort: 0,+ probePort: 0,+ maxBodyBytes: 4096,+ maxVisualBodyBytes: 8192,+ maxStreamDurationMs: 1_800_000,+ maxDeadlineMs: 1_800_000,+ upstreamIdleTimeoutMs: 600_000,+ heartbeatIntervalMs: 15_000,+ jwtIssuer: 'control-api',+ jwtPublicKey: publicKey,+ executionEnabled: true,+ controlApiBaseUrl: '',+ controlApiServiceName: 'grok-llm-proxy',+ controlApiServiceToken: '',+ }+}++function platformToken(): string {+ return jwt.sign(+ {+ sub: 'default/research-host',+ hostRefs: ['research-host'],+ workflowControlScopes: ['llm:grok:execute'],+ scope: 'workflow:approval:request',+ },+ privateKey,+ { algorithm: 'RS256', issuer: 'control-api', audience: 'workflow-approvals', expiresIn: 60 }+ )+}++function platformTokenForOtherPrincipal(): string {+ return jwt.sign(+ {+ sub: 'default/other-host',+ hostRefs: ['research-host'],+ workflowControlScopes: ['llm:grok:execute'],+ scope: 'workflow:approval:request',+ },+ privateKey,+ { algorithm: 'RS256', issuer: 'control-api', audience: 'workflow-approvals', expiresIn: 60 }+ )+}++async function waitFor(predicate: () => boolean, label: string): Promise<void> {+ const started = Date.now()+ while (!predicate()) {+ if (Date.now() - started > 5_000) throw new Error(label)+ await new Promise(resolve => setTimeout(resolve, 5))+ }+}++const sockets = new Set<ReturnType<typeof connectTcp>>()++function stalledVisualBody(+ port: number,+ declared: number,+ holdParser = false,+ extraHeaders = '',+ prefix = '{"executionTicket":'+): { socket: ReturnType<typeof connectTcp>; response: Promise<number> } {+ let owner: ReturnType<typeof connectTcp> | undefined+ const response = new Promise<number>((resolve, reject) => {+ owner = connectTcp(port, '127.0.0.1', () => {+ owner?.write(+ `POST ${COMPLETIONS_PATH} HTTP/1.1\r\n` ++ `Host: 127.0.0.1:${port}\r\n` ++ `Authorization: Bearer ${platformToken()}\r\n` ++ `Content-Type: application/json\r\n` ++ (holdParser ? 'X-Visual-Parser-Hold: 1\r\n' : '') ++ extraHeaders ++ `Content-Length: ${declared}\r\n` ++ `\r\n` ++ prefix+ )+ })+ sockets.add(owner)+ owner.once('close', () => sockets.delete(owner!))+ owner.setTimeout(5_000, () => {+ owner?.destroy()+ reject(new Error('the stalled visual body produced no response'))+ })+ const chunks: Buffer[] = []+ owner.on('data', data => {+ chunks.push(data)+ const match = /^HTTP\/1\.1 (\d+)/.exec(Buffer.concat(chunks).toString('utf8'))+ if (match) resolve(Number(match[1]))+ })+ owner.on('error', err => {+ if ((err as NodeJS.ErrnoException).code !== 'ECONNRESET') reject(err)+ })+ })+ return { socket: owner!, response }+}++function invalidTicketBody(): string {+ const content = 'x'.repeat(5000)+ return JSON.stringify({+ executionTicket: 'invalid-ticket',+ requestHash: 'a'.repeat(64),+ request: {+ schemaVersion: 'grok-completion-request.v2',+ requestId: 'req-visual-ownership',+ idempotencyKey: 'idem-visual-ownership',+ provider: 'grok-subscription',+ model: 'grok-4.6',+ messages: [{ role: 'user', content }],+ },+ })+}++function postVisualRaw(+ port: number,+ body: string,+ holdParser = false+): { socket: ReturnType<typeof connectTcp>; response: Promise<number> } {+ let owner: ReturnType<typeof connectTcp> | undefined+ const response = new Promise<number>((resolve, reject) => {+ owner = connectTcp(port, '127.0.0.1', () => {+ owner?.write(+ `POST ${COMPLETIONS_PATH} HTTP/1.1\r\n` ++ `Host: 127.0.0.1:${port}\r\n` ++ `Authorization: Bearer ${platformToken()}\r\n` ++ `Content-Type: application/json\r\n` ++ (holdParser ? 'X-Visual-Parser-Hold: 1\r\n' : '') ++ `Content-Length: ${Buffer.byteLength(body)}\r\n` ++ `\r\n` ++ body+ )+ })+ sockets.add(owner)+ owner.once('close', () => sockets.delete(owner!))+ owner.setTimeout(5_000, () => {+ owner?.destroy()+ reject(new Error('the visual request produced no response'))+ })+ const chunks: Buffer[] = []+ owner.on('data', data => {+ chunks.push(data)+ const match = /^HTTP\/1\.1 (\d+)/.exec(Buffer.concat(chunks).toString('utf8'))+ if (match) resolve(Number(match[1]))+ })+ owner.on('error', err => {+ if ((err as NodeJS.ErrnoException).code !== 'ECONNRESET') reject(err)+ })+ })+ return { socket: owner!, response }+}++async function postVisualStatus(port: number, body: string): Promise<number> {+ const request = postVisualRaw(port, body)+ const status = await request.response+ request.socket.destroy()+ return status+}++describe('grok visual body read ownership', () => {+ const servers: Array<ReturnType<typeof createProxyApps>> = []+ const listeners: Array<ReturnType<typeof createServer>> = []++ beforeEach(() => parserBarrier.reset())++ afterEach(async () => {+ parserBarrier.release()+ vi.restoreAllMocks()+ for (const socket of sockets) socket.destroy()+ await Promise.all(+ listeners+ .splice(0)+ .map(+ listener =>+ new Promise<void>((resolve, reject) =>+ listener.close(err => (err ? reject(err) : resolve()))+ )+ )+ )+ await Promise.all(servers.splice(0).map(server => server.close()))+ await waitFor(+ () => visualStreamGate.snapshot().running === 0 && visualStreamGate.snapshot().queued === 0,+ 'the visual gate did not drain after the ownership test'+ )+ })++ it('keeps the visual slot and principal share until the timed-out parser terminates', async () => {+ const apps = createProxyApps(config(), {+ bodyReadDeadlineMs: 200,+ visualReadCloseGraceMs: 10,+ })+ servers.push(apps)+ const listener = createServer(apps.runtimeApp).listen(0)+ listeners.push(listener)+ const address = listener.address()+ if (!address || typeof address === 'string') throw new Error('listener has no port')++ const readers = Array.from({ length: VISUAL_PER_HOST_MAX_ADMITTED }, () =>+ stalledVisualBody(address.port, 5000, true)+ )+ await waitFor(+ () =>+ visualStreamGate.snapshot().running + visualStreamGate.snapshot().queued ===+ VISUAL_PER_HOST_MAX_ADMITTED,+ 'the stalled visual bodies did not fill the principal share'+ )+ const statuses = await Promise.all(+ readers.slice(0, VISUAL_STREAM_LIMITS.maxConcurrentStreams).map(reader => reader.response)+ )+ expect(statuses).toEqual(+ Array.from({ length: VISUAL_STREAM_LIMITS.maxConcurrentStreams }, () => 408)+ )+ expect(visualStreamGate.snapshot().running + visualStreamGate.snapshot().queued).toBe(+ VISUAL_PER_HOST_MAX_ADMITTED+ )++ const shareRefusal = await postVisualStatus(address.port, invalidTicketBody())+ expect(shareRefusal).toBe(503)+ // A different principal can queue, but the reader must not start until the+ // existing parser callbacks finish. This observes reader entry, not merely+ // the absence of an HTTP response from the next request.+ const nextSocket = connectTcp(address.port, '127.0.0.1')+ sockets.add(nextSocket)+ nextSocket.once('close', () => sockets.delete(nextSocket))+ const nextStatus = new Promise<number>((resolve, reject) => {+ nextSocket.once('connect', () => {+ const body = invalidTicketBody()+ nextSocket.write(+ `POST ${COMPLETIONS_PATH} HTTP/1.1\r\n` ++ `Host: 127.0.0.1:${address.port}\r\n` ++ `Authorization: Bearer ${platformTokenForOtherPrincipal()}\r\n` ++ `Content-Type: application/json\r\n` ++ `Content-Length: ${Buffer.byteLength(body)}\r\n\r\n${body}`+ )+ })+ nextSocket.on('data', data => {+ const match = /^HTTP\/1\.1 (\d+)/.exec(data.toString('utf8'))+ if (match) resolve(Number(match[1]))+ })+ nextSocket.once('error', reject)+ })+ await waitFor(+ () =>+ visualStreamGate.snapshot().queued ===+ VISUAL_PER_HOST_MAX_ADMITTED - VISUAL_STREAM_LIMITS.maxConcurrentStreams + 1,+ 'the next principal did not queue behind the retained parsers'+ )+ expect(parserBarrier.reads).toHaveLength(VISUAL_STREAM_LIMITS.maxConcurrentStreams)++ for (const reader of readers) {+ reader.response.catch(() => undefined)+ reader.socket.destroy()+ }+ await waitFor(+ () => parserBarrier.settledCount === VISUAL_STREAM_LIMITS.maxConcurrentStreams,+ 'the held parsers did not terminate'+ )+ expect(visualStreamGate.snapshot().running).toBe(VISUAL_STREAM_LIMITS.maxConcurrentStreams)+ expect(visualStreamGate.snapshot().queued).toBeGreaterThanOrEqual(1)+ expect(parserBarrier.reads).toHaveLength(VISUAL_STREAM_LIMITS.maxConcurrentStreams)+ parserBarrier.release()+ expect(await nextStatus).toBe(403)+ nextSocket.destroy()+ await waitFor(+ () => visualStreamGate.snapshot().running === 0 && visualStreamGate.snapshot().queued === 0,+ 'the timed-out visual reader did not terminate and release its admission'+ )+ expect(+ parserBarrier.reads+ .slice(0, VISUAL_STREAM_LIMITS.maxConcurrentStreams)+ .every(read => read.returned && read.req.body === undefined)+ ).toBe(true)+ })++ it('uses the close backstop when the visual client disappears before the deadline', async () => {+ const apps = createProxyApps(config(), {+ bodyReadDeadlineMs: 5_000,+ visualReadCloseGraceMs: 10,+ })+ servers.push(apps)+ const listener = createServer(apps.runtimeApp).listen(0)+ listeners.push(listener)+ const address = listener.address()+ if (!address || typeof address === 'string') throw new Error('listener has no port')++ const stalled = stalledVisualBody(address.port, 5000)+ await waitFor(+ () => visualStreamGate.snapshot().running === 1,+ 'the visual body did not take the slot'+ )+ stalled.socket.destroy()+ stalled.response.catch(() => undefined)+ await waitFor(+ () => visualStreamGate.snapshot().running === 0 && visualStreamGate.snapshot().queued === 0,+ 'a disconnected visual reader retained its admission after the close backstop'+ )+ })++ it('releases the visual admission after a parser error', async () => {+ const apps = createProxyApps(config())+ servers.push(apps)+ const listener = createServer(apps.runtimeApp).listen(0)+ listeners.push(listener)+ const address = listener.address()+ if (!address || typeof address === 'string') throw new Error('listener has no port')++ const status = await postVisualStatus(address.port, `{"executionTicket":${' '.repeat(5000)}`)+ expect(status).toBe(400)+ await waitFor(+ () => visualStreamGate.snapshot().running === 0 && visualStreamGate.snapshot().queued === 0,+ 'a malformed visual body did not release its admission'+ )+ })++ it('transfers a successfully parsed visual body to the route and releases it after the response', async () => {+ const apps = createProxyApps(config())+ servers.push(apps)+ const listener = createServer(apps.runtimeApp).listen(0)+ listeners.push(listener)+ const address = listener.address()+ if (!address || typeof address === 'string') throw new Error('listener has no port')++ const status = await postVisualStatus(address.port, invalidTicketBody())+ expect(status).toBe(403)+ await waitFor(+ () => visualStreamGate.snapshot().running === 0 && visualStreamGate.snapshot().queued === 0,+ 'a parsed visual body did not release its admission after the route response'+ )+ })+ it('stops the native reader independently when a non-reading peer never finishes the response', async () => {+ const apps = createProxyApps(config(), { bodyReadDeadlineMs: 40, visualReadCloseGraceMs: 15 })+ servers.push(apps)+ const listener = createServer(apps.runtimeApp).listen(0)+ listeners.push(listener)+ const address = listener.address()+ if (!address || typeof address === 'string') throw new Error('listener has no port')++ const stalled = stalledVisualBody(address.port, 5000, false, 'X-Visual-Never-Finish: 1\r\n')+ stalled.response.catch(() => undefined)+ stalled.socket.pause()+ await waitFor(() => parserBarrier.reads.length === 1, 'the native visual reader did not start')+ const read = parserBarrier.reads[0]!+ await waitFor(() => read.returned, 'the independent close backstop did not terminate raw-body')+ expect(read.endCalls).toBe(1)+ expect(read.finishes).toBe(0)+ expect(read.req.destroyed).toBe(true)+ expect(read.req.body).toBeUndefined()+ expect(visualStreamGate.snapshot()).toEqual({ running: 0, queued: 0 })+ expect(await postVisualStatus(address.port, invalidTicketBody())).toBe(403)+ })++ it('clears a late successful native parse after its read deadline without dispatching it', async () => {+ const apps = createProxyApps(config(), { bodyReadDeadlineMs: 40, visualReadCloseGraceMs: 1000 })+ servers.push(apps)+ const listener = createServer(apps.runtimeApp).listen(0)+ listeners.push(listener)+ const address = listener.address()+ if (!address || typeof address === 'string') throw new Error('listener has no port')++ const body = invalidTicketBody()+ const split = 128+ const stalled = stalledVisualBody(+ address.port,+ Buffer.byteLength(body),+ false,+ 'X-Visual-Never-Finish: 1\r\n',+ body.slice(0, split)+ )+ stalled.response.catch(() => undefined)+ await waitFor(+ () => parserBarrier.reads[0]?.endCalls === 1,+ 'the visual reader never reached its deadline'+ )+ const read = parserBarrier.reads[0]!+ expect(read.nativeSettled).toBe(false)+ expect(visualStreamGate.snapshot().running).toBe(1)+ stalled.socket.write(body.slice(split))+ await waitFor(() => read.returned, 'the late native parser callback did not run')+ expect(read.error).toBeUndefined()+ expect(read.req.body).toBeUndefined()+ expect(visualStreamGate.snapshot()).toEqual({ running: 0, queued: 0 })+ stalled.socket.destroy()+ expect(await postVisualStatus(address.port, invalidTicketBody())).toBe(403)+ })++ it('clears the retained parsed body after response close before its callback returns', async () => {+ const apps = createProxyApps(config(), { bodyReadDeadlineMs: 5000, visualReadCloseGraceMs: 15 })+ servers.push(apps)+ const listener = createServer(apps.runtimeApp).listen(0)+ listeners.push(listener)+ const address = listener.address()+ if (!address || typeof address === 'string') throw new Error('listener has no port')++ const request = postVisualRaw(address.port, invalidTicketBody(), true)+ request.response.catch(() => undefined)+ await waitFor(() => parserBarrier.settledCount === 1, 'the complete body was never parsed')+ const read = parserBarrier.reads[0]!+ expect(read.req.body).toBeDefined()+ expect(read.returned).toBe(false)+ request.socket.destroy()+ await waitFor(() => read.res.destroyed, 'the response did not close')+ expect(visualStreamGate.snapshot().running).toBe(1)+ parserBarrier.release()+ await waitFor(() => read.returned, 'the disconnected parser callback did not run')+ expect(read.req.body).toBeUndefined()+ expect(visualStreamGate.snapshot()).toEqual({ running: 0, queued: 0 })+ expect(await postVisualStatus(address.port, invalidTicketBody())).toBe(403)+ })++ it.each([0, -1, 1.5, Number.NaN, Number.POSITIVE_INFINITY, 2147483648])(+ 'rejects invalid native reader clocks (%s)',+ value => {+ expect(() => createProxyApps(config(), { bodyReadDeadlineMs: value })).toThrow(RangeError)+ expect(() => createProxyApps(config(), { visualReadCloseGraceMs: value })).toThrow(RangeError)+ }+ )+})diff --git a/mcp-host/package-lock.json b/mcp-host/package-lock.jsonindex 7a702113a..3d9a15dd4 100644--- a/mcp-host/package-lock.json+++ b/mcp-host/package-lock.json@@ -1,12 +1,12 @@{"name": "clerum-mcp-host",- "version": "0.1.486",+ "version": "0.1.487","lockfileVersion": 3,"requires": true,"packages": {"": {"name": "clerum-mcp-host",- "version": "0.1.486",+ "version": "0.1.487","license": "SEE LICENSE IN LICENSE","dependencies": {"@anthropic-ai/sdk": "^0.32.0",diff --git a/mcp-host/package.json b/mcp-host/package.jsonindex 668c9959f..edc364693 100644--- a/mcp-host/package.json+++ b/mcp-host/package.json@@ -1,6 +1,6 @@{"name": "clerum-mcp-host",- "version": "0.1.486",+ "version": "0.1.487","description": "MCP Host - reads Host CRD and provides LLM access via OpenAI/Claude","main": "dist/main.js","engines": {diff --git a/mcp-host/src/llm/__tests__/codexLlmProxyClient.test.ts b/mcp-host/src/llm/__tests__/codexLlmProxyClient.test.tsindex 507a86543..a7bb1faab 100644--- a/mcp-host/src/llm/__tests__/codexLlmProxyClient.test.ts+++ b/mcp-host/src/llm/__tests__/codexLlmProxyClient.test.ts@@ -10,6 +10,7 @@ import { LlmErrorCode } from '../../core/errors'import {CodexLlmProxyClient,CodexProxyError,+ codexProxyErrorMessage,resolveCodexProxyRuntimeUrl,} from '../codexLlmProxyClient'import { CodexSubscriptionProvider } from '../codexSubscription'@@ -435,6 +436,109 @@ describe('CodexLlmProxyClient', () => {expect(classifyFailoverClass(classified.code, classified.retryable)).toBeNull()})+ it.each(['visual_host_share', 'visual_gate', 'proxy_capacity_exceeded'] as const)(+ 'decodes the local %s admission code without failover and keeps the client usable',+ async code => {+ const input = {+ executionTicket: 'ticket-123456',+ requestHash: 'a'.repeat(64),+ request: {},+ }+ const fetchFn = vi+ .fn()+ .mockResolvedValueOnce(+ Response.json({ error: code, reason: 'untrusted upstream prose' }, { status: 503 })+ )+ .mockResolvedValueOnce(new Response(sse([{ type: 'done', outcome: 'success' }])))+ const client = new CodexLlmProxyClient({+ runtimeUrl: 'http://proxy/internal/runtime/v1/codex/completions',+ readPlatformJwt: () => 'platform-jwt',+ fetchFn,+ })+ const err = await client.stream(input).then(+ () => undefined,+ (e: unknown) => e+ )+ expect(fetchFn).toHaveBeenCalledTimes(1)+ expect(err).toBeInstanceOf(CodexProxyError)+ if (!(err instanceof CodexProxyError)) throw new Error('expected CodexProxyError')+ expect(err).toMatchObject({+ code,+ message: codexProxyErrorMessage(code, 503),+ dispatched: true,+ })+ expect(err.message).not.toContain('untrusted upstream prose')++ const provider = new CodexSubscriptionProvider('gpt-5.3-codex', {} as never)+ const classified = provider.classifyError(err)+ expect(classified.code).toBe(LlmErrorCode.ApiCallFailed)+ expect(classified.retryable).toBe(false)+ expect(classified.providerCode).toBe(code)+ expect(classifyFailoverClass(classified.code, classified.retryable)).toBeNull()++ const next = await client.stream(input)+ expect(next.outcome).toBe('success')+ expect(fetchFn).toHaveBeenCalledTimes(2)++ const outage = provider.classifyError(+ new CodexProxyError('provider_unavailable', 'real upstream outage')+ )+ expect(outage.code).toBe(LlmErrorCode.ModelOverloaded)+ expect(outage.retryable).toBe(true)+ expect(classifyFailoverClass(outage.code, outage.retryable)).toBe('provider_unavailable')+ }+ )++ it.each([+ [+ 'unknown_machine_code',+ JSON.stringify({ error: 'future_proxy_error' }),+ LlmErrorCode.ApiCallFailed,+ false,+ 'future_proxy_error',+ ],+ [+ 'reason_only',+ JSON.stringify({ reason: 'visual_gate' }),+ LlmErrorCode.ModelOverloaded,+ true,+ 'provider_unavailable',+ ],+ ['malformed_json', '{', LlmErrorCode.ModelOverloaded, true, 'provider_unavailable'],+ ] as const)(+ 'keeps wire %s separate from trusted local refusal provenance',+ async (_label, body, code, retryable, providerCode) => {+ const fetchFn = vi+ .fn()+ .mockResolvedValue(+ new Response(body, { status: 503, headers: { 'content-type': 'application/json' } })+ )+ const proxy = new CodexLlmProxyClient({+ runtimeUrl: 'http://proxy/internal/runtime/v1/codex/completions',+ readPlatformJwt: () => 'platform-jwt',+ fetchFn,+ })+ const error = await proxy+ .stream({+ executionTicket: 'ticket-123456',+ requestHash: 'a'.repeat(64),+ request: {},+ })+ .then(+ () => undefined,+ (caught: unknown) => caught+ )+ expect(error).toBeInstanceOf(CodexProxyError)+ expect(fetchFn).toHaveBeenCalledTimes(1)+ const provider = new CodexSubscriptionProvider('gpt-5.3-codex', {} as never)+ const classified = provider.classifyError(error)+ expect(classified).toMatchObject({ code, retryable, providerCode })+ expect(classifyFailoverClass(classified.code, classified.retryable)).toBe(+ retryable ? 'provider_unavailable' : null+ )+ }+ )+// T-TE-1 (D4) — the proxy passes control-api's redeem refusal through as// `403 { error: 'ticket_expired' }` when the ticket died while the request// was queued. Nothing reached the provider, so a re-authorized retry is thediff --git a/mcp-host/src/llm/__tests__/codexSubscription.test.ts b/mcp-host/src/llm/__tests__/codexSubscription.test.tsindex 1ab985615..0bfb929ea 100644--- a/mcp-host/src/llm/__tests__/codexSubscription.test.ts+++ b/mcp-host/src/llm/__tests__/codexSubscription.test.ts@@ -69,6 +69,37 @@ function deps(overrides?: {}describe('CodexSubscriptionProvider', () => {+ it.each(['visual_host_share', 'visual_gate', 'proxy_capacity_exceeded'] as const)(+ 'keeps the provider usable after a local %s admission refusal',+ async code => {+ const wired = deps({+ stream: vi+ .fn()+ .mockRejectedValueOnce(+ new CodexProxyError(code, 'proxy admission is full', { dispatched: true })+ )+ .mockResolvedValueOnce({ text: 'next text turn', toolCalls: [], outcome: 'success' }),+ })+ const provider = new CodexSubscriptionProvider('gpt-5.3-codex', wired as never)++ await expect(+ provider.completeSingleTurn([{ role: 'user', content: 'image' }])+ ).rejects.toMatchObject({ code })+ const result = await provider.completeSingleTurn([{ role: 'user', content: 'next text' }])+ expect(result.content).toBe('next text turn')+ expect(wired.authorize).toHaveBeenCalledTimes(2)+ expect(wired.stream).toHaveBeenCalledTimes(2)++ const classified = provider.classifyError(+ new CodexProxyError(code, 'proxy admission is full', { dispatched: true })+ )+ expect(classified.code).toBe(LlmErrorCode.ApiCallFailed)+ expect(classified.retryable).toBe(false)+ expect(classified.providerCode).toBe(code)+ expect(classifyFailoverClass(classified.code, classified.retryable)).toBeNull()+ }+ )+it.each(['unknown', 'canceled', 'error'])('never returns executable calls from a %s batch',async outcome => {diff --git a/mcp-host/src/llm/__tests__/grokLlmProxyClient.test.ts b/mcp-host/src/llm/__tests__/grokLlmProxyClient.test.tsindex fa3484335..c1cef922e 100644--- a/mcp-host/src/llm/__tests__/grokLlmProxyClient.test.ts+++ b/mcp-host/src/llm/__tests__/grokLlmProxyClient.test.ts@@ -545,6 +545,90 @@ describe('GrokLlmProxyClient', () => {expect(classifyFailoverClass(classified.code, classified.retryable)).toBeNull()})+ it.each(['visual_host_share', 'visual_gate', 'proxy_capacity_exceeded'] as const)(+ 'decodes the local %s admission code without failover and keeps the client usable',+ async code => {+ const fetchFn = vi+ .fn()+ .mockResolvedValueOnce(+ Response.json({ error: code, reason: 'untrusted upstream prose' }, { status: 503 })+ )+ .mockResolvedValueOnce(new Response(sse([{ type: 'done', outcome: 'success' }])))+ const grok = client(fetchFn)+ const err = await grok.stream(STREAM_INPUT).then(+ () => undefined,+ (e: unknown) => e+ )+ expect(fetchFn).toHaveBeenCalledTimes(1)+ expect(err).toBeInstanceOf(GrokProxyError)+ if (!(err instanceof GrokProxyError)) throw new Error('expected GrokProxyError')+ expect(err).toMatchObject({+ code,+ message: grokProxyErrorMessage(code, 503),+ dispatched: true,+ })+ expect(err.message).not.toContain('untrusted upstream prose')++ const provider = new GrokSubscriptionProvider('grok-4.6', {} as never)+ const classified = provider.classifyError(err)+ expect(classified.code).toBe(LlmErrorCode.ApiCallFailed)+ expect(classified.retryable).toBe(false)+ expect(classified.providerCode).toBe(code)+ expect(classifyFailoverClass(classified.code, classified.retryable)).toBeNull()++ const next = await grok.stream(STREAM_INPUT)+ expect(next.outcome).toBe('success')+ expect(fetchFn).toHaveBeenCalledTimes(2)++ const outage = provider.classifyError(+ new GrokProxyError('provider_unavailable', 'real upstream outage')+ )+ expect(outage.code).toBe(LlmErrorCode.ModelOverloaded)+ expect(outage.retryable).toBe(true)+ expect(classifyFailoverClass(outage.code, outage.retryable)).toBe('provider_unavailable')+ }+ )++ it.each([+ [+ 'unknown_machine_code',+ JSON.stringify({ error: 'future_proxy_error' }),+ LlmErrorCode.ApiCallFailed,+ false,+ 'future_proxy_error',+ ],+ [+ 'reason_only',+ JSON.stringify({ reason: 'visual_gate' }),+ LlmErrorCode.ModelOverloaded,+ true,+ 'provider_unavailable',+ ],+ ['malformed_json', '{', LlmErrorCode.ModelOverloaded, true, 'provider_unavailable'],+ ] as const)(+ 'keeps wire %s separate from trusted local refusal provenance',+ async (_label, body, code, retryable, providerCode) => {+ const fetchFn = vi+ .fn()+ .mockResolvedValue(+ new Response(body, { status: 503, headers: { 'content-type': 'application/json' } })+ )+ const proxy = client(fetchFn)+ const error = await proxy.stream(STREAM_INPUT).then(+ () => undefined,+ (caught: unknown) => caught+ )+ expect(error).toBeInstanceOf(GrokProxyError)+ expect(fetchFn).toHaveBeenCalledTimes(1)+ const provider = new GrokSubscriptionProvider('grok-4.6', {} as never)+ const classified = provider.classifyError(error)+ expect(classified).toMatchObject({ code, retryable, providerCode })+ expect(classifyFailoverClass(classified.code, classified.retryable)).toBe(+ retryable ? 'provider_unavailable' : null+ )+ }+ )+// T-TE-1 (D4) — the proxy passes control-api's redeem refusal through as// `403 { error: 'ticket_expired' }` when the ticket died while the request// was queued. Nothing reached the provider, so a re-authorized retry is thediff --git a/mcp-host/src/llm/__tests__/grokSubscription.test.ts b/mcp-host/src/llm/__tests__/grokSubscription.test.tsindex ff97676d7..14d9581ec 100644--- a/mcp-host/src/llm/__tests__/grokSubscription.test.ts+++ b/mcp-host/src/llm/__tests__/grokSubscription.test.ts@@ -47,6 +47,8 @@ function deps(overrides?: {return {authorizer: { authorize },proxy: { stream },+ authorize,+ stream,attemptContext: vi.fn(() => ({policyRevision: 1,policyHash: 'b'.repeat(64),@@ -56,6 +58,37 @@ function deps(overrides?: {}describe('GrokSubscriptionProvider', () => {+ it.each(['visual_host_share', 'visual_gate', 'proxy_capacity_exceeded'] as const)(+ 'keeps the provider usable after a local %s admission refusal',+ async code => {+ const wired = deps({+ stream: vi+ .fn()+ .mockRejectedValueOnce(+ new GrokProxyError(code, 'proxy admission is full', { dispatched: true })+ )+ .mockResolvedValueOnce({ text: 'next text turn', toolCalls: [], outcome: 'success' }),+ })+ const provider = new GrokSubscriptionProvider('grok-4.6', wired as never)++ await expect(+ provider.completeSingleTurn([{ role: 'user', content: 'image' }])+ ).rejects.toMatchObject({ code })+ const result = await provider.completeSingleTurn([{ role: 'user', content: 'next text' }])+ expect(result.content).toBe('next text turn')+ expect(wired.authorize).toHaveBeenCalledTimes(2)+ expect(wired.stream).toHaveBeenCalledTimes(2)++ const classified = provider.classifyError(+ new GrokProxyError(code, 'proxy admission is full', { dispatched: true })+ )+ expect(classified.code).toBe(LlmErrorCode.ApiCallFailed)+ expect(classified.retryable).toBe(false)+ expect(classified.providerCode).toBe(code)+ expect(classifyFailoverClass(classified.code, classified.retryable)).toBeNull()+ }+ )+it.each(['unknown', 'canceled', 'error'])('never returns executable calls from a %s batch',async outcome => {diff --git a/mcp-host/src/llm/codexLlmProxyClient.ts b/mcp-host/src/llm/codexLlmProxyClient.tsindex a4b34519d..ecbc61f0d 100644--- a/mcp-host/src/llm/codexLlmProxyClient.ts+++ b/mcp-host/src/llm/codexLlmProxyClient.ts@@ -9,6 +9,26 @@ import { upstreamRejectedStatus } from './upstreamRejected'export const CODEX_PROXY_COMPLETIONS_PATH = '/internal/runtime/v1/codex/completions'+/**+ * Operator-facing text for a proxy denial. The three local-capacity codes say+ * what the caller can do; they are not upstream outage diagnostics.+ */+export function codexProxyErrorMessage(code: string, status?: number): string {+ if (+ code === 'visual_host_share' ||+ code === 'visual_gate' ||+ code === 'proxy_capacity_exceeded'+ ) {+ return 'Codex proxy admission is full. Wait for an active request to finish or send fewer concurrent requests before retrying; this is not a Codex outage.'+ }+ if (code === 'payload_too_large') {+ return 'Codex request is too large; use fewer or smaller images, or reduce context'+ }+ return status === undefined+ ? `proxy stream failed with ${code}`+ : `proxy stream failed with ${status} (${code})`+}+export type CodexProxyErrorOptions = {/*** Whether a request had already left this process when the error was@@ -166,7 +186,9 @@ export class CodexLlmProxyClient {return this.streamOnce(input, false)}const payload = (await response.json().catch(() => ({}))) as Record<string, unknown>- // A 429 is a rate limit, not a provider outage (G1-6, G1-11, #720): the+ // Only the proxy-owned machine `error` field identifies the outcome; an+ // upstream `reason` string is never promoted to trusted provenance. A+ // 429 is a rate limit, not a provider outage (G1-6, G1-11, #720): the// proxy answers `rate_limited` itself, and a 429 with no JSON code or a// reason phrase is read the same way, so only a machine code a 429// carries replaces `rate_limited`.@@ -178,16 +200,10 @@ export class CodexLlmProxyClient {: typeof payload.error === 'string'? payload.error: 'provider_unavailable'- throw new CodexProxyError(- code,- code === 'payload_too_large'- ? 'Codex request is too large; use fewer or smaller images, or reduce context'- : `proxy stream failed with ${response.status} (${code})`,- {- retryAfterMs: response.status === 429 ? retryAfterMs(response) : undefined,- upstreamStatus: upstreamRejectedStatus(code, payload.upstreamStatus),- }- )+ throw new CodexProxyError(code, codexProxyErrorMessage(code, response.status), {+ retryAfterMs: response.status === 429 ? retryAfterMs(response) : undefined,+ upstreamStatus: upstreamRejectedStatus(code, payload.upstreamStatus),+ })}if (!response.body) {throw new CodexProxyError('provider_unavailable', 'proxy stream had no body')@@ -221,7 +237,7 @@ async function readProxySse(body: ReadableStream<Uint8Array>): Promise<CodexProxif (!line) continueconst frame = JSON.parse(line.slice(6)) as CodexProxyFrameif (frame.type === 'error') {- throw new CodexProxyError(frame.code, `proxy stream failed with ${frame.code}`, {+ throw new CodexProxyError(frame.code, codexProxyErrorMessage(frame.code), {upstreamStatus: upstreamRejectedStatus(frame.code, frame.upstreamStatus),})}diff --git a/mcp-host/src/llm/failover/__tests__/engine.test.ts b/mcp-host/src/llm/failover/__tests__/engine.test.tsindex 193ff062f..d373ca322 100644--- a/mcp-host/src/llm/failover/__tests__/engine.test.ts+++ b/mcp-host/src/llm/failover/__tests__/engine.test.ts@@ -102,6 +102,21 @@ describe('FailoverEngine', () => {expect(metricInc).not.toHaveBeenCalled()})+ it('keeps the primary eligible after a local admission failure', async () => {+ const e = engine()+ const local = llmError(LlmErrorCode.ApiCallFailed, false)+ const firstBuild = vi.fn(() => () => Promise.reject(local))+ await expect(e.run(PRIMARY, firstBuild, classify)).rejects.toBe(local)+ expect(firstBuild).toHaveBeenCalledTimes(1)+ expect(onSwitch).not.toHaveBeenCalled()+ expect(metricInc).not.toHaveBeenCalled()++ const nextBuild = vi.fn(target => () => Promise.resolve(target.kind))+ await expect(e.run(PRIMARY, nextBuild, classify)).resolves.toBe('primary')+ expect(nextBuild).toHaveBeenCalledTimes(1)+ expect(e.servedBy()).toEqual({ ...PRIMARY, fallback: false })+ })+it('respects a restricted triggerOn: an out-of-set class propagates', async () => {const e = engine(policy({ triggerOn: ['auth'] }))const err = llmError(LlmErrorCode.RateLimited, true) // rate_limited not in triggerOndiff --git a/mcp-host/src/llm/failover/__tests__/subscriptionAdmission.integration.test.ts b/mcp-host/src/llm/failover/__tests__/subscriptionAdmission.integration.test.tsnew file mode 100644index 000000000..ce8fd6183--- /dev/null+++ b/mcp-host/src/llm/failover/__tests__/subscriptionAdmission.integration.test.ts@@ -0,0 +1,358 @@+import { afterEach, describe, expect, it, vi } from 'vitest'+import jwt from 'jsonwebtoken'+import { generateKeyPairSync, randomUUID } from 'node:crypto'+import { type RequestListener, createServer } from 'node:http'+import { type Socket, connect as connectTcp } from 'node:net'+import { LlmErrorCode } from '../../../core/errors'+import { CodexLlmProxyClient } from '../../codexLlmProxyClient'+import { CodexSubscriptionProvider } from '../../codexSubscription'+import { GrokLlmProxyClient } from '../../grokLlmProxyClient'+import { GrokSubscriptionProvider } from '../../grokSubscription'+import { type ClassifiedLike, FailoverEngine } from '../engine'+import type { FailoverTarget, ModelPair } from '../types'++type Gate = {+ acquire(signal?: AbortSignal, deadlineAt?: number): Promise<() => void>+ snapshot(): { running: number; queued: number }+}+type Budget = { readonly inFlightBytes: number; readonly queued: number }+type ProxyLimitModule = {+ streamGate: Gate+ visualStreamGate: Gate+ STREAM_LIMITS: { maxConcurrentStreams: number; maxQueuedRequests: number }+ VISUAL_STREAM_LIMITS: { maxConcurrentStreams: number; maxQueuedRequests: number }+ VISUAL_PER_HOST_MAX_ADMITTED: number+ BodyBudget: {+ prototype: {+ acquire(bytes: number, signal?: AbortSignal, deadlineAt?: number): Promise<() => void>+ }+ }+}+type ProxyModule = {+ createProxyApps(+ config: Record<string, unknown>,+ deps: Record<string, unknown>+ ): { runtimeApp: RequestListener; close(): Promise<void> }+}+type StreamInput = {+ executionTicket: string+ requestHash: string+ redeem(input: unknown): Promise<unknown>+ onFrame(frame: { type: 'text'; text: string }): Promise<unknown>+}+type LocalProducer =+ | 'visual_host_share'+ | 'visual_gate'+ | 'stream_queue_full'+ | 'stream_queue_wait'+ | 'body_queue_full'+ | 'demotion_queue_full'+const LOCAL_PRODUCERS: LocalProducer[] = [+ 'visual_host_share',+ 'visual_gate',+ 'stream_queue_full',+ 'stream_queue_wait',+ 'body_queue_full',+ 'demotion_queue_full',+]+const { privateKey, publicKey } = generateKeyPairSync('rsa', {+ modulusLength: 2048,+ publicKeyEncoding: { type: 'spki', format: 'pem' },+ privateKeyEncoding: { type: 'pkcs8', format: 'pem' },+})+async function until(predicate: () => boolean, label: string): Promise<void> {+ const deadline = Date.now() + 5000+ while (!predicate()) {+ if (Date.now() >= deadline) throw new Error(label)+ await new Promise(resolve => setTimeout(resolve, 5))+ }+}+const cleanup: Array<() => Promise<void>> = []+afterEach(async () => {+ for (const close of cleanup.splice(0).reverse()) await close()+ vi.restoreAllMocks()+})+async function fixture(variant: 'grok' | 'codex') {+ // Runtime imports preserve Host's production compiler boundary. This test+ // exercises actual proxy HTTP producers, Host decoding and failover state.+ const base = `../../../../../${variant}-llm-proxy/src/`+ const proxyModule = await vi.importActual<ProxyModule>(`${base}server.ts`)+ const limits = await vi.importActual<ProxyLimitModule>(`${base}requestLimits.ts`)+ const model = variant === 'grok' ? 'grok-4.6' : 'gpt-5.3-codex'+ const provider = `${variant}-subscription`+ const platform = jwt.sign(+ {+ sub: 'default/admission-host',+ hostRefs: ['admission-host'],+ workflowControlScopes: [`llm:${variant}:execute`],+ scope: 'workflow:approval:request',+ },+ privateKey,+ { algorithm: 'RS256', issuer: 'control-api', audience: 'workflow-approvals', expiresIn: 60 }+ )+ const redeem = vi.fn(async () => undefined)+ // Refusals must never reach this transport seam. A later accepted route uses+ // it as a positive route/Host witness, not as real upstream compatibility.+ const stream = vi.fn(async (input: StreamInput) => {+ await input.redeem({ executionTicket: input.executionTicket, requestHash: input.requestHash })+ await input.onFrame({ type: 'text', text: 'primary route recovered' })+ return { outcome: 'success' as const }+ })+ const apps = proxyModule.createProxyApps(+ {+ runtimePort: 0,+ adminPort: 0,+ probePort: 0,+ maxBodyBytes: 4096,+ maxVisualBodyBytes: 8192,+ maxStreamDurationMs: 1000,+ maxDeadlineMs: 1000,+ upstreamIdleTimeoutMs: 1000,+ heartbeatIntervalMs: 15000,+ jwtIssuer: 'control-api',+ jwtPublicKey: publicKey,+ executionEnabled: true,+ controlApiBaseUrl: '',+ controlApiServiceName: `${variant}-llm-proxy`,+ controlApiServiceToken: '',+ },+ { controlApiClient: { redeem }, streamCompletion: stream, bodyReadDeadlineMs: 5000 }+ )+ const listener = createServer(apps.runtimeApp).listen(0, '127.0.0.1')+ await new Promise<void>(resolve => listener.once('listening', resolve))+ const address = listener.address()+ if (!address || typeof address === 'string') throw new Error('the proxy has no local listener')+ const port = address.port+ const path = `/internal/runtime/v1/${variant}/completions`+ const url = `http://127.0.0.1:${port}${path}`+ const sockets: Socket[] = []+ const held: Array<() => void> = []+ const queued: Array<Promise<void>> = []+ let restoreQueueClock: (() => void) | undefined+ let padded = false+ const hostFetch = vi.fn<typeof fetch>().mockImplementation((input, init) =>+ fetch(input, {+ ...init,+ // A padded caller can have large wire bytes and a small logical V1 body.+ ...(padded ? { body: `${String(init?.body)}${' '.repeat(5000)}` } : {}),+ signal: init?.signal ?? AbortSignal.timeout(5000),+ })+ )+ const clientOptions = { runtimeUrl: url, readPlatformJwt: () => platform, fetchFn: hostFetch }+ const client =+ variant === 'grok'+ ? new GrokLlmProxyClient(clientOptions)+ : new CodexLlmProxyClient(clientOptions)+ const authorize = vi.fn(async (input: { requestHash: string }) => {+ const id = randomUUID()+ return {+ providerAttemptId: id,+ requestHash: input.requestHash,+ executionTicket: jwt.sign(+ {+ jti: id,+ typ: `${variant}-execution-ticket`,+ hostRef: 'admission-host',+ model,+ requestHash: input.requestHash,+ providerAttemptId: id,+ },+ privateKey,+ {+ algorithm: 'RS256',+ issuer: 'control-api',+ audience: `${variant}-llm-proxy`,+ expiresIn: 60,+ }+ ),+ expiresAt: new Date(Date.now() + 60000).toISOString(),+ }+ })+ const deps = {+ authorizer: { authorize },+ proxy: client,+ attemptContext: () => ({+ policyRevision: 1,+ policyHash: 'b'.repeat(64),+ hostRef: 'admission-host',+ }),+ }+ const primaryProvider =+ variant === 'grok'+ ? new GrokSubscriptionProvider(model, deps as never)+ : new CodexSubscriptionProvider(model, deps as never)+ const primary: ModelPair = { provider, model }+ const onSwitch = vi.fn(),+ metricInc = vi.fn()+ const engine = new FailoverEngine(+ {+ cooldownSeconds: 300,+ triggerOn: ['provider_unavailable', 'rate_limited'],+ fallbacks: [{ provider: 'openai', model: 'gpt-5.4' }],+ },+ { onSwitch, metricInc }+ )+ const fallback = vi.fn(async () => ({ content: 'fallback' }))+ const built: FailoverTarget[] = []+ let classified: ClassifiedLike | undefined+ const run = (content: string) =>+ engine.run<{ content: string }>(+ primary,+ target => {+ built.push(target)+ return target.kind === 'primary'+ ? () => primaryProvider.completeSingleTurn([{ role: 'user', content }])+ : fallback+ },+ err => {+ classified = primaryProvider.classifyError(err)+ return classified+ }+ )+ const stopCapacity = async () => {+ padded = false+ for (const socket of sockets.splice(0)) socket.destroy()+ for (const release of held.splice(0)) release()+ await Promise.all(queued.splice(0))+ restoreQueueClock?.()+ restoreQueueClock = undefined+ await until(() => {+ const ordinary = limits.streamGate.snapshot(),+ visual = limits.visualStreamGate.snapshot()+ return (+ ordinary.running === 0 &&+ ordinary.queued === 0 &&+ visual.running === 0 &&+ visual.queued === 0+ )+ }, 'the proxy admission fixture did not drain')+ }+ cleanup.push(async () => {+ await stopCapacity()+ listener.closeAllConnections()+ await new Promise<void>((resolve, reject) =>+ listener.close(err => (err ? reject(err) : resolve()))+ )+ await apps.close()+ })+ function park(count: number, declared: number): void {+ for (let index = 0; index < count; index += 1) {+ const socket = connectTcp(port, '127.0.0.1')+ sockets.push(socket)+ socket.once('connect', () =>+ socket.write(+ `POST ${path} HTTP/1.1\r\nHost: 127.0.0.1:${port}\r\n` ++ `Authorization: Bearer ${platform}\r\nContent-Type: application/json\r\nContent-Length: ${declared}\r\n\r\n{`+ )+ )+ socket.on('error', err => {+ if ((err as NodeJS.ErrnoException).code !== 'ECONNRESET') throw err+ })+ socket.resume()+ }+ }+ async function fill(gate: Gate, width: number, waiters: number): Promise<void> {+ for (let index = 0; index < width; index += 1) held.push(await gate.acquire())+ for (let index = 0; index < waiters; index += 1)+ queued.push(gate.acquire().then(release => release()))+ await until(() => gate.snapshot().queued === waiters, 'the actual stream gate was not filled')+ }+ async function block(kind: LocalProducer): Promise<void> {+ if (kind === 'visual_host_share') {+ park(limits.VISUAL_PER_HOST_MAX_ADMITTED, 6000)+ await until(() => {+ const s = limits.visualStreamGate.snapshot()+ return s.running + s.queued === limits.VISUAL_PER_HOST_MAX_ADMITTED+ }, 'the actual principal share was not filled')+ } else if (kind === 'visual_gate') {+ await fill(+ limits.visualStreamGate,+ limits.VISUAL_STREAM_LIMITS.maxConcurrentStreams,+ limits.VISUAL_STREAM_LIMITS.maxQueuedRequests+ )+ } else if (kind === 'stream_queue_full') {+ await fill(+ limits.streamGate,+ limits.STREAM_LIMITS.maxConcurrentStreams,+ limits.STREAM_LIMITS.maxQueuedRequests+ )+ } else if (kind === 'stream_queue_wait') {+ // Shorten only the actual gate clock; producer and HTTP path stay real.+ const oldClock: unknown = Reflect.get(limits.streamGate, 'maxQueueWaitMs')+ Reflect.set(limits.streamGate, 'maxQueueWaitMs', 35)+ restoreQueueClock = () => {+ Reflect.set(limits.streamGate, 'maxQueueWaitMs', oldClock)+ }+ await fill(limits.streamGate, limits.STREAM_LIMITS.maxConcurrentStreams, 0)+ } else {+ const acquisitions = vi.spyOn(limits.BodyBudget.prototype, 'acquire')+ park(3, 4096)+ await until(+ () => (acquisitions.mock.contexts[0] as Budget | undefined)?.inFlightBytes === 3 * 4096,+ 'the ordinary readers did not take the byte budget'+ )+ const budget = acquisitions.mock.contexts[0] as Budget+ park(limits.STREAM_LIMITS.maxQueuedRequests, 4096)+ await until(+ () => budget.queued === limits.STREAM_LIMITS.maxQueuedRequests,+ 'the actual body queue was not filled'+ )+ padded = kind === 'demotion_queue_full'+ }+ }+ return {+ block,+ stopCapacity,+ run,+ engine,+ primary,+ authorize,+ redeem,+ stream,+ hostFetch,+ fallback,+ built,+ onSwitch,+ metricInc,+ classified: () => classified,+ }+}+describe.each(['grok', 'codex'] as const)(+ '%s subscription HTTP admission and failover',+ variant => {+ it.each(LOCAL_PRODUCERS)(+ 'local %s keeps the next turn on primary without redeem or failover',+ async kind => {+ const f = await fixture(variant)+ await f.block(kind)+ const content = kind.startsWith('visual_') ? 'x'.repeat(5000) : 'bounded text'+ const code = kind.startsWith('visual_') ? kind : 'proxy_capacity_exceeded'+ await expect(f.run(content)).rejects.toMatchObject({ code, dispatched: true })+ expect(f.classified()).toMatchObject({+ code: LlmErrorCode.ApiCallFailed,+ retryable: false,+ providerCode: code,+ providerDispatched: true,+ })+ expect(f.authorize).toHaveBeenCalledTimes(1)+ expect(f.hostFetch).toHaveBeenCalledTimes(1)+ expect(f.redeem).not.toHaveBeenCalled()+ expect(f.stream).not.toHaveBeenCalled()+ expect(f.built.map(target => target.kind)).toEqual(['primary'])+ expect(f.fallback).not.toHaveBeenCalled()+ expect(f.onSwitch).not.toHaveBeenCalled()+ expect(f.metricInc).not.toHaveBeenCalled()+ expect(f.engine.planTargets(f.primary)[0]?.kind).toBe('primary')+ await f.stopCapacity()+ expect((await f.run('next ordinary text')).content).toBe('primary route recovered')+ expect(f.authorize).toHaveBeenCalledTimes(2)+ expect(f.hostFetch).toHaveBeenCalledTimes(2)+ expect(f.redeem).toHaveBeenCalledTimes(1)+ expect(f.stream).toHaveBeenCalledTimes(1)+ expect(f.built.map(target => target.kind)).toEqual(['primary', 'primary'])+ expect(f.fallback).not.toHaveBeenCalled()+ expect(f.engine.servedBy()).toEqual({ ...f.primary, fallback: false })+ }+ )+ }+)diff --git a/mcp-host/src/llm/grokLlmProxyClient.ts b/mcp-host/src/llm/grokLlmProxyClient.tsindex 045a19333..8b92d3669 100644--- a/mcp-host/src/llm/grokLlmProxyClient.ts+++ b/mcp-host/src/llm/grokLlmProxyClient.ts@@ -15,6 +15,13 @@ export const GROK_PROXY_COMPLETIONS_PATH = '/internal/runtime/v1/grok/completion* sentence that says what to do; everything else keeps the diagnostic shape.*/export function grokProxyErrorMessage(code: string, status?: number): string {+ if (+ code === 'visual_host_share' ||+ code === 'visual_gate' ||+ code === 'proxy_capacity_exceeded'+ ) {+ return 'Grok proxy admission is full. Wait for an active request to finish or send fewer concurrent requests before retrying; this is not a Grok outage.'+ }if (code === 'client_upgrade_required') {return 'Grok subscription inference is unavailable: xAI now requires a newer Grok client version than this deployment sends. An operator can set GROK_LLM_PROXY_CLIENT_VERSION to a current Grok Build release, or contact support — retrying will not help.'}@@ -194,7 +201,9 @@ export class GrokLlmProxyClient {return this.streamOnce(input, false)}const payload = (await response.json().catch(() => ({}))) as Record<string, unknown>- // A 413 with no JSON code comes from the gateway in front of the proxy+ // Only the proxy-owned machine `error` field identifies the outcome; an+ // upstream `reason` string is never promoted to trusted provenance. A+ // 413 with no JSON code comes from the gateway in front of the proxy// (nginx `client_max_body_size`): a size refusal of this request, never a// provider outage (#739). A code the 413 carries still wins. A 429 is a// rate limit (G1-6, G1-11, #720): the proxy answers `rate_limited`diff --git a/scripts/ci/pinned-vitest-cases/codex-llm-proxy.json b/scripts/ci/pinned-vitest-cases/codex-llm-proxy.jsonindex d52dd3907..d890e99f2 100644--- a/scripts/ci/pinned-vitest-cases/codex-llm-proxy.json+++ b/scripts/ci/pinned-vitest-cases/codex-llm-proxy.json@@ -49,16 +49,32 @@},{"suite": "server.security.test.ts",- "fullName": "codex-llm-proxy ticket-aware stream-gate wait (#739 D1-bis) visual admission overflow answers 503 provider_unavailable and logs visual_gate"+ "fullName": "codex-llm-proxy ticket-aware stream-gate wait (#739 D1-bis) visual admission overflow answers 503 visual_gate and logs visual_gate"+ },+ {+ "suite": "server.security.test.ts",+ "fullName": "codex-llm-proxy attempt telemetry (l) logs the request-limit reason and labels its failure metric request_limit"+ },+ {+ "suite": "bodyAdmission.test.ts",+ "fullName": "codex-llm-proxy body admission (#731 R3-2) T-R3-2c answers proxy_capacity_exceeded once the admission queue is full"},{"suite": "streamGate.handoff.test.ts","fullName": "visual stream-gate handoff releases the visual slot for a whitespace-padded small V2"},+ {+ "suite": "streamGate.handoff.test.ts",+ "fullName": "visual stream-gate handoff visual per-host share admits at most four visual entries per principal while another principal proceeds"+ },{"suite": "streamGate.handoff.test.ts","fullName": "visual stream-gate handoff visual per-host share releases the principal share when the visual queue is full before grant"},+ {+ "suite": "visualBodyReadOwnership.test.ts",+ "fullName": "codex visual body read ownership keeps the visual slot and principal share until the timed-out parser terminates"+ },{"suite": "streamGate.handoff.test.ts","fullName": "visual stream-gate handoff visual slot lifetime frees the queue place of a visual request aborted while it waited"@@ -66,5 +82,25 @@{"suite": "streamGate.handoff.test.ts","fullName": "visual stream-gate handoff visual slot lifetime holds the queue place of a disconnected large-body waiter until its grant"+ },+ {+ "suite": "visualBodyReadOwnership.test.ts",+ "fullName": "codex visual body read ownership stops the native reader independently when a non-reading peer never finishes the response"+ },+ {+ "suite": "visualBodyReadOwnership.test.ts",+ "fullName": "codex visual body read ownership clears a late successful native parse after its read deadline without dispatching it"+ },+ {+ "suite": "visualBodyReadOwnership.test.ts",+ "fullName": "codex visual body read ownership clears the retained parsed body after response close before its callback returns"+ },+ {+ "suite": "server.security.test.ts",+ "fullName": "codex-llm-proxy attempt telemetry normalizes real transport upstream 503 claiming visual_gate to provider_unavailable"+ },+ {+ "suite": "streamLimitsFreeze.test.ts",+ "fullName": "codex-subscription stream limits freeze rejects open or mutable admission code producers while reading the closed production union"}]diff --git a/scripts/ci/pinned-vitest-cases/grok-llm-proxy.json b/scripts/ci/pinned-vitest-cases/grok-llm-proxy.jsonindex 27b8b6296..06f231acb 100644--- a/scripts/ci/pinned-vitest-cases/grok-llm-proxy.json+++ b/scripts/ci/pinned-vitest-cases/grok-llm-proxy.json@@ -51,14 +51,30 @@"suite": "server.security.test.ts","fullName": "grok-llm-proxy attempt telemetry (r4-l1b) logs a catalog DNS lookup failure by cause code on both admin routes"},+ {+ "suite": "server.security.test.ts",+ "fullName": "grok-llm-proxy attempt telemetry (rl1) logs the request-limit reason and labels its failure metric request_limit"+ },+ {+ "suite": "bodyAdmission.test.ts",+ "fullName": "grok-llm-proxy body admission (#731 R3-2) T-R3-2c-grok answers proxy_capacity_exceeded once the admission queue is full"+ },{"suite": "streamGate.handoff.test.ts","fullName": "grok visual stream-gate handoff releases the visual slot for a whitespace-padded small V2"},+ {+ "suite": "streamGate.handoff.test.ts",+ "fullName": "grok visual stream-gate handoff visual per-host share admits at most two visual entries per principal while another principal proceeds"+ },{"suite": "streamGate.handoff.test.ts","fullName": "grok visual stream-gate handoff visual per-host share releases the principal share when the visual queue is full before grant"},+ {+ "suite": "visualBodyReadOwnership.test.ts",+ "fullName": "grok visual body read ownership keeps the visual slot and principal share until the timed-out parser terminates"+ },{"suite": "streamGate.handoff.test.ts","fullName": "grok visual stream-gate handoff visual slot lifetime frees the queue place of a visual request aborted while it waited"@@ -66,5 +82,25 @@{"suite": "streamGate.handoff.test.ts","fullName": "grok visual stream-gate handoff visual slot lifetime holds the queue place of a disconnected large-body waiter until its grant"+ },+ {+ "suite": "visualBodyReadOwnership.test.ts",+ "fullName": "grok visual body read ownership stops the native reader independently when a non-reading peer never finishes the response"+ },+ {+ "suite": "visualBodyReadOwnership.test.ts",+ "fullName": "grok visual body read ownership clears a late successful native parse after its read deadline without dispatching it"+ },+ {+ "suite": "visualBodyReadOwnership.test.ts",+ "fullName": "grok visual body read ownership clears the retained parsed body after response close before its callback returns"+ },+ {+ "suite": "server.security.test.ts",+ "fullName": "grok-llm-proxy attempt telemetry normalizes real transport upstream 503 claiming visual_gate to provider_unavailable"+ },+ {+ "suite": "contractFreeze.test.ts",+ "fullName": "grok-subscription contract freeze rejects open or mutable admission code producers while reading the closed production union"}]diff --git a/scripts/ci/pinned-vitest-cases/mcp-host.json b/scripts/ci/pinned-vitest-cases/mcp-host.jsonnew file mode 100644index 000000000..706f5489e--- /dev/null+++ b/scripts/ci/pinned-vitest-cases/mcp-host.json@@ -0,0 +1,74 @@+[+ {+ "suite": "subscriptionAdmission.integration.test.ts",+ "fullName": "grok subscription HTTP admission and failover local visual_host_share keeps the next turn on primary without redeem or failover"+ },+ {+ "suite": "subscriptionAdmission.integration.test.ts",+ "fullName": "grok subscription HTTP admission and failover local visual_gate keeps the next turn on primary without redeem or failover"+ },+ {+ "suite": "subscriptionAdmission.integration.test.ts",+ "fullName": "grok subscription HTTP admission and failover local stream_queue_full keeps the next turn on primary without redeem or failover"+ },+ {+ "suite": "subscriptionAdmission.integration.test.ts",+ "fullName": "grok subscription HTTP admission and failover local stream_queue_wait keeps the next turn on primary without redeem or failover"+ },+ {+ "suite": "subscriptionAdmission.integration.test.ts",+ "fullName": "grok subscription HTTP admission and failover local body_queue_full keeps the next turn on primary without redeem or failover"+ },+ {+ "suite": "subscriptionAdmission.integration.test.ts",+ "fullName": "grok subscription HTTP admission and failover local demotion_queue_full keeps the next turn on primary without redeem or failover"+ },+ {+ "suite": "subscriptionAdmission.integration.test.ts",+ "fullName": "codex subscription HTTP admission and failover local visual_host_share keeps the next turn on primary without redeem or failover"+ },+ {+ "suite": "subscriptionAdmission.integration.test.ts",+ "fullName": "codex subscription HTTP admission and failover local visual_gate keeps the next turn on primary without redeem or failover"+ },+ {+ "suite": "subscriptionAdmission.integration.test.ts",+ "fullName": "codex subscription HTTP admission and failover local stream_queue_full keeps the next turn on primary without redeem or failover"+ },+ {+ "suite": "subscriptionAdmission.integration.test.ts",+ "fullName": "codex subscription HTTP admission and failover local stream_queue_wait keeps the next turn on primary without redeem or failover"+ },+ {+ "suite": "subscriptionAdmission.integration.test.ts",+ "fullName": "codex subscription HTTP admission and failover local body_queue_full keeps the next turn on primary without redeem or failover"+ },+ {+ "suite": "subscriptionAdmission.integration.test.ts",+ "fullName": "codex subscription HTTP admission and failover local demotion_queue_full keeps the next turn on primary without redeem or failover"+ },+ {+ "suite": "grokLlmProxyClient.test.ts",+ "fullName": "GrokLlmProxyClient keeps wire unknown_machine_code separate from trusted local refusal provenance"+ },+ {+ "suite": "grokLlmProxyClient.test.ts",+ "fullName": "GrokLlmProxyClient keeps wire reason_only separate from trusted local refusal provenance"+ },+ {+ "suite": "grokLlmProxyClient.test.ts",+ "fullName": "GrokLlmProxyClient keeps wire malformed_json separate from trusted local refusal provenance"+ },+ {+ "suite": "codexLlmProxyClient.test.ts",+ "fullName": "CodexLlmProxyClient keeps wire unknown_machine_code separate from trusted local refusal provenance"+ },+ {+ "suite": "codexLlmProxyClient.test.ts",+ "fullName": "CodexLlmProxyClient keeps wire reason_only separate from trusted local refusal provenance"+ },+ {+ "suite": "codexLlmProxyClient.test.ts",+ "fullName": "CodexLlmProxyClient keeps wire malformed_json separate from trusted local refusal provenance"+ }+]diff --git a/scripts/tests/test-codex-subscription-t0.sh b/scripts/tests/test-codex-subscription-t0.shindex 3eb98357a..641b4b6a6 100755--- a/scripts/tests/test-codex-subscription-t0.sh+++ b/scripts/tests/test-codex-subscription-t0.sh@@ -242,6 +242,9 @@ run_group "control-api" "control-api" \"test/routes.admin.codexSubscription.test.ts" \"test/routes.admin.codexSubscription.hostWrite.test.ts" \"test/routes.auth.codexSubscriptionCallback.test.ts" \+ "test/routes.mcp-host.llmProviderAttempts.bodyAdmission.test.ts" \+ "test/llmProviderAttemptBodyAdmission.test.ts" \+ "test/llmProviderAttemptAuthorizer.cancellation.test.ts" \"test/routes.mcp-host.llmProviderAttempts.test.ts" \"test/routes.adminRecipes.test.ts" \"test/routes.adminPluginWorkloadSdk.test.ts" \@@ -300,6 +303,7 @@ run_group "codex-llm-proxy" "codex-llm-proxy" \"test/sseBackpressure.test.ts" \"test/sseHeartbeat.test.ts" \"test/streamGate.handoff.test.ts" \+ "test/visualBodyReadOwnership.test.ts" \"test/streamLimitsFreeze.test.ts" \"test/toolNameMap.test.ts"@@ -325,6 +329,7 @@ run_group "mcp-host" "mcp-host" \"src/llm/__tests__/codexSubscription.test.ts" \"src/llm/__tests__/codexLlmProxyClient.test.ts" \"src/llm/__tests__/subscriptionRequestHash.test.ts" \+ "src/llm/failover/__tests__/subscriptionAdmission.integration.test.ts" \"src/llm/__tests__/providerAttemptAuthorizer.test.ts" \"src/llm/hostLlmBinding.test.ts" \"src/config/configStore.test.ts" \diff --git a/scripts/tests/test-grok-subscription-t0.sh b/scripts/tests/test-grok-subscription-t0.shindex 08a51666d..a0536876d 100755--- a/scripts/tests/test-grok-subscription-t0.sh+++ b/scripts/tests/test-grok-subscription-t0.sh@@ -252,6 +252,7 @@ run_group "grok-llm-proxy" "grok-llm-proxy" \"test/sseBackpressure.test.ts" \"test/sseHeartbeat.test.ts" \"test/streamGate.handoff.test.ts" \+ "test/visualBodyReadOwnership.test.ts" \"test/toolNameMap.test.ts" \"test/upstreamErrorHint.test.ts"@@ -273,6 +274,9 @@ run_group "control-api grok" "control-api" \"test/subscriptionGrantIdentity.test.ts" \"test/llmProviders.test.ts" \"test/routes.admin.grokSubscription.test.ts" \+ "test/routes.mcp-host.llmProviderAttempts.bodyAdmission.test.ts" \+ "test/llmProviderAttemptBodyAdmission.test.ts" \+ "test/llmProviderAttemptAuthorizer.cancellation.test.ts" \"test/routes.internal.llmProviderAttempts.grok.test.ts" \"test/routes.adminPluginWorkloadSdk.test.ts" \"test/routes.adminRecipes.test.ts" \@@ -296,6 +300,7 @@ run_group "mcp-host grok" "mcp-host" \"src/llm/__tests__/imageSource.test.ts" \"src/llm/__tests__/grokLlmProxyClient.test.ts" \"src/llm/__tests__/imageInput.test.ts" \+ "src/llm/failover/__tests__/subscriptionAdmission.integration.test.ts" \"src/llm/__tests__/providerAttemptAuthorizer.test.ts" \"src/core/adapters/__tests__/llmImageCompatibility.test.ts" \"src/agent/__tests__/taskExecutor.test.ts" \diff --git a/tests/e2e/fixtures/codex-subscription/sanitized-upstream-contract.json b/tests/e2e/fixtures/codex-subscription/sanitized-upstream-contract.jsonindex 1bea6b6e4..af548bbd7 100644--- a/tests/e2e/fixtures/codex-subscription/sanitized-upstream-contract.json+++ b/tests/e2e/fixtures/codex-subscription/sanitized-upstream-contract.json@@ -85,7 +85,10 @@"disabled","Unauthorized","not_found",- "internal_error"+ "internal_error",+ "visual_host_share",+ "visual_gate",+ "proxy_capacity_exceeded"],"termsAndTestAccount": {"automationAuthorized": true,diff --git a/tests/e2e/fixtures/grok-subscription/sanitized-upstream-contract.json b/tests/e2e/fixtures/grok-subscription/sanitized-upstream-contract.jsonindex bf373a3c7..1b8242b42 100644--- a/tests/e2e/fixtures/grok-subscription/sanitized-upstream-contract.json+++ b/tests/e2e/fixtures/grok-subscription/sanitized-upstream-contract.json@@ -102,7 +102,10 @@"host_binding_mismatch","Unauthorized","not_found",- "internal_error"+ "internal_error",+ "visual_host_share",+ "visual_gate",+ "proxy_capacity_exceeded"],"termsAndTestAccount": {"automationAuthorized": false,diff --git a/tests/e2e/fixtures/package-lock.json b/tests/e2e/fixtures/package-lock.jsonindex 8c88caac1..2e21df64b 100644--- a/tests/e2e/fixtures/package-lock.json+++ b/tests/e2e/fixtures/package-lock.json@@ -1,12 +1,12 @@{"name": "@clerum/e2e-test-fixtures",- "version": "1.0.100",+ "version": "1.0.101","lockfileVersion": 3,"requires": true,"packages": {"": {"name": "@clerum/e2e-test-fixtures",- "version": "1.0.100",+ "version": "1.0.101","dependencies": {"@kubernetes/client-node": "^1.0.0"},diff --git a/tests/e2e/fixtures/package.json b/tests/e2e/fixtures/package.jsonindex b80adc934..0523a0d75 100644--- a/tests/e2e/fixtures/package.json+++ b/tests/e2e/fixtures/package.json@@ -1,6 +1,6 @@{"name": "@clerum/e2e-test-fixtures",- "version": "1.0.100",+ "version": "1.0.101","description": "E2E test fixtures and utilities for Clerum testing","type": "module","main": "index.ts",