Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
feat(llm-proxy): give every principal a fair visual gate share
Alfredo Lopez6d790e374b7a
diff --git a/codex-llm-proxy/src/auth/platformJwtVerifier.ts b/codex-llm-proxy/src/auth/platformJwtVerifier.tsindex 3fd38321c..211ba02be 100644--- a/codex-llm-proxy/src/auth/platformJwtVerifier.ts+++ b/codex-llm-proxy/src/auth/platformJwtVerifier.ts@@ -25,12 +25,17 @@ export function verifyPlatformJwt(if (!Array.isArray(claims.hostRefs) || !Array.isArray(claims.workflowControlScopes)) {return null}+ // A11.7 V3: `sub` becomes part of the per-principal visual admission key,+ // so a missing, empty or non-string sub must fail verification instead of+ // collapsing every such caller onto the literal principal "undefined".+ const sub = claims.sub+ if (typeof sub !== 'string' || sub === '') return nullif (typeof claims.exp !== 'number') return nullif (claims.scope !== 'workflow:approval:request') return nullif (claims.hostRefs.some(ref => String(ref) === '*')) return nullif (!claims.workflowControlScopes.includes('llm:codex:execute')) return nullreturn {- sub: String(claims.sub),+ sub,hostRefs: claims.hostRefs.map(String),workflowControlScopes: claims.workflowControlScopes.map(String),}diff --git a/codex-llm-proxy/src/requestLimits.ts b/codex-llm-proxy/src/requestLimits.tsindex 792e73c93..1ef4f7db0 100644--- a/codex-llm-proxy/src/requestLimits.ts+++ b/codex-llm-proxy/src/requestLimits.ts@@ -89,6 +89,16 @@ export const VISUAL_STREAM_LIMITS = {maxQueuedRequests: 8,} as const+/**+ * Fair share of the visual gate's entries — running plus queued — that one+ * platform principal (`sub` plus sorted `hostRefs`) may hold at once. Above the+ * share, further visual requests from that principal are refused 503+ * `provider_unavailable` with log reason `visual_host_share`, so one principal+ * cannot fill a gate that every other host still needs. The gate widths above+ * are unchanged; the share only bounds how much of them one principal occupies.+ */+export const VISUAL_PER_HOST_MAX_ADMITTED = 4+// Largest delay setTimeout honors; Node fires anything above it after 1 ms.const MAX_TIMER_DELAY_MS = 2_147_483_647@@ -131,7 +141,10 @@ export class TicketLifeError extends RequestLimitError {}}-export function assertBoundedDeadline(deadlineMs: number | undefined, maxDeadlineMs: number): number {+export function assertBoundedDeadline(+ deadlineMs: number | undefined,+ maxDeadlineMs: number+): number {// Without a valid maximum, Math.min below would return NaN for a valid// request deadline instead of refusing it.if (!Number.isInteger(maxDeadlineMs) || maxDeadlineMs <= 0) {@@ -141,7 +154,12 @@ export function assertBoundedDeadline(deadlineMs: number | undefined, maxDeadlinif (!Number.isFinite(requested) || !Number.isInteger(requested) || requested <= 0) {throw new RequestLimitError('deadline is invalid', 'invalid')}- return Math.min(requested, maxDeadlineMs, STREAM_LIMITS.maxStreamDurationMs, CONTRACT_LIMITS.maxDeadlineMs)+ return Math.min(+ requested,+ maxDeadlineMs,+ STREAM_LIMITS.maxStreamDurationMs,+ CONTRACT_LIMITS.maxDeadlineMs+ )}export function assertBoundedIdleTimeout(idleTimeoutMs: number | undefined): number {@@ -173,7 +191,11 @@ export class StreamGate {// 1 ms timer, which would refuse every queued waiter at once instead of// after the configured wait. Fractions are refused so the bound stays a// whole number of milliseconds.- if (!Number.isInteger(maxQueueWaitMs) || maxQueueWaitMs <= 0 || maxQueueWaitMs > MAX_TIMER_DELAY_MS) {+ if (+ !Number.isInteger(maxQueueWaitMs) ||+ maxQueueWaitMs <= 0 ||+ maxQueueWaitMs > MAX_TIMER_DELAY_MS+ ) {throw new RangeError(`maxQueueWaitMs must be an integer in 1..${MAX_TIMER_DELAY_MS}`)}}@@ -197,7 +219,8 @@ export class StreamGate {}if (signal?.aborted) throw new RequestLimitError('stream request was aborted', 'aborted')if (this.running >= this.maxConcurrent) {- if (this.queued >= this.maxQueued) throw new RequestLimitError('stream queue is full', 'queue_full')+ if (this.queued >= this.maxQueued)+ throw new RequestLimitError('stream queue is full', 'queue_full')this.queued += 1try {await new Promise<void>((resolve, reject) => {@@ -315,7 +338,9 @@ export class BodyBudget {throw new RangeError(`a body of ${bytes} bytes cannot fit a budget of ${this.capacityBytes}`)}if (deadlineAt !== undefined && !Number.isFinite(deadlineAt)) {- throw new RangeError(`a body admission deadline must be a finite epoch time, got ${deadlineAt}`)+ throw new RangeError(+ `a body admission deadline must be a finite epoch time, got ${deadlineAt}`+ )}if (signal?.aborted) throw new RequestLimitError('body admission was aborted', 'aborted')if (this.waiters.length === 0 && this.inFlight + bytes <= this.capacityBytes) {@@ -368,7 +393,10 @@ export class BodyBudget {/** Grant queued bodies in arrival order while the head of the queue fits. */private drain(): void {- while (this.waiters.length > 0 && this.inFlight + this.waiters[0]!.bytes <= this.capacityBytes) {+ while (+ this.waiters.length > 0 &&+ this.inFlight + this.waiters[0]!.bytes <= this.capacityBytes+ ) {const waiter = this.waiters.shift()!waiter.grant(this.take(waiter.bytes))}diff --git a/codex-llm-proxy/src/server.ts b/codex-llm-proxy/src/server.tsindex c381b4056..f9e9eef02 100644--- a/codex-llm-proxy/src/server.ts+++ b/codex-llm-proxy/src/server.ts@@ -9,18 +9,23 @@ import { rateLimit } from 'express-rate-limit'import { type Server, createServer } from 'node:http'import { Registry, collectDefaultMetrics } from 'prom-client'import { z } from 'zod'-import { verifyAdminPermit } from './auth/adminPermitVerifier.js'import {- isExpiredExecutionTicket,- verifyExecutionTicket,-} from './auth/executionTicketVerifier.js'+ BODY_STRUCTURE_LIMITS,+ LIMITS,+ SCHEMA_VERSION_V2,+ createBodyStructureVerify,+ measureNonImageCompletionBytes,+ requestBodyLimitBytes,+} from '@clerum/llm-provider-attempt-contract'+import { verifyAdminPermit } from './auth/adminPermitVerifier.js'+import { isExpiredExecutionTicket, verifyExecutionTicket } from './auth/executionTicketVerifier.js'import { type PlatformJwtClaims, verifyPlatformJwt } from './auth/platformJwtVerifier.js'import {CodexTransportError,+ UpstreamTimeoutError,listCodexModels,streamCodexCompletion,testCodexConnection,- UpstreamTimeoutError,} from './codexTransport.js'import type { CodexLlmProxyConfig } from './config.js'import { ControlApiClient, ControlApiClientError } from './controlApiClient.js'@@ -31,14 +36,6 @@ import {type OriginPolicyOptions,defaultAddressLookup,} from './originPolicy.js'-import {- BODY_STRUCTURE_LIMITS,- LIMITS,- SCHEMA_VERSION_V2,- createBodyStructureVerify,- measureNonImageCompletionBytes,- requestBodyLimitBytes,-} from '@clerum/llm-provider-attempt-contract'import {BODY_READ_DEADLINE_MS,BodyBudget,@@ -47,6 +44,7 @@ import {RequestLimitError,STREAM_LIMITS,TicketLifeError,+ VISUAL_PER_HOST_MAX_ADMITTED,streamGate,visualStreamGate,} from './requestLimits.js'@@ -303,6 +301,9 @@ export function createProxyApps(// One budget for both apps: every body this process reads counts against it.const bodyBudget = new BodyBudget(IN_FLIGHT_BODY_BUDGET_BODIES * config.maxBodyBytes)const bodyReadDeadlineMs = deps.bodyReadDeadlineMs ?? BODY_READ_DEADLINE_MS+ // Per-principal occupancy of the visual gate (running plus queued entries).+ // Keyed on the verified platform principal: sub plus sorted hostRefs.+ const visualPrincipalAdmissions = new Map<string, number>()const runtimeApp = express()// R9-1: `inflate: false` on every parser. The budgets count the declared wire@@ -355,8 +356,41 @@ export function createProxyApps(ordinaryJson(req, res, next)return}+ // B4 / A11.7 V3: one principal gets a fair share of the visual gate, so a+ // single host cannot fill the entries every other host also needs. The+ // share is taken before the gate wait, so queued entries count too.+ const platform = req.codexPlatform+ if (!platform) {+ // Fail closed: admission without verified claims must never fall back to+ // a shared per-principal key. platformGate runs before this middleware.+ reject(res, 401, 'Unauthorized')+ return+ }+ const visualPrincipal = JSON.stringify([platform.sub, ...[...platform.hostRefs].sort()])+ const admittedByPrincipal = visualPrincipalAdmissions.get(visualPrincipal) ?? 0+ if (admittedByPrincipal >= VISUAL_PER_HOST_MAX_ADMITTED) {+ logger.warn(+ {+ event: 'codex_proxy_admission_refused',+ reason: 'visual_host_share',+ limit: VISUAL_PER_HOST_MAX_ADMITTED,+ },+ 'admission refused'+ )+ reject(res, 503, 'provider_unavailable')+ return+ }+ visualPrincipalAdmissions.set(visualPrincipal, admittedByPrincipal + 1)void (async () => {let release: (() => void) | undefined+ let principalShareHeld = true+ const releasePrincipalShare = (): void => {+ if (!principalShareHeld) return+ principalShareHeld = false+ const remaining = (visualPrincipalAdmissions.get(visualPrincipal) ?? 1) - 1+ if (remaining > 0) visualPrincipalAdmissions.set(visualPrincipal, remaining)+ else visualPrincipalAdmissions.delete(visualPrincipal)+ }const parseAbort = new AbortController()const abortParse = (): void => parseAbort.abort()req.once('aborted', abortParse)@@ -364,6 +398,7 @@ export function createProxyApps(release = await visualStreamGate.acquire(parseAbort.signal, req.codexAdmissionDeadlineAt)} catch (err) {req.off('aborted', abortParse)+ releasePrincipalShare()// The client left while queued: nobody reads a refusal, and a departed// client is not gate saturation, so it is not logged as `visual_gate`.// Same rule as the ordinary path's `!abort.signal.aborted`.@@ -386,6 +421,7 @@ export function createProxyApps(}req.off('aborted', abortParse)const releaseVisual = (): void => {+ releasePrincipalShare()req.codexStreamRelease?.()req.codexStreamRelease = undefined}@@ -461,7 +497,8 @@ export function createProxyApps(// the same envelope allowance control-api's authorizer grants its wrapper.if (wholeBodyBytes > Math.min(configuredLimit, envelopeLimit) ||- measureNonImageCompletionBytes(req.body) > LIMITS.maxRequestBodyBytes + ENVELOPE_ALLOWANCE_BYTES+ measureNonImageCompletionBytes(req.body) >+ LIMITS.maxRequestBodyBytes + ENVELOPE_ALLOWANCE_BYTES) {releaseAdmission()reject(res, 413, 'payload_too_large')diff --git a/codex-llm-proxy/test/requestLimits.test.ts b/codex-llm-proxy/test/requestLimits.test.tsindex c8a2cb38a..d96428a31 100644--- a/codex-llm-proxy/test/requestLimits.test.ts+++ b/codex-llm-proxy/test/requestLimits.test.ts@@ -1,12 +1,13 @@-import { getEventListeners } from 'node:events'import { describe, expect, it, vi } from 'vitest'+import { getEventListeners } from 'node:events'import { CONTROL_API_REQUEST_TIMEOUT_MS } from '../src/controlApiClient.js'import {- assertBoundedDeadline,RequestLimitError,STREAM_LIMITS,StreamGate,+ VISUAL_PER_HOST_MAX_ADMITTED,VISUAL_STREAM_LIMITS,+ assertBoundedDeadline,} from '../src/requestLimits.js'import { DEFAULT_HEARTBEAT_INTERVAL_MS, MAX_HEARTBEAT_INTERVAL_MS } from '../src/sseHeartbeat.js'@@ -379,15 +380,18 @@ describe('StreamGate', () => {// Witness: the smallest sizes are accepted, including a gate with no queue.expect(() => new StreamGate(1, 0)).not.toThrow()for (const maxConcurrent of [0, -1, 1.5, Number.NaN]) {- expect(() => new StreamGate(maxConcurrent, 1), `maxConcurrent=${maxConcurrent}`).toThrow(RangeError)+ expect(() => new StreamGate(maxConcurrent, 1), `maxConcurrent=${maxConcurrent}`).toThrow(+ RangeError+ )}for (const maxQueued of [-1, 1.5, Number.NaN]) {expect(() => new StreamGate(1, maxQueued), `maxQueued=${maxQueued}`).toThrow(RangeError)}})- it('pins the visual 2/8 sibling against the ordinary 8/16 stream gate', () => {+ it('pins the visual 2/8 sibling, its per-host share, and the ordinary 8/16 gate', () => {expect(VISUAL_STREAM_LIMITS).toEqual({ maxConcurrentStreams: 2, maxQueuedRequests: 8 })+ expect(VISUAL_PER_HOST_MAX_ADMITTED).toBe(4)expect(STREAM_LIMITS.maxConcurrentStreams).toBe(8)expect(STREAM_LIMITS.maxQueuedRequests).toBe(16)})diff --git a/codex-llm-proxy/test/streamGate.handoff.test.ts b/codex-llm-proxy/test/streamGate.handoff.test.tsindex 690b0da07..339156a15 100644--- a/codex-llm-proxy/test/streamGate.handoff.test.ts+++ b/codex-llm-proxy/test/streamGate.handoff.test.ts@@ -1,22 +1,23 @@+import { afterEach, describe, expect, it, vi } from 'vitest'+import jwt from 'jsonwebtoken'import { generateKeyPairSync } from 'node:crypto'import { createServer } from 'node:http'import { connect as connectTcp } from 'node:net'-import jwt from 'jsonwebtoken'-import { afterEach, describe, expect, it, vi } from 'vitest'import {hashCodexCompletionRequest,parseCodexCompletionRequest,} from '@clerum/llm-provider-attempt-contract'import { type CodexLlmProxyConfig } from '../src/config.js'import { logger } from '../src/logger.js'-import { createProxyApps } from '../src/server.js'import {RequestLimitError,STREAM_LIMITS,+ VISUAL_PER_HOST_MAX_ADMITTED,VISUAL_STREAM_LIMITS,streamGate,visualStreamGate,} from '../src/requestLimits.js'+import { createProxyApps } from '../src/server.js'const COMPLETIONS_PATH = '/internal/runtime/v1/codex/completions'@@ -143,13 +144,14 @@ function completionPayload(async function postCompletion(port: number,schemaVersion: 'codex-completion-request.v1' | 'codex-completion-request.v2',- content = 'hi'+ content = 'hi',+ hostRefs: string[] = ['research-host']): Promise<Response> {- const { token, body } = completionPayload(schemaVersion, content)+ const { body } = completionPayload(schemaVersion, content)return fetch(`http://127.0.0.1:${port}/internal/runtime/v1/codex/completions`, {method: 'POST',headers: {- Authorization: `Bearer ${token}`,+ Authorization: `Bearer ${platformToken(hostRefs)}`,'content-type': 'application/json',},body,@@ -289,12 +291,14 @@ describe('visual stream-gate handoff', () => {for (const hang of hangs.splice(0)) hang.release()for (const server of serversToClose.splice(0)) await server.close()await Promise.all(- listeners.splice(0).map(- listener =>- new Promise<void>((resolve, reject) =>- listener.close(err => (err ? reject(err) : resolve()))- )- )+ listeners+ .splice(0)+ .map(+ listener =>+ new Promise<void>((resolve, reject) =>+ listener.close(err => (err ? reject(err) : resolve()))+ )+ ))await waitFor(() =>@@ -341,8 +345,11 @@ describe('visual stream-gate handoff', () => {'small V1 waited on the visual gate instead of the ordinary stream gate')- const queued = Array.from({ length: VISUAL_STREAM_LIMITS.maxQueuedRequests }, () =>- postCompletion(port, 'codex-completion-request.v2', largeContent)+ // One principal can hold at most four visual entries, so each queued+ // request mints a distinct principal; the subject here is the global gate+ // width, not the per-host share.+ const queued = Array.from({ length: VISUAL_STREAM_LIMITS.maxQueuedRequests }, (_, index) =>+ postCompletion(port, 'codex-completion-request.v2', largeContent, [`visual-queue-${index}`]))await waitFor(() => visualStreamGate.snapshot().queued === VISUAL_STREAM_LIMITS.maxQueuedRequests,@@ -459,6 +466,109 @@ describe('visual stream-gate handoff', () => {await response})+ describe('visual per-host share', () => {+ function invalidTicketBody(maxBodyBytes: number): string {+ const body = JSON.stringify({+ executionTicket: 'invalid-ticket',+ requestHash: 'a'.repeat(64),+ request: completionRequest('codex-completion-request.v2', 'x'.repeat(maxBodyBytes)),+ })+ expect(Buffer.byteLength(body)).toBeGreaterThan(maxBodyBytes)+ return body+ }++ it('refuses a platform token whose sub is not a non-empty string with 401', async () => {+ const port = listen(createProxyApps(config()))+ const claims = {+ hostRefs: ['research-host'],+ workflowControlScopes: ['llm:codex:execute'],+ scope: 'workflow:approval:request',+ }+ for (const bearerToken of [+ sign(claims, 'workflow-approvals'),+ sign({ ...claims, sub: '' }, 'workflow-approvals'),+ sign({ ...claims, sub: 403 }, 'workflow-approvals'),+ ]) {+ const res = await postBody(port, bearerToken, JSON.stringify({ probe: 1 }))+ expect(res.status).toBe(401)+ expect(await res.json()).toEqual({ error: 'Unauthorized' })+ }+ // Liveness witness: none of those requests reached body admission.+ expect(visualStreamGate.snapshot()).toEqual({ running: 0, queued: 0 })+ })++ it('admits at most four visual entries per principal while another principal proceeds', async () => {+ expect(VISUAL_PER_HOST_MAX_ADMITTED).toBe(4)+ const maxBodyBytes =+ Buffer.byteLength(completionPayload('codex-completion-request.v1').body) + 512+ const hang = hangStream()+ hangs.push(hang)+ const acquire = vi.spyOn(visualStreamGate, 'acquire')+ const warn = vi.spyOn(logger, 'warn')+ const port = listen(+ createProxyApps(config({ maxBodyBytes, maxVisualBodyBytes: 1024 * 1024 }), {+ streamCompletion: hang.impl,+ })+ )++ // Host A's first two entries run and its next two queue; all four count+ // against the same principal (sub plus hostRefs).+ const first = postCompletion(port, 'codex-completion-request.v2', 'x'.repeat(maxBodyBytes))+ const second = postCompletion(port, 'codex-completion-request.v2', 'x'.repeat(maxBodyBytes))+ await waitFor(+ () => visualStreamGate.snapshot().running === 2,+ 'host A did not take both running visual slots'+ )+ const third = postBody(port, platformToken(), invalidTicketBody(maxBodyBytes))+ const fourth = postBody(port, platformToken(), invalidTicketBody(maxBodyBytes))+ await waitFor(+ () => visualStreamGate.snapshot().queued === 2,+ 'host A did not take two queued visual entries'+ )+ expect(acquire).toHaveBeenCalledTimes(4)++ const fifth = await postBody(port, platformToken(), invalidTicketBody(maxBodyBytes))+ expect(fifth.status).toBe(503)+ expect(await fifth.json()).toEqual({ error: 'provider_unavailable' })+ expect(warn).toHaveBeenCalledWith(+ expect.objectContaining({+ event: 'codex_proxy_admission_refused',+ reason: 'visual_host_share',+ }),+ 'admission refused'+ )+ // The share refusal happens before the global gate is touched.+ expect(acquire).toHaveBeenCalledTimes(4)++ const otherHost = postBody(+ port,+ platformToken(['other-host']),+ invalidTicketBody(maxBodyBytes)+ )+ await waitFor(+ () => visualStreamGate.snapshot().queued === 3,+ 'host B was not admitted while host A held its full share'+ )++ hang.release()+ await first+ await second+ const thirdRes = await third+ expect(thirdRes.status).toBe(403)+ expect(await thirdRes.json()).toEqual({ error: 'ticket_invalid' })+ const otherRes = await otherHost+ expect(otherRes.status).toBe(403)+ expect(await otherRes.json()).toEqual({ error: 'ticket_invalid' })+ await waitFor(+ () => visualStreamGate.snapshot().running === 0 && visualStreamGate.snapshot().queued === 0,+ 'the visual gate did not drain'+ )+ // Host A's share was released with its entries, so a fresh A request is+ // admitted again and reaches the ticket check.+ await expectNextVisualAdmitted(port, maxBodyBytes)+ })+ })+describe('visual slot lifetime', () => {function smallCaps(): number {return Buffer.byteLength(completionPayload('codex-completion-request.v1').body) + 512@@ -466,8 +576,7 @@ describe('visual stream-gate handoff', () => {async function expectGateEmpty(): Promise<void> {await waitFor(- () =>- visualStreamGate.snapshot().running === 0 && visualStreamGate.snapshot().queued === 0,+ () => visualStreamGate.snapshot().running === 0 && visualStreamGate.snapshot().queued === 0,'the visual slot was not released')expect(visualStreamGate.snapshot()).toEqual({ running: 0, queued: 0 })@@ -476,7 +585,9 @@ describe('visual stream-gate handoff', () => {it('answers 400 to a deeply nested visual body and frees the slot', async () => {const maxBodyBytes = smallCaps()const acquire = vi.spyOn(visualStreamGate, 'acquire')- const port = listen(createProxyApps(config({ maxBodyBytes, maxVisualBodyBytes: 1024 * 1024 })))+ const port = listen(+ createProxyApps(config({ maxBodyBytes, maxVisualBodyBytes: 1024 * 1024 }))+ )// JSON.parse accepts this depth; JSON.stringify and the contract's byte// measurement throw RangeError on it.const depth = 20_000@@ -497,7 +608,9 @@ describe('visual stream-gate handoff', () => {it('frees the slot when the handler throws before releasing it', async () => {const maxBodyBytes = smallCaps()const acquire = vi.spyOn(visualStreamGate, 'acquire')- const port = listen(createProxyApps(config({ maxBodyBytes, maxVisualBodyBytes: 1024 * 1024 })))+ const port = listen(+ createProxyApps(config({ maxBodyBytes, maxVisualBodyBytes: 1024 * 1024 }))+ )const body = JSON.stringify({executionTicket: 'invalid-ticket',requestHash: 'a'.repeat(64),@@ -615,9 +728,7 @@ describe('visual stream-gate handoff', () => {const maxBodyBytes = smallCaps()const acquire = vi.spyOn(visualStreamGate, 'acquire')const port = listen(- createProxyApps(- config({ maxBodyBytes, maxVisualBodyBytes: 1024 * 1024, ...row.overrides })- )+ createProxyApps(config({ maxBodyBytes, maxVisualBodyBytes: 1024 * 1024, ...row.overrides })))const payload = row.body(maxBodyBytes)expect(Buffer.byteLength(payload.body)).toBeGreaterThan(maxBodyBytes)@@ -662,11 +773,17 @@ describe('visual stream-gate handoff', () => {body: payload.body,signal: abort.signal,}).catch((err: unknown) => err)- await waitFor(() => visualStreamGate.snapshot().queued === 1, 'the second request did not queue')+ await waitFor(+ () => visualStreamGate.snapshot().queued === 1,+ 'the second request did not queue'+ )const warn = vi.spyOn(logger, 'warn')abort.abort()expect(await waiting).toBeInstanceOf(Error)- await waitFor(() => visualStreamGate.snapshot().queued === 0, 'the aborted waiter kept its place')+ await waitFor(+ () => visualStreamGate.snapshot().queued === 0,+ 'the aborted waiter kept its place'+ )expect(acquire).toHaveBeenCalledTimes(3)// Witness: the waiter's acquire ended on the client's abort, so the// catch around it ran. A departed client is not gate saturation and@@ -726,7 +843,10 @@ describe('visual stream-gate handoff', () => {const code = (err as NodeJS.ErrnoException).codeif (code !== 'ECONNRESET' && code !== 'EPIPE') throw err})- await waitFor(() => visualStreamGate.snapshot().queued === 1, 'the large waiter did not queue')+ await waitFor(+ () => visualStreamGate.snapshot().queued === 1,+ 'the large waiter did not queue'+ )const closed = new Promise<void>(resolve => waiter.once('close', () => resolve()))waiter.destroy()await closeddiff --git a/grok-llm-proxy/src/auth/platformJwtVerifier.ts b/grok-llm-proxy/src/auth/platformJwtVerifier.tsindex 0888088d4..6b1044714 100644--- a/grok-llm-proxy/src/auth/platformJwtVerifier.ts+++ b/grok-llm-proxy/src/auth/platformJwtVerifier.ts@@ -30,12 +30,17 @@ export function verifyPlatformJwt(if (!Array.isArray(claims.hostRefs) || !Array.isArray(claims.workflowControlScopes)) {return null}+ // A11.7 V3: `sub` becomes part of the per-principal visual admission key,+ // so a missing, empty or non-string sub must fail verification instead of+ // collapsing every such caller onto the literal principal "undefined".+ const sub = claims.sub+ if (typeof sub !== 'string' || sub === '') return nullif (typeof claims.exp !== 'number') return nullif (claims.scope !== 'workflow:approval:request') return nullif (claims.hostRefs.some(ref => String(ref) === '*')) return nullif (!claims.workflowControlScopes.includes('llm:grok:execute')) return nullreturn {- sub: String(claims.sub),+ sub,hostRefs: claims.hostRefs.map(String),workflowControlScopes: claims.workflowControlScopes.map(String),}diff --git a/grok-llm-proxy/src/requestLimits.ts b/grok-llm-proxy/src/requestLimits.tsindex aa98f8fb0..6ec2ef0f5 100644--- a/grok-llm-proxy/src/requestLimits.ts+++ b/grok-llm-proxy/src/requestLimits.ts@@ -81,6 +81,16 @@ export const VISUAL_STREAM_LIMITS = {maxQueuedRequests: 4,} as const+/**+ * Fair share of the visual gate's entries — running plus queued — that one+ * platform principal (`sub` plus sorted `hostRefs`) may hold at once. Above the+ * share, further visual requests from that principal are refused 503+ * `provider_unavailable` with log reason `visual_host_share`, so one principal+ * cannot fill a gate that every other host still needs. The gate widths above+ * are unchanged; the share only bounds how much of them one principal occupies.+ */+export const VISUAL_PER_HOST_MAX_ADMITTED = 2+// Largest delay setTimeout honors; Node fires anything above it after 1 ms.const MAX_TIMER_DELAY_MS = 2_147_483_647@@ -173,7 +183,11 @@ export class StreamGate {// 1 ms timer, which would refuse every queued waiter at once instead of// after the configured wait. Fractions are refused so the bound stays a// whole number of milliseconds.- if (!Number.isInteger(maxQueueWaitMs) || maxQueueWaitMs <= 0 || maxQueueWaitMs > MAX_TIMER_DELAY_MS) {+ if (+ !Number.isInteger(maxQueueWaitMs) ||+ maxQueueWaitMs <= 0 ||+ maxQueueWaitMs > MAX_TIMER_DELAY_MS+ ) {throw new RangeError(`maxQueueWaitMs must be an integer in 1..${MAX_TIMER_DELAY_MS}`)}}@@ -197,7 +211,8 @@ export class StreamGate {}if (signal?.aborted) throw new RequestLimitError('stream request was aborted', 'aborted')if (this.running >= this.maxConcurrent) {- if (this.queued >= this.maxQueued) throw new RequestLimitError('stream queue is full', 'queue_full')+ if (this.queued >= this.maxQueued)+ throw new RequestLimitError('stream queue is full', 'queue_full')this.queued += 1try {await new Promise<void>((resolve, reject) => {@@ -315,7 +330,9 @@ export class BodyBudget {throw new RangeError(`a body of ${bytes} bytes cannot fit a budget of ${this.capacityBytes}`)}if (deadlineAt !== undefined && !Number.isFinite(deadlineAt)) {- throw new RangeError(`a body admission deadline must be a finite epoch time, got ${deadlineAt}`)+ throw new RangeError(+ `a body admission deadline must be a finite epoch time, got ${deadlineAt}`+ )}if (signal?.aborted) throw new RequestLimitError('body admission was aborted', 'aborted')if (this.waiters.length === 0 && this.inFlight + bytes <= this.capacityBytes) {@@ -368,7 +385,10 @@ export class BodyBudget {/** Grant queued bodies in arrival order while the head of the queue fits. */private drain(): void {- while (this.waiters.length > 0 && this.inFlight + this.waiters[0]!.bytes <= this.capacityBytes) {+ while (+ this.waiters.length > 0 &&+ this.inFlight + this.waiters[0]!.bytes <= this.capacityBytes+ ) {const waiter = this.waiters.shift()!waiter.grant(this.take(waiter.bytes))}diff --git a/grok-llm-proxy/src/server.ts b/grok-llm-proxy/src/server.tsindex e5a77172c..cb945abd2 100644--- a/grok-llm-proxy/src/server.ts+++ b/grok-llm-proxy/src/server.ts@@ -7,6 +7,8 @@ import express, {} from 'express'import { rateLimit } from 'express-rate-limit'import { type Server, createServer } from 'node:http'+import { Registry, collectDefaultMetrics } from 'prom-client'+import { z } from 'zod'import {BODY_STRUCTURE_LIMITS,LIMITS,@@ -15,22 +17,17 @@ import {measureNonImageCompletionBytes,requestBodyLimitBytes,} from '@clerum/grok-provider-attempt-contract'-import { Registry, collectDefaultMetrics } from 'prom-client'-import { z } from 'zod'import { verifyAdminPermit } from './auth/adminPermitVerifier.js'-import {- isExpiredExecutionTicket,- verifyExecutionTicket,-} from './auth/executionTicketVerifier.js'+import { isExpiredExecutionTicket, verifyExecutionTicket } from './auth/executionTicketVerifier.js'import { type PlatformJwtClaims, verifyPlatformJwt } from './auth/platformJwtVerifier.js'import type { GrokLlmProxyConfig } from './config.js'import { ControlApiClient, ControlApiClientError } from './controlApiClient.js'import {GrokTransportError,+ UpstreamTimeoutError,listGrokModels,streamGrokCompletion,testGrokConnection,- UpstreamTimeoutError,} from './grokTransport.js'import { logger } from './logger.js'import { createProxyMetrics } from './metrics.js'@@ -47,6 +44,7 @@ import {RequestLimitError,STREAM_LIMITS,TicketLifeError,+ VISUAL_PER_HOST_MAX_ADMITTED,streamGate,visualStreamGate,} from './requestLimits.js'@@ -331,6 +329,9 @@ export function createProxyApps(// One budget for both apps: every body this process reads counts against it.const bodyBudget = new BodyBudget(IN_FLIGHT_BODY_BUDGET_BODIES * config.maxBodyBytes)const bodyReadDeadlineMs = deps.bodyReadDeadlineMs ?? BODY_READ_DEADLINE_MS+ // Per-principal occupancy of the visual gate (running plus queued entries).+ // Keyed on the verified platform principal: sub plus sorted hostRefs.+ const visualPrincipalAdmissions = new Map<string, number>()const runtimeApp = express()// R9-M-B: the token is checked from the header before any budget is taken or@@ -391,8 +392,41 @@ export function createProxyApps(ordinaryJson(req, res, next)return}+ // B4 / A11.7 V3: one principal gets a fair share of the visual gate, so a+ // single host cannot fill the entries every other host also needs. The+ // share is taken before the gate wait, so queued entries count too.+ const platform = req.grokPlatform+ if (!platform) {+ // Fail closed: admission without verified claims must never fall back to+ // a shared per-principal key. platformGate runs before this middleware.+ reject(res, 401, 'Unauthorized')+ return+ }+ const visualPrincipal = JSON.stringify([platform.sub, ...[...platform.hostRefs].sort()])+ const admittedByPrincipal = visualPrincipalAdmissions.get(visualPrincipal) ?? 0+ if (admittedByPrincipal >= VISUAL_PER_HOST_MAX_ADMITTED) {+ logger.warn(+ {+ event: 'grok_proxy_admission_refused',+ reason: 'visual_host_share',+ limit: VISUAL_PER_HOST_MAX_ADMITTED,+ },+ 'admission refused'+ )+ reject(res, 503, 'provider_unavailable')+ return+ }+ visualPrincipalAdmissions.set(visualPrincipal, admittedByPrincipal + 1)void (async () => {let release: (() => void) | undefined+ let principalShareHeld = true+ const releasePrincipalShare = (): void => {+ if (!principalShareHeld) return+ principalShareHeld = false+ const remaining = (visualPrincipalAdmissions.get(visualPrincipal) ?? 1) - 1+ if (remaining > 0) visualPrincipalAdmissions.set(visualPrincipal, remaining)+ else visualPrincipalAdmissions.delete(visualPrincipal)+ }const parseAbort = new AbortController()const abortParse = (): void => parseAbort.abort()req.once('aborted', abortParse)@@ -400,6 +434,7 @@ export function createProxyApps(release = await visualStreamGate.acquire(parseAbort.signal, req.grokAdmissionDeadlineAt)} catch (err) {req.off('aborted', abortParse)+ releasePrincipalShare()// The client left while queued: nobody reads a refusal, and a departed// client is not gate saturation, so it is not logged as `visual_gate`.// Same rule as the ordinary path's `!abort.signal.aborted`.@@ -422,6 +457,7 @@ export function createProxyApps(}req.off('aborted', abortParse)const releaseVisual = (): void => {+ releasePrincipalShare()req.grokStreamRelease?.()req.grokStreamRelease = undefined}@@ -503,7 +539,8 @@ export function createProxyApps(// the same envelope allowance control-api's authorizer grants its wrapper.if (wholeBodyBytes > Math.min(configuredLimit, envelopeLimit) ||- measureNonImageCompletionBytes(req.body) > LIMITS.maxRequestBodyBytes + ENVELOPE_ALLOWANCE_BYTES+ measureNonImageCompletionBytes(req.body) >+ LIMITS.maxRequestBodyBytes + ENVELOPE_ALLOWANCE_BYTES) {releaseAdmission()reject(res, 413, 'payload_too_large')diff --git a/grok-llm-proxy/test/requestLimits.test.ts b/grok-llm-proxy/test/requestLimits.test.tsindex 0c1202848..1e5152137 100644--- a/grok-llm-proxy/test/requestLimits.test.ts+++ b/grok-llm-proxy/test/requestLimits.test.ts@@ -1,12 +1,13 @@-import { getEventListeners } from 'node:events'import { describe, expect, it, vi } from 'vitest'+import { getEventListeners } from 'node:events'import { CONTROL_API_REQUEST_TIMEOUT_MS } from '../src/controlApiClient.js'import {- assertBoundedDeadline,RequestLimitError,STREAM_LIMITS,StreamGate,+ VISUAL_PER_HOST_MAX_ADMITTED,VISUAL_STREAM_LIMITS,+ assertBoundedDeadline,visualStreamGate,} from '../src/requestLimits.js'import { DEFAULT_HEARTBEAT_INTERVAL_MS, MAX_HEARTBEAT_INTERVAL_MS } from '../src/sseHeartbeat.js'@@ -386,7 +387,9 @@ describe('StreamGate', () => {// Witness: the smallest sizes are accepted, including a gate with no queue.expect(() => new StreamGate(1, 0)).not.toThrow()for (const maxConcurrent of [0, -1, 1.5, Number.NaN]) {- expect(() => new StreamGate(maxConcurrent, 1), `maxConcurrent=${maxConcurrent}`).toThrow(RangeError)+ expect(() => new StreamGate(maxConcurrent, 1), `maxConcurrent=${maxConcurrent}`).toThrow(+ RangeError+ )}for (const maxQueued of [-1, 1.5, Number.NaN]) {expect(() => new StreamGate(1, maxQueued), `maxQueued=${maxQueued}`).toThrow(RangeError)@@ -396,8 +399,9 @@ describe('StreamGate', () => {// A V2 body above the ordinary cap keeps its image bytes resident for the// whole upstream stream, so it takes a 1-wide sibling of the 8-wide gate.// Widening it needs a new memory measurement against the 1Gi limit.- it('pins the visual 1/4 sibling against the ordinary 8/16 stream gate', () => {+ it('pins the visual 1/4 sibling, its per-host share, and the ordinary 8/16 gate', () => {expect(VISUAL_STREAM_LIMITS).toEqual({ maxConcurrentStreams: 1, maxQueuedRequests: 4 })+ expect(VISUAL_PER_HOST_MAX_ADMITTED).toBe(2)expect(STREAM_LIMITS.maxConcurrentStreams).toBe(8)expect(STREAM_LIMITS.maxQueuedRequests).toBe(16)})diff --git a/grok-llm-proxy/test/streamGate.handoff.test.ts b/grok-llm-proxy/test/streamGate.handoff.test.tsindex 0b3957c86..002bc4200 100644--- a/grok-llm-proxy/test/streamGate.handoff.test.ts+++ b/grok-llm-proxy/test/streamGate.handoff.test.ts@@ -1,22 +1,23 @@+import { afterEach, describe, expect, it, vi } from 'vitest'+import jwt from 'jsonwebtoken'import { generateKeyPairSync } from 'node:crypto'import { createServer } from 'node:http'import { connect as connectTcp } from 'node:net'-import jwt from 'jsonwebtoken'-import { afterEach, describe, expect, it, vi } from 'vitest'import {hashGrokCompletionRequest,parseGrokCompletionRequest,} from '@clerum/grok-provider-attempt-contract'import { type GrokLlmProxyConfig } from '../src/config.js'import { logger } from '../src/logger.js'-import { createProxyApps } from '../src/server.js'import {RequestLimitError,STREAM_LIMITS,+ VISUAL_PER_HOST_MAX_ADMITTED,VISUAL_STREAM_LIMITS,streamGate,visualStreamGate,} from '../src/requestLimits.js'+import { createProxyApps } from '../src/server.js'const { privateKey, publicKey } = generateKeyPairSync('rsa', {modulusLength: 2048,@@ -142,13 +143,14 @@ function completionPayload(async function postCompletion(port: number,schemaVersion: SchemaVersion,- content = 'hi'+ content = 'hi',+ hostRefs: string[] = ['research-host']): Promise<Response> {- const { token, body } = completionPayload(schemaVersion, content)+ const { body } = completionPayload(schemaVersion, content)return fetch(`http://127.0.0.1:${port}${COMPLETIONS_PATH}`, {method: 'POST',headers: {- Authorization: `Bearer ${token}`,+ Authorization: `Bearer ${platformToken(hostRefs)}`,'content-type': 'application/json',},body,@@ -279,12 +281,14 @@ describe('grok visual stream-gate handoff', () => {for (const hang of hangs.splice(0)) hang.release()for (const server of serversToClose.splice(0)) await server.close()await Promise.all(- listeners.splice(0).map(- listener =>- new Promise<void>((resolve, reject) =>- listener.close(err => (err ? reject(err) : resolve()))- )- )+ listeners+ .splice(0)+ .map(+ listener =>+ new Promise<void>((resolve, reject) =>+ listener.close(err => (err ? reject(err) : resolve()))+ )+ ))await waitFor(() =>@@ -335,8 +339,11 @@ describe('grok visual stream-gate handoff', () => {'small V1 waited on the visual gate instead of the ordinary stream gate')- const queued = Array.from({ length: VISUAL_STREAM_LIMITS.maxQueuedRequests }, () =>- postCompletion(port, 'grok-completion-request.v2', largeContent)+ // One principal can hold at most two visual entries, so each queued+ // request mints a distinct principal; the subject here is the global gate+ // width, not the per-host share.+ const queued = Array.from({ length: VISUAL_STREAM_LIMITS.maxQueuedRequests }, (_, index) =>+ postCompletion(port, 'grok-completion-request.v2', largeContent, [`visual-queue-${index}`]))await waitFor(() => visualStreamGate.snapshot().queued === VISUAL_STREAM_LIMITS.maxQueuedRequests,@@ -449,6 +456,106 @@ describe('grok visual stream-gate handoff', () => {await response})+ describe('visual per-host share', () => {+ function invalidTicketBody(maxBodyBytes: number): string {+ const body = JSON.stringify({+ executionTicket: 'invalid-ticket',+ requestHash: 'a'.repeat(64),+ request: completionRequest('grok-completion-request.v2', 'x'.repeat(maxBodyBytes)),+ })+ expect(Buffer.byteLength(body)).toBeGreaterThan(maxBodyBytes)+ return body+ }++ it('refuses a platform token whose sub is not a non-empty string with 401', async () => {+ const port = listen(createProxyApps(config()))+ const claims = {+ hostRefs: ['research-host'],+ workflowControlScopes: ['llm:grok:execute'],+ scope: 'workflow:approval:request',+ }+ for (const bearerToken of [+ sign(claims, 'workflow-approvals'),+ sign({ ...claims, sub: '' }, 'workflow-approvals'),+ sign({ ...claims, sub: 403 }, 'workflow-approvals'),+ ]) {+ const res = await postBody(port, bearerToken, JSON.stringify({ probe: 1 }))+ expect(res.status).toBe(401)+ expect(await res.json()).toEqual({ error: 'Unauthorized' })+ }+ // Liveness witness: none of those requests reached body admission.+ expect(visualStreamGate.snapshot()).toEqual({ running: 0, queued: 0 })+ })++ it('admits at most two visual entries per principal while another principal proceeds', async () => {+ expect(VISUAL_PER_HOST_MAX_ADMITTED).toBe(2)+ const maxBodyBytes =+ Buffer.byteLength(completionPayload('grok-completion-request.v1').body) + 512+ const hang = hangStream()+ hangs.push(hang)+ const acquire = vi.spyOn(visualStreamGate, 'acquire')+ const warn = vi.spyOn(logger, 'warn')+ const port = listen(+ createProxyApps(config({ maxBodyBytes, maxVisualBodyBytes: 1024 * 1024 }), {+ streamCompletion: hang.impl,+ })+ )++ // Host A's first entry runs and its second queues; both count against+ // the same principal (sub plus hostRefs).+ const first = postCompletion(port, 'grok-completion-request.v2', 'x'.repeat(maxBodyBytes))+ await waitFor(+ () => visualStreamGate.snapshot().running === 1,+ 'host A did not take the running visual slot'+ )+ const second = postBody(port, platformToken(), invalidTicketBody(maxBodyBytes))+ await waitFor(+ () => visualStreamGate.snapshot().queued === 1,+ 'host A did not take a queued visual entry'+ )+ expect(acquire).toHaveBeenCalledTimes(2)++ const third = await postBody(port, platformToken(), invalidTicketBody(maxBodyBytes))+ expect(third.status).toBe(503)+ expect(await third.json()).toEqual({ error: 'provider_unavailable' })+ expect(warn).toHaveBeenCalledWith(+ expect.objectContaining({+ event: 'grok_proxy_admission_refused',+ reason: 'visual_host_share',+ }),+ 'admission refused'+ )+ // The share refusal happens before the global gate is touched.+ expect(acquire).toHaveBeenCalledTimes(2)++ const otherHost = postBody(+ port,+ platformToken(['other-host']),+ invalidTicketBody(maxBodyBytes)+ )+ await waitFor(+ () => visualStreamGate.snapshot().queued === 2,+ 'host B was not admitted while host A held its full share'+ )++ hang.release()+ await first+ const secondRes = await second+ expect(secondRes.status).toBe(403)+ expect(await secondRes.json()).toEqual({ error: 'ticket_invalid' })+ const otherRes = await otherHost+ expect(otherRes.status).toBe(403)+ expect(await otherRes.json()).toEqual({ error: 'ticket_invalid' })+ await waitFor(+ () => visualStreamGate.snapshot().running === 0 && visualStreamGate.snapshot().queued === 0,+ 'the visual gate did not drain'+ )+ // Host A's share was released with its entries, so a fresh A request is+ // admitted again and reaches the ticket check.+ await expectNextVisualAdmitted(port, maxBodyBytes)+ })+ })+describe('visual slot lifetime', () => {function smallCaps(): number {return Buffer.byteLength(completionPayload('grok-completion-request.v1').body) + 512@@ -456,8 +563,7 @@ describe('grok visual stream-gate handoff', () => {async function expectGateEmpty(): Promise<void> {await waitFor(- () =>- visualStreamGate.snapshot().running === 0 && visualStreamGate.snapshot().queued === 0,+ () => visualStreamGate.snapshot().running === 0 && visualStreamGate.snapshot().queued === 0,'the visual slot was not released')expect(visualStreamGate.snapshot()).toEqual({ running: 0, queued: 0 })@@ -466,7 +572,9 @@ describe('grok visual stream-gate handoff', () => {it('answers 400 to a deeply nested visual body and frees the slot', async () => {const maxBodyBytes = smallCaps()const acquire = vi.spyOn(visualStreamGate, 'acquire')- const port = listen(createProxyApps(config({ maxBodyBytes, maxVisualBodyBytes: 1024 * 1024 })))+ const port = listen(+ createProxyApps(config({ maxBodyBytes, maxVisualBodyBytes: 1024 * 1024 }))+ )// JSON.parse accepts this depth; JSON.stringify and the contract's byte// measurement throw RangeError on it.const depth = 20_000@@ -487,7 +595,9 @@ describe('grok visual stream-gate handoff', () => {it('frees the slot when the handler throws before releasing it', async () => {const maxBodyBytes = smallCaps()const acquire = vi.spyOn(visualStreamGate, 'acquire')- const port = listen(createProxyApps(config({ maxBodyBytes, maxVisualBodyBytes: 1024 * 1024 })))+ const port = listen(+ createProxyApps(config({ maxBodyBytes, maxVisualBodyBytes: 1024 * 1024 }))+ )const body = JSON.stringify({executionTicket: 'invalid-ticket',requestHash: 'a'.repeat(64),@@ -602,9 +712,7 @@ describe('grok visual stream-gate handoff', () => {const maxBodyBytes = smallCaps()const acquire = vi.spyOn(visualStreamGate, 'acquire')const port = listen(- createProxyApps(- config({ maxBodyBytes, maxVisualBodyBytes: 1024 * 1024, ...row.overrides })- )+ createProxyApps(config({ maxBodyBytes, maxVisualBodyBytes: 1024 * 1024, ...row.overrides })))const { token, body } = row.body(maxBodyBytes)expect(Buffer.byteLength(body)).toBeGreaterThan(maxBodyBytes)@@ -643,11 +751,17 @@ describe('grok visual stream-gate handoff', () => {body,signal: abort.signal,}).catch((err: unknown) => err)- await waitFor(() => visualStreamGate.snapshot().queued === 1, 'the second request did not queue')+ await waitFor(+ () => visualStreamGate.snapshot().queued === 1,+ 'the second request did not queue'+ )const warn = vi.spyOn(logger, 'warn')abort.abort()expect(await waiting).toBeInstanceOf(Error)- await waitFor(() => visualStreamGate.snapshot().queued === 0, 'the aborted waiter kept its place')+ await waitFor(+ () => visualStreamGate.snapshot().queued === 0,+ 'the aborted waiter kept its place'+ )expect(acquire).toHaveBeenCalledTimes(2)// Witness: the waiter's acquire ended on the client's abort, so the// catch around it ran. A departed client is not gate saturation and@@ -702,7 +816,10 @@ describe('grok visual stream-gate handoff', () => {const code = (err as NodeJS.ErrnoException).codeif (code !== 'ECONNRESET' && code !== 'EPIPE') throw err})- await waitFor(() => visualStreamGate.snapshot().queued === 1, 'the large waiter did not queue')+ await waitFor(+ () => visualStreamGate.snapshot().queued === 1,+ 'the large waiter did not queue'+ )const closed = new Promise<void>(resolve => waiter.once('close', () => resolve()))waiter.destroy()await closed