Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
fix(control-api): admit text authorizes without the retained-body slot
Alfredo Lopez62d6ca85f497
diff --git a/control-api/package-lock.json b/control-api/package-lock.jsonindex 844a7a064..6792f55e7 100644--- a/control-api/package-lock.json+++ b/control-api/package-lock.json@@ -1,12 +1,12 @@{"name": "clerum-control-api",- "version": "0.1.715",+ "version": "0.1.716","lockfileVersion": 3,"requires": true,"packages": {"": {"name": "clerum-control-api",- "version": "0.1.715",+ "version": "0.1.716","dependencies": {"@clerum/display-field": "file:../packages/display-field","@clerum/grok-provider-attempt-contract": "file:../packages/grok-provider-attempt-contract",diff --git a/control-api/package.json b/control-api/package.jsonindex 5d49e9968..8ee1b9f0e 100644--- a/control-api/package.json+++ b/control-api/package.json@@ -1,6 +1,6 @@{"name": "clerum-control-api",- "version": "0.1.715",+ "version": "0.1.716","description": "Internal Clerum control plane REST API + UI for managing CRDs and secrets","main": "dist/main.js","engines": {diff --git a/control-api/src/middleware/llmProviderAttemptAdmissionLimits.ts b/control-api/src/middleware/llmProviderAttemptAdmissionLimits.tsindex 986211881..30a09fbde 100644--- a/control-api/src/middleware/llmProviderAttemptAdmissionLimits.ts+++ b/control-api/src/middleware/llmProviderAttemptAdmissionLimits.ts@@ -1,13 +1,29 @@import type { AuthorizeBodyAdmissionPolicy } from './llmProviderAttemptBodyAdmission.js'-// Experimental candidate for the authorized PR #806 cgroup calibration.-// These values are not certified for deployment. Keep this candidate local-// and unpublished until the combined authorize/GFS workload selects and-// records a count, deadlines, heap policy and non-heap margin below 768 MiB.+// Ships as an explicit uncertified ceiling; see #813. The combined+// authorize/GFS workload has not yet selected and recorded a count, deadlines,+// heap policy and non-heap margin below 768 MiB in three fresh cgroups.// The policy is process-wide; every Node worker needs its own measured budget.+//+// It governs only the retained path: bodies declared above the text authorize+// envelope, chunked bodies and unparseable lengths. Text-only authorizes are+// bounded by their own parser limit and never take a unit (see the route).+//+// maxQueued, maxPerPrincipal and queueWaitMs bound latency and fairness, not+// memory: queued requests are paused before any body read. Two waiters allow+// one same-principal overlap plus one other principal; the running+queued+// share stops one principal holding all three positions. queueWaitMs is the+// longest a healthy holder can keep the unit (its read deadline plus its work+// deadline), so a waiter is refused only when the holder outlives both.+const READ_DEADLINE_MS = 10_000+const WORK_DEADLINE_MS = 30_000+export const LLM_PROVIDER_ATTEMPT_ADMISSION_POLICY: AuthorizeBodyAdmissionPolicy = Object.freeze({maxInFlight: 1,- readDeadlineMs: 10_000,- workDeadlineMs: 30_000,+ maxQueued: 2,+ maxPerPrincipal: 2,+ queueWaitMs: READ_DEADLINE_MS + WORK_DEADLINE_MS,+ readDeadlineMs: READ_DEADLINE_MS,+ workDeadlineMs: WORK_DEADLINE_MS,closeGraceMs: 250,})diff --git a/control-api/src/middleware/llmProviderAttemptBodyAdmission.ts b/control-api/src/middleware/llmProviderAttemptBodyAdmission.tsindex a6b819f89..ab17e454e 100644--- a/control-api/src/middleware/llmProviderAttemptBodyAdmission.ts+++ b/control-api/src/middleware/llmProviderAttemptBodyAdmission.ts@@ -1,13 +1,26 @@import type { Request, RequestHandler, Response } from 'express'import { type Logger, rootLogger } from '../observability/logger.js'+import type { McpHostAccessClaims } from '../utils/auth/mcpHostJwtToken.js'export type AuthorizeBodyAdmissionPolicy = Readonly<{maxInFlight: number+ maxQueued: number+ maxPerPrincipal: number+ queueWaitMs: numberreadDeadlineMs: numberworkDeadlineMs: numbercloseGraceMs: number}>+// The route supplies these claims only after requireMcpHostJwt succeeds.+// Neither body fields nor unverified headers supply admission identity.+export type AuthorizeBodyAdmissionPrincipal = Readonly<+ Pick<McpHostAccessClaims, 'sub' | 'hostRefs'>+>++type AdmissionWaiter = { grant: () => void }+type AdmissionRefusal = 'principal_share' | 'queue_full' | 'queue_wait'+export class AuthorizeWorkInterrupted extends Error {constructor(readonly code: 'authorize_timeout' | 'authorize_aborted') {super(code)@@ -29,59 +42,122 @@ const log = rootLogger.child({ module: LOG_MODULE })* Owns one full permitted authorize body from before parsing until both the* parser callback and the signal-aware authorize work have unwound.*- * The count is deliberately not based on Content-Length: chunked bodies remain- * accepted, framing is enforced by Node/body-parser, and every admitted- * request is charged the same full retained-body unit.+ * The route decides which requests are charged (`run`) and which are not+ * (`runUncharged`). Every charged request costs the same full retained-body+ * unit whatever its Content-Length, so chunked bodies stay accepted and+ * framing stays with Node/body-parser. Both paths keep the read deadline, the+ * work deadline and disconnect cancellation.*/export class AuthorizeBodyAdmission {private inFlight = 0+ private readonly waiters: AdmissionWaiter[] = []+ private readonly principalUse = new Map<string, number>()+ private draining = falseprivate readonly policy: AuthorizeBodyAdmissionPolicyconstructor(policy: AuthorizeBodyAdmissionPolicy) {if (!Number.isSafeInteger(policy.maxInFlight) || policy.maxInFlight <= 0) {throw new TypeError('maxInFlight must be a positive safe integer')}- for (const value of [policy.readDeadlineMs, policy.workDeadlineMs, policy.closeGraceMs]) {+ if (!Number.isSafeInteger(policy.maxQueued) || policy.maxQueued < 0) {+ throw new TypeError('maxQueued must be a non-negative safe integer')+ }+ if (!Number.isSafeInteger(policy.maxPerPrincipal) || policy.maxPerPrincipal <= 0) {+ throw new TypeError('maxPerPrincipal must be a positive safe integer')+ }+ for (const value of [+ policy.queueWaitMs,+ policy.readDeadlineMs,+ policy.workDeadlineMs,+ policy.closeGraceMs,+ ]) {if (!Number.isSafeInteger(value) || value <= 0 || value > MAX_TIMER_DELAY_MS) {throw new TypeError('authorize timers must be positive signed-32-bit integers')}}this.policy = Object.freeze({maxInFlight: policy.maxInFlight,+ maxQueued: policy.maxQueued,+ maxPerPrincipal: policy.maxPerPrincipal,+ queueWaitMs: policy.queueWaitMs,readDeadlineMs: policy.readDeadlineMs,workDeadlineMs: policy.workDeadlineMs,closeGraceMs: policy.closeGraceMs,})}- tryAcquire(): (() => void) | null {- if (this.inFlight >= this.policy.maxInFlight) return null+ tryAcquire(principal: AuthorizeBodyAdmissionPrincipal): (() => void) | null {+ const key = principalKey(principal)+ // Reserve released units for existing waiters synchronously. An arrival+ // cannot jump the FIFO while a granted waiter is awaiting its continuation.+ if (+ this.waiters.length > 0 ||+ this.inFlight >= this.policy.maxInFlight ||+ (this.principalUse.get(key) ?? 0) >= this.policy.maxPerPrincipal+ )+ return null+ this.retainPrincipal(key)this.inFlight += 1+ return this.releaseOwner(key)+ }++ private releaseOwner(key: string): () => void {let held = truereturn () => {if (!held) returnheld = falsethis.inFlight -= 1+ this.releasePrincipal(key)+ this.drainWaiters()}}- snapshot(): Readonly<{ inFlight: number }> {- return { inFlight: this.inFlight }+ snapshot(): Readonly<{ inFlight: number; queued: number; principals: number }> {+ return {+ inFlight: this.inFlight,+ queued: this.waiters.length,+ principals: this.principalUse.size,+ }}async run(+ req: Request,+ res: Response,+ parser: RequestHandler,+ work: (signal: AbortSignal) => Promise<void>,+ principal: AuthorizeBodyAdmissionPrincipal+ ): Promise<void> {+ const release = await this.waitForBodyOwner(req, res, principal)+ if (!release) return+ await this.runOwned(req, res, parser, work, release)+ }++ // For bodies the route has bounded below the retained-body unit. They take+ // no unit and never queue; the parser's own limit bounds what they retain.+ async runUncharged(req: Request,res: Response,parser: RequestHandler,work: (signal: AbortSignal) => Promise<void>): Promise<void> {- const release = this.tryAcquire()- if (!release) {- this.refuseBeforeRead(req, res)- return- }+ if (cannotStartBodyRead(req, res)) return+ await this.runOwned(req, res, parser, work, () => {})+ }+ private async runOwned(+ req: Request,+ res: Response,+ parser: RequestHandler,+ work: (signal: AbortSignal) => Promise<void>,+ release: () => void+ ): Promise<void> {try {+ // A waiter may disconnect after a grant but before this continuation.+ // It must unwind the granted unit without ever installing a body reader.+ if (cannotStartBodyRead(req, res)) {+ if (!req.destroyed) req.destroy()+ return+ }await parseToCompletion(req, res, parser, this.policy)if (req.aborted || res.destroyed || res.writableEnded) return@@ -114,7 +190,109 @@ export class AuthorizeBodyAdmission {}}- private refuseBeforeRead(req: Request, res: Response): void {+ private async waitForBodyOwner(+ req: Request,+ res: Response,+ principal: AuthorizeBodyAdmissionPrincipal+ ): Promise<(() => void) | null> {+ if (cannotStartBodyRead(req, res)) return null+ const release = this.tryAcquire(principal)+ if (release) return release++ const key = principalKey(principal)+ if ((this.principalUse.get(key) ?? 0) >= this.policy.maxPerPrincipal) {+ this.refuseBeforeRead(req, res, 'principal_share')+ return null+ }+ if (this.waiters.length >= this.policy.maxQueued) {+ this.refuseBeforeRead(req, res, 'queue_full')+ return null+ }++ req.pause()+ this.retainPrincipal(key)+ const deadline = performance.now() + this.policy.queueWaitMs+ return new Promise((resolve, reject) => {+ let pending = true+ let waitTimer: ReturnType<typeof setTimeout> | undefined+ const finishWaiting = (): boolean => {+ if (!pending) return false+ pending = false+ if (waitTimer !== undefined) clearTimeout(waitTimer)+ req.off('aborted', abortWaiting)+ res.off('close', closeWaiting)+ const index = this.waiters.indexOf(waiter)+ if (index !== -1) this.waiters.splice(index, 1)+ return true+ }+ const cancelWaiting = (reason: 'aborted' | 'queue_wait'): void => {+ if (!finishWaiting()) return+ this.releasePrincipal(key)+ try {+ if (reason === 'queue_wait') this.refuseBeforeRead(req, res, reason)+ else if (!req.destroyed) req.destroy()+ resolve(null)+ } catch (err) {+ reject(err)+ } finally {+ this.drainWaiters()+ }+ }+ const abortWaiting = (): void => cancelWaiting('aborted')+ const closeWaiting = (): void => {+ if (!res.writableFinished) abortWaiting()+ }+ const waiter: AdmissionWaiter = {+ grant: () => {+ if (!pending) return+ if (cannotStartBodyRead(req, res)) {+ abortWaiting()+ return+ }+ // Timers can run late under load. A release cannot revive an+ // already expired waiter before its timer callback is delivered.+ if (performance.now() >= deadline) {+ cancelWaiting('queue_wait')+ return+ }+ if (!finishWaiting()) return+ this.inFlight += 1+ // The principal already owns this queued share: transfer it to+ // running rather than charging twice or briefly returning it.+ resolve(this.releaseOwner(key))+ },+ }+ this.waiters.push(waiter)+ req.once('aborted', abortWaiting)+ res.once('close', closeWaiting)+ waitTimer = setTimeout(() => cancelWaiting('queue_wait'), this.policy.queueWaitMs)+ if (cannotStartBodyRead(req, res)) abortWaiting()+ })+ }++ private retainPrincipal(key: string): void {+ this.principalUse.set(key, (this.principalUse.get(key) ?? 0) + 1)+ }++ private releasePrincipal(key: string): void {+ const remaining = (this.principalUse.get(key) ?? 0) - 1+ if (remaining === 0) this.principalUse.delete(key)+ else this.principalUse.set(key, remaining)+ }++ private drainWaiters(): void {+ if (this.draining) return+ this.draining = true+ try {+ while (this.inFlight < this.policy.maxInFlight && this.waiters.length > 0) {+ this.waiters[0].grant()+ }+ } finally {+ this.draining = false+ }+ }++ private refuseBeforeRead(req: Request, res: Response, refusal: AdmissionRefusal): void {const inFlight = this.inFlightreq.pause()let stopped = false@@ -137,8 +315,13 @@ export class AuthorizeBodyAdmission {{event: 'llm_provider_attempt_admission_refused',reason: 'authorize_capacity_exceeded',+ refusal,inFlight,maxInFlight: this.policy.maxInFlight,+ queued: this.waiters.length,+ maxQueued: this.policy.maxQueued,+ maxPerPrincipal: this.policy.maxPerPrincipal,+ queueWaitMs: this.policy.queueWaitMs,},'authorize admission refused before body read')@@ -151,6 +334,17 @@ export class AuthorizeBodyAdmission {}}+function principalKey(principal: AuthorizeBodyAdmissionPrincipal): string {+ return JSON.stringify([principal.sub, [...new Set(principal.hostRefs)].sort()])+}++// A request stream auto-destroys once its body has been read to the end, so+// `destroyed` means a disconnect only while the body is still unread. An+// already-read body goes to the parser, which skips it.+function cannotStartBodyRead(req: Request, res: Response): boolean {+ return req.aborted || (req.destroyed && !req.readableEnded) || res.destroyed || res.writableEnded+}+function removeParserBody(err: unknown): void {if (err && typeof err === 'object' && 'body' in err) {delete (err as ParserError).body@@ -233,6 +427,9 @@ function parseToCompletion(try {parser(req, res, finish)+ // A queued IncomingMessage was explicitly paused. Resume only after+ // the admitted parser has installed its body reader, never on grant.+ if (!settled && !readStopped) req.resume()} catch (err) {finish(err)}diff --git a/control-api/src/routes/mcp-host/llmProviderAttempts.routes.ts b/control-api/src/routes/mcp-host/llmProviderAttempts.routes.tsindex e25e9b1c0..e8a80500b 100644--- a/control-api/src/routes/mcp-host/llmProviderAttempts.routes.ts+++ b/control-api/src/routes/mcp-host/llmProviderAttempts.routes.ts@@ -1,10 +1,13 @@import express, { type NextFunction, type Request, type Response, Router } from 'express'+import type { IncomingHttpHeaders } from 'node:http'import {BODY_STRUCTURE_LIMITS as GROK_BODY_STRUCTURE_LIMITS,+ ENVELOPE_ALLOWANCE_BYTES as GROK_ENVELOPE_ALLOWANCE_BYTES,LIMITS as GROK_LIMITS,} from '@clerum/grok-provider-attempt-contract'import {BODY_STRUCTURE_LIMITS,+ ENVELOPE_ALLOWANCE_BYTES,LIMITS,createBodyStructureVerify,} from '@clerum/llm-provider-attempt-contract'@@ -164,36 +167,69 @@ export async function resolveHostAssignedAssignment(}}+// The text authorize envelope: the non-image request budget plus the envelope+// allowance, the same cap each contract's structural scan uses.+export const AUTHORIZE_TEXT_BODY_BYTES = Math.max(+ LIMITS.maxRequestBodyBytes + ENVELOPE_ALLOWANCE_BYTES,+ GROK_LIMITS.maxRequestBodyBytes + GROK_ENVELOPE_ALLOWANCE_BYTES+)++export type AuthorizeBudget = 'ordinary' | 'retained'++// Node frames a body at exactly its Content-Length, and the ordinary parser+// refuses a declared length above its limit before reading, so a declared+// length at or below the text envelope bounds what the request can retain.+// Only those requests skip the retained-body unit. Chunked bodies and lengths+// that are not plain digits take the retained path, so a framing the bound+// cannot read fails closed. A request with neither header has no body.+export function selectAuthorizeBudget(headers: IncomingHttpHeaders): AuthorizeBudget {+ if (headers['transfer-encoding'] !== undefined) return 'retained'+ const declared = headers['content-length']+ if (declared === undefined) return 'ordinary'+ if (!/^\d+$/.test(declared)) return 'retained'+ return Number(declared) <= AUTHORIZE_TEXT_BODY_BYTES ? 'ordinary' : 'retained'+}+export function createMcpHostLlmProviderAttemptRoutes(gateway: K8sGateway): Router {const router = Router()// Shared by both providers: preserve the larger visual envelope (Codex// #660, Grok #784) and each authorizer's provider-specific limit. Refuse// encoded bodies and scan raw structure before JSON.parse allocates objects.- const authorizeBodyParser = express.json({+ const retainedBodyParser = express.json({limit: Math.max(LIMITS.maxVisualRequestBodyBytes, GROK_LIMITS.maxVisualRequestBodyBytes),inflate: false,verify: verifyBodyStructure,})+ const ordinaryBodyParser = express.json({+ limit: AUTHORIZE_TEXT_BODY_BYTES,+ inflate: false,+ verify: verifyBodyStructure,+ })router.post('/mcp-host/llm/provider-attempts/authorize',...llmProviderAttemptAuthorizeRateLimits(),requireMcpHostJwt,asyncHandler(async (req: Request, res: Response) => {- try {- await authorizeBodyAdmission.run(req, res, authorizeBodyParser, async signal => {- signal.throwIfAborted()- const claims = req.mcpHostJwt- if (!claims) {- res.status(401).json({ error: 'Unauthorized' })- return- }- const result = await authorizeLlmProviderAttempt(claims, req.body, {- signal,- resolveAssignment: hostRef => resolveHostAssignedAssignment(gateway, hostRef, signal),- })- signal.throwIfAborted()- res.status(200).json(result)+ const claims = req.mcpHostJwt+ if (!claims) {+ res.status(401).json({ error: 'Unauthorized' })+ return+ }+ const work = async (signal: AbortSignal): Promise<void> => {+ signal.throwIfAborted()+ const result = await authorizeLlmProviderAttempt(claims, req.body, {+ signal,+ resolveAssignment: hostRef => resolveHostAssignedAssignment(gateway, hostRef, signal),})+ signal.throwIfAborted()+ res.status(200).json(result)+ }+ try {+ if (selectAuthorizeBudget(req.headers) === 'ordinary') {+ await authorizeBodyAdmission.runUncharged(req, res, ordinaryBodyParser, work)+ } else {+ await authorizeBodyAdmission.run(req, res, retainedBodyParser, work, claims)+ }} catch (err) {if (err instanceof AuthorizeWorkInterrupted) {// A closed transport only requests cancellation. run() has awaiteddiff --git a/control-api/test/llmProviderAttemptBodyAdmission.test.ts b/control-api/test/llmProviderAttemptBodyAdmission.test.tsindex 4a9e2f983..5057250ea 100644--- a/control-api/test/llmProviderAttemptBodyAdmission.test.ts+++ b/control-api/test/llmProviderAttemptBodyAdmission.test.ts@@ -3,16 +3,21 @@ import express, { type Request, type RequestHandler, type Response } from 'expreimport { EventEmitter } from 'node:events'import { type Server, createServer } from 'node:http'import net from 'node:net'+import { LLM_PROVIDER_ATTEMPT_ADMISSION_POLICY } from '../src/middleware/llmProviderAttemptAdmissionLimits.js'import {AuthorizeBodyAdmission,AuthorizeWorkInterrupted,} from '../src/middleware/llmProviderAttemptBodyAdmission.js'+const PRINCIPAL = { sub: 'unit-principal', hostRefs: ['unit-host'] }+const OTHER_PRINCIPAL = { sub: 'other-principal', hostRefs: ['other-host'] }+type FakeRequest = EventEmitter & {body?: unknownaborted: booleandestroyed: booleanpause: ReturnType<typeof vi.fn>+ resume: ReturnType<typeof vi.fn>destroy: ReturnType<typeof vi.fn>log?: FakeRequestLogger}@@ -43,6 +48,7 @@ function fakeRequest(): FakeRequest {request.aborted = falserequest.destroyed = falserequest.pause = vi.fn()+ request.resume = vi.fn()request.destroy = vi.fn(() => {request.destroyed = truerequest.aborted = true@@ -138,19 +144,40 @@ function readRawResponse(describe('AuthorizeBodyAdmission policy and idempotence', () => {it('rejects invalid counts and timer values and freezes the accepted policy', () => {- const valid = { maxInFlight: 1, readDeadlineMs: 20, workDeadlineMs: 20, closeGraceMs: 5 }+ const valid = {+ maxInFlight: 1,+ maxQueued: 0,+ maxPerPrincipal: 2,+ queueWaitMs: 100,+ readDeadlineMs: 20,+ workDeadlineMs: 20,+ closeGraceMs: 5,+ }const gate = new AuthorizeBodyAdmission(valid)expect(() => {;(gate as unknown as { policy: { maxInFlight: number } }).policy.maxInFlight = 99}).toThrow(TypeError)- const first = gate.tryAcquire()+ const first = gate.tryAcquire(PRINCIPAL)expect(first).not.toBeNull()- expect(gate.tryAcquire()).toBeNull()+ expect(gate.tryAcquire(PRINCIPAL)).toBeNull()first?.()expect(() => new AuthorizeBodyAdmission({ ...valid, maxInFlight: 0 })).toThrow(TypeError)expect(() => new AuthorizeBodyAdmission({ ...valid, maxInFlight: 1.5 })).toThrow(TypeError)- for (const key of ['readDeadlineMs', 'workDeadlineMs', 'closeGraceMs'] as const) {+ for (const value of [-1, 1.5, Number.MAX_SAFE_INTEGER + 1]) {+ expect(() => new AuthorizeBodyAdmission({ ...valid, maxQueued: value })).toThrow(TypeError)+ }+ for (const value of [0, 1.5, Number.MAX_SAFE_INTEGER + 1]) {+ expect(() => new AuthorizeBodyAdmission({ ...valid, maxPerPrincipal: value })).toThrow(+ TypeError+ )+ }+ for (const key of [+ 'queueWaitMs',+ 'readDeadlineMs',+ 'workDeadlineMs',+ 'closeGraceMs',+ ] as const) {expect(() => new AuthorizeBodyAdmission({ ...valid, [key]: 0 })).toThrow(TypeError)expect(() => new AuthorizeBodyAdmission({ ...valid, [key]: 2_147_483_648 })).toThrow(TypeError@@ -161,20 +188,23 @@ describe('AuthorizeBodyAdmission policy and idempotence', () => {it('an old release cannot free the slot occupied by a newer request', () => {const gate = new AuthorizeBodyAdmission({maxInFlight: 1,+ maxQueued: 0,+ maxPerPrincipal: 2,+ queueWaitMs: 100,readDeadlineMs: 100,workDeadlineMs: 100,closeGraceMs: 10,})- const releaseA = gate.tryAcquire()+ const releaseA = gate.tryAcquire(PRINCIPAL)expect(releaseA).not.toBeNull()- expect(gate.tryAcquire()).toBeNull()+ expect(gate.tryAcquire(PRINCIPAL)).toBeNull()releaseA?.()- const releaseB = gate.tryAcquire()+ const releaseB = gate.tryAcquire(PRINCIPAL)expect(releaseB).not.toBeNull()releaseA?.()- expect(gate.tryAcquire()).toBeNull()+ expect(gate.tryAcquire(PRINCIPAL)).toBeNull()releaseB?.()- const releaseC = gate.tryAcquire()+ const releaseC = gate.tryAcquire(PRINCIPAL)expect(releaseC).not.toBeNull()releaseC?.()})@@ -182,6 +212,9 @@ describe('AuthorizeBodyAdmission policy and idempotence', () => {it('clears retained request bodies and ignores duplicate parser callbacks', async () => {const gate = new AuthorizeBodyAdmission({maxInFlight: 1,+ maxQueued: 0,+ maxPerPrincipal: 2,+ queueWaitMs: 100,readDeadlineMs: 100,workDeadlineMs: 100,closeGraceMs: 10,@@ -195,11 +228,565 @@ describe('AuthorizeBodyAdmission policy and idempotence', () => {})const work = vi.fn(async () => undefined)- await gate.run(req as never, res as never, parser, work)+ await gate.run(req as never, res as never, parser, work, PRINCIPAL)expect(work).toHaveBeenCalledOnce()expect(req.body).toBeUndefined()- expect(gate.tryAcquire()).not.toBeNull()+ expect(gate.tryAcquire(PRINCIPAL)).not.toBeNull()+ })+})++describe('AuthorizeBodyAdmission FIFO and verified-principal share', () => {+ function queuedGate(+ overrides: Partial<ConstructorParameters<typeof AuthorizeBodyAdmission>[0]> = {}+ ) {+ return new AuthorizeBodyAdmission({+ maxInFlight: 1,+ maxQueued: 2,+ maxPerPrincipal: 2,+ queueWaitMs: 1000,+ readDeadlineMs: 1000,+ workDeadlineMs: 1000,+ closeGraceMs: 5,+ ...overrides,+ })+ }++ it('pauses a healthy overlap before parsing and runs it exactly once after the full unit is released', async () => {+ const gate = queuedGate()+ const holder = gate.tryAcquire(PRINCIPAL)+ const req = fakeRequest()+ const res = fakeResponse()+ const parse = vi.fn<RequestHandler>((request, _response, next) => {+ request.body = { queued: true }+ next()+ })+ const work = vi.fn(async () => undefined)+ const waiting = gate.run(req as never, res as never, parse, work, OTHER_PRINCIPAL)++ expect(gate.snapshot()).toEqual({ inFlight: 1, queued: 1, principals: 2 })+ expect(req.pause).toHaveBeenCalledOnce()+ expect(req.resume).not.toHaveBeenCalled()+ expect(req.body).toBeUndefined()+ expect(parse).not.toHaveBeenCalled()+ expect(work).not.toHaveBeenCalled()+ expect(res.status).not.toHaveBeenCalled()++ holder?.()+ // The grant is charged before the continuation can install its parser.+ expect(gate.snapshot()).toEqual({ inFlight: 1, queued: 0, principals: 1 })+ expect(gate.tryAcquire(PRINCIPAL)).toBeNull()+ await waiting+ expect(parse).toHaveBeenCalledOnce()+ expect(work).toHaveBeenCalledOnce()+ expect(req.body).toBeUndefined()+ expect(gate.snapshot()).toEqual({ inFlight: 0, queued: 0, principals: 0 })+ expect(req.listenerCount('aborted')).toBe(0)+ expect(res.listenerCount('close')).toBe(0)+ })++ it('never lets an arrival leapfrog the FIFO and refuses a full queue without a body reader', async () => {+ const gate = queuedGate()+ const holder = gate.tryAcquire(PRINCIPAL)+ const order: string[] = []+ const finishB = deferred()+ const finishC = deferred()+ const startedB = deferred()+ const startedC = deferred()+ const parse = vi.fn<RequestHandler>((_request, _response, next) => next())+ const reqB = fakeRequest()+ const resB = fakeResponse()+ const b = gate.run(+ reqB as never,+ resB as never,+ parse,+ async () => {+ order.push('b')+ startedB.resolve()+ await finishB.promise+ },+ OTHER_PRINCIPAL+ )+ const reqC = fakeRequest()+ const resC = fakeResponse()+ const c = gate.run(+ reqC as never,+ resC as never,+ parse,+ async () => {+ order.push('c')+ startedC.resolve()+ await finishC.promise+ },+ { sub: 'third', hostRefs: ['third-host'] }+ )+ const fullReq = fakeRequest()+ const fullRes = fakeResponse()+ const fullParser = vi.fn()+ const refused = gate.run(+ fullReq as never,+ fullRes as never,+ fullParser,+ async () => undefined,+ {+ sub: 'fourth',+ hostRefs: ['fourth-host'],+ }+ )+ expect(fullRes.body).toEqual({ error: 'authorize_capacity_exceeded' })+ expect(fullParser).not.toHaveBeenCalled()+ await refused+ fullRes.emit('finish')++ holder?.()+ const reqD = fakeRequest()+ const resD = fakeResponse()+ const d = gate.run(+ reqD as never,+ resD as never,+ parse,+ async () => {+ order.push('d')+ },+ { sub: 'fourth', hostRefs: ['fourth-host'] }+ )+ await startedB.promise+ expect(order).toEqual(['b'])+ expect(gate.snapshot()).toEqual({ inFlight: 1, queued: 2, principals: 3 })+ finishB.resolve()+ await startedC.promise+ expect(order).toEqual(['b', 'c'])+ finishC.resolve()+ await Promise.all([b, c, d])+ expect(order).toEqual(['b', 'c', 'd'])+ expect(gate.snapshot()).toEqual({ inFlight: 0, queued: 0, principals: 0 })+ })++ it('charges running plus queued to canonical verified sub and hostRefs, independent of body and provider', async () => {+ const gate = queuedGate()+ const binding = { sub: 'same-sub', hostRefs: ['host-b', 'host-a'] }+ const holder = gate.tryAcquire(binding)+ const req = fakeRequest()+ const res = fakeResponse()+ const parser = vi.fn<RequestHandler>((_request, _response, next) => next())+ const waiting = gate.run(req as never, res as never, parser, async () => undefined, {+ sub: 'same-sub',+ hostRefs: ['host-a', 'host-b', 'host-a'],+ })+ const rejectedReq = fakeRequest()+ rejectedReq.body = { hostRef: 'forged-other-host', provider: 'grok-subscription' }+ const rejectedRes = fakeResponse()+ const rejectedParser = vi.fn()+ const rejected = gate.run(+ rejectedReq as never,+ rejectedRes as never,+ rejectedParser,+ async () => undefined,+ { sub: 'same-sub', hostRefs: ['host-b', 'host-a'] }+ )+ expect(rejectedRes.body).toEqual({ error: 'authorize_capacity_exceeded' })+ expect(rejectedParser).not.toHaveBeenCalled()+ await rejected+ rejectedRes.emit('finish')++ // A different signed subject has a distinct share despite identical refs.+ const distinctReq = fakeRequest()+ const distinctRes = fakeResponse()+ const distinct = gate.run(+ distinctReq as never,+ distinctRes as never,+ parser,+ async () => undefined,+ { sub: 'different-sub', hostRefs: ['host-a', 'host-b'] }+ )+ expect(gate.snapshot()).toEqual({ inFlight: 1, queued: 2, principals: 2 })+ holder?.()+ await Promise.all([waiting, distinct])+ const recovered = gate.tryAcquire(binding)+ expect(recovered).not.toBeNull()+ recovered?.()+ expect(gate.snapshot()).toEqual({ inFlight: 0, queued: 0, principals: 0 })+ })++ it.each(['aborted', 'close'] as const)(+ 'removes a queued %s once, cleans its wait hooks and serves the next request',+ async event => {+ vi.useFakeTimers()+ try {+ const gate = queuedGate()+ const holder = gate.tryAcquire(PRINCIPAL)+ const req = fakeRequest()+ const res = fakeResponse()+ const parser = vi.fn<RequestHandler>((_request, _response, next) => next())+ const work = vi.fn(async () => undefined)+ const aborted = gate.run(req as never, res as never, parser, work, OTHER_PRINCIPAL)+ const emitter = event === 'aborted' ? req : res+ emitter.emit(event)+ emitter.emit(event)+ await aborted++ expect(req.destroy).toHaveBeenCalledOnce()+ expect(parser).not.toHaveBeenCalled()+ expect(work).not.toHaveBeenCalled()+ expect(res.status).not.toHaveBeenCalled()+ expect(req.listenerCount('aborted')).toBe(0)+ expect(res.listenerCount('close')).toBe(0)+ expect(vi.getTimerCount()).toBe(0)+ expect(gate.snapshot()).toEqual({ inFlight: 1, queued: 0, principals: 1 })++ const next = gate.run(+ fakeRequest() as never,+ fakeResponse() as never,+ parser,+ work,+ OTHER_PRINCIPAL+ )+ holder?.()+ await next+ expect(parser).toHaveBeenCalledOnce()+ expect(work).toHaveBeenCalledOnce()+ expect(gate.snapshot()).toEqual({ inFlight: 0, queued: 0, principals: 0 })+ } finally {+ vi.useRealTimers()+ }+ }+ )++ it('expires a queued request with a finite unread-body backstop and ignores the late expiry callback after recovery', async () => {+ vi.useFakeTimers()+ const clock = vi.spyOn(globalThis, 'setTimeout')+ try {+ const gate = queuedGate({ queueWaitMs: 20 })+ const holder = gate.tryAcquire(PRINCIPAL)+ const req = fakeRequest()+ const res = fakeResponse()+ const parser = vi.fn<RequestHandler>((_request, _response, next) => next())+ const work = vi.fn(async () => undefined)+ const expired = gate.run(req as never, res as never, parser, work, OTHER_PRINCIPAL)+ const expiry = clock.mock.calls.find(([, delay]) => delay === 20)?.[0]+ expect(expiry).toBeTypeOf('function')+ await vi.advanceTimersByTimeAsync(20)+ await expired+ expect(res.body).toEqual({ error: 'authorize_capacity_exceeded' })+ expect(parser).not.toHaveBeenCalled()+ expect(work).not.toHaveBeenCalled()+ expect(req.destroy).not.toHaveBeenCalled()+ expect(req.listenerCount('aborted')).toBe(0)+ expect(gate.snapshot()).toEqual({ inFlight: 1, queued: 0, principals: 1 })+ await vi.advanceTimersByTimeAsync(5)+ expect(req.destroy).toHaveBeenCalledOnce()+ expect(res.listenerCount('finish')).toBe(0)+ expect(res.listenerCount('close')).toBe(0)+ expect(vi.getTimerCount()).toBe(0)++ const next = gate.run(+ fakeRequest() as never,+ fakeResponse() as never,+ parser,+ work,+ OTHER_PRINCIPAL+ )+ holder?.()+ await next+ if (typeof expiry !== 'function') throw new Error('queue expiry callback was not installed')+ expiry()+ res.emit('close')+ req.emit('aborted')+ expect(req.destroy).toHaveBeenCalledOnce()+ expect(work).toHaveBeenCalledOnce()+ expect(gate.snapshot()).toEqual({ inFlight: 0, queued: 0, principals: 0 })+ expect(vi.getTimerCount()).toBe(0)+ } finally {+ clock.mockRestore()+ vi.useRealTimers()+ }+ })++ it('does not revive an expired FIFO head when its timer is delayed until after a release', async () => {+ vi.useFakeTimers()+ try {+ const gate = queuedGate({ queueWaitMs: 20 })+ const holder = gate.tryAcquire(PRINCIPAL)+ const req = fakeRequest()+ const res = fakeResponse()+ const parser = vi.fn<RequestHandler>((_request, _response, next) => next())+ const waiting = gate.run(+ req as never,+ res as never,+ parser,+ async () => undefined,+ OTHER_PRINCIPAL+ )+ const monotonic = vi+ .spyOn(globalThis.performance, 'now')+ .mockReturnValue(Number.MAX_SAFE_INTEGER)+ try {+ holder?.()+ await waiting+ expect(res.body).toEqual({ error: 'authorize_capacity_exceeded' })+ expect(parser).not.toHaveBeenCalled()+ expect(gate.snapshot()).toEqual({ inFlight: 0, queued: 0, principals: 0 })+ res.emit('finish')+ expect(vi.getTimerCount()).toBe(0)+ } finally {+ monotonic.mockRestore()+ }+ } finally {+ vi.useRealTimers()+ }+ })++ it('cancels a grant-to-parser disconnect without starting a reader or leaking a principal share', async () => {+ const gate = queuedGate()+ const holder = gate.tryAcquire(PRINCIPAL)+ const req = fakeRequest()+ const res = fakeResponse()+ const parser = vi.fn()+ const work = vi.fn(async () => undefined)+ const waiting = gate.run(req as never, res as never, parser, work, OTHER_PRINCIPAL)+ holder?.()+ res.destroyed = true+ res.emit('close')+ await waiting+ expect(parser).not.toHaveBeenCalled()+ expect(work).not.toHaveBeenCalled()+ expect(req.resume).not.toHaveBeenCalled()+ expect(req.destroy).toHaveBeenCalledOnce()+ expect(gate.snapshot()).toEqual({ inFlight: 0, queued: 0, principals: 0 })+ })++ it('keeps a queued reader paused after parser transport close until the actual parser callback unwinds', async () => {+ const gate = queuedGate()+ const req = fakeRequest()+ const res = fakeResponse()+ const parserStarted = deferred()+ let finishParser!: (error?: unknown) => void+ const parser: RequestHandler = (request, _response, next) => {+ request.body = { retainedByParser: true }+ finishParser = next+ parserStarted.resolve()+ }+ const parserError = Object.assign(new Error('closed parser'), { type: 'request.aborted' })+ const activeWork = vi.fn(async () => undefined)+ const active = gate+ .run(req as never, res as never, parser, activeWork, PRINCIPAL)+ .catch(error => error)+ await parserStarted.promise+ const queuedReq = fakeRequest()+ const queuedRes = fakeResponse()+ const queuedParser = vi.fn<RequestHandler>((_request, _response, next) => next())+ const queuedWork = vi.fn(async () => undefined)+ const queued = gate.run(+ queuedReq as never,+ queuedRes as never,+ queuedParser,+ queuedWork,+ OTHER_PRINCIPAL+ )+ res.emit('close')+ expect(req.destroy).toHaveBeenCalledOnce()+ expect(req.body).toEqual({ retainedByParser: true })+ expect(gate.snapshot()).toEqual({ inFlight: 1, queued: 1, principals: 2 })+ expect(queuedParser).not.toHaveBeenCalled()+ expect(queuedReq.resume).not.toHaveBeenCalled()+ finishParser(parserError)+ expect(await active).toBe(parserError)+ await queued+ finishParser(new Error('late duplicate callback'))+ expect(activeWork).not.toHaveBeenCalled()+ expect(req.body).toBeUndefined()+ expect(queuedParser).toHaveBeenCalledOnce()+ expect(queuedWork).toHaveBeenCalledOnce()+ expect(req.listenerCount('aborted')).toBe(0)+ expect(res.listenerCount('close')).toBe(0)+ expect(gate.snapshot()).toEqual({ inFlight: 0, queued: 0, principals: 0 })+ })++ it('keeps queued readers paused through active abort and only grants after actual work cleanup', async () => {+ const gate = queuedGate()+ const req = fakeRequest()+ const res = fakeResponse()+ const parse = vi.fn<RequestHandler>((_request, _response, next) => next())+ const activeStarted = deferred()+ const abortObserved = deferred()+ const permitCleanup = deferred()+ const active = gate+ .run(+ req as never,+ res as never,+ parse,+ async signal => {+ activeStarted.resolve()+ await new Promise<void>(resolve =>+ signal.addEventListener(+ 'abort',+ () => {+ abortObserved.resolve()+ resolve()+ },+ { once: true }+ )+ )+ await permitCleanup.promise+ signal.throwIfAborted()+ },+ PRINCIPAL+ )+ .catch(error => error)+ await activeStarted.promise+ const queuedReq = fakeRequest()+ const queuedRes = fakeResponse()+ const queuedParse = vi.fn<RequestHandler>((_request, _response, next) => next())+ const queuedWork = vi.fn(async () => undefined)+ const queued = gate.run(+ queuedReq as never,+ queuedRes as never,+ queuedParse,+ queuedWork,+ OTHER_PRINCIPAL+ )+ res.emit('close')+ await abortObserved.promise+ expect(gate.snapshot()).toEqual({ inFlight: 1, queued: 1, principals: 2 })+ expect(queuedParse).not.toHaveBeenCalled()+ expect(queuedWork).not.toHaveBeenCalled()+ expect(queuedReq.resume).not.toHaveBeenCalled()+ permitCleanup.resolve()+ const interrupted = await active+ expect(interrupted).toBeInstanceOf(AuthorizeWorkInterrupted)+ expect(interrupted.code).toBe('authorize_aborted')+ await queued+ expect(queuedWork).toHaveBeenCalledOnce()+ expect(gate.snapshot()).toEqual({ inFlight: 0, queued: 0, principals: 0 })+ })+})++describe('AuthorizeBodyAdmission production queue wait', () => {+ const PRODUCTION = LLM_PROVIDER_ATTEMPT_ADMISSION_POLICY++ it('derives the queue wait from the read and work deadlines a healthy holder can use', () => {+ expect(PRODUCTION.queueWaitMs).toBe(PRODUCTION.readDeadlineMs + PRODUCTION.workDeadlineMs)+ expect(PRODUCTION.queueWaitMs).toBe(40_000)+ })++ function queueBehindHolder() {+ const gate = new AuthorizeBodyAdmission(PRODUCTION)+ const holder = gate.tryAcquire(PRINCIPAL)+ expect(holder).not.toBeNull()+ const req = fakeRequest()+ const res = fakeResponse()+ const requestLogger = fakeRequestLogger()+ req.log = requestLogger.logger+ const parser = vi.fn<RequestHandler>((request, _response, next) => {+ request.body = { queued: true }+ next()+ })+ const work = vi.fn(async () => undefined)+ const waiting = gate.run(req as never, res as never, parser, work, OTHER_PRINCIPAL)+ expect(gate.snapshot()).toEqual({ inFlight: 1, queued: 1, principals: 2 })+ return { gate, holder: holder!, req, res, requestLogger, parser, work, waiting }+ }++ it('refuses the queued request with queue_wait exactly at 40000 ms', async () => {+ vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout', 'performance'] })+ try {+ const queued = queueBehindHolder()+ vi.advanceTimersByTime(PRODUCTION.queueWaitMs - 1)+ expect(queued.gate.snapshot()).toEqual({ inFlight: 1, queued: 1, principals: 2 })+ expect(queued.res.status).not.toHaveBeenCalled()++ vi.advanceTimersByTime(1)+ await queued.waiting+ expect(queued.res.statusCode).toBe(503)+ expect(queued.res.body).toEqual({ error: 'authorize_capacity_exceeded' })+ expect(queued.requestLogger.child.warn).toHaveBeenCalledWith(+ expect.objectContaining({+ event: 'llm_provider_attempt_admission_refused',+ refusal: 'queue_wait',+ queueWaitMs: 40_000,+ }),+ 'authorize admission refused before body read'+ )+ expect(queued.parser).not.toHaveBeenCalled()+ expect(queued.work).not.toHaveBeenCalled()+ expect(queued.gate.snapshot()).toEqual({ inFlight: 1, queued: 0, principals: 1 })+ queued.holder()+ expect(queued.gate.snapshot()).toEqual({ inFlight: 0, queued: 0, principals: 0 })+ } finally {+ vi.useRealTimers()+ }+ })++ it('grants the queued request when the holder releases at 39999 ms', async () => {+ vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout', 'performance'] })+ try {+ const queued = queueBehindHolder()+ vi.advanceTimersByTime(PRODUCTION.queueWaitMs - 1)+ queued.holder()+ await queued.waiting+ expect(queued.parser).toHaveBeenCalledOnce()+ expect(queued.work).toHaveBeenCalledOnce()+ expect(queued.res.status).not.toHaveBeenCalled()+ expect(queued.requestLogger.child.warn).not.toHaveBeenCalled()+ expect(queued.gate.snapshot()).toEqual({ inFlight: 0, queued: 0, principals: 0 })+ } finally {+ vi.useRealTimers()+ }+ })+})++describe('AuthorizeBodyAdmission uncharged path', () => {+ it('runs the parser and work without taking a unit, queueing or touching principal shares', async () => {+ const gate = new AuthorizeBodyAdmission(LLM_PROVIDER_ATTEMPT_ADMISSION_POLICY)+ const holder = gate.tryAcquire(PRINCIPAL)+ const req = fakeRequest()+ const res = fakeResponse()+ const workStarted = deferred()+ const permitWork = deferred()+ const parser = vi.fn<RequestHandler>((request, _response, next) => {+ request.body = { text: true }+ next()+ })+ let bodySeenByWork: unknown+ const running = gate.runUncharged(req as never, res as never, parser, async () => {+ bodySeenByWork = req.body+ workStarted.resolve()+ await permitWork.promise+ })+ await workStarted.promise+ // The unit is held by another request, yet the uncharged request runs.+ expect(bodySeenByWork).toEqual({ text: true })+ expect(gate.snapshot()).toEqual({ inFlight: 1, queued: 0, principals: 1 })+ expect(req.pause).not.toHaveBeenCalled()+ permitWork.resolve()+ await running+ expect(req.body).toBeUndefined()+ expect(gate.snapshot()).toEqual({ inFlight: 1, queued: 0, principals: 1 })+ holder?.()+ expect(gate.snapshot()).toEqual({ inFlight: 0, queued: 0, principals: 0 })+ })++ it('keeps the work deadline on the uncharged path', async () => {+ vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout'] })+ try {+ const gate = new AuthorizeBodyAdmission(LLM_PROVIDER_ATTEMPT_ADMISSION_POLICY)+ const req = fakeRequest()+ const res = fakeResponse()+ const parser = vi.fn<RequestHandler>((_request, _response, next) => next())+ const aborted = deferred()+ const running = gate.runUncharged(req as never, res as never, parser, async signal => {+ signal.addEventListener('abort', () => aborted.resolve(), { once: true })+ await aborted.promise+ })+ const outcome = running.catch(error => error)+ await vi.advanceTimersByTimeAsync(LLM_PROVIDER_ATTEMPT_ADMISSION_POLICY.workDeadlineMs)+ const error = await outcome+ expect(error).toBeInstanceOf(AuthorizeWorkInterrupted)+ expect((error as AuthorizeWorkInterrupted).code).toBe('authorize_timeout')+ expect(parser).toHaveBeenCalledOnce()+ } finally {+ vi.useRealTimers()+ }})})@@ -241,6 +828,9 @@ describe('AuthorizeBodyAdmission parser lifecycle', () => {])('propagates the $name parser error after removing its raw body', async case_ => {const gate = new AuthorizeBodyAdmission({maxInFlight: 1,+ maxQueued: 0,+ maxPerPrincipal: 2,+ queueWaitMs: 100,readDeadlineMs: 100,workDeadlineMs: 100,closeGraceMs: 10,@@ -252,9 +842,9 @@ describe('AuthorizeBodyAdmission parser lifecycle', () => {})const work = vi.fn(async () => undefined)- await expect(gate.run(req as never, res as never, parser as RequestHandler, work)).rejects.toBe(- case_.error- )+ await expect(+ gate.run(req as never, res as never, parser as RequestHandler, work, PRINCIPAL)+ ).rejects.toBe(case_.error)expect(work).not.toHaveBeenCalled()expect(case_.error).not.toHaveProperty('body')@@ -263,12 +853,15 @@ describe('AuthorizeBodyAdmission parser lifecycle', () => {} else {expect(res.setHeader).not.toHaveBeenCalledWith('Connection', 'close')}- expect(gate.tryAcquire()).not.toBeNull()+ expect(gate.tryAcquire(PRINCIPAL)).not.toBeNull()})it('stops a stalled read through closeGrace even when response finish never arrives', async () => {const gate = new AuthorizeBodyAdmission({maxInFlight: 1,+ maxQueued: 0,+ maxPerPrincipal: 2,+ queueWaitMs: 100,readDeadlineMs: 15,workDeadlineMs: 100,closeGraceMs: 5,@@ -282,7 +875,7 @@ describe('AuthorizeBodyAdmission parser lifecycle', () => {})const work = vi.fn(async () => undefined)- const running = gate.run(req as never, res as never, parser, work)+ const running = gate.run(req as never, res as never, parser, work, PRINCIPAL)await vi.waitFor(() => expect(res.body).toEqual({ error: 'request_timeout' }))await running@@ -291,7 +884,7 @@ describe('AuthorizeBodyAdmission parser lifecycle', () => {expect(work).not.toHaveBeenCalled()expect(EventEmitter.listenerCount(res, 'close')).toBe(0)expect(EventEmitter.listenerCount(res, 'finish')).toBe(0)- expect(gate.tryAcquire()).not.toBeNull()+ expect(gate.tryAcquire(PRINCIPAL)).not.toBeNull()expect(req.log?.child).toHaveBeenCalledWith({module: 'llm-provider-attempt-body-admission',})@@ -308,11 +901,14 @@ describe('AuthorizeBodyAdmission parser lifecycle', () => {it('bounds a never-finishing capacity refusal without reading or starting work', async () => {const gate = new AuthorizeBodyAdmission({maxInFlight: 1,+ maxQueued: 0,+ maxPerPrincipal: 2,+ queueWaitMs: 100,readDeadlineMs: 100,workDeadlineMs: 100,closeGraceMs: 5,})- const holder = gate.tryAcquire()+ const holder = gate.tryAcquire(PRINCIPAL)expect(holder).not.toBeNull()const req = fakeRequest()const requestLogger = fakeRequestLogger()@@ -321,7 +917,7 @@ describe('AuthorizeBodyAdmission parser lifecycle', () => {const parser = vi.fn()const work = vi.fn(async () => undefined)- await gate.run(req as never, res as never, parser as RequestHandler, work)+ await gate.run(req as never, res as never, parser as RequestHandler, work, PRINCIPAL)await vi.waitFor(() => expect(req.destroy).toHaveBeenCalledOnce())expect(req.pause).toHaveBeenCalledOnce()@@ -331,7 +927,7 @@ describe('AuthorizeBodyAdmission parser lifecycle', () => {expect(work).not.toHaveBeenCalled()expect(EventEmitter.listenerCount(res, 'close')).toBe(0)expect(EventEmitter.listenerCount(res, 'finish')).toBe(0)- expect(gate.tryAcquire()).toBeNull()+ expect(gate.tryAcquire(PRINCIPAL)).toBeNull()expect(req.log?.child).toHaveBeenCalledWith({module: 'llm-provider-attempt-body-admission',})@@ -341,17 +937,23 @@ describe('AuthorizeBodyAdmission parser lifecycle', () => {reason: 'authorize_capacity_exceeded',inFlight: 1,maxInFlight: 1,+ maxQueued: 0,+ maxPerPrincipal: 2,+ queueWaitMs: 100,}),'authorize admission refused before body read')holder?.()- expect(gate.tryAcquire()).not.toBeNull()+ expect(gate.tryAcquire(PRINCIPAL)).not.toBeNull()})it('uses close during parsing only to stop the reader and never starts work', async () => {const gate = new AuthorizeBodyAdmission({maxInFlight: 1,+ maxQueued: 0,+ maxPerPrincipal: 2,+ queueWaitMs: 100,readDeadlineMs: 100,workDeadlineMs: 100,closeGraceMs: 10,@@ -364,13 +966,13 @@ describe('AuthorizeBodyAdmission parser lifecycle', () => {const work = vi.fn(async () => undefined)await Promise.all([- gate.run(req as never, res as never, parser, work),+ gate.run(req as never, res as never, parser, work, PRINCIPAL),Promise.resolve().then(() => res.emit('close')),])expect(req.destroy).toHaveBeenCalledOnce()expect(work).not.toHaveBeenCalled()- expect(gate.tryAcquire()).not.toBeNull()+ expect(gate.tryAcquire(PRINCIPAL)).not.toBeNull()})})@@ -378,6 +980,9 @@ describe('AuthorizeBodyAdmission work cancellation', () => {it('preserves typed authorize_timeout after real signal-aware work settles', async () => {const gate = new AuthorizeBodyAdmission({maxInFlight: 1,+ maxQueued: 0,+ maxPerPrincipal: 2,+ queueWaitMs: 100,readDeadlineMs: 100,workDeadlineMs: 15,closeGraceMs: 10,@@ -393,18 +998,21 @@ describe('AuthorizeBodyAdmission work cancellation', () => {)const error = await gate- .run(req as never, res as never, parser as RequestHandler, work)+ .run(req as never, res as never, parser as RequestHandler, work, PRINCIPAL).catch(err => err)expect(error).toBeInstanceOf(AuthorizeWorkInterrupted)expect((error as AuthorizeWorkInterrupted).code).toBe('authorize_timeout')expect(EventEmitter.listenerCount(res, 'close')).toBe(0)- expect(gate.tryAcquire()).not.toBeNull()+ expect(gate.tryAcquire(PRINCIPAL)).not.toBeNull()})it('does not free a work owner on client close until signal-aware work unwinds', async () => {const gate = new AuthorizeBodyAdmission({maxInFlight: 1,+ maxQueued: 0,+ maxPerPrincipal: 2,+ queueWaitMs: 100,readDeadlineMs: 1000,workDeadlineMs: 5000,closeGraceMs: 10,@@ -432,23 +1040,28 @@ describe('AuthorizeBodyAdmission work cancellation', () => {})const running = gate- .run(req as never, res as never, parser as RequestHandler, signal =>- work(signal).catch((err: AuthorizeWorkInterrupted) => {- workSettled.resolve(err)- throw err- })+ .run(+ req as never,+ res as never,+ parser as RequestHandler,+ signal =>+ work(signal).catch((err: AuthorizeWorkInterrupted) => {+ workSettled.resolve(err)+ throw err+ }),+ PRINCIPAL).catch(error => error)await workStarted.promiseres.emit('close')await closeObserved.promise- expect(gate.tryAcquire()).toBeNull()+ expect(gate.tryAcquire(PRINCIPAL)).toBeNull()allowWorkUnwind.resolve()const error = (await Promise.all([running, workSettled.promise]))[1]expect(error.code).toBe('authorize_aborted')expect(req.body).toBeUndefined()- expect(gate.tryAcquire()).not.toBeNull()+ expect(gate.tryAcquire(PRINCIPAL)).not.toBeNull()})})@@ -456,6 +1069,9 @@ describe('AuthorizeBodyAdmission HTTP fixture', () => {it('preserves chunked JSON and releases after work completes', async () => {const gate = new AuthorizeBodyAdmission({maxInFlight: 1,+ maxQueued: 0,+ maxPerPrincipal: 2,+ queueWaitMs: 100,readDeadlineMs: 1000,workDeadlineMs: 1000,closeGraceMs: 10,@@ -466,7 +1082,7 @@ describe('AuthorizeBodyAdmission HTTP fixture', () => {})const app = express()app.post('/test', (req, res, next) => {- void gate.run(req, res, parser, signal => work(req, res, signal)).catch(next)+ void gate.run(req, res, parser, signal => work(req, res, signal), PRINCIPAL).catch(next)})await withServer(app, async port => {@@ -484,12 +1100,15 @@ describe('AuthorizeBodyAdmission HTTP fixture', () => {})expect(work).toHaveBeenCalledOnce()- expect(gate.tryAcquire()).not.toBeNull()+ expect(gate.tryAcquire(PRINCIPAL)).not.toBeNull()})it('refuses an over-cap request before parsing and keeps a disconnected work owner charged', async () => {const gate = new AuthorizeBodyAdmission({maxInFlight: 1,+ maxQueued: 0,+ maxPerPrincipal: 2,+ queueWaitMs: 100,readDeadlineMs: 1000,workDeadlineMs: 5000,closeGraceMs: 10,@@ -521,11 +1140,16 @@ describe('AuthorizeBodyAdmission HTTP fixture', () => {const app = express()app.post('/test', (req, res, next) => {void gate- .run(req, res, parser, signal =>- work(req, res, signal).catch((err: AuthorizeWorkInterrupted) => {- workSettled.resolve(err)- throw err- })+ .run(+ req,+ res,+ parser,+ signal =>+ work(req, res, signal).catch((err: AuthorizeWorkInterrupted) => {+ workSettled.resolve(err)+ throw err+ }),+ PRINCIPAL).catch(next)})@@ -584,6 +1208,9 @@ describe('AuthorizeBodyAdmission HTTP fixture', () => {it('releases only after the real body-parser callback follows a destroyed stalled request', async () => {const gate = new AuthorizeBodyAdmission({maxInFlight: 1,+ maxQueued: 0,+ maxPerPrincipal: 2,+ queueWaitMs: 100,readDeadlineMs: 5000,workDeadlineMs: 1000,closeGraceMs: 10,@@ -599,7 +1226,9 @@ describe('AuthorizeBodyAdmission HTTP fixture', () => {})const app = express()app.post('/test', (req, res, next) => {- void gate.run(req, res, wrappedParser, signal => work(req, res, signal)).catch(next)+ void gate+ .run(req, res, wrappedParser, signal => work(req, res, signal), PRINCIPAL)+ .catch(next)})await withServer(app, async port => {diff --git a/control-api/test/routes.mcp-host.llmProviderAttempts.bodyAdmission.test.ts b/control-api/test/routes.mcp-host.llmProviderAttempts.bodyAdmission.test.tsindex cccda197f..00b678e9a 100644--- a/control-api/test/routes.mcp-host.llmProviderAttempts.bodyAdmission.test.ts+++ b/control-api/test/routes.mcp-host.llmProviderAttempts.bodyAdmission.test.ts@@ -1,11 +1,16 @@-import { beforeEach, describe, expect, it, vi } from 'vitest'+import { beforeEach, describe, expect, it, onTestFinished, vi } from 'vitest'import type { Request } from 'express'import { type IncomingMessage, createServer, request as httpRequest } from 'node:http'+import net from 'node:net'import { LLM_PROVIDER_ATTEMPT_ADMISSION_POLICY } from '../src/middleware/llmProviderAttemptAdmissionLimits.js'import {AuthorizeBodyAdmission,AuthorizeWorkInterrupted,} from '../src/middleware/llmProviderAttemptBodyAdmission.js'+import {+ AUTHORIZE_TEXT_BODY_BYTES,+ selectAuthorizeBudget,+} from '../src/routes/mcp-host/llmProviderAttempts.routes.js'import * as authorizer from '../src/services/llmProviderAttemptAuthorizer.js'import * as rateLimiter from '../src/services/rateLimiterService.js'import { issueMcpHostAccessJwt } from '../src/utils/auth/mcpHostJwtToken.js'@@ -68,13 +73,29 @@ type Observation = {type Reply = { status: number; headers: IncomingMessage['headers']; body: string }-function headers(host: string, provider: 'codex' | 'grok' = 'codex') {+function textHeaders(host: string, provider: 'codex' | 'grok' = 'codex'): Record<string, string> {const issued = issueMcpHostAccessJwt('default', host, [host], {workflowControlScopes: [provider === 'codex' ? 'llm:codex:execute' : 'llm:grok:execute'],})return { Authorization: `Bearer ${issued.token}`, 'content-type': 'application/json' }}+// The retained-ownership cases compete for the retained-body unit. A small+// body with a declared length takes the uncharged ordinary path, so these+// requests are sent chunked, which selectAuthorizeBudget charges.+function headers(host: string, provider: 'codex' | 'grok' = 'codex'): Record<string, string> {+ return { ...textHeaders(host, provider), 'transfer-encoding': 'chunked' }+}++// Headers-only retained requests declare a length just above the text+// envelope, the smallest declared length that takes the unit.+function declaredRetainedHeaders(+ host: string,+ provider: 'codex' | 'grok' = 'codex'+): Record<string, string> {+ return { ...textHeaders(host, provider), 'content-length': String(AUTHORIZE_TEXT_BODY_BYTES + 1) }+}+function send(url: string, requestHeaders: Record<string, string>, body?: string) {let client!: ReturnType<typeof httpRequest>const response = new Promise<Reply>((resolve, reject) => {@@ -161,20 +182,44 @@ async function withApps(}}+// The production queue wait outlives a healthy holder (read plus work+// deadline). Cases that observe a queued request being refused shorten only+// that timer, so the refusal arrives within the fixture deadline.+function shortenQueueWait() {+ const nativeSetTimeout = globalThis.setTimeout+ return vi+ .spyOn(globalThis, 'setTimeout')+ .mockImplementation((fn, ms, ...args) =>+ nativeSetTimeout(+ fn,+ ms === LLM_PROVIDER_ATTEMPT_ADMISSION_POLICY.queueWaitMs ? 40 : ms,+ ...args+ )+ )+}+function observeHeldOwner() {const settled = deferred()+ let admission: AuthorizeBodyAdmission | undefinedconst original = AuthorizeBodyAdmission.prototype.runconst spy = vi.spyOn(AuthorizeBodyAdmission.prototype, 'run').mockImplementation(async function (this: AuthorizeBodyAdmission,...args: Parameters<AuthorizeBodyAdmission['run']>) {+ admission = thistry {return await original.apply(this, args)} finally {if (args[0].headers['x-admission-case'] === 'held') settled.resolve()}})- return { settled: settled.promise, spy }+ return {+ settled: settled.promise,+ spy,+ get admission() {+ return admission+ },+ }}beforeEach(() => {@@ -198,6 +243,7 @@ describe('createApp retained authorize-body ownership', () => {const permitUnwind = deferred()const owner = observeHeldOwner()const parse = vi.spyOn(JSON, 'parse')+ const clock = shortenQueueWait()vi.mocked(authorizer.authorizeLlmProviderAttempt).mockImplementationOnce(async (_claims, _body, deps) => {started.resolve()@@ -237,10 +283,10 @@ describe('createApp retained authorize-body ownership', () => {// The same refusal must also arrive from headers alone. An owner// that first buffers the next body would stall this request.- const headersOnly = await send(otherApp.url, {- ...headers('admission-headers-only-host'),- 'content-length': '4096',- }).response+ const headersOnly = await send(+ otherApp.url,+ declaredRetainedHeaders('admission-headers-only-host')+ ).responseexpect(headersOnly.status).toBe(503)expect(otherApp.observations[1]).toMatchObject({bodyDataSubscriptions: 0,@@ -292,11 +338,355 @@ describe('createApp retained authorize-body ownership', () => {}})} finally {+ clock.mockRestore()parse.mockRestore()owner.spy.mockRestore()}})+ for (const provider of ['codex', 'grok'] as const) {+ it(`queues a healthy overlapping chunked ${provider} request across apps before any read and executes it once`, async () => {+ const started = deferred()+ const permitUnwind = deferred()+ const owner = observeHeldOwner()+ vi.mocked(authorizer.authorizeLlmProviderAttempt).mockImplementationOnce(+ async (_claims, _body, deps) => {+ started.resolve()+ await permitUnwind.promise+ deps?.signal?.throwIfAborted()+ return SUCCESS+ }+ )+ try {+ await withApps(2, async ([firstApp, otherApp]) => {+ const first = send(+ firstApp.url,+ { ...headers(`healthy-first-${provider}-host`), 'x-admission-case': 'held' },+ '{"first":true}'+ )+ const firstOutcome = first.response.catch(error => error)+ const queuedBody = {+ hostRef: `healthy-queued-${provider}-host`,+ request: { provider: `${provider}-subscription` },+ }+ let queued: ReturnType<typeof send> | undefined+ let queuedOutcome: Promise<Reply | Error> | undefined+ try {+ await started.promise+ queued = send(otherApp.url, {+ ...headers(`healthy-queued-${provider}-host`, provider),+ 'transfer-encoding': 'chunked',+ })+ queuedOutcome = queued.response.catch(error => error)+ const encoded = JSON.stringify(queuedBody)+ queued.client.write(encoded.slice(0, 12))+ queued.client.end(encoded.slice(12))+ await vi.waitFor(() =>+ expect(owner.admission?.snapshot()).toMatchObject({ inFlight: 1, queued: 1 })+ )+ expect(otherApp.observations[0]).toMatchObject({+ bodyDataSubscriptions: 0,+ readBytesBeforeResponse: 0,+ })+ expect(authorizer.authorizeLlmProviderAttempt).toHaveBeenCalledOnce()++ permitUnwind.resolve()+ const replies = await Promise.all([firstOutcome, queuedOutcome])+ expect(replies).toEqual([+ expect.objectContaining({ status: 200 }),+ expect.objectContaining({ status: 200 }),+ ])+ expect(JSON.parse((replies[1] as Reply).body)).toEqual(SUCCESS)+ expect(otherApp.observations[0].bodyDataSubscriptions).toBe(1)+ expect(authorizer.authorizeLlmProviderAttempt).toHaveBeenCalledTimes(2)+ expect(authorizer.authorizeLlmProviderAttempt).toHaveBeenNthCalledWith(+ 2,+ expect.objectContaining({ hostRefs: [`healthy-queued-${provider}-host`] }),+ queuedBody,+ expect.objectContaining({ signal: expect.any(AbortSignal) })+ )+ expect((otherApp.observations[0].request as Request).body).toBeUndefined()+ expect(owner.admission?.snapshot()).toEqual({ inFlight: 0, queued: 0, principals: 0 })+ } finally {+ permitUnwind.resolve()+ first.client.destroy()+ queued?.client.destroy()+ await Promise.all([firstOutcome, queuedOutcome])+ }+ })+ } finally {+ owner.spy.mockRestore()+ }+ })+ }++ it('uses the verified JWT share across providers and ignores a different body host before parsing', async () => {+ const started = deferred()+ const permitUnwind = deferred()+ const owner = observeHeldOwner()+ vi.mocked(authorizer.authorizeLlmProviderAttempt).mockImplementationOnce(+ async (_claims, _body, deps) => {+ started.resolve()+ await permitUnwind.promise+ deps?.signal?.throwIfAborted()+ return SUCCESS+ }+ )+ try {+ await withApps(1, async ([app]) => {+ const first = send(+ app.url,+ { ...headers('verified-share-host'), 'x-admission-case': 'held' },+ '{}'+ )+ const firstOutcome = first.response.catch(error => error)+ let queued: ReturnType<typeof send> | undefined+ let queuedOutcome: Promise<Reply | Error> | undefined+ let rejected: ReturnType<typeof send> | undefined+ let rejectedOutcome: Promise<Reply | Error> | undefined+ try {+ await started.promise+ queued = send(+ app.url,+ headers('verified-share-host', 'grok'),+ '{"hostRef":"forged-second-host","request":{"provider":"grok-subscription"}}'+ )+ queuedOutcome = queued.response.catch(error => error)+ await vi.waitFor(() =>+ expect(owner.admission?.snapshot()).toMatchObject({+ inFlight: 1,+ queued: 1,+ principals: 1,+ })+ )+ rejected = send(+ app.url,+ headers('verified-share-host'),+ '{"hostRef":"forged-third-host"}'+ )+ rejectedOutcome = rejected.response.catch(error => error)+ await vi.waitFor(() => expect(owner.spy).toHaveBeenCalledTimes(3))+ expect(owner.admission?.snapshot()).toEqual({ inFlight: 1, queued: 1, principals: 1 })+ const refused = (await rejectedOutcome) as Reply+ expect(refused.status).toBe(503)+ expect(JSON.parse(refused.body)).toEqual({ error: 'authorize_capacity_exceeded' })+ expect(app.observations[1]).toMatchObject({+ bodyDataSubscriptions: 0,+ readBytesBeforeResponse: 0,+ })+ expect(app.observations[2]).toMatchObject({+ bodyDataSubscriptions: 0,+ readBytesBeforeResponse: 0,+ })+ expect(authorizer.authorizeLlmProviderAttempt).toHaveBeenCalledOnce()++ permitUnwind.resolve()+ expect(await firstOutcome).toMatchObject({ status: 200 })+ expect(await queuedOutcome).toMatchObject({ status: 200 })+ const recovered = await send(app.url, headers('verified-share-host'), '{}').response+ expect(recovered.status).toBe(200)+ expect(authorizer.authorizeLlmProviderAttempt).toHaveBeenCalledTimes(3)+ expect(owner.admission?.snapshot()).toEqual({ inFlight: 0, queued: 0, principals: 0 })+ } finally {+ permitUnwind.resolve()+ first.client.destroy()+ queued?.client.destroy()+ rejected?.client.destroy()+ await Promise.all([firstOutcome, queuedOutcome, rejectedOutcome])+ }+ })+ } finally {+ owner.spy.mockRestore()+ }+ })++ it('removes a queued disconnect without parsing and returns that share for a later healthy overlap', async () => {+ const started = deferred()+ const permitUnwind = deferred()+ const owner = observeHeldOwner()+ vi.mocked(authorizer.authorizeLlmProviderAttempt).mockImplementationOnce(+ async (_claims, _body, deps) => {+ started.resolve()+ await permitUnwind.promise+ deps?.signal?.throwIfAborted()+ return SUCCESS+ }+ )+ try {+ await withApps(1, async ([app]) => {+ const first = send(+ app.url,+ { ...headers('queued-disconnect-holder'), 'x-admission-case': 'held' },+ '{}'+ )+ const firstOutcome = first.response.catch(error => error)+ let cancelled: ReturnType<typeof send> | undefined+ let cancelledOutcome: Promise<Reply | Error> | undefined+ let recovered: ReturnType<typeof send> | undefined+ let recoveredOutcome: Promise<Reply | Error> | undefined+ try {+ await started.promise+ cancelled = send(app.url, headers('queued-disconnect-host'), '{"cancelled":true}')+ cancelledOutcome = cancelled.response.catch(error => error)+ await vi.waitFor(() =>+ expect(owner.admission?.snapshot()).toMatchObject({ inFlight: 1, queued: 1 })+ )+ cancelled.client.destroy()+ await cancelledOutcome+ await vi.waitFor(() =>+ expect(owner.admission?.snapshot()).toEqual({ inFlight: 1, queued: 0, principals: 1 })+ )+ expect(app.observations[1]).toMatchObject({+ bodyDataSubscriptions: 0,+ readBytesBeforeResponse: 0,+ })+ expect(app.observations[1].request.destroyed).toBe(true)+ expect(authorizer.authorizeLlmProviderAttempt).toHaveBeenCalledOnce()+ recovered = send(app.url, headers('queued-disconnect-host'), '{"recovered":true}')+ recoveredOutcome = recovered.response.catch(error => error)+ await vi.waitFor(() =>+ expect(owner.admission?.snapshot()).toMatchObject({ inFlight: 1, queued: 1 })+ )+ permitUnwind.resolve()+ expect(await firstOutcome).toMatchObject({ status: 200 })+ expect(await recoveredOutcome).toMatchObject({ status: 200 })+ expect(authorizer.authorizeLlmProviderAttempt).toHaveBeenCalledTimes(2)+ expect(owner.admission?.snapshot()).toEqual({ inFlight: 0, queued: 0, principals: 0 })+ } finally {+ permitUnwind.resolve()+ first.client.destroy()+ cancelled?.client.destroy()+ recovered?.client.destroy()+ await Promise.all([firstOutcome, cancelledOutcome, recoveredOutcome])+ }+ })+ } finally {+ owner.spy.mockRestore()+ }+ })++ it('expires queued headers without reading them and preserves terminal local saturation followed by recovery', async () => {+ const started = deferred()+ const permitUnwind = deferred()+ const owner = observeHeldOwner()+ const nativeSetTimeout = globalThis.setTimeout+ const clock = vi+ .spyOn(globalThis, 'setTimeout')+ .mockImplementation((fn, ms, ...args) =>+ nativeSetTimeout(+ fn,+ ms === LLM_PROVIDER_ATTEMPT_ADMISSION_POLICY.queueWaitMs ? 40 : ms,+ ...args+ )+ )+ vi.mocked(authorizer.authorizeLlmProviderAttempt).mockImplementationOnce(+ async (_claims, _body, deps) => {+ started.resolve()+ await permitUnwind.promise+ deps?.signal?.throwIfAborted()+ return SUCCESS+ }+ )+ try {+ await withApps(1, async ([app]) => {+ const first = send(+ app.url,+ { ...headers('queued-expiry-holder'), 'x-admission-case': 'held' },+ '{}'+ )+ const firstOutcome = first.response.catch(error => error)+ try {+ await started.promise+ const expired = await send(app.url, declaredRetainedHeaders('queued-expiry-host'))+ .response+ expect(expired.status).toBe(503)+ expect(JSON.parse(expired.body)).toEqual({ error: 'authorize_capacity_exceeded' })+ expect(expired.headers.connection).toBe('close')+ expect(app.observations[1]).toMatchObject({+ bodyDataSubscriptions: 0,+ readBytesBeforeResponse: 0,+ })+ await vi.waitFor(() => expect(app.observations[1].request.destroyed).toBe(true))+ expect(owner.admission?.snapshot()).toEqual({ inFlight: 1, queued: 0, principals: 1 })+ expect(authorizer.authorizeLlmProviderAttempt).toHaveBeenCalledOnce()+ permitUnwind.resolve()+ expect(await firstOutcome).toMatchObject({ status: 200 })+ const recovered = await send(app.url, headers('queued-expiry-host'), '{}').response+ expect(recovered.status).toBe(200)+ expect(authorizer.authorizeLlmProviderAttempt).toHaveBeenCalledTimes(2)+ } finally {+ permitUnwind.resolve()+ first.client.destroy()+ await firstOutcome+ }+ })+ } finally {+ clock.mockRestore()+ owner.spy.mockRestore()+ }+ })++ it('keeps a queued reader paused after active disconnect until physical work unwind and then completes it once', async () => {+ const started = deferred()+ const abortObserved = deferred()+ const permitUnwind = deferred()+ const owner = observeHeldOwner()+ vi.mocked(authorizer.authorizeLlmProviderAttempt).mockImplementationOnce(+ async (_claims, _body, deps) => {+ started.resolve()+ deps?.signal?.addEventListener('abort', () => abortObserved.resolve(), { once: true })+ await permitUnwind.promise+ deps?.signal?.throwIfAborted()+ return SUCCESS+ }+ )+ try {+ await withApps(2, async ([firstApp, otherApp]) => {+ const first = send(+ firstApp.url,+ { ...headers('active-disconnect-holder'), 'x-admission-case': 'held' },+ '{}'+ )+ const firstOutcome = first.response.catch(error => error)+ let queued: ReturnType<typeof send> | undefined+ let queuedOutcome: Promise<Reply | Error> | undefined+ try {+ await started.promise+ queued = send(+ otherApp.url,+ headers('active-disconnect-queued-host', 'grok'),+ '{"queued":true}'+ )+ queuedOutcome = queued.response.catch(error => error)+ await vi.waitFor(() =>+ expect(owner.admission?.snapshot()).toMatchObject({ inFlight: 1, queued: 1 })+ )+ first.client.destroy()+ await abortObserved.promise+ expect(otherApp.observations[0]).toMatchObject({+ bodyDataSubscriptions: 0,+ readBytesBeforeResponse: 0,+ })+ expect(authorizer.authorizeLlmProviderAttempt).toHaveBeenCalledOnce()+ expect(owner.admission?.snapshot()).toEqual({ inFlight: 1, queued: 1, principals: 2 })+ permitUnwind.resolve()+ await owner.settled+ expect(await queuedOutcome).toMatchObject({ status: 200 })+ expect(authorizer.authorizeLlmProviderAttempt).toHaveBeenCalledTimes(2)+ expect(firstApp.observations[0].responseWritesAfterClose).toBe(0)+ expect(owner.admission?.snapshot()).toEqual({ inFlight: 0, queued: 0, principals: 0 })+ } finally {+ permitUnwind.resolve()+ first.client.destroy()+ queued?.client.destroy()+ await Promise.all([firstOutcome, queuedOutcome])+ }+ })+ } finally {+ owner.spy.mockRestore()+ }+ })+it('runs both rate limits and JWT before taking any body owner', async () => {const acquire = vi.spyOn(AuthorizeBodyAdmission.prototype, 'tryAcquire')try {@@ -339,6 +729,8 @@ describe('createApp retained authorize-body ownership', () => {for (const provider of ['codex', 'grok'] as const) {it(`preserves chunked ${provider} JSON through the actual app and releases the body`, async () => {+ const clock = shortenQueueWait()+ onTestFinished(() => clock.mockRestore())await withApps(1, async ([app]) => {const body = JSON.stringify({request: { provider: `${provider}-subscription`, chunked: true },@@ -500,8 +892,7 @@ describe('createApp retained authorize-body ownership', () => {try {await withApps(1, async ([app]) => {const sent = send(app.url, {- ...headers('read-timeout-host'),- 'content-length': '128',+ ...declaredRetainedHeaders('read-timeout-host'),'x-admission-case': 'held',})const response = await sent.response@@ -545,3 +936,185 @@ describe('Host and Recipe assignment owner cancellation', () => {})}})++describe('createApp declared-length authorize budget', () => {+ it('pins the text envelope to each contract text budget plus its envelope allowance', () => {+ expect(AUTHORIZE_TEXT_BODY_BYTES).toBe(8_388_608 + 16_384)+ })++ it.each([+ ['no body headers', {}, 'ordinary'],+ ['zero declared length', { 'content-length': '0' }, 'ordinary'],+ ['declared length at the text envelope', { 'content-length': '8404992' }, 'ordinary'],+ ['declared length one byte above the envelope', { 'content-length': '8404993' }, 'retained'],+ ['declared visual length', { 'content-length': String(35 * 1024 * 1024) }, 'retained'],+ ['chunked body', { 'transfer-encoding': 'chunked' }, 'retained'],+ [+ 'chunked body with a small declared length',+ { 'transfer-encoding': 'chunked', 'content-length': '2' },+ 'retained',+ ],+ ['non-numeric declared length', { 'content-length': '12abc' }, 'retained'],+ ['negative declared length', { 'content-length': '-1' }, 'retained'],+ ['empty declared length', { 'content-length': '' }, 'retained'],+ ] as const)('selects the %s budget', (_name, requestHeaders, expected) => {+ expect(selectAuthorizeBudget(requestHeaders)).toBe(expected)+ })++ function observeAdmission() {+ let admission: AuthorizeBodyAdmission | undefined+ const charged = vi.fn()+ const uncharged = vi.fn()+ const run = AuthorizeBodyAdmission.prototype.run+ const runUncharged = AuthorizeBodyAdmission.prototype.runUncharged+ const runSpy = vi.spyOn(AuthorizeBodyAdmission.prototype, 'run').mockImplementation(function (+ this: AuthorizeBodyAdmission,+ ...args: Parameters<AuthorizeBodyAdmission['run']>+ ) {+ admission = this+ charged(args[0].headers['x-admission-case'])+ return run.apply(this, args)+ })+ const unchargedSpy = vi+ .spyOn(AuthorizeBodyAdmission.prototype, 'runUncharged')+ .mockImplementation(function (+ this: AuthorizeBodyAdmission,+ ...args: Parameters<AuthorizeBodyAdmission['runUncharged']>+ ) {+ admission = this+ uncharged(args[0].headers['x-admission-case'])+ return runUncharged.apply(this, args)+ })+ onTestFinished(() => {+ runSpy.mockRestore()+ unchargedSpy.mockRestore()+ })+ return {+ charged,+ uncharged,+ get admission() {+ if (!admission) throw new Error('no admission observed')+ return admission+ },+ }+ }++ it('authorizes a text request from another Host while a declared visual body stalls the unit', async () => {+ const observed = observeAdmission()+ const textBody = '{"text":"hi"}'+ let inFlightDuringText: number | undefined+ vi.mocked(authorizer.authorizeLlmProviderAttempt).mockImplementationOnce(async claims => {+ inFlightDuringText = observed.admission.snapshot().inFlight+ expect(claims.hostRefs).toEqual(['text-host'])+ return SUCCESS+ })+ await withApps(2, async ([visualApp, textApp]) => {+ // Headers only: the declared body never arrives, so this request holds+ // the retained unit inside its read deadline.+ const stalled = send(visualApp.url, {+ ...declaredRetainedHeaders('stalled-visual-host'),+ 'x-admission-case': 'stalled',+ })+ const stalledOutcome = stalled.response.catch(error => error)+ try {+ await vi.waitFor(() => expect(observed.charged).toHaveBeenCalledWith('stalled'))+ await vi.waitFor(() => expect(observed.admission.snapshot().inFlight).toBe(1))++ const text = await send(+ textApp.url,+ { ...textHeaders('text-host'), 'x-admission-case': 'text' },+ textBody+ ).response+ expect(text.status).toBe(200)+ expect(JSON.parse(text.body)).toEqual(SUCCESS)+ expect(observed.uncharged).toHaveBeenCalledWith('text')+ expect(authorizer.authorizeLlmProviderAttempt).toHaveBeenCalledOnce()+ expect(authorizer.authorizeLlmProviderAttempt).toHaveBeenCalledWith(+ expect.objectContaining({ hostRefs: ['text-host'] }),+ JSON.parse(textBody),+ expect.objectContaining({ signal: expect.any(AbortSignal) })+ )+ // The stalled visual request still holds the only unit.+ expect(inFlightDuringText).toBe(1)+ expect(observed.admission.snapshot().inFlight).toBe(1)+ } finally {+ stalled.client.destroy()+ await stalledOutcome+ }+ await vi.waitFor(() => expect(observed.admission.snapshot().inFlight).toBe(0))+ })+ })++ it('never charges a text request while its authorize work is running', async () => {+ const observed = observeAdmission()+ const workStarted = deferred()+ const permitWork = deferred()+ let snapshotDuringWork: ReturnType<AuthorizeBodyAdmission['snapshot']> | undefined+ vi.mocked(authorizer.authorizeLlmProviderAttempt).mockImplementationOnce(async () => {+ snapshotDuringWork = observed.admission.snapshot()+ workStarted.resolve()+ await permitWork.promise+ return SUCCESS+ })+ await withApps(1, async ([app]) => {+ const sent = send(app.url, textHeaders('uncharged-host'), '{"text":"uncharged"}')+ await workStarted.promise+ expect(app.observations[0].bodyDataSubscriptions).toBe(1)+ expect(snapshotDuringWork).toEqual({ inFlight: 0, queued: 0, principals: 0 })+ expect(observed.uncharged).toHaveBeenCalledOnce()+ expect(observed.charged).not.toHaveBeenCalled()+ permitWork.resolve()+ expect((await sent.response).status).toBe(200)+ })+ })++ it('reads only the declared bytes of a lying text request', async () => {+ const observed = observeAdmission()+ await withApps(1, async ([app]) => {+ const { port, pathname } = new URL(app.url)+ // Content-Length covers only an incomplete JSON prefix. The bytes after+ // it are a complete pipelined request, so Node frames them as a second+ // request instead of failing the connection, and the first response is+ // the route's own answer to the 13 declared bytes.+ const declared = '{"a":"bbbbbbb'+ const surplus = [+ `GET ${pathname} HTTP/1.1`,+ 'Host: 127.0.0.1',+ 'Connection: close',+ '',+ '',+ ].join('\r\n')+ const requestHeaders = textHeaders('lying-length-host')+ const socket = net.connect(Number(port), '127.0.0.1')+ const chunks: Buffer[] = []+ const ended = new Promise<string>((resolve, reject) => {+ socket.on('data', chunk => chunks.push(Buffer.from(chunk)))+ socket.on('error', reject)+ socket.on('end', () => resolve(Buffer.concat(chunks).toString('utf8')))+ })+ socket.write(+ [+ `POST ${pathname} HTTP/1.1`,+ 'Host: 127.0.0.1',+ `Authorization: ${requestHeaders.Authorization}`,+ 'Content-Type: application/json',+ `Content-Length: ${Buffer.byteLength(declared)}`,+ '',+ declared + surplus,+ ].join('\r\n')+ )+ const raw = await ended+ expect(Buffer.byteLength(declared)).toBe(13)+ const statuses = [...raw.matchAll(/HTTP\/1\.1 (\d{3}) /g)].map(match => Number(match[1]))+ // The route refuses the 13-byte prefix; the surplus GET carries no JWT.+ expect(statuses).toEqual([400, 401])+ const [first, second] = raw.split(/(?=HTTP\/1\.1 401 )/)+ expect(first).toContain('{"error":"invalid_request"}')+ expect(second).toContain('{"error":"Unauthorized"}')+ expect(observed.uncharged).toHaveBeenCalledOnce()+ expect(observed.charged).not.toHaveBeenCalled()+ expect(authorizer.authorizeLlmProviderAttempt).not.toHaveBeenCalled()+ expect(observed.admission.snapshot()).toEqual({ inFlight: 0, queued: 0, principals: 0 })+ })+ })+})diff --git a/mcp-host/package-lock.json b/mcp-host/package-lock.jsonindex d74aa345a..e523568e4 100644--- a/mcp-host/package-lock.json+++ b/mcp-host/package-lock.json@@ -1,12 +1,12 @@{"name": "clerum-mcp-host",- "version": "0.1.488",+ "version": "0.1.489","lockfileVersion": 3,"requires": true,"packages": {"": {"name": "clerum-mcp-host",- "version": "0.1.488",+ "version": "0.1.489","license": "SEE LICENSE IN LICENSE","dependencies": {"@anthropic-ai/sdk": "^0.32.0",diff --git a/mcp-host/package.json b/mcp-host/package.jsonindex cef9cdc9c..36bf0bf79 100644--- a/mcp-host/package.json+++ b/mcp-host/package.json@@ -1,6 +1,6 @@{"name": "clerum-mcp-host",- "version": "0.1.488",+ "version": "0.1.489","description": "MCP Host - reads Host CRD and provides LLM access via OpenAI/Claude","main": "dist/main.js","engines": {diff --git a/mcp-host/src/llm/__tests__/providerAttemptAuthorizer.test.ts b/mcp-host/src/llm/__tests__/providerAttemptAuthorizer.test.tsindex e251044c8..d0a267b1d 100644--- a/mcp-host/src/llm/__tests__/providerAttemptAuthorizer.test.ts+++ b/mcp-host/src/llm/__tests__/providerAttemptAuthorizer.test.ts@@ -504,7 +504,7 @@ describe('ProviderAttemptAuthorizer', () => {it.each([['authorize_capacity_exceeded',- 'Too many requests are active. Wait for them to finish or send fewer concurrent requests.',+ 'Authorization is busy with other large requests, which share one capacity across all Hosts. Nothing was sent to the provider; wait for them to finish, then try again.',],['authorize_timeout',diff --git a/mcp-host/src/llm/providerAttemptAuthorizer.ts b/mcp-host/src/llm/providerAttemptAuthorizer.tsindex 938287ac7..d7b12317f 100644--- a/mcp-host/src/llm/providerAttemptAuthorizer.ts+++ b/mcp-host/src/llm/providerAttemptAuthorizer.ts@@ -210,7 +210,7 @@ export class ProviderAttemptAuthorizer {code === 'payload_too_large'? `${contract.label} request is too large; use fewer or smaller images, or reduce context`: code === 'authorize_capacity_exceeded'- ? 'Too many requests are active. Wait for them to finish or send fewer concurrent requests.'+ ? 'Authorization is busy with other large requests, which share one capacity across all Hosts. Nothing was sent to the provider; wait for them to finish, then try again.': code === 'authorize_timeout'? 'Request authorization timed out. Wait for active requests to finish, then try again.': `authorize failed with ${response.status}`,