Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
fix(control-api): discard a refused authorize body before answering 503
Alfredo Lopez0637ccfe45fd
diff --git a/control-api/package-lock.json b/control-api/package-lock.jsonindex def1061e8..e29bc25e5 100644--- a/control-api/package-lock.json+++ b/control-api/package-lock.json@@ -1,12 +1,12 @@{"name": "clerum-control-api",- "version": "0.1.725",+ "version": "0.1.726","lockfileVersion": 3,"requires": true,"packages": {"": {"name": "clerum-control-api",- "version": "0.1.725",+ "version": "0.1.726","dependencies": {"@clerum/display-field": "file:../packages/display-field","@clerum/grok-provider-attempt-contract": "file:../packages/grok-provider-attempt-contract",diff --git a/control-api/package.json b/control-api/package.jsonindex 80a38aff5..0f608d265 100644--- a/control-api/package.json+++ b/control-api/package.json@@ -1,6 +1,6 @@{"name": "clerum-control-api",- "version": "0.1.725",+ "version": "0.1.726","description": "Internal Clerum control plane REST API + UI for managing CRDs and secrets","main": "dist/main.js","engines": {diff --git a/control-api/src/middleware/llmProviderAttemptAdmissionLimits.ts b/control-api/src/middleware/llmProviderAttemptAdmissionLimits.tsindex 62a161817..a90f7d9ff 100644--- a/control-api/src/middleware/llmProviderAttemptAdmissionLimits.ts+++ b/control-api/src/middleware/llmProviderAttemptAdmissionLimits.ts@@ -1,3 +1,5 @@+import { LIMITS as GROK_LIMITS } from '@clerum/grok-provider-attempt-contract'+import { LIMITS } from '@clerum/llm-provider-attempt-contract'import type { AuthorizeBodyAdmissionPolicy } from './llmProviderAttemptBodyAdmission.js'// Ships as an explicit uncertified ceiling; see #813. The combined@@ -24,6 +26,11 @@ import type { AuthorizeBodyAdmissionPolicy } from './llmProviderAttemptBodyAdmis// once in the same provider after it, re-uploading the body, and the second// refusal is terminal. It equals the read deadline, the bound on a slow// upload, and stays below the Host's 30 s retry ceiling. Uncertified (#813).+//+// maxDiscardBodyBytes is the retained parser's limit. A capacity refusal reads+// and discards up to that many bytes before it answers, so the gateway reads+// the 503 instead of failing its write on a reset (502). Discarded bytes are+// never retained, so this costs network and CPU, not memory.const READ_DEADLINE_MS = 10_000const WORK_DEADLINE_MS = 30_000@@ -36,4 +43,8 @@ export const LLM_PROVIDER_ATTEMPT_ADMISSION_POLICY: AuthorizeBodyAdmissionPolicyworkDeadlineMs: WORK_DEADLINE_MS,closeGraceMs: 250,retryAfterSeconds: READ_DEADLINE_MS / 1000,+ maxDiscardBodyBytes: Math.max(+ LIMITS.maxVisualRequestBodyBytes,+ GROK_LIMITS.maxVisualRequestBodyBytes+ ),})diff --git a/control-api/src/middleware/llmProviderAttemptBodyAdmission.ts b/control-api/src/middleware/llmProviderAttemptBodyAdmission.tsindex fa6c79213..f4f6f4a00 100644--- a/control-api/src/middleware/llmProviderAttemptBodyAdmission.ts+++ b/control-api/src/middleware/llmProviderAttemptBodyAdmission.ts@@ -12,6 +12,8 @@ export type AuthorizeBodyAdmissionPolicy = Readonly<{closeGraceMs: number/** `Retry-After` on a capacity refusal; the Host retries once after it. */retryAfterSeconds: number+ /** Most bytes a capacity refusal reads and discards before it answers. */+ maxDiscardBodyBytes: number}>// The route supplies these claims only after requireMcpHostJwt succeeds.@@ -85,6 +87,9 @@ export class AuthorizeBodyAdmission {) {throw new TypeError('retryAfterSeconds must be an integer from 1 to 3600')}+ if (!Number.isSafeInteger(policy.maxDiscardBodyBytes) || policy.maxDiscardBodyBytes <= 0) {+ throw new TypeError('maxDiscardBodyBytes must be a positive safe integer')+ }this.policy = Object.freeze({maxInFlight: policy.maxInFlight,maxQueued: policy.maxQueued,@@ -94,6 +99,7 @@ export class AuthorizeBodyAdmission {workDeadlineMs: policy.workDeadlineMs,closeGraceMs: policy.closeGraceMs,retryAfterSeconds: policy.retryAfterSeconds,+ maxDiscardBodyBytes: policy.maxDiscardBodyBytes,})}@@ -303,24 +309,18 @@ export class AuthorizeBodyAdmission {}}+ // The refusal reads and discards the body before it answers. Closing a+ // socket with unread request bytes sends a TCP reset, and the gateway+ // (nginx, which buffers the whole body and is still writing it) can then+ // fail its write before it reads the queued 503. It answers 502 instead,+ // mapped to control_plane_unavailable, and the Host never sees the+ // retryable code. Discarding retains nothing, so the refusal still takes no+ // admission unit. A body that does not end within the read deadline, or+ // that declares or sends more than maxDiscardBodyBytes, is answered at once+ // and closed, as before.private refuseBeforeRead(req: Request, res: Response, refusal: AdmissionRefusal): void {const inFlight = this.inFlightreq.pause()- let stopped = false- let closeTimer: ReturnType<typeof setTimeout> | undefined- const stopRejectedBody = (): void => {- if (stopped) return- stopped = true- if (closeTimer !== undefined) clearTimeout(closeTimer)- res.off('finish', stopRejectedBody)- res.off('close', stopRejectedBody)- if (!req.destroyed) req.destroy()- }- // A blocked or pipelined peer can leave the 503 response unable to reach- // finish/close. The unread request body still needs a bounded stop.- closeTimer = setTimeout(stopRejectedBody, this.policy.closeGraceMs)- res.once('finish', stopRejectedBody)- res.once('close', stopRejectedBody)res.setHeader('Connection', 'close')// Nothing was read or recorded, so the Host may retry the same attempt// once after this delay (same provider, no failover).@@ -339,12 +339,116 @@ export class AuthorizeBodyAdmission {},'authorize admission refused before body read')- try {- res.status(503).json({ error: 'authorize_capacity_exceeded' })- } catch (err) {- stopRejectedBody()- throw err+ this.discardThenAnswer(req, res)+ }++ private discardThenAnswer(req: Request, res: Response): void {+ let settled = false+ let discarded = 0+ let readTimer: ReturnType<typeof setTimeout> | undefined+ let closeTimer: ReturnType<typeof setTimeout> | undefined+ const { closeGraceMs, maxDiscardBodyBytes, readDeadlineMs } = this.policy++ const detachReader = (): void => {+ if (readTimer !== undefined) clearTimeout(readTimer)+ req.off('data', discard)+ req.off('end', answerDiscarded)+ req.off('aborted', abandon)+ req.off('error', abandon)+ res.off('close', abandonUnfinished)+ }+ const clearClose = (): void => {+ if (closeTimer !== undefined) clearTimeout(closeTimer)+ res.off('finish', stopSocket)+ res.off('close', stopSocket)+ res.off('finish', clearClose)+ res.off('close', clearClose)+ }+ const stopSocket = (): void => {+ clearClose()+ if (!req.destroyed) req.destroy()+ }+ // Inside an event callback a throw would be uncaught, so it is logged and+ // the socket closed; on the synchronous path it propagates to the caller.+ const send = (rethrow: boolean): void => {+ try {+ res.status(503).json({ error: 'authorize_capacity_exceeded' })+ } catch (err) {+ stopSocket()+ if (rethrow) throw err+ requestLog(req).error(+ { event: 'llm_provider_attempt_admission_refusal_write_failed', err },+ 'authorize capacity refusal could not be written'+ )+ }+ }+ // Unread bytes remain: answer now and close the socket. A blocked or+ // pipelined peer can leave the 503 unable to reach finish/close, so the+ // close is also bounded by closeGraceMs.+ const answerAndClose = (+ reason: 'declared_length' | 'discard_limit' | 'read_deadline',+ rethrow: boolean+ ): void => {+ if (settled) return+ settled = true+ detachReader()+ req.pause()+ requestLog(req).warn(+ {+ event: 'llm_provider_attempt_admission_discard_stopped',+ reason,+ discardedBytes: discarded,+ maxDiscardBodyBytes,+ },+ 'authorize capacity refusal closed before the body ended'+ )+ closeTimer = setTimeout(stopSocket, closeGraceMs)+ res.once('finish', stopSocket)+ res.once('close', stopSocket)+ send(rethrow)+ }+ // The body was read to its end, so Node's Connection: close ends the+ // socket without a reset. closeGraceMs still bounds a peer that never+ // reads the 503.+ const answerDiscarded = (): void => {+ if (settled) return+ settled = true+ detachReader()+ closeTimer = setTimeout(stopSocket, closeGraceMs)+ res.once('finish', clearClose)+ res.once('close', clearClose)+ send(false)+ }+ const abandon = (): void => {+ if (settled) return+ settled = true+ detachReader()+ if (!req.destroyed) req.destroy()+ }+ const abandonUnfinished = (): void => {+ if (!res.writableFinished) abandon()+ }+ const discard = (chunk: Buffer): void => {+ discarded += chunk.length+ if (discarded > maxDiscardBodyBytes) answerAndClose('discard_limit', false)+ }++ const declared = req.headers?.['content-length']+ if (+ typeof declared === 'string' &&+ /^\d+$/.test(declared) &&+ Number(declared) > maxDiscardBodyBytes+ ) {+ answerAndClose('declared_length', true)+ return}+ readTimer = setTimeout(() => answerAndClose('read_deadline', false), readDeadlineMs)+ req.on('data', discard)+ req.once('end', answerDiscarded)+ req.once('aborted', abandon)+ req.once('error', abandon)+ res.once('close', abandonUnfinished)+ req.resume()}}diff --git a/control-api/test/llmProviderAttemptBodyAdmission.test.ts b/control-api/test/llmProviderAttemptBodyAdmission.test.tsindex e52333980..7ec8cacaa 100644--- a/control-api/test/llmProviderAttemptBodyAdmission.test.ts+++ b/control-api/test/llmProviderAttemptBodyAdmission.test.ts@@ -14,6 +14,7 @@ const OTHER_PRINCIPAL = { sub: 'other-principal', hostRefs: ['other-host'] }type FakeRequest = EventEmitter & {body?: unknown+ headers?: Record<string, string>aborted: booleandestroyed: booleanpause: ReturnType<typeof vi.fn>@@ -24,6 +25,7 @@ type FakeRequest = EventEmitter & {type FakeChildLogger = {warn: ReturnType<typeof vi.fn>+ error: ReturnType<typeof vi.fn>}type FakeRequestLogger = {@@ -77,7 +79,7 @@ function fakeResponse(): FakeResponse {}function fakeRequestLogger(): { logger: FakeRequestLogger; child: FakeChildLogger } {- const childLogger: FakeChildLogger = { warn: vi.fn() }+ const childLogger: FakeChildLogger = { warn: vi.fn(), error: vi.fn() }return {logger: { child: vi.fn(() => childLogger) },child: childLogger,@@ -142,6 +144,99 @@ function readRawResponse(return result}+type LargeBodyOutcome = {+ status: number+ headers: string+ body: string+ bytesQueued: number+ bytesAcknowledged: number+ bytesQueuedAtFirstResponseByte: number | undefined+ ended: boolean+ error: Error | undefined+}++// Writes the head and then the body in chunks, honouring backpressure, while+// reading the response concurrently. It never ends its own side, so a server+// FIN surfaces as `end` and a reset as `error` (EPIPE/ECONNRESET).+async function sendLargeBody(+ port: number,+ head: string,+ totalBytes: number,+ chunkBytes: number,+ closeDeadlineMs: number+): Promise<LargeBodyOutcome> {+ const socket = net.connect(port, '127.0.0.1')+ const chunks: Buffer[] = []+ let bytesQueued = 0+ let bytesAcknowledged = 0+ let bytesQueuedAtFirstResponseByte: number | undefined+ let ended = false+ let error: Error | undefined+ socket.on('data', chunk => {+ bytesQueuedAtFirstResponseByte ??= bytesQueued+ chunks.push(Buffer.from(chunk))+ })+ socket.on('end', () => {+ ended = true+ })+ socket.on('error', err => {+ error = err+ })+ const closed = new Promise<void>(resolve => socket.once('close', () => resolve()))+ await new Promise<void>((resolve, reject) => {+ socket.once('connect', resolve)+ socket.once('error', reject)+ })+ socket.write(head)+ const chunk = Buffer.alloc(chunkBytes, 0x20)+ while (bytesQueued < totalBytes && error === undefined && !socket.destroyed) {+ const size = Math.min(chunkBytes, totalBytes - bytesQueued)+ bytesQueued += size+ const flushed = socket.write(chunk.subarray(0, size), err => {+ if (!err) bytesAcknowledged += size+ })+ if (!flushed) {+ await new Promise<void>(resolve => {+ const done = (): void => {+ socket.off('drain', done)+ socket.off('close', done)+ resolve()+ }+ socket.once('drain', done)+ socket.once('close', done)+ })+ }+ }+ let deadline: ReturnType<typeof setTimeout> | undefined+ try {+ await Promise.race([+ closed,+ new Promise<never>((_resolve, reject) => {+ deadline = setTimeout(+ () => reject(new Error(`large-body client did not close within ${closeDeadlineMs} ms`)),+ closeDeadlineMs+ )+ }),+ ])+ } finally {+ if (deadline !== undefined) clearTimeout(deadline)+ socket.destroy()+ }+ const text = Buffer.concat(chunks).toString('utf8')+ const headerEnd = text.indexOf('\r\n\r\n')+ const headers = headerEnd === -1 ? text : text.slice(0, headerEnd)+ return {+ status: Number(/^HTTP\/1\.1 (\d+)/m.exec(headers)?.[1]),+ headers,+ body: headerEnd === -1 ? '' : text.slice(headerEnd + 4),+ bytesQueued,+ bytesAcknowledged,+ bytesQueuedAtFirstResponseByte,+ ended,+ error,+ }+}+describe('AuthorizeBodyAdmission policy and idempotence', () => {it('rejects invalid counts and timer values and freezes the accepted policy', () => {const valid = {@@ -153,6 +248,7 @@ describe('AuthorizeBodyAdmission policy and idempotence', () => {workDeadlineMs: 20,closeGraceMs: 5,retryAfterSeconds: 1,+ maxDiscardBodyBytes: 1_048_576,}const gate = new AuthorizeBodyAdmission(valid)expect(() => {@@ -191,6 +287,11 @@ describe('AuthorizeBodyAdmission policy and idempotence', () => {TypeError)}+ for (const value of [0, -1, 1.5, Number.MAX_SAFE_INTEGER + 1]) {+ expect(() => new AuthorizeBodyAdmission({ ...valid, maxDiscardBodyBytes: value })).toThrow(+ TypeError+ )+ }})it('an old release cannot free the slot occupied by a newer request', () => {@@ -203,6 +304,7 @@ describe('AuthorizeBodyAdmission policy and idempotence', () => {workDeadlineMs: 100,closeGraceMs: 10,retryAfterSeconds: 1,+ maxDiscardBodyBytes: 1_048_576,})const releaseA = gate.tryAcquire(PRINCIPAL)expect(releaseA).not.toBeNull()@@ -228,6 +330,7 @@ describe('AuthorizeBodyAdmission policy and idempotence', () => {workDeadlineMs: 100,closeGraceMs: 10,retryAfterSeconds: 1,+ maxDiscardBodyBytes: 1_048_576,})const req = fakeRequest()const res = fakeResponse()@@ -259,6 +362,7 @@ describe('AuthorizeBodyAdmission FIFO and verified-principal share', () => {workDeadlineMs: 1000,closeGraceMs: 5,retryAfterSeconds: 1,+ maxDiscardBodyBytes: 1_048_576,...overrides,})}@@ -296,7 +400,7 @@ describe('AuthorizeBodyAdmission FIFO and verified-principal share', () => {expect(res.listenerCount('close')).toBe(0)})- it('never lets an arrival leapfrog the FIFO and refuses a full queue without a body reader', async () => {+ it('never lets an arrival leapfrog the FIFO and refuses a full queue without parsing or retaining the body', async () => {const gate = queuedGate()const holder = gate.tryAcquire(PRINCIPAL)const order: string[] = []@@ -344,8 +448,15 @@ describe('AuthorizeBodyAdmission FIFO and verified-principal share', () => {hostRefs: ['fourth-host'],})- expect(fullRes.body).toEqual({ error: 'authorize_capacity_exceeded' })+ // The refusal discards the body to its end before it answers.expect(fullRes.setHeader).toHaveBeenCalledWith('Retry-After', '1')+ expect(fullReq.resume).toHaveBeenCalledOnce()+ expect(fullRes.body).toBeUndefined()+ fullReq.emit('data', Buffer.from('{"discarded":true}'))+ expect(fullRes.body).toBeUndefined()+ fullReq.emit('end')+ expect(fullRes.body).toEqual({ error: 'authorize_capacity_exceeded' })+ expect(fullReq.body).toBeUndefined()expect(fullParser).not.toHaveBeenCalled()await refusedfullRes.emit('finish')@@ -396,9 +507,12 @@ describe('AuthorizeBodyAdmission FIFO and verified-principal share', () => {async () => undefined,{ sub: 'same-sub', hostRefs: ['host-b', 'host-a'] })- expect(rejectedRes.body).toEqual({ error: 'authorize_capacity_exceeded' })// principal_share invites the Host's single retry like every capacity refusal.expect(rejectedRes.setHeader).toHaveBeenCalledWith('Retry-After', '1')+ expect(rejectedReq.resume).toHaveBeenCalledOnce()+ expect(rejectedRes.body).toBeUndefined()+ rejectedReq.emit('end')+ expect(rejectedRes.body).toEqual({ error: 'authorize_capacity_exceeded' })expect(rejectedParser).not.toHaveBeenCalled()await rejectedrejectedRes.emit('finish')@@ -516,12 +630,23 @@ describe('AuthorizeBodyAdmission FIFO and verified-principal share', () => {expect(expiry).toBeTypeOf('function')await vi.advanceTimersByTimeAsync(20)await expired+ // The queue hooks are gone; the refusal's discard reader replaced them.+ expect(req.resume).toHaveBeenCalledOnce()+ expect(req.listenerCount('data')).toBe(1)+ expect(req.listenerCount('aborted')).toBe(1)+ expect(res.body).toBeUndefined()+ expect(gate.snapshot()).toEqual({ inFlight: 1, queued: 0, principals: 1 })+ // The body never ends: the read deadline (1000 ms) is its finite backstop.+ await vi.advanceTimersByTimeAsync(999)+ expect(res.body).toBeUndefined()+ await vi.advanceTimersByTimeAsync(1)expect(res.body).toEqual({ error: 'authorize_capacity_exceeded' })expect(parser).not.toHaveBeenCalled()expect(work).not.toHaveBeenCalled()expect(req.destroy).not.toHaveBeenCalled()+ expect(req.listenerCount('data')).toBe(0)expect(req.listenerCount('aborted')).toBe(0)- expect(gate.snapshot()).toEqual({ inFlight: 1, queued: 0, principals: 1 })+ // Response finish never arrives: closeGrace (5 ms) closes the socket.await vi.advanceTimersByTimeAsync(5)expect(req.destroy).toHaveBeenCalledOnce()expect(res.listenerCount('finish')).toBe(0)@@ -572,10 +697,15 @@ describe('AuthorizeBodyAdmission FIFO and verified-principal share', () => {try {holder?.()await waiting+ expect(req.resume).toHaveBeenCalledOnce()+ expect(res.body).toBeUndefined()+ req.emit('end')expect(res.body).toEqual({ error: 'authorize_capacity_exceeded' })expect(parser).not.toHaveBeenCalled()expect(gate.snapshot()).toEqual({ inFlight: 0, queued: 0, principals: 0 })res.emit('finish')+ // A body read to its end is not destroyed; Connection: close ends it.+ expect(req.destroy).not.toHaveBeenCalled()expect(vi.getTimerCount()).toBe(0)} finally {monotonic.mockRestore()@@ -746,6 +876,9 @@ describe('AuthorizeBodyAdmission production queue wait', () => {vi.advanceTimersByTime(1)await queued.waiting+ expect(queued.req.resume).toHaveBeenCalledOnce()+ expect(queued.res.status).not.toHaveBeenCalled()+ queued.req.emit('end')expect(queued.res.statusCode).toBe(503)expect(queued.res.body).toEqual({ error: 'authorize_capacity_exceeded' })expect(queued.requestLogger.child.warn).toHaveBeenCalledWith(@@ -884,6 +1017,7 @@ describe('AuthorizeBodyAdmission parser lifecycle', () => {workDeadlineMs: 100,closeGraceMs: 10,retryAfterSeconds: 1,+ maxDiscardBodyBytes: 1_048_576,})const req = fakeRequest()const res = fakeResponse()@@ -916,6 +1050,7 @@ describe('AuthorizeBodyAdmission parser lifecycle', () => {workDeadlineMs: 100,closeGraceMs: 5,retryAfterSeconds: 1,+ maxDiscardBodyBytes: 1_048_576,})const req = fakeRequest()const requestLogger = fakeRequestLogger()@@ -949,7 +1084,7 @@ describe('AuthorizeBodyAdmission parser lifecycle', () => {)})- it('bounds a never-finishing capacity refusal without reading or starting work', async () => {+ it('bounds a never-ending, never-finishing capacity refusal at the read deadline without parsing or starting work', async () => {const gate = new AuthorizeBodyAdmission({maxInFlight: 1,maxQueued: 0,@@ -959,6 +1094,7 @@ describe('AuthorizeBodyAdmission parser lifecycle', () => {workDeadlineMs: 100,closeGraceMs: 5,retryAfterSeconds: 1,+ maxDiscardBodyBytes: 1_048_576,})const holder = gate.tryAcquire(PRINCIPAL)expect(holder).not.toBeNull()@@ -970,15 +1106,24 @@ describe('AuthorizeBodyAdmission parser lifecycle', () => {const work = vi.fn(async () => undefined)await gate.run(req as never, res as never, parser as RequestHandler, work, PRINCIPAL)+ // The refusal discards; the body never ends and finish never arrives.+ expect(req.resume).toHaveBeenCalledOnce()+ expect(res.body).toBeUndefined()+ req.emit('data', Buffer.from('partial'))await vi.waitFor(() => expect(req.destroy).toHaveBeenCalledOnce())- expect(req.pause).toHaveBeenCalledOnce()+ // Paused once by the refusal and once more when the discard stops.+ expect(req.pause).toHaveBeenCalledTimes(2)expect(res.setHeader).toHaveBeenCalledWith('Connection', 'close')expect(res.body).toEqual({ error: 'authorize_capacity_exceeded' })+ expect(req.body).toBeUndefined()expect(parser).not.toHaveBeenCalled()expect(work).not.toHaveBeenCalled()expect(EventEmitter.listenerCount(res, 'close')).toBe(0)expect(EventEmitter.listenerCount(res, 'finish')).toBe(0)+ for (const event of ['data', 'end', 'aborted', 'error']) {+ expect(req.listenerCount(event)).toBe(0)+ }expect(gate.tryAcquire(PRINCIPAL)).toBeNull()expect(req.log?.child).toHaveBeenCalledWith({module: 'llm-provider-attempt-body-admission',@@ -995,6 +1140,15 @@ describe('AuthorizeBodyAdmission parser lifecycle', () => {}),'authorize admission refused before body read')+ expect(requestLogger.child.warn).toHaveBeenCalledWith(+ {+ event: 'llm_provider_attempt_admission_discard_stopped',+ reason: 'read_deadline',+ discardedBytes: Buffer.byteLength('partial'),+ maxDiscardBodyBytes: 1_048_576,+ },+ 'authorize capacity refusal closed before the body ended'+ )holder?.()expect(gate.tryAcquire(PRINCIPAL)).not.toBeNull()@@ -1010,6 +1164,7 @@ describe('AuthorizeBodyAdmission parser lifecycle', () => {workDeadlineMs: 100,closeGraceMs: 10,retryAfterSeconds: 1,+ maxDiscardBodyBytes: 1_048_576,})const req = fakeRequest()const res = fakeResponse()@@ -1040,6 +1195,7 @@ describe('AuthorizeBodyAdmission work cancellation', () => {workDeadlineMs: 15,closeGraceMs: 10,retryAfterSeconds: 1,+ maxDiscardBodyBytes: 1_048_576,})const req = fakeRequest()const res = fakeResponse()@@ -1071,6 +1227,7 @@ describe('AuthorizeBodyAdmission work cancellation', () => {workDeadlineMs: 5000,closeGraceMs: 10,retryAfterSeconds: 1,+ maxDiscardBodyBytes: 1_048_576,})const req = fakeRequest()const res = fakeResponse()@@ -1120,6 +1277,362 @@ describe('AuthorizeBodyAdmission work cancellation', () => {})})+describe('AuthorizeBodyAdmission capacity refusal discard', () => {+ const CLOSE_GRACE_MS = 5+ const READ_DEADLINE_MS = 100+ const REQUEST_EVENTS = ['data', 'end', 'aborted', 'error'] as const+ const STOPPED_MESSAGE = 'authorize capacity refusal closed before the body ended'++ function refusingGate(maxDiscardBodyBytes = 1_048_576) {+ const gate = new AuthorizeBodyAdmission({+ maxInFlight: 1,+ maxQueued: 0,+ maxPerPrincipal: 2,+ queueWaitMs: 100,+ readDeadlineMs: READ_DEADLINE_MS,+ workDeadlineMs: 100,+ closeGraceMs: CLOSE_GRACE_MS,+ retryAfterSeconds: 1,+ maxDiscardBodyBytes,+ })+ const holder = gate.tryAcquire(PRINCIPAL)+ expect(holder).not.toBeNull()+ return { gate, holder: holder! }+ }++ function refusalFixture(headers?: Record<string, string>) {+ const req = fakeRequest()+ if (headers) req.headers = headers+ const requestLogger = fakeRequestLogger()+ req.log = requestLogger.logger+ const res = fakeResponse()+ const parser = vi.fn()+ const work = vi.fn(async () => undefined)+ return { req, res, parser, work, warn: requestLogger.child.warn }+ }++ // Runs a queue_full refusal and checks what every capacity refusal does at+ // once: close-and-retry headers and the refusal log, before any answer.+ async function refuse(gate: AuthorizeBodyAdmission, headers?: Record<string, string>) {+ const fixture = refusalFixture(headers)+ await gate.run(+ fixture.req as never,+ fixture.res as never,+ fixture.parser as RequestHandler,+ fixture.work,+ OTHER_PRINCIPAL+ )+ expect(fixture.res.setHeader).toHaveBeenCalledWith('Connection', 'close')+ expect(fixture.res.setHeader).toHaveBeenCalledWith('Retry-After', '1')+ expect(fixture.warn).toHaveBeenCalledWith(+ expect.objectContaining({+ event: 'llm_provider_attempt_admission_refused',+ reason: 'authorize_capacity_exceeded',+ refusal: 'queue_full',+ }),+ 'authorize admission refused before body read'+ )+ return fixture+ }++ function discardStoppedLogs(warn: ReturnType<typeof vi.fn>) {+ return warn.mock.calls.filter(+ ([fields]) =>+ (fields as { event?: unknown } | undefined)?.event ===+ 'llm_provider_attempt_admission_discard_stopped'+ )+ }++ function finishResponse(res: FakeResponse) {+ res.writableEnded = true+ res.writableFinished = true+ res.emit('finish')+ }++ function expectNothingRetained(fixture: ReturnType<typeof refusalFixture>) {+ expect(fixture.req.body).toBeUndefined()+ expect(fixture.parser).not.toHaveBeenCalled()+ expect(fixture.work).not.toHaveBeenCalled()+ }++ it('answers only after the discarded body ends and does not destroy a fully read request on finish', async () => {+ vi.useFakeTimers()+ try {+ const { gate, holder } = refusingGate()+ const fixture = await refuse(gate)+ const { req, res } = fixture+ expect(req.resume).toHaveBeenCalledOnce()+ expect(req.listenerCount('data')).toBe(1)+ req.emit('data', Buffer.alloc(1024))+ req.emit('data', Buffer.alloc(2048))+ expect(res.status).not.toHaveBeenCalled()+ expect(res.body).toBeUndefined()++ req.emit('end')+ expect(res.statusCode).toBe(503)+ expect(res.body).toEqual({ error: 'authorize_capacity_exceeded' })+ expect(res.json).toHaveBeenCalledOnce()+ expectNothingRetained(fixture)+ expect(discardStoppedLogs(fixture.warn)).toEqual([])+ for (const event of REQUEST_EVENTS) expect(req.listenerCount(event)).toBe(0)+ // The refusal took no unit: only the holder is charged.+ expect(gate.snapshot()).toEqual({ inFlight: 1, queued: 0, principals: 1 })+ // Only the closeGrace backstop is pending, cleared by finish.+ expect(res.listenerCount('finish')).toBe(1)+ expect(vi.getTimerCount()).toBe(1)++ finishResponse(res)+ res.emit('close')+ expect(res.listenerCount('finish')).toBe(0)+ expect(res.listenerCount('close')).toBe(0)+ expect(vi.getTimerCount()).toBe(0)+ await vi.advanceTimersByTimeAsync(CLOSE_GRACE_MS * 10)+ expect(req.destroy).not.toHaveBeenCalled()+ expect(res.json).toHaveBeenCalledOnce()++ holder()+ expect(gate.snapshot()).toEqual({ inFlight: 0, queued: 0, principals: 0 })+ } finally {+ vi.useRealTimers()+ }+ })++ it('destroys a fully discarded refusal through closeGrace when response finish never arrives', async () => {+ vi.useFakeTimers()+ try {+ const { gate } = refusingGate()+ const fixture = await refuse(gate)+ const { req, res } = fixture+ req.emit('data', Buffer.from('{}'))+ req.emit('end')+ expect(res.body).toEqual({ error: 'authorize_capacity_exceeded' })+ expect(vi.getTimerCount()).toBe(1)++ await vi.advanceTimersByTimeAsync(CLOSE_GRACE_MS - 1)+ expect(req.destroy).not.toHaveBeenCalled()+ await vi.advanceTimersByTimeAsync(1)+ expect(req.destroy).toHaveBeenCalledOnce()+ expect(res.listenerCount('finish')).toBe(0)+ expect(res.listenerCount('close')).toBe(0)+ expect(vi.getTimerCount()).toBe(0)+ expect(discardStoppedLogs(fixture.warn)).toEqual([])+ expectNothingRetained(fixture)+ } finally {+ vi.useRealTimers()+ }+ })++ it('discards a body of exactly maxDiscardBodyBytes and answers on end', async () => {+ const { gate } = refusingGate(10)+ const fixture = await refuse(gate)+ fixture.req.emit('data', Buffer.alloc(4))+ fixture.req.emit('data', Buffer.alloc(6))+ expect(fixture.res.body).toBeUndefined()+ fixture.req.emit('end')+ expect(fixture.res.body).toEqual({ error: 'authorize_capacity_exceeded' })+ expect(discardStoppedLogs(fixture.warn)).toEqual([])+ finishResponse(fixture.res)+ expect(fixture.req.destroy).not.toHaveBeenCalled()+ })++ it('answers at once and closes when the discarded bytes exceed maxDiscardBodyBytes', async () => {+ vi.useFakeTimers()+ try {+ const { gate } = refusingGate(10)+ const fixture = await refuse(gate)+ const { req, res } = fixture+ req.emit('data', Buffer.alloc(6))+ expect(res.body).toBeUndefined()+ req.emit('data', Buffer.alloc(5))++ expect(res.statusCode).toBe(503)+ expect(res.body).toEqual({ error: 'authorize_capacity_exceeded' })+ // Paused by the refusal and again when the discard stops.+ expect(req.pause).toHaveBeenCalledTimes(2)+ for (const event of REQUEST_EVENTS) expect(req.listenerCount(event)).toBe(0)+ expect(discardStoppedLogs(fixture.warn)).toEqual([+ [+ {+ event: 'llm_provider_attempt_admission_discard_stopped',+ reason: 'discard_limit',+ discardedBytes: 11,+ maxDiscardBodyBytes: 10,+ },+ STOPPED_MESSAGE,+ ],+ ])+ expect(req.destroy).not.toHaveBeenCalled()+ finishResponse(res)+ expect(req.destroy).toHaveBeenCalledOnce()+ expect(vi.getTimerCount()).toBe(0)+ req.emit('data', Buffer.alloc(1))+ req.emit('end')+ expect(res.json).toHaveBeenCalledOnce()+ expectNothingRetained(fixture)+ } finally {+ vi.useRealTimers()+ }+ })++ it('answers from headers alone and closes when Content-Length declares more than maxDiscardBodyBytes', async () => {+ vi.useFakeTimers()+ try {+ const { gate } = refusingGate(10)+ const fixture = await refuse(gate, { 'content-length': '11' })+ const { req, res } = fixture+ expect(res.statusCode).toBe(503)+ expect(res.body).toEqual({ error: 'authorize_capacity_exceeded' })+ expect(req.resume).not.toHaveBeenCalled()+ for (const event of REQUEST_EVENTS) expect(req.listenerCount(event)).toBe(0)+ expect(discardStoppedLogs(fixture.warn)).toEqual([+ [+ {+ event: 'llm_provider_attempt_admission_discard_stopped',+ reason: 'declared_length',+ discardedBytes: 0,+ maxDiscardBodyBytes: 10,+ },+ STOPPED_MESSAGE,+ ],+ ])+ expect(req.destroy).not.toHaveBeenCalled()+ finishResponse(res)+ expect(req.destroy).toHaveBeenCalledOnce()+ expect(vi.getTimerCount()).toBe(0)+ expectNothingRetained(fixture)+ } finally {+ vi.useRealTimers()+ }+ })++ it.each(['10', '0x20', '1e3', '11 '])(+ 'discards instead of trusting Content-Length %j that is at the cap or not plain digits',+ async declared => {+ const { gate } = refusingGate(10)+ const fixture = await refuse(gate, { 'content-length': declared })+ expect(fixture.req.resume).toHaveBeenCalledOnce()+ expect(fixture.res.body).toBeUndefined()+ fixture.req.emit('end')+ expect(fixture.res.body).toEqual({ error: 'authorize_capacity_exceeded' })+ expect(discardStoppedLogs(fixture.warn)).toEqual([])+ finishResponse(fixture.res)+ }+ )++ it('propagates a synchronous refusal write error on the declared-length path and closes the socket', async () => {+ vi.useFakeTimers()+ try {+ const { gate } = refusingGate(10)+ const fixture = refusalFixture({ 'content-length': '11' })+ const writeError = new Error('refusal write failed')+ fixture.res.json.mockImplementation(() => {+ throw writeError+ })+ await expect(+ gate.run(+ fixture.req as never,+ fixture.res as never,+ fixture.parser as RequestHandler,+ fixture.work,+ OTHER_PRINCIPAL+ )+ ).rejects.toBe(writeError)+ expect(fixture.res.json).toHaveBeenCalledOnce()+ expect(fixture.req.destroy).toHaveBeenCalledOnce()+ expect(discardStoppedLogs(fixture.warn)).toHaveLength(1)+ expect(fixture.res.listenerCount('finish')).toBe(0)+ expect(fixture.res.listenerCount('close')).toBe(0)+ expect(vi.getTimerCount()).toBe(0)+ expectNothingRetained(fixture)+ expect(gate.snapshot()).toEqual({ inFlight: 1, queued: 0, principals: 1 })+ } finally {+ vi.useRealTimers()+ }+ })++ it('answers at the read deadline and closes when the discarded body never ends', async () => {+ vi.useFakeTimers()+ try {+ const { gate } = refusingGate()+ const fixture = await refuse(gate)+ const { req, res } = fixture+ req.emit('data', Buffer.alloc(3))+ await vi.advanceTimersByTimeAsync(READ_DEADLINE_MS - 1)+ expect(res.status).not.toHaveBeenCalled()+ expect(req.listenerCount('data')).toBe(1)++ await vi.advanceTimersByTimeAsync(1)+ expect(res.statusCode).toBe(503)+ expect(res.body).toEqual({ error: 'authorize_capacity_exceeded' })+ expect(req.pause).toHaveBeenCalledTimes(2)+ for (const event of REQUEST_EVENTS) expect(req.listenerCount(event)).toBe(0)+ expect(discardStoppedLogs(fixture.warn)).toEqual([+ [+ {+ event: 'llm_provider_attempt_admission_discard_stopped',+ reason: 'read_deadline',+ discardedBytes: 3,+ maxDiscardBodyBytes: 1_048_576,+ },+ STOPPED_MESSAGE,+ ],+ ])+ expect(req.destroy).not.toHaveBeenCalled()+ finishResponse(res)+ expect(req.destroy).toHaveBeenCalledOnce()+ expect(vi.getTimerCount()).toBe(0)+ req.emit('end')+ expect(res.json).toHaveBeenCalledOnce()+ expectNothingRetained(fixture)+ } finally {+ vi.useRealTimers()+ }+ })++ it.each([+ { source: 'request', event: 'aborted' },+ { source: 'request', event: 'error' },+ { source: 'response', event: 'close' },+ ] as const)(+ 'destroys the request without answering when the $source emits $event during the discard',+ async ({ source, event }) => {+ vi.useFakeTimers()+ try {+ const { gate } = refusingGate()+ const fixture = await refuse(gate)+ const { req, res } = fixture+ // Liveness: the discard reader is installed and consuming.+ expect(req.resume).toHaveBeenCalledOnce()+ expect(req.listenerCount('data')).toBe(1)+ req.emit('data', Buffer.alloc(4))+ expect(vi.getTimerCount()).toBe(1)++ if (source === 'request') {+ req.emit(event, ...(event === 'error' ? [new Error('client reset')] : []))+ } else {+ res.emit(event)+ }++ expect(req.destroy).toHaveBeenCalledOnce()+ expect(res.status).not.toHaveBeenCalled()+ expect(res.json).not.toHaveBeenCalled()+ expect(discardStoppedLogs(fixture.warn)).toEqual([])+ for (const name of REQUEST_EVENTS) expect(req.listenerCount(name)).toBe(0)+ expect(res.listenerCount('close')).toBe(0)+ expect(res.listenerCount('finish')).toBe(0)+ expect(vi.getTimerCount()).toBe(0)+ req.emit('end')+ await vi.advanceTimersByTimeAsync(READ_DEADLINE_MS * 2)+ expect(res.json).not.toHaveBeenCalled()+ expect(req.destroy).toHaveBeenCalledOnce()+ expectNothingRetained(fixture)+ } finally {+ vi.useRealTimers()+ }+ }+ )+})+describe('AuthorizeBodyAdmission HTTP fixture', () => {it('preserves chunked JSON and releases after work completes', async () => {const gate = new AuthorizeBodyAdmission({@@ -1131,6 +1644,7 @@ describe('AuthorizeBodyAdmission HTTP fixture', () => {workDeadlineMs: 1000,closeGraceMs: 10,retryAfterSeconds: 1,+ maxDiscardBodyBytes: 1_048_576,})const parser = express.json({ limit: '1kb', inflate: false })const work = vi.fn(async (req: express.Request, res: Response, _signal: AbortSignal) => {@@ -1169,6 +1683,7 @@ describe('AuthorizeBodyAdmission HTTP fixture', () => {workDeadlineMs: 5000,closeGraceMs: 10,retryAfterSeconds: 1,+ maxDiscardBodyBytes: 1_048_576,})const parser = express.json({ limit: '1kb', inflate: false })const workStarted = deferred()@@ -1273,6 +1788,7 @@ describe('AuthorizeBodyAdmission HTTP fixture', () => {workDeadlineMs: 1000,closeGraceMs: 10,retryAfterSeconds: 1,+ maxDiscardBodyBytes: 1_048_576,})const parser = express.json({ limit: '1kb', inflate: false })const parserStarted = deferred()@@ -1316,4 +1832,64 @@ describe('AuthorizeBodyAdmission HTTP fixture', () => {expect(work).toHaveBeenCalledOnce()})++ // A gateway such as nginx writes the whole buffered body before it reads the+ // answer. The refusal must let that write complete and end the connection+ // with FIN; a reset would turn the retryable 503 into a gateway 502.+ it('lets a refused client write its whole large body, then answers 503 with Retry-After and closes with FIN', async () => {+ const BODY_BYTES = 8 * 1024 * 1024+ const CHUNK_BYTES = 64 * 1024+ const gate = new AuthorizeBodyAdmission({+ maxInFlight: 1,+ maxQueued: 0,+ maxPerPrincipal: 2,+ queueWaitMs: 100,+ readDeadlineMs: 10_000,+ workDeadlineMs: 1000,+ closeGraceMs: 10,+ retryAfterSeconds: 1,+ maxDiscardBodyBytes: 16 * 1024 * 1024,+ })+ const holder = gate.tryAcquire(PRINCIPAL)+ expect(holder).not.toBeNull()+ const parser = vi.fn<RequestHandler>()+ const work = vi.fn(async () => undefined)+ const refusedRequests: express.Request[] = []+ const app = express()+ app.post('/test', (req, res, next) => {+ refusedRequests.push(req)+ void gate.run(req, res, parser, work, OTHER_PRINCIPAL).catch(next)+ })++ await withServer(app, async port => {+ const outcome = await sendLargeBody(+ port,+ 'POST /test HTTP/1.1\r\n' ++ 'Host: localhost\r\n' ++ 'Content-Type: application/json\r\n' ++ `Content-Length: ${BODY_BYTES}\r\n\r\n`,+ BODY_BYTES,+ CHUNK_BYTES,+ 10_000+ )+ expect(outcome.error).toBeUndefined()+ expect(outcome.bytesQueued).toBe(BODY_BYTES)+ expect(outcome.bytesAcknowledged).toBe(BODY_BYTES)+ // The answer only follows the last body byte.+ expect(outcome.bytesQueuedAtFirstResponseByte).toBe(BODY_BYTES)+ expect(outcome.ended).toBe(true)+ expect(outcome.status).toBe(503)+ expect(JSON.parse(outcome.body)).toEqual({ error: 'authorize_capacity_exceeded' })+ expect(outcome.headers.toLowerCase()).toContain('\r\nconnection: close')+ expect(outcome.headers.toLowerCase()).toMatch(/\r\nretry-after: 1(\r\n|$)/)+ })++ expect(refusedRequests).toHaveLength(1)+ expect(refusedRequests[0].body).toBeUndefined()+ expect(parser).not.toHaveBeenCalled()+ expect(work).not.toHaveBeenCalled()+ expect(gate.snapshot()).toEqual({ inFlight: 1, queued: 0, principals: 1 })+ holder?.()+ expect(gate.snapshot()).toEqual({ inFlight: 0, queued: 0, principals: 0 })+ }, 20_000)})diff --git a/control-api/test/routes.mcp-host.llmProviderAttempts.bodyAdmission.test.ts b/control-api/test/routes.mcp-host.llmProviderAttempts.bodyAdmission.test.tsindex 772814016..b8abf1e42 100644--- a/control-api/test/routes.mcp-host.llmProviderAttempts.bodyAdmission.test.ts+++ b/control-api/test/routes.mcp-host.llmProviderAttempts.bodyAdmission.test.ts@@ -68,6 +68,9 @@ type Observation = {request: IncomingMessagebodyDataSubscriptions: numberreadBytesBeforeResponse: number+ // Every body byte delivered to a data listener before the response ended,+ // whether through read() or emitted directly to a flowing stream.+ dataBytesBeforeResponse: numberresponseWritesAfterClose: number}@@ -96,6 +99,15 @@ function declaredRetainedHeaders(return { ...textHeaders(host, provider), 'content-length': String(AUTHORIZE_TEXT_BODY_BYTES + 1) }}+// A capacity refusal discards the body before it answers, except for a+// declared length above the discard cap, which it answers from headers alone.+function overDiscardCapHeaders(host: string): Record<string, string> {+ return {+ ...textHeaders(host),+ 'content-length': String(LLM_PROVIDER_ATTEMPT_ADMISSION_POLICY.maxDiscardBodyBytes + 1),+ }+}+function send(url: string, requestHeaders: Record<string, string>, body?: string) {let client!: ReturnType<typeof httpRequest>const response = new Promise<Reply>((resolve, reject) => {@@ -129,6 +141,7 @@ async function withApps(request: req,bodyDataSubscriptions: 0,readBytesBeforeResponse: 0,+ dataBytesBeforeResponse: 0,responseWritesAfterClose: 0,}observations.push(observed)@@ -151,6 +164,18 @@ async function withApps(observed.readBytesBeforeResponse += value.lengthreturn value}+ // A flowing stream can emit a chunk without calling read(), so data+ // delivery is counted at emit.+ const emit = req.emit+ req.emit = function (this: typeof req, event: string | symbol, ...args: unknown[]) {+ const [chunk] = args+ if (event === 'data' && !res.writableEnded) {+ observed.dataBytesBeforeResponse += Buffer.isBuffer(chunk)+ ? chunk.length+ : Buffer.byteLength(String(chunk))+ }+ return emit.call(this, event, ...args)+ } as typeof req.emitconst on = req.onvi.spyOn(req, 'on').mockImplementation(function (this: typeof req,@@ -277,23 +302,28 @@ describe('createApp retained authorize-body ownership', () => {expect(blocked.headers['retry-after']).toBe(String(LLM_PROVIDER_ATTEMPT_ADMISSION_POLICY.retryAfterSeconds))+ // The refusal discards the whole body (one counting listener) before+ // it answers; nothing reaches JSON.parse or the authorizer.expect(otherApp.observations[0]).toMatchObject({- bodyDataSubscriptions: 0,- readBytesBeforeResponse: 0,+ bodyDataSubscriptions: 1,+ dataBytesBeforeResponse: Buffer.byteLength(rejectedBody),})expect(parse.mock.calls.map(([text]) => text)).not.toContain(rejectedBody)expect(authorizer.authorizeLlmProviderAttempt).toHaveBeenCalledOnce()- // The same refusal must also arrive from headers alone. An owner- // that first buffers the next body would stall this request.+ // A declared body above the discard cap is refused from headers+ // alone. An owner that first read that body would stall this request.const headersOnly = await send(otherApp.url,- declaredRetainedHeaders('admission-headers-only-host')+ overDiscardCapHeaders('admission-headers-only-host')).responseexpect(headersOnly.status).toBe(503)+ expect(JSON.parse(headersOnly.body)).toEqual({ error: 'authorize_capacity_exceeded' })+ expect(headersOnly.headers.connection).toBe('close')expect(otherApp.observations[1]).toMatchObject({bodyDataSubscriptions: 0,readBytesBeforeResponse: 0,+ dataBytesBeforeResponse: 0,})const malformed = '{unauthenticated while full'@@ -307,6 +337,7 @@ describe('createApp retained authorize-body ownership', () => {expect(otherApp.observations[2]).toMatchObject({bodyDataSubscriptions: 0,readBytesBeforeResponse: 0,+ dataBytesBeforeResponse: 0,})first.client.destroy()@@ -317,8 +348,8 @@ describe('createApp retained authorize-body ownership', () => {expect(JSON.parse(stillHeld.body)).toEqual({ error: 'authorize_capacity_exceeded' })expect(parse.mock.calls.map(([text]) => text)).not.toContain(rejectedBody)expect(otherApp.observations[3]).toMatchObject({- bodyDataSubscriptions: 0,- readBytesBeforeResponse: 0,+ bodyDataSubscriptions: 1,+ dataBytesBeforeResponse: Buffer.byteLength(rejectedBody),})expect(authorizer.authorizeLlmProviderAttempt).toHaveBeenCalledOnce()@@ -462,25 +493,26 @@ describe('createApp retained authorize-body ownership', () => {principals: 1,}))- rejected = send(- app.url,- headers('verified-share-host'),- '{"hostRef":"forged-third-host"}'- )+ const rejectedBody = '{"hostRef":"forged-third-host"}'+ rejected = send(app.url, headers('verified-share-host'), rejectedBody)rejectedOutcome = rejected.response.catch(error => error)await vi.waitFor(() => expect(owner.spy).toHaveBeenCalledTimes(3))expect(owner.admission?.snapshot()).toEqual({ inFlight: 1, queued: 1, principals: 1 })const refused = (await rejectedOutcome) as Replyexpect(refused.status).toBe(503)expect(JSON.parse(refused.body)).toEqual({ error: 'authorize_capacity_exceeded' })+ // The queued request is still paused and unread; the refused one was+ // discarded to its end by one counting listener and never parsed.expect(app.observations[1]).toMatchObject({bodyDataSubscriptions: 0,readBytesBeforeResponse: 0,+ dataBytesBeforeResponse: 0,})expect(app.observations[2]).toMatchObject({- bodyDataSubscriptions: 0,- readBytesBeforeResponse: 0,+ bodyDataSubscriptions: 1,+ dataBytesBeforeResponse: Buffer.byteLength(rejectedBody),})+ expect((app.observations[2].request as Request).body).toBeUndefined()expect(authorizer.authorizeLlmProviderAttempt).toHaveBeenCalledOnce()permitUnwind.resolve()@@ -568,20 +600,44 @@ describe('createApp retained authorize-body ownership', () => {}})- it('expires queued headers without reading them and preserves terminal local saturation followed by recovery', async () => {+ it('expires queued headers without parsing them, refuses them at the discard read deadline and preserves terminal local saturation followed by recovery', async () => {const started = deferred()const permitUnwind = deferred()const owner = observeHeldOwner()const nativeSetTimeout = globalThis.setTimeout- const clock = vi- .spyOn(globalThis, 'setTimeout')- .mockImplementation((fn, ms, ...args) =>- nativeSetTimeout(- fn,- ms === LLM_PROVIDER_ATTEMPT_ADMISSION_POLICY.queueWaitMs ? 40 : ms,+ // Shortens the queue wait, and only the first read deadline installed by+ // the queue expiry (the refusal's discard deadline), so the holder and the+ // recovery request keep the production read clock. Fired timers are counted.+ let queueExpirations = 0+ let shortenNextReadDeadline = false+ let discardDeadlinesFired = 0+ const clock = vi.spyOn(globalThis, 'setTimeout').mockImplementation((fn, ms, ...args) => {+ const callback = fn as (...callbackArgs: unknown[]) => void+ if (ms === LLM_PROVIDER_ATTEMPT_ADMISSION_POLICY.queueWaitMs) {+ return nativeSetTimeout(+ (...callbackArgs: unknown[]) => {+ queueExpirations += 1+ shortenNextReadDeadline = true+ callback(...callbackArgs)+ shortenNextReadDeadline = false+ },+ 40,...args)- )+ }+ if (shortenNextReadDeadline && ms === LLM_PROVIDER_ATTEMPT_ADMISSION_POLICY.readDeadlineMs) {+ shortenNextReadDeadline = false+ return nativeSetTimeout(+ (...callbackArgs: unknown[]) => {+ discardDeadlinesFired += 1+ callback(...callbackArgs)+ },+ 80,+ ...args+ )+ }+ return nativeSetTimeout(fn, ms, ...args)+ })vi.mocked(authorizer.authorizeLlmProviderAttempt).mockImplementationOnce(async (_claims, _body, deps) => {started.resolve()@@ -600,16 +656,22 @@ describe('createApp retained authorize-body ownership', () => {const firstOutcome = first.response.catch(error => error)try {await started.promise+ // Headers only: the queue expires, the refusal waits for a body that+ // never comes and answers when its read deadline fires.const expired = await send(app.url, declaredRetainedHeaders('queued-expiry-host')).response+ expect(queueExpirations).toBe(1)+ expect(discardDeadlinesFired).toBe(1)expect(expired.status).toBe(503)expect(JSON.parse(expired.body)).toEqual({ error: 'authorize_capacity_exceeded' })expect(expired.headers.connection).toBe('close')expect(expired.headers['retry-after']).toBe('10')expect(app.observations[1]).toMatchObject({- bodyDataSubscriptions: 0,+ bodyDataSubscriptions: 1,readBytesBeforeResponse: 0,+ dataBytesBeforeResponse: 0,})+ expect((app.observations[1].request as Request).body).toBeUndefined()await vi.waitFor(() => expect(app.observations[1].request.destroyed).toBe(true))expect(owner.admission?.snapshot()).toEqual({ inFlight: 1, queued: 0, principals: 1 })expect(authorizer.authorizeLlmProviderAttempt).toHaveBeenCalledOnce()@@ -768,10 +830,13 @@ describe('createApp retained authorize-body ownership', () => {).responseexpect(refused.status).toBe(503)expect(JSON.parse(refused.body)).toEqual({ error: 'authorize_capacity_exceeded' })+ // Discarded to its end by one counting listener, never parsed.expect(app.observations[1]).toMatchObject({- bodyDataSubscriptions: 0,- readBytesBeforeResponse: 0,+ bodyDataSubscriptions: 1,+ dataBytesBeforeResponse: 2,})+ expect((app.observations[1].request as Request).body).toBeUndefined()+ expect(authorizer.authorizeLlmProviderAttempt).toHaveBeenCalledOnce()permitUnwind.resolve()const response = await firstOutcome