Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
test(llm-proxy): pin one admission clock and the ticket's remaining life (#739)
Alfredo Lopez57f46227dd20
diff --git a/codex-llm-proxy/test/bodyAdmission.test.ts b/codex-llm-proxy/test/bodyAdmission.test.tsindex 037d4529f..027ab5db5 100644--- a/codex-llm-proxy/test/bodyAdmission.test.ts+++ b/codex-llm-proxy/test/bodyAdmission.test.ts@@ -34,6 +34,7 @@ import {// The admission budget holds this many bodies of the configured maximum size.IN_FLIGHT_BODY_BUDGET_BODIES as BUDGET_BODIES,STREAM_LIMITS,+ streamGate,} from '../src/requestLimits.js'import { type ProxyRuntimeDeps, createProxyApps } from '../src/server.js'@@ -49,12 +50,12 @@ const { privateKey, publicKey } = generateKeyPairSync('rsa', {const HOST_REF = 'research-host'const COMPLETIONS_PATH = '/internal/runtime/v1/codex/completions'-function sign(payload: Record<string, unknown>, audience: string): string {+function sign(payload: Record<string, unknown>, audience: string, expiresIn = 60): string {return jwt.sign(payload, privateKey, {algorithm: 'RS256',issuer: 'control-api',audience,- expiresIn: 60,+ expiresIn,})}@@ -88,8 +89,11 @@ const platformToken = sign('workflow-approvals')-/** A runtime envelope whose request carries one user message of `contentChars`. */-function completionBody(contentChars: number, attempt: string): string {+/**+ * A runtime envelope whose request carries one user message of `contentChars`.+ * The execution ticket lives `ticketLifetimeSeconds`.+ */+function completionBody(contentChars: number, attempt: string, ticketLifetimeSeconds = 60): string {const raw = {schemaVersion: 'codex-completion-request.v1',requestId: `req-${attempt}`,@@ -111,7 +115,8 @@ function completionBody(contentChars: number, attempt: string): string {requestHash,providerAttemptId: `att-${attempt}`,},- 'codex-llm-proxy'+ 'codex-llm-proxy',+ ticketLifetimeSeconds),requestHash,request: raw,@@ -641,6 +646,85 @@ describe('codex-llm-proxy authentication before admission and body-read deadline}, 30_000)})+/**+ * Polls `condition` on real time (`performance.now` is never faked here);+ * fails loudly when it does not hold within `ms`.+ */+async function until(condition: () => boolean, what: string, ms = 5_000): Promise<void> {+ const deadline = performance.now() + ms+ while (!condition()) {+ if (performance.now() > deadline) throw new Error(`${what} did not happen within ${ms} ms`)+ await sleep(10)+ }+}++/**+ * #739 D1 — one admission clock per request. Every wait a request performs is+ * bounded by the instant it arrived plus `maxQueueWaitMs`, so time spent in the+ * body budget is not granted again by the stream gate.+ */+describe('codex-llm-proxy one admission clock (#739 D1)', () => {+ // A small body limit keeps this cheap; the budget scales with it.+ const maxBodyBytes = 16 * 1024++ it('T-AC-4 answers a request that waited 50 s for the body budget 503 at 60 s from arrival, not 110 s', async () => {+ const bodyWaitMs = 50_000+ // The stalled holders are answered 408 at `bodyWaitMs`, which is what frees+ // the budget for the queued request.+ const proxy = await heldProxy(maxBodyBytes, { bodyReadDeadlineMs: bodyWaitMs })+ proxy.releaseAll()+ vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout', 'Date'] })+ const stalled = Array.from({ length: BUDGET_BODIES }, () =>+ stalledPost(proxy.port, { declaredBytes: maxBodyBytes, token: platformToken })+ )+ const slots: Array<() => void> = []+ let queued: ReturnType<typeof stalledPost> | undefined+ try {+ await until(() => proxy.arrived() >= BUDGET_BODIES, 'the stalled holders arriving')+ await sleep(50)+ for (let i = 0; i < STREAM_LIMITS.maxConcurrentStreams; i += 1) {+ slots.push(await streamGate.acquire())+ }+ // The ticket outlives every wait below, so only the admission clock can+ // end this request.+ const body = completionBody(12_000, 'one-clock', 300)+ expect(Buffer.byteLength(body)).toBeLessThanOrEqual(maxBodyBytes)+ const arrivedAt = Date.now()+ queued = stalledPost(proxy.port, {+ declaredBytes: Buffer.byteLength(body),+ token: platformToken,+ sent: body,+ })+ await until(() => proxy.arrived() >= BUDGET_BODIES + 1, 'the queued request arriving')+ await sleep(50)++ await vi.advanceTimersByTimeAsync(bodyWaitMs)+ for (const reply of await Promise.all(stalled.map(s => within(s.reply, 2_000)))) {+ expect(reply.status).toBe(408)+ }+ // Witness: the queued body was granted, parsed and verified, and now+ // waits at the saturated stream gate.+ await until(() => streamGate.snapshot().queued === 1, 'the request queueing at the stream gate')++ await vi.advanceTimersByTimeAsync(STREAM_LIMITS.maxQueueWaitMs - bodyWaitMs - 1)+ expect((await within(queued.reply, 200)).status).toBe(0)+ await vi.advanceTimersByTimeAsync(1)+ const refused = await within(queued.reply, 2_000)+ expect(refused.status).toBe(503)+ expect(JSON.parse(refused.body)).toEqual({ error: 'provider_unavailable' })+ expect(Date.now() - arrivedAt).toBe(STREAM_LIMITS.maxQueueWaitMs)+ expect(streamGate.snapshot().queued).toBe(0)+ expect(proxy.redeemed()).toBe(0)+ } finally {+ vi.useRealTimers()+ for (const s of stalled) s.destroy()+ queued?.destroy()+ for (const release of slots) release()+ await proxy.close()+ }+ }, 30_000)+})+/*** POST `body` with the given `Content-Encoding`. Keep-alive, so a reply sent* before the body is read is not raced by the client's own write.diff --git a/codex-llm-proxy/test/bodyBudget.test.ts b/codex-llm-proxy/test/bodyBudget.test.tsindex b8b18f7da..fe238ce46 100644--- a/codex-llm-proxy/test/bodyBudget.test.ts+++ b/codex-llm-proxy/test/bodyBudget.test.ts@@ -1,5 +1,11 @@-import { describe, expect, it } from 'vitest'-import { BodyBudget, RequestLimitError } from '../src/requestLimits.js'+import { describe, expect, it, vi } from 'vitest'+import { LIMITS } from '@clerum/llm-provider-attempt-contract'+import {+ BodyBudget,+ DEFAULT_MAX_BODY_BYTES,+ IN_FLIGHT_BODY_BUDGET_BODIES,+ RequestLimitError,+} from '../src/requestLimits.js'async function isPending(promise: Promise<unknown>): Promise<boolean> {const marker = Symbol('pending')@@ -68,3 +74,78 @@ describe('BodyBudget (#731 R3-2)', () => {expect(budget.inFlightBytes).toBe(10)})})++/**+ * #739 D1 — a queued body waits only until the request's admission deadline,+ * the same instant every later wait of that request is bounded by.+ */+describe('BodyBudget admission deadline (#739 D1)', () => {+ it('T-AC-1 rejects a waiter blocked behind three full grants at its deadline and frees its queue slot', async () => {+ vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout', 'Date'] })+ try {+ const budget = new BodyBudget(30, 4)+ await budget.acquire(10)+ await budget.acquire(10)+ await budget.acquire(10)+ const deadlineAt = Date.now() + 1_000+ let outcome: unknown+ budget.acquire(10, undefined, deadlineAt).then(+ () => {+ outcome = 'granted'+ },+ (err: unknown) => {+ outcome = err+ }+ )+ // Witness: the body really queued behind the full budget.+ expect(budget.queued).toBe(1)+ await vi.advanceTimersByTimeAsync(999)+ expect(outcome).toBeUndefined()+ expect(budget.queued).toBe(1)+ await vi.advanceTimersByTimeAsync(1)+ expect(outcome).toBeInstanceOf(RequestLimitError)+ expect((outcome as Error).message).toBe('body admission wait exceeded')+ expect(budget.queued).toBe(0)+ expect(budget.inFlightBytes).toBe(30)+ } finally {+ vi.useRealTimers()+ }+ })++ it('T-AC-2 ends head-of-line blocking: at the head waiter deadline the smaller body behind it is granted', async () => {+ vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout', 'Date'] })+ try {+ const budget = new BodyBudget(IN_FLIGHT_BODY_BUDGET_BODIES * DEFAULT_MAX_BODY_BYTES)+ for (let i = 0; i < IN_FLIGHT_BODY_BUDGET_BODIES; i += 1) {+ await budget.acquire(LIMITS.maxRequestBodyBytes)+ }+ const full = IN_FLIGHT_BODY_BUDGET_BODIES * LIMITS.maxRequestBodyBytes+ // Fixture check: 48 KiB are left, so 64 KiB waits and 16 KiB would fit.+ expect(IN_FLIGHT_BODY_BUDGET_BODIES * DEFAULT_MAX_BODY_BYTES - full).toBe(48 * 1024)+ let headOutcome: unknown+ budget.acquire(64 * 1024, undefined, Date.now() + 1_000).then(+ () => {+ headOutcome = 'granted'+ },+ (err: unknown) => {+ headOutcome = err+ }+ )+ let grantedWith: number | undefined+ void budget.acquire(16 * 1024, undefined, Date.now() + 60_000).then(() => {+ grantedWith = budget.inFlightBytes+ })+ expect(budget.queued).toBe(2)+ await vi.advanceTimersByTimeAsync(999)+ // FIFO: the 16 KiB body still waits behind the head although it fits.+ expect(grantedWith).toBeUndefined()+ await vi.advanceTimersByTimeAsync(1)+ expect(headOutcome).toBeInstanceOf(RequestLimitError)+ expect((headOutcome as Error).message).toBe('body admission wait exceeded')+ expect(grantedWith).toBe(full + 16 * 1024)+ expect(budget.queued).toBe(0)+ } finally {+ vi.useRealTimers()+ }+ })+})diff --git a/codex-llm-proxy/test/executionTicketVerifier.test.ts b/codex-llm-proxy/test/executionTicketVerifier.test.tsnew file mode 100644index 000000000..ef29ce2b0--- /dev/null+++ b/codex-llm-proxy/test/executionTicketVerifier.test.ts@@ -0,0 +1,40 @@+import { generateKeyPairSync } from 'node:crypto'+import jwt from 'jsonwebtoken'+import { describe, expect, it } from 'vitest'+import { verifyExecutionTicket } from '../src/auth/executionTicketVerifier.js'+import type { CodexLlmProxyConfig } from '../src/config.js'++const { privateKey, publicKey } = generateKeyPairSync('rsa', {+ modulusLength: 2048,+ publicKeyEncoding: { type: 'spki', format: 'pem' },+ privateKeyEncoding: { type: 'pkcs8', format: 'pem' },+})++const verifierConfig = { jwtIssuer: 'control-api', jwtPublicKey: publicKey } as CodexLlmProxyConfig++const CLAIMS = {+ jti: '66666666-6666-4666-8666-666666666666',+ typ: 'codex-execution-ticket',+ hostRef: 'research-host',+ model: 'gpt-5.1',+ requestHash: 'c'.repeat(64),+ providerAttemptId: 'att-verifier',+}++function signTicket(payload: Record<string, unknown>): string {+ return jwt.sign(payload, privateKey, {+ algorithm: 'RS256',+ issuer: 'control-api',+ audience: 'codex-llm-proxy',+ })+}++describe('verifyExecutionTicket (#739 D1-bis)', () => {+ it('T-AC-6 returns the signed exp as expiresAtMs and still refuses a ticket without exp', () => {+ const exp = Math.floor(Date.now() / 1000) + 45+ const verified = verifyExecutionTicket(signTicket({ ...CLAIMS, exp }), verifierConfig)+ expect(verified).toEqual({ ...CLAIMS, expiresAtMs: exp * 1000 })++ expect(verifyExecutionTicket(signTicket(CLAIMS), verifierConfig)).toBeNull()+ })+})diff --git a/codex-llm-proxy/test/requestLimits.test.ts b/codex-llm-proxy/test/requestLimits.test.tsindex b06dc6f84..f20c6b198 100644--- a/codex-llm-proxy/test/requestLimits.test.ts+++ b/codex-llm-proxy/test/requestLimits.test.ts@@ -14,22 +14,23 @@ import { DEFAULT_HEARTBEAT_INTERVAL_MS, MAX_HEARTBEAT_INTERVAL_MS } from '../srcconst HOST_UNDICI_TIMEOUT_MS = 300_000describe('stream timing invariant', () => {- it('reaches the first keepalive before the Host HTTP client times out', () => {+ it('T-KI-1 reaches the first keepalive before the Host HTTP client times out', () => {expect(STREAM_LIMITS.maxStreamDurationMs).toBe(1_800_000)expect(STREAM_LIMITS.maxQueueWaitMs).toBe(60_000)+ expect(CONTROL_API_REQUEST_TIMEOUT_MS).toBe(15_000)// Nothing is written while a request waits for a slot or for the redeem,// so both must end, and one heartbeat interval pass, inside the timeout.- // A visual request can wait twice: on visualStreamGate before the body is- // parsed, then on streamGate when the parsed body no longer needs the- // visual slot. Both gates share the maxQueueWaitMs default. The interval- // is configurable, so the largest one config accepts is the one pinned.+ // Every wait a request performs (the body budget, the visual gate, the+ // stream gate) is bounded by one admission clock of maxQueueWaitMs from+ // arrival (#739 D1), so the queue time counts once. The interval is+ // configurable, so the largest one config accepts is the one pinned.expect(MAX_HEARTBEAT_INTERVAL_MS).toBe(60_000)expect(DEFAULT_HEARTBEAT_INTERVAL_MS).toBeLessThanOrEqual(MAX_HEARTBEAT_INTERVAL_MS)- expect(- 2 * STREAM_LIMITS.maxQueueWaitMs +- CONTROL_API_REQUEST_TIMEOUT_MS +- MAX_HEARTBEAT_INTERVAL_MS- ).toBeLessThan(HOST_UNDICI_TIMEOUT_MS)+ const firstKeepaliveBy =+ STREAM_LIMITS.maxQueueWaitMs + CONTROL_API_REQUEST_TIMEOUT_MS + MAX_HEARTBEAT_INTERVAL_MS+ expect(firstKeepaliveBy).toBe(135_000)+ expect(HOST_UNDICI_TIMEOUT_MS).toBe(300_000)+ expect(firstKeepaliveBy).toBeLessThan(HOST_UNDICI_TIMEOUT_MS)})})@@ -120,6 +121,26 @@ describe('StreamGate', () => {releaseNext()})+ it('T-AC-3 rejects a waiter at its admission deadline when that comes before maxQueueWaitMs', async () => {+ const gate = new StreamGate(1, 1, 60_000)+ const release = await gate.acquire()+ const started = Date.now()+ const queued = gate.acquire(undefined, started + 50)+ // Witness: the waiter really queued, it was not refused at once.+ expect(gate.snapshot().queued).toBe(1)+ await expect(+ Promise.race([+ queued,+ new Promise<never>((_, reject) =>+ setTimeout(() => reject(new Error('the admission deadline did not end the wait')), 1_000)+ ),+ ])+ ).rejects.toMatchObject({ name: 'RequestLimitError', message: 'stream queue wait exceeded' })+ expect(Date.now() - started).toBeGreaterThanOrEqual(45)+ expect(gate.snapshot().queued).toBe(0)+ release()+ })+it('pins the visual 2/8 sibling against the ordinary 8/16 stream gate', () => {expect(VISUAL_STREAM_LIMITS).toEqual({ maxConcurrentStreams: 2, maxQueuedRequests: 8 })expect(STREAM_LIMITS.maxConcurrentStreams).toBe(8)diff --git a/codex-llm-proxy/test/server.security.test.ts b/codex-llm-proxy/test/server.security.test.tsindex 4244d31b8..441fc4afa 100644--- a/codex-llm-proxy/test/server.security.test.ts+++ b/codex-llm-proxy/test/server.security.test.ts@@ -23,6 +23,7 @@ import {DEFAULT_MAX_BODY_BYTES,ENVELOPE_ALLOWANCE_BYTES,RequestLimitError,+ STREAM_LIMITS,streamGate,} from '../src/requestLimits.js'import { createProxyApps } from '../src/server.js'@@ -1385,3 +1386,226 @@ describe('codex-llm-proxy attempt telemetry', () => {}})})++/**+ * #739 D1-bis — the stream-gate wait also ends when the execution ticket dies.+ * A request still queued at the ticket's `exp` could only be redeemed into a+ * certain `ticket_expired`, so it is answered as a capacity refusal instead,+ * with no redeem. The margin is zero: a request whose ticket is still alive+ * when a slot frees is served exactly as before.+ */+describe('codex-llm-proxy ticket-aware stream-gate wait (#739 D1-bis)', () => {+ /** Captured before any test fakes the clock, so waits stay on real time. */+ const realSetTimeout = globalThis.setTimeout+ const realSleep = (ms: number) => new Promise<void>(resolve => realSetTimeout(resolve, ms))+ const lookup = async () => [{ address: '1.2.3.4', family: 4 }]+ const COMPLETIONS = '/internal/runtime/v1/codex/completions'++ async function until(condition: () => boolean, what: string): Promise<void> {+ const deadline = performance.now() + 5_000+ while (!condition()) {+ if (performance.now() > deadline) throw new Error(`${what} did not happen within 5 s`)+ await realSleep(10)+ }+ }++ function withinReal<T>(promise: Promise<T>, ms: number): Promise<T | undefined> {+ return Promise.race([promise, realSleep(ms).then(() => undefined)])+ }++ /** An envelope whose ticket expires exactly `ticketLifeMs` from the (fake) now. */+ function envelope(providerAttemptId: string, ticketLifeMs: number): Record<string, unknown> {+ const raw: Record<string, unknown> = {+ schemaVersion: 'codex-completion-request.v1',+ requestId: `req-${providerAttemptId}`,+ idempotencyKey: `idem-${providerAttemptId}`,+ provider: 'codex-subscription',+ model: 'gpt-5.1',+ messages: [{ role: 'user', content: 'hello' }],+ }+ const parsed = parseCodexCompletionRequestV1(raw)+ if (!parsed.ok) throw new Error(parsed.message)+ const requestHash = hashCodexCompletionRequestV1(parsed.value)+ const executionTicket = jwt.sign(+ {+ jti: '77777777-7777-4777-8777-777777777777',+ typ: 'codex-execution-ticket',+ hostRef: 'research-host',+ model: 'gpt-5.1',+ requestHash,+ providerAttemptId,+ exp: (Date.now() + ticketLifeMs) / 1000,+ },+ privateKey,+ { algorithm: 'RS256', issuer: 'control-api', audience: 'codex-llm-proxy' }+ )+ return { executionTicket, requestHash, request: raw }+ }++ function countingClient(redeemAnswer: 'granted' | 'ticket_expired'): {+ client: ControlApiClient+ redeems: () => number+ } {+ let redeems = 0+ const client = {+ async redeem(): Promise<RedeemAttemptSuccess> {+ redeems += 1+ if (redeemAnswer === 'ticket_expired') {+ throw new ControlApiClientError('ticket_expired', 'control API request denied')+ }+ return {+ accessToken: 'test-access-ticket-life',+ chatgptAccountId: 'acct-ticket-life',+ transport: {+ protocolVersion: 'codex-subscription-transport.v1',+ completionsOrigin: CODEX_COMPLETIONS_ORIGIN,+ catalogOrigin: 'https://chatgpt.com/backend-api/codex/models?client_version=1.0.0',+ operation: 'completion_stream',+ servedModel: 'gpt-5.1',+ maxStreamDurationMs: 1_800_000,+ },+ expiryClass: 'short_lived',+ attemptReceipt: 'd'.repeat(64),+ }+ },+ async finalize(input: {+ receipt: { providerAttemptId: string; outcome: FinalizeAttemptSuccess['outcome'] }+ }): Promise<FinalizeAttemptSuccess> {+ return {+ providerAttemptId: input.receipt.providerAttemptId,+ outcome: input.receipt.outcome,+ duplicate: false,+ }+ },+ } as unknown as ControlApiClient+ return { client, redeems: () => redeems }+ }++ const upstream = (async () =>+ new Response(+ 'data: {"type":"response.output_text.delta","delta":"t0"}\n\n' ++ 'data: {"type":"response.completed"}\n\n',+ { status: 200, headers: { 'content-type': 'text/event-stream' } }+ )) as typeof fetch++ function failureCount(metricsText: string, code: string): number {+ const line = metricsText+ .split('\n')+ .find(row => row.startsWith(`codex_proxy_attempt_failures_total{code="${code}"}`))+ return line ? Number(line.split(' ').pop()) : 0+ }++ /** Fakes the clock on a whole second, so a ticket's `exp` lands on an exact instant. */+ function fakeClock(): void {+ vi.useFakeTimers({+ now: Math.ceil(Date.now() / 1000) * 1000,+ toFake: ['setTimeout', 'clearTimeout', 'Date'],+ })+ }++ /**+ * Frees the held slots and lets any waiter still polling the shared gate+ * finish on the fake clock, so the module gate is empty for the next test.+ */+ async function releaseAndDrain(slots: Array<() => void>): Promise<void> {+ for (const release of slots.splice(0)) release()+ await vi.advanceTimersByTimeAsync(STREAM_LIMITS.maxQueueWaitMs)+ vi.useRealTimers()+ }++ async function saturate(): Promise<Array<() => void>> {+ const slots: Array<() => void> = []+ for (let i = 0; i < STREAM_LIMITS.maxConcurrentStreams; i += 1) {+ slots.push(await streamGate.acquire())+ }+ return slots+ }++ it('T-AC-5 answers 503 provider_unavailable at the ticket expiry, without a redeem, while the gate stays saturated', async () => {+ fakeClock()+ const warn = vi.spyOn(logger, 'warn')+ let slots: Array<() => void> = []+ try {+ slots = await saturate()+ const control = countingClient('ticket_expired')+ const apps = createProxyApps(config({ maxBodyBytes: 65_536 }), {+ controlApiClient: control.client,+ fetchFn: upstream,+ lookup,+ })+ const ticketLifeMs = 20_000+ const reply = request(apps.runtimeApp)+ .post(COMPLETIONS)+ .set('Authorization', `Bearer ${platformToken()}`)+ .send(envelope('att-ticket-life', ticketLifeMs))+ .then(res => res)+ await until(() => streamGate.snapshot().queued === 1, 'the request queueing at the stream gate')++ await vi.advanceTimersByTimeAsync(ticketLifeMs - 1)+ // Witness: the request waited for the ticket's whole life, it was not+ // refused on arrival.+ expect(await withinReal(reply, 100)).toBeUndefined()+ await vi.advanceTimersByTimeAsync(1)+ const res = await withinReal(reply, 2_000)+ expect(res?.status).toBe(503)+ expect(res?.body).toEqual({ error: 'provider_unavailable' })+ expect(streamGate.snapshot().queued).toBe(0)++ // A slot that frees after the ticket died is never used to redeem it.+ slots.pop()?.()+ await vi.advanceTimersByTimeAsync(STREAM_LIMITS.maxQueueWaitMs)+ expect(control.redeems()).toBe(0)++ const refusals = warn.mock.calls+ .map(call => call[0] as unknown as Record<string, unknown>)+ .filter(entry => entry?.event === 'codex_proxy_admission_refused')+ expect(refusals).toEqual([+ {+ event: 'codex_proxy_admission_refused',+ reason: 'ticket_life',+ providerAttemptId: 'att-ticket-life',+ hostRef: 'research-host',+ },+ ])+ const metricsText = (await request(apps.probeApp).get('/metrics')).text+ expect(failureCount(metricsText, 'provider_unavailable')).toBe(1)+ expect(failureCount(metricsText, 'request_limit')).toBe(0)+ } finally {+ await releaseAndDrain(slots)+ warn.mockRestore()+ }+ }, 30_000)++ it('T-PAR-TK serves a request whose ticket is still alive when the gate frees at +50 s of a 60 s ticket', async () => {+ fakeClock()+ let slots: Array<() => void> = []+ try {+ slots = await saturate()+ const control = countingClient('granted')+ const apps = createProxyApps(config({ maxBodyBytes: 65_536 }), {+ controlApiClient: control.client,+ fetchFn: upstream,+ lookup,+ })+ const reply = request(apps.runtimeApp)+ .post(COMPLETIONS)+ .set('Authorization', `Bearer ${platformToken()}`)+ .send(envelope('att-ticket-alive', 60_000))+ .then(res => res)+ await until(() => streamGate.snapshot().queued === 1, 'the request queueing at the stream gate')++ await vi.advanceTimersByTimeAsync(50_000)+ expect(await withinReal(reply, 100)).toBeUndefined()+ slots.pop()?.()+ // One stream-gate poll interval.+ await vi.advanceTimersByTimeAsync(10)+ const res = await withinReal(reply, 2_000)+ expect(res?.status).toBe(200)+ expect(res?.text).toContain('data: {"type":"text","text":"t0"}')+ expect(res?.text).toContain('data: {"type":"done","outcome":"success"}')+ expect(control.redeems()).toBe(1)+ } finally {+ await releaseAndDrain(slots)+ }+ }, 30_000)+})diff --git a/grok-llm-proxy/test/bodyAdmission.test.ts b/grok-llm-proxy/test/bodyAdmission.test.tsindex 50897da2a..3eb572d25 100644--- a/grok-llm-proxy/test/bodyAdmission.test.ts+++ b/grok-llm-proxy/test/bodyAdmission.test.ts@@ -34,6 +34,7 @@ import {// The admission budget holds this many bodies of the configured maximum size.IN_FLIGHT_BODY_BUDGET_BODIES as BUDGET_BODIES,STREAM_LIMITS,+ streamGate,} from '../src/requestLimits.js'import { type ProxyRuntimeDeps, createProxyApps } from '../src/server.js'@@ -49,12 +50,12 @@ const { privateKey, publicKey } = generateKeyPairSync('rsa', {const HOST_REF = 'research-host'const COMPLETIONS_PATH = '/internal/runtime/v1/grok/completions'-function sign(payload: Record<string, unknown>, audience: string): string {+function sign(payload: Record<string, unknown>, audience: string, expiresIn = 60): string {return jwt.sign(payload, privateKey, {algorithm: 'RS256',issuer: 'control-api',audience,- expiresIn: 60,+ expiresIn,})}@@ -87,8 +88,11 @@ const platformToken = sign('workflow-approvals')-/** A runtime envelope whose request carries one user message of `contentChars`. */-function completionBody(contentChars: number, attempt: string): string {+/**+ * A runtime envelope whose request carries one user message of `contentChars`.+ * The execution ticket lives `ticketLifetimeSeconds`.+ */+function completionBody(contentChars: number, attempt: string, ticketLifetimeSeconds = 60): string {const raw = {schemaVersion: 'grok-completion-request.v1',requestId: `req-${attempt}`,@@ -110,7 +114,8 @@ function completionBody(contentChars: number, attempt: string): string {requestHash,providerAttemptId: `att-${attempt}`,},- 'grok-llm-proxy'+ 'grok-llm-proxy',+ ticketLifetimeSeconds),requestHash,request: raw,@@ -637,6 +642,89 @@ describe('grok-llm-proxy authentication before admission and body-read deadline}, 30_000)})+/**+ * Polls `condition` on real time (`performance.now` is never faked here);+ * fails loudly when it does not hold within `ms`.+ */+async function until(condition: () => boolean, what: string, ms = 5_000): Promise<void> {+ const deadline = performance.now() + ms+ while (!condition()) {+ if (performance.now() > deadline) throw new Error(`${what} did not happen within ${ms} ms`)+ await sleep(10)+ }+}++/**+ * #739 D1 — one admission clock per request. Every wait a request performs is+ * bounded by the instant it arrived plus `maxQueueWaitMs`, so time spent in the+ * body budget is not granted again by the stream gate.+ */+describe('grok-llm-proxy one admission clock (#739 D1)', () => {+ // A small body limit keeps this cheap; the budget scales with it.+ const maxBodyBytes = 16 * 1024++ it('T-AC-4-grok answers a request that waited 50 s for the body budget 503 at 60 s from arrival, not 110 s', async () => {+ const bodyWaitMs = 50_000+ // The stalled holders are answered 408 at `bodyWaitMs`, which is what frees+ // the budget for the queued request.+ const proxy = await heldProxy(maxBodyBytes, { bodyReadDeadlineMs: bodyWaitMs })+ proxy.releaseAll()+ vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout', 'Date'] })+ const stalled = Array.from({ length: BUDGET_BODIES }, () =>+ stalledPost(proxy.port, { declaredBytes: maxBodyBytes, token: platformToken })+ )+ const slots: Array<() => void> = []+ let queued: ReturnType<typeof stalledPost> | undefined+ let acquire: ReturnType<typeof vi.spyOn> | undefined+ try {+ await until(() => proxy.arrived() >= BUDGET_BODIES, 'the stalled holders arriving')+ await sleep(50)+ for (let i = 0; i < STREAM_LIMITS.maxConcurrentStreams; i += 1) {+ slots.push(await streamGate.acquire())+ }+ // Pass-through spy: it only counts the handler's calls into the gate.+ acquire = vi.spyOn(streamGate, 'acquire')+ // The ticket outlives every wait below, so only the admission clock can+ // end this request.+ const body = completionBody(12_000, 'one-clock', 300)+ expect(Buffer.byteLength(body)).toBeLessThanOrEqual(maxBodyBytes)+ const arrivedAt = Date.now()+ queued = stalledPost(proxy.port, {+ declaredBytes: Buffer.byteLength(body),+ token: platformToken,+ sent: body,+ })+ await until(() => proxy.arrived() >= BUDGET_BODIES + 1, 'the queued request arriving')+ await sleep(50)++ await vi.advanceTimersByTimeAsync(bodyWaitMs)+ for (const reply of await Promise.all(stalled.map(s => within(s.reply, 2_000)))) {+ expect(reply.status).toBe(408)+ }+ // Witness: the queued body was granted, parsed and verified, and now+ // waits at the saturated stream gate.+ const spy = acquire+ await until(() => spy.mock.calls.length === 1, 'the request queueing at the stream gate')++ await vi.advanceTimersByTimeAsync(STREAM_LIMITS.maxQueueWaitMs - bodyWaitMs - 1)+ expect((await within(queued.reply, 200)).status).toBe(0)+ await vi.advanceTimersByTimeAsync(1)+ const refused = await within(queued.reply, 2_000)+ expect(refused.status).toBe(503)+ expect(JSON.parse(refused.body)).toEqual({ error: 'provider_unavailable' })+ expect(Date.now() - arrivedAt).toBe(STREAM_LIMITS.maxQueueWaitMs)+ expect(proxy.redeemed()).toBe(0)+ } finally {+ acquire?.mockRestore()+ vi.useRealTimers()+ for (const s of stalled) s.destroy()+ queued?.destroy()+ for (const release of slots) release()+ await proxy.close()+ }+ }, 30_000)+})+/*** POST `body` with the given `Content-Encoding`. Keep-alive, so a reply sent* before the body is read is not raced by the client's own write.diff --git a/grok-llm-proxy/test/bodyBudget.test.ts b/grok-llm-proxy/test/bodyBudget.test.tsindex c4878ebd9..7bfc89c83 100644--- a/grok-llm-proxy/test/bodyBudget.test.ts+++ b/grok-llm-proxy/test/bodyBudget.test.ts@@ -1,5 +1,11 @@-import { describe, expect, it } from 'vitest'-import { BodyBudget, RequestLimitError } from '../src/requestLimits.js'+import { describe, expect, it, vi } from 'vitest'+import { LIMITS } from '@clerum/grok-provider-attempt-contract'+import {+ BodyBudget,+ DEFAULT_MAX_BODY_BYTES,+ IN_FLIGHT_BODY_BUDGET_BODIES,+ RequestLimitError,+} from '../src/requestLimits.js'async function isPending(promise: Promise<unknown>): Promise<boolean> {const marker = Symbol('pending')@@ -68,3 +74,78 @@ describe('BodyBudget (#731 R3-2)', () => {expect(budget.inFlightBytes).toBe(10)})})++/**+ * #739 D1 — a queued body waits only until the request's admission deadline,+ * the same instant every later wait of that request is bounded by.+ */+describe('BodyBudget admission deadline (#739 D1)', () => {+ it('T-AC-1-grok rejects a waiter blocked behind three full grants at its deadline and frees its queue slot', async () => {+ vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout', 'Date'] })+ try {+ const budget = new BodyBudget(30, 4)+ await budget.acquire(10)+ await budget.acquire(10)+ await budget.acquire(10)+ const deadlineAt = Date.now() + 1_000+ let outcome: unknown+ budget.acquire(10, undefined, deadlineAt).then(+ () => {+ outcome = 'granted'+ },+ (err: unknown) => {+ outcome = err+ }+ )+ // Witness: the body really queued behind the full budget.+ expect(budget.queued).toBe(1)+ await vi.advanceTimersByTimeAsync(999)+ expect(outcome).toBeUndefined()+ expect(budget.queued).toBe(1)+ await vi.advanceTimersByTimeAsync(1)+ expect(outcome).toBeInstanceOf(RequestLimitError)+ expect((outcome as Error).message).toBe('body admission wait exceeded')+ expect(budget.queued).toBe(0)+ expect(budget.inFlightBytes).toBe(30)+ } finally {+ vi.useRealTimers()+ }+ })++ it('T-AC-2-grok ends head-of-line blocking: at the head waiter deadline the smaller body behind it is granted', async () => {+ vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout', 'Date'] })+ try {+ const budget = new BodyBudget(IN_FLIGHT_BODY_BUDGET_BODIES * DEFAULT_MAX_BODY_BYTES)+ for (let i = 0; i < IN_FLIGHT_BODY_BUDGET_BODIES; i += 1) {+ await budget.acquire(LIMITS.maxRequestBodyBytes)+ }+ const full = IN_FLIGHT_BODY_BUDGET_BODIES * LIMITS.maxRequestBodyBytes+ // Fixture check: 48 KiB are left, so 64 KiB waits and 16 KiB would fit.+ expect(IN_FLIGHT_BODY_BUDGET_BODIES * DEFAULT_MAX_BODY_BYTES - full).toBe(48 * 1024)+ let headOutcome: unknown+ budget.acquire(64 * 1024, undefined, Date.now() + 1_000).then(+ () => {+ headOutcome = 'granted'+ },+ (err: unknown) => {+ headOutcome = err+ }+ )+ let grantedWith: number | undefined+ void budget.acquire(16 * 1024, undefined, Date.now() + 60_000).then(() => {+ grantedWith = budget.inFlightBytes+ })+ expect(budget.queued).toBe(2)+ await vi.advanceTimersByTimeAsync(999)+ // FIFO: the 16 KiB body still waits behind the head although it fits.+ expect(grantedWith).toBeUndefined()+ await vi.advanceTimersByTimeAsync(1)+ expect(headOutcome).toBeInstanceOf(RequestLimitError)+ expect((headOutcome as Error).message).toBe('body admission wait exceeded')+ expect(grantedWith).toBe(full + 16 * 1024)+ expect(budget.queued).toBe(0)+ } finally {+ vi.useRealTimers()+ }+ })+})diff --git a/grok-llm-proxy/test/executionTicketVerifier.test.ts b/grok-llm-proxy/test/executionTicketVerifier.test.tsnew file mode 100644index 000000000..e315d0467--- /dev/null+++ b/grok-llm-proxy/test/executionTicketVerifier.test.ts@@ -0,0 +1,40 @@+import { generateKeyPairSync } from 'node:crypto'+import jwt from 'jsonwebtoken'+import { describe, expect, it } from 'vitest'+import { verifyExecutionTicket } from '../src/auth/executionTicketVerifier.js'+import type { GrokLlmProxyConfig } from '../src/config.js'++const { privateKey, publicKey } = generateKeyPairSync('rsa', {+ modulusLength: 2048,+ publicKeyEncoding: { type: 'spki', format: 'pem' },+ privateKeyEncoding: { type: 'pkcs8', format: 'pem' },+})++const verifierConfig = { jwtIssuer: 'control-api', jwtPublicKey: publicKey } as GrokLlmProxyConfig++const CLAIMS = {+ jti: '77777777-7777-4777-8777-777777777777',+ typ: 'grok-execution-ticket',+ hostRef: 'research-host',+ model: 'grok-4.6',+ requestHash: 'c'.repeat(64),+ providerAttemptId: 'att-verifier',+}++function signTicket(payload: Record<string, unknown>): string {+ return jwt.sign(payload, privateKey, {+ algorithm: 'RS256',+ issuer: 'control-api',+ audience: 'grok-llm-proxy',+ })+}++describe('verifyExecutionTicket (#739 D1-bis)', () => {+ it('T-AC-6-grok returns the signed exp as expiresAtMs and still refuses a ticket without exp', () => {+ const exp = Math.floor(Date.now() / 1000) + 45+ const verified = verifyExecutionTicket(signTicket({ ...CLAIMS, exp }), verifierConfig)+ expect(verified).toEqual({ ...CLAIMS, expiresAtMs: exp * 1000 })++ expect(verifyExecutionTicket(signTicket(CLAIMS), verifierConfig)).toBeNull()+ })+})diff --git a/grok-llm-proxy/test/requestLimits.test.ts b/grok-llm-proxy/test/requestLimits.test.tsindex dcee1d5f6..593970115 100644--- a/grok-llm-proxy/test/requestLimits.test.ts+++ b/grok-llm-proxy/test/requestLimits.test.ts@@ -13,18 +13,23 @@ import { DEFAULT_HEARTBEAT_INTERVAL_MS, MAX_HEARTBEAT_INTERVAL_MS } from '../srcconst HOST_UNDICI_TIMEOUT_MS = 300_000describe('stream timing invariant', () => {- it('reaches the first keepalive before the Host HTTP client times out', () => {+ it('T-KI-1-grok reaches the first keepalive before the Host HTTP client times out', () => {expect(STREAM_LIMITS.maxStreamDurationMs).toBe(1_800_000)expect(STREAM_LIMITS.maxQueueWaitMs).toBe(60_000)+ expect(CONTROL_API_REQUEST_TIMEOUT_MS).toBe(15_000)// Nothing is written while a request waits for a slot or for the redeem,// so both must end, and one heartbeat interval pass, inside the timeout.- // The interval is configurable, so the largest one config accepts is the- // one pinned.+ // Every wait a request performs (the body budget, the stream gate) is+ // bounded by one admission clock of maxQueueWaitMs from arrival (#739 D1),+ // so the queue time counts once. The interval is configurable, so the+ // largest one config accepts is the one pinned.expect(MAX_HEARTBEAT_INTERVAL_MS).toBe(60_000)expect(DEFAULT_HEARTBEAT_INTERVAL_MS).toBeLessThanOrEqual(MAX_HEARTBEAT_INTERVAL_MS)- expect(+ const firstKeepaliveBy =STREAM_LIMITS.maxQueueWaitMs + CONTROL_API_REQUEST_TIMEOUT_MS + MAX_HEARTBEAT_INTERVAL_MS- ).toBeLessThan(HOST_UNDICI_TIMEOUT_MS)+ expect(firstKeepaliveBy).toBe(135_000)+ expect(HOST_UNDICI_TIMEOUT_MS).toBe(300_000)+ expect(firstKeepaliveBy).toBeLessThan(HOST_UNDICI_TIMEOUT_MS)})})@@ -114,4 +119,30 @@ describe('StreamGate', () => {const releaseNext = await nextreleaseNext()})++ it('T-AC-3-grok rejects a waiter at its admission deadline when that comes before maxQueueWaitMs', async () => {+ const gate = new StreamGate(1, 1, 60_000)+ const release = await gate.acquire()+ const started = Date.now()+ const queued = gate.acquire(undefined, started + 50)+ // Witness: the waiter really queued, so the one queue slot is taken.+ await expect(gate.acquire()).rejects.toMatchObject({+ name: 'RequestLimitError',+ message: 'stream queue is full',+ })+ await expect(+ Promise.race([+ queued,+ new Promise<never>((_, reject) =>+ setTimeout(() => reject(new Error('the admission deadline did not end the wait')), 1_000)+ ),+ ])+ ).rejects.toMatchObject({ name: 'RequestLimitError', message: 'stream queue wait exceeded' })+ expect(Date.now() - started).toBeGreaterThanOrEqual(45)+ // The queue slot is free again, so a new waiter queues instead of "queue is full".+ const next = gate.acquire()+ release()+ const releaseNext = await next+ releaseNext()+ })})diff --git a/grok-llm-proxy/test/server.security.test.ts b/grok-llm-proxy/test/server.security.test.tsindex de67dd4d8..9d7cc7d32 100644--- a/grok-llm-proxy/test/server.security.test.ts+++ b/grok-llm-proxy/test/server.security.test.ts@@ -20,7 +20,12 @@ import {import { MAX_TOOL_CALL_ARGUMENT_BYTES } from '../src/grokTransport.js'import { REDACT_PATHS, logger } from '../src/logger.js'import { GROK_CATALOG_ORIGIN, GROK_COMPLETIONS_ORIGIN } from '../src/originPolicy.js'-import { ENVELOPE_ALLOWANCE_BYTES, RequestLimitError, streamGate } from '../src/requestLimits.js'+import {+ ENVELOPE_ALLOWANCE_BYTES,+ RequestLimitError,+ STREAM_LIMITS,+ streamGate,+} from '../src/requestLimits.js'import { createProxyApps } from '../src/server.js'const { privateKey, publicKey } = generateKeyPairSync('rsa', {@@ -1393,3 +1398,233 @@ describe('grok-llm-proxy attempt telemetry', () => {}})})++/**+ * #739 D1-bis — the stream-gate wait also ends when the execution ticket dies.+ * A request still queued at the ticket's `exp` could only be redeemed into a+ * certain `ticket_expired`, so it is answered as a capacity refusal instead,+ * with no redeem. The margin is zero: a request whose ticket is still alive+ * when a slot frees is served exactly as before.+ */+describe('grok-llm-proxy ticket-aware stream-gate wait (#739 D1-bis)', () => {+ /** Captured before any test fakes the clock, so waits stay on real time. */+ const realSetTimeout = globalThis.setTimeout+ const realSleep = (ms: number) => new Promise<void>(resolve => realSetTimeout(resolve, ms))+ const lookup = async () => [{ address: '1.2.3.4', family: 4 }]+ const COMPLETIONS = '/internal/runtime/v1/grok/completions'++ async function until(condition: () => boolean, what: string): Promise<void> {+ const deadline = performance.now() + 5_000+ while (!condition()) {+ if (performance.now() > deadline) throw new Error(`${what} did not happen within 5 s`)+ await realSleep(10)+ }+ }++ function withinReal<T>(promise: Promise<T>, ms: number): Promise<T | undefined> {+ return Promise.race([promise, realSleep(ms).then(() => undefined)])+ }++ /** An envelope whose ticket expires exactly `ticketLifeMs` from the (fake) now. */+ function envelope(providerAttemptId: string, ticketLifeMs: number): Record<string, unknown> {+ const raw: Record<string, unknown> = {+ schemaVersion: 'grok-completion-request.v1',+ requestId: `req-${providerAttemptId}`,+ idempotencyKey: `idem-${providerAttemptId}`,+ provider: 'grok-subscription',+ model: 'gpt-5.1',+ messages: [{ role: 'user', content: 'hello' }],+ }+ const parsed = parseGrokCompletionRequestV1(raw)+ if (!parsed.ok) throw new Error(parsed.message)+ const requestHash = hashGrokCompletionRequestV1(parsed.value)+ const executionTicket = jwt.sign(+ {+ jti: '77777777-7777-4777-8777-777777777777',+ typ: 'grok-execution-ticket',+ hostRef: 'research-host',+ model: 'gpt-5.1',+ requestHash,+ providerAttemptId,+ exp: (Date.now() + ticketLifeMs) / 1000,+ },+ privateKey,+ { algorithm: 'RS256', issuer: 'control-api', audience: 'grok-llm-proxy' }+ )+ return { executionTicket, requestHash, request: raw }+ }++ function countingClient(redeemAnswer: 'granted' | 'ticket_expired'): {+ client: ControlApiClient+ redeems: () => number+ } {+ let redeems = 0+ const client = {+ async redeem(): Promise<RedeemAttemptSuccess> {+ redeems += 1+ if (redeemAnswer === 'ticket_expired') {+ throw new ControlApiClientError('ticket_expired', 'control API request denied')+ }+ return {+ accessToken: 'test-access-ticket-life',+ transport: {+ protocolVersion: 'grok-subscription-transport.v1',+ completionsOrigin: GROK_COMPLETIONS_ORIGIN,+ catalogOrigin: GROK_CATALOG_ORIGIN,+ operation: 'completion_stream',+ servedModel: 'gpt-5.1',+ maxStreamDurationMs: 1_800_000,+ },+ expiryClass: 'short_lived',+ attemptReceipt: 'd'.repeat(64),+ }+ },+ async finalize(input: {+ receipt: { providerAttemptId: string; outcome: FinalizeAttemptSuccess['outcome'] }+ }): Promise<FinalizeAttemptSuccess> {+ return {+ providerAttemptId: input.receipt.providerAttemptId,+ outcome: input.receipt.outcome,+ duplicate: false,+ }+ },+ } as unknown as ControlApiClient+ return { client, redeems: () => redeems }+ }++ const upstream = (async () =>+ new Response(+ 'data: {"type":"response.output_text.delta","delta":"t0"}\n\n' ++ 'data: {"type":"response.completed"}\n\n',+ { status: 200, headers: { 'content-type': 'text/event-stream' } }+ )) as typeof fetch++ function failureCount(metricsText: string, code: string): number {+ const line = metricsText+ .split('\n')+ .find(row => row.startsWith(`grok_proxy_attempt_failures_total{code="${code}"}`))+ return line ? Number(line.split(' ').pop()) : 0+ }++ /** Fakes the clock on a whole second, so a ticket's `exp` lands on an exact instant. */+ function fakeClock(): void {+ vi.useFakeTimers({+ now: Math.ceil(Date.now() / 1000) * 1000,+ toFake: ['setTimeout', 'clearTimeout', 'Date'],+ })+ }++ /**+ * Frees the held slots and lets any waiter still polling the shared gate+ * finish on the fake clock, so the module gate is empty for the next test.+ */+ async function releaseAndDrain(slots: Array<() => void>): Promise<void> {+ for (const release of slots.splice(0)) release()+ await vi.advanceTimersByTimeAsync(STREAM_LIMITS.maxQueueWaitMs)+ vi.useRealTimers()+ }++ async function saturate(): Promise<Array<() => void>> {+ const slots: Array<() => void> = []+ for (let i = 0; i < STREAM_LIMITS.maxConcurrentStreams; i += 1) {+ slots.push(await streamGate.acquire())+ }+ return slots+ }++ it('T-AC-5-grok answers 503 provider_unavailable at the ticket expiry, without a redeem, while the gate stays saturated', async () => {+ fakeClock()+ const warn = vi.spyOn(logger, 'warn')+ let slots: Array<() => void> = []+ let acquire: ReturnType<typeof vi.spyOn> | undefined+ try {+ slots = await saturate()+ // Pass-through spy: it only counts the handler's calls into the gate.+ const spy = vi.spyOn(streamGate, 'acquire')+ acquire = spy+ const control = countingClient('ticket_expired')+ const apps = createProxyApps(config({ maxBodyBytes: 65_536 }), {+ controlApiClient: control.client,+ fetchFn: upstream,+ lookup,+ })+ const ticketLifeMs = 20_000+ const reply = request(apps.runtimeApp)+ .post(COMPLETIONS)+ .set('Authorization', `Bearer ${platformToken()}`)+ .send(envelope('att-ticket-life', ticketLifeMs))+ .then(res => res)+ await until(() => spy.mock.calls.length === 1, 'the request queueing at the stream gate')++ await vi.advanceTimersByTimeAsync(ticketLifeMs - 1)+ // Witness: the request waited for the ticket's whole life, it was not+ // refused on arrival.+ expect(await withinReal(reply, 100)).toBeUndefined()+ await vi.advanceTimersByTimeAsync(1)+ const res = await withinReal(reply, 2_000)+ expect(res?.status).toBe(503)+ expect(res?.body).toEqual({ error: 'provider_unavailable' })++ // A slot that frees after the ticket died is never used to redeem it.+ slots.pop()?.()+ await vi.advanceTimersByTimeAsync(STREAM_LIMITS.maxQueueWaitMs)+ expect(control.redeems()).toBe(0)++ const refusals = warn.mock.calls+ .map(call => call[0] as unknown as Record<string, unknown>)+ .filter(entry => entry?.event === 'grok_proxy_admission_refused')+ expect(refusals).toEqual([+ {+ event: 'grok_proxy_admission_refused',+ reason: 'ticket_life',+ providerAttemptId: 'att-ticket-life',+ hostRef: 'research-host',+ },+ ])+ const metricsText = (await request(apps.probeApp).get('/metrics')).text+ expect(failureCount(metricsText, 'provider_unavailable')).toBe(1)+ expect(failureCount(metricsText, 'request_limit')).toBe(0)+ } finally {+ acquire?.mockRestore()+ await releaseAndDrain(slots)+ warn.mockRestore()+ }+ }, 30_000)++ it('T-PAR-TK-grok serves a request whose ticket is still alive when the gate frees at +50 s of a 60 s ticket', async () => {+ fakeClock()+ let slots: Array<() => void> = []+ let acquire: ReturnType<typeof vi.spyOn> | undefined+ try {+ slots = await saturate()+ const spy = vi.spyOn(streamGate, 'acquire')+ acquire = spy+ const control = countingClient('granted')+ const apps = createProxyApps(config({ maxBodyBytes: 65_536 }), {+ controlApiClient: control.client,+ fetchFn: upstream,+ lookup,+ })+ const reply = request(apps.runtimeApp)+ .post(COMPLETIONS)+ .set('Authorization', `Bearer ${platformToken()}`)+ .send(envelope('att-ticket-alive', 60_000))+ .then(res => res)+ await until(() => spy.mock.calls.length === 1, 'the request queueing at the stream gate')++ await vi.advanceTimersByTimeAsync(50_000)+ expect(await withinReal(reply, 100)).toBeUndefined()+ slots.pop()?.()+ // One stream-gate poll interval.+ await vi.advanceTimersByTimeAsync(10)+ const res = await withinReal(reply, 2_000)+ expect(res?.status).toBe(200)+ expect(res?.text).toContain('data: {"type":"text","text":"t0"}')+ expect(res?.text).toContain('data: {"type":"done","outcome":"success"}')+ expect(control.redeems()).toBe(1)+ } finally {+ acquire?.mockRestore()+ await releaseAndDrain(slots)+ }+ }, 30_000)+})