Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript

fix(llm-proxy): make the visual gate wait-out refusal deterministic

Alfredo Lopez96d0a865ec2e
diff --git a/codex-llm-proxy/src/requestLimits.ts b/codex-llm-proxy/src/requestLimits.ts
index 1093ab0be..21349f36c 100644
--- a/codex-llm-proxy/src/requestLimits.ts
+++ b/codex-llm-proxy/src/requestLimits.ts
@@ -234,11 +234,32 @@ export class StreamGate {
* the slot count, so a slot that frees after the bound does not admit it
* even when the event loop stalled across the bound. A free slot is granted
* at once whatever the deadline; the caller checks a deadline already past.
+ *
+ * The refusal names which bound ran out. The gate's own bound is
+ * `maxQueueWaitMs` after the enqueue, or after `queueWaitStartedAt` (epoch
+ * ms) when the caller passes it: the visual gate's own wait is the request's
+ * admission clock, so its caller passes the arrival instant. Without
+ * `queueWaitStartedAt`, a `deadlineAt` at or before the own bound governs
+ * (`deadline`, `provider_unavailable`). With it, the deadline and the own
+ * bound are measured from the same instant, so a `deadlineAt` equal to the
+ * own bound is that bound restated and only a strictly earlier one governs.
+ * Every other expiry is the gate's queue wait (`queue_wait`,
+ * `queueCapacityCode`). The decision does not depend on how many
+ * milliseconds passed between the caller's stamp and the enqueue.
*/
- async acquire(signal?: AbortSignal, deadlineAt?: number): Promise<() => void> {
+ async acquire(
+ signal?: AbortSignal,
+ deadlineAt?: number,
+ queueWaitStartedAt?: number
+ ): Promise<() => void> {
if (deadlineAt !== undefined && !Number.isFinite(deadlineAt)) {
throw new RangeError(`a stream gate deadline must be a finite epoch time, got ${deadlineAt}`)
}
+ if (queueWaitStartedAt !== undefined && !Number.isFinite(queueWaitStartedAt)) {
+ throw new RangeError(
+ `a stream gate queue wait start must be a finite epoch time, got ${queueWaitStartedAt}`
+ )
+ }
if (signal?.aborted) throw new RequestLimitError('stream request was aborted', 'aborted')
if (this.running >= this.maxConcurrent) {
if (this.queued >= this.maxQueued)
@@ -257,12 +278,14 @@ export class StreamGate {
reject(new RequestLimitError('stream request was aborted', 'aborted'))
}
const queuedAt = performance.now()
- const waitBoundMs =
- deadlineAt === undefined
- ? this.maxQueueWaitMs
- : Math.min(this.maxQueueWaitMs, deadlineAt - Date.now())
- // The caller's deadline governs only when it is the nearer bound.
- const deadlineGoverns = deadlineAt !== undefined && waitBoundMs < this.maxQueueWaitMs
+ const enqueuedAt = Date.now()
+ const ownBoundAt = (queueWaitStartedAt ?? enqueuedAt) + this.maxQueueWaitMs
+ const deadlineGoverns =
+ deadlineAt !== undefined &&
+ (queueWaitStartedAt === undefined ? deadlineAt <= ownBoundAt : deadlineAt < ownBoundAt)
+ // No waiter stays queued longer than maxQueueWaitMs after the enqueue.
+ const boundAt = deadlineAt !== undefined && deadlineGoverns ? deadlineAt : ownBoundAt
+ const waitBoundMs = Math.min(this.maxQueueWaitMs, boundAt - enqueuedAt)
const expire = () => {
settle()
reject(
diff --git a/codex-llm-proxy/src/server.ts b/codex-llm-proxy/src/server.ts
index 5dc5ec2d3..a0d337ae8 100644
--- a/codex-llm-proxy/src/server.ts
+++ b/codex-llm-proxy/src/server.ts
@@ -59,6 +59,12 @@ type AdmittedRequest = Request & {
* request performs (body budget, visual gate, stream gate) ends by then.
*/
codexAdmissionDeadlineAt?: number
+ /**
+ * The arrival instant the admission clock was stamped from, in epoch ms.
+ * The visual gate measures its own queue wait from it, so its refusal
+ * identity does not depend on when the request reached the gate.
+ */
+ codexAdmissionArrivedAt?: number
/**
* #739 D2 — the body's budget reservation, set once it was granted. The
* handler releases it when the upstream accepted the request; the
@@ -200,7 +206,9 @@ function bodyAdmission(
parse: RequestHandler
) {
return (req: AdmittedRequest, res: Response, next: NextFunction): void => {
- const admissionDeadlineAt = Date.now() + STREAM_LIMITS.maxQueueWaitMs
+ const arrivedAt = Date.now()
+ const admissionDeadlineAt = arrivedAt + STREAM_LIMITS.maxQueueWaitMs
+ req.codexAdmissionArrivedAt = arrivedAt
req.codexAdmissionDeadlineAt = admissionDeadlineAt
if (req.headers['transfer-encoding'] !== undefined) {
reject(res, 411, 'length_required')
@@ -414,6 +422,13 @@ export function createProxyApps(
reject(res, 401, 'Unauthorized')
return
}
+ // bodyAdmission stamps both before it hands the request here.
+ const arrivedAt = req.codexAdmissionArrivedAt
+ const admissionDeadlineAt = req.codexAdmissionDeadlineAt
+ if (arrivedAt === undefined || admissionDeadlineAt === undefined) {
+ next(new Error('visual admission was reached without body admission'))
+ return
+ }
const visualPrincipal = JSON.stringify([platform.sub, ...[...platform.hostRefs].sort()])
const admittedByPrincipal = visualPrincipalAdmissions.get(visualPrincipal) ?? 0
if (admittedByPrincipal >= VISUAL_PER_HOST_MAX_ADMITTED) {
@@ -457,7 +472,10 @@ export function createProxyApps(
const abortParse = (): void => parseAbort.abort()
req.once('aborted', abortParse)
try {
- release = await visualStreamGate.acquire(parseAbort.signal, req.codexAdmissionDeadlineAt)
+ // The visual gate's own wait is the admission clock: it is measured
+ // from arrival, so a waiter that waits it out is refused as
+ // `visual_gate` however many ms passed before it reached the gate.
+ release = await visualStreamGate.acquire(parseAbort.signal, admissionDeadlineAt, arrivedAt)
} catch (err) {
req.off('aborted', abortParse)
releasePrincipalShare()
diff --git a/codex-llm-proxy/test/requestLimits.test.ts b/codex-llm-proxy/test/requestLimits.test.ts
index 9edbfb5f6..abd77dcd5 100644
--- a/codex-llm-proxy/test/requestLimits.test.ts
+++ b/codex-llm-proxy/test/requestLimits.test.ts
@@ -429,3 +429,160 @@ describe('StreamGate', () => {
expect(visualStreamGate.snapshot()).toEqual({ running: 0, queued: 0 })
})
})
+
+/**
+ * Records how an acquire settled: `admitted`, or the refusal's kind and wire
+ * code. Unset while the waiter is still queued.
+ */
+function settled(waiter: Promise<() => void>) {
+ const state: { outcome?: 'admitted' | { kind: string; code: string } } = {}
+ const done = waiter.then(
+ release => {
+ state.outcome = 'admitted'
+ release()
+ },
+ (err: RequestLimitError) => {
+ state.outcome = { kind: err.kind, code: err.code }
+ }
+ )
+ return { state, done }
+}
+
+describe('StreamGate refusal identity', () => {
+ const FAKE_CLOCK = {
+ toFake: ['setTimeout', 'clearTimeout', 'Date', 'performance'] as Array<
+ 'setTimeout' | 'clearTimeout' | 'Date' | 'performance'
+ >,
+ }
+ const WAIT_MS = STREAM_LIMITS.maxQueueWaitMs
+
+ // The visual gate's own wait is the request's admission clock, so its caller
+ // passes the arrival instant. The refusal must not depend on how many ms
+ // passed between the arrival stamp and the enqueue.
+ it('refuses visual waiters enqueued 0, 1 and 5 ms after arrival as visual_gate at arrival + maxQueueWaitMs', async () => {
+ vi.useFakeTimers(FAKE_CLOCK)
+ try {
+ const gate = new StreamGate(1, 4, WAIT_MS, 'visual_gate')
+ const held = await gate.acquire()
+ const arrivedAt = Date.now()
+ const admissionDeadlineAt = arrivedAt + WAIT_MS
+ const atZero = settled(gate.acquire(undefined, admissionDeadlineAt, arrivedAt))
+ await vi.advanceTimersByTimeAsync(1)
+ const atOne = settled(gate.acquire(undefined, admissionDeadlineAt, arrivedAt))
+ await vi.advanceTimersByTimeAsync(4)
+ const atFive = settled(gate.acquire(undefined, admissionDeadlineAt, arrivedAt))
+
+ await vi.advanceTimersByTimeAsync(WAIT_MS - 5 - 1)
+ // Witness: one millisecond before the admission clock runs out, all
+ // three waiters are still queued, so none was refused early.
+ expect(Date.now()).toBe(admissionDeadlineAt - 1)
+ expect(gate.snapshot()).toEqual({ running: 1, queued: 3 })
+ expect([atZero.state.outcome, atOne.state.outcome, atFive.state.outcome]).toEqual([
+ undefined,
+ undefined,
+ undefined,
+ ])
+
+ await vi.advanceTimersByTimeAsync(1)
+ await Promise.all([atZero.done, atOne.done, atFive.done])
+ expect(Date.now()).toBe(admissionDeadlineAt)
+ const queueWait = { kind: 'queue_wait', code: 'visual_gate' }
+ expect(atZero.state.outcome).toEqual(queueWait)
+ expect(atOne.state.outcome).toEqual(queueWait)
+ expect(atFive.state.outcome).toEqual(queueWait)
+ expect(gate.snapshot()).toEqual({ running: 1, queued: 0 })
+ held()
+ } finally {
+ vi.useRealTimers()
+ }
+ })
+
+ it('refuses a visual waiter whose caller deadline is strictly earlier than arrival + maxQueueWaitMs as deadline', async () => {
+ vi.useFakeTimers(FAKE_CLOCK)
+ try {
+ const gate = new StreamGate(1, 4, WAIT_MS, 'visual_gate')
+ const held = await gate.acquire()
+ const arrivedAt = Date.now()
+ await vi.advanceTimersByTimeAsync(5)
+ const callerDeadlineAt = arrivedAt + WAIT_MS - 1
+ const waiter = settled(gate.acquire(undefined, callerDeadlineAt, arrivedAt))
+
+ await vi.advanceTimersByTimeAsync(callerDeadlineAt - 1 - Date.now())
+ // Witness: still queued one millisecond before the caller's deadline.
+ expect(waiter.state.outcome).toBeUndefined()
+ expect(gate.snapshot()).toEqual({ running: 1, queued: 1 })
+
+ await vi.advanceTimersByTimeAsync(1)
+ await waiter.done
+ expect(Date.now()).toBe(callerDeadlineAt)
+ expect(waiter.state.outcome).toEqual({ kind: 'deadline', code: 'provider_unavailable' })
+ held()
+ } finally {
+ vi.useRealTimers()
+ }
+ })
+
+ // The ordinary gate measures its own bound from the enqueue. A caller
+ // deadline that lands exactly on that bound is the admission clock running
+ // out, which is answered with provider_unavailable.
+ it('refuses an ordinary waiter whose caller deadline ties its own bound as deadline', async () => {
+ vi.useFakeTimers(FAKE_CLOCK)
+ try {
+ const gate = new StreamGate(1, 1)
+ const held = await gate.acquire()
+ const enqueuedAt = Date.now()
+ const waiter = settled(gate.acquire(undefined, enqueuedAt + WAIT_MS))
+
+ await vi.advanceTimersByTimeAsync(WAIT_MS - 1)
+ // Witness: still queued one millisecond before the shared bound.
+ expect(waiter.state.outcome).toBeUndefined()
+ expect(gate.snapshot()).toEqual({ running: 1, queued: 1 })
+
+ await vi.advanceTimersByTimeAsync(1)
+ await waiter.done
+ expect(Date.now() - enqueuedAt).toBe(WAIT_MS)
+ expect(waiter.state.outcome).toEqual({ kind: 'deadline', code: 'provider_unavailable' })
+ held()
+ } finally {
+ vi.useRealTimers()
+ }
+ })
+
+ it('refuses an ordinary waiter whose caller deadline is 1 ms past its own bound as queue_wait', async () => {
+ vi.useFakeTimers(FAKE_CLOCK)
+ try {
+ const gate = new StreamGate(1, 1)
+ const held = await gate.acquire()
+ const enqueuedAt = Date.now()
+ const waiter = settled(gate.acquire(undefined, enqueuedAt + WAIT_MS + 1))
+
+ await vi.advanceTimersByTimeAsync(WAIT_MS - 1)
+ // Witness: still queued one millisecond before the gate's own bound.
+ expect(waiter.state.outcome).toBeUndefined()
+ expect(gate.snapshot()).toEqual({ running: 1, queued: 1 })
+
+ await vi.advanceTimersByTimeAsync(1)
+ await waiter.done
+ // The gate's own bound ended the wait, not the later caller deadline.
+ expect(Date.now() - enqueuedAt).toBe(WAIT_MS)
+ expect(waiter.state.outcome).toEqual({ kind: 'queue_wait', code: 'proxy_capacity_exceeded' })
+ held()
+ } finally {
+ vi.useRealTimers()
+ }
+ })
+
+ it('refuses a queue wait start that is not a finite epoch time', async () => {
+ const gate = new StreamGate(1, 1)
+ for (const start of [Number.NaN, Number.POSITIVE_INFINITY]) {
+ await expect(gate.acquire(undefined, undefined, start), String(start)).rejects.toThrow(
+ RangeError
+ )
+ }
+ // Witness: the refusal took no slot, and a finite start is accepted.
+ expect(gate.snapshot()).toEqual({ running: 0, queued: 0 })
+ const release = await gate.acquire(undefined, undefined, Date.now())
+ expect(gate.snapshot()).toEqual({ running: 1, queued: 0 })
+ release()
+ })
+})
diff --git a/codex-llm-proxy/test/server.security.test.ts b/codex-llm-proxy/test/server.security.test.ts
index bbcf213c6..5cd4a0380 100644
--- a/codex-llm-proxy/test/server.security.test.ts
+++ b/codex-llm-proxy/test/server.security.test.ts
@@ -2552,6 +2552,88 @@ describe('codex-llm-proxy ticket-aware stream-gate wait (#739 D1-bis)', () => {
warn.mockRestore()
}
}, 30_000)
+
+ // The visual gate's own wait is the admission clock stamped at arrival. A
+ // body that reaches the gate some ms after arrival and waits that clock out
+ // is refused as the gate's own queue wait, `visual_gate`: the same answer as
+ // a body that reached the gate in the arrival millisecond.
+ it.each([0, 5])(
+ 'T-VG-QW answers 503 visual_gate when a body enqueued %i ms after arrival waits out the admission clock',
+ async lagMs => {
+ fakeClock()
+ const warn = vi.spyOn(logger, 'warn')
+ const held: Array<() => void> = []
+ let acquire: { mockRestore: () => void } | undefined
+ try {
+ for (let i = 0; i < VISUAL_STREAM_LIMITS.maxConcurrentStreams; i += 1) {
+ held.push(await visualStreamGate.acquire())
+ }
+ const enqueue = visualStreamGate.acquire.bind(visualStreamGate)
+ // Moves the clock `lagMs` between the arrival stamp and the enqueue,
+ // then calls the real gate.
+ const spy = vi
+ .spyOn(visualStreamGate, 'acquire')
+ .mockImplementation((signal, deadlineAt, queueWaitStartedAt) => {
+ vi.setSystemTime(Date.now() + lagMs)
+ return enqueue(signal, deadlineAt, queueWaitStartedAt)
+ })
+ acquire = spy
+ const control = countingClient('granted')
+ const apps = createProxyApps(config({ maxBodyBytes: 65_536 }), {
+ controlApiClient: control.client,
+ fetchFn: upstream,
+ lookup,
+ })
+ const arrivedAt = Date.now()
+ const reply = request(apps.runtimeApp)
+ .post(COMPLETIONS)
+ .set('Authorization', `Bearer ${platformToken()}`)
+ .send(visualEnvelope(`att-visual-wait-${lagMs}`, 60_000))
+ .then(res => res)
+ await until(
+ () => spy.mock.calls.length === 1 && visualStreamGate.snapshot().queued === 1,
+ 'the body queueing at the visual gate'
+ )
+ // Witness: the gate was handed the admission clock and the arrival
+ // instant, and it enqueued the body `lagMs` after arrival.
+ expect(spy.mock.calls[0]?.slice(1)).toEqual([
+ arrivedAt + STREAM_LIMITS.maxQueueWaitMs,
+ arrivedAt,
+ ])
+ expect(Date.now()).toBe(arrivedAt + lagMs)
+
+ await vi.advanceTimersByTimeAsync(STREAM_LIMITS.maxQueueWaitMs - lagMs - 1)
+ // Witness: still queued one millisecond before the admission clock
+ // runs out, so the body was not refused early.
+ expect(await withinReal(reply, 100)).toBeUndefined()
+ expect(visualStreamGate.snapshot().queued).toBe(1)
+ await vi.advanceTimersByTimeAsync(1)
+ const res = await withinReal(reply, 2_000)
+ expect(Date.now()).toBe(arrivedAt + STREAM_LIMITS.maxQueueWaitMs)
+ expect(res?.status).toBe(503)
+ expect(res?.body).toEqual({ error: 'visual_gate' })
+ expect(control.redeems()).toBe(0)
+ const refusals = warn.mock.calls
+ .map(call => call[0] as unknown as Record<string, unknown>)
+ .filter(entry => entry?.event === 'codex_proxy_admission_refused')
+ expect(refusals).toEqual([
+ {
+ event: 'codex_proxy_admission_refused',
+ reason: 'visual_gate',
+ code: 'visual_gate',
+ detail: 'stream queue wait exceeded',
+ },
+ ])
+ expect(visualStreamGate.snapshot()).toEqual({ running: held.length, queued: 0 })
+ } finally {
+ acquire?.mockRestore()
+ for (const release of held.splice(0)) release()
+ vi.useRealTimers()
+ warn.mockRestore()
+ }
+ },
+ 30_000
+ )
})
/**
diff --git a/grok-llm-proxy/src/requestLimits.ts b/grok-llm-proxy/src/requestLimits.ts
index ecded6a9b..32c206d03 100644
--- a/grok-llm-proxy/src/requestLimits.ts
+++ b/grok-llm-proxy/src/requestLimits.ts
@@ -226,11 +226,32 @@ export class StreamGate {
* the slot count, so a slot that frees after the bound does not admit it
* even when the event loop stalled across the bound. A free slot is granted
* at once whatever the deadline; the caller checks a deadline already past.
+ *
+ * The refusal names which bound ran out. The gate's own bound is
+ * `maxQueueWaitMs` after the enqueue, or after `queueWaitStartedAt` (epoch
+ * ms) when the caller passes it: the visual gate's own wait is the request's
+ * admission clock, so its caller passes the arrival instant. Without
+ * `queueWaitStartedAt`, a `deadlineAt` at or before the own bound governs
+ * (`deadline`, `provider_unavailable`). With it, the deadline and the own
+ * bound are measured from the same instant, so a `deadlineAt` equal to the
+ * own bound is that bound restated and only a strictly earlier one governs.
+ * Every other expiry is the gate's queue wait (`queue_wait`,
+ * `queueCapacityCode`). The decision does not depend on how many
+ * milliseconds passed between the caller's stamp and the enqueue.
*/
- async acquire(signal?: AbortSignal, deadlineAt?: number): Promise<() => void> {
+ async acquire(
+ signal?: AbortSignal,
+ deadlineAt?: number,
+ queueWaitStartedAt?: number
+ ): Promise<() => void> {
if (deadlineAt !== undefined && !Number.isFinite(deadlineAt)) {
throw new RangeError(`a stream gate deadline must be a finite epoch time, got ${deadlineAt}`)
}
+ if (queueWaitStartedAt !== undefined && !Number.isFinite(queueWaitStartedAt)) {
+ throw new RangeError(
+ `a stream gate queue wait start must be a finite epoch time, got ${queueWaitStartedAt}`
+ )
+ }
if (signal?.aborted) throw new RequestLimitError('stream request was aborted', 'aborted')
if (this.running >= this.maxConcurrent) {
if (this.queued >= this.maxQueued)
@@ -249,12 +270,14 @@ export class StreamGate {
reject(new RequestLimitError('stream request was aborted', 'aborted'))
}
const queuedAt = performance.now()
- const waitBoundMs =
- deadlineAt === undefined
- ? this.maxQueueWaitMs
- : Math.min(this.maxQueueWaitMs, deadlineAt - Date.now())
- // The caller's deadline governs only when it is the nearer bound.
- const deadlineGoverns = deadlineAt !== undefined && waitBoundMs < this.maxQueueWaitMs
+ const enqueuedAt = Date.now()
+ const ownBoundAt = (queueWaitStartedAt ?? enqueuedAt) + this.maxQueueWaitMs
+ const deadlineGoverns =
+ deadlineAt !== undefined &&
+ (queueWaitStartedAt === undefined ? deadlineAt <= ownBoundAt : deadlineAt < ownBoundAt)
+ // No waiter stays queued longer than maxQueueWaitMs after the enqueue.
+ const boundAt = deadlineAt !== undefined && deadlineGoverns ? deadlineAt : ownBoundAt
+ const waitBoundMs = Math.min(this.maxQueueWaitMs, boundAt - enqueuedAt)
const expire = () => {
settle()
reject(
diff --git a/grok-llm-proxy/src/server.ts b/grok-llm-proxy/src/server.ts
index e560f8161..7336026ab 100644
--- a/grok-llm-proxy/src/server.ts
+++ b/grok-llm-proxy/src/server.ts
@@ -59,6 +59,12 @@ type AdmittedRequest = Request & {
* request performs (body budget, visual gate, stream gate) ends by then.
*/
grokAdmissionDeadlineAt?: number
+ /**
+ * The arrival instant the admission clock was stamped from, in epoch ms.
+ * The visual gate measures its own queue wait from it, so its refusal
+ * identity does not depend on when the request reached the gate.
+ */
+ grokAdmissionArrivedAt?: number
/**
* #739 D2 — the body's budget reservation, set once it was granted. The
* handler releases it when the upstream accepted the request; the
@@ -203,7 +209,9 @@ function bodyAdmission(
parse: RequestHandler
) {
return (req: AdmittedRequest, res: Response, next: NextFunction): void => {
- const admissionDeadlineAt = Date.now() + STREAM_LIMITS.maxQueueWaitMs
+ const arrivedAt = Date.now()
+ const admissionDeadlineAt = arrivedAt + STREAM_LIMITS.maxQueueWaitMs
+ req.grokAdmissionArrivedAt = arrivedAt
req.grokAdmissionDeadlineAt = admissionDeadlineAt
if (req.headers['transfer-encoding'] !== undefined) {
reject(res, 411, 'length_required')
@@ -422,6 +430,13 @@ export function createProxyApps(
reject(res, 401, 'Unauthorized')
return
}
+ // bodyAdmission stamps both before it hands the request here.
+ const arrivedAt = req.grokAdmissionArrivedAt
+ const admissionDeadlineAt = req.grokAdmissionDeadlineAt
+ if (arrivedAt === undefined || admissionDeadlineAt === undefined) {
+ next(new Error('visual admission was reached without body admission'))
+ return
+ }
const visualPrincipal = JSON.stringify([platform.sub, ...[...platform.hostRefs].sort()])
const admittedByPrincipal = visualPrincipalAdmissions.get(visualPrincipal) ?? 0
if (admittedByPrincipal >= VISUAL_PER_HOST_MAX_ADMITTED) {
@@ -465,7 +480,10 @@ export function createProxyApps(
const abortParse = (): void => parseAbort.abort()
req.once('aborted', abortParse)
try {
- release = await visualStreamGate.acquire(parseAbort.signal, req.grokAdmissionDeadlineAt)
+ // The visual gate's own wait is the admission clock: it is measured
+ // from arrival, so a waiter that waits it out is refused as
+ // `visual_gate` however many ms passed before it reached the gate.
+ release = await visualStreamGate.acquire(parseAbort.signal, admissionDeadlineAt, arrivedAt)
} catch (err) {
req.off('aborted', abortParse)
releasePrincipalShare()
diff --git a/grok-llm-proxy/test/requestLimits.test.ts b/grok-llm-proxy/test/requestLimits.test.ts
index e5a3df4d7..bb7938658 100644
--- a/grok-llm-proxy/test/requestLimits.test.ts
+++ b/grok-llm-proxy/test/requestLimits.test.ts
@@ -441,3 +441,160 @@ describe('StreamGate', () => {
expect(visualStreamGate.snapshot()).toEqual({ running: 0, queued: 0 })
})
})
+
+/**
+ * Records how an acquire settled: `admitted`, or the refusal's kind and wire
+ * code. Unset while the waiter is still queued.
+ */
+function settled(waiter: Promise<() => void>) {
+ const state: { outcome?: 'admitted' | { kind: string; code: string } } = {}
+ const done = waiter.then(
+ release => {
+ state.outcome = 'admitted'
+ release()
+ },
+ (err: RequestLimitError) => {
+ state.outcome = { kind: err.kind, code: err.code }
+ }
+ )
+ return { state, done }
+}
+
+describe('StreamGate refusal identity', () => {
+ const FAKE_CLOCK = {
+ toFake: ['setTimeout', 'clearTimeout', 'Date', 'performance'] as Array<
+ 'setTimeout' | 'clearTimeout' | 'Date' | 'performance'
+ >,
+ }
+ const WAIT_MS = STREAM_LIMITS.maxQueueWaitMs
+
+ // The visual gate's own wait is the request's admission clock, so its caller
+ // passes the arrival instant. The refusal must not depend on how many ms
+ // passed between the arrival stamp and the enqueue.
+ it('refuses visual waiters enqueued 0, 1 and 5 ms after arrival as visual_gate at arrival + maxQueueWaitMs', async () => {
+ vi.useFakeTimers(FAKE_CLOCK)
+ try {
+ const gate = new StreamGate(1, 4, WAIT_MS, 'visual_gate')
+ const held = await gate.acquire()
+ const arrivedAt = Date.now()
+ const admissionDeadlineAt = arrivedAt + WAIT_MS
+ const atZero = settled(gate.acquire(undefined, admissionDeadlineAt, arrivedAt))
+ await vi.advanceTimersByTimeAsync(1)
+ const atOne = settled(gate.acquire(undefined, admissionDeadlineAt, arrivedAt))
+ await vi.advanceTimersByTimeAsync(4)
+ const atFive = settled(gate.acquire(undefined, admissionDeadlineAt, arrivedAt))
+
+ await vi.advanceTimersByTimeAsync(WAIT_MS - 5 - 1)
+ // Witness: one millisecond before the admission clock runs out, all
+ // three waiters are still queued, so none was refused early.
+ expect(Date.now()).toBe(admissionDeadlineAt - 1)
+ expect(gate.snapshot()).toEqual({ running: 1, queued: 3 })
+ expect([atZero.state.outcome, atOne.state.outcome, atFive.state.outcome]).toEqual([
+ undefined,
+ undefined,
+ undefined,
+ ])
+
+ await vi.advanceTimersByTimeAsync(1)
+ await Promise.all([atZero.done, atOne.done, atFive.done])
+ expect(Date.now()).toBe(admissionDeadlineAt)
+ const queueWait = { kind: 'queue_wait', code: 'visual_gate' }
+ expect(atZero.state.outcome).toEqual(queueWait)
+ expect(atOne.state.outcome).toEqual(queueWait)
+ expect(atFive.state.outcome).toEqual(queueWait)
+ expect(gate.snapshot()).toEqual({ running: 1, queued: 0 })
+ held()
+ } finally {
+ vi.useRealTimers()
+ }
+ })
+
+ it('refuses a visual waiter whose caller deadline is strictly earlier than arrival + maxQueueWaitMs as deadline', async () => {
+ vi.useFakeTimers(FAKE_CLOCK)
+ try {
+ const gate = new StreamGate(1, 4, WAIT_MS, 'visual_gate')
+ const held = await gate.acquire()
+ const arrivedAt = Date.now()
+ await vi.advanceTimersByTimeAsync(5)
+ const callerDeadlineAt = arrivedAt + WAIT_MS - 1
+ const waiter = settled(gate.acquire(undefined, callerDeadlineAt, arrivedAt))
+
+ await vi.advanceTimersByTimeAsync(callerDeadlineAt - 1 - Date.now())
+ // Witness: still queued one millisecond before the caller's deadline.
+ expect(waiter.state.outcome).toBeUndefined()
+ expect(gate.snapshot()).toEqual({ running: 1, queued: 1 })
+
+ await vi.advanceTimersByTimeAsync(1)
+ await waiter.done
+ expect(Date.now()).toBe(callerDeadlineAt)
+ expect(waiter.state.outcome).toEqual({ kind: 'deadline', code: 'provider_unavailable' })
+ held()
+ } finally {
+ vi.useRealTimers()
+ }
+ })
+
+ // The ordinary gate measures its own bound from the enqueue. A caller
+ // deadline that lands exactly on that bound is the admission clock running
+ // out, which is answered with provider_unavailable.
+ it('refuses an ordinary waiter whose caller deadline ties its own bound as deadline', async () => {
+ vi.useFakeTimers(FAKE_CLOCK)
+ try {
+ const gate = new StreamGate(1, 1)
+ const held = await gate.acquire()
+ const enqueuedAt = Date.now()
+ const waiter = settled(gate.acquire(undefined, enqueuedAt + WAIT_MS))
+
+ await vi.advanceTimersByTimeAsync(WAIT_MS - 1)
+ // Witness: still queued one millisecond before the shared bound.
+ expect(waiter.state.outcome).toBeUndefined()
+ expect(gate.snapshot()).toEqual({ running: 1, queued: 1 })
+
+ await vi.advanceTimersByTimeAsync(1)
+ await waiter.done
+ expect(Date.now() - enqueuedAt).toBe(WAIT_MS)
+ expect(waiter.state.outcome).toEqual({ kind: 'deadline', code: 'provider_unavailable' })
+ held()
+ } finally {
+ vi.useRealTimers()
+ }
+ })
+
+ it('refuses an ordinary waiter whose caller deadline is 1 ms past its own bound as queue_wait', async () => {
+ vi.useFakeTimers(FAKE_CLOCK)
+ try {
+ const gate = new StreamGate(1, 1)
+ const held = await gate.acquire()
+ const enqueuedAt = Date.now()
+ const waiter = settled(gate.acquire(undefined, enqueuedAt + WAIT_MS + 1))
+
+ await vi.advanceTimersByTimeAsync(WAIT_MS - 1)
+ // Witness: still queued one millisecond before the gate's own bound.
+ expect(waiter.state.outcome).toBeUndefined()
+ expect(gate.snapshot()).toEqual({ running: 1, queued: 1 })
+
+ await vi.advanceTimersByTimeAsync(1)
+ await waiter.done
+ // The gate's own bound ended the wait, not the later caller deadline.
+ expect(Date.now() - enqueuedAt).toBe(WAIT_MS)
+ expect(waiter.state.outcome).toEqual({ kind: 'queue_wait', code: 'proxy_capacity_exceeded' })
+ held()
+ } finally {
+ vi.useRealTimers()
+ }
+ })
+
+ it('refuses a queue wait start that is not a finite epoch time', async () => {
+ const gate = new StreamGate(1, 1)
+ for (const start of [Number.NaN, Number.POSITIVE_INFINITY]) {
+ await expect(gate.acquire(undefined, undefined, start), String(start)).rejects.toThrow(
+ RangeError
+ )
+ }
+ // Witness: the refusal took no slot, and a finite start is accepted.
+ expect(gate.snapshot()).toEqual({ running: 0, queued: 0 })
+ const release = await gate.acquire(undefined, undefined, Date.now())
+ expect(gate.snapshot()).toEqual({ running: 1, queued: 0 })
+ release()
+ })
+})
diff --git a/grok-llm-proxy/test/server.security.test.ts b/grok-llm-proxy/test/server.security.test.ts
index 0c9be5a13..6044f0617 100644
--- a/grok-llm-proxy/test/server.security.test.ts
+++ b/grok-llm-proxy/test/server.security.test.ts
@@ -29,6 +29,7 @@ import {
ENVELOPE_ALLOWANCE_BYTES,
RequestLimitError,
STREAM_LIMITS,
+ VISUAL_STREAM_LIMITS,
streamGate,
visualStreamGate,
} from '../src/requestLimits.js'
@@ -2644,6 +2645,88 @@ describe('grok-llm-proxy ticket-aware stream-gate wait (#739 D1-bis)', () => {
warn.mockRestore()
}
}, 30_000)
+
+ // The visual gate's own wait is the admission clock stamped at arrival. A
+ // body that reaches the gate some ms after arrival and waits that clock out
+ // is refused as the gate's own queue wait, `visual_gate`: the same answer as
+ // a body that reached the gate in the arrival millisecond.
+ it.each([0, 5])(
+ 'T-VG-QW-grok answers 503 visual_gate when a body enqueued %i ms after arrival waits out the admission clock',
+ async lagMs => {
+ fakeClock()
+ const warn = vi.spyOn(logger, 'warn')
+ const held: Array<() => void> = []
+ let acquire: { mockRestore: () => void } | undefined
+ try {
+ for (let i = 0; i < VISUAL_STREAM_LIMITS.maxConcurrentStreams; i += 1) {
+ held.push(await visualStreamGate.acquire())
+ }
+ const enqueue = visualStreamGate.acquire.bind(visualStreamGate)
+ // Moves the clock `lagMs` between the arrival stamp and the enqueue,
+ // then calls the real gate.
+ const spy = vi
+ .spyOn(visualStreamGate, 'acquire')
+ .mockImplementation((signal, deadlineAt, queueWaitStartedAt) => {
+ vi.setSystemTime(Date.now() + lagMs)
+ return enqueue(signal, deadlineAt, queueWaitStartedAt)
+ })
+ acquire = spy
+ const control = countingClient('granted')
+ const apps = createProxyApps(config({ maxBodyBytes: 65_536 }), {
+ controlApiClient: control.client,
+ fetchFn: upstream,
+ lookup,
+ })
+ const arrivedAt = Date.now()
+ const reply = request(apps.runtimeApp)
+ .post(COMPLETIONS)
+ .set('Authorization', `Bearer ${platformToken()}`)
+ .send(visualEnvelope(`att-visual-wait-${lagMs}`, 60_000))
+ .then(res => res)
+ await until(
+ () => spy.mock.calls.length === 1 && visualStreamGate.snapshot().queued === 1,
+ 'the body queueing at the visual gate'
+ )
+ // Witness: the gate was handed the admission clock and the arrival
+ // instant, and it enqueued the body `lagMs` after arrival.
+ expect(spy.mock.calls[0]?.slice(1)).toEqual([
+ arrivedAt + STREAM_LIMITS.maxQueueWaitMs,
+ arrivedAt,
+ ])
+ expect(Date.now()).toBe(arrivedAt + lagMs)
+
+ await vi.advanceTimersByTimeAsync(STREAM_LIMITS.maxQueueWaitMs - lagMs - 1)
+ // Witness: still queued one millisecond before the admission clock
+ // runs out, so the body was not refused early.
+ expect(await withinReal(reply, 100)).toBeUndefined()
+ expect(visualStreamGate.snapshot().queued).toBe(1)
+ await vi.advanceTimersByTimeAsync(1)
+ const res = await withinReal(reply, 2_000)
+ expect(Date.now()).toBe(arrivedAt + STREAM_LIMITS.maxQueueWaitMs)
+ expect(res?.status).toBe(503)
+ expect(res?.body).toEqual({ error: 'visual_gate' })
+ expect(control.redeems()).toBe(0)
+ const refusals = warn.mock.calls
+ .map(call => call[0] as unknown as Record<string, unknown>)
+ .filter(entry => entry?.event === 'grok_proxy_admission_refused')
+ expect(refusals).toEqual([
+ {
+ event: 'grok_proxy_admission_refused',
+ reason: 'visual_gate',
+ code: 'visual_gate',
+ detail: 'stream queue wait exceeded',
+ },
+ ])
+ expect(visualStreamGate.snapshot()).toEqual({ running: held.length, queued: 0 })
+ } finally {
+ acquire?.mockRestore()
+ for (const release of held.splice(0)) release()
+ vi.useRealTimers()
+ warn.mockRestore()
+ }
+ },
+ 30_000
+ )
})
/**