Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript

fix(llm-proxy): grant stream gate slots in arrival order

Alfredo Lopez0f664517e295
diff --git a/codex-llm-proxy/src/requestLimits.ts b/codex-llm-proxy/src/requestLimits.ts
index 21349f36c..601679e81 100644
--- a/codex-llm-proxy/src/requestLimits.ts
+++ b/codex-llm-proxy/src/requestLimits.ts
@@ -191,9 +191,16 @@ export function assertBoundedIdleTimeout(idleTimeoutMs: number | undefined): num
return Math.min(requested, STREAM_LIMITS.upstreamIdleTimeoutMs)
}
+/**
+ * A queued stream-gate caller. `grant` hands it the slot a release passes on
+ * and returns false, after refusing it, when its wait already reached its
+ * bound.
+ */
+type StreamWaiter = { grant: (release: () => void) => boolean }
+
export class StreamGate {
private running = 0
- private queued = 0
+ private readonly waiters: StreamWaiter[] = []
constructor(
private readonly maxConcurrent: number = STREAM_LIMITS.maxConcurrentStreams,
@@ -223,17 +230,21 @@ export class StreamGate {
}
/**
- * Take a stream slot. When `signal` aborts (client disconnected) while the
- * caller is still queued, the waiter is rejected and its queue slot freed, so
- * a dropped client never proceeds to redeem an attempt. A waiter still
- * queued after `maxQueueWaitMs`, or at `deadlineAt` (epoch ms) when that
- * comes first, is rejected the same way (#739 D1). The bound is fixed when
- * the caller queues, as the smaller of the two, and measured on the
- * monotonic clock from then on. A deadline timer settles the waiter at the
- * bound, and a poll that runs after the bound checks the elapsed wait before
- * the slot count, so a slot that frees after the bound does not admit it
- * even when the event loop stalled across the bound. A free slot is granted
- * at once whatever the deadline; the caller checks a deadline already past.
+ * Take a stream slot. Waiters are granted in arrival order: a release hands
+ * its slot straight to the oldest waiter, and a new caller queues behind the
+ * waiters instead of taking a slot ahead of them. The returned release is
+ * idempotent. When `signal` aborts (client disconnected) while the caller is
+ * still queued, the waiter is rejected and removed, so a dropped client never
+ * proceeds to redeem an attempt. A waiter still queued after
+ * `maxQueueWaitMs`, or at `deadlineAt` (epoch ms) when that comes first, is
+ * rejected and removed the same way (#739 D1). The bound is fixed when the
+ * caller queues, as the smaller of the two, and measured on the monotonic
+ * clock from then on. A deadline timer settles the waiter at the bound, and
+ * a release that reaches a waiter after the bound checks the elapsed wait
+ * first and passes the slot on to the next waiter, so a slot that frees after
+ * the bound does not admit it even when the event loop stalled across the
+ * bound. A free slot with nobody queued is granted at once whatever the
+ * deadline; the caller checks a deadline already past.
*
* The refusal names which bound ran out. The gate's own bound is
* `maxQueueWaitMs` after the enqueue, or after `queueWaitStartedAt` (epoch
@@ -261,72 +272,79 @@ export class StreamGate {
)
}
if (signal?.aborted) throw new RequestLimitError('stream request was aborted', 'aborted')
- if (this.running >= this.maxConcurrent) {
- if (this.queued >= this.maxQueued)
- throw new RequestLimitError('stream queue is full', 'queue_full', this.queueCapacityCode)
- this.queued += 1
- try {
- await new Promise<void>((resolve, reject) => {
- let poll: ReturnType<typeof setTimeout> | undefined
- const settle = () => {
- if (poll !== undefined) clearTimeout(poll)
- clearTimeout(deadline)
- signal?.removeEventListener('abort', onAbort)
- }
- const onAbort = () => {
- settle()
- reject(new RequestLimitError('stream request was aborted', 'aborted'))
- }
- const queuedAt = performance.now()
- const enqueuedAt = Date.now()
- const ownBoundAt = (queueWaitStartedAt ?? enqueuedAt) + this.maxQueueWaitMs
- const deadlineGoverns =
- deadlineAt !== undefined &&
- (queueWaitStartedAt === undefined ? deadlineAt <= ownBoundAt : deadlineAt < ownBoundAt)
- // No waiter stays queued longer than maxQueueWaitMs after the enqueue.
- const boundAt = deadlineAt !== undefined && deadlineGoverns ? deadlineAt : ownBoundAt
- const waitBoundMs = Math.min(this.maxQueueWaitMs, boundAt - enqueuedAt)
- const expire = () => {
- settle()
- reject(
- new RequestLimitError(
- 'stream queue wait exceeded',
- deadlineGoverns ? 'deadline' : 'queue_wait',
- deadlineGoverns ? 'provider_unavailable' : this.queueCapacityCode
- )
- )
- }
- const deadline = setTimeout(expire, Math.max(0, waitBoundMs))
- const wait = () => {
- // After the event loop stalls, a poll and the deadline can both be
- // overdue, and Node runs the poll first because it was due first.
- if (performance.now() - queuedAt >= waitBoundMs) {
- expire()
- return
- }
- if (this.running < this.maxConcurrent) {
- settle()
- resolve()
- return
- }
- poll = setTimeout(wait, 10)
+ if (this.waiters.length === 0 && this.running < this.maxConcurrent) {
+ this.running += 1
+ return this.slotRelease()
+ }
+ if (this.waiters.length >= this.maxQueued)
+ throw new RequestLimitError('stream queue is full', 'queue_full', this.queueCapacityCode)
+ return new Promise<() => void>((resolve, reject) => {
+ const leave = () => {
+ const index = this.waiters.indexOf(waiter)
+ if (index !== -1) this.waiters.splice(index, 1)
+ clearTimeout(deadline)
+ signal?.removeEventListener('abort', onAbort)
+ }
+ const onAbort = () => {
+ leave()
+ reject(new RequestLimitError('stream request was aborted', 'aborted'))
+ }
+ const queuedAt = performance.now()
+ const enqueuedAt = Date.now()
+ const ownBoundAt = (queueWaitStartedAt ?? enqueuedAt) + this.maxQueueWaitMs
+ const deadlineGoverns =
+ deadlineAt !== undefined &&
+ (queueWaitStartedAt === undefined ? deadlineAt <= ownBoundAt : deadlineAt < ownBoundAt)
+ // No waiter stays queued longer than maxQueueWaitMs after the enqueue.
+ const boundAt = deadlineAt !== undefined && deadlineGoverns ? deadlineAt : ownBoundAt
+ const waitBoundMs = Math.min(this.maxQueueWaitMs, boundAt - enqueuedAt)
+ const expire = () => {
+ leave()
+ reject(
+ new RequestLimitError(
+ 'stream queue wait exceeded',
+ deadlineGoverns ? 'deadline' : 'queue_wait',
+ deadlineGoverns ? 'provider_unavailable' : this.queueCapacityCode
+ )
+ )
+ }
+ const waiter: StreamWaiter = {
+ grant: release => {
+ // After the event loop stalls, a release can run while the deadline
+ // timer is overdue but has not fired yet.
+ if (performance.now() - queuedAt >= waitBoundMs) {
+ expire()
+ return false
}
- signal?.addEventListener('abort', onAbort, { once: true })
- wait()
- })
- } finally {
- this.queued -= 1
+ leave()
+ resolve(release)
+ return true
+ },
}
- }
- this.running += 1
+ const deadline = setTimeout(expire, Math.max(0, waitBoundMs))
+ signal?.addEventListener('abort', onAbort, { once: true })
+ this.waiters.push(waiter)
+ if (waitBoundMs <= 0) expire()
+ })
+ }
+
+ private slotRelease(): () => void {
+ let released = false
return () => {
- this.running = Math.max(0, this.running - 1)
+ if (released) return
+ released = true
+ // The slot passes to the oldest waiter still inside its bound, so
+ // `running` stays the same; it is freed only when nobody can take it.
+ while (this.waiters.length > 0) {
+ if (this.waiters[0]!.grant(this.slotRelease())) return
+ }
+ this.running -= 1
}
}
/** Observable occupancy for tests. Production callers must not branch on this. */
snapshot(): { running: number; queued: number } {
- return { running: this.running, queued: this.queued }
+ return { running: this.running, queued: this.waiters.length }
}
}
diff --git a/codex-llm-proxy/test/requestLimits.test.ts b/codex-llm-proxy/test/requestLimits.test.ts
index 8c6bbfa68..b1d22487a 100644
--- a/codex-llm-proxy/test/requestLimits.test.ts
+++ b/codex-llm-proxy/test/requestLimits.test.ts
@@ -123,7 +123,7 @@ describe('StreamGate', () => {
releaseNext()
})
- it('clears the deadline and the poll when a queued client aborts', async () => {
+ it('clears the deadline when a queued client aborts', async () => {
vi.useFakeTimers()
try {
const gate = new StreamGate(1, 1, 55)
@@ -131,10 +131,10 @@ describe('StreamGate', () => {
const abort = new AbortController()
const { state, waiter } = track(gate, abort.signal)
await vi.advanceTimersByTimeAsync(20)
- // Witness: the waiter is queued with its deadline, its poll and its
- // abort listener live.
+ // Witness: the waiter is queued with its deadline and its abort
+ // listener live, and the gate keeps no other timer.
expect(state.outcome).toBeUndefined()
- expect(vi.getTimerCount()).toBe(2)
+ expect(vi.getTimerCount()).toBe(1)
expect(getEventListeners(abort.signal, 'abort')).toHaveLength(1)
abort.abort()
await vi.advanceTimersByTimeAsync(0)
@@ -193,9 +193,8 @@ describe('StreamGate', () => {
})
// The admission deadline shortens the bound the gate's own timer enforces.
- // 55 ms falls between two 10 ms polls, so only the deadline timer can
- // settle the waiter at 55 ms; a timer armed with maxQueueWaitMs would leave
- // it to the 60 ms poll.
+ // A timer armed with maxQueueWaitMs would leave the waiter queued past
+ // 55 ms; only the deadline timer settles it there.
it('T-AC-3b settles at deadlineAt on the deadline timer, not on the next poll', async () => {
vi.useFakeTimers()
try {
@@ -242,9 +241,9 @@ describe('StreamGate', () => {
}
})
- // The gate polls for a free slot every 10 ms. A bound that is not a multiple
- // of 10 falls between two polls, so the next two cases separate a deadline
- // the gate enforces on its own timer from one it only notices on the next poll.
+ // The gate keeps one timer per waiter, its deadline. A release hands the
+ // slot to the oldest waiter in the same turn, so the bound and a release
+ // are the only events that settle a waiter.
it('rejects at maxQueueWaitMs and leaves no timer or abort listener behind', async () => {
vi.useFakeTimers()
try {
@@ -254,10 +253,10 @@ describe('StreamGate', () => {
const { state, waiter } = track(gate, abort.signal)
await vi.advanceTimersByTimeAsync(54)
// Witness: one millisecond before the bound the waiter is still queued,
- // with its deadline, its poll and its abort listener live.
+ // with its deadline and its abort listener live, and no other timer.
expect(state.outcome).toBeUndefined()
expect(gate.snapshot()).toEqual({ running: 1, queued: 1 })
- expect(vi.getTimerCount()).toBe(2)
+ expect(vi.getTimerCount()).toBe(1)
expect(getEventListeners(abort.signal, 'abort')).toHaveLength(1)
await vi.advanceTimersByTimeAsync(1)
expect(state.outcome).toBe('stream queue wait exceeded')
@@ -277,48 +276,51 @@ describe('StreamGate', () => {
}
})
- it('settles at maxQueueWaitMs when a slot frees between the last poll and the bound', async () => {
+ it('hands a slot that frees 3 ms before the bound to the waiter in the same turn', async () => {
vi.useFakeTimers()
try {
const gate = new StreamGate(1, 1, 55)
const release = await gate.acquire()
const { state, waiter } = track(gate)
- // Last poll at 50 ms sees the slot taken; it frees at 52 ms.
await vi.advanceTimersByTimeAsync(52)
- release()
- // Witness: nothing has settled the waiter yet.
+ // Witness: still queued 3 ms before the bound.
expect(state.outcome).toBeUndefined()
- await vi.advanceTimersByTimeAsync(3)
- // The bound is a deadline: the waiter is settled at 55 ms, not at the
- // 60 ms poll, and the deadline does not look at the slot count.
- expect(state.outcome).toBe('stream queue wait exceeded')
- await waiter
+ release()
+ await vi.advanceTimersByTimeAsync(0)
+ // No poll interval separates the release from the grant, and the grant
+ // cleared the deadline timer.
+ expect(state.outcome).toBe('admitted')
+ expect(gate.snapshot()).toEqual({ running: 1, queued: 0 })
+ expect(vi.getTimerCount()).toBe(0)
+ ;(await waiter)?.()
expect(gate.snapshot()).toEqual({ running: 0, queued: 0 })
} finally {
vi.useRealTimers()
}
})
- it('refuses a waiter when the poll due at the bound finds a free slot', async () => {
+ it('refuses a waiter whose bound and a release fall due in the same millisecond', async () => {
vi.useFakeTimers()
try {
const gate = new StreamGate(1, 1, 50)
const release = await gate.acquire()
const { state, waiter } = track(gate)
- await vi.advanceTimersByTimeAsync(45)
- release()
+ // Armed after the waiter's deadline timer with the same due time, so the
+ // deadline runs first.
+ setTimeout(release, 50)
+ await vi.advanceTimersByTimeAsync(49)
expect(state.outcome).toBeUndefined()
- // The deadline and the 50 ms poll fall due together; the poll must not
- // admit a waiter whose wait has reached the bound.
- await vi.advanceTimersByTimeAsync(5)
+ await vi.advanceTimersByTimeAsync(1)
expect(state.outcome).toBe('stream queue wait exceeded')
await waiter
+ // The release found nobody queued, so the slot was freed, not leaked.
+ expect(gate.snapshot()).toEqual({ running: 0, queued: 0 })
} finally {
vi.useRealTimers()
}
})
- it('refuses a waiter whose poll runs after the bound because the event loop stalled', async () => {
+ it('refuses a waiter that a release reaches after the bound because the event loop stalled', async () => {
const gate = new StreamGate(1, 1, 500)
const release = await gate.acquire()
const { state, waiter } = track(gate)
@@ -327,18 +329,19 @@ describe('StreamGate', () => {
// pause between the two reads would come out of the 30 ms margin.
const queuedAt = performance.now()
await new Promise(resolve => setTimeout(resolve, 20))
- // Witness: the poll has run and found the slot taken. The 500 ms bound
- // leaves 25x the 20 ms wait, so a paused runner does not fire the deadline
- // first; without this witness the deadline could do the rejecting and the
- // test would pass with the elapsed check removed.
+ // Witness: the waiter is still queued. The 500 ms bound leaves 25x the
+ // 20 ms wait, so a paused runner does not fire the deadline first.
expect(state.outcome).toBeUndefined()
// Hold the event loop past the bound, then free the slot before any timer
- // can run. The next poll and the deadline are now both overdue, and Node
- // runs the poll first because it was due first.
+ // can run. The deadline is overdue, so only the elapsed check in the
+ // hand-off can refuse the waiter.
while (performance.now() - queuedAt < 530) {
// busy wait
}
release()
+ // Settled in the release turn, before the deadline timer could run: the
+ // hand-off refused the waiter and, with nobody left, freed the slot.
+ expect(gate.snapshot()).toEqual({ running: 0, queued: 0 })
;(await waiter)?.()
expect(state.outcome).toBe('stream queue wait exceeded')
})
@@ -368,6 +371,196 @@ describe('StreamGate', () => {
}
})
+ // Grant order. A release hands its slot to the oldest waiter in the same
+ // turn, and a caller that arrives while anyone is queued queues behind them.
+ describe('FIFO hand-off', () => {
+ function named(
+ gate: StreamGate,
+ name: string,
+ order: string[],
+ signal?: AbortSignal,
+ deadlineAt?: number
+ ) {
+ const state: { outcome?: string; release?: () => void } = {}
+ const waiter = gate.acquire(signal, deadlineAt).then(
+ release => {
+ order.push(name)
+ state.outcome = 'admitted'
+ state.release = release
+ },
+ (err: RequestLimitError) => {
+ state.outcome = err.kind
+ }
+ )
+ return { state, waiter }
+ }
+
+ it('grants the older of two waiters queued at different times first', async () => {
+ vi.useFakeTimers()
+ try {
+ const gate = new StreamGate(1, 2, 60_000)
+ const hold = await gate.acquire()
+ const order: string[] = []
+ const older = named(gate, 'older', order)
+ await vi.advanceTimersByTimeAsync(5)
+ const newer = named(gate, 'newer', order)
+ await vi.advanceTimersByTimeAsync(7)
+ // Witness: both are queued behind the held slot.
+ expect(gate.snapshot()).toEqual({ running: 1, queued: 2 })
+ hold()
+ await vi.advanceTimersByTimeAsync(20)
+ expect(order).toEqual(['older'])
+ expect(newer.state.outcome).toBeUndefined()
+ expect(gate.snapshot()).toEqual({ running: 1, queued: 1 })
+ older.state.release!()
+ await vi.advanceTimersByTimeAsync(0)
+ expect(order).toEqual(['older', 'newer'])
+ newer.state.release!()
+ await Promise.all([older.waiter, newer.waiter])
+ expect(gate.snapshot()).toEqual({ running: 0, queued: 0 })
+ } finally {
+ vi.useRealTimers()
+ }
+ })
+
+ it('queues a caller that arrives in the release turn behind the waiter', async () => {
+ vi.useFakeTimers()
+ try {
+ const gate = new StreamGate(1, 2, 60_000)
+ const hold = await gate.acquire()
+ const order: string[] = []
+ const queued = named(gate, 'queued', order)
+ await vi.advanceTimersByTimeAsync(3)
+ hold()
+ const late = named(gate, 'late', order)
+ // The released slot went to the queued waiter, so the late caller
+ // queued instead of taking it.
+ expect(gate.snapshot()).toEqual({ running: 1, queued: 1 })
+ await vi.advanceTimersByTimeAsync(20)
+ expect(order).toEqual(['queued'])
+ expect(late.state.outcome).toBeUndefined()
+ queued.state.release!()
+ await vi.advanceTimersByTimeAsync(0)
+ expect(order).toEqual(['queued', 'late'])
+ late.state.release!()
+ await Promise.all([queued.waiter, late.waiter])
+ expect(gate.snapshot()).toEqual({ running: 0, queued: 0 })
+ } finally {
+ vi.useRealTimers()
+ }
+ })
+
+ it('skips an aborted head and grants the waiter behind it', async () => {
+ vi.useFakeTimers()
+ try {
+ const gate = new StreamGate(1, 2, 60_000)
+ const hold = await gate.acquire()
+ const order: string[] = []
+ const abort = new AbortController()
+ const head = named(gate, 'head', order, abort.signal)
+ const next = named(gate, 'next', order)
+ abort.abort()
+ await vi.advanceTimersByTimeAsync(0)
+ expect(head.state.outcome).toBe('aborted')
+ // Only the aborted waiter left the queue.
+ expect(gate.snapshot()).toEqual({ running: 1, queued: 1 })
+ hold()
+ await vi.advanceTimersByTimeAsync(0)
+ expect(order).toEqual(['next'])
+ next.state.release!()
+ await Promise.all([head.waiter, next.waiter])
+ expect(gate.snapshot()).toEqual({ running: 0, queued: 0 })
+ } finally {
+ vi.useRealTimers()
+ }
+ })
+
+ it('removes only the waiter whose deadline expires and returns its queue place', async () => {
+ vi.useFakeTimers()
+ try {
+ const gate = new StreamGate(1, 2, 60_000)
+ const hold = await gate.acquire()
+ const order: string[] = []
+ const early = named(gate, 'early', order, undefined, Date.now() + 30)
+ const later = named(gate, 'later', order)
+ await vi.advanceTimersByTimeAsync(29)
+ // Witness: both are queued one millisecond before the first deadline.
+ expect(gate.snapshot()).toEqual({ running: 1, queued: 2 })
+ await expect(gate.acquire()).rejects.toMatchObject({ kind: 'queue_full' })
+ await vi.advanceTimersByTimeAsync(1)
+ expect(early.state.outcome).toBe('deadline')
+ expect(later.state.outcome).toBeUndefined()
+ expect(gate.snapshot()).toEqual({ running: 1, queued: 1 })
+ // The freed queue place takes a new waiter, which queues behind `later`.
+ const third = named(gate, 'third', order)
+ expect(gate.snapshot()).toEqual({ running: 1, queued: 2 })
+ hold()
+ await vi.advanceTimersByTimeAsync(0)
+ expect(order).toEqual(['later'])
+ later.state.release!()
+ await vi.advanceTimersByTimeAsync(0)
+ expect(order).toEqual(['later', 'third'])
+ third.state.release!()
+ await Promise.all([early.waiter, later.waiter, third.waiter])
+ expect(gate.snapshot()).toEqual({ running: 0, queued: 0 })
+ } finally {
+ vi.useRealTimers()
+ }
+ })
+
+ it('passes a slot past a head whose bound a stalled event loop overran', async () => {
+ const gate = new StreamGate(1, 2, 5_000)
+ const hold = await gate.acquire()
+ const order: string[] = []
+ const startedAt = Date.now()
+ const head = named(gate, 'head', order, undefined, startedAt + 50)
+ const next = named(gate, 'next', order)
+ const queuedAt = performance.now()
+ // Hold the event loop past the head's bound so its deadline timer
+ // cannot run before the release.
+ while (performance.now() - queuedAt < 80) {
+ // busy wait
+ }
+ hold()
+ // The hand-off refused the overdue head and granted the next waiter in
+ // the same turn: one slot running, nobody queued.
+ expect(gate.snapshot()).toEqual({ running: 1, queued: 0 })
+ await Promise.resolve()
+ await head.waiter
+ expect(head.state.outcome).toBe('deadline')
+ await vi.waitFor(() => expect(order).toEqual(['next']))
+ next.state.release!()
+ await next.waiter
+ expect(gate.snapshot()).toEqual({ running: 0, queued: 0 })
+ })
+
+ it('ignores a second call of the same release', async () => {
+ vi.useFakeTimers()
+ try {
+ const gate = new StreamGate(1, 2, 60_000)
+ const hold = await gate.acquire()
+ const order: string[] = []
+ const first = named(gate, 'first', order)
+ const second = named(gate, 'second', order)
+ hold()
+ hold()
+ await vi.advanceTimersByTimeAsync(0)
+ // One release handed over one slot.
+ expect(order).toEqual(['first'])
+ expect(gate.snapshot()).toEqual({ running: 1, queued: 1 })
+ first.state.release!()
+ await vi.advanceTimersByTimeAsync(0)
+ expect(order).toEqual(['first', 'second'])
+ second.state.release!()
+ second.state.release!()
+ await Promise.all([first.waiter, second.waiter])
+ expect(gate.snapshot()).toEqual({ running: 0, queued: 0 })
+ } finally {
+ vi.useRealTimers()
+ }
+ })
+ })
+
it('refuses a maxQueueWaitMs that setTimeout cannot honor', () => {
// Witness: the bounds themselves are accepted.
expect(() => new StreamGate(1, 1, 1)).not.toThrow()
@@ -425,8 +618,8 @@ describe('StreamGate', () => {
// Queue capacity: the caller after maxQueuedRequests is refused.
await expect(visualStreamGate.acquire()).rejects.toMatchObject({ kind: 'queue_full' })
for (const release of held) release()
- // The gate grants a freed slot to whichever waiter polls first, so each
- // waiter releases its slot as soon as it gets one, in any order.
+ // Each waiter releases its slot as soon as it gets one, so the cleanup
+ // does not depend on the grant order.
await Promise.all(queued.map(async waiter => (await waiter)()))
expect(visualStreamGate.snapshot()).toEqual({ running: 0, queued: 0 })
})
diff --git a/codex-llm-proxy/test/server.security.test.ts b/codex-llm-proxy/test/server.security.test.ts
index e2d8b59b9..e02e44b16 100644
--- a/codex-llm-proxy/test/server.security.test.ts
+++ b/codex-llm-proxy/test/server.security.test.ts
@@ -2516,9 +2516,8 @@ describe('codex-llm-proxy ticket-aware stream-gate wait (#739 D1-bis)', () => {
const warn = vi.spyOn(logger, 'warn')
const held: Array<() => void> = []
const waiting: Array<Promise<() => void>> = []
- // The gate grants a freed slot to whichever waiter polls first, not in
- // queue order. Cleanup therefore aborts every waiter before it frees the
- // held slots, so no waiter can take a slot that cleanup would then wait on.
+ // Cleanup aborts every waiter before it frees the held slots, so no
+ // waiter takes a slot that cleanup would then wait on.
const drain = new AbortController()
try {
for (let i = 0; i < VISUAL_STREAM_LIMITS.maxConcurrentStreams; i += 1) {
diff --git a/grok-llm-proxy/src/requestLimits.ts b/grok-llm-proxy/src/requestLimits.ts
index 32c206d03..2e8e62884 100644
--- a/grok-llm-proxy/src/requestLimits.ts
+++ b/grok-llm-proxy/src/requestLimits.ts
@@ -183,9 +183,16 @@ export function assertBoundedIdleTimeout(idleTimeoutMs: number | undefined): num
return Math.min(requested, STREAM_LIMITS.upstreamIdleTimeoutMs)
}
+/**
+ * A queued stream-gate caller. `grant` hands it the slot a release passes on
+ * and returns false, after refusing it, when its wait already reached its
+ * bound.
+ */
+type StreamWaiter = { grant: (release: () => void) => boolean }
+
export class StreamGate {
private running = 0
- private queued = 0
+ private readonly waiters: StreamWaiter[] = []
constructor(
private readonly maxConcurrent: number = STREAM_LIMITS.maxConcurrentStreams,
@@ -215,17 +222,21 @@ export class StreamGate {
}
/**
- * Take a stream slot. When `signal` aborts (client disconnected) while the
- * caller is still queued, the waiter is rejected and its queue slot freed, so
- * a dropped client never proceeds to redeem an attempt. A waiter still
- * queued after `maxQueueWaitMs`, or at `deadlineAt` (epoch ms) when that
- * comes first, is rejected the same way (#739 D1). The bound is fixed when
- * the caller queues, as the smaller of the two, and measured on the
- * monotonic clock from then on. A deadline timer settles the waiter at the
- * bound, and a poll that runs after the bound checks the elapsed wait before
- * the slot count, so a slot that frees after the bound does not admit it
- * even when the event loop stalled across the bound. A free slot is granted
- * at once whatever the deadline; the caller checks a deadline already past.
+ * Take a stream slot. Waiters are granted in arrival order: a release hands
+ * its slot straight to the oldest waiter, and a new caller queues behind the
+ * waiters instead of taking a slot ahead of them. The returned release is
+ * idempotent. When `signal` aborts (client disconnected) while the caller is
+ * still queued, the waiter is rejected and removed, so a dropped client never
+ * proceeds to redeem an attempt. A waiter still queued after
+ * `maxQueueWaitMs`, or at `deadlineAt` (epoch ms) when that comes first, is
+ * rejected and removed the same way (#739 D1). The bound is fixed when the
+ * caller queues, as the smaller of the two, and measured on the monotonic
+ * clock from then on. A deadline timer settles the waiter at the bound, and
+ * a release that reaches a waiter after the bound checks the elapsed wait
+ * first and passes the slot on to the next waiter, so a slot that frees after
+ * the bound does not admit it even when the event loop stalled across the
+ * bound. A free slot with nobody queued is granted at once whatever the
+ * deadline; the caller checks a deadline already past.
*
* The refusal names which bound ran out. The gate's own bound is
* `maxQueueWaitMs` after the enqueue, or after `queueWaitStartedAt` (epoch
@@ -253,72 +264,79 @@ export class StreamGate {
)
}
if (signal?.aborted) throw new RequestLimitError('stream request was aborted', 'aborted')
- if (this.running >= this.maxConcurrent) {
- if (this.queued >= this.maxQueued)
- throw new RequestLimitError('stream queue is full', 'queue_full', this.queueCapacityCode)
- this.queued += 1
- try {
- await new Promise<void>((resolve, reject) => {
- let poll: ReturnType<typeof setTimeout> | undefined
- const settle = () => {
- if (poll !== undefined) clearTimeout(poll)
- clearTimeout(deadline)
- signal?.removeEventListener('abort', onAbort)
- }
- const onAbort = () => {
- settle()
- reject(new RequestLimitError('stream request was aborted', 'aborted'))
- }
- const queuedAt = performance.now()
- const enqueuedAt = Date.now()
- const ownBoundAt = (queueWaitStartedAt ?? enqueuedAt) + this.maxQueueWaitMs
- const deadlineGoverns =
- deadlineAt !== undefined &&
- (queueWaitStartedAt === undefined ? deadlineAt <= ownBoundAt : deadlineAt < ownBoundAt)
- // No waiter stays queued longer than maxQueueWaitMs after the enqueue.
- const boundAt = deadlineAt !== undefined && deadlineGoverns ? deadlineAt : ownBoundAt
- const waitBoundMs = Math.min(this.maxQueueWaitMs, boundAt - enqueuedAt)
- const expire = () => {
- settle()
- reject(
- new RequestLimitError(
- 'stream queue wait exceeded',
- deadlineGoverns ? 'deadline' : 'queue_wait',
- deadlineGoverns ? 'provider_unavailable' : this.queueCapacityCode
- )
- )
- }
- const deadline = setTimeout(expire, Math.max(0, waitBoundMs))
- const wait = () => {
- // After the event loop stalls, a poll and the deadline can both be
- // overdue, and Node runs the poll first because it was due first.
- if (performance.now() - queuedAt >= waitBoundMs) {
- expire()
- return
- }
- if (this.running < this.maxConcurrent) {
- settle()
- resolve()
- return
- }
- poll = setTimeout(wait, 10)
+ if (this.waiters.length === 0 && this.running < this.maxConcurrent) {
+ this.running += 1
+ return this.slotRelease()
+ }
+ if (this.waiters.length >= this.maxQueued)
+ throw new RequestLimitError('stream queue is full', 'queue_full', this.queueCapacityCode)
+ return new Promise<() => void>((resolve, reject) => {
+ const leave = () => {
+ const index = this.waiters.indexOf(waiter)
+ if (index !== -1) this.waiters.splice(index, 1)
+ clearTimeout(deadline)
+ signal?.removeEventListener('abort', onAbort)
+ }
+ const onAbort = () => {
+ leave()
+ reject(new RequestLimitError('stream request was aborted', 'aborted'))
+ }
+ const queuedAt = performance.now()
+ const enqueuedAt = Date.now()
+ const ownBoundAt = (queueWaitStartedAt ?? enqueuedAt) + this.maxQueueWaitMs
+ const deadlineGoverns =
+ deadlineAt !== undefined &&
+ (queueWaitStartedAt === undefined ? deadlineAt <= ownBoundAt : deadlineAt < ownBoundAt)
+ // No waiter stays queued longer than maxQueueWaitMs after the enqueue.
+ const boundAt = deadlineAt !== undefined && deadlineGoverns ? deadlineAt : ownBoundAt
+ const waitBoundMs = Math.min(this.maxQueueWaitMs, boundAt - enqueuedAt)
+ const expire = () => {
+ leave()
+ reject(
+ new RequestLimitError(
+ 'stream queue wait exceeded',
+ deadlineGoverns ? 'deadline' : 'queue_wait',
+ deadlineGoverns ? 'provider_unavailable' : this.queueCapacityCode
+ )
+ )
+ }
+ const waiter: StreamWaiter = {
+ grant: release => {
+ // After the event loop stalls, a release can run while the deadline
+ // timer is overdue but has not fired yet.
+ if (performance.now() - queuedAt >= waitBoundMs) {
+ expire()
+ return false
}
- signal?.addEventListener('abort', onAbort, { once: true })
- wait()
- })
- } finally {
- this.queued -= 1
+ leave()
+ resolve(release)
+ return true
+ },
}
- }
- this.running += 1
+ const deadline = setTimeout(expire, Math.max(0, waitBoundMs))
+ signal?.addEventListener('abort', onAbort, { once: true })
+ this.waiters.push(waiter)
+ if (waitBoundMs <= 0) expire()
+ })
+ }
+
+ private slotRelease(): () => void {
+ let released = false
return () => {
- this.running = Math.max(0, this.running - 1)
+ if (released) return
+ released = true
+ // The slot passes to the oldest waiter still inside its bound, so
+ // `running` stays the same; it is freed only when nobody can take it.
+ while (this.waiters.length > 0) {
+ if (this.waiters[0]!.grant(this.slotRelease())) return
+ }
+ this.running -= 1
}
}
/** Observable occupancy for tests. Production callers must not branch on this. */
snapshot(): { running: number; queued: number } {
- return { running: this.running, queued: this.queued }
+ return { running: this.running, queued: this.waiters.length }
}
}
diff --git a/grok-llm-proxy/test/requestLimits.test.ts b/grok-llm-proxy/test/requestLimits.test.ts
index e451b11bf..05c0decb0 100644
--- a/grok-llm-proxy/test/requestLimits.test.ts
+++ b/grok-llm-proxy/test/requestLimits.test.ts
@@ -123,7 +123,7 @@ describe('StreamGate', () => {
releaseNext()
})
- it('clears the deadline and the poll when a queued client aborts', async () => {
+ it('clears the deadline when a queued client aborts', async () => {
vi.useFakeTimers()
try {
const gate = new StreamGate(1, 1, 55)
@@ -131,10 +131,10 @@ describe('StreamGate', () => {
const abort = new AbortController()
const { state, waiter } = track(gate, abort.signal)
await vi.advanceTimersByTimeAsync(20)
- // Witness: the waiter is queued with its deadline, its poll and its
- // abort listener live.
+ // Witness: the waiter is queued with its deadline and its abort
+ // listener live, and the gate keeps no other timer.
expect(state.outcome).toBeUndefined()
- expect(vi.getTimerCount()).toBe(2)
+ expect(vi.getTimerCount()).toBe(1)
expect(getEventListeners(abort.signal, 'abort')).toHaveLength(1)
abort.abort()
await vi.advanceTimersByTimeAsync(0)
@@ -198,9 +198,8 @@ describe('StreamGate', () => {
})
// The admission deadline shortens the bound the gate's own timer enforces.
- // 55 ms falls between two 10 ms polls, so only the deadline timer can
- // settle the waiter at 55 ms; a timer armed with maxQueueWaitMs would leave
- // it to the 60 ms poll.
+ // A timer armed with maxQueueWaitMs would leave the waiter queued past
+ // 55 ms; only the deadline timer settles it there.
it('T-AC-3b-grok settles at deadlineAt on the deadline timer, not on the next poll', async () => {
vi.useFakeTimers()
try {
@@ -252,9 +251,9 @@ describe('StreamGate', () => {
}
})
- // The gate polls for a free slot every 10 ms. A bound that is not a multiple
- // of 10 falls between two polls, so the next two cases separate a deadline
- // the gate enforces on its own timer from one it only notices on the next poll.
+ // The gate keeps one timer per waiter, its deadline. A release hands the
+ // slot to the oldest waiter in the same turn, so the bound and a release
+ // are the only events that settle a waiter.
it('rejects at maxQueueWaitMs and leaves no timer or abort listener behind', async () => {
vi.useFakeTimers()
try {
@@ -264,9 +263,9 @@ describe('StreamGate', () => {
const { state, waiter } = track(gate, abort.signal)
await vi.advanceTimersByTimeAsync(54)
// Witness: one millisecond before the bound the waiter is still queued,
- // with its deadline, its poll and its abort listener live.
+ // with its deadline and its abort listener live, and no other timer.
expect(state.outcome).toBeUndefined()
- expect(vi.getTimerCount()).toBe(2)
+ expect(vi.getTimerCount()).toBe(1)
expect(getEventListeners(abort.signal, 'abort')).toHaveLength(1)
await vi.advanceTimersByTimeAsync(1)
expect(state.outcome).toBe('stream queue wait exceeded')
@@ -285,47 +284,51 @@ describe('StreamGate', () => {
}
})
- it('settles at maxQueueWaitMs when a slot frees between the last poll and the bound', async () => {
+ it('hands a slot that frees 3 ms before the bound to the waiter in the same turn', async () => {
vi.useFakeTimers()
try {
const gate = new StreamGate(1, 1, 55)
const release = await gate.acquire()
const { state, waiter } = track(gate)
- // Last poll at 50 ms sees the slot taken; it frees at 52 ms.
await vi.advanceTimersByTimeAsync(52)
- release()
- // Witness: nothing has settled the waiter yet.
+ // Witness: still queued 3 ms before the bound.
expect(state.outcome).toBeUndefined()
- await vi.advanceTimersByTimeAsync(3)
- // The bound is a deadline: the waiter is settled at 55 ms, not at the
- // 60 ms poll, and the deadline does not look at the slot count.
- expect(state.outcome).toBe('stream queue wait exceeded')
- await waiter
+ release()
+ await vi.advanceTimersByTimeAsync(0)
+ // No poll interval separates the release from the grant, and the grant
+ // cleared the deadline timer.
+ expect(state.outcome).toBe('admitted')
+ expect(gate.snapshot()).toEqual({ running: 1, queued: 0 })
+ expect(vi.getTimerCount()).toBe(0)
+ ;(await waiter)?.()
+ expect(gate.snapshot()).toEqual({ running: 0, queued: 0 })
} finally {
vi.useRealTimers()
}
})
- it('refuses a waiter when the poll due at the bound finds a free slot', async () => {
+ it('refuses a waiter whose bound and a release fall due in the same millisecond', async () => {
vi.useFakeTimers()
try {
const gate = new StreamGate(1, 1, 50)
const release = await gate.acquire()
const { state, waiter } = track(gate)
- await vi.advanceTimersByTimeAsync(45)
- release()
+ // Armed after the waiter's deadline timer with the same due time, so the
+ // deadline runs first.
+ setTimeout(release, 50)
+ await vi.advanceTimersByTimeAsync(49)
expect(state.outcome).toBeUndefined()
- // The deadline and the 50 ms poll fall due together; the poll must not
- // admit a waiter whose wait has reached the bound.
- await vi.advanceTimersByTimeAsync(5)
+ await vi.advanceTimersByTimeAsync(1)
expect(state.outcome).toBe('stream queue wait exceeded')
await waiter
+ // The release found nobody queued, so the slot was freed, not leaked.
+ expect(gate.snapshot()).toEqual({ running: 0, queued: 0 })
} finally {
vi.useRealTimers()
}
})
- it('refuses a waiter whose poll runs after the bound because the event loop stalled', async () => {
+ it('refuses a waiter that a release reaches after the bound because the event loop stalled', async () => {
const gate = new StreamGate(1, 1, 500)
const release = await gate.acquire()
const { state, waiter } = track(gate)
@@ -334,18 +337,19 @@ describe('StreamGate', () => {
// pause between the two reads would come out of the 30 ms margin.
const queuedAt = performance.now()
await new Promise(resolve => setTimeout(resolve, 20))
- // Witness: the poll has run and found the slot taken. The 500 ms bound
- // leaves 25x the 20 ms wait, so a paused runner does not fire the deadline
- // first; without this witness the deadline could do the rejecting and the
- // test would pass with the elapsed check removed.
+ // Witness: the waiter is still queued. The 500 ms bound leaves 25x the
+ // 20 ms wait, so a paused runner does not fire the deadline first.
expect(state.outcome).toBeUndefined()
// Hold the event loop past the bound, then free the slot before any timer
- // can run. The next poll and the deadline are now both overdue, and Node
- // runs the poll first because it was due first.
+ // can run. The deadline is overdue, so only the elapsed check in the
+ // hand-off can refuse the waiter.
while (performance.now() - queuedAt < 530) {
// busy wait
}
release()
+ // Settled in the release turn, before the deadline timer could run: the
+ // hand-off refused the waiter and, with nobody left, freed the slot.
+ expect(gate.snapshot()).toEqual({ running: 0, queued: 0 })
;(await waiter)?.()
expect(state.outcome).toBe('stream queue wait exceeded')
})
@@ -374,6 +378,196 @@ describe('StreamGate', () => {
}
})
+ // Grant order. A release hands its slot to the oldest waiter in the same
+ // turn, and a caller that arrives while anyone is queued queues behind them.
+ describe('FIFO hand-off', () => {
+ function named(
+ gate: StreamGate,
+ name: string,
+ order: string[],
+ signal?: AbortSignal,
+ deadlineAt?: number
+ ) {
+ const state: { outcome?: string; release?: () => void } = {}
+ const waiter = gate.acquire(signal, deadlineAt).then(
+ release => {
+ order.push(name)
+ state.outcome = 'admitted'
+ state.release = release
+ },
+ (err: RequestLimitError) => {
+ state.outcome = err.kind
+ }
+ )
+ return { state, waiter }
+ }
+
+ it('grants the older of two waiters queued at different times first', async () => {
+ vi.useFakeTimers()
+ try {
+ const gate = new StreamGate(1, 2, 60_000)
+ const hold = await gate.acquire()
+ const order: string[] = []
+ const older = named(gate, 'older', order)
+ await vi.advanceTimersByTimeAsync(5)
+ const newer = named(gate, 'newer', order)
+ await vi.advanceTimersByTimeAsync(7)
+ // Witness: both are queued behind the held slot.
+ expect(gate.snapshot()).toEqual({ running: 1, queued: 2 })
+ hold()
+ await vi.advanceTimersByTimeAsync(20)
+ expect(order).toEqual(['older'])
+ expect(newer.state.outcome).toBeUndefined()
+ expect(gate.snapshot()).toEqual({ running: 1, queued: 1 })
+ older.state.release!()
+ await vi.advanceTimersByTimeAsync(0)
+ expect(order).toEqual(['older', 'newer'])
+ newer.state.release!()
+ await Promise.all([older.waiter, newer.waiter])
+ expect(gate.snapshot()).toEqual({ running: 0, queued: 0 })
+ } finally {
+ vi.useRealTimers()
+ }
+ })
+
+ it('queues a caller that arrives in the release turn behind the waiter', async () => {
+ vi.useFakeTimers()
+ try {
+ const gate = new StreamGate(1, 2, 60_000)
+ const hold = await gate.acquire()
+ const order: string[] = []
+ const queued = named(gate, 'queued', order)
+ await vi.advanceTimersByTimeAsync(3)
+ hold()
+ const late = named(gate, 'late', order)
+ // The released slot went to the queued waiter, so the late caller
+ // queued instead of taking it.
+ expect(gate.snapshot()).toEqual({ running: 1, queued: 1 })
+ await vi.advanceTimersByTimeAsync(20)
+ expect(order).toEqual(['queued'])
+ expect(late.state.outcome).toBeUndefined()
+ queued.state.release!()
+ await vi.advanceTimersByTimeAsync(0)
+ expect(order).toEqual(['queued', 'late'])
+ late.state.release!()
+ await Promise.all([queued.waiter, late.waiter])
+ expect(gate.snapshot()).toEqual({ running: 0, queued: 0 })
+ } finally {
+ vi.useRealTimers()
+ }
+ })
+
+ it('skips an aborted head and grants the waiter behind it', async () => {
+ vi.useFakeTimers()
+ try {
+ const gate = new StreamGate(1, 2, 60_000)
+ const hold = await gate.acquire()
+ const order: string[] = []
+ const abort = new AbortController()
+ const head = named(gate, 'head', order, abort.signal)
+ const next = named(gate, 'next', order)
+ abort.abort()
+ await vi.advanceTimersByTimeAsync(0)
+ expect(head.state.outcome).toBe('aborted')
+ // Only the aborted waiter left the queue.
+ expect(gate.snapshot()).toEqual({ running: 1, queued: 1 })
+ hold()
+ await vi.advanceTimersByTimeAsync(0)
+ expect(order).toEqual(['next'])
+ next.state.release!()
+ await Promise.all([head.waiter, next.waiter])
+ expect(gate.snapshot()).toEqual({ running: 0, queued: 0 })
+ } finally {
+ vi.useRealTimers()
+ }
+ })
+
+ it('removes only the waiter whose deadline expires and returns its queue place', async () => {
+ vi.useFakeTimers()
+ try {
+ const gate = new StreamGate(1, 2, 60_000)
+ const hold = await gate.acquire()
+ const order: string[] = []
+ const early = named(gate, 'early', order, undefined, Date.now() + 30)
+ const later = named(gate, 'later', order)
+ await vi.advanceTimersByTimeAsync(29)
+ // Witness: both are queued one millisecond before the first deadline.
+ expect(gate.snapshot()).toEqual({ running: 1, queued: 2 })
+ await expect(gate.acquire()).rejects.toMatchObject({ kind: 'queue_full' })
+ await vi.advanceTimersByTimeAsync(1)
+ expect(early.state.outcome).toBe('deadline')
+ expect(later.state.outcome).toBeUndefined()
+ expect(gate.snapshot()).toEqual({ running: 1, queued: 1 })
+ // The freed queue place takes a new waiter, which queues behind `later`.
+ const third = named(gate, 'third', order)
+ expect(gate.snapshot()).toEqual({ running: 1, queued: 2 })
+ hold()
+ await vi.advanceTimersByTimeAsync(0)
+ expect(order).toEqual(['later'])
+ later.state.release!()
+ await vi.advanceTimersByTimeAsync(0)
+ expect(order).toEqual(['later', 'third'])
+ third.state.release!()
+ await Promise.all([early.waiter, later.waiter, third.waiter])
+ expect(gate.snapshot()).toEqual({ running: 0, queued: 0 })
+ } finally {
+ vi.useRealTimers()
+ }
+ })
+
+ it('passes a slot past a head whose bound a stalled event loop overran', async () => {
+ const gate = new StreamGate(1, 2, 5_000)
+ const hold = await gate.acquire()
+ const order: string[] = []
+ const startedAt = Date.now()
+ const head = named(gate, 'head', order, undefined, startedAt + 50)
+ const next = named(gate, 'next', order)
+ const queuedAt = performance.now()
+ // Hold the event loop past the head's bound so its deadline timer
+ // cannot run before the release.
+ while (performance.now() - queuedAt < 80) {
+ // busy wait
+ }
+ hold()
+ // The hand-off refused the overdue head and granted the next waiter in
+ // the same turn: one slot running, nobody queued.
+ expect(gate.snapshot()).toEqual({ running: 1, queued: 0 })
+ await Promise.resolve()
+ await head.waiter
+ expect(head.state.outcome).toBe('deadline')
+ await vi.waitFor(() => expect(order).toEqual(['next']))
+ next.state.release!()
+ await next.waiter
+ expect(gate.snapshot()).toEqual({ running: 0, queued: 0 })
+ })
+
+ it('ignores a second call of the same release', async () => {
+ vi.useFakeTimers()
+ try {
+ const gate = new StreamGate(1, 2, 60_000)
+ const hold = await gate.acquire()
+ const order: string[] = []
+ const first = named(gate, 'first', order)
+ const second = named(gate, 'second', order)
+ hold()
+ hold()
+ await vi.advanceTimersByTimeAsync(0)
+ // One release handed over one slot.
+ expect(order).toEqual(['first'])
+ expect(gate.snapshot()).toEqual({ running: 1, queued: 1 })
+ first.state.release!()
+ await vi.advanceTimersByTimeAsync(0)
+ expect(order).toEqual(['first', 'second'])
+ second.state.release!()
+ second.state.release!()
+ await Promise.all([first.waiter, second.waiter])
+ expect(gate.snapshot()).toEqual({ running: 0, queued: 0 })
+ } finally {
+ vi.useRealTimers()
+ }
+ })
+ })
+
it('refuses a maxQueueWaitMs that setTimeout cannot honor', () => {
// Witness: the bounds themselves are accepted.
expect(() => new StreamGate(1, 1, 1)).not.toThrow()
@@ -419,8 +613,8 @@ describe('StreamGate', () => {
expect(gate.snapshot()).toEqual({ running: 1, queued: 4 })
await expect(gate.acquire()).rejects.toBeInstanceOf(RequestLimitError)
held()
- // The gate grants a freed slot to whichever waiter polls first, so each
- // waiter releases its slot as soon as it gets one, in any order.
+ // Each waiter releases its slot as soon as it gets one, so the cleanup
+ // does not depend on the grant order.
await Promise.all(queued.map(async waiter => (await waiter)()))
expect(gate.snapshot()).toEqual({ running: 0, queued: 0 })
})
@@ -439,8 +633,8 @@ describe('StreamGate', () => {
// Queue capacity: the caller after maxQueuedRequests is refused.
await expect(visualStreamGate.acquire()).rejects.toMatchObject({ kind: 'queue_full' })
for (const release of held) release()
- // The gate grants a freed slot to whichever waiter polls first, so each
- // waiter releases its slot as soon as it gets one, in any order.
+ // Each waiter releases its slot as soon as it gets one, so the cleanup
+ // does not depend on the grant order.
await Promise.all(queued.map(async waiter => (await waiter)()))
expect(visualStreamGate.snapshot()).toEqual({ running: 0, queued: 0 })
})