Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript

fix(llm-proxy): enforce the stream queue wait bound on its own timer

Alfredo Lopez8fb48c07ab03
diff --git a/codex-llm-proxy/src/requestLimits.ts b/codex-llm-proxy/src/requestLimits.ts
index 237b8738d..4b9cf2285 100644
--- a/codex-llm-proxy/src/requestLimits.ts
+++ b/codex-llm-proxy/src/requestLimits.ts
@@ -25,6 +25,9 @@ export const VISUAL_STREAM_LIMITS = {
maxQueuedRequests: 8,
} as const
+// Largest delay setTimeout honors; Node fires anything above it after 1 ms.
+const MAX_TIMER_DELAY_MS = 2_147_483_647
+
export class RequestLimitError extends Error {
readonly code = 'provider_unavailable'
constructor(message: string) {
@@ -62,13 +65,20 @@ export class StreamGate {
private readonly maxConcurrent: number = STREAM_LIMITS.maxConcurrentStreams,
private readonly maxQueued: number = STREAM_LIMITS.maxQueuedRequests,
private readonly maxQueueWaitMs: number = STREAM_LIMITS.maxQueueWaitMs
- ) {}
+ ) {
+ // setTimeout fires after 1 ms for any delay above 2^31 - 1, which would
+ // refuse every queued waiter at once instead of after the configured wait.
+ if (!Number.isInteger(maxQueueWaitMs) || maxQueueWaitMs <= 0 || maxQueueWaitMs > MAX_TIMER_DELAY_MS) {
+ throw new RangeError(`maxQueueWaitMs must be an integer in 1..${MAX_TIMER_DELAY_MS}`)
+ }
+ }
/**
* Take a stream slot. When `signal` aborts (client disconnected) while the
* caller is still queued, the waiter is rejected and its queue slot freed, so
* a dropped client never proceeds to redeem an attempt. A waiter still
- * queued after `maxQueueWaitMs` is rejected the same way.
+ * queued after `maxQueueWaitMs` is rejected the same way, on its own timer:
+ * a slot that frees after the bound and before the next poll does not admit it.
*/
async acquire(signal?: AbortSignal): Promise<() => void> {
if (signal?.aborted) throw new RequestLimitError('stream request was aborted')
@@ -76,25 +86,28 @@ export class StreamGate {
if (this.queued >= this.maxQueued) throw new RequestLimitError('stream queue is full')
this.queued += 1
try {
- const queuedAt = Date.now()
await new Promise<void>((resolve, reject) => {
- let timer: ReturnType<typeof setTimeout> | undefined
+ let poll: ReturnType<typeof setTimeout> | undefined
+ const settle = () => {
+ if (poll !== undefined) clearTimeout(poll)
+ clearTimeout(deadline)
+ signal?.removeEventListener('abort', onAbort)
+ }
const onAbort = () => {
- if (timer !== undefined) clearTimeout(timer)
+ settle()
reject(new RequestLimitError('stream request was aborted'))
}
+ const deadline = setTimeout(() => {
+ settle()
+ reject(new RequestLimitError('stream queue wait exceeded'))
+ }, this.maxQueueWaitMs)
const wait = () => {
if (this.running < this.maxConcurrent) {
- signal?.removeEventListener('abort', onAbort)
+ settle()
resolve()
return
}
- if (Date.now() - queuedAt >= this.maxQueueWaitMs) {
- signal?.removeEventListener('abort', onAbort)
- reject(new RequestLimitError('stream queue wait exceeded'))
- return
- }
- timer = setTimeout(wait, 10)
+ poll = setTimeout(wait, 10)
}
signal?.addEventListener('abort', onAbort, { once: true })
wait()
diff --git a/codex-llm-proxy/test/requestLimits.test.ts b/codex-llm-proxy/test/requestLimits.test.ts
index b06dc6f84..b5f7ce363 100644
--- a/codex-llm-proxy/test/requestLimits.test.ts
+++ b/codex-llm-proxy/test/requestLimits.test.ts
@@ -1,4 +1,4 @@
-import { describe, expect, it } from 'vitest'
+import { describe, expect, it, vi } from 'vitest'
import { CONTROL_API_REQUEST_TIMEOUT_MS } from '../src/controlApiClient.js'
import {
assertBoundedDeadline,
@@ -120,6 +120,114 @@ describe('StreamGate', () => {
releaseNext()
})
+ // The gate polls for a free slot every 10 ms. A bound that is not a multiple
+ // of 10 falls between two polls, so these two cases separate a deadline the
+ // gate enforces on its own timer from one it only notices on the next poll.
+ it('rejects at maxQueueWaitMs even when a slot frees before the next poll', async () => {
+ vi.useFakeTimers()
+ try {
+ const gate = new StreamGate(1, 1, 55)
+ const release = await gate.acquire()
+ let outcome: string | undefined
+ const waiter = gate.acquire().then(
+ () => {
+ outcome = 'admitted'
+ },
+ (err: Error) => {
+ outcome = err.message
+ }
+ )
+ await vi.advanceTimersByTimeAsync(54)
+ // Witness: the waiter is still queued one millisecond before the bound.
+ expect(outcome).toBeUndefined()
+ expect(gate.snapshot()).toEqual({ running: 1, queued: 1 })
+ await vi.advanceTimersByTimeAsync(1)
+ expect(outcome).toBe('stream queue wait exceeded')
+ // The poll stopped with the rejection: no timer is left behind.
+ expect(vi.getTimerCount()).toBe(0)
+ // A slot that frees after the bound and before the next poll (60 ms)
+ // must not reach the waiter that was already refused.
+ await vi.advanceTimersByTimeAsync(2)
+ release()
+ await vi.advanceTimersByTimeAsync(10)
+ await waiter
+ expect(outcome).toBe('stream queue wait exceeded')
+ expect(gate.snapshot()).toEqual({ running: 0, queued: 0 })
+ } finally {
+ vi.useRealTimers()
+ }
+ })
+
+ it('admits a waiter whose slot frees before maxQueueWaitMs', async () => {
+ vi.useFakeTimers()
+ try {
+ const gate = new StreamGate(1, 1, 55)
+ const release = await gate.acquire()
+ let outcome: string | undefined
+ const waiter = gate.acquire().then(
+ next => {
+ outcome = 'admitted'
+ return next
+ },
+ (err: Error) => {
+ outcome = err.message
+ return undefined
+ }
+ )
+ await vi.advanceTimersByTimeAsync(45)
+ release()
+ await vi.advanceTimersByTimeAsync(5)
+ expect(outcome).toBe('admitted')
+ // The deadline timer was cleared on admission, before it could fire.
+ expect(vi.getTimerCount()).toBe(0)
+ await vi.advanceTimersByTimeAsync(100)
+ expect(outcome).toBe('admitted')
+ expect(gate.snapshot()).toEqual({ running: 1, queued: 0 })
+ ;(await waiter)?.()
+ } finally {
+ vi.useRealTimers()
+ }
+ })
+
+ it('settles at maxQueueWaitMs when a slot frees between the last poll and the bound', async () => {
+ vi.useFakeTimers()
+ try {
+ const gate = new StreamGate(1, 1, 55)
+ const release = await gate.acquire()
+ let outcome: string | undefined
+ const waiter = gate.acquire().then(
+ () => {
+ outcome = 'admitted'
+ },
+ (err: Error) => {
+ outcome = err.message
+ }
+ )
+ // Last poll at 50 ms sees the slot taken; it frees at 52 ms.
+ await vi.advanceTimersByTimeAsync(52)
+ release()
+ // Witness: nothing has settled the waiter yet.
+ expect(outcome).toBeUndefined()
+ await vi.advanceTimersByTimeAsync(3)
+ // The bound is a deadline: the waiter is settled at 55 ms, not at the
+ // 60 ms poll, and the deadline does not look at the slot count.
+ expect(outcome).toBe('stream queue wait exceeded')
+ await waiter
+ expect(gate.snapshot()).toEqual({ running: 0, queued: 0 })
+ } finally {
+ vi.useRealTimers()
+ }
+ })
+
+ it('refuses a maxQueueWaitMs that setTimeout cannot honor', () => {
+ // Witness: the bounds themselves are accepted.
+ expect(() => new StreamGate(1, 1, 1)).not.toThrow()
+ expect(() => new StreamGate(1, 1, 2_147_483_647)).not.toThrow()
+ for (const maxQueueWaitMs of [0, -1, 1.5, Number.NaN, 2_147_483_648]) {
+ expect(() => new StreamGate(1, 1, maxQueueWaitMs), String(maxQueueWaitMs)).toThrow(RangeError)
+ }
+ })
+
it('pins the visual 2/8 sibling against the ordinary 8/16 stream gate', () => {
expect(VISUAL_STREAM_LIMITS).toEqual({ maxConcurrentStreams: 2, maxQueuedRequests: 8 })
expect(STREAM_LIMITS.maxConcurrentStreams).toBe(8)
diff --git a/grok-llm-proxy/src/requestLimits.ts b/grok-llm-proxy/src/requestLimits.ts
index c6c9d304c..1353fa7d9 100644
--- a/grok-llm-proxy/src/requestLimits.ts
+++ b/grok-llm-proxy/src/requestLimits.ts
@@ -13,6 +13,9 @@ export const STREAM_LIMITS = {
upstreamIdleTimeoutMs: 600_000,
} as const
+// Largest delay setTimeout honors; Node fires anything above it after 1 ms.
+const MAX_TIMER_DELAY_MS = 2_147_483_647
+
export class RequestLimitError extends Error {
readonly code = 'provider_unavailable'
constructor(message: string) {
@@ -58,13 +61,20 @@ export class StreamGate {
private readonly maxConcurrent: number = STREAM_LIMITS.maxConcurrentStreams,
private readonly maxQueued: number = STREAM_LIMITS.maxQueuedRequests,
private readonly maxQueueWaitMs: number = STREAM_LIMITS.maxQueueWaitMs
- ) {}
+ ) {
+ // setTimeout fires after 1 ms for any delay above 2^31 - 1, which would
+ // refuse every queued waiter at once instead of after the configured wait.
+ if (!Number.isInteger(maxQueueWaitMs) || maxQueueWaitMs <= 0 || maxQueueWaitMs > MAX_TIMER_DELAY_MS) {
+ throw new RangeError(`maxQueueWaitMs must be an integer in 1..${MAX_TIMER_DELAY_MS}`)
+ }
+ }
/**
* Take a stream slot. When `signal` aborts (client disconnected) while the
* caller is still queued, the waiter is rejected and its queue slot freed, so
* a dropped client never proceeds to redeem an attempt. A waiter still
- * queued after `maxQueueWaitMs` is rejected the same way.
+ * queued after `maxQueueWaitMs` is rejected the same way, on its own timer:
+ * a slot that frees after the bound and before the next poll does not admit it.
*/
async acquire(signal?: AbortSignal): Promise<() => void> {
if (signal?.aborted) throw new RequestLimitError('stream request was aborted')
@@ -72,25 +82,28 @@ export class StreamGate {
if (this.queued >= this.maxQueued) throw new RequestLimitError('stream queue is full')
this.queued += 1
try {
- const queuedAt = Date.now()
await new Promise<void>((resolve, reject) => {
- let timer: ReturnType<typeof setTimeout> | undefined
+ let poll: ReturnType<typeof setTimeout> | undefined
+ const settle = () => {
+ if (poll !== undefined) clearTimeout(poll)
+ clearTimeout(deadline)
+ signal?.removeEventListener('abort', onAbort)
+ }
const onAbort = () => {
- if (timer !== undefined) clearTimeout(timer)
+ settle()
reject(new RequestLimitError('stream request was aborted'))
}
+ const deadline = setTimeout(() => {
+ settle()
+ reject(new RequestLimitError('stream queue wait exceeded'))
+ }, this.maxQueueWaitMs)
const wait = () => {
if (this.running < this.maxConcurrent) {
- signal?.removeEventListener('abort', onAbort)
+ settle()
resolve()
return
}
- if (Date.now() - queuedAt >= this.maxQueueWaitMs) {
- signal?.removeEventListener('abort', onAbort)
- reject(new RequestLimitError('stream queue wait exceeded'))
- return
- }
- timer = setTimeout(wait, 10)
+ poll = setTimeout(wait, 10)
}
signal?.addEventListener('abort', onAbort, { once: true })
wait()
diff --git a/grok-llm-proxy/test/requestLimits.test.ts b/grok-llm-proxy/test/requestLimits.test.ts
index dcee1d5f6..52b8ce4e3 100644
--- a/grok-llm-proxy/test/requestLimits.test.ts
+++ b/grok-llm-proxy/test/requestLimits.test.ts
@@ -1,4 +1,4 @@
-import { describe, expect, it } from 'vitest'
+import { describe, expect, it, vi } from 'vitest'
import { CONTROL_API_REQUEST_TIMEOUT_MS } from '../src/controlApiClient.js'
import {
assertBoundedDeadline,
@@ -114,4 +114,108 @@ describe('StreamGate', () => {
const releaseNext = await next
releaseNext()
})
+
+ // The gate polls for a free slot every 10 ms. A bound that is not a multiple
+ // of 10 falls between two polls, so these two cases separate a deadline the
+ // gate enforces on its own timer from one it only notices on the next poll.
+ it('rejects at maxQueueWaitMs even when a slot frees before the next poll', async () => {
+ vi.useFakeTimers()
+ try {
+ const gate = new StreamGate(1, 1, 55)
+ const release = await gate.acquire()
+ let outcome: string | undefined
+ const waiter = gate.acquire().then(
+ () => {
+ outcome = 'admitted'
+ },
+ (err: Error) => {
+ outcome = err.message
+ }
+ )
+ await vi.advanceTimersByTimeAsync(54)
+ // Witness: the waiter is still queued one millisecond before the bound.
+ expect(outcome).toBeUndefined()
+ await vi.advanceTimersByTimeAsync(1)
+ expect(outcome).toBe('stream queue wait exceeded')
+ // The poll stopped with the rejection: no timer is left behind.
+ expect(vi.getTimerCount()).toBe(0)
+ // A slot that frees after the bound and before the next poll (60 ms)
+ // must not reach the waiter that was already refused.
+ await vi.advanceTimersByTimeAsync(2)
+ release()
+ await vi.advanceTimersByTimeAsync(10)
+ await waiter
+ expect(outcome).toBe('stream queue wait exceeded')
+ } finally {
+ vi.useRealTimers()
+ }
+ })
+
+ it('settles at maxQueueWaitMs when a slot frees between the last poll and the bound', async () => {
+ vi.useFakeTimers()
+ try {
+ const gate = new StreamGate(1, 1, 55)
+ const release = await gate.acquire()
+ let outcome: string | undefined
+ const waiter = gate.acquire().then(
+ () => {
+ outcome = 'admitted'
+ },
+ (err: Error) => {
+ outcome = err.message
+ }
+ )
+ // Last poll at 50 ms sees the slot taken; it frees at 52 ms.
+ await vi.advanceTimersByTimeAsync(52)
+ release()
+ // Witness: nothing has settled the waiter yet.
+ expect(outcome).toBeUndefined()
+ await vi.advanceTimersByTimeAsync(3)
+ // The bound is a deadline: the waiter is settled at 55 ms, not at the
+ // 60 ms poll, and the deadline does not look at the slot count.
+ expect(outcome).toBe('stream queue wait exceeded')
+ await waiter
+ } finally {
+ vi.useRealTimers()
+ }
+ })
+
+ it('refuses a maxQueueWaitMs that setTimeout cannot honor', () => {
+ // Witness: the bounds themselves are accepted.
+ expect(() => new StreamGate(1, 1, 1)).not.toThrow()
+ expect(() => new StreamGate(1, 1, 2_147_483_647)).not.toThrow()
+ for (const maxQueueWaitMs of [0, -1, 1.5, Number.NaN, 2_147_483_648]) {
+ expect(() => new StreamGate(1, 1, maxQueueWaitMs), String(maxQueueWaitMs)).toThrow(RangeError)
+ }
+ })
+
+ it('admits a waiter whose slot frees before maxQueueWaitMs', async () => {
+ vi.useFakeTimers()
+ try {
+ const gate = new StreamGate(1, 1, 55)
+ const release = await gate.acquire()
+ let outcome: string | undefined
+ const waiter = gate.acquire().then(
+ next => {
+ outcome = 'admitted'
+ return next
+ },
+ (err: Error) => {
+ outcome = err.message
+ return undefined
+ }
+ )
+ await vi.advanceTimersByTimeAsync(45)
+ release()
+ await vi.advanceTimersByTimeAsync(5)
+ expect(outcome).toBe('admitted')
+ // The deadline timer was cleared on admission, before it could fire.
+ expect(vi.getTimerCount()).toBe(0)
+ await vi.advanceTimersByTimeAsync(100)
+ expect(outcome).toBe('admitted')
+ ;(await waiter)?.()
+ } finally {
+ vi.useRealTimers()
+ }
+ })
})