Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
fix(llm-proxy): enforce the stream queue wait bound on its own timer
Alfredo Lopez8fb48c07ab03
diff --git a/codex-llm-proxy/src/requestLimits.ts b/codex-llm-proxy/src/requestLimits.tsindex 237b8738d..4b9cf2285 100644--- a/codex-llm-proxy/src/requestLimits.ts+++ b/codex-llm-proxy/src/requestLimits.ts@@ -25,6 +25,9 @@ export const VISUAL_STREAM_LIMITS = {maxQueuedRequests: 8,} as const+// Largest delay setTimeout honors; Node fires anything above it after 1 ms.+const MAX_TIMER_DELAY_MS = 2_147_483_647+export class RequestLimitError extends Error {readonly code = 'provider_unavailable'constructor(message: string) {@@ -62,13 +65,20 @@ export class StreamGate {private readonly maxConcurrent: number = STREAM_LIMITS.maxConcurrentStreams,private readonly maxQueued: number = STREAM_LIMITS.maxQueuedRequests,private readonly maxQueueWaitMs: number = STREAM_LIMITS.maxQueueWaitMs- ) {}+ ) {+ // setTimeout fires after 1 ms for any delay above 2^31 - 1, which would+ // refuse every queued waiter at once instead of after the configured wait.+ if (!Number.isInteger(maxQueueWaitMs) || maxQueueWaitMs <= 0 || maxQueueWaitMs > MAX_TIMER_DELAY_MS) {+ throw new RangeError(`maxQueueWaitMs must be an integer in 1..${MAX_TIMER_DELAY_MS}`)+ }+ }/*** Take a stream slot. When `signal` aborts (client disconnected) while the* caller is still queued, the waiter is rejected and its queue slot freed, so* a dropped client never proceeds to redeem an attempt. A waiter still- * queued after `maxQueueWaitMs` is rejected the same way.+ * queued after `maxQueueWaitMs` is rejected the same way, on its own timer:+ * a slot that frees after the bound and before the next poll does not admit it.*/async acquire(signal?: AbortSignal): Promise<() => void> {if (signal?.aborted) throw new RequestLimitError('stream request was aborted')@@ -76,25 +86,28 @@ export class StreamGate {if (this.queued >= this.maxQueued) throw new RequestLimitError('stream queue is full')this.queued += 1try {- const queuedAt = Date.now()await new Promise<void>((resolve, reject) => {- let timer: ReturnType<typeof setTimeout> | undefined+ let poll: ReturnType<typeof setTimeout> | undefined+ const settle = () => {+ if (poll !== undefined) clearTimeout(poll)+ clearTimeout(deadline)+ signal?.removeEventListener('abort', onAbort)+ }const onAbort = () => {- if (timer !== undefined) clearTimeout(timer)+ settle()reject(new RequestLimitError('stream request was aborted'))}+ const deadline = setTimeout(() => {+ settle()+ reject(new RequestLimitError('stream queue wait exceeded'))+ }, this.maxQueueWaitMs)const wait = () => {if (this.running < this.maxConcurrent) {- signal?.removeEventListener('abort', onAbort)+ settle()resolve()return}- if (Date.now() - queuedAt >= this.maxQueueWaitMs) {- signal?.removeEventListener('abort', onAbort)- reject(new RequestLimitError('stream queue wait exceeded'))- return- }- timer = setTimeout(wait, 10)+ poll = setTimeout(wait, 10)}signal?.addEventListener('abort', onAbort, { once: true })wait()diff --git a/codex-llm-proxy/test/requestLimits.test.ts b/codex-llm-proxy/test/requestLimits.test.tsindex b06dc6f84..b5f7ce363 100644--- a/codex-llm-proxy/test/requestLimits.test.ts+++ b/codex-llm-proxy/test/requestLimits.test.ts@@ -1,4 +1,4 @@-import { describe, expect, it } from 'vitest'+import { describe, expect, it, vi } from 'vitest'import { CONTROL_API_REQUEST_TIMEOUT_MS } from '../src/controlApiClient.js'import {assertBoundedDeadline,@@ -120,6 +120,114 @@ describe('StreamGate', () => {releaseNext()})+ // The gate polls for a free slot every 10 ms. A bound that is not a multiple+ // of 10 falls between two polls, so these two cases separate a deadline the+ // gate enforces on its own timer from one it only notices on the next poll.+ it('rejects at maxQueueWaitMs even when a slot frees before the next poll', async () => {+ vi.useFakeTimers()+ try {+ const gate = new StreamGate(1, 1, 55)+ const release = await gate.acquire()+ let outcome: string | undefined+ const waiter = gate.acquire().then(+ () => {+ outcome = 'admitted'+ },+ (err: Error) => {+ outcome = err.message+ }+ )+ await vi.advanceTimersByTimeAsync(54)+ // Witness: the waiter is still queued one millisecond before the bound.+ expect(outcome).toBeUndefined()+ expect(gate.snapshot()).toEqual({ running: 1, queued: 1 })+ await vi.advanceTimersByTimeAsync(1)+ expect(outcome).toBe('stream queue wait exceeded')+ // The poll stopped with the rejection: no timer is left behind.+ expect(vi.getTimerCount()).toBe(0)+ // A slot that frees after the bound and before the next poll (60 ms)+ // must not reach the waiter that was already refused.+ await vi.advanceTimersByTimeAsync(2)+ release()+ await vi.advanceTimersByTimeAsync(10)+ await waiter+ expect(outcome).toBe('stream queue wait exceeded')+ expect(gate.snapshot()).toEqual({ running: 0, queued: 0 })+ } finally {+ vi.useRealTimers()+ }+ })++ it('admits a waiter whose slot frees before maxQueueWaitMs', async () => {+ vi.useFakeTimers()+ try {+ const gate = new StreamGate(1, 1, 55)+ const release = await gate.acquire()+ let outcome: string | undefined+ const waiter = gate.acquire().then(+ next => {+ outcome = 'admitted'+ return next+ },+ (err: Error) => {+ outcome = err.message+ return undefined+ }+ )+ await vi.advanceTimersByTimeAsync(45)+ release()+ await vi.advanceTimersByTimeAsync(5)+ expect(outcome).toBe('admitted')+ // The deadline timer was cleared on admission, before it could fire.+ expect(vi.getTimerCount()).toBe(0)+ await vi.advanceTimersByTimeAsync(100)+ expect(outcome).toBe('admitted')+ expect(gate.snapshot()).toEqual({ running: 1, queued: 0 })+ ;(await waiter)?.()+ } finally {+ vi.useRealTimers()+ }+ })++ it('settles at maxQueueWaitMs when a slot frees between the last poll and the bound', async () => {+ vi.useFakeTimers()+ try {+ const gate = new StreamGate(1, 1, 55)+ const release = await gate.acquire()+ let outcome: string | undefined+ const waiter = gate.acquire().then(+ () => {+ outcome = 'admitted'+ },+ (err: Error) => {+ outcome = err.message+ }+ )+ // Last poll at 50 ms sees the slot taken; it frees at 52 ms.+ await vi.advanceTimersByTimeAsync(52)+ release()+ // Witness: nothing has settled the waiter yet.+ expect(outcome).toBeUndefined()+ await vi.advanceTimersByTimeAsync(3)+ // The bound is a deadline: the waiter is settled at 55 ms, not at the+ // 60 ms poll, and the deadline does not look at the slot count.+ expect(outcome).toBe('stream queue wait exceeded')+ await waiter+ expect(gate.snapshot()).toEqual({ running: 0, queued: 0 })+ } finally {+ vi.useRealTimers()+ }+ })++ it('refuses a maxQueueWaitMs that setTimeout cannot honor', () => {+ // Witness: the bounds themselves are accepted.+ expect(() => new StreamGate(1, 1, 1)).not.toThrow()+ expect(() => new StreamGate(1, 1, 2_147_483_647)).not.toThrow()+ for (const maxQueueWaitMs of [0, -1, 1.5, Number.NaN, 2_147_483_648]) {+ expect(() => new StreamGate(1, 1, maxQueueWaitMs), String(maxQueueWaitMs)).toThrow(RangeError)+ }+ })+it('pins the visual 2/8 sibling against the ordinary 8/16 stream gate', () => {expect(VISUAL_STREAM_LIMITS).toEqual({ maxConcurrentStreams: 2, maxQueuedRequests: 8 })expect(STREAM_LIMITS.maxConcurrentStreams).toBe(8)diff --git a/grok-llm-proxy/src/requestLimits.ts b/grok-llm-proxy/src/requestLimits.tsindex c6c9d304c..1353fa7d9 100644--- a/grok-llm-proxy/src/requestLimits.ts+++ b/grok-llm-proxy/src/requestLimits.ts@@ -13,6 +13,9 @@ export const STREAM_LIMITS = {upstreamIdleTimeoutMs: 600_000,} as const+// Largest delay setTimeout honors; Node fires anything above it after 1 ms.+const MAX_TIMER_DELAY_MS = 2_147_483_647+export class RequestLimitError extends Error {readonly code = 'provider_unavailable'constructor(message: string) {@@ -58,13 +61,20 @@ export class StreamGate {private readonly maxConcurrent: number = STREAM_LIMITS.maxConcurrentStreams,private readonly maxQueued: number = STREAM_LIMITS.maxQueuedRequests,private readonly maxQueueWaitMs: number = STREAM_LIMITS.maxQueueWaitMs- ) {}+ ) {+ // setTimeout fires after 1 ms for any delay above 2^31 - 1, which would+ // refuse every queued waiter at once instead of after the configured wait.+ if (!Number.isInteger(maxQueueWaitMs) || maxQueueWaitMs <= 0 || maxQueueWaitMs > MAX_TIMER_DELAY_MS) {+ throw new RangeError(`maxQueueWaitMs must be an integer in 1..${MAX_TIMER_DELAY_MS}`)+ }+ }/*** Take a stream slot. When `signal` aborts (client disconnected) while the* caller is still queued, the waiter is rejected and its queue slot freed, so* a dropped client never proceeds to redeem an attempt. A waiter still- * queued after `maxQueueWaitMs` is rejected the same way.+ * queued after `maxQueueWaitMs` is rejected the same way, on its own timer:+ * a slot that frees after the bound and before the next poll does not admit it.*/async acquire(signal?: AbortSignal): Promise<() => void> {if (signal?.aborted) throw new RequestLimitError('stream request was aborted')@@ -72,25 +82,28 @@ export class StreamGate {if (this.queued >= this.maxQueued) throw new RequestLimitError('stream queue is full')this.queued += 1try {- const queuedAt = Date.now()await new Promise<void>((resolve, reject) => {- let timer: ReturnType<typeof setTimeout> | undefined+ let poll: ReturnType<typeof setTimeout> | undefined+ const settle = () => {+ if (poll !== undefined) clearTimeout(poll)+ clearTimeout(deadline)+ signal?.removeEventListener('abort', onAbort)+ }const onAbort = () => {- if (timer !== undefined) clearTimeout(timer)+ settle()reject(new RequestLimitError('stream request was aborted'))}+ const deadline = setTimeout(() => {+ settle()+ reject(new RequestLimitError('stream queue wait exceeded'))+ }, this.maxQueueWaitMs)const wait = () => {if (this.running < this.maxConcurrent) {- signal?.removeEventListener('abort', onAbort)+ settle()resolve()return}- if (Date.now() - queuedAt >= this.maxQueueWaitMs) {- signal?.removeEventListener('abort', onAbort)- reject(new RequestLimitError('stream queue wait exceeded'))- return- }- timer = setTimeout(wait, 10)+ poll = setTimeout(wait, 10)}signal?.addEventListener('abort', onAbort, { once: true })wait()diff --git a/grok-llm-proxy/test/requestLimits.test.ts b/grok-llm-proxy/test/requestLimits.test.tsindex dcee1d5f6..52b8ce4e3 100644--- a/grok-llm-proxy/test/requestLimits.test.ts+++ b/grok-llm-proxy/test/requestLimits.test.ts@@ -1,4 +1,4 @@-import { describe, expect, it } from 'vitest'+import { describe, expect, it, vi } from 'vitest'import { CONTROL_API_REQUEST_TIMEOUT_MS } from '../src/controlApiClient.js'import {assertBoundedDeadline,@@ -114,4 +114,108 @@ describe('StreamGate', () => {const releaseNext = await nextreleaseNext()})++ // The gate polls for a free slot every 10 ms. A bound that is not a multiple+ // of 10 falls between two polls, so these two cases separate a deadline the+ // gate enforces on its own timer from one it only notices on the next poll.+ it('rejects at maxQueueWaitMs even when a slot frees before the next poll', async () => {+ vi.useFakeTimers()+ try {+ const gate = new StreamGate(1, 1, 55)+ const release = await gate.acquire()+ let outcome: string | undefined+ const waiter = gate.acquire().then(+ () => {+ outcome = 'admitted'+ },+ (err: Error) => {+ outcome = err.message+ }+ )+ await vi.advanceTimersByTimeAsync(54)+ // Witness: the waiter is still queued one millisecond before the bound.+ expect(outcome).toBeUndefined()+ await vi.advanceTimersByTimeAsync(1)+ expect(outcome).toBe('stream queue wait exceeded')+ // The poll stopped with the rejection: no timer is left behind.+ expect(vi.getTimerCount()).toBe(0)+ // A slot that frees after the bound and before the next poll (60 ms)+ // must not reach the waiter that was already refused.+ await vi.advanceTimersByTimeAsync(2)+ release()+ await vi.advanceTimersByTimeAsync(10)+ await waiter+ expect(outcome).toBe('stream queue wait exceeded')+ } finally {+ vi.useRealTimers()+ }+ })++ it('settles at maxQueueWaitMs when a slot frees between the last poll and the bound', async () => {+ vi.useFakeTimers()+ try {+ const gate = new StreamGate(1, 1, 55)+ const release = await gate.acquire()+ let outcome: string | undefined+ const waiter = gate.acquire().then(+ () => {+ outcome = 'admitted'+ },+ (err: Error) => {+ outcome = err.message+ }+ )+ // Last poll at 50 ms sees the slot taken; it frees at 52 ms.+ await vi.advanceTimersByTimeAsync(52)+ release()+ // Witness: nothing has settled the waiter yet.+ expect(outcome).toBeUndefined()+ await vi.advanceTimersByTimeAsync(3)+ // The bound is a deadline: the waiter is settled at 55 ms, not at the+ // 60 ms poll, and the deadline does not look at the slot count.+ expect(outcome).toBe('stream queue wait exceeded')+ await waiter+ } finally {+ vi.useRealTimers()+ }+ })++ it('refuses a maxQueueWaitMs that setTimeout cannot honor', () => {+ // Witness: the bounds themselves are accepted.+ expect(() => new StreamGate(1, 1, 1)).not.toThrow()+ expect(() => new StreamGate(1, 1, 2_147_483_647)).not.toThrow()+ for (const maxQueueWaitMs of [0, -1, 1.5, Number.NaN, 2_147_483_648]) {+ expect(() => new StreamGate(1, 1, maxQueueWaitMs), String(maxQueueWaitMs)).toThrow(RangeError)+ }+ })++ it('admits a waiter whose slot frees before maxQueueWaitMs', async () => {+ vi.useFakeTimers()+ try {+ const gate = new StreamGate(1, 1, 55)+ const release = await gate.acquire()+ let outcome: string | undefined+ const waiter = gate.acquire().then(+ next => {+ outcome = 'admitted'+ return next+ },+ (err: Error) => {+ outcome = err.message+ return undefined+ }+ )+ await vi.advanceTimersByTimeAsync(45)+ release()+ await vi.advanceTimersByTimeAsync(5)+ expect(outcome).toBe('admitted')+ // The deadline timer was cleared on admission, before it could fire.+ expect(vi.getTimerCount()).toBe(0)+ await vi.advanceTimersByTimeAsync(100)+ expect(outcome).toBe('admitted')+ ;(await waiter)?.()+ } finally {+ vi.useRealTimers()+ }+ })})