Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
fix(llm-proxy): keep the visual slot until hand-off unwinds on client close (r11-M1)
claude[bot]a8f5cf5155d8
diff --git a/codex-llm-proxy/src/server.ts b/codex-llm-proxy/src/server.tsindex 09d1b2262..5232c4563 100644--- a/codex-llm-proxy/src/server.ts+++ b/codex-llm-proxy/src/server.ts@@ -441,7 +441,16 @@ export function createProxyApps(}, bodyReadDeadlineMs)res.once('close', () => {clearTimeout(readDeadline)- releaseVisual()+ // Backstop only while the slot is still held here. Once the handler+ // takes ownership it clears `req.codexStreamRelease` (hand-off) or the+ // demotion path releases it through `releaseAdmission`; in both cases+ // the handler's own `finally` frees the slot when it finishes+ // unwinding. Releasing here after hand-off would free the visual slot+ // and the principal share while the handler still holds the parsed+ // body during its (up to ~30 s) unwind, admitting another large body+ // over the gate width. Guarding on the field preserves the+ // `d5826f385` demotion fix (that path never runs this hook).+ if (req.codexStreamRelease) releaseVisual()})visualJson(req, res, err => {clearTimeout(readDeadline)diff --git a/codex-llm-proxy/test/streamGate.handoff.test.ts b/codex-llm-proxy/test/streamGate.handoff.test.tsindex 1fbfb438a..00b0566d7 100644--- a/codex-llm-proxy/test/streamGate.handoff.test.ts+++ b/codex-llm-proxy/test/streamGate.handoff.test.ts@@ -890,5 +890,68 @@ describe('visual stream-gate handoff', () => {hang.release()await holder})++ // M1 regression: after the handler takes a visual slot for the stream+ // (hand-off), a client disconnect must not free the slot or the principal+ // share early. The `res.close` backstop only runs while the slot is still+ // held in selectTransportBudget; once the handler owns it the slot is freed+ // by the handler's own `finally` when it finishes unwinding. Freeing it on+ // close after hand-off would admit another large body over the gate width+ // while the disconnected request's parsed body is still resident.+ it('keeps the visual slot held when the streaming client disconnects after hand-off', async () => {+ expect(VISUAL_STREAM_LIMITS.maxConcurrentStreams).toBe(2)+ const maxBodyBytes = smallCaps()+ const acquire = vi.spyOn(visualStreamGate, 'acquire')+ const hang = hangStream()+ hangs.push(hang)+ const port = listen(+ createProxyApps(config({ maxBodyBytes, maxVisualBodyBytes: 1024 * 1024 }), {+ streamCompletion: hang.impl,+ })+ )+ // Fill both visual slots: A hands off under an abortable client, B under+ // a normal one. Both use the default host so they clear host-binding and+ // reach the stream. Two admissions for one principal stay under the+ // per-host share of four.+ const payloadA = largeValid(maxBodyBytes)+ const abort = new AbortController()+ const holderA = fetch(`http://127.0.0.1:${port}${COMPLETIONS_PATH}`, {+ method: 'POST',+ headers: {+ Authorization: `Bearer ${payloadA.token}`,+ 'content-type': 'application/json',+ },+ body: payloadA.body,+ signal: abort.signal,+ }).catch((err: unknown) => err)+ const holderB = postCompletion(port, 'codex-completion-request.v2', 'x'.repeat(maxBodyBytes))+ await waitFor(+ () => visualStreamGate.snapshot().running === 2,+ 'the two large V2 bodies did not hold both visual slots'+ )+ // A third large V2 queues behind the two slots, so the queue depth is a+ // live witness of whether a slot stays held. It is the third admission for+ // the principal, still under the per-host share of four.+ const waiter = postCompletion(port, 'codex-completion-request.v2', 'x'.repeat(maxBodyBytes))+ await waitFor(+ () => visualStreamGate.snapshot().queued === 1,+ 'the third large V2 did not queue behind the held slots'+ )++ // Holder A's client disconnects while its handler is still unwinding (the+ // hang holds it). Its slot and the queue place must both persist.+ abort.abort()+ expect(await holderA).toBeInstanceOf(Error)+ // Give the close backstop time to (wrongly) fire before the assertion.+ await new Promise(resolve => setTimeout(resolve, 300))+ expect(visualStreamGate.snapshot()).toEqual({ running: 2, queued: 1 })+ expect(acquire).toHaveBeenCalledTimes(3)++ // Releasing the handlers frees the slots; only then is the waiter admitted.+ hang.release()+ await Promise.all([holderB, waiter])+ await expectGateEmpty()+ await expectNextVisualAdmitted(port, maxBodyBytes)+ })})})diff --git a/grok-llm-proxy/src/server.ts b/grok-llm-proxy/src/server.tsindex 31e67b494..68ba8a417 100644--- a/grok-llm-proxy/src/server.ts+++ b/grok-llm-proxy/src/server.ts@@ -477,7 +477,16 @@ export function createProxyApps(}, bodyReadDeadlineMs)res.once('close', () => {clearTimeout(readDeadline)- releaseVisual()+ // Backstop only while the slot is still held here. Once the handler+ // takes ownership it clears `req.grokStreamRelease` (hand-off) or the+ // demotion path releases it through `releaseAdmission`; in both cases+ // the handler's own `finally` frees the slot when it finishes+ // unwinding. Releasing here after hand-off would free the visual slot+ // and the principal share while the handler still holds the parsed+ // body during its (up to ~30 s) unwind, admitting another large body+ // over the gate width. Guarding on the field preserves the+ // `d5826f385` demotion fix (that path never runs this hook).+ if (req.grokStreamRelease) releaseVisual()})visualJson(req, res, err => {clearTimeout(readDeadline)diff --git a/grok-llm-proxy/test/streamGate.handoff.test.ts b/grok-llm-proxy/test/streamGate.handoff.test.tsindex 002bc4200..9b5e33968 100644--- a/grok-llm-proxy/test/streamGate.handoff.test.ts+++ b/grok-llm-proxy/test/streamGate.handoff.test.ts@@ -862,5 +862,66 @@ describe('grok visual stream-gate handoff', () => {hang.release()await holder})++ // M1 regression: after the handler takes the visual slot for the stream+ // (hand-off), a client disconnect must not free the slot or the principal+ // share early. The `res.close` backstop only runs while the slot is still+ // held in selectTransportBudget; once the handler owns it the slot is freed+ // by the handler's own `finally` when it finishes unwinding. Freeing it on+ // close after hand-off would admit another large body over the gate width+ // while the disconnected request's parsed body is still resident.+ it('keeps the visual slot held when the streaming client disconnects after hand-off', async () => {+ const maxBodyBytes = smallCaps()+ const acquire = vi.spyOn(visualStreamGate, 'acquire')+ const hang = hangStream()+ hangs.push(hang)+ const port = listen(+ createProxyApps(config({ maxBodyBytes, maxVisualBodyBytes: 1024 * 1024 }), {+ streamCompletion: hang.impl,+ })+ )+ // The first large V2 hands off to the stream and holds the single slot.+ const abort = new AbortController()+ const holder = fetch(`http://127.0.0.1:${port}${COMPLETIONS_PATH}`, {+ method: 'POST',+ headers: {+ Authorization: `Bearer ${platformToken()}`,+ 'content-type': 'application/json',+ },+ body: largeValid(maxBodyBytes).body,+ signal: abort.signal,+ }).catch((err: unknown) => err)+ await waitFor(+ () => visualStreamGate.snapshot().running === 1,+ 'the large V2 did not hold the visual gate'+ )+ // A second large V2 from a distinct principal queues behind the one slot,+ // so the queue depth is a live witness of whether the slot stays held.+ const waiter = postCompletion(+ port,+ 'grok-completion-request.v2',+ 'x'.repeat(maxBodyBytes),+ ['visual-queue-m1']+ )+ await waitFor(+ () => visualStreamGate.snapshot().queued === 1,+ 'the second large V2 did not queue behind the held slot'+ )++ // The streaming client disconnects while the handler is still unwinding+ // (the hang holds it). The slot and the queue place must both persist.+ abort.abort()+ expect(await holder).toBeInstanceOf(Error)+ // Give the close backstop time to (wrongly) fire before the assertion.+ await new Promise(resolve => setTimeout(resolve, 300))+ expect(visualStreamGate.snapshot()).toEqual({ running: 1, queued: 1 })+ expect(acquire).toHaveBeenCalledTimes(2)++ // Releasing the handler frees the slot; only then is the waiter admitted.+ hang.release()+ await waiter+ await expectGateEmpty()+ await expectNextVisualAdmitted(port, maxBodyBytes)+ })})})