Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript

fix(llm-proxy): keep the visual slot until hand-off unwinds on client close (r11-M1)

claude[bot]a8f5cf5155d8
diff --git a/codex-llm-proxy/src/server.ts b/codex-llm-proxy/src/server.ts
index 09d1b2262..5232c4563 100644
--- a/codex-llm-proxy/src/server.ts
+++ b/codex-llm-proxy/src/server.ts
@@ -441,7 +441,16 @@ export function createProxyApps(
}, bodyReadDeadlineMs)
res.once('close', () => {
clearTimeout(readDeadline)
- releaseVisual()
+ // Backstop only while the slot is still held here. Once the handler
+ // takes ownership it clears `req.codexStreamRelease` (hand-off) or the
+ // demotion path releases it through `releaseAdmission`; in both cases
+ // the handler's own `finally` frees the slot when it finishes
+ // unwinding. Releasing here after hand-off would free the visual slot
+ // and the principal share while the handler still holds the parsed
+ // body during its (up to ~30 s) unwind, admitting another large body
+ // over the gate width. Guarding on the field preserves the
+ // `d5826f385` demotion fix (that path never runs this hook).
+ if (req.codexStreamRelease) releaseVisual()
})
visualJson(req, res, err => {
clearTimeout(readDeadline)
diff --git a/codex-llm-proxy/test/streamGate.handoff.test.ts b/codex-llm-proxy/test/streamGate.handoff.test.ts
index 1fbfb438a..00b0566d7 100644
--- a/codex-llm-proxy/test/streamGate.handoff.test.ts
+++ b/codex-llm-proxy/test/streamGate.handoff.test.ts
@@ -890,5 +890,68 @@ describe('visual stream-gate handoff', () => {
hang.release()
await holder
})
+
+ // M1 regression: after the handler takes a visual slot for the stream
+ // (hand-off), a client disconnect must not free the slot or the principal
+ // share early. The `res.close` backstop only runs while the slot is still
+ // held in selectTransportBudget; once the handler owns it the slot is freed
+ // by the handler's own `finally` when it finishes unwinding. Freeing it on
+ // close after hand-off would admit another large body over the gate width
+ // while the disconnected request's parsed body is still resident.
+ it('keeps the visual slot held when the streaming client disconnects after hand-off', async () => {
+ expect(VISUAL_STREAM_LIMITS.maxConcurrentStreams).toBe(2)
+ const maxBodyBytes = smallCaps()
+ const acquire = vi.spyOn(visualStreamGate, 'acquire')
+ const hang = hangStream()
+ hangs.push(hang)
+ const port = listen(
+ createProxyApps(config({ maxBodyBytes, maxVisualBodyBytes: 1024 * 1024 }), {
+ streamCompletion: hang.impl,
+ })
+ )
+ // Fill both visual slots: A hands off under an abortable client, B under
+ // a normal one. Both use the default host so they clear host-binding and
+ // reach the stream. Two admissions for one principal stay under the
+ // per-host share of four.
+ const payloadA = largeValid(maxBodyBytes)
+ const abort = new AbortController()
+ const holderA = fetch(`http://127.0.0.1:${port}${COMPLETIONS_PATH}`, {
+ method: 'POST',
+ headers: {
+ Authorization: `Bearer ${payloadA.token}`,
+ 'content-type': 'application/json',
+ },
+ body: payloadA.body,
+ signal: abort.signal,
+ }).catch((err: unknown) => err)
+ const holderB = postCompletion(port, 'codex-completion-request.v2', 'x'.repeat(maxBodyBytes))
+ await waitFor(
+ () => visualStreamGate.snapshot().running === 2,
+ 'the two large V2 bodies did not hold both visual slots'
+ )
+ // A third large V2 queues behind the two slots, so the queue depth is a
+ // live witness of whether a slot stays held. It is the third admission for
+ // the principal, still under the per-host share of four.
+ const waiter = postCompletion(port, 'codex-completion-request.v2', 'x'.repeat(maxBodyBytes))
+ await waitFor(
+ () => visualStreamGate.snapshot().queued === 1,
+ 'the third large V2 did not queue behind the held slots'
+ )
+
+ // Holder A's client disconnects while its handler is still unwinding (the
+ // hang holds it). Its slot and the queue place must both persist.
+ abort.abort()
+ expect(await holderA).toBeInstanceOf(Error)
+ // Give the close backstop time to (wrongly) fire before the assertion.
+ await new Promise(resolve => setTimeout(resolve, 300))
+ expect(visualStreamGate.snapshot()).toEqual({ running: 2, queued: 1 })
+ expect(acquire).toHaveBeenCalledTimes(3)
+
+ // Releasing the handlers frees the slots; only then is the waiter admitted.
+ hang.release()
+ await Promise.all([holderB, waiter])
+ await expectGateEmpty()
+ await expectNextVisualAdmitted(port, maxBodyBytes)
+ })
})
})
diff --git a/grok-llm-proxy/src/server.ts b/grok-llm-proxy/src/server.ts
index 31e67b494..68ba8a417 100644
--- a/grok-llm-proxy/src/server.ts
+++ b/grok-llm-proxy/src/server.ts
@@ -477,7 +477,16 @@ export function createProxyApps(
}, bodyReadDeadlineMs)
res.once('close', () => {
clearTimeout(readDeadline)
- releaseVisual()
+ // Backstop only while the slot is still held here. Once the handler
+ // takes ownership it clears `req.grokStreamRelease` (hand-off) or the
+ // demotion path releases it through `releaseAdmission`; in both cases
+ // the handler's own `finally` frees the slot when it finishes
+ // unwinding. Releasing here after hand-off would free the visual slot
+ // and the principal share while the handler still holds the parsed
+ // body during its (up to ~30 s) unwind, admitting another large body
+ // over the gate width. Guarding on the field preserves the
+ // `d5826f385` demotion fix (that path never runs this hook).
+ if (req.grokStreamRelease) releaseVisual()
})
visualJson(req, res, err => {
clearTimeout(readDeadline)
diff --git a/grok-llm-proxy/test/streamGate.handoff.test.ts b/grok-llm-proxy/test/streamGate.handoff.test.ts
index 002bc4200..9b5e33968 100644
--- a/grok-llm-proxy/test/streamGate.handoff.test.ts
+++ b/grok-llm-proxy/test/streamGate.handoff.test.ts
@@ -862,5 +862,66 @@ describe('grok visual stream-gate handoff', () => {
hang.release()
await holder
})
+
+ // M1 regression: after the handler takes the visual slot for the stream
+ // (hand-off), a client disconnect must not free the slot or the principal
+ // share early. The `res.close` backstop only runs while the slot is still
+ // held in selectTransportBudget; once the handler owns it the slot is freed
+ // by the handler's own `finally` when it finishes unwinding. Freeing it on
+ // close after hand-off would admit another large body over the gate width
+ // while the disconnected request's parsed body is still resident.
+ it('keeps the visual slot held when the streaming client disconnects after hand-off', async () => {
+ const maxBodyBytes = smallCaps()
+ const acquire = vi.spyOn(visualStreamGate, 'acquire')
+ const hang = hangStream()
+ hangs.push(hang)
+ const port = listen(
+ createProxyApps(config({ maxBodyBytes, maxVisualBodyBytes: 1024 * 1024 }), {
+ streamCompletion: hang.impl,
+ })
+ )
+ // The first large V2 hands off to the stream and holds the single slot.
+ const abort = new AbortController()
+ const holder = fetch(`http://127.0.0.1:${port}${COMPLETIONS_PATH}`, {
+ method: 'POST',
+ headers: {
+ Authorization: `Bearer ${platformToken()}`,
+ 'content-type': 'application/json',
+ },
+ body: largeValid(maxBodyBytes).body,
+ signal: abort.signal,
+ }).catch((err: unknown) => err)
+ await waitFor(
+ () => visualStreamGate.snapshot().running === 1,
+ 'the large V2 did not hold the visual gate'
+ )
+ // A second large V2 from a distinct principal queues behind the one slot,
+ // so the queue depth is a live witness of whether the slot stays held.
+ const waiter = postCompletion(
+ port,
+ 'grok-completion-request.v2',
+ 'x'.repeat(maxBodyBytes),
+ ['visual-queue-m1']
+ )
+ await waitFor(
+ () => visualStreamGate.snapshot().queued === 1,
+ 'the second large V2 did not queue behind the held slot'
+ )
+
+ // The streaming client disconnects while the handler is still unwinding
+ // (the hang holds it). The slot and the queue place must both persist.
+ abort.abort()
+ expect(await holder).toBeInstanceOf(Error)
+ // Give the close backstop time to (wrongly) fire before the assertion.
+ await new Promise(resolve => setTimeout(resolve, 300))
+ expect(visualStreamGate.snapshot()).toEqual({ running: 1, queued: 1 })
+ expect(acquire).toHaveBeenCalledTimes(2)
+
+ // Releasing the handler frees the slot; only then is the waiter admitted.
+ hang.release()
+ await waiter
+ await expectGateEmpty()
+ await expectNextVisualAdmitted(port, maxBodyBytes)
+ })
})
})