Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript

fix(codex-llm-proxy): free the visual slot on every pre-handler exit

Alfredo Lopezf5fe76d5aa41
diff --git a/codex-llm-proxy/src/server.ts b/codex-llm-proxy/src/server.ts
index 8b9e0ba72..c381b4056 100644
--- a/codex-llm-proxy/src/server.ts
+++ b/codex-llm-proxy/src/server.ts
@@ -364,7 +364,20 @@ export function createProxyApps(
release = await visualStreamGate.acquire(parseAbort.signal, req.codexAdmissionDeadlineAt)
} catch (err) {
req.off('aborted', abortParse)
+ // The client left while queued: nobody reads a refusal, and a departed
+ // client is not gate saturation, so it is not logged as `visual_gate`.
+ // Same rule as the ordinary path's `!abort.signal.aborted`.
+ if (err instanceof RequestLimitError && err.kind === 'aborted') return
if (err instanceof RequestLimitError) {
+ logger.warn(
+ {
+ event: 'codex_proxy_admission_refused',
+ reason: 'visual_gate',
+ code: err.code,
+ detail: err.message,
+ },
+ 'admission refused'
+ )
reject(res, 503, err.code)
return
}
@@ -372,11 +385,31 @@ export function createProxyApps(
return
}
req.off('aborted', abortParse)
+ const releaseVisual = (): void => {
+ req.codexStreamRelease?.()
+ req.codexStreamRelease = undefined
+ }
req.codexStreamRelease = release
+ let expired = false
+ const readDeadline = setTimeout(() => {
+ expired = true
+ releaseVisual()
+ if (res.headersSent) return
+ // The rest of the body is never read, so the connection cannot be reused.
+ res.setHeader('connection', 'close')
+ reject(res, 408, 'request_timeout')
+ }, bodyReadDeadlineMs)
+ res.once('close', () => {
+ clearTimeout(readDeadline)
+ releaseVisual()
+ })
visualJson(req, res, err => {
+ clearTimeout(readDeadline)
+ // The 408 already answered this request; the parser's late error
+ // (the body aborted by the closed connection) has no one to reach.
+ if (expired) return
if (err) {
- req.codexStreamRelease?.()
- req.codexStreamRelease = undefined
+ releaseVisual()
next(err)
return
}
diff --git a/codex-llm-proxy/test/streamGate.handoff.test.ts b/codex-llm-proxy/test/streamGate.handoff.test.ts
index 88d121eb2..690b0da07 100644
--- a/codex-llm-proxy/test/streamGate.handoff.test.ts
+++ b/codex-llm-proxy/test/streamGate.handoff.test.ts
@@ -2,20 +2,24 @@ import { generateKeyPairSync } from 'node:crypto'
import { createServer } from 'node:http'
import { connect as connectTcp } from 'node:net'
import jwt from 'jsonwebtoken'
-import { afterEach, describe, expect, it } from 'vitest'
+import { afterEach, describe, expect, it, vi } from 'vitest'
import {
hashCodexCompletionRequest,
parseCodexCompletionRequest,
} from '@clerum/llm-provider-attempt-contract'
import { type CodexLlmProxyConfig } from '../src/config.js'
+import { logger } from '../src/logger.js'
import { createProxyApps } from '../src/server.js'
import {
+ RequestLimitError,
STREAM_LIMITS,
VISUAL_STREAM_LIMITS,
streamGate,
visualStreamGate,
} from '../src/requestLimits.js'
+const COMPLETIONS_PATH = '/internal/runtime/v1/codex/completions'
+
const { privateKey, publicKey } = generateKeyPairSync('rsa', {
modulusLength: 2048,
publicKeyEncoding: { type: 'spki', format: 'pem' },
@@ -52,11 +56,11 @@ function sign(payload: Record<string, unknown>, audience: string): string {
})
}
-function platformToken(): string {
+function platformToken(hostRefs: string[] = ['research-host']): string {
return sign(
{
sub: 'default/research-host',
- hostRefs: ['research-host'],
+ hostRefs,
workflowControlScopes: ['llm:codex:execute'],
scope: 'workflow:approval:request',
},
@@ -197,12 +201,91 @@ function postChunked(port: number, token: string): Promise<number> {
})
}
+function postBody(port: number, bearerToken: string, body: string): Promise<Response> {
+ return fetch(`http://127.0.0.1:${port}${COMPLETIONS_PATH}`, {
+ method: 'POST',
+ headers: {
+ Authorization: `Bearer ${bearerToken}`,
+ 'content-type': 'application/json',
+ },
+ body,
+ })
+}
+
+// Declares `declared` body bytes, sends `sent` of them and then stalls, so the
+// server has to answer before the body completes.
+function postStalled(
+ port: number,
+ bearerToken: string,
+ declared: number,
+ sent: string
+): Promise<{ status: number; head: string }> {
+ return new Promise((resolve, reject) => {
+ const socket = connectTcp(port, '127.0.0.1', () => {
+ socket.write(
+ `POST ${COMPLETIONS_PATH} HTTP/1.1\r\n` +
+ `Host: 127.0.0.1:${port}\r\n` +
+ `Authorization: Bearer ${bearerToken}\r\n` +
+ `Content-Type: application/json\r\n` +
+ `Content-Length: ${declared}\r\n` +
+ `\r\n` +
+ sent
+ )
+ })
+ const chunks: Buffer[] = []
+ socket.setTimeout(3_000, () => {
+ socket.destroy()
+ reject(new Error('the server did not answer a stalled visual body'))
+ })
+ socket.on('data', data => {
+ chunks.push(data)
+ const raw = Buffer.concat(chunks).toString('utf8')
+ const end = raw.indexOf('\r\n\r\n')
+ if (end === -1) return
+ socket.destroy()
+ const match = /^HTTP\/1\.1 (\d+)/.exec(raw)
+ if (!match) {
+ reject(new Error(`no status in ${raw.slice(0, 180)}`))
+ return
+ }
+ resolve({ status: Number(match[1]), head: raw.slice(0, end) })
+ })
+ socket.on('error', err => {
+ if ((err as NodeJS.ErrnoException).code !== 'ECONNRESET') reject(err)
+ })
+ })
+}
+
+// A large V2 with an unusable ticket: admitted through the visual gate, then
+// refused at the ticket check. It can only answer 403 when a visual slot is free.
+async function expectNextVisualAdmitted(port: number, maxBodyBytes: number): Promise<void> {
+ const body = JSON.stringify({
+ executionTicket: 'invalid-ticket',
+ requestHash: 'a'.repeat(64),
+ request: completionRequest('codex-completion-request.v2', 'x'.repeat(maxBodyBytes)),
+ })
+ expect(Buffer.byteLength(body)).toBeGreaterThan(maxBodyBytes)
+ const res = await postBody(port, platformToken(), body)
+ expect(res.status).toBe(403)
+ expect(await res.json()).toEqual({ error: 'ticket_invalid' })
+}
+
describe('visual stream-gate handoff', () => {
const hangs: Array<{ release: () => void }> = []
const serversToClose: Array<{ close: () => Promise<void> }> = []
const listeners: Array<ReturnType<typeof createServer>> = []
+ function listen(servers: ReturnType<typeof createProxyApps>): number {
+ serversToClose.push(servers)
+ const listener = createServer(servers.runtimeApp).listen(0)
+ listeners.push(listener)
+ const address = listener.address()
+ if (!address || typeof address === 'string') throw new Error('listener has no port')
+ return address.port
+ }
+
afterEach(async () => {
+ vi.restoreAllMocks()
for (const hang of hangs.splice(0)) hang.release()
for (const server of serversToClose.splice(0)) await server.close()
await Promise.all(
@@ -375,4 +458,317 @@ describe('visual stream-gate handoff', () => {
hang.release()
await response
})
+
+ describe('visual slot lifetime', () => {
+ function smallCaps(): number {
+ return Buffer.byteLength(completionPayload('codex-completion-request.v1').body) + 512
+ }
+
+ async function expectGateEmpty(): Promise<void> {
+ await waitFor(
+ () =>
+ visualStreamGate.snapshot().running === 0 && visualStreamGate.snapshot().queued === 0,
+ 'the visual slot was not released'
+ )
+ expect(visualStreamGate.snapshot()).toEqual({ running: 0, queued: 0 })
+ }
+
+ it('answers 400 to a deeply nested visual body and frees the slot', async () => {
+ const maxBodyBytes = smallCaps()
+ const acquire = vi.spyOn(visualStreamGate, 'acquire')
+ const port = listen(createProxyApps(config({ maxBodyBytes, maxVisualBodyBytes: 1024 * 1024 })))
+ // JSON.parse accepts this depth; JSON.stringify and the contract's byte
+ // measurement throw RangeError on it.
+ const depth = 20_000
+ const body =
+ `{"executionTicket":"invalid-ticket","requestHash":"${'a'.repeat(64)}",` +
+ `"request":{"schemaVersion":"codex-completion-request.v2",` +
+ `"deep":${'['.repeat(depth)}${']'.repeat(depth)}}}`
+ expect(Buffer.byteLength(body)).toBeGreaterThan(maxBodyBytes)
+
+ const res = await postBody(port, platformToken(), body)
+ expect(res.status).toBe(400)
+ expect(await res.json()).toEqual({ error: 'invalid_request' })
+ expect(acquire).toHaveBeenCalledTimes(1)
+ await expectGateEmpty()
+ await expectNextVisualAdmitted(port, maxBodyBytes)
+ })
+
+ it('frees the slot when the handler throws before releasing it', async () => {
+ const maxBodyBytes = smallCaps()
+ const acquire = vi.spyOn(visualStreamGate, 'acquire')
+ const port = listen(createProxyApps(config({ maxBodyBytes, maxVisualBodyBytes: 1024 * 1024 })))
+ const body = JSON.stringify({
+ executionTicket: 'invalid-ticket',
+ requestHash: 'a'.repeat(64),
+ request: completionRequest('codex-completion-request.v2', 'x'.repeat(maxBodyBytes)),
+ poison: true,
+ })
+ expect(Buffer.byteLength(body)).toBeGreaterThan(maxBodyBytes)
+ // Any throw between the grant and the handler's explicit releases reaches
+ // the error handler; only the response's close event can free the slot.
+ const stringify = JSON.stringify.bind(JSON)
+ const poisoned = vi.spyOn(JSON, 'stringify').mockImplementation(((
+ value: unknown,
+ ...rest: unknown[]
+ ) => {
+ if (value !== null && typeof value === 'object' && 'poison' in value) {
+ throw new RangeError('Maximum call stack size exceeded')
+ }
+ return (stringify as (...args: unknown[]) => string)(value, ...rest)
+ }) as typeof JSON.stringify)
+
+ const res = await postBody(port, platformToken(), body)
+ expect(res.status).toBe(500)
+ expect(await res.json()).toEqual({ error: 'internal_error' })
+ expect(poisoned).toHaveBeenCalledWith(expect.objectContaining({ poison: true }))
+ expect(acquire).toHaveBeenCalledTimes(1)
+ poisoned.mockRestore()
+ await expectGateEmpty()
+ await expectNextVisualAdmitted(port, maxBodyBytes)
+ })
+
+ it('answers 408 to a stalled visual body within the read deadline and frees the slot', async () => {
+ const maxBodyBytes = smallCaps()
+ const acquire = vi.spyOn(visualStreamGate, 'acquire')
+ const port = listen(
+ createProxyApps(config({ maxBodyBytes, maxVisualBodyBytes: 1024 * 1024 }), {
+ bodyReadDeadlineMs: 100,
+ })
+ )
+ const declared = maxBodyBytes + 1024
+ const answer = await postStalled(port, platformToken(), declared, '{"executionTicket":')
+ expect(answer.status).toBe(408)
+ expect(answer.head).toMatch(/^connection: close$/im)
+ expect(acquire).toHaveBeenCalledTimes(1)
+ await expectGateEmpty()
+ await expectNextVisualAdmitted(port, maxBodyBytes)
+ })
+
+ type RefusalRow = {
+ name: string
+ status: number
+ error: string
+ overrides?: Partial<CodexLlmProxyConfig>
+ body: (maxBodyBytes: number) => { auth: string; body: string }
+ }
+
+ const largeValid = (maxBodyBytes: number) =>
+ completionPayload('codex-completion-request.v2', 'x'.repeat(maxBodyBytes))
+
+ const refusals: RefusalRow[] = [
+ {
+ name: 'malformed JSON',
+ status: 400,
+ error: 'invalid_request',
+ body: maxBodyBytes => ({
+ auth: platformToken(),
+ body: `{"pad":"${'x'.repeat(maxBodyBytes)}`,
+ }),
+ },
+ {
+ name: 'an unknown envelope field',
+ status: 400,
+ error: 'unknown_field',
+ body: maxBodyBytes => {
+ const valid = largeValid(maxBodyBytes)
+ return {
+ auth: valid.token,
+ body: JSON.stringify({ ...JSON.parse(valid.body), extra: 1 }),
+ }
+ },
+ },
+ {
+ name: 'an invalid ticket',
+ status: 403,
+ error: 'ticket_invalid',
+ body: maxBodyBytes => {
+ const valid = largeValid(maxBodyBytes)
+ return {
+ auth: valid.token,
+ body: JSON.stringify({ ...JSON.parse(valid.body), executionTicket: 'invalid-ticket' }),
+ }
+ },
+ },
+ {
+ name: 'a host binding mismatch',
+ status: 403,
+ error: 'host_binding_mismatch',
+ body: maxBodyBytes => ({
+ auth: platformToken(['other-host']),
+ body: largeValid(maxBodyBytes).body,
+ }),
+ },
+ {
+ name: 'execution disabled',
+ status: 404,
+ error: 'disabled',
+ overrides: { executionEnabled: false },
+ body: maxBodyBytes => {
+ const valid = largeValid(maxBodyBytes)
+ return { auth: valid.token, body: valid.body }
+ },
+ },
+ ]
+
+ it.each(refusals)('frees the visual slot after refusing $name', async row => {
+ const maxBodyBytes = smallCaps()
+ const acquire = vi.spyOn(visualStreamGate, 'acquire')
+ const port = listen(
+ createProxyApps(
+ config({ maxBodyBytes, maxVisualBodyBytes: 1024 * 1024, ...row.overrides })
+ )
+ )
+ const payload = row.body(maxBodyBytes)
+ expect(Buffer.byteLength(payload.body)).toBeGreaterThan(maxBodyBytes)
+
+ const res = await postBody(port, payload.auth, payload.body)
+ expect(res.status).toBe(row.status)
+ expect(await res.json()).toEqual({ error: row.error })
+ expect(acquire).toHaveBeenCalledTimes(1)
+ await expectGateEmpty()
+ // The ticket check runs before the execution flag, so this also holds
+ // for the disabled row.
+ await expectNextVisualAdmitted(port, maxBodyBytes)
+ })
+
+ // A body this small fits the socket buffer Node keeps reading, so the
+ // client's disconnect fires `aborted` while it waits. A larger body is the
+ // next test.
+ it('frees the queue place of a visual request aborted while it waited', async () => {
+ const maxBodyBytes = smallCaps()
+ const acquire = vi.spyOn(visualStreamGate, 'acquire')
+ const hang = hangStream()
+ hangs.push(hang)
+ const port = listen(
+ createProxyApps(config({ maxBodyBytes, maxVisualBodyBytes: 1024 * 1024 }), {
+ streamCompletion: hang.impl,
+ })
+ )
+ // Codex admits two visual streams, so both entries must be held before a
+ // third request can queue behind them.
+ const holderA = postCompletion(port, 'codex-completion-request.v2', 'x'.repeat(maxBodyBytes))
+ const holderB = postCompletion(port, 'codex-completion-request.v2', 'x'.repeat(maxBodyBytes))
+ await waitFor(
+ () => visualStreamGate.snapshot().running === 2,
+ 'no pair of streams held the gate'
+ )
+
+ const abort = new AbortController()
+ const payload = largeValid(maxBodyBytes)
+ const waiting = fetch(`http://127.0.0.1:${port}${COMPLETIONS_PATH}`, {
+ method: 'POST',
+ headers: { Authorization: `Bearer ${payload.token}`, 'content-type': 'application/json' },
+ body: payload.body,
+ signal: abort.signal,
+ }).catch((err: unknown) => err)
+ await waitFor(() => visualStreamGate.snapshot().queued === 1, 'the second request did not queue')
+ const warn = vi.spyOn(logger, 'warn')
+ abort.abort()
+ expect(await waiting).toBeInstanceOf(Error)
+ await waitFor(() => visualStreamGate.snapshot().queued === 0, 'the aborted waiter kept its place')
+ expect(acquire).toHaveBeenCalledTimes(3)
+ // Witness: the waiter's acquire ended on the client's abort, so the
+ // catch around it ran. A departed client is not gate saturation and
+ // must not be logged as `visual_gate`.
+ const waiterAcquire = acquire.mock.results[2]?.value as Promise<unknown>
+ const acquireError = await waiterAcquire.catch((err: unknown) => err)
+ expect(acquireError).toBeInstanceOf(RequestLimitError)
+ expect((acquireError as RequestLimitError).kind).toBe('aborted')
+ expect(warn).not.toHaveBeenCalledWith(
+ expect.objectContaining({ event: 'codex_proxy_admission_refused', reason: 'visual_gate' }),
+ 'admission refused'
+ )
+
+ hang.release()
+ await holderA
+ await holderB
+ await expectGateEmpty()
+ await expectNextVisualAdmitted(port, maxBodyBytes)
+ })
+
+ // Once the unread body exceeds the request's buffer, Node stops reading the
+ // socket, so a queued client's disconnect is not seen and `aborted` never
+ // fires. The place is held until the grant or the admission deadline. At the
+ // grant Node reads the bytes that already reached the server; here that is
+ // the whole body, so the dead client's request runs through the handler,
+ // which frees the slot. This pins that bound.
+ it('holds the queue place of a disconnected large-body waiter until its grant', async () => {
+ const maxBodyBytes = smallCaps()
+ const acquire = vi.spyOn(visualStreamGate, 'acquire')
+ const hang = hangStream()
+ hangs.push(hang)
+ const port = listen(
+ createProxyApps(config({ maxBodyBytes, maxVisualBodyBytes: 4 * 1024 * 1024 }), {
+ streamCompletion: hang.impl,
+ })
+ )
+ const holderA = postCompletion(port, 'codex-completion-request.v2', 'x'.repeat(maxBodyBytes))
+ const holderB = postCompletion(port, 'codex-completion-request.v2', 'x'.repeat(maxBodyBytes))
+ await waitFor(
+ () => visualStreamGate.snapshot().running === 2,
+ 'no pair of streams held the gate'
+ )
+
+ const payload = largeValid(2 * 1024 * 1024)
+ const waiter = connectTcp(port, '127.0.0.1', () => {
+ waiter.write(
+ `POST ${COMPLETIONS_PATH} HTTP/1.1\r\n` +
+ `Host: 127.0.0.1:${port}\r\n` +
+ `Authorization: Bearer ${payload.token}\r\n` +
+ `Content-Type: application/json\r\n` +
+ `Content-Length: ${Buffer.byteLength(payload.body)}\r\n` +
+ `\r\n`
+ )
+ waiter.write(payload.body)
+ })
+ waiter.on('error', err => {
+ const code = (err as NodeJS.ErrnoException).code
+ if (code !== 'ECONNRESET' && code !== 'EPIPE') throw err
+ })
+ await waitFor(() => visualStreamGate.snapshot().queued === 1, 'the large waiter did not queue')
+ const closed = new Promise<void>(resolve => waiter.once('close', () => resolve()))
+ waiter.destroy()
+ await closed
+ expect(waiter.destroyed).toBe(true)
+
+ await new Promise(resolve => setTimeout(resolve, 300))
+ expect(visualStreamGate.snapshot()).toEqual({ running: 2, queued: 1 })
+ expect(acquire).toHaveBeenCalledTimes(3)
+
+ hang.release()
+ await holderA
+ await holderB
+ await expectGateEmpty()
+ await expectNextVisualAdmitted(port, maxBodyBytes)
+ })
+
+ it('refuses a declared length past the visual ceiling with 413 without queueing', async () => {
+ const maxBodyBytes = smallCaps()
+ const maxVisualBodyBytes = 64 * 1024
+ const acquire = vi.spyOn(visualStreamGate, 'acquire')
+ const hang = hangStream()
+ hangs.push(hang)
+ const port = listen(
+ createProxyApps(config({ maxBodyBytes, maxVisualBodyBytes }), {
+ streamCompletion: hang.impl,
+ })
+ )
+ const holder = postCompletion(port, 'codex-completion-request.v2', 'x'.repeat(maxBodyBytes))
+ await waitFor(() => visualStreamGate.snapshot().running === 1, 'no stream held the gate')
+
+ // With the slot held, a body that reached the gate would queue and wait;
+ // the declared length alone must answer 413 first.
+ const over = largeValid(maxVisualBodyBytes)
+ expect(Buffer.byteLength(over.body)).toBeGreaterThan(maxVisualBodyBytes)
+ const res = await postBody(port, over.token, over.body)
+ expect(res.status).toBe(413)
+ expect(await res.json()).toEqual({ error: 'payload_too_large' })
+ expect(visualStreamGate.snapshot()).toEqual({ running: 1, queued: 0 })
+ expect(acquire).toHaveBeenCalledTimes(1)
+
+ hang.release()
+ await holder
+ })
+ })
})