Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
fix(codex-llm-proxy): free the visual slot on every pre-handler exit
Alfredo Lopezf5fe76d5aa41
diff --git a/codex-llm-proxy/src/server.ts b/codex-llm-proxy/src/server.tsindex 8b9e0ba72..c381b4056 100644--- a/codex-llm-proxy/src/server.ts+++ b/codex-llm-proxy/src/server.ts@@ -364,7 +364,20 @@ export function createProxyApps(release = await visualStreamGate.acquire(parseAbort.signal, req.codexAdmissionDeadlineAt)} catch (err) {req.off('aborted', abortParse)+ // The client left while queued: nobody reads a refusal, and a departed+ // client is not gate saturation, so it is not logged as `visual_gate`.+ // Same rule as the ordinary path's `!abort.signal.aborted`.+ if (err instanceof RequestLimitError && err.kind === 'aborted') returnif (err instanceof RequestLimitError) {+ logger.warn(+ {+ event: 'codex_proxy_admission_refused',+ reason: 'visual_gate',+ code: err.code,+ detail: err.message,+ },+ 'admission refused'+ )reject(res, 503, err.code)return}@@ -372,11 +385,31 @@ export function createProxyApps(return}req.off('aborted', abortParse)+ const releaseVisual = (): void => {+ req.codexStreamRelease?.()+ req.codexStreamRelease = undefined+ }req.codexStreamRelease = release+ let expired = false+ const readDeadline = setTimeout(() => {+ expired = true+ releaseVisual()+ if (res.headersSent) return+ // The rest of the body is never read, so the connection cannot be reused.+ res.setHeader('connection', 'close')+ reject(res, 408, 'request_timeout')+ }, bodyReadDeadlineMs)+ res.once('close', () => {+ clearTimeout(readDeadline)+ releaseVisual()+ })visualJson(req, res, err => {+ clearTimeout(readDeadline)+ // The 408 already answered this request; the parser's late error+ // (the body aborted by the closed connection) has no one to reach.+ if (expired) returnif (err) {- req.codexStreamRelease?.()- req.codexStreamRelease = undefined+ releaseVisual()next(err)return}diff --git a/codex-llm-proxy/test/streamGate.handoff.test.ts b/codex-llm-proxy/test/streamGate.handoff.test.tsindex 88d121eb2..690b0da07 100644--- a/codex-llm-proxy/test/streamGate.handoff.test.ts+++ b/codex-llm-proxy/test/streamGate.handoff.test.ts@@ -2,20 +2,24 @@ import { generateKeyPairSync } from 'node:crypto'import { createServer } from 'node:http'import { connect as connectTcp } from 'node:net'import jwt from 'jsonwebtoken'-import { afterEach, describe, expect, it } from 'vitest'+import { afterEach, describe, expect, it, vi } from 'vitest'import {hashCodexCompletionRequest,parseCodexCompletionRequest,} from '@clerum/llm-provider-attempt-contract'import { type CodexLlmProxyConfig } from '../src/config.js'+import { logger } from '../src/logger.js'import { createProxyApps } from '../src/server.js'import {+ RequestLimitError,STREAM_LIMITS,VISUAL_STREAM_LIMITS,streamGate,visualStreamGate,} from '../src/requestLimits.js'+const COMPLETIONS_PATH = '/internal/runtime/v1/codex/completions'+const { privateKey, publicKey } = generateKeyPairSync('rsa', {modulusLength: 2048,publicKeyEncoding: { type: 'spki', format: 'pem' },@@ -52,11 +56,11 @@ function sign(payload: Record<string, unknown>, audience: string): string {})}-function platformToken(): string {+function platformToken(hostRefs: string[] = ['research-host']): string {return sign({sub: 'default/research-host',- hostRefs: ['research-host'],+ hostRefs,workflowControlScopes: ['llm:codex:execute'],scope: 'workflow:approval:request',},@@ -197,12 +201,91 @@ function postChunked(port: number, token: string): Promise<number> {})}+function postBody(port: number, bearerToken: string, body: string): Promise<Response> {+ return fetch(`http://127.0.0.1:${port}${COMPLETIONS_PATH}`, {+ method: 'POST',+ headers: {+ Authorization: `Bearer ${bearerToken}`,+ 'content-type': 'application/json',+ },+ body,+ })+}++// Declares `declared` body bytes, sends `sent` of them and then stalls, so the+// server has to answer before the body completes.+function postStalled(+ port: number,+ bearerToken: string,+ declared: number,+ sent: string+): Promise<{ status: number; head: string }> {+ return new Promise((resolve, reject) => {+ const socket = connectTcp(port, '127.0.0.1', () => {+ socket.write(+ `POST ${COMPLETIONS_PATH} HTTP/1.1\r\n` ++ `Host: 127.0.0.1:${port}\r\n` ++ `Authorization: Bearer ${bearerToken}\r\n` ++ `Content-Type: application/json\r\n` ++ `Content-Length: ${declared}\r\n` ++ `\r\n` ++ sent+ )+ })+ const chunks: Buffer[] = []+ socket.setTimeout(3_000, () => {+ socket.destroy()+ reject(new Error('the server did not answer a stalled visual body'))+ })+ socket.on('data', data => {+ chunks.push(data)+ const raw = Buffer.concat(chunks).toString('utf8')+ const end = raw.indexOf('\r\n\r\n')+ if (end === -1) return+ socket.destroy()+ const match = /^HTTP\/1\.1 (\d+)/.exec(raw)+ if (!match) {+ reject(new Error(`no status in ${raw.slice(0, 180)}`))+ return+ }+ resolve({ status: Number(match[1]), head: raw.slice(0, end) })+ })+ socket.on('error', err => {+ if ((err as NodeJS.ErrnoException).code !== 'ECONNRESET') reject(err)+ })+ })+}++// A large V2 with an unusable ticket: admitted through the visual gate, then+// refused at the ticket check. It can only answer 403 when a visual slot is free.+async function expectNextVisualAdmitted(port: number, maxBodyBytes: number): Promise<void> {+ const body = JSON.stringify({+ executionTicket: 'invalid-ticket',+ requestHash: 'a'.repeat(64),+ request: completionRequest('codex-completion-request.v2', 'x'.repeat(maxBodyBytes)),+ })+ expect(Buffer.byteLength(body)).toBeGreaterThan(maxBodyBytes)+ const res = await postBody(port, platformToken(), body)+ expect(res.status).toBe(403)+ expect(await res.json()).toEqual({ error: 'ticket_invalid' })+}+describe('visual stream-gate handoff', () => {const hangs: Array<{ release: () => void }> = []const serversToClose: Array<{ close: () => Promise<void> }> = []const listeners: Array<ReturnType<typeof createServer>> = []+ function listen(servers: ReturnType<typeof createProxyApps>): number {+ serversToClose.push(servers)+ const listener = createServer(servers.runtimeApp).listen(0)+ listeners.push(listener)+ const address = listener.address()+ if (!address || typeof address === 'string') throw new Error('listener has no port')+ return address.port+ }+afterEach(async () => {+ vi.restoreAllMocks()for (const hang of hangs.splice(0)) hang.release()for (const server of serversToClose.splice(0)) await server.close()await Promise.all(@@ -375,4 +458,317 @@ describe('visual stream-gate handoff', () => {hang.release()await response})++ describe('visual slot lifetime', () => {+ function smallCaps(): number {+ return Buffer.byteLength(completionPayload('codex-completion-request.v1').body) + 512+ }++ async function expectGateEmpty(): Promise<void> {+ await waitFor(+ () =>+ visualStreamGate.snapshot().running === 0 && visualStreamGate.snapshot().queued === 0,+ 'the visual slot was not released'+ )+ expect(visualStreamGate.snapshot()).toEqual({ running: 0, queued: 0 })+ }++ it('answers 400 to a deeply nested visual body and frees the slot', async () => {+ const maxBodyBytes = smallCaps()+ const acquire = vi.spyOn(visualStreamGate, 'acquire')+ const port = listen(createProxyApps(config({ maxBodyBytes, maxVisualBodyBytes: 1024 * 1024 })))+ // JSON.parse accepts this depth; JSON.stringify and the contract's byte+ // measurement throw RangeError on it.+ const depth = 20_000+ const body =+ `{"executionTicket":"invalid-ticket","requestHash":"${'a'.repeat(64)}",` ++ `"request":{"schemaVersion":"codex-completion-request.v2",` ++ `"deep":${'['.repeat(depth)}${']'.repeat(depth)}}}`+ expect(Buffer.byteLength(body)).toBeGreaterThan(maxBodyBytes)++ const res = await postBody(port, platformToken(), body)+ expect(res.status).toBe(400)+ expect(await res.json()).toEqual({ error: 'invalid_request' })+ expect(acquire).toHaveBeenCalledTimes(1)+ await expectGateEmpty()+ await expectNextVisualAdmitted(port, maxBodyBytes)+ })++ it('frees the slot when the handler throws before releasing it', async () => {+ const maxBodyBytes = smallCaps()+ const acquire = vi.spyOn(visualStreamGate, 'acquire')+ const port = listen(createProxyApps(config({ maxBodyBytes, maxVisualBodyBytes: 1024 * 1024 })))+ const body = JSON.stringify({+ executionTicket: 'invalid-ticket',+ requestHash: 'a'.repeat(64),+ request: completionRequest('codex-completion-request.v2', 'x'.repeat(maxBodyBytes)),+ poison: true,+ })+ expect(Buffer.byteLength(body)).toBeGreaterThan(maxBodyBytes)+ // Any throw between the grant and the handler's explicit releases reaches+ // the error handler; only the response's close event can free the slot.+ const stringify = JSON.stringify.bind(JSON)+ const poisoned = vi.spyOn(JSON, 'stringify').mockImplementation(((+ value: unknown,+ ...rest: unknown[]+ ) => {+ if (value !== null && typeof value === 'object' && 'poison' in value) {+ throw new RangeError('Maximum call stack size exceeded')+ }+ return (stringify as (...args: unknown[]) => string)(value, ...rest)+ }) as typeof JSON.stringify)++ const res = await postBody(port, platformToken(), body)+ expect(res.status).toBe(500)+ expect(await res.json()).toEqual({ error: 'internal_error' })+ expect(poisoned).toHaveBeenCalledWith(expect.objectContaining({ poison: true }))+ expect(acquire).toHaveBeenCalledTimes(1)+ poisoned.mockRestore()+ await expectGateEmpty()+ await expectNextVisualAdmitted(port, maxBodyBytes)+ })++ it('answers 408 to a stalled visual body within the read deadline and frees the slot', async () => {+ const maxBodyBytes = smallCaps()+ const acquire = vi.spyOn(visualStreamGate, 'acquire')+ const port = listen(+ createProxyApps(config({ maxBodyBytes, maxVisualBodyBytes: 1024 * 1024 }), {+ bodyReadDeadlineMs: 100,+ })+ )+ const declared = maxBodyBytes + 1024+ const answer = await postStalled(port, platformToken(), declared, '{"executionTicket":')+ expect(answer.status).toBe(408)+ expect(answer.head).toMatch(/^connection: close$/im)+ expect(acquire).toHaveBeenCalledTimes(1)+ await expectGateEmpty()+ await expectNextVisualAdmitted(port, maxBodyBytes)+ })++ type RefusalRow = {+ name: string+ status: number+ error: string+ overrides?: Partial<CodexLlmProxyConfig>+ body: (maxBodyBytes: number) => { auth: string; body: string }+ }++ const largeValid = (maxBodyBytes: number) =>+ completionPayload('codex-completion-request.v2', 'x'.repeat(maxBodyBytes))++ const refusals: RefusalRow[] = [+ {+ name: 'malformed JSON',+ status: 400,+ error: 'invalid_request',+ body: maxBodyBytes => ({+ auth: platformToken(),+ body: `{"pad":"${'x'.repeat(maxBodyBytes)}`,+ }),+ },+ {+ name: 'an unknown envelope field',+ status: 400,+ error: 'unknown_field',+ body: maxBodyBytes => {+ const valid = largeValid(maxBodyBytes)+ return {+ auth: valid.token,+ body: JSON.stringify({ ...JSON.parse(valid.body), extra: 1 }),+ }+ },+ },+ {+ name: 'an invalid ticket',+ status: 403,+ error: 'ticket_invalid',+ body: maxBodyBytes => {+ const valid = largeValid(maxBodyBytes)+ return {+ auth: valid.token,+ body: JSON.stringify({ ...JSON.parse(valid.body), executionTicket: 'invalid-ticket' }),+ }+ },+ },+ {+ name: 'a host binding mismatch',+ status: 403,+ error: 'host_binding_mismatch',+ body: maxBodyBytes => ({+ auth: platformToken(['other-host']),+ body: largeValid(maxBodyBytes).body,+ }),+ },+ {+ name: 'execution disabled',+ status: 404,+ error: 'disabled',+ overrides: { executionEnabled: false },+ body: maxBodyBytes => {+ const valid = largeValid(maxBodyBytes)+ return { auth: valid.token, body: valid.body }+ },+ },+ ]++ it.each(refusals)('frees the visual slot after refusing $name', async row => {+ const maxBodyBytes = smallCaps()+ const acquire = vi.spyOn(visualStreamGate, 'acquire')+ const port = listen(+ createProxyApps(+ config({ maxBodyBytes, maxVisualBodyBytes: 1024 * 1024, ...row.overrides })+ )+ )+ const payload = row.body(maxBodyBytes)+ expect(Buffer.byteLength(payload.body)).toBeGreaterThan(maxBodyBytes)++ const res = await postBody(port, payload.auth, payload.body)+ expect(res.status).toBe(row.status)+ expect(await res.json()).toEqual({ error: row.error })+ expect(acquire).toHaveBeenCalledTimes(1)+ await expectGateEmpty()+ // The ticket check runs before the execution flag, so this also holds+ // for the disabled row.+ await expectNextVisualAdmitted(port, maxBodyBytes)+ })++ // A body this small fits the socket buffer Node keeps reading, so the+ // client's disconnect fires `aborted` while it waits. A larger body is the+ // next test.+ it('frees the queue place of a visual request aborted while it waited', async () => {+ const maxBodyBytes = smallCaps()+ const acquire = vi.spyOn(visualStreamGate, 'acquire')+ const hang = hangStream()+ hangs.push(hang)+ const port = listen(+ createProxyApps(config({ maxBodyBytes, maxVisualBodyBytes: 1024 * 1024 }), {+ streamCompletion: hang.impl,+ })+ )+ // Codex admits two visual streams, so both entries must be held before a+ // third request can queue behind them.+ const holderA = postCompletion(port, 'codex-completion-request.v2', 'x'.repeat(maxBodyBytes))+ const holderB = postCompletion(port, 'codex-completion-request.v2', 'x'.repeat(maxBodyBytes))+ await waitFor(+ () => visualStreamGate.snapshot().running === 2,+ 'no pair of streams held the gate'+ )++ const abort = new AbortController()+ const payload = largeValid(maxBodyBytes)+ const waiting = fetch(`http://127.0.0.1:${port}${COMPLETIONS_PATH}`, {+ method: 'POST',+ headers: { Authorization: `Bearer ${payload.token}`, 'content-type': 'application/json' },+ body: payload.body,+ signal: abort.signal,+ }).catch((err: unknown) => err)+ await waitFor(() => visualStreamGate.snapshot().queued === 1, 'the second request did not queue')+ const warn = vi.spyOn(logger, 'warn')+ abort.abort()+ expect(await waiting).toBeInstanceOf(Error)+ await waitFor(() => visualStreamGate.snapshot().queued === 0, 'the aborted waiter kept its place')+ expect(acquire).toHaveBeenCalledTimes(3)+ // Witness: the waiter's acquire ended on the client's abort, so the+ // catch around it ran. A departed client is not gate saturation and+ // must not be logged as `visual_gate`.+ const waiterAcquire = acquire.mock.results[2]?.value as Promise<unknown>+ const acquireError = await waiterAcquire.catch((err: unknown) => err)+ expect(acquireError).toBeInstanceOf(RequestLimitError)+ expect((acquireError as RequestLimitError).kind).toBe('aborted')+ expect(warn).not.toHaveBeenCalledWith(+ expect.objectContaining({ event: 'codex_proxy_admission_refused', reason: 'visual_gate' }),+ 'admission refused'+ )++ hang.release()+ await holderA+ await holderB+ await expectGateEmpty()+ await expectNextVisualAdmitted(port, maxBodyBytes)+ })++ // Once the unread body exceeds the request's buffer, Node stops reading the+ // socket, so a queued client's disconnect is not seen and `aborted` never+ // fires. The place is held until the grant or the admission deadline. At the+ // grant Node reads the bytes that already reached the server; here that is+ // the whole body, so the dead client's request runs through the handler,+ // which frees the slot. This pins that bound.+ it('holds the queue place of a disconnected large-body waiter until its grant', async () => {+ const maxBodyBytes = smallCaps()+ const acquire = vi.spyOn(visualStreamGate, 'acquire')+ const hang = hangStream()+ hangs.push(hang)+ const port = listen(+ createProxyApps(config({ maxBodyBytes, maxVisualBodyBytes: 4 * 1024 * 1024 }), {+ streamCompletion: hang.impl,+ })+ )+ const holderA = postCompletion(port, 'codex-completion-request.v2', 'x'.repeat(maxBodyBytes))+ const holderB = postCompletion(port, 'codex-completion-request.v2', 'x'.repeat(maxBodyBytes))+ await waitFor(+ () => visualStreamGate.snapshot().running === 2,+ 'no pair of streams held the gate'+ )++ const payload = largeValid(2 * 1024 * 1024)+ const waiter = connectTcp(port, '127.0.0.1', () => {+ waiter.write(+ `POST ${COMPLETIONS_PATH} HTTP/1.1\r\n` ++ `Host: 127.0.0.1:${port}\r\n` ++ `Authorization: Bearer ${payload.token}\r\n` ++ `Content-Type: application/json\r\n` ++ `Content-Length: ${Buffer.byteLength(payload.body)}\r\n` ++ `\r\n`+ )+ waiter.write(payload.body)+ })+ waiter.on('error', err => {+ const code = (err as NodeJS.ErrnoException).code+ if (code !== 'ECONNRESET' && code !== 'EPIPE') throw err+ })+ await waitFor(() => visualStreamGate.snapshot().queued === 1, 'the large waiter did not queue')+ const closed = new Promise<void>(resolve => waiter.once('close', () => resolve()))+ waiter.destroy()+ await closed+ expect(waiter.destroyed).toBe(true)++ await new Promise(resolve => setTimeout(resolve, 300))+ expect(visualStreamGate.snapshot()).toEqual({ running: 2, queued: 1 })+ expect(acquire).toHaveBeenCalledTimes(3)++ hang.release()+ await holderA+ await holderB+ await expectGateEmpty()+ await expectNextVisualAdmitted(port, maxBodyBytes)+ })++ it('refuses a declared length past the visual ceiling with 413 without queueing', async () => {+ const maxBodyBytes = smallCaps()+ const maxVisualBodyBytes = 64 * 1024+ const acquire = vi.spyOn(visualStreamGate, 'acquire')+ const hang = hangStream()+ hangs.push(hang)+ const port = listen(+ createProxyApps(config({ maxBodyBytes, maxVisualBodyBytes }), {+ streamCompletion: hang.impl,+ })+ )+ const holder = postCompletion(port, 'codex-completion-request.v2', 'x'.repeat(maxBodyBytes))+ await waitFor(() => visualStreamGate.snapshot().running === 1, 'no stream held the gate')++ // With the slot held, a body that reached the gate would queue and wait;+ // the declared length alone must answer 413 first.+ const over = largeValid(maxVisualBodyBytes)+ expect(Buffer.byteLength(over.body)).toBeGreaterThan(maxVisualBodyBytes)+ const res = await postBody(port, over.token, over.body)+ expect(res.status).toBe(413)+ expect(await res.json()).toEqual({ error: 'payload_too_large' })+ expect(visualStreamGate.snapshot()).toEqual({ running: 1, queued: 0 })+ expect(acquire).toHaveBeenCalledTimes(1)++ hang.release()+ await holder+ })+ })})