Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
import express, {
type Express,
type NextFunction,
type Request,
type RequestHandler,
type Response,
} from 'express'
import { rateLimit } from 'express-rate-limit'
import { type Server, createServer } from 'node:http'
import { Registry, collectDefaultMetrics } from 'prom-client'
import { z } from 'zod'
import {
BODY_STRUCTURE_LIMITS,
LIMITS,
SCHEMA_VERSION_V2,
createBodyStructureVerify,
measureNonImageCompletionBytes,
requestBodyLimitBytes,
} from '@clerum/llm-provider-attempt-contract'
import { verifyAdminPermit } from './auth/adminPermitVerifier.js'
import { isExpiredExecutionTicket, verifyExecutionTicket } from './auth/executionTicketVerifier.js'
import { type PlatformJwtClaims, verifyPlatformJwt } from './auth/platformJwtVerifier.js'
import {
CodexTransportError,
type StreamCodexCompletionInput,
UpstreamTimeoutError,
listCodexModels,
streamCodexCompletion,
testCodexConnection,
} from './codexTransport.js'
import type { CodexLlmProxyConfig } from './config.js'
import { ControlApiClient, ControlApiClientError, fetchCauseCode } from './controlApiClient.js'
import { logger } from './logger.js'
import { createProxyMetrics } from './metrics.js'
import {
OriginDeniedError,
type OriginPolicyOptions,
defaultAddressLookup,
} from './originPolicy.js'
import {
BODY_READ_DEADLINE_MS,
BodyBudget,
ENVELOPE_ALLOWANCE_BYTES,
IN_FLIGHT_BODY_BUDGET_BODIES,
RequestLimitError,
STREAM_LIMITS,
TicketLifeError,
VISUAL_PER_HOST_MAX_ADMITTED,
VISUAL_READ_CLOSE_GRACE_MS,
streamGate,
visualStreamGate,
} from './requestLimits.js'
import { startSseHeartbeat } from './sseHeartbeat.js'
type AdmittedRequest = Request & {
/**
* #739 D1 — the request's one admission clock, stamped by body admission:
* arrival + `STREAM_LIMITS.maxQueueWaitMs`, in epoch ms. Every wait the
* request performs (body budget, visual gate, stream gate) ends by then.
*/
codexAdmissionDeadlineAt?: number
/**
* The arrival instant the admission clock was stamped from, in epoch ms.
* The visual gate measures its own queue wait from it, so its refusal
* identity does not depend on when the request reached the gate.
*/
codexAdmissionArrivedAt?: number
/**
* #739 D2 — the body's budget reservation, set once it was granted. The
* handler releases it when the upstream accepted the request; the
* response's `close` event releases it on every other path. Idempotent.
*/
codexBodyRelease?: () => void
}
type GatedRequest = AdmittedRequest & {
codexStreamRelease?: () => void
/** Set by the platform gate before body admission (R9-M-B). */
codexPlatform?: PlatformJwtClaims
}
const COMPLETION_KEYS = new Set(['executionTicket', 'requestHash', 'request', 'deadlineMs'])
const ADMIN_KEYS = new Set(['accessToken'])
const COMPLETION_PATH = '/internal/runtime/v1/codex/completions'
const completionBodySchema = z
.object({
// A signed JWT of a few hundred bytes; the bound keeps an oversized
// string from reaching jwt.verify.
executionTicket: z.string().min(1).max(ENVELOPE_ALLOWANCE_BYTES),
requestHash: z.string().regex(/^[a-f0-9]{64}$/),
request: z.object({}).passthrough(),
deadlineMs: z.number().int().positive().optional(),
})
.strict()
const adminBodySchema = z.object({ accessToken: z.string().min(1) }).strict()
function bearer(req: Request): string {
const raw = String(req.header('authorization') || '')
return raw.replace(/^bearer\s+/i, '').trim()
}
function contentLengthBytes(req: Request): number | null {
const header = req.headers['content-length']
if (typeof header !== 'string' || !/^[0-9]+$/.test(header)) return null
const value = Number(header)
return Number.isSafeInteger(value) ? value : null
}
function isRecord(value: unknown): value is Record<string, unknown> {
return value !== null && typeof value === 'object' && !Array.isArray(value)
}
/**
* `type` is the fixed error type of a refused body (body-parser's or the
* structure scan's), never a message or any part of the body.
*/
function reject(res: Response, status: number, code: string, type?: string): void {
if (res.headersSent) return
logger.warn(
type === undefined
? { event: 'codex_proxy_denied', code }
: { event: 'codex_proxy_denied', code, type },
'request denied'
)
res.status(status).json({ error: code })
}
// Every JSON parser scans the raw body before JSON.parse. JSON.parse
// allocates one heap object per container, so a body within the byte limit
// can still exhaust the heap; the scan refuses a body denser, more nested or
// with more containers than any request the contract accepts. Its depth bound
// (LIMITS.maxNestingDepth + 6) also keeps JSON.stringify and the contract's
// byte measurement from throwing RangeError on a deep body after parse.
const verifyBodyStructure = createBodyStructureVerify(BODY_STRUCTURE_LIMITS)
function boundedErrorHandler(err: unknown, _req: Request, res: Response, _next: () => void): void {
const typed = err as { type?: string; status?: number; body?: unknown }
// The refusals below share one response code per status, so the logged type
// is what tells a structural 413 from a byte-limit 413.
const type = typeof typed?.type === 'string' ? typed.type : undefined
if (typed?.type === 'entity.too.large' || typed?.status === 413) {
reject(res, 413, 'payload_too_large', type)
return
}
// R9-1: the parsers run with `inflate: false`, so body-parser refuses an
// encoded body before reading it.
if (typed?.type === 'encoding.unsupported' || typed?.type === 'charset.unsupported') {
reject(res, 415, 'unsupported_media_type', type)
return
}
if (err instanceof SyntaxError || typed?.type === 'body.structure.too.deep') {
reject(res, 400, 'invalid_request', type)
return
}
// The client closed the connection before it sent the whole body. Nobody
// reads a response, so none is written; the budget or visual slot is
// released by the response's `close` hook.
if (typed?.type === 'request.aborted') {
logger.info({ event: 'codex_proxy_request_aborted', type }, 'client aborted the request body')
return
}
// body-parser attaches the raw body to an error thrown by `verify` or by
// JSON.parse. Such an error is logged by its type and status only.
if (typed?.body !== undefined) {
logger.error(
{ event: 'codex_proxy_error', type: typed.type, status: typed.status },
'unhandled request error'
)
} else {
logger.error({ event: 'codex_proxy_error', err }, 'unhandled request error')
}
reject(res, 500, 'internal_error')
}
/**
* #731 R3-2 — memory-bounded admission around the body parser `parse`. It runs
* after the caller's token was checked (R9-M-B), so an anonymous caller never
* takes budget. A body is read only once its declared `Content-Length` fits
* the shared byte budget, so the bodies in memory are bounded by bytes rather
* than by the stream gate's request count. A `Transfer-Encoding` body is
* refused with 411 instead of being read unbounded; a non-numeric
* `Content-Length` is refused with 400. A request with neither header has no
* body under HTTP/1.1 (RFC 9112 §6.3): `parse` reads nothing, and bytes sent
* after its headers are parsed as the next request, which Node answers 400.
* Bodies over the limit go to `parse` without a reservation: on the admin app
* express.json answers 413 from the header without buffering them, and on the
* runtime app
* `selectTransportBudget` either answers 413 or admits them through the visual
* gate. A granted body must be read and parsed within `readDeadlineMs` of the
* grant, or it is answered 408 `request_timeout`, its reservation released and
* its connection closed. Otherwise the reservation is held until the upstream
* accepted the completion (#739 D2: the handler releases
* `req.codexBodyRelease` once the upstream fetch resolved, when the body has
* been written), or until the response closes on every path that never
* reaches the upstream. A queued waiter is dropped if the client leaves
* first; a full queue gets the stream gate's overload response. #739 D1: the
* request's admission clock is stamped here, at arrival, and a waiter still
* queued when it runs out gets the same overload response.
*/
function bodyAdmission(
budget: BodyBudget,
maxBodyBytes: number,
readDeadlineMs: number,
parse: RequestHandler
) {
return (req: AdmittedRequest, res: Response, next: NextFunction): void => {
const arrivedAt = Date.now()
const admissionDeadlineAt = arrivedAt + STREAM_LIMITS.maxQueueWaitMs
req.codexAdmissionArrivedAt = arrivedAt
req.codexAdmissionDeadlineAt = admissionDeadlineAt
if (req.headers['transfer-encoding'] !== undefined) {
reject(res, 411, 'length_required')
return
}
const declared = req.headers['content-length']
if (declared === undefined) {
parse(req, res, next)
return
}
if (!/^\d+$/.test(declared)) {
reject(res, 400, 'invalid_request')
return
}
const bytes = Number(declared)
if (bytes === 0 || bytes > maxBodyBytes) {
parse(req, res, next)
return
}
const abort = new AbortController()
let release: (() => void) | undefined
let readDeadline: ReturnType<typeof setTimeout> | undefined
res.once('close', () => {
clearTimeout(readDeadline)
if (release) release()
else abort.abort()
})
budget.acquire(bytes, abort.signal, admissionDeadlineAt).then(
granted => {
if (abort.signal.aborted) {
granted()
return
}
release = granted
req.codexBodyRelease = granted
let expired = false
readDeadline = setTimeout(() => {
expired = true
granted()
if (res.headersSent) return
// The rest of the body is never read, so the connection cannot be reused.
res.setHeader('connection', 'close')
reject(res, 408, 'request_timeout')
}, readDeadlineMs)
parse(req, res, err => {
clearTimeout(readDeadline)
// The 408 already answered this request; the parser's late error
// (the body aborted by the closed connection) has no one to reach.
if (expired) return
next(err)
})
},
(err: unknown) => {
if (!(err instanceof RequestLimitError)) {
next(err)
return
}
if (abort.signal.aborted) return
// RequestLimitError messages are fixed strings with no request data.
logger.warn(
{
event: 'codex_proxy_admission_refused',
reason: 'body_budget',
code: err.code,
kind: err.kind,
detail: err.message,
},
'admission refused'
)
reject(res, 503, err.code)
}
)
}
}
export type ProxyRuntimeDeps = {
controlApiClient?: ControlApiClient
fetchFn?: typeof fetch
/**
* Test seam for the DNS half of the origin policy. Production always uses
* `defaultAddressLookup`; hermetic e2e injects a resolver so the frozen
* chatgpt.com origin check runs without live DNS. The URL freeze in
* `assertAllowedUpstreamUrl` is unaffected by this seam.
*/
lookup?: OriginPolicyOptions['lookup']
/** Test seam: hang or observe a stream without contacting ChatGPT. */
streamCompletion?: typeof streamCodexCompletion
/** Test seam for the body-read deadline. Production uses `BODY_READ_DEADLINE_MS`. */
bodyReadDeadlineMs?: number
/** Test seam for the bounded visual-parser close backstop. */
visualReadCloseGraceMs?: number
}
export type ProxyServers = {
runtime: Server
admin: Server
probe: Server
runtimeApp: Express
adminApp: Express
probeApp: Express
close: () => Promise<void>
}
export function createProxyApps(
config: CodexLlmProxyConfig,
deps: ProxyRuntimeDeps = {}
): ProxyServers {
const metricsRegistry = new Registry()
collectDefaultMetrics({ register: metricsRegistry })
const metrics = createProxyMetrics(metricsRegistry)
const client =
deps.controlApiClient ??
new ControlApiClient({
baseUrl: config.controlApiBaseUrl,
serviceName: config.controlApiServiceName,
serviceToken: config.controlApiServiceToken,
})
const fetchFn = deps.fetchFn ?? fetch
const lookup = deps.lookup ?? defaultAddressLookup
const runtimeRateLimit = rateLimit({
windowMs: 60_000,
limit: 60,
standardHeaders: 'draft-7',
legacyHeaders: false,
// G1-2 (#720): the Host reads a JSON error code. The library sets
// Retry-After and the draft-7 headers before it calls the handler.
handler: (_req, res) => reject(res, 429, 'rate_limited'),
})
const adminRateLimit = rateLimit({
windowMs: 60_000,
limit: 30,
standardHeaders: 'draft-7',
legacyHeaders: false,
})
// One budget for both apps: every body this process reads counts against it.
const bodyBudget = new BodyBudget(IN_FLIGHT_BODY_BUDGET_BODIES * config.maxBodyBytes)
const bodyReadDeadlineMs = deps.bodyReadDeadlineMs ?? BODY_READ_DEADLINE_MS
const visualReadCloseGraceMs = deps.visualReadCloseGraceMs ?? VISUAL_READ_CLOSE_GRACE_MS
for (const [name, value] of Object.entries({ bodyReadDeadlineMs, visualReadCloseGraceMs })) {
if (!Number.isSafeInteger(value) || value <= 0 || value > 2_147_483_647) {
throw new RangeError(`${name} must be a positive signed-32-bit integer`)
}
}
// Per-principal occupancy of the visual gate (running plus queued entries).
// Keyed on the verified platform principal: sub plus sorted hostRefs.
const visualPrincipalAdmissions = new Map<string, number>()
const visualRequestOwners = new WeakSet<GatedRequest>()
const runtimeApp = express()
// R9-1: `inflate: false` on every parser. The budgets count the declared wire
// length, so an encoded body is refused (415) instead of inflated past it.
const ordinaryJson = express.json({
limit: config.maxBodyBytes,
inflate: false,
verify: verifyBodyStructure,
})
const visualJson = express.json({
limit: config.maxVisualBodyBytes,
inflate: false,
verify: verifyBodyStructure,
})
// R9-M-B: the token is checked from the header before any budget is taken or
// any body byte is read, so an anonymous caller cannot hold a reservation.
// It runs after the rate limiter, so it cannot be forced ahead of the limit.
const platformGate = (req: GatedRequest, res: Response, next: NextFunction): void => {
const token = bearer(req)
if (verifyAdminPermit(token, config)) {
reject(res, 403, 'insufficient_scope')
return
}
const platform = verifyPlatformJwt(token, config)
if (!platform) {
reject(res, 401, 'Unauthorized')
return
}
req.codexPlatform = platform
next()
}
// The parser behind body admission. Every request here carries a verified
// platform JWT.
//
// A granted visual slot is released only after parser completion (or by the
// route's stream finally); a response close starts a bounded destruction
// backstop instead of freeing a live parser owner.
// schemaVersion is not known until the body is parsed, so the visual gate
// must not be taken for every platform JWT. Only a declared Content-Length
// above the ordinary cap can be a visual envelope. Those bodies take the
// 2-wide gate (the image bytes stay resident for the ChatGPT stream). Every
// smaller body, including every valid V1, stays on the ordinary parser and
// later the 8-wide stream gate. A missing length cannot be upgraded: body
// admission already refused a chunked body with 411, and a request with
// neither header has no body, so only a declared length takes a visual slot.
const selectTransportBudget = (req: GatedRequest, res: Response, next: NextFunction): void => {
const declared = contentLengthBytes(req)
if (declared !== null && declared > config.maxVisualBodyBytes) {
reject(res, 413, 'payload_too_large')
return
}
if (declared === null || declared <= config.maxBodyBytes) {
ordinaryJson(req, res, next)
return
}
// One principal gets a fair share of the visual gate, so a
// single host cannot fill the entries every other host also needs. The
// share is taken before the gate wait, so queued entries count too.
const platform = req.codexPlatform
if (!platform) {
// Fail closed: admission without verified claims must never fall back to
// a shared per-principal key. platformGate runs before this middleware.
reject(res, 401, 'Unauthorized')
return
}
// bodyAdmission stamps both before it hands the request here.
const arrivedAt = req.codexAdmissionArrivedAt
const admissionDeadlineAt = req.codexAdmissionDeadlineAt
if (arrivedAt === undefined || admissionDeadlineAt === undefined) {
next(new Error('visual admission was reached without body admission'))
return
}
const visualPrincipal = JSON.stringify([platform.sub, ...[...platform.hostRefs].sort()])
const admittedByPrincipal = visualPrincipalAdmissions.get(visualPrincipal) ?? 0
if (admittedByPrincipal >= VISUAL_PER_HOST_MAX_ADMITTED) {
logger.warn(
{
event: 'codex_proxy_admission_refused',
reason: 'visual_host_share',
limit: VISUAL_PER_HOST_MAX_ADMITTED,
sub: platform.sub,
hostRefs: [...platform.hostRefs].sort(),
},
'admission refused'
)
reject(res, 503, 'visual_host_share')
return
}
visualPrincipalAdmissions.set(visualPrincipal, admittedByPrincipal + 1)
void (async () => {
let release: (() => void) | undefined
let principalShareHeld = true
const releasePrincipalShare = (): void => {
if (!principalShareHeld) return
principalShareHeld = false
const held = visualPrincipalAdmissions.get(visualPrincipal)
if (held === undefined) {
// A held share always has an entry, so a missing one is a counting
// defect. This runs from `close` handlers, where a throw has no
// caller to reach, so it is logged and the release completes.
logger.error(
{ event: 'codex_proxy_error', reason: 'principal_share_missing', sub: platform.sub },
'visual principal share missing at release'
)
visualPrincipalAdmissions.delete(visualPrincipal)
return
}
const remaining = held - 1
if (remaining > 0) visualPrincipalAdmissions.set(visualPrincipal, remaining)
else visualPrincipalAdmissions.delete(visualPrincipal)
}
const parseAbort = new AbortController()
const abortParse = (): void => parseAbort.abort()
req.once('aborted', abortParse)
try {
// The visual gate's own wait is the admission clock: it is measured
// from arrival, so a waiter that waits it out is refused as
// `visual_gate` however many ms passed before it reached the gate.
release = await visualStreamGate.acquire(parseAbort.signal, admissionDeadlineAt, arrivedAt)
} catch (err) {
req.off('aborted', abortParse)
releasePrincipalShare()
// The client left while queued: nobody reads a refusal, and a departed
// client is not gate saturation, so it is not logged as `visual_gate`.
// Same rule as the ordinary path's `!abort.signal.aborted`.
if (err instanceof RequestLimitError && err.kind === 'aborted') return
if (err instanceof RequestLimitError) {
logger.warn(
{
event: 'codex_proxy_admission_refused',
reason: 'visual_gate',
code: err.code,
detail: err.message,
},
'admission refused'
)
reject(res, 503, err.code)
return
}
next(err)
return
}
req.off('aborted', abortParse)
const releaseVisual = (): void => {
releasePrincipalShare()
release?.()
release = undefined
req.codexStreamRelease = undefined
}
req.codexStreamRelease = releaseVisual
let cancelled = false
let parserSettled = false
let closeGrace: ReturnType<typeof setTimeout> | undefined
const cancelParser = (): void => {
if (parserSettled || closeGrace !== undefined) return
closeGrace = setTimeout(() => {
if (!parserSettled) req.destroy()
}, visualReadCloseGraceMs)
}
const readDeadline = setTimeout(() => {
cancelled = true
cancelParser()
if (res.headersSent) return
// The rest of the body is never read, so the connection cannot be reused.
res.setHeader('connection', 'close')
reject(res, 408, 'request_timeout')
}, bodyReadDeadlineMs)
const onResponseClose = (): void => {
cancelled = true
clearTimeout(readDeadline)
cancelParser()
}
res.once('close', onResponseClose)
visualJson(req, res, err => {
parserSettled = true
clearTimeout(readDeadline)
if (closeGrace !== undefined) clearTimeout(closeGrace)
res.off('close', onResponseClose)
if (cancelled || err) {
req.body = undefined
if (err !== null && typeof err === 'object' && 'body' in err) {
delete (err as { body?: unknown }).body
}
}
// Capacity remains charged until body-parser has actually unwound. On
// success, `req.codexStreamRelease` transfers ownership to the handler,
// which releases it in its outer `finally`; refusals release here.
if (cancelled || err) releaseVisual()
if (cancelled) return
if (err) {
next(err)
return
}
res.once('close', () => {
// A synchronous handler throw can happen before its async owner
// installs cancellation and a finally block. Once that owner has
// taken over, response close must not release its live work.
if (!visualRequestOwners.has(req)) releaseVisual()
})
next()
})
})()
}
// Order: rate limit, token, body admission around the parser, handler.
const runtimeAdmission = bodyAdmission(
bodyBudget,
config.maxBodyBytes,
bodyReadDeadlineMs,
selectTransportBudget
)
runtimeApp.post(COMPLETION_PATH, runtimeRateLimit, platformGate, runtimeAdmission, (req, res) => {
const gated = req as GatedRequest
const releaseAdmission = (): void => {
gated.codexStreamRelease?.()
gated.codexStreamRelease = undefined
}
const platform = gated.codexPlatform
if (!platform) {
releaseAdmission()
throw new Error('the completion route was reached without the platform gate')
}
const admissionDeadlineAt = gated.codexAdmissionDeadlineAt
if (admissionDeadlineAt === undefined) {
releaseAdmission()
throw new Error('the completion route was reached without body admission')
}
if (!req.is('application/json')) {
releaseAdmission()
reject(res, 415, 'unsupported_media_type')
return
}
const request = isRecord(req.body) ? req.body.request : undefined
const visualDeclared = isRecord(request) && request.schemaVersion === SCHEMA_VERSION_V2
const configuredLimit = visualDeclared ? config.maxVisualBodyBytes : config.maxBodyBytes
const wholeBodyBytes = Buffer.byteLength(JSON.stringify(req.body ?? {}), 'utf8')
// A V1 envelope carries the ticket, the hash and the deadline beside a
// request that may itself sit at the contract cap (#731), so its limit adds
// the envelope allowance. A V2 envelope stays on the contract's visual
// ceiling, the same one buildCodexProxyEnvelope enforces.
const envelopeLimit = visualDeclared
? requestBodyLimitBytes(request)
: requestBodyLimitBytes(request) + ENVELOPE_ALLOWANCE_BYTES
// Declaring V2 raises only the image budget. Text, tools and wrapper
// fields stay on the contract's maxRequestBodyBytes non-image ceiling, plus
// the same envelope allowance control-api's authorizer grants its wrapper.
// The min stays: `createProxyApps` accepts a config that did not pass
// `loadConfig` (bodyAdmission.test.ts builds one with a 16 KiB body limit).
if (
wholeBodyBytes > Math.min(configuredLimit, envelopeLimit) ||
measureNonImageCompletionBytes(req.body) >
LIMITS.maxRequestBodyBytes + ENVELOPE_ALLOWANCE_BYTES
) {
releaseAdmission()
reject(res, 413, 'payload_too_large')
return
}
const extra = Object.keys(req.body ?? {}).find(key => !COMPLETION_KEYS.has(key))
if (extra) {
releaseAdmission()
reject(res, 400, 'unknown_field')
return
}
const parsed = completionBodySchema.safeParse(req.body)
if (!parsed.success) {
releaseAdmission()
reject(res, 400, 'invalid_request')
return
}
if (parsed.data.deadlineMs !== undefined && parsed.data.deadlineMs > config.maxDeadlineMs) {
releaseAdmission()
reject(res, 400, 'invalid_request')
return
}
const ticket = verifyExecutionTicket(parsed.data.executionTicket, config)
if (!ticket) {
releaseAdmission()
// R17-2: body admission can wait up to maxQueueWaitMs, the ticket TTL,
// before this read, so an honest ticket can arrive expired. The Host
// retries ticket_expired with a fresh authorization.
if (isExpiredExecutionTicket(parsed.data.executionTicket, config)) {
reject(res, 403, 'ticket_expired')
} else {
reject(res, 403, 'ticket_invalid')
}
return
}
if (platform.hostRefs.includes('*') || !platform.hostRefs.includes(ticket.hostRef)) {
releaseAdmission()
reject(res, 403, 'host_binding_mismatch')
return
}
if (!config.executionEnabled) {
releaseAdmission()
reject(res, 404, 'disabled')
return
}
void (async () => {
let release: (() => void) | undefined
const abort = new AbortController()
// After express.json() the incoming request is already complete. Listening
// to req 'close' aborts the ChatGPT hop on every call (3–12ms canceled).
// Abort only when the client drops the response before we finish writing.
abortWhenClientDisconnects(req, res, abort)
visualRequestOwners.add(gated)
// One `codex_proxy_attempt_finished` line per attempt. Identifiers and
// counts only: never the body, ticket, frames, tool names or arguments.
const attempt = {
providerAttemptId: ticket.providerAttemptId,
hostRef: ticket.hostRef,
model: ticket.model,
requestHash: ticket.requestHash,
}
const attemptStarted = Date.now()
let toolCalls = 0
let textChunks = 0
let heartbeats = 0
// Set once the redeem succeeded. Every exit path below calls it before
// it ends the response: a tick after `res.end()` would write after end.
let stopHeartbeat: (() => void) | undefined
try {
// A visual slot exists only when Content-Length exceeded the ordinary
// cap. Keep it for the ChatGPT stream only when the parsed V2 body still
// exceeds that cap (image bytes stay resident). Padding, V1, and small
// V2 release it and take the 8-wide gate.
// #739 D1-bis: every admission wait also ends when the ticket dies. A
// request still queued at `exp` could only be redeemed into
// ticket_expired, so it is refused here, before any redeem. The margin
// is zero: a ticket still alive when a slot frees is served as before.
const deadlineAt = Math.min(admissionDeadlineAt, ticket.expiresAtMs)
const refuseTicketLife = (): never => {
logger.warn(
{
event: 'codex_proxy_admission_refused',
reason: 'ticket_life',
providerAttemptId: ticket.providerAttemptId,
hostRef: ticket.hostRef,
},
'admission refused'
)
throw new TicketLifeError()
}
const waitWithinTicketLife = async (
acquire: () => Promise<() => void>
): Promise<() => void> => {
if (deadlineAt <= Date.now()) {
if (ticket.expiresAtMs <= admissionDeadlineAt) refuseTicketLife()
throw new RequestLimitError('admission deadline exceeded', 'deadline')
}
try {
return await acquire()
} catch (err) {
// The wait ended on its own deadline, and that deadline was the
// ticket's expiry: comparing clocks here misses a timer that fired
// a millisecond early.
if (
err instanceof RequestLimitError &&
err.kind === 'deadline' &&
!abort.signal.aborted &&
ticket.expiresAtMs <= admissionDeadlineAt
) {
refuseTicketLife()
}
throw err
}
}
if (visualDeclared && wholeBodyBytes > config.maxBodyBytes) {
release = gated.codexStreamRelease
gated.codexStreamRelease = undefined
} else if (gated.codexStreamRelease) {
// A demoted body was read through the visual gate, so it holds
// no body-budget reservation. It takes one for its compact size
// before it gives up the visual slot, so the 8-wide stream gate
// never holds more resident bodies than the budget admits. Every
// request takes its locks in the same order (visual, budget,
// stream), so no two requests wait on each other.
try {
gated.codexBodyRelease = await waitWithinTicketLife(() =>
bodyBudget.acquire(wholeBodyBytes, abort.signal, deadlineAt)
)
} finally {
releaseAdmission()
}
} else {
releaseAdmission()
}
if (!release) {
release = await waitWithinTicketLife(() => streamGate.acquire(abort.signal, deadlineAt))
}
res.status(200)
res.setHeader('content-type', 'text/event-stream')
res.setHeader('cache-control', 'no-cache')
const started = Date.now()
const stream = deps.streamCompletion ?? streamCodexCompletion
const streamInput: StreamCodexCompletionInput = {
executionTicket: parsed.data.executionTicket,
requestHash: parsed.data.requestHash,
request: parsed.data.request,
deadlineMs: parsed.data.deadlineMs,
maxDeadlineMs: Math.min(config.maxDeadlineMs, config.maxStreamDurationMs),
upstreamIdleTimeoutMs: config.upstreamIdleTimeoutMs,
ticket: {
jti: ticket.jti,
hostRef: ticket.hostRef,
model: ticket.model,
requestHash: ticket.requestHash,
providerAttemptId: ticket.providerAttemptId,
},
signal: abort.signal,
redeem: input => client.redeem(input),
onRedeemed: () => {
stopHeartbeat = startSseHeartbeat(res, abort.signal, config.heartbeatIntervalMs, () => {
heartbeats += 1
})
abort.signal.addEventListener('abort', stopHeartbeat, { once: true })
},
// #739 D2: the body has been written upstream, so its budget
// reservation ends here rather than with the stream. Both the raw
// body and the parsed copy are dropped below (#806 Q1).
onUpstreamAccepted: () => {
gated.codexBodyRelease?.()
gated.codexBodyRelease = undefined
// Every reference to the parsed tree ends with the reservation: a
// live stream keeps this frame, `parsed` and `streamInput` for as
// long as the upstream answers, and an 8-wide gate of dense trees
// would otherwise outlast the budget that admitted them.
req.body = undefined
parsed.data.request = {}
streamInput.request = undefined
},
finalize: input => client.finalize(input),
fetchFn,
lookup,
onFrame: frame => {
if (frame.type === 'tool_call') toolCalls += 1
else textChunks += 1
return writeSseChunk(res, `data: ${JSON.stringify(frame)}\n\n`, abort.signal)
},
}
const result = await stream(streamInput)
stopHeartbeat?.()
res.write(
`data: ${JSON.stringify({ type: 'done', outcome: result.outcome, ...(result.usage ? { usage: result.usage } : {}) })}\n\n`
)
metrics.observeAttempt(result.outcome, 'completion_stream')
metrics.observeStream(Date.now() - started)
const finished = {
event: 'codex_proxy_attempt_finished',
...attempt,
outcome: result.outcome,
deliveredAs: 'sse_done',
toolCalls,
textChunks,
heartbeats,
durationMs: Date.now() - attemptStarted,
...(result.usage ? { usage: result.usage } : {}),
}
if (result.outcome === 'success') logger.info(finished, 'codex attempt finished')
else logger.warn(finished, 'codex attempt finished')
res.end()
} catch (err) {
stopHeartbeat?.()
const mapped = mapError(err)
metrics.observeAttempt('error', 'completion_stream')
// Queue capacity has a terminal local wire code; the request_limit
// metric also includes abort/deadline kinds that keep their existing
// wire semantics. Ticket expiry remains provider_unavailable.
metrics.observeAttemptFailure(
err instanceof RequestLimitError && !(err instanceof TicketLifeError)
? 'request_limit'
: failureLabel(mapped.code)
)
if (err instanceof UpstreamTimeoutError) metrics.observeUpstreamTimeout(err.kind)
const deliveredAs = res.headersSent ? 'sse_error' : 'http_status'
const causeCode = err instanceof ControlApiClientError ? err.causeCode : fetchCauseCode(err)
logger.warn(
{
event: 'codex_proxy_attempt_finished',
...attempt,
outcome: 'failed',
code: mapped.code,
// An invalid_request message is the contract parser's, which
// names caller-supplied fields; the code alone is logged for it.
...(err instanceof CodexTransportError && err.code !== 'invalid_request'
? { reason: err.message, ...(err.details ? { details: err.details } : {}) }
: {}),
// RequestLimitError messages are fixed strings with no request data.
...(err instanceof RequestLimitError ? { reason: err.message } : {}),
// An error no class above maps is a handler defect answered 503:
// the error itself is logged, or nothing would show it.
...(isMappedError(err) ? {} : { err }),
// G1-4: a failed fetch's cause code only, never its message or URL.
...(causeCode ? { causeCode } : {}),
deliveredAs,
...(deliveredAs === 'http_status' ? { httpStatus: mapped.status } : {}),
toolCalls,
textChunks,
heartbeats,
durationMs: Date.now() - attemptStarted,
},
'codex attempt finished'
)
// R1-H2: the upstream status of an upstream_rejected travels on both
// paths, so the Host can tell an entitlement refusal from a 404.
const upstreamStatus =
err instanceof CodexTransportError && err.code === 'upstream_rejected'
? err.details?.upstreamStatus
: undefined
const rejectedBy = typeof upstreamStatus === 'number' ? { upstreamStatus } : {}
if (deliveredAs === 'sse_error') {
res.write(
`data: ${JSON.stringify({ type: 'error', code: mapped.code, ...rejectedBy })}\n\n`
)
res.end()
return
}
// Nothing was written yet, so the staged SSE headers can still be
// replaced; res.json() keeps an existing content-type.
res.removeHeader('cache-control')
res.setHeader('content-type', 'application/json; charset=utf-8')
// G1-1: an upstream Retry-After reaches the Host only on this path; an
// SSE error frame carries the code and, for upstream_rejected, the
// upstream status, never Retry-After.
const retryAfter =
err instanceof CodexTransportError ? err.details?.retryAfterSeconds : undefined
if (typeof retryAfter === 'number') res.setHeader('retry-after', String(retryAfter))
res.status(mapped.status).json({ error: mapped.code, ...rejectedBy })
} finally {
stopHeartbeat?.()
release?.()
// Defence in depth for an early throw after this async owner took over
// but before the visual release was moved into `release`.
gated.codexStreamRelease?.()
gated.codexStreamRelease = undefined
// A demoted body's reservation, when the stream ended before the body
// was written upstream. Budgeted bodies are also released on close.
gated.codexBodyRelease?.()
gated.codexBodyRelease = undefined
}
})()
})
runtimeApp.use((_req, res) => reject(res, 404, 'not_found'))
runtimeApp.use(boundedErrorHandler)
const adminApp = express()
// R9-M-B: the admin permit is checked before any budget is taken, exactly as
// the platform JWT is on the runtime app.
const adminGate =
(operation: 'catalog_list' | 'connection_test') =>
(req: Request, res: Response, next: NextFunction): void => {
const token = bearer(req)
if (verifyExecutionTicket(token, config)) {
reject(res, 403, 'insufficient_scope')
return
}
if (!verifyAdminPermit(token, config, operation)) {
reject(res, 401, 'Unauthorized')
return
}
next()
}
const adminAdmission = bodyAdmission(
bodyBudget,
config.maxBodyBytes,
bodyReadDeadlineMs,
express.json({ limit: config.maxBodyBytes, inflate: false, verify: verifyBodyStructure })
)
const adminHandler = (kind: 'models' | 'test') => (req: Request, res: Response) => {
if (!req.is('application/json')) {
reject(res, 415, 'unsupported_media_type')
return
}
if (!config.executionEnabled) {
reject(res, 404, 'disabled')
return
}
const extra = Object.keys(req.body ?? {}).find(key => !ADMIN_KEYS.has(key))
if (extra) {
reject(res, 400, 'unknown_field')
return
}
const parsed = adminBodySchema.safeParse(req.body)
if (!parsed.success) {
reject(res, 400, 'invalid_request')
return
}
void (async () => {
try {
if (kind === 'models') {
const listed = await listCodexModels({
accessToken: parsed.data.accessToken,
fetchFn,
lookup,
})
res.status(200).json(listed)
return
}
const tested = await testCodexConnection({
accessToken: parsed.data.accessToken,
fetchFn,
lookup,
})
res.status(200).json(tested)
} catch (err) {
const mapped = mapError(err)
reject(res, mapped.status, mapped.code)
}
})()
}
adminApp.post(
'/internal/admin/v1/codex/models',
adminRateLimit,
adminGate('catalog_list'),
adminAdmission,
adminHandler('models')
)
adminApp.post(
'/internal/admin/v1/codex/test',
adminRateLimit,
adminGate('connection_test'),
adminAdmission,
adminHandler('test')
)
adminApp.use((_req, res) => reject(res, 404, 'not_found'))
adminApp.use(boundedErrorHandler)
const probeApp = express()
probeApp.get('/healthz', (_req, res) => res.status(200).json({ ok: true }))
probeApp.get('/readyz', (_req, res) => res.status(200).json({ ok: true }))
probeApp.get('/metrics', async (_req, res) => {
res.set('content-type', metricsRegistry.contentType)
res.status(200).send(await metricsRegistry.metrics())
})
probeApp.use((_req, res) => reject(res, 404, 'not_found'))
const runtime = createServer(runtimeApp)
const admin = createServer(adminApp)
const probe = createServer(probeApp)
return {
runtime,
admin,
probe,
runtimeApp,
adminApp,
probeApp,
close: async () => {
await Promise.all([runtime, admin, probe].map(server => closeServer(server)))
},
}
}
export function startProxy(config: CodexLlmProxyConfig): ProxyServers {
const servers = createProxyApps(config)
servers.runtime.listen(config.runtimePort)
servers.admin.listen(config.adminPort)
servers.probe.listen(config.probePort)
logger.info(
{
event: 'codex_proxy_listen',
runtimePort: config.runtimePort,
adminPort: config.adminPort,
probePort: config.probePort,
},
'codex-llm-proxy listeners ready'
)
return servers
}
// Every code the proxy or control-api is known to send. The failure metric
// uses this as its label allowlist because a control-api error body is not
// bounded by the proxy.
const ATTEMPT_ERROR_STATUS: Record<string, number> = {
invalid_request: 400,
// The upstream's own status class for a request over the context window (#731).
context_length_exceeded: 400,
payload_too_large: 413,
Unauthorized: 401,
origin_denied: 403,
request_hash_mismatch: 403,
ticket_invalid: 403,
ticket_expired: 403,
no_grant: 403,
host_binding_mismatch: 403,
model_not_allowed: 403,
insufficient_scope: 403,
disabled: 404,
// A reserved body that was not read within BODY_READ_DEADLINE_MS (R9-M-B).
request_timeout: 408,
ticket_replayed: 409,
// An upstream 429 (G1-1, #720); its Retry-After travels as a header.
rate_limited: 429,
tool_call_limit_exceeded: 422,
// An upstream 4xx the same request would get again (G1-3, #720). Not 502:
// the gateways answer 502 when nothing behind them answered (R1-H2).
upstream_rejected: 422,
// A redeem no control-plane process answered (G1-4, #720); control-api may
// still have received it when the gateway's 502 mapping produced the code.
control_plane_unavailable: 503,
invalid_tool_arguments: 422,
// The attempt ran for its whole stream budget. Retrying the same request
// would spend the same budget again, so it is a gateway timeout, not 503.
stream_duration_exceeded: 504,
connection_unavailable: 503,
visual_host_share: 503,
visual_gate: 503,
proxy_capacity_exceeded: 503,
provider_unavailable: 503,
sse_buffer_exceeded: 503,
invalid_receipt: 503,
conflict: 503,
}
// A control-api code is an unbounded string, so both readers of the table must
// ignore inherited names: `ATTEMPT_ERROR_STATUS['constructor']` is a function,
// not undefined, and `??` would pass it straight to `res.status()`.
function attemptErrorStatus(code: string): number {
return Object.hasOwn(ATTEMPT_ERROR_STATUS, code) ? ATTEMPT_ERROR_STATUS[code]! : 503
}
function failureLabel(code: string): string {
return Object.hasOwn(ATTEMPT_ERROR_STATUS, code) ? code : 'other'
}
type MappedError =
| OriginDeniedError
| RequestLimitError
| CodexTransportError
| ControlApiClientError
/**
* True for the errors `mapError` answers with a status of their own. Anything
* else is a defect in the handler; it is answered 503 like an outage, so the
* caller logs the error itself to keep it visible.
*/
function isMappedError(err: unknown): err is MappedError {
return (
err instanceof OriginDeniedError ||
err instanceof RequestLimitError ||
err instanceof CodexTransportError ||
err instanceof ControlApiClientError
)
}
function mapError(err: unknown): { status: number; code: string } {
if (!isMappedError(err)) return { status: 503, code: 'provider_unavailable' }
if (err instanceof OriginDeniedError) return { status: 403, code: 'origin_denied' }
return { status: attemptErrorStatus(err.code), code: err.code }
}
/**
* Write one SSE chunk and honor socket backpressure. Returns undefined when
* the chunk was accepted (hot path stays synchronous); otherwise a promise that
* settles on 'drain', or when the client closes / the stream aborts so a
* departed consumer never pins the stream slot.
*/
export function writeSseChunk(
res: Response,
chunk: string,
signal: AbortSignal
): Promise<void> | undefined {
if (res.write(chunk)) return undefined
return new Promise<void>(resolve => {
const done = () => {
res.off('drain', done)
res.off('close', done)
signal.removeEventListener('abort', done)
resolve()
}
res.on('drain', done)
res.on('close', done)
signal.addEventListener('abort', done, { once: true })
if (signal.aborted || res.destroyed) done()
})
}
export function abortWhenClientDisconnects(
req: Request,
res: Response,
abort: AbortController
): void {
const cancel = () => {
if (!res.writableEnded) abort.abort()
}
res.on('close', cancel)
req.on('aborted', cancel)
}
function closeServer(server: Server): Promise<void> {
return new Promise(resolve => {
server.close(() => resolve())
})
}