Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript

fix(proxy): release the parsed request when the upstream accepts the stream

Alfredo Lopezd29cadea43cb
diff --git a/codex-llm-proxy/src/codexTransport.ts b/codex-llm-proxy/src/codexTransport.ts
index 40a1adccb..e2d611c95 100644
--- a/codex-llm-proxy/src/codexTransport.ts
+++ b/codex-llm-proxy/src/codexTransport.ts
@@ -116,9 +116,23 @@ export type StreamCodexCompletionResult = {
usage?: SafeUsage
}
-export async function streamCodexCompletion(
- input: StreamCodexCompletionInput
-): Promise<StreamCodexCompletionResult> {
+/**
+ * The validated request, handed from the pre-dispatch checks to the upstream
+ * write. It is emptied as soon as the upstream body is built, so a live stream
+ * never keeps the parsed tree (up to LIMITS.maxRequestElements values) alive.
+ */
+type HeldRequest = { request: CodexCompletionRequest | undefined; requestId: string }
+
+/**
+ * Every check that needs the whole request, in a synchronous function: its
+ * locals end with the call, and only the model, the bounded deadline and the
+ * holder survive into the long-lived async frame of the stream.
+ */
+function prepareCodexCompletion(input: StreamCodexCompletionInput): {
+ held: HeldRequest
+ model: string
+ boundedDeadlineMs: number
+} {
const parsed = parseCodexCompletionRequest(input.request)
if (!parsed.ok) {
throw new CodexTransportError(
@@ -146,6 +160,17 @@ export async function streamCodexCompletion(
input.deadlineMs ?? request.deadlineMs,
input.maxDeadlineMs
)
+ return {
+ held: { request, requestId: request.requestId },
+ model: request.model,
+ boundedDeadlineMs,
+ }
+}
+
+export async function streamCodexCompletion(
+ input: StreamCodexCompletionInput
+): Promise<StreamCodexCompletionResult> {
+ const { held, model, boundedDeadlineMs } = prepareCodexCompletion(input)
// A client that disconnected before dispatch must not consume the ticket:
// nothing was redeemed, so there is no attempt receipt to finalize.
if (input.signal?.aborted) {
@@ -154,11 +179,11 @@ export async function streamCodexCompletion(
const redeemed = await input.redeem({
executionTicket: input.executionTicket,
requestHash: input.requestHash,
- model: request.model,
+ model,
hostRef: input.ticket.hostRef,
operation: 'completion_stream',
})
- if (redeemed.transport.servedModel !== request.model) {
+ if (redeemed.transport.servedModel !== model) {
await finalizeQuietly(input, redeemed, 'error')
throw new CodexTransportError('model_not_allowed', 'served model does not match the request')
}
@@ -172,7 +197,7 @@ export async function streamCodexCompletion(
const started = Date.now()
try {
const streamed = await readUpstreamStream({
- request,
+ held,
accessToken,
chatgptAccountId: redeemed.chatgptAccountId,
deadlineMs,
@@ -243,7 +268,7 @@ async function finalizeQuietly(
}
async function readUpstreamStream(input: {
- request: CodexCompletionRequest
+ held: HeldRequest
accessToken: string
chatgptAccountId?: string
deadlineMs: number
@@ -329,6 +354,25 @@ class UpstreamDeadline {
}
}
+/**
+ * Serialize the upstream body and empty the holder. Synchronous, so the tree
+ * and the intermediate payload are unreachable once it returns; only the
+ * serialized string and the tool-name map outlive the call.
+ */
+function buildUpstreamBody(held: HeldRequest): { names: ToolNameMap; body: string } {
+ const request = held.request
+ if (!request) throw new Error('the upstream request was already released')
+ const names = new ToolNameMap([
+ ...(request.tools ?? []).map(tool => tool.name),
+ ...request.messages.flatMap(message =>
+ message.role === 'assistant' ? (message.toolCalls ?? []).map(call => call.name) : []
+ ),
+ ])
+ const body = JSON.stringify(toUpstreamPayload(request, names))
+ held.request = undefined
+ return { names, body }
+}
+
async function dispatchUpstreamStream(
input: Parameters<typeof readUpstreamStream>[0],
deadline: UpstreamDeadline
@@ -338,7 +382,7 @@ async function dispatchUpstreamStream(
const headers = chatgptUpstreamHeaders(input.accessToken, {
'content-type': 'application/json',
accept: 'text/event-stream',
- session_id: input.request.requestId,
+ session_id: input.held.requestId,
...(input.chatgptAccountId ? { 'chatgpt-account-id': input.chatgptAccountId } : {}),
})
if (!headers['chatgpt-account-id']) {
@@ -347,12 +391,7 @@ async function dispatchUpstreamStream(
'Codex access token is missing ChatGPT account id'
)
}
- const names = new ToolNameMap([
- ...(input.request.tools ?? []).map(tool => tool.name),
- ...input.request.messages.flatMap(message =>
- message.role === 'assistant' ? (message.toolCalls ?? []).map(call => call.name) : []
- ),
- ])
+ const { names, body } = buildUpstreamBody(input.held)
const response = await deadline.waitUpstream(
fetchFrozenOrigin({
url,
@@ -362,7 +401,7 @@ async function dispatchUpstreamStream(
method: 'POST',
signal,
headers,
- body: JSON.stringify(toUpstreamPayload(input.request, names)),
+ body,
},
}),
signal
diff --git a/codex-llm-proxy/src/server.ts b/codex-llm-proxy/src/server.ts
index 00e330d7b..dfb134489 100644
--- a/codex-llm-proxy/src/server.ts
+++ b/codex-llm-proxy/src/server.ts
@@ -22,6 +22,7 @@ import { isExpiredExecutionTicket, verifyExecutionTicket } from './auth/executio
import { type PlatformJwtClaims, verifyPlatformJwt } from './auth/platformJwtVerifier.js'
import {
CodexTransportError,
+ type StreamCodexCompletionInput,
UpstreamTimeoutError,
listCodexModels,
streamCodexCompletion,
@@ -699,7 +700,7 @@ export function createProxyApps(
res.setHeader('cache-control', 'no-cache')
const started = Date.now()
const stream = deps.streamCompletion ?? streamCodexCompletion
- const result = await stream({
+ const streamInput: StreamCodexCompletionInput = {
executionTicket: parsed.data.executionTicket,
requestHash: parsed.data.requestHash,
request: parsed.data.request,
@@ -727,7 +728,13 @@ export function createProxyApps(
onUpstreamAccepted: () => {
gated.codexBodyRelease?.()
gated.codexBodyRelease = undefined
+ // Every reference to the parsed tree ends with the reservation: a
+ // live stream keeps this frame, `parsed` and `streamInput` for as
+ // long as the upstream answers, and an 8-wide gate of dense trees
+ // would otherwise outlast the budget that admitted them.
req.body = undefined
+ parsed.data.request = {}
+ streamInput.request = undefined
},
finalize: input => client.finalize(input),
fetchFn,
@@ -737,7 +744,8 @@ export function createProxyApps(
else textChunks += 1
return writeSseChunk(res, `data: ${JSON.stringify(frame)}\n\n`, abort.signal)
},
- })
+ }
+ const result = await stream(streamInput)
stopHeartbeat?.()
res.write(
`data: ${JSON.stringify({ type: 'done', outcome: result.outcome, ...(result.usage ? { usage: result.usage } : {}) })}\n\n`
diff --git a/codex-llm-proxy/test/releaseAtAcceptance.test.ts b/codex-llm-proxy/test/releaseAtAcceptance.test.ts
new file mode 100644
index 000000000..5793dfcd5
--- /dev/null
+++ b/codex-llm-proxy/test/releaseAtAcceptance.test.ts
@@ -0,0 +1,291 @@
+import { generateKeyPairSync } from 'node:crypto'
+import { createServer } from 'node:http'
+import { runInNewContext } from 'node:vm'
+import { setFlagsFromString } from 'node:v8'
+import jwt from 'jsonwebtoken'
+import { afterEach, describe, expect, it, vi } from 'vitest'
+import { hashCodexCompletionRequest } from '@clerum/llm-provider-attempt-contract'
+import type { CodexLlmProxyConfig } from '../src/config.js'
+import type { RedeemAttemptSuccess } from '../src/controlApiClient.js'
+import {
+ type StreamCodexCompletionInput,
+ streamCodexCompletion,
+} from '../src/codexTransport.js'
+import { CODEX_COMPLETIONS_ORIGIN } from '../src/originPolicy.js'
+import { streamGate, visualStreamGate } from '../src/requestLimits.js'
+import { type ProxyRuntimeDeps, createProxyApps } from '../src/server.js'
+
+// A live stream keeps the handler frame, the transport frame and the input
+// object for as long as the upstream answers (V8 keeps the locals of a
+// suspended async function). If any of them still names the parsed request
+// after the upstream accepted the body, one body-sized tree stays resident per
+// live stream, and the 8-wide stream gate multiplies it. These tests hold the
+// tree only through a WeakRef and require a forced GC to collect it.
+
+// The transport streams from the contract's parsed copy, not from the object
+// the caller passed, so a WeakRef on the caller's tree cannot see a holder of
+// the copy. This wrapper records the copy's tools array as each parse returns.
+const parsedCopies = vi.hoisted(() => ({ refs: [] as WeakRef<object>[] }))
+
+vi.mock('@clerum/llm-provider-attempt-contract', async importOriginal => {
+ const actual = await importOriginal<typeof import('@clerum/llm-provider-attempt-contract')>()
+ return {
+ ...actual,
+ parseCodexCompletionRequest: (input: unknown) => {
+ const result = actual.parseCodexCompletionRequest(input)
+ if (result.ok && Array.isArray(result.value.tools)) {
+ parsedCopies.refs.push(new WeakRef(result.value.tools))
+ }
+ return result
+ },
+ }
+})
+
+setFlagsFromString('--expose-gc')
+const collect = runInNewContext('gc') as () => void
+
+async function collectedWithin(ref: WeakRef<object>): Promise<boolean> {
+ for (let attempt = 0; attempt < 10; attempt += 1) {
+ await new Promise(resolve => setTimeout(resolve, 10))
+ collect()
+ if (ref.deref() === undefined) return true
+ }
+ return false
+}
+
+const { privateKey, publicKey } = generateKeyPairSync('rsa', {
+ modulusLength: 2048,
+ publicKeyEncoding: { type: 'spki', format: 'pem' },
+ privateKeyEncoding: { type: 'pkcs8', format: 'pem' },
+})
+
+function sign(payload: Record<string, unknown>, audience: string): string {
+ return jwt.sign(payload, privateKey, {
+ algorithm: 'RS256',
+ issuer: 'control-api',
+ audience,
+ expiresIn: 60,
+ })
+}
+
+function config(): CodexLlmProxyConfig {
+ return {
+ runtimePort: 8080,
+ adminPort: 8081,
+ probePort: 9090,
+ maxBodyBytes: 1_048_576,
+ maxVisualBodyBytes: 24 * 1024 * 1024,
+ maxStreamDurationMs: 1_800_000,
+ maxDeadlineMs: 1_800_000,
+ upstreamIdleTimeoutMs: 600_000,
+ heartbeatIntervalMs: 15_000,
+ jwtIssuer: 'control-api',
+ jwtPublicKey: publicKey,
+ executionEnabled: true,
+ controlApiBaseUrl: '',
+ controlApiServiceName: 'codex-llm-proxy',
+ controlApiServiceToken: '',
+ }
+}
+
+const MODEL = 'gpt-5.1'
+const REQUEST = {
+ schemaVersion: 'codex-completion-request.v1' as const,
+ requestId: 'req-release-001',
+ idempotencyKey: 'idem-release-001',
+ provider: 'codex-subscription' as const,
+ model: MODEL,
+ messages: [{ role: 'user' as const, content: 'hello' }],
+ tools: [
+ {
+ name: 'release__probe',
+ description: 'probe',
+ parameters: { type: 'object', properties: { q: { type: 'string' } } },
+ },
+ ],
+}
+
+function redeemSuccess(): RedeemAttemptSuccess {
+ const encoded = Buffer.from(
+ JSON.stringify({
+ sub: 'live',
+ 'https://api.openai.com/auth': { chatgpt_account_id: 'acct_live_1' },
+ })
+ ).toString('base64url')
+ return {
+ accessToken: `hdr.${encoded}.sig`,
+ transport: {
+ protocolVersion: 'codex-subscription-transport.v1',
+ completionsOrigin: CODEX_COMPLETIONS_ORIGIN,
+ catalogOrigin: 'https://chatgpt.com/backend-api/codex/models?client_version=1.0.0',
+ operation: 'completion_stream',
+ servedModel: MODEL,
+ maxStreamDurationMs: 1_800_000,
+ },
+ expiryClass: 'short_lived',
+ attemptReceipt: 'a'.repeat(64),
+ }
+}
+
+/**
+ * Builds the input in its own frame, so the test's async frame never holds the
+ * tree: only `input.request` and the WeakRef name it.
+ */
+function transportInput(witness: {
+ fetchReached: boolean
+ collectedInFetch: boolean
+ parsedCollectedInFetch: boolean
+}): {
+ input: StreamCodexCompletionInput
+ ref: WeakRef<object>
+} {
+ const requestHash = hashCodexCompletionRequest(REQUEST)
+ const tree = JSON.parse(JSON.stringify(REQUEST)) as typeof REQUEST
+ // The tools array is what the contract parse and the upstream payload read.
+ const ref = new WeakRef(tree.tools)
+ const input: StreamCodexCompletionInput = {
+ executionTicket: 'ticket-release',
+ requestHash,
+ request: tree,
+ ticket: {
+ jti: 'jti-release',
+ hostRef: 'research-host',
+ model: MODEL,
+ requestHash,
+ providerAttemptId: 'att-release',
+ },
+ maxDeadlineMs: 300_000,
+ redeem: async () => redeemSuccess(),
+ finalize: async () => ({
+ providerAttemptId: 'att-release',
+ outcome: 'success' as const,
+ duplicate: false,
+ }),
+ fetchFn: (async () => {
+ // Runs after the upstream body was built; the stream is not read yet.
+ witness.fetchReached = true
+ witness.collectedInFetch = await collectedWithin(ref)
+ witness.parsedCollectedInFetch = await collectedWithin(parsedCopies.refs[0])
+ return new Response('data: {"type":"response.completed"}\n\n', {
+ status: 200,
+ headers: { 'content-type': 'text/event-stream' },
+ })
+ }) as typeof fetch,
+ lookup: async () => [{ address: '104.18.32.47', family: 4 }],
+ }
+ return { input, ref }
+}
+
+describe('the parsed request is released once the upstream body is built', () => {
+ it('the transport keeps no reference to the tree while it waits on the upstream', async () => {
+ parsedCopies.refs.length = 0
+ const witness = { fetchReached: false, collectedInFetch: false, parsedCollectedInFetch: false }
+ const { input, ref } = transportInput(witness)
+ const pending = streamCodexCompletion(input)
+ // The server drops its own reference at acceptance; the transport's own
+ // holders are what this test measures, so the input's is dropped here.
+ input.request = undefined
+ // Liveness witnesses: the parse ran and produced its own copy, and that
+ // copy was alive until the collector could run.
+ expect(parsedCopies.refs).toHaveLength(1)
+ expect(parsedCopies.refs[0].deref()).toBeDefined()
+ expect(ref.deref()).toBeDefined()
+ await pending
+ // Liveness witness: the fetch was reached, so the body had been built.
+ expect(witness.fetchReached).toBe(true)
+ expect(witness.collectedInFetch).toBe(true)
+ expect(witness.parsedCollectedInFetch).toBe(true)
+ })
+})
+
+describe('the handler releases the parsed request at onUpstreamAccepted', () => {
+ const serversToClose: Array<{ close: () => Promise<void> }> = []
+ const listeners: Array<ReturnType<typeof createServer>> = []
+
+ afterEach(async () => {
+ vi.restoreAllMocks()
+ for (const server of serversToClose.splice(0)) await server.close()
+ await Promise.all(
+ listeners.splice(0).map(
+ listener =>
+ new Promise<void>((resolve, reject) =>
+ listener.close(err => (err ? reject(err) : resolve()))
+ )
+ )
+ )
+ expect(visualStreamGate.snapshot()).toMatchObject({ running: 0, queued: 0 })
+ expect(streamGate.snapshot()).toMatchObject({ running: 0, queued: 0 })
+ })
+
+ it('leaves the tree collectable while the stream stays open', async () => {
+ const witness: {
+ entered: boolean
+ measured: boolean
+ aliveBeforeAccept: boolean
+ collectedAfterAccept: boolean
+ } = { entered: false, measured: false, aliveBeforeAccept: false, collectedAfterAccept: false }
+ let finish: () => void = () => undefined
+ const stream: NonNullable<ProxyRuntimeDeps['streamCompletion']> = async input => {
+ witness.entered = true
+ // The tools array is shared by the zod copy the stream receives and by the
+ // body parser's original, so it is alive while either one is referenced.
+ const ref = new WeakRef((input.request as { tools: object }).tools)
+ witness.aliveBeforeAccept = ref.deref() !== undefined
+ input.onUpstreamAccepted?.()
+ witness.collectedAfterAccept = await collectedWithin(ref)
+ const held = new Promise<void>(resolve => {
+ finish = resolve
+ })
+ witness.measured = true
+ await held
+ return { outcome: 'canceled' as const }
+ }
+ const servers = createProxyApps(config(), { streamCompletion: stream })
+ serversToClose.push(servers)
+ const listener = createServer(servers.runtimeApp).listen(0)
+ listeners.push(listener)
+ const address = listener.address()
+ if (!address || typeof address === 'string') throw new Error('listener has no port')
+
+ const ticket = sign(
+ {
+ jti: '22222222-2222-4222-8222-000000000001',
+ typ: 'codex-execution-ticket',
+ hostRef: 'research-host',
+ model: MODEL,
+ requestHash: 'a'.repeat(64),
+ providerAttemptId: 'att-release-server',
+ },
+ 'codex-llm-proxy'
+ )
+ const platform = sign(
+ {
+ sub: 'default/research-host',
+ hostRefs: ['research-host'],
+ workflowControlScopes: ['llm:codex:execute'],
+ scope: 'workflow:approval:request',
+ },
+ 'workflow-approvals'
+ )
+ const response = fetch(`http://127.0.0.1:${address.port}/internal/runtime/v1/codex/completions`, {
+ method: 'POST',
+ headers: { Authorization: `Bearer ${platform}`, 'content-type': 'application/json' },
+ body: JSON.stringify({ executionTicket: ticket, requestHash: 'a'.repeat(64), request: REQUEST }),
+ })
+ const started = Date.now()
+ while (!witness.measured) {
+ if (Date.now() - started > 5_000) throw new Error('the stream seam was not reached')
+ await new Promise(resolve => setTimeout(resolve, 10))
+ }
+ try {
+ // Liveness witnesses: the seam was entered, the tree was alive when the
+ // stub took its reference, and the acceptance hook ran.
+ expect(witness.entered).toBe(true)
+ expect(witness.aliveBeforeAccept).toBe(true)
+ expect(witness.collectedAfterAccept).toBe(true)
+ } finally {
+ finish()
+ }
+ expect((await response).status).toBe(200)
+ })
+})
diff --git a/grok-llm-proxy/src/grokTransport.ts b/grok-llm-proxy/src/grokTransport.ts
index b59799d98..d4ba739df 100644
--- a/grok-llm-proxy/src/grokTransport.ts
+++ b/grok-llm-proxy/src/grokTransport.ts
@@ -144,9 +144,23 @@ export type StreamGrokCompletionResult = {
usage?: SafeUsage
}
-export async function streamGrokCompletion(
- input: StreamGrokCompletionInput
-): Promise<StreamGrokCompletionResult> {
+/**
+ * The validated request, handed from the pre-dispatch checks to the upstream
+ * write. It is emptied as soon as the upstream body is built, so a live stream
+ * never keeps the parsed tree (up to LIMITS.maxRequestElements values) alive.
+ */
+type HeldRequest = { request: GrokCompletionRequest | undefined }
+
+/**
+ * Every check that needs the whole request, in a synchronous function: its
+ * locals end with the call, and only the model, the bounded deadline and the
+ * holder survive into the long-lived async frame of the stream.
+ */
+function prepareGrokCompletion(input: StreamGrokCompletionInput): {
+ held: HeldRequest
+ model: string
+ boundedDeadlineMs: number
+} {
const parsed = parseGrokCompletionRequest(input.request)
if (!parsed.ok) {
// A `size` refusal (a byte budget, or the container, member or element
@@ -178,6 +192,13 @@ export async function streamGrokCompletion(
input.deadlineMs ?? request.deadlineMs,
input.maxDeadlineMs
)
+ return { held: { request }, model: request.model, boundedDeadlineMs }
+}
+
+export async function streamGrokCompletion(
+ input: StreamGrokCompletionInput
+): Promise<StreamGrokCompletionResult> {
+ const { held, model, boundedDeadlineMs } = prepareGrokCompletion(input)
// A client that disconnected before dispatch must not consume the ticket:
// nothing was redeemed, so there is no attempt receipt to finalize.
if (input.signal?.aborted) {
@@ -186,11 +207,11 @@ export async function streamGrokCompletion(
const redeemed = await input.redeem({
executionTicket: input.executionTicket,
requestHash: input.requestHash,
- model: request.model,
+ model,
hostRef: input.ticket.hostRef,
operation: 'completion_stream',
})
- if (redeemed.transport.servedModel !== request.model) {
+ if (redeemed.transport.servedModel !== model) {
await finalizeQuietly(input, redeemed, 'error')
throw new GrokTransportError('model_not_allowed', 'served model does not match the request')
}
@@ -204,7 +225,7 @@ export async function streamGrokCompletion(
const started = Date.now()
try {
const streamed = await readUpstreamStream({
- request,
+ held,
accessToken,
deadlineMs,
idleTimeoutMs,
@@ -301,7 +322,7 @@ export async function readUpstreamErrorHint(response: {
}
async function readUpstreamStream(input: {
- request: GrokCompletionRequest
+ held: HeldRequest
accessToken: string
deadlineMs: number
idleTimeoutMs: number
@@ -386,6 +407,25 @@ class UpstreamDeadline {
}
}
+/**
+ * Serialize the upstream body and empty the holder. Synchronous, so the tree
+ * and the intermediate payload are unreachable once it returns; only the
+ * serialized string and the tool-name map outlive the call.
+ */
+function buildUpstreamBody(held: HeldRequest): { names: ToolNameMap; body: string } {
+ const request = held.request
+ if (!request) throw new Error('the upstream request was already released')
+ const names = new ToolNameMap([
+ ...(request.tools ?? []).map(tool => tool.name),
+ ...request.messages.flatMap(message =>
+ message.role === 'assistant' ? (message.toolCalls ?? []).map(call => call.name) : []
+ ),
+ ])
+ const body = JSON.stringify(toUpstreamPayload(request, names))
+ held.request = undefined
+ return { names, body }
+}
+
async function dispatchUpstreamStream(
input: Parameters<typeof readUpstreamStream>[0],
deadline: UpstreamDeadline
@@ -396,12 +436,7 @@ async function dispatchUpstreamStream(
'content-type': 'application/json',
accept: 'text/event-stream',
})
- const names = new ToolNameMap([
- ...(input.request.tools ?? []).map(tool => tool.name),
- ...input.request.messages.flatMap(message =>
- message.role === 'assistant' ? (message.toolCalls ?? []).map(call => call.name) : []
- ),
- ])
+ const { names, body } = buildUpstreamBody(input.held)
const response = await deadline.waitUpstream(
fetchFrozenOrigin({
url,
@@ -411,7 +446,7 @@ async function dispatchUpstreamStream(
method: 'POST',
signal,
headers,
- body: JSON.stringify(toUpstreamPayload(input.request, names)),
+ body,
},
}),
signal
diff --git a/grok-llm-proxy/src/server.ts b/grok-llm-proxy/src/server.ts
index 034661742..be029ba47 100644
--- a/grok-llm-proxy/src/server.ts
+++ b/grok-llm-proxy/src/server.ts
@@ -24,6 +24,7 @@ import type { GrokLlmProxyConfig } from './config.js'
import { ControlApiClient, ControlApiClientError, fetchCauseCode } from './controlApiClient.js'
import {
GrokTransportError,
+ type StreamGrokCompletionInput,
UpstreamTimeoutError,
listGrokModels,
streamGrokCompletion,
@@ -711,7 +712,7 @@ export function createProxyApps(
res.setHeader('cache-control', 'no-cache')
const started = Date.now()
const stream = deps.streamCompletion ?? streamGrokCompletion
- const result = await stream({
+ const streamInput: StreamGrokCompletionInput = {
executionTicket: parsed.data.executionTicket,
requestHash: parsed.data.requestHash,
request: parsed.data.request,
@@ -739,7 +740,13 @@ export function createProxyApps(
onUpstreamAccepted: () => {
gated.grokBodyRelease?.()
gated.grokBodyRelease = undefined
+ // Every reference to the parsed tree ends with the reservation: a
+ // live stream keeps this frame, `parsed` and `streamInput` for as
+ // long as the upstream answers, and an 8-wide gate of dense trees
+ // would otherwise outlast the budget that admitted them.
req.body = undefined
+ parsed.data.request = {}
+ streamInput.request = undefined
},
finalize: input => client.finalize(input),
fetchFn,
@@ -749,7 +756,8 @@ export function createProxyApps(
else textChunks += 1
return writeSseChunk(res, `data: ${JSON.stringify(frame)}\n\n`, abort.signal)
},
- })
+ }
+ const result = await stream(streamInput)
stopHeartbeat?.()
res.write(
`data: ${JSON.stringify({ type: 'done', outcome: result.outcome, ...(result.usage ? { usage: result.usage } : {}) })}\n\n`
diff --git a/grok-llm-proxy/test/releaseAtAcceptance.test.ts b/grok-llm-proxy/test/releaseAtAcceptance.test.ts
new file mode 100644
index 000000000..71c1ee202
--- /dev/null
+++ b/grok-llm-proxy/test/releaseAtAcceptance.test.ts
@@ -0,0 +1,291 @@
+import { generateKeyPairSync } from 'node:crypto'
+import { createServer } from 'node:http'
+import { runInNewContext } from 'node:vm'
+import { setFlagsFromString } from 'node:v8'
+import jwt from 'jsonwebtoken'
+import { afterEach, describe, expect, it, vi } from 'vitest'
+import { hashGrokCompletionRequest } from '@clerum/grok-provider-attempt-contract'
+import type { GrokLlmProxyConfig } from '../src/config.js'
+import type { RedeemAttemptSuccess } from '../src/controlApiClient.js'
+import {
+ type StreamGrokCompletionInput,
+ streamGrokCompletion,
+} from '../src/grokTransport.js'
+import { GROK_COMPLETIONS_ORIGIN } from '../src/originPolicy.js'
+import { streamGate, visualStreamGate } from '../src/requestLimits.js'
+import { type ProxyRuntimeDeps, createProxyApps } from '../src/server.js'
+
+// A live stream keeps the handler frame, the transport frame and the input
+// object for as long as the upstream answers (V8 keeps the locals of a
+// suspended async function). If any of them still names the parsed request
+// after the upstream accepted the body, one body-sized tree stays resident per
+// live stream, and the 8-wide stream gate multiplies it. These tests hold the
+// tree only through a WeakRef and require a forced GC to collect it.
+
+// The transport streams from the contract's parsed copy, not from the object
+// the caller passed, so a WeakRef on the caller's tree cannot see a holder of
+// the copy. This wrapper records the copy's tools array as each parse returns.
+const parsedCopies = vi.hoisted(() => ({ refs: [] as WeakRef<object>[] }))
+
+vi.mock('@clerum/grok-provider-attempt-contract', async importOriginal => {
+ const actual = await importOriginal<typeof import('@clerum/grok-provider-attempt-contract')>()
+ return {
+ ...actual,
+ parseGrokCompletionRequest: (input: unknown) => {
+ const result = actual.parseGrokCompletionRequest(input)
+ if (result.ok && Array.isArray(result.value.tools)) {
+ parsedCopies.refs.push(new WeakRef(result.value.tools))
+ }
+ return result
+ },
+ }
+})
+
+setFlagsFromString('--expose-gc')
+const collect = runInNewContext('gc') as () => void
+
+async function collectedWithin(ref: WeakRef<object>): Promise<boolean> {
+ for (let attempt = 0; attempt < 10; attempt += 1) {
+ await new Promise(resolve => setTimeout(resolve, 10))
+ collect()
+ if (ref.deref() === undefined) return true
+ }
+ return false
+}
+
+const { privateKey, publicKey } = generateKeyPairSync('rsa', {
+ modulusLength: 2048,
+ publicKeyEncoding: { type: 'spki', format: 'pem' },
+ privateKeyEncoding: { type: 'pkcs8', format: 'pem' },
+})
+
+function sign(payload: Record<string, unknown>, audience: string): string {
+ return jwt.sign(payload, privateKey, {
+ algorithm: 'RS256',
+ issuer: 'control-api',
+ audience,
+ expiresIn: 60,
+ })
+}
+
+function config(): GrokLlmProxyConfig {
+ return {
+ runtimePort: 8080,
+ adminPort: 8081,
+ probePort: 9090,
+ maxBodyBytes: 1_048_576,
+ maxVisualBodyBytes: 35 * 1024 * 1024,
+ maxStreamDurationMs: 1_800_000,
+ maxDeadlineMs: 1_800_000,
+ upstreamIdleTimeoutMs: 600_000,
+ heartbeatIntervalMs: 15_000,
+ jwtIssuer: 'control-api',
+ jwtPublicKey: publicKey,
+ executionEnabled: true,
+ controlApiBaseUrl: '',
+ controlApiServiceName: 'grok-llm-proxy',
+ controlApiServiceToken: '',
+ }
+}
+
+const MODEL = 'grok-4.6'
+const REQUEST = {
+ schemaVersion: 'grok-completion-request.v1' as const,
+ requestId: 'req-release-001',
+ idempotencyKey: 'idem-release-001',
+ provider: 'grok-subscription' as const,
+ model: MODEL,
+ messages: [{ role: 'user' as const, content: 'hello' }],
+ tools: [
+ {
+ name: 'release__probe',
+ description: 'probe',
+ parameters: { type: 'object', properties: { q: { type: 'string' } } },
+ },
+ ],
+}
+
+function redeemSuccess(): RedeemAttemptSuccess {
+ const encoded = Buffer.from(
+ JSON.stringify({
+ sub: 'live',
+ 'https://api.x.ai/auth': { grok_account_id: 'acct_live_1' },
+ })
+ ).toString('base64url')
+ return {
+ accessToken: `hdr.${encoded}.sig`,
+ transport: {
+ protocolVersion: 'grok-subscription-transport.v1',
+ completionsOrigin: GROK_COMPLETIONS_ORIGIN,
+ catalogOrigin: 'https://cli-chat-proxy.grok.com/v1/models',
+ operation: 'completion_stream',
+ servedModel: MODEL,
+ maxStreamDurationMs: 1_800_000,
+ },
+ expiryClass: 'short_lived',
+ attemptReceipt: 'a'.repeat(64),
+ }
+}
+
+/**
+ * Builds the input in its own frame, so the test's async frame never holds the
+ * tree: only `input.request` and the WeakRef name it.
+ */
+function transportInput(witness: {
+ fetchReached: boolean
+ collectedInFetch: boolean
+ parsedCollectedInFetch: boolean
+}): {
+ input: StreamGrokCompletionInput
+ ref: WeakRef<object>
+} {
+ const requestHash = hashGrokCompletionRequest(REQUEST)
+ const tree = JSON.parse(JSON.stringify(REQUEST)) as typeof REQUEST
+ // The tools array is what the contract parse and the upstream payload read.
+ const ref = new WeakRef(tree.tools)
+ const input: StreamGrokCompletionInput = {
+ executionTicket: 'ticket-release',
+ requestHash,
+ request: tree,
+ ticket: {
+ jti: 'jti-release',
+ hostRef: 'research-host',
+ model: MODEL,
+ requestHash,
+ providerAttemptId: 'att-release',
+ },
+ maxDeadlineMs: 300_000,
+ redeem: async () => redeemSuccess(),
+ finalize: async () => ({
+ providerAttemptId: 'att-release',
+ outcome: 'success' as const,
+ duplicate: false,
+ }),
+ fetchFn: (async () => {
+ // Runs after the upstream body was built; the stream is not read yet.
+ witness.fetchReached = true
+ witness.collectedInFetch = await collectedWithin(ref)
+ witness.parsedCollectedInFetch = await collectedWithin(parsedCopies.refs[0])
+ return new Response('data: {"type":"response.completed"}\n\n', {
+ status: 200,
+ headers: { 'content-type': 'text/event-stream' },
+ })
+ }) as typeof fetch,
+ lookup: async () => [{ address: '104.18.32.47', family: 4 }],
+ }
+ return { input, ref }
+}
+
+describe('the parsed request is released once the upstream body is built', () => {
+ it('the transport keeps no reference to the tree while it waits on the upstream', async () => {
+ parsedCopies.refs.length = 0
+ const witness = { fetchReached: false, collectedInFetch: false, parsedCollectedInFetch: false }
+ const { input, ref } = transportInput(witness)
+ const pending = streamGrokCompletion(input)
+ // The server drops its own reference at acceptance; the transport's own
+ // holders are what this test measures, so the input's is dropped here.
+ input.request = undefined
+ // Liveness witnesses: the parse ran and produced its own copy, and that
+ // copy was alive until the collector could run.
+ expect(parsedCopies.refs).toHaveLength(1)
+ expect(parsedCopies.refs[0].deref()).toBeDefined()
+ expect(ref.deref()).toBeDefined()
+ await pending
+ // Liveness witness: the fetch was reached, so the body had been built.
+ expect(witness.fetchReached).toBe(true)
+ expect(witness.collectedInFetch).toBe(true)
+ expect(witness.parsedCollectedInFetch).toBe(true)
+ })
+})
+
+describe('the handler releases the parsed request at onUpstreamAccepted', () => {
+ const serversToClose: Array<{ close: () => Promise<void> }> = []
+ const listeners: Array<ReturnType<typeof createServer>> = []
+
+ afterEach(async () => {
+ vi.restoreAllMocks()
+ for (const server of serversToClose.splice(0)) await server.close()
+ await Promise.all(
+ listeners.splice(0).map(
+ listener =>
+ new Promise<void>((resolve, reject) =>
+ listener.close(err => (err ? reject(err) : resolve()))
+ )
+ )
+ )
+ expect(visualStreamGate.snapshot()).toMatchObject({ running: 0, queued: 0 })
+ expect(streamGate.snapshot()).toMatchObject({ running: 0, queued: 0 })
+ })
+
+ it('leaves the tree collectable while the stream stays open', async () => {
+ const witness: {
+ entered: boolean
+ measured: boolean
+ aliveBeforeAccept: boolean
+ collectedAfterAccept: boolean
+ } = { entered: false, measured: false, aliveBeforeAccept: false, collectedAfterAccept: false }
+ let finish: () => void = () => undefined
+ const stream: NonNullable<ProxyRuntimeDeps['streamCompletion']> = async input => {
+ witness.entered = true
+ // The tools array is shared by the zod copy the stream receives and by the
+ // body parser's original, so it is alive while either one is referenced.
+ const ref = new WeakRef((input.request as { tools: object }).tools)
+ witness.aliveBeforeAccept = ref.deref() !== undefined
+ input.onUpstreamAccepted?.()
+ witness.collectedAfterAccept = await collectedWithin(ref)
+ const held = new Promise<void>(resolve => {
+ finish = resolve
+ })
+ witness.measured = true
+ await held
+ return { outcome: 'canceled' as const }
+ }
+ const servers = createProxyApps(config(), { streamCompletion: stream })
+ serversToClose.push(servers)
+ const listener = createServer(servers.runtimeApp).listen(0)
+ listeners.push(listener)
+ const address = listener.address()
+ if (!address || typeof address === 'string') throw new Error('listener has no port')
+
+ const ticket = sign(
+ {
+ jti: '22222222-2222-4222-8222-000000000001',
+ typ: 'grok-execution-ticket',
+ hostRef: 'research-host',
+ model: MODEL,
+ requestHash: 'a'.repeat(64),
+ providerAttemptId: 'att-release-server',
+ },
+ 'grok-llm-proxy'
+ )
+ const platform = sign(
+ {
+ sub: 'default/research-host',
+ hostRefs: ['research-host'],
+ workflowControlScopes: ['llm:grok:execute'],
+ scope: 'workflow:approval:request',
+ },
+ 'workflow-approvals'
+ )
+ const response = fetch(`http://127.0.0.1:${address.port}/internal/runtime/v1/grok/completions`, {
+ method: 'POST',
+ headers: { Authorization: `Bearer ${platform}`, 'content-type': 'application/json' },
+ body: JSON.stringify({ executionTicket: ticket, requestHash: 'a'.repeat(64), request: REQUEST }),
+ })
+ const started = Date.now()
+ while (!witness.measured) {
+ if (Date.now() - started > 5_000) throw new Error('the stream seam was not reached')
+ await new Promise(resolve => setTimeout(resolve, 10))
+ }
+ try {
+ // Liveness witnesses: the seam was entered, the tree was alive when the
+ // stub took its reference, and the acceptance hook ran.
+ expect(witness.entered).toBe(true)
+ expect(witness.aliveBeforeAccept).toBe(true)
+ expect(witness.collectedAfterAccept).toBe(true)
+ } finally {
+ finish()
+ }
+ expect((await response).status).toBe(200)
+ })
+})
diff --git a/scripts/tests/test-codex-subscription-t0.sh b/scripts/tests/test-codex-subscription-t0.sh
index 00791989f..3eb98357a 100755
--- a/scripts/tests/test-codex-subscription-t0.sh
+++ b/scripts/tests/test-codex-subscription-t0.sh
@@ -293,6 +293,7 @@ run_group "codex-llm-proxy" "codex-llm-proxy" \
"test/metrics.test.ts" \
"test/originPolicy.test.ts" \
"test/redaction.test.ts" \
+ "test/releaseAtAcceptance.test.ts" \
"test/requestLimits.test.ts" \
"test/runtimePath.hermetic.e2e.test.ts" \
"test/server.security.test.ts" \
diff --git a/scripts/tests/test-grok-subscription-t0.sh b/scripts/tests/test-grok-subscription-t0.sh
index 165d83169..08a51666d 100755
--- a/scripts/tests/test-grok-subscription-t0.sh
+++ b/scripts/tests/test-grok-subscription-t0.sh
@@ -245,6 +245,7 @@ run_group "grok-llm-proxy" "grok-llm-proxy" \
"test/metrics.test.ts" \
"test/originPolicy.test.ts" \
"test/redaction.test.ts" \
+ "test/releaseAtAcceptance.test.ts" \
"test/requestLimits.test.ts" \
"test/runtimePath.hermetic.e2e.test.ts" \
"test/server.security.test.ts" \