Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
fix(proxy): release the parsed request when the upstream accepts the stream
Alfredo Lopezd29cadea43cb
diff --git a/codex-llm-proxy/src/codexTransport.ts b/codex-llm-proxy/src/codexTransport.tsindex 40a1adccb..e2d611c95 100644--- a/codex-llm-proxy/src/codexTransport.ts+++ b/codex-llm-proxy/src/codexTransport.ts@@ -116,9 +116,23 @@ export type StreamCodexCompletionResult = {usage?: SafeUsage}-export async function streamCodexCompletion(- input: StreamCodexCompletionInput-): Promise<StreamCodexCompletionResult> {+/**+ * The validated request, handed from the pre-dispatch checks to the upstream+ * write. It is emptied as soon as the upstream body is built, so a live stream+ * never keeps the parsed tree (up to LIMITS.maxRequestElements values) alive.+ */+type HeldRequest = { request: CodexCompletionRequest | undefined; requestId: string }++/**+ * Every check that needs the whole request, in a synchronous function: its+ * locals end with the call, and only the model, the bounded deadline and the+ * holder survive into the long-lived async frame of the stream.+ */+function prepareCodexCompletion(input: StreamCodexCompletionInput): {+ held: HeldRequest+ model: string+ boundedDeadlineMs: number+} {const parsed = parseCodexCompletionRequest(input.request)if (!parsed.ok) {throw new CodexTransportError(@@ -146,6 +160,17 @@ export async function streamCodexCompletion(input.deadlineMs ?? request.deadlineMs,input.maxDeadlineMs)+ return {+ held: { request, requestId: request.requestId },+ model: request.model,+ boundedDeadlineMs,+ }+}++export async function streamCodexCompletion(+ input: StreamCodexCompletionInput+): Promise<StreamCodexCompletionResult> {+ const { held, model, boundedDeadlineMs } = prepareCodexCompletion(input)// A client that disconnected before dispatch must not consume the ticket:// nothing was redeemed, so there is no attempt receipt to finalize.if (input.signal?.aborted) {@@ -154,11 +179,11 @@ export async function streamCodexCompletion(const redeemed = await input.redeem({executionTicket: input.executionTicket,requestHash: input.requestHash,- model: request.model,+ model,hostRef: input.ticket.hostRef,operation: 'completion_stream',})- if (redeemed.transport.servedModel !== request.model) {+ if (redeemed.transport.servedModel !== model) {await finalizeQuietly(input, redeemed, 'error')throw new CodexTransportError('model_not_allowed', 'served model does not match the request')}@@ -172,7 +197,7 @@ export async function streamCodexCompletion(const started = Date.now()try {const streamed = await readUpstreamStream({- request,+ held,accessToken,chatgptAccountId: redeemed.chatgptAccountId,deadlineMs,@@ -243,7 +268,7 @@ async function finalizeQuietly(}async function readUpstreamStream(input: {- request: CodexCompletionRequest+ held: HeldRequestaccessToken: stringchatgptAccountId?: stringdeadlineMs: number@@ -329,6 +354,25 @@ class UpstreamDeadline {}}+/**+ * Serialize the upstream body and empty the holder. Synchronous, so the tree+ * and the intermediate payload are unreachable once it returns; only the+ * serialized string and the tool-name map outlive the call.+ */+function buildUpstreamBody(held: HeldRequest): { names: ToolNameMap; body: string } {+ const request = held.request+ if (!request) throw new Error('the upstream request was already released')+ const names = new ToolNameMap([+ ...(request.tools ?? []).map(tool => tool.name),+ ...request.messages.flatMap(message =>+ message.role === 'assistant' ? (message.toolCalls ?? []).map(call => call.name) : []+ ),+ ])+ const body = JSON.stringify(toUpstreamPayload(request, names))+ held.request = undefined+ return { names, body }+}+async function dispatchUpstreamStream(input: Parameters<typeof readUpstreamStream>[0],deadline: UpstreamDeadline@@ -338,7 +382,7 @@ async function dispatchUpstreamStream(const headers = chatgptUpstreamHeaders(input.accessToken, {'content-type': 'application/json',accept: 'text/event-stream',- session_id: input.request.requestId,+ session_id: input.held.requestId,...(input.chatgptAccountId ? { 'chatgpt-account-id': input.chatgptAccountId } : {}),})if (!headers['chatgpt-account-id']) {@@ -347,12 +391,7 @@ async function dispatchUpstreamStream('Codex access token is missing ChatGPT account id')}- const names = new ToolNameMap([- ...(input.request.tools ?? []).map(tool => tool.name),- ...input.request.messages.flatMap(message =>- message.role === 'assistant' ? (message.toolCalls ?? []).map(call => call.name) : []- ),- ])+ const { names, body } = buildUpstreamBody(input.held)const response = await deadline.waitUpstream(fetchFrozenOrigin({url,@@ -362,7 +401,7 @@ async function dispatchUpstreamStream(method: 'POST',signal,headers,- body: JSON.stringify(toUpstreamPayload(input.request, names)),+ body,},}),signaldiff --git a/codex-llm-proxy/src/server.ts b/codex-llm-proxy/src/server.tsindex 00e330d7b..dfb134489 100644--- a/codex-llm-proxy/src/server.ts+++ b/codex-llm-proxy/src/server.ts@@ -22,6 +22,7 @@ import { isExpiredExecutionTicket, verifyExecutionTicket } from './auth/executioimport { type PlatformJwtClaims, verifyPlatformJwt } from './auth/platformJwtVerifier.js'import {CodexTransportError,+ type StreamCodexCompletionInput,UpstreamTimeoutError,listCodexModels,streamCodexCompletion,@@ -699,7 +700,7 @@ export function createProxyApps(res.setHeader('cache-control', 'no-cache')const started = Date.now()const stream = deps.streamCompletion ?? streamCodexCompletion- const result = await stream({+ const streamInput: StreamCodexCompletionInput = {executionTicket: parsed.data.executionTicket,requestHash: parsed.data.requestHash,request: parsed.data.request,@@ -727,7 +728,13 @@ export function createProxyApps(onUpstreamAccepted: () => {gated.codexBodyRelease?.()gated.codexBodyRelease = undefined+ // Every reference to the parsed tree ends with the reservation: a+ // live stream keeps this frame, `parsed` and `streamInput` for as+ // long as the upstream answers, and an 8-wide gate of dense trees+ // would otherwise outlast the budget that admitted them.req.body = undefined+ parsed.data.request = {}+ streamInput.request = undefined},finalize: input => client.finalize(input),fetchFn,@@ -737,7 +744,8 @@ export function createProxyApps(else textChunks += 1return writeSseChunk(res, `data: ${JSON.stringify(frame)}\n\n`, abort.signal)},- })+ }+ const result = await stream(streamInput)stopHeartbeat?.()res.write(`data: ${JSON.stringify({ type: 'done', outcome: result.outcome, ...(result.usage ? { usage: result.usage } : {}) })}\n\n`diff --git a/codex-llm-proxy/test/releaseAtAcceptance.test.ts b/codex-llm-proxy/test/releaseAtAcceptance.test.tsnew file mode 100644index 000000000..5793dfcd5--- /dev/null+++ b/codex-llm-proxy/test/releaseAtAcceptance.test.ts@@ -0,0 +1,291 @@+import { generateKeyPairSync } from 'node:crypto'+import { createServer } from 'node:http'+import { runInNewContext } from 'node:vm'+import { setFlagsFromString } from 'node:v8'+import jwt from 'jsonwebtoken'+import { afterEach, describe, expect, it, vi } from 'vitest'+import { hashCodexCompletionRequest } from '@clerum/llm-provider-attempt-contract'+import type { CodexLlmProxyConfig } from '../src/config.js'+import type { RedeemAttemptSuccess } from '../src/controlApiClient.js'+import {+ type StreamCodexCompletionInput,+ streamCodexCompletion,+} from '../src/codexTransport.js'+import { CODEX_COMPLETIONS_ORIGIN } from '../src/originPolicy.js'+import { streamGate, visualStreamGate } from '../src/requestLimits.js'+import { type ProxyRuntimeDeps, createProxyApps } from '../src/server.js'++// A live stream keeps the handler frame, the transport frame and the input+// object for as long as the upstream answers (V8 keeps the locals of a+// suspended async function). If any of them still names the parsed request+// after the upstream accepted the body, one body-sized tree stays resident per+// live stream, and the 8-wide stream gate multiplies it. These tests hold the+// tree only through a WeakRef and require a forced GC to collect it.++// The transport streams from the contract's parsed copy, not from the object+// the caller passed, so a WeakRef on the caller's tree cannot see a holder of+// the copy. This wrapper records the copy's tools array as each parse returns.+const parsedCopies = vi.hoisted(() => ({ refs: [] as WeakRef<object>[] }))++vi.mock('@clerum/llm-provider-attempt-contract', async importOriginal => {+ const actual = await importOriginal<typeof import('@clerum/llm-provider-attempt-contract')>()+ return {+ ...actual,+ parseCodexCompletionRequest: (input: unknown) => {+ const result = actual.parseCodexCompletionRequest(input)+ if (result.ok && Array.isArray(result.value.tools)) {+ parsedCopies.refs.push(new WeakRef(result.value.tools))+ }+ return result+ },+ }+})++setFlagsFromString('--expose-gc')+const collect = runInNewContext('gc') as () => void++async function collectedWithin(ref: WeakRef<object>): Promise<boolean> {+ for (let attempt = 0; attempt < 10; attempt += 1) {+ await new Promise(resolve => setTimeout(resolve, 10))+ collect()+ if (ref.deref() === undefined) return true+ }+ return false+}++const { privateKey, publicKey } = generateKeyPairSync('rsa', {+ modulusLength: 2048,+ publicKeyEncoding: { type: 'spki', format: 'pem' },+ privateKeyEncoding: { type: 'pkcs8', format: 'pem' },+})++function sign(payload: Record<string, unknown>, audience: string): string {+ return jwt.sign(payload, privateKey, {+ algorithm: 'RS256',+ issuer: 'control-api',+ audience,+ expiresIn: 60,+ })+}++function config(): CodexLlmProxyConfig {+ return {+ runtimePort: 8080,+ adminPort: 8081,+ probePort: 9090,+ maxBodyBytes: 1_048_576,+ maxVisualBodyBytes: 24 * 1024 * 1024,+ maxStreamDurationMs: 1_800_000,+ maxDeadlineMs: 1_800_000,+ upstreamIdleTimeoutMs: 600_000,+ heartbeatIntervalMs: 15_000,+ jwtIssuer: 'control-api',+ jwtPublicKey: publicKey,+ executionEnabled: true,+ controlApiBaseUrl: '',+ controlApiServiceName: 'codex-llm-proxy',+ controlApiServiceToken: '',+ }+}++const MODEL = 'gpt-5.1'+const REQUEST = {+ schemaVersion: 'codex-completion-request.v1' as const,+ requestId: 'req-release-001',+ idempotencyKey: 'idem-release-001',+ provider: 'codex-subscription' as const,+ model: MODEL,+ messages: [{ role: 'user' as const, content: 'hello' }],+ tools: [+ {+ name: 'release__probe',+ description: 'probe',+ parameters: { type: 'object', properties: { q: { type: 'string' } } },+ },+ ],+}++function redeemSuccess(): RedeemAttemptSuccess {+ const encoded = Buffer.from(+ JSON.stringify({+ sub: 'live',+ 'https://api.openai.com/auth': { chatgpt_account_id: 'acct_live_1' },+ })+ ).toString('base64url')+ return {+ accessToken: `hdr.${encoded}.sig`,+ transport: {+ protocolVersion: 'codex-subscription-transport.v1',+ completionsOrigin: CODEX_COMPLETIONS_ORIGIN,+ catalogOrigin: 'https://chatgpt.com/backend-api/codex/models?client_version=1.0.0',+ operation: 'completion_stream',+ servedModel: MODEL,+ maxStreamDurationMs: 1_800_000,+ },+ expiryClass: 'short_lived',+ attemptReceipt: 'a'.repeat(64),+ }+}++/**+ * Builds the input in its own frame, so the test's async frame never holds the+ * tree: only `input.request` and the WeakRef name it.+ */+function transportInput(witness: {+ fetchReached: boolean+ collectedInFetch: boolean+ parsedCollectedInFetch: boolean+}): {+ input: StreamCodexCompletionInput+ ref: WeakRef<object>+} {+ const requestHash = hashCodexCompletionRequest(REQUEST)+ const tree = JSON.parse(JSON.stringify(REQUEST)) as typeof REQUEST+ // The tools array is what the contract parse and the upstream payload read.+ const ref = new WeakRef(tree.tools)+ const input: StreamCodexCompletionInput = {+ executionTicket: 'ticket-release',+ requestHash,+ request: tree,+ ticket: {+ jti: 'jti-release',+ hostRef: 'research-host',+ model: MODEL,+ requestHash,+ providerAttemptId: 'att-release',+ },+ maxDeadlineMs: 300_000,+ redeem: async () => redeemSuccess(),+ finalize: async () => ({+ providerAttemptId: 'att-release',+ outcome: 'success' as const,+ duplicate: false,+ }),+ fetchFn: (async () => {+ // Runs after the upstream body was built; the stream is not read yet.+ witness.fetchReached = true+ witness.collectedInFetch = await collectedWithin(ref)+ witness.parsedCollectedInFetch = await collectedWithin(parsedCopies.refs[0])+ return new Response('data: {"type":"response.completed"}\n\n', {+ status: 200,+ headers: { 'content-type': 'text/event-stream' },+ })+ }) as typeof fetch,+ lookup: async () => [{ address: '104.18.32.47', family: 4 }],+ }+ return { input, ref }+}++describe('the parsed request is released once the upstream body is built', () => {+ it('the transport keeps no reference to the tree while it waits on the upstream', async () => {+ parsedCopies.refs.length = 0+ const witness = { fetchReached: false, collectedInFetch: false, parsedCollectedInFetch: false }+ const { input, ref } = transportInput(witness)+ const pending = streamCodexCompletion(input)+ // The server drops its own reference at acceptance; the transport's own+ // holders are what this test measures, so the input's is dropped here.+ input.request = undefined+ // Liveness witnesses: the parse ran and produced its own copy, and that+ // copy was alive until the collector could run.+ expect(parsedCopies.refs).toHaveLength(1)+ expect(parsedCopies.refs[0].deref()).toBeDefined()+ expect(ref.deref()).toBeDefined()+ await pending+ // Liveness witness: the fetch was reached, so the body had been built.+ expect(witness.fetchReached).toBe(true)+ expect(witness.collectedInFetch).toBe(true)+ expect(witness.parsedCollectedInFetch).toBe(true)+ })+})++describe('the handler releases the parsed request at onUpstreamAccepted', () => {+ const serversToClose: Array<{ close: () => Promise<void> }> = []+ const listeners: Array<ReturnType<typeof createServer>> = []++ afterEach(async () => {+ vi.restoreAllMocks()+ for (const server of serversToClose.splice(0)) await server.close()+ await Promise.all(+ listeners.splice(0).map(+ listener =>+ new Promise<void>((resolve, reject) =>+ listener.close(err => (err ? reject(err) : resolve()))+ )+ )+ )+ expect(visualStreamGate.snapshot()).toMatchObject({ running: 0, queued: 0 })+ expect(streamGate.snapshot()).toMatchObject({ running: 0, queued: 0 })+ })++ it('leaves the tree collectable while the stream stays open', async () => {+ const witness: {+ entered: boolean+ measured: boolean+ aliveBeforeAccept: boolean+ collectedAfterAccept: boolean+ } = { entered: false, measured: false, aliveBeforeAccept: false, collectedAfterAccept: false }+ let finish: () => void = () => undefined+ const stream: NonNullable<ProxyRuntimeDeps['streamCompletion']> = async input => {+ witness.entered = true+ // The tools array is shared by the zod copy the stream receives and by the+ // body parser's original, so it is alive while either one is referenced.+ const ref = new WeakRef((input.request as { tools: object }).tools)+ witness.aliveBeforeAccept = ref.deref() !== undefined+ input.onUpstreamAccepted?.()+ witness.collectedAfterAccept = await collectedWithin(ref)+ const held = new Promise<void>(resolve => {+ finish = resolve+ })+ witness.measured = true+ await held+ return { outcome: 'canceled' as const }+ }+ const servers = createProxyApps(config(), { streamCompletion: stream })+ serversToClose.push(servers)+ const listener = createServer(servers.runtimeApp).listen(0)+ listeners.push(listener)+ const address = listener.address()+ if (!address || typeof address === 'string') throw new Error('listener has no port')++ const ticket = sign(+ {+ jti: '22222222-2222-4222-8222-000000000001',+ typ: 'codex-execution-ticket',+ hostRef: 'research-host',+ model: MODEL,+ requestHash: 'a'.repeat(64),+ providerAttemptId: 'att-release-server',+ },+ 'codex-llm-proxy'+ )+ const platform = sign(+ {+ sub: 'default/research-host',+ hostRefs: ['research-host'],+ workflowControlScopes: ['llm:codex:execute'],+ scope: 'workflow:approval:request',+ },+ 'workflow-approvals'+ )+ const response = fetch(`http://127.0.0.1:${address.port}/internal/runtime/v1/codex/completions`, {+ method: 'POST',+ headers: { Authorization: `Bearer ${platform}`, 'content-type': 'application/json' },+ body: JSON.stringify({ executionTicket: ticket, requestHash: 'a'.repeat(64), request: REQUEST }),+ })+ const started = Date.now()+ while (!witness.measured) {+ if (Date.now() - started > 5_000) throw new Error('the stream seam was not reached')+ await new Promise(resolve => setTimeout(resolve, 10))+ }+ try {+ // Liveness witnesses: the seam was entered, the tree was alive when the+ // stub took its reference, and the acceptance hook ran.+ expect(witness.entered).toBe(true)+ expect(witness.aliveBeforeAccept).toBe(true)+ expect(witness.collectedAfterAccept).toBe(true)+ } finally {+ finish()+ }+ expect((await response).status).toBe(200)+ })+})diff --git a/grok-llm-proxy/src/grokTransport.ts b/grok-llm-proxy/src/grokTransport.tsindex b59799d98..d4ba739df 100644--- a/grok-llm-proxy/src/grokTransport.ts+++ b/grok-llm-proxy/src/grokTransport.ts@@ -144,9 +144,23 @@ export type StreamGrokCompletionResult = {usage?: SafeUsage}-export async function streamGrokCompletion(- input: StreamGrokCompletionInput-): Promise<StreamGrokCompletionResult> {+/**+ * The validated request, handed from the pre-dispatch checks to the upstream+ * write. It is emptied as soon as the upstream body is built, so a live stream+ * never keeps the parsed tree (up to LIMITS.maxRequestElements values) alive.+ */+type HeldRequest = { request: GrokCompletionRequest | undefined }++/**+ * Every check that needs the whole request, in a synchronous function: its+ * locals end with the call, and only the model, the bounded deadline and the+ * holder survive into the long-lived async frame of the stream.+ */+function prepareGrokCompletion(input: StreamGrokCompletionInput): {+ held: HeldRequest+ model: string+ boundedDeadlineMs: number+} {const parsed = parseGrokCompletionRequest(input.request)if (!parsed.ok) {// A `size` refusal (a byte budget, or the container, member or element@@ -178,6 +192,13 @@ export async function streamGrokCompletion(input.deadlineMs ?? request.deadlineMs,input.maxDeadlineMs)+ return { held: { request }, model: request.model, boundedDeadlineMs }+}++export async function streamGrokCompletion(+ input: StreamGrokCompletionInput+): Promise<StreamGrokCompletionResult> {+ const { held, model, boundedDeadlineMs } = prepareGrokCompletion(input)// A client that disconnected before dispatch must not consume the ticket:// nothing was redeemed, so there is no attempt receipt to finalize.if (input.signal?.aborted) {@@ -186,11 +207,11 @@ export async function streamGrokCompletion(const redeemed = await input.redeem({executionTicket: input.executionTicket,requestHash: input.requestHash,- model: request.model,+ model,hostRef: input.ticket.hostRef,operation: 'completion_stream',})- if (redeemed.transport.servedModel !== request.model) {+ if (redeemed.transport.servedModel !== model) {await finalizeQuietly(input, redeemed, 'error')throw new GrokTransportError('model_not_allowed', 'served model does not match the request')}@@ -204,7 +225,7 @@ export async function streamGrokCompletion(const started = Date.now()try {const streamed = await readUpstreamStream({- request,+ held,accessToken,deadlineMs,idleTimeoutMs,@@ -301,7 +322,7 @@ export async function readUpstreamErrorHint(response: {}async function readUpstreamStream(input: {- request: GrokCompletionRequest+ held: HeldRequestaccessToken: stringdeadlineMs: numberidleTimeoutMs: number@@ -386,6 +407,25 @@ class UpstreamDeadline {}}+/**+ * Serialize the upstream body and empty the holder. Synchronous, so the tree+ * and the intermediate payload are unreachable once it returns; only the+ * serialized string and the tool-name map outlive the call.+ */+function buildUpstreamBody(held: HeldRequest): { names: ToolNameMap; body: string } {+ const request = held.request+ if (!request) throw new Error('the upstream request was already released')+ const names = new ToolNameMap([+ ...(request.tools ?? []).map(tool => tool.name),+ ...request.messages.flatMap(message =>+ message.role === 'assistant' ? (message.toolCalls ?? []).map(call => call.name) : []+ ),+ ])+ const body = JSON.stringify(toUpstreamPayload(request, names))+ held.request = undefined+ return { names, body }+}+async function dispatchUpstreamStream(input: Parameters<typeof readUpstreamStream>[0],deadline: UpstreamDeadline@@ -396,12 +436,7 @@ async function dispatchUpstreamStream('content-type': 'application/json',accept: 'text/event-stream',})- const names = new ToolNameMap([- ...(input.request.tools ?? []).map(tool => tool.name),- ...input.request.messages.flatMap(message =>- message.role === 'assistant' ? (message.toolCalls ?? []).map(call => call.name) : []- ),- ])+ const { names, body } = buildUpstreamBody(input.held)const response = await deadline.waitUpstream(fetchFrozenOrigin({url,@@ -411,7 +446,7 @@ async function dispatchUpstreamStream(method: 'POST',signal,headers,- body: JSON.stringify(toUpstreamPayload(input.request, names)),+ body,},}),signaldiff --git a/grok-llm-proxy/src/server.ts b/grok-llm-proxy/src/server.tsindex 034661742..be029ba47 100644--- a/grok-llm-proxy/src/server.ts+++ b/grok-llm-proxy/src/server.ts@@ -24,6 +24,7 @@ import type { GrokLlmProxyConfig } from './config.js'import { ControlApiClient, ControlApiClientError, fetchCauseCode } from './controlApiClient.js'import {GrokTransportError,+ type StreamGrokCompletionInput,UpstreamTimeoutError,listGrokModels,streamGrokCompletion,@@ -711,7 +712,7 @@ export function createProxyApps(res.setHeader('cache-control', 'no-cache')const started = Date.now()const stream = deps.streamCompletion ?? streamGrokCompletion- const result = await stream({+ const streamInput: StreamGrokCompletionInput = {executionTicket: parsed.data.executionTicket,requestHash: parsed.data.requestHash,request: parsed.data.request,@@ -739,7 +740,13 @@ export function createProxyApps(onUpstreamAccepted: () => {gated.grokBodyRelease?.()gated.grokBodyRelease = undefined+ // Every reference to the parsed tree ends with the reservation: a+ // live stream keeps this frame, `parsed` and `streamInput` for as+ // long as the upstream answers, and an 8-wide gate of dense trees+ // would otherwise outlast the budget that admitted them.req.body = undefined+ parsed.data.request = {}+ streamInput.request = undefined},finalize: input => client.finalize(input),fetchFn,@@ -749,7 +756,8 @@ export function createProxyApps(else textChunks += 1return writeSseChunk(res, `data: ${JSON.stringify(frame)}\n\n`, abort.signal)},- })+ }+ const result = await stream(streamInput)stopHeartbeat?.()res.write(`data: ${JSON.stringify({ type: 'done', outcome: result.outcome, ...(result.usage ? { usage: result.usage } : {}) })}\n\n`diff --git a/grok-llm-proxy/test/releaseAtAcceptance.test.ts b/grok-llm-proxy/test/releaseAtAcceptance.test.tsnew file mode 100644index 000000000..71c1ee202--- /dev/null+++ b/grok-llm-proxy/test/releaseAtAcceptance.test.ts@@ -0,0 +1,291 @@+import { generateKeyPairSync } from 'node:crypto'+import { createServer } from 'node:http'+import { runInNewContext } from 'node:vm'+import { setFlagsFromString } from 'node:v8'+import jwt from 'jsonwebtoken'+import { afterEach, describe, expect, it, vi } from 'vitest'+import { hashGrokCompletionRequest } from '@clerum/grok-provider-attempt-contract'+import type { GrokLlmProxyConfig } from '../src/config.js'+import type { RedeemAttemptSuccess } from '../src/controlApiClient.js'+import {+ type StreamGrokCompletionInput,+ streamGrokCompletion,+} from '../src/grokTransport.js'+import { GROK_COMPLETIONS_ORIGIN } from '../src/originPolicy.js'+import { streamGate, visualStreamGate } from '../src/requestLimits.js'+import { type ProxyRuntimeDeps, createProxyApps } from '../src/server.js'++// A live stream keeps the handler frame, the transport frame and the input+// object for as long as the upstream answers (V8 keeps the locals of a+// suspended async function). If any of them still names the parsed request+// after the upstream accepted the body, one body-sized tree stays resident per+// live stream, and the 8-wide stream gate multiplies it. These tests hold the+// tree only through a WeakRef and require a forced GC to collect it.++// The transport streams from the contract's parsed copy, not from the object+// the caller passed, so a WeakRef on the caller's tree cannot see a holder of+// the copy. This wrapper records the copy's tools array as each parse returns.+const parsedCopies = vi.hoisted(() => ({ refs: [] as WeakRef<object>[] }))++vi.mock('@clerum/grok-provider-attempt-contract', async importOriginal => {+ const actual = await importOriginal<typeof import('@clerum/grok-provider-attempt-contract')>()+ return {+ ...actual,+ parseGrokCompletionRequest: (input: unknown) => {+ const result = actual.parseGrokCompletionRequest(input)+ if (result.ok && Array.isArray(result.value.tools)) {+ parsedCopies.refs.push(new WeakRef(result.value.tools))+ }+ return result+ },+ }+})++setFlagsFromString('--expose-gc')+const collect = runInNewContext('gc') as () => void++async function collectedWithin(ref: WeakRef<object>): Promise<boolean> {+ for (let attempt = 0; attempt < 10; attempt += 1) {+ await new Promise(resolve => setTimeout(resolve, 10))+ collect()+ if (ref.deref() === undefined) return true+ }+ return false+}++const { privateKey, publicKey } = generateKeyPairSync('rsa', {+ modulusLength: 2048,+ publicKeyEncoding: { type: 'spki', format: 'pem' },+ privateKeyEncoding: { type: 'pkcs8', format: 'pem' },+})++function sign(payload: Record<string, unknown>, audience: string): string {+ return jwt.sign(payload, privateKey, {+ algorithm: 'RS256',+ issuer: 'control-api',+ audience,+ expiresIn: 60,+ })+}++function config(): GrokLlmProxyConfig {+ return {+ runtimePort: 8080,+ adminPort: 8081,+ probePort: 9090,+ maxBodyBytes: 1_048_576,+ maxVisualBodyBytes: 35 * 1024 * 1024,+ maxStreamDurationMs: 1_800_000,+ maxDeadlineMs: 1_800_000,+ upstreamIdleTimeoutMs: 600_000,+ heartbeatIntervalMs: 15_000,+ jwtIssuer: 'control-api',+ jwtPublicKey: publicKey,+ executionEnabled: true,+ controlApiBaseUrl: '',+ controlApiServiceName: 'grok-llm-proxy',+ controlApiServiceToken: '',+ }+}++const MODEL = 'grok-4.6'+const REQUEST = {+ schemaVersion: 'grok-completion-request.v1' as const,+ requestId: 'req-release-001',+ idempotencyKey: 'idem-release-001',+ provider: 'grok-subscription' as const,+ model: MODEL,+ messages: [{ role: 'user' as const, content: 'hello' }],+ tools: [+ {+ name: 'release__probe',+ description: 'probe',+ parameters: { type: 'object', properties: { q: { type: 'string' } } },+ },+ ],+}++function redeemSuccess(): RedeemAttemptSuccess {+ const encoded = Buffer.from(+ JSON.stringify({+ sub: 'live',+ 'https://api.x.ai/auth': { grok_account_id: 'acct_live_1' },+ })+ ).toString('base64url')+ return {+ accessToken: `hdr.${encoded}.sig`,+ transport: {+ protocolVersion: 'grok-subscription-transport.v1',+ completionsOrigin: GROK_COMPLETIONS_ORIGIN,+ catalogOrigin: 'https://cli-chat-proxy.grok.com/v1/models',+ operation: 'completion_stream',+ servedModel: MODEL,+ maxStreamDurationMs: 1_800_000,+ },+ expiryClass: 'short_lived',+ attemptReceipt: 'a'.repeat(64),+ }+}++/**+ * Builds the input in its own frame, so the test's async frame never holds the+ * tree: only `input.request` and the WeakRef name it.+ */+function transportInput(witness: {+ fetchReached: boolean+ collectedInFetch: boolean+ parsedCollectedInFetch: boolean+}): {+ input: StreamGrokCompletionInput+ ref: WeakRef<object>+} {+ const requestHash = hashGrokCompletionRequest(REQUEST)+ const tree = JSON.parse(JSON.stringify(REQUEST)) as typeof REQUEST+ // The tools array is what the contract parse and the upstream payload read.+ const ref = new WeakRef(tree.tools)+ const input: StreamGrokCompletionInput = {+ executionTicket: 'ticket-release',+ requestHash,+ request: tree,+ ticket: {+ jti: 'jti-release',+ hostRef: 'research-host',+ model: MODEL,+ requestHash,+ providerAttemptId: 'att-release',+ },+ maxDeadlineMs: 300_000,+ redeem: async () => redeemSuccess(),+ finalize: async () => ({+ providerAttemptId: 'att-release',+ outcome: 'success' as const,+ duplicate: false,+ }),+ fetchFn: (async () => {+ // Runs after the upstream body was built; the stream is not read yet.+ witness.fetchReached = true+ witness.collectedInFetch = await collectedWithin(ref)+ witness.parsedCollectedInFetch = await collectedWithin(parsedCopies.refs[0])+ return new Response('data: {"type":"response.completed"}\n\n', {+ status: 200,+ headers: { 'content-type': 'text/event-stream' },+ })+ }) as typeof fetch,+ lookup: async () => [{ address: '104.18.32.47', family: 4 }],+ }+ return { input, ref }+}++describe('the parsed request is released once the upstream body is built', () => {+ it('the transport keeps no reference to the tree while it waits on the upstream', async () => {+ parsedCopies.refs.length = 0+ const witness = { fetchReached: false, collectedInFetch: false, parsedCollectedInFetch: false }+ const { input, ref } = transportInput(witness)+ const pending = streamGrokCompletion(input)+ // The server drops its own reference at acceptance; the transport's own+ // holders are what this test measures, so the input's is dropped here.+ input.request = undefined+ // Liveness witnesses: the parse ran and produced its own copy, and that+ // copy was alive until the collector could run.+ expect(parsedCopies.refs).toHaveLength(1)+ expect(parsedCopies.refs[0].deref()).toBeDefined()+ expect(ref.deref()).toBeDefined()+ await pending+ // Liveness witness: the fetch was reached, so the body had been built.+ expect(witness.fetchReached).toBe(true)+ expect(witness.collectedInFetch).toBe(true)+ expect(witness.parsedCollectedInFetch).toBe(true)+ })+})++describe('the handler releases the parsed request at onUpstreamAccepted', () => {+ const serversToClose: Array<{ close: () => Promise<void> }> = []+ const listeners: Array<ReturnType<typeof createServer>> = []++ afterEach(async () => {+ vi.restoreAllMocks()+ for (const server of serversToClose.splice(0)) await server.close()+ await Promise.all(+ listeners.splice(0).map(+ listener =>+ new Promise<void>((resolve, reject) =>+ listener.close(err => (err ? reject(err) : resolve()))+ )+ )+ )+ expect(visualStreamGate.snapshot()).toMatchObject({ running: 0, queued: 0 })+ expect(streamGate.snapshot()).toMatchObject({ running: 0, queued: 0 })+ })++ it('leaves the tree collectable while the stream stays open', async () => {+ const witness: {+ entered: boolean+ measured: boolean+ aliveBeforeAccept: boolean+ collectedAfterAccept: boolean+ } = { entered: false, measured: false, aliveBeforeAccept: false, collectedAfterAccept: false }+ let finish: () => void = () => undefined+ const stream: NonNullable<ProxyRuntimeDeps['streamCompletion']> = async input => {+ witness.entered = true+ // The tools array is shared by the zod copy the stream receives and by the+ // body parser's original, so it is alive while either one is referenced.+ const ref = new WeakRef((input.request as { tools: object }).tools)+ witness.aliveBeforeAccept = ref.deref() !== undefined+ input.onUpstreamAccepted?.()+ witness.collectedAfterAccept = await collectedWithin(ref)+ const held = new Promise<void>(resolve => {+ finish = resolve+ })+ witness.measured = true+ await held+ return { outcome: 'canceled' as const }+ }+ const servers = createProxyApps(config(), { streamCompletion: stream })+ serversToClose.push(servers)+ const listener = createServer(servers.runtimeApp).listen(0)+ listeners.push(listener)+ const address = listener.address()+ if (!address || typeof address === 'string') throw new Error('listener has no port')++ const ticket = sign(+ {+ jti: '22222222-2222-4222-8222-000000000001',+ typ: 'grok-execution-ticket',+ hostRef: 'research-host',+ model: MODEL,+ requestHash: 'a'.repeat(64),+ providerAttemptId: 'att-release-server',+ },+ 'grok-llm-proxy'+ )+ const platform = sign(+ {+ sub: 'default/research-host',+ hostRefs: ['research-host'],+ workflowControlScopes: ['llm:grok:execute'],+ scope: 'workflow:approval:request',+ },+ 'workflow-approvals'+ )+ const response = fetch(`http://127.0.0.1:${address.port}/internal/runtime/v1/grok/completions`, {+ method: 'POST',+ headers: { Authorization: `Bearer ${platform}`, 'content-type': 'application/json' },+ body: JSON.stringify({ executionTicket: ticket, requestHash: 'a'.repeat(64), request: REQUEST }),+ })+ const started = Date.now()+ while (!witness.measured) {+ if (Date.now() - started > 5_000) throw new Error('the stream seam was not reached')+ await new Promise(resolve => setTimeout(resolve, 10))+ }+ try {+ // Liveness witnesses: the seam was entered, the tree was alive when the+ // stub took its reference, and the acceptance hook ran.+ expect(witness.entered).toBe(true)+ expect(witness.aliveBeforeAccept).toBe(true)+ expect(witness.collectedAfterAccept).toBe(true)+ } finally {+ finish()+ }+ expect((await response).status).toBe(200)+ })+})diff --git a/scripts/tests/test-codex-subscription-t0.sh b/scripts/tests/test-codex-subscription-t0.shindex 00791989f..3eb98357a 100755--- a/scripts/tests/test-codex-subscription-t0.sh+++ b/scripts/tests/test-codex-subscription-t0.sh@@ -293,6 +293,7 @@ run_group "codex-llm-proxy" "codex-llm-proxy" \"test/metrics.test.ts" \"test/originPolicy.test.ts" \"test/redaction.test.ts" \+ "test/releaseAtAcceptance.test.ts" \"test/requestLimits.test.ts" \"test/runtimePath.hermetic.e2e.test.ts" \"test/server.security.test.ts" \diff --git a/scripts/tests/test-grok-subscription-t0.sh b/scripts/tests/test-grok-subscription-t0.shindex 165d83169..08a51666d 100755--- a/scripts/tests/test-grok-subscription-t0.sh+++ b/scripts/tests/test-grok-subscription-t0.sh@@ -245,6 +245,7 @@ run_group "grok-llm-proxy" "grok-llm-proxy" \"test/metrics.test.ts" \"test/originPolicy.test.ts" \"test/redaction.test.ts" \+ "test/releaseAtAcceptance.test.ts" \"test/requestLimits.test.ts" \"test/runtimePath.hermetic.e2e.test.ts" \"test/server.security.test.ts" \