Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
fix(llm-proxy): release the body budget on upstream acceptance, keep the close backstop (#739)
Alfredo Lopez6a8a2514f5cb
diff --git a/codex-llm-proxy/src/codexTransport.ts b/codex-llm-proxy/src/codexTransport.tsindex ae31219a1..54d562923 100644--- a/codex-llm-proxy/src/codexTransport.ts+++ b/codex-llm-proxy/src/codexTransport.ts@@ -89,6 +89,13 @@ export type StreamCodexCompletionInput = {* heartbeat here, so a denied redeem still answers with an HTTP status.*/onRedeemed?: () => void+ /**+ * #739 D2 — called once, as soon as the upstream fetch resolved and before+ * its status is inspected: the whole request body has been written by then,+ * whatever the answer. The server releases the body's budget reservation+ * here. Not called when the fetch rejects or never resolves.+ */+ onUpstreamAccepted?: () => voidfinalize: (input: {attemptReceipt: stringreceipt: {@@ -174,6 +181,7 @@ export async function streamCodexCompletion(fetchFn: input.fetchFn,lookup: input.lookup,onFrame: input.onFrame,+ onUpstreamAccepted: input.onUpstreamAccepted,})outcome = streamed.outcomeusage = streamed.usage@@ -244,6 +252,7 @@ async function readUpstreamStream(input: {fetchFn: typeof fetchlookup?: OriginPolicyOptions['lookup']onFrame?: FrameSink+ onUpstreamAccepted?: () => void}): Promise<StreamCodexCompletionResult> {const deadline = new UpstreamDeadline(input.deadlineMs, input.idleTimeoutMs)try {@@ -358,6 +367,7 @@ async function dispatchUpstreamStream(}),signal)+ input.onUpstreamAccepted?.()if (!response.ok || !response.body) {const credentialRejected = response.status === 401 || response.status === 403const errorBody =diff --git a/codex-llm-proxy/src/server.ts b/codex-llm-proxy/src/server.tsindex d810be81a..e2e492358 100644--- a/codex-llm-proxy/src/server.ts+++ b/codex-llm-proxy/src/server.ts@@ -54,6 +54,12 @@ type AdmittedRequest = Request & {* request performs (body budget, visual gate, stream gate) ends by then.*/codexAdmissionDeadlineAt?: number+ /**+ * #739 D2 — the body's budget reservation, set once it was granted. The+ * handler releases it when the upstream accepted the request; the+ * response's `close` event releases it on every other path. Idempotent.+ */+ codexBodyRelease?: () => void}type GatedRequest = AdmittedRequest & {@@ -132,9 +138,11 @@ function boundedErrorHandler(err: unknown, _req: Request, res: Response, _next:* `selectTransportBudget` either answers 413 or admits them through the visual* gate. A granted body must be read and parsed within `readDeadlineMs` of the* grant, or it is answered 408 `request_timeout`, its reservation released and- * its connection closed. Otherwise the reservation is held until the response- * closes, which for a completion is the whole stream (up to- * `maxStreamDurationMs`). A queued waiter is dropped if the client leaves+ * its connection closed. Otherwise the reservation is held until the upstream+ * accepted the completion (#739 D2: the handler releases+ * `req.codexBodyRelease` once the upstream fetch resolved, when the body has+ * been written), or until the response closes on every path that never+ * reaches the upstream. A queued waiter is dropped if the client leaves* first; a full queue gets the stream gate's overload response. #739 D1: the* request's admission clock is stamped here, at arrival, and a waiter still* queued when it runs out gets the same overload response.@@ -181,6 +189,7 @@ function bodyAdmission(return}release = granted+ req.codexBodyRelease = grantedlet expired = falsereadDeadline = setTimeout(() => {expired = true@@ -520,6 +529,14 @@ export function createProxyApps(})abort.signal.addEventListener('abort', stopHeartbeat, { once: true })},+ // #739 D2: the body has been written upstream, so its budget+ // reservation ends here rather than with the stream. The parsed+ // copy the transport streams from stays; the raw one is dropped.+ onUpstreamAccepted: () => {+ gated.codexBodyRelease?.()+ gated.codexBodyRelease = undefined+ req.body = undefined+ },finalize: input => client.finalize(input),fetchFn,lookup,diff --git a/grok-llm-proxy/src/grokTransport.ts b/grok-llm-proxy/src/grokTransport.tsindex 47997499b..346b55651 100644--- a/grok-llm-proxy/src/grokTransport.ts+++ b/grok-llm-proxy/src/grokTransport.ts@@ -116,6 +116,13 @@ export type StreamGrokCompletionInput = {* heartbeat here, so a denied redeem still answers with an HTTP status.*/onRedeemed?: () => void+ /**+ * #739 D2 — called once, as soon as the upstream fetch resolved and before+ * its status is inspected: the whole request body has been written by then,+ * whatever the answer. The server releases the body's budget reservation+ * here. Not called when the fetch rejects or never resolves.+ */+ onUpstreamAccepted?: () => voidfinalize: (input: {attemptReceipt: stringreceipt: {@@ -191,6 +198,7 @@ export async function streamGrokCompletion(fetchFn: input.fetchFn,lookup: input.lookup,onFrame: input.onFrame,+ onUpstreamAccepted: input.onUpstreamAccepted,})outcome = streamed.outcomeusage = streamed.usage@@ -287,6 +295,7 @@ async function readUpstreamStream(input: {fetchFn: typeof fetchlookup?: OriginPolicyOptions['lookup']onFrame?: FrameSink+ onUpstreamAccepted?: () => void}): Promise<StreamGrokCompletionResult> {const deadline = new UpstreamDeadline(input.deadlineMs, input.idleTimeoutMs)try {@@ -393,6 +402,7 @@ async function dispatchUpstreamStream(}),signal)+ input.onUpstreamAccepted?.()if (!response.ok || !response.body) {const errorBody = await readUpstreamErrorBody(response)logger.warn(diff --git a/grok-llm-proxy/src/server.ts b/grok-llm-proxy/src/server.tsindex 7d28ca8fe..1a80aad2f 100644--- a/grok-llm-proxy/src/server.ts+++ b/grok-llm-proxy/src/server.ts@@ -46,6 +46,12 @@ type AdmittedRequest = Request & {* request performs (body budget, stream gate) ends by then.*/grokAdmissionDeadlineAt?: number+ /**+ * #739 D2 — the body's budget reservation, set once it was granted. The+ * handler releases it when the upstream accepted the request; the+ * response's `close` event releases it on every other path. Idempotent.+ */+ grokBodyRelease?: () => void}type GatedRequest = AdmittedRequest & {@@ -111,8 +117,10 @@ function boundedErrorHandler(err: unknown, _req: Request, res: Response, _next:* buffering them. A granted body must be read and parsed within* `readDeadlineMs` of the grant, or it is answered 408 `request_timeout`, its* reservation released and its connection closed. Otherwise the reservation is- * held until the response closes, which for a completion is the whole stream- * (up to `maxStreamDurationMs`). A queued waiter is dropped if the client+ * held until the upstream accepted the completion (#739 D2: the handler+ * releases `req.grokBodyRelease` once the upstream fetch resolved, when the+ * body has been written), or until the response closes on every path that+ * never reaches the upstream. A queued waiter is dropped if the client* leaves first; a full queue gets the stream gate's overload response. #739* D1: the request's admission clock is stamped here, at arrival, and a waiter* still queued when it runs out gets the same overload response.@@ -159,6 +167,7 @@ function bodyAdmission(return}release = granted+ req.grokBodyRelease = grantedlet expired = falsereadDeadline = setTimeout(() => {expired = true@@ -394,6 +403,15 @@ export function createProxyApps(})abort.signal.addEventListener('abort', stopHeartbeat, { once: true })},+ // #739 D2: the body has been written upstream, so its budget+ // reservation ends here rather than with the stream. The parsed+ // copy the transport streams from stays; the raw one is dropped.+ onUpstreamAccepted: () => {+ const admitted = req as GatedRequest+ admitted.grokBodyRelease?.()+ admitted.grokBodyRelease = undefined+ req.body = undefined+ },finalize: input => client.finalize(input),fetchFn,lookup,