Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript

fix(llm-proxy): release the body budget on upstream acceptance, keep the close backstop (#739)

Alfredo Lopez6a8a2514f5cb
diff --git a/codex-llm-proxy/src/codexTransport.ts b/codex-llm-proxy/src/codexTransport.ts
index ae31219a1..54d562923 100644
--- a/codex-llm-proxy/src/codexTransport.ts
+++ b/codex-llm-proxy/src/codexTransport.ts
@@ -89,6 +89,13 @@ export type StreamCodexCompletionInput = {
* heartbeat here, so a denied redeem still answers with an HTTP status.
*/
onRedeemed?: () => void
+ /**
+ * #739 D2 — called once, as soon as the upstream fetch resolved and before
+ * its status is inspected: the whole request body has been written by then,
+ * whatever the answer. The server releases the body's budget reservation
+ * here. Not called when the fetch rejects or never resolves.
+ */
+ onUpstreamAccepted?: () => void
finalize: (input: {
attemptReceipt: string
receipt: {
@@ -174,6 +181,7 @@ export async function streamCodexCompletion(
fetchFn: input.fetchFn,
lookup: input.lookup,
onFrame: input.onFrame,
+ onUpstreamAccepted: input.onUpstreamAccepted,
})
outcome = streamed.outcome
usage = streamed.usage
@@ -244,6 +252,7 @@ async function readUpstreamStream(input: {
fetchFn: typeof fetch
lookup?: OriginPolicyOptions['lookup']
onFrame?: FrameSink
+ onUpstreamAccepted?: () => void
}): Promise<StreamCodexCompletionResult> {
const deadline = new UpstreamDeadline(input.deadlineMs, input.idleTimeoutMs)
try {
@@ -358,6 +367,7 @@ async function dispatchUpstreamStream(
}),
signal
)
+ input.onUpstreamAccepted?.()
if (!response.ok || !response.body) {
const credentialRejected = response.status === 401 || response.status === 403
const errorBody =
diff --git a/codex-llm-proxy/src/server.ts b/codex-llm-proxy/src/server.ts
index d810be81a..e2e492358 100644
--- a/codex-llm-proxy/src/server.ts
+++ b/codex-llm-proxy/src/server.ts
@@ -54,6 +54,12 @@ type AdmittedRequest = Request & {
* request performs (body budget, visual gate, stream gate) ends by then.
*/
codexAdmissionDeadlineAt?: number
+ /**
+ * #739 D2 — the body's budget reservation, set once it was granted. The
+ * handler releases it when the upstream accepted the request; the
+ * response's `close` event releases it on every other path. Idempotent.
+ */
+ codexBodyRelease?: () => void
}
type GatedRequest = AdmittedRequest & {
@@ -132,9 +138,11 @@ function boundedErrorHandler(err: unknown, _req: Request, res: Response, _next:
* `selectTransportBudget` either answers 413 or admits them through the visual
* gate. A granted body must be read and parsed within `readDeadlineMs` of the
* grant, or it is answered 408 `request_timeout`, its reservation released and
- * its connection closed. Otherwise the reservation is held until the response
- * closes, which for a completion is the whole stream (up to
- * `maxStreamDurationMs`). A queued waiter is dropped if the client leaves
+ * its connection closed. Otherwise the reservation is held until the upstream
+ * accepted the completion (#739 D2: the handler releases
+ * `req.codexBodyRelease` once the upstream fetch resolved, when the body has
+ * been written), or until the response closes on every path that never
+ * reaches the upstream. A queued waiter is dropped if the client leaves
* first; a full queue gets the stream gate's overload response. #739 D1: the
* request's admission clock is stamped here, at arrival, and a waiter still
* queued when it runs out gets the same overload response.
@@ -181,6 +189,7 @@ function bodyAdmission(
return
}
release = granted
+ req.codexBodyRelease = granted
let expired = false
readDeadline = setTimeout(() => {
expired = true
@@ -520,6 +529,14 @@ export function createProxyApps(
})
abort.signal.addEventListener('abort', stopHeartbeat, { once: true })
},
+ // #739 D2: the body has been written upstream, so its budget
+ // reservation ends here rather than with the stream. The parsed
+ // copy the transport streams from stays; the raw one is dropped.
+ onUpstreamAccepted: () => {
+ gated.codexBodyRelease?.()
+ gated.codexBodyRelease = undefined
+ req.body = undefined
+ },
finalize: input => client.finalize(input),
fetchFn,
lookup,
diff --git a/grok-llm-proxy/src/grokTransport.ts b/grok-llm-proxy/src/grokTransport.ts
index 47997499b..346b55651 100644
--- a/grok-llm-proxy/src/grokTransport.ts
+++ b/grok-llm-proxy/src/grokTransport.ts
@@ -116,6 +116,13 @@ export type StreamGrokCompletionInput = {
* heartbeat here, so a denied redeem still answers with an HTTP status.
*/
onRedeemed?: () => void
+ /**
+ * #739 D2 — called once, as soon as the upstream fetch resolved and before
+ * its status is inspected: the whole request body has been written by then,
+ * whatever the answer. The server releases the body's budget reservation
+ * here. Not called when the fetch rejects or never resolves.
+ */
+ onUpstreamAccepted?: () => void
finalize: (input: {
attemptReceipt: string
receipt: {
@@ -191,6 +198,7 @@ export async function streamGrokCompletion(
fetchFn: input.fetchFn,
lookup: input.lookup,
onFrame: input.onFrame,
+ onUpstreamAccepted: input.onUpstreamAccepted,
})
outcome = streamed.outcome
usage = streamed.usage
@@ -287,6 +295,7 @@ async function readUpstreamStream(input: {
fetchFn: typeof fetch
lookup?: OriginPolicyOptions['lookup']
onFrame?: FrameSink
+ onUpstreamAccepted?: () => void
}): Promise<StreamGrokCompletionResult> {
const deadline = new UpstreamDeadline(input.deadlineMs, input.idleTimeoutMs)
try {
@@ -393,6 +402,7 @@ async function dispatchUpstreamStream(
}),
signal
)
+ input.onUpstreamAccepted?.()
if (!response.ok || !response.body) {
const errorBody = await readUpstreamErrorBody(response)
logger.warn(
diff --git a/grok-llm-proxy/src/server.ts b/grok-llm-proxy/src/server.ts
index 7d28ca8fe..1a80aad2f 100644
--- a/grok-llm-proxy/src/server.ts
+++ b/grok-llm-proxy/src/server.ts
@@ -46,6 +46,12 @@ type AdmittedRequest = Request & {
* request performs (body budget, stream gate) ends by then.
*/
grokAdmissionDeadlineAt?: number
+ /**
+ * #739 D2 — the body's budget reservation, set once it was granted. The
+ * handler releases it when the upstream accepted the request; the
+ * response's `close` event releases it on every other path. Idempotent.
+ */
+ grokBodyRelease?: () => void
}
type GatedRequest = AdmittedRequest & {
@@ -111,8 +117,10 @@ function boundedErrorHandler(err: unknown, _req: Request, res: Response, _next:
* buffering them. A granted body must be read and parsed within
* `readDeadlineMs` of the grant, or it is answered 408 `request_timeout`, its
* reservation released and its connection closed. Otherwise the reservation is
- * held until the response closes, which for a completion is the whole stream
- * (up to `maxStreamDurationMs`). A queued waiter is dropped if the client
+ * held until the upstream accepted the completion (#739 D2: the handler
+ * releases `req.grokBodyRelease` once the upstream fetch resolved, when the
+ * body has been written), or until the response closes on every path that
+ * never reaches the upstream. A queued waiter is dropped if the client
* leaves first; a full queue gets the stream gate's overload response. #739
* D1: the request's admission clock is stamped here, at arrival, and a waiter
* still queued when it runs out gets the same overload response.
@@ -159,6 +167,7 @@ function bodyAdmission(
return
}
release = granted
+ req.grokBodyRelease = granted
let expired = false
readDeadline = setTimeout(() => {
expired = true
@@ -394,6 +403,15 @@ export function createProxyApps(
})
abort.signal.addEventListener('abort', stopHeartbeat, { once: true })
},
+ // #739 D2: the body has been written upstream, so its budget
+ // reservation ends here rather than with the stream. The parsed
+ // copy the transport streams from stays; the raw one is dropped.
+ onUpstreamAccepted: () => {
+ const admitted = req as GatedRequest
+ admitted.grokBodyRelease?.()
+ admitted.grokBodyRelease = undefined
+ req.body = undefined
+ },
finalize: input => client.finalize(input),
fetchFn,
lookup,