Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript

fix: settle and classify every oauth-broker provider attempt

Alfredo Lopez6b9722341cf1
diff --git a/mcp-host/package-lock.json b/mcp-host/package-lock.json
index 9aced9b84..b7eb7f138 100644
--- a/mcp-host/package-lock.json
+++ b/mcp-host/package-lock.json
@@ -1,12 +1,12 @@
{
"name": "clerum-mcp-host",
- "version": "0.1.310",
+ "version": "0.1.311",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "clerum-mcp-host",
- "version": "0.1.310",
+ "version": "0.1.311",
"license": "SEE LICENSE IN LICENSE",
"dependencies": {
"@anthropic-ai/sdk": "^0.32.0",
diff --git a/mcp-host/package.json b/mcp-host/package.json
index 548b1b092..c9c428555 100644
--- a/mcp-host/package.json
+++ b/mcp-host/package.json
@@ -1,6 +1,6 @@
{
"name": "clerum-mcp-host",
- "version": "0.1.310",
+ "version": "0.1.311",
"description": "MCP Host - reads Host CRD and provides LLM access via OpenAI/Claude",
"main": "dist/main.js",
"engines": {
diff --git a/mcp-host/src/llm/__tests__/codexLlmProxyClient.test.ts b/mcp-host/src/llm/__tests__/codexLlmProxyClient.test.ts
index c6a5548ca..583455635 100644
--- a/mcp-host/src/llm/__tests__/codexLlmProxyClient.test.ts
+++ b/mcp-host/src/llm/__tests__/codexLlmProxyClient.test.ts
@@ -144,4 +144,60 @@ describe('CodexLlmProxyClient', () => {
})
).rejects.toMatchObject({ code: 'origin_denied' })
})
+
+ it('marks only the pre-stream abort as never dispatched', async () => {
+ const runtimeUrl = resolveCodexProxyRuntimeUrl(
+ 'http://codex-llm-proxy.control-plane.svc.cluster.local:8080'
+ )
+
+ const abortedClient = new CodexLlmProxyClient({
+ runtimeUrl,
+ readPlatformJwt: () => 'platform-jwt',
+ fetchFn: vi.fn() as unknown as typeof fetch,
+ })
+ await expect(
+ abortedClient.stream({
+ executionTicket: 'ticket-123456',
+ requestHash: 'a'.repeat(64),
+ request: {},
+ signal: AbortSignal.abort(),
+ })
+ ).rejects.toMatchObject({ code: 'canceled', dispatched: false })
+
+ // The request left the process before control-api answered, so a denial is
+ // not proof that nothing was billed.
+ const deniedClient = new CodexLlmProxyClient({
+ runtimeUrl,
+ readPlatformJwt: () => 'platform-jwt',
+ fetchFn: vi.fn().mockResolvedValue({
+ ok: false,
+ status: 403,
+ json: async () => ({ error: 'origin_denied' }),
+ }) as unknown as typeof fetch,
+ })
+ await expect(
+ deniedClient.stream({
+ executionTicket: 'ticket-123456',
+ requestHash: 'a'.repeat(64),
+ request: {},
+ })
+ ).rejects.toMatchObject({ code: 'origin_denied', dispatched: true })
+
+ // An error frame arrives mid-stream: the upstream call is already running.
+ const framedClient = new CodexLlmProxyClient({
+ runtimeUrl,
+ readPlatformJwt: () => 'platform-jwt',
+ fetchFn: vi.fn().mockResolvedValue({
+ ok: true,
+ body: sse([{ type: 'error', code: 'rate_limited' }]),
+ }) as unknown as typeof fetch,
+ })
+ await expect(
+ framedClient.stream({
+ executionTicket: 'ticket-123456',
+ requestHash: 'a'.repeat(64),
+ request: {},
+ })
+ ).rejects.toMatchObject({ code: 'rate_limited', dispatched: true })
+ })
})
diff --git a/mcp-host/src/llm/__tests__/codexSubscription.test.ts b/mcp-host/src/llm/__tests__/codexSubscription.test.ts
index fef628868..3bf79c795 100644
--- a/mcp-host/src/llm/__tests__/codexSubscription.test.ts
+++ b/mcp-host/src/llm/__tests__/codexSubscription.test.ts
@@ -279,4 +279,55 @@ describe('CodexSubscriptionProvider', () => {
expect(limited.retryable).toBe(true)
expect(classifyFailoverClass(limited.code, limited.retryable)).toBe('rate_limited')
})
+
+ it('records whether a classified Codex failure had already left the process', () => {
+ const provider = new CodexSubscriptionProvider('gpt-5.3-codex', deps() as never)
+
+ // Every CodexAuthorizeError throw site precedes `proxy.stream`.
+ expect(
+ provider.classifyError(new CodexAuthorizeError('no_grant', 'revoked')).providerDispatched
+ ).toBe(false)
+ expect(
+ provider.classifyError(new CodexAuthorizeError('budget_denied', 'over budget'))
+ .providerDispatched
+ ).toBe(false)
+ expect(
+ provider.classifyError(new CodexAuthorizeError('insufficient_scope', 'scope'))
+ .providerDispatched
+ ).toBe(false)
+
+ // The two pre-stream aborts are the only proxy errors that prove it.
+ expect(
+ provider.classifyError(new CodexProxyError('canceled', 'aborted before authorize', false))
+ .providerDispatched
+ ).toBe(false)
+
+ // Anything else on the proxy happened after the request was issued.
+ expect(
+ provider.classifyError(new CodexProxyError('provider_unavailable', 'upstream 5xx'))
+ .providerDispatched
+ ).toBe(true)
+ expect(
+ provider.classifyError(new CodexProxyError('rate_limited', 'upstream 429')).providerDispatched
+ ).toBe(true)
+
+ // An unrecognized error is not evidence that no call was made.
+ expect(provider.classifyError(new Error('who knows')).providerDispatched).toBeUndefined()
+ })
+
+ it('shares the caller deadline with the authorize hop', async () => {
+ const wired = deps()
+ const provider = new CodexSubscriptionProvider('gpt-5.3-codex', wired as never)
+ const controller = new AbortController()
+
+ await provider.completeSingleTurn([{ role: 'user', content: 'hi' }], {
+ signal: controller.signal,
+ })
+ expect(wired.authorize).toHaveBeenCalledWith(expect.any(Object), {
+ signal: controller.signal,
+ })
+
+ await provider.completeSingleTurn([{ role: 'user', content: 'hi' }])
+ expect(wired.authorize).toHaveBeenNthCalledWith(2, expect.any(Object), {})
+ })
})
diff --git a/mcp-host/src/llm/__tests__/providerAttemptAuthorizer.test.ts b/mcp-host/src/llm/__tests__/providerAttemptAuthorizer.test.ts
index 53f63847d..bf699138d 100644
--- a/mcp-host/src/llm/__tests__/providerAttemptAuthorizer.test.ts
+++ b/mcp-host/src/llm/__tests__/providerAttemptAuthorizer.test.ts
@@ -131,4 +131,70 @@ describe('ProviderAttemptAuthorizer', () => {
expect(fetchFn.mock.calls[1][1].headers.authorization).toBe('Bearer fresh-jwt')
expect(result.executionTicket).toBe('ticket-123456')
})
+
+ it('forwards the caller deadline to the authorize hop and to its post-401 retry', async () => {
+ const controller = new AbortController()
+ const refreshOnUnauthorized = vi.fn().mockResolvedValue(undefined)
+ const fetchFn = vi
+ .fn()
+ .mockResolvedValueOnce({
+ ok: false,
+ status: 401,
+ json: async () => ({ error: 'Unauthorized' }),
+ })
+ .mockResolvedValueOnce({ ok: true, json: async () => validAuthorize })
+ const authorizer = new ProviderAttemptAuthorizer({
+ authorizeUrl: resolveCodexAuthorizeUrl('http://gateway:8092'),
+ readPlatformJwt: () => 'platform-jwt',
+ refreshOnUnauthorized,
+ fetchFn: fetchFn as unknown as typeof fetch,
+ })
+
+ await authorizer.authorize(
+ {
+ request: { schemaVersion: 'codex-completion-request.v1' },
+ invocationId: 'inv-1',
+ attemptGeneration: 1,
+ providerAttemptIndex: 1,
+ policyRevision: 1,
+ policyHash: 'b'.repeat(64),
+ },
+ { signal: controller.signal }
+ )
+ // The retry after a refresh must not outlive the deadline that bounded the
+ // first hop — a second, unbounded call would be a second clock.
+ expect(fetchFn.mock.calls[0][1].signal).toBe(controller.signal)
+ expect(fetchFn.mock.calls[1][1].signal).toBe(controller.signal)
+ })
+
+ it('fails loudly on an already-expired deadline without rotating credentials', async () => {
+ const refreshOnUnauthorized = vi.fn()
+ // Faithful to real fetch: an aborted signal rejects before any response.
+ const fetchFn = vi.fn(async (_url: string, init: RequestInit) => {
+ if (init.signal?.aborted) throw new DOMException('operation aborted', 'AbortError')
+ throw new Error('fetch must not proceed with an aborted signal')
+ })
+ const authorizer = new ProviderAttemptAuthorizer({
+ authorizeUrl: resolveCodexAuthorizeUrl('http://gateway:8092'),
+ readPlatformJwt: () => 'platform-jwt',
+ refreshOnUnauthorized,
+ fetchFn: fetchFn as unknown as typeof fetch,
+ })
+
+ await expect(
+ authorizer.authorize(
+ {
+ request: {},
+ invocationId: 'inv-1',
+ attemptGeneration: 1,
+ providerAttemptIndex: 1,
+ policyRevision: 1,
+ policyHash: 'b'.repeat(64),
+ },
+ { signal: AbortSignal.abort() }
+ )
+ ).rejects.toThrow(/aborted/i)
+ expect(fetchFn).toHaveBeenCalledTimes(1)
+ expect(refreshOnUnauthorized).not.toHaveBeenCalled()
+ })
})
diff --git a/mcp-host/src/llm/codexLlmProxyClient.ts b/mcp-host/src/llm/codexLlmProxyClient.ts
index 940a174a4..91a5e5aa6 100644
--- a/mcp-host/src/llm/codexLlmProxyClient.ts
+++ b/mcp-host/src/llm/codexLlmProxyClient.ts
@@ -1,9 +1,16 @@
export const CODEX_PROXY_COMPLETIONS_PATH = '/internal/runtime/v1/codex/completions'
export class CodexProxyError extends Error {
+ /**
+ * `dispatched` records whether a request had already left this process when
+ * the error was raised. It defaults to `true` because every construction
+ * site except the pre-stream abort happens after the fetch was issued, and
+ * the safe default is the one that keeps the attempt fenced.
+ */
constructor(
readonly code: string,
- message: string
+ message: string,
+ readonly dispatched: boolean = true
) {
super(message)
this.name = 'CodexProxyError'
@@ -50,7 +57,7 @@ export class CodexLlmProxyClient {
signal?: AbortSignal
}): Promise<CodexProxyStreamResult> {
if (input.signal?.aborted) {
- throw new CodexProxyError('canceled', 'aborted before proxy stream')
+ throw new CodexProxyError('canceled', 'aborted before proxy stream', false)
}
return this.streamOnce(input, Boolean(this.options.refreshOnUnauthorized))
}
diff --git a/mcp-host/src/llm/codexSubscription.ts b/mcp-host/src/llm/codexSubscription.ts
index d1b2cbc4b..e6c6a81ba 100644
--- a/mcp-host/src/llm/codexSubscription.ts
+++ b/mcp-host/src/llm/codexSubscription.ts
@@ -115,6 +115,16 @@ export class CodexSubscriptionProvider implements SingleTurnProvider {
classifyError(err: unknown): ClassifiedError {
const code =
err instanceof CodexAuthorizeError || err instanceof CodexProxyError ? err.code : undefined
+ // Every CodexAuthorizeError throw site precedes `proxy.stream`, so an
+ // authorize failure proves nothing was dispatched. A CodexProxyError knows
+ // whether its own request had left the process. Anything else stays
+ // undefined — an unrecognized error is not evidence of a missing call.
+ const providerDispatched =
+ err instanceof CodexAuthorizeError
+ ? false
+ : err instanceof CodexProxyError
+ ? err.dispatched
+ : undefined
if (
code === 'insufficient_scope' ||
code === 'no_grant' ||
@@ -126,6 +136,7 @@ export class CodexSubscriptionProvider implements SingleTurnProvider {
retryable: false,
message: err instanceof Error ? err.message : String(err),
providerCode: code,
+ ...(providerDispatched !== undefined ? { providerDispatched } : {}),
}
}
if (code === 'budget_denied') {
@@ -134,6 +145,7 @@ export class CodexSubscriptionProvider implements SingleTurnProvider {
retryable: false,
message: err instanceof Error ? err.message : String(err),
providerCode: code,
+ ...(providerDispatched !== undefined ? { providerDispatched } : {}),
}
}
if (code === 'model_not_allowed') {
@@ -142,6 +154,7 @@ export class CodexSubscriptionProvider implements SingleTurnProvider {
retryable: false,
message: err instanceof Error ? err.message : String(err),
providerCode: code,
+ ...(providerDispatched !== undefined ? { providerDispatched } : {}),
}
}
if (code === 'rate_limited') {
@@ -150,6 +163,7 @@ export class CodexSubscriptionProvider implements SingleTurnProvider {
retryable: true,
message: err instanceof Error ? err.message : String(err),
providerCode: code,
+ ...(providerDispatched !== undefined ? { providerDispatched } : {}),
}
}
if (code === 'provider_unavailable' || code === 'connection_unavailable') {
@@ -158,6 +172,7 @@ export class CodexSubscriptionProvider implements SingleTurnProvider {
retryable: true,
message: err instanceof Error ? err.message : String(err),
providerCode: code,
+ ...(providerDispatched !== undefined ? { providerDispatched } : {}),
}
}
if (code) {
@@ -166,6 +181,7 @@ export class CodexSubscriptionProvider implements SingleTurnProvider {
retryable: false,
message: err instanceof Error ? err.message : String(err),
providerCode: code,
+ ...(providerDispatched !== undefined ? { providerDispatched } : {}),
}
}
return classifyUnknown(err)
@@ -236,7 +252,7 @@ export class CodexSubscriptionProvider implements SingleTurnProvider {
}
) {
if (options?.signal?.aborted) {
- throw new CodexProxyError('canceled', 'aborted before authorize')
+ throw new CodexProxyError('canceled', 'aborted before authorize', false)
}
const request = this.buildRequest(messages, tools, options)
const requestHash = hashCodexCompletionRequestV1(request)
@@ -249,23 +265,29 @@ export class CodexSubscriptionProvider implements SingleTurnProvider {
throw new CodexAuthorizeError('no_grant', 'Codex catalog policy binding is missing')
}
const providerAttemptIndex = context.providerAttemptIndex ?? this.nextProviderAttemptIndex++
- const authorized = await this.deps.authorizer.authorize({
- request,
- requestHash,
- invocationId: context.invocationId ?? request.requestId,
- attemptGeneration: context.attemptGeneration ?? 1,
- providerAttemptIndex,
- policyRevision: context.policyRevision,
- policyHash: context.policyHash,
- hostRef: context.hostRef,
- recipeNamespace: context.recipeNamespace,
- recipeName: context.recipeName,
- userId: context.userId,
- ...(context.pluginWorkloadSdkProviderAttemptId
- ? { pluginWorkloadSdkProviderAttemptId: context.pluginWorkloadSdkProviderAttemptId }
- : {}),
- ...(context.targetRef ? { targetRef: context.targetRef } : {}),
- })
+ const authorized = await this.deps.authorizer.authorize(
+ {
+ request,
+ requestHash,
+ invocationId: context.invocationId ?? request.requestId,
+ attemptGeneration: context.attemptGeneration ?? 1,
+ providerAttemptIndex,
+ policyRevision: context.policyRevision,
+ policyHash: context.policyHash,
+ hostRef: context.hostRef,
+ recipeNamespace: context.recipeNamespace,
+ recipeName: context.recipeName,
+ userId: context.userId,
+ ...(context.pluginWorkloadSdkProviderAttemptId
+ ? { pluginWorkloadSdkProviderAttemptId: context.pluginWorkloadSdkProviderAttemptId }
+ : {}),
+ ...(context.targetRef ? { targetRef: context.targetRef } : {}),
+ },
+ // Authorize shares the caller's deadline. Without this the attempt could
+ // keep a cancelled request alive on the control-api hop while the bridge
+ // has already given up on it.
+ { ...(options?.signal ? { signal: options.signal } : {}) }
+ )
if (!('accessToken' in authorized)) {
// Bound: authorize returns ticket material only.
}
diff --git a/mcp-host/src/llm/failover/engine.ts b/mcp-host/src/llm/failover/engine.ts
index 68857b632..a091f8484 100644
--- a/mcp-host/src/llm/failover/engine.ts
+++ b/mcp-host/src/llm/failover/engine.ts
@@ -42,6 +42,12 @@ export interface ClassifiedLike {
retryable: boolean
/** Provider-native code (e.g. `budget_denied`) used for terminal gates. */
providerCode?: string
+ /**
+ * Mirrors `ClassifiedError.providerDispatched`: `false` only when the
+ * classifier proved no call left the process. `undefined` is unknown and
+ * must be read as dispatched.
+ */
+ providerDispatched?: boolean
}
export interface FailoverEngineOptions {
diff --git a/mcp-host/src/llm/providerAttemptAuthorizer.ts b/mcp-host/src/llm/providerAttemptAuthorizer.ts
index a47b8c6ee..0fd18520b 100644
--- a/mcp-host/src/llm/providerAttemptAuthorizer.ts
+++ b/mcp-host/src/llm/providerAttemptAuthorizer.ts
@@ -48,18 +48,28 @@ export class ProviderAttemptAuthorizer {
}
}
- async authorize(body: AuthorizeAttemptBody): Promise<{
+ /**
+ * The caller's `signal` is the only clock on this hop. Authorize deliberately
+ * has no timeout of its own: the bridge already bounds the whole attempt, and
+ * a second independent deadline would make two different answers possible for
+ * "did this attempt still have time?".
+ */
+ async authorize(
+ body: AuthorizeAttemptBody,
+ options?: { signal?: AbortSignal }
+ ): Promise<{
providerAttemptId: string
requestHash: string
executionTicket: string
expiresAt: string
}> {
- return this.authorizeOnce(body, Boolean(this.options.refreshOnUnauthorized))
+ return this.authorizeOnce(body, Boolean(this.options.refreshOnUnauthorized), options?.signal)
}
private async authorizeOnce(
body: AuthorizeAttemptBody,
- retryOnUnauthorized: boolean
+ retryOnUnauthorized: boolean,
+ signal?: AbortSignal
): Promise<{
providerAttemptId: string
requestHash: string
@@ -78,12 +88,13 @@ export class ProviderAttemptAuthorizer {
'content-type': 'application/json',
},
body: JSON.stringify(body),
+ ...(signal ? { signal } : {}),
})
const payload = (await response.json().catch(() => ({}))) as Record<string, unknown>
if (!response.ok) {
if (response.status === 401 && retryOnUnauthorized && this.options.refreshOnUnauthorized) {
await this.options.refreshOnUnauthorized()
- return this.authorizeOnce(body, false)
+ return this.authorizeOnce(body, false, signal)
}
const code = typeof payload.error === 'string' ? payload.error : 'provider_unavailable'
throw new CodexAuthorizeError(code, `authorize failed with ${response.status}`)
diff --git a/mcp-host/src/llm/types.ts b/mcp-host/src/llm/types.ts
index 1766381b4..4afd896fb 100644
--- a/mcp-host/src/llm/types.ts
+++ b/mcp-host/src/llm/types.ts
@@ -45,6 +45,18 @@ export interface ClassifiedError {
*/
httpStatus?: number
providerCode?: string
+ /**
+ * Execution-phase signal. `false` means the classifier PROVED that no call
+ * left this process (the attempt died before authorize or before the proxy
+ * stream), so the physical attempt cannot have billed. `true` means a call
+ * left the process or may have. `undefined` means the classifier does not
+ * know — consumers must assume the expensive direction (dispatched).
+ *
+ * Only a classifier that can prove the negative may set `false`: marking a
+ * dispatched attempt as `false` would leave the idempotency key revivable
+ * and permit a second billable call.
+ */
+ providerDispatched?: boolean
}
/**
diff --git a/mcp-host/src/pluginWorkloadSdk/promptBridge/controlApiClient.test.ts b/mcp-host/src/pluginWorkloadSdk/promptBridge/controlApiClient.test.ts
index 464f6ef74..9ec3361f7 100644
--- a/mcp-host/src/pluginWorkloadSdk/promptBridge/controlApiClient.test.ts
+++ b/mcp-host/src/pluginWorkloadSdk/promptBridge/controlApiClient.test.ts
@@ -544,6 +544,97 @@ describe('PluginWorkloadSdkControlApiClient', () => {
})
})
+ it('surfaces ledger_pending as the retryable-with-reason finalize triple', async () => {
+ // The handler retries a finalize ONLY on this exact triple. If control-api
+ // ever stops sending `reason`, or the client stops mapping it, the retry
+ // silently disappears — so the triple is asserted end to end here.
+ const fetchImpl = vi.fn().mockResolvedValue(
+ jsonResponse(409, {
+ error: 'provider_unavailable',
+ message: 'linked Codex attempt has not finalized usage yet',
+ retryable: true,
+ reason: 'provider_unavailable',
+ })
+ )
+ const client = makeClient(fetchImpl as unknown as typeof fetch)
+ const failure = await client
+ .finalizePromptBridge({
+ recipeNamespace: 'sandbox-recipes',
+ recipeName: 'r1',
+ invocationId: 'inv-1',
+ attemptGeneration: 1,
+ providerAttemptId: 'attempt-1',
+ providerAttemptIndex: 1,
+ status: 'complete',
+ reason: 'provider_completed',
+ target,
+ })
+ .catch((err: unknown) => err)
+ expect(failure).toBeInstanceOf(PluginWorkloadError)
+ expect(failure).toMatchObject({
+ code: 'provider_unavailable',
+ retryable: true,
+ reason: 'provider_unavailable',
+ })
+ // A 409 is not retried by the transport itself; the ledger retry lives in
+ // the handler, where the deadline can bound it.
+ expect(fetchImpl).toHaveBeenCalledTimes(1)
+ })
+
+ it('threads a caller deadline into the finalize request and stops retrying once aborted', async () => {
+ const controller = new AbortController()
+ const fetchImpl = vi.fn().mockResolvedValue(
+ jsonResponse(201, {
+ invocationId: 'inv-1',
+ providerAttemptId: 'attempt-1',
+ status: 'complete',
+ outcome: 'exact',
+ idempotent: false,
+ usageAccepted: true,
+ })
+ )
+ const client = makeClient(fetchImpl as unknown as typeof fetch)
+ await client.finalizePromptBridge(
+ {
+ recipeNamespace: 'sandbox-recipes',
+ recipeName: 'r1',
+ invocationId: 'inv-1',
+ attemptGeneration: 1,
+ providerAttemptId: 'attempt-1',
+ providerAttemptIndex: 1,
+ status: 'complete',
+ reason: 'provider_completed',
+ target,
+ },
+ { signal: controller.signal }
+ )
+ const [, init] = fetchImpl.mock.calls[0] as [string, RequestInit]
+ expect(init.signal).toBeInstanceOf(AbortSignal)
+ expect(init.signal?.aborted).toBe(false)
+
+ // An already-expired deadline must not buy a single transport attempt: the
+ // AbortError would otherwise be swallowed by the network-retry branch.
+ const abortedFetch = vi.fn()
+ const abortedClient = makeClient(abortedFetch as unknown as typeof fetch)
+ await expect(
+ abortedClient.finalizePromptBridge(
+ {
+ recipeNamespace: 'sandbox-recipes',
+ recipeName: 'r1',
+ invocationId: 'inv-1',
+ attemptGeneration: 1,
+ providerAttemptId: 'attempt-1',
+ providerAttemptIndex: 1,
+ status: 'complete',
+ reason: 'provider_completed',
+ target,
+ },
+ { signal: AbortSignal.abort() }
+ )
+ ).rejects.toMatchObject({ code: 'provider_unavailable', retryable: true })
+ expect(abortedFetch).not.toHaveBeenCalled()
+ })
+
it('submitClientNotification maps event_type_not_allowed', async () => {
const fetchImpl = vi
.fn()
diff --git a/mcp-host/src/pluginWorkloadSdk/promptBridge/controlApiClient.ts b/mcp-host/src/pluginWorkloadSdk/promptBridge/controlApiClient.ts
index 8abe68351..efeff34e2 100644
--- a/mcp-host/src/pluginWorkloadSdk/promptBridge/controlApiClient.ts
+++ b/mcp-host/src/pluginWorkloadSdk/promptBridge/controlApiClient.ts
@@ -319,9 +319,9 @@ export class PluginWorkloadSdkControlApiClient {
* no refresh hook) propagates the error so a genuinely bad credential still
* surfaces instead of looping.
*/
- private async post(path: string, body: unknown): Promise<unknown> {
+ private async post(path: string, body: unknown, signal?: AbortSignal): Promise<unknown> {
try {
- return await this.postOnce(path, body)
+ return await this.postOnce(path, body, signal)
} catch (err) {
if (
err instanceof PluginWorkloadError &&
@@ -329,7 +329,7 @@ export class PluginWorkloadSdkControlApiClient {
this.opts.refreshOnUnauthorized
) {
await this.triggerRefresh()
- return this.postOnce(path, body)
+ return this.postOnce(path, body, signal)
}
throw err
}
@@ -363,7 +363,7 @@ export class PluginWorkloadSdkControlApiClient {
}
}
- private async postOnce(path: string, body: unknown): Promise<unknown> {
+ private async postOnce(path: string, body: unknown, signal?: AbortSignal): Promise<unknown> {
if (!this.breaker.allow()) {
throw new PluginWorkloadError(
'provider_unavailable',
@@ -379,6 +379,10 @@ export class PluginWorkloadSdkControlApiClient {
let lastError: unknown = null
for (let attempt = 0; attempt <= maxRetries; attempt++) {
+ // An aborted caller deadline is not a transient network failure: without
+ // this the `catch` below would swallow every AbortError as retryable and
+ // burn the remaining attempts after the operation was already over.
+ if (signal?.aborted) break
if (attempt > 0) await sleep(baseDelay * 2 ** (attempt - 1))
let response: Response
try {
@@ -389,7 +393,12 @@ export class PluginWorkloadSdkControlApiClient {
Authorization: `Bearer ${this.opts.getAccessToken()}`,
},
body: JSON.stringify(body),
- signal: AbortSignal.timeout(15_000),
+ // The per-request timeout still bounds one hop; a caller-supplied
+ // signal additionally bounds the whole operation across retries, so
+ // a finalize deadline cannot be outlived by this loop.
+ signal: signal
+ ? AbortSignal.any([AbortSignal.timeout(15_000), signal])
+ : AbortSignal.timeout(15_000),
})
} catch (err) {
// Network failure — retryable.
@@ -818,25 +827,28 @@ export class PluginWorkloadSdkControlApiClient {
* unknown spend. This replaces separate status + usage calls for the
* stepless runtime; the route is idempotent on providerAttemptId.
*/
- async finalizePromptBridge(body: {
- recipeNamespace: string
- recipeName: string
- invocationId: string
- attemptGeneration: number
- providerAttemptId: string
- providerAttemptIndex: number
- status: 'complete' | 'failed' | 'provider_unavailable'
- reason: string
- target: PromptBridgeTarget
- usage?: {
- llmSecretName: string
- callerRef: string
- fallbackUsed: boolean
- attemptCount: number
- inputTokens: number
- outputTokens: number
- }
- }): Promise<FinalizePromptBridgeResponse> {
+ async finalizePromptBridge(
+ body: {
+ recipeNamespace: string
+ recipeName: string
+ invocationId: string
+ attemptGeneration: number
+ providerAttemptId: string
+ providerAttemptIndex: number
+ status: 'complete' | 'failed' | 'provider_unavailable'
+ reason: string
+ target: PromptBridgeTarget
+ usage?: {
+ llmSecretName: string
+ callerRef: string
+ fallbackUsed: boolean
+ attemptCount: number
+ inputTokens: number
+ outputTokens: number
+ }
+ },
+ options?: { signal?: AbortSignal }
+ ): Promise<FinalizePromptBridgeResponse> {
const result = await this.post(
`/api/v1/mcp-host/plugin-workload-sdk/invocations/${encodeURIComponent(body.invocationId)}/finalize`,
{
@@ -850,7 +862,8 @@ export class PluginWorkloadSdkControlApiClient {
reason: body.reason,
target: body.target,
...(body.usage ? { usage: body.usage } : {}),
- }
+ },
+ options?.signal
)
if (!isFinalizePromptBridgeResponse(result)) {
throw new PluginWorkloadError(
diff --git a/mcp-host/src/pluginWorkloadSdk/promptBridge/handler.test.ts b/mcp-host/src/pluginWorkloadSdk/promptBridge/handler.test.ts
index 78a6f98a8..f965bd516 100644
--- a/mcp-host/src/pluginWorkloadSdk/promptBridge/handler.test.ts
+++ b/mcp-host/src/pluginWorkloadSdk/promptBridge/handler.test.ts
@@ -3,9 +3,11 @@ import { PluginWorkloadError } from '../domain/errors'
import type { PromptBridgeTarget } from '../domain/types'
import type { PluginWorkloadSdkControlApiClient } from './controlApiClient'
import {
+ FINALIZE_LEDGER_RETRY_DELAYS_MS,
PromptBridgeHandler,
type PromptBridgeHandlerDeps,
- finalizeLedgerRetryDelayMs,
+ finalizeWithLedgerRetry,
+ isLedgerPendingError,
} from './handler'
import type { LlmBridge } from './llmBridge'
@@ -327,7 +329,8 @@ describe('PromptBridgeHandler', () => {
status: 'failed',
reason: 'credential_unavailable',
target: primary,
- })
+ }),
+ expect.objectContaining({ signal: expect.any(AbortSignal) })
)
expect(report).not.toHaveBeenCalled()
})
@@ -401,7 +404,8 @@ describe('PromptBridgeHandler', () => {
status: 'complete',
target: fallback,
usage: expect.objectContaining({ inputTokens: 10, outputTokens: 5 }),
- })
+ }),
+ expect.objectContaining({ signal: expect.any(AbortSignal) })
)
expect(report).not.toHaveBeenCalled()
expect(onUsage).not.toHaveBeenCalled()
@@ -444,16 +448,222 @@ describe('PromptBridgeHandler', () => {
status: 'provider_unavailable',
reason: 'outcome_unknown',
target: primary,
- })
+ }),
+ expect.objectContaining({ signal: expect.any(AbortSignal) })
)
expect(report).not.toHaveBeenCalled()
})
- it('uses seconds-scale backoff for ledger_pending retries', () => {
- expect(finalizeLedgerRetryDelayMs(0)).toBe(500)
- expect(finalizeLedgerRetryDelayMs(1)).toBe(1000)
- expect(finalizeLedgerRetryDelayMs(2)).toBe(2000)
- expect(finalizeLedgerRetryDelayMs(3)).toBe(4000)
+ it('bounds ledger_pending patience to a fixed four-attempt table', () => {
+ expect(FINALIZE_LEDGER_RETRY_DELAYS_MS).toEqual([500, 1000, 2000])
+ // Four attempts, three waits: the table length IS the retry budget.
+ expect(FINALIZE_LEDGER_RETRY_DELAYS_MS.reduce((sum, ms) => sum + ms, 0)).toBe(3_500)
+ })
+
+ it('recognises only the ledger_pending triple as a retryable finalize error', () => {
+ expect(
+ isLedgerPendingError(
+ new PluginWorkloadError(
+ 'provider_unavailable',
+ 'linked Codex attempt has not finalized usage yet',
+ true,
+ 'provider_unavailable'
+ )
+ )
+ ).toBe(true)
+ // Retryable but reasonless: control-api 5xx, open breaker, exhausted
+ // transport retries. Those already burnt their own backoff inside postOnce.
+ expect(
+ isLedgerPendingError(
+ new PluginWorkloadError('provider_unavailable', 'control-api responded 503', true)
+ )
+ ).toBe(false)
+ expect(
+ isLedgerPendingError(
+ new PluginWorkloadError(
+ 'provider_unavailable',
+ 'not retryable',
+ false,
+ 'provider_unavailable'
+ )
+ )
+ ).toBe(false)
+ expect(
+ isLedgerPendingError(
+ new PluginWorkloadError('provider_unavailable', 'other reason', true, 'timeout')
+ )
+ ).toBe(false)
+ expect(isLedgerPendingError(new Error('plain'))).toBe(false)
+ })
+
+ it('does not retry a retryable finalize error that is not ledger_pending', async () => {
+ const finalize = vi
+ .fn<FinalizePromptBridge>()
+ .mockRejectedValue(
+ new PluginWorkloadError('provider_unavailable', 'control-api responded 503', true)
+ )
+ const complete = vi.fn().mockResolvedValue({
+ model: fallback.model,
+ servedTarget: fallback,
+ fallbackUsed: true,
+ attemptCount: 2,
+ llmSecretName: 'provider-secret',
+ providerAttemptId: 'attempt-1',
+ providerAttemptIndex: 2,
+ content: 'summary text',
+ usage: { inputTokens: 10, outputTokens: 5 },
+ finishReason: 'complete',
+ })
+ const { handler } = makeDeps({ complete, finalize })
+
+ await expect(handler.handle(validBody, 'api')).rejects.toMatchObject({
+ code: 'provider_unavailable',
+ reason: 'outcome_unknown',
+ })
+ // One attempt on the complete path, then one on the error path (N-11).
+ // Neither retries: the error is retryable but carries no `reason`.
+ expect(finalize).toHaveBeenCalledTimes(2)
+ expect(finalize).toHaveBeenNthCalledWith(
+ 1,
+ expect.objectContaining({ status: 'complete' }),
+ expect.objectContaining({ signal: expect.any(AbortSignal) })
+ )
+ expect(finalize).toHaveBeenNthCalledWith(
+ 2,
+ expect.objectContaining({ status: 'provider_unavailable' }),
+ expect.objectContaining({ signal: expect.any(AbortSignal) })
+ )
+ })
+
+ it('exhausts exactly the table budget on a persistent ledger_pending', async () => {
+ const finalize = vi
+ .fn<FinalizePromptBridge>()
+ .mockRejectedValue(
+ new PluginWorkloadError(
+ 'provider_unavailable',
+ 'linked Codex attempt has not finalized usage yet',
+ true,
+ 'provider_unavailable'
+ )
+ )
+ const complete = vi.fn().mockResolvedValue({
+ model: fallback.model,
+ servedTarget: fallback,
+ fallbackUsed: true,
+ attemptCount: 2,
+ llmSecretName: 'provider-secret',
+ providerAttemptId: 'attempt-1',
+ providerAttemptIndex: 2,
+ content: 'summary text',
+ usage: { inputTokens: 10, outputTokens: 5 },
+ finishReason: 'complete',
+ })
+ const { handler } = makeDeps({ complete, finalize })
+
+ vi.useFakeTimers()
+ try {
+ // Attach the rejection handler before advancing the clock, or the
+ // in-flight rejection surfaces as an unhandled rejection.
+ const settled = expect(handler.handle(validBody, 'api')).rejects.toMatchObject({
+ code: 'provider_unavailable',
+ reason: 'outcome_unknown',
+ })
+ await vi.runAllTimersAsync()
+ await settled
+ } finally {
+ vi.useRealTimers()
+ }
+ // Four attempts per finalize path, never five. The complete path exhausts
+ // its budget, then the error path (N-11) settles the same receipt with a
+ // budget of its own — the documented worst case of two bounded deadlines.
+ expect(finalize).toHaveBeenCalledTimes((FINALIZE_LEDGER_RETRY_DELAYS_MS.length + 1) * 2)
+ })
+
+ it('surfaces ledger_pending without another wait once its deadline is aborted', async () => {
+ const pendingError = new PluginWorkloadError(
+ 'provider_unavailable',
+ 'linked Codex attempt has not finalized usage yet',
+ true,
+ 'provider_unavailable'
+ )
+ const finalize = vi.fn<FinalizePromptBridge>().mockRejectedValue(pendingError)
+
+ await expect(
+ finalizeWithLedgerRetry(
+ finalize,
+ {
+ invocationId: 'inv-1',
+ attemptGeneration: 1,
+ providerAttemptId: 'attempt-1',
+ providerAttemptIndex: 1,
+ status: 'complete',
+ reason: 'provider_completed',
+ target: primary,
+ },
+ AbortSignal.abort()
+ )
+ ).rejects.toBe(pendingError)
+ // An expired deadline buys neither another attempt nor another sleep, and
+ // the real pending error reaches the caller instead of a synthetic one.
+ expect(finalize).toHaveBeenCalledTimes(1)
+ })
+
+ it('retries ledger_pending on the error path with the same bounded budget', async () => {
+ const finalize = vi
+ .fn<FinalizePromptBridge>()
+ .mockRejectedValueOnce(
+ new PluginWorkloadError(
+ 'provider_unavailable',
+ 'linked Codex attempt has not finalized usage yet',
+ true,
+ 'provider_unavailable'
+ )
+ )
+ .mockResolvedValueOnce({
+ invocationId: 'inv-1',
+ providerAttemptId: 'attempt-1',
+ status: 'provider_unavailable',
+ outcome: 'unknown',
+ idempotent: false,
+ usageAccepted: false,
+ })
+ const complete = vi.fn().mockRejectedValue(
+ new PluginWorkloadError(
+ 'provider_unavailable',
+ 'provider outcome unknown',
+ false,
+ 'outcome_unknown',
+ true,
+ {
+ providerAttemptId: 'attempt-1',
+ providerAttemptIndex: 1,
+ target: primary,
+ attemptCount: 1,
+ fallbackUsed: false,
+ }
+ )
+ )
+ const { handler, report } = makeDeps({ complete, finalize })
+
+ vi.useFakeTimers()
+ try {
+ const settled = expect(handler.handle(validBody, 'api')).rejects.toMatchObject({
+ code: 'provider_unavailable',
+ reason: 'outcome_unknown',
+ })
+ await vi.runAllTimersAsync()
+ await settled
+ } finally {
+ vi.useRealTimers()
+ }
+ expect(finalize).toHaveBeenCalledTimes(2)
+ expect(finalize).toHaveBeenNthCalledWith(
+ 2,
+ expect.objectContaining({ status: 'provider_unavailable' }),
+ expect.objectContaining({ signal: expect.any(AbortSignal) })
+ )
+ // The retry succeeded, so the legacy invocation-status report stays silent.
+ expect(report).not.toHaveBeenCalled()
})
it('retries a retryable complete finalization before treating the outcome as unknown', async () => {
@@ -501,8 +711,16 @@ describe('PromptBridgeHandler', () => {
vi.useRealTimers()
}
expect(finalize).toHaveBeenCalledTimes(2)
- expect(finalize).toHaveBeenNthCalledWith(1, expect.objectContaining({ status: 'complete' }))
- expect(finalize).toHaveBeenNthCalledWith(2, expect.objectContaining({ status: 'complete' }))
+ expect(finalize).toHaveBeenNthCalledWith(
+ 1,
+ expect.objectContaining({ status: 'complete' }),
+ expect.objectContaining({ signal: expect.any(AbortSignal) })
+ )
+ expect(finalize).toHaveBeenNthCalledWith(
+ 2,
+ expect.objectContaining({ status: 'complete' }),
+ expect.objectContaining({ signal: expect.any(AbortSignal) })
+ )
expect(report).not.toHaveBeenCalled()
})
@@ -537,10 +755,15 @@ describe('PromptBridgeHandler', () => {
reason: 'outcome_unknown',
})
expect(finalize).toHaveBeenCalledTimes(2)
- expect(finalize).toHaveBeenNthCalledWith(1, expect.objectContaining({ status: 'complete' }))
+ expect(finalize).toHaveBeenNthCalledWith(
+ 1,
+ expect.objectContaining({ status: 'complete' }),
+ expect.objectContaining({ signal: expect.any(AbortSignal) })
+ )
expect(finalize).toHaveBeenNthCalledWith(
2,
- expect.objectContaining({ status: 'provider_unavailable', reason: 'outcome_unknown' })
+ expect.objectContaining({ status: 'provider_unavailable', reason: 'outcome_unknown' }),
+ expect.objectContaining({ signal: expect.any(AbortSignal) })
)
expect(report).not.toHaveBeenCalled()
})
diff --git a/mcp-host/src/pluginWorkloadSdk/promptBridge/handler.ts b/mcp-host/src/pluginWorkloadSdk/promptBridge/handler.ts
index b78dbde30..dc333f53f 100644
--- a/mcp-host/src/pluginWorkloadSdk/promptBridge/handler.ts
+++ b/mcp-host/src/pluginWorkloadSdk/promptBridge/handler.ts
@@ -36,35 +36,78 @@ import type { LlmBridge } from './llmBridge'
const IDEMPOTENCY_KEY_RE = new RegExp(DEFAULT_IDEMPOTENCY_KEY_PATTERN)
const PURPOSE_SET = new Set<string>(PLUGIN_WORKLOAD_SDK_PURPOSES)
const MAX_METADATA_BYTES = 16 * 1024
-const FINALIZE_LEDGER_RETRIES = 4
-/** Seconds-scale backoff so a Codex usage frame can land before ledger_pending expires. */
-export function finalizeLedgerRetryDelayMs(attempt: number): number {
- return Math.min(4_000, 500 * 2 ** attempt)
+/**
+ * Four attempts, three waits (3.5 s total). `ledger_pending` at this point is a
+ * commit race of milliseconds between the proxy's own finalize — which runs in
+ * its `finally`, before the `done` frame — and this read. The 900 s the sweeper
+ * waits is the sweeper's patience, not a client budget: a proxy that already
+ * exhausted its attempts will not land no matter how long the host waits here.
+ *
+ * No jitter: finalizations serialize per recipe on the advisory lock rather
+ * than competing in a thundering herd, and a deterministic table is the part a
+ * test can actually assert.
+ */
+export const FINALIZE_LEDGER_RETRY_DELAYS_MS = [500, 1_000, 2_000] as const
+
+/**
+ * Whole-finalize deadline, retries included. Worst case after the provider has
+ * already answered is two of these (complete path, then error path) instead of
+ * the 65-250 s an unfiltered retry of every retryable error could reach.
+ */
+const FINALIZE_DEADLINE_MS = 30_000
+
+/**
+ * `ledger_pending` is the only 409 the finalize route marks retryable AND
+ * names with a `reason`. Every other retryable finalize failure (5xx, open
+ * breaker, exhausted transport retries, unexpected response shape) arrives
+ * WITHOUT a reason and has already burnt four fetches with backoff inside
+ * `postOnce`; repeating those four more times outside is amplification with no
+ * new information. The triple is asserted as a contract in the client tests.
+ */
+export function isLedgerPendingError(err: unknown): boolean {
+ return (
+ err instanceof PluginWorkloadError &&
+ err.code === 'provider_unavailable' &&
+ err.retryable === true &&
+ err.reason === 'provider_unavailable'
+ )
+}
+
+/** Resolves on the timer or on abort, whichever comes first. */
+function sleepUnlessAborted(ms: number, signal: AbortSignal): Promise<void> {
+ return new Promise(resolve => {
+ let timer: NodeJS.Timeout
+ const finish = () => {
+ clearTimeout(timer)
+ signal.removeEventListener('abort', finish)
+ resolve()
+ }
+ timer = setTimeout(finish, ms)
+ signal.addEventListener('abort', finish, { once: true })
+ })
}
-async function finalizeCompleteWithLedgerRetry(
+export async function finalizeWithLedgerRetry(
finalize: NonNullable<PromptBridgeHandlerDeps['finalizePromptBridge']>,
- input: Parameters<NonNullable<PromptBridgeHandlerDeps['finalizePromptBridge']>>[0]
+ input: Parameters<NonNullable<PromptBridgeHandlerDeps['finalizePromptBridge']>>[0],
+ signal: AbortSignal
): Promise<void> {
- let lastError: unknown
- for (let attempt = 0; attempt < FINALIZE_LEDGER_RETRIES; attempt++) {
+ for (let attempt = 0; ; attempt++) {
try {
- await finalize(input)
+ await finalize(input, { signal })
return
} catch (error) {
- lastError = error
- if (
- !(error instanceof PluginWorkloadError) ||
- error.retryable !== true ||
- attempt === FINALIZE_LEDGER_RETRIES - 1
- ) {
+ if (attempt >= FINALIZE_LEDGER_RETRY_DELAYS_MS.length || !isLedgerPendingError(error)) {
throw error
}
- await new Promise(resolve => setTimeout(resolve, finalizeLedgerRetryDelayMs(attempt)))
+ // The deadline surfaces the real pending error rather than a synthetic
+ // one, so the caller's fence sees why the ledger never settled.
+ if (signal.aborted) throw error
+ await sleepUnlessAborted(FINALIZE_LEDGER_RETRY_DELAYS_MS[attempt], signal)
+ if (signal.aborted) throw error
}
}
- throw lastError
}
/**
@@ -110,23 +153,26 @@ export interface PromptBridgeHandlerDeps {
/** Public provider/model identity from the live spec.agent binding. */
getBootstrapTarget?: () => { provider: string; model: string } | null
/** SDK-only atomic accounting boundary; workflow mode keeps the legacy reporter. */
- finalizePromptBridge?: (input: {
- invocationId: string
- attemptGeneration: number
- providerAttemptId: string
- providerAttemptIndex: number
- status: 'complete' | 'failed' | 'provider_unavailable'
- reason: string
- target: PromptBridgeTarget
- usage?: {
- llmSecretName: string
- callerRef: string
- fallbackUsed: boolean
- attemptCount: number
- inputTokens: number
- outputTokens: number
- }
- }) => Promise<unknown>
+ finalizePromptBridge?: (
+ input: {
+ invocationId: string
+ attemptGeneration: number
+ providerAttemptId: string
+ providerAttemptIndex: number
+ status: 'complete' | 'failed' | 'provider_unavailable'
+ reason: string
+ target: PromptBridgeTarget
+ usage?: {
+ llmSecretName: string
+ callerRef: string
+ fallbackUsed: boolean
+ attemptCount: number
+ inputTokens: number
+ outputTokens: number
+ }
+ },
+ options?: { signal?: AbortSignal }
+ ) => Promise<unknown>
onUsage?: (usage: {
invocationId: string
attemptGeneration: number
@@ -418,26 +464,30 @@ export class PromptBridgeHandler {
providerAttemptIndex >= 1
) {
try {
- await finalizeCompleteWithLedgerRetry(this.deps.finalizePromptBridge, {
- invocationId: authorized.invocationId,
- attemptGeneration: authorized.attemptGeneration,
- providerAttemptId,
- providerAttemptIndex,
- status: 'complete',
- reason:
- providerAttemptAcknowledgement === 'failed'
- ? 'provider_completion_acknowledgement_failed'
- : 'provider_completed',
- target: completion.servedTarget,
- usage: {
- llmSecretName: completion.llmSecretName,
- callerRef,
- fallbackUsed: completion.fallbackUsed,
- attemptCount: completion.attemptCount,
- inputTokens: completion.usage.inputTokens,
- outputTokens: completion.usage.outputTokens,
+ await finalizeWithLedgerRetry(
+ this.deps.finalizePromptBridge,
+ {
+ invocationId: authorized.invocationId,
+ attemptGeneration: authorized.attemptGeneration,
+ providerAttemptId,
+ providerAttemptIndex,
+ status: 'complete',
+ reason:
+ providerAttemptAcknowledgement === 'failed'
+ ? 'provider_completion_acknowledgement_failed'
+ : 'provider_completed',
+ target: completion.servedTarget,
+ usage: {
+ llmSecretName: completion.llmSecretName,
+ callerRef,
+ fallbackUsed: completion.fallbackUsed,
+ attemptCount: completion.attemptCount,
+ inputTokens: completion.usage.inputTokens,
+ outputTokens: completion.usage.outputTokens,
+ },
},
- })
+ AbortSignal.timeout(FINALIZE_DEADLINE_MS)
+ )
} catch {
// Returning provider content before durable finalization would hide
// a possible spend. Surface an explicit unknown outcome instead.
@@ -535,16 +585,24 @@ export class PromptBridgeHandler {
err instanceof PluginWorkloadError && err.providerMayHaveExecuted
? 'provider_unavailable'
: 'failed'
- await this.deps.finalizePromptBridge({
- invocationId: authorized.invocationId,
- attemptGeneration: authorized.attemptGeneration,
- providerAttemptId: unknownProviderAttempt.providerAttemptId,
- providerAttemptIndex: unknownProviderAttempt.providerAttemptIndex,
- status: finalizationStatus,
- reason:
- err instanceof PluginWorkloadError && err.reason ? err.reason : 'outcome_unknown',
- target: unknownProviderAttempt.target,
- })
+ // N-11: the error path settles the same ledger row as the complete
+ // path, so it needs the same `ledger_pending` patience and the same
+ // bounded deadline. Losing this attempt to a millisecond commit race
+ // would silently drop the spend for an attempt that already ran.
+ await finalizeWithLedgerRetry(
+ this.deps.finalizePromptBridge,
+ {
+ invocationId: authorized.invocationId,
+ attemptGeneration: authorized.attemptGeneration,
+ providerAttemptId: unknownProviderAttempt.providerAttemptId,
+ providerAttemptIndex: unknownProviderAttempt.providerAttemptIndex,
+ status: finalizationStatus,
+ reason:
+ err instanceof PluginWorkloadError && err.reason ? err.reason : 'outcome_unknown',
+ target: unknownProviderAttempt.target,
+ },
+ AbortSignal.timeout(FINALIZE_DEADLINE_MS)
+ )
providerAttemptFinalized = true
} catch {
console.warn('[PluginWorkloadSdk] provider spend finalization failed')
diff --git a/mcp-host/src/pluginWorkloadSdk/promptBridge/llmBridge.failover.test.ts b/mcp-host/src/pluginWorkloadSdk/promptBridge/llmBridge.failover.test.ts
index 5ec256ffe..c5b95924d 100644
--- a/mcp-host/src/pluginWorkloadSdk/promptBridge/llmBridge.failover.test.ts
+++ b/mcp-host/src/pluginWorkloadSdk/promptBridge/llmBridge.failover.test.ts
@@ -33,7 +33,12 @@ const fallback: PromptBridgeTarget = {
class FakeProvider implements Partial<SingleTurnProvider> {
constructor(
private readonly behavior: () => Promise<typeof OK>,
- private readonly classification: { code: LlmErrorCode; retryable: boolean } = {
+ private readonly classification: {
+ code: LlmErrorCode
+ retryable: boolean
+ providerCode?: string
+ providerDispatched?: boolean
+ } = {
code: LlmErrorCode.RateLimited,
retryable: true,
}
@@ -782,3 +787,318 @@ describe('LlmBridge authorized multi-provider fallback', () => {
expect(provider.completeSingleTurn).not.toHaveBeenCalled()
})
})
+
+const codexTarget: PromptBridgeTarget = {
+ targetRef: 'codex-primary',
+ provider: 'codex-subscription',
+ model: 'gpt-5.1',
+ credentialSlot: '',
+ connectionRef: 'team-plus',
+}
+
+function installCodexBinding(): void {
+ replaceSdkOnlyCodexBinding({
+ connectionKey: 'team-plus',
+ catalogRevision: 1,
+ credentialRevision: 0,
+ model: codexTarget.model,
+ bindingHash: computeCodexPolicyHash({
+ model: codexTarget.model,
+ catalogRevision: 1,
+ credentialRevision: 0,
+ connectionKey: 'team-plus',
+ }),
+ })
+}
+
+/** Codex reserves index 1; any api-key fallback behind it reserves index 2. */
+function codexTicketIssuer() {
+ return {
+ issue: vi.fn(async ({ target }: { target: PromptBridgeTarget }) => ({
+ credentialTicket: target.provider === 'codex-subscription' ? '' : `fresh-${target.targetRef}`,
+ providerAttemptId: `sdk-${target.targetRef}`,
+ providerAttemptIndex: target.provider === 'codex-subscription' ? 1 : 2,
+ })),
+ }
+}
+
+const codexReceipt = {
+ providerAttemptId: `sdk-${codexTarget.targetRef}`,
+ providerAttemptIndex: 1,
+ target: codexTarget,
+ attemptCount: 1,
+ fallbackUsed: false,
+}
+
+describe('LlmBridge oauth-broker terminal accounting', () => {
+ afterEach(() => {
+ replaceSdkOnlyCodexBinding(null)
+ })
+
+ it('keeps a pre-dispatch Codex budget denial revivable and carries its receipt', async () => {
+ installCodexBinding()
+ const codex = new FakeProvider(() => Promise.reject(new Error('budget denied')), {
+ code: LlmErrorCode.InsufficientQuota,
+ retryable: false,
+ providerCode: 'budget_denied',
+ providerDispatched: false,
+ })
+ const second = new FakeProvider(() => Promise.resolve(OK))
+ const { bridge, providerCalls } = makeBridge({
+ [codexTarget.model]: codex,
+ [fallback.model]: second,
+ })
+ const providerAttemptReporter = { report: vi.fn().mockResolvedValue(undefined) }
+
+ await expect(
+ bridge.complete({
+ ...request,
+ targets: [{ target: codexTarget }, { target: fallback }],
+ credentialTicketIssuer: codexTicketIssuer(),
+ providerAttemptReporter,
+ })
+ ).rejects.toMatchObject({
+ code: 'provider_unavailable',
+ reason: 'insufficient_quota',
+ providerMayHaveExecuted: false,
+ providerAttempt: codexReceipt,
+ })
+ // A denial that never reached the provider must stay terminal for the
+ // invocation and revivable for the idempotency key.
+ expect(providerCalls).toEqual([`${codexTarget.provider}/${codexTarget.model}`])
+ expect(second.completeSingleTurn).not.toHaveBeenCalled()
+ expect(providerAttemptReporter.report).toHaveBeenCalledWith({
+ providerAttemptId: codexReceipt.providerAttemptId,
+ providerAttemptIndex: 1,
+ status: 'failed',
+ })
+ })
+
+ it('fences a dispatched Codex no_grant as an ambiguous physical outcome', async () => {
+ installCodexBinding()
+ const codex = new FakeProvider(() => Promise.reject(new Error('grant revoked mid-flight')), {
+ code: LlmErrorCode.AuthenticationFailed,
+ retryable: false,
+ providerCode: 'no_grant',
+ providerDispatched: true,
+ })
+ const second = new FakeProvider(() => Promise.resolve(OK))
+ const { bridge } = makeBridge({
+ [codexTarget.model]: codex,
+ [fallback.model]: second,
+ })
+ const providerAttemptReporter = { report: vi.fn().mockResolvedValue(undefined) }
+
+ await expect(
+ bridge.complete({
+ ...request,
+ targets: [{ target: codexTarget }, { target: fallback }],
+ credentialTicketIssuer: codexTicketIssuer(),
+ providerAttemptReporter,
+ })
+ ).rejects.toMatchObject({
+ code: 'provider_unavailable',
+ providerMayHaveExecuted: true,
+ providerAttempt: codexReceipt,
+ })
+ expect(second.completeSingleTurn).not.toHaveBeenCalled()
+ // The physical status must agree with providerMayHaveExecuted; a hardcoded
+ // `failed` here would leave the key revivable after a possible charge.
+ expect(providerAttemptReporter.report).toHaveBeenCalledWith({
+ providerAttemptId: codexReceipt.providerAttemptId,
+ providerAttemptIndex: 1,
+ status: 'provider_unavailable',
+ })
+ })
+
+ it.each(['insufficient_scope', 'host_binding_mismatch'])(
+ 'closes a pre-dispatch Codex %s terminally with its receipt',
+ async providerCode => {
+ installCodexBinding()
+ const codex = new FakeProvider(() => Promise.reject(new Error(providerCode)), {
+ code: LlmErrorCode.AuthenticationFailed,
+ retryable: false,
+ providerCode,
+ providerDispatched: false,
+ })
+ const second = new FakeProvider(() => Promise.resolve(OK))
+ const { bridge } = makeBridge({
+ [codexTarget.model]: codex,
+ [fallback.model]: second,
+ })
+ const providerAttemptReporter = { report: vi.fn().mockResolvedValue(undefined) }
+
+ await expect(
+ bridge.complete({
+ ...request,
+ targets: [{ target: codexTarget }, { target: fallback }],
+ credentialTicketIssuer: codexTicketIssuer(),
+ providerAttemptReporter,
+ })
+ ).rejects.toMatchObject({
+ code: 'provider_unavailable',
+ providerMayHaveExecuted: false,
+ providerAttempt: codexReceipt,
+ })
+ expect(second.completeSingleTurn).not.toHaveBeenCalled()
+ expect(providerAttemptReporter.report).toHaveBeenCalledWith({
+ providerAttemptId: codexReceipt.providerAttemptId,
+ providerAttemptIndex: 1,
+ status: 'failed',
+ })
+ }
+ )
+
+ it('carries the reserved receipt on a terminal generic Codex failure', async () => {
+ installCodexBinding()
+ const codex = new FakeProvider(() => Promise.reject(new Error('codex unavailable')))
+ const { bridge } = makeBridge({ [codexTarget.model]: codex })
+ const providerAttemptReporter = { report: vi.fn().mockResolvedValue(undefined) }
+
+ await expect(
+ bridge.complete({
+ ...request,
+ targets: [{ target: codexTarget }],
+ credentialTicketIssuer: codexTicketIssuer(),
+ providerAttemptReporter,
+ })
+ ).rejects.toMatchObject({
+ code: 'provider_unavailable',
+ reason: 'rate_limited',
+ providerAttempt: codexReceipt,
+ })
+ expect(providerAttemptReporter.report).toHaveBeenCalledWith({
+ providerAttemptId: codexReceipt.providerAttemptId,
+ providerAttemptIndex: 1,
+ status: 'provider_unavailable',
+ })
+ })
+
+ it('fences a Codex budget denial whose acknowledgement was lost', async () => {
+ installCodexBinding()
+ const codex = new FakeProvider(() => Promise.reject(new Error('budget denied')), {
+ code: LlmErrorCode.InsufficientQuota,
+ retryable: false,
+ providerCode: 'budget_denied',
+ providerDispatched: false,
+ })
+ const second = new FakeProvider(() => Promise.resolve(OK))
+ const { bridge } = makeBridge({
+ [codexTarget.model]: codex,
+ [fallback.model]: second,
+ })
+ const providerAttemptReporter = {
+ report: vi.fn().mockRejectedValue(new Error('control-api unreachable')),
+ }
+
+ await expect(
+ bridge.complete({
+ ...request,
+ targets: [{ target: codexTarget }, { target: fallback }],
+ credentialTicketIssuer: codexTicketIssuer(),
+ providerAttemptReporter,
+ })
+ ).rejects.toMatchObject({
+ code: 'provider_unavailable',
+ reason: 'outcome_unknown',
+ providerMayHaveExecuted: true,
+ providerAttempt: codexReceipt,
+ })
+ expect(second.completeSingleTurn).not.toHaveBeenCalled()
+ })
+
+ it('treats an unclassified Codex dispatch state as executed', async () => {
+ installCodexBinding()
+ const codex = new FakeProvider(() => Promise.reject(new Error('unrecognized failure')), {
+ code: LlmErrorCode.ModelOverloaded,
+ retryable: true,
+ })
+ const { bridge } = makeBridge({ [codexTarget.model]: codex })
+ const providerAttemptReporter = { report: vi.fn().mockResolvedValue(undefined) }
+
+ await expect(
+ bridge.complete({
+ ...request,
+ targets: [{ target: codexTarget }],
+ credentialTicketIssuer: codexTicketIssuer(),
+ providerAttemptReporter,
+ })
+ ).rejects.toMatchObject({
+ code: 'provider_unavailable',
+ providerMayHaveExecuted: true,
+ })
+ expect(providerAttemptReporter.report).toHaveBeenCalledWith({
+ providerAttemptId: codexReceipt.providerAttemptId,
+ providerAttemptIndex: 1,
+ status: 'provider_unavailable',
+ })
+ })
+
+ it('honours the acknowledgement mode on a served Codex attempt', async () => {
+ installCodexBinding()
+ const codex = new FakeProvider(async () => ({
+ ...OK,
+ usage_reported: false,
+ usage: { input_tokens: 0, output_tokens: 0, total_tokens: 0 },
+ }))
+ const { bridge } = makeBridge({ [codexTarget.model]: codex })
+ const providerAttemptReporter = { report: vi.fn().mockResolvedValue(undefined) }
+
+ const atomic = await bridge.complete({
+ ...request,
+ targets: [{ target: codexTarget }],
+ acknowledgementMode: 'atomic_terminal_finalization',
+ credentialTicketIssuer: codexTicketIssuer(),
+ providerAttemptReporter,
+ })
+ expect(atomic.providerAttemptAcknowledgement).toBe('owned_by_finalizer')
+ expect(providerAttemptReporter.report).not.toHaveBeenCalled()
+
+ const perAttempt = await bridge.complete({
+ ...request,
+ targets: [{ target: codexTarget }],
+ acknowledgementMode: 'per_attempt',
+ credentialTicketIssuer: codexTicketIssuer(),
+ providerAttemptReporter,
+ })
+ expect(perAttempt.providerAttemptAcknowledgement).toBe('confirmed')
+ expect(providerAttemptReporter.report).toHaveBeenCalledWith({
+ providerAttemptId: codexReceipt.providerAttemptId,
+ providerAttemptIndex: 1,
+ status: 'complete',
+ })
+ })
+
+ it('reports a pre-dispatch eligible Codex failure as failed before failing over', async () => {
+ installCodexBinding()
+ const codex = new FakeProvider(() => Promise.reject(new Error('aborted before authorize')), {
+ code: LlmErrorCode.ModelOverloaded,
+ retryable: true,
+ providerDispatched: false,
+ })
+ const second = new FakeProvider(() => Promise.resolve(OK))
+ const { bridge, providerCalls } = makeBridge({
+ [codexTarget.model]: codex,
+ [fallback.model]: second,
+ })
+ const providerAttemptReporter = { report: vi.fn().mockResolvedValue(undefined) }
+
+ const result = await bridge.complete({
+ ...request,
+ targets: [{ target: codexTarget }, { target: fallback }],
+ credentialTicketIssuer: codexTicketIssuer(),
+ providerAttemptReporter,
+ })
+
+ expect(result).toMatchObject({ servedTarget: fallback, fallbackUsed: true, attemptCount: 2 })
+ expect(providerCalls).toEqual([
+ `${codexTarget.provider}/${codexTarget.model}`,
+ `${fallback.provider}/${fallback.model}`,
+ ])
+ expect(providerAttemptReporter.report).toHaveBeenNthCalledWith(1, {
+ providerAttemptId: codexReceipt.providerAttemptId,
+ providerAttemptIndex: 1,
+ status: 'failed',
+ })
+ })
+})
diff --git a/mcp-host/src/pluginWorkloadSdk/promptBridge/llmBridge.ts b/mcp-host/src/pluginWorkloadSdk/promptBridge/llmBridge.ts
index a3387eb9b..ed16f8711 100644
--- a/mcp-host/src/pluginWorkloadSdk/promptBridge/llmBridge.ts
+++ b/mcp-host/src/pluginWorkloadSdk/promptBridge/llmBridge.ts
@@ -666,6 +666,26 @@ export class LlmBridge {
breaker,
captured
)
+ // Receipt ownership is decided by the caller's mode, exactly as on the
+ // api-key lane. Only atomic finalization defers the terminal ACK; a
+ // workflow-mode Codex attempt must still be acknowledged here, and a
+ // lost acknowledgement must surface as `failed` rather than be hidden
+ // behind the finalizer's name.
+ let providerAttemptAcknowledgement: 'confirmed' | 'owned_by_finalizer' | 'failed' =
+ 'confirmed'
+ if (request.acknowledgementMode === 'atomic_terminal_finalization') {
+ providerAttemptAcknowledgement = 'owned_by_finalizer'
+ } else if (ticket.providerAttemptId && ticket.providerAttemptIndex !== undefined) {
+ const acknowledged = await reportProviderAttemptBestEffort(
+ request.providerAttemptReporter,
+ {
+ providerAttemptId: ticket.providerAttemptId,
+ providerAttemptIndex: ticket.providerAttemptIndex,
+ status: 'complete',
+ }
+ )
+ providerAttemptAcknowledgement = acknowledged ? 'confirmed' : 'failed'
+ }
return {
kind: 'result',
result: {
@@ -676,35 +696,19 @@ export class LlmBridge {
llmSecretName: '',
providerAttemptId: ticket.providerAttemptId,
providerAttemptIndex: ticket.providerAttemptIndex,
- providerAttemptAcknowledgement: 'owned_by_finalizer',
+ providerAttemptAcknowledgement,
},
}
} catch (error) {
- if (
- error instanceof ClassifiedProviderError &&
- (error.classified.providerCode === 'budget_denied' ||
- error.classified.providerCode === 'no_grant' ||
- error.classified.providerCode === 'host_binding_mismatch' ||
- error.classified.providerCode === 'insufficient_scope' ||
- error.classified.code === LlmErrorCode.AuthenticationFailed)
- ) {
- if (ticket.providerAttemptId && ticket.providerAttemptIndex !== undefined) {
- await reportProviderAttemptBestEffort(request.providerAttemptReporter, {
- providerAttemptId: ticket.providerAttemptId,
- providerAttemptIndex: ticket.providerAttemptIndex,
- status: 'failed',
- })
- }
- return {
- kind: 'error',
- error,
- terminal: true,
- }
- }
+ // One physical status for one attempt. The grant/scope/budget failures
+ // used to be acknowledged as `failed` from a branch of their own, which
+ // could contradict `providerMayHaveExecuted` and leave a possibly billed
+ // Codex call revivable. `providerMayHaveExecuted` is now the single
+ // source of that decision for every oauth failure.
+ const providerMayHaveExecuted =
+ (error instanceof PluginWorkloadError && error.providerMayHaveExecuted) ||
+ (error instanceof ClassifiedProviderError && error.providerMayHaveExecuted)
if (ticket.providerAttemptId && ticket.providerAttemptIndex !== undefined) {
- const providerMayHaveExecuted =
- (error instanceof PluginWorkloadError && error.providerMayHaveExecuted) ||
- (error instanceof ClassifiedProviderError && error.providerMayHaveExecuted)
const acknowledged = await reportProviderAttemptBestEffort(
request.providerAttemptReporter,
{
@@ -731,12 +735,30 @@ export class LlmBridge {
? withProviderAttemptContext(error, providerAttemptContext)
: error
}
+ // Grant, scope, binding and budget denials never advance to another
+ // target: they are operator/configuration signals about this invocation,
+ // not transient provider weather.
+ const forcedTerminal =
+ error.classified.providerCode === 'budget_denied' ||
+ error.classified.providerCode === 'no_grant' ||
+ error.classified.providerCode === 'host_binding_mismatch' ||
+ error.classified.providerCode === 'insufficient_scope' ||
+ error.classified.code === LlmErrorCode.AuthenticationFailed
const failureClass = classifyFailoverClass(error.classified.code, error.classified.retryable)
const eligible = failureClass !== null && triggerOn.has(failureClass)
return {
kind: 'error',
- error,
- terminal: !eligible || index === request.targets.length - 1,
+ // The ticket already reserved this physical attempt, so the receipt
+ // must travel with the error. Without it the atomic finalizer closes
+ // the logical invocation while the spend row for the attempt is never
+ // written — the same contract the api-key lane applies.
+ error: providerAttemptContext
+ ? new ClassifiedProviderError(
+ error.classified,
+ withProviderAttemptContext(error, providerAttemptContext)
+ )
+ : error,
+ terminal: forcedTerminal || !eligible || index === request.targets.length - 1,
}
}
}
@@ -861,17 +883,23 @@ export class LlmBridge {
code: classified.code,
retryable: classified.retryable,
...(classified.providerCode ? { providerCode: classified.providerCode } : {}),
+ ...(classified.providerDispatched !== undefined
+ ? { providerDispatched: classified.providerDispatched }
+ : {}),
},
new PluginWorkloadError(
'provider_unavailable',
`LLM provider error: ${classified.code}`,
classified.retryable,
reason,
- // This branch runs only after the provider call was entered. Even a
- // provider-declared auth/rate-limit response is not proof that no
- // billable work occurred, so it must remain fenced as an ambiguous
- // physical outcome rather than reviving the idempotency key.
- true
+ // This branch runs only after the provider call was entered, so an
+ // unknown dispatch state must read as executed: a provider-declared
+ // auth/rate-limit response is not proof that no billable work
+ // occurred. Only a classifier that PROVED no call left the process
+ // may revive the idempotency key — mislabelling a dispatched attempt
+ // as not-executed would permit a second charge, while the opposite
+ // mistake only costs revivability.
+ classified.providerDispatched ?? true
)
)
} finally {
diff --git a/mcp-host/src/pluginWorkloadSdk/server/index.ts b/mcp-host/src/pluginWorkloadSdk/server/index.ts
index ef1159d2c..e18cfd3a8 100644
--- a/mcp-host/src/pluginWorkloadSdk/server/index.ts
+++ b/mcp-host/src/pluginWorkloadSdk/server/index.ts
@@ -360,12 +360,15 @@ export function maybeCreatePluginWorkloadSdkServer(
: undefined,
finalizePromptBridge:
config.pluginWorkloadSdkRuntimeMode === 'sdk-only'
- ? input =>
- controlApiClient.finalizePromptBridge({
- recipeNamespace: binding.recipeNamespace,
- recipeName: binding.recipeName,
- ...input,
- })
+ ? (input, options) =>
+ controlApiClient.finalizePromptBridge(
+ {
+ recipeNamespace: binding.recipeNamespace,
+ recipeName: binding.recipeName,
+ ...input,
+ },
+ options
+ )
: undefined,
onUsage: usage => {
if (!usageReporter) return