Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
fix: settle and classify every oauth-broker provider attempt
Alfredo Lopez6b9722341cf1
diff --git a/mcp-host/package-lock.json b/mcp-host/package-lock.jsonindex 9aced9b84..b7eb7f138 100644--- a/mcp-host/package-lock.json+++ b/mcp-host/package-lock.json@@ -1,12 +1,12 @@{"name": "clerum-mcp-host",- "version": "0.1.310",+ "version": "0.1.311","lockfileVersion": 3,"requires": true,"packages": {"": {"name": "clerum-mcp-host",- "version": "0.1.310",+ "version": "0.1.311","license": "SEE LICENSE IN LICENSE","dependencies": {"@anthropic-ai/sdk": "^0.32.0",diff --git a/mcp-host/package.json b/mcp-host/package.jsonindex 548b1b092..c9c428555 100644--- a/mcp-host/package.json+++ b/mcp-host/package.json@@ -1,6 +1,6 @@{"name": "clerum-mcp-host",- "version": "0.1.310",+ "version": "0.1.311","description": "MCP Host - reads Host CRD and provides LLM access via OpenAI/Claude","main": "dist/main.js","engines": {diff --git a/mcp-host/src/llm/__tests__/codexLlmProxyClient.test.ts b/mcp-host/src/llm/__tests__/codexLlmProxyClient.test.tsindex c6a5548ca..583455635 100644--- a/mcp-host/src/llm/__tests__/codexLlmProxyClient.test.ts+++ b/mcp-host/src/llm/__tests__/codexLlmProxyClient.test.ts@@ -144,4 +144,60 @@ describe('CodexLlmProxyClient', () => {})).rejects.toMatchObject({ code: 'origin_denied' })})++ it('marks only the pre-stream abort as never dispatched', async () => {+ const runtimeUrl = resolveCodexProxyRuntimeUrl(+ 'http://codex-llm-proxy.control-plane.svc.cluster.local:8080'+ )++ const abortedClient = new CodexLlmProxyClient({+ runtimeUrl,+ readPlatformJwt: () => 'platform-jwt',+ fetchFn: vi.fn() as unknown as typeof fetch,+ })+ await expect(+ abortedClient.stream({+ executionTicket: 'ticket-123456',+ requestHash: 'a'.repeat(64),+ request: {},+ signal: AbortSignal.abort(),+ })+ ).rejects.toMatchObject({ code: 'canceled', dispatched: false })++ // The request left the process before control-api answered, so a denial is+ // not proof that nothing was billed.+ const deniedClient = new CodexLlmProxyClient({+ runtimeUrl,+ readPlatformJwt: () => 'platform-jwt',+ fetchFn: vi.fn().mockResolvedValue({+ ok: false,+ status: 403,+ json: async () => ({ error: 'origin_denied' }),+ }) as unknown as typeof fetch,+ })+ await expect(+ deniedClient.stream({+ executionTicket: 'ticket-123456',+ requestHash: 'a'.repeat(64),+ request: {},+ })+ ).rejects.toMatchObject({ code: 'origin_denied', dispatched: true })++ // An error frame arrives mid-stream: the upstream call is already running.+ const framedClient = new CodexLlmProxyClient({+ runtimeUrl,+ readPlatformJwt: () => 'platform-jwt',+ fetchFn: vi.fn().mockResolvedValue({+ ok: true,+ body: sse([{ type: 'error', code: 'rate_limited' }]),+ }) as unknown as typeof fetch,+ })+ await expect(+ framedClient.stream({+ executionTicket: 'ticket-123456',+ requestHash: 'a'.repeat(64),+ request: {},+ })+ ).rejects.toMatchObject({ code: 'rate_limited', dispatched: true })+ })})diff --git a/mcp-host/src/llm/__tests__/codexSubscription.test.ts b/mcp-host/src/llm/__tests__/codexSubscription.test.tsindex fef628868..3bf79c795 100644--- a/mcp-host/src/llm/__tests__/codexSubscription.test.ts+++ b/mcp-host/src/llm/__tests__/codexSubscription.test.ts@@ -279,4 +279,55 @@ describe('CodexSubscriptionProvider', () => {expect(limited.retryable).toBe(true)expect(classifyFailoverClass(limited.code, limited.retryable)).toBe('rate_limited')})++ it('records whether a classified Codex failure had already left the process', () => {+ const provider = new CodexSubscriptionProvider('gpt-5.3-codex', deps() as never)++ // Every CodexAuthorizeError throw site precedes `proxy.stream`.+ expect(+ provider.classifyError(new CodexAuthorizeError('no_grant', 'revoked')).providerDispatched+ ).toBe(false)+ expect(+ provider.classifyError(new CodexAuthorizeError('budget_denied', 'over budget'))+ .providerDispatched+ ).toBe(false)+ expect(+ provider.classifyError(new CodexAuthorizeError('insufficient_scope', 'scope'))+ .providerDispatched+ ).toBe(false)++ // The two pre-stream aborts are the only proxy errors that prove it.+ expect(+ provider.classifyError(new CodexProxyError('canceled', 'aborted before authorize', false))+ .providerDispatched+ ).toBe(false)++ // Anything else on the proxy happened after the request was issued.+ expect(+ provider.classifyError(new CodexProxyError('provider_unavailable', 'upstream 5xx'))+ .providerDispatched+ ).toBe(true)+ expect(+ provider.classifyError(new CodexProxyError('rate_limited', 'upstream 429')).providerDispatched+ ).toBe(true)++ // An unrecognized error is not evidence that no call was made.+ expect(provider.classifyError(new Error('who knows')).providerDispatched).toBeUndefined()+ })++ it('shares the caller deadline with the authorize hop', async () => {+ const wired = deps()+ const provider = new CodexSubscriptionProvider('gpt-5.3-codex', wired as never)+ const controller = new AbortController()++ await provider.completeSingleTurn([{ role: 'user', content: 'hi' }], {+ signal: controller.signal,+ })+ expect(wired.authorize).toHaveBeenCalledWith(expect.any(Object), {+ signal: controller.signal,+ })++ await provider.completeSingleTurn([{ role: 'user', content: 'hi' }])+ expect(wired.authorize).toHaveBeenNthCalledWith(2, expect.any(Object), {})+ })})diff --git a/mcp-host/src/llm/__tests__/providerAttemptAuthorizer.test.ts b/mcp-host/src/llm/__tests__/providerAttemptAuthorizer.test.tsindex 53f63847d..bf699138d 100644--- a/mcp-host/src/llm/__tests__/providerAttemptAuthorizer.test.ts+++ b/mcp-host/src/llm/__tests__/providerAttemptAuthorizer.test.ts@@ -131,4 +131,70 @@ describe('ProviderAttemptAuthorizer', () => {expect(fetchFn.mock.calls[1][1].headers.authorization).toBe('Bearer fresh-jwt')expect(result.executionTicket).toBe('ticket-123456')})++ it('forwards the caller deadline to the authorize hop and to its post-401 retry', async () => {+ const controller = new AbortController()+ const refreshOnUnauthorized = vi.fn().mockResolvedValue(undefined)+ const fetchFn = vi+ .fn()+ .mockResolvedValueOnce({+ ok: false,+ status: 401,+ json: async () => ({ error: 'Unauthorized' }),+ })+ .mockResolvedValueOnce({ ok: true, json: async () => validAuthorize })+ const authorizer = new ProviderAttemptAuthorizer({+ authorizeUrl: resolveCodexAuthorizeUrl('http://gateway:8092'),+ readPlatformJwt: () => 'platform-jwt',+ refreshOnUnauthorized,+ fetchFn: fetchFn as unknown as typeof fetch,+ })++ await authorizer.authorize(+ {+ request: { schemaVersion: 'codex-completion-request.v1' },+ invocationId: 'inv-1',+ attemptGeneration: 1,+ providerAttemptIndex: 1,+ policyRevision: 1,+ policyHash: 'b'.repeat(64),+ },+ { signal: controller.signal }+ )+ // The retry after a refresh must not outlive the deadline that bounded the+ // first hop — a second, unbounded call would be a second clock.+ expect(fetchFn.mock.calls[0][1].signal).toBe(controller.signal)+ expect(fetchFn.mock.calls[1][1].signal).toBe(controller.signal)+ })++ it('fails loudly on an already-expired deadline without rotating credentials', async () => {+ const refreshOnUnauthorized = vi.fn()+ // Faithful to real fetch: an aborted signal rejects before any response.+ const fetchFn = vi.fn(async (_url: string, init: RequestInit) => {+ if (init.signal?.aborted) throw new DOMException('operation aborted', 'AbortError')+ throw new Error('fetch must not proceed with an aborted signal')+ })+ const authorizer = new ProviderAttemptAuthorizer({+ authorizeUrl: resolveCodexAuthorizeUrl('http://gateway:8092'),+ readPlatformJwt: () => 'platform-jwt',+ refreshOnUnauthorized,+ fetchFn: fetchFn as unknown as typeof fetch,+ })++ await expect(+ authorizer.authorize(+ {+ request: {},+ invocationId: 'inv-1',+ attemptGeneration: 1,+ providerAttemptIndex: 1,+ policyRevision: 1,+ policyHash: 'b'.repeat(64),+ },+ { signal: AbortSignal.abort() }+ )+ ).rejects.toThrow(/aborted/i)+ expect(fetchFn).toHaveBeenCalledTimes(1)+ expect(refreshOnUnauthorized).not.toHaveBeenCalled()+ })})diff --git a/mcp-host/src/llm/codexLlmProxyClient.ts b/mcp-host/src/llm/codexLlmProxyClient.tsindex 940a174a4..91a5e5aa6 100644--- a/mcp-host/src/llm/codexLlmProxyClient.ts+++ b/mcp-host/src/llm/codexLlmProxyClient.ts@@ -1,9 +1,16 @@export const CODEX_PROXY_COMPLETIONS_PATH = '/internal/runtime/v1/codex/completions'export class CodexProxyError extends Error {+ /**+ * `dispatched` records whether a request had already left this process when+ * the error was raised. It defaults to `true` because every construction+ * site except the pre-stream abort happens after the fetch was issued, and+ * the safe default is the one that keeps the attempt fenced.+ */constructor(readonly code: string,- message: string+ message: string,+ readonly dispatched: boolean = true) {super(message)this.name = 'CodexProxyError'@@ -50,7 +57,7 @@ export class CodexLlmProxyClient {signal?: AbortSignal}): Promise<CodexProxyStreamResult> {if (input.signal?.aborted) {- throw new CodexProxyError('canceled', 'aborted before proxy stream')+ throw new CodexProxyError('canceled', 'aborted before proxy stream', false)}return this.streamOnce(input, Boolean(this.options.refreshOnUnauthorized))}diff --git a/mcp-host/src/llm/codexSubscription.ts b/mcp-host/src/llm/codexSubscription.tsindex d1b2cbc4b..e6c6a81ba 100644--- a/mcp-host/src/llm/codexSubscription.ts+++ b/mcp-host/src/llm/codexSubscription.ts@@ -115,6 +115,16 @@ export class CodexSubscriptionProvider implements SingleTurnProvider {classifyError(err: unknown): ClassifiedError {const code =err instanceof CodexAuthorizeError || err instanceof CodexProxyError ? err.code : undefined+ // Every CodexAuthorizeError throw site precedes `proxy.stream`, so an+ // authorize failure proves nothing was dispatched. A CodexProxyError knows+ // whether its own request had left the process. Anything else stays+ // undefined — an unrecognized error is not evidence of a missing call.+ const providerDispatched =+ err instanceof CodexAuthorizeError+ ? false+ : err instanceof CodexProxyError+ ? err.dispatched+ : undefinedif (code === 'insufficient_scope' ||code === 'no_grant' ||@@ -126,6 +136,7 @@ export class CodexSubscriptionProvider implements SingleTurnProvider {retryable: false,message: err instanceof Error ? err.message : String(err),providerCode: code,+ ...(providerDispatched !== undefined ? { providerDispatched } : {}),}}if (code === 'budget_denied') {@@ -134,6 +145,7 @@ export class CodexSubscriptionProvider implements SingleTurnProvider {retryable: false,message: err instanceof Error ? err.message : String(err),providerCode: code,+ ...(providerDispatched !== undefined ? { providerDispatched } : {}),}}if (code === 'model_not_allowed') {@@ -142,6 +154,7 @@ export class CodexSubscriptionProvider implements SingleTurnProvider {retryable: false,message: err instanceof Error ? err.message : String(err),providerCode: code,+ ...(providerDispatched !== undefined ? { providerDispatched } : {}),}}if (code === 'rate_limited') {@@ -150,6 +163,7 @@ export class CodexSubscriptionProvider implements SingleTurnProvider {retryable: true,message: err instanceof Error ? err.message : String(err),providerCode: code,+ ...(providerDispatched !== undefined ? { providerDispatched } : {}),}}if (code === 'provider_unavailable' || code === 'connection_unavailable') {@@ -158,6 +172,7 @@ export class CodexSubscriptionProvider implements SingleTurnProvider {retryable: true,message: err instanceof Error ? err.message : String(err),providerCode: code,+ ...(providerDispatched !== undefined ? { providerDispatched } : {}),}}if (code) {@@ -166,6 +181,7 @@ export class CodexSubscriptionProvider implements SingleTurnProvider {retryable: false,message: err instanceof Error ? err.message : String(err),providerCode: code,+ ...(providerDispatched !== undefined ? { providerDispatched } : {}),}}return classifyUnknown(err)@@ -236,7 +252,7 @@ export class CodexSubscriptionProvider implements SingleTurnProvider {}) {if (options?.signal?.aborted) {- throw new CodexProxyError('canceled', 'aborted before authorize')+ throw new CodexProxyError('canceled', 'aborted before authorize', false)}const request = this.buildRequest(messages, tools, options)const requestHash = hashCodexCompletionRequestV1(request)@@ -249,23 +265,29 @@ export class CodexSubscriptionProvider implements SingleTurnProvider {throw new CodexAuthorizeError('no_grant', 'Codex catalog policy binding is missing')}const providerAttemptIndex = context.providerAttemptIndex ?? this.nextProviderAttemptIndex++- const authorized = await this.deps.authorizer.authorize({- request,- requestHash,- invocationId: context.invocationId ?? request.requestId,- attemptGeneration: context.attemptGeneration ?? 1,- providerAttemptIndex,- policyRevision: context.policyRevision,- policyHash: context.policyHash,- hostRef: context.hostRef,- recipeNamespace: context.recipeNamespace,- recipeName: context.recipeName,- userId: context.userId,- ...(context.pluginWorkloadSdkProviderAttemptId- ? { pluginWorkloadSdkProviderAttemptId: context.pluginWorkloadSdkProviderAttemptId }- : {}),- ...(context.targetRef ? { targetRef: context.targetRef } : {}),- })+ const authorized = await this.deps.authorizer.authorize(+ {+ request,+ requestHash,+ invocationId: context.invocationId ?? request.requestId,+ attemptGeneration: context.attemptGeneration ?? 1,+ providerAttemptIndex,+ policyRevision: context.policyRevision,+ policyHash: context.policyHash,+ hostRef: context.hostRef,+ recipeNamespace: context.recipeNamespace,+ recipeName: context.recipeName,+ userId: context.userId,+ ...(context.pluginWorkloadSdkProviderAttemptId+ ? { pluginWorkloadSdkProviderAttemptId: context.pluginWorkloadSdkProviderAttemptId }+ : {}),+ ...(context.targetRef ? { targetRef: context.targetRef } : {}),+ },+ // Authorize shares the caller's deadline. Without this the attempt could+ // keep a cancelled request alive on the control-api hop while the bridge+ // has already given up on it.+ { ...(options?.signal ? { signal: options.signal } : {}) }+ )if (!('accessToken' in authorized)) {// Bound: authorize returns ticket material only.}diff --git a/mcp-host/src/llm/failover/engine.ts b/mcp-host/src/llm/failover/engine.tsindex 68857b632..a091f8484 100644--- a/mcp-host/src/llm/failover/engine.ts+++ b/mcp-host/src/llm/failover/engine.ts@@ -42,6 +42,12 @@ export interface ClassifiedLike {retryable: boolean/** Provider-native code (e.g. `budget_denied`) used for terminal gates. */providerCode?: string+ /**+ * Mirrors `ClassifiedError.providerDispatched`: `false` only when the+ * classifier proved no call left the process. `undefined` is unknown and+ * must be read as dispatched.+ */+ providerDispatched?: boolean}export interface FailoverEngineOptions {diff --git a/mcp-host/src/llm/providerAttemptAuthorizer.ts b/mcp-host/src/llm/providerAttemptAuthorizer.tsindex a47b8c6ee..0fd18520b 100644--- a/mcp-host/src/llm/providerAttemptAuthorizer.ts+++ b/mcp-host/src/llm/providerAttemptAuthorizer.ts@@ -48,18 +48,28 @@ export class ProviderAttemptAuthorizer {}}- async authorize(body: AuthorizeAttemptBody): Promise<{+ /**+ * The caller's `signal` is the only clock on this hop. Authorize deliberately+ * has no timeout of its own: the bridge already bounds the whole attempt, and+ * a second independent deadline would make two different answers possible for+ * "did this attempt still have time?".+ */+ async authorize(+ body: AuthorizeAttemptBody,+ options?: { signal?: AbortSignal }+ ): Promise<{providerAttemptId: stringrequestHash: stringexecutionTicket: stringexpiresAt: string}> {- return this.authorizeOnce(body, Boolean(this.options.refreshOnUnauthorized))+ return this.authorizeOnce(body, Boolean(this.options.refreshOnUnauthorized), options?.signal)}private async authorizeOnce(body: AuthorizeAttemptBody,- retryOnUnauthorized: boolean+ retryOnUnauthorized: boolean,+ signal?: AbortSignal): Promise<{providerAttemptId: stringrequestHash: string@@ -78,12 +88,13 @@ export class ProviderAttemptAuthorizer {'content-type': 'application/json',},body: JSON.stringify(body),+ ...(signal ? { signal } : {}),})const payload = (await response.json().catch(() => ({}))) as Record<string, unknown>if (!response.ok) {if (response.status === 401 && retryOnUnauthorized && this.options.refreshOnUnauthorized) {await this.options.refreshOnUnauthorized()- return this.authorizeOnce(body, false)+ return this.authorizeOnce(body, false, signal)}const code = typeof payload.error === 'string' ? payload.error : 'provider_unavailable'throw new CodexAuthorizeError(code, `authorize failed with ${response.status}`)diff --git a/mcp-host/src/llm/types.ts b/mcp-host/src/llm/types.tsindex 1766381b4..4afd896fb 100644--- a/mcp-host/src/llm/types.ts+++ b/mcp-host/src/llm/types.ts@@ -45,6 +45,18 @@ export interface ClassifiedError {*/httpStatus?: numberproviderCode?: string+ /**+ * Execution-phase signal. `false` means the classifier PROVED that no call+ * left this process (the attempt died before authorize or before the proxy+ * stream), so the physical attempt cannot have billed. `true` means a call+ * left the process or may have. `undefined` means the classifier does not+ * know — consumers must assume the expensive direction (dispatched).+ *+ * Only a classifier that can prove the negative may set `false`: marking a+ * dispatched attempt as `false` would leave the idempotency key revivable+ * and permit a second billable call.+ */+ providerDispatched?: boolean}/**diff --git a/mcp-host/src/pluginWorkloadSdk/promptBridge/controlApiClient.test.ts b/mcp-host/src/pluginWorkloadSdk/promptBridge/controlApiClient.test.tsindex 464f6ef74..9ec3361f7 100644--- a/mcp-host/src/pluginWorkloadSdk/promptBridge/controlApiClient.test.ts+++ b/mcp-host/src/pluginWorkloadSdk/promptBridge/controlApiClient.test.ts@@ -544,6 +544,97 @@ describe('PluginWorkloadSdkControlApiClient', () => {})})+ it('surfaces ledger_pending as the retryable-with-reason finalize triple', async () => {+ // The handler retries a finalize ONLY on this exact triple. If control-api+ // ever stops sending `reason`, or the client stops mapping it, the retry+ // silently disappears — so the triple is asserted end to end here.+ const fetchImpl = vi.fn().mockResolvedValue(+ jsonResponse(409, {+ error: 'provider_unavailable',+ message: 'linked Codex attempt has not finalized usage yet',+ retryable: true,+ reason: 'provider_unavailable',+ })+ )+ const client = makeClient(fetchImpl as unknown as typeof fetch)+ const failure = await client+ .finalizePromptBridge({+ recipeNamespace: 'sandbox-recipes',+ recipeName: 'r1',+ invocationId: 'inv-1',+ attemptGeneration: 1,+ providerAttemptId: 'attempt-1',+ providerAttemptIndex: 1,+ status: 'complete',+ reason: 'provider_completed',+ target,+ })+ .catch((err: unknown) => err)+ expect(failure).toBeInstanceOf(PluginWorkloadError)+ expect(failure).toMatchObject({+ code: 'provider_unavailable',+ retryable: true,+ reason: 'provider_unavailable',+ })+ // A 409 is not retried by the transport itself; the ledger retry lives in+ // the handler, where the deadline can bound it.+ expect(fetchImpl).toHaveBeenCalledTimes(1)+ })++ it('threads a caller deadline into the finalize request and stops retrying once aborted', async () => {+ const controller = new AbortController()+ const fetchImpl = vi.fn().mockResolvedValue(+ jsonResponse(201, {+ invocationId: 'inv-1',+ providerAttemptId: 'attempt-1',+ status: 'complete',+ outcome: 'exact',+ idempotent: false,+ usageAccepted: true,+ })+ )+ const client = makeClient(fetchImpl as unknown as typeof fetch)+ await client.finalizePromptBridge(+ {+ recipeNamespace: 'sandbox-recipes',+ recipeName: 'r1',+ invocationId: 'inv-1',+ attemptGeneration: 1,+ providerAttemptId: 'attempt-1',+ providerAttemptIndex: 1,+ status: 'complete',+ reason: 'provider_completed',+ target,+ },+ { signal: controller.signal }+ )+ const [, init] = fetchImpl.mock.calls[0] as [string, RequestInit]+ expect(init.signal).toBeInstanceOf(AbortSignal)+ expect(init.signal?.aborted).toBe(false)++ // An already-expired deadline must not buy a single transport attempt: the+ // AbortError would otherwise be swallowed by the network-retry branch.+ const abortedFetch = vi.fn()+ const abortedClient = makeClient(abortedFetch as unknown as typeof fetch)+ await expect(+ abortedClient.finalizePromptBridge(+ {+ recipeNamespace: 'sandbox-recipes',+ recipeName: 'r1',+ invocationId: 'inv-1',+ attemptGeneration: 1,+ providerAttemptId: 'attempt-1',+ providerAttemptIndex: 1,+ status: 'complete',+ reason: 'provider_completed',+ target,+ },+ { signal: AbortSignal.abort() }+ )+ ).rejects.toMatchObject({ code: 'provider_unavailable', retryable: true })+ expect(abortedFetch).not.toHaveBeenCalled()+ })+it('submitClientNotification maps event_type_not_allowed', async () => {const fetchImpl = vi.fn()diff --git a/mcp-host/src/pluginWorkloadSdk/promptBridge/controlApiClient.ts b/mcp-host/src/pluginWorkloadSdk/promptBridge/controlApiClient.tsindex 8abe68351..efeff34e2 100644--- a/mcp-host/src/pluginWorkloadSdk/promptBridge/controlApiClient.ts+++ b/mcp-host/src/pluginWorkloadSdk/promptBridge/controlApiClient.ts@@ -319,9 +319,9 @@ export class PluginWorkloadSdkControlApiClient {* no refresh hook) propagates the error so a genuinely bad credential still* surfaces instead of looping.*/- private async post(path: string, body: unknown): Promise<unknown> {+ private async post(path: string, body: unknown, signal?: AbortSignal): Promise<unknown> {try {- return await this.postOnce(path, body)+ return await this.postOnce(path, body, signal)} catch (err) {if (err instanceof PluginWorkloadError &&@@ -329,7 +329,7 @@ export class PluginWorkloadSdkControlApiClient {this.opts.refreshOnUnauthorized) {await this.triggerRefresh()- return this.postOnce(path, body)+ return this.postOnce(path, body, signal)}throw err}@@ -363,7 +363,7 @@ export class PluginWorkloadSdkControlApiClient {}}- private async postOnce(path: string, body: unknown): Promise<unknown> {+ private async postOnce(path: string, body: unknown, signal?: AbortSignal): Promise<unknown> {if (!this.breaker.allow()) {throw new PluginWorkloadError('provider_unavailable',@@ -379,6 +379,10 @@ export class PluginWorkloadSdkControlApiClient {let lastError: unknown = nullfor (let attempt = 0; attempt <= maxRetries; attempt++) {+ // An aborted caller deadline is not a transient network failure: without+ // this the `catch` below would swallow every AbortError as retryable and+ // burn the remaining attempts after the operation was already over.+ if (signal?.aborted) breakif (attempt > 0) await sleep(baseDelay * 2 ** (attempt - 1))let response: Responsetry {@@ -389,7 +393,12 @@ export class PluginWorkloadSdkControlApiClient {Authorization: `Bearer ${this.opts.getAccessToken()}`,},body: JSON.stringify(body),- signal: AbortSignal.timeout(15_000),+ // The per-request timeout still bounds one hop; a caller-supplied+ // signal additionally bounds the whole operation across retries, so+ // a finalize deadline cannot be outlived by this loop.+ signal: signal+ ? AbortSignal.any([AbortSignal.timeout(15_000), signal])+ : AbortSignal.timeout(15_000),})} catch (err) {// Network failure — retryable.@@ -818,25 +827,28 @@ export class PluginWorkloadSdkControlApiClient {* unknown spend. This replaces separate status + usage calls for the* stepless runtime; the route is idempotent on providerAttemptId.*/- async finalizePromptBridge(body: {- recipeNamespace: string- recipeName: string- invocationId: string- attemptGeneration: number- providerAttemptId: string- providerAttemptIndex: number- status: 'complete' | 'failed' | 'provider_unavailable'- reason: string- target: PromptBridgeTarget- usage?: {- llmSecretName: string- callerRef: string- fallbackUsed: boolean- attemptCount: number- inputTokens: number- outputTokens: number- }- }): Promise<FinalizePromptBridgeResponse> {+ async finalizePromptBridge(+ body: {+ recipeNamespace: string+ recipeName: string+ invocationId: string+ attemptGeneration: number+ providerAttemptId: string+ providerAttemptIndex: number+ status: 'complete' | 'failed' | 'provider_unavailable'+ reason: string+ target: PromptBridgeTarget+ usage?: {+ llmSecretName: string+ callerRef: string+ fallbackUsed: boolean+ attemptCount: number+ inputTokens: number+ outputTokens: number+ }+ },+ options?: { signal?: AbortSignal }+ ): Promise<FinalizePromptBridgeResponse> {const result = await this.post(`/api/v1/mcp-host/plugin-workload-sdk/invocations/${encodeURIComponent(body.invocationId)}/finalize`,{@@ -850,7 +862,8 @@ export class PluginWorkloadSdkControlApiClient {reason: body.reason,target: body.target,...(body.usage ? { usage: body.usage } : {}),- }+ },+ options?.signal)if (!isFinalizePromptBridgeResponse(result)) {throw new PluginWorkloadError(diff --git a/mcp-host/src/pluginWorkloadSdk/promptBridge/handler.test.ts b/mcp-host/src/pluginWorkloadSdk/promptBridge/handler.test.tsindex 78a6f98a8..f965bd516 100644--- a/mcp-host/src/pluginWorkloadSdk/promptBridge/handler.test.ts+++ b/mcp-host/src/pluginWorkloadSdk/promptBridge/handler.test.ts@@ -3,9 +3,11 @@ import { PluginWorkloadError } from '../domain/errors'import type { PromptBridgeTarget } from '../domain/types'import type { PluginWorkloadSdkControlApiClient } from './controlApiClient'import {+ FINALIZE_LEDGER_RETRY_DELAYS_MS,PromptBridgeHandler,type PromptBridgeHandlerDeps,- finalizeLedgerRetryDelayMs,+ finalizeWithLedgerRetry,+ isLedgerPendingError,} from './handler'import type { LlmBridge } from './llmBridge'@@ -327,7 +329,8 @@ describe('PromptBridgeHandler', () => {status: 'failed',reason: 'credential_unavailable',target: primary,- })+ }),+ expect.objectContaining({ signal: expect.any(AbortSignal) }))expect(report).not.toHaveBeenCalled()})@@ -401,7 +404,8 @@ describe('PromptBridgeHandler', () => {status: 'complete',target: fallback,usage: expect.objectContaining({ inputTokens: 10, outputTokens: 5 }),- })+ }),+ expect.objectContaining({ signal: expect.any(AbortSignal) }))expect(report).not.toHaveBeenCalled()expect(onUsage).not.toHaveBeenCalled()@@ -444,16 +448,222 @@ describe('PromptBridgeHandler', () => {status: 'provider_unavailable',reason: 'outcome_unknown',target: primary,- })+ }),+ expect.objectContaining({ signal: expect.any(AbortSignal) }))expect(report).not.toHaveBeenCalled()})- it('uses seconds-scale backoff for ledger_pending retries', () => {- expect(finalizeLedgerRetryDelayMs(0)).toBe(500)- expect(finalizeLedgerRetryDelayMs(1)).toBe(1000)- expect(finalizeLedgerRetryDelayMs(2)).toBe(2000)- expect(finalizeLedgerRetryDelayMs(3)).toBe(4000)+ it('bounds ledger_pending patience to a fixed four-attempt table', () => {+ expect(FINALIZE_LEDGER_RETRY_DELAYS_MS).toEqual([500, 1000, 2000])+ // Four attempts, three waits: the table length IS the retry budget.+ expect(FINALIZE_LEDGER_RETRY_DELAYS_MS.reduce((sum, ms) => sum + ms, 0)).toBe(3_500)+ })++ it('recognises only the ledger_pending triple as a retryable finalize error', () => {+ expect(+ isLedgerPendingError(+ new PluginWorkloadError(+ 'provider_unavailable',+ 'linked Codex attempt has not finalized usage yet',+ true,+ 'provider_unavailable'+ )+ )+ ).toBe(true)+ // Retryable but reasonless: control-api 5xx, open breaker, exhausted+ // transport retries. Those already burnt their own backoff inside postOnce.+ expect(+ isLedgerPendingError(+ new PluginWorkloadError('provider_unavailable', 'control-api responded 503', true)+ )+ ).toBe(false)+ expect(+ isLedgerPendingError(+ new PluginWorkloadError(+ 'provider_unavailable',+ 'not retryable',+ false,+ 'provider_unavailable'+ )+ )+ ).toBe(false)+ expect(+ isLedgerPendingError(+ new PluginWorkloadError('provider_unavailable', 'other reason', true, 'timeout')+ )+ ).toBe(false)+ expect(isLedgerPendingError(new Error('plain'))).toBe(false)+ })++ it('does not retry a retryable finalize error that is not ledger_pending', async () => {+ const finalize = vi+ .fn<FinalizePromptBridge>()+ .mockRejectedValue(+ new PluginWorkloadError('provider_unavailable', 'control-api responded 503', true)+ )+ const complete = vi.fn().mockResolvedValue({+ model: fallback.model,+ servedTarget: fallback,+ fallbackUsed: true,+ attemptCount: 2,+ llmSecretName: 'provider-secret',+ providerAttemptId: 'attempt-1',+ providerAttemptIndex: 2,+ content: 'summary text',+ usage: { inputTokens: 10, outputTokens: 5 },+ finishReason: 'complete',+ })+ const { handler } = makeDeps({ complete, finalize })++ await expect(handler.handle(validBody, 'api')).rejects.toMatchObject({+ code: 'provider_unavailable',+ reason: 'outcome_unknown',+ })+ // One attempt on the complete path, then one on the error path (N-11).+ // Neither retries: the error is retryable but carries no `reason`.+ expect(finalize).toHaveBeenCalledTimes(2)+ expect(finalize).toHaveBeenNthCalledWith(+ 1,+ expect.objectContaining({ status: 'complete' }),+ expect.objectContaining({ signal: expect.any(AbortSignal) })+ )+ expect(finalize).toHaveBeenNthCalledWith(+ 2,+ expect.objectContaining({ status: 'provider_unavailable' }),+ expect.objectContaining({ signal: expect.any(AbortSignal) })+ )+ })++ it('exhausts exactly the table budget on a persistent ledger_pending', async () => {+ const finalize = vi+ .fn<FinalizePromptBridge>()+ .mockRejectedValue(+ new PluginWorkloadError(+ 'provider_unavailable',+ 'linked Codex attempt has not finalized usage yet',+ true,+ 'provider_unavailable'+ )+ )+ const complete = vi.fn().mockResolvedValue({+ model: fallback.model,+ servedTarget: fallback,+ fallbackUsed: true,+ attemptCount: 2,+ llmSecretName: 'provider-secret',+ providerAttemptId: 'attempt-1',+ providerAttemptIndex: 2,+ content: 'summary text',+ usage: { inputTokens: 10, outputTokens: 5 },+ finishReason: 'complete',+ })+ const { handler } = makeDeps({ complete, finalize })++ vi.useFakeTimers()+ try {+ // Attach the rejection handler before advancing the clock, or the+ // in-flight rejection surfaces as an unhandled rejection.+ const settled = expect(handler.handle(validBody, 'api')).rejects.toMatchObject({+ code: 'provider_unavailable',+ reason: 'outcome_unknown',+ })+ await vi.runAllTimersAsync()+ await settled+ } finally {+ vi.useRealTimers()+ }+ // Four attempts per finalize path, never five. The complete path exhausts+ // its budget, then the error path (N-11) settles the same receipt with a+ // budget of its own — the documented worst case of two bounded deadlines.+ expect(finalize).toHaveBeenCalledTimes((FINALIZE_LEDGER_RETRY_DELAYS_MS.length + 1) * 2)+ })++ it('surfaces ledger_pending without another wait once its deadline is aborted', async () => {+ const pendingError = new PluginWorkloadError(+ 'provider_unavailable',+ 'linked Codex attempt has not finalized usage yet',+ true,+ 'provider_unavailable'+ )+ const finalize = vi.fn<FinalizePromptBridge>().mockRejectedValue(pendingError)++ await expect(+ finalizeWithLedgerRetry(+ finalize,+ {+ invocationId: 'inv-1',+ attemptGeneration: 1,+ providerAttemptId: 'attempt-1',+ providerAttemptIndex: 1,+ status: 'complete',+ reason: 'provider_completed',+ target: primary,+ },+ AbortSignal.abort()+ )+ ).rejects.toBe(pendingError)+ // An expired deadline buys neither another attempt nor another sleep, and+ // the real pending error reaches the caller instead of a synthetic one.+ expect(finalize).toHaveBeenCalledTimes(1)+ })++ it('retries ledger_pending on the error path with the same bounded budget', async () => {+ const finalize = vi+ .fn<FinalizePromptBridge>()+ .mockRejectedValueOnce(+ new PluginWorkloadError(+ 'provider_unavailable',+ 'linked Codex attempt has not finalized usage yet',+ true,+ 'provider_unavailable'+ )+ )+ .mockResolvedValueOnce({+ invocationId: 'inv-1',+ providerAttemptId: 'attempt-1',+ status: 'provider_unavailable',+ outcome: 'unknown',+ idempotent: false,+ usageAccepted: false,+ })+ const complete = vi.fn().mockRejectedValue(+ new PluginWorkloadError(+ 'provider_unavailable',+ 'provider outcome unknown',+ false,+ 'outcome_unknown',+ true,+ {+ providerAttemptId: 'attempt-1',+ providerAttemptIndex: 1,+ target: primary,+ attemptCount: 1,+ fallbackUsed: false,+ }+ )+ )+ const { handler, report } = makeDeps({ complete, finalize })++ vi.useFakeTimers()+ try {+ const settled = expect(handler.handle(validBody, 'api')).rejects.toMatchObject({+ code: 'provider_unavailable',+ reason: 'outcome_unknown',+ })+ await vi.runAllTimersAsync()+ await settled+ } finally {+ vi.useRealTimers()+ }+ expect(finalize).toHaveBeenCalledTimes(2)+ expect(finalize).toHaveBeenNthCalledWith(+ 2,+ expect.objectContaining({ status: 'provider_unavailable' }),+ expect.objectContaining({ signal: expect.any(AbortSignal) })+ )+ // The retry succeeded, so the legacy invocation-status report stays silent.+ expect(report).not.toHaveBeenCalled()})it('retries a retryable complete finalization before treating the outcome as unknown', async () => {@@ -501,8 +711,16 @@ describe('PromptBridgeHandler', () => {vi.useRealTimers()}expect(finalize).toHaveBeenCalledTimes(2)- expect(finalize).toHaveBeenNthCalledWith(1, expect.objectContaining({ status: 'complete' }))- expect(finalize).toHaveBeenNthCalledWith(2, expect.objectContaining({ status: 'complete' }))+ expect(finalize).toHaveBeenNthCalledWith(+ 1,+ expect.objectContaining({ status: 'complete' }),+ expect.objectContaining({ signal: expect.any(AbortSignal) })+ )+ expect(finalize).toHaveBeenNthCalledWith(+ 2,+ expect.objectContaining({ status: 'complete' }),+ expect.objectContaining({ signal: expect.any(AbortSignal) })+ )expect(report).not.toHaveBeenCalled()})@@ -537,10 +755,15 @@ describe('PromptBridgeHandler', () => {reason: 'outcome_unknown',})expect(finalize).toHaveBeenCalledTimes(2)- expect(finalize).toHaveBeenNthCalledWith(1, expect.objectContaining({ status: 'complete' }))+ expect(finalize).toHaveBeenNthCalledWith(+ 1,+ expect.objectContaining({ status: 'complete' }),+ expect.objectContaining({ signal: expect.any(AbortSignal) })+ )expect(finalize).toHaveBeenNthCalledWith(2,- expect.objectContaining({ status: 'provider_unavailable', reason: 'outcome_unknown' })+ expect.objectContaining({ status: 'provider_unavailable', reason: 'outcome_unknown' }),+ expect.objectContaining({ signal: expect.any(AbortSignal) }))expect(report).not.toHaveBeenCalled()})diff --git a/mcp-host/src/pluginWorkloadSdk/promptBridge/handler.ts b/mcp-host/src/pluginWorkloadSdk/promptBridge/handler.tsindex b78dbde30..dc333f53f 100644--- a/mcp-host/src/pluginWorkloadSdk/promptBridge/handler.ts+++ b/mcp-host/src/pluginWorkloadSdk/promptBridge/handler.ts@@ -36,35 +36,78 @@ import type { LlmBridge } from './llmBridge'const IDEMPOTENCY_KEY_RE = new RegExp(DEFAULT_IDEMPOTENCY_KEY_PATTERN)const PURPOSE_SET = new Set<string>(PLUGIN_WORKLOAD_SDK_PURPOSES)const MAX_METADATA_BYTES = 16 * 1024-const FINALIZE_LEDGER_RETRIES = 4-/** Seconds-scale backoff so a Codex usage frame can land before ledger_pending expires. */-export function finalizeLedgerRetryDelayMs(attempt: number): number {- return Math.min(4_000, 500 * 2 ** attempt)+/**+ * Four attempts, three waits (3.5 s total). `ledger_pending` at this point is a+ * commit race of milliseconds between the proxy's own finalize — which runs in+ * its `finally`, before the `done` frame — and this read. The 900 s the sweeper+ * waits is the sweeper's patience, not a client budget: a proxy that already+ * exhausted its attempts will not land no matter how long the host waits here.+ *+ * No jitter: finalizations serialize per recipe on the advisory lock rather+ * than competing in a thundering herd, and a deterministic table is the part a+ * test can actually assert.+ */+export const FINALIZE_LEDGER_RETRY_DELAYS_MS = [500, 1_000, 2_000] as const++/**+ * Whole-finalize deadline, retries included. Worst case after the provider has+ * already answered is two of these (complete path, then error path) instead of+ * the 65-250 s an unfiltered retry of every retryable error could reach.+ */+const FINALIZE_DEADLINE_MS = 30_000++/**+ * `ledger_pending` is the only 409 the finalize route marks retryable AND+ * names with a `reason`. Every other retryable finalize failure (5xx, open+ * breaker, exhausted transport retries, unexpected response shape) arrives+ * WITHOUT a reason and has already burnt four fetches with backoff inside+ * `postOnce`; repeating those four more times outside is amplification with no+ * new information. The triple is asserted as a contract in the client tests.+ */+export function isLedgerPendingError(err: unknown): boolean {+ return (+ err instanceof PluginWorkloadError &&+ err.code === 'provider_unavailable' &&+ err.retryable === true &&+ err.reason === 'provider_unavailable'+ )+}++/** Resolves on the timer or on abort, whichever comes first. */+function sleepUnlessAborted(ms: number, signal: AbortSignal): Promise<void> {+ return new Promise(resolve => {+ let timer: NodeJS.Timeout+ const finish = () => {+ clearTimeout(timer)+ signal.removeEventListener('abort', finish)+ resolve()+ }+ timer = setTimeout(finish, ms)+ signal.addEventListener('abort', finish, { once: true })+ })}-async function finalizeCompleteWithLedgerRetry(+export async function finalizeWithLedgerRetry(finalize: NonNullable<PromptBridgeHandlerDeps['finalizePromptBridge']>,- input: Parameters<NonNullable<PromptBridgeHandlerDeps['finalizePromptBridge']>>[0]+ input: Parameters<NonNullable<PromptBridgeHandlerDeps['finalizePromptBridge']>>[0],+ signal: AbortSignal): Promise<void> {- let lastError: unknown- for (let attempt = 0; attempt < FINALIZE_LEDGER_RETRIES; attempt++) {+ for (let attempt = 0; ; attempt++) {try {- await finalize(input)+ await finalize(input, { signal })return} catch (error) {- lastError = error- if (- !(error instanceof PluginWorkloadError) ||- error.retryable !== true ||- attempt === FINALIZE_LEDGER_RETRIES - 1- ) {+ if (attempt >= FINALIZE_LEDGER_RETRY_DELAYS_MS.length || !isLedgerPendingError(error)) {throw error}- await new Promise(resolve => setTimeout(resolve, finalizeLedgerRetryDelayMs(attempt)))+ // The deadline surfaces the real pending error rather than a synthetic+ // one, so the caller's fence sees why the ledger never settled.+ if (signal.aborted) throw error+ await sleepUnlessAborted(FINALIZE_LEDGER_RETRY_DELAYS_MS[attempt], signal)+ if (signal.aborted) throw error}}- throw lastError}/**@@ -110,23 +153,26 @@ export interface PromptBridgeHandlerDeps {/** Public provider/model identity from the live spec.agent binding. */getBootstrapTarget?: () => { provider: string; model: string } | null/** SDK-only atomic accounting boundary; workflow mode keeps the legacy reporter. */- finalizePromptBridge?: (input: {- invocationId: string- attemptGeneration: number- providerAttemptId: string- providerAttemptIndex: number- status: 'complete' | 'failed' | 'provider_unavailable'- reason: string- target: PromptBridgeTarget- usage?: {- llmSecretName: string- callerRef: string- fallbackUsed: boolean- attemptCount: number- inputTokens: number- outputTokens: number- }- }) => Promise<unknown>+ finalizePromptBridge?: (+ input: {+ invocationId: string+ attemptGeneration: number+ providerAttemptId: string+ providerAttemptIndex: number+ status: 'complete' | 'failed' | 'provider_unavailable'+ reason: string+ target: PromptBridgeTarget+ usage?: {+ llmSecretName: string+ callerRef: string+ fallbackUsed: boolean+ attemptCount: number+ inputTokens: number+ outputTokens: number+ }+ },+ options?: { signal?: AbortSignal }+ ) => Promise<unknown>onUsage?: (usage: {invocationId: stringattemptGeneration: number@@ -418,26 +464,30 @@ export class PromptBridgeHandler {providerAttemptIndex >= 1) {try {- await finalizeCompleteWithLedgerRetry(this.deps.finalizePromptBridge, {- invocationId: authorized.invocationId,- attemptGeneration: authorized.attemptGeneration,- providerAttemptId,- providerAttemptIndex,- status: 'complete',- reason:- providerAttemptAcknowledgement === 'failed'- ? 'provider_completion_acknowledgement_failed'- : 'provider_completed',- target: completion.servedTarget,- usage: {- llmSecretName: completion.llmSecretName,- callerRef,- fallbackUsed: completion.fallbackUsed,- attemptCount: completion.attemptCount,- inputTokens: completion.usage.inputTokens,- outputTokens: completion.usage.outputTokens,+ await finalizeWithLedgerRetry(+ this.deps.finalizePromptBridge,+ {+ invocationId: authorized.invocationId,+ attemptGeneration: authorized.attemptGeneration,+ providerAttemptId,+ providerAttemptIndex,+ status: 'complete',+ reason:+ providerAttemptAcknowledgement === 'failed'+ ? 'provider_completion_acknowledgement_failed'+ : 'provider_completed',+ target: completion.servedTarget,+ usage: {+ llmSecretName: completion.llmSecretName,+ callerRef,+ fallbackUsed: completion.fallbackUsed,+ attemptCount: completion.attemptCount,+ inputTokens: completion.usage.inputTokens,+ outputTokens: completion.usage.outputTokens,+ },},- })+ AbortSignal.timeout(FINALIZE_DEADLINE_MS)+ )} catch {// Returning provider content before durable finalization would hide// a possible spend. Surface an explicit unknown outcome instead.@@ -535,16 +585,24 @@ export class PromptBridgeHandler {err instanceof PluginWorkloadError && err.providerMayHaveExecuted? 'provider_unavailable': 'failed'- await this.deps.finalizePromptBridge({- invocationId: authorized.invocationId,- attemptGeneration: authorized.attemptGeneration,- providerAttemptId: unknownProviderAttempt.providerAttemptId,- providerAttemptIndex: unknownProviderAttempt.providerAttemptIndex,- status: finalizationStatus,- reason:- err instanceof PluginWorkloadError && err.reason ? err.reason : 'outcome_unknown',- target: unknownProviderAttempt.target,- })+ // N-11: the error path settles the same ledger row as the complete+ // path, so it needs the same `ledger_pending` patience and the same+ // bounded deadline. Losing this attempt to a millisecond commit race+ // would silently drop the spend for an attempt that already ran.+ await finalizeWithLedgerRetry(+ this.deps.finalizePromptBridge,+ {+ invocationId: authorized.invocationId,+ attemptGeneration: authorized.attemptGeneration,+ providerAttemptId: unknownProviderAttempt.providerAttemptId,+ providerAttemptIndex: unknownProviderAttempt.providerAttemptIndex,+ status: finalizationStatus,+ reason:+ err instanceof PluginWorkloadError && err.reason ? err.reason : 'outcome_unknown',+ target: unknownProviderAttempt.target,+ },+ AbortSignal.timeout(FINALIZE_DEADLINE_MS)+ )providerAttemptFinalized = true} catch {console.warn('[PluginWorkloadSdk] provider spend finalization failed')diff --git a/mcp-host/src/pluginWorkloadSdk/promptBridge/llmBridge.failover.test.ts b/mcp-host/src/pluginWorkloadSdk/promptBridge/llmBridge.failover.test.tsindex 5ec256ffe..c5b95924d 100644--- a/mcp-host/src/pluginWorkloadSdk/promptBridge/llmBridge.failover.test.ts+++ b/mcp-host/src/pluginWorkloadSdk/promptBridge/llmBridge.failover.test.ts@@ -33,7 +33,12 @@ const fallback: PromptBridgeTarget = {class FakeProvider implements Partial<SingleTurnProvider> {constructor(private readonly behavior: () => Promise<typeof OK>,- private readonly classification: { code: LlmErrorCode; retryable: boolean } = {+ private readonly classification: {+ code: LlmErrorCode+ retryable: boolean+ providerCode?: string+ providerDispatched?: boolean+ } = {code: LlmErrorCode.RateLimited,retryable: true,}@@ -782,3 +787,318 @@ describe('LlmBridge authorized multi-provider fallback', () => {expect(provider.completeSingleTurn).not.toHaveBeenCalled()})})++const codexTarget: PromptBridgeTarget = {+ targetRef: 'codex-primary',+ provider: 'codex-subscription',+ model: 'gpt-5.1',+ credentialSlot: '',+ connectionRef: 'team-plus',+}++function installCodexBinding(): void {+ replaceSdkOnlyCodexBinding({+ connectionKey: 'team-plus',+ catalogRevision: 1,+ credentialRevision: 0,+ model: codexTarget.model,+ bindingHash: computeCodexPolicyHash({+ model: codexTarget.model,+ catalogRevision: 1,+ credentialRevision: 0,+ connectionKey: 'team-plus',+ }),+ })+}++/** Codex reserves index 1; any api-key fallback behind it reserves index 2. */+function codexTicketIssuer() {+ return {+ issue: vi.fn(async ({ target }: { target: PromptBridgeTarget }) => ({+ credentialTicket: target.provider === 'codex-subscription' ? '' : `fresh-${target.targetRef}`,+ providerAttemptId: `sdk-${target.targetRef}`,+ providerAttemptIndex: target.provider === 'codex-subscription' ? 1 : 2,+ })),+ }+}++const codexReceipt = {+ providerAttemptId: `sdk-${codexTarget.targetRef}`,+ providerAttemptIndex: 1,+ target: codexTarget,+ attemptCount: 1,+ fallbackUsed: false,+}++describe('LlmBridge oauth-broker terminal accounting', () => {+ afterEach(() => {+ replaceSdkOnlyCodexBinding(null)+ })++ it('keeps a pre-dispatch Codex budget denial revivable and carries its receipt', async () => {+ installCodexBinding()+ const codex = new FakeProvider(() => Promise.reject(new Error('budget denied')), {+ code: LlmErrorCode.InsufficientQuota,+ retryable: false,+ providerCode: 'budget_denied',+ providerDispatched: false,+ })+ const second = new FakeProvider(() => Promise.resolve(OK))+ const { bridge, providerCalls } = makeBridge({+ [codexTarget.model]: codex,+ [fallback.model]: second,+ })+ const providerAttemptReporter = { report: vi.fn().mockResolvedValue(undefined) }++ await expect(+ bridge.complete({+ ...request,+ targets: [{ target: codexTarget }, { target: fallback }],+ credentialTicketIssuer: codexTicketIssuer(),+ providerAttemptReporter,+ })+ ).rejects.toMatchObject({+ code: 'provider_unavailable',+ reason: 'insufficient_quota',+ providerMayHaveExecuted: false,+ providerAttempt: codexReceipt,+ })+ // A denial that never reached the provider must stay terminal for the+ // invocation and revivable for the idempotency key.+ expect(providerCalls).toEqual([`${codexTarget.provider}/${codexTarget.model}`])+ expect(second.completeSingleTurn).not.toHaveBeenCalled()+ expect(providerAttemptReporter.report).toHaveBeenCalledWith({+ providerAttemptId: codexReceipt.providerAttemptId,+ providerAttemptIndex: 1,+ status: 'failed',+ })+ })++ it('fences a dispatched Codex no_grant as an ambiguous physical outcome', async () => {+ installCodexBinding()+ const codex = new FakeProvider(() => Promise.reject(new Error('grant revoked mid-flight')), {+ code: LlmErrorCode.AuthenticationFailed,+ retryable: false,+ providerCode: 'no_grant',+ providerDispatched: true,+ })+ const second = new FakeProvider(() => Promise.resolve(OK))+ const { bridge } = makeBridge({+ [codexTarget.model]: codex,+ [fallback.model]: second,+ })+ const providerAttemptReporter = { report: vi.fn().mockResolvedValue(undefined) }++ await expect(+ bridge.complete({+ ...request,+ targets: [{ target: codexTarget }, { target: fallback }],+ credentialTicketIssuer: codexTicketIssuer(),+ providerAttemptReporter,+ })+ ).rejects.toMatchObject({+ code: 'provider_unavailable',+ providerMayHaveExecuted: true,+ providerAttempt: codexReceipt,+ })+ expect(second.completeSingleTurn).not.toHaveBeenCalled()+ // The physical status must agree with providerMayHaveExecuted; a hardcoded+ // `failed` here would leave the key revivable after a possible charge.+ expect(providerAttemptReporter.report).toHaveBeenCalledWith({+ providerAttemptId: codexReceipt.providerAttemptId,+ providerAttemptIndex: 1,+ status: 'provider_unavailable',+ })+ })++ it.each(['insufficient_scope', 'host_binding_mismatch'])(+ 'closes a pre-dispatch Codex %s terminally with its receipt',+ async providerCode => {+ installCodexBinding()+ const codex = new FakeProvider(() => Promise.reject(new Error(providerCode)), {+ code: LlmErrorCode.AuthenticationFailed,+ retryable: false,+ providerCode,+ providerDispatched: false,+ })+ const second = new FakeProvider(() => Promise.resolve(OK))+ const { bridge } = makeBridge({+ [codexTarget.model]: codex,+ [fallback.model]: second,+ })+ const providerAttemptReporter = { report: vi.fn().mockResolvedValue(undefined) }++ await expect(+ bridge.complete({+ ...request,+ targets: [{ target: codexTarget }, { target: fallback }],+ credentialTicketIssuer: codexTicketIssuer(),+ providerAttemptReporter,+ })+ ).rejects.toMatchObject({+ code: 'provider_unavailable',+ providerMayHaveExecuted: false,+ providerAttempt: codexReceipt,+ })+ expect(second.completeSingleTurn).not.toHaveBeenCalled()+ expect(providerAttemptReporter.report).toHaveBeenCalledWith({+ providerAttemptId: codexReceipt.providerAttemptId,+ providerAttemptIndex: 1,+ status: 'failed',+ })+ }+ )++ it('carries the reserved receipt on a terminal generic Codex failure', async () => {+ installCodexBinding()+ const codex = new FakeProvider(() => Promise.reject(new Error('codex unavailable')))+ const { bridge } = makeBridge({ [codexTarget.model]: codex })+ const providerAttemptReporter = { report: vi.fn().mockResolvedValue(undefined) }++ await expect(+ bridge.complete({+ ...request,+ targets: [{ target: codexTarget }],+ credentialTicketIssuer: codexTicketIssuer(),+ providerAttemptReporter,+ })+ ).rejects.toMatchObject({+ code: 'provider_unavailable',+ reason: 'rate_limited',+ providerAttempt: codexReceipt,+ })+ expect(providerAttemptReporter.report).toHaveBeenCalledWith({+ providerAttemptId: codexReceipt.providerAttemptId,+ providerAttemptIndex: 1,+ status: 'provider_unavailable',+ })+ })++ it('fences a Codex budget denial whose acknowledgement was lost', async () => {+ installCodexBinding()+ const codex = new FakeProvider(() => Promise.reject(new Error('budget denied')), {+ code: LlmErrorCode.InsufficientQuota,+ retryable: false,+ providerCode: 'budget_denied',+ providerDispatched: false,+ })+ const second = new FakeProvider(() => Promise.resolve(OK))+ const { bridge } = makeBridge({+ [codexTarget.model]: codex,+ [fallback.model]: second,+ })+ const providerAttemptReporter = {+ report: vi.fn().mockRejectedValue(new Error('control-api unreachable')),+ }++ await expect(+ bridge.complete({+ ...request,+ targets: [{ target: codexTarget }, { target: fallback }],+ credentialTicketIssuer: codexTicketIssuer(),+ providerAttemptReporter,+ })+ ).rejects.toMatchObject({+ code: 'provider_unavailable',+ reason: 'outcome_unknown',+ providerMayHaveExecuted: true,+ providerAttempt: codexReceipt,+ })+ expect(second.completeSingleTurn).not.toHaveBeenCalled()+ })++ it('treats an unclassified Codex dispatch state as executed', async () => {+ installCodexBinding()+ const codex = new FakeProvider(() => Promise.reject(new Error('unrecognized failure')), {+ code: LlmErrorCode.ModelOverloaded,+ retryable: true,+ })+ const { bridge } = makeBridge({ [codexTarget.model]: codex })+ const providerAttemptReporter = { report: vi.fn().mockResolvedValue(undefined) }++ await expect(+ bridge.complete({+ ...request,+ targets: [{ target: codexTarget }],+ credentialTicketIssuer: codexTicketIssuer(),+ providerAttemptReporter,+ })+ ).rejects.toMatchObject({+ code: 'provider_unavailable',+ providerMayHaveExecuted: true,+ })+ expect(providerAttemptReporter.report).toHaveBeenCalledWith({+ providerAttemptId: codexReceipt.providerAttemptId,+ providerAttemptIndex: 1,+ status: 'provider_unavailable',+ })+ })++ it('honours the acknowledgement mode on a served Codex attempt', async () => {+ installCodexBinding()+ const codex = new FakeProvider(async () => ({+ ...OK,+ usage_reported: false,+ usage: { input_tokens: 0, output_tokens: 0, total_tokens: 0 },+ }))+ const { bridge } = makeBridge({ [codexTarget.model]: codex })+ const providerAttemptReporter = { report: vi.fn().mockResolvedValue(undefined) }++ const atomic = await bridge.complete({+ ...request,+ targets: [{ target: codexTarget }],+ acknowledgementMode: 'atomic_terminal_finalization',+ credentialTicketIssuer: codexTicketIssuer(),+ providerAttemptReporter,+ })+ expect(atomic.providerAttemptAcknowledgement).toBe('owned_by_finalizer')+ expect(providerAttemptReporter.report).not.toHaveBeenCalled()++ const perAttempt = await bridge.complete({+ ...request,+ targets: [{ target: codexTarget }],+ acknowledgementMode: 'per_attempt',+ credentialTicketIssuer: codexTicketIssuer(),+ providerAttemptReporter,+ })+ expect(perAttempt.providerAttemptAcknowledgement).toBe('confirmed')+ expect(providerAttemptReporter.report).toHaveBeenCalledWith({+ providerAttemptId: codexReceipt.providerAttemptId,+ providerAttemptIndex: 1,+ status: 'complete',+ })+ })++ it('reports a pre-dispatch eligible Codex failure as failed before failing over', async () => {+ installCodexBinding()+ const codex = new FakeProvider(() => Promise.reject(new Error('aborted before authorize')), {+ code: LlmErrorCode.ModelOverloaded,+ retryable: true,+ providerDispatched: false,+ })+ const second = new FakeProvider(() => Promise.resolve(OK))+ const { bridge, providerCalls } = makeBridge({+ [codexTarget.model]: codex,+ [fallback.model]: second,+ })+ const providerAttemptReporter = { report: vi.fn().mockResolvedValue(undefined) }++ const result = await bridge.complete({+ ...request,+ targets: [{ target: codexTarget }, { target: fallback }],+ credentialTicketIssuer: codexTicketIssuer(),+ providerAttemptReporter,+ })++ expect(result).toMatchObject({ servedTarget: fallback, fallbackUsed: true, attemptCount: 2 })+ expect(providerCalls).toEqual([+ `${codexTarget.provider}/${codexTarget.model}`,+ `${fallback.provider}/${fallback.model}`,+ ])+ expect(providerAttemptReporter.report).toHaveBeenNthCalledWith(1, {+ providerAttemptId: codexReceipt.providerAttemptId,+ providerAttemptIndex: 1,+ status: 'failed',+ })+ })+})diff --git a/mcp-host/src/pluginWorkloadSdk/promptBridge/llmBridge.ts b/mcp-host/src/pluginWorkloadSdk/promptBridge/llmBridge.tsindex a3387eb9b..ed16f8711 100644--- a/mcp-host/src/pluginWorkloadSdk/promptBridge/llmBridge.ts+++ b/mcp-host/src/pluginWorkloadSdk/promptBridge/llmBridge.ts@@ -666,6 +666,26 @@ export class LlmBridge {breaker,captured)+ // Receipt ownership is decided by the caller's mode, exactly as on the+ // api-key lane. Only atomic finalization defers the terminal ACK; a+ // workflow-mode Codex attempt must still be acknowledged here, and a+ // lost acknowledgement must surface as `failed` rather than be hidden+ // behind the finalizer's name.+ let providerAttemptAcknowledgement: 'confirmed' | 'owned_by_finalizer' | 'failed' =+ 'confirmed'+ if (request.acknowledgementMode === 'atomic_terminal_finalization') {+ providerAttemptAcknowledgement = 'owned_by_finalizer'+ } else if (ticket.providerAttemptId && ticket.providerAttemptIndex !== undefined) {+ const acknowledged = await reportProviderAttemptBestEffort(+ request.providerAttemptReporter,+ {+ providerAttemptId: ticket.providerAttemptId,+ providerAttemptIndex: ticket.providerAttemptIndex,+ status: 'complete',+ }+ )+ providerAttemptAcknowledgement = acknowledged ? 'confirmed' : 'failed'+ }return {kind: 'result',result: {@@ -676,35 +696,19 @@ export class LlmBridge {llmSecretName: '',providerAttemptId: ticket.providerAttemptId,providerAttemptIndex: ticket.providerAttemptIndex,- providerAttemptAcknowledgement: 'owned_by_finalizer',+ providerAttemptAcknowledgement,},}} catch (error) {- if (- error instanceof ClassifiedProviderError &&- (error.classified.providerCode === 'budget_denied' ||- error.classified.providerCode === 'no_grant' ||- error.classified.providerCode === 'host_binding_mismatch' ||- error.classified.providerCode === 'insufficient_scope' ||- error.classified.code === LlmErrorCode.AuthenticationFailed)- ) {- if (ticket.providerAttemptId && ticket.providerAttemptIndex !== undefined) {- await reportProviderAttemptBestEffort(request.providerAttemptReporter, {- providerAttemptId: ticket.providerAttemptId,- providerAttemptIndex: ticket.providerAttemptIndex,- status: 'failed',- })- }- return {- kind: 'error',- error,- terminal: true,- }- }+ // One physical status for one attempt. The grant/scope/budget failures+ // used to be acknowledged as `failed` from a branch of their own, which+ // could contradict `providerMayHaveExecuted` and leave a possibly billed+ // Codex call revivable. `providerMayHaveExecuted` is now the single+ // source of that decision for every oauth failure.+ const providerMayHaveExecuted =+ (error instanceof PluginWorkloadError && error.providerMayHaveExecuted) ||+ (error instanceof ClassifiedProviderError && error.providerMayHaveExecuted)if (ticket.providerAttemptId && ticket.providerAttemptIndex !== undefined) {- const providerMayHaveExecuted =- (error instanceof PluginWorkloadError && error.providerMayHaveExecuted) ||- (error instanceof ClassifiedProviderError && error.providerMayHaveExecuted)const acknowledged = await reportProviderAttemptBestEffort(request.providerAttemptReporter,{@@ -731,12 +735,30 @@ export class LlmBridge {? withProviderAttemptContext(error, providerAttemptContext): error}+ // Grant, scope, binding and budget denials never advance to another+ // target: they are operator/configuration signals about this invocation,+ // not transient provider weather.+ const forcedTerminal =+ error.classified.providerCode === 'budget_denied' ||+ error.classified.providerCode === 'no_grant' ||+ error.classified.providerCode === 'host_binding_mismatch' ||+ error.classified.providerCode === 'insufficient_scope' ||+ error.classified.code === LlmErrorCode.AuthenticationFailedconst failureClass = classifyFailoverClass(error.classified.code, error.classified.retryable)const eligible = failureClass !== null && triggerOn.has(failureClass)return {kind: 'error',- error,- terminal: !eligible || index === request.targets.length - 1,+ // The ticket already reserved this physical attempt, so the receipt+ // must travel with the error. Without it the atomic finalizer closes+ // the logical invocation while the spend row for the attempt is never+ // written — the same contract the api-key lane applies.+ error: providerAttemptContext+ ? new ClassifiedProviderError(+ error.classified,+ withProviderAttemptContext(error, providerAttemptContext)+ )+ : error,+ terminal: forcedTerminal || !eligible || index === request.targets.length - 1,}}}@@ -861,17 +883,23 @@ export class LlmBridge {code: classified.code,retryable: classified.retryable,...(classified.providerCode ? { providerCode: classified.providerCode } : {}),+ ...(classified.providerDispatched !== undefined+ ? { providerDispatched: classified.providerDispatched }+ : {}),},new PluginWorkloadError('provider_unavailable',`LLM provider error: ${classified.code}`,classified.retryable,reason,- // This branch runs only after the provider call was entered. Even a- // provider-declared auth/rate-limit response is not proof that no- // billable work occurred, so it must remain fenced as an ambiguous- // physical outcome rather than reviving the idempotency key.- true+ // This branch runs only after the provider call was entered, so an+ // unknown dispatch state must read as executed: a provider-declared+ // auth/rate-limit response is not proof that no billable work+ // occurred. Only a classifier that PROVED no call left the process+ // may revive the idempotency key — mislabelling a dispatched attempt+ // as not-executed would permit a second charge, while the opposite+ // mistake only costs revivability.+ classified.providerDispatched ?? true))} finally {diff --git a/mcp-host/src/pluginWorkloadSdk/server/index.ts b/mcp-host/src/pluginWorkloadSdk/server/index.tsindex ef1159d2c..e18cfd3a8 100644--- a/mcp-host/src/pluginWorkloadSdk/server/index.ts+++ b/mcp-host/src/pluginWorkloadSdk/server/index.ts@@ -360,12 +360,15 @@ export function maybeCreatePluginWorkloadSdkServer(: undefined,finalizePromptBridge:config.pluginWorkloadSdkRuntimeMode === 'sdk-only'- ? input =>- controlApiClient.finalizePromptBridge({- recipeNamespace: binding.recipeNamespace,- recipeName: binding.recipeName,- ...input,- })+ ? (input, options) =>+ controlApiClient.finalizePromptBridge(+ {+ recipeNamespace: binding.recipeNamespace,+ recipeName: binding.recipeName,+ ...input,+ },+ options+ ): undefined,onUsage: usage => {if (!usageReporter) return