Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
fix: fence Codex ACK loss, sweeper races, and leftover mcp-host JWTs
Alfredo Lopez785d3ba4c445
diff --git a/control-api/package-lock.json b/control-api/package-lock.jsonindex aaedd9871..6b04124cf 100644--- a/control-api/package-lock.json+++ b/control-api/package-lock.json@@ -1,12 +1,12 @@{"name": "clerum-control-api",- "version": "0.1.530",+ "version": "0.1.531","lockfileVersion": 3,"requires": true,"packages": {"": {"name": "clerum-control-api",- "version": "0.1.530",+ "version": "0.1.531","dependencies": {"@clerum/display-field": "file:../packages/display-field","@clerum/image-policy": "file:../packages/image-policy",diff --git a/control-api/package.json b/control-api/package.jsonindex bddadf99e..94132485b 100644--- a/control-api/package.json+++ b/control-api/package.json@@ -1,6 +1,6 @@{"name": "clerum-control-api",- "version": "0.1.530",+ "version": "0.1.531","description": "Internal Clerum control plane REST API + UI for managing CRDs and secrets","main": "dist/main.js","engines": {diff --git a/control-api/src/routes/mcp-host/plugin-workload-sdk.routes.ts b/control-api/src/routes/mcp-host/plugin-workload-sdk.routes.tsindex fbe2913bf..1f75d5bd2 100644--- a/control-api/src/routes/mcp-host/plugin-workload-sdk.routes.ts+++ b/control-api/src/routes/mcp-host/plugin-workload-sdk.routes.ts@@ -718,11 +718,17 @@ export function createMcpHostPluginWorkloadSdkRoutes(): Router {isClientNotificationsPolicyReady(clientNotificationsGrant) &&(await hasUsableClientNotificationRecipients(clientNotificationsGrant)))- const primaryTarget = grant?.promptTargets[0] ?? null+ const alignedPrimary =+ grant &&+ grant.promptTargets[0] &&+ grant.defaultTargetRef !== null &&+ grant.promptTargets[0].targetRef === grant.defaultTargetRef+ ? grant.promptTargets[0]+ : nullconst reservationOnlyOauthBroker = Boolean(- primaryTarget &&- isLlmProviderId(primaryTarget.provider) &&- PROVIDER_AUTH_MODE[primaryTarget.provider] === 'oauth-broker'+ alignedPrimary &&+ isLlmProviderId(alignedPrimary.provider) &&+ PROVIDER_AUTH_MODE[alignedPrimary.provider] === 'oauth-broker')// Old mcp-host/WRC require capabilities.contractVersion === 2. Advertise// 3 only for oauth-broker defaults so those binaries fail at reconcile@@ -739,9 +745,9 @@ export function createMcpHostPluginWorkloadSdkRoutes(): Router {policyRevision: grant?.policyRevision ?? 0,policyHash: v2Ready && grant ? hashPromptTargetPolicy(grant) : null,defaultTargetRef: grant?.defaultTargetRef ?? null,- defaultProvider: primaryTarget?.provider ?? null,- defaultModel: primaryTarget?.model ?? null,- defaultConnectionRef: primaryTarget?.connectionRef ?? null,+ defaultProvider: alignedPrimary?.provider ?? null,+ defaultModel: alignedPrimary?.model ?? null,+ defaultConnectionRef: alignedPrimary?.connectionRef ?? null,v2Ready,...(reservationOnlyOauthBroker ? { reservationOnlyOauthBroker: true } : {}),clientNotificationsPolicyState: clientNotificationsGrant?.policyState ?? 'missing',diff --git a/control-api/src/services/llmProviderAttemptStore.ts b/control-api/src/services/llmProviderAttemptStore.tsindex 1e1c2d494..326701aa6 100644--- a/control-api/src/services/llmProviderAttemptStore.ts+++ b/control-api/src/services/llmProviderAttemptStore.ts@@ -35,6 +35,24 @@ export type LlmProviderAttemptRow = LlmProviderAttemptInsert & {createdAt: Date}+export function isLinkedCodexUsageReady(+ row: Pick<LlmProviderAttemptRow, 'outcome' | 'usageInputTokens' | 'usageOutputTokens'>+): boolean {+ return row.outcome === 'success' && row.usageInputTokens != null && row.usageOutputTokens != null+}++/** Linked Codex still owns exact usage; sweepers must not freeze SDK spend. */+export function isLinkedCodexInFlightWithoutUsage(+ row: Pick<LlmProviderAttemptRow, 'status' | 'outcome' | 'usageInputTokens' | 'usageOutputTokens'>+): boolean {+ if (isLinkedCodexUsageReady(row)) return false+ return (+ row.status === 'authorized' ||+ row.status === 'redeemed' ||+ (row.status === 'finalized' && row.outcome === 'success')+ )+}+export async function applyLlmProviderAttemptSchema(db: DbClient): Promise<void> {await db.query(`CREATE TABLE IF NOT EXISTS llm_provider_attempts (diff --git a/control-api/src/services/pluginWorkloadSdkDb.ts b/control-api/src/services/pluginWorkloadSdkDb.tsindex be4f43d7b..d2b2ae4e1 100644--- a/control-api/src/services/pluginWorkloadSdkDb.ts+++ b/control-api/src/services/pluginWorkloadSdkDb.ts@@ -1,7 +1,12 @@import { createHash, randomUUID } from 'node:crypto'+import { PROVIDER_AUTH_MODE, isLlmProviderId } from '@clerum/llm-providers'import { type DbClient, type DbTransactionClient, pool, withTransaction } from '../db.js'import type { AdministrativeEventSubmitterPrincipalV1 } from '../middleware/tracingSubmitterAuth.js'import { stableStringify } from '../utils/stableStringify.js'+import {+ isLinkedCodexInFlightWithoutUsage,+ loadLlmProviderAttemptBySdkAttemptId,+} from './llmProviderAttemptStore.js'import {type ControlApiPermissionChange,appendControlApiPermissionEventsInTransaction,@@ -2123,21 +2128,30 @@ export async function failStaleInvocationsInTransaction(db: DbClient): Promise<number> {const timeoutSeconds = Math.max(1, Math.floor(olderThanSeconds))- const result = await db.query(- `UPDATE plugin_workload_sdk_invocations- SET status = 'provider_unavailable', updated_at = now(), completed_at = now()- WHERE status = 'in_progress'- AND (- (lease_expires_at IS NOT NULL AND lease_expires_at < now())- OR (- lease_expires_at IS NULL- AND updated_at < now() - interval '1 second' * $1- )- )- RETURNING id, attempt_generation`,+ const stale = await db.query(+ `SELECT id, attempt_generation, recipe_namespace, recipe_name+ FROM plugin_workload_sdk_invocations+ WHERE status = 'in_progress'+ AND (+ (lease_expires_at IS NOT NULL AND lease_expires_at < now())+ OR (+ lease_expires_at IS NULL+ AND updated_at < now() - interval '1 second' * $1+ )+ )+ FOR UPDATE`,[timeoutSeconds])- for (const row of result.rows as Array<{ id: string; attempt_generation: number }>) {+ let closed = 0+ for (const row of stale.rows as Array<{+ id: string+ attempt_generation: number+ recipe_namespace: string+ recipe_name: string+ }>) {+ await db.query(`SELECT pg_advisory_xact_lock(hashtext($1)::bigint)`, [+ `plugin_workload_sdk:${row.recipe_namespace}/${row.recipe_name}`,+ ])const physicalAttempts = await db.query(`SELECT id, recipe_namespace, recipe_name, attempt_generation,attempt_index, target_ref, provider, model, credential_slot@@ -2148,6 +2162,35 @@ export async function failStaleInvocationsInTransaction(FOR UPDATE`,[row.id, row.attempt_generation])+ let linkedCodexInFlight = false+ for (const attempt of physicalAttempts.rows as Array<Record<string, unknown>>) {+ if (+ !isLlmProviderId(String(attempt.provider)) ||+ PROVIDER_AUTH_MODE[String(attempt.provider)] !== 'oauth-broker'+ ) {+ continue+ }+ const linked = await loadLlmProviderAttemptBySdkAttemptId(db, String(attempt.id))+ if (linked && isLinkedCodexInFlightWithoutUsage(linked)) {+ linkedCodexInFlight = true+ break+ }+ }+ // Codex still owns exact usage. Leave the invocation leased so finalize+ // can 409 ledger_pending instead of freezing an unknown spend row.+ if (linkedCodexInFlight) continue++ const marked = await db.query(+ `UPDATE plugin_workload_sdk_invocations+ SET status = 'provider_unavailable', updated_at = now(), completed_at = now()+ WHERE id = $1+ AND attempt_generation = $2+ AND status = 'in_progress'+ RETURNING id`,+ [row.id, row.attempt_generation]+ )+ if ((marked.rowCount ?? 0) !== 1) continue+for (const attempt of physicalAttempts.rows as Array<Record<string, unknown>>) {// The host has disappeared, so no current JWT can prove a host_ref.// Persist an unattributed unknown outcome instead of inventing one@@ -2188,8 +2231,9 @@ export async function failStaleInvocationsInTransaction(AND status IN ('reserved', 'in_progress')`,[row.id, row.attempt_generation])+ closed += 1}- return result.rowCount ?? 0+ return closed}export interface ListInvocationsFilter {diff --git a/control-api/src/services/pluginWorkloadSdkFinalization.ts b/control-api/src/services/pluginWorkloadSdkFinalization.tsindex 7a9f5a6e0..abe1bb0ec 100644--- a/control-api/src/services/pluginWorkloadSdkFinalization.ts+++ b/control-api/src/services/pluginWorkloadSdkFinalization.ts@@ -1,6 +1,10 @@import { PROVIDER_AUTH_MODE, isLlmProviderId } from '@clerum/llm-providers'import type { DbClient } from '../db.js'-import { loadLlmProviderAttemptBySdkAttemptId } from './llmProviderAttemptStore.js'+import {+ isLinkedCodexInFlightWithoutUsage,+ isLinkedCodexUsageReady,+ loadLlmProviderAttemptBySdkAttemptId,+} from './llmProviderAttemptStore.js'import { withTraceIngestTransaction } from './tracing/pools.js'import { projectAcceptedUsageEvents } from './tracing/usageProjection.js'import { ingestUsageEventsInTransaction } from './usageEvents.js'@@ -167,12 +171,7 @@ function resolvePromptBridgeOutcome(if (!oauthBroker) {return status === 'complete' ? 'exact' : status === 'failed' ? 'not_executed' : 'unknown'}- if (- status === 'complete' &&- linked?.outcome === 'success' &&- linked.usageInputTokens != null &&- linked.usageOutputTokens != null- ) {+ if (status === 'complete' && linked && isLinkedCodexUsageReady(linked)) {return 'exact'}if (status === 'failed' && !linked) return 'not_executed'@@ -284,7 +283,24 @@ export async function finalizePromptBridgeInTransaction([input.providerAttemptId])const existing = existingResult.rows[0] as SpendOutcomeRow | undefined- if (existing) return mapExistingOutcome(existing, input)+ if (existing) {+ if (+ isOauthBrokerProvider(existing.provider) &&+ input.status === 'complete' &&+ existing.outcome === 'unknown'+ ) {+ const linkedCodex = await loadLlmProviderAttemptBySdkAttemptId(db, input.providerAttemptId)+ if (linkedCodex && isLinkedCodexInFlightWithoutUsage(linkedCodex)) {+ throw new PromptBridgeFinalizationError(+ 'ledger_pending',+ 'linked Codex attempt has not finalized usage yet',+ 409,+ true+ )+ }+ }+ return mapExistingOutcome(existing, input)+ }const invocationResult = await db.query(`SELECT id, recipe_namespace, recipe_name, method, status, attempt_generation@@ -391,15 +407,7 @@ export async function finalizePromptBridgeInTransaction(? await loadLlmProviderAttemptBySdkAttemptId(db, input.providerAttemptId): nullif (oauthBroker && input.status === 'complete' && linkedCodex) {- const usageReady =- linkedCodex.outcome === 'success' &&- linkedCodex.usageInputTokens != null &&- linkedCodex.usageOutputTokens != null- const inFlight =- linkedCodex.status === 'authorized' ||- linkedCodex.status === 'redeemed' ||- (linkedCodex.status === 'finalized' && linkedCodex.outcome === 'success' && !usageReady)- if (!usageReady && inFlight) {+ if (isLinkedCodexInFlightWithoutUsage(linkedCodex)) {throw new PromptBridgeFinalizationError('ledger_pending','linked Codex attempt has not finalized usage yet',diff --git a/control-api/test/routes.mcp-host.plugin-workload-sdk.test.ts b/control-api/test/routes.mcp-host.plugin-workload-sdk.test.tsindex e26388e00..16a32f055 100644--- a/control-api/test/routes.mcp-host.plugin-workload-sdk.test.ts+++ b/control-api/test/routes.mcp-host.plugin-workload-sdk.test.ts@@ -326,6 +326,58 @@ describe('GET /mcp-host/plugin-workload-sdk/capabilities', () => {v2Ready: true,})})++ it('does not advertise oauth-broker v3 when promptTargets[0] is not the default', async () => {+ vi.mocked(sdkDb.findGrant).mockResolvedValue({+ id: 'grant-misaligned-capabilities',+ recipeNamespace: NS,+ recipeName: RECIPE,+ capabilityFamily: 'promptBridge',+ provider: 'codex-subscription',+ allowedModels: ['gpt-5.1'],+ allowedEventTypes: [],+ allowedTargetRefs: [],+ allowedUserRefs: [],+ allowedCallers: ['api'],+ quotaLimits: {},+ modelPolicies: {},+ promptTargets: [+ {+ targetRef: 'primary-codex',+ provider: 'codex-subscription',+ model: 'gpt-5.1',+ credentialSlot: '',+ connectionRef: 'team-plus',+ },+ {+ targetRef: 'fallback-openai',+ provider: 'openai',+ model: 'gpt-5.4-mini',+ credentialSlot: 'openai-api-key',+ },+ ],+ defaultTargetRef: 'fallback-openai',+ policyState: 'active',+ policyRevision: 2,+ createdAt: '2026-08-03T00:00:00.000Z',+ updatedAt: '2026-08-03T00:00:00.000Z',+ })++ const res = await request(buildApp())+ .get('/mcp-host/plugin-workload-sdk/capabilities')+ .set('Authorization', `Bearer ${issueSdkToken()}`)++ expect(res.status).toBe(200)+ expect(res.body).toMatchObject({+ contractVersion: 2,+ defaultTargetRef: 'fallback-openai',+ defaultProvider: null,+ defaultModel: null,+ defaultConnectionRef: null,+ v2Ready: false,+ })+ expect(res.body.reservationOnlyOauthBroker).toBeUndefined()+ })})describe('POST /mcp-host/plugin-workload-sdk/prompt-bridge/v2', () => {diff --git a/control-api/test/services.llmProviderAttemptStore.helpers.test.ts b/control-api/test/services.llmProviderAttemptStore.helpers.test.tsnew file mode 100644index 000000000..61c46467e--- /dev/null+++ b/control-api/test/services.llmProviderAttemptStore.helpers.test.ts@@ -0,0 +1,74 @@+import { describe, expect, it } from 'vitest'+import {+ isLinkedCodexInFlightWithoutUsage,+ isLinkedCodexUsageReady,+} from '../src/services/llmProviderAttemptStore.js'++describe('linked Codex usage predicates', () => {+ it('treats success plus both token counts as usage-ready', () => {+ expect(+ isLinkedCodexUsageReady({+ outcome: 'success',+ usageInputTokens: 3,+ usageOutputTokens: 2,+ })+ ).toBe(true)+ expect(+ isLinkedCodexUsageReady({+ outcome: 'success',+ usageInputTokens: 3,+ usageOutputTokens: null,+ })+ ).toBe(false)+ expect(+ isLinkedCodexUsageReady({+ outcome: 'unknown',+ usageInputTokens: 3,+ usageOutputTokens: 2,+ })+ ).toBe(false)+ })++ it('keeps authorized, redeemed, and usage-less success finalized in flight', () => {+ expect(+ isLinkedCodexInFlightWithoutUsage({+ status: 'authorized',+ outcome: null,+ usageInputTokens: null,+ usageOutputTokens: null,+ })+ ).toBe(true)+ expect(+ isLinkedCodexInFlightWithoutUsage({+ status: 'redeemed',+ outcome: null,+ usageInputTokens: null,+ usageOutputTokens: null,+ })+ ).toBe(true)+ expect(+ isLinkedCodexInFlightWithoutUsage({+ status: 'finalized',+ outcome: 'success',+ usageInputTokens: null,+ usageOutputTokens: null,+ })+ ).toBe(true)+ expect(+ isLinkedCodexInFlightWithoutUsage({+ status: 'finalized',+ outcome: 'success',+ usageInputTokens: 3,+ usageOutputTokens: 2,+ })+ ).toBe(false)+ expect(+ isLinkedCodexInFlightWithoutUsage({+ status: 'finalized',+ outcome: 'error',+ usageInputTokens: null,+ usageOutputTokens: null,+ })+ ).toBe(false)+ })+})diff --git a/control-api/test/services.pluginWorkloadSdkDb.test.ts b/control-api/test/services.pluginWorkloadSdkDb.test.tsindex 6b8669294..31f75bf02 100644--- a/control-api/test/services.pluginWorkloadSdkDb.test.ts+++ b/control-api/test/services.pluginWorkloadSdkDb.test.ts@@ -400,9 +400,17 @@ describe('invocation retry lifecycle timestamps', () => {it('sweeps by the latest retry timestamp rather than the original reservation', async () => {vi.mocked(pool.query).mockResolvedValueOnce({- rows: [{ id: 'inv-1', attempt_generation: 2 }],+ rows: [+ {+ id: 'inv-1',+ attempt_generation: 2,+ recipe_namespace: 'sandbox-recipes',+ recipe_name: 'sdk-recipe',+ },+ ],rowCount: 1,} as never)+ .mockResolvedValueOnce({ rows: [], rowCount: 1 } as never).mockResolvedValueOnce({rows: [{@@ -419,27 +427,104 @@ describe('invocation retry lifecycle timestamps', () => {],rowCount: 1,} as never)- .mockResolvedValue({ rows: [], rowCount: 1 } as never)+ .mockResolvedValue({ rows: [{ id: 'inv-1' }], rowCount: 1 } as never)await expect(failStaleInvocations(150)).resolves.toBe(1)const [sql] = vi.mocked(pool.query).mock.calls[0] as unknown as [string, unknown[]]- expect(sql).toContain("SET status = 'provider_unavailable'")+ expect(sql).toContain('FROM plugin_workload_sdk_invocations')expect(sql).toContain('lease_expires_at < now()')expect(sql).toContain('lease_expires_at IS NULL')expect(sql).toContain('updated_at < now()')- expect(sql).toContain('RETURNING id, attempt_generation')- expect(pool.query).toHaveBeenCalledTimes(5)- expect(vi.mocked(pool.query).mock.calls[1]?.[0] as string).toContain(- 'FROM plugin_workload_sdk_provider_attempts'- )+ expect(sql).toContain('FOR UPDATE')+ expect(pool.query).toHaveBeenCalledTimes(7)+ expect(vi.mocked(pool.query).mock.calls[1]?.[0] as string).toContain('pg_advisory_xact_lock')expect(vi.mocked(pool.query).mock.calls[2]?.[0] as string).toContain(- 'INSERT INTO plugin_workload_sdk_spend_outcomes'+ 'FROM plugin_workload_sdk_provider_attempts')expect(vi.mocked(pool.query).mock.calls[3]?.[0] as string).toContain("SET status = 'provider_unavailable'")expect(vi.mocked(pool.query).mock.calls[4]?.[0] as string).toContain(+ 'INSERT INTO plugin_workload_sdk_spend_outcomes'+ )+ expect(vi.mocked(pool.query).mock.calls[5]?.[0] as string).toContain("SET status = 'provider_unavailable'")+ expect(vi.mocked(pool.query).mock.calls[6]?.[0] as string).toContain(+ "SET status = 'provider_unavailable'"+ )+ })++ it('does not freeze oauth spend while a linked Codex attempt is still in flight', async () => {+ vi.mocked(pool.query)+ .mockResolvedValueOnce({+ rows: [+ {+ id: 'inv-codex',+ attempt_generation: 1,+ recipe_namespace: 'sandbox-recipes',+ recipe_name: 'sdk-recipe',+ },+ ],+ rowCount: 1,+ } as never)+ .mockResolvedValueOnce({ rows: [], rowCount: 1 } as never)+ .mockResolvedValueOnce({+ rows: [+ {+ id: 'attempt-codex',+ recipe_namespace: 'sandbox-recipes',+ recipe_name: 'sdk-recipe',+ attempt_generation: 1,+ attempt_index: 1,+ target_ref: 'primary-codex',+ provider: 'codex-subscription',+ model: 'gpt-5.1',+ credential_slot: '',+ },+ ],+ rowCount: 1,+ } as never)+ .mockResolvedValueOnce({+ rows: [+ {+ id: 'codex-1',+ caller_kind: 'recipe',+ host_ref: 'sandbox-recipes/sdk-recipe',+ recipe_namespace: 'sandbox-recipes',+ recipe_name: 'sdk-recipe',+ invocation_id: 'inv-codex',+ attempt_generation: 1,+ provider_attempt_index: 1,+ provider: 'codex-subscription',+ model: 'gpt-5.1',+ request_hash: 'hash',+ policy_revision: 1,+ policy_hash: 'a'.repeat(64),+ budget_reservation_id: 'budget-1',+ connection_revision: 1,+ plugin_workload_sdk_provider_attempt_id: 'attempt-codex',+ status: 'authorized',+ outcome: null,+ usage_input_tokens: null,+ usage_output_tokens: null,+ created_at: new Date('2026-09-01T00:00:00.000Z'),+ },+ ],+ rowCount: 1,+ } as never)+ await expect(failStaleInvocations(150)).resolves.toBe(0)+ expect(+ vi+ .mocked(pool.query)+ .mock.calls.some(([sql]: [string]) =>+ sql.includes('INSERT INTO plugin_workload_sdk_spend_outcomes')+ )+ ).toBe(false)+ expect(+ vi+ .mocked(pool.query)+ .mock.calls.some(([sql]: [string]) => sql.includes("SET status = 'provider_unavailable'"))+ ).toBe(false)})})diff --git a/control-api/test/services.pluginWorkloadSdkFinalization.realPostgres.integration.test.ts b/control-api/test/services.pluginWorkloadSdkFinalization.realPostgres.integration.test.tsindex 4b27e2ee2..581875699 100644--- a/control-api/test/services.pluginWorkloadSdkFinalization.realPostgres.integration.test.ts+++ b/control-api/test/services.pluginWorkloadSdkFinalization.realPostgres.integration.test.ts@@ -2,6 +2,7 @@ import { afterAll, beforeAll, describe, expect, it } from 'vitest'import { randomBytes, randomUUID } from 'node:crypto'import { Pool, type PoolClient } from 'pg'import { initDb } from '../src/db.js'+import { insertLlmProviderAttempt } from '../src/services/llmProviderAttemptStore.js'import { failStaleInvocationsInTransaction } from '../src/services/pluginWorkloadSdkDb.js'import {PromptBridgeFinalizationError,@@ -428,4 +429,56 @@ describeRealPostgres('Plugin Workload SDK finalization on real PostgreSQL', () =sweeperClient.release()}})++ it('does not freeze oauth spend while a linked Codex attempt is still in flight', async () => {+ const client = await dbPool.connect()+ try {+ await begin(client)+ const input = await seedAttempt(client, { leaseExpired: true })+ await client.query(+ `UPDATE plugin_workload_sdk_provider_attempts+ SET provider = 'codex-subscription',+ model = 'gpt-5.1',+ credential_slot = '',+ target_ref = 'primary-codex'+ WHERE id = $1`,+ [input.providerAttemptId]+ )+ await insertLlmProviderAttempt(client, {+ callerKind: 'recipe',+ hostRef: input.hostRef,+ recipeNamespace: input.recipeNamespace,+ recipeName: input.recipeName,+ invocationId: input.invocationId,+ attemptGeneration: 1,+ providerAttemptIndex: 1,+ model: 'gpt-5.1',+ requestHash: 'a'.repeat(64),+ policyRevision: 1,+ policyHash: 'b'.repeat(64),+ budgetReservationId: randomUUID(),+ connectionRevision: 1,+ pluginWorkloadSdkProviderAttemptId: input.providerAttemptId,+ })+ const closed = await failStaleInvocationsInTransaction(1, client)+ await commit(client)++ expect(closed).toBe(0)+ const spend = await dbPool.query(+ `SELECT count(*)::int AS count+ FROM plugin_workload_sdk_spend_outcomes+ WHERE provider_attempt_id = $1`,+ [input.providerAttemptId]+ )+ expect(spend.rows[0]?.count).toBe(0)+ const invocation = await dbPool.query(+ `SELECT status FROM plugin_workload_sdk_invocations WHERE id = $1`,+ [input.invocationId]+ )+ expect(invocation.rows[0]?.status).toBe('in_progress')+ } finally {+ await client.query('ROLLBACK').catch(() => undefined)+ client.release()+ }+ })})diff --git a/control-api/test/services.pluginWorkloadSdkFinalization.test.ts b/control-api/test/services.pluginWorkloadSdkFinalization.test.tsindex 89eef2f17..5898921d3 100644--- a/control-api/test/services.pluginWorkloadSdkFinalization.test.ts+++ b/control-api/test/services.pluginWorkloadSdkFinalization.test.ts@@ -723,7 +723,7 @@ describe('Plugin Workload SDK promptBridge finalization', () => {outputTokens: 2,},},- dbWithRows([], row) as never+ dbWithRows([], row, []) as never)).resolves.toMatchObject({idempotent: true,@@ -732,6 +732,73 @@ describe('Plugin Workload SDK promptBridge finalization', () => {})})+ it('refuses to replay a sweeper unknown while the linked Codex attempt is still in flight', async () => {+ const row = {+ provider_attempt_id: IDS.providerAttempt,+ invocation_id: IDS.invocation,+ recipe_namespace: inputBase.recipeNamespace,+ recipe_name: inputBase.recipeName,+ attempt_generation: 1,+ attempt_index: 1,+ target_ref: 'codex-primary',+ host_ref: inputBase.hostRef,+ provider: 'codex-subscription',+ model: 'gpt-5.1',+ credential_slot: '',+ outcome: 'unknown',+ input_tokens: null,+ output_tokens: null,+ }+ await expect(+ finalizePromptBridgeInTransaction(+ {+ ...inputBase,+ target: {+ targetRef: 'codex-primary',+ provider: 'codex-subscription',+ model: 'gpt-5.1',+ credentialSlot: '',+ },+ status: 'complete',+ usage: {+ llmSecretName: '',+ callerRef: 'scanner',+ fallbackUsed: false,+ attemptCount: 1,+ inputTokens: 4,+ outputTokens: 2,+ },+ },+ dbWithRows([], row, {+ id: '33333333-3333-4333-8333-333333333333',+ caller_kind: 'recipe',+ host_ref: inputBase.hostRef,+ recipe_namespace: inputBase.recipeNamespace,+ recipe_name: inputBase.recipeName,+ invocation_id: IDS.invocation,+ attempt_generation: 1,+ provider_attempt_index: 1,+ provider: 'codex-subscription',+ model: 'gpt-5.1',+ request_hash: 'hash',+ policy_revision: 1,+ policy_hash: 'a'.repeat(64),+ budget_reservation_id: 'budget-1',+ connection_revision: 1,+ status: 'authorized',+ outcome: null,+ usage_input_tokens: null,+ usage_output_tokens: null,+ created_at: new Date('2026-08-03T00:00:00.000Z'),+ }) as never+ )+ ).rejects.toMatchObject({+ code: 'ledger_pending',+ httpStatus: 409,+ retryable: true,+ })+ })+it('rejects empty static-key credential material from the persisted provider', async () => {await expect(finalizePromptBridgeInTransaction(diff --git a/mcp-host/package-lock.json b/mcp-host/package-lock.jsonindex 388765e66..d4ac7c746 100644--- a/mcp-host/package-lock.json+++ b/mcp-host/package-lock.json@@ -1,12 +1,12 @@{"name": "clerum-mcp-host",- "version": "0.1.308",+ "version": "0.1.309","lockfileVersion": 3,"requires": true,"packages": {"": {"name": "clerum-mcp-host",- "version": "0.1.308",+ "version": "0.1.309","license": "SEE LICENSE IN LICENSE","dependencies": {"@anthropic-ai/sdk": "^0.32.0",diff --git a/mcp-host/package.json b/mcp-host/package.jsonindex b2399685e..39bb36d85 100644--- a/mcp-host/package.json+++ b/mcp-host/package.json@@ -1,6 +1,6 @@{"name": "clerum-mcp-host",- "version": "0.1.308",+ "version": "0.1.309","description": "MCP Host - reads Host CRD and provides LLM access via OpenAI/Claude","main": "dist/main.js","engines": {diff --git a/mcp-host/src/pluginWorkloadSdk/bootstrapIdentity.test.ts b/mcp-host/src/pluginWorkloadSdk/bootstrapIdentity.test.tsindex abce781b4..0a5b33bd4 100644--- a/mcp-host/src/pluginWorkloadSdk/bootstrapIdentity.test.ts+++ b/mcp-host/src/pluginWorkloadSdk/bootstrapIdentity.test.ts@@ -1,4 +1,5 @@import { describe, expect, it, vi } from 'vitest'+import { CODEX_UNASSIGNED_CONNECTION_KEY } from '@clerum/codex-catalog-projection'import { computeCodexPolicyHash } from '@clerum/llm-provider-attempt-contract'import {configurePluginWorkloadSdkBootstrapIdentity,@@ -37,6 +38,24 @@ describe('readVerifiedSdkOnlyCodexBinding', () => {binding)})++ it('rejects the shared unassigned connection sentinel', () => {+ expect(+ readVerifiedSdkOnlyCodexBinding(+ {+ ...binding,+ connectionKey: CODEX_UNASSIGNED_CONNECTION_KEY,+ bindingHash: computeCodexPolicyHash({+ model,+ catalogRevision: 4,+ credentialRevision: 1,+ connectionKey: CODEX_UNASSIGNED_CONNECTION_KEY,+ }),+ },+ model+ )+ ).toBeNull()+ })})describe('Plugin Workload SDK bootstrap identity', () => {diff --git a/mcp-host/src/pluginWorkloadSdk/promptBridge/llmBridge.failover.test.ts b/mcp-host/src/pluginWorkloadSdk/promptBridge/llmBridge.failover.test.tsindex 47ae46b74..5ec256ffe 100644--- a/mcp-host/src/pluginWorkloadSdk/promptBridge/llmBridge.failover.test.ts+++ b/mcp-host/src/pluginWorkloadSdk/promptBridge/llmBridge.failover.test.ts@@ -676,6 +676,63 @@ describe('LlmBridge authorized multi-provider fallback', () => {})})+ it('does not fail over a Codex target when the post-dispatch acknowledgement is lost', async () => {+ const codex: PromptBridgeTarget = {+ targetRef: 'codex-primary',+ provider: 'codex-subscription',+ model: 'gpt-5.1',+ credentialSlot: '',+ connectionRef: 'team-plus',+ }+ replaceSdkOnlyCodexBinding({+ connectionKey: 'team-plus',+ catalogRevision: 1,+ credentialRevision: 0,+ model: 'gpt-5.1',+ bindingHash: computeCodexPolicyHash({+ model: 'gpt-5.1',+ catalogRevision: 1,+ credentialRevision: 0,+ connectionKey: 'team-plus',+ }),+ })+ const first = new FakeProvider(() => Promise.reject(new Error('codex unavailable')))+ const second = new FakeProvider(() => Promise.resolve(OK))+ const { bridge, providerCalls } = makeBridge({+ 'gpt-5.1': first,+ [fallback.model]: second,+ })+ const providerAttemptReporter = {+ report: vi.fn().mockRejectedValue(new Error('control-api unreachable')),+ }+ const issuer = {+ issue: vi.fn(async ({ target }: { target: PromptBridgeTarget }) => ({+ credentialTicket:+ target.provider === 'codex-subscription' ? '' : `fresh-${target.targetRef}`,+ providerAttemptId: `sdk-${target.targetRef}`,+ providerAttemptIndex: target.provider === 'codex-subscription' ? 1 : 2,+ })),+ }++ await expect(+ bridge.complete({+ ...request,+ targets: [{ target: codex }, { target: fallback }],+ credentialTicketIssuer: issuer,+ providerAttemptReporter,+ })+ ).rejects.toMatchObject({+ code: 'provider_unavailable',+ retryable: false,+ reason: 'outcome_unknown',+ providerMayHaveExecuted: true,+ })+ expect(first.completeSingleTurn).toHaveBeenCalledOnce()+ expect(second.completeSingleTurn).not.toHaveBeenCalled()+ expect(providerCalls).toEqual(['codex-subscription/gpt-5.1'])+ expect(issuer.issue).toHaveBeenCalledTimes(1)+ })+it('does not execute a Codex target while its circuit is open', async () => {const codex: PromptBridgeTarget = {targetRef: 'codex-primary',diff --git a/mcp-host/src/pluginWorkloadSdk/promptBridge/llmBridge.ts b/mcp-host/src/pluginWorkloadSdk/promptBridge/llmBridge.tsindex d37fd60ec..a3387eb9b 100644--- a/mcp-host/src/pluginWorkloadSdk/promptBridge/llmBridge.ts+++ b/mcp-host/src/pluginWorkloadSdk/promptBridge/llmBridge.ts@@ -705,11 +705,26 @@ export class LlmBridge {const providerMayHaveExecuted =(error instanceof PluginWorkloadError && error.providerMayHaveExecuted) ||(error instanceof ClassifiedProviderError && error.providerMayHaveExecuted)- await reportProviderAttemptBestEffort(request.providerAttemptReporter, {- providerAttemptId: ticket.providerAttemptId,- providerAttemptIndex: ticket.providerAttemptIndex,- status: providerMayHaveExecuted ? 'provider_unavailable' : 'failed',- })+ const acknowledged = await reportProviderAttemptBestEffort(+ request.providerAttemptReporter,+ {+ providerAttemptId: ticket.providerAttemptId,+ providerAttemptIndex: ticket.providerAttemptIndex,+ status: providerMayHaveExecuted ? 'provider_unavailable' : 'failed',+ }+ )+ // Mirror the non-oauth fence: a lost receipt after dispatch is not a+ // safe failover signal. Codex may already have billed.+ if (!acknowledged) {+ return {+ kind: 'error',+ error: new ClassifiedProviderError(+ { code: LlmErrorCode.ApiCallFailed, retryable: false },+ providerOutcomeUnknownError(providerAttemptContext)+ ),+ terminal: true,+ }+ }}if (!(error instanceof ClassifiedProviderError)) {throw providerAttemptContextdiff --git a/mcp-host/src/pluginWorkloadSdk/sdkOnlyCodexBinding.test.ts b/mcp-host/src/pluginWorkloadSdk/sdkOnlyCodexBinding.test.tsnew file mode 100644index 000000000..7547384a2--- /dev/null+++ b/mcp-host/src/pluginWorkloadSdk/sdkOnlyCodexBinding.test.ts@@ -0,0 +1,51 @@+import { describe, expect, it } from 'vitest'+import { CODEX_UNASSIGNED_CONNECTION_KEY } from '@clerum/codex-catalog-projection'+import { computeCodexPolicyHash } from '@clerum/llm-provider-attempt-contract'+import {+ isPluginWorkloadSdkCodexBindingProof,+ readVerifiedSdkOnlyCodexBinding,+} from './sdkOnlyCodexBinding'++describe('mcp-host sdk-only Codex binding proof', () => {+ const model = 'gpt-5.6-luna'+ const proof = {+ connectionKey: 'team-plus',+ catalogRevision: 4,+ credentialRevision: 1,+ model,+ bindingHash: computeCodexPolicyHash({+ model,+ catalogRevision: 4,+ credentialRevision: 1,+ connectionKey: 'team-plus',+ }),+ }++ it('rejects the unassigned connection sentinel', () => {+ expect(+ isPluginWorkloadSdkCodexBindingProof({+ ...proof,+ connectionKey: CODEX_UNASSIGNED_CONNECTION_KEY,+ })+ ).toBe(false)+ expect(+ readVerifiedSdkOnlyCodexBinding(+ {+ ...proof,+ connectionKey: CODEX_UNASSIGNED_CONNECTION_KEY,+ bindingHash: computeCodexPolicyHash({+ model,+ catalogRevision: 4,+ credentialRevision: 1,+ connectionKey: CODEX_UNASSIGNED_CONNECTION_KEY,+ }),+ },+ model+ )+ ).toBeNull()+ })++ it('strips extra keys after the hash verifies', () => {+ expect(readVerifiedSdkOnlyCodexBinding({ ...proof, extra: 'drop-me' }, model)).toEqual(proof)+ })+})diff --git a/mcp-host/src/pluginWorkloadSdk/sdkOnlyCodexBinding.ts b/mcp-host/src/pluginWorkloadSdk/sdkOnlyCodexBinding.tsindex f849ae577..3e3d2f8f5 100644--- a/mcp-host/src/pluginWorkloadSdk/sdkOnlyCodexBinding.ts+++ b/mcp-host/src/pluginWorkloadSdk/sdkOnlyCodexBinding.ts@@ -1,4 +1,7 @@-import type { CodexPolicyBinding } from '@clerum/codex-catalog-projection'+import {+ CODEX_UNASSIGNED_CONNECTION_KEY,+ type CodexPolicyBinding,+} from '@clerum/codex-catalog-projection'import { computeCodexPolicyHash } from '@clerum/llm-provider-attempt-contract'const POLICY_HASH_RE = /^[a-f0-9]{64}$/@@ -46,7 +49,7 @@ export function isPluginWorkloadSdkCodexBindingProof(return (typeof candidate.connectionKey === 'string' &&candidate.connectionKey.trim().length > 0 &&- candidate.connectionKey !== 'unassigned' &&+ candidate.connectionKey !== CODEX_UNASSIGNED_CONNECTION_KEY &&Number.isInteger(candidate.catalogRevision) &&Number(candidate.catalogRevision) >= 1 &&Number.isInteger(candidate.credentialRevision) &&diff --git a/workflow-recipes/package-lock.json b/workflow-recipes/package-lock.jsonindex 19d142c68..714918528 100644--- a/workflow-recipes/package-lock.json+++ b/workflow-recipes/package-lock.json@@ -1,12 +1,12 @@{"name": "workload-recipes-controller",- "version": "0.1.323",+ "version": "0.1.324","lockfileVersion": 3,"requires": true,"packages": {"": {"name": "workload-recipes-controller",- "version": "0.1.323",+ "version": "0.1.324","license": "SEE LICENSE IN LICENSE","dependencies": {"@aws-sdk/client-s3": "^3.1009.0",diff --git a/workflow-recipes/package.json b/workflow-recipes/package.jsonindex 7907fb24a..06224611e 100644--- a/workflow-recipes/package.json+++ b/workflow-recipes/package.json@@ -1,6 +1,6 @@{"name": "workload-recipes-controller",- "version": "0.1.323",+ "version": "0.1.324","description": "Workload Recipes Controller (WRC) - WorkloadRecipe lifecycle management and MCP server interface","main": "dist/main.js","engines": {diff --git a/workflow-recipes/src/reconciler/pluginWorkloadSdkValidator.test.ts b/workflow-recipes/src/reconciler/pluginWorkloadSdkValidator.test.tsindex c1d2ed55e..29b9184d0 100644--- a/workflow-recipes/src/reconciler/pluginWorkloadSdkValidator.test.ts+++ b/workflow-recipes/src/reconciler/pluginWorkloadSdkValidator.test.ts@@ -594,6 +594,43 @@ describe('buildPluginWorkloadSdkStatus', () => {})})+ it('names a stale Codex bootstrap contract when the binding proof is absent', () => {+ const projection = buildPluginWorkloadSdkStatus({+ spec: baseSpec(+ { promptBridge: {} },+ { agent: { provider: 'codex-subscription', model: 'gpt-5.6-luna' } }+ ),+ existingConditions: undefined,+ phase: 'active',+ featureFlagEnabled: true,+ bootstrapProof: {+ ready: true,+ contractVersion: 2,+ podUid: 'pod-uid-1',+ provider: 'codex-subscription',+ model: 'gpt-5.6-luna',+ verifiedAt: NOW,+ },+ now: NOW,+ })+ expect(projection.conditions).toEqual(+ expect.arrayContaining([+ expect.objectContaining({+ type: PLUGIN_WORKLOAD_SDK_CONDITION_TYPE,+ status: 'False',+ reason: 'PolicyNotConfigured',+ message:+ 'Plugin Workload SDK promptBridge policy is not ready (codex_bootstrap_contract_stale)',+ }),+ expect.objectContaining({+ type: PLUGIN_WORKLOAD_SDK_POLICY_PENDING_CONDITION_TYPE,+ status: 'True',+ reason: 'codex_bootstrap_contract_stale',+ }),+ ])+ )+ })+it('does not validate a clientNotifications-only recipe before its grant is proven', () => {const projection = buildPluginWorkloadSdkStatus({spec: baseSpec(diff --git a/workflow-recipes/src/reconciler/pluginWorkloadSdkValidator.ts b/workflow-recipes/src/reconciler/pluginWorkloadSdkValidator.tsindex 4f435fdb9..e67b7ad28 100644--- a/workflow-recipes/src/reconciler/pluginWorkloadSdkValidator.ts+++ b/workflow-recipes/src/reconciler/pluginWorkloadSdkValidator.ts@@ -361,7 +361,13 @@ export function buildPluginWorkloadSdkStatus(args: {? (bootstrapProof?.clientNotificationsPolicyReason ??bootstrapProof?.clientNotificationsPolicyState ??'unknown')- : (bootstrapProof?.policyReason ?? bootstrapProof?.policyState ?? 'unknown')+ : (bootstrapProof?.policyReason ??+ bootstrapProof?.policyState ??+ (codexBindingPending+ ? bootstrapProof?.contractVersion !== 3+ ? 'codex_bootstrap_contract_stale'+ : 'codex_execution_binding_missing'+ : 'unknown'))const message =policyReason === 'grant_missing'? `Plugin Workload SDK ${family} is awaiting an operator grant`diff --git a/workflow-recipes/src/workflow/pluginWorkloadSdkPodDrift.test.ts b/workflow-recipes/src/workflow/pluginWorkloadSdkPodDrift.test.tsindex e2b129d45..1c957d48f 100644--- a/workflow-recipes/src/workflow/pluginWorkloadSdkPodDrift.test.ts+++ b/workflow-recipes/src/workflow/pluginWorkloadSdkPodDrift.test.ts@@ -63,6 +63,27 @@ describe('readMcpHostPodDriftStateIfExists', () => {})})+ it('reads the runtime token generation used after a failed remint roll', async () => {+ const coreApi = {+ readNamespacedPod: vi.fn().mockResolvedValue({+ metadata: {+ annotations: {+ 'clerum.io/mcp-host-runtime-token-generation': '3',+ },+ },+ spec: { containers: [{ name: 'mcp-host', image: 'registry.example/mcp-host:current' }] },+ }),+ }++ await expect(+ readMcpHostPodDriftStateIfExists(coreApi as never, POD_NAME, NAMESPACE)+ ).resolves.toEqual({+ image: 'registry.example/mcp-host:current',+ tokenGeneration: '3',+ deleting: false,+ })+ })+it('returns undefined when the pod disappeared before the drift read', async () => {const coreApi = {readNamespacedPod: vi.fn().mockRejectedValue({ code: 404 }),diff --git a/workflow-recipes/src/workflow/pluginWorkloadSdkPodDrift.ts b/workflow-recipes/src/workflow/pluginWorkloadSdkPodDrift.tsindex 4a9cd7808..75fd9f93c 100644--- a/workflow-recipes/src/workflow/pluginWorkloadSdkPodDrift.ts+++ b/workflow-recipes/src/workflow/pluginWorkloadSdkPodDrift.ts@@ -1,10 +1,12 @@import * as k8s from '@kubernetes/client-node'import { getErrorCode } from '../reconciler/k8sErrors'import { PLUGIN_WORKLOAD_SDK_RUNTIME_CONTRACT_HASH_ANNOTATION } from './podFactory'+import { MCP_HOST_RUNTIME_TOKEN_GENERATION_ANNOTATION } from './secretFactory'export type McpHostPodDriftState = {image?: stringruntimeContractHash?: string+ tokenGeneration?: stringdeleting: boolean}@@ -23,9 +25,12 @@ export async function readMcpHostPodDriftStateIfExists(const mcpHostContainer = pod.spec?.containers?.find(container => container.name === 'mcp-host')const runtimeContractHash =pod.metadata?.annotations?.[PLUGIN_WORKLOAD_SDK_RUNTIME_CONTRACT_HASH_ANNOTATION]+ const tokenGeneration =+ pod.metadata?.annotations?.[MCP_HOST_RUNTIME_TOKEN_GENERATION_ANNOTATION]return {image: mcpHostContainer?.image,...(runtimeContractHash ? { runtimeContractHash } : {}),+ ...(tokenGeneration ? { tokenGeneration } : {}),deleting: Boolean(pod.metadata?.deletionTimestamp),}} catch (error: unknown) {diff --git a/workflow-recipes/src/workflow/pluginWorkloadSdkProvisioner.imageDrift.test.ts b/workflow-recipes/src/workflow/pluginWorkloadSdkProvisioner.imageDrift.test.tsindex 69b9f1e38..5dbc713e4 100644--- a/workflow-recipes/src/workflow/pluginWorkloadSdkProvisioner.imageDrift.test.ts+++ b/workflow-recipes/src/workflow/pluginWorkloadSdkProvisioner.imageDrift.test.ts@@ -16,19 +16,21 @@ const RECIPE = 'demo'function podWithPhase(image: string,phase: 'Running' | 'Pending',- opts: { deleting?: boolean; runtimeContractHash?: string } = {}+ opts: { deleting?: boolean; runtimeContractHash?: string; tokenGeneration?: string } = {}) {+ const annotations = {+ ...(opts.runtimeContractHash+ ? { 'clerum.io/plugin-workload-sdk-runtime-contract-hash': opts.runtimeContractHash }+ : {}),+ ...(opts.tokenGeneration+ ? { 'clerum.io/mcp-host-runtime-token-generation': opts.tokenGeneration }+ : {}),+ }return {metadata: {uid: 'pod-uid-1',...(opts.deleting ? { deletionTimestamp: '2026-06-19T18:00:00Z' } : {}),- ...(opts.runtimeContractHash- ? {- annotations: {- 'clerum.io/plugin-workload-sdk-runtime-contract-hash': opts.runtimeContractHash,- },- }- : {}),+ ...(Object.keys(annotations).length > 0 ? { annotations } : {}),},spec: { containers: [{ name: 'mcp-host', image }] },status: { phase, conditions: [], containerStatuses: [] },@@ -41,10 +43,21 @@ function makeProvisioner(opts: {deleting?: booleanruntimeContractHash?: string- tokenRefresh?: { reminted: boolean; reason?: 'scope' | 'binding' | 'ttl' }+ tokenRefresh?: {+ reminted: boolean+ reason?: 'scope' | 'binding' | 'ttl'+ tokenGeneration?: string+ }+ tokenGeneration?: string} = {}) {- const readNamespacedPod = vi.fn().mockResolvedValue(podWithPhase(podImage, phase, opts))+ const readNamespacedPod = vi.fn().mockResolvedValue(+ podWithPhase(podImage, phase, {+ deleting: opts.deleting,+ runtimeContractHash: opts.runtimeContractHash,+ tokenGeneration: opts.tokenGeneration,+ })+ )const deleteNamespacedPod = vi.fn().mockResolvedValue({})const createNamespacedPod = vi.fn().mockResolvedValue({})const coreApi = {@@ -550,4 +563,53 @@ describe('ensureEagerSdkMcpHost runtime token roll', () => {expect(deleteNamespacedPod).not.toHaveBeenCalled()})++ it('rolls a healthy eager mcp-host when Secret token generation drifted after a failed roll', async () => {+ const { provisioner, deleteNamespacedPod, createNamespacedPod } = makeProvisioner(+ DESIRED_IMAGE,+ 'Running',+ {+ runtimeContractHash: desiredRuntimeContractHash(),+ tokenGeneration: '1',+ tokenRefresh: { reminted: false, tokenGeneration: '2' },+ }+ )++ const status = await provisioner.ensureEagerSdkMcpHost(+ RECIPE,+ 'recipe-uid',+ SANDBOX_NS,+ RECIPE,+ SPEC,+ RUNTIME,+ { mcpHostPhase: 'Running' }+ )++ expect(status).toBe('deploying')+ expect(deleteNamespacedPod).toHaveBeenCalledWith({+ name: `${RECIPE}-mcp-host`,+ namespace: SANDBOX_NS,+ })+ expect(createNamespacedPod).not.toHaveBeenCalled()+ })++ it('does not roll when Secret and pod token generations already match', async () => {+ const { provisioner, deleteNamespacedPod } = makeProvisioner(DESIRED_IMAGE, 'Running', {+ runtimeContractHash: desiredRuntimeContractHash(),+ tokenGeneration: '2',+ tokenRefresh: { reminted: false, tokenGeneration: '2' },+ })++ await provisioner.ensureEagerSdkMcpHost(+ RECIPE,+ 'recipe-uid',+ SANDBOX_NS,+ RECIPE,+ SPEC,+ RUNTIME,+ { mcpHostPhase: 'Running' }+ )++ expect(deleteNamespacedPod).not.toHaveBeenCalled()+ })})diff --git a/workflow-recipes/src/workflow/pluginWorkloadSdkProvisioner.tokenRoll.test.ts b/workflow-recipes/src/workflow/pluginWorkloadSdkProvisioner.tokenRoll.test.tsnew file mode 100644index 000000000..466928a97--- /dev/null+++ b/workflow-recipes/src/workflow/pluginWorkloadSdkProvisioner.tokenRoll.test.ts@@ -0,0 +1,29 @@+import { describe, expect, it } from 'vitest'+import {+ eagerMcpHostPodTokenGenerationDrift,+ eagerMcpHostRequiresTokenRoll,+} from './pluginWorkloadSdkProvisioner'++describe('eager mcp-host token roll predicates', () => {+ it('rolls only on this-pass scope or binding remint', () => {+ expect(eagerMcpHostRequiresTokenRoll({ reminted: true, reason: 'scope' })).toBe(true)+ expect(eagerMcpHostRequiresTokenRoll({ reminted: true, reason: 'binding' })).toBe(true)+ expect(eagerMcpHostRequiresTokenRoll({ reminted: true, reason: 'ttl' })).toBe(false)+ expect(eagerMcpHostRequiresTokenRoll({ reminted: false })).toBe(false)+ expect(eagerMcpHostRequiresTokenRoll(undefined)).toBe(false)+ })++ it('rolls a leftover pod when the Secret generation residue drifted', () => {+ expect(+ eagerMcpHostPodTokenGenerationDrift({ reminted: false, tokenGeneration: '2' }, '1')+ ).toBe(true)+ expect(+ eagerMcpHostPodTokenGenerationDrift({ reminted: false, tokenGeneration: '2' }, undefined)+ ).toBe(true)+ expect(+ eagerMcpHostPodTokenGenerationDrift({ reminted: false, tokenGeneration: '2' }, '2')+ ).toBe(false)+ expect(eagerMcpHostPodTokenGenerationDrift({ reminted: false }, '1')).toBe(false)+ expect(eagerMcpHostPodTokenGenerationDrift(undefined, '1')).toBe(false)+ })+})diff --git a/workflow-recipes/src/workflow/pluginWorkloadSdkProvisioner.ts b/workflow-recipes/src/workflow/pluginWorkloadSdkProvisioner.tsindex 6e2b94dda..00543a548 100644--- a/workflow-recipes/src/workflow/pluginWorkloadSdkProvisioner.ts+++ b/workflow-recipes/src/workflow/pluginWorkloadSdkProvisioner.ts@@ -51,6 +51,8 @@ export type McpHostRuntimeTokenRefreshReason = 'scope' | 'binding' | 'ttl'export type McpHostRuntimeTokenRefreshResult = {reminted: booleanreason?: McpHostRuntimeTokenRefreshReason+ /** Secret annotation; eager pods must match or they still carry a stale env JWT. */+ tokenGeneration?: string}export const NO_MCP_HOST_RUNTIME_TOKEN_REFRESH: McpHostRuntimeTokenRefreshResult = {@@ -64,6 +66,16 @@ export function eagerMcpHostRequiresTokenRoll(return result?.reminted === true && (result.reason === 'scope' || result.reason === 'binding')}+/** Prior remint patched the Secret; the running process may still have the old JWT. */+export function eagerMcpHostPodTokenGenerationDrift(+ result: McpHostRuntimeTokenRefreshResult | void,+ podGeneration?: string+): boolean {+ const expected = result?.tokenGeneration+ if (!expected) return false+ return podGeneration !== expected+}+export interface EagerSdkBootstrapProof {ready: truecontractVersion: 2 | 3@@ -257,6 +269,9 @@ export class PluginWorkloadSdkProvisioner {mountWorkflowOutput: false,pluginWorkloadSdkCapabilities: capabilities,pluginWorkloadSdkRuntimeMode: 'sdk-only',+ ...(tokenRefresh?.tokenGeneration+ ? { runtimeTokenGeneration: tokenRefresh.tokenGeneration }+ : {}),})const desiredRuntimeContractHash = pluginWorkloadSdkRuntimeContractHash(desiredMcpHostPod)@@ -278,14 +293,11 @@ export class PluginWorkloadSdkProvisioner {}}- // Runtime JWT scope/binding remint. Access/refresh tokens are env-injected- // on a bare Pod, so a Secret patch does not update process.env. HCC rolls- // on contract_changed; WRC must do the same or authorize keeps the first-boot- // JWT (insufficient_scope after reservation-only Codex).- if (- eagerMcpHostRequiresTokenRoll(tokenRefresh) &&- (mcpHostPhase === 'Running' || mcpHostPhase === 'Pending')- ) {+ // Runtime JWT scope/binding remint, plus a Secret/Pod generation residue+ // when a prior remint patched the Secret but the roll failed. Access/refresh+ // tokens are env-injected on a bare Pod; Secret scopes matching is not proof+ // that process.env was refreshed.+ if (mcpHostPhase === 'Running' || mcpHostPhase === 'Pending') {const runningPod = await readMcpHostPodDriftStateIfExists(this.deps.coreApi,`${recipeName}-mcp-host`,@@ -294,32 +306,22 @@ export class PluginWorkloadSdkProvisioner {if (runningPod?.deleting) {return 'deploying'}- log.info('Plugin Workload SDK eager mcp-host runtime token drift; rolling Pod', {- reason: tokenRefresh?.reason,- })- await this.rollEagerSdkMcpHostPod(recipeName)- return 'deploying'- }-- // Image-drift roll. The eager SDK mcp-host is a bare Pod (no Deployment),- // so a platform mcp-host image bump never reaches a long-lived recipe on- // its own: createIfNotExists is a no-op while the stale pod stays healthy,- // and there is no owning controller to perform a rolling update. A recipe- // deployed before an mcp-host release therefore keeps serving SDK routes- // from the old image (e.g. a pre-recipients-endpoint pod returns 404 for- // GET /sdk/v1/client-notifications/recipients). Roll the pod so the next- // reconcile recreates it from config.mcpHostImage. Safe on this path: the- // eager pod hosts only the always-on SDK server and produces no run- // artifacts, and no triggered run is in flight (see ensureEagerSdkMcpHost).- if (mcpHostPhase === 'Running' || mcpHostPhase === 'Pending') {- const runningPod = await readMcpHostPodDriftStateIfExists(- this.deps.coreApi,- `${recipeName}-mcp-host`,- this.deps.config.sandboxNamespace- )- if (runningPod?.deleting) {+ const tokenRoll =+ eagerMcpHostRequiresTokenRoll(tokenRefresh) ||+ eagerMcpHostPodTokenGenerationDrift(tokenRefresh, runningPod?.tokenGeneration)+ if (tokenRoll) {+ log.info('Plugin Workload SDK eager mcp-host runtime token drift; rolling Pod', {+ reason: tokenRefresh?.reason,+ secretTokenGeneration: tokenRefresh?.tokenGeneration,+ podTokenGeneration: runningPod?.tokenGeneration,+ })+ await this.rollEagerSdkMcpHostPod(recipeName)return 'deploying'}+ // Image-drift roll. The eager SDK mcp-host is a bare Pod (no Deployment),+ // so a platform mcp-host image bump never reaches a long-lived recipe on+ // its own: createIfNotExists is a no-op while the stale pod stays healthy,+ // and there is no owning controller to perform a rolling update.const imageDrift = runningPod?.image !== this.deps.config.mcpHostImageconst runtimeContractDrift = runningPod?.runtimeContractHash !== desiredRuntimeContractHashif (runningPod && (imageDrift || runtimeContractDrift)) {diff --git a/workflow-recipes/src/workflow/podFactory.ts b/workflow-recipes/src/workflow/podFactory.tsindex 07d6401ca..06afbdf40 100644--- a/workflow-recipes/src/workflow/podFactory.ts+++ b/workflow-recipes/src/workflow/podFactory.ts@@ -22,6 +22,7 @@ import {DEFAULT_CUSTOM_COORDINATOR_ACTIVE_DEADLINE_SECONDS,WORKFLOW_OUTPUT_PVC_NAME,} from './runtimeConstants'+import { MCP_HOST_RUNTIME_TOKEN_GENERATION_ANNOTATION } from './secretFactory'import { AgentSpec, WorkflowConfig } from './types'/** Directory where the mcp-host mounts per-caller SDK bearer tokens (Secret volume). */@@ -600,6 +601,8 @@ export interface McpHostPodOptions {pluginWorkloadSdkEnabled?: boolean/** Runtime contract used by mcp-host usage attribution. */pluginWorkloadSdkRuntimeMode?: 'workflow' | 'sdk-only'+ /** Secret residue copied onto the eager pod after a scope/binding remint. */+ runtimeTokenGeneration?: string}export function buildMcpHostPod(@@ -886,6 +889,9 @@ export function buildMcpHostPod(...(pod.metadata?.annotations ?? {}),[PLUGIN_WORKLOAD_SDK_RUNTIME_CONTRACT_HASH_ANNOTATION]:pluginWorkloadSdkRuntimeContractHash(pod),+ ...(options.runtimeTokenGeneration+ ? { [MCP_HOST_RUNTIME_TOKEN_GENERATION_ANNOTATION]: options.runtimeTokenGeneration }+ : {}),},}}diff --git a/workflow-recipes/src/workflow/sdkOnlyCodexBinding.test.ts b/workflow-recipes/src/workflow/sdkOnlyCodexBinding.test.tsindex d0c182229..2c834ba76 100644--- a/workflow-recipes/src/workflow/sdkOnlyCodexBinding.test.ts+++ b/workflow-recipes/src/workflow/sdkOnlyCodexBinding.test.ts@@ -40,6 +40,29 @@ describe('deriveSdkOnlyCodexBinding', () => {expect(isPluginWorkloadSdkCodexBindingProof(binding)).toBe(true)})+ it('returns null when the assigned catalog excludes the requested model', () => {+ expect(+ deriveSdkOnlyCodexBinding({+ provider: 'codex-subscription',+ model: 'gpt-5.6-luna',+ connectionKey: 'team-plus',+ configMap: {+ metadata: {+ annotations: {+ 'clerum.io/codex-connections': JSON.stringify({+ 'team-plus': {+ models: ['gpt-5.1'],+ catalogRevision: 3,+ connectionRevision: 1,+ },+ }),+ },+ },+ },+ })+ ).toBeNull()+ })+it('returns null for non-Codex providers and unassigned grants', () => {expect(deriveSdkOnlyCodexBinding({diff --git a/workflow-recipes/src/workflow/secretFactory.ts b/workflow-recipes/src/workflow/secretFactory.tsindex c8a99172d..10443a829 100644--- a/workflow-recipes/src/workflow/secretFactory.ts+++ b/workflow-recipes/src/workflow/secretFactory.ts@@ -13,6 +13,24 @@ import { JwtTokenFactory } from './jwtTokenFactory'import { pluginWorkloadSdkTokenSecretKey } from './pluginWorkloadSdkTokens'import { buildPluginWorkloadSdkTokenSecretName } from './resourceNames'+/** Secret/Pod residue so a failed eager roll cannot leave a stale env JWT. */+export const MCP_HOST_RUNTIME_TOKEN_GENERATION_ANNOTATION =+ 'clerum.io/mcp-host-runtime-token-generation'++export function readMcpHostRuntimeTokenGeneration(+ secret: { metadata?: { annotations?: Record<string, string> | null } | null } | undefined+): string | undefined {+ const raw = secret?.metadata?.annotations?.[MCP_HOST_RUNTIME_TOKEN_GENERATION_ANNOTATION]+ const trimmed = typeof raw === 'string' ? raw.trim() : ''+ return trimmed.length > 0 ? trimmed : undefined+}++export function nextMcpHostRuntimeTokenGeneration(current?: string): string {+ const parsed = Number(current)+ if (Number.isInteger(parsed) && parsed >= 1) return String(parsed + 1)+ return '1'+}+export function buildCoordinatorTokenSecret(recipeName: string,mcpHostToken: string | undefined,diff --git a/workflow-recipes/src/workflow/workflowReconciler.codexScopeProvenance.test.ts b/workflow-recipes/src/workflow/workflowReconciler.codexScopeProvenance.test.tsindex f725ce689..04bad1016 100644--- a/workflow-recipes/src/workflow/workflowReconciler.codexScopeProvenance.test.ts+++ b/workflow-recipes/src/workflow/workflowReconciler.codexScopeProvenance.test.ts@@ -387,11 +387,17 @@ describe('WorkflowReconciler Codex scope provenance', () => {expect(issueMcpHostRuntimeTokens).toHaveBeenCalled()expect(issuedScopes()).not.toContain('llm:codex:execute')- expect(crashRecoveryMocks.deletePodIfExists).toHaveBeenCalledWith(+ expect(crashRecoveryMocks.deletePodIfExists).not.toHaveBeenCalledWith(expect.anything(),'codex-recipe-mcp-host',sandboxNamespace)+ const remintPatch = coreApi.patchNamespacedSecret.mock.calls.at(-1)?.[0] as {+ body?: { metadata?: { annotations?: Record<string, string> } }+ }+ expect(+ remintPatch.body?.metadata?.annotations?.['clerum.io/mcp-host-runtime-token-generation']+ ).toBe('1')expect(networkingApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledWith({name: CODEX_PROXY_POLICY,namespace: sandboxNamespace,@@ -440,7 +446,7 @@ describe('WorkflowReconciler Codex scope provenance', () => {expect(issueMcpHostRuntimeTokens).toHaveBeenCalled()expect(issuedScopes()).not.toContain('llm:codex:execute')- expect(crashRecoveryMocks.deletePodIfExists).toHaveBeenCalledWith(+ expect(crashRecoveryMocks.deletePodIfExists).not.toHaveBeenCalledWith(expect.anything(),'codex-recipe-mcp-host',sandboxNamespacediff --git a/workflow-recipes/src/workflow/workflowReconciler.ts b/workflow-recipes/src/workflow/workflowReconciler.tsindex 201d65a31..1dd970072 100644--- a/workflow-recipes/src/workflow/workflowReconciler.ts+++ b/workflow-recipes/src/workflow/workflowReconciler.ts@@ -141,7 +141,13 @@ import {reconcileScheduling,} from './schedulingHandler'import { deriveSdkOnlyCodexBinding } from './sdkOnlyCodexBinding'-import { buildMcpHostRuntimeTokenSecret, createCoordinatorTokens } from './secretFactory'+import {+ MCP_HOST_RUNTIME_TOKEN_GENERATION_ANNOTATION,+ buildMcpHostRuntimeTokenSecret,+ createCoordinatorTokens,+ nextMcpHostRuntimeTokenGeneration,+ readMcpHostRuntimeTokenGeneration,+} from './secretFactory'import { SNIPPET_RUN_KEYS } from './snippetRunSchema'import {CyclicDependencyError,@@ -4812,7 +4818,10 @@ export class WorkflowReconciler {gfsAccessExp - nowSecs < refreshBeforeSecondsif (!needsMcpHostRuntimeRefresh && !needsWorkflowControlRefresh && !needsGfsAccessRefresh) {- return NO_MCP_HOST_RUNTIME_TOKEN_REFRESH+ const tokenGeneration = readMcpHostRuntimeTokenGeneration(existing)+ return tokenGeneration+ ? { reminted: false, tokenGeneration }+ : NO_MCP_HOST_RUNTIME_TOKEN_REFRESH}const refreshReason: McpHostRuntimeTokenRefreshReason =@@ -4885,12 +4894,26 @@ export class WorkflowReconciler {data['mcp-host-gfs-token'] = Buffer.from(gfs.token).toString('base64')}+ const currentGeneration = readMcpHostRuntimeTokenGeneration(existing)+ const tokenGeneration =+ refreshReason === 'scope' || refreshReason === 'binding'+ ? nextMcpHostRuntimeTokenGeneration(currentGeneration)+ : currentGenerationawait this.deps.coreApi.patchNamespacedSecret({name: `wf-${recipeName}-mcp-host-runtime-tokens`,namespace: this.deps.config.sandboxNamespace,body: {data,+ ...(tokenGeneration && (refreshReason === 'scope' || refreshReason === 'binding')+ ? {+ metadata: {+ annotations: {+ [MCP_HOST_RUNTIME_TOKEN_GENERATION_ANNOTATION]: tokenGeneration,+ },+ },+ }+ : {}),},},{ middleware: [k8s.setHeaderMiddleware('Content-Type', 'application/merge-patch+json')] }@@ -4898,16 +4921,16 @@ export class WorkflowReconciler {this.log.info(`Refreshed workflow auth Secret for ${recipeName} (mcpHostRuntime=${needsMcpHostRuntimeRefresh}, mcpHostControl=${needsWorkflowControlRefresh}, reason=${refreshReason})`)- if (refreshReason === 'scope' || refreshReason === 'binding') {- // Access/refresh JWTs are env-injected. A Secret patch does not update- // the running process; refresh/reissue copy the stale scopes.- await deletePodIfExists(- this.deps.coreApi,- `${recipeName}-mcp-host`,- this.deps.config.sandboxNamespace- )+ // Do not delete ${recipeName}-mcp-host here. This helper also runs on the+ // in-progress step-based credential refresh path, where a mid-run pod+ // delete is not recreated. Eager sdk-only hosts roll via+ // eagerMcpHostRequiresTokenRoll → rollEagerSdkMcpHostPod, including when+ // this pass only reports the Secret generation residue.+ return {+ reminted: true,+ reason: refreshReason,+ ...(tokenGeneration ? { tokenGeneration } : {}),}- return { reminted: true, reason: refreshReason }} catch (err) {this.log.error(`Failed to refresh mcpHost runtime tokens for ${recipeName}`, {error: err instanceof Error ? err.message : String(err),diff --git a/workflow-recipes/tests/unit/workflow/secretFactory.mcpHostRuntimeTokens.test.ts b/workflow-recipes/tests/unit/workflow/secretFactory.mcpHostRuntimeTokens.test.tsindex 6bdae10f1..34c71bbe9 100644--- a/workflow-recipes/tests/unit/workflow/secretFactory.mcpHostRuntimeTokens.test.ts+++ b/workflow-recipes/tests/unit/workflow/secretFactory.mcpHostRuntimeTokens.test.ts@@ -2,6 +2,8 @@ import { describe, expect, it } from 'vitest'import {buildCoordinatorTokenSecret,buildMcpHostRuntimeTokenSecret,+ nextMcpHostRuntimeTokenGeneration,+ readMcpHostRuntimeTokenGeneration,} from '../../../src/workflow/secretFactory'describe('Secret Factory — mcpHost Runtime Tokens', () => {@@ -74,6 +76,17 @@ describe('Secret Factory — mcpHost Runtime Tokens', () => {expect(secret.metadata!.ownerReferences).toBeUndefined()})+ it('bumps the runtime token generation from a missing or invalid residue', () => {+ expect(nextMcpHostRuntimeTokenGeneration(undefined)).toBe('1')+ expect(nextMcpHostRuntimeTokenGeneration('1')).toBe('2')+ expect(readMcpHostRuntimeTokenGeneration(undefined)).toBeUndefined()+ expect(+ readMcpHostRuntimeTokenGeneration({+ metadata: { annotations: { 'clerum.io/mcp-host-runtime-token-generation': '4' } },+ })+ ).toBe('4')+ })+// ── Existing Secret unaffected ───────────────────────────────────────it('does NOT modify the existing coordinator-token Secret', () => {