Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript

fix: fence Codex ACK loss, sweeper races, and leftover mcp-host JWTs

Alfredo Lopez785d3ba4c445
diff --git a/control-api/package-lock.json b/control-api/package-lock.json
index aaedd9871..6b04124cf 100644
--- a/control-api/package-lock.json
+++ b/control-api/package-lock.json
@@ -1,12 +1,12 @@
{
"name": "clerum-control-api",
- "version": "0.1.530",
+ "version": "0.1.531",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "clerum-control-api",
- "version": "0.1.530",
+ "version": "0.1.531",
"dependencies": {
"@clerum/display-field": "file:../packages/display-field",
"@clerum/image-policy": "file:../packages/image-policy",
diff --git a/control-api/package.json b/control-api/package.json
index bddadf99e..94132485b 100644
--- a/control-api/package.json
+++ b/control-api/package.json
@@ -1,6 +1,6 @@
{
"name": "clerum-control-api",
- "version": "0.1.530",
+ "version": "0.1.531",
"description": "Internal Clerum control plane REST API + UI for managing CRDs and secrets",
"main": "dist/main.js",
"engines": {
diff --git a/control-api/src/routes/mcp-host/plugin-workload-sdk.routes.ts b/control-api/src/routes/mcp-host/plugin-workload-sdk.routes.ts
index fbe2913bf..1f75d5bd2 100644
--- a/control-api/src/routes/mcp-host/plugin-workload-sdk.routes.ts
+++ b/control-api/src/routes/mcp-host/plugin-workload-sdk.routes.ts
@@ -718,11 +718,17 @@ export function createMcpHostPluginWorkloadSdkRoutes(): Router {
isClientNotificationsPolicyReady(clientNotificationsGrant) &&
(await hasUsableClientNotificationRecipients(clientNotificationsGrant))
)
- const primaryTarget = grant?.promptTargets[0] ?? null
+ const alignedPrimary =
+ grant &&
+ grant.promptTargets[0] &&
+ grant.defaultTargetRef !== null &&
+ grant.promptTargets[0].targetRef === grant.defaultTargetRef
+ ? grant.promptTargets[0]
+ : null
const reservationOnlyOauthBroker = Boolean(
- primaryTarget &&
- isLlmProviderId(primaryTarget.provider) &&
- PROVIDER_AUTH_MODE[primaryTarget.provider] === 'oauth-broker'
+ alignedPrimary &&
+ isLlmProviderId(alignedPrimary.provider) &&
+ PROVIDER_AUTH_MODE[alignedPrimary.provider] === 'oauth-broker'
)
// Old mcp-host/WRC require capabilities.contractVersion === 2. Advertise
// 3 only for oauth-broker defaults so those binaries fail at reconcile
@@ -739,9 +745,9 @@ export function createMcpHostPluginWorkloadSdkRoutes(): Router {
policyRevision: grant?.policyRevision ?? 0,
policyHash: v2Ready && grant ? hashPromptTargetPolicy(grant) : null,
defaultTargetRef: grant?.defaultTargetRef ?? null,
- defaultProvider: primaryTarget?.provider ?? null,
- defaultModel: primaryTarget?.model ?? null,
- defaultConnectionRef: primaryTarget?.connectionRef ?? null,
+ defaultProvider: alignedPrimary?.provider ?? null,
+ defaultModel: alignedPrimary?.model ?? null,
+ defaultConnectionRef: alignedPrimary?.connectionRef ?? null,
v2Ready,
...(reservationOnlyOauthBroker ? { reservationOnlyOauthBroker: true } : {}),
clientNotificationsPolicyState: clientNotificationsGrant?.policyState ?? 'missing',
diff --git a/control-api/src/services/llmProviderAttemptStore.ts b/control-api/src/services/llmProviderAttemptStore.ts
index 1e1c2d494..326701aa6 100644
--- a/control-api/src/services/llmProviderAttemptStore.ts
+++ b/control-api/src/services/llmProviderAttemptStore.ts
@@ -35,6 +35,24 @@ export type LlmProviderAttemptRow = LlmProviderAttemptInsert & {
createdAt: Date
}
+export function isLinkedCodexUsageReady(
+ row: Pick<LlmProviderAttemptRow, 'outcome' | 'usageInputTokens' | 'usageOutputTokens'>
+): boolean {
+ return row.outcome === 'success' && row.usageInputTokens != null && row.usageOutputTokens != null
+}
+
+/** Linked Codex still owns exact usage; sweepers must not freeze SDK spend. */
+export function isLinkedCodexInFlightWithoutUsage(
+ row: Pick<LlmProviderAttemptRow, 'status' | 'outcome' | 'usageInputTokens' | 'usageOutputTokens'>
+): boolean {
+ if (isLinkedCodexUsageReady(row)) return false
+ return (
+ row.status === 'authorized' ||
+ row.status === 'redeemed' ||
+ (row.status === 'finalized' && row.outcome === 'success')
+ )
+}
+
export async function applyLlmProviderAttemptSchema(db: DbClient): Promise<void> {
await db.query(`
CREATE TABLE IF NOT EXISTS llm_provider_attempts (
diff --git a/control-api/src/services/pluginWorkloadSdkDb.ts b/control-api/src/services/pluginWorkloadSdkDb.ts
index be4f43d7b..d2b2ae4e1 100644
--- a/control-api/src/services/pluginWorkloadSdkDb.ts
+++ b/control-api/src/services/pluginWorkloadSdkDb.ts
@@ -1,7 +1,12 @@
import { createHash, randomUUID } from 'node:crypto'
+import { PROVIDER_AUTH_MODE, isLlmProviderId } from '@clerum/llm-providers'
import { type DbClient, type DbTransactionClient, pool, withTransaction } from '../db.js'
import type { AdministrativeEventSubmitterPrincipalV1 } from '../middleware/tracingSubmitterAuth.js'
import { stableStringify } from '../utils/stableStringify.js'
+import {
+ isLinkedCodexInFlightWithoutUsage,
+ loadLlmProviderAttemptBySdkAttemptId,
+} from './llmProviderAttemptStore.js'
import {
type ControlApiPermissionChange,
appendControlApiPermissionEventsInTransaction,
@@ -2123,21 +2128,30 @@ export async function failStaleInvocationsInTransaction(
db: DbClient
): Promise<number> {
const timeoutSeconds = Math.max(1, Math.floor(olderThanSeconds))
- const result = await db.query(
- `UPDATE plugin_workload_sdk_invocations
- SET status = 'provider_unavailable', updated_at = now(), completed_at = now()
- WHERE status = 'in_progress'
- AND (
- (lease_expires_at IS NOT NULL AND lease_expires_at < now())
- OR (
- lease_expires_at IS NULL
- AND updated_at < now() - interval '1 second' * $1
- )
- )
- RETURNING id, attempt_generation`,
+ const stale = await db.query(
+ `SELECT id, attempt_generation, recipe_namespace, recipe_name
+ FROM plugin_workload_sdk_invocations
+ WHERE status = 'in_progress'
+ AND (
+ (lease_expires_at IS NOT NULL AND lease_expires_at < now())
+ OR (
+ lease_expires_at IS NULL
+ AND updated_at < now() - interval '1 second' * $1
+ )
+ )
+ FOR UPDATE`,
[timeoutSeconds]
)
- for (const row of result.rows as Array<{ id: string; attempt_generation: number }>) {
+ let closed = 0
+ for (const row of stale.rows as Array<{
+ id: string
+ attempt_generation: number
+ recipe_namespace: string
+ recipe_name: string
+ }>) {
+ await db.query(`SELECT pg_advisory_xact_lock(hashtext($1)::bigint)`, [
+ `plugin_workload_sdk:${row.recipe_namespace}/${row.recipe_name}`,
+ ])
const physicalAttempts = await db.query(
`SELECT id, recipe_namespace, recipe_name, attempt_generation,
attempt_index, target_ref, provider, model, credential_slot
@@ -2148,6 +2162,35 @@ export async function failStaleInvocationsInTransaction(
FOR UPDATE`,
[row.id, row.attempt_generation]
)
+ let linkedCodexInFlight = false
+ for (const attempt of physicalAttempts.rows as Array<Record<string, unknown>>) {
+ if (
+ !isLlmProviderId(String(attempt.provider)) ||
+ PROVIDER_AUTH_MODE[String(attempt.provider)] !== 'oauth-broker'
+ ) {
+ continue
+ }
+ const linked = await loadLlmProviderAttemptBySdkAttemptId(db, String(attempt.id))
+ if (linked && isLinkedCodexInFlightWithoutUsage(linked)) {
+ linkedCodexInFlight = true
+ break
+ }
+ }
+ // Codex still owns exact usage. Leave the invocation leased so finalize
+ // can 409 ledger_pending instead of freezing an unknown spend row.
+ if (linkedCodexInFlight) continue
+
+ const marked = await db.query(
+ `UPDATE plugin_workload_sdk_invocations
+ SET status = 'provider_unavailable', updated_at = now(), completed_at = now()
+ WHERE id = $1
+ AND attempt_generation = $2
+ AND status = 'in_progress'
+ RETURNING id`,
+ [row.id, row.attempt_generation]
+ )
+ if ((marked.rowCount ?? 0) !== 1) continue
+
for (const attempt of physicalAttempts.rows as Array<Record<string, unknown>>) {
// The host has disappeared, so no current JWT can prove a host_ref.
// Persist an unattributed unknown outcome instead of inventing one
@@ -2188,8 +2231,9 @@ export async function failStaleInvocationsInTransaction(
AND status IN ('reserved', 'in_progress')`,
[row.id, row.attempt_generation]
)
+ closed += 1
}
- return result.rowCount ?? 0
+ return closed
}
export interface ListInvocationsFilter {
diff --git a/control-api/src/services/pluginWorkloadSdkFinalization.ts b/control-api/src/services/pluginWorkloadSdkFinalization.ts
index 7a9f5a6e0..abe1bb0ec 100644
--- a/control-api/src/services/pluginWorkloadSdkFinalization.ts
+++ b/control-api/src/services/pluginWorkloadSdkFinalization.ts
@@ -1,6 +1,10 @@
import { PROVIDER_AUTH_MODE, isLlmProviderId } from '@clerum/llm-providers'
import type { DbClient } from '../db.js'
-import { loadLlmProviderAttemptBySdkAttemptId } from './llmProviderAttemptStore.js'
+import {
+ isLinkedCodexInFlightWithoutUsage,
+ isLinkedCodexUsageReady,
+ loadLlmProviderAttemptBySdkAttemptId,
+} from './llmProviderAttemptStore.js'
import { withTraceIngestTransaction } from './tracing/pools.js'
import { projectAcceptedUsageEvents } from './tracing/usageProjection.js'
import { ingestUsageEventsInTransaction } from './usageEvents.js'
@@ -167,12 +171,7 @@ function resolvePromptBridgeOutcome(
if (!oauthBroker) {
return status === 'complete' ? 'exact' : status === 'failed' ? 'not_executed' : 'unknown'
}
- if (
- status === 'complete' &&
- linked?.outcome === 'success' &&
- linked.usageInputTokens != null &&
- linked.usageOutputTokens != null
- ) {
+ if (status === 'complete' && linked && isLinkedCodexUsageReady(linked)) {
return 'exact'
}
if (status === 'failed' && !linked) return 'not_executed'
@@ -284,7 +283,24 @@ export async function finalizePromptBridgeInTransaction(
[input.providerAttemptId]
)
const existing = existingResult.rows[0] as SpendOutcomeRow | undefined
- if (existing) return mapExistingOutcome(existing, input)
+ if (existing) {
+ if (
+ isOauthBrokerProvider(existing.provider) &&
+ input.status === 'complete' &&
+ existing.outcome === 'unknown'
+ ) {
+ const linkedCodex = await loadLlmProviderAttemptBySdkAttemptId(db, input.providerAttemptId)
+ if (linkedCodex && isLinkedCodexInFlightWithoutUsage(linkedCodex)) {
+ throw new PromptBridgeFinalizationError(
+ 'ledger_pending',
+ 'linked Codex attempt has not finalized usage yet',
+ 409,
+ true
+ )
+ }
+ }
+ return mapExistingOutcome(existing, input)
+ }
const invocationResult = await db.query(
`SELECT id, recipe_namespace, recipe_name, method, status, attempt_generation
@@ -391,15 +407,7 @@ export async function finalizePromptBridgeInTransaction(
? await loadLlmProviderAttemptBySdkAttemptId(db, input.providerAttemptId)
: null
if (oauthBroker && input.status === 'complete' && linkedCodex) {
- const usageReady =
- linkedCodex.outcome === 'success' &&
- linkedCodex.usageInputTokens != null &&
- linkedCodex.usageOutputTokens != null
- const inFlight =
- linkedCodex.status === 'authorized' ||
- linkedCodex.status === 'redeemed' ||
- (linkedCodex.status === 'finalized' && linkedCodex.outcome === 'success' && !usageReady)
- if (!usageReady && inFlight) {
+ if (isLinkedCodexInFlightWithoutUsage(linkedCodex)) {
throw new PromptBridgeFinalizationError(
'ledger_pending',
'linked Codex attempt has not finalized usage yet',
diff --git a/control-api/test/routes.mcp-host.plugin-workload-sdk.test.ts b/control-api/test/routes.mcp-host.plugin-workload-sdk.test.ts
index e26388e00..16a32f055 100644
--- a/control-api/test/routes.mcp-host.plugin-workload-sdk.test.ts
+++ b/control-api/test/routes.mcp-host.plugin-workload-sdk.test.ts
@@ -326,6 +326,58 @@ describe('GET /mcp-host/plugin-workload-sdk/capabilities', () => {
v2Ready: true,
})
})
+
+ it('does not advertise oauth-broker v3 when promptTargets[0] is not the default', async () => {
+ vi.mocked(sdkDb.findGrant).mockResolvedValue({
+ id: 'grant-misaligned-capabilities',
+ recipeNamespace: NS,
+ recipeName: RECIPE,
+ capabilityFamily: 'promptBridge',
+ provider: 'codex-subscription',
+ allowedModels: ['gpt-5.1'],
+ allowedEventTypes: [],
+ allowedTargetRefs: [],
+ allowedUserRefs: [],
+ allowedCallers: ['api'],
+ quotaLimits: {},
+ modelPolicies: {},
+ promptTargets: [
+ {
+ targetRef: 'primary-codex',
+ provider: 'codex-subscription',
+ model: 'gpt-5.1',
+ credentialSlot: '',
+ connectionRef: 'team-plus',
+ },
+ {
+ targetRef: 'fallback-openai',
+ provider: 'openai',
+ model: 'gpt-5.4-mini',
+ credentialSlot: 'openai-api-key',
+ },
+ ],
+ defaultTargetRef: 'fallback-openai',
+ policyState: 'active',
+ policyRevision: 2,
+ createdAt: '2026-08-03T00:00:00.000Z',
+ updatedAt: '2026-08-03T00:00:00.000Z',
+ })
+
+ const res = await request(buildApp())
+ .get('/mcp-host/plugin-workload-sdk/capabilities')
+ .set('Authorization', `Bearer ${issueSdkToken()}`)
+
+ expect(res.status).toBe(200)
+ expect(res.body).toMatchObject({
+ contractVersion: 2,
+ defaultTargetRef: 'fallback-openai',
+ defaultProvider: null,
+ defaultModel: null,
+ defaultConnectionRef: null,
+ v2Ready: false,
+ })
+ expect(res.body.reservationOnlyOauthBroker).toBeUndefined()
+ })
})
describe('POST /mcp-host/plugin-workload-sdk/prompt-bridge/v2', () => {
diff --git a/control-api/test/services.llmProviderAttemptStore.helpers.test.ts b/control-api/test/services.llmProviderAttemptStore.helpers.test.ts
new file mode 100644
index 000000000..61c46467e
--- /dev/null
+++ b/control-api/test/services.llmProviderAttemptStore.helpers.test.ts
@@ -0,0 +1,74 @@
+import { describe, expect, it } from 'vitest'
+import {
+ isLinkedCodexInFlightWithoutUsage,
+ isLinkedCodexUsageReady,
+} from '../src/services/llmProviderAttemptStore.js'
+
+describe('linked Codex usage predicates', () => {
+ it('treats success plus both token counts as usage-ready', () => {
+ expect(
+ isLinkedCodexUsageReady({
+ outcome: 'success',
+ usageInputTokens: 3,
+ usageOutputTokens: 2,
+ })
+ ).toBe(true)
+ expect(
+ isLinkedCodexUsageReady({
+ outcome: 'success',
+ usageInputTokens: 3,
+ usageOutputTokens: null,
+ })
+ ).toBe(false)
+ expect(
+ isLinkedCodexUsageReady({
+ outcome: 'unknown',
+ usageInputTokens: 3,
+ usageOutputTokens: 2,
+ })
+ ).toBe(false)
+ })
+
+ it('keeps authorized, redeemed, and usage-less success finalized in flight', () => {
+ expect(
+ isLinkedCodexInFlightWithoutUsage({
+ status: 'authorized',
+ outcome: null,
+ usageInputTokens: null,
+ usageOutputTokens: null,
+ })
+ ).toBe(true)
+ expect(
+ isLinkedCodexInFlightWithoutUsage({
+ status: 'redeemed',
+ outcome: null,
+ usageInputTokens: null,
+ usageOutputTokens: null,
+ })
+ ).toBe(true)
+ expect(
+ isLinkedCodexInFlightWithoutUsage({
+ status: 'finalized',
+ outcome: 'success',
+ usageInputTokens: null,
+ usageOutputTokens: null,
+ })
+ ).toBe(true)
+ expect(
+ isLinkedCodexInFlightWithoutUsage({
+ status: 'finalized',
+ outcome: 'success',
+ usageInputTokens: 3,
+ usageOutputTokens: 2,
+ })
+ ).toBe(false)
+ expect(
+ isLinkedCodexInFlightWithoutUsage({
+ status: 'finalized',
+ outcome: 'error',
+ usageInputTokens: null,
+ usageOutputTokens: null,
+ })
+ ).toBe(false)
+ })
+})
diff --git a/control-api/test/services.pluginWorkloadSdkDb.test.ts b/control-api/test/services.pluginWorkloadSdkDb.test.ts
index 6b8669294..31f75bf02 100644
--- a/control-api/test/services.pluginWorkloadSdkDb.test.ts
+++ b/control-api/test/services.pluginWorkloadSdkDb.test.ts
@@ -400,9 +400,17 @@ describe('invocation retry lifecycle timestamps', () => {
it('sweeps by the latest retry timestamp rather than the original reservation', async () => {
vi.mocked(pool.query)
.mockResolvedValueOnce({
- rows: [{ id: 'inv-1', attempt_generation: 2 }],
+ rows: [
+ {
+ id: 'inv-1',
+ attempt_generation: 2,
+ recipe_namespace: 'sandbox-recipes',
+ recipe_name: 'sdk-recipe',
+ },
+ ],
rowCount: 1,
} as never)
+ .mockResolvedValueOnce({ rows: [], rowCount: 1 } as never)
.mockResolvedValueOnce({
rows: [
{
@@ -419,27 +427,104 @@ describe('invocation retry lifecycle timestamps', () => {
],
rowCount: 1,
} as never)
- .mockResolvedValue({ rows: [], rowCount: 1 } as never)
+ .mockResolvedValue({ rows: [{ id: 'inv-1' }], rowCount: 1 } as never)
await expect(failStaleInvocations(150)).resolves.toBe(1)
const [sql] = vi.mocked(pool.query).mock.calls[0] as unknown as [string, unknown[]]
- expect(sql).toContain("SET status = 'provider_unavailable'")
+ expect(sql).toContain('FROM plugin_workload_sdk_invocations')
expect(sql).toContain('lease_expires_at < now()')
expect(sql).toContain('lease_expires_at IS NULL')
expect(sql).toContain('updated_at < now()')
- expect(sql).toContain('RETURNING id, attempt_generation')
- expect(pool.query).toHaveBeenCalledTimes(5)
- expect(vi.mocked(pool.query).mock.calls[1]?.[0] as string).toContain(
- 'FROM plugin_workload_sdk_provider_attempts'
- )
+ expect(sql).toContain('FOR UPDATE')
+ expect(pool.query).toHaveBeenCalledTimes(7)
+ expect(vi.mocked(pool.query).mock.calls[1]?.[0] as string).toContain('pg_advisory_xact_lock')
expect(vi.mocked(pool.query).mock.calls[2]?.[0] as string).toContain(
- 'INSERT INTO plugin_workload_sdk_spend_outcomes'
+ 'FROM plugin_workload_sdk_provider_attempts'
)
expect(vi.mocked(pool.query).mock.calls[3]?.[0] as string).toContain(
"SET status = 'provider_unavailable'"
)
expect(vi.mocked(pool.query).mock.calls[4]?.[0] as string).toContain(
+ 'INSERT INTO plugin_workload_sdk_spend_outcomes'
+ )
+ expect(vi.mocked(pool.query).mock.calls[5]?.[0] as string).toContain(
"SET status = 'provider_unavailable'"
)
+ expect(vi.mocked(pool.query).mock.calls[6]?.[0] as string).toContain(
+ "SET status = 'provider_unavailable'"
+ )
+ })
+
+ it('does not freeze oauth spend while a linked Codex attempt is still in flight', async () => {
+ vi.mocked(pool.query)
+ .mockResolvedValueOnce({
+ rows: [
+ {
+ id: 'inv-codex',
+ attempt_generation: 1,
+ recipe_namespace: 'sandbox-recipes',
+ recipe_name: 'sdk-recipe',
+ },
+ ],
+ rowCount: 1,
+ } as never)
+ .mockResolvedValueOnce({ rows: [], rowCount: 1 } as never)
+ .mockResolvedValueOnce({
+ rows: [
+ {
+ id: 'attempt-codex',
+ recipe_namespace: 'sandbox-recipes',
+ recipe_name: 'sdk-recipe',
+ attempt_generation: 1,
+ attempt_index: 1,
+ target_ref: 'primary-codex',
+ provider: 'codex-subscription',
+ model: 'gpt-5.1',
+ credential_slot: '',
+ },
+ ],
+ rowCount: 1,
+ } as never)
+ .mockResolvedValueOnce({
+ rows: [
+ {
+ id: 'codex-1',
+ caller_kind: 'recipe',
+ host_ref: 'sandbox-recipes/sdk-recipe',
+ recipe_namespace: 'sandbox-recipes',
+ recipe_name: 'sdk-recipe',
+ invocation_id: 'inv-codex',
+ attempt_generation: 1,
+ provider_attempt_index: 1,
+ provider: 'codex-subscription',
+ model: 'gpt-5.1',
+ request_hash: 'hash',
+ policy_revision: 1,
+ policy_hash: 'a'.repeat(64),
+ budget_reservation_id: 'budget-1',
+ connection_revision: 1,
+ plugin_workload_sdk_provider_attempt_id: 'attempt-codex',
+ status: 'authorized',
+ outcome: null,
+ usage_input_tokens: null,
+ usage_output_tokens: null,
+ created_at: new Date('2026-09-01T00:00:00.000Z'),
+ },
+ ],
+ rowCount: 1,
+ } as never)
+ await expect(failStaleInvocations(150)).resolves.toBe(0)
+ expect(
+ vi
+ .mocked(pool.query)
+ .mock.calls.some(([sql]: [string]) =>
+ sql.includes('INSERT INTO plugin_workload_sdk_spend_outcomes')
+ )
+ ).toBe(false)
+ expect(
+ vi
+ .mocked(pool.query)
+ .mock.calls.some(([sql]: [string]) => sql.includes("SET status = 'provider_unavailable'"))
+ ).toBe(false)
})
})
diff --git a/control-api/test/services.pluginWorkloadSdkFinalization.realPostgres.integration.test.ts b/control-api/test/services.pluginWorkloadSdkFinalization.realPostgres.integration.test.ts
index 4b27e2ee2..581875699 100644
--- a/control-api/test/services.pluginWorkloadSdkFinalization.realPostgres.integration.test.ts
+++ b/control-api/test/services.pluginWorkloadSdkFinalization.realPostgres.integration.test.ts
@@ -2,6 +2,7 @@ import { afterAll, beforeAll, describe, expect, it } from 'vitest'
import { randomBytes, randomUUID } from 'node:crypto'
import { Pool, type PoolClient } from 'pg'
import { initDb } from '../src/db.js'
+import { insertLlmProviderAttempt } from '../src/services/llmProviderAttemptStore.js'
import { failStaleInvocationsInTransaction } from '../src/services/pluginWorkloadSdkDb.js'
import {
PromptBridgeFinalizationError,
@@ -428,4 +429,56 @@ describeRealPostgres('Plugin Workload SDK finalization on real PostgreSQL', () =
sweeperClient.release()
}
})
+
+ it('does not freeze oauth spend while a linked Codex attempt is still in flight', async () => {
+ const client = await dbPool.connect()
+ try {
+ await begin(client)
+ const input = await seedAttempt(client, { leaseExpired: true })
+ await client.query(
+ `UPDATE plugin_workload_sdk_provider_attempts
+ SET provider = 'codex-subscription',
+ model = 'gpt-5.1',
+ credential_slot = '',
+ target_ref = 'primary-codex'
+ WHERE id = $1`,
+ [input.providerAttemptId]
+ )
+ await insertLlmProviderAttempt(client, {
+ callerKind: 'recipe',
+ hostRef: input.hostRef,
+ recipeNamespace: input.recipeNamespace,
+ recipeName: input.recipeName,
+ invocationId: input.invocationId,
+ attemptGeneration: 1,
+ providerAttemptIndex: 1,
+ model: 'gpt-5.1',
+ requestHash: 'a'.repeat(64),
+ policyRevision: 1,
+ policyHash: 'b'.repeat(64),
+ budgetReservationId: randomUUID(),
+ connectionRevision: 1,
+ pluginWorkloadSdkProviderAttemptId: input.providerAttemptId,
+ })
+ const closed = await failStaleInvocationsInTransaction(1, client)
+ await commit(client)
+
+ expect(closed).toBe(0)
+ const spend = await dbPool.query(
+ `SELECT count(*)::int AS count
+ FROM plugin_workload_sdk_spend_outcomes
+ WHERE provider_attempt_id = $1`,
+ [input.providerAttemptId]
+ )
+ expect(spend.rows[0]?.count).toBe(0)
+ const invocation = await dbPool.query(
+ `SELECT status FROM plugin_workload_sdk_invocations WHERE id = $1`,
+ [input.invocationId]
+ )
+ expect(invocation.rows[0]?.status).toBe('in_progress')
+ } finally {
+ await client.query('ROLLBACK').catch(() => undefined)
+ client.release()
+ }
+ })
})
diff --git a/control-api/test/services.pluginWorkloadSdkFinalization.test.ts b/control-api/test/services.pluginWorkloadSdkFinalization.test.ts
index 89eef2f17..5898921d3 100644
--- a/control-api/test/services.pluginWorkloadSdkFinalization.test.ts
+++ b/control-api/test/services.pluginWorkloadSdkFinalization.test.ts
@@ -723,7 +723,7 @@ describe('Plugin Workload SDK promptBridge finalization', () => {
outputTokens: 2,
},
},
- dbWithRows([], row) as never
+ dbWithRows([], row, []) as never
)
).resolves.toMatchObject({
idempotent: true,
@@ -732,6 +732,73 @@ describe('Plugin Workload SDK promptBridge finalization', () => {
})
})
+ it('refuses to replay a sweeper unknown while the linked Codex attempt is still in flight', async () => {
+ const row = {
+ provider_attempt_id: IDS.providerAttempt,
+ invocation_id: IDS.invocation,
+ recipe_namespace: inputBase.recipeNamespace,
+ recipe_name: inputBase.recipeName,
+ attempt_generation: 1,
+ attempt_index: 1,
+ target_ref: 'codex-primary',
+ host_ref: inputBase.hostRef,
+ provider: 'codex-subscription',
+ model: 'gpt-5.1',
+ credential_slot: '',
+ outcome: 'unknown',
+ input_tokens: null,
+ output_tokens: null,
+ }
+ await expect(
+ finalizePromptBridgeInTransaction(
+ {
+ ...inputBase,
+ target: {
+ targetRef: 'codex-primary',
+ provider: 'codex-subscription',
+ model: 'gpt-5.1',
+ credentialSlot: '',
+ },
+ status: 'complete',
+ usage: {
+ llmSecretName: '',
+ callerRef: 'scanner',
+ fallbackUsed: false,
+ attemptCount: 1,
+ inputTokens: 4,
+ outputTokens: 2,
+ },
+ },
+ dbWithRows([], row, {
+ id: '33333333-3333-4333-8333-333333333333',
+ caller_kind: 'recipe',
+ host_ref: inputBase.hostRef,
+ recipe_namespace: inputBase.recipeNamespace,
+ recipe_name: inputBase.recipeName,
+ invocation_id: IDS.invocation,
+ attempt_generation: 1,
+ provider_attempt_index: 1,
+ provider: 'codex-subscription',
+ model: 'gpt-5.1',
+ request_hash: 'hash',
+ policy_revision: 1,
+ policy_hash: 'a'.repeat(64),
+ budget_reservation_id: 'budget-1',
+ connection_revision: 1,
+ status: 'authorized',
+ outcome: null,
+ usage_input_tokens: null,
+ usage_output_tokens: null,
+ created_at: new Date('2026-08-03T00:00:00.000Z'),
+ }) as never
+ )
+ ).rejects.toMatchObject({
+ code: 'ledger_pending',
+ httpStatus: 409,
+ retryable: true,
+ })
+ })
+
it('rejects empty static-key credential material from the persisted provider', async () => {
await expect(
finalizePromptBridgeInTransaction(
diff --git a/mcp-host/package-lock.json b/mcp-host/package-lock.json
index 388765e66..d4ac7c746 100644
--- a/mcp-host/package-lock.json
+++ b/mcp-host/package-lock.json
@@ -1,12 +1,12 @@
{
"name": "clerum-mcp-host",
- "version": "0.1.308",
+ "version": "0.1.309",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "clerum-mcp-host",
- "version": "0.1.308",
+ "version": "0.1.309",
"license": "SEE LICENSE IN LICENSE",
"dependencies": {
"@anthropic-ai/sdk": "^0.32.0",
diff --git a/mcp-host/package.json b/mcp-host/package.json
index b2399685e..39bb36d85 100644
--- a/mcp-host/package.json
+++ b/mcp-host/package.json
@@ -1,6 +1,6 @@
{
"name": "clerum-mcp-host",
- "version": "0.1.308",
+ "version": "0.1.309",
"description": "MCP Host - reads Host CRD and provides LLM access via OpenAI/Claude",
"main": "dist/main.js",
"engines": {
diff --git a/mcp-host/src/pluginWorkloadSdk/bootstrapIdentity.test.ts b/mcp-host/src/pluginWorkloadSdk/bootstrapIdentity.test.ts
index abce781b4..0a5b33bd4 100644
--- a/mcp-host/src/pluginWorkloadSdk/bootstrapIdentity.test.ts
+++ b/mcp-host/src/pluginWorkloadSdk/bootstrapIdentity.test.ts
@@ -1,4 +1,5 @@
import { describe, expect, it, vi } from 'vitest'
+import { CODEX_UNASSIGNED_CONNECTION_KEY } from '@clerum/codex-catalog-projection'
import { computeCodexPolicyHash } from '@clerum/llm-provider-attempt-contract'
import {
configurePluginWorkloadSdkBootstrapIdentity,
@@ -37,6 +38,24 @@ describe('readVerifiedSdkOnlyCodexBinding', () => {
binding
)
})
+
+ it('rejects the shared unassigned connection sentinel', () => {
+ expect(
+ readVerifiedSdkOnlyCodexBinding(
+ {
+ ...binding,
+ connectionKey: CODEX_UNASSIGNED_CONNECTION_KEY,
+ bindingHash: computeCodexPolicyHash({
+ model,
+ catalogRevision: 4,
+ credentialRevision: 1,
+ connectionKey: CODEX_UNASSIGNED_CONNECTION_KEY,
+ }),
+ },
+ model
+ )
+ ).toBeNull()
+ })
})
describe('Plugin Workload SDK bootstrap identity', () => {
diff --git a/mcp-host/src/pluginWorkloadSdk/promptBridge/llmBridge.failover.test.ts b/mcp-host/src/pluginWorkloadSdk/promptBridge/llmBridge.failover.test.ts
index 47ae46b74..5ec256ffe 100644
--- a/mcp-host/src/pluginWorkloadSdk/promptBridge/llmBridge.failover.test.ts
+++ b/mcp-host/src/pluginWorkloadSdk/promptBridge/llmBridge.failover.test.ts
@@ -676,6 +676,63 @@ describe('LlmBridge authorized multi-provider fallback', () => {
})
})
+ it('does not fail over a Codex target when the post-dispatch acknowledgement is lost', async () => {
+ const codex: PromptBridgeTarget = {
+ targetRef: 'codex-primary',
+ provider: 'codex-subscription',
+ model: 'gpt-5.1',
+ credentialSlot: '',
+ connectionRef: 'team-plus',
+ }
+ replaceSdkOnlyCodexBinding({
+ connectionKey: 'team-plus',
+ catalogRevision: 1,
+ credentialRevision: 0,
+ model: 'gpt-5.1',
+ bindingHash: computeCodexPolicyHash({
+ model: 'gpt-5.1',
+ catalogRevision: 1,
+ credentialRevision: 0,
+ connectionKey: 'team-plus',
+ }),
+ })
+ const first = new FakeProvider(() => Promise.reject(new Error('codex unavailable')))
+ const second = new FakeProvider(() => Promise.resolve(OK))
+ const { bridge, providerCalls } = makeBridge({
+ 'gpt-5.1': first,
+ [fallback.model]: second,
+ })
+ const providerAttemptReporter = {
+ report: vi.fn().mockRejectedValue(new Error('control-api unreachable')),
+ }
+ const issuer = {
+ issue: vi.fn(async ({ target }: { target: PromptBridgeTarget }) => ({
+ credentialTicket:
+ target.provider === 'codex-subscription' ? '' : `fresh-${target.targetRef}`,
+ providerAttemptId: `sdk-${target.targetRef}`,
+ providerAttemptIndex: target.provider === 'codex-subscription' ? 1 : 2,
+ })),
+ }
+
+ await expect(
+ bridge.complete({
+ ...request,
+ targets: [{ target: codex }, { target: fallback }],
+ credentialTicketIssuer: issuer,
+ providerAttemptReporter,
+ })
+ ).rejects.toMatchObject({
+ code: 'provider_unavailable',
+ retryable: false,
+ reason: 'outcome_unknown',
+ providerMayHaveExecuted: true,
+ })
+ expect(first.completeSingleTurn).toHaveBeenCalledOnce()
+ expect(second.completeSingleTurn).not.toHaveBeenCalled()
+ expect(providerCalls).toEqual(['codex-subscription/gpt-5.1'])
+ expect(issuer.issue).toHaveBeenCalledTimes(1)
+ })
+
it('does not execute a Codex target while its circuit is open', async () => {
const codex: PromptBridgeTarget = {
targetRef: 'codex-primary',
diff --git a/mcp-host/src/pluginWorkloadSdk/promptBridge/llmBridge.ts b/mcp-host/src/pluginWorkloadSdk/promptBridge/llmBridge.ts
index d37fd60ec..a3387eb9b 100644
--- a/mcp-host/src/pluginWorkloadSdk/promptBridge/llmBridge.ts
+++ b/mcp-host/src/pluginWorkloadSdk/promptBridge/llmBridge.ts
@@ -705,11 +705,26 @@ export class LlmBridge {
const providerMayHaveExecuted =
(error instanceof PluginWorkloadError && error.providerMayHaveExecuted) ||
(error instanceof ClassifiedProviderError && error.providerMayHaveExecuted)
- await reportProviderAttemptBestEffort(request.providerAttemptReporter, {
- providerAttemptId: ticket.providerAttemptId,
- providerAttemptIndex: ticket.providerAttemptIndex,
- status: providerMayHaveExecuted ? 'provider_unavailable' : 'failed',
- })
+ const acknowledged = await reportProviderAttemptBestEffort(
+ request.providerAttemptReporter,
+ {
+ providerAttemptId: ticket.providerAttemptId,
+ providerAttemptIndex: ticket.providerAttemptIndex,
+ status: providerMayHaveExecuted ? 'provider_unavailable' : 'failed',
+ }
+ )
+ // Mirror the non-oauth fence: a lost receipt after dispatch is not a
+ // safe failover signal. Codex may already have billed.
+ if (!acknowledged) {
+ return {
+ kind: 'error',
+ error: new ClassifiedProviderError(
+ { code: LlmErrorCode.ApiCallFailed, retryable: false },
+ providerOutcomeUnknownError(providerAttemptContext)
+ ),
+ terminal: true,
+ }
+ }
}
if (!(error instanceof ClassifiedProviderError)) {
throw providerAttemptContext
diff --git a/mcp-host/src/pluginWorkloadSdk/sdkOnlyCodexBinding.test.ts b/mcp-host/src/pluginWorkloadSdk/sdkOnlyCodexBinding.test.ts
new file mode 100644
index 000000000..7547384a2
--- /dev/null
+++ b/mcp-host/src/pluginWorkloadSdk/sdkOnlyCodexBinding.test.ts
@@ -0,0 +1,51 @@
+import { describe, expect, it } from 'vitest'
+import { CODEX_UNASSIGNED_CONNECTION_KEY } from '@clerum/codex-catalog-projection'
+import { computeCodexPolicyHash } from '@clerum/llm-provider-attempt-contract'
+import {
+ isPluginWorkloadSdkCodexBindingProof,
+ readVerifiedSdkOnlyCodexBinding,
+} from './sdkOnlyCodexBinding'
+
+describe('mcp-host sdk-only Codex binding proof', () => {
+ const model = 'gpt-5.6-luna'
+ const proof = {
+ connectionKey: 'team-plus',
+ catalogRevision: 4,
+ credentialRevision: 1,
+ model,
+ bindingHash: computeCodexPolicyHash({
+ model,
+ catalogRevision: 4,
+ credentialRevision: 1,
+ connectionKey: 'team-plus',
+ }),
+ }
+
+ it('rejects the unassigned connection sentinel', () => {
+ expect(
+ isPluginWorkloadSdkCodexBindingProof({
+ ...proof,
+ connectionKey: CODEX_UNASSIGNED_CONNECTION_KEY,
+ })
+ ).toBe(false)
+ expect(
+ readVerifiedSdkOnlyCodexBinding(
+ {
+ ...proof,
+ connectionKey: CODEX_UNASSIGNED_CONNECTION_KEY,
+ bindingHash: computeCodexPolicyHash({
+ model,
+ catalogRevision: 4,
+ credentialRevision: 1,
+ connectionKey: CODEX_UNASSIGNED_CONNECTION_KEY,
+ }),
+ },
+ model
+ )
+ ).toBeNull()
+ })
+
+ it('strips extra keys after the hash verifies', () => {
+ expect(readVerifiedSdkOnlyCodexBinding({ ...proof, extra: 'drop-me' }, model)).toEqual(proof)
+ })
+})
diff --git a/mcp-host/src/pluginWorkloadSdk/sdkOnlyCodexBinding.ts b/mcp-host/src/pluginWorkloadSdk/sdkOnlyCodexBinding.ts
index f849ae577..3e3d2f8f5 100644
--- a/mcp-host/src/pluginWorkloadSdk/sdkOnlyCodexBinding.ts
+++ b/mcp-host/src/pluginWorkloadSdk/sdkOnlyCodexBinding.ts
@@ -1,4 +1,7 @@
-import type { CodexPolicyBinding } from '@clerum/codex-catalog-projection'
+import {
+ CODEX_UNASSIGNED_CONNECTION_KEY,
+ type CodexPolicyBinding,
+} from '@clerum/codex-catalog-projection'
import { computeCodexPolicyHash } from '@clerum/llm-provider-attempt-contract'
const POLICY_HASH_RE = /^[a-f0-9]{64}$/
@@ -46,7 +49,7 @@ export function isPluginWorkloadSdkCodexBindingProof(
return (
typeof candidate.connectionKey === 'string' &&
candidate.connectionKey.trim().length > 0 &&
- candidate.connectionKey !== 'unassigned' &&
+ candidate.connectionKey !== CODEX_UNASSIGNED_CONNECTION_KEY &&
Number.isInteger(candidate.catalogRevision) &&
Number(candidate.catalogRevision) >= 1 &&
Number.isInteger(candidate.credentialRevision) &&
diff --git a/workflow-recipes/package-lock.json b/workflow-recipes/package-lock.json
index 19d142c68..714918528 100644
--- a/workflow-recipes/package-lock.json
+++ b/workflow-recipes/package-lock.json
@@ -1,12 +1,12 @@
{
"name": "workload-recipes-controller",
- "version": "0.1.323",
+ "version": "0.1.324",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "workload-recipes-controller",
- "version": "0.1.323",
+ "version": "0.1.324",
"license": "SEE LICENSE IN LICENSE",
"dependencies": {
"@aws-sdk/client-s3": "^3.1009.0",
diff --git a/workflow-recipes/package.json b/workflow-recipes/package.json
index 7907fb24a..06224611e 100644
--- a/workflow-recipes/package.json
+++ b/workflow-recipes/package.json
@@ -1,6 +1,6 @@
{
"name": "workload-recipes-controller",
- "version": "0.1.323",
+ "version": "0.1.324",
"description": "Workload Recipes Controller (WRC) - WorkloadRecipe lifecycle management and MCP server interface",
"main": "dist/main.js",
"engines": {
diff --git a/workflow-recipes/src/reconciler/pluginWorkloadSdkValidator.test.ts b/workflow-recipes/src/reconciler/pluginWorkloadSdkValidator.test.ts
index c1d2ed55e..29b9184d0 100644
--- a/workflow-recipes/src/reconciler/pluginWorkloadSdkValidator.test.ts
+++ b/workflow-recipes/src/reconciler/pluginWorkloadSdkValidator.test.ts
@@ -594,6 +594,43 @@ describe('buildPluginWorkloadSdkStatus', () => {
})
})
+ it('names a stale Codex bootstrap contract when the binding proof is absent', () => {
+ const projection = buildPluginWorkloadSdkStatus({
+ spec: baseSpec(
+ { promptBridge: {} },
+ { agent: { provider: 'codex-subscription', model: 'gpt-5.6-luna' } }
+ ),
+ existingConditions: undefined,
+ phase: 'active',
+ featureFlagEnabled: true,
+ bootstrapProof: {
+ ready: true,
+ contractVersion: 2,
+ podUid: 'pod-uid-1',
+ provider: 'codex-subscription',
+ model: 'gpt-5.6-luna',
+ verifiedAt: NOW,
+ },
+ now: NOW,
+ })
+ expect(projection.conditions).toEqual(
+ expect.arrayContaining([
+ expect.objectContaining({
+ type: PLUGIN_WORKLOAD_SDK_CONDITION_TYPE,
+ status: 'False',
+ reason: 'PolicyNotConfigured',
+ message:
+ 'Plugin Workload SDK promptBridge policy is not ready (codex_bootstrap_contract_stale)',
+ }),
+ expect.objectContaining({
+ type: PLUGIN_WORKLOAD_SDK_POLICY_PENDING_CONDITION_TYPE,
+ status: 'True',
+ reason: 'codex_bootstrap_contract_stale',
+ }),
+ ])
+ )
+ })
+
it('does not validate a clientNotifications-only recipe before its grant is proven', () => {
const projection = buildPluginWorkloadSdkStatus({
spec: baseSpec(
diff --git a/workflow-recipes/src/reconciler/pluginWorkloadSdkValidator.ts b/workflow-recipes/src/reconciler/pluginWorkloadSdkValidator.ts
index 4f435fdb9..e67b7ad28 100644
--- a/workflow-recipes/src/reconciler/pluginWorkloadSdkValidator.ts
+++ b/workflow-recipes/src/reconciler/pluginWorkloadSdkValidator.ts
@@ -361,7 +361,13 @@ export function buildPluginWorkloadSdkStatus(args: {
? (bootstrapProof?.clientNotificationsPolicyReason ??
bootstrapProof?.clientNotificationsPolicyState ??
'unknown')
- : (bootstrapProof?.policyReason ?? bootstrapProof?.policyState ?? 'unknown')
+ : (bootstrapProof?.policyReason ??
+ bootstrapProof?.policyState ??
+ (codexBindingPending
+ ? bootstrapProof?.contractVersion !== 3
+ ? 'codex_bootstrap_contract_stale'
+ : 'codex_execution_binding_missing'
+ : 'unknown'))
const message =
policyReason === 'grant_missing'
? `Plugin Workload SDK ${family} is awaiting an operator grant`
diff --git a/workflow-recipes/src/workflow/pluginWorkloadSdkPodDrift.test.ts b/workflow-recipes/src/workflow/pluginWorkloadSdkPodDrift.test.ts
index e2b129d45..1c957d48f 100644
--- a/workflow-recipes/src/workflow/pluginWorkloadSdkPodDrift.test.ts
+++ b/workflow-recipes/src/workflow/pluginWorkloadSdkPodDrift.test.ts
@@ -63,6 +63,27 @@ describe('readMcpHostPodDriftStateIfExists', () => {
})
})
+ it('reads the runtime token generation used after a failed remint roll', async () => {
+ const coreApi = {
+ readNamespacedPod: vi.fn().mockResolvedValue({
+ metadata: {
+ annotations: {
+ 'clerum.io/mcp-host-runtime-token-generation': '3',
+ },
+ },
+ spec: { containers: [{ name: 'mcp-host', image: 'registry.example/mcp-host:current' }] },
+ }),
+ }
+
+ await expect(
+ readMcpHostPodDriftStateIfExists(coreApi as never, POD_NAME, NAMESPACE)
+ ).resolves.toEqual({
+ image: 'registry.example/mcp-host:current',
+ tokenGeneration: '3',
+ deleting: false,
+ })
+ })
+
it('returns undefined when the pod disappeared before the drift read', async () => {
const coreApi = {
readNamespacedPod: vi.fn().mockRejectedValue({ code: 404 }),
diff --git a/workflow-recipes/src/workflow/pluginWorkloadSdkPodDrift.ts b/workflow-recipes/src/workflow/pluginWorkloadSdkPodDrift.ts
index 4a9cd7808..75fd9f93c 100644
--- a/workflow-recipes/src/workflow/pluginWorkloadSdkPodDrift.ts
+++ b/workflow-recipes/src/workflow/pluginWorkloadSdkPodDrift.ts
@@ -1,10 +1,12 @@
import * as k8s from '@kubernetes/client-node'
import { getErrorCode } from '../reconciler/k8sErrors'
import { PLUGIN_WORKLOAD_SDK_RUNTIME_CONTRACT_HASH_ANNOTATION } from './podFactory'
+import { MCP_HOST_RUNTIME_TOKEN_GENERATION_ANNOTATION } from './secretFactory'
export type McpHostPodDriftState = {
image?: string
runtimeContractHash?: string
+ tokenGeneration?: string
deleting: boolean
}
@@ -23,9 +25,12 @@ export async function readMcpHostPodDriftStateIfExists(
const mcpHostContainer = pod.spec?.containers?.find(container => container.name === 'mcp-host')
const runtimeContractHash =
pod.metadata?.annotations?.[PLUGIN_WORKLOAD_SDK_RUNTIME_CONTRACT_HASH_ANNOTATION]
+ const tokenGeneration =
+ pod.metadata?.annotations?.[MCP_HOST_RUNTIME_TOKEN_GENERATION_ANNOTATION]
return {
image: mcpHostContainer?.image,
...(runtimeContractHash ? { runtimeContractHash } : {}),
+ ...(tokenGeneration ? { tokenGeneration } : {}),
deleting: Boolean(pod.metadata?.deletionTimestamp),
}
} catch (error: unknown) {
diff --git a/workflow-recipes/src/workflow/pluginWorkloadSdkProvisioner.imageDrift.test.ts b/workflow-recipes/src/workflow/pluginWorkloadSdkProvisioner.imageDrift.test.ts
index 69b9f1e38..5dbc713e4 100644
--- a/workflow-recipes/src/workflow/pluginWorkloadSdkProvisioner.imageDrift.test.ts
+++ b/workflow-recipes/src/workflow/pluginWorkloadSdkProvisioner.imageDrift.test.ts
@@ -16,19 +16,21 @@ const RECIPE = 'demo'
function podWithPhase(
image: string,
phase: 'Running' | 'Pending',
- opts: { deleting?: boolean; runtimeContractHash?: string } = {}
+ opts: { deleting?: boolean; runtimeContractHash?: string; tokenGeneration?: string } = {}
) {
+ const annotations = {
+ ...(opts.runtimeContractHash
+ ? { 'clerum.io/plugin-workload-sdk-runtime-contract-hash': opts.runtimeContractHash }
+ : {}),
+ ...(opts.tokenGeneration
+ ? { 'clerum.io/mcp-host-runtime-token-generation': opts.tokenGeneration }
+ : {}),
+ }
return {
metadata: {
uid: 'pod-uid-1',
...(opts.deleting ? { deletionTimestamp: '2026-06-19T18:00:00Z' } : {}),
- ...(opts.runtimeContractHash
- ? {
- annotations: {
- 'clerum.io/plugin-workload-sdk-runtime-contract-hash': opts.runtimeContractHash,
- },
- }
- : {}),
+ ...(Object.keys(annotations).length > 0 ? { annotations } : {}),
},
spec: { containers: [{ name: 'mcp-host', image }] },
status: { phase, conditions: [], containerStatuses: [] },
@@ -41,10 +43,21 @@ function makeProvisioner(
opts: {
deleting?: boolean
runtimeContractHash?: string
- tokenRefresh?: { reminted: boolean; reason?: 'scope' | 'binding' | 'ttl' }
+ tokenRefresh?: {
+ reminted: boolean
+ reason?: 'scope' | 'binding' | 'ttl'
+ tokenGeneration?: string
+ }
+ tokenGeneration?: string
} = {}
) {
- const readNamespacedPod = vi.fn().mockResolvedValue(podWithPhase(podImage, phase, opts))
+ const readNamespacedPod = vi.fn().mockResolvedValue(
+ podWithPhase(podImage, phase, {
+ deleting: opts.deleting,
+ runtimeContractHash: opts.runtimeContractHash,
+ tokenGeneration: opts.tokenGeneration,
+ })
+ )
const deleteNamespacedPod = vi.fn().mockResolvedValue({})
const createNamespacedPod = vi.fn().mockResolvedValue({})
const coreApi = {
@@ -550,4 +563,53 @@ describe('ensureEagerSdkMcpHost runtime token roll', () => {
expect(deleteNamespacedPod).not.toHaveBeenCalled()
})
+
+ it('rolls a healthy eager mcp-host when Secret token generation drifted after a failed roll', async () => {
+ const { provisioner, deleteNamespacedPod, createNamespacedPod } = makeProvisioner(
+ DESIRED_IMAGE,
+ 'Running',
+ {
+ runtimeContractHash: desiredRuntimeContractHash(),
+ tokenGeneration: '1',
+ tokenRefresh: { reminted: false, tokenGeneration: '2' },
+ }
+ )
+
+ const status = await provisioner.ensureEagerSdkMcpHost(
+ RECIPE,
+ 'recipe-uid',
+ SANDBOX_NS,
+ RECIPE,
+ SPEC,
+ RUNTIME,
+ { mcpHostPhase: 'Running' }
+ )
+
+ expect(status).toBe('deploying')
+ expect(deleteNamespacedPod).toHaveBeenCalledWith({
+ name: `${RECIPE}-mcp-host`,
+ namespace: SANDBOX_NS,
+ })
+ expect(createNamespacedPod).not.toHaveBeenCalled()
+ })
+
+ it('does not roll when Secret and pod token generations already match', async () => {
+ const { provisioner, deleteNamespacedPod } = makeProvisioner(DESIRED_IMAGE, 'Running', {
+ runtimeContractHash: desiredRuntimeContractHash(),
+ tokenGeneration: '2',
+ tokenRefresh: { reminted: false, tokenGeneration: '2' },
+ })
+
+ await provisioner.ensureEagerSdkMcpHost(
+ RECIPE,
+ 'recipe-uid',
+ SANDBOX_NS,
+ RECIPE,
+ SPEC,
+ RUNTIME,
+ { mcpHostPhase: 'Running' }
+ )
+
+ expect(deleteNamespacedPod).not.toHaveBeenCalled()
+ })
})
diff --git a/workflow-recipes/src/workflow/pluginWorkloadSdkProvisioner.tokenRoll.test.ts b/workflow-recipes/src/workflow/pluginWorkloadSdkProvisioner.tokenRoll.test.ts
new file mode 100644
index 000000000..466928a97
--- /dev/null
+++ b/workflow-recipes/src/workflow/pluginWorkloadSdkProvisioner.tokenRoll.test.ts
@@ -0,0 +1,29 @@
+import { describe, expect, it } from 'vitest'
+import {
+ eagerMcpHostPodTokenGenerationDrift,
+ eagerMcpHostRequiresTokenRoll,
+} from './pluginWorkloadSdkProvisioner'
+
+describe('eager mcp-host token roll predicates', () => {
+ it('rolls only on this-pass scope or binding remint', () => {
+ expect(eagerMcpHostRequiresTokenRoll({ reminted: true, reason: 'scope' })).toBe(true)
+ expect(eagerMcpHostRequiresTokenRoll({ reminted: true, reason: 'binding' })).toBe(true)
+ expect(eagerMcpHostRequiresTokenRoll({ reminted: true, reason: 'ttl' })).toBe(false)
+ expect(eagerMcpHostRequiresTokenRoll({ reminted: false })).toBe(false)
+ expect(eagerMcpHostRequiresTokenRoll(undefined)).toBe(false)
+ })
+
+ it('rolls a leftover pod when the Secret generation residue drifted', () => {
+ expect(
+ eagerMcpHostPodTokenGenerationDrift({ reminted: false, tokenGeneration: '2' }, '1')
+ ).toBe(true)
+ expect(
+ eagerMcpHostPodTokenGenerationDrift({ reminted: false, tokenGeneration: '2' }, undefined)
+ ).toBe(true)
+ expect(
+ eagerMcpHostPodTokenGenerationDrift({ reminted: false, tokenGeneration: '2' }, '2')
+ ).toBe(false)
+ expect(eagerMcpHostPodTokenGenerationDrift({ reminted: false }, '1')).toBe(false)
+ expect(eagerMcpHostPodTokenGenerationDrift(undefined, '1')).toBe(false)
+ })
+})
diff --git a/workflow-recipes/src/workflow/pluginWorkloadSdkProvisioner.ts b/workflow-recipes/src/workflow/pluginWorkloadSdkProvisioner.ts
index 6e2b94dda..00543a548 100644
--- a/workflow-recipes/src/workflow/pluginWorkloadSdkProvisioner.ts
+++ b/workflow-recipes/src/workflow/pluginWorkloadSdkProvisioner.ts
@@ -51,6 +51,8 @@ export type McpHostRuntimeTokenRefreshReason = 'scope' | 'binding' | 'ttl'
export type McpHostRuntimeTokenRefreshResult = {
reminted: boolean
reason?: McpHostRuntimeTokenRefreshReason
+ /** Secret annotation; eager pods must match or they still carry a stale env JWT. */
+ tokenGeneration?: string
}
export const NO_MCP_HOST_RUNTIME_TOKEN_REFRESH: McpHostRuntimeTokenRefreshResult = {
@@ -64,6 +66,16 @@ export function eagerMcpHostRequiresTokenRoll(
return result?.reminted === true && (result.reason === 'scope' || result.reason === 'binding')
}
+/** Prior remint patched the Secret; the running process may still have the old JWT. */
+export function eagerMcpHostPodTokenGenerationDrift(
+ result: McpHostRuntimeTokenRefreshResult | void,
+ podGeneration?: string
+): boolean {
+ const expected = result?.tokenGeneration
+ if (!expected) return false
+ return podGeneration !== expected
+}
+
export interface EagerSdkBootstrapProof {
ready: true
contractVersion: 2 | 3
@@ -257,6 +269,9 @@ export class PluginWorkloadSdkProvisioner {
mountWorkflowOutput: false,
pluginWorkloadSdkCapabilities: capabilities,
pluginWorkloadSdkRuntimeMode: 'sdk-only',
+ ...(tokenRefresh?.tokenGeneration
+ ? { runtimeTokenGeneration: tokenRefresh.tokenGeneration }
+ : {}),
}
)
const desiredRuntimeContractHash = pluginWorkloadSdkRuntimeContractHash(desiredMcpHostPod)
@@ -278,14 +293,11 @@ export class PluginWorkloadSdkProvisioner {
}
}
- // Runtime JWT scope/binding remint. Access/refresh tokens are env-injected
- // on a bare Pod, so a Secret patch does not update process.env. HCC rolls
- // on contract_changed; WRC must do the same or authorize keeps the first-boot
- // JWT (insufficient_scope after reservation-only Codex).
- if (
- eagerMcpHostRequiresTokenRoll(tokenRefresh) &&
- (mcpHostPhase === 'Running' || mcpHostPhase === 'Pending')
- ) {
+ // Runtime JWT scope/binding remint, plus a Secret/Pod generation residue
+ // when a prior remint patched the Secret but the roll failed. Access/refresh
+ // tokens are env-injected on a bare Pod; Secret scopes matching is not proof
+ // that process.env was refreshed.
+ if (mcpHostPhase === 'Running' || mcpHostPhase === 'Pending') {
const runningPod = await readMcpHostPodDriftStateIfExists(
this.deps.coreApi,
`${recipeName}-mcp-host`,
@@ -294,32 +306,22 @@ export class PluginWorkloadSdkProvisioner {
if (runningPod?.deleting) {
return 'deploying'
}
- log.info('Plugin Workload SDK eager mcp-host runtime token drift; rolling Pod', {
- reason: tokenRefresh?.reason,
- })
- await this.rollEagerSdkMcpHostPod(recipeName)
- return 'deploying'
- }
-
- // Image-drift roll. The eager SDK mcp-host is a bare Pod (no Deployment),
- // so a platform mcp-host image bump never reaches a long-lived recipe on
- // its own: createIfNotExists is a no-op while the stale pod stays healthy,
- // and there is no owning controller to perform a rolling update. A recipe
- // deployed before an mcp-host release therefore keeps serving SDK routes
- // from the old image (e.g. a pre-recipients-endpoint pod returns 404 for
- // GET /sdk/v1/client-notifications/recipients). Roll the pod so the next
- // reconcile recreates it from config.mcpHostImage. Safe on this path: the
- // eager pod hosts only the always-on SDK server and produces no run
- // artifacts, and no triggered run is in flight (see ensureEagerSdkMcpHost).
- if (mcpHostPhase === 'Running' || mcpHostPhase === 'Pending') {
- const runningPod = await readMcpHostPodDriftStateIfExists(
- this.deps.coreApi,
- `${recipeName}-mcp-host`,
- this.deps.config.sandboxNamespace
- )
- if (runningPod?.deleting) {
+ const tokenRoll =
+ eagerMcpHostRequiresTokenRoll(tokenRefresh) ||
+ eagerMcpHostPodTokenGenerationDrift(tokenRefresh, runningPod?.tokenGeneration)
+ if (tokenRoll) {
+ log.info('Plugin Workload SDK eager mcp-host runtime token drift; rolling Pod', {
+ reason: tokenRefresh?.reason,
+ secretTokenGeneration: tokenRefresh?.tokenGeneration,
+ podTokenGeneration: runningPod?.tokenGeneration,
+ })
+ await this.rollEagerSdkMcpHostPod(recipeName)
return 'deploying'
}
+ // Image-drift roll. The eager SDK mcp-host is a bare Pod (no Deployment),
+ // so a platform mcp-host image bump never reaches a long-lived recipe on
+ // its own: createIfNotExists is a no-op while the stale pod stays healthy,
+ // and there is no owning controller to perform a rolling update.
const imageDrift = runningPod?.image !== this.deps.config.mcpHostImage
const runtimeContractDrift = runningPod?.runtimeContractHash !== desiredRuntimeContractHash
if (runningPod && (imageDrift || runtimeContractDrift)) {
diff --git a/workflow-recipes/src/workflow/podFactory.ts b/workflow-recipes/src/workflow/podFactory.ts
index 07d6401ca..06afbdf40 100644
--- a/workflow-recipes/src/workflow/podFactory.ts
+++ b/workflow-recipes/src/workflow/podFactory.ts
@@ -22,6 +22,7 @@ import {
DEFAULT_CUSTOM_COORDINATOR_ACTIVE_DEADLINE_SECONDS,
WORKFLOW_OUTPUT_PVC_NAME,
} from './runtimeConstants'
+import { MCP_HOST_RUNTIME_TOKEN_GENERATION_ANNOTATION } from './secretFactory'
import { AgentSpec, WorkflowConfig } from './types'
/** Directory where the mcp-host mounts per-caller SDK bearer tokens (Secret volume). */
@@ -600,6 +601,8 @@ export interface McpHostPodOptions {
pluginWorkloadSdkEnabled?: boolean
/** Runtime contract used by mcp-host usage attribution. */
pluginWorkloadSdkRuntimeMode?: 'workflow' | 'sdk-only'
+ /** Secret residue copied onto the eager pod after a scope/binding remint. */
+ runtimeTokenGeneration?: string
}
export function buildMcpHostPod(
@@ -886,6 +889,9 @@ export function buildMcpHostPod(
...(pod.metadata?.annotations ?? {}),
[PLUGIN_WORKLOAD_SDK_RUNTIME_CONTRACT_HASH_ANNOTATION]:
pluginWorkloadSdkRuntimeContractHash(pod),
+ ...(options.runtimeTokenGeneration
+ ? { [MCP_HOST_RUNTIME_TOKEN_GENERATION_ANNOTATION]: options.runtimeTokenGeneration }
+ : {}),
},
}
}
diff --git a/workflow-recipes/src/workflow/sdkOnlyCodexBinding.test.ts b/workflow-recipes/src/workflow/sdkOnlyCodexBinding.test.ts
index d0c182229..2c834ba76 100644
--- a/workflow-recipes/src/workflow/sdkOnlyCodexBinding.test.ts
+++ b/workflow-recipes/src/workflow/sdkOnlyCodexBinding.test.ts
@@ -40,6 +40,29 @@ describe('deriveSdkOnlyCodexBinding', () => {
expect(isPluginWorkloadSdkCodexBindingProof(binding)).toBe(true)
})
+ it('returns null when the assigned catalog excludes the requested model', () => {
+ expect(
+ deriveSdkOnlyCodexBinding({
+ provider: 'codex-subscription',
+ model: 'gpt-5.6-luna',
+ connectionKey: 'team-plus',
+ configMap: {
+ metadata: {
+ annotations: {
+ 'clerum.io/codex-connections': JSON.stringify({
+ 'team-plus': {
+ models: ['gpt-5.1'],
+ catalogRevision: 3,
+ connectionRevision: 1,
+ },
+ }),
+ },
+ },
+ },
+ })
+ ).toBeNull()
+ })
+
it('returns null for non-Codex providers and unassigned grants', () => {
expect(
deriveSdkOnlyCodexBinding({
diff --git a/workflow-recipes/src/workflow/secretFactory.ts b/workflow-recipes/src/workflow/secretFactory.ts
index c8a99172d..10443a829 100644
--- a/workflow-recipes/src/workflow/secretFactory.ts
+++ b/workflow-recipes/src/workflow/secretFactory.ts
@@ -13,6 +13,24 @@ import { JwtTokenFactory } from './jwtTokenFactory'
import { pluginWorkloadSdkTokenSecretKey } from './pluginWorkloadSdkTokens'
import { buildPluginWorkloadSdkTokenSecretName } from './resourceNames'
+/** Secret/Pod residue so a failed eager roll cannot leave a stale env JWT. */
+export const MCP_HOST_RUNTIME_TOKEN_GENERATION_ANNOTATION =
+ 'clerum.io/mcp-host-runtime-token-generation'
+
+export function readMcpHostRuntimeTokenGeneration(
+ secret: { metadata?: { annotations?: Record<string, string> | null } | null } | undefined
+): string | undefined {
+ const raw = secret?.metadata?.annotations?.[MCP_HOST_RUNTIME_TOKEN_GENERATION_ANNOTATION]
+ const trimmed = typeof raw === 'string' ? raw.trim() : ''
+ return trimmed.length > 0 ? trimmed : undefined
+}
+
+export function nextMcpHostRuntimeTokenGeneration(current?: string): string {
+ const parsed = Number(current)
+ if (Number.isInteger(parsed) && parsed >= 1) return String(parsed + 1)
+ return '1'
+}
+
export function buildCoordinatorTokenSecret(
recipeName: string,
mcpHostToken: string | undefined,
diff --git a/workflow-recipes/src/workflow/workflowReconciler.codexScopeProvenance.test.ts b/workflow-recipes/src/workflow/workflowReconciler.codexScopeProvenance.test.ts
index f725ce689..04bad1016 100644
--- a/workflow-recipes/src/workflow/workflowReconciler.codexScopeProvenance.test.ts
+++ b/workflow-recipes/src/workflow/workflowReconciler.codexScopeProvenance.test.ts
@@ -387,11 +387,17 @@ describe('WorkflowReconciler Codex scope provenance', () => {
expect(issueMcpHostRuntimeTokens).toHaveBeenCalled()
expect(issuedScopes()).not.toContain('llm:codex:execute')
- expect(crashRecoveryMocks.deletePodIfExists).toHaveBeenCalledWith(
+ expect(crashRecoveryMocks.deletePodIfExists).not.toHaveBeenCalledWith(
expect.anything(),
'codex-recipe-mcp-host',
sandboxNamespace
)
+ const remintPatch = coreApi.patchNamespacedSecret.mock.calls.at(-1)?.[0] as {
+ body?: { metadata?: { annotations?: Record<string, string> } }
+ }
+ expect(
+ remintPatch.body?.metadata?.annotations?.['clerum.io/mcp-host-runtime-token-generation']
+ ).toBe('1')
expect(networkingApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledWith({
name: CODEX_PROXY_POLICY,
namespace: sandboxNamespace,
@@ -440,7 +446,7 @@ describe('WorkflowReconciler Codex scope provenance', () => {
expect(issueMcpHostRuntimeTokens).toHaveBeenCalled()
expect(issuedScopes()).not.toContain('llm:codex:execute')
- expect(crashRecoveryMocks.deletePodIfExists).toHaveBeenCalledWith(
+ expect(crashRecoveryMocks.deletePodIfExists).not.toHaveBeenCalledWith(
expect.anything(),
'codex-recipe-mcp-host',
sandboxNamespace
diff --git a/workflow-recipes/src/workflow/workflowReconciler.ts b/workflow-recipes/src/workflow/workflowReconciler.ts
index 201d65a31..1dd970072 100644
--- a/workflow-recipes/src/workflow/workflowReconciler.ts
+++ b/workflow-recipes/src/workflow/workflowReconciler.ts
@@ -141,7 +141,13 @@ import {
reconcileScheduling,
} from './schedulingHandler'
import { deriveSdkOnlyCodexBinding } from './sdkOnlyCodexBinding'
-import { buildMcpHostRuntimeTokenSecret, createCoordinatorTokens } from './secretFactory'
+import {
+ MCP_HOST_RUNTIME_TOKEN_GENERATION_ANNOTATION,
+ buildMcpHostRuntimeTokenSecret,
+ createCoordinatorTokens,
+ nextMcpHostRuntimeTokenGeneration,
+ readMcpHostRuntimeTokenGeneration,
+} from './secretFactory'
import { SNIPPET_RUN_KEYS } from './snippetRunSchema'
import {
CyclicDependencyError,
@@ -4812,7 +4818,10 @@ export class WorkflowReconciler {
gfsAccessExp - nowSecs < refreshBeforeSeconds
if (!needsMcpHostRuntimeRefresh && !needsWorkflowControlRefresh && !needsGfsAccessRefresh) {
- return NO_MCP_HOST_RUNTIME_TOKEN_REFRESH
+ const tokenGeneration = readMcpHostRuntimeTokenGeneration(existing)
+ return tokenGeneration
+ ? { reminted: false, tokenGeneration }
+ : NO_MCP_HOST_RUNTIME_TOKEN_REFRESH
}
const refreshReason: McpHostRuntimeTokenRefreshReason =
@@ -4885,12 +4894,26 @@ export class WorkflowReconciler {
data['mcp-host-gfs-token'] = Buffer.from(gfs.token).toString('base64')
}
+ const currentGeneration = readMcpHostRuntimeTokenGeneration(existing)
+ const tokenGeneration =
+ refreshReason === 'scope' || refreshReason === 'binding'
+ ? nextMcpHostRuntimeTokenGeneration(currentGeneration)
+ : currentGeneration
await this.deps.coreApi.patchNamespacedSecret(
{
name: `wf-${recipeName}-mcp-host-runtime-tokens`,
namespace: this.deps.config.sandboxNamespace,
body: {
data,
+ ...(tokenGeneration && (refreshReason === 'scope' || refreshReason === 'binding')
+ ? {
+ metadata: {
+ annotations: {
+ [MCP_HOST_RUNTIME_TOKEN_GENERATION_ANNOTATION]: tokenGeneration,
+ },
+ },
+ }
+ : {}),
},
},
{ middleware: [k8s.setHeaderMiddleware('Content-Type', 'application/merge-patch+json')] }
@@ -4898,16 +4921,16 @@ export class WorkflowReconciler {
this.log.info(
`Refreshed workflow auth Secret for ${recipeName} (mcpHostRuntime=${needsMcpHostRuntimeRefresh}, mcpHostControl=${needsWorkflowControlRefresh}, reason=${refreshReason})`
)
- if (refreshReason === 'scope' || refreshReason === 'binding') {
- // Access/refresh JWTs are env-injected. A Secret patch does not update
- // the running process; refresh/reissue copy the stale scopes.
- await deletePodIfExists(
- this.deps.coreApi,
- `${recipeName}-mcp-host`,
- this.deps.config.sandboxNamespace
- )
+ // Do not delete ${recipeName}-mcp-host here. This helper also runs on the
+ // in-progress step-based credential refresh path, where a mid-run pod
+ // delete is not recreated. Eager sdk-only hosts roll via
+ // eagerMcpHostRequiresTokenRoll → rollEagerSdkMcpHostPod, including when
+ // this pass only reports the Secret generation residue.
+ return {
+ reminted: true,
+ reason: refreshReason,
+ ...(tokenGeneration ? { tokenGeneration } : {}),
}
- return { reminted: true, reason: refreshReason }
} catch (err) {
this.log.error(`Failed to refresh mcpHost runtime tokens for ${recipeName}`, {
error: err instanceof Error ? err.message : String(err),
diff --git a/workflow-recipes/tests/unit/workflow/secretFactory.mcpHostRuntimeTokens.test.ts b/workflow-recipes/tests/unit/workflow/secretFactory.mcpHostRuntimeTokens.test.ts
index 6bdae10f1..34c71bbe9 100644
--- a/workflow-recipes/tests/unit/workflow/secretFactory.mcpHostRuntimeTokens.test.ts
+++ b/workflow-recipes/tests/unit/workflow/secretFactory.mcpHostRuntimeTokens.test.ts
@@ -2,6 +2,8 @@ import { describe, expect, it } from 'vitest'
import {
buildCoordinatorTokenSecret,
buildMcpHostRuntimeTokenSecret,
+ nextMcpHostRuntimeTokenGeneration,
+ readMcpHostRuntimeTokenGeneration,
} from '../../../src/workflow/secretFactory'
describe('Secret Factory — mcpHost Runtime Tokens', () => {
@@ -74,6 +76,17 @@ describe('Secret Factory — mcpHost Runtime Tokens', () => {
expect(secret.metadata!.ownerReferences).toBeUndefined()
})
+ it('bumps the runtime token generation from a missing or invalid residue', () => {
+ expect(nextMcpHostRuntimeTokenGeneration(undefined)).toBe('1')
+ expect(nextMcpHostRuntimeTokenGeneration('1')).toBe('2')
+ expect(readMcpHostRuntimeTokenGeneration(undefined)).toBeUndefined()
+ expect(
+ readMcpHostRuntimeTokenGeneration({
+ metadata: { annotations: { 'clerum.io/mcp-host-runtime-token-generation': '4' } },
+ })
+ ).toBe('4')
+ })
+
// ── Existing Secret unaffected ───────────────────────────────────────
it('does NOT modify the existing coordinator-token Secret', () => {