Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
import * as k8s from '@kubernetes/client-node'
import { randomBytes } from 'node:crypto'
import { type Logger, createLogger } from '../observability/logger'
import { getErrorCode } from '../reconciler/k8sErrors'
import type { NetworkPolicyConflictReason } from '../reconciler/networkPolicyConvergence'
import { effectiveWorkflowContextRefForSpec } from '../reconciler/workflowContext'
import type { WorkflowRecipeSpec } from '../types'
import { resolveEagerSdkMcpHostAgent } from './agentResolution'
import type { CodexRecipeVerdict } from './codexRecipeVerdict'
import {
deletePodIfExists,
getContainerWaitingReason,
getPodPhase,
getPodReadiness,
isRecoverableContainerWaitingReason,
} from './crashRecovery'
import type { JwtTokenFactory } from './jwtTokenFactory'
import type { ModelConfigHandler } from './modelConfigHandler'
import { readMcpHostPodDriftStateIfExists } from './pluginWorkloadSdkPodDrift'
import {
pluginWorkloadSdkSecretTokensMatch,
pluginWorkloadSdkTokenSecretKey,
resolvePluginWorkloadSdkAllowedCallers,
} from './pluginWorkloadSdkTokens'
import {
buildMcpHostPod,
declaredPluginWorkloadSdkCapabilities,
pluginWorkloadSdkRuntimeContractHash,
recipeDeclaresGrokSubscription,
} from './podFactory'
import {
PLUGIN_WORKLOAD_SDK_LEGACY_TOKEN_DATA_KEY,
buildMcpHostUrl,
buildPluginWorkloadSdkTokenSecretName,
} from './resourceNames'
import type { WorkflowRuntimePlan } from './runtimePlan'
import {
type PluginWorkloadSdkCodexBindingProof,
readVerifiedSdkOnlyCodexBinding,
} from './sdkOnlyCodexBinding'
import { readVerifiedSdkOnlyGrokBinding } from './sdkOnlyGrokBinding'
import { buildPluginWorkloadSdkTokenSecret } from './secretFactory'
import type { WorkflowConfig } from './types'
export type EagerSdkMcpHostStatus =
| 'ready'
| 'awaiting_policy'
| 'deploying'
| 'failed'
| 'provider_unavailable'
/**
* What one policy apply could not converge. A conflict names a live policy
* another controller owns; WRC leaves it untouched. A pending retry means a
* policy was left unwritten for a later pass: it was terminating, or still
* contended after the bounded apply rounds. An apply prunes nothing, so it
* says nothing about the prune or the legacy policy.
*/
export type WorkflowNetworkPolicyApplySummary = {
conflicts: { policy: string; reason: NetworkPolicyConflictReason }[]
retryPending: boolean
}
/**
* What a prune pass established about the run-lane policies the spec no
* longer wants: `converged` when every DELETE it owed landed (2xx or 404),
* `pending` when one did not (a non-404 DELETE), and `unevaluated` when its
* LIST failed, so it could not tell which policies were owed a DELETE at all.
*/
export type NetworkPolicyPruneFact = 'converged' | 'pending' | 'unevaluated'
/**
* What a pass established about the legacy
* `<recipe>-mcp-servers-egress-internet` policy, deleted by name because it
* may carry no labels: `removed` when its DELETE landed (2xx or 404) or an
* earlier DELETE already did, `pending` when it did not.
*/
export type NetworkPolicyLegacyFact = 'removed' | 'pending'
/**
* What one full `applyWorkflowNetworkPolicies` pass (apply, prune and legacy
* delete) established. Every field is required: an absent fact is never
* read as "carry the published one".
*/
export type NetworkPolicyPassSummary = WorkflowNetworkPolicyApplySummary & {
prune: NetworkPolicyPruneFact
legacy: NetworkPolicyLegacyFact
}
export type EagerSdkMcpHostResult = {
status: EagerSdkMcpHostStatus
/**
* Undefined when the pass returned before applying the policies. An empty
* summary would claim every policy converged and clear a published conflict.
*/
networkPolicies?: NetworkPolicyPassSummary
}
/** Why the mcp-host runtime JWT Secret was reminted. */
export type McpHostRuntimeTokenRefreshReason = 'scope' | 'binding' | 'ttl'
export type McpHostRuntimeTokenRefreshResult = {
reminted: boolean
reason?: McpHostRuntimeTokenRefreshReason
/** Secret annotation; eager pods must match or they still carry a stale env JWT. */
tokenGeneration?: string
}
export const NO_MCP_HOST_RUNTIME_TOKEN_REFRESH: McpHostRuntimeTokenRefreshResult = {
reminted: false,
}
/** Env-injected access/refresh JWTs only update on a new process. */
export function eagerMcpHostRequiresTokenRoll(
result: McpHostRuntimeTokenRefreshResult | void
): boolean {
return result?.reminted === true && (result.reason === 'scope' || result.reason === 'binding')
}
/** Prior remint patched the Secret; the running process may still have the old JWT. */
export function eagerMcpHostPodTokenGenerationDrift(
result: McpHostRuntimeTokenRefreshResult | void,
podGeneration?: string
): boolean {
const expected = result?.tokenGeneration
if (!expected) return false
return podGeneration !== expected
}
export interface EagerSdkBootstrapProof {
ready: true
contractVersion: 2 | 3
podUid: string
codexBinding?: PluginWorkloadSdkCodexBindingProof
subscriptionBinding?: PluginWorkloadSdkCodexBindingProof
provider?: string
model?: string
policyReady?: boolean
policyState?: string
policyReason?: string
/** Optional snapshot when a grant already exists; request authorization
* always re-reads and validates the current policy. */
policyRevision?: number
policyHash?: string
defaultTargetRef?: string
defaultProvider?: string
defaultModel?: string
clientNotificationsPolicyReady?: boolean
clientNotificationsPolicyState?: string
clientNotificationsPolicyReason?: string
verifiedAt: string
}
/**
* Consecutive eager SDK bootstrap failures tolerated before the provider is
* surfaced as unavailable instead of an indefinite `deploying`. Mirrors the
* crash-recovery MAX_ATTEMPTS bound so a permanently-failing provider broker
* (e.g. a bad secretRef) is not indistinguishable from "still booting".
*/
const MAX_EAGER_CONFIGURE_ATTEMPTS = 3
type EagerConfigureFailureBudget = {
key: string
count: number
}
export type PluginWorkloadSdkProvisionerDeps = {
coreApi: k8s.CoreV1Api
config: WorkflowConfig
tokenFactory: JwtTokenFactory
modelConfigHandler?: ModelConfigHandler
log: Logger
ensureMcpHostSecrets: (
namespace: string,
recipeName: string,
runtimeScopeRecipeName: string,
spec: WorkflowRecipeSpec,
recipeUid: string | undefined,
codexVerdict: CodexRecipeVerdict
) => Promise<McpHostRuntimeTokenRefreshResult | void>
applyWorkflowNetworkPolicies: (
recipeName: string,
recipeUid: string,
spec: WorkflowRecipeSpec,
runtime: WorkflowRuntimePlan,
awaitsTriggeredRun: boolean,
codexProjection: CodexRecipeVerdict['projection'],
eagerSdkMcpHost: boolean,
grokProjection?: CodexRecipeVerdict['grokProjection']
) => Promise<NetworkPolicyPassSummary>
ensureMcpHostHeadlessService: (recipeName: string) => Promise<void>
createIfNotExists: (createFn: () => Promise<unknown>, label: string) => Promise<boolean>
safeDelete: (deleteFn: () => Promise<unknown>) => Promise<void>
}
/**
* Plugin Workload SDK provisioning: eager mcp-host lifecycle, readiness gates,
* and per-caller workload token Secrets. Extracted from WorkflowReconciler to
* keep the reconciler file bounded while preserving the existing behavior.
*/
export class PluginWorkloadSdkProvisioner {
/** Last fresh Control API proof, tied to the current pod UID. */
private readonly eagerSdkBootstrapProofByRecipe = new Map<string, EagerSdkBootstrapProof>()
/**
* Consecutive eager SDK bootstrap failures per recipe and stable bootstrap
* identity. The budget counts only repeated failures for the same
* `<podUid>:<provider>:<model>`; a pod replacement or agent model/provider
* change starts from one without needing WRC to observe the roll directly.
* Reset on a successful bootstrap (and on recipe cleanup). Once the matching
* key reaches MAX_EAGER_CONFIGURE_ATTEMPTS the provisioner projects
* `provider_unavailable` instead of an indefinite `deploying`. Lost on WRC
* restart, which just re-attempts from zero (bootstrap is idempotent).
*/
private readonly eagerSdkConfigureFailuresByRecipe = new Map<
string,
EagerConfigureFailureBudget
>()
constructor(private readonly deps: PluginWorkloadSdkProvisionerDeps) {}
clearRecipeState(recipeName: string): void {
this.eagerSdkBootstrapProofByRecipe.delete(recipeName)
this.eagerSdkConfigureFailuresByRecipe.delete(recipeName)
}
getBootstrapProof(recipeName: string): EagerSdkBootstrapProof | undefined {
return this.eagerSdkBootstrapProofByRecipe.get(recipeName)
}
/**
* Deletes the eager mcp-host pod (wedge recovery or image-drift roll) and
* clears the per-recipe bootstrap failure budget. The failure map is also
* identity-keyed, but explicit WRC rolls are a clear boundary where the prior
* pod's broker state is no longer relevant.
*/
private async rollEagerSdkMcpHostPod(recipeName: string): Promise<void> {
await deletePodIfExists(
this.deps.coreApi,
`${recipeName}-mcp-host`,
this.deps.config.sandboxNamespace
)
this.eagerSdkBootstrapProofByRecipe.delete(recipeName)
this.eagerSdkConfigureFailuresByRecipe.delete(recipeName)
}
/**
* Plugin Workload SDK eager mcp-host lifecycle (Option A).
*
* Creates the recipe mcp-host before any run is triggered so the always-on
* SDK server (:8099) is reachable, then — only when the recipe declares the
* promptBridge family — publishes the public provider/model bootstrap binding
* via a credential-free endpoint. Credentials stay behind the per-attempt
* WRC broker; clientNotifications-only recipes need no provider bootstrap.
*
* Idempotent: safe to call on every reconcile. The mcp-host bootstrap is an
* identity-only handshake, so re-delivering the same provider/model after a
* pod restart simply re-arms the SDK LLM binding.
*
* Known limit: the NetworkPolicy apply summary is returned only when the
* function returns. If the pod half throws after the apply, the summary is
* discarded: ownership conflicts the apply already evaluated are not
* published, and its `retryPending` is lost for that pass.
*/
async ensureEagerSdkMcpHost(
recipeName: string,
recipeUid: string,
namespace: string,
runtimeScopeRecipeName: string,
spec: WorkflowRecipeSpec,
runtime: WorkflowRuntimePlan,
opts: {
mcpHostPhase: string | undefined
/**
* The caller's ONE Codex verdict for this pass. Required, and
* deliberately not a pair of optional derived fields: an optional
* `codexBindingUndecidable` defaulting to false is a flag a future
* caller can silently omit, which is precisely how a binding-less v3
* configure reached a live host.
*/
codexVerdict: CodexRecipeVerdict
}
): Promise<EagerSdkMcpHostResult> {
const log = createLogger('wrc', recipeName)
const capabilities = declaredPluginWorkloadSdkCapabilities(spec.pluginWorkloadSdk)
const requiresPromptBridge = capabilities.includes('promptBridge')
const mcpHostAgent = resolveEagerSdkMcpHostAgent(spec)
if (!mcpHostAgent && requiresPromptBridge) {
log.warn(
'Plugin Workload SDK promptBridge declared but no agent is resolvable; ' +
'declare spec.agent with provider+model to bind the eager mcp-host'
)
return { status: 'failed' }
}
const tokenRefresh = await this.deps.ensureMcpHostSecrets(
namespace,
recipeName,
runtimeScopeRecipeName,
spec,
recipeUid,
opts.codexVerdict
)
const networkPolicies = await this.deps.applyWorkflowNetworkPolicies(
recipeName,
recipeUid,
spec,
runtime,
/* awaitsTriggeredRun */ true,
opts.codexVerdict.projection,
/* eagerSdkMcpHost */ true,
opts.codexVerdict.grokProjection
)
const status = await this.provisionEagerSdkMcpHostPod(
recipeName,
namespace,
runtimeScopeRecipeName,
spec,
opts,
{ log, capabilities, requiresPromptBridge, mcpHostAgent, tokenRefresh }
)
return { status, networkPolicies }
}
/** Pod lifecycle and bootstrap half of `ensureEagerSdkMcpHost`, after the policy apply. */
private async provisionEagerSdkMcpHostPod(
recipeName: string,
namespace: string,
runtimeScopeRecipeName: string,
spec: WorkflowRecipeSpec,
opts: { mcpHostPhase: string | undefined; codexVerdict: CodexRecipeVerdict },
pass: {
log: Logger
capabilities: ReturnType<typeof declaredPluginWorkloadSdkCapabilities>
requiresPromptBridge: boolean
mcpHostAgent: ReturnType<typeof resolveEagerSdkMcpHostAgent>
tokenRefresh: McpHostRuntimeTokenRefreshResult | void
}
): Promise<EagerSdkMcpHostStatus> {
const { log, capabilities, requiresPromptBridge, mcpHostAgent, tokenRefresh } = pass
await this.deps.ensureMcpHostHeadlessService(recipeName)
// Build the desired Pod once per reconcile so drift checks compare the
// complete immutable runtime contract, not only the image string. This
// catches same-image upgrades of the sdk-only env/dispatch contract.
const desiredMcpHostPod = buildMcpHostPod(
recipeName,
mcpHostAgent,
this.deps.config,
runtimeScopeRecipeName,
namespace,
undefined,
undefined,
effectiveWorkflowContextRefForSpec(recipeName, spec),
{
mountWorkflowOutput: false,
pluginWorkloadSdkCapabilities: capabilities,
pluginWorkloadSdkRuntimeMode: 'sdk-only',
grokSubscriptionEnabled: this.deps.config.grokSubscriptionEnabled === true,
recipeAgentProvider: mcpHostAgent?.provider,
recipeDeclaresGrok: recipeDeclaresGrokSubscription(spec),
...(tokenRefresh?.tokenGeneration
? { runtimeTokenGeneration: tokenRefresh.tokenGeneration }
: {}),
}
)
const desiredRuntimeContractHash = pluginWorkloadSdkRuntimeContractHash(desiredMcpHostPod)
let mcpHostPhase = opts.mcpHostPhase
if (mcpHostPhase === 'Running' || mcpHostPhase === 'Pending') {
const waitingReason = await getContainerWaitingReason(
this.deps.coreApi,
`${recipeName}-mcp-host`,
this.deps.config.sandboxNamespace
)
if (isRecoverableContainerWaitingReason(mcpHostPhase, waitingReason)) {
log.warn('Plugin Workload SDK eager mcp-host is wedged; recreating', {
phase: mcpHostPhase,
reason: waitingReason,
})
await this.rollEagerSdkMcpHostPod(recipeName)
mcpHostPhase = undefined
}
}
// Runtime JWT scope/binding remint, plus a Secret/Pod generation residue
// when a prior remint patched the Secret but the roll failed. Access/refresh
// tokens are env-injected on a bare Pod; Secret scopes matching is not proof
// that process.env was refreshed.
if (mcpHostPhase === 'Running' || mcpHostPhase === 'Pending') {
const runningPod = await readMcpHostPodDriftStateIfExists(
this.deps.coreApi,
`${recipeName}-mcp-host`,
this.deps.config.sandboxNamespace
)
if (runningPod?.deleting) {
return 'deploying'
}
const tokenRoll =
eagerMcpHostRequiresTokenRoll(tokenRefresh) ||
eagerMcpHostPodTokenGenerationDrift(tokenRefresh, runningPod?.tokenGeneration)
if (tokenRoll) {
log.info('Plugin Workload SDK eager mcp-host runtime token drift; rolling Pod', {
reason: tokenRefresh?.reason,
secretTokenGeneration: tokenRefresh?.tokenGeneration,
podTokenGeneration: runningPod?.tokenGeneration,
})
await this.rollEagerSdkMcpHostPod(recipeName)
return 'deploying'
}
// Image-drift roll. The eager SDK mcp-host is a bare Pod (no Deployment),
// so a platform mcp-host image bump never reaches a long-lived recipe on
// its own: createIfNotExists is a no-op while the stale pod stays healthy,
// and there is no owning controller to perform a rolling update.
const imageDrift = runningPod?.image !== this.deps.config.mcpHostImage
const runtimeContractDrift = runningPod?.runtimeContractHash !== desiredRuntimeContractHash
if (runningPod && (imageDrift || runtimeContractDrift)) {
log.info('Plugin Workload SDK eager mcp-host runtime contract drift; rolling Pod', {
runningImage: runningPod.image,
desiredImage: this.deps.config.mcpHostImage,
observedRuntimeContractHash: runningPod.runtimeContractHash,
desiredRuntimeContractHash,
})
await this.rollEagerSdkMcpHostPod(recipeName)
// Requeue: the pod is recreated from the platform image on a later
// reconcile once the prior pod has terminated. Returning here avoids
// racing createIfNotExists against the still-terminating pod and
// avoids bootstrapping a pod that is about to be replaced.
return 'deploying'
}
}
if (!mcpHostPhase || mcpHostPhase === 'Failed' || mcpHostPhase === 'Unknown') {
await this.deps.createIfNotExists(
() =>
this.deps.coreApi.createNamespacedPod({
namespace: this.deps.config.sandboxNamespace,
body: desiredMcpHostPod,
}),
`Pod "${recipeName}-mcp-host"`
)
}
const readiness = await this.readinessForEagerSdkMcpHost(recipeName)
const promptBridge = capabilities.includes('promptBridge')
const clientNotifications = capabilities.includes('clientNotifications')
if (!promptBridge && !clientNotifications) {
this.eagerSdkBootstrapProofByRecipe.delete(recipeName)
return readiness.status
}
const modelConfigHandler = this.deps.modelConfigHandler
if (requiresPromptBridge && !modelConfigHandler) {
this.eagerSdkBootstrapProofByRecipe.delete(recipeName)
return 'provider_unavailable'
}
if (readiness.status !== 'ready') {
this.eagerSdkBootstrapProofByRecipe.delete(recipeName)
return readiness.status
}
// A provider-free notification host can expose its route once the
// namespace-bound runtime is Ready; there is no LLM identity to broker.
// Production normally supplies the modelConfigHandler for a policy proof,
// but absence must not turn a valid notification-only recipe into a fake
// provider failure.
if (!requiresPromptBridge && !modelConfigHandler) {
this.eagerSdkBootstrapProofByRecipe.delete(recipeName)
return readiness.status
}
if (!modelConfigHandler) return 'provider_unavailable'
if (!readiness.uid) {
log.warn(
'Plugin Workload SDK eager mcp-host reported Ready without a pod UID; waiting for stable bootstrap identity'
)
return 'deploying'
}
if (promptBridge && !mcpHostAgent) return 'failed'
const brokerProjectionUncertain =
mcpHostAgent?.provider === 'codex-subscription'
? opts.codexVerdict.projection.eligibility === 'uncertain'
: mcpHostAgent?.provider === 'grok-subscription'
? opts.codexVerdict.grokProjection.eligibility === 'uncertain'
: false
if (brokerProjectionUncertain && promptBridge) {
const existing = this.eagerSdkBootstrapProofByRecipe.get(recipeName)
if (existing && existing.podUid !== readiness.uid) {
// Proof of a pod that has since been replaced. The new pod has never
// been configured, so its cached predecessor's proof must not be read
// as "this host already holds a Codex binding".
log.info('Plugin Workload SDK bootstrap proof belongs to a replaced pod; discarding', {
proofPodUid: existing.podUid,
podUid: readiness.uid,
})
this.eagerSdkBootstrapProofByRecipe.delete(recipeName)
return 'awaiting_policy'
}
if (
!existing ||
existing.policyReady === false ||
existing.contractVersion !== 3 ||
// The agent's own slot only: a proof holding the other broker's
// binding says nothing about this host's grant.
!(mcpHostAgent?.provider === 'grok-subscription'
? existing.subscriptionBinding
: existing.codexBinding) ||
existing.policyReason === 'codex_execution_binding_missing' ||
existing.policyReason === 'execution_binding_missing'
) {
return 'awaiting_policy'
}
return 'ready'
}
// Both the skip decision above and this binding are fields of the SAME
// verdict the caller computed once for this pass. The provisioner never
// re-derives either after its own awaits.
const resolvedCodexBinding = opts.codexVerdict.hostBinding
const resolvedGrokBinding = opts.codexVerdict.grokBinding
const resolvedBrokerBinding =
mcpHostAgent?.provider === 'grok-subscription' ? resolvedGrokBinding : resolvedCodexBinding
const capabilityFamily = promptBridge ? 'promptBridge' : 'clientNotifications'
const configureKey = `${readiness.uid}:${capabilityFamily}:${mcpHostAgent?.provider ?? 'none'}:${mcpHostAgent?.model ?? 'none'}:${resolvedBrokerBinding?.bindingHash ?? 'none'}`
try {
const wrcConfigureToken = await this.deps.tokenFactory.signWrcConfigureToken(
recipeName,
namespace
)
const mcpHostEndpoint = buildMcpHostUrl(recipeName, this.deps.config.sandboxNamespace)
const result =
capabilityFamily === 'promptBridge'
? await modelConfigHandler.configurePluginWorkloadSdkBootstrap(
mcpHostAgent!.provider,
mcpHostAgent!.model,
mcpHostEndpoint,
wrcConfigureToken,
'promptBridge',
resolvedBrokerBinding
)
: await modelConfigHandler.configurePluginWorkloadSdkBootstrap(
undefined,
undefined,
mcpHostEndpoint,
wrcConfigureToken,
'clientNotifications'
)
if (result.status >= 400) {
// A pre-v3 mcp-host image answers the Codex bootstrap with a v2
// identity, which the broker rejects. That is a stale workload image,
// not a broker outage: charging it to the configure budget renders as
// "the configured provider is unavailable" and sends the operator
// looking for a credential problem. Record the stale contract as the
// pending policy reason instead.
if (
promptBridge &&
(mcpHostAgent?.provider === 'codex-subscription' ||
mcpHostAgent?.provider === 'grok-subscription') &&
result.body?.policyReason === 'codex_bootstrap_contract_stale'
) {
log.warn(
'Plugin Workload SDK eager mcp-host answered the Codex bootstrap with a pre-v3 contract',
{ status: result.status, podUid: readiness.uid }
)
this.eagerSdkBootstrapProofByRecipe.set(recipeName, {
ready: true,
contractVersion: 2,
podUid: readiness.uid,
provider: mcpHostAgent.provider,
model: mcpHostAgent.model,
policyReady: false,
policyState: 'contract_stale',
policyReason: 'codex_bootstrap_contract_stale',
verifiedAt: new Date().toISOString(),
})
return 'awaiting_policy'
}
this.eagerSdkBootstrapProofByRecipe.delete(recipeName)
return this.recordEagerConfigureFailure(recipeName, configureKey, log, {
reason: 'Plugin Workload SDK eager mcp-host bootstrap returned an error',
detail: { status: result.status },
})
}
const proof = parseEagerSdkBootstrapProof(result.body, readiness.uid, mcpHostAgent?.model)
if (!proof) {
this.eagerSdkBootstrapProofByRecipe.delete(recipeName)
return this.recordEagerConfigureFailure(recipeName, configureKey, log, {
reason: 'Plugin Workload SDK bootstrap response omitted identity proof',
detail: { status: result.status },
})
}
// The host must echo back exactly the binding WRC minted. A different
// binding is a broken host (or a mixed-up pod), not a pending policy:
// accepting it would let the recipe execute under a policy WRC never
// authorized. The hash covers all five binding fields.
//
// ORDER IS LOAD-BEARING: this check must stay BELOW the stale-contract
// arm in the `status >= 400` branch above. A pre-v3 host answers with no
// codexBinding at all, so against a non-null minted binding it compares
// `null !== <hash>` and would be charged here as a mismatched echo —
// converging on `provider_unavailable` and burying the stale-image reason
// under the very symptom that arm exists to replace. Moving this earlier
// silently reopens that.
if (
promptBridge &&
mcpHostAgent?.provider === 'codex-subscription' &&
(proof.codexBinding?.bindingHash ?? null) !== (resolvedCodexBinding?.bindingHash ?? null)
) {
this.eagerSdkBootstrapProofByRecipe.delete(recipeName)
return this.recordEagerConfigureFailure(recipeName, configureKey, log, {
reason:
'Plugin Workload SDK bootstrap echoed a Codex binding that does not match the minted binding',
detail: {
status: result.status,
mintedBindingHash: resolvedCodexBinding?.bindingHash ?? null,
echoedBindingHash: proof.codexBinding?.bindingHash ?? null,
},
})
}
if (
promptBridge &&
mcpHostAgent?.provider === 'grok-subscription' &&
(proof.subscriptionBinding?.bindingHash ?? null) !==
(resolvedGrokBinding?.bindingHash ?? null)
) {
this.eagerSdkBootstrapProofByRecipe.delete(recipeName)
return this.recordEagerConfigureFailure(recipeName, configureKey, log, {
reason:
'Plugin Workload SDK bootstrap echoed a Grok binding that does not match the minted binding',
detail: {
status: result.status,
mintedBindingHash: resolvedGrokBinding?.bindingHash ?? null,
echoedBindingHash: proof.subscriptionBinding?.bindingHash ?? null,
},
})
}
this.eagerSdkBootstrapProofByRecipe.set(recipeName, proof)
this.eagerSdkConfigureFailuresByRecipe.delete(recipeName)
if (promptBridge && proof.policyReady === false) {
return 'awaiting_policy'
}
if (
promptBridge &&
mcpHostAgent?.provider === 'codex-subscription' &&
(proof.contractVersion !== 3 ||
!proof.codexBinding ||
proof.policyReason === 'codex_execution_binding_missing')
) {
return 'awaiting_policy'
}
if (
promptBridge &&
mcpHostAgent?.provider === 'grok-subscription' &&
(proof.contractVersion !== 3 ||
!proof.subscriptionBinding ||
proof.policyReason === 'execution_binding_missing')
) {
return 'awaiting_policy'
}
if (clientNotifications && proof.clientNotificationsPolicyReady === false) {
return 'awaiting_policy'
}
} catch (err) {
this.eagerSdkBootstrapProofByRecipe.delete(recipeName)
return this.recordEagerConfigureFailure(recipeName, configureKey, log, {
reason: 'Plugin Workload SDK eager mcp-host bootstrap failed',
detail: { error: err instanceof Error ? err.message : String(err) },
})
}
return 'ready'
}
async readinessForEagerSdkMcpHost(
recipeName: string
): Promise<{ status: EagerSdkMcpHostStatus; uid?: string; ready: boolean }> {
const phase = await getPodPhase(
this.deps.coreApi,
`${recipeName}-mcp-host`,
this.deps.config.sandboxNamespace
)
if (phase === 'Failed' || phase === 'Unknown') {
return { status: 'failed', ready: false }
}
const readiness = await getPodReadiness(
this.deps.coreApi,
`${recipeName}-mcp-host`,
this.deps.config.sandboxNamespace
)
if (readiness.ready) {
return { status: 'ready', uid: readiness.uid, ready: true }
}
return { status: 'deploying', uid: readiness.uid, ready: false }
}
/**
* Records a consecutive eager SDK bootstrap failure and decides the projected
* status. Below MAX_EAGER_CONFIGURE_ATTEMPTS the recipe stays `deploying` so
* the level-triggered reconcile keeps retrying (a transient mcp-host blip or
* slow boot recovers on its own). Once the bound is reached, the failure is a
* persistent broker problem (e.g. a bad secretRef) and is surfaced as
* `provider_unavailable` so it is no longer indistinguishable from "still
* booting". Mirrors the crash-recovery MAX_ATTEMPTS bound.
*/
private recordEagerConfigureFailure(
recipeName: string,
configureKey: string,
log: Logger,
info: { reason: string; detail: Record<string, unknown> }
): EagerSdkMcpHostStatus {
const current = this.eagerSdkConfigureFailuresByRecipe.get(recipeName)
const attempts = current?.key === configureKey ? current.count + 1 : 1
this.eagerSdkConfigureFailuresByRecipe.set(recipeName, { key: configureKey, count: attempts })
if (attempts >= MAX_EAGER_CONFIGURE_ATTEMPTS) {
log.warn(info.reason, {
...info.detail,
configureKey,
attempts,
maxAttempts: MAX_EAGER_CONFIGURE_ATTEMPTS,
status: 'provider_unavailable',
})
return 'provider_unavailable'
}
log.warn(info.reason, {
...info.detail,
configureKey,
attempts,
maxAttempts: MAX_EAGER_CONFIGURE_ATTEMPTS,
})
// Not ready until the provider is brokered; stay pending so the
// level-triggered reconcile retries the bootstrap.
return 'deploying'
}
/**
* Provisions the recipe-scoped Plugin Workload SDK token Secret (plan
* §3.6). Each allowed caller workload gets its own key (`caller-<id>`) so
* the SDK server can derive caller identity from the bearer token. Syncs
* keys on every reconcile; delete the Secret to rotate all tokens.
*/
async ensurePluginWorkloadSdkTokenSecret(
recipeName: string,
spec: WorkflowRecipeSpec
): Promise<void> {
const secretName = buildPluginWorkloadSdkTokenSecretName(recipeName)
const sandboxNamespace = this.deps.config.sandboxNamespace
const allowedCallers = resolvePluginWorkloadSdkAllowedCallers(spec)
if (allowedCallers.length === 0) {
await this.deletePluginWorkloadSdkTokenSecret(recipeName)
return
}
let existingSecret: k8s.V1Secret | undefined
try {
existingSecret = await this.deps.coreApi.readNamespacedSecret({
name: secretName,
namespace: sandboxNamespace,
})
} catch (err) {
if (getErrorCode(err) !== 404) throw err
}
const existingDecoded: Record<string, string> = {}
for (const [key, value] of Object.entries(existingSecret?.data ?? {})) {
existingDecoded[key] = Buffer.from(value, 'base64').toString('utf8')
}
const tokensByCaller: Record<string, string> = {}
const legacyToken = existingDecoded[PLUGIN_WORKLOAD_SDK_LEGACY_TOKEN_DATA_KEY]
for (const callerRef of allowedCallers) {
const key = pluginWorkloadSdkTokenSecretKey(callerRef)
const existing = existingDecoded[key]
if (existing) {
tokensByCaller[callerRef] = existing
} else if (legacyToken && allowedCallers.length === 1) {
tokensByCaller[callerRef] = legacyToken
} else {
tokensByCaller[callerRef] = randomBytes(32).toString('base64url')
}
}
const desired = buildPluginWorkloadSdkTokenSecret(recipeName, tokensByCaller, sandboxNamespace)
if (!existingSecret) {
try {
await this.deps.coreApi.createNamespacedSecret({
namespace: sandboxNamespace,
body: desired,
})
this.deps.log.info(`Created Secret "${secretName}"`)
} catch (err) {
if (getErrorCode(err) !== 409) throw err
this.deps.log.info(`Secret "${secretName}" already exists (skip create)`)
}
return
}
if (pluginWorkloadSdkSecretTokensMatch(existingDecoded, tokensByCaller)) {
return
}
await this.deps.coreApi.replaceNamespacedSecret({
name: secretName,
namespace: sandboxNamespace,
body: desired,
})
this.deps.log.info(`Synced Secret "${secretName}" (${allowedCallers.length} caller token(s))`)
}
/** Tears down the recipe-scoped SDK workload token Secret (recipe cleanup). */
async deletePluginWorkloadSdkTokenSecret(recipeName: string): Promise<void> {
await this.deps.safeDelete(() =>
this.deps.coreApi.deleteNamespacedSecret({
name: buildPluginWorkloadSdkTokenSecretName(recipeName),
namespace: this.deps.config.sandboxNamespace,
})
)
}
}
function parseEagerSdkBootstrapProof(
body: Record<string, unknown>,
podUid: string,
/** Agent model WRC bootstrapped; a binding for another model is not ours. */
expectedModel: string | undefined
): EagerSdkBootstrapProof | null {
if (body.capabilityFamily === 'clientNotifications') {
if (
body.configured !== true ||
body.ready !== true ||
body.contractVersion !== 2 ||
body.capabilityFamily !== 'clientNotifications' ||
typeof body.policyReady !== 'boolean' ||
typeof body.policyState !== 'string'
) {
return null
}
return {
ready: true,
contractVersion: 2,
podUid,
clientNotificationsPolicyReady: body.policyReady,
clientNotificationsPolicyState: body.policyState,
...(typeof body.policyReason === 'string'
? { clientNotificationsPolicyReason: body.policyReason }
: {}),
verifiedAt: new Date().toISOString(),
}
}
if (
body.configured !== true ||
body.ready !== true ||
(body.contractVersion !== 2 && body.contractVersion !== 3) ||
typeof body.provider !== 'string' ||
typeof body.model !== 'string' ||
body.provider.length === 0 ||
body.model.length === 0
) {
return null
}
if (
(body.provider === 'codex-subscription' || body.provider === 'grok-subscription') &&
body.contractVersion !== 3
) {
return null
}
// Strict slots, matching mcp-host's bootstrap identity: Codex proof lives
// only in `codexBinding`, Grok proof only in `subscriptionBinding`. Neither
// provider's proof ever carries the other slot.
const codexBinding =
body.provider === 'codex-subscription'
? parseCodexBindingProof(body.codexBinding, expectedModel)
: undefined
const subscriptionBinding =
body.provider === 'grok-subscription'
? parseGrokBindingProof(body.subscriptionBinding, expectedModel)
: undefined
if (
body.provider === 'codex-subscription' &&
body.policyReason === 'codex_execution_binding_missing'
) {
return {
ready: true,
contractVersion: 3,
podUid,
provider: body.provider,
model: body.model,
policyReady: false,
policyState: typeof body.policyState === 'string' ? body.policyState : 'binding_missing',
policyReason: 'codex_execution_binding_missing',
verifiedAt: new Date().toISOString(),
}
}
if (body.provider === 'grok-subscription' && body.policyReason === 'execution_binding_missing') {
return {
ready: true,
contractVersion: 3,
podUid,
provider: body.provider,
model: body.model,
policyReady: false,
policyState: typeof body.policyState === 'string' ? body.policyState : 'binding_missing',
policyReason: 'execution_binding_missing',
verifiedAt: new Date().toISOString(),
}
}
if (body.provider === 'codex-subscription' && !codexBinding) {
return null
}
if (body.provider === 'grok-subscription' && !subscriptionBinding) {
return null
}
const hasPolicyProof =
Number.isSafeInteger(body.policyRevision) &&
Number(body.policyRevision) >= 1 &&
typeof body.policyHash === 'string' &&
/^[a-f0-9]{64}$/.test(body.policyHash) &&
typeof body.defaultTargetRef === 'string' &&
body.defaultTargetRef.length > 0 &&
typeof body.defaultProvider === 'string' &&
body.defaultProvider === body.provider &&
typeof body.defaultModel === 'string' &&
body.defaultModel === body.model
return {
ready: true,
contractVersion: body.contractVersion === 3 ? 3 : 2,
podUid,
provider: body.provider,
model: body.model,
...(codexBinding ? { codexBinding } : {}),
...(subscriptionBinding ? { subscriptionBinding } : {}),
policyReady:
body.policyReady !== false &&
hasPolicyProof &&
(body.provider !== 'codex-subscription' || Boolean(codexBinding)) &&
(body.provider !== 'grok-subscription' || Boolean(subscriptionBinding)),
policyState: typeof body.policyState === 'string' ? body.policyState : 'unknown',
...(typeof body.policyReason === 'string' ? { policyReason: body.policyReason } : {}),
verifiedAt: new Date().toISOString(),
...(hasPolicyProof
? {
policyRevision: Number(body.policyRevision),
policyHash: body.policyHash as string,
defaultTargetRef: body.defaultTargetRef as string,
defaultProvider: body.defaultProvider as string,
defaultModel: body.defaultModel as string,
}
: {}),
...(typeof body.clientNotificationsPolicyReady === 'boolean'
? { clientNotificationsPolicyReady: body.clientNotificationsPolicyReady }
: {}),
...(typeof body.clientNotificationsPolicyState === 'string'
? { clientNotificationsPolicyState: body.clientNotificationsPolicyState }
: {}),
...(typeof body.clientNotificationsPolicyReason === 'string'
? { clientNotificationsPolicyReason: body.clientNotificationsPolicyReason }
: {}),
}
}
function parseCodexBindingProof(
value: unknown,
expectedModel: string | undefined
): PluginWorkloadSdkCodexBindingProof | undefined {
// No expected model means no pin is possible, and an unpinned binding could
// be for another model. Refuse the proof instead of accepting it blind.
if (!expectedModel) return undefined
return readVerifiedSdkOnlyCodexBinding(value, expectedModel) ?? undefined
}
function parseGrokBindingProof(
value: unknown,
expectedModel: string | undefined
): PluginWorkloadSdkCodexBindingProof | undefined {
if (!expectedModel) return undefined
return readVerifiedSdkOnlyGrokBinding(value, expectedModel) ?? undefined
}