Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
/**
* Workflow Reconciler — handles recipes with `spec.steps[]`.
*
* Orchestrates the creation of:
* - Coordinator Pod + mcp_host Pod (Two-Pod Model)
* - JWT Secrets for inter-component auth
* - SOUL.md ConfigMap from object storage
* - Workflow config ConfigMap
* - Headless Service for mcp_host DNS
* - NetworkPolicies for inter-Pod communication
*/
import * as k8s from '@kubernetes/client-node'
import { decodeJwt } from 'jose'
import { randomBytes } from 'node:crypto'
import { isIP } from 'node:net'
import type { Pool } from 'pg'
import { GROK_EXECUTE_SCOPE } from '@clerum/codex-catalog-projection'
import {
hasInvalidDigest,
hasLatestTag,
hasUnsafeImageReferenceSyntax,
hasValidSha256Digest,
matchesAllowedImagePrefix,
} from '@clerum/image-policy'
import { mintRecipeHostGfsToken } from '../gfsBinding'
import { createLogger } from '../observability/logger'
import { CRD_GROUP, CRD_VERSION, WORKFLOWRECIPE_PLURAL } from '../reconciler/crdConstants'
import {
deleteOutcomeFields,
observeNamespacedDelete,
shouldRecordDelete,
} from '../reconciler/deleteOutcome'
import {
ResourceVanishedAfterConflictError,
RetryableReconcileError,
getErrorCode,
isRetryableInfraError,
} from '../reconciler/k8sErrors'
import {
type NetworkPolicyConflictReason,
type NetworkPolicyConvergenceDecision,
buildNetworkPolicyReplacement,
classifyOwnerlessNetworkPolicyOwnership,
decideNetworkPolicyConvergence,
} from '../reconciler/networkPolicyConvergence'
import {
resolveStatefulSetHeadlessServiceName,
resolveWorkloadMcpServerLabel,
resolveWorkloadRuntimeResourceName,
} from '../reconciler/resourceBuilder'
import { classifySecretAccess } from '../reconciler/secretOwnership'
import { effectiveWorkflowContextRefForSpec } from '../reconciler/workflowContext'
import type {
StatusCondition,
WorkflowRecipeGfsScope,
WorkflowRecipeSpec,
WorkflowSnippetRunSpec,
} from '../types'
import { resolveEagerSdkMcpHostAgent, resolveMcpHostAgent } from './agentResolution'
import {
CODEX_EXECUTE_SCOPE,
type CodexCatalogSnapshot,
type CodexExecutionProjection,
} from './codexExecutionProjection'
import {
type CodexAllowlistView,
type CodexRecipeVerdict,
type CodexReconcileContext,
projectCodexRecipeVerdict,
} from './codexRecipeVerdict'
import {
type PodPresence,
type PodReadiness,
deletePodIfExists,
evaluateCompletedRuntimePodRecovery,
evaluateCrashRecovery,
getContainerWaitingReason,
getPodPhase,
getPodPresence,
getPodReadiness,
isRecoverableContainerWaitingReason,
waitForPodDeletion,
} from './crashRecovery'
import { JwtTokenFactory } from './jwtTokenFactory'
import {
ALLOWED_MODELS_CONFIGMAP_NAME,
ALLOWLIST_CONFIGMAP_NAMESPACE,
CODEX_UNASSIGNED_CONNECTION_KEY,
parseAllowedModelsSnapshot,
snapshotFromConfigMapError,
} from './llmAllowedModelsSnapshot'
import {
type EffectiveWorkflowControlScope,
type WorkflowControlScope,
issueMcpHostRuntimeTokens,
issueMcpHostWorkflowControlToken,
} from './mcpHostRuntimeTokenIssuerClient'
import type { ModelConfigHandler } from './modelConfigHandler'
import {
NetworkPolicyConfig,
buildRunLaneNetworkPolicyCatalog,
buildWorkflowNetworkPolicies,
} from './networkPolicyFactory'
import { ObjectStorageClient, StorageCredentials, StorageRef } from './objectStorageClient'
import {
type EagerSdkBootstrapProof,
type McpHostRuntimeTokenRefreshReason,
type McpHostRuntimeTokenRefreshResult,
NO_MCP_HOST_RUNTIME_TOKEN_REFRESH,
type NetworkPolicyLegacyFact,
type NetworkPolicyPassSummary,
type NetworkPolicyPruneFact,
PluginWorkloadSdkProvisioner,
type WorkflowNetworkPolicyApplySummary,
} from './pluginWorkloadSdkProvisioner'
import type {
PluginWorkloadSdkRevocationClient,
PluginWorkloadSdkRevocationReceipt,
} from './pluginWorkloadSdkRevocationClient'
import {
type SnippetRunnerSecretAlias,
WORKFLOW_OUTPUT_CLAIM_LABEL,
WORKFLOW_OUTPUT_SCOPE_LABEL,
buildArtifactReaderHeadlessService,
buildArtifactReaderPod,
buildCoordinatorPod,
buildMcpHostHeadlessService,
buildMcpHostPod,
buildMcpHostRouteAliasHeadlessService,
buildSnippetRunnerHeadlessService,
buildSnippetRunnerPod,
buildWorkflowOutputAnchorPod,
buildWorkflowOutputAnchorPodName,
buildWorkflowOutputPreparePod,
buildWorkflowOutputPreparePodName,
declaredPluginWorkloadSdkCapabilities,
recipeDeclaresGrokSubscription,
workflowOutputLabelValue,
} from './podFactory'
import {
buildArtifactReaderServiceName,
buildArtifactReaderUrl,
buildMcpHostRouteAliasServiceName,
buildMcpHostServiceName,
buildMcpHostUrl,
buildPluginWorkloadSdkTokenSecretName,
buildSnippetRunnerServiceName,
} from './resourceNames'
import {
CUSTOM_COORDINATOR_ACTIVE_DEADLINE_BUFFER_SECONDS,
DEFAULT_CUSTOM_COORDINATOR_ACTIVE_DEADLINE_SECONDS,
DEFAULT_STEP_TIMEOUT_SECONDS,
MAX_CUSTOM_COORDINATOR_ACTIVE_DEADLINE_SECONDS,
WORKFLOW_OUTPUT_PVC_NAME,
} from './runtimeConstants'
import {
isAllowedRuntimeHttpEgressCidr,
resolveRuntimeHttpEgressCidrs,
} from './runtimeEgressResolver'
import type { RuntimeHttpEgressResolver } from './runtimeEgressResolver'
import {
type WorkflowRuntimeComponent,
type WorkflowRuntimePlan,
buildWorkflowOutputPvcName,
deriveWorkflowRuntimePlan,
getCustomCoordinatorImage,
hasSnippetSteps,
isSnippetRun,
needsWorkflowMcpHost,
} from './runtimePlan'
import {
type SchedulingRecipe,
WORKFLOW_TEAM_ID_LABEL,
deleteScheduling,
reconcileScheduling,
} from './schedulingHandler'
import {
MCP_HOST_RUNTIME_TOKEN_GENERATION_ANNOTATION,
buildMcpHostRuntimeTokenSecret,
createCoordinatorTokens,
nextMcpHostRuntimeTokenGeneration,
readMcpHostRuntimeTokenGeneration,
} from './secretFactory'
import { SNIPPET_RUN_KEYS } from './snippetRunSchema'
import {
CyclicDependencyError,
UnknownDependencyError,
buildExecutionGroups,
} from './stepDependencyGraph'
import {
AgentSpec,
RecipeClassification,
StepStatus,
WorkflowConfig,
WorkflowExecutionStatus,
WorkflowPhase,
} from './types'
import { validateWorkflowRecipeLimits } from './workflowLimits'
const DEFAULT_WORKFLOW_OUTPUT_STORAGE_SIZE = '256Mi'
const WORKFLOW_OUTPUT_PVC_DELETE_WAIT_MS = 30_000
const WORKFLOW_OUTPUT_PVC_DELETE_POLL_MS = 1_000
const MAX_WORKFLOW_OUTPUT_PREPARE_ATTEMPTS = 3
const MIN_RUNTIME_TOKEN_FILE_REFRESH_LEAD_SECONDS = 75
const RUNTIME_TOKEN_REFRESH_EXPIRY_SAFETY_SECONDS = 5
const MAX_SNIPPET_CODE_LENGTH = 20_000
const SNIPPET_SECRET_ALIAS_RE = /^[A-Za-z][A-Za-z0-9_]{0,63}$/
const CONTROL_PLANE_ADMIN_USAGE_USER_PREFIX = 'admin-ui/'
const DEFAULT_RUNTIME_HTTP_EGRESS_DNS_OVERLAP_SECONDS = 5 * 60
const MAX_RUNTIME_HTTP_EGRESS_ALLOWED_HOSTS = 20
const MAX_RUNTIME_HTTP_EGRESS_POLICY_CIDRS = 64
const RUNTIME_HTTP_EGRESS_CURRENT_CIDRS_ANNOTATION = 'clerum.io/runtime-http-egress-current-cidrs'
const RUNTIME_HTTP_EGRESS_PREVIOUS_CIDRS_ANNOTATION = 'clerum.io/runtime-http-egress-previous-cidrs'
const RUNTIME_HTTP_EGRESS_PREVIOUS_EXPIRES_AT_ANNOTATION =
'clerum.io/runtime-http-egress-previous-expires-at'
const RUNTIME_HTTP_EGRESS_PREVIOUS_CIDR_EXPIRIES_ANNOTATION =
'clerum.io/runtime-http-egress-previous-cidr-expiries'
const RUNTIME_HTTP_EGRESS_RESOLVED_AT_ANNOTATION = 'clerum.io/runtime-http-egress-resolved-at'
// The run lane authors these keys even when desired omits them, so a stale
// live value is drift and the replacement drops it.
const RUNTIME_HTTP_EGRESS_OWNED_ANNOTATIONS: ReadonlySet<string> = new Set([
RUNTIME_HTTP_EGRESS_CURRENT_CIDRS_ANNOTATION,
RUNTIME_HTTP_EGRESS_PREVIOUS_CIDRS_ANNOTATION,
RUNTIME_HTTP_EGRESS_PREVIOUS_EXPIRES_AT_ANNOTATION,
RUNTIME_HTTP_EGRESS_PREVIOUS_CIDR_EXPIRIES_ANNOTATION,
RUNTIME_HTTP_EGRESS_RESOLVED_AT_ANNOTATION,
])
type RunLaneNetworkPolicyApplyResult =
| { policy: string; action: 'created' | 'replaced' | 'unchanged' }
| {
policy: string
action: 'retry'
reason: 'terminating' | 'contended'
}
| { policy: string; action: 'conflict'; reason: NetworkPolicyConflictReason }
/**
* Which Codex/Grok proxy policy survives a prune that does not desire it: a
* proxy whose verdict is uncertain. The skip log names it `reason: 'uncertain'`.
*/
interface RunLaneProxyKeepReasons {
codex: boolean
grok: boolean
}
// Read-then-write rounds per NetworkPolicy and pass. Three rounds is at most
// three writes, the same as the create plus two replaces it replaced.
const NETWORK_POLICY_APPLY_ROUNDS = 3
const MCP_HOST_READINESS_WAIT_TIMEOUT_MS = 4 * 60_000
const DNS_SUBDOMAIN_RE =
/^[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?(?:\.[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?)*$/
const WORKFLOW_OUTPUT_EXTERNAL_CLAIM_LABEL = 'clerum.io/workflow-output-external'
type ExternalWorkflowOutputPvcState =
| 'ready'
| 'terminating'
| 'missing'
| 'wrc-managed-conflict'
| 'unauthorized'
export interface PluginWorkloadSdkCleanupOptions {
/**
* Hybrid workflow recipes share their mcp-host, runtime token Secret and
* workflow network policies with the SDK lane. Remove only SDK-owned
* authority when the capability is edited out; the workflow reconciler
* remains the owner of shared runtime resources.
*/
preserveWorkflowRuntime?: boolean
/**
* Identity for the live legacy-delete ledger. It shares the gone-set and
* backoff with `reconcile()` and the WRC retry; it never erases either.
*/
recipeUid?: string
/**
* True only while the recipe itself is being deleted. A completed cleanup
* may then drop the process-local legacy ledger entry; a pending cleanup
* must retain it so the finalizer retry resumes from the same window.
*/
recipeDeleted?: boolean
}
export class LegacyNetworkPolicyDeletePendingError extends Error {
constructor(
recipeName: string,
readonly retryAfterMs: number
) {
super(`Legacy mcp-servers internet egress NetworkPolicy delete is pending for ${recipeName}`)
this.name = 'LegacyNetworkPolicyDeletePendingError'
}
}
function buildLegacyMcpServersInternetEgressPolicyName(recipeName: string): string {
return `${recipeName}-mcp-servers-egress-internet`
}
export const NETWORK_POLICY_OWNERSHIP_CONDITION_TYPE = 'WorkflowNetworkPolicyOwnership'
/**
* Published only as `False` while the run-lane policies have not converged,
* and removed once they have. It carries two independent facts (see
* `networkPolicyMarkerFacts`):
*
* - apply (`converged | pending`): `pending` when a desired policy was left
* unwritten (terminating or contended). The pass requeues, and the WRC
* short-circuits of a running workflow, which never reach `reconcile()`,
* apply the policies again while it is set.
* - prune (`converged | pending | unevaluated`, see `NetworkPolicyPruneFact`):
* `pending` when a DELETE the prune owed did not land, `unevaluated` when
* its LIST failed. No requeue for either, since nothing but `reconcile()`
* prunes. Only a later `reconcile()` prune that lands, the `unmanaged`
* writer after a full SDK teardown, or the finalizer clears it; every other
* path carries it over (see `networkPolicyMarkerConditions`).
*
* Each combination with a fact that is not converged has its own reason
* (`RetryPending`, `PrunePending`, `PruneUnevaluated`, `RetryAndPrunePending`,
* `RetryAndPruneUnevaluated`), and current facts are read back from the reason.
* The frozen historical combined marker is normalized by the decoder below.
*/
export const NETWORK_POLICIES_CONVERGED_CONDITION_TYPE = 'WorkflowNetworkPoliciesConverged'
/** The two facts the WorkflowNetworkPoliciesConverged marker carries. */
export interface NetworkPolicyMarkerFacts {
apply: 'converged' | 'pending'
prune: NetworkPolicyPruneFact
}
const CONVERGED_NETWORK_POLICY_MARKER_FACTS = {
apply: 'converged',
prune: 'converged',
} as const satisfies NetworkPolicyMarkerFacts
/**
* One current reason per combination of facts. Current messages are prose for
* operators. The decoder separately recognizes the exact historical combined
* marker emitted before those facts had distinct reasons.
*/
const NETWORK_POLICY_MARKER_REASONS = {
RetryPending: {
facts: { apply: 'pending', prune: 'converged' },
message: 'One or more run-lane NetworkPolicies are pending a retry (terminating or contended)',
},
PrunePending: {
facts: { apply: 'converged', prune: 'pending' },
message:
'One or more run-lane NetworkPolicies the spec no longer wants are pending a delete; the next reconcile() pass or the finalizer removes them',
},
PruneUnevaluated: {
facts: { apply: 'converged', prune: 'unevaluated' },
message:
'The run-lane NetworkPolicies the spec no longer wants could not be listed, so none was pruned; the next reconcile() pass or the finalizer removes them',
},
RetryAndPrunePending: {
facts: { apply: 'pending', prune: 'pending' },
message:
'One or more run-lane NetworkPolicies are pending a retry (terminating or contended), and one or more the spec no longer wants are pending a delete',
},
RetryAndPruneUnevaluated: {
facts: { apply: 'pending', prune: 'unevaluated' },
message:
'One or more run-lane NetworkPolicies are pending a retry (terminating or contended), and the ones the spec no longer wants could not be listed for a prune',
},
} as const satisfies Record<string, { facts: NetworkPolicyMarkerFacts; message: string }>
/**
* The only historical message this controller interprets. Before every fact
* had its own reason, `af0152924` encoded apply-plus-prune as `RetryPending`
* with exactly this prose. Equality is deliberate: substring or reworded
* messages do not gain a prune fact.
*/
const HISTORICAL_RETRY_AND_PRUNE_PENDING_MESSAGE =
'One or more run-lane NetworkPolicies are pending a retry (terminating or contended), and one or more the spec no longer wants are pending a delete'
type NetworkPolicyMarkerReason = keyof typeof NETWORK_POLICY_MARKER_REASONS
function isNetworkPolicyMarkerReason(
reason: string | undefined
): reason is NetworkPolicyMarkerReason {
return reason !== undefined && Object.hasOwn(NETWORK_POLICY_MARKER_REASONS, reason)
}
/** The reason for `facts`, or `undefined` when both are converged. */
function networkPolicyMarkerReason(
facts: NetworkPolicyMarkerFacts
): NetworkPolicyMarkerReason | undefined {
const retry = facts.apply === 'pending'
switch (facts.prune) {
case 'converged':
return retry ? 'RetryPending' : undefined
case 'pending':
return retry ? 'RetryAndPrunePending' : 'PrunePending'
case 'unevaluated':
return retry ? 'RetryAndPruneUnevaluated' : 'PruneUnevaluated'
default: {
const unreachable: never = facts.prune
throw new Error(`Unknown NetworkPolicy prune fact: ${String(unreachable)}`)
}
}
}
/**
* Published only as `False`/`DeletePending` while the legacy
* `<recipe>-mcp-servers-egress-internet` policy is pending a delete (see
* `NetworkPolicyLegacyFact`), and removed once it is gone. The fact is read
* back from the reason alone.
*/
export const LEGACY_NETWORK_POLICY_REMOVED_CONDITION_TYPE = 'WorkflowLegacyNetworkPolicyRemoved'
const LEGACY_NETWORK_POLICY_REASONS = {
DeletePending: {
fact: 'pending',
message:
'The legacy mcp-servers internet egress NetworkPolicy is pending a delete; a later pass or the finalizer removes it',
},
} as const satisfies Record<string, { fact: NetworkPolicyLegacyFact; message: string }>
type LegacyNetworkPolicyReason = keyof typeof LEGACY_NETWORK_POLICY_REASONS
function isLegacyNetworkPolicyReason(
reason: string | undefined
): reason is LegacyNetworkPolicyReason {
return reason !== undefined && Object.hasOwn(LEGACY_NETWORK_POLICY_REASONS, reason)
}
/**
* Merged by its own group in `patchStatus`, apart from the workflow-output
* group: a pass that never reached the policy apply leaves the field
* undefined, and the published conditions must survive that pass's patch.
*/
export const NETWORK_POLICY_OWNERSHIP_CONDITION_TYPES = new Set([
NETWORK_POLICY_OWNERSHIP_CONDITION_TYPE,
NETWORK_POLICIES_CONVERGED_CONDITION_TYPE,
LEGACY_NETWORK_POLICY_REMOVED_CONDITION_TYPE,
])
/**
* Whether `fresh` differs from the conditions of the NetworkPolicy group
* already in `existing`, by status, reason and message. `lastTransitionTime`
* and every type outside the group are ignored, so a pass that republishes the
* same conditions opens no status patch.
*/
export function networkPolicyConditionsChanged(
existing: StatusCondition[] | undefined,
fresh: StatusCondition[]
): boolean {
const signal = (conditions: StatusCondition[]) =>
conditions
.filter(c => NETWORK_POLICY_OWNERSHIP_CONDITION_TYPES.has(c.type))
.map(c => `${c.type}\u0000${c.status}\u0000${c.reason ?? ''}\u0000${c.message ?? ''}`)
.sort()
.join('\u0001')
return signal(existing ?? []) !== signal(fresh)
}
/**
* True when the published marker says an apply is pending a retry. A marker
* that carries only a pending prune does not count: the retry prunes nothing,
* so reapplying for it would do no work.
*/
export function hasNetworkPolicyRetryPendingMarker(
conditions: StatusCondition[] | undefined
): boolean {
return networkPolicyMarkerFacts(conditions).apply === 'pending'
}
/**
* True while the published group says the legacy mcp-servers internet policy
* is pending a delete (`WorkflowLegacyNetworkPolicyRemoved=False/DeletePending`).
* Only this reason enters the `legacy-retry` writer; a reason this controller
* did not write is never interpreted.
*/
export function hasLegacyNetworkPolicyDeletePendingMarker(
conditions: StatusCondition[] | undefined
): boolean {
const published = publishedMarker(
conditions,
LEGACY_NETWORK_POLICY_REMOVED_CONDITION_TYPE,
isLegacyNetworkPolicyReason
)
return (
published.kind === 'known' &&
LEGACY_NETWORK_POLICY_REASONS[published.condition.reason as LegacyNetworkPolicyReason].fact ===
'pending'
)
}
/**
* Backoff between two DELETEs of the legacy mcp-servers internet policy for
* one recipe after a DELETE that did not land: 60 s, doubling up to 1 h. A
* transient 403 does not leave a false DeletePending for hours, a sustained
* one costs one DELETE per hour per recipe, and a converged recipe costs none.
*/
export const LEGACY_NETWORK_POLICY_DELETE_BACKOFF_BASE_MS = 60_000
export const LEGACY_NETWORK_POLICY_DELETE_BACKOFF_MAX_MS = 3_600_000
/**
* Reads the facts a published marker carries. This and
* `buildNetworkPolicyConvergedCondition` are the only places that encode what
* the marker means. A `False` marker with any other reason was not written by
* this controller and reads as converged facts.
*/
export function networkPolicyMarkerFacts(
conditions: StatusCondition[] | undefined
): NetworkPolicyMarkerFacts {
const marker = (conditions ?? []).find(
c => c.type === NETWORK_POLICIES_CONVERGED_CONDITION_TYPE && c.status === 'False'
)
if (!isNetworkPolicyMarkerReason(marker?.reason)) {
return { ...CONVERGED_NETWORK_POLICY_MARKER_FACTS }
}
if (
marker.reason === 'RetryPending' &&
marker.message === HISTORICAL_RETRY_AND_PRUNE_PENDING_MESSAGE
) {
return { apply: 'pending', prune: 'pending' }
}
return { ...NETWORK_POLICY_MARKER_REASONS[marker.reason].facts }
}
/**
* The marker for `facts`, or `undefined` when both are converged (the marker
* is then removed). `lastTransitionTime` follows `status`: a marker that stays
* `False` keeps the time it was first published, whichever fact it now names.
*/
export function buildNetworkPolicyConvergedCondition(
facts: NetworkPolicyMarkerFacts,
now: string,
existingConditions?: StatusCondition[]
): StatusCondition | undefined {
const reason = networkPolicyMarkerReason(facts)
if (reason === undefined) return undefined
const existing = existingConditions?.find(
c => c.type === NETWORK_POLICIES_CONVERGED_CONDITION_TYPE && c.status === 'False'
)
return {
type: NETWORK_POLICIES_CONVERGED_CONDITION_TYPE,
status: 'False',
reason,
message: NETWORK_POLICY_MARKER_REASONS[reason].message,
lastTransitionTime: existing?.lastTransitionTime ?? now,
}
}
export const WORKFLOW_OUTPUT_CONDITION_TYPES = new Set([
'WorkflowOutputRwoCompatibility',
'WorkflowOutputWrcManagedLifecycle',
'WorkflowOutputPrepareGate',
'WorkflowOutputExternalClaim',
'WorkflowOutputLegacyGlobalClaim',
])
/**
* The ownership condition for one policy apply pass: `False` naming every
* policy another controller owns, or `[]` when no policy is owned by another
* controller (a pending retry does not count as a conflict). The result goes in
* `networkPolicyOwnershipConditions`, where `[]` removes a previously published
* condition and `undefined` keeps it. `lastTransitionTime` follows `status`, as
* the Kubernetes condition convention has it: a change in the set of
* conflicting policies rewrites `message` and keeps the time the conflict began.
*/
export function buildNetworkPolicyOwnershipConditions(
summary: WorkflowNetworkPolicyApplySummary,
now: string,
existingConditions?: StatusCondition[]
): StatusCondition[] {
if (summary.conflicts.length === 0) return []
const conflicts = summary.conflicts
.map(conflict => `${conflict.policy} (${conflict.reason})`)
.sort()
.join(', ')
const existing = existingConditions?.find(
c => c.type === NETWORK_POLICY_OWNERSHIP_CONDITION_TYPE && c.status === 'False'
)
return [
{
type: NETWORK_POLICY_OWNERSHIP_CONDITION_TYPE,
status: 'False',
reason: 'OwnershipConflict',
message: `NetworkPolicy ownership conflict: ${conflicts}`,
lastTransitionTime: existing?.lastTransitionTime ?? now,
},
]
}
/**
* The legacy marker for `fact`, or `undefined` once the policy is removed.
* `lastTransitionTime` is kept while the marker stays `False`.
*/
function buildLegacyNetworkPolicyCondition(
fact: NetworkPolicyLegacyFact,
now: string,
existingConditions?: StatusCondition[]
): StatusCondition | undefined {
switch (fact) {
case 'removed':
return undefined
case 'pending': {
const existing = existingConditions?.find(
c => c.type === LEGACY_NETWORK_POLICY_REMOVED_CONDITION_TYPE && c.status === 'False'
)
return {
type: LEGACY_NETWORK_POLICY_REMOVED_CONDITION_TYPE,
status: 'False',
reason: 'DeletePending',
message: LEGACY_NETWORK_POLICY_REASONS.DeletePending.message,
lastTransitionTime: existing?.lastTransitionTime ?? now,
}
}
default: {
const unreachable: never = fact
throw new Error(`Unknown legacy NetworkPolicy fact: ${String(unreachable)}`)
}
}
}
/**
* Who writes the NetworkPolicy condition group, and so which facts it
* observed (Table A on `retryRunLaneNetworkPolicies`):
*
* - `reconcile`: a full pass (run lane, eager lane, SDK-only lane with a
* summary). Observes the apply, the prune and the legacy delete.
* - `apply-retry`: the mid-run retry. Observes the apply; carries the prune
* and legacy facts.
* - `legacy-retry`: the legacy delete the WRC retries before its
* short-circuits while `DeletePending` is published. Observes the legacy
* fact; carries the apply and prune facts and the ownership condition.
* - `terminal`: the run ended, so a pending apply is dropped; carries the
* prune and legacy facts and the ownership condition.
* - `unmanaged`: the kill switch, the SDK-only lane without a runtime and
* the capability removal. Written only after a teardown that deleted every
* SDK policy, the legacy one by name included, so it clears the group.
*/
export type NetworkPolicyMarkerWriter =
| { kind: 'reconcile'; summary: NetworkPolicyPassSummary }
| { kind: 'apply-retry'; summary: WorkflowNetworkPolicyApplySummary }
| { kind: 'legacy-retry'; legacy: NetworkPolicyLegacyFact }
| { kind: 'terminal' }
| { kind: 'unmanaged' }
/**
* A marker a carrying writer found published. `unknown` is a condition of a
* marker type whose status or reason this controller did not write: it is
* never interpreted, only kept as it is.
*/
type PublishedMarker =
| { kind: 'absent' }
| { kind: 'known'; condition: StatusCondition }
| { kind: 'unknown'; condition: StatusCondition }
function publishedMarker(
existingConditions: StatusCondition[] | undefined,
type: string,
isKnownReason: (reason: string | undefined) => boolean
): PublishedMarker {
const condition = (existingConditions ?? []).find(c => c.type === type)
if (condition === undefined) return { kind: 'absent' }
if (condition.status === 'False' && isKnownReason(condition.reason)) {
return { kind: 'known', condition }
}
return { kind: 'unknown', condition }
}
function warnUnknownNetworkPolicyMarker(
writer: NetworkPolicyMarkerWriter['kind'],
condition: StatusCondition
): void {
createLogger('wrc', 'network-policy-marker').warn(
'Keeping a NetworkPolicy marker with a reason this controller did not write',
{ writer, type: condition.type, status: condition.status, reason: condition.reason }
)
}
/** The converged marker a carrying writer publishes for its observed apply. */
function carriedConvergedMarker(
writer: NetworkPolicyMarkerWriter['kind'],
apply: NetworkPolicyMarkerFacts['apply'],
existingConditions: StatusCondition[] | undefined,
now: string
): StatusCondition | undefined {
const published = publishedMarker(
existingConditions,
NETWORK_POLICIES_CONVERGED_CONDITION_TYPE,
isNetworkPolicyMarkerReason
)
if (published.kind === 'unknown') {
warnUnknownNetworkPolicyMarker(writer, published.condition)
return published.condition
}
const { prune } = networkPolicyMarkerFacts(existingConditions)
return buildNetworkPolicyConvergedCondition({ apply, prune }, now, existingConditions)
}
/** The legacy marker a carrying writer publishes: the published one, as is. */
function carriedLegacyMarker(
writer: NetworkPolicyMarkerWriter['kind'],
existingConditions: StatusCondition[] | undefined
): StatusCondition | undefined {
const published = publishedMarker(
existingConditions,
LEGACY_NETWORK_POLICY_REMOVED_CONDITION_TYPE,
isLegacyNetworkPolicyReason
)
switch (published.kind) {
case 'absent':
return undefined
case 'known':
return published.condition
case 'unknown':
warnUnknownNetworkPolicyMarker(writer, published.condition)
return published.condition
default: {
const unreachable: never = published
throw new Error(`Unknown published marker: ${String(unreachable)}`)
}
}
}
function presentConditions(conditions: (StatusCondition | undefined)[]): StatusCondition[] {
return conditions.filter((c): c is StatusCondition => c !== undefined)
}
/**
* The whole NetworkPolicy condition group `writer` publishes: the ownership
* condition, the converged marker and the legacy marker, each only while it
* is `False`. `[]` removes the group. Every writer goes through here, so each
* fact is observed, carried or cleared exactly as Table A says.
*/
export function networkPolicyMarkerConditions(
writer: NetworkPolicyMarkerWriter,
existingConditions: StatusCondition[] | undefined,
now: string
): StatusCondition[] {
switch (writer.kind) {
case 'reconcile': {
const { summary } = writer
return presentConditions([
...buildNetworkPolicyOwnershipConditions(summary, now, existingConditions),
buildNetworkPolicyConvergedCondition(
{ apply: summary.retryPending ? 'pending' : 'converged', prune: summary.prune },
now,
existingConditions
),
buildLegacyNetworkPolicyCondition(summary.legacy, now, existingConditions),
])
}
case 'apply-retry': {
const { summary } = writer
return presentConditions([
...buildNetworkPolicyOwnershipConditions(summary, now, existingConditions),
carriedConvergedMarker(
writer.kind,
summary.retryPending ? 'pending' : 'converged',
existingConditions,
now
),
carriedLegacyMarker(writer.kind, existingConditions),
])
}
case 'legacy-retry':
return presentConditions([
...(existingConditions ?? []).filter(
c => c.type === NETWORK_POLICY_OWNERSHIP_CONDITION_TYPE
),
carriedConvergedMarker(
writer.kind,
networkPolicyMarkerFacts(existingConditions).apply,
existingConditions,
now
),
buildLegacyNetworkPolicyCondition(writer.legacy, now, existingConditions),
])
case 'terminal':
return presentConditions([
...(existingConditions ?? []).filter(
c => c.type === NETWORK_POLICY_OWNERSHIP_CONDITION_TYPE
),
carriedConvergedMarker(writer.kind, 'converged', existingConditions, now),
carriedLegacyMarker(writer.kind, existingConditions),
])
case 'unmanaged':
return []
default: {
const unreachable: never = writer
throw new Error(`Unknown NetworkPolicy marker writer: ${String(unreachable)}`)
}
}
}
/**
* The status fields one full pass contributes, shared by the run lane, the
* eager lane and the SDK-only lane so all three translate a summary the same
* way (the `reconcile` writer). No summary (the pass returned before the
* apply) yields neither field, which keeps the published conditions. A
* summary always yields the whole group, `[]` included. Only a pending retry
* sets the retry flag (and with it a requeue).
*/
export function translateNetworkPolicyPassSummary(
summary: NetworkPolicyPassSummary | undefined,
existingConditions: StatusCondition[] | undefined,
now: string
): Pick<WorkflowReconcileResult, 'networkPolicyOwnershipConditions' | 'networkPolicyRetryPending'> {
if (summary === undefined) return {}
return {
networkPolicyOwnershipConditions: networkPolicyMarkerConditions(
{ kind: 'reconcile', summary },
existingConditions,
now
),
...(summary.retryPending ? { networkPolicyRetryPending: true } : {}),
}
}
interface CoordinatorTokenRefreshOptions {
includeMcpHostToken?: boolean
useCustomCoordinatorWrcToken?: boolean
includeSnippetRunnerToken?: boolean
gfsToken?: string
gfsSubject?: string
gfsScopes?: WorkflowRecipeGfsScope[]
}
interface RuntimeHttpEgressPolicyState {
currentCidrs: string[]
effectiveCidrs: string[]
annotations: Record<string, string>
}
type RuntimeHttpEgressPolicyRead =
| { kind: 'absent' }
| { kind: 'foreign' }
| { kind: 'terminating'; annotations: Record<string, string> }
| { kind: 'live'; annotations: Record<string, string> }
type PublicHttpEgressClass = 'exact-host' | 'public-web'
function delay(ms: number): Promise<void> {
return new Promise(resolve => setTimeout(resolve, ms))
}
function pluginWorkloadSdkPolicyReason(
spec: WorkflowRecipeSpec,
proof: EagerSdkBootstrapProof | undefined
): string {
if (spec.pluginWorkloadSdk?.promptBridge && proof?.policyReady === false) {
return proof.policyReason ?? proof.policyState ?? 'unknown'
}
if (
spec.pluginWorkloadSdk?.clientNotifications &&
proof?.clientNotificationsPolicyReady === false
) {
return (
proof.clientNotificationsPolicyReason ?? proof.clientNotificationsPolicyState ?? 'unknown'
)
}
return (
proof?.policyReason ??
proof?.clientNotificationsPolicyReason ??
proof?.policyState ??
proof?.clientNotificationsPolicyState ??
'unknown'
)
}
function normalizeCidrs(cidrs: string[]): string[] {
return [...new Set(cidrs.map(cidr => cidr.trim()).filter(Boolean))].sort()
}
function serializeCidrs(cidrs: string[]): string {
return normalizeCidrs(cidrs).join(',')
}
function parseCidrsAnnotation(value: string | undefined): string[] {
if (!value) return []
return normalizeCidrs(value.split(','))
}
function parseTrustedRuntimeHttpEgressCidrsAnnotation(value: string | undefined): string[] {
return parseCidrsAnnotation(value).filter(isAllowedRuntimeHttpEgressCidr)
}
function serializeRuntimeHttpEgressCidrExpiries(expiries: Map<string, Date>): string {
const ordered = [...expiries.entries()]
.filter(([cidr]) => isAllowedRuntimeHttpEgressCidr(cidr))
.sort(([a], [b]) => a.localeCompare(b))
.map(([cidr, expiresAt]) => [cidr, expiresAt.toISOString()])
return JSON.stringify(Object.fromEntries(ordered))
}
function parseTrustedRuntimeHttpEgressCidrExpiriesAnnotation(
value: string | undefined,
now: Date,
maxFutureMs?: number
): Map<string, Date> {
const expiries = new Map<string, Date>()
if (!value) return expiries
try {
const parsed = JSON.parse(value) as unknown
if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) return expiries
for (const [cidr, expiresAtText] of Object.entries(parsed)) {
if (!isAllowedRuntimeHttpEgressCidr(cidr) || typeof expiresAtText !== 'string') continue
const expiresAt = parseFutureDate(expiresAtText, now, maxFutureMs)
if (expiresAt) expiries.set(cidr, expiresAt)
}
} catch {
return expiries
}
return expiries
}
function parseFutureDate(
value: string | undefined,
now: Date,
maxFutureMs?: number
): Date | undefined {
if (!value) return undefined
const parsedMs = Date.parse(value)
if (!Number.isFinite(parsedMs)) return undefined
if (maxFutureMs !== undefined && parsedMs > now.getTime() + maxFutureMs) return undefined
const parsed = new Date(parsedMs)
return parsed.getTime() > now.getTime() ? parsed : undefined
}
function parseDate(value: string | undefined): Date | undefined {
if (!value) return undefined
const parsedMs = Date.parse(value)
return Number.isFinite(parsedMs) ? new Date(parsedMs) : undefined
}
function latestDate(a: Date | undefined, b: Date | undefined): Date | undefined {
if (!a) return b
if (!b) return a
return a.getTime() >= b.getTime() ? a : b
}
/**
* The latest live resolved-at when every policy the apply writes exists and
* already carries the computed CIDR state. Undefined when a policy is missing,
* any CIDR state annotation changed (a previous-CIDR overlap expiry included),
* or no live value parses, so the caller stamps the current time.
*/
function unchangedRuntimeHttpEgressResolvedAt(
liveAnnotationSets: Array<Record<string, string> | undefined>,
computed: Record<string, string>
): Date | undefined {
let latest: Date | undefined
for (const live of liveAnnotationSets) {
if (!live) return undefined
for (const key of RUNTIME_HTTP_EGRESS_OWNED_ANNOTATIONS) {
if (key === RUNTIME_HTTP_EGRESS_RESOLVED_AT_ANNOTATION) continue
if (live[key] !== computed[key]) return undefined
}
latest = latestDate(latest, parseDate(live[RUNTIME_HTTP_EGRESS_RESOLVED_AT_ANNOTATION]))
}
return latest
}
function addRuntimeHttpEgressPreviousExpiry(
expiries: Map<string, Date>,
cidr: string,
expiresAt: Date
): void {
if (!isAllowedRuntimeHttpEgressCidr(cidr)) return
const existing = expiries.get(cidr)
if (!existing || expiresAt.getTime() > existing.getTime()) {
expiries.set(cidr, expiresAt)
}
}
const GFS_SCOPE_ORDER: WorkflowRecipeGfsScope[] = [
'gfs.read',
'gfs.write',
'gfs.delete',
'gfs.manage_acl',
'gfs.share',
]
const WORKFLOW_CONTROL_SCOPE_ORDER: WorkflowControlScope[] = [
'workflow:list',
'workflow:read',
'workflow:trigger',
'workflow:approval:resolve',
'workflow:approval:decide',
'plugin-workload-sdk',
]
const EFFECTIVE_WORKFLOW_CONTROL_SCOPE_ORDER: EffectiveWorkflowControlScope[] = [
...WORKFLOW_CONTROL_SCOPE_ORDER,
'llm:codex:execute',
'llm:grok:execute',
]
export type { CodexReconcileContext } from './codexRecipeVerdict'
function orderedScopesEqual<T extends string>(actual: T[], expected: T[]): boolean {
if (actual.length !== expected.length) return false
return actual.every((scope, index) => scope === expected[index])
}
function workflowControlScopesEqual(
actual: EffectiveWorkflowControlScope[],
expected: EffectiveWorkflowControlScope[]
): boolean {
return orderedScopesEqual(actual, expected)
}
function gfsScopesEqual(
actual: WorkflowRecipeGfsScope[],
expected: WorkflowRecipeGfsScope[]
): boolean {
return orderedScopesEqual(actual, expected)
}
function workflowHasGfsPublishTargets(spec: WorkflowRecipeSpec): boolean {
return (spec.gfs?.publishTargets ?? []).length > 0
}
function isCodexSnapshotTimeout(error: unknown): boolean {
const code = (error as { code?: unknown }).code
if (code === 'ETIMEDOUT' || code === 'UND_ERR_CONNECT_TIMEOUT' || code === 'ABORT_ERR') {
return true
}
const name = (error as { name?: string }).name
if (name === 'AbortError' || name === 'TimeoutError') return true
const message = error instanceof Error ? error.message : String(error)
return /timed?\s*out/i.test(message)
}
function deriveRecipeHostGfsScopes(spec: WorkflowRecipeSpec): WorkflowRecipeGfsScope[] {
// Workflow/plugin hosts get a read ceiling by default so operators can grant
// readable GFS inputs after install without requiring a recipe spec rewrite.
// The database-backed GFS grant store remains the authority; this token scope
// only caps what a host may use if a matching grant exists.
const scopes = new Set<WorkflowRecipeGfsScope>(['gfs.read'])
if ((spec.gfs?.publishTargets ?? []).length > 0) {
scopes.add('gfs.write')
}
for (const mount of spec.gfs?.mounts ?? []) {
for (const scope of mount.scopes ?? []) {
if ((GFS_SCOPE_ORDER as string[]).includes(scope)) {
scopes.add(scope)
}
}
}
return GFS_SCOPE_ORDER.filter(scope => scopes.has(scope))
}
function deriveWorkflowControlScopes(
spec: WorkflowRecipeSpec,
opts: { pluginWorkloadSdkEnabled?: boolean } = {}
): WorkflowControlScope[] {
// Declarative workflow/SDK scopes only. `llm:codex:execute` is derive-only
// from the Codex eligibility projection and must never be minted here.
const scopes = new Set<WorkflowControlScope>()
// Plugin Workload SDK (plan §3.6): only recipes that declare the
// capability — and only while the feature flag is on — receive the scope
// the control-api authorizer requires (scope_denied otherwise).
if (opts.pluginWorkloadSdkEnabled && spec.pluginWorkloadSdk) {
scopes.add('plugin-workload-sdk')
}
for (const step of spec.steps ?? []) {
// Workflow broker privileges are security boundaries, so derive scopes only
// from explicitly named tools. Wildcards/globs remain fail-closed.
for (const tool of step.allowedTools?.include ?? []) {
if (tool === 'clerum__list_workflows' || tool === 'workflow_list') {
scopes.add('workflow:list')
}
if (
tool === 'clerum__read_workflow' ||
tool === 'workflow_status' ||
tool === 'workflow_health'
) {
scopes.add('workflow:read')
}
if (tool === 'clerum__trigger_workflow' || tool === 'workflow_trigger') {
scopes.add('workflow:trigger')
scopes.add('workflow:approval:resolve')
scopes.add('workflow:approval:decide')
}
}
if (step.requiresApproval) {
scopes.add('workflow:approval:resolve')
scopes.add('workflow:approval:decide')
}
}
return WORKFLOW_CONTROL_SCOPE_ORDER.filter(scope => scopes.has(scope))
}
function hasCompleteAgent(
agent: AgentSpec | { provider?: string; model?: string } | undefined
): boolean {
return Boolean(agent?.provider && agent.model)
}
function requestedCustomCoordinatorActiveDeadlineSeconds(spec: WorkflowRecipeSpec): number {
const steps = spec.steps ?? []
if (steps.length === 0) return DEFAULT_CUSTOM_COORDINATOR_ACTIVE_DEADLINE_SECONDS
const declaredTimeouts = steps.reduce((sum, step) => {
const timeout = Number.isInteger(step.timeoutSeconds)
? Math.max(1, step.timeoutSeconds ?? DEFAULT_STEP_TIMEOUT_SECONDS)
: DEFAULT_STEP_TIMEOUT_SECONDS
return sum + timeout
}, 0)
return Math.max(
DEFAULT_CUSTOM_COORDINATOR_ACTIVE_DEADLINE_SECONDS,
declaredTimeouts + CUSTOM_COORDINATOR_ACTIVE_DEADLINE_BUFFER_SECONDS
)
}
function resolveCustomCoordinatorActiveDeadlineSeconds(spec: WorkflowRecipeSpec): number {
return Math.min(
requestedCustomCoordinatorActiveDeadlineSeconds(spec),
MAX_CUSTOM_COORDINATOR_ACTIVE_DEADLINE_SECONDS
)
}
function validateCustomCoordinatorRuntimePolicy(spec: WorkflowRecipeSpec): string | undefined {
const requestedDeadline = requestedCustomCoordinatorActiveDeadlineSeconds(spec)
if (requestedDeadline > MAX_CUSTOM_COORDINATOR_ACTIVE_DEADLINE_SECONDS) {
return (
'custom coordinator step timeouts exceed the maximum active deadline ' +
`of ${MAX_CUSTOM_COORDINATOR_ACTIVE_DEADLINE_SECONDS}s`
)
}
return undefined
}
function resolveSnippetSecretRefName(
spec: WorkflowRecipeSpec,
secretName: string,
resourceInstances?: Record<string, string>
): string {
const resourceIds = new Set((spec.resources ?? []).map(resource => resource.id))
if (!resourceIds.has(secretName)) return secretName
return resourceInstances?.[secretName] ?? secretName
}
function collectSnippetSecretAliases(
spec: WorkflowRecipeSpec,
resourceInstances?: Record<string, string>
): SnippetRunnerSecretAlias[] {
const byAlias = new Map<string, SnippetRunnerSecretAlias>()
for (const step of spec.steps ?? []) {
if (!isSnippetRun(step.run)) continue
for (const item of step.run.capabilities?.secrets ?? []) {
const existing = byAlias.get(item.alias)
const next = {
alias: item.alias,
secretName: resolveSnippetSecretRefName(spec, item.secretRef.name, resourceInstances),
secretKey: item.secretRef.key,
}
if (
existing &&
(existing.secretName !== next.secretName || existing.secretKey !== next.secretKey)
) {
throw new Error(`snippet secret alias "${item.alias}" maps to multiple secret refs`)
}
byAlias.set(item.alias, next)
}
}
return [...byAlias.values()]
}
function collectSnippetSecretRefs(
spec: WorkflowRecipeSpec,
resourceInstances?: Record<string, string>
): SnippetRunnerSecretAlias[] {
const byRef = new Map<string, SnippetRunnerSecretAlias>()
for (const item of collectSnippetSecretAliases(spec, resourceInstances)) {
byRef.set(`${item.secretName}/${item.secretKey}`, item)
}
return [...byRef.values()]
}
function collectSnippetMcpServerIds(spec: WorkflowRecipeSpec): string[] {
const ids = new Set<string>()
for (const step of spec.steps ?? []) {
if (!isSnippetRun(step.run)) continue
for (const server of step.run.capabilities?.mcp?.servers ?? []) ids.add(server)
}
return [...ids]
}
function collectSnippetDatabaseWorkloadIds(spec: WorkflowRecipeSpec): string[] {
const ids = new Set<string>()
for (const step of spec.steps ?? []) {
if (!isSnippetRun(step.run)) continue
for (const workload of step.run.capabilities?.mongo?.workloads ?? []) ids.add(workload)
for (const workload of step.run.capabilities?.postgres?.workloads ?? []) ids.add(workload)
}
return [...ids]
}
function hasSnippetPublicHttpEgress(spec: WorkflowRecipeSpec): boolean {
// NetworkPolicy is pod-scoped. Per-step host enforcement remains in the snippet SDK.
const egressClass = runtimeHttpEgressClass(spec)
if (egressClass === 'public-web') {
return (spec.steps ?? []).some(
step => isSnippetRun(step.run) && step.run.capabilities?.http?.egressClass === 'public-web'
)
}
return Boolean(
spec.runtimeEgress?.http?.allowedHosts?.length &&
(spec.steps ?? []).some(
step => isSnippetRun(step.run) && Boolean(step.run.capabilities?.http?.allowedHosts?.length)
)
)
}
function hasCustomCoordinatorPublicHttpEgress(spec: WorkflowRecipeSpec): boolean {
if (!spec.coordinatorImage?.trim()) return false
return runtimeHttpEgressClass(spec) === 'public-web'
? spec.runtimeEgress?.http?.egressClass === 'public-web'
: Boolean(spec.runtimeEgress?.http?.allowedHosts?.length)
}
function runtimeHttpEgressClass(spec: WorkflowRecipeSpec): PublicHttpEgressClass {
return spec.runtimeEgress?.http?.egressClass === 'public-web' ? 'public-web' : 'exact-host'
}
function validatePublicHttpHost(host: string): string | undefined {
if (host !== host.trim()) return `HTTP egress host "${host}" must not include whitespace`
if (host.length < 1 || host.length > 253) return `HTTP egress host "${host}" length is invalid`
if (host !== host.toLowerCase()) return `HTTP egress host "${host}" must be lowercase`
if (host.includes('*')) return `HTTP egress host "${host}" must not contain wildcards`
if (host.includes('/') || host.includes(':')) {
return `HTTP egress host "${host}" must be a hostname, not a URL or host:port`
}
if (isIP(host) !== 0) return `HTTP egress host "${host}" must not be an IP literal`
if (
host === 'localhost' ||
host.endsWith('.localhost') ||
host.endsWith('.local') ||
host.endsWith('.internal') ||
host.endsWith('.svc') ||
host.endsWith('.cluster.local') ||
host === 'kubernetes.default' ||
host === 'metadata.goog'
) {
return `HTTP egress host "${host}" must be a public DNS hostname`
}
if (!host.includes('.')) return `HTTP egress host "${host}" must be a public DNS hostname`
const labels = host.split('.')
if (labels.some(label => label.length === 0 || label.length > 63)) {
return `HTTP egress host "${host}" has an invalid DNS label`
}
if (labels.some(label => !/^[A-Za-z0-9](?:[A-Za-z0-9-]*[A-Za-z0-9])?$/.test(label))) {
return `HTTP egress host "${host}" must be a DNS hostname`
}
return undefined
}
function validateRuntimeEgressPolicy(spec: WorkflowRecipeSpec): string | undefined {
const egressClass = spec.runtimeEgress?.http?.egressClass ?? 'exact-host'
if (egressClass !== 'exact-host' && egressClass !== 'public-web') {
return `runtimeEgress.http.egressClass must be exact-host or public-web`
}
const hosts = spec.runtimeEgress?.http?.allowedHosts ?? []
if (egressClass === 'public-web') {
if (hosts.length > 0) {
return `runtimeEgress.http.allowedHosts must be omitted when egressClass is public-web`
}
return undefined
}
if (hosts.length > MAX_RUNTIME_HTTP_EGRESS_ALLOWED_HOSTS) {
return `runtimeEgress.http.allowedHosts must contain at most ${MAX_RUNTIME_HTTP_EGRESS_ALLOWED_HOSTS} hosts`
}
const seen = new Set<string>()
for (const host of hosts) {
const error = validatePublicHttpHost(host)
if (error) return error
if (seen.has(host)) return `HTTP egress host "${host}" is duplicated`
seen.add(host)
}
return undefined
}
async function readCoordinatorRuntimeIfExists(
coreApi: k8s.CoreV1Api,
name: string,
namespace: string
): Promise<{ image?: string; tier?: string } | undefined> {
try {
const pod = await coreApi.readNamespacedPod({ name, namespace })
return {
image: pod.spec?.containers?.[0]?.image,
tier: pod.metadata?.labels?.['clerum.io/coordinator-tier'],
}
} catch (error: unknown) {
if (getErrorCode(error) === 404) return undefined
throw error
}
}
/**
* Resolve `clerum.io/mcpserver` pod-label values for NetworkPolicy selection.
* Workflow workload-spawned servers use their runtime-safe label value;
* external McpServer CRDs use the endpoint hostname.
*/
export function resolveMcpServerFullNames(
recipeName: string,
mcpServers: Array<{ id: string; endpoint?: string }> | undefined,
workloadIdsWithTransport?: ReadonlySet<string>,
workloadMcpServerLabels?: ReadonlyMap<string, string>
): string[] {
const workloadIds = workloadIdsWithTransport ?? new Set<string>()
return (mcpServers ?? []).map(s => {
// Workload-spawned: pod label follows resourceBuilder and must stay
// Kubernetes-label-safe for child recipes with long parent names.
if (workloadIds.has(s.id)) {
return workloadMcpServerLabels?.get(s.id) ?? `${recipeName}-${s.id}`
}
// External McpServer CRD: pod label equals the endpoint hostname (McpServer.metadata.name).
if (s.endpoint) {
const m = s.endpoint.match(/^https?:\/\/([^.:/]+)/)
return m ? m[1] : s.id
}
// Legacy fallback (no endpoint, not a workload): use `{recipeName}-{id}`.
return `${recipeName}-${s.id}`
})
}
export interface WorkflowReconcileResult {
phase: string
message: string
workflowPhase?: WorkflowPhase
clearWorkflowExecution?: boolean
workflowConditions?: StatusCondition[]
/**
* The NetworkPolicy condition group this pass's policy apply produced
* (`networkPolicyMarkerConditions`): the ownership condition and the
* converged and legacy markers. Undefined when the pass returned before the
* apply, which keeps the published group; `[]` when every fact converged
* and no policy is owned by another controller, which removes it.
*/
networkPolicyOwnershipConditions?: StatusCondition[]
/**
* Set when a transient K8s API blip aborted reconcile: the caller must NOT
* patch the CRD status (preserve the current execution state) and should
* requeue. Propagated up through WRC's ReconcileResult.
*/
skipStatusPatch?: boolean
/** Fresh Control API v2 proof tied to the current eager mcp-host pod. */
pluginWorkloadSdkBootstrapProof?: EagerSdkBootstrapProof
/** Eager host identity is ready, but prompt policy awaits operator action. */
pluginWorkloadSdkPolicyPending?: boolean
/**
* A NetworkPolicy this pass left unwritten and wants retried: it is
* terminating, or it was still contended after the bounded apply rounds.
* The caller requeues and publishes the RetryPending marker, which lets the
* recipe reconciler's short-circuits reapply the policies (see its requeue
* comment).
*/
networkPolicyRetryPending?: boolean
}
function mcpHostReadinessMessage(readiness: PodReadiness): string {
const details = [`phase=${readiness.phase ?? 'missing'}`]
if (readiness.waitingReason) details.push(`waiting=${readiness.waitingReason}`)
if (readiness.schedulingReason) details.push(`scheduling=${readiness.schedulingReason}`)
return `Waiting for mcp-host pod to become Ready (${details.join(', ')})`
}
function workflowOutputPrepareWaitMessage(podName: string, readiness: PodReadiness): string {
const details = [`phase=${readiness.phase ?? 'missing'}`]
if (readiness.waitingReason) details.push(`waiting=${readiness.waitingReason}`)
if (readiness.schedulingReason) details.push(`scheduling=${readiness.schedulingReason}`)
return `Waiting for workflow output prepare pod "${podName}" to complete before starting runtime pods (${details.join(', ')})`
}
function workflowOutputAnchorWaitMessage(podName: string, readiness: PodReadiness): string {
const details = [`phase=${readiness.phase ?? 'missing'}`]
if (readiness.waitingReason) details.push(`waiting=${readiness.waitingReason}`)
if (readiness.schedulingReason) details.push(`scheduling=${readiness.schedulingReason}`)
return `Waiting for workflow output anchor pod "${podName}" to mount the output PVC before starting prepare/runtime pods (${details.join(', ')})`
}
function workflowPhaseForMcpHostReadinessWait(
execution: WorkflowExecutionStatus | undefined
): WorkflowPhase {
return execution?.phase === 'recovering' ? 'recovering' : 'initializing'
}
function hasMcpHostReadinessWaitExpired(
execution: WorkflowExecutionStatus | undefined,
nowMs = Date.now()
): boolean {
if (!execution?.startedAt) return false
const startedAtMs = Date.parse(execution.startedAt)
return Number.isFinite(startedAtMs) && nowMs - startedAtMs >= MCP_HOST_READINESS_WAIT_TIMEOUT_MS
}
function trimOutputClaimName(spec: WorkflowRecipeSpec | undefined): string | undefined {
const claimName = spec?.output?.claimName?.trim()
return claimName || undefined
}
function buildStatusCondition(
type: string,
reason: string,
message: string,
lastTransitionTime: string
): StatusCondition {
return { type, status: 'True', reason, message, lastTransitionTime }
}
function buildWorkflowOutputConditions(
runtime: WorkflowRuntimePlan,
now: string,
existingConditions?: StatusCondition[]
): StatusCondition[] {
const output = runtime.output
if (!output.mountRequired) return []
const transitionTime = (type: string): string =>
existingConditions?.find(c => c.type === type && c.status === 'True')?.lastTransitionTime ?? now
const conditions: StatusCondition[] = [
buildStatusCondition(
'WorkflowOutputRwoCompatibility',
'RwoOutputClaimCoLocation',
`Workflow output claim "${output.claimName}" uses RWO compatibility mode: concurrent runs for this recipe are scheduled with required pod affinity to one node.`,
transitionTime('WorkflowOutputRwoCompatibility')
),
]
if (output.claimOwnership === 'wrc-managed') {
conditions.push(
buildStatusCondition(
'WorkflowOutputWrcManagedLifecycle',
'WrcManagedOutputClaim',
`WRC manages workflow output PVC "${output.claimName}". Deleting the parent WorkflowRecipe deletes PVC-backed artifact bytes while DB run history remains.`,
transitionTime('WorkflowOutputWrcManagedLifecycle')
)
)
conditions.push(
buildStatusCondition(
'WorkflowOutputPrepareGate',
'RunScopedOutputPrepared',
`WRC prepares run-scoped workflow output directories on "${output.claimName}" before starting runtime pods so non-root workflow containers can write artifacts.`,
transitionTime('WorkflowOutputPrepareGate')
)
)
} else {
conditions.push(
buildStatusCondition(
'WorkflowOutputExternalClaim',
'ExternalOutputClaim',
`WorkflowRecipe output reuses existing PVC "${output.claimName}". WRC will not create, resize, or delete this claim.`,
transitionTime('WorkflowOutputExternalClaim')
)
)
}
if (output.claimName === WORKFLOW_OUTPUT_PVC_NAME) {
conditions.push(
buildStatusCondition(
'WorkflowOutputLegacyGlobalClaim',
'LegacyGlobalOutputClaim',
`Workflow output is using legacy global PVC "${WORKFLOW_OUTPUT_PVC_NAME}". On GKE RWO this is a compatibility path, not the target state.`,
transitionTime('WorkflowOutputLegacyGlobalClaim')
)
)
}
return conditions
}
function isPreCoordinatorWorkflowPhase(execution: WorkflowExecutionStatus | undefined): boolean {
const phase = execution?.phase
return !phase || phase === 'pending' || phase === 'initializing' || phase === 'recovering'
}
function evaluateWorkflowOutputPrepareRecovery(
currentExecution: WorkflowExecutionStatus | undefined
):
| { action: 'replace'; message: string; newAttempt: number }
| { action: 'fail'; message: string } {
const attempt = currentExecution?.attempt ?? 0
if (attempt >= MAX_WORKFLOW_OUTPUT_PREPARE_ATTEMPTS) {
return {
action: 'fail',
message: `Workflow output prepare pod failed after ${MAX_WORKFLOW_OUTPUT_PREPARE_ATTEMPTS} recovery attempts`,
}
}
return {
action: 'replace',
message: `Workflow output prepare pod failed before runtime pod startup; recreating it (attempt ${attempt + 1}/${MAX_WORKFLOW_OUTPUT_PREPARE_ATTEMPTS})`,
newAttempt: attempt + 1,
}
}
// Re-export canonical CRD type for consumers (tests, etc.)
export type { WorkflowRecipeSpec } from '../types'
export interface WorkflowReconcilerDeps {
coreApi: k8s.CoreV1Api
customApi: k8s.CustomObjectsApi
networkingApi: k8s.NetworkingV1Api
/**
* Postgres pool used by the DB-backed scheduling handler. Optional so unit
* tests that don't touch `spec.scheduling` can omit it — tests that exercise
* the scheduling path provide a mock.
*/
pgPool?: Pool
config: WorkflowConfig
tokenFactory: JwtTokenFactory
resolveRuntimeHttpEgressCidrs?: RuntimeHttpEgressResolver
/**
* WRC-side Secret Broker used to POST /configure to a recipe mcp-host with
* the provider API key. Optional so unit tests and dev mode can omit it;
* the Plugin Workload SDK eager-configure path no-ops without it (the
* mcp-host stays unconfigured until a triggered run injects a model).
*/
modelConfigHandler?: ModelConfigHandler
/**
* Recipe-scoped Control API revocation boundary for SDK-only teardown.
* Teardown is fail-closed when this client is absent; production always
* wires the HTTP implementation and tests provide a deterministic stub.
*/
pluginWorkloadSdkRevocationClient: PluginWorkloadSdkRevocationClient
}
export class WorkflowReconciler {
private readonly log = createLogger('wrc', 'reconciler')
private readonly pluginWorkloadSdkProvisioner: PluginWorkloadSdkProvisioner
/**
* The last allowlist refresh as ONE value. `configMap` is kept raw so each
* recipe's projection can re-parse it with that recipe's assigned grant key
* (HCC keeps the same shape for Hosts), and it is cleared on a failed read so
* the projection sees `missing`. They are a single field because they must be
* read together: a `configMap` from one refresh paired with a verdict from
* another is how a concurrently-reconciled recipe turns a failed read into a
* binding-less v3 configure that wipes a live mcp-host binding.
* Readers capture `this.codexView` once and use only that snapshot.
*
* There is deliberately NO derived boolean here. Every
* consumer reads a field of the ONE `CodexRecipeVerdict` a pass computes
* with `projectCodexRecipeVerdict`. Converging one dimension at a time —
* first `readOk`, then `snapshotError` — always left the next one free to
* diverge; provenance was that next one.
*
* Cleared on a failed read so grant decisions stay fail-closed. Do not send
* a binding-less v3 configure in that window — skip configure so mcp-host
* keeps a live binding. Symmetrically (see `resolveEffectiveControlScopes`),
* an unreadable ConfigMap must not revoke the Codex scope either: with no
* data there is no policy decision to apply in either direction.
*/
private codexView: {
configMap?: k8s.V1ConfigMap
snapshot: CodexCatalogSnapshot
} = { snapshot: { flagEnabled: false } }
private readonly codexContexts = new Map<string, CodexReconcileContext>()
/** B3(c): one process-lifetime DELETE of the leftover mcp-servers internet NP. Keyed by recipe uid. */
private readonly prunedLegacyMcpServersInternetEgress = new Set<string>()
/**
* R4-L4: the backoff after a legacy DELETE that did not land, keyed like the
* gone set. `notBeforeMs` is the earliest time the next DELETE may be sent.
*/
private readonly legacyMcpServersInternetEgressBackoff = new Map<
string,
{ delayMs: number; notBeforeMs: number }
>()
constructor(private readonly deps: WorkflowReconcilerDeps) {
this.pluginWorkloadSdkProvisioner = new PluginWorkloadSdkProvisioner({
coreApi: deps.coreApi,
config: deps.config,
tokenFactory: deps.tokenFactory,
modelConfigHandler: deps.modelConfigHandler,
log: this.log,
ensureMcpHostSecrets: (
namespace,
recipeName,
runtimeScopeRecipeName,
spec,
recipeUid,
codexVerdict
) =>
this.ensureMcpHostSecrets(
namespace,
recipeName,
runtimeScopeRecipeName,
spec,
recipeUid,
codexVerdict
),
applyWorkflowNetworkPolicies: (
recipeName,
recipeUid,
spec,
runtime,
awaitsTriggeredRun,
codexProjection,
eagerSdkMcpHost,
grokProjection
) =>
this.applyWorkflowNetworkPolicies(
recipeName,
recipeUid,
spec,
runtime,
awaitsTriggeredRun,
codexProjection,
eagerSdkMcpHost,
grokProjection
),
ensureMcpHostHeadlessService: recipeName => this.ensureMcpHostHeadlessService(recipeName),
createIfNotExists: (createFn, label) => this.createIfNotExists(createFn, label),
safeDelete: deleteFn => this.safeDelete(deleteFn),
})
}
setCodexReconcileContext(context: CodexReconcileContext | null, recipeUid?: string): void {
if (!context) {
const uid = recipeUid?.trim()
if (uid) this.codexContexts.delete(uid)
return
}
this.codexContexts.set(context.recipeUid, context)
}
private resolveCodexContext(
recipeUid: string,
recipeName: string,
runtimeScopeRecipeName: string
): CodexReconcileContext {
const stored = this.codexContexts.get(recipeUid)
if (stored) return stored
return {
recipeUid,
recipeName,
runtimeScopeRecipeName,
claimedParent: false,
parentSpec: null,
connectionKey: CODEX_UNASSIGNED_CONNECTION_KEY,
}
}
/**
* The single Codex verdict for one reconcile pass.
*
* `view` is a REQUIRED parameter and never defaults to `this.codexView`:
* the caller must capture the allowlist refresh once and hand the same
* object to every consumer. Reading the live field here is exactly how the
* scope path and the configure path ended up on two different snapshots
* within one pass (scope dimension).
*/
private codexVerdictFor(
spec: WorkflowRecipeSpec,
recipeUid: string,
recipeName: string,
runtimeScopeRecipeName: string,
view: CodexAllowlistView
): CodexRecipeVerdict {
const context = this.resolveCodexContext(recipeUid, recipeName, runtimeScopeRecipeName)
return projectCodexRecipeVerdict({
ownSpec: spec,
context,
hostAgent: resolveEagerSdkMcpHostAgent(spec),
view,
// The WRC master switch is an input of the ONE Grok verdict, so scopes,
// grok-proxy egress and the SDK bootstrap binding agree with the pod env
// (`buildMcpHostPod` gates on the same flag).
grokSubscriptionEnabled: this.deps.config.grokSubscriptionEnabled === true,
log: this.log,
})
}
/**
* Effective control scopes plus whether the Codex decision rests on data.
*
* `codexScopeUncertain` is true when the projection could not decide —
* an unreadable allowlist ConfigMap, or a spec whose Codex provenance could
* not be established. In that state `derivedScopes` is empty, which is
* indistinguishable from a real revocation; the token refresh uses the flag
* to preserve whatever the live JWT already carries instead of reminting
* (and rolling the pod) over a transient read failure. A readable ConfigMap
* that says `reauth-required` is a decision, not uncertainty: it still
* withdraws the scope.
*/
private resolveEffectiveControlScopes(
spec: WorkflowRecipeSpec,
verdict: CodexRecipeVerdict
): {
scopes: EffectiveWorkflowControlScope[]
codexScopeUncertain: boolean
grokScopeUncertain: boolean
} {
const workflow = deriveWorkflowControlScopes(spec, {
pluginWorkloadSdkEnabled: this.deps.config.pluginWorkloadSdkEnabled,
})
const derived = [
...verdict.projection.derivedScopes,
...(verdict.grokProjection?.derivedScopes ?? []),
].filter(scope => !workflow.includes(scope as WorkflowControlScope))
return {
scopes: [...workflow, ...derived] as EffectiveWorkflowControlScope[],
codexScopeUncertain: verdict.projection.eligibility === 'uncertain',
grokScopeUncertain: verdict.grokProjection?.eligibility === 'uncertain',
}
}
private async refreshCodexSnapshot(): Promise<{
configMap?: k8s.V1ConfigMap
snapshot: CodexCatalogSnapshot
}> {
try {
const cm = await this.deps.coreApi.readNamespacedConfigMap({
name: ALLOWED_MODELS_CONFIGMAP_NAME,
namespace: ALLOWLIST_CONFIGMAP_NAMESPACE,
})
this.codexView = { configMap: cm, snapshot: parseAllowedModelsSnapshot(cm) }
return this.codexView
} catch (err) {
if (isCodexSnapshotTimeout(err)) {
this.codexView = { snapshot: snapshotFromConfigMapError('timeout') }
this.log.warn('Codex allowlist ConfigMap read timed out; failing closed', {
configMap: ALLOWED_MODELS_CONFIGMAP_NAME,
namespace: ALLOWLIST_CONFIGMAP_NAMESPACE,
})
return this.codexView
}
const code = getErrorCode(err)
if (code === 401 || code === 403) {
this.codexView = { snapshot: snapshotFromConfigMapError('forbidden') }
this.log.warn('Codex allowlist ConfigMap read forbidden; failing closed', {
configMap: ALLOWED_MODELS_CONFIGMAP_NAME,
namespace: ALLOWLIST_CONFIGMAP_NAMESPACE,
statusCode: code,
})
return this.codexView
}
this.codexView = { snapshot: snapshotFromConfigMapError('missing') }
this.log.warn('Codex allowlist ConfigMap unavailable; failing closed', {
configMap: ALLOWED_MODELS_CONFIGMAP_NAME,
namespace: ALLOWLIST_CONFIGMAP_NAMESPACE,
statusCode: code,
})
return this.codexView
}
}
private get runtimeTokenRefreshBeforeSeconds(): number {
return this.deps.config.runtimeTokenRefreshBeforeSeconds
}
private get runtimeTokenFileRefreshBeforeSeconds(): number {
const maxLead = Math.max(
1,
this.deps.config.runtimeTokenTtlSeconds - RUNTIME_TOKEN_REFRESH_EXPIRY_SAFETY_SECONDS
)
return Math.min(
maxLead,
Math.max(this.runtimeTokenRefreshBeforeSeconds, MIN_RUNTIME_TOKEN_FILE_REFRESH_LEAD_SECONDS)
)
}
validateWorkflowSpec(spec: WorkflowRecipeSpec): string | undefined {
const maxWorkflowSteps = this.deps.config.maxWorkflowSteps
const stepCount = spec.steps?.length ?? 0
if (stepCount > maxWorkflowSteps) {
return `WorkflowRecipe has ${stepCount} steps, exceeding WRC_MAX_WORKFLOW_STEPS=${maxWorkflowSteps}`
}
const validationError = this.validateExecutableSteps(spec)
if (validationError) return validationError
const outputValidationError = this.validateWorkflowOutputSpec(spec)
if (outputValidationError) return outputValidationError
const runtimeEgressPolicyError = validateRuntimeEgressPolicy(spec)
if (runtimeEgressPolicyError) return runtimeEgressPolicyError
const coordinatorImageOverride = getCustomCoordinatorImage(spec)
const imagePolicyError = this.validateCustomCoordinatorImagePolicy(coordinatorImageOverride)
if (imagePolicyError) return imagePolicyError
if (coordinatorImageOverride) {
const runtimePolicyError = validateCustomCoordinatorRuntimePolicy(spec)
if (runtimePolicyError) return runtimePolicyError
}
if (hasSnippetSteps(spec) && this.deps.config.enableSnippetRuntime === false) {
return 'snippet workflow runtime is disabled'
}
return undefined
}
/**
* Reconcile the always-on Plugin Workload SDK host for a recipe that has no
* executable workflow steps. This deliberately does not call `reconcile()`:
* SDK-only workloads must not acquire coordinator, run, step, or output-PVC
* lifecycle merely because they share the WorkflowRecipe CRD.
*/
async reconcilePluginWorkloadSdkOnly(
recipeName: string,
recipeUid: string,
namespace: string,
spec: WorkflowRecipeSpec,
runtimeScopeRecipeName = recipeName
): Promise<{
phase: 'active' | 'awaiting_policy' | 'deploying' | 'failed' | 'provider_unavailable'
message: string
pluginWorkloadSdkBootstrapProof?: EagerSdkBootstrapProof
/** Undefined when the eager host returned before applying the policies. */
networkPolicies?: NetworkPolicyPassSummary
}> {
if (!this.deps.config.pluginWorkloadSdkEnabled || !spec.pluginWorkloadSdk) {
// Nothing is applied here, so no summary: an empty one would read as
// "evaluated, no conflict" and clear a published ownership condition.
return {
phase: 'active',
message: 'Plugin Workload SDK runtime disabled',
}
}
if ((spec.steps?.length ?? 0) > 0) {
throw new Error('SDK-only runtime requires spec.steps to be absent or empty')
}
const codexView = await this.refreshCodexSnapshot()
// One verdict per pass. Scope, binding and the configure skip are fields of
// this object, not three derivations that must be kept in agreement.
const codexVerdict = this.codexVerdictFor(
spec,
recipeUid,
recipeName,
runtimeScopeRecipeName,
codexView
)
const runtime = deriveWorkflowRuntimePlan(spec, {
recipeName,
runtimeScopeRecipeName,
pluginWorkloadSdkEnabled: true,
})
const mcpHostPhase = await getPodPhase(
this.deps.coreApi,
`${recipeName}-mcp-host`,
this.deps.config.sandboxNamespace
)
const { status, networkPolicies } =
await this.pluginWorkloadSdkProvisioner.ensureEagerSdkMcpHost(
recipeName,
recipeUid,
namespace,
runtimeScopeRecipeName,
spec,
runtime,
{ mcpHostPhase, codexVerdict }
)
const bootstrapProof = this.pluginWorkloadSdkProvisioner.getBootstrapProof(recipeName)
switch (status) {
case 'ready':
return {
phase: 'active',
message: 'Plugin Workload SDK mcp-host registered',
pluginWorkloadSdkBootstrapProof: bootstrapProof,
networkPolicies,
}
case 'awaiting_policy':
return {
phase: 'awaiting_policy',
message: `Plugin Workload SDK operator policy pending (${pluginWorkloadSdkPolicyReason(spec, bootstrapProof)})`,
pluginWorkloadSdkBootstrapProof: bootstrapProof,
networkPolicies,
}
case 'deploying':
return {
phase: 'deploying',
message: 'Plugin Workload SDK mcp-host starting',
networkPolicies,
}
case 'provider_unavailable':
return {
phase: 'provider_unavailable',
message: 'Plugin Workload SDK mcp-host provider unavailable',
networkPolicies,
}
case 'failed':
return {
phase: 'failed',
message: 'Plugin Workload SDK mcp-host could not start',
networkPolicies,
}
}
}
/**
* Revoke and physically fence the Plugin Workload SDK capability. This is
* capability-level cleanup, deliberately independent from workflow
* coordinator teardown so both SDK-only and hybrid recipes share the same
* authorization-fencing invariant.
*/
async cleanupPluginWorkloadSdk(
recipeName: string,
options: PluginWorkloadSdkCleanupOptions = {}
): Promise<void> {
const ns = this.deps.config.sandboxNamespace
const preserveWorkflowRuntime = options.preserveWorkflowRuntime === true
this.pluginWorkloadSdkProvisioner.clearRecipeState(recipeName)
const sdkNetworkPolicyNames = [
`${recipeName}-workload-to-mcp-host-sdk-ingress`,
`${recipeName}-workload-to-mcp-host-sdk-egress`,
`${recipeName}-mcp-host-to-wrc-sdk-broker`,
]
const sharedNetworkPolicyNames = [
`${recipeName}-mcp-host-to-servers`,
`${recipeName}-mcp-host-to-llm-api`,
`${recipeName}-mcp-host-to-gfs`,
`${recipeName}-mcp-host-to-approval-gateway`,
`${recipeName}-mcp-host-to-codex-proxy`,
`${recipeName}-mcp-host-to-grok-proxy`,
]
const networkPolicyNames = preserveWorkflowRuntime
? sdkNetworkPolicyNames
: [...sdkNetworkPolicyNames, ...sharedNetworkPolicyNames]
let revocationReceipt: PluginWorkloadSdkRevocationReceipt | null = null
const revocationClient = this.deps.pluginWorkloadSdkRevocationClient
if (!revocationClient) {
throw new Error(
'Plugin Workload SDK revocation client is not configured; refusing to tear down an unfenced capability'
)
}
try {
// Authorization fencing is the first state transition. If Control API
// cannot revoke (or reports a CAS conflict), retain the finalizer and
// every owned runtime object for a retry; deleting locally first would
// hide an unfenced broker grant and make recovery non-deterministic.
revocationReceipt = await revocationClient.revoke(ns, recipeName)
if (revocationReceipt.state === 'conflict') {
throw new Error(`Plugin Workload SDK revocation conflict for recipe "${recipeName}"`)
}
} catch (error) {
this.log.error('Plugin Workload SDK control-plane revocation failed', {
recipeName,
error: error instanceof Error ? error.message : String(error),
})
throw error
}
// Close the service and network-policy edges before removing the pod or
// token Secrets. This ordering is the revocation boundary: a live pod
// cannot keep receiving broker traffic while its credentials are being
// deleted, and a failed delete is requeued instead of reported as disabled.
if (!preserveWorkflowRuntime) {
await this.teardownDelete(() =>
this.deps.coreApi.deleteNamespacedService({
name: buildMcpHostServiceName(recipeName),
namespace: ns,
})
)
}
await Promise.all([
...sdkNetworkPolicyNames.map(name =>
this.teardownDelete(() =>
this.deps.networkingApi.deleteNamespacedNetworkPolicy({ name, namespace: ns })
)
),
...(preserveWorkflowRuntime
? []
: sharedNetworkPolicyNames.map(name =>
this.teardownDelete(() =>
this.deps.networkingApi.deleteNamespacedNetworkPolicy({ name, namespace: ns })
)
)),
// Older SDK-only reconciles used the shared workflow policy factory and
// left coordinator-shaped policies behind. Sweep the complete recipe
// label set so upgrades and dynamic policy names cannot retain a stale
// isolation lane. The selector is recipe-scoped and only matches WRC-
// managed policies, never the namespace-wide baseline policies.
...(preserveWorkflowRuntime
? []
: [
this.teardownDelete(() =>
this.deleteNetworkPoliciesByLabelSelector(
ns,
`clerum.io/recipe=${recipeName},clerum.io/managed-by=wrc`,
buildLegacyMcpServersInternetEgressPolicyName(recipeName)
)
),
this.teardownDelete(() =>
this.deleteNetworkPoliciesByLabelSelector(
this.deps.config.mcpServerNamespace,
`clerum.io/recipe=${recipeName},clerum.io/managed-by=wrc`,
buildLegacyMcpServersInternetEgressPolicyName(recipeName)
)
),
]),
])
if (!preserveWorkflowRuntime) {
await this.teardownDelete(() =>
this.deps.coreApi.deleteNamespacedPod({
name: `${recipeName}-mcp-host`,
namespace: ns,
propagationPolicy: 'Background',
})
)
}
await Promise.all([
this.teardownDelete(() =>
this.deps.coreApi.deleteNamespacedSecret({
name: buildPluginWorkloadSdkTokenSecretName(recipeName),
namespace: ns,
})
),
...(preserveWorkflowRuntime
? []
: [
// The runtime JWT Secret is created for eager SDK hosts as well as
// workflow hosts. SDK-only capability removal must revoke it with
// the host; hybrid workflow cleanup owns the shared Secret.
this.teardownDelete(() =>
this.deps.coreApi.deleteNamespacedSecret({
name: `wf-${recipeName}-mcp-host-runtime-tokens`,
namespace: ns,
})
),
]),
])
if (!preserveWorkflowRuntime) {
// The legacy internet policy may carry no labels, while a labeled copy
// is excluded from the sweeps above. Both forms go through the one live
// ledger so a completed cleanup can prove `unmanaged`. It runs after the
// host pod and token Secrets are gone: a pending legacy DELETE must not
// hold the SDK credentials in place.
await this.cleanupLegacyMcpServersInternetEgress(
recipeName,
options.recipeUid,
options.recipeDeleted === true
)
}
await this.assertPluginWorkloadSdkResourcesAbsent(recipeName, ns, networkPolicyNames, {
preserveWorkflowRuntime,
})
if (revocationReceipt.state === 'revoking') {
if (!revocationReceipt.revocationId) {
throw new Error(`Plugin Workload SDK revocation receipt for ${recipeName} has no epoch`)
}
const finalized = await revocationClient.finalize(
ns,
recipeName,
revocationReceipt.revocationId
)
// `missing` is also terminal: an operator may have removed the grant
// after the revocation epoch was fenced. There is no remaining broker
// authority to finalize in that case, so retaining the recipe finalizer
// would turn a safe, idempotent cleanup into a permanent wedge.
if (finalized.state !== 'disabled' && finalized.state !== 'missing') {
throw new Error(
`Plugin Workload SDK revocation for ${recipeName} was not confirmed disabled or absent`
)
}
}
if (options.recipeDeleted === true) {
this.forgetLegacyMcpServersInternetEgress(recipeName, options.recipeUid)
}
}
private async assertPluginWorkloadSdkResourcesAbsent(
recipeName: string,
namespace: string,
networkPolicyNames: string[],
options: PluginWorkloadSdkCleanupOptions = {}
): Promise<void> {
// Keep the API calls lazy until every precondition (notably pod deletion)
// has completed. Starting a read and then awaiting the pod poll leaves a
// rejection from an already-absent object temporarily unhandled; Node can
// terminate the controller before the later Promise.allSettled observes it.
// Lazy checks preserve the fail-closed rule while making teardown safe for
// the normal idempotent 404 path.
const checks: Array<() => Promise<unknown>> = [
() =>
this.deps.coreApi.readNamespacedSecret({
name: buildPluginWorkloadSdkTokenSecretName(recipeName),
namespace,
}),
...networkPolicyNames.map(
name => () => this.deps.networkingApi.readNamespacedNetworkPolicy({ name, namespace })
),
]
if (!options.preserveWorkflowRuntime) {
const podGone = await waitForPodDeletion(
this.deps.coreApi,
`${recipeName}-mcp-host`,
namespace,
{
timeoutMs: 5_000,
pollIntervalMs: 250,
}
)
if (!podGone)
throw new Error(`Plugin Workload SDK pod for ${recipeName} still exists after teardown`)
checks.push(
() =>
this.deps.coreApi.readNamespacedSecret({
name: `wf-${recipeName}-mcp-host-runtime-tokens`,
namespace,
}),
() =>
this.deps.coreApi.readNamespacedService({
name: buildMcpHostServiceName(recipeName),
namespace,
}),
() =>
this.deps.coreApi.readNamespacedEndpoints({
name: buildMcpHostServiceName(recipeName),
namespace,
}),
() =>
this.deps.networkingApi.listNamespacedNetworkPolicy({
namespace,
labelSelector: `clerum.io/recipe=${recipeName},clerum.io/managed-by=wrc`,
}),
() =>
this.deps.networkingApi.listNamespacedNetworkPolicy({
namespace: this.deps.config.mcpServerNamespace,
labelSelector: `clerum.io/recipe=${recipeName},clerum.io/managed-by=wrc`,
})
)
}
const results = await Promise.allSettled(checks.map(check => check()))
const unexpected = results.find(result => {
if (result.status !== 'fulfilled') return false
const value = result.value as { items?: unknown[] } | undefined
return Array.isArray(value?.items) ? value.items.length > 0 : true
})
if (unexpected) {
throw new Error(`Plugin Workload SDK resource for ${recipeName} still exists after teardown`)
}
const failed = results.find(
(result): result is PromiseRejectedResult =>
result.status === 'rejected' && getErrorCode(result.reason) !== 404
)
if (failed) throw failed.reason
}
async reconcile(
recipeName: string,
recipeUid: string,
namespace: string,
spec: WorkflowRecipeSpec,
currentStatus?: {
workflowExecution?: WorkflowExecutionStatus
steps?: StepStatus[]
conditions?: StatusCondition[]
resourceInstances?: Record<string, string>
},
resolvedInputs?: Record<string, unknown>,
runtimeScopeRecipeName = recipeName,
workflowRunId?: string,
workflowTeamId?: string,
workflowActorId?: string,
workflowActorType?: string
): Promise<WorkflowReconcileResult> {
const preflightError = this.validateWorkflowSpec(spec)
if (preflightError) return { phase: 'failed', message: preflightError, workflowPhase: 'failed' }
// Capture the allowlist refresh ONCE for this pass and
// compute the verdict from it. Every consumer below — eager configure, the
// run path's secrets, and the NetworkPolicy decision — reads this object
// instead of `this.codexView`, which a concurrent recipe can replace
// between two of them.
const codexView = await this.refreshCodexSnapshot()
const codexVerdict = this.codexVerdictFor(
spec,
recipeUid,
recipeName,
runtimeScopeRecipeName,
codexView
)
// A triggered run reconciles the run-scoped child CR; snippet Secret ownership
// belongs to the verified parent scope (WRC-resolved runtimeScopeRecipeName),
// never the child name. Non-triggered recipes default the scope to recipeName.
const secretPreflightError = await this.validateSnippetSecretRefs(
runtimeScopeRecipeName,
spec,
currentStatus?.resourceInstances
)
if (secretPreflightError) {
return { phase: 'failed', message: secretPreflightError, workflowPhase: 'failed' }
}
const runtime = deriveWorkflowRuntimePlan(spec, {
recipeName,
runtimeScopeRecipeName,
workflowRunId: workflowRunId?.trim(),
pluginWorkloadSdkEnabled: this.deps.config.pluginWorkloadSdkEnabled,
})
const classification = runtime.classification
const needsMcpHost = runtime.mcpHost.required
const needsSnippetRunner = runtime.snippetRunner.required
const usesCustomCoordinator = runtime.coordinator.kind === 'custom'
const needsArtifactReader = runtime.artifactReader.required
const workflowOutputClaimOwnership = runtime.output.claimOwnership
const workflowOutputClaimName = runtime.output.mountRequired ? runtime.output.claimName : ''
const workflowOutputSubPath = runtime.output.mountRequired ? runtime.output.subPath : ''
const desiredCoordinatorImage =
runtime.coordinator.imageOverride ?? this.deps.config.coordinatorImage
const workflowRunIdValue = workflowRunId?.trim()
const workflowActorIdValue = workflowActorId?.trim()
const workflowUsageUserId =
workflowActorType === 'user'
? workflowActorIdValue
: workflowActorType === 'admin' && workflowActorIdValue
? `${CONTROL_PLANE_ADMIN_USAGE_USER_PREFIX}${workflowActorIdValue}`
: undefined
// eagerMcpHostForSdk: no run is triggered yet, but the recipe declares
// pluginWorkloadSdk, so the mcp-host must be always-on to host the SDK
// server (:8099). ONLY the mcp-host starts; the coordinator and run-scoped
// pods still defer — the coordinator requires a CLERUM_WORKFLOW_RUN_ID and
// crashes without one.
const hasPluginWorkloadSdk =
this.deps.config.pluginWorkloadSdkEnabled && !!spec.pluginWorkloadSdk
const awaitsTriggeredRun = needsMcpHost && !workflowRunIdValue
const eagerMcpHostForSdk = hasPluginWorkloadSdk && !workflowRunIdValue
const workflowConditions = buildWorkflowOutputConditions(
runtime,
new Date().toISOString(),
currentStatus?.conditions
)
// Empty until this pass applies the policies, then what the apply found.
// A return before the apply leaves it empty, and patchStatus keeps the
// published condition.
let networkPolicyStatus: ReturnType<typeof translateNetworkPolicyPassSummary> = {}
const withWorkflowConditions = (result: WorkflowReconcileResult): WorkflowReconcileResult => ({
...result,
workflowConditions,
...networkPolicyStatus,
})
const outputAnchorPodName = runtime.output.anchorRequired
? buildWorkflowOutputAnchorPodName(runtimeScopeRecipeName)
: undefined
const needsWorkflowOutputPrepare = runtime.output.prepareRequired && !awaitsTriggeredRun
const outputPreparePodName = needsWorkflowOutputPrepare
? buildWorkflowOutputPreparePodName(recipeName)
: undefined
const log = createLogger('wrc', recipeName)
log.info(`Reconciling workflow`, {
classification,
needsMcpHost,
awaitsTriggeredRun,
needsSnippetRunner,
customCoordinator: usesCustomCoordinator,
})
try {
// Check crash recovery for existing Pods
const coordPresence = await getPodPresence(
this.deps.coreApi,
`${recipeName}-coordinator`,
this.deps.config.sandboxNamespace
)
let coordPhase = coordPresence.kind === 'present' ? coordPresence.phase : undefined
const coordWaitingReason = await getContainerWaitingReason(
this.deps.coreApi,
`${recipeName}-coordinator`,
this.deps.config.sandboxNamespace
)
let outputAnchorPhase = outputAnchorPodName
? await getPodPhase(
this.deps.coreApi,
outputAnchorPodName,
this.deps.config.sandboxNamespace
)
: undefined
let outputPreparePhase = outputPreparePodName
? await getPodPhase(
this.deps.coreApi,
outputPreparePodName,
this.deps.config.sandboxNamespace
)
: undefined
const mcpHostPresence: PodPresence = needsMcpHost
? await getPodPresence(
this.deps.coreApi,
`${recipeName}-mcp-host`,
this.deps.config.sandboxNamespace
)
: { kind: 'unread' }
let mcpHostPhase = mcpHostPresence.kind === 'present' ? mcpHostPresence.phase : undefined
let artifactReaderPhase = needsArtifactReader
? await getPodPhase(
this.deps.coreApi,
`${recipeName}-artifact-reader`,
this.deps.config.sandboxNamespace
)
: undefined
let snippetRunnerPhase = needsSnippetRunner
? await getPodPhase(
this.deps.coreApi,
`${recipeName}-snippet-runner`,
this.deps.config.sandboxNamespace
)
: undefined
const mcpHostWaitingReason = needsMcpHost
? await getContainerWaitingReason(
this.deps.coreApi,
`${recipeName}-mcp-host`,
this.deps.config.sandboxNamespace
)
: undefined
if (coordPhase) {
const existingCoordinator = await readCoordinatorRuntimeIfExists(
this.deps.coreApi,
`${recipeName}-coordinator`,
this.deps.config.sandboxNamespace
)
const desiredCoordinatorTier = usesCustomCoordinator ? 'custom' : 'builtin'
if (existingCoordinator?.tier && existingCoordinator.tier !== desiredCoordinatorTier) {
return withWorkflowConditions({
phase: 'failed',
message:
'coordinator tier is immutable after the workflow runtime pod has been created',
workflowPhase: 'failed',
})
}
if (
usesCustomCoordinator &&
existingCoordinator?.image &&
existingCoordinator.image !== desiredCoordinatorImage
) {
return withWorkflowConditions({
phase: 'failed',
message:
'coordinator image is immutable after the workflow runtime pod has been created',
workflowPhase: 'failed',
})
}
}
if (awaitsTriggeredRun) {
const samePassPresence: Partial<Record<WorkflowRuntimeComponent, PodPresence>> = {
'workflow-coordinator': coordPresence,
...(mcpHostPresence.kind !== 'unread' ? { 'workflow-mcp-host': mcpHostPresence } : {}),
}
for (const component of runtime.cleanup.deleteBeforeTriggeredRun) {
// Plugin Workload SDK: keep the mcp-host alive across the
// awaiting-trigger window — it hosts the always-on SDK server.
// Only the coordinator (and run-scoped pods) are torn down here.
if (eagerMcpHostForSdk && component === 'workflow-mcp-host') continue
// B3(d): skip DELETE only when the same-pass GET was a 404. A live
// pod with an empty status.phase is present and must still be deleted.
const presence = samePassPresence[component]
if (presence?.kind === 'absent') {
log.debug('Skipping runtime component DELETE; same-pass GET was absent', {
recipe: recipeName,
component,
presence: presence.kind,
})
continue
}
await this.deleteRuntimeComponentIfExists(recipeName, component)
}
// Plugin Workload SDK: bring up the mcp-host (and broker the provider
// for promptBridge) before the run-scoped output/PVC gating below — a
// pure-SDK recipe has no run yet, so it must not wait on run output
// infrastructure to expose the always-on SDK server.
if (eagerMcpHostForSdk) {
// The eager mcp-host hosts only the always-on SDK server; it produces
// no run artifacts yet, so it must NOT mount the workflow-output PVC
// (that PVC is provisioned by the triggered-run path). Mounting it
// here would wedge the pod in Pending — the PVC does not exist until a
// run starts. The triggered run later recreates the pod with the mount.
// Reuse the verdict this pass already computed rather than deriving a
// second one from the same view. Two derivations cannot disagree here
// (identical arguments, `codexView` is const), but a second one is a
// seam waiting to drift the moment either call site's arguments move —
// and it emits the uncertain-provenance warning twice per pass.
// Resolving the binding inside the provisioner (after its own awaits)
// let a concurrently-reconciled recipe's failed refresh hand over a
// null binding that read as decidable — a binding-less v3 configure
// that wipes the live host binding.
const { status: eagerStatus, networkPolicies: eagerNetworkPolicies } =
await this.pluginWorkloadSdkProvisioner.ensureEagerSdkMcpHost(
recipeName,
recipeUid,
namespace,
runtimeScopeRecipeName,
spec,
runtime,
{ mcpHostPhase, codexVerdict }
)
const eagerBootstrapProof =
this.pluginWorkloadSdkProvisioner.getBootstrapProof(recipeName)
// No summary means the host returned before the apply, so the pass
// cannot say whether a conflict is gone. A `failed` pod after the
// apply still carries a real summary and publishes it.
networkPolicyStatus = translateNetworkPolicyPassSummary(
eagerNetworkPolicies,
currentStatus?.conditions,
new Date().toISOString()
)
if (eagerStatus === 'failed') {
return withWorkflowConditions({
phase: 'failed',
message: 'Plugin Workload SDK mcp-host could not start',
workflowPhase: 'failed',
})
}
if (eagerStatus === 'provider_unavailable') {
// The eager mcp-host is Ready but its provider could not be brokered
// after repeated /configure attempts (e.g. a bad secretRef). Stay
// active so the level-triggered reconcile keeps retrying — the
// recipe self-heals once the broker config is fixed — but surface a
// distinct condition so this is no longer indistinguishable from
// "still booting".
const providerUnavailableMessage = `Plugin Workload SDK mcp-host provider unavailable: /configure failed repeatedly (${classification})`
return {
...withWorkflowConditions({
phase: 'active',
message: providerUnavailableMessage,
clearWorkflowExecution: true,
}),
workflowConditions: [
...workflowConditions,
buildStatusCondition(
'PluginWorkloadSdkProviderUnavailable',
'EagerConfigureFailed',
providerUnavailableMessage,
new Date().toISOString()
),
],
}
}
const eagerPolicyPending = eagerStatus === 'awaiting_policy'
const pluginWorkloadSdkDeclared =
spec.pluginWorkloadSdk?.promptBridge !== undefined ||
spec.pluginWorkloadSdk?.clientNotifications !== undefined
const promptBridgeDeclared = spec.pluginWorkloadSdk?.promptBridge !== undefined
const eagerReady =
(eagerStatus === 'ready' || eagerPolicyPending) &&
(!pluginWorkloadSdkDeclared ||
!promptBridgeDeclared ||
eagerBootstrapProof !== undefined)
if (eagerStatus === 'ready' && promptBridgeDeclared && !eagerBootstrapProof) {
return withWorkflowConditions({
phase: 'deploying',
message: `Plugin Workload SDK bootstrap policy proof pending (${classification})`,
clearWorkflowExecution: true,
})
}
const eagerMessage = eagerPolicyPending
? `Plugin Workload SDK operator policy pending (${pluginWorkloadSdkPolicyReason(spec, eagerBootstrapProof)}) (${classification})`
: eagerReady
? `Plugin Workload SDK mcp-host registered (${classification})`
: `Plugin Workload SDK mcp-host starting (${classification})`
return withWorkflowConditions({
// While the eager mcp-host is still booting (eagerStatus ===
// 'deploying'), return 'deploying' — NOT 'active' — so the watcher
// requeues at the fixed progress interval and re-runs the eager
// /configure once the pod becomes Ready. Returning 'active' here
// drops the requeue (requeueAfterMs=undefined at workflowRecipe
// Reconciler), so a pod that becomes Ready AFTER this reconcile is
// never re-reconciled and stays "waiting for /configure" forever —
// the SDK provider is never armed and promptBridge fails with
// provider_unavailable. 'ready' projects 'active' (configured).
phase: eagerReady ? 'active' : 'deploying',
message: eagerMessage,
clearWorkflowExecution: true,
pluginWorkloadSdkBootstrapProof: eagerBootstrapProof,
pluginWorkloadSdkPolicyPending: eagerPolicyPending,
})
}
}
// If coordinator Pod has failed, evaluate crash recovery
if (
!awaitsTriggeredRun &&
(coordPhase === 'Failed' ||
coordPhase === 'Unknown' ||
isRecoverableContainerWaitingReason(coordPhase, coordWaitingReason))
) {
const recovery = evaluateCrashRecovery(
coordPhase,
currentStatus?.workflowExecution,
coordWaitingReason
)
if (recovery.action === 'fail') {
return withWorkflowConditions({
phase: 'failed',
message: recovery.message,
workflowPhase: 'failed',
})
}
if (recovery.action === 'replace') {
log.info(recovery.message)
// Persist attempt counter to CRD status BEFORE deleting the Pod —
// without this, attempt stays at 0 and MAX_ATTEMPTS guard never fires.
const existingExecution = currentStatus?.workflowExecution ?? {}
await this.deps.customApi.patchNamespacedCustomObjectStatus(
{
group: CRD_GROUP,
version: CRD_VERSION,
namespace,
plural: WORKFLOWRECIPE_PLURAL,
name: recipeName,
body: {
status: {
workflowExecution: {
...existingExecution,
phase: recovery.newPhase,
attempt: recovery.newAttempt,
},
},
},
},
{
middleware: [k8s.setHeaderMiddleware('Content-Type', 'application/merge-patch+json')],
}
)
for (const component of runtime.cleanup.deleteAfterCoordinatorReplacement) {
await this.deleteRuntimeComponentIfExists(recipeName, component)
if (component === 'workflow-mcp-host') mcpHostPhase = undefined
if (component === 'workflow-artifact-reader') artifactReaderPhase = undefined
if (component === 'workflow-snippet-runner') snippetRunnerPhase = undefined
}
await deletePodIfExists(
this.deps.coreApi,
`${recipeName}-coordinator`,
this.deps.config.sandboxNamespace
)
coordPhase = undefined
// Fall through to Pod creation below
}
}
// Similarly for mcp_host Pod
const isPreCoordinatorPhase = isPreCoordinatorWorkflowPhase(currentStatus?.workflowExecution)
if (
needsMcpHost &&
!awaitsTriggeredRun &&
(mcpHostPhase === 'Failed' ||
mcpHostPhase === 'Unknown' ||
(isPreCoordinatorPhase &&
(mcpHostPhase === 'Succeeded' ||
isRecoverableContainerWaitingReason(mcpHostPhase, mcpHostWaitingReason))))
) {
const recovery =
mcpHostPhase === 'Succeeded'
? evaluateCompletedRuntimePodRecovery(currentStatus?.workflowExecution, 'mcp_host')
: evaluateCrashRecovery(
mcpHostPhase,
currentStatus?.workflowExecution,
mcpHostWaitingReason
)
if (recovery.action === 'fail') {
return withWorkflowConditions({
phase: 'failed',
message: recovery.message,
workflowPhase: 'failed',
})
}
if (recovery.action === 'replace') {
log.info(`mcp_host: ${recovery.message}`)
const existingExecution = currentStatus?.workflowExecution ?? {}
await this.deps.customApi.patchNamespacedCustomObjectStatus(
{
group: CRD_GROUP,
version: CRD_VERSION,
namespace,
plural: WORKFLOWRECIPE_PLURAL,
name: recipeName,
body: {
status: {
workflowExecution: {
...existingExecution,
phase: recovery.newPhase,
attempt: recovery.newAttempt,
},
},
},
},
{
middleware: [k8s.setHeaderMiddleware('Content-Type', 'application/merge-patch+json')],
}
)
await deletePodIfExists(
this.deps.coreApi,
`${recipeName}-mcp-host`,
this.deps.config.sandboxNamespace
)
mcpHostPhase = undefined
}
}
const coordinatorGfsScopes = deriveRecipeHostGfsScopes(spec)
const coordinatorGfsSubject = `host:3rd:${namespace}/${runtimeScopeRecipeName}`
const coordinatorGfsBinding = workflowHasGfsPublishTargets(spec)
? await mintRecipeHostGfsToken(namespace, runtimeScopeRecipeName, {
scopes: coordinatorGfsScopes,
})
: undefined
const coordinatorGfsToken = coordinatorGfsBinding ? coordinatorGfsBinding.token : undefined
// 1. Create coordinator token Secret
const tokenSecret = await createCoordinatorTokens(
recipeName,
this.deps.tokenFactory,
this.deps.config.sandboxNamespace,
{
...runtime.tokens.coordinator,
['gfs' + 'To' + 'ken']: coordinatorGfsToken,
}
)
// WRC→mcp-host tokens are now minted fresh per request inside
// configureModel()/getArtifact() — nothing is stored in-process anymore.
const tokenSecretCreated = await this.createIfNotExists(
() =>
this.deps.coreApi.createNamespacedSecret({
namespace: this.deps.config.sandboxNamespace,
body: tokenSecret,
}),
`Secret "wf-${recipeName}-coordinator-token"`
)
if (!tokenSecretCreated) {
await this.refreshCoordinatorTokenIfExpiring(recipeName, namespace, {
...runtime.tokens.coordinator,
['gfs' + 'To' + 'ken']: coordinatorGfsToken,
gfsSubject: coordinatorGfsSubject,
gfsScopes: coordinatorGfsScopes,
})
}
if (needsMcpHost && !awaitsTriggeredRun) {
await this.ensureMcpHostSecrets(
namespace,
recipeName,
runtimeScopeRecipeName,
spec,
recipeUid,
codexVerdict
)
}
if (runtime.output.ensurePvc) {
const outputPvcReady = await this.ensureWorkflowOutputPvc(
runtimeScopeRecipeName,
runtime.output.claimName,
spec
)
if (!outputPvcReady) {
return withWorkflowConditions({
phase: 'deploying',
message: `Workflow output PVC "${runtime.output.claimName}" is terminating; waiting for Kubernetes to finish deletion before creating runtime pods`,
workflowPhase: 'initializing',
})
}
} else if (workflowOutputClaimOwnership === 'external') {
const externalPvcState = await this.isExternalWorkflowOutputPvcReady(
workflowOutputClaimName,
runtimeScopeRecipeName
)
if (externalPvcState !== 'ready') {
if (externalPvcState === 'wrc-managed-conflict') {
return withWorkflowConditions({
phase: 'failed',
message: `External workflow output PVC "${workflowOutputClaimName}" is managed by WRC; choose an operator-owned PVC or remove spec.output.claimName`,
workflowPhase: 'failed',
})
}
if (externalPvcState === 'unauthorized') {
return withWorkflowConditions({
phase: 'failed',
message:
`External workflow output PVC "${workflowOutputClaimName}" is not labeled for workflow output scope "${runtimeScopeRecipeName}". ` +
`Add labels ${WORKFLOW_OUTPUT_EXTERNAL_CLAIM_LABEL}=true, ${WORKFLOW_OUTPUT_CLAIM_LABEL}=${workflowOutputLabelValue(workflowOutputClaimName)}, and ${WORKFLOW_OUTPUT_SCOPE_LABEL}=${workflowOutputLabelValue(runtimeScopeRecipeName)} or remove spec.output.claimName.`,
workflowPhase: 'failed',
})
}
return withWorkflowConditions({
phase: 'deploying',
message:
externalPvcState === 'terminating'
? `External workflow output PVC "${workflowOutputClaimName}" is terminating; waiting before creating runtime pods`
: `External workflow output PVC "${workflowOutputClaimName}" was not found in ${this.deps.config.sandboxNamespace}; create it or remove spec.output.claimName`,
workflowPhase: 'initializing',
})
}
}
if (runtime.output.anchorRequired && outputAnchorPodName) {
if (outputAnchorPhase === 'Failed' || outputAnchorPhase === 'Unknown') {
await deletePodIfExists(
this.deps.coreApi,
outputAnchorPodName,
this.deps.config.sandboxNamespace
)
outputAnchorPhase = undefined
}
if (!outputAnchorPhase) {
const anchorPod = buildWorkflowOutputAnchorPod(runtimeScopeRecipeName, this.deps.config, {
workflowOutputClaimName,
})
await this.createIfNotExists(
() =>
this.deps.coreApi.createNamespacedPod({
namespace: this.deps.config.sandboxNamespace,
body: anchorPod,
}),
`Pod "${outputAnchorPodName}"`
)
if (!awaitsTriggeredRun) {
return withWorkflowConditions({
phase: 'deploying',
message: `Created workflow output anchor pod "${outputAnchorPodName}"; waiting for it to mount the output PVC before starting prepare/runtime pods`,
workflowPhase: 'initializing',
})
}
}
if (!awaitsTriggeredRun && outputAnchorPhase !== 'Running') {
const readiness = await getPodReadiness(
this.deps.coreApi,
outputAnchorPodName,
this.deps.config.sandboxNamespace
)
return withWorkflowConditions({
phase: 'deploying',
message: workflowOutputAnchorWaitMessage(outputAnchorPodName, readiness),
workflowPhase:
currentStatus?.workflowExecution?.phase === 'recovering'
? 'recovering'
: 'initializing',
})
}
}
if (needsWorkflowOutputPrepare && outputPreparePodName) {
if (outputPreparePhase === 'Failed') {
const recovery = evaluateWorkflowOutputPrepareRecovery(currentStatus?.workflowExecution)
if (recovery.action === 'fail') {
return withWorkflowConditions({
phase: 'failed',
message: recovery.message,
workflowPhase: 'failed',
})
}
const existingExecution = currentStatus?.workflowExecution ?? {}
await this.deps.customApi.patchNamespacedCustomObjectStatus(
{
group: CRD_GROUP,
version: CRD_VERSION,
namespace,
plural: WORKFLOWRECIPE_PLURAL,
name: recipeName,
body: {
status: {
workflowExecution: {
...existingExecution,
phase: 'recovering',
attempt: recovery.newAttempt,
message: recovery.message,
},
},
},
},
{
middleware: [k8s.setHeaderMiddleware('Content-Type', 'application/merge-patch+json')],
}
)
await deletePodIfExists(
this.deps.coreApi,
outputPreparePodName,
this.deps.config.sandboxNamespace
)
return withWorkflowConditions({
phase: 'deploying',
message: recovery.message,
workflowPhase: 'recovering',
})
}
if (outputPreparePhase === 'Unknown') {
await deletePodIfExists(
this.deps.coreApi,
outputPreparePodName,
this.deps.config.sandboxNamespace
)
outputPreparePhase = undefined
}
if (!outputPreparePhase) {
const preparePod = buildWorkflowOutputPreparePod(recipeName, this.deps.config, {
workflowOutputClaimName,
workflowOutputSubPath,
workflowOutputScope: runtimeScopeRecipeName,
})
await this.createIfNotExists(
() =>
this.deps.coreApi.createNamespacedPod({
namespace: this.deps.config.sandboxNamespace,
body: preparePod,
}),
`Pod "${outputPreparePodName}"`
)
return withWorkflowConditions({
phase: 'deploying',
message: `Created workflow output prepare pod "${outputPreparePodName}"; waiting before starting runtime pods`,
workflowPhase: 'initializing',
})
}
if (outputPreparePhase !== 'Succeeded') {
const readiness = await getPodReadiness(
this.deps.coreApi,
outputPreparePodName,
this.deps.config.sandboxNamespace
)
return withWorkflowConditions({
phase: 'deploying',
message: workflowOutputPrepareWaitMessage(outputPreparePodName, readiness),
workflowPhase: 'initializing',
})
}
}
// 2. Create SOUL.md ConfigMap
await this.ensureSoulConfigMap(recipeName, spec)
// 3. Create workflow config ConfigMap
await this.ensureWorkflowConfigMap(recipeName, recipeUid, spec, resolvedInputs)
if (needsMcpHost && !awaitsTriggeredRun) {
await this.ensureMcpHostHeadlessService(recipeName)
await this.ensureHeadlessServiceExists(
buildMcpHostRouteAliasServiceName(recipeName, this.deps.config.sandboxNamespace),
buildMcpHostRouteAliasHeadlessService(recipeName, this.deps.config.sandboxNamespace)
)
}
if (needsArtifactReader && !awaitsTriggeredRun) {
await this.ensureHeadlessServiceExists(
buildArtifactReaderServiceName(recipeName),
buildArtifactReaderHeadlessService(recipeName, this.deps.config.sandboxNamespace)
)
}
if (needsSnippetRunner && !awaitsTriggeredRun) {
await this.ensureHeadlessServiceExists(
buildSnippetRunnerServiceName(recipeName),
buildSnippetRunnerHeadlessService(recipeName, this.deps.config.sandboxNamespace)
)
}
// 5. Create NetworkPolicies before Pods. The namespace default-deny is
// already active, and custom coordinators can call WRC immediately on
// startup. Applying allow policies first avoids a startup race where the
// first status/probe request is blocked before coord-to-wrc exists.
const runLaneNetworkPolicies = await this.applyWorkflowNetworkPolicies(
recipeName,
recipeUid,
spec,
runtime,
awaitsTriggeredRun,
codexVerdict.projection,
false,
codexVerdict.grokProjection
)
networkPolicyStatus = translateNetworkPolicyPassSummary(
runLaneNetworkPolicies,
currentStatus?.conditions,
new Date().toISOString()
)
// 6. Create Pods — mcp-host FIRST, then coordinator. If the coordinator
// resolves DNS before mcp-host's EndpointSlice exists, undici caches the
// NXDOMAIN for 5-30s and startup latency balloons.
//
// NOTE: artifact-producing workflows mount `/output` from run-scoped
// storage. The WRC-managed claim uses
// `workflow-output/<parentRecipe>/<runId>` for child runs; explicit PVC
// output uses the same run-scoped layout inside the dedicated claim.
// Eager→run transition: a recipe that ran the eager SDK path created the
// mcp-host WITHOUT the workflow-output mount (no PVC existed yet). When a
// run is now triggered and needs that mount, the still-running eager pod
// would be 409-skipped by createIfNotExists and the run would have no
// output volume. Detect the mount-less pod and delete it so it is rebuilt
// with the mount below. Scoped to the pre-coordinator phase so it only
// fires as a run starts — never on an already-running/completed workflow.
if (
needsMcpHost &&
!awaitsTriggeredRun &&
isPreCoordinatorPhase &&
mcpHostPhase &&
runtime.pods.mcpHost?.mountWorkflowOutput &&
!(await this.mcpHostPodHasWorkflowOutputMount(recipeName))
) {
await deletePodIfExists(
this.deps.coreApi,
`${recipeName}-mcp-host`,
this.deps.config.sandboxNamespace
)
const priorPodPhase = await getPodPhase(
this.deps.coreApi,
`${recipeName}-mcp-host`,
this.deps.config.sandboxNamespace
)
if (priorPodPhase === 'Running' || priorPodPhase === 'Pending') {
return withWorkflowConditions({
phase: 'deploying',
message: 'Waiting for prior eager mcp-host pod to terminate before run-scoped recreate',
workflowPhase: currentStatus?.workflowExecution?.phase ?? 'initializing',
})
}
mcpHostPhase = undefined
}
if (
needsMcpHost &&
!awaitsTriggeredRun &&
(!mcpHostPhase || mcpHostPhase === 'Failed' || mcpHostPhase === 'Unknown')
) {
const mcpHostAgent = this.resolveMcpHostAgent(spec)
if (!mcpHostAgent) {
return withWorkflowConditions({
phase: 'failed',
message: 'Workflow spec missing required agent configuration',
workflowPhase: 'failed',
})
}
const mcpHostPod = buildMcpHostPod(
recipeName,
mcpHostAgent,
this.deps.config,
runtimeScopeRecipeName,
namespace,
runtime.pods.mcpHost!.workflowOutputClaimName,
runtime.pods.mcpHost!.workflowOutputSubPath,
effectiveWorkflowContextRefForSpec(recipeName, spec),
{
gfsScopes: coordinatorGfsScopes,
mountWorkflowOutput: runtime.pods.mcpHost!.mountWorkflowOutput,
workflowOutputScope: runtime.pods.mcpHost!.workflowOutputScope,
pluginWorkloadSdkCapabilities: this.deps.config.pluginWorkloadSdkEnabled
? declaredPluginWorkloadSdkCapabilities(spec.pluginWorkloadSdk)
: [],
grokSubscriptionEnabled: this.deps.config.grokSubscriptionEnabled === true,
recipeAgentProvider: spec.agent?.provider,
recipeDeclaresGrok: recipeDeclaresGrokSubscription(spec),
}
)
await this.createIfNotExists(
() =>
this.deps.coreApi.createNamespacedPod({
namespace: this.deps.config.sandboxNamespace,
body: mcpHostPod,
}),
`Pod "${recipeName}-mcp-host"`
)
}
if (needsMcpHost && !awaitsTriggeredRun && isPreCoordinatorPhase) {
const readiness = await getPodReadiness(
this.deps.coreApi,
`${recipeName}-mcp-host`,
this.deps.config.sandboxNamespace
)
if (!readiness.ready) {
const message = mcpHostReadinessMessage(readiness)
if (hasMcpHostReadinessWaitExpired(currentStatus?.workflowExecution)) {
return withWorkflowConditions({
phase: 'failed',
message: `${message}; readiness deadline exceeded`,
workflowPhase: 'failed',
})
}
return withWorkflowConditions({
phase: 'deploying',
message,
workflowPhase: workflowPhaseForMcpHostReadinessWait(currentStatus?.workflowExecution),
})
}
}
if (
!awaitsTriggeredRun &&
needsArtifactReader &&
(!artifactReaderPhase ||
artifactReaderPhase === 'Failed' ||
artifactReaderPhase === 'Unknown')
) {
if (artifactReaderPhase === 'Failed' || artifactReaderPhase === 'Unknown') {
await deletePodIfExists(
this.deps.coreApi,
`${recipeName}-artifact-reader`,
this.deps.config.sandboxNamespace
)
artifactReaderPhase = undefined
}
const artifactReaderPod = buildArtifactReaderPod(
recipeName,
this.deps.config,
runtime.pods.artifactReader!
)
await this.createIfNotExists(
() =>
this.deps.coreApi.createNamespacedPod({
namespace: this.deps.config.sandboxNamespace,
body: artifactReaderPod,
}),
`Pod "${recipeName}-artifact-reader"`
)
}
if (
!awaitsTriggeredRun &&
needsSnippetRunner &&
(!snippetRunnerPhase || snippetRunnerPhase === 'Failed' || snippetRunnerPhase === 'Unknown')
) {
if (snippetRunnerPhase === 'Failed' || snippetRunnerPhase === 'Unknown') {
await deletePodIfExists(
this.deps.coreApi,
`${recipeName}-snippet-runner`,
this.deps.config.sandboxNamespace
)
snippetRunnerPhase = undefined
}
let secretAliases: SnippetRunnerSecretAlias[]
try {
secretAliases = collectSnippetSecretAliases(spec, currentStatus?.resourceInstances)
} catch (err) {
return withWorkflowConditions({
phase: 'failed',
message: err instanceof Error ? err.message : String(err),
workflowPhase: 'failed',
})
}
const snippetRunnerPod = buildSnippetRunnerPod(recipeName, this.deps.config, {
...runtime.pods.snippetRunner!,
secretAliases,
})
await this.createIfNotExists(
() =>
this.deps.coreApi.createNamespacedPod({
namespace: this.deps.config.sandboxNamespace,
body: snippetRunnerPod,
}),
`Pod "${recipeName}-snippet-runner"`
)
}
if (
!awaitsTriggeredRun &&
(!coordPhase || coordPhase === 'Failed' || coordPhase === 'Unknown')
) {
const coordPod = buildCoordinatorPod(recipeName, this.deps.config, {
...runtime.pods.coordinator,
activeDeadlineSeconds: usesCustomCoordinator
? resolveCustomCoordinatorActiveDeadlineSeconds(spec)
: undefined,
workflowRunId: workflowRunIdValue,
workflowTeamId,
workflowUserId: workflowUsageUserId,
needsGfsPublish: workflowHasGfsPublishTargets(spec),
})
await this.createIfNotExists(
() =>
this.deps.coreApi.createNamespacedPod({
namespace: this.deps.config.sandboxNamespace,
body: coordPod,
}),
`Pod "${recipeName}-coordinator"`
)
}
// 7. Set recipe-type annotation
await this.setRecipeTypeAnnotation(recipeName, namespace, classification)
// 8. Reconcile scheduling — DB-backed worker, not a K8s CronJob. The WRC
// only keeps `workflow_schedules` in sync; control-api's worker owns the
// advance-loop and the row-lock advisory coordination.
//
// Canonical shape is `spec.triggers.schedule`. Legacy `spec.scheduling`
// is still accepted for backwards compat — normalize to a single
// SchedulingSpec before calling reconcileScheduling().
const scheduleTrigger = spec.triggers?.schedule
const normalizedScheduling = scheduleTrigger
? {
cron: scheduleTrigger.cron,
timezone: scheduleTrigger.timezone,
concurrencyPolicy: scheduleTrigger.concurrencyPolicy,
suspend: scheduleTrigger.suspend,
}
: spec.scheduling
if (!this.deps.pgPool) {
if (normalizedScheduling) {
log.warn('schedule present but pgPool not wired — skipping schedule reconcile')
}
} else {
// Denormalize the on-demand actor allow-list into the schedule row
// so the control-api worker can gate scheduled fires without a
// cross-store JOIN back to the CRD. An unset / empty list is
// treated as "no restriction" (matches admin endpoint semantics).
const allowedActors =
spec.triggers?.onDemand?.allowedActors && spec.triggers.onDemand.allowedActors.length > 0
? spec.triggers.onDemand.allowedActors
: null
const schedRecipe: SchedulingRecipe = {
metadata: {
name: recipeName,
namespace,
uid: recipeUid,
labels: workflowTeamId ? { [WORKFLOW_TEAM_ID_LABEL]: workflowTeamId } : undefined,
},
spec: {
scheduling: normalizedScheduling,
allowedActors,
runRetention: spec.runRetention ?? null,
},
}
const schedResult = await reconcileScheduling(this.deps.pgPool, schedRecipe)
if (normalizedScheduling || schedResult.action === 'deleted') {
log.info(`Scheduling: ${schedResult.action}`)
}
}
if (awaitsTriggeredRun) {
return withWorkflowConditions({
phase: 'active',
message: `Workflow trigger infrastructure registered (${classification})`,
clearWorkflowExecution: true,
})
}
// Determine phase — narrow from CRD's open string to closed WorkflowPhase union.
// Unknown/stale phase values fall back to "initializing" to prevent propagation.
const rawPhase = currentStatus?.workflowExecution?.phase
const KNOWN_PHASES: readonly string[] = [
'pending',
'initializing',
'running',
'completed',
'failed',
'recovering',
'cancelled',
]
const workflowPhase: WorkflowPhase =
rawPhase && KNOWN_PHASES.includes(rawPhase) ? (rawPhase as WorkflowPhase) : 'initializing'
// Phase transition: when the coordinator reports the workflow as completed/failed,
// the CRD must advance beyond "deploying" so the UI and reconciler reflect final state.
// "completed" → "active" (infrastructure still exists, execution done)
// "failed" → "failed" (surfaces failure in CRD status)
let recipePhase = 'deploying'
if (workflowPhase === 'running' || workflowPhase === 'completed') recipePhase = 'active'
else if (workflowPhase === 'failed' || workflowPhase === 'cancelled') recipePhase = 'failed'
return withWorkflowConditions({
phase: recipePhase,
message: `Workflow infrastructure created (${classification})`,
workflowPhase,
})
} catch (error) {
// Transient K8s API blip (connect ETIMEDOUT, ECONNRESET, 5xx, 429, …)
// during workflow infrastructure reconcile. Do NOT latch the workflow to
// the terminal `failed` phase — that would brick a recoverable run with no
// retry. Preserve the current workflow execution phase/message and signal
// skipStatusPatch so WRC leaves status untouched and requeues. Mirrors the
// outer WRC catch-all (isRetryableInfraError); same classifier. A step that
// throws RetryableReconcileError (e.g. a Service that vanished after a
// create conflict) has declared itself transient and is treated the same.
if (error instanceof RetryableReconcileError || isRetryableInfraError(error)) {
log.warn(`Transient infra error reconciling workflow — will retry, not failing`, {
error: error instanceof Error ? error.message : String(error),
})
const preserved = currentStatus?.workflowExecution
const preservedPhase = preserved?.phase as WorkflowPhase | undefined
const workflowPhase: WorkflowPhase = preservedPhase ?? 'recovering'
// Keep the recipe phase consistent with the preserved execution phase;
// a transient blip must never surface `failed` to the recipe/UI.
let recipePhase = 'deploying'
if (workflowPhase === 'running' || workflowPhase === 'completed') recipePhase = 'active'
return {
phase: recipePhase,
message: preserved?.message ?? String(error),
workflowPhase,
skipStatusPatch: true,
}
}
log.error(`Failed to reconcile workflow`, {
error: error instanceof Error ? error.message : String(error),
})
return withWorkflowConditions({
phase: 'failed',
message: String(error),
workflowPhase: 'failed',
})
}
}
private async ensureWorkflowOutputPvc(
recipeName: string,
claimName: string,
spec: WorkflowRecipeSpec
): Promise<boolean> {
const pvc: k8s.V1PersistentVolumeClaim = {
apiVersion: 'v1',
kind: 'PersistentVolumeClaim',
metadata: {
name: claimName,
namespace: this.deps.config.sandboxNamespace,
labels: {
'clerum.io/recipe': recipeName,
'clerum.io/component': 'workflow-output',
'clerum.io/managed-by': 'wrc',
[WORKFLOW_OUTPUT_CLAIM_LABEL]: workflowOutputLabelValue(claimName),
[WORKFLOW_OUTPUT_SCOPE_LABEL]: workflowOutputLabelValue(recipeName),
},
},
spec: {
accessModes: ['ReadWriteOnce'],
resources: {
requests: {
storage: spec.output?.storageSize ?? DEFAULT_WORKFLOW_OUTPUT_STORAGE_SIZE,
},
},
},
}
try {
await this.deps.coreApi.createNamespacedPersistentVolumeClaim({
namespace: this.deps.config.sandboxNamespace,
body: pvc,
})
this.log.info(`Created PersistentVolumeClaim "${claimName}"`)
return true
} catch (error: unknown) {
if (getErrorCode(error) !== 409) throw error
}
const existing = await this.readWorkflowOutputPvc(claimName)
if (!existing) {
try {
await this.deps.coreApi.createNamespacedPersistentVolumeClaim({
namespace: this.deps.config.sandboxNamespace,
body: pvc,
})
this.log.info(`Created PersistentVolumeClaim "${claimName}" after delete race`)
return true
} catch (error: unknown) {
if (getErrorCode(error) !== 409) throw error
}
const racedExisting = await this.readWorkflowOutputPvc(claimName)
if (!racedExisting || racedExisting.metadata?.deletionTimestamp) {
this.log.warn(`PersistentVolumeClaim "${claimName}" is terminating after create race`)
return false
}
this.assertWorkflowOutputPvcManagedByScope(racedExisting, claimName, recipeName)
this.log.info(`PersistentVolumeClaim "${claimName}" already exists (skip)`)
return true
}
if (existing.metadata?.deletionTimestamp) {
this.log.warn(`PersistentVolumeClaim "${claimName}" is terminating; waiting to recreate`)
const waitResult = await this.waitForWorkflowOutputPvcDeletion(claimName)
if (waitResult === 'ready') {
const readyExisting = await this.readWorkflowOutputPvc(claimName)
if (!readyExisting || readyExisting.metadata?.deletionTimestamp) {
this.log.warn(`PersistentVolumeClaim "${claimName}" changed while waiting for deletion`)
return false
}
this.assertWorkflowOutputPvcManagedByScope(readyExisting, claimName, recipeName)
this.log.info(`PersistentVolumeClaim "${claimName}" already exists (skip)`)
return true
}
if (waitResult === 'terminating') {
this.log.warn(`PersistentVolumeClaim "${claimName}" is still terminating after wait`)
return false
}
try {
await this.deps.coreApi.createNamespacedPersistentVolumeClaim({
namespace: this.deps.config.sandboxNamespace,
body: pvc,
})
this.log.info(`Created PersistentVolumeClaim "${claimName}" after terminating PVC deleted`)
return true
} catch (error: unknown) {
if (getErrorCode(error) !== 409) throw error
}
const racedExisting = await this.readWorkflowOutputPvc(claimName)
if (!racedExisting || racedExisting.metadata?.deletionTimestamp) {
this.log.warn(`PersistentVolumeClaim "${claimName}" is terminating after recreate race`)
return false
}
this.assertWorkflowOutputPvcManagedByScope(racedExisting, claimName, recipeName)
this.log.info(`PersistentVolumeClaim "${claimName}" already exists (skip)`)
return true
}
this.assertWorkflowOutputPvcManagedByScope(existing, claimName, recipeName)
this.log.info(`PersistentVolumeClaim "${claimName}" already exists (skip)`)
return true
}
private assertWorkflowOutputPvcManagedByScope(
pvc: k8s.V1PersistentVolumeClaim,
claimName: string,
recipeName: string
): void {
const labels = pvc.metadata?.labels ?? {}
const expectedClaim = workflowOutputLabelValue(claimName)
const expectedScope = workflowOutputLabelValue(recipeName)
const mismatches: string[] = []
if (labels['clerum.io/managed-by'] !== 'wrc') mismatches.push('managed-by')
if (labels['clerum.io/component'] !== 'workflow-output') mismatches.push('component')
if (labels[WORKFLOW_OUTPUT_CLAIM_LABEL] !== expectedClaim) {
mismatches.push(WORKFLOW_OUTPUT_CLAIM_LABEL)
}
if (labels[WORKFLOW_OUTPUT_SCOPE_LABEL] !== expectedScope) {
mismatches.push(WORKFLOW_OUTPUT_SCOPE_LABEL)
}
if (mismatches.length === 0) return
throw new Error(
`Workflow output PVC "${claimName}" already exists but is not managed by WRC for workflow output scope "${recipeName}" (${mismatches.join(', ')})`
)
}
private async waitForWorkflowOutputPvcDeletion(
claimName: string
): Promise<'deleted' | 'ready' | 'terminating'> {
const deadline = Date.now() + WORKFLOW_OUTPUT_PVC_DELETE_WAIT_MS
while (Date.now() < deadline) {
await delay(WORKFLOW_OUTPUT_PVC_DELETE_POLL_MS)
const pvc = await this.readWorkflowOutputPvc(claimName)
if (!pvc) return 'deleted'
if (!pvc.metadata?.deletionTimestamp) return 'ready'
}
return 'terminating'
}
private async readWorkflowOutputPvc(
claimName: string
): Promise<k8s.V1PersistentVolumeClaim | undefined> {
try {
return await this.deps.coreApi.readNamespacedPersistentVolumeClaim({
name: claimName,
namespace: this.deps.config.sandboxNamespace,
})
} catch (error: unknown) {
if (getErrorCode(error) === 404) return undefined
throw error
}
}
private async isExternalWorkflowOutputPvcReady(
claimName: string,
runtimeScopeRecipeName: string
): Promise<ExternalWorkflowOutputPvcState> {
const pvc = await this.readWorkflowOutputPvc(claimName)
if (!pvc) return 'missing'
if (pvc.metadata?.deletionTimestamp) return 'terminating'
const labels = pvc.metadata?.labels ?? {}
const expectedClaim = workflowOutputLabelValue(claimName)
const expectedScope = workflowOutputLabelValue(runtimeScopeRecipeName)
if (labels['clerum.io/managed-by'] === 'wrc') return 'wrc-managed-conflict'
if (
labels[WORKFLOW_OUTPUT_EXTERNAL_CLAIM_LABEL] !== 'true' ||
labels[WORKFLOW_OUTPUT_CLAIM_LABEL] !== expectedClaim ||
labels[WORKFLOW_OUTPUT_SCOPE_LABEL] !== expectedScope
) {
return 'unauthorized'
}
return 'ready'
}
private async deleteRuntimeComponentIfExists(
recipeName: string,
component: WorkflowRuntimeComponent
): Promise<void> {
const podSuffixByComponent: Record<WorkflowRuntimeComponent, string> = {
'workflow-coordinator': 'coordinator',
'workflow-mcp-host': 'mcp-host',
'workflow-artifact-reader': 'artifact-reader',
'workflow-snippet-runner': 'snippet-runner',
}
await deletePodIfExists(
this.deps.coreApi,
`${recipeName}-${podSuffixByComponent[component]}`,
this.deps.config.sandboxNamespace
)
}
/**
* Tear down a run's COMPUTE pods once it reaches a terminal phase, while
* PRESERVING the artifact-reader pod, the workflow-output PVC, and the
* run-scoped WorkflowRecipe CR. The artifact-reader keeps serving `/output`
* (PVC-backed) downloads until the archive-cron deletes the CR at its TTL —
* so dropping mcp-host/coordinator/snippet-runner here frees CPU/RAM without
* losing any artifacts.
*
* DELETE-first and 404-safe (`deletePodIfExists`). This path is outside B3
* (E.4(e), §7.16): do not add a GET just to skip the DELETE. The archive
* TTL still owns leftover objects. Deliberately EXCLUDES
* `workflow-artifact-reader`.
*
* The component list is an explicit literal (not derived from
* `runtime.cleanup.*`) on purpose: the "artifact-reader is preserved"
* invariant must be obvious and asserted in one place. Folding it into the
* `deriveWorkflowRuntimePlan` model would couple terminal teardown to the
* crash-recovery cleanup lists for a larger, riskier change with no behaviour
* gain — the exclusion is pinned by a dedicated unit test instead.
*/
async teardownComputePodsForTerminalRun(recipeName: string): Promise<void> {
const computeComponents: WorkflowRuntimeComponent[] = [
'workflow-coordinator',
'workflow-mcp-host',
'workflow-snippet-runner',
]
for (const component of computeComponents) {
await this.deleteRuntimeComponentIfExists(recipeName, component)
}
}
async ensureMcpHostRuntimeCredentials(
recipeNamespace: string,
recipeName: string,
spec: WorkflowRecipeSpec,
runtimeScopeRecipeName = recipeName,
recipeUid?: string
): Promise<void> {
const runtime = deriveWorkflowRuntimePlan(spec, {
recipeName,
runtimeScopeRecipeName,
pluginWorkloadSdkEnabled: this.deps.config.pluginWorkloadSdkEnabled,
})
if (!runtime.mcpHost.required) return
const codexView = await this.refreshCodexSnapshot()
await this.ensureMcpHostSecrets(
recipeNamespace,
recipeName,
runtimeScopeRecipeName,
spec,
recipeUid,
this.codexVerdictFor(spec, recipeUid ?? '', recipeName, runtimeScopeRecipeName, codexView)
)
}
/**
* Reapply the run-lane NetworkPolicies outside reconcile(). The running and
* active short-circuits return before reconcile(), so a policy a previous
* pass left pending (terminating or contended) would otherwise stay as it is
* until the run ends. Builds the desired set with a verdict computed now and
* applies it, but deletes nothing: the run's pods are live and may still use
* any lane the spec wanted when they were created, so a revoke here cuts
* their traffic. That includes the legacy mcp-servers internet policy, which
* this path keeps. The summary is apply-only, so this path
* is the `apply-retry` writer of Table A below: it observes the apply and
* carries the published prune and legacy facts.
*
* What each path does to a run-lane policy (verdict = Codex/Grok
* eligibility; it governs only the proxy lanes, every other catalog lane
* follows the spec alone; legacy = `<recipe>-mcp-servers-egress-internet`):
*
* | path | proxy required | proxy uncertain | proxy retired | other lane, not desired | legacy |
* | reconcile() pass | applied | kept if live | pruned | pruned | deleted |
* | mid-run retry (this) | applied | kept if live | kept | kept | kept |
* | legacy retry | untouched | untouched | untouched | untouched | deleted, backoff |
* | terminal teardown | kept | kept | kept | kept | kept |
* | finalizer | deleted | deleted | deleted | deleted | deleted |
*
* Table A, what each writer publishes (`networkPolicyMarkerConditions`).
* The facts are A (apply: `converged | pending`), P (prune: `converged |
* pending | unevaluated`, `unevaluated` being a failed LIST) and L (legacy:
* `removed | pending`), read back from the condition reasons alone:
*
* | A | P | `WorkflowNetworkPoliciesConverged` | requeue |
* | converged | converged | absent | none |
* | pending | converged | False/`RetryPending` | the retry's |
* | converged | pending | False/`PrunePending` | none |
* | converged | unevaluated | False/`PruneUnevaluated` | none |
* | pending | pending | False/`RetryAndPrunePending` | as RetryPending |
* | pending | unevaluated | False/`RetryAndPruneUnevaluated` | as RetryPending |
*
* | L | `WorkflowLegacyNetworkPolicyRemoved` |
* | removed | absent |
* | pending | False/`DeletePending` |
*
* | writer | A | P | L | clears |
* | `reconcile` (run lane, eager, SDK with a summary) | observed | observed | observed | what converged |
* | SDK with a runtime and no summary | not written | not written | not written | nothing |
* | `apply-retry` (this, in-progress and active) | observed | carried | carried | A only |
* | `legacy-retry` (WRC, before the short-circuits) | carried | carried | observed, backoff | L only |
* | `terminal` | forced converged | carried | carried | A |
* | `unmanaged` (kill switch, SDK lane without a runtime, capability removal) | converged | converged | removed | everything |
* | finalizer | the object is deleted | | | |
*
* The legacy retry runs in the WRC before limit validation, every
* short-circuit and the early token reap, only while `DeletePending` is
* published. It, the reconcile() pass and a full SDK cleanup share one
* backoff per recipe: after a DELETE that did
* not land, the next one waits `min(prev * 2 || 60 s, 1 h)`, and inside
* that window the delete answers `pending` without a request. A `removed`
* answer clears L with a conditions-only patch; `pending` writes nothing.
* The watcher's legacy-only callback requeues after the rest of the window;
* it does not force a full reconcile. A cleanup that must remove the legacy
* policy but observes `pending` fails before `unmanaged` and requeues the
* full cleanup at the same window. Finalizer cleanup is the last chance:
* it sends through the same ledger even inside a window, then records and
* forgets state only after the complete cleanup succeeds.
*
* A carrying writer keeps a marker whose reason it did not write as it is
* and warns; `reconcile` overwrites it. `unmanaged` is written only after
* `cleanupPluginWorkloadSdk` without `preserveWorkflowRuntime` succeeded:
* its label sweeps exclude the legacy name, and the shared live ledger
* deletes that name once; a failure or backoff fails the pass before the
* writer runs. A pending or
* unevaluated prune is not retried outside reconcile(): only reconcile()
* can compute the desired set, and the finalizer's label sweep removes
* every run-lane leftover. On the finalizer, a non-404 DELETE is logged by
* `safeDelete` and not rethrown, so the finalizer completes.
*
* Historical `af0152924` markers are read only from their reason, except
* the one exact `RetryPending` combined message defined beside the decoder.
* Its ambiguous `PrunePending` cannot be safely promoted to L: current
* ordinary prune failures use the same reason, so they require a real
* reconcile() observation before the legacy retry can run.
*
* A desired lane is applied on both apply paths; an uncertain verdict never
* creates a proxy. Terminal teardown deletes a run-scoped run's compute pods
* and its cross-namespace transport workloads, and, for every terminal
* recipe, revokes `<recipe>-coordinator-to-gfs` unless the run completed and
* that policy may open; that policy is outside the run-lane catalog. It
* deletes no run-lane policy. The finalizer deletes a fixed list of run-lane
* names and sweeps by the recipe labels in the sandbox and mcp-server
* namespaces; the legacy policy predates those labels and is deleted by
* name in the mcp-server namespace.
*
* reconcile(), and so its prune, also runs while the run's pods are live:
* the WRC falls through to it for a run in `initializing` or `recovering`,
* and for any in-progress run while transport network readiness is pending.
*/
async retryRunLaneNetworkPolicies(
recipeName: string,
recipeUid: string,
spec: WorkflowRecipeSpec,
runtimeScopeRecipeName = recipeName,
workflowRunId?: string
): Promise<WorkflowNetworkPolicyApplySummary> {
const runId = workflowRunId?.trim()
const runtime = deriveWorkflowRuntimePlan(spec, {
recipeName,
runtimeScopeRecipeName,
workflowRunId: runId,
pluginWorkloadSdkEnabled: this.deps.config.pluginWorkloadSdkEnabled,
})
const awaitsTriggeredRun = runtime.mcpHost.required && !runId
const codexView = await this.refreshCodexSnapshot()
const codexVerdict = this.codexVerdictFor(
spec,
recipeUid,
recipeName,
runtimeScopeRecipeName,
codexView
)
const { policies } = await this.buildWorkflowNetworkPoliciesForSpec(
recipeName,
recipeUid,
spec,
runtime,
awaitsTriggeredRun,
codexVerdict.projection,
false,
codexVerdict.grokProjection
)
return this.applyNetworkPolicyList(policies)
}
async ensureCoordinatorRuntimeCredentials(
recipeNamespace: string,
recipeName: string,
spec: WorkflowRecipeSpec,
runtimeScopeRecipeName = recipeName
): Promise<void> {
const runtime = deriveWorkflowRuntimePlan(spec, {
recipeName,
runtimeScopeRecipeName,
pluginWorkloadSdkEnabled: this.deps.config.pluginWorkloadSdkEnabled,
})
const coordinatorGfsScopes = deriveRecipeHostGfsScopes(spec)
const coordinatorGfsSubject = `host:3rd:${recipeNamespace}/${runtimeScopeRecipeName}`
const coordinatorGfsBinding = workflowHasGfsPublishTargets(spec)
? await mintRecipeHostGfsToken(recipeNamespace, runtimeScopeRecipeName, {
scopes: coordinatorGfsScopes,
})
: undefined
await this.refreshCoordinatorTokenIfExpiring(recipeName, recipeNamespace, {
...runtime.tokens.coordinator,
gfsToken: coordinatorGfsBinding ? coordinatorGfsBinding.token : undefined,
gfsSubject: coordinatorGfsSubject,
gfsScopes: coordinatorGfsScopes,
})
}
/**
* Reapply the runtime HTTP egress policies outside reconcile(). Returns what
* the apply could not converge: this path has no status to publish it on, so
* the caller logs it.
*/
async refreshRuntimeHttpEgressNetworkPolicies(
_recipeNamespace: string,
recipeName: string,
recipeUid: string,
spec: WorkflowRecipeSpec,
_runtimeScopeRecipeName = recipeName
): Promise<WorkflowNetworkPolicyApplySummary> {
if (!this.needsRuntimeHttpEgressRefresh(spec)) return { conflicts: [], retryPending: false }
let policies: k8s.V1NetworkPolicy[]
try {
policies = await this.buildRuntimeHttpEgressNetworkPoliciesForSpec(
recipeName,
recipeUid,
spec
)
} catch (error) {
await this.pruneExpiredRuntimeHttpEgressNetworkPolicyOverlaps(recipeName, recipeUid, spec)
throw error
}
return this.applyNetworkPolicyList(policies)
}
private async buildRuntimeHttpEgressNetworkPoliciesForSpec(
recipeName: string,
recipeUid: string,
spec: WorkflowRecipeSpec
): Promise<k8s.V1NetworkPolicy[]> {
const runtimeHttpEgressPolicyNames = this.runtimeHttpEgressPolicyNames(recipeName, spec)
if (runtimeHttpEgressPolicyNames.length === 0) return []
const runtimeHttpEgressState =
runtimeHttpEgressClass(spec) === 'exact-host'
? await this.resolveRuntimeHttpEgressPolicyState(
this.deps.config.sandboxNamespace,
runtimeHttpEgressPolicyNames,
spec.runtimeEgress?.http?.allowedHosts ?? []
)
: { currentCidrs: [], effectiveCidrs: [], annotations: {} }
return this.buildRuntimeHttpEgressNetworkPoliciesForState(
recipeName,
recipeUid,
spec,
runtimeHttpEgressPolicyNames,
runtimeHttpEgressState
)
}
private buildRuntimeHttpEgressNetworkPoliciesForState(
recipeName: string,
recipeUid: string,
spec: WorkflowRecipeSpec,
runtimeHttpEgressPolicyNames: string[],
runtimeHttpEgressState: RuntimeHttpEgressPolicyState
): k8s.V1NetworkPolicy[] {
const hasCustomCoordinatorHttpEgress = hasCustomCoordinatorPublicHttpEgress(spec)
const hasSnippetHttpEgress = hasSnippetPublicHttpEgress(spec)
const egressClass = runtimeHttpEgressClass(spec)
const workloadIdsWithTransport = new Set(
(spec.workloads ?? []).filter(w => w.transport != null).map(w => w.id)
)
const workloadMcpServerLabels = this.resolveTransportWorkloadMcpServerLabels(
recipeName,
recipeUid,
spec
)
const snippetMcpServerFullNames = hasSnippetHttpEgress
? this.resolveSnippetMcpServerFullNames(
recipeName,
recipeUid,
spec,
workloadIdsWithTransport,
workloadMcpServerLabels
)
: []
const runtimeHttpPolicyNameSet = new Set(runtimeHttpEgressPolicyNames)
const policies = buildWorkflowNetworkPolicies(
{
recipeName,
sandboxNamespace: this.deps.config.sandboxNamespace,
controlPlaneNamespace: 'control-plane',
mcpServerNamespace: this.deps.config.mcpServerNamespace,
wrcPort: 8082,
mcpHostPort: 8080,
artifactReaderPort: 8080,
snippetRunnerPort: 8095,
includeMcpHost: false,
includeCoordinatorGfs: false,
includeArtifactReader: false,
includeSnippetRunner: hasSnippetHttpEgress,
coordinatorPublicHttpEgress: hasCustomCoordinatorHttpEgress,
coordinatorPublicHttpEgressClass: egressClass,
coordinatorPublicHttpEgressCidrs: hasCustomCoordinatorHttpEgress
? egressClass === 'exact-host'
? runtimeHttpEgressState.effectiveCidrs
: undefined
: undefined,
coordinatorWorkloadEgress: hasCustomCoordinatorHttpEgress
? this.resolveCustomCoordinatorWorkloadNetworkBindings(recipeName, recipeUid, spec)
: undefined,
snippetRunnerPublicHttpEgress: hasSnippetHttpEgress,
snippetRunnerPublicHttpEgressClass: egressClass,
snippetRunnerPublicHttpEgressCidrs: hasSnippetHttpEgress
? egressClass === 'exact-host'
? runtimeHttpEgressState.effectiveCidrs
: undefined
: undefined,
snippetRunnerWorkloadEgress: hasSnippetHttpEgress
? this.resolveSnippetWorkloadNetworkBindings(recipeName, recipeUid, spec)
: undefined,
},
[],
snippetMcpServerFullNames
).filter(policy => runtimeHttpPolicyNameSet.has(policy.metadata?.name ?? ''))
this.annotateRuntimeHttpEgressPolicies(
policies,
runtimeHttpPolicyNameSet,
runtimeHttpEgressState.annotations
)
return policies
}
private async pruneExpiredRuntimeHttpEgressNetworkPolicyOverlaps(
recipeName: string,
recipeUid: string,
spec: WorkflowRecipeSpec
): Promise<void> {
const runtimeHttpEgressPolicyNames = this.runtimeHttpEgressPolicyNames(recipeName, spec)
if (runtimeHttpEgressPolicyNames.length === 0) return
if (runtimeHttpEgressClass(spec) === 'public-web') return
const state = await this.readExistingRuntimeHttpEgressPolicyState(
this.deps.config.sandboxNamespace,
runtimeHttpEgressPolicyNames
)
if (!state) return
const policies = this.buildRuntimeHttpEgressNetworkPoliciesForState(
recipeName,
recipeUid,
spec,
runtimeHttpEgressPolicyNames,
state
)
const pruned = await this.applyNetworkPolicyList(policies)
// The refresh rethrows the DNS error after this prune, so this line is the
// only record that a policy still carries the CIDRs the prune meant to drop.
// A conflict is warn-logged by applyNetworkPolicy and not published here:
// this path runs outside reconcile() and writes no status.
if (pruned.retryPending) {
this.log.error(
'DNS-failure prune left a NetworkPolicy pending a retry; a later refresh retries it',
{ recipeName }
)
}
}
private legacyMcpServersInternetEgressKey(recipeName: string, recipeUid?: string): string {
return recipeUid && recipeUid.length > 0 ? recipeUid : recipeName
}
private forgetLegacyMcpServersInternetEgress(recipeName: string, recipeUid?: string): void {
this.prunedLegacyMcpServersInternetEgress.delete(recipeName)
this.legacyMcpServersInternetEgressBackoff.delete(recipeName)
if (recipeUid) {
this.prunedLegacyMcpServersInternetEgress.delete(recipeUid)
this.legacyMcpServersInternetEgressBackoff.delete(recipeUid)
}
}
private async cleanupLegacyMcpServersInternetEgress(
recipeName: string,
recipeUid?: string,
lastChance = false
): Promise<void> {
const fact = await this.pruneLegacyMcpServersInternetEgressPolicy(
recipeName,
recipeUid,
lastChance
)
if (fact === 'pending') {
throw new LegacyNetworkPolicyDeletePendingError(
recipeName,
Math.max(1, this.legacyMcpServersInternetEgressRetryDelayMs(recipeName, recipeUid ?? ''))
)
}
}
/**
* R4-L4: the legacy delete the WRC retries before its short-circuits while
* `DeletePending` is published (the `legacy-retry` writer). The same delete
* and backoff as the reconcile() pass: inside the backoff window it answers
* `pending` without a request.
*/
async retryLegacyMcpServersInternetEgressDelete(
recipeName: string,
recipeUid: string
): Promise<NetworkPolicyLegacyFact> {
return this.pruneLegacyMcpServersInternetEgressPolicy(recipeName, recipeUid)
}
/**
* R4-L4: how long until the next legacy DELETE may be sent for this recipe;
* 0 when no backoff is running. The WRC requeues a pass that leaves
* `DeletePending` published after this delay.
*/
legacyMcpServersInternetEgressRetryDelayMs(recipeName: string, recipeUid: string): number {
const backoff = this.legacyMcpServersInternetEgressBackoff.get(
this.legacyMcpServersInternetEgressKey(recipeName, recipeUid)
)
if (backoff === undefined) return 0
return Math.max(0, backoff.notBeforeMs - Date.now())
}
private async pruneLegacyMcpServersInternetEgressPolicy(
recipeName: string,
recipeUid?: string,
lastChance = false
): Promise<NetworkPolicyLegacyFact> {
const key = this.legacyMcpServersInternetEgressKey(recipeName, recipeUid)
if (this.prunedLegacyMcpServersInternetEgress.has(key)) {
this.log.debug('Skipping legacy mcp-servers internet NP delete; already observed gone', {
recipe: recipeName,
})
return 'removed'
}
const backoff = this.legacyMcpServersInternetEgressBackoff.get(key)
if (backoff !== undefined && !lastChance && Date.now() < backoff.notBeforeMs) {
this.log.debug('Skipping legacy mcp-servers internet NP delete; inside its backoff window', {
recipe: recipeName,
retryInMs: backoff.notBeforeMs - Date.now(),
})
return 'pending'
}
const outcome = await observeNamespacedDelete(() =>
this.deps.networkingApi.deleteNamespacedNetworkPolicy({
name: buildLegacyMcpServersInternetEgressPolicyName(recipeName),
namespace: this.deps.config.mcpServerNamespace,
})
)
if (outcome.kind === 'failed') {
const delayMs = Math.min(
backoff === undefined ? LEGACY_NETWORK_POLICY_DELETE_BACKOFF_BASE_MS : backoff.delayMs * 2,
LEGACY_NETWORK_POLICY_DELETE_BACKOFF_MAX_MS
)
this.legacyMcpServersInternetEgressBackoff.set(key, {
delayMs,
notBeforeMs: Date.now() + delayMs,
})
this.log.error('Legacy mcp-servers internet NP delete failed', {
recipe: recipeName,
...deleteOutcomeFields(outcome),
retryInMs: delayMs,
err: outcome.error,
})
return 'pending'
}
this.legacyMcpServersInternetEgressBackoff.delete(key)
this.log.info('Legacy mcp-servers internet NP delete', {
recipe: recipeName,
...deleteOutcomeFields(outcome),
})
if (shouldRecordDelete(outcome)) {
this.prunedLegacyMcpServersInternetEgress.add(key)
}
return 'removed'
}
private needsRuntimeHttpEgressRefresh(spec: WorkflowRecipeSpec): boolean {
return hasCustomCoordinatorPublicHttpEgress(spec) || hasSnippetPublicHttpEgress(spec)
}
private runtimeHttpEgressPolicyNames(recipeName: string, spec: WorkflowRecipeSpec): string[] {
const names: string[] = []
if (hasCustomCoordinatorPublicHttpEgress(spec)) names.push(`${recipeName}-coord-to-wrc`)
if (hasSnippetPublicHttpEgress(spec)) names.push(`${recipeName}-snippet-runner-egress`)
return names
}
private async buildWorkflowNetworkPoliciesForSpec(
recipeName: string,
recipeUid: string,
spec: WorkflowRecipeSpec,
runtime: WorkflowRuntimePlan,
awaitsTriggeredRun: boolean,
/**
* The pass's Codex projection. Recomputing it here read the
* live allowlist view after this method's own awaits, so a NetworkPolicy
* decision could rest on a different snapshot than the binding did.
*/
codexProjection: CodexExecutionProjection,
eagerSdkMcpHost = false,
grokProjection: CodexExecutionProjection & { requiresGrokProxyEgress?: boolean } = {
...codexProjection,
derivedScopes: [],
requiresCodexProxyEgress: false,
requiresGrokProxyEgress: false,
eligibility: 'ineligible',
reason: 'static_only',
}
): Promise<{ policies: k8s.V1NetworkPolicy[]; catalog: Set<string> }> {
const runtimeHttpEgressPolicyNames = this.runtimeHttpEgressPolicyNames(recipeName, spec)
const runtimeHttpEgressState =
runtimeHttpEgressPolicyNames.length > 0 && runtimeHttpEgressClass(spec) === 'exact-host'
? await this.resolveRuntimeHttpEgressPolicyState(
this.deps.config.sandboxNamespace,
runtimeHttpEgressPolicyNames,
spec.runtimeEgress?.http?.allowedHosts ?? []
)
: undefined
const hasCustomCoordinatorHttpEgress = hasCustomCoordinatorPublicHttpEgress(spec)
const hasSnippetHttpEgress = hasSnippetPublicHttpEgress(spec)
const egressClass = runtimeHttpEgressClass(spec)
// "Is the mcp-host allowed to be live in this reconcile?" — true on the
// run path (not awaiting) and on the eager Plugin Workload SDK path, which
// deploys the mcp-host (and its NetworkPolicy lanes) before any run is
// triggered. The artifact-reader/snippet-runner stay run-only.
const mcpHostLaneLive = !awaitsTriggeredRun || eagerSdkMcpHost
const includeMcpHost = runtime.network.includeMcpHost && mcpHostLaneLive
const npConfig: NetworkPolicyConfig = {
recipeName,
sandboxNamespace: this.deps.config.sandboxNamespace,
controlPlaneNamespace: 'control-plane',
mcpServerNamespace: this.deps.config.mcpServerNamespace,
wrcPort: 8082,
mcpHostPort: 8080,
artifactReaderPort: 8080,
snippetRunnerPort: 8095,
includeMcpHost,
includeCodexProxyEgress: codexProjection.requiresCodexProxyEgress && includeMcpHost,
includeGrokProxyEgress: grokProjection.requiresGrokProxyEgress === true && includeMcpHost,
// A stepless eager SDK host has no coordinator pod. Keep the mcp-host
// control/egress lanes, but do not manufacture coordinator policies
// whose selectors can never match a real workload.
includeCoordinator: !eagerSdkMcpHost,
// The outer WorkflowRecipe reconciler owns this declarative lane so it can
// revoke before phase short-circuits while gating creation on provenance.
includeCoordinatorGfs: false,
includeArtifactReader: runtime.network.includeArtifactReader && !awaitsTriggeredRun,
includeSnippetRunner: runtime.network.includeSnippetRunner && !awaitsTriggeredRun,
// Plugin Workload SDK (plan §6.1): open the SDK lane when the recipe
// declares the capability, the feature flag is on, and the recipe
// mcp-host is deployed — including the eager (pre-run) SDK path.
pluginWorkloadSdkSandboxAccess:
this.deps.config.pluginWorkloadSdkEnabled === true &&
spec.pluginWorkloadSdk != null &&
runtime.network.includeMcpHost &&
mcpHostLaneLive,
coordinatorPublicHttpEgress: hasCustomCoordinatorHttpEgress,
coordinatorPublicHttpEgressClass: egressClass,
coordinatorPublicHttpEgressCidrs: hasCustomCoordinatorHttpEgress
? egressClass === 'exact-host'
? runtimeHttpEgressState?.effectiveCidrs
: undefined
: undefined,
coordinatorWorkloadEgress: this.resolveCustomCoordinatorWorkloadNetworkBindings(
recipeName,
recipeUid,
spec
),
snippetRunnerPublicHttpEgress: hasSnippetHttpEgress,
snippetRunnerPublicHttpEgressClass: egressClass,
snippetRunnerPublicHttpEgressCidrs: hasSnippetHttpEgress
? egressClass === 'exact-host'
? runtimeHttpEgressState?.effectiveCidrs
: undefined
: undefined,
snippetRunnerWorkloadEgress: this.resolveSnippetWorkloadNetworkBindings(
recipeName,
recipeUid,
spec
),
}
const workloadIdsWithTransport = new Set(
(spec.workloads ?? []).filter(w => w.transport != null).map(w => w.id)
)
const workloadMcpServerLabels = this.resolveTransportWorkloadMcpServerLabels(
recipeName,
recipeUid,
spec
)
const mcpServerFullNames = resolveMcpServerFullNames(
recipeName,
spec.mcpServers,
workloadIdsWithTransport,
workloadMcpServerLabels
)
const snippetMcpServerFullNames = this.resolveSnippetMcpServerFullNames(
recipeName,
recipeUid,
spec,
workloadIdsWithTransport,
workloadMcpServerLabels
)
const policies = buildWorkflowNetworkPolicies(
npConfig,
mcpServerFullNames,
snippetMcpServerFullNames
)
if (runtimeHttpEgressState) {
this.annotateRuntimeHttpEgressPolicies(
policies,
new Set(runtimeHttpEgressPolicyNames),
runtimeHttpEgressState.annotations
)
}
const catalog = buildRunLaneNetworkPolicyCatalog(
npConfig,
mcpServerFullNames,
snippetMcpServerFullNames
)
return { policies, catalog }
}
/**
* Build, apply, and legacy-prune the workflow NetworkPolicies in one unit.
* Shared by the triggered-run path and the eager SDK path (the latter passes
* eagerSdkMcpHost=true to open the mcp-host + SDK lanes before any run). The
* legacy prune runs on both paths so the eager mcp-host doesn't keep a stale
* internet-egress policy the run path would have removed.
*/
private async applyWorkflowNetworkPolicies(
recipeName: string,
recipeUid: string,
spec: WorkflowRecipeSpec,
runtime: WorkflowRuntimePlan,
awaitsTriggeredRun: boolean,
codexProjection: CodexExecutionProjection,
eagerSdkMcpHost = false,
grokProjection?: CodexExecutionProjection & { requiresGrokProxyEgress?: boolean }
): Promise<NetworkPolicyPassSummary> {
const { policies, catalog } = await this.buildWorkflowNetworkPoliciesForSpec(
recipeName,
recipeUid,
spec,
runtime,
awaitsTriggeredRun,
codexProjection,
eagerSdkMcpHost,
grokProjection
)
return this.applyAndPruneRunLaneNetworkPolicies(recipeName, recipeUid, policies, catalog, {
codex: codexProjection.eligibility === 'uncertain',
grok: grokProjection?.eligibility === 'uncertain',
})
}
private async applyAndPruneRunLaneNetworkPolicies(
recipeName: string,
recipeUid: string,
policies: k8s.V1NetworkPolicy[],
catalog: Set<string>,
keepProxies: RunLaneProxyKeepReasons
): Promise<NetworkPolicyPassSummary> {
const applied = await this.applyNetworkPolicyList(policies)
const policyNames = new Set(policies.map(policy => policy.metadata?.name))
const prune = await this.pruneUndesiredRunLaneNetworkPolicies(
recipeName,
policyNames,
catalog,
keepProxies
)
const legacy = await this.pruneLegacyMcpServersInternetEgressPolicy(recipeName, recipeUid)
return { ...applied, prune, legacy }
}
/**
* Drop run-lane NetworkPolicies that exist and are no longer desired.
* Two-term selector only (`managed-by=wrc`): a single-term recipe selector
* would also return the recipes-lane policies in the same namespace.
* Universe is the factory catalog (GFS never in it), not LIST \ desired.
* A Codex/Grok proxy whose verdict is uncertain survives even when it is
* absent from this pass's desired set. A listed policy with an
* ownerReference is skipped, matching the apply's ownership veto. Only
* reconcile() prunes; the mid-run retry never calls this (see
* retryRunLaneNetworkPolicies for the per-path table). A failed LIST
* evaluated nothing and answers `unevaluated`; a DELETE that did not land
* answers `pending`.
*/
private async pruneUndesiredRunLaneNetworkPolicies(
recipeName: string,
desiredNames: Set<string | undefined>,
catalog: Set<string>,
keepProxies: RunLaneProxyKeepReasons
): Promise<NetworkPolicyPruneFact> {
const namespace = this.deps.config.sandboxNamespace
const labelSelector = `clerum.io/recipe=${recipeName},clerum.io/managed-by=wrc`
let list: { items?: k8s.V1NetworkPolicy[] }
try {
list = await this.deps.networkingApi.listNamespacedNetworkPolicy({
namespace,
labelSelector,
})
} catch (error: unknown) {
this.log.error('Run-lane NP prune LIST failed; apply outcome is kept', {
recipe: recipeName,
err: error,
})
return 'unevaluated'
}
const codexProxyPolicyName = `${recipeName}-mcp-host-to-codex-proxy`
const grokProxyPolicyName = `${recipeName}-mcp-host-to-grok-proxy`
let deletePending = false
for (const policy of list.items ?? []) {
const name = policy.metadata?.name
if (!name || desiredNames.has(name)) continue
const keep =
name === codexProxyPolicyName
? keepProxies.codex
: name === grokProxyPolicyName
? keepProxies.grok
: false
if (keep) {
this.log.debug('Skipping run-lane NP prune', {
recipe: recipeName,
policy: name,
reason: 'uncertain',
})
continue
}
if (!catalog.has(name)) {
this.log.debug('Skipping run-lane NP prune', {
recipe: recipeName,
policy: name,
reason: 'not-in-catalog',
})
continue
}
// Same ownership veto the run-lane apply uses: a policy under our name
// that carries an ownerReference belongs to someone else, and the apply
// already refuses to replace it.
const ownership = classifyOwnerlessNetworkPolicyOwnership(policy)
if (ownership.kind !== 'owned') {
this.log.warn('Skipping run-lane NP prune', {
recipe: recipeName,
policy: name,
reason: ownership.kind === 'conflict' ? ownership.reason : ownership.kind,
})
continue
}
const outcome = await observeNamespacedDelete(() =>
this.deps.networkingApi.deleteNamespacedNetworkPolicy({ name, namespace })
)
if (outcome.kind === 'failed') {
this.log.error('Run-lane NP prune delete failed', {
recipe: recipeName,
policy: name,
...deleteOutcomeFields(outcome),
err: outcome.error,
})
deletePending = true
}
}
return deletePending ? 'pending' : 'converged'
}
/**
* Apply each policy and collect what the pass could not converge. A conflict
* or a pending retry leaves that one policy as it is and the loop moves on;
* each caller decides how to surface both, so neither fails the pass.
*/
private async applyNetworkPolicyList(
policies: k8s.V1NetworkPolicy[]
): Promise<WorkflowNetworkPolicyApplySummary> {
const summary: WorkflowNetworkPolicyApplySummary = { conflicts: [], retryPending: false }
for (const policy of policies) {
const applied = await this.applyNetworkPolicy(policy)
if (applied.action === 'conflict') {
summary.conflicts.push({ policy: applied.policy, reason: applied.reason })
} else if (applied.action === 'retry') {
summary.retryPending = true
}
}
return summary
}
private async resolveRuntimeHttpEgressPolicyState(
namespace: string,
policyNames: string[],
hosts: string[]
): Promise<RuntimeHttpEgressPolicyState> {
const now = new Date()
const currentCidrs = normalizeCidrs(
await (this.deps.resolveRuntimeHttpEgressCidrs ?? resolveRuntimeHttpEgressCidrs)(hosts)
)
const currentSerialized = serializeCidrs(currentCidrs)
const previousExpiries = new Map<string, Date>()
const liveAnnotationSets: Array<Record<string, string> | undefined> = []
// Coordinator and snippet policies share one WorkflowRecipe runtimeEgress contract.
// Merge valid annotations from all matching policies so a partial refresh or restart
// preserves the widest still-active overlap window before writing identical state back.
// Taking the latest expiration avoids one policy shortening another policy's DNS rollover.
for (const policyName of policyNames) {
const live = await this.readRuntimeHttpEgressPolicy(namespace, policyName)
// The apply leaves a foreign or terminating policy unwritten, so its
// annotations stop following DNS. Merging them reopened the overlap for
// its stale CIDR and re-stamped resolved-at on every pass; counting it as
// missing would re-stamp too. It is left out of the state entirely.
if (live.kind === 'foreign' || live.kind === 'terminating') continue
const annotations = live.kind === 'live' ? live.annotations : undefined
liveAnnotationSets.push(annotations)
if (!annotations) continue
const existingCurrent = parseCidrsAnnotation(
annotations[RUNTIME_HTTP_EGRESS_CURRENT_CIDRS_ANNOTATION]
)
if (existingCurrent.length > 0 && serializeCidrs(existingCurrent) !== currentSerialized) {
const rolloverExpiresAt = new Date(now.getTime() + this.runtimeHttpEgressOverlapMs)
for (const cidr of existingCurrent.filter(isAllowedRuntimeHttpEgressCidr)) {
addRuntimeHttpEgressPreviousExpiry(previousExpiries, cidr, rolloverExpiresAt)
}
}
const existingPreviousExpiresAt = parseFutureDate(
annotations[RUNTIME_HTTP_EGRESS_PREVIOUS_EXPIRES_AT_ANNOTATION],
now,
this.maxTrustedRuntimeHttpEgressPreviousExpiryMs
)
const hasPreviousCidrExpiriesAnnotation =
annotations[RUNTIME_HTTP_EGRESS_PREVIOUS_CIDR_EXPIRIES_ANNOTATION] !== undefined
const existingPreviousExpiries = parseTrustedRuntimeHttpEgressCidrExpiriesAnnotation(
annotations[RUNTIME_HTTP_EGRESS_PREVIOUS_CIDR_EXPIRIES_ANNOTATION],
now,
this.maxTrustedRuntimeHttpEgressPreviousExpiryMs
)
for (const cidr of parseTrustedRuntimeHttpEgressCidrsAnnotation(
annotations[RUNTIME_HTTP_EGRESS_PREVIOUS_CIDRS_ANNOTATION]
)) {
const expiresAt = hasPreviousCidrExpiriesAnnotation
? existingPreviousExpiries.get(cidr)
: existingPreviousExpiresAt
if (expiresAt) {
addRuntimeHttpEgressPreviousExpiry(previousExpiries, cidr, expiresAt)
}
}
}
const activePreviousEntries = [...previousExpiries.entries()].filter(
([, expiresAt]) => expiresAt.getTime() > now.getTime()
)
const activePreviousCidrs = normalizeCidrs(activePreviousEntries.map(([cidr]) => cidr))
const effectiveCidrs = normalizeCidrs([...currentCidrs, ...activePreviousCidrs])
if (effectiveCidrs.length > MAX_RUNTIME_HTTP_EGRESS_POLICY_CIDRS) {
throw new Error(
`runtime HTTP egress resolved ${effectiveCidrs.length} CIDRs, exceeding max ${MAX_RUNTIME_HTTP_EGRESS_POLICY_CIDRS}`
)
}
const annotations: Record<string, string> = {
[RUNTIME_HTTP_EGRESS_CURRENT_CIDRS_ANNOTATION]: currentSerialized,
}
if (activePreviousCidrs.length > 0) {
const activePreviousExpiries = new Map(activePreviousEntries)
const previousExpiresAt = [...activePreviousExpiries.values()].reduce<Date | undefined>(
latestDate,
undefined
)
annotations[RUNTIME_HTTP_EGRESS_PREVIOUS_CIDRS_ANNOTATION] =
serializeCidrs(activePreviousCidrs)
annotations[RUNTIME_HTTP_EGRESS_PREVIOUS_EXPIRES_AT_ANNOTATION] =
previousExpiresAt!.toISOString()
annotations[RUNTIME_HTTP_EGRESS_PREVIOUS_CIDR_EXPIRIES_ANNOTATION] =
serializeRuntimeHttpEgressCidrExpiries(activePreviousExpiries)
}
// Observability only; policy decisions use current/previous CIDR annotations above.
// The key records the last change of the CIDR state annotations, not the last resolution:
// stamping every pass made each refresh a replace of an otherwise converged policy.
// A pass that finds any sibling policy missing, or no parseable live value,
// stamps the current time on every policy, so the siblings are replaced once.
annotations[RUNTIME_HTTP_EGRESS_RESOLVED_AT_ANNOTATION] = (
unchangedRuntimeHttpEgressResolvedAt(liveAnnotationSets, annotations) ?? now
).toISOString()
return { currentCidrs, effectiveCidrs, annotations }
}
private async readExistingRuntimeHttpEgressPolicyState(
namespace: string,
policyNames: string[]
): Promise<RuntimeHttpEgressPolicyState | undefined> {
const now = new Date()
const currentCidrs = new Set<string>()
const previousExpiries = new Map<string, Date>()
let latestResolvedAt: Date | undefined
let prunedPrevious = false
for (const policyName of policyNames) {
const live = await this.readRuntimeHttpEgressPolicy(namespace, policyName)
// A foreign policy's CIDRs are not this workflow's state. A terminating
// one still counts: the prune only drops expired entries, and its apply
// defers that policy and reports the pending retry.
if (live.kind === 'absent' || live.kind === 'foreign') continue
const annotations = live.annotations
for (const cidr of parseTrustedRuntimeHttpEgressCidrsAnnotation(
annotations[RUNTIME_HTTP_EGRESS_CURRENT_CIDRS_ANNOTATION]
)) {
currentCidrs.add(cidr)
}
latestResolvedAt = latestDate(
latestResolvedAt,
parseDate(annotations[RUNTIME_HTTP_EGRESS_RESOLVED_AT_ANNOTATION])
)
const existingPreviousExpiresAt = parseFutureDate(
annotations[RUNTIME_HTTP_EGRESS_PREVIOUS_EXPIRES_AT_ANNOTATION],
now,
this.maxTrustedRuntimeHttpEgressPreviousExpiryMs
)
const hasPreviousCidrExpiriesAnnotation =
annotations[RUNTIME_HTTP_EGRESS_PREVIOUS_CIDR_EXPIRIES_ANNOTATION] !== undefined
const existingPreviousExpiries = parseTrustedRuntimeHttpEgressCidrExpiriesAnnotation(
annotations[RUNTIME_HTTP_EGRESS_PREVIOUS_CIDR_EXPIRIES_ANNOTATION],
now,
this.maxTrustedRuntimeHttpEgressPreviousExpiryMs
)
const rawPreviousCidrs = parseCidrsAnnotation(
annotations[RUNTIME_HTTP_EGRESS_PREVIOUS_CIDRS_ANNOTATION]
)
for (const cidr of rawPreviousCidrs) {
if (!isAllowedRuntimeHttpEgressCidr(cidr)) {
prunedPrevious = true
continue
}
const expiresAt = hasPreviousCidrExpiriesAnnotation
? existingPreviousExpiries.get(cidr)
: existingPreviousExpiresAt
if (expiresAt && expiresAt.getTime() > now.getTime()) {
addRuntimeHttpEgressPreviousExpiry(previousExpiries, cidr, expiresAt)
} else {
prunedPrevious = true
}
}
}
if (!prunedPrevious) return undefined
const trustedCurrentCidrs = normalizeCidrs([...currentCidrs])
if (trustedCurrentCidrs.length === 0) return undefined
const activePreviousEntries = [...previousExpiries.entries()].filter(
([, expiresAt]) => expiresAt.getTime() > now.getTime()
)
const activePreviousCidrs = normalizeCidrs(activePreviousEntries.map(([cidr]) => cidr))
const effectiveCidrs = normalizeCidrs([...trustedCurrentCidrs, ...activePreviousCidrs])
const annotations: Record<string, string> = {
[RUNTIME_HTTP_EGRESS_CURRENT_CIDRS_ANNOTATION]: serializeCidrs(trustedCurrentCidrs),
}
if (latestResolvedAt) {
annotations[RUNTIME_HTTP_EGRESS_RESOLVED_AT_ANNOTATION] = latestResolvedAt.toISOString()
}
if (activePreviousCidrs.length > 0) {
const activePreviousExpiries = new Map(activePreviousEntries)
const previousExpiresAt = [...activePreviousExpiries.values()].reduce<Date | undefined>(
latestDate,
undefined
)
annotations[RUNTIME_HTTP_EGRESS_PREVIOUS_CIDRS_ANNOTATION] =
serializeCidrs(activePreviousCidrs)
annotations[RUNTIME_HTTP_EGRESS_PREVIOUS_EXPIRES_AT_ANNOTATION] =
previousExpiresAt!.toISOString()
annotations[RUNTIME_HTTP_EGRESS_PREVIOUS_CIDR_EXPIRIES_ANNOTATION] =
serializeRuntimeHttpEgressCidrExpiries(activePreviousExpiries)
}
return { currentCidrs: trustedCurrentCidrs, effectiveCidrs, annotations }
}
private get runtimeHttpEgressOverlapMs(): number {
return (
(this.deps.config.runtimeEgressDnsOverlapSeconds ??
DEFAULT_RUNTIME_HTTP_EGRESS_DNS_OVERLAP_SECONDS) * 1_000
)
}
private get maxTrustedRuntimeHttpEgressPreviousExpiryMs(): number {
return this.runtimeHttpEgressOverlapMs * 2
}
/**
* Read a run-lane policy and classify it the way applyNetworkPolicy will. The
* run-lane policies carry no ownerReferences, so the ownership veto of
* decideNetworkPolicyConvergence reduces to the ownerless classification.
*/
private async readRuntimeHttpEgressPolicy(
namespace: string,
name: string
): Promise<RuntimeHttpEgressPolicyRead> {
const policy = await this.readNetworkPolicyOrNull(name, namespace)
if (!policy) return { kind: 'absent' }
if (classifyOwnerlessNetworkPolicyOwnership(policy).kind !== 'owned') {
return { kind: 'foreign' }
}
const annotations = policy.metadata?.annotations ?? {}
return policy.metadata?.deletionTimestamp
? { kind: 'terminating', annotations }
: { kind: 'live', annotations }
}
private annotateRuntimeHttpEgressPolicies(
policies: k8s.V1NetworkPolicy[],
names: Set<string>,
annotations: Record<string, string>
): void {
for (const policy of policies) {
if (!names.has(policy.metadata?.name ?? '')) continue
policy.metadata = {
...policy.metadata,
annotations: {
...(policy.metadata?.annotations ?? {}),
...annotations,
},
}
}
}
async reconcileDelete(
recipeName: string,
recipeNamespace?: string,
spec?: WorkflowRecipeSpec,
recipeUid?: string
): Promise<void> {
const ns = this.deps.config.sandboxNamespace
const log = createLogger('wrc', recipeName)
log.info(`Deleting workflow resources`)
this.pluginWorkloadSdkProvisioner.clearRecipeState(recipeName)
this.forgetLegacyMcpServersInternetEgress(recipeName, recipeUid)
// Best-effort cleanup of the recipe's subdirectory on the workflow output PVC.
// Must run BEFORE pod deletion while mcp-host is still reachable; failures
// are logged and swallowed so a transient HTTP error does not strand the
// CRD finalizer. A small amount of stale data in the recipe output scope is
// preferable to an undeletable WorkflowRecipe.
// recipeNamespace is optional on reconcileDelete's signature (legacy); fall
// back to the sandbox default when the caller didn't supply it. After the
// full CRD watcher propagation every path carries the namespace.
await this.cleanupRecipeArtifacts(recipeName, recipeNamespace ?? ns)
// Idempotent delete of the workflow_schedules row. No-op if the recipe
// never had scheduling or the row was already cleared.
if (this.deps.pgPool && recipeNamespace) {
try {
await deleteScheduling(this.deps.pgPool, recipeNamespace, recipeName)
} catch (err) {
log.error('Failed to delete workflow_schedules row', {
error: err instanceof Error ? err.message : String(err),
})
}
// Cleanup trigger grants, approval target allowlists, and live approval
// requests. Without this, a
// recipe-delete + recipe-recreate cycle with the same name inherits
// stale user/team grant, team approval target rows, or pending/approved
// approval decisions from the prior incarnation.
//
// Pattern mirrors the scheduling delete above — same pool, same
// ns/name scoping, same idempotent semantics. Failure logged and
// swallowed: stranding the CRD finalizer over a transient DB blip
// is worse than an orphaned grant row that a human can DELETE by
// hand if needed.
try {
const client = await this.deps.pgPool.connect()
try {
await client.query('BEGIN')
await client.query(
`DELETE FROM user_workflow_triggers
WHERE recipe_namespace = $1 AND recipe_name = $2`,
[recipeNamespace, recipeName]
)
await client.query(
`DELETE FROM team_workflow_triggers
WHERE recipe_namespace = $1 AND recipe_name = $2`,
[recipeNamespace, recipeName]
)
await client.query(
`DELETE FROM workflow_recipe_allowed_teams
WHERE recipe_namespace = $1 AND recipe_name = $2`,
[recipeNamespace, recipeName]
)
const legacyAllowedUsers = await client.query(
`SELECT to_regclass('public.workflow_recipe_allowed_users')::text AS "regclass"`
)
const legacyAllowedUsersRow = legacyAllowedUsers.rows[0] as
| { regclass?: string | null }
| undefined
if (legacyAllowedUsersRow?.regclass) {
await client.query(
`DELETE FROM workflow_recipe_allowed_users
WHERE recipe_namespace = $1 AND recipe_name = $2`,
[recipeNamespace, recipeName]
)
}
await client.query(
`UPDATE workflow_approval_requests
SET status = 'cancelled',
cancelled_at = COALESCE(cancelled_at, NOW()),
cancelled_by = COALESCE(cancelled_by, 'workflow-recipe-delete')
WHERE recipe_namespace = $1
AND recipe_name = $2
AND status IN ('pending', 'approved')`,
[recipeNamespace, recipeName]
)
await client.query('COMMIT')
} catch (err) {
await client.query('ROLLBACK').catch(rollbackErr => {
log.error('Failed to rollback workflow trigger policy cleanup', {
error: rollbackErr instanceof Error ? rollbackErr.message : String(rollbackErr),
})
})
throw err
} finally {
client.release()
}
} catch (err) {
log.error('Failed to delete workflow trigger policy rows', {
error: err instanceof Error ? err.message : String(err),
})
}
}
// Parallelize all independent deletions with allSettled so one failure
// doesn't prevent the remaining cleanup attempts. Runtime pod/PVC deletion
// failures are rethrown after all attempts so the CRD finalizer retries.
const npNames = [
`${recipeName}-coord-to-mcp-host`,
`${recipeName}-coord-to-mcp-host-ingress`,
`${recipeName}-coord-to-wrc`,
`${recipeName}-wrc-to-mcp-host`,
`${recipeName}-wrc-to-artifact-reader`,
`${recipeName}-mcp-host-to-servers`,
`${recipeName}-mcp-host-to-llm-api`,
`${recipeName}-mcp-host-to-approval-gateway`,
`${recipeName}-mcp-host-to-codex-proxy`,
`${recipeName}-mcp-host-to-grok-proxy`,
`${recipeName}-coord-to-snippet-runner`,
`${recipeName}-coord-to-snippet-runner-ingress`,
`${recipeName}-snippet-runner-egress`,
]
// NP-02: wf-mcp-host-ingress lives in mcpServerNamespace, not sandboxNamespace.
// NP-03: now one NP per mcp-server (named `wf-mcp-ingress-${mcpServerName}`).
// List-by-label is the cleanest cleanup — all managed NPs have recipe label.
// Per-mcp-server NPs are deleted via label selector below. The legacy
// internet policy predates those labels, so it is deleted by name (R4-L4).
const mcpServerNpNames: string[] = [buildLegacyMcpServersInternetEgressPolicyName(recipeName)]
const mcpNs = this.deps.config.mcpServerNamespace
const cleanupTasks: Array<{ label: string; run: () => Promise<void> }> = [
{
label: `Pod "${recipeName}-coordinator"`,
run: () => deletePodIfExists(this.deps.coreApi, `${recipeName}-coordinator`, ns),
},
{
label: `Pod "${recipeName}-mcp-host"`,
run: () => deletePodIfExists(this.deps.coreApi, `${recipeName}-mcp-host`, ns),
},
{
label: `Pod "${recipeName}-artifact-reader"`,
run: () => deletePodIfExists(this.deps.coreApi, `${recipeName}-artifact-reader`, ns),
},
{
label: `Pod "${recipeName}-snippet-runner"`,
run: () => deletePodIfExists(this.deps.coreApi, `${recipeName}-snippet-runner`, ns),
},
{
label: `Pod "${buildWorkflowOutputPreparePodName(recipeName)}"`,
run: () =>
deletePodIfExists(this.deps.coreApi, buildWorkflowOutputPreparePodName(recipeName), ns),
},
{
label: `Pod "${buildWorkflowOutputAnchorPodName(recipeName)}"`,
run: () =>
deletePodIfExists(this.deps.coreApi, buildWorkflowOutputAnchorPodName(recipeName), ns),
},
{
label: `Secret "wf-${recipeName}-coordinator-token"`,
run: () =>
this.safeDelete(() =>
this.deps.coreApi.deleteNamespacedSecret({
name: `wf-${recipeName}-coordinator-token`,
namespace: ns,
})
),
},
{
label: `Secret "wf-${recipeName}-mcp-host-runtime-tokens"`,
run: () =>
this.safeDelete(() =>
this.deps.coreApi.deleteNamespacedSecret({
name: `wf-${recipeName}-mcp-host-runtime-tokens`,
namespace: ns,
})
),
},
{
// Plugin Workload SDK workload token — provisioned by the eager SDK path
// (ensurePluginWorkloadSdkTokenSecret); tear it down with the recipe.
label: `Secret "${buildPluginWorkloadSdkTokenSecretName(recipeName)}"`,
run: () => this.pluginWorkloadSdkProvisioner.deletePluginWorkloadSdkTokenSecret(recipeName),
},
{
label: `ConfigMap "wf-${recipeName}-soul-md"`,
run: () =>
this.safeDelete(() =>
this.deps.coreApi.deleteNamespacedConfigMap({
name: `wf-${recipeName}-soul-md`,
namespace: ns,
})
),
},
{
label: `ConfigMap "${recipeName}-workflow-config"`,
run: () =>
this.safeDelete(() =>
this.deps.coreApi.deleteNamespacedConfigMap({
name: `${recipeName}-workflow-config`,
namespace: ns,
})
),
},
{
label: `Service "${buildMcpHostServiceName(recipeName)}"`,
run: () =>
this.safeDelete(() =>
this.deps.coreApi.deleteNamespacedService({
name: buildMcpHostServiceName(recipeName),
namespace: ns,
})
),
},
{
label: `Service "${buildMcpHostRouteAliasServiceName(recipeName, ns)}"`,
run: () =>
this.safeDelete(() =>
this.deps.coreApi.deleteNamespacedService({
name: buildMcpHostRouteAliasServiceName(recipeName, ns),
namespace: ns,
})
),
},
{
label: `Service "${buildArtifactReaderServiceName(recipeName)}"`,
run: () =>
this.safeDelete(() =>
this.deps.coreApi.deleteNamespacedService({
name: buildArtifactReaderServiceName(recipeName),
namespace: ns,
})
),
},
{
label: `Service "${buildSnippetRunnerServiceName(recipeName)}"`,
run: () =>
this.safeDelete(() =>
this.deps.coreApi.deleteNamespacedService({
name: buildSnippetRunnerServiceName(recipeName),
namespace: ns,
})
),
},
...npNames.map(npName => ({
label: `NetworkPolicy "${npName}"`,
run: () =>
this.safeDelete(() =>
this.deps.networkingApi.deleteNamespacedNetworkPolicy({ name: npName, namespace: ns })
),
})),
// Snippet per-workload ingress policies are generated from workload
// resource names and may be hash-suffixed when long. Delete by the same
// exact WRC recipe labels used at creation so dynamic names cannot linger.
{
label: 'NetworkPolicies by sandbox recipe label',
run: () =>
this.safeDelete(() =>
this.deleteNetworkPoliciesByLabelSelector(
ns,
`clerum.io/recipe=${recipeName},clerum.io/managed-by=wrc`
)
),
},
...mcpServerNpNames.map(npName => ({
label: `NetworkPolicy "${npName}"`,
run: () =>
this.safeDelete(() =>
this.deps.networkingApi.deleteNamespacedNetworkPolicy({
name: npName,
namespace: mcpNs,
})
),
})),
// NP-03: Delete per-mcp-server ingress NPs by label selector (wf-mcp-ingress-*).
{
label: 'NetworkPolicies by mcp-server recipe label',
run: () =>
this.safeDelete(() =>
this.deleteNetworkPoliciesByLabelSelector(
mcpNs,
`clerum.io/recipe=${recipeName},clerum.io/managed-by=wrc`
)
),
},
// Clean up per-workload transport Services in mcp-server. Agentic
// recipes WITHOUT contextRef have their Services created by WRC directly
// (not via HCC delegation), and adjustManifestNamespace strips ownerRefs
// because the recipe CRD lives in sandbox-recipes while the Service
// lives in mcp-server (cross-namespace ownerRefs are invalid). K8s GC
// will not collect them on CRD delete, so we sweep them here.
//
// Label selector uses `clerum.io/managed-by=workflow-recipes` (NOT
// `wrc`) because that is the value emitted by `standardLabels()` in
// resourceBuilder.ts; the `wrc` value above only tags NetworkPolicy
// objects that workflowReconciler itself builds.
{
label: 'Transport Services by mcp-server recipe label',
run: () =>
this.safeDelete(() =>
this.deps.coreApi.deleteCollectionNamespacedService({
namespace: mcpNs,
labelSelector: `clerum.io/recipe=${recipeName},clerum.io/managed-by=workflow-recipes`,
})
),
},
]
// Child runs use the parent recipe's workflow output PVC, so deleting a
// child may 404 here. The parent PVC is deleted with the parent recipe.
// Explicit external output claims are operator-owned even if they reuse
// the generated claim name, so never delete when the deleted CRD declares
// spec.output.claimName.
if (!trimOutputClaimName(spec)) {
const pvcName = buildWorkflowOutputPvcName(recipeName)
cleanupTasks.push({
label: `PersistentVolumeClaim "${pvcName}"`,
run: () => this.deletePersistentVolumeClaimIfExists(pvcName, ns),
})
}
const cleanupResults = await Promise.allSettled(cleanupTasks.map(task => task.run()))
const failures = cleanupResults.flatMap((result, index) =>
result.status === 'rejected'
? [
{
label: cleanupTasks[index]?.label ?? `cleanup task ${index}`,
reason: result.reason,
},
]
: []
)
if (failures.length > 0) {
for (const failure of failures) {
log.error('Workflow runtime resource cleanup failed', {
resource: failure.label,
error: failure.reason instanceof Error ? failure.reason.message : String(failure.reason),
})
}
throw new Error(
`Failed to delete workflow runtime resources: ${failures.map(f => f.label).join(', ')}`
)
}
}
private async deleteNetworkPoliciesByLabelSelector(
namespace: string,
labelSelector: string,
excludeName?: string
): Promise<void> {
const list = await this.deps.networkingApi.listNamespacedNetworkPolicy({
namespace,
labelSelector,
})
await Promise.all(
(list.items ?? [])
.map(policy => policy.metadata?.name)
.filter((name): name is string => Boolean(name))
.filter(name => name !== excludeName)
.map(async name => {
try {
await this.deps.networkingApi.deleteNamespacedNetworkPolicy({ name, namespace })
} catch (error: unknown) {
// A concurrent reconciler or the API server may have removed the
// policy between list and delete. That is already the desired
// postcondition; every other error must keep the revocation
// transaction in progress and trigger a retry.
if (getErrorCode(error) !== 404) throw error
}
})
)
}
/**
* Best-effort artifact cleanup before pod teardown. Prefer artifact-reader
* because it exists for every `/output`-backed run; fall back to mcp-host for
* older or partially reconciled runs. Failures are swallowed so a transient
* HTTP error never blocks the finalizer.
*/
private async cleanupRecipeArtifacts(recipeName: string, recipeNamespace: string): Promise<void> {
const log = createLogger('wrc', recipeName)
const candidates = [
{
component: 'artifact-reader',
serviceName: buildArtifactReaderServiceName(recipeName),
url: buildArtifactReaderUrl(recipeName, this.deps.config.sandboxNamespace),
},
{
component: 'mcp-host',
serviceName: buildMcpHostServiceName(recipeName),
url: buildMcpHostUrl(recipeName, this.deps.config.sandboxNamespace),
},
]
const availableCandidates: typeof candidates = []
for (const candidate of candidates) {
try {
await this.deps.coreApi.readNamespacedService({
name: candidate.serviceName,
namespace: this.deps.config.sandboxNamespace,
})
} catch (err) {
if (getErrorCode(err) === 404) {
log.info(`artifact cleanup: ${candidate.component} service absent; skipping target`)
continue
}
log.warn(`artifact cleanup: failed to check ${candidate.component} service`, {
error: err instanceof Error ? err.message : String(err),
})
}
availableCandidates.push(candidate)
}
if (availableCandidates.length === 0) {
log.info('artifact cleanup: no artifact cleanup service found; skipping HTTP cleanup')
return
}
let token: string
try {
token = await this.deps.tokenFactory.signWrcArtifactDeleteToken(recipeName, recipeNamespace)
} catch (err) {
log.warn('artifact cleanup: failed to mint delete token; skipping', {
error: err instanceof Error ? err.message : String(err),
})
return
}
for (const candidate of availableCandidates) {
const target = `${candidate.url}/api/v1/workflow/artifacts`
try {
const res = await fetch(target, {
method: 'DELETE',
headers: { authorization: `Bearer ${token}` },
signal: AbortSignal.timeout(5_000),
})
if (!res.ok) {
log.warn(`artifact cleanup: ${candidate.component} returned non-2xx`, {
status: res.status,
})
continue
}
log.info(`artifact cleanup: ${candidate.component} /artifacts DELETE succeeded`)
// Once `/output` cleanup succeeds, remaining mcp-host artifacts are pod-local /tmp scratch.
// Pod deletion clears that ephemeral data; there is no second durable store to clean.
return
} catch (err) {
log.warn(`artifact cleanup: ${candidate.component} DELETE request failed`, {
error: err instanceof Error ? err.message : String(err),
})
}
}
}
private validateCustomCoordinatorImagePolicy(image: string | undefined): string | undefined {
if (!image) return undefined
if (this.deps.config.enableCustomCoordinatorImage !== true) {
return 'custom coordinator images are disabled'
}
if (hasUnsafeImageReferenceSyntax(image)) {
return 'custom coordinator image reference is invalid'
}
if (hasLatestTag(image)) {
return 'custom coordinator image must not use :latest'
}
if (hasInvalidDigest(image)) {
return 'custom coordinator image digest must be a valid sha256 digest'
}
if (this.deps.config.requireCoordinatorImageDigest === true && !hasValidSha256Digest(image)) {
return 'custom coordinator image must include a valid sha256 digest'
}
const prefixes = this.deps.config.allowedCoordinatorImagePrefixes ?? []
if (prefixes.length === 0) {
return 'custom coordinator image allowlist is empty'
}
if (!prefixes.some(prefix => matchesAllowedImagePrefix(image, prefix))) {
return 'custom coordinator image is not allowed by WRC policy'
}
return undefined
}
/**
* Provision the mcp-host's backing Secrets: the runtime-token Secret (always)
* and the Plugin Workload SDK workload-token Secret (when the SDK is enabled
* and declared). Shared by the triggered-run path and the eager SDK path so
* the two cannot drift on which Secrets the pod needs to boot Ready.
*/
private async ensureMcpHostSecrets(
namespace: string,
recipeName: string,
runtimeScopeRecipeName: string,
spec: WorkflowRecipeSpec,
recipeUid: string | undefined,
// The SAME verdict the caller used for the configure decision.
// Recomputing it here read `this.codexView` live, so a concurrent refresh
// could give the binding one snapshot and the scopes another.
codexVerdict: CodexRecipeVerdict
): Promise<McpHostRuntimeTokenRefreshResult> {
const effectiveScopes = this.resolveEffectiveControlScopes(spec, codexVerdict)
const tokenRefresh = await this.ensureMcpHostRuntimeTokenSecret(
namespace,
recipeName,
runtimeScopeRecipeName,
effectiveScopes.scopes,
deriveRecipeHostGfsScopes(spec),
effectiveScopes.codexScopeUncertain,
effectiveScopes.grokScopeUncertain
)
if (this.deps.config.pluginWorkloadSdkEnabled && spec.pluginWorkloadSdk) {
await this.pluginWorkloadSdkProvisioner.ensurePluginWorkloadSdkTokenSecret(recipeName, spec)
}
return tokenRefresh
}
/**
* Converge the mcp-host headless Service that backs its in-cluster DNS name
* (wf-<recipe>-mcp-host). Shared by the triggered-run and eager SDK paths;
* without it the SDK endpoint and coordinator→mcp-host calls fail at DNS.
*
* Reads first and writes only when needed: creates the Service when it is
* absent, and replaces it when its selector or ports have drifted. A live
* Service that matches is left untouched. A replace that conflicts with a
* concurrent write is re-judged against a fresh read and retried once.
*/
private async ensureMcpHostHeadlessService(recipeName: string): Promise<void> {
const headlessSvc = buildMcpHostHeadlessService(recipeName, this.deps.config.sandboxNamespace)
const namespace = this.deps.config.sandboxNamespace
const name = buildMcpHostServiceName(recipeName)
// Read first: this method runs on every eager reconcile, and a POST used as
// an existence probe is an apiserver write (409) on every pass.
let existing = await this.readServiceIfExists(name, namespace)
if (!existing) {
try {
await this.deps.coreApi.createNamespacedService({ namespace, body: headlessSvc })
this.log.info(`Created Headless Service "${name}"`)
return
} catch (error: unknown) {
if (getErrorCode(error) !== 409) throw error
// Another writer created it between the read and the POST.
existing = await this.readServiceIfExists(name, namespace)
if (!existing) {
// ...and it was deleted again before the re-read. There is no live
// Service to judge, so this pass stops and asks for a fresh one.
throw new ResourceVanishedAfterConflictError(`Headless Service "${name}"`, {
cause: error,
})
}
}
}
// Skip the replace when the live Service already matches the desired
// spec — an unconditional PUT churns resourceVersions and apiserver writes
// for no drift.
const normalizePorts = (ports: k8s.V1ServicePort[] | undefined) =>
(ports ?? []).map(p => ({
name: p.name ?? null,
port: p.port,
targetPort: p.targetPort ?? null,
protocol: p.protocol ?? 'TCP',
}))
const desiredSpec = JSON.stringify({
selector: headlessSvc.spec?.selector ?? null,
ports: normalizePorts(headlessSvc.spec?.ports),
})
// A 409 on the replace means another writer changed the Service after the
// read, so its resourceVersion is stale. Re-read, judge the new live
// Service, and retry once, as applyNetworkPolicy does.
for (let attempt = 0; attempt < 2; attempt += 1) {
const existingSpec = JSON.stringify({
selector: existing.spec?.selector ?? null,
ports: normalizePorts(existing.spec?.ports),
})
if (desiredSpec === existingSpec) {
return
}
const updatedSvc: k8s.V1Service = {
...headlessSvc,
metadata: {
...headlessSvc.metadata,
resourceVersion: existing.metadata?.resourceVersion,
},
spec: {
...headlessSvc.spec,
clusterIP: existing.spec?.clusterIP ?? headlessSvc.spec?.clusterIP,
},
}
try {
await this.deps.coreApi.replaceNamespacedService({ name, namespace, body: updatedSvc })
this.log.info(`Updated Headless Service "${name}"`)
return
} catch (error: unknown) {
if (getErrorCode(error) !== 409) throw error
if (attempt === 1) {
// Contention, not a defect: another writer changed the Service again
// between the re-read and the retry. Ask for a fresh pass instead of
// failing the run, as the vanished-after-conflict path does.
throw new RetryableReconcileError(
`Headless Service "${name}" replace conflicted on both attempts; a fresh reconciliation is required`,
{ cause: error }
)
}
const reread = await this.readServiceIfExists(name, namespace)
if (!reread) {
throw new RetryableReconcileError(
`Headless Service "${name}" disappeared after a replace conflict; a fresh reconciliation is required`,
{ cause: error }
)
}
existing = reread
}
}
}
/**
* Create-only headless Service: an existing Service is left as it is, and
* the read keeps the POST from running as an existence probe on every pass.
*/
private async ensureHeadlessServiceExists(name: string, body: k8s.V1Service): Promise<void> {
const namespace = this.deps.config.sandboxNamespace
if (await this.readServiceIfExists(name, namespace)) return
await this.createIfNotExists(
() => this.deps.coreApi.createNamespacedService({ namespace, body }),
`Headless Service "${name}"`
)
}
private async readServiceIfExists(
name: string,
namespace: string
): Promise<k8s.V1Service | undefined> {
try {
return await this.deps.coreApi.readNamespacedService({ name, namespace })
} catch (error: unknown) {
if (getErrorCode(error) === 404) return undefined
throw error
}
}
/**
* True when the existing mcp-host pod carries the workflow-output mount (its
* presence is signalled by the output-claim label). Eager SDK pods are built
* without the mount, so a missing label means the pod must be recreated before
* a triggered run that needs output. Returns true on read failure (fail-safe:
* don't churn a pod we can't inspect).
*/
private async mcpHostPodHasWorkflowOutputMount(recipeName: string): Promise<boolean> {
try {
const pod = await this.deps.coreApi.readNamespacedPod({
name: `${recipeName}-mcp-host`,
namespace: this.deps.config.sandboxNamespace,
})
return pod.metadata?.labels?.[WORKFLOW_OUTPUT_CLAIM_LABEL] !== undefined
} catch {
return true
}
}
private resolveMcpHostAgent(spec: WorkflowRecipeSpec): AgentSpec | undefined {
return resolveMcpHostAgent(spec)
}
private validateWorkflowOutputSpec(spec: WorkflowRecipeSpec): string | undefined {
const claimName = trimOutputClaimName(spec)
if (!claimName) return undefined
if (spec.output?.destination !== 'pvc') {
return 'spec.output.claimName requires spec.output.destination=pvc'
}
if (claimName.length > 253 || !DNS_SUBDOMAIN_RE.test(claimName)) {
return 'spec.output.claimName must be a valid Kubernetes PVC name'
}
const namespace = spec.output?.namespace?.trim()
if (namespace && namespace !== this.deps.config.sandboxNamespace) {
return `spec.output.namespace must be omitted or match ${this.deps.config.sandboxNamespace}; workflow output PVCs are mounted in the runtime namespace`
}
return undefined
}
private validateExecutableSteps(spec: WorkflowRecipeSpec): string | undefined {
const limitError = validateWorkflowRecipeLimits(spec, this.deps.config)
if (limitError) return limitError
const steps = spec.steps ?? []
try {
buildExecutionGroups(steps.map(step => ({ id: step.id, dependsOn: step.dependsOn ?? [] })))
} catch (err) {
if (err instanceof CyclicDependencyError) {
return `dependency cycle detected: ${err.cycle.join(' -> ')}`
}
if (err instanceof UnknownDependencyError) {
return `step "${err.stepId}" depends on unknown step "${err.unknownDependency}"`
}
throw err
}
for (const step of steps) {
const hasRun = step.run !== undefined
const hasInstruction = typeof step.instruction === 'string' && step.instruction.trim() !== ''
const hasCustomCoordinator = getCustomCoordinatorImage(spec) !== undefined
const hasBrokerAgent = hasCompleteAgent(spec.agent) || hasCompleteAgent(step.agent)
const needsBroker =
hasInstruction ||
Boolean(step.agent) ||
Boolean(step.mcpServers?.length) ||
Boolean(step.requiresApproval)
if (hasRun && hasInstruction) {
return `step "${step.id}" cannot configure both run and instruction`
}
if (!hasCustomCoordinator && hasRun === hasInstruction) {
return `step "${step.id}" must configure exactly one of run or instruction`
}
if (hasRun) {
const run = step.run
if (step.agent) {
return `step "${step.id}" cannot configure an agent when run is set`
}
if (step.requiresApproval) {
return `step "${step.id}" cannot require approval when run is set`
}
if (!isSnippetRun(run)) return `step "${step.id}" run.type must be snippet`
const snippetError = this.validateSnippetStep(step.id, run, spec)
if (snippetError) return snippetError
} else if (hasCustomCoordinator && needsBroker && !hasBrokerAgent) {
if (step.requiresApproval) {
return `step "${step.id}" requires an agent configuration for approval`
}
return `step "${step.id}" requires an agent configuration for broker-backed custom execution`
} else if (hasInstruction && !spec.agent && (!step.agent?.provider || !step.agent.model)) {
return `step "${step.id}" requires an agent configuration`
} else if (step.requiresApproval && !needsWorkflowMcpHost(spec)) {
return `step "${step.id}" requires an agentic broker for approval`
}
}
return undefined
}
private validateSnippetStep(
stepId: string,
run: WorkflowSnippetRunSpec,
spec: WorkflowRecipeSpec
): string | undefined {
for (const key of Object.keys(run as unknown as Record<string, unknown>)) {
if (!SNIPPET_RUN_KEYS.has(key))
return `step "${stepId}" run contains unsupported field "${key}"`
}
if (run.language !== 'typescript') {
return `step "${stepId}" snippet language must be typescript`
}
if (typeof run.code !== 'string' || run.code.trim() === '') {
return `step "${stepId}" snippet code must be a non-empty string`
}
if (run.code.length > MAX_SNIPPET_CODE_LENGTH) {
return `step "${stepId}" snippet code exceeds ${MAX_SNIPPET_CODE_LENGTH} characters`
}
const workloadIds = new Set((spec.workloads ?? []).map(workload => workload.id))
const mcpServerIds = new Set((spec.mcpServers ?? []).map(server => server.id))
for (const workload of spec.workloads ?? []) {
if (workload.transport) mcpServerIds.add(workload.id)
}
const mongoAccess = run.capabilities?.mongo?.access
if (run.capabilities?.mongo && mongoAccess !== 'read' && mongoAccess !== 'readWrite') {
return `step "${stepId}" mongo capability must declare access read or readWrite`
}
for (const workload of run.capabilities?.mongo?.workloads ?? []) {
if (!workloadIds.has(workload)) {
return `step "${stepId}" references undeclared mongo workload "${workload}"`
}
}
const postgresAccess = run.capabilities?.postgres?.access
if (run.capabilities?.postgres && postgresAccess !== 'read' && postgresAccess !== 'readWrite') {
return `step "${stepId}" postgres capability must declare access read or readWrite`
}
for (const workload of run.capabilities?.postgres?.workloads ?? []) {
if (!workloadIds.has(workload)) {
return `step "${stepId}" references undeclared postgres workload "${workload}"`
}
}
const secretAliases = new Set<string>()
for (const secret of run.capabilities?.secrets ?? []) {
if (!SNIPPET_SECRET_ALIAS_RE.test(secret.alias)) {
return `step "${stepId}" has invalid snippet secret alias "${secret.alias}"`
}
if (secretAliases.has(secret.alias)) {
return `step "${stepId}" declares duplicate snippet secret alias "${secret.alias}"`
}
secretAliases.add(secret.alias)
if (!secret.secretRef?.name || !secret.secretRef?.key) {
return `step "${stepId}" snippet secret alias "${secret.alias}" must reference name and key`
}
if (secret.secretRef.name.startsWith('wf-')) {
return `step "${stepId}" cannot reference platform-managed secret "${secret.secretRef.name}"`
}
}
const workflowHttpClass = runtimeHttpEgressClass(spec)
const requestedHttpClass = run.capabilities?.http?.egressClass ?? 'exact-host'
if (requestedHttpClass !== 'exact-host' && requestedHttpClass !== 'public-web') {
return `step "${stepId}" HTTP egressClass must be exact-host or public-web`
}
const workflowHttpHosts = new Set(spec.runtimeEgress?.http?.allowedHosts ?? [])
const requestedHttpHosts = run.capabilities?.http?.allowedHosts ?? []
if (requestedHttpClass === 'public-web') {
if (workflowHttpClass !== 'public-web') {
return `step "${stepId}" HTTP egressClass public-web requires spec.runtimeEgress.http.egressClass public-web`
}
if (requestedHttpHosts.length > 0) {
return `step "${stepId}" HTTP allowedHosts must be omitted when egressClass is public-web`
}
}
if (
requestedHttpClass === 'exact-host' &&
workflowHttpClass === 'public-web' &&
requestedHttpHosts.length > 0
) {
return `step "${stepId}" HTTP allowedHosts cannot be used when spec.runtimeEgress.http.egressClass is public-web`
}
for (const host of requestedHttpHosts) {
const hostError = validatePublicHttpHost(host)
if (hostError) return `step "${stepId}" ${hostError}`
if (!workflowHttpHosts.has(host)) {
return `step "${stepId}" HTTP host "${host}" must be declared in spec.runtimeEgress.http.allowedHosts`
}
}
const requestedMcpServers = run.capabilities?.mcp?.servers ?? []
for (const server of requestedMcpServers) {
if (!mcpServerIds.has(server)) {
return `step "${stepId}" references undeclared MCP server "${server}"`
}
}
if (requestedMcpServers.length > 0) {
const allowedTools = run.capabilities?.mcp?.allowedTools?.include ?? []
if (allowedTools.length === 0) {
return `step "${stepId}" must declare explicit MCP allowedTools`
}
for (const tool of allowedTools) {
if (tool.includes('*')) {
return `step "${stepId}" snippet MCP allowedTools cannot include wildcards`
}
if (!requestedMcpServers.some(server => tool.startsWith(`${server}__`))) {
return `step "${stepId}" snippet MCP tool "${tool}" must be scoped to an allowed server`
}
}
}
return undefined
}
private async validateSnippetSecretRefs(
recipeName: string,
spec: WorkflowRecipeSpec,
resourceInstances?: Record<string, string>
): Promise<string | undefined> {
const ns = this.deps.config.sandboxNamespace
const read = (name: string, namespace: string) =>
this.deps.coreApi.readNamespacedSecret({ name, namespace })
for (const secret of collectSnippetSecretRefs(spec, resourceInstances)) {
// Issue #637 — route snippet capability Secrets through the SAME ownership
// chokepoint as every other surface (classifySecretAccess), so the snippet
// path cannot drift from the shared ownership/error semantics. The
// snippet-runner pod mounts this Secret as a `secretKeyRef` env var in
// sandbox-recipes; without this gate a recipe could name another recipe's
// co-tenant Secret and exfiltrate its keys. Fail closed on ownership BEFORE
// the key-presence check.
const access = await classifySecretAccess(
read,
getErrorCode,
secret.secretName,
ns,
recipeName
)
if (access.state === 'denied') {
return (
`snippet secret "${secret.secretName}" is not accessible to recipe ` +
`"${recipeName}" — it requires clerum.io/shared=true or ` +
`clerum.io/owner-recipe=${recipeName}`
)
}
if (access.state === 'error') {
// Ownership could not be verified (non-404 read error) — fail closed; the
// outer reconcile maps the throw to a retryable/degraded outcome.
throw new Error(
`snippet secret "${secret.secretName}" ownership could not be verified in namespace "${ns}"`
)
}
if (access.state === 'missing') {
return `snippet secret "${secret.secretName}" was not found in namespace "${ns}"`
}
if (!access.keys.has(secret.secretKey)) {
return (
`snippet secret key "${secret.secretKey}" was not found in Secret ` +
`"${secret.secretName}" in namespace "${ns}"`
)
}
}
return undefined
}
private resolveSnippetWorkloadNetworkBindings(
recipeName: string,
recipeUid: string,
spec: WorkflowRecipeSpec
): Array<{ resourceName: string; port: number }> {
const requestedIds = new Set(collectSnippetDatabaseWorkloadIds(spec))
if (requestedIds.size === 0) return []
const recipeRef = {
apiVersion: 'clerum.io/v1alpha1' as const,
kind: 'WorkflowRecipe' as const,
metadata: {
name: recipeName,
namespace: this.deps.config.sandboxNamespace,
uid: recipeUid,
},
spec,
}
return (spec.workloads ?? [])
.filter(workload => requestedIds.has(workload.id))
.map(workload => ({
resourceName: resolveWorkloadRuntimeResourceName(recipeRef, workload),
port: workload.port ?? (this.isPostgresSnippetWorkload(spec, workload.id) ? 5432 : 27017),
}))
}
private resolveCustomCoordinatorWorkloadNetworkBindings(
recipeName: string,
recipeUid: string,
spec: WorkflowRecipeSpec
): Array<{ resourceName: string; port: number }> {
if (!spec.coordinatorImage?.trim()) return []
const recipeRef = {
apiVersion: 'clerum.io/v1alpha1' as const,
kind: 'WorkflowRecipe' as const,
metadata: {
name: recipeName,
namespace: this.deps.config.sandboxNamespace,
uid: recipeUid,
},
spec,
}
return (spec.workloads ?? [])
.filter(workload => !workload.transport && typeof workload.port === 'number')
.map(workload => ({
resourceName: resolveWorkloadRuntimeResourceName(recipeRef, workload),
port: workload.port!,
}))
}
private isPostgresSnippetWorkload(spec: WorkflowRecipeSpec, workloadId: string): boolean {
return (spec.steps ?? []).some(
step =>
isSnippetRun(step.run) && step.run.capabilities?.postgres?.workloads?.includes(workloadId)
)
}
private resolveSnippetMcpServerFullNames(
recipeName: string,
recipeUid: string,
spec: WorkflowRecipeSpec,
workloadIdsWithTransport: ReadonlySet<string>,
workloadMcpServerLabels = this.resolveTransportWorkloadMcpServerLabels(
recipeName,
recipeUid,
spec
)
): string[] {
const requestedIds = new Set(collectSnippetMcpServerIds(spec))
if (requestedIds.size === 0) return []
const selected = new Map<string, { id: string; endpoint?: string }>()
for (const server of spec.mcpServers ?? []) {
if (requestedIds.has(server.id)) selected.set(server.id, server)
}
for (const workload of spec.workloads ?? []) {
if (requestedIds.has(workload.id) && workload.transport) {
selected.set(workload.id, { id: workload.id })
}
}
return resolveMcpServerFullNames(
recipeName,
[...selected.values()],
workloadIdsWithTransport,
workloadMcpServerLabels
)
}
private resolveTransportWorkloadMcpServerLabels(
recipeName: string,
recipeUid: string,
spec: WorkflowRecipeSpec
): Map<string, string> {
const recipeRef = {
apiVersion: 'clerum.io/v1alpha1' as const,
kind: 'WorkflowRecipe' as const,
metadata: {
name: recipeName,
namespace: this.deps.config.sandboxNamespace,
uid: recipeUid,
},
spec,
}
return new Map(
(spec.workloads ?? [])
.filter(workload => workload.transport != null)
.map(workload => [workload.id, resolveWorkloadMcpServerLabel(recipeRef, workload)])
)
}
// ─── SOUL.md ConfigMap ──────────────────────────────────────────────
private async ensureSoulConfigMap(recipeName: string, spec: WorkflowRecipeSpec): Promise<void> {
let soulContent =
'# Default SOUL.md\nYou are a helpful AI assistant executing workflow steps.\n'
if (spec.agent?.soulRef?.storageRef) {
try {
// Read storage credentials from control-plane
const credSecret = await this.deps.coreApi.readNamespacedSecret({
name: 'clerum-storage-credentials',
namespace: 'control-plane',
})
const rawAccessKey = credSecret.data?.['access-key']
const rawSecretKey = credSecret.data?.['secret-key']
if (!rawAccessKey || !rawSecretKey) {
throw new Error('Storage credentials Secret missing access-key or secret-key fields')
}
const creds: StorageCredentials = {
accessKey: Buffer.from(rawAccessKey, 'base64').toString(),
secretKey: Buffer.from(rawSecretKey, 'base64').toString(),
}
const ref: StorageRef = {
...spec.agent.soulRef.storageRef,
provider: spec.agent.soulRef.storageRef.provider ?? 's3',
}
const client = new ObjectStorageClient(creds, ref)
const content = await client.download(recipeName)
soulContent = content.toString('utf-8')
} catch (error) {
createLogger('wrc', recipeName).error(`Failed to download SOUL.md`, {
error: error instanceof Error ? error.message : String(error),
})
// Proceed with default — non-fatal
}
}
const configMap: k8s.V1ConfigMap = {
apiVersion: 'v1',
kind: 'ConfigMap',
metadata: {
name: `wf-${recipeName}-soul-md`,
namespace: this.deps.config.sandboxNamespace,
labels: {
'clerum.io/recipe': recipeName,
'clerum.io/managed-by': 'wrc',
},
// No ownerRef: cross-namespace ownerRefs are deleted by K8s GC (1.24+).
// Cleanup handled explicitly by reconcileDelete() finalizer.
},
data: {
'SOUL.md': soulContent,
},
}
await this.createIfNotExists(
() =>
this.deps.coreApi.createNamespacedConfigMap({
namespace: this.deps.config.sandboxNamespace,
body: configMap,
}),
`ConfigMap "wf-${recipeName}-soul-md"`
)
}
// ─── Workflow Config ConfigMap ──────────────────────────────────────
private async ensureWorkflowConfigMap(
recipeName: string,
recipeUid: string,
spec: WorkflowRecipeSpec,
resolvedInputs?: Record<string, unknown>
): Promise<void> {
// Substitute {{inputs.KEY}} placeholders in step instructions with resolved input values.
// This ensures the coordinator Pod receives concrete values rather than template literals.
const substituteInputs = (text: string): string => {
if (!resolvedInputs) return text
return text.replace(/\{\{inputs\.([\w-]+)\}\}/g, (_match, key: string) => {
const val = resolvedInputs[key]
return val !== undefined ? String(val) : `{{inputs.${key}}}`
})
}
const effectiveInputs = resolvedInputs ?? spec.inputs
const recipeRef = {
apiVersion: 'clerum.io/v1alpha1' as const,
kind: 'WorkflowRecipe' as const,
metadata: {
name: recipeName,
namespace: this.deps.config.sandboxNamespace,
uid: recipeUid,
},
spec,
}
// Build server registry from explicit mcpServers plus transport workloads.
// The runner reads this mounted config as the source of truth for manual SDK
// MCP calls; the coordinator request is only an invocation and cannot widen
// endpoints or tools at runtime.
const resolvedMcpServersForConfig = new Map<
string,
{ id: string; endpoint: string; transport?: string }
>()
const serverRegistry = new Map<string, { name: string; url: string }>()
for (const srv of (spec.mcpServers ?? []) as Array<{
id: string
endpoint?: string
transport?: string
}>) {
if (!srv.id || !srv.endpoint) continue
resolvedMcpServersForConfig.set(srv.id, {
id: srv.id,
endpoint: srv.endpoint,
...(srv.transport && { transport: srv.transport }),
})
serverRegistry.set(srv.id, { name: srv.id, url: srv.endpoint })
}
for (const workload of spec.workloads ?? []) {
if (!workload.transport || !workload.port) continue
const resourceName = resolveWorkloadRuntimeResourceName(recipeRef, workload)
const path = workload.transport.path ?? '/mcp'
const endpoint = `http://${resourceName}.${this.deps.config.mcpServerNamespace}.svc.cluster.local:${workload.port}${path}`
resolvedMcpServersForConfig.set(workload.id, {
id: workload.id,
endpoint,
transport: workload.transport.type,
})
serverRegistry.set(workload.id, { name: workload.id, url: endpoint })
}
const resolvedWorkloads = (spec.workloads ?? []).map(workload => {
const resourceName = resolveWorkloadRuntimeResourceName(recipeRef, workload)
const serviceName =
workload.type === 'statefulset'
? resolveStatefulSetHeadlessServiceName(recipeRef, workload)
: resourceName
return {
id: workload.id,
type: workload.type,
port: workload.port,
serviceName,
resourceName,
namespace: this.deps.config.sandboxNamespace,
host: `${serviceName}.${this.deps.config.sandboxNamespace}.svc.cluster.local`,
...(workload.transport && { transport: workload.transport }),
}
})
const workflowConfig = {
name: recipeName,
namespace: this.deps.config.sandboxNamespace,
recipeName,
workloads: resolvedWorkloads,
...(spec.coordinatorImage && { coordinatorImage: spec.coordinatorImage }),
...(spec.runtimeEgress && { runtimeEgress: spec.runtimeEgress }),
...(effectiveInputs && { inputs: effectiveInputs }),
...(spec.inputContract && { inputContract: spec.inputContract }),
...(spec.agent && {
agent: {
provider: spec.agent.provider,
model: spec.agent.model,
},
}),
steps: (spec.steps ?? []).map(s => ({
id: s.id,
...(s.instruction && { instruction: substituteInputs(s.instruction) }),
...(s.run && {
run: {
type: 'snippet',
language: s.run.language,
code: s.run.code,
...(s.run.capabilities && { capabilities: s.run.capabilities }),
},
}),
dependsOn: s.dependsOn ?? [],
timeoutSeconds: s.timeoutSeconds ?? 300,
backoffSeconds: s.backoffSeconds ?? 30,
...(s.maxRetries !== undefined && { maxRetries: s.maxRetries }),
// Resolve string ID references → { name, url } objects so the coordinator can
// connect to the correct MCP server endpoint without knowing K8s service names.
mcpServers: (s.mcpServers ?? [])
.map((ref: unknown) => {
if (typeof ref === 'string') {
const resolved = serverRegistry.get(ref)
if (!resolved) {
this.log.warn(`Step "${s.id}" references unknown mcpServer ID "${ref}" — skipping`)
return null
}
return resolved
}
return ref as { name: string; url: string }
})
.filter((x): x is { name: string; url: string } => x !== null),
...(s.allowedTools && { allowedTools: s.allowedTools }),
...(s.maxIterations && { maxIterations: s.maxIterations }),
...(s.toolChoice !== undefined && { toolChoice: s.toolChoice }),
...(s.agent && {
agent: {
...(s.agent.provider && { provider: s.agent.provider }),
...(s.agent.model && { model: s.agent.model }),
...(s.agent.soul && { soul: s.agent.soul }),
},
}),
...(s.requiresApproval && { requiresApproval: s.requiresApproval }),
})),
mcpServers: [...resolvedMcpServersForConfig.values()],
output: spec.output ?? { destination: 'stdout' },
}
const configMapName = `${recipeName}-workflow-config`
const ns = this.deps.config.sandboxNamespace
const configMap: k8s.V1ConfigMap = {
apiVersion: 'v1',
kind: 'ConfigMap',
metadata: {
name: configMapName,
namespace: ns,
labels: {
'clerum.io/recipe': recipeName,
'clerum.io/managed-by': 'wrc',
},
// No ownerRef: cross-namespace ownerRefs are deleted by K8s GC (1.24+).
// Cleanup handled explicitly by reconcileDelete() finalizer.
},
data: {
'config.json': JSON.stringify(workflowConfig, null, 2),
},
}
// Use create-or-replace so that spec changes (new workloads, corrected endpoints)
// are reflected in the ConfigMap and picked up by the next coordinator Pod.
try {
await this.deps.coreApi.createNamespacedConfigMap({ namespace: ns, body: configMap })
this.log.info(`Created ConfigMap "${configMapName}"`)
} catch (createErr) {
if (getErrorCode(createErr) !== 409) throw createErr
const existing = await this.deps.coreApi.readNamespacedConfigMap({
name: configMapName,
namespace: ns,
})
configMap.metadata!.resourceVersion = existing.metadata?.resourceVersion
await this.deps.coreApi.replaceNamespacedConfigMap({
name: configMapName,
namespace: ns,
body: configMap,
})
this.log.info(`Updated ConfigMap "${configMapName}"`)
}
}
// ─── Recipe-Type Annotation ─────────────────────────────────────────
private async setRecipeTypeAnnotation(
recipeName: string,
namespace: string,
classification: RecipeClassification
): Promise<void> {
try {
// Merge-patch creates the annotations map when it does not exist yet.
await this.deps.customApi.patchNamespacedCustomObject(
{
group: CRD_GROUP,
version: CRD_VERSION,
namespace,
plural: WORKFLOWRECIPE_PLURAL,
name: recipeName,
body: {
metadata: {
annotations: {
'clerum.io/recipe-type': classification,
},
},
},
},
{ middleware: [k8s.setHeaderMiddleware('Content-Type', 'application/merge-patch+json')] }
)
} catch {
// Non-fatal — annotation is informational
}
}
// ─── Helpers ────────────────────────────────────────────────────────
private async createIfNotExists(
createFn: () => Promise<unknown>,
label: string
): Promise<boolean> {
try {
await createFn()
this.log.info(`Created ${label}`)
return true
} catch (error: unknown) {
if (getErrorCode(error) === 409) {
this.log.info(`${label} already exists (skip)`)
return false
} else {
throw error
}
}
}
private async readNetworkPolicyOrNull(
name: string,
namespace: string
): Promise<k8s.V1NetworkPolicy | null> {
try {
return await this.deps.networkingApi.readNamespacedNetworkPolicy({ name, namespace })
} catch (error: unknown) {
if (getErrorCode(error) === 404) return null
throw error
}
}
/**
* Read first, then write only when the live object differs from desired. The
* write is a full-object PUT: desired spec and annotations, plus the live
* labels, finalizers and annotations outside the owned key set. Owned
* annotations that desired no longer carries are therefore removed, and
* foreign ones are kept.
*
* Each round reads the policy again. An absent policy is created; a present
* one goes through `decideNetworkPolicyConvergence`. A 409 on the create, or
* a 404 or 409 on the PUT, means another writer moved the object between
* the read and the write, so the next round reads it again. A policy that
* disappears is therefore recreated in the same pass. After
* `NETWORK_POLICY_APPLY_ROUNDS` rounds the policy is reported as
* `retry/contended`; that bound matches the write count of the previous
* create-then-replace-twice apply.
*
* Ownership conflicts, terminating objects and contention are returned, not
* thrown: the workflow reconcile catch turns every error it does not
* classify as transient into a terminal `failed`. Any other API error still
* propagates.
*/
private async applyNetworkPolicy(
policy: k8s.V1NetworkPolicy
): Promise<RunLaneNetworkPolicyApplyResult> {
const name = policy.metadata?.name
const namespace = policy.metadata?.namespace
if (!name || !namespace)
throw new Error('NetworkPolicy metadata.name and namespace are required')
for (let round = 0; round < NETWORK_POLICY_APPLY_ROUNDS; round += 1) {
const existing = await this.readNetworkPolicyOrNull(name, namespace)
if (!existing) {
try {
await this.deps.networkingApi.createNamespacedNetworkPolicy({ namespace, body: policy })
this.log.info(`Created NetworkPolicy "${name}"`)
return { policy: name, action: 'created' }
} catch (error: unknown) {
// Another writer created it after our read: read it again.
if (getErrorCode(error) !== 409) throw error
continue
}
}
const decision: NetworkPolicyConvergenceDecision = decideNetworkPolicyConvergence(
'workload-egress',
policy,
existing,
RUNTIME_HTTP_EGRESS_OWNED_ANNOTATIONS
)
switch (decision.action) {
case 'unchanged':
this.log.info(`Unchanged NetworkPolicy "${name}"; skipping update`, { namespace })
return { policy: name, action: 'unchanged' }
case 'conflict':
this.log.warn(`NetworkPolicy "${name}" is not owned by this workflow; leaving it`, {
namespace,
reason: decision.reason,
})
return { policy: name, action: 'conflict', reason: decision.reason }
case 'retry':
this.log.warn(`NetworkPolicy "${name}" cannot be written yet; retrying later`, {
namespace,
reason: decision.reason,
})
return { policy: name, action: 'retry', reason: decision.reason }
case 'replace':
try {
await this.deps.networkingApi.replaceNamespacedNetworkPolicy({
name,
namespace,
body: buildNetworkPolicyReplacement(
policy,
existing,
RUNTIME_HTTP_EGRESS_OWNED_ANNOTATIONS
),
})
this.log.info(`Updated NetworkPolicy "${name}"`, { namespace, reason: decision.reason })
return { policy: name, action: 'replaced' }
} catch (error: unknown) {
// 404: deleted after our read. 409: changed after our read. Either
// way the next round reads it again and decides from that.
const code = getErrorCode(error)
if (code !== 404 && code !== 409) throw error
}
break
default: {
// Unreachable while every action has a case; a new action fails to
// compile here instead of falling through to a write.
const unhandled: never = decision
throw new Error(
`NetworkPolicy "${name}": unhandled convergence action ${String(
(unhandled as { action: unknown }).action
)}`
)
}
}
}
// Contention, not a defect: another writer moved the policy after every
// one of our reads. Leave it for a later pass.
this.log.warn(
`NetworkPolicy "${name}" is still contended after ${NETWORK_POLICY_APPLY_ROUNDS} write attempts; retrying later`,
{ namespace, reason: 'contended' }
)
return { policy: name, action: 'retry', reason: 'contended' }
}
private async safeDelete(deleteFn: () => Promise<unknown>): Promise<void> {
try {
await deleteFn()
} catch (error: unknown) {
if (getErrorCode(error) !== 404) {
this.log.error('Delete failed', {
error: error instanceof Error ? error.message : String(error),
})
}
}
}
/**
* SDK capability teardown is a revocation boundary, not best-effort
* housekeeping. Ignore only an already-absent object; propagate every other
* failure so the reconciler keeps the previous capability state and
* requeues instead of publishing a false `disabled` result.
*/
private async teardownDelete(deleteFn: () => Promise<unknown>): Promise<void> {
try {
await deleteFn()
} catch (error: unknown) {
if (getErrorCode(error) === 404) return
throw error
}
}
private async deletePersistentVolumeClaimIfExists(
name: string,
namespace: string
): Promise<void> {
try {
await this.deps.coreApi.deleteNamespacedPersistentVolumeClaim({ name, namespace })
} catch (error: unknown) {
if (getErrorCode(error) !== 404) throw error
}
}
/** Replaces coordinator JWTs in-place when either expires near the refresh window. */
private async refreshCoordinatorTokenIfExpiring(
recipeName: string,
recipeNamespace: string,
options: CoordinatorTokenRefreshOptions = {}
): Promise<void> {
try {
const includeMcpHostToken = options.includeMcpHostToken ?? true
const expectedWrcSubject = options.useCustomCoordinatorWrcToken
? 'custom-coordinator'
: 'coordinator'
const existing = await this.deps.coreApi.readNamespacedSecret({
name: `wf-${recipeName}-coordinator-token`,
namespace: recipeNamespace,
})
const rawMcpHost = existing.data?.['mcp-host-token']
const rawWrc = existing.data?.['wrc-token']
const rawSnippetRunner = existing.data?.['snippet-runner-token']
const rawGfs = existing.data?.['gfs-token']
const gfsJwt = rawGfs ? Buffer.from(rawGfs, 'base64' as BufferEncoding).toString('utf-8') : ''
const mcpHostJwt = rawMcpHost ? Buffer.from(rawMcpHost, 'base64').toString('utf-8') : ''
const wrcJwt = rawWrc ? Buffer.from(rawWrc, 'base64').toString('utf-8') : ''
const mcpHostExp = includeMcpHostToken ? WorkflowReconciler.jwtExp(mcpHostJwt) : 0
const wrcExp = WorkflowReconciler.jwtExp(wrcJwt)
const wrcSubject = WorkflowReconciler.jwtSubject(wrcJwt)
const nowSecs = Math.floor(Date.now() / 1000)
const refreshBeforeSeconds = this.runtimeTokenFileRefreshBeforeSeconds
const expectedGfsSubject = options.gfsSubject ?? `host:3rd:${recipeNamespace}/${recipeName}`
const expectedGfsScopes = options.gfsScopes ?? []
const gfsSubject = rawGfs ? WorkflowReconciler.jwtSubject(gfsJwt) : ''
const gfsScopes = rawGfs ? WorkflowReconciler.jwtGfsScopes(gfsJwt) : []
const gfsSubjectMismatch =
Boolean(rawGfs) && Boolean(options.gfsSubject) && gfsSubject !== expectedGfsSubject
const gfsScopesMismatch =
Boolean(rawGfs) &&
Boolean(options.gfsScopes) &&
!gfsScopesEqual(gfsScopes, expectedGfsScopes)
const gfsExp = options.gfsToken ? WorkflowReconciler.jwtExp(gfsJwt) : 0
const needsGfsToken =
Boolean(options.gfsToken) &&
(gfsExp === 0 ||
gfsExp - nowSecs < refreshBeforeSeconds ||
gfsSubjectMismatch ||
gfsScopesMismatch)
const shouldPruneGfsToken = !options.gfsToken && Boolean(rawGfs)
const needsMcpHostRefresh =
includeMcpHostToken && (mcpHostExp === 0 || mcpHostExp - nowSecs < refreshBeforeSeconds)
const needsWrcRefresh =
wrcExp === 0 || wrcExp - nowSecs < refreshBeforeSeconds || wrcSubject !== expectedWrcSubject
const shouldPruneMcpHostToken = !includeMcpHostToken && Boolean(rawMcpHost)
const needsSnippetRunnerToken =
options.includeSnippetRunnerToken === true && !rawSnippetRunner
const shouldPruneSnippetRunnerToken =
options.includeSnippetRunnerToken !== true && Boolean(rawSnippetRunner)
if (
needsMcpHostRefresh ||
needsWrcRefresh ||
shouldPruneMcpHostToken ||
needsSnippetRunnerToken ||
shouldPruneSnippetRunnerToken ||
needsGfsToken ||
shouldPruneGfsToken
) {
const data: Record<string, string | null> = {}
if (needsMcpHostRefresh) {
const freshMcpHost = await this.deps.tokenFactory.signCoordinatorToMcpHostToken(
recipeName,
recipeNamespace
)
data['mcp-host-token'] = Buffer.from(freshMcpHost).toString('base64')
} else if (shouldPruneMcpHostToken) {
data['mcp-host-token'] = null
}
if (needsWrcRefresh) {
const freshWrc = options.useCustomCoordinatorWrcToken
? await this.deps.tokenFactory.signCustomCoordinatorToWrcToken(
recipeName,
recipeNamespace
)
: await this.deps.tokenFactory.signCoordinatorToWrcToken(recipeName, recipeNamespace)
data['wrc-token'] = Buffer.from(freshWrc).toString('base64')
}
if (needsSnippetRunnerToken) {
data['snippet-runner-token'] = Buffer.from(
randomBytes(32).toString('base64url')
).toString('base64')
} else if (shouldPruneSnippetRunnerToken) {
data['snippet-runner-token'] = null
}
if (needsGfsToken && options.gfsToken) {
data['gfs-token'] = Buffer.from(options.gfsToken).toString('base64' as BufferEncoding)
} else if (shouldPruneGfsToken) {
data['gfs-token'] = null
}
await this.deps.coreApi.patchNamespacedSecret(
{
name: `wf-${recipeName}-coordinator-token`,
namespace: recipeNamespace,
body: {
data,
},
},
{ middleware: [k8s.setHeaderMiddleware('Content-Type', 'application/merge-patch+json')] }
)
const mcpHostRemaining =
includeMcpHostToken && mcpHostExp > 0
? `${mcpHostExp - nowSecs}s`
: includeMcpHostToken
? 'unknown(parse error)'
: 'not required'
const wrcRemaining = wrcExp > 0 ? `${wrcExp - nowSecs}s` : 'unknown(parse error)'
this.log.info(
`Refreshed coordinator tokens for ${recipeName} (mcp-host exp in ${mcpHostRemaining}, wrc exp in ${wrcRemaining})`
)
}
} catch (err) {
this.log.error(`Failed to refresh coordinator tokens for ${recipeName}`, {
error: err instanceof Error ? err.message : String(err),
})
}
}
/**
* Provisions the mcp-host-runtime-token Secret. JWT claims stay scoped to the
* WorkflowRecipe namespace; the network call to the approval gateway is
* deferred until the Secret is missing or expiring.
*/
private async ensureMcpHostRuntimeTokenSecret(
recipeNamespace: string,
recipeName: string,
runtimeScopeRecipeName = recipeName,
workflowControlScopes: EffectiveWorkflowControlScope[] = [],
gfsScopes: WorkflowRecipeGfsScope[] = ['gfs.read'],
codexScopeUncertain = false,
grokScopeUncertain = false
): Promise<McpHostRuntimeTokenRefreshResult> {
const secretName = `wf-${recipeName}-mcp-host-runtime-tokens`
const sandboxNamespace = this.deps.config.sandboxNamespace
try {
const existing = await this.deps.coreApi.readNamespacedSecret({
name: secretName,
namespace: sandboxNamespace,
})
return await this.refreshMcpHostRuntimeTokensIfExpiring(
recipeNamespace,
recipeName,
runtimeScopeRecipeName,
workflowControlScopes,
gfsScopes,
existing,
codexScopeUncertain,
grokScopeUncertain
)
} catch (err) {
if (getErrorCode(err) !== 404) throw err
}
// First mint: there is no live JWT whose Codex decision could be preserved,
// so an uncertain snapshot mints without the scope (fail-closed). The real
// remint happens once the ConfigMap is readable again.
const mcpHostRuntimeTokens = await issueMcpHostRuntimeTokens(
recipeNamespace,
runtimeScopeRecipeName,
workflowControlScopes
)
const mcpHostControlToken = mcpHostRuntimeTokens.mcpHostControlToken
const gfs = await mintRecipeHostGfsToken(recipeNamespace, runtimeScopeRecipeName, {
scopes: gfsScopes,
})
const mcpHostRuntimeSecret = buildMcpHostRuntimeTokenSecret(
recipeName,
mcpHostRuntimeTokens.accessToken,
mcpHostRuntimeTokens.refreshToken,
sandboxNamespace,
mcpHostControlToken,
gfs.token
)
try {
await this.deps.coreApi.createNamespacedSecret({
namespace: sandboxNamespace,
body: mcpHostRuntimeSecret,
})
this.log.info(`Created Secret "${secretName}"`)
return NO_MCP_HOST_RUNTIME_TOKEN_REFRESH
} catch (err) {
if (getErrorCode(err) !== 409) throw err
const existing = await this.deps.coreApi.readNamespacedSecret({
name: secretName,
namespace: sandboxNamespace,
})
const tokenRefresh = await this.refreshMcpHostRuntimeTokensIfExpiring(
recipeNamespace,
recipeName,
runtimeScopeRecipeName,
workflowControlScopes,
gfsScopes,
existing,
codexScopeUncertain,
grokScopeUncertain
)
this.log.info(`Secret "${secretName}" already exists (skip)`)
return tokenRefresh
}
}
/**
* Refreshes the mcp-host-runtime-token Secret if either stored JWT is missing or will expire
* within the configured runtime refresh window. Refresh failures are fatal to reconcile so the
* mounted Secret never silently drifts stale after a pod restart.
*
* `codexScopeUncertain` marks a projection that had no data to decide on (see
* `resolveEffectiveControlScopes`). In that state the expected scope set
* inherits `llm:codex:execute` from the live JWT instead of dropping it: a
* transient ConfigMap read failure otherwise looks exactly like a revocation
* and remints with reason `scope`, which rolls the eager pod, discards its
* bootstrap proof, and rolls it a second time when the ConfigMap returns.
* Every other scope is still compared, and a readable ConfigMap that denies
* the grant still withdraws the scope and rolls the pod.
*/
private async refreshMcpHostRuntimeTokensIfExpiring(
recipeNamespace: string,
recipeName: string,
runtimeScopeRecipeName: string,
requestedWorkflowControlScopes: EffectiveWorkflowControlScope[],
expectedGfsScopes: WorkflowRecipeGfsScope[],
existing: k8s.V1Secret,
codexScopeUncertain = false,
grokScopeUncertain = false
): Promise<McpHostRuntimeTokenRefreshResult> {
const rawAccess = existing.data?.['mcp-host-runtime-access-token']
const rawRefresh = existing.data?.['mcp-host-runtime-refresh-token']
const accessJwt = rawAccess ? Buffer.from(rawAccess, 'base64').toString('utf-8') : ''
const refreshJwt = rawRefresh ? Buffer.from(rawRefresh, 'base64').toString('utf-8') : ''
const accessExp = WorkflowReconciler.jwtExp(accessJwt)
const refreshExp = WorkflowReconciler.jwtExp(refreshJwt)
const accessBinding = WorkflowReconciler.jwtRuntimeBinding(accessJwt)
const refreshBinding = WorkflowReconciler.jwtRuntimeBinding(refreshJwt)
const accessScopes = WorkflowReconciler.jwtWorkflowControlScopes(
accessJwt,
'workflowControlScopes'
)
const refreshScopes = WorkflowReconciler.jwtWorkflowControlScopes(
refreshJwt,
'workflowControlScopes'
)
// Uncertain neither grants nor revokes: with no readable catalog the Codex
// decision already in the live access JWT is carried forward, so a failed
// ConfigMap read cannot masquerade as a revocation. Applies to the issued
// tokens as well — expecting a scope the mint would drop would remint on
// every pass.
const preservedCodexScope =
codexScopeUncertain &&
accessScopes.includes(CODEX_EXECUTE_SCOPE) &&
!requestedWorkflowControlScopes.includes(CODEX_EXECUTE_SCOPE)
if (preservedCodexScope) {
this.log.warn('Codex catalog is undecidable; preserving the live Codex scope', {
recipeName,
runtimeScopeRecipeName,
scope: CODEX_EXECUTE_SCOPE,
})
}
const preservedGrokScope =
grokScopeUncertain &&
accessScopes.includes(GROK_EXECUTE_SCOPE) &&
!requestedWorkflowControlScopes.includes(GROK_EXECUTE_SCOPE)
if (preservedGrokScope) {
this.log.warn('Grok catalog is undecidable; preserving the live Grok scope', {
recipeName,
runtimeScopeRecipeName,
scope: GROK_EXECUTE_SCOPE,
})
}
const workflowControlScopes: EffectiveWorkflowControlScope[] = [
...requestedWorkflowControlScopes,
...(preservedCodexScope ? [CODEX_EXECUTE_SCOPE] : []),
...(preservedGrokScope ? [GROK_EXECUTE_SCOPE] : []),
]
const rawMcpHostControl = existing.data?.['mcp-host-workflow-control-token']
const mcpHostControlJwt = rawMcpHostControl
? Buffer.from(rawMcpHostControl, 'base64').toString('utf-8')
: ''
const mcpHostControlExp = WorkflowReconciler.jwtExp(mcpHostControlJwt)
const rawGfsAccess = existing.data?.['mcp-host-gfs-token']
const gfsAccessJwt = rawGfsAccess ? Buffer.from(rawGfsAccess, 'base64').toString('utf-8') : ''
const gfsAccessExp = WorkflowReconciler.jwtExp(gfsAccessJwt)
const gfsAccessSub = WorkflowReconciler.jwtSubject(gfsAccessJwt)
const gfsAccessScopes = WorkflowReconciler.jwtGfsScopes(gfsAccessJwt)
const controlBinding = WorkflowReconciler.jwtRuntimeBinding(mcpHostControlJwt)
const controlScopes = WorkflowReconciler.jwtWorkflowControlScopes(mcpHostControlJwt)
const nowSecs = Math.floor(Date.now() / 1000)
const expectedRuntimeHostRef = `${recipeNamespace}/${runtimeScopeRecipeName}`
const expectedControlHostRef = expectedRuntimeHostRef
const expectedGfsSubject = `host:3rd:${recipeNamespace}/${runtimeScopeRecipeName}`
const runtimeBindingMismatch =
!WorkflowReconciler.matchesRuntimeBinding(
accessBinding,
recipeNamespace,
runtimeScopeRecipeName,
expectedRuntimeHostRef
) ||
!WorkflowReconciler.matchesRuntimeBinding(
refreshBinding,
recipeNamespace,
runtimeScopeRecipeName,
expectedRuntimeHostRef
)
const controlBindingMismatch =
!!rawMcpHostControl &&
!WorkflowReconciler.matchesRuntimeBinding(
controlBinding,
recipeNamespace,
runtimeScopeRecipeName,
expectedControlHostRef
)
const controlScopesMismatch = !workflowControlScopesEqual(controlScopes, workflowControlScopes)
const runtimeScopesMismatch =
!workflowControlScopesEqual(accessScopes, workflowControlScopes) ||
!workflowControlScopesEqual(refreshScopes, workflowControlScopes)
const gfsSubjectMismatch = !!rawGfsAccess && gfsAccessSub !== expectedGfsSubject
const gfsScopesMismatch = !gfsScopesEqual(gfsAccessScopes, expectedGfsScopes)
const refreshBeforeSeconds = this.runtimeTokenFileRefreshBeforeSeconds
const needsMcpHostRuntimeRefresh =
accessExp === 0 ||
refreshExp === 0 ||
runtimeBindingMismatch ||
runtimeScopesMismatch ||
accessExp - nowSecs < refreshBeforeSeconds ||
refreshExp - nowSecs < refreshBeforeSeconds
const needsWorkflowControlRefresh =
mcpHostControlExp === 0 ||
controlBindingMismatch ||
controlScopesMismatch ||
mcpHostControlExp - nowSecs < refreshBeforeSeconds
const needsGfsAccessRefresh =
gfsAccessExp === 0 ||
gfsSubjectMismatch ||
gfsScopesMismatch ||
gfsAccessExp - nowSecs < refreshBeforeSeconds
if (!needsMcpHostRuntimeRefresh && !needsWorkflowControlRefresh && !needsGfsAccessRefresh) {
const tokenGeneration = readMcpHostRuntimeTokenGeneration(existing)
return tokenGeneration
? { reminted: false, tokenGeneration }
: NO_MCP_HOST_RUNTIME_TOKEN_REFRESH
}
const refreshReason: McpHostRuntimeTokenRefreshReason =
runtimeScopesMismatch || controlScopesMismatch
? 'scope'
: runtimeBindingMismatch || controlBindingMismatch
? 'binding'
: 'ttl'
if (
runtimeBindingMismatch ||
runtimeScopesMismatch ||
controlBindingMismatch ||
controlScopesMismatch ||
gfsSubjectMismatch ||
gfsScopesMismatch
) {
this.log.warn('mcpHost runtime token binding drift detected; reissuing tokens', {
recipeName,
runtimeScopeRecipeName,
accessBinding,
refreshBinding,
accessScopes,
refreshScopes,
controlBinding,
controlScopes,
expectedControlScopes: workflowControlScopes,
gfsAccessSub,
expectedGfsSubject,
gfsAccessScopes,
expectedGfsScopes,
})
}
try {
const data: Record<string, string> = {}
const refreshedMcpHostRuntimeTokens = needsMcpHostRuntimeRefresh
? await issueMcpHostRuntimeTokens(
recipeNamespace,
runtimeScopeRecipeName,
workflowControlScopes
)
: null
if (refreshedMcpHostRuntimeTokens) {
data['mcp-host-runtime-access-token'] = Buffer.from(
refreshedMcpHostRuntimeTokens.accessToken
).toString('base64')
data['mcp-host-runtime-refresh-token'] = Buffer.from(
refreshedMcpHostRuntimeTokens.refreshToken
).toString('base64')
}
if (needsWorkflowControlRefresh || refreshedMcpHostRuntimeTokens) {
const mcpHostControlToken = refreshedMcpHostRuntimeTokens
? refreshedMcpHostRuntimeTokens.mcpHostControlToken
: await this.issueMcpHostControlToken(
recipeNamespace,
runtimeScopeRecipeName,
workflowControlScopes
)
data['mcp-host-workflow-control-token'] =
Buffer.from(mcpHostControlToken).toString('base64')
}
if (needsGfsAccessRefresh) {
const gfs = await mintRecipeHostGfsToken(recipeNamespace, runtimeScopeRecipeName, {
scopes: expectedGfsScopes,
})
data['mcp-host-gfs-token'] = Buffer.from(gfs.token).toString('base64')
}
const currentGeneration = readMcpHostRuntimeTokenGeneration(existing)
const tokenGeneration =
refreshReason === 'scope' || refreshReason === 'binding'
? nextMcpHostRuntimeTokenGeneration(currentGeneration)
: currentGeneration
await this.deps.coreApi.patchNamespacedSecret(
{
name: `wf-${recipeName}-mcp-host-runtime-tokens`,
namespace: this.deps.config.sandboxNamespace,
body: {
data,
...(tokenGeneration && (refreshReason === 'scope' || refreshReason === 'binding')
? {
metadata: {
annotations: {
[MCP_HOST_RUNTIME_TOKEN_GENERATION_ANNOTATION]: tokenGeneration,
},
},
}
: {}),
},
},
{ middleware: [k8s.setHeaderMiddleware('Content-Type', 'application/merge-patch+json')] }
)
this.log.info(
`Refreshed workflow auth Secret for ${recipeName} (mcpHostRuntime=${needsMcpHostRuntimeRefresh}, mcpHostControl=${needsWorkflowControlRefresh}, reason=${refreshReason})`
)
// Do not delete ${recipeName}-mcp-host here. This helper also runs on the
// in-progress step-based credential refresh path, where a mid-run pod
// delete is not recreated. Eager sdk-only hosts roll via
// eagerMcpHostRequiresTokenRoll → rollEagerSdkMcpHostPod, including when
// this pass only reports the Secret generation residue.
return {
reminted: true,
reason: refreshReason,
...(tokenGeneration ? { tokenGeneration } : {}),
}
} catch (err) {
this.log.error(`Failed to refresh mcpHost runtime tokens for ${recipeName}`, {
error: err instanceof Error ? err.message : String(err),
})
throw err
}
}
private async issueMcpHostControlToken(
recipeNamespace: string,
recipeName: string,
workflowControlScopes: EffectiveWorkflowControlScope[]
): Promise<string> {
try {
return await issueMcpHostWorkflowControlToken(
recipeNamespace,
recipeName,
workflowControlScopes
)
} catch (err) {
this.log.warn(
`Failed to issue mcpHost control token for "${recipeName}" — workflow tools will be unavailable: ${err instanceof Error ? err.message : err}`
)
throw err
}
}
/** Decodes the `exp` claim from a JWT without signature verification. Returns 0 on parse error. */
private static jwtExp(jwt: string): number {
try {
const payload = decodeJwt(jwt)
return typeof payload.exp === 'number' ? payload.exp : 0
} catch {
return 0
}
}
private static jwtRuntimeBinding(jwt: string): {
recipeNamespace: string
recipeName: string
hostRef: string
} | null {
try {
const payload = decodeJwt(jwt)
const hostRefs = Array.isArray(payload.hostRefs) ? payload.hostRefs : []
const hostRef = typeof hostRefs[0] === 'string' ? hostRefs[0].trim() : ''
const recipeNamespace =
typeof payload.recipeNamespace === 'string' ? payload.recipeNamespace.trim() : ''
const recipeName = typeof payload.recipeName === 'string' ? payload.recipeName.trim() : ''
if (!recipeNamespace || !recipeName || !hostRef) return null
return { recipeNamespace, recipeName, hostRef }
} catch {
return null
}
}
private static jwtWorkflowControlScopes(
jwt: string,
claim: 'scopes' | 'workflowControlScopes' = 'scopes'
): EffectiveWorkflowControlScope[] {
try {
const payload = decodeJwt(jwt)
const scopes = Array.isArray(payload[claim]) ? payload[claim] : []
const normalized: EffectiveWorkflowControlScope[] = []
const seen = new Set<string>()
for (const scope of scopes) {
if (
typeof scope === 'string' &&
(EFFECTIVE_WORKFLOW_CONTROL_SCOPE_ORDER as string[]).includes(scope) &&
!seen.has(scope)
) {
seen.add(scope)
normalized.push(scope as EffectiveWorkflowControlScope)
}
}
return EFFECTIVE_WORKFLOW_CONTROL_SCOPE_ORDER.filter(scope => normalized.includes(scope))
} catch {
return []
}
}
private static jwtGfsScopes(jwt: string): WorkflowRecipeGfsScope[] {
try {
const payload = decodeJwt(jwt)
const scopes = Array.isArray(payload.scopes) ? payload.scopes : []
const normalized: WorkflowRecipeGfsScope[] = []
const seen = new Set<string>()
for (const scope of scopes) {
if (
typeof scope === 'string' &&
(GFS_SCOPE_ORDER as string[]).includes(scope) &&
!seen.has(scope)
) {
seen.add(scope)
normalized.push(scope as WorkflowRecipeGfsScope)
}
}
return GFS_SCOPE_ORDER.filter(scope => normalized.includes(scope))
} catch {
return []
}
}
/** Decodes the JWT subject without signature verification. Returns empty string on parse error. */
private static jwtSubject(jwt: string): string {
try {
const payload = decodeJwt(jwt)
return typeof payload.sub === 'string' ? payload.sub : ''
} catch {
return ''
}
}
private static matchesRuntimeBinding(
binding: { recipeNamespace: string; recipeName: string; hostRef: string } | null,
recipeNamespace: string,
recipeName: string,
hostRef: string
): boolean {
return (
binding?.recipeNamespace === recipeNamespace &&
binding.recipeName === recipeName &&
binding.hostRef === hostRef
)
}
}