Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
/**
* Kubernetes client for watching WorkflowRecipe CRDs.
* Follows the context-mapper/src/k8sClient.ts pattern.
*
* Production: Watches WorkflowRecipe CRDs via K8s Watch API with auto-reconnect.
* Dev mode: Reads recipes from CLERUM_RECIPES env var JSON.
*
* Fase 4 (DB-first runs): the legacy `WorkflowRunReconciler` (CRD watch +
* child creation + reaper) is replaced by `DbRunProcessor`, which consumes
* `workflow_runs` rows via Postgres LISTEN/NOTIFY. The child-recipe creation
* logic moves here as a `ChildRecipeCreator` adapter that closes over
* `customApi`, keeping the processor fully K8s-unaware.
*/
import * as k8s from '@kubernetes/client-node'
import type { Pool } from 'pg'
import { loadConfig } from './config'
import { createGrantUpdateListenerPool, getPool, initDb } from './db'
import {
type GovernedTraceReporter,
type WorkflowInfrastructureTelemetryProjection,
createGovernedTraceReporter,
} from './governedTraceReporter'
import { K8sSecretWatchLoop } from './k8sSecretWatchLoop'
import { createLogger } from './observability/logger'
import { CRD_GROUP, CRD_VERSION, WORKFLOWRECIPE_PLURAL } from './reconciler/crdConstants'
import {
type ChildRecipeCreator,
type ChildRecipeExistenceChecker,
type DbRunProcessor,
type DbRunRow,
createDbRunProcessor,
} from './reconciler/dbRunProcessor'
import {
type GrantUpdateListener,
createGrantUpdateListener,
} from './reconciler/grantUpdateListener'
import { getErrorCode } from './reconciler/k8sErrors'
import { RecipeEventQueue } from './reconciler/recipeEventQueue'
import {
parseClusterLocalFqdn,
recipeHasBackgroundAccessClient,
} from './reconciler/resourceBuilder'
import { SecretReverseIndex } from './reconciler/secretReverseIndex'
import { SecretWatcher } from './reconciler/secretWatcher'
import {
type ReconcileResult,
TRANSIENT_REQUEUE_BASE_MS,
TRANSPORT_NETWORK_CONDITION_TYPE,
WorkflowRecipeReconciler,
hasPendingTransportNetworkReadiness,
} from './reconciler/workflowRecipeReconciler'
import {
type StatusCondition,
WorkflowRecipeCRD,
WorkflowRecipeSpec,
type WorkloadStatus,
} from './types'
import { createDbRunChildRecipe } from './workflow/dbRunChildRecipeCreator'
import type { JwtTokenFactory } from './workflow/jwtTokenFactory'
import {
CODEX_CONNECTION_REF_ANNOTATION,
SUBSCRIPTION_CONNECTION_REF_ANNOTATION,
} from './workflow/llmAllowedModelsSnapshot'
import { networkPolicyConditionsChanged } from './workflow/workflowReconciler'
function grantIdentityFingerprint(
recipe: { metadata?: { annotations?: Record<string, string> } } | undefined
): string {
const annotations = recipe?.metadata?.annotations ?? {}
return `${annotations[SUBSCRIPTION_CONNECTION_REF_ANNOTATION] ?? ''}\0${annotations[CODEX_CONNECTION_REF_ANNOTATION] ?? ''}`
}
const PLURAL = 'workflowrecipes'
const MIN_RUNTIME_CREDENTIAL_REFRESH_INTERVAL_MS = 5_000
const MAX_RUNTIME_CREDENTIAL_REFRESH_INTERVAL_MS = 30_000
const WORKLOAD_STATUS_REFRESH_INTERVAL_MS = 30_000
const WORKFLOW_RUN_ID_LABEL = 'clerum.io/workflow-run-id'
// issue #375: how long a published Plugin Workload SDK `verifiedAt` may age
// before a steady-state reconcile refreshes it, even when every semantic field
// is unchanged. `verifiedAt` is recomputed on every bootstrap-proof parse, so
// without a throttle a fresh timestamp each pass would patch on every reconcile
// (≤30s) and spin a MODIFIED→reconcile loop. 5 min keeps the liveness signal
// current while capping steady-state churn at ≤1 patch / 5 min per recipe.
const VERIFIED_AT_REFRESH_MS = 5 * 60_000
// Bound the fan-out of the periodic external-egress requeue: the per-recipe
// eventQueue already serializes each recipe, but without a global cap a refresh
// pass over N recipes would enqueue N reconciles at once. This caps the number of
// reconciles started concurrently per pass; the rest queue and drain in order.
const EXTERNAL_EGRESS_REFRESH_MAX_CONCURRENCY = 8
export interface WorkflowRecipeWatchObject {
metadata: {
name: string
namespace?: string
uid?: string
resourceVersion?: string
generation?: number
deletionTimestamp?: string
finalizers?: string[]
annotations?: Record<string, string>
labels?: Record<string, string>
ownerReferences?: WorkflowRecipeCRD['metadata']['ownerReferences']
}
spec: WorkflowRecipeSpec
status?: WorkflowRecipeCRD['status']
}
export function workflowRecipeFromWatchObject(
apiObj: WorkflowRecipeWatchObject,
fallbackNamespace: string
): WorkflowRecipeCRD {
return {
apiVersion: `${CRD_GROUP}/${CRD_VERSION}` as 'clerum.io/v1alpha1',
kind: 'WorkflowRecipe',
metadata: {
name: apiObj.metadata.name,
namespace: apiObj.metadata.namespace || fallbackNamespace,
uid: apiObj.metadata.uid,
resourceVersion: apiObj.metadata.resourceVersion,
generation: apiObj.metadata.generation,
finalizers: apiObj.metadata.finalizers,
deletionTimestamp: apiObj.metadata.deletionTimestamp,
annotations: apiObj.metadata.annotations,
labels: apiObj.metadata.labels,
ownerReferences: apiObj.metadata.ownerReferences,
},
spec: apiObj.spec,
status: apiObj.status,
}
}
export function shouldPatchRecipeStatus(
recipe: WorkflowRecipeCRD,
result: ReconcileResult
): boolean {
if (result.skipStatusPatch) return false
if (recipe.status?.phase !== result.phase) return true
if (recipe.status?.message !== result.message) return true
if (workloadStatusesChanged(recipe, result)) return true
// Merge-patching status maps preserves fields omitted by the next patch.
// Keep the SDK capability message aligned with its owned condition so a
// previous status-only refresh cannot leave a contradictory
// `state=validated` + `message=bootstrap not ready` record behind.
if (pluginWorkloadSdkStatusMessageChanged(recipe)) return true
// issue #375: publish a computed Plugin Workload SDK capability transition
// (e.g. awaiting_policy→validated) even when nothing else observable changed.
// Without this the reconciler computes the new state but no patch is emitted
// and the CRD keeps proclaiming the stale state until an incidental diff.
if (pluginWorkloadSdkProjectionChanged(recipe, result)) return true
if (
ownedConditionsChanged(recipe.status?.conditions, result.internalDependencyConditions, [
'InternalDependenciesReady',
])
) {
return true
}
// Parent workflow recipes may remain active while stale execution status is
// cleared. That still needs one status patch even though phase is unchanged.
if (result.clearWorkflowExecution && recipe.status?.workflowExecution) return true
if (
result.transportNetworkConditions !== undefined &&
ownedConditionsChanged(recipe.status?.conditions, result.transportNetworkConditions, [
TRANSPORT_NETWORK_CONDITION_TYPE,
])
)
return true
// A NetworkPolicy ownership conflict or a pending retry changes neither
// phase nor message, so without this clause those conditions would not be
// published or cleared on a pass where nothing else changed. An undefined
// field means the pass never reached the apply, so it does not force a patch.
if (
result.networkPolicyOwnershipConditions !== undefined &&
networkPolicyConditionsChanged(
recipe.status?.conditions,
result.networkPolicyOwnershipConditions
)
)
return true
return false
}
/**
* issue #375: decide whether the freshly-computed Plugin Workload SDK capability
* projection (carried on `result.pluginWorkloadSdkProjection`, built once by the
* watcher via the reconciler) differs from what is currently persisted in
* `status.pluginWorkloadSdk` in a way that must be published.
*
* Semantic fields are compared explicitly (NEVER a deep-equal that would drag in
* timestamps): `state`, `message`, `policyRevision`, `policyHash`,
* `defaultTargetRef`, `bootstrapPodUid`, `bootstrapContractVersion`,
* `bootstrapProvider`, `bootstrapModel`, and the `promptBridge`/
* `clientNotifications` family flags. A change in any of them (e.g.
* awaiting_policy→validated on grant creation, validated→awaiting_policy on
* revocation, or an in-place re-bootstrap against a corrected model/provider
* with the same pod) forces a patch immediately — `bootstrapProvider`/
* `bootstrapModel`/`bootstrapContractVersion` are persisted values control-api
* reads at runtime, NOT timestamps, so they must not hide behind the
* `verifiedAt` throttle.
*
* `verifiedAt` and `validatedAt` are timestamp-derived and EXCLUDED from
* semantic equality — otherwise every reconcile would look "changed". Instead,
* when the projection is otherwise identical, `verifiedAt` is refreshed on a
* throttle: patch only when the newly-computed `verifiedAt` is strictly NEWER
* than the persisted one AND the persisted one is older than
* `VERIFIED_AT_REFRESH_MS` (see the inline INVARIANT). The WINDOW clause is the
* primary loop-stopper: `verifiedAt` is recomputed fresh on essentially every
* proof parse (so the projected timestamp is normally NEWER, not equal), but
* right after a refresh `persisted ≈ now`, so `now - persisted` is under the
* threshold and no patch fires until the 5-min window reopens — bounding steady
* state to ≤1 patch / 5 min. The `projected > persisted` guard is NOT that
* bound; it only defends the edge where a future projection carries a STALE
* (equal-or-older) timestamp, which must never trigger a patch.
*
* A `undefined` projection (paths that do not run the SDK lane, e.g. the
* workload-status-only refresh) short-circuits to false — that pass has no SDK
* opinion and must not force a patch.
*/
function pluginWorkloadSdkProjectionChanged(
recipe: WorkflowRecipeCRD,
result: ReconcileResult,
nowMs: number = Date.now()
): boolean {
const projected = result.pluginWorkloadSdkProjection?.capability
if (projected === undefined) return false
const current = recipe.status?.pluginWorkloadSdk
// `null` means the projection wants to CLEAR status.pluginWorkloadSdk
// (capability removed from spec). Patch only if something is still persisted.
if (projected === null) return current !== undefined
if (!current) return true
const nullable = (value: string | number | null | undefined): string | number | null =>
value ?? null
if (current.state !== projected.state) return true
if (nullable(current.message) !== nullable(projected.message)) return true
if (nullable(current.policyRevision) !== nullable(projected.policyRevision)) return true
if (nullable(current.policyHash) !== nullable(projected.policyHash)) return true
if (nullable(current.defaultTargetRef) !== nullable(projected.defaultTargetRef)) return true
if (nullable(current.bootstrapPodUid) !== nullable(projected.bootstrapPodUid)) return true
if (nullable(current.bootstrapContractVersion) !== nullable(projected.bootstrapContractVersion))
return true
if (nullable(current.bootstrapProvider) !== nullable(projected.bootstrapProvider)) return true
if (nullable(current.bootstrapModel) !== nullable(projected.bootstrapModel)) return true
if (current.promptBridge !== projected.promptBridge) return true
if (current.clientNotifications !== projected.clientNotifications) return true
// Semantic equality holds. Refresh verifiedAt on a throttle so the liveness
// signal stays current without a per-reconcile patch loop. INVARIANT: only
// refresh when the newly-computed verifiedAt is strictly NEWER than the
// persisted one AND the persisted one is older than VERIFIED_AT_REFRESH_MS.
// The WINDOW clause (nowMs - persistedMs > VERIFIED_AT_REFRESH_MS) is the
// PRIMARY loop-stopper: verifiedAt is recomputed fresh on essentially every
// proof parse, so the projected timestamp is normally NEWER — the loop is not
// stopped by the two eventually becoming equal. It is stopped because right
// after a refresh persistedMs ≈ nowMs, so (nowMs - persistedMs) is under the
// threshold and no patch fires until the 5-min window reopens (≤1 patch/5min).
// The projectedMs > persistedMs guard is NOT that bound; it only defends the
// edge where a future projection carries a STALE (equal-or-older) timestamp,
// which must never trigger a patch.
if (!projected.verifiedAt) return false
if (!current.verifiedAt) return true
const projectedMs = Date.parse(projected.verifiedAt)
// Cannot reason about an unparseable projected timestamp — do not patch.
if (Number.isNaN(projectedMs)) return false
const persistedMs = Date.parse(current.verifiedAt)
// A corrupt persisted timestamp is refreshed once from a valid projection.
if (Number.isNaN(persistedMs)) return true
return projectedMs > persistedMs && nowMs - persistedMs > VERIFIED_AT_REFRESH_MS
}
function pluginWorkloadSdkStatusMessageChanged(recipe: WorkflowRecipeCRD): boolean {
const capability = recipe.status?.pluginWorkloadSdk
if (!capability) return false
const condition = recipe.status?.conditions?.find(
candidate => candidate.type === 'PluginWorkloadSdkCapability'
)
if (typeof condition?.message !== 'string') return false
return capability.message !== condition.message
}
function workflowRunIdForTelemetry(recipe: WorkflowRecipeCRD): string | undefined {
const runId = recipe.metadata.labels?.[WORKFLOW_RUN_ID_LABEL]?.trim()
return runId || undefined
}
export function reconcileOutcomeTelemetryProjection(
recipe: WorkflowRecipeCRD,
result: ReconcileResult,
occurredAt = new Date().toISOString()
): WorkflowInfrastructureTelemetryProjection | undefined {
const runId = workflowRunIdForTelemetry(recipe)
if (!runId) return undefined
return {
sourceEventId: `workflow-reconcile:${recipe.metadata.namespace}:${recipe.metadata.name}:generation:${recipe.metadata.generation ?? 'unknown'}:${result.phase}`,
occurredAt,
telemetryType: 'reconcile_outcome',
runId,
payload: {
phase: result.phase,
status: result.skipStatusPatch ? 'skipped' : 'patched',
},
}
}
export function controllerErrorTelemetryProjection(
recipe: WorkflowRecipeCRD,
error: unknown,
occurredAt = new Date().toISOString()
): WorkflowInfrastructureTelemetryProjection | undefined {
const runId = workflowRunIdForTelemetry(recipe)
if (!runId) return undefined
const errorClass =
error instanceof Error && error.name.trim()
? error.name
: typeof error === 'object' && error !== null
? error.constructor?.name || 'unknown'
: 'unknown'
return {
sourceEventId: `workflow-controller-error:${recipe.metadata.namespace}:${recipe.metadata.name}:generation:${recipe.metadata.generation ?? 'unknown'}:${errorClass}`,
occurredAt,
telemetryType: 'controller_error',
runId,
payload: {
phase: recipe.status?.phase,
status: 'error',
error_class: errorClass,
},
}
}
function enqueueInfrastructureTelemetryBestEffort(
traceReporter: GovernedTraceReporter | null,
projection: WorkflowInfrastructureTelemetryProjection | undefined
): void {
if (!traceReporter || !projection) return
try {
traceReporter.enqueueInfrastructureTelemetry(projection)
} catch (error) {
createLogger('wrc', 'workflow-recipes').error(
'[WR-K8s] Infrastructure telemetry enqueue failed:',
{ err: error }
)
}
}
function workloadStatusesChanged(recipe: WorkflowRecipeCRD, result: ReconcileResult): boolean {
const existing = normalizeExistingWorkloadStatuses(recipe.status?.workloads)
const fresh = normalizeResultWorkloadStatuses(recipe, result)
if (existing.length !== fresh.length) return true
return fresh.some((status, index) => !sameWorkloadStatus(existing[index], status))
}
function normalizeExistingWorkloadStatuses(
statuses: WorkloadStatus[] | undefined
): WorkloadStatus[] {
return (statuses ?? []).map(status => ({
id: status.id,
type: status.type,
phase: status.phase,
ready: status.ready,
message: status.message,
}))
}
function normalizeResultWorkloadStatuses(
recipe: WorkflowRecipeCRD,
result: ReconcileResult
): WorkloadStatus[] {
return result.workloadStatuses.map(status => {
const def = (recipe.spec.workloads ?? []).find(workload => workload.id === status.id)
return {
id: status.id,
type: def?.type ?? 'deployment',
phase: status.phase,
ready: status.ready,
message: status.message,
}
})
}
function sameWorkloadStatus(a: WorkloadStatus, b: WorkloadStatus): boolean {
return (
a.id === b.id &&
a.type === b.type &&
a.phase === b.phase &&
a.ready === b.ready &&
a.message === b.message
)
}
function ownedConditionsChanged(
existing: StatusCondition[] | undefined,
fresh: StatusCondition[] | undefined,
ownedTypes: string[]
): boolean {
const existingByType = new Map(existing?.map(condition => [condition.type, condition]) ?? [])
const freshByType = new Map(fresh?.map(condition => [condition.type, condition]) ?? [])
return ownedTypes.some(type => {
const existingCondition = existingByType.get(type)
const freshCondition = freshByType.get(type)
if (!freshCondition) return existingCondition !== undefined
if (!existingCondition) return true
return !sameConditionSignal(existingCondition, freshCondition)
})
}
function sameConditionSignal(a: StatusCondition, b: StatusCondition): boolean {
return a.status === b.status && a.reason === b.reason && a.message === b.message
}
export function runtimeCredentialRefreshIntervalMs(config: {
runtimeTokenRefreshBeforeSeconds: number
}): number {
return Math.max(
MIN_RUNTIME_CREDENTIAL_REFRESH_INTERVAL_MS,
Math.min(
MAX_RUNTIME_CREDENTIAL_REFRESH_INTERVAL_MS,
Math.floor((config.runtimeTokenRefreshBeforeSeconds * 1_000) / 4)
)
)
}
export function workflowNeedsRuntimeCredentialRefresh(recipe: WorkflowRecipeCRD): boolean {
const isWorkflow = (recipe.spec.steps ?? []).length > 0
if (!isWorkflow || recipe.metadata.deletionTimestamp) return false
const workflowPhase = recipe.status?.workflowExecution?.phase
return (
workflowPhase === 'initializing' ||
workflowPhase === 'running' ||
workflowPhase === 'recovering'
)
}
export function workflowNeedsInfrastructureReconcile(recipe: WorkflowRecipeCRD): boolean {
const isWorkflow = (recipe.spec.steps ?? []).length > 0
if (recipe.metadata.deletionTimestamp) return false
const workflowPhase = recipe.status?.workflowExecution?.phase
// Restore the retry after a process restart, including stepless stdio recipes
// whose egress is exclusively HCC-owned. Pod readiness is not admission proof.
if (
hasPendingTransportNetworkReadiness(recipe) &&
!['failed', 'deprecated', 'rollback-failed'].includes(recipe.status?.phase ?? '') &&
!['completed', 'failed', 'cancelled'].includes(workflowPhase ?? '')
)
return true
// SDK-only recipes are deliberately stepless, but their eager mcp-host and
// bootstrap proof still need the full reconciler lane. The generic workload
// status-refresh path cannot represent that proof and would overwrite a
// validated SDK capability with BootstrapNotReady while leaving phase=active.
// Keep this level-triggered retry independent from workflowExecution.
if (!isWorkflow && recipe.spec.pluginWorkloadSdk) return true
if (!isWorkflow) return false
if (workflowPhase === 'initializing' || workflowPhase === 'recovering') return true
// Plugin Workload SDK eager path: the recipe sits in phase=active with no run
// (workflowExecution cleared), but the WRC must keep reconciling so the eager
// mcp-host stays alive (recreated after any pod death — restartPolicy: Never)
// and, for promptBridge, its provider gets (re-)configured once Ready. Both
// families (promptBridge AND clientNotifications-only) run the always-on eager
// SDK mcp-host, so both must re-trigger. The reconcile + /configure are
// idempotent, so a periodic level-triggered retry is safe. Without this, a
// clientNotifications-only eager mcp-host whose pod dies is never recreated
// until an unrelated spec edit, and promptBridge stays provider_unavailable.
if (recipe.spec.pluginWorkloadSdk && workflowPhase === undefined) return true
return false
}
export function workflowNeedsTerminalRunTeardown(recipe: WorkflowRecipeCRD): boolean {
const isWorkflow = (recipe.spec.steps ?? []).length > 0
if (!isWorkflow || recipe.metadata.deletionTimestamp) return false
if (!recipe.metadata.labels?.['clerum.io/workflow-run-id']?.trim()) return false
const workflowPhase = recipe.status?.workflowExecution?.phase
return (
workflowPhase === 'completed' || workflowPhase === 'failed' || workflowPhase === 'cancelled'
)
}
export function workflowNeedsWorkloadStatusRefresh(recipe: WorkflowRecipeCRD): boolean {
if (recipe.metadata.deletionTimestamp) return false
if ((recipe.spec.steps ?? []).length > 0) return false
// SDK-only recipes have capability state (including the eager bootstrap
// proof) that is owned by the full reconciler. A status-only workload read
// has no proof to carry and would corrupt the persisted SDK projection.
if (recipe.spec.pluginWorkloadSdk) return false
if ((recipe.spec.workloads ?? []).length === 0) return false
return recipe.status?.phase === 'active' || recipe.status?.phase === 'degraded'
}
/**
* Issue #637 — does this recipe project any name-based Secret (`envSecret` or
* `imagePullSecrets`) that the ownership gate must keep re-validating? Used as a
* periodic backstop: the SecretWatcher is the fast, event-driven revocation path,
* but a deterministic re-classification every refresh interval guarantees a Secret
* re-labeled foreign in ANY of the three runtime namespaces is revoked even if a
* watch event was delayed or coalesced.
*/
export function recipeReferencesNamedSecrets(recipe: WorkflowRecipeCRD): boolean {
return (recipe.spec.workloads ?? []).some(
w => Boolean(w.envSecret?.name) || (w.imagePullSecrets?.length ?? 0) > 0
)
}
/**
* Issue #637 — a steady-state WORKFLOW recipe (steps>0) that projects a name-based
* Secret needs the SAME periodic ownership re-classification that pure-workload
* recipes already get through `workflowNeedsWorkloadStatusRefresh`. That predicate
* deliberately excludes workflow recipes (their status is workflowExecution-driven,
* not workload-observation-driven), so without this the periodic backstop covered
* only non-workflow recipes: a Secret re-labeled foreign during a long-running
* workflow was revoked solely by the event-driven SecretWatcher, with no
* deterministic timer fallback if a watch event was delayed or coalesced. This gate
* lets such workflow recipes through to an OWNERSHIP-ONLY re-reconcile (no workload
* status patch), reusing the already-hardened steady-state revocation path
* (`revokeOrRequeueSteadyWorkflow`).
*/
export function workflowNeedsOwnershipBackstop(recipe: WorkflowRecipeCRD): boolean {
if (recipe.metadata.deletionTimestamp) return false
if ((recipe.spec.steps ?? []).length === 0) return false
return recipeReferencesNamedSecrets(recipe)
}
/**
* issue #299 §3.2/§5 — does this recipe declare at least one exact-host EXTERNAL
* egress target that the WRC accumulator folds into a sliding-window policy?
*
* Two accumulator-driven sources qualify:
* - `spec.ui.egress.external[]` — sandbox-ui external FQDNs (always exact-host).
* - a NON-transport workload's `egressBindings[]` whose `dns` is an external FQDN
* (i.e. NOT `<svc>.<ns>.svc.cluster.local`). Validation guarantees non-transport
* workloads use exact-host egress, so any external-FQDN binding here is
* exact-host. The UI workload's own bindings are skipped (its egress is declared
* via `spec.ui.egress`, and the reconciler's workload loop skips it too).
*
* Transport-workload egress is intentionally EXCLUDED: it is delegated to HCC via
* the McpServer CRD, not folded by this accumulator, so re-enqueueing on its
* account would do no accumulator work.
*
* The predicate is pure and cheap (no DNS, no apiserver) so the refresh timer can
* filter the cached recipe set synchronously each tick.
*/
export function recipeHasExternalExactHostEgress(recipe: WorkflowRecipeCRD): boolean {
if (recipe.metadata.deletionTimestamp) return false
if ((recipe.spec.ui?.egress?.external ?? []).length > 0) return true
const uiWorkloadId = recipe.spec.ui?.workloadRef
return (recipe.spec.workloads ?? []).some(workload => {
if (workload.transport) return false
if (uiWorkloadId && workload.id === uiWorkloadId) return false
return (workload.egressBindings ?? []).some(
binding =>
Boolean(binding.dns) &&
binding.port != null &&
parseClusterLocalFqdn(binding.dns as string) === null
)
})
}
/**
* issue #299 §3.2/§5 — the external-egress refresh cadence in ms.
*
* `clamp(refreshInterval, floor)`: never tick faster than the floor. `loadConfig`
* already enforces `floor <= interval <= overlap/2`, but clamping here keeps the
* timer correct even if a caller passes an unvalidated config (tests, future
* callers), so a sub-floor interval can never turn the timer into a busy loop.
*/
export function externalEgressRefreshIntervalMs(
config: {
externalEgressRefreshIntervalSeconds: number
externalEgressRefreshFloorSeconds: number
},
minObservedTtlMs = Infinity
): number {
const floorMs = config.externalEgressRefreshFloorSeconds * 1000
const configuredMs = config.externalEgressRefreshIntervalSeconds * 1000
// H2 (issue #299): advance the refresh to <= observed TTL/2 so a rotating
// low-TTL host is sampled every rotation. A finite observed TTL can only make
// the refresh FASTER than the configured interval, never slower; the floor is
// the hard lower bound to avoid a hot-loop.
const ttlAwareMs = Number.isFinite(minObservedTtlMs)
? Math.min(configuredMs, minObservedTtlMs / 2)
: configuredMs
return Math.max(ttlAwareMs, floorMs)
}
/**
* Run `worker` over `items` with at most `limit` promises in flight at once.
* Deterministic worker-pool: `limit` runners pull from a shared cursor until the
* list drains. Used to bound the external-egress refresh fan-out.
*/
async function runWithConcurrency<T>(
items: T[],
limit: number,
worker: (item: T) => Promise<void>
): Promise<void> {
let cursor = 0
const runnerCount = Math.min(Math.max(limit, 1), items.length)
const runners = Array.from({ length: runnerCount }, async () => {
while (cursor < items.length) {
const item = items[cursor++]
await worker(item)
}
})
await Promise.all(runners)
}
export interface WorkflowRecipeProvider {
getAllRecipes(): WorkflowRecipeCRD[]
start(): Promise<void>
stop(): Promise<void>
/** JWT signer initialized at startup (null if signing-key Secret is absent). */
getTokenFactory(): JwtTokenFactory | null
/** Best-effort governed trace reporter shared by WRC status paths. */
getTraceReporter(): GovernedTraceReporter | null
/** DB-first run processor — null in dev mode or when pool is unavailable. */
getDbRunProcessor(): DbRunProcessor | null
}
// One refresh pass over every cached recipe that opts into backgroundAccess.
// Exported so a unit test can exercise it without booting the watcher / Watch.
export async function rotateBrokerTokensOnce(
recipes: Iterable<WorkflowRecipeCRD>,
refresh: (recipe: WorkflowRecipeCRD) => Promise<void>,
onError?: (recipe: WorkflowRecipeCRD, err: unknown) => void
): Promise<void> {
for (const recipe of recipes) {
if (!recipeHasBackgroundAccessClient(recipe)) continue
try {
await refresh(recipe)
} catch (err) {
onError?.(recipe, err)
}
}
}
// JWT TTL=600s, refresh-before window=300s in the reconciler. 60s ticks give
// 5 chances to refresh before expiry and keep per-recipe load to one secret
// read + (rare) one write per minute.
const BROKER_TOKEN_ROTATION_INTERVAL_MS = 60_000
/**
* Production watcher — watches WorkflowRecipe CRDs and triggers reconciliation.
*/
export class WorkflowRecipeWatcher implements WorkflowRecipeProvider {
private watch: k8s.Watch
private kc: k8s.KubeConfig
private customApi: k8s.CustomObjectsApi
private watchRequest: { abort: () => void } | null = null
private recipes: Map<string, WorkflowRecipeCRD> = new Map()
private reconciler: WorkflowRecipeReconciler
private dbRunProcessor: DbRunProcessor | null = null
// issue #375 (P3): event-driven grant→WRC nudge. Started only when the WRC has
// a DB (same gate as dbRunProcessor); without a DB the listener never starts
// and the level-triggered reconcile remains the sole convergence path.
private grantUpdateListener: GrantUpdateListener | null = null
// issue #375 (M4, jozer review): the LISTEN session pins its client for the
// process lifetime, so it gets its OWN single-connection pool and consumes
// zero slots of the shared `poolMax` (default 4) pool that dbRunProcessor,
// leader election, and run-sync all contend for. Owned (and ended) by this
// watcher because it exists solely for the grant-update listener.
private grantUpdateListenerPool: Pool | null = null
private config = loadConfig()
private stopped = false
private runtimeCredentialRefreshTimer: ReturnType<typeof setInterval> | null = null
private runtimeCredentialRefreshRunning = false
private workloadStatusRefreshTimer: ReturnType<typeof setInterval> | null = null
private workloadStatusRefreshRunning = false
// issue #299 — periodic external-egress requeue. GitHub (and other single-A-record
// rotators) change their resolved IP without bumping the CRD generation, so the
// accumulator's sliding window only converges if we re-reconcile on a timer.
private externalEgressRefreshTimer: ReturnType<typeof setTimeout> | null = null
private externalEgressRefreshRunning = false
private eventQueue = new RecipeEventQueue((key, error) => {
createLogger('wrc', 'workflow-recipes').error('Unhandled recipe event failure', {
key,
err: error,
})
})
private secretReverseIndex = new SecretReverseIndex()
private secretWatchLoops: K8sSecretWatchLoop[] = []
private brokerRotationTimer: NodeJS.Timeout | null = null
private traceReporter: GovernedTraceReporter | null = createGovernedTraceReporter(
this.config.governedTracingEnabled
)
// Per-recipe deterministic retry for transient (skipStatusPatch) reconcile
// results: no status patch ⇒ no MODIFIED event, so we must schedule the retry
// ourselves with bounded exponential backoff. Cleared on any non-transient
// result or DELETE.
private transientRetries = new Map<
string,
{
timer: ReturnType<typeof setTimeout>
attempts: number
recipeUid?: string
}
>()
private static readonly TRANSIENT_RETRY_MAX_MS = 60_000
constructor(kc: k8s.KubeConfig, pool?: Pool | null) {
this.kc = kc
this.watch = new k8s.Watch(kc)
this.customApi = kc.makeApiClient(k8s.CustomObjectsApi)
this.reconciler = new WorkflowRecipeReconciler(kc, undefined, {
secretReverseIndex: this.secretReverseIndex,
})
if (pool && this.config.db) {
this.dbRunProcessor = createDbRunProcessor({
instanceId: this.config.db.instanceId,
pool,
runPollMs: this.config.db.runPollMs,
createChildRecipe: this.makeChildRecipeCreator(),
childRecipeExists: this.makeChildRecipeExists(),
onRunStarted: runId => {
this.traceReporter?.enqueueWorkflowLifecycle({
sourceEventId: `workflow-run:${runId}:start`,
occurredAt: new Date().toISOString(),
runId,
eventType: 'run_start',
payload: { phase: 'Running' },
})
},
onRunTerminal: (runId, phase) => {
this.traceReporter?.enqueueWorkflowLifecycle({
sourceEventId: `workflow-run:${runId}:terminal`,
occurredAt: new Date().toISOString(),
runId,
eventType: 'run_end',
payload: { phase },
})
},
})
// issue #375 (P3): on a grant-update NOTIFY, force-reconcile the affected
// recipe through the SAME path the 30s watchdog uses. The dispatch body is
// a named method so it is unit-testable without standing up a DB.
// issue #375 (M4): the listener LISTENs on a DEDICATED single-connection
// pool, never the shared one — a lifetime-pinned LISTEN checkout would
// permanently burn one of the shared pool's 4 slots (dbRunProcessor
// already pins another).
this.grantUpdateListenerPool = createGrantUpdateListenerPool(this.config.db)
this.grantUpdateListener = createGrantUpdateListener({
pool: this.grantUpdateListenerPool,
// `capabilityFamily` is accepted for forward-compat and intentionally
// unused: the dispatch force-reconciles the whole recipe from the DB
// source of truth (all families recomputed), and authorization is
// per-invocation in control-api, never derived from the notification.
onGrantUpdate: ({ recipeNamespace, recipeName }) =>
this.handleGrantUpdateNotification(recipeNamespace, recipeName),
})
}
}
/**
* issue #375 (P3): dispatch a grant-update NOTIFY into a forced reconcile of
* the affected recipe — the SAME path the 30s watchdog uses. Kept as a named
* method (not an inline closure) so it is unit-testable without a live DB.
*/
private handleGrantUpdateNotification(recipeNamespace: string, recipeName: string): void {
const known = this.recipes.get(recipeName)
if (!known || known.metadata.namespace !== recipeNamespace) {
// issue #375 (H4): make the discard diagnosable. control-api's emitted
// recipeNamespace and the WRC's watched sandboxNamespace come from
// independent env vars; if they ever diverge, 100% of NOTIFYs would be
// dropped and the feature would silently degrade to 30s polling. Log names
// only — no secrets.
createLogger('wrc', recipeName).warn('Discarding grant-update NOTIFY', {
recipeNamespace,
recipeName,
value3: known
? `namespace mismatch (watching "${known.metadata.namespace}")`
: 'unknown recipe',
})
return
}
void this.eventQueue.enqueue(recipeName, () => {
// issue #375 (H5): reconcile the CURRENT recipe. A newer MODIFIED may have
// landed between NOTIFY receipt and this queued job running
// (handleRecipeEvent updates this.recipes), so re-read the cache instead of
// closing over the stale snapshot. Re-check the guard on the fresh read —
// the recipe may have been deleted, or its namespace changed, while queued.
const cached = this.recipes.get(recipeName)
if (!cached || cached.metadata.namespace !== recipeNamespace) return Promise.resolve()
return this.handleRecipeEvent('MODIFIED', cached, { forceReconcile: true })
})
}
getAllRecipes(): WorkflowRecipeCRD[] {
return [...this.recipes.values()]
}
getTokenFactory(): JwtTokenFactory | null {
// Exposed by the reconciler once initializeWorkflow() has run.
// Will be null until the signing-key Secret has been read successfully.
return this.reconciler.tokenFactory
}
getTraceReporter(): GovernedTraceReporter | null {
return this.traceReporter
}
getDbRunProcessor(): DbRunProcessor | null {
return this.dbRunProcessor
}
/**
* Builds the closure the DB run processor invokes when it needs to
* materialize a child WorkflowRecipe for a Pending run. This is the ONLY
* bridge between the DB-driven run lifecycle and the K8s API.
*/
private makeChildRecipeCreator(): ChildRecipeCreator {
return async (run: DbRunRow) => createDbRunChildRecipe(this.customApi, run)
}
private makeChildRecipeExists(): ChildRecipeExistenceChecker {
return async ({ name, namespace }) => {
try {
await this.customApi.getNamespacedCustomObject({
group: CRD_GROUP,
version: CRD_VERSION,
namespace,
plural: WORKFLOWRECIPE_PLURAL,
name,
})
return true
} catch (err) {
if (getErrorCode(err) === 404) return false
throw err
}
}
}
async start(): Promise<void> {
createLogger('wrc', 'workflow-recipes').info('[WR-K8s] Starting WorkflowRecipe watcher')
// Initialize workflow subsystem by reading the signing key from control-plane.
// Non-fatal: if the Secret doesn't exist, workflow recipes will gracefully fail
// but traditional workload recipes continue to work.
await this.tryInitializeWorkflow()
await this.startWatch()
await this.startSecretWatch()
this.startBrokerTokenRotationLoop()
this.startRuntimeCredentialRefreshLoop()
this.startWorkloadStatusRefreshLoop()
this.startExternalEgressRefreshLoop()
if (this.dbRunProcessor) {
await this.dbRunProcessor.start()
}
if (this.grantUpdateListener) {
await this.grantUpdateListener.start()
}
}
// Refreshes opted-in recipes' broker-token Secrets between CRD events.
// Serializes per recipe through the same queue as watch events so a
// refresh never races with an in-flight reconcile.
private startBrokerTokenRotationLoop(): void {
const tick = () => {
void rotateBrokerTokensOnce(
this.recipes.values(),
recipe =>
this.eventQueue.enqueue(recipe.metadata.name, () =>
this.reconciler.ensureOAuthBrokerTokenSecret(recipe)
),
(recipe, err) => {
createLogger('wrc', recipe.metadata.name).error('Broker-token refresh failed', {
name: recipe.metadata.name,
err,
})
}
)
}
this.brokerRotationTimer = setInterval(tick, BROKER_TOKEN_ROTATION_INTERVAL_MS)
// Node treats setInterval as a ref by default — without unref(), the loop
// would keep the process alive during graceful shutdown windows.
this.brokerRotationTimer.unref?.()
}
private async startSecretWatch(): Promise<void> {
// Issue #637 — watch Secrets in EVERY namespace a recipe's workloads can mount
// them from (transport → mcp-server, ui.workloadRef → sandbox-ui, rest →
// sandbox-recipes), so a Secret created or re-labeled foreign in ANY of them
// fans out a re-reconcile and the ownership gate re-evaluates (revocation /
// deferred-foreign-creation). Watching only sandbox-recipes left an mcp-server
// or sandbox-ui Secret change unobserved, so an already-rendered `missing`/
// previously-allowed ref could turn foreign without re-classification.
//
// Each namespace gets its OWN SecretWatcher so the per-name last-seen dedup
// state never collides across namespaces (the same Secret name can exist in
// two of them); all share the reverse index + the enqueue callback. Fan-out is
// by name, so a change over-triggers harmless re-reconciles at worst.
const watchedNamespaces = [
...new Set([
this.config.sandboxNamespace,
this.config.namespace,
this.config.sandboxUiNamespace,
]),
]
for (const namespace of watchedNamespaces) {
const handler = new SecretWatcher(
this.secretReverseIndex,
name => this.triggerSecretDrivenReconcile(name),
10_000,
recipeName => this.handleOAuthBrokerTokenAdded(namespace, recipeName)
)
const loop = new K8sSecretWatchLoop(this.kc, namespace, handler)
this.secretWatchLoops.push(loop)
await loop.start()
}
}
/**
* B3(a): a broker token Secret appeared. The token lives only in the sandbox
* namespace, so an ADDED for a same-named Secret elsewhere is ignored.
*
* The delete ledger is always re-armed. A recipe with no backgroundAccess
* client must not have the token at all, so it is also re-reconciled to reap
* it: the broker rotation loop skips such recipes and the ownership backstop
* only covers recipes that reference named Secrets, so nothing else would
* issue the delete before the next CRD event. An opted-in recipe legitimately
* owns its token and only invalidates, because a watch reconnect replays
* ADDED for every broker token and must not force a reconcile storm.
*
* A recipe not yet in the cache is only re-armed, not enqueued. At startup
* `startWatch` returns once the watch is open, without waiting for the
* queued recipe ADDED events to run, and the Secret watch starts right
* after, so a token ADDED can land first. The recipe's own queued event then reconciles with
* the ledger armed. A pass that returns before the token reap leaves the
* token to the next one that reaches it: after a CRD event, a recipe watch
* restart replaying ADDED, or a Secret watch reconnect replaying the token
* ADDED once the recipe is cached.
*/
private handleOAuthBrokerTokenAdded(namespace: string, recipeName: string): void {
if (namespace !== this.config.sandboxNamespace) return
this.reconciler.invalidateOAuthBrokerSecretLedger(recipeName)
const recipe = this.recipes.get(recipeName)
if (recipe && !recipeHasBackgroundAccessClient(recipe)) {
this.triggerSecretDrivenReconcile(recipeName)
}
}
/**
* Enqueue a re-reconcile for a recipe whose referenced Secret's key-set
* just changed. The cached recipe object is reused — no extra GET against
* the apiserver. If the recipe is no longer in our cache (e.g. it was
* deleted between the Secret event and the debounce fire), we skip.
*/
private triggerSecretDrivenReconcile(recipeName: string): void {
const recipe = this.recipes.get(recipeName)
if (!recipe) return
// Issue #637 — a Secret re-label does NOT bump the recipe's metadata.generation,
// so a plain 'MODIFIED' event is classified status-only and short-circuits BEFORE
// reconcile() (and thus before the ownership gate `revokeOrRequeueSteadyWorkflow`).
// forceReconcile makes the event-driven revocation actually re-evaluate ownership;
// without it a foreign re-label is only caught by the slower 30s periodic backstop,
// not the ~10s watcher debounce this path exists to provide.
this.eventQueue.enqueue(recipeName, () =>
this.handleRecipeEvent('MODIFIED', recipe, { forceReconcile: true })
)
}
private async tryInitializeWorkflow(): Promise<void> {
const SIGNING_KEY_SECRET = 'clerum-wrc-signing-key'
const SIGNING_KEY_NS = this.config.controlPlaneNamespace
try {
const coreApi = this.kc.makeApiClient(k8s.CoreV1Api)
const secret = await coreApi.readNamespacedSecret({
name: SIGNING_KEY_SECRET,
namespace: SIGNING_KEY_NS,
})
const privateKeyB64 = secret.data?.['private.pem']
const publicKeyB64 = secret.data?.['public.pem']
if (!privateKeyB64) {
createLogger('wrc', 'workflow-recipes').warn(
'Signing Secret lacks private.pem; workflow disabled',
{ SIGNING_KEY_SECRET }
)
return
}
if (!publicKeyB64) {
createLogger('wrc', 'workflow-recipes').warn(
'Signing Secret lacks public.pem; JWT verification disabled',
{ SIGNING_KEY_SECRET }
)
}
const privateKeyPem = Buffer.from(privateKeyB64, 'base64').toString('utf-8')
const publicKeyPem = publicKeyB64
? Buffer.from(publicKeyB64, 'base64').toString('utf-8')
: undefined
await this.reconciler.initializeWorkflow(privateKeyPem, publicKeyPem)
createLogger('wrc', 'workflow-recipes').info(
'[WR-K8s] Workflow subsystem initialized from signing key'
)
} catch (error: unknown) {
const code = (error as { response?: { statusCode?: number } })?.response?.statusCode
if (code === 404 || code === 403) {
createLogger('wrc', 'workflow-recipes').warn(
'Signing Secret not found or forbidden; workflow disabled',
{ code }
)
} else {
createLogger('wrc', 'workflow-recipes').warn(
'[WR-K8s] Failed to read signing key — workflow disabled:',
{ err: error }
)
}
}
}
private async startWatch(): Promise<void> {
// Design invariant: WorkflowRecipe CRDs are watched only in sandbox-recipes.
// Transport workloads may render McpServer children in mcp-server, but the
// recipe CRD itself is never stored or reconciled from that namespace.
const path = `/apis/${CRD_GROUP}/${CRD_VERSION}/namespaces/${this.config.sandboxNamespace}/${PLURAL}`
const watchCallback = (type: string, apiObj: WorkflowRecipeWatchObject) => {
const recipe = workflowRecipeFromWatchObject(apiObj, this.config.sandboxNamespace)
createLogger('wrc', recipe.metadata.name).info('Received recipe watch event', {
type,
name: recipe.metadata.name,
})
this.eventQueue.enqueue(recipe.metadata.name, () => this.handleRecipeEvent(type, recipe))
}
const doneCallback = (err: Error | null) => {
if (this.stopped) return
if (err) {
createLogger('wrc', 'workflow-recipes').error('[WR-K8s] Watch error:', { err })
}
createLogger('wrc', 'workflow-recipes').info('[WR-K8s] Watch ended, restarting...')
setTimeout(() => this.startWatch(), err ? 5000 : 1000)
}
this.watchRequest = await this.watch.watch(path, {}, watchCallback, doneCallback)
}
private async handleRecipeEvent(
type: string,
recipe: WorkflowRecipeCRD,
options: { forceReconcile?: boolean } = {}
): Promise<void> {
if (type === 'ADDED' || type === 'MODIFIED') {
const scheduledUid = this.transientRetries.get(recipe.metadata.name)?.recipeUid
if (scheduledUid && scheduledUid !== recipe.metadata.uid) {
this.clearTransientRetry(recipe.metadata.name)
}
let dbSyncAttempted = false
if (recipe.metadata.deletionTimestamp) {
this.clearTransientRetry(recipe.metadata.name)
// Resource is being deleted — cleanup cross-namespace resources, then remove finalizer.
createLogger('wrc', recipe.metadata.name).info('Cleaning up recipe finalizer', {
name: recipe.metadata.name,
})
try {
await this.reconciler.reconcileDelete(recipe)
await this.reconciler.removeFinalizer(recipe)
} catch (error) {
if (getErrorCode(error) === 404) {
createLogger('wrc', recipe.metadata.name).warn(
'Skipping finalizer cleanup because the recipe is gone',
{ name: recipe.metadata.name }
)
} else {
createLogger('wrc', recipe.metadata.name).error('Finalizer cleanup failed', {
name: recipe.metadata.name,
err: error,
})
}
}
this.recipes.delete(recipe.metadata.name)
return
}
if (
type === 'MODIFIED' &&
!options.forceReconcile &&
this.isStatusOnlyModifiedEvent(recipe)
) {
this.recipes.set(recipe.metadata.name, recipe)
const needsTerminalRunTeardown = workflowNeedsTerminalRunTeardown(recipe)
// A status-only MODIFIED event (generation unchanged) is exactly how a
// workflow run's terminal completion and per-step transitions arrive —
// Kubernetes bumps .metadata.generation only on .spec changes, never on
// .status. Skipping the expensive full reconcile here is correct, but we
// MUST still drive the CRD->DB sync, otherwise a completed run's DB row
// stays Running forever (UI shows "running", dbRunProcessor keeps logging
// "orphaned running run reclaimed"). syncFromRecipeExecution is DB-only
// (no CRD patch ⇒ no MODIFIED event ⇒ no reconcile loop); on terminal it
// flips Running→Succeeded|Failed|Canceled, on non-terminal it bumps the
// last_reconciled_at heartbeat that suppresses orphan reclaim.
if (this.dbRunProcessor) {
try {
await this.dbRunProcessor.syncFromRecipeExecution(recipe)
} catch (error) {
createLogger('wrc', recipe.metadata.name).error(
'Database sync failed for a status-only event',
{ name: recipe.metadata.name, err: error }
)
} finally {
dbSyncAttempted = true
}
}
if (needsTerminalRunTeardown) {
createLogger('wrc', recipe.metadata.name).info(
'Forcing reconciliation of terminal run status for compute teardown',
{ name: recipe.metadata.name }
)
} else {
return
}
}
// Normal reconciliation — ensure finalizer first for cross-namespace safety.
this.recipes.set(recipe.metadata.name, recipe)
try {
if (!(await this.reconciler.isRecipeStillActive(recipe))) {
createLogger('wrc', recipe.metadata.name).warn(
'Skipping stale event because the recipe is gone or deleting',
{ name: recipe.metadata.name }
)
this.recipes.delete(recipe.metadata.name)
return
}
if (this.dbRunProcessor && !dbSyncAttempted) {
await this.dbRunProcessor.syncFromRecipeExecution(recipe)
}
await this.reconciler.ensureFinalizer(recipe)
if (!(await this.reconciler.isRecipeStillActive(recipe))) {
createLogger('wrc', recipe.metadata.name).warn(
'Skipping reconciliation because deletion began after the finalizer check',
{ name: recipe.metadata.name }
)
this.recipes.delete(recipe.metadata.name)
return
}
const result = await this.reconciler.reconcile(recipe)
// Patch only when observable status changes. This avoids reconcile loops
// while still allowing active parent recipes to clear stale execution state.
const willPatch = shouldPatchRecipeStatus(recipe, result)
// issue #375 (P4): surface a computed-vs-published Plugin Workload SDK
// state divergence — old→new plus whether a patch was emitted. With P1
// this is rare; its value is forensic (it would have discriminated
// "transition computed but not published" from "never computed" in
// minutes) and it verifies P1 in the field. Only states are logged — no
// tokens/secrets (sec-003).
const projectedSdkState = result.pluginWorkloadSdkProjection?.capability?.state
const persistedSdkState = recipe.status?.pluginWorkloadSdk?.state
// The `validated → capability:null` cleanup case is intentionally NOT
// forensically logged here (projectedSdkState is undefined for a cleared
// capability); its clearing PATCH is still emitted via
// `pluginWorkloadSdkProjectionChanged`, so the publish is covered.
if (projectedSdkState !== undefined && projectedSdkState !== persistedSdkState) {
createLogger('wrc', recipe.metadata.name).info('Plugin Workload SDK state changed', {
name: recipe.metadata.name,
value2: persistedSdkState ?? 'none',
projectedSdkState,
willPatch,
})
}
if (willPatch) {
await this.reconciler.patchStatus(recipe, result)
enqueueInfrastructureTelemetryBestEffort(
this.traceReporter,
reconcileOutcomeTelemetryProjection(recipe, result)
)
}
// Transient (skipStatusPatch) results write no status ⇒ emit no MODIFIED
// event, so schedule a deterministic backoff retry ourselves. Any other
// result clears the per-recipe backoff.
if (result.requeueAfterMs && !this.stopped) {
this.scheduleTransientRetry(
recipe.metadata.name,
result.requeueAfterMs,
result.requeueFixedInterval ?? false,
result.requeueOperation
)
} else {
this.clearTransientRetry(recipe.metadata.name)
}
} catch (error) {
if (getErrorCode(error) === 404) {
createLogger('wrc', recipe.metadata.name).warn(
'Reconciliation skipped because the recipe is gone',
{ name: recipe.metadata.name }
)
this.recipes.delete(recipe.metadata.name)
return
}
createLogger('wrc', recipe.metadata.name).error('Reconciliation failed', {
name: recipe.metadata.name,
err: error,
})
enqueueInfrastructureTelemetryBestEffort(
this.traceReporter,
controllerErrorTelemetryProjection(recipe, error)
)
// issue #375 (P2): a transient (non-404) failure used to return here
// WITHOUT re-arming the retry chain, so a thrown reconcile killed the 5s
// self-heal and only the 30s watchdog could recover. Re-schedule the
// bounded exponential backoff retry so the recipe re-reconciles on its
// own; persistent failure still backs off toward TRANSIENT_RETRY_MAX_MS,
// with the watchdog as the second net. The 404 branch above already
// cleared and returned, so it never reaches here.
if (!this.stopped) {
this.scheduleTransientRetry(recipe.metadata.name, TRANSIENT_REQUEUE_BASE_MS, false)
}
}
} else if (type === 'DELETED') {
// Resource is gone (finalizer already removed) — just clean cache.
this.clearTransientRetry(recipe.metadata.name)
this.recipes.delete(recipe.metadata.name)
}
}
private isStatusOnlyModifiedEvent(recipe: WorkflowRecipeCRD): boolean {
const cached = this.recipes.get(recipe.metadata.name)
// POST /admin/recipes/:name/retry requests the canonical failed -> candidate
// transition through /status, without changing spec/generation. Run that
// transition once; subsequent candidate/status updates remain coalesced.
if (cached?.status?.phase === 'failed' && recipe.status?.phase === 'candidate') return false
const cachedGeneration = cached?.metadata.generation
const nextGeneration = recipe.metadata.generation
if (
cachedGeneration === undefined ||
nextGeneration === undefined ||
cachedGeneration !== nextGeneration
) {
return false
}
// Grant identity is a metadata annotation and does not bump generation.
// Treat it as a real reconcile, not a status-only skip.
const cachedRef = grantIdentityFingerprint(cached)
const nextRef = grantIdentityFingerprint(recipe)
return cachedRef === nextRef
}
/**
* Schedule a timer-driven re-enqueue for `name`. The retry re-reconciles the
* cached recipe; if it is still transient/in-progress, handleRecipeEvent
* reschedules. A non-transient result (or DELETE / stop) calls
* clearTransientRetry and clears the entry.
*
* Two modes (single entry per recipe name, coalesced — any existing timer is
* cleared before scheduling the next):
*
* - `fixedInterval === false` (default, transient ERROR path): bounded
* EXPONENTIAL backoff — `delay = min(baseMs * 2**attempts, MAX)` — and the
* attempt counter grows so genuine flakiness backs off toward the cap.
*
* - `fixedInterval === true` (steady-state PROGRESS path, phase==='deploying'
* waiting on a Pod we created): FIXED `delay = baseMs` with `attempts` RESET
* to 0. Progress is not an error and must not inherit the error backoff
* curve — exponential growth would stretch the poll cadence toward the cap
* and re-expose the 240s mcp-host readiness deadline this requeue beats.
* Resetting attempts means a later GENUINE transient error starts backoff
* fresh from base rather than from a counter inflated by normal progress.
*
* Exposed for tests via the watcher's event path.
*/
private scheduleTransientRetry(
name: string,
baseMs: number,
fixedInterval = false,
operation?: ReconcileResult['requeueOperation']
): void {
const existing = this.transientRetries.get(name)
// Fixed-interval (progress) requeues reset the backoff counter; transient
// (error) requeues grow it.
const attempts = fixedInterval ? 0 : (existing?.attempts ?? 0)
if (existing) clearTimeout(existing.timer)
const owner = operation ? this.recipes.get(name) : undefined
if (operation && (!owner?.metadata.uid || owner.metadata.deletionTimestamp)) {
this.transientRetries.delete(name)
return
}
const delay = fixedInterval
? baseMs
: Math.min(baseMs * 2 ** attempts, WorkflowRecipeWatcher.TRANSIENT_RETRY_MAX_MS)
const timer = setTimeout(() => {
if (this.stopped) return
if (operation && owner) {
void this.eventQueue.enqueue(name, () =>
this.retryPendingLegacyNetworkPolicyDelete(
name,
owner.metadata.namespace,
owner.metadata.uid!,
timer
)
)
return
}
const cached = this.recipes.get(name)
if (!cached) {
this.transientRetries.delete(name)
return
}
const label = fixedInterval ? 'progress' : 'transient'
createLogger('wrc', 'workflow-recipes').info('Retrying recipe reconciliation', {
label,
name,
value3: attempts + 1,
delay,
})
void this.eventQueue.enqueue(name, () =>
this.handleRecipeEvent('MODIFIED', cached, { forceReconcile: true })
)
}, delay)
timer.unref?.()
// Fixed-interval progress requeues keep attempts pinned at 0 (no backoff);
// transient-error requeues preserve a growing attempt count so they back off.
// NOTE: a fixed 5s interval × N concurrent deploying runs is N/5 reconciles
// per second. We accept that load tradeoff for prompt progress; the inner
// reconcile advances past the waiting state once the awaited Pod succeeds,
// and the 240s readiness deadline / dbRunProcessor max-duration paths
// terminate any run permanently stuck in deploying, so this is bounded.
this.transientRetries.set(name, {
timer,
attempts: fixedInterval ? 0 : attempts + 1,
...(operation ? { recipeUid: owner!.metadata.uid } : {}),
})
}
/**
* Runs after the recipe's queue wait. The API read is authoritative for UID,
* deletion and policy markers; this callback never invokes the full recipe
* pipeline, token reap or terminal runtime teardown.
*/
private async retryPendingLegacyNetworkPolicyDelete(
name: string,
namespace: string,
uid: string,
timer: ReturnType<typeof setTimeout>
): Promise<void> {
const current = () => !this.stopped && this.transientRetries.get(name)?.timer === timer
if (!current()) return
try {
const apiObject = (await this.customApi.getNamespacedCustomObject({
group: CRD_GROUP,
version: CRD_VERSION,
namespace,
plural: WORKFLOWRECIPE_PLURAL,
name,
})) as WorkflowRecipeWatchObject
if (!current()) return
if (
apiObject.metadata.name !== name ||
apiObject.metadata.namespace !== namespace ||
apiObject.metadata.uid !== uid ||
apiObject.metadata.deletionTimestamp
) {
this.clearTransientRetry(name)
return
}
const latest = workflowRecipeFromWatchObject(apiObject, namespace)
const result = await this.reconciler.retryPendingLegacyNetworkPolicyDeleteOnly(latest)
if (!current()) return
const cached = this.recipes.get(name)
if (!cached || cached.metadata.uid !== uid || cached.metadata.deletionTimestamp) {
this.clearTransientRetry(name)
return
}
if (result.requeueAfterMs) {
this.scheduleTransientRetry(
name,
result.requeueAfterMs,
true,
'legacy-network-policy-delete'
)
} else {
this.clearTransientRetry(name)
}
} catch (error) {
if (!current()) return
if (getErrorCode(error) === 404) {
this.clearTransientRetry(name)
return
}
createLogger('wrc', name).error('Legacy NetworkPolicy delete retry failed', {
name,
err: error,
})
// A status conflict or transient API failure keeps this same narrow
// operation on the existing bounded error backoff; it cannot replay runtime cleanup.
this.scheduleTransientRetry(
name,
TRANSIENT_REQUEUE_BASE_MS,
false,
'legacy-network-policy-delete'
)
}
}
private clearTransientRetry(name: string): void {
const existing = this.transientRetries.get(name)
if (existing) {
clearTimeout(existing.timer)
this.transientRetries.delete(name)
}
}
private startRuntimeCredentialRefreshLoop(): void {
if (this.runtimeCredentialRefreshTimer) return
const intervalMs = runtimeCredentialRefreshIntervalMs(this.config)
this.runtimeCredentialRefreshTimer = setInterval(() => {
if (this.runtimeCredentialRefreshRunning) return
this.runtimeCredentialRefreshRunning = true
void this.refreshRuntimeCredentialsForInProgressRecipes().finally(() => {
this.runtimeCredentialRefreshRunning = false
})
}, intervalMs)
this.runtimeCredentialRefreshTimer.unref?.()
}
private startWorkloadStatusRefreshLoop(): void {
if (this.workloadStatusRefreshTimer) return
// Child workload readiness can drift without a WorkflowRecipe CRD event.
// Re-enqueue steady active/degraded workload recipes so reconcile observes
// Kubernetes status and patches degradation or recovery.
this.workloadStatusRefreshTimer = setInterval(() => {
if (this.workloadStatusRefreshRunning) return
this.workloadStatusRefreshRunning = true
void this.refreshWorkloadStatusesForSteadyRecipes().finally(() => {
this.workloadStatusRefreshRunning = false
})
}, WORKLOAD_STATUS_REFRESH_INTERVAL_MS)
this.workloadStatusRefreshTimer.unref?.()
}
/**
* issue #299 §3.2/§5 — start the periodic external-egress requeue.
*
* A recipe's exact-host external egress (`spec.ui.egress.external` or a
* non-transport workload's external `egressBindings`) resolves to IPs that a
* single-A-record provider (GitHub: api.github.com, ~15s TTL over
* 140.82.112.0/20) rotates WITHOUT bumping the CRD generation. Generation-only
* reconcile triggers therefore never re-run the accumulator, and the
* sliding-window egress set on the NetworkPolicy stops covering the pod's next
* resolution. This timer re-enqueues a FULL reconcile (forceReconcile — a plain
* MODIFIED would short-circuit as status-only) for every cached recipe with such
* egress, so the accumulator folds each fresh DNS snapshot into the window.
*
* Overlapping passes are impossible by construction: the next delay is armed in
* the pass's `.finally`, so a slow pass simply pushes the next tick out rather
* than stacking. `externalEgressRefreshRunning` is an in-flight indicator (read
* by tests); per-recipe coalescing is handled by the eventQueue.
*/
private startExternalEgressRefreshLoop(): void {
if (this.externalEgressRefreshTimer) return
// Self-rescheduling (not a fixed setInterval) so the delay can advance to
// <= observed TTL/2 (H2, issue #299) as low-TTL hosts are seen. Rescheduling
// AFTER each pass completes also guarantees no overlapping refreshes.
const scheduleNext = (): void => {
if (this.stopped) return
const delayMs = externalEgressRefreshIntervalMs(
this.config,
this.reconciler.externalEgressRefreshMinTtlMs
)
this.externalEgressRefreshTimer = setTimeout(() => {
this.externalEgressRefreshRunning = true
void this.refreshExternalEgressForActiveRecipes().finally(() => {
this.externalEgressRefreshRunning = false
scheduleNext()
})
}, delayMs)
this.externalEgressRefreshTimer.unref?.()
}
scheduleNext()
}
private async refreshExternalEgressForActiveRecipes(): Promise<void> {
if (this.stopped) return
const started = Date.now()
const selected = [...this.recipes.keys()].filter(name => {
const recipe = this.recipes.get(name)
return recipe ? recipeHasExternalExactHostEgress(recipe) : false
})
if (selected.length === 0) return
let enqueued = 0
await runWithConcurrency(selected, EXTERNAL_EGRESS_REFRESH_MAX_CONCURRENCY, async name => {
if (this.stopped) return
const latest = this.recipes.get(name)
if (!latest) return
try {
await this.eventQueue.enqueue(name, () =>
// Generation is unchanged for a DNS-rotation refresh, so force the full
// reconcile; otherwise handleRecipeEvent classifies this synthetic
// MODIFIED as status-only and returns before the accumulator runs.
this.handleRecipeEvent('MODIFIED', latest, { forceReconcile: true })
)
enqueued++
} catch (error) {
createLogger('wrc', 'workflow-recipes').error('External egress refresh failed', {
name,
err: error,
})
}
})
// Counts and duration only — never the resolved FQDNs/IPs or any Secret.
// This line summarises a FLEET-WIDE sweep: naming every host of every recipe
// here would dump the whole fleet's egress onto one line, every pass, with no
// way to attribute an address back to a policy. The per-host record lives
// where the set is decided and written instead — `logResolvedEgressSet` in
// workflowRecipeReconciler.ts, one entry per FQDN, only when the set changed
// and only once the policy has landed (#299). Secrets stay out of both.
createLogger('wrc', 'workflow-recipes').info(
'Re-enqueued recipes for external egress refresh',
{ enqueued, length: selected.length, value3: Date.now() - started }
)
}
private async refreshWorkloadStatusesForSteadyRecipes(): Promise<void> {
if (this.stopped) return
const refreshes = [...this.recipes.keys()].map(async recipeName => {
const latest = this.recipes.get(recipeName)
if (!latest) return
const needsStatusRefresh = workflowNeedsWorkloadStatusRefresh(latest)
// Issue #637 — workflow recipes are excluded from the status refresh but must
// still get a periodic ownership re-classification (their only other revocation
// path is the event-driven SecretWatcher).
const needsOwnershipBackstop = workflowNeedsOwnershipBackstop(latest)
if (!needsStatusRefresh && !needsOwnershipBackstop) return
try {
await this.eventQueue.enqueue(recipeName, () =>
this.refreshWorkloadStatusOnly(latest, { ownershipOnly: !needsStatusRefresh })
)
} catch (error) {
createLogger('wrc', recipeName).error('Workload status refresh failed', {
recipeName,
err: error,
})
}
})
await Promise.all(refreshes)
}
private async refreshWorkloadStatusOnly(
recipe: WorkflowRecipeCRD,
opts: { ownershipOnly?: boolean } = {}
): Promise<void> {
if (opts.ownershipOnly) {
// Issue #637 — workflow-recipe ownership backstop. Skip the workload-status
// observe/patch (a workflow's status is workflowExecution-driven) and force a
// reconcile so the steady-state ownership gate re-classifies and revokes a
// Secret re-labeled foreign even if the SecretWatcher event was missed.
await this.handleRecipeEvent('MODIFIED', recipe, { forceReconcile: true })
return
}
const result = await this.reconciler.observeCurrentWorkloadStatus(recipe)
if (shouldPatchRecipeStatus(recipe, result)) {
await this.reconciler.patchStatus(recipe, result)
enqueueInfrastructureTelemetryBestEffort(
this.traceReporter,
reconcileOutcomeTelemetryProjection(recipe, result)
)
}
// Issue #637 — periodically force a full reconcile for recipes that project a
// name-based Secret, so the ownership gate re-classifies and a Secret re-labeled
// foreign in mcp-server/sandbox-ui is revoked DETERMINISTICALLY (within this
// interval) even if the SecretWatcher's event-driven fan-out was delayed or
// coalesced. The full reconcile is idempotent for an unchanged recipe.
const ownershipRecheck = recipeReferencesNamedSecrets(recipe)
if (ownershipRecheck || result.workloadStatuses.some(status => status.ready === false)) {
await this.handleRecipeEvent('MODIFIED', recipe, { forceReconcile: true })
}
}
private async refreshRuntimeCredentialsForInProgressRecipes(): Promise<void> {
if (this.stopped) return
const refreshes = [...this.recipes.keys()].map(async recipeName => {
const latest = this.recipes.get(recipeName)
if (!latest) return
try {
if (workflowNeedsRuntimeCredentialRefresh(latest)) {
await this.reconciler.refreshInProgressWorkflowRuntimeCredentials(latest)
}
if (workflowNeedsInfrastructureReconcile(latest)) {
await this.eventQueue.enqueue(recipeName, () =>
// Infrastructure retries are timer-driven with an unchanged CRD
// generation. Force the full reconcile; otherwise the watcher
// classifies this synthetic MODIFIED event as status-only and
// returns before re-running the eager SDK bootstrap lane.
this.handleRecipeEvent('MODIFIED', latest, { forceReconcile: true })
)
}
} catch (error) {
createLogger('wrc', recipeName).error(
'Runtime credential refresh or infrastructure retry failed',
{ recipeName, err: error }
)
}
})
await Promise.all(refreshes)
}
async stop(): Promise<void> {
this.stopped = true
for (const { timer } of this.transientRetries.values()) clearTimeout(timer)
this.transientRetries.clear()
if (this.runtimeCredentialRefreshTimer) {
clearInterval(this.runtimeCredentialRefreshTimer)
this.runtimeCredentialRefreshTimer = null
}
if (this.workloadStatusRefreshTimer) {
clearInterval(this.workloadStatusRefreshTimer)
this.workloadStatusRefreshTimer = null
}
if (this.externalEgressRefreshTimer) {
clearTimeout(this.externalEgressRefreshTimer)
this.externalEgressRefreshTimer = null
}
if (this.watchRequest) {
createLogger('wrc', 'workflow-recipes').info('[WR-K8s] Stopping watch')
this.watchRequest.abort()
this.watchRequest = null
}
for (const loop of this.secretWatchLoops) {
loop.stop()
}
this.secretWatchLoops = []
if (this.brokerRotationTimer) {
clearInterval(this.brokerRotationTimer)
this.brokerRotationTimer = null
}
if (this.dbRunProcessor) {
await this.dbRunProcessor.stop()
}
if (this.grantUpdateListener) {
await this.grantUpdateListener.stop()
}
if (this.grantUpdateListenerPool) {
// issue #375 (M4): the watcher owns the dedicated LISTEN pool — end it on
// shutdown so its single connection is closed with the process.
await this.grantUpdateListenerPool.end()
this.grantUpdateListenerPool = null
}
if (this.traceReporter) {
const result = await this.traceReporter.stopAndDrain()
if (!result.drained) {
createLogger('wrc', 'workflow-recipes').warn(
'[WR-K8s] Governed trace reporter shutdown timed out',
{
value1: {
dropped: result.dropped,
},
}
)
}
}
}
}
/**
* Dev mode provider — reads recipes from CLERUM_RECIPES env var.
*/
export class DevWorkflowRecipeProvider implements WorkflowRecipeProvider {
private recipes: Map<string, WorkflowRecipeCRD> = new Map()
getAllRecipes(): WorkflowRecipeCRD[] {
return [...this.recipes.values()]
}
getTokenFactory(): JwtTokenFactory | null {
return null
}
getTraceReporter(): GovernedTraceReporter | null {
return null
}
getDbRunProcessor(): DbRunProcessor | null {
return null
}
async start(): Promise<void> {
const raw = process.env.CLERUM_RECIPES
if (!raw) {
createLogger('wrc', 'workflow-recipes').info(
'[WR-Dev] No CLERUM_RECIPES env var — starting with empty recipe list'
)
return
}
try {
const recipes = JSON.parse(raw) as WorkflowRecipeCRD[]
for (const recipe of recipes) {
this.recipes.set(recipe.metadata.name, recipe)
createLogger('wrc', recipe.metadata.name).info('Loaded development recipe', {
name: recipe.metadata.name,
})
}
createLogger('wrc', 'workflow-recipes').info('Loaded development recipes', {
size: this.recipes.size,
})
} catch (error) {
createLogger('wrc', 'workflow-recipes').error('[WR-Dev] Failed to parse CLERUM_RECIPES:', {
err: error,
})
}
}
async stop(): Promise<void> {
createLogger('wrc', 'workflow-recipes').info('[WR-Dev] Stopping dev provider')
}
}
/**
* Create the appropriate provider based on mode. In production the Postgres
* pool is initialized here so the WRC replica has a single shared connection
* pool spanning `DbRunProcessor`, leader election, and any future archive
* helpers. A missing `config.db` (legacy/test) downgrades the watcher to
* "no DB processor" — the CRD watch still works.
*/
export function createWorkflowRecipeProvider(): WorkflowRecipeProvider {
const config = loadConfig()
if (config.devMode) {
createLogger('wrc', 'workflow-recipes').info('[WR-Provider] Creating dev mode provider')
return new DevWorkflowRecipeProvider()
}
const kc = new k8s.KubeConfig()
kc.loadFromDefault()
let pool: Pool | null = null
if (config.db) {
try {
pool = initDb(config.db)
} catch (err) {
createLogger('wrc', 'workflow-recipes').warn(
'[WR-Provider] initDb() failed — DB run processor disabled:',
{ err }
)
pool = null
}
}
// Allow tests and early bootstrap to retrieve the already-initialized pool
// without re-calling initDb().
if (!pool) {
try {
pool = getPool()
} catch {
pool = null
}
}
createLogger('wrc', 'workflow-recipes').info('[WR-Provider] Creating K8s watcher provider')
return new WorkflowRecipeWatcher(kc, pool)
}