Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
/**
* McpServer Reconciler - manages Kubernetes Deployments and Services
* based on McpServer CRD state.
*
* Responsibilities:
* - Create Deployment + Service when a new McpServer CRD appears
* - Update Deployment when an McpServer CRD is modified
* - Delete Deployment + Service when an McpServer CRD is removed
* - Validate that referenced secrets exist before creating deployments
* - Full reconciliation on startup (sync desired vs actual state)
*/
import * as k8s from '@kubernetes/client-node'
import { IntOrString } from '@kubernetes/client-node/dist/types.js'
import { createHash } from 'crypto'
import { classifyPluginImage } from '@clerum/image-policy'
import {
NETWORK_READY_ANNOTATION,
NETWORK_READY_GENERATION_ANNOTATION,
PRE_DEPLOY_ANNOTATION,
} from '@clerum/network-policy-core'
import { isWorkflowRecipeDefaultAllowedCapability } from '@clerum/workflow-recipe-capability-policy'
import { config } from './config'
import { MANAGED_BY_LABEL, MANAGED_BY_VALUE, MCPSERVER_LABEL } from './constants'
import { hccLogger } from './logger'
import {
confirmAuthoritativeMcpServerAbsence,
sameMcpServerDesiredRevision,
shouldFailClosedForSecretFailure,
} from './mcpServerSafety'
import { createsTotal, mcpserverMissingSecret } from './metrics'
import { McpServerCRD, McpServerStatus } from './types'
import {
canonicalStringify,
configMapMatchesDesired,
deploymentMatchesDesired,
ensureResource,
getErrorCode,
observeCreate,
observeExistenceRead,
preserveDeploymentAnnotations,
preserveObjectAnnotations,
preserveServiceAssignedFields,
replaceWithConflictRetry,
serviceMatchesDesired,
} from './utils'
/**
* Result of validating an envSecret reference. When `ok` is false, `reason`
* classifies the failure so the caller can surface it via status conditions.
*/
export type SecretValidationResult =
| {
ok: true
/**
* sha256 of the validated Secret's content (issue #223). Empty when the
* McpServer has no envSecret. Travels into the pod template as the
* credentials-revision annotation, which is what makes a credential
* rotation roll the pod: env vars come from `secretKeyRef` and Kubernetes
* only injects those when the container starts.
*
* It is derived from the very Secret this validation read, so the digest
* can never describe a different revision than the one that was checked.
*/
revision: string
}
| {
ok: false
reason: 'SecretNotFound' | 'SecretMissingKey' | 'SecretAccessDenied' | 'ReadError'
message: string
}
/**
* Condition written to `status.conditions[]` on an McpServer CRD.
* `status` is the K8s-standard string tri-state, NOT a boolean.
*/
export interface McpServerCondition {
type: string
status: 'True' | 'False' | 'Unknown'
reason: string
message: string
lastTransitionTime: string
observedGeneration?: number
}
type McpServerReconcilerDeps = {
appsApi?: k8s.AppsV1Api
coreApi?: k8s.CoreV1Api
customApi?: k8s.CustomObjectsApi
/**
* TEST-ONLY escape hatch. When the inventory-authority fence is not wired
* (setInventoryAuthority / setResolveCurrentServer never called), mutation
* fences fail CLOSED by default. Unit tests that exercise the reconciler
* without a watcher opt in to the legacy assume-current behavior here.
* Production construction (k8sClient) must never set this.
*/
assumeInventoryAuthorityWhenUnconfigured?: boolean
}
export type McpServerInventoryAuthoritySnapshot = {
known: boolean
generation: number
}
export type McpServerFullReconcileOptions = {
runEffect?: (serverName: string, work: () => Promise<void>) => Promise<void>
maxConcurrency?: number
// Restrict create/update and orphan cleanup to these names. desiredServers
// must still be the whole inventory so out-of-scope servers are never
// mistaken for orphans.
scope?: ReadonlySet<string>
}
export type McpServerMutationOptions = {
isCurrent?: () => boolean
}
// G2: Pre-deploy handshake constants — imported from @clerum/network-policy-core
// (single source of truth shared with WRC; a divergence is a compile error rather
// than a silent 30s fail-open timeout). clerum.io/network-ready-observed-generation
// (Issue #408) is the generation HCC stamps when acking, so WRC can reject a stale ack.
// Issue #223: sha256 of the envSecret content, carried on the POD TEMPLATE (not
// the Deployment object) so that changing it changes the pod template hash and
// Kubernetes performs a rolling restart on its own. Same mechanism the per-Host
// channel-readers use in hostReconciler.ts.
const CREDENTIALS_REVISION_ANNOTATION = 'clerum.io/credentials-revision'
// Codex P0 fix (PR #101): sanitization constants for remote.authHeaders.
// Only standard HTTP header token chars — letters, digits, and '-'. Rejecting
// anything else prevents smuggling headers like `X: foo\r\nAuthorization`.
const HEADER_NAME_RE = /^[A-Za-z0-9-]+$/
// Bound the value length so a pathological CRD cannot blow up the ConfigMap.
const MAX_AUTH_HEADER_VALUE_LENGTH = 2048
const WRC_OWNED_RUNTIME_READY_REASON = 'WrcOwnedRuntimeRegistered'
// Keep fleet admission aligned with the external-egress convergence lane:
// enough parallelism for independent servers without allocating one pending
// effect promise per fleet member.
const DEFAULT_FULL_RECONCILE_MAX_CONCURRENCY = 10
// Status is a read/merge/write transaction shared with other controllers.
// Bound optimistic-conflict retries so a hot object cannot stall reconciliation.
const STATUS_CONDITION_WRITE_MAX_ATTEMPTS = 3
/** Types HCC no longer writes and strips from every status write (#606). A future writer must not reuse `NetworkReady`; remove the entry first. */
const RETIRED_STATUS_CONDITION_TYPES = ['NetworkReady'] as const
export const RUNTIME_NOT_DESIRED_DISABLED_MESSAGE =
'McpServer is disabled; HCC does not run its runtime'
export const RUNTIME_NOT_DESIRED_FAIL_CLOSED_MESSAGE =
'Env Secret validation failed; HCC does not run its runtime'
function isRetiredStatusConditionType(type: string): boolean {
return (RETIRED_STATUS_CONDITION_TYPES as readonly string[]).includes(type)
}
type StatusWriteDecline = 'record-obligation' | 'silent'
type StatusWriteDeclineArg = () => StatusWriteDecline
type StatusWriteOutcome = 'wrote' | 'unchanged' | 'declined'
type RuntimeWithdrawKind = 'disabled' | 'fail-closed'
type DeploymentDeleteOutcome = 'gone' | 'denied' | 'foreign'
type RetractManagedRuntimeResult = {
retracted: boolean
cleanupComplete: boolean
foreignRuntime: boolean
runtimeGone: boolean
}
function shouldStopAfterIncompleteRetract(
retract: RetractManagedRuntimeResult,
isCurrent: () => boolean
): boolean {
if (retract.retracted || retract.foreignRuntime) return false
return !(retract.runtimeGone && isCurrent())
}
function runtimeNotDesiredMessage(kind: RuntimeWithdrawKind): string {
switch (kind) {
case 'disabled':
return RUNTIME_NOT_DESIRED_DISABLED_MESSAGE
case 'fail-closed':
return RUNTIME_NOT_DESIRED_FAIL_CLOSED_MESSAGE
default: {
const _exhaustive: never = kind
return _exhaustive
}
}
}
function throwCleanupFailures(failures: unknown[], message: string): void {
if (failures.length > 0) {
throw new AggregateError(failures, message)
}
}
async function mapSettledWithConcurrency<T, TResult>(
items: readonly T[],
maxConcurrency: number,
work: (item: T, index: number) => Promise<TResult>
): Promise<Array<PromiseSettledResult<TResult>>> {
if (!Number.isInteger(maxConcurrency) || maxConcurrency < 1) {
throw new RangeError('maxConcurrency must be a positive integer')
}
const results = new Array<PromiseSettledResult<TResult>>(items.length)
let nextIndex = 0
const worker = async (): Promise<void> => {
while (nextIndex < items.length) {
const index = nextIndex
nextIndex += 1
try {
results[index] = {
status: 'fulfilled',
value: await work(items[index], index),
}
} catch (reason) {
results[index] = { status: 'rejected', reason }
}
}
}
const workerCount = Math.min(maxConcurrency, items.length)
await Promise.all(Array.from({ length: workerCount }, () => worker()))
return results
}
/**
* State of the readiness poll window for one McpServer (issue #223).
*
* A "window" is one bounded observation of a single Deployment rollout:
* maxAttempts ticks of intervalMs each. The window survives repeated
* reconciles — see `readinessPolls` below for why that is load-bearing.
*/
interface ReadinessPollState {
/**
* Timer for the next tick; undefined while a tick is executing its awaits
* (the handle is consumed the moment the callback fires) and once the
* window reached a terminal verdict.
*/
timer: ReturnType<typeof setTimeout> | undefined
/**
* True while a tick is executing. `timer` alone cannot distinguish "tick
* running RIGHT NOW" from "window ended", and the difference is
* load-bearing for pollReadiness() re-entrancy (see `recheckRequested`).
*/
ticking: boolean
/**
* Set by pollReadiness() when it is called while a tick is in flight. The
* caller (a reconcile) may have JUST rewritten the Deployment — e.g. a
* corrected credential rotation bumping the generation — and the in-flight
* tick's read, started earlier, cannot have seen that write. While this
* flag is set the tick must NOT commit a terminal verdict (ready or
* exhausted): it defers one interval so the next read, which starts after
* that Deployment write, observes the new generation and resets the
* window. Without this, the exhausting tick could publish RolloutIncomplete
* with the SUPERSEDED generation's numbers and — since McpServer has no
* periodic resync or Deployment watch — nothing would ever re-arm the poll
* for the new rollout (issue #223).
*/
recheckRequested: boolean
/**
* Deployment generation the current window is watching; null until the
* first tick reads it. A change means a NEW rollout (e.g. another
* credential rotation), which resets the window so the new rollout gets
* its own full budget.
*/
generation: number | null
/** Ticks consumed by the current window. */
attempts: number
/** True once the terminal RolloutIncomplete verdict was written for `generation`. */
exhausted: boolean
/**
* Rollout numbers from the exhausting tick. reconcile() re-writes them so
* the terminal condition stays sticky instead of being downgraded to
* WaitingForReplicas by the very reconcile the terminal write triggers.
*/
detail: string
/**
* Mutation fence and CRD object of the LATEST reconcile that owns this
* window. pollReadiness() REBINDS both on every call that finds the window
* live: a superseding reconcile (e.g. the fullReconcile after a routine
* McpServer watch recovery bumps the inventory-authority generation) becomes
* the window's owner, so the next tick evaluates the CURRENT fence instead of
* the arming reconcile's stale closure. Without the rebind the stale fence
* self-terminates the window with no re-arm and — status-only MODIFIED events
* being suppressed, with no periodic McpServer resync — DeploymentReady=False
* sticks forever for a healthy rollout (the #223 rotation-timeout symptom,
* reintroduced under a routine reconnect).
*
* Two ORTHOGONAL supersession guards protect this window; do not assume one
* covers the other (that assumption is exactly how the F1 race was born):
* - Identity retirement: beginStatusTracking()/clearStatus() retire the
* window on any CRD uid or CRD-generation change (spec edit, recreate),
* so a surviving window always has the caller's status identity — a
* rebind can never adopt a recreate or a superseded desired revision.
* - In-tick generation reset + recheckRequested: a credential rotation
* bumps neither CRD uid nor CRD generation (it changes the Deployment's
* metadata.generation only), so identity retirement does NOT fire; the
* rotation is instead guarded by the tick's live Deployment read (a new
* generation resets attempts/exhausted) and by recheckRequested deferring
* a mid-flight tick's terminal verdict past the rotation's write.
* A rebind never touches the budget fields (generation/attempts/exhausted/
* detail) above.
*/
isCurrent: () => boolean
server: McpServerCRD
/**
* Snapshot of `readinessRetirements` at window create time (#606). A later
* retirement increments the name's counter; the tick must not publish after
* that, even if this state was already deleted from `readinessPolls`.
*/
retirementEpoch: number
}
export class McpServerReconciler {
private readonly appsApi: k8s.AppsV1Api
private readonly coreApi: k8s.CoreV1Api
private readonly customApi: k8s.CustomObjectsApi
/**
* Snapshot the McpServer LIST→WATCH authority owned by McpServerWatcher.
* The fail-closed default prevents orphan cleanup until the watcher wires
* the real authority source.
*/
private inventoryAuthority: () => McpServerInventoryAuthoritySnapshot = () => ({
known: false,
generation: 0,
})
private inventoryAuthorityConfigured = false
private resolveCurrentServer?: (name: string) => McpServerCRD | undefined
/** See McpServerReconcilerDeps: test-only opt-in to assume-current when the
* inventory-authority fence is unwired. Production leaves it false. */
private readonly assumeInventoryAuthorityWhenUnconfigured: boolean
private warnedUnconfiguredAuthority = false
/** Tracks the deployment status of each managed McpServer. */
private readonly statusMap: Map<string, McpServerStatus> = new Map()
private readonly statusIdentities: Map<string, Pick<McpServerCRD, 'uid' | 'generation'>> =
new Map()
/** G7: Tracks the initial `managed` value per McpServer to enforce immutability. */
private readonly managedSnapshot: Map<string, boolean> = new Map()
/**
* Readiness poll state per server name (issue #223, E2 status-update-loop).
*
* Entries outlive individual reconciles ON PURPOSE. Every status write this
* reconciler makes fires a MODIFIED watch event on the McpServer, which
* fires another reconcile, which calls pollReadiness() again. When that
* call cancelled and re-armed the poll (the pre-fix behavior), the attempt
* budget restarted on every status write, so a rollout that never converges
* NEVER exhausted its budget and the terminal RolloutIncomplete verdict was
* never written to the CRD. This map lets pollReadiness() detect a live
* window and leave it alone, and lets reconcile() keep an exhausted verdict
* sticky for the same Deployment generation.
*/
private readonly readinessPolls: Map<string, ReadinessPollState> = new Map()
/**
* Retirement epoch per server name (#606). Incremented when a disabled or
* fail-closed reconcile retracts the readiness window. `clearStatus` does
* not delete entries: a recreated name keeps the counter so stale ticks fail
* the epoch check in addition to the uid fence.
*/
private readonly readinessRetirements = new Map<string, number>()
/**
* Tail promise of the in-flight reconcile chain per server name. Serializes
* concurrent reconcile() calls for the same McpServer so the CRD watch and
* SecretInformer can't race to replaceNamespacedDeployment() and produce
* spurious 409 "object has been modified" errors.
*/
private readonly inFlight: Map<string, Promise<void>> = new Map()
// A fulfilled reconcile promise can mean an authority guard retired work or
// a resource API error was reported through status. Keep that distinction
// independently of discovery readiness and the per-name promise queue.
private readonly pendingReconciliations = new Map<string, { namespace: string }>()
hasIncompleteReconciliation(): boolean {
return this.pendingReconciliations.size > 0
}
incompleteReconciliationNames(): string[] {
return [...this.pendingReconciliations.keys()]
}
hasPendingReconciliation(): boolean {
return (
this.hasIncompleteReconciliation() ||
this.inFlight.size > 0 ||
[...this.readinessPolls.values()].some(
poll => (poll.timer !== undefined || poll.ticking) && this.readinessPollStillDesired(poll)
)
)
}
private readinessPollStillDesired(poll: ReadinessPollState): boolean {
const current = this.resolveCurrentServer?.(poll.server.name)
return (
this.resolveCurrentServer === undefined ||
(current !== undefined && sameMcpServerDesiredRevision(poll.server, current))
)
}
private retainRetiredReadinessPoll(poll: ReadinessPollState): void {
if (this.readinessPollStillDesired(poll)) {
this.pendingReconciliations.set(poll.server.name, { namespace: poll.server.namespace })
}
}
/**
* Retract the live readiness window for a name that will no longer run a
* runtime (#606). Increments the retirement epoch first so an in-flight tick
* that already deleted its map entry still fails `retiredNow()`. Clears any
* pending timer. Does not record an incomplete-publication obligation. Poll
* ticks pass an `onDecline` factory evaluated at decline time: silent only
* when `retiredNow()` is true, otherwise the revision fence still records
* an obligation.
*/
private retireReadinessWindow(name: string): void {
this.readinessRetirements.set(name, (this.readinessRetirements.get(name) ?? 0) + 1)
const poll = this.readinessPolls.get(name)
if (poll === undefined) return
if (poll.timer !== undefined) {
clearTimeout(poll.timer)
poll.timer = undefined
}
this.readinessPolls.delete(name)
}
constructor(kc: k8s.KubeConfig, deps?: McpServerReconcilerDeps) {
this.appsApi = deps?.appsApi ?? kc.makeApiClient(k8s.AppsV1Api)
this.coreApi = deps?.coreApi ?? kc.makeApiClient(k8s.CoreV1Api)
this.customApi = deps?.customApi ?? kc.makeApiClient(k8s.CustomObjectsApi)
this.assumeInventoryAuthorityWhenUnconfigured =
deps?.assumeInventoryAuthorityWhenUnconfigured ?? false
}
/**
* Late-bound authority source for destructive full-reconcile cleanup.
* A full pass may finish reconciling desired servers, but it may delete an
* orphan only while the same authoritative inventory generation remains
* active from pass start through delete admission.
*/
setInventoryAuthority(fn: () => McpServerInventoryAuthoritySnapshot): void {
this.inventoryAuthority = fn
this.inventoryAuthorityConfigured = true
}
/**
* Resolve current cache membership at effect admission. Full-reconcile
* snapshots select candidates only; they must never overwrite or delete a
* same-name CRD accepted by the continuing watch.
*/
setResolveCurrentServer(fn: (name: string) => McpServerCRD | undefined): void {
this.resolveCurrentServer = fn
}
/**
* Verdict for a mutation fence evaluated while the inventory authority is NOT
* wired. Fail closed (deny) unless a test explicitly opted in; deny loudly
* once, because an unconfigured fence in a running controller is a
* construction bug, never a runtime condition.
*/
private currentWhenUnconfigured(): boolean {
if (this.assumeInventoryAuthorityWhenUnconfigured) return true
if (!this.warnedUnconfiguredAuthority) {
this.warnedUnconfiguredAuthority = true
console.error(
'[Reconciler] inventory authority not configured — mutation fences fail closed. ' +
'Wire setInventoryAuthority() and setResolveCurrentServer() before reconciling.'
)
}
return false
}
// ─── Status ────────────────────────────────────────────────────────
/**
* Get deployment status for an McpServer.
* Returns current in-memory status when this process has reconciled the
* server. After a controller restart, a generation-matched persisted Ready
* condition is the only safe fallback: a legacy condition without
* observedGeneration, or a condition for an older spec generation, cannot
* prove that the current desired runtime is ready.
*/
getStatus(serverOrName: string | McpServerCRD): McpServerStatus {
const name = typeof serverOrName === 'string' ? serverOrName : serverOrName.name
const current = this.statusMap.get(name)
if (current && (typeof serverOrName === 'string' || this.statusIdentityMatches(serverOrName))) {
return typeof serverOrName === 'string' ? current : { ...current, authoritative: true }
}
if (typeof serverOrName !== 'string' && serverOrName.generation !== undefined) {
const readyConditions =
serverOrName.status?.conditions?.filter(condition => condition.type === 'Ready') ?? []
const persistedReady =
readyConditions.length === 1 &&
readyConditions[0].observedGeneration === serverOrName.generation
? readyConditions[0]
: undefined
if (persistedReady) {
const ready = persistedReady.status === 'True'
return {
deployed: ready,
ready,
message: persistedReady.message,
authoritative: true,
}
}
}
return {
deployed: false,
ready: false,
message: 'Not reconciled yet',
authoritative: false,
}
}
private statusIdentityMatches(server: McpServerCRD): boolean {
const identity = this.statusIdentities.get(server.name)
return (
identity !== undefined &&
identity.uid === server.uid &&
identity.generation === server.generation
)
}
private beginStatusTracking(server: McpServerCRD): void {
const previousIdentity = this.statusIdentities.get(server.name)
if (!this.statusIdentityMatches(server)) {
this.statusMap.delete(server.name)
// A new CRD identity (uid/generation) retires any poll window armed for
// the previous desired revision: its verdicts would describe a rollout
// that no longer represents the desired state. The reconcile for the
// new identity arms its own window after its Deployment write.
const poll = this.readinessPolls.get(server.name)
if (poll) {
if (poll.timer !== undefined) clearTimeout(poll.timer)
this.readinessPolls.delete(server.name)
}
if (previousIdentity !== undefined && previousIdentity.uid !== server.uid) {
this.managedSnapshot.delete(server.name)
}
}
this.statusIdentities.set(server.name, {
uid: server.uid,
generation: server.generation,
})
}
/**
* Update the status of an McpServer after reconciliation.
*/
private setStatus(name: string, status: McpServerStatus): void {
this.statusMap.set(name, status)
}
/**
* Remove status tracking for a deleted McpServer.
*/
private clearStatus(name: string): void {
this.statusMap.delete(name)
this.statusIdentities.delete(name)
this.managedSnapshot.delete(name)
const poll = this.readinessPolls.get(name)
if (poll) {
if (poll.timer !== undefined) clearTimeout(poll.timer)
this.readinessPolls.delete(name)
}
}
/**
* Read a Deployment's rollout state, generation-aware.
*
* `readyReplicas > 0` is NOT enough (issue #223): during a rolling update the
* OLD pod is still Ready, so a new pod that never starts would be reported as
* healthy. A rollout has actually converged only when the controller has
* observed the current generation AND every replica is updated, ready, and
* available.
*
* `totalReplicas === updatedReplicas` is the part that catches a FAILED
* rollout of a credential rotation (issue #223, E2): with replicas=1 and the
* default RollingUpdate (maxSurge=1, maxUnavailable=0), a new pod that
* crash-loops on a bad credential never becomes Ready, so Kubernetes keeps
* the OLD pod. That old pod holds readyReplicas=1 and availableReplicas=1, so
* unavailableReplicas stays 0 and updatedReplicas reaches 1 — every other
* signal looks converged. Only `status.replicas` (total managed pods) reveals
* the leftover: it stays at 2 while the new pod crash-loops, and drops to 1
* only once the rollout truly completes. Without this check the poll would
* report a failed rotation as healthy and never write RolloutIncomplete.
*
* `detail` carries the numbers into status conditions so a stuck rollout says
* what is stuck instead of just "not ready".
*
* `generation` is the Deployment's `metadata.generation`, exposed so the
* readiness poll can tell "same rollout, keep counting" apart from "NEW
* rollout, fresh window" (issue #223, E2). It plays no part in the
* readiness verdict above. null when the Deployment could not be read.
*/
private async readDeploymentRollout(
name: string,
namespace: string
): Promise<{ ready: boolean; detail: string; generation: number | null }> {
try {
const deployment = await this.appsApi.readNamespacedDeployment({ name, namespace })
const desired = deployment.spec?.replicas ?? 1
const generation = deployment.metadata?.generation ?? 0
const observedGeneration = deployment.status?.observedGeneration ?? 0
const updatedReplicas = deployment.status?.updatedReplicas ?? 0
const readyReplicas = deployment.status?.readyReplicas ?? 0
const totalReplicas = deployment.status?.replicas ?? 0
const unavailableReplicas = deployment.status?.unavailableReplicas ?? 0
const ready =
observedGeneration >= generation &&
updatedReplicas === desired &&
readyReplicas === desired &&
totalReplicas === updatedReplicas &&
unavailableReplicas === 0
return {
ready,
generation,
detail:
`generation ${observedGeneration}/${generation}, ` +
`updated ${updatedReplicas}/${desired}, ready ${readyReplicas}/${desired}, ` +
`total ${totalReplicas}, unavailable ${unavailableReplicas}`,
}
} catch (error: unknown) {
const message = error instanceof Error ? error.message : String(error)
return { ready: false, generation: null, detail: `Deployment read failed: ${message}` }
}
}
/**
* Poll deployment readiness until ready or maxAttempts reached.
*
* Re-entrancy is the whole point (issue #223, E2 status-update-loop):
* reconcile() runs after every MODIFIED watch event — including the events
* caused by our OWN status writes — and calls this on every pass where the
* Deployment is not ready yet. A live window is therefore NEVER cancelled
* or re-armed here; it keeps counting toward its budget so a rollout that
* never converges reaches the terminal RolloutIncomplete verdict after
* maxAttempts × intervalMs. A NEW rollout (bumped Deployment generation,
* e.g. another credential rotation) is detected inside the poll tick
* itself, which resets the window so the new rollout gets its own budget.
*
* A call that lands while a tick is EXECUTING (mid-await) is not silently
* dropped either: it flags `recheckRequested` so the tick defers any
* terminal verdict until a read taken AFTER the caller's Deployment write
* confirms the generation — otherwise the exhausting tick could seal the
* window with a superseded generation's numbers and the new rollout would
* never be observed (see ReadinessPollState.recheckRequested).
*/
private pollReadiness(
server: McpServerCRD,
intervalMs = 5000,
maxAttempts = 24,
isCurrent: () => boolean = () => true
): void {
const { name, namespace } = server
const previous = this.readinessPolls.get(name)
if (previous !== undefined && (previous.timer !== undefined || previous.ticking)) {
// A window is already running for this server — leave its TIMER and
// BUDGET alone (re-arming would reset the attempt budget: the E2
// status-update-loop). But this caller is now the window's owner: rebind
// the mutation fence (and the same-identity server object) so the next
// tick evaluates the CURRENT fence instead of the arming reconcile's
// stale closure, which would otherwise self-terminate the window with no
// re-arm after a routine watch-generation bump.
previous.isCurrent = isCurrent
previous.server = server
// If its tick is executing RIGHT NOW, the tick's Deployment read may
// predate the Deployment write that triggered THIS call (a new
// generation), so a silent no-op could let the tick end the window on
// stale data. Record that a re-check is owed; the tick honors it before
// committing any terminal verdict.
if (previous.ticking) {
previous.recheckRequested = true
}
return
}
// Fresh window. Carry over the generation and terminal verdict of a
// previous exhausted window: if the Deployment has NOT changed since,
// the new window must not mistake it for a new rollout, and reconcile()
// keeps reporting the sticky RolloutIncomplete meanwhile.
const state: ReadinessPollState = {
timer: undefined,
ticking: false,
recheckRequested: false,
generation: previous?.generation ?? null,
attempts: 0,
exhausted: previous?.exhausted ?? false,
detail: previous?.detail ?? '',
isCurrent,
server,
retirementEpoch: this.readinessRetirements.get(name) ?? 0,
}
this.readinessPolls.set(name, state)
const poll = async (): Promise<void> => {
// The timer handle was consumed the moment this callback fired; reflect
// that, and mark the tick as in flight so a concurrent pollReadiness()
// can tell "tick running" apart from "window ended" during our awaits.
state.timer = undefined
state.ticking = true
const retiredNow = (): boolean =>
(this.readinessRetirements.get(name) ?? 0) !== state.retirementEpoch
const publishIfCurrent = (): boolean => state.isCurrent() && !retiredNow()
const pollWriteDecline = (): StatusWriteDecline =>
retiredNow() ? 'silent' : 'record-obligation'
// Mutation fencing: a superseded reconcile (desired-revision or
// inventory-authority change) must not keep observing — or writing
// status — on behalf of a stale desired state. End the window quietly;
// the current reconcile arms its own window after its Deployment write.
if (!state.isCurrent()) {
this.retainRetiredReadinessPoll(state)
state.ticking = false
if (this.readinessPolls.get(name) === state) this.readinessPolls.delete(name)
return
}
const rollout = await this.readDeploymentRollout(name, namespace)
if (!state.isCurrent() || !this.statusIdentityMatches(state.server)) {
this.retainRetiredReadinessPoll(state)
state.ticking = false
if (this.readinessPolls.get(name) === state) this.readinessPolls.delete(name)
return
}
// The McpServer may have been deleted (clearStatus) while we were
// reading. A cancelled window must not keep ticking or write status
// for a CRD that is gone.
if (this.readinessPolls.get(name) !== state) return
// Generation change ⇒ a NEW rollout: it deserves its own full budget,
// and any terminal verdict for the old generation no longer applies.
// A failed read (generation null) never resets the window — otherwise
// a flapping API server would grant infinite budget.
if (rollout.generation !== null && rollout.generation !== state.generation) {
state.generation = rollout.generation
state.attempts = 0
state.exhausted = false
state.detail = ''
}
state.attempts++
// A pollReadiness() call landed while this tick was mid-await: the
// Deployment may have been rewritten AFTER our read (that reconcile
// writes the Deployment BEFORE calling pollReadiness()), so a terminal
// verdict from this tick's data could blame a superseded generation —
// and nothing else would ever re-arm the poll for the new one. Defer
// the verdict one interval: the next read starts after that Deployment
// write, so it MUST observe the new generation and reset the window
// above. A same-generation echo merely delays the verdict by one tick;
// the budget is NOT reset, so the E2 status-update-loop stays fixed.
if (state.recheckRequested) {
state.recheckRequested = false
if (rollout.ready || state.attempts >= maxAttempts) {
state.ticking = false
state.timer = setTimeout(poll, intervalMs)
return
}
}
if (rollout.ready) {
state.ticking = false
this.setStatus(name, { deployed: true, ready: true, message: 'Running' })
this.readinessPolls.delete(name)
console.log(
`[Reconciler] McpServer "${name}" is now ready (after ${state.attempts} poll(s))`
)
// The poll MUST write the CRD, not just in-memory status (issue #223).
// With generation-aware readiness the check right after a replace is
// always false, so this is the ONLY place a converged rollout is ever
// recorded — without it every successful credential rotation would sit
// at DeploymentReady=False forever and the UI would report a timeout.
if (retiredNow()) return
await this.updateStatusConditions(
state.server,
true,
undefined,
publishIfCurrent,
pollWriteDecline
)
// Readiness is decoupled from fleet convergence: the poll also
// upgrades the persisted, generation-matched Ready condition so a
// controller restart can trust the recorded readiness (see getStatus).
if (retiredNow()) return
await this.writeStatusCondition(
state.server,
{
type: 'Ready',
status: 'True',
reason: 'ReconcileSuccess',
message: 'Deployment created',
},
publishIfCurrent,
pollWriteDecline
)
return
}
if (state.attempts >= maxAttempts) {
// Terminal verdict for THIS generation. The entry stays in the map
// (timer undefined since tick start, ticking cleared here — BEFORE
// the status write below, so a reconcile arriving during that await
// opens a fresh observation window instead of flagging a re-check
// this dying tick would never honor) so reconcile() keeps the verdict
// sticky and a later pollReadiness() call may re-observe.
state.ticking = false
state.exhausted = true
state.detail = rollout.detail
console.warn(
`[Reconciler] McpServer "${name}" not ready after ${maxAttempts} polls — giving up (${rollout.detail})`
)
// Give up loudly, on the CRD: a stuck rollout must be visible to the
// operator with the numbers that prove it, never left as a stale
// condition or a silent Unknown.
if (retiredNow()) return
await this.updateStatusConditions(
state.server,
false,
rollout.detail,
publishIfCurrent,
pollWriteDecline
)
return
}
// Schedule next poll
state.ticking = false
state.timer = setTimeout(poll, intervalMs)
}
state.timer = setTimeout(poll, intervalMs)
}
// ─── Secret Validation ──────────────────────────────────────────────
/**
* Validate that the referenced secret exists and contains all required keys.
* Returns a discriminated result describing success or the exact failure mode.
*/
async validateSecret(server: McpServerCRD): Promise<SecretValidationResult> {
// No envSecret: nothing to hash, and no credentials annotation is written.
if (!server.spec.envSecret) return { ok: true, revision: '' }
const secretName = server.spec.envSecret.name
try {
const secret = await this.coreApi.readNamespacedSecret({
name: secretName,
namespace: server.namespace,
})
const data = secret.data || {}
const revision = createHash('sha256')
.update(canonicalStringify(data as Record<string, unknown>))
.digest('hex')
for (const keyMapping of server.spec.envSecret.keys) {
if (!(keyMapping.secretKey in data)) {
const message =
`Secret "${secretName}" is missing required key "${keyMapping.secretKey}" ` +
`(needed for env var ${keyMapping.envVar})`
console.error(`[Reconciler] ${message} in McpServer ${server.name}`)
return { ok: false, reason: 'SecretMissingKey', message }
}
}
console.log(`[Reconciler] Secret "${secretName}" validated for McpServer ${server.name}`)
return { ok: true, revision }
} catch (error: unknown) {
const code = getErrorCode(error)
if (code === 404) {
const message = `Secret "${secretName}" not found in namespace "${server.namespace}"`
console.error(
`[Reconciler] ${message}. McpServer ${server.name} will NOT be deployed. Create the secret first.`
)
return { ok: false, reason: 'SecretNotFound', message }
}
if (code === 401 || code === 403) {
const message =
`Access denied reading Secret "${secretName}" in namespace "${server.namespace}" ` +
`(K8s API ${code})`
console.error(`[Reconciler] ${message}. McpServer ${server.name} will be failed closed.`)
return { ok: false, reason: 'SecretAccessDenied', message }
}
const errMsg = error instanceof Error ? error.message : String(error)
console.error(`[Reconciler] Failed to read secret "${secretName}":`, error)
return {
ok: false,
reason: 'ReadError',
message: `Failed to read Secret "${secretName}": ${errMsg}`,
}
}
}
// ─── Resource Builders ──────────────────────────────────────────────
// ─── Remote Egress Proxy ─────────────────────────────────────────
/**
* Check if this McpServer is a remote egress proxy (has spec.remote.baseUrl).
*/
private isRemote(server: McpServerCRD): boolean {
return !!server.spec.remote?.baseUrl
}
/**
* Build nginx.conf ConfigMap for a remote MCP egress proxy.
* Generates an nginx reverse proxy config that forwards to the external baseUrl.
*/
/**
* Validate and sanitize remote baseUrl to prevent SSRF and nginx.conf injection.
* Returns safe URL reconstructed from parsed components.
*/
private sanitizeRemoteUrl(rawUrl: string): { safeUrl: string; host: string; path: string } {
const parsed = new URL(rawUrl) // throws on malformed — no silent fallback
if (parsed.protocol !== 'https:') {
throw new Error(`remote.baseUrl must use HTTPS (got ${parsed.protocol})`)
}
const host = parsed.hostname
// Block IP-literal URLs entirely (IPv4 and IPv6) — require DNS hostnames only
// This prevents IPv6 SSRF bypasses (::1, ::ffff:7f00:1, fd00::, etc.)
const bareHost = host.replace(/^\[|\]$/g, '')
if (/^[\d.:[\]]+$/.test(bareHost) || bareHost.includes(':')) {
throw new Error(`remote.baseUrl must use a DNS hostname, not an IP address: ${host}`)
}
// Block internal cluster services (SSRF prevention)
if (
host.endsWith('.svc.cluster.local') ||
host.endsWith('.svc') ||
host === 'kubernetes.default'
) {
throw new Error(`remote.baseUrl must not point to internal cluster services: ${host}`)
}
// Block RFC1918, link-local, loopback (defense in depth for edge cases)
const PRIVATE_IP = /^(10\.|172\.(1[6-9]|2\d|3[01])\.|192\.168\.|127\.|169\.254\.|0\.)/
if (PRIVATE_IP.test(host)) {
throw new Error(`remote.baseUrl must not point to private/reserved IP ranges: ${host}`)
}
// Reconstruct safe URL from parsed components (strips injected chars like newlines)
const path = parsed.pathname
const safeUrl = parsed.origin + path + parsed.search
return { safeUrl, host: parsed.host, path }
}
/**
* Codex P0 fix (PR #101): sanitize + validate authHeaders from the McpServer CRD
* before emitting them as nginx `proxy_set_header` lines. Prevents nginx config
* injection (breakouts from the quoted string) and HTTP header smuggling.
*
* Returns a safe { header, valueTemplate } tuple with the valueTemplate already
* escaped for nginx double-quoted string context, or null if the entry is
* rejected (a console.warn is emitted describing why).
*/
private sanitizeAuthHeader(h: {
header: string
valueTemplate: string
}): { header: string; valueTemplate: string } | null {
// Validate header name — HTTP token chars only (RFC 7230 subset we allow).
if (!HEADER_NAME_RE.test(h.header)) {
console.warn(`[HCC] Rejecting authHeader with invalid name: ${h.header.slice(0, 32)}`)
return null
}
// Bound the value length so a pathological CRD cannot blow up the ConfigMap.
if (h.valueTemplate.length > MAX_AUTH_HEADER_VALUE_LENGTH) {
console.warn(`[HCC] Rejecting authHeader value too long for header: ${h.header}`)
return null
}
// CR/LF/NUL allow HTTP request smuggling and nginx directive injection.
if (/[\r\n\0]/.test(h.valueTemplate)) {
console.warn(`[HCC] Rejecting authHeader with CR/LF/NUL in value for header: ${h.header}`)
return null
}
// Escape backslashes (first, to avoid double-escaping) then double quotes so
// the value is safe inside nginx's `"..."` string context.
const escaped = h.valueTemplate.replace(/\\/g, '\\\\').replace(/"/g, '\\"')
return { header: h.header, valueTemplate: escaped }
}
private buildNginxConfigMap(server: McpServerCRD): k8s.V1ConfigMap {
const {
safeUrl,
host: upstreamHost,
path: upstreamPath,
} = this.sanitizeRemoteUrl(server.spec.remote!.baseUrl)
const transportPort = server.spec.transport.port || 3000
// Codex P1 fix (PR #101): forward credential secrets as upstream auth headers.
// Each authHeader produces a `proxy_set_header` line; ${VAR} placeholders in
// valueTemplate are resolved at pod startup by Clerum's nginx egress proxy
// /etc/nginx/templates/ envsubst (env vars from envSecret are already mounted
// into the egress-proxy container by buildDeployment).
//
// Codex P0 fix (PR #101): each entry is sanitized + validated via
// sanitizeAuthHeader() before emission so an operator-authored CRD cannot
// break out of the quoted string to inject arbitrary nginx config, and
// cannot smuggle HTTP headers via CR/LF.
const sanitizedAuthHeaders = (server.spec.remote!.authHeaders ?? [])
.map(h => this.sanitizeAuthHeader(h))
.filter((h): h is { header: string; valueTemplate: string } => h !== null)
const authHeaderLines = sanitizedAuthHeaders
.map(h => ` proxy_set_header ${h.header} "${h.valueTemplate}";`)
.join('\n')
// ── D-2 Authorization precedence/clearing ──
// Exactly one Authorization directive is emitted, by precedence, because the
// security decision here is non-obvious:
// oauth (and no static Authorization) → passthrough the per-user Bearer that
// mcp-host injects ($http_authorization);
// authHeaders declares Authorization → emit nothing here — the operator's
// static credential (already rendered by
// authHeaderLines) wins;
// otherwise (static remote) → clear it ("") so a static remote never
// forwards an incoming Authorization by accident.
// An oauth server that also statically pins Authorization is a contradiction:
// the static value would shadow the per-user Bearer. Reject the reconcile.
//
// Two different detections on purpose:
// - CONFLICT (oauth) is keyed on the RAW authHeaders: an oauth server must not
// declare Authorization at all, even a malformed value that sanitization would
// drop — otherwise the reject could be bypassed by injecting a bad value.
// - The clear-vs-operator-wins decision is keyed on the SANITIZED/EMITTED set:
// if the operator's Authorization entry was dropped by sanitizeAuthHeader (bad
// name/length/CR-LF-NUL), no static line is rendered, so we must still CLEAR —
// keying this on the raw list would leave the incoming Authorization forwarded.
const isOAuth = server.spec.auth?.type === 'oauth'
const rawDeclaresAuthorizationHeader = (server.spec.remote!.authHeaders ?? []).some(
h => h.header.toLowerCase() === 'authorization'
)
const emitsAuthorizationHeader = sanitizedAuthHeaders.some(
h => h.header.toLowerCase() === 'authorization'
)
if (isOAuth && rawDeclaresAuthorizationHeader) {
hccLogger.error(
'Rejecting remote reconcile: oauth server pins a static Authorization header',
{
scope: '[Reconciler]',
mcpServer: server.name,
namespace: server.namespace,
}
)
throw new Error(
`remote MCP server "${server.name}" uses auth.type 'oauth' but also declares a static ` +
'Authorization header in spec.remote.authHeaders; the per-user Bearer must pass through — ' +
'remove the Authorization entry'
)
}
const authorizationDirective = isOAuth
? ' proxy_set_header Authorization $http_authorization;'
: emitsAuthorizationHeader
? ''
: ' proxy_set_header Authorization "";'
const authorizationBlock = authorizationDirective
? `\n # ── Authorization precedence (D-2): ${
isOAuth
? 'passthrough the per-user Bearer'
: 'clear — a static remote never forwards Authorization'
} ──\n${authorizationDirective}\n`
: ''
const nginxConf = `# Auto-generated by Clerum HCC for remote MCP server: ${server.name}
# Stored as .conf.template; Clerum's nginx egress proxy runs envsubst at startup.
server {
listen ${transportPort};
# Mirror the upstream endpoint path locally so nginx URI replacement keeps
# the remote endpoint path exactly once.
location ${upstreamPath} {
proxy_pass ${safeUrl};
proxy_set_header Host ${upstreamHost};
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolved) ──\n' + authHeaderLines + '\n' : ''}${authorizationBlock}
# SSE/streaming support
proxy_buffering off;
proxy_cache off;
proxy_connect_timeout 10s;
proxy_read_timeout 86400s;
proxy_send_timeout 86400s;
chunked_transfer_encoding on;
# TLS to upstream
proxy_ssl_verify on;
proxy_ssl_server_name on;
proxy_ssl_trusted_certificate /etc/ssl/certs/ca-certificates.crt;
# nginx defaults proxy_ssl_verify_depth to 1, which only admits a chain
# with a single untrusted intermediate. CAs mid-root-rollover serve a
# chain that reaches a trusted anchor through a cross-signed root, so the
# leaf sits two intermediates below the trust store: Let's Encrypt via
# "Root YE"/"Root YR" (cross-signed by ISRG Root X2/X1) and Microsoft via
# "Microsoft TLS ECC Root G2" (cross-signed by MS ECC Root 2017). At depth
# 1 nginx rejects those with "(20:unable to get local issuer certificate)"
# even though the CA bundle is complete — it looks like a missing root but
# is purely the depth cap. Raising the depth keeps verification fully on.
proxy_ssl_verify_depth 3;
}
# Health check endpoint
location /health {
return 200 'ok';
add_header Content-Type text/plain;
}
}`
const labels: Record<string, string> = {
app: server.name,
[MANAGED_BY_LABEL]: MANAGED_BY_VALUE,
[MCPSERVER_LABEL]: server.name,
'clerum.io/component': 'egress-proxy',
}
return {
apiVersion: 'v1',
kind: 'ConfigMap',
metadata: {
name: `${server.name}-nginx-conf`,
namespace: server.namespace,
labels,
...(server.uid && {
ownerReferences: [
{
apiVersion: 'clerum.io/v1alpha1',
kind: 'McpServer',
name: server.name,
uid: server.uid,
controller: true,
blockOwnerDeletion: true,
},
],
}),
},
data: {
// Stored as .conf.template so Clerum's nginx egress proxy runs envsubst at
// startup, resolving ${VAR} placeholders from env (sourced from envSecret).
'default.conf.template': nginxConf,
},
}
}
/**
* Ensure ConfigMap exists and is up to date (create or replace).
*/
private async ensureConfigMap(
server: McpServerCRD,
isCurrent: () => boolean = () => true
): Promise<void> {
const cm = this.buildNginxConfigMap(server)
const name = `${server.name}-nginx-conf`
const namespace = server.namespace
await ensureResource({
mutationAllowed: isCurrent,
read: () =>
observeExistenceRead('ConfigMap', () =>
this.coreApi.readNamespacedConfigMap({ name, namespace })
),
create: async () => {
await observeCreate('ConfigMap', () =>
this.coreApi.createNamespacedConfigMap({ namespace, body: cm })
)
hccLogger.info('nginx ConfigMap created', {
scope: '[Reconciler]',
configMap: name,
namespace,
})
},
converge: read =>
replaceWithConflictRetry({
description: `nginx ConfigMap "${name}"`,
logPrefix: '[Reconciler]',
body: cm,
mergeExisting: preserveObjectAnnotations,
isUpToDate: configMapMatchesDesired,
mutationAllowed: isCurrent,
read,
replace: body => this.coreApi.replaceNamespacedConfigMap({ name, namespace, body }),
}),
onSkipped: () => createsTotal.inc({ kind: 'ConfigMap', outcome: 'skipped' }),
})
}
/**
* Build a Deployment manifest for an McpServer.
*/
/**
* Sanitize CRD spec fields to prevent developer-supplied CRDs from
* influencing platform-controlled deployment behavior.
*
* Policy: the platform ALWAYS wins. CRD fields that could escalate
* privileges, override security, or DoS the cluster are stripped or
* overridden with platform defaults.
*/
private sanitizeCrdSpec(server: McpServerCRD): void {
// ── CRITICAL: imagePullPolicy — platform decides, not the CRD ──
// Remove CRD-supplied value; buildDeployment uses config.mcpServerImagePullPolicy
delete server.spec.imagePullPolicy
// ── CRITICAL: prevent root execution ──
if (server.spec.security) {
if (server.spec.security.runAsUser !== undefined && server.spec.security.runAsUser < 1) {
console.warn(
`[Security] McpServer "${server.name}": runAsUser=${server.spec.security.runAsUser} rejected (must be >= 1)`
)
server.spec.security.runAsUser = 1000
}
if (server.spec.security.runAsGroup !== undefined && server.spec.security.runAsGroup < 1) {
console.warn(
`[Security] McpServer "${server.name}": runAsGroup=${server.spec.security.runAsGroup} rejected (must be >= 1)`
)
server.spec.security.runAsGroup = 1000
}
if (server.spec.security.fsGroup !== undefined && server.spec.security.fsGroup < 1) {
console.warn(
`[Security] McpServer "${server.name}": fsGroup=${server.spec.security.fsGroup} rejected (must be >= 1)`
)
server.spec.security.fsGroup = 1000
}
// ── CRITICAL: strip dangerous capabilities ──
if (server.spec.security.addCapabilities) {
const original = server.spec.security.addCapabilities
server.spec.security.addCapabilities = original.filter(cap => {
if (!isWorkflowRecipeDefaultAllowedCapability(cap)) {
console.warn(
`[Security] McpServer "${server.name}": capability "${cap}" stripped (forbidden)`
)
return false
}
return true
})
}
}
// ── HIGH: sanitize env vars — block dangerous names ──
const FORBIDDEN_ENV = new Set([
'LD_PRELOAD',
'LD_LIBRARY_PATH',
'PATH',
'NODE_OPTIONS',
'PYTHONPATH',
'JAVA_TOOL_OPTIONS',
'KUBECONFIG',
'KUBERNETES_SERVICE_HOST',
'KUBERNETES_SERVICE_PORT',
])
if (server.spec.env) {
server.spec.env = server.spec.env.filter(e => {
if (FORBIDDEN_ENV.has(e.name.toUpperCase())) {
console.warn(
`[Security] McpServer "${server.name}": env "${e.name}" stripped (forbidden)`
)
return false
}
return true
})
}
// ── MEDIUM: clamp resources to platform limits ──
if (server.spec.resources) {
const MAX_CPU = '4000m'
const MAX_MEM = '8Gi'
if (server.spec.resources.limits) {
if (server.spec.resources.limits.cpu && parseInt(server.spec.resources.limits.cpu) > 4000) {
server.spec.resources.limits.cpu = MAX_CPU
}
if (
server.spec.resources.limits.memory &&
parseInt(server.spec.resources.limits.memory) > 8192
) {
server.spec.resources.limits.memory = MAX_MEM
}
}
}
}
private buildDeployment(server: McpServerCRD, credentialsRevision = ''): k8s.V1Deployment {
// ── Platform hardening: sanitize an ephemeral copy before building PodSpec ──
// The caller retains this object as the LIST -> WATCH desired-state cache.
// Mutating it here makes a later status-only watch event look like a
// user-driven spec revision, which invalidates the authoritative policy
// revocation marker during startup. Sanitization remains mandatory for the
// generated PodSpec, but must never rewrite the observed CRD inventory.
server = structuredClone(server)
this.sanitizeCrdSpec(server)
const transportPort = server.spec.transport.port || 3000
const isStdio = server.spec.transport.type === 'stdio'
const isRemote = this.isRemote(server)
const healthCheckPort = server.spec.healthCheck?.port
// --- Env vars ---
const env: k8s.V1EnvVar[] = []
// Env vars derived from CRD fields via envMapping
const mapping = server.spec.envMapping
if (mapping) {
if (mapping.transport) {
env.push({ name: mapping.transport, value: 'http' })
}
if (mapping.httpHost) {
env.push({ name: mapping.httpHost, value: '0.0.0.0' })
}
if (mapping.httpPort) {
env.push({ name: mapping.httpPort, value: String(transportPort) })
}
if (mapping.healthCheckHost) {
env.push({ name: mapping.healthCheckHost, value: '0.0.0.0' })
}
if (mapping.healthCheckPort && healthCheckPort) {
env.push({ name: mapping.healthCheckPort, value: String(healthCheckPort) })
}
const sc = server.spec.serverConfig
if (mapping.readOnly && sc?.readOnly !== undefined) {
env.push({ name: mapping.readOnly, value: String(sc.readOnly) })
}
if (mapping.loggers && sc?.loggers) {
env.push({ name: mapping.loggers, value: sc.loggers })
}
if (mapping.telemetry && sc?.telemetry) {
env.push({ name: mapping.telemetry, value: sc.telemetry })
}
}
// Additional plain env vars from CRD (server-specific extras)
if (server.spec.env) {
for (const e of server.spec.env) {
env.push({ name: e.name, value: e.value })
}
}
// Secret-backed env vars
if (server.spec.envSecret) {
for (const keyMapping of server.spec.envSecret.keys) {
env.push({
name: keyMapping.envVar,
valueFrom: {
secretKeyRef: {
name: server.spec.envSecret.name,
key: keyMapping.secretKey,
},
},
})
}
}
// Remote servers: ensure authHeaders ${VAR} placeholders have env vars
// defined, even when credentials are optional and envSecret is absent.
// Without this, nginx envsubst fails with "unknown variable" on startup.
if (isRemote && server.spec.remote?.authHeaders) {
const definedVars = new Set(env.map(e => e.name))
for (const ah of server.spec.remote.authHeaders) {
const matches = ah.valueTemplate.match(/\$\{(\w+)\}/g)
if (!matches) continue
for (const m of matches) {
const varName = m.slice(2, -1)
if (!definedVars.has(varName)) {
env.push({ name: varName, value: '' })
definedVars.add(varName)
}
}
}
}
// --- Ports ---
const ports: k8s.V1ContainerPort[] = [
{ name: 'http', containerPort: transportPort, protocol: 'TCP' },
]
if (healthCheckPort) {
ports.push({ name: 'healthcheck', containerPort: healthCheckPort, protocol: 'TCP' })
}
// --- Probes ---
const probePort = healthCheckPort ? 'healthcheck' : 'http'
// --- Resources ---
let resources: k8s.V1ResourceRequirements | undefined
if (server.spec.resources) {
resources = {}
if (server.spec.resources.requests) {
resources.requests = {}
if (server.spec.resources.requests.memory)
resources.requests['memory'] = server.spec.resources.requests.memory
if (server.spec.resources.requests.cpu)
resources.requests['cpu'] = server.spec.resources.requests.cpu
}
if (server.spec.resources.limits) {
resources.limits = {}
if (server.spec.resources.limits.memory)
resources.limits['memory'] = server.spec.resources.limits.memory
if (server.spec.resources.limits.cpu)
resources.limits['cpu'] = server.spec.resources.limits.cpu
}
}
// --- Labels ---
const labels: Record<string, string> = {
app: server.name,
[MANAGED_BY_LABEL]: MANAGED_BY_VALUE,
[MCPSERVER_LABEL]: server.name,
}
const runAsUser = server.spec.security?.runAsUser ?? 1000
const runAsGroup = server.spec.security?.runAsGroup ?? runAsUser
const hardenedContainerSecurityContext: k8s.V1SecurityContext = {
allowPrivilegeEscalation: false,
runAsNonRoot: true,
runAsUser,
runAsGroup,
capabilities: { drop: ['ALL'] },
seccompProfile: { type: 'RuntimeDefault' },
}
return {
apiVersion: 'apps/v1',
kind: 'Deployment',
metadata: {
name: server.name,
namespace: server.namespace,
labels,
// G5: ownerRef to McpServer CRD for garbage collection
...(server.uid && {
ownerReferences: [
{
apiVersion: 'clerum.io/v1alpha1',
kind: 'McpServer',
name: server.name,
uid: server.uid,
controller: true,
blockOwnerDeletion: true,
},
],
}),
},
spec: {
replicas: 1,
selector: {
matchLabels: { app: server.name },
},
template: {
metadata: {
labels,
// Issue #223: rotating a credential changes this digest, which
// changes the pod template, which is what makes Kubernetes roll the
// pod. Omitted when there is no envSecret — an annotation with an
// empty value would be one more field to drift on for connectors
// that have no credentials at all.
...(credentialsRevision && {
annotations: { [CREDENTIALS_REVISION_ANNOTATION]: credentialsRevision },
}),
},
spec: {
automountServiceAccountToken: false,
enableServiceLinks: false,
// G1: Apply security context from McpServer CRD spec
securityContext: {
runAsNonRoot: true,
runAsUser,
runAsGroup,
seccompProfile: { type: 'RuntimeDefault' },
...(server.spec.security?.fsGroup !== undefined && {
fsGroup: server.spec.security.fsGroup,
}),
},
// G6: Apply imagePullSecrets from McpServer CRD spec
// Codex P2 fix (PR #101): CRD schema mandates [{name: string}] per K8s
// LocalObjectReference; producers (mcpDelegation, registry install) now
// normalize to objects, so we pass through directly without double-wrapping.
...(server.spec.imagePullSecrets?.length
? {
imagePullSecrets: server.spec.imagePullSecrets,
}
: {}),
initContainers:
isStdio && !isRemote
? [
{
name: 'copy-mcp-app',
image: server.spec.image,
imagePullPolicy: config.mcpServerImagePullPolicy,
command: [
'sh',
'-c',
'cp -r /app/* /mcp-bin/ 2>/dev/null; cp -r /mcp-app/* /mcp-bin/ 2>/dev/null; true',
],
securityContext: hardenedContainerSecurityContext,
volumeMounts: [{ name: 'mcp-bin', mountPath: '/mcp-bin' }],
},
]
: undefined,
containers: [
// Remote egress proxy: nginx container with ConfigMap mount
...(isRemote
? [
{
name: 'egress-proxy',
image: config.egressProxyImage,
imagePullPolicy: 'IfNotPresent' as const,
ports: [
{ name: 'http', containerPort: transportPort, protocol: 'TCP' as const },
],
env: env.length > 0 ? env : undefined,
// Codex P1 fix (PR #101): mount as templates so the nginx proxy
// entrypoint runs envsubst at startup and resolves ${VAR}
// placeholders in auth headers using env from envSecret.
volumeMounts: [
{ name: 'nginx-conf', mountPath: '/etc/nginx/templates', readOnly: true },
],
livenessProbe: {
httpGet: { path: '/health', port: transportPort as unknown as IntOrString },
initialDelaySeconds: 5,
periodSeconds: 15,
},
readinessProbe: {
httpGet: { path: '/health', port: transportPort as unknown as IntOrString },
initialDelaySeconds: 3,
periodSeconds: 10,
},
resources: {
requests: { memory: '32Mi', cpu: '25m' },
limits: { memory: '64Mi', cpu: '100m' },
},
securityContext: {
runAsNonRoot: true,
runAsUser: 101,
runAsGroup: 101,
capabilities: { drop: ['ALL'] },
allowPrivilegeEscalation: false,
seccompProfile: { type: 'RuntimeDefault' },
},
},
]
: []),
// Local HTTP server (not stdio, not remote)
...(isStdio || isRemote
? []
: [
{
name: 'mcp-server',
image: server.spec.image,
imagePullPolicy: config.mcpServerImagePullPolicy,
command: server.spec.command || undefined,
args: server.spec.args || undefined,
ports,
env: env.length > 0 ? env : undefined,
// G1: Per-container security context — capabilities.add must be at container level
securityContext: {
allowPrivilegeEscalation: false,
runAsNonRoot: true,
runAsUser,
runAsGroup,
seccompProfile: { type: 'RuntimeDefault' },
capabilities: {
drop: ['ALL'],
...(server.spec.security?.addCapabilities?.length
? { add: server.spec.security.addCapabilities }
: {}),
},
},
livenessProbe: {
tcpSocket: { port: probePort as IntOrString },
initialDelaySeconds: 10,
periodSeconds: 15,
},
readinessProbe: {
tcpSocket: { port: probePort as IntOrString },
initialDelaySeconds: 5,
periodSeconds: 10,
},
resources,
},
]),
...(isStdio
? [
{
name: 'stdio-bridge',
image: config.stdioBridgeImage, // G10: configurable bridge image
imagePullPolicy: config.mcpServerImagePullPolicy,
env: [
{
name: 'STDIO_COMMAND',
value: (server.spec.command || ['/mcp-bin/mcp-server'])[0],
},
{
name: 'STDIO_ARGS',
value: JSON.stringify(
server.spec.args || (server.spec.command || []).slice(1)
),
},
{ name: 'BRIDGE_PORT', value: String(transportPort) },
...env,
],
ports: [{ name: 'http', containerPort: transportPort, protocol: 'TCP' }],
securityContext: hardenedContainerSecurityContext,
livenessProbe: {
httpGet: { path: '/health', port: transportPort as unknown as IntOrString },
initialDelaySeconds: 15,
periodSeconds: 15,
},
readinessProbe: {
httpGet: { path: '/health', port: transportPort as unknown as IntOrString },
initialDelaySeconds: 10,
periodSeconds: 10,
},
volumeMounts: [{ name: 'mcp-bin', mountPath: '/mcp-bin' }],
// G4: Independent resource limits for stdio-bridge sidecar
resources: {
requests: config.stdioBridgeResources.requests,
limits: config.stdioBridgeResources.limits,
},
},
]
: []),
],
volumes: (() => {
const v: k8s.V1Volume[] = []
if (isStdio) v.push({ name: 'mcp-bin', emptyDir: {} })
if (isRemote)
v.push({ name: 'nginx-conf', configMap: { name: `${server.name}-nginx-conf` } })
return v.length > 0 ? v : undefined
})(),
},
},
},
}
}
/**
* Build a Service manifest for an McpServer.
*/
private buildService(server: McpServerCRD): k8s.V1Service {
const transportPort = server.spec.transport.port || 3000
return {
apiVersion: 'v1',
kind: 'Service',
metadata: {
name: server.name,
namespace: server.namespace,
labels: {
app: server.name,
[MANAGED_BY_LABEL]: MANAGED_BY_VALUE,
[MCPSERVER_LABEL]: server.name,
},
// G5: ownerRef to McpServer CRD for garbage collection
...(server.uid && {
ownerReferences: [
{
apiVersion: 'clerum.io/v1alpha1',
kind: 'McpServer',
name: server.name,
uid: server.uid,
controller: true,
blockOwnerDeletion: true,
},
],
}),
},
spec: {
type: 'ClusterIP',
ports: [
{
port: transportPort,
targetPort: 'http' as IntOrString,
protocol: 'TCP',
name: 'http',
},
],
selector: { app: server.name },
},
}
}
// ─── Reconciliation Actions ─────────────────────────────────────────
/**
* Ensure a Deployment exists and is up to date.
*/
private async ensureDeployment(
server: McpServerCRD,
credentialsRevision = '',
isCurrent: () => boolean = () => true
): Promise<void> {
const deployment = this.buildDeployment(server, credentialsRevision)
await ensureResource({
mutationAllowed: isCurrent,
read: () =>
observeExistenceRead('Deployment', () =>
this.appsApi.readNamespacedDeployment({
name: server.name,
namespace: server.namespace,
})
),
create: async () => {
await observeCreate('Deployment', () =>
this.appsApi.createNamespacedDeployment({
namespace: server.namespace,
body: deployment,
})
)
hccLogger.info('Deployment created', {
scope: '[Reconciler]',
deployment: server.name,
namespace: server.namespace,
})
},
converge: read =>
replaceWithConflictRetry({
description: `Deployment "${server.name}"`,
logPrefix: '[Reconciler]',
body: deployment,
mergeExisting: preserveDeploymentAnnotations,
isUpToDate: deploymentMatchesDesired,
read,
replace: body =>
this.appsApi.replaceNamespacedDeployment({
name: server.name,
namespace: server.namespace,
body,
}),
mutationAllowed: isCurrent,
}),
onSkipped: () => createsTotal.inc({ kind: 'Deployment', outcome: 'skipped' }),
})
}
/**
* Ensure a Service exists and is up to date.
*/
private async ensureService(
server: McpServerCRD,
isCurrent: () => boolean = () => true
): Promise<void> {
const service = this.buildService(server)
await ensureResource({
mutationAllowed: isCurrent,
read: () =>
observeExistenceRead('Service', () =>
this.coreApi.readNamespacedService({
name: server.name,
namespace: server.namespace,
})
),
create: () =>
observeCreate('Service', () =>
this.coreApi.createNamespacedService({
namespace: server.namespace,
body: service,
})
),
converge: read =>
replaceWithConflictRetry({
description: `Service "${server.name}"`,
logPrefix: '[Reconciler]',
body: service,
mergeExisting: preserveServiceAssignedFields,
isUpToDate: serviceMatchesDesired,
read,
replace: body =>
this.coreApi.replaceNamespacedService({
name: server.name,
namespace: server.namespace,
body,
}),
mutationAllowed: isCurrent,
}),
onSkipped: () => createsTotal.inc({ kind: 'Service', outcome: 'skipped' }),
})
}
/**
* Delete HCC-owned Deployment, Service, and remote ConfigMap for an McpServer.
* Name matches are not enough: WRC-owned `managed:false` runtimes can share
* names with McpServer CRDs, so every delete verifies HCC ownership labels.
*/
private async deleteResources(
name: string,
namespace: string,
deleteAllowed?: () => Promise<boolean>
): Promise<void> {
const failures: unknown[] = []
for (const cleanup of [
() => this.deleteDeploymentIfHccOwned(name, namespace, deleteAllowed),
() => this.deleteConfigMapIfHccOwned(`${name}-nginx-conf`, namespace, name, deleteAllowed),
() => this.deleteServiceIfHccOwned(name, namespace, deleteAllowed),
]) {
try {
await cleanup()
} catch (error) {
failures.push(error)
}
}
throwCleanupFailures(
failures,
`Failed to delete runtime Kubernetes resources for McpServer "${name}"`
)
}
private isHccOwnedMcpResource(
resource: { metadata?: { labels?: Record<string, string> } },
serverName: string
): boolean {
const labels = resource.metadata?.labels ?? {}
return labels[MANAGED_BY_LABEL] === MANAGED_BY_VALUE && labels[MCPSERVER_LABEL] === serverName
}
private async deleteDeploymentIfHccOwned(
name: string,
namespace: string,
deleteAllowed?: () => Promise<boolean>
): Promise<DeploymentDeleteOutcome> {
let deployment: k8s.V1Deployment
try {
deployment = await this.appsApi.readNamespacedDeployment({ name, namespace })
} catch (error: unknown) {
if (getErrorCode(error) === 404) {
hccLogger.info('Deployment already gone', { name })
return 'gone'
}
hccLogger.error('Failed to read Deployment ownership', { name, err: error })
throw error
}
if (!this.isHccOwnedMcpResource(deployment, name)) {
hccLogger.warn('Skipping Deployment delete — not HCC-owned', { name })
return 'foreign'
}
if (deleteAllowed && !(await deleteAllowed())) return 'denied'
try {
await this.appsApi.deleteNamespacedDeployment({ name, namespace })
hccLogger.info('Deleted Deployment', { name })
return 'gone'
} catch (error: unknown) {
if (getErrorCode(error) === 404) {
hccLogger.info('Deployment already gone', { name })
return 'gone'
}
hccLogger.error('Failed to delete Deployment', { name, err: error })
throw error
}
}
/**
* Delete leftover HCC-owned ConfigMap and Service after the Deployment
* outcome is known. Does not write status.
*/
private async deleteHccOwnedRuntimeSiblings(
name: string,
namespace: string,
deleteAllowed?: () => Promise<boolean>
): Promise<void> {
const siblingFailures: unknown[] = []
for (const cleanup of [
() => this.deleteConfigMapIfHccOwned(`${name}-nginx-conf`, namespace, name, deleteAllowed),
() => this.deleteServiceIfHccOwned(name, namespace, deleteAllowed),
]) {
try {
await cleanup()
} catch (error) {
siblingFailures.push(error)
}
}
throwCleanupFailures(
siblingFailures,
`Failed to delete runtime Kubernetes resources for McpServer "${name}"`
)
}
/**
* Withdraw an HCC-owned runtime (#606): once the Deployment is confirmed
* gone, retire the poll, persist DeploymentReady=False/RuntimeNotDesired,
* then finish sibling deletes. Sibling failure does not roll back the
* retract. Retire before the retract write so an in-flight tick cannot
* republish True after the retract PATCH commits but before the epoch
* bumps. A foreign (WRC-owned) Deployment retires the window without
* writing RuntimeNotDesired.
*/
private async retractManagedRuntime(
server: McpServerCRD,
kind: RuntimeWithdrawKind,
isCurrent: () => boolean
): Promise<RetractManagedRuntimeResult> {
const name = server.name
const namespace = server.namespace
const deleteAllowed = async (): Promise<boolean> => isCurrent()
let cleanupComplete = true
const gatedDeleteAllowed = async (): Promise<boolean> => {
const allowed = await deleteAllowed()
if (!allowed) cleanupComplete = false
return allowed
}
let deploymentOutcome: DeploymentDeleteOutcome
try {
deploymentOutcome = await this.deleteDeploymentIfHccOwned(name, namespace, deleteAllowed)
} catch (error) {
const failures: unknown[] = [error]
for (const cleanup of [
() => this.deleteConfigMapIfHccOwned(`${name}-nginx-conf`, namespace, name, deleteAllowed),
() => this.deleteServiceIfHccOwned(name, namespace, deleteAllowed),
]) {
try {
await cleanup()
} catch (siblingError) {
failures.push(siblingError)
}
}
throwCleanupFailures(
failures,
`Failed to delete runtime Kubernetes resources for McpServer "${name}"`
)
throw new Error(`Failed to delete runtime Kubernetes resources for McpServer "${name}"`)
}
switch (deploymentOutcome) {
case 'denied':
return {
retracted: false,
cleanupComplete: false,
foreignRuntime: false,
runtimeGone: false,
}
case 'foreign':
this.retireReadinessWindow(name)
await this.deleteHccOwnedRuntimeSiblings(name, namespace, gatedDeleteAllowed)
return {
retracted: false,
cleanupComplete: cleanupComplete && isCurrent(),
foreignRuntime: true,
runtimeGone: false,
}
case 'gone':
break
default: {
const _exhaustive: never = deploymentOutcome
return _exhaustive
}
}
this.retireReadinessWindow(name)
const retractWrite = await this.writeStatusCondition(
server,
{
type: 'DeploymentReady',
status: 'False',
reason: 'RuntimeNotDesired',
message: runtimeNotDesiredMessage(kind),
},
isCurrent
)
const epoch = this.readinessRetirements.get(name)
if (retractWrite === 'declined') {
hccLogger.warn('Retract write declined', { name, kind, epoch, outcome: retractWrite })
} else {
hccLogger.info('Retracted managed runtime', { name, kind, epoch, outcome: retractWrite })
}
await this.deleteHccOwnedRuntimeSiblings(name, namespace, gatedDeleteAllowed)
if (retractWrite === 'declined') {
return {
retracted: false,
cleanupComplete: false,
foreignRuntime: false,
runtimeGone: true,
}
}
return {
retracted: true,
cleanupComplete: cleanupComplete && isCurrent(),
foreignRuntime: false,
runtimeGone: true,
}
}
private async deleteConfigMapIfHccOwned(
configMapName: string,
namespace: string,
serverName: string,
deleteAllowed?: () => Promise<boolean>
): Promise<void> {
let configMap: k8s.V1ConfigMap
try {
configMap = await this.coreApi.readNamespacedConfigMap({ name: configMapName, namespace })
} catch (error: unknown) {
if (getErrorCode(error) === 404) {
console.log(`[Reconciler] ConfigMap "${configMapName}" already gone`)
} else {
console.error(`[Reconciler] Failed to read ConfigMap "${configMapName}" ownership:`, error)
throw error
}
return
}
if (!this.isHccOwnedMcpResource(configMap, serverName)) {
console.warn(`[Reconciler] Skipping ConfigMap "${configMapName}" delete — not HCC-owned`)
return
}
if (deleteAllowed && !(await deleteAllowed())) return
try {
await this.coreApi.deleteNamespacedConfigMap({ name: configMapName, namespace })
console.log(`[Reconciler] Deleted nginx ConfigMap "${configMapName}"`)
} catch (error: unknown) {
if (getErrorCode(error) === 404) {
console.log(`[Reconciler] ConfigMap "${configMapName}" already gone`)
} else {
console.error(`[Reconciler] Failed to delete ConfigMap "${configMapName}":`, error)
throw error
}
}
}
private async deleteServiceIfHccOwned(
name: string,
namespace: string,
deleteAllowed?: () => Promise<boolean>
): Promise<void> {
let service: k8s.V1Service
try {
service = await this.coreApi.readNamespacedService({ name, namespace })
} catch (error: unknown) {
if (getErrorCode(error) === 404) {
console.log(`[Reconciler] Service "${name}" already gone`)
} else {
console.error(`[Reconciler] Failed to read Service "${name}" ownership:`, error)
throw error
}
return
}
if (!this.isHccOwnedMcpResource(service, name)) {
console.warn(`[Reconciler] Skipping Service "${name}" delete — not HCC-owned`)
return
}
if (deleteAllowed && !(await deleteAllowed())) return
try {
await this.coreApi.deleteNamespacedService({ name, namespace })
console.log(`[Reconciler] Deleted Service "${name}"`)
} catch (error: unknown) {
if (getErrorCode(error) === 404) {
console.log(`[Reconciler] Service "${name}" already gone`)
} else {
console.error(`[Reconciler] Failed to delete Service "${name}":`, error)
throw error
}
}
}
private async deleteRuntimeResources(
name: string,
namespace: string,
deleteAllowed?: () => Promise<boolean>
): Promise<boolean> {
// A guarded no-op is incomplete even if authority returns before the
// outer call settles. Ownership-based skips remain legitimate outcomes.
let complete = true
await this.deleteResources(
name,
namespace,
deleteAllowed
? async () => {
const allowed = await deleteAllowed()
if (!allowed) complete = false
return allowed
}
: undefined
)
return complete
}
// ─── CRD Status & Annotation Patching ──────────────────────────────
/**
* G2: Set `clerum.io/network-ready: "true"` annotation on McpServer CRD.
* Completes the pre-deploy handshake so WRC knows NetworkPolicies are applied.
*/
private async setNetworkReadyAnnotation(
server: McpServerCRD,
isCurrent: () => boolean = () => true
): Promise<void> {
// Use merge-patch (plain object body) instead of JSON Patch (array body).
// JSON Patch op:"add" on /metadata/annotations/<key> requires /metadata/annotations
// to already exist — it fails with 422 when the CRD was created without annotations.
// Merge-patch creates the map if absent. Errors are re-thrown so the caller (reconcile)
// knows the handshake was not completed — WRC will time out and fail rather than proceed
// silently without NetworkPolicies in place.
if (!isCurrent()) return
await this.customApi.patchNamespacedCustomObject(
{
group: 'clerum.io',
version: 'v1alpha1',
namespace: server.namespace,
plural: 'mcpservers',
name: server.name,
body: {
metadata: {
annotations: {
[NETWORK_READY_ANNOTATION]: 'true',
// Issue #408: stamp the acked generation so WRC's waitForNetworkReady
// can reject a stale ack from a previous generation.
...(typeof server.generation === 'number'
? { [NETWORK_READY_GENERATION_ANNOTATION]: String(server.generation) }
: {}),
},
},
},
},
{
middleware: [k8s.setHeaderMiddleware('Content-Type', 'application/merge-patch+json')],
}
)
}
/**
* G11: Update DeploymentReady on the McpServer status subresource.
* The condition describes Deployment rollout. WRC does not watch this type;
* it reads the `clerum.io/network-ready` annotation.
*/
private async updateStatusConditions(
server: McpServerCRD,
deploymentReady: boolean,
/**
* Rollout numbers from readDeploymentRollout, appended to the failure
* message so a stuck rollout reports WHAT is stuck (issue #223). Callers
* that only know the verdict omit it.
*/
rolloutDetail?: string,
isCurrent: () => boolean = () => true,
onDecline: StatusWriteDeclineArg = () => 'record-obligation'
): Promise<void> {
try {
const deploymentWrote = await this.writeStatusCondition(
server,
{
type: 'DeploymentReady',
status: deploymentReady ? 'True' : 'False',
reason: deploymentReady
? 'ReplicasAvailable'
: rolloutDetail
? 'RolloutIncomplete'
: 'WaitingForReplicas',
message: rolloutDetail
? `Rollout did not converge — ${rolloutDetail}`
: deploymentReady
? 'Deployment has ready replicas'
: 'Waiting for pods to become ready',
},
isCurrent,
onDecline
)
if (deploymentWrote === 'wrote') {
hccLogger.info('Updated status conditions', {
name: server.name,
deploymentReady,
})
}
} catch (error) {
// Status subresource may not exist yet — log but don't fail reconciliation
hccLogger.warn('Failed to update status conditions', { name: server.name, err: error })
}
}
/**
* Write (or merge) a single status condition on the McpServer CRD.
*
* Merge semantics:
* - Condition with same `type` and unchanged `status` → keep existing
* `lastTransitionTime`, refresh `reason` / `message`.
* - `DeploymentReady` with the same `status` but a new `reason` or
* `message` → bump `lastTransitionTime` so recovery after
* `RuntimeNotDesired` is visible to #223 consumers.
* - Condition with same `type` but different `status` → bump
* `lastTransitionTime` to now.
* - New type → append with fresh `lastTransitionTime`.
* - Any write also strips retired types (NetworkReady). If a retired
* type is still present, the patch is forced even when the requested
* condition is otherwise unchanged.
*
* Side-effect: updates the `clerum_hcc_mcpserver_missing_secret` gauge for
* `SecretResolved` conditions (1 when False, 0 when True).
*
* `onDecline` is a factory evaluated at decline time. The default records
* an incomplete-publication obligation. Poll ticks pass a factory that is
* `'silent'` only while `retiredNow()` is true.
*
* Returns `'wrote'` after a successful patch, `'unchanged'` when the
* requested condition already matches and no retired type remains, or
* `'declined'` when the fence or a write error stops the update.
*
* Failure modes:
* - 404 (CRD deleted mid-reconcile) → log + `decline()`.
* - 409/422 (optimistic conflict) → re-read, re-merge, bounded retry.
* - Other errors or exhausted retries → log + `decline()`.
*/
async writeStatusCondition(
server: McpServerCRD,
condition: Omit<McpServerCondition, 'lastTransitionTime'>,
isCurrent: () => boolean = () => true,
onDecline: StatusWriteDeclineArg = () => 'record-obligation'
): Promise<StatusWriteOutcome> {
const decline = (): StatusWriteOutcome =>
onDecline() === 'silent' ? 'declined' : this.recordIncompletePublication(server)
if (!isCurrent()) return decline()
const now = new Date().toISOString()
// Update the missing-secret gauge for SecretResolved conditions.
// Do this before any no-op short-circuit so the gauge stays current
// across reconciles even when we skip the API patch.
if (condition.type === 'SecretResolved' && server.spec.envSecret) {
const labels = {
namespace: server.namespace,
name: server.name,
secret_name: server.spec.envSecret.name,
}
mcpserverMissingSecret.set(labels, condition.status === 'False' ? 1 : 0)
}
// Optimistic-concurrency loop (M2). The patch REPLACES the whole
// /status/conditions array with a value built from a snapshot read just
// before. Without a version guard, a concurrent writer — the readiness poll
// and a reconcile can both write status for the same server, and the poll
// timer is NOT serialized through `inFlight` — would be silently clobbered
// by whichever patch lands second, dropping the other's condition. A `test`
// op on metadata.resourceVersion makes the API server reject the patch (422)
// if anything changed since our read; we then re-read and retry, so no
// writer's condition is lost.
for (let attempt = 1; attempt <= STATUS_CONDITION_WRITE_MAX_ATTEMPTS; attempt += 1) {
if (!isCurrent()) return decline()
// Re-read on every optimistic-conflict retry. Merging against the
// latest resourceVersion preserves conditions written by peer
// controllers between our read and patch.
let existingConditions: McpServerCondition[] = []
let hasStatusObject = false
let resourceVersion: string | undefined
try {
const current = (await this.customApi.getNamespacedCustomObjectStatus({
group: 'clerum.io',
version: 'v1alpha1',
namespace: server.namespace,
plural: 'mcpservers',
name: server.name,
})) as {
metadata?: { resourceVersion?: string }
status?: { conditions?: McpServerCondition[] }
}
hasStatusObject = typeof current.status === 'object' && current.status !== null
existingConditions = current.status?.conditions ?? []
resourceVersion = current.metadata?.resourceVersion
} catch (error) {
if (getErrorCode(error) === 404) {
console.warn(
`[Reconciler] McpServer "${server.name}" deleted mid-reconcile — skipping status update`
)
return decline()
}
console.warn(
`[Reconciler] Failed to read status for "${server.name}" — skipping status update for type "${condition.type}" to avoid clobbering other status fields:`,
error
)
return decline()
}
if (!isCurrent()) return decline()
const prior = existingConditions.find(c => c.type === condition.type)
const observedGeneration = condition.observedGeneration ?? server.generation
const unchanged =
prior !== undefined &&
prior.status === condition.status &&
prior.reason === condition.reason &&
prior.message === condition.message &&
prior.observedGeneration === observedGeneration &&
!existingConditions.some(existing => isRetiredStatusConditionType(existing.type))
// Skip the patch when the condition is identical to what's already on
// the CRD. Without this, every reconcile bumps the McpServer's
// resourceVersion, fires another watch event, triggers another
// reconcile — a tight self-loop that also amplifies NetworkPolicy
// optimistic-lock contention downstream.
if (unchanged) {
return 'unchanged'
}
const refreshDeploymentReadyTime =
condition.type === 'DeploymentReady' &&
prior !== undefined &&
(prior.reason !== condition.reason || prior.message !== condition.message)
const lastTransitionTime =
prior && prior.status === condition.status && !refreshDeploymentReadyTime
? prior.lastTransitionTime
: now
const merged: McpServerCondition = {
type: condition.type,
status: condition.status,
reason: condition.reason,
message: condition.message,
lastTransitionTime,
...(observedGeneration !== undefined && { observedGeneration }),
}
const nextConditions = [
...existingConditions.filter(
existing =>
existing.type !== condition.type && !isRetiredStatusConditionType(existing.type)
),
merged,
]
const writePreconditions: Array<{
op: 'test'
path: string
value: string | number
}> = []
if (server.uid !== undefined) {
writePreconditions.push({
op: 'test',
path: '/metadata/uid',
value: server.uid,
})
}
if (server.generation !== undefined) {
writePreconditions.push({
op: 'test',
path: '/metadata/generation',
value: server.generation,
})
}
if (resourceVersion !== undefined) {
writePreconditions.push({
op: 'test',
path: '/metadata/resourceVersion',
value: resourceVersion,
})
}
const statusPatch = [
...writePreconditions,
...(hasStatusObject
? [{ op: 'add', path: '/status/conditions', value: nextConditions }]
: [{ op: 'add', path: '/status', value: { conditions: nextConditions } }]),
]
try {
// The generated client prefers JSON Patch for this endpoint. Send an
// actual patch document so status writes keep working across client
// upgrades instead of relying on merge-patch object bodies.
if (!isCurrent()) return decline()
await this.customApi.patchNamespacedCustomObjectStatus({
group: 'clerum.io',
version: 'v1alpha1',
namespace: server.namespace,
plural: 'mcpservers',
name: server.name,
body: statusPatch,
})
return 'wrote'
} catch (error) {
const errorCode = getErrorCode(error)
if (errorCode === 404) {
console.warn(
`[Reconciler] McpServer "${server.name}" deleted mid-reconcile — status patch skipped`
)
return decline()
}
// 409 (Conflict) or 422 (a `test` op failed) mean a concurrent writer
// won the race. Re-read and retry rather than lose our condition. Any
// other error, or the last attempt, falls through to the best-effort
// log-and-swallow below.
if (
(errorCode === 409 || errorCode === 422) &&
attempt < STATUS_CONDITION_WRITE_MAX_ATTEMPTS
) {
continue
}
console.warn(
`[Reconciler] Failed to write status condition ${condition.type}=${condition.status} on "${server.name}" (attempt ${attempt}/${STATUS_CONDITION_WRITE_MAX_ATTEMPTS}):`,
error
)
return decline()
}
}
return decline()
}
private recordIncompletePublication(server: McpServerCRD): 'declined' {
// A declined or failed write replaces the obligation so an enclosing
// reconcile cannot clear that failure on a later successful sibling write.
this.pendingReconciliations.set(server.name, { namespace: server.namespace })
return 'declined'
}
// ─── Public API ─────────────────────────────────────────────────────
/**
* Reconcile a created or modified McpServer.
*
* Concurrent calls for the same server name are serialized through
* `inFlight` so we never run two reconciles in parallel for one server.
* A prior failure does not block the next call; the chain absorbs it so
* the next reconcile still runs.
*/
async reconcile(server: McpServerCRD, options: McpServerMutationOptions = {}): Promise<void> {
const capturedAuthority = this.inventoryAuthority()
const callerIsCurrent = options.isCurrent ?? (() => true)
const isCurrent = () => {
if (!callerIsCurrent()) return false
if (!this.inventoryAuthorityConfigured) return this.currentWhenUnconfigured()
if (!this.authorityMatches(capturedAuthority)) return false
const current = this.resolveCurrentServer?.(server.name)
return current !== undefined && sameMcpServerDesiredRevision(server, current)
}
const key = server.name
const obligation = { namespace: server.namespace }
this.pendingReconciliations.set(key, obligation)
const prev = this.inFlight.get(key) ?? Promise.resolve()
const next = prev
.catch(() => undefined)
.then(async () => {
if (!isCurrent()) return
const completed = await this.performReconcile(server, isCurrent)
if (completed && this.pendingReconciliations.get(key) === obligation) {
this.pendingReconciliations.delete(key)
}
})
this.inFlight.set(key, next)
try {
await next
} finally {
if (this.inFlight.get(key) === next) {
this.inFlight.delete(key)
}
}
}
private async performReconcile(server: McpServerCRD, isCurrent: () => boolean): Promise<boolean> {
if (!isCurrent()) return false
this.beginStatusTracking(server)
// G7: Enforce managed field immutability — once set, cannot change
if (!isCurrent()) return false
const currentManaged = server.spec.managed ?? true
const previousManaged = this.managedSnapshot.get(server.name)
if (previousManaged !== undefined && previousManaged !== currentManaged) {
console.error(
`[Reconciler] McpServer "${server.name}": managed field changed from ${previousManaged} to ${currentManaged}. ` +
`This is not allowed — delete and recreate the McpServer to change ownership.`
)
if (!isCurrent()) return false
this.setStatus(server.name, {
deployed: false,
ready: false,
message: `managed field is immutable (was ${previousManaged}, attempted ${currentManaged})`,
})
return isCurrent()
}
if (!isCurrent()) return false
this.managedSnapshot.set(server.name, currentManaged)
// If disabled, respect ownership before any cleanup. managed:false means
// WRC owns runtime resources; HCC only marks discovery status disabled.
if (server.spec.enabled === false) {
let cleanupComplete = true
let retract: RetractManagedRuntimeResult | undefined
if (currentManaged) {
console.log(
`[Reconciler] McpServer "${server.name}" is disabled — removing HCC-owned resources`
)
retract = await this.retractManagedRuntime(server, 'disabled', isCurrent)
if (shouldStopAfterIncompleteRetract(retract, isCurrent)) return false
cleanupComplete = retract.cleanupComplete
} else {
console.log(
`[Reconciler] McpServer "${server.name}" is disabled but WRC-owned; skipping runtime cleanup`
)
}
if (!isCurrent()) return false
this.setStatus(server.name, { deployed: false, ready: false, message: 'Disabled' })
await this.writeStatusCondition(
server,
{
type: 'Ready',
status: 'False',
reason: 'Disabled',
message: 'McpServer is disabled',
},
isCurrent
)
if (retract && !retract.retracted && !retract.foreignRuntime) return false
return cleanupComplete && isCurrent()
}
if (!currentManaged) {
return this.reconcileWrcOwnedServer(server, isCurrent)
}
// Validate secret before creating/updating
const secretResult = await this.validateSecret(server)
if (!isCurrent()) return false
if (!secretResult.ok) {
console.error(
`[Reconciler] Skipping deployment of "${server.name}" — secret validation failed`
)
const withdrawRuntime = shouldFailClosedForSecretFailure(secretResult.reason)
let cleanupComplete = true
let retract: RetractManagedRuntimeResult | undefined
if (withdrawRuntime) {
retract = await this.retractManagedRuntime(server, 'fail-closed', isCurrent)
if (shouldStopAfterIncompleteRetract(retract, isCurrent)) return false
cleanupComplete = retract.cleanupComplete
} else {
console.warn(
`[Reconciler] Preserving existing runtime for "${server.name}" after transient Secret read failure`
)
}
if (!isCurrent()) return false
this.setStatus(server.name, {
deployed: false,
ready: false,
message: secretResult.message,
})
await this.writeStatusCondition(
server,
{
type: 'SecretResolved',
status: 'False',
reason: secretResult.reason,
message: secretResult.message,
},
isCurrent
)
await this.writeStatusCondition(
server,
{
type: 'Ready',
status: 'False',
reason: 'SecretValidationFailed',
message: secretResult.message,
},
isCurrent
)
if (retract && !retract.retracted && !retract.foreignRuntime) return false
return withdrawRuntime && cleanupComplete && isCurrent()
}
// Ensure resources exist and are up to date
try {
// Remote egress proxy: create nginx ConfigMap before Deployment
if (this.isRemote(server)) {
// The CRD image remains desired-state input owned by its author. Remote
// workloads use the platform-owned proxy image selected in
// buildDeployment(); rewriting spec.image here would bump generation
// and retire this reconcile before it can create the runtime resources.
await this.ensureConfigMap(server, isCurrent)
} else {
// Phase 2.3: plugin image-host allowlist. Audit mode (default) logs a
// would-be denial and still builds; enforce mode blocks the workload.
const decision = classifyPluginImage(server.spec.image, {
allowedPrefixes: config.allowedPluginImagePrefixes,
rejectLatest: false,
})
if (!decision.ok) {
const message = `Image "${server.spec.image}" is not permitted by the plugin image allowlist (${decision.reason})`
if (config.enforcePluginImageAllowlist) {
console.warn(`[Reconciler] ${message} — blocking "${server.name}" (enforce mode)`)
await this.writeStatusCondition(
server,
{
type: 'Ready',
status: 'False',
reason: 'ImageNotAllowed',
message,
},
isCurrent
)
if (!isCurrent()) return false
this.setStatus(server.name, { deployed: false, ready: false, message })
return isCurrent()
}
console.warn(
`[Reconciler] ${message} for "${server.name}" — audit mode, allowing (set CONTEXT_MAPPER_ENFORCE_IMAGE_ALLOWLIST=true to block)`
)
}
}
if (!isCurrent()) return false
await this.ensureService(server, isCurrent)
if (!isCurrent()) return false
await this.ensureDeployment(server, secretResult.revision, isCurrent)
} catch (err) {
if (!isCurrent()) return false
this.setStatus(server.name, {
deployed: false,
ready: false,
message: `Resource sync failed: ${err instanceof Error ? err.message : String(err)}`,
})
return false
}
// G2: Complete pre-deploy handshake — set network-ready annotation
// after Deployment + Service are created, signaling WRC that
// NetworkPolicies are in place and workload can proceed.
if (
server.annotations?.[PRE_DEPLOY_ANNOTATION] === 'true' &&
(server.annotations?.[NETWORK_READY_ANNOTATION] !== 'true' ||
// Issue #408: re-ack when the stamped generation is stale (spec changed since
// the last ack). A non-numeric generation keeps the legacy behavior.
(typeof server.generation === 'number' &&
server.annotations?.[NETWORK_READY_GENERATION_ANNOTATION] !== String(server.generation)))
) {
try {
await this.setNetworkReadyAnnotation(server, isCurrent)
} catch (err) {
if (!isCurrent()) return false
this.setStatus(server.name, {
deployed: false,
ready: false,
message: `Pre-deploy handshake failed: ${err instanceof Error ? err.message : String(err)}`,
})
return false
}
}
// Check readiness after reconciliation
const rollout = await this.readDeploymentRollout(server.name, server.namespace)
if (!isCurrent()) return false
const ready = rollout.ready
this.setStatus(server.name, {
deployed: true,
ready,
message: ready ? 'Running' : 'Deployment created, waiting for pods to become ready',
})
// G11: Update status conditions on McpServer CRD. If the readiness poll
// already exhausted its budget for THIS Deployment generation, keep the
// terminal RolloutIncomplete verdict sticky: the terminal status write
// itself fires a MODIFIED watch event and thus this very code path, and
// writing a bare "waiting" condition here would downgrade the verdict
// milliseconds after it was published — the operator (and the E2
// scenario) would never observe the rollout failure. A NEW rollout
// (different generation) drops the stickiness and reports fresh.
const pollState = this.readinessPolls.get(server.name)
const exhaustedDetail =
!ready &&
pollState !== undefined &&
pollState.exhausted &&
rollout.generation !== null &&
pollState.generation === rollout.generation
? pollState.detail
: undefined
await this.updateStatusConditions(server, ready, exhaustedDetail, isCurrent)
// PR-B B1: record SecretResolved=True + Ready=True after a successful
// reconcile. This is additive to updateStatusConditions (DeploymentReady);
// the merge logic in writeStatusCondition preserves the existing
// conditions by type.
if (server.spec.envSecret) {
await this.writeStatusCondition(
server,
{
type: 'SecretResolved',
status: 'True',
reason: 'SecretFound',
message: 'Secret resolved and validated',
},
isCurrent
)
}
await this.writeStatusCondition(
server,
{
type: 'Ready',
status: ready ? 'True' : 'Unknown',
reason: ready ? 'ReconcileSuccess' : 'WaitingForReplicas',
message: ready
? 'Deployment created'
: 'Deployment created, waiting for pods to become ready',
},
isCurrent
)
// If not ready yet, poll until it becomes ready. With generation-aware
// readiness this is the normal path right after any Deployment write —
// including a credential rotation — so the poll owns the terminal verdict
// on the CRD.
//
// Post-terminal re-observation (intentional, self-terminating): after a
// terminal RolloutIncomplete, the status write that recorded it fires a
// MODIFIED watch → performReconcile → here again with !ready, arming ONE more
// poll window. For a genuinely stuck deployment the terminal condition re-write
// is byte-identical (the detail is a pure function of the replica tuple, no
// timestamps), so writeStatusCondition's `unchanged` short-circuit skips the
// patch → no watch event → no further re-arm. The only thing that re-arms again
// is a deployment whose replica tuple perpetually OSCILLATES, and that is
// bounded to one reconcile per poll window (~120s) — not a tight loop, and (per
// pollReadiness's atomic check-then-set) not a timer leak.
if (!ready && isCurrent()) {
this.pollReadiness(server, 5000, 24, isCurrent)
}
return isCurrent()
}
private async reconcileWrcOwnedServer(
server: McpServerCRD,
isCurrent: () => boolean
): Promise<boolean> {
if (!isCurrent()) return false
console.log(
`[Reconciler] McpServer "${server.name}" is WRC-owned (managed:false); ` +
'skipping HCC runtime creation/deletion and updating discovery status.'
)
const secretResult = await this.validateSecret(server)
if (!isCurrent()) return false
if (!secretResult.ok) {
this.setStatus(server.name, {
deployed: true,
ready: false,
message: secretResult.message,
})
if (server.spec.envSecret) {
await this.writeStatusCondition(
server,
{
type: 'SecretResolved',
status: 'False',
reason: secretResult.reason,
message: secretResult.message,
},
isCurrent
)
}
await this.writeStatusCondition(
server,
{
type: 'Ready',
status: 'False',
reason: 'SecretValidationFailed',
message: secretResult.message,
},
isCurrent
)
return shouldFailClosedForSecretFailure(secretResult.reason) && isCurrent()
}
if (!isCurrent()) return false
this.setStatus(server.name, {
deployed: true,
ready: true,
message: 'WRC-owned runtime registered',
})
if (server.spec.envSecret) {
await this.writeStatusCondition(
server,
{
type: 'SecretResolved',
status: 'True',
reason: 'SecretFound',
message: 'Secret resolved and validated',
},
isCurrent
)
}
await this.writeStatusCondition(
server,
{
type: 'Ready',
status: 'True',
reason: WRC_OWNED_RUNTIME_READY_REASON,
message: 'WRC-owned runtime registered for HCC discovery',
},
isCurrent
)
return isCurrent()
}
/**
* Reconcile a deleted McpServer.
*/
async reconcileDelete(name: string, namespace: string): Promise<void> {
console.log(`[Reconciler] McpServer "${name}" deleted — cleaning up resources`)
const capturedAuthority = this.inventoryAuthority()
const obligation = { namespace }
this.pendingReconciliations.set(name, obligation)
const complete = await this.deleteRuntimeResources(name, namespace, () =>
this.orphanDeleteAllowed(name, namespace, capturedAuthority)
)
if (complete && this.pendingReconciliations.get(name) === obligation) {
this.pendingReconciliations.delete(name)
this.clearStatus(name)
}
}
private authorityMatches(captured: McpServerInventoryAuthoritySnapshot): boolean {
const current = this.inventoryAuthority()
return captured.known && current.known && captured.generation === current.generation
}
/**
* A cache miss is not proof of absence: the API watch may have accepted an
* ADDED event that has not reached this process yet. Confirm a 404 from the
* API and then recheck both cache membership and LIST→WATCH authority
* immediately before every destructive delete.
*/
private async orphanDeleteAllowed(
name: string,
namespace: string,
capturedAuthority: McpServerInventoryAuthoritySnapshot
): Promise<boolean> {
return confirmAuthoritativeMcpServerAbsence({
inventoryAuthoritative: () => this.authorityMatches(capturedAuthority),
resolveCurrent: () => this.resolveCurrentServer?.(name),
readCurrent: () =>
this.customApi.getNamespacedCustomObject({
group: 'clerum.io',
version: 'v1alpha1',
namespace,
plural: 'mcpservers',
name,
}),
})
}
/**
* Full reconciliation pass — sync desired state with actual state.
* Called on startup to handle CRDs that were created while the skill-mapper was down.
*/
async fullReconcile(
desiredServers: McpServerCRD[],
options: McpServerFullReconcileOptions = {}
): Promise<void> {
console.log(
`[Reconciler] Running full reconciliation for ${desiredServers.length} McpServer(s)`
)
const capturedAuthority = this.inventoryAuthority()
// Get all deployments managed by skill-mapper
const existingDeployments = await this.listManagedDeployments()
const desiredNames = new Set(
desiredServers
.filter(s => s.spec.enabled !== false && (s.spec.managed ?? true))
.map(s => s.name)
)
const runEffect =
options.runEffect ?? ((_serverName: string, work: () => Promise<void>) => work())
const maxConcurrency = options.maxConcurrency ?? DEFAULT_FULL_RECONCILE_MAX_CONCURRENCY
const inScope = (name: string) => options.scope === undefined || options.scope.has(name)
// Create or update desired servers independently. runEffect owns each
// per-server lane, so a slow fleet member cannot become a process-wide
// startup barrier. The finite worker pool admits no more than the configured
// number of effects at once, while still waiting for every desired result
// before reporting aggregated failures or moving to orphan cleanup.
const scopedServers = desiredServers.filter(server => inScope(server.name))
const desiredResults = await mapSettledWithConcurrency(scopedServers, maxConcurrency, server =>
runEffect(server.name, async () => {
const current = this.resolveCurrentServer ? this.resolveCurrentServer(server.name) : server
// Cache mutation happens synchronously before its keyed watch effect is
// queued. If this startup item was superseded, skip it and let the
// newer queued event perform external-egress-before-runtime using its
// own spec; reconciling the newer runtime here could otherwise race
// ahead of that safety policy.
if (!current || !sameMcpServerDesiredRevision(server, current)) return
const isCurrent = () => {
if (!this.inventoryAuthorityConfigured) {
if (!this.currentWhenUnconfigured()) return false
} else if (!this.authorityMatches(capturedAuthority)) {
return false
}
const latest = this.resolveCurrentServer?.(current.name)
return latest
? sameMcpServerDesiredRevision(current, latest)
: this.resolveCurrentServer === undefined && this.currentWhenUnconfigured()
}
await this.reconcile(current, { isCurrent })
})
)
throwCleanupFailures(
desiredResults
.filter((result): result is PromiseRejectedResult => result.status === 'rejected')
.map(result => result.reason),
'Failed to reconcile desired McpServers'
)
// Delete orphaned deployments (managed by us but no longer desired)
const cleanupTargets = new Map(
existingDeployments.map(deployment => [
deployment.metadata?.name || '',
deployment.metadata?.namespace || config.namespace,
])
)
const presentNames = new Set(desiredServers.map(server => server.name))
for (const [name, pending] of this.pendingReconciliations) {
if (!presentNames.has(name)) cleanupTargets.set(name, pending.namespace)
}
for (const [name, namespace] of cleanupTargets) {
if (!desiredNames.has(name) && inScope(name)) {
if (!presentNames.has(name)) this.pendingReconciliations.set(name, { namespace })
await runEffect(name, async () => {
if (!(await this.orphanDeleteAllowed(name, namespace, capturedAuthority))) {
console.warn(
`[Reconciler] Deferring orphan cleanup for "${name}": McpServer exists or inventory authority is unknown or changed`
)
return
}
console.log(`[Reconciler] Removing orphaned resources for "${name}"`)
await this.reconcileDelete(name, namespace)
})
}
}
console.log('[Reconciler] Full reconciliation complete')
}
/**
* List all Deployments managed by skill-mapper (by label selector).
*/
private async listManagedDeployments(): Promise<k8s.V1Deployment[]> {
try {
const response = await this.appsApi.listNamespacedDeployment({
namespace: config.namespace,
labelSelector: `${MANAGED_BY_LABEL}=${MANAGED_BY_VALUE}`,
})
return response.items || []
} catch (error) {
console.error('[Reconciler] Failed to list managed deployments:', error)
throw error
}
}
}