Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript
114 KB
Raw Blame
/**
 * McpServer Reconciler - manages Kubernetes Deployments and Services
 * based on McpServer CRD state.
 *
 * Responsibilities:
 * - Create Deployment + Service when a new McpServer CRD appears
 * - Update Deployment when an McpServer CRD is modified
 * - Delete Deployment + Service when an McpServer CRD is removed
 * - Validate that referenced secrets exist before creating deployments
 * - Full reconciliation on startup (sync desired vs actual state)
 */
import * as k8s from '@kubernetes/client-node'
import { IntOrString } from '@kubernetes/client-node/dist/types.js'
import { createHash } from 'crypto'
import { classifyPluginImage } from '@clerum/image-policy'
import {
  NETWORK_READY_ANNOTATION,
  NETWORK_READY_GENERATION_ANNOTATION,
  PRE_DEPLOY_ANNOTATION,
} from '@clerum/network-policy-core'
import { isWorkflowRecipeDefaultAllowedCapability } from '@clerum/workflow-recipe-capability-policy'
import { config } from './config'
import { MANAGED_BY_LABEL, MANAGED_BY_VALUE, MCPSERVER_LABEL } from './constants'
import { hccLogger } from './logger'
import {
  confirmAuthoritativeMcpServerAbsence,
  sameMcpServerDesiredRevision,
  shouldFailClosedForSecretFailure,
} from './mcpServerSafety'
import { createsTotal, mcpserverMissingSecret } from './metrics'
import { McpServerCRD, McpServerStatus } from './types'
import {
  canonicalStringify,
  configMapMatchesDesired,
  deploymentMatchesDesired,
  ensureResource,
  getErrorCode,
  observeCreate,
  observeExistenceRead,
  preserveDeploymentAnnotations,
  preserveObjectAnnotations,
  preserveServiceAssignedFields,
  replaceWithConflictRetry,
  serviceMatchesDesired,
} from './utils'

/**
 * Result of validating an envSecret reference. When `ok` is false, `reason`
 * classifies the failure so the caller can surface it via status conditions.
 */
export type SecretValidationResult =
  | {
      ok: true
      /**
       * sha256 of the validated Secret's content (issue #223). Empty when the
       * McpServer has no envSecret. Travels into the pod template as the
       * credentials-revision annotation, which is what makes a credential
       * rotation roll the pod: env vars come from `secretKeyRef` and Kubernetes
       * only injects those when the container starts.
       *
       * It is derived from the very Secret this validation read, so the digest
       * can never describe a different revision than the one that was checked.
       */
      revision: string
    }
  | {
      ok: false
      reason: 'SecretNotFound' | 'SecretMissingKey' | 'SecretAccessDenied' | 'ReadError'
      message: string
    }

/**
 * Condition written to `status.conditions[]` on an McpServer CRD.
 * `status` is the K8s-standard string tri-state, NOT a boolean.
 */
export interface McpServerCondition {
  type: string
  status: 'True' | 'False' | 'Unknown'
  reason: string
  message: string
  lastTransitionTime: string
  observedGeneration?: number
}

type McpServerReconcilerDeps = {
  appsApi?: k8s.AppsV1Api
  coreApi?: k8s.CoreV1Api
  customApi?: k8s.CustomObjectsApi
  /**
   * TEST-ONLY escape hatch. When the inventory-authority fence is not wired
   * (setInventoryAuthority / setResolveCurrentServer never called), mutation
   * fences fail CLOSED by default. Unit tests that exercise the reconciler
   * without a watcher opt in to the legacy assume-current behavior here.
   * Production construction (k8sClient) must never set this.
   */
  assumeInventoryAuthorityWhenUnconfigured?: boolean
}

export type McpServerInventoryAuthoritySnapshot = {
  known: boolean
  generation: number
}

export type McpServerFullReconcileOptions = {
  runEffect?: (serverName: string, work: () => Promise<void>) => Promise<void>
  maxConcurrency?: number
  // Restrict create/update and orphan cleanup to these names. desiredServers
  // must still be the whole inventory so out-of-scope servers are never
  // mistaken for orphans.
  scope?: ReadonlySet<string>
}

export type McpServerMutationOptions = {
  isCurrent?: () => boolean
}

// G2: Pre-deploy handshake constants — imported from @clerum/network-policy-core
// (single source of truth shared with WRC; a divergence is a compile error rather
// than a silent 30s fail-open timeout). clerum.io/network-ready-observed-generation
// (Issue #408) is the generation HCC stamps when acking, so WRC can reject a stale ack.

// Issue #223: sha256 of the envSecret content, carried on the POD TEMPLATE (not
// the Deployment object) so that changing it changes the pod template hash and
// Kubernetes performs a rolling restart on its own. Same mechanism the per-Host
// channel-readers use in hostReconciler.ts.
const CREDENTIALS_REVISION_ANNOTATION = 'clerum.io/credentials-revision'

// Codex P0 fix (PR #101): sanitization constants for remote.authHeaders.
// Only standard HTTP header token chars — letters, digits, and '-'. Rejecting
// anything else prevents smuggling headers like `X: foo\r\nAuthorization`.
const HEADER_NAME_RE = /^[A-Za-z0-9-]+$/
// Bound the value length so a pathological CRD cannot blow up the ConfigMap.
const MAX_AUTH_HEADER_VALUE_LENGTH = 2048
const WRC_OWNED_RUNTIME_READY_REASON = 'WrcOwnedRuntimeRegistered'
// Keep fleet admission aligned with the external-egress convergence lane:
// enough parallelism for independent servers without allocating one pending
// effect promise per fleet member.
const DEFAULT_FULL_RECONCILE_MAX_CONCURRENCY = 10
// Status is a read/merge/write transaction shared with other controllers.
// Bound optimistic-conflict retries so a hot object cannot stall reconciliation.
const STATUS_CONDITION_WRITE_MAX_ATTEMPTS = 3
/** Types HCC no longer writes and strips from every status write (#606). A future writer must not reuse `NetworkReady`; remove the entry first. */
const RETIRED_STATUS_CONDITION_TYPES = ['NetworkReady'] as const
export const RUNTIME_NOT_DESIRED_DISABLED_MESSAGE =
  'McpServer is disabled; HCC does not run its runtime'
export const RUNTIME_NOT_DESIRED_FAIL_CLOSED_MESSAGE =
  'Env Secret validation failed; HCC does not run its runtime'

function isRetiredStatusConditionType(type: string): boolean {
  return (RETIRED_STATUS_CONDITION_TYPES as readonly string[]).includes(type)
}

type StatusWriteDecline = 'record-obligation' | 'silent'
type StatusWriteDeclineArg = () => StatusWriteDecline
type StatusWriteOutcome = 'wrote' | 'unchanged' | 'declined'
type RuntimeWithdrawKind = 'disabled' | 'fail-closed'
type DeploymentDeleteOutcome = 'gone' | 'denied' | 'foreign'

type RetractManagedRuntimeResult = {
  retracted: boolean
  cleanupComplete: boolean
  foreignRuntime: boolean
  runtimeGone: boolean
}

function shouldStopAfterIncompleteRetract(
  retract: RetractManagedRuntimeResult,
  isCurrent: () => boolean
): boolean {
  if (retract.retracted || retract.foreignRuntime) return false
  return !(retract.runtimeGone && isCurrent())
}

function runtimeNotDesiredMessage(kind: RuntimeWithdrawKind): string {
  switch (kind) {
    case 'disabled':
      return RUNTIME_NOT_DESIRED_DISABLED_MESSAGE
    case 'fail-closed':
      return RUNTIME_NOT_DESIRED_FAIL_CLOSED_MESSAGE
    default: {
      const _exhaustive: never = kind
      return _exhaustive
    }
  }
}

function throwCleanupFailures(failures: unknown[], message: string): void {
  if (failures.length > 0) {
    throw new AggregateError(failures, message)
  }
}

async function mapSettledWithConcurrency<T, TResult>(
  items: readonly T[],
  maxConcurrency: number,
  work: (item: T, index: number) => Promise<TResult>
): Promise<Array<PromiseSettledResult<TResult>>> {
  if (!Number.isInteger(maxConcurrency) || maxConcurrency < 1) {
    throw new RangeError('maxConcurrency must be a positive integer')
  }

  const results = new Array<PromiseSettledResult<TResult>>(items.length)
  let nextIndex = 0
  const worker = async (): Promise<void> => {
    while (nextIndex < items.length) {
      const index = nextIndex
      nextIndex += 1
      try {
        results[index] = {
          status: 'fulfilled',
          value: await work(items[index], index),
        }
      } catch (reason) {
        results[index] = { status: 'rejected', reason }
      }
    }
  }

  const workerCount = Math.min(maxConcurrency, items.length)
  await Promise.all(Array.from({ length: workerCount }, () => worker()))
  return results
}

/**
 * State of the readiness poll window for one McpServer (issue #223).
 *
 * A "window" is one bounded observation of a single Deployment rollout:
 * maxAttempts ticks of intervalMs each. The window survives repeated
 * reconciles — see `readinessPolls` below for why that is load-bearing.
 */
interface ReadinessPollState {
  /**
   * Timer for the next tick; undefined while a tick is executing its awaits
   * (the handle is consumed the moment the callback fires) and once the
   * window reached a terminal verdict.
   */
  timer: ReturnType<typeof setTimeout> | undefined
  /**
   * True while a tick is executing. `timer` alone cannot distinguish "tick
   * running RIGHT NOW" from "window ended", and the difference is
   * load-bearing for pollReadiness() re-entrancy (see `recheckRequested`).
   */
  ticking: boolean
  /**
   * Set by pollReadiness() when it is called while a tick is in flight. The
   * caller (a reconcile) may have JUST rewritten the Deployment — e.g. a
   * corrected credential rotation bumping the generation — and the in-flight
   * tick's read, started earlier, cannot have seen that write. While this
   * flag is set the tick must NOT commit a terminal verdict (ready or
   * exhausted): it defers one interval so the next read, which starts after
   * that Deployment write, observes the new generation and resets the
   * window. Without this, the exhausting tick could publish RolloutIncomplete
   * with the SUPERSEDED generation's numbers and — since McpServer has no
   * periodic resync or Deployment watch — nothing would ever re-arm the poll
   * for the new rollout (issue #223).
   */
  recheckRequested: boolean
  /**
   * Deployment generation the current window is watching; null until the
   * first tick reads it. A change means a NEW rollout (e.g. another
   * credential rotation), which resets the window so the new rollout gets
   * its own full budget.
   */
  generation: number | null
  /** Ticks consumed by the current window. */
  attempts: number
  /** True once the terminal RolloutIncomplete verdict was written for `generation`. */
  exhausted: boolean
  /**
   * Rollout numbers from the exhausting tick. reconcile() re-writes them so
   * the terminal condition stays sticky instead of being downgraded to
   * WaitingForReplicas by the very reconcile the terminal write triggers.
   */
  detail: string
  /**
   * Mutation fence and CRD object of the LATEST reconcile that owns this
   * window. pollReadiness() REBINDS both on every call that finds the window
   * live: a superseding reconcile (e.g. the fullReconcile after a routine
   * McpServer watch recovery bumps the inventory-authority generation) becomes
   * the window's owner, so the next tick evaluates the CURRENT fence instead of
   * the arming reconcile's stale closure. Without the rebind the stale fence
   * self-terminates the window with no re-arm and — status-only MODIFIED events
   * being suppressed, with no periodic McpServer resync — DeploymentReady=False
   * sticks forever for a healthy rollout (the #223 rotation-timeout symptom,
   * reintroduced under a routine reconnect).
   *
   * Two ORTHOGONAL supersession guards protect this window; do not assume one
   * covers the other (that assumption is exactly how the F1 race was born):
   *   - Identity retirement: beginStatusTracking()/clearStatus() retire the
   *     window on any CRD uid or CRD-generation change (spec edit, recreate),
   *     so a surviving window always has the caller's status identity — a
   *     rebind can never adopt a recreate or a superseded desired revision.
   *   - In-tick generation reset + recheckRequested: a credential rotation
   *     bumps neither CRD uid nor CRD generation (it changes the Deployment's
   *     metadata.generation only), so identity retirement does NOT fire; the
   *     rotation is instead guarded by the tick's live Deployment read (a new
   *     generation resets attempts/exhausted) and by recheckRequested deferring
   *     a mid-flight tick's terminal verdict past the rotation's write.
   * A rebind never touches the budget fields (generation/attempts/exhausted/
   * detail) above.
   */
  isCurrent: () => boolean
  server: McpServerCRD
  /**
   * Snapshot of `readinessRetirements` at window create time (#606). A later
   * retirement increments the name's counter; the tick must not publish after
   * that, even if this state was already deleted from `readinessPolls`.
   */
  retirementEpoch: number
}

export class McpServerReconciler {
  private readonly appsApi: k8s.AppsV1Api
  private readonly coreApi: k8s.CoreV1Api
  private readonly customApi: k8s.CustomObjectsApi

  /**
   * Snapshot the McpServer LIST→WATCH authority owned by McpServerWatcher.
   * The fail-closed default prevents orphan cleanup until the watcher wires
   * the real authority source.
   */
  private inventoryAuthority: () => McpServerInventoryAuthoritySnapshot = () => ({
    known: false,
    generation: 0,
  })
  private inventoryAuthorityConfigured = false
  private resolveCurrentServer?: (name: string) => McpServerCRD | undefined
  /** See McpServerReconcilerDeps: test-only opt-in to assume-current when the
   * inventory-authority fence is unwired. Production leaves it false. */
  private readonly assumeInventoryAuthorityWhenUnconfigured: boolean
  private warnedUnconfiguredAuthority = false

  /** Tracks the deployment status of each managed McpServer. */
  private readonly statusMap: Map<string, McpServerStatus> = new Map()
  private readonly statusIdentities: Map<string, Pick<McpServerCRD, 'uid' | 'generation'>> =
    new Map()

  /** G7: Tracks the initial `managed` value per McpServer to enforce immutability. */
  private readonly managedSnapshot: Map<string, boolean> = new Map()

  /**
   * Readiness poll state per server name (issue #223, E2 status-update-loop).
   *
   * Entries outlive individual reconciles ON PURPOSE. Every status write this
   * reconciler makes fires a MODIFIED watch event on the McpServer, which
   * fires another reconcile, which calls pollReadiness() again. When that
   * call cancelled and re-armed the poll (the pre-fix behavior), the attempt
   * budget restarted on every status write, so a rollout that never converges
   * NEVER exhausted its budget and the terminal RolloutIncomplete verdict was
   * never written to the CRD. This map lets pollReadiness() detect a live
   * window and leave it alone, and lets reconcile() keep an exhausted verdict
   * sticky for the same Deployment generation.
   */
  private readonly readinessPolls: Map<string, ReadinessPollState> = new Map()
  /**
   * Retirement epoch per server name (#606). Incremented when a disabled or
   * fail-closed reconcile retracts the readiness window. `clearStatus` does
   * not delete entries: a recreated name keeps the counter so stale ticks fail
   * the epoch check in addition to the uid fence.
   */
  private readonly readinessRetirements = new Map<string, number>()

  /**
   * Tail promise of the in-flight reconcile chain per server name. Serializes
   * concurrent reconcile() calls for the same McpServer so the CRD watch and
   * SecretInformer can't race to replaceNamespacedDeployment() and produce
   * spurious 409 "object has been modified" errors.
   */
  private readonly inFlight: Map<string, Promise<void>> = new Map()

  // A fulfilled reconcile promise can mean an authority guard retired work or
  // a resource API error was reported through status. Keep that distinction
  // independently of discovery readiness and the per-name promise queue.
  private readonly pendingReconciliations = new Map<string, { namespace: string }>()

  hasIncompleteReconciliation(): boolean {
    return this.pendingReconciliations.size > 0
  }

  incompleteReconciliationNames(): string[] {
    return [...this.pendingReconciliations.keys()]
  }

  hasPendingReconciliation(): boolean {
    return (
      this.hasIncompleteReconciliation() ||
      this.inFlight.size > 0 ||
      [...this.readinessPolls.values()].some(
        poll => (poll.timer !== undefined || poll.ticking) && this.readinessPollStillDesired(poll)
      )
    )
  }

  private readinessPollStillDesired(poll: ReadinessPollState): boolean {
    const current = this.resolveCurrentServer?.(poll.server.name)
    return (
      this.resolveCurrentServer === undefined ||
      (current !== undefined && sameMcpServerDesiredRevision(poll.server, current))
    )
  }

  private retainRetiredReadinessPoll(poll: ReadinessPollState): void {
    if (this.readinessPollStillDesired(poll)) {
      this.pendingReconciliations.set(poll.server.name, { namespace: poll.server.namespace })
    }
  }

  /**
   * Retract the live readiness window for a name that will no longer run a
   * runtime (#606). Increments the retirement epoch first so an in-flight tick
   * that already deleted its map entry still fails `retiredNow()`. Clears any
   * pending timer. Does not record an incomplete-publication obligation. Poll
   * ticks pass an `onDecline` factory evaluated at decline time: silent only
   * when `retiredNow()` is true, otherwise the revision fence still records
   * an obligation.
   */
  private retireReadinessWindow(name: string): void {
    this.readinessRetirements.set(name, (this.readinessRetirements.get(name) ?? 0) + 1)
    const poll = this.readinessPolls.get(name)
    if (poll === undefined) return
    if (poll.timer !== undefined) {
      clearTimeout(poll.timer)
      poll.timer = undefined
    }
    this.readinessPolls.delete(name)
  }

  constructor(kc: k8s.KubeConfig, deps?: McpServerReconcilerDeps) {
    this.appsApi = deps?.appsApi ?? kc.makeApiClient(k8s.AppsV1Api)
    this.coreApi = deps?.coreApi ?? kc.makeApiClient(k8s.CoreV1Api)
    this.customApi = deps?.customApi ?? kc.makeApiClient(k8s.CustomObjectsApi)
    this.assumeInventoryAuthorityWhenUnconfigured =
      deps?.assumeInventoryAuthorityWhenUnconfigured ?? false
  }

  /**
   * Late-bound authority source for destructive full-reconcile cleanup.
   * A full pass may finish reconciling desired servers, but it may delete an
   * orphan only while the same authoritative inventory generation remains
   * active from pass start through delete admission.
   */
  setInventoryAuthority(fn: () => McpServerInventoryAuthoritySnapshot): void {
    this.inventoryAuthority = fn
    this.inventoryAuthorityConfigured = true
  }

  /**
   * Resolve current cache membership at effect admission. Full-reconcile
   * snapshots select candidates only; they must never overwrite or delete a
   * same-name CRD accepted by the continuing watch.
   */
  setResolveCurrentServer(fn: (name: string) => McpServerCRD | undefined): void {
    this.resolveCurrentServer = fn
  }

  /**
   * Verdict for a mutation fence evaluated while the inventory authority is NOT
   * wired. Fail closed (deny) unless a test explicitly opted in; deny loudly
   * once, because an unconfigured fence in a running controller is a
   * construction bug, never a runtime condition.
   */
  private currentWhenUnconfigured(): boolean {
    if (this.assumeInventoryAuthorityWhenUnconfigured) return true
    if (!this.warnedUnconfiguredAuthority) {
      this.warnedUnconfiguredAuthority = true
      console.error(
        '[Reconciler] inventory authority not configured — mutation fences fail closed. ' +
          'Wire setInventoryAuthority() and setResolveCurrentServer() before reconciling.'
      )
    }
    return false
  }

  // ─── Status ────────────────────────────────────────────────────────

  /**
   * Get deployment status for an McpServer.
   * Returns current in-memory status when this process has reconciled the
   * server. After a controller restart, a generation-matched persisted Ready
   * condition is the only safe fallback: a legacy condition without
   * observedGeneration, or a condition for an older spec generation, cannot
   * prove that the current desired runtime is ready.
   */
  getStatus(serverOrName: string | McpServerCRD): McpServerStatus {
    const name = typeof serverOrName === 'string' ? serverOrName : serverOrName.name
    const current = this.statusMap.get(name)
    if (current && (typeof serverOrName === 'string' || this.statusIdentityMatches(serverOrName))) {
      return typeof serverOrName === 'string' ? current : { ...current, authoritative: true }
    }

    if (typeof serverOrName !== 'string' && serverOrName.generation !== undefined) {
      const readyConditions =
        serverOrName.status?.conditions?.filter(condition => condition.type === 'Ready') ?? []
      const persistedReady =
        readyConditions.length === 1 &&
        readyConditions[0].observedGeneration === serverOrName.generation
          ? readyConditions[0]
          : undefined
      if (persistedReady) {
        const ready = persistedReady.status === 'True'
        return {
          deployed: ready,
          ready,
          message: persistedReady.message,
          authoritative: true,
        }
      }
    }

    return {
      deployed: false,
      ready: false,
      message: 'Not reconciled yet',
      authoritative: false,
    }
  }

  private statusIdentityMatches(server: McpServerCRD): boolean {
    const identity = this.statusIdentities.get(server.name)
    return (
      identity !== undefined &&
      identity.uid === server.uid &&
      identity.generation === server.generation
    )
  }

  private beginStatusTracking(server: McpServerCRD): void {
    const previousIdentity = this.statusIdentities.get(server.name)
    if (!this.statusIdentityMatches(server)) {
      this.statusMap.delete(server.name)
      // A new CRD identity (uid/generation) retires any poll window armed for
      // the previous desired revision: its verdicts would describe a rollout
      // that no longer represents the desired state. The reconcile for the
      // new identity arms its own window after its Deployment write.
      const poll = this.readinessPolls.get(server.name)
      if (poll) {
        if (poll.timer !== undefined) clearTimeout(poll.timer)
        this.readinessPolls.delete(server.name)
      }
      if (previousIdentity !== undefined && previousIdentity.uid !== server.uid) {
        this.managedSnapshot.delete(server.name)
      }
    }
    this.statusIdentities.set(server.name, {
      uid: server.uid,
      generation: server.generation,
    })
  }

  /**
   * Update the status of an McpServer after reconciliation.
   */
  private setStatus(name: string, status: McpServerStatus): void {
    this.statusMap.set(name, status)
  }

  /**
   * Remove status tracking for a deleted McpServer.
   */
  private clearStatus(name: string): void {
    this.statusMap.delete(name)
    this.statusIdentities.delete(name)
    this.managedSnapshot.delete(name)
    const poll = this.readinessPolls.get(name)
    if (poll) {
      if (poll.timer !== undefined) clearTimeout(poll.timer)
      this.readinessPolls.delete(name)
    }
  }

  /**
   * Read a Deployment's rollout state, generation-aware.
   *
   * `readyReplicas > 0` is NOT enough (issue #223): during a rolling update the
   * OLD pod is still Ready, so a new pod that never starts would be reported as
   * healthy. A rollout has actually converged only when the controller has
   * observed the current generation AND every replica is updated, ready, and
   * available.
   *
   * `totalReplicas === updatedReplicas` is the part that catches a FAILED
   * rollout of a credential rotation (issue #223, E2): with replicas=1 and the
   * default RollingUpdate (maxSurge=1, maxUnavailable=0), a new pod that
   * crash-loops on a bad credential never becomes Ready, so Kubernetes keeps
   * the OLD pod. That old pod holds readyReplicas=1 and availableReplicas=1, so
   * unavailableReplicas stays 0 and updatedReplicas reaches 1 — every other
   * signal looks converged. Only `status.replicas` (total managed pods) reveals
   * the leftover: it stays at 2 while the new pod crash-loops, and drops to 1
   * only once the rollout truly completes. Without this check the poll would
   * report a failed rotation as healthy and never write RolloutIncomplete.
   *
   * `detail` carries the numbers into status conditions so a stuck rollout says
   * what is stuck instead of just "not ready".
   *
   * `generation` is the Deployment's `metadata.generation`, exposed so the
   * readiness poll can tell "same rollout, keep counting" apart from "NEW
   * rollout, fresh window" (issue #223, E2). It plays no part in the
   * readiness verdict above. null when the Deployment could not be read.
   */
  private async readDeploymentRollout(
    name: string,
    namespace: string
  ): Promise<{ ready: boolean; detail: string; generation: number | null }> {
    try {
      const deployment = await this.appsApi.readNamespacedDeployment({ name, namespace })
      const desired = deployment.spec?.replicas ?? 1
      const generation = deployment.metadata?.generation ?? 0
      const observedGeneration = deployment.status?.observedGeneration ?? 0
      const updatedReplicas = deployment.status?.updatedReplicas ?? 0
      const readyReplicas = deployment.status?.readyReplicas ?? 0
      const totalReplicas = deployment.status?.replicas ?? 0
      const unavailableReplicas = deployment.status?.unavailableReplicas ?? 0

      const ready =
        observedGeneration >= generation &&
        updatedReplicas === desired &&
        readyReplicas === desired &&
        totalReplicas === updatedReplicas &&
        unavailableReplicas === 0

      return {
        ready,
        generation,
        detail:
          `generation ${observedGeneration}/${generation}, ` +
          `updated ${updatedReplicas}/${desired}, ready ${readyReplicas}/${desired}, ` +
          `total ${totalReplicas}, unavailable ${unavailableReplicas}`,
      }
    } catch (error: unknown) {
      const message = error instanceof Error ? error.message : String(error)
      return { ready: false, generation: null, detail: `Deployment read failed: ${message}` }
    }
  }

  /**
   * Poll deployment readiness until ready or maxAttempts reached.
   *
   * Re-entrancy is the whole point (issue #223, E2 status-update-loop):
   * reconcile() runs after every MODIFIED watch event — including the events
   * caused by our OWN status writes — and calls this on every pass where the
   * Deployment is not ready yet. A live window is therefore NEVER cancelled
   * or re-armed here; it keeps counting toward its budget so a rollout that
   * never converges reaches the terminal RolloutIncomplete verdict after
   * maxAttempts × intervalMs. A NEW rollout (bumped Deployment generation,
   * e.g. another credential rotation) is detected inside the poll tick
   * itself, which resets the window so the new rollout gets its own budget.
   *
   * A call that lands while a tick is EXECUTING (mid-await) is not silently
   * dropped either: it flags `recheckRequested` so the tick defers any
   * terminal verdict until a read taken AFTER the caller's Deployment write
   * confirms the generation — otherwise the exhausting tick could seal the
   * window with a superseded generation's numbers and the new rollout would
   * never be observed (see ReadinessPollState.recheckRequested).
   */
  private pollReadiness(
    server: McpServerCRD,
    intervalMs = 5000,
    maxAttempts = 24,
    isCurrent: () => boolean = () => true
  ): void {
    const { name, namespace } = server

    const previous = this.readinessPolls.get(name)
    if (previous !== undefined && (previous.timer !== undefined || previous.ticking)) {
      // A window is already running for this server — leave its TIMER and
      // BUDGET alone (re-arming would reset the attempt budget: the E2
      // status-update-loop). But this caller is now the window's owner: rebind
      // the mutation fence (and the same-identity server object) so the next
      // tick evaluates the CURRENT fence instead of the arming reconcile's
      // stale closure, which would otherwise self-terminate the window with no
      // re-arm after a routine watch-generation bump.
      previous.isCurrent = isCurrent
      previous.server = server
      // If its tick is executing RIGHT NOW, the tick's Deployment read may
      // predate the Deployment write that triggered THIS call (a new
      // generation), so a silent no-op could let the tick end the window on
      // stale data. Record that a re-check is owed; the tick honors it before
      // committing any terminal verdict.
      if (previous.ticking) {
        previous.recheckRequested = true
      }
      return
    }

    // Fresh window. Carry over the generation and terminal verdict of a
    // previous exhausted window: if the Deployment has NOT changed since,
    // the new window must not mistake it for a new rollout, and reconcile()
    // keeps reporting the sticky RolloutIncomplete meanwhile.
    const state: ReadinessPollState = {
      timer: undefined,
      ticking: false,
      recheckRequested: false,
      generation: previous?.generation ?? null,
      attempts: 0,
      exhausted: previous?.exhausted ?? false,
      detail: previous?.detail ?? '',
      isCurrent,
      server,
      retirementEpoch: this.readinessRetirements.get(name) ?? 0,
    }
    this.readinessPolls.set(name, state)

    const poll = async (): Promise<void> => {
      // The timer handle was consumed the moment this callback fired; reflect
      // that, and mark the tick as in flight so a concurrent pollReadiness()
      // can tell "tick running" apart from "window ended" during our awaits.
      state.timer = undefined
      state.ticking = true
      const retiredNow = (): boolean =>
        (this.readinessRetirements.get(name) ?? 0) !== state.retirementEpoch
      const publishIfCurrent = (): boolean => state.isCurrent() && !retiredNow()
      const pollWriteDecline = (): StatusWriteDecline =>
        retiredNow() ? 'silent' : 'record-obligation'
      // Mutation fencing: a superseded reconcile (desired-revision or
      // inventory-authority change) must not keep observing — or writing
      // status — on behalf of a stale desired state. End the window quietly;
      // the current reconcile arms its own window after its Deployment write.
      if (!state.isCurrent()) {
        this.retainRetiredReadinessPoll(state)
        state.ticking = false
        if (this.readinessPolls.get(name) === state) this.readinessPolls.delete(name)
        return
      }
      const rollout = await this.readDeploymentRollout(name, namespace)
      if (!state.isCurrent() || !this.statusIdentityMatches(state.server)) {
        this.retainRetiredReadinessPoll(state)
        state.ticking = false
        if (this.readinessPolls.get(name) === state) this.readinessPolls.delete(name)
        return
      }

      // The McpServer may have been deleted (clearStatus) while we were
      // reading. A cancelled window must not keep ticking or write status
      // for a CRD that is gone.
      if (this.readinessPolls.get(name) !== state) return

      // Generation change ⇒ a NEW rollout: it deserves its own full budget,
      // and any terminal verdict for the old generation no longer applies.
      // A failed read (generation null) never resets the window — otherwise
      // a flapping API server would grant infinite budget.
      if (rollout.generation !== null && rollout.generation !== state.generation) {
        state.generation = rollout.generation
        state.attempts = 0
        state.exhausted = false
        state.detail = ''
      }
      state.attempts++

      // A pollReadiness() call landed while this tick was mid-await: the
      // Deployment may have been rewritten AFTER our read (that reconcile
      // writes the Deployment BEFORE calling pollReadiness()), so a terminal
      // verdict from this tick's data could blame a superseded generation —
      // and nothing else would ever re-arm the poll for the new one. Defer
      // the verdict one interval: the next read starts after that Deployment
      // write, so it MUST observe the new generation and reset the window
      // above. A same-generation echo merely delays the verdict by one tick;
      // the budget is NOT reset, so the E2 status-update-loop stays fixed.
      if (state.recheckRequested) {
        state.recheckRequested = false
        if (rollout.ready || state.attempts >= maxAttempts) {
          state.ticking = false
          state.timer = setTimeout(poll, intervalMs)
          return
        }
      }

      if (rollout.ready) {
        state.ticking = false
        this.setStatus(name, { deployed: true, ready: true, message: 'Running' })
        this.readinessPolls.delete(name)
        console.log(
          `[Reconciler] McpServer "${name}" is now ready (after ${state.attempts} poll(s))`
        )
        // The poll MUST write the CRD, not just in-memory status (issue #223).
        // With generation-aware readiness the check right after a replace is
        // always false, so this is the ONLY place a converged rollout is ever
        // recorded — without it every successful credential rotation would sit
        // at DeploymentReady=False forever and the UI would report a timeout.
        if (retiredNow()) return
        await this.updateStatusConditions(
          state.server,
          true,
          undefined,
          publishIfCurrent,
          pollWriteDecline
        )
        // Readiness is decoupled from fleet convergence: the poll also
        // upgrades the persisted, generation-matched Ready condition so a
        // controller restart can trust the recorded readiness (see getStatus).
        if (retiredNow()) return
        await this.writeStatusCondition(
          state.server,
          {
            type: 'Ready',
            status: 'True',
            reason: 'ReconcileSuccess',
            message: 'Deployment created',
          },
          publishIfCurrent,
          pollWriteDecline
        )
        return
      }

      if (state.attempts >= maxAttempts) {
        // Terminal verdict for THIS generation. The entry stays in the map
        // (timer undefined since tick start, ticking cleared here — BEFORE
        // the status write below, so a reconcile arriving during that await
        // opens a fresh observation window instead of flagging a re-check
        // this dying tick would never honor) so reconcile() keeps the verdict
        // sticky and a later pollReadiness() call may re-observe.
        state.ticking = false
        state.exhausted = true
        state.detail = rollout.detail
        console.warn(
          `[Reconciler] McpServer "${name}" not ready after ${maxAttempts} polls — giving up (${rollout.detail})`
        )
        // Give up loudly, on the CRD: a stuck rollout must be visible to the
        // operator with the numbers that prove it, never left as a stale
        // condition or a silent Unknown.
        if (retiredNow()) return
        await this.updateStatusConditions(
          state.server,
          false,
          rollout.detail,
          publishIfCurrent,
          pollWriteDecline
        )
        return
      }

      // Schedule next poll
      state.ticking = false
      state.timer = setTimeout(poll, intervalMs)
    }

    state.timer = setTimeout(poll, intervalMs)
  }

  // ─── Secret Validation ──────────────────────────────────────────────

  /**
   * Validate that the referenced secret exists and contains all required keys.
   * Returns a discriminated result describing success or the exact failure mode.
   */
  async validateSecret(server: McpServerCRD): Promise<SecretValidationResult> {
    // No envSecret: nothing to hash, and no credentials annotation is written.
    if (!server.spec.envSecret) return { ok: true, revision: '' }

    const secretName = server.spec.envSecret.name

    try {
      const secret = await this.coreApi.readNamespacedSecret({
        name: secretName,
        namespace: server.namespace,
      })

      const data = secret.data || {}
      const revision = createHash('sha256')
        .update(canonicalStringify(data as Record<string, unknown>))
        .digest('hex')

      for (const keyMapping of server.spec.envSecret.keys) {
        if (!(keyMapping.secretKey in data)) {
          const message =
            `Secret "${secretName}" is missing required key "${keyMapping.secretKey}" ` +
            `(needed for env var ${keyMapping.envVar})`
          console.error(`[Reconciler] ${message} in McpServer ${server.name}`)
          return { ok: false, reason: 'SecretMissingKey', message }
        }
      }

      console.log(`[Reconciler] Secret "${secretName}" validated for McpServer ${server.name}`)
      return { ok: true, revision }
    } catch (error: unknown) {
      const code = getErrorCode(error)
      if (code === 404) {
        const message = `Secret "${secretName}" not found in namespace "${server.namespace}"`
        console.error(
          `[Reconciler] ${message}. McpServer ${server.name} will NOT be deployed. Create the secret first.`
        )
        return { ok: false, reason: 'SecretNotFound', message }
      }
      if (code === 401 || code === 403) {
        const message =
          `Access denied reading Secret "${secretName}" in namespace "${server.namespace}" ` +
          `(K8s API ${code})`
        console.error(`[Reconciler] ${message}. McpServer ${server.name} will be failed closed.`)
        return { ok: false, reason: 'SecretAccessDenied', message }
      }
      const errMsg = error instanceof Error ? error.message : String(error)
      console.error(`[Reconciler] Failed to read secret "${secretName}":`, error)
      return {
        ok: false,
        reason: 'ReadError',
        message: `Failed to read Secret "${secretName}": ${errMsg}`,
      }
    }
  }

  // ─── Resource Builders ──────────────────────────────────────────────

  // ─── Remote Egress Proxy ─────────────────────────────────────────

  /**
   * Check if this McpServer is a remote egress proxy (has spec.remote.baseUrl).
   */
  private isRemote(server: McpServerCRD): boolean {
    return !!server.spec.remote?.baseUrl
  }

  /**
   * Build nginx.conf ConfigMap for a remote MCP egress proxy.
   * Generates an nginx reverse proxy config that forwards to the external baseUrl.
   */
  /**
   * Validate and sanitize remote baseUrl to prevent SSRF and nginx.conf injection.
   * Returns safe URL reconstructed from parsed components.
   */
  private sanitizeRemoteUrl(rawUrl: string): { safeUrl: string; host: string; path: string } {
    const parsed = new URL(rawUrl) // throws on malformed — no silent fallback
    if (parsed.protocol !== 'https:') {
      throw new Error(`remote.baseUrl must use HTTPS (got ${parsed.protocol})`)
    }
    const host = parsed.hostname
    // Block IP-literal URLs entirely (IPv4 and IPv6) — require DNS hostnames only
    // This prevents IPv6 SSRF bypasses (::1, ::ffff:7f00:1, fd00::, etc.)
    const bareHost = host.replace(/^\[|\]$/g, '')
    if (/^[\d.:[\]]+$/.test(bareHost) || bareHost.includes(':')) {
      throw new Error(`remote.baseUrl must use a DNS hostname, not an IP address: ${host}`)
    }
    // Block internal cluster services (SSRF prevention)
    if (
      host.endsWith('.svc.cluster.local') ||
      host.endsWith('.svc') ||
      host === 'kubernetes.default'
    ) {
      throw new Error(`remote.baseUrl must not point to internal cluster services: ${host}`)
    }
    // Block RFC1918, link-local, loopback (defense in depth for edge cases)
    const PRIVATE_IP = /^(10\.|172\.(1[6-9]|2\d|3[01])\.|192\.168\.|127\.|169\.254\.|0\.)/
    if (PRIVATE_IP.test(host)) {
      throw new Error(`remote.baseUrl must not point to private/reserved IP ranges: ${host}`)
    }
    // Reconstruct safe URL from parsed components (strips injected chars like newlines)
    const path = parsed.pathname
    const safeUrl = parsed.origin + path + parsed.search
    return { safeUrl, host: parsed.host, path }
  }

  /**
   * Codex P0 fix (PR #101): sanitize + validate authHeaders from the McpServer CRD
   * before emitting them as nginx `proxy_set_header` lines. Prevents nginx config
   * injection (breakouts from the quoted string) and HTTP header smuggling.
   *
   * Returns a safe { header, valueTemplate } tuple with the valueTemplate already
   * escaped for nginx double-quoted string context, or null if the entry is
   * rejected (a console.warn is emitted describing why).
   */
  private sanitizeAuthHeader(h: {
    header: string
    valueTemplate: string
  }): { header: string; valueTemplate: string } | null {
    // Validate header name — HTTP token chars only (RFC 7230 subset we allow).
    if (!HEADER_NAME_RE.test(h.header)) {
      console.warn(`[HCC] Rejecting authHeader with invalid name: ${h.header.slice(0, 32)}`)
      return null
    }
    // Bound the value length so a pathological CRD cannot blow up the ConfigMap.
    if (h.valueTemplate.length > MAX_AUTH_HEADER_VALUE_LENGTH) {
      console.warn(`[HCC] Rejecting authHeader value too long for header: ${h.header}`)
      return null
    }
    // CR/LF/NUL allow HTTP request smuggling and nginx directive injection.
    if (/[\r\n\0]/.test(h.valueTemplate)) {
      console.warn(`[HCC] Rejecting authHeader with CR/LF/NUL in value for header: ${h.header}`)
      return null
    }
    // Escape backslashes (first, to avoid double-escaping) then double quotes so
    // the value is safe inside nginx's `"..."` string context.
    const escaped = h.valueTemplate.replace(/\\/g, '\\\\').replace(/"/g, '\\"')
    return { header: h.header, valueTemplate: escaped }
  }

  private buildNginxConfigMap(server: McpServerCRD): k8s.V1ConfigMap {
    const {
      safeUrl,
      host: upstreamHost,
      path: upstreamPath,
    } = this.sanitizeRemoteUrl(server.spec.remote!.baseUrl)
    const transportPort = server.spec.transport.port || 3000

    // Codex P1 fix (PR #101): forward credential secrets as upstream auth headers.
    // Each authHeader produces a `proxy_set_header` line; ${VAR} placeholders in
    // valueTemplate are resolved at pod startup by Clerum's nginx egress proxy
    // /etc/nginx/templates/ envsubst (env vars from envSecret are already mounted
    // into the egress-proxy container by buildDeployment).
    //
    // Codex P0 fix (PR #101): each entry is sanitized + validated via
    // sanitizeAuthHeader() before emission so an operator-authored CRD cannot
    // break out of the quoted string to inject arbitrary nginx config, and
    // cannot smuggle HTTP headers via CR/LF.
    const sanitizedAuthHeaders = (server.spec.remote!.authHeaders ?? [])
      .map(h => this.sanitizeAuthHeader(h))
      .filter((h): h is { header: string; valueTemplate: string } => h !== null)
    const authHeaderLines = sanitizedAuthHeaders
      .map(h => `        proxy_set_header ${h.header} "${h.valueTemplate}";`)
      .join('\n')

    // ── D-2 Authorization precedence/clearing ──
    // Exactly one Authorization directive is emitted, by precedence, because the
    // security decision here is non-obvious:
    //   oauth (and no static Authorization) → passthrough the per-user Bearer that
    //                                          mcp-host injects ($http_authorization);
    //   authHeaders declares Authorization  → emit nothing here — the operator's
    //                                          static credential (already rendered by
    //                                          authHeaderLines) wins;
    //   otherwise (static remote)           → clear it ("") so a static remote never
    //                                          forwards an incoming Authorization by accident.
    // An oauth server that also statically pins Authorization is a contradiction:
    // the static value would shadow the per-user Bearer. Reject the reconcile.
    //
    // Two different detections on purpose:
    //   - CONFLICT (oauth) is keyed on the RAW authHeaders: an oauth server must not
    //     declare Authorization at all, even a malformed value that sanitization would
    //     drop — otherwise the reject could be bypassed by injecting a bad value.
    //   - The clear-vs-operator-wins decision is keyed on the SANITIZED/EMITTED set:
    //     if the operator's Authorization entry was dropped by sanitizeAuthHeader (bad
    //     name/length/CR-LF-NUL), no static line is rendered, so we must still CLEAR —
    //     keying this on the raw list would leave the incoming Authorization forwarded.
    const isOAuth = server.spec.auth?.type === 'oauth'
    const rawDeclaresAuthorizationHeader = (server.spec.remote!.authHeaders ?? []).some(
      h => h.header.toLowerCase() === 'authorization'
    )
    const emitsAuthorizationHeader = sanitizedAuthHeaders.some(
      h => h.header.toLowerCase() === 'authorization'
    )
    if (isOAuth && rawDeclaresAuthorizationHeader) {
      hccLogger.error(
        'Rejecting remote reconcile: oauth server pins a static Authorization header',
        {
          scope: '[Reconciler]',
          mcpServer: server.name,
          namespace: server.namespace,
        }
      )
      throw new Error(
        `remote MCP server "${server.name}" uses auth.type 'oauth' but also declares a static ` +
          'Authorization header in spec.remote.authHeaders; the per-user Bearer must pass through — ' +
          'remove the Authorization entry'
      )
    }
    const authorizationDirective = isOAuth
      ? '        proxy_set_header Authorization $http_authorization;'
      : emitsAuthorizationHeader
        ? ''
        : '        proxy_set_header Authorization "";'
    const authorizationBlock = authorizationDirective
      ? `\n        # ── Authorization precedence (D-2): ${
          isOAuth
            ? 'passthrough the per-user Bearer'
            : 'clear — a static remote never forwards Authorization'
        } ──\n${authorizationDirective}\n`
      : ''

    const nginxConf = `# Auto-generated by Clerum HCC for remote MCP server: ${server.name}
# Stored as .conf.template; Clerum's nginx egress proxy runs envsubst at startup.
server {
    listen ${transportPort};

    # Mirror the upstream endpoint path locally so nginx URI replacement keeps
    # the remote endpoint path exactly once.
    location ${upstreamPath} {
        proxy_pass ${safeUrl};
        proxy_set_header Host ${upstreamHost};
        proxy_set_header X-Real-IP $remote_addr;
        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
        proxy_set_header X-Forwarded-Proto $scheme;
${authHeaderLines ? '\n        # ── Credential auth headers (envsubst-resolved) ──\n' + authHeaderLines + '\n' : ''}${authorizationBlock}
        # SSE/streaming support
        proxy_buffering off;
        proxy_cache off;
        proxy_connect_timeout 10s;
        proxy_read_timeout 86400s;
        proxy_send_timeout 86400s;
        chunked_transfer_encoding on;

        # TLS to upstream
        proxy_ssl_verify on;
        proxy_ssl_server_name on;
        proxy_ssl_trusted_certificate /etc/ssl/certs/ca-certificates.crt;
        # nginx defaults proxy_ssl_verify_depth to 1, which only admits a chain
        # with a single untrusted intermediate. CAs mid-root-rollover serve a
        # chain that reaches a trusted anchor through a cross-signed root, so the
        # leaf sits two intermediates below the trust store: Let's Encrypt via
        # "Root YE"/"Root YR" (cross-signed by ISRG Root X2/X1) and Microsoft via
        # "Microsoft TLS ECC Root G2" (cross-signed by MS ECC Root 2017). At depth
        # 1 nginx rejects those with "(20:unable to get local issuer certificate)"
        # even though the CA bundle is complete — it looks like a missing root but
        # is purely the depth cap. Raising the depth keeps verification fully on.
        proxy_ssl_verify_depth 3;
    }

    # Health check endpoint
    location /health {
        return 200 'ok';
        add_header Content-Type text/plain;
    }
}`

    const labels: Record<string, string> = {
      app: server.name,
      [MANAGED_BY_LABEL]: MANAGED_BY_VALUE,
      [MCPSERVER_LABEL]: server.name,
      'clerum.io/component': 'egress-proxy',
    }

    return {
      apiVersion: 'v1',
      kind: 'ConfigMap',
      metadata: {
        name: `${server.name}-nginx-conf`,
        namespace: server.namespace,
        labels,
        ...(server.uid && {
          ownerReferences: [
            {
              apiVersion: 'clerum.io/v1alpha1',
              kind: 'McpServer',
              name: server.name,
              uid: server.uid,
              controller: true,
              blockOwnerDeletion: true,
            },
          ],
        }),
      },
      data: {
        // Stored as .conf.template so Clerum's nginx egress proxy runs envsubst at
        // startup, resolving ${VAR} placeholders from env (sourced from envSecret).
        'default.conf.template': nginxConf,
      },
    }
  }

  /**
   * Ensure ConfigMap exists and is up to date (create or replace).
   */
  private async ensureConfigMap(
    server: McpServerCRD,
    isCurrent: () => boolean = () => true
  ): Promise<void> {
    const cm = this.buildNginxConfigMap(server)
    const name = `${server.name}-nginx-conf`
    const namespace = server.namespace
    await ensureResource({
      mutationAllowed: isCurrent,
      read: () =>
        observeExistenceRead('ConfigMap', () =>
          this.coreApi.readNamespacedConfigMap({ name, namespace })
        ),
      create: async () => {
        await observeCreate('ConfigMap', () =>
          this.coreApi.createNamespacedConfigMap({ namespace, body: cm })
        )
        hccLogger.info('nginx ConfigMap created', {
          scope: '[Reconciler]',
          configMap: name,
          namespace,
        })
      },
      converge: read =>
        replaceWithConflictRetry({
          description: `nginx ConfigMap "${name}"`,
          logPrefix: '[Reconciler]',
          body: cm,
          mergeExisting: preserveObjectAnnotations,
          isUpToDate: configMapMatchesDesired,
          mutationAllowed: isCurrent,
          read,
          replace: body => this.coreApi.replaceNamespacedConfigMap({ name, namespace, body }),
        }),
      onSkipped: () => createsTotal.inc({ kind: 'ConfigMap', outcome: 'skipped' }),
    })
  }

  /**
   * Build a Deployment manifest for an McpServer.
   */
  /**
   * Sanitize CRD spec fields to prevent developer-supplied CRDs from
   * influencing platform-controlled deployment behavior.
   *
   * Policy: the platform ALWAYS wins. CRD fields that could escalate
   * privileges, override security, or DoS the cluster are stripped or
   * overridden with platform defaults.
   */
  private sanitizeCrdSpec(server: McpServerCRD): void {
    // ── CRITICAL: imagePullPolicy — platform decides, not the CRD ──
    // Remove CRD-supplied value; buildDeployment uses config.mcpServerImagePullPolicy
    delete server.spec.imagePullPolicy

    // ── CRITICAL: prevent root execution ──
    if (server.spec.security) {
      if (server.spec.security.runAsUser !== undefined && server.spec.security.runAsUser < 1) {
        console.warn(
          `[Security] McpServer "${server.name}": runAsUser=${server.spec.security.runAsUser} rejected (must be >= 1)`
        )
        server.spec.security.runAsUser = 1000
      }
      if (server.spec.security.runAsGroup !== undefined && server.spec.security.runAsGroup < 1) {
        console.warn(
          `[Security] McpServer "${server.name}": runAsGroup=${server.spec.security.runAsGroup} rejected (must be >= 1)`
        )
        server.spec.security.runAsGroup = 1000
      }
      if (server.spec.security.fsGroup !== undefined && server.spec.security.fsGroup < 1) {
        console.warn(
          `[Security] McpServer "${server.name}": fsGroup=${server.spec.security.fsGroup} rejected (must be >= 1)`
        )
        server.spec.security.fsGroup = 1000
      }

      // ── CRITICAL: strip dangerous capabilities ──
      if (server.spec.security.addCapabilities) {
        const original = server.spec.security.addCapabilities
        server.spec.security.addCapabilities = original.filter(cap => {
          if (!isWorkflowRecipeDefaultAllowedCapability(cap)) {
            console.warn(
              `[Security] McpServer "${server.name}": capability "${cap}" stripped (forbidden)`
            )
            return false
          }
          return true
        })
      }
    }

    // ── HIGH: sanitize env vars — block dangerous names ──
    const FORBIDDEN_ENV = new Set([
      'LD_PRELOAD',
      'LD_LIBRARY_PATH',
      'PATH',
      'NODE_OPTIONS',
      'PYTHONPATH',
      'JAVA_TOOL_OPTIONS',
      'KUBECONFIG',
      'KUBERNETES_SERVICE_HOST',
      'KUBERNETES_SERVICE_PORT',
    ])
    if (server.spec.env) {
      server.spec.env = server.spec.env.filter(e => {
        if (FORBIDDEN_ENV.has(e.name.toUpperCase())) {
          console.warn(
            `[Security] McpServer "${server.name}": env "${e.name}" stripped (forbidden)`
          )
          return false
        }
        return true
      })
    }

    // ── MEDIUM: clamp resources to platform limits ──
    if (server.spec.resources) {
      const MAX_CPU = '4000m'
      const MAX_MEM = '8Gi'
      if (server.spec.resources.limits) {
        if (server.spec.resources.limits.cpu && parseInt(server.spec.resources.limits.cpu) > 4000) {
          server.spec.resources.limits.cpu = MAX_CPU
        }
        if (
          server.spec.resources.limits.memory &&
          parseInt(server.spec.resources.limits.memory) > 8192
        ) {
          server.spec.resources.limits.memory = MAX_MEM
        }
      }
    }
  }

  private buildDeployment(server: McpServerCRD, credentialsRevision = ''): k8s.V1Deployment {
    // ── Platform hardening: sanitize an ephemeral copy before building PodSpec ──
    // The caller retains this object as the LIST -> WATCH desired-state cache.
    // Mutating it here makes a later status-only watch event look like a
    // user-driven spec revision, which invalidates the authoritative policy
    // revocation marker during startup. Sanitization remains mandatory for the
    // generated PodSpec, but must never rewrite the observed CRD inventory.
    server = structuredClone(server)
    this.sanitizeCrdSpec(server)

    const transportPort = server.spec.transport.port || 3000
    const isStdio = server.spec.transport.type === 'stdio'
    const isRemote = this.isRemote(server)
    const healthCheckPort = server.spec.healthCheck?.port

    // --- Env vars ---
    const env: k8s.V1EnvVar[] = []

    // Env vars derived from CRD fields via envMapping
    const mapping = server.spec.envMapping
    if (mapping) {
      if (mapping.transport) {
        env.push({ name: mapping.transport, value: 'http' })
      }
      if (mapping.httpHost) {
        env.push({ name: mapping.httpHost, value: '0.0.0.0' })
      }
      if (mapping.httpPort) {
        env.push({ name: mapping.httpPort, value: String(transportPort) })
      }
      if (mapping.healthCheckHost) {
        env.push({ name: mapping.healthCheckHost, value: '0.0.0.0' })
      }
      if (mapping.healthCheckPort && healthCheckPort) {
        env.push({ name: mapping.healthCheckPort, value: String(healthCheckPort) })
      }

      const sc = server.spec.serverConfig
      if (mapping.readOnly && sc?.readOnly !== undefined) {
        env.push({ name: mapping.readOnly, value: String(sc.readOnly) })
      }
      if (mapping.loggers && sc?.loggers) {
        env.push({ name: mapping.loggers, value: sc.loggers })
      }
      if (mapping.telemetry && sc?.telemetry) {
        env.push({ name: mapping.telemetry, value: sc.telemetry })
      }
    }

    // Additional plain env vars from CRD (server-specific extras)
    if (server.spec.env) {
      for (const e of server.spec.env) {
        env.push({ name: e.name, value: e.value })
      }
    }

    // Secret-backed env vars
    if (server.spec.envSecret) {
      for (const keyMapping of server.spec.envSecret.keys) {
        env.push({
          name: keyMapping.envVar,
          valueFrom: {
            secretKeyRef: {
              name: server.spec.envSecret.name,
              key: keyMapping.secretKey,
            },
          },
        })
      }
    }

    // Remote servers: ensure authHeaders ${VAR} placeholders have env vars
    // defined, even when credentials are optional and envSecret is absent.
    // Without this, nginx envsubst fails with "unknown variable" on startup.
    if (isRemote && server.spec.remote?.authHeaders) {
      const definedVars = new Set(env.map(e => e.name))
      for (const ah of server.spec.remote.authHeaders) {
        const matches = ah.valueTemplate.match(/\$\{(\w+)\}/g)
        if (!matches) continue
        for (const m of matches) {
          const varName = m.slice(2, -1)
          if (!definedVars.has(varName)) {
            env.push({ name: varName, value: '' })
            definedVars.add(varName)
          }
        }
      }
    }

    // --- Ports ---
    const ports: k8s.V1ContainerPort[] = [
      { name: 'http', containerPort: transportPort, protocol: 'TCP' },
    ]
    if (healthCheckPort) {
      ports.push({ name: 'healthcheck', containerPort: healthCheckPort, protocol: 'TCP' })
    }

    // --- Probes ---
    const probePort = healthCheckPort ? 'healthcheck' : 'http'

    // --- Resources ---
    let resources: k8s.V1ResourceRequirements | undefined
    if (server.spec.resources) {
      resources = {}
      if (server.spec.resources.requests) {
        resources.requests = {}
        if (server.spec.resources.requests.memory)
          resources.requests['memory'] = server.spec.resources.requests.memory
        if (server.spec.resources.requests.cpu)
          resources.requests['cpu'] = server.spec.resources.requests.cpu
      }
      if (server.spec.resources.limits) {
        resources.limits = {}
        if (server.spec.resources.limits.memory)
          resources.limits['memory'] = server.spec.resources.limits.memory
        if (server.spec.resources.limits.cpu)
          resources.limits['cpu'] = server.spec.resources.limits.cpu
      }
    }

    // --- Labels ---
    const labels: Record<string, string> = {
      app: server.name,
      [MANAGED_BY_LABEL]: MANAGED_BY_VALUE,
      [MCPSERVER_LABEL]: server.name,
    }
    const runAsUser = server.spec.security?.runAsUser ?? 1000
    const runAsGroup = server.spec.security?.runAsGroup ?? runAsUser
    const hardenedContainerSecurityContext: k8s.V1SecurityContext = {
      allowPrivilegeEscalation: false,
      runAsNonRoot: true,
      runAsUser,
      runAsGroup,
      capabilities: { drop: ['ALL'] },
      seccompProfile: { type: 'RuntimeDefault' },
    }

    return {
      apiVersion: 'apps/v1',
      kind: 'Deployment',
      metadata: {
        name: server.name,
        namespace: server.namespace,
        labels,
        // G5: ownerRef to McpServer CRD for garbage collection
        ...(server.uid && {
          ownerReferences: [
            {
              apiVersion: 'clerum.io/v1alpha1',
              kind: 'McpServer',
              name: server.name,
              uid: server.uid,
              controller: true,
              blockOwnerDeletion: true,
            },
          ],
        }),
      },
      spec: {
        replicas: 1,
        selector: {
          matchLabels: { app: server.name },
        },
        template: {
          metadata: {
            labels,
            // Issue #223: rotating a credential changes this digest, which
            // changes the pod template, which is what makes Kubernetes roll the
            // pod. Omitted when there is no envSecret — an annotation with an
            // empty value would be one more field to drift on for connectors
            // that have no credentials at all.
            ...(credentialsRevision && {
              annotations: { [CREDENTIALS_REVISION_ANNOTATION]: credentialsRevision },
            }),
          },
          spec: {
            automountServiceAccountToken: false,
            enableServiceLinks: false,
            // G1: Apply security context from McpServer CRD spec
            securityContext: {
              runAsNonRoot: true,
              runAsUser,
              runAsGroup,
              seccompProfile: { type: 'RuntimeDefault' },
              ...(server.spec.security?.fsGroup !== undefined && {
                fsGroup: server.spec.security.fsGroup,
              }),
            },
            // G6: Apply imagePullSecrets from McpServer CRD spec
            // Codex P2 fix (PR #101): CRD schema mandates [{name: string}] per K8s
            // LocalObjectReference; producers (mcpDelegation, registry install) now
            // normalize to objects, so we pass through directly without double-wrapping.
            ...(server.spec.imagePullSecrets?.length
              ? {
                  imagePullSecrets: server.spec.imagePullSecrets,
                }
              : {}),
            initContainers:
              isStdio && !isRemote
                ? [
                    {
                      name: 'copy-mcp-app',
                      image: server.spec.image,
                      imagePullPolicy: config.mcpServerImagePullPolicy,
                      command: [
                        'sh',
                        '-c',
                        'cp -r /app/* /mcp-bin/ 2>/dev/null; cp -r /mcp-app/* /mcp-bin/ 2>/dev/null; true',
                      ],
                      securityContext: hardenedContainerSecurityContext,
                      volumeMounts: [{ name: 'mcp-bin', mountPath: '/mcp-bin' }],
                    },
                  ]
                : undefined,
            containers: [
              // Remote egress proxy: nginx container with ConfigMap mount
              ...(isRemote
                ? [
                    {
                      name: 'egress-proxy',
                      image: config.egressProxyImage,
                      imagePullPolicy: 'IfNotPresent' as const,
                      ports: [
                        { name: 'http', containerPort: transportPort, protocol: 'TCP' as const },
                      ],
                      env: env.length > 0 ? env : undefined,
                      // Codex P1 fix (PR #101): mount as templates so the nginx proxy
                      // entrypoint runs envsubst at startup and resolves ${VAR}
                      // placeholders in auth headers using env from envSecret.
                      volumeMounts: [
                        { name: 'nginx-conf', mountPath: '/etc/nginx/templates', readOnly: true },
                      ],
                      livenessProbe: {
                        httpGet: { path: '/health', port: transportPort as unknown as IntOrString },
                        initialDelaySeconds: 5,
                        periodSeconds: 15,
                      },
                      readinessProbe: {
                        httpGet: { path: '/health', port: transportPort as unknown as IntOrString },
                        initialDelaySeconds: 3,
                        periodSeconds: 10,
                      },
                      resources: {
                        requests: { memory: '32Mi', cpu: '25m' },
                        limits: { memory: '64Mi', cpu: '100m' },
                      },
                      securityContext: {
                        runAsNonRoot: true,
                        runAsUser: 101,
                        runAsGroup: 101,
                        capabilities: { drop: ['ALL'] },
                        allowPrivilegeEscalation: false,
                        seccompProfile: { type: 'RuntimeDefault' },
                      },
                    },
                  ]
                : []),
              // Local HTTP server (not stdio, not remote)
              ...(isStdio || isRemote
                ? []
                : [
                    {
                      name: 'mcp-server',
                      image: server.spec.image,
                      imagePullPolicy: config.mcpServerImagePullPolicy,
                      command: server.spec.command || undefined,
                      args: server.spec.args || undefined,
                      ports,
                      env: env.length > 0 ? env : undefined,
                      // G1: Per-container security context — capabilities.add must be at container level
                      securityContext: {
                        allowPrivilegeEscalation: false,
                        runAsNonRoot: true,
                        runAsUser,
                        runAsGroup,
                        seccompProfile: { type: 'RuntimeDefault' },
                        capabilities: {
                          drop: ['ALL'],
                          ...(server.spec.security?.addCapabilities?.length
                            ? { add: server.spec.security.addCapabilities }
                            : {}),
                        },
                      },
                      livenessProbe: {
                        tcpSocket: { port: probePort as IntOrString },
                        initialDelaySeconds: 10,
                        periodSeconds: 15,
                      },
                      readinessProbe: {
                        tcpSocket: { port: probePort as IntOrString },
                        initialDelaySeconds: 5,
                        periodSeconds: 10,
                      },
                      resources,
                    },
                  ]),
              ...(isStdio
                ? [
                    {
                      name: 'stdio-bridge',
                      image: config.stdioBridgeImage, // G10: configurable bridge image
                      imagePullPolicy: config.mcpServerImagePullPolicy,
                      env: [
                        {
                          name: 'STDIO_COMMAND',
                          value: (server.spec.command || ['/mcp-bin/mcp-server'])[0],
                        },
                        {
                          name: 'STDIO_ARGS',
                          value: JSON.stringify(
                            server.spec.args || (server.spec.command || []).slice(1)
                          ),
                        },
                        { name: 'BRIDGE_PORT', value: String(transportPort) },
                        ...env,
                      ],
                      ports: [{ name: 'http', containerPort: transportPort, protocol: 'TCP' }],
                      securityContext: hardenedContainerSecurityContext,
                      livenessProbe: {
                        httpGet: { path: '/health', port: transportPort as unknown as IntOrString },
                        initialDelaySeconds: 15,
                        periodSeconds: 15,
                      },
                      readinessProbe: {
                        httpGet: { path: '/health', port: transportPort as unknown as IntOrString },
                        initialDelaySeconds: 10,
                        periodSeconds: 10,
                      },
                      volumeMounts: [{ name: 'mcp-bin', mountPath: '/mcp-bin' }],
                      // G4: Independent resource limits for stdio-bridge sidecar
                      resources: {
                        requests: config.stdioBridgeResources.requests,
                        limits: config.stdioBridgeResources.limits,
                      },
                    },
                  ]
                : []),
            ],
            volumes: (() => {
              const v: k8s.V1Volume[] = []
              if (isStdio) v.push({ name: 'mcp-bin', emptyDir: {} })
              if (isRemote)
                v.push({ name: 'nginx-conf', configMap: { name: `${server.name}-nginx-conf` } })
              return v.length > 0 ? v : undefined
            })(),
          },
        },
      },
    }
  }

  /**
   * Build a Service manifest for an McpServer.
   */
  private buildService(server: McpServerCRD): k8s.V1Service {
    const transportPort = server.spec.transport.port || 3000

    return {
      apiVersion: 'v1',
      kind: 'Service',
      metadata: {
        name: server.name,
        namespace: server.namespace,
        labels: {
          app: server.name,
          [MANAGED_BY_LABEL]: MANAGED_BY_VALUE,
          [MCPSERVER_LABEL]: server.name,
        },
        // G5: ownerRef to McpServer CRD for garbage collection
        ...(server.uid && {
          ownerReferences: [
            {
              apiVersion: 'clerum.io/v1alpha1',
              kind: 'McpServer',
              name: server.name,
              uid: server.uid,
              controller: true,
              blockOwnerDeletion: true,
            },
          ],
        }),
      },
      spec: {
        type: 'ClusterIP',
        ports: [
          {
            port: transportPort,
            targetPort: 'http' as IntOrString,
            protocol: 'TCP',
            name: 'http',
          },
        ],
        selector: { app: server.name },
      },
    }
  }

  // ─── Reconciliation Actions ─────────────────────────────────────────

  /**
   * Ensure a Deployment exists and is up to date.
   */
  private async ensureDeployment(
    server: McpServerCRD,
    credentialsRevision = '',
    isCurrent: () => boolean = () => true
  ): Promise<void> {
    const deployment = this.buildDeployment(server, credentialsRevision)

    await ensureResource({
      mutationAllowed: isCurrent,
      read: () =>
        observeExistenceRead('Deployment', () =>
          this.appsApi.readNamespacedDeployment({
            name: server.name,
            namespace: server.namespace,
          })
        ),
      create: async () => {
        await observeCreate('Deployment', () =>
          this.appsApi.createNamespacedDeployment({
            namespace: server.namespace,
            body: deployment,
          })
        )
        hccLogger.info('Deployment created', {
          scope: '[Reconciler]',
          deployment: server.name,
          namespace: server.namespace,
        })
      },
      converge: read =>
        replaceWithConflictRetry({
          description: `Deployment "${server.name}"`,
          logPrefix: '[Reconciler]',
          body: deployment,
          mergeExisting: preserveDeploymentAnnotations,
          isUpToDate: deploymentMatchesDesired,
          read,
          replace: body =>
            this.appsApi.replaceNamespacedDeployment({
              name: server.name,
              namespace: server.namespace,
              body,
            }),
          mutationAllowed: isCurrent,
        }),
      onSkipped: () => createsTotal.inc({ kind: 'Deployment', outcome: 'skipped' }),
    })
  }

  /**
   * Ensure a Service exists and is up to date.
   */
  private async ensureService(
    server: McpServerCRD,
    isCurrent: () => boolean = () => true
  ): Promise<void> {
    const service = this.buildService(server)

    await ensureResource({
      mutationAllowed: isCurrent,
      read: () =>
        observeExistenceRead('Service', () =>
          this.coreApi.readNamespacedService({
            name: server.name,
            namespace: server.namespace,
          })
        ),
      create: () =>
        observeCreate('Service', () =>
          this.coreApi.createNamespacedService({
            namespace: server.namespace,
            body: service,
          })
        ),
      converge: read =>
        replaceWithConflictRetry({
          description: `Service "${server.name}"`,
          logPrefix: '[Reconciler]',
          body: service,
          mergeExisting: preserveServiceAssignedFields,
          isUpToDate: serviceMatchesDesired,
          read,
          replace: body =>
            this.coreApi.replaceNamespacedService({
              name: server.name,
              namespace: server.namespace,
              body,
            }),
          mutationAllowed: isCurrent,
        }),
      onSkipped: () => createsTotal.inc({ kind: 'Service', outcome: 'skipped' }),
    })
  }

  /**
   * Delete HCC-owned Deployment, Service, and remote ConfigMap for an McpServer.
   * Name matches are not enough: WRC-owned `managed:false` runtimes can share
   * names with McpServer CRDs, so every delete verifies HCC ownership labels.
   */
  private async deleteResources(
    name: string,
    namespace: string,
    deleteAllowed?: () => Promise<boolean>
  ): Promise<void> {
    const failures: unknown[] = []
    for (const cleanup of [
      () => this.deleteDeploymentIfHccOwned(name, namespace, deleteAllowed),
      () => this.deleteConfigMapIfHccOwned(`${name}-nginx-conf`, namespace, name, deleteAllowed),
      () => this.deleteServiceIfHccOwned(name, namespace, deleteAllowed),
    ]) {
      try {
        await cleanup()
      } catch (error) {
        failures.push(error)
      }
    }
    throwCleanupFailures(
      failures,
      `Failed to delete runtime Kubernetes resources for McpServer "${name}"`
    )
  }

  private isHccOwnedMcpResource(
    resource: { metadata?: { labels?: Record<string, string> } },
    serverName: string
  ): boolean {
    const labels = resource.metadata?.labels ?? {}
    return labels[MANAGED_BY_LABEL] === MANAGED_BY_VALUE && labels[MCPSERVER_LABEL] === serverName
  }

  private async deleteDeploymentIfHccOwned(
    name: string,
    namespace: string,
    deleteAllowed?: () => Promise<boolean>
  ): Promise<DeploymentDeleteOutcome> {
    let deployment: k8s.V1Deployment
    try {
      deployment = await this.appsApi.readNamespacedDeployment({ name, namespace })
    } catch (error: unknown) {
      if (getErrorCode(error) === 404) {
        hccLogger.info('Deployment already gone', { name })
        return 'gone'
      }
      hccLogger.error('Failed to read Deployment ownership', { name, err: error })
      throw error
    }

    if (!this.isHccOwnedMcpResource(deployment, name)) {
      hccLogger.warn('Skipping Deployment delete — not HCC-owned', { name })
      return 'foreign'
    }
    if (deleteAllowed && !(await deleteAllowed())) return 'denied'

    try {
      await this.appsApi.deleteNamespacedDeployment({ name, namespace })
      hccLogger.info('Deleted Deployment', { name })
      return 'gone'
    } catch (error: unknown) {
      if (getErrorCode(error) === 404) {
        hccLogger.info('Deployment already gone', { name })
        return 'gone'
      }
      hccLogger.error('Failed to delete Deployment', { name, err: error })
      throw error
    }
  }

  /**
   * Delete leftover HCC-owned ConfigMap and Service after the Deployment
   * outcome is known. Does not write status.
   */
  private async deleteHccOwnedRuntimeSiblings(
    name: string,
    namespace: string,
    deleteAllowed?: () => Promise<boolean>
  ): Promise<void> {
    const siblingFailures: unknown[] = []
    for (const cleanup of [
      () => this.deleteConfigMapIfHccOwned(`${name}-nginx-conf`, namespace, name, deleteAllowed),
      () => this.deleteServiceIfHccOwned(name, namespace, deleteAllowed),
    ]) {
      try {
        await cleanup()
      } catch (error) {
        siblingFailures.push(error)
      }
    }
    throwCleanupFailures(
      siblingFailures,
      `Failed to delete runtime Kubernetes resources for McpServer "${name}"`
    )
  }

  /**
   * Withdraw an HCC-owned runtime (#606): once the Deployment is confirmed
   * gone, retire the poll, persist DeploymentReady=False/RuntimeNotDesired,
   * then finish sibling deletes. Sibling failure does not roll back the
   * retract. Retire before the retract write so an in-flight tick cannot
   * republish True after the retract PATCH commits but before the epoch
   * bumps. A foreign (WRC-owned) Deployment retires the window without
   * writing RuntimeNotDesired.
   */
  private async retractManagedRuntime(
    server: McpServerCRD,
    kind: RuntimeWithdrawKind,
    isCurrent: () => boolean
  ): Promise<RetractManagedRuntimeResult> {
    const name = server.name
    const namespace = server.namespace
    const deleteAllowed = async (): Promise<boolean> => isCurrent()
    let cleanupComplete = true
    const gatedDeleteAllowed = async (): Promise<boolean> => {
      const allowed = await deleteAllowed()
      if (!allowed) cleanupComplete = false
      return allowed
    }

    let deploymentOutcome: DeploymentDeleteOutcome
    try {
      deploymentOutcome = await this.deleteDeploymentIfHccOwned(name, namespace, deleteAllowed)
    } catch (error) {
      const failures: unknown[] = [error]
      for (const cleanup of [
        () => this.deleteConfigMapIfHccOwned(`${name}-nginx-conf`, namespace, name, deleteAllowed),
        () => this.deleteServiceIfHccOwned(name, namespace, deleteAllowed),
      ]) {
        try {
          await cleanup()
        } catch (siblingError) {
          failures.push(siblingError)
        }
      }
      throwCleanupFailures(
        failures,
        `Failed to delete runtime Kubernetes resources for McpServer "${name}"`
      )
      throw new Error(`Failed to delete runtime Kubernetes resources for McpServer "${name}"`)
    }

    switch (deploymentOutcome) {
      case 'denied':
        return {
          retracted: false,
          cleanupComplete: false,
          foreignRuntime: false,
          runtimeGone: false,
        }
      case 'foreign':
        this.retireReadinessWindow(name)
        await this.deleteHccOwnedRuntimeSiblings(name, namespace, gatedDeleteAllowed)
        return {
          retracted: false,
          cleanupComplete: cleanupComplete && isCurrent(),
          foreignRuntime: true,
          runtimeGone: false,
        }
      case 'gone':
        break
      default: {
        const _exhaustive: never = deploymentOutcome
        return _exhaustive
      }
    }

    this.retireReadinessWindow(name)
    const retractWrite = await this.writeStatusCondition(
      server,
      {
        type: 'DeploymentReady',
        status: 'False',
        reason: 'RuntimeNotDesired',
        message: runtimeNotDesiredMessage(kind),
      },
      isCurrent
    )
    const epoch = this.readinessRetirements.get(name)
    if (retractWrite === 'declined') {
      hccLogger.warn('Retract write declined', { name, kind, epoch, outcome: retractWrite })
    } else {
      hccLogger.info('Retracted managed runtime', { name, kind, epoch, outcome: retractWrite })
    }

    await this.deleteHccOwnedRuntimeSiblings(name, namespace, gatedDeleteAllowed)
    if (retractWrite === 'declined') {
      return {
        retracted: false,
        cleanupComplete: false,
        foreignRuntime: false,
        runtimeGone: true,
      }
    }
    return {
      retracted: true,
      cleanupComplete: cleanupComplete && isCurrent(),
      foreignRuntime: false,
      runtimeGone: true,
    }
  }

  private async deleteConfigMapIfHccOwned(
    configMapName: string,
    namespace: string,
    serverName: string,
    deleteAllowed?: () => Promise<boolean>
  ): Promise<void> {
    let configMap: k8s.V1ConfigMap
    try {
      configMap = await this.coreApi.readNamespacedConfigMap({ name: configMapName, namespace })
    } catch (error: unknown) {
      if (getErrorCode(error) === 404) {
        console.log(`[Reconciler] ConfigMap "${configMapName}" already gone`)
      } else {
        console.error(`[Reconciler] Failed to read ConfigMap "${configMapName}" ownership:`, error)
        throw error
      }
      return
    }

    if (!this.isHccOwnedMcpResource(configMap, serverName)) {
      console.warn(`[Reconciler] Skipping ConfigMap "${configMapName}" delete — not HCC-owned`)
      return
    }
    if (deleteAllowed && !(await deleteAllowed())) return

    try {
      await this.coreApi.deleteNamespacedConfigMap({ name: configMapName, namespace })
      console.log(`[Reconciler] Deleted nginx ConfigMap "${configMapName}"`)
    } catch (error: unknown) {
      if (getErrorCode(error) === 404) {
        console.log(`[Reconciler] ConfigMap "${configMapName}" already gone`)
      } else {
        console.error(`[Reconciler] Failed to delete ConfigMap "${configMapName}":`, error)
        throw error
      }
    }
  }

  private async deleteServiceIfHccOwned(
    name: string,
    namespace: string,
    deleteAllowed?: () => Promise<boolean>
  ): Promise<void> {
    let service: k8s.V1Service
    try {
      service = await this.coreApi.readNamespacedService({ name, namespace })
    } catch (error: unknown) {
      if (getErrorCode(error) === 404) {
        console.log(`[Reconciler] Service "${name}" already gone`)
      } else {
        console.error(`[Reconciler] Failed to read Service "${name}" ownership:`, error)
        throw error
      }
      return
    }

    if (!this.isHccOwnedMcpResource(service, name)) {
      console.warn(`[Reconciler] Skipping Service "${name}" delete — not HCC-owned`)
      return
    }
    if (deleteAllowed && !(await deleteAllowed())) return

    try {
      await this.coreApi.deleteNamespacedService({ name, namespace })
      console.log(`[Reconciler] Deleted Service "${name}"`)
    } catch (error: unknown) {
      if (getErrorCode(error) === 404) {
        console.log(`[Reconciler] Service "${name}" already gone`)
      } else {
        console.error(`[Reconciler] Failed to delete Service "${name}":`, error)
        throw error
      }
    }
  }

  private async deleteRuntimeResources(
    name: string,
    namespace: string,
    deleteAllowed?: () => Promise<boolean>
  ): Promise<boolean> {
    // A guarded no-op is incomplete even if authority returns before the
    // outer call settles. Ownership-based skips remain legitimate outcomes.
    let complete = true
    await this.deleteResources(
      name,
      namespace,
      deleteAllowed
        ? async () => {
            const allowed = await deleteAllowed()
            if (!allowed) complete = false
            return allowed
          }
        : undefined
    )
    return complete
  }

  // ─── CRD Status & Annotation Patching ──────────────────────────────

  /**
   * G2: Set `clerum.io/network-ready: "true"` annotation on McpServer CRD.
   * Completes the pre-deploy handshake so WRC knows NetworkPolicies are applied.
   */
  private async setNetworkReadyAnnotation(
    server: McpServerCRD,
    isCurrent: () => boolean = () => true
  ): Promise<void> {
    // Use merge-patch (plain object body) instead of JSON Patch (array body).
    // JSON Patch op:"add" on /metadata/annotations/<key> requires /metadata/annotations
    // to already exist — it fails with 422 when the CRD was created without annotations.
    // Merge-patch creates the map if absent. Errors are re-thrown so the caller (reconcile)
    // knows the handshake was not completed — WRC will time out and fail rather than proceed
    // silently without NetworkPolicies in place.
    if (!isCurrent()) return
    await this.customApi.patchNamespacedCustomObject(
      {
        group: 'clerum.io',
        version: 'v1alpha1',
        namespace: server.namespace,
        plural: 'mcpservers',
        name: server.name,
        body: {
          metadata: {
            annotations: {
              [NETWORK_READY_ANNOTATION]: 'true',
              // Issue #408: stamp the acked generation so WRC's waitForNetworkReady
              // can reject a stale ack from a previous generation.
              ...(typeof server.generation === 'number'
                ? { [NETWORK_READY_GENERATION_ANNOTATION]: String(server.generation) }
                : {}),
            },
          },
        },
      },
      {
        middleware: [k8s.setHeaderMiddleware('Content-Type', 'application/merge-patch+json')],
      }
    )
  }

  /**
   * G11: Update DeploymentReady on the McpServer status subresource.
   * The condition describes Deployment rollout. WRC does not watch this type;
   * it reads the `clerum.io/network-ready` annotation.
   */
  private async updateStatusConditions(
    server: McpServerCRD,
    deploymentReady: boolean,
    /**
     * Rollout numbers from readDeploymentRollout, appended to the failure
     * message so a stuck rollout reports WHAT is stuck (issue #223). Callers
     * that only know the verdict omit it.
     */
    rolloutDetail?: string,
    isCurrent: () => boolean = () => true,
    onDecline: StatusWriteDeclineArg = () => 'record-obligation'
  ): Promise<void> {
    try {
      const deploymentWrote = await this.writeStatusCondition(
        server,
        {
          type: 'DeploymentReady',
          status: deploymentReady ? 'True' : 'False',
          reason: deploymentReady
            ? 'ReplicasAvailable'
            : rolloutDetail
              ? 'RolloutIncomplete'
              : 'WaitingForReplicas',
          message: rolloutDetail
            ? `Rollout did not converge — ${rolloutDetail}`
            : deploymentReady
              ? 'Deployment has ready replicas'
              : 'Waiting for pods to become ready',
        },
        isCurrent,
        onDecline
      )
      if (deploymentWrote === 'wrote') {
        hccLogger.info('Updated status conditions', {
          name: server.name,
          deploymentReady,
        })
      }
    } catch (error) {
      // Status subresource may not exist yet — log but don't fail reconciliation
      hccLogger.warn('Failed to update status conditions', { name: server.name, err: error })
    }
  }

  /**
   * Write (or merge) a single status condition on the McpServer CRD.
   *
   * Merge semantics:
   *   - Condition with same `type` and unchanged `status` → keep existing
   *     `lastTransitionTime`, refresh `reason` / `message`.
   *   - `DeploymentReady` with the same `status` but a new `reason` or
   *     `message` → bump `lastTransitionTime` so recovery after
   *     `RuntimeNotDesired` is visible to #223 consumers.
   *   - Condition with same `type` but different `status` → bump
   *     `lastTransitionTime` to now.
   *   - New type → append with fresh `lastTransitionTime`.
   *   - Any write also strips retired types (NetworkReady). If a retired
   *     type is still present, the patch is forced even when the requested
   *     condition is otherwise unchanged.
   *
   * Side-effect: updates the `clerum_hcc_mcpserver_missing_secret` gauge for
   * `SecretResolved` conditions (1 when False, 0 when True).
   *
   * `onDecline` is a factory evaluated at decline time. The default records
   * an incomplete-publication obligation. Poll ticks pass a factory that is
   * `'silent'` only while `retiredNow()` is true.
   *
   * Returns `'wrote'` after a successful patch, `'unchanged'` when the
   * requested condition already matches and no retired type remains, or
   * `'declined'` when the fence or a write error stops the update.
   *
   * Failure modes:
   *   - 404 (CRD deleted mid-reconcile) → log + `decline()`.
   *   - 409/422 (optimistic conflict) → re-read, re-merge, bounded retry.
   *   - Other errors or exhausted retries → log + `decline()`.
   */
  async writeStatusCondition(
    server: McpServerCRD,
    condition: Omit<McpServerCondition, 'lastTransitionTime'>,
    isCurrent: () => boolean = () => true,
    onDecline: StatusWriteDeclineArg = () => 'record-obligation'
  ): Promise<StatusWriteOutcome> {
    const decline = (): StatusWriteOutcome =>
      onDecline() === 'silent' ? 'declined' : this.recordIncompletePublication(server)
    if (!isCurrent()) return decline()
    const now = new Date().toISOString()

    // Update the missing-secret gauge for SecretResolved conditions.
    // Do this before any no-op short-circuit so the gauge stays current
    // across reconciles even when we skip the API patch.
    if (condition.type === 'SecretResolved' && server.spec.envSecret) {
      const labels = {
        namespace: server.namespace,
        name: server.name,
        secret_name: server.spec.envSecret.name,
      }
      mcpserverMissingSecret.set(labels, condition.status === 'False' ? 1 : 0)
    }

    // Optimistic-concurrency loop (M2). The patch REPLACES the whole
    // /status/conditions array with a value built from a snapshot read just
    // before. Without a version guard, a concurrent writer — the readiness poll
    // and a reconcile can both write status for the same server, and the poll
    // timer is NOT serialized through `inFlight` — would be silently clobbered
    // by whichever patch lands second, dropping the other's condition. A `test`
    // op on metadata.resourceVersion makes the API server reject the patch (422)
    // if anything changed since our read; we then re-read and retry, so no
    // writer's condition is lost.
    for (let attempt = 1; attempt <= STATUS_CONDITION_WRITE_MAX_ATTEMPTS; attempt += 1) {
      if (!isCurrent()) return decline()
      // Re-read on every optimistic-conflict retry. Merging against the
      // latest resourceVersion preserves conditions written by peer
      // controllers between our read and patch.
      let existingConditions: McpServerCondition[] = []
      let hasStatusObject = false
      let resourceVersion: string | undefined
      try {
        const current = (await this.customApi.getNamespacedCustomObjectStatus({
          group: 'clerum.io',
          version: 'v1alpha1',
          namespace: server.namespace,
          plural: 'mcpservers',
          name: server.name,
        })) as {
          metadata?: { resourceVersion?: string }
          status?: { conditions?: McpServerCondition[] }
        }
        hasStatusObject = typeof current.status === 'object' && current.status !== null
        existingConditions = current.status?.conditions ?? []
        resourceVersion = current.metadata?.resourceVersion
      } catch (error) {
        if (getErrorCode(error) === 404) {
          console.warn(
            `[Reconciler] McpServer "${server.name}" deleted mid-reconcile — skipping status update`
          )
          return decline()
        }
        console.warn(
          `[Reconciler] Failed to read status for "${server.name}" — skipping status update for type "${condition.type}" to avoid clobbering other status fields:`,
          error
        )
        return decline()
      }
      if (!isCurrent()) return decline()

      const prior = existingConditions.find(c => c.type === condition.type)
      const observedGeneration = condition.observedGeneration ?? server.generation
      const unchanged =
        prior !== undefined &&
        prior.status === condition.status &&
        prior.reason === condition.reason &&
        prior.message === condition.message &&
        prior.observedGeneration === observedGeneration &&
        !existingConditions.some(existing => isRetiredStatusConditionType(existing.type))

      // Skip the patch when the condition is identical to what's already on
      // the CRD. Without this, every reconcile bumps the McpServer's
      // resourceVersion, fires another watch event, triggers another
      // reconcile — a tight self-loop that also amplifies NetworkPolicy
      // optimistic-lock contention downstream.
      if (unchanged) {
        return 'unchanged'
      }

      const refreshDeploymentReadyTime =
        condition.type === 'DeploymentReady' &&
        prior !== undefined &&
        (prior.reason !== condition.reason || prior.message !== condition.message)
      const lastTransitionTime =
        prior && prior.status === condition.status && !refreshDeploymentReadyTime
          ? prior.lastTransitionTime
          : now

      const merged: McpServerCondition = {
        type: condition.type,
        status: condition.status,
        reason: condition.reason,
        message: condition.message,
        lastTransitionTime,
        ...(observedGeneration !== undefined && { observedGeneration }),
      }

      const nextConditions = [
        ...existingConditions.filter(
          existing =>
            existing.type !== condition.type && !isRetiredStatusConditionType(existing.type)
        ),
        merged,
      ]

      const writePreconditions: Array<{
        op: 'test'
        path: string
        value: string | number
      }> = []
      if (server.uid !== undefined) {
        writePreconditions.push({
          op: 'test',
          path: '/metadata/uid',
          value: server.uid,
        })
      }
      if (server.generation !== undefined) {
        writePreconditions.push({
          op: 'test',
          path: '/metadata/generation',
          value: server.generation,
        })
      }
      if (resourceVersion !== undefined) {
        writePreconditions.push({
          op: 'test',
          path: '/metadata/resourceVersion',
          value: resourceVersion,
        })
      }
      const statusPatch = [
        ...writePreconditions,
        ...(hasStatusObject
          ? [{ op: 'add', path: '/status/conditions', value: nextConditions }]
          : [{ op: 'add', path: '/status', value: { conditions: nextConditions } }]),
      ]

      try {
        // The generated client prefers JSON Patch for this endpoint. Send an
        // actual patch document so status writes keep working across client
        // upgrades instead of relying on merge-patch object bodies.
        if (!isCurrent()) return decline()
        await this.customApi.patchNamespacedCustomObjectStatus({
          group: 'clerum.io',
          version: 'v1alpha1',
          namespace: server.namespace,
          plural: 'mcpservers',
          name: server.name,
          body: statusPatch,
        })
        return 'wrote'
      } catch (error) {
        const errorCode = getErrorCode(error)
        if (errorCode === 404) {
          console.warn(
            `[Reconciler] McpServer "${server.name}" deleted mid-reconcile — status patch skipped`
          )
          return decline()
        }
        // 409 (Conflict) or 422 (a `test` op failed) mean a concurrent writer
        // won the race. Re-read and retry rather than lose our condition. Any
        // other error, or the last attempt, falls through to the best-effort
        // log-and-swallow below.
        if (
          (errorCode === 409 || errorCode === 422) &&
          attempt < STATUS_CONDITION_WRITE_MAX_ATTEMPTS
        ) {
          continue
        }
        console.warn(
          `[Reconciler] Failed to write status condition ${condition.type}=${condition.status} on "${server.name}" (attempt ${attempt}/${STATUS_CONDITION_WRITE_MAX_ATTEMPTS}):`,
          error
        )
        return decline()
      }
    }
    return decline()
  }

  private recordIncompletePublication(server: McpServerCRD): 'declined' {
    // A declined or failed write replaces the obligation so an enclosing
    // reconcile cannot clear that failure on a later successful sibling write.
    this.pendingReconciliations.set(server.name, { namespace: server.namespace })
    return 'declined'
  }

  // ─── Public API ─────────────────────────────────────────────────────

  /**
   * Reconcile a created or modified McpServer.
   *
   * Concurrent calls for the same server name are serialized through
   * `inFlight` so we never run two reconciles in parallel for one server.
   * A prior failure does not block the next call; the chain absorbs it so
   * the next reconcile still runs.
   */
  async reconcile(server: McpServerCRD, options: McpServerMutationOptions = {}): Promise<void> {
    const capturedAuthority = this.inventoryAuthority()
    const callerIsCurrent = options.isCurrent ?? (() => true)
    const isCurrent = () => {
      if (!callerIsCurrent()) return false
      if (!this.inventoryAuthorityConfigured) return this.currentWhenUnconfigured()
      if (!this.authorityMatches(capturedAuthority)) return false
      const current = this.resolveCurrentServer?.(server.name)
      return current !== undefined && sameMcpServerDesiredRevision(server, current)
    }
    const key = server.name
    const obligation = { namespace: server.namespace }
    this.pendingReconciliations.set(key, obligation)
    const prev = this.inFlight.get(key) ?? Promise.resolve()
    const next = prev
      .catch(() => undefined)
      .then(async () => {
        if (!isCurrent()) return
        const completed = await this.performReconcile(server, isCurrent)
        if (completed && this.pendingReconciliations.get(key) === obligation) {
          this.pendingReconciliations.delete(key)
        }
      })
    this.inFlight.set(key, next)
    try {
      await next
    } finally {
      if (this.inFlight.get(key) === next) {
        this.inFlight.delete(key)
      }
    }
  }

  private async performReconcile(server: McpServerCRD, isCurrent: () => boolean): Promise<boolean> {
    if (!isCurrent()) return false
    this.beginStatusTracking(server)

    // G7: Enforce managed field immutability — once set, cannot change
    if (!isCurrent()) return false
    const currentManaged = server.spec.managed ?? true
    const previousManaged = this.managedSnapshot.get(server.name)
    if (previousManaged !== undefined && previousManaged !== currentManaged) {
      console.error(
        `[Reconciler] McpServer "${server.name}": managed field changed from ${previousManaged} to ${currentManaged}. ` +
          `This is not allowed — delete and recreate the McpServer to change ownership.`
      )
      if (!isCurrent()) return false
      this.setStatus(server.name, {
        deployed: false,
        ready: false,
        message: `managed field is immutable (was ${previousManaged}, attempted ${currentManaged})`,
      })
      return isCurrent()
    }
    if (!isCurrent()) return false
    this.managedSnapshot.set(server.name, currentManaged)

    // If disabled, respect ownership before any cleanup. managed:false means
    // WRC owns runtime resources; HCC only marks discovery status disabled.
    if (server.spec.enabled === false) {
      let cleanupComplete = true
      let retract: RetractManagedRuntimeResult | undefined
      if (currentManaged) {
        console.log(
          `[Reconciler] McpServer "${server.name}" is disabled — removing HCC-owned resources`
        )
        retract = await this.retractManagedRuntime(server, 'disabled', isCurrent)
        if (shouldStopAfterIncompleteRetract(retract, isCurrent)) return false
        cleanupComplete = retract.cleanupComplete
      } else {
        console.log(
          `[Reconciler] McpServer "${server.name}" is disabled but WRC-owned; skipping runtime cleanup`
        )
      }
      if (!isCurrent()) return false
      this.setStatus(server.name, { deployed: false, ready: false, message: 'Disabled' })
      await this.writeStatusCondition(
        server,
        {
          type: 'Ready',
          status: 'False',
          reason: 'Disabled',
          message: 'McpServer is disabled',
        },
        isCurrent
      )
      if (retract && !retract.retracted && !retract.foreignRuntime) return false
      return cleanupComplete && isCurrent()
    }

    if (!currentManaged) {
      return this.reconcileWrcOwnedServer(server, isCurrent)
    }

    // Validate secret before creating/updating
    const secretResult = await this.validateSecret(server)
    if (!isCurrent()) return false
    if (!secretResult.ok) {
      console.error(
        `[Reconciler] Skipping deployment of "${server.name}" — secret validation failed`
      )
      const withdrawRuntime = shouldFailClosedForSecretFailure(secretResult.reason)
      let cleanupComplete = true
      let retract: RetractManagedRuntimeResult | undefined
      if (withdrawRuntime) {
        retract = await this.retractManagedRuntime(server, 'fail-closed', isCurrent)
        if (shouldStopAfterIncompleteRetract(retract, isCurrent)) return false
        cleanupComplete = retract.cleanupComplete
      } else {
        console.warn(
          `[Reconciler] Preserving existing runtime for "${server.name}" after transient Secret read failure`
        )
      }
      if (!isCurrent()) return false
      this.setStatus(server.name, {
        deployed: false,
        ready: false,
        message: secretResult.message,
      })
      await this.writeStatusCondition(
        server,
        {
          type: 'SecretResolved',
          status: 'False',
          reason: secretResult.reason,
          message: secretResult.message,
        },
        isCurrent
      )
      await this.writeStatusCondition(
        server,
        {
          type: 'Ready',
          status: 'False',
          reason: 'SecretValidationFailed',
          message: secretResult.message,
        },
        isCurrent
      )
      if (retract && !retract.retracted && !retract.foreignRuntime) return false
      return withdrawRuntime && cleanupComplete && isCurrent()
    }

    // Ensure resources exist and are up to date
    try {
      // Remote egress proxy: create nginx ConfigMap before Deployment
      if (this.isRemote(server)) {
        // The CRD image remains desired-state input owned by its author. Remote
        // workloads use the platform-owned proxy image selected in
        // buildDeployment(); rewriting spec.image here would bump generation
        // and retire this reconcile before it can create the runtime resources.
        await this.ensureConfigMap(server, isCurrent)
      } else {
        // Phase 2.3: plugin image-host allowlist. Audit mode (default) logs a
        // would-be denial and still builds; enforce mode blocks the workload.
        const decision = classifyPluginImage(server.spec.image, {
          allowedPrefixes: config.allowedPluginImagePrefixes,
          rejectLatest: false,
        })
        if (!decision.ok) {
          const message = `Image "${server.spec.image}" is not permitted by the plugin image allowlist (${decision.reason})`
          if (config.enforcePluginImageAllowlist) {
            console.warn(`[Reconciler] ${message} — blocking "${server.name}" (enforce mode)`)
            await this.writeStatusCondition(
              server,
              {
                type: 'Ready',
                status: 'False',
                reason: 'ImageNotAllowed',
                message,
              },
              isCurrent
            )
            if (!isCurrent()) return false
            this.setStatus(server.name, { deployed: false, ready: false, message })
            return isCurrent()
          }
          console.warn(
            `[Reconciler] ${message} for "${server.name}" — audit mode, allowing (set CONTEXT_MAPPER_ENFORCE_IMAGE_ALLOWLIST=true to block)`
          )
        }
      }
      if (!isCurrent()) return false
      await this.ensureService(server, isCurrent)
      if (!isCurrent()) return false
      await this.ensureDeployment(server, secretResult.revision, isCurrent)
    } catch (err) {
      if (!isCurrent()) return false
      this.setStatus(server.name, {
        deployed: false,
        ready: false,
        message: `Resource sync failed: ${err instanceof Error ? err.message : String(err)}`,
      })
      return false
    }

    // G2: Complete pre-deploy handshake — set network-ready annotation
    // after Deployment + Service are created, signaling WRC that
    // NetworkPolicies are in place and workload can proceed.
    if (
      server.annotations?.[PRE_DEPLOY_ANNOTATION] === 'true' &&
      (server.annotations?.[NETWORK_READY_ANNOTATION] !== 'true' ||
        // Issue #408: re-ack when the stamped generation is stale (spec changed since
        // the last ack). A non-numeric generation keeps the legacy behavior.
        (typeof server.generation === 'number' &&
          server.annotations?.[NETWORK_READY_GENERATION_ANNOTATION] !== String(server.generation)))
    ) {
      try {
        await this.setNetworkReadyAnnotation(server, isCurrent)
      } catch (err) {
        if (!isCurrent()) return false
        this.setStatus(server.name, {
          deployed: false,
          ready: false,
          message: `Pre-deploy handshake failed: ${err instanceof Error ? err.message : String(err)}`,
        })
        return false
      }
    }

    // Check readiness after reconciliation
    const rollout = await this.readDeploymentRollout(server.name, server.namespace)
    if (!isCurrent()) return false
    const ready = rollout.ready
    this.setStatus(server.name, {
      deployed: true,
      ready,
      message: ready ? 'Running' : 'Deployment created, waiting for pods to become ready',
    })

    // G11: Update status conditions on McpServer CRD. If the readiness poll
    // already exhausted its budget for THIS Deployment generation, keep the
    // terminal RolloutIncomplete verdict sticky: the terminal status write
    // itself fires a MODIFIED watch event and thus this very code path, and
    // writing a bare "waiting" condition here would downgrade the verdict
    // milliseconds after it was published — the operator (and the E2
    // scenario) would never observe the rollout failure. A NEW rollout
    // (different generation) drops the stickiness and reports fresh.
    const pollState = this.readinessPolls.get(server.name)
    const exhaustedDetail =
      !ready &&
      pollState !== undefined &&
      pollState.exhausted &&
      rollout.generation !== null &&
      pollState.generation === rollout.generation
        ? pollState.detail
        : undefined
    await this.updateStatusConditions(server, ready, exhaustedDetail, isCurrent)

    // PR-B B1: record SecretResolved=True + Ready=True after a successful
    // reconcile. This is additive to updateStatusConditions (DeploymentReady);
    // the merge logic in writeStatusCondition preserves the existing
    // conditions by type.
    if (server.spec.envSecret) {
      await this.writeStatusCondition(
        server,
        {
          type: 'SecretResolved',
          status: 'True',
          reason: 'SecretFound',
          message: 'Secret resolved and validated',
        },
        isCurrent
      )
    }
    await this.writeStatusCondition(
      server,
      {
        type: 'Ready',
        status: ready ? 'True' : 'Unknown',
        reason: ready ? 'ReconcileSuccess' : 'WaitingForReplicas',
        message: ready
          ? 'Deployment created'
          : 'Deployment created, waiting for pods to become ready',
      },
      isCurrent
    )

    // If not ready yet, poll until it becomes ready. With generation-aware
    // readiness this is the normal path right after any Deployment write —
    // including a credential rotation — so the poll owns the terminal verdict
    // on the CRD.
    //
    // Post-terminal re-observation (intentional, self-terminating): after a
    // terminal RolloutIncomplete, the status write that recorded it fires a
    // MODIFIED watch → performReconcile → here again with !ready, arming ONE more
    // poll window. For a genuinely stuck deployment the terminal condition re-write
    // is byte-identical (the detail is a pure function of the replica tuple, no
    // timestamps), so writeStatusCondition's `unchanged` short-circuit skips the
    // patch → no watch event → no further re-arm. The only thing that re-arms again
    // is a deployment whose replica tuple perpetually OSCILLATES, and that is
    // bounded to one reconcile per poll window (~120s) — not a tight loop, and (per
    // pollReadiness's atomic check-then-set) not a timer leak.
    if (!ready && isCurrent()) {
      this.pollReadiness(server, 5000, 24, isCurrent)
    }
    return isCurrent()
  }

  private async reconcileWrcOwnedServer(
    server: McpServerCRD,
    isCurrent: () => boolean
  ): Promise<boolean> {
    if (!isCurrent()) return false
    console.log(
      `[Reconciler] McpServer "${server.name}" is WRC-owned (managed:false); ` +
        'skipping HCC runtime creation/deletion and updating discovery status.'
    )

    const secretResult = await this.validateSecret(server)
    if (!isCurrent()) return false
    if (!secretResult.ok) {
      this.setStatus(server.name, {
        deployed: true,
        ready: false,
        message: secretResult.message,
      })
      if (server.spec.envSecret) {
        await this.writeStatusCondition(
          server,
          {
            type: 'SecretResolved',
            status: 'False',
            reason: secretResult.reason,
            message: secretResult.message,
          },
          isCurrent
        )
      }
      await this.writeStatusCondition(
        server,
        {
          type: 'Ready',
          status: 'False',
          reason: 'SecretValidationFailed',
          message: secretResult.message,
        },
        isCurrent
      )
      return shouldFailClosedForSecretFailure(secretResult.reason) && isCurrent()
    }

    if (!isCurrent()) return false
    this.setStatus(server.name, {
      deployed: true,
      ready: true,
      message: 'WRC-owned runtime registered',
    })
    if (server.spec.envSecret) {
      await this.writeStatusCondition(
        server,
        {
          type: 'SecretResolved',
          status: 'True',
          reason: 'SecretFound',
          message: 'Secret resolved and validated',
        },
        isCurrent
      )
    }
    await this.writeStatusCondition(
      server,
      {
        type: 'Ready',
        status: 'True',
        reason: WRC_OWNED_RUNTIME_READY_REASON,
        message: 'WRC-owned runtime registered for HCC discovery',
      },
      isCurrent
    )
    return isCurrent()
  }

  /**
   * Reconcile a deleted McpServer.
   */
  async reconcileDelete(name: string, namespace: string): Promise<void> {
    console.log(`[Reconciler] McpServer "${name}" deleted — cleaning up resources`)
    const capturedAuthority = this.inventoryAuthority()
    const obligation = { namespace }
    this.pendingReconciliations.set(name, obligation)
    const complete = await this.deleteRuntimeResources(name, namespace, () =>
      this.orphanDeleteAllowed(name, namespace, capturedAuthority)
    )
    if (complete && this.pendingReconciliations.get(name) === obligation) {
      this.pendingReconciliations.delete(name)
      this.clearStatus(name)
    }
  }

  private authorityMatches(captured: McpServerInventoryAuthoritySnapshot): boolean {
    const current = this.inventoryAuthority()
    return captured.known && current.known && captured.generation === current.generation
  }

  /**
   * A cache miss is not proof of absence: the API watch may have accepted an
   * ADDED event that has not reached this process yet. Confirm a 404 from the
   * API and then recheck both cache membership and LIST→WATCH authority
   * immediately before every destructive delete.
   */
  private async orphanDeleteAllowed(
    name: string,
    namespace: string,
    capturedAuthority: McpServerInventoryAuthoritySnapshot
  ): Promise<boolean> {
    return confirmAuthoritativeMcpServerAbsence({
      inventoryAuthoritative: () => this.authorityMatches(capturedAuthority),
      resolveCurrent: () => this.resolveCurrentServer?.(name),
      readCurrent: () =>
        this.customApi.getNamespacedCustomObject({
          group: 'clerum.io',
          version: 'v1alpha1',
          namespace,
          plural: 'mcpservers',
          name,
        }),
    })
  }

  /**
   * Full reconciliation pass — sync desired state with actual state.
   * Called on startup to handle CRDs that were created while the skill-mapper was down.
   */
  async fullReconcile(
    desiredServers: McpServerCRD[],
    options: McpServerFullReconcileOptions = {}
  ): Promise<void> {
    console.log(
      `[Reconciler] Running full reconciliation for ${desiredServers.length} McpServer(s)`
    )
    const capturedAuthority = this.inventoryAuthority()

    // Get all deployments managed by skill-mapper
    const existingDeployments = await this.listManagedDeployments()
    const desiredNames = new Set(
      desiredServers
        .filter(s => s.spec.enabled !== false && (s.spec.managed ?? true))
        .map(s => s.name)
    )

    const runEffect =
      options.runEffect ?? ((_serverName: string, work: () => Promise<void>) => work())
    const maxConcurrency = options.maxConcurrency ?? DEFAULT_FULL_RECONCILE_MAX_CONCURRENCY
    const inScope = (name: string) => options.scope === undefined || options.scope.has(name)

    // Create or update desired servers independently. runEffect owns each
    // per-server lane, so a slow fleet member cannot become a process-wide
    // startup barrier. The finite worker pool admits no more than the configured
    // number of effects at once, while still waiting for every desired result
    // before reporting aggregated failures or moving to orphan cleanup.
    const scopedServers = desiredServers.filter(server => inScope(server.name))
    const desiredResults = await mapSettledWithConcurrency(scopedServers, maxConcurrency, server =>
      runEffect(server.name, async () => {
        const current = this.resolveCurrentServer ? this.resolveCurrentServer(server.name) : server
        // Cache mutation happens synchronously before its keyed watch effect is
        // queued. If this startup item was superseded, skip it and let the
        // newer queued event perform external-egress-before-runtime using its
        // own spec; reconciling the newer runtime here could otherwise race
        // ahead of that safety policy.
        if (!current || !sameMcpServerDesiredRevision(server, current)) return
        const isCurrent = () => {
          if (!this.inventoryAuthorityConfigured) {
            if (!this.currentWhenUnconfigured()) return false
          } else if (!this.authorityMatches(capturedAuthority)) {
            return false
          }
          const latest = this.resolveCurrentServer?.(current.name)
          return latest
            ? sameMcpServerDesiredRevision(current, latest)
            : this.resolveCurrentServer === undefined && this.currentWhenUnconfigured()
        }
        await this.reconcile(current, { isCurrent })
      })
    )
    throwCleanupFailures(
      desiredResults
        .filter((result): result is PromiseRejectedResult => result.status === 'rejected')
        .map(result => result.reason),
      'Failed to reconcile desired McpServers'
    )

    // Delete orphaned deployments (managed by us but no longer desired)
    const cleanupTargets = new Map(
      existingDeployments.map(deployment => [
        deployment.metadata?.name || '',
        deployment.metadata?.namespace || config.namespace,
      ])
    )
    const presentNames = new Set(desiredServers.map(server => server.name))
    for (const [name, pending] of this.pendingReconciliations) {
      if (!presentNames.has(name)) cleanupTargets.set(name, pending.namespace)
    }
    for (const [name, namespace] of cleanupTargets) {
      if (!desiredNames.has(name) && inScope(name)) {
        if (!presentNames.has(name)) this.pendingReconciliations.set(name, { namespace })
        await runEffect(name, async () => {
          if (!(await this.orphanDeleteAllowed(name, namespace, capturedAuthority))) {
            console.warn(
              `[Reconciler] Deferring orphan cleanup for "${name}": McpServer exists or inventory authority is unknown or changed`
            )
            return
          }
          console.log(`[Reconciler] Removing orphaned resources for "${name}"`)
          await this.reconcileDelete(name, namespace)
        })
      }
    }

    console.log('[Reconciler] Full reconciliation complete')
  }

  /**
   * List all Deployments managed by skill-mapper (by label selector).
   */
  private async listManagedDeployments(): Promise<k8s.V1Deployment[]> {
    try {
      const response = await this.appsApi.listNamespacedDeployment({
        namespace: config.namespace,
        labelSelector: `${MANAGED_BY_LABEL}=${MANAGED_BY_VALUE}`,
      })
      return response.items || []
    } catch (error) {
      console.error('[Reconciler] Failed to list managed deployments:', error)
      throw error
    }
  }
}