Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript
254 KB
Raw Blame
import * as k8s from '@kubernetes/client-node'
import { IntOrString } from '@kubernetes/client-node/dist/types.js'
import { createHash, randomUUID } from 'crypto'
import { isDeepStrictEqual } from 'node:util'
import * as path from 'path'
import type { AdministrativeOutcomeReporter } from './administrativeOutcomeReporter'
import {
  type CodexCatalogSnapshot,
  type CodexExecutionProjection,
  GROK_EXECUTE_SCOPE,
  assignedHostCodexConnectionRef,
  projectCodexExecution,
  projectGrokExecution,
} from './codexExecutionProjection'
import { config } from './config'
import { HOST_LABEL, MANAGED_BY_LABEL, MANAGED_BY_VALUE } from './constants'
import { mintHostGfsToken } from './gfsHostBinding'
import { GFS_HOST_SCOPES } from './gfsHostPolicy'
import { makeExpectedHostGfsSubject } from './gfsHostSubject'
import type {
  HccInfrastructureTelemetryPayload,
  HostLookupReference,
  InfrastructureTelemetryReporter,
} from './infrastructureTelemetryReporter'
import {
  HOST_K8S_REQUEST_TIMEOUT_CODE,
  HostK8sRequestTimeoutError,
  makeHostK8sApiClient,
} from './k8s/hostK8sApiClient'
import {
  SFS_LABEL,
  SFS_NAMESPACE_LABEL,
  WFC_APP_LABEL,
  sharedFileSystemHash,
} from './k8s/sharedFileSystemFactory'
import { kubernetesQuantitiesEqual } from './kubernetesQuantity'
import {
  ALLOWED_MODELS_CONFIGMAP_NAME,
  parseAllowedModelsSnapshot,
  parseGrokAllowedModelsSnapshot,
  snapshotForAssignedCodexGrant,
  snapshotFromConfigMapError,
} from './llmAllowedModelsSnapshot'
import { hccLogger } from './logger'
import { issueMcpHostRuntimeTokens } from './mcpHostRuntimeTokenIssuerClient'
import {
  createsTotal,
  hostCleanupDeferredTotal,
  hostDeleteCleanupTotal,
  hostFleetBenignSupersessionsTotal,
  hostReconcileDurationSeconds,
  hostReconcileInFlight,
  hostReconcileQueueWaitSeconds,
} from './metrics'
import {
  MCP_HOST_GFS_TOKEN_SECRET_KEY,
  MCP_HOST_RUNTIME_TOKEN_SECRET_ACCESS_KEY,
  MCP_HOST_RUNTIME_TOKEN_SECRET_CONTROL_KEY,
  MCP_HOST_RUNTIME_TOKEN_SECRET_REFRESH_KEY,
  buildMcpHostRuntimeTokenSecret,
  mcpHostRuntimeTokenSecretName,
} from './secretFactory'
import {
  buildWorkspaceLayoutMergeContainer,
  decodeJwtExpMs,
  effectiveRotateBeforeMs,
  resolveStatelessImagePullPolicy,
} from './statelessDeployment'
import { EffectiveHostLifecycle, SuspendFromHeartbeatOutcome } from './statelessLifecycle.types'
import { ReflectHostOutcomeFn, StatelessLifecycleExecutor } from './statelessLifecycleExecutor'
import {
  CommunicationChannelCRD,
  EffectiveMcpHostControlScope,
  HostCRD,
  HostChannelReaderStatus,
  HostRuntimeControlScope,
  HostRuntimeStatus,
  HostWorkflowControlScope,
  HostWorkflowControlSpec,
} from './types'
import {
  type ResourceApplyResult,
  applyNetworkPolicy,
  canonicalStringify,
  canonicalizeValue,
  deploymentMatchesDesired,
  ensureResource,
  getErrorCode,
  observeCreate,
  observeExistenceRead,
  preserveDeploymentAnnotations,
  preserveServiceAssignedFields,
  replaceWithConflictRetry,
  serviceMatchesDesired,
} from './utils'

export type { EffectiveHostLifecycle } from './statelessLifecycle.types'

const HOST_GROUP = 'clerum.io'
const HOST_VERSION = 'v1alpha1'
const HOST_PLURAL = 'hosts'

/**
 * Lane label for a Host reconcile, used only for low-cardinality telemetry:
 * `urgent` (a direct watch ADDED/MODIFIED/DELETED/wake event), `retry` (a
 * bounded watch-reconcile retry), or `fleet` (a bounded background full/
 * lifecycle worker). Never a host name, user, or team.
 */
export type HostReconcileSource = 'urgent' | 'retry' | 'fleet'

/** Immutable Host watch authority snapshot captured at a full-pass boundary. */
export type HostWatchAuthoritySnapshot = { known: boolean; generation: number }

/**
 * Immutable lease for every inventory that contributes to a Host mutation.
 * Host resources come from the Host watch; mounted SharedFileSystems come from
 * the Context watch. Both are CONTENT revisions (they move only on a real
 * desired-state change), not channel generations — a watch reconnect that
 * re-LISTs the identical inventory must not retire an in-flight Host lease, or
 * sustained watch-churn starves every queued reconcile. Keeping both revisions
 * still prevents a false-stable Host-only lease when Context desired state
 * changes during queued work.
 */
export type HostMutationAuthoritySnapshot = {
  known: boolean
  hostRevision: number
  contextRevision: number
}

class HostInventoryAuthorityUnavailableError extends Error {
  constructor(
    action: string,
    captured: HostMutationAuthoritySnapshot,
    current: HostMutationAuthoritySnapshot
  ) {
    super(
      `Host inventory authority changed before ${action} admission ` +
        `(captured known=${captured.known} hostRevision=${captured.hostRevision} ` +
        `contextRevision=${captured.contextRevision}, current known=${current.known} ` +
        `hostRevision=${current.hostRevision} contextRevision=${current.contextRevision})`
    )
    this.name = 'HostInventoryAuthorityUnavailableError'
  }
}

class HostMutationIdentityChangedError extends Error {
  constructor(action: string, hostName: string) {
    super(`Host identity changed before ${action} admission for "${hostName}"`)
    this.name = 'HostMutationIdentityChangedError'
  }
}

class RuntimeBootstrapPreparationUnavailableError extends Error {
  constructor() {
    super('Waiting for a prepared runtime bootstrap before changing resources')
    this.name = 'RuntimeBootstrapPreparationUnavailableError'
  }
}

class HostMutationSpecRevisionChangedError extends Error {
  constructor(action: string, hostName: string) {
    super(`Host spec generation changed before ${action} admission for "${hostName}"`)
    this.name = 'HostMutationSpecRevisionChangedError'
  }
}

class HostMutationDependencyChangedError extends Error {
  constructor(action: string, hostName: string) {
    super(`Host mutation dependency changed before ${action} admission for "${hostName}"`)
    this.name = 'HostMutationDependencyChangedError'
  }
}

/**
 * The four benign mid-pass supersession signals. revalidateHostMutationBoundary()
 * throws these when the watch observes a newer spec/uid/authority-generation while
 * a reconcile is in flight; a fresh reconcile runs afterwards, so the retired pass
 * is a WITHDRAWAL, not a hard failure. Matched by `error.name` (string), NOT
 * `instanceof`: the classes are unexported and statelessLifecycleExecutor.ts
 * synthesizes name-equivalent plain Errors for the same races.
 */
const BENIGN_SUPERSESSION_ERROR_NAMES: ReadonlySet<string> = new Set([
  'HostInventoryAuthorityUnavailableError',
  'HostMutationIdentityChangedError',
  'HostMutationSpecRevisionChangedError',
  'HostMutationDependencyChangedError',
])

function isBenignSupersessionError(error: unknown): error is Error {
  return error instanceof Error && BENIGN_SUPERSESSION_ERROR_NAMES.has(error.name)
}

/**
 * The exact §13.2 destructive-cleanup predicate. Deletion of an orphan Host's
 * owned bundle is permitted ONLY when watch authority is known, the watch
 * generation captured at pass start still matches the current generation, the
 * current cache omits the Host, AND a fresh authoritative read confirmed a 404.
 * Any weaker evidence defers.
 */
export function destructiveCleanupAllowed(input: {
  watchAuthorityKnown: boolean
  capturedWatchGeneration: number
  currentWatchGeneration: number
  currentCacheOmitsHost: boolean
  freshAuthoritativeRead: 'confirmed404' | 'present' | 'error'
}): boolean {
  return (
    input.watchAuthorityKnown &&
    input.capturedWatchGeneration === input.currentWatchGeneration &&
    input.currentCacheOmitsHost &&
    input.freshAuthoritativeRead === 'confirmed404'
  )
}

/**
 * Dependency-free bounded worker pool. Dispatches `keys` across at most
 * `concurrency` lanes; each lane pulls the next key, runs `worker`, and
 * collects (never swallows) failures so aggregation stays complete. Lane
 * startup is ordered so the first `concurrency` keys begin in input order.
 */
async function runBoundedHostWorkers(
  keys: string[],
  concurrency: number,
  worker: (key: string) => Promise<void>
): Promise<unknown[]> {
  const failures: unknown[] = []
  if (keys.length === 0) return failures
  let cursor = 0
  const runLane = async (): Promise<void> => {
    for (;;) {
      const index = cursor
      cursor += 1
      if (index >= keys.length) return
      try {
        await worker(keys[index])
      } catch (error) {
        failures.push(error)
      }
    }
  }
  const lanes = Math.max(1, Math.min(concurrency, keys.length))
  const runners: Promise<void>[] = []
  for (let i = 0; i < lanes; i += 1) runners.push(runLane())
  await Promise.all(runners)
  return failures
}

/** A managed resource considered for orphan cleanup, keyed by owning Host. */
type CleanupCandidate = { owner: string | undefined; kind: string; name: string }

/**
 * The workflow-control scopes a first-party Host's mcp-host control token needs
 * for the full channel-workflow surface (list/read/trigger + the Telegram/Slack
 * provider-identity resolve + approval decide). Matches the canonical Host
 * overlay instances (deploy/overlays/<env>/instances/host.yaml).
 */
export const DEFAULT_FIRST_PARTY_WORKFLOW_CONTROL_SCOPES: HostWorkflowControlScope[] = [
  'workflow:list',
  'workflow:read',
  'workflow:trigger',
  'workflow:approval:resolve',
  'workflow:approval:decide',
]

/**
 * Resolve the workflow-control scopes declared by a Host.
 *
 * Existing channel-bearing Hosts historically omitted workflowControl while
 * still needing the first-party workflow approval surface. Preserve that
 * server-side default only when the entire block is absent and channel ingress
 * is present. An explicit `scopes` array is honored exactly as declared,
 * including `[]` as an intentional opt-out from workflow access.
 */
export function resolveWorkflowControlScopes(
  workflowControl: HostWorkflowControlSpec | undefined,
  options: { hasChannelIngress?: boolean } = {}
): HostWorkflowControlScope[] {
  if (workflowControl?.scopes !== undefined) {
    return [...workflowControl.scopes]
  }
  if (!workflowControl && options.hasChannelIngress === true) {
    return [...DEFAULT_FIRST_PARTY_WORKFLOW_CONTROL_SCOPES]
  }
  return []
}

/**
 * The scope HCC appends for a Host runtime that fronts an enabled
 * `auth.type: oauth` mcp-server. control-api's OAuth user-token broker
 * (`POST /api/v1/mcp-oauth/user-token`) requires the caller's mcp-host control
 * JWT to carry this scope; it is derive-only (never user-declarable) — see
 * {@link HostRuntimeControlScope}.
 */
export const OAUTH_USER_TOKEN_SCOPE = 'oauth:user-token' as const

/**
 * Resolve the EFFECTIVE runtime control scopes minted into the mcp-host control
 * JWT: the declarable workflow-control scopes plus the derive-only
 * `oauth:user-token` when the Host fronts an enabled oauth mcp-server.
 *
 * This is the SINGLE source of truth shared by BOTH the mint path
 * (`resolveWorkflowControlScopesForHost`) and the drift hash
 * (`runtimeTokenScopeHash`), so the scopes carried by the token and the scopes
 * hashed for change-detection can never diverge.
 */
export function resolveRuntimeControlScopes(
  workflowControl: HostWorkflowControlSpec | undefined,
  options: { hasChannelIngress?: boolean; frontsOAuthServer?: boolean } = {}
): HostRuntimeControlScope[] {
  const scopes: HostRuntimeControlScope[] = [
    ...resolveWorkflowControlScopes(workflowControl, {
      hasChannelIngress: options.hasChannelIngress,
    }),
  ]
  if (options.frontsOAuthServer === true) {
    scopes.push(OAUTH_USER_TOKEN_SCOPE)
  }
  return scopes
}

/**
 * One SharedFileSystem mount injected into a Host's mcp-host Deployment.
 * The PVC is mounted RO at the requested path; mcp-host's built-in
 * clerum__context_files_* tools read CLERUM_CONTEXT_FILES_MOUNTS to learn
 * what's mounted where.
 */
export interface ResolvedSfsMount {
  /** SharedFileSystem.metadata.name (also the JWT `sharedFileSystem` claim). */
  name: string
  /** SharedFileSystem.metadata.namespace — always mcp-host in v1. */
  namespace: string
  /** PVC name that the per-SFS wfc and the Host both mount. */
  pvcName: string
  /** Path inside the mcp-host container where the PVC is mounted (RO). */
  mountPath: string
}

export type ResolveContextMountsFn = (host: HostCRD) => Promise<ResolvedSfsMount[]>

/**
 * Resolve whether a Host's referenced Context fronts at least one ENABLED
 * `auth.type: oauth` mcp-server. Wired in production from the McpServerWatcher
 * (which owns the Context + McpServer caches) so HCC can decide whether to
 * request the derive-only `oauth:user-token` scope without HostReconciler
 * needing direct K8s access to non-Host CRDs. Default (unwired) returns false.
 */
export type HostFrontsOAuthServerFn = (host: HostCRD) => Promise<boolean>

const CONTEXT_LABEL = 'clerum.io/context'
const CONTEXT_MOUNT_PATH_PATTERN = /^\/[a-zA-Z0-9_.][a-zA-Z0-9_.\/-]*$/
const RUNTIME_TOKEN_REVISION_ANNOTATION = 'clerum.io/runtime-token-revision'
// Rolls the mcp-host pod when Host.spec.guardrails changes. mcp-host re-resolves
// the guardrails block (installed-hook refs, built-ins, limits) live off its Host
// watch, but a watch that has lapsed leaves the running agent on the block it last
// saw; the roll is the delivery path that does not depend on that watch. Stamping
// a hash of the guardrails spec onto the pod template makes any change flip the
// template → rolling restart → mcp-host re-reads guardrails at boot. Removal is
// handled by preserveHostDeploymentAnnotations, since an omitted key would
// otherwise survive the annotation merge and produce no diff.
// Mirrors the runtime-token / credentials revision annotations.
const GUARDRAILS_REVISION_ANNOTATION = 'clerum.io/guardrails-revision'

/**
 * Deep, key-sorted JSON for a stable content hash. Object keys are sorted at
 * EVERY level — unlike `canonicalStringify`, which sorts only the top level —
 * so a guardrails block that differs only in nested key order (e.g. the API
 * returning `hooks` phases or a hook ref's fields in a different order) hashes
 * identically and does not roll mcp-host for nothing. Array order is PRESERVED:
 * hook precedence within a phase is semantic, so reordering it is a real change.
 */
function deepStableStringify(value: unknown): string {
  const norm = (v: unknown): unknown => {
    if (Array.isArray(v)) return v.map(norm)
    if (v && typeof v === 'object') {
      const out: Record<string, unknown> = {}
      for (const k of Object.keys(v as Record<string, unknown>).sort()) {
        out[k] = norm((v as Record<string, unknown>)[k])
      }
      return out
    }
    return v
  }
  return JSON.stringify(norm(value))
}
const RUNTIME_TOKEN_SECRET_REVISION_ANNOTATION = 'clerum.io/runtime-token-secret-revision'
const RUNTIME_TOKEN_ISSUED_AT_ANNOTATION = 'clerum.io/runtime-token-issued-at'
const RUNTIME_TOKEN_REFRESH_EXPIRES_AT_ANNOTATION = 'clerum.io/runtime-token-refresh-expires-at'
const RUNTIME_TOKEN_REFRESH_BEFORE_ANNOTATION = 'clerum.io/runtime-token-refresh-before'
const GFS_TOKEN_EXPIRES_AT_ANNOTATION = 'clerum.io/gfs-token-expires-at'
const GFS_TOKEN_REFRESH_BEFORE_ANNOTATION = 'clerum.io/gfs-token-refresh-before'
const GFS_TOKEN_EXPECTED_SUBJECT_ANNOTATION = 'clerum.io/gfs-token-expected-subject'
const GFS_TOKEN_CAPABILITY_SET_HASH_ANNOTATION = 'clerum.io/gfs-token-capability-set-hash'
const GFS_TOKEN_HOST_UID_ANNOTATION = 'clerum.io/gfs-token-host-uid'
const HOST_UID_ANNOTATION = 'clerum.io/host-uid'
const GFS_TOKEN_HOST_GENERATION_ANNOTATION = 'clerum.io/gfs-token-host-generation'
const RUNTIME_TOKEN_HOST_BINDING_HASH_ANNOTATION = 'clerum.io/runtime-token-host-binding-hash'
const RUNTIME_TOKEN_SCOPE_HASH_ANNOTATION = 'clerum.io/runtime-token-scope-hash'
const RUNTIME_TOKEN_ISSUER_ANNOTATION = 'clerum.io/runtime-token-issuer'
const RUNTIME_TOKEN_AUDIENCE_ANNOTATION = 'clerum.io/runtime-token-audience'
const RUNTIME_TOKEN_SCHEMA_VERSION_ANNOTATION = 'clerum.io/runtime-token-schema-version'
const RUNTIME_TOKEN_BOOTSTRAP_STATE_ANNOTATION = 'clerum.io/runtime-token-bootstrap-state'
const RUNTIME_TOKEN_ROLLOUT_REQUIRED_ANNOTATION = 'clerum.io/runtime-token-rollout-required'
const RUNTIME_TOKEN_BOOTSTRAP_DEPLOYMENT_UID_ANNOTATION =
  'clerum.io/runtime-token-bootstrap-deployment-uid'
const RUNTIME_TOKEN_BOOTSTRAP_APPLIED_REVISION_ANNOTATION =
  'clerum.io/runtime-token-bootstrap-applied-revision'
const RUNTIME_TOKEN_BOOTSTRAP_READY_REVISION_ANNOTATION =
  'clerum.io/runtime-token-bootstrap-ready-revision'
const RUNTIME_TOKEN_HAS_CHANNEL_INGRESS_ANNOTATION = 'clerum.io/runtime-token-has-channel-ingress'
const RUNTIME_TOKEN_FRONTS_OAUTH_ANNOTATION = 'clerum.io/runtime-token-fronts-oauth-server'
const RUNTIME_TOKEN_ISSUER = 'control-api'
const RUNTIME_TOKEN_AUDIENCE = 'host-context-controller,workflow-approvals'
// v3 extends the existing first-party access/refresh token material with the
// exact HCC audience, immutable MCP credential capability, and live Host UID.
// Existing Secrets therefore fail the contract check and rotate onto the
// caller-bound contract instead of remaining workflow-only.
const RUNTIME_TOKEN_SCHEMA_VERSION = '3'
const RUNTIME_TOKEN_BOOTSTRAP_STATE_FRESH = 'fresh'
const RUNTIME_TOKEN_BOOTSTRAP_STATE_CONSUMED = 'consumed'
// Deployments affected by the historical stringData/data hashing bug can carry
// the empty-payload revision. Roll them once so the pod template converges to
// the normalized persisted Secret revision.
const LEGACY_EMPTY_RUNTIME_TOKEN_REVISION = createHash('sha256')
  .update(JSON.stringify([]))
  .digest('hex')
const WORKFLOW_TOKEN_MOUNT_PATH = '/var/run/clerum/workflow-tokens'
const MCP_HOST_WORKFLOW_CONTROL_TOKEN_FILE_PATH = `${WORKFLOW_TOKEN_MOUNT_PATH}/${MCP_HOST_RUNTIME_TOKEN_SECRET_CONTROL_KEY}`
const MCP_HOST_RUNTIME_AUTH_STATE_PATH = '/var/run/clerum/workflow-auth'
/**
 * Restricted-PSA-compliant container securityContext shared by the mcp-host
 * container and the workspace-layout-merge initContainer. MCC shared-tenant
 * namespaces (mcp-host-<slug>) enforce the `restricted` PodSecurity standard;
 * without these fields the ReplicaSet gets FailedCreate and never schedules a
 * pod. UID/GID 1001 = the image's baked-in `nodejs` user; it must match the
 * pod-level securityContext and the existing workspace PVC ownership so
 * re-rolls don't EACCES. readOnlyRootFilesystem is intentionally NOT set —
 * mcp-host writes outside its mounted /tmp + workspace volumes and restricted
 * PSA does not require it.
 */
function hostContainerSecurityContext(): k8s.V1SecurityContext {
  return {
    allowPrivilegeEscalation: false,
    runAsNonRoot: true,
    runAsUser: 1001,
    runAsGroup: 1001,
    capabilities: { drop: ['ALL'] },
    seccompProfile: { type: 'RuntimeDefault' },
  }
}
const log = hccLogger.child({ module: 'host-reconciler' })

function throwCleanupFailures(failures: unknown[], message: string): void {
  if (failures.length > 0) {
    throw new AggregateError(failures, message)
  }
}

export class HostFleetReconcileError extends AggregateError {
  constructor(
    readonly hostFailures: unknown[],
    readonly cleanupFailures: unknown[],
    message = 'Host fleet reconciliation completed with errors'
  ) {
    super([...hostFailures, ...cleanupFailures], message)
    this.name = 'HostFleetReconcileError'
  }
}

type HostReconcilerDeps = {
  appsApi?: k8s.AppsV1Api
  coreApi?: k8s.CoreV1Api
  networkingApi?: k8s.NetworkingV1Api
  rbacApi?: k8s.RbacAuthorizationV1Api
  customApi?: k8s.CustomObjectsApi
  /** Injectable clock so tests get deterministic condition lastTransitionTime. */
  now?: () => Date
  /** Stable per-enqueue identity; the reporter reuses it for transport retries. */
  newTelemetryOccurrenceId?: () => string
  /**
   * Resolve which SharedFileSystem PVCs to mount into a Host's mcp-host
   * Deployment. The default (no override) returns []. Production wires this
   * up from the McpServerWatcher in k8sClient.ts so it can read the Context
   * referenced by the Host without HostReconciler needing direct K8s access
   * to non-Host CRDs.
   */
  resolveContextMounts?: ResolveContextMountsFn
  /**
   * Resolve whether the Host fronts an enabled `auth.type: oauth` mcp-server,
   * gating the derive-only `oauth:user-token` runtime scope. Wired in
   * production by `McpServerWatcher` over `getServerInfosByContext`. Default
   * (no override) returns false — fail closed: no oauth scope until wired.
   */
  hostFrontsOAuthServer?: HostFrontsOAuthServerFn
  /**
   * Count CommunicationChannels referencing this Host. Used by
   * `buildChannelReaderDeployment` to compute `spec.replicas`
   * (0 when no CCs, 1 when 1+). Wired by `McpServerWatcher` from
   * its `communicationChannels` cache. Default returns 0 — Hosts
   * without a wired counter scale to replicas=0.
   */
  countCommunicationChannels?: (hostName: string) => number
  /**
   * Find every CommunicationChannel whose `spec.hostRef` equals the given
   * host name. Used by `patchChannelReaderRevisionAnnotation` to discover
   * which Secret(s) a host depends on. Wired by `McpServerWatcher` from
   * its `communicationChannels` cache.
   */
  findCommunicationChannelsByHostRef?: (host: string) => CommunicationChannelCRD[]
  /**
   * Find every CommunicationChannel whose `spec.credentialsSecretRef.name`
   * equals the given Secret name. Used by the Secret informer (Task 9) to
   * route rotation events to the affected hosts. Wired by
   * `McpServerWatcher` from its `communicationChannels` cache.
   */
  findCommunicationChannelsByCredentialsSecretName?: (name: string) => CommunicationChannelCRD[]
  /**
   * B2: Returns true once the McpServerWatcher has completed its CC
   * initial-list startup block. HostReconciler uses this to avoid
   * scaling an existing channel-reader Deployment to 0 when the cache
   * is still empty (e.g. during startup or after a CC-load failure).
   * Default returns false — safe: preserves existing replicas until wired.
   */
  isCommunicationChannelCacheSynced?: () => boolean
  infrastructureTelemetryReporter?: InfrastructureTelemetryReporter
  administrativeOutcomeReporter?: AdministrativeOutcomeReporter
}

type HostSecretValidationResult =
  | { ok: true }
  | { ok: false; reason: 'SecretNotFound' | 'SecretAccessDenied' | 'ReadError'; message: string }

type BootstrapOptions = {
  forceFreshForWake?: boolean
  forceFreshForDeploymentChange?: boolean
  targetSuspended?: boolean
  prepareForResourceChange?: boolean
  refreshGfsOnly?: boolean
  preserveDeploymentTemplateOnWake?: boolean
}

type RuntimeTokenProvision = {
  revision: string
  scopeHash: string
  deploymentUid?: string
  freshBootstrapPrepared?: boolean
}

// Reported as the reconcile_outcome transition `gfs_token:<outcome>` (#328).
type GfsTokenLifecycleOutcome = 'minted' | 'rotated' | 'reused' | 'failed'

type DeploymentMutationState = {
  lifecycle: EffectiveHostLifecycle
  runtimeTokenRevision: string
  freshBootstrapPrepared?: boolean
  // Captured in the same synchronous step as the stable scope-hash check so
  // the pod's Grok factory switch and the minted llm:grok:execute scope agree.
  grokExecutionEnabled: boolean
}

export class HostReconciler {
  private readonly appsApi: k8s.AppsV1Api
  private readonly coreApi: k8s.CoreV1Api
  private readonly networkingApi: k8s.NetworkingV1Api
  private readonly rbacApi: k8s.RbacAuthorizationV1Api
  /**
   * Lazily constructed (see the `customApi` getter): only lifecycle-enabled
   * Hosts touch the CustomObjects API, and existing tests construct the
   * reconciler with a bare KubeConfig stub that cannot makeApiClient.
   */
  private customApiInstance: k8s.CustomObjectsApi | undefined
  private readonly kubeConfig: k8s.KubeConfig
  private readonly now: () => Date
  private readonly newTelemetryOccurrenceId: () => string
  private readonly statusMap: Map<string, HostRuntimeStatus> = new Map()
  private codexSnapshot: CodexCatalogSnapshot = { flagEnabled: false }
  private lastCodexConfigMap: k8s.V1ConfigMap | undefined
  private readonly readinessTimers: Map<string, ReturnType<typeof setTimeout>> = new Map()
  private readonly reconcileGenerations = new Map<string, number>()
  private readonly statusGenerations = new Map<string, number>()
  /**
   * host.name → image whose pull-policy refusal was already error-logged.
   * The periodic resync rebuilds the Deployment forever; the operator needs
   * the error once per distinct misconfigured image (the durable condition
   * carries the long-lived signal).
   */
  private readonly pullPolicyRejectionLogged: Map<string, string> = new Map()
  /**
   * Stateless lifecycle execution (Stages 2–6), extracted to
   * StatelessLifecycleExecutor: reconcile-time assessment + durable status
   * writes, the heartbeat suspend/drain executors, the wake fast-path, and
   * the per-host reconcile serialization chain.
   */
  private readonly lifecycle: StatelessLifecycleExecutor
  private resolveContextMounts: ResolveContextMountsFn
  private hostFrontsOAuthServerFn: HostFrontsOAuthServerFn
  private countCommunicationChannels: (hostName: string) => number
  private findCommunicationChannelsByHostRef: (host: string) => CommunicationChannelCRD[]
  private findCommunicationChannelsByCredentialsSecretName: (
    name: string
  ) => CommunicationChannelCRD[]
  private readonly infrastructureTelemetryReporter?: InfrastructureTelemetryReporter
  private readonly administrativeOutcomeReporter?: AdministrativeOutcomeReporter
  private readonly gfsTokenLifecycleEvidence = new Map<string, GfsTokenLifecycleOutcome>()
  // B2: whether the CC cache initial-list has completed. Defaults to false
  // (safe: preserves existing Deployment replicas until wired by McpServerWatcher).
  private ccCacheSyncedFn: () => boolean = () => false
  /**
   * Resolve the current cached Host object by name. Wired by McpServerWatcher
   * from its live Host cache so a fleet worker reconciles the freshest spec and
   * so orphan cleanup compares candidates with the CURRENT inventory rather than
   * the pass snapshot. `null` (unwired) means callers fall back to the pass
   * snapshot for reconcile, and cleanup stays fail-closed (authority unknown).
   */
  private resolveCurrentHost: ((name: string) => HostCRD | undefined) | null = null
  /**
   * H2: late-bound cache reflector. Wired by McpServerWatcher next to
   * setResolveCurrentHost; null (standalone) means lifecycle outcomes reflect
   * only onto the caller's own object (legacy contract).
   */
  private reflectHostOutcome: ReflectHostOutcomeFn | null = null
  /**
   * Snapshot the current Host watch authority + generation. Wired by
   * McpServerWatcher. Default is fail-closed (unknown) so orphan cleanup never
   * runs until the real authority getter is installed.
   */
  private hostWatchAuthority: () => HostWatchAuthoritySnapshot = () => ({
    known: false,
    generation: 0,
  })
  /**
   * Positive Host mutations depend on Host plus Context authority. The default
   * preserves the standalone HostReconciler contract by deriving a Host-only
   * lease; McpServerWatcher wires the complete composite lease.
   */
  private hostMutationAuthority: () => HostMutationAuthoritySnapshot = () => {
    const host = this.hostWatchAuthority()
    return {
      known: host.known,
      // Standalone contract: no separate desired-revision plumbing here, so the
      // Host watch generation stands in as the Host-only lease identity. The
      // McpServerWatcher wires the real composite content revisions in production.
      hostRevision: host.generation,
      contextRevision: 0,
    }
  }
  /**
   * Exact revisions of the cross-resource inputs selected by a Host. Watch
   * generations fence LIST -> WATCH authority, but normal MODIFIED events stay
   * in the same generation. The watcher supplies the selected Context,
   * referenced SharedFileSystems plus mountability, and Host-scoped
   * CommunicationChannels so unrelated fleet changes do not retire this lease.
   */
  private resolveHostMutationDependencies: ((host: HostCRD) => readonly unknown[]) | null = null

  constructor(kc: k8s.KubeConfig, deps?: HostReconcilerDeps) {
    this.appsApi =
      deps?.appsApi ?? makeHostK8sApiClient(kc, k8s.AppsV1Api, config.hostK8sRequestTimeoutMs)
    this.coreApi =
      deps?.coreApi ?? makeHostK8sApiClient(kc, k8s.CoreV1Api, config.hostK8sRequestTimeoutMs)
    this.networkingApi =
      deps?.networkingApi ??
      makeHostK8sApiClient(kc, k8s.NetworkingV1Api, config.hostK8sRequestTimeoutMs)
    this.rbacApi =
      deps?.rbacApi ??
      makeHostK8sApiClient(kc, k8s.RbacAuthorizationV1Api, config.hostK8sRequestTimeoutMs)
    this.kubeConfig = kc
    this.customApiInstance = deps?.customApi
    this.now = deps?.now ?? (() => new Date())
    this.newTelemetryOccurrenceId = deps?.newTelemetryOccurrenceId ?? randomUUID
    this.resolveContextMounts = deps?.resolveContextMounts ?? (async () => [])
    this.hostFrontsOAuthServerFn = deps?.hostFrontsOAuthServer ?? (async () => false)
    this.countCommunicationChannels = deps?.countCommunicationChannels ?? (() => 0)
    this.findCommunicationChannelsByHostRef = deps?.findCommunicationChannelsByHostRef ?? (() => [])
    this.findCommunicationChannelsByCredentialsSecretName =
      deps?.findCommunicationChannelsByCredentialsSecretName ?? (() => [])
    this.infrastructureTelemetryReporter = deps?.infrastructureTelemetryReporter
    this.administrativeOutcomeReporter = deps?.administrativeOutcomeReporter
    if (deps?.isCommunicationChannelCacheSynced) {
      this.ccCacheSyncedFn = deps.isCommunicationChannelCacheSynced
    }
    this.lifecycle = new StatelessLifecycleExecutor({
      appsApi: this.appsApi,
      coreApi: this.coreApi,
      getCustomApi: () => this.customApi,
      now: this.now,
      countCommunicationChannels: hostName => this.countCommunicationChannels(hostName),
      isCommunicationChannelCacheSynced: () => this.ccCacheSyncedFn(),
      reconcileCore: (host, revalidate) => this.reconcileCore(host, revalidate),
      prepareHostMutationAdmission: (action, host) =>
        this.prepareHostMutationAdmission(action, host),
      reflectHostOutcome: (name, uid, apply) => this.reflectHostOutcome?.(name, uid, apply),
      onLifecycleStatusCommitted: (host, lifecycle) => {
        // A uid-less snapshot never reaches a lifecycle status commit, so this
        // branch is the compiler's requirement rather than an observed path.
        // It shares the emitter guard instead of assuming that stays true.
        const hostLookupReference = this.hostTelemetryReference(host, {
          telemetryType: 'health_transition',
          state: lifecycle.state,
        })
        if (hostLookupReference === undefined) return
        const occurredAt = this.now().toISOString()
        this.infrastructureTelemetryReporter?.enqueueHealthTransition({
          sourceEventId: `hcc-health-transition:${this.newTelemetryOccurrenceId()}`,
          occurredAt,
          hostLookupReference,
          payload: { transition: `lifecycle:${lifecycle.state}`, state: lifecycle.state },
        })
      },
    })
  }

  /** CustomObjects API client for Host /status writes (lazily constructed). */
  private get customApi(): k8s.CustomObjectsApi {
    if (!this.customApiInstance) {
      this.customApiInstance = makeHostK8sApiClient(
        this.kubeConfig,
        k8s.CustomObjectsApi,
        config.hostK8sRequestTimeoutMs
      )
    }
    return this.customApiInstance
  }

  /**
   * Late-bound setter so the McpServerWatcher (which owns both the Context
   * and SharedFileSystem caches) can wire up the mount resolver after this
   * reconciler is constructed.
   */
  setResolveContextMounts(fn: ResolveContextMountsFn): void {
    this.resolveContextMounts = fn
  }

  /**
   * Late-bound setter so the McpServerWatcher (which owns the Context and
   * McpServer caches) can wire up the oauth-server resolver after this
   * reconciler is constructed. Pattern mirrors setResolveContextMounts.
   */
  setHostFrontsOAuthServer(fn: HostFrontsOAuthServerFn): void {
    this.hostFrontsOAuthServerFn = fn
  }

  /**
   * Whether this Host fronts an enabled `auth.type: oauth` mcp-server, gating
   * the derive-only `oauth:user-token` runtime scope. A read that failed after
   * its retries is reported as unobserved instead of collapsing to `false`:
   * "scope unknown" is not "scope revoked". Callers decide what an unknown
   * answer means — the issuance path fails closed only where that cannot strip
   * a grant the running pod holds, and the deployment drift guard
   * (`currentScopeMatchesProvision`) never forces a re-mint on it. An uncertain
   * read never grants the scope.
   *
   * Each call is an INDEPENDENT live point-in-time read (mirrors
   * `hasChannelIngress`); the value is deliberately not cached across a
   * reconcile. The issuance path resolves it once and threads the same bool
   * into the mint-scope derive and the drift hash so they never diverge; the
   * deployment drift guard re-reads it on purpose to catch a scope change
   * mid-reconcile.
   */
  private async observeFrontsOAuthServer(
    host: HostCRD
  ): Promise<
    | { observed: true; frontsOAuthServer: boolean }
    | { observed: false; attempts: number; error: unknown }
  > {
    // Bounded retry that absorbs a TRANSIENT apiserver blip without touching the
    // authoritative-`false` path. Contract of the underlying resolver: a real
    // scope change (the server genuinely stopped fronting an oauth mcp-server)
    // RETURNS `false` without throwing; only a transient read failure (apiserver
    // blip) THROWS. So we retry ONLY on throw — any RETURNED value (true or
    // false) is authoritative and is returned immediately, so a healthy call
    // resolves on the first attempt with zero added overhead and an authoritative
    // `false` is never re-tried. Before this retry, a transient throw collapsed
    // to `false` exactly like an authoritative `false`, dropping the
    // oauth:user-token scope → runtime-token re-mint + Deployment rollout, then a
    // flip back on recovery → churn, once per OAuth Host per reconcile.
    const MAX_ATTEMPTS = 3
    const BACKOFF_BASE_MS = 50
    let lastErr: unknown
    for (let attempt = 1; attempt <= MAX_ATTEMPTS; attempt++) {
      try {
        return { observed: true, frontsOAuthServer: await this.hostFrontsOAuthServerFn(host) }
      } catch (err) {
        lastErr = err
        if (attempt < MAX_ATTEMPTS) {
          // Small linear backoff (50ms, 100ms) to let a transient blip clear.
          await new Promise(resolve => setTimeout(resolve, BACKOFF_BASE_MS * attempt))
        }
      }
    }
    return { observed: false, attempts: MAX_ATTEMPTS, error: lastErr }
  }

  /**
   * Late-bound setter so the McpServerWatcher (which owns the
   * communicationChannels cache) can wire up the counter after this
   * reconciler is constructed. Pattern mirrors setResolveContextMounts.
   */
  setCountCommunicationChannels(fn: (hostName: string) => number): void {
    this.countCommunicationChannels = fn
  }

  /**
   * B2: Late-bound setter so McpServerWatcher can wire up the CC cache-sync
   * flag after this reconciler is constructed. Pattern mirrors
   * setCountCommunicationChannels.
   */
  setIsCommunicationChannelCacheSynced(fn: () => boolean): void {
    this.ccCacheSyncedFn = fn
  }

  /**
   * Late-bound setter for the live Host cache resolver. A fleet worker resolves
   * the current cached Host at execution time (skipping absent/superseded
   * entries), and orphan cleanup uses it to compare candidates with the CURRENT
   * inventory. Wired by McpServerWatcher from its `hosts` cache.
   */
  setResolveCurrentHost(fn: (name: string) => HostCRD | undefined): void {
    this.resolveCurrentHost = fn
  }

  /**
   * H2: late-bound setter for the guarded cache reflector. Wired by
   * McpServerWatcher from its `hosts` cache (uid-guarded apply). Pattern
   * mirrors setResolveCurrentHost.
   */
  setReflectHostOutcome(fn: ReflectHostOutcomeFn): void {
    this.reflectHostOutcome = fn
  }

  /**
   * Late-bound setter for the Host watch authority/generation snapshot. Wired by
   * McpServerWatcher so orphan cleanup can require known authority and a stable
   * watch generation before deleting anything.
   */
  setHostWatchAuthority(fn: () => HostWatchAuthoritySnapshot): void {
    this.hostWatchAuthority = fn
  }

  setHostMutationAuthority(fn: () => HostMutationAuthoritySnapshot): void {
    this.hostMutationAuthority = fn
  }

  setResolveHostMutationDependencies(fn: (host: HostCRD) => readonly unknown[]): void {
    this.resolveHostMutationDependencies = fn
  }

  /**
   * A standalone HostReconciler receives an explicit Host object and has no
   * watch-backed cache to fence. Once a live-cache resolver is wired (the
   * production McpServerWatcher path), every mutation captures and revalidates
   * Host LIST -> WATCH authority at its actual serializer admission boundary.
   */
  private captureHostMutationAuthority(): HostMutationAuthoritySnapshot | null {
    return this.resolveCurrentHost === null ? null : this.hostMutationAuthority()
  }

  private requireHostMutationAuthority(
    action: string,
    captured: HostMutationAuthoritySnapshot | null
  ): void {
    if (captured === null) return
    const current = this.hostMutationAuthority()
    if (
      !captured.known ||
      !current.known ||
      captured.hostRevision !== current.hostRevision ||
      captured.contextRevision !== current.contextRevision
    ) {
      throw new HostInventoryAuthorityUnavailableError(action, captured, current)
    }
  }

  /**
   * Heartbeat writers own their serializer inside StatelessLifecycleExecutor.
   * Capture authority here at dispatch, then let the returned closure re-check
   * it and resolve the current same-UID Host when that serializer admits the
   * work. This preserves one queue per Host without a nested-lock deadlock.
   */
  private prepareHostMutationAdmission(action: string, requested: HostCRD): () => HostCRD {
    const capturedAuthority = this.captureHostMutationAuthority()
    const capturedDependencies = this.resolveHostMutationDependencies?.(requested)
    return this.makeHostMutationAdmission(
      action,
      requested,
      capturedAuthority,
      capturedDependencies
    )
  }

  private makeHostMutationAdmission(
    action: string,
    requested: HostCRD,
    capturedAuthority: HostMutationAuthoritySnapshot | null,
    capturedDependencies: readonly unknown[] | undefined
  ): () => HostCRD {
    return () => {
      this.requireHostMutationAuthority(action, capturedAuthority)
      if (this.resolveCurrentHost === null) return requested
      const current = this.resolveCurrentHost(requested.name)
      if (
        current === undefined ||
        requested.uid === undefined ||
        current.uid === undefined ||
        requested.uid !== current.uid
      ) {
        throw new HostMutationIdentityChangedError(action, requested.name)
      }
      if (
        requested.generation === undefined ||
        current.generation === undefined ||
        requested.generation !== current.generation
      ) {
        throw new HostMutationSpecRevisionChangedError(action, requested.name)
      }
      if (this.resolveHostMutationDependencies !== null) {
        const currentDependencies = this.resolveHostMutationDependencies(current)
        const sameDependencies =
          capturedDependencies !== undefined &&
          capturedDependencies.length === currentDependencies.length &&
          capturedDependencies.every((value, index) =>
            isDeepStrictEqual(value, currentDependencies[index])
          )
        if (!sameDependencies) {
          throw new HostMutationDependencyChangedError(action, requested.name)
        }
      }
      // McpServerWatcher owns the canonical LIST/WATCH cache object. Mutation
      // bodies historically reflect status into their admitted object for
      // same-pass decisions, so detach the working snapshot whenever a live
      // cache resolver is wired. Standalone callers without a resolver retain
      // the legacy caller-visible reflection contract.
      return structuredClone(current)
    }
  }

  /**
   * Late-bound setter for the host-ref lookup used by
   * `patchChannelReaderRevisionAnnotation` to discover Secret(s) referenced
   * by a host's CCs. Wired by `McpServerWatcher` from its
   * `communicationChannels` cache. Pattern mirrors setCountCommunicationChannels.
   */
  setFindCommunicationChannelsByHostRef(fn: (host: string) => CommunicationChannelCRD[]): void {
    this.findCommunicationChannelsByHostRef = fn
  }

  /**
   * Late-bound setter for the Secret-name lookup used by the Secret
   * informer (Task 9) to route rotation events to the affected hosts.
   * Wired by `McpServerWatcher` from its `communicationChannels` cache.
   */
  setFindCommunicationChannelsByCredentialsSecretName(
    fn: (name: string) => CommunicationChannelCRD[]
  ): void {
    this.findCommunicationChannelsByCredentialsSecretName = fn
  }

  /**
   * Volume name used by both the volumes[] entry and each volumeMounts[]
   * entry that references it. Stable per (sfs.name, sfs.namespace) so a
   * Deployment template diff only flips when the SFS list changes — not
   * when the same set is reordered.
   */
  static contextMountVolumeName(sfs: { name: string; namespace: string }): string {
    return `ctxfs-${sharedFileSystemHash(sfs)}`
  }

  static contextMountPathRejectionReason(
    mountPath: string,
    reservedMountPaths: string[]
  ): string | null {
    const normalizedMountPath = HostReconciler.normalizeContextMountPath(mountPath)
    if (!normalizedMountPath) {
      return (
        'mountPath must be an absolute POSIX path matching ' + CONTEXT_MOUNT_PATH_PATTERN.source
      )
    }

    const reserved = new Set(
      reservedMountPaths
        .map(HostReconciler.normalizeContainerMountPath)
        .filter((path): path is string => Boolean(path))
    )
    if (reserved.has(normalizedMountPath)) {
      return `mountPath conflicts with reserved Host mount "${normalizedMountPath}"`
    }

    return null
  }

  private static normalizeContextMountPath(mountPath: string): string | null {
    if (
      !CONTEXT_MOUNT_PATH_PATTERN.test(mountPath) ||
      mountPath.split('/').some(segment => segment === '..')
    ) {
      return null
    }
    return HostReconciler.normalizeContainerMountPath(mountPath)
  }

  private static normalizeContainerMountPath(mountPath: string): string | null {
    if (!mountPath.startsWith('/') || mountPath.includes('\\') || mountPath.includes('\0')) {
      return null
    }
    const normalized = path.posix.normalize(mountPath)
    return normalized.length > 1 ? normalized.replace(/\/+$/, '') : normalized
  }

  // ─── Per-Host RBAC scaffolding ────────────────────────────────────────

  /** Per-Host ServiceAccount name. */
  private hostSaName(host: HostCRD): string {
    return `host-${host.name}-sa`
  }

  /** Per-Host RBAC Role + RoleBinding name (shared between the two). */
  private hostRoleName(host: HostCRD): string {
    return `host-${host.name}-config-reader`
  }

  private rbacLabels(host: HostCRD): Record<string, string> {
    return {
      [MANAGED_BY_LABEL]: MANAGED_BY_VALUE,
      [HOST_LABEL]: host.name,
    }
  }

  private static isHccOwnedHostResource(
    resource: { metadata?: { labels?: Record<string, string> } },
    hostName: string
  ): boolean {
    const labels = resource.metadata?.labels ?? {}
    return labels[MANAGED_BY_LABEL] === MANAGED_BY_VALUE && labels[HOST_LABEL] === hostName
  }

  private async deleteIfHccOwned<T extends { metadata?: { labels?: Record<string, string> } }>(
    kind: string,
    name: string,
    namespace: string,
    hostName: string,
    read: () => Promise<T>,
    remove: () => Promise<unknown>
  ): Promise<void> {
    let resource: T
    try {
      resource = await read()
    } catch (error) {
      if (getErrorCode(error) === 404) return
      log.error('Failed to read Host resource ownership', { kind, name, err: error })
      throw error
    }

    if (!HostReconciler.isHccOwnedHostResource(resource, hostName)) {
      log.warn('Skipping Host resource delete - not HCC-owned', { kind, name, host: hostName })
      return
    }

    try {
      await remove()
      log.info('Deleted Host resource', { kind, name, namespace })
    } catch (error) {
      if (getErrorCode(error) !== 404) {
        log.error('Failed to delete Host resource', { kind, name, err: error })
        throw error
      }
    }
  }

  /**
   * Ensure a per-Host ServiceAccount exists. Idempotent.
   */
  private async ensureHostServiceAccount(host: HostCRD, revalidate?: () => void): Promise<void> {
    const name = this.hostSaName(host)
    const body: k8s.V1ServiceAccount = {
      apiVersion: 'v1',
      kind: 'ServiceAccount',
      metadata: { name, namespace: host.namespace, labels: this.rbacLabels(host) },
    }
    const mutationAllowed = () => {
      revalidate?.()
      return true
    }
    await ensureResource({
      mutationAllowed,
      read: () =>
        observeExistenceRead('ServiceAccount', () =>
          this.coreApi.readNamespacedServiceAccount({ name, namespace: host.namespace })
        ),
      create: () =>
        observeCreate('ServiceAccount', () =>
          this.coreApi.createNamespacedServiceAccount({ namespace: host.namespace, body })
        ),
      // Existing ServiceAccounts have always been retained without an update.
      converge: async read => {
        try {
          await read()
        } catch (error) {
          if (error != null && getErrorCode(error) === 404) return false
          throw error
        }
      },
      onSkipped: () => createsTotal.inc({ kind: 'ServiceAccount', outcome: 'skipped' }),
    })
  }

  /**
   * Per-Host Role granting `get/watch/list` ONLY on this Host's resources
   * (Host CRD, env ConfigMap/Secret, LLM Secret named in spec.secretRef).
   * On secretRef change the resourceNames are rewritten so Host A can never
   * read Host B's Secret.
   */
  private async ensureHostRole(host: HostCRD, revalidate?: () => void): Promise<void> {
    const name = this.hostRoleName(host)
    const body: k8s.V1Role = {
      apiVersion: 'rbac.authorization.k8s.io/v1',
      kind: 'Role',
      metadata: { name, namespace: host.namespace, labels: this.rbacLabels(host) },
      rules: [
        {
          apiGroups: ['clerum.io'],
          resources: ['hosts'],
          resourceNames: [host.name],
          verbs: ['get', 'watch', 'list'],
        },
        {
          // Per-Host env ConfigMap + the cluster-wide R3 allowlist ConfigMap
          // (clerum-llm-allowed-models). The allowlist is a single shared object
          // materialized by control-api; every Host may read it. resourceNames
          // are rewritten on every reconcile, so this propagates to existing Roles.
          apiGroups: [''],
          resources: ['configmaps'],
          resourceNames: [`host-${host.name}-env`, 'clerum-llm-allowed-models'],
          verbs: ['get', 'watch', 'list'],
        },
        {
          // READ-ONLY on the mcp-host-runtime-token Secret: granting write here would
          // let the pod rotate its own credentials (privilege escalation).
          // Writes happen via HCC's own ServiceAccount.
          apiGroups: [''],
          resources: ['secrets'],
          resourceNames: [
            host.spec.secretRef,
            `host-${host.name}-env-secret`,
            mcpHostRuntimeTokenSecretName(host),
          ].filter((name): name is string => Boolean(name)),
          verbs: ['get', 'watch', 'list'],
        },
      ],
    }
    const mutationAllowed = () => {
      revalidate?.()
      return true
    }
    await ensureResource({
      mutationAllowed,
      read: () =>
        observeExistenceRead('Role', () =>
          this.rbacApi.readNamespacedRole({ name, namespace: host.namespace })
        ),
      create: () =>
        observeCreate('Role', () =>
          this.rbacApi.createNamespacedRole({ namespace: host.namespace, body })
        ),
      converge: async read => {
        try {
          // Retry optimistic-lock contention, but never continue provisioning
          // with stale permissions after the bounded retry budget is exhausted.
          await replaceWithConflictRetry({
            description: `Role "${name}"`,
            logPrefix: '[HostReconciler]',
            body,
            read,
            mutationAllowed,
            isUpToDate: roleMatchesDesired,
            replace: next =>
              this.rbacApi.replaceNamespacedRole({ name, namespace: host.namespace, body: next }),
          })
        } catch (error) {
          // Preserve the existing disappearance policy, without swallowing
          // authorization, transport, or terminal conflict failures.
          if (error != null && getErrorCode(error) === 404) return false
          throw error
        }
      },
      onSkipped: () => createsTotal.inc({ kind: 'Role', outcome: 'skipped' }),
    })
  }

  /**
   * Ensure a per-Host RoleBinding binding the per-Host Role to the per-Host SA.
   */
  private async ensureHostRoleBinding(host: HostCRD, revalidate?: () => void): Promise<void> {
    const name = this.hostRoleName(host)
    const body: k8s.V1RoleBinding = {
      apiVersion: 'rbac.authorization.k8s.io/v1',
      kind: 'RoleBinding',
      metadata: { name, namespace: host.namespace, labels: this.rbacLabels(host) },
      subjects: [
        {
          kind: 'ServiceAccount',
          name: this.hostSaName(host),
          namespace: host.namespace,
        },
      ],
      roleRef: {
        apiGroup: 'rbac.authorization.k8s.io',
        kind: 'Role',
        name,
      },
    }
    const mutationAllowed = () => {
      revalidate?.()
      return true
    }
    await ensureResource({
      mutationAllowed,
      read: () =>
        observeExistenceRead('RoleBinding', () =>
          this.rbacApi.readNamespacedRoleBinding({ name, namespace: host.namespace })
        ),
      create: () =>
        observeCreate('RoleBinding', () =>
          this.rbacApi.createNamespacedRoleBinding({ namespace: host.namespace, body })
        ),
      // Preserve the existing binding; this path does not update subjects/roleRef.
      converge: async read => {
        try {
          await read()
        } catch (error) {
          if (error != null && getErrorCode(error) === 404) return false
          throw error
        }
      },
      onSkipped: () => createsTotal.inc({ kind: 'RoleBinding', outcome: 'skipped' }),
    })
  }

  /**
   * Ensures the mcp-host-runtime-token Secret for a 1st-party mcp-host pod.
   * Throws after 3 failed attempts (1s/2s/4s); the Host stays unready
   * until the cause is fixed.
   */
  private static runtimeTokenSecretRevision(data: Record<string, string> | undefined): string {
    const canonical = JSON.stringify(
      Object.entries(data ?? {}).sort(([left], [right]) => left.localeCompare(right))
    )
    return createHash('sha256').update(canonical).digest('hex')
  }

  private static secretDataForKeys(
    secret: Pick<k8s.V1Secret, 'data' | 'stringData'>,
    keys: string[]
  ): Record<string, string> | null {
    if (secret.data && keys.every(key => typeof secret.data?.[key] === 'string')) {
      return Object.fromEntries(keys.map(key => [key, secret.data![key]]))
    }
    if (secret.stringData && keys.every(key => typeof secret.stringData?.[key] === 'string')) {
      return Object.fromEntries(
        keys.map(key => [key, Buffer.from(secret.stringData![key], 'utf8').toString('base64')])
      )
    }
    return null
  }

  private static runtimeTokenSecretData(
    secret: Pick<k8s.V1Secret, 'data' | 'stringData'>
  ): Record<string, string> | null {
    return HostReconciler.secretDataForKeys(secret, [
      MCP_HOST_RUNTIME_TOKEN_SECRET_ACCESS_KEY,
      MCP_HOST_RUNTIME_TOKEN_SECRET_REFRESH_KEY,
      MCP_HOST_RUNTIME_TOKEN_SECRET_CONTROL_KEY,
      MCP_HOST_GFS_TOKEN_SECRET_KEY,
    ])
  }

  private static runtimeTokenSecretRevisionFromSecret(
    secret: Pick<k8s.V1Secret, 'data' | 'stringData'>
  ): string | null {
    const data = HostReconciler.runtimeTokenSecretData(secret)
    return data ? HostReconciler.runtimeTokenSecretRevision(data) : null
  }

  private static shortHash(value: unknown): string {
    return createHash('sha256').update(JSON.stringify(value)).digest('hex')
  }

  private static runtimeTokenHostBindingHash(host: HostCRD): string {
    return HostReconciler.shortHash({ namespace: host.namespace, host: host.name })
  }

  private static runtimeTokenScopeHash(
    host: HostCRD,
    hasChannelIngress = false,
    frontsOAuthServer = false,
    projection?: CodexExecutionProjection,
    grokProjection?: CodexExecutionProjection
  ): string {
    // Hash the EFFECTIVE (resolved) scopes — what actually gets minted into the
    // control token — so change-detection matches the default-fallback applied
    // at issuance (otherwise a null workflowControl hashes as [] while the token
    // carries the first-party defaults, and the two drift). Uses the SAME
    // `resolveRuntimeControlScopes` as the mint path (see
    // `resolveEffectiveControlScopesForHost`) so the hashed set and the minted
    // set — including the derive-only `oauth:user-token` and broker
    // projections' derived scopes — can never diverge.
    const runtimeScopes = [
      ...resolveRuntimeControlScopes(host.spec.workflowControl, {
        hasChannelIngress,
        frontsOAuthServer,
      }),
    ].sort()
    const codexDerived = projection?.derivedScopes ?? []
    const grokDerived = grokProjection?.derivedScopes ?? []
    if (codexDerived.length === 0 && grokDerived.length === 0) {
      return HostReconciler.shortHash(runtimeScopes)
    }
    // Codex-only Hosts must keep the pre-Grok hash input so an HCC upgrade
    // does not remint every existing Codex runtime token.
    if (grokDerived.length === 0) {
      return HostReconciler.shortHash({
        scopes: [...runtimeScopes, ...codexDerived].sort(),
        drift: projection?.driftHashInput,
      })
    }
    return HostReconciler.shortHash({
      scopes: [...runtimeScopes, ...codexDerived, ...grokDerived].sort(),
      drift: {
        codex: projection?.driftHashInput,
        grok: grokProjection?.driftHashInput,
      },
    })
  }

  private static gfsCapabilitySetHash(): string {
    return HostReconciler.shortHash([...GFS_HOST_SCOPES].sort())
  }

  /**
   * The key carries the uid alongside namespace/name: a Host deleted and
   * recreated under the same name is a different object, and an entry keyed by
   * the name alone would be reported as the new object's rotation (#696).
   */
  private static gfsLifecycleEvidenceKey(
    host: Pick<HostCRD, 'namespace' | 'name' | 'uid'>
  ): string | undefined {
    if (host.uid === undefined) return undefined
    return `${HostReconciler.gfsLifecycleEvidencePrefix(host.namespace, host.name)}${host.uid}`
  }

  private static gfsLifecycleEvidencePrefix(namespace: string, name: string): string {
    return `${namespace}/${name}/`
  }

  /**
   * Without a uid the objects cannot be told apart, so nothing is recorded: an
   * entry under an incomplete key would be claimed by whichever Host answers
   * to the name next. The real mappers copy metadata.uid from the API server,
   * so this warns rather than papering over a snapshot that lost it.
   */
  private recordGfsLifecycleEvidence(host: HostCRD, outcome: GfsTokenLifecycleOutcome): void {
    const key = HostReconciler.gfsLifecycleEvidenceKey(host)
    if (key === undefined) {
      log.warn('skipping gfs token lifecycle evidence: Host snapshot has no uid', {
        host: host.name,
        namespace: host.namespace,
        outcome,
      })
      return
    }
    this.gfsTokenLifecycleEvidence.set(key, outcome)
  }

  /**
   * Drops one object's entry. A caller holding the snapshot it reconciled
   * knows which uid it recorded under, so it must not reach for the prefix
   * sweep below: that would also destroy a sibling uid's evidence at the same
   * name, which is the cross-object confusion the uid in the key exists to
   * prevent (#696).
   */
  private clearGfsLifecycleEvidenceForObject(
    host: Pick<HostCRD, 'namespace' | 'name' | 'uid'>
  ): void {
    const key = HostReconciler.gfsLifecycleEvidenceKey(host)
    if (key !== undefined) this.gfsTokenLifecycleEvidence.delete(key)
  }

  /**
   * Drops every object's entry at a namespace/name. Only for callers handed a
   * name with no snapshot behind it, which is `reconcileDelete`: it cannot know
   * which uid the departing object had.
   */
  private clearGfsLifecycleEvidence(namespace: string, name: string): void {
    const prefix = HostReconciler.gfsLifecycleEvidencePrefix(namespace, name)
    for (const key of this.gfsTokenLifecycleEvidence.keys()) {
      if (key.startsWith(prefix)) this.gfsTokenLifecycleEvidence.delete(key)
    }
  }

  private static effectiveBootstrapRefreshBeforeSec(refreshTtlSec: number): number {
    const configured = Math.max(0, config.mcpHostBootstrapRefreshBeforeSec ?? 0)
    const resyncFloor = Math.max(0, config.hostResyncIntervalSec ?? 0) * 3
    const requested = Math.max(configured, resyncFloor)
    if (refreshTtlSec <= 1) return 0
    return Math.min(requested, refreshTtlSec - 1)
  }

  private static runtimeTokenSecretAnnotations(
    host: HostCRD,
    tokens: Awaited<ReturnType<typeof issueMcpHostRuntimeTokens>>,
    secretRevision: string,
    nowMs: number,
    gfsTokenTtlSec: number,
    hasChannelIngress = false,
    frontsOAuthServer = false,
    preservedHostUid?: string,
    projection?: CodexExecutionProjection,
    grokProjection?: CodexExecutionProjection
  ): Record<string, string> {
    const refreshTtlSec = Number.isFinite(tokens.refreshExpiresInSeconds)
      ? Math.max(0, tokens.refreshExpiresInSeconds)
      : 0
    const refreshExpiresAtMs = nowMs + refreshTtlSec * 1000
    const refreshBeforeSec = HostReconciler.effectiveBootstrapRefreshBeforeSec(refreshTtlSec)
    const refreshBeforeMs = refreshTtlSec > 0 ? refreshExpiresAtMs - refreshBeforeSec * 1000 : nowMs
    const annotations: Record<string, string> = {
      [RUNTIME_TOKEN_SECRET_REVISION_ANNOTATION]: secretRevision,
      [RUNTIME_TOKEN_ISSUED_AT_ANNOTATION]: new Date(nowMs).toISOString(),
      [RUNTIME_TOKEN_REFRESH_EXPIRES_AT_ANNOTATION]: new Date(refreshExpiresAtMs).toISOString(),
      [RUNTIME_TOKEN_REFRESH_BEFORE_ANNOTATION]: new Date(refreshBeforeMs).toISOString(),
      [RUNTIME_TOKEN_HOST_BINDING_HASH_ANNOTATION]:
        HostReconciler.runtimeTokenHostBindingHash(host),
      [RUNTIME_TOKEN_SCOPE_HASH_ANNOTATION]: HostReconciler.runtimeTokenScopeHash(
        host,
        hasChannelIngress,
        frontsOAuthServer,
        projection,
        grokProjection
      ),
      [RUNTIME_TOKEN_ISSUER_ANNOTATION]: RUNTIME_TOKEN_ISSUER,
      [RUNTIME_TOKEN_AUDIENCE_ANNOTATION]: RUNTIME_TOKEN_AUDIENCE,
      [RUNTIME_TOKEN_SCHEMA_VERSION_ANNOTATION]: RUNTIME_TOKEN_SCHEMA_VERSION,
      [GFS_TOKEN_EXPECTED_SUBJECT_ANNOTATION]:
        makeExpectedHostGfsSubject(host.namespace, host.name) ?? '',
      [GFS_TOKEN_CAPABILITY_SET_HASH_ANNOTATION]: HostReconciler.gfsCapabilitySetHash(),
    }
    const hostUid = host.uid ?? preservedHostUid
    if (hostUid) annotations[GFS_TOKEN_HOST_UID_ANNOTATION] = hostUid
    if (host.generation !== undefined) {
      annotations[GFS_TOKEN_HOST_GENERATION_ANNOTATION] = String(host.generation)
    }
    const gfsTtlSec = Number.isFinite(gfsTokenTtlSec) ? Math.max(0, gfsTokenTtlSec) : 0
    if (gfsTtlSec > 0) {
      const gfsExpiresAtMs = nowMs + gfsTtlSec * 1000
      const gfsRefreshBeforeSec = HostReconciler.effectiveBootstrapRefreshBeforeSec(gfsTtlSec)
      annotations[GFS_TOKEN_EXPIRES_AT_ANNOTATION] = new Date(gfsExpiresAtMs).toISOString()
      annotations[GFS_TOKEN_REFRESH_BEFORE_ANNOTATION] = new Date(
        gfsExpiresAtMs - gfsRefreshBeforeSec * 1000
      ).toISOString()
    }
    return annotations
  }

  private static runtimeTokenRefreshDecision(
    host: HostCRD,
    secret: k8s.V1Secret,
    nowMs: number,
    hasChannelIngress = false,
    frontsOAuthServer = false,
    projection?: CodexExecutionProjection,
    grokProjection?: CodexExecutionProjection
  ): { refresh: boolean; rolloutRequired: boolean; reason: string; refreshTokenExpMs?: number } {
    const labels = secret.metadata?.labels ?? {}
    if (labels[MANAGED_BY_LABEL] !== MANAGED_BY_VALUE || labels[HOST_LABEL] !== host.name) {
      // Reuse is only safe for Secrets HCC itself wrote for this Host. A
      // foreign or unlabeled Secret is replaced with a freshly issued,
      // HCC-owned one, and the pod rolls onto the trusted material.
      return { refresh: true, rolloutRequired: true, reason: 'not_hcc_owned' }
    }

    const data = HostReconciler.runtimeTokenSecretData(secret)
    if (!data) return { refresh: true, rolloutRequired: false, reason: 'malformed_secret' }

    const annotations = secret.metadata?.annotations ?? {}
    const expectedHostBindingHash = HostReconciler.runtimeTokenHostBindingHash(host)
    const expectedScopeHash = HostReconciler.runtimeTokenScopeHash(
      host,
      hasChannelIngress,
      frontsOAuthServer,
      projection,
      grokProjection
    )
    const hasContractMetadata =
      RUNTIME_TOKEN_HOST_BINDING_HASH_ANNOTATION in annotations ||
      RUNTIME_TOKEN_SCOPE_HASH_ANNOTATION in annotations ||
      RUNTIME_TOKEN_ISSUER_ANNOTATION in annotations ||
      RUNTIME_TOKEN_AUDIENCE_ANNOTATION in annotations ||
      RUNTIME_TOKEN_SCHEMA_VERSION_ANNOTATION in annotations
    if (!hasContractMetadata) {
      return { refresh: true, rolloutRequired: false, reason: 'missing_refresh_metadata' }
    }
    const contractChanged =
      annotations[RUNTIME_TOKEN_HOST_BINDING_HASH_ANNOTATION] !== expectedHostBindingHash ||
      annotations[RUNTIME_TOKEN_SCOPE_HASH_ANNOTATION] !== expectedScopeHash ||
      annotations[RUNTIME_TOKEN_ISSUER_ANNOTATION] !== RUNTIME_TOKEN_ISSUER ||
      annotations[RUNTIME_TOKEN_AUDIENCE_ANNOTATION] !== RUNTIME_TOKEN_AUDIENCE ||
      annotations[RUNTIME_TOKEN_SCHEMA_VERSION_ANNOTATION] !== RUNTIME_TOKEN_SCHEMA_VERSION
    if (contractChanged) {
      return { refresh: true, rolloutRequired: true, reason: 'contract_changed' }
    }

    const expectedGfsSubject = makeExpectedHostGfsSubject(host.namespace, host.name)
    if (!expectedGfsSubject) {
      return { refresh: true, rolloutRequired: true, reason: 'invalid_gfs_subject' }
    }
    if (annotations[GFS_TOKEN_EXPECTED_SUBJECT_ANNOTATION] !== expectedGfsSubject) {
      return { refresh: true, rolloutRequired: true, reason: 'gfs_subject_changed' }
    }
    if (
      annotations[GFS_TOKEN_CAPABILITY_SET_HASH_ANNOTATION] !==
      HostReconciler.gfsCapabilitySetHash()
    ) {
      return { refresh: true, rolloutRequired: true, reason: 'gfs_capability_set_changed' }
    }
    if (host.uid && annotations[GFS_TOKEN_HOST_UID_ANNOTATION] !== host.uid) {
      return { refresh: true, rolloutRequired: true, reason: 'gfs_host_uid_changed' }
    }

    // Scheduling cross-check against the refresh token's REAL expiry. The
    // `exp` claim is decoded from the JWT payload WITHOUT signature
    // verification -- HCC must never verify third-party tokens (that is
    // control-api's job); parsing `exp` to schedule rotation is metadata
    // inspection, not verification. Fail toward a fresh token when the claim
    // cannot be parsed.
    const refreshTokenExpMs = decodeJwtExpMs(
      Buffer.from(data[MCP_HOST_RUNTIME_TOKEN_SECRET_REFRESH_KEY], 'base64').toString('utf8')
    )
    if (refreshTokenExpMs === null) {
      return { refresh: true, rolloutRequired: false, reason: 'unparsable_refresh_token' }
    }
    const issuedAtMs = Date.parse(annotations[RUNTIME_TOKEN_ISSUED_AT_ANNOTATION] ?? '')
    const rotateBeforeMs = effectiveRotateBeforeMs(
      refreshTokenExpMs,
      Number.isFinite(issuedAtMs) ? issuedAtMs : null
    )
    if (nowMs >= refreshTokenExpMs - rotateBeforeMs) {
      return {
        refresh: true,
        rolloutRequired: false,
        reason: 'refresh_token_near_expiry',
        refreshTokenExpMs,
      }
    }

    const refreshBefore = Date.parse(annotations[RUNTIME_TOKEN_REFRESH_BEFORE_ANNOTATION] ?? '')
    if (!Number.isFinite(refreshBefore)) {
      return { refresh: true, rolloutRequired: false, reason: 'missing_refresh_metadata' }
    }
    if (nowMs >= refreshBefore) {
      return {
        refresh: true,
        rolloutRequired: false,
        reason: 'refresh_before_reached',
        refreshTokenExpMs,
      }
    }
    const gfsRefreshBefore = Date.parse(annotations[GFS_TOKEN_REFRESH_BEFORE_ANNOTATION] ?? '')
    if (!Number.isFinite(gfsRefreshBefore)) {
      return { refresh: true, rolloutRequired: false, reason: 'missing_gfs_refresh_metadata' }
    }
    if (nowMs >= gfsRefreshBefore) {
      return { refresh: true, rolloutRequired: false, reason: 'gfs_refresh_before_reached' }
    }
    return { refresh: false, rolloutRequired: false, reason: 'current', refreshTokenExpMs }
  }

  private async readHostDeploymentOrNull(host: HostCRD): Promise<k8s.V1Deployment | null> {
    try {
      return await this.appsApi.readNamespacedDeployment({
        name: host.name,
        namespace: host.namespace,
      })
    } catch (err) {
      if (getErrorCode(err) === 404) return null
      throw err
    }
  }

  private static deploymentRuntimeTokenRevision(deployment: k8s.V1Deployment | null): string {
    return (
      deployment?.spec?.template?.metadata?.annotations?.[RUNTIME_TOKEN_REVISION_ANNOTATION] ?? ''
    )
  }

  private static deploymentReady(deployment: k8s.V1Deployment | null): boolean {
    return (deployment?.status?.readyReplicas ?? 0) > 0
  }

  /** Bootstrap acknowledgement requires Ready replicas from the applied generation. */
  private static deploymentBootstrapConverged(deployment: k8s.V1Deployment | null): boolean {
    const generation = deployment?.metadata?.generation
    const replicas = deployment?.spec?.replicas ?? 1
    const status = deployment?.status
    return (
      generation !== undefined &&
      generation > 0 &&
      (status?.observedGeneration ?? 0) >= generation &&
      replicas > 0 &&
      status?.updatedReplicas === replicas &&
      status?.readyReplicas === replicas &&
      status?.replicas === replicas
    )
  }

  private static deploymentResourcesMatchHost(
    deployment: k8s.V1Deployment,
    host: HostCRD
  ): boolean {
    const resources = deployment.spec?.template.spec?.containers.find(
      container => container.name === 'mcp-host'
    )?.resources
    const desired =
      host.spec.desktop?.browser || host.spec.desktop?.x11
        ? config.desktopResources
        : config.hostResources
    let matches = true
    for (const kind of ['requests', 'limits'] as const) {
      const actual = resources?.[kind]
      const expected = desired[kind]
      if (!isDeepStrictEqual(Object.keys(actual ?? {}).sort(), Object.keys(expected).sort())) {
        matches = false
      }
      for (const [key, value] of Object.entries(expected)) {
        // Validate desired quantities even when a field is absent. A lexical
        // PUT stays governed by the exact Deployment comparator, but equal
        // Kubernetes amounts do not start a new Pod or need new credentials.
        kubernetesQuantitiesEqual(value, value)
        if (actual?.[key] === undefined || !kubernetesQuantitiesEqual(actual[key], value)) {
          matches = false
        }
      }
    }
    return matches
  }

  /**
   * The binding covers the Deployment UID and its runtime-token-revision
   * annotation only. It does not bind the rest of the pod template, so a
   * template edit that keeps both values still matches.
   *
   * This is deliberate: there is no pod-template hash. The record and the
   * Deployment belong to the same Host, and any principal that can edit the
   * pod template can already read the runtime-token Secret, so a template hash
   * would add no isolation. A hash would also have to be computed on the
   * apiserver-defaulted object, which never equals the applied body, so it
   * would fail to match and re-mint the credentials on every reconcile.
   */
  private static bootstrapBindingMatchesDeployment(
    record: Pick<k8s.V1Secret, 'metadata'>,
    deployment: k8s.V1Deployment | null
  ): boolean {
    const deploymentIdentity = deployment?.metadata?.uid
    const boundDeploymentIdentity =
      record.metadata?.annotations?.[RUNTIME_TOKEN_BOOTSTRAP_DEPLOYMENT_UID_ANNOTATION]
    const appliedRevision = HostReconciler.deploymentRuntimeTokenRevision(deployment)
    const boundAppliedRevision =
      record.metadata?.annotations?.[RUNTIME_TOKEN_BOOTSTRAP_APPLIED_REVISION_ANNOTATION]
    return (
      !!deploymentIdentity &&
      boundDeploymentIdentity === deploymentIdentity &&
      appliedRevision !== '' &&
      boundAppliedRevision === appliedRevision
    )
  }

  private static bootstrapAppliedToDeployment(
    record: k8s.V1Secret,
    deployment: k8s.V1Deployment | null,
    contentRevision: string
  ): boolean {
    if (HostReconciler.bootstrapBindingMatchesDeployment(record, deployment)) return true
    // An existing target binding must never be adopted by a same-name/new-UID
    // Deployment. Older unbound records can be bound by their applied revision.
    if (record.metadata?.annotations?.[RUNTIME_TOKEN_BOOTSTRAP_DEPLOYMENT_UID_ANNOTATION]) {
      return false
    }
    return (
      contentRevision !== '' &&
      HostReconciler.deploymentRuntimeTokenRevision(deployment) === contentRevision
    )
  }

  private static bootstrapObservationUpdates(
    host: HostCRD,
    record: k8s.V1Secret,
    deployment: k8s.V1Deployment | null,
    contentRevision: string
  ): Record<string, string> {
    if (
      record.metadata?.annotations?.[RUNTIME_TOKEN_BOOTSTRAP_STATE_ANNOTATION] !==
        RUNTIME_TOKEN_BOOTSTRAP_STATE_FRESH ||
      !HostReconciler.deploymentBelongsToHost(deployment, host) ||
      !HostReconciler.bootstrapAppliedToDeployment(record, deployment, contentRevision)
    )
      return {}
    const current = record.metadata?.annotations ?? {}
    const revision = HostReconciler.deploymentRuntimeTokenRevision(deployment)
    const desired: Record<string, string> = {
      [RUNTIME_TOKEN_BOOTSTRAP_DEPLOYMENT_UID_ANNOTATION]: deployment!.metadata!.uid!,
      [RUNTIME_TOKEN_BOOTSTRAP_APPLIED_REVISION_ANNOTATION]: revision,
      // Application is distinct from consumption: GFS content may change
      // while the same bootstrap waits for strict acknowledgement.
      [RUNTIME_TOKEN_ROLLOUT_REQUIRED_ANNOTATION]: 'false',
    }
    if (HostReconciler.deploymentBootstrapConverged(deployment)) {
      desired[RUNTIME_TOKEN_BOOTSTRAP_STATE_ANNOTATION] = RUNTIME_TOKEN_BOOTSTRAP_STATE_CONSUMED
    } else if (
      (deployment!.spec?.replicas ?? 1) > 0 &&
      (deployment!.metadata!.generation ?? 0) > 0 &&
      (deployment!.status?.observedGeneration ?? 0) >= deployment!.metadata!.generation! &&
      (deployment!.status?.updatedReplicas ?? 0) > 0 &&
      HostReconciler.deploymentReady(deployment)
    ) {
      // Aggregate Ready is only possible-use evidence, not an acknowledgement.
      // Persist it for this UID/revision so a later loss of Ready can recover
      // once; uninterrupted NotReady startup never acquires this observation.
      desired[RUNTIME_TOKEN_BOOTSTRAP_READY_REVISION_ANNOTATION] = revision
    }
    return Object.fromEntries(
      Object.entries(desired).filter(([key, value]) => current[key] !== value)
    )
  }

  private static credentialRecordBelongsToHost(
    record: { metadata?: { labels?: Record<string, string> } },
    host: HostCRD
  ): boolean {
    return HostReconciler.isHccOwnedHostResource(record, host.name)
  }

  private static runtimeRecordIsComplete(
    record: Pick<k8s.V1Secret, 'data' | 'stringData'>
  ): boolean {
    return HostReconciler.runtimeTokenSecretData(record) !== null
  }

  private static retainedBoolean(record: k8s.V1Secret, key: string): boolean | null {
    const value = record.metadata?.annotations?.[key]
    return value === 'true' ? true : value === 'false' ? false : null
  }

  private static deploymentBelongsToHost(
    deployment: k8s.V1Deployment | null,
    host: HostCRD
  ): deployment is k8s.V1Deployment {
    return (
      !!deployment &&
      HostReconciler.credentialRecordBelongsToHost(deployment, host) &&
      deployment.metadata?.name === host.name &&
      deployment.metadata?.namespace === host.namespace &&
      !!deployment.metadata?.uid &&
      !!deployment.metadata?.resourceVersion &&
      !deployment.metadata?.deletionTimestamp &&
      !!deployment.spec?.template?.spec &&
      !!host.uid &&
      deployment.metadata?.annotations?.[HOST_UID_ANNOTATION] === host.uid
    )
  }

  private static credentialRenewalWindowReached(
    record: k8s.V1Secret,
    nowMs: number,
    annotation: string
  ): boolean {
    const refreshBefore = Date.parse(record.metadata?.annotations?.[annotation] ?? '')
    return !Number.isFinite(refreshBefore) || nowMs >= refreshBefore
  }

  /**
   * True when this reconcile will (re)start a pod that reads the
   * mcp-host-runtime-token Secret fresh at boot -- a missing Deployment, a
   * Deployment scaled to 0 (the wake / scale 0->1 path), or a Deployment with
   * no Ready replica (a pod is (re)starting). In every such case the booting
   * pod bootstraps its refresh token from the Secret env-var material, so that
   * material MUST be freshly minted and un-revoked.
   *
   * The mcp-host runtime refresh token is single-use-rotating: control-api
   * revokes the prior JTI on every successful refresh. Once a running pod has
   * rotated, the copy still sitting in this HCC-owned Secret is REVOKED.
   * Reusing it (because its decoded exp is far out) hands a booting pod a
   * revoked refresh token -- it can never refresh its 300s access token,
   * readiness 503s forever, and the pod never becomes Ready. Reuse is only
   * safe for a pod that stays up and never re-reads the Secret (it uses its
   * in-memory rotated tokens), which is exactly deploymentReady && replicas>=1.
   *
   * A wake bootstrap already bound to this exact Deployment UID and applied
   * runtime revision is the one startup exception: its replica-only wake starts
   * that Deployment with the newly mounted material. Any identity or revision
   * change must mint again.
   */
  private static deploymentNeedsFreshBootstrap(
    deployment: k8s.V1Deployment | null,
    currentRevision: string,
    credentialRecord: k8s.V1Secret | null,
    options: BootstrapOptions
  ): boolean {
    if (options.forceFreshForWake || options.forceFreshForDeploymentChange || !deployment)
      return true
    // This reconcile will converge the Deployment to zero, so no Pod will
    // consume the stored material before the next verified wake.
    if (options.targetSuspended === true) return false
    if ((deployment.spec?.replicas ?? 1) === 0) return true
    const annotations = credentialRecord?.metadata?.annotations ?? {}
    const fresh =
      annotations[RUNTIME_TOKEN_BOOTSTRAP_STATE_ANNOTATION] === RUNTIME_TOKEN_BOOTSTRAP_STATE_FRESH
    const boundUid = annotations[RUNTIME_TOKEN_BOOTSTRAP_DEPLOYMENT_UID_ANNOTATION]
    if (boundUid && boundUid !== deployment.metadata?.uid) return true
    if (HostReconciler.deploymentReady(deployment)) return false
    if (
      fresh &&
      credentialRecord &&
      HostReconciler.bootstrapBindingMatchesDeployment(credentialRecord, deployment) &&
      annotations[RUNTIME_TOKEN_BOOTSTRAP_READY_REVISION_ANNOTATION] ===
        HostReconciler.deploymentRuntimeTokenRevision(deployment)
    )
      return true
    if (
      fresh &&
      credentialRecord &&
      (HostReconciler.bootstrapBindingMatchesDeployment(credentialRecord, deployment) ||
        (boundUid === deployment.metadata?.uid &&
          annotations[RUNTIME_TOKEN_ROLLOUT_REQUIRED_ANNOTATION] === 'true'))
    ) {
      return false
    }
    return (
      credentialRecord?.metadata?.annotations?.[RUNTIME_TOKEN_BOOTSTRAP_STATE_ANNOTATION] !==
        RUNTIME_TOKEN_BOOTSTRAP_STATE_FRESH ||
      HostReconciler.deploymentRuntimeTokenRevision(deployment) !== currentRevision
    )
  }

  private static shouldRollForRuntimeSecret(
    deployment: k8s.V1Deployment | null,
    force: boolean
  ): boolean {
    if (!deployment) return true
    if (force) return true
    if (!HostReconciler.deploymentReady(deployment)) return true
    const revision = HostReconciler.deploymentRuntimeTokenRevision(deployment)
    if (!revision) return true
    return revision === LEGACY_EMPTY_RUNTIME_TOKEN_REVISION
  }

  private static hasFreshBootstrapSecretNewerThanHealthyDeployment(
    secret: k8s.V1Secret,
    secretRevision: string,
    deployment: k8s.V1Deployment | null,
    nowMs: number
  ): boolean {
    if (!HostReconciler.deploymentReady(deployment)) return false
    const deploymentRevision = HostReconciler.deploymentRuntimeTokenRevision(deployment)
    if (!deploymentRevision || deploymentRevision === secretRevision) return false
    const refreshExpiresAt = Date.parse(
      secret.metadata?.annotations?.[RUNTIME_TOKEN_REFRESH_EXPIRES_AT_ANNOTATION] ?? ''
    )
    return Number.isFinite(refreshExpiresAt) && nowMs < refreshExpiresAt
  }

  private static deploymentRestartedAfterRuntimeSecretIssued(
    secret: k8s.V1Secret,
    deployment: k8s.V1Deployment | null
  ): boolean {
    const restartedAt = Date.parse(
      deployment?.spec?.template?.metadata?.annotations?.['kubectl.kubernetes.io/restartedAt'] ?? ''
    )
    if (!Number.isFinite(restartedAt)) return false

    const issuedAt = Date.parse(
      secret.metadata?.annotations?.[RUNTIME_TOKEN_ISSUED_AT_ANNOTATION] ?? ''
    )
    return Number.isFinite(issuedAt) && restartedAt > issuedAt
  }

  private hasChannelIngress(host: HostCRD): boolean {
    return (host.spec.channels?.length ?? 0) > 0 || this.countCommunicationChannels(host.name) > 0
  }

  private resolveWorkflowControlScopesForHost(
    host: HostCRD,
    hasChannelIngress = this.hasChannelIngress(host),
    frontsOAuthServer = false
  ): HostRuntimeControlScope[] {
    // Delegate to the SAME resolver the drift hash uses so the minted scope set
    // and the hashed scope set can never diverge (a divergence re-issues the
    // token every reconcile).
    const scopes = resolveRuntimeControlScopes(host.spec.workflowControl, {
      hasChannelIngress,
      frontsOAuthServer,
    })
    const hasWorkflowScope = scopes.some(scope => scope !== OAUTH_USER_TOKEN_SCOPE)
    if (hasChannelIngress && !hasWorkflowScope) {
      log.warn('channel-bearing Host resolved with no workflow-control scopes', {
        host: host.name,
        namespace: host.namespace,
      })
    }
    return scopes
  }

  private projectCodexForHost(host: HostCRD): CodexExecutionProjection {
    const connectionKey = assignedHostCodexConnectionRef(host.spec.model?.connectionRef)
    const snapshot = snapshotForAssignedCodexGrant(
      connectionKey,
      this.lastCodexConfigMap,
      this.codexSnapshot
    )
    return projectCodexExecution(host.spec, snapshot)
  }

  private projectGrokForHost(host: HostCRD): CodexExecutionProjection & {
    requiresGrokProxyEgress: boolean
  } {
    const connectionKey = assignedHostCodexConnectionRef(host.spec.model?.connectionRef)
    const snapshot = parseGrokAllowedModelsSnapshot(this.lastCodexConfigMap, connectionKey)
    return projectGrokExecution(host.spec, snapshot)
  }

  private hostDerivesGrokExecution(host: HostCRD): boolean {
    return this.projectGrokForHost(host).derivedScopes.includes(GROK_EXECUTE_SCOPE)
  }

  private resolveEffectiveControlScopesForHost(
    host: HostCRD,
    frontsOAuthServer: boolean,
    hasChannelIngress = this.hasChannelIngress(host)
  ): EffectiveMcpHostControlScope[] {
    const workflow = this.resolveWorkflowControlScopesForHost(
      host,
      hasChannelIngress,
      frontsOAuthServer
    )
    const derived = [
      ...this.projectCodexForHost(host).derivedScopes,
      ...this.projectGrokForHost(host).derivedScopes,
    ].filter(scope => !workflow.includes(scope as HostWorkflowControlScope))
    return [...workflow, ...derived] as EffectiveMcpHostControlScope[]
  }

  private runtimeScopeHashFor(
    host: HostCRD,
    frontsOAuthServer: boolean,
    hasChannelIngress = this.hasChannelIngress(host)
  ): string {
    return HostReconciler.runtimeTokenScopeHash(
      host,
      hasChannelIngress,
      frontsOAuthServer,
      this.projectCodexForHost(host),
      this.projectGrokForHost(host)
    )
  }

  /**
   * Deployment drift guard: does the provisioned credential still match the
   * Host's current scope contract? An OAuth read that failed after its retries
   * is "scope unknown", so it matches a provision minted under either OAuth
   * answer instead of collapsing to `false` and forcing a re-mint that would
   * strip oauth:user-token and roll the Deployment. The issuance path owns the
   * decision to narrow the grant (see `ensureMcpHostRuntimeTokenSecret`).
   */
  private currentScopeMatchesProvision(
    host: HostCRD,
    provision: RuntimeTokenProvision,
    live: Awaited<ReturnType<HostReconciler['observeFrontsOAuthServer']>>
  ): boolean {
    if (live.observed) {
      return provision.scopeHash === this.runtimeScopeHashFor(host, live.frontsOAuthServer)
    }
    return (
      provision.scopeHash === this.runtimeScopeHashFor(host, true) ||
      provision.scopeHash === this.runtimeScopeHashFor(host, false)
    )
  }

  private async refreshCodexSnapshot(): Promise<void> {
    try {
      const cm = await this.coreApi.readNamespacedConfigMap({
        name: ALLOWED_MODELS_CONFIGMAP_NAME,
        namespace: config.hccTargetNamespace,
      })
      this.lastCodexConfigMap = cm
      this.codexSnapshot = parseAllowedModelsSnapshot(cm)
    } catch (err) {
      const timeout =
        err instanceof HostK8sRequestTimeoutError ||
        (err as { code?: string }).code === HOST_K8S_REQUEST_TIMEOUT_CODE
      if (timeout) {
        this.lastCodexConfigMap = undefined
        this.codexSnapshot = snapshotFromConfigMapError('timeout')
        log.warn('Codex allowlist ConfigMap read timed out; failing closed', {
          configMap: ALLOWED_MODELS_CONFIGMAP_NAME,
          namespace: config.hccTargetNamespace,
        })
        return
      }
      const code = getErrorCode(err)
      if (code === 401 || code === 403) {
        this.lastCodexConfigMap = undefined
        this.codexSnapshot = snapshotFromConfigMapError('forbidden')
        log.warn('Codex allowlist ConfigMap read forbidden; failing closed', {
          configMap: ALLOWED_MODELS_CONFIGMAP_NAME,
          namespace: config.hccTargetNamespace,
          statusCode: code,
        })
        return
      }
      this.lastCodexConfigMap = undefined
      this.codexSnapshot = snapshotFromConfigMapError('missing')
      log.warn('Codex allowlist ConfigMap unavailable; failing closed', {
        configMap: ALLOWED_MODELS_CONFIGMAP_NAME,
        namespace: config.hccTargetNamespace,
        statusCode: code,
      })
    }
  }

  private async ensureMcpHostRuntimeTokenSecret(
    host: HostCRD,
    options: BootstrapOptions = {}
  ): Promise<RuntimeTokenProvision | null> {
    let lastErr: unknown = null
    // The held GFS-only renewal runs inside a reconcile that keeps a Ready
    // runtime in place; it gets one attempt so a failing renewal never blocks
    // that reconcile on backoff. The next hold pass retries it.
    const maxAttempts = options.refreshGfsOnly ? 1 : 3
    for (let attempt = 1; attempt <= maxAttempts; attempt++) {
      try {
        const expectedGfsSubject = makeExpectedHostGfsSubject(host.namespace, host.name)
        if (!expectedGfsSubject) {
          throw new Error('Host GFS token provisioning requires a trusted namespace and name')
        }
        const name = mcpHostRuntimeTokenSecretName(host)
        let existing: k8s.V1Secret | null = null
        try {
          existing = await observeExistenceRead('Secret', () =>
            this.coreApi.readNamespacedSecret({ name, namespace: host.namespace })
          )
        } catch (err) {
          if (getErrorCode(err) !== 404) throw err
        }
        const deployment = await this.readHostDeploymentOrNull(host)
        const existingRevision = existing
          ? HostReconciler.runtimeTokenSecretRevisionFromSecret(existing)
          : null
        const nowMs = Date.now()
        const preserveWakeTemplate =
          options.preserveDeploymentTemplateOnWake === true &&
          !!deployment &&
          HostReconciler.deploymentBelongsToHost(deployment, host) &&
          (deployment.spec?.replicas ?? 1) === 0 &&
          HostReconciler.deploymentRuntimeTokenRevision(deployment) !== ''
        if (options.refreshGfsOnly) {
          const trustedDeployment =
            HostReconciler.deploymentBelongsToHost(deployment, host) &&
            (deployment?.spec?.replicas ?? 0) > 0
          if (
            !existing ||
            !HostReconciler.credentialRecordBelongsToHost(existing, host) ||
            !host.uid ||
            existing.metadata?.annotations?.[GFS_TOKEN_HOST_UID_ANNOTATION] !== host.uid ||
            !HostReconciler.runtimeRecordIsComplete(existing) ||
            !trustedDeployment
          ) {
            log.warn(
              'deferring held-runtime credential renewal because runtime identity is not verified',
              {
                host: host.name,
                namespace: host.namespace,
              }
            )
            return null
          }
          const annotationUpdates = HostReconciler.bootstrapObservationUpdates(
            host,
            existing,
            deployment,
            existingRevision ?? ''
          )
          const renewalDue = HostReconciler.credentialRenewalWindowReached(
            existing,
            nowMs,
            GFS_TOKEN_REFRESH_BEFORE_ANNOTATION
          )
          // A pending strict acknowledgement cannot starve independent GFS
          // renewal. Bind the applied bootstrap before changing the full
          // content hash; runtime bytes and the applied revision stay intact.
          if (!renewalDue || !HostReconciler.deploymentReady(deployment)) {
            if (Object.keys(annotationUpdates).length > 0) {
              await this.coreApi.replaceNamespacedSecret({
                name,
                namespace: host.namespace,
                body: {
                  ...existing,
                  metadata: {
                    ...existing.metadata,
                    annotations: {
                      ...(existing.metadata?.annotations ?? {}),
                      ...annotationUpdates,
                    },
                  },
                },
              })
            }
            if (
              !HostReconciler.deploymentReady(deployment) &&
              !HostReconciler.bootstrapBindingMatchesDeployment(existing, deployment)
            )
              return null
            this.recordGfsLifecycleEvidence(host, 'reused')
            return {
              revision:
                HostReconciler.deploymentRuntimeTokenRevision(deployment) || existingRevision || '',
              scopeHash:
                existing.metadata?.annotations?.[RUNTIME_TOKEN_SCOPE_HASH_ANNOTATION] ?? '',
            }
          }

          const gfs = await mintHostGfsToken({ name: host.name, namespace: host.namespace })
          const data = { ...(HostReconciler.runtimeTokenSecretData(existing) ?? {}) }
          data[MCP_HOST_GFS_TOKEN_SECRET_KEY] = Buffer.from(gfs.token).toString('base64')
          const gfsExpiresAtMs = nowMs + Math.max(0, gfs.expiresInSeconds) * 1000
          const gfsRefreshBeforeSec = HostReconciler.effectiveBootstrapRefreshBeforeSec(
            Math.max(0, gfs.expiresInSeconds)
          )
          const nextRevision = HostReconciler.runtimeTokenSecretRevision(data)
          await this.coreApi.replaceNamespacedSecret({
            name,
            namespace: host.namespace,
            body: {
              ...existing,
              data,
              metadata: {
                ...existing.metadata,
                annotations: {
                  ...(existing.metadata?.annotations ?? {}),
                  ...annotationUpdates,
                  [GFS_TOKEN_EXPECTED_SUBJECT_ANNOTATION]:
                    makeExpectedHostGfsSubject(host.namespace, host.name) ?? '',
                  [GFS_TOKEN_CAPABILITY_SET_HASH_ANNOTATION]: HostReconciler.gfsCapabilitySetHash(),
                  [GFS_TOKEN_EXPIRES_AT_ANNOTATION]: new Date(gfsExpiresAtMs).toISOString(),
                  [GFS_TOKEN_REFRESH_BEFORE_ANNOTATION]: new Date(
                    gfsExpiresAtMs - gfsRefreshBeforeSec * 1000
                  ).toISOString(),
                  [RUNTIME_TOKEN_SECRET_REVISION_ANNOTATION]: nextRevision,
                },
              },
            },
          })
          this.recordGfsLifecycleEvidence(host, 'rotated')
          return {
            revision:
              HostReconciler.deploymentRuntimeTokenRevision(deployment) || existingRevision || '',
            scopeHash: existing.metadata?.annotations?.[RUNTIME_TOKEN_SCOPE_HASH_ANNOTATION] ?? '',
          }
        }
        const bootstrapIsFresh =
          existing?.metadata?.annotations?.[RUNTIME_TOKEN_BOOTSTRAP_STATE_ANNOTATION] ===
          RUNTIME_TOKEN_BOOTSTRAP_STATE_FRESH
        const retainedAnnotations = existing?.metadata?.annotations ?? {}
        const cacheSynced = this.ccCacheSyncedFn()
        const retainedScopeIsTrusted =
          !!existing &&
          HostReconciler.credentialRecordBelongsToHost(existing, host) &&
          !!host.uid &&
          retainedAnnotations[GFS_TOKEN_HOST_UID_ANNOTATION] === host.uid
        const retainedChannelIngress =
          retainedScopeIsTrusted && existing
            ? HostReconciler.retainedBoolean(existing, RUNTIME_TOKEN_HAS_CHANNEL_INGRESS_ANNOTATION)
            : null
        const retainedFrontsOAuth =
          retainedScopeIsTrusted && existing
            ? HostReconciler.retainedBoolean(existing, RUNTIME_TOKEN_FRONTS_OAUTH_ANNOTATION)
            : null
        const hasChannelIngress = cacheSynced
          ? this.hasChannelIngress(host)
          : retainedChannelIngress === true
        if (!cacheSynced && retainedChannelIngress === null) {
          log.warn(
            'skipping runtime token mint during channel cache loss without retained scope observation',
            {
              host: host.name,
              namespace: host.namespace,
              observation: 'hasChannelIngress',
            }
          )
          return null
        }
        const projection = this.projectCodexForHost(host)
        const grokProjection = this.projectGrokForHost(host)
        const decideRefresh = (
          frontsOAuth: boolean
        ): {
          refresh: boolean
          rolloutRequired: boolean
          reason: string
          refreshTokenExpMs?: number
        } => {
          let decided: {
            refresh: boolean
            rolloutRequired: boolean
            reason: string
            refreshTokenExpMs?: number
          } = existing
            ? HostReconciler.runtimeTokenRefreshDecision(
                host,
                existing,
                nowMs,
                hasChannelIngress,
                frontsOAuth,
                projection,
                grokProjection
              )
            : { refresh: true, rolloutRequired: false, reason: 'missing_secret' }
          if (
            existing &&
            HostReconciler.deploymentRestartedAfterRuntimeSecretIssued(existing, deployment)
          ) {
            // A manual restart after the runtime Secret was issued starts pods with
            // env-var bootstrap tokens from that older Secret. Force a re-issue and
            // rollout so every restarted pod observes a fresh, unconsumed token pair.
            decided = {
              refresh: true,
              rolloutRequired: true,
              reason: 'deployment_restarted_after_secret_issued',
            }
          }

          // Revoked-on-wake guard: the mcp-host runtime refresh token is
          // single-use-rotating (control-api revokes the prior JTI on every
          // refresh). A pod that is about to boot -- wake (scale 0->1), missing
          // Deployment, or a not-Ready (re)starting pod -- reads its bootstrap
          // refresh token from THIS Secret. If the pre-suspend pod already
          // rotated, the copy in the Secret is REVOKED, so an exp-based "reuse"
          // hands the booting pod a dead token and readiness 503s forever. When
          // a new pod will consume the Secret, force a fresh mint + rollout even
          // though the decoded exp is far out. Reuse stays correct only for a
          // running, Ready pod that never re-reads the Secret (churn fix).
          if (
            existingRevision &&
            decided.refresh === false &&
            HostReconciler.deploymentNeedsFreshBootstrap(
              deployment,
              existingRevision,
              existing,
              options
            )
          ) {
            decided = {
              refresh: true,
              rolloutRequired: true,
              reason: 'fresh_mint_for_booting_pod',
              ...(decided.refreshTokenExpMs !== undefined
                ? { refreshTokenExpMs: decided.refreshTokenExpMs }
                : {}),
            }
          }
          return decided
        }

        // Resolve ONCE per issuance and thread the SAME frontsOAuthServer bool and
        // codex projection into the mint-scope derive, the refresh decision, the
        // scope hash and the stored annotation, so the minted token and the drift
        // hash never diverge.
        //
        // The OAuth observation reads the McpServer cache, which does not depend
        // on the CommunicationChannel cache, so it stays live during channel
        // cache loss: an admin who removed the OAuth mcp-server while the channel
        // watch was down must not have `oauth:user-token` re-granted from the
        // retained annotation on a held wake.
        //
        // A live read that failed after its retries is "scope unknown", not
        // "scope revoked". It resolves to `false` (fail closed) only when that
        // cannot narrow a grant the running pod holds: the retained value is
        // `false`, or there is no trusted retained value while channel authority
        // is synced. A retained `true` under synced authority defers the decision
        // while the credential needs no renewal, so an McpServer relist does not
        // strip oauth:user-token and roll the Deployment; once the credential must
        // be renewed, the renewal mints without the unobserved scope. Under channel
        // cache loss the mint is skipped instead.
        const liveFrontsOAuth = await this.observeFrontsOAuthServer(host)
        let observedFrontsOAuth: boolean
        if (liveFrontsOAuth.observed) {
          observedFrontsOAuth = liveFrontsOAuth.frontsOAuthServer
        } else {
          const observationFailure = {
            host: host.name,
            namespace: host.namespace,
            observation: 'frontsOAuthServer',
            attempts: liveFrontsOAuth.attempts,
            err:
              liveFrontsOAuth.error instanceof Error
                ? liveFrontsOAuth.error.message
                : String(liveFrontsOAuth.error),
          }
          if (retainedFrontsOAuth === false || (cacheSynced && retainedFrontsOAuth !== true)) {
            log.warn(
              'failed to resolve whether Host fronts an oauth mcp-server after retries; failing closed (no oauth:user-token scope)',
              observationFailure
            )
            observedFrontsOAuth = false
          } else if (cacheSynced) {
            if (!decideRefresh(true).refresh) {
              log.warn(
                'deferring runtime token decision: OAuth observation unavailable and retained scope grants oauth:user-token',
                observationFailure
              )
              return null
            }
            log.warn(
              'renewing runtime token without oauth:user-token: OAuth observation unavailable when the credential must be renewed',
              observationFailure
            )
            observedFrontsOAuth = false
          } else {
            log.warn(
              'skipping runtime token mint during channel cache loss without an authoritative OAuth observation',
              observationFailure
            )
            return null
          }
        }
        const scopeHash = HostReconciler.runtimeTokenScopeHash(
          host,
          hasChannelIngress,
          observedFrontsOAuth,
          projection,
          grokProjection
        )
        let decision = decideRefresh(observedFrontsOAuth)
        const resourcesWillChange =
          options.prepareForResourceChange === true &&
          HostReconciler.deploymentBelongsToHost(deployment, host) &&
          (deployment.spec?.replicas ?? 1) > 0 &&
          !HostReconciler.deploymentResourcesMatchHost(deployment!, host)
        const bootstrapApplied =
          !!existing &&
          !!existingRevision &&
          HostReconciler.bootstrapAppliedToDeployment(existing, deployment, existingRevision)
        const boundUid =
          existing?.metadata?.annotations?.[RUNTIME_TOKEN_BOOTSTRAP_DEPLOYMENT_UID_ANNOTATION]
        // A routine renewal can be consumed by a container restart without
        // changing the applied revision. Reuse only explicit rollout preparation.
        const usablePendingPreparation =
          bootstrapIsFresh &&
          existing?.metadata?.annotations?.[RUNTIME_TOKEN_ROLLOUT_REQUIRED_ANNOTATION] === 'true' &&
          !bootstrapApplied &&
          (!boundUid || boundUid === deployment?.metadata?.uid)
        if (resourcesWillChange) {
          // Resource preparation is optional until scope authority is observed;
          // it must not participate in the intrinsic renewal/OAuth fallback above.
          // Compose a real transition with an already due renewal in one issuance.
          decision = {
            ...decision,
            refresh: decision.refresh || !usablePendingPreparation,
            rolloutRequired: true,
            reason: decision.refresh ? decision.reason : 'fresh_mint_for_resource_change',
          }
        }

        if (existing && existingRevision && !decision.refresh) {
          const deploymentRevision = HostReconciler.deploymentRuntimeTokenRevision(deployment)
          const rolloutMarker =
            existing.metadata?.annotations?.[RUNTIME_TOKEN_ROLLOUT_REQUIRED_ANNOTATION]
          const boundTargetRevision =
            existing.metadata?.annotations?.[RUNTIME_TOKEN_BOOTSTRAP_APPLIED_REVISION_ANNOTATION]
          const preparedRevision =
            bootstrapIsFresh && boundUid === deployment?.metadata?.uid && boundTargetRevision
              ? boundTargetRevision
              : existingRevision
          const bootstrapBoundToAppliedDeployment =
            HostReconciler.bootstrapBindingMatchesDeployment(existing, deployment)
          const bootstrapPendingForAppliedDeployment = bootstrapIsFresh && bootstrapApplied
          const bootstrapConsumedForAppliedDeployment =
            existing.metadata?.annotations?.[RUNTIME_TOKEN_BOOTSTRAP_STATE_ANNOTATION] ===
              RUNTIME_TOKEN_BOOTSTRAP_STATE_CONSUMED && bootstrapBoundToAppliedDeployment
          const rolloutPending =
            !bootstrapApplied &&
            (resourcesWillChange ||
              rolloutMarker === 'true' ||
              (rolloutMarker === undefined &&
                bootstrapIsFresh &&
                deploymentRevision !== '' &&
                deploymentRevision !== existingRevision))
          const annotationUpdates = HostReconciler.bootstrapObservationUpdates(
            host,
            existing,
            deployment,
            existingRevision
          )
          if (resourcesWillChange && usablePendingPreparation && !bootstrapApplied) {
            // Older preparations can already carry the rollout marker without
            // a target binding. Persist that binding before delivering them.
            const preparationAnnotations = {
              [RUNTIME_TOKEN_ROLLOUT_REQUIRED_ANNOTATION]: 'true',
              [RUNTIME_TOKEN_BOOTSTRAP_DEPLOYMENT_UID_ANNOTATION]: deployment!.metadata!.uid!,
              [RUNTIME_TOKEN_BOOTSTRAP_APPLIED_REVISION_ANNOTATION]: preparedRevision,
            }
            for (const [key, value] of Object.entries(preparationAnnotations)) {
              if (existing.metadata?.annotations?.[key] !== value) annotationUpdates[key] = value
            }
          }
          if (cacheSynced) {
            const currentIngress =
              existing.metadata?.annotations?.[RUNTIME_TOKEN_HAS_CHANNEL_INGRESS_ANNOTATION]
            if (currentIngress !== String(hasChannelIngress)) {
              annotationUpdates[RUNTIME_TOKEN_HAS_CHANNEL_INGRESS_ANNOTATION] =
                String(hasChannelIngress)
            }
            const currentOAuth =
              existing.metadata?.annotations?.[RUNTIME_TOKEN_FRONTS_OAUTH_ANNOTATION]
            if (currentOAuth !== String(observedFrontsOAuth)) {
              annotationUpdates[RUNTIME_TOKEN_FRONTS_OAUTH_ANNOTATION] = String(observedFrontsOAuth)
            }
          }
          if (
            host.generation !== undefined &&
            existing.metadata?.annotations?.[GFS_TOKEN_HOST_GENERATION_ANNOTATION] !==
              String(host.generation)
          ) {
            annotationUpdates[GFS_TOKEN_HOST_GENERATION_ANNOTATION] = String(host.generation)
          }
          if (Object.keys(annotationUpdates).length > 0) {
            await this.coreApi.replaceNamespacedSecret({
              name,
              namespace: host.namespace,
              body: {
                ...existing,
                metadata: {
                  ...existing.metadata,
                  annotations: {
                    ...(existing.metadata?.annotations ?? {}),
                    ...annotationUpdates,
                  },
                },
              },
            })
          }
          const refreshExpInHours =
            typeof decision.refreshTokenExpMs === 'number'
              ? Math.round(((decision.refreshTokenExpMs - nowMs) / 3_600_000) * 10) / 10
              : null
          log.info('reusing mcp-host-runtime-token Secret', {
            host: host.name,
            namespace: host.namespace,
            resourceName: name,
            refreshExpInHours,
          })
          this.recordGfsLifecycleEvidence(host, 'reused')
          const selectedRevision =
            rolloutPending ||
            (!bootstrapPendingForAppliedDeployment &&
              !bootstrapConsumedForAppliedDeployment &&
              HostReconciler.shouldRollForRuntimeSecret(deployment, false))
              ? preparedRevision
              : deploymentRevision || existingRevision
          return {
            revision: selectedRevision,
            scopeHash,
            deploymentUid: deployment?.metadata?.uid,
            freshBootstrapPrepared:
              bootstrapIsFresh &&
              !bootstrapApplied &&
              (annotationUpdates[RUNTIME_TOKEN_BOOTSTRAP_DEPLOYMENT_UID_ANNOTATION] ?? boundUid) ===
                deployment?.metadata?.uid &&
              (annotationUpdates[RUNTIME_TOKEN_BOOTSTRAP_APPLIED_REVISION_ANNOTATION] ??
                boundTargetRevision) === selectedRevision,
          }
        }

        if (
          !resourcesWillChange &&
          existing &&
          existingRevision &&
          (decision.reason === 'refresh_before_reached' ||
            decision.reason === 'refresh_token_near_expiry') &&
          HostReconciler.hasFreshBootstrapSecretNewerThanHealthyDeployment(
            existing,
            existingRevision,
            deployment,
            nowMs
          )
        ) {
          log.info('preserved fresh mcp-host-runtime-token Secret for healthy deployment', {
            host: host.name,
            namespace: host.namespace,
            resourceName: name,
            reason: decision.reason,
          })
          return {
            revision: HostReconciler.deploymentRuntimeTokenRevision(deployment) || existingRevision,
            scopeHash,
            deploymentUid: deployment?.metadata?.uid,
          }
        }

        const tokens = await issueMcpHostRuntimeTokens(
          host.name,
          host.uid ?? '',
          this.resolveEffectiveControlScopesForHost(host, observedFrontsOAuth, hasChannelIngress)
        )
        const gfs = await mintHostGfsToken({ name: host.name, namespace: host.namespace })
        const body = buildMcpHostRuntimeTokenSecret(
          host,
          tokens.accessToken,
          tokens.refreshToken,
          tokens.mcpHostControlToken,
          gfs.token
        )
        const revision = HostReconciler.runtimeTokenSecretRevisionFromSecret(body)
        if (!revision) {
          throw new Error('mcp-host-runtime-token Secret body missing required credential keys')
        }
        const rolloutRequired =
          !preserveWakeTemplate &&
          (!existing ||
            HostReconciler.shouldRollForRuntimeSecret(deployment, decision.rolloutRequired))
        body.metadata = {
          ...body.metadata,
          annotations: {
            ...(body.metadata?.annotations ?? {}),
            ...HostReconciler.runtimeTokenSecretAnnotations(
              host,
              tokens,
              revision,
              nowMs,
              gfs.expiresInSeconds,
              hasChannelIngress,
              observedFrontsOAuth,
              existing?.metadata?.annotations?.[GFS_TOKEN_HOST_UID_ANNOTATION],
              projection,
              grokProjection
            ),
            [RUNTIME_TOKEN_BOOTSTRAP_STATE_ANNOTATION]: RUNTIME_TOKEN_BOOTSTRAP_STATE_FRESH,
            [RUNTIME_TOKEN_ROLLOUT_REQUIRED_ANNOTATION]: rolloutRequired ? 'true' : 'false',
            ...((preserveWakeTemplate || rolloutRequired) &&
            HostReconciler.deploymentBelongsToHost(deployment, host)
              ? {
                  [RUNTIME_TOKEN_BOOTSTRAP_DEPLOYMENT_UID_ANNOTATION]:
                    deployment.metadata?.uid ?? '',
                  [RUNTIME_TOKEN_BOOTSTRAP_APPLIED_REVISION_ANNOTATION]: preserveWakeTemplate
                    ? HostReconciler.deploymentRuntimeTokenRevision(deployment)
                    : revision,
                }
              : {}),
            [RUNTIME_TOKEN_HAS_CHANNEL_INGRESS_ANNOTATION]: String(hasChannelIngress),
            [RUNTIME_TOKEN_FRONTS_OAUTH_ANNOTATION]: String(observedFrontsOAuth),
          },
        }

        if (!existing) {
          await observeCreate('Secret', () =>
            this.coreApi.createNamespacedSecret({ namespace: host.namespace, body })
          )
          this.recordGfsLifecycleEvidence(host, 'minted')
          log.info('created mcp-host-runtime-token Secret', {
            host: host.name,
            namespace: host.namespace,
            resourceName: name,
            reason: decision.reason,
          })
          return {
            revision: preserveWakeTemplate
              ? HostReconciler.deploymentRuntimeTokenRevision(deployment) || revision
              : revision,
            scopeHash,
            deploymentUid: deployment?.metadata?.uid,
            freshBootstrapPrepared:
              rolloutRequired && HostReconciler.deploymentBelongsToHost(deployment, host),
          }
        }

        const replaceBody = {
          ...body,
          metadata: {
            ...body.metadata,
            resourceVersion: existing.metadata?.resourceVersion,
          },
        }
        await this.coreApi.replaceNamespacedSecret({
          name,
          namespace: host.namespace,
          body: replaceBody,
        })
        this.recordGfsLifecycleEvidence(host, 'rotated')
        log.info('rotated mcp-host-runtime-token Secret', {
          host: host.name,
          namespace: host.namespace,
          resourceName: name,
          reason: decision.reason,
          rolloutRequired,
        })
        return {
          revision: preserveWakeTemplate
            ? HostReconciler.deploymentRuntimeTokenRevision(deployment) || revision
            : rolloutRequired
              ? revision
              : HostReconciler.deploymentRuntimeTokenRevision(deployment) || revision,
          scopeHash,
          deploymentUid: deployment?.metadata?.uid,
          freshBootstrapPrepared:
            rolloutRequired && HostReconciler.deploymentBelongsToHost(deployment, host),
        }
      } catch (err) {
        lastErr = err
        if (makeExpectedHostGfsSubject(host.namespace, host.name)) {
          this.recordGfsLifecycleEvidence(host, 'failed')
        }
        if (err instanceof Error && err.name === 'GfsHostTokenValidationError') break
        const delayMs = 1000 * Math.pow(2, attempt - 1)
        log.error('mcp-host-runtime-token Secret attempt failed', {
          host: host.name,
          namespace: host.namespace,
          attempt,
          maxAttempts,
          error: String(err),
        })
        if (attempt < maxAttempts) {
          await new Promise(resolve => setTimeout(resolve, delayMs))
        }
      }
    }
    throw new Error(
      `Failed to ensure mcp-host-runtime-token Secret for host "${host.name}" after ${maxAttempts} attempt${maxAttempts === 1 ? '' : 's'}: ${String(lastErr)}`
    )
  }

  private async provisionRuntimeTokenRevision(
    host: HostCRD,
    options: BootstrapOptions
  ): Promise<RuntimeTokenProvision | null> {
    try {
      return await this.ensureMcpHostRuntimeTokenSecret(host, options)
    } catch (err) {
      log.error('mcpHost runtime token provisioning failed', { host: host.name, err })
      this.setStatus(host.name, {
        deployed: false,
        ready: false,
        message: 'mcpHost runtime token provisioning failed',
      })
      throw err
    }
  }

  /**
   * Delete per-Host RBAC. Called from reconcileDelete. 404s are swallowed.
   */
  private async deleteHostRbac(name: string, namespace: string): Promise<void> {
    const saName = `host-${name}-sa`
    const roleName = `host-${name}-config-reader`
    const failures: unknown[] = []
    for (const cleanup of [
      () =>
        this.deleteIfHccOwned(
          'RoleBinding',
          roleName,
          namespace,
          name,
          () => this.rbacApi.readNamespacedRoleBinding({ name: roleName, namespace }),
          () => this.rbacApi.deleteNamespacedRoleBinding({ name: roleName, namespace })
        ),
      () =>
        this.deleteIfHccOwned(
          'Role',
          roleName,
          namespace,
          name,
          () => this.rbacApi.readNamespacedRole({ name: roleName, namespace }),
          () => this.rbacApi.deleteNamespacedRole({ name: roleName, namespace })
        ),
      () =>
        this.deleteIfHccOwned(
          'ServiceAccount',
          saName,
          namespace,
          name,
          () => this.coreApi.readNamespacedServiceAccount({ name: saName, namespace }),
          () => this.coreApi.deleteNamespacedServiceAccount({ name: saName, namespace })
        ),
    ]) {
      try {
        await cleanup()
      } catch (err) {
        failures.push(err)
      }
    }
    throwCleanupFailures(failures, `Failed to delete RBAC for Host "${name}"`)
  }

  private async deleteMcpHostRuntimeTokenSecret(name: string, namespace: string): Promise<void> {
    const secretName = `host-${name}-mcp-host-runtime-tokens`
    try {
      const secret = await this.coreApi.readNamespacedSecret({ name: secretName, namespace })
      const labels = secret.metadata?.labels ?? {}
      if (labels[MANAGED_BY_LABEL] !== MANAGED_BY_VALUE || labels[HOST_LABEL] !== name) {
        log.warn('Skipping runtime Secret delete - not HCC-owned', {
          name: secretName,
          host: name,
        })
        return
      }
      await this.coreApi.deleteNamespacedSecret({ name: secretName, namespace })
    } catch (err) {
      if (getErrorCode(err) !== 404) {
        log.error('Failed to delete runtime Secret', { name: secretName, err })
        throw err
      }
    }
  }

  private async deleteLegacyChannelReaderRuntimeAuth(name: string): Promise<void> {
    const suffix = ['mcp', 'host', 'runtime', 'auth'].join('-')
    const nameToDelete = `channel-reader-${name}-${suffix}`
    try {
      const obj = await this.coreApi.readNamespacedSecret({
        name: nameToDelete,
        namespace: config.channelsNamespace,
      })
      const labels = obj.metadata?.labels ?? {}
      const owned =
        labels[MANAGED_BY_LABEL] === MANAGED_BY_VALUE &&
        labels[HOST_LABEL] === name &&
        labels['clerum.io/component'] === 'channel-reader' &&
        labels['clerum.io/secret-purpose'] === suffix
      if (!owned) {
        log.warn('Skipping legacy runtime auth cleanup; labels do not prove HCC ownership', {
          name: nameToDelete,
        })
        return
      }
      await this.coreApi.deleteNamespacedSecret({
        name: nameToDelete,
        namespace: config.channelsNamespace,
      })
    } catch (err) {
      if (getErrorCode(err) !== 404) {
        log.error('Failed legacy runtime auth cleanup', { name: nameToDelete, err })
        throw err
      }
    }
  }

  getStatus(name: string): HostRuntimeStatus {
    return (
      this.statusMap.get(name) ?? { deployed: false, ready: false, message: 'Not reconciled yet' }
    )
  }

  /**
   * Track which hosts have desktop enabled.
   * Updated during reconcile, cleared during reconcileDelete.
   */
  private readonly desktopHosts: Set<string> = new Set()

  hasDesktop(hostRef: string): boolean {
    return this.desktopHosts.has(hostRef)
  }

  private setStatus(name: string, status: HostRuntimeStatus): void {
    this.statusMap.set(name, status)
    this.statusGenerations.set(name, this.reconcileGenerations.get(name) ?? 0)
  }

  private advanceReconcileGeneration(name: string): number {
    const generation = (this.reconcileGenerations.get(name) ?? 0) + 1
    this.reconcileGenerations.set(name, generation)
    return generation
  }

  private clearStatus(name: string): void {
    this.statusMap.delete(name)
    this.reconcileGenerations.delete(name)
    this.statusGenerations.delete(name)
    this.lifecycle.clearHost(name)
    const timer = this.readinessTimers.get(name)
    if (timer) {
      clearTimeout(timer)
      this.readinessTimers.delete(name)
    }
  }

  /**
   * Undefined when the snapshot carries no uid, which `HostCRD` documents as a
   * legal state. Sending the reference anyway would earn a 400 the reporter
   * treats as terminal, so the event would be dropped — and, unlike the
   * administrative path, re-sent and dropped again on every pass, because this
   * reporter keeps no `seen` set to fall silent. Every telemetry emitter goes
   * through here so the drop is decided, and named, in one place (#693).
   */
  private hostTelemetryReference(
    host: HostCRD,
    context: Record<string, string>
  ): HostLookupReference | undefined {
    if (host.uid === undefined) {
      log.warn('skipping host telemetry: Host snapshot has no uid', {
        host: host.name,
        namespace: host.namespace,
        ...context,
      })
      return undefined
    }
    return {
      name: host.name,
      namespace: host.namespace,
      ...(host.generation !== undefined ? { generation: host.generation } : {}),
      uid: host.uid,
    }
  }

  private enqueueHostTelemetry(
    host: HostCRD,
    telemetryType: 'lifecycle_transition' | 'reconcile_outcome' | 'controller_error',
    reasonCode: string,
    payload: HccInfrastructureTelemetryPayload
  ): void {
    const hostLookupReference = this.hostTelemetryReference(host, { telemetryType, reasonCode })
    if (hostLookupReference === undefined) return
    const occurredAt = this.now().toISOString()
    const eventPayload = {
      resource_class: 'Host',
      reason_code: reasonCode,
      ...payload,
    }
    if (telemetryType === 'reconcile_outcome') {
      this.infrastructureTelemetryReporter?.enqueue({
        occurredAt,
        telemetryType,
        hostLookupReference,
        payload: eventPayload,
      })
      return
    }
    this.infrastructureTelemetryReporter?.enqueue({
      sourceEventId: `hcc-${telemetryType}:${this.newTelemetryOccurrenceId()}`,
      occurredAt,
      telemetryType,
      hostLookupReference,
      payload: eventPayload,
    })
  }

  private enqueueControllerError(host: HostCRD, reasonCode: string, error: unknown): void {
    this.enqueueHostTelemetry(host, 'controller_error', reasonCode, {
      status: 'failed',
      error_class: error instanceof Error ? error.name : typeof error,
    })
    this.enqueueAdministrativeOutcome(host, 'failed', reasonCode)
  }

  private enqueueReconcileOutcome(host: HostCRD): void {
    const status = this.getStatus(host.name)
    const succeeded = status.deployed && status.ready
    this.enqueueReconcileTelemetry(host, succeeded ? 'ready' : 'not_ready', {
      status: succeeded ? 'succeeded' : 'failed',
      phase: status.deployed ? 'deployed' : 'not_deployed',
      state: status.ready ? 'ready' : 'not_ready',
    })
    if (succeeded) this.enqueueAdministrativeOutcome(host, 'succeeded', 'reconciled')
  }

  /**
   * A pass that threw may not have written any status yet, so the status in
   * memory can still be the previous pass's. Report the failure itself, not
   * that status, and never an administrative `succeeded` (#327).
   */
  private enqueueReconcileExceptionOutcome(host: HostCRD): void {
    this.enqueueReconcileTelemetry(host, 'reconcile_exception', { status: 'failed' })
  }

  private enqueueReconcileTelemetry(
    host: HostCRD,
    reasonCode: string,
    payload: HccInfrastructureTelemetryPayload
  ): void {
    const evidenceKey = HostReconciler.gfsLifecycleEvidenceKey(host)
    const gfsOutcome = evidenceKey ? this.gfsTokenLifecycleEvidence.get(evidenceKey) : undefined
    if (evidenceKey) this.gfsTokenLifecycleEvidence.delete(evidenceKey)
    this.enqueueHostTelemetry(host, 'reconcile_outcome', reasonCode, {
      ...payload,
      // control-api allowlists `transition`, not gfs_* keys (#328).
      ...(gfsOutcome ? { transition: `gfs_token:${gfsOutcome}` } : {}),
    })
  }

  private enqueueAdministrativeOutcome(
    host: HostCRD,
    outcome: 'succeeded' | 'failed',
    reasonCode: string
  ): void {
    const operationId = host.annotations?.['clerum.io/administrative-intent-id']
    if (!operationId || host.generation === undefined) return
    if (host.uid === undefined) {
      // control-api binds the outcome to the live object's metadata.uid and
      // refuses a reference without one, so an event sent now would be a 400
      // this reporter treats as terminal. Drop it here and say why (#694).
      log.warn('skipping administrative outcome: Host snapshot has no uid', {
        host: host.name,
        namespace: host.namespace,
        outcome,
      })
      return
    }
    this.administrativeOutcomeReporter?.enqueueHostOutcome({
      // v2: the identity gained the Host uid. sourceStatusRef feeds the
      // server's payload hash, so keeping the v1 key would make every live
      // key answer 409 and the uid-bearing claim would never be stored (#694).
      sourceEventId: `hcc-admin-outcome-v2:${operationId}:${host.generation}:${host.uid}:${outcome}`,
      occurredAt: this.now().toISOString(),
      hostRef: {
        name: host.name,
        namespace: host.namespace,
        generation: host.generation,
        uid: host.uid,
      },
      outcome,
      reasonCode,
    })
  }

  private pvcName(host: HostCRD): string {
    return `${host.name}-workspace`
  }

  // ─── Stateless lifecycle (Stage 2) ──────────────────────────────────
  //
  // Assessment, durable status writes, the heartbeat executors
  // (StatelessLifecycleReconcilerPort), the wake fast-path, and the
  // per-host serialization chain live in StatelessLifecycleExecutor
  // (statelessLifecycleExecutor.ts). The methods below are thin delegates
  // so external callers keep their HostReconciler entry points.

  /** Public view of the synchronous effective-lifecycle derivation. */
  getEffectiveLifecycle(host: HostCRD): EffectiveHostLifecycle {
    return this.lifecycle.getEffectiveLifecycle(host)
  }

  /** Public view of the clerum.io/wake-requested generation parser. */
  getWakeRequestedGeneration(host: HostCRD): number {
    return this.lifecycle.getWakeRequestedGeneration(host)
  }

  /** FRESH GET (not the watch cache) of a Host — see StatelessLifecycleExecutor. */
  async readFreshHost(host: HostCRD): Promise<HostCRD> {
    return this.lifecycle.readFreshHost(host)
  }

  /**
   * Suspend a stateless Host after the drained report — see
   * StatelessLifecycleExecutor. `entryWakeHandledGeneration` is the AP-1
   * generation epoch captured by the tracker when the suspend was decided.
   */
  async suspendHostFromHeartbeat(
    host: HostCRD,
    reason: string,
    entryWakeHandledGeneration: number
  ): Promise<SuspendFromHeartbeatOutcome> {
    return this.lifecycle.suspendHostFromHeartbeat(host, reason, entryWakeHandledGeneration)
  }

  /** Publish the D8 suspend-blocked reason — see StatelessLifecycleExecutor. */
  async publishSuspendBlockedReason(host: HostCRD, reason: string): Promise<void> {
    return this.lifecycle.publishSuspendBlockedReason(host, reason)
  }

  /**
   * Persist the tracker's drain decision durably — see
   * StatelessLifecycleExecutor. `entryWakeHandledGeneration` is the AP-1
   * generation epoch the tracker captured when the drain was decided.
   */
  async markHostDrainingFromHeartbeat(
    host: HostCRD,
    entryWakeHandledGeneration: number
  ): Promise<void> {
    return this.lifecycle.markHostDrainingFromHeartbeat(host, entryWakeHandledGeneration)
  }

  /** Tracker-side cancel-drain — see StatelessLifecycleExecutor. */
  async markHostActiveFromHeartbeat(host: HostCRD): Promise<void> {
    return this.lifecycle.markHostActiveFromHeartbeat(host)
  }

  /** Resolve a Host pod's creationTimestamp by pod UID (max-uptime ceiling). */
  async findPodCreationTimestamp(host: HostCRD, podUid: string): Promise<Date | null> {
    return this.lifecycle.findPodCreationTimestamp(host, podUid)
  }

  /**
   * Pull policy applied to this stateless Host's pod (Stage 6, W5). The
   * operator's resolved policy is preserved (never overridden to an
   * unpullable Always for a node-local image). When IfNotPresent is used with
   * a mutable image, the stale-cached-image-on-wake risk is warned once per
   * (host, image) and carried durably by the StatelessPullPolicyRejected
   * condition (assessLifecycle) — operator-visible, non-breaking.
   */
  private statelessImagePullPolicyForHost(
    host: HostCRD,
    image: string
  ): 'Always' | 'IfNotPresent' | 'Never' {
    const resolution = resolveStatelessImagePullPolicy(image)
    if (resolution.mutableImageRisk === undefined) {
      this.pullPolicyRejectionLogged.delete(host.name)
      return resolution.policy
    }
    if (this.pullPolicyRejectionLogged.get(host.name) !== image) {
      this.pullPolicyRejectionLogged.set(host.name, image)
      log.warn(
        'Stateless Host runs a mutable image reference (no @sha256: digest, tag not sha-<gitsha>): a node with a stale cached image serves old code on wake. Pin an immutable reference to eliminate the risk.',
        { host: host.name, imagePullPolicy: resolution.policy, image }
      )
    }
    return resolution.policy
  }

  async validateHostSecret(host: HostCRD): Promise<HostSecretValidationResult> {
    if (!host.spec.secretRef) {
      return { ok: true }
    }
    try {
      await this.coreApi.readNamespacedSecret({
        namespace: host.namespace,
        name: host.spec.secretRef,
      })
      return { ok: true }
    } catch (error) {
      const code = getErrorCode(error)
      if (code === 404) {
        const message = `Secret "${host.spec.secretRef}" not found in namespace "${host.namespace}"`
        log.error('Host Secret not found; Host will not be deployed', {
          host: host.name,
          detail: message,
        })
        return { ok: false, reason: 'SecretNotFound', message }
      }
      if (code === 401 || code === 403) {
        const message =
          `Access denied reading Secret "${host.spec.secretRef}" in namespace "${host.namespace}" ` +
          `(K8s API ${code})`
        log.error('Host Secret access denied; Host will be failed closed', {
          host: host.name,
          detail: message,
        })
        return { ok: false, reason: 'SecretAccessDenied', message }
      }
      const message = `Failed to validate Secret "${host.spec.secretRef}" for host "${host.name}"`
      log.error('Failed to validate Host Secret', { host: host.name, detail: message, err: error })
      return { ok: false, reason: 'ReadError', message }
    }
  }

  private buildPvc(host: HostCRD): k8s.V1PersistentVolumeClaim {
    const pvcName = this.pvcName(host)
    return {
      apiVersion: 'v1',
      kind: 'PersistentVolumeClaim',
      metadata: {
        name: pvcName,
        namespace: host.namespace,
        labels: {
          [MANAGED_BY_LABEL]: MANAGED_BY_VALUE,
          [HOST_LABEL]: host.name,
        },
      },
      spec: {
        storageClassName: config.hostWorkspaceStorageClassName,
        accessModes: ['ReadWriteOnce'],
        resources: {
          requests: {
            storage: config.hostWorkspaceStorageSize,
          },
        },
      },
    }
  }

  /**
   * Build the per-Host channel-reader Deployment manifest. Mirrors the static
   * deploy/base/channels/channel-reader.yaml shape, but per-Host:
   *   - name      = channel-reader-<host>
   *   - namespace = config.channelsNamespace
   *   - env       = CLERUM_HOST_REF=<host>
   *   - annotation clerum.io/credentials-revision=<sha256> when provided
   *
   * `secretRevision` is sha256(canonicalJSON(secret.data)); empty string
   * means "no Secret yet — boot with optional envFrom; SecretInformer will
   * patch the annotation later, triggering a roll".
   */
  private buildChannelReaderDeployment(
    host: HostCRD,
    secretRevision: string,
    existingReplicas?: number
  ): k8s.V1Deployment {
    const name = `channel-reader-${host.name}`
    const annotations: Record<string, string> = {}
    if (secretRevision) {
      annotations['clerum.io/credentials-revision'] = secretRevision
    }
    // CRITICAL: app.kubernetes.io/name MUST be unique per Host.
    //
    // The legacy static Deployment in deploy/base/channels/channel-reader.yaml
    // uses selector.matchLabels: app.kubernetes.io/name=channel-reader. K8s
    // Deployment selectors match supersets, so if per-Host pods carried the
    // plain "channel-reader" value, the static Deployment's ReplicaSet would
    // try to adopt them and delete N-1 every reconcile to converge on its
    // single-replica goal — producing a hot loop where per-Host Deployments
    // recreate pods that the static one immediately deletes.
    //
    // Per the K8s recommended-labels semantics, each per-Host Deployment is
    // a distinct application instance, so the unique-per-Host value is
    // appropriate. Coexistence with the static Deployment during migration
    // works correctly. The static Deployment will be retired in a follow-up.
    const appKubernetesName = `channel-reader-${host.name}`
    const labels = {
      app: 'channel-reader',
      'app.kubernetes.io/name': appKubernetesName,
      'app.kubernetes.io/part-of': 'clerum',
      [HOST_LABEL]: host.name,
      [MANAGED_BY_LABEL]: MANAGED_BY_VALUE,
    }
    // Pods carry the managed-by label so the per-Host NetworkPolicies can
    // require it in their podSelectors and refuse to apply to pods spoofed
    // into the channels namespace by another controller. Note: NOT included
    // in `selector.matchLabels` below — Deployment selectors are immutable
    // after create, so adding it there would require delete+recreate of
    // existing per-Host Deployments. Pod-template labels are mutable; new
    // pods on the next rollout get the label, NetworkPolicies match.
    const podLabels = {
      app: 'channel-reader',
      'app.kubernetes.io/name': appKubernetesName,
      'app.kubernetes.io/part-of': 'clerum',
      [HOST_LABEL]: host.name,
      [MANAGED_BY_LABEL]: MANAGED_BY_VALUE,
    }
    // Replica count is driven by CommunicationChannel count for this Host (#281):
    // 0 CCs → no pod runs (saves ~128Mi RAM on idle Hosts); 1+ CCs → one pod
    // polls. McpServerWatcher injects the counter via setCountCommunicationChannels;
    // the default returns 0, so any Host whose counter hasn't been wired up
    // scales to 0 (safe — Telegram polling halts but no resource leak).
    //
    // B2 — guard against spurious scale-to-0 during startup:
    // If the CC cache has NOT finished its initial list (ccCacheSyncedFn returns
    // false) AND we know the existing Deployment already has replicas=N, preserve
    // those replicas rather than forcing them to 0. This prevents a restart of
    // HCC from momentarily killing a live channel-reader before the CC cache is
    // populated. Once the cache is synced the normal ccCount logic takes over.
    const ccCount = this.countCommunicationChannels(host.name)
    const cacheIsSynced = this.ccCacheSyncedFn()
    const desiredReplicas =
      !cacheIsSynced && existingReplicas !== undefined ? existingReplicas : ccCount > 0 ? 1 : 0
    return {
      apiVersion: 'apps/v1',
      kind: 'Deployment',
      metadata: { name, namespace: config.channelsNamespace, labels },
      spec: {
        replicas: desiredReplicas,
        selector: { matchLabels: { app: 'channel-reader', [HOST_LABEL]: host.name } },
        template: {
          metadata: {
            labels: podLabels,
            ...(Object.keys(annotations).length > 0 ? { annotations } : {}),
          },
          spec: {
            serviceAccountName: 'clerum-channel-reader',
            containers: [
              {
                name: 'channel-reader',
                image: config.channelReaderImage,
                imagePullPolicy: config.channelReaderImagePullPolicy,
                ports: [
                  {
                    name: 'handoff',
                    containerPort: config.channelReaderHandoffPort,
                    protocol: 'TCP',
                  },
                ],
                envFrom: [{ configMapRef: { name: 'clerum-channel-reader-config' } }],
                env: [
                  { name: 'CLERUM_HOST_REF', value: host.name },
                  // Inject the pod's own namespace via the downward API so the
                  // channel-reader always discovers CommunicationChannels and
                  // credential Secrets in its own namespace (channels-<slug>
                  // under MCC, or "channels" in the default stack). This explicit
                  // env wins over any envFrom ConfigMap value, making the
                  // namespace correct by construction regardless of the ConfigMap.
                  {
                    name: 'CLERUM_NAMESPACE',
                    valueFrom: { fieldRef: { fieldPath: 'metadata.namespace' } },
                  },
                  // The mcp-host Service (buildService) is named host.name in
                  // host.namespace (mcp-host-<slug> under MCC), type ClusterIP,
                  // exposing the `http` port = config.hostPort. The channel-reader
                  // lives in channels-<slug>, so the URL MUST cross namespaces and
                  // use host.namespace — not config.channelsNamespace.
                  {
                    name: 'CLERUM_MCP_HOST_URL',
                    value: `http://${host.name}.${host.namespace}.svc.cluster.local:${config.hostPort}`,
                  },
                  {
                    name: 'CLERUM_CHANNEL_READER_HANDOFF_TOKEN',
                    valueFrom: {
                      secretKeyRef: {
                        name: 'workflow-approval-request-reader-credentials',
                        key: 'channel-reader-handoff-token',
                        optional: true,
                      },
                    },
                  },
                ],
                resources: {
                  requests: { cpu: '50m', memory: '128Mi' },
                  limits: { cpu: '200m', memory: '256Mi' },
                },
                securityContext: {
                  allowPrivilegeEscalation: false,
                  readOnlyRootFilesystem: true,
                  runAsNonRoot: true,
                  runAsUser: 1000,
                  runAsGroup: 1000,
                  // Restricted-PSA-compliant: MCC shared-tenant namespaces
                  // (channels-<slug>) enforce the `restricted` PodSecurity
                  // standard, which requires capabilities.drop=[ALL] and
                  // seccompProfile=RuntimeDefault.
                  capabilities: { drop: ['ALL'] },
                  seccompProfile: { type: 'RuntimeDefault' },
                },
              },
            ],
          },
        },
      },
    }
  }

  private buildChannelReaderService(host: HostCRD): k8s.V1Service {
    const name = `channel-reader-${host.name}`
    return {
      apiVersion: 'v1',
      kind: 'Service',
      metadata: {
        name,
        namespace: config.channelsNamespace,
        labels: {
          app: 'channel-reader',
          'app.kubernetes.io/name': name,
          'app.kubernetes.io/part-of': 'clerum',
          [HOST_LABEL]: host.name,
          [MANAGED_BY_LABEL]: MANAGED_BY_VALUE,
        },
      },
      spec: {
        type: 'ClusterIP',
        selector: {
          app: 'channel-reader',
          [HOST_LABEL]: host.name,
          [MANAGED_BY_LABEL]: MANAGED_BY_VALUE,
        },
        ports: [
          {
            name: 'handoff',
            port: config.channelReaderHandoffPort,
            targetPort: 'handoff' as IntOrString,
            protocol: 'TCP',
          },
        ],
      },
    }
  }

  private async reconcileChannelReaderService(
    host: HostCRD,
    revalidate?: () => void
  ): Promise<void> {
    const name = `channel-reader-${host.name}`
    const ns = config.channelsNamespace
    const service = this.buildChannelReaderService(host)
    const mutationAllowed = () => {
      revalidate?.()
      return true
    }
    await ensureResource({
      mutationAllowed,
      read: () =>
        observeExistenceRead('Service', () =>
          this.coreApi.readNamespacedService({ name, namespace: ns })
        ),
      create: () =>
        observeCreate('Service', () =>
          this.coreApi.createNamespacedService({ namespace: ns, body: service })
        ),
      onSkipped: () => createsTotal.inc({ kind: 'Service', outcome: 'skipped' }),
      converge: read =>
        replaceWithConflictRetry({
          description: `channel-reader Service "${name}"`,
          logPrefix: '[HostReconciler]',
          body: service,
          mergeExisting: preserveServiceAssignedFields,
          isUpToDate: serviceMatchesDesired,
          mutationAllowed,
          read,
          replace: body => this.coreApi.replaceNamespacedService({ name, namespace: ns, body }),
        }),
    })
  }

  /**
   * Idempotent reconcile of the per-Host channel-reader Deployment. Mirrors
   * the PVC create-or-replace pattern at hostReconciler.ts:497-519.
   *
   * - Reads existing per-host credentials Secret to seed the
   *   clerum.io/credentials-revision pod template annotation, so the initial
   *   pod boots with the right hash and doesn't get rolled twice when
   *   SecretInformer fires later.
   * - Reads before creation and compares a server-normalized Deployment; it
   *   replaces only meaningful drift. Conflict retries rebuild against the
   *   latest live replica count. A Deployment owned by a different host is
   *   never overwritten: the initial read skips, and a retry-time ownership
   *   change throws so write_skips_total does not count the refusal as a
   *   healthy no-op.
   */
  private async reconcileChannelReaderDeployment(
    host: HostCRD,
    revalidate?: () => void
  ): Promise<void> {
    const name = `channel-reader-${host.name}`
    const ns = config.channelsNamespace
    const revision = await this.computeChannelReaderRevisionForHost(host.name)
    const desired = this.buildChannelReaderDeployment(host, revision)
    const mutationAllowed = () => {
      revalidate?.()
      return true
    }
    await ensureResource({
      mutationAllowed,
      read: () =>
        observeExistenceRead('Deployment', () =>
          this.appsApi.readNamespacedDeployment({ name, namespace: ns })
        ),
      create: () =>
        observeCreate('Deployment', () =>
          this.appsApi.createNamespacedDeployment({ namespace: ns, body: desired })
        ),
      onSkipped: () => createsTotal.inc({ kind: 'Deployment', outcome: 'skipped' }),
      converge: async (read, observed) => {
        // A post-create-conflict read still throws on 404. Later retry reads
        // keep the existing benign-disappearance policy in the retry helper.
        const existing = observed ?? (await read())
        const ownerHost = existing.metadata?.labels?.[HOST_LABEL]
        if (ownerHost && ownerHost !== host.name) {
          log.warn('Channel-reader Deployment belongs to another Host; refusing update', { name })
          return false
        }
        // ensureResource already supplies a consume-once snapshot for the
        // initial presence path. Only POST409 needs its owner-check read retained.
        let snapshot: k8s.V1Deployment | undefined = observed === undefined ? existing : undefined
        await replaceWithConflictRetry({
          description: `channel-reader Deployment "${name}"`,
          logPrefix: '[HostReconciler]',
          body: this.buildChannelReaderDeployment(host, revision, existing.spec?.replicas),
          mutationAllowed,
          read: () => {
            if (snapshot !== undefined) {
              const first = snapshot
              snapshot = undefined
              return Promise.resolve(first)
            }
            return read()
          },
          replace: body => this.appsApi.replaceNamespacedDeployment({ name, namespace: ns, body }),
          mergeExisting: (_body, fresh) => {
            const desiredWithLiveReplicas = this.buildChannelReaderDeployment(
              host,
              revision,
              fresh.spec?.replicas
            )
            desiredWithLiveReplicas.metadata = {
              ...desiredWithLiveReplicas.metadata,
              resourceVersion: fresh.metadata?.resourceVersion,
            }
            return preserveChannelReaderDeploymentAnnotations(desiredWithLiveReplicas, fresh)
          },
          validateExisting: fresh => {
            const freshOwnerHost = fresh.metadata?.labels?.[HOST_LABEL]
            if (freshOwnerHost && freshOwnerHost !== host.name) {
              throw new Error(
                `channel-reader Deployment ownership changed to host "${freshOwnerHost}"; refusing replace`
              )
            }
          },
          isUpToDate: deploymentMatchesDesired,
        })
      },
    })
  }

  /**
   * Patch the `clerum.io/credentials-revision` annotation on the per-Host
   * channel-reader Deployment so K8s rolls the pod. Revision = sha256 of
   * the host's credentials Secret content (empty when no per-CC Secret
   * resolves yet). Idempotent on unchanged input — patches the same value
   * are no-ops at the K8s layer.
   *
   * Strategic-merge-patch preserves any other annotations on the pod
   * template. 404 on the Deployment is benign (no per-Host pod for that
   * host yet — McpServerWatcher's fullReconcile will create one when the
   * Host CRD is present).
   */
  async patchChannelReaderRevisionAnnotation(hostName: string): Promise<void> {
    const capturedAuthority = this.captureHostMutationAuthority()
    const depName = `channel-reader-${hostName}`
    try {
      await this.lifecycle.serializeByHost(hostName, async () => {
        const action = `channel-reader "${hostName}" revision patch`
        this.requireHostMutationAuthority(action, capturedAuthority)

        const resolveCurrentHost = this.resolveCurrentHost
        const admittedHost = resolveCurrentHost?.(hostName)
        if (resolveCurrentHost !== null && admittedHost === undefined) return
        const revalidate = admittedHost
          ? this.makeHostMutationAdmission(
              action,
              admittedHost,
              capturedAuthority,
              this.resolveHostMutationDependencies?.(admittedHost)
            )
          : () => this.requireHostMutationAuthority(action, capturedAuthority)

        revalidate()
        const revision = await this.computeChannelReaderRevisionForHost(hostName)
        // The revision calculation can await multiple Secret reads. Revalidate
        // Host LIST/WATCH authority plus UID and generation immediately before
        // entering the Kubernetes mutation, while still holding this Host lane.
        revalidate()
        const patchBody = {
          spec: {
            template: {
              metadata: {
                annotations: { 'clerum.io/credentials-revision': revision },
              },
            },
          },
        }
        await this.appsApi.patchNamespacedDeployment(
          { name: depName, namespace: config.channelsNamespace, body: patchBody },
          {
            middleware: [
              k8s.setHeaderMiddleware('Content-Type', 'application/strategic-merge-patch+json'),
            ],
          }
        )
        log.info('Patched channel-reader credentials-revision', {
          deployment: depName,
          revision: revision || '(empty)',
        })
      })
    } catch (err) {
      if (getErrorCode(err) === 404) return
      log.error('Failed to reconcile channel-reader credentials-revision', {
        deployment: depName,
        err,
      })
    }
  }

  /**
   * Compute the revision for a host by hashing the data of the Secret(s)
   * referenced by all CCs that resolve to this host. Used by
   * patchChannelReaderRevisionAnnotation. Returns '' when no Secret is
   * resolvable yet.
   *
   * Wiring: a `findCommunicationChannelsByHostRef` dependency is injected
   * via setFindCommunicationChannelsByHostRef from McpServerWatcher.
   */
  private async computeChannelReaderRevisionForHost(hostName: string): Promise<string> {
    const ccs = this.findCommunicationChannelsByHostRef(hostName)
    const secretNames = new Set<string>()
    for (const cc of ccs) {
      if (cc.spec.credentialsSecretRef?.name) secretNames.add(cc.spec.credentialsSecretRef.name)
    }
    if (secretNames.size === 0) return ''

    const sortedNames = [...secretNames].sort()
    const combined: Record<string, Record<string, string>> = {}
    for (const name of sortedNames) {
      try {
        const secret = await this.coreApi.readNamespacedSecret({
          name,
          namespace: config.channelsNamespace,
        })
        combined[name] = (secret.data ?? {}) as Record<string, string>
      } catch (err) {
        if (getErrorCode(err) === 404) continue
        throw err
      }
    }
    const canonical = canonicalStringify(combined)
    return createHash('sha256').update(canonical).digest('hex')
  }

  /**
   * Called by the channels-namespace SecretInformer (started in main.ts) on
   * Secret rotation events. Looks up which CommunicationChannels reference
   * the rotated Secret via `findCommunicationChannelsByCredentialsSecretName`
   * (CC cache reverse-map), collects the affected hosts, and patches each
   * host's channel-reader Deployment annotation exactly once via
   * `patchChannelReaderRevisionAnnotation`.
   *
   * Cross-namespace events are skipped early — only Secrets in the
   * configured channelsNamespace are interesting (this avoids spurious
   * cache lookups when the informer fires for unrelated Secrets).
   *
   * A single Secret can map to multiple hosts (operators can share a
   * Secret across CCs of different Hosts via raw YAML); each affected host
   * is patched once even if it owns multiple CCs that reference the same
   * Secret.
   *
   * When no CC references the Secret, this is a no-op — the Secret is
   * either orphaned or owned by a different subsystem.
   *
   * Per-host error handling lives in `patchChannelReaderRevisionAnnotation`
   * (swallows Deployment 404 / Secret 404, logs others). This method does
   * NOT throw, so the SecretInformer event loop continues regardless of
   * per-host failures.
   */
  async reconcileChannelReaderRevision(secretName: string, secretNamespace: string): Promise<void> {
    if (secretNamespace !== config.channelsNamespace) return

    const affectedCCs = this.findCommunicationChannelsByCredentialsSecretName(secretName)
    if (affectedCCs.length === 0) return

    const affectedHosts = new Set<string>(affectedCCs.map(cc => cc.spec.hostRef))
    // Addendum 2: each affected Host patches only its own `channel-reader-<host>`
    // Deployment annotation — a disjoint, order-independent, idempotent write set
    // per Host (§10.4 inventory). Fan out through the bounded worker pool
    // (reusing HCC_HOST_FULL_RECONCILE_CONCURRENCY) so a shared credentials-Secret
    // change touching many Hosts is not head-of-line blocked, while never issuing
    // an unbounded burst at the K8s API. patchChannelReaderRevisionAnnotation
    // still swallows its own per-Host errors, so this method continues not to
    // throw (runBoundedHostWorkers never rethrows).
    await runBoundedHostWorkers(
      Array.from(affectedHosts),
      config.hostFullReconcileConcurrency,
      hostName => this.patchChannelReaderRevisionAnnotation(hostName)
    )
  }

  buildDeployment(
    host: HostCRD,
    mounts: ResolvedSfsMount[] = [],
    runtimeTokenRevision = '',
    lifecycle?: EffectiveHostLifecycle,
    grokExecutionEnabled = this.hostDerivesGrokExecution(host)
  ): k8s.V1Deployment {
    const labels: Record<string, string> = {
      app: host.name,
      [MANAGED_BY_LABEL]: MANAGED_BY_VALUE,
      [HOST_LABEL]: host.name,
      [CONTEXT_LABEL]: host.spec.contextRef,
    }

    const isDesktop = !!(host.spec.desktop?.browser || host.spec.desktop?.x11)
    // reconcile() passes the full async lifecycle assessment; direct callers
    // (and tests) fall back to the synchronously derivable view.
    const effectiveLifecycle = lifecycle ?? this.lifecycle.effectiveLifecycleFromCache(host)
    const isStateless = effectiveLifecycle.stateless
    const image = isDesktop ? config.desktopImage : config.hostImage
    const resources = isDesktop ? config.desktopResources : config.hostResources
    // Stage 6 (W5): stateless pods may override the pull policy (a mutable
    // image is reported, not refused — see resolveStatelessImagePullPolicy). Non-stateless pods keep the
    // global policy so their manifest stays byte-identical.
    const imagePullPolicy = isStateless
      ? this.statelessImagePullPolicyForHost(host, image)
      : config.hostImagePullPolicy

    // Ports
    const ports: Array<{ name: string; containerPort: number; protocol: string }> = [
      { name: 'http', containerPort: config.hostPort, protocol: 'TCP' },
    ]
    if (isDesktop) {
      ports.push({ name: 'desktop', containerPort: config.desktopPort, protocol: 'TCP' })
    }

    // When desktop is enabled, mount the workspace at /config/workspace so the
    // XFCE user (abc, UID 1001) can browse the same files the agent writes to.
    const workspacePath = isDesktop ? '/config/workspace' : config.hostWorkspacePath
    const reservedMountPaths = [
      workspacePath,
      '/tmp',
      WORKFLOW_TOKEN_MOUNT_PATH,
      MCP_HOST_RUNTIME_AUTH_STATE_PATH,
    ]
    const contextMounts = mounts.filter(m => {
      const reason = HostReconciler.contextMountPathRejectionReason(m.mountPath, reservedMountPaths)
      if (reason) {
        log.warn('Skipping invalid SharedFileSystem context mount', {
          host: host.name,
          sharedFileSystem: m.name,
          mountPath: m.mountPath,
          reason,
        })
        return false
      }
      return true
    })

    // Env
    const env: k8s.V1EnvVar[] = [
      { name: 'CLERUM_HOST_NAME', value: host.name },
      {
        name: 'CLERUM_NAMESPACE',
        valueFrom: { fieldRef: { fieldPath: 'metadata.namespace' } },
      },
      { name: 'CLERUM_WORKSPACE_PATH', value: workspacePath },
      // Changing spec.secretRef changes the pod template and rolls the host.
      { name: 'CLERUM_LLM_SECRET_REF', value: host.spec.secretRef ?? '' },
      // mcpHost runtime token env vars. Names match WRC's podFactory.ts so mcp-host
      // sees the same shape regardless of which controller provisioned it.
      {
        name: 'MCP_HOST_RUNTIME_ACCESS_TOKEN',
        valueFrom: {
          secretKeyRef: {
            name: mcpHostRuntimeTokenSecretName(host),
            key: MCP_HOST_RUNTIME_TOKEN_SECRET_ACCESS_KEY,
          },
        },
      },
      {
        name: 'MCP_HOST_RUNTIME_REFRESH_TOKEN',
        valueFrom: {
          secretKeyRef: {
            name: mcpHostRuntimeTokenSecretName(host),
            key: MCP_HOST_RUNTIME_TOKEN_SECRET_REFRESH_KEY,
          },
        },
      },
      {
        name: 'MCP_HOST_WORKFLOW_CONTROL_TOKEN',
        valueFrom: {
          secretKeyRef: {
            name: mcpHostRuntimeTokenSecretName(host),
            key: MCP_HOST_RUNTIME_TOKEN_SECRET_CONTROL_KEY,
          },
        },
      },
      {
        name: 'MCP_HOST_WORKFLOW_CONTROL_TOKEN_FILE',
        value: MCP_HOST_WORKFLOW_CONTROL_TOKEN_FILE_PATH,
      },
      {
        name: 'MCP_HOST_RUNTIME_AUTH_STATE_DIR',
        value: MCP_HOST_RUNTIME_AUTH_STATE_PATH,
      },
      { name: 'MCP_HOST_GATEWAY_URL', value: config.mcpHostGatewayUrl },
    ]
    // mcp-host refuses to construct the grok-subscription provider unless this
    // switch is on. Emit it only when this Host's Grok projection mints
    // llm:grok:execute (same gate as the scope and grok-proxy egress policy),
    // so an eligibility flip changes the pod template and rolls the Host.
    // Non-Grok Hosts keep a byte-identical template.
    if (grokExecutionEnabled) {
      env.push({ name: 'MCP_HOST_GROK_SUBSCRIPTION_ENABLED', value: 'true' })
    }
    // One storage layout for every lifecycle (issue #825): the conversation
    // database lives at the workspace PVC root, so a lifecycle flip changes
    // compute only and never where the history is read from.
    env.push(
      { name: 'CLERUM_SESSION_STORE', value: 'sqlite' },
      { name: 'CLERUM_SESSION_DB_DIR', value: workspacePath }
    )
    if (isStateless) {
      env.push(
        { name: 'CLERUM_STATELESS_LIFECYCLE', value: 'true' },
        // Stage 3: the heartbeat emitter requires its own pod UID so HCC
        // can discard heartbeats from stale pods after a wake.
        {
          name: 'CLERUM_POD_UID',
          valueFrom: { fieldRef: { fieldPath: 'metadata.uid' } },
        }
      )
    }
    if (isDesktop) {
      env.push(
        { name: 'CLERUM_DESKTOP_X11', value: String(host.spec.desktop?.x11 ?? false) },
        { name: 'CLERUM_DESKTOP_BROWSER', value: String(host.spec.desktop?.browser ?? false) },
        { name: 'PUID', value: '1001' },
        { name: 'PGID', value: '1001' },
        { name: 'TZ', value: 'UTC' },
        { name: 'CUSTOM_PORT', value: String(config.desktopPort) }
      )
    }
    if (contextMounts.length > 0) {
      // Sort by name for stable JSON ordering across reconciles — otherwise a
      // map iteration reorder would flip the env value and trigger an unnecessary
      // pod restart.
      const sorted = [...contextMounts].sort((a, b) => a.name.localeCompare(b.name))
      env.push({
        name: 'CLERUM_CONTEXT_FILES_MOUNTS',
        value: JSON.stringify(
          sorted.map(m => ({
            name: m.name,
            namespace: m.namespace,
            mountPath: m.mountPath,
            pvcName: m.pvcName,
          }))
        ),
      })
    }
    const podAnnotations: Record<string, string> = {}
    if (runtimeTokenRevision) {
      podAnnotations[RUNTIME_TOKEN_REVISION_ANNOTATION] = runtimeTokenRevision
    }
    // Roll the mcp-host pod whenever the guardrails block changes. mcp-host does
    // re-resolve guardrails live, but only while its Host watch is connected, so
    // the roll is what delivers the change to an agent whose watch has lapsed.
    // Stamp a hash of the block: present→changed flips the value, and removal
    // omits the key — which only reaches the pod because
    // preserveHostDeploymentAnnotations strips it from the merge instead of
    // letting the live value survive (an omitted key alone produces no diff).
    if (host.spec.guardrails) {
      podAnnotations[GUARDRAILS_REVISION_ANNOTATION] = createHash('sha256')
        .update(deepStableStringify(host.spec.guardrails))
        .digest('hex')
    }

    // Security context — desktop needs runAsNonRoot: false for s6-overlay init,
    // which is fundamentally incompatible with the `restricted` PodSecurity
    // standard. Desktop Hosts therefore won't schedule in MCC shared-tenant
    // namespaces under restricted PSA — a separate known limitation, out of
    // scope here. Non-desktop hosts emit a restricted-PSA-compliant pod-level
    // securityContext so the ReplicaSet schedules in restricted namespaces.
    const securityContext = isDesktop
      ? { runAsNonRoot: false, seccompProfile: { type: 'RuntimeDefault' }, fsGroup: 1001 }
      : {
          runAsNonRoot: true,
          // UID/GID 1001 matches the mcp-host image's baked-in `nodejs` user
          // (Dockerfile / .slim / .full all create it at 1001). Running as the
          // image user keeps the running UID aligned with the workspace PVC
          // ownership written by prior rolls — re-rolling existing 1st-party
          // hosts (chatllm, etc.) does not hit EACCES on workspace/memory file
          // rewrites. fsGroup stays 1001 so fresh-PVC tenants still work.
          runAsUser: 1001,
          runAsGroup: 1001,
          fsGroup: 1001,
          seccompProfile: { type: 'RuntimeDefault' },
        }

    // Startup probe — desktop image needs longer initial delay for s6 boot + XFCE + mcp-host launch.
    // Stateless wake-budget tuning (Stage 6, W7): a woken pod should turn
    // Ready as soon as it can answer, so the stateless startup probe drops
    // the initial delay and polls every 2s. The total startup allowance is
    // preserved: non-stateless allows 10 + 5s × 60 = 310s, stateless allows
    // 0 + 2s × 155 = 310s (stateless is never desktop — assessLifecycle
    // rejects spec.desktop). Non-stateless pods keep the exact legacy probe.
    const startupProbe = isStateless
      ? {
          httpGet: { path: '/v1/runtime/live', port: 'http' as IntOrString },
          initialDelaySeconds: 0,
          periodSeconds: 2,
          timeoutSeconds: 2,
          failureThreshold: 155,
        }
      : {
          httpGet: { path: '/v1/runtime/live', port: 'http' as IntOrString },
          initialDelaySeconds: isDesktop ? 30 : 10,
          periodSeconds: 5,
          timeoutSeconds: 2,
          failureThreshold: isDesktop ? 120 : 60,
        }

    return {
      apiVersion: 'apps/v1',
      kind: 'Deployment',
      metadata: {
        name: host.name,
        namespace: host.namespace,
        labels,
        ...(host.uid ? { annotations: { [HOST_UID_ANNOTATION]: host.uid } } : {}),
      },
      spec: {
        // A suspended stateless Host scales to 0 on EVERY reconcile path
        // (event + resync): the state lives in the CRD status, so neither a
        // routine reconcile nor an HCC restart resurrects it.
        replicas: isStateless && effectiveLifecycle.state === 'suspended' ? 0 : 1,
        // Recreate: the old Pod is gone before the new one starts, so two
        // Pods never share the RWO workspace PVC (a RollingUpdate with
        // maxSurge 0 still starts the new Pod while the old one terminates).
        strategy: { type: 'Recreate' },
        selector: {
          matchLabels: { app: host.name },
        },
        template: {
          metadata: {
            labels,
            ...(Object.keys(podAnnotations).length > 0 ? { annotations: podAnnotations } : {}),
          },
          spec: {
            // Per-Host ServiceAccount provisioned by
            // ensureHostServiceAccount() — bound to a narrow Role that lets
            // mcp-host watch only its own Host CRD, env CM/Secret, and LLM
            // Secret.
            serviceAccountName: this.hostSaName(host),
            // A stateless Host may be recreated after a user interaction.
            // Give only that on-demand workload the interactive class so its
            // wake can preempt explicitly lower-priority batch work. Keep
            // stateful Hosts unclassed at the default service-equivalent
            // priority to preserve their existing PodSpec.
            ...(isStateless ? { priorityClassName: 'clerum-interactive-host' } : {}),
            imagePullSecrets: config.hostImagePullSecretName
              ? [{ name: config.hostImagePullSecretName }]
              : undefined,
            // While CONTEXT_MAPPER_WORKSPACE_LAYOUT_MERGE is on, every Host
            // merges a PVC split by the earlier stateless layout back to the
            // root layout before mcp-host starts; on any other tree the init
            // exits 0 without touching it.
            initContainers: config.workspaceLayoutMerge
              ? [
                  buildWorkspaceLayoutMergeContainer(
                    image,
                    imagePullPolicy,
                    hostContainerSecurityContext()
                  ),
                ]
              : undefined,
            containers: [
              {
                name: 'mcp-host',
                image,
                imagePullPolicy,
                ports,
                envFrom: [{ configMapRef: { name: config.hostConfigMapName } }],
                env,
                volumeMounts: [
                  { name: 'workspace', mountPath: workspacePath },
                  {
                    name: 'mcp-host-runtime-tokens',
                    mountPath: WORKFLOW_TOKEN_MOUNT_PATH,
                    readOnly: true,
                  },
                  { name: 'workflow-auth-state', mountPath: MCP_HOST_RUNTIME_AUTH_STATE_PATH },
                  { name: 'tmp', mountPath: '/tmp' },
                  ...contextMounts.map(m => ({
                    name: HostReconciler.contextMountVolumeName(m),
                    mountPath: m.mountPath,
                    readOnly: true,
                  })),
                ],
                startupProbe,
                livenessProbe: {
                  httpGet: { path: '/v1/runtime/live', port: 'http' as IntOrString },
                  initialDelaySeconds: 45,
                  periodSeconds: 30,
                  timeoutSeconds: 2,
                  failureThreshold: 3,
                },
                readinessProbe: {
                  httpGet: { path: '/v1/runtime/health', port: 'http' as IntOrString },
                  // Stateless (Stage 6, W7): the startup probe already gates
                  // readiness checks until the pod is live, so the fixed 20s
                  // readiness delay only pads the wake budget — drop it.
                  initialDelaySeconds: isStateless ? 0 : 20,
                  periodSeconds: 10,
                  timeoutSeconds: 2,
                  failureThreshold: 6,
                },
                resources: {
                  requests: {
                    memory: resources.requests.memory,
                    cpu: resources.requests.cpu,
                  },
                  limits: {
                    memory: resources.limits.memory,
                    cpu: resources.limits.cpu,
                  },
                },
                securityContext: hostContainerSecurityContext(),
              },
            ],
            securityContext,
            // #592: co-locate the mcp-host pod on the same node as each mounted
            // SharedFileSystem's wfc pod. The SFS PVC is RWO, so the RW writer
            // (wfc) and these RO consumers must share a node. Mounts are gated on
            // the SFS PVC being Bound (resolveContextMounts → isMountable); under
            // WaitForFirstConsumer a Bound PVC means the wfc (first consumer) was
            // already scheduled, so its affinity target exists. Limit: a Host
            // mounting 2+ SFS whose wfc landed on different nodes is unschedulable
            // (surfaced as Degraded) — that case needs RWX (deferred) and is not
            // used today (Contexts mount 0-1 SFS).
            //
            // LATENT (see issue #602 — RWX is not enabled on any cluster today):
            // this podAffinity is UNCONDITIONAL — it applies to every mount
            // regardless of access mode. If RWX is ever enabled and a Host mounts
            // an explicit ReadWriteMany SFS, this `required` hostname affinity
            // would still force its RO consumers onto the wfc's single node,
            // defeating RWX's multi-node-reader benefit. It is harmless today: no
            // RWX StorageClass exists on dev/prod, so an RWX PVC never binds →
            // the isMountable (PVC-Bound) gate keeps it out of contextMounts → it
            // never reaches this affinity. WHEN RWX lands: plumb the access mode
            // into ResolvedSfsMount and apply this term only to RWO mounts (RWX
            // mounts must get NO podAffinity). Do NOT add that conditioning now —
            // it would be untestable without an RWX StorageClass. See #602.
            ...(contextMounts.length > 0
              ? {
                  affinity: {
                    podAffinity: {
                      requiredDuringSchedulingIgnoredDuringExecution: contextMounts.map(m => ({
                        labelSelector: {
                          matchLabels: {
                            app: WFC_APP_LABEL,
                            [SFS_LABEL]: m.name,
                            [SFS_NAMESPACE_LABEL]: m.namespace,
                          },
                        },
                        topologyKey: 'kubernetes.io/hostname',
                      })),
                    },
                  },
                }
              : {}),
            volumes: [
              { name: 'workspace', persistentVolumeClaim: { claimName: this.pvcName(host) } },
              {
                name: 'mcp-host-runtime-tokens',
                secret: { secretName: mcpHostRuntimeTokenSecretName(host) },
              },
              { name: 'workflow-auth-state', emptyDir: { sizeLimit: '16Mi' } },
              { name: 'tmp', emptyDir: {} },
              ...contextMounts.map(m => ({
                name: HostReconciler.contextMountVolumeName(m),
                persistentVolumeClaim: { claimName: m.pvcName, readOnly: true },
              })),
            ],
          },
        },
      },
    }
  }

  private buildService(host: HostCRD): k8s.V1Service {
    const isDesktop = !!(host.spec.desktop?.browser || host.spec.desktop?.x11)

    const ports: Array<{ name: string; port: number; targetPort: IntOrString; protocol: string }> =
      [
        {
          name: 'http',
          port: config.hostPort,
          targetPort: 'http' as IntOrString,
          protocol: 'TCP',
        },
      ]

    if (isDesktop) {
      ports.push({
        name: 'desktop',
        port: config.desktopPort,
        targetPort: 'desktop' as IntOrString,
        protocol: 'TCP',
      })
    }

    return {
      apiVersion: 'v1',
      kind: 'Service',
      metadata: {
        name: host.name,
        namespace: host.namespace,
        labels: {
          app: host.name,
          [MANAGED_BY_LABEL]: MANAGED_BY_VALUE,
          [HOST_LABEL]: host.name,
        },
      },
      spec: {
        type: 'ClusterIP',
        selector: { app: host.name },
        ports,
      },
    }
  }

  private async checkDeploymentReady(name: string, namespace: string): Promise<boolean> {
    try {
      const deployment = await this.appsApi.readNamespacedDeployment({ name, namespace })
      return (deployment.status?.readyReplicas ?? 0) > 0
    } catch {
      return false
    }
  }

  /**
   * Best-effort read of the per-Host channel-reader Deployment status.
   * Never throws — a read failure returns a degraded status so the caller
   * can surface it without blocking mcp-host reconcile.
   */
  private async checkChannelReaderStatus(
    hostName: string,
    ccCount: number
  ): Promise<HostChannelReaderStatus> {
    const depName = `channel-reader-${hostName}`
    const ns = config.channelsNamespace
    const expected = ccCount > 0

    try {
      const dep = await this.appsApi.readNamespacedDeployment({ name: depName, namespace: ns })
      const readyReplicas = dep.status?.readyReplicas ?? 0
      const desiredReplicas = dep.spec?.replicas ?? 0
      const ready = readyReplicas >= 1

      if (!expected) {
        // No CCs — Deployment may legitimately be scaled to 0.
        return {
          expected: false,
          ready: desiredReplicas > 0 && ready,
          message:
            desiredReplicas > 0 && ready
              ? 'Running (no CommunicationChannels)'
              : 'Scaled to 0 (no CommunicationChannels)',
        }
      }

      return {
        expected: true,
        ready,
        message: ready ? 'Running' : 'Deployment exists but no ready replicas',
      }
    } catch (err) {
      const code = getErrorCode(err)
      if (code === 404) {
        return {
          expected,
          ready: false,
          message: expected ? 'Deployment not found' : 'Not deployed (no CommunicationChannels)',
        }
      }
      // Unknown error — surface it but don't throw.
      log.warn('Failed to read channel-reader Deployment for status', { deployment: depName, err })
      return {
        expected,
        ready: false,
        message: `Status read error: ${(err as Error).message}`,
      }
    }
  }

  private pollReadiness(
    name: string,
    namespace: string,
    intervalMs = 5000,
    maxAttempts = 12
  ): void {
    const pollGeneration = this.reconcileGenerations.get(name) ?? 0
    const existing = this.readinessTimers.get(name)
    if (existing) {
      clearTimeout(existing)
      this.readinessTimers.delete(name)
    }

    let attempts = 0

    const poll = async () => {
      attempts++
      const ready = await this.checkDeploymentReady(name, namespace)
      if (ready) {
        if (
          (this.reconcileGenerations.get(name) ?? 0) !== pollGeneration ||
          (this.statusGenerations.get(name) ?? 0) > pollGeneration
        ) {
          this.readinessTimers.delete(name)
          return
        }
        this.setStatus(name, { deployed: true, ready: true, message: 'Running' })
        this.readinessTimers.delete(name)
        return
      }
      if (attempts >= maxAttempts) {
        this.readinessTimers.delete(name)
        return
      }
      const timer = setTimeout(poll, intervalMs)
      this.readinessTimers.set(name, timer)
    }

    const timer = setTimeout(poll, intervalMs)
    this.readinessTimers.set(name, timer)
  }

  private async ensurePvc(host: HostCRD, revalidate?: () => void): Promise<boolean> {
    const pvc = this.buildPvc(host)
    const name = this.pvcName(host)
    let createFailed = false
    const mutationAllowed = () => {
      revalidate?.()
      return true
    }
    // Initial GET errors must propagate; retain only the established
    // non-throwing POST and convergence failures of this PVC writer.
    const result = await ensureResource({
      mutationAllowed,
      read: () =>
        observeExistenceRead('PersistentVolumeClaim', () =>
          this.coreApi.readNamespacedPersistentVolumeClaim({ namespace: host.namespace, name })
        ),
      create: async () => {
        try {
          await observeCreate('PersistentVolumeClaim', () =>
            this.coreApi.createNamespacedPersistentVolumeClaim({
              namespace: host.namespace,
              body: pvc,
            })
          )
        } catch (error) {
          if (error != null && getErrorCode(error) === 409) throw error
          createFailed = true
          log.error('Failed to create Host PVC', { host: host.name, err: error })
        }
      },
      converge: async read => {
        try {
          const existing = await read()
          revalidate?.()
          if (existing.spec?.volumeName) return
          pvc.metadata!.resourceVersion = existing.metadata?.resourceVersion
          await this.coreApi.replaceNamespacedPersistentVolumeClaim({
            namespace: host.namespace,
            name,
            body: pvc,
          })
        } catch (error) {
          if (isBenignSupersessionError(error)) throw error
          log.error('Failed to update Host PVC', { host: host.name, err: error })
          return false
        }
      },
      onSkipped: () => createsTotal.inc({ kind: 'PersistentVolumeClaim', outcome: 'skipped' }),
    })
    return !createFailed && this.resourceApplySucceeded(result)
  }

  private async ensureService(host: HostCRD, revalidate?: () => void): Promise<boolean> {
    const service = this.buildService(host)
    let createFailed = false
    const mutationAllowed = () => {
      revalidate?.()
      return true
    }
    // Initial read errors propagate. Only the existing POST/convergence paths
    // retain their historical non-throwing failures.
    const result = await ensureResource({
      mutationAllowed,
      read: () =>
        observeExistenceRead('Service', () =>
          this.coreApi.readNamespacedService({ namespace: host.namespace, name: host.name })
        ),
      create: async () => {
        try {
          await observeCreate('Service', () =>
            this.coreApi.createNamespacedService({ namespace: host.namespace, body: service })
          )
        } catch (error) {
          if (error != null && getErrorCode(error) === 409) throw error
          createFailed = true
          log.error('Failed to create Host Service', { host: host.name, err: error })
        }
      },
      onSkipped: () => createsTotal.inc({ kind: 'Service', outcome: 'skipped' }),
      converge: async read => {
        try {
          await replaceWithConflictRetry({
            description: `Service "${host.name}"`,
            logPrefix: '[HostReconciler]',
            body: service,
            mergeExisting: preserveServiceAssignedFields,
            isUpToDate: serviceMatchesDesired,
            mutationAllowed,
            read,
            replace: body =>
              this.coreApi.replaceNamespacedService({
                namespace: host.namespace,
                name: host.name,
                body,
              }),
          })
        } catch (error) {
          if (isBenignSupersessionError(error)) throw error
          log.error('Failed to update Host Service', { host: host.name, err: error })
          return false
        }
      },
    })
    return !createFailed && this.resourceApplySucceeded(result)
  }

  private resourceApplySucceeded(result: ResourceApplyResult): boolean {
    return result === 'created' || result === 'replaced' || result === 'up_to_date'
  }

  private async ensureDeployment(
    host: HostCRD,
    mounts: ResolvedSfsMount[],
    runtimeTokenRevision: string | undefined,
    lifecycle?: EffectiveHostLifecycle,
    resolveStateBeforeMutation?: (observed?: k8s.V1Deployment) => Promise<DeploymentMutationState>,
    revalidate?: () => void
  ): Promise<boolean> {
    let observedDeployment: k8s.V1Deployment | undefined
    let holdingTemplate = false
    let applied = true
    const buildDesiredDeployment = async (): Promise<k8s.V1Deployment | null> => {
      const state = resolveStateBeforeMutation
        ? await resolveStateBeforeMutation(observedDeployment)
        : null
      const effective = state?.lifecycle ?? lifecycle
      const resourcesDiffer =
        !!observedDeployment &&
        HostReconciler.deploymentBelongsToHost(observedDeployment, host) &&
        !HostReconciler.deploymentResourcesMatchHost(observedDeployment, host)
      const suspendingWithResourceDrift =
        effective?.stateless === true &&
        effective.state === 'suspended' &&
        effective.suspensionBlocked !== true &&
        resourcesDiffer
      holdingTemplate = effective?.suspensionBlocked === true || suspendingWithResourceDrift
      if (holdingTemplate) {
        if (!observedDeployment) return null
        const existing = observedDeployment
        if (
          !HostReconciler.isHccOwnedHostResource(existing, host.name) ||
          existing.metadata?.name !== host.name ||
          existing.metadata?.namespace !== host.namespace ||
          !existing.metadata?.uid ||
          !existing.metadata?.resourceVersion ||
          existing.metadata.deletionTimestamp ||
          !existing.spec?.template?.spec
        ) {
          throw new Error(`Cannot preserve an unverified Deployment for Host "${host.name}"`)
        }
        const annotatedHostUid = existing.metadata.annotations?.[HOST_UID_ANNOTATION]
        const hostUidVerified = !!host.uid && annotatedHostUid === host.uid
        if (!hostUidVerified) {
          throw new Error(`Cannot preserve an unverified Deployment for Host "${host.name}"`)
        }
        // Preserve UID/resourceVersion and every applied field. A real suspend
        // must still reach zero when resource policy changes; its next wake
        // prepares credentials before applying that deferred template change.
        // Conflicts re-read identity before retrying the replica-only mutation.
        const replicas = suspendingWithResourceDrift
          ? 0
          : effective?.allowScaleUpDuringHold
            ? 1
            : (existing.spec.replicas ?? 1)
        return {
          ...existing,
          spec: { ...existing.spec, replicas },
        }
      }
      if (
        observedDeployment &&
        resourcesDiffer &&
        (observedDeployment.spec?.replicas ?? 1) > 0 &&
        effective?.state === 'active' &&
        state?.freshBootstrapPrepared !== true
      ) {
        // Readiness cannot gate a resource repair that is needed for scheduling.
        // Delivery is gated on persisted, fresh material for the observed target.
        throw new RuntimeBootstrapPreparationUnavailableError()
      }
      const desired = this.buildDeployment(
        host,
        mounts,
        state?.runtimeTokenRevision ?? runtimeTokenRevision,
        effective,
        state?.grokExecutionEnabled ?? this.hostDerivesGrokExecution(host)
      )
      if (observedDeployment && resourcesDiffer && effective?.state === 'draining') {
        // Defer only resource policy while draining. An intrinsic credential
        // recovery must still deliver its prepared runtime revision.
        const appliedResources = observedDeployment.spec!.template.spec!.containers.find(
          container => container.name === 'mcp-host'
        )?.resources
        const desiredContainer = desired.spec!.template.spec!.containers.find(
          container => container.name === 'mcp-host'
        )!
        desiredContainer.resources = structuredClone(appliedResources)
      }
      return desired
    }
    const mutationAllowed = () => {
      revalidate?.()
      return true
    }
    const result = await ensureResource({
      mutationAllowed,
      read: async () => {
        observedDeployment = await observeExistenceRead('Deployment', () =>
          this.appsApi.readNamespacedDeployment({ namespace: host.namespace, name: host.name })
        )
        return observedDeployment
      },
      create: async () => {
        // The initial GET may outlive the lifecycle or token-scope observation.
        const deployment = await buildDesiredDeployment()
        revalidate?.()
        if (!deployment) {
          applied = false
          return
        }
        return observeCreate('Deployment', () =>
          this.appsApi.createNamespacedDeployment({ namespace: host.namespace, body: deployment })
        )
      },
      onSkipped: () => createsTotal.inc({ kind: 'Deployment', outcome: 'skipped' }),
      converge: read =>
        replaceWithConflictRetry({
          description: `Deployment "${host.name}"`,
          logPrefix: '[HostReconciler]',
          resolveBody: async () => {
            const desired = await buildDesiredDeployment()
            if (!desired) throw new Error('Deployment disappeared during preservation')
            return desired
          },
          mergeExisting: (desired, existing) =>
            holdingTemplate ? desired : preserveHostDeploymentAnnotations(desired, existing),
          isUpToDate: deploymentMatchesDesired,
          mutationAllowed,
          read,
          replace: body =>
            this.appsApi.replaceNamespacedDeployment({
              namespace: host.namespace,
              name: host.name,
              body,
            }),
        }),
    })
    return applied && this.resourceApplySucceeded(result)
  }

  private async deleteRuntimeResources(name: string, namespace: string): Promise<void> {
    const failures: unknown[] = []

    // Desktop NetworkPolicy is created idempotently in ensureDesktopNetworkPolicy;
    // attempt deletion unconditionally so non-desktop hosts (no policy) just hit 404.
    // Kept for backward-compatible cleanup of policies created before rpc-proxy
    // host egress became per-Host.
    const desktopPolicyName = `allow-rpc-proxy-desktop-${name}`
    for (const cleanup of [
      () =>
        this.deleteIfHccOwned(
          'Deployment',
          name,
          namespace,
          name,
          () => this.appsApi.readNamespacedDeployment({ name, namespace }),
          () => this.appsApi.deleteNamespacedDeployment({ name, namespace })
        ),
      () =>
        this.deleteIfHccOwned(
          'Service',
          name,
          namespace,
          name,
          () => this.coreApi.readNamespacedService({ name, namespace }),
          () => this.coreApi.deleteNamespacedService({ name, namespace })
        ),
      () =>
        this.deleteIfHccOwned(
          'NetworkPolicy',
          desktopPolicyName,
          namespace,
          name,
          () =>
            this.networkingApi.readNamespacedNetworkPolicy({ name: desktopPolicyName, namespace }),
          () =>
            this.networkingApi.deleteNamespacedNetworkPolicy({ name: desktopPolicyName, namespace })
        ),
    ]) {
      try {
        await cleanup()
      } catch (error) {
        failures.push(error)
      }
    }
    throwCleanupFailures(
      failures,
      `Failed to delete runtime Kubernetes resources for Host "${name}"`
    )
  }

  private async deleteWorkspacePvc(name: string, namespace: string): Promise<void> {
    const pvcName = `${name}-workspace`
    await this.deleteIfHccOwned(
      'PersistentVolumeClaim',
      pvcName,
      namespace,
      name,
      () => this.coreApi.readNamespacedPersistentVolumeClaim({ name: pvcName, namespace }),
      () => this.coreApi.deleteNamespacedPersistentVolumeClaim({ name: pvcName, namespace })
    )
  }

  private async deleteResources(
    name: string,
    namespace: string,
    options: { deleteWorkspacePvc?: boolean } = {}
  ): Promise<void> {
    const failures: unknown[] = []
    const cleanups = [() => this.deleteRuntimeResources(name, namespace)]
    if (options.deleteWorkspacePvc ?? true) {
      cleanups.push(() => this.deleteWorkspacePvc(name, namespace))
    }
    for (const cleanup of cleanups) {
      try {
        await cleanup()
      } catch (error) {
        failures.push(error)
      }
    }
    throwCleanupFailures(failures, `Failed to delete Kubernetes resources for Host "${name}"`)
  }

  /**
   * Cascade delete the per-Host channel-reader Deployment in the channels
   * namespace. 404 is swallowed (already gone); non-404 errors abort cleanup
   * so revocation retries instead of silently leaving a reader alive.
   */
  private async deleteChannelReaderDeployment(name: string): Promise<void> {
    const depName = `channel-reader-${name}`
    await this.deleteIfHccOwned(
      'Deployment',
      depName,
      config.channelsNamespace,
      name,
      () =>
        this.appsApi.readNamespacedDeployment({
          name: depName,
          namespace: config.channelsNamespace,
        }),
      () =>
        this.appsApi.deleteNamespacedDeployment({
          name: depName,
          namespace: config.channelsNamespace,
        })
    )
  }

  private async deleteChannelReaderService(name: string): Promise<void> {
    const serviceName = `channel-reader-${name}`
    await this.deleteIfHccOwned(
      'Service',
      serviceName,
      config.channelsNamespace,
      name,
      () =>
        this.coreApi.readNamespacedService({
          name: serviceName,
          namespace: config.channelsNamespace,
        }),
      () =>
        this.coreApi.deleteNamespacedService({
          name: serviceName,
          namespace: config.channelsNamespace,
        })
    )
  }

  /**
   * One-shot startup sweep: delete the legacy static `clerum-channel-reader`
   * Deployment in the channels namespace if it still exists.
   *
   * The legacy static Deployment was the only channel-reader before the
   * per-Host pattern landed. Per-Host Deployments now own that role
   * (channel-reader-<host>). With both alive, the two pods compete on the
   * same Telegram bot token's getUpdates long-poll and Telegram returns
   * 409 Conflict to whichever loses — empirical evidence on minikube +
   * example-dev is captured in #273.
   *
   * Idempotent on every HCC startup: 404 means already gone (the steady
   * state we want), so we log nothing. A successful delete emits an
   * `AUDIT:` line for operator monitoring. Other errors are logged but
   * NOT thrown — startup continues so per-Host reconciles still run.
   *
   * Public so main.ts can call it once at boot, before the reconcile
   * loop starts (preventing the brief window where both pods could
   * poll concurrently).
   */
  async sweepLegacyStaticChannelReader(): Promise<void> {
    const depName = 'clerum-channel-reader'
    const ns = config.channelsNamespace
    try {
      await this.appsApi.deleteNamespacedDeployment({ name: depName, namespace: ns })
      // Structured audit line — operators tail this to confirm the legacy
      // pod is gone post-deploy. Mirrors the format used by
      // sweepOrphanChannelReaderResources for consistency.
      log.info('AUDIT: legacy-sweep deleted Deployment', { name: depName, namespace: ns })
    } catch (err) {
      const code = getErrorCode(err)
      if (code === 404) return // already gone — steady state, no log
      log.error('legacy-sweep failed to delete Deployment', { name: depName, namespace: ns, err })
    }
  }

  private async ensureDesktopNetworkPolicy(
    host: HostCRD
  ): Promise<'not-required' | ResourceApplyResult> {
    const isDesktop = !!(host.spec.desktop?.browser || host.spec.desktop?.x11)
    if (!isDesktop) return 'not-required'

    const policyName = `allow-rpc-proxy-desktop-${host.name}`
    const policy: k8s.V1NetworkPolicy = {
      apiVersion: 'networking.k8s.io/v1',
      kind: 'NetworkPolicy',
      metadata: {
        name: policyName,
        namespace: host.namespace,
        labels: {
          [MANAGED_BY_LABEL]: MANAGED_BY_VALUE,
          [HOST_LABEL]: host.name,
          'clerum.io/policy-type': 'desktop-ingress',
        },
      },
      spec: {
        podSelector: { matchLabels: { app: host.name } },
        policyTypes: ['Ingress'],
        ingress: [
          {
            _from: [
              {
                namespaceSelector: {
                  matchLabels: { 'kubernetes.io/metadata.name': config.rpcProxyNamespace },
                },
                podSelector: { matchLabels: { app: 'rpc-proxy' } },
              },
            ],
            ports: [{ port: config.desktopPort, protocol: 'TCP' }],
          },
        ],
      },
    }

    return applyNetworkPolicy(
      this.networkingApi,
      policyName,
      host.namespace,
      policy,
      '[HostReconciler]'
    )
  }

  /**
   * Per-Host rpc-proxy egress policy. Pins rpc-proxy traffic to exactly one
   * mcp-host pod instead of relying on namespace-wide rpc-proxy -> mcp-host
   * egress. Desktop hosts also expose the desktop service port through the
   * same host-scoped egress boundary.
   */
  private async ensureRpcProxyHostEgressNetworkPolicy(host: HostCRD): Promise<ResourceApplyResult> {
    const isDesktop = !!(host.spec.desktop?.browser || host.spec.desktop?.x11)
    const policyName = `rpc-proxy-${host.name}-egress-mcp-host`
    const ports: k8s.V1NetworkPolicyPort[] = [{ port: config.hostPort, protocol: 'TCP' }]
    if (isDesktop) {
      ports.push({ port: config.desktopPort, protocol: 'TCP' })
    }
    const policy: k8s.V1NetworkPolicy = {
      apiVersion: 'networking.k8s.io/v1',
      kind: 'NetworkPolicy',
      metadata: {
        name: policyName,
        namespace: config.rpcProxyNamespace,
        labels: {
          [MANAGED_BY_LABEL]: MANAGED_BY_VALUE,
          [HOST_LABEL]: host.name,
          'clerum.io/policy-type': 'rpc-proxy-host-egress',
        },
      },
      spec: {
        podSelector: { matchLabels: { app: 'rpc-proxy' } },
        policyTypes: ['Egress'],
        egress: [
          {
            to: [
              {
                namespaceSelector: {
                  matchLabels: { 'kubernetes.io/metadata.name': host.namespace },
                },
                podSelector: {
                  matchLabels: {
                    [HOST_LABEL]: host.name,
                    [MANAGED_BY_LABEL]: MANAGED_BY_VALUE,
                  },
                },
              },
            ],
            ports,
          },
        ],
      },
    }

    return applyNetworkPolicy(
      this.networkingApi,
      policyName,
      config.rpcProxyNamespace,
      policy,
      '[HostReconciler]'
    )
  }

  /**
   * Per-Host channel-reader egress policy. Pins channel-reader-<host> pods to
   * its bound mcp-host on :8080. Provider approvals and verification continue
   * through mcp-host; channel-reader must not reach control-api gateways.
   */
  private async ensureChannelReaderEgressNetworkPolicy(
    host: HostCRD
  ): Promise<ResourceApplyResult> {
    const policyName = `channel-reader-${host.name}-egress`
    const policy: k8s.V1NetworkPolicy = {
      apiVersion: 'networking.k8s.io/v1',
      kind: 'NetworkPolicy',
      metadata: {
        name: policyName,
        namespace: config.channelsNamespace,
        labels: {
          [MANAGED_BY_LABEL]: MANAGED_BY_VALUE,
          [HOST_LABEL]: host.name,
          'clerum.io/policy-type': 'channel-reader-egress',
        },
      },
      spec: {
        podSelector: {
          matchLabels: {
            app: 'channel-reader',
            [HOST_LABEL]: host.name,
            [MANAGED_BY_LABEL]: MANAGED_BY_VALUE,
          },
        },
        policyTypes: ['Egress'],
        egress: [
          {
            to: [
              {
                namespaceSelector: {
                  matchLabels: { 'kubernetes.io/metadata.name': config.hostNamespace },
                },
                podSelector: {
                  matchLabels: { [HOST_LABEL]: host.name },
                },
              },
            ],
            ports: [{ port: config.hostPort, protocol: 'TCP' }],
          },
        ],
      },
    }

    // Re-throw on failure: per-Host NPs are one half of the
    // channel-reader security boundary. Route-level JWT auth is the other
    // half, so an NP apply error still MUST surface in HostRuntimeStatus
    // rather than be silently swallowed.
    return applyNetworkPolicy(
      this.networkingApi,
      policyName,
      config.channelsNamespace,
      policy,
      '[HostReconciler]'
    )
  }

  /**
   * Per-Host workflow-approval-reader egress policy. Slack interactive callbacks
   * are verified by control-api against a CommunicationChannel, then forwarded
   * to that channel's bound Host for the authoritative provider-decision path.
   */
  private async ensureWorkflowApprovalReaderHostEgressNetworkPolicy(
    host: HostCRD
  ): Promise<ResourceApplyResult> {
    const policyName = `workflow-approval-reader-${host.name}-egress-mcp-host`
    const policy: k8s.V1NetworkPolicy = {
      apiVersion: 'networking.k8s.io/v1',
      kind: 'NetworkPolicy',
      metadata: {
        name: policyName,
        namespace: config.channelsNamespace,
        labels: {
          [MANAGED_BY_LABEL]: MANAGED_BY_VALUE,
          [HOST_LABEL]: host.name,
          'clerum.io/policy-type': 'workflow-approval-reader-host-egress',
        },
      },
      spec: {
        podSelector: {
          matchLabels: { 'app.kubernetes.io/name': 'workflow-approval-request-reader' },
        },
        policyTypes: ['Egress'],
        egress: [
          {
            to: [
              {
                namespaceSelector: {
                  matchLabels: { 'kubernetes.io/metadata.name': host.namespace },
                },
                podSelector: {
                  matchLabels: {
                    [HOST_LABEL]: host.name,
                    [MANAGED_BY_LABEL]: MANAGED_BY_VALUE,
                  },
                },
              },
            ],
            ports: [{ port: config.hostPort, protocol: 'TCP' }],
          },
        ],
      },
    }

    return applyNetworkPolicy(
      this.networkingApi,
      policyName,
      config.channelsNamespace,
      policy,
      '[HostReconciler]'
    )
  }

  /**
   * Per-Host mcp-host ingress policy for channel-reader. Pins mcp-host-<host>
   * pod ingress on :8080 to ONLY accept connections from channel-reader-<host>
   * pods (label match).
   */
  private async ensureMcpHostIngressNetworkPolicy(host: HostCRD): Promise<ResourceApplyResult> {
    const policyName = `mcp-host-${host.name}-ingress-channel-reader`
    const policy: k8s.V1NetworkPolicy = {
      apiVersion: 'networking.k8s.io/v1',
      kind: 'NetworkPolicy',
      metadata: {
        name: policyName,
        namespace: host.namespace,
        labels: {
          [MANAGED_BY_LABEL]: MANAGED_BY_VALUE,
          [HOST_LABEL]: host.name,
          'clerum.io/policy-type': 'channel-reader-ingress',
        },
      },
      spec: {
        podSelector: {
          matchLabels: {
            [HOST_LABEL]: host.name,
            [MANAGED_BY_LABEL]: MANAGED_BY_VALUE,
          },
        },
        policyTypes: ['Ingress'],
        ingress: [
          {
            _from: [
              {
                namespaceSelector: {
                  matchLabels: { 'kubernetes.io/metadata.name': config.channelsNamespace },
                },
                podSelector: {
                  matchLabels: {
                    app: 'channel-reader',
                    [HOST_LABEL]: host.name,
                    [MANAGED_BY_LABEL]: MANAGED_BY_VALUE,
                  },
                },
              },
            ],
            ports: [{ port: config.hostPort, protocol: 'TCP' }],
          },
        ],
      },
    }

    // Re-throw on failure: per-Host NPs are one half of the
    // channel-reader↔mcp-host security boundary. Route-level JWT auth is
    // the other half, so an apply error MUST surface in HostRuntimeStatus
    // rather than be silently swallowed.
    return applyNetworkPolicy(
      this.networkingApi,
      policyName,
      host.namespace,
      policy,
      '[HostReconciler]'
    )
  }

  /**
   * Per-Host mcp-host ingress policy for workflow-approval-reader. This is the
   * provider webhook counterpart to channel-reader ingress and stays pinned to
   * the Host resolved from the verified CommunicationChannel target.
   */
  private async ensureWorkflowApprovalReaderMcpHostIngressNetworkPolicy(
    host: HostCRD
  ): Promise<ResourceApplyResult> {
    const policyName = `mcp-host-${host.name}-ingress-workflow-approval-reader`
    const policy: k8s.V1NetworkPolicy = {
      apiVersion: 'networking.k8s.io/v1',
      kind: 'NetworkPolicy',
      metadata: {
        name: policyName,
        namespace: host.namespace,
        labels: {
          [MANAGED_BY_LABEL]: MANAGED_BY_VALUE,
          [HOST_LABEL]: host.name,
          'clerum.io/policy-type': 'workflow-approval-reader-host-ingress',
        },
      },
      spec: {
        podSelector: {
          matchLabels: {
            [HOST_LABEL]: host.name,
            [MANAGED_BY_LABEL]: MANAGED_BY_VALUE,
          },
        },
        policyTypes: ['Ingress'],
        ingress: [
          {
            _from: [
              {
                namespaceSelector: {
                  matchLabels: { 'kubernetes.io/metadata.name': config.channelsNamespace },
                },
                podSelector: {
                  matchLabels: {
                    'app.kubernetes.io/name': 'workflow-approval-request-reader',
                  },
                },
              },
            ],
            ports: [{ port: config.hostPort, protocol: 'TCP' }],
          },
        ],
      },
    }

    return applyNetworkPolicy(
      this.networkingApi,
      policyName,
      host.namespace,
      policy,
      '[HostReconciler]'
    )
  }

  /**
   * Per-Host mcp-host ingress policy for rpc-proxy. This replaces the legacy
   * namespace-wide mcp-host ingress rule for rpc-proxy. Desktop hosts include
   * the desktop port, but only for the selected Host pod.
   */
  private async ensureRpcProxyMcpHostIngressNetworkPolicy(
    host: HostCRD
  ): Promise<ResourceApplyResult> {
    const isDesktop = !!(host.spec.desktop?.browser || host.spec.desktop?.x11)
    const policyName = `mcp-host-${host.name}-ingress-rpc-proxy`
    const ports: k8s.V1NetworkPolicyPort[] = [{ port: config.hostPort, protocol: 'TCP' }]
    if (isDesktop) {
      ports.push({ port: config.desktopPort, protocol: 'TCP' })
    }
    const policy: k8s.V1NetworkPolicy = {
      apiVersion: 'networking.k8s.io/v1',
      kind: 'NetworkPolicy',
      metadata: {
        name: policyName,
        namespace: host.namespace,
        labels: {
          [MANAGED_BY_LABEL]: MANAGED_BY_VALUE,
          [HOST_LABEL]: host.name,
          'clerum.io/policy-type': 'rpc-proxy-host-ingress',
        },
      },
      spec: {
        podSelector: {
          matchLabels: {
            [HOST_LABEL]: host.name,
            [MANAGED_BY_LABEL]: MANAGED_BY_VALUE,
          },
        },
        policyTypes: ['Ingress'],
        ingress: [
          {
            _from: [
              {
                namespaceSelector: {
                  matchLabels: { 'kubernetes.io/metadata.name': config.rpcProxyNamespace },
                },
                podSelector: { matchLabels: { app: 'rpc-proxy' } },
              },
            ],
            ports,
          },
        ],
      },
    }

    return applyNetworkPolicy(
      this.networkingApi,
      policyName,
      host.namespace,
      policy,
      '[HostReconciler]'
    )
  }

  /**
   * Per-Host mcp-host egress to gfsc. The runtime token grants authority; this
   * NetworkPolicy only opens the required transport lane from the selected Host
   * pod to the governed GFS controller service.
   */
  private async ensureMcpHostGfsEgressNetworkPolicy(host: HostCRD): Promise<ResourceApplyResult> {
    const target = { namespace: config.gfsNamespace, port: config.gfscPort }
    const policyName = `mcp-host-${host.name}-egress-gfs`
    const policy: k8s.V1NetworkPolicy = {
      apiVersion: 'networking.k8s.io/v1',
      kind: 'NetworkPolicy',
      metadata: {
        name: policyName,
        namespace: host.namespace,
        labels: {
          [MANAGED_BY_LABEL]: MANAGED_BY_VALUE,
          [HOST_LABEL]: host.name,
          'clerum.io/policy-type': 'gfs-egress',
        },
      },
      spec: {
        podSelector: {
          matchLabels: {
            [HOST_LABEL]: host.name,
            [MANAGED_BY_LABEL]: MANAGED_BY_VALUE,
          },
        },
        policyTypes: ['Egress'],
        egress: [
          {
            to: [
              {
                namespaceSelector: {
                  matchLabels: { 'kubernetes.io/metadata.name': target.namespace },
                },
                podSelector: {
                  matchLabels: { app: 'gfs-controller' },
                },
              },
            ],
            ports: [{ port: target.port, protocol: 'TCP' }],
          },
        ],
      },
    }

    return applyNetworkPolicy(
      this.networkingApi,
      policyName,
      host.namespace,
      policy,
      '[HostReconciler]'
    )
  }

  /**
   * Per-Host mcp-host egress to the static Codex LLM proxy runtime listener.
   * Derived from the same Codex projection that mints `llm:codex:execute`.
   */
  private async ensureMcpHostCodexProxyEgressNetworkPolicy(
    host: HostCRD
  ): Promise<ResourceApplyResult> {
    const policyName = `mcp-host-${host.name}-egress-codex-proxy`
    const policy: k8s.V1NetworkPolicy = {
      apiVersion: 'networking.k8s.io/v1',
      kind: 'NetworkPolicy',
      metadata: {
        name: policyName,
        namespace: host.namespace,
        labels: {
          [MANAGED_BY_LABEL]: MANAGED_BY_VALUE,
          [HOST_LABEL]: host.name,
          'clerum.io/policy-type': 'codex-proxy-egress',
        },
      },
      spec: {
        podSelector: {
          matchLabels: {
            [HOST_LABEL]: host.name,
            [MANAGED_BY_LABEL]: MANAGED_BY_VALUE,
          },
        },
        policyTypes: ['Egress'],
        egress: [
          {
            to: [
              {
                namespaceSelector: {
                  matchLabels: { 'kubernetes.io/metadata.name': config.controlPlaneNamespace },
                },
                podSelector: {
                  matchLabels: { app: 'codex-llm-proxy' },
                },
              },
            ],
            ports: [{ port: 8080, protocol: 'TCP' }],
          },
        ],
      },
    }

    return applyNetworkPolicy(
      this.networkingApi,
      policyName,
      host.namespace,
      policy,
      '[HostReconciler]'
    )
  }

  private async deleteMcpHostCodexProxyEgressNetworkPolicy(host: HostCRD): Promise<void> {
    const policyName = `mcp-host-${host.name}-egress-codex-proxy`
    await this.deleteIfHccOwned(
      'NetworkPolicy',
      policyName,
      host.namespace,
      host.name,
      () =>
        this.networkingApi.readNamespacedNetworkPolicy({
          name: policyName,
          namespace: host.namespace,
        }),
      () =>
        this.networkingApi.deleteNamespacedNetworkPolicy({
          name: policyName,
          namespace: host.namespace,
        })
    )
  }

  private async reconcileMcpHostCodexProxyEgressNetworkPolicy(
    host: HostCRD
  ): Promise<'not-required' | ResourceApplyResult> {
    const projection = this.projectCodexForHost(host)
    if (projection.eligibility === 'uncertain') return 'not-required'
    if (projection.requiresCodexProxyEgress) {
      return this.ensureMcpHostCodexProxyEgressNetworkPolicy(host)
    }
    await this.deleteMcpHostCodexProxyEgressNetworkPolicy(host)
    return 'not-required'
  }

  private async ensureMcpHostGrokProxyEgressNetworkPolicy(
    host: HostCRD
  ): Promise<ResourceApplyResult> {
    const policyName = `mcp-host-${host.name}-egress-grok-proxy`
    const policy: k8s.V1NetworkPolicy = {
      apiVersion: 'networking.k8s.io/v1',
      kind: 'NetworkPolicy',
      metadata: {
        name: policyName,
        namespace: host.namespace,
        labels: {
          [MANAGED_BY_LABEL]: MANAGED_BY_VALUE,
          [HOST_LABEL]: host.name,
          'clerum.io/policy-type': 'grok-proxy-egress',
        },
      },
      spec: {
        podSelector: {
          matchLabels: {
            [HOST_LABEL]: host.name,
            [MANAGED_BY_LABEL]: MANAGED_BY_VALUE,
          },
        },
        policyTypes: ['Egress'],
        egress: [
          {
            to: [
              {
                namespaceSelector: {
                  matchLabels: { 'kubernetes.io/metadata.name': config.controlPlaneNamespace },
                },
                podSelector: {
                  matchLabels: { app: 'grok-llm-proxy' },
                },
              },
            ],
            ports: [{ port: 8080, protocol: 'TCP' }],
          },
        ],
      },
    }
    return applyNetworkPolicy(
      this.networkingApi,
      policyName,
      host.namespace,
      policy,
      '[HostReconciler]'
    )
  }

  private async deleteMcpHostGrokProxyEgressNetworkPolicy(host: HostCRD): Promise<void> {
    const policyName = `mcp-host-${host.name}-egress-grok-proxy`
    await this.deleteIfHccOwned(
      'NetworkPolicy',
      policyName,
      host.namespace,
      host.name,
      () =>
        this.networkingApi.readNamespacedNetworkPolicy({
          name: policyName,
          namespace: host.namespace,
        }),
      () =>
        this.networkingApi.deleteNamespacedNetworkPolicy({
          name: policyName,
          namespace: host.namespace,
        })
    )
  }

  private async reconcileMcpHostGrokProxyEgressNetworkPolicy(
    host: HostCRD
  ): Promise<'not-required' | ResourceApplyResult> {
    const projection = this.projectGrokForHost(host)
    if (projection.eligibility === 'uncertain') return 'not-required'
    if (projection.requiresGrokProxyEgress) {
      return this.ensureMcpHostGrokProxyEgressNetworkPolicy(host)
    }
    await this.deleteMcpHostGrokProxyEgressNetworkPolicy(host)
    return 'not-required'
  }

  /**
   * Delete per-Host NetworkPolicies created by this reconciler.
   * 404-tolerant — safe to call even if NPs were never created or already gone.
   */
  private async deleteHostNetworkPolicies(name: string, namespace: string): Promise<void> {
    const failures: unknown[] = []
    const policies = [
      { name: `mcp-host-${name}-ingress-channel-reader`, namespace },
      { name: `mcp-host-${name}-ingress-workflow-approval-reader`, namespace },
      { name: `mcp-host-${name}-ingress-rpc-proxy`, namespace },
      { name: `mcp-host-${name}-egress-gfs`, namespace },
      { name: `mcp-host-${name}-egress-codex-proxy`, namespace },
      { name: `mcp-host-${name}-egress-grok-proxy`, namespace },
      { name: `mcp-host-${name}-egress-llm-hooks`, namespace },
      { name: `channel-reader-${name}-egress`, namespace: config.channelsNamespace },
      {
        name: `workflow-approval-reader-${name}-egress-mcp-host`,
        namespace: config.channelsNamespace,
      },
      { name: `rpc-proxy-${name}-egress-mcp-host`, namespace: config.rpcProxyNamespace },
    ]
    for (const policy of policies) {
      try {
        await this.deleteIfHccOwned(
          'NetworkPolicy',
          policy.name,
          policy.namespace,
          name,
          () =>
            this.networkingApi.readNamespacedNetworkPolicy({
              name: policy.name,
              namespace: policy.namespace,
            }),
          () =>
            this.networkingApi.deleteNamespacedNetworkPolicy({
              name: policy.name,
              namespace: policy.namespace,
            })
        )
      } catch (error) {
        failures.push(error)
      }
    }
    throwCleanupFailures(failures, `Failed to delete NetworkPolicies for Host "${name}"`)
  }

  private async deleteHostRuntimeResources(
    name: string,
    namespace: string,
    options: { deleteWorkspacePvc?: boolean } = {}
  ): Promise<void> {
    const failures: unknown[] = []
    for (const cleanup of [
      () => this.deleteResources(name, namespace, options),
      () => this.deleteHostRbac(name, namespace),
      () => this.deleteMcpHostRuntimeTokenSecret(name, namespace),
      () => this.deleteLegacyChannelReaderRuntimeAuth(name),
      () => this.deleteChannelReaderService(name),
      () => this.deleteChannelReaderDeployment(name),
      () => this.deleteHostNetworkPolicies(name, namespace),
    ]) {
      try {
        await cleanup()
      } catch (error) {
        failures.push(error)
      }
    }
    throwCleanupFailures(failures, `Failed to delete runtime resources for Host "${name}"`)
  }

  async reconcile(host: HostCRD, source: HostReconcileSource = 'urgent'): Promise<void> {
    const dispatchedAt = Date.now()
    const capturedAuthority = this.captureHostMutationAuthority()
    const capturedDependencies = this.resolveHostMutationDependencies?.(host)
    return this.lifecycle.serializeByHost(host.name, async () => {
      this.advanceReconcileGeneration(host.name)
      this.requireHostMutationAuthority(`Host "${host.name}" reconcile`, capturedAuthority)
      const admittedHost = this.resolveCurrentHost ? this.resolveCurrentHost(host.name) : host
      if (!admittedHost) return
      const revalidate = this.makeHostMutationAdmission(
        `Host "${host.name}" reconcile`,
        admittedHost,
        capturedAuthority,
        capturedDependencies
      )
      const admittedAt = Date.now()
      hostReconcileInFlight.inc({ lane: source })
      try {
        await this.reconcileCore(admittedHost, revalidate)
        this.enqueueReconcileOutcome(admittedHost)
        this.observeReconcileLatency(source, 'success', dispatchedAt, admittedAt)
      } catch (error) {
        if (isBenignSupersessionError(error)) {
          // A newer spec/uid/authority-generation superseded this pass mid-flight;
          // a fresh reconcile runs after. Model it as a withdrawal (superseded),
          // NOT a hard failure — do NOT emit controller_error/reconcile_exception
          // or an administrative 'failed' outcome. Callers already treat the
          // rethrow as a retire (reconcileDelete → 'superseded', watch callers
          // only log the error), so the rethrow is preserved.
          // This exit emits no outcome, so drop the GFS evidence this pass
          // recorded: a leftover would be reported by the next pass over this
          // object. The clear is scoped to this snapshot's uid, so a sibling
          // object at the same name keeps its own evidence. The stateless
          // suspension path also calls reconcileCore without emitting an
          // outcome and does not clear the entry (#696).
          this.clearGfsLifecycleEvidenceForObject(admittedHost)
          this.observeReconcileLatency(source, 'superseded', dispatchedAt, admittedAt)
          throw error
        }
        this.enqueueControllerError(admittedHost, 'reconcile_exception', error)
        this.enqueueReconcileExceptionOutcome(admittedHost)
        this.observeReconcileLatency(source, 'error', dispatchedAt, admittedAt)
        throw error
      } finally {
        hostReconcileInFlight.dec({ lane: source })
      }
    })
  }

  private observeReconcileLatency(
    source: HostReconcileSource,
    outcome: 'success' | 'error' | 'superseded',
    dispatchedAt: number,
    admittedAt: number
  ): void {
    hostReconcileQueueWaitSeconds.observe(
      { lane: source, outcome },
      Math.max(0, admittedAt - dispatchedAt) / 1000
    )
    hostReconcileDurationSeconds.observe(
      { source, outcome },
      Math.max(0, Date.now() - admittedAt) / 1000
    )
  }

  private async reconcileCore(host: HostCRD, revalidate?: () => HostCRD): Promise<void> {
    // A per-Host serializer orders work, but it does not invalidate an older
    // reconcile when the watch cache observes a newer spec while this body is
    // awaiting Kubernetes or issuer I/O. Re-check the captured authority,
    // identity, and spec generation at every material mutation boundary.
    // Individual API writes remain atomic at the Kubernetes resource level;
    // these checks prevent subsequent stale writes from spanning an observed
    // generation change.
    const revalidateHostMutationBoundary = (): void => {
      revalidate?.()
    }

    revalidateHostMutationBoundary()
    // Wake fast-path (Stage 4.3) BEFORE the heavy reconcile body: reconciles
    // are serialized PER HOST (serializeByHost), so a pending wake for THIS
    // Host must not wait behind token issuance, NetworkPolicies or the
    // channel-reader work below in this same chain. Other Hosts no longer gate
    // it — the process-wide convergence tail was removed. The periodic resync
    // funnels through this same method, so a watch event dropped on disconnect
    // is recovered here.
    const forceFreshForWake = (await this.lifecycle.handleWakeFastPath(host)) === true
    revalidateHostMutationBoundary()
    await this.refreshCodexSnapshot()
    revalidateHostMutationBoundary()

    // Track whether this host has desktop enabled
    const isDesktop = !!(host.spec.desktop?.browser || host.spec.desktop?.x11)
    if (isDesktop) {
      this.desktopHosts.add(host.name)
    } else {
      this.desktopHosts.delete(host.name)
    }

    const secretResult = await this.validateHostSecret(host)
    revalidateHostMutationBoundary()
    if (!secretResult.ok) {
      if (
        secretResult.reason === 'SecretNotFound' ||
        secretResult.reason === 'SecretAccessDenied'
      ) {
        // This branch deletes every ephemeral runtime resource. The Secret
        // decision may have blocked on the API while Host watch authority or
        // the Host UID/spec generation changed. Revalidate at the destructive
        // commit boundary so a stale secretRef can never tear down the current
        // runtime; the recovered/current Host reconcile will decide again.
        revalidateHostMutationBoundary()
        await this.deleteHostRuntimeResources(host.name, host.namespace, {
          deleteWorkspacePvc: false,
        })
      } else {
        log.warn('Preserving existing runtime after transient Secret read failure', {
          host: host.name,
        })
      }
      this.setStatus(host.name, {
        deployed: false,
        ready: false,
        message: secretResult.message,
      })
      this.enqueueControllerError(host, secretResult.reason, new Error(secretResult.message))
      return
    }

    // Per-Host SA + Role + RoleBinding must exist BEFORE the Deployment,
    // otherwise the kubelet can't mount the SA token and the pod will
    // crash-loop until RBAC is created.
    revalidateHostMutationBoundary()
    await this.ensureHostServiceAccount(host, revalidateHostMutationBoundary)
    revalidateHostMutationBoundary()
    await this.ensureHostRole(host, revalidateHostMutationBoundary)
    revalidateHostMutationBoundary()
    await this.ensureHostRoleBinding(host, revalidateHostMutationBoundary)

    let mounts: ResolvedSfsMount[] = []
    try {
      mounts = await this.resolveContextMounts(host)
    } catch (err) {
      // A failure to resolve mounts should not block the rest of the Host
      // reconciliation — the pod will still come up without the SFS mounts,
      // and a subsequent reconcile (Context update or SFS becoming Ready)
      // will inject them.
      log.error('Failed to resolve context mounts', { host: host.name, err })
    }
    revalidateHostMutationBoundary()
    // Stateless lifecycle (Stage 2): assess enable/reject and persist the
    // durable state to the Host status subresource BEFORE building the
    // Deployment, so replicas derive from the same assessment.
    let lifecycle = await this.lifecycle.assessLifecycle(host, mounts)
    revalidateHostMutationBoundary()
    // Drained-pre-scale guard (Stage 4.3): a pending wake must abort the
    // suspension IMMEDIATELY before replicas:0 derives from this assessment
    // — see StatelessLifecycleExecutor.resolveWakeBeforeScaleDown. A null
    // result means the fresh guard read failed: skip the scale-down this
    // pass (the periodic resync retries) rather than scaling to 0 on stale
    // data.
    const guardedLifecycle = await this.lifecycle.resolveWakeBeforeScaleDown(host, lifecycle)
    revalidateHostMutationBoundary()
    if (guardedLifecycle === null) {
      return
    }
    lifecycle = guardedLifecycle
    revalidateHostMutationBoundary()
    const lifecycleStatusCommitted = await this.lifecycle.writeLifecycleStatusToCluster(
      host,
      lifecycle
    )
    revalidateHostMutationBoundary()
    if (lifecycleStatusCommitted) {
      this.enqueueHostTelemetry(host, 'lifecycle_transition', lifecycle.effective.state, {
        status: 'committed',
        transition: `lifecycle:${lifecycle.effective.state}`,
        state: lifecycle.effective.state,
        phase: lifecycle.effective.stateless ? 'stateless' : 'stateful',
        summary: lifecycle.lifecycle.reason,
      })
    }

    let pvcApplied = true
    let serviceApplied = true
    const resourceErrors: unknown[] = []
    const npFailures: string[] = []
    const ensureIndependentResource = async (
      kind: 'Host PVC' | 'Host Service',
      reconcile: () => Promise<boolean>
    ): Promise<boolean> => {
      try {
        return await reconcile()
      } catch (error) {
        if (isBenignSupersessionError(error)) throw error
        resourceErrors.push(error)
        log.error('Failed to reconcile Host resource', {
          host: host.name,
          resource: kind,
          err: error,
        })
        return false
      }
    }
    const holdAppliedRuntime = async (wakeRequested: boolean): Promise<void> => {
      const liveDeployment = await this.readHostDeploymentOrNull(host)
      // A legacy Deployment without a runtime-token-revision annotation is not
      // a preserved wake target, so it stays at zero replicas until channel
      // authority returns and the normal path rewrites its template.
      const appliedHostRuntime =
        HostReconciler.deploymentBelongsToHost(liveDeployment, host) &&
        HostReconciler.deploymentRuntimeTokenRevision(liveDeployment) !== ''
      const liveReplicas = liveDeployment?.spec?.replicas ?? 1
      const preservedWakeTarget = appliedHostRuntime && liveReplicas === 0
      // A suspension recorded in status whose scale-down never reached the
      // Deployment leaves the applied runtime running. A wake on it needs no
      // mint and no replica change: it is held like any running runtime.
      const wakeOnRunningRuntime = wakeRequested && appliedHostRuntime && liveReplicas > 0
      const runtimeIsReady =
        !!liveDeployment &&
        HostReconciler.deploymentBelongsToHost(liveDeployment, host) &&
        HostReconciler.deploymentReady(liveDeployment)
      const scaleUpForRecovery = lifecycle.effective.state === 'active' && preservedWakeTarget
      const allowScaleUp = preservedWakeTarget && (wakeRequested || scaleUpForRecovery)
      if (
        !liveDeployment ||
        ((wakeRequested || scaleUpForRecovery) && !preservedWakeTarget && !wakeOnRunningRuntime)
      ) {
        this.setStatus(host.name, {
          deployed: !!liveDeployment,
          ready: false,
          message:
            'Waiting for a verified applied runtime before waking during CommunicationChannel inventory loss',
        })
        return
      }
      let provisioningRequired = false
      if (allowScaleUp) {
        // A pod starting from zero will consume bootstrap credentials. Mint and
        // persist them before the replica-only update; a failed mint aborts here.
        provisioningRequired = true
      } else if (
        (lifecycle.effective.state === 'active' || wakeOnRunningRuntime) &&
        runtimeIsReady
      ) {
        let provision: RuntimeTokenProvision | null
        try {
          provision = await this.provisionRuntimeTokenRevision(host, {
            targetSuspended: false,
            refreshGfsOnly: true,
          })
        } catch (error) {
          // The Ready runtime keeps serving on the credential it holds; a failed
          // renewal is reported, not turned into an undeployed Host. The next
          // hold pass renews again.
          log.warn('held runtime credential renewal failed; keeping the Ready runtime', {
            host: host.name,
            namespace: host.namespace,
            err: error instanceof Error ? error.message : String(error),
          })
          // Telemetry only: this pass still ends with the runtime deployed and
          // Ready, so the administrative outcome it reports is `succeeded`. A
          // `failed` outcome here would give one intent both outcomes.
          this.enqueueHostTelemetry(host, 'controller_error', 'RuntimeCredentialRenewalFailed', {
            status: 'failed',
            error_class: error instanceof Error ? error.name : typeof error,
          })
          this.setStatus(host.name, {
            deployed: true,
            ready: runtimeIsReady,
            message: 'Held runtime credential renewal failed; runtime kept',
          })
          return
        }
        if (!provision) {
          this.setStatus(host.name, {
            deployed: true,
            ready: false,
            message: 'Held runtime credential renewal is not safely available',
          })
          return
        }
        revalidateHostMutationBoundary()
      }
      if (provisioningRequired) {
        const provision = await this.provisionRuntimeTokenRevision(host, {
          forceFreshForWake: true,
          // This mint exists only to scale the Host up from zero, so the
          // target is never suspended.
          targetSuspended: false,
          preserveDeploymentTemplateOnWake: true,
        })
        if (!provision) {
          this.setStatus(host.name, {
            deployed: !!liveDeployment,
            ready: false,
            message: 'Waiting for authoritative scope observation',
          })
          return
        }
        revalidateHostMutationBoundary()
      }
      const applied = await this.ensureDeployment(
        host,
        mounts,
        undefined,
        { ...lifecycle.effective, allowScaleUpDuringHold: allowScaleUp },
        undefined,
        revalidateHostMutationBoundary
      )
      revalidateHostMutationBoundary()
      const runtimeExpectedToRun =
        lifecycle.effective.state !== 'suspended' || allowScaleUp || wakeOnRunningRuntime
      if (runtimeExpectedToRun) {
        this.lifecycle.markHostNotSuspended(host.name)
      }
      const deploymentReady =
        (lifecycle.effective.state !== 'suspended' || wakeOnRunningRuntime) &&
        applied &&
        (await this.checkDeploymentReady(host.name, host.namespace))
      revalidateHostMutationBoundary()
      const ready = deploymentReady && pvcApplied && serviceApplied && npFailures.length === 0
      // PVC/Service read errors were already thrown before the hold; only
      // non-throwing convergence failures reach this point.
      const failures = [
        ...(!pvcApplied ? ['Host PVC did not converge'] : []),
        ...(!serviceApplied ? ['Host Service did not converge'] : []),
        ...npFailures,
      ]
      this.advanceReconcileGeneration(host.name)
      const timer = this.readinessTimers.get(host.name)
      if (timer) {
        clearTimeout(timer)
        this.readinessTimers.delete(host.name)
      }
      this.setStatus(host.name, {
        deployed: applied,
        ready,
        message: !applied
          ? 'Waiting for CommunicationChannel inventory before creating runtime'
          : failures.length > 0
            ? `degraded — Host runtime boundary incomplete while preserving applied runtime (${failures.join('; ')})`
            : !runtimeExpectedToRun
              ? 'CommunicationChannel inventory unavailable; preserving suspended Host replicas'
              : deploymentReady
                ? 'CommunicationChannel inventory unavailable; preserving applied runtime'
                : 'CommunicationChannel inventory unavailable; preserved applied runtime is not Ready',
      })
      // Mirror the normal path: a runtime expected to run (including a held
      // wake from zero) is polled until Ready; a degraded boundary is not,
      // because the poll would overwrite the degraded verdict with "Running".
      if (applied && !deploymentReady && failures.length === 0 && runtimeExpectedToRun) {
        this.pollReadiness(host.name, host.namespace)
      }
    }

    revalidateHostMutationBoundary()
    pvcApplied = await ensureIndependentResource('Host PVC', () =>
      this.ensurePvc(host, revalidateHostMutationBoundary)
    )
    revalidateHostMutationBoundary()
    serviceApplied = await ensureIndependentResource('Host Service', () =>
      this.ensureService(host, revalidateHostMutationBoundary)
    )

    // NetworkPolicies before Deployments. Calico/Cilium evaluate egress and
    // ingress against the policies that exist when the connection is opened,
    // so any pod created before its policy comes up runs briefly with the
    // namespace's deny-all default. The channel-reader path is fail-closed
    // (deny-all blocks message delivery), but creating the policy first
    // makes rollout deterministic and avoids transient readiness failures.
    //
    // The per-Host NPs are the network boundary for this direct runtime hop.
    // Route-level edge context and ownership checks remain the application
    // boundary. An NP apply failure means the network boundary is missing, so
    // surface it via HostRuntimeStatus instead of swallowing it.
    const ensureNetworkPolicy = async (
      name: string,
      reconcile: () => Promise<'not-required' | ResourceApplyResult>
    ): Promise<void> => {
      try {
        revalidateHostMutationBoundary()
        const result = await reconcile()
        revalidateHostMutationBoundary()
        if (result !== 'not-required' && !this.resourceApplySucceeded(result)) {
          throw new Error(`apply did not converge (${result})`)
        }
      } catch (err) {
        log.error('Failed to ensure Host NetworkPolicy', { host: host.name, policy: name, err })
        npFailures.push(`${name}: ${err instanceof Error ? err.message : String(err)}`)
      }
    }
    await ensureNetworkPolicy('mcp-host ingress', () =>
      this.ensureMcpHostIngressNetworkPolicy(host)
    )
    await ensureNetworkPolicy('mcp-host GFS egress', () =>
      this.ensureMcpHostGfsEgressNetworkPolicy(host)
    )
    await ensureNetworkPolicy('mcp-host Codex proxy egress', () =>
      this.reconcileMcpHostCodexProxyEgressNetworkPolicy(host)
    )
    await ensureNetworkPolicy('mcp-host Grok proxy egress', () =>
      this.reconcileMcpHostGrokProxyEgressNetworkPolicy(host)
    )
    // The mcp-host→llm-hooks egress policy is now owned by LlmHookReconciler
    // (per-host, scoped to referenced hook pods — N1/N7); host-delete cleanup
    // of `mcp-host-<host>-egress-llm-hooks` stays in deleteHostNetworkPolicies.
    await ensureNetworkPolicy('workflow approval reader ingress', () =>
      this.ensureWorkflowApprovalReaderMcpHostIngressNetworkPolicy(host)
    )
    await ensureNetworkPolicy('rpc-proxy mcp-host ingress', () =>
      this.ensureRpcProxyMcpHostIngressNetworkPolicy(host)
    )
    await ensureNetworkPolicy('rpc-proxy Host egress', () =>
      this.ensureRpcProxyHostEgressNetworkPolicy(host)
    )
    await ensureNetworkPolicy('desktop ingress', () => this.ensureDesktopNetworkPolicy(host))
    await ensureNetworkPolicy('channel-reader egress', () =>
      this.ensureChannelReaderEgressNetworkPolicy(host)
    )
    await ensureNetworkPolicy('workflow approval reader egress', () =>
      this.ensureWorkflowApprovalReaderHostEgressNetworkPolicy(host)
    )
    revalidateHostMutationBoundary()

    // Resolve policy once before bootstrap so token scopes and wake/suspend
    // decisions use the latest channel state already observed.
    const bootstrapLifecycle = this.lifecycle.enforceCommunicationChannelPolicyBeforeDeployment(
      host.name,
      lifecycle
    )
    if (bootstrapLifecycle !== lifecycle) {
      lifecycle = bootstrapLifecycle
      revalidateHostMutationBoundary()
      await this.lifecycle.writeLifecycleStatusToCluster(host, lifecycle)
    }
    revalidateHostMutationBoundary()

    // PVC/Service errors abort before any credential or Deployment mutation,
    // on the hold path as on the normal path.
    if (resourceErrors.length === 1) throw resourceErrors[0]
    if (resourceErrors.length > 1) {
      throw new AggregateError(
        resourceErrors,
        `Failed to reconcile Host resources for "${host.name}"`
      )
    }

    if (lifecycle.effective.suspensionBlocked) {
      await holdAppliedRuntime(forceFreshForWake)
      return
    }

    // Bootstrap captures the scope contract used for issuance. The Deployment
    // guard below compares that contract with the live channel cache after this
    // potentially slow I/O and before every create/replace attempt.
    let runtimeTokenProvision = await this.provisionRuntimeTokenRevision(host, {
      forceFreshForWake,
      targetSuspended: lifecycle.effective.stateless && lifecycle.effective.state === 'suspended',
      prepareForResourceChange: lifecycle.effective.state === 'active',
    })
    if (!runtimeTokenProvision) {
      const liveDeployment = await this.readHostDeploymentOrNull(host)
      this.setStatus(host.name, {
        deployed: !!liveDeployment,
        ready: false,
        message: 'Waiting for authoritative scope observation',
      })
      return
    }
    const requireRuntimeTokenProvision = (): RuntimeTokenProvision => {
      if (!runtimeTokenProvision) {
        const error = new Error('Waiting for authoritative scope observation')
        error.name = 'RuntimeScopeObservationUnavailableError'
        throw error
      }
      return runtimeTokenProvision
    }
    revalidateHostMutationBoundary()

    // replaceWithConflictRetry may wait and re-read after a 409, so each body is
    // rebuilt from a stable lifecycle plus token-scope pair before mutation.
    const resolveDeploymentLifecycle = async (): Promise<EffectiveHostLifecycle> => {
      revalidateHostMutationBoundary()
      const deploymentLifecycle = this.lifecycle.enforceCommunicationChannelPolicyBeforeDeployment(
        host.name,
        lifecycle
      )
      if (deploymentLifecycle !== lifecycle) {
        lifecycle = deploymentLifecycle
        revalidateHostMutationBoundary()
        await this.lifecycle.writeLifecycleStatusToCluster(host, lifecycle)
      }
      revalidateHostMutationBoundary()
      return lifecycle.effective
    }

    const ensureCurrentRuntimeTokenScope = async (): Promise<void> => {
      for (let attempt = 1; attempt <= 3; attempt++) {
        revalidateHostMutationBoundary()
        const currentOAuth = await this.observeFrontsOAuthServer(host)
        if (this.currentScopeMatchesProvision(host, requireRuntimeTokenProvision(), currentOAuth)) {
          return
        }

        runtimeTokenProvision = await this.provisionRuntimeTokenRevision(host, {
          forceFreshForWake: false,
          targetSuspended:
            lifecycle.effective.stateless && lifecycle.effective.state === 'suspended',
          prepareForResourceChange: lifecycle.effective.state === 'active',
        })
        if (!runtimeTokenProvision) {
          const error = new Error('Waiting for authoritative scope observation')
          error.name = 'RuntimeScopeObservationUnavailableError'
          throw error
        }
        revalidateHostMutationBoundary()
        const postProvisionOAuth = await this.observeFrontsOAuthServer(host)
        if (
          this.currentScopeMatchesProvision(
            host,
            requireRuntimeTokenProvision(),
            postProvisionOAuth
          )
        ) {
          return
        }

        log.warn('CommunicationChannel scope contract changed during token provisioning', {
          host: host.name,
          namespace: host.namespace,
          attempt,
        })
      }
      throw new Error(
        `CommunicationChannel scope contract did not stabilize before Deployment mutation for host "${host.name}"`
      )
    }

    const resolveDeploymentState = async (
      observed?: k8s.V1Deployment
    ): Promise<DeploymentMutationState> => {
      for (let attempt = 1; attempt <= 3; attempt++) {
        revalidateHostMutationBoundary()
        const beforeScope = await resolveDeploymentLifecycle()
        if (beforeScope.suspensionBlocked) {
          return {
            lifecycle: beforeScope,
            runtimeTokenRevision: requireRuntimeTokenProvision().revision,
            grokExecutionEnabled: this.hostDerivesGrokExecution(host),
          }
        }
        await ensureCurrentRuntimeTokenScope()
        const deploymentIdentityChanged =
          !!observed?.metadata?.uid &&
          !!requireRuntimeTokenProvision().deploymentUid &&
          observed.metadata.uid !== requireRuntimeTokenProvision().deploymentUid
        const observedResourceTransition =
          !!observed &&
          beforeScope.state === 'active' &&
          HostReconciler.deploymentBelongsToHost(observed, host) &&
          (observed.spec?.replicas ?? 1) > 0 &&
          !HostReconciler.deploymentResourcesMatchHost(observed, host)
        if (deploymentIdentityChanged || observedResourceTransition) {
          // A CAS retry or existence read can reveal a new UID or resource drift.
          // Prepare against this observation before rebuilding the Pod template;
          // persisted preparation is reused for retries against the same target.
          runtimeTokenProvision = await this.provisionRuntimeTokenRevision(host, {
            forceFreshForDeploymentChange: deploymentIdentityChanged,
            targetSuspended: beforeScope.stateless && beforeScope.state === 'suspended',
            prepareForResourceChange: beforeScope.state === 'active',
          })
          requireRuntimeTokenProvision()
          revalidateHostMutationBoundary()
        }
        const effective = await resolveDeploymentLifecycle()
        const currentOAuth = await this.observeFrontsOAuthServer(host)
        // The OAuth lookup can await I/O after the last lifecycle observation.
        // Retry assessment rather than committing a now-obsolete mode.
        if (
          this.lifecycle.enforceCommunicationChannelPolicyBeforeDeployment(host.name, lifecycle) !==
          lifecycle
        ) {
          continue
        }
        if (this.currentScopeMatchesProvision(host, requireRuntimeTokenProvision(), currentOAuth)) {
          revalidateHostMutationBoundary()
          return {
            lifecycle: effective,
            runtimeTokenRevision: requireRuntimeTokenProvision().revision,
            freshBootstrapPrepared: requireRuntimeTokenProvision().freshBootstrapPrepared,
            grokExecutionEnabled: this.hostDerivesGrokExecution(host),
          }
        }
        log.warn('CommunicationChannel scope contract changed before Deployment mutation', {
          host: host.name,
          namespace: host.namespace,
          attempt,
        })
      }
      throw new Error(
        `CommunicationChannel lifecycle and token scope did not stabilize before Deployment mutation for host "${host.name}"`
      )
    }

    revalidateHostMutationBoundary()
    let deploymentApplied: boolean
    try {
      deploymentApplied = await this.ensureDeployment(
        host,
        mounts,
        runtimeTokenProvision.revision,
        lifecycle.effective,
        resolveDeploymentState,
        revalidateHostMutationBoundary
      )
    } catch (error) {
      if (
        error instanceof Error &&
        (error.name === 'RuntimeScopeObservationUnavailableError' ||
          error.name === 'RuntimeBootstrapPreparationUnavailableError')
      ) {
        const liveDeployment = await this.readHostDeploymentOrNull(host)
        this.setStatus(host.name, {
          deployed: !!liveDeployment,
          ready: false,
          message:
            error.name === 'RuntimeBootstrapPreparationUnavailableError'
              ? error.message
              : 'Waiting for authoritative scope observation',
        })
        return
      }
      throw error
    }
    revalidateHostMutationBoundary()

    if (!deploymentApplied) {
      this.setStatus(host.name, {
        deployed: false,
        ready: false,
        message: 'Waiting for CommunicationChannel inventory before creating runtime',
      })
      return
    }

    const suspended = lifecycle.effective.stateless && lifecycle.effective.state === 'suspended'
    if (suspended) {
      this.lifecycle.recordSuspendedApplied(host.name)
    } else {
      this.lifecycle.markHostNotSuspended(host.name)
    }
    const ready = suspended ? false : await this.checkDeploymentReady(host.name, host.namespace)
    revalidateHostMutationBoundary()
    // If any per-Host NP failed to apply, mark the host as degraded:
    // deployed remains true (the pod is in place), but ready=false +
    // a message that names the missing security boundary. Operators see
    // this in `kubectl get host ... -o yaml` under status.runtime.message.
    const baseMessage = suspended
      ? 'Suspended (stateless lifecycle, replicas=0)'
      : ready
        ? 'Running'
        : 'Deployment created, waiting for pods to become ready'
    const message = npFailures.length
      ? `degraded — NetworkPolicy boundary missing (${npFailures.join('; ')})`
      : baseMessage
    this.setStatus(host.name, {
      deployed: true,
      ready: ready && npFailures.length === 0,
      message,
    })
    // Do not schedule a readiness poll when NetworkPolicy has degraded the
    // runtime: the same-generation poll would overwrite the degraded verdict
    // with "Running" once the Deployment alone becomes Ready.
    if (!ready && !suspended && npFailures.length === 0) {
      this.pollReadiness(host.name, host.namespace)
    }

    // Materialize the per-Host channel-reader Deployment. The egress NP that
    // pins this pod to mcp-host has already been applied above. A failure does
    // not remove the converged mcp-host pod, but it rejects this reconcile so
    // the caller can retry and is surfaced in HostRuntimeStatus.
    //
    // computeChannelReaderRevisionForHost re-throws on Secret read errors
    // other than 404 (RBAC drift, 503, etc.), so this catch handles a real
    // expected error path — not just defense in depth.
    const channelReaderFailures: unknown[] = []
    for (const reconcileResource of [
      () => this.reconcileChannelReaderService(host, revalidateHostMutationBoundary),
      () => this.reconcileChannelReaderDeployment(host, revalidateHostMutationBoundary),
    ]) {
      try {
        revalidateHostMutationBoundary()
        await reconcileResource()
      } catch (err) {
        channelReaderFailures.push(err)
      }
    }
    const channelReaderError = channelReaderFailures
      .map(error => (error instanceof Error ? error.message : String(error)))
      .join('; ')
    if (channelReaderError) {
      const prev = this.getStatus(host.name)
      this.setStatus(host.name, {
        ...prev,
        message: `channel-reader: ${channelReaderError}`,
      })
    }

    // Surface channel-reader Deployment readiness in HostRuntimeStatus.
    // Best-effort — checkChannelReaderStatus never throws.
    const ccCount = this.countCommunicationChannels(host.name)
    const channelReaderStatus = await this.checkChannelReaderStatus(host.name, ccCount)
    revalidateHostMutationBoundary()
    if (channelReaderFailures.length > 0) {
      // Overlay the reconcile error onto the status message so both signals
      // are visible (the deploy error may explain why the Deployment is missing).
      channelReaderStatus.message = `reconcile error: ${channelReaderError}`
      channelReaderStatus.ready = false
      this.enqueueControllerError(
        host,
        'channel_reader_reconcile_failed',
        new Error(channelReaderError)
      )
    }
    const prev = this.getStatus(host.name)
    this.setStatus(host.name, { ...prev, channelReader: channelReaderStatus })
    throwCleanupFailures(
      channelReaderFailures,
      `Failed to converge channel-reader resources for Host "${host.name}"`
    )
  }

  /**
   * Delete a Host's runtime bundle.
   *
   * `opts.skipIf` is an OPTIONAL fence evaluated INSIDE the per-Host serializer,
   * immediately before anything destructive runs. Callers whose "this Host is
   * gone" decision was made BEFORE admission (the watch-recovery diff, which
   * compares LIST-time names) pass it so a same-name Host recreated during the
   * admission window is not wiped: the recreation's ADDED sets the cache and
   * enters this Host's chain FIRST, its reconcile rebuilds the bundle, and only
   * then does the stale delete reach the front of the strict-FIFO queue.
   * Ownership labels cannot discriminate here — the rebuilt bundle carries
   * identical `clerum.io/host` + managed-by labels — and no uid is available on
   * that path, so cache presence at admission is the correct fence. This
   * mirrors the F2/#827 in-serializer re-check in collectHostCleanupFailures.
   *
   * Callers that omit `opts` drop only the extra `skipIf` term; the
   * resolveCurrentHost supersede fence still gates the delete.
   */
  async reconcileDelete(
    name: string,
    namespace: string,
    opts?: { skipIf?: () => boolean }
  ): Promise<void> {
    const capturedAuthority = this.captureHostMutationAuthority()
    // A DELETED event must not race an in-flight create/update for the same
    // Host: route it through the same per-Host serializer as reconcile(). The
    // global convergence tail no longer provides this ordering.
    await this.lifecycle.serializeByHost(name, async () => {
      this.requireHostMutationAuthority(`Host "${name}" delete`, capturedAuthority)
      if (this.resolveCurrentHost?.(name) !== undefined || opts?.skipIf?.()) {
        // Same vocabulary the watch path already reports for this condition
        // (reconcileHostWatchEvent), so the skip is observable, not silent.
        hostDeleteCleanupTotal.inc({ outcome: 'superseded' })
        return
      }
      await this.deleteHostRuntimeResources(name, namespace)
      this.clearStatus(name)
      this.desktopHosts.delete(name)
      // The delete path has no uid to key by, so every entry for this
      // namespace/name goes: the object is gone and no outcome will carry its
      // evidence (#696).
      this.clearGfsLifecycleEvidence(namespace, name)
    })
  }

  /**
   * Reconcile the fleet with bounded cross-Host concurrency. Dispatches Host
   * KEYS (not stale object snapshots): each worker resolves the CURRENT cached
   * Host at execution time and skips entries the cache no longer holds
   * (superseded by a delete/newer event). Failures are aggregated completely.
   */
  private async collectHostReconcileFailures(
    desiredHosts: HostCRD[],
    source: HostReconcileSource
  ): Promise<unknown[]> {
    const snapshotByName = new Map(desiredHosts.map(h => [h.name, h]))
    const resolve = this.resolveCurrentHost ?? ((name: string) => snapshotByName.get(name))
    const keys = desiredHosts.map(h => h.name)
    return runBoundedHostWorkers(keys, config.hostFullReconcileConcurrency, async name => {
      const host = resolve(name)
      if (!host) return // absent/superseded since the pass captured its snapshot
      try {
        await this.reconcile(host, source)
      } catch (error) {
        if (isBenignSupersessionError(error)) {
          // Watch-race withdrawal is not a host failure.
          hostFleetBenignSupersessionsTotal.inc({ error: error.name })
          log.warn('Fleet Host reconcile withdrawn as benign supersession', {
            err: error,
            errorName: error.name,
          })
          return
        }
        log.error('Fleet reconcile failed for Host', { host: name, err: error })
        throw error
      }
    })
  }

  /**
   * Orphan cleanup with the §10.5 authority contract (replaces the accidental
   * safety the removed global tail used to provide):
   *  - runs only with known watch authority and the SAME generation captured at
   *    pass start;
   *  - compares candidates with the CURRENT cache, not the pass snapshot;
   *  - groups candidates by owning logical Host from ownership labels; a
   *    candidate with no derivable owner is NEVER deleted (deferred + recorded);
   *  - performs ONE fresh authoritative Host read per orphan Host; a confirmed
   *    404 (while authority stays valid) authorizes deleting that Host's entire
   *    owned bundle INSIDE the Host's per-Host serializer;
   *  - defers (with a bounded reason) on unknown authority, generation drift,
   *    watch loss, a present read, or a failed read.
   */
  private async collectHostCleanupFailures(
    capturedAuthority: HostWatchAuthoritySnapshot
  ): Promise<unknown[]> {
    const failures: unknown[] = []
    const authorityValid = (): boolean => {
      const current = this.hostWatchAuthority()
      return (
        capturedAuthority.known &&
        current.known &&
        capturedAuthority.generation === current.generation
      )
    }

    if (!authorityValid()) {
      const current = this.hostWatchAuthority()
      const reason =
        !capturedAuthority.known || !current.known ? 'authority_unknown' : 'generation_changed'
      hostCleanupDeferredTotal.inc({ reason })
      log.warn('Deferring orphan cleanup', { reason })
      return failures
    }

    let candidates: CleanupCandidate[]
    try {
      candidates = await this.gatherCleanupCandidates(failures)
    } catch (error) {
      failures.push(error)
      return failures
    }

    const cacheHasHost = (name: string): boolean =>
      this.resolveCurrentHost ? this.resolveCurrentHost(name) !== undefined : false

    const orphanHosts = new Set<string>()
    for (const candidate of candidates) {
      if (!candidate.owner) {
        // Never delete a resource whose owning Host cannot be derived.
        hostCleanupDeferredTotal.inc({ reason: 'no_owner_label' })
        log.warn('Deferring orphan candidate: no derivable owning Host', {
          kind: candidate.kind,
          name: candidate.name,
        })
        continue
      }
      if (cacheHasHost(candidate.owner)) continue // still present → retain
      orphanHosts.add(candidate.owner)
    }

    // §10.5 destructive cleanup runs per orphan logical Host. Each Host's owned
    // write set is disjoint (§10.4 inventory) and each destructive bundle runs
    // inside that Host's own `serializeByHost` chain, so distinct orphan Hosts
    // are provably independent. Addendum 2: dispatch them through the same
    // bounded worker pool as the fleet (reusing HCC_HOST_FULL_RECONCILE_CONCURRENCY)
    // instead of a serial `for … await` — never an unbounded fan-out at the K8s
    // API. Same-Host ordering is still guaranteed by serializeByHost, and delete
    // failures still aggregate completely because runBoundedHostWorkers collects
    // every lane's throw.
    const cleanupFailures = await runBoundedHostWorkers(
      Array.from(orphanHosts),
      config.hostFullReconcileConcurrency,
      async hostName => {
        if (!authorityValid()) {
          hostCleanupDeferredTotal.inc({ reason: 'watch_lost' })
          log.warn('Deferring orphan cleanup: watch authority lost', { host: hostName })
          return
        }
        const presence = await this.readHostPresence(hostName)
        if (presence === 'present') return // reappeared → retain
        if (presence === 'error') {
          hostCleanupDeferredTotal.inc({ reason: 'fresh_read_failed' })
          return
        }
        if (
          !destructiveCleanupAllowed({
            watchAuthorityKnown: this.hostWatchAuthority().known,
            capturedWatchGeneration: capturedAuthority.generation,
            currentWatchGeneration: this.hostWatchAuthority().generation,
            currentCacheOmitsHost: !cacheHasHost(hostName),
            // Derived, not hardcoded. `presence` is provably 'absent' here (the
            // 'present' and 'error' branches returned above), so this evaluates
            // to the identical literal and the runtime behavior is unchanged.
            // Deriving it makes the mapping total: if readHostPresence ever
            // grows a fourth outcome, the else-branch type stops satisfying
            // freshAuthoritativeRead's union and the compiler rejects it —
            // instead of silently labelling a non-404 read as 'confirmed404'
            // in a value that is DECISIONAL (a conjunct of
            // destructiveCleanupAllowed, :94-95), not telemetry.
            freshAuthoritativeRead: presence === 'absent' ? 'confirmed404' : presence,
          })
        ) {
          hostCleanupDeferredTotal.inc({ reason: 'watch_lost' })
          return
        }
        try {
          await this.lifecycle.serializeByHost(hostName, async () => {
            // The fresh 404 and the cache omission are only destructive
            // authority while the same LIST -> WATCH generation remains live.
            // Revalidate inside the serializer: queued Host work can otherwise
            // outlive that generation and delete a same-name recreation whose
            // ADDED event has not reached the retired cache.
            if (!authorityValid()) {
              hostCleanupDeferredTotal.inc({ reason: 'watch_lost' })
              return
            }
            // F2 / #827 TOCTOU: a same-name Host recreated between the fresh read
            // above and admission into this serializer updates the current cache
            // (its ADDED event sets the cache before its reconcile runs, and that
            // reconcile serializes ahead of this delete). Re-check the current
            // cache INSIDE the serializer so a just-recreated Host's bundle is
            // never deleted by this stale cleanup.
            if (cacheHasHost(hostName)) {
              hostCleanupDeferredTotal.inc({ reason: 'recreated' })
              return
            }
            await this.deleteHostRuntimeResources(hostName, config.hostNamespace)
          })
        } catch (error) {
          log.error('Fleet cleanup failed for orphan Host', { host: hostName, err: error })
          throw error
        }
      }
    )
    failures.push(...cleanupFailures)
    return failures
  }

  /**
   * List every managed resource that could be an orphan and derive its owning
   * logical Host from ownership labels. List failures are collected into
   * `listFailures` (reported by the caller) so a partial discovery still cleans
   * what it could confidently attribute. Candidates never drive a delete on
   * their own — the caller applies the fresh-read authority gate.
   */
  private async gatherCleanupCandidates(listFailures: unknown[]): Promise<CleanupCandidate[]> {
    const candidates: CleanupCandidate[] = []
    const pushOwner = (kind: string, name: string | undefined, owner: string | undefined): void => {
      if (!name) return
      candidates.push({ kind, name, owner })
    }

    try {
      for (const deployment of await this.listManagedHostDeployments()) {
        pushOwner(
          'Deployment',
          deployment.metadata?.name,
          deployment.metadata?.labels?.[HOST_LABEL]
        )
      }
    } catch (error) {
      log.error('Failed to list managed host deployments', { err: error })
      listFailures.push(error)
    }

    const channelsNs = config.channelsNamespace
    const managed = (labels: Record<string, string> | undefined): boolean =>
      labels?.[MANAGED_BY_LABEL] === MANAGED_BY_VALUE
    try {
      const list = await this.appsApi.listNamespacedDeployment({ namespace: channelsNs })
      for (const dep of list.items ?? []) {
        const labels = dep.metadata?.labels
        if (labels?.app !== 'channel-reader' || !managed(labels)) continue
        pushOwner('ChannelReaderDeployment', dep.metadata?.name, labels[HOST_LABEL])
      }
    } catch (error) {
      log.warn('channel-reader Deployment candidate list failed', { err: error })
      listFailures.push(error)
    }
    try {
      const list = await this.coreApi.listNamespacedService({ namespace: channelsNs })
      for (const svc of list.items ?? []) {
        const labels = svc.metadata?.labels
        if (labels?.app !== 'channel-reader' || !managed(labels)) continue
        pushOwner('ChannelReaderService', svc.metadata?.name, labels[HOST_LABEL])
      }
    } catch (error) {
      log.warn('channel-reader Service candidate list failed', { err: error })
      listFailures.push(error)
    }
    try {
      const list = await this.coreApi.listNamespacedSecret({
        namespace: channelsNs,
        labelSelector: [
          `${MANAGED_BY_LABEL}=${MANAGED_BY_VALUE}`,
          'clerum.io/component=channel-reader',
        ].join(','),
      })
      for (const sec of list.items ?? []) {
        const labels = sec.metadata?.labels
        if (!managed(labels) || labels?.['clerum.io/component'] !== 'channel-reader') continue
        pushOwner('ChannelReaderSecret', sec.metadata?.name, labels[HOST_LABEL])
      }
    } catch (error) {
      log.warn('channel-reader Secret candidate list failed', { err: error })
      listFailures.push(error)
    }

    // #827 (Addendum 4 item 4): partial-leftover discovery. A bundle whose
    // Deployment — and even its NetworkPolicies — are already gone must still be
    // discovered from ANY surviving owned resource. List every owned kind HCC
    // writes in the host namespace by ownership label and derive the owner from
    // clerum.io/host; a candidate without a derivable owner is never deleted by
    // the caller (§10.5).
    const hostNs = config.hostNamespace
    const managedSelector = `${MANAGED_BY_LABEL}=${MANAGED_BY_VALUE}`
    const gatherOwnedKind = async (
      kind: string,
      list: () => Promise<{
        items?: Array<{ metadata?: { name?: string; labels?: Record<string, string> } }>
      }>
    ): Promise<void> => {
      try {
        const res = await list()
        for (const item of res.items ?? []) {
          if (!managed(item.metadata?.labels)) continue
          pushOwner(kind, item.metadata?.name, item.metadata?.labels?.[HOST_LABEL])
        }
      } catch (error) {
        log.warn('Owned resource candidate list failed', { kind, namespace: hostNs, err: error })
        listFailures.push(error)
      }
    }
    await gatherOwnedKind('Service', () =>
      this.coreApi.listNamespacedService({ namespace: hostNs, labelSelector: managedSelector })
    )
    await gatherOwnedKind('PersistentVolumeClaim', () =>
      this.coreApi.listNamespacedPersistentVolumeClaim({
        namespace: hostNs,
        labelSelector: managedSelector,
      })
    )
    await gatherOwnedKind('ServiceAccount', () =>
      this.coreApi.listNamespacedServiceAccount({
        namespace: hostNs,
        labelSelector: managedSelector,
      })
    )
    await gatherOwnedKind('Secret', () =>
      this.coreApi.listNamespacedSecret({ namespace: hostNs, labelSelector: managedSelector })
    )
    await gatherOwnedKind('Role', () =>
      this.rbacApi.listNamespacedRole({ namespace: hostNs, labelSelector: managedSelector })
    )
    await gatherOwnedKind('RoleBinding', () =>
      this.rbacApi.listNamespacedRoleBinding({ namespace: hostNs, labelSelector: managedSelector })
    )

    for (const ns of [config.channelsNamespace, config.hostNamespace, config.rpcProxyNamespace]) {
      try {
        const list = await this.networkingApi.listNamespacedNetworkPolicy({
          namespace: ns,
          labelSelector: `${MANAGED_BY_LABEL}=${MANAGED_BY_VALUE}`,
        })
        for (const np of list.items ?? []) {
          pushOwner('NetworkPolicy', np.metadata?.name, np.metadata?.labels?.[HOST_LABEL])
        }
      } catch (error) {
        log.warn('NetworkPolicy candidate list failed', { namespace: ns, err: error })
        listFailures.push(error)
      }
    }

    return candidates
  }

  /**
   * One fresh authoritative Host read. `absent` is a confirmed 404 (the Host is
   * genuinely gone). `present` means the Host reappeared. `error` (timeout,
   * forbidden, 5xx) is deliberately NOT surfaced as a pass failure — the caller
   * defers with a bounded reason, matching the §13.2 cleanup matrix.
   */
  private async readHostPresence(name: string): Promise<'absent' | 'present' | 'error'> {
    try {
      await this.customApi.getNamespacedCustomObject({
        group: HOST_GROUP,
        version: HOST_VERSION,
        namespace: config.hostNamespace,
        plural: HOST_PLURAL,
        name,
      })
      return 'present'
    } catch (error) {
      if (getErrorCode(error) === 404) return 'absent'
      log.warn('Fresh authoritative read for orphan Host failed', { host: name, err: error })
      return 'error'
    }
  }

  async reconcileHosts(desiredHosts: HostCRD[]): Promise<void> {
    const hostFailures = await this.collectHostReconcileFailures(desiredHosts, 'fleet')
    if (hostFailures.length > 0) {
      throw new HostFleetReconcileError(hostFailures, [])
    }
  }

  async fullReconcile(desiredHosts: HostCRD[]): Promise<void> {
    // Capture watch authority + generation at pass start (§10.5 step 1) so
    // cleanup can prove nothing shifted underneath it before deleting.
    const capturedAuthority = this.hostWatchAuthority()
    const hostFailures = await this.collectHostReconcileFailures(desiredHosts, 'fleet')
    // Cleanup runs only after every fleet worker has settled.
    const cleanupFailures = await this.collectHostCleanupFailures(capturedAuthority)
    if (hostFailures.length > 0 || cleanupFailures.length > 0) {
      throw new HostFleetReconcileError(hostFailures, cleanupFailures)
    }
  }

  private async listManagedHostDeployments(): Promise<k8s.V1Deployment[]> {
    const response = await this.appsApi.listNamespacedDeployment({
      namespace: config.hostNamespace,
      labelSelector: `${MANAGED_BY_LABEL}=${MANAGED_BY_VALUE}`,
    })
    return (response.items || []).filter(item => item.metadata?.labels?.[HOST_LABEL])
  }
}

/**
 * True when the desired Role matches the live object on the fields HCC authors:
 * `rbacLabels` and `rules`. Server metadata and annotations are ignored — the
 * builder does not write annotations, and comparing them would force a PUT.
 *
 * Label comparison is a subset of the keys HCC authors. Extra live labels
 * (Kyverno/Gatekeeper add-labels, ArgoCD instance, cost tags) must not force a
 * PUT that strips them and re-enters the write loop. A missing or changed
 * authored key still fail-opens to write, so a later third `rbacLabels` key
 * still lands. A key HCC stops authoring is treated as extra-live and will
 * not PUT until some other authored field drifts; the previous exact-map
 * compare used to retract it. `isHccOwnedHostResource` keys on the same two
 * labels today.
 *
 * Object keys inside each rule are canonicalized: client-node rebuilds
 * V1PolicyRule in attributeTypeMap order (resourceNames before resources), so a
 * raw JSON.stringify never matches a live GET (#307). Rule arrays, verbs, and
 * resourceNames stay in author order. Missing rules or labels, or any compare
 * failure, returns false (fail-open-to-write).
 */
function roleMatchesDesired(desired: k8s.V1Role, existing: k8s.V1Role): boolean {
  try {
    if (!desired.rules || !existing.rules) return false
    const desiredLabels = desired.metadata?.labels
    const existingLabels = existing.metadata?.labels
    if (!desiredLabels || !existingLabels) return false
    const authored: Record<string, string> = {}
    const liveAuthored: Record<string, string> = {}
    for (const key of Object.keys(desiredLabels)) {
      if (existingLabels[key] === undefined) return false
      authored[key] = desiredLabels[key]
      liveAuthored[key] = existingLabels[key]
    }
    return (
      JSON.stringify(canonicalizeValue({ labels: authored, rules: desired.rules })) ===
      JSON.stringify(canonicalizeValue({ labels: liveAuthored, rules: existing.rules }))
    )
  } catch {
    return false
  }
}

/**
 * Preserve operational pod-template annotations (kubectl restart markers,
 * operator edits) while keeping CONTROLLER-OWNED ones authoritative: a key in
 * `controllerOwned` that the desired template omits is REMOVED from the merge
 * rather than surviving from the live object.
 *
 * Without this, "the field went away" cannot be expressed at all. The merge is
 * `{...existing, ...desired}`, so an omitted key keeps its old value, the merged
 * object then compares equal to what is live, and `replaceWithConflictRetry`
 * skips the write entirely — no template diff, no rolling restart, no write.
 */
export function preserveDeploymentAnnotationsExcept(
  desired: k8s.V1Deployment,
  existing: k8s.V1Deployment,
  controllerOwned: readonly string[]
): k8s.V1Deployment {
  const preserved = preserveDeploymentAnnotations(desired, existing)
  const desiredAnnotations = desired.spec?.template?.metadata?.annotations
  const dropped = controllerOwned.filter(key => desiredAnnotations?.[key] === undefined)
  if (dropped.length === 0) return preserved

  const spec = preserved.spec
  const template = spec?.template
  if (!spec || !template) return preserved

  const annotations = { ...(template.metadata?.annotations ?? {}) }
  for (const key of dropped) delete annotations[key]
  return {
    ...preserved,
    spec: {
      ...spec,
      template: {
        ...template,
        metadata: {
          ...template.metadata,
          annotations: Object.keys(annotations).length > 0 ? annotations : undefined,
        },
      },
    },
  }
}

/**
 * Host Deployment merge. `clerum.io/guardrails-revision` is controller-owned:
 * dropping the whole `spec.guardrails` block omits it from the desired template,
 * and that removal MUST reach the pod. mcp-host re-resolves guardrails live, but
 * only while its Host watch is connected — the pod roll is what delivers the
 * change to an agent whose watch has lapsed, so a removal that silently produced
 * no diff left the agent enforcing guardrails the operator had uninstalled.
 */
function preserveHostDeploymentAnnotations(
  desired: k8s.V1Deployment,
  existing: k8s.V1Deployment
): k8s.V1Deployment {
  return preserveDeploymentAnnotationsExcept(desired, existing, [GUARDRAILS_REVISION_ANNOTATION])
}

/**
 * Preserve operational annotations without retaining the channel-reader
 * revision when HCC intentionally omits it from the desired pod template.
 * That annotation is controller-owned and must be cleared when no backing
 * CommunicationChannel Secret is resolvable.
 */
function preserveChannelReaderDeploymentAnnotations(
  desired: k8s.V1Deployment,
  existing: k8s.V1Deployment
): k8s.V1Deployment {
  return preserveDeploymentAnnotationsExcept(desired, existing, ['clerum.io/credentials-revision'])
}