Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
import * as k8s from '@kubernetes/client-node'
import { IntOrString } from '@kubernetes/client-node/dist/types.js'
import { createHash, randomUUID } from 'crypto'
import { isDeepStrictEqual } from 'node:util'
import * as path from 'path'
import type { AdministrativeOutcomeReporter } from './administrativeOutcomeReporter'
import {
type CodexCatalogSnapshot,
type CodexExecutionProjection,
GROK_EXECUTE_SCOPE,
assignedHostCodexConnectionRef,
projectCodexExecution,
projectGrokExecution,
} from './codexExecutionProjection'
import { config } from './config'
import { HOST_LABEL, MANAGED_BY_LABEL, MANAGED_BY_VALUE } from './constants'
import { mintHostGfsToken } from './gfsHostBinding'
import { GFS_HOST_SCOPES } from './gfsHostPolicy'
import { makeExpectedHostGfsSubject } from './gfsHostSubject'
import type {
HccInfrastructureTelemetryPayload,
HostLookupReference,
InfrastructureTelemetryReporter,
} from './infrastructureTelemetryReporter'
import {
HOST_K8S_REQUEST_TIMEOUT_CODE,
HostK8sRequestTimeoutError,
makeHostK8sApiClient,
} from './k8s/hostK8sApiClient'
import {
SFS_LABEL,
SFS_NAMESPACE_LABEL,
WFC_APP_LABEL,
sharedFileSystemHash,
} from './k8s/sharedFileSystemFactory'
import { kubernetesQuantitiesEqual } from './kubernetesQuantity'
import {
ALLOWED_MODELS_CONFIGMAP_NAME,
parseAllowedModelsSnapshot,
parseGrokAllowedModelsSnapshot,
snapshotForAssignedCodexGrant,
snapshotFromConfigMapError,
} from './llmAllowedModelsSnapshot'
import { hccLogger } from './logger'
import { issueMcpHostRuntimeTokens } from './mcpHostRuntimeTokenIssuerClient'
import {
createsTotal,
hostCleanupDeferredTotal,
hostDeleteCleanupTotal,
hostFleetBenignSupersessionsTotal,
hostReconcileDurationSeconds,
hostReconcileInFlight,
hostReconcileQueueWaitSeconds,
} from './metrics'
import {
MCP_HOST_GFS_TOKEN_SECRET_KEY,
MCP_HOST_RUNTIME_TOKEN_SECRET_ACCESS_KEY,
MCP_HOST_RUNTIME_TOKEN_SECRET_CONTROL_KEY,
MCP_HOST_RUNTIME_TOKEN_SECRET_REFRESH_KEY,
buildMcpHostRuntimeTokenSecret,
mcpHostRuntimeTokenSecretName,
} from './secretFactory'
import {
buildWorkspaceLayoutMergeContainer,
decodeJwtExpMs,
effectiveRotateBeforeMs,
resolveStatelessImagePullPolicy,
} from './statelessDeployment'
import { EffectiveHostLifecycle, SuspendFromHeartbeatOutcome } from './statelessLifecycle.types'
import { ReflectHostOutcomeFn, StatelessLifecycleExecutor } from './statelessLifecycleExecutor'
import {
CommunicationChannelCRD,
EffectiveMcpHostControlScope,
HostCRD,
HostChannelReaderStatus,
HostRuntimeControlScope,
HostRuntimeStatus,
HostWorkflowControlScope,
HostWorkflowControlSpec,
} from './types'
import {
type ResourceApplyResult,
applyNetworkPolicy,
canonicalStringify,
canonicalizeValue,
deploymentMatchesDesired,
ensureResource,
getErrorCode,
observeCreate,
observeExistenceRead,
preserveDeploymentAnnotations,
preserveServiceAssignedFields,
replaceWithConflictRetry,
serviceMatchesDesired,
} from './utils'
export type { EffectiveHostLifecycle } from './statelessLifecycle.types'
const HOST_GROUP = 'clerum.io'
const HOST_VERSION = 'v1alpha1'
const HOST_PLURAL = 'hosts'
/**
* Lane label for a Host reconcile, used only for low-cardinality telemetry:
* `urgent` (a direct watch ADDED/MODIFIED/DELETED/wake event), `retry` (a
* bounded watch-reconcile retry), or `fleet` (a bounded background full/
* lifecycle worker). Never a host name, user, or team.
*/
export type HostReconcileSource = 'urgent' | 'retry' | 'fleet'
/** Immutable Host watch authority snapshot captured at a full-pass boundary. */
export type HostWatchAuthoritySnapshot = { known: boolean; generation: number }
/**
* Immutable lease for every inventory that contributes to a Host mutation.
* Host resources come from the Host watch; mounted SharedFileSystems come from
* the Context watch. Both are CONTENT revisions (they move only on a real
* desired-state change), not channel generations — a watch reconnect that
* re-LISTs the identical inventory must not retire an in-flight Host lease, or
* sustained watch-churn starves every queued reconcile. Keeping both revisions
* still prevents a false-stable Host-only lease when Context desired state
* changes during queued work.
*/
export type HostMutationAuthoritySnapshot = {
known: boolean
hostRevision: number
contextRevision: number
}
class HostInventoryAuthorityUnavailableError extends Error {
constructor(
action: string,
captured: HostMutationAuthoritySnapshot,
current: HostMutationAuthoritySnapshot
) {
super(
`Host inventory authority changed before ${action} admission ` +
`(captured known=${captured.known} hostRevision=${captured.hostRevision} ` +
`contextRevision=${captured.contextRevision}, current known=${current.known} ` +
`hostRevision=${current.hostRevision} contextRevision=${current.contextRevision})`
)
this.name = 'HostInventoryAuthorityUnavailableError'
}
}
class HostMutationIdentityChangedError extends Error {
constructor(action: string, hostName: string) {
super(`Host identity changed before ${action} admission for "${hostName}"`)
this.name = 'HostMutationIdentityChangedError'
}
}
class RuntimeBootstrapPreparationUnavailableError extends Error {
constructor() {
super('Waiting for a prepared runtime bootstrap before changing resources')
this.name = 'RuntimeBootstrapPreparationUnavailableError'
}
}
class HostMutationSpecRevisionChangedError extends Error {
constructor(action: string, hostName: string) {
super(`Host spec generation changed before ${action} admission for "${hostName}"`)
this.name = 'HostMutationSpecRevisionChangedError'
}
}
class HostMutationDependencyChangedError extends Error {
constructor(action: string, hostName: string) {
super(`Host mutation dependency changed before ${action} admission for "${hostName}"`)
this.name = 'HostMutationDependencyChangedError'
}
}
/**
* The four benign mid-pass supersession signals. revalidateHostMutationBoundary()
* throws these when the watch observes a newer spec/uid/authority-generation while
* a reconcile is in flight; a fresh reconcile runs afterwards, so the retired pass
* is a WITHDRAWAL, not a hard failure. Matched by `error.name` (string), NOT
* `instanceof`: the classes are unexported and statelessLifecycleExecutor.ts
* synthesizes name-equivalent plain Errors for the same races.
*/
const BENIGN_SUPERSESSION_ERROR_NAMES: ReadonlySet<string> = new Set([
'HostInventoryAuthorityUnavailableError',
'HostMutationIdentityChangedError',
'HostMutationSpecRevisionChangedError',
'HostMutationDependencyChangedError',
])
function isBenignSupersessionError(error: unknown): error is Error {
return error instanceof Error && BENIGN_SUPERSESSION_ERROR_NAMES.has(error.name)
}
/**
* The exact §13.2 destructive-cleanup predicate. Deletion of an orphan Host's
* owned bundle is permitted ONLY when watch authority is known, the watch
* generation captured at pass start still matches the current generation, the
* current cache omits the Host, AND a fresh authoritative read confirmed a 404.
* Any weaker evidence defers.
*/
export function destructiveCleanupAllowed(input: {
watchAuthorityKnown: boolean
capturedWatchGeneration: number
currentWatchGeneration: number
currentCacheOmitsHost: boolean
freshAuthoritativeRead: 'confirmed404' | 'present' | 'error'
}): boolean {
return (
input.watchAuthorityKnown &&
input.capturedWatchGeneration === input.currentWatchGeneration &&
input.currentCacheOmitsHost &&
input.freshAuthoritativeRead === 'confirmed404'
)
}
/**
* Dependency-free bounded worker pool. Dispatches `keys` across at most
* `concurrency` lanes; each lane pulls the next key, runs `worker`, and
* collects (never swallows) failures so aggregation stays complete. Lane
* startup is ordered so the first `concurrency` keys begin in input order.
*/
async function runBoundedHostWorkers(
keys: string[],
concurrency: number,
worker: (key: string) => Promise<void>
): Promise<unknown[]> {
const failures: unknown[] = []
if (keys.length === 0) return failures
let cursor = 0
const runLane = async (): Promise<void> => {
for (;;) {
const index = cursor
cursor += 1
if (index >= keys.length) return
try {
await worker(keys[index])
} catch (error) {
failures.push(error)
}
}
}
const lanes = Math.max(1, Math.min(concurrency, keys.length))
const runners: Promise<void>[] = []
for (let i = 0; i < lanes; i += 1) runners.push(runLane())
await Promise.all(runners)
return failures
}
/** A managed resource considered for orphan cleanup, keyed by owning Host. */
type CleanupCandidate = { owner: string | undefined; kind: string; name: string }
/**
* The workflow-control scopes a first-party Host's mcp-host control token needs
* for the full channel-workflow surface (list/read/trigger + the Telegram/Slack
* provider-identity resolve + approval decide). Matches the canonical Host
* overlay instances (deploy/overlays/<env>/instances/host.yaml).
*/
export const DEFAULT_FIRST_PARTY_WORKFLOW_CONTROL_SCOPES: HostWorkflowControlScope[] = [
'workflow:list',
'workflow:read',
'workflow:trigger',
'workflow:approval:resolve',
'workflow:approval:decide',
]
/**
* Resolve the workflow-control scopes declared by a Host.
*
* Existing channel-bearing Hosts historically omitted workflowControl while
* still needing the first-party workflow approval surface. Preserve that
* server-side default only when the entire block is absent and channel ingress
* is present. An explicit `scopes` array is honored exactly as declared,
* including `[]` as an intentional opt-out from workflow access.
*/
export function resolveWorkflowControlScopes(
workflowControl: HostWorkflowControlSpec | undefined,
options: { hasChannelIngress?: boolean } = {}
): HostWorkflowControlScope[] {
if (workflowControl?.scopes !== undefined) {
return [...workflowControl.scopes]
}
if (!workflowControl && options.hasChannelIngress === true) {
return [...DEFAULT_FIRST_PARTY_WORKFLOW_CONTROL_SCOPES]
}
return []
}
/**
* The scope HCC appends for a Host runtime that fronts an enabled
* `auth.type: oauth` mcp-server. control-api's OAuth user-token broker
* (`POST /api/v1/mcp-oauth/user-token`) requires the caller's mcp-host control
* JWT to carry this scope; it is derive-only (never user-declarable) — see
* {@link HostRuntimeControlScope}.
*/
export const OAUTH_USER_TOKEN_SCOPE = 'oauth:user-token' as const
/**
* Resolve the EFFECTIVE runtime control scopes minted into the mcp-host control
* JWT: the declarable workflow-control scopes plus the derive-only
* `oauth:user-token` when the Host fronts an enabled oauth mcp-server.
*
* This is the SINGLE source of truth shared by BOTH the mint path
* (`resolveWorkflowControlScopesForHost`) and the drift hash
* (`runtimeTokenScopeHash`), so the scopes carried by the token and the scopes
* hashed for change-detection can never diverge.
*/
export function resolveRuntimeControlScopes(
workflowControl: HostWorkflowControlSpec | undefined,
options: { hasChannelIngress?: boolean; frontsOAuthServer?: boolean } = {}
): HostRuntimeControlScope[] {
const scopes: HostRuntimeControlScope[] = [
...resolveWorkflowControlScopes(workflowControl, {
hasChannelIngress: options.hasChannelIngress,
}),
]
if (options.frontsOAuthServer === true) {
scopes.push(OAUTH_USER_TOKEN_SCOPE)
}
return scopes
}
/**
* One SharedFileSystem mount injected into a Host's mcp-host Deployment.
* The PVC is mounted RO at the requested path; mcp-host's built-in
* clerum__context_files_* tools read CLERUM_CONTEXT_FILES_MOUNTS to learn
* what's mounted where.
*/
export interface ResolvedSfsMount {
/** SharedFileSystem.metadata.name (also the JWT `sharedFileSystem` claim). */
name: string
/** SharedFileSystem.metadata.namespace — always mcp-host in v1. */
namespace: string
/** PVC name that the per-SFS wfc and the Host both mount. */
pvcName: string
/** Path inside the mcp-host container where the PVC is mounted (RO). */
mountPath: string
}
export type ResolveContextMountsFn = (host: HostCRD) => Promise<ResolvedSfsMount[]>
/**
* Resolve whether a Host's referenced Context fronts at least one ENABLED
* `auth.type: oauth` mcp-server. Wired in production from the McpServerWatcher
* (which owns the Context + McpServer caches) so HCC can decide whether to
* request the derive-only `oauth:user-token` scope without HostReconciler
* needing direct K8s access to non-Host CRDs. Default (unwired) returns false.
*/
export type HostFrontsOAuthServerFn = (host: HostCRD) => Promise<boolean>
const CONTEXT_LABEL = 'clerum.io/context'
const CONTEXT_MOUNT_PATH_PATTERN = /^\/[a-zA-Z0-9_.][a-zA-Z0-9_.\/-]*$/
const RUNTIME_TOKEN_REVISION_ANNOTATION = 'clerum.io/runtime-token-revision'
// Rolls the mcp-host pod when Host.spec.guardrails changes. mcp-host re-resolves
// the guardrails block (installed-hook refs, built-ins, limits) live off its Host
// watch, but a watch that has lapsed leaves the running agent on the block it last
// saw; the roll is the delivery path that does not depend on that watch. Stamping
// a hash of the guardrails spec onto the pod template makes any change flip the
// template → rolling restart → mcp-host re-reads guardrails at boot. Removal is
// handled by preserveHostDeploymentAnnotations, since an omitted key would
// otherwise survive the annotation merge and produce no diff.
// Mirrors the runtime-token / credentials revision annotations.
const GUARDRAILS_REVISION_ANNOTATION = 'clerum.io/guardrails-revision'
/**
* Deep, key-sorted JSON for a stable content hash. Object keys are sorted at
* EVERY level — unlike `canonicalStringify`, which sorts only the top level —
* so a guardrails block that differs only in nested key order (e.g. the API
* returning `hooks` phases or a hook ref's fields in a different order) hashes
* identically and does not roll mcp-host for nothing. Array order is PRESERVED:
* hook precedence within a phase is semantic, so reordering it is a real change.
*/
function deepStableStringify(value: unknown): string {
const norm = (v: unknown): unknown => {
if (Array.isArray(v)) return v.map(norm)
if (v && typeof v === 'object') {
const out: Record<string, unknown> = {}
for (const k of Object.keys(v as Record<string, unknown>).sort()) {
out[k] = norm((v as Record<string, unknown>)[k])
}
return out
}
return v
}
return JSON.stringify(norm(value))
}
const RUNTIME_TOKEN_SECRET_REVISION_ANNOTATION = 'clerum.io/runtime-token-secret-revision'
const RUNTIME_TOKEN_ISSUED_AT_ANNOTATION = 'clerum.io/runtime-token-issued-at'
const RUNTIME_TOKEN_REFRESH_EXPIRES_AT_ANNOTATION = 'clerum.io/runtime-token-refresh-expires-at'
const RUNTIME_TOKEN_REFRESH_BEFORE_ANNOTATION = 'clerum.io/runtime-token-refresh-before'
const GFS_TOKEN_EXPIRES_AT_ANNOTATION = 'clerum.io/gfs-token-expires-at'
const GFS_TOKEN_REFRESH_BEFORE_ANNOTATION = 'clerum.io/gfs-token-refresh-before'
const GFS_TOKEN_EXPECTED_SUBJECT_ANNOTATION = 'clerum.io/gfs-token-expected-subject'
const GFS_TOKEN_CAPABILITY_SET_HASH_ANNOTATION = 'clerum.io/gfs-token-capability-set-hash'
const GFS_TOKEN_HOST_UID_ANNOTATION = 'clerum.io/gfs-token-host-uid'
const HOST_UID_ANNOTATION = 'clerum.io/host-uid'
const GFS_TOKEN_HOST_GENERATION_ANNOTATION = 'clerum.io/gfs-token-host-generation'
const RUNTIME_TOKEN_HOST_BINDING_HASH_ANNOTATION = 'clerum.io/runtime-token-host-binding-hash'
const RUNTIME_TOKEN_SCOPE_HASH_ANNOTATION = 'clerum.io/runtime-token-scope-hash'
const RUNTIME_TOKEN_ISSUER_ANNOTATION = 'clerum.io/runtime-token-issuer'
const RUNTIME_TOKEN_AUDIENCE_ANNOTATION = 'clerum.io/runtime-token-audience'
const RUNTIME_TOKEN_SCHEMA_VERSION_ANNOTATION = 'clerum.io/runtime-token-schema-version'
const RUNTIME_TOKEN_BOOTSTRAP_STATE_ANNOTATION = 'clerum.io/runtime-token-bootstrap-state'
const RUNTIME_TOKEN_ROLLOUT_REQUIRED_ANNOTATION = 'clerum.io/runtime-token-rollout-required'
const RUNTIME_TOKEN_BOOTSTRAP_DEPLOYMENT_UID_ANNOTATION =
'clerum.io/runtime-token-bootstrap-deployment-uid'
const RUNTIME_TOKEN_BOOTSTRAP_APPLIED_REVISION_ANNOTATION =
'clerum.io/runtime-token-bootstrap-applied-revision'
const RUNTIME_TOKEN_BOOTSTRAP_READY_REVISION_ANNOTATION =
'clerum.io/runtime-token-bootstrap-ready-revision'
const RUNTIME_TOKEN_HAS_CHANNEL_INGRESS_ANNOTATION = 'clerum.io/runtime-token-has-channel-ingress'
const RUNTIME_TOKEN_FRONTS_OAUTH_ANNOTATION = 'clerum.io/runtime-token-fronts-oauth-server'
const RUNTIME_TOKEN_ISSUER = 'control-api'
const RUNTIME_TOKEN_AUDIENCE = 'host-context-controller,workflow-approvals'
// v3 extends the existing first-party access/refresh token material with the
// exact HCC audience, immutable MCP credential capability, and live Host UID.
// Existing Secrets therefore fail the contract check and rotate onto the
// caller-bound contract instead of remaining workflow-only.
const RUNTIME_TOKEN_SCHEMA_VERSION = '3'
const RUNTIME_TOKEN_BOOTSTRAP_STATE_FRESH = 'fresh'
const RUNTIME_TOKEN_BOOTSTRAP_STATE_CONSUMED = 'consumed'
// Deployments affected by the historical stringData/data hashing bug can carry
// the empty-payload revision. Roll them once so the pod template converges to
// the normalized persisted Secret revision.
const LEGACY_EMPTY_RUNTIME_TOKEN_REVISION = createHash('sha256')
.update(JSON.stringify([]))
.digest('hex')
const WORKFLOW_TOKEN_MOUNT_PATH = '/var/run/clerum/workflow-tokens'
const MCP_HOST_WORKFLOW_CONTROL_TOKEN_FILE_PATH = `${WORKFLOW_TOKEN_MOUNT_PATH}/${MCP_HOST_RUNTIME_TOKEN_SECRET_CONTROL_KEY}`
const MCP_HOST_RUNTIME_AUTH_STATE_PATH = '/var/run/clerum/workflow-auth'
/**
* Restricted-PSA-compliant container securityContext shared by the mcp-host
* container and the workspace-layout-merge initContainer. MCC shared-tenant
* namespaces (mcp-host-<slug>) enforce the `restricted` PodSecurity standard;
* without these fields the ReplicaSet gets FailedCreate and never schedules a
* pod. UID/GID 1001 = the image's baked-in `nodejs` user; it must match the
* pod-level securityContext and the existing workspace PVC ownership so
* re-rolls don't EACCES. readOnlyRootFilesystem is intentionally NOT set —
* mcp-host writes outside its mounted /tmp + workspace volumes and restricted
* PSA does not require it.
*/
function hostContainerSecurityContext(): k8s.V1SecurityContext {
return {
allowPrivilegeEscalation: false,
runAsNonRoot: true,
runAsUser: 1001,
runAsGroup: 1001,
capabilities: { drop: ['ALL'] },
seccompProfile: { type: 'RuntimeDefault' },
}
}
const log = hccLogger.child({ module: 'host-reconciler' })
function throwCleanupFailures(failures: unknown[], message: string): void {
if (failures.length > 0) {
throw new AggregateError(failures, message)
}
}
export class HostFleetReconcileError extends AggregateError {
constructor(
readonly hostFailures: unknown[],
readonly cleanupFailures: unknown[],
message = 'Host fleet reconciliation completed with errors'
) {
super([...hostFailures, ...cleanupFailures], message)
this.name = 'HostFleetReconcileError'
}
}
type HostReconcilerDeps = {
appsApi?: k8s.AppsV1Api
coreApi?: k8s.CoreV1Api
networkingApi?: k8s.NetworkingV1Api
rbacApi?: k8s.RbacAuthorizationV1Api
customApi?: k8s.CustomObjectsApi
/** Injectable clock so tests get deterministic condition lastTransitionTime. */
now?: () => Date
/** Stable per-enqueue identity; the reporter reuses it for transport retries. */
newTelemetryOccurrenceId?: () => string
/**
* Resolve which SharedFileSystem PVCs to mount into a Host's mcp-host
* Deployment. The default (no override) returns []. Production wires this
* up from the McpServerWatcher in k8sClient.ts so it can read the Context
* referenced by the Host without HostReconciler needing direct K8s access
* to non-Host CRDs.
*/
resolveContextMounts?: ResolveContextMountsFn
/**
* Resolve whether the Host fronts an enabled `auth.type: oauth` mcp-server,
* gating the derive-only `oauth:user-token` runtime scope. Wired in
* production by `McpServerWatcher` over `getServerInfosByContext`. Default
* (no override) returns false — fail closed: no oauth scope until wired.
*/
hostFrontsOAuthServer?: HostFrontsOAuthServerFn
/**
* Count CommunicationChannels referencing this Host. Used by
* `buildChannelReaderDeployment` to compute `spec.replicas`
* (0 when no CCs, 1 when 1+). Wired by `McpServerWatcher` from
* its `communicationChannels` cache. Default returns 0 — Hosts
* without a wired counter scale to replicas=0.
*/
countCommunicationChannels?: (hostName: string) => number
/**
* Find every CommunicationChannel whose `spec.hostRef` equals the given
* host name. Used by `patchChannelReaderRevisionAnnotation` to discover
* which Secret(s) a host depends on. Wired by `McpServerWatcher` from
* its `communicationChannels` cache.
*/
findCommunicationChannelsByHostRef?: (host: string) => CommunicationChannelCRD[]
/**
* Find every CommunicationChannel whose `spec.credentialsSecretRef.name`
* equals the given Secret name. Used by the Secret informer (Task 9) to
* route rotation events to the affected hosts. Wired by
* `McpServerWatcher` from its `communicationChannels` cache.
*/
findCommunicationChannelsByCredentialsSecretName?: (name: string) => CommunicationChannelCRD[]
/**
* B2: Returns true once the McpServerWatcher has completed its CC
* initial-list startup block. HostReconciler uses this to avoid
* scaling an existing channel-reader Deployment to 0 when the cache
* is still empty (e.g. during startup or after a CC-load failure).
* Default returns false — safe: preserves existing replicas until wired.
*/
isCommunicationChannelCacheSynced?: () => boolean
infrastructureTelemetryReporter?: InfrastructureTelemetryReporter
administrativeOutcomeReporter?: AdministrativeOutcomeReporter
}
type HostSecretValidationResult =
| { ok: true }
| { ok: false; reason: 'SecretNotFound' | 'SecretAccessDenied' | 'ReadError'; message: string }
type BootstrapOptions = {
forceFreshForWake?: boolean
forceFreshForDeploymentChange?: boolean
targetSuspended?: boolean
prepareForResourceChange?: boolean
refreshGfsOnly?: boolean
preserveDeploymentTemplateOnWake?: boolean
}
type RuntimeTokenProvision = {
revision: string
scopeHash: string
deploymentUid?: string
freshBootstrapPrepared?: boolean
}
// Reported as the reconcile_outcome transition `gfs_token:<outcome>` (#328).
type GfsTokenLifecycleOutcome = 'minted' | 'rotated' | 'reused' | 'failed'
type DeploymentMutationState = {
lifecycle: EffectiveHostLifecycle
runtimeTokenRevision: string
freshBootstrapPrepared?: boolean
// Captured in the same synchronous step as the stable scope-hash check so
// the pod's Grok factory switch and the minted llm:grok:execute scope agree.
grokExecutionEnabled: boolean
}
export class HostReconciler {
private readonly appsApi: k8s.AppsV1Api
private readonly coreApi: k8s.CoreV1Api
private readonly networkingApi: k8s.NetworkingV1Api
private readonly rbacApi: k8s.RbacAuthorizationV1Api
/**
* Lazily constructed (see the `customApi` getter): only lifecycle-enabled
* Hosts touch the CustomObjects API, and existing tests construct the
* reconciler with a bare KubeConfig stub that cannot makeApiClient.
*/
private customApiInstance: k8s.CustomObjectsApi | undefined
private readonly kubeConfig: k8s.KubeConfig
private readonly now: () => Date
private readonly newTelemetryOccurrenceId: () => string
private readonly statusMap: Map<string, HostRuntimeStatus> = new Map()
private codexSnapshot: CodexCatalogSnapshot = { flagEnabled: false }
private lastCodexConfigMap: k8s.V1ConfigMap | undefined
private readonly readinessTimers: Map<string, ReturnType<typeof setTimeout>> = new Map()
private readonly reconcileGenerations = new Map<string, number>()
private readonly statusGenerations = new Map<string, number>()
/**
* host.name → image whose pull-policy refusal was already error-logged.
* The periodic resync rebuilds the Deployment forever; the operator needs
* the error once per distinct misconfigured image (the durable condition
* carries the long-lived signal).
*/
private readonly pullPolicyRejectionLogged: Map<string, string> = new Map()
/**
* Stateless lifecycle execution (Stages 2–6), extracted to
* StatelessLifecycleExecutor: reconcile-time assessment + durable status
* writes, the heartbeat suspend/drain executors, the wake fast-path, and
* the per-host reconcile serialization chain.
*/
private readonly lifecycle: StatelessLifecycleExecutor
private resolveContextMounts: ResolveContextMountsFn
private hostFrontsOAuthServerFn: HostFrontsOAuthServerFn
private countCommunicationChannels: (hostName: string) => number
private findCommunicationChannelsByHostRef: (host: string) => CommunicationChannelCRD[]
private findCommunicationChannelsByCredentialsSecretName: (
name: string
) => CommunicationChannelCRD[]
private readonly infrastructureTelemetryReporter?: InfrastructureTelemetryReporter
private readonly administrativeOutcomeReporter?: AdministrativeOutcomeReporter
private readonly gfsTokenLifecycleEvidence = new Map<string, GfsTokenLifecycleOutcome>()
// B2: whether the CC cache initial-list has completed. Defaults to false
// (safe: preserves existing Deployment replicas until wired by McpServerWatcher).
private ccCacheSyncedFn: () => boolean = () => false
/**
* Resolve the current cached Host object by name. Wired by McpServerWatcher
* from its live Host cache so a fleet worker reconciles the freshest spec and
* so orphan cleanup compares candidates with the CURRENT inventory rather than
* the pass snapshot. `null` (unwired) means callers fall back to the pass
* snapshot for reconcile, and cleanup stays fail-closed (authority unknown).
*/
private resolveCurrentHost: ((name: string) => HostCRD | undefined) | null = null
/**
* H2: late-bound cache reflector. Wired by McpServerWatcher next to
* setResolveCurrentHost; null (standalone) means lifecycle outcomes reflect
* only onto the caller's own object (legacy contract).
*/
private reflectHostOutcome: ReflectHostOutcomeFn | null = null
/**
* Snapshot the current Host watch authority + generation. Wired by
* McpServerWatcher. Default is fail-closed (unknown) so orphan cleanup never
* runs until the real authority getter is installed.
*/
private hostWatchAuthority: () => HostWatchAuthoritySnapshot = () => ({
known: false,
generation: 0,
})
/**
* Positive Host mutations depend on Host plus Context authority. The default
* preserves the standalone HostReconciler contract by deriving a Host-only
* lease; McpServerWatcher wires the complete composite lease.
*/
private hostMutationAuthority: () => HostMutationAuthoritySnapshot = () => {
const host = this.hostWatchAuthority()
return {
known: host.known,
// Standalone contract: no separate desired-revision plumbing here, so the
// Host watch generation stands in as the Host-only lease identity. The
// McpServerWatcher wires the real composite content revisions in production.
hostRevision: host.generation,
contextRevision: 0,
}
}
/**
* Exact revisions of the cross-resource inputs selected by a Host. Watch
* generations fence LIST -> WATCH authority, but normal MODIFIED events stay
* in the same generation. The watcher supplies the selected Context,
* referenced SharedFileSystems plus mountability, and Host-scoped
* CommunicationChannels so unrelated fleet changes do not retire this lease.
*/
private resolveHostMutationDependencies: ((host: HostCRD) => readonly unknown[]) | null = null
constructor(kc: k8s.KubeConfig, deps?: HostReconcilerDeps) {
this.appsApi =
deps?.appsApi ?? makeHostK8sApiClient(kc, k8s.AppsV1Api, config.hostK8sRequestTimeoutMs)
this.coreApi =
deps?.coreApi ?? makeHostK8sApiClient(kc, k8s.CoreV1Api, config.hostK8sRequestTimeoutMs)
this.networkingApi =
deps?.networkingApi ??
makeHostK8sApiClient(kc, k8s.NetworkingV1Api, config.hostK8sRequestTimeoutMs)
this.rbacApi =
deps?.rbacApi ??
makeHostK8sApiClient(kc, k8s.RbacAuthorizationV1Api, config.hostK8sRequestTimeoutMs)
this.kubeConfig = kc
this.customApiInstance = deps?.customApi
this.now = deps?.now ?? (() => new Date())
this.newTelemetryOccurrenceId = deps?.newTelemetryOccurrenceId ?? randomUUID
this.resolveContextMounts = deps?.resolveContextMounts ?? (async () => [])
this.hostFrontsOAuthServerFn = deps?.hostFrontsOAuthServer ?? (async () => false)
this.countCommunicationChannels = deps?.countCommunicationChannels ?? (() => 0)
this.findCommunicationChannelsByHostRef = deps?.findCommunicationChannelsByHostRef ?? (() => [])
this.findCommunicationChannelsByCredentialsSecretName =
deps?.findCommunicationChannelsByCredentialsSecretName ?? (() => [])
this.infrastructureTelemetryReporter = deps?.infrastructureTelemetryReporter
this.administrativeOutcomeReporter = deps?.administrativeOutcomeReporter
if (deps?.isCommunicationChannelCacheSynced) {
this.ccCacheSyncedFn = deps.isCommunicationChannelCacheSynced
}
this.lifecycle = new StatelessLifecycleExecutor({
appsApi: this.appsApi,
coreApi: this.coreApi,
getCustomApi: () => this.customApi,
now: this.now,
countCommunicationChannels: hostName => this.countCommunicationChannels(hostName),
isCommunicationChannelCacheSynced: () => this.ccCacheSyncedFn(),
reconcileCore: (host, revalidate) => this.reconcileCore(host, revalidate),
prepareHostMutationAdmission: (action, host) =>
this.prepareHostMutationAdmission(action, host),
reflectHostOutcome: (name, uid, apply) => this.reflectHostOutcome?.(name, uid, apply),
onLifecycleStatusCommitted: (host, lifecycle) => {
// A uid-less snapshot never reaches a lifecycle status commit, so this
// branch is the compiler's requirement rather than an observed path.
// It shares the emitter guard instead of assuming that stays true.
const hostLookupReference = this.hostTelemetryReference(host, {
telemetryType: 'health_transition',
state: lifecycle.state,
})
if (hostLookupReference === undefined) return
const occurredAt = this.now().toISOString()
this.infrastructureTelemetryReporter?.enqueueHealthTransition({
sourceEventId: `hcc-health-transition:${this.newTelemetryOccurrenceId()}`,
occurredAt,
hostLookupReference,
payload: { transition: `lifecycle:${lifecycle.state}`, state: lifecycle.state },
})
},
})
}
/** CustomObjects API client for Host /status writes (lazily constructed). */
private get customApi(): k8s.CustomObjectsApi {
if (!this.customApiInstance) {
this.customApiInstance = makeHostK8sApiClient(
this.kubeConfig,
k8s.CustomObjectsApi,
config.hostK8sRequestTimeoutMs
)
}
return this.customApiInstance
}
/**
* Late-bound setter so the McpServerWatcher (which owns both the Context
* and SharedFileSystem caches) can wire up the mount resolver after this
* reconciler is constructed.
*/
setResolveContextMounts(fn: ResolveContextMountsFn): void {
this.resolveContextMounts = fn
}
/**
* Late-bound setter so the McpServerWatcher (which owns the Context and
* McpServer caches) can wire up the oauth-server resolver after this
* reconciler is constructed. Pattern mirrors setResolveContextMounts.
*/
setHostFrontsOAuthServer(fn: HostFrontsOAuthServerFn): void {
this.hostFrontsOAuthServerFn = fn
}
/**
* Whether this Host fronts an enabled `auth.type: oauth` mcp-server, gating
* the derive-only `oauth:user-token` runtime scope. A read that failed after
* its retries is reported as unobserved instead of collapsing to `false`:
* "scope unknown" is not "scope revoked". Callers decide what an unknown
* answer means — the issuance path fails closed only where that cannot strip
* a grant the running pod holds, and the deployment drift guard
* (`currentScopeMatchesProvision`) never forces a re-mint on it. An uncertain
* read never grants the scope.
*
* Each call is an INDEPENDENT live point-in-time read (mirrors
* `hasChannelIngress`); the value is deliberately not cached across a
* reconcile. The issuance path resolves it once and threads the same bool
* into the mint-scope derive and the drift hash so they never diverge; the
* deployment drift guard re-reads it on purpose to catch a scope change
* mid-reconcile.
*/
private async observeFrontsOAuthServer(
host: HostCRD
): Promise<
| { observed: true; frontsOAuthServer: boolean }
| { observed: false; attempts: number; error: unknown }
> {
// Bounded retry that absorbs a TRANSIENT apiserver blip without touching the
// authoritative-`false` path. Contract of the underlying resolver: a real
// scope change (the server genuinely stopped fronting an oauth mcp-server)
// RETURNS `false` without throwing; only a transient read failure (apiserver
// blip) THROWS. So we retry ONLY on throw — any RETURNED value (true or
// false) is authoritative and is returned immediately, so a healthy call
// resolves on the first attempt with zero added overhead and an authoritative
// `false` is never re-tried. Before this retry, a transient throw collapsed
// to `false` exactly like an authoritative `false`, dropping the
// oauth:user-token scope → runtime-token re-mint + Deployment rollout, then a
// flip back on recovery → churn, once per OAuth Host per reconcile.
const MAX_ATTEMPTS = 3
const BACKOFF_BASE_MS = 50
let lastErr: unknown
for (let attempt = 1; attempt <= MAX_ATTEMPTS; attempt++) {
try {
return { observed: true, frontsOAuthServer: await this.hostFrontsOAuthServerFn(host) }
} catch (err) {
lastErr = err
if (attempt < MAX_ATTEMPTS) {
// Small linear backoff (50ms, 100ms) to let a transient blip clear.
await new Promise(resolve => setTimeout(resolve, BACKOFF_BASE_MS * attempt))
}
}
}
return { observed: false, attempts: MAX_ATTEMPTS, error: lastErr }
}
/**
* Late-bound setter so the McpServerWatcher (which owns the
* communicationChannels cache) can wire up the counter after this
* reconciler is constructed. Pattern mirrors setResolveContextMounts.
*/
setCountCommunicationChannels(fn: (hostName: string) => number): void {
this.countCommunicationChannels = fn
}
/**
* B2: Late-bound setter so McpServerWatcher can wire up the CC cache-sync
* flag after this reconciler is constructed. Pattern mirrors
* setCountCommunicationChannels.
*/
setIsCommunicationChannelCacheSynced(fn: () => boolean): void {
this.ccCacheSyncedFn = fn
}
/**
* Late-bound setter for the live Host cache resolver. A fleet worker resolves
* the current cached Host at execution time (skipping absent/superseded
* entries), and orphan cleanup uses it to compare candidates with the CURRENT
* inventory. Wired by McpServerWatcher from its `hosts` cache.
*/
setResolveCurrentHost(fn: (name: string) => HostCRD | undefined): void {
this.resolveCurrentHost = fn
}
/**
* H2: late-bound setter for the guarded cache reflector. Wired by
* McpServerWatcher from its `hosts` cache (uid-guarded apply). Pattern
* mirrors setResolveCurrentHost.
*/
setReflectHostOutcome(fn: ReflectHostOutcomeFn): void {
this.reflectHostOutcome = fn
}
/**
* Late-bound setter for the Host watch authority/generation snapshot. Wired by
* McpServerWatcher so orphan cleanup can require known authority and a stable
* watch generation before deleting anything.
*/
setHostWatchAuthority(fn: () => HostWatchAuthoritySnapshot): void {
this.hostWatchAuthority = fn
}
setHostMutationAuthority(fn: () => HostMutationAuthoritySnapshot): void {
this.hostMutationAuthority = fn
}
setResolveHostMutationDependencies(fn: (host: HostCRD) => readonly unknown[]): void {
this.resolveHostMutationDependencies = fn
}
/**
* A standalone HostReconciler receives an explicit Host object and has no
* watch-backed cache to fence. Once a live-cache resolver is wired (the
* production McpServerWatcher path), every mutation captures and revalidates
* Host LIST -> WATCH authority at its actual serializer admission boundary.
*/
private captureHostMutationAuthority(): HostMutationAuthoritySnapshot | null {
return this.resolveCurrentHost === null ? null : this.hostMutationAuthority()
}
private requireHostMutationAuthority(
action: string,
captured: HostMutationAuthoritySnapshot | null
): void {
if (captured === null) return
const current = this.hostMutationAuthority()
if (
!captured.known ||
!current.known ||
captured.hostRevision !== current.hostRevision ||
captured.contextRevision !== current.contextRevision
) {
throw new HostInventoryAuthorityUnavailableError(action, captured, current)
}
}
/**
* Heartbeat writers own their serializer inside StatelessLifecycleExecutor.
* Capture authority here at dispatch, then let the returned closure re-check
* it and resolve the current same-UID Host when that serializer admits the
* work. This preserves one queue per Host without a nested-lock deadlock.
*/
private prepareHostMutationAdmission(action: string, requested: HostCRD): () => HostCRD {
const capturedAuthority = this.captureHostMutationAuthority()
const capturedDependencies = this.resolveHostMutationDependencies?.(requested)
return this.makeHostMutationAdmission(
action,
requested,
capturedAuthority,
capturedDependencies
)
}
private makeHostMutationAdmission(
action: string,
requested: HostCRD,
capturedAuthority: HostMutationAuthoritySnapshot | null,
capturedDependencies: readonly unknown[] | undefined
): () => HostCRD {
return () => {
this.requireHostMutationAuthority(action, capturedAuthority)
if (this.resolveCurrentHost === null) return requested
const current = this.resolveCurrentHost(requested.name)
if (
current === undefined ||
requested.uid === undefined ||
current.uid === undefined ||
requested.uid !== current.uid
) {
throw new HostMutationIdentityChangedError(action, requested.name)
}
if (
requested.generation === undefined ||
current.generation === undefined ||
requested.generation !== current.generation
) {
throw new HostMutationSpecRevisionChangedError(action, requested.name)
}
if (this.resolveHostMutationDependencies !== null) {
const currentDependencies = this.resolveHostMutationDependencies(current)
const sameDependencies =
capturedDependencies !== undefined &&
capturedDependencies.length === currentDependencies.length &&
capturedDependencies.every((value, index) =>
isDeepStrictEqual(value, currentDependencies[index])
)
if (!sameDependencies) {
throw new HostMutationDependencyChangedError(action, requested.name)
}
}
// McpServerWatcher owns the canonical LIST/WATCH cache object. Mutation
// bodies historically reflect status into their admitted object for
// same-pass decisions, so detach the working snapshot whenever a live
// cache resolver is wired. Standalone callers without a resolver retain
// the legacy caller-visible reflection contract.
return structuredClone(current)
}
}
/**
* Late-bound setter for the host-ref lookup used by
* `patchChannelReaderRevisionAnnotation` to discover Secret(s) referenced
* by a host's CCs. Wired by `McpServerWatcher` from its
* `communicationChannels` cache. Pattern mirrors setCountCommunicationChannels.
*/
setFindCommunicationChannelsByHostRef(fn: (host: string) => CommunicationChannelCRD[]): void {
this.findCommunicationChannelsByHostRef = fn
}
/**
* Late-bound setter for the Secret-name lookup used by the Secret
* informer (Task 9) to route rotation events to the affected hosts.
* Wired by `McpServerWatcher` from its `communicationChannels` cache.
*/
setFindCommunicationChannelsByCredentialsSecretName(
fn: (name: string) => CommunicationChannelCRD[]
): void {
this.findCommunicationChannelsByCredentialsSecretName = fn
}
/**
* Volume name used by both the volumes[] entry and each volumeMounts[]
* entry that references it. Stable per (sfs.name, sfs.namespace) so a
* Deployment template diff only flips when the SFS list changes — not
* when the same set is reordered.
*/
static contextMountVolumeName(sfs: { name: string; namespace: string }): string {
return `ctxfs-${sharedFileSystemHash(sfs)}`
}
static contextMountPathRejectionReason(
mountPath: string,
reservedMountPaths: string[]
): string | null {
const normalizedMountPath = HostReconciler.normalizeContextMountPath(mountPath)
if (!normalizedMountPath) {
return (
'mountPath must be an absolute POSIX path matching ' + CONTEXT_MOUNT_PATH_PATTERN.source
)
}
const reserved = new Set(
reservedMountPaths
.map(HostReconciler.normalizeContainerMountPath)
.filter((path): path is string => Boolean(path))
)
if (reserved.has(normalizedMountPath)) {
return `mountPath conflicts with reserved Host mount "${normalizedMountPath}"`
}
return null
}
private static normalizeContextMountPath(mountPath: string): string | null {
if (
!CONTEXT_MOUNT_PATH_PATTERN.test(mountPath) ||
mountPath.split('/').some(segment => segment === '..')
) {
return null
}
return HostReconciler.normalizeContainerMountPath(mountPath)
}
private static normalizeContainerMountPath(mountPath: string): string | null {
if (!mountPath.startsWith('/') || mountPath.includes('\\') || mountPath.includes('\0')) {
return null
}
const normalized = path.posix.normalize(mountPath)
return normalized.length > 1 ? normalized.replace(/\/+$/, '') : normalized
}
// ─── Per-Host RBAC scaffolding ────────────────────────────────────────
/** Per-Host ServiceAccount name. */
private hostSaName(host: HostCRD): string {
return `host-${host.name}-sa`
}
/** Per-Host RBAC Role + RoleBinding name (shared between the two). */
private hostRoleName(host: HostCRD): string {
return `host-${host.name}-config-reader`
}
private rbacLabels(host: HostCRD): Record<string, string> {
return {
[MANAGED_BY_LABEL]: MANAGED_BY_VALUE,
[HOST_LABEL]: host.name,
}
}
private static isHccOwnedHostResource(
resource: { metadata?: { labels?: Record<string, string> } },
hostName: string
): boolean {
const labels = resource.metadata?.labels ?? {}
return labels[MANAGED_BY_LABEL] === MANAGED_BY_VALUE && labels[HOST_LABEL] === hostName
}
private async deleteIfHccOwned<T extends { metadata?: { labels?: Record<string, string> } }>(
kind: string,
name: string,
namespace: string,
hostName: string,
read: () => Promise<T>,
remove: () => Promise<unknown>
): Promise<void> {
let resource: T
try {
resource = await read()
} catch (error) {
if (getErrorCode(error) === 404) return
log.error('Failed to read Host resource ownership', { kind, name, err: error })
throw error
}
if (!HostReconciler.isHccOwnedHostResource(resource, hostName)) {
log.warn('Skipping Host resource delete - not HCC-owned', { kind, name, host: hostName })
return
}
try {
await remove()
log.info('Deleted Host resource', { kind, name, namespace })
} catch (error) {
if (getErrorCode(error) !== 404) {
log.error('Failed to delete Host resource', { kind, name, err: error })
throw error
}
}
}
/**
* Ensure a per-Host ServiceAccount exists. Idempotent.
*/
private async ensureHostServiceAccount(host: HostCRD, revalidate?: () => void): Promise<void> {
const name = this.hostSaName(host)
const body: k8s.V1ServiceAccount = {
apiVersion: 'v1',
kind: 'ServiceAccount',
metadata: { name, namespace: host.namespace, labels: this.rbacLabels(host) },
}
const mutationAllowed = () => {
revalidate?.()
return true
}
await ensureResource({
mutationAllowed,
read: () =>
observeExistenceRead('ServiceAccount', () =>
this.coreApi.readNamespacedServiceAccount({ name, namespace: host.namespace })
),
create: () =>
observeCreate('ServiceAccount', () =>
this.coreApi.createNamespacedServiceAccount({ namespace: host.namespace, body })
),
// Existing ServiceAccounts have always been retained without an update.
converge: async read => {
try {
await read()
} catch (error) {
if (error != null && getErrorCode(error) === 404) return false
throw error
}
},
onSkipped: () => createsTotal.inc({ kind: 'ServiceAccount', outcome: 'skipped' }),
})
}
/**
* Per-Host Role granting `get/watch/list` ONLY on this Host's resources
* (Host CRD, env ConfigMap/Secret, LLM Secret named in spec.secretRef).
* On secretRef change the resourceNames are rewritten so Host A can never
* read Host B's Secret.
*/
private async ensureHostRole(host: HostCRD, revalidate?: () => void): Promise<void> {
const name = this.hostRoleName(host)
const body: k8s.V1Role = {
apiVersion: 'rbac.authorization.k8s.io/v1',
kind: 'Role',
metadata: { name, namespace: host.namespace, labels: this.rbacLabels(host) },
rules: [
{
apiGroups: ['clerum.io'],
resources: ['hosts'],
resourceNames: [host.name],
verbs: ['get', 'watch', 'list'],
},
{
// Per-Host env ConfigMap + the cluster-wide R3 allowlist ConfigMap
// (clerum-llm-allowed-models). The allowlist is a single shared object
// materialized by control-api; every Host may read it. resourceNames
// are rewritten on every reconcile, so this propagates to existing Roles.
apiGroups: [''],
resources: ['configmaps'],
resourceNames: [`host-${host.name}-env`, 'clerum-llm-allowed-models'],
verbs: ['get', 'watch', 'list'],
},
{
// READ-ONLY on the mcp-host-runtime-token Secret: granting write here would
// let the pod rotate its own credentials (privilege escalation).
// Writes happen via HCC's own ServiceAccount.
apiGroups: [''],
resources: ['secrets'],
resourceNames: [
host.spec.secretRef,
`host-${host.name}-env-secret`,
mcpHostRuntimeTokenSecretName(host),
].filter((name): name is string => Boolean(name)),
verbs: ['get', 'watch', 'list'],
},
],
}
const mutationAllowed = () => {
revalidate?.()
return true
}
await ensureResource({
mutationAllowed,
read: () =>
observeExistenceRead('Role', () =>
this.rbacApi.readNamespacedRole({ name, namespace: host.namespace })
),
create: () =>
observeCreate('Role', () =>
this.rbacApi.createNamespacedRole({ namespace: host.namespace, body })
),
converge: async read => {
try {
// Retry optimistic-lock contention, but never continue provisioning
// with stale permissions after the bounded retry budget is exhausted.
await replaceWithConflictRetry({
description: `Role "${name}"`,
logPrefix: '[HostReconciler]',
body,
read,
mutationAllowed,
isUpToDate: roleMatchesDesired,
replace: next =>
this.rbacApi.replaceNamespacedRole({ name, namespace: host.namespace, body: next }),
})
} catch (error) {
// Preserve the existing disappearance policy, without swallowing
// authorization, transport, or terminal conflict failures.
if (error != null && getErrorCode(error) === 404) return false
throw error
}
},
onSkipped: () => createsTotal.inc({ kind: 'Role', outcome: 'skipped' }),
})
}
/**
* Ensure a per-Host RoleBinding binding the per-Host Role to the per-Host SA.
*/
private async ensureHostRoleBinding(host: HostCRD, revalidate?: () => void): Promise<void> {
const name = this.hostRoleName(host)
const body: k8s.V1RoleBinding = {
apiVersion: 'rbac.authorization.k8s.io/v1',
kind: 'RoleBinding',
metadata: { name, namespace: host.namespace, labels: this.rbacLabels(host) },
subjects: [
{
kind: 'ServiceAccount',
name: this.hostSaName(host),
namespace: host.namespace,
},
],
roleRef: {
apiGroup: 'rbac.authorization.k8s.io',
kind: 'Role',
name,
},
}
const mutationAllowed = () => {
revalidate?.()
return true
}
await ensureResource({
mutationAllowed,
read: () =>
observeExistenceRead('RoleBinding', () =>
this.rbacApi.readNamespacedRoleBinding({ name, namespace: host.namespace })
),
create: () =>
observeCreate('RoleBinding', () =>
this.rbacApi.createNamespacedRoleBinding({ namespace: host.namespace, body })
),
// Preserve the existing binding; this path does not update subjects/roleRef.
converge: async read => {
try {
await read()
} catch (error) {
if (error != null && getErrorCode(error) === 404) return false
throw error
}
},
onSkipped: () => createsTotal.inc({ kind: 'RoleBinding', outcome: 'skipped' }),
})
}
/**
* Ensures the mcp-host-runtime-token Secret for a 1st-party mcp-host pod.
* Throws after 3 failed attempts (1s/2s/4s); the Host stays unready
* until the cause is fixed.
*/
private static runtimeTokenSecretRevision(data: Record<string, string> | undefined): string {
const canonical = JSON.stringify(
Object.entries(data ?? {}).sort(([left], [right]) => left.localeCompare(right))
)
return createHash('sha256').update(canonical).digest('hex')
}
private static secretDataForKeys(
secret: Pick<k8s.V1Secret, 'data' | 'stringData'>,
keys: string[]
): Record<string, string> | null {
if (secret.data && keys.every(key => typeof secret.data?.[key] === 'string')) {
return Object.fromEntries(keys.map(key => [key, secret.data![key]]))
}
if (secret.stringData && keys.every(key => typeof secret.stringData?.[key] === 'string')) {
return Object.fromEntries(
keys.map(key => [key, Buffer.from(secret.stringData![key], 'utf8').toString('base64')])
)
}
return null
}
private static runtimeTokenSecretData(
secret: Pick<k8s.V1Secret, 'data' | 'stringData'>
): Record<string, string> | null {
return HostReconciler.secretDataForKeys(secret, [
MCP_HOST_RUNTIME_TOKEN_SECRET_ACCESS_KEY,
MCP_HOST_RUNTIME_TOKEN_SECRET_REFRESH_KEY,
MCP_HOST_RUNTIME_TOKEN_SECRET_CONTROL_KEY,
MCP_HOST_GFS_TOKEN_SECRET_KEY,
])
}
private static runtimeTokenSecretRevisionFromSecret(
secret: Pick<k8s.V1Secret, 'data' | 'stringData'>
): string | null {
const data = HostReconciler.runtimeTokenSecretData(secret)
return data ? HostReconciler.runtimeTokenSecretRevision(data) : null
}
private static shortHash(value: unknown): string {
return createHash('sha256').update(JSON.stringify(value)).digest('hex')
}
private static runtimeTokenHostBindingHash(host: HostCRD): string {
return HostReconciler.shortHash({ namespace: host.namespace, host: host.name })
}
private static runtimeTokenScopeHash(
host: HostCRD,
hasChannelIngress = false,
frontsOAuthServer = false,
projection?: CodexExecutionProjection,
grokProjection?: CodexExecutionProjection
): string {
// Hash the EFFECTIVE (resolved) scopes — what actually gets minted into the
// control token — so change-detection matches the default-fallback applied
// at issuance (otherwise a null workflowControl hashes as [] while the token
// carries the first-party defaults, and the two drift). Uses the SAME
// `resolveRuntimeControlScopes` as the mint path (see
// `resolveEffectiveControlScopesForHost`) so the hashed set and the minted
// set — including the derive-only `oauth:user-token` and broker
// projections' derived scopes — can never diverge.
const runtimeScopes = [
...resolveRuntimeControlScopes(host.spec.workflowControl, {
hasChannelIngress,
frontsOAuthServer,
}),
].sort()
const codexDerived = projection?.derivedScopes ?? []
const grokDerived = grokProjection?.derivedScopes ?? []
if (codexDerived.length === 0 && grokDerived.length === 0) {
return HostReconciler.shortHash(runtimeScopes)
}
// Codex-only Hosts must keep the pre-Grok hash input so an HCC upgrade
// does not remint every existing Codex runtime token.
if (grokDerived.length === 0) {
return HostReconciler.shortHash({
scopes: [...runtimeScopes, ...codexDerived].sort(),
drift: projection?.driftHashInput,
})
}
return HostReconciler.shortHash({
scopes: [...runtimeScopes, ...codexDerived, ...grokDerived].sort(),
drift: {
codex: projection?.driftHashInput,
grok: grokProjection?.driftHashInput,
},
})
}
private static gfsCapabilitySetHash(): string {
return HostReconciler.shortHash([...GFS_HOST_SCOPES].sort())
}
/**
* The key carries the uid alongside namespace/name: a Host deleted and
* recreated under the same name is a different object, and an entry keyed by
* the name alone would be reported as the new object's rotation (#696).
*/
private static gfsLifecycleEvidenceKey(
host: Pick<HostCRD, 'namespace' | 'name' | 'uid'>
): string | undefined {
if (host.uid === undefined) return undefined
return `${HostReconciler.gfsLifecycleEvidencePrefix(host.namespace, host.name)}${host.uid}`
}
private static gfsLifecycleEvidencePrefix(namespace: string, name: string): string {
return `${namespace}/${name}/`
}
/**
* Without a uid the objects cannot be told apart, so nothing is recorded: an
* entry under an incomplete key would be claimed by whichever Host answers
* to the name next. The real mappers copy metadata.uid from the API server,
* so this warns rather than papering over a snapshot that lost it.
*/
private recordGfsLifecycleEvidence(host: HostCRD, outcome: GfsTokenLifecycleOutcome): void {
const key = HostReconciler.gfsLifecycleEvidenceKey(host)
if (key === undefined) {
log.warn('skipping gfs token lifecycle evidence: Host snapshot has no uid', {
host: host.name,
namespace: host.namespace,
outcome,
})
return
}
this.gfsTokenLifecycleEvidence.set(key, outcome)
}
/**
* Drops one object's entry. A caller holding the snapshot it reconciled
* knows which uid it recorded under, so it must not reach for the prefix
* sweep below: that would also destroy a sibling uid's evidence at the same
* name, which is the cross-object confusion the uid in the key exists to
* prevent (#696).
*/
private clearGfsLifecycleEvidenceForObject(
host: Pick<HostCRD, 'namespace' | 'name' | 'uid'>
): void {
const key = HostReconciler.gfsLifecycleEvidenceKey(host)
if (key !== undefined) this.gfsTokenLifecycleEvidence.delete(key)
}
/**
* Drops every object's entry at a namespace/name. Only for callers handed a
* name with no snapshot behind it, which is `reconcileDelete`: it cannot know
* which uid the departing object had.
*/
private clearGfsLifecycleEvidence(namespace: string, name: string): void {
const prefix = HostReconciler.gfsLifecycleEvidencePrefix(namespace, name)
for (const key of this.gfsTokenLifecycleEvidence.keys()) {
if (key.startsWith(prefix)) this.gfsTokenLifecycleEvidence.delete(key)
}
}
private static effectiveBootstrapRefreshBeforeSec(refreshTtlSec: number): number {
const configured = Math.max(0, config.mcpHostBootstrapRefreshBeforeSec ?? 0)
const resyncFloor = Math.max(0, config.hostResyncIntervalSec ?? 0) * 3
const requested = Math.max(configured, resyncFloor)
if (refreshTtlSec <= 1) return 0
return Math.min(requested, refreshTtlSec - 1)
}
private static runtimeTokenSecretAnnotations(
host: HostCRD,
tokens: Awaited<ReturnType<typeof issueMcpHostRuntimeTokens>>,
secretRevision: string,
nowMs: number,
gfsTokenTtlSec: number,
hasChannelIngress = false,
frontsOAuthServer = false,
preservedHostUid?: string,
projection?: CodexExecutionProjection,
grokProjection?: CodexExecutionProjection
): Record<string, string> {
const refreshTtlSec = Number.isFinite(tokens.refreshExpiresInSeconds)
? Math.max(0, tokens.refreshExpiresInSeconds)
: 0
const refreshExpiresAtMs = nowMs + refreshTtlSec * 1000
const refreshBeforeSec = HostReconciler.effectiveBootstrapRefreshBeforeSec(refreshTtlSec)
const refreshBeforeMs = refreshTtlSec > 0 ? refreshExpiresAtMs - refreshBeforeSec * 1000 : nowMs
const annotations: Record<string, string> = {
[RUNTIME_TOKEN_SECRET_REVISION_ANNOTATION]: secretRevision,
[RUNTIME_TOKEN_ISSUED_AT_ANNOTATION]: new Date(nowMs).toISOString(),
[RUNTIME_TOKEN_REFRESH_EXPIRES_AT_ANNOTATION]: new Date(refreshExpiresAtMs).toISOString(),
[RUNTIME_TOKEN_REFRESH_BEFORE_ANNOTATION]: new Date(refreshBeforeMs).toISOString(),
[RUNTIME_TOKEN_HOST_BINDING_HASH_ANNOTATION]:
HostReconciler.runtimeTokenHostBindingHash(host),
[RUNTIME_TOKEN_SCOPE_HASH_ANNOTATION]: HostReconciler.runtimeTokenScopeHash(
host,
hasChannelIngress,
frontsOAuthServer,
projection,
grokProjection
),
[RUNTIME_TOKEN_ISSUER_ANNOTATION]: RUNTIME_TOKEN_ISSUER,
[RUNTIME_TOKEN_AUDIENCE_ANNOTATION]: RUNTIME_TOKEN_AUDIENCE,
[RUNTIME_TOKEN_SCHEMA_VERSION_ANNOTATION]: RUNTIME_TOKEN_SCHEMA_VERSION,
[GFS_TOKEN_EXPECTED_SUBJECT_ANNOTATION]:
makeExpectedHostGfsSubject(host.namespace, host.name) ?? '',
[GFS_TOKEN_CAPABILITY_SET_HASH_ANNOTATION]: HostReconciler.gfsCapabilitySetHash(),
}
const hostUid = host.uid ?? preservedHostUid
if (hostUid) annotations[GFS_TOKEN_HOST_UID_ANNOTATION] = hostUid
if (host.generation !== undefined) {
annotations[GFS_TOKEN_HOST_GENERATION_ANNOTATION] = String(host.generation)
}
const gfsTtlSec = Number.isFinite(gfsTokenTtlSec) ? Math.max(0, gfsTokenTtlSec) : 0
if (gfsTtlSec > 0) {
const gfsExpiresAtMs = nowMs + gfsTtlSec * 1000
const gfsRefreshBeforeSec = HostReconciler.effectiveBootstrapRefreshBeforeSec(gfsTtlSec)
annotations[GFS_TOKEN_EXPIRES_AT_ANNOTATION] = new Date(gfsExpiresAtMs).toISOString()
annotations[GFS_TOKEN_REFRESH_BEFORE_ANNOTATION] = new Date(
gfsExpiresAtMs - gfsRefreshBeforeSec * 1000
).toISOString()
}
return annotations
}
private static runtimeTokenRefreshDecision(
host: HostCRD,
secret: k8s.V1Secret,
nowMs: number,
hasChannelIngress = false,
frontsOAuthServer = false,
projection?: CodexExecutionProjection,
grokProjection?: CodexExecutionProjection
): { refresh: boolean; rolloutRequired: boolean; reason: string; refreshTokenExpMs?: number } {
const labels = secret.metadata?.labels ?? {}
if (labels[MANAGED_BY_LABEL] !== MANAGED_BY_VALUE || labels[HOST_LABEL] !== host.name) {
// Reuse is only safe for Secrets HCC itself wrote for this Host. A
// foreign or unlabeled Secret is replaced with a freshly issued,
// HCC-owned one, and the pod rolls onto the trusted material.
return { refresh: true, rolloutRequired: true, reason: 'not_hcc_owned' }
}
const data = HostReconciler.runtimeTokenSecretData(secret)
if (!data) return { refresh: true, rolloutRequired: false, reason: 'malformed_secret' }
const annotations = secret.metadata?.annotations ?? {}
const expectedHostBindingHash = HostReconciler.runtimeTokenHostBindingHash(host)
const expectedScopeHash = HostReconciler.runtimeTokenScopeHash(
host,
hasChannelIngress,
frontsOAuthServer,
projection,
grokProjection
)
const hasContractMetadata =
RUNTIME_TOKEN_HOST_BINDING_HASH_ANNOTATION in annotations ||
RUNTIME_TOKEN_SCOPE_HASH_ANNOTATION in annotations ||
RUNTIME_TOKEN_ISSUER_ANNOTATION in annotations ||
RUNTIME_TOKEN_AUDIENCE_ANNOTATION in annotations ||
RUNTIME_TOKEN_SCHEMA_VERSION_ANNOTATION in annotations
if (!hasContractMetadata) {
return { refresh: true, rolloutRequired: false, reason: 'missing_refresh_metadata' }
}
const contractChanged =
annotations[RUNTIME_TOKEN_HOST_BINDING_HASH_ANNOTATION] !== expectedHostBindingHash ||
annotations[RUNTIME_TOKEN_SCOPE_HASH_ANNOTATION] !== expectedScopeHash ||
annotations[RUNTIME_TOKEN_ISSUER_ANNOTATION] !== RUNTIME_TOKEN_ISSUER ||
annotations[RUNTIME_TOKEN_AUDIENCE_ANNOTATION] !== RUNTIME_TOKEN_AUDIENCE ||
annotations[RUNTIME_TOKEN_SCHEMA_VERSION_ANNOTATION] !== RUNTIME_TOKEN_SCHEMA_VERSION
if (contractChanged) {
return { refresh: true, rolloutRequired: true, reason: 'contract_changed' }
}
const expectedGfsSubject = makeExpectedHostGfsSubject(host.namespace, host.name)
if (!expectedGfsSubject) {
return { refresh: true, rolloutRequired: true, reason: 'invalid_gfs_subject' }
}
if (annotations[GFS_TOKEN_EXPECTED_SUBJECT_ANNOTATION] !== expectedGfsSubject) {
return { refresh: true, rolloutRequired: true, reason: 'gfs_subject_changed' }
}
if (
annotations[GFS_TOKEN_CAPABILITY_SET_HASH_ANNOTATION] !==
HostReconciler.gfsCapabilitySetHash()
) {
return { refresh: true, rolloutRequired: true, reason: 'gfs_capability_set_changed' }
}
if (host.uid && annotations[GFS_TOKEN_HOST_UID_ANNOTATION] !== host.uid) {
return { refresh: true, rolloutRequired: true, reason: 'gfs_host_uid_changed' }
}
// Scheduling cross-check against the refresh token's REAL expiry. The
// `exp` claim is decoded from the JWT payload WITHOUT signature
// verification -- HCC must never verify third-party tokens (that is
// control-api's job); parsing `exp` to schedule rotation is metadata
// inspection, not verification. Fail toward a fresh token when the claim
// cannot be parsed.
const refreshTokenExpMs = decodeJwtExpMs(
Buffer.from(data[MCP_HOST_RUNTIME_TOKEN_SECRET_REFRESH_KEY], 'base64').toString('utf8')
)
if (refreshTokenExpMs === null) {
return { refresh: true, rolloutRequired: false, reason: 'unparsable_refresh_token' }
}
const issuedAtMs = Date.parse(annotations[RUNTIME_TOKEN_ISSUED_AT_ANNOTATION] ?? '')
const rotateBeforeMs = effectiveRotateBeforeMs(
refreshTokenExpMs,
Number.isFinite(issuedAtMs) ? issuedAtMs : null
)
if (nowMs >= refreshTokenExpMs - rotateBeforeMs) {
return {
refresh: true,
rolloutRequired: false,
reason: 'refresh_token_near_expiry',
refreshTokenExpMs,
}
}
const refreshBefore = Date.parse(annotations[RUNTIME_TOKEN_REFRESH_BEFORE_ANNOTATION] ?? '')
if (!Number.isFinite(refreshBefore)) {
return { refresh: true, rolloutRequired: false, reason: 'missing_refresh_metadata' }
}
if (nowMs >= refreshBefore) {
return {
refresh: true,
rolloutRequired: false,
reason: 'refresh_before_reached',
refreshTokenExpMs,
}
}
const gfsRefreshBefore = Date.parse(annotations[GFS_TOKEN_REFRESH_BEFORE_ANNOTATION] ?? '')
if (!Number.isFinite(gfsRefreshBefore)) {
return { refresh: true, rolloutRequired: false, reason: 'missing_gfs_refresh_metadata' }
}
if (nowMs >= gfsRefreshBefore) {
return { refresh: true, rolloutRequired: false, reason: 'gfs_refresh_before_reached' }
}
return { refresh: false, rolloutRequired: false, reason: 'current', refreshTokenExpMs }
}
private async readHostDeploymentOrNull(host: HostCRD): Promise<k8s.V1Deployment | null> {
try {
return await this.appsApi.readNamespacedDeployment({
name: host.name,
namespace: host.namespace,
})
} catch (err) {
if (getErrorCode(err) === 404) return null
throw err
}
}
private static deploymentRuntimeTokenRevision(deployment: k8s.V1Deployment | null): string {
return (
deployment?.spec?.template?.metadata?.annotations?.[RUNTIME_TOKEN_REVISION_ANNOTATION] ?? ''
)
}
private static deploymentReady(deployment: k8s.V1Deployment | null): boolean {
return (deployment?.status?.readyReplicas ?? 0) > 0
}
/** Bootstrap acknowledgement requires Ready replicas from the applied generation. */
private static deploymentBootstrapConverged(deployment: k8s.V1Deployment | null): boolean {
const generation = deployment?.metadata?.generation
const replicas = deployment?.spec?.replicas ?? 1
const status = deployment?.status
return (
generation !== undefined &&
generation > 0 &&
(status?.observedGeneration ?? 0) >= generation &&
replicas > 0 &&
status?.updatedReplicas === replicas &&
status?.readyReplicas === replicas &&
status?.replicas === replicas
)
}
private static deploymentResourcesMatchHost(
deployment: k8s.V1Deployment,
host: HostCRD
): boolean {
const resources = deployment.spec?.template.spec?.containers.find(
container => container.name === 'mcp-host'
)?.resources
const desired =
host.spec.desktop?.browser || host.spec.desktop?.x11
? config.desktopResources
: config.hostResources
let matches = true
for (const kind of ['requests', 'limits'] as const) {
const actual = resources?.[kind]
const expected = desired[kind]
if (!isDeepStrictEqual(Object.keys(actual ?? {}).sort(), Object.keys(expected).sort())) {
matches = false
}
for (const [key, value] of Object.entries(expected)) {
// Validate desired quantities even when a field is absent. A lexical
// PUT stays governed by the exact Deployment comparator, but equal
// Kubernetes amounts do not start a new Pod or need new credentials.
kubernetesQuantitiesEqual(value, value)
if (actual?.[key] === undefined || !kubernetesQuantitiesEqual(actual[key], value)) {
matches = false
}
}
}
return matches
}
/**
* The binding covers the Deployment UID and its runtime-token-revision
* annotation only. It does not bind the rest of the pod template, so a
* template edit that keeps both values still matches.
*
* This is deliberate: there is no pod-template hash. The record and the
* Deployment belong to the same Host, and any principal that can edit the
* pod template can already read the runtime-token Secret, so a template hash
* would add no isolation. A hash would also have to be computed on the
* apiserver-defaulted object, which never equals the applied body, so it
* would fail to match and re-mint the credentials on every reconcile.
*/
private static bootstrapBindingMatchesDeployment(
record: Pick<k8s.V1Secret, 'metadata'>,
deployment: k8s.V1Deployment | null
): boolean {
const deploymentIdentity = deployment?.metadata?.uid
const boundDeploymentIdentity =
record.metadata?.annotations?.[RUNTIME_TOKEN_BOOTSTRAP_DEPLOYMENT_UID_ANNOTATION]
const appliedRevision = HostReconciler.deploymentRuntimeTokenRevision(deployment)
const boundAppliedRevision =
record.metadata?.annotations?.[RUNTIME_TOKEN_BOOTSTRAP_APPLIED_REVISION_ANNOTATION]
return (
!!deploymentIdentity &&
boundDeploymentIdentity === deploymentIdentity &&
appliedRevision !== '' &&
boundAppliedRevision === appliedRevision
)
}
private static bootstrapAppliedToDeployment(
record: k8s.V1Secret,
deployment: k8s.V1Deployment | null,
contentRevision: string
): boolean {
if (HostReconciler.bootstrapBindingMatchesDeployment(record, deployment)) return true
// An existing target binding must never be adopted by a same-name/new-UID
// Deployment. Older unbound records can be bound by their applied revision.
if (record.metadata?.annotations?.[RUNTIME_TOKEN_BOOTSTRAP_DEPLOYMENT_UID_ANNOTATION]) {
return false
}
return (
contentRevision !== '' &&
HostReconciler.deploymentRuntimeTokenRevision(deployment) === contentRevision
)
}
private static bootstrapObservationUpdates(
host: HostCRD,
record: k8s.V1Secret,
deployment: k8s.V1Deployment | null,
contentRevision: string
): Record<string, string> {
if (
record.metadata?.annotations?.[RUNTIME_TOKEN_BOOTSTRAP_STATE_ANNOTATION] !==
RUNTIME_TOKEN_BOOTSTRAP_STATE_FRESH ||
!HostReconciler.deploymentBelongsToHost(deployment, host) ||
!HostReconciler.bootstrapAppliedToDeployment(record, deployment, contentRevision)
)
return {}
const current = record.metadata?.annotations ?? {}
const revision = HostReconciler.deploymentRuntimeTokenRevision(deployment)
const desired: Record<string, string> = {
[RUNTIME_TOKEN_BOOTSTRAP_DEPLOYMENT_UID_ANNOTATION]: deployment!.metadata!.uid!,
[RUNTIME_TOKEN_BOOTSTRAP_APPLIED_REVISION_ANNOTATION]: revision,
// Application is distinct from consumption: GFS content may change
// while the same bootstrap waits for strict acknowledgement.
[RUNTIME_TOKEN_ROLLOUT_REQUIRED_ANNOTATION]: 'false',
}
if (HostReconciler.deploymentBootstrapConverged(deployment)) {
desired[RUNTIME_TOKEN_BOOTSTRAP_STATE_ANNOTATION] = RUNTIME_TOKEN_BOOTSTRAP_STATE_CONSUMED
} else if (
(deployment!.spec?.replicas ?? 1) > 0 &&
(deployment!.metadata!.generation ?? 0) > 0 &&
(deployment!.status?.observedGeneration ?? 0) >= deployment!.metadata!.generation! &&
(deployment!.status?.updatedReplicas ?? 0) > 0 &&
HostReconciler.deploymentReady(deployment)
) {
// Aggregate Ready is only possible-use evidence, not an acknowledgement.
// Persist it for this UID/revision so a later loss of Ready can recover
// once; uninterrupted NotReady startup never acquires this observation.
desired[RUNTIME_TOKEN_BOOTSTRAP_READY_REVISION_ANNOTATION] = revision
}
return Object.fromEntries(
Object.entries(desired).filter(([key, value]) => current[key] !== value)
)
}
private static credentialRecordBelongsToHost(
record: { metadata?: { labels?: Record<string, string> } },
host: HostCRD
): boolean {
return HostReconciler.isHccOwnedHostResource(record, host.name)
}
private static runtimeRecordIsComplete(
record: Pick<k8s.V1Secret, 'data' | 'stringData'>
): boolean {
return HostReconciler.runtimeTokenSecretData(record) !== null
}
private static retainedBoolean(record: k8s.V1Secret, key: string): boolean | null {
const value = record.metadata?.annotations?.[key]
return value === 'true' ? true : value === 'false' ? false : null
}
private static deploymentBelongsToHost(
deployment: k8s.V1Deployment | null,
host: HostCRD
): deployment is k8s.V1Deployment {
return (
!!deployment &&
HostReconciler.credentialRecordBelongsToHost(deployment, host) &&
deployment.metadata?.name === host.name &&
deployment.metadata?.namespace === host.namespace &&
!!deployment.metadata?.uid &&
!!deployment.metadata?.resourceVersion &&
!deployment.metadata?.deletionTimestamp &&
!!deployment.spec?.template?.spec &&
!!host.uid &&
deployment.metadata?.annotations?.[HOST_UID_ANNOTATION] === host.uid
)
}
private static credentialRenewalWindowReached(
record: k8s.V1Secret,
nowMs: number,
annotation: string
): boolean {
const refreshBefore = Date.parse(record.metadata?.annotations?.[annotation] ?? '')
return !Number.isFinite(refreshBefore) || nowMs >= refreshBefore
}
/**
* True when this reconcile will (re)start a pod that reads the
* mcp-host-runtime-token Secret fresh at boot -- a missing Deployment, a
* Deployment scaled to 0 (the wake / scale 0->1 path), or a Deployment with
* no Ready replica (a pod is (re)starting). In every such case the booting
* pod bootstraps its refresh token from the Secret env-var material, so that
* material MUST be freshly minted and un-revoked.
*
* The mcp-host runtime refresh token is single-use-rotating: control-api
* revokes the prior JTI on every successful refresh. Once a running pod has
* rotated, the copy still sitting in this HCC-owned Secret is REVOKED.
* Reusing it (because its decoded exp is far out) hands a booting pod a
* revoked refresh token -- it can never refresh its 300s access token,
* readiness 503s forever, and the pod never becomes Ready. Reuse is only
* safe for a pod that stays up and never re-reads the Secret (it uses its
* in-memory rotated tokens), which is exactly deploymentReady && replicas>=1.
*
* A wake bootstrap already bound to this exact Deployment UID and applied
* runtime revision is the one startup exception: its replica-only wake starts
* that Deployment with the newly mounted material. Any identity or revision
* change must mint again.
*/
private static deploymentNeedsFreshBootstrap(
deployment: k8s.V1Deployment | null,
currentRevision: string,
credentialRecord: k8s.V1Secret | null,
options: BootstrapOptions
): boolean {
if (options.forceFreshForWake || options.forceFreshForDeploymentChange || !deployment)
return true
// This reconcile will converge the Deployment to zero, so no Pod will
// consume the stored material before the next verified wake.
if (options.targetSuspended === true) return false
if ((deployment.spec?.replicas ?? 1) === 0) return true
const annotations = credentialRecord?.metadata?.annotations ?? {}
const fresh =
annotations[RUNTIME_TOKEN_BOOTSTRAP_STATE_ANNOTATION] === RUNTIME_TOKEN_BOOTSTRAP_STATE_FRESH
const boundUid = annotations[RUNTIME_TOKEN_BOOTSTRAP_DEPLOYMENT_UID_ANNOTATION]
if (boundUid && boundUid !== deployment.metadata?.uid) return true
if (HostReconciler.deploymentReady(deployment)) return false
if (
fresh &&
credentialRecord &&
HostReconciler.bootstrapBindingMatchesDeployment(credentialRecord, deployment) &&
annotations[RUNTIME_TOKEN_BOOTSTRAP_READY_REVISION_ANNOTATION] ===
HostReconciler.deploymentRuntimeTokenRevision(deployment)
)
return true
if (
fresh &&
credentialRecord &&
(HostReconciler.bootstrapBindingMatchesDeployment(credentialRecord, deployment) ||
(boundUid === deployment.metadata?.uid &&
annotations[RUNTIME_TOKEN_ROLLOUT_REQUIRED_ANNOTATION] === 'true'))
) {
return false
}
return (
credentialRecord?.metadata?.annotations?.[RUNTIME_TOKEN_BOOTSTRAP_STATE_ANNOTATION] !==
RUNTIME_TOKEN_BOOTSTRAP_STATE_FRESH ||
HostReconciler.deploymentRuntimeTokenRevision(deployment) !== currentRevision
)
}
private static shouldRollForRuntimeSecret(
deployment: k8s.V1Deployment | null,
force: boolean
): boolean {
if (!deployment) return true
if (force) return true
if (!HostReconciler.deploymentReady(deployment)) return true
const revision = HostReconciler.deploymentRuntimeTokenRevision(deployment)
if (!revision) return true
return revision === LEGACY_EMPTY_RUNTIME_TOKEN_REVISION
}
private static hasFreshBootstrapSecretNewerThanHealthyDeployment(
secret: k8s.V1Secret,
secretRevision: string,
deployment: k8s.V1Deployment | null,
nowMs: number
): boolean {
if (!HostReconciler.deploymentReady(deployment)) return false
const deploymentRevision = HostReconciler.deploymentRuntimeTokenRevision(deployment)
if (!deploymentRevision || deploymentRevision === secretRevision) return false
const refreshExpiresAt = Date.parse(
secret.metadata?.annotations?.[RUNTIME_TOKEN_REFRESH_EXPIRES_AT_ANNOTATION] ?? ''
)
return Number.isFinite(refreshExpiresAt) && nowMs < refreshExpiresAt
}
private static deploymentRestartedAfterRuntimeSecretIssued(
secret: k8s.V1Secret,
deployment: k8s.V1Deployment | null
): boolean {
const restartedAt = Date.parse(
deployment?.spec?.template?.metadata?.annotations?.['kubectl.kubernetes.io/restartedAt'] ?? ''
)
if (!Number.isFinite(restartedAt)) return false
const issuedAt = Date.parse(
secret.metadata?.annotations?.[RUNTIME_TOKEN_ISSUED_AT_ANNOTATION] ?? ''
)
return Number.isFinite(issuedAt) && restartedAt > issuedAt
}
private hasChannelIngress(host: HostCRD): boolean {
return (host.spec.channels?.length ?? 0) > 0 || this.countCommunicationChannels(host.name) > 0
}
private resolveWorkflowControlScopesForHost(
host: HostCRD,
hasChannelIngress = this.hasChannelIngress(host),
frontsOAuthServer = false
): HostRuntimeControlScope[] {
// Delegate to the SAME resolver the drift hash uses so the minted scope set
// and the hashed scope set can never diverge (a divergence re-issues the
// token every reconcile).
const scopes = resolveRuntimeControlScopes(host.spec.workflowControl, {
hasChannelIngress,
frontsOAuthServer,
})
const hasWorkflowScope = scopes.some(scope => scope !== OAUTH_USER_TOKEN_SCOPE)
if (hasChannelIngress && !hasWorkflowScope) {
log.warn('channel-bearing Host resolved with no workflow-control scopes', {
host: host.name,
namespace: host.namespace,
})
}
return scopes
}
private projectCodexForHost(host: HostCRD): CodexExecutionProjection {
const connectionKey = assignedHostCodexConnectionRef(host.spec.model?.connectionRef)
const snapshot = snapshotForAssignedCodexGrant(
connectionKey,
this.lastCodexConfigMap,
this.codexSnapshot
)
return projectCodexExecution(host.spec, snapshot)
}
private projectGrokForHost(host: HostCRD): CodexExecutionProjection & {
requiresGrokProxyEgress: boolean
} {
const connectionKey = assignedHostCodexConnectionRef(host.spec.model?.connectionRef)
const snapshot = parseGrokAllowedModelsSnapshot(this.lastCodexConfigMap, connectionKey)
return projectGrokExecution(host.spec, snapshot)
}
private hostDerivesGrokExecution(host: HostCRD): boolean {
return this.projectGrokForHost(host).derivedScopes.includes(GROK_EXECUTE_SCOPE)
}
private resolveEffectiveControlScopesForHost(
host: HostCRD,
frontsOAuthServer: boolean,
hasChannelIngress = this.hasChannelIngress(host)
): EffectiveMcpHostControlScope[] {
const workflow = this.resolveWorkflowControlScopesForHost(
host,
hasChannelIngress,
frontsOAuthServer
)
const derived = [
...this.projectCodexForHost(host).derivedScopes,
...this.projectGrokForHost(host).derivedScopes,
].filter(scope => !workflow.includes(scope as HostWorkflowControlScope))
return [...workflow, ...derived] as EffectiveMcpHostControlScope[]
}
private runtimeScopeHashFor(
host: HostCRD,
frontsOAuthServer: boolean,
hasChannelIngress = this.hasChannelIngress(host)
): string {
return HostReconciler.runtimeTokenScopeHash(
host,
hasChannelIngress,
frontsOAuthServer,
this.projectCodexForHost(host),
this.projectGrokForHost(host)
)
}
/**
* Deployment drift guard: does the provisioned credential still match the
* Host's current scope contract? An OAuth read that failed after its retries
* is "scope unknown", so it matches a provision minted under either OAuth
* answer instead of collapsing to `false` and forcing a re-mint that would
* strip oauth:user-token and roll the Deployment. The issuance path owns the
* decision to narrow the grant (see `ensureMcpHostRuntimeTokenSecret`).
*/
private currentScopeMatchesProvision(
host: HostCRD,
provision: RuntimeTokenProvision,
live: Awaited<ReturnType<HostReconciler['observeFrontsOAuthServer']>>
): boolean {
if (live.observed) {
return provision.scopeHash === this.runtimeScopeHashFor(host, live.frontsOAuthServer)
}
return (
provision.scopeHash === this.runtimeScopeHashFor(host, true) ||
provision.scopeHash === this.runtimeScopeHashFor(host, false)
)
}
private async refreshCodexSnapshot(): Promise<void> {
try {
const cm = await this.coreApi.readNamespacedConfigMap({
name: ALLOWED_MODELS_CONFIGMAP_NAME,
namespace: config.hccTargetNamespace,
})
this.lastCodexConfigMap = cm
this.codexSnapshot = parseAllowedModelsSnapshot(cm)
} catch (err) {
const timeout =
err instanceof HostK8sRequestTimeoutError ||
(err as { code?: string }).code === HOST_K8S_REQUEST_TIMEOUT_CODE
if (timeout) {
this.lastCodexConfigMap = undefined
this.codexSnapshot = snapshotFromConfigMapError('timeout')
log.warn('Codex allowlist ConfigMap read timed out; failing closed', {
configMap: ALLOWED_MODELS_CONFIGMAP_NAME,
namespace: config.hccTargetNamespace,
})
return
}
const code = getErrorCode(err)
if (code === 401 || code === 403) {
this.lastCodexConfigMap = undefined
this.codexSnapshot = snapshotFromConfigMapError('forbidden')
log.warn('Codex allowlist ConfigMap read forbidden; failing closed', {
configMap: ALLOWED_MODELS_CONFIGMAP_NAME,
namespace: config.hccTargetNamespace,
statusCode: code,
})
return
}
this.lastCodexConfigMap = undefined
this.codexSnapshot = snapshotFromConfigMapError('missing')
log.warn('Codex allowlist ConfigMap unavailable; failing closed', {
configMap: ALLOWED_MODELS_CONFIGMAP_NAME,
namespace: config.hccTargetNamespace,
statusCode: code,
})
}
}
private async ensureMcpHostRuntimeTokenSecret(
host: HostCRD,
options: BootstrapOptions = {}
): Promise<RuntimeTokenProvision | null> {
let lastErr: unknown = null
// The held GFS-only renewal runs inside a reconcile that keeps a Ready
// runtime in place; it gets one attempt so a failing renewal never blocks
// that reconcile on backoff. The next hold pass retries it.
const maxAttempts = options.refreshGfsOnly ? 1 : 3
for (let attempt = 1; attempt <= maxAttempts; attempt++) {
try {
const expectedGfsSubject = makeExpectedHostGfsSubject(host.namespace, host.name)
if (!expectedGfsSubject) {
throw new Error('Host GFS token provisioning requires a trusted namespace and name')
}
const name = mcpHostRuntimeTokenSecretName(host)
let existing: k8s.V1Secret | null = null
try {
existing = await observeExistenceRead('Secret', () =>
this.coreApi.readNamespacedSecret({ name, namespace: host.namespace })
)
} catch (err) {
if (getErrorCode(err) !== 404) throw err
}
const deployment = await this.readHostDeploymentOrNull(host)
const existingRevision = existing
? HostReconciler.runtimeTokenSecretRevisionFromSecret(existing)
: null
const nowMs = Date.now()
const preserveWakeTemplate =
options.preserveDeploymentTemplateOnWake === true &&
!!deployment &&
HostReconciler.deploymentBelongsToHost(deployment, host) &&
(deployment.spec?.replicas ?? 1) === 0 &&
HostReconciler.deploymentRuntimeTokenRevision(deployment) !== ''
if (options.refreshGfsOnly) {
const trustedDeployment =
HostReconciler.deploymentBelongsToHost(deployment, host) &&
(deployment?.spec?.replicas ?? 0) > 0
if (
!existing ||
!HostReconciler.credentialRecordBelongsToHost(existing, host) ||
!host.uid ||
existing.metadata?.annotations?.[GFS_TOKEN_HOST_UID_ANNOTATION] !== host.uid ||
!HostReconciler.runtimeRecordIsComplete(existing) ||
!trustedDeployment
) {
log.warn(
'deferring held-runtime credential renewal because runtime identity is not verified',
{
host: host.name,
namespace: host.namespace,
}
)
return null
}
const annotationUpdates = HostReconciler.bootstrapObservationUpdates(
host,
existing,
deployment,
existingRevision ?? ''
)
const renewalDue = HostReconciler.credentialRenewalWindowReached(
existing,
nowMs,
GFS_TOKEN_REFRESH_BEFORE_ANNOTATION
)
// A pending strict acknowledgement cannot starve independent GFS
// renewal. Bind the applied bootstrap before changing the full
// content hash; runtime bytes and the applied revision stay intact.
if (!renewalDue || !HostReconciler.deploymentReady(deployment)) {
if (Object.keys(annotationUpdates).length > 0) {
await this.coreApi.replaceNamespacedSecret({
name,
namespace: host.namespace,
body: {
...existing,
metadata: {
...existing.metadata,
annotations: {
...(existing.metadata?.annotations ?? {}),
...annotationUpdates,
},
},
},
})
}
if (
!HostReconciler.deploymentReady(deployment) &&
!HostReconciler.bootstrapBindingMatchesDeployment(existing, deployment)
)
return null
this.recordGfsLifecycleEvidence(host, 'reused')
return {
revision:
HostReconciler.deploymentRuntimeTokenRevision(deployment) || existingRevision || '',
scopeHash:
existing.metadata?.annotations?.[RUNTIME_TOKEN_SCOPE_HASH_ANNOTATION] ?? '',
}
}
const gfs = await mintHostGfsToken({ name: host.name, namespace: host.namespace })
const data = { ...(HostReconciler.runtimeTokenSecretData(existing) ?? {}) }
data[MCP_HOST_GFS_TOKEN_SECRET_KEY] = Buffer.from(gfs.token).toString('base64')
const gfsExpiresAtMs = nowMs + Math.max(0, gfs.expiresInSeconds) * 1000
const gfsRefreshBeforeSec = HostReconciler.effectiveBootstrapRefreshBeforeSec(
Math.max(0, gfs.expiresInSeconds)
)
const nextRevision = HostReconciler.runtimeTokenSecretRevision(data)
await this.coreApi.replaceNamespacedSecret({
name,
namespace: host.namespace,
body: {
...existing,
data,
metadata: {
...existing.metadata,
annotations: {
...(existing.metadata?.annotations ?? {}),
...annotationUpdates,
[GFS_TOKEN_EXPECTED_SUBJECT_ANNOTATION]:
makeExpectedHostGfsSubject(host.namespace, host.name) ?? '',
[GFS_TOKEN_CAPABILITY_SET_HASH_ANNOTATION]: HostReconciler.gfsCapabilitySetHash(),
[GFS_TOKEN_EXPIRES_AT_ANNOTATION]: new Date(gfsExpiresAtMs).toISOString(),
[GFS_TOKEN_REFRESH_BEFORE_ANNOTATION]: new Date(
gfsExpiresAtMs - gfsRefreshBeforeSec * 1000
).toISOString(),
[RUNTIME_TOKEN_SECRET_REVISION_ANNOTATION]: nextRevision,
},
},
},
})
this.recordGfsLifecycleEvidence(host, 'rotated')
return {
revision:
HostReconciler.deploymentRuntimeTokenRevision(deployment) || existingRevision || '',
scopeHash: existing.metadata?.annotations?.[RUNTIME_TOKEN_SCOPE_HASH_ANNOTATION] ?? '',
}
}
const bootstrapIsFresh =
existing?.metadata?.annotations?.[RUNTIME_TOKEN_BOOTSTRAP_STATE_ANNOTATION] ===
RUNTIME_TOKEN_BOOTSTRAP_STATE_FRESH
const retainedAnnotations = existing?.metadata?.annotations ?? {}
const cacheSynced = this.ccCacheSyncedFn()
const retainedScopeIsTrusted =
!!existing &&
HostReconciler.credentialRecordBelongsToHost(existing, host) &&
!!host.uid &&
retainedAnnotations[GFS_TOKEN_HOST_UID_ANNOTATION] === host.uid
const retainedChannelIngress =
retainedScopeIsTrusted && existing
? HostReconciler.retainedBoolean(existing, RUNTIME_TOKEN_HAS_CHANNEL_INGRESS_ANNOTATION)
: null
const retainedFrontsOAuth =
retainedScopeIsTrusted && existing
? HostReconciler.retainedBoolean(existing, RUNTIME_TOKEN_FRONTS_OAUTH_ANNOTATION)
: null
const hasChannelIngress = cacheSynced
? this.hasChannelIngress(host)
: retainedChannelIngress === true
if (!cacheSynced && retainedChannelIngress === null) {
log.warn(
'skipping runtime token mint during channel cache loss without retained scope observation',
{
host: host.name,
namespace: host.namespace,
observation: 'hasChannelIngress',
}
)
return null
}
const projection = this.projectCodexForHost(host)
const grokProjection = this.projectGrokForHost(host)
const decideRefresh = (
frontsOAuth: boolean
): {
refresh: boolean
rolloutRequired: boolean
reason: string
refreshTokenExpMs?: number
} => {
let decided: {
refresh: boolean
rolloutRequired: boolean
reason: string
refreshTokenExpMs?: number
} = existing
? HostReconciler.runtimeTokenRefreshDecision(
host,
existing,
nowMs,
hasChannelIngress,
frontsOAuth,
projection,
grokProjection
)
: { refresh: true, rolloutRequired: false, reason: 'missing_secret' }
if (
existing &&
HostReconciler.deploymentRestartedAfterRuntimeSecretIssued(existing, deployment)
) {
// A manual restart after the runtime Secret was issued starts pods with
// env-var bootstrap tokens from that older Secret. Force a re-issue and
// rollout so every restarted pod observes a fresh, unconsumed token pair.
decided = {
refresh: true,
rolloutRequired: true,
reason: 'deployment_restarted_after_secret_issued',
}
}
// Revoked-on-wake guard: the mcp-host runtime refresh token is
// single-use-rotating (control-api revokes the prior JTI on every
// refresh). A pod that is about to boot -- wake (scale 0->1), missing
// Deployment, or a not-Ready (re)starting pod -- reads its bootstrap
// refresh token from THIS Secret. If the pre-suspend pod already
// rotated, the copy in the Secret is REVOKED, so an exp-based "reuse"
// hands the booting pod a dead token and readiness 503s forever. When
// a new pod will consume the Secret, force a fresh mint + rollout even
// though the decoded exp is far out. Reuse stays correct only for a
// running, Ready pod that never re-reads the Secret (churn fix).
if (
existingRevision &&
decided.refresh === false &&
HostReconciler.deploymentNeedsFreshBootstrap(
deployment,
existingRevision,
existing,
options
)
) {
decided = {
refresh: true,
rolloutRequired: true,
reason: 'fresh_mint_for_booting_pod',
...(decided.refreshTokenExpMs !== undefined
? { refreshTokenExpMs: decided.refreshTokenExpMs }
: {}),
}
}
return decided
}
// Resolve ONCE per issuance and thread the SAME frontsOAuthServer bool and
// codex projection into the mint-scope derive, the refresh decision, the
// scope hash and the stored annotation, so the minted token and the drift
// hash never diverge.
//
// The OAuth observation reads the McpServer cache, which does not depend
// on the CommunicationChannel cache, so it stays live during channel
// cache loss: an admin who removed the OAuth mcp-server while the channel
// watch was down must not have `oauth:user-token` re-granted from the
// retained annotation on a held wake.
//
// A live read that failed after its retries is "scope unknown", not
// "scope revoked". It resolves to `false` (fail closed) only when that
// cannot narrow a grant the running pod holds: the retained value is
// `false`, or there is no trusted retained value while channel authority
// is synced. A retained `true` under synced authority defers the decision
// while the credential needs no renewal, so an McpServer relist does not
// strip oauth:user-token and roll the Deployment; once the credential must
// be renewed, the renewal mints without the unobserved scope. Under channel
// cache loss the mint is skipped instead.
const liveFrontsOAuth = await this.observeFrontsOAuthServer(host)
let observedFrontsOAuth: boolean
if (liveFrontsOAuth.observed) {
observedFrontsOAuth = liveFrontsOAuth.frontsOAuthServer
} else {
const observationFailure = {
host: host.name,
namespace: host.namespace,
observation: 'frontsOAuthServer',
attempts: liveFrontsOAuth.attempts,
err:
liveFrontsOAuth.error instanceof Error
? liveFrontsOAuth.error.message
: String(liveFrontsOAuth.error),
}
if (retainedFrontsOAuth === false || (cacheSynced && retainedFrontsOAuth !== true)) {
log.warn(
'failed to resolve whether Host fronts an oauth mcp-server after retries; failing closed (no oauth:user-token scope)',
observationFailure
)
observedFrontsOAuth = false
} else if (cacheSynced) {
if (!decideRefresh(true).refresh) {
log.warn(
'deferring runtime token decision: OAuth observation unavailable and retained scope grants oauth:user-token',
observationFailure
)
return null
}
log.warn(
'renewing runtime token without oauth:user-token: OAuth observation unavailable when the credential must be renewed',
observationFailure
)
observedFrontsOAuth = false
} else {
log.warn(
'skipping runtime token mint during channel cache loss without an authoritative OAuth observation',
observationFailure
)
return null
}
}
const scopeHash = HostReconciler.runtimeTokenScopeHash(
host,
hasChannelIngress,
observedFrontsOAuth,
projection,
grokProjection
)
let decision = decideRefresh(observedFrontsOAuth)
const resourcesWillChange =
options.prepareForResourceChange === true &&
HostReconciler.deploymentBelongsToHost(deployment, host) &&
(deployment.spec?.replicas ?? 1) > 0 &&
!HostReconciler.deploymentResourcesMatchHost(deployment!, host)
const bootstrapApplied =
!!existing &&
!!existingRevision &&
HostReconciler.bootstrapAppliedToDeployment(existing, deployment, existingRevision)
const boundUid =
existing?.metadata?.annotations?.[RUNTIME_TOKEN_BOOTSTRAP_DEPLOYMENT_UID_ANNOTATION]
// A routine renewal can be consumed by a container restart without
// changing the applied revision. Reuse only explicit rollout preparation.
const usablePendingPreparation =
bootstrapIsFresh &&
existing?.metadata?.annotations?.[RUNTIME_TOKEN_ROLLOUT_REQUIRED_ANNOTATION] === 'true' &&
!bootstrapApplied &&
(!boundUid || boundUid === deployment?.metadata?.uid)
if (resourcesWillChange) {
// Resource preparation is optional until scope authority is observed;
// it must not participate in the intrinsic renewal/OAuth fallback above.
// Compose a real transition with an already due renewal in one issuance.
decision = {
...decision,
refresh: decision.refresh || !usablePendingPreparation,
rolloutRequired: true,
reason: decision.refresh ? decision.reason : 'fresh_mint_for_resource_change',
}
}
if (existing && existingRevision && !decision.refresh) {
const deploymentRevision = HostReconciler.deploymentRuntimeTokenRevision(deployment)
const rolloutMarker =
existing.metadata?.annotations?.[RUNTIME_TOKEN_ROLLOUT_REQUIRED_ANNOTATION]
const boundTargetRevision =
existing.metadata?.annotations?.[RUNTIME_TOKEN_BOOTSTRAP_APPLIED_REVISION_ANNOTATION]
const preparedRevision =
bootstrapIsFresh && boundUid === deployment?.metadata?.uid && boundTargetRevision
? boundTargetRevision
: existingRevision
const bootstrapBoundToAppliedDeployment =
HostReconciler.bootstrapBindingMatchesDeployment(existing, deployment)
const bootstrapPendingForAppliedDeployment = bootstrapIsFresh && bootstrapApplied
const bootstrapConsumedForAppliedDeployment =
existing.metadata?.annotations?.[RUNTIME_TOKEN_BOOTSTRAP_STATE_ANNOTATION] ===
RUNTIME_TOKEN_BOOTSTRAP_STATE_CONSUMED && bootstrapBoundToAppliedDeployment
const rolloutPending =
!bootstrapApplied &&
(resourcesWillChange ||
rolloutMarker === 'true' ||
(rolloutMarker === undefined &&
bootstrapIsFresh &&
deploymentRevision !== '' &&
deploymentRevision !== existingRevision))
const annotationUpdates = HostReconciler.bootstrapObservationUpdates(
host,
existing,
deployment,
existingRevision
)
if (resourcesWillChange && usablePendingPreparation && !bootstrapApplied) {
// Older preparations can already carry the rollout marker without
// a target binding. Persist that binding before delivering them.
const preparationAnnotations = {
[RUNTIME_TOKEN_ROLLOUT_REQUIRED_ANNOTATION]: 'true',
[RUNTIME_TOKEN_BOOTSTRAP_DEPLOYMENT_UID_ANNOTATION]: deployment!.metadata!.uid!,
[RUNTIME_TOKEN_BOOTSTRAP_APPLIED_REVISION_ANNOTATION]: preparedRevision,
}
for (const [key, value] of Object.entries(preparationAnnotations)) {
if (existing.metadata?.annotations?.[key] !== value) annotationUpdates[key] = value
}
}
if (cacheSynced) {
const currentIngress =
existing.metadata?.annotations?.[RUNTIME_TOKEN_HAS_CHANNEL_INGRESS_ANNOTATION]
if (currentIngress !== String(hasChannelIngress)) {
annotationUpdates[RUNTIME_TOKEN_HAS_CHANNEL_INGRESS_ANNOTATION] =
String(hasChannelIngress)
}
const currentOAuth =
existing.metadata?.annotations?.[RUNTIME_TOKEN_FRONTS_OAUTH_ANNOTATION]
if (currentOAuth !== String(observedFrontsOAuth)) {
annotationUpdates[RUNTIME_TOKEN_FRONTS_OAUTH_ANNOTATION] = String(observedFrontsOAuth)
}
}
if (
host.generation !== undefined &&
existing.metadata?.annotations?.[GFS_TOKEN_HOST_GENERATION_ANNOTATION] !==
String(host.generation)
) {
annotationUpdates[GFS_TOKEN_HOST_GENERATION_ANNOTATION] = String(host.generation)
}
if (Object.keys(annotationUpdates).length > 0) {
await this.coreApi.replaceNamespacedSecret({
name,
namespace: host.namespace,
body: {
...existing,
metadata: {
...existing.metadata,
annotations: {
...(existing.metadata?.annotations ?? {}),
...annotationUpdates,
},
},
},
})
}
const refreshExpInHours =
typeof decision.refreshTokenExpMs === 'number'
? Math.round(((decision.refreshTokenExpMs - nowMs) / 3_600_000) * 10) / 10
: null
log.info('reusing mcp-host-runtime-token Secret', {
host: host.name,
namespace: host.namespace,
resourceName: name,
refreshExpInHours,
})
this.recordGfsLifecycleEvidence(host, 'reused')
const selectedRevision =
rolloutPending ||
(!bootstrapPendingForAppliedDeployment &&
!bootstrapConsumedForAppliedDeployment &&
HostReconciler.shouldRollForRuntimeSecret(deployment, false))
? preparedRevision
: deploymentRevision || existingRevision
return {
revision: selectedRevision,
scopeHash,
deploymentUid: deployment?.metadata?.uid,
freshBootstrapPrepared:
bootstrapIsFresh &&
!bootstrapApplied &&
(annotationUpdates[RUNTIME_TOKEN_BOOTSTRAP_DEPLOYMENT_UID_ANNOTATION] ?? boundUid) ===
deployment?.metadata?.uid &&
(annotationUpdates[RUNTIME_TOKEN_BOOTSTRAP_APPLIED_REVISION_ANNOTATION] ??
boundTargetRevision) === selectedRevision,
}
}
if (
!resourcesWillChange &&
existing &&
existingRevision &&
(decision.reason === 'refresh_before_reached' ||
decision.reason === 'refresh_token_near_expiry') &&
HostReconciler.hasFreshBootstrapSecretNewerThanHealthyDeployment(
existing,
existingRevision,
deployment,
nowMs
)
) {
log.info('preserved fresh mcp-host-runtime-token Secret for healthy deployment', {
host: host.name,
namespace: host.namespace,
resourceName: name,
reason: decision.reason,
})
return {
revision: HostReconciler.deploymentRuntimeTokenRevision(deployment) || existingRevision,
scopeHash,
deploymentUid: deployment?.metadata?.uid,
}
}
const tokens = await issueMcpHostRuntimeTokens(
host.name,
host.uid ?? '',
this.resolveEffectiveControlScopesForHost(host, observedFrontsOAuth, hasChannelIngress)
)
const gfs = await mintHostGfsToken({ name: host.name, namespace: host.namespace })
const body = buildMcpHostRuntimeTokenSecret(
host,
tokens.accessToken,
tokens.refreshToken,
tokens.mcpHostControlToken,
gfs.token
)
const revision = HostReconciler.runtimeTokenSecretRevisionFromSecret(body)
if (!revision) {
throw new Error('mcp-host-runtime-token Secret body missing required credential keys')
}
const rolloutRequired =
!preserveWakeTemplate &&
(!existing ||
HostReconciler.shouldRollForRuntimeSecret(deployment, decision.rolloutRequired))
body.metadata = {
...body.metadata,
annotations: {
...(body.metadata?.annotations ?? {}),
...HostReconciler.runtimeTokenSecretAnnotations(
host,
tokens,
revision,
nowMs,
gfs.expiresInSeconds,
hasChannelIngress,
observedFrontsOAuth,
existing?.metadata?.annotations?.[GFS_TOKEN_HOST_UID_ANNOTATION],
projection,
grokProjection
),
[RUNTIME_TOKEN_BOOTSTRAP_STATE_ANNOTATION]: RUNTIME_TOKEN_BOOTSTRAP_STATE_FRESH,
[RUNTIME_TOKEN_ROLLOUT_REQUIRED_ANNOTATION]: rolloutRequired ? 'true' : 'false',
...((preserveWakeTemplate || rolloutRequired) &&
HostReconciler.deploymentBelongsToHost(deployment, host)
? {
[RUNTIME_TOKEN_BOOTSTRAP_DEPLOYMENT_UID_ANNOTATION]:
deployment.metadata?.uid ?? '',
[RUNTIME_TOKEN_BOOTSTRAP_APPLIED_REVISION_ANNOTATION]: preserveWakeTemplate
? HostReconciler.deploymentRuntimeTokenRevision(deployment)
: revision,
}
: {}),
[RUNTIME_TOKEN_HAS_CHANNEL_INGRESS_ANNOTATION]: String(hasChannelIngress),
[RUNTIME_TOKEN_FRONTS_OAUTH_ANNOTATION]: String(observedFrontsOAuth),
},
}
if (!existing) {
await observeCreate('Secret', () =>
this.coreApi.createNamespacedSecret({ namespace: host.namespace, body })
)
this.recordGfsLifecycleEvidence(host, 'minted')
log.info('created mcp-host-runtime-token Secret', {
host: host.name,
namespace: host.namespace,
resourceName: name,
reason: decision.reason,
})
return {
revision: preserveWakeTemplate
? HostReconciler.deploymentRuntimeTokenRevision(deployment) || revision
: revision,
scopeHash,
deploymentUid: deployment?.metadata?.uid,
freshBootstrapPrepared:
rolloutRequired && HostReconciler.deploymentBelongsToHost(deployment, host),
}
}
const replaceBody = {
...body,
metadata: {
...body.metadata,
resourceVersion: existing.metadata?.resourceVersion,
},
}
await this.coreApi.replaceNamespacedSecret({
name,
namespace: host.namespace,
body: replaceBody,
})
this.recordGfsLifecycleEvidence(host, 'rotated')
log.info('rotated mcp-host-runtime-token Secret', {
host: host.name,
namespace: host.namespace,
resourceName: name,
reason: decision.reason,
rolloutRequired,
})
return {
revision: preserveWakeTemplate
? HostReconciler.deploymentRuntimeTokenRevision(deployment) || revision
: rolloutRequired
? revision
: HostReconciler.deploymentRuntimeTokenRevision(deployment) || revision,
scopeHash,
deploymentUid: deployment?.metadata?.uid,
freshBootstrapPrepared:
rolloutRequired && HostReconciler.deploymentBelongsToHost(deployment, host),
}
} catch (err) {
lastErr = err
if (makeExpectedHostGfsSubject(host.namespace, host.name)) {
this.recordGfsLifecycleEvidence(host, 'failed')
}
if (err instanceof Error && err.name === 'GfsHostTokenValidationError') break
const delayMs = 1000 * Math.pow(2, attempt - 1)
log.error('mcp-host-runtime-token Secret attempt failed', {
host: host.name,
namespace: host.namespace,
attempt,
maxAttempts,
error: String(err),
})
if (attempt < maxAttempts) {
await new Promise(resolve => setTimeout(resolve, delayMs))
}
}
}
throw new Error(
`Failed to ensure mcp-host-runtime-token Secret for host "${host.name}" after ${maxAttempts} attempt${maxAttempts === 1 ? '' : 's'}: ${String(lastErr)}`
)
}
private async provisionRuntimeTokenRevision(
host: HostCRD,
options: BootstrapOptions
): Promise<RuntimeTokenProvision | null> {
try {
return await this.ensureMcpHostRuntimeTokenSecret(host, options)
} catch (err) {
log.error('mcpHost runtime token provisioning failed', { host: host.name, err })
this.setStatus(host.name, {
deployed: false,
ready: false,
message: 'mcpHost runtime token provisioning failed',
})
throw err
}
}
/**
* Delete per-Host RBAC. Called from reconcileDelete. 404s are swallowed.
*/
private async deleteHostRbac(name: string, namespace: string): Promise<void> {
const saName = `host-${name}-sa`
const roleName = `host-${name}-config-reader`
const failures: unknown[] = []
for (const cleanup of [
() =>
this.deleteIfHccOwned(
'RoleBinding',
roleName,
namespace,
name,
() => this.rbacApi.readNamespacedRoleBinding({ name: roleName, namespace }),
() => this.rbacApi.deleteNamespacedRoleBinding({ name: roleName, namespace })
),
() =>
this.deleteIfHccOwned(
'Role',
roleName,
namespace,
name,
() => this.rbacApi.readNamespacedRole({ name: roleName, namespace }),
() => this.rbacApi.deleteNamespacedRole({ name: roleName, namespace })
),
() =>
this.deleteIfHccOwned(
'ServiceAccount',
saName,
namespace,
name,
() => this.coreApi.readNamespacedServiceAccount({ name: saName, namespace }),
() => this.coreApi.deleteNamespacedServiceAccount({ name: saName, namespace })
),
]) {
try {
await cleanup()
} catch (err) {
failures.push(err)
}
}
throwCleanupFailures(failures, `Failed to delete RBAC for Host "${name}"`)
}
private async deleteMcpHostRuntimeTokenSecret(name: string, namespace: string): Promise<void> {
const secretName = `host-${name}-mcp-host-runtime-tokens`
try {
const secret = await this.coreApi.readNamespacedSecret({ name: secretName, namespace })
const labels = secret.metadata?.labels ?? {}
if (labels[MANAGED_BY_LABEL] !== MANAGED_BY_VALUE || labels[HOST_LABEL] !== name) {
log.warn('Skipping runtime Secret delete - not HCC-owned', {
name: secretName,
host: name,
})
return
}
await this.coreApi.deleteNamespacedSecret({ name: secretName, namespace })
} catch (err) {
if (getErrorCode(err) !== 404) {
log.error('Failed to delete runtime Secret', { name: secretName, err })
throw err
}
}
}
private async deleteLegacyChannelReaderRuntimeAuth(name: string): Promise<void> {
const suffix = ['mcp', 'host', 'runtime', 'auth'].join('-')
const nameToDelete = `channel-reader-${name}-${suffix}`
try {
const obj = await this.coreApi.readNamespacedSecret({
name: nameToDelete,
namespace: config.channelsNamespace,
})
const labels = obj.metadata?.labels ?? {}
const owned =
labels[MANAGED_BY_LABEL] === MANAGED_BY_VALUE &&
labels[HOST_LABEL] === name &&
labels['clerum.io/component'] === 'channel-reader' &&
labels['clerum.io/secret-purpose'] === suffix
if (!owned) {
log.warn('Skipping legacy runtime auth cleanup; labels do not prove HCC ownership', {
name: nameToDelete,
})
return
}
await this.coreApi.deleteNamespacedSecret({
name: nameToDelete,
namespace: config.channelsNamespace,
})
} catch (err) {
if (getErrorCode(err) !== 404) {
log.error('Failed legacy runtime auth cleanup', { name: nameToDelete, err })
throw err
}
}
}
getStatus(name: string): HostRuntimeStatus {
return (
this.statusMap.get(name) ?? { deployed: false, ready: false, message: 'Not reconciled yet' }
)
}
/**
* Track which hosts have desktop enabled.
* Updated during reconcile, cleared during reconcileDelete.
*/
private readonly desktopHosts: Set<string> = new Set()
hasDesktop(hostRef: string): boolean {
return this.desktopHosts.has(hostRef)
}
private setStatus(name: string, status: HostRuntimeStatus): void {
this.statusMap.set(name, status)
this.statusGenerations.set(name, this.reconcileGenerations.get(name) ?? 0)
}
private advanceReconcileGeneration(name: string): number {
const generation = (this.reconcileGenerations.get(name) ?? 0) + 1
this.reconcileGenerations.set(name, generation)
return generation
}
private clearStatus(name: string): void {
this.statusMap.delete(name)
this.reconcileGenerations.delete(name)
this.statusGenerations.delete(name)
this.lifecycle.clearHost(name)
const timer = this.readinessTimers.get(name)
if (timer) {
clearTimeout(timer)
this.readinessTimers.delete(name)
}
}
/**
* Undefined when the snapshot carries no uid, which `HostCRD` documents as a
* legal state. Sending the reference anyway would earn a 400 the reporter
* treats as terminal, so the event would be dropped — and, unlike the
* administrative path, re-sent and dropped again on every pass, because this
* reporter keeps no `seen` set to fall silent. Every telemetry emitter goes
* through here so the drop is decided, and named, in one place (#693).
*/
private hostTelemetryReference(
host: HostCRD,
context: Record<string, string>
): HostLookupReference | undefined {
if (host.uid === undefined) {
log.warn('skipping host telemetry: Host snapshot has no uid', {
host: host.name,
namespace: host.namespace,
...context,
})
return undefined
}
return {
name: host.name,
namespace: host.namespace,
...(host.generation !== undefined ? { generation: host.generation } : {}),
uid: host.uid,
}
}
private enqueueHostTelemetry(
host: HostCRD,
telemetryType: 'lifecycle_transition' | 'reconcile_outcome' | 'controller_error',
reasonCode: string,
payload: HccInfrastructureTelemetryPayload
): void {
const hostLookupReference = this.hostTelemetryReference(host, { telemetryType, reasonCode })
if (hostLookupReference === undefined) return
const occurredAt = this.now().toISOString()
const eventPayload = {
resource_class: 'Host',
reason_code: reasonCode,
...payload,
}
if (telemetryType === 'reconcile_outcome') {
this.infrastructureTelemetryReporter?.enqueue({
occurredAt,
telemetryType,
hostLookupReference,
payload: eventPayload,
})
return
}
this.infrastructureTelemetryReporter?.enqueue({
sourceEventId: `hcc-${telemetryType}:${this.newTelemetryOccurrenceId()}`,
occurredAt,
telemetryType,
hostLookupReference,
payload: eventPayload,
})
}
private enqueueControllerError(host: HostCRD, reasonCode: string, error: unknown): void {
this.enqueueHostTelemetry(host, 'controller_error', reasonCode, {
status: 'failed',
error_class: error instanceof Error ? error.name : typeof error,
})
this.enqueueAdministrativeOutcome(host, 'failed', reasonCode)
}
private enqueueReconcileOutcome(host: HostCRD): void {
const status = this.getStatus(host.name)
const succeeded = status.deployed && status.ready
this.enqueueReconcileTelemetry(host, succeeded ? 'ready' : 'not_ready', {
status: succeeded ? 'succeeded' : 'failed',
phase: status.deployed ? 'deployed' : 'not_deployed',
state: status.ready ? 'ready' : 'not_ready',
})
if (succeeded) this.enqueueAdministrativeOutcome(host, 'succeeded', 'reconciled')
}
/**
* A pass that threw may not have written any status yet, so the status in
* memory can still be the previous pass's. Report the failure itself, not
* that status, and never an administrative `succeeded` (#327).
*/
private enqueueReconcileExceptionOutcome(host: HostCRD): void {
this.enqueueReconcileTelemetry(host, 'reconcile_exception', { status: 'failed' })
}
private enqueueReconcileTelemetry(
host: HostCRD,
reasonCode: string,
payload: HccInfrastructureTelemetryPayload
): void {
const evidenceKey = HostReconciler.gfsLifecycleEvidenceKey(host)
const gfsOutcome = evidenceKey ? this.gfsTokenLifecycleEvidence.get(evidenceKey) : undefined
if (evidenceKey) this.gfsTokenLifecycleEvidence.delete(evidenceKey)
this.enqueueHostTelemetry(host, 'reconcile_outcome', reasonCode, {
...payload,
// control-api allowlists `transition`, not gfs_* keys (#328).
...(gfsOutcome ? { transition: `gfs_token:${gfsOutcome}` } : {}),
})
}
private enqueueAdministrativeOutcome(
host: HostCRD,
outcome: 'succeeded' | 'failed',
reasonCode: string
): void {
const operationId = host.annotations?.['clerum.io/administrative-intent-id']
if (!operationId || host.generation === undefined) return
if (host.uid === undefined) {
// control-api binds the outcome to the live object's metadata.uid and
// refuses a reference without one, so an event sent now would be a 400
// this reporter treats as terminal. Drop it here and say why (#694).
log.warn('skipping administrative outcome: Host snapshot has no uid', {
host: host.name,
namespace: host.namespace,
outcome,
})
return
}
this.administrativeOutcomeReporter?.enqueueHostOutcome({
// v2: the identity gained the Host uid. sourceStatusRef feeds the
// server's payload hash, so keeping the v1 key would make every live
// key answer 409 and the uid-bearing claim would never be stored (#694).
sourceEventId: `hcc-admin-outcome-v2:${operationId}:${host.generation}:${host.uid}:${outcome}`,
occurredAt: this.now().toISOString(),
hostRef: {
name: host.name,
namespace: host.namespace,
generation: host.generation,
uid: host.uid,
},
outcome,
reasonCode,
})
}
private pvcName(host: HostCRD): string {
return `${host.name}-workspace`
}
// ─── Stateless lifecycle (Stage 2) ──────────────────────────────────
//
// Assessment, durable status writes, the heartbeat executors
// (StatelessLifecycleReconcilerPort), the wake fast-path, and the
// per-host serialization chain live in StatelessLifecycleExecutor
// (statelessLifecycleExecutor.ts). The methods below are thin delegates
// so external callers keep their HostReconciler entry points.
/** Public view of the synchronous effective-lifecycle derivation. */
getEffectiveLifecycle(host: HostCRD): EffectiveHostLifecycle {
return this.lifecycle.getEffectiveLifecycle(host)
}
/** Public view of the clerum.io/wake-requested generation parser. */
getWakeRequestedGeneration(host: HostCRD): number {
return this.lifecycle.getWakeRequestedGeneration(host)
}
/** FRESH GET (not the watch cache) of a Host — see StatelessLifecycleExecutor. */
async readFreshHost(host: HostCRD): Promise<HostCRD> {
return this.lifecycle.readFreshHost(host)
}
/**
* Suspend a stateless Host after the drained report — see
* StatelessLifecycleExecutor. `entryWakeHandledGeneration` is the AP-1
* generation epoch captured by the tracker when the suspend was decided.
*/
async suspendHostFromHeartbeat(
host: HostCRD,
reason: string,
entryWakeHandledGeneration: number
): Promise<SuspendFromHeartbeatOutcome> {
return this.lifecycle.suspendHostFromHeartbeat(host, reason, entryWakeHandledGeneration)
}
/** Publish the D8 suspend-blocked reason — see StatelessLifecycleExecutor. */
async publishSuspendBlockedReason(host: HostCRD, reason: string): Promise<void> {
return this.lifecycle.publishSuspendBlockedReason(host, reason)
}
/**
* Persist the tracker's drain decision durably — see
* StatelessLifecycleExecutor. `entryWakeHandledGeneration` is the AP-1
* generation epoch the tracker captured when the drain was decided.
*/
async markHostDrainingFromHeartbeat(
host: HostCRD,
entryWakeHandledGeneration: number
): Promise<void> {
return this.lifecycle.markHostDrainingFromHeartbeat(host, entryWakeHandledGeneration)
}
/** Tracker-side cancel-drain — see StatelessLifecycleExecutor. */
async markHostActiveFromHeartbeat(host: HostCRD): Promise<void> {
return this.lifecycle.markHostActiveFromHeartbeat(host)
}
/** Resolve a Host pod's creationTimestamp by pod UID (max-uptime ceiling). */
async findPodCreationTimestamp(host: HostCRD, podUid: string): Promise<Date | null> {
return this.lifecycle.findPodCreationTimestamp(host, podUid)
}
/**
* Pull policy applied to this stateless Host's pod (Stage 6, W5). The
* operator's resolved policy is preserved (never overridden to an
* unpullable Always for a node-local image). When IfNotPresent is used with
* a mutable image, the stale-cached-image-on-wake risk is warned once per
* (host, image) and carried durably by the StatelessPullPolicyRejected
* condition (assessLifecycle) — operator-visible, non-breaking.
*/
private statelessImagePullPolicyForHost(
host: HostCRD,
image: string
): 'Always' | 'IfNotPresent' | 'Never' {
const resolution = resolveStatelessImagePullPolicy(image)
if (resolution.mutableImageRisk === undefined) {
this.pullPolicyRejectionLogged.delete(host.name)
return resolution.policy
}
if (this.pullPolicyRejectionLogged.get(host.name) !== image) {
this.pullPolicyRejectionLogged.set(host.name, image)
log.warn(
'Stateless Host runs a mutable image reference (no @sha256: digest, tag not sha-<gitsha>): a node with a stale cached image serves old code on wake. Pin an immutable reference to eliminate the risk.',
{ host: host.name, imagePullPolicy: resolution.policy, image }
)
}
return resolution.policy
}
async validateHostSecret(host: HostCRD): Promise<HostSecretValidationResult> {
if (!host.spec.secretRef) {
return { ok: true }
}
try {
await this.coreApi.readNamespacedSecret({
namespace: host.namespace,
name: host.spec.secretRef,
})
return { ok: true }
} catch (error) {
const code = getErrorCode(error)
if (code === 404) {
const message = `Secret "${host.spec.secretRef}" not found in namespace "${host.namespace}"`
log.error('Host Secret not found; Host will not be deployed', {
host: host.name,
detail: message,
})
return { ok: false, reason: 'SecretNotFound', message }
}
if (code === 401 || code === 403) {
const message =
`Access denied reading Secret "${host.spec.secretRef}" in namespace "${host.namespace}" ` +
`(K8s API ${code})`
log.error('Host Secret access denied; Host will be failed closed', {
host: host.name,
detail: message,
})
return { ok: false, reason: 'SecretAccessDenied', message }
}
const message = `Failed to validate Secret "${host.spec.secretRef}" for host "${host.name}"`
log.error('Failed to validate Host Secret', { host: host.name, detail: message, err: error })
return { ok: false, reason: 'ReadError', message }
}
}
private buildPvc(host: HostCRD): k8s.V1PersistentVolumeClaim {
const pvcName = this.pvcName(host)
return {
apiVersion: 'v1',
kind: 'PersistentVolumeClaim',
metadata: {
name: pvcName,
namespace: host.namespace,
labels: {
[MANAGED_BY_LABEL]: MANAGED_BY_VALUE,
[HOST_LABEL]: host.name,
},
},
spec: {
storageClassName: config.hostWorkspaceStorageClassName,
accessModes: ['ReadWriteOnce'],
resources: {
requests: {
storage: config.hostWorkspaceStorageSize,
},
},
},
}
}
/**
* Build the per-Host channel-reader Deployment manifest. Mirrors the static
* deploy/base/channels/channel-reader.yaml shape, but per-Host:
* - name = channel-reader-<host>
* - namespace = config.channelsNamespace
* - env = CLERUM_HOST_REF=<host>
* - annotation clerum.io/credentials-revision=<sha256> when provided
*
* `secretRevision` is sha256(canonicalJSON(secret.data)); empty string
* means "no Secret yet — boot with optional envFrom; SecretInformer will
* patch the annotation later, triggering a roll".
*/
private buildChannelReaderDeployment(
host: HostCRD,
secretRevision: string,
existingReplicas?: number
): k8s.V1Deployment {
const name = `channel-reader-${host.name}`
const annotations: Record<string, string> = {}
if (secretRevision) {
annotations['clerum.io/credentials-revision'] = secretRevision
}
// CRITICAL: app.kubernetes.io/name MUST be unique per Host.
//
// The legacy static Deployment in deploy/base/channels/channel-reader.yaml
// uses selector.matchLabels: app.kubernetes.io/name=channel-reader. K8s
// Deployment selectors match supersets, so if per-Host pods carried the
// plain "channel-reader" value, the static Deployment's ReplicaSet would
// try to adopt them and delete N-1 every reconcile to converge on its
// single-replica goal — producing a hot loop where per-Host Deployments
// recreate pods that the static one immediately deletes.
//
// Per the K8s recommended-labels semantics, each per-Host Deployment is
// a distinct application instance, so the unique-per-Host value is
// appropriate. Coexistence with the static Deployment during migration
// works correctly. The static Deployment will be retired in a follow-up.
const appKubernetesName = `channel-reader-${host.name}`
const labels = {
app: 'channel-reader',
'app.kubernetes.io/name': appKubernetesName,
'app.kubernetes.io/part-of': 'clerum',
[HOST_LABEL]: host.name,
[MANAGED_BY_LABEL]: MANAGED_BY_VALUE,
}
// Pods carry the managed-by label so the per-Host NetworkPolicies can
// require it in their podSelectors and refuse to apply to pods spoofed
// into the channels namespace by another controller. Note: NOT included
// in `selector.matchLabels` below — Deployment selectors are immutable
// after create, so adding it there would require delete+recreate of
// existing per-Host Deployments. Pod-template labels are mutable; new
// pods on the next rollout get the label, NetworkPolicies match.
const podLabels = {
app: 'channel-reader',
'app.kubernetes.io/name': appKubernetesName,
'app.kubernetes.io/part-of': 'clerum',
[HOST_LABEL]: host.name,
[MANAGED_BY_LABEL]: MANAGED_BY_VALUE,
}
// Replica count is driven by CommunicationChannel count for this Host (#281):
// 0 CCs → no pod runs (saves ~128Mi RAM on idle Hosts); 1+ CCs → one pod
// polls. McpServerWatcher injects the counter via setCountCommunicationChannels;
// the default returns 0, so any Host whose counter hasn't been wired up
// scales to 0 (safe — Telegram polling halts but no resource leak).
//
// B2 — guard against spurious scale-to-0 during startup:
// If the CC cache has NOT finished its initial list (ccCacheSyncedFn returns
// false) AND we know the existing Deployment already has replicas=N, preserve
// those replicas rather than forcing them to 0. This prevents a restart of
// HCC from momentarily killing a live channel-reader before the CC cache is
// populated. Once the cache is synced the normal ccCount logic takes over.
const ccCount = this.countCommunicationChannels(host.name)
const cacheIsSynced = this.ccCacheSyncedFn()
const desiredReplicas =
!cacheIsSynced && existingReplicas !== undefined ? existingReplicas : ccCount > 0 ? 1 : 0
return {
apiVersion: 'apps/v1',
kind: 'Deployment',
metadata: { name, namespace: config.channelsNamespace, labels },
spec: {
replicas: desiredReplicas,
selector: { matchLabels: { app: 'channel-reader', [HOST_LABEL]: host.name } },
template: {
metadata: {
labels: podLabels,
...(Object.keys(annotations).length > 0 ? { annotations } : {}),
},
spec: {
serviceAccountName: 'clerum-channel-reader',
containers: [
{
name: 'channel-reader',
image: config.channelReaderImage,
imagePullPolicy: config.channelReaderImagePullPolicy,
ports: [
{
name: 'handoff',
containerPort: config.channelReaderHandoffPort,
protocol: 'TCP',
},
],
envFrom: [{ configMapRef: { name: 'clerum-channel-reader-config' } }],
env: [
{ name: 'CLERUM_HOST_REF', value: host.name },
// Inject the pod's own namespace via the downward API so the
// channel-reader always discovers CommunicationChannels and
// credential Secrets in its own namespace (channels-<slug>
// under MCC, or "channels" in the default stack). This explicit
// env wins over any envFrom ConfigMap value, making the
// namespace correct by construction regardless of the ConfigMap.
{
name: 'CLERUM_NAMESPACE',
valueFrom: { fieldRef: { fieldPath: 'metadata.namespace' } },
},
// The mcp-host Service (buildService) is named host.name in
// host.namespace (mcp-host-<slug> under MCC), type ClusterIP,
// exposing the `http` port = config.hostPort. The channel-reader
// lives in channels-<slug>, so the URL MUST cross namespaces and
// use host.namespace — not config.channelsNamespace.
{
name: 'CLERUM_MCP_HOST_URL',
value: `http://${host.name}.${host.namespace}.svc.cluster.local:${config.hostPort}`,
},
{
name: 'CLERUM_CHANNEL_READER_HANDOFF_TOKEN',
valueFrom: {
secretKeyRef: {
name: 'workflow-approval-request-reader-credentials',
key: 'channel-reader-handoff-token',
optional: true,
},
},
},
],
resources: {
requests: { cpu: '50m', memory: '128Mi' },
limits: { cpu: '200m', memory: '256Mi' },
},
securityContext: {
allowPrivilegeEscalation: false,
readOnlyRootFilesystem: true,
runAsNonRoot: true,
runAsUser: 1000,
runAsGroup: 1000,
// Restricted-PSA-compliant: MCC shared-tenant namespaces
// (channels-<slug>) enforce the `restricted` PodSecurity
// standard, which requires capabilities.drop=[ALL] and
// seccompProfile=RuntimeDefault.
capabilities: { drop: ['ALL'] },
seccompProfile: { type: 'RuntimeDefault' },
},
},
],
},
},
},
}
}
private buildChannelReaderService(host: HostCRD): k8s.V1Service {
const name = `channel-reader-${host.name}`
return {
apiVersion: 'v1',
kind: 'Service',
metadata: {
name,
namespace: config.channelsNamespace,
labels: {
app: 'channel-reader',
'app.kubernetes.io/name': name,
'app.kubernetes.io/part-of': 'clerum',
[HOST_LABEL]: host.name,
[MANAGED_BY_LABEL]: MANAGED_BY_VALUE,
},
},
spec: {
type: 'ClusterIP',
selector: {
app: 'channel-reader',
[HOST_LABEL]: host.name,
[MANAGED_BY_LABEL]: MANAGED_BY_VALUE,
},
ports: [
{
name: 'handoff',
port: config.channelReaderHandoffPort,
targetPort: 'handoff' as IntOrString,
protocol: 'TCP',
},
],
},
}
}
private async reconcileChannelReaderService(
host: HostCRD,
revalidate?: () => void
): Promise<void> {
const name = `channel-reader-${host.name}`
const ns = config.channelsNamespace
const service = this.buildChannelReaderService(host)
const mutationAllowed = () => {
revalidate?.()
return true
}
await ensureResource({
mutationAllowed,
read: () =>
observeExistenceRead('Service', () =>
this.coreApi.readNamespacedService({ name, namespace: ns })
),
create: () =>
observeCreate('Service', () =>
this.coreApi.createNamespacedService({ namespace: ns, body: service })
),
onSkipped: () => createsTotal.inc({ kind: 'Service', outcome: 'skipped' }),
converge: read =>
replaceWithConflictRetry({
description: `channel-reader Service "${name}"`,
logPrefix: '[HostReconciler]',
body: service,
mergeExisting: preserveServiceAssignedFields,
isUpToDate: serviceMatchesDesired,
mutationAllowed,
read,
replace: body => this.coreApi.replaceNamespacedService({ name, namespace: ns, body }),
}),
})
}
/**
* Idempotent reconcile of the per-Host channel-reader Deployment. Mirrors
* the PVC create-or-replace pattern at hostReconciler.ts:497-519.
*
* - Reads existing per-host credentials Secret to seed the
* clerum.io/credentials-revision pod template annotation, so the initial
* pod boots with the right hash and doesn't get rolled twice when
* SecretInformer fires later.
* - Reads before creation and compares a server-normalized Deployment; it
* replaces only meaningful drift. Conflict retries rebuild against the
* latest live replica count. A Deployment owned by a different host is
* never overwritten: the initial read skips, and a retry-time ownership
* change throws so write_skips_total does not count the refusal as a
* healthy no-op.
*/
private async reconcileChannelReaderDeployment(
host: HostCRD,
revalidate?: () => void
): Promise<void> {
const name = `channel-reader-${host.name}`
const ns = config.channelsNamespace
const revision = await this.computeChannelReaderRevisionForHost(host.name)
const desired = this.buildChannelReaderDeployment(host, revision)
const mutationAllowed = () => {
revalidate?.()
return true
}
await ensureResource({
mutationAllowed,
read: () =>
observeExistenceRead('Deployment', () =>
this.appsApi.readNamespacedDeployment({ name, namespace: ns })
),
create: () =>
observeCreate('Deployment', () =>
this.appsApi.createNamespacedDeployment({ namespace: ns, body: desired })
),
onSkipped: () => createsTotal.inc({ kind: 'Deployment', outcome: 'skipped' }),
converge: async (read, observed) => {
// A post-create-conflict read still throws on 404. Later retry reads
// keep the existing benign-disappearance policy in the retry helper.
const existing = observed ?? (await read())
const ownerHost = existing.metadata?.labels?.[HOST_LABEL]
if (ownerHost && ownerHost !== host.name) {
log.warn('Channel-reader Deployment belongs to another Host; refusing update', { name })
return false
}
// ensureResource already supplies a consume-once snapshot for the
// initial presence path. Only POST409 needs its owner-check read retained.
let snapshot: k8s.V1Deployment | undefined = observed === undefined ? existing : undefined
await replaceWithConflictRetry({
description: `channel-reader Deployment "${name}"`,
logPrefix: '[HostReconciler]',
body: this.buildChannelReaderDeployment(host, revision, existing.spec?.replicas),
mutationAllowed,
read: () => {
if (snapshot !== undefined) {
const first = snapshot
snapshot = undefined
return Promise.resolve(first)
}
return read()
},
replace: body => this.appsApi.replaceNamespacedDeployment({ name, namespace: ns, body }),
mergeExisting: (_body, fresh) => {
const desiredWithLiveReplicas = this.buildChannelReaderDeployment(
host,
revision,
fresh.spec?.replicas
)
desiredWithLiveReplicas.metadata = {
...desiredWithLiveReplicas.metadata,
resourceVersion: fresh.metadata?.resourceVersion,
}
return preserveChannelReaderDeploymentAnnotations(desiredWithLiveReplicas, fresh)
},
validateExisting: fresh => {
const freshOwnerHost = fresh.metadata?.labels?.[HOST_LABEL]
if (freshOwnerHost && freshOwnerHost !== host.name) {
throw new Error(
`channel-reader Deployment ownership changed to host "${freshOwnerHost}"; refusing replace`
)
}
},
isUpToDate: deploymentMatchesDesired,
})
},
})
}
/**
* Patch the `clerum.io/credentials-revision` annotation on the per-Host
* channel-reader Deployment so K8s rolls the pod. Revision = sha256 of
* the host's credentials Secret content (empty when no per-CC Secret
* resolves yet). Idempotent on unchanged input — patches the same value
* are no-ops at the K8s layer.
*
* Strategic-merge-patch preserves any other annotations on the pod
* template. 404 on the Deployment is benign (no per-Host pod for that
* host yet — McpServerWatcher's fullReconcile will create one when the
* Host CRD is present).
*/
async patchChannelReaderRevisionAnnotation(hostName: string): Promise<void> {
const capturedAuthority = this.captureHostMutationAuthority()
const depName = `channel-reader-${hostName}`
try {
await this.lifecycle.serializeByHost(hostName, async () => {
const action = `channel-reader "${hostName}" revision patch`
this.requireHostMutationAuthority(action, capturedAuthority)
const resolveCurrentHost = this.resolveCurrentHost
const admittedHost = resolveCurrentHost?.(hostName)
if (resolveCurrentHost !== null && admittedHost === undefined) return
const revalidate = admittedHost
? this.makeHostMutationAdmission(
action,
admittedHost,
capturedAuthority,
this.resolveHostMutationDependencies?.(admittedHost)
)
: () => this.requireHostMutationAuthority(action, capturedAuthority)
revalidate()
const revision = await this.computeChannelReaderRevisionForHost(hostName)
// The revision calculation can await multiple Secret reads. Revalidate
// Host LIST/WATCH authority plus UID and generation immediately before
// entering the Kubernetes mutation, while still holding this Host lane.
revalidate()
const patchBody = {
spec: {
template: {
metadata: {
annotations: { 'clerum.io/credentials-revision': revision },
},
},
},
}
await this.appsApi.patchNamespacedDeployment(
{ name: depName, namespace: config.channelsNamespace, body: patchBody },
{
middleware: [
k8s.setHeaderMiddleware('Content-Type', 'application/strategic-merge-patch+json'),
],
}
)
log.info('Patched channel-reader credentials-revision', {
deployment: depName,
revision: revision || '(empty)',
})
})
} catch (err) {
if (getErrorCode(err) === 404) return
log.error('Failed to reconcile channel-reader credentials-revision', {
deployment: depName,
err,
})
}
}
/**
* Compute the revision for a host by hashing the data of the Secret(s)
* referenced by all CCs that resolve to this host. Used by
* patchChannelReaderRevisionAnnotation. Returns '' when no Secret is
* resolvable yet.
*
* Wiring: a `findCommunicationChannelsByHostRef` dependency is injected
* via setFindCommunicationChannelsByHostRef from McpServerWatcher.
*/
private async computeChannelReaderRevisionForHost(hostName: string): Promise<string> {
const ccs = this.findCommunicationChannelsByHostRef(hostName)
const secretNames = new Set<string>()
for (const cc of ccs) {
if (cc.spec.credentialsSecretRef?.name) secretNames.add(cc.spec.credentialsSecretRef.name)
}
if (secretNames.size === 0) return ''
const sortedNames = [...secretNames].sort()
const combined: Record<string, Record<string, string>> = {}
for (const name of sortedNames) {
try {
const secret = await this.coreApi.readNamespacedSecret({
name,
namespace: config.channelsNamespace,
})
combined[name] = (secret.data ?? {}) as Record<string, string>
} catch (err) {
if (getErrorCode(err) === 404) continue
throw err
}
}
const canonical = canonicalStringify(combined)
return createHash('sha256').update(canonical).digest('hex')
}
/**
* Called by the channels-namespace SecretInformer (started in main.ts) on
* Secret rotation events. Looks up which CommunicationChannels reference
* the rotated Secret via `findCommunicationChannelsByCredentialsSecretName`
* (CC cache reverse-map), collects the affected hosts, and patches each
* host's channel-reader Deployment annotation exactly once via
* `patchChannelReaderRevisionAnnotation`.
*
* Cross-namespace events are skipped early — only Secrets in the
* configured channelsNamespace are interesting (this avoids spurious
* cache lookups when the informer fires for unrelated Secrets).
*
* A single Secret can map to multiple hosts (operators can share a
* Secret across CCs of different Hosts via raw YAML); each affected host
* is patched once even if it owns multiple CCs that reference the same
* Secret.
*
* When no CC references the Secret, this is a no-op — the Secret is
* either orphaned or owned by a different subsystem.
*
* Per-host error handling lives in `patchChannelReaderRevisionAnnotation`
* (swallows Deployment 404 / Secret 404, logs others). This method does
* NOT throw, so the SecretInformer event loop continues regardless of
* per-host failures.
*/
async reconcileChannelReaderRevision(secretName: string, secretNamespace: string): Promise<void> {
if (secretNamespace !== config.channelsNamespace) return
const affectedCCs = this.findCommunicationChannelsByCredentialsSecretName(secretName)
if (affectedCCs.length === 0) return
const affectedHosts = new Set<string>(affectedCCs.map(cc => cc.spec.hostRef))
// Addendum 2: each affected Host patches only its own `channel-reader-<host>`
// Deployment annotation — a disjoint, order-independent, idempotent write set
// per Host (§10.4 inventory). Fan out through the bounded worker pool
// (reusing HCC_HOST_FULL_RECONCILE_CONCURRENCY) so a shared credentials-Secret
// change touching many Hosts is not head-of-line blocked, while never issuing
// an unbounded burst at the K8s API. patchChannelReaderRevisionAnnotation
// still swallows its own per-Host errors, so this method continues not to
// throw (runBoundedHostWorkers never rethrows).
await runBoundedHostWorkers(
Array.from(affectedHosts),
config.hostFullReconcileConcurrency,
hostName => this.patchChannelReaderRevisionAnnotation(hostName)
)
}
buildDeployment(
host: HostCRD,
mounts: ResolvedSfsMount[] = [],
runtimeTokenRevision = '',
lifecycle?: EffectiveHostLifecycle,
grokExecutionEnabled = this.hostDerivesGrokExecution(host)
): k8s.V1Deployment {
const labels: Record<string, string> = {
app: host.name,
[MANAGED_BY_LABEL]: MANAGED_BY_VALUE,
[HOST_LABEL]: host.name,
[CONTEXT_LABEL]: host.spec.contextRef,
}
const isDesktop = !!(host.spec.desktop?.browser || host.spec.desktop?.x11)
// reconcile() passes the full async lifecycle assessment; direct callers
// (and tests) fall back to the synchronously derivable view.
const effectiveLifecycle = lifecycle ?? this.lifecycle.effectiveLifecycleFromCache(host)
const isStateless = effectiveLifecycle.stateless
const image = isDesktop ? config.desktopImage : config.hostImage
const resources = isDesktop ? config.desktopResources : config.hostResources
// Stage 6 (W5): stateless pods may override the pull policy (a mutable
// image is reported, not refused — see resolveStatelessImagePullPolicy). Non-stateless pods keep the
// global policy so their manifest stays byte-identical.
const imagePullPolicy = isStateless
? this.statelessImagePullPolicyForHost(host, image)
: config.hostImagePullPolicy
// Ports
const ports: Array<{ name: string; containerPort: number; protocol: string }> = [
{ name: 'http', containerPort: config.hostPort, protocol: 'TCP' },
]
if (isDesktop) {
ports.push({ name: 'desktop', containerPort: config.desktopPort, protocol: 'TCP' })
}
// When desktop is enabled, mount the workspace at /config/workspace so the
// XFCE user (abc, UID 1001) can browse the same files the agent writes to.
const workspacePath = isDesktop ? '/config/workspace' : config.hostWorkspacePath
const reservedMountPaths = [
workspacePath,
'/tmp',
WORKFLOW_TOKEN_MOUNT_PATH,
MCP_HOST_RUNTIME_AUTH_STATE_PATH,
]
const contextMounts = mounts.filter(m => {
const reason = HostReconciler.contextMountPathRejectionReason(m.mountPath, reservedMountPaths)
if (reason) {
log.warn('Skipping invalid SharedFileSystem context mount', {
host: host.name,
sharedFileSystem: m.name,
mountPath: m.mountPath,
reason,
})
return false
}
return true
})
// Env
const env: k8s.V1EnvVar[] = [
{ name: 'CLERUM_HOST_NAME', value: host.name },
{
name: 'CLERUM_NAMESPACE',
valueFrom: { fieldRef: { fieldPath: 'metadata.namespace' } },
},
{ name: 'CLERUM_WORKSPACE_PATH', value: workspacePath },
// Changing spec.secretRef changes the pod template and rolls the host.
{ name: 'CLERUM_LLM_SECRET_REF', value: host.spec.secretRef ?? '' },
// mcpHost runtime token env vars. Names match WRC's podFactory.ts so mcp-host
// sees the same shape regardless of which controller provisioned it.
{
name: 'MCP_HOST_RUNTIME_ACCESS_TOKEN',
valueFrom: {
secretKeyRef: {
name: mcpHostRuntimeTokenSecretName(host),
key: MCP_HOST_RUNTIME_TOKEN_SECRET_ACCESS_KEY,
},
},
},
{
name: 'MCP_HOST_RUNTIME_REFRESH_TOKEN',
valueFrom: {
secretKeyRef: {
name: mcpHostRuntimeTokenSecretName(host),
key: MCP_HOST_RUNTIME_TOKEN_SECRET_REFRESH_KEY,
},
},
},
{
name: 'MCP_HOST_WORKFLOW_CONTROL_TOKEN',
valueFrom: {
secretKeyRef: {
name: mcpHostRuntimeTokenSecretName(host),
key: MCP_HOST_RUNTIME_TOKEN_SECRET_CONTROL_KEY,
},
},
},
{
name: 'MCP_HOST_WORKFLOW_CONTROL_TOKEN_FILE',
value: MCP_HOST_WORKFLOW_CONTROL_TOKEN_FILE_PATH,
},
{
name: 'MCP_HOST_RUNTIME_AUTH_STATE_DIR',
value: MCP_HOST_RUNTIME_AUTH_STATE_PATH,
},
{ name: 'MCP_HOST_GATEWAY_URL', value: config.mcpHostGatewayUrl },
]
// mcp-host refuses to construct the grok-subscription provider unless this
// switch is on. Emit it only when this Host's Grok projection mints
// llm:grok:execute (same gate as the scope and grok-proxy egress policy),
// so an eligibility flip changes the pod template and rolls the Host.
// Non-Grok Hosts keep a byte-identical template.
if (grokExecutionEnabled) {
env.push({ name: 'MCP_HOST_GROK_SUBSCRIPTION_ENABLED', value: 'true' })
}
// One storage layout for every lifecycle (issue #825): the conversation
// database lives at the workspace PVC root, so a lifecycle flip changes
// compute only and never where the history is read from.
env.push(
{ name: 'CLERUM_SESSION_STORE', value: 'sqlite' },
{ name: 'CLERUM_SESSION_DB_DIR', value: workspacePath }
)
if (isStateless) {
env.push(
{ name: 'CLERUM_STATELESS_LIFECYCLE', value: 'true' },
// Stage 3: the heartbeat emitter requires its own pod UID so HCC
// can discard heartbeats from stale pods after a wake.
{
name: 'CLERUM_POD_UID',
valueFrom: { fieldRef: { fieldPath: 'metadata.uid' } },
}
)
}
if (isDesktop) {
env.push(
{ name: 'CLERUM_DESKTOP_X11', value: String(host.spec.desktop?.x11 ?? false) },
{ name: 'CLERUM_DESKTOP_BROWSER', value: String(host.spec.desktop?.browser ?? false) },
{ name: 'PUID', value: '1001' },
{ name: 'PGID', value: '1001' },
{ name: 'TZ', value: 'UTC' },
{ name: 'CUSTOM_PORT', value: String(config.desktopPort) }
)
}
if (contextMounts.length > 0) {
// Sort by name for stable JSON ordering across reconciles — otherwise a
// map iteration reorder would flip the env value and trigger an unnecessary
// pod restart.
const sorted = [...contextMounts].sort((a, b) => a.name.localeCompare(b.name))
env.push({
name: 'CLERUM_CONTEXT_FILES_MOUNTS',
value: JSON.stringify(
sorted.map(m => ({
name: m.name,
namespace: m.namespace,
mountPath: m.mountPath,
pvcName: m.pvcName,
}))
),
})
}
const podAnnotations: Record<string, string> = {}
if (runtimeTokenRevision) {
podAnnotations[RUNTIME_TOKEN_REVISION_ANNOTATION] = runtimeTokenRevision
}
// Roll the mcp-host pod whenever the guardrails block changes. mcp-host does
// re-resolve guardrails live, but only while its Host watch is connected, so
// the roll is what delivers the change to an agent whose watch has lapsed.
// Stamp a hash of the block: present→changed flips the value, and removal
// omits the key — which only reaches the pod because
// preserveHostDeploymentAnnotations strips it from the merge instead of
// letting the live value survive (an omitted key alone produces no diff).
if (host.spec.guardrails) {
podAnnotations[GUARDRAILS_REVISION_ANNOTATION] = createHash('sha256')
.update(deepStableStringify(host.spec.guardrails))
.digest('hex')
}
// Security context — desktop needs runAsNonRoot: false for s6-overlay init,
// which is fundamentally incompatible with the `restricted` PodSecurity
// standard. Desktop Hosts therefore won't schedule in MCC shared-tenant
// namespaces under restricted PSA — a separate known limitation, out of
// scope here. Non-desktop hosts emit a restricted-PSA-compliant pod-level
// securityContext so the ReplicaSet schedules in restricted namespaces.
const securityContext = isDesktop
? { runAsNonRoot: false, seccompProfile: { type: 'RuntimeDefault' }, fsGroup: 1001 }
: {
runAsNonRoot: true,
// UID/GID 1001 matches the mcp-host image's baked-in `nodejs` user
// (Dockerfile / .slim / .full all create it at 1001). Running as the
// image user keeps the running UID aligned with the workspace PVC
// ownership written by prior rolls — re-rolling existing 1st-party
// hosts (chatllm, etc.) does not hit EACCES on workspace/memory file
// rewrites. fsGroup stays 1001 so fresh-PVC tenants still work.
runAsUser: 1001,
runAsGroup: 1001,
fsGroup: 1001,
seccompProfile: { type: 'RuntimeDefault' },
}
// Startup probe — desktop image needs longer initial delay for s6 boot + XFCE + mcp-host launch.
// Stateless wake-budget tuning (Stage 6, W7): a woken pod should turn
// Ready as soon as it can answer, so the stateless startup probe drops
// the initial delay and polls every 2s. The total startup allowance is
// preserved: non-stateless allows 10 + 5s × 60 = 310s, stateless allows
// 0 + 2s × 155 = 310s (stateless is never desktop — assessLifecycle
// rejects spec.desktop). Non-stateless pods keep the exact legacy probe.
const startupProbe = isStateless
? {
httpGet: { path: '/v1/runtime/live', port: 'http' as IntOrString },
initialDelaySeconds: 0,
periodSeconds: 2,
timeoutSeconds: 2,
failureThreshold: 155,
}
: {
httpGet: { path: '/v1/runtime/live', port: 'http' as IntOrString },
initialDelaySeconds: isDesktop ? 30 : 10,
periodSeconds: 5,
timeoutSeconds: 2,
failureThreshold: isDesktop ? 120 : 60,
}
return {
apiVersion: 'apps/v1',
kind: 'Deployment',
metadata: {
name: host.name,
namespace: host.namespace,
labels,
...(host.uid ? { annotations: { [HOST_UID_ANNOTATION]: host.uid } } : {}),
},
spec: {
// A suspended stateless Host scales to 0 on EVERY reconcile path
// (event + resync): the state lives in the CRD status, so neither a
// routine reconcile nor an HCC restart resurrects it.
replicas: isStateless && effectiveLifecycle.state === 'suspended' ? 0 : 1,
// Recreate: the old Pod is gone before the new one starts, so two
// Pods never share the RWO workspace PVC (a RollingUpdate with
// maxSurge 0 still starts the new Pod while the old one terminates).
strategy: { type: 'Recreate' },
selector: {
matchLabels: { app: host.name },
},
template: {
metadata: {
labels,
...(Object.keys(podAnnotations).length > 0 ? { annotations: podAnnotations } : {}),
},
spec: {
// Per-Host ServiceAccount provisioned by
// ensureHostServiceAccount() — bound to a narrow Role that lets
// mcp-host watch only its own Host CRD, env CM/Secret, and LLM
// Secret.
serviceAccountName: this.hostSaName(host),
// A stateless Host may be recreated after a user interaction.
// Give only that on-demand workload the interactive class so its
// wake can preempt explicitly lower-priority batch work. Keep
// stateful Hosts unclassed at the default service-equivalent
// priority to preserve their existing PodSpec.
...(isStateless ? { priorityClassName: 'clerum-interactive-host' } : {}),
imagePullSecrets: config.hostImagePullSecretName
? [{ name: config.hostImagePullSecretName }]
: undefined,
// While CONTEXT_MAPPER_WORKSPACE_LAYOUT_MERGE is on, every Host
// merges a PVC split by the earlier stateless layout back to the
// root layout before mcp-host starts; on any other tree the init
// exits 0 without touching it.
initContainers: config.workspaceLayoutMerge
? [
buildWorkspaceLayoutMergeContainer(
image,
imagePullPolicy,
hostContainerSecurityContext()
),
]
: undefined,
containers: [
{
name: 'mcp-host',
image,
imagePullPolicy,
ports,
envFrom: [{ configMapRef: { name: config.hostConfigMapName } }],
env,
volumeMounts: [
{ name: 'workspace', mountPath: workspacePath },
{
name: 'mcp-host-runtime-tokens',
mountPath: WORKFLOW_TOKEN_MOUNT_PATH,
readOnly: true,
},
{ name: 'workflow-auth-state', mountPath: MCP_HOST_RUNTIME_AUTH_STATE_PATH },
{ name: 'tmp', mountPath: '/tmp' },
...contextMounts.map(m => ({
name: HostReconciler.contextMountVolumeName(m),
mountPath: m.mountPath,
readOnly: true,
})),
],
startupProbe,
livenessProbe: {
httpGet: { path: '/v1/runtime/live', port: 'http' as IntOrString },
initialDelaySeconds: 45,
periodSeconds: 30,
timeoutSeconds: 2,
failureThreshold: 3,
},
readinessProbe: {
httpGet: { path: '/v1/runtime/health', port: 'http' as IntOrString },
// Stateless (Stage 6, W7): the startup probe already gates
// readiness checks until the pod is live, so the fixed 20s
// readiness delay only pads the wake budget — drop it.
initialDelaySeconds: isStateless ? 0 : 20,
periodSeconds: 10,
timeoutSeconds: 2,
failureThreshold: 6,
},
resources: {
requests: {
memory: resources.requests.memory,
cpu: resources.requests.cpu,
},
limits: {
memory: resources.limits.memory,
cpu: resources.limits.cpu,
},
},
securityContext: hostContainerSecurityContext(),
},
],
securityContext,
// #592: co-locate the mcp-host pod on the same node as each mounted
// SharedFileSystem's wfc pod. The SFS PVC is RWO, so the RW writer
// (wfc) and these RO consumers must share a node. Mounts are gated on
// the SFS PVC being Bound (resolveContextMounts → isMountable); under
// WaitForFirstConsumer a Bound PVC means the wfc (first consumer) was
// already scheduled, so its affinity target exists. Limit: a Host
// mounting 2+ SFS whose wfc landed on different nodes is unschedulable
// (surfaced as Degraded) — that case needs RWX (deferred) and is not
// used today (Contexts mount 0-1 SFS).
//
// LATENT (see issue #602 — RWX is not enabled on any cluster today):
// this podAffinity is UNCONDITIONAL — it applies to every mount
// regardless of access mode. If RWX is ever enabled and a Host mounts
// an explicit ReadWriteMany SFS, this `required` hostname affinity
// would still force its RO consumers onto the wfc's single node,
// defeating RWX's multi-node-reader benefit. It is harmless today: no
// RWX StorageClass exists on dev/prod, so an RWX PVC never binds →
// the isMountable (PVC-Bound) gate keeps it out of contextMounts → it
// never reaches this affinity. WHEN RWX lands: plumb the access mode
// into ResolvedSfsMount and apply this term only to RWO mounts (RWX
// mounts must get NO podAffinity). Do NOT add that conditioning now —
// it would be untestable without an RWX StorageClass. See #602.
...(contextMounts.length > 0
? {
affinity: {
podAffinity: {
requiredDuringSchedulingIgnoredDuringExecution: contextMounts.map(m => ({
labelSelector: {
matchLabels: {
app: WFC_APP_LABEL,
[SFS_LABEL]: m.name,
[SFS_NAMESPACE_LABEL]: m.namespace,
},
},
topologyKey: 'kubernetes.io/hostname',
})),
},
},
}
: {}),
volumes: [
{ name: 'workspace', persistentVolumeClaim: { claimName: this.pvcName(host) } },
{
name: 'mcp-host-runtime-tokens',
secret: { secretName: mcpHostRuntimeTokenSecretName(host) },
},
{ name: 'workflow-auth-state', emptyDir: { sizeLimit: '16Mi' } },
{ name: 'tmp', emptyDir: {} },
...contextMounts.map(m => ({
name: HostReconciler.contextMountVolumeName(m),
persistentVolumeClaim: { claimName: m.pvcName, readOnly: true },
})),
],
},
},
},
}
}
private buildService(host: HostCRD): k8s.V1Service {
const isDesktop = !!(host.spec.desktop?.browser || host.spec.desktop?.x11)
const ports: Array<{ name: string; port: number; targetPort: IntOrString; protocol: string }> =
[
{
name: 'http',
port: config.hostPort,
targetPort: 'http' as IntOrString,
protocol: 'TCP',
},
]
if (isDesktop) {
ports.push({
name: 'desktop',
port: config.desktopPort,
targetPort: 'desktop' as IntOrString,
protocol: 'TCP',
})
}
return {
apiVersion: 'v1',
kind: 'Service',
metadata: {
name: host.name,
namespace: host.namespace,
labels: {
app: host.name,
[MANAGED_BY_LABEL]: MANAGED_BY_VALUE,
[HOST_LABEL]: host.name,
},
},
spec: {
type: 'ClusterIP',
selector: { app: host.name },
ports,
},
}
}
private async checkDeploymentReady(name: string, namespace: string): Promise<boolean> {
try {
const deployment = await this.appsApi.readNamespacedDeployment({ name, namespace })
return (deployment.status?.readyReplicas ?? 0) > 0
} catch {
return false
}
}
/**
* Best-effort read of the per-Host channel-reader Deployment status.
* Never throws — a read failure returns a degraded status so the caller
* can surface it without blocking mcp-host reconcile.
*/
private async checkChannelReaderStatus(
hostName: string,
ccCount: number
): Promise<HostChannelReaderStatus> {
const depName = `channel-reader-${hostName}`
const ns = config.channelsNamespace
const expected = ccCount > 0
try {
const dep = await this.appsApi.readNamespacedDeployment({ name: depName, namespace: ns })
const readyReplicas = dep.status?.readyReplicas ?? 0
const desiredReplicas = dep.spec?.replicas ?? 0
const ready = readyReplicas >= 1
if (!expected) {
// No CCs — Deployment may legitimately be scaled to 0.
return {
expected: false,
ready: desiredReplicas > 0 && ready,
message:
desiredReplicas > 0 && ready
? 'Running (no CommunicationChannels)'
: 'Scaled to 0 (no CommunicationChannels)',
}
}
return {
expected: true,
ready,
message: ready ? 'Running' : 'Deployment exists but no ready replicas',
}
} catch (err) {
const code = getErrorCode(err)
if (code === 404) {
return {
expected,
ready: false,
message: expected ? 'Deployment not found' : 'Not deployed (no CommunicationChannels)',
}
}
// Unknown error — surface it but don't throw.
log.warn('Failed to read channel-reader Deployment for status', { deployment: depName, err })
return {
expected,
ready: false,
message: `Status read error: ${(err as Error).message}`,
}
}
}
private pollReadiness(
name: string,
namespace: string,
intervalMs = 5000,
maxAttempts = 12
): void {
const pollGeneration = this.reconcileGenerations.get(name) ?? 0
const existing = this.readinessTimers.get(name)
if (existing) {
clearTimeout(existing)
this.readinessTimers.delete(name)
}
let attempts = 0
const poll = async () => {
attempts++
const ready = await this.checkDeploymentReady(name, namespace)
if (ready) {
if (
(this.reconcileGenerations.get(name) ?? 0) !== pollGeneration ||
(this.statusGenerations.get(name) ?? 0) > pollGeneration
) {
this.readinessTimers.delete(name)
return
}
this.setStatus(name, { deployed: true, ready: true, message: 'Running' })
this.readinessTimers.delete(name)
return
}
if (attempts >= maxAttempts) {
this.readinessTimers.delete(name)
return
}
const timer = setTimeout(poll, intervalMs)
this.readinessTimers.set(name, timer)
}
const timer = setTimeout(poll, intervalMs)
this.readinessTimers.set(name, timer)
}
private async ensurePvc(host: HostCRD, revalidate?: () => void): Promise<boolean> {
const pvc = this.buildPvc(host)
const name = this.pvcName(host)
let createFailed = false
const mutationAllowed = () => {
revalidate?.()
return true
}
// Initial GET errors must propagate; retain only the established
// non-throwing POST and convergence failures of this PVC writer.
const result = await ensureResource({
mutationAllowed,
read: () =>
observeExistenceRead('PersistentVolumeClaim', () =>
this.coreApi.readNamespacedPersistentVolumeClaim({ namespace: host.namespace, name })
),
create: async () => {
try {
await observeCreate('PersistentVolumeClaim', () =>
this.coreApi.createNamespacedPersistentVolumeClaim({
namespace: host.namespace,
body: pvc,
})
)
} catch (error) {
if (error != null && getErrorCode(error) === 409) throw error
createFailed = true
log.error('Failed to create Host PVC', { host: host.name, err: error })
}
},
converge: async read => {
try {
const existing = await read()
revalidate?.()
if (existing.spec?.volumeName) return
pvc.metadata!.resourceVersion = existing.metadata?.resourceVersion
await this.coreApi.replaceNamespacedPersistentVolumeClaim({
namespace: host.namespace,
name,
body: pvc,
})
} catch (error) {
if (isBenignSupersessionError(error)) throw error
log.error('Failed to update Host PVC', { host: host.name, err: error })
return false
}
},
onSkipped: () => createsTotal.inc({ kind: 'PersistentVolumeClaim', outcome: 'skipped' }),
})
return !createFailed && this.resourceApplySucceeded(result)
}
private async ensureService(host: HostCRD, revalidate?: () => void): Promise<boolean> {
const service = this.buildService(host)
let createFailed = false
const mutationAllowed = () => {
revalidate?.()
return true
}
// Initial read errors propagate. Only the existing POST/convergence paths
// retain their historical non-throwing failures.
const result = await ensureResource({
mutationAllowed,
read: () =>
observeExistenceRead('Service', () =>
this.coreApi.readNamespacedService({ namespace: host.namespace, name: host.name })
),
create: async () => {
try {
await observeCreate('Service', () =>
this.coreApi.createNamespacedService({ namespace: host.namespace, body: service })
)
} catch (error) {
if (error != null && getErrorCode(error) === 409) throw error
createFailed = true
log.error('Failed to create Host Service', { host: host.name, err: error })
}
},
onSkipped: () => createsTotal.inc({ kind: 'Service', outcome: 'skipped' }),
converge: async read => {
try {
await replaceWithConflictRetry({
description: `Service "${host.name}"`,
logPrefix: '[HostReconciler]',
body: service,
mergeExisting: preserveServiceAssignedFields,
isUpToDate: serviceMatchesDesired,
mutationAllowed,
read,
replace: body =>
this.coreApi.replaceNamespacedService({
namespace: host.namespace,
name: host.name,
body,
}),
})
} catch (error) {
if (isBenignSupersessionError(error)) throw error
log.error('Failed to update Host Service', { host: host.name, err: error })
return false
}
},
})
return !createFailed && this.resourceApplySucceeded(result)
}
private resourceApplySucceeded(result: ResourceApplyResult): boolean {
return result === 'created' || result === 'replaced' || result === 'up_to_date'
}
private async ensureDeployment(
host: HostCRD,
mounts: ResolvedSfsMount[],
runtimeTokenRevision: string | undefined,
lifecycle?: EffectiveHostLifecycle,
resolveStateBeforeMutation?: (observed?: k8s.V1Deployment) => Promise<DeploymentMutationState>,
revalidate?: () => void
): Promise<boolean> {
let observedDeployment: k8s.V1Deployment | undefined
let holdingTemplate = false
let applied = true
const buildDesiredDeployment = async (): Promise<k8s.V1Deployment | null> => {
const state = resolveStateBeforeMutation
? await resolveStateBeforeMutation(observedDeployment)
: null
const effective = state?.lifecycle ?? lifecycle
const resourcesDiffer =
!!observedDeployment &&
HostReconciler.deploymentBelongsToHost(observedDeployment, host) &&
!HostReconciler.deploymentResourcesMatchHost(observedDeployment, host)
const suspendingWithResourceDrift =
effective?.stateless === true &&
effective.state === 'suspended' &&
effective.suspensionBlocked !== true &&
resourcesDiffer
holdingTemplate = effective?.suspensionBlocked === true || suspendingWithResourceDrift
if (holdingTemplate) {
if (!observedDeployment) return null
const existing = observedDeployment
if (
!HostReconciler.isHccOwnedHostResource(existing, host.name) ||
existing.metadata?.name !== host.name ||
existing.metadata?.namespace !== host.namespace ||
!existing.metadata?.uid ||
!existing.metadata?.resourceVersion ||
existing.metadata.deletionTimestamp ||
!existing.spec?.template?.spec
) {
throw new Error(`Cannot preserve an unverified Deployment for Host "${host.name}"`)
}
const annotatedHostUid = existing.metadata.annotations?.[HOST_UID_ANNOTATION]
const hostUidVerified = !!host.uid && annotatedHostUid === host.uid
if (!hostUidVerified) {
throw new Error(`Cannot preserve an unverified Deployment for Host "${host.name}"`)
}
// Preserve UID/resourceVersion and every applied field. A real suspend
// must still reach zero when resource policy changes; its next wake
// prepares credentials before applying that deferred template change.
// Conflicts re-read identity before retrying the replica-only mutation.
const replicas = suspendingWithResourceDrift
? 0
: effective?.allowScaleUpDuringHold
? 1
: (existing.spec.replicas ?? 1)
return {
...existing,
spec: { ...existing.spec, replicas },
}
}
if (
observedDeployment &&
resourcesDiffer &&
(observedDeployment.spec?.replicas ?? 1) > 0 &&
effective?.state === 'active' &&
state?.freshBootstrapPrepared !== true
) {
// Readiness cannot gate a resource repair that is needed for scheduling.
// Delivery is gated on persisted, fresh material for the observed target.
throw new RuntimeBootstrapPreparationUnavailableError()
}
const desired = this.buildDeployment(
host,
mounts,
state?.runtimeTokenRevision ?? runtimeTokenRevision,
effective,
state?.grokExecutionEnabled ?? this.hostDerivesGrokExecution(host)
)
if (observedDeployment && resourcesDiffer && effective?.state === 'draining') {
// Defer only resource policy while draining. An intrinsic credential
// recovery must still deliver its prepared runtime revision.
const appliedResources = observedDeployment.spec!.template.spec!.containers.find(
container => container.name === 'mcp-host'
)?.resources
const desiredContainer = desired.spec!.template.spec!.containers.find(
container => container.name === 'mcp-host'
)!
desiredContainer.resources = structuredClone(appliedResources)
}
return desired
}
const mutationAllowed = () => {
revalidate?.()
return true
}
const result = await ensureResource({
mutationAllowed,
read: async () => {
observedDeployment = await observeExistenceRead('Deployment', () =>
this.appsApi.readNamespacedDeployment({ namespace: host.namespace, name: host.name })
)
return observedDeployment
},
create: async () => {
// The initial GET may outlive the lifecycle or token-scope observation.
const deployment = await buildDesiredDeployment()
revalidate?.()
if (!deployment) {
applied = false
return
}
return observeCreate('Deployment', () =>
this.appsApi.createNamespacedDeployment({ namespace: host.namespace, body: deployment })
)
},
onSkipped: () => createsTotal.inc({ kind: 'Deployment', outcome: 'skipped' }),
converge: read =>
replaceWithConflictRetry({
description: `Deployment "${host.name}"`,
logPrefix: '[HostReconciler]',
resolveBody: async () => {
const desired = await buildDesiredDeployment()
if (!desired) throw new Error('Deployment disappeared during preservation')
return desired
},
mergeExisting: (desired, existing) =>
holdingTemplate ? desired : preserveHostDeploymentAnnotations(desired, existing),
isUpToDate: deploymentMatchesDesired,
mutationAllowed,
read,
replace: body =>
this.appsApi.replaceNamespacedDeployment({
namespace: host.namespace,
name: host.name,
body,
}),
}),
})
return applied && this.resourceApplySucceeded(result)
}
private async deleteRuntimeResources(name: string, namespace: string): Promise<void> {
const failures: unknown[] = []
// Desktop NetworkPolicy is created idempotently in ensureDesktopNetworkPolicy;
// attempt deletion unconditionally so non-desktop hosts (no policy) just hit 404.
// Kept for backward-compatible cleanup of policies created before rpc-proxy
// host egress became per-Host.
const desktopPolicyName = `allow-rpc-proxy-desktop-${name}`
for (const cleanup of [
() =>
this.deleteIfHccOwned(
'Deployment',
name,
namespace,
name,
() => this.appsApi.readNamespacedDeployment({ name, namespace }),
() => this.appsApi.deleteNamespacedDeployment({ name, namespace })
),
() =>
this.deleteIfHccOwned(
'Service',
name,
namespace,
name,
() => this.coreApi.readNamespacedService({ name, namespace }),
() => this.coreApi.deleteNamespacedService({ name, namespace })
),
() =>
this.deleteIfHccOwned(
'NetworkPolicy',
desktopPolicyName,
namespace,
name,
() =>
this.networkingApi.readNamespacedNetworkPolicy({ name: desktopPolicyName, namespace }),
() =>
this.networkingApi.deleteNamespacedNetworkPolicy({ name: desktopPolicyName, namespace })
),
]) {
try {
await cleanup()
} catch (error) {
failures.push(error)
}
}
throwCleanupFailures(
failures,
`Failed to delete runtime Kubernetes resources for Host "${name}"`
)
}
private async deleteWorkspacePvc(name: string, namespace: string): Promise<void> {
const pvcName = `${name}-workspace`
await this.deleteIfHccOwned(
'PersistentVolumeClaim',
pvcName,
namespace,
name,
() => this.coreApi.readNamespacedPersistentVolumeClaim({ name: pvcName, namespace }),
() => this.coreApi.deleteNamespacedPersistentVolumeClaim({ name: pvcName, namespace })
)
}
private async deleteResources(
name: string,
namespace: string,
options: { deleteWorkspacePvc?: boolean } = {}
): Promise<void> {
const failures: unknown[] = []
const cleanups = [() => this.deleteRuntimeResources(name, namespace)]
if (options.deleteWorkspacePvc ?? true) {
cleanups.push(() => this.deleteWorkspacePvc(name, namespace))
}
for (const cleanup of cleanups) {
try {
await cleanup()
} catch (error) {
failures.push(error)
}
}
throwCleanupFailures(failures, `Failed to delete Kubernetes resources for Host "${name}"`)
}
/**
* Cascade delete the per-Host channel-reader Deployment in the channels
* namespace. 404 is swallowed (already gone); non-404 errors abort cleanup
* so revocation retries instead of silently leaving a reader alive.
*/
private async deleteChannelReaderDeployment(name: string): Promise<void> {
const depName = `channel-reader-${name}`
await this.deleteIfHccOwned(
'Deployment',
depName,
config.channelsNamespace,
name,
() =>
this.appsApi.readNamespacedDeployment({
name: depName,
namespace: config.channelsNamespace,
}),
() =>
this.appsApi.deleteNamespacedDeployment({
name: depName,
namespace: config.channelsNamespace,
})
)
}
private async deleteChannelReaderService(name: string): Promise<void> {
const serviceName = `channel-reader-${name}`
await this.deleteIfHccOwned(
'Service',
serviceName,
config.channelsNamespace,
name,
() =>
this.coreApi.readNamespacedService({
name: serviceName,
namespace: config.channelsNamespace,
}),
() =>
this.coreApi.deleteNamespacedService({
name: serviceName,
namespace: config.channelsNamespace,
})
)
}
/**
* One-shot startup sweep: delete the legacy static `clerum-channel-reader`
* Deployment in the channels namespace if it still exists.
*
* The legacy static Deployment was the only channel-reader before the
* per-Host pattern landed. Per-Host Deployments now own that role
* (channel-reader-<host>). With both alive, the two pods compete on the
* same Telegram bot token's getUpdates long-poll and Telegram returns
* 409 Conflict to whichever loses — empirical evidence on minikube +
* example-dev is captured in #273.
*
* Idempotent on every HCC startup: 404 means already gone (the steady
* state we want), so we log nothing. A successful delete emits an
* `AUDIT:` line for operator monitoring. Other errors are logged but
* NOT thrown — startup continues so per-Host reconciles still run.
*
* Public so main.ts can call it once at boot, before the reconcile
* loop starts (preventing the brief window where both pods could
* poll concurrently).
*/
async sweepLegacyStaticChannelReader(): Promise<void> {
const depName = 'clerum-channel-reader'
const ns = config.channelsNamespace
try {
await this.appsApi.deleteNamespacedDeployment({ name: depName, namespace: ns })
// Structured audit line — operators tail this to confirm the legacy
// pod is gone post-deploy. Mirrors the format used by
// sweepOrphanChannelReaderResources for consistency.
log.info('AUDIT: legacy-sweep deleted Deployment', { name: depName, namespace: ns })
} catch (err) {
const code = getErrorCode(err)
if (code === 404) return // already gone — steady state, no log
log.error('legacy-sweep failed to delete Deployment', { name: depName, namespace: ns, err })
}
}
private async ensureDesktopNetworkPolicy(
host: HostCRD
): Promise<'not-required' | ResourceApplyResult> {
const isDesktop = !!(host.spec.desktop?.browser || host.spec.desktop?.x11)
if (!isDesktop) return 'not-required'
const policyName = `allow-rpc-proxy-desktop-${host.name}`
const policy: k8s.V1NetworkPolicy = {
apiVersion: 'networking.k8s.io/v1',
kind: 'NetworkPolicy',
metadata: {
name: policyName,
namespace: host.namespace,
labels: {
[MANAGED_BY_LABEL]: MANAGED_BY_VALUE,
[HOST_LABEL]: host.name,
'clerum.io/policy-type': 'desktop-ingress',
},
},
spec: {
podSelector: { matchLabels: { app: host.name } },
policyTypes: ['Ingress'],
ingress: [
{
_from: [
{
namespaceSelector: {
matchLabels: { 'kubernetes.io/metadata.name': config.rpcProxyNamespace },
},
podSelector: { matchLabels: { app: 'rpc-proxy' } },
},
],
ports: [{ port: config.desktopPort, protocol: 'TCP' }],
},
],
},
}
return applyNetworkPolicy(
this.networkingApi,
policyName,
host.namespace,
policy,
'[HostReconciler]'
)
}
/**
* Per-Host rpc-proxy egress policy. Pins rpc-proxy traffic to exactly one
* mcp-host pod instead of relying on namespace-wide rpc-proxy -> mcp-host
* egress. Desktop hosts also expose the desktop service port through the
* same host-scoped egress boundary.
*/
private async ensureRpcProxyHostEgressNetworkPolicy(host: HostCRD): Promise<ResourceApplyResult> {
const isDesktop = !!(host.spec.desktop?.browser || host.spec.desktop?.x11)
const policyName = `rpc-proxy-${host.name}-egress-mcp-host`
const ports: k8s.V1NetworkPolicyPort[] = [{ port: config.hostPort, protocol: 'TCP' }]
if (isDesktop) {
ports.push({ port: config.desktopPort, protocol: 'TCP' })
}
const policy: k8s.V1NetworkPolicy = {
apiVersion: 'networking.k8s.io/v1',
kind: 'NetworkPolicy',
metadata: {
name: policyName,
namespace: config.rpcProxyNamespace,
labels: {
[MANAGED_BY_LABEL]: MANAGED_BY_VALUE,
[HOST_LABEL]: host.name,
'clerum.io/policy-type': 'rpc-proxy-host-egress',
},
},
spec: {
podSelector: { matchLabels: { app: 'rpc-proxy' } },
policyTypes: ['Egress'],
egress: [
{
to: [
{
namespaceSelector: {
matchLabels: { 'kubernetes.io/metadata.name': host.namespace },
},
podSelector: {
matchLabels: {
[HOST_LABEL]: host.name,
[MANAGED_BY_LABEL]: MANAGED_BY_VALUE,
},
},
},
],
ports,
},
],
},
}
return applyNetworkPolicy(
this.networkingApi,
policyName,
config.rpcProxyNamespace,
policy,
'[HostReconciler]'
)
}
/**
* Per-Host channel-reader egress policy. Pins channel-reader-<host> pods to
* its bound mcp-host on :8080. Provider approvals and verification continue
* through mcp-host; channel-reader must not reach control-api gateways.
*/
private async ensureChannelReaderEgressNetworkPolicy(
host: HostCRD
): Promise<ResourceApplyResult> {
const policyName = `channel-reader-${host.name}-egress`
const policy: k8s.V1NetworkPolicy = {
apiVersion: 'networking.k8s.io/v1',
kind: 'NetworkPolicy',
metadata: {
name: policyName,
namespace: config.channelsNamespace,
labels: {
[MANAGED_BY_LABEL]: MANAGED_BY_VALUE,
[HOST_LABEL]: host.name,
'clerum.io/policy-type': 'channel-reader-egress',
},
},
spec: {
podSelector: {
matchLabels: {
app: 'channel-reader',
[HOST_LABEL]: host.name,
[MANAGED_BY_LABEL]: MANAGED_BY_VALUE,
},
},
policyTypes: ['Egress'],
egress: [
{
to: [
{
namespaceSelector: {
matchLabels: { 'kubernetes.io/metadata.name': config.hostNamespace },
},
podSelector: {
matchLabels: { [HOST_LABEL]: host.name },
},
},
],
ports: [{ port: config.hostPort, protocol: 'TCP' }],
},
],
},
}
// Re-throw on failure: per-Host NPs are one half of the
// channel-reader security boundary. Route-level JWT auth is the other
// half, so an NP apply error still MUST surface in HostRuntimeStatus
// rather than be silently swallowed.
return applyNetworkPolicy(
this.networkingApi,
policyName,
config.channelsNamespace,
policy,
'[HostReconciler]'
)
}
/**
* Per-Host workflow-approval-reader egress policy. Slack interactive callbacks
* are verified by control-api against a CommunicationChannel, then forwarded
* to that channel's bound Host for the authoritative provider-decision path.
*/
private async ensureWorkflowApprovalReaderHostEgressNetworkPolicy(
host: HostCRD
): Promise<ResourceApplyResult> {
const policyName = `workflow-approval-reader-${host.name}-egress-mcp-host`
const policy: k8s.V1NetworkPolicy = {
apiVersion: 'networking.k8s.io/v1',
kind: 'NetworkPolicy',
metadata: {
name: policyName,
namespace: config.channelsNamespace,
labels: {
[MANAGED_BY_LABEL]: MANAGED_BY_VALUE,
[HOST_LABEL]: host.name,
'clerum.io/policy-type': 'workflow-approval-reader-host-egress',
},
},
spec: {
podSelector: {
matchLabels: { 'app.kubernetes.io/name': 'workflow-approval-request-reader' },
},
policyTypes: ['Egress'],
egress: [
{
to: [
{
namespaceSelector: {
matchLabels: { 'kubernetes.io/metadata.name': host.namespace },
},
podSelector: {
matchLabels: {
[HOST_LABEL]: host.name,
[MANAGED_BY_LABEL]: MANAGED_BY_VALUE,
},
},
},
],
ports: [{ port: config.hostPort, protocol: 'TCP' }],
},
],
},
}
return applyNetworkPolicy(
this.networkingApi,
policyName,
config.channelsNamespace,
policy,
'[HostReconciler]'
)
}
/**
* Per-Host mcp-host ingress policy for channel-reader. Pins mcp-host-<host>
* pod ingress on :8080 to ONLY accept connections from channel-reader-<host>
* pods (label match).
*/
private async ensureMcpHostIngressNetworkPolicy(host: HostCRD): Promise<ResourceApplyResult> {
const policyName = `mcp-host-${host.name}-ingress-channel-reader`
const policy: k8s.V1NetworkPolicy = {
apiVersion: 'networking.k8s.io/v1',
kind: 'NetworkPolicy',
metadata: {
name: policyName,
namespace: host.namespace,
labels: {
[MANAGED_BY_LABEL]: MANAGED_BY_VALUE,
[HOST_LABEL]: host.name,
'clerum.io/policy-type': 'channel-reader-ingress',
},
},
spec: {
podSelector: {
matchLabels: {
[HOST_LABEL]: host.name,
[MANAGED_BY_LABEL]: MANAGED_BY_VALUE,
},
},
policyTypes: ['Ingress'],
ingress: [
{
_from: [
{
namespaceSelector: {
matchLabels: { 'kubernetes.io/metadata.name': config.channelsNamespace },
},
podSelector: {
matchLabels: {
app: 'channel-reader',
[HOST_LABEL]: host.name,
[MANAGED_BY_LABEL]: MANAGED_BY_VALUE,
},
},
},
],
ports: [{ port: config.hostPort, protocol: 'TCP' }],
},
],
},
}
// Re-throw on failure: per-Host NPs are one half of the
// channel-reader↔mcp-host security boundary. Route-level JWT auth is
// the other half, so an apply error MUST surface in HostRuntimeStatus
// rather than be silently swallowed.
return applyNetworkPolicy(
this.networkingApi,
policyName,
host.namespace,
policy,
'[HostReconciler]'
)
}
/**
* Per-Host mcp-host ingress policy for workflow-approval-reader. This is the
* provider webhook counterpart to channel-reader ingress and stays pinned to
* the Host resolved from the verified CommunicationChannel target.
*/
private async ensureWorkflowApprovalReaderMcpHostIngressNetworkPolicy(
host: HostCRD
): Promise<ResourceApplyResult> {
const policyName = `mcp-host-${host.name}-ingress-workflow-approval-reader`
const policy: k8s.V1NetworkPolicy = {
apiVersion: 'networking.k8s.io/v1',
kind: 'NetworkPolicy',
metadata: {
name: policyName,
namespace: host.namespace,
labels: {
[MANAGED_BY_LABEL]: MANAGED_BY_VALUE,
[HOST_LABEL]: host.name,
'clerum.io/policy-type': 'workflow-approval-reader-host-ingress',
},
},
spec: {
podSelector: {
matchLabels: {
[HOST_LABEL]: host.name,
[MANAGED_BY_LABEL]: MANAGED_BY_VALUE,
},
},
policyTypes: ['Ingress'],
ingress: [
{
_from: [
{
namespaceSelector: {
matchLabels: { 'kubernetes.io/metadata.name': config.channelsNamespace },
},
podSelector: {
matchLabels: {
'app.kubernetes.io/name': 'workflow-approval-request-reader',
},
},
},
],
ports: [{ port: config.hostPort, protocol: 'TCP' }],
},
],
},
}
return applyNetworkPolicy(
this.networkingApi,
policyName,
host.namespace,
policy,
'[HostReconciler]'
)
}
/**
* Per-Host mcp-host ingress policy for rpc-proxy. This replaces the legacy
* namespace-wide mcp-host ingress rule for rpc-proxy. Desktop hosts include
* the desktop port, but only for the selected Host pod.
*/
private async ensureRpcProxyMcpHostIngressNetworkPolicy(
host: HostCRD
): Promise<ResourceApplyResult> {
const isDesktop = !!(host.spec.desktop?.browser || host.spec.desktop?.x11)
const policyName = `mcp-host-${host.name}-ingress-rpc-proxy`
const ports: k8s.V1NetworkPolicyPort[] = [{ port: config.hostPort, protocol: 'TCP' }]
if (isDesktop) {
ports.push({ port: config.desktopPort, protocol: 'TCP' })
}
const policy: k8s.V1NetworkPolicy = {
apiVersion: 'networking.k8s.io/v1',
kind: 'NetworkPolicy',
metadata: {
name: policyName,
namespace: host.namespace,
labels: {
[MANAGED_BY_LABEL]: MANAGED_BY_VALUE,
[HOST_LABEL]: host.name,
'clerum.io/policy-type': 'rpc-proxy-host-ingress',
},
},
spec: {
podSelector: {
matchLabels: {
[HOST_LABEL]: host.name,
[MANAGED_BY_LABEL]: MANAGED_BY_VALUE,
},
},
policyTypes: ['Ingress'],
ingress: [
{
_from: [
{
namespaceSelector: {
matchLabels: { 'kubernetes.io/metadata.name': config.rpcProxyNamespace },
},
podSelector: { matchLabels: { app: 'rpc-proxy' } },
},
],
ports,
},
],
},
}
return applyNetworkPolicy(
this.networkingApi,
policyName,
host.namespace,
policy,
'[HostReconciler]'
)
}
/**
* Per-Host mcp-host egress to gfsc. The runtime token grants authority; this
* NetworkPolicy only opens the required transport lane from the selected Host
* pod to the governed GFS controller service.
*/
private async ensureMcpHostGfsEgressNetworkPolicy(host: HostCRD): Promise<ResourceApplyResult> {
const target = { namespace: config.gfsNamespace, port: config.gfscPort }
const policyName = `mcp-host-${host.name}-egress-gfs`
const policy: k8s.V1NetworkPolicy = {
apiVersion: 'networking.k8s.io/v1',
kind: 'NetworkPolicy',
metadata: {
name: policyName,
namespace: host.namespace,
labels: {
[MANAGED_BY_LABEL]: MANAGED_BY_VALUE,
[HOST_LABEL]: host.name,
'clerum.io/policy-type': 'gfs-egress',
},
},
spec: {
podSelector: {
matchLabels: {
[HOST_LABEL]: host.name,
[MANAGED_BY_LABEL]: MANAGED_BY_VALUE,
},
},
policyTypes: ['Egress'],
egress: [
{
to: [
{
namespaceSelector: {
matchLabels: { 'kubernetes.io/metadata.name': target.namespace },
},
podSelector: {
matchLabels: { app: 'gfs-controller' },
},
},
],
ports: [{ port: target.port, protocol: 'TCP' }],
},
],
},
}
return applyNetworkPolicy(
this.networkingApi,
policyName,
host.namespace,
policy,
'[HostReconciler]'
)
}
/**
* Per-Host mcp-host egress to the static Codex LLM proxy runtime listener.
* Derived from the same Codex projection that mints `llm:codex:execute`.
*/
private async ensureMcpHostCodexProxyEgressNetworkPolicy(
host: HostCRD
): Promise<ResourceApplyResult> {
const policyName = `mcp-host-${host.name}-egress-codex-proxy`
const policy: k8s.V1NetworkPolicy = {
apiVersion: 'networking.k8s.io/v1',
kind: 'NetworkPolicy',
metadata: {
name: policyName,
namespace: host.namespace,
labels: {
[MANAGED_BY_LABEL]: MANAGED_BY_VALUE,
[HOST_LABEL]: host.name,
'clerum.io/policy-type': 'codex-proxy-egress',
},
},
spec: {
podSelector: {
matchLabels: {
[HOST_LABEL]: host.name,
[MANAGED_BY_LABEL]: MANAGED_BY_VALUE,
},
},
policyTypes: ['Egress'],
egress: [
{
to: [
{
namespaceSelector: {
matchLabels: { 'kubernetes.io/metadata.name': config.controlPlaneNamespace },
},
podSelector: {
matchLabels: { app: 'codex-llm-proxy' },
},
},
],
ports: [{ port: 8080, protocol: 'TCP' }],
},
],
},
}
return applyNetworkPolicy(
this.networkingApi,
policyName,
host.namespace,
policy,
'[HostReconciler]'
)
}
private async deleteMcpHostCodexProxyEgressNetworkPolicy(host: HostCRD): Promise<void> {
const policyName = `mcp-host-${host.name}-egress-codex-proxy`
await this.deleteIfHccOwned(
'NetworkPolicy',
policyName,
host.namespace,
host.name,
() =>
this.networkingApi.readNamespacedNetworkPolicy({
name: policyName,
namespace: host.namespace,
}),
() =>
this.networkingApi.deleteNamespacedNetworkPolicy({
name: policyName,
namespace: host.namespace,
})
)
}
private async reconcileMcpHostCodexProxyEgressNetworkPolicy(
host: HostCRD
): Promise<'not-required' | ResourceApplyResult> {
const projection = this.projectCodexForHost(host)
if (projection.eligibility === 'uncertain') return 'not-required'
if (projection.requiresCodexProxyEgress) {
return this.ensureMcpHostCodexProxyEgressNetworkPolicy(host)
}
await this.deleteMcpHostCodexProxyEgressNetworkPolicy(host)
return 'not-required'
}
private async ensureMcpHostGrokProxyEgressNetworkPolicy(
host: HostCRD
): Promise<ResourceApplyResult> {
const policyName = `mcp-host-${host.name}-egress-grok-proxy`
const policy: k8s.V1NetworkPolicy = {
apiVersion: 'networking.k8s.io/v1',
kind: 'NetworkPolicy',
metadata: {
name: policyName,
namespace: host.namespace,
labels: {
[MANAGED_BY_LABEL]: MANAGED_BY_VALUE,
[HOST_LABEL]: host.name,
'clerum.io/policy-type': 'grok-proxy-egress',
},
},
spec: {
podSelector: {
matchLabels: {
[HOST_LABEL]: host.name,
[MANAGED_BY_LABEL]: MANAGED_BY_VALUE,
},
},
policyTypes: ['Egress'],
egress: [
{
to: [
{
namespaceSelector: {
matchLabels: { 'kubernetes.io/metadata.name': config.controlPlaneNamespace },
},
podSelector: {
matchLabels: { app: 'grok-llm-proxy' },
},
},
],
ports: [{ port: 8080, protocol: 'TCP' }],
},
],
},
}
return applyNetworkPolicy(
this.networkingApi,
policyName,
host.namespace,
policy,
'[HostReconciler]'
)
}
private async deleteMcpHostGrokProxyEgressNetworkPolicy(host: HostCRD): Promise<void> {
const policyName = `mcp-host-${host.name}-egress-grok-proxy`
await this.deleteIfHccOwned(
'NetworkPolicy',
policyName,
host.namespace,
host.name,
() =>
this.networkingApi.readNamespacedNetworkPolicy({
name: policyName,
namespace: host.namespace,
}),
() =>
this.networkingApi.deleteNamespacedNetworkPolicy({
name: policyName,
namespace: host.namespace,
})
)
}
private async reconcileMcpHostGrokProxyEgressNetworkPolicy(
host: HostCRD
): Promise<'not-required' | ResourceApplyResult> {
const projection = this.projectGrokForHost(host)
if (projection.eligibility === 'uncertain') return 'not-required'
if (projection.requiresGrokProxyEgress) {
return this.ensureMcpHostGrokProxyEgressNetworkPolicy(host)
}
await this.deleteMcpHostGrokProxyEgressNetworkPolicy(host)
return 'not-required'
}
/**
* Delete per-Host NetworkPolicies created by this reconciler.
* 404-tolerant — safe to call even if NPs were never created or already gone.
*/
private async deleteHostNetworkPolicies(name: string, namespace: string): Promise<void> {
const failures: unknown[] = []
const policies = [
{ name: `mcp-host-${name}-ingress-channel-reader`, namespace },
{ name: `mcp-host-${name}-ingress-workflow-approval-reader`, namespace },
{ name: `mcp-host-${name}-ingress-rpc-proxy`, namespace },
{ name: `mcp-host-${name}-egress-gfs`, namespace },
{ name: `mcp-host-${name}-egress-codex-proxy`, namespace },
{ name: `mcp-host-${name}-egress-grok-proxy`, namespace },
{ name: `mcp-host-${name}-egress-llm-hooks`, namespace },
{ name: `channel-reader-${name}-egress`, namespace: config.channelsNamespace },
{
name: `workflow-approval-reader-${name}-egress-mcp-host`,
namespace: config.channelsNamespace,
},
{ name: `rpc-proxy-${name}-egress-mcp-host`, namespace: config.rpcProxyNamespace },
]
for (const policy of policies) {
try {
await this.deleteIfHccOwned(
'NetworkPolicy',
policy.name,
policy.namespace,
name,
() =>
this.networkingApi.readNamespacedNetworkPolicy({
name: policy.name,
namespace: policy.namespace,
}),
() =>
this.networkingApi.deleteNamespacedNetworkPolicy({
name: policy.name,
namespace: policy.namespace,
})
)
} catch (error) {
failures.push(error)
}
}
throwCleanupFailures(failures, `Failed to delete NetworkPolicies for Host "${name}"`)
}
private async deleteHostRuntimeResources(
name: string,
namespace: string,
options: { deleteWorkspacePvc?: boolean } = {}
): Promise<void> {
const failures: unknown[] = []
for (const cleanup of [
() => this.deleteResources(name, namespace, options),
() => this.deleteHostRbac(name, namespace),
() => this.deleteMcpHostRuntimeTokenSecret(name, namespace),
() => this.deleteLegacyChannelReaderRuntimeAuth(name),
() => this.deleteChannelReaderService(name),
() => this.deleteChannelReaderDeployment(name),
() => this.deleteHostNetworkPolicies(name, namespace),
]) {
try {
await cleanup()
} catch (error) {
failures.push(error)
}
}
throwCleanupFailures(failures, `Failed to delete runtime resources for Host "${name}"`)
}
async reconcile(host: HostCRD, source: HostReconcileSource = 'urgent'): Promise<void> {
const dispatchedAt = Date.now()
const capturedAuthority = this.captureHostMutationAuthority()
const capturedDependencies = this.resolveHostMutationDependencies?.(host)
return this.lifecycle.serializeByHost(host.name, async () => {
this.advanceReconcileGeneration(host.name)
this.requireHostMutationAuthority(`Host "${host.name}" reconcile`, capturedAuthority)
const admittedHost = this.resolveCurrentHost ? this.resolveCurrentHost(host.name) : host
if (!admittedHost) return
const revalidate = this.makeHostMutationAdmission(
`Host "${host.name}" reconcile`,
admittedHost,
capturedAuthority,
capturedDependencies
)
const admittedAt = Date.now()
hostReconcileInFlight.inc({ lane: source })
try {
await this.reconcileCore(admittedHost, revalidate)
this.enqueueReconcileOutcome(admittedHost)
this.observeReconcileLatency(source, 'success', dispatchedAt, admittedAt)
} catch (error) {
if (isBenignSupersessionError(error)) {
// A newer spec/uid/authority-generation superseded this pass mid-flight;
// a fresh reconcile runs after. Model it as a withdrawal (superseded),
// NOT a hard failure — do NOT emit controller_error/reconcile_exception
// or an administrative 'failed' outcome. Callers already treat the
// rethrow as a retire (reconcileDelete → 'superseded', watch callers
// only log the error), so the rethrow is preserved.
// This exit emits no outcome, so drop the GFS evidence this pass
// recorded: a leftover would be reported by the next pass over this
// object. The clear is scoped to this snapshot's uid, so a sibling
// object at the same name keeps its own evidence. The stateless
// suspension path also calls reconcileCore without emitting an
// outcome and does not clear the entry (#696).
this.clearGfsLifecycleEvidenceForObject(admittedHost)
this.observeReconcileLatency(source, 'superseded', dispatchedAt, admittedAt)
throw error
}
this.enqueueControllerError(admittedHost, 'reconcile_exception', error)
this.enqueueReconcileExceptionOutcome(admittedHost)
this.observeReconcileLatency(source, 'error', dispatchedAt, admittedAt)
throw error
} finally {
hostReconcileInFlight.dec({ lane: source })
}
})
}
private observeReconcileLatency(
source: HostReconcileSource,
outcome: 'success' | 'error' | 'superseded',
dispatchedAt: number,
admittedAt: number
): void {
hostReconcileQueueWaitSeconds.observe(
{ lane: source, outcome },
Math.max(0, admittedAt - dispatchedAt) / 1000
)
hostReconcileDurationSeconds.observe(
{ source, outcome },
Math.max(0, Date.now() - admittedAt) / 1000
)
}
private async reconcileCore(host: HostCRD, revalidate?: () => HostCRD): Promise<void> {
// A per-Host serializer orders work, but it does not invalidate an older
// reconcile when the watch cache observes a newer spec while this body is
// awaiting Kubernetes or issuer I/O. Re-check the captured authority,
// identity, and spec generation at every material mutation boundary.
// Individual API writes remain atomic at the Kubernetes resource level;
// these checks prevent subsequent stale writes from spanning an observed
// generation change.
const revalidateHostMutationBoundary = (): void => {
revalidate?.()
}
revalidateHostMutationBoundary()
// Wake fast-path (Stage 4.3) BEFORE the heavy reconcile body: reconciles
// are serialized PER HOST (serializeByHost), so a pending wake for THIS
// Host must not wait behind token issuance, NetworkPolicies or the
// channel-reader work below in this same chain. Other Hosts no longer gate
// it — the process-wide convergence tail was removed. The periodic resync
// funnels through this same method, so a watch event dropped on disconnect
// is recovered here.
const forceFreshForWake = (await this.lifecycle.handleWakeFastPath(host)) === true
revalidateHostMutationBoundary()
await this.refreshCodexSnapshot()
revalidateHostMutationBoundary()
// Track whether this host has desktop enabled
const isDesktop = !!(host.spec.desktop?.browser || host.spec.desktop?.x11)
if (isDesktop) {
this.desktopHosts.add(host.name)
} else {
this.desktopHosts.delete(host.name)
}
const secretResult = await this.validateHostSecret(host)
revalidateHostMutationBoundary()
if (!secretResult.ok) {
if (
secretResult.reason === 'SecretNotFound' ||
secretResult.reason === 'SecretAccessDenied'
) {
// This branch deletes every ephemeral runtime resource. The Secret
// decision may have blocked on the API while Host watch authority or
// the Host UID/spec generation changed. Revalidate at the destructive
// commit boundary so a stale secretRef can never tear down the current
// runtime; the recovered/current Host reconcile will decide again.
revalidateHostMutationBoundary()
await this.deleteHostRuntimeResources(host.name, host.namespace, {
deleteWorkspacePvc: false,
})
} else {
log.warn('Preserving existing runtime after transient Secret read failure', {
host: host.name,
})
}
this.setStatus(host.name, {
deployed: false,
ready: false,
message: secretResult.message,
})
this.enqueueControllerError(host, secretResult.reason, new Error(secretResult.message))
return
}
// Per-Host SA + Role + RoleBinding must exist BEFORE the Deployment,
// otherwise the kubelet can't mount the SA token and the pod will
// crash-loop until RBAC is created.
revalidateHostMutationBoundary()
await this.ensureHostServiceAccount(host, revalidateHostMutationBoundary)
revalidateHostMutationBoundary()
await this.ensureHostRole(host, revalidateHostMutationBoundary)
revalidateHostMutationBoundary()
await this.ensureHostRoleBinding(host, revalidateHostMutationBoundary)
let mounts: ResolvedSfsMount[] = []
try {
mounts = await this.resolveContextMounts(host)
} catch (err) {
// A failure to resolve mounts should not block the rest of the Host
// reconciliation — the pod will still come up without the SFS mounts,
// and a subsequent reconcile (Context update or SFS becoming Ready)
// will inject them.
log.error('Failed to resolve context mounts', { host: host.name, err })
}
revalidateHostMutationBoundary()
// Stateless lifecycle (Stage 2): assess enable/reject and persist the
// durable state to the Host status subresource BEFORE building the
// Deployment, so replicas derive from the same assessment.
let lifecycle = await this.lifecycle.assessLifecycle(host, mounts)
revalidateHostMutationBoundary()
// Drained-pre-scale guard (Stage 4.3): a pending wake must abort the
// suspension IMMEDIATELY before replicas:0 derives from this assessment
// — see StatelessLifecycleExecutor.resolveWakeBeforeScaleDown. A null
// result means the fresh guard read failed: skip the scale-down this
// pass (the periodic resync retries) rather than scaling to 0 on stale
// data.
const guardedLifecycle = await this.lifecycle.resolveWakeBeforeScaleDown(host, lifecycle)
revalidateHostMutationBoundary()
if (guardedLifecycle === null) {
return
}
lifecycle = guardedLifecycle
revalidateHostMutationBoundary()
const lifecycleStatusCommitted = await this.lifecycle.writeLifecycleStatusToCluster(
host,
lifecycle
)
revalidateHostMutationBoundary()
if (lifecycleStatusCommitted) {
this.enqueueHostTelemetry(host, 'lifecycle_transition', lifecycle.effective.state, {
status: 'committed',
transition: `lifecycle:${lifecycle.effective.state}`,
state: lifecycle.effective.state,
phase: lifecycle.effective.stateless ? 'stateless' : 'stateful',
summary: lifecycle.lifecycle.reason,
})
}
let pvcApplied = true
let serviceApplied = true
const resourceErrors: unknown[] = []
const npFailures: string[] = []
const ensureIndependentResource = async (
kind: 'Host PVC' | 'Host Service',
reconcile: () => Promise<boolean>
): Promise<boolean> => {
try {
return await reconcile()
} catch (error) {
if (isBenignSupersessionError(error)) throw error
resourceErrors.push(error)
log.error('Failed to reconcile Host resource', {
host: host.name,
resource: kind,
err: error,
})
return false
}
}
const holdAppliedRuntime = async (wakeRequested: boolean): Promise<void> => {
const liveDeployment = await this.readHostDeploymentOrNull(host)
// A legacy Deployment without a runtime-token-revision annotation is not
// a preserved wake target, so it stays at zero replicas until channel
// authority returns and the normal path rewrites its template.
const appliedHostRuntime =
HostReconciler.deploymentBelongsToHost(liveDeployment, host) &&
HostReconciler.deploymentRuntimeTokenRevision(liveDeployment) !== ''
const liveReplicas = liveDeployment?.spec?.replicas ?? 1
const preservedWakeTarget = appliedHostRuntime && liveReplicas === 0
// A suspension recorded in status whose scale-down never reached the
// Deployment leaves the applied runtime running. A wake on it needs no
// mint and no replica change: it is held like any running runtime.
const wakeOnRunningRuntime = wakeRequested && appliedHostRuntime && liveReplicas > 0
const runtimeIsReady =
!!liveDeployment &&
HostReconciler.deploymentBelongsToHost(liveDeployment, host) &&
HostReconciler.deploymentReady(liveDeployment)
const scaleUpForRecovery = lifecycle.effective.state === 'active' && preservedWakeTarget
const allowScaleUp = preservedWakeTarget && (wakeRequested || scaleUpForRecovery)
if (
!liveDeployment ||
((wakeRequested || scaleUpForRecovery) && !preservedWakeTarget && !wakeOnRunningRuntime)
) {
this.setStatus(host.name, {
deployed: !!liveDeployment,
ready: false,
message:
'Waiting for a verified applied runtime before waking during CommunicationChannel inventory loss',
})
return
}
let provisioningRequired = false
if (allowScaleUp) {
// A pod starting from zero will consume bootstrap credentials. Mint and
// persist them before the replica-only update; a failed mint aborts here.
provisioningRequired = true
} else if (
(lifecycle.effective.state === 'active' || wakeOnRunningRuntime) &&
runtimeIsReady
) {
let provision: RuntimeTokenProvision | null
try {
provision = await this.provisionRuntimeTokenRevision(host, {
targetSuspended: false,
refreshGfsOnly: true,
})
} catch (error) {
// The Ready runtime keeps serving on the credential it holds; a failed
// renewal is reported, not turned into an undeployed Host. The next
// hold pass renews again.
log.warn('held runtime credential renewal failed; keeping the Ready runtime', {
host: host.name,
namespace: host.namespace,
err: error instanceof Error ? error.message : String(error),
})
// Telemetry only: this pass still ends with the runtime deployed and
// Ready, so the administrative outcome it reports is `succeeded`. A
// `failed` outcome here would give one intent both outcomes.
this.enqueueHostTelemetry(host, 'controller_error', 'RuntimeCredentialRenewalFailed', {
status: 'failed',
error_class: error instanceof Error ? error.name : typeof error,
})
this.setStatus(host.name, {
deployed: true,
ready: runtimeIsReady,
message: 'Held runtime credential renewal failed; runtime kept',
})
return
}
if (!provision) {
this.setStatus(host.name, {
deployed: true,
ready: false,
message: 'Held runtime credential renewal is not safely available',
})
return
}
revalidateHostMutationBoundary()
}
if (provisioningRequired) {
const provision = await this.provisionRuntimeTokenRevision(host, {
forceFreshForWake: true,
// This mint exists only to scale the Host up from zero, so the
// target is never suspended.
targetSuspended: false,
preserveDeploymentTemplateOnWake: true,
})
if (!provision) {
this.setStatus(host.name, {
deployed: !!liveDeployment,
ready: false,
message: 'Waiting for authoritative scope observation',
})
return
}
revalidateHostMutationBoundary()
}
const applied = await this.ensureDeployment(
host,
mounts,
undefined,
{ ...lifecycle.effective, allowScaleUpDuringHold: allowScaleUp },
undefined,
revalidateHostMutationBoundary
)
revalidateHostMutationBoundary()
const runtimeExpectedToRun =
lifecycle.effective.state !== 'suspended' || allowScaleUp || wakeOnRunningRuntime
if (runtimeExpectedToRun) {
this.lifecycle.markHostNotSuspended(host.name)
}
const deploymentReady =
(lifecycle.effective.state !== 'suspended' || wakeOnRunningRuntime) &&
applied &&
(await this.checkDeploymentReady(host.name, host.namespace))
revalidateHostMutationBoundary()
const ready = deploymentReady && pvcApplied && serviceApplied && npFailures.length === 0
// PVC/Service read errors were already thrown before the hold; only
// non-throwing convergence failures reach this point.
const failures = [
...(!pvcApplied ? ['Host PVC did not converge'] : []),
...(!serviceApplied ? ['Host Service did not converge'] : []),
...npFailures,
]
this.advanceReconcileGeneration(host.name)
const timer = this.readinessTimers.get(host.name)
if (timer) {
clearTimeout(timer)
this.readinessTimers.delete(host.name)
}
this.setStatus(host.name, {
deployed: applied,
ready,
message: !applied
? 'Waiting for CommunicationChannel inventory before creating runtime'
: failures.length > 0
? `degraded — Host runtime boundary incomplete while preserving applied runtime (${failures.join('; ')})`
: !runtimeExpectedToRun
? 'CommunicationChannel inventory unavailable; preserving suspended Host replicas'
: deploymentReady
? 'CommunicationChannel inventory unavailable; preserving applied runtime'
: 'CommunicationChannel inventory unavailable; preserved applied runtime is not Ready',
})
// Mirror the normal path: a runtime expected to run (including a held
// wake from zero) is polled until Ready; a degraded boundary is not,
// because the poll would overwrite the degraded verdict with "Running".
if (applied && !deploymentReady && failures.length === 0 && runtimeExpectedToRun) {
this.pollReadiness(host.name, host.namespace)
}
}
revalidateHostMutationBoundary()
pvcApplied = await ensureIndependentResource('Host PVC', () =>
this.ensurePvc(host, revalidateHostMutationBoundary)
)
revalidateHostMutationBoundary()
serviceApplied = await ensureIndependentResource('Host Service', () =>
this.ensureService(host, revalidateHostMutationBoundary)
)
// NetworkPolicies before Deployments. Calico/Cilium evaluate egress and
// ingress against the policies that exist when the connection is opened,
// so any pod created before its policy comes up runs briefly with the
// namespace's deny-all default. The channel-reader path is fail-closed
// (deny-all blocks message delivery), but creating the policy first
// makes rollout deterministic and avoids transient readiness failures.
//
// The per-Host NPs are the network boundary for this direct runtime hop.
// Route-level edge context and ownership checks remain the application
// boundary. An NP apply failure means the network boundary is missing, so
// surface it via HostRuntimeStatus instead of swallowing it.
const ensureNetworkPolicy = async (
name: string,
reconcile: () => Promise<'not-required' | ResourceApplyResult>
): Promise<void> => {
try {
revalidateHostMutationBoundary()
const result = await reconcile()
revalidateHostMutationBoundary()
if (result !== 'not-required' && !this.resourceApplySucceeded(result)) {
throw new Error(`apply did not converge (${result})`)
}
} catch (err) {
log.error('Failed to ensure Host NetworkPolicy', { host: host.name, policy: name, err })
npFailures.push(`${name}: ${err instanceof Error ? err.message : String(err)}`)
}
}
await ensureNetworkPolicy('mcp-host ingress', () =>
this.ensureMcpHostIngressNetworkPolicy(host)
)
await ensureNetworkPolicy('mcp-host GFS egress', () =>
this.ensureMcpHostGfsEgressNetworkPolicy(host)
)
await ensureNetworkPolicy('mcp-host Codex proxy egress', () =>
this.reconcileMcpHostCodexProxyEgressNetworkPolicy(host)
)
await ensureNetworkPolicy('mcp-host Grok proxy egress', () =>
this.reconcileMcpHostGrokProxyEgressNetworkPolicy(host)
)
// The mcp-host→llm-hooks egress policy is now owned by LlmHookReconciler
// (per-host, scoped to referenced hook pods — N1/N7); host-delete cleanup
// of `mcp-host-<host>-egress-llm-hooks` stays in deleteHostNetworkPolicies.
await ensureNetworkPolicy('workflow approval reader ingress', () =>
this.ensureWorkflowApprovalReaderMcpHostIngressNetworkPolicy(host)
)
await ensureNetworkPolicy('rpc-proxy mcp-host ingress', () =>
this.ensureRpcProxyMcpHostIngressNetworkPolicy(host)
)
await ensureNetworkPolicy('rpc-proxy Host egress', () =>
this.ensureRpcProxyHostEgressNetworkPolicy(host)
)
await ensureNetworkPolicy('desktop ingress', () => this.ensureDesktopNetworkPolicy(host))
await ensureNetworkPolicy('channel-reader egress', () =>
this.ensureChannelReaderEgressNetworkPolicy(host)
)
await ensureNetworkPolicy('workflow approval reader egress', () =>
this.ensureWorkflowApprovalReaderHostEgressNetworkPolicy(host)
)
revalidateHostMutationBoundary()
// Resolve policy once before bootstrap so token scopes and wake/suspend
// decisions use the latest channel state already observed.
const bootstrapLifecycle = this.lifecycle.enforceCommunicationChannelPolicyBeforeDeployment(
host.name,
lifecycle
)
if (bootstrapLifecycle !== lifecycle) {
lifecycle = bootstrapLifecycle
revalidateHostMutationBoundary()
await this.lifecycle.writeLifecycleStatusToCluster(host, lifecycle)
}
revalidateHostMutationBoundary()
// PVC/Service errors abort before any credential or Deployment mutation,
// on the hold path as on the normal path.
if (resourceErrors.length === 1) throw resourceErrors[0]
if (resourceErrors.length > 1) {
throw new AggregateError(
resourceErrors,
`Failed to reconcile Host resources for "${host.name}"`
)
}
if (lifecycle.effective.suspensionBlocked) {
await holdAppliedRuntime(forceFreshForWake)
return
}
// Bootstrap captures the scope contract used for issuance. The Deployment
// guard below compares that contract with the live channel cache after this
// potentially slow I/O and before every create/replace attempt.
let runtimeTokenProvision = await this.provisionRuntimeTokenRevision(host, {
forceFreshForWake,
targetSuspended: lifecycle.effective.stateless && lifecycle.effective.state === 'suspended',
prepareForResourceChange: lifecycle.effective.state === 'active',
})
if (!runtimeTokenProvision) {
const liveDeployment = await this.readHostDeploymentOrNull(host)
this.setStatus(host.name, {
deployed: !!liveDeployment,
ready: false,
message: 'Waiting for authoritative scope observation',
})
return
}
const requireRuntimeTokenProvision = (): RuntimeTokenProvision => {
if (!runtimeTokenProvision) {
const error = new Error('Waiting for authoritative scope observation')
error.name = 'RuntimeScopeObservationUnavailableError'
throw error
}
return runtimeTokenProvision
}
revalidateHostMutationBoundary()
// replaceWithConflictRetry may wait and re-read after a 409, so each body is
// rebuilt from a stable lifecycle plus token-scope pair before mutation.
const resolveDeploymentLifecycle = async (): Promise<EffectiveHostLifecycle> => {
revalidateHostMutationBoundary()
const deploymentLifecycle = this.lifecycle.enforceCommunicationChannelPolicyBeforeDeployment(
host.name,
lifecycle
)
if (deploymentLifecycle !== lifecycle) {
lifecycle = deploymentLifecycle
revalidateHostMutationBoundary()
await this.lifecycle.writeLifecycleStatusToCluster(host, lifecycle)
}
revalidateHostMutationBoundary()
return lifecycle.effective
}
const ensureCurrentRuntimeTokenScope = async (): Promise<void> => {
for (let attempt = 1; attempt <= 3; attempt++) {
revalidateHostMutationBoundary()
const currentOAuth = await this.observeFrontsOAuthServer(host)
if (this.currentScopeMatchesProvision(host, requireRuntimeTokenProvision(), currentOAuth)) {
return
}
runtimeTokenProvision = await this.provisionRuntimeTokenRevision(host, {
forceFreshForWake: false,
targetSuspended:
lifecycle.effective.stateless && lifecycle.effective.state === 'suspended',
prepareForResourceChange: lifecycle.effective.state === 'active',
})
if (!runtimeTokenProvision) {
const error = new Error('Waiting for authoritative scope observation')
error.name = 'RuntimeScopeObservationUnavailableError'
throw error
}
revalidateHostMutationBoundary()
const postProvisionOAuth = await this.observeFrontsOAuthServer(host)
if (
this.currentScopeMatchesProvision(
host,
requireRuntimeTokenProvision(),
postProvisionOAuth
)
) {
return
}
log.warn('CommunicationChannel scope contract changed during token provisioning', {
host: host.name,
namespace: host.namespace,
attempt,
})
}
throw new Error(
`CommunicationChannel scope contract did not stabilize before Deployment mutation for host "${host.name}"`
)
}
const resolveDeploymentState = async (
observed?: k8s.V1Deployment
): Promise<DeploymentMutationState> => {
for (let attempt = 1; attempt <= 3; attempt++) {
revalidateHostMutationBoundary()
const beforeScope = await resolveDeploymentLifecycle()
if (beforeScope.suspensionBlocked) {
return {
lifecycle: beforeScope,
runtimeTokenRevision: requireRuntimeTokenProvision().revision,
grokExecutionEnabled: this.hostDerivesGrokExecution(host),
}
}
await ensureCurrentRuntimeTokenScope()
const deploymentIdentityChanged =
!!observed?.metadata?.uid &&
!!requireRuntimeTokenProvision().deploymentUid &&
observed.metadata.uid !== requireRuntimeTokenProvision().deploymentUid
const observedResourceTransition =
!!observed &&
beforeScope.state === 'active' &&
HostReconciler.deploymentBelongsToHost(observed, host) &&
(observed.spec?.replicas ?? 1) > 0 &&
!HostReconciler.deploymentResourcesMatchHost(observed, host)
if (deploymentIdentityChanged || observedResourceTransition) {
// A CAS retry or existence read can reveal a new UID or resource drift.
// Prepare against this observation before rebuilding the Pod template;
// persisted preparation is reused for retries against the same target.
runtimeTokenProvision = await this.provisionRuntimeTokenRevision(host, {
forceFreshForDeploymentChange: deploymentIdentityChanged,
targetSuspended: beforeScope.stateless && beforeScope.state === 'suspended',
prepareForResourceChange: beforeScope.state === 'active',
})
requireRuntimeTokenProvision()
revalidateHostMutationBoundary()
}
const effective = await resolveDeploymentLifecycle()
const currentOAuth = await this.observeFrontsOAuthServer(host)
// The OAuth lookup can await I/O after the last lifecycle observation.
// Retry assessment rather than committing a now-obsolete mode.
if (
this.lifecycle.enforceCommunicationChannelPolicyBeforeDeployment(host.name, lifecycle) !==
lifecycle
) {
continue
}
if (this.currentScopeMatchesProvision(host, requireRuntimeTokenProvision(), currentOAuth)) {
revalidateHostMutationBoundary()
return {
lifecycle: effective,
runtimeTokenRevision: requireRuntimeTokenProvision().revision,
freshBootstrapPrepared: requireRuntimeTokenProvision().freshBootstrapPrepared,
grokExecutionEnabled: this.hostDerivesGrokExecution(host),
}
}
log.warn('CommunicationChannel scope contract changed before Deployment mutation', {
host: host.name,
namespace: host.namespace,
attempt,
})
}
throw new Error(
`CommunicationChannel lifecycle and token scope did not stabilize before Deployment mutation for host "${host.name}"`
)
}
revalidateHostMutationBoundary()
let deploymentApplied: boolean
try {
deploymentApplied = await this.ensureDeployment(
host,
mounts,
runtimeTokenProvision.revision,
lifecycle.effective,
resolveDeploymentState,
revalidateHostMutationBoundary
)
} catch (error) {
if (
error instanceof Error &&
(error.name === 'RuntimeScopeObservationUnavailableError' ||
error.name === 'RuntimeBootstrapPreparationUnavailableError')
) {
const liveDeployment = await this.readHostDeploymentOrNull(host)
this.setStatus(host.name, {
deployed: !!liveDeployment,
ready: false,
message:
error.name === 'RuntimeBootstrapPreparationUnavailableError'
? error.message
: 'Waiting for authoritative scope observation',
})
return
}
throw error
}
revalidateHostMutationBoundary()
if (!deploymentApplied) {
this.setStatus(host.name, {
deployed: false,
ready: false,
message: 'Waiting for CommunicationChannel inventory before creating runtime',
})
return
}
const suspended = lifecycle.effective.stateless && lifecycle.effective.state === 'suspended'
if (suspended) {
this.lifecycle.recordSuspendedApplied(host.name)
} else {
this.lifecycle.markHostNotSuspended(host.name)
}
const ready = suspended ? false : await this.checkDeploymentReady(host.name, host.namespace)
revalidateHostMutationBoundary()
// If any per-Host NP failed to apply, mark the host as degraded:
// deployed remains true (the pod is in place), but ready=false +
// a message that names the missing security boundary. Operators see
// this in `kubectl get host ... -o yaml` under status.runtime.message.
const baseMessage = suspended
? 'Suspended (stateless lifecycle, replicas=0)'
: ready
? 'Running'
: 'Deployment created, waiting for pods to become ready'
const message = npFailures.length
? `degraded — NetworkPolicy boundary missing (${npFailures.join('; ')})`
: baseMessage
this.setStatus(host.name, {
deployed: true,
ready: ready && npFailures.length === 0,
message,
})
// Do not schedule a readiness poll when NetworkPolicy has degraded the
// runtime: the same-generation poll would overwrite the degraded verdict
// with "Running" once the Deployment alone becomes Ready.
if (!ready && !suspended && npFailures.length === 0) {
this.pollReadiness(host.name, host.namespace)
}
// Materialize the per-Host channel-reader Deployment. The egress NP that
// pins this pod to mcp-host has already been applied above. A failure does
// not remove the converged mcp-host pod, but it rejects this reconcile so
// the caller can retry and is surfaced in HostRuntimeStatus.
//
// computeChannelReaderRevisionForHost re-throws on Secret read errors
// other than 404 (RBAC drift, 503, etc.), so this catch handles a real
// expected error path — not just defense in depth.
const channelReaderFailures: unknown[] = []
for (const reconcileResource of [
() => this.reconcileChannelReaderService(host, revalidateHostMutationBoundary),
() => this.reconcileChannelReaderDeployment(host, revalidateHostMutationBoundary),
]) {
try {
revalidateHostMutationBoundary()
await reconcileResource()
} catch (err) {
channelReaderFailures.push(err)
}
}
const channelReaderError = channelReaderFailures
.map(error => (error instanceof Error ? error.message : String(error)))
.join('; ')
if (channelReaderError) {
const prev = this.getStatus(host.name)
this.setStatus(host.name, {
...prev,
message: `channel-reader: ${channelReaderError}`,
})
}
// Surface channel-reader Deployment readiness in HostRuntimeStatus.
// Best-effort — checkChannelReaderStatus never throws.
const ccCount = this.countCommunicationChannels(host.name)
const channelReaderStatus = await this.checkChannelReaderStatus(host.name, ccCount)
revalidateHostMutationBoundary()
if (channelReaderFailures.length > 0) {
// Overlay the reconcile error onto the status message so both signals
// are visible (the deploy error may explain why the Deployment is missing).
channelReaderStatus.message = `reconcile error: ${channelReaderError}`
channelReaderStatus.ready = false
this.enqueueControllerError(
host,
'channel_reader_reconcile_failed',
new Error(channelReaderError)
)
}
const prev = this.getStatus(host.name)
this.setStatus(host.name, { ...prev, channelReader: channelReaderStatus })
throwCleanupFailures(
channelReaderFailures,
`Failed to converge channel-reader resources for Host "${host.name}"`
)
}
/**
* Delete a Host's runtime bundle.
*
* `opts.skipIf` is an OPTIONAL fence evaluated INSIDE the per-Host serializer,
* immediately before anything destructive runs. Callers whose "this Host is
* gone" decision was made BEFORE admission (the watch-recovery diff, which
* compares LIST-time names) pass it so a same-name Host recreated during the
* admission window is not wiped: the recreation's ADDED sets the cache and
* enters this Host's chain FIRST, its reconcile rebuilds the bundle, and only
* then does the stale delete reach the front of the strict-FIFO queue.
* Ownership labels cannot discriminate here — the rebuilt bundle carries
* identical `clerum.io/host` + managed-by labels — and no uid is available on
* that path, so cache presence at admission is the correct fence. This
* mirrors the F2/#827 in-serializer re-check in collectHostCleanupFailures.
*
* Callers that omit `opts` drop only the extra `skipIf` term; the
* resolveCurrentHost supersede fence still gates the delete.
*/
async reconcileDelete(
name: string,
namespace: string,
opts?: { skipIf?: () => boolean }
): Promise<void> {
const capturedAuthority = this.captureHostMutationAuthority()
// A DELETED event must not race an in-flight create/update for the same
// Host: route it through the same per-Host serializer as reconcile(). The
// global convergence tail no longer provides this ordering.
await this.lifecycle.serializeByHost(name, async () => {
this.requireHostMutationAuthority(`Host "${name}" delete`, capturedAuthority)
if (this.resolveCurrentHost?.(name) !== undefined || opts?.skipIf?.()) {
// Same vocabulary the watch path already reports for this condition
// (reconcileHostWatchEvent), so the skip is observable, not silent.
hostDeleteCleanupTotal.inc({ outcome: 'superseded' })
return
}
await this.deleteHostRuntimeResources(name, namespace)
this.clearStatus(name)
this.desktopHosts.delete(name)
// The delete path has no uid to key by, so every entry for this
// namespace/name goes: the object is gone and no outcome will carry its
// evidence (#696).
this.clearGfsLifecycleEvidence(namespace, name)
})
}
/**
* Reconcile the fleet with bounded cross-Host concurrency. Dispatches Host
* KEYS (not stale object snapshots): each worker resolves the CURRENT cached
* Host at execution time and skips entries the cache no longer holds
* (superseded by a delete/newer event). Failures are aggregated completely.
*/
private async collectHostReconcileFailures(
desiredHosts: HostCRD[],
source: HostReconcileSource
): Promise<unknown[]> {
const snapshotByName = new Map(desiredHosts.map(h => [h.name, h]))
const resolve = this.resolveCurrentHost ?? ((name: string) => snapshotByName.get(name))
const keys = desiredHosts.map(h => h.name)
return runBoundedHostWorkers(keys, config.hostFullReconcileConcurrency, async name => {
const host = resolve(name)
if (!host) return // absent/superseded since the pass captured its snapshot
try {
await this.reconcile(host, source)
} catch (error) {
if (isBenignSupersessionError(error)) {
// Watch-race withdrawal is not a host failure.
hostFleetBenignSupersessionsTotal.inc({ error: error.name })
log.warn('Fleet Host reconcile withdrawn as benign supersession', {
err: error,
errorName: error.name,
})
return
}
log.error('Fleet reconcile failed for Host', { host: name, err: error })
throw error
}
})
}
/**
* Orphan cleanup with the §10.5 authority contract (replaces the accidental
* safety the removed global tail used to provide):
* - runs only with known watch authority and the SAME generation captured at
* pass start;
* - compares candidates with the CURRENT cache, not the pass snapshot;
* - groups candidates by owning logical Host from ownership labels; a
* candidate with no derivable owner is NEVER deleted (deferred + recorded);
* - performs ONE fresh authoritative Host read per orphan Host; a confirmed
* 404 (while authority stays valid) authorizes deleting that Host's entire
* owned bundle INSIDE the Host's per-Host serializer;
* - defers (with a bounded reason) on unknown authority, generation drift,
* watch loss, a present read, or a failed read.
*/
private async collectHostCleanupFailures(
capturedAuthority: HostWatchAuthoritySnapshot
): Promise<unknown[]> {
const failures: unknown[] = []
const authorityValid = (): boolean => {
const current = this.hostWatchAuthority()
return (
capturedAuthority.known &&
current.known &&
capturedAuthority.generation === current.generation
)
}
if (!authorityValid()) {
const current = this.hostWatchAuthority()
const reason =
!capturedAuthority.known || !current.known ? 'authority_unknown' : 'generation_changed'
hostCleanupDeferredTotal.inc({ reason })
log.warn('Deferring orphan cleanup', { reason })
return failures
}
let candidates: CleanupCandidate[]
try {
candidates = await this.gatherCleanupCandidates(failures)
} catch (error) {
failures.push(error)
return failures
}
const cacheHasHost = (name: string): boolean =>
this.resolveCurrentHost ? this.resolveCurrentHost(name) !== undefined : false
const orphanHosts = new Set<string>()
for (const candidate of candidates) {
if (!candidate.owner) {
// Never delete a resource whose owning Host cannot be derived.
hostCleanupDeferredTotal.inc({ reason: 'no_owner_label' })
log.warn('Deferring orphan candidate: no derivable owning Host', {
kind: candidate.kind,
name: candidate.name,
})
continue
}
if (cacheHasHost(candidate.owner)) continue // still present → retain
orphanHosts.add(candidate.owner)
}
// §10.5 destructive cleanup runs per orphan logical Host. Each Host's owned
// write set is disjoint (§10.4 inventory) and each destructive bundle runs
// inside that Host's own `serializeByHost` chain, so distinct orphan Hosts
// are provably independent. Addendum 2: dispatch them through the same
// bounded worker pool as the fleet (reusing HCC_HOST_FULL_RECONCILE_CONCURRENCY)
// instead of a serial `for … await` — never an unbounded fan-out at the K8s
// API. Same-Host ordering is still guaranteed by serializeByHost, and delete
// failures still aggregate completely because runBoundedHostWorkers collects
// every lane's throw.
const cleanupFailures = await runBoundedHostWorkers(
Array.from(orphanHosts),
config.hostFullReconcileConcurrency,
async hostName => {
if (!authorityValid()) {
hostCleanupDeferredTotal.inc({ reason: 'watch_lost' })
log.warn('Deferring orphan cleanup: watch authority lost', { host: hostName })
return
}
const presence = await this.readHostPresence(hostName)
if (presence === 'present') return // reappeared → retain
if (presence === 'error') {
hostCleanupDeferredTotal.inc({ reason: 'fresh_read_failed' })
return
}
if (
!destructiveCleanupAllowed({
watchAuthorityKnown: this.hostWatchAuthority().known,
capturedWatchGeneration: capturedAuthority.generation,
currentWatchGeneration: this.hostWatchAuthority().generation,
currentCacheOmitsHost: !cacheHasHost(hostName),
// Derived, not hardcoded. `presence` is provably 'absent' here (the
// 'present' and 'error' branches returned above), so this evaluates
// to the identical literal and the runtime behavior is unchanged.
// Deriving it makes the mapping total: if readHostPresence ever
// grows a fourth outcome, the else-branch type stops satisfying
// freshAuthoritativeRead's union and the compiler rejects it —
// instead of silently labelling a non-404 read as 'confirmed404'
// in a value that is DECISIONAL (a conjunct of
// destructiveCleanupAllowed, :94-95), not telemetry.
freshAuthoritativeRead: presence === 'absent' ? 'confirmed404' : presence,
})
) {
hostCleanupDeferredTotal.inc({ reason: 'watch_lost' })
return
}
try {
await this.lifecycle.serializeByHost(hostName, async () => {
// The fresh 404 and the cache omission are only destructive
// authority while the same LIST -> WATCH generation remains live.
// Revalidate inside the serializer: queued Host work can otherwise
// outlive that generation and delete a same-name recreation whose
// ADDED event has not reached the retired cache.
if (!authorityValid()) {
hostCleanupDeferredTotal.inc({ reason: 'watch_lost' })
return
}
// F2 / #827 TOCTOU: a same-name Host recreated between the fresh read
// above and admission into this serializer updates the current cache
// (its ADDED event sets the cache before its reconcile runs, and that
// reconcile serializes ahead of this delete). Re-check the current
// cache INSIDE the serializer so a just-recreated Host's bundle is
// never deleted by this stale cleanup.
if (cacheHasHost(hostName)) {
hostCleanupDeferredTotal.inc({ reason: 'recreated' })
return
}
await this.deleteHostRuntimeResources(hostName, config.hostNamespace)
})
} catch (error) {
log.error('Fleet cleanup failed for orphan Host', { host: hostName, err: error })
throw error
}
}
)
failures.push(...cleanupFailures)
return failures
}
/**
* List every managed resource that could be an orphan and derive its owning
* logical Host from ownership labels. List failures are collected into
* `listFailures` (reported by the caller) so a partial discovery still cleans
* what it could confidently attribute. Candidates never drive a delete on
* their own — the caller applies the fresh-read authority gate.
*/
private async gatherCleanupCandidates(listFailures: unknown[]): Promise<CleanupCandidate[]> {
const candidates: CleanupCandidate[] = []
const pushOwner = (kind: string, name: string | undefined, owner: string | undefined): void => {
if (!name) return
candidates.push({ kind, name, owner })
}
try {
for (const deployment of await this.listManagedHostDeployments()) {
pushOwner(
'Deployment',
deployment.metadata?.name,
deployment.metadata?.labels?.[HOST_LABEL]
)
}
} catch (error) {
log.error('Failed to list managed host deployments', { err: error })
listFailures.push(error)
}
const channelsNs = config.channelsNamespace
const managed = (labels: Record<string, string> | undefined): boolean =>
labels?.[MANAGED_BY_LABEL] === MANAGED_BY_VALUE
try {
const list = await this.appsApi.listNamespacedDeployment({ namespace: channelsNs })
for (const dep of list.items ?? []) {
const labels = dep.metadata?.labels
if (labels?.app !== 'channel-reader' || !managed(labels)) continue
pushOwner('ChannelReaderDeployment', dep.metadata?.name, labels[HOST_LABEL])
}
} catch (error) {
log.warn('channel-reader Deployment candidate list failed', { err: error })
listFailures.push(error)
}
try {
const list = await this.coreApi.listNamespacedService({ namespace: channelsNs })
for (const svc of list.items ?? []) {
const labels = svc.metadata?.labels
if (labels?.app !== 'channel-reader' || !managed(labels)) continue
pushOwner('ChannelReaderService', svc.metadata?.name, labels[HOST_LABEL])
}
} catch (error) {
log.warn('channel-reader Service candidate list failed', { err: error })
listFailures.push(error)
}
try {
const list = await this.coreApi.listNamespacedSecret({
namespace: channelsNs,
labelSelector: [
`${MANAGED_BY_LABEL}=${MANAGED_BY_VALUE}`,
'clerum.io/component=channel-reader',
].join(','),
})
for (const sec of list.items ?? []) {
const labels = sec.metadata?.labels
if (!managed(labels) || labels?.['clerum.io/component'] !== 'channel-reader') continue
pushOwner('ChannelReaderSecret', sec.metadata?.name, labels[HOST_LABEL])
}
} catch (error) {
log.warn('channel-reader Secret candidate list failed', { err: error })
listFailures.push(error)
}
// #827 (Addendum 4 item 4): partial-leftover discovery. A bundle whose
// Deployment — and even its NetworkPolicies — are already gone must still be
// discovered from ANY surviving owned resource. List every owned kind HCC
// writes in the host namespace by ownership label and derive the owner from
// clerum.io/host; a candidate without a derivable owner is never deleted by
// the caller (§10.5).
const hostNs = config.hostNamespace
const managedSelector = `${MANAGED_BY_LABEL}=${MANAGED_BY_VALUE}`
const gatherOwnedKind = async (
kind: string,
list: () => Promise<{
items?: Array<{ metadata?: { name?: string; labels?: Record<string, string> } }>
}>
): Promise<void> => {
try {
const res = await list()
for (const item of res.items ?? []) {
if (!managed(item.metadata?.labels)) continue
pushOwner(kind, item.metadata?.name, item.metadata?.labels?.[HOST_LABEL])
}
} catch (error) {
log.warn('Owned resource candidate list failed', { kind, namespace: hostNs, err: error })
listFailures.push(error)
}
}
await gatherOwnedKind('Service', () =>
this.coreApi.listNamespacedService({ namespace: hostNs, labelSelector: managedSelector })
)
await gatherOwnedKind('PersistentVolumeClaim', () =>
this.coreApi.listNamespacedPersistentVolumeClaim({
namespace: hostNs,
labelSelector: managedSelector,
})
)
await gatherOwnedKind('ServiceAccount', () =>
this.coreApi.listNamespacedServiceAccount({
namespace: hostNs,
labelSelector: managedSelector,
})
)
await gatherOwnedKind('Secret', () =>
this.coreApi.listNamespacedSecret({ namespace: hostNs, labelSelector: managedSelector })
)
await gatherOwnedKind('Role', () =>
this.rbacApi.listNamespacedRole({ namespace: hostNs, labelSelector: managedSelector })
)
await gatherOwnedKind('RoleBinding', () =>
this.rbacApi.listNamespacedRoleBinding({ namespace: hostNs, labelSelector: managedSelector })
)
for (const ns of [config.channelsNamespace, config.hostNamespace, config.rpcProxyNamespace]) {
try {
const list = await this.networkingApi.listNamespacedNetworkPolicy({
namespace: ns,
labelSelector: `${MANAGED_BY_LABEL}=${MANAGED_BY_VALUE}`,
})
for (const np of list.items ?? []) {
pushOwner('NetworkPolicy', np.metadata?.name, np.metadata?.labels?.[HOST_LABEL])
}
} catch (error) {
log.warn('NetworkPolicy candidate list failed', { namespace: ns, err: error })
listFailures.push(error)
}
}
return candidates
}
/**
* One fresh authoritative Host read. `absent` is a confirmed 404 (the Host is
* genuinely gone). `present` means the Host reappeared. `error` (timeout,
* forbidden, 5xx) is deliberately NOT surfaced as a pass failure — the caller
* defers with a bounded reason, matching the §13.2 cleanup matrix.
*/
private async readHostPresence(name: string): Promise<'absent' | 'present' | 'error'> {
try {
await this.customApi.getNamespacedCustomObject({
group: HOST_GROUP,
version: HOST_VERSION,
namespace: config.hostNamespace,
plural: HOST_PLURAL,
name,
})
return 'present'
} catch (error) {
if (getErrorCode(error) === 404) return 'absent'
log.warn('Fresh authoritative read for orphan Host failed', { host: name, err: error })
return 'error'
}
}
async reconcileHosts(desiredHosts: HostCRD[]): Promise<void> {
const hostFailures = await this.collectHostReconcileFailures(desiredHosts, 'fleet')
if (hostFailures.length > 0) {
throw new HostFleetReconcileError(hostFailures, [])
}
}
async fullReconcile(desiredHosts: HostCRD[]): Promise<void> {
// Capture watch authority + generation at pass start (§10.5 step 1) so
// cleanup can prove nothing shifted underneath it before deleting.
const capturedAuthority = this.hostWatchAuthority()
const hostFailures = await this.collectHostReconcileFailures(desiredHosts, 'fleet')
// Cleanup runs only after every fleet worker has settled.
const cleanupFailures = await this.collectHostCleanupFailures(capturedAuthority)
if (hostFailures.length > 0 || cleanupFailures.length > 0) {
throw new HostFleetReconcileError(hostFailures, cleanupFailures)
}
}
private async listManagedHostDeployments(): Promise<k8s.V1Deployment[]> {
const response = await this.appsApi.listNamespacedDeployment({
namespace: config.hostNamespace,
labelSelector: `${MANAGED_BY_LABEL}=${MANAGED_BY_VALUE}`,
})
return (response.items || []).filter(item => item.metadata?.labels?.[HOST_LABEL])
}
}
/**
* True when the desired Role matches the live object on the fields HCC authors:
* `rbacLabels` and `rules`. Server metadata and annotations are ignored — the
* builder does not write annotations, and comparing them would force a PUT.
*
* Label comparison is a subset of the keys HCC authors. Extra live labels
* (Kyverno/Gatekeeper add-labels, ArgoCD instance, cost tags) must not force a
* PUT that strips them and re-enters the write loop. A missing or changed
* authored key still fail-opens to write, so a later third `rbacLabels` key
* still lands. A key HCC stops authoring is treated as extra-live and will
* not PUT until some other authored field drifts; the previous exact-map
* compare used to retract it. `isHccOwnedHostResource` keys on the same two
* labels today.
*
* Object keys inside each rule are canonicalized: client-node rebuilds
* V1PolicyRule in attributeTypeMap order (resourceNames before resources), so a
* raw JSON.stringify never matches a live GET (#307). Rule arrays, verbs, and
* resourceNames stay in author order. Missing rules or labels, or any compare
* failure, returns false (fail-open-to-write).
*/
function roleMatchesDesired(desired: k8s.V1Role, existing: k8s.V1Role): boolean {
try {
if (!desired.rules || !existing.rules) return false
const desiredLabels = desired.metadata?.labels
const existingLabels = existing.metadata?.labels
if (!desiredLabels || !existingLabels) return false
const authored: Record<string, string> = {}
const liveAuthored: Record<string, string> = {}
for (const key of Object.keys(desiredLabels)) {
if (existingLabels[key] === undefined) return false
authored[key] = desiredLabels[key]
liveAuthored[key] = existingLabels[key]
}
return (
JSON.stringify(canonicalizeValue({ labels: authored, rules: desired.rules })) ===
JSON.stringify(canonicalizeValue({ labels: liveAuthored, rules: existing.rules }))
)
} catch {
return false
}
}
/**
* Preserve operational pod-template annotations (kubectl restart markers,
* operator edits) while keeping CONTROLLER-OWNED ones authoritative: a key in
* `controllerOwned` that the desired template omits is REMOVED from the merge
* rather than surviving from the live object.
*
* Without this, "the field went away" cannot be expressed at all. The merge is
* `{...existing, ...desired}`, so an omitted key keeps its old value, the merged
* object then compares equal to what is live, and `replaceWithConflictRetry`
* skips the write entirely — no template diff, no rolling restart, no write.
*/
export function preserveDeploymentAnnotationsExcept(
desired: k8s.V1Deployment,
existing: k8s.V1Deployment,
controllerOwned: readonly string[]
): k8s.V1Deployment {
const preserved = preserveDeploymentAnnotations(desired, existing)
const desiredAnnotations = desired.spec?.template?.metadata?.annotations
const dropped = controllerOwned.filter(key => desiredAnnotations?.[key] === undefined)
if (dropped.length === 0) return preserved
const spec = preserved.spec
const template = spec?.template
if (!spec || !template) return preserved
const annotations = { ...(template.metadata?.annotations ?? {}) }
for (const key of dropped) delete annotations[key]
return {
...preserved,
spec: {
...spec,
template: {
...template,
metadata: {
...template.metadata,
annotations: Object.keys(annotations).length > 0 ? annotations : undefined,
},
},
},
}
}
/**
* Host Deployment merge. `clerum.io/guardrails-revision` is controller-owned:
* dropping the whole `spec.guardrails` block omits it from the desired template,
* and that removal MUST reach the pod. mcp-host re-resolves guardrails live, but
* only while its Host watch is connected — the pod roll is what delivers the
* change to an agent whose watch has lapsed, so a removal that silently produced
* no diff left the agent enforcing guardrails the operator had uninstalled.
*/
function preserveHostDeploymentAnnotations(
desired: k8s.V1Deployment,
existing: k8s.V1Deployment
): k8s.V1Deployment {
return preserveDeploymentAnnotationsExcept(desired, existing, [GUARDRAILS_REVISION_ANNOTATION])
}
/**
* Preserve operational annotations without retaining the channel-reader
* revision when HCC intentionally omits it from the desired pod template.
* That annotation is controller-owned and must be cleared when no backing
* CommunicationChannel Secret is resolvable.
*/
function preserveChannelReaderDeploymentAnnotations(
desired: k8s.V1Deployment,
existing: k8s.V1Deployment
): k8s.V1Deployment {
return preserveDeploymentAnnotationsExcept(desired, existing, ['clerum.io/credentials-revision'])
}