Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
/**
* Kubernetes client for watching McpServer CRDs.
* Also provides a dev mode provider for local testing.
*
* In production, the McpServerWatcher:
* 1. Watches McpServer CRDs for changes
* 2. Maintains an in-memory cache for the REST API
* 3. Triggers the reconciler to create/update/delete Deployments + Services
*/
import * as k8s from '@kubernetes/client-node'
import {
type AdministrativeOutcomeReporter,
createAdministrativeOutcomeReporter,
} from './administrativeOutcomeReporter'
import { BindingDef, BindingPolicyReconciler } from './bindingPolicyReconciler'
import { config } from './config'
import {
ExternalEgressConvergenceCoordinator,
type ExternalEgressRetryHandle,
type ExternalEgressWatchEventType,
} from './externalEgressConvergenceCoordinator'
import { gfsDefaultFactoryConfig } from './gfsConfig'
import { GfsReconciler } from './gfsReconciler'
import { ControlApiGfsSeedClient } from './gfsSeedClient'
import {
type HostFleetReconcileMode,
type HostFleetReconcileRequest,
HostFleetScheduler,
} from './hostFleetScheduler'
import {
HostFleetReconcileError,
type HostReconcileSource,
HostReconciler,
type ResolvedSfsMount,
} from './hostReconciler'
import {
type InfrastructureTelemetryReporter,
createInfrastructureTelemetryReporter,
} from './infrastructureTelemetryReporter'
import { K8sGfsApi } from './k8s/gfsK8sApi'
import { makeHostK8sApiClient } from './k8s/hostK8sApiClient'
import { pvcName as sfsPvcName } from './k8s/sharedFileSystemFactory'
import { LlmHookReconciler, computePodKey, referencedHookIds } from './llmHookReconciler'
import { hccLogger } from './logger'
import type {
AuthorityContext,
AuthorityHost,
AuthorityMcpServer,
AuthoritySecret,
AuthoritySecretMetadata,
McpAuthorizationStore,
} from './mcpAuthorization'
import {
confirmAuthoritativeMcpServerAbsence,
isMcpServerStatusOnlyUpdate,
runtimeDesired,
sameMcpServerDesiredRevision,
sameMcpServerPolicyRevision,
} from './mcpServerSafety'
import {
contextMetadataOnlyEventsTotal,
hostDeleteCleanupTotal,
hostFleetLifecycleCatchTotal,
hostFleetRequestsTotal,
hostWatchRecoverySeconds,
initialConvergenceEffectsDroppedTotal,
initialConvergenceLastSuccessTimestampSeconds,
initialConvergencePassDurationSeconds,
initialConvergencePassResultsTotal,
initialConvergenceRetriesTotal,
initialConvergenceSwallowedTotal,
netPolDefaultsOnlyTickDurationSeconds,
netPolDefaultsOnlyTicksTotal,
netPolResyncTicksSkippedTotal,
} from './metrics'
import {
DESIRED_NETWORKPOLICY_INVENTORY_CHANGED_MESSAGE,
NetworkPolicyReconciler,
sameContextDesiredRevision,
} from './networkPolicyReconciler'
import type { ReadinessInventoryDetail } from './readinessGate'
import { McpServerReconciler } from './reconciler'
import { SharedFileSystemReconciler } from './sharedFileSystemReconciler'
import {
CommunicationChannelCRD,
ContextCRD,
ContextSpec,
GlobalFileSystemCRD,
GlobalFileSystemSpec,
HostCRD,
HostSpec,
LlmHookCRD,
LlmHookSpec,
McpServerCRD,
McpServerInfo,
McpServerSpec,
SharedFileSystemCRD,
SharedFileSystemSpec,
} from './types'
import { getErrorCode } from './utils'
// Only initialize K8s client if not in dev mode
let customObjectsApi: k8s.CustomObjectsApi | null = null
let hostCustomObjectsApi: k8s.CustomObjectsApi | null = null
let coreApi: k8s.CoreV1Api | null = null
let kc: k8s.KubeConfig | null = null
if (!config.devMode) {
kc = new k8s.KubeConfig()
kc.loadFromDefault()
customObjectsApi = kc.makeApiClient(k8s.CustomObjectsApi)
hostCustomObjectsApi = makeHostK8sApiClient(
kc,
k8s.CustomObjectsApi,
config.hostK8sRequestTimeoutMs
)
coreApi = kc.makeApiClient(k8s.CoreV1Api)
}
/**
* Return the shared KubeConfig used by production clients (null in dev mode).
* Exposed so auxiliary watchers (e.g. SecretInformer) can reuse the same config.
*/
export function getKubeConfig(): k8s.KubeConfig | null {
return kc
}
const GROUP = 'clerum.io'
const VERSION = 'v1alpha1'
const PLURAL_MCPSERVERS = 'mcpservers'
const PLURAL_CONTEXTS = 'contexts'
const PLURAL_HOSTS = 'hosts'
const PLURAL_SHAREDFILESYSTEMS = 'sharedfilesystems'
const PLURAL_GLOBALFILESYSTEMS = 'globalfilesystems'
const PLURAL_COMMUNICATIONCHANNELS = 'communicationchannels'
const PLURAL_LLMHOOKS = 'llmhooks'
// Before readiness was decoupled, a failed initial McpServer or NetworkPolicy
// sweep made provider.start() fail and Kubernetes restarted HCC. These retries
// retain that convergence guarantee now that the sweeps run in the background.
const INITIAL_CONVERGENCE_RETRY_DELAYS_MS = [5000, 15000, 30000, 60000, 300000]
const COMMUNICATION_CHANNEL_CACHE_RECOVERY_RETRY_MS = 5000
// Retry-after-failure pacing for the three hardened inventory lanes
// (McpServer/Context/Host) is exponential with full jitter, mirroring the
// client-go reflector standard. During a long apiserver outage (a GKE zonal
// control-plane upgrade: minutes of every watch down and every re-LIST
// failing) a fixed interval hammers the recovering apiserver, and — worse —
// the HCC's ~3-6 streams synchronize after a simultaneous cut and hit it in
// phase. delay = min(BASE * 2^(failures-1), CAP), then FULL jitter
// (random() * delay; AWS-style full jitter maximizes de-correlation across
// the streams, which is exactly the goal) floored at MIN so jitter can never
// reintroduce a 0-delay busy-loop. A successful recovery resets the ladder.
const WATCH_RECOVERY_BACKOFF_BASE_MS = 1000
const WATCH_RECOVERY_BACKOFF_CAP_MS = 30000
const WATCH_RECOVERY_BACKOFF_MIN_MS = WATCH_RECOVERY_BACKOFF_BASE_MS / 2
// Anti-busy-loop floor for IMMEDIATE watch-close recovery. The first re-LIST
// after an isolated watch close runs immediately (sub-second readiness blip
// instead of the old fixed ~5.5s), but a close arriving within this floor of
// the last successful recovery is demoted onto the paced retry timer above so
// a degraded apiserver churning its watches is never hammered with
// back-to-back LISTs.
const WATCH_CLOSE_RECOVERY_FLOOR_MS = 1000
const HOST_WATCH_RECONCILE_RETRY_DELAYS_MS = [5000, 15000, 30000]
// Wake-pending Hosts get immediate per-Host admission after watch recovery
// (§10.2 step 7) rather than waiting for the background fleet pass.
const WAKE_REQUESTED_ANNOTATION = 'clerum.io/wake-requested'
/**
* Next retry-after-failure delay for a hardened inventory lane. `retryAfterMs`
* (from an HTTP 429 Retry-After, i.e. GKE API Priority & Fairness telling us
* exactly when to come back) overrides the computed backoff, clamped into
* [MIN, CAP] so a throttled lane can neither busy-loop nor stall for an hour
* on a pathological header while readiness is failing closed.
*/
function computeWatchRecoveryRetryDelayMs(
consecutiveFailures: number,
retryAfterMs?: number
): number {
if (retryAfterMs !== undefined) {
return Math.min(
Math.max(retryAfterMs, WATCH_RECOVERY_BACKOFF_MIN_MS),
WATCH_RECOVERY_BACKOFF_CAP_MS
)
}
const attempt = Math.max(1, consecutiveFailures)
const computed = Math.min(
WATCH_RECOVERY_BACKOFF_BASE_MS * 2 ** (attempt - 1),
WATCH_RECOVERY_BACKOFF_CAP_MS
)
return Math.max(WATCH_RECOVERY_BACKOFF_MIN_MS, Math.random() * computed)
}
/**
* Extract Retry-After (milliseconds) from an HTTP 429 error. The
* @kubernetes/client-node ApiException carries the response headers as a
* plain string map; header-name casing is transport-dependent, so the lookup
* is case-insensitive. Only the delta-seconds form is honored — the HTTP-date
* form is not parsed (APF emits delta-seconds; a date would need clock-skew
* handling for marginal value) and falls through to the computed backoff.
*/
function getRetryAfterMs(error: unknown): number | undefined {
if (getErrorCode(error) !== 429) return undefined
const headers = (error as { headers?: unknown }).headers
if (headers === null || typeof headers !== 'object') return undefined
for (const [name, value] of Object.entries(headers)) {
if (name.toLowerCase() !== 'retry-after') continue
const seconds = Number(value)
if (Number.isFinite(seconds) && seconds >= 0) return seconds * 1000
return undefined
}
return undefined
}
type CommunicationChannelSnapshot = {
channels: CommunicationChannelCRD[]
resourceVersion?: string
}
type McpServerSnapshot = {
servers: McpServerCRD[]
resourceVersion?: string
}
type ContextSnapshot = {
contexts: ContextCRD[]
resourceVersion?: string
}
type HostSnapshot = {
hosts: HostCRD[]
resourceVersion?: string
}
type HostInventoryRecoveryCause = 'cold-start' | 'watch-recovery'
type HostWatchEventType = 'ADDED' | 'MODIFIED' | 'DELETED'
type InitialConvergenceLane = 'McpServer' | 'NetworkPolicy'
type NetworkPolicyConvergenceCause =
| 'mcp-recovery'
| 'context-recovery'
| 'periodic-resync'
| 'startup'
| 'retry'
| 'mcp-change'
| 'context-change'
| 'context-reconcile-failure'
type CompletedInventoryRevision = { contextRevision: number; serverRevision: number }
// A scoped McpServer pass retries only names left incomplete by a pass whose
// effects all ran under this inventory revision. 'full' covers the fleet.
type McpServerConvergenceRequest =
| 'full'
| (CompletedInventoryRevision & { names: ReadonlySet<string> })
function widenMcpServerConvergence(
current: McpServerConvergenceRequest | undefined,
next: McpServerConvergenceRequest
): McpServerConvergenceRequest {
if (current === undefined) return next
if (
current === 'full' ||
next === 'full' ||
current.contextRevision !== next.contextRevision ||
current.serverRevision !== next.serverRevision
) {
return 'full'
}
return { ...next, names: new Set([...current.names, ...next.names]) }
}
type InitialConvergencePassResult =
| 'certified'
| 'aborted-authority'
| 'deferred-unsynced'
| 'aborted-bump'
| 'failed'
function observeInitialNetworkPolicyPass(
startedAtMs: number,
result: InitialConvergencePassResult
): void {
const seconds = Math.max(0, (Date.now() - startedAtMs) / 1000)
initialConvergencePassResultsTotal.inc({ lane: 'NetworkPolicy', result })
initialConvergencePassDurationSeconds.observe({ lane: 'NetworkPolicy', result }, seconds)
}
function isDesiredNetworkPolicyInventoryInterruption(error: unknown): boolean {
if (error instanceof AggregateError) {
return (
error.errors.length > 0 && error.errors.every(isDesiredNetworkPolicyInventoryInterruption)
)
}
return error instanceof Error && error.message === DESIRED_NETWORKPOLICY_INVENTORY_CHANGED_MESSAGE
}
type NetworkPolicySafetyCertificate = {
// VESTIGIAL — retained for record shape only, never read in any decision.
// The certificate is identified SOLELY by content revision (the PR #382 fix);
// the generation fields survive as diagnostics and MUST NOT gate readiness
// (reading them would reintroduce the channel-identity livelock).
contextGeneration: number
serverGeneration: number
contextRevision: number
serverRevision: number
}
type ActiveInitialConvergenceRun = {
trailingRequested: boolean
promise: Promise<void>
}
type HostInventoryRecoveryRequest = {
convergenceReason: string
ccLifecycleGeneration?: number
cause: HostInventoryRecoveryCause
}
type ActiveHostInventoryRecovery = {
request: HostInventoryRecoveryRequest
promise: Promise<HostCRD[]>
}
function mergeHostInventoryRecoveryRequest(
target: HostInventoryRecoveryRequest,
requested: HostInventoryRecoveryRequest
): void {
// Cold start is the stronger admission mode: its snapshot is consumed only
// by the bounded full pass, whereas watch recovery also dispatches urgent
// per-Host work. Preserve that mode when either joined caller requires it.
if (requested.cause === 'cold-start' || target.cause !== 'cold-start') {
target.convergenceReason = requested.convergenceReason
}
if (target.cause === 'cold-start' || requested.cause === 'cold-start') {
target.cause = 'cold-start'
}
if (requested.ccLifecycleGeneration !== undefined) {
target.ccLifecycleGeneration =
target.ccLifecycleGeneration === undefined
? requested.ccLifecycleGeneration
: Math.max(target.ccLifecycleGeneration, requested.ccLifecycleGeneration)
}
}
/**
* Serializes a reconciliation domain without turning a rejected operation into
* a permanent queue failure. Kubernetes Watch invokes async callbacks without
* awaiting the preceding callback, while a full reconciliation can delete
* orphaned resources owned by the same domain. A single ordered effect stream
* therefore makes a later watch event the final writer after an older sweep.
*/
class SerializedReconciliationQueue {
private tail: Promise<void> = Promise.resolve()
enqueue(work: () => Promise<void>): Promise<void> {
const result = this.tail.then(work)
this.tail = result.then(
() => undefined,
() => undefined
)
return result
}
}
type KeyedReconciliationQueue = {
queue: SerializedReconciliationQueue
references: number
}
type McpServerWatchObject = {
metadata: {
name: string
namespace?: string
uid?: string
generation?: number
annotations?: Record<string, string>
labels?: Record<string, string>
}
spec: McpServerSpec
status?: McpServerCRD['status']
}
/**
* Interface for McpServer providers (K8s or Dev).
*/
export interface McpServerProvider {
/** Get all cached CRDs (internal use). */
getAllServers(): McpServerCRD[]
/** Get curated server info for all servers (API consumers). */
getAllServerInfos(): McpServerInfo[]
/** Get curated server info filtered by context (API consumers). Reads the Context CRD to determine allowed servers. */
getServerInfosByContext(contextRef: string): Promise<McpServerInfo[]>
/** Set callback for when servers change. */
onChange(callback: () => void): void
start(): Promise<void>
stop(): Promise<void>
}
/**
* List all McpServer CRDs in the namespace.
*/
async function listMcpServerSnapshot(): Promise<McpServerSnapshot> {
if (!customObjectsApi) {
throw new Error('K8s client not initialized - are you in dev mode?')
}
try {
console.log(`[K8s] Listing all McpServers in namespace ${config.namespace}`)
const response = await customObjectsApi.listNamespacedCustomObject({
group: GROUP,
version: VERSION,
namespace: config.namespace,
plural: PLURAL_MCPSERVERS,
})
const list = response as {
metadata?: { resourceVersion?: string }
items: Array<{
metadata: {
name: string
namespace?: string
uid?: string
generation?: number
annotations?: Record<string, string>
labels?: Record<string, string>
}
spec: McpServerSpec
status?: McpServerCRD['status']
}>
}
const servers = list.items.map(item => ({
name: item.metadata.name,
namespace: item.metadata.namespace || config.namespace,
uid: item.metadata.uid,
generation: item.metadata.generation,
annotations: item.metadata.annotations,
labels: item.metadata.labels,
spec: item.spec,
status: item.status,
}))
console.log(`[K8s] Found ${servers.length} McpServer(s)`)
return { servers, resourceVersion: list.metadata?.resourceVersion }
} catch (error) {
console.error('[K8s] Failed to list McpServers:', error)
throw error
}
}
export async function listAllMcpServers(): Promise<McpServerCRD[]> {
return (await listMcpServerSnapshot()).servers
}
/**
* List all Context CRDs in the namespace.
*/
async function listContextSnapshot(): Promise<ContextSnapshot> {
if (!customObjectsApi) {
throw new Error('K8s client not initialized - are you in dev mode?')
}
try {
console.log(`[K8s] Listing all Contexts in namespace ${config.namespace}`)
const response = await customObjectsApi.listNamespacedCustomObject({
group: GROUP,
version: VERSION,
namespace: config.namespace,
plural: PLURAL_CONTEXTS,
})
const list = response as {
metadata?: { resourceVersion?: string }
items: Array<{
metadata: {
name: string
namespace?: string
uid?: string
generation?: number
}
spec: ContextSpec
}>
}
const contexts = list.items.map(item => ({
name: item.metadata.name,
namespace: item.metadata.namespace || config.namespace,
uid: item.metadata.uid,
generation: item.metadata.generation,
spec: item.spec,
}))
console.log(`[K8s] Found ${contexts.length} Context(s)`)
return { contexts, resourceVersion: list.metadata?.resourceVersion }
} catch (error) {
console.error('[K8s] Failed to list Contexts:', error)
throw error
}
}
export async function listAllContexts(): Promise<ContextCRD[]> {
return (await listContextSnapshot()).contexts
}
/**
* List all Host CRDs in host namespace.
*/
async function listHostSnapshot(): Promise<HostSnapshot> {
if (!hostCustomObjectsApi) {
throw new Error('K8s client not initialized - are you in dev mode?')
}
try {
console.log(`[K8s] Listing all Hosts in namespace ${config.hostNamespace}`)
const response = await hostCustomObjectsApi.listNamespacedCustomObject({
group: GROUP,
version: VERSION,
namespace: config.hostNamespace,
plural: PLURAL_HOSTS,
})
const list = response as {
metadata?: { resourceVersion?: string }
items: Array<{
metadata: {
name: string
namespace?: string
uid?: string
generation?: number
resourceVersion?: string
annotations?: Record<string, string>
}
spec: HostSpec
status?: HostCRD['status']
}>
}
const hosts = list.items.map(item => ({
name: item.metadata.name,
namespace: item.metadata.namespace || config.hostNamespace,
uid: item.metadata.uid,
generation: item.metadata.generation,
resourceVersion: item.metadata.resourceVersion,
annotations: item.metadata.annotations,
spec: item.spec,
status: item.status,
}))
console.log(`[K8s] Found ${hosts.length} Host(s)`)
return { hosts, resourceVersion: list.metadata?.resourceVersion }
} catch (error) {
console.error('[K8s] Failed to list Hosts:', error)
throw error
}
}
export async function listAllHosts(): Promise<HostCRD[]> {
return (await listHostSnapshot()).hosts
}
/**
* List all SharedFileSystem CRDs in the mcp-host namespace (the only namespace
* SharedFileSystems are allowed to live in, per CRD validation).
*/
export async function listAllSharedFileSystems(): Promise<SharedFileSystemCRD[]> {
if (!hostCustomObjectsApi) {
throw new Error('K8s client not initialized - are you in dev mode?')
}
try {
console.log(`[K8s] Listing all SharedFileSystems in namespace ${config.hostNamespace}`)
// Deadline-bearing client, like listHostSnapshot: the cold-start Host fleet
// pass waits on this inventory, so an apiserver that never answers would
// otherwise strand the fleet behind an already-certified readiness — and
// leave the SharedFileSystem watch, which starts after this await, dead.
const response = await hostCustomObjectsApi.listNamespacedCustomObject({
group: GROUP,
version: VERSION,
namespace: config.hostNamespace,
plural: PLURAL_SHAREDFILESYSTEMS,
})
const list = response as {
items: Array<{
metadata: { name: string; namespace?: string }
spec: SharedFileSystemSpec
}>
}
const sfses = list.items.map(item => ({
name: item.metadata.name,
namespace: item.metadata.namespace || config.hostNamespace,
spec: item.spec,
}))
console.log(`[K8s] Found ${sfses.length} SharedFileSystem(s)`)
return sfses
} catch (error) {
console.error('[K8s] Failed to list SharedFileSystems:', error)
throw error
}
}
/**
* List the GlobalFileSystem CRDs in the gfs namespace. gfs is a cluster
* singleton (one governed drive); this lists 0 or 1 in the core.
*/
export async function listAllGlobalFileSystems(): Promise<GlobalFileSystemCRD[]> {
if (!customObjectsApi) {
throw new Error('K8s client not initialized - are you in dev mode?')
}
const namespace = gfsDefaultFactoryConfig().gfsNamespace
try {
const response = await customObjectsApi.listNamespacedCustomObject({
group: GROUP,
version: VERSION,
namespace,
plural: PLURAL_GLOBALFILESYSTEMS,
})
const list = response as {
items: Array<{
metadata: { name: string; namespace?: string }
spec: GlobalFileSystemSpec
status?: GlobalFileSystemCRD['status']
}>
}
return list.items.map(item => ({
name: item.metadata.name,
namespace: item.metadata.namespace || namespace,
spec: item.spec,
status: item.status,
}))
} catch (error) {
console.error('[K8s] Failed to list GlobalFileSystems:', error)
throw error
}
}
/**
* List all CommunicationChannel CRDs in the channels namespace.
*
* HCC reads only spec.hostRef to count CCs per Host (#281).
* The per-Host channel-reader pod's own watch handles full CC config
* (telegram[], email[], slack[]).
*/
export async function listAllCommunicationChannels(): Promise<CommunicationChannelCRD[]> {
return (await listCommunicationChannelSnapshot()).channels
}
/**
* List a complete CommunicationChannel snapshot together with the resource
* version from which a watch can continue. The snapshot and watch must be
* paired: a raw watch alone cannot prove that the cache contains channels
* created before the controller connected.
*/
async function listCommunicationChannelSnapshot(): Promise<CommunicationChannelSnapshot> {
if (!customObjectsApi) {
throw new Error('K8s client not initialized - are you in dev mode?')
}
try {
console.log(`[K8s] Listing all CommunicationChannels in namespace ${config.channelsNamespace}`)
const response = await customObjectsApi.listNamespacedCustomObject({
group: GROUP,
version: VERSION,
namespace: config.channelsNamespace,
plural: PLURAL_COMMUNICATIONCHANNELS,
})
const list = response as {
metadata?: { resourceVersion?: string }
items: Array<{
metadata: { name: string; namespace?: string }
spec: { hostRef: string; credentialsSecretRef?: { name: string } }
}>
}
const ccs = list.items.map(item => ({
name: item.metadata.name,
namespace: item.metadata.namespace || config.channelsNamespace,
spec: {
hostRef: item.spec.hostRef,
...(item.spec.credentialsSecretRef?.name
? { credentialsSecretRef: { name: item.spec.credentialsSecretRef.name } }
: {}),
},
}))
console.log(`[K8s] Found ${ccs.length} CommunicationChannel(s)`)
return { channels: ccs, resourceVersion: list.metadata?.resourceVersion }
} catch (error) {
console.error('[K8s] Failed to list CommunicationChannels:', error)
throw error
}
}
/**
* List all LlmHook CRDs in the llm-hooks namespace.
*/
export async function listAllLlmHooks(): Promise<LlmHookCRD[]> {
if (!customObjectsApi) {
throw new Error('K8s client not initialized - are you in dev mode?')
}
try {
console.log(`[K8s] Listing all LlmHooks in namespace ${config.llmHooksNamespace}`)
const response = await customObjectsApi.listNamespacedCustomObject({
group: GROUP,
version: VERSION,
namespace: config.llmHooksNamespace,
plural: PLURAL_LLMHOOKS,
})
const list = response as {
items: Array<{
metadata: {
name: string
namespace?: string
uid?: string
generation?: number
annotations?: Record<string, string>
labels?: Record<string, string>
}
spec: LlmHookSpec
status?: LlmHookCRD['status']
}>
}
const hooks = list.items.map(item => ({
name: item.metadata.name,
namespace: item.metadata.namespace || config.llmHooksNamespace,
uid: item.metadata.uid,
generation: item.metadata.generation,
annotations: item.metadata.annotations,
labels: item.metadata.labels,
spec: item.spec,
status: item.status,
}))
console.log(`[K8s] Found ${hooks.length} LlmHook(s)`)
return hooks
} catch (error) {
console.error('[K8s] Failed to list LlmHooks:', error)
throw error
}
}
/**
* Read a Context CRD by contextId.
* Returns the allowed McpServer names, or null if not found.
*/
export async function getContext(contextId: string): Promise<ContextCRD | null> {
if (!customObjectsApi) {
throw new Error('K8s client not initialized - are you in dev mode?')
}
try {
console.log(`[K8s] Reading Context CRD: ${contextId}`)
const response = await customObjectsApi.getNamespacedCustomObject({
group: GROUP,
version: VERSION,
namespace: config.namespace,
plural: PLURAL_CONTEXTS,
name: contextId,
})
const obj = response as {
metadata: { name: string; namespace?: string }
spec: ContextSpec
}
return {
name: obj.metadata.name,
namespace: obj.metadata.namespace || config.namespace,
spec: obj.spec,
}
} catch (error) {
if (getErrorCode(error) === 404) {
console.warn(`[K8s] Context CRD not found: ${contextId}`)
return null
}
console.error(`[K8s] Failed to read Context CRD:`, error)
throw error
}
}
/**
* McpServer watcher - watches for changes to McpServer CRDs and
* triggers the reconciler to manage Deployments + Services.
*/
export class McpServerWatcher implements McpServerProvider {
private watch: k8s.Watch
private mcpWatchRequest: { abort: () => void } | null = null
private ctxWatchRequest: { abort: () => void } | null = null
private hostWatchRequest: { abort: () => void } | null = null
private sfsWatchRequest: { abort: () => void } | null = null
private gfsWatchRequest: { abort: () => void } | null = null
private ccWatchRequest: { abort: () => void } | null = null
private llmHookWatchRequest: { abort: () => void } | null = null
private servers: Map<string, McpServerCRD> = new Map()
private hosts: Map<string, HostCRD> = new Map()
private contexts: Map<string, ContextCRD> = new Map()
private sharedFileSystems: Map<string, SharedFileSystemCRD> = new Map()
private globalFileSystems: Map<string, GlobalFileSystemCRD> = new Map()
private communicationChannels: Map<string, CommunicationChannelCRD> = new Map()
private llmHooks: Map<string, LlmHookCRD> = new Map()
private mcpWatchGeneration = 0
private contextWatchGeneration = 0
private mcpServerDesiredRevision = 0
private contextDesiredRevision = 0
private hostDesiredRevision = 0
// Readiness covers authoritative revocation, not additive fleet completion.
// Generations bind the completed safety sweep to the exact LIST -> WATCH
// pair that supplied its absence decisions. A recovered watch invalidates
// this marker until the new inventory's orphan-allow sweep completes.
// Sentinel -1 = "no authoritative revocation has certified yet". A real
// desired revision starts at 0, so the readiness gate stays closed until the
// first recordNetworkPolicySafetyCertificate writes a real revision here —
// the startup role the (now removed) generation equalities used to play.
private networkPolicyRevocationContextRevision = -1
private networkPolicyRevocationServerRevision = -1
// Safety certification precedes additive policy work and runtime effects.
// Only successful, current completions discharge these separate obligations.
private completedNetworkPolicyRevision: CompletedInventoryRevision | null = null
private completedRuntimeRevision: CompletedInventoryRevision | null = null
private networkPolicyRepairPending = false
private runtimeRepairPending = false
private pendingMcpServerConvergence: McpServerConvergenceRequest | undefined
private mcpServerRetryRequest: McpServerConvergenceRequest | undefined
private runtimeInventoryGeneration: number | null = null
private readonly pendingNetworkPolicyCauses = new Set<NetworkPolicyConvergenceCause>()
private networkPolicyPassSequence = 0
private activeNetworkPolicyPass: {
passId: number
causes: NetworkPolicyConvergenceCause[]
trailing: boolean
contextRevision: number
serverRevision: number
} | null = null
private mcpServerCacheSynced = false
private contextCacheSynced = false
private mcpServerCacheRecoveryTimer: ReturnType<typeof setTimeout> | null = null
private contextCacheRecoveryTimer: ReturnType<typeof setTimeout> | null = null
private mcpServerCacheRecoveryInFlight: Promise<boolean> | null = null
private contextCacheRecoveryInFlight: Promise<boolean> | null = null
// Instant (Date.now) at which the last SUCCESSFUL recovery re-established
// each inventory watch. attempt*CacheRecovery compares against these to
// apply WATCH_CLOSE_RECOVERY_FLOOR_MS: a close spaced beyond the floor gets
// an immediate re-LIST; a burst close within it falls back to the timer.
private lastMcpServerWatchRecoveryAt = Number.NEGATIVE_INFINITY
private lastContextWatchRecoveryAt = Number.NEGATIVE_INFINITY
private lastHostWatchRecoveryAt = Number.NEGATIVE_INFINITY
// Consecutive recovery FAILURES per hardened lane, driving the exponential
// backoff ladder in schedule*CacheRecovery. Reset to 0 by every successful
// recovery so an isolated later close starts again at the base delay. The
// paired retryAfterMs (captured from an HTTP 429 Retry-After) overrides the
// computed delay for exactly the next scheduled retry.
private mcpServerWatchRecoveryFailures = 0
private mcpServerWatchRecoveryRetryAfterMs: number | undefined
private contextWatchRecoveryFailures = 0
private contextWatchRecoveryRetryAfterMs: number | undefined
private hostWatchRecoveryFailures = 0
private hostWatchRecoveryRetryAfterMs: number | undefined
// A watch callback can still settle after its stream reports completion.
// Host reconciles change the mcp-host pod template, so stale callbacks must
// never replay an older Host spec after a replacement watch is active.
private hostWatchGeneration = 0
// Event revisions fence async retries within the current resourceVersion-
// continuing stream. Watch generation fences callbacks from retired streams.
private hostWatchRevision = 0
private readonly latestHostWatchEventRevisions = new Map<string, number>()
private readonly hostWatchRetryTimers = new Map<string, ReturnType<typeof setTimeout>>()
private readonly hostWatchRetryAttempts = new Map<string, number>()
private hostCacheSynced = false
private hostCacheRecoveryTimer: ReturnType<typeof setTimeout> | null = null
private sfsWatchRestartTimer: ReturnType<typeof setTimeout> | null = null
private gfsWatchRestartTimer: ReturnType<typeof setTimeout> | null = null
// Host watch LIST-to-WATCH recovery is a dedicated, deduplicated operation
// (§10.2). Concurrent recovery signals reuse the in-flight promise so exactly
// one LIST + WATCH is installed. Independent per-Host events no longer share a
// process-wide convergence tail — they enter the per-Host serializer directly.
private hostRecoveryInFlight: ActiveHostInventoryRecovery | null = null
private hostCacheRecoveryIntent: HostInventoryRecoveryRequest | null = null
// Only the cold-start Host fleet pass waits for the initial SFS inventory.
// The inventory itself remains asynchronous with respect to provider.start
// and NetworkPolicy readiness certification.
private initialHostFleetSfsInventory: Promise<boolean> | null = null
// B2: tracks whether the CC snapshot is paired with a continuing watch.
// Set to true only while a complete snapshot is paired with a live watch;
// used by HostReconciler to make fail-closed lifecycle decisions.
private ccCacheSynced = false
private ccWatchGeneration = 0
private ccCacheRecoveryTimer: ReturnType<typeof setTimeout> | null = null
private ccCacheRecoveryInFlight: Promise<boolean> | null = null
private readonly hostFleetScheduler = new HostFleetScheduler({
perform: request => this.performHostFleetReconcile(request),
recordRequest: result => hostFleetRequestsTotal.inc({ result }),
})
private get ccLifecycleGeneration(): number {
return this.hostFleetScheduler.currentLifecycleGeneration
}
private get ccAppliedLifecycleGeneration(): number {
return this.hostFleetScheduler.appliedLifecycleGeneration
}
private hostResyncInFlight: Promise<void> | null = null
private changeCallback?: () => void
private stopped = false
private reconciler: McpServerReconciler
private hostReconciler: HostReconciler
private netPolReconciler: NetworkPolicyReconciler
private llmHookReconciler: LlmHookReconciler
private bindingReconciler: BindingPolicyReconciler
private sharedFileSystemReconciler: SharedFileSystemReconciler
private gfsReconciler: GfsReconciler
private readonly infrastructureTelemetryReporter?: InfrastructureTelemetryReporter
private readonly administrativeOutcomeReporter?: AdministrativeOutcomeReporter
private readonly externalEgressCoordinator = new ExternalEgressConvergenceCoordinator({
listServers: () => [...this.servers.values()],
getCurrentServer: name => this.servers.get(name),
inventoryAuthoritative: () => !this.stopped && this.mcpServerCacheSynced,
sameDesiredRevision: sameMcpServerPolicyRevision,
enqueue: (server, work) => this.enqueueMcpServerReconciliation(server, work),
mutate: (type, server, options) => this.performExternalEgressMutation(type, server, options),
replay: (type, server, retry) =>
this.reconcileMcpServerWatchEvent(type, server, this.mcpWatchGeneration, retry),
externalEgressRefreshMinTtlMs: () => this.netPolReconciler.externalEgressRefreshMinTtlMs,
})
private readonly initialConvergenceRetryTimers = new Map<
InitialConvergenceLane,
ReturnType<typeof setTimeout>
>()
private readonly initialConvergenceRetryAttempts = new Map<InitialConvergenceLane, number>()
private readonly initialConvergenceRuns = new Map<
InitialConvergenceLane,
ActiveInitialConvergenceRun
>()
// Every resource effect serializes by resource identity, so one slow fleet
// member cannot block unrelated live events while same-resource updates
// still retain last-writer ordering.
private readonly mcpServerReconciliationQueues = new Map<string, KeyedReconciliationQueue>()
private readonly contextReconciliationQueues = new Map<string, KeyedReconciliationQueue>()
private readonly sharedFileSystemReconciliationQueues = new Map<
string,
KeyedReconciliationQueue
>()
private readonly globalFileSystemReconciliationQueues = new Map<
string,
KeyedReconciliationQueue
>()
private sharedFileSystemCacheRevision = 0
private globalFileSystemCacheRevision = 0
// Periodic resync timer: K8s watches can drop events on long disconnects;
// running fullReconcile every N minutes guarantees mcp-host-runtime-token Secret
// rotation eventually catches up even after a missed MODIFIED event.
// Disabled when interval <= 0 (tests).
private resyncTimer: ReturnType<typeof setInterval> | null = null
// Periodic SharedFileSystem resync (#592): the SFS watch fires only on SFS CRD
// changes, not on PVC binding / wfc pod readiness, so a SharedFileSystem that
// reported Initializing/Degraded needs a periodic re-reconcile to converge to a
// truthful Ready. Disabled when interval <= 0 (tests).
private sfsResyncTimer: ReturnType<typeof setInterval> | null = null
// Periodic LlmHook resync: drives the reference-counted orphan sweep and
// readiness convergence when the watch drops events (guardrails phase-4 §3).
private llmHookResyncTimer: ReturnType<typeof setInterval> | null = null
// Periodic GlobalFileSystem resync: like SFS, the gfs watch fires only on the
// CRD changing — not on the gfsc writer Deployment becoming Available — so a
// GlobalFileSystem stuck at Initializing converges to Ready (and seeds its
// root directories) once the writer is up. Disabled when interval <= 0 (tests).
private gfsResyncTimer: ReturnType<typeof setInterval> | null = null
// Periodic NetworkPolicy resync (#478): re-enters the coordinated
// single-flight pass. Disabled by default (interval <= 0); startup still
// runs one convergence. Never call netPolReconciler.fullReconcile from here.
private netPolResyncTimer: ReturnType<typeof setInterval> | null = null
// Periodic L0/L1 defaults-only tick (#488). Disabled by default
// (interval <= 0). Never shares the full-pass in-flight skip; write sets
// are disjoint from the typed sweep.
private netPolDefaultsResyncTimer: ReturnType<typeof setInterval> | null = null
// Serializes overlapping defaults-only ticks. Distinct from
// initialConvergenceRuns.get('NetworkPolicy').
private netPolDefaultsOnlyRun: Promise<void> | null = null
// Set by a timer tick so the next coordinated NetworkPolicy pass re-applies
// L0/L1 defaults (class D). Startup and event-driven entry leave this false
// — start() already ran ensureDefaultPolicies as a bootstrap barrier.
private netPolConvergenceEnsureDefaults = false
constructor() {
if (!kc) {
throw new Error('K8s client not initialized - are you in dev mode?')
}
this.watch = new k8s.Watch(kc)
this.reconciler = new McpServerReconciler(kc)
this.reconciler.setInventoryAuthority(() => ({
known: this.mcpServerCacheSynced,
generation: this.mcpWatchGeneration,
}))
this.reconciler.setResolveCurrentServer(name => this.servers.get(name))
const infrastructureTelemetryReporter = config.controlApiBaseUrl
? createInfrastructureTelemetryReporter(config.governedTracingEnabled, {
baseUrl: config.controlApiBaseUrl,
})
: undefined
const administrativeOutcomeReporter = config.controlApiBaseUrl
? createAdministrativeOutcomeReporter(config.governedTracingEnabled, {
baseUrl: config.controlApiBaseUrl,
})
: undefined
this.infrastructureTelemetryReporter = infrastructureTelemetryReporter
this.administrativeOutcomeReporter = administrativeOutcomeReporter
this.hostReconciler = new HostReconciler(kc, {
infrastructureTelemetryReporter: this.infrastructureTelemetryReporter,
administrativeOutcomeReporter: this.administrativeOutcomeReporter,
})
this.netPolReconciler = new NetworkPolicyReconciler(kc, this.servers)
// LlmHook reconciler shares the live hook + host caches so it can recompute
// pod-key member sets and the Host→LlmHook reverse index on every reconcile.
this.llmHookReconciler = new LlmHookReconciler(kc, this.llmHooks, this.hosts)
this.bindingReconciler = new BindingPolicyReconciler(kc, config.namespace)
this.sharedFileSystemReconciler = new SharedFileSystemReconciler(kc)
// gfs (Global File System) — DISTINCT from SharedFileSystem. The reconcile
// logic (GfsReconciler) is K8s-agnostic; K8sGfsApi is the real cluster
// binding. Env-driven factory config (gfs namespace, gfsc image, etc.).
this.gfsReconciler = new GfsReconciler(
new K8sGfsApi(
kc.makeApiClient(k8s.CoreV1Api),
kc.makeApiClient(k8s.AppsV1Api),
kc.makeApiClient(k8s.NetworkingV1Api),
kc.makeApiClient(k8s.PolicyV1Api),
kc.makeApiClient(k8s.CustomObjectsApi)
),
gfsDefaultFactoryConfig(),
new ControlApiGfsSeedClient()
)
// Wire the cross-CRD lookup the HostReconciler needs to mount each
// referenced SharedFileSystem RO into the per-Host mcp-host pod.
this.hostReconciler.setResolveContextMounts(host => this.resolveContextMounts(host))
// Wire the oauth-server probe the HostReconciler uses to gate the
// derive-only `oauth:user-token` runtime scope: true iff the Host's Context
// fronts an enabled `auth.type: oauth` mcp-server. Reuses the same
// Context-scoped allow-list projection as mcp-host discovery.
this.hostReconciler.setHostFrontsOAuthServer(host => this.hostFrontsOAuthServer(host))
// #281: HostReconciler drives channel-reader Deployment replicas from
// the CC count below, populated by the CommunicationChannel watch
// (see startCommunicationChannelWatch).
this.hostReconciler.setCountCommunicationChannels(host => this.countCommunicationChannels(host))
// B2: Wire the CC cache-sync flag so HostReconciler can gate replica
// scale-down on whether the initial CC list has completed.
this.hostReconciler.setIsCommunicationChannelCacheSynced(() =>
this.isCommunicationChannelCacheSynced()
)
// A fleet worker resolves the freshest cached Host at execution (skipping
// entries a delete/newer event has since removed), and orphan cleanup
// compares candidates with the CURRENT inventory rather than a pass snapshot.
this.hostReconciler.setResolveCurrentHost(name => this.hosts.get(name))
// H2: reflect committed lifecycle outcomes onto the CURRENT cache entry —
// but never onto a same-name recreation. uid === undefined fails closed
// (production watch objects always carry uid).
this.hostReconciler.setReflectHostOutcome((name, uid, apply) => {
const cached = this.hosts.get(name)
if (!cached || uid === undefined || cached.uid !== uid) return
apply(cached)
})
// Orphan cleanup requires known watch authority and a stable watch
// generation; expose both so cleanup fail-closes while authority is unknown
// or the watch has been retired mid-pass.
this.hostReconciler.setHostWatchAuthority(() => ({
known: this.hostCacheSynced,
generation: this.hostWatchGeneration,
}))
this.hostReconciler.setHostMutationAuthority(() => ({
known: this.hostCacheSynced && this.contextCacheSynced,
hostRevision: this.hostDesiredRevision,
contextRevision: this.contextDesiredRevision,
}))
this.hostReconciler.setResolveHostMutationDependencies(host => {
const context = this.contexts.get(host.spec.contextRef)
// Lease desired semantics, not cache-object identity or observed status.
// Kubernetes watch events replace objects and status-only changes may
// legitimately advance resourceVersion without changing Host inputs.
const desiredRevision = <T extends { name: string; namespace: string; spec: unknown }>(
resource: T | undefined
) =>
resource
? { name: resource.name, namespace: resource.namespace, spec: resource.spec }
: undefined
const dependencies: unknown[] = [
desiredRevision(context),
this.ccCacheSynced,
this.ccWatchGeneration,
]
for (const ref of context?.spec.sharedFileSystems ?? []) {
const sfs = this.sharedFileSystems.get(ref.name)
dependencies.push(
desiredRevision(sfs),
sfs ? this.sharedFileSystemReconciler.isMountable(sfs) : false
)
}
const channels = this.findCommunicationChannelsByHostRef(host.name).sort((a, b) =>
a.name.localeCompare(b.name)
)
dependencies.push(...channels.map(channel => desiredRevision(channel)))
return dependencies
})
// Per-CC credentials Secret migration: HostReconciler computes the
// channel-reader credentials-revision annotation by hashing the Secret(s)
// referenced by all CCs of a host. Wire both finders so
// patchChannelReaderRevisionAnnotation (CC events) and
// reconcileChannelReaderRevision (Secret rotation events) can look up
// affected hosts from our in-process CC cache.
this.hostReconciler.setFindCommunicationChannelsByHostRef(host =>
this.findCommunicationChannelsByHostRef(host)
)
this.hostReconciler.setFindCommunicationChannelsByCredentialsSecretName(name =>
this.findCommunicationChannelsByCredentialsSecretName(name)
)
// SFS status.mountedByContexts is computed by the SFS reconciler at
// every reconcile by asking us which Contexts currently reference it.
this.sharedFileSystemReconciler.setListContextRefs(sfsName =>
this.contextsReferencingSfs(sfsName)
)
}
/**
* B2: Returns true while the CC cache is backed by a complete snapshot and
* its resourceVersion-continuing watch. HostReconciler gates scale-down and
* stateless eligibility on this flag.
*/
isCommunicationChannelCacheSynced(): boolean {
return this.ccCacheSynced
}
/**
* Readiness authority covers the inventories that back HCC discovery,
* Context policy decisions, and Host API/runtime ownership. Communication
* Channel loss is intentionally excluded: that subsystem already fails safe
* by preserving replicas and disabling stateless suspension until recovery.
* SFS/GFS retain their established eventual-resync contract.
*/
getReadinessInventoryDetail(): ReadinessInventoryDetail {
return {
stopped: this.stopped,
mcpServerCacheSynced: this.mcpServerCacheSynced,
contextCacheSynced: this.contextCacheSynced,
hostCacheSynced: this.hostCacheSynced,
safetyInventoryCertified: this.netPolReconciler.hasCertifiedSafetyInventory(),
contextRevisionAligned:
this.networkPolicyRevocationContextRevision === this.contextDesiredRevision,
serverRevisionAligned:
this.networkPolicyRevocationServerRevision === this.mcpServerDesiredRevision,
}
}
isReadinessInventoryAuthoritative(): boolean {
const detail = this.getReadinessInventoryDetail()
return (
!detail.stopped &&
detail.mcpServerCacheSynced &&
detail.contextCacheSynced &&
detail.hostCacheSynced &&
// Certification is pinned to CONTENT identity (the desired-revision
// counters), not CHANNEL identity (the watch-generation counters). A
// "Premature close" reconnect re-LISTs the same inventory and bumps the
// generation without changing desired state; gating readiness on the
// generation there livelocked the certificate forever under sustained GKE
// watch-churn (the clerum-dev incident) while the underlying safety
// decision stayed valid. The revision only moves when the diffing snapshot
// installer observes a real desired-state change, so a same-content
// reconnect no longer re-closes the gate, while a genuine change still
// forces re-certification. Losing a delete fence bumps no revision, so
// hasCertifiedSafetyInventory() remains the fence for that one failure
// mode that no equality below can see.
detail.safetyInventoryCertified &&
detail.contextRevisionAligned &&
detail.serverRevisionAligned
)
}
/**
* Host-managed runtime mutation requires a LIST snapshot paired with its
* continuing WATCH. This narrower predicate is also used by non-HTTP effect
* producers while the broader readiness predicate additionally covers
* McpServer and Context discovery authority.
*/
isHostInventoryAuthoritative(): boolean {
return !this.stopped && this.hostCacheSynced
}
/**
* Host runtime state is derived from both the Host and Context inventories:
* Context owns the SharedFileSystem mount set rendered into each Host pod.
* Mutation must therefore stop when either LIST -> WATCH pair is unavailable.
*/
private isHostEffectInventoryAuthoritative(): boolean {
return this.isHostInventoryAuthoritative() && this.contextCacheSynced
}
/**
* Fail closed at every Host-effect admission boundary while recovery owns
* inventory authority. Preserve the reason/generation in the retry intent so
* the successful LIST -> WATCH recovery's full pass applies the deferred
* effect from current state instead of acting on the stale cache.
*/
private admitHostDependentEffects(
convergenceReason: string,
ccLifecycleGeneration?: number
): boolean {
if (this.isHostEffectInventoryAuthoritative()) return true
if (!this.stopped && !this.hostCacheSynced) {
this.scheduleHostCacheRecovery({
convergenceReason,
ccLifecycleGeneration,
cause: 'watch-recovery',
})
}
return false
}
private retireMcpServerWatch(expectedGeneration?: number): boolean {
if (expectedGeneration !== undefined && expectedGeneration !== this.mcpWatchGeneration) {
return false
}
this.mcpServerCacheSynced = false
this.mcpWatchGeneration += 1
this.mcpWatchRequest?.abort()
this.mcpWatchRequest = null
return true
}
private retireContextWatch(expectedGeneration?: number): boolean {
if (expectedGeneration !== undefined && expectedGeneration !== this.contextWatchGeneration) {
return false
}
this.contextCacheSynced = false
this.contextWatchGeneration += 1
this.ctxWatchRequest?.abort()
this.ctxWatchRequest = null
return true
}
private async enqueueKeyedReconciliation(
queues: Map<string, KeyedReconciliationQueue>,
key: string,
work: () => Promise<void>
): Promise<void> {
let entry = queues.get(key)
if (!entry) {
entry = { queue: new SerializedReconciliationQueue(), references: 0 }
queues.set(key, entry)
}
entry.references++
try {
await entry.queue.enqueue(async () => {
if (this.stopped) return
await work()
})
} finally {
entry.references--
if (entry.references === 0 && queues.get(key) === entry) queues.delete(key)
}
}
private enqueueMcpServerReconciliation(
server: Pick<McpServerCRD, 'name' | 'namespace'>,
work: () => Promise<void>
): Promise<void> {
return this.enqueueKeyedReconciliation(
this.mcpServerReconciliationQueues,
`${server.namespace}/${server.name}`,
work
)
}
private enqueueContextReconciliation(
contextId: string,
work: () => Promise<void>
): Promise<void> {
return this.enqueueKeyedReconciliation(this.contextReconciliationQueues, contextId, work)
}
private enqueueContextIdentityReconciliation(
contextIds: string[],
work: () => Promise<void>
): Promise<void> {
const keys = [...new Set(contextIds)].sort()
const enter = (index: number): Promise<void> => {
const key = keys[index]
if (!key) return work()
return this.enqueueContextReconciliation(key, () => enter(index + 1))
}
return enter(0)
}
private hasMcpServerInventoryAuthority(watchGeneration: number): boolean {
return this.mcpServerCacheSynced && this.mcpWatchGeneration === watchGeneration
}
private hasContextInventoryAuthority(watchGeneration: number): boolean {
return this.contextCacheSynced && this.contextWatchGeneration === watchGeneration
}
private async mcpServerAbsentForDelete(
name: string,
namespace: string,
watchGeneration: number
): Promise<boolean> {
if (!customObjectsApi) return false
return confirmAuthoritativeMcpServerAbsence({
inventoryAuthoritative: () => this.hasMcpServerInventoryAuthority(watchGeneration),
resolveCurrent: () => this.servers.get(name),
readCurrent: () =>
customObjectsApi.getNamespacedCustomObject({
group: GROUP,
version: VERSION,
namespace,
plural: PLURAL_MCPSERVERS,
name,
}),
})
}
private async authorizeMcpServerDeleteOrRetry(
server: McpServerCRD,
watchGeneration: number
): Promise<boolean> {
try {
const authorized = await this.mcpServerAbsentForDelete(
server.name,
server.namespace,
watchGeneration
)
if (!authorized && !this.hasMcpServerInventoryAuthority(watchGeneration)) {
console.warn(
`[K8s] McpServer inventory authority changed while deleting ${server.name}; ` +
'full cleanup deferred until retry'
)
this.scheduleExternalEgressRetry('DELETED', server)
this.changeCallback?.()
}
return authorized
} catch (error) {
console.error(
`[K8s] Authoritative absence check failed for deleted McpServer ${server.name}; ` +
'cleanup blocked until retry:',
error
)
this.scheduleExternalEgressRetry('DELETED', server)
this.changeCallback?.()
return false
}
}
private async contextAbsentForDelete(
name: string,
namespace: string,
watchGeneration: number
): Promise<boolean> {
if (
!this.contextCacheSynced ||
this.contextWatchGeneration !== watchGeneration ||
this.contexts.has(name) ||
!customObjectsApi
) {
return false
}
try {
await customObjectsApi.getNamespacedCustomObject({
group: GROUP,
version: VERSION,
namespace,
plural: PLURAL_CONTEXTS,
name,
})
return false
} catch (error: unknown) {
if (getErrorCode(error) !== 404) throw error
}
return (
this.contextCacheSynced &&
this.contextWatchGeneration === watchGeneration &&
!this.contexts.has(name)
)
}
private async contextIdAbsentForDelete(
contextId: string,
watchGeneration: number
): Promise<boolean> {
const inventoryAuthoritative = () =>
this.contextCacheSynced && this.contextWatchGeneration === watchGeneration
const resolveCurrent = () =>
[...this.contexts.values()].find(context => context.spec.contextId === contextId)
if (!customObjectsApi || !inventoryAuthoritative() || resolveCurrent()) return false
const response = (await customObjectsApi.listNamespacedCustomObject({
group: GROUP,
version: VERSION,
namespace: config.namespace,
plural: PLURAL_CONTEXTS,
})) as { items?: Array<{ spec?: { contextId?: string } }> }
if ((response.items ?? []).some(context => context.spec?.contextId === contextId)) return false
return inventoryAuthoritative() && !resolveCurrent()
}
private enqueueSharedFileSystemReconciliation(
name: string,
work: () => Promise<void>
): Promise<void> {
return this.enqueueKeyedReconciliation(this.sharedFileSystemReconciliationQueues, name, work)
}
private enqueueGlobalFileSystemReconciliation(
name: string,
work: () => Promise<void>
): Promise<void> {
return this.enqueueKeyedReconciliation(this.globalFileSystemReconciliationQueues, name, work)
}
/**
* Fleet passes acquire every selected object lane in stable order. This
* preserves their snapshot-wide ordering without making two unrelated live
* watch events wait behind one another.
*/
private enqueueKeyedFleetReconciliation(
queues: Map<string, KeyedReconciliationQueue>,
keys: string[],
work: () => Promise<void>
): Promise<void> {
const orderedKeys = [...new Set(keys)].sort()
const enter = (index: number): Promise<void> => {
const key = orderedKeys[index]
if (!key) return work()
return this.enqueueKeyedReconciliation(queues, key, () => enter(index + 1))
}
return enter(0)
}
private enqueueSharedFileSystemFleetReconciliation(
names: string[],
work: () => Promise<void>
): Promise<void> {
return this.enqueueKeyedFleetReconciliation(
this.sharedFileSystemReconciliationQueues,
names,
work
)
}
private enqueueGlobalFileSystemFleetReconciliation(
names: string[],
work: () => Promise<void>
): Promise<void> {
return this.enqueueKeyedFleetReconciliation(
this.globalFileSystemReconciliationQueues,
names,
work
)
}
private requireInventoryResourceVersion(
kind: 'McpServer' | 'Context',
resourceVersion: string | undefined
): string {
if (!resourceVersion) {
throw new Error(`${kind} snapshot missing resourceVersion`)
}
return resourceVersion
}
// A re-LIST after a watch reconnect ("Premature close") reinstalls the whole
// cache. Diff the incoming snapshot against the current cache BEFORE replacing
// it and bump the desired revision iff the desired state actually changed
// (an add, a removal, or a spec change under the canonical comparator). This
// is the counterpart to the watch-event revision bump: it makes the desired
// revision a monotonic CONTENT identity that survives channel loss, so the
// safety certificate can gate on content instead of the churning generation.
// Same-content reconnect → no bump → certification survives the churn. Real
// change carried by the re-LIST → bump → forced re-certification (fail-closed).
private mcpServerSnapshotChangesDesiredState(snapshot: McpServerSnapshot): boolean {
if (snapshot.servers.length !== this.servers.size) return true
for (const server of snapshot.servers) {
const previous = this.servers.get(server.name)
if (previous === undefined || !sameMcpServerPolicyRevision(previous, server)) {
return true
}
}
return false
}
private installMcpServerSnapshot(snapshot: McpServerSnapshot): void {
if (this.mcpServerSnapshotChangesDesiredState(snapshot)) {
this.mcpServerDesiredRevision += 1
}
this.servers.clear()
for (const server of snapshot.servers) {
this.servers.set(server.name, server)
}
}
private contextSnapshotChangesDesiredState(snapshot: ContextSnapshot): boolean {
if (snapshot.contexts.length !== this.contexts.size) return true
for (const context of snapshot.contexts) {
const previous = this.contexts.get(context.name)
if (previous === undefined || !sameContextDesiredRevision(previous, context)) {
return true
}
}
return false
}
private installContextSnapshot(snapshot: ContextSnapshot): void {
if (this.contextSnapshotChangesDesiredState(snapshot)) {
this.contextDesiredRevision += 1
}
this.contexts.clear()
for (const context of snapshot.contexts) {
this.contexts.set(context.name, context)
}
}
private async restartMcpServerWatch(snapshot: McpServerSnapshot): Promise<number> {
const resourceVersion = this.requireInventoryResourceVersion(
'McpServer',
snapshot.resourceVersion
)
this.mcpServerCacheSynced = false
this.installMcpServerSnapshot(snapshot)
const watchGeneration = await this.startMcpServerWatch(resourceVersion)
if (
this.stopped ||
watchGeneration !== this.mcpWatchGeneration ||
this.mcpWatchRequest === null
) {
throw new Error('McpServer snapshot could not be paired with an active watch')
}
this.mcpServerCacheSynced = true
return watchGeneration
}
private async restartContextWatch(snapshot: ContextSnapshot): Promise<number> {
const resourceVersion = this.requireInventoryResourceVersion(
'Context',
snapshot.resourceVersion
)
this.contextCacheSynced = false
this.installContextSnapshot(snapshot)
const watchGeneration = await this.startContextWatch(resourceVersion)
if (
this.stopped ||
watchGeneration !== this.contextWatchGeneration ||
this.ctxWatchRequest === null
) {
throw new Error('Context snapshot could not be paired with an active watch')
}
this.contextCacheSynced = true
return watchGeneration
}
private recoverMcpServerInventoryAndWatch(): Promise<boolean> {
if (this.stopped) return Promise.resolve(false)
if (this.mcpServerCacheRecoveryInFlight) return this.mcpServerCacheRecoveryInFlight
const recovery = (async () => {
const before = this.currentDesiredInventoryRevision()
try {
const snapshot = await listMcpServerSnapshot()
if (this.stopped) return false
await this.restartMcpServerWatch(snapshot)
if (this.stopped) return false
this.lastMcpServerWatchRecoveryAt = Date.now()
this.mcpServerWatchRecoveryFailures = 0
this.mcpServerWatchRecoveryRetryAfterMs = undefined
this.changeCallback?.()
if (this.shouldRequestNetworkPolicyRecovery('McpServer', before)) {
void this.runInitialNetworkPolicyConvergence({ cause: 'mcp-recovery' })
}
return true
} catch (error) {
this.mcpServerCacheSynced = false
this.mcpServerWatchRecoveryFailures += 1
this.mcpServerWatchRecoveryRetryAfterMs = getRetryAfterMs(error)
hccLogger.error('[K8s] McpServer cache recovery failed:', { err: error })
return false
}
})()
this.mcpServerCacheRecoveryInFlight = recovery
void recovery.finally(() => {
if (this.mcpServerCacheRecoveryInFlight === recovery) {
this.mcpServerCacheRecoveryInFlight = null
}
})
return recovery
}
private recoverContextInventoryAndWatch(): Promise<boolean> {
if (this.stopped) return Promise.resolve(false)
if (this.contextCacheRecoveryInFlight) return this.contextCacheRecoveryInFlight
const recovery = (async () => {
const before = this.currentDesiredInventoryRevision()
try {
const snapshot = await listContextSnapshot()
if (this.stopped) return false
await this.restartContextWatch(snapshot)
if (this.stopped) return false
this.lastContextWatchRecoveryAt = Date.now()
this.contextWatchRecoveryFailures = 0
this.contextWatchRecoveryRetryAfterMs = undefined
this.hostFleetScheduler.bumpInputRevision()
if (this.shouldRequestNetworkPolicyRecovery('Context', before)) {
void this.runInitialNetworkPolicyConvergence({ cause: 'context-recovery' })
}
void this.runInitialSharedFileSystemConvergence()
if (this.hostCacheSynced) {
const pendingCcLifecycleGeneration =
this.ccLifecycleGeneration > this.ccAppliedLifecycleGeneration
? this.ccLifecycleGeneration
: undefined
void this.requestHostFleetReconcile(
'Context cache recovery',
pendingCcLifecycleGeneration,
'full'
)
}
return true
} catch (error) {
this.contextCacheSynced = false
this.contextWatchRecoveryFailures += 1
this.contextWatchRecoveryRetryAfterMs = getRetryAfterMs(error)
hccLogger.error('[K8s] Context cache recovery failed:', { err: error })
return false
}
})()
this.contextCacheRecoveryInFlight = recovery
void recovery.finally(() => {
if (this.contextCacheRecoveryInFlight === recovery) {
this.contextCacheRecoveryInFlight = null
}
})
return recovery
}
/**
* One recovery attempt NOW (immediate re-LIST on watch close), with the
* jittered exponential-backoff timer kept purely as retry-after-failure
* pacing. The anti-busy-loop floor demotes a close that arrives within
* WATCH_CLOSE_RECOVERY_FLOOR_MS of the last successful recovery back onto
* the paced timer. Dedup against concurrent attempts lives in
* recoverMcpServerInventoryAndWatch (mcpServerCacheRecoveryInFlight).
*/
private attemptMcpServerCacheRecovery(): void {
if (this.stopped) return
if (Date.now() - this.lastMcpServerWatchRecoveryAt < WATCH_CLOSE_RECOVERY_FLOOR_MS) {
this.scheduleMcpServerCacheRecovery()
return
}
void this.recoverMcpServerInventoryAndWatch().then(recovered => {
if (!recovered) this.scheduleMcpServerCacheRecovery()
})
}
/** Context analogue of attemptMcpServerCacheRecovery. */
private attemptContextCacheRecovery(): void {
if (this.stopped) return
if (Date.now() - this.lastContextWatchRecoveryAt < WATCH_CLOSE_RECOVERY_FLOOR_MS) {
this.scheduleContextCacheRecovery()
return
}
void this.recoverContextInventoryAndWatch().then(recovered => {
if (!recovered) this.scheduleContextCacheRecovery()
})
}
/**
* Host analogue: one attempt NOW unless inside the floor. The retry re-arm
* on failure lives in performHostInventoryRecovery (scheduleHostCacheRecovery
* before rethrow) and the intent/in-flight merge lives in
* recoverHostInventoryAndWatch, so this helper only decides immediate vs
* paced.
*/
private attemptHostCacheRecovery(): void {
if (this.stopped) return
if (Date.now() - this.lastHostWatchRecoveryAt < WATCH_CLOSE_RECOVERY_FLOOR_MS) {
this.scheduleHostCacheRecovery()
return
}
void this.recoverHostInventoryAndWatch().catch(() => undefined)
}
private scheduleMcpServerCacheRecovery(): void {
if (this.stopped || this.mcpServerCacheRecoveryTimer) return
const delayMs = computeWatchRecoveryRetryDelayMs(
this.mcpServerWatchRecoveryFailures,
this.mcpServerWatchRecoveryRetryAfterMs
)
// Retry-After applies to exactly the retry it throttled; the next failure
// recaptures it (or falls back to the backoff ladder).
this.mcpServerWatchRecoveryRetryAfterMs = undefined
this.mcpServerCacheRecoveryTimer = setTimeout(() => {
this.mcpServerCacheRecoveryTimer = null
this.attemptMcpServerCacheRecovery()
}, delayMs)
}
private scheduleContextCacheRecovery(): void {
if (this.stopped || this.contextCacheRecoveryTimer) return
const delayMs = computeWatchRecoveryRetryDelayMs(
this.contextWatchRecoveryFailures,
this.contextWatchRecoveryRetryAfterMs
)
this.contextWatchRecoveryRetryAfterMs = undefined
this.contextCacheRecoveryTimer = setTimeout(() => {
this.contextCacheRecoveryTimer = null
this.attemptContextCacheRecovery()
}, delayMs)
}
private installCommunicationChannelSnapshot(snapshot: CommunicationChannelSnapshot): void {
this.communicationChannels.clear()
for (const channel of snapshot.channels) {
this.communicationChannels.set(channel.name, channel)
}
}
private beginCommunicationChannelLifecycleTransition(): number {
return this.hostFleetScheduler.beginLifecycleTransition()
}
private markCommunicationChannelLifecycleApplied(generation: number): void {
this.hostFleetScheduler.markLifecycleApplied(generation)
}
private scheduleCommunicationChannelFleetRetry(request: HostFleetReconcileRequest): void {
this.hostFleetScheduler.scheduleLifecycleRetry(request)
}
/**
* Rebuild the channel cache from an authoritative list after startup or watch
* recovery. Until the paired watch is connected, keep stateless eligibility
* fail-closed; a partial stream cannot prove there are no channels.
*/
private async recoverCommunicationChannelCache(): Promise<boolean> {
if (this.stopped) return false
if (this.ccCacheRecoveryInFlight) return this.ccCacheRecoveryInFlight
if (this.ccCacheRecoveryTimer) {
clearTimeout(this.ccCacheRecoveryTimer)
this.ccCacheRecoveryTimer = null
}
const recovery = (async (): Promise<boolean> => {
this.ccCacheSynced = false
try {
const snapshot = await listCommunicationChannelSnapshot()
if (this.stopped) return false
const watchGeneration = await this.restartCommunicationChannelWatch(snapshot)
if (
this.stopped ||
watchGeneration !== this.ccWatchGeneration ||
this.ccWatchRequest === null
) {
return false
}
this.ccCacheSynced = true
const lifecycleGeneration = this.beginCommunicationChannelLifecycleTransition()
console.log(
`[K8s] Recovered ${snapshot.channels.length} CommunicationChannel(s) into cache ` +
'(ccCacheSynced=true)'
)
void this.requestHostFleetReconcile('CommunicationChannel recovery', lifecycleGeneration)
return true
} catch (error) {
hccLogger.error(
'[K8s] CommunicationChannel cache recovery failed; preserving durable Host lifecycle state:',
{ err: error }
)
return false
}
})()
this.ccCacheRecoveryInFlight = recovery
try {
return await recovery
} finally {
if (this.ccCacheRecoveryInFlight === recovery) {
this.ccCacheRecoveryInFlight = null
}
if (!this.stopped && !this.ccCacheSynced) {
this.scheduleCommunicationChannelCacheRecovery()
}
}
}
private scheduleCommunicationChannelCacheRecovery(): void {
if (this.stopped || this.ccCacheRecoveryTimer) return
this.ccCacheRecoveryTimer = setTimeout(() => {
this.ccCacheRecoveryTimer = null
void this.recoverCommunicationChannelCache()
}, COMMUNICATION_CHANNEL_CACHE_RECOVERY_RETRY_MS)
}
private clearAllHostWatchRetries(): void {
for (const timer of this.hostWatchRetryTimers.values()) clearTimeout(timer)
this.hostWatchRetryTimers.clear()
this.hostWatchRetryAttempts.clear()
this.latestHostWatchEventRevisions.clear()
}
private retireHostWatch(): void {
this.hostCacheSynced = false
this.hostWatchGeneration += 1
if (this.hostWatchRequest) {
this.hostWatchRequest.abort()
this.hostWatchRequest = null
}
this.clearAllHostWatchRetries()
}
// Host content identity for the diffing installer and the watch-event bump.
// Kubernetes bumps metadata.generation only on a spec change (not on the
// status writes HCC itself makes), and the uid distinguishes a delete+recreate
// of the same name. Same (uid, generation) = same desired Host state.
// SCOPE: unlike the McpServer/Context comparators (which hash spec+labels+
// annotations), this is blind to Host annotations/labels — correct today
// because no Host annotation/label is load-bearing for a mutation (the wake
// annotation drives dispatch, not template content). If a Host annotation or
// label ever becomes mutation-relevant, extend this to hash it, or a
// reconnect that re-LISTs an unchanged (uid, generation) would skip the bump.
private sameHostDesiredRevision(previous: HostCRD, current: HostCRD): boolean {
return previous.uid === current.uid && previous.generation === current.generation
}
private hostSnapshotChangesDesiredState(snapshot: HostSnapshot): boolean {
if (snapshot.hosts.length !== this.hosts.size) return true
for (const host of snapshot.hosts) {
const previous = this.hosts.get(host.name)
if (previous === undefined || !this.sameHostDesiredRevision(previous, host)) return true
}
return false
}
private installHostSnapshot(snapshot: HostSnapshot): void {
// Same diffing rule as the Context/McpServer installers: a Premature-close
// reconnect that re-LISTs the identical Host inventory leaves hostDesiredRevision
// untouched, so the Host mutation-authority fence no longer starves every
// queued reconcile with HostInventoryAuthorityUnavailableError under churn.
if (this.hostSnapshotChangesDesiredState(snapshot)) this.hostDesiredRevision += 1
this.hosts.clear()
for (const host of snapshot.hosts) this.hosts.set(host.name, host)
}
/**
* Dedicated, deduplicated Host watch LIST-to-WATCH recovery (§10.2). Watch
* authority recovers independently of the background fleet convergence pass:
* concurrent recovery signals reuse the in-flight promise so exactly one LIST
* + WATCH is installed, and authority is declared known ONLY after the WATCH
* is established. On success it enqueues discovered new/changed/wake-pending
* Hosts through their per-Host chains (step 7) and requests — without awaiting
* — a coalesced background full pass (step 8).
*/
private recoverHostInventoryAndWatch(
convergenceReason = 'Host watch recovery convergence',
ccLifecycleGeneration?: number,
cause: HostInventoryRecoveryCause = 'watch-recovery'
): Promise<HostCRD[]> {
if (this.stopped) return Promise.resolve([])
const requested = { convergenceReason, ccLifecycleGeneration, cause }
// A scheduled retry and an immediate caller are two entrances to the same
// monotonic recovery intent. Adopt the pending request before choosing or
// joining the active operation so a direct recovery cannot erase a stronger
// cold-start cause or newer lifecycle generation when it cancels the timer.
const request = this.hostCacheRecoveryIntent
? { ...this.hostCacheRecoveryIntent }
: { ...requested }
if (this.hostCacheRecoveryIntent) {
mergeHostInventoryRecoveryRequest(request, requested)
this.hostCacheRecoveryIntent = null
if (this.hostCacheRecoveryTimer) {
clearTimeout(this.hostCacheRecoveryTimer)
this.hostCacheRecoveryTimer = null
}
}
if (this.hostRecoveryInFlight) {
mergeHostInventoryRecoveryRequest(this.hostRecoveryInFlight.request, request)
return this.hostRecoveryInFlight.promise
}
const recovery = this.performHostInventoryRecovery(request)
const active = { request, promise: recovery }
this.hostRecoveryInFlight = active
const clear = (): void => {
if (this.hostRecoveryInFlight === active) this.hostRecoveryInFlight = null
}
void recovery.then(clear, clear)
return recovery
}
private async performHostInventoryRecovery(
request: HostInventoryRecoveryRequest
): Promise<HostCRD[]> {
const startedAt = Date.now()
// Capture the pre-recovery inventory so step 7 can distinguish genuinely
// new/changed Hosts from ones the fleet pass will converge anyway.
const previousNames = new Set(this.hosts.keys())
const previousGenerations = new Map<string, number | undefined>(
[...this.hosts.values()].map(host => [host.name, host.generation])
)
// #827: capture identities so recovery can tell a same-name recreation from
// a genuine disappearance and reconcile the former rather than delete it.
const previousUids = new Map<string, string | undefined>(
[...this.hosts.values()].map(host => [host.name, host.uid])
)
this.retireHostWatch()
try {
const listStartedAt = Date.now()
const snapshot = await listHostSnapshot()
if (this.stopped) return []
if (!snapshot.resourceVersion) {
throw new Error('Host snapshot missing resourceVersion')
}
hostWatchRecoverySeconds.observe(
{ phase: 'list', outcome: 'success' },
(Date.now() - listStartedAt) / 1000
)
this.installHostSnapshot(snapshot)
const watchStartedAt = Date.now()
const watchGeneration = await this.startHostWatch(snapshot.resourceVersion)
if (
this.stopped ||
watchGeneration !== this.hostWatchGeneration ||
this.hostWatchRequest === null
) {
throw new Error('Host snapshot could not be paired with an active watch')
}
// Authority is known ONLY after the WATCH is installed.
this.hostCacheSynced = true
this.lastHostWatchRecoveryAt = Date.now()
this.hostWatchRecoveryFailures = 0
this.hostWatchRecoveryRetryAfterMs = undefined
hostWatchRecoverySeconds.observe(
{ phase: 'watch', outcome: 'success' },
(Date.now() - watchStartedAt) / 1000
)
hostWatchRecoverySeconds.observe(
{ phase: 'total', outcome: 'success' },
(Date.now() - startedAt) / 1000
)
// Effect producers can discover newer work while LIST/WATCH recovery is
// already in flight. Fold that scheduled intent into this successful
// operation before cancelling its timer so no lifecycle generation,
// convergence reason, or stronger cold-start cause is discarded.
if (this.hostCacheRecoveryIntent) {
mergeHostInventoryRecoveryRequest(request, this.hostCacheRecoveryIntent)
}
if (this.hostCacheRecoveryTimer) {
clearTimeout(this.hostCacheRecoveryTimer)
this.hostCacheRecoveryTimer = null
}
this.hostCacheRecoveryIntent = null
if (request.cause === 'watch-recovery') {
this.enqueueRecoveredUrgentHosts(
snapshot.hosts,
previousNames,
previousGenerations,
previousUids
)
// Addendum 4 (#827): a Host present before recovery and absent from the
// fresh authoritative snapshot genuinely disappeared. Enqueue an immediate
// per-Host delete cleanup so a DELETE lost to watch retirement is never
// silently dropped.
this.enqueueRecoveredHostDeletes(snapshot.hosts, previousNames)
}
// Request a coalesced background full pass, but do NOT await it before
// declaring watch recovery complete. The cold-start pass alone waits for
// the concurrent SFS LIST: otherwise it can write every Host template
// without its declared mounts, then immediately roll the fleet again
// when SFS discovery finishes. NetworkPolicy readiness and start() stay
// independent from that finite inventory request.
const requestFleetPass = () => {
if (this.stopped) return
void this.requestHostFleetReconcile(
request.convergenceReason,
request.ccLifecycleGeneration,
'full'
)
}
const initialSfsInventory =
request.cause === 'cold-start' ? this.initialHostFleetSfsInventory : null
if (initialSfsInventory) {
// Waiting for the inventory is deliberate: a Host fleet pass without it
// writes mount-less templates and immediately rerolls the whole fleet.
// The wait cannot strand the fleet, because the LIST underneath runs on
// the deadline-bearing client: an apiserver that never answers aborts
// the request, `startSharedFileSystemInventoryAndWatch` logs and
// resolves false, and this settles. One path, one pass.
void initialSfsInventory.then(() => requestFleetPass())
} else {
requestFleetPass()
}
return snapshot.hosts
} catch (error) {
// Keep authority unknown, record the failure, and use the existing
// bounded retry policy.
this.hostCacheSynced = false
hostWatchRecoverySeconds.observe(
{ phase: 'total', outcome: 'failure' },
(Date.now() - startedAt) / 1000
)
if (!this.stopped) {
this.hostWatchRecoveryFailures += 1
this.hostWatchRecoveryRetryAfterMs = getRetryAfterMs(error)
console.error('[K8s] Host watch recovery failed:', error)
this.scheduleHostCacheRecovery(request)
}
throw error
}
}
/**
* §10.2 step 7: give genuinely new, changed, or wake-pending Hosts immediate
* per-Host admission after recovery rather than waiting for the background
* fleet pass. Each dispatch resolves the CURRENT cached Host at execution and
* enters that Host's serializer, so it is ordered against any concurrent watch
* event for the same Host.
*/
private enqueueRecoveredUrgentHosts(
hosts: HostCRD[],
previousNames: Set<string>,
previousGenerations: Map<string, number | undefined>,
previousUids: Map<string, string | undefined>
): void {
for (const host of hosts) {
const isNew = !previousNames.has(host.name)
const previousGeneration = previousGenerations.get(host.name)
const changed =
previousGeneration !== undefined &&
host.generation !== undefined &&
host.generation !== previousGeneration
// #827: a same-name Host recreated during the watch gap carries a fresh
// uid; its generation may even reset to 1 and look "unchanged", so an
// identity change is itself urgent work.
const previousUid = previousUids.get(host.name)
const recreated =
previousUid !== undefined && host.uid !== undefined && host.uid !== previousUid
const wakePending = host.annotations?.[WAKE_REQUESTED_ANNOTATION] !== undefined
if (!isNew && !changed && !recreated && !wakePending) continue
void this.dispatchUrgentHostReconcile(host.name)
}
}
private async dispatchUrgentHostReconcile(name: string): Promise<void> {
if (this.stopped) return
if (!this.admitHostDependentEffects('Recovered urgent Host convergence')) return
const host = this.hosts.get(name)
if (!host) return
try {
await this.hostReconciler.reconcile(host, 'urgent')
} catch (error) {
console.error(`[K8s] Urgent Host reconcile failed for "${name}":`, error)
}
}
/**
* §10.2 + Addendum 4 (#827): a Host present in the pre-recovery inventory but
* ABSENT from the fresh authoritative snapshot has genuinely disappeared. An
* accepted DELETED event invalidated when the watch retired is thereby
* replaced by this authoritative absence check and never silently lost. Each
* confirmed deletion is dispatched IMMEDIATELY through its own per-Host chain
* (reconcileDelete serializes internally) — never queued behind other Hosts'
* convergence; the compensating full-pass sweep remains only a safety net. A
* same-name recreation is present in the snapshot by name and so is never in
* the disappeared set (it is reconciled by enqueueRecoveredUrgentHosts).
*/
private enqueueRecoveredHostDeletes(hosts: HostCRD[], previousNames: Set<string>): void {
const snapshotNames = new Set(hosts.map(host => host.name))
for (const name of previousNames) {
if (snapshotNames.has(name)) continue // still present (or recreated) → not a disappearance
hostDeleteCleanupTotal.inc({ outcome: 'queued' })
void this.dispatchRecoveredHostDelete(name)
}
}
private async dispatchRecoveredHostDelete(name: string): Promise<void> {
if (this.stopped) return
if (!this.admitHostDependentEffects('Recovered Host deletion convergence')) return
// The fresh authoritative LIST already confirmed absence and the cache no
// longer holds this Host, so cleanup is authorized. Route it through the
// per-Host serializer (reconcileDelete → serializeByHost) so an older
// in-flight reconcile for the same Host cannot recreate its resources after
// cleanup.
hostDeleteCleanupTotal.inc({ outcome: 'confirmed' })
try {
// F2/#827 TOCTOU parity: absence was resolved at LIST time, but the
// recreation's ADDED can be delivered while the watch replays from the
// snapshot's resourceVersion — that callback sets `this.hosts`
// SYNCHRONOUSLY and enters this Host's chain FIRST, while this
// fire-and-forget dispatch queues the delete SECOND. serializeByHost is
// strict FIFO, so without this fence the rebuilt bundle (workspace PVC,
// per-Host RBAC, runtime-token Secret, channel-reader resources,
// NetworkPolicies) would be created and then wiped. Ownership labels
// cannot discriminate — the rebuilt bundle carries identical labels — and
// a name absent from the snapshot has no uid to compare, so live-cache
// presence AT ADMISSION is the fence, exactly as in the F2 orphan sweep.
await this.hostReconciler.reconcileDelete(name, config.hostNamespace, {
skipIf: () => this.hosts.has(name),
})
hostDeleteCleanupTotal.inc({ outcome: 'completed' })
} catch (error) {
hostDeleteCleanupTotal.inc({ outcome: 'retried' })
console.error(
`[K8s] Recovered Host delete cleanup failed for "${name}"; the safety-net sweep will retry:`,
error
)
}
}
private scheduleHostCacheRecovery(
requested: HostInventoryRecoveryRequest = {
convergenceReason: 'Host watch recovery convergence',
cause: 'watch-recovery',
}
): void {
if (this.stopped) return
if (this.hostCacheRecoveryIntent) {
mergeHostInventoryRecoveryRequest(this.hostCacheRecoveryIntent, requested)
} else {
this.hostCacheRecoveryIntent = { ...requested }
}
if (this.hostCacheRecoveryTimer) return
const delayMs = computeWatchRecoveryRetryDelayMs(
this.hostWatchRecoveryFailures,
this.hostWatchRecoveryRetryAfterMs
)
this.hostWatchRecoveryRetryAfterMs = undefined
this.hostCacheRecoveryTimer = setTimeout(() => {
this.hostCacheRecoveryTimer = null
const request = this.hostCacheRecoveryIntent ?? requested
this.hostCacheRecoveryIntent = null
// Watch recovery is an independent operation, no longer coupled to a full
// fleet pass. Recovery itself requests the background convergence pass.
void this.recoverHostInventoryAndWatch(
request.convergenceReason,
request.ccLifecycleGeneration,
request.cause
).catch(() => undefined)
}, delayMs)
}
private isCurrentHostWatchEvent(
type: HostWatchEventType,
host: HostCRD,
eventRevision: number
): boolean {
if (this.latestHostWatchEventRevisions.get(host.name) !== eventRevision) return false
if (type === 'DELETED') return !this.hosts.has(host.name)
const current = this.hosts.get(host.name)
if (!current) return false
// #827 identity fence: a same-name Host that was deleted and recreated
// carries a fresh metadata.uid. If the cache already holds a DIFFERENT
// identity for this name, this event describes a superseded object and must
// not act on the current one. uid complements (never replaces) the
// generation fence below and the §10.5 fresh read on the cleanup path.
if (current.uid !== undefined && host.uid !== undefined && current.uid !== host.uid) {
return false
}
if (current.generation === undefined || host.generation === undefined) return true
return current.generation <= host.generation
}
private clearHostWatchRetry(name: string): void {
const timer = this.hostWatchRetryTimers.get(name)
if (timer) clearTimeout(timer)
this.hostWatchRetryTimers.delete(name)
this.hostWatchRetryAttempts.delete(name)
}
private completeHostWatchEvent(name: string, eventRevision: number): void {
if (this.latestHostWatchEventRevisions.get(name) !== eventRevision) return
this.clearHostWatchRetry(name)
this.latestHostWatchEventRevisions.delete(name)
}
private async reconcileHostWatchEvent(
type: HostWatchEventType,
host: HostCRD,
eventRevision: number,
source: HostReconcileSource = 'urgent'
): Promise<void> {
if (this.stopped) return
if (!this.admitHostDependentEffects('Host watch event convergence')) return
if (!this.isCurrentHostWatchEvent(type, host, eventRevision)) {
// #827: a DELETE that is no longer current because the cache now holds a
// (recreated) same-name Host is a superseded delete — record it and do
// NOT delete the new identity's resources.
if (type === 'DELETED' && this.hosts.has(host.name)) {
hostDeleteCleanupTotal.inc({ outcome: 'superseded' })
}
return
}
if (type === 'DELETED') {
await this.hostReconciler.reconcileDelete(host.name, host.namespace)
} else {
await this.hostReconciler.reconcile(host, source)
}
}
private scheduleHostWatchReconcileRetry(
type: HostWatchEventType,
host: HostCRD,
eventRevision: number
): void {
if (
this.stopped ||
!this.isCurrentHostWatchEvent(type, host, eventRevision) ||
this.hostWatchRetryTimers.has(host.name)
) {
return
}
const attempt = (this.hostWatchRetryAttempts.get(host.name) ?? 0) + 1
const retryDelay = HOST_WATCH_RECONCILE_RETRY_DELAYS_MS[attempt - 1]
if (retryDelay === undefined) {
console.error(
`[K8s] Host watch reconciliation retry exhausted for ${host.name} after ${HOST_WATCH_RECONCILE_RETRY_DELAYS_MS.length} attempts`
)
this.completeHostWatchEvent(host.name, eventRevision)
return
}
this.hostWatchRetryAttempts.set(host.name, attempt)
console.warn(
`[K8s] Scheduling Host watch reconciliation retry ${attempt}/${HOST_WATCH_RECONCILE_RETRY_DELAYS_MS.length} ` +
`for ${host.name} in ${retryDelay}ms`
)
const timer = setTimeout(() => {
this.hostWatchRetryTimers.delete(host.name)
void this.retryHostWatchReconcile(type, host, eventRevision)
}, retryDelay)
this.hostWatchRetryTimers.set(host.name, timer)
}
private async retryHostWatchReconcile(
type: HostWatchEventType,
host: HostCRD,
eventRevision: number
): Promise<void> {
if (this.stopped || !this.isCurrentHostWatchEvent(type, host, eventRevision)) {
this.completeHostWatchEvent(host.name, eventRevision)
return
}
try {
// F1: a retry is a distinct reconcile lane from first-attempt load — an
// operator must be able to tell a retry storm from fresh admission.
await this.reconcileHostWatchEvent(type, host, eventRevision, 'retry')
this.completeHostWatchEvent(host.name, eventRevision)
} catch (error) {
console.error(`[K8s] Host watch reconciliation retry failed for ${host.name}:`, error)
this.scheduleHostWatchReconcileRetry(type, host, eventRevision)
}
}
private performHostFleetReconcile(request: HostFleetReconcileRequest): Promise<void> {
return this.performHostFleetReconcileOnce(request)
}
private async performHostFleetReconcileOnce(request: HostFleetReconcileRequest): Promise<void> {
if (this.stopped) return
const { reason, mode, ccLifecycleGeneration } = request
if (
mode === 'lifecycle' &&
ccLifecycleGeneration !== undefined &&
ccLifecycleGeneration !== this.ccLifecycleGeneration
) {
return
}
try {
// Watch authority recovers as an independent, deduplicated operation. A
// fleet pass only needs to ensure the LIST -> WATCH is established when
// continuity was lost; it never drives the watch restart on every pass
// (that would re-LIST forever). Recovery, once synced, requests its own
// background convergence pass.
if (!this.hostCacheSynced) {
await this.recoverHostInventoryAndWatch()
if (this.stopped) return
}
if (!this.isHostEffectInventoryAuthoritative()) return
// Reconcile the CURRENT cache. Fleet workers dispatch Host keys and
// resolve the freshest cached spec at execution (see collectHostReconcile
// Failures); only fullReconcile runs the authority-gated orphan cleanup.
const hosts = [...this.hosts.values()]
if (mode === 'full') {
console.log(`[K8s] Reconciling ${hosts.length} Host(s) after ${reason}`)
await this.hostReconciler.fullReconcile(hosts)
} else {
if (
this.stopped ||
(ccLifecycleGeneration !== undefined &&
ccLifecycleGeneration !== this.ccLifecycleGeneration)
) {
return
}
console.log(`[K8s] Reconciling ${hosts.length} Host(s) for lifecycle after ${reason}`)
await this.hostReconciler.reconcileHosts(hosts)
}
if (!this.stopped) {
if (ccLifecycleGeneration !== undefined) {
this.markCommunicationChannelLifecycleApplied(ccLifecycleGeneration)
}
console.log(`[K8s] Completed Host reconciliation after ${reason}`)
}
} catch (error) {
hostFleetRequestsTotal.inc({ result: 'failed' })
console.error(`[K8s] Host reconciliation after ${reason} failed:`, error)
if (!this.stopped && ccLifecycleGeneration !== undefined) {
if (error instanceof HostFleetReconcileError && error.hostFailures.length === 0) {
hostFleetLifecycleCatchTotal.inc({ decision: 'applied' })
this.markCommunicationChannelLifecycleApplied(ccLifecycleGeneration)
} else {
// Retry Host convergence without coupling the lifecycle ladder to
// orphan cleanup. A later periodic full pass retries cleanup work.
hostFleetLifecycleCatchTotal.inc({ decision: 'retry' })
this.scheduleCommunicationChannelFleetRetry({
...request,
mode: error instanceof HostFleetReconcileError ? 'lifecycle' : request.mode,
})
}
}
} finally {
// The paired watch can end while fullReconcile is still active. Its
// timer may then be coalesced into this same pass, so every exit must
// re-arm recovery while inventory continuity remains unknown.
if (!this.stopped && !this.hostCacheSynced) {
this.scheduleHostCacheRecovery()
}
}
}
/**
* Cache transitions, periodic resyncs, and retries can request a fleet pass
* concurrently. Generic and covered same-generation requests reuse existing
* work; a full inventory pass queues behind lifecycle-only work when needed.
* Only the latest newer lifecycle generation may occupy one trailing slot.
* Each caller waits only for the pass that covers its request, so startup
* cannot be extended forever by a sustained retry stream.
*/
private requestHostFleetReconcile(
reason: string,
ccLifecycleGeneration?: number,
mode?: HostFleetReconcileMode
): Promise<void> {
return this.hostFleetScheduler.request(reason, ccLifecycleGeneration, mode)
}
private contextsReferencingSfs(sfsName: string): Array<{ namespace: string; name: string }> {
const out: Array<{ namespace: string; name: string }> = []
for (const ctx of this.contexts.values()) {
if (ctx.spec.sharedFileSystems?.some(r => r.name === sfsName)) {
out.push({ namespace: ctx.namespace, name: ctx.name })
}
}
return out
}
/**
* Count CommunicationChannels whose spec.hostRef matches the given host name.
*
* Wired into HostReconciler via setCountCommunicationChannels so
* buildChannelReaderDeployment can drive per-Host channel-reader Deployment
* replicas based on whether the Host has any active channels (#281).
*
* Returns 0 for unknown hosts — safe fallback when:
* - CC ADDED event arrives before its Host CRD is in HCC's cache (race;
* fullReconcile will catch up on next cycle).
* - Host has genuinely no CCs (the intended scale-to-0 case).
*/
countCommunicationChannels(hostName: string): number {
let n = 0
for (const cc of this.communicationChannels.values()) {
if (cc.spec.hostRef === hostName) n++
}
return n
}
/**
* Return all CCs whose spec.credentialsSecretRef.name matches the given
* Secret name. Used by HCC's channels-ns SecretInformer to find which
* per-Host channel-reader Deployments to annotation-roll on rotation.
* Order undefined; callers must sort if they care.
*/
findCommunicationChannelsByCredentialsSecretName(name: string): CommunicationChannelCRD[] {
const out: CommunicationChannelCRD[] = []
for (const cc of this.communicationChannels.values()) {
if (cc.spec.credentialsSecretRef?.name === name) out.push(cc)
}
return out
}
/**
* Return all CCs whose spec.hostRef matches. Used by the CC informer
* handler to figure out which channel-reader Deployment to annotation-roll
* on rebind (old host) AND newly-arrived CC (new host).
*/
findCommunicationChannelsByHostRef(hostRef: string): CommunicationChannelCRD[] {
const out: CommunicationChannelCRD[] = []
for (const cc of this.communicationChannels.values()) {
if (cc.spec.hostRef === hostRef) out.push(cc)
}
return out
}
private reconcileSharedFileSystemsReferencedByContext(
prevContext: ContextCRD | undefined,
nextContext: ContextCRD | undefined,
isCurrent: () => boolean = () => true
): Promise<void> {
const names = [
...(prevContext?.spec.sharedFileSystems ?? []).map(ref => ref.name),
...(nextContext?.spec.sharedFileSystems ?? []).map(ref => ref.name),
]
return this.enqueueSharedFileSystemFleetReconciliation(names, async () => {
if (!isCurrent()) return
await this.reconcileSharedFileSystemsReferencedByContextCore(
prevContext,
nextContext,
isCurrent
)
})
}
private async reconcileSharedFileSystemsReferencedByContextCore(
prevContext: ContextCRD | undefined,
nextContext: ContextCRD | undefined,
isCurrent: () => boolean
): Promise<void> {
const names = new Set<string>()
for (const r of prevContext?.spec.sharedFileSystems ?? []) names.add(r.name)
for (const r of nextContext?.spec.sharedFileSystems ?? []) names.add(r.name)
for (const name of names) {
if (!isCurrent()) return
const sfs = this.sharedFileSystems.get(name)
if (!sfs) continue
try {
await this.sharedFileSystemReconciler.reconcile(sfs)
} catch (err) {
console.error(`[K8s] SFS re-reconcile after Context change failed for "${name}":`, err)
}
}
}
/**
* Find every Host whose Context declares a reference to the given
* SharedFileSystem and trigger a fresh reconcile so the mcp-host
* Deployment template picks up (or drops) the volume + volumeMount.
*/
private async reconcileHostsReferencingSfs(sfsName: string): Promise<void> {
if (!this.admitHostDependentEffects(`SharedFileSystem "${sfsName}" Host convergence`)) {
return
}
const affectedContexts = new Set<string>()
for (const ctx of this.contexts.values()) {
if (ctx.spec.sharedFileSystems?.some(r => r.name === sfsName)) {
affectedContexts.add(ctx.name)
}
}
if (affectedContexts.size === 0) return
for (const host of this.hosts.values()) {
if (!affectedContexts.has(host.spec.contextRef)) continue
try {
await this.hostReconciler.reconcile(host)
} catch (err) {
console.error(`[K8s] Host re-reconcile failed for "${host.name}":`, err)
}
}
}
/**
* Reconcile Hosts affected by a Context change only while the Host cache is
* authoritative. A recovery full pass consumes the current Context cache, so
* deferral loses no effect and never acts on an unpaired Host LIST.
*/
private async reconcileHostsReferencingContext(
contextName: string,
isCurrent: () => boolean = () => true
): Promise<void> {
if (!isCurrent()) return
if (!this.admitHostDependentEffects(`Context "${contextName}" Host convergence`)) return
for (const host of this.hosts.values()) {
if (!isCurrent()) return
if (host.spec.contextRef !== contextName) continue
await this.hostReconciler.reconcile(host)
}
}
/**
* Re-reconcile the Host whose name matches the given hostRef. Called by the
* CommunicationChannel watch handler so per-Host channel-reader Deployment
* replicas converge to the new CC count (#281).
*
* No-op when the Host is unknown (CC arrived before Host CRD, or for a
* Host that has been deleted). The next fullReconcile recovers.
*
* Errors are logged and reported as `false` so the watch remains alive while
* its caller schedules bounded fleet convergence.
*/
private async reconcileHostsReferencingCC(hostRef: string): Promise<boolean> {
if (!this.admitHostDependentEffects(`CommunicationChannel Host "${hostRef}" convergence`)) {
return false
}
const host = this.hosts.get(hostRef)
if (!host) return true
try {
await this.hostReconciler.reconcile(host)
return true
} catch (err) {
console.error(`[K8s] Host re-reconcile after CC change failed for "${host.name}":`, err)
return false
}
}
/**
* Resolve `host.spec.contextRef → context.spec.sharedFileSystems[] →
* SharedFileSystem PVC name` from the in-process informer caches. Returns
* one entry per SharedFileSystem ref the Context declares; entries with
* no matching SharedFileSystem CRD are skipped (and logged) so the pod
* can still come up while the operator catches up.
*/
/**
* True iff the Host's referenced Context fronts at least one ENABLED
* `auth.type: oauth` mcp-server. HostReconciler uses this to gate the
* derive-only `oauth:user-token` runtime scope. Reads the same
* Context-scoped allow-list projection (`getServerInfosByContext`, which
* already filters to enabled + allowed servers) that mcp-host discovery uses,
* so HCC does not need a second cross-CRD read path. HostReconciler wraps this
* call in a fail-closed guard, so a thrown read there yields no oauth scope.
*
* A retired McpServer watch keeps its last `servers` cache, so an answer read
* from it is only an observation while the same LIST -> WATCH generation stays
* authoritative across the whole read. Otherwise this throws: a server an
* admin disabled during the outage must not re-grant the scope from the stale
* cache, and HostReconciler treats the throw as an unobserved read.
*/
private async hostFrontsOAuthServer(host: HostCRD): Promise<boolean> {
const watchGeneration = this.mcpWatchGeneration
if (!this.hasMcpServerInventoryAuthority(watchGeneration)) {
throw new Error('McpServer inventory is not authoritative; OAuth scope is unobserved')
}
const servers = await this.getServerInfosByContext(host.spec.contextRef)
if (!this.hasMcpServerInventoryAuthority(watchGeneration)) {
throw new Error('McpServer inventory lost authority during the OAuth scope read')
}
return servers.some(server => server.enabled && server.auth?.type === 'oauth')
}
private async resolveContextMounts(host: HostCRD): Promise<ResolvedSfsMount[]> {
const context = this.contexts.get(host.spec.contextRef)
const refs = context?.spec.sharedFileSystems ?? []
if (refs.length === 0) return []
const out: ResolvedSfsMount[] = []
for (const ref of refs) {
const sfs = this.sharedFileSystems.get(ref.name)
if (!sfs) {
console.warn(
`[K8s] Context "${host.spec.contextRef}" references SharedFileSystem ` +
`"${ref.name}" which is not yet known to HCC; skipping mount for now`
)
continue
}
// #592: only mount once the SFS PVC is Bound (the volume is provisioned and
// its node is determined). Under WaitForFirstConsumer a Bound PVC means the
// wfc — the first consumer — has already been scheduled, so the podAffinity
// term (added in buildDeployment) has a target node and the RWO mount is
// valid from the co-located node; mounting before the PVC binds would leave
// the mcp-host Pending or risk a cross-node mount failure.
//
// Gating on Bound — NOT on phase==='Ready' — is deliberate: the mcp-host
// mounts the PVC directly read-only and never talks to the wfc HTTP server,
// so a transient wfc restart (readyReplicas→0, phase flips to Initializing/
// Degraded while the PVC stays Bound) must NOT rip the mount out of a
// running consumer pod. The mount is therefore RETAINED across wfc blips and
// only dropped when the volume is genuinely unusable (PVC not Bound) or the
// Context stops referencing the SFS. The not-mountable→mountable transition
// re-reconciles the Host (reconcileHostsReferencingSfs / periodic resync).
if (
!this.sharedFileSystemReconciler.isMountable({ name: sfs.name, namespace: sfs.namespace })
) {
const phase = this.sharedFileSystemReconciler.getStatus({
name: sfs.name,
namespace: sfs.namespace,
}).phase
console.log(
`[K8s] SharedFileSystem "${ref.name}" PVC is not Bound yet (phase=${phase ?? 'unknown'}); ` +
`deferring RO mount for Host "${host.name}" until the volume binds`
)
continue
}
out.push({
name: sfs.name,
namespace: sfs.namespace,
pvcName: sfsPvcName(sfs),
mountPath: ref.mountPath,
})
}
return out
}
/**
* Get the SharedFileSystem reconciler — exposed so the host reconciler
* (when it lands in the next commit) can resolve PVC names by SFS name
* without a fresh K8s round-trip.
*/
getSharedFileSystemReconciler(): SharedFileSystemReconciler {
return this.sharedFileSystemReconciler
}
/**
* Get the host reconciler (for server-level desktop checks).
*/
getHostReconciler(): HostReconciler {
return this.hostReconciler
}
/**
* Look up a cached Host CRD by name (Stage 3: the stateless lifecycle
* tracker resolves the heartbeat's hostRef through this).
*/
getHost(name: string): HostCRD | undefined {
if (!this.isHostInventoryAuthoritative()) return undefined
return this.hosts.get(name)
}
/**
* Route channel credentials-Secret events through the same Host authority
* boundary as watch, Context, SFS, and heartbeat effects.
*/
async reconcileChannelReaderRevision(secretName: string, secretNamespace: string): Promise<void> {
const convergenceReason = `CommunicationChannel Secret "${secretNamespace}/${secretName}" Host convergence`
if (!this.ccCacheSynced) {
console.warn(
`[K8s] Deferring ${convergenceReason}; CommunicationChannel cache is not authoritative`
)
return
}
if (!this.admitHostDependentEffects(convergenceReason)) {
return
}
await this.hostReconciler.reconcileChannelReaderRevision(secretName, secretNamespace)
}
/**
* Trigger a reconcile for any cached McpServer whose `envSecret.name`
* matches the given Secret. Used by SecretInformer to react to Secret
* lifecycle changes (create/update/delete) without waiting for a CRD
* event.
*/
async reconcileByEnvSecret(secretName: string, secretNamespace: string): Promise<void> {
const inventoryGeneration = this.mcpWatchGeneration
if (!this.hasMcpServerInventoryAuthority(inventoryGeneration)) return
const selected = [...this.servers.values()].filter(
server => server.namespace === secretNamespace && server.spec.envSecret?.name === secretName
)
await Promise.all(
selected.map(server =>
this.enqueueMcpServerReconciliation(server, async () => {
if (!this.hasMcpServerInventoryAuthority(inventoryGeneration)) return
const current = this.servers.get(server.name)
if (
!current ||
current.namespace !== secretNamespace ||
current.spec.envSecret?.name !== secretName
) {
return
}
try {
console.log(
`[K8s] Re-reconciling McpServer "${current.name}" after Secret "${secretName}" change`
)
await this.reconciler.reconcile(current, {
isCurrent: () => {
if (!this.hasMcpServerInventoryAuthority(inventoryGeneration)) return false
const latest = this.servers.get(current.name)
return (
latest !== undefined &&
latest.namespace === secretNamespace &&
latest.spec.envSecret?.name === secretName &&
sameMcpServerDesiredRevision(current, latest)
)
},
})
} catch (err) {
console.error(`[K8s] Secret-triggered reconcile failed for "${current.name}":`, err)
}
})
)
)
}
/**
* Trigger a reconcile for any cached image `LlmHook` whose `envSecret` matches
* the changed Secret (spec §8.2). A rotation (same name, new contents) leaves
* the pod key unchanged but re-stamps the credentials-revision on the pod
* template → rolling restart onto the new credential — mirroring
* `reconcileByEnvSecret` for McpServer. Driven by the llm-hooks SecretInformer.
*/
async reconcileLlmHookByEnvSecret(secretName: string, secretNamespace: string): Promise<void> {
if (secretNamespace !== config.llmHooksNamespace) return
for (const hook of this.llmHooks.values()) {
if (hook.spec.target?.image?.envSecret !== secretName) continue
try {
console.log(
`[K8s] Re-reconciling LlmHook "${hook.name}" after Secret "${secretName}" change`
)
// Pod key is unchanged by a contents-only rotation, so pass it as the
// previous key (no stale-workload teardown) — the reconcile re-stamps the
// credentials-revision and rolls the shared pod.
await this.llmHookReconciler.reconcile(hook, computePodKey(hook))
} catch (err) {
console.error(`[K8s] Secret-triggered LlmHook reconcile failed for "${hook.name}":`, err)
}
}
}
/**
* Get all cached servers.
*/
getAllServers(): McpServerCRD[] {
return [...this.servers.values()]
}
/**
* Convert a CRD into curated McpServerInfo (strips deployment internals, adds status).
*/
private toServerInfo(server: McpServerCRD): McpServerInfo {
// Managed stdio servers are deployed with the stdio-bridge sidecar, which
// exposes StreamableHTTP on the transport port. Report streamableHttp to
// mcp-host so it uses the correct client transport to connect.
const transport = { ...server.spec.transport }
if (transport.type === 'stdio' && server.spec.managed !== false) {
transport.type = 'streamableHttp'
}
return {
name: server.name,
description: server.spec.description,
contextRef: server.spec.contextRef,
transport,
auth: server.spec.auth,
// Discovery metadata only: the OAuth block is projected verbatim, but
// mcp-host does NOT read it — per-connection partition dispatch is driven
// by `authKind` (derived separately from the AuthorityMcpServer store),
// so this field currently has no consumer. Token is NEVER mounted (O4).
oauth: server.spec.oauth,
enabled: server.spec.enabled !== false,
status: this.reconciler.getStatus(server),
}
}
/**
* Get curated server info for all servers (for API consumers).
*/
getAllServerInfos(): McpServerInfo[] {
return this.getAllServers().map(s => this.toServerInfo(s))
}
/**
* Get curated server info filtered by context (for API consumers).
* Reads the Context CRD to determine which McpServers are allowed.
*/
async getServerInfosByContext(contextRef: string): Promise<McpServerInfo[]> {
const context = await getContext(contextRef)
if (!context) {
console.warn(`[Provider] Context "${contextRef}" not found — returning no servers`)
return []
}
const allowedNames = new Set(context.spec.mcpServers)
console.log(
`[Provider] Context "${contextRef}" allows servers: [${context.spec.mcpServers.join(', ')}]`
)
return this.getAllServers()
.filter(s => allowedNames.has(s.name) && s.spec.enabled !== false)
.map(s => this.toServerInfo(s))
}
/**
* Set callback for when servers change.
*/
onChange(callback: () => void): void {
this.changeCallback = callback
}
/**
* Start watching for changes and run initial reconciliation.
*/
async start(): Promise<void> {
// L0/L1 isolation and removal of legacy permissive policies are mandatory
// bootstrap barriers. They are a fixed-size safety operation, independent
// of fleet size, and must succeed before HCC can watch or reconcile runtimes.
await this.netPolReconciler.ensureDefaultPolicies()
// Discovery and policy caches become authoritative only when each complete
// LIST snapshot is paired with a watch continuing from its opaque collection
// resourceVersion. Full fleet convergence happens later in the background.
// A transient LIST/WATCH failure on either inventory must not kill the
// retry that restart*Watch already armed (it sets *CacheSynced=false before
// it can throw). Mirror the Host lane below: log, keep the flag false, and
// schedule in-process recovery. The HTTP server stays live, dynamic
// readiness stays 503, and recovery promotes readiness only after a fresh
// LIST is paired with its continuing WATCH.
try {
const initialServerSnapshot = await listMcpServerSnapshot()
await this.restartMcpServerWatch(initialServerSnapshot)
} catch (error) {
this.mcpServerCacheSynced = false
console.error(
'[K8s] Initial McpServer inventory is unavailable; HCC remains unready while in-process recovery continues:',
error
)
}
if (!this.mcpServerCacheSynced) {
this.scheduleMcpServerCacheRecovery()
}
try {
const initialContextSnapshot = await listContextSnapshot()
await this.restartContextWatch(initialContextSnapshot)
} catch (error) {
this.contextCacheSynced = false
console.error(
'[K8s] Initial Context inventory is unavailable; HCC remains unready while in-process recovery continues:',
error
)
}
if (!this.contextCacheSynced) {
this.scheduleContextCacheRecovery()
}
// Host templates resolve Context SharedFileSystem references from this
// cache. Inventory is therefore a fixed startup boundary, not part of the
// unbounded SFS reconciliation fleet: letting the first Host pass run
// without it writes mount-less templates and immediately rolls the entire
// Host fleet a second time once discovery catches up.
const initialSharedFileSystemInventory = this.startSharedFileSystemInventoryAndWatch()
this.initialHostFleetSfsInventory = initialSharedFileSystemInventory
// ── CommunicationChannel snapshot + watch — MUST complete before Host fullReconcile ──
// A stateless Host may suspend only after this controller has a complete
// CommunicationChannel snapshot and a watch continuing from that snapshot's
// resourceVersion. A raw watch cannot prove it saw pre-existing channels.
let initialCCSnapshot: CommunicationChannelSnapshot | undefined
try {
initialCCSnapshot = await listCommunicationChannelSnapshot()
this.installCommunicationChannelSnapshot(initialCCSnapshot)
} catch (error) {
console.error(
'[K8s] CommunicationChannel initial load failed; ccCacheSynced remains false ' +
'(B2 preserves channel-reader replicas, preserving durable Host lifecycle state):',
error
)
}
try {
const watchGeneration = await this.startCommunicationChannelWatch(
initialCCSnapshot?.resourceVersion
)
if (
initialCCSnapshot?.resourceVersion &&
watchGeneration === this.ccWatchGeneration &&
this.ccWatchRequest !== null
) {
this.ccCacheSynced = true
console.log(
`[K8s] Loaded ${initialCCSnapshot.channels.length} CommunicationChannel(s) into cache ` +
'(ccCacheSynced=true)'
)
} else if (initialCCSnapshot) {
console.error(
'[K8s] CommunicationChannel snapshot could not be paired with an active watch; ' +
'ccCacheSynced remains false'
)
}
} catch (error) {
this.ccCacheSynced = false
console.error(
'[K8s] CommunicationChannel watch failed to start; ccCacheSynced remains false:',
error
)
}
if (!this.ccCacheSynced) {
this.scheduleCommunicationChannelCacheRecovery()
}
console.log(
`[K8s] Starting initial Host background convergence... (ccCacheSynced=${this.ccCacheSynced})`
)
const initialLifecycleGeneration =
this.ccLifecycleGeneration === 0
? this.beginCommunicationChannelLifecycleTransition()
: this.ccLifecycleGeneration
try {
await this.recoverHostInventoryAndWatch(
'initial Host reconciliation',
initialLifecycleGeneration,
'cold-start'
)
} catch (error) {
// Host inventory is an essential readiness authority, but a transient
// LIST/WATCH failure must not kill the retry that recovery just armed.
// Continue startup with hostCacheSynced=false: the HTTP server stays live,
// dynamic readiness remains 503, and recovery promotes readiness only
// after a fresh LIST is paired with its continuing WATCH.
console.warn(
'[K8s] Initial Host inventory is unavailable; HCC remains unready while in-process recovery continues:',
error
)
}
// ── LlmHook initial load + reconciliation (guardrails phase-4) ──
// Runs AFTER the Host cache is populated so the shared hook pods' NetworkPolicy
// ingress reflects the current Host→LlmHook reverse index on the first pass.
// The GFS lane and the four pre-existing watches moved to dev's per-resource
// background lanes (startGlobalFileSystemBackgroundLane and friends); only the
// LlmHook lane is new here, so only it survives this merge.
try {
const initialHooks = await listAllLlmHooks()
for (const hook of initialHooks) {
this.llmHooks.set(hook.name, hook)
}
console.log('[K8s] Running initial LlmHook reconciliation...')
await this.llmHookReconciler.fullReconcile(initialHooks)
} catch (error) {
console.error(
'[K8s] Skipping initial LlmHook reconciliation because discovery failed:',
error
)
}
await this.startLlmHookWatch()
const resyncSec = config.hostResyncIntervalSec
if (resyncSec > 0) {
this.resyncTimer = setInterval(() => {
void this.runHostResync()
}, resyncSec * 1000)
console.log(`[K8s] Host periodic resync enabled (every ${resyncSec}s)`)
} else {
console.warn(
'[K8s] Host periodic resync disabled; runtime-auth degraded mcp-host pods will not self-heal through controller-driven rollout unless another Host event triggers reconciliation.'
)
}
const sfsResyncSec = config.sfsResyncIntervalSec
if (sfsResyncSec > 0) {
this.sfsResyncTimer = setInterval(() => {
void this.runSfsResync()
}, sfsResyncSec * 1000)
console.log(`[K8s] SharedFileSystem periodic resync enabled (every ${sfsResyncSec}s)`)
} else {
console.warn(
'[K8s] SharedFileSystem periodic resync disabled; a SharedFileSystem stuck in Initializing/Degraded will not auto-recover to Ready until another SFS event triggers reconciliation (#592).'
)
}
const llmHookResyncSec = config.llmHookResyncIntervalSec
if (llmHookResyncSec > 0) {
this.llmHookResyncTimer = setInterval(() => {
void this.runLlmHookResync()
}, llmHookResyncSec * 1000)
console.log(`[K8s] LlmHook periodic resync enabled (every ${llmHookResyncSec}s)`)
} else {
console.warn(
'[K8s] LlmHook periodic resync disabled; label-orphaned hook workloads will not be swept until another LlmHook event triggers reconciliation.'
)
}
const gfsResyncSec = config.gfsResyncIntervalSec
if (gfsResyncSec > 0) {
this.gfsResyncTimer = setInterval(() => {
void this.runGfsResync()
}, gfsResyncSec * 1000)
console.log(`[K8s] GlobalFileSystem periodic resync enabled (every ${gfsResyncSec}s)`)
} else {
console.warn(
'[K8s] GlobalFileSystem periodic resync disabled; a GlobalFileSystem stuck in Initializing will not auto-recover to Ready (nor seed its root directories) until another gfs event triggers reconciliation.'
)
}
// Interval > 0 is an ops enable, not a merge default. Issue #478: with
// this skip-if-in-flight guard the chosen enable is 1500s. Without it the
// floor would be 3600s. 300s is not a valid enable while passes last
// minutes (treadmill ⇔ pass duration ≥ period). Event-driven coalescing
// is unchanged; only this timer trigger is filtered.
const netPolResyncSec = config.netPolResyncIntervalSec
if (netPolResyncSec > 0) {
this.netPolResyncTimer = setInterval(() => {
if (this.initialConvergenceRuns.has('NetworkPolicy')) {
netPolResyncTicksSkippedTotal.inc({ reason: 'pass-in-flight' })
return
}
void this.runInitialNetworkPolicyConvergence({
ensureDefaults: true,
cause: 'periodic-resync',
})
}, netPolResyncSec * 1000)
hccLogger.info('[K8s] NetworkPolicy periodic resync enabled', {
intervalSeconds: netPolResyncSec,
})
} else {
hccLogger.warn(
'[K8s] NetworkPolicy periodic resync disabled; dropped watch events will not self-heal stale NetworkPolicy allows through controller-driven convergence until another Context or McpServer event triggers reconciliation.'
)
}
// Class-D cheap path (#488). Interval > 0 is an ops enable, not a merge
// default. This timer never consults initialConvergenceRuns and never
// calls runInitialNetworkPolicyConvergence / fullReconcile.
const netPolDefaultsResyncSec = config.netPolDefaultsResyncIntervalSec
if (netPolDefaultsResyncSec > 0) {
this.netPolDefaultsResyncTimer = setInterval(() => {
void this.runNetworkPolicyDefaultsOnly()
}, netPolDefaultsResyncSec * 1000)
console.log(
`[K8s] NetworkPolicy defaults-only resync enabled (every ${netPolDefaultsResyncSec}s)`
)
}
const externalEgressResyncSec = config.externalEgressResyncIntervalSec
// #205 delegates external-egress periodic resync to the convergence
// coordinator. Its runResyncCore drives reconcileExternalEgress, so the
// #299 sliding-window accumulation still converges on DNS rotation between
// McpServer events. The coordinator owns cadence with a self-rescheduling
// timer that advances to <= observed TTL/2 (H2, issue #299), bounded by the
// configured floor.
this.externalEgressCoordinator.startPeriodicResync(
externalEgressResyncSec,
config.externalEgressRefreshFloorSec
)
// Full convergence is observable and retains each lane's existing retry or
// periodic-resync contract, but it no longer extends provider.start().
// NetworkPolicy safety starts first and launches McpServer convergence from
// its certification callback, before additive policy convergence completes.
// This prevents the startup egress refresh from racing or undoing the safety
// pass while keeping readiness independent from both additive fleets.
void this.runInitialNetworkPolicyConvergence({ cause: 'startup' })
void initialSharedFileSystemInventory.then(inventoryComplete => {
if (inventoryComplete && !this.stopped) {
void this.runInitialSharedFileSystemConvergence()
}
})
void this.startGlobalFileSystemBackgroundLane()
}
private async startSharedFileSystemInventoryAndWatch(): Promise<boolean> {
let inventoryComplete = false
try {
const initialSfses = await listAllSharedFileSystems()
inventoryComplete = true
for (const sfs of initialSfses) this.sharedFileSystems.set(sfs.name, sfs)
} catch (error) {
console.error(
'[K8s] Skipping initial SharedFileSystem background convergence because discovery failed:',
error
)
}
// The cold-start Host fleet pass gates on this promise, so it must settle on
// the inventory alone. `k8s.Watch` carries no transport deadline — the
// client builds `AbortSignal.any([controller, timeout])` and then overwrites
// it with the bare controller — so awaiting the watch start here would
// re-strand the fleet behind an already-certified readiness, which is the
// exact failure bounding the LIST was meant to close.
void this.startSharedFileSystemWatch().catch(error => {
console.error('[K8s] SharedFileSystem background watch failed to start:', error)
this.scheduleSharedFileSystemWatchRestart(5000)
})
if (inventoryComplete && !this.stopped) {
return true
}
return false
}
private async startGlobalFileSystemBackgroundLane(): Promise<void> {
let inventoryComplete = false
try {
const initialGfses = await listAllGlobalFileSystems()
inventoryComplete = true
for (const gfs of initialGfses) this.globalFileSystems.set(gfs.name, gfs)
} catch (error) {
console.error(
'[K8s] Skipping initial GlobalFileSystem background convergence because discovery failed:',
error
)
}
try {
await this.startGlobalFileSystemWatch()
} catch (error) {
console.error('[K8s] GlobalFileSystem background watch failed to start:', error)
this.scheduleGlobalFileSystemWatchRestart(5000)
}
if (inventoryComplete && !this.stopped) {
void this.runInitialGlobalFileSystemConvergence()
}
}
private currentDesiredInventoryRevision(): CompletedInventoryRevision {
return {
contextRevision: this.contextDesiredRevision,
serverRevision: this.mcpServerDesiredRevision,
}
}
private matchesCurrentInventory(revision: CompletedInventoryRevision | null): boolean {
return (
revision !== null &&
revision.contextRevision === this.contextDesiredRevision &&
revision.serverRevision === this.mcpServerDesiredRevision
)
}
private shouldRequestNetworkPolicyRecovery(
kind: 'McpServer' | 'Context',
before: CompletedInventoryRevision
): boolean {
let reason:
| 'identical-complete'
| 'relevant-change'
| 'no-authority'
| 'work-pending'
| 'resync-disabled'
const deferred = kind === 'McpServer' && !this.contextCacheSynced
if (deferred) reason = 'no-authority'
else if (!this.matchesCurrentInventory(before)) reason = 'relevant-change'
else if (this.netPolResyncTimer === null) reason = 'resync-disabled'
else if (this.currentNetworkPolicySafetyCertificate() === null) reason = 'no-authority'
else if (
!this.matchesCurrentInventory(this.completedNetworkPolicyRevision) ||
!this.matchesCurrentInventory(this.completedRuntimeRevision) ||
this.networkPolicyRepairPending ||
this.runtimeRepairPending ||
this.reconciler.hasPendingReconciliation() ||
(this.runtimeInventoryGeneration !== null &&
this.runtimeInventoryGeneration !== this.mcpWatchGeneration) ||
this.initialConvergenceRetryTimers.has('NetworkPolicy') ||
this.initialConvergenceRetryTimers.has('McpServer')
) {
reason = 'work-pending'
} else reason = 'identical-complete'
const request = !deferred && reason !== 'identical-complete'
hccLogger.info('[K8s] NetworkPolicy recovery decision', {
event: 'networkpolicy-recovery-decision',
kind,
decision: deferred ? 'defer' : request ? 'request' : 'skip',
reason,
...this.currentDesiredInventoryRevision(),
})
return request
}
private runInitialMcpServerConvergence(
request: McpServerConvergenceRequest = 'full'
): Promise<void> {
this.pendingMcpServerConvergence = widenMcpServerConvergence(
this.pendingMcpServerConvergence,
request
)
return this.runInitialConvergence('McpServer')
}
private currentNetworkPolicySafetyCertificate(): NetworkPolicySafetyCertificate | null {
if (
this.stopped ||
!this.contextCacheSynced ||
!this.mcpServerCacheSynced ||
!this.netPolReconciler.hasCertifiedSafetyInventory() ||
// Content identity, not channel identity — see isReadinessInventoryAuthoritative.
this.networkPolicyRevocationContextRevision !== this.contextDesiredRevision ||
this.networkPolicyRevocationServerRevision !== this.mcpServerDesiredRevision
) {
return null
}
return {
contextGeneration: this.contextWatchGeneration,
serverGeneration: this.mcpWatchGeneration,
contextRevision: this.contextDesiredRevision,
serverRevision: this.mcpServerDesiredRevision,
}
}
private recordNetworkPolicySafetyCertificate(
certificate: NetworkPolicySafetyCertificate
): boolean {
if (
this.stopped ||
!this.contextCacheSynced ||
!this.mcpServerCacheSynced ||
// Content identity, not channel identity — a Premature-close reconnect
// that re-LISTs the same inventory must not invalidate this certificate.
this.contextDesiredRevision !== certificate.contextRevision ||
this.mcpServerDesiredRevision !== certificate.serverRevision
) {
return false
}
this.networkPolicyRevocationContextRevision = certificate.contextRevision
this.networkPolicyRevocationServerRevision = certificate.serverRevision
return true
}
private isNetworkPolicySafetyCertificateCurrent(
certificate: NetworkPolicySafetyCertificate
): boolean {
const current = this.currentNetworkPolicySafetyCertificate()
return (
current !== null &&
// Content identity, not channel identity — see isReadinessInventoryAuthoritative.
current.contextRevision === certificate.contextRevision &&
current.serverRevision === certificate.serverRevision
)
}
private async runInitialMcpServerConvergenceCore(): Promise<void> {
const request = this.pendingMcpServerConvergence ?? 'full'
this.pendingMcpServerConvergence = undefined
const safetyCertificate = this.currentNetworkPolicySafetyCertificate()
// A retired run may settle normally without invoking work. Keep its repair
// obligation until an authoritative pass actually completes every effect.
this.runtimeRepairPending = true
if (safetyCertificate === null) return
let effectsComplete = true
// A scoped retry completes only the pass it came from. Once the inventory
// has moved, the rest of the fleet may need work too, so run a full pass.
const scope =
request !== 'full' && this.matchesCurrentInventory(request) ? request.names : undefined
const inventoryGeneration = this.mcpWatchGeneration
this.runtimeInventoryGeneration = inventoryGeneration
const inventoryAuthoritative = () =>
this.hasMcpServerInventoryAuthority(inventoryGeneration) &&
this.isNetworkPolicySafetyCertificateCurrent(safetyCertificate)
try {
const initialServers = [...this.servers.values()]
const initialExternalEgressGates = this.externalEgressCoordinator.prepareStartupGates(
scope ? initialServers.filter(server => scope.has(server.name)) : initialServers,
inventoryAuthoritative
)
if (this.stopped) return
if (scope) {
hccLogger.info('[K8s] Running scoped McpServer background reconciliation retry...', {
servers: [...scope].sort(),
})
} else {
hccLogger.info('[K8s] Running initial McpServer background reconciliation...')
}
await this.reconciler.fullReconcile(initialServers, {
scope,
runEffect: async (serverName, work) => {
const selected = this.servers.get(serverName)
const laneOwner = selected ?? {
name: serverName,
namespace: config.namespace,
}
if (!(await initialExternalEgressGates.waitFor(laneOwner))) {
effectsComplete = false
return
}
if (!inventoryAuthoritative()) {
effectsComplete = false
return
}
await this.enqueueMcpServerReconciliation(laneOwner, async () => {
if (!inventoryAuthoritative()) {
effectsComplete = false
return
}
await work()
})
},
})
if (!effectsComplete || !inventoryAuthoritative()) {
this.scheduleInitialConvergenceRetry('McpServer')
return
}
if (this.reconciler.hasIncompleteReconciliation()) {
// Every offered effect ran under this certificate, so only individual
// servers are unfinished (for example a Deployment write rejected by
// admission). Retry just those names: one persistently failing server
// must not re-reconcile the whole fleet at the retry cap.
this.scheduleInitialConvergenceRetry('McpServer', {
contextRevision: safetyCertificate.contextRevision,
serverRevision: safetyCertificate.serverRevision,
names: new Set(this.reconciler.incompleteReconciliationNames()),
})
return
}
this.completedRuntimeRevision = safetyCertificate
this.runtimeRepairPending = false
initialConvergenceLastSuccessTimestampSeconds.set({ lane: 'McpServer' }, Date.now() / 1000)
this.clearInitialConvergenceRetry('McpServer')
} catch (error) {
hccLogger.error('[K8s] Initial McpServer background reconciliation failed:', { err: error })
this.scheduleInitialConvergenceRetry('McpServer')
} finally {
this.runtimeInventoryGeneration = null
}
}
private runInitialNetworkPolicyConvergence(options: {
ensureDefaults?: boolean
cause: NetworkPolicyConvergenceCause
}): Promise<void> {
if (options?.ensureDefaults) {
this.netPolConvergenceEnsureDefaults = true
}
const cause = options.cause
this.pendingNetworkPolicyCauses.add(cause)
hccLogger.info('[K8s] NetworkPolicy convergence requested', {
event: 'networkpolicy-request',
cause,
...this.currentDesiredInventoryRevision(),
admission: this.activeNetworkPolicyPass === null ? 'before-capture' : 'trailing',
})
return this.runInitialConvergence('NetworkPolicy')
}
/**
* Class-D defaults-only tick (#488).
*
* `ensureDefaultPolicies` writes only L0/L1: `ensureDefaultDeny`,
* `ensureInfrastructurePolicies`, `ensureAllowContextMapperApi`, and the
* two `deleteLegacyStaticPolicy` calls. The fleet sweep lists typed
* `policy-type ∈ {context-allow, rpc-proxy-egress, external-egress}`.
* Baseline never appears in that list, so a concurrent full NetPol pass
* is safe. Do not re-apply HARD REQUIREMENT 2 here and do not share
* `initialConvergenceRuns.has('NetworkPolicy')`.
*
* Overlapping defaults-only ticks serialize: a second tick increments
* `netPolResyncTicksSkippedTotal{reason="defaults-only-in-flight"}` and
* joins the in-flight run. Never increment `pass-in-flight` from here.
* Each executed tick records `netPolDefaultsOnlyTicksTotal` and
* `netPolDefaultsOnlyTickDurationSeconds` with result success|error.
*/
private runNetworkPolicyDefaultsOnly(): Promise<void> {
if (this.netPolDefaultsOnlyRun) {
netPolResyncTicksSkippedTotal.inc({ reason: 'defaults-only-in-flight' })
return this.netPolDefaultsOnlyRun
}
const run = (async () => {
const startedAtMs = Date.now()
let result: 'success' | 'error' = 'success'
try {
await this.netPolReconciler.ensureDefaultPolicies()
} catch (error) {
result = 'error'
console.error('[K8s] NetworkPolicy defaults-only tick failed:', error)
} finally {
const seconds = Math.max(0, (Date.now() - startedAtMs) / 1000)
netPolDefaultsOnlyTicksTotal.inc({ result })
netPolDefaultsOnlyTickDurationSeconds.observe({ result }, seconds)
}
})()
this.netPolDefaultsOnlyRun = run
void run.finally(() => {
if (this.netPolDefaultsOnlyRun === run) {
this.netPolDefaultsOnlyRun = null
}
})
return run
}
private runInitialConvergence(lane: InitialConvergenceLane): Promise<void> {
const active = this.initialConvergenceRuns.get(lane)
if (active) {
active.trailingRequested = true
return active.promise
}
const run: ActiveInitialConvergenceRun = {
trailingRequested: false,
promise: Promise.resolve(),
}
this.initialConvergenceRuns.set(lane, run)
run.promise = Promise.resolve()
.then(async () => {
let trailing = false
do {
run.trailingRequested = false
if (lane === 'McpServer') {
await this.runInitialMcpServerConvergenceCore()
} else {
const pass = {
passId: ++this.networkPolicyPassSequence,
causes: [...this.pendingNetworkPolicyCauses],
trailing,
...this.currentDesiredInventoryRevision(),
}
this.pendingNetworkPolicyCauses.clear()
this.activeNetworkPolicyPass = pass
hccLogger.info('[K8s] NetworkPolicy pass started', {
event: 'networkpolicy-pass-start',
...pass,
})
try {
await this.runInitialNetworkPolicyConvergenceCore()
} finally {
this.activeNetworkPolicyPass = null
}
}
trailing = true
} while (!this.stopped && run.trailingRequested)
// Retire the active slot synchronously before this promise settles.
// A request arriving in the following settlement microtask must start
// a new run instead of attaching to a pass whose loop already exited.
if (this.initialConvergenceRuns.get(lane) === run) {
this.initialConvergenceRuns.delete(lane)
}
})
.finally(() => {
// Errors bypass the normal retirement above.
if (this.initialConvergenceRuns.get(lane) === run) {
this.initialConvergenceRuns.delete(lane)
}
})
return run.promise
}
private async runInitialNetworkPolicyConvergenceCore(): Promise<void> {
// A NetworkPolicy full pass combines authoritative Context/McpServer allow
// revocation with additive Context policy effects. External egress
// creation and DNS refresh have one separate owner: the startup/resync
// coordinator. Running safety from a partial inventory would be neither a
// full convergence nor safe evidence of success. Recovery of either
// missing LIST -> WATCH pair schedules a fresh current-cache pass.
const startedAtMs = Date.now()
const observePass = (result: InitialConvergencePassResult) => {
observeInitialNetworkPolicyPass(startedAtMs, result)
hccLogger.info('[K8s] NetworkPolicy pass finished', {
event: 'networkpolicy-pass-result',
...this.activeNetworkPolicyPass,
result,
})
}
if (!this.contextCacheSynced || !this.mcpServerCacheSynced) {
const unsynced = [
!this.contextCacheSynced ? 'Context' : undefined,
!this.mcpServerCacheSynced ? 'McpServer' : undefined,
]
.filter((name): name is string => name !== undefined)
.join(' and ')
hccLogger.warn(
`[K8s] NetworkPolicy convergence request deferred: caches unsynced (${unsynced})`
)
initialConvergenceSwallowedTotal.inc({ lane: 'NetworkPolicy', sink: 'unsynced' })
observePass('deferred-unsynced')
this.networkPolicyRepairPending = true
this.scheduleInitialConvergenceRetry('NetworkPolicy')
// A timer tick may have requested defaults. This path never reached
// fullReconcile, so drop the flag: event/retry entry stays false.
this.netPolConvergenceEnsureDefaults = false
return
}
let effectsComplete = true
let authoritativeRevocationCompleted = false
let safetyCertificate: NetworkPolicySafetyCertificate | undefined
try {
const initialContexts = [...this.contexts.values()]
const initialServers = [...this.servers.values()]
const contextInventoryGeneration = this.contextWatchGeneration
const serverInventoryGeneration = this.mcpWatchGeneration
const capturedSafetyCertificate: NetworkPolicySafetyCertificate = {
contextGeneration: contextInventoryGeneration,
serverGeneration: serverInventoryGeneration,
contextRevision: this.contextDesiredRevision,
serverRevision: this.mcpServerDesiredRevision,
}
safetyCertificate = capturedSafetyCertificate
const serverInventoryComplete = this.mcpServerCacheSynced
const contextInventoryAuthoritative = () =>
this.contextCacheSynced &&
this.contextDesiredRevision === capturedSafetyCertificate.contextRevision
const serverInventoryAuthoritative = () =>
this.mcpServerCacheSynced &&
this.mcpServerDesiredRevision === capturedSafetyCertificate.serverRevision
hccLogger.info('[K8s] Running initial NetworkPolicy background reconciliation...')
const ensureDefaults = this.netPolConvergenceEnsureDefaults
this.netPolConvergenceEnsureDefaults = false
await this.netPolReconciler.fullReconcile(initialContexts, initialServers, {
serverInventoryComplete,
ensureDefaults,
contextInventoryAuthoritative,
serverInventoryAuthoritative,
runContextEffect: (contextId, work) =>
this.enqueueContextReconciliation(contextId, async () => {
if (!contextInventoryAuthoritative() || !serverInventoryAuthoritative()) {
effectsComplete = false
this.networkPolicyRepairPending = true
initialConvergenceEffectsDroppedTotal.inc({
lane: 'NetworkPolicy',
kind: 'context',
})
return
}
await work()
}),
runServerEffect: (serverName, work) => {
const selected = this.servers.get(serverName)
return this.enqueueMcpServerReconciliation(
selected ?? { name: serverName, namespace: config.namespace },
async () => {
if (!serverInventoryAuthoritative()) {
effectsComplete = false
this.networkPolicyRepairPending = true
initialConvergenceEffectsDroppedTotal.inc({
lane: 'NetworkPolicy',
kind: 'server',
})
return
}
await work()
}
)
},
resolveCurrentContext: name => this.contexts.get(name),
resolveCurrentContextById: contextId =>
[...this.contexts.values()].find(context => context.spec.contextId === contextId),
resolveCurrentServer: name => this.servers.get(name),
contextDesiredRevision: () => this.contextDesiredRevision,
serverDesiredRevision: () => this.mcpServerDesiredRevision,
// B3 piece-2: queue an additive recreation for every server whose
// external-egress allow the authoritative pass revoked — fired at the
// deletion, so an abort/throw cannot strand a revoked-but-desired allow.
// The coordinator maps against the current cache (absent → DELETED, no
// recreate) and rebuilds from the current spec with fresh DNS.
onExternalEgressRevoked: server =>
this.externalEgressCoordinator.scheduleRetry('MODIFIED', server),
onAuthoritativeRevocationComplete: () => {
if (!this.recordNetworkPolicySafetyCertificate(capturedSafetyCertificate)) return
authoritativeRevocationCompleted = true
// Start runtime convergence at the safety boundary, before this full
// pass enters additive Context work. Startup egress gates may now
// preserve only policies certified by this exact authoritative
// inventory generation.
void this.runInitialMcpServerConvergence()
},
})
if (
!effectsComplete ||
!authoritativeRevocationCompleted ||
!contextInventoryAuthoritative() ||
!serverInventoryAuthoritative()
) {
hccLogger.warn(
'[K8s] pass ended without certifying: inventory authority lost',
this.networkPolicyInventoryMovement(capturedSafetyCertificate)
)
initialConvergenceSwallowedTotal.inc({ lane: 'NetworkPolicy', sink: 'authority-lost' })
observePass('aborted-authority')
this.networkPolicyRepairPending = true
this.scheduleInitialConvergenceRetry('NetworkPolicy')
return
}
this.completedNetworkPolicyRevision = capturedSafetyCertificate
this.networkPolicyRepairPending = false
initialConvergenceLastSuccessTimestampSeconds.set(
{ lane: 'NetworkPolicy' },
Date.now() / 1000
)
this.clearInitialConvergenceRetry('NetworkPolicy')
observePass('certified')
} catch (error) {
hccLogger.error(
authoritativeRevocationCompleted
? '[K8s] Initial NetworkPolicy post-certification additive reconciliation failed:'
: '[K8s] Initial NetworkPolicy background reconciliation failed:',
{ err: error }
)
const inventoryInterruption =
safetyCertificate !== undefined && isDesiredNetworkPolicyInventoryInterruption(error)
const movement = safetyCertificate
? this.networkPolicyInventoryMovement(safetyCertificate)
: undefined
const result: InitialConvergencePassResult =
inventoryInterruption && movement
? movement.contextMoved || movement.serverMoved
? 'aborted-bump'
: 'aborted-authority'
: 'failed'
if (result === 'aborted-bump') {
hccLogger.warn('[K8s] pass ended without certifying: desired inventory changed', movement)
} else if (result === 'aborted-authority') {
hccLogger.warn('[K8s] pass ended without certifying: inventory authority lost', movement)
}
observePass(result)
this.networkPolicyRepairPending = true
this.scheduleInitialConvergenceRetry('NetworkPolicy')
}
}
private networkPolicyInventoryMovement(certificate: NetworkPolicySafetyCertificate): {
contextMoved: boolean
serverMoved: boolean
contextCacheSynced: boolean
mcpServerCacheSynced: boolean
} {
return {
contextMoved: this.contextDesiredRevision !== certificate.contextRevision,
serverMoved: this.mcpServerDesiredRevision !== certificate.serverRevision,
contextCacheSynced: this.contextCacheSynced,
mcpServerCacheSynced: this.mcpServerCacheSynced,
}
}
private clearInitialConvergenceRetry(lane: InitialConvergenceLane): void {
const timer = this.initialConvergenceRetryTimers.get(lane)
if (timer) {
clearTimeout(timer)
this.initialConvergenceRetryTimers.delete(lane)
}
this.initialConvergenceRetryAttempts.delete(lane)
if (lane === 'McpServer') this.mcpServerRetryRequest = undefined
}
private scheduleInitialConvergenceRetry(
lane: InitialConvergenceLane,
mcpServerRequest: McpServerConvergenceRequest = 'full'
): void {
if (this.stopped) return
if (lane === 'McpServer') {
// An armed retry widens to cover every request made before it fires.
this.mcpServerRetryRequest = widenMcpServerConvergence(
this.mcpServerRetryRequest,
mcpServerRequest
)
}
if (this.initialConvergenceRetryTimers.has(lane)) return
const attempt = (this.initialConvergenceRetryAttempts.get(lane) ?? 0) + 1
const delayMs =
INITIAL_CONVERGENCE_RETRY_DELAYS_MS[
Math.min(attempt - 1, INITIAL_CONVERGENCE_RETRY_DELAYS_MS.length - 1)
]
this.initialConvergenceRetryAttempts.set(lane, attempt)
initialConvergenceRetriesTotal.inc({ lane })
console.warn(
`[K8s] Scheduling initial ${lane} background convergence retry ${attempt} in ${delayMs}ms`
)
const timer = setTimeout(() => {
this.initialConvergenceRetryTimers.delete(lane)
if (this.stopped) return
if (lane === 'McpServer') {
const request = this.mcpServerRetryRequest ?? 'full'
this.mcpServerRetryRequest = undefined
void this.runInitialMcpServerConvergence(request)
return
}
void this.runInitialNetworkPolicyConvergence({ cause: 'retry' })
}, delayMs)
this.initialConvergenceRetryTimers.set(lane, timer)
}
private runInitialSharedFileSystemConvergence(): Promise<void> {
const names = [...this.sharedFileSystems.keys()]
return this.enqueueSharedFileSystemFleetReconciliation(names, () =>
this.runInitialSharedFileSystemConvergenceCore()
)
}
private async runInitialSharedFileSystemConvergenceCore(): Promise<void> {
try {
const initialSfses = [...this.sharedFileSystems.values()]
console.log('[K8s] Running initial SharedFileSystem background reconciliation...')
await this.sharedFileSystemReconciler.fullReconcile(initialSfses)
} catch (error) {
console.error('[K8s] Initial SharedFileSystem background reconciliation failed:', error)
}
}
private runInitialGlobalFileSystemConvergence(): Promise<void> {
const names = [...this.globalFileSystems.keys()]
return this.enqueueGlobalFileSystemFleetReconciliation(names, () =>
this.runInitialGlobalFileSystemConvergenceCore()
)
}
private async runInitialGlobalFileSystemConvergenceCore(): Promise<void> {
try {
const initialGfses = [...this.globalFileSystems.values()]
console.log('[K8s] Running initial GlobalFileSystem background reconciliation...')
await this.gfsReconciler.fullReconcile(initialGfses)
} catch (error) {
console.error('[K8s] Initial GlobalFileSystem background reconciliation failed:', error)
}
}
/**
* Start watching SharedFileSystem CRDs in the mcp-host namespace.
*/
private async startSharedFileSystemWatch(): Promise<void> {
const path = `/apis/${GROUP}/${VERSION}/namespaces/${config.hostNamespace}/${PLURAL_SHAREDFILESYSTEMS}`
console.log(`[K8s] Starting SharedFileSystem watch`)
const watchCallback = async (
type: string,
apiObj: { metadata: { name: string; namespace?: string }; spec: SharedFileSystemSpec }
) => {
if (type === 'BOOKMARK') return
const sfs: SharedFileSystemCRD = {
name: apiObj.metadata.name,
namespace: apiObj.metadata.namespace || config.hostNamespace,
spec: apiObj.spec,
}
console.log(`[K8s] SharedFileSystem watch event: ${type} for ${sfs.name}`)
if (type === 'ADDED' || type === 'MODIFIED') {
this.sharedFileSystems.set(sfs.name, sfs)
} else if (type === 'DELETED') {
this.sharedFileSystems.delete(sfs.name)
}
this.sharedFileSystemCacheRevision += 1
await this.enqueueSharedFileSystemReconciliation(sfs.name, async () => {
try {
if (type === 'ADDED' || type === 'MODIFIED') {
await this.sharedFileSystemReconciler.reconcile(sfs)
} else if (type === 'DELETED') {
await this.sharedFileSystemReconciler.reconcileDelete(sfs.name, sfs.namespace, sfs.spec)
}
} catch (error) {
console.error(`[K8s] SharedFileSystem reconciliation failed for ${sfs.name}:`, error)
}
// Re-reconcile any Host whose Context references this SharedFileSystem.
// ADDED/MODIFIED: pick up the now-resolvable mount; DELETED: drop it on
// the next mcp-host pod template diff so the pod stops failing to mount.
try {
await this.reconcileHostsReferencingSfs(sfs.name)
} catch (error) {
console.error(
`[K8s] Failed to re-reconcile Hosts referencing SharedFileSystem ${sfs.name}:`,
error
)
}
})
}
const doneCallback = (err: Error | null) => {
if (this.stopped) return
this.sfsWatchRequest = null
if (err) {
console.error('[K8s] SharedFileSystem watch error:', err)
}
console.log('[K8s] SharedFileSystem watch ended, restarting...')
this.scheduleSharedFileSystemWatchRestart(err ? 5000 : 1000)
}
this.sfsWatchRequest = await this.watch.watch(path, {}, watchCallback, doneCallback)
}
private scheduleSharedFileSystemWatchRestart(delayMs: number): void {
if (this.stopped || this.sfsWatchRestartTimer) return
this.sfsWatchRestartTimer = setTimeout(() => {
this.sfsWatchRestartTimer = null
if (this.stopped) return
void this.startSharedFileSystemWatch().catch(error => {
console.error('[K8s] SharedFileSystem background watch restart failed:', error)
this.scheduleSharedFileSystemWatchRestart(5000)
})
}, delayMs)
}
/**
* Watch the GlobalFileSystem singleton (gfs). ADDED/MODIFIED → reconcile the
* drive stack; DELETED → tear it down (respecting retainOnDelete). DISTINCT
* from the SharedFileSystem watch — gfs lives in its own namespace and has no
* Context cross-reference.
*/
private async startGlobalFileSystemWatch(): Promise<void> {
const namespace = gfsDefaultFactoryConfig().gfsNamespace
const path = `/apis/${GROUP}/${VERSION}/namespaces/${namespace}/${PLURAL_GLOBALFILESYSTEMS}`
console.log(`[K8s] Starting GlobalFileSystem watch`)
const watchCallback = async (
type: string,
apiObj: {
metadata: { name: string; namespace?: string }
spec: GlobalFileSystemSpec
status?: GlobalFileSystemCRD['status']
}
) => {
if (type === 'BOOKMARK') return
const gfs: GlobalFileSystemCRD = {
name: apiObj.metadata.name,
namespace: apiObj.metadata.namespace || namespace,
spec: apiObj.spec,
status: apiObj.status,
}
console.log(`[K8s] GlobalFileSystem watch event: ${type} for ${gfs.name}`)
if (type === 'ADDED' || type === 'MODIFIED') {
this.globalFileSystems.set(gfs.name, gfs)
} else if (type === 'DELETED') {
this.globalFileSystems.delete(gfs.name)
}
this.globalFileSystemCacheRevision += 1
await this.enqueueGlobalFileSystemReconciliation(gfs.name, async () => {
try {
if (type === 'ADDED' || type === 'MODIFIED') {
await this.gfsReconciler.reconcile(gfs)
} else if (type === 'DELETED') {
await this.gfsReconciler.reconcileDelete(gfs)
}
} catch (error) {
console.error(`[K8s] GlobalFileSystem reconciliation failed for ${gfs.name}:`, error)
}
})
}
const doneCallback = (err: Error | null) => {
if (this.stopped) return
this.gfsWatchRequest = null
if (err) {
console.error('[K8s] GlobalFileSystem watch error:', err)
}
console.log('[K8s] GlobalFileSystem watch ended, restarting...')
this.scheduleGlobalFileSystemWatchRestart(err ? 5000 : 1000)
}
this.gfsWatchRequest = await this.watch.watch(path, {}, watchCallback, doneCallback)
}
private scheduleGlobalFileSystemWatchRestart(delayMs: number): void {
if (this.stopped || this.gfsWatchRestartTimer) return
this.gfsWatchRestartTimer = setTimeout(() => {
this.gfsWatchRestartTimer = null
if (this.stopped) return
void this.startGlobalFileSystemWatch().catch(error => {
console.error('[K8s] GlobalFileSystem background watch restart failed:', error)
this.scheduleGlobalFileSystemWatchRestart(5000)
})
}, delayMs)
}
/**
* Extracted so unit tests can invoke the handler without spinning up a real
* K8s watch. Production calls this from `startCommunicationChannelWatch` and
* wires it into `this.watch.watch(...)`.
*/
private getCommunicationChannelWatchCallback(): (
type: string,
apiObj: {
metadata: { name: string; namespace?: string }
spec: { hostRef: string; credentialsSecretRef?: { name: string } }
}
) => Promise<void> {
return async (type, apiObj) => {
if (type === 'BOOKMARK') return
const cc: CommunicationChannelCRD = {
name: apiObj.metadata.name,
namespace: apiObj.metadata.namespace || config.channelsNamespace,
spec: {
hostRef: apiObj.spec.hostRef,
...(apiObj.spec.credentialsSecretRef?.name
? { credentialsSecretRef: { name: apiObj.spec.credentialsSecretRef.name } }
: {}),
},
}
console.log(`[K8s] CommunicationChannel watch event: ${type} for ${cc.name}`)
// Track the previous hostRef to handle MODIFIED-with-hostRef-change
// (R5 in the spec): both old and new Hosts must be re-reconciled.
const previous = this.communicationChannels.get(cc.name)
const previousHostRef = previous?.spec.hostRef
if (type === 'ADDED' || type === 'MODIFIED') {
this.communicationChannels.set(cc.name, cc)
} else if (type === 'DELETED') {
this.communicationChannels.delete(cc.name)
} else {
// BOOKMARK or unknown — nothing to reconcile.
return
}
// Reconcile the affected Host(s). For MODIFIED with hostRef change,
// reconcile both. Duplicate reconciles for the same hostRef are
// collapsed via a Set. reconcileHostsReferencingCC is no-throw and
// reports failures so the watch loop stays alive while recovery is queued.
//
// Additionally patch the per-Host channel-reader Deployment's
// credentials-revision annotation so the pod rolls when its CC set
// (or any referenced credentials Secret) changes. Order between the
// two calls doesn't matter — both end up patching the same Deployment
// and patches are idempotent.
const affectedHostRefs = new Set<string>([cc.spec.hostRef])
if (previousHostRef && previousHostRef !== cc.spec.hostRef) {
affectedHostRefs.add(previousHostRef)
}
if (!this.isHostInventoryAuthoritative()) {
const lifecycleGeneration = this.beginCommunicationChannelLifecycleTransition()
this.admitHostDependentEffects(
'CommunicationChannel event Host convergence',
lifecycleGeneration
)
return
}
let needsFleetRetry = false
for (const hostRef of affectedHostRefs) {
if (!(await this.reconcileHostsReferencingCC(hostRef))) {
needsFleetRetry = true
}
try {
await this.hostReconciler.patchChannelReaderRevisionAnnotation(hostRef)
} catch (err) {
console.error(
`[K8s] channel-reader revision patch after CC change failed for "${hostRef}":`,
err
)
needsFleetRetry = true
}
}
if (needsFleetRetry) {
const lifecycleGeneration = this.beginCommunicationChannelLifecycleTransition()
void this.requestHostFleetReconcile(
'CommunicationChannel event convergence fallback',
lifecycleGeneration,
'lifecycle'
)
}
}
}
/**
* Start watching CommunicationChannel CRDs in the channels namespace.
*
* Drives per-Host channel-reader Deployment replicas via
* HostReconciler.countCommunicationChannels (#281). Mirrors the
* SharedFileSystem watch — auto-restart on disconnect.
*/
private async startCommunicationChannelWatch(resourceVersion?: string): Promise<number> {
const watchGeneration = ++this.ccWatchGeneration
await this.openCommunicationChannelWatch(resourceVersion, watchGeneration)
return watchGeneration
}
private async restartCommunicationChannelWatch(
snapshot: CommunicationChannelSnapshot
): Promise<number> {
if (!snapshot.resourceVersion) {
throw new Error('CommunicationChannel snapshot missing resourceVersion')
}
const watchGeneration = ++this.ccWatchGeneration
if (this.ccWatchRequest) {
this.ccWatchRequest.abort()
this.ccWatchRequest = null
}
this.installCommunicationChannelSnapshot(snapshot)
await this.openCommunicationChannelWatch(snapshot.resourceVersion, watchGeneration)
return watchGeneration
}
private async openCommunicationChannelWatch(
resourceVersion: string | undefined,
watchGeneration: number
): Promise<void> {
const path = `/apis/${GROUP}/${VERSION}/namespaces/${config.channelsNamespace}/${PLURAL_COMMUNICATIONCHANNELS}`
console.log(`[K8s] Starting CommunicationChannel watch`)
let watchEnded = false
const applyWatchEvent = this.getCommunicationChannelWatchCallback()
const watchCallback = async (
type: string,
apiObj: {
metadata: { name: string; namespace?: string }
spec: { hostRef: string; credentialsSecretRef?: { name: string } }
}
) => {
if (watchEnded || this.stopped || watchGeneration !== this.ccWatchGeneration) return
await applyWatchEvent(type, apiObj)
}
const doneCallback = (err: Error | null) => {
if (watchEnded || this.stopped || watchGeneration !== this.ccWatchGeneration) return
watchEnded = true
this.ccWatchRequest = null
this.ccCacheSynced = false
const lifecycleGeneration = this.beginCommunicationChannelLifecycleTransition()
if (err) {
console.error('[K8s] CommunicationChannel watch error:', err)
}
hccLogger.info(
'[K8s] CommunicationChannel watch ended; preserving durable Host lifecycle state until snapshot recovery'
)
void this.requestHostFleetReconcile(
'CommunicationChannel watch interruption',
lifecycleGeneration
)
this.scheduleCommunicationChannelCacheRecovery()
}
const request = await this.watch.watch(
path,
resourceVersion ? { resourceVersion } : {},
watchCallback,
doneCallback
)
if (this.stopped || watchGeneration !== this.ccWatchGeneration || watchEnded) {
request.abort()
if (!this.stopped && watchGeneration === this.ccWatchGeneration && watchEnded) {
throw new Error('CommunicationChannel watch ended before request initialization completed')
}
return
}
this.ccWatchRequest = request
}
private runHostResync(): Promise<void> {
if (this.stopped) return Promise.resolve()
if (this.hostResyncInFlight) return this.hostResyncInFlight
const run = this.performHostResync()
this.hostResyncInFlight = run
const settle = () => {
if (this.hostResyncInFlight === run) this.hostResyncInFlight = null
}
void run.then(settle, settle)
return run
}
private async performHostResync(): Promise<void> {
if (!this.ccCacheSynced) {
await this.recoverCommunicationChannelCache()
return
}
const pendingLifecycleGeneration =
this.ccLifecycleGeneration !== this.ccAppliedLifecycleGeneration
? this.ccLifecycleGeneration
: undefined
if (pendingLifecycleGeneration !== undefined) {
await this.requestHostFleetReconcile(
'Periodic lifecycle convergence',
pendingLifecycleGeneration,
'lifecycle'
)
if (this.ccAppliedLifecycleGeneration !== pendingLifecycleGeneration) return
}
await this.requestHostFleetReconcile('Periodic resync', undefined, 'full')
}
/**
* Periodic SharedFileSystem resync (#592). The SFS watch only fires on SFS CRD
* changes, not on the PVC binding or the wfc pod becoming Ready. Re-running
* fullReconcile every N seconds lets a SharedFileSystem that reported
* Initializing/Degraded (PVC still binding under WaitForFirstConsumer, or the
* wfc pod still starting) transition to a truthful Ready once the volume binds
* and the pod serves — without waiting for an unrelated SFS event.
*/
/**
* Periodic GlobalFileSystem re-reconcile. The gfs watch only fires on the CRD
* changing, so a GlobalFileSystem that reported Initializing (writer not yet
* Available at first reconcile) needs this to converge to Ready and to run its
* one-time root-directory seed once the writer is up. Idempotent; per-item
* failures are logged by fullReconcile, never thrown.
*/
private async runGfsResync(): Promise<void> {
if (this.stopped) return
try {
const cacheRevisionAtListStart = this.globalFileSystemCacheRevision
const gfses = await listAllGlobalFileSystems()
const names = [
...this.globalFileSystems.keys(),
...gfses.map(globalFileSystem => globalFileSystem.name),
]
return this.enqueueGlobalFileSystemFleetReconciliation(names, () =>
this.runGfsResyncCore(gfses, cacheRevisionAtListStart)
)
} catch (err) {
console.error(
`[K8s] GlobalFileSystem periodic resync failed: ${
err instanceof Error ? err.message : String(err)
}`
)
}
}
private async runGfsResyncCore(
listedGfses?: GlobalFileSystemCRD[],
listedAtCacheRevision?: number
): Promise<void> {
if (this.stopped) return
try {
const cacheRevisionAtListStart = listedAtCacheRevision ?? this.globalFileSystemCacheRevision
const gfses = listedGfses ?? (await listAllGlobalFileSystems())
let reconcileInventory: GlobalFileSystemCRD[]
if (cacheRevisionAtListStart === this.globalFileSystemCacheRevision) {
this.globalFileSystems.clear()
for (const gfs of gfses) this.globalFileSystems.set(gfs.name, gfs)
reconcileInventory = gfses
} else {
reconcileInventory = [...this.globalFileSystems.values()]
}
await this.gfsReconciler.fullReconcile(reconcileInventory)
} catch (err) {
console.error(
`[K8s] GlobalFileSystem periodic resync failed: ${
err instanceof Error ? err.message : String(err)
}`
)
}
}
private async runSfsResync(): Promise<void> {
if (this.stopped) return
try {
const cacheRevisionAtListStart = this.sharedFileSystemCacheRevision
const sfses = await listAllSharedFileSystems()
const names = [
...this.sharedFileSystems.keys(),
...sfses.map(sharedFileSystem => sharedFileSystem.name),
]
return this.enqueueSharedFileSystemFleetReconciliation(names, () =>
this.runSfsResyncCore(sfses, cacheRevisionAtListStart)
)
} catch (error) {
console.error('[K8s] Periodic SharedFileSystem resync failed:', error)
}
}
private async runSfsResyncCore(
listedSfses?: SharedFileSystemCRD[],
listedAtCacheRevision?: number
): Promise<void> {
if (this.stopped) return
try {
const cacheRevisionAtListStart = listedAtCacheRevision ?? this.sharedFileSystemCacheRevision
const sfses = listedSfses ?? (await listAllSharedFileSystems())
let reconcileInventory: SharedFileSystemCRD[]
if (cacheRevisionAtListStart === this.sharedFileSystemCacheRevision) {
this.sharedFileSystems.clear()
for (const sfs of sfses) this.sharedFileSystems.set(sfs.name, sfs)
reconcileInventory = sfses
} else {
reconcileInventory = [...this.sharedFileSystems.values()]
}
// Capture each SFS's mountability (PVC Bound) BEFORE reconciling so we can
// detect a flip. resolveContextMounts() injects the RO mount + podAffinity
// only while the SFS is mountable, so a mountable⇄not-mountable flip changes
// the mount set of every consuming mcp-host pod. (Keyed on mountability, NOT
// on phase==='Ready': a transient wfc readiness dip while the PVC stays
// Bound must NOT re-roll consumers — see resolveContextMounts.)
const wasMountable = new Map<string, boolean>()
for (const sfs of reconcileInventory) {
wasMountable.set(sfs.name, this.sharedFileSystemReconciler.isMountable(sfs))
}
await this.sharedFileSystemReconciler.fullReconcile(reconcileInventory)
// #592 gap fix: heal the consuming mcp-host Deployments on a mountability
// flip — inject the mount + affinity once the PVC binds, or drop it if the
// volume becomes genuinely unusable. This does NOT rely on the status-
// subresource MODIFIED self-event, which the apiserver suppresses for a
// no-op patch and which our dirty check makes even less likely to fire — so
// an already-Bound SFS picked up on HCC restart still re-injects its mount
// within one resync interval instead of waiting for the (much longer) Host
// resync.
for (const sfs of reconcileInventory) {
const nowMountable = this.sharedFileSystemReconciler.isMountable(sfs)
if (nowMountable !== (wasMountable.get(sfs.name) ?? false)) {
await this.reconcileHostsReferencingSfs(sfs.name)
}
}
} catch (error) {
console.error('[K8s] Periodic SharedFileSystem resync failed:', error)
}
}
/**
* Start watching McpServer CRDs.
*/
private getMcpServerWatchCallback(
watchGeneration = this.mcpWatchGeneration
): (type: string, apiObj: McpServerWatchObject) => Promise<void> {
return async (type, apiObj) => {
if (type === 'BOOKMARK') return
if (this.stopped || watchGeneration !== this.mcpWatchGeneration) return
const server: McpServerCRD = {
name: apiObj.metadata.name,
namespace: apiObj.metadata.namespace || config.namespace,
uid: apiObj.metadata.uid,
generation: apiObj.metadata.generation,
annotations: apiObj.metadata.annotations,
labels: apiObj.metadata.labels,
spec: apiObj.spec,
status: apiObj.status,
}
console.log(`[K8s] McpServer watch event: ${type} for ${server.name}`)
const previous = this.servers.get(server.name)
// Update cache
let desiredStateChanged = false
if (type === 'ADDED' || type === 'MODIFIED') {
desiredStateChanged =
previous === undefined || !sameMcpServerPolicyRevision(previous, server)
this.servers.set(server.name, server)
} else if (type === 'DELETED') {
if (!previous?.uid || !server.uid || previous.uid === server.uid) {
desiredStateChanged = this.servers.delete(server.name)
}
}
if (desiredStateChanged) {
this.mcpServerDesiredRevision += 1
void this.runInitialNetworkPolicyConvergence({ cause: 'mcp-change' })
}
// Most status writes do not change desired state. A runtime eligibility
// edge does change policy intent: propagate it without re-entering the
// runtime owner inline. The scheduled replay also retries runtime
// reconciliation. Discovery receives every update; UID/spec/annotation/
// label changes remain live.
if (type === 'MODIFIED' && previous && isMcpServerStatusOnlyUpdate(previous, server)) {
if (runtimeDesired(previous) !== runtimeDesired(server)) {
// The revision bump above retires old policy work and requests a
// certified safety pass; replay also retries per-server additions.
this.externalEgressCoordinator.scheduleRetry('MODIFIED', server)
try {
await this.reconcileContextsForMcpServer(server, watchGeneration)
} catch (err) {
// The certified fleet pass and the server replay remain scheduled.
hccLogger.error('Runtime verdict policy propagation failed', {
serverName: server.name,
err,
})
}
}
this.changeCallback?.()
return
}
await this.enqueueMcpServerReconciliation(server, () =>
this.reconcileMcpServerWatchEvent(
type as ExternalEgressWatchEventType,
server,
watchGeneration
)
)
}
}
private async reconcileContextsForMcpServer(
server: McpServerCRD,
watchGeneration: number
): Promise<void> {
const contextGeneration = this.contextWatchGeneration
const isCurrent = (): boolean => {
const current = this.servers.get(server.name)
return (
this.mcpServerWatchEffectIsCurrent(server, watchGeneration) &&
current !== undefined &&
sameMcpServerPolicyRevision(server, current) &&
this.hasContextInventoryAuthority(contextGeneration)
)
}
for (const selected of this.contexts.values()) {
if (!isCurrent()) return
if (!selected.spec.mcpServers?.includes(server.name)) continue
const contextId = selected.spec.contextId
await this.enqueueContextReconciliation(contextId, async () => {
if (!isCurrent()) return
const current = this.contexts.get(selected.name)
if (
current?.spec.contextId !== contextId ||
!current.spec.mcpServers?.includes(server.name)
)
return
hccLogger.debug('Reconciling Context policies after McpServer change', {
serverName: server.name,
contextName: current.name,
contextId,
})
const completed = await this.netPolReconciler.reconcileContext(current, {
isCurrent: () => isCurrent() && this.contexts.get(current.name) === current,
})
if (completed === false && isCurrent()) {
this.scheduleExternalEgressRetry('MODIFIED', server)
}
})
}
}
private async reconcileMcpServerWatchEvent(
type: ExternalEgressWatchEventType,
server: McpServerCRD,
watchGeneration: number,
retry?: ExternalEgressRetryHandle
): Promise<void> {
if (this.stopped || watchGeneration !== this.mcpWatchGeneration) return
if (retry && !retry.isCurrent()) return
if (retry && !this.hasMcpServerInventoryAuthority(watchGeneration)) {
this.scheduleExternalEgressRetry(type, server)
return
}
const current = this.servers.get(server.name)
if (type === 'DELETED') {
// A same-name recreation may already be accepted while this callback
// waits for its per-server effect. The newer identity owns the key.
if (current) return
} else {
if (!current || current.namespace !== server.namespace) return
server = current
}
const deleteAllowed = () =>
this.mcpServerAbsentForDelete(server.name, server.namespace, watchGeneration)
// External egress is part of the workload's pre-start contract. Reconcile
// it before HCC creates or updates any managed runtime Deployment so a
// stdio MCP with egressBindings cannot start before ExternalEgressReady.
let retryCompletion: ExternalEgressRetryHandle | undefined
if (type === 'ADDED' || type === 'MODIFIED') {
try {
retryCompletion = await this.runExternalEgressOnce(type, server, {
retry,
isCurrent: () =>
this.mcpServerWatchEffectIsCurrent(server, watchGeneration) &&
(!retry || this.hasMcpServerInventoryAuthority(watchGeneration)),
})
} catch (error) {
console.error(
`[K8s] External egress reconciliation failed for ${server.name}; runtime reconciliation blocked:`,
error
)
this.scheduleExternalEgressRetry(type, server)
this.changeCallback?.()
return
}
} else {
if (!(await this.authorizeMcpServerDeleteOrRetry(server, watchGeneration))) return
try {
retryCompletion = await this.runExternalEgressOnce(type, server, {
deleteAllowed,
retry,
isCurrent: () =>
!this.stopped &&
watchGeneration === this.mcpWatchGeneration &&
(!retry || this.hasMcpServerInventoryAuthority(watchGeneration)),
})
} catch (error) {
console.error(
`[K8s] External egress reconciliation failed for ${server.name}; runtime reconciliation blocked:`,
error
)
this.scheduleExternalEgressRetry(type, server)
this.changeCallback?.()
return
}
}
if (type === 'ADDED' || type === 'MODIFIED') {
if (retry && !this.hasMcpServerInventoryAuthority(watchGeneration)) {
this.scheduleExternalEgressRetry(type, server)
return
}
if (!this.mcpServerWatchEffectIsCurrent(server, watchGeneration)) return
server = this.servers.get(server.name)!
} else if (!(await this.authorizeMcpServerDeleteOrRetry(server, watchGeneration))) {
return
}
// Trigger deployment reconciliation.
try {
if (type === 'ADDED' || type === 'MODIFIED') {
const currentServer = server
await this.reconciler.reconcile(currentServer, {
isCurrent: () => this.mcpServerWatchEffectIsCurrent(currentServer, watchGeneration),
})
} else if (type === 'DELETED') {
await this.reconciler.reconcileDelete(server.name, server.namespace)
}
} catch (error) {
console.error(`[K8s] Reconciliation failed for ${server.name}:`, error)
this.scheduleExternalEgressRetry(type, server)
this.changeCallback?.()
return
}
if (type === 'ADDED' || type === 'MODIFIED') {
if (!this.mcpServerWatchEffectIsCurrent(server, watchGeneration)) return
server = this.servers.get(server.name)!
} else if (!(await this.authorizeMcpServerDeleteOrRetry(server, watchGeneration))) {
return
}
// Trigger binding policy reconciliation (L3 ingress/egress).
try {
if (type === 'ADDED' || type === 'MODIFIED') {
const bindingsJson = server.annotations?.['clerum.io/recipe-bindings']
if (bindingsJson) {
const bindings: BindingDef[] = JSON.parse(bindingsJson)
const recipeName = server.labels?.['clerum.io/recipe'] ?? server.name
const mcpWorkloadName = server.labels?.['clerum.io/workload'] ?? server.name
const bindingServer = server
await this.bindingReconciler.reconcileBindings(
recipeName,
bindings,
mcpWorkloadName,
bindingServer.name,
{
isCurrent: () => this.mcpServerWatchEffectIsCurrent(bindingServer, watchGeneration),
}
)
if (!this.mcpServerWatchEffectIsCurrent(server, watchGeneration)) return
server = this.servers.get(server.name)!
}
await this.reconcileContextsForMcpServer(server, watchGeneration)
} else if (type === 'DELETED') {
const recipeName = server.labels?.['clerum.io/recipe'] ?? server.name
await this.bindingReconciler.cleanupBindings(recipeName, { deleteAllowed })
}
} catch (error) {
console.error(`[K8s] Binding/egress reconciliation failed for ${server.name}:`, error)
this.scheduleExternalEgressRetry(type, server)
this.changeCallback?.()
return
}
retryCompletion?.complete()
this.changeCallback?.()
}
private async startMcpServerWatch(resourceVersion: string): Promise<number> {
this.requireInventoryResourceVersion('McpServer', resourceVersion)
const path = `/apis/${GROUP}/${VERSION}/namespaces/${config.namespace}/${PLURAL_MCPSERVERS}`
console.log(`[K8s] Starting McpServer watch`)
const watchGeneration = ++this.mcpWatchGeneration
if (this.mcpWatchRequest) {
this.mcpWatchRequest.abort()
this.mcpWatchRequest = null
}
let watchEnded = false
const watchCallback = this.getMcpServerWatchCallback(watchGeneration)
const doneCallback = (err: Error | null) => {
if (watchEnded) return
watchEnded = true
if (this.stopped || watchGeneration !== this.mcpWatchGeneration) return
if (!this.retireMcpServerWatch(watchGeneration)) return
if (err) {
console.error('[K8s] McpServer watch error:', err)
}
console.log('[K8s] McpServer watch ended; recovering authoritative inventory...')
this.attemptMcpServerCacheRecovery()
}
const request = await this.watch.watch(
path,
{ resourceVersion, allowWatchBookmarks: true },
watchCallback,
doneCallback
)
if (this.stopped || watchGeneration !== this.mcpWatchGeneration || watchEnded) {
request.abort()
return watchGeneration
}
this.mcpWatchRequest = request
return watchGeneration
}
private mcpServerWatchEffectIsCurrent(expected: McpServerCRD, watchGeneration: number): boolean {
if (this.stopped || watchGeneration !== this.mcpWatchGeneration) return false
const current = this.servers.get(expected.name)
return current !== undefined && sameMcpServerDesiredRevision(expected, current)
}
private runExternalEgressOnce(
type: ExternalEgressWatchEventType,
server: McpServerCRD,
options: {
deleteAllowed?: () => Promise<boolean>
retry?: ExternalEgressRetryHandle
isCurrent?: () => boolean
} = {}
): Promise<ExternalEgressRetryHandle | undefined> {
return this.externalEgressCoordinator.reconcile(type, server, options)
}
private scheduleExternalEgressRetry(type: string, server: McpServerCRD): void {
this.externalEgressCoordinator.scheduleRetry(type, server)
}
private async performExternalEgressMutation(
type: ExternalEgressWatchEventType,
server: McpServerCRD,
options: {
deleteAllowed?: () => Promise<boolean>
isCurrent: () => boolean
}
): Promise<void> {
if (type === 'DELETED') {
const deleteAllowed = async (): Promise<boolean> => {
if (!options.isCurrent()) return false
if (options.deleteAllowed && !(await options.deleteAllowed())) return false
return options.isCurrent()
}
await this.netPolReconciler.cleanupExternalEgress(
server.name,
server.namespace,
undefined,
deleteAllowed
)
return
}
await this.netPolReconciler.reconcileExternalEgress(server, {
isCurrent: options.isCurrent,
})
}
private getLlmHookWatchCallback(): (
type: string,
apiObj: {
metadata: {
name: string
namespace?: string
uid?: string
generation?: number
annotations?: Record<string, string>
labels?: Record<string, string>
}
spec: LlmHookSpec
status?: LlmHookCRD['status']
}
) => Promise<void> {
return async (type, apiObj) => {
const hook: LlmHookCRD = {
name: apiObj.metadata.name,
namespace: apiObj.metadata.namespace || config.llmHooksNamespace,
uid: apiObj.metadata.uid,
generation: apiObj.metadata.generation,
annotations: apiObj.metadata.annotations,
labels: apiObj.metadata.labels,
spec: apiObj.spec,
status: apiObj.status,
}
console.log(`[K8s] LlmHook watch event: ${type} for ${hook.name}`)
// Compute the pod key the CR had BEFORE this event so an image bump can
// chain teardown of the old pod key with ensure of the new one (§4), and
// a delete can GC the workload the CR was a member of.
const previous = this.llmHooks.get(hook.name)
const previousPodKey = previous ? computePodKey(previous) : null
if (type === 'ADDED' || type === 'MODIFIED') {
this.llmHooks.set(hook.name, hook)
} else if (type === 'DELETED') {
this.llmHooks.delete(hook.name)
}
try {
if (type === 'ADDED' || type === 'MODIFIED') {
await this.llmHookReconciler.reconcile(hook, previousPodKey)
} else if (type === 'DELETED') {
await this.llmHookReconciler.reconcileDelete(hook.name, previousPodKey)
}
} catch (error) {
console.error(`[K8s] LlmHook reconciliation failed for ${hook.name}:`, error)
}
}
}
/**
* Start watching LlmHook CRDs in the llm-hooks namespace.
*/
private async startLlmHookWatch(): Promise<void> {
const path = `/apis/${GROUP}/${VERSION}/namespaces/${config.llmHooksNamespace}/${PLURAL_LLMHOOKS}`
console.log(`[K8s] Starting LlmHook watch`)
const watchCallback = this.getLlmHookWatchCallback()
const doneCallback = (err: Error | null) => {
if (this.stopped) return
if (err) {
console.error('[K8s] LlmHook watch error:', err)
}
console.log('[K8s] LlmHook watch ended, restarting...')
setTimeout(() => this.startLlmHookWatch(), err ? 5000 : 1000)
}
this.llmHookWatchRequest = await this.watch.watch(path, {}, watchCallback, doneCallback)
}
/** Periodic LlmHook resync: full reconcile drives the orphan sweep (§3). */
private async runLlmHookResync(): Promise<void> {
if (this.stopped) return
try {
await this.llmHookReconciler.fullReconcile([...this.llmHooks.values()])
} catch (error) {
console.error('[K8s] LlmHook periodic resync failed:', error)
}
}
/**
* Start watching Context CRDs for NetworkPolicy reconciliation.
*/
private async startContextWatch(resourceVersion: string): Promise<number> {
this.requireInventoryResourceVersion('Context', resourceVersion)
const path = `/apis/${GROUP}/${VERSION}/namespaces/${config.namespace}/${PLURAL_CONTEXTS}`
console.log(`[K8s] Starting Context watch`)
const watchGeneration = ++this.contextWatchGeneration
if (this.ctxWatchRequest) {
this.ctxWatchRequest.abort()
this.ctxWatchRequest = null
}
let watchEnded = false
const watchCallback = async (
type: string,
apiObj: {
metadata: {
name: string
namespace?: string
uid?: string
generation?: number
}
spec: ContextSpec
}
) => {
if (type === 'BOOKMARK') return
if (this.stopped || watchGeneration !== this.contextWatchGeneration) return
const context: ContextCRD = {
name: apiObj.metadata.name,
namespace: apiObj.metadata.namespace || config.namespace,
uid: apiObj.metadata.uid,
generation: apiObj.metadata.generation,
spec: apiObj.spec,
}
console.log(`[K8s] Context watch event: ${type} for ${context.name}`)
// Capture the previous spec before mutating the cache so we can
// re-reconcile any SFS that *was* referenced but is no longer.
const previous = this.contexts.get(context.name)
// A scoped Context delta may replace only a prior, fully authoritative
// safety certificate. Capturing it before this event changes desired
// state proves there was no outstanding Context or McpServer revocation
// work that the scoped reconciliation could accidentally certify.
const previousSafetyCertificate = this.currentNetworkPolicySafetyCertificate()
// Cache the context for cross-resource re-reconciliation
let desiredStateChanged = false
let deltaSafetyCertificate: NetworkPolicySafetyCertificate | undefined
if (type === 'ADDED' || type === 'MODIFIED') {
desiredStateChanged =
previous === undefined || !sameContextDesiredRevision(previous, context)
// Keep the cached object identity when desired revision did not move.
// Scoped revoke uses `contexts.get(name) === selectedContext`; replacing
// the entry on every label/status MODIFIED aborts an in-flight pass.
// `desiredStateChanged` was just computed as
// `previous === undefined || !sameContextDesiredRevision(...)`, so the
// bootstrap case already implies it; a second disjunct here was dead
// (#568 review, jozer-rami minors).
if (desiredStateChanged) {
this.contexts.set(context.name, context)
}
} else if (type === 'DELETED') {
desiredStateChanged = this.contexts.delete(context.name)
}
if (desiredStateChanged) {
this.contextDesiredRevision += 1
// A same-identity Context MODIFIED event can complete its own stale
// allow revocation below. Once that exact delta is current, it is safe
// to restore readiness without waiting for an older pass's additive
// fleet. Deletes and identity changes remain fail-closed until the
// authoritative full pass certifies their broader cleanup.
if (
type === 'MODIFIED' &&
previous?.uid === context.uid &&
previous?.spec.contextId === context.spec.contextId &&
previousSafetyCertificate !== null
) {
deltaSafetyCertificate = {
contextGeneration: previousSafetyCertificate.contextGeneration,
serverGeneration: previousSafetyCertificate.serverGeneration,
contextRevision: this.contextDesiredRevision,
serverRevision: previousSafetyCertificate.serverRevision,
}
}
void this.runInitialNetworkPolicyConvergence({ cause: 'context-change' })
}
// Metadata-only Context MODIFIED (label ping-pong, annotation churn)
// does not change desired revision. Skip SFS propagate and Host fan-out
// — that accidental resync is the #460 Loop A storm. Keep scoped
// NetworkPolicy revoke: without a delta certificate honorsLostFence is
// false, so a lost delete fence still throws into full convergence.
// Periodic resync (default 0 / #478) is not the healer for this path.
// No changeCallback here. The Context watch never notified before this PR,
// and adding the notification on THIS path would have fired it exactly
// where nothing changed — the label ping-pong this gate exists to silence —
// while a real desired-state change on the same watch still would not fire
// it. The only subscriber is a `console.log` in main.ts that reports the
// McpServer cache, so for a Context event the line is wrong, and the
// `getAllServers()` behind it is an O(n) rebuild on the hot path this PR
// removes work from (#568 review: jozer-rami #2 / zach88 R1-M1, converged).
const metadataOnlyModified = type === 'MODIFIED' && !desiredStateChanged
// The absorption must not be invisible: if the Loop A storm ever returns,
// HCC now silently eats it. A COUNTER rather than a log line, because this
// fires on exactly the churn that made the logs unreadable in the first
// place (#492) — a per-event line would reproduce the problem it is meant
// to make observable (claude[bot] audit, finding 1).
if (metadataOnlyModified) contextMetadataOnlyEventsTotal.inc()
// Re-reconcile every SFS this Context referenced before or after the
// change so SharedFileSystem.status.mountedByContexts stays in sync
// without waiting for an SFS-level event.
const nextForSfs = type === 'DELETED' ? undefined : context
const contextInventoryAuthoritative = () => this.hasContextInventoryAuthority(watchGeneration)
const serverInventoryGeneration = this.mcpWatchGeneration
if (!metadataOnlyModified) {
void this.reconcileSharedFileSystemsReferencedByContext(
previous,
nextForSfs,
contextInventoryAuthoritative
)
}
const previousContextId = previous?.spec.contextId
const effectContextIds =
previousContextId && previousContextId !== context.spec.contextId
? [previousContextId, context.spec.contextId]
: [context.spec.contextId]
await this.enqueueContextIdentityReconciliation(effectContextIds, async () => {
if (this.stopped || watchGeneration !== this.contextWatchGeneration) return
try {
if (previousContextId && previousContextId !== context.spec.contextId) {
const oldIdDeleteAllowed = () =>
this.contextIdAbsentForDelete(previousContextId, watchGeneration)
if (await oldIdDeleteAllowed()) {
await this.netPolReconciler.reconcileDeleteContext(
previousContextId,
oldIdDeleteAllowed
)
}
}
if (this.stopped || watchGeneration !== this.contextWatchGeneration) return
const current = this.contexts.get(context.name)
if (type === 'DELETED') {
if (current) return
} else if (current?.spec.contextId !== context.spec.contextId) {
return
}
if (type === 'ADDED' || type === 'MODIFIED') {
const selectedContext = current!
// A scoped delta may only replace the safety certificate when its
// own stale-allow revocation ran to completion — `reconcileContext`
// aborts mid-pass whenever its authority fence breaks — and when
// the last authoritative pass certified its namespace-wide
// inventory, which a label-scoped delta cannot vouch for.
// `deltaSafetyCertificate` is only ever built for MODIFIED, so this
// is the one branch where it can be recorded.
const scopedRevocationCompleted = await this.netPolReconciler.reconcileContext(
selectedContext,
{
isCurrent: () =>
contextInventoryAuthoritative() &&
this.hasMcpServerInventoryAuthority(serverInventoryGeneration) &&
this.contexts.get(selectedContext.name) === selectedContext,
// Only opt in when there is a certificate to withhold. Without
// one the result below is discarded, so a reported lost fence
// would vanish: no throw to reach the retry. Metadata-only
// MODIFIED never builds a certificate, so this stays false and
// a 409 still reaches full convergence without Host/SFS fan-out.
honorsLostFence: deltaSafetyCertificate !== undefined,
}
)
if (deltaSafetyCertificate && scopedRevocationCompleted) {
if (this.netPolReconciler.hasCertifiedSafetyInventory()) {
this.recordNetworkPolicySafetyCertificate(deltaSafetyCertificate)
} else {
console.warn(
`[K8s] Not certifying the scoped delta for context "${context.spec.contextId}": the last authoritative safety pass ended without certifying its namespace-wide inventory`
)
}
}
} else if (type === 'DELETED') {
const deleteAllowed = () =>
this.contextAbsentForDelete(context.name, context.namespace, watchGeneration)
if (!(await deleteAllowed())) return
await this.netPolReconciler.reconcileDeleteContext(
context.spec.contextId,
deleteAllowed
)
}
} catch (error) {
hccLogger.error(
`[K8s] NetworkPolicy reconciliation failed for context ${context.name}:`,
{
contextName: context.name,
err: error,
}
)
void this.runInitialNetworkPolicyConvergence({ cause: 'context-reconcile-failure' })
return
}
if (this.stopped || watchGeneration !== this.contextWatchGeneration) return
if (metadataOnlyModified) {
return
}
// Re-reconcile any Host that points at this Context so that changes to
// spec.sharedFileSystems[] propagate into the mcp-host pod template.
try {
await this.reconcileHostsReferencingContext(context.name, contextInventoryAuthoritative)
} catch (error) {
console.error(
`[K8s] Host re-reconcile after Context "${context.name}" change failed:`,
error
)
}
})
}
const doneCallback = (err: Error | null) => {
if (watchEnded) return
watchEnded = true
if (this.stopped || watchGeneration !== this.contextWatchGeneration) return
if (!this.retireContextWatch(watchGeneration)) return
if (err) {
console.error('[K8s] Context watch error:', err)
}
console.log('[K8s] Context watch ended; recovering authoritative inventory...')
this.attemptContextCacheRecovery()
}
const request = await this.watch.watch(
path,
{ resourceVersion, allowWatchBookmarks: true },
watchCallback,
doneCallback
)
if (this.stopped || watchGeneration !== this.contextWatchGeneration || watchEnded) {
request.abort()
return watchGeneration
}
this.ctxWatchRequest = request
return watchGeneration
}
/**
* Start watching Host CRDs for runtime reconciliation.
*/
private async startHostWatch(resourceVersion: string): Promise<number> {
if (!resourceVersion) {
throw new Error('Host watch requires a snapshot resourceVersion')
}
const path = `/apis/${GROUP}/${VERSION}/namespaces/${config.hostNamespace}/${PLURAL_HOSTS}`
const watchGeneration = ++this.hostWatchGeneration
if (this.hostWatchRequest) {
this.hostWatchRequest.abort()
this.hostWatchRequest = null
}
console.log(`[K8s] Starting Host watch`)
let watchEnded = false
const watchCallback = async (
type: string,
apiObj: {
metadata: {
name: string
namespace?: string
uid?: string
generation?: number
resourceVersion?: string
annotations?: Record<string, string>
}
spec: HostSpec
status?: HostCRD['status']
}
) => {
if (watchEnded || this.stopped || watchGeneration !== this.hostWatchGeneration) return
const host: HostCRD = {
name: apiObj.metadata.name,
namespace: apiObj.metadata.namespace || config.hostNamespace,
uid: apiObj.metadata.uid,
generation: apiObj.metadata.generation,
resourceVersion: apiObj.metadata.resourceVersion,
annotations: apiObj.metadata.annotations,
spec: apiObj.spec,
status: apiObj.status,
}
console.log(`[K8s] Host watch event: ${type} for ${host.name}`)
if (type !== 'ADDED' && type !== 'MODIFIED' && type !== 'DELETED') return
const eventType: HostWatchEventType = type
this.clearHostWatchRetry(host.name)
const eventRevision = ++this.hostWatchRevision
this.latestHostWatchEventRevisions.set(host.name, eventRevision)
// Advance hostDesiredRevision only on a real desired-state change (the
// content identity the mutation-authority fence reads), separate from the
// per-event hostWatchRevision counter above.
const previousHost = this.hosts.get(host.name)
let hostDesiredStateChanged = false
// Same pre-event snapshot, aliased for the Host→LlmHook reverse index: a
// removed reference must re-reconcile the (now smaller) NetworkPolicy
// ingress set for the affected hook pod keys.
const previousHostForHooks = previousHost
if (eventType === 'ADDED' || eventType === 'MODIFIED') {
hostDesiredStateChanged =
previousHost === undefined || !this.sameHostDesiredRevision(previousHost, host)
this.hosts.set(host.name, host)
} else {
hostDesiredStateChanged = this.hosts.delete(host.name)
}
if (hostDesiredStateChanged) this.hostDesiredRevision += 1
try {
// Direct dispatch (§10.3): reconcileHostWatchEvent enters the per-Host
// serializer inside reconcile()/reconcileDelete(). The removed
// process-wide convergence tail no longer gates independent Hosts.
await this.reconcileHostWatchEvent(eventType, host, eventRevision)
this.completeHostWatchEvent(host.name, eventRevision)
} catch (error) {
console.error(`[K8s] Host reconciliation failed for ${host.name}:`, error)
this.scheduleHostWatchReconcileRetry(eventType, host, eventRevision)
}
// Fan out to the LlmHook NetworkPolicy ingress (§5): re-reconcile the hook
// pod keys this Host references now (or referenced before), so ingress
// admits exactly the current set of mcp-hosts.
const affectedHookIds = new Set<string>([
...referencedHookIds(previousHostForHooks),
...(eventType === 'DELETED' ? [] : referencedHookIds(host)),
])
if (affectedHookIds.size > 0) {
try {
await this.llmHookReconciler.reconcileNetworkPoliciesForHooks([...affectedHookIds])
} catch (error) {
console.error(
`[K8s] LlmHook NetworkPolicy fan-out after Host "${host.name}" change failed:`,
error
)
}
}
// Keep this Host's scoped egress-to-hooks policy in sync with its CURRENT
// references (N1/N7) — in particular the "dropped the last hook reference"
// case, where the policy must be removed. On DELETE the Host reconciler's
// deleteHostNetworkPolicies removes it by name.
if (eventType !== 'DELETED') {
try {
await this.llmHookReconciler.reconcileHostEgress(host)
} catch (error) {
console.error(`[K8s] Host egress-to-hooks reconcile for "${host.name}" failed:`, error)
}
}
}
const doneCallback = (err: Error | null) => {
if (watchEnded || this.stopped || watchGeneration !== this.hostWatchGeneration) return
watchEnded = true
this.hostWatchRequest = null
this.hostCacheSynced = false
this.hostWatchGeneration += 1
this.clearAllHostWatchRetries()
if (err) {
console.error('[K8s] Host watch error:', err)
}
console.log('[K8s] Host watch ended; rebuilding the Host snapshot before watch recovery')
this.attemptHostCacheRecovery()
}
const request = await this.watch.watch(path, { resourceVersion }, watchCallback, doneCallback)
if (this.stopped || watchGeneration !== this.hostWatchGeneration || watchEnded) {
request.abort()
if (!this.stopped && watchGeneration !== this.hostWatchGeneration && watchEnded) {
throw new Error('Host watch ended before request initialization completed')
}
return watchGeneration
}
this.hostWatchRequest = request
return watchGeneration
}
/**
* Stop all watches.
*/
async stop(): Promise<void> {
this.stopped = true
this.externalEgressCoordinator.stop()
this.retireMcpServerWatch()
this.retireContextWatch()
this.ccCacheSynced = false
this.hostCacheSynced = false
this.hostWatchGeneration += 1
this.ccWatchGeneration += 1
this.hostFleetScheduler.stop()
if (this.ccCacheRecoveryTimer) {
clearTimeout(this.ccCacheRecoveryTimer)
this.ccCacheRecoveryTimer = null
}
if (this.hostCacheRecoveryTimer) {
clearTimeout(this.hostCacheRecoveryTimer)
this.hostCacheRecoveryTimer = null
}
if (this.sfsWatchRestartTimer) {
clearTimeout(this.sfsWatchRestartTimer)
this.sfsWatchRestartTimer = null
}
if (this.gfsWatchRestartTimer) {
clearTimeout(this.gfsWatchRestartTimer)
this.gfsWatchRestartTimer = null
}
this.hostCacheRecoveryIntent = null
if (this.mcpServerCacheRecoveryTimer) {
clearTimeout(this.mcpServerCacheRecoveryTimer)
this.mcpServerCacheRecoveryTimer = null
}
if (this.contextCacheRecoveryTimer) {
clearTimeout(this.contextCacheRecoveryTimer)
this.contextCacheRecoveryTimer = null
}
this.clearAllHostWatchRetries()
if (this.resyncTimer) {
clearInterval(this.resyncTimer)
this.resyncTimer = null
}
if (this.sfsResyncTimer) {
clearInterval(this.sfsResyncTimer)
this.sfsResyncTimer = null
}
if (this.llmHookResyncTimer) {
clearInterval(this.llmHookResyncTimer)
this.llmHookResyncTimer = null
}
if (this.gfsResyncTimer) {
clearInterval(this.gfsResyncTimer)
this.gfsResyncTimer = null
}
if (this.netPolResyncTimer) {
clearInterval(this.netPolResyncTimer)
this.netPolResyncTimer = null
}
if (this.netPolDefaultsResyncTimer) {
clearInterval(this.netPolDefaultsResyncTimer)
this.netPolDefaultsResyncTimer = null
}
this.netPolDefaultsOnlyRun = null
this.netPolConvergenceEnsureDefaults = false
for (const timer of this.initialConvergenceRetryTimers.values()) {
clearTimeout(timer)
}
this.initialConvergenceRetryTimers.clear()
this.initialConvergenceRetryAttempts.clear()
if (this.mcpWatchRequest) {
console.log('[K8s] Stopping McpServer watch')
this.mcpWatchRequest.abort()
this.mcpWatchRequest = null
}
if (this.ctxWatchRequest) {
console.log('[K8s] Stopping Context watch')
this.ctxWatchRequest.abort()
this.ctxWatchRequest = null
}
if (this.hostWatchRequest) {
console.log('[K8s] Stopping Host watch')
this.hostWatchRequest.abort()
this.hostWatchRequest = null
}
if (this.sfsWatchRequest) {
console.log('[K8s] Stopping SharedFileSystem watch')
this.sfsWatchRequest.abort()
this.sfsWatchRequest = null
}
if (this.gfsWatchRequest) {
console.log('[K8s] Stopping GlobalFileSystem watch')
this.gfsWatchRequest.abort()
this.gfsWatchRequest = null
}
if (this.ccWatchRequest) {
console.log('[K8s] Stopping CommunicationChannel watch')
this.ccWatchRequest.abort()
this.ccWatchRequest = null
}
if (this.llmHookWatchRequest) {
console.log('[K8s] Stopping LlmHook watch')
this.llmHookWatchRequest.abort()
this.llmHookWatchRequest = null
}
await Promise.allSettled([
this.infrastructureTelemetryReporter?.stop(),
this.administrativeOutcomeReporter?.stop(),
])
}
}
/**
* Dev mode McpServer provider - uses servers from environment variable.
* No reconciliation in dev mode (no K8s Deployments, Services, or NetworkPolicies).
*
* Context-based filtering mirrors production behaviour:
* - If CLERUM_CONTEXTS is set, getServerInfosByContext reads those contexts
* and returns only the servers listed in the matching context's mcpServers list.
* - If CLERUM_CONTEXTS is NOT set, it falls back to matching by contextRef on the server.
*/
export class DevMcpServerProvider implements McpServerProvider {
private servers: Map<string, McpServerCRD> = new Map()
private contexts: Map<string, ContextCRD> = new Map()
private changeCallback?: () => void
getAllServers(): McpServerCRD[] {
return [...this.servers.values()]
}
private toServerInfo(server: McpServerCRD): McpServerInfo {
return {
name: server.name,
description: server.spec.description,
contextRef: server.spec.contextRef,
transport: server.spec.transport,
auth: server.spec.auth,
enabled: server.spec.enabled !== false,
status: { deployed: true, ready: true, message: 'Dev mode' },
}
}
getAllServerInfos(): McpServerInfo[] {
return this.getAllServers().map(s => this.toServerInfo(s))
}
/**
* Get servers for a context.
* If CLERUM_CONTEXTS was provided, uses the context's mcpServers allow-list
* (mirrors production behaviour). Otherwise falls back to contextRef matching.
*/
async getServerInfosByContext(contextRef: string): Promise<McpServerInfo[]> {
const context = this.contexts.get(contextRef)
if (context) {
// Production-like behaviour: filter by the Context's mcpServers list
const allowedNames = new Set(context.spec.mcpServers)
console.log(
`[Dev] Context "${contextRef}" allows servers: [${context.spec.mcpServers.join(', ')}]`
)
return this.getAllServers()
.filter(s => allowedNames.has(s.name) && s.spec.enabled !== false)
.map(s => this.toServerInfo(s))
}
if (this.contexts.size > 0) {
// Contexts are loaded but this one doesn't exist — return empty
console.warn(`[Dev] Context "${contextRef}" not found — returning no servers`)
return []
}
// Fallback: no contexts loaded, filter by contextRef on the server
console.log(`[Dev] No CLERUM_CONTEXTS set — falling back to contextRef matching`)
return this.getAllServers()
.filter(s => s.spec.contextRef === contextRef && s.spec.enabled !== false)
.map(s => this.toServerInfo(s))
}
onChange(callback: () => void): void {
this.changeCallback = callback
}
async start(): Promise<void> {
// Load MCP servers
console.log(`[Dev] Loading MCP servers from CLERUM_MCP_SERVERS`)
for (const server of config.devMcpServers) {
const serverWithNamespace: McpServerCRD = {
...server,
namespace: server.namespace || 'dev',
}
this.servers.set(server.name, serverWithNamespace)
console.log(`[Dev] Loaded server: ${server.name} (context: ${server.spec.contextRef})`)
}
console.log(`[Dev] Loaded ${this.servers.size} MCP server(s)`)
// Load contexts
if (config.devContexts.length > 0) {
console.log(`[Dev] Loading Contexts from CLERUM_CONTEXTS`)
for (const ctx of config.devContexts) {
this.contexts.set(ctx.spec.contextId, ctx)
console.log(
`[Dev] Loaded context: ${ctx.name} → mcpServers=[${ctx.spec.mcpServers.join(', ')}]`
)
}
console.log(`[Dev] Loaded ${this.contexts.size} Context(s)`)
} else {
console.log(
`[Dev] No CLERUM_CONTEXTS set — context filtering will fall back to contextRef matching`
)
}
}
async stop(): Promise<void> {
console.log('[Dev] Stopping dev provider')
}
/** Add a server dynamically (useful for testing). */
addServer(server: McpServerCRD): void {
this.servers.set(server.name, server)
this.changeCallback?.()
}
/** Remove a server dynamically (useful for testing). */
removeServer(name: string): void {
this.servers.delete(name)
this.changeCallback?.()
}
/** Add a context dynamically (useful for testing). */
addContext(context: ContextCRD): void {
this.contexts.set(context.spec.contextId, context)
this.changeCallback?.()
}
/** Remove a context dynamically (useful for testing). */
removeContext(contextId: string): void {
this.contexts.delete(contextId)
this.changeCallback?.()
}
}
/**
* Create the appropriate provider based on mode.
*/
export function createMcpServerProvider(): McpServerProvider {
if (config.devMode) {
console.log('[Provider] Creating dev mode provider')
return new DevMcpServerProvider()
} else {
console.log('[Provider] Creating K8s watcher provider (with reconciler)')
return new McpServerWatcher()
}
}
function authorizationMetadata(metadata: {
uid?: string
resourceVersion?: string
deletionTimestamp?: string | Date
}): { uid: string; resourceVersion: string; deletionTimestamp?: string } {
return {
uid: metadata.uid ?? '',
resourceVersion: metadata.resourceVersion ?? '',
...(metadata.deletionTimestamp
? { deletionTimestamp: new Date(metadata.deletionTimestamp).toISOString() }
: {}),
}
}
export function isMcpAuthorizationNotFound(error: unknown): boolean {
return getErrorCode(error) === 404
}
/**
* Live Kubernetes authority used only by the protected v2 Host MCP routes.
* Reads are deliberately not served from watch caches: every successful
* credential operation is fenced against current object UIDs/resourceVersions.
*/
export function createMcpAuthorizationStore(provider: McpServerProvider): McpAuthorizationStore {
if (config.devMode || !customObjectsApi || !hostCustomObjectsApi || !coreApi) {
return {
async readHost() {
return null
},
async readContext() {
return null
},
async readMcpServer() {
return null
},
async readSecretMetadata() {
return null
},
async readSecret() {
return null
},
}
}
return {
async readHost(name: string): Promise<AuthorityHost | null> {
try {
const object = (await hostCustomObjectsApi!.getNamespacedCustomObject({
group: GROUP,
version: VERSION,
namespace: config.hostNamespace,
plural: PLURAL_HOSTS,
name,
})) as {
metadata: {
name: string
namespace?: string
uid?: string
resourceVersion?: string
deletionTimestamp?: string
}
spec: HostSpec
}
return {
name: object.metadata.name,
namespace: object.metadata.namespace ?? config.hostNamespace,
metadata: authorizationMetadata(object.metadata),
contextRef: object.spec.contextRef,
}
} catch (error) {
if (isMcpAuthorizationNotFound(error)) return null
throw error
}
},
async readContext(name: string): Promise<AuthorityContext | null> {
try {
const object = (await customObjectsApi!.getNamespacedCustomObject({
group: GROUP,
version: VERSION,
namespace: config.namespace,
plural: PLURAL_CONTEXTS,
name,
})) as {
metadata: {
name: string
namespace?: string
uid?: string
resourceVersion?: string
deletionTimestamp?: string
}
spec: ContextSpec
}
return {
name: object.metadata.name,
namespace: object.metadata.namespace ?? config.namespace,
metadata: authorizationMetadata(object.metadata),
mcpServers: [...object.spec.mcpServers],
}
} catch (error) {
if (isMcpAuthorizationNotFound(error)) return null
throw error
}
},
async readMcpServer(name: string): Promise<AuthorityMcpServer | null> {
try {
const object = (await customObjectsApi!.getNamespacedCustomObject({
group: GROUP,
version: VERSION,
namespace: config.namespace,
plural: PLURAL_MCPSERVERS,
name,
})) as McpServerWatchObject & {
metadata: McpServerWatchObject['metadata'] & {
resourceVersion?: string
deletionTimestamp?: string
}
}
const status = provider.getAllServerInfos().find(server => server.name === name)
?.status ?? {
deployed: false,
ready: false,
}
return {
name: object.metadata.name,
namespace: object.metadata.namespace ?? config.namespace,
metadata: authorizationMetadata(object.metadata),
description: object.spec.description,
transport: { ...object.spec.transport },
auth: object.spec.auth ? { ...object.spec.auth } : undefined,
// grantScope drives the inventory authKind derivation (mini-spec 10 §3.1).
oauth: object.spec.oauth ? { ...object.spec.oauth } : undefined,
// Single source of truth for remote-ness, mirrors the reconciler's isRemote.
remote: !!object.spec.remote?.baseUrl,
// Transport quirk derived alongside remote (mini-spec 19 §D-8): the
// resource advertised bearer_methods_supported:["body"]. Non-secret;
// projected omit-when-false to the inventory in listServers.
bearerInBody: !!object.spec.oauth?.bearerInBody,
enabled: object.spec.enabled !== false,
status,
}
} catch (error) {
if (isMcpAuthorizationNotFound(error)) return null
throw error
}
},
async readSecretMetadata(name: string): Promise<AuthoritySecretMetadata | null> {
try {
// CoreV1Api returns a Secret object, but this boundary intentionally
// drops `data` immediately. Inventory authorization can therefore use
// UID/resourceVersion without making credential bytes available to the
// service layer or its DTO/logging path.
const object = await coreApi!.readNamespacedSecret({ name, namespace: config.namespace })
return {
name: object.metadata?.name ?? name,
namespace: object.metadata?.namespace ?? config.namespace,
metadata: authorizationMetadata({
uid: object.metadata?.uid,
resourceVersion: object.metadata?.resourceVersion,
deletionTimestamp: object.metadata?.deletionTimestamp,
}),
}
} catch (error) {
if (isMcpAuthorizationNotFound(error)) return null
throw error
}
},
async readSecret(name: string): Promise<AuthoritySecret | null> {
try {
const object = await coreApi!.readNamespacedSecret({ name, namespace: config.namespace })
return {
name: object.metadata?.name ?? name,
namespace: object.metadata?.namespace ?? config.namespace,
metadata: authorizationMetadata({
uid: object.metadata?.uid,
resourceVersion: object.metadata?.resourceVersion,
deletionTimestamp: object.metadata?.deletionTimestamp,
}),
data: { ...(object.data ?? {}) },
}
} catch (error) {
if (isMcpAuthorizationNotFound(error)) return null
throw error
}
},
}
}