Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript

chore: sync np-08 mcp-token branch with origin/dev @ 1b845a3d

Alfredo Lopez5ed13f26928d
diff --cc host-context-controller/src/k8sClient.test.ts
index 472b15d00,fb1ac6231..362c5954b
--- a/host-context-controller/src/k8sClient.test.ts
+++ b/host-context-controller/src/k8sClient.test.ts
@@@ -3,14 -5,15 +5,17 @@@ import { HostFleetReconcileError } fro
import { HostK8sRequestTimeoutError } from './k8s/hostK8sApiClient'
import {
McpServerWatcher,
+ createMcpAuthorizationStore,
getContext,
+ isMcpAuthorizationNotFound,
- externalEgressResyncDelayMs,
listAllCommunicationChannels,
+ listAllGlobalFileSystems,
listAllHosts,
+ listAllSharedFileSystems,
} from './k8sClient'
- import type { HostCRD } from './types'
+ import { registry } from './metrics'
+ import { ContextMapperServer } from './server'
+ import type { HostCRD, McpServerCRD } from './types'
function deferred<T = void>(): {
promise: Promise<T>
@@@ -31,20 -117,24 +119,26 @@@ const mocks = vi.hoisted(() =>
const getNamespacedCustomObject = vi
.fn()
.mockRejectedValue(Object.assign(new Error('not found'), { code: 404 }))
+ const readNamespacedSecret = vi.fn()
const ensureDefaultPolicies = vi.fn().mockResolvedValue(undefined)
- const netPolFullReconcile = vi.fn().mockResolvedValue(undefined)
+ const hasCertifiedSafetyInventory = vi.fn().mockReturnValue(true)
+ const netPolFullReconcile = vi.fn().mockImplementation(async (...args: unknown[]) => {
+ const options = args[2] as { onAuthoritativeRevocationComplete?: () => void } | undefined
+ options?.onAuthoritativeRevocationComplete?.()
+ })
const serverFullReconcile = vi.fn().mockResolvedValue(undefined)
const hostFullReconcile = vi.fn().mockResolvedValue(undefined)
const hostReconcileHosts = vi.fn().mockResolvedValue(undefined)
const sfsFullReconcile = vi.fn().mockResolvedValue(undefined)
const watch = vi.fn().mockResolvedValue({ abort: vi.fn() })
const hostListCallOptions = vi.fn()
+ const sharedFileSystemListCallOptions = vi.fn()
const createAdministrativeOutcomeReporter = vi.fn().mockReturnValue(undefined)
return {
+ hasCertifiedSafetyInventory,
listNamespacedCustomObject,
getNamespacedCustomObject,
+ readNamespacedSecret,
ensureDefaultPolicies,
netPolFullReconcile,
serverFullReconcile,
@@@ -97,12 -190,14 +194,15 @@@ vi.mock('@kubernetes/client-node', () =
makeApiClient(api: unknown): unknown {
if (api === CustomObjectsApi) {
return {
+ getNamespacedCustomObject: mocks.getNamespacedCustomObject,
listNamespacedCustomObject: async (request: { plural?: string }, options?: unknown) => {
if (request.plural === 'hosts') mocks.hostListCallOptions(options)
+ if (request.plural === 'sharedfilesystems')
+ mocks.sharedFileSystemListCallOptions(options)
const response = await mocks.listNamespacedCustomObject(request)
if (
- request.plural === 'hosts' &&
+ request.plural &&
+ ['hosts', 'mcpservers', 'contexts'].includes(request.plural) &&
response &&
typeof response === 'object' &&
!('metadata' in response)
@@@ -1032,50 -2139,3801 +2209,3815 @@@ describe('McpServerWatcher startup', (
watcher.stop()
})
- it('starts CC watch even when initial CC list fails (graceful degradation)', async () => {
- mocks.listNamespacedCustomObject.mockImplementation(async ({ plural }: { plural: string }) => {
- if (plural === 'communicationchannels') throw new Error('CC list failed')
- if (plural === 'mcpservers') return { items: [] }
- if (plural === 'contexts') return { items: [] }
- if (plural === 'hosts') return { items: [] }
- if (plural === 'sharedfilesystems') return { items: [] }
- return { items: [] }
- })
-
+ it('updates discovery without re-reconciling a status-only McpServer watch event', async () => {
+ const previousServer = {
+ name: 'status-server',
+ namespace: 'mcp-server',
+ uid: 'status-server-uid',
+ generation: 7,
+ annotations: { 'clerum.io/recipe': 'status-recipe' },
+ labels: { 'clerum.io/workload': 'status-server' },
+ spec: {
+ contextRef: 'default',
+ image: 'clerum/status-server:test',
+ transport: { type: 'streamableHttp' as const, port: 3000 },
+ },
+ }
const watcher = new McpServerWatcher()
+ ;(watcher as any).servers.set(previousServer.name, previousServer)
+ const changed = vi.fn()
+ watcher.onChange(changed)
+ const reconciler = (watcher as any).reconciler
+ const netPol = (watcher as any).netPolReconciler
+ const callback = (watcher as any).getMcpServerWatchCallback()
+ expect((watcher as any).mcpServerDesiredRevision).toBe(0)
- vi.spyOn(watcher as any, 'startMcpServerWatch').mockResolvedValue(undefined)
- vi.spyOn(watcher as any, 'startContextWatch').mockResolvedValue(undefined)
- vi.spyOn(watcher as any, 'startSharedFileSystemWatch').mockResolvedValue(undefined)
- vi.spyOn(watcher as any, 'startGlobalFileSystemWatch').mockResolvedValue(undefined)
- const ccWatchSpy = vi
- .spyOn(watcher as any, 'startCommunicationChannelWatch')
- .mockResolvedValue(undefined)
- const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {})
+ await callback('MODIFIED', {
+ metadata: {
+ name: previousServer.name,
+ namespace: previousServer.namespace,
+ uid: previousServer.uid,
+ generation: previousServer.generation,
+ annotations: previousServer.annotations,
+ labels: previousServer.labels,
+ },
+ spec: previousServer.spec,
+ status: { conditions: [{ type: 'Ready', status: 'True' }] },
+ })
- await expect(watcher.start()).resolves.toBeUndefined()
- expect(ccWatchSpy).toHaveBeenCalledTimes(1)
- expect(watcher.isCommunicationChannelCacheSynced()).toBe(false)
- expect(errorSpy).toHaveBeenCalledWith(
- '[K8s] CommunicationChannel initial load failed; ccCacheSynced remains false ' +
- '(B2 preserves channel-reader replicas and holds stateless lifecycle active):',
- expect.any(Error)
- )
+ expect(watcher.getAllServers()[0]?.status).toEqual({
+ conditions: [{ type: 'Ready', status: 'True' }],
+ })
+ expect(changed).toHaveBeenCalledOnce()
+ expect(reconciler.reconcile).not.toHaveBeenCalled()
+ expect(netPol.reconcileExternalEgress).not.toHaveBeenCalled()
+ expect((watcher as any).mcpServerDesiredRevision).toBe(0)
- errorSpy.mockRestore()
- watcher.stop()
+ await callback('MODIFIED', {
+ metadata: {
+ name: previousServer.name,
+ namespace: previousServer.namespace,
+ uid: previousServer.uid,
+ generation: 8,
+ annotations: previousServer.annotations,
+ labels: previousServer.labels,
+ },
+ spec: { ...previousServer.spec, image: 'clerum/status-server:next' },
+ })
+ expect(reconciler.reconcile).toHaveBeenCalledOnce()
+ expect((watcher as any).mcpServerDesiredRevision).toBe(1)
+ await watcher.stop()
})
- })
- describe('McpServerWatcher CommunicationChannel cache recovery', () => {
- async function flushMicrotasks(): Promise<void> {
- for (let i = 0; i < 8; i += 1) await Promise.resolve()
- }
+ it('does not advance desired revision for the controller-owned network-ready handshake', async () => {
+ const previousServer = {
+ name: 'network-ready-server',
+ namespace: 'mcp-server',
+ uid: 'network-ready-server-uid',
+ generation: 7,
+ annotations: { 'clerum.io/pre-deploy': 'true' },
+ labels: { 'clerum.io/workload': 'network-ready-server' },
+ spec: {
+ contextRef: 'default',
+ image: 'clerum/network-ready-server:test',
+ transport: { type: 'streamableHttp' as const, port: 3000 },
+ },
+ }
+ const watcher = new McpServerWatcher()
+ ;(watcher as any).servers.set(previousServer.name, previousServer)
+ const callback = (watcher as any).getMcpServerWatchCallback()
+
+ await callback('MODIFIED', {
+ metadata: {
+ name: previousServer.name,
+ namespace: previousServer.namespace,
+ uid: previousServer.uid,
+ generation: previousServer.generation,
+ annotations: {
+ ...previousServer.annotations,
+ 'clerum.io/network-ready': 'true',
+ },
+ labels: previousServer.labels,
+ },
+ spec: previousServer.spec,
+ })
+
+ expect((watcher as any).mcpServerDesiredRevision).toBe(0)
+ expect((watcher as any).reconciler.reconcile).not.toHaveBeenCalled()
+ await watcher.stop()
+ })
+
+ it('advances McpServer desired revision for owner addition and deletion', async () => {
+ const serverObject = {
+ metadata: {
+ name: 'revision-server',
+ namespace: 'mcp-server',
+ uid: 'revision-server-uid',
+ generation: 1,
+ },
+ spec: {
+ contextRef: 'default',
+ image: 'clerum/revision-server:test',
+ transport: { type: 'streamableHttp' as const, port: 3000 },
+ },
+ }
+ const watcher = new McpServerWatcher()
+ const callback = (watcher as any).getMcpServerWatchCallback()
+
+ expect((watcher as any).mcpServerDesiredRevision).toBe(0)
+ await callback('ADDED', serverObject)
+ expect((watcher as any).mcpServerDesiredRevision).toBe(1)
+ await callback('DELETED', serverObject)
+ expect((watcher as any).mcpServerDesiredRevision).toBe(2)
+
+ await watcher.stop()
+ })
+
+ it('advances Context desired revision only when the authoritative desired cache changes', async () => {
+ const watcher = new McpServerWatcher()
+ let contextWatchCallback:
+ | ((
+ type: string,
+ apiObj: {
+ metadata: { name: string; namespace: string }
+ spec: { contextId: string; mcpServers: string[] }
+ }
+ ) => Promise<void>)
+ | undefined
+ mocks.watch.mockImplementationOnce(async (_path, _options, callback) => {
+ contextWatchCallback = callback
+ return { abort: vi.fn() }
+ })
+ await (watcher as any).startContextWatch('context-desired-revision-rv')
+ ;(watcher as any).contextCacheSynced = true
+ ;(watcher as any).mcpServerCacheSynced = true
+
+ const added = {
+ metadata: { name: 'revision-context', namespace: 'mcp-server' },
+ spec: { contextId: 'revision-context', mcpServers: [] as string[] },
+ }
+ expect((watcher as any).contextDesiredRevision).toBe(0)
+ await contextWatchCallback!('ADDED', added)
+ expect((watcher as any).contextDesiredRevision).toBe(1)
+
+ await contextWatchCallback!('MODIFIED', added)
+ expect((watcher as any).contextDesiredRevision).toBe(1)
+
+ const modified = {
+ ...added,
+ spec: { ...added.spec, mcpServers: ['revision-server'] },
+ }
+ await contextWatchCallback!('MODIFIED', modified)
+ expect((watcher as any).contextDesiredRevision).toBe(2)
+
+ await contextWatchCallback!('DELETED', modified)
+ expect((watcher as any).contextDesiredRevision).toBe(3)
+
+ await watcher.stop()
+ })
+
+ it('uses only the bounded fleet pass for the cold-start Host snapshot', async () => {
+ const initialHosts = Array.from({ length: 50 }, (_, index) => ({
+ metadata: {
+ name: `initial-host-${index}`,
+ namespace: 'mcp-host',
+ uid: `initial-host-uid-${index}`,
+ generation: 1,
+ },
+ spec: {
+ host: `initial-host-${index}`,
+ contextRef: 'default',
+ secretRef: 'host-secret',
+ },
+ }))
+ mocks.listNamespacedCustomObject.mockImplementation(async ({ plural }: { plural: string }) => {
+ if (plural === 'hosts') {
+ return {
+ metadata: { resourceVersion: 'initial-host-rv' },
+ items: initialHosts,
+ }
+ }
+ if (plural === 'communicationchannels') {
+ return { metadata: { resourceVersion: 'initial-cc-rv' }, items: [] }
+ }
+ return { items: [] }
+ })
+ const watcher = new McpServerWatcher()
+ const urgentReconcile = vi.spyOn(watcher.getHostReconciler(), 'reconcile')
+
+ await watcher.start()
+ await vi.waitFor(() =>
+ expect(mocks.hostFullReconcile).toHaveBeenCalledWith(
+ initialHosts.map(host =>
+ expect.objectContaining({
+ name: host.metadata.name,
+ generation: host.metadata.generation,
+ })
+ )
+ )
+ )
+ await flushMicrotasks()
+
+ expect(mocks.hostFullReconcile).toHaveBeenCalledOnce()
+ expect(urgentReconcile).not.toHaveBeenCalled()
+ await watcher.stop()
+ })
+
+ it('admits a McpServer watch reconciliation while initial convergence is blocked on another server', async () => {
+ const staleServer = {
+ name: 'stale-server',
+ namespace: 'mcp-server',
+ spec: {
+ contextRef: 'default',
+ image: 'clerum/stale-server:test',
+ transport: { type: 'streamableHttp' as const, port: 3000 },
+ egressBindings: [{ dns: 'stale.example', port: 443 }],
+ },
+ }
+ const currentServer = {
+ metadata: { name: 'current-server', namespace: 'mcp-server' },
+ spec: {
+ contextRef: 'default',
+ image: 'clerum/current-server:test',
+ transport: { type: 'streamableHttp' as const, port: 3000 },
+ },
+ }
+ const egressStarted = deferred()
+ const releaseEgress = deferred()
+ const reconciliationOrder: string[] = []
+ const watcher = new McpServerWatcher()
+ ;(watcher as any).mcpServerCacheSynced = true
+ ;(watcher as any).contextCacheSynced = true
+ markNetworkPolicyRevocationAuthoritative(watcher)
+ ;(watcher as any).servers.set(staleServer.name, staleServer)
+ const netPol = (watcher as any).netPolReconciler
+ netPol.reconcileExternalEgress.mockImplementationOnce(async () => {
+ egressStarted.resolve(undefined)
+ await releaseEgress.promise
+ })
+ mocks.serverFullReconcile.mockImplementationOnce(async (servers, options) => {
+ await Promise.all(
+ servers.map((selected: { name: string }) =>
+ options.runEffect(selected.name, async () => {
+ reconciliationOrder.push('initial')
+ })
+ )
+ )
+ })
+ const reconciler = (watcher as any).reconciler
+ reconciler.reconcile.mockImplementationOnce(async () => {
+ reconciliationOrder.push('watch')
+ })
+
+ const convergence = (watcher as any).runInitialMcpServerConvergence()
+ await egressStarted.promise
+ const watchConvergence = (watcher as any).getMcpServerWatchCallback()('MODIFIED', currentServer)
+
+ await flushMicrotasks()
+ expect(reconciler.reconcile).toHaveBeenCalledWith(
+ expect.objectContaining({ name: 'current-server' }),
+ { isCurrent: expect.any(Function) }
+ )
+
+ releaseEgress.resolve(undefined)
+ await Promise.all([convergence, watchConvergence])
+
+ expect(reconciliationOrder).toEqual(['watch'])
+ expect(reconciler.reconcile).toHaveBeenLastCalledWith(
+ expect.objectContaining({ name: 'current-server' }),
+ { isCurrent: expect.any(Function) }
+ )
+ await watcher.stop()
+ })
+
+ it('continues a same-server reconciliation lane after a rejected operation', async () => {
+ const watcher = new McpServerWatcher()
+ const server = { name: 'retry-safe-server', namespace: 'mcp-server' }
+ const order: string[] = []
+ const firstStarted = deferred()
+ const rejectFirst = deferred()
+
+ const rejected = (watcher as any).enqueueMcpServerReconciliation(server, async () => {
+ order.push('rejected')
+ firstStarted.resolve(undefined)
+ await rejectFirst.promise
+ throw new Error('transient lane failure')
+ }) as Promise<void>
+ await firstStarted.promise
+ const recovered = (watcher as any).enqueueMcpServerReconciliation(server, async () => {
+ order.push('recovered')
+ }) as Promise<void>
+ rejectFirst.resolve(undefined)
+
+ await expect(rejected).rejects.toThrow('transient lane failure')
+ await expect(recovered).resolves.toBeUndefined()
+ expect(order).toEqual(['rejected', 'recovered'])
+ expect((watcher as any).mcpServerReconciliationQueues.size).toBe(0)
+ await watcher.stop()
+ })
+
+ it('revalidates env-Secret matches inside each server lane before reconciling', async () => {
+ const currentServer = {
+ name: 'current-secret-server',
+ namespace: 'mcp-server',
+ spec: {
+ contextRef: 'default',
+ image: 'clerum/current-secret-server:test',
+ transport: { type: 'streamableHttp' as const, port: 3000 },
+ envSecret: { name: 'shared-env' },
+ },
+ }
+ const staleServer = {
+ name: 'stale-secret-server',
+ namespace: 'mcp-server',
+ spec: {
+ contextRef: 'default',
+ image: 'clerum/stale-secret-server:test',
+ transport: { type: 'streamableHttp' as const, port: 3000 },
+ envSecret: { name: 'shared-env' },
+ },
+ }
+ const unrelatedServer = {
+ name: 'unrelated-secret-server',
+ namespace: 'mcp-server',
+ spec: {
+ contextRef: 'default',
+ image: 'clerum/unrelated-secret-server:test',
+ transport: { type: 'streamableHttp' as const, port: 3000 },
+ envSecret: { name: 'other-env' },
+ },
+ }
+ const blockerStarted = deferred()
+ const releaseBlocker = deferred()
+ const watcher = new McpServerWatcher()
+ markMcpServerInventoryAuthoritative(watcher)
+ const reconciler = (watcher as any).reconciler
+ for (const server of [currentServer, staleServer, unrelatedServer]) {
+ ;(watcher as any).servers.set(server.name, server)
+ }
+
+ const blocker = (watcher as any).enqueueMcpServerReconciliation(staleServer, async () => {
+ blockerStarted.resolve(undefined)
+ await releaseBlocker.promise
+ })
+ await blockerStarted.promise
+
+ const convergence = watcher.reconcileByEnvSecret('shared-env', 'mcp-server')
+ ;(watcher as any).servers.set(staleServer.name, {
+ ...staleServer,
+ spec: { ...staleServer.spec, envSecret: { name: 'replacement-env' } },
+ })
+ await vi.waitFor(() =>
+ expect(reconciler.reconcile).toHaveBeenCalledWith(
+ expect.objectContaining({ name: currentServer.name }),
+ { isCurrent: expect.any(Function) }
+ )
+ )
+ releaseBlocker.resolve(undefined)
+ await Promise.all([blocker, convergence])
+
+ expect(reconciler.reconcile).toHaveBeenCalledTimes(1)
+ expect(
+ reconciler.reconcile.mock.calls.some(
+ ([server]: [McpServerCRD]) => server.name === staleServer.name
+ )
+ ).toBe(false)
+ expect(
+ reconciler.reconcile.mock.calls.some(
+ ([server]: [McpServerCRD]) => server.name === unrelatedServer.name
+ )
+ ).toBe(false)
+ await watcher.stop()
+ })
+
+ it('defers env-Secret reconciliation until the McpServer inventory is authoritative', async () => {
+ const selected = {
+ name: 'authority-fenced-secret-server',
+ namespace: 'mcp-server',
+ spec: {
+ contextRef: 'default',
+ image: 'clerum/authority-fenced-secret-server:test',
+ transport: { type: 'streamableHttp' as const, port: 3000 },
+ envSecret: { name: 'shared-env' },
+ },
+ }
+ const watcher = new McpServerWatcher()
+ const reconciler = (watcher as any).reconciler
+ ;(watcher as any).servers.set(selected.name, selected)
+ ;(watcher as any).mcpServerCacheSynced = false
+
+ await watcher.reconcileByEnvSecret('shared-env', 'mcp-server')
+
+ expect(reconciler.reconcile).not.toHaveBeenCalled()
+
+ markMcpServerInventoryAuthoritative(watcher)
+ await watcher.reconcileByEnvSecret('shared-env', 'mcp-server')
+
+ expect(reconciler.reconcile).toHaveBeenCalledOnce()
+ expect(reconciler.reconcile).toHaveBeenCalledWith(
+ expect.objectContaining({ name: selected.name }),
+ { isCurrent: expect.any(Function) }
+ )
+ await watcher.stop()
+ })
+
+ it('fences Context-triggered SharedFileSystem convergence to the current Context lease', async () => {
+ const context = {
+ name: 'sfs-context',
+ namespace: 'mcp-server',
+ spec: {
+ contextId: 'sfs-context',
+ sharedFileSystems: [{ name: 'shared-data', mountPath: '/shared' }],
+ },
+ }
+ const sharedFileSystem = {
+ name: 'shared-data',
+ namespace: 'mcp-host',
+ spec: {},
+ }
+ const watcher = new McpServerWatcher()
+ const reconciler = (watcher as any).sharedFileSystemReconciler
+ ;(watcher as any).sharedFileSystems.set(sharedFileSystem.name, sharedFileSystem)
+
+ await (watcher as any).reconcileSharedFileSystemsReferencedByContext(
+ undefined,
+ context,
+ () => false
+ )
+
+ expect(reconciler.reconcile).not.toHaveBeenCalled()
+
+ await (watcher as any).reconcileSharedFileSystemsReferencedByContext(
+ undefined,
+ context,
+ () => true
+ )
+
+ expect(reconciler.reconcile).toHaveBeenCalledOnce()
+ expect(reconciler.reconcile).toHaveBeenCalledWith(
+ expect.objectContaining({ name: sharedFileSystem.name })
+ )
+ await watcher.stop()
+ })
+
+ it('fences Context-triggered Host convergence to the current Context lease', async () => {
+ const watcher = new McpServerWatcher()
+ markHostInventoryAuthoritative(watcher)
+ ;(watcher as any).hosts.set('context-host', {
+ name: 'context-host',
+ namespace: 'mcp-host',
+ spec: {
+ host: 'context-host',
+ contextRef: 'context-a',
+ secretRef: 'host-secret',
+ },
+ })
+ const hostReconciler = (watcher as any).hostReconciler
+
+ await (watcher as any).reconcileHostsReferencingContext('context-a', () => false)
+
+ expect(hostReconciler.reconcile).not.toHaveBeenCalled()
+
+ await (watcher as any).reconcileHostsReferencingContext('context-a', () => true)
+
+ expect(hostReconciler.reconcile).toHaveBeenCalledOnce()
+ expect(hostReconciler.reconcile).toHaveBeenCalledWith(
+ expect.objectContaining({ name: 'context-host' })
+ )
+ await watcher.stop()
+ })
+
+ it('keeps env-Secret reconciliation a no-op after stop', async () => {
+ const selected = {
+ name: 'stopped-secret-server',
+ namespace: 'mcp-server',
+ spec: {
+ contextRef: 'default',
+ image: 'clerum/stopped-secret-server:test',
+ transport: { type: 'streamableHttp' as const, port: 3000 },
+ envSecret: { name: 'shared-env' },
+ },
+ }
+ const watcher = new McpServerWatcher()
+ const reconciler = (watcher as any).reconciler
+ ;(watcher as any).servers.set(selected.name, selected)
+
+ await watcher.stop()
+ await watcher.reconcileByEnvSecret('shared-env', 'mcp-server')
+
+ expect(reconciler.reconcile).not.toHaveBeenCalled()
+ })
+
+ it('admits an initial server without egress while another initial server waits on DNS', async () => {
+ const dnsBlockedServer = {
+ name: 'dns-blocked-server',
+ namespace: 'mcp-server',
+ uid: 'dns-blocked-server-uid',
+ generation: 1,
+ spec: {
+ contextRef: 'default',
+ image: 'clerum/dns-blocked-server:test',
+ transport: { type: 'streamableHttp' as const, port: 3000 },
+ egressBindings: [{ dns: 'blocked.example', port: 443 }],
+ },
+ }
+ const independentServer = {
+ name: 'independent-server',
+ namespace: 'mcp-server',
+ uid: 'independent-server-uid',
+ generation: 1,
+ spec: {
+ contextRef: 'default',
+ image: 'clerum/independent-server:test',
+ transport: { type: 'streamableHttp' as const, port: 3000 },
+ },
+ }
+ const egressStarted = deferred()
+ const releaseEgress = deferred()
+ const watcher = new McpServerWatcher()
+ ;(watcher as any).mcpServerCacheSynced = true
+ ;(watcher as any).contextCacheSynced = true
+ markNetworkPolicyRevocationAuthoritative(watcher)
+ ;(watcher as any).servers.set(dnsBlockedServer.name, dnsBlockedServer)
+ ;(watcher as any).servers.set(independentServer.name, independentServer)
+ const netPol = (watcher as any).netPolReconciler
+ const reconciler = (watcher as any).reconciler
+ netPol.reconcileExternalEgress.mockImplementationOnce(async () => {
+ egressStarted.resolve(undefined)
+ await releaseEgress.promise
+ })
+ mocks.serverFullReconcile.mockImplementationOnce(
+ async (
+ servers: Array<typeof dnsBlockedServer | typeof independentServer>,
+ options: { runEffect: (serverName: string, work: () => Promise<void>) => Promise<void> }
+ ) => {
+ await Promise.all(
+ servers.map(server => options.runEffect(server.name, () => reconciler.reconcile(server)))
+ )
+ }
+ )
+
+ const convergence = (watcher as any).runInitialMcpServerConvergence()
+ await egressStarted.promise
+ await flushMicrotasks()
+
+ expect(reconciler.reconcile).toHaveBeenCalledWith(independentServer)
+ expect(reconciler.reconcile).not.toHaveBeenCalledWith(dnsBlockedServer)
+
+ releaseEgress.resolve(undefined)
+ await convergence
+
+ expect(reconciler.reconcile).toHaveBeenCalledWith(dnsBlockedServer)
+ await watcher.stop()
+ })
+
+ it('admits a Context watch reconciliation while initial policy convergence is blocked', async () => {
+ const staleContext = {
+ name: 'stale-context',
+ namespace: 'mcp-server',
+ spec: { contextId: 'stale-context', mcpServers: [] },
+ }
+ const currentContext = {
+ metadata: { name: 'current-context', namespace: 'mcp-server' },
+ spec: { contextId: 'current-context', mcpServers: [] },
+ }
+ const fullReconcileStarted = deferred()
+ const releaseFullReconcile = deferred()
+ const reconciliationOrder: string[] = []
+ const watcher = new McpServerWatcher()
+ ;(watcher as any).mcpServerCacheSynced = true
+ ;(watcher as any).contextCacheSynced = true
+ ;(watcher as any).contexts.set(staleContext.name, staleContext)
+ mocks.netPolFullReconcile.mockImplementationOnce(async () => {
+ fullReconcileStarted.resolve(undefined)
+ await releaseFullReconcile.promise
+ reconciliationOrder.push('initial')
+ })
+ const netPol = (watcher as any).netPolReconciler
+ netPol.reconcileContext.mockImplementationOnce(async () => {
+ reconciliationOrder.push('watch')
+ })
+ let contextWatchCallback:
+ | ((type: string, apiObj: typeof currentContext) => Promise<void>)
+ | undefined
+ mocks.watch.mockImplementationOnce(async (_path, _options, callback) => {
+ contextWatchCallback = callback
+ return { abort: vi.fn() }
+ })
+ await (watcher as any).startContextWatch('context-queue-rv')
+
+ const convergence = (watcher as any).runInitialNetworkPolicyConvergence(true)
+ await fullReconcileStarted.promise
+ const watchConvergence = contextWatchCallback!('MODIFIED', currentContext)
+
+ await flushMicrotasks()
+ expect(netPol.reconcileContext).toHaveBeenCalledWith(
+ expect.objectContaining({ name: 'current-context' }),
+ expect.objectContaining({ isCurrent: expect.any(Function) })
+ )
+
+ releaseFullReconcile.resolve(undefined)
+ await Promise.all([convergence, watchConvergence])
+
+ expect(reconciliationOrder).toEqual(['watch', 'initial'])
+ expect(netPol.reconcileContext).toHaveBeenLastCalledWith(
+ expect.objectContaining({ name: 'current-context' }),
+ expect.objectContaining({ isCurrent: expect.any(Function) })
+ )
+ await watcher.stop()
+ })
+
+ it('makes a recreated Context the last writer after same-contextId orphan cleanup', async () => {
+ const contextName = 'recreated-context-resource'
+ const contextId = 'recreated-context'
+ const recreatedContext = {
+ metadata: { name: contextName, namespace: 'mcp-server' },
+ spec: {
+ contextId,
+ description: 'recreated',
+ mcpServers: [],
+ },
+ }
+ const orphanCleanupStarted = deferred()
+ const releaseOrphanCleanup = deferred()
+ const order: string[] = []
+ const watcher = new McpServerWatcher()
+ ;(watcher as any).mcpServerCacheSynced = true
+ ;(watcher as any).contextCacheSynced = true
+ mocks.netPolFullReconcile.mockImplementationOnce(
+ async (
+ _contexts: unknown[],
+ _servers: unknown[],
+ options: {
+ runContextEffect: (effectContextId: string, work: () => Promise<void>) => Promise<void>
+ }
+ ) => {
+ await options.runContextEffect(contextId, async () => {
+ orphanCleanupStarted.resolve(undefined)
+ await releaseOrphanCleanup.promise
+ order.push('orphan-delete')
+ })
+ }
+ )
+ const netPol = (watcher as any).netPolReconciler
+ netPol.reconcileContext.mockImplementationOnce(async () => {
+ order.push('live-recreate')
+ })
+ let contextWatchCallback:
+ | ((type: string, apiObj: typeof recreatedContext) => Promise<void>)
+ | undefined
+ mocks.watch.mockImplementationOnce(async (_path, _options, callback) => {
+ contextWatchCallback = callback
+ return { abort: vi.fn() }
+ })
+ await (watcher as any).startContextWatch('recreated-context-rv')
+
+ const convergence = (watcher as any).runInitialNetworkPolicyConvergence()
+ await orphanCleanupStarted.promise
+ const recreation = contextWatchCallback!('ADDED', recreatedContext)
+ await flushMicrotasks()
+
+ expect(netPol.reconcileContext).not.toHaveBeenCalled()
+
+ releaseOrphanCleanup.resolve(undefined)
+ await Promise.all([convergence, recreation])
+
+ expect(order).toEqual(['orphan-delete', 'live-recreate'])
+ expect(netPol.reconcileContext).toHaveBeenLastCalledWith(
+ expect.objectContaining({
+ name: contextName,
+ spec: expect.objectContaining({ contextId, description: 'recreated' }),
+ }),
+ expect.objectContaining({ isCurrent: expect.any(Function) })
+ )
+ await watcher.stop()
+ })
+
+ it('retires old contextId policies before reconciling a mutable Context identity', async () => {
+ const oldContext = {
+ name: 'mutable-context-resource',
+ namespace: 'mcp-server',
+ spec: {
+ contextId: 'old-context-id',
+ mcpServers: ['redis-tools'],
+ },
+ }
+ const newContext = {
+ metadata: { name: oldContext.name, namespace: oldContext.namespace },
+ spec: {
+ contextId: 'new-context-id',
+ mcpServers: ['redis-tools'],
+ },
+ }
+ mocks.listNamespacedCustomObject.mockImplementation(async ({ plural }: { plural: string }) => {
+ if (plural === 'contexts') {
+ return {
+ metadata: { resourceVersion: 'mutable-context-rv' },
+ items: [newContext],
+ }
+ }
+ return { items: [] }
+ })
+ let contextWatchCallback:
+ | ((type: string, apiObj: typeof newContext) => Promise<void>)
+ | undefined
+ mocks.watch.mockImplementationOnce(async (_path, _options, callback) => {
+ contextWatchCallback = callback
+ return { abort: vi.fn() }
+ })
+ const watcher = new McpServerWatcher()
+ ;(watcher as any).contexts.set(oldContext.name, oldContext)
+ ;(watcher as any).contextCacheSynced = true
+ const netPol = (watcher as any).netPolReconciler
+ const order: string[] = []
+ netPol.reconcileDeleteContext.mockImplementation(async (contextId: string) => {
+ order.push(`delete:${contextId}`)
+ })
+ netPol.reconcileContext.mockImplementation(async (context: typeof oldContext) => {
+ order.push(`reconcile:${context.spec.contextId}`)
+ })
+ await (watcher as any).startContextWatch('mutable-context-rv')
+
+ await contextWatchCallback!('MODIFIED', newContext)
+
+ expect(order).toEqual(['delete:old-context-id', 'reconcile:new-context-id'])
+ expect(netPol.reconcileDeleteContext).toHaveBeenCalledWith(
+ 'old-context-id',
+ expect.any(Function)
+ )
+ await watcher.stop()
+ })
+
+ it('retires queued Context policy work immediately when its watch ends', async () => {
+ const context = {
+ metadata: { name: 'retired-context-resource', namespace: 'mcp-server' },
+ spec: { contextId: 'retired-context-id', mcpServers: [] },
+ }
+ const blockerStarted = deferred()
+ const releaseBlocker = deferred()
+ let contextWatchCallback: ((type: string, apiObj: typeof context) => Promise<void>) | undefined
+ let contextWatchDone: ((error: Error | null) => void) | undefined
+ mocks.watch.mockImplementationOnce(async (_path, _options, callback, done) => {
+ contextWatchCallback = callback
+ contextWatchDone = done
+ return { abort: vi.fn() }
+ })
+ const watcher = new McpServerWatcher()
+ const netPol = (watcher as any).netPolReconciler
+ await (watcher as any).startContextWatch('retired-context-rv')
+ const blocker = (watcher as any).enqueueContextReconciliation(
+ context.spec.contextId,
+ async () => {
+ blockerStarted.resolve(undefined)
+ await releaseBlocker.promise
+ }
+ )
+ await blockerStarted.promise
+
+ const event = contextWatchCallback!('MODIFIED', context)
+ contextWatchDone!(null)
+ releaseBlocker.resolve(undefined)
+ await Promise.all([blocker, event])
+
+ expect(netPol.reconcileContext).not.toHaveBeenCalled()
+ await watcher.stop()
+ })
+
+ it('serializes McpServer-driven Context policy work with a newer Context watch revision', async () => {
+ const server = {
+ metadata: {
+ name: 'context-policy-server',
+ namespace: 'mcp-server',
+ uid: 'context-policy-server-uid',
+ generation: 1,
+ },
+ spec: {
+ contextRef: 'context-policy',
+ image: 'clerum/context-policy-server:test',
+ transport: { type: 'streamableHttp' as const, port: 3000 },
+ },
+ }
+ const oldContext = {
+ name: 'context-policy-resource',
+ namespace: 'mcp-server',
+ spec: {
+ contextId: 'context-policy',
+ description: 'old',
+ mcpServers: [server.metadata.name],
+ },
+ }
+ const newContext = {
+ metadata: { name: oldContext.name, namespace: oldContext.namespace },
+ spec: {
+ ...oldContext.spec,
+ description: 'new',
+ },
+ }
+ const oldStarted = deferred()
+ const releaseOld = deferred()
+ const order: string[] = []
+ let oldLease: (() => boolean) | undefined
+ let contextWatchCallback:
+ | ((type: string, apiObj: typeof newContext) => Promise<void>)
+ | undefined
+ mocks.watch.mockImplementationOnce(async (_path, _options, callback) => {
+ contextWatchCallback = callback
+ return { abort: vi.fn() }
+ })
+ const watcher = new McpServerWatcher()
+ ;(watcher as any).contexts.set(oldContext.name, oldContext)
+ const netPol = (watcher as any).netPolReconciler
+ netPol.reconcileContext.mockImplementation(
+ async (context: typeof oldContext, options: { isCurrent: () => boolean }) => {
+ if (context.spec.description === 'old') {
+ oldLease = options.isCurrent
+ oldStarted.resolve(undefined)
+ await releaseOld.promise
+ order.push('old')
+ return
+ }
+ order.push('new')
+ }
+ )
+ await (watcher as any).startContextWatch('context-policy-rv')
+
+ const serverEvent = (watcher as any).getMcpServerWatchCallback()('MODIFIED', server)
+ await oldStarted.promise
+ const contextEvent = contextWatchCallback!('MODIFIED', newContext)
+ await flushMicrotasks()
+
+ expect(netPol.reconcileContext).toHaveBeenCalledTimes(1)
+ expect(oldLease?.()).toBe(false)
+ releaseOld.resolve(undefined)
+ await Promise.all([serverEvent, contextEvent])
+
+ expect(order).toEqual(['old', 'new'])
+ expect(netPol.reconcileContext).toHaveBeenLastCalledWith(
+ expect.objectContaining({
+ spec: expect.objectContaining({ description: 'new' }),
+ }),
+ expect.objectContaining({ isCurrent: expect.any(Function) })
+ )
+ await watcher.stop()
+ })
+
+ it('retires a blocked Context policy lease when MODIFIED replaces it in the same watch', async () => {
+ const oldContext = {
+ metadata: { name: 'same-stream-context', namespace: 'mcp-server' },
+ spec: {
+ contextId: 'same-stream-context',
+ description: 'old',
+ mcpServers: ['old-server'],
+ },
+ }
+ const newContext = {
+ metadata: oldContext.metadata,
+ spec: {
+ ...oldContext.spec,
+ description: 'new',
+ mcpServers: ['new-server'],
+ },
+ }
+ const mutationStarted = deferred()
+ const releaseMutation = deferred()
+ let oldLease: (() => boolean) | undefined
+ let contextWatchCallback:
+ | ((type: string, apiObj: typeof oldContext) => Promise<void>)
+ | undefined
+ mocks.watch.mockImplementationOnce(async (_path, _options, callback) => {
+ contextWatchCallback = callback
+ return { abort: vi.fn() }
+ })
+ const watcher = new McpServerWatcher()
+ const netPol = (watcher as any).netPolReconciler
+ netPol.reconcileContext.mockImplementationOnce(
+ async (_context: unknown, options: { isCurrent: () => boolean }) => {
+ oldLease = options.isCurrent
+ mutationStarted.resolve()
+ await releaseMutation.promise
+ }
+ )
+ await (watcher as any).startContextWatch('same-stream-context-rv')
+ ;(watcher as any).contextCacheSynced = true
+ ;(watcher as any).mcpServerCacheSynced = true
+
+ const oldReconcile = contextWatchCallback!('MODIFIED', oldContext)
+ await mutationStarted.promise
+ expect(oldLease?.()).toBe(true)
+
+ const newReconcile = contextWatchCallback!('MODIFIED', newContext)
+ expect(oldLease?.()).toBe(false)
+
+ releaseMutation.resolve()
+ await Promise.all([oldReconcile, newReconcile])
+ await watcher.stop()
+ })
+
+ it('keeps live control-plane events responsive during a periodic external-egress fleet resync', async () => {
+ const server = {
+ name: 'periodic-egress-server',
+ namespace: 'mcp-server',
+ uid: 'periodic-egress-uid',
+ generation: 1,
+ spec: {
+ contextRef: 'default',
+ image: 'clerum/periodic-egress:test',
+ transport: { type: 'streamableHttp' as const, port: 3000 },
+ egressBindings: [{ dns: 'periodic.example', port: 443 }],
+ },
+ }
+ const context = {
+ metadata: { name: 'live-context', namespace: 'mcp-server' },
+ spec: { contextId: 'live-context', mcpServers: [] },
+ }
+ const resyncStarted = deferred()
+ const releaseResync = deferred()
+ let contextWatchCallback: ((type: string, apiObj: typeof context) => Promise<void>) | undefined
+ mocks.watch.mockImplementationOnce(async (_path, _options, callback) => {
+ contextWatchCallback = callback
+ return { abort: vi.fn() }
+ })
+ const randomSpy = vi.spyOn(Math, 'random').mockReturnValue(0)
+ const watcher = new McpServerWatcher()
+ markMcpServerInventoryAuthoritative(watcher)
+ ;(watcher as any).servers.set(server.name, server)
+ const netPol = (watcher as any).netPolReconciler
+ netPol.reconcileExternalEgress.mockImplementationOnce(async () => {
+ resyncStarted.resolve(undefined)
+ await releaseResync.promise
+ })
+ await (watcher as any).startContextWatch('live-context-rv')
+
+ const firstResync = (watcher as any).externalEgressCoordinator.runResync() as Promise<void>
+ const overlappingResync = (
+ watcher as any
+ ).externalEgressCoordinator.runResync() as Promise<void>
+ expect(overlappingResync).toBe(firstResync)
+ await resyncStarted.promise
+
+ const liveEvent = contextWatchCallback!('MODIFIED', context)
+ await flushMicrotasks()
+ expect(netPol.reconcileContext).toHaveBeenCalledWith(
+ expect.objectContaining({ name: 'live-context' }),
+ expect.objectContaining({ isCurrent: expect.any(Function) })
+ )
+
+ releaseResync.resolve(undefined)
+ await Promise.all([firstResync, liveEvent])
+ await watcher.stop()
+ randomSpy.mockRestore()
+ })
+
+ it('retires a blocked periodic egress lease when MODIFIED replaces it in the same watch', async () => {
+ const oldServer = {
+ name: 'same-stream-egress',
+ namespace: 'mcp-server',
+ uid: 'same-stream-egress-uid',
+ generation: 1,
+ spec: {
+ contextRef: 'default',
+ image: 'clerum/same-stream-egress:old',
+ transport: { type: 'streamableHttp' as const, port: 3000 },
+ egressBindings: [{ dns: 'old.example', port: 443 }],
+ },
+ }
+ const newServer = {
+ metadata: {
+ name: oldServer.name,
+ namespace: oldServer.namespace,
+ uid: oldServer.uid,
+ generation: 2,
+ },
+ spec: {
+ ...oldServer.spec,
+ image: 'clerum/same-stream-egress:new',
+ egressBindings: [{ dns: 'new.example', port: 443 }],
+ },
+ }
+ const mutationStarted = deferred()
+ const releaseMutation = deferred()
+ let oldLease: (() => boolean) | undefined
+ const watcher = new McpServerWatcher()
+ markMcpServerInventoryAuthoritative(watcher)
+ ;(watcher as any).servers.set(oldServer.name, oldServer)
+ const netPol = (watcher as any).netPolReconciler
+ netPol.reconcileExternalEgress.mockImplementationOnce(
+ async (_server: unknown, options: { isCurrent: () => boolean }) => {
+ oldLease = options.isCurrent
+ mutationStarted.resolve()
+ await releaseMutation.promise
+ }
+ )
+ vi.spyOn(Math, 'random').mockReturnValue(0)
+
+ const periodic = (watcher as any).externalEgressCoordinator.runResync() as Promise<void>
+ await mutationStarted.promise
+ expect(oldLease?.()).toBe(true)
+
+ const replacement = (watcher as any).getMcpServerWatchCallback()('MODIFIED', newServer)
+ expect(oldLease?.()).toBe(false)
+
+ releaseMutation.resolve()
+ await Promise.all([periodic, replacement])
+ await watcher.stop()
+ })
+
+ it('uses the latest cached McpServer after periodic external-egress jitter', async () => {
+ vi.useFakeTimers()
+ const staleServer = {
+ name: 'periodic-egress-server',
+ namespace: 'mcp-server',
+ uid: 'periodic-egress-uid',
+ generation: 1,
+ spec: {
+ contextRef: 'default',
+ image: 'clerum/periodic-egress:old',
+ transport: { type: 'streamableHttp' as const, port: 3000 },
+ egressBindings: [{ dns: 'old.example', port: 443 }],
+ },
+ }
+ const currentServer = {
+ metadata: {
+ name: staleServer.name,
+ namespace: staleServer.namespace,
+ uid: staleServer.uid,
+ generation: 2,
+ },
+ spec: {
+ ...staleServer.spec,
+ image: 'clerum/periodic-egress:new',
+ egressBindings: [{ dns: 'new.example', port: 443 }],
+ },
+ }
+ const randomSpy = vi.spyOn(Math, 'random').mockReturnValue(0.5)
+ const watcher = new McpServerWatcher()
+ ;(watcher as any).servers.set(staleServer.name, staleServer)
+ const netPol = (watcher as any).netPolReconciler
+
+ const periodic = (watcher as any).externalEgressCoordinator.runResync() as Promise<void>
+ await flushMicrotasks()
+ await (watcher as any).getMcpServerWatchCallback()('MODIFIED', currentServer)
+ await vi.advanceTimersByTimeAsync(2500)
+ await periodic
+
+ expect(netPol.reconcileExternalEgress).toHaveBeenLastCalledWith(
+ expect.objectContaining({
+ name: staleServer.name,
+ generation: 2,
+ spec: expect.objectContaining({
+ egressBindings: [{ dns: 'new.example', port: 443 }],
+ }),
+ }),
+ { isCurrent: expect.any(Function) }
+ )
+ await watcher.stop()
+ randomSpy.mockRestore()
+ })
+
+ it('queues same-object SharedFileSystem and GlobalFileSystem watch effects behind initial passes', async () => {
+ const staleSfs = { name: 'stale-sfs', namespace: 'mcp-host', spec: {} }
+ const currentSfs = { metadata: { name: 'stale-sfs', namespace: 'mcp-host' }, spec: {} }
+ const staleGfs = { name: 'stale-gfs', namespace: 'mcp-host', spec: {} }
+ const currentGfs = {
+ metadata: { name: 'stale-gfs', namespace: 'mcp-host' },
+ spec: {},
+ status: {
+ phase: 'Ready',
+ pvcName: 'gfs-drive',
+ serviceName: 'gfsc',
+ serviceUrl: 'http://gfsc.gfs.svc.cluster.local:8087',
+ },
+ }
+ const sfsInitialStarted = deferred()
+ const releaseSfsInitial = deferred()
+ const gfsInitialStarted = deferred()
+ const releaseGfsInitial = deferred()
+ const sfsOrder: string[] = []
+ const gfsOrder: string[] = []
+ const watcher = new McpServerWatcher()
+ ;(watcher as any).sharedFileSystems.set(staleSfs.name, staleSfs)
+ ;(watcher as any).globalFileSystems.set(staleGfs.name, staleGfs)
+
+ mocks.sfsFullReconcile.mockImplementationOnce(async () => {
+ sfsInitialStarted.resolve(undefined)
+ await releaseSfsInitial.promise
+ sfsOrder.push('initial')
+ })
+ const sfsReconciler = (watcher as any).sharedFileSystemReconciler
+ sfsReconciler.reconcile.mockImplementationOnce(async () => {
+ sfsOrder.push('watch')
+ })
+ const gfsReconciler = (watcher as any).gfsReconciler
+ vi.spyOn(gfsReconciler, 'fullReconcile').mockImplementationOnce(async () => {
+ gfsInitialStarted.resolve(undefined)
+ await releaseGfsInitial.promise
+ gfsOrder.push('initial')
+ })
+ const gfsReconcile = vi.spyOn(gfsReconciler, 'reconcile').mockImplementationOnce(async () => {
+ gfsOrder.push('watch')
+ })
+
+ let sfsWatchCallback: ((type: string, apiObj: typeof currentSfs) => Promise<void>) | undefined
+ let gfsWatchCallback: ((type: string, apiObj: typeof currentGfs) => Promise<void>) | undefined
+ mocks.watch
+ .mockImplementationOnce(async (_path, _options, callback) => {
+ sfsWatchCallback = callback
+ return { abort: vi.fn() }
+ })
+ .mockImplementationOnce(async (_path, _options, callback) => {
+ gfsWatchCallback = callback
+ return { abort: vi.fn() }
+ })
+ await (watcher as any).startSharedFileSystemWatch()
+ await (watcher as any).startGlobalFileSystemWatch()
+
+ const sfsInitial = (watcher as any).runInitialSharedFileSystemConvergence()
+ const gfsInitial = (watcher as any).runInitialGlobalFileSystemConvergence()
+ await Promise.all([sfsInitialStarted.promise, gfsInitialStarted.promise])
+ const sfsWatch = sfsWatchCallback!('MODIFIED', currentSfs)
+ const gfsWatch = gfsWatchCallback!('MODIFIED', currentGfs)
+
+ await flushMicrotasks()
+ expect(sfsReconciler.reconcile).not.toHaveBeenCalled()
+ expect(gfsReconcile).not.toHaveBeenCalled()
+
+ releaseSfsInitial.resolve(undefined)
+ releaseGfsInitial.resolve(undefined)
+ await Promise.all([sfsInitial, gfsInitial, sfsWatch, gfsWatch])
+
+ expect(sfsOrder).toEqual(['initial', 'watch'])
+ expect(gfsOrder).toEqual(['initial', 'watch'])
+ expect(gfsReconcile).toHaveBeenCalledWith(
+ expect.objectContaining({ status: currentGfs.status })
+ )
+ await watcher.stop()
+ })
+
+ it('does not let a periodic SFS or GFS inventory overwrite a newer watch cache entry', async () => {
+ const staleSfs = { metadata: { name: 'stale-sfs', namespace: 'mcp-host' }, spec: {} }
+ const currentSfs = { metadata: { name: 'current-sfs', namespace: 'mcp-host' }, spec: {} }
+ const staleGfs = { metadata: { name: 'stale-gfs', namespace: 'mcp-host' }, spec: {} }
+ const currentGfs = { metadata: { name: 'current-gfs', namespace: 'mcp-host' }, spec: {} }
+ const sfsListStarted = deferred()
+ const releaseSfsList = deferred<unknown>()
+ const gfsListStarted = deferred()
+ const releaseGfsList = deferred<unknown>()
+ const watcher = new McpServerWatcher()
+ mocks.listNamespacedCustomObject.mockImplementation(async ({ plural }: { plural: string }) => {
+ if (plural === 'sharedfilesystems') {
+ sfsListStarted.resolve(undefined)
+ return releaseSfsList.promise
+ }
+ if (plural === 'globalfilesystems') {
+ gfsListStarted.resolve(undefined)
+ return releaseGfsList.promise
+ }
+ return { items: [] }
+ })
+ const gfsReconciler = (watcher as any).gfsReconciler
+ vi.spyOn(gfsReconciler, 'fullReconcile').mockResolvedValue(undefined)
+
+ let sfsWatchCallback: ((type: string, apiObj: typeof currentSfs) => Promise<void>) | undefined
+ let gfsWatchCallback: ((type: string, apiObj: typeof currentGfs) => Promise<void>) | undefined
+ mocks.watch
+ .mockImplementationOnce(async (_path, _options, callback) => {
+ sfsWatchCallback = callback
+ return { abort: vi.fn() }
+ })
+ .mockImplementationOnce(async (_path, _options, callback) => {
+ gfsWatchCallback = callback
+ return { abort: vi.fn() }
+ })
+ await (watcher as any).startSharedFileSystemWatch()
+ await (watcher as any).startGlobalFileSystemWatch()
+
+ const sfsResync = (watcher as any).runSfsResync()
+ const gfsResync = (watcher as any).runGfsResync()
+ await Promise.all([sfsListStarted.promise, gfsListStarted.promise])
+ const sfsWatch = sfsWatchCallback!('MODIFIED', currentSfs)
+ const gfsWatch = gfsWatchCallback!('MODIFIED', currentGfs)
+ releaseSfsList.resolve({ items: [staleSfs] })
+ releaseGfsList.resolve({ items: [staleGfs] })
+ await Promise.all([sfsResync, gfsResync, sfsWatch, gfsWatch])
+
+ expect(mocks.sfsFullReconcile).toHaveBeenCalledWith([
+ expect.objectContaining({ name: 'current-sfs' }),
+ ])
+ expect(gfsReconciler.fullReconcile).toHaveBeenCalledWith([
+ expect.objectContaining({ name: 'current-gfs' }),
+ ])
+ expect((watcher as any).sharedFileSystems.get('current-sfs')).toEqual(
+ expect.objectContaining({ name: 'current-sfs' })
+ )
+ expect((watcher as any).sharedFileSystems.has('stale-sfs')).toBe(false)
+ expect((watcher as any).globalFileSystems.get('current-gfs')).toEqual(
+ expect.objectContaining({ name: 'current-gfs' })
+ )
+ expect((watcher as any).globalFileSystems.has('stale-gfs')).toBe(false)
+ await watcher.stop()
+ })
+
+ it('refreshes SFS and GFS caches and effects from an uncontested periodic inventory', async () => {
+ const listedSfs = {
+ metadata: { name: 'listed-sfs', namespace: 'mcp-host' },
+ spec: { capacity: '1Gi' },
+ }
+ const listedGfs = {
+ metadata: { name: 'listed-gfs', namespace: 'gfs' },
+ spec: { capacity: '10Gi' },
+ }
+ mocks.listNamespacedCustomObject.mockImplementation(async ({ plural }: { plural: string }) => {
+ if (plural === 'sharedfilesystems') return { items: [listedSfs] }
+ if (plural === 'globalfilesystems') return { items: [listedGfs] }
+ return { items: [] }
+ })
+ const watcher = new McpServerWatcher()
+ const gfsReconciler = (watcher as any).gfsReconciler
+ const gfsFullReconcile = vi.spyOn(gfsReconciler, 'fullReconcile').mockResolvedValue(undefined)
+
+ await Promise.all([(watcher as any).runSfsResync(), (watcher as any).runGfsResync()])
+
+ expect((watcher as any).sharedFileSystems.get('listed-sfs')).toEqual(
+ expect.objectContaining({ name: 'listed-sfs', spec: listedSfs.spec })
+ )
+ expect((watcher as any).globalFileSystems.get('listed-gfs')).toEqual(
+ expect.objectContaining({ name: 'listed-gfs', spec: listedGfs.spec })
+ )
+ expect(mocks.sfsFullReconcile).toHaveBeenCalledWith([
+ expect.objectContaining({ name: 'listed-sfs', spec: listedSfs.spec }),
+ ])
+ expect(gfsFullReconcile).toHaveBeenCalledWith([
+ expect.objectContaining({ name: 'listed-gfs', spec: listedGfs.spec }),
+ ])
+ await watcher.stop()
+ })
+
+ it('does not run queued fleet effects after the watcher stops', async () => {
+ const initialServer = {
+ name: 'initial-server',
+ namespace: 'mcp-server',
+ spec: {
+ contextRef: 'default',
+ image: 'clerum/initial-server:test',
+ transport: { type: 'streamableHttp' as const, port: 3000 },
+ egressBindings: [{ dns: 'initial.example', port: 443 }],
+ },
+ }
+ const queuedServer = {
+ metadata: { name: 'queued-server', namespace: 'mcp-server' },
+ spec: {
+ contextRef: 'default',
+ image: 'clerum/queued-server:test',
+ transport: { type: 'streamableHttp' as const, port: 3000 },
+ },
+ }
+ const egressStarted = deferred()
+ const releaseEgress = deferred()
+ const watcher = new McpServerWatcher()
+ ;(watcher as any).mcpServerCacheSynced = true
+ ;(watcher as any).contextCacheSynced = true
+ markNetworkPolicyRevocationAuthoritative(watcher)
+ ;(watcher as any).servers.set(initialServer.name, initialServer)
+ const netPol = (watcher as any).netPolReconciler
+ netPol.reconcileExternalEgress.mockImplementationOnce(async () => {
+ egressStarted.resolve(undefined)
+ await releaseEgress.promise
+ })
+ const reconciler = (watcher as any).reconciler
+ mocks.serverFullReconcile.mockImplementationOnce(async (servers, options) => {
+ await Promise.all(
+ servers.map((selected: { name: string }) =>
+ options.runEffect(selected.name, () => reconciler.reconcile(selected))
+ )
+ )
+ })
+
+ const initial = (watcher as any).runInitialMcpServerConvergence()
+ await egressStarted.promise
+ const queuedWatch = (watcher as any).getMcpServerWatchCallback()('MODIFIED', queuedServer)
+
+ await watcher.stop()
+ releaseEgress.resolve(undefined)
+ await Promise.all([initial, queuedWatch])
+
+ expect(mocks.serverFullReconcile).toHaveBeenCalledOnce()
+ expect(reconciler.reconcile).not.toHaveBeenCalled()
+ })
+
+ it('coalesces overlapping McpServer fleet passes into one trailing current-cache pass', async () => {
+ const firstPassStarted = deferred()
+ const releaseFirstPass = deferred()
+ const snapshots: string[][] = []
+ const watcher = new McpServerWatcher()
+ ;(watcher as any).mcpServerCacheSynced = true
+ ;(watcher as any).contextCacheSynced = true
+ markNetworkPolicyRevocationAuthoritative(watcher)
+ ;(watcher as any).servers.set('first-server', {
+ name: 'first-server',
+ namespace: 'mcp-server',
+ spec: {
+ contextRef: 'default',
+ image: 'clerum/first-server:test',
+ transport: { type: 'streamableHttp' as const, port: 3000 },
+ },
+ })
+ mocks.serverFullReconcile
+ .mockImplementationOnce(async (servers: Array<{ name: string }>) => {
+ snapshots.push(servers.map(server => server.name))
+ firstPassStarted.resolve(undefined)
+ await releaseFirstPass.promise
+ })
+ .mockImplementationOnce(async (servers: Array<{ name: string }>) => {
+ snapshots.push(servers.map(server => server.name))
+ })
+
+ const first = (watcher as any).runInitialMcpServerConvergence()
+ await firstPassStarted.promise
+ ;(watcher as any).servers.set('second-server', {
+ name: 'second-server',
+ namespace: 'mcp-server',
+ spec: {
+ contextRef: 'default',
+ image: 'clerum/second-server:test',
+ transport: { type: 'streamableHttp' as const, port: 3000 },
+ },
+ })
+ const second = (watcher as any).runInitialMcpServerConvergence()
+ const third = (watcher as any).runInitialMcpServerConvergence()
+
+ expect(mocks.serverFullReconcile).toHaveBeenCalledTimes(1)
+ releaseFirstPass.resolve(undefined)
+ await Promise.all([first, second, third])
+
+ expect(mocks.serverFullReconcile).toHaveBeenCalledTimes(2)
+ expect(snapshots).toEqual([['first-server'], ['first-server', 'second-server']])
+ await watcher.stop()
+ })
+
+ it('coalesces overlapping NetworkPolicy fleet passes into one trailing current-cache pass', async () => {
+ const firstPassStarted = deferred()
+ const releaseFirstPass = deferred()
+ const snapshots: string[][] = []
+ const watcher = new McpServerWatcher()
+ ;(watcher as any).contextCacheSynced = true
+ ;(watcher as any).mcpServerCacheSynced = true
+ ;(watcher as any).contexts.set('first-context', {
+ name: 'first-context',
+ namespace: 'mcp-server',
+ spec: { contextId: 'first-context', mcpServers: [] },
+ })
+ mocks.netPolFullReconcile
+ .mockImplementationOnce(async (contexts: Array<{ name: string }>) => {
+ snapshots.push(contexts.map(context => context.name))
+ firstPassStarted.resolve(undefined)
+ await releaseFirstPass.promise
+ })
+ .mockImplementationOnce(async (contexts: Array<{ name: string }>) => {
+ snapshots.push(contexts.map(context => context.name))
+ })
+
+ const first = (watcher as any).runInitialNetworkPolicyConvergence()
+ await firstPassStarted.promise
+ ;(watcher as any).contexts.set('second-context', {
+ name: 'second-context',
+ namespace: 'mcp-server',
+ spec: { contextId: 'second-context', mcpServers: [] },
+ })
+ const second = (watcher as any).runInitialNetworkPolicyConvergence()
+ const third = (watcher as any).runInitialNetworkPolicyConvergence()
+
+ expect(mocks.netPolFullReconcile).toHaveBeenCalledTimes(1)
+ releaseFirstPass.resolve(undefined)
+ await Promise.all([first, second, third])
+
+ expect(mocks.netPolFullReconcile).toHaveBeenCalledTimes(2)
+ expect(snapshots).toEqual([['first-context'], ['first-context', 'second-context']])
+ await watcher.stop()
+ })
+
+ it('does not lose an initial convergence request queued in the settlement microtask', async () => {
+ const watcher = new McpServerWatcher()
+ let trailing: Promise<void> | undefined
+ const convergenceCore = vi
+ .fn()
+ .mockImplementationOnce(
+ () =>
+ ({
+ then: (resolve: () => void) => {
+ resolve()
+ queueMicrotask(() => {
+ trailing = (watcher as any).runInitialMcpServerConvergence()
+ })
+ },
+ }) as PromiseLike<void>
+ )
+ .mockResolvedValueOnce(undefined)
+ ;(watcher as any).runInitialMcpServerConvergenceCore = convergenceCore
+
+ const first = (watcher as any).runInitialMcpServerConvergence()
+ await first
+ await vi.waitFor(() => expect(trailing).toBeDefined())
+ await trailing
+
+ expect(convergenceCore).toHaveBeenCalledTimes(2)
+ await watcher.stop()
+ })
+
+ it('retains one keyed queue until every same-key waiter has drained', async () => {
+ const watcher = new McpServerWatcher()
+ const firstStarted = deferred()
+ const releaseFirst = deferred()
+ const effects: string[] = []
+ const queues = (watcher as any).contextReconciliationQueues as Map<
+ string,
+ { references: number }
+ >
+
+ const first = (watcher as any).enqueueContextReconciliation('shared', async () => {
+ effects.push('first')
+ firstStarted.resolve(undefined)
+ await releaseFirst.promise
+ })
+ await firstStarted.promise
+ const second = (watcher as any).enqueueContextReconciliation('shared', async () => {
+ effects.push('second')
+ })
+
+ expect(queues.get('shared')?.references).toBe(2)
+ expect(effects).toEqual(['first'])
+ releaseFirst.resolve(undefined)
+ await Promise.all([first, second])
+ expect(effects).toEqual(['first', 'second'])
+ expect(queues.size).toBe(0)
+ await watcher.stop()
+ })
+
+ it('does not execute newly enqueued keyed work after stop', async () => {
+ const watcher = new McpServerWatcher()
+ const work = vi.fn()
+ ;(watcher as any).stopped = true
+
+ await (watcher as any).enqueueContextReconciliation('stopped', work)
+
+ expect(work).not.toHaveBeenCalled()
+ expect((watcher as any).contextReconciliationQueues.size).toBe(0)
+ })
+
+ it('acquires multi-context identity keys in one deterministic sorted order', async () => {
+ const watcher = new McpServerWatcher()
+ const acquired: string[] = []
+ vi.spyOn(watcher as any, 'enqueueContextReconciliation').mockImplementation(
+ async (...args: unknown[]) => {
+ const [key, work] = args as [string, () => Promise<void>]
+ acquired.push(key)
+ await work()
+ }
+ )
+
+ await (watcher as any).enqueueContextIdentityReconciliation(
+ ['z-context', 'a-context', 'm-context', 'z-context'],
+ async () => undefined
+ )
+
+ expect(acquired).toEqual(['a-context', 'm-context', 'z-context'])
+ await watcher.stop()
+ })
+
+ it('fences an McpServer fleet pass when its LIST-to-WATCH authority is retired', async () => {
+ const egressStarted = deferred()
+ const releaseEgress = deferred()
+ const watcher = new McpServerWatcher()
+ ;(watcher as any).mcpServerCacheSynced = true
+ ;(watcher as any).contextCacheSynced = true
+ ;(watcher as any).mcpWatchGeneration = 7
+ markNetworkPolicyRevocationAuthoritative(watcher)
+ ;(watcher as any).initialConvergenceRetryAttempts.set('McpServer', 3)
+ ;(watcher as any).servers.set('stale-server', {
+ name: 'stale-server',
+ namespace: 'mcp-server',
+ generation: 1,
+ spec: {
+ contextRef: 'default',
+ image: 'clerum/stale-server:v1',
+ transport: { type: 'streamableHttp' as const, port: 3000 },
+ egressBindings: [{ dns: 'stale.example', port: 443 }],
+ },
+ })
+ const netPol = (watcher as any).netPolReconciler
+ const reconciler = (watcher as any).reconciler
+ netPol.reconcileExternalEgress.mockImplementationOnce(async () => {
+ egressStarted.resolve(undefined)
+ await releaseEgress.promise
+ })
+ mocks.serverFullReconcile.mockImplementation(async (servers, options) => {
+ await Promise.all(
+ servers.map((selected: { name: string }) =>
+ options.runEffect(selected.name, () => reconciler.reconcile(selected))
+ )
+ )
+ })
+
+ const stalePass = (watcher as any).runInitialMcpServerConvergence()
+ await egressStarted.promise
+ ;(watcher as any).mcpServerCacheSynced = false
+ ;(watcher as any).mcpWatchGeneration = 8
+ const trailing = (watcher as any).runInitialMcpServerConvergence()
+ releaseEgress.resolve(undefined)
+ await Promise.all([stalePass, trailing])
+
+ expect(reconciler.reconcile).not.toHaveBeenCalled()
+ expect((watcher as any).initialConvergenceRetryAttempts.get('McpServer')).toBe(3)
+ ;(watcher as any).servers.set('stale-server', {
+ name: 'stale-server',
+ namespace: 'mcp-server',
+ generation: 2,
+ spec: {
+ contextRef: 'default',
+ image: 'clerum/stale-server:v2',
+ transport: { type: 'streamableHttp' as const, port: 3000 },
+ egressBindings: [{ dns: 'current.example', port: 443 }],
+ },
+ })
+ ;(watcher as any).mcpServerCacheSynced = true
+ markNetworkPolicyRevocationAuthoritative(watcher)
+ await (watcher as any).runInitialMcpServerConvergence()
+
+ expect(reconciler.reconcile).toHaveBeenCalledOnce()
+ expect(reconciler.reconcile).toHaveBeenCalledWith(
+ expect.objectContaining({
+ generation: 2,
+ spec: expect.objectContaining({ image: 'clerum/stale-server:v2' }),
+ })
+ )
+ expect((watcher as any).initialConvergenceRetryAttempts.has('McpServer')).toBe(false)
+ await watcher.stop()
+ })
+
+ it('fences NetworkPolicy positive effects when their inventory leases are retired', async () => {
+ const firstPassStarted = deferred()
+ const releaseFirstPass = deferred()
+ const appliedContexts: string[] = []
+ const appliedServers: string[] = []
+ let pass = 0
+ const watcher = new McpServerWatcher()
+ ;(watcher as any).contextCacheSynced = true
+ ;(watcher as any).mcpServerCacheSynced = true
+ ;(watcher as any).contextWatchGeneration = 11
+ ;(watcher as any).mcpWatchGeneration = 13
+ ;(watcher as any).initialConvergenceRetryAttempts.set('NetworkPolicy', 4)
+ ;(watcher as any).contexts.set('stale-context', {
+ name: 'stale-context',
+ namespace: 'mcp-server',
+ spec: { contextId: 'stale-context', mcpServers: ['stale-server'] },
+ })
+ ;(watcher as any).servers.set('stale-server', {
+ name: 'stale-server',
+ namespace: 'mcp-server',
+ spec: {
+ contextRef: 'stale-context',
+ image: 'clerum/stale-server:v1',
+ transport: { type: 'streamableHttp' as const, port: 3000 },
+ },
+ })
+ mocks.netPolFullReconcile.mockImplementation(async (contexts, servers, options) => {
+ pass += 1
+ if (pass === 1) {
+ firstPassStarted.resolve(undefined)
+ await releaseFirstPass.promise
+ }
+ await Promise.all(
+ contexts.map((context: { spec: { contextId: string } }) =>
+ options.runContextEffect(context.spec.contextId, async () => {
+ appliedContexts.push(context.spec.contextId)
+ })
+ )
+ )
+ await Promise.all(
+ servers.map((server: { name: string }) =>
+ options.runServerEffect(server.name, async () => {
+ appliedServers.push(server.name)
+ })
+ )
+ )
+ })
+
+ const stalePass = (watcher as any).runInitialNetworkPolicyConvergence()
+ await firstPassStarted.promise
+ ;(watcher as any).contextCacheSynced = false
+ ;(watcher as any).mcpServerCacheSynced = false
+ ;(watcher as any).contextWatchGeneration = 12
+ ;(watcher as any).mcpWatchGeneration = 14
+ const trailing = (watcher as any).runInitialNetworkPolicyConvergence()
+ releaseFirstPass.resolve(undefined)
+ await Promise.all([stalePass, trailing])
+
+ expect(appliedContexts).toEqual([])
+ expect(appliedServers).toEqual([])
+ expect((watcher as any).initialConvergenceRetryAttempts.get('NetworkPolicy')).toBe(4)
+ ;(watcher as any).contexts.clear()
+ ;(watcher as any).servers.clear()
+ ;(watcher as any).contexts.set('current-context', {
+ name: 'current-context',
+ namespace: 'mcp-server',
+ spec: { contextId: 'current-context', mcpServers: ['current-server'] },
+ })
+ ;(watcher as any).servers.set('current-server', {
+ name: 'current-server',
+ namespace: 'mcp-server',
+ spec: {
+ contextRef: 'current-context',
+ image: 'clerum/current-server:v2',
+ transport: { type: 'streamableHttp' as const, port: 3000 },
+ },
+ })
+ ;(watcher as any).contextCacheSynced = true
+ ;(watcher as any).mcpServerCacheSynced = true
+ await (watcher as any).runInitialNetworkPolicyConvergence()
+
+ expect(appliedContexts).toEqual(['current-context'])
+ expect(appliedServers).toEqual(['current-server'])
+ expect((watcher as any).initialConvergenceRetryAttempts.has('NetworkPolicy')).toBe(false)
+ await watcher.stop()
+ })
+
+ it('does not certify NetworkPolicy revocation after authority is lost at the callback boundary', async () => {
+ const watcher = new McpServerWatcher()
+ ;(watcher as any).contextCacheSynced = true
+ ;(watcher as any).mcpServerCacheSynced = true
+ ;(watcher as any).contextWatchGeneration = 21
+ ;(watcher as any).mcpWatchGeneration = 34
+ mocks.netPolFullReconcile.mockImplementationOnce(async (_contexts, _servers, options) => {
+ ;(watcher as any).contextCacheSynced = false
+ options.onAuthoritativeRevocationComplete()
+ })
+
+ await (watcher as any).runInitialNetworkPolicyConvergence()
+
+ // Authority lost at the callback boundary (contextCacheSynced=false) → record
+ // refuses, so the content-identity revocation counters stay at the "never
+ // certified" sentinel rather than adopting the captured revision.
+ expect((watcher as any).networkPolicyRevocationContextRevision).toBe(-1)
+ expect((watcher as any).networkPolicyRevocationServerRevision).toBe(-1)
+ await watcher.stop()
+ })
+
+ it('wires onExternalEgressRevoked to schedule a MODIFIED external-egress retry (B3)', async () => {
+ const watcher = new McpServerWatcher()
+ ;(watcher as any).contextCacheSynced = true
+ ;(watcher as any).mcpServerCacheSynced = true
+ ;(watcher as any).contextWatchGeneration = 21
+ ;(watcher as any).mcpWatchGeneration = 34
+ const server = {
+ name: 'openai-mcp',
+ namespace: 'mcp-server',
+ spec: {
+ contextRef: 'dev',
+ image: 'openai:latest',
+ transport: { type: 'streamableHttp' as const, port: 3000 },
+ egressBindings: [{ dns: 'api.openai.com', port: 443 }],
+ },
+ }
+ const retrySpy = vi
+ .spyOn((watcher as any).externalEgressCoordinator, 'scheduleRetry')
+ .mockImplementation(() => undefined)
+ let captured: { onExternalEgressRevoked?: (s: unknown) => void } | undefined
+ mocks.netPolFullReconcile.mockImplementationOnce(
+ async (_contexts: unknown[], _servers: unknown[], options: typeof captured) => {
+ captured = options
+ }
+ )
+
+ await (watcher as any).runInitialNetworkPolicyConvergence()
+
+ expect(captured?.onExternalEgressRevoked).toBeTypeOf('function')
+ captured!.onExternalEgressRevoked!(server)
+ expect(retrySpy).toHaveBeenCalledWith('MODIFIED', server)
+ await watcher.stop()
+ })
+
+ it('does not start runtime fleet convergence without a current NetworkPolicy safety certificate', async () => {
+ const watcher = new McpServerWatcher()
+ ;(watcher as any).contextCacheSynced = true
+ ;(watcher as any).mcpServerCacheSynced = true
+ ;(watcher as any).contextWatchGeneration = 4
+ ;(watcher as any).mcpWatchGeneration = 7
+
+ await (watcher as any).runInitialMcpServerConvergence()
+ expect(mocks.serverFullReconcile).not.toHaveBeenCalled()
+
+ await (watcher as any).runInitialNetworkPolicyConvergence()
+ await vi.waitFor(() => expect(mocks.serverFullReconcile).toHaveBeenCalledOnce())
+
+ await watcher.stop()
+ })
+
+ it('does not start runtime fleet convergence while the safety inventory is uncertified', async () => {
+ // M1: with the revocation counters aligned, the ONLY thing that can null the
+ // certificate is the lost safety fence. Fence lost -> cert null -> the fleet
+ // lane must not run; fence restored -> it runs exactly once.
+ const watcher = new McpServerWatcher()
+ ;(watcher as any).contextCacheSynced = true
+ ;(watcher as any).mcpServerCacheSynced = true
+ ;(watcher as any).contextWatchGeneration = 4
+ ;(watcher as any).mcpWatchGeneration = 7
+ markNetworkPolicyRevocationAuthoritative(watcher)
+ try {
+ mocks.hasCertifiedSafetyInventory.mockReturnValue(false)
+ await (watcher as any).runInitialMcpServerConvergence()
+ expect(mocks.serverFullReconcile).not.toHaveBeenCalled()
+
+ mocks.hasCertifiedSafetyInventory.mockReturnValue(true)
+ await (watcher as any).runInitialMcpServerConvergence()
+ expect(mocks.serverFullReconcile).toHaveBeenCalledOnce()
+ } finally {
+ // vi.clearAllMocks() clears calls, not implementations — restore explicitly
+ // or every later test would re-gate on a stale false.
+ mocks.hasCertifiedSafetyInventory.mockReturnValue(true)
+ await watcher.stop()
+ }
+ })
+
+ it('invalidates readiness when either desired policy revision advances', async () => {
+ const watcher = new McpServerWatcher()
+ ;(watcher as any).hostCacheSynced = true
+ ;(watcher as any).contextCacheSynced = true
+ ;(watcher as any).mcpServerCacheSynced = true
+ ;(watcher as any).contextWatchGeneration = 4
+ ;(watcher as any).mcpWatchGeneration = 7
+ markNetworkPolicyRevocationAuthoritative(watcher)
+ expect(watcher.isReadinessInventoryAuthoritative()).toBe(true)
+ ;(watcher as any).mcpServerDesiredRevision += 1
+ expect(watcher.isReadinessInventoryAuthoritative()).toBe(false)
+
+ markNetworkPolicyRevocationAuthoritative(watcher)
+ expect(watcher.isReadinessInventoryAuthoritative()).toBe(true)
+ ;(watcher as any).contextDesiredRevision += 1
+ expect(watcher.isReadinessInventoryAuthoritative()).toBe(false)
+
+ await watcher.stop()
+ })
+
+ it('retires readiness and requests a safety pass before a changed McpServer effect settles', async () => {
+ const previous = {
+ metadata: {
+ name: 'restrictive-change',
+ namespace: 'mcp-server',
+ uid: 'restrictive-change-uid',
+ generation: 1,
+ },
+ spec: {
+ contextRef: 'default',
+ image: 'clerum/restrictive-change:test',
+ transport: { type: 'streamableHttp' as const, port: 3000 },
+ egressBindings: [{ dns: 'api.example.com', port: 443, protocol: 'TCP' as const }],
+ },
+ }
+ const current = {
+ ...previous,
+ metadata: { ...previous.metadata, generation: 2 },
+ spec: {
+ ...previous.spec,
+ egressBindings: [{ dns: 'api.example.com', port: 443, protocol: 'UDP' as const }],
+ },
+ }
+ const watcher = new McpServerWatcher()
+ ;(watcher as any).hostCacheSynced = true
+ ;(watcher as any).contextCacheSynced = true
+ ;(watcher as any).mcpServerCacheSynced = true
+ ;(watcher as any).contextWatchGeneration = 4
+ ;(watcher as any).mcpWatchGeneration = 7
+ ;(watcher as any).servers.set(previous.metadata.name, {
+ name: previous.metadata.name,
+ namespace: previous.metadata.namespace,
+ uid: previous.metadata.uid,
+ generation: previous.metadata.generation,
+ spec: previous.spec,
+ })
+ markNetworkPolicyRevocationAuthoritative(watcher)
+ const safetyPass = vi
+ .spyOn(watcher as any, 'runInitialNetworkPolicyConvergence')
+ .mockResolvedValue(undefined)
+ ;(watcher as any).netPolReconciler.reconcileExternalEgress.mockRejectedValueOnce(
+ new Error('replacement policy unavailable')
+ )
+ const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {})
+
+ await (watcher as any).getMcpServerWatchCallback(7)('MODIFIED', current)
+
+ expect(watcher.isReadinessInventoryAuthoritative()).toBe(false)
+ expect(safetyPass).toHaveBeenCalled()
+ await watcher.stop()
+ errorSpy.mockRestore()
+ })
+
+ it('fences runtime effects when a desired-revision change retires the safety certificate', async () => {
+ const selected = {
+ name: 'certificate-fenced-server',
+ namespace: 'mcp-server',
+ spec: {
+ contextRef: 'default',
+ image: 'clerum/certificate-fenced:test',
+ transport: { type: 'streamableHttp' as const, port: 3000 },
+ egressBindings: [{ dns: 'certificate-fenced.example', port: 443 }],
+ },
+ }
+ const egressStarted = deferred()
+ const releaseEgress = deferred()
+ const runtimeEffect = vi.fn()
+ const watcher = new McpServerWatcher()
+ ;(watcher as any).contextCacheSynced = true
+ ;(watcher as any).mcpServerCacheSynced = true
+ ;(watcher as any).contextWatchGeneration = 11
+ ;(watcher as any).mcpWatchGeneration = 13
+ markNetworkPolicyRevocationAuthoritative(watcher)
+ ;(watcher as any).servers.set(selected.name, selected)
+ ;(watcher as any).netPolReconciler.reconcileExternalEgress.mockImplementationOnce(async () => {
+ egressStarted.resolve(undefined)
+ await releaseEgress.promise
+ })
+ mocks.serverFullReconcile.mockImplementationOnce(async (servers, options) => {
+ await Promise.all(
+ servers.map((server: { name: string }) => options.runEffect(server.name, runtimeEffect))
+ )
+ })
+
+ const convergence = (watcher as any).runInitialMcpServerConvergence()
+ await egressStarted.promise
+ // A real desired-state change lands mid-convergence: the content identity
+ // moves, retiring the captured safety certificate. A watch reconnect that
+ // did NOT change content (generation-only bump) would correctly NOT fence —
+ // that is the whole point of the content-identity gate — so the fence must
+ // key on the revision, which this change advances.
+ ;(watcher as any).contextDesiredRevision += 1
+ releaseEgress.resolve(undefined)
+ await convergence
+
+ expect(runtimeEffect).not.toHaveBeenCalled()
+ await watcher.stop()
+ })
+
+ it.each([
+ ['SharedFileSystem', 'enqueueSharedFileSystemReconciliation'],
+ ['GlobalFileSystem', 'enqueueGlobalFileSystemReconciliation'],
+ ] as const)(
+ 'does not admit queued %s effects after the watcher stops',
+ async (_, enqueueMethod) => {
+ const firstStarted = deferred()
+ const releaseFirst = deferred()
+ const watcher = new McpServerWatcher()
+ let queuedEffectRan = false
+
+ const first = (watcher as any)[enqueueMethod]('same-object', async () => {
+ firstStarted.resolve(undefined)
+ await releaseFirst.promise
+ }) as Promise<void>
+ await firstStarted.promise
+ const queued = (watcher as any)[enqueueMethod]('same-object', async () => {
+ queuedEffectRan = true
+ }) as Promise<void>
+
+ await watcher.stop()
+ releaseFirst.resolve(undefined)
+ await Promise.all([first, queued])
+
+ expect(queuedEffectRan).toBe(false)
+ }
+ )
+
+ it.each([
+ ['SharedFileSystem', 'enqueueSharedFileSystemReconciliation'],
+ ['GlobalFileSystem', 'enqueueGlobalFileSystemReconciliation'],
+ ] as const)(
+ 'serializes the same %s identity without blocking an unrelated fleet member',
+ async (_, enqueueMethod) => {
+ const firstStarted = deferred()
+ const releaseFirst = deferred()
+ const secondStarted = deferred()
+ const unrelatedStarted = deferred()
+ const watcher = new McpServerWatcher()
+ let secondRan = false
+
+ const first = (watcher as any)[enqueueMethod]('slow', async () => {
+ firstStarted.resolve(undefined)
+ await releaseFirst.promise
+ }) as Promise<void>
+ await firstStarted.promise
+ const second = (watcher as any)[enqueueMethod]('slow', async () => {
+ secondRan = true
+ secondStarted.resolve(undefined)
+ }) as Promise<void>
+ const unrelated = (watcher as any)[enqueueMethod]('independent', async () => {
+ unrelatedStarted.resolve(undefined)
+ }) as Promise<void>
+
+ await unrelatedStarted.promise
+ expect(secondRan).toBe(false)
+ releaseFirst.resolve(undefined)
+ await Promise.all([first, second, unrelated])
+ await expect(secondStarted.promise).resolves.toBeUndefined()
+ await watcher.stop()
+ }
+ )
+
+ it('keeps a keyed lane alive until a third waiter has consumed the same queue entry', async () => {
+ const firstStarted = deferred()
+ const releaseFirst = deferred()
+ const secondStarted = deferred()
+ const releaseSecond = deferred()
+ const thirdStarted = deferred()
+ let thirdRan = false
+ const watcher = new McpServerWatcher()
+
+ const first = (watcher as any).enqueueContextReconciliation('shared', async () => {
+ firstStarted.resolve(undefined)
+ await releaseFirst.promise
+ })
+ await firstStarted.promise
+ const second = (watcher as any).enqueueContextReconciliation('shared', async () => {
+ secondStarted.resolve(undefined)
+ await releaseSecond.promise
+ })
+ const third = (watcher as any).enqueueContextReconciliation('shared', async () => {
+ thirdRan = true
+ thirdStarted.resolve(undefined)
+ })
+
+ expect((watcher as any).contextReconciliationQueues.get('shared').references).toBe(3)
+ releaseFirst.resolve(undefined)
+ await secondStarted.promise
+ expect(thirdRan).toBe(false)
+ expect((watcher as any).contextReconciliationQueues.get('shared').references).toBe(2)
+ releaseSecond.resolve(undefined)
+ await Promise.all([first, second, third])
+ expect((watcher as any).contextReconciliationQueues.has('shared')).toBe(false)
+ await watcher.stop()
+ })
+
+ it('caps initial convergence retry delay after attempts exceed the schedule', async () => {
+ vi.useFakeTimers()
+ const watcher = new McpServerWatcher()
+ const convergence = vi
+ .spyOn(watcher as any, 'runInitialMcpServerConvergence')
+ .mockResolvedValue(undefined)
+ ;(watcher as any).initialConvergenceRetryAttempts.set('McpServer', 99)
+ ;(watcher as any).scheduleInitialConvergenceRetry('McpServer')
+ await vi.advanceTimersByTimeAsync(299_999)
+ expect(convergence).not.toHaveBeenCalled()
+ await vi.advanceTimersByTimeAsync(1)
+ expect(convergence).toHaveBeenCalledOnce()
+ await watcher.stop()
+ })
+
+ it('defers NetworkPolicy full convergence until both inventories are authoritative', async () => {
+ const watcher = new McpServerWatcher()
+ ;(watcher as any).contextCacheSynced = true
+ ;(watcher as any).mcpServerCacheSynced = false
+ ;(watcher as any).initialConvergenceRetryAttempts.set('NetworkPolicy', 2)
+
+ await (watcher as any).runInitialNetworkPolicyConvergence()
+
+ expect(mocks.netPolFullReconcile).not.toHaveBeenCalled()
+ expect((watcher as any).initialConvergenceRetryAttempts.get('NetworkPolicy')).toBe(2)
+ ;(watcher as any).mcpServerCacheSynced = true
+ await (watcher as any).runInitialNetworkPolicyConvergence()
+
+ expect(mocks.netPolFullReconcile).toHaveBeenCalledOnce()
+ expect(mocks.netPolFullReconcile).toHaveBeenLastCalledWith(
+ [],
+ [],
+ expect.objectContaining({
+ serverInventoryComplete: true,
+ ensureDefaults: false,
+ contextInventoryAuthoritative: expect.any(Function),
+ serverInventoryAuthoritative: expect.any(Function),
+ })
+ )
+ expect((watcher as any).initialConvergenceRetryAttempts.has('NetworkPolicy')).toBe(false)
+ await watcher.stop()
+ })
+
+ it('fences NetworkPolicy orphan cleanup to the inventory generations captured by the pass', async () => {
+ const watcher = new McpServerWatcher()
+ ;(watcher as any).contextCacheSynced = true
+ ;(watcher as any).mcpServerCacheSynced = true
+ ;(watcher as any).contextWatchGeneration = 17
+ ;(watcher as any).mcpWatchGeneration = 23
+ let options:
+ | {
+ contextInventoryAuthoritative: () => boolean
+ serverInventoryAuthoritative: () => boolean
+ }
+ | undefined
+ mocks.netPolFullReconcile.mockImplementationOnce(
+ async (
+ _contexts: unknown[],
+ _servers: unknown[],
+ received: {
+ contextInventoryAuthoritative: () => boolean
+ serverInventoryAuthoritative: () => boolean
+ }
+ ) => {
+ options = received
+ }
+ )
+
+ await (watcher as any).runInitialNetworkPolicyConvergence()
+
+ expect(options?.contextInventoryAuthoritative()).toBe(true)
+ expect(options?.serverInventoryAuthoritative()).toBe(true)
+
+ // A recovered watch may already be authoritative again, but it is a
+ // different snapshot generation and cannot authorize cleanup by this pass.
+ ;(watcher as any).contextWatchGeneration = 18
+ ;(watcher as any).mcpWatchGeneration = 24
+
+ expect(options?.contextInventoryAuthoritative()).toBe(false)
+ expect(options?.serverInventoryAuthoritative()).toBe(false)
+ await watcher.stop()
+ })
+
+ it('escalates and resets failed initial McpServer convergence without delaying bootstrap', async () => {
+ vi.useFakeTimers()
+ const server = {
+ metadata: { name: 'initial-runtime', namespace: 'mcp-server' },
+ spec: {
+ contextRef: 'default',
+ image: 'clerum/initial-runtime:test',
+ transport: { type: 'streamableHttp' as const, port: 3000 },
+ },
+ }
+ mocks.listNamespacedCustomObject.mockImplementation(async ({ plural }: { plural: string }) => {
+ if (plural === 'mcpservers') return { items: [server] }
+ if (plural === 'contexts') return { items: [] }
+ if (plural === 'communicationchannels') {
+ return { metadata: { resourceVersion: 'mcp-retry-rv' }, items: [] }
+ }
+ return { items: [] }
+ })
+ mocks.serverFullReconcile
+ .mockRejectedValueOnce(new Error('initial runtime reconciliation failed'))
+ .mockRejectedValueOnce(new Error('initial runtime reconciliation still unavailable'))
+ .mockResolvedValueOnce(undefined)
+ const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {})
+ const retryMetricBefore = await readInitialConvergenceMetric(
+ 'clerum_hcc_initial_convergence_retries_total',
+ 'McpServer'
+ )
+ const watcher = new McpServerWatcher()
+ stubAuthoritativeInventoryWatch(watcher, 'McpServer')
+ stubAuthoritativeInventoryWatch(watcher, 'Context')
+ vi.spyOn(watcher as any, 'startSharedFileSystemWatch').mockResolvedValue(undefined)
+ vi.spyOn(watcher as any, 'startGlobalFileSystemWatch').mockResolvedValue(undefined)
+ vi.spyOn(watcher as any, 'startCommunicationChannelWatch').mockResolvedValue(undefined)
+
+ await watcher.start()
+ await vi.advanceTimersByTimeAsync(0)
+ expect(mocks.serverFullReconcile).toHaveBeenCalledTimes(1)
+
+ await vi.advanceTimersByTimeAsync(5000)
+ expect(mocks.serverFullReconcile).toHaveBeenCalledTimes(2)
+
+ await vi.advanceTimersByTimeAsync(14_999)
+ expect(mocks.serverFullReconcile).toHaveBeenCalledTimes(2)
+ await vi.advanceTimersByTimeAsync(1)
+ expect(mocks.serverFullReconcile).toHaveBeenCalledTimes(3)
+ expect((watcher as any).initialConvergenceRetryAttempts.has('McpServer')).toBe(false)
+ expect((watcher as any).initialConvergenceRetryTimers.has('McpServer')).toBe(false)
+ expect(
+ await readInitialConvergenceMetric(
+ 'clerum_hcc_initial_convergence_retries_total',
+ 'McpServer'
+ )
+ ).toBe(retryMetricBefore + 2)
+ expect(
+ await readInitialConvergenceMetric(
+ 'clerum_hcc_initial_convergence_last_success_timestamp_seconds',
+ 'McpServer'
+ )
+ ).toBeGreaterThan(0)
+
+ await watcher.stop()
+ errorSpy.mockRestore()
+ })
+
+ it('cancels a pending initial convergence retry when the watcher stops', async () => {
+ vi.useFakeTimers()
+ const watcher = new McpServerWatcher()
+ ;(watcher as any).mcpServerCacheSynced = true
+ ;(watcher as any).contextCacheSynced = true
+ markNetworkPolicyRevocationAuthoritative(watcher)
+ mocks.serverFullReconcile.mockRejectedValueOnce(
+ new Error('initial runtime reconciliation unavailable')
+ )
+ const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {})
+
+ await (watcher as any).runInitialMcpServerConvergence()
+
+ expect(mocks.serverFullReconcile).toHaveBeenCalledTimes(1)
+ expect((watcher as any).initialConvergenceRetryAttempts.get('McpServer')).toBe(1)
+ expect((watcher as any).initialConvergenceRetryTimers.has('McpServer')).toBe(true)
+
+ await watcher.stop()
+ await vi.advanceTimersByTimeAsync(300_000)
+
+ expect(mocks.serverFullReconcile).toHaveBeenCalledTimes(1)
+ expect((watcher as any).initialConvergenceRetryAttempts.size).toBe(0)
+ expect((watcher as any).initialConvergenceRetryTimers.size).toBe(0)
+ errorSpy.mockRestore()
+ })
+
+ it('retries failed initial NetworkPolicy convergence from the current caches', async () => {
+ vi.useFakeTimers()
+ const context = {
+ metadata: { name: 'initial-context', namespace: 'mcp-server' },
+ spec: { contextId: 'initial-context', mcpServers: [] },
+ }
+ mocks.listNamespacedCustomObject.mockImplementation(async ({ plural }: { plural: string }) => {
+ if (plural === 'mcpservers') return { items: [] }
+ if (plural === 'contexts') return { items: [context] }
+ if (plural === 'communicationchannels') {
+ return { metadata: { resourceVersion: 'netpol-retry-rv' }, items: [] }
+ }
+ return { items: [] }
+ })
+ mocks.netPolFullReconcile
+ .mockRejectedValueOnce(new Error('initial NetworkPolicy reconciliation failed'))
+ .mockImplementationOnce(
+ async (
+ _contexts: unknown,
+ _servers: unknown,
+ options?: { onAuthoritativeRevocationComplete?: () => void }
+ ) => {
+ options?.onAuthoritativeRevocationComplete?.()
+ }
+ )
+ const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {})
+ const watcher = new McpServerWatcher()
+ stubAuthoritativeInventoryWatch(watcher, 'McpServer')
+ stubAuthoritativeInventoryWatch(watcher, 'Context')
+ vi.spyOn(watcher as any, 'startSharedFileSystemWatch').mockResolvedValue(undefined)
+ vi.spyOn(watcher as any, 'startGlobalFileSystemWatch').mockResolvedValue(undefined)
+ vi.spyOn(watcher as any, 'startCommunicationChannelWatch').mockResolvedValue(undefined)
+
+ await watcher.start()
+ await vi.advanceTimersByTimeAsync(0)
+ expect(mocks.netPolFullReconcile).toHaveBeenCalledTimes(1)
+ expect(watcher.isReadinessInventoryAuthoritative()).toBe(false)
+
+ await vi.advanceTimersByTimeAsync(5000)
+ expect(mocks.netPolFullReconcile).toHaveBeenCalledTimes(2)
+ expect(watcher.isReadinessInventoryAuthoritative()).toBe(true)
+ expect(mocks.netPolFullReconcile).toHaveBeenLastCalledWith(
+ [expect.objectContaining({ name: 'initial-context' })],
+ [],
+ expect.objectContaining({
+ serverInventoryComplete: true,
+ ensureDefaults: false,
+ contextInventoryAuthoritative: expect.any(Function),
+ serverInventoryAuthoritative: expect.any(Function),
+ })
+ )
+
+ await watcher.stop()
+ errorSpy.mockRestore()
+ })
+
+ it('does not report a post-certification additive failure as a readiness safety failure', async () => {
+ vi.useFakeTimers()
+ const additiveFailure = new AggregateError(
+ [new Error('Context policy API unavailable')],
+ 'One or more additive Context NetworkPolicy reconciliations failed'
+ )
+ mocks.netPolFullReconcile.mockImplementationOnce(
+ async (
+ _contexts: unknown,
+ _servers: unknown,
+ options?: { onAuthoritativeRevocationComplete?: () => void }
+ ) => {
+ options?.onAuthoritativeRevocationComplete?.()
+ throw additiveFailure
+ }
+ )
+ const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {})
+ const watcher = new McpServerWatcher()
+ stubAuthoritativeInventoryWatch(watcher, 'McpServer')
+ stubAuthoritativeInventoryWatch(watcher, 'Context')
+ vi.spyOn(watcher as any, 'startSharedFileSystemWatch').mockResolvedValue(undefined)
+ vi.spyOn(watcher as any, 'startGlobalFileSystemWatch').mockResolvedValue(undefined)
+ vi.spyOn(watcher as any, 'startCommunicationChannelWatch').mockResolvedValue(undefined)
+
+ await watcher.start()
+ await vi.advanceTimersByTimeAsync(0)
+
+ expect(watcher.isReadinessInventoryAuthoritative()).toBe(true)
+ expect(errorSpy).toHaveBeenCalledWith(
+ '[K8s] Initial NetworkPolicy post-certification additive reconciliation failed:',
+ additiveFailure
+ )
+ expect(errorSpy).not.toHaveBeenCalledWith(
+ '[K8s] Initial NetworkPolicy background reconciliation failed:',
+ additiveFailure
+ )
+ expect((watcher as any).initialConvergenceRetryTimers.has('NetworkPolicy')).toBe(true)
+
+ await watcher.stop()
+ errorSpy.mockRestore()
+ })
+
+ it.each([
+ ['McpServer', 'startMcpServerWatch'],
+ ['Context', 'startContextWatch'],
+ ] as const)(
+ 'keeps safe bootstrap authority-gated when the %s watch cannot be established',
+ async (_kind, method) => {
+ const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {})
+ const warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => {})
+ const watcher = new McpServerWatcher()
+ if (method === 'startMcpServerWatch') {
+ vi.spyOn(watcher as any, method).mockRejectedValue(new Error(`${method} unavailable`))
+ } else {
+ stubAuthoritativeInventoryWatch(watcher, 'McpServer')
+ }
+ if (method === 'startContextWatch') {
+ vi.spyOn(watcher as any, method).mockRejectedValue(new Error(`${method} unavailable`))
+ } else if (method !== 'startMcpServerWatch') {
+ stubAuthoritativeInventoryWatch(watcher, 'Context')
+ }
+ vi.spyOn(watcher as any, 'startCommunicationChannelWatch').mockResolvedValue(undefined)
+ const recoveryTimerField =
+ method === 'startMcpServerWatch'
+ ? 'mcpServerCacheRecoveryTimer'
+ : 'contextCacheRecoveryTimer'
+
+ try {
+ await expect(watcher.start()).resolves.toBeUndefined()
+ expect(watcher.isReadinessInventoryAuthoritative()).toBe(false)
+ expect((watcher as any)[recoveryTimerField]).not.toBeNull()
+ expect(mocks.netPolFullReconcile).not.toHaveBeenCalled()
+ } finally {
+ await watcher.stop()
+ errorSpy.mockRestore()
+ warnSpy.mockRestore()
+ }
+ }
+ )
+
+ it.each([
+ ['SharedFileSystem', 'startSharedFileSystemWatch'],
+ ['GlobalFileSystem', 'startGlobalFileSystemWatch'],
+ ] as const)(
+ 'isolates, logs, and retries an initial %s background watch rejection',
+ async (kind, method) => {
+ vi.useFakeTimers()
+ const failure = new Error(`${method} unavailable`)
+ const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {})
+ const watcher = new McpServerWatcher()
+ stubAuthoritativeInventoryWatch(watcher, 'McpServer')
+ stubAuthoritativeInventoryWatch(watcher, 'Context')
+ const rejectedWatch = vi
+ .spyOn(watcher as any, method)
+ .mockRejectedValueOnce(failure)
+ .mockResolvedValueOnce(undefined)
+ const otherMethod =
+ method === 'startSharedFileSystemWatch'
+ ? 'startGlobalFileSystemWatch'
+ : 'startSharedFileSystemWatch'
+ vi.spyOn(watcher as any, otherMethod).mockResolvedValue(undefined)
+ vi.spyOn(watcher as any, 'startCommunicationChannelWatch').mockResolvedValue(undefined)
+
+ await expect(watcher.start()).resolves.toBeUndefined()
+ await vi.advanceTimersByTimeAsync(0)
+
+ expect(errorSpy).toHaveBeenCalledWith(
+ `[K8s] ${kind} background watch failed to start:`,
+ failure
+ )
+ expect(rejectedWatch).toHaveBeenCalledOnce()
+ expect(mocks.netPolFullReconcile).toHaveBeenCalledOnce()
+
+ await vi.advanceTimersByTimeAsync(4_999)
+ expect(rejectedWatch).toHaveBeenCalledOnce()
+ await vi.advanceTimersByTimeAsync(1)
+ expect(rejectedWatch).toHaveBeenCalledTimes(2)
+
+ await watcher.stop()
+ errorSpy.mockRestore()
+ }
+ )
+
+ it('applies the request deadline wrapper to the finite Host inventory LIST', async () => {
+ mocks.listNamespacedCustomObject.mockImplementation(async ({ plural }: { plural: string }) => {
+ if (plural === 'hosts') return { items: [] }
+ return { items: [] }
+ })
+
+ await expect(listAllHosts()).resolves.toEqual([])
+
+ expect(mocks.hostListCallOptions).toHaveBeenCalledWith(
+ expect.objectContaining({
+ middleware: [expect.objectContaining({ pre: expect.any(Function) })],
+ middlewareMergeStrategy: 'append',
+ })
+ )
+ })
+
+ it('preserves Host UID from the Kubernetes inventory boundary', async () => {
+ mocks.listNamespacedCustomObject.mockImplementation(async ({ plural }: { plural: string }) => {
+ if (plural === 'hosts') {
+ return {
+ items: [
+ {
+ metadata: {
+ name: 'uid-host',
+ namespace: 'mcp-host',
+ uid: '9f43826a-4031-4a31-93af-a3dd8fcfe805',
+ generation: 4,
+ },
+ spec: { host: 'uid-host', contextRef: 'context-a', secretRef: 'host-secret' },
+ },
+ ],
+ }
+ }
+ return { items: [] }
+ })
+
+ await expect(listAllHosts()).resolves.toEqual([
+ expect.objectContaining({
+ name: 'uid-host',
+ uid: '9f43826a-4031-4a31-93af-a3dd8fcfe805',
+ generation: 4,
+ }),
+ ])
+ })
+
+ it('preserves observed GlobalFileSystem status from the Kubernetes inventory boundary', async () => {
+ const status = {
+ phase: 'Ready',
+ pvcName: 'gfs-drive',
+ serviceName: 'gfsc',
+ serviceUrl: 'http://gfsc.gfs.svc.cluster.local:8087',
+ }
+ mocks.listNamespacedCustomObject.mockImplementation(async ({ plural }: { plural: string }) => {
+ if (plural === 'globalfilesystems') {
+ return {
+ items: [
+ {
+ metadata: { name: 'gfs', namespace: 'gfs' },
+ spec: {},
+ status,
+ },
+ ],
+ }
+ }
+ return { items: [] }
+ })
+
+ await expect(listAllGlobalFileSystems()).resolves.toEqual([
+ expect.objectContaining({ name: 'gfs', namespace: 'gfs', status }),
+ ])
+ })
+
+ it('reconciles startup external egress before runtime full reconciliation', async () => {
+ const eventLog: string[] = []
+ const server = {
+ metadata: { name: 'web-search', namespace: 'mcp-server', generation: 4 },
+ spec: {
+ contextRef: 'wf-research',
+ image: 'clerum/web-search:test',
+ transport: { type: 'streamableHttp' as const, port: 3000 },
+ egressBindings: [{ dns: 'duckduckgo.com', port: 443 }],
+ },
+ }
+
+ mocks.listNamespacedCustomObject.mockImplementation(async ({ plural }: { plural: string }) => {
+ if (plural === 'mcpservers') return { items: [server] }
+ if (plural === 'contexts') return { items: [] }
+ if (plural === 'hosts') return { items: [] }
+ if (plural === 'sharedfilesystems') return { items: [] }
+ if (plural === 'communicationchannels')
+ return { metadata: { resourceVersion: '1' }, items: [] }
+ return { items: [] }
+ })
+ mocks.ensureDefaultPolicies.mockImplementation(async () => {
+ eventLog.push('defaults')
+ })
+ const watcher = new McpServerWatcher()
+ const reconciler = (watcher as any).reconciler
+ mocks.serverFullReconcile.mockImplementation(async (servers, options) => {
+ await Promise.all(
+ servers.map((selected: { name: string }) =>
+ options.runEffect(selected.name, async () => {
+ eventLog.push('runtime')
+ await reconciler.reconcile(selected)
+ })
+ )
+ )
+ })
+ const netPol = (watcher as any).netPolReconciler
+ netPol.reconcileExternalEgress.mockImplementation(async () => {
+ eventLog.push('egress')
+ })
+ stubAuthoritativeInventoryWatch(watcher, 'McpServer')
+ stubAuthoritativeInventoryWatch(watcher, 'Context')
+ vi.spyOn(watcher as any, 'startSharedFileSystemWatch').mockImplementation(async () => undefined)
+ vi.spyOn(watcher as any, 'startGlobalFileSystemWatch').mockImplementation(async () => undefined)
+ vi.spyOn(watcher as any, 'startCommunicationChannelWatch').mockImplementation(
+ async () => undefined
+ )
+
+ await watcher.start()
+
+ await vi.waitFor(() => expect(eventLog).toContain('runtime'))
+ expect(eventLog.slice(0, 3)).toEqual(['defaults', 'egress', 'runtime'])
+ expect(mocks.serverFullReconcile).toHaveBeenCalledWith(
+ [expect.objectContaining({ name: 'web-search' })],
+ { runEffect: expect.any(Function) }
+ )
+
+ watcher.stop()
+ })
+
+ it('skips startup runtime reconciliation for servers whose external egress is not ready', async () => {
+ vi.useFakeTimers()
+ const server = {
+ metadata: { name: 'web-search', namespace: 'mcp-server', generation: 4 },
+ spec: {
+ contextRef: 'wf-research',
+ image: 'clerum/web-search:test',
+ transport: { type: 'streamableHttp' as const, port: 3000 },
+ egressBindings: [{ dns: 'duckduckgo.com', port: 443 }],
+ },
+ }
+
+ mocks.listNamespacedCustomObject.mockImplementation(async ({ plural }: { plural: string }) => {
+ if (plural === 'mcpservers') return { items: [server] }
+ if (plural === 'contexts') return { items: [] }
+ if (plural === 'hosts') return { items: [] }
+ if (plural === 'sharedfilesystems') return { items: [] }
+ if (plural === 'communicationchannels')
+ return { metadata: { resourceVersion: '1' }, items: [] }
+ return { items: [] }
+ })
+
+ const watcher = new McpServerWatcher()
+ const reconciler = (watcher as any).reconciler
+ mocks.serverFullReconcile.mockImplementation(async (servers, options) => {
+ await Promise.all(
+ servers.map((selected: { name: string }) =>
+ options.runEffect(selected.name, () => reconciler.reconcile(selected))
+ )
+ )
+ })
+ const netPol = (watcher as any).netPolReconciler
+ const dnsFailure = new Error('dns resolution failed')
+ netPol.reconcileExternalEgress.mockRejectedValueOnce(dnsFailure)
+ const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {})
+ const warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => {})
+ stubAuthoritativeInventoryWatch(watcher, 'McpServer')
+ stubAuthoritativeInventoryWatch(watcher, 'Context')
+ vi.spyOn(watcher as any, 'startSharedFileSystemWatch').mockImplementation(async () => undefined)
+ vi.spyOn(watcher as any, 'startGlobalFileSystemWatch').mockImplementation(async () => undefined)
+ vi.spyOn(watcher as any, 'startCommunicationChannelWatch').mockImplementation(
+ async () => undefined
+ )
+
+ await watcher.start()
+
+ await vi.waitFor(() => expect(mocks.serverFullReconcile).toHaveBeenCalledOnce())
+ expect(mocks.serverFullReconcile).toHaveBeenCalledWith(
+ [expect.objectContaining({ name: 'web-search' })],
+ { runEffect: expect.any(Function) }
+ )
+ expect(netPol.reconcileExternalEgress).toHaveBeenCalledTimes(1)
+ expect(reconciler.reconcile).not.toHaveBeenCalled()
+ expect(watcher.isReadinessInventoryAuthoritative()).toBe(true)
+ expect((watcher as any).initialConvergenceRetryTimers.has('NetworkPolicy')).toBe(false)
+ expect(errorSpy).not.toHaveBeenCalledWith(
+ '[K8s] Initial NetworkPolicy background reconciliation failed:',
+ dnsFailure
+ )
+
+ watcher.stop()
+ errorSpy.mockRestore()
+ warnSpy.mockRestore()
+ })
+ })
+
+ describe('getContext error semantics', () => {
+ beforeEach(() => {
+ vi.clearAllMocks()
+ })
+
+ it('preserves not-found semantics only for an authoritative 404', async () => {
+ mocks.getNamespacedCustomObject.mockRejectedValueOnce(
+ Object.assign(new Error('context absent'), { code: 404 })
+ )
+
+ await expect(getContext('missing-context')).resolves.toBeNull()
+ })
+
+ it('propagates non-404 API failures instead of converting discovery to an empty fleet', async () => {
+ const unavailable = Object.assign(new Error('apiserver unavailable'), {
+ response: { statusCode: 503 },
+ })
+ mocks.getNamespacedCustomObject.mockRejectedValueOnce(unavailable)
+
+ await expect(getContext('production-context')).rejects.toBe(unavailable)
+ })
+ })
+
++describe('MCP authorization Kubernetes not-found classification', () => {
++ it.each([{ code: 404 }, { response: { statusCode: 404 } }])(
++ 'recognizes Kubernetes 404 shape %#',
++ error => {
++ expect(isMcpAuthorizationNotFound(error)).toBe(true)
++ }
++ )
++
++ it('does not classify authority failures as not-found', () => {
++ expect(isMcpAuthorizationNotFound({ code: 403 })).toBe(false)
++ expect(isMcpAuthorizationNotFound({ response: { statusCode: 503 } })).toBe(false)
++ })
++})
++
+ describe('McpServerWatcher external egress retries', () => {
+ const serverObject = {
+ metadata: { name: 'redis-tools', namespace: 'mcp-server' },
+ spec: {
+ contextRef: 'context1',
+ image: 'redis-tools:test',
+ transport: { type: 'streamableHttp' as const, port: 3000 },
+ egressBindings: [{ dns: 'example.com', port: 443 }],
+ },
+ }
+
+ beforeEach(() => {
+ vi.clearAllMocks()
+ vi.useFakeTimers()
+ })
+
+ afterEach(() => {
+ vi.useRealTimers()
+ vi.restoreAllMocks()
+ })
+
+ it('retries failed external egress reconciliation from an ADDED event', async () => {
+ const watcher = new McpServerWatcher()
+ markMcpServerInventoryAuthoritative(watcher)
+ const netPol = (watcher as any).netPolReconciler
+ const reconciler = (watcher as any).reconciler
+ const bindingReconciler = (watcher as any).bindingReconciler
+ const serverWithBindings = {
+ ...serverObject,
+ metadata: {
+ ...serverObject.metadata,
+ annotations: {
+ 'clerum.io/recipe-bindings': JSON.stringify([
+ { source: 'redis-tools', target: 'downstream' },
+ ]),
+ },
+ },
+ }
+ ;(watcher as any).contexts.set('context1', {
+ name: 'context1',
+ namespace: 'mcp-server',
+ spec: { contextId: 'context1', mcpServers: ['redis-tools'] },
+ })
+ netPol.reconcileExternalEgress.mockRejectedValueOnce(new Error('api temporarily unavailable'))
+ netPol.reconcileExternalEgress.mockResolvedValue(undefined)
+ vi.spyOn(console, 'error').mockImplementation(() => {})
+ vi.spyOn(console, 'warn').mockImplementation(() => {})
+
+ const watchCallback = (watcher as any).getMcpServerWatchCallback()
+ await watchCallback('ADDED', serverWithBindings)
+
+ expect(netPol.reconcileExternalEgress).toHaveBeenCalledTimes(1)
+ expect(bindingReconciler.reconcileBindings).not.toHaveBeenCalled()
+ expect(netPol.reconcileContext).not.toHaveBeenCalled()
+
+ await vi.advanceTimersByTimeAsync(5000)
+
+ expect(netPol.reconcileExternalEgress).toHaveBeenCalledTimes(2)
+ expect(reconciler.reconcile).toHaveBeenCalledTimes(1)
+ expect(reconciler.reconcile).toHaveBeenCalledWith(
+ expect.objectContaining({ name: 'redis-tools' }),
+ { isCurrent: expect.any(Function) }
+ )
+ expect(bindingReconciler.reconcileBindings).toHaveBeenCalledTimes(1)
+ expect(netPol.reconcileContext).toHaveBeenCalledWith(
+ expect.objectContaining({
+ spec: expect.objectContaining({ contextId: 'context1' }),
+ }),
+ expect.objectContaining({ isCurrent: expect.any(Function) })
+ )
+
+ watcher.stop()
+ })
+
+ it('retires a binding-policy lease when a same-watch MODIFIED supersedes the server', async () => {
+ const watcher = new McpServerWatcher()
+ markMcpServerInventoryAuthoritative(watcher)
+ const bindingReconciler = (watcher as any).bindingReconciler
+ const bindingStarted = deferred()
+ const releaseBinding = deferred()
+ bindingReconciler.reconcileBindings.mockImplementationOnce(async () => {
+ bindingStarted.resolve(undefined)
+ await releaseBinding.promise
+ })
+ const annotations = {
+ 'clerum.io/recipe-bindings': JSON.stringify([
+ { from: 'redis-tools', to: 'downstream', port: 443 },
+ ]),
+ }
+ const original = {
+ ...serverObject,
+ metadata: {
+ ...serverObject.metadata,
+ uid: 'redis-tools-uid',
+ generation: 1,
+ annotations,
+ },
+ }
+ const replacement = {
+ ...original,
+ metadata: { ...original.metadata, generation: 2 },
+ spec: { ...original.spec, image: 'redis-tools:v2' },
+ }
+ const watchCallback = (watcher as any).getMcpServerWatchCallback()
+
+ const originalEffect = watchCallback('ADDED', original)
+ await bindingStarted.promise
+ const replacementEffect = watchCallback('MODIFIED', replacement)
+
+ const originalOptions = bindingReconciler.reconcileBindings.mock.calls[0][4]
+ expect(originalOptions).toEqual({ isCurrent: expect.any(Function) })
+ expect(originalOptions.isCurrent()).toBe(false)
+
+ releaseBinding.resolve(undefined)
+ await Promise.all([originalEffect, replacementEffect])
+
+ expect(bindingReconciler.reconcileBindings).toHaveBeenCalledTimes(2)
+ expect(bindingReconciler.reconcileBindings).toHaveBeenLastCalledWith(
+ 'redis-tools',
+ expect.any(Array),
+ 'redis-tools',
+ 'redis-tools',
+ { isCurrent: expect.any(Function) }
+ )
+ await watcher.stop()
+ })
+
+ it('retires binding cleanup when a same-name McpServer is recreated during DELETE', async () => {
+ const watcher = new McpServerWatcher()
+ ;(watcher as any).mcpServerCacheSynced = true
+ const bindingReconciler = (watcher as any).bindingReconciler
+ const cleanupStarted = deferred()
+ const releaseCleanup = deferred()
+ bindingReconciler.cleanupBindings.mockImplementationOnce(async () => {
+ cleanupStarted.resolve(undefined)
+ await releaseCleanup.promise
+ })
+ const deleted = {
+ ...serverObject,
+ metadata: {
+ ...serverObject.metadata,
+ uid: 'redis-tools-old-uid',
+ generation: 1,
+ },
+ }
+ const recreated = {
+ ...serverObject,
+ metadata: {
+ ...serverObject.metadata,
+ uid: 'redis-tools-new-uid',
+ generation: 1,
+ },
+ spec: { ...serverObject.spec, image: 'redis-tools:recreated' },
+ }
+ const watchCallback = (watcher as any).getMcpServerWatchCallback()
+
+ const deleteEffect = watchCallback('DELETED', deleted)
+ await cleanupStarted.promise
+ const recreateEffect = watchCallback('ADDED', recreated)
+
+ const cleanupOptions = bindingReconciler.cleanupBindings.mock.calls[0][1]
+ expect(cleanupOptions).toEqual({ deleteAllowed: expect.any(Function) })
+ await expect(cleanupOptions.deleteAllowed()).resolves.toBe(false)
+
+ releaseCleanup.resolve(undefined)
+ await Promise.all([deleteEffect, recreateEffect])
+
+ expect(bindingReconciler.cleanupBindings).toHaveBeenCalledTimes(1)
+ await watcher.stop()
+ })
+
+ it('retries the complete pipeline when a no-egress runtime reconcile fails', async () => {
+ const watcher = new McpServerWatcher()
+ markMcpServerInventoryAuthoritative(watcher)
+ const reconciler = (watcher as any).reconciler
+ const noEgressServer = {
+ ...serverObject,
+ spec: {
+ ...serverObject.spec,
+ egressBindings: undefined,
+ },
+ }
+ reconciler.reconcile
+ .mockRejectedValueOnce(new Error('deployment temporarily unavailable'))
+ .mockResolvedValue(undefined)
+ vi.spyOn(console, 'error').mockImplementation(() => {})
+ vi.spyOn(console, 'warn').mockImplementation(() => {})
+
+ await (watcher as any).getMcpServerWatchCallback()('ADDED', noEgressServer)
+ expect(reconciler.reconcile).toHaveBeenCalledTimes(1)
+
+ await vi.advanceTimersByTimeAsync(5000)
+
+ expect(reconciler.reconcile).toHaveBeenCalledTimes(2)
+ watcher.stop()
+ })
+
+ it('retries runtime reconciliation when external egress is ready but runtime reconcile fails', async () => {
+ const watcher = new McpServerWatcher()
+ markMcpServerInventoryAuthoritative(watcher)
+ const netPol = (watcher as any).netPolReconciler
+ const reconciler = (watcher as any).reconciler
+ netPol.reconcileExternalEgress.mockResolvedValue(undefined)
+ reconciler.reconcile.mockRejectedValueOnce(new Error('deployment temporarily unavailable'))
+ reconciler.reconcile.mockResolvedValue(undefined)
+ vi.spyOn(console, 'error').mockImplementation(() => {})
+ vi.spyOn(console, 'warn').mockImplementation(() => {})
+
+ const watchCallback = (watcher as any).getMcpServerWatchCallback()
+ await watchCallback('ADDED', serverObject)
+
+ expect(netPol.reconcileExternalEgress).toHaveBeenCalledTimes(1)
+ expect(reconciler.reconcile).toHaveBeenCalledTimes(1)
+
+ await vi.advanceTimersByTimeAsync(5000)
+
+ expect(netPol.reconcileExternalEgress).toHaveBeenCalledTimes(2)
+ expect(reconciler.reconcile).toHaveBeenCalledTimes(2)
+ expect(reconciler.reconcile).toHaveBeenLastCalledWith(
+ expect.objectContaining({ name: 'redis-tools' }),
+ { isCurrent: expect.any(Function) }
+ )
+
+ watcher.stop()
+ })
+
+ it('keeps retry attempts bounded when runtime fails after an external egress retry succeeds', async () => {
+ const watcher = new McpServerWatcher()
+ markMcpServerInventoryAuthoritative(watcher)
+ const netPol = (watcher as any).netPolReconciler
+ const reconciler = (watcher as any).reconciler
+ netPol.reconcileExternalEgress.mockRejectedValueOnce(new Error('api temporarily unavailable'))
+ netPol.reconcileExternalEgress.mockResolvedValue(undefined)
+ reconciler.reconcile.mockRejectedValueOnce(new Error('deployment temporarily unavailable'))
+ reconciler.reconcile.mockResolvedValue(undefined)
+ vi.spyOn(console, 'error').mockImplementation(() => {})
+ vi.spyOn(console, 'warn').mockImplementation(() => {})
+
+ const watchCallback = (watcher as any).getMcpServerWatchCallback()
+ await watchCallback('ADDED', serverObject)
+
+ expect(netPol.reconcileExternalEgress).toHaveBeenCalledTimes(1)
+ expect(reconciler.reconcile).not.toHaveBeenCalled()
+
+ await vi.advanceTimersByTimeAsync(5000)
+
+ expect(netPol.reconcileExternalEgress).toHaveBeenCalledTimes(2)
+ expect(reconciler.reconcile).toHaveBeenCalledTimes(1)
+
+ await vi.advanceTimersByTimeAsync(15000)
+
+ expect(netPol.reconcileExternalEgress).toHaveBeenCalledTimes(3)
+ expect(reconciler.reconcile).toHaveBeenCalledTimes(2)
+ expect(reconciler.reconcile).toHaveBeenLastCalledWith(
+ expect.objectContaining({ name: 'redis-tools' }),
+ { isCurrent: expect.any(Function) }
+ )
+
+ watcher.stop()
+ })
+
+ it('continues a pending runtime pipeline during periodic external egress resync', async () => {
+ const watcher = new McpServerWatcher()
+ markMcpServerInventoryAuthoritative(watcher)
+ const netPol = (watcher as any).netPolReconciler
+ const reconciler = (watcher as any).reconciler
+ netPol.reconcileExternalEgress.mockResolvedValue(undefined)
+ reconciler.reconcile.mockRejectedValueOnce(new Error('deployment temporarily unavailable'))
+ reconciler.reconcile.mockResolvedValue(undefined)
+ vi.spyOn(console, 'error').mockImplementation(() => {})
+ vi.spyOn(console, 'warn').mockImplementation(() => {})
+ vi.spyOn(console, 'log').mockImplementation(() => {})
+ vi.spyOn(Math, 'random').mockReturnValue(0)
+
+ const watchCallback = (watcher as any).getMcpServerWatchCallback()
+ await watchCallback('ADDED', serverObject)
+
+ expect(netPol.reconcileExternalEgress).toHaveBeenCalledTimes(1)
+ expect(reconciler.reconcile).toHaveBeenCalledTimes(1)
+
+ await (watcher as any).externalEgressCoordinator.runResync()
+
+ expect(netPol.reconcileExternalEgress).toHaveBeenCalledTimes(2)
+ expect(reconciler.reconcile).toHaveBeenCalledTimes(2)
+
+ await vi.advanceTimersByTimeAsync(5000)
+ expect(netPol.reconcileExternalEgress).toHaveBeenCalledTimes(2)
+ expect(reconciler.reconcile).toHaveBeenCalledTimes(2)
+
+ watcher.stop()
+ })
+
+ it('keeps retrying an ADDED pipeline at the capped delay until it recovers', async () => {
+ const watcher = new McpServerWatcher()
+ markMcpServerInventoryAuthoritative(watcher)
+ const netPol = (watcher as any).netPolReconciler
+ const reconciler = (watcher as any).reconciler
+ netPol.reconcileExternalEgress
+ .mockRejectedValueOnce(new Error('initial outage'))
+ .mockRejectedValueOnce(new Error('outage after 5s'))
+ .mockRejectedValueOnce(new Error('outage after 15s'))
+ .mockRejectedValueOnce(new Error('outage after 30s'))
+ .mockResolvedValue(undefined)
+ vi.spyOn(console, 'error').mockImplementation(() => {})
+ vi.spyOn(console, 'warn').mockImplementation(() => {})
+
+ await (watcher as any).getMcpServerWatchCallback()('ADDED', serverObject)
+ await vi.advanceTimersByTimeAsync(5000)
+ await vi.advanceTimersByTimeAsync(15000)
+ await vi.advanceTimersByTimeAsync(30000)
+
+ expect(netPol.reconcileExternalEgress).toHaveBeenCalledTimes(4)
+ expect(reconciler.reconcile).not.toHaveBeenCalled()
+
+ await vi.advanceTimersByTimeAsync(30000)
+
+ expect(netPol.reconcileExternalEgress).toHaveBeenCalledTimes(5)
+ expect(reconciler.reconcile).toHaveBeenCalledTimes(1)
+ watcher.stop()
+ })
+
+ it('keeps retrying a partial DELETE pipeline at the capped delay until it recovers', async () => {
+ const watcher = new McpServerWatcher()
+ ;(watcher as any).mcpServerCacheSynced = true
+ const netPol = (watcher as any).netPolReconciler
+ const reconciler = (watcher as any).reconciler
+ const bindingReconciler = (watcher as any).bindingReconciler
+ netPol.cleanupExternalEgress
+ .mockRejectedValueOnce(new Error('initial cleanup outage'))
+ .mockRejectedValueOnce(new Error('cleanup outage after 5s'))
+ .mockRejectedValueOnce(new Error('cleanup outage after 15s'))
+ .mockRejectedValueOnce(new Error('cleanup outage after 30s'))
+ .mockResolvedValue(undefined)
+ vi.spyOn(console, 'error').mockImplementation(() => {})
+ vi.spyOn(console, 'warn').mockImplementation(() => {})
+
+ await (watcher as any).getMcpServerWatchCallback()('DELETED', serverObject)
+ await vi.advanceTimersByTimeAsync(5000)
+ await vi.advanceTimersByTimeAsync(15000)
+ await vi.advanceTimersByTimeAsync(30000)
+
+ expect(netPol.cleanupExternalEgress).toHaveBeenCalledTimes(4)
+ expect(reconciler.reconcileDelete).not.toHaveBeenCalled()
+ expect(bindingReconciler.cleanupBindings).not.toHaveBeenCalled()
+
+ await vi.advanceTimersByTimeAsync(30000)
+
+ expect(netPol.cleanupExternalEgress).toHaveBeenCalledTimes(5)
+ expect(reconciler.reconcileDelete).toHaveBeenCalledWith('redis-tools', 'mcp-server')
+ expect(bindingReconciler.cleanupBindings).toHaveBeenCalledWith('redis-tools', {
+ deleteAllowed: expect.any(Function),
+ })
+ watcher.stop()
+ })
+
+ it('bounds concurrent durable retries across McpServers', async () => {
+ const watcher = new McpServerWatcher()
+ markMcpServerInventoryAuthoritative(watcher)
+ const reconciler = (watcher as any).reconciler
+ const release = deferred()
+ let active = 0
+ let maxActive = 0
+ reconciler.reconcile.mockImplementation(async () => {
+ active += 1
+ maxActive = Math.max(maxActive, active)
+ await release.promise
+ active -= 1
+ })
+ vi.spyOn(console, 'warn').mockImplementation(() => {})
+
+ for (let index = 0; index < 11; index += 1) {
+ const server = {
+ name: `server-${index}`,
+ namespace: 'mcp-server',
+ spec: {
+ ...serverObject.spec,
+ egressBindings: undefined,
+ },
+ }
+ ;(watcher as any).servers.set(server.name, server)
+ ;(watcher as any).scheduleExternalEgressRetry('ADDED', server)
+ }
+
+ await vi.advanceTimersByTimeAsync(5000)
+ await flushMicrotasks()
+
+ expect(maxActive).toBe(10)
+
+ release.resolve(undefined)
+ await flushMicrotasks()
+ watcher.stop()
+ })
+
+ it('replays a retry against the watcher generation current when the timer executes', async () => {
+ const watcher = new McpServerWatcher()
+ markMcpServerInventoryAuthoritative(watcher)
+ const selected = {
+ name: 'generation-fenced-server',
+ namespace: 'mcp-server',
+ spec: {
+ ...serverObject.spec,
+ egressBindings: undefined,
+ },
+ }
+ ;(watcher as any).servers.set(selected.name, selected)
+ ;(watcher as any).mcpWatchGeneration = 41
+ const replay = vi
+ .spyOn(watcher as any, 'reconcileMcpServerWatchEvent')
+ .mockResolvedValue(undefined)
+ vi.spyOn(console, 'warn').mockImplementation(() => {})
+ ;(watcher as any).scheduleExternalEgressRetry('ADDED', selected)
+ ;(watcher as any).mcpWatchGeneration = 42
+ await vi.advanceTimersByTimeAsync(5000)
+
+ expect(replay).toHaveBeenCalledWith(
+ 'MODIFIED',
+ expect.objectContaining({ name: selected.name }),
+ 42,
+ expect.objectContaining({
+ isCurrent: expect.any(Function),
+ complete: expect.any(Function),
+ })
+ )
+ watcher.stop()
+ })
+
+ it('reconciles external egress before runtime reconciliation on ADDED events', async () => {
+ const watcher = new McpServerWatcher()
+ const netPol = (watcher as any).netPolReconciler
+ const reconciler = (watcher as any).reconciler
+ const order: string[] = []
+ netPol.reconcileExternalEgress.mockImplementation(async () => {
+ order.push('egress')
+ })
+ reconciler.reconcile.mockImplementation(async () => {
+ order.push('runtime')
+ })
+
+ const watchCallback = (watcher as any).getMcpServerWatchCallback()
+ await watchCallback('ADDED', serverObject)
+
+ expect(order).toEqual(['egress', 'runtime'])
+
+ watcher.stop()
+ })
+
+ it('does not reconcile runtime resources when external egress pre-start reconciliation fails', async () => {
+ const watcher = new McpServerWatcher()
+ const netPol = (watcher as any).netPolReconciler
+ const reconciler = (watcher as any).reconciler
+ netPol.reconcileExternalEgress.mockRejectedValueOnce(new Error('api temporarily unavailable'))
+ vi.spyOn(console, 'error').mockImplementation(() => {})
+ vi.spyOn(console, 'warn').mockImplementation(() => {})
+
+ const watchCallback = (watcher as any).getMcpServerWatchCallback()
+ await watchCallback('ADDED', serverObject)
+
+ expect(reconciler.reconcile).not.toHaveBeenCalled()
+
+ watcher.stop()
+ })
+
+ it('retries failed external egress cleanup from a DELETED event', async () => {
+ const watcher = new McpServerWatcher()
+ ;(watcher as any).mcpServerCacheSynced = true
+ const netPol = (watcher as any).netPolReconciler
+ const reconciler = (watcher as any).reconciler
+ const bindingReconciler = (watcher as any).bindingReconciler
+ netPol.cleanupExternalEgress.mockRejectedValueOnce(new Error('delete temporarily unavailable'))
+ netPol.cleanupExternalEgress.mockResolvedValue(undefined)
+ vi.spyOn(console, 'error').mockImplementation(() => {})
+ vi.spyOn(console, 'warn').mockImplementation(() => {})
+
+ const watchCallback = (watcher as any).getMcpServerWatchCallback()
+ await watchCallback('DELETED', serverObject)
+
+ expect(netPol.cleanupExternalEgress).toHaveBeenCalledTimes(1)
+
+ await vi.advanceTimersByTimeAsync(5000)
+
+ expect(netPol.cleanupExternalEgress).toHaveBeenCalledTimes(2)
+ expect(netPol.cleanupExternalEgress).toHaveBeenLastCalledWith(
+ 'redis-tools',
+ 'mcp-server',
+ undefined,
+ expect.any(Function)
+ )
+ expect(reconciler.reconcileDelete).toHaveBeenCalledWith('redis-tools', 'mcp-server')
+ expect(bindingReconciler.cleanupBindings).toHaveBeenCalledWith('redis-tools', {
+ deleteAllowed: expect.any(Function),
+ })
+
+ watcher.stop()
+ })
+
+ it('retries the full DELETE pipeline when authoritative absence is temporarily unavailable', async () => {
+ const watcher = new McpServerWatcher()
+ ;(watcher as any).mcpServerCacheSynced = true
+ const netPol = (watcher as any).netPolReconciler
+ const reconciler = (watcher as any).reconciler
+ const bindingReconciler = (watcher as any).bindingReconciler
+ const unavailable = Object.assign(new Error('apiserver temporarily unavailable'), {
+ response: { statusCode: 503 },
+ })
+ mocks.getNamespacedCustomObject.mockRejectedValueOnce(unavailable)
+ vi.spyOn(console, 'error').mockImplementation(() => {})
+ vi.spyOn(console, 'warn').mockImplementation(() => {})
+
+ const watchCallback = (watcher as any).getMcpServerWatchCallback()
+ await expect(watchCallback('DELETED', serverObject)).resolves.toBeUndefined()
+
+ expect(mocks.getNamespacedCustomObject).toHaveBeenCalledTimes(1)
+ expect(netPol.cleanupExternalEgress).not.toHaveBeenCalled()
+ expect(reconciler.reconcileDelete).not.toHaveBeenCalled()
+ expect(bindingReconciler.cleanupBindings).not.toHaveBeenCalled()
+
+ await vi.advanceTimersByTimeAsync(5000)
+
+ expect(netPol.cleanupExternalEgress).toHaveBeenCalledTimes(1)
+ expect(reconciler.reconcileDelete).toHaveBeenCalledWith('redis-tools', 'mcp-server')
+ expect(bindingReconciler.cleanupBindings).toHaveBeenCalledWith('redis-tools', {
+ deleteAllowed: expect.any(Function),
+ })
+ expect(mocks.getNamespacedCustomObject).toHaveBeenCalledTimes(4)
+
+ watcher.stop()
+ })
+
+ it('retries the full DELETE pipeline when inventory authority is lost between cleanup stages', async () => {
+ const watcher = new McpServerWatcher()
+ ;(watcher as any).mcpServerCacheSynced = true
+ const netPol = (watcher as any).netPolReconciler
+ const reconciler = (watcher as any).reconciler
+ const bindingReconciler = (watcher as any).bindingReconciler
+ reconciler.reconcileDelete.mockImplementationOnce(async () => {
+ ;(watcher as any).mcpServerCacheSynced = false
+ })
+ vi.spyOn(console, 'warn').mockImplementation(() => {})
+
+ const watchCallback = (watcher as any).getMcpServerWatchCallback()
+ await expect(watchCallback('DELETED', serverObject)).resolves.toBeUndefined()
+
+ expect(netPol.cleanupExternalEgress).toHaveBeenCalledTimes(1)
+ expect(reconciler.reconcileDelete).toHaveBeenCalledWith('redis-tools', 'mcp-server')
+ expect(bindingReconciler.cleanupBindings).not.toHaveBeenCalled()
+
+ await vi.advanceTimersByTimeAsync(5000)
+
+ expect(netPol.cleanupExternalEgress).toHaveBeenCalledTimes(1)
+ expect(reconciler.reconcileDelete).toHaveBeenCalledTimes(1)
+ expect(bindingReconciler.cleanupBindings).not.toHaveBeenCalled()
+ ;(watcher as any).mcpServerCacheSynced = true
+ await vi.advanceTimersByTimeAsync(14_999)
+ expect(netPol.cleanupExternalEgress).toHaveBeenCalledTimes(1)
+ await vi.advanceTimersByTimeAsync(1)
+
+ expect(netPol.cleanupExternalEgress).toHaveBeenCalledTimes(2)
+ expect(reconciler.reconcileDelete).toHaveBeenCalledTimes(2)
+ expect(bindingReconciler.cleanupBindings).toHaveBeenCalledWith('redis-tools', {
+ deleteAllowed: expect.any(Function),
+ })
+
+ watcher.stop()
+ })
+
+ it('retires queued McpServer work immediately when its watch ends', async () => {
+ const blockerStarted = deferred()
+ const releaseBlocker = deferred()
+ let watchCallback: ((type: string, apiObj: typeof serverObject) => Promise<void>) | undefined
+ let watchDone: ((error: Error | null) => void) | undefined
+ mocks.watch.mockImplementationOnce(async (_path, _options, callback, done) => {
+ watchCallback = callback
+ watchDone = done
+ return { abort: vi.fn() }
+ })
+ const watcher = new McpServerWatcher()
+ const netPol = (watcher as any).netPolReconciler
+ await (watcher as any).startMcpServerWatch('mcp-watch-rv')
+ const blocker = (watcher as any).enqueueMcpServerReconciliation(
+ { name: 'redis-tools', namespace: 'mcp-server' },
+ async () => {
+ blockerStarted.resolve(undefined)
+ await releaseBlocker.promise
+ }
+ )
+ await blockerStarted.promise
+
+ const event = watchCallback!('ADDED', serverObject)
+ watchDone!(null)
+ releaseBlocker.resolve(undefined)
+ await Promise.all([blocker, event])
+
+ expect(netPol.reconcileExternalEgress).not.toHaveBeenCalled()
+ await watcher.stop()
+ })
+
+ it('retries the recreated server when an older DELETE retry timer already owns the key', async () => {
+ const deletedServer = {
+ metadata: {
+ name: 'redis-tools',
+ namespace: 'mcp-server',
+ uid: 'redis-tools-old-uid',
+ generation: 1,
+ },
+ spec: serverObject.spec,
+ }
+ const recreatedServer = {
+ metadata: {
+ name: 'redis-tools',
+ namespace: 'mcp-server',
+ uid: 'redis-tools-new-uid',
+ generation: 1,
+ },
+ spec: { ...serverObject.spec, image: 'redis-tools:recreated' },
+ }
+ const watcher = new McpServerWatcher()
+ ;(watcher as any).mcpServerCacheSynced = true
+ const netPol = (watcher as any).netPolReconciler
+ const reconciler = (watcher as any).reconciler
+ netPol.cleanupExternalEgress.mockRejectedValueOnce(new Error('cleanup unavailable'))
+ netPol.reconcileExternalEgress
+ .mockRejectedValueOnce(new Error('recreated policy unavailable'))
+ .mockResolvedValue(undefined)
+ vi.spyOn(console, 'error').mockImplementation(() => {})
+ vi.spyOn(console, 'warn').mockImplementation(() => {})
+ const watchCallback = (watcher as any).getMcpServerWatchCallback()
+
+ await watchCallback('DELETED', deletedServer)
+ await watchCallback('ADDED', recreatedServer)
+ await vi.advanceTimersByTimeAsync(5000)
+
+ expect(netPol.cleanupExternalEgress).toHaveBeenCalledTimes(1)
+ expect(netPol.reconcileExternalEgress).toHaveBeenCalledTimes(2)
+ expect(reconciler.reconcile).toHaveBeenCalledOnce()
+ expect(reconciler.reconcile).toHaveBeenCalledWith(
+ expect.objectContaining({
+ uid: 'redis-tools-new-uid',
+ generation: 1,
+ spec: expect.objectContaining({ image: 'redis-tools:recreated' }),
+ }),
+ { isCurrent: expect.any(Function) }
+ )
+
+ watcher.stop()
+ })
+
+ it('retries the latest DELETE cleanup when an older ADD retry timer already owns the key', async () => {
+ const server = {
+ metadata: {
+ name: 'redis-tools',
+ namespace: 'mcp-server',
+ uid: 'redis-tools-uid',
+ generation: 3,
+ },
+ spec: serverObject.spec,
+ }
+ const watcher = new McpServerWatcher()
+ ;(watcher as any).mcpServerCacheSynced = true
+ const netPol = (watcher as any).netPolReconciler
+ netPol.reconcileExternalEgress.mockRejectedValueOnce(new Error('policy unavailable'))
+ netPol.cleanupExternalEgress
+ .mockRejectedValueOnce(new Error('cleanup unavailable'))
+ .mockResolvedValue(undefined)
+ vi.spyOn(console, 'error').mockImplementation(() => {})
+ vi.spyOn(console, 'warn').mockImplementation(() => {})
+ const watchCallback = (watcher as any).getMcpServerWatchCallback()
+
+ await watchCallback('ADDED', server)
+ await watchCallback('DELETED', server)
+ await vi.advanceTimersByTimeAsync(5000)
+
+ expect(netPol.reconcileExternalEgress).toHaveBeenCalledTimes(1)
+ expect(netPol.cleanupExternalEgress).toHaveBeenCalledTimes(2)
+
+ watcher.stop()
+ })
+
+ it('does not continue to runtime or bindings after stop releases a live DNS hold', async () => {
+ const egressStarted = deferred()
+ const releaseEgress = deferred()
+ const server = {
+ metadata: {
+ name: 'redis-tools',
+ namespace: 'mcp-server',
+ uid: 'redis-tools-uid',
+ generation: 1,
+ annotations: {
+ 'clerum.io/recipe-bindings': JSON.stringify([
+ { source: 'redis-tools', target: 'downstream' },
+ ]),
+ },
+ },
+ spec: serverObject.spec,
+ }
+ const watcher = new McpServerWatcher()
+ const netPol = (watcher as any).netPolReconciler
+ const reconciler = (watcher as any).reconciler
+ const bindingReconciler = (watcher as any).bindingReconciler
+ netPol.reconcileExternalEgress.mockImplementationOnce(async () => {
+ egressStarted.resolve(undefined)
+ await releaseEgress.promise
+ })
+ const watchCallback = (watcher as any).getMcpServerWatchCallback()
+
+ const reconcile = watchCallback('ADDED', server)
+ await egressStarted.promise
+ await watcher.stop()
+ releaseEgress.resolve(undefined)
+ await reconcile
+
+ expect(reconciler.reconcile).not.toHaveBeenCalled()
+ expect(bindingReconciler.reconcileBindings).not.toHaveBeenCalled()
+ })
+
+ it('does not run an older runtime revision after newer desired state arrives during egress', async () => {
+ const egressStarted = deferred()
+ const releaseEgress = deferred()
+ const oldServer = {
+ metadata: {
+ name: 'redis-tools',
+ namespace: 'mcp-server',
+ uid: 'redis-tools-uid',
+ generation: 1,
+ },
+ spec: { ...serverObject.spec, image: 'redis-tools:old' },
+ }
+ const newServer = {
+ metadata: {
+ name: 'redis-tools',
+ namespace: 'mcp-server',
+ uid: 'redis-tools-uid',
+ generation: 2,
+ },
+ spec: { ...serverObject.spec, image: 'redis-tools:new' },
+ }
+ const watcher = new McpServerWatcher()
+ const netPol = (watcher as any).netPolReconciler
+ const reconciler = (watcher as any).reconciler
+ netPol.reconcileExternalEgress
+ .mockImplementationOnce(async () => {
+ egressStarted.resolve(undefined)
+ await releaseEgress.promise
+ })
+ .mockResolvedValue(undefined)
+ const watchCallback = (watcher as any).getMcpServerWatchCallback()
+
+ const oldReconcile = watchCallback('ADDED', oldServer)
+ await egressStarted.promise
+ const newReconcile = watchCallback('MODIFIED', newServer)
+ releaseEgress.resolve(undefined)
+ await Promise.all([oldReconcile, newReconcile])
+
+ expect(reconciler.reconcile).toHaveBeenCalledOnce()
+ expect(reconciler.reconcile).toHaveBeenCalledWith(
+ expect.objectContaining({
+ generation: 2,
+ spec: expect.objectContaining({ image: 'redis-tools:new' }),
+ }),
+ { isCurrent: expect.any(Function) }
+ )
+
+ watcher.stop()
+ })
+
+ it('does not resume periodic external egress work from jitter after stop', async () => {
+ const watcher = new McpServerWatcher()
+ const netPol = (watcher as any).netPolReconciler
+ ;(watcher as any).servers.set('redis-tools', {
+ name: 'redis-tools',
+ namespace: 'mcp-server',
+ spec: serverObject.spec,
+ })
+ vi.spyOn(Math, 'random').mockReturnValue(0.5)
+
+ const resync = (watcher as any).externalEgressCoordinator.runResync()
+ await flushMicrotasks()
+ await watcher.stop()
+ await vi.advanceTimersByTimeAsync(2500)
+ await resync
+
+ expect(netPol.reconcileExternalEgress).not.toHaveBeenCalled()
+ })
+
+ it('periodic resync reuses external egress reconciliation for cached servers with bindings', async () => {
+ const watcher = new McpServerWatcher()
+ markMcpServerInventoryAuthoritative(watcher)
+ const netPol = (watcher as any).netPolReconciler
+ netPol.reconcileExternalEgress.mockResolvedValue(undefined)
+ vi.spyOn(Math, 'random').mockReturnValue(0)
+ ;(watcher as any).servers.set('redis-tools', {
+ name: 'redis-tools',
+ namespace: 'mcp-server',
+ spec: serverObject.spec,
+ })
+
+ await (watcher as any).externalEgressCoordinator.runResync()
+
+ expect(netPol.reconcileExternalEgress).toHaveBeenCalledTimes(1)
+
+ watcher.stop()
+ })
+
+ it('does not run parallel external egress reconciles for the same McpServer', async () => {
+ const watcher = new McpServerWatcher()
+ const netPol = (watcher as any).netPolReconciler
+ const mutationStarted = deferred()
+ const releaseMutation = deferred()
+ const server = {
+ name: 'redis-tools',
+ namespace: 'mcp-server',
+ spec: serverObject.spec,
+ }
+ ;(watcher as any).servers.set('redis-tools', server)
+ netPol.reconcileExternalEgress.mockImplementationOnce(async () => {
+ mutationStarted.resolve(undefined)
+ await releaseMutation.promise
+ })
+ vi.spyOn(Math, 'random').mockReturnValue(0)
+
+ const active = (watcher as any).runExternalEgressOnce('MODIFIED', server)
+ await mutationStarted.promise
+ await (watcher as any).externalEgressCoordinator.runResync()
+
+ expect(netPol.reconcileExternalEgress).toHaveBeenCalledOnce()
+
+ releaseMutation.resolve(undefined)
+ await active
+ watcher.stop()
+ })
+
+ it('waits for already in-flight external egress before reconciling the current server', async () => {
+ const watcher = new McpServerWatcher()
+ const netPol = (watcher as any).netPolReconciler
+ const mutationStarted = deferred()
+ const releaseMutation = deferred()
+ const server = {
+ name: 'redis-tools',
+ namespace: 'mcp-server',
+ spec: serverObject.spec,
+ }
+ ;(watcher as any).servers.set(server.name, server)
+ netPol.reconcileExternalEgress.mockImplementationOnce(async () => {
+ mutationStarted.resolve(undefined)
+ await releaseMutation.promise
+ })
+ const warn = vi.spyOn(console, 'warn').mockImplementation(() => {})
+
+ const active = (watcher as any).runExternalEgressOnce('MODIFIED', server)
+ await mutationStarted.promise
+ const waiting = (watcher as any).runExternalEgressOnce('MODIFIED', server)
+ await flushMicrotasks()
+
+ expect(netPol.reconcileExternalEgress).toHaveBeenCalledOnce()
+ expect(warn).toHaveBeenCalledWith(
+ '[K8s] Waiting for external egress reconcile for mcp-server/redis-tools; already in flight'
+ )
+
+ releaseMutation.resolve(undefined)
+ await Promise.all([active, waiting])
+ expect(netPol.reconcileExternalEgress).toHaveBeenCalledTimes(2)
+
+ warn.mockRestore()
+ watcher.stop()
+ })
+
+ it('does not start another external egress mutation after stop releases an in-flight wait', async () => {
+ const watcher = new McpServerWatcher()
+ const netPol = (watcher as any).netPolReconciler
+ const mutationStarted = deferred()
+ const releaseMutation = deferred()
+ const server = {
+ name: 'redis-tools',
+ namespace: 'mcp-server',
+ spec: serverObject.spec,
+ }
+ ;(watcher as any).servers.set(server.name, server)
+ netPol.reconcileExternalEgress.mockImplementationOnce(async () => {
+ mutationStarted.resolve(undefined)
+ await releaseMutation.promise
+ })
+
+ const active = (watcher as any).runExternalEgressOnce('MODIFIED', server)
+ await mutationStarted.promise
+ const waiting = (watcher as any).runExternalEgressOnce('MODIFIED', server)
+ await flushMicrotasks()
+ await watcher.stop()
+ releaseMutation.resolve(undefined)
+ await Promise.all([active, waiting])
+
+ expect(netPol.reconcileExternalEgress).toHaveBeenCalledOnce()
+ })
+ })
+
+ describe('McpServerWatcher.countCommunicationChannels', () => {
+ it('returns 0 for an unknown host', () => {
+ const watcher = new McpServerWatcher()
+ expect(watcher.countCommunicationChannels('unknown-host')).toBe(0)
+ })
+
+ it('counts CCs whose spec.hostRef matches the given host', () => {
+ const watcher = new McpServerWatcher()
+ // Seed the private cache directly — the real watcher populates from K8s events.
+ const ccs = (watcher as any).communicationChannels as Map<string, { spec: { hostRef: string } }>
+ ccs.set('cc-1', { name: 'cc-1', namespace: 'channels', spec: { hostRef: 'marketing' } } as any)
+ ccs.set('cc-2', { name: 'cc-2', namespace: 'channels', spec: { hostRef: 'marketing' } } as any)
+ ccs.set('cc-3', { name: 'cc-3', namespace: 'channels', spec: { hostRef: 'trader' } } as any)
+ expect(watcher.countCommunicationChannels('marketing')).toBe(2)
+ expect(watcher.countCommunicationChannels('trader')).toBe(1)
+ expect(watcher.countCommunicationChannels('chatllm')).toBe(0)
+ })
+ })
+
+ describe('listAllCommunicationChannels', () => {
+ beforeEach(() => {
+ vi.clearAllMocks()
+ })
+
+ it('lists CommunicationChannel CRDs in the channels namespace', async () => {
+ const fixtureItems = [
+ {
+ metadata: { name: 'gcp-channel', namespace: 'channels' },
+ spec: { hostRef: 'josue-agent' },
+ },
+ {
+ metadata: { name: 'tgtestjose2', namespace: 'channels' },
+ spec: { hostRef: 'development' },
+ },
+ ]
+ mocks.listNamespacedCustomObject.mockResolvedValueOnce({ items: fixtureItems })
+
+ const result = await listAllCommunicationChannels()
+
+ expect(mocks.listNamespacedCustomObject).toHaveBeenCalledWith({
+ group: 'clerum.io',
+ version: 'v1alpha1',
+ namespace: 'channels',
+ plural: 'communicationchannels',
+ })
+ expect(result).toHaveLength(2)
+ expect(result[0]).toEqual({
+ name: 'gcp-channel',
+ namespace: 'channels',
+ spec: { hostRef: 'josue-agent' },
+ })
+ expect(result[1].spec.hostRef).toBe('development')
+ })
+ })
+
+ describe('McpServerWatcher wires CC counter into HostReconciler', () => {
+ it('HostReconciler.countCommunicationChannels reads from the watcher cache', () => {
+ const watcher = new McpServerWatcher()
+ const ccs = (watcher as any).communicationChannels as Map<string, { spec: { hostRef: string } }>
+ ccs.set('cc-1', { name: 'cc-1', namespace: 'channels', spec: { hostRef: 'alpha-host' } } as any)
+ ccs.set('cc-2', { name: 'cc-2', namespace: 'channels', spec: { hostRef: 'alpha-host' } } as any)
+ const hostReconciler = watcher.getHostReconciler()
+ // Same private field access pattern the hostReconciler tests use.
+ expect((hostReconciler as any).countCommunicationChannels('alpha-host')).toBe(2)
+ expect((hostReconciler as any).countCommunicationChannels('beta-host')).toBe(0)
+ })
+ })
+
+ describe('McpServerWatcher Host mutation dependency wiring', () => {
+ it('H2: wires a uid-guarded cache reflector into HostReconciler', () => {
+ const watcher = new McpServerWatcher()
+ const reflect = (
+ watcher.getHostReconciler() as unknown as {
+ _reflectHostOutcomeFn:
+ | ((name: string, uid: string | undefined, apply: (target: HostCRD) => void) => void)
+ | null
+ }
+ )._reflectHostOutcomeFn
+ if (!reflect) {
+ throw new Error('setReflectHostOutcome was never wired by the McpServerWatcher constructor')
+ }
+ const entry = {
+ name: 'h2-host',
+ namespace: 'mcp-host',
+ uid: 'uid-1',
+ spec: { host: 'h2-host', contextRef: 'c', secretRef: 's' },
+ } as unknown as HostCRD
+ ;(watcher as unknown as { hosts: Map<string, HostCRD> }).hosts.set('h2-host', entry)
+ const apply = vi.fn()
+ reflect('h2-host', 'uid-1', apply) // positive: current entry, matching uid
+ expect(apply).toHaveBeenCalledTimes(1)
+ expect(apply).toHaveBeenCalledWith(entry)
+ apply.mockClear()
+ reflect('h2-host', 'uid-2', apply) // recreation: same name, different uid → skip
+ expect(apply).not.toHaveBeenCalled()
+ reflect('h2-host', undefined, apply) // fail-closed on missing uid
+ expect(apply).not.toHaveBeenCalled()
+ reflect('absent-host', 'uid-1', apply) // absent entry → skip
+ expect(apply).not.toHaveBeenCalled()
+ })
+
+ it('tracks only the selected Context, referenced SFS state, and Host-scoped channels', () => {
+ const watcher = new McpServerWatcher()
+ const host = {
+ name: 'alpha-host',
+ namespace: 'mcp-host',
+ spec: { contextRef: 'alpha-context' },
+ } as HostCRD
+ const context = {
+ name: 'alpha-context',
+ namespace: 'mcp-server',
+ spec: {
+ contextId: 'alpha-context',
+ sharedFileSystems: [{ name: 'documents', mountPath: '/contexts/documents' }],
+ },
+ }
+ const documentsV1 = {
+ name: 'documents',
+ namespace: 'mcp-host',
+ spec: { storageClassName: 'fast-v1' },
+ }
+ const alphaChannelV1 = {
+ name: 'telegram-alpha',
+ namespace: 'channels',
+ spec: { hostRef: 'alpha-host' },
+ }
+ ;(watcher as any).contexts.set(context.name, context)
+ ;(watcher as any).sharedFileSystems.set(documentsV1.name, documentsV1)
+ ;(watcher as any).communicationChannels.set(alphaChannelV1.name, alphaChannelV1)
+ ;(watcher as any).ccCacheSynced = true
+ ;(watcher as any).ccWatchGeneration = 7
+ const sfsReconciler = (watcher as any).sharedFileSystemReconciler
+ sfsReconciler.isMountable.mockReturnValue(true)
+ const hostReconciler = watcher.getHostReconciler() as any
+ const resolveDependencies = hostReconciler._hostMutationDependenciesFn
+
+ const original = resolveDependencies(host)
+ expect(original).toEqual([
+ { name: context.name, namespace: context.namespace, spec: context.spec },
+ true,
+ 7,
+ { name: documentsV1.name, namespace: documentsV1.namespace, spec: documentsV1.spec },
+ true,
+ {
+ name: alphaChannelV1.name,
+ namespace: alphaChannelV1.namespace,
+ spec: alphaChannelV1.spec,
+ },
+ ])
+ ;(watcher as any).sharedFileSystems.set('unrelated', {
+ name: 'unrelated',
+ namespace: 'mcp-host',
+ spec: {},
+ })
+ ;(watcher as any).communicationChannels.set('telegram-beta', {
+ name: 'telegram-beta',
+ namespace: 'channels',
+ spec: { hostRef: 'beta-host' },
+ })
+ const afterUnrelatedChanges = resolveDependencies(host)
+ expect(afterUnrelatedChanges).toEqual(original)
+
+ const documentsV2 = {
+ ...documentsV1,
+ spec: { storageClassName: 'fast-v2' },
+ }
+ ;(watcher as any).sharedFileSystems.set(documentsV2.name, documentsV2)
+ const afterSfsRevision = resolveDependencies(host)
+ expect(afterSfsRevision[3]).toEqual({
+ name: documentsV2.name,
+ namespace: documentsV2.namespace,
+ spec: documentsV2.spec,
+ })
+ expect(afterSfsRevision[3]).not.toEqual(original[3])
+
+ sfsReconciler.isMountable.mockReturnValue(false)
+ const afterMountabilityChange = resolveDependencies(host)
+ expect(afterMountabilityChange[4]).toBe(false)
+
+ const alphaChannelV2 = {
+ ...alphaChannelV1,
+ spec: {
+ ...alphaChannelV1.spec,
+ credentialsSecretRef: { name: 'telegram-alpha-v2' },
+ },
+ }
+ ;(watcher as any).communicationChannels.set(alphaChannelV2.name, alphaChannelV2)
+ const afterChannelRevision = resolveDependencies(host)
+ expect(afterChannelRevision.at(-1)).toEqual({
+ name: alphaChannelV2.name,
+ namespace: alphaChannelV2.namespace,
+ spec: alphaChannelV2.spec,
+ })
+ expect(afterChannelRevision.at(-1)).not.toEqual(original.at(-1))
+ })
+ })
+
+ describe('McpServerWatcher.reconcileHostsReferencingCC', () => {
+ it('re-reconciles the Host whose name matches the CC hostRef', async () => {
+ const watcher = new McpServerWatcher()
+ markHostInventoryAuthoritative(watcher)
+ const hosts = (watcher as any).hosts as Map<string, { name: string; spec: any }>
+ hosts.set('marketing', {
+ name: 'marketing',
+ namespace: 'mcp-host',
+ spec: { host: 'marketing', contextRef: 'c', secretRef: 's' },
+ } as any)
+ const reconcileSpy = vi
+ .spyOn(watcher.getHostReconciler(), 'reconcile')
+ .mockResolvedValue(undefined)
+
+ await (watcher as any).reconcileHostsReferencingCC('marketing')
+
+ expect(reconcileSpy).toHaveBeenCalledTimes(1)
+ expect(reconcileSpy.mock.calls[0][0]).toMatchObject({ name: 'marketing' })
+ })
+
+ it('is a no-op when the hostRef does not match any cached Host', async () => {
+ const watcher = new McpServerWatcher()
+ markHostInventoryAuthoritative(watcher)
+ const reconcileSpy = vi
+ .spyOn(watcher.getHostReconciler(), 'reconcile')
+ .mockResolvedValue(undefined)
+
+ await (watcher as any).reconcileHostsReferencingCC('does-not-exist')
+
+ expect(reconcileSpy).not.toHaveBeenCalled()
+ })
+
+ it('reports reconcile errors without throwing so the watch can schedule recovery', async () => {
+ const watcher = new McpServerWatcher()
+ markHostInventoryAuthoritative(watcher)
+ const hosts = (watcher as any).hosts as Map<string, { name: string; spec: any }>
+ hosts.set('marketing', {
+ name: 'marketing',
+ namespace: 'mcp-host',
+ spec: { host: 'marketing', contextRef: 'c', secretRef: 's' },
+ } as any)
+ vi.spyOn(watcher.getHostReconciler(), 'reconcile').mockRejectedValue(new Error('boom'))
+ const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {})
+
+ await expect((watcher as any).reconcileHostsReferencingCC('marketing')).resolves.toBe(false)
+ expect(errorSpy).toHaveBeenCalled()
+
+ errorSpy.mockRestore()
+ })
+
+ it('schedules lifecycle convergence when a targeted CC event reconcile fails', async () => {
+ const watcher = new McpServerWatcher()
+ markHostInventoryAuthoritative(watcher)
+ ;(watcher as any).hosts.set('marketing', {
+ name: 'marketing',
+ namespace: 'mcp-host',
+ spec: { host: 'marketing', contextRef: 'c', secretRef: 's' },
+ })
+ vi.spyOn(watcher.getHostReconciler(), 'reconcile').mockRejectedValue(new Error('temporary'))
+ const requestFleet = vi
+ .spyOn(watcher as any, 'requestHostFleetReconcile')
+ .mockResolvedValue(undefined)
+ const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {})
+
+ const watchCallback = (watcher as any).getCommunicationChannelWatchCallback()
+ await watchCallback('ADDED', {
+ metadata: { name: 'marketing-channel', namespace: 'channels' },
+ spec: { hostRef: 'marketing' },
+ })
+
+ expect(requestFleet).toHaveBeenCalledWith(
+ 'CommunicationChannel event convergence fallback',
+ 1,
+ 'lifecycle'
+ )
+ errorSpy.mockRestore()
+ watcher.stop()
+ })
+ })
+
+ describe('McpServerWatcher.start ordering (#281 R6-bis)', () => {
+ beforeEach(() => {
+ vi.clearAllMocks()
+ mocks.watch.mockReset().mockResolvedValue({ abort: vi.fn() })
+ mocks.hostFullReconcile.mockReset().mockResolvedValue(undefined)
+ })
+
+ it('populates CC cache and starts CC watch BEFORE fullReconcile(initialHosts)', async () => {
+ const eventLog: string[] = []
+
+ // The existing mocking infrastructure routes all list calls through
+ // mocks.listNamespacedCustomObject keyed by `plural`. We intercept per-plural
+ // to record when each fires relative to fullReconcile.
+ mocks.listNamespacedCustomObject.mockImplementation(async ({ plural }: { plural: string }) => {
+ if (plural === 'communicationchannels') {
+ eventLog.push('listAllCommunicationChannels')
+ return {
+ metadata: { resourceVersion: '104' },
+ items: [
+ { metadata: { name: 'cc-1', namespace: 'channels' }, spec: { hostRef: 'marketing' } },
+ ],
+ }
+ }
+ if (plural === 'mcpservers') return { items: [] }
+ if (plural === 'contexts') return { items: [] }
+ if (plural === 'hosts') return { items: [] }
+ if (plural === 'sharedfilesystems') return { items: [] }
+ return { items: [] }
+ })
+
+ // Wrap hostFullReconcile to record when it fires.
+ mocks.hostFullReconcile.mockImplementation(async () => {
+ eventLog.push('fullReconcile')
+ })
+
+ mocks.watch.mockImplementation(async () => {
+ eventLog.push('startCommunicationChannelWatch')
+ return { abort: vi.fn() }
+ })
+ const watcher = new McpServerWatcher()
+
+ stubAuthoritativeInventoryWatch(watcher, 'McpServer')
+ stubAuthoritativeInventoryWatch(watcher, 'Context')
+ vi.spyOn(watcher as any, 'startSharedFileSystemWatch').mockResolvedValue(undefined)
+ vi.spyOn(watcher as any, 'startGlobalFileSystemWatch').mockResolvedValue(undefined)
+ const ccWatchSpy = vi.spyOn(watcher as any, 'startCommunicationChannelWatch')
+
+ await watcher.start()
+
+ // The critical ordering: the paired CC snapshot/watch must be established
+ // before any Host can make a stateless eligibility decision.
+ const idxList = eventLog.indexOf('listAllCommunicationChannels')
+ const idxWatch = eventLog.indexOf('startCommunicationChannelWatch')
+ const idxReconcile = eventLog.indexOf('fullReconcile')
+ expect(idxList).toBeGreaterThanOrEqual(0)
+ expect(idxWatch).toBeGreaterThanOrEqual(0)
+ expect(idxReconcile).toBeGreaterThanOrEqual(0)
+ expect(idxList).toBeLessThan(idxReconcile)
+ expect(idxWatch).toBeLessThan(idxReconcile)
+ expect(ccWatchSpy).toHaveBeenCalledWith('104')
+
+ // Cache populated by the initial LIST.
+ const cache = (watcher as any).communicationChannels as Map<string, any>
+ expect(cache.size).toBe(1)
+ expect(cache.get('cc-1')?.spec.hostRef).toBe('marketing')
+ expect(watcher.isCommunicationChannelCacheSynced()).toBe(true)
+ watcher.stop()
+ })
+
+ it('starts CC watch even when initial CC list fails (graceful degradation)', async () => {
+ mocks.listNamespacedCustomObject.mockImplementation(async ({ plural }: { plural: string }) => {
+ if (plural === 'communicationchannels') throw new Error('CC list failed')
+ if (plural === 'mcpservers') return { items: [] }
+ if (plural === 'contexts') return { items: [] }
+ if (plural === 'hosts') return { items: [] }
+ if (plural === 'sharedfilesystems') return { items: [] }
+ return { items: [] }
+ })
+
+ const watcher = new McpServerWatcher()
+
+ stubAuthoritativeInventoryWatch(watcher, 'McpServer')
+ stubAuthoritativeInventoryWatch(watcher, 'Context')
+ vi.spyOn(watcher as any, 'startSharedFileSystemWatch').mockResolvedValue(undefined)
+ vi.spyOn(watcher as any, 'startGlobalFileSystemWatch').mockResolvedValue(undefined)
+ const ccWatchSpy = vi
+ .spyOn(watcher as any, 'startCommunicationChannelWatch')
+ .mockResolvedValue(undefined)
+ const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {})
+
+ await expect(watcher.start()).resolves.toBeUndefined()
+ expect(ccWatchSpy).toHaveBeenCalledTimes(1)
+ expect(watcher.isCommunicationChannelCacheSynced()).toBe(false)
+ expect(errorSpy).toHaveBeenCalledWith(
+ '[K8s] CommunicationChannel initial load failed; ccCacheSynced remains false ' +
+ '(B2 preserves channel-reader replicas and holds stateless lifecycle active):',
+ expect.any(Error)
+ )
+
+ errorSpy.mockRestore()
+ watcher.stop()
+ })
+ })
+
+ describe('McpServerWatcher CommunicationChannel cache recovery', () => {
+ async function flushMicrotasks(): Promise<void> {
+ for (let i = 0; i < 8; i += 1) await Promise.resolve()
+ }
function communicationChannelWatchCalls(): typeof mocks.watch.mock.calls {
return mocks.watch.mock.calls.filter(([path]) =>
diff --cc host-context-controller/src/k8sClient.ts
index 0044519cf,d4ffc430b..c39b5c3d1
--- a/host-context-controller/src/k8sClient.ts
+++ b/host-context-controller/src/k8sClient.ts
@@@ -30,16 -40,20 +40,28 @@@ import
import { K8sGfsApi } from './k8s/gfsK8sApi'
import { makeHostK8sApiClient } from './k8s/hostK8sApiClient'
import { pvcName as sfsPvcName } from './k8s/sharedFileSystemFactory'
+import type {
+ AuthorityContext,
+ AuthorityHost,
+ AuthorityMcpServer,
+ AuthoritySecret,
+ AuthoritySecretMetadata,
+ McpAuthorizationStore,
+} from './mcpAuthorization'
- import { hostDeleteCleanupTotal, hostFleetRequestsTotal, hostWatchRecoverySeconds } from './metrics'
- import { NetworkPolicyReconciler } from './networkPolicyReconciler'
+ import { LlmHookReconciler, computePodKey, referencedHookIds } from './llmHookReconciler'
+ import {
+ confirmAuthoritativeMcpServerAbsence,
+ isMcpServerStatusOnlyUpdate,
+ sameMcpServerDesiredRevision,
+ } from './mcpServerSafety'
+ import {
+ hostDeleteCleanupTotal,
+ hostFleetRequestsTotal,
+ hostWatchRecoverySeconds,
+ initialConvergenceLastSuccessTimestampSeconds,
+ initialConvergenceRetriesTotal,
+ } from './metrics'
+ import { NetworkPolicyReconciler, sameContextDesiredRevision } from './networkPolicyReconciler'
import { McpServerReconciler } from './reconciler'
import { SharedFileSystemReconciler } from './sharedFileSystemReconciler'
import {
diff --cc host-context-controller/src/main.ts
index b898e0414,a1c168adf..d8cd848b5
--- a/host-context-controller/src/main.ts
+++ b/host-context-controller/src/main.ts
@@@ -19,8 -18,7 +19,9 @@@ import
createMcpServerProvider,
getKubeConfig,
} from './k8sClient'
+import { McpApiAuthenticator } from './mcpApiAuthentication'
+import { McpAuthorizationService } from './mcpAuthorization'
+ import { resolveHostAuthoritativeFn, resolveProviderAuthoritativeFn } from './readinessGate'
import { ContextMapperServer } from './server'
import { StatelessLifecycleTracker } from './statelessLifecycleTracker'
import {
@@@ -170,8 -163,8 +183,10 @@@ async function main(): Promise<void>
config.port,
hostReconciler,
hasDesktopFn,
+ providerAuthoritativeFn,
- hostAuthoritativeFn
++ hostAuthoritativeFn,
+ mcpAuthenticator,
+ mcpAuthorization
)
await server.start()
diff --cc host-context-controller/src/server.test.ts
index 87506e4f9,0ab7d03aa..300882d6f
--- a/host-context-controller/src/server.test.ts
+++ b/host-context-controller/src/server.test.ts
@@@ -80,91 -70,7 +80,93 @@@ async function invoke
return res
}
-describe('ContextMapperServer readiness', () => {
+const principal: VerifiedMcpHostPrincipal = {
+ subject: 'mcp-host/standalone',
+ hostName: 'host-a',
+ hostUid: 'host-uid-a',
+ namespace: 'mcp-host',
+ jti: 'jti-a',
+ issuedAt: 1,
+ expiresAt: 4_000_000_000,
+ audiences: ['host-context-controller'],
+}
+
+class FakeAuthorityStore implements McpAuthorizationStore {
+ secretValueReads = 0
+ private readonly host: AuthorityHost = {
+ name: 'host-a',
+ namespace: 'mcp-host',
+ metadata: { uid: 'host-uid-a', resourceVersion: '1' },
+ contextRef: 'context-a',
+ }
+ private readonly context: AuthorityContext = {
+ name: 'context-a',
+ namespace: 'mcp-server',
+ metadata: { uid: 'context-uid-a', resourceVersion: '2' },
+ mcpServers: ['server-a'],
+ }
+ private readonly mcpServer: AuthorityMcpServer = {
+ name: 'server-a',
+ namespace: 'mcp-server',
+ metadata: { uid: 'server-uid-a', resourceVersion: '3' },
+ transport: { type: 'streamableHttp', url: 'http://server-a/mcp' },
+ auth: { type: 'bearer', secretRef: 'server-a-auth', secretKey: 'token' },
+ enabled: true,
+ status: { deployed: true, ready: true },
+ }
+ private readonly secret: AuthoritySecret = {
+ name: 'server-a-auth',
+ namespace: 'mcp-server',
+ metadata: { uid: 'secret-uid-a', resourceVersion: '4' },
+ data: { token: Buffer.from('credential-value').toString('base64') },
+ }
+
+ async readHost() {
+ return this.host
+ }
+
+ async readContext() {
+ return this.context
+ }
+
+ async readMcpServer() {
+ return this.mcpServer
+ }
+
+ async readSecretMetadata() {
+ return {
+ name: this.secret.name,
+ namespace: this.secret.namespace,
+ metadata: this.secret.metadata,
+ }
+ }
+
+ async readSecret() {
+ this.secretValueReads += 1
+ return this.secret
+ }
+}
+
+function protectedServer(store = new FakeAuthorityStore()): {
+ server: ContextMapperServer
+ store: FakeAuthorityStore
+} {
+ const authenticator = { authenticate: () => principal } as unknown as McpApiAuthenticator
+ const server = new ContextMapperServer(
+ new FakeProvider(),
+ 0,
+ undefined,
+ undefined,
++ () => true,
++ () => true,
+ authenticator,
+ new McpAuthorizationService(store)
+ )
+ server.setReady(true)
+ return { server, store }
+}
+
+describe('ContextMapperServer', () => {
let server: ContextMapperServer | null = null
afterEach(async () => {
@@@ -184,215 -106,207 +202,401 @@@
response = await invoke(server, '/api/v1/mcpservers')
expect(response.statusCode).toBe(503)
- expect(JSON.parse(response.body)).toEqual({
- error: 'Service Unavailable',
- message: 'Context mapper is still starting',
- })
server.setReady(true)
+ response = await invoke(server, '/api/v1/mcpservers')
+ expect(response.statusCode).toBe(200)
+ expect(JSON.parse(response.body)).toMatchObject({ servers: [], contextRef: '*' })
+ })
- response = await invoke(server, '/ready')
+ it('keeps the temporary global v1 inventory metadata-only and non-cacheable', async () => {
+ server = new ContextMapperServer(
+ new FakeProvider([
+ {
+ name: 'server-a',
+ description: 'must not be exposed by the legacy system inventory',
+ contextRef: 'context-a',
+ transport: {
+ type: 'streamableHttp',
+ url: 'http://server-a/mcp',
+ secretRef: 'nested-secret-must-not-leak',
+ } as McpServerInfo['transport'],
+ auth: { type: 'bearer', secretRef: 'server-a-auth', secretKey: 'token' },
+ enabled: true,
+ status: { deployed: true, ready: true, message: 'internal status detail' },
+ },
+ ]),
+ 0,
+ undefined,
+ undefined,
+ () => true
+ )
+ server.setReady(true)
+
+ const response = await invoke(server, '/api/v1/mcpservers')
expect(response.statusCode).toBe(200)
- expect(JSON.parse(response.body)).toEqual({ status: 'ready', ready: true })
+ expect(response.headers['Cache-Control']).toBe('no-store, private')
+ expect(JSON.parse(response.body).servers[0]).not.toHaveProperty('auth')
+ expect(JSON.parse(response.body).servers[0]).not.toHaveProperty('description')
+ expect(response.body).not.toContain('secretRef')
+ expect(response.body).not.toContain('secretKey')
+ expect(response.body).not.toContain('nested-secret-must-not-leak')
+ expect(response.body).not.toContain('internal status detail')
+ })
- response = await invoke(server, '/api/v1/mcpservers')
+ it.each(['/api/v1/mcpservers/context/context-a', '/api/v1/mcpservers/server-a/auth'])(
+ 'returns an application-level 410 tombstone for legacy route %s',
+ async path => {
+ server = new ContextMapperServer(new FakeProvider(), 0)
+ server.setReady(true)
+ const response = await invoke(server, path)
+ expect(response.statusCode).toBe(410)
+ expect(JSON.parse(response.body)).toEqual({ error: 'gone' })
+ expect(response.headers['Cache-Control']).toBe('no-store, private')
+ }
+ )
+
+ it('tombstones legacy Host-selected routes for every method before readiness and CORS', async () => {
+ server = new ContextMapperServer(new FakeProvider(), 0, undefined, undefined, () => true)
+
+ for (const [path, method] of [
+ ['/api/v1/mcpservers/context/context-a', 'OPTIONS'],
+ ['/api/v1/mcpservers/server-a/auth', 'POST'],
+ ] as const) {
+ const response = await invoke(server, path, { method })
+ expect(response.statusCode).toBe(410)
+ expect(JSON.parse(response.body)).toEqual({ error: 'gone' })
+ expect(response.headers['Access-Control-Allow-Origin']).toBeUndefined()
+ expect(response.headers['Cache-Control']).toBe('no-store, private')
+ }
+ })
+
+ it('serves scoped v2 inventory without wildcard browser CORS or Secret values', async () => {
+ const protectedRoute = protectedServer()
+ server = protectedRoute.server
+ const response = await invoke(server, '/api/v2/hosts/self/mcpservers')
+ expect(response.statusCode).toBe(200)
+ expect(response.headers['Access-Control-Allow-Origin']).toBeUndefined()
+ expect(response.headers['Cache-Control']).toBe('no-store, private')
+ const body = JSON.parse(response.body)
+ expect(body.servers).toEqual([
+ expect.objectContaining({ name: 'server-a', authRequired: true }),
+ ])
+ expect(response.body).not.toContain('context-a')
+ expect(response.body).not.toContain('server-a-auth')
+ expect(response.body).not.toContain('credential-value')
+ expect(protectedRoute.store.secretValueReads).toBe(0)
+ })
+
+ it('keeps direct protected-route startup and not-found errors generic and non-cacheable', async () => {
+ server = new ContextMapperServer(new FakeProvider(), 0, undefined, undefined, () => true)
+
+ let response = await invoke(server, '/api/v2/hosts/self/mcpservers')
+ expect(response.statusCode).toBe(503)
+ expect(JSON.parse(response.body)).toEqual({ error: 'authorization_unavailable' })
+ expect(response.headers['Cache-Control']).toBe('no-store, private')
+
+ server.setReady(true)
+ response = await invoke(server, '/api/v2/hosts/self/mcpservers/unknown')
+ expect(response.statusCode).toBe(404)
+ expect(JSON.parse(response.body)).toEqual({ error: 'not_found' })
+ expect(response.headers['Cache-Control']).toBe('no-store, private')
+ })
+
+ it('accepts only the exact credential body and returns the fenced token DTO', async () => {
+ const protectedRoute = protectedServer()
+ server = protectedRoute.server
+ const response = await invoke(server, '/api/v2/hosts/self/mcpservers/credential', {
+ method: 'POST',
+ headers: { 'content-type': 'application/json' },
+ body: JSON.stringify({ serverName: 'server-a' }),
+ })
expect(response.statusCode).toBe(200)
- expect(JSON.parse(response.body)).toMatchObject({
- servers: [],
- contextRef: '*',
+ expect(JSON.parse(response.body)).toEqual({
+ token: 'credential-value',
+ credentialRevision: expect.any(String),
+ })
+ expect(protectedRoute.store.secretValueReads).toBe(1)
+
+ const invalid = await invoke(server, '/api/v2/hosts/self/mcpservers/credential', {
+ method: 'POST',
+ headers: { 'content-type': 'application/json' },
+ body: JSON.stringify({ serverName: 'server-a', contextRef: 'context-b' }),
+ })
+ expect(invalid.statusCode).toBe(400)
+ expect(protectedRoute.store.secretValueReads).toBe(1)
+ })
+
+ it('normalizes protected route method, media-type, and body-limit failures without a Secret read', async () => {
+ const protectedRoute = protectedServer()
+ server = protectedRoute.server
+
+ let response = await invoke(server, '/api/v2/hosts/self/mcpservers', { method: 'POST' })
+ expect(response.statusCode).toBe(405)
+ expect(JSON.parse(response.body)).toEqual({ error: 'method_not_allowed' })
+ expect(response.headers.Allow).toBe('GET')
+
+ response = await invoke(server, '/api/v2/hosts/self/mcpservers/credential', {
+ method: 'POST',
+ headers: { 'content-type': 'text/plain' },
+ body: '{}',
+ })
+ expect(response.statusCode).toBe(415)
+ expect(JSON.parse(response.body)).toEqual({ error: 'unsupported_media_type' })
+
+ response = await invoke(server, '/api/v2/hosts/self/mcpservers/credential', {
+ method: 'POST',
+ headers: { 'content-type': 'application/json' },
+ body: JSON.stringify({ serverName: 'a'.repeat(1_025) }),
+ })
+ expect(response.statusCode).toBe(413)
+ expect(JSON.parse(response.body)).toEqual({ error: 'payload_too_large' })
+ expect(response.headers['Cache-Control']).toBe('no-store, private')
+ expect(protectedRoute.store.secretValueReads).toBe(0)
+ })
+
+ it('fails closed when the protected Host verifier is not configured', async () => {
+ server = new ContextMapperServer(new FakeProvider(), 0)
+ server.setReady(true)
+
+ const response = await invoke(server, '/api/v2/hosts/self/mcpservers')
+ expect(response.statusCode).toBe(503)
+ expect(JSON.parse(response.body)).toEqual({ error: 'authorization_unavailable' })
+ expect(response.headers['Cache-Control']).toBe('no-store, private')
+ })
+
+ it('maps a rebound Host identity to a generic non-cacheable 401', async () => {
+ const authenticator = { authenticate: () => principal } as unknown as McpApiAuthenticator
+ const authorization = {
+ listServers: async () => {
+ throw new McpAuthorizationError('unauthorized')
+ },
+ } as unknown as McpAuthorizationService
+ server = new ContextMapperServer(
+ new FakeProvider(),
+ 0,
+ undefined,
+ undefined,
++ () => true,
++ () => true,
+ authenticator,
+ authorization
+ )
+ server.setReady(true)
+
+ const response = await invoke(server, '/api/v2/hosts/self/mcpservers')
+ expect(response.statusCode).toBe(401)
+ expect(JSON.parse(response.body)).toEqual({ error: 'unauthorized' })
+ expect(response.headers['WWW-Authenticate']).toBe('Bearer realm="host-context-controller"')
+ expect(response.headers['Cache-Control']).toBe('no-store, private')
+ expect(response.headers.Pragma).toBe('no-cache')
+ })
+})
+
+describe('McpHostApiRateLimiter', () => {
+ let server: ContextMapperServer | null = null
+
+ afterEach(async () => {
+ await server?.stop()
+ server = null
+ vi.restoreAllMocks()
+ })
+
+ it('limits each signed Host identity/action independently and resets after one minute', () => {
+ let now = 1_000
+ const limiter = new McpHostApiRateLimiter(1, () => now)
+ expect(limiter.consume('host-uid-a', 'credential')).toEqual({ allowed: true })
+ expect(limiter.consume('host-uid-a', 'inventory')).toEqual({ allowed: true })
+ expect(limiter.consume('host-uid-b', 'credential')).toEqual({ allowed: true })
+ expect(limiter.consume('host-uid-a', 'credential')).toEqual({
+ allowed: false,
+ retryAfterSeconds: 60,
})
+ now += 60_000
+ expect(limiter.consume('host-uid-a', 'credential')).toEqual({ allowed: true })
})
+
+ it('tracks provider authority dynamically after warm-up and fails API requests closed', async () => {
+ let providerAuthoritative = true
+ server = new ContextMapperServer(
+ new FakeProvider(),
+ 0,
+ undefined,
+ undefined,
+ () => providerAuthoritative
+ )
+
+ let response = await invoke(server, '/ready')
+ expect(response.statusCode).toBe(503)
+ expect(JSON.parse(response.body)).toEqual({ status: 'starting', ready: false })
+
+ server.setReady(true)
+
+ response = await invoke(server, '/ready')
+ expect(response.statusCode).toBe(200)
+ expect(JSON.parse(response.body)).toEqual({ status: 'ready', ready: true })
+
+ providerAuthoritative = false
+
+ response = await invoke(server, '/ready')
+ expect(response.statusCode).toBe(503)
+ expect(JSON.parse(response.body)).toEqual({ status: 'degraded', ready: false })
+
+ response = await invoke(server, '/health')
+ expect(response.statusCode).toBe(200)
+ expect(JSON.parse(response.body)).toEqual({
+ status: 'ok',
+ ready: false,
+ })
+
+ response = await invoke(server, '/api/v1/mcpservers')
+ expect(response.statusCode).toBe(503)
+ expect(JSON.parse(response.body)).toEqual({
+ error: 'Service Unavailable',
+ message: 'Context mapper provider inventory is not authoritative',
+ })
+
+ providerAuthoritative = true
+
+ response = await invoke(server, '/ready')
+ expect(response.statusCode).toBe(200)
+ expect(JSON.parse(response.body)).toEqual({ status: 'ready', ready: true })
+
+ response = await invoke(server, '/api/v1/mcpservers')
+ expect(response.statusCode).toBe(200)
+ })
+
+ it('fails readiness closed when the provider authority check throws', async () => {
+ const authorityError = new Error('authority unavailable')
+ const errorLog = vi.spyOn(console, 'error').mockImplementation(() => undefined)
+ server = new ContextMapperServer(new FakeProvider(), 0, undefined, undefined, () => {
+ throw authorityError
+ })
+ server.setReady(true)
+
+ const response = await invoke(server, '/ready')
+
+ expect(response.statusCode).toBe(503)
+ expect(JSON.parse(response.body)).toEqual({ status: 'degraded', ready: false })
+ expect(errorLog).toHaveBeenCalledWith(
+ '[Server] Provider authority readiness check failed:',
+ authorityError
+ )
+ })
+
+ it('withdraws readiness as soon as shutdown begins', async () => {
+ server = new ContextMapperServer(new FakeProvider(), 0, undefined, undefined, () => true)
+ server.setReady(true)
+
+ expect((await invoke(server, '/ready')).statusCode).toBe(200)
+
+ await server.stop()
+
+ const response = await invoke(server, '/ready')
+ expect(response.statusCode).toBe(503)
+ expect(JSON.parse(response.body)).toEqual({ status: 'starting', ready: false })
+ })
+ })
+
+ describe('ContextMapperServer desktop status authority', () => {
+ let server: ContextMapperServer | null = null
+
+ // A Host reconciler that reports a running desktop, so the only thing that can
+ // withhold a 200 is the Host-authority gate under test.
+ const runningReconciler = {
+ getStatus: () => ({ deployed: true, ready: true, message: '' }),
+ hasDesktop: () => true,
+ } as unknown as ConstructorParameters<typeof ContextMapperServer>[2]
+ const hasDesktopFn = () => true
+
+ afterEach(async () => {
+ await server?.stop()
+ server = null
+ vi.restoreAllMocks()
+ })
+
+ it('serves desktop status on Host authority alone even when the provider inventory is not authoritative', async () => {
+ // The heart of H1: desktop needs only Host authority. A degraded
+ // McpServer/Context lane (providerAuthoritative=false) must NOT 503 desktop,
+ // while /api/v1/mcpservers still 503s on the same server.
+ server = new ContextMapperServer(
+ new FakeProvider(),
+ 0,
+ runningReconciler,
+ hasDesktopFn,
+ () => false, // provider inventory NOT authoritative
+ () => true // Host inventory authoritative
+ )
+ server.setReady(true)
+
+ const desktop = await invoke(server, '/api/v1/desktop/host-a')
+ expect(desktop.statusCode).toBe(200)
+ expect(JSON.parse(desktop.body)).toEqual({ status: 'running', hostRef: 'host-a' })
+
+ const mcpservers = await invoke(server, '/api/v1/mcpservers')
+ expect(mcpservers.statusCode).toBe(503)
+ })
+
+ it('fails desktop closed (503, never 200 inactive) when Host inventory is not authoritative', async () => {
+ server = new ContextMapperServer(
+ new FakeProvider(),
+ 0,
+ runningReconciler,
+ hasDesktopFn,
+ () => true, // provider authoritative — must not rescue desktop
+ () => false // Host inventory NOT authoritative
+ )
+ server.setReady(true)
+
+ const desktop = await invoke(server, '/api/v1/desktop/host-a')
+ expect(desktop.statusCode).toBe(503)
+ expect(JSON.parse(desktop.body)).toEqual({
+ error: 'Service Unavailable',
+ message: 'Host inventory is not authoritative',
+ })
+ })
+
+ it('serves desktop status 200 in the dev wiring (no watcher), never 503 (R2-M1)', async () => {
+ // Wiring regression for R2-M1: main.ts resolves both authority gates from the
+ // same watcher. In dev (DevMcpServerProvider, watcher=null) the OLD wiring left
+ // the Host gate undefined, so the server's fail-closed default pinned every
+ // /api/v1/desktop/* at 503 after setReady(true) — where the base answered 200
+ // {status:'inactive'}. Exercise the REAL resolvers with a null watcher, exactly
+ // as main.ts does, so a revert to the undefined wiring turns this 200 into 503.
+ const providerAuthoritativeFn = resolveProviderAuthoritativeFn(null)
+ const hostAuthoritativeFn = resolveHostAuthoritativeFn(null)
+ server = new ContextMapperServer(
+ new FakeProvider(),
+ 0,
+ runningReconciler,
+ hasDesktopFn,
+ providerAuthoritativeFn,
+ hostAuthoritativeFn
+ )
+ server.setReady(true)
+
+ const desktop = await invoke(server, '/api/v1/desktop/host-a')
+ expect(desktop.statusCode).toBe(200)
+ expect(JSON.parse(desktop.body)).toEqual({ status: 'running', hostRef: 'host-a' })
+ })
+
+ it('fails desktop closed when no Host-authority gate is wired (default)', async () => {
+ // Omitting the 6th arg must default to fail-closed, exactly like the
+ // provider gate — no type error catches the omission.
+ server = new ContextMapperServer(
+ new FakeProvider(),
+ 0,
+ runningReconciler,
+ hasDesktopFn,
+ () => true
+ )
+ server.setReady(true)
+
+ const desktop = await invoke(server, '/api/v1/desktop/host-a')
+ expect(desktop.statusCode).toBe(503)
+ expect(JSON.parse(desktop.body)).toEqual({
+ error: 'Service Unavailable',
+ message: 'Host inventory is not authoritative',
+ })
+ })
})
diff --cc host-context-controller/src/server.ts
index 6fa2393d7,03f576dcf..00b5df722
--- a/host-context-controller/src/server.ts
+++ b/host-context-controller/src/server.ts
@@@ -5,83 -5,21 +5,89 @@@ import * as http from 'http
import { config } from './config'
import { HostReconciler } from './hostReconciler'
import { McpServerProvider } from './k8sClient'
-import { registry } from './metrics'
-import { ErrorResponse, McpServersResponse } from './types'
+import { hccLogger } from './logger'
+import {
+ McpApiAuthenticationError,
+ McpApiAuthenticator,
+ type VerifiedMcpHostPrincipal,
+} from './mcpApiAuthentication'
+import {
+ McpAuthorizationError,
+ McpAuthorizationService,
+ toPublicMcpTransport,
+} from './mcpAuthorization'
+import { mcpHostApiRequestsTotal, registry } from './metrics'
+import { McpServersResponse } from './types'
+
+const MCP_SELECTOR_RE = /^[a-z0-9]([-a-z0-9.]*[a-z0-9])?$/
+const MCP_REQUEST_BODY_LIMIT = 1024
+const mcpApiLog = hccLogger.child({ module: 'mcp-host-api' })
+type McpHostApiAction = 'inventory' | 'credential'
+
+type RateLimitEntry = { windowStartMs: number; count: number }
+
+/** Per-Host, per-action fixed-window limiter applied only after JWT validation. */
+export class McpHostApiRateLimiter {
+ private readonly entries = new Map<string, RateLimitEntry>()
+
+ constructor(
+ private readonly maxPerMinute: number,
+ private readonly now: () => number = Date.now
+ ) {}
+
+ consume(
+ hostUid: string,
+ action: McpHostApiAction
+ ): { allowed: true } | { allowed: false; retryAfterSeconds: number } {
+ const nowMs = this.now()
+ const key = `${hostUid}\u0000${action}`
+ const existing = this.entries.get(key)
+ if (!existing || nowMs - existing.windowStartMs >= 60_000) {
+ this.entries.set(key, { windowStartMs: nowMs, count: 1 })
+ this.prune(nowMs)
+ return { allowed: true }
+ }
+ if (existing.count >= this.maxPerMinute) {
+ return {
+ allowed: false,
+ retryAfterSeconds: Math.max(
+ 1,
+ Math.ceil((60_000 - (nowMs - existing.windowStartMs)) / 1000)
+ ),
+ }
+ }
+ existing.count += 1
+ return { allowed: true }
+ }
+
+ private prune(nowMs: number): void {
+ if (this.entries.size <= 10_000) return
+ for (const [key, entry] of this.entries) {
+ if (nowMs - entry.windowStartMs >= 60_000) this.entries.delete(key)
+ }
+ while (this.entries.size > 10_000) {
+ const oldest = this.entries.keys().next()
+ if (oldest.done) break
+ this.entries.delete(oldest.value)
+ }
+ }
+}
+ type ReadinessState =
+ | { ready: true; status: 'ready' }
+ | { ready: false; status: 'starting' | 'degraded'; message: string }
+
export class ContextMapperServer {
private server: http.Server | null = null
private provider: McpServerProvider
private port: number
private hostReconciler: HostReconciler | null
private hasDesktopFn: ((hostRef: string) => boolean) | null
+ private mcpAuthenticator: McpApiAuthenticator | null
+ private mcpAuthorization: McpAuthorizationService | null
+ private mcpRateLimiter: McpHostApiRateLimiter
+ private providerAuthoritativeFn: () => boolean
+ private hostAuthoritativeFn: () => boolean
private ready = false
constructor(
@@@ -89,16 -27,34 +95,39 @@@
port: number = config.port,
hostReconciler?: HostReconciler,
hasDesktopFn?: (hostRef: string) => boolean,
+ // Fails closed. /ready is the assertion that no stale allow is live, and a
+ // caller that omits this gate gets no type error, so the default must
+ // withhold readiness rather than grant it.
+ providerAuthoritativeFn: () => boolean = () => false,
+ // Desktop status needs ONLY Host inventory authority, not the full
+ // readiness inventory. Fails closed like providerAuthoritativeFn: a caller
+ // that omits it gets a 503 on desktop rather than a wrong 200 'inactive'.
- hostAuthoritativeFn: () => boolean = () => false
++ hostAuthoritativeFn: () => boolean = () => false,
+ mcpAuthenticator?: McpApiAuthenticator,
+ mcpAuthorization?: McpAuthorizationService
) {
this.provider = provider
this.port = port
this.hostReconciler = hostReconciler ?? null
this.hasDesktopFn = hasDesktopFn ?? null
+ this.mcpAuthenticator = mcpAuthenticator ?? null
+ this.mcpAuthorization = mcpAuthorization ?? null
+ this.mcpRateLimiter = new McpHostApiRateLimiter(config.mcpHostApiRateLimitPerMinute)
+ this.providerAuthoritativeFn = providerAuthoritativeFn
+ this.hostAuthoritativeFn = hostAuthoritativeFn
+ }
+
+ /**
+ * Host inventory authority, fail-closed on error — the desktop route's only
+ * authority gate. Same shape as the provider check in getReadinessState().
+ */
+ private safeHostAuthoritative(): boolean {
+ try {
+ return this.hostAuthoritativeFn()
+ } catch (err) {
+ console.error('[Server] Host authority check failed:', err)
+ return false
+ }
}
/**
@@@ -108,30 -64,35 +137,54 @@@
this.ready = ready
}
+ private getReadinessState(): ReadinessState {
+ if (!this.ready) {
+ return {
+ ready: false,
+ status: 'starting',
+ message: 'Context mapper is still starting',
+ }
+ }
+
+ try {
+ if (this.providerAuthoritativeFn()) {
+ return { ready: true, status: 'ready' }
+ }
+ } catch (err) {
+ console.error('[Server] Provider authority readiness check failed:', err)
+ }
+
+ return {
+ ready: false,
+ status: 'degraded',
+ message: 'Context mapper provider inventory is not authoritative',
+ }
+ }
+
private async handleRequest(req: http.IncomingMessage, res: http.ServerResponse): Promise<void> {
- // CORS headers
- res.setHeader('Access-Control-Allow-Origin', '*')
- res.setHeader('Access-Control-Allow-Methods', 'GET, OPTIONS')
- res.setHeader('Access-Control-Allow-Headers', 'Content-Type')
+ const url = new URL(req.url || '/', `http://localhost:${this.port}`)
+ const isLegacyHostSelectedMcpRoute =
+ url.pathname.startsWith('/api/v1/mcpservers/context/') ||
+ (url.pathname.startsWith('/api/v1/mcpservers/') && url.pathname.endsWith('/auth'))
+ const isProtectedMcpRoute =
+ url.pathname.startsWith('/api/v2/hosts/self/mcpservers') || isLegacyHostSelectedMcpRoute
+
+ // The Host credential surface is service-to-service and deliberately has
+ // no browser wildcard CORS grant. Preserve the historical public CORS
+ // behavior only for unrelated/legacy endpoints.
+ if (!isProtectedMcpRoute) {
+ res.setHeader('Access-Control-Allow-Origin', '*')
+ res.setHeader('Access-Control-Allow-Methods', 'GET, OPTIONS')
+ res.setHeader('Access-Control-Allow-Headers', 'Content-Type')
+ }
+
+ // Retire caller-selected Context/credential routes at the application
+ // boundary for every method, including preflight, and independently of
+ // readiness. This remains a defense if a caller bypasses the gateway.
+ if (isLegacyHostSelectedMcpRoute) {
+ this.sendProtectedJson(res, 410, { error: 'gone' })
+ return
+ }
if (req.method === 'OPTIONS') {
res.writeHead(204)
@@@ -168,14 -136,30 +226,37 @@@
return
}
- if (!this.ready) {
+ // Desktop status: Host inventory is the ONLY authority this route needs, so
+ // it is gated here — ABOVE the blanket readiness gate — on Host authority
+ // alone. A degraded McpServer/Context lane must NOT 503 desktop, and a
+ // degraded Host lane must 503 it (never answer 200 'inactive' from a stale
+ // cache). See the E3 design note's endpoint→authority matrix.
+ if (req.method === 'GET' && url.pathname.startsWith('/api/v1/desktop/')) {
+ if (!this.ready || !this.safeHostAuthoritative()) {
+ this.sendJson(res, 503, {
+ error: 'Service Unavailable',
+ message: 'Host inventory is not authoritative',
+ })
+ return
+ }
+ if (!this.checkDesktopAuth(req, res)) return
+ const hostRef = url.pathname.replace('/api/v1/desktop/', '')
+ await this.handleDesktopStatus(res, hostRef)
+ return
+ }
+
+ const readiness = this.getReadinessState()
+ if (!readiness.ready) {
++ // NP-08: protected MCP routes get an opaque, uniform authorization error
++ // instead of the generic readiness message, preserving the caller-binding
++ // contract even while the authority inventory is still warming up.
+ if (isProtectedMcpRoute) {
+ this.sendProtectedJson(res, 503, { error: 'authorization_unavailable' })
+ return
+ }
this.sendJson(res, 503, {
error: 'Service Unavailable',
- message: 'Context mapper is still starting',
+ message: readiness.message,
})
return
}
@@@ -204,19 -188,7 +285,16 @@@
return
}
- // Desktop status (requires service token)
- if (req.method === 'GET' && url.pathname.startsWith('/api/v1/desktop/')) {
- if (!this.checkDesktopAuth(req, res)) return
- const hostRef = url.pathname.replace('/api/v1/desktop/', '')
- await this.handleDesktopStatus(res, hostRef)
- return
- }
++ // Desktop status is handled above (Host-authority-gated, ABOVE the readiness
++ // gate). The v1 caller-selected Context/credential routes are retired at the
++ // application boundary (410) earlier in this handler; NP-08 removed their
++ // handlers, so no route block belongs here.
+
// 404
+ if (isProtectedMcpRoute) {
+ this.sendProtectedJson(res, 404, { error: 'not_found' })
+ return
+ }
this.sendJson(res, 404, { error: 'Not Found', message: 'Endpoint not found' })
}
@@@ -471,11 -268,12 +549,12 @@@
endpoints: {
'GET /': 'This information page',
'GET /health': "Health check endpoint - returns { status: 'ok', ready: boolean }",
- 'GET /ready': 'Readiness endpoint - returns 200 only after warm-up completes',
+ 'GET /ready':
+ 'Readiness endpoint - returns 200 after warm-up while provider inventory is authoritative',
'GET /api/v1/mcpservers': 'List all McpServer resources',
- 'GET /api/v1/mcpservers/context/:contextRef':
- 'List McpServers filtered by contextRef (only enabled servers)',
- 'GET /api/v1/mcpservers/:name/auth': 'Get authentication token for a specific McpServer',
+ 'GET /api/v2/hosts/self/mcpservers': 'List MCP servers for the authenticated Host',
+ 'POST /api/v2/hosts/self/mcpservers/credential':
+ 'Get an authorized credential for the authenticated Host',
},
examples: {
listAll: `curl http://localhost:${this.port}/api/v1/mcpservers`,
diff --cc host-context-controller/src/types.ts
index d71de8eb0,dfc8f722c..69d66fb56
--- a/host-context-controller/src/types.ts
+++ b/host-context-controller/src/types.ts
@@@ -507,13 -549,14 +549,16 @@@ export interface McpServerStatus
deployed: boolean
/** Whether the Deployment has at least one ready replica. */
ready: boolean
- /** Human-readable status message. */
- message?: string
/**
- * Whether this readiness observation is current and authoritative. Omitted
- * when the status source cannot make that guarantee.
- * Whether HCC can tie this status to the current McpServer identity and
- * generation. False means discovery is authoritative but runtime status has
- * not yet been re-established after controller startup.
++ * Whether this readiness observation is current and authoritative — HCC can
++ * tie the status to the current McpServer identity and generation. Omitted
++ * when the status source cannot make that guarantee; false means discovery
++ * is authoritative but runtime status has not yet been re-established after
++ * controller startup.
*/
authoritative?: boolean
+ /** Human-readable status message. */
+ message?: string
}
/**
diff --cc mcp-host/src/config.ts
index a597ed40f,ec6c16b64..66a42d4b9
--- a/mcp-host/src/config.ts
+++ b/mcp-host/src/config.ts
@@@ -556,19 -604,17 +608,25 @@@ export const config: Config =
// Context Mapper poll interval (default 30 seconds)
contextMapperPollInterval: parseInt(getEnv('CLERUM_CONTEXT_MAPPER_POLL_INTERVAL', '30000')!, 10),
+ // Bound transient HCC/Kubernetes authority outages. Identity failures revoke
+ // immediately; 5xx/transport failures may preserve the last good fleet only
+ // within this finite window.
+ hccAuthorityMaxStalenessMs: Math.min(
+ getEnvNumber('HCC_AUTHORITY_MAX_STALENESS_MS', 60_000),
+ 60_000
+ ),
+
// MCP status heartbeat. Defaults to 30 seconds so a single missed tick does
// not trip desktop staleness.
- mcpStatusHeartbeatInterval: parseInt(
- getEnv('CLERUM_MCP_STATUS_HEARTBEAT_INTERVAL', '30000')!,
- 10
+ mcpStatusHeartbeatInterval: parseMcpStatusHeartbeatDuration(
+ 'CLERUM_MCP_STATUS_HEARTBEAT_INTERVAL',
+ getEnv('CLERUM_MCP_STATUS_HEARTBEAT_INTERVAL'),
+ 30_000
+ ),
+ mcpStatusHeartbeatTimeoutMs: parseMcpStatusHeartbeatDuration(
+ 'CLERUM_MCP_STATUS_HEARTBEAT_TIMEOUT_MS',
+ getEnv('CLERUM_MCP_STATUS_HEARTBEAT_TIMEOUT_MS'),
+ 25_000
),
// Agent configuration
diff --cc mcp-host/src/main.ts
index 8fa84d964,20686d117..7bbc6a305
--- a/mcp-host/src/main.ts
+++ b/mcp-host/src/main.ts
@@@ -2668,11 -2566,15 +2720,16 @@@ async function shutdown(signal: string)
// Stop components in order
stopRuntimeAuthProactiveRefresh()
hostWatcher?.stop()
+ llmHookWatcher?.stop()
configStore?.stop()
stopContextMapperPolling()
+ mcpAuthorityStalenessDeadline.clear()
stopMcpStatusHeartbeat()
statelessHeartbeat?.stop()
+ if (guardrailResolveTimer) {
+ clearInterval(guardrailResolveTimer)
+ guardrailResolveTimer = null
+ }
if (agent) {
await agent.stop()