Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
chore: sync np-08 mcp-token branch with origin/dev @ 1b845a3d
Alfredo Lopez5ed13f26928d
diff --cc host-context-controller/src/k8sClient.test.tsindex 472b15d00,fb1ac6231..362c5954b--- a/host-context-controller/src/k8sClient.test.ts+++ b/host-context-controller/src/k8sClient.test.ts@@@ -3,14 -5,15 +5,17 @@@ import { HostFleetReconcileError } froimport { HostK8sRequestTimeoutError } from './k8s/hostK8sApiClient'import {McpServerWatcher,+ createMcpAuthorizationStore,getContext,+ isMcpAuthorizationNotFound,- externalEgressResyncDelayMs,listAllCommunicationChannels,+ listAllGlobalFileSystems,listAllHosts,+ listAllSharedFileSystems,} from './k8sClient'- import type { HostCRD } from './types'+ import { registry } from './metrics'+ import { ContextMapperServer } from './server'+ import type { HostCRD, McpServerCRD } from './types'function deferred<T = void>(): {promise: Promise<T>@@@ -31,20 -117,24 +119,26 @@@ const mocks = vi.hoisted(() =>const getNamespacedCustomObject = vi.fn().mockRejectedValue(Object.assign(new Error('not found'), { code: 404 }))+ const readNamespacedSecret = vi.fn()const ensureDefaultPolicies = vi.fn().mockResolvedValue(undefined)- const netPolFullReconcile = vi.fn().mockResolvedValue(undefined)+ const hasCertifiedSafetyInventory = vi.fn().mockReturnValue(true)+ const netPolFullReconcile = vi.fn().mockImplementation(async (...args: unknown[]) => {+ const options = args[2] as { onAuthoritativeRevocationComplete?: () => void } | undefined+ options?.onAuthoritativeRevocationComplete?.()+ })const serverFullReconcile = vi.fn().mockResolvedValue(undefined)const hostFullReconcile = vi.fn().mockResolvedValue(undefined)const hostReconcileHosts = vi.fn().mockResolvedValue(undefined)const sfsFullReconcile = vi.fn().mockResolvedValue(undefined)const watch = vi.fn().mockResolvedValue({ abort: vi.fn() })const hostListCallOptions = vi.fn()+ const sharedFileSystemListCallOptions = vi.fn()const createAdministrativeOutcomeReporter = vi.fn().mockReturnValue(undefined)return {+ hasCertifiedSafetyInventory,listNamespacedCustomObject,getNamespacedCustomObject,+ readNamespacedSecret,ensureDefaultPolicies,netPolFullReconcile,serverFullReconcile,@@@ -97,12 -190,14 +194,15 @@@ vi.mock('@kubernetes/client-node', () =makeApiClient(api: unknown): unknown {if (api === CustomObjectsApi) {return {+ getNamespacedCustomObject: mocks.getNamespacedCustomObject,listNamespacedCustomObject: async (request: { plural?: string }, options?: unknown) => {if (request.plural === 'hosts') mocks.hostListCallOptions(options)+ if (request.plural === 'sharedfilesystems')+ mocks.sharedFileSystemListCallOptions(options)const response = await mocks.listNamespacedCustomObject(request)if (- request.plural === 'hosts' &&+ request.plural &&+ ['hosts', 'mcpservers', 'contexts'].includes(request.plural) &&response &&typeof response === 'object' &&!('metadata' in response)@@@ -1032,50 -2139,3801 +2209,3815 @@@ describe('McpServerWatcher startup', (watcher.stop()})- it('starts CC watch even when initial CC list fails (graceful degradation)', async () => {- mocks.listNamespacedCustomObject.mockImplementation(async ({ plural }: { plural: string }) => {- if (plural === 'communicationchannels') throw new Error('CC list failed')- if (plural === 'mcpservers') return { items: [] }- if (plural === 'contexts') return { items: [] }- if (plural === 'hosts') return { items: [] }- if (plural === 'sharedfilesystems') return { items: [] }- return { items: [] }- })-+ it('updates discovery without re-reconciling a status-only McpServer watch event', async () => {+ const previousServer = {+ name: 'status-server',+ namespace: 'mcp-server',+ uid: 'status-server-uid',+ generation: 7,+ annotations: { 'clerum.io/recipe': 'status-recipe' },+ labels: { 'clerum.io/workload': 'status-server' },+ spec: {+ contextRef: 'default',+ image: 'clerum/status-server:test',+ transport: { type: 'streamableHttp' as const, port: 3000 },+ },+ }const watcher = new McpServerWatcher()+ ;(watcher as any).servers.set(previousServer.name, previousServer)+ const changed = vi.fn()+ watcher.onChange(changed)+ const reconciler = (watcher as any).reconciler+ const netPol = (watcher as any).netPolReconciler+ const callback = (watcher as any).getMcpServerWatchCallback()+ expect((watcher as any).mcpServerDesiredRevision).toBe(0)- vi.spyOn(watcher as any, 'startMcpServerWatch').mockResolvedValue(undefined)- vi.spyOn(watcher as any, 'startContextWatch').mockResolvedValue(undefined)- vi.spyOn(watcher as any, 'startSharedFileSystemWatch').mockResolvedValue(undefined)- vi.spyOn(watcher as any, 'startGlobalFileSystemWatch').mockResolvedValue(undefined)- const ccWatchSpy = vi- .spyOn(watcher as any, 'startCommunicationChannelWatch')- .mockResolvedValue(undefined)- const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {})+ await callback('MODIFIED', {+ metadata: {+ name: previousServer.name,+ namespace: previousServer.namespace,+ uid: previousServer.uid,+ generation: previousServer.generation,+ annotations: previousServer.annotations,+ labels: previousServer.labels,+ },+ spec: previousServer.spec,+ status: { conditions: [{ type: 'Ready', status: 'True' }] },+ })- await expect(watcher.start()).resolves.toBeUndefined()- expect(ccWatchSpy).toHaveBeenCalledTimes(1)- expect(watcher.isCommunicationChannelCacheSynced()).toBe(false)- expect(errorSpy).toHaveBeenCalledWith(- '[K8s] CommunicationChannel initial load failed; ccCacheSynced remains false ' +- '(B2 preserves channel-reader replicas and holds stateless lifecycle active):',- expect.any(Error)- )+ expect(watcher.getAllServers()[0]?.status).toEqual({+ conditions: [{ type: 'Ready', status: 'True' }],+ })+ expect(changed).toHaveBeenCalledOnce()+ expect(reconciler.reconcile).not.toHaveBeenCalled()+ expect(netPol.reconcileExternalEgress).not.toHaveBeenCalled()+ expect((watcher as any).mcpServerDesiredRevision).toBe(0)- errorSpy.mockRestore()- watcher.stop()+ await callback('MODIFIED', {+ metadata: {+ name: previousServer.name,+ namespace: previousServer.namespace,+ uid: previousServer.uid,+ generation: 8,+ annotations: previousServer.annotations,+ labels: previousServer.labels,+ },+ spec: { ...previousServer.spec, image: 'clerum/status-server:next' },+ })+ expect(reconciler.reconcile).toHaveBeenCalledOnce()+ expect((watcher as any).mcpServerDesiredRevision).toBe(1)+ await watcher.stop()})- })- describe('McpServerWatcher CommunicationChannel cache recovery', () => {- async function flushMicrotasks(): Promise<void> {- for (let i = 0; i < 8; i += 1) await Promise.resolve()- }+ it('does not advance desired revision for the controller-owned network-ready handshake', async () => {+ const previousServer = {+ name: 'network-ready-server',+ namespace: 'mcp-server',+ uid: 'network-ready-server-uid',+ generation: 7,+ annotations: { 'clerum.io/pre-deploy': 'true' },+ labels: { 'clerum.io/workload': 'network-ready-server' },+ spec: {+ contextRef: 'default',+ image: 'clerum/network-ready-server:test',+ transport: { type: 'streamableHttp' as const, port: 3000 },+ },+ }+ const watcher = new McpServerWatcher()+ ;(watcher as any).servers.set(previousServer.name, previousServer)+ const callback = (watcher as any).getMcpServerWatchCallback()++ await callback('MODIFIED', {+ metadata: {+ name: previousServer.name,+ namespace: previousServer.namespace,+ uid: previousServer.uid,+ generation: previousServer.generation,+ annotations: {+ ...previousServer.annotations,+ 'clerum.io/network-ready': 'true',+ },+ labels: previousServer.labels,+ },+ spec: previousServer.spec,+ })++ expect((watcher as any).mcpServerDesiredRevision).toBe(0)+ expect((watcher as any).reconciler.reconcile).not.toHaveBeenCalled()+ await watcher.stop()+ })++ it('advances McpServer desired revision for owner addition and deletion', async () => {+ const serverObject = {+ metadata: {+ name: 'revision-server',+ namespace: 'mcp-server',+ uid: 'revision-server-uid',+ generation: 1,+ },+ spec: {+ contextRef: 'default',+ image: 'clerum/revision-server:test',+ transport: { type: 'streamableHttp' as const, port: 3000 },+ },+ }+ const watcher = new McpServerWatcher()+ const callback = (watcher as any).getMcpServerWatchCallback()++ expect((watcher as any).mcpServerDesiredRevision).toBe(0)+ await callback('ADDED', serverObject)+ expect((watcher as any).mcpServerDesiredRevision).toBe(1)+ await callback('DELETED', serverObject)+ expect((watcher as any).mcpServerDesiredRevision).toBe(2)++ await watcher.stop()+ })++ it('advances Context desired revision only when the authoritative desired cache changes', async () => {+ const watcher = new McpServerWatcher()+ let contextWatchCallback:+ | ((+ type: string,+ apiObj: {+ metadata: { name: string; namespace: string }+ spec: { contextId: string; mcpServers: string[] }+ }+ ) => Promise<void>)+ | undefined+ mocks.watch.mockImplementationOnce(async (_path, _options, callback) => {+ contextWatchCallback = callback+ return { abort: vi.fn() }+ })+ await (watcher as any).startContextWatch('context-desired-revision-rv')+ ;(watcher as any).contextCacheSynced = true+ ;(watcher as any).mcpServerCacheSynced = true++ const added = {+ metadata: { name: 'revision-context', namespace: 'mcp-server' },+ spec: { contextId: 'revision-context', mcpServers: [] as string[] },+ }+ expect((watcher as any).contextDesiredRevision).toBe(0)+ await contextWatchCallback!('ADDED', added)+ expect((watcher as any).contextDesiredRevision).toBe(1)++ await contextWatchCallback!('MODIFIED', added)+ expect((watcher as any).contextDesiredRevision).toBe(1)++ const modified = {+ ...added,+ spec: { ...added.spec, mcpServers: ['revision-server'] },+ }+ await contextWatchCallback!('MODIFIED', modified)+ expect((watcher as any).contextDesiredRevision).toBe(2)++ await contextWatchCallback!('DELETED', modified)+ expect((watcher as any).contextDesiredRevision).toBe(3)++ await watcher.stop()+ })++ it('uses only the bounded fleet pass for the cold-start Host snapshot', async () => {+ const initialHosts = Array.from({ length: 50 }, (_, index) => ({+ metadata: {+ name: `initial-host-${index}`,+ namespace: 'mcp-host',+ uid: `initial-host-uid-${index}`,+ generation: 1,+ },+ spec: {+ host: `initial-host-${index}`,+ contextRef: 'default',+ secretRef: 'host-secret',+ },+ }))+ mocks.listNamespacedCustomObject.mockImplementation(async ({ plural }: { plural: string }) => {+ if (plural === 'hosts') {+ return {+ metadata: { resourceVersion: 'initial-host-rv' },+ items: initialHosts,+ }+ }+ if (plural === 'communicationchannels') {+ return { metadata: { resourceVersion: 'initial-cc-rv' }, items: [] }+ }+ return { items: [] }+ })+ const watcher = new McpServerWatcher()+ const urgentReconcile = vi.spyOn(watcher.getHostReconciler(), 'reconcile')++ await watcher.start()+ await vi.waitFor(() =>+ expect(mocks.hostFullReconcile).toHaveBeenCalledWith(+ initialHosts.map(host =>+ expect.objectContaining({+ name: host.metadata.name,+ generation: host.metadata.generation,+ })+ )+ )+ )+ await flushMicrotasks()++ expect(mocks.hostFullReconcile).toHaveBeenCalledOnce()+ expect(urgentReconcile).not.toHaveBeenCalled()+ await watcher.stop()+ })++ it('admits a McpServer watch reconciliation while initial convergence is blocked on another server', async () => {+ const staleServer = {+ name: 'stale-server',+ namespace: 'mcp-server',+ spec: {+ contextRef: 'default',+ image: 'clerum/stale-server:test',+ transport: { type: 'streamableHttp' as const, port: 3000 },+ egressBindings: [{ dns: 'stale.example', port: 443 }],+ },+ }+ const currentServer = {+ metadata: { name: 'current-server', namespace: 'mcp-server' },+ spec: {+ contextRef: 'default',+ image: 'clerum/current-server:test',+ transport: { type: 'streamableHttp' as const, port: 3000 },+ },+ }+ const egressStarted = deferred()+ const releaseEgress = deferred()+ const reconciliationOrder: string[] = []+ const watcher = new McpServerWatcher()+ ;(watcher as any).mcpServerCacheSynced = true+ ;(watcher as any).contextCacheSynced = true+ markNetworkPolicyRevocationAuthoritative(watcher)+ ;(watcher as any).servers.set(staleServer.name, staleServer)+ const netPol = (watcher as any).netPolReconciler+ netPol.reconcileExternalEgress.mockImplementationOnce(async () => {+ egressStarted.resolve(undefined)+ await releaseEgress.promise+ })+ mocks.serverFullReconcile.mockImplementationOnce(async (servers, options) => {+ await Promise.all(+ servers.map((selected: { name: string }) =>+ options.runEffect(selected.name, async () => {+ reconciliationOrder.push('initial')+ })+ )+ )+ })+ const reconciler = (watcher as any).reconciler+ reconciler.reconcile.mockImplementationOnce(async () => {+ reconciliationOrder.push('watch')+ })++ const convergence = (watcher as any).runInitialMcpServerConvergence()+ await egressStarted.promise+ const watchConvergence = (watcher as any).getMcpServerWatchCallback()('MODIFIED', currentServer)++ await flushMicrotasks()+ expect(reconciler.reconcile).toHaveBeenCalledWith(+ expect.objectContaining({ name: 'current-server' }),+ { isCurrent: expect.any(Function) }+ )++ releaseEgress.resolve(undefined)+ await Promise.all([convergence, watchConvergence])++ expect(reconciliationOrder).toEqual(['watch'])+ expect(reconciler.reconcile).toHaveBeenLastCalledWith(+ expect.objectContaining({ name: 'current-server' }),+ { isCurrent: expect.any(Function) }+ )+ await watcher.stop()+ })++ it('continues a same-server reconciliation lane after a rejected operation', async () => {+ const watcher = new McpServerWatcher()+ const server = { name: 'retry-safe-server', namespace: 'mcp-server' }+ const order: string[] = []+ const firstStarted = deferred()+ const rejectFirst = deferred()++ const rejected = (watcher as any).enqueueMcpServerReconciliation(server, async () => {+ order.push('rejected')+ firstStarted.resolve(undefined)+ await rejectFirst.promise+ throw new Error('transient lane failure')+ }) as Promise<void>+ await firstStarted.promise+ const recovered = (watcher as any).enqueueMcpServerReconciliation(server, async () => {+ order.push('recovered')+ }) as Promise<void>+ rejectFirst.resolve(undefined)++ await expect(rejected).rejects.toThrow('transient lane failure')+ await expect(recovered).resolves.toBeUndefined()+ expect(order).toEqual(['rejected', 'recovered'])+ expect((watcher as any).mcpServerReconciliationQueues.size).toBe(0)+ await watcher.stop()+ })++ it('revalidates env-Secret matches inside each server lane before reconciling', async () => {+ const currentServer = {+ name: 'current-secret-server',+ namespace: 'mcp-server',+ spec: {+ contextRef: 'default',+ image: 'clerum/current-secret-server:test',+ transport: { type: 'streamableHttp' as const, port: 3000 },+ envSecret: { name: 'shared-env' },+ },+ }+ const staleServer = {+ name: 'stale-secret-server',+ namespace: 'mcp-server',+ spec: {+ contextRef: 'default',+ image: 'clerum/stale-secret-server:test',+ transport: { type: 'streamableHttp' as const, port: 3000 },+ envSecret: { name: 'shared-env' },+ },+ }+ const unrelatedServer = {+ name: 'unrelated-secret-server',+ namespace: 'mcp-server',+ spec: {+ contextRef: 'default',+ image: 'clerum/unrelated-secret-server:test',+ transport: { type: 'streamableHttp' as const, port: 3000 },+ envSecret: { name: 'other-env' },+ },+ }+ const blockerStarted = deferred()+ const releaseBlocker = deferred()+ const watcher = new McpServerWatcher()+ markMcpServerInventoryAuthoritative(watcher)+ const reconciler = (watcher as any).reconciler+ for (const server of [currentServer, staleServer, unrelatedServer]) {+ ;(watcher as any).servers.set(server.name, server)+ }++ const blocker = (watcher as any).enqueueMcpServerReconciliation(staleServer, async () => {+ blockerStarted.resolve(undefined)+ await releaseBlocker.promise+ })+ await blockerStarted.promise++ const convergence = watcher.reconcileByEnvSecret('shared-env', 'mcp-server')+ ;(watcher as any).servers.set(staleServer.name, {+ ...staleServer,+ spec: { ...staleServer.spec, envSecret: { name: 'replacement-env' } },+ })+ await vi.waitFor(() =>+ expect(reconciler.reconcile).toHaveBeenCalledWith(+ expect.objectContaining({ name: currentServer.name }),+ { isCurrent: expect.any(Function) }+ )+ )+ releaseBlocker.resolve(undefined)+ await Promise.all([blocker, convergence])++ expect(reconciler.reconcile).toHaveBeenCalledTimes(1)+ expect(+ reconciler.reconcile.mock.calls.some(+ ([server]: [McpServerCRD]) => server.name === staleServer.name+ )+ ).toBe(false)+ expect(+ reconciler.reconcile.mock.calls.some(+ ([server]: [McpServerCRD]) => server.name === unrelatedServer.name+ )+ ).toBe(false)+ await watcher.stop()+ })++ it('defers env-Secret reconciliation until the McpServer inventory is authoritative', async () => {+ const selected = {+ name: 'authority-fenced-secret-server',+ namespace: 'mcp-server',+ spec: {+ contextRef: 'default',+ image: 'clerum/authority-fenced-secret-server:test',+ transport: { type: 'streamableHttp' as const, port: 3000 },+ envSecret: { name: 'shared-env' },+ },+ }+ const watcher = new McpServerWatcher()+ const reconciler = (watcher as any).reconciler+ ;(watcher as any).servers.set(selected.name, selected)+ ;(watcher as any).mcpServerCacheSynced = false++ await watcher.reconcileByEnvSecret('shared-env', 'mcp-server')++ expect(reconciler.reconcile).not.toHaveBeenCalled()++ markMcpServerInventoryAuthoritative(watcher)+ await watcher.reconcileByEnvSecret('shared-env', 'mcp-server')++ expect(reconciler.reconcile).toHaveBeenCalledOnce()+ expect(reconciler.reconcile).toHaveBeenCalledWith(+ expect.objectContaining({ name: selected.name }),+ { isCurrent: expect.any(Function) }+ )+ await watcher.stop()+ })++ it('fences Context-triggered SharedFileSystem convergence to the current Context lease', async () => {+ const context = {+ name: 'sfs-context',+ namespace: 'mcp-server',+ spec: {+ contextId: 'sfs-context',+ sharedFileSystems: [{ name: 'shared-data', mountPath: '/shared' }],+ },+ }+ const sharedFileSystem = {+ name: 'shared-data',+ namespace: 'mcp-host',+ spec: {},+ }+ const watcher = new McpServerWatcher()+ const reconciler = (watcher as any).sharedFileSystemReconciler+ ;(watcher as any).sharedFileSystems.set(sharedFileSystem.name, sharedFileSystem)++ await (watcher as any).reconcileSharedFileSystemsReferencedByContext(+ undefined,+ context,+ () => false+ )++ expect(reconciler.reconcile).not.toHaveBeenCalled()++ await (watcher as any).reconcileSharedFileSystemsReferencedByContext(+ undefined,+ context,+ () => true+ )++ expect(reconciler.reconcile).toHaveBeenCalledOnce()+ expect(reconciler.reconcile).toHaveBeenCalledWith(+ expect.objectContaining({ name: sharedFileSystem.name })+ )+ await watcher.stop()+ })++ it('fences Context-triggered Host convergence to the current Context lease', async () => {+ const watcher = new McpServerWatcher()+ markHostInventoryAuthoritative(watcher)+ ;(watcher as any).hosts.set('context-host', {+ name: 'context-host',+ namespace: 'mcp-host',+ spec: {+ host: 'context-host',+ contextRef: 'context-a',+ secretRef: 'host-secret',+ },+ })+ const hostReconciler = (watcher as any).hostReconciler++ await (watcher as any).reconcileHostsReferencingContext('context-a', () => false)++ expect(hostReconciler.reconcile).not.toHaveBeenCalled()++ await (watcher as any).reconcileHostsReferencingContext('context-a', () => true)++ expect(hostReconciler.reconcile).toHaveBeenCalledOnce()+ expect(hostReconciler.reconcile).toHaveBeenCalledWith(+ expect.objectContaining({ name: 'context-host' })+ )+ await watcher.stop()+ })++ it('keeps env-Secret reconciliation a no-op after stop', async () => {+ const selected = {+ name: 'stopped-secret-server',+ namespace: 'mcp-server',+ spec: {+ contextRef: 'default',+ image: 'clerum/stopped-secret-server:test',+ transport: { type: 'streamableHttp' as const, port: 3000 },+ envSecret: { name: 'shared-env' },+ },+ }+ const watcher = new McpServerWatcher()+ const reconciler = (watcher as any).reconciler+ ;(watcher as any).servers.set(selected.name, selected)++ await watcher.stop()+ await watcher.reconcileByEnvSecret('shared-env', 'mcp-server')++ expect(reconciler.reconcile).not.toHaveBeenCalled()+ })++ it('admits an initial server without egress while another initial server waits on DNS', async () => {+ const dnsBlockedServer = {+ name: 'dns-blocked-server',+ namespace: 'mcp-server',+ uid: 'dns-blocked-server-uid',+ generation: 1,+ spec: {+ contextRef: 'default',+ image: 'clerum/dns-blocked-server:test',+ transport: { type: 'streamableHttp' as const, port: 3000 },+ egressBindings: [{ dns: 'blocked.example', port: 443 }],+ },+ }+ const independentServer = {+ name: 'independent-server',+ namespace: 'mcp-server',+ uid: 'independent-server-uid',+ generation: 1,+ spec: {+ contextRef: 'default',+ image: 'clerum/independent-server:test',+ transport: { type: 'streamableHttp' as const, port: 3000 },+ },+ }+ const egressStarted = deferred()+ const releaseEgress = deferred()+ const watcher = new McpServerWatcher()+ ;(watcher as any).mcpServerCacheSynced = true+ ;(watcher as any).contextCacheSynced = true+ markNetworkPolicyRevocationAuthoritative(watcher)+ ;(watcher as any).servers.set(dnsBlockedServer.name, dnsBlockedServer)+ ;(watcher as any).servers.set(independentServer.name, independentServer)+ const netPol = (watcher as any).netPolReconciler+ const reconciler = (watcher as any).reconciler+ netPol.reconcileExternalEgress.mockImplementationOnce(async () => {+ egressStarted.resolve(undefined)+ await releaseEgress.promise+ })+ mocks.serverFullReconcile.mockImplementationOnce(+ async (+ servers: Array<typeof dnsBlockedServer | typeof independentServer>,+ options: { runEffect: (serverName: string, work: () => Promise<void>) => Promise<void> }+ ) => {+ await Promise.all(+ servers.map(server => options.runEffect(server.name, () => reconciler.reconcile(server)))+ )+ }+ )++ const convergence = (watcher as any).runInitialMcpServerConvergence()+ await egressStarted.promise+ await flushMicrotasks()++ expect(reconciler.reconcile).toHaveBeenCalledWith(independentServer)+ expect(reconciler.reconcile).not.toHaveBeenCalledWith(dnsBlockedServer)++ releaseEgress.resolve(undefined)+ await convergence++ expect(reconciler.reconcile).toHaveBeenCalledWith(dnsBlockedServer)+ await watcher.stop()+ })++ it('admits a Context watch reconciliation while initial policy convergence is blocked', async () => {+ const staleContext = {+ name: 'stale-context',+ namespace: 'mcp-server',+ spec: { contextId: 'stale-context', mcpServers: [] },+ }+ const currentContext = {+ metadata: { name: 'current-context', namespace: 'mcp-server' },+ spec: { contextId: 'current-context', mcpServers: [] },+ }+ const fullReconcileStarted = deferred()+ const releaseFullReconcile = deferred()+ const reconciliationOrder: string[] = []+ const watcher = new McpServerWatcher()+ ;(watcher as any).mcpServerCacheSynced = true+ ;(watcher as any).contextCacheSynced = true+ ;(watcher as any).contexts.set(staleContext.name, staleContext)+ mocks.netPolFullReconcile.mockImplementationOnce(async () => {+ fullReconcileStarted.resolve(undefined)+ await releaseFullReconcile.promise+ reconciliationOrder.push('initial')+ })+ const netPol = (watcher as any).netPolReconciler+ netPol.reconcileContext.mockImplementationOnce(async () => {+ reconciliationOrder.push('watch')+ })+ let contextWatchCallback:+ | ((type: string, apiObj: typeof currentContext) => Promise<void>)+ | undefined+ mocks.watch.mockImplementationOnce(async (_path, _options, callback) => {+ contextWatchCallback = callback+ return { abort: vi.fn() }+ })+ await (watcher as any).startContextWatch('context-queue-rv')++ const convergence = (watcher as any).runInitialNetworkPolicyConvergence(true)+ await fullReconcileStarted.promise+ const watchConvergence = contextWatchCallback!('MODIFIED', currentContext)++ await flushMicrotasks()+ expect(netPol.reconcileContext).toHaveBeenCalledWith(+ expect.objectContaining({ name: 'current-context' }),+ expect.objectContaining({ isCurrent: expect.any(Function) })+ )++ releaseFullReconcile.resolve(undefined)+ await Promise.all([convergence, watchConvergence])++ expect(reconciliationOrder).toEqual(['watch', 'initial'])+ expect(netPol.reconcileContext).toHaveBeenLastCalledWith(+ expect.objectContaining({ name: 'current-context' }),+ expect.objectContaining({ isCurrent: expect.any(Function) })+ )+ await watcher.stop()+ })++ it('makes a recreated Context the last writer after same-contextId orphan cleanup', async () => {+ const contextName = 'recreated-context-resource'+ const contextId = 'recreated-context'+ const recreatedContext = {+ metadata: { name: contextName, namespace: 'mcp-server' },+ spec: {+ contextId,+ description: 'recreated',+ mcpServers: [],+ },+ }+ const orphanCleanupStarted = deferred()+ const releaseOrphanCleanup = deferred()+ const order: string[] = []+ const watcher = new McpServerWatcher()+ ;(watcher as any).mcpServerCacheSynced = true+ ;(watcher as any).contextCacheSynced = true+ mocks.netPolFullReconcile.mockImplementationOnce(+ async (+ _contexts: unknown[],+ _servers: unknown[],+ options: {+ runContextEffect: (effectContextId: string, work: () => Promise<void>) => Promise<void>+ }+ ) => {+ await options.runContextEffect(contextId, async () => {+ orphanCleanupStarted.resolve(undefined)+ await releaseOrphanCleanup.promise+ order.push('orphan-delete')+ })+ }+ )+ const netPol = (watcher as any).netPolReconciler+ netPol.reconcileContext.mockImplementationOnce(async () => {+ order.push('live-recreate')+ })+ let contextWatchCallback:+ | ((type: string, apiObj: typeof recreatedContext) => Promise<void>)+ | undefined+ mocks.watch.mockImplementationOnce(async (_path, _options, callback) => {+ contextWatchCallback = callback+ return { abort: vi.fn() }+ })+ await (watcher as any).startContextWatch('recreated-context-rv')++ const convergence = (watcher as any).runInitialNetworkPolicyConvergence()+ await orphanCleanupStarted.promise+ const recreation = contextWatchCallback!('ADDED', recreatedContext)+ await flushMicrotasks()++ expect(netPol.reconcileContext).not.toHaveBeenCalled()++ releaseOrphanCleanup.resolve(undefined)+ await Promise.all([convergence, recreation])++ expect(order).toEqual(['orphan-delete', 'live-recreate'])+ expect(netPol.reconcileContext).toHaveBeenLastCalledWith(+ expect.objectContaining({+ name: contextName,+ spec: expect.objectContaining({ contextId, description: 'recreated' }),+ }),+ expect.objectContaining({ isCurrent: expect.any(Function) })+ )+ await watcher.stop()+ })++ it('retires old contextId policies before reconciling a mutable Context identity', async () => {+ const oldContext = {+ name: 'mutable-context-resource',+ namespace: 'mcp-server',+ spec: {+ contextId: 'old-context-id',+ mcpServers: ['redis-tools'],+ },+ }+ const newContext = {+ metadata: { name: oldContext.name, namespace: oldContext.namespace },+ spec: {+ contextId: 'new-context-id',+ mcpServers: ['redis-tools'],+ },+ }+ mocks.listNamespacedCustomObject.mockImplementation(async ({ plural }: { plural: string }) => {+ if (plural === 'contexts') {+ return {+ metadata: { resourceVersion: 'mutable-context-rv' },+ items: [newContext],+ }+ }+ return { items: [] }+ })+ let contextWatchCallback:+ | ((type: string, apiObj: typeof newContext) => Promise<void>)+ | undefined+ mocks.watch.mockImplementationOnce(async (_path, _options, callback) => {+ contextWatchCallback = callback+ return { abort: vi.fn() }+ })+ const watcher = new McpServerWatcher()+ ;(watcher as any).contexts.set(oldContext.name, oldContext)+ ;(watcher as any).contextCacheSynced = true+ const netPol = (watcher as any).netPolReconciler+ const order: string[] = []+ netPol.reconcileDeleteContext.mockImplementation(async (contextId: string) => {+ order.push(`delete:${contextId}`)+ })+ netPol.reconcileContext.mockImplementation(async (context: typeof oldContext) => {+ order.push(`reconcile:${context.spec.contextId}`)+ })+ await (watcher as any).startContextWatch('mutable-context-rv')++ await contextWatchCallback!('MODIFIED', newContext)++ expect(order).toEqual(['delete:old-context-id', 'reconcile:new-context-id'])+ expect(netPol.reconcileDeleteContext).toHaveBeenCalledWith(+ 'old-context-id',+ expect.any(Function)+ )+ await watcher.stop()+ })++ it('retires queued Context policy work immediately when its watch ends', async () => {+ const context = {+ metadata: { name: 'retired-context-resource', namespace: 'mcp-server' },+ spec: { contextId: 'retired-context-id', mcpServers: [] },+ }+ const blockerStarted = deferred()+ const releaseBlocker = deferred()+ let contextWatchCallback: ((type: string, apiObj: typeof context) => Promise<void>) | undefined+ let contextWatchDone: ((error: Error | null) => void) | undefined+ mocks.watch.mockImplementationOnce(async (_path, _options, callback, done) => {+ contextWatchCallback = callback+ contextWatchDone = done+ return { abort: vi.fn() }+ })+ const watcher = new McpServerWatcher()+ const netPol = (watcher as any).netPolReconciler+ await (watcher as any).startContextWatch('retired-context-rv')+ const blocker = (watcher as any).enqueueContextReconciliation(+ context.spec.contextId,+ async () => {+ blockerStarted.resolve(undefined)+ await releaseBlocker.promise+ }+ )+ await blockerStarted.promise++ const event = contextWatchCallback!('MODIFIED', context)+ contextWatchDone!(null)+ releaseBlocker.resolve(undefined)+ await Promise.all([blocker, event])++ expect(netPol.reconcileContext).not.toHaveBeenCalled()+ await watcher.stop()+ })++ it('serializes McpServer-driven Context policy work with a newer Context watch revision', async () => {+ const server = {+ metadata: {+ name: 'context-policy-server',+ namespace: 'mcp-server',+ uid: 'context-policy-server-uid',+ generation: 1,+ },+ spec: {+ contextRef: 'context-policy',+ image: 'clerum/context-policy-server:test',+ transport: { type: 'streamableHttp' as const, port: 3000 },+ },+ }+ const oldContext = {+ name: 'context-policy-resource',+ namespace: 'mcp-server',+ spec: {+ contextId: 'context-policy',+ description: 'old',+ mcpServers: [server.metadata.name],+ },+ }+ const newContext = {+ metadata: { name: oldContext.name, namespace: oldContext.namespace },+ spec: {+ ...oldContext.spec,+ description: 'new',+ },+ }+ const oldStarted = deferred()+ const releaseOld = deferred()+ const order: string[] = []+ let oldLease: (() => boolean) | undefined+ let contextWatchCallback:+ | ((type: string, apiObj: typeof newContext) => Promise<void>)+ | undefined+ mocks.watch.mockImplementationOnce(async (_path, _options, callback) => {+ contextWatchCallback = callback+ return { abort: vi.fn() }+ })+ const watcher = new McpServerWatcher()+ ;(watcher as any).contexts.set(oldContext.name, oldContext)+ const netPol = (watcher as any).netPolReconciler+ netPol.reconcileContext.mockImplementation(+ async (context: typeof oldContext, options: { isCurrent: () => boolean }) => {+ if (context.spec.description === 'old') {+ oldLease = options.isCurrent+ oldStarted.resolve(undefined)+ await releaseOld.promise+ order.push('old')+ return+ }+ order.push('new')+ }+ )+ await (watcher as any).startContextWatch('context-policy-rv')++ const serverEvent = (watcher as any).getMcpServerWatchCallback()('MODIFIED', server)+ await oldStarted.promise+ const contextEvent = contextWatchCallback!('MODIFIED', newContext)+ await flushMicrotasks()++ expect(netPol.reconcileContext).toHaveBeenCalledTimes(1)+ expect(oldLease?.()).toBe(false)+ releaseOld.resolve(undefined)+ await Promise.all([serverEvent, contextEvent])++ expect(order).toEqual(['old', 'new'])+ expect(netPol.reconcileContext).toHaveBeenLastCalledWith(+ expect.objectContaining({+ spec: expect.objectContaining({ description: 'new' }),+ }),+ expect.objectContaining({ isCurrent: expect.any(Function) })+ )+ await watcher.stop()+ })++ it('retires a blocked Context policy lease when MODIFIED replaces it in the same watch', async () => {+ const oldContext = {+ metadata: { name: 'same-stream-context', namespace: 'mcp-server' },+ spec: {+ contextId: 'same-stream-context',+ description: 'old',+ mcpServers: ['old-server'],+ },+ }+ const newContext = {+ metadata: oldContext.metadata,+ spec: {+ ...oldContext.spec,+ description: 'new',+ mcpServers: ['new-server'],+ },+ }+ const mutationStarted = deferred()+ const releaseMutation = deferred()+ let oldLease: (() => boolean) | undefined+ let contextWatchCallback:+ | ((type: string, apiObj: typeof oldContext) => Promise<void>)+ | undefined+ mocks.watch.mockImplementationOnce(async (_path, _options, callback) => {+ contextWatchCallback = callback+ return { abort: vi.fn() }+ })+ const watcher = new McpServerWatcher()+ const netPol = (watcher as any).netPolReconciler+ netPol.reconcileContext.mockImplementationOnce(+ async (_context: unknown, options: { isCurrent: () => boolean }) => {+ oldLease = options.isCurrent+ mutationStarted.resolve()+ await releaseMutation.promise+ }+ )+ await (watcher as any).startContextWatch('same-stream-context-rv')+ ;(watcher as any).contextCacheSynced = true+ ;(watcher as any).mcpServerCacheSynced = true++ const oldReconcile = contextWatchCallback!('MODIFIED', oldContext)+ await mutationStarted.promise+ expect(oldLease?.()).toBe(true)++ const newReconcile = contextWatchCallback!('MODIFIED', newContext)+ expect(oldLease?.()).toBe(false)++ releaseMutation.resolve()+ await Promise.all([oldReconcile, newReconcile])+ await watcher.stop()+ })++ it('keeps live control-plane events responsive during a periodic external-egress fleet resync', async () => {+ const server = {+ name: 'periodic-egress-server',+ namespace: 'mcp-server',+ uid: 'periodic-egress-uid',+ generation: 1,+ spec: {+ contextRef: 'default',+ image: 'clerum/periodic-egress:test',+ transport: { type: 'streamableHttp' as const, port: 3000 },+ egressBindings: [{ dns: 'periodic.example', port: 443 }],+ },+ }+ const context = {+ metadata: { name: 'live-context', namespace: 'mcp-server' },+ spec: { contextId: 'live-context', mcpServers: [] },+ }+ const resyncStarted = deferred()+ const releaseResync = deferred()+ let contextWatchCallback: ((type: string, apiObj: typeof context) => Promise<void>) | undefined+ mocks.watch.mockImplementationOnce(async (_path, _options, callback) => {+ contextWatchCallback = callback+ return { abort: vi.fn() }+ })+ const randomSpy = vi.spyOn(Math, 'random').mockReturnValue(0)+ const watcher = new McpServerWatcher()+ markMcpServerInventoryAuthoritative(watcher)+ ;(watcher as any).servers.set(server.name, server)+ const netPol = (watcher as any).netPolReconciler+ netPol.reconcileExternalEgress.mockImplementationOnce(async () => {+ resyncStarted.resolve(undefined)+ await releaseResync.promise+ })+ await (watcher as any).startContextWatch('live-context-rv')++ const firstResync = (watcher as any).externalEgressCoordinator.runResync() as Promise<void>+ const overlappingResync = (+ watcher as any+ ).externalEgressCoordinator.runResync() as Promise<void>+ expect(overlappingResync).toBe(firstResync)+ await resyncStarted.promise++ const liveEvent = contextWatchCallback!('MODIFIED', context)+ await flushMicrotasks()+ expect(netPol.reconcileContext).toHaveBeenCalledWith(+ expect.objectContaining({ name: 'live-context' }),+ expect.objectContaining({ isCurrent: expect.any(Function) })+ )++ releaseResync.resolve(undefined)+ await Promise.all([firstResync, liveEvent])+ await watcher.stop()+ randomSpy.mockRestore()+ })++ it('retires a blocked periodic egress lease when MODIFIED replaces it in the same watch', async () => {+ const oldServer = {+ name: 'same-stream-egress',+ namespace: 'mcp-server',+ uid: 'same-stream-egress-uid',+ generation: 1,+ spec: {+ contextRef: 'default',+ image: 'clerum/same-stream-egress:old',+ transport: { type: 'streamableHttp' as const, port: 3000 },+ egressBindings: [{ dns: 'old.example', port: 443 }],+ },+ }+ const newServer = {+ metadata: {+ name: oldServer.name,+ namespace: oldServer.namespace,+ uid: oldServer.uid,+ generation: 2,+ },+ spec: {+ ...oldServer.spec,+ image: 'clerum/same-stream-egress:new',+ egressBindings: [{ dns: 'new.example', port: 443 }],+ },+ }+ const mutationStarted = deferred()+ const releaseMutation = deferred()+ let oldLease: (() => boolean) | undefined+ const watcher = new McpServerWatcher()+ markMcpServerInventoryAuthoritative(watcher)+ ;(watcher as any).servers.set(oldServer.name, oldServer)+ const netPol = (watcher as any).netPolReconciler+ netPol.reconcileExternalEgress.mockImplementationOnce(+ async (_server: unknown, options: { isCurrent: () => boolean }) => {+ oldLease = options.isCurrent+ mutationStarted.resolve()+ await releaseMutation.promise+ }+ )+ vi.spyOn(Math, 'random').mockReturnValue(0)++ const periodic = (watcher as any).externalEgressCoordinator.runResync() as Promise<void>+ await mutationStarted.promise+ expect(oldLease?.()).toBe(true)++ const replacement = (watcher as any).getMcpServerWatchCallback()('MODIFIED', newServer)+ expect(oldLease?.()).toBe(false)++ releaseMutation.resolve()+ await Promise.all([periodic, replacement])+ await watcher.stop()+ })++ it('uses the latest cached McpServer after periodic external-egress jitter', async () => {+ vi.useFakeTimers()+ const staleServer = {+ name: 'periodic-egress-server',+ namespace: 'mcp-server',+ uid: 'periodic-egress-uid',+ generation: 1,+ spec: {+ contextRef: 'default',+ image: 'clerum/periodic-egress:old',+ transport: { type: 'streamableHttp' as const, port: 3000 },+ egressBindings: [{ dns: 'old.example', port: 443 }],+ },+ }+ const currentServer = {+ metadata: {+ name: staleServer.name,+ namespace: staleServer.namespace,+ uid: staleServer.uid,+ generation: 2,+ },+ spec: {+ ...staleServer.spec,+ image: 'clerum/periodic-egress:new',+ egressBindings: [{ dns: 'new.example', port: 443 }],+ },+ }+ const randomSpy = vi.spyOn(Math, 'random').mockReturnValue(0.5)+ const watcher = new McpServerWatcher()+ ;(watcher as any).servers.set(staleServer.name, staleServer)+ const netPol = (watcher as any).netPolReconciler++ const periodic = (watcher as any).externalEgressCoordinator.runResync() as Promise<void>+ await flushMicrotasks()+ await (watcher as any).getMcpServerWatchCallback()('MODIFIED', currentServer)+ await vi.advanceTimersByTimeAsync(2500)+ await periodic++ expect(netPol.reconcileExternalEgress).toHaveBeenLastCalledWith(+ expect.objectContaining({+ name: staleServer.name,+ generation: 2,+ spec: expect.objectContaining({+ egressBindings: [{ dns: 'new.example', port: 443 }],+ }),+ }),+ { isCurrent: expect.any(Function) }+ )+ await watcher.stop()+ randomSpy.mockRestore()+ })++ it('queues same-object SharedFileSystem and GlobalFileSystem watch effects behind initial passes', async () => {+ const staleSfs = { name: 'stale-sfs', namespace: 'mcp-host', spec: {} }+ const currentSfs = { metadata: { name: 'stale-sfs', namespace: 'mcp-host' }, spec: {} }+ const staleGfs = { name: 'stale-gfs', namespace: 'mcp-host', spec: {} }+ const currentGfs = {+ metadata: { name: 'stale-gfs', namespace: 'mcp-host' },+ spec: {},+ status: {+ phase: 'Ready',+ pvcName: 'gfs-drive',+ serviceName: 'gfsc',+ serviceUrl: 'http://gfsc.gfs.svc.cluster.local:8087',+ },+ }+ const sfsInitialStarted = deferred()+ const releaseSfsInitial = deferred()+ const gfsInitialStarted = deferred()+ const releaseGfsInitial = deferred()+ const sfsOrder: string[] = []+ const gfsOrder: string[] = []+ const watcher = new McpServerWatcher()+ ;(watcher as any).sharedFileSystems.set(staleSfs.name, staleSfs)+ ;(watcher as any).globalFileSystems.set(staleGfs.name, staleGfs)++ mocks.sfsFullReconcile.mockImplementationOnce(async () => {+ sfsInitialStarted.resolve(undefined)+ await releaseSfsInitial.promise+ sfsOrder.push('initial')+ })+ const sfsReconciler = (watcher as any).sharedFileSystemReconciler+ sfsReconciler.reconcile.mockImplementationOnce(async () => {+ sfsOrder.push('watch')+ })+ const gfsReconciler = (watcher as any).gfsReconciler+ vi.spyOn(gfsReconciler, 'fullReconcile').mockImplementationOnce(async () => {+ gfsInitialStarted.resolve(undefined)+ await releaseGfsInitial.promise+ gfsOrder.push('initial')+ })+ const gfsReconcile = vi.spyOn(gfsReconciler, 'reconcile').mockImplementationOnce(async () => {+ gfsOrder.push('watch')+ })++ let sfsWatchCallback: ((type: string, apiObj: typeof currentSfs) => Promise<void>) | undefined+ let gfsWatchCallback: ((type: string, apiObj: typeof currentGfs) => Promise<void>) | undefined+ mocks.watch+ .mockImplementationOnce(async (_path, _options, callback) => {+ sfsWatchCallback = callback+ return { abort: vi.fn() }+ })+ .mockImplementationOnce(async (_path, _options, callback) => {+ gfsWatchCallback = callback+ return { abort: vi.fn() }+ })+ await (watcher as any).startSharedFileSystemWatch()+ await (watcher as any).startGlobalFileSystemWatch()++ const sfsInitial = (watcher as any).runInitialSharedFileSystemConvergence()+ const gfsInitial = (watcher as any).runInitialGlobalFileSystemConvergence()+ await Promise.all([sfsInitialStarted.promise, gfsInitialStarted.promise])+ const sfsWatch = sfsWatchCallback!('MODIFIED', currentSfs)+ const gfsWatch = gfsWatchCallback!('MODIFIED', currentGfs)++ await flushMicrotasks()+ expect(sfsReconciler.reconcile).not.toHaveBeenCalled()+ expect(gfsReconcile).not.toHaveBeenCalled()++ releaseSfsInitial.resolve(undefined)+ releaseGfsInitial.resolve(undefined)+ await Promise.all([sfsInitial, gfsInitial, sfsWatch, gfsWatch])++ expect(sfsOrder).toEqual(['initial', 'watch'])+ expect(gfsOrder).toEqual(['initial', 'watch'])+ expect(gfsReconcile).toHaveBeenCalledWith(+ expect.objectContaining({ status: currentGfs.status })+ )+ await watcher.stop()+ })++ it('does not let a periodic SFS or GFS inventory overwrite a newer watch cache entry', async () => {+ const staleSfs = { metadata: { name: 'stale-sfs', namespace: 'mcp-host' }, spec: {} }+ const currentSfs = { metadata: { name: 'current-sfs', namespace: 'mcp-host' }, spec: {} }+ const staleGfs = { metadata: { name: 'stale-gfs', namespace: 'mcp-host' }, spec: {} }+ const currentGfs = { metadata: { name: 'current-gfs', namespace: 'mcp-host' }, spec: {} }+ const sfsListStarted = deferred()+ const releaseSfsList = deferred<unknown>()+ const gfsListStarted = deferred()+ const releaseGfsList = deferred<unknown>()+ const watcher = new McpServerWatcher()+ mocks.listNamespacedCustomObject.mockImplementation(async ({ plural }: { plural: string }) => {+ if (plural === 'sharedfilesystems') {+ sfsListStarted.resolve(undefined)+ return releaseSfsList.promise+ }+ if (plural === 'globalfilesystems') {+ gfsListStarted.resolve(undefined)+ return releaseGfsList.promise+ }+ return { items: [] }+ })+ const gfsReconciler = (watcher as any).gfsReconciler+ vi.spyOn(gfsReconciler, 'fullReconcile').mockResolvedValue(undefined)++ let sfsWatchCallback: ((type: string, apiObj: typeof currentSfs) => Promise<void>) | undefined+ let gfsWatchCallback: ((type: string, apiObj: typeof currentGfs) => Promise<void>) | undefined+ mocks.watch+ .mockImplementationOnce(async (_path, _options, callback) => {+ sfsWatchCallback = callback+ return { abort: vi.fn() }+ })+ .mockImplementationOnce(async (_path, _options, callback) => {+ gfsWatchCallback = callback+ return { abort: vi.fn() }+ })+ await (watcher as any).startSharedFileSystemWatch()+ await (watcher as any).startGlobalFileSystemWatch()++ const sfsResync = (watcher as any).runSfsResync()+ const gfsResync = (watcher as any).runGfsResync()+ await Promise.all([sfsListStarted.promise, gfsListStarted.promise])+ const sfsWatch = sfsWatchCallback!('MODIFIED', currentSfs)+ const gfsWatch = gfsWatchCallback!('MODIFIED', currentGfs)+ releaseSfsList.resolve({ items: [staleSfs] })+ releaseGfsList.resolve({ items: [staleGfs] })+ await Promise.all([sfsResync, gfsResync, sfsWatch, gfsWatch])++ expect(mocks.sfsFullReconcile).toHaveBeenCalledWith([+ expect.objectContaining({ name: 'current-sfs' }),+ ])+ expect(gfsReconciler.fullReconcile).toHaveBeenCalledWith([+ expect.objectContaining({ name: 'current-gfs' }),+ ])+ expect((watcher as any).sharedFileSystems.get('current-sfs')).toEqual(+ expect.objectContaining({ name: 'current-sfs' })+ )+ expect((watcher as any).sharedFileSystems.has('stale-sfs')).toBe(false)+ expect((watcher as any).globalFileSystems.get('current-gfs')).toEqual(+ expect.objectContaining({ name: 'current-gfs' })+ )+ expect((watcher as any).globalFileSystems.has('stale-gfs')).toBe(false)+ await watcher.stop()+ })++ it('refreshes SFS and GFS caches and effects from an uncontested periodic inventory', async () => {+ const listedSfs = {+ metadata: { name: 'listed-sfs', namespace: 'mcp-host' },+ spec: { capacity: '1Gi' },+ }+ const listedGfs = {+ metadata: { name: 'listed-gfs', namespace: 'gfs' },+ spec: { capacity: '10Gi' },+ }+ mocks.listNamespacedCustomObject.mockImplementation(async ({ plural }: { plural: string }) => {+ if (plural === 'sharedfilesystems') return { items: [listedSfs] }+ if (plural === 'globalfilesystems') return { items: [listedGfs] }+ return { items: [] }+ })+ const watcher = new McpServerWatcher()+ const gfsReconciler = (watcher as any).gfsReconciler+ const gfsFullReconcile = vi.spyOn(gfsReconciler, 'fullReconcile').mockResolvedValue(undefined)++ await Promise.all([(watcher as any).runSfsResync(), (watcher as any).runGfsResync()])++ expect((watcher as any).sharedFileSystems.get('listed-sfs')).toEqual(+ expect.objectContaining({ name: 'listed-sfs', spec: listedSfs.spec })+ )+ expect((watcher as any).globalFileSystems.get('listed-gfs')).toEqual(+ expect.objectContaining({ name: 'listed-gfs', spec: listedGfs.spec })+ )+ expect(mocks.sfsFullReconcile).toHaveBeenCalledWith([+ expect.objectContaining({ name: 'listed-sfs', spec: listedSfs.spec }),+ ])+ expect(gfsFullReconcile).toHaveBeenCalledWith([+ expect.objectContaining({ name: 'listed-gfs', spec: listedGfs.spec }),+ ])+ await watcher.stop()+ })++ it('does not run queued fleet effects after the watcher stops', async () => {+ const initialServer = {+ name: 'initial-server',+ namespace: 'mcp-server',+ spec: {+ contextRef: 'default',+ image: 'clerum/initial-server:test',+ transport: { type: 'streamableHttp' as const, port: 3000 },+ egressBindings: [{ dns: 'initial.example', port: 443 }],+ },+ }+ const queuedServer = {+ metadata: { name: 'queued-server', namespace: 'mcp-server' },+ spec: {+ contextRef: 'default',+ image: 'clerum/queued-server:test',+ transport: { type: 'streamableHttp' as const, port: 3000 },+ },+ }+ const egressStarted = deferred()+ const releaseEgress = deferred()+ const watcher = new McpServerWatcher()+ ;(watcher as any).mcpServerCacheSynced = true+ ;(watcher as any).contextCacheSynced = true+ markNetworkPolicyRevocationAuthoritative(watcher)+ ;(watcher as any).servers.set(initialServer.name, initialServer)+ const netPol = (watcher as any).netPolReconciler+ netPol.reconcileExternalEgress.mockImplementationOnce(async () => {+ egressStarted.resolve(undefined)+ await releaseEgress.promise+ })+ const reconciler = (watcher as any).reconciler+ mocks.serverFullReconcile.mockImplementationOnce(async (servers, options) => {+ await Promise.all(+ servers.map((selected: { name: string }) =>+ options.runEffect(selected.name, () => reconciler.reconcile(selected))+ )+ )+ })++ const initial = (watcher as any).runInitialMcpServerConvergence()+ await egressStarted.promise+ const queuedWatch = (watcher as any).getMcpServerWatchCallback()('MODIFIED', queuedServer)++ await watcher.stop()+ releaseEgress.resolve(undefined)+ await Promise.all([initial, queuedWatch])++ expect(mocks.serverFullReconcile).toHaveBeenCalledOnce()+ expect(reconciler.reconcile).not.toHaveBeenCalled()+ })++ it('coalesces overlapping McpServer fleet passes into one trailing current-cache pass', async () => {+ const firstPassStarted = deferred()+ const releaseFirstPass = deferred()+ const snapshots: string[][] = []+ const watcher = new McpServerWatcher()+ ;(watcher as any).mcpServerCacheSynced = true+ ;(watcher as any).contextCacheSynced = true+ markNetworkPolicyRevocationAuthoritative(watcher)+ ;(watcher as any).servers.set('first-server', {+ name: 'first-server',+ namespace: 'mcp-server',+ spec: {+ contextRef: 'default',+ image: 'clerum/first-server:test',+ transport: { type: 'streamableHttp' as const, port: 3000 },+ },+ })+ mocks.serverFullReconcile+ .mockImplementationOnce(async (servers: Array<{ name: string }>) => {+ snapshots.push(servers.map(server => server.name))+ firstPassStarted.resolve(undefined)+ await releaseFirstPass.promise+ })+ .mockImplementationOnce(async (servers: Array<{ name: string }>) => {+ snapshots.push(servers.map(server => server.name))+ })++ const first = (watcher as any).runInitialMcpServerConvergence()+ await firstPassStarted.promise+ ;(watcher as any).servers.set('second-server', {+ name: 'second-server',+ namespace: 'mcp-server',+ spec: {+ contextRef: 'default',+ image: 'clerum/second-server:test',+ transport: { type: 'streamableHttp' as const, port: 3000 },+ },+ })+ const second = (watcher as any).runInitialMcpServerConvergence()+ const third = (watcher as any).runInitialMcpServerConvergence()++ expect(mocks.serverFullReconcile).toHaveBeenCalledTimes(1)+ releaseFirstPass.resolve(undefined)+ await Promise.all([first, second, third])++ expect(mocks.serverFullReconcile).toHaveBeenCalledTimes(2)+ expect(snapshots).toEqual([['first-server'], ['first-server', 'second-server']])+ await watcher.stop()+ })++ it('coalesces overlapping NetworkPolicy fleet passes into one trailing current-cache pass', async () => {+ const firstPassStarted = deferred()+ const releaseFirstPass = deferred()+ const snapshots: string[][] = []+ const watcher = new McpServerWatcher()+ ;(watcher as any).contextCacheSynced = true+ ;(watcher as any).mcpServerCacheSynced = true+ ;(watcher as any).contexts.set('first-context', {+ name: 'first-context',+ namespace: 'mcp-server',+ spec: { contextId: 'first-context', mcpServers: [] },+ })+ mocks.netPolFullReconcile+ .mockImplementationOnce(async (contexts: Array<{ name: string }>) => {+ snapshots.push(contexts.map(context => context.name))+ firstPassStarted.resolve(undefined)+ await releaseFirstPass.promise+ })+ .mockImplementationOnce(async (contexts: Array<{ name: string }>) => {+ snapshots.push(contexts.map(context => context.name))+ })++ const first = (watcher as any).runInitialNetworkPolicyConvergence()+ await firstPassStarted.promise+ ;(watcher as any).contexts.set('second-context', {+ name: 'second-context',+ namespace: 'mcp-server',+ spec: { contextId: 'second-context', mcpServers: [] },+ })+ const second = (watcher as any).runInitialNetworkPolicyConvergence()+ const third = (watcher as any).runInitialNetworkPolicyConvergence()++ expect(mocks.netPolFullReconcile).toHaveBeenCalledTimes(1)+ releaseFirstPass.resolve(undefined)+ await Promise.all([first, second, third])++ expect(mocks.netPolFullReconcile).toHaveBeenCalledTimes(2)+ expect(snapshots).toEqual([['first-context'], ['first-context', 'second-context']])+ await watcher.stop()+ })++ it('does not lose an initial convergence request queued in the settlement microtask', async () => {+ const watcher = new McpServerWatcher()+ let trailing: Promise<void> | undefined+ const convergenceCore = vi+ .fn()+ .mockImplementationOnce(+ () =>+ ({+ then: (resolve: () => void) => {+ resolve()+ queueMicrotask(() => {+ trailing = (watcher as any).runInitialMcpServerConvergence()+ })+ },+ }) as PromiseLike<void>+ )+ .mockResolvedValueOnce(undefined)+ ;(watcher as any).runInitialMcpServerConvergenceCore = convergenceCore++ const first = (watcher as any).runInitialMcpServerConvergence()+ await first+ await vi.waitFor(() => expect(trailing).toBeDefined())+ await trailing++ expect(convergenceCore).toHaveBeenCalledTimes(2)+ await watcher.stop()+ })++ it('retains one keyed queue until every same-key waiter has drained', async () => {+ const watcher = new McpServerWatcher()+ const firstStarted = deferred()+ const releaseFirst = deferred()+ const effects: string[] = []+ const queues = (watcher as any).contextReconciliationQueues as Map<+ string,+ { references: number }+ >++ const first = (watcher as any).enqueueContextReconciliation('shared', async () => {+ effects.push('first')+ firstStarted.resolve(undefined)+ await releaseFirst.promise+ })+ await firstStarted.promise+ const second = (watcher as any).enqueueContextReconciliation('shared', async () => {+ effects.push('second')+ })++ expect(queues.get('shared')?.references).toBe(2)+ expect(effects).toEqual(['first'])+ releaseFirst.resolve(undefined)+ await Promise.all([first, second])+ expect(effects).toEqual(['first', 'second'])+ expect(queues.size).toBe(0)+ await watcher.stop()+ })++ it('does not execute newly enqueued keyed work after stop', async () => {+ const watcher = new McpServerWatcher()+ const work = vi.fn()+ ;(watcher as any).stopped = true++ await (watcher as any).enqueueContextReconciliation('stopped', work)++ expect(work).not.toHaveBeenCalled()+ expect((watcher as any).contextReconciliationQueues.size).toBe(0)+ })++ it('acquires multi-context identity keys in one deterministic sorted order', async () => {+ const watcher = new McpServerWatcher()+ const acquired: string[] = []+ vi.spyOn(watcher as any, 'enqueueContextReconciliation').mockImplementation(+ async (...args: unknown[]) => {+ const [key, work] = args as [string, () => Promise<void>]+ acquired.push(key)+ await work()+ }+ )++ await (watcher as any).enqueueContextIdentityReconciliation(+ ['z-context', 'a-context', 'm-context', 'z-context'],+ async () => undefined+ )++ expect(acquired).toEqual(['a-context', 'm-context', 'z-context'])+ await watcher.stop()+ })++ it('fences an McpServer fleet pass when its LIST-to-WATCH authority is retired', async () => {+ const egressStarted = deferred()+ const releaseEgress = deferred()+ const watcher = new McpServerWatcher()+ ;(watcher as any).mcpServerCacheSynced = true+ ;(watcher as any).contextCacheSynced = true+ ;(watcher as any).mcpWatchGeneration = 7+ markNetworkPolicyRevocationAuthoritative(watcher)+ ;(watcher as any).initialConvergenceRetryAttempts.set('McpServer', 3)+ ;(watcher as any).servers.set('stale-server', {+ name: 'stale-server',+ namespace: 'mcp-server',+ generation: 1,+ spec: {+ contextRef: 'default',+ image: 'clerum/stale-server:v1',+ transport: { type: 'streamableHttp' as const, port: 3000 },+ egressBindings: [{ dns: 'stale.example', port: 443 }],+ },+ })+ const netPol = (watcher as any).netPolReconciler+ const reconciler = (watcher as any).reconciler+ netPol.reconcileExternalEgress.mockImplementationOnce(async () => {+ egressStarted.resolve(undefined)+ await releaseEgress.promise+ })+ mocks.serverFullReconcile.mockImplementation(async (servers, options) => {+ await Promise.all(+ servers.map((selected: { name: string }) =>+ options.runEffect(selected.name, () => reconciler.reconcile(selected))+ )+ )+ })++ const stalePass = (watcher as any).runInitialMcpServerConvergence()+ await egressStarted.promise+ ;(watcher as any).mcpServerCacheSynced = false+ ;(watcher as any).mcpWatchGeneration = 8+ const trailing = (watcher as any).runInitialMcpServerConvergence()+ releaseEgress.resolve(undefined)+ await Promise.all([stalePass, trailing])++ expect(reconciler.reconcile).not.toHaveBeenCalled()+ expect((watcher as any).initialConvergenceRetryAttempts.get('McpServer')).toBe(3)+ ;(watcher as any).servers.set('stale-server', {+ name: 'stale-server',+ namespace: 'mcp-server',+ generation: 2,+ spec: {+ contextRef: 'default',+ image: 'clerum/stale-server:v2',+ transport: { type: 'streamableHttp' as const, port: 3000 },+ egressBindings: [{ dns: 'current.example', port: 443 }],+ },+ })+ ;(watcher as any).mcpServerCacheSynced = true+ markNetworkPolicyRevocationAuthoritative(watcher)+ await (watcher as any).runInitialMcpServerConvergence()++ expect(reconciler.reconcile).toHaveBeenCalledOnce()+ expect(reconciler.reconcile).toHaveBeenCalledWith(+ expect.objectContaining({+ generation: 2,+ spec: expect.objectContaining({ image: 'clerum/stale-server:v2' }),+ })+ )+ expect((watcher as any).initialConvergenceRetryAttempts.has('McpServer')).toBe(false)+ await watcher.stop()+ })++ it('fences NetworkPolicy positive effects when their inventory leases are retired', async () => {+ const firstPassStarted = deferred()+ const releaseFirstPass = deferred()+ const appliedContexts: string[] = []+ const appliedServers: string[] = []+ let pass = 0+ const watcher = new McpServerWatcher()+ ;(watcher as any).contextCacheSynced = true+ ;(watcher as any).mcpServerCacheSynced = true+ ;(watcher as any).contextWatchGeneration = 11+ ;(watcher as any).mcpWatchGeneration = 13+ ;(watcher as any).initialConvergenceRetryAttempts.set('NetworkPolicy', 4)+ ;(watcher as any).contexts.set('stale-context', {+ name: 'stale-context',+ namespace: 'mcp-server',+ spec: { contextId: 'stale-context', mcpServers: ['stale-server'] },+ })+ ;(watcher as any).servers.set('stale-server', {+ name: 'stale-server',+ namespace: 'mcp-server',+ spec: {+ contextRef: 'stale-context',+ image: 'clerum/stale-server:v1',+ transport: { type: 'streamableHttp' as const, port: 3000 },+ },+ })+ mocks.netPolFullReconcile.mockImplementation(async (contexts, servers, options) => {+ pass += 1+ if (pass === 1) {+ firstPassStarted.resolve(undefined)+ await releaseFirstPass.promise+ }+ await Promise.all(+ contexts.map((context: { spec: { contextId: string } }) =>+ options.runContextEffect(context.spec.contextId, async () => {+ appliedContexts.push(context.spec.contextId)+ })+ )+ )+ await Promise.all(+ servers.map((server: { name: string }) =>+ options.runServerEffect(server.name, async () => {+ appliedServers.push(server.name)+ })+ )+ )+ })++ const stalePass = (watcher as any).runInitialNetworkPolicyConvergence()+ await firstPassStarted.promise+ ;(watcher as any).contextCacheSynced = false+ ;(watcher as any).mcpServerCacheSynced = false+ ;(watcher as any).contextWatchGeneration = 12+ ;(watcher as any).mcpWatchGeneration = 14+ const trailing = (watcher as any).runInitialNetworkPolicyConvergence()+ releaseFirstPass.resolve(undefined)+ await Promise.all([stalePass, trailing])++ expect(appliedContexts).toEqual([])+ expect(appliedServers).toEqual([])+ expect((watcher as any).initialConvergenceRetryAttempts.get('NetworkPolicy')).toBe(4)+ ;(watcher as any).contexts.clear()+ ;(watcher as any).servers.clear()+ ;(watcher as any).contexts.set('current-context', {+ name: 'current-context',+ namespace: 'mcp-server',+ spec: { contextId: 'current-context', mcpServers: ['current-server'] },+ })+ ;(watcher as any).servers.set('current-server', {+ name: 'current-server',+ namespace: 'mcp-server',+ spec: {+ contextRef: 'current-context',+ image: 'clerum/current-server:v2',+ transport: { type: 'streamableHttp' as const, port: 3000 },+ },+ })+ ;(watcher as any).contextCacheSynced = true+ ;(watcher as any).mcpServerCacheSynced = true+ await (watcher as any).runInitialNetworkPolicyConvergence()++ expect(appliedContexts).toEqual(['current-context'])+ expect(appliedServers).toEqual(['current-server'])+ expect((watcher as any).initialConvergenceRetryAttempts.has('NetworkPolicy')).toBe(false)+ await watcher.stop()+ })++ it('does not certify NetworkPolicy revocation after authority is lost at the callback boundary', async () => {+ const watcher = new McpServerWatcher()+ ;(watcher as any).contextCacheSynced = true+ ;(watcher as any).mcpServerCacheSynced = true+ ;(watcher as any).contextWatchGeneration = 21+ ;(watcher as any).mcpWatchGeneration = 34+ mocks.netPolFullReconcile.mockImplementationOnce(async (_contexts, _servers, options) => {+ ;(watcher as any).contextCacheSynced = false+ options.onAuthoritativeRevocationComplete()+ })++ await (watcher as any).runInitialNetworkPolicyConvergence()++ // Authority lost at the callback boundary (contextCacheSynced=false) → record+ // refuses, so the content-identity revocation counters stay at the "never+ // certified" sentinel rather than adopting the captured revision.+ expect((watcher as any).networkPolicyRevocationContextRevision).toBe(-1)+ expect((watcher as any).networkPolicyRevocationServerRevision).toBe(-1)+ await watcher.stop()+ })++ it('wires onExternalEgressRevoked to schedule a MODIFIED external-egress retry (B3)', async () => {+ const watcher = new McpServerWatcher()+ ;(watcher as any).contextCacheSynced = true+ ;(watcher as any).mcpServerCacheSynced = true+ ;(watcher as any).contextWatchGeneration = 21+ ;(watcher as any).mcpWatchGeneration = 34+ const server = {+ name: 'openai-mcp',+ namespace: 'mcp-server',+ spec: {+ contextRef: 'dev',+ image: 'openai:latest',+ transport: { type: 'streamableHttp' as const, port: 3000 },+ egressBindings: [{ dns: 'api.openai.com', port: 443 }],+ },+ }+ const retrySpy = vi+ .spyOn((watcher as any).externalEgressCoordinator, 'scheduleRetry')+ .mockImplementation(() => undefined)+ let captured: { onExternalEgressRevoked?: (s: unknown) => void } | undefined+ mocks.netPolFullReconcile.mockImplementationOnce(+ async (_contexts: unknown[], _servers: unknown[], options: typeof captured) => {+ captured = options+ }+ )++ await (watcher as any).runInitialNetworkPolicyConvergence()++ expect(captured?.onExternalEgressRevoked).toBeTypeOf('function')+ captured!.onExternalEgressRevoked!(server)+ expect(retrySpy).toHaveBeenCalledWith('MODIFIED', server)+ await watcher.stop()+ })++ it('does not start runtime fleet convergence without a current NetworkPolicy safety certificate', async () => {+ const watcher = new McpServerWatcher()+ ;(watcher as any).contextCacheSynced = true+ ;(watcher as any).mcpServerCacheSynced = true+ ;(watcher as any).contextWatchGeneration = 4+ ;(watcher as any).mcpWatchGeneration = 7++ await (watcher as any).runInitialMcpServerConvergence()+ expect(mocks.serverFullReconcile).not.toHaveBeenCalled()++ await (watcher as any).runInitialNetworkPolicyConvergence()+ await vi.waitFor(() => expect(mocks.serverFullReconcile).toHaveBeenCalledOnce())++ await watcher.stop()+ })++ it('does not start runtime fleet convergence while the safety inventory is uncertified', async () => {+ // M1: with the revocation counters aligned, the ONLY thing that can null the+ // certificate is the lost safety fence. Fence lost -> cert null -> the fleet+ // lane must not run; fence restored -> it runs exactly once.+ const watcher = new McpServerWatcher()+ ;(watcher as any).contextCacheSynced = true+ ;(watcher as any).mcpServerCacheSynced = true+ ;(watcher as any).contextWatchGeneration = 4+ ;(watcher as any).mcpWatchGeneration = 7+ markNetworkPolicyRevocationAuthoritative(watcher)+ try {+ mocks.hasCertifiedSafetyInventory.mockReturnValue(false)+ await (watcher as any).runInitialMcpServerConvergence()+ expect(mocks.serverFullReconcile).not.toHaveBeenCalled()++ mocks.hasCertifiedSafetyInventory.mockReturnValue(true)+ await (watcher as any).runInitialMcpServerConvergence()+ expect(mocks.serverFullReconcile).toHaveBeenCalledOnce()+ } finally {+ // vi.clearAllMocks() clears calls, not implementations — restore explicitly+ // or every later test would re-gate on a stale false.+ mocks.hasCertifiedSafetyInventory.mockReturnValue(true)+ await watcher.stop()+ }+ })++ it('invalidates readiness when either desired policy revision advances', async () => {+ const watcher = new McpServerWatcher()+ ;(watcher as any).hostCacheSynced = true+ ;(watcher as any).contextCacheSynced = true+ ;(watcher as any).mcpServerCacheSynced = true+ ;(watcher as any).contextWatchGeneration = 4+ ;(watcher as any).mcpWatchGeneration = 7+ markNetworkPolicyRevocationAuthoritative(watcher)+ expect(watcher.isReadinessInventoryAuthoritative()).toBe(true)+ ;(watcher as any).mcpServerDesiredRevision += 1+ expect(watcher.isReadinessInventoryAuthoritative()).toBe(false)++ markNetworkPolicyRevocationAuthoritative(watcher)+ expect(watcher.isReadinessInventoryAuthoritative()).toBe(true)+ ;(watcher as any).contextDesiredRevision += 1+ expect(watcher.isReadinessInventoryAuthoritative()).toBe(false)++ await watcher.stop()+ })++ it('retires readiness and requests a safety pass before a changed McpServer effect settles', async () => {+ const previous = {+ metadata: {+ name: 'restrictive-change',+ namespace: 'mcp-server',+ uid: 'restrictive-change-uid',+ generation: 1,+ },+ spec: {+ contextRef: 'default',+ image: 'clerum/restrictive-change:test',+ transport: { type: 'streamableHttp' as const, port: 3000 },+ egressBindings: [{ dns: 'api.example.com', port: 443, protocol: 'TCP' as const }],+ },+ }+ const current = {+ ...previous,+ metadata: { ...previous.metadata, generation: 2 },+ spec: {+ ...previous.spec,+ egressBindings: [{ dns: 'api.example.com', port: 443, protocol: 'UDP' as const }],+ },+ }+ const watcher = new McpServerWatcher()+ ;(watcher as any).hostCacheSynced = true+ ;(watcher as any).contextCacheSynced = true+ ;(watcher as any).mcpServerCacheSynced = true+ ;(watcher as any).contextWatchGeneration = 4+ ;(watcher as any).mcpWatchGeneration = 7+ ;(watcher as any).servers.set(previous.metadata.name, {+ name: previous.metadata.name,+ namespace: previous.metadata.namespace,+ uid: previous.metadata.uid,+ generation: previous.metadata.generation,+ spec: previous.spec,+ })+ markNetworkPolicyRevocationAuthoritative(watcher)+ const safetyPass = vi+ .spyOn(watcher as any, 'runInitialNetworkPolicyConvergence')+ .mockResolvedValue(undefined)+ ;(watcher as any).netPolReconciler.reconcileExternalEgress.mockRejectedValueOnce(+ new Error('replacement policy unavailable')+ )+ const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {})++ await (watcher as any).getMcpServerWatchCallback(7)('MODIFIED', current)++ expect(watcher.isReadinessInventoryAuthoritative()).toBe(false)+ expect(safetyPass).toHaveBeenCalled()+ await watcher.stop()+ errorSpy.mockRestore()+ })++ it('fences runtime effects when a desired-revision change retires the safety certificate', async () => {+ const selected = {+ name: 'certificate-fenced-server',+ namespace: 'mcp-server',+ spec: {+ contextRef: 'default',+ image: 'clerum/certificate-fenced:test',+ transport: { type: 'streamableHttp' as const, port: 3000 },+ egressBindings: [{ dns: 'certificate-fenced.example', port: 443 }],+ },+ }+ const egressStarted = deferred()+ const releaseEgress = deferred()+ const runtimeEffect = vi.fn()+ const watcher = new McpServerWatcher()+ ;(watcher as any).contextCacheSynced = true+ ;(watcher as any).mcpServerCacheSynced = true+ ;(watcher as any).contextWatchGeneration = 11+ ;(watcher as any).mcpWatchGeneration = 13+ markNetworkPolicyRevocationAuthoritative(watcher)+ ;(watcher as any).servers.set(selected.name, selected)+ ;(watcher as any).netPolReconciler.reconcileExternalEgress.mockImplementationOnce(async () => {+ egressStarted.resolve(undefined)+ await releaseEgress.promise+ })+ mocks.serverFullReconcile.mockImplementationOnce(async (servers, options) => {+ await Promise.all(+ servers.map((server: { name: string }) => options.runEffect(server.name, runtimeEffect))+ )+ })++ const convergence = (watcher as any).runInitialMcpServerConvergence()+ await egressStarted.promise+ // A real desired-state change lands mid-convergence: the content identity+ // moves, retiring the captured safety certificate. A watch reconnect that+ // did NOT change content (generation-only bump) would correctly NOT fence —+ // that is the whole point of the content-identity gate — so the fence must+ // key on the revision, which this change advances.+ ;(watcher as any).contextDesiredRevision += 1+ releaseEgress.resolve(undefined)+ await convergence++ expect(runtimeEffect).not.toHaveBeenCalled()+ await watcher.stop()+ })++ it.each([+ ['SharedFileSystem', 'enqueueSharedFileSystemReconciliation'],+ ['GlobalFileSystem', 'enqueueGlobalFileSystemReconciliation'],+ ] as const)(+ 'does not admit queued %s effects after the watcher stops',+ async (_, enqueueMethod) => {+ const firstStarted = deferred()+ const releaseFirst = deferred()+ const watcher = new McpServerWatcher()+ let queuedEffectRan = false++ const first = (watcher as any)[enqueueMethod]('same-object', async () => {+ firstStarted.resolve(undefined)+ await releaseFirst.promise+ }) as Promise<void>+ await firstStarted.promise+ const queued = (watcher as any)[enqueueMethod]('same-object', async () => {+ queuedEffectRan = true+ }) as Promise<void>++ await watcher.stop()+ releaseFirst.resolve(undefined)+ await Promise.all([first, queued])++ expect(queuedEffectRan).toBe(false)+ }+ )++ it.each([+ ['SharedFileSystem', 'enqueueSharedFileSystemReconciliation'],+ ['GlobalFileSystem', 'enqueueGlobalFileSystemReconciliation'],+ ] as const)(+ 'serializes the same %s identity without blocking an unrelated fleet member',+ async (_, enqueueMethod) => {+ const firstStarted = deferred()+ const releaseFirst = deferred()+ const secondStarted = deferred()+ const unrelatedStarted = deferred()+ const watcher = new McpServerWatcher()+ let secondRan = false++ const first = (watcher as any)[enqueueMethod]('slow', async () => {+ firstStarted.resolve(undefined)+ await releaseFirst.promise+ }) as Promise<void>+ await firstStarted.promise+ const second = (watcher as any)[enqueueMethod]('slow', async () => {+ secondRan = true+ secondStarted.resolve(undefined)+ }) as Promise<void>+ const unrelated = (watcher as any)[enqueueMethod]('independent', async () => {+ unrelatedStarted.resolve(undefined)+ }) as Promise<void>++ await unrelatedStarted.promise+ expect(secondRan).toBe(false)+ releaseFirst.resolve(undefined)+ await Promise.all([first, second, unrelated])+ await expect(secondStarted.promise).resolves.toBeUndefined()+ await watcher.stop()+ }+ )++ it('keeps a keyed lane alive until a third waiter has consumed the same queue entry', async () => {+ const firstStarted = deferred()+ const releaseFirst = deferred()+ const secondStarted = deferred()+ const releaseSecond = deferred()+ const thirdStarted = deferred()+ let thirdRan = false+ const watcher = new McpServerWatcher()++ const first = (watcher as any).enqueueContextReconciliation('shared', async () => {+ firstStarted.resolve(undefined)+ await releaseFirst.promise+ })+ await firstStarted.promise+ const second = (watcher as any).enqueueContextReconciliation('shared', async () => {+ secondStarted.resolve(undefined)+ await releaseSecond.promise+ })+ const third = (watcher as any).enqueueContextReconciliation('shared', async () => {+ thirdRan = true+ thirdStarted.resolve(undefined)+ })++ expect((watcher as any).contextReconciliationQueues.get('shared').references).toBe(3)+ releaseFirst.resolve(undefined)+ await secondStarted.promise+ expect(thirdRan).toBe(false)+ expect((watcher as any).contextReconciliationQueues.get('shared').references).toBe(2)+ releaseSecond.resolve(undefined)+ await Promise.all([first, second, third])+ expect((watcher as any).contextReconciliationQueues.has('shared')).toBe(false)+ await watcher.stop()+ })++ it('caps initial convergence retry delay after attempts exceed the schedule', async () => {+ vi.useFakeTimers()+ const watcher = new McpServerWatcher()+ const convergence = vi+ .spyOn(watcher as any, 'runInitialMcpServerConvergence')+ .mockResolvedValue(undefined)+ ;(watcher as any).initialConvergenceRetryAttempts.set('McpServer', 99)+ ;(watcher as any).scheduleInitialConvergenceRetry('McpServer')+ await vi.advanceTimersByTimeAsync(299_999)+ expect(convergence).not.toHaveBeenCalled()+ await vi.advanceTimersByTimeAsync(1)+ expect(convergence).toHaveBeenCalledOnce()+ await watcher.stop()+ })++ it('defers NetworkPolicy full convergence until both inventories are authoritative', async () => {+ const watcher = new McpServerWatcher()+ ;(watcher as any).contextCacheSynced = true+ ;(watcher as any).mcpServerCacheSynced = false+ ;(watcher as any).initialConvergenceRetryAttempts.set('NetworkPolicy', 2)++ await (watcher as any).runInitialNetworkPolicyConvergence()++ expect(mocks.netPolFullReconcile).not.toHaveBeenCalled()+ expect((watcher as any).initialConvergenceRetryAttempts.get('NetworkPolicy')).toBe(2)+ ;(watcher as any).mcpServerCacheSynced = true+ await (watcher as any).runInitialNetworkPolicyConvergence()++ expect(mocks.netPolFullReconcile).toHaveBeenCalledOnce()+ expect(mocks.netPolFullReconcile).toHaveBeenLastCalledWith(+ [],+ [],+ expect.objectContaining({+ serverInventoryComplete: true,+ ensureDefaults: false,+ contextInventoryAuthoritative: expect.any(Function),+ serverInventoryAuthoritative: expect.any(Function),+ })+ )+ expect((watcher as any).initialConvergenceRetryAttempts.has('NetworkPolicy')).toBe(false)+ await watcher.stop()+ })++ it('fences NetworkPolicy orphan cleanup to the inventory generations captured by the pass', async () => {+ const watcher = new McpServerWatcher()+ ;(watcher as any).contextCacheSynced = true+ ;(watcher as any).mcpServerCacheSynced = true+ ;(watcher as any).contextWatchGeneration = 17+ ;(watcher as any).mcpWatchGeneration = 23+ let options:+ | {+ contextInventoryAuthoritative: () => boolean+ serverInventoryAuthoritative: () => boolean+ }+ | undefined+ mocks.netPolFullReconcile.mockImplementationOnce(+ async (+ _contexts: unknown[],+ _servers: unknown[],+ received: {+ contextInventoryAuthoritative: () => boolean+ serverInventoryAuthoritative: () => boolean+ }+ ) => {+ options = received+ }+ )++ await (watcher as any).runInitialNetworkPolicyConvergence()++ expect(options?.contextInventoryAuthoritative()).toBe(true)+ expect(options?.serverInventoryAuthoritative()).toBe(true)++ // A recovered watch may already be authoritative again, but it is a+ // different snapshot generation and cannot authorize cleanup by this pass.+ ;(watcher as any).contextWatchGeneration = 18+ ;(watcher as any).mcpWatchGeneration = 24++ expect(options?.contextInventoryAuthoritative()).toBe(false)+ expect(options?.serverInventoryAuthoritative()).toBe(false)+ await watcher.stop()+ })++ it('escalates and resets failed initial McpServer convergence without delaying bootstrap', async () => {+ vi.useFakeTimers()+ const server = {+ metadata: { name: 'initial-runtime', namespace: 'mcp-server' },+ spec: {+ contextRef: 'default',+ image: 'clerum/initial-runtime:test',+ transport: { type: 'streamableHttp' as const, port: 3000 },+ },+ }+ mocks.listNamespacedCustomObject.mockImplementation(async ({ plural }: { plural: string }) => {+ if (plural === 'mcpservers') return { items: [server] }+ if (plural === 'contexts') return { items: [] }+ if (plural === 'communicationchannels') {+ return { metadata: { resourceVersion: 'mcp-retry-rv' }, items: [] }+ }+ return { items: [] }+ })+ mocks.serverFullReconcile+ .mockRejectedValueOnce(new Error('initial runtime reconciliation failed'))+ .mockRejectedValueOnce(new Error('initial runtime reconciliation still unavailable'))+ .mockResolvedValueOnce(undefined)+ const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {})+ const retryMetricBefore = await readInitialConvergenceMetric(+ 'clerum_hcc_initial_convergence_retries_total',+ 'McpServer'+ )+ const watcher = new McpServerWatcher()+ stubAuthoritativeInventoryWatch(watcher, 'McpServer')+ stubAuthoritativeInventoryWatch(watcher, 'Context')+ vi.spyOn(watcher as any, 'startSharedFileSystemWatch').mockResolvedValue(undefined)+ vi.spyOn(watcher as any, 'startGlobalFileSystemWatch').mockResolvedValue(undefined)+ vi.spyOn(watcher as any, 'startCommunicationChannelWatch').mockResolvedValue(undefined)++ await watcher.start()+ await vi.advanceTimersByTimeAsync(0)+ expect(mocks.serverFullReconcile).toHaveBeenCalledTimes(1)++ await vi.advanceTimersByTimeAsync(5000)+ expect(mocks.serverFullReconcile).toHaveBeenCalledTimes(2)++ await vi.advanceTimersByTimeAsync(14_999)+ expect(mocks.serverFullReconcile).toHaveBeenCalledTimes(2)+ await vi.advanceTimersByTimeAsync(1)+ expect(mocks.serverFullReconcile).toHaveBeenCalledTimes(3)+ expect((watcher as any).initialConvergenceRetryAttempts.has('McpServer')).toBe(false)+ expect((watcher as any).initialConvergenceRetryTimers.has('McpServer')).toBe(false)+ expect(+ await readInitialConvergenceMetric(+ 'clerum_hcc_initial_convergence_retries_total',+ 'McpServer'+ )+ ).toBe(retryMetricBefore + 2)+ expect(+ await readInitialConvergenceMetric(+ 'clerum_hcc_initial_convergence_last_success_timestamp_seconds',+ 'McpServer'+ )+ ).toBeGreaterThan(0)++ await watcher.stop()+ errorSpy.mockRestore()+ })++ it('cancels a pending initial convergence retry when the watcher stops', async () => {+ vi.useFakeTimers()+ const watcher = new McpServerWatcher()+ ;(watcher as any).mcpServerCacheSynced = true+ ;(watcher as any).contextCacheSynced = true+ markNetworkPolicyRevocationAuthoritative(watcher)+ mocks.serverFullReconcile.mockRejectedValueOnce(+ new Error('initial runtime reconciliation unavailable')+ )+ const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {})++ await (watcher as any).runInitialMcpServerConvergence()++ expect(mocks.serverFullReconcile).toHaveBeenCalledTimes(1)+ expect((watcher as any).initialConvergenceRetryAttempts.get('McpServer')).toBe(1)+ expect((watcher as any).initialConvergenceRetryTimers.has('McpServer')).toBe(true)++ await watcher.stop()+ await vi.advanceTimersByTimeAsync(300_000)++ expect(mocks.serverFullReconcile).toHaveBeenCalledTimes(1)+ expect((watcher as any).initialConvergenceRetryAttempts.size).toBe(0)+ expect((watcher as any).initialConvergenceRetryTimers.size).toBe(0)+ errorSpy.mockRestore()+ })++ it('retries failed initial NetworkPolicy convergence from the current caches', async () => {+ vi.useFakeTimers()+ const context = {+ metadata: { name: 'initial-context', namespace: 'mcp-server' },+ spec: { contextId: 'initial-context', mcpServers: [] },+ }+ mocks.listNamespacedCustomObject.mockImplementation(async ({ plural }: { plural: string }) => {+ if (plural === 'mcpservers') return { items: [] }+ if (plural === 'contexts') return { items: [context] }+ if (plural === 'communicationchannels') {+ return { metadata: { resourceVersion: 'netpol-retry-rv' }, items: [] }+ }+ return { items: [] }+ })+ mocks.netPolFullReconcile+ .mockRejectedValueOnce(new Error('initial NetworkPolicy reconciliation failed'))+ .mockImplementationOnce(+ async (+ _contexts: unknown,+ _servers: unknown,+ options?: { onAuthoritativeRevocationComplete?: () => void }+ ) => {+ options?.onAuthoritativeRevocationComplete?.()+ }+ )+ const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {})+ const watcher = new McpServerWatcher()+ stubAuthoritativeInventoryWatch(watcher, 'McpServer')+ stubAuthoritativeInventoryWatch(watcher, 'Context')+ vi.spyOn(watcher as any, 'startSharedFileSystemWatch').mockResolvedValue(undefined)+ vi.spyOn(watcher as any, 'startGlobalFileSystemWatch').mockResolvedValue(undefined)+ vi.spyOn(watcher as any, 'startCommunicationChannelWatch').mockResolvedValue(undefined)++ await watcher.start()+ await vi.advanceTimersByTimeAsync(0)+ expect(mocks.netPolFullReconcile).toHaveBeenCalledTimes(1)+ expect(watcher.isReadinessInventoryAuthoritative()).toBe(false)++ await vi.advanceTimersByTimeAsync(5000)+ expect(mocks.netPolFullReconcile).toHaveBeenCalledTimes(2)+ expect(watcher.isReadinessInventoryAuthoritative()).toBe(true)+ expect(mocks.netPolFullReconcile).toHaveBeenLastCalledWith(+ [expect.objectContaining({ name: 'initial-context' })],+ [],+ expect.objectContaining({+ serverInventoryComplete: true,+ ensureDefaults: false,+ contextInventoryAuthoritative: expect.any(Function),+ serverInventoryAuthoritative: expect.any(Function),+ })+ )++ await watcher.stop()+ errorSpy.mockRestore()+ })++ it('does not report a post-certification additive failure as a readiness safety failure', async () => {+ vi.useFakeTimers()+ const additiveFailure = new AggregateError(+ [new Error('Context policy API unavailable')],+ 'One or more additive Context NetworkPolicy reconciliations failed'+ )+ mocks.netPolFullReconcile.mockImplementationOnce(+ async (+ _contexts: unknown,+ _servers: unknown,+ options?: { onAuthoritativeRevocationComplete?: () => void }+ ) => {+ options?.onAuthoritativeRevocationComplete?.()+ throw additiveFailure+ }+ )+ const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {})+ const watcher = new McpServerWatcher()+ stubAuthoritativeInventoryWatch(watcher, 'McpServer')+ stubAuthoritativeInventoryWatch(watcher, 'Context')+ vi.spyOn(watcher as any, 'startSharedFileSystemWatch').mockResolvedValue(undefined)+ vi.spyOn(watcher as any, 'startGlobalFileSystemWatch').mockResolvedValue(undefined)+ vi.spyOn(watcher as any, 'startCommunicationChannelWatch').mockResolvedValue(undefined)++ await watcher.start()+ await vi.advanceTimersByTimeAsync(0)++ expect(watcher.isReadinessInventoryAuthoritative()).toBe(true)+ expect(errorSpy).toHaveBeenCalledWith(+ '[K8s] Initial NetworkPolicy post-certification additive reconciliation failed:',+ additiveFailure+ )+ expect(errorSpy).not.toHaveBeenCalledWith(+ '[K8s] Initial NetworkPolicy background reconciliation failed:',+ additiveFailure+ )+ expect((watcher as any).initialConvergenceRetryTimers.has('NetworkPolicy')).toBe(true)++ await watcher.stop()+ errorSpy.mockRestore()+ })++ it.each([+ ['McpServer', 'startMcpServerWatch'],+ ['Context', 'startContextWatch'],+ ] as const)(+ 'keeps safe bootstrap authority-gated when the %s watch cannot be established',+ async (_kind, method) => {+ const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {})+ const warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => {})+ const watcher = new McpServerWatcher()+ if (method === 'startMcpServerWatch') {+ vi.spyOn(watcher as any, method).mockRejectedValue(new Error(`${method} unavailable`))+ } else {+ stubAuthoritativeInventoryWatch(watcher, 'McpServer')+ }+ if (method === 'startContextWatch') {+ vi.spyOn(watcher as any, method).mockRejectedValue(new Error(`${method} unavailable`))+ } else if (method !== 'startMcpServerWatch') {+ stubAuthoritativeInventoryWatch(watcher, 'Context')+ }+ vi.spyOn(watcher as any, 'startCommunicationChannelWatch').mockResolvedValue(undefined)+ const recoveryTimerField =+ method === 'startMcpServerWatch'+ ? 'mcpServerCacheRecoveryTimer'+ : 'contextCacheRecoveryTimer'++ try {+ await expect(watcher.start()).resolves.toBeUndefined()+ expect(watcher.isReadinessInventoryAuthoritative()).toBe(false)+ expect((watcher as any)[recoveryTimerField]).not.toBeNull()+ expect(mocks.netPolFullReconcile).not.toHaveBeenCalled()+ } finally {+ await watcher.stop()+ errorSpy.mockRestore()+ warnSpy.mockRestore()+ }+ }+ )++ it.each([+ ['SharedFileSystem', 'startSharedFileSystemWatch'],+ ['GlobalFileSystem', 'startGlobalFileSystemWatch'],+ ] as const)(+ 'isolates, logs, and retries an initial %s background watch rejection',+ async (kind, method) => {+ vi.useFakeTimers()+ const failure = new Error(`${method} unavailable`)+ const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {})+ const watcher = new McpServerWatcher()+ stubAuthoritativeInventoryWatch(watcher, 'McpServer')+ stubAuthoritativeInventoryWatch(watcher, 'Context')+ const rejectedWatch = vi+ .spyOn(watcher as any, method)+ .mockRejectedValueOnce(failure)+ .mockResolvedValueOnce(undefined)+ const otherMethod =+ method === 'startSharedFileSystemWatch'+ ? 'startGlobalFileSystemWatch'+ : 'startSharedFileSystemWatch'+ vi.spyOn(watcher as any, otherMethod).mockResolvedValue(undefined)+ vi.spyOn(watcher as any, 'startCommunicationChannelWatch').mockResolvedValue(undefined)++ await expect(watcher.start()).resolves.toBeUndefined()+ await vi.advanceTimersByTimeAsync(0)++ expect(errorSpy).toHaveBeenCalledWith(+ `[K8s] ${kind} background watch failed to start:`,+ failure+ )+ expect(rejectedWatch).toHaveBeenCalledOnce()+ expect(mocks.netPolFullReconcile).toHaveBeenCalledOnce()++ await vi.advanceTimersByTimeAsync(4_999)+ expect(rejectedWatch).toHaveBeenCalledOnce()+ await vi.advanceTimersByTimeAsync(1)+ expect(rejectedWatch).toHaveBeenCalledTimes(2)++ await watcher.stop()+ errorSpy.mockRestore()+ }+ )++ it('applies the request deadline wrapper to the finite Host inventory LIST', async () => {+ mocks.listNamespacedCustomObject.mockImplementation(async ({ plural }: { plural: string }) => {+ if (plural === 'hosts') return { items: [] }+ return { items: [] }+ })++ await expect(listAllHosts()).resolves.toEqual([])++ expect(mocks.hostListCallOptions).toHaveBeenCalledWith(+ expect.objectContaining({+ middleware: [expect.objectContaining({ pre: expect.any(Function) })],+ middlewareMergeStrategy: 'append',+ })+ )+ })++ it('preserves Host UID from the Kubernetes inventory boundary', async () => {+ mocks.listNamespacedCustomObject.mockImplementation(async ({ plural }: { plural: string }) => {+ if (plural === 'hosts') {+ return {+ items: [+ {+ metadata: {+ name: 'uid-host',+ namespace: 'mcp-host',+ uid: '9f43826a-4031-4a31-93af-a3dd8fcfe805',+ generation: 4,+ },+ spec: { host: 'uid-host', contextRef: 'context-a', secretRef: 'host-secret' },+ },+ ],+ }+ }+ return { items: [] }+ })++ await expect(listAllHosts()).resolves.toEqual([+ expect.objectContaining({+ name: 'uid-host',+ uid: '9f43826a-4031-4a31-93af-a3dd8fcfe805',+ generation: 4,+ }),+ ])+ })++ it('preserves observed GlobalFileSystem status from the Kubernetes inventory boundary', async () => {+ const status = {+ phase: 'Ready',+ pvcName: 'gfs-drive',+ serviceName: 'gfsc',+ serviceUrl: 'http://gfsc.gfs.svc.cluster.local:8087',+ }+ mocks.listNamespacedCustomObject.mockImplementation(async ({ plural }: { plural: string }) => {+ if (plural === 'globalfilesystems') {+ return {+ items: [+ {+ metadata: { name: 'gfs', namespace: 'gfs' },+ spec: {},+ status,+ },+ ],+ }+ }+ return { items: [] }+ })++ await expect(listAllGlobalFileSystems()).resolves.toEqual([+ expect.objectContaining({ name: 'gfs', namespace: 'gfs', status }),+ ])+ })++ it('reconciles startup external egress before runtime full reconciliation', async () => {+ const eventLog: string[] = []+ const server = {+ metadata: { name: 'web-search', namespace: 'mcp-server', generation: 4 },+ spec: {+ contextRef: 'wf-research',+ image: 'clerum/web-search:test',+ transport: { type: 'streamableHttp' as const, port: 3000 },+ egressBindings: [{ dns: 'duckduckgo.com', port: 443 }],+ },+ }++ mocks.listNamespacedCustomObject.mockImplementation(async ({ plural }: { plural: string }) => {+ if (plural === 'mcpservers') return { items: [server] }+ if (plural === 'contexts') return { items: [] }+ if (plural === 'hosts') return { items: [] }+ if (plural === 'sharedfilesystems') return { items: [] }+ if (plural === 'communicationchannels')+ return { metadata: { resourceVersion: '1' }, items: [] }+ return { items: [] }+ })+ mocks.ensureDefaultPolicies.mockImplementation(async () => {+ eventLog.push('defaults')+ })+ const watcher = new McpServerWatcher()+ const reconciler = (watcher as any).reconciler+ mocks.serverFullReconcile.mockImplementation(async (servers, options) => {+ await Promise.all(+ servers.map((selected: { name: string }) =>+ options.runEffect(selected.name, async () => {+ eventLog.push('runtime')+ await reconciler.reconcile(selected)+ })+ )+ )+ })+ const netPol = (watcher as any).netPolReconciler+ netPol.reconcileExternalEgress.mockImplementation(async () => {+ eventLog.push('egress')+ })+ stubAuthoritativeInventoryWatch(watcher, 'McpServer')+ stubAuthoritativeInventoryWatch(watcher, 'Context')+ vi.spyOn(watcher as any, 'startSharedFileSystemWatch').mockImplementation(async () => undefined)+ vi.spyOn(watcher as any, 'startGlobalFileSystemWatch').mockImplementation(async () => undefined)+ vi.spyOn(watcher as any, 'startCommunicationChannelWatch').mockImplementation(+ async () => undefined+ )++ await watcher.start()++ await vi.waitFor(() => expect(eventLog).toContain('runtime'))+ expect(eventLog.slice(0, 3)).toEqual(['defaults', 'egress', 'runtime'])+ expect(mocks.serverFullReconcile).toHaveBeenCalledWith(+ [expect.objectContaining({ name: 'web-search' })],+ { runEffect: expect.any(Function) }+ )++ watcher.stop()+ })++ it('skips startup runtime reconciliation for servers whose external egress is not ready', async () => {+ vi.useFakeTimers()+ const server = {+ metadata: { name: 'web-search', namespace: 'mcp-server', generation: 4 },+ spec: {+ contextRef: 'wf-research',+ image: 'clerum/web-search:test',+ transport: { type: 'streamableHttp' as const, port: 3000 },+ egressBindings: [{ dns: 'duckduckgo.com', port: 443 }],+ },+ }++ mocks.listNamespacedCustomObject.mockImplementation(async ({ plural }: { plural: string }) => {+ if (plural === 'mcpservers') return { items: [server] }+ if (plural === 'contexts') return { items: [] }+ if (plural === 'hosts') return { items: [] }+ if (plural === 'sharedfilesystems') return { items: [] }+ if (plural === 'communicationchannels')+ return { metadata: { resourceVersion: '1' }, items: [] }+ return { items: [] }+ })++ const watcher = new McpServerWatcher()+ const reconciler = (watcher as any).reconciler+ mocks.serverFullReconcile.mockImplementation(async (servers, options) => {+ await Promise.all(+ servers.map((selected: { name: string }) =>+ options.runEffect(selected.name, () => reconciler.reconcile(selected))+ )+ )+ })+ const netPol = (watcher as any).netPolReconciler+ const dnsFailure = new Error('dns resolution failed')+ netPol.reconcileExternalEgress.mockRejectedValueOnce(dnsFailure)+ const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {})+ const warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => {})+ stubAuthoritativeInventoryWatch(watcher, 'McpServer')+ stubAuthoritativeInventoryWatch(watcher, 'Context')+ vi.spyOn(watcher as any, 'startSharedFileSystemWatch').mockImplementation(async () => undefined)+ vi.spyOn(watcher as any, 'startGlobalFileSystemWatch').mockImplementation(async () => undefined)+ vi.spyOn(watcher as any, 'startCommunicationChannelWatch').mockImplementation(+ async () => undefined+ )++ await watcher.start()++ await vi.waitFor(() => expect(mocks.serverFullReconcile).toHaveBeenCalledOnce())+ expect(mocks.serverFullReconcile).toHaveBeenCalledWith(+ [expect.objectContaining({ name: 'web-search' })],+ { runEffect: expect.any(Function) }+ )+ expect(netPol.reconcileExternalEgress).toHaveBeenCalledTimes(1)+ expect(reconciler.reconcile).not.toHaveBeenCalled()+ expect(watcher.isReadinessInventoryAuthoritative()).toBe(true)+ expect((watcher as any).initialConvergenceRetryTimers.has('NetworkPolicy')).toBe(false)+ expect(errorSpy).not.toHaveBeenCalledWith(+ '[K8s] Initial NetworkPolicy background reconciliation failed:',+ dnsFailure+ )++ watcher.stop()+ errorSpy.mockRestore()+ warnSpy.mockRestore()+ })+ })++ describe('getContext error semantics', () => {+ beforeEach(() => {+ vi.clearAllMocks()+ })++ it('preserves not-found semantics only for an authoritative 404', async () => {+ mocks.getNamespacedCustomObject.mockRejectedValueOnce(+ Object.assign(new Error('context absent'), { code: 404 })+ )++ await expect(getContext('missing-context')).resolves.toBeNull()+ })++ it('propagates non-404 API failures instead of converting discovery to an empty fleet', async () => {+ const unavailable = Object.assign(new Error('apiserver unavailable'), {+ response: { statusCode: 503 },+ })+ mocks.getNamespacedCustomObject.mockRejectedValueOnce(unavailable)++ await expect(getContext('production-context')).rejects.toBe(unavailable)+ })+ })+++describe('MCP authorization Kubernetes not-found classification', () => {++ it.each([{ code: 404 }, { response: { statusCode: 404 } }])(++ 'recognizes Kubernetes 404 shape %#',++ error => {++ expect(isMcpAuthorizationNotFound(error)).toBe(true)++ }++ )++++ it('does not classify authority failures as not-found', () => {++ expect(isMcpAuthorizationNotFound({ code: 403 })).toBe(false)++ expect(isMcpAuthorizationNotFound({ response: { statusCode: 503 } })).toBe(false)++ })++})+++ describe('McpServerWatcher external egress retries', () => {+ const serverObject = {+ metadata: { name: 'redis-tools', namespace: 'mcp-server' },+ spec: {+ contextRef: 'context1',+ image: 'redis-tools:test',+ transport: { type: 'streamableHttp' as const, port: 3000 },+ egressBindings: [{ dns: 'example.com', port: 443 }],+ },+ }++ beforeEach(() => {+ vi.clearAllMocks()+ vi.useFakeTimers()+ })++ afterEach(() => {+ vi.useRealTimers()+ vi.restoreAllMocks()+ })++ it('retries failed external egress reconciliation from an ADDED event', async () => {+ const watcher = new McpServerWatcher()+ markMcpServerInventoryAuthoritative(watcher)+ const netPol = (watcher as any).netPolReconciler+ const reconciler = (watcher as any).reconciler+ const bindingReconciler = (watcher as any).bindingReconciler+ const serverWithBindings = {+ ...serverObject,+ metadata: {+ ...serverObject.metadata,+ annotations: {+ 'clerum.io/recipe-bindings': JSON.stringify([+ { source: 'redis-tools', target: 'downstream' },+ ]),+ },+ },+ }+ ;(watcher as any).contexts.set('context1', {+ name: 'context1',+ namespace: 'mcp-server',+ spec: { contextId: 'context1', mcpServers: ['redis-tools'] },+ })+ netPol.reconcileExternalEgress.mockRejectedValueOnce(new Error('api temporarily unavailable'))+ netPol.reconcileExternalEgress.mockResolvedValue(undefined)+ vi.spyOn(console, 'error').mockImplementation(() => {})+ vi.spyOn(console, 'warn').mockImplementation(() => {})++ const watchCallback = (watcher as any).getMcpServerWatchCallback()+ await watchCallback('ADDED', serverWithBindings)++ expect(netPol.reconcileExternalEgress).toHaveBeenCalledTimes(1)+ expect(bindingReconciler.reconcileBindings).not.toHaveBeenCalled()+ expect(netPol.reconcileContext).not.toHaveBeenCalled()++ await vi.advanceTimersByTimeAsync(5000)++ expect(netPol.reconcileExternalEgress).toHaveBeenCalledTimes(2)+ expect(reconciler.reconcile).toHaveBeenCalledTimes(1)+ expect(reconciler.reconcile).toHaveBeenCalledWith(+ expect.objectContaining({ name: 'redis-tools' }),+ { isCurrent: expect.any(Function) }+ )+ expect(bindingReconciler.reconcileBindings).toHaveBeenCalledTimes(1)+ expect(netPol.reconcileContext).toHaveBeenCalledWith(+ expect.objectContaining({+ spec: expect.objectContaining({ contextId: 'context1' }),+ }),+ expect.objectContaining({ isCurrent: expect.any(Function) })+ )++ watcher.stop()+ })++ it('retires a binding-policy lease when a same-watch MODIFIED supersedes the server', async () => {+ const watcher = new McpServerWatcher()+ markMcpServerInventoryAuthoritative(watcher)+ const bindingReconciler = (watcher as any).bindingReconciler+ const bindingStarted = deferred()+ const releaseBinding = deferred()+ bindingReconciler.reconcileBindings.mockImplementationOnce(async () => {+ bindingStarted.resolve(undefined)+ await releaseBinding.promise+ })+ const annotations = {+ 'clerum.io/recipe-bindings': JSON.stringify([+ { from: 'redis-tools', to: 'downstream', port: 443 },+ ]),+ }+ const original = {+ ...serverObject,+ metadata: {+ ...serverObject.metadata,+ uid: 'redis-tools-uid',+ generation: 1,+ annotations,+ },+ }+ const replacement = {+ ...original,+ metadata: { ...original.metadata, generation: 2 },+ spec: { ...original.spec, image: 'redis-tools:v2' },+ }+ const watchCallback = (watcher as any).getMcpServerWatchCallback()++ const originalEffect = watchCallback('ADDED', original)+ await bindingStarted.promise+ const replacementEffect = watchCallback('MODIFIED', replacement)++ const originalOptions = bindingReconciler.reconcileBindings.mock.calls[0][4]+ expect(originalOptions).toEqual({ isCurrent: expect.any(Function) })+ expect(originalOptions.isCurrent()).toBe(false)++ releaseBinding.resolve(undefined)+ await Promise.all([originalEffect, replacementEffect])++ expect(bindingReconciler.reconcileBindings).toHaveBeenCalledTimes(2)+ expect(bindingReconciler.reconcileBindings).toHaveBeenLastCalledWith(+ 'redis-tools',+ expect.any(Array),+ 'redis-tools',+ 'redis-tools',+ { isCurrent: expect.any(Function) }+ )+ await watcher.stop()+ })++ it('retires binding cleanup when a same-name McpServer is recreated during DELETE', async () => {+ const watcher = new McpServerWatcher()+ ;(watcher as any).mcpServerCacheSynced = true+ const bindingReconciler = (watcher as any).bindingReconciler+ const cleanupStarted = deferred()+ const releaseCleanup = deferred()+ bindingReconciler.cleanupBindings.mockImplementationOnce(async () => {+ cleanupStarted.resolve(undefined)+ await releaseCleanup.promise+ })+ const deleted = {+ ...serverObject,+ metadata: {+ ...serverObject.metadata,+ uid: 'redis-tools-old-uid',+ generation: 1,+ },+ }+ const recreated = {+ ...serverObject,+ metadata: {+ ...serverObject.metadata,+ uid: 'redis-tools-new-uid',+ generation: 1,+ },+ spec: { ...serverObject.spec, image: 'redis-tools:recreated' },+ }+ const watchCallback = (watcher as any).getMcpServerWatchCallback()++ const deleteEffect = watchCallback('DELETED', deleted)+ await cleanupStarted.promise+ const recreateEffect = watchCallback('ADDED', recreated)++ const cleanupOptions = bindingReconciler.cleanupBindings.mock.calls[0][1]+ expect(cleanupOptions).toEqual({ deleteAllowed: expect.any(Function) })+ await expect(cleanupOptions.deleteAllowed()).resolves.toBe(false)++ releaseCleanup.resolve(undefined)+ await Promise.all([deleteEffect, recreateEffect])++ expect(bindingReconciler.cleanupBindings).toHaveBeenCalledTimes(1)+ await watcher.stop()+ })++ it('retries the complete pipeline when a no-egress runtime reconcile fails', async () => {+ const watcher = new McpServerWatcher()+ markMcpServerInventoryAuthoritative(watcher)+ const reconciler = (watcher as any).reconciler+ const noEgressServer = {+ ...serverObject,+ spec: {+ ...serverObject.spec,+ egressBindings: undefined,+ },+ }+ reconciler.reconcile+ .mockRejectedValueOnce(new Error('deployment temporarily unavailable'))+ .mockResolvedValue(undefined)+ vi.spyOn(console, 'error').mockImplementation(() => {})+ vi.spyOn(console, 'warn').mockImplementation(() => {})++ await (watcher as any).getMcpServerWatchCallback()('ADDED', noEgressServer)+ expect(reconciler.reconcile).toHaveBeenCalledTimes(1)++ await vi.advanceTimersByTimeAsync(5000)++ expect(reconciler.reconcile).toHaveBeenCalledTimes(2)+ watcher.stop()+ })++ it('retries runtime reconciliation when external egress is ready but runtime reconcile fails', async () => {+ const watcher = new McpServerWatcher()+ markMcpServerInventoryAuthoritative(watcher)+ const netPol = (watcher as any).netPolReconciler+ const reconciler = (watcher as any).reconciler+ netPol.reconcileExternalEgress.mockResolvedValue(undefined)+ reconciler.reconcile.mockRejectedValueOnce(new Error('deployment temporarily unavailable'))+ reconciler.reconcile.mockResolvedValue(undefined)+ vi.spyOn(console, 'error').mockImplementation(() => {})+ vi.spyOn(console, 'warn').mockImplementation(() => {})++ const watchCallback = (watcher as any).getMcpServerWatchCallback()+ await watchCallback('ADDED', serverObject)++ expect(netPol.reconcileExternalEgress).toHaveBeenCalledTimes(1)+ expect(reconciler.reconcile).toHaveBeenCalledTimes(1)++ await vi.advanceTimersByTimeAsync(5000)++ expect(netPol.reconcileExternalEgress).toHaveBeenCalledTimes(2)+ expect(reconciler.reconcile).toHaveBeenCalledTimes(2)+ expect(reconciler.reconcile).toHaveBeenLastCalledWith(+ expect.objectContaining({ name: 'redis-tools' }),+ { isCurrent: expect.any(Function) }+ )++ watcher.stop()+ })++ it('keeps retry attempts bounded when runtime fails after an external egress retry succeeds', async () => {+ const watcher = new McpServerWatcher()+ markMcpServerInventoryAuthoritative(watcher)+ const netPol = (watcher as any).netPolReconciler+ const reconciler = (watcher as any).reconciler+ netPol.reconcileExternalEgress.mockRejectedValueOnce(new Error('api temporarily unavailable'))+ netPol.reconcileExternalEgress.mockResolvedValue(undefined)+ reconciler.reconcile.mockRejectedValueOnce(new Error('deployment temporarily unavailable'))+ reconciler.reconcile.mockResolvedValue(undefined)+ vi.spyOn(console, 'error').mockImplementation(() => {})+ vi.spyOn(console, 'warn').mockImplementation(() => {})++ const watchCallback = (watcher as any).getMcpServerWatchCallback()+ await watchCallback('ADDED', serverObject)++ expect(netPol.reconcileExternalEgress).toHaveBeenCalledTimes(1)+ expect(reconciler.reconcile).not.toHaveBeenCalled()++ await vi.advanceTimersByTimeAsync(5000)++ expect(netPol.reconcileExternalEgress).toHaveBeenCalledTimes(2)+ expect(reconciler.reconcile).toHaveBeenCalledTimes(1)++ await vi.advanceTimersByTimeAsync(15000)++ expect(netPol.reconcileExternalEgress).toHaveBeenCalledTimes(3)+ expect(reconciler.reconcile).toHaveBeenCalledTimes(2)+ expect(reconciler.reconcile).toHaveBeenLastCalledWith(+ expect.objectContaining({ name: 'redis-tools' }),+ { isCurrent: expect.any(Function) }+ )++ watcher.stop()+ })++ it('continues a pending runtime pipeline during periodic external egress resync', async () => {+ const watcher = new McpServerWatcher()+ markMcpServerInventoryAuthoritative(watcher)+ const netPol = (watcher as any).netPolReconciler+ const reconciler = (watcher as any).reconciler+ netPol.reconcileExternalEgress.mockResolvedValue(undefined)+ reconciler.reconcile.mockRejectedValueOnce(new Error('deployment temporarily unavailable'))+ reconciler.reconcile.mockResolvedValue(undefined)+ vi.spyOn(console, 'error').mockImplementation(() => {})+ vi.spyOn(console, 'warn').mockImplementation(() => {})+ vi.spyOn(console, 'log').mockImplementation(() => {})+ vi.spyOn(Math, 'random').mockReturnValue(0)++ const watchCallback = (watcher as any).getMcpServerWatchCallback()+ await watchCallback('ADDED', serverObject)++ expect(netPol.reconcileExternalEgress).toHaveBeenCalledTimes(1)+ expect(reconciler.reconcile).toHaveBeenCalledTimes(1)++ await (watcher as any).externalEgressCoordinator.runResync()++ expect(netPol.reconcileExternalEgress).toHaveBeenCalledTimes(2)+ expect(reconciler.reconcile).toHaveBeenCalledTimes(2)++ await vi.advanceTimersByTimeAsync(5000)+ expect(netPol.reconcileExternalEgress).toHaveBeenCalledTimes(2)+ expect(reconciler.reconcile).toHaveBeenCalledTimes(2)++ watcher.stop()+ })++ it('keeps retrying an ADDED pipeline at the capped delay until it recovers', async () => {+ const watcher = new McpServerWatcher()+ markMcpServerInventoryAuthoritative(watcher)+ const netPol = (watcher as any).netPolReconciler+ const reconciler = (watcher as any).reconciler+ netPol.reconcileExternalEgress+ .mockRejectedValueOnce(new Error('initial outage'))+ .mockRejectedValueOnce(new Error('outage after 5s'))+ .mockRejectedValueOnce(new Error('outage after 15s'))+ .mockRejectedValueOnce(new Error('outage after 30s'))+ .mockResolvedValue(undefined)+ vi.spyOn(console, 'error').mockImplementation(() => {})+ vi.spyOn(console, 'warn').mockImplementation(() => {})++ await (watcher as any).getMcpServerWatchCallback()('ADDED', serverObject)+ await vi.advanceTimersByTimeAsync(5000)+ await vi.advanceTimersByTimeAsync(15000)+ await vi.advanceTimersByTimeAsync(30000)++ expect(netPol.reconcileExternalEgress).toHaveBeenCalledTimes(4)+ expect(reconciler.reconcile).not.toHaveBeenCalled()++ await vi.advanceTimersByTimeAsync(30000)++ expect(netPol.reconcileExternalEgress).toHaveBeenCalledTimes(5)+ expect(reconciler.reconcile).toHaveBeenCalledTimes(1)+ watcher.stop()+ })++ it('keeps retrying a partial DELETE pipeline at the capped delay until it recovers', async () => {+ const watcher = new McpServerWatcher()+ ;(watcher as any).mcpServerCacheSynced = true+ const netPol = (watcher as any).netPolReconciler+ const reconciler = (watcher as any).reconciler+ const bindingReconciler = (watcher as any).bindingReconciler+ netPol.cleanupExternalEgress+ .mockRejectedValueOnce(new Error('initial cleanup outage'))+ .mockRejectedValueOnce(new Error('cleanup outage after 5s'))+ .mockRejectedValueOnce(new Error('cleanup outage after 15s'))+ .mockRejectedValueOnce(new Error('cleanup outage after 30s'))+ .mockResolvedValue(undefined)+ vi.spyOn(console, 'error').mockImplementation(() => {})+ vi.spyOn(console, 'warn').mockImplementation(() => {})++ await (watcher as any).getMcpServerWatchCallback()('DELETED', serverObject)+ await vi.advanceTimersByTimeAsync(5000)+ await vi.advanceTimersByTimeAsync(15000)+ await vi.advanceTimersByTimeAsync(30000)++ expect(netPol.cleanupExternalEgress).toHaveBeenCalledTimes(4)+ expect(reconciler.reconcileDelete).not.toHaveBeenCalled()+ expect(bindingReconciler.cleanupBindings).not.toHaveBeenCalled()++ await vi.advanceTimersByTimeAsync(30000)++ expect(netPol.cleanupExternalEgress).toHaveBeenCalledTimes(5)+ expect(reconciler.reconcileDelete).toHaveBeenCalledWith('redis-tools', 'mcp-server')+ expect(bindingReconciler.cleanupBindings).toHaveBeenCalledWith('redis-tools', {+ deleteAllowed: expect.any(Function),+ })+ watcher.stop()+ })++ it('bounds concurrent durable retries across McpServers', async () => {+ const watcher = new McpServerWatcher()+ markMcpServerInventoryAuthoritative(watcher)+ const reconciler = (watcher as any).reconciler+ const release = deferred()+ let active = 0+ let maxActive = 0+ reconciler.reconcile.mockImplementation(async () => {+ active += 1+ maxActive = Math.max(maxActive, active)+ await release.promise+ active -= 1+ })+ vi.spyOn(console, 'warn').mockImplementation(() => {})++ for (let index = 0; index < 11; index += 1) {+ const server = {+ name: `server-${index}`,+ namespace: 'mcp-server',+ spec: {+ ...serverObject.spec,+ egressBindings: undefined,+ },+ }+ ;(watcher as any).servers.set(server.name, server)+ ;(watcher as any).scheduleExternalEgressRetry('ADDED', server)+ }++ await vi.advanceTimersByTimeAsync(5000)+ await flushMicrotasks()++ expect(maxActive).toBe(10)++ release.resolve(undefined)+ await flushMicrotasks()+ watcher.stop()+ })++ it('replays a retry against the watcher generation current when the timer executes', async () => {+ const watcher = new McpServerWatcher()+ markMcpServerInventoryAuthoritative(watcher)+ const selected = {+ name: 'generation-fenced-server',+ namespace: 'mcp-server',+ spec: {+ ...serverObject.spec,+ egressBindings: undefined,+ },+ }+ ;(watcher as any).servers.set(selected.name, selected)+ ;(watcher as any).mcpWatchGeneration = 41+ const replay = vi+ .spyOn(watcher as any, 'reconcileMcpServerWatchEvent')+ .mockResolvedValue(undefined)+ vi.spyOn(console, 'warn').mockImplementation(() => {})+ ;(watcher as any).scheduleExternalEgressRetry('ADDED', selected)+ ;(watcher as any).mcpWatchGeneration = 42+ await vi.advanceTimersByTimeAsync(5000)++ expect(replay).toHaveBeenCalledWith(+ 'MODIFIED',+ expect.objectContaining({ name: selected.name }),+ 42,+ expect.objectContaining({+ isCurrent: expect.any(Function),+ complete: expect.any(Function),+ })+ )+ watcher.stop()+ })++ it('reconciles external egress before runtime reconciliation on ADDED events', async () => {+ const watcher = new McpServerWatcher()+ const netPol = (watcher as any).netPolReconciler+ const reconciler = (watcher as any).reconciler+ const order: string[] = []+ netPol.reconcileExternalEgress.mockImplementation(async () => {+ order.push('egress')+ })+ reconciler.reconcile.mockImplementation(async () => {+ order.push('runtime')+ })++ const watchCallback = (watcher as any).getMcpServerWatchCallback()+ await watchCallback('ADDED', serverObject)++ expect(order).toEqual(['egress', 'runtime'])++ watcher.stop()+ })++ it('does not reconcile runtime resources when external egress pre-start reconciliation fails', async () => {+ const watcher = new McpServerWatcher()+ const netPol = (watcher as any).netPolReconciler+ const reconciler = (watcher as any).reconciler+ netPol.reconcileExternalEgress.mockRejectedValueOnce(new Error('api temporarily unavailable'))+ vi.spyOn(console, 'error').mockImplementation(() => {})+ vi.spyOn(console, 'warn').mockImplementation(() => {})++ const watchCallback = (watcher as any).getMcpServerWatchCallback()+ await watchCallback('ADDED', serverObject)++ expect(reconciler.reconcile).not.toHaveBeenCalled()++ watcher.stop()+ })++ it('retries failed external egress cleanup from a DELETED event', async () => {+ const watcher = new McpServerWatcher()+ ;(watcher as any).mcpServerCacheSynced = true+ const netPol = (watcher as any).netPolReconciler+ const reconciler = (watcher as any).reconciler+ const bindingReconciler = (watcher as any).bindingReconciler+ netPol.cleanupExternalEgress.mockRejectedValueOnce(new Error('delete temporarily unavailable'))+ netPol.cleanupExternalEgress.mockResolvedValue(undefined)+ vi.spyOn(console, 'error').mockImplementation(() => {})+ vi.spyOn(console, 'warn').mockImplementation(() => {})++ const watchCallback = (watcher as any).getMcpServerWatchCallback()+ await watchCallback('DELETED', serverObject)++ expect(netPol.cleanupExternalEgress).toHaveBeenCalledTimes(1)++ await vi.advanceTimersByTimeAsync(5000)++ expect(netPol.cleanupExternalEgress).toHaveBeenCalledTimes(2)+ expect(netPol.cleanupExternalEgress).toHaveBeenLastCalledWith(+ 'redis-tools',+ 'mcp-server',+ undefined,+ expect.any(Function)+ )+ expect(reconciler.reconcileDelete).toHaveBeenCalledWith('redis-tools', 'mcp-server')+ expect(bindingReconciler.cleanupBindings).toHaveBeenCalledWith('redis-tools', {+ deleteAllowed: expect.any(Function),+ })++ watcher.stop()+ })++ it('retries the full DELETE pipeline when authoritative absence is temporarily unavailable', async () => {+ const watcher = new McpServerWatcher()+ ;(watcher as any).mcpServerCacheSynced = true+ const netPol = (watcher as any).netPolReconciler+ const reconciler = (watcher as any).reconciler+ const bindingReconciler = (watcher as any).bindingReconciler+ const unavailable = Object.assign(new Error('apiserver temporarily unavailable'), {+ response: { statusCode: 503 },+ })+ mocks.getNamespacedCustomObject.mockRejectedValueOnce(unavailable)+ vi.spyOn(console, 'error').mockImplementation(() => {})+ vi.spyOn(console, 'warn').mockImplementation(() => {})++ const watchCallback = (watcher as any).getMcpServerWatchCallback()+ await expect(watchCallback('DELETED', serverObject)).resolves.toBeUndefined()++ expect(mocks.getNamespacedCustomObject).toHaveBeenCalledTimes(1)+ expect(netPol.cleanupExternalEgress).not.toHaveBeenCalled()+ expect(reconciler.reconcileDelete).not.toHaveBeenCalled()+ expect(bindingReconciler.cleanupBindings).not.toHaveBeenCalled()++ await vi.advanceTimersByTimeAsync(5000)++ expect(netPol.cleanupExternalEgress).toHaveBeenCalledTimes(1)+ expect(reconciler.reconcileDelete).toHaveBeenCalledWith('redis-tools', 'mcp-server')+ expect(bindingReconciler.cleanupBindings).toHaveBeenCalledWith('redis-tools', {+ deleteAllowed: expect.any(Function),+ })+ expect(mocks.getNamespacedCustomObject).toHaveBeenCalledTimes(4)++ watcher.stop()+ })++ it('retries the full DELETE pipeline when inventory authority is lost between cleanup stages', async () => {+ const watcher = new McpServerWatcher()+ ;(watcher as any).mcpServerCacheSynced = true+ const netPol = (watcher as any).netPolReconciler+ const reconciler = (watcher as any).reconciler+ const bindingReconciler = (watcher as any).bindingReconciler+ reconciler.reconcileDelete.mockImplementationOnce(async () => {+ ;(watcher as any).mcpServerCacheSynced = false+ })+ vi.spyOn(console, 'warn').mockImplementation(() => {})++ const watchCallback = (watcher as any).getMcpServerWatchCallback()+ await expect(watchCallback('DELETED', serverObject)).resolves.toBeUndefined()++ expect(netPol.cleanupExternalEgress).toHaveBeenCalledTimes(1)+ expect(reconciler.reconcileDelete).toHaveBeenCalledWith('redis-tools', 'mcp-server')+ expect(bindingReconciler.cleanupBindings).not.toHaveBeenCalled()++ await vi.advanceTimersByTimeAsync(5000)++ expect(netPol.cleanupExternalEgress).toHaveBeenCalledTimes(1)+ expect(reconciler.reconcileDelete).toHaveBeenCalledTimes(1)+ expect(bindingReconciler.cleanupBindings).not.toHaveBeenCalled()+ ;(watcher as any).mcpServerCacheSynced = true+ await vi.advanceTimersByTimeAsync(14_999)+ expect(netPol.cleanupExternalEgress).toHaveBeenCalledTimes(1)+ await vi.advanceTimersByTimeAsync(1)++ expect(netPol.cleanupExternalEgress).toHaveBeenCalledTimes(2)+ expect(reconciler.reconcileDelete).toHaveBeenCalledTimes(2)+ expect(bindingReconciler.cleanupBindings).toHaveBeenCalledWith('redis-tools', {+ deleteAllowed: expect.any(Function),+ })++ watcher.stop()+ })++ it('retires queued McpServer work immediately when its watch ends', async () => {+ const blockerStarted = deferred()+ const releaseBlocker = deferred()+ let watchCallback: ((type: string, apiObj: typeof serverObject) => Promise<void>) | undefined+ let watchDone: ((error: Error | null) => void) | undefined+ mocks.watch.mockImplementationOnce(async (_path, _options, callback, done) => {+ watchCallback = callback+ watchDone = done+ return { abort: vi.fn() }+ })+ const watcher = new McpServerWatcher()+ const netPol = (watcher as any).netPolReconciler+ await (watcher as any).startMcpServerWatch('mcp-watch-rv')+ const blocker = (watcher as any).enqueueMcpServerReconciliation(+ { name: 'redis-tools', namespace: 'mcp-server' },+ async () => {+ blockerStarted.resolve(undefined)+ await releaseBlocker.promise+ }+ )+ await blockerStarted.promise++ const event = watchCallback!('ADDED', serverObject)+ watchDone!(null)+ releaseBlocker.resolve(undefined)+ await Promise.all([blocker, event])++ expect(netPol.reconcileExternalEgress).not.toHaveBeenCalled()+ await watcher.stop()+ })++ it('retries the recreated server when an older DELETE retry timer already owns the key', async () => {+ const deletedServer = {+ metadata: {+ name: 'redis-tools',+ namespace: 'mcp-server',+ uid: 'redis-tools-old-uid',+ generation: 1,+ },+ spec: serverObject.spec,+ }+ const recreatedServer = {+ metadata: {+ name: 'redis-tools',+ namespace: 'mcp-server',+ uid: 'redis-tools-new-uid',+ generation: 1,+ },+ spec: { ...serverObject.spec, image: 'redis-tools:recreated' },+ }+ const watcher = new McpServerWatcher()+ ;(watcher as any).mcpServerCacheSynced = true+ const netPol = (watcher as any).netPolReconciler+ const reconciler = (watcher as any).reconciler+ netPol.cleanupExternalEgress.mockRejectedValueOnce(new Error('cleanup unavailable'))+ netPol.reconcileExternalEgress+ .mockRejectedValueOnce(new Error('recreated policy unavailable'))+ .mockResolvedValue(undefined)+ vi.spyOn(console, 'error').mockImplementation(() => {})+ vi.spyOn(console, 'warn').mockImplementation(() => {})+ const watchCallback = (watcher as any).getMcpServerWatchCallback()++ await watchCallback('DELETED', deletedServer)+ await watchCallback('ADDED', recreatedServer)+ await vi.advanceTimersByTimeAsync(5000)++ expect(netPol.cleanupExternalEgress).toHaveBeenCalledTimes(1)+ expect(netPol.reconcileExternalEgress).toHaveBeenCalledTimes(2)+ expect(reconciler.reconcile).toHaveBeenCalledOnce()+ expect(reconciler.reconcile).toHaveBeenCalledWith(+ expect.objectContaining({+ uid: 'redis-tools-new-uid',+ generation: 1,+ spec: expect.objectContaining({ image: 'redis-tools:recreated' }),+ }),+ { isCurrent: expect.any(Function) }+ )++ watcher.stop()+ })++ it('retries the latest DELETE cleanup when an older ADD retry timer already owns the key', async () => {+ const server = {+ metadata: {+ name: 'redis-tools',+ namespace: 'mcp-server',+ uid: 'redis-tools-uid',+ generation: 3,+ },+ spec: serverObject.spec,+ }+ const watcher = new McpServerWatcher()+ ;(watcher as any).mcpServerCacheSynced = true+ const netPol = (watcher as any).netPolReconciler+ netPol.reconcileExternalEgress.mockRejectedValueOnce(new Error('policy unavailable'))+ netPol.cleanupExternalEgress+ .mockRejectedValueOnce(new Error('cleanup unavailable'))+ .mockResolvedValue(undefined)+ vi.spyOn(console, 'error').mockImplementation(() => {})+ vi.spyOn(console, 'warn').mockImplementation(() => {})+ const watchCallback = (watcher as any).getMcpServerWatchCallback()++ await watchCallback('ADDED', server)+ await watchCallback('DELETED', server)+ await vi.advanceTimersByTimeAsync(5000)++ expect(netPol.reconcileExternalEgress).toHaveBeenCalledTimes(1)+ expect(netPol.cleanupExternalEgress).toHaveBeenCalledTimes(2)++ watcher.stop()+ })++ it('does not continue to runtime or bindings after stop releases a live DNS hold', async () => {+ const egressStarted = deferred()+ const releaseEgress = deferred()+ const server = {+ metadata: {+ name: 'redis-tools',+ namespace: 'mcp-server',+ uid: 'redis-tools-uid',+ generation: 1,+ annotations: {+ 'clerum.io/recipe-bindings': JSON.stringify([+ { source: 'redis-tools', target: 'downstream' },+ ]),+ },+ },+ spec: serverObject.spec,+ }+ const watcher = new McpServerWatcher()+ const netPol = (watcher as any).netPolReconciler+ const reconciler = (watcher as any).reconciler+ const bindingReconciler = (watcher as any).bindingReconciler+ netPol.reconcileExternalEgress.mockImplementationOnce(async () => {+ egressStarted.resolve(undefined)+ await releaseEgress.promise+ })+ const watchCallback = (watcher as any).getMcpServerWatchCallback()++ const reconcile = watchCallback('ADDED', server)+ await egressStarted.promise+ await watcher.stop()+ releaseEgress.resolve(undefined)+ await reconcile++ expect(reconciler.reconcile).not.toHaveBeenCalled()+ expect(bindingReconciler.reconcileBindings).not.toHaveBeenCalled()+ })++ it('does not run an older runtime revision after newer desired state arrives during egress', async () => {+ const egressStarted = deferred()+ const releaseEgress = deferred()+ const oldServer = {+ metadata: {+ name: 'redis-tools',+ namespace: 'mcp-server',+ uid: 'redis-tools-uid',+ generation: 1,+ },+ spec: { ...serverObject.spec, image: 'redis-tools:old' },+ }+ const newServer = {+ metadata: {+ name: 'redis-tools',+ namespace: 'mcp-server',+ uid: 'redis-tools-uid',+ generation: 2,+ },+ spec: { ...serverObject.spec, image: 'redis-tools:new' },+ }+ const watcher = new McpServerWatcher()+ const netPol = (watcher as any).netPolReconciler+ const reconciler = (watcher as any).reconciler+ netPol.reconcileExternalEgress+ .mockImplementationOnce(async () => {+ egressStarted.resolve(undefined)+ await releaseEgress.promise+ })+ .mockResolvedValue(undefined)+ const watchCallback = (watcher as any).getMcpServerWatchCallback()++ const oldReconcile = watchCallback('ADDED', oldServer)+ await egressStarted.promise+ const newReconcile = watchCallback('MODIFIED', newServer)+ releaseEgress.resolve(undefined)+ await Promise.all([oldReconcile, newReconcile])++ expect(reconciler.reconcile).toHaveBeenCalledOnce()+ expect(reconciler.reconcile).toHaveBeenCalledWith(+ expect.objectContaining({+ generation: 2,+ spec: expect.objectContaining({ image: 'redis-tools:new' }),+ }),+ { isCurrent: expect.any(Function) }+ )++ watcher.stop()+ })++ it('does not resume periodic external egress work from jitter after stop', async () => {+ const watcher = new McpServerWatcher()+ const netPol = (watcher as any).netPolReconciler+ ;(watcher as any).servers.set('redis-tools', {+ name: 'redis-tools',+ namespace: 'mcp-server',+ spec: serverObject.spec,+ })+ vi.spyOn(Math, 'random').mockReturnValue(0.5)++ const resync = (watcher as any).externalEgressCoordinator.runResync()+ await flushMicrotasks()+ await watcher.stop()+ await vi.advanceTimersByTimeAsync(2500)+ await resync++ expect(netPol.reconcileExternalEgress).not.toHaveBeenCalled()+ })++ it('periodic resync reuses external egress reconciliation for cached servers with bindings', async () => {+ const watcher = new McpServerWatcher()+ markMcpServerInventoryAuthoritative(watcher)+ const netPol = (watcher as any).netPolReconciler+ netPol.reconcileExternalEgress.mockResolvedValue(undefined)+ vi.spyOn(Math, 'random').mockReturnValue(0)+ ;(watcher as any).servers.set('redis-tools', {+ name: 'redis-tools',+ namespace: 'mcp-server',+ spec: serverObject.spec,+ })++ await (watcher as any).externalEgressCoordinator.runResync()++ expect(netPol.reconcileExternalEgress).toHaveBeenCalledTimes(1)++ watcher.stop()+ })++ it('does not run parallel external egress reconciles for the same McpServer', async () => {+ const watcher = new McpServerWatcher()+ const netPol = (watcher as any).netPolReconciler+ const mutationStarted = deferred()+ const releaseMutation = deferred()+ const server = {+ name: 'redis-tools',+ namespace: 'mcp-server',+ spec: serverObject.spec,+ }+ ;(watcher as any).servers.set('redis-tools', server)+ netPol.reconcileExternalEgress.mockImplementationOnce(async () => {+ mutationStarted.resolve(undefined)+ await releaseMutation.promise+ })+ vi.spyOn(Math, 'random').mockReturnValue(0)++ const active = (watcher as any).runExternalEgressOnce('MODIFIED', server)+ await mutationStarted.promise+ await (watcher as any).externalEgressCoordinator.runResync()++ expect(netPol.reconcileExternalEgress).toHaveBeenCalledOnce()++ releaseMutation.resolve(undefined)+ await active+ watcher.stop()+ })++ it('waits for already in-flight external egress before reconciling the current server', async () => {+ const watcher = new McpServerWatcher()+ const netPol = (watcher as any).netPolReconciler+ const mutationStarted = deferred()+ const releaseMutation = deferred()+ const server = {+ name: 'redis-tools',+ namespace: 'mcp-server',+ spec: serverObject.spec,+ }+ ;(watcher as any).servers.set(server.name, server)+ netPol.reconcileExternalEgress.mockImplementationOnce(async () => {+ mutationStarted.resolve(undefined)+ await releaseMutation.promise+ })+ const warn = vi.spyOn(console, 'warn').mockImplementation(() => {})++ const active = (watcher as any).runExternalEgressOnce('MODIFIED', server)+ await mutationStarted.promise+ const waiting = (watcher as any).runExternalEgressOnce('MODIFIED', server)+ await flushMicrotasks()++ expect(netPol.reconcileExternalEgress).toHaveBeenCalledOnce()+ expect(warn).toHaveBeenCalledWith(+ '[K8s] Waiting for external egress reconcile for mcp-server/redis-tools; already in flight'+ )++ releaseMutation.resolve(undefined)+ await Promise.all([active, waiting])+ expect(netPol.reconcileExternalEgress).toHaveBeenCalledTimes(2)++ warn.mockRestore()+ watcher.stop()+ })++ it('does not start another external egress mutation after stop releases an in-flight wait', async () => {+ const watcher = new McpServerWatcher()+ const netPol = (watcher as any).netPolReconciler+ const mutationStarted = deferred()+ const releaseMutation = deferred()+ const server = {+ name: 'redis-tools',+ namespace: 'mcp-server',+ spec: serverObject.spec,+ }+ ;(watcher as any).servers.set(server.name, server)+ netPol.reconcileExternalEgress.mockImplementationOnce(async () => {+ mutationStarted.resolve(undefined)+ await releaseMutation.promise+ })++ const active = (watcher as any).runExternalEgressOnce('MODIFIED', server)+ await mutationStarted.promise+ const waiting = (watcher as any).runExternalEgressOnce('MODIFIED', server)+ await flushMicrotasks()+ await watcher.stop()+ releaseMutation.resolve(undefined)+ await Promise.all([active, waiting])++ expect(netPol.reconcileExternalEgress).toHaveBeenCalledOnce()+ })+ })++ describe('McpServerWatcher.countCommunicationChannels', () => {+ it('returns 0 for an unknown host', () => {+ const watcher = new McpServerWatcher()+ expect(watcher.countCommunicationChannels('unknown-host')).toBe(0)+ })++ it('counts CCs whose spec.hostRef matches the given host', () => {+ const watcher = new McpServerWatcher()+ // Seed the private cache directly — the real watcher populates from K8s events.+ const ccs = (watcher as any).communicationChannels as Map<string, { spec: { hostRef: string } }>+ ccs.set('cc-1', { name: 'cc-1', namespace: 'channels', spec: { hostRef: 'marketing' } } as any)+ ccs.set('cc-2', { name: 'cc-2', namespace: 'channels', spec: { hostRef: 'marketing' } } as any)+ ccs.set('cc-3', { name: 'cc-3', namespace: 'channels', spec: { hostRef: 'trader' } } as any)+ expect(watcher.countCommunicationChannels('marketing')).toBe(2)+ expect(watcher.countCommunicationChannels('trader')).toBe(1)+ expect(watcher.countCommunicationChannels('chatllm')).toBe(0)+ })+ })++ describe('listAllCommunicationChannels', () => {+ beforeEach(() => {+ vi.clearAllMocks()+ })++ it('lists CommunicationChannel CRDs in the channels namespace', async () => {+ const fixtureItems = [+ {+ metadata: { name: 'gcp-channel', namespace: 'channels' },+ spec: { hostRef: 'josue-agent' },+ },+ {+ metadata: { name: 'tgtestjose2', namespace: 'channels' },+ spec: { hostRef: 'development' },+ },+ ]+ mocks.listNamespacedCustomObject.mockResolvedValueOnce({ items: fixtureItems })++ const result = await listAllCommunicationChannels()++ expect(mocks.listNamespacedCustomObject).toHaveBeenCalledWith({+ group: 'clerum.io',+ version: 'v1alpha1',+ namespace: 'channels',+ plural: 'communicationchannels',+ })+ expect(result).toHaveLength(2)+ expect(result[0]).toEqual({+ name: 'gcp-channel',+ namespace: 'channels',+ spec: { hostRef: 'josue-agent' },+ })+ expect(result[1].spec.hostRef).toBe('development')+ })+ })++ describe('McpServerWatcher wires CC counter into HostReconciler', () => {+ it('HostReconciler.countCommunicationChannels reads from the watcher cache', () => {+ const watcher = new McpServerWatcher()+ const ccs = (watcher as any).communicationChannels as Map<string, { spec: { hostRef: string } }>+ ccs.set('cc-1', { name: 'cc-1', namespace: 'channels', spec: { hostRef: 'alpha-host' } } as any)+ ccs.set('cc-2', { name: 'cc-2', namespace: 'channels', spec: { hostRef: 'alpha-host' } } as any)+ const hostReconciler = watcher.getHostReconciler()+ // Same private field access pattern the hostReconciler tests use.+ expect((hostReconciler as any).countCommunicationChannels('alpha-host')).toBe(2)+ expect((hostReconciler as any).countCommunicationChannels('beta-host')).toBe(0)+ })+ })++ describe('McpServerWatcher Host mutation dependency wiring', () => {+ it('H2: wires a uid-guarded cache reflector into HostReconciler', () => {+ const watcher = new McpServerWatcher()+ const reflect = (+ watcher.getHostReconciler() as unknown as {+ _reflectHostOutcomeFn:+ | ((name: string, uid: string | undefined, apply: (target: HostCRD) => void) => void)+ | null+ }+ )._reflectHostOutcomeFn+ if (!reflect) {+ throw new Error('setReflectHostOutcome was never wired by the McpServerWatcher constructor')+ }+ const entry = {+ name: 'h2-host',+ namespace: 'mcp-host',+ uid: 'uid-1',+ spec: { host: 'h2-host', contextRef: 'c', secretRef: 's' },+ } as unknown as HostCRD+ ;(watcher as unknown as { hosts: Map<string, HostCRD> }).hosts.set('h2-host', entry)+ const apply = vi.fn()+ reflect('h2-host', 'uid-1', apply) // positive: current entry, matching uid+ expect(apply).toHaveBeenCalledTimes(1)+ expect(apply).toHaveBeenCalledWith(entry)+ apply.mockClear()+ reflect('h2-host', 'uid-2', apply) // recreation: same name, different uid → skip+ expect(apply).not.toHaveBeenCalled()+ reflect('h2-host', undefined, apply) // fail-closed on missing uid+ expect(apply).not.toHaveBeenCalled()+ reflect('absent-host', 'uid-1', apply) // absent entry → skip+ expect(apply).not.toHaveBeenCalled()+ })++ it('tracks only the selected Context, referenced SFS state, and Host-scoped channels', () => {+ const watcher = new McpServerWatcher()+ const host = {+ name: 'alpha-host',+ namespace: 'mcp-host',+ spec: { contextRef: 'alpha-context' },+ } as HostCRD+ const context = {+ name: 'alpha-context',+ namespace: 'mcp-server',+ spec: {+ contextId: 'alpha-context',+ sharedFileSystems: [{ name: 'documents', mountPath: '/contexts/documents' }],+ },+ }+ const documentsV1 = {+ name: 'documents',+ namespace: 'mcp-host',+ spec: { storageClassName: 'fast-v1' },+ }+ const alphaChannelV1 = {+ name: 'telegram-alpha',+ namespace: 'channels',+ spec: { hostRef: 'alpha-host' },+ }+ ;(watcher as any).contexts.set(context.name, context)+ ;(watcher as any).sharedFileSystems.set(documentsV1.name, documentsV1)+ ;(watcher as any).communicationChannels.set(alphaChannelV1.name, alphaChannelV1)+ ;(watcher as any).ccCacheSynced = true+ ;(watcher as any).ccWatchGeneration = 7+ const sfsReconciler = (watcher as any).sharedFileSystemReconciler+ sfsReconciler.isMountable.mockReturnValue(true)+ const hostReconciler = watcher.getHostReconciler() as any+ const resolveDependencies = hostReconciler._hostMutationDependenciesFn++ const original = resolveDependencies(host)+ expect(original).toEqual([+ { name: context.name, namespace: context.namespace, spec: context.spec },+ true,+ 7,+ { name: documentsV1.name, namespace: documentsV1.namespace, spec: documentsV1.spec },+ true,+ {+ name: alphaChannelV1.name,+ namespace: alphaChannelV1.namespace,+ spec: alphaChannelV1.spec,+ },+ ])+ ;(watcher as any).sharedFileSystems.set('unrelated', {+ name: 'unrelated',+ namespace: 'mcp-host',+ spec: {},+ })+ ;(watcher as any).communicationChannels.set('telegram-beta', {+ name: 'telegram-beta',+ namespace: 'channels',+ spec: { hostRef: 'beta-host' },+ })+ const afterUnrelatedChanges = resolveDependencies(host)+ expect(afterUnrelatedChanges).toEqual(original)++ const documentsV2 = {+ ...documentsV1,+ spec: { storageClassName: 'fast-v2' },+ }+ ;(watcher as any).sharedFileSystems.set(documentsV2.name, documentsV2)+ const afterSfsRevision = resolveDependencies(host)+ expect(afterSfsRevision[3]).toEqual({+ name: documentsV2.name,+ namespace: documentsV2.namespace,+ spec: documentsV2.spec,+ })+ expect(afterSfsRevision[3]).not.toEqual(original[3])++ sfsReconciler.isMountable.mockReturnValue(false)+ const afterMountabilityChange = resolveDependencies(host)+ expect(afterMountabilityChange[4]).toBe(false)++ const alphaChannelV2 = {+ ...alphaChannelV1,+ spec: {+ ...alphaChannelV1.spec,+ credentialsSecretRef: { name: 'telegram-alpha-v2' },+ },+ }+ ;(watcher as any).communicationChannels.set(alphaChannelV2.name, alphaChannelV2)+ const afterChannelRevision = resolveDependencies(host)+ expect(afterChannelRevision.at(-1)).toEqual({+ name: alphaChannelV2.name,+ namespace: alphaChannelV2.namespace,+ spec: alphaChannelV2.spec,+ })+ expect(afterChannelRevision.at(-1)).not.toEqual(original.at(-1))+ })+ })++ describe('McpServerWatcher.reconcileHostsReferencingCC', () => {+ it('re-reconciles the Host whose name matches the CC hostRef', async () => {+ const watcher = new McpServerWatcher()+ markHostInventoryAuthoritative(watcher)+ const hosts = (watcher as any).hosts as Map<string, { name: string; spec: any }>+ hosts.set('marketing', {+ name: 'marketing',+ namespace: 'mcp-host',+ spec: { host: 'marketing', contextRef: 'c', secretRef: 's' },+ } as any)+ const reconcileSpy = vi+ .spyOn(watcher.getHostReconciler(), 'reconcile')+ .mockResolvedValue(undefined)++ await (watcher as any).reconcileHostsReferencingCC('marketing')++ expect(reconcileSpy).toHaveBeenCalledTimes(1)+ expect(reconcileSpy.mock.calls[0][0]).toMatchObject({ name: 'marketing' })+ })++ it('is a no-op when the hostRef does not match any cached Host', async () => {+ const watcher = new McpServerWatcher()+ markHostInventoryAuthoritative(watcher)+ const reconcileSpy = vi+ .spyOn(watcher.getHostReconciler(), 'reconcile')+ .mockResolvedValue(undefined)++ await (watcher as any).reconcileHostsReferencingCC('does-not-exist')++ expect(reconcileSpy).not.toHaveBeenCalled()+ })++ it('reports reconcile errors without throwing so the watch can schedule recovery', async () => {+ const watcher = new McpServerWatcher()+ markHostInventoryAuthoritative(watcher)+ const hosts = (watcher as any).hosts as Map<string, { name: string; spec: any }>+ hosts.set('marketing', {+ name: 'marketing',+ namespace: 'mcp-host',+ spec: { host: 'marketing', contextRef: 'c', secretRef: 's' },+ } as any)+ vi.spyOn(watcher.getHostReconciler(), 'reconcile').mockRejectedValue(new Error('boom'))+ const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {})++ await expect((watcher as any).reconcileHostsReferencingCC('marketing')).resolves.toBe(false)+ expect(errorSpy).toHaveBeenCalled()++ errorSpy.mockRestore()+ })++ it('schedules lifecycle convergence when a targeted CC event reconcile fails', async () => {+ const watcher = new McpServerWatcher()+ markHostInventoryAuthoritative(watcher)+ ;(watcher as any).hosts.set('marketing', {+ name: 'marketing',+ namespace: 'mcp-host',+ spec: { host: 'marketing', contextRef: 'c', secretRef: 's' },+ })+ vi.spyOn(watcher.getHostReconciler(), 'reconcile').mockRejectedValue(new Error('temporary'))+ const requestFleet = vi+ .spyOn(watcher as any, 'requestHostFleetReconcile')+ .mockResolvedValue(undefined)+ const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {})++ const watchCallback = (watcher as any).getCommunicationChannelWatchCallback()+ await watchCallback('ADDED', {+ metadata: { name: 'marketing-channel', namespace: 'channels' },+ spec: { hostRef: 'marketing' },+ })++ expect(requestFleet).toHaveBeenCalledWith(+ 'CommunicationChannel event convergence fallback',+ 1,+ 'lifecycle'+ )+ errorSpy.mockRestore()+ watcher.stop()+ })+ })++ describe('McpServerWatcher.start ordering (#281 R6-bis)', () => {+ beforeEach(() => {+ vi.clearAllMocks()+ mocks.watch.mockReset().mockResolvedValue({ abort: vi.fn() })+ mocks.hostFullReconcile.mockReset().mockResolvedValue(undefined)+ })++ it('populates CC cache and starts CC watch BEFORE fullReconcile(initialHosts)', async () => {+ const eventLog: string[] = []++ // The existing mocking infrastructure routes all list calls through+ // mocks.listNamespacedCustomObject keyed by `plural`. We intercept per-plural+ // to record when each fires relative to fullReconcile.+ mocks.listNamespacedCustomObject.mockImplementation(async ({ plural }: { plural: string }) => {+ if (plural === 'communicationchannels') {+ eventLog.push('listAllCommunicationChannels')+ return {+ metadata: { resourceVersion: '104' },+ items: [+ { metadata: { name: 'cc-1', namespace: 'channels' }, spec: { hostRef: 'marketing' } },+ ],+ }+ }+ if (plural === 'mcpservers') return { items: [] }+ if (plural === 'contexts') return { items: [] }+ if (plural === 'hosts') return { items: [] }+ if (plural === 'sharedfilesystems') return { items: [] }+ return { items: [] }+ })++ // Wrap hostFullReconcile to record when it fires.+ mocks.hostFullReconcile.mockImplementation(async () => {+ eventLog.push('fullReconcile')+ })++ mocks.watch.mockImplementation(async () => {+ eventLog.push('startCommunicationChannelWatch')+ return { abort: vi.fn() }+ })+ const watcher = new McpServerWatcher()++ stubAuthoritativeInventoryWatch(watcher, 'McpServer')+ stubAuthoritativeInventoryWatch(watcher, 'Context')+ vi.spyOn(watcher as any, 'startSharedFileSystemWatch').mockResolvedValue(undefined)+ vi.spyOn(watcher as any, 'startGlobalFileSystemWatch').mockResolvedValue(undefined)+ const ccWatchSpy = vi.spyOn(watcher as any, 'startCommunicationChannelWatch')++ await watcher.start()++ // The critical ordering: the paired CC snapshot/watch must be established+ // before any Host can make a stateless eligibility decision.+ const idxList = eventLog.indexOf('listAllCommunicationChannels')+ const idxWatch = eventLog.indexOf('startCommunicationChannelWatch')+ const idxReconcile = eventLog.indexOf('fullReconcile')+ expect(idxList).toBeGreaterThanOrEqual(0)+ expect(idxWatch).toBeGreaterThanOrEqual(0)+ expect(idxReconcile).toBeGreaterThanOrEqual(0)+ expect(idxList).toBeLessThan(idxReconcile)+ expect(idxWatch).toBeLessThan(idxReconcile)+ expect(ccWatchSpy).toHaveBeenCalledWith('104')++ // Cache populated by the initial LIST.+ const cache = (watcher as any).communicationChannels as Map<string, any>+ expect(cache.size).toBe(1)+ expect(cache.get('cc-1')?.spec.hostRef).toBe('marketing')+ expect(watcher.isCommunicationChannelCacheSynced()).toBe(true)+ watcher.stop()+ })++ it('starts CC watch even when initial CC list fails (graceful degradation)', async () => {+ mocks.listNamespacedCustomObject.mockImplementation(async ({ plural }: { plural: string }) => {+ if (plural === 'communicationchannels') throw new Error('CC list failed')+ if (plural === 'mcpservers') return { items: [] }+ if (plural === 'contexts') return { items: [] }+ if (plural === 'hosts') return { items: [] }+ if (plural === 'sharedfilesystems') return { items: [] }+ return { items: [] }+ })++ const watcher = new McpServerWatcher()++ stubAuthoritativeInventoryWatch(watcher, 'McpServer')+ stubAuthoritativeInventoryWatch(watcher, 'Context')+ vi.spyOn(watcher as any, 'startSharedFileSystemWatch').mockResolvedValue(undefined)+ vi.spyOn(watcher as any, 'startGlobalFileSystemWatch').mockResolvedValue(undefined)+ const ccWatchSpy = vi+ .spyOn(watcher as any, 'startCommunicationChannelWatch')+ .mockResolvedValue(undefined)+ const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {})++ await expect(watcher.start()).resolves.toBeUndefined()+ expect(ccWatchSpy).toHaveBeenCalledTimes(1)+ expect(watcher.isCommunicationChannelCacheSynced()).toBe(false)+ expect(errorSpy).toHaveBeenCalledWith(+ '[K8s] CommunicationChannel initial load failed; ccCacheSynced remains false ' ++ '(B2 preserves channel-reader replicas and holds stateless lifecycle active):',+ expect.any(Error)+ )++ errorSpy.mockRestore()+ watcher.stop()+ })+ })++ describe('McpServerWatcher CommunicationChannel cache recovery', () => {+ async function flushMicrotasks(): Promise<void> {+ for (let i = 0; i < 8; i += 1) await Promise.resolve()+ }function communicationChannelWatchCalls(): typeof mocks.watch.mock.calls {return mocks.watch.mock.calls.filter(([path]) =>diff --cc host-context-controller/src/k8sClient.tsindex 0044519cf,d4ffc430b..c39b5c3d1--- a/host-context-controller/src/k8sClient.ts+++ b/host-context-controller/src/k8sClient.ts@@@ -30,16 -40,20 +40,28 @@@ importimport { K8sGfsApi } from './k8s/gfsK8sApi'import { makeHostK8sApiClient } from './k8s/hostK8sApiClient'import { pvcName as sfsPvcName } from './k8s/sharedFileSystemFactory'+import type {+ AuthorityContext,+ AuthorityHost,+ AuthorityMcpServer,+ AuthoritySecret,+ AuthoritySecretMetadata,+ McpAuthorizationStore,+} from './mcpAuthorization'- import { hostDeleteCleanupTotal, hostFleetRequestsTotal, hostWatchRecoverySeconds } from './metrics'- import { NetworkPolicyReconciler } from './networkPolicyReconciler'+ import { LlmHookReconciler, computePodKey, referencedHookIds } from './llmHookReconciler'+ import {+ confirmAuthoritativeMcpServerAbsence,+ isMcpServerStatusOnlyUpdate,+ sameMcpServerDesiredRevision,+ } from './mcpServerSafety'+ import {+ hostDeleteCleanupTotal,+ hostFleetRequestsTotal,+ hostWatchRecoverySeconds,+ initialConvergenceLastSuccessTimestampSeconds,+ initialConvergenceRetriesTotal,+ } from './metrics'+ import { NetworkPolicyReconciler, sameContextDesiredRevision } from './networkPolicyReconciler'import { McpServerReconciler } from './reconciler'import { SharedFileSystemReconciler } from './sharedFileSystemReconciler'import {diff --cc host-context-controller/src/main.tsindex b898e0414,a1c168adf..d8cd848b5--- a/host-context-controller/src/main.ts+++ b/host-context-controller/src/main.ts@@@ -19,8 -18,7 +19,9 @@@ importcreateMcpServerProvider,getKubeConfig,} from './k8sClient'+import { McpApiAuthenticator } from './mcpApiAuthentication'+import { McpAuthorizationService } from './mcpAuthorization'+ import { resolveHostAuthoritativeFn, resolveProviderAuthoritativeFn } from './readinessGate'import { ContextMapperServer } from './server'import { StatelessLifecycleTracker } from './statelessLifecycleTracker'import {@@@ -170,8 -163,8 +183,10 @@@ async function main(): Promise<void>config.port,hostReconciler,hasDesktopFn,+ providerAuthoritativeFn,- hostAuthoritativeFn++ hostAuthoritativeFn,+ mcpAuthenticator,+ mcpAuthorization)await server.start()diff --cc host-context-controller/src/server.test.tsindex 87506e4f9,0ab7d03aa..300882d6f--- a/host-context-controller/src/server.test.ts+++ b/host-context-controller/src/server.test.ts@@@ -80,91 -70,7 +80,93 @@@ async function invokereturn res}-describe('ContextMapperServer readiness', () => {+const principal: VerifiedMcpHostPrincipal = {+ subject: 'mcp-host/standalone',+ hostName: 'host-a',+ hostUid: 'host-uid-a',+ namespace: 'mcp-host',+ jti: 'jti-a',+ issuedAt: 1,+ expiresAt: 4_000_000_000,+ audiences: ['host-context-controller'],+}++class FakeAuthorityStore implements McpAuthorizationStore {+ secretValueReads = 0+ private readonly host: AuthorityHost = {+ name: 'host-a',+ namespace: 'mcp-host',+ metadata: { uid: 'host-uid-a', resourceVersion: '1' },+ contextRef: 'context-a',+ }+ private readonly context: AuthorityContext = {+ name: 'context-a',+ namespace: 'mcp-server',+ metadata: { uid: 'context-uid-a', resourceVersion: '2' },+ mcpServers: ['server-a'],+ }+ private readonly mcpServer: AuthorityMcpServer = {+ name: 'server-a',+ namespace: 'mcp-server',+ metadata: { uid: 'server-uid-a', resourceVersion: '3' },+ transport: { type: 'streamableHttp', url: 'http://server-a/mcp' },+ auth: { type: 'bearer', secretRef: 'server-a-auth', secretKey: 'token' },+ enabled: true,+ status: { deployed: true, ready: true },+ }+ private readonly secret: AuthoritySecret = {+ name: 'server-a-auth',+ namespace: 'mcp-server',+ metadata: { uid: 'secret-uid-a', resourceVersion: '4' },+ data: { token: Buffer.from('credential-value').toString('base64') },+ }++ async readHost() {+ return this.host+ }++ async readContext() {+ return this.context+ }++ async readMcpServer() {+ return this.mcpServer+ }++ async readSecretMetadata() {+ return {+ name: this.secret.name,+ namespace: this.secret.namespace,+ metadata: this.secret.metadata,+ }+ }++ async readSecret() {+ this.secretValueReads += 1+ return this.secret+ }+}++function protectedServer(store = new FakeAuthorityStore()): {+ server: ContextMapperServer+ store: FakeAuthorityStore+} {+ const authenticator = { authenticate: () => principal } as unknown as McpApiAuthenticator+ const server = new ContextMapperServer(+ new FakeProvider(),+ 0,+ undefined,+ undefined,++ () => true,++ () => true,+ authenticator,+ new McpAuthorizationService(store)+ )+ server.setReady(true)+ return { server, store }+}++describe('ContextMapperServer', () => {let server: ContextMapperServer | null = nullafterEach(async () => {@@@ -184,215 -106,207 +202,401 @@@response = await invoke(server, '/api/v1/mcpservers')expect(response.statusCode).toBe(503)- expect(JSON.parse(response.body)).toEqual({- error: 'Service Unavailable',- message: 'Context mapper is still starting',- })server.setReady(true)+ response = await invoke(server, '/api/v1/mcpservers')+ expect(response.statusCode).toBe(200)+ expect(JSON.parse(response.body)).toMatchObject({ servers: [], contextRef: '*' })+ })- response = await invoke(server, '/ready')+ it('keeps the temporary global v1 inventory metadata-only and non-cacheable', async () => {+ server = new ContextMapperServer(+ new FakeProvider([+ {+ name: 'server-a',+ description: 'must not be exposed by the legacy system inventory',+ contextRef: 'context-a',+ transport: {+ type: 'streamableHttp',+ url: 'http://server-a/mcp',+ secretRef: 'nested-secret-must-not-leak',+ } as McpServerInfo['transport'],+ auth: { type: 'bearer', secretRef: 'server-a-auth', secretKey: 'token' },+ enabled: true,+ status: { deployed: true, ready: true, message: 'internal status detail' },+ },+ ]),+ 0,+ undefined,+ undefined,+ () => true+ )+ server.setReady(true)++ const response = await invoke(server, '/api/v1/mcpservers')expect(response.statusCode).toBe(200)- expect(JSON.parse(response.body)).toEqual({ status: 'ready', ready: true })+ expect(response.headers['Cache-Control']).toBe('no-store, private')+ expect(JSON.parse(response.body).servers[0]).not.toHaveProperty('auth')+ expect(JSON.parse(response.body).servers[0]).not.toHaveProperty('description')+ expect(response.body).not.toContain('secretRef')+ expect(response.body).not.toContain('secretKey')+ expect(response.body).not.toContain('nested-secret-must-not-leak')+ expect(response.body).not.toContain('internal status detail')+ })- response = await invoke(server, '/api/v1/mcpservers')+ it.each(['/api/v1/mcpservers/context/context-a', '/api/v1/mcpservers/server-a/auth'])(+ 'returns an application-level 410 tombstone for legacy route %s',+ async path => {+ server = new ContextMapperServer(new FakeProvider(), 0)+ server.setReady(true)+ const response = await invoke(server, path)+ expect(response.statusCode).toBe(410)+ expect(JSON.parse(response.body)).toEqual({ error: 'gone' })+ expect(response.headers['Cache-Control']).toBe('no-store, private')+ }+ )++ it('tombstones legacy Host-selected routes for every method before readiness and CORS', async () => {+ server = new ContextMapperServer(new FakeProvider(), 0, undefined, undefined, () => true)++ for (const [path, method] of [+ ['/api/v1/mcpservers/context/context-a', 'OPTIONS'],+ ['/api/v1/mcpservers/server-a/auth', 'POST'],+ ] as const) {+ const response = await invoke(server, path, { method })+ expect(response.statusCode).toBe(410)+ expect(JSON.parse(response.body)).toEqual({ error: 'gone' })+ expect(response.headers['Access-Control-Allow-Origin']).toBeUndefined()+ expect(response.headers['Cache-Control']).toBe('no-store, private')+ }+ })++ it('serves scoped v2 inventory without wildcard browser CORS or Secret values', async () => {+ const protectedRoute = protectedServer()+ server = protectedRoute.server+ const response = await invoke(server, '/api/v2/hosts/self/mcpservers')+ expect(response.statusCode).toBe(200)+ expect(response.headers['Access-Control-Allow-Origin']).toBeUndefined()+ expect(response.headers['Cache-Control']).toBe('no-store, private')+ const body = JSON.parse(response.body)+ expect(body.servers).toEqual([+ expect.objectContaining({ name: 'server-a', authRequired: true }),+ ])+ expect(response.body).not.toContain('context-a')+ expect(response.body).not.toContain('server-a-auth')+ expect(response.body).not.toContain('credential-value')+ expect(protectedRoute.store.secretValueReads).toBe(0)+ })++ it('keeps direct protected-route startup and not-found errors generic and non-cacheable', async () => {+ server = new ContextMapperServer(new FakeProvider(), 0, undefined, undefined, () => true)++ let response = await invoke(server, '/api/v2/hosts/self/mcpservers')+ expect(response.statusCode).toBe(503)+ expect(JSON.parse(response.body)).toEqual({ error: 'authorization_unavailable' })+ expect(response.headers['Cache-Control']).toBe('no-store, private')++ server.setReady(true)+ response = await invoke(server, '/api/v2/hosts/self/mcpservers/unknown')+ expect(response.statusCode).toBe(404)+ expect(JSON.parse(response.body)).toEqual({ error: 'not_found' })+ expect(response.headers['Cache-Control']).toBe('no-store, private')+ })++ it('accepts only the exact credential body and returns the fenced token DTO', async () => {+ const protectedRoute = protectedServer()+ server = protectedRoute.server+ const response = await invoke(server, '/api/v2/hosts/self/mcpservers/credential', {+ method: 'POST',+ headers: { 'content-type': 'application/json' },+ body: JSON.stringify({ serverName: 'server-a' }),+ })expect(response.statusCode).toBe(200)- expect(JSON.parse(response.body)).toMatchObject({- servers: [],- contextRef: '*',+ expect(JSON.parse(response.body)).toEqual({+ token: 'credential-value',+ credentialRevision: expect.any(String),+ })+ expect(protectedRoute.store.secretValueReads).toBe(1)++ const invalid = await invoke(server, '/api/v2/hosts/self/mcpservers/credential', {+ method: 'POST',+ headers: { 'content-type': 'application/json' },+ body: JSON.stringify({ serverName: 'server-a', contextRef: 'context-b' }),+ })+ expect(invalid.statusCode).toBe(400)+ expect(protectedRoute.store.secretValueReads).toBe(1)+ })++ it('normalizes protected route method, media-type, and body-limit failures without a Secret read', async () => {+ const protectedRoute = protectedServer()+ server = protectedRoute.server++ let response = await invoke(server, '/api/v2/hosts/self/mcpservers', { method: 'POST' })+ expect(response.statusCode).toBe(405)+ expect(JSON.parse(response.body)).toEqual({ error: 'method_not_allowed' })+ expect(response.headers.Allow).toBe('GET')++ response = await invoke(server, '/api/v2/hosts/self/mcpservers/credential', {+ method: 'POST',+ headers: { 'content-type': 'text/plain' },+ body: '{}',+ })+ expect(response.statusCode).toBe(415)+ expect(JSON.parse(response.body)).toEqual({ error: 'unsupported_media_type' })++ response = await invoke(server, '/api/v2/hosts/self/mcpservers/credential', {+ method: 'POST',+ headers: { 'content-type': 'application/json' },+ body: JSON.stringify({ serverName: 'a'.repeat(1_025) }),+ })+ expect(response.statusCode).toBe(413)+ expect(JSON.parse(response.body)).toEqual({ error: 'payload_too_large' })+ expect(response.headers['Cache-Control']).toBe('no-store, private')+ expect(protectedRoute.store.secretValueReads).toBe(0)+ })++ it('fails closed when the protected Host verifier is not configured', async () => {+ server = new ContextMapperServer(new FakeProvider(), 0)+ server.setReady(true)++ const response = await invoke(server, '/api/v2/hosts/self/mcpservers')+ expect(response.statusCode).toBe(503)+ expect(JSON.parse(response.body)).toEqual({ error: 'authorization_unavailable' })+ expect(response.headers['Cache-Control']).toBe('no-store, private')+ })++ it('maps a rebound Host identity to a generic non-cacheable 401', async () => {+ const authenticator = { authenticate: () => principal } as unknown as McpApiAuthenticator+ const authorization = {+ listServers: async () => {+ throw new McpAuthorizationError('unauthorized')+ },+ } as unknown as McpAuthorizationService+ server = new ContextMapperServer(+ new FakeProvider(),+ 0,+ undefined,+ undefined,++ () => true,++ () => true,+ authenticator,+ authorization+ )+ server.setReady(true)++ const response = await invoke(server, '/api/v2/hosts/self/mcpservers')+ expect(response.statusCode).toBe(401)+ expect(JSON.parse(response.body)).toEqual({ error: 'unauthorized' })+ expect(response.headers['WWW-Authenticate']).toBe('Bearer realm="host-context-controller"')+ expect(response.headers['Cache-Control']).toBe('no-store, private')+ expect(response.headers.Pragma).toBe('no-cache')+ })+})++describe('McpHostApiRateLimiter', () => {+ let server: ContextMapperServer | null = null++ afterEach(async () => {+ await server?.stop()+ server = null+ vi.restoreAllMocks()+ })++ it('limits each signed Host identity/action independently and resets after one minute', () => {+ let now = 1_000+ const limiter = new McpHostApiRateLimiter(1, () => now)+ expect(limiter.consume('host-uid-a', 'credential')).toEqual({ allowed: true })+ expect(limiter.consume('host-uid-a', 'inventory')).toEqual({ allowed: true })+ expect(limiter.consume('host-uid-b', 'credential')).toEqual({ allowed: true })+ expect(limiter.consume('host-uid-a', 'credential')).toEqual({+ allowed: false,+ retryAfterSeconds: 60,})+ now += 60_000+ expect(limiter.consume('host-uid-a', 'credential')).toEqual({ allowed: true })})++ it('tracks provider authority dynamically after warm-up and fails API requests closed', async () => {+ let providerAuthoritative = true+ server = new ContextMapperServer(+ new FakeProvider(),+ 0,+ undefined,+ undefined,+ () => providerAuthoritative+ )++ let response = await invoke(server, '/ready')+ expect(response.statusCode).toBe(503)+ expect(JSON.parse(response.body)).toEqual({ status: 'starting', ready: false })++ server.setReady(true)++ response = await invoke(server, '/ready')+ expect(response.statusCode).toBe(200)+ expect(JSON.parse(response.body)).toEqual({ status: 'ready', ready: true })++ providerAuthoritative = false++ response = await invoke(server, '/ready')+ expect(response.statusCode).toBe(503)+ expect(JSON.parse(response.body)).toEqual({ status: 'degraded', ready: false })++ response = await invoke(server, '/health')+ expect(response.statusCode).toBe(200)+ expect(JSON.parse(response.body)).toEqual({+ status: 'ok',+ ready: false,+ })++ response = await invoke(server, '/api/v1/mcpservers')+ expect(response.statusCode).toBe(503)+ expect(JSON.parse(response.body)).toEqual({+ error: 'Service Unavailable',+ message: 'Context mapper provider inventory is not authoritative',+ })++ providerAuthoritative = true++ response = await invoke(server, '/ready')+ expect(response.statusCode).toBe(200)+ expect(JSON.parse(response.body)).toEqual({ status: 'ready', ready: true })++ response = await invoke(server, '/api/v1/mcpservers')+ expect(response.statusCode).toBe(200)+ })++ it('fails readiness closed when the provider authority check throws', async () => {+ const authorityError = new Error('authority unavailable')+ const errorLog = vi.spyOn(console, 'error').mockImplementation(() => undefined)+ server = new ContextMapperServer(new FakeProvider(), 0, undefined, undefined, () => {+ throw authorityError+ })+ server.setReady(true)++ const response = await invoke(server, '/ready')++ expect(response.statusCode).toBe(503)+ expect(JSON.parse(response.body)).toEqual({ status: 'degraded', ready: false })+ expect(errorLog).toHaveBeenCalledWith(+ '[Server] Provider authority readiness check failed:',+ authorityError+ )+ })++ it('withdraws readiness as soon as shutdown begins', async () => {+ server = new ContextMapperServer(new FakeProvider(), 0, undefined, undefined, () => true)+ server.setReady(true)++ expect((await invoke(server, '/ready')).statusCode).toBe(200)++ await server.stop()++ const response = await invoke(server, '/ready')+ expect(response.statusCode).toBe(503)+ expect(JSON.parse(response.body)).toEqual({ status: 'starting', ready: false })+ })+ })++ describe('ContextMapperServer desktop status authority', () => {+ let server: ContextMapperServer | null = null++ // A Host reconciler that reports a running desktop, so the only thing that can+ // withhold a 200 is the Host-authority gate under test.+ const runningReconciler = {+ getStatus: () => ({ deployed: true, ready: true, message: '' }),+ hasDesktop: () => true,+ } as unknown as ConstructorParameters<typeof ContextMapperServer>[2]+ const hasDesktopFn = () => true++ afterEach(async () => {+ await server?.stop()+ server = null+ vi.restoreAllMocks()+ })++ it('serves desktop status on Host authority alone even when the provider inventory is not authoritative', async () => {+ // The heart of H1: desktop needs only Host authority. A degraded+ // McpServer/Context lane (providerAuthoritative=false) must NOT 503 desktop,+ // while /api/v1/mcpservers still 503s on the same server.+ server = new ContextMapperServer(+ new FakeProvider(),+ 0,+ runningReconciler,+ hasDesktopFn,+ () => false, // provider inventory NOT authoritative+ () => true // Host inventory authoritative+ )+ server.setReady(true)++ const desktop = await invoke(server, '/api/v1/desktop/host-a')+ expect(desktop.statusCode).toBe(200)+ expect(JSON.parse(desktop.body)).toEqual({ status: 'running', hostRef: 'host-a' })++ const mcpservers = await invoke(server, '/api/v1/mcpservers')+ expect(mcpservers.statusCode).toBe(503)+ })++ it('fails desktop closed (503, never 200 inactive) when Host inventory is not authoritative', async () => {+ server = new ContextMapperServer(+ new FakeProvider(),+ 0,+ runningReconciler,+ hasDesktopFn,+ () => true, // provider authoritative — must not rescue desktop+ () => false // Host inventory NOT authoritative+ )+ server.setReady(true)++ const desktop = await invoke(server, '/api/v1/desktop/host-a')+ expect(desktop.statusCode).toBe(503)+ expect(JSON.parse(desktop.body)).toEqual({+ error: 'Service Unavailable',+ message: 'Host inventory is not authoritative',+ })+ })++ it('serves desktop status 200 in the dev wiring (no watcher), never 503 (R2-M1)', async () => {+ // Wiring regression for R2-M1: main.ts resolves both authority gates from the+ // same watcher. In dev (DevMcpServerProvider, watcher=null) the OLD wiring left+ // the Host gate undefined, so the server's fail-closed default pinned every+ // /api/v1/desktop/* at 503 after setReady(true) — where the base answered 200+ // {status:'inactive'}. Exercise the REAL resolvers with a null watcher, exactly+ // as main.ts does, so a revert to the undefined wiring turns this 200 into 503.+ const providerAuthoritativeFn = resolveProviderAuthoritativeFn(null)+ const hostAuthoritativeFn = resolveHostAuthoritativeFn(null)+ server = new ContextMapperServer(+ new FakeProvider(),+ 0,+ runningReconciler,+ hasDesktopFn,+ providerAuthoritativeFn,+ hostAuthoritativeFn+ )+ server.setReady(true)++ const desktop = await invoke(server, '/api/v1/desktop/host-a')+ expect(desktop.statusCode).toBe(200)+ expect(JSON.parse(desktop.body)).toEqual({ status: 'running', hostRef: 'host-a' })+ })++ it('fails desktop closed when no Host-authority gate is wired (default)', async () => {+ // Omitting the 6th arg must default to fail-closed, exactly like the+ // provider gate — no type error catches the omission.+ server = new ContextMapperServer(+ new FakeProvider(),+ 0,+ runningReconciler,+ hasDesktopFn,+ () => true+ )+ server.setReady(true)++ const desktop = await invoke(server, '/api/v1/desktop/host-a')+ expect(desktop.statusCode).toBe(503)+ expect(JSON.parse(desktop.body)).toEqual({+ error: 'Service Unavailable',+ message: 'Host inventory is not authoritative',+ })+ })})diff --cc host-context-controller/src/server.tsindex 6fa2393d7,03f576dcf..00b5df722--- a/host-context-controller/src/server.ts+++ b/host-context-controller/src/server.ts@@@ -5,83 -5,21 +5,89 @@@ import * as http from 'httpimport { config } from './config'import { HostReconciler } from './hostReconciler'import { McpServerProvider } from './k8sClient'-import { registry } from './metrics'-import { ErrorResponse, McpServersResponse } from './types'+import { hccLogger } from './logger'+import {+ McpApiAuthenticationError,+ McpApiAuthenticator,+ type VerifiedMcpHostPrincipal,+} from './mcpApiAuthentication'+import {+ McpAuthorizationError,+ McpAuthorizationService,+ toPublicMcpTransport,+} from './mcpAuthorization'+import { mcpHostApiRequestsTotal, registry } from './metrics'+import { McpServersResponse } from './types'++const MCP_SELECTOR_RE = /^[a-z0-9]([-a-z0-9.]*[a-z0-9])?$/+const MCP_REQUEST_BODY_LIMIT = 1024+const mcpApiLog = hccLogger.child({ module: 'mcp-host-api' })+type McpHostApiAction = 'inventory' | 'credential'++type RateLimitEntry = { windowStartMs: number; count: number }++/** Per-Host, per-action fixed-window limiter applied only after JWT validation. */+export class McpHostApiRateLimiter {+ private readonly entries = new Map<string, RateLimitEntry>()++ constructor(+ private readonly maxPerMinute: number,+ private readonly now: () => number = Date.now+ ) {}++ consume(+ hostUid: string,+ action: McpHostApiAction+ ): { allowed: true } | { allowed: false; retryAfterSeconds: number } {+ const nowMs = this.now()+ const key = `${hostUid}\u0000${action}`+ const existing = this.entries.get(key)+ if (!existing || nowMs - existing.windowStartMs >= 60_000) {+ this.entries.set(key, { windowStartMs: nowMs, count: 1 })+ this.prune(nowMs)+ return { allowed: true }+ }+ if (existing.count >= this.maxPerMinute) {+ return {+ allowed: false,+ retryAfterSeconds: Math.max(+ 1,+ Math.ceil((60_000 - (nowMs - existing.windowStartMs)) / 1000)+ ),+ }+ }+ existing.count += 1+ return { allowed: true }+ }++ private prune(nowMs: number): void {+ if (this.entries.size <= 10_000) return+ for (const [key, entry] of this.entries) {+ if (nowMs - entry.windowStartMs >= 60_000) this.entries.delete(key)+ }+ while (this.entries.size > 10_000) {+ const oldest = this.entries.keys().next()+ if (oldest.done) break+ this.entries.delete(oldest.value)+ }+ }+}+ type ReadinessState =+ | { ready: true; status: 'ready' }+ | { ready: false; status: 'starting' | 'degraded'; message: string }+export class ContextMapperServer {private server: http.Server | null = nullprivate provider: McpServerProviderprivate port: numberprivate hostReconciler: HostReconciler | nullprivate hasDesktopFn: ((hostRef: string) => boolean) | null+ private mcpAuthenticator: McpApiAuthenticator | null+ private mcpAuthorization: McpAuthorizationService | null+ private mcpRateLimiter: McpHostApiRateLimiter+ private providerAuthoritativeFn: () => boolean+ private hostAuthoritativeFn: () => booleanprivate ready = falseconstructor(@@@ -89,16 -27,34 +95,39 @@@port: number = config.port,hostReconciler?: HostReconciler,hasDesktopFn?: (hostRef: string) => boolean,+ // Fails closed. /ready is the assertion that no stale allow is live, and a+ // caller that omits this gate gets no type error, so the default must+ // withhold readiness rather than grant it.+ providerAuthoritativeFn: () => boolean = () => false,+ // Desktop status needs ONLY Host inventory authority, not the full+ // readiness inventory. Fails closed like providerAuthoritativeFn: a caller+ // that omits it gets a 503 on desktop rather than a wrong 200 'inactive'.- hostAuthoritativeFn: () => boolean = () => false++ hostAuthoritativeFn: () => boolean = () => false,+ mcpAuthenticator?: McpApiAuthenticator,+ mcpAuthorization?: McpAuthorizationService) {this.provider = providerthis.port = portthis.hostReconciler = hostReconciler ?? nullthis.hasDesktopFn = hasDesktopFn ?? null+ this.mcpAuthenticator = mcpAuthenticator ?? null+ this.mcpAuthorization = mcpAuthorization ?? null+ this.mcpRateLimiter = new McpHostApiRateLimiter(config.mcpHostApiRateLimitPerMinute)+ this.providerAuthoritativeFn = providerAuthoritativeFn+ this.hostAuthoritativeFn = hostAuthoritativeFn+ }++ /**+ * Host inventory authority, fail-closed on error — the desktop route's only+ * authority gate. Same shape as the provider check in getReadinessState().+ */+ private safeHostAuthoritative(): boolean {+ try {+ return this.hostAuthoritativeFn()+ } catch (err) {+ console.error('[Server] Host authority check failed:', err)+ return false+ }}/**@@@ -108,30 -64,35 +137,54 @@@this.ready = ready}+ private getReadinessState(): ReadinessState {+ if (!this.ready) {+ return {+ ready: false,+ status: 'starting',+ message: 'Context mapper is still starting',+ }+ }++ try {+ if (this.providerAuthoritativeFn()) {+ return { ready: true, status: 'ready' }+ }+ } catch (err) {+ console.error('[Server] Provider authority readiness check failed:', err)+ }++ return {+ ready: false,+ status: 'degraded',+ message: 'Context mapper provider inventory is not authoritative',+ }+ }+private async handleRequest(req: http.IncomingMessage, res: http.ServerResponse): Promise<void> {- // CORS headers- res.setHeader('Access-Control-Allow-Origin', '*')- res.setHeader('Access-Control-Allow-Methods', 'GET, OPTIONS')- res.setHeader('Access-Control-Allow-Headers', 'Content-Type')+ const url = new URL(req.url || '/', `http://localhost:${this.port}`)+ const isLegacyHostSelectedMcpRoute =+ url.pathname.startsWith('/api/v1/mcpservers/context/') ||+ (url.pathname.startsWith('/api/v1/mcpservers/') && url.pathname.endsWith('/auth'))+ const isProtectedMcpRoute =+ url.pathname.startsWith('/api/v2/hosts/self/mcpservers') || isLegacyHostSelectedMcpRoute++ // The Host credential surface is service-to-service and deliberately has+ // no browser wildcard CORS grant. Preserve the historical public CORS+ // behavior only for unrelated/legacy endpoints.+ if (!isProtectedMcpRoute) {+ res.setHeader('Access-Control-Allow-Origin', '*')+ res.setHeader('Access-Control-Allow-Methods', 'GET, OPTIONS')+ res.setHeader('Access-Control-Allow-Headers', 'Content-Type')+ }++ // Retire caller-selected Context/credential routes at the application+ // boundary for every method, including preflight, and independently of+ // readiness. This remains a defense if a caller bypasses the gateway.+ if (isLegacyHostSelectedMcpRoute) {+ this.sendProtectedJson(res, 410, { error: 'gone' })+ return+ }if (req.method === 'OPTIONS') {res.writeHead(204)@@@ -168,14 -136,30 +226,37 @@@return}- if (!this.ready) {+ // Desktop status: Host inventory is the ONLY authority this route needs, so+ // it is gated here — ABOVE the blanket readiness gate — on Host authority+ // alone. A degraded McpServer/Context lane must NOT 503 desktop, and a+ // degraded Host lane must 503 it (never answer 200 'inactive' from a stale+ // cache). See the E3 design note's endpoint→authority matrix.+ if (req.method === 'GET' && url.pathname.startsWith('/api/v1/desktop/')) {+ if (!this.ready || !this.safeHostAuthoritative()) {+ this.sendJson(res, 503, {+ error: 'Service Unavailable',+ message: 'Host inventory is not authoritative',+ })+ return+ }+ if (!this.checkDesktopAuth(req, res)) return+ const hostRef = url.pathname.replace('/api/v1/desktop/', '')+ await this.handleDesktopStatus(res, hostRef)+ return+ }++ const readiness = this.getReadinessState()+ if (!readiness.ready) {++ // NP-08: protected MCP routes get an opaque, uniform authorization error++ // instead of the generic readiness message, preserving the caller-binding++ // contract even while the authority inventory is still warming up.+ if (isProtectedMcpRoute) {+ this.sendProtectedJson(res, 503, { error: 'authorization_unavailable' })+ return+ }this.sendJson(res, 503, {error: 'Service Unavailable',- message: 'Context mapper is still starting',+ message: readiness.message,})return}@@@ -204,19 -188,7 +285,16 @@@return}- // Desktop status (requires service token)- if (req.method === 'GET' && url.pathname.startsWith('/api/v1/desktop/')) {- if (!this.checkDesktopAuth(req, res)) return- const hostRef = url.pathname.replace('/api/v1/desktop/', '')- await this.handleDesktopStatus(res, hostRef)- return- }++ // Desktop status is handled above (Host-authority-gated, ABOVE the readiness++ // gate). The v1 caller-selected Context/credential routes are retired at the++ // application boundary (410) earlier in this handler; NP-08 removed their++ // handlers, so no route block belongs here.+// 404+ if (isProtectedMcpRoute) {+ this.sendProtectedJson(res, 404, { error: 'not_found' })+ return+ }this.sendJson(res, 404, { error: 'Not Found', message: 'Endpoint not found' })}@@@ -471,11 -268,12 +549,12 @@@endpoints: {'GET /': 'This information page','GET /health': "Health check endpoint - returns { status: 'ok', ready: boolean }",- 'GET /ready': 'Readiness endpoint - returns 200 only after warm-up completes',+ 'GET /ready':+ 'Readiness endpoint - returns 200 after warm-up while provider inventory is authoritative','GET /api/v1/mcpservers': 'List all McpServer resources',- 'GET /api/v1/mcpservers/context/:contextRef':- 'List McpServers filtered by contextRef (only enabled servers)',- 'GET /api/v1/mcpservers/:name/auth': 'Get authentication token for a specific McpServer',+ 'GET /api/v2/hosts/self/mcpservers': 'List MCP servers for the authenticated Host',+ 'POST /api/v2/hosts/self/mcpservers/credential':+ 'Get an authorized credential for the authenticated Host',},examples: {listAll: `curl http://localhost:${this.port}/api/v1/mcpservers`,diff --cc host-context-controller/src/types.tsindex d71de8eb0,dfc8f722c..69d66fb56--- a/host-context-controller/src/types.ts+++ b/host-context-controller/src/types.ts@@@ -507,13 -549,14 +549,16 @@@ export interface McpServerStatusdeployed: boolean/** Whether the Deployment has at least one ready replica. */ready: boolean- /** Human-readable status message. */- message?: string/**- * Whether this readiness observation is current and authoritative. Omitted- * when the status source cannot make that guarantee.- * Whether HCC can tie this status to the current McpServer identity and- * generation. False means discovery is authoritative but runtime status has- * not yet been re-established after controller startup.++ * Whether this readiness observation is current and authoritative — HCC can++ * tie the status to the current McpServer identity and generation. Omitted++ * when the status source cannot make that guarantee; false means discovery++ * is authoritative but runtime status has not yet been re-established after++ * controller startup.*/authoritative?: boolean+ /** Human-readable status message. */+ message?: string}/**diff --cc mcp-host/src/config.tsindex a597ed40f,ec6c16b64..66a42d4b9--- a/mcp-host/src/config.ts+++ b/mcp-host/src/config.ts@@@ -556,19 -604,17 +608,25 @@@ export const config: Config =// Context Mapper poll interval (default 30 seconds)contextMapperPollInterval: parseInt(getEnv('CLERUM_CONTEXT_MAPPER_POLL_INTERVAL', '30000')!, 10),+ // Bound transient HCC/Kubernetes authority outages. Identity failures revoke+ // immediately; 5xx/transport failures may preserve the last good fleet only+ // within this finite window.+ hccAuthorityMaxStalenessMs: Math.min(+ getEnvNumber('HCC_AUTHORITY_MAX_STALENESS_MS', 60_000),+ 60_000+ ),+// MCP status heartbeat. Defaults to 30 seconds so a single missed tick does// not trip desktop staleness.- mcpStatusHeartbeatInterval: parseInt(- getEnv('CLERUM_MCP_STATUS_HEARTBEAT_INTERVAL', '30000')!,- 10+ mcpStatusHeartbeatInterval: parseMcpStatusHeartbeatDuration(+ 'CLERUM_MCP_STATUS_HEARTBEAT_INTERVAL',+ getEnv('CLERUM_MCP_STATUS_HEARTBEAT_INTERVAL'),+ 30_000+ ),+ mcpStatusHeartbeatTimeoutMs: parseMcpStatusHeartbeatDuration(+ 'CLERUM_MCP_STATUS_HEARTBEAT_TIMEOUT_MS',+ getEnv('CLERUM_MCP_STATUS_HEARTBEAT_TIMEOUT_MS'),+ 25_000),// Agent configurationdiff --cc mcp-host/src/main.tsindex 8fa84d964,20686d117..7bbc6a305--- a/mcp-host/src/main.ts+++ b/mcp-host/src/main.ts@@@ -2668,11 -2566,15 +2720,16 @@@ async function shutdown(signal: string)// Stop components in orderstopRuntimeAuthProactiveRefresh()hostWatcher?.stop()+ llmHookWatcher?.stop()configStore?.stop()stopContextMapperPolling()+ mcpAuthorityStalenessDeadline.clear()stopMcpStatusHeartbeat()statelessHeartbeat?.stop()+ if (guardrailResolveTimer) {+ clearInterval(guardrailResolveTimer)+ guardrailResolveTimer = null+ }if (agent) {await agent.stop()