Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
Merge branch 'feat/mcp-server-oauth' into feat/mcp-server-oauth-host
Diego Zacariasa729a8fa5f29
diff --cc host-context-controller/src/hostReconciler.tsindex d891b7abf,94008e22f..203831263--- a/host-context-controller/src/hostReconciler.ts+++ b/host-context-controller/src/hostReconciler.ts@@@ -3872,10 -4044,10 +4172,11 @@@ export class HostReconcilerconst ensureCurrentRuntimeTokenScope = async (): Promise<void> => {for (let attempt = 1; attempt <= 3; attempt++) {+ revalidateHostMutationBoundary()const currentScopeHash = HostReconciler.runtimeTokenScopeHash(host,- this.hasChannelIngress(host)+ this.hasChannelIngress(host),+ await this.frontsOAuthServer(host))if (runtimeTokenProvision.scopeHash === currentScopeHash) return@@@ -3884,10 -4056,10 +4185,11 @@@targetSuspended:lifecycle.effective.stateless && lifecycle.effective.state === 'suspended',})+ revalidateHostMutationBoundary()const postProvisionScopeHash = HostReconciler.runtimeTokenScopeHash(host,- this.hasChannelIngress(host)+ this.hasChannelIngress(host),+ await this.frontsOAuthServer(host))if (runtimeTokenProvision.scopeHash === postProvisionScopeHash) return@@@ -3909,10 -4082,10 +4212,11 @@@const effective = await resolveDeploymentLifecycle()const currentScopeHash = HostReconciler.runtimeTokenScopeHash(host,- this.hasChannelIngress(host)+ this.hasChannelIngress(host),+ await this.frontsOAuthServer(host))if (runtimeTokenProvision.scopeHash === currentScopeHash) {+ revalidateHostMutationBoundary()return {lifecycle: effective,runtimeTokenRevision: runtimeTokenProvision.revision,diff --cc host-context-controller/src/k8sClient.test.tsindex 50b15a673,fb1ac6231..aa9159e07--- a/host-context-controller/src/k8sClient.test.ts+++ b/host-context-controller/src/k8sClient.test.ts@@@ -3237,524 -8856,988 +8857,1051 @@@ describe('McpServerWatcher Host watch greturn { abort: vi.fn() }})- mocks.listNamespacedCustomObject.mockImplementation(async ({ plural }: { plural: string }) => {- if (plural !== 'hosts') return { items: [] }- return {- metadata: { resourceVersion: 'opaque-rv-recreated' },- items: [- {- metadata: { name: 'recreated-host', namespace: 'mcp-host', generation: 1 },- spec: { host: 'recreated-host', lifecycle: { stateless: true } },- },- ],- }+ const watcher = newContextAuthoritativeWatcher()+ const reconcile = vi+ .spyOn(watcher.getHostReconciler(), 'reconcile')+ .mockRejectedValueOnce(new Error('stale event failed'))+ .mockResolvedValue(undefined)+ await (watcher as any).startHostWatch('test-host-rv')+ if (!hostWatchCallback) throw new Error('Host watch callback was not installed')+ markHostInventoryAuthoritative(watcher)++ await hostWatchCallback('ADDED', {+ metadata: { name: 'superseded-host', namespace: 'mcp-host', generation: 1 },+ spec: { host: 'superseded-host', lifecycle: { stateless: true } },+ })+ await hostWatchCallback('MODIFIED', {+ metadata: { name: 'superseded-host', namespace: 'mcp-host', generation: 2 },+ spec: { host: 'superseded-host', lifecycle: { stateless: true } },+ })+ await vi.advanceTimersByTimeAsync(60000)++ expect(reconcile).toHaveBeenCalledTimes(2)+ expect(reconcile.mock.calls[1][0]).toMatchObject({ generation: 2 })+ expect((watcher as any).hostWatchRetryTimers.size).toBe(0)+ watcher.stop()+ })+ })++ describe('McpServerWatcher.startCommunicationChannelWatch', () => {+ it('caches ADDED events and triggers Host reconcile', async () => {+ const watcher = new McpServerWatcher()+ markHostInventoryAuthoritative(watcher)+ const hosts = (watcher as any).hosts as Map<string, any>+ hosts.set('marketing', {+ name: 'marketing',+ namespace: 'mcp-host',+ spec: { host: 'marketing', contextRef: 'c', secretRef: 's' },+ })+ const reconcileSpy = vi+ .spyOn(watcher.getHostReconciler(), 'reconcile')+ .mockResolvedValue(undefined)++ const watchCallback = (watcher as any).getCommunicationChannelWatchCallback()+ await watchCallback('ADDED', {+ metadata: { name: 'cc-marketing', namespace: 'channels' },+ spec: { hostRef: 'marketing' },+ })++ const cache = (watcher as any).communicationChannels as Map<string, any>+ expect(cache.get('cc-marketing')?.spec.hostRef).toBe('marketing')+ expect(reconcileSpy).toHaveBeenCalledWith(expect.objectContaining({ name: 'marketing' }))+ })++ it('removes from cache on DELETED and triggers Host reconcile', async () => {+ const watcher = new McpServerWatcher()+ markHostInventoryAuthoritative(watcher)+ const cache = (watcher as any).communicationChannels as Map<string, any>+ cache.set('cc-marketing', {+ name: 'cc-marketing',+ namespace: 'channels',+ spec: { hostRef: 'marketing' },+ })+ const hosts = (watcher as any).hosts as Map<string, any>+ hosts.set('marketing', {+ name: 'marketing',+ namespace: 'mcp-host',+ spec: { host: 'marketing', contextRef: 'c', secretRef: 's' },+ })+ const reconcileSpy = vi+ .spyOn(watcher.getHostReconciler(), 'reconcile')+ .mockResolvedValue(undefined)++ const watchCallback = (watcher as any).getCommunicationChannelWatchCallback()+ await watchCallback('DELETED', {+ metadata: { name: 'cc-marketing', namespace: 'channels' },+ spec: { hostRef: 'marketing' },+ })++ expect(cache.has('cc-marketing')).toBe(false)+ expect(reconcileSpy).toHaveBeenCalled()+ })++ it('on MODIFIED with hostRef change, reconciles BOTH old and new hosts', async () => {+ const watcher = new McpServerWatcher()+ markHostInventoryAuthoritative(watcher)+ const cache = (watcher as any).communicationChannels as Map<string, any>+ cache.set('cc-1', {+ name: 'cc-1',+ namespace: 'channels',+ spec: { hostRef: 'old-host' },+ })+ const hosts = (watcher as any).hosts as Map<string, any>+ hosts.set('old-host', {+ name: 'old-host',+ namespace: 'mcp-host',+ spec: { host: 'old-host', contextRef: 'c', secretRef: 's' },+ })+ hosts.set('new-host', {+ name: 'new-host',+ namespace: 'mcp-host',+ spec: { host: 'new-host', contextRef: 'c', secretRef: 's' },+ })+ const reconcileSpy = vi+ .spyOn(watcher.getHostReconciler(), 'reconcile')+ .mockResolvedValue(undefined)++ const watchCallback = (watcher as any).getCommunicationChannelWatchCallback()+ await watchCallback('MODIFIED', {+ metadata: { name: 'cc-1', namespace: 'channels' },+ spec: { hostRef: 'new-host' },+ })++ expect(reconcileSpy).toHaveBeenCalledTimes(2)+ const names = reconcileSpy.mock.calls.map(c => (c[0] as any).name).sort()+ expect(names).toEqual(['new-host', 'old-host'])+ })++ it('ignores unknown event types (e.g. BOOKMARK) without mutating cache or reconciling', async () => {+ const watcher = new McpServerWatcher()+ const cache = (watcher as any).communicationChannels as Map<string, any>+ cache.set('cc-1', {+ name: 'cc-1',+ namespace: 'channels',+ spec: { hostRef: 'marketing' },+ })+ const hosts = (watcher as any).hosts as Map<string, any>+ hosts.set('marketing', {+ name: 'marketing',+ namespace: 'mcp-host',+ spec: { host: 'marketing', contextRef: 'c', secretRef: 's' },+ })+ const reconcileSpy = vi+ .spyOn(watcher.getHostReconciler(), 'reconcile')+ .mockResolvedValue(undefined)++ const watchCallback = (watcher as any).getCommunicationChannelWatchCallback()+ await watchCallback('BOOKMARK', {+ metadata: { name: 'cc-1', namespace: 'channels' },+ spec: { hostRef: 'marketing' },})++ expect(cache.get('cc-1')?.spec.hostRef).toBe('marketing')+ expect(reconcileSpy).not.toHaveBeenCalled()+ })+ })++ describe('McpServerWatcher.startCommunicationChannelWatch — channel-reader revision rolls', () => {+ it('on CC ADDED, patches the new host channel-reader revision annotation', async () => {const watcher = new McpServerWatcher()- const reconcileDelete = vi- .spyOn(watcher.getHostReconciler(), 'reconcileDelete')+ markHostInventoryAuthoritative(watcher)+ const hosts = (watcher as any).hosts as Map<string, any>+ hosts.set('marketing', {+ name: 'marketing',+ namespace: 'mcp-host',+ spec: { host: 'marketing', contextRef: 'c', secretRef: 's' },+ })+ const patchSpy = vi+ .spyOn(watcher.getHostReconciler(), 'patchChannelReaderRevisionAnnotation').mockResolvedValue(undefined)- await (watcher as any).startHostWatch('test-host-rv')- await (watcher as any).requestHostFleetReconcile('Periodic resync', undefined, 'full')- await callbacks[0]('DELETED', {- metadata: { name: 'recreated-host', namespace: 'mcp-host', generation: 8 },- spec: { host: 'recreated-host', lifecycle: { stateless: false } },+ const watchCallback = (watcher as any).getCommunicationChannelWatchCallback()+ await watchCallback('ADDED', {+ metadata: { name: 'cc-marketing', namespace: 'channels' },+ spec: { hostRef: 'marketing' },})- expect(callbacks).toHaveLength(2)- expect(reconcileDelete).not.toHaveBeenCalled()- expect((watcher as any).hosts.get('recreated-host')).toMatchObject({- spec: { lifecycle: { stateless: true } },- })- watcher.stop()+ expect(patchSpy).toHaveBeenCalledTimes(1)+ expect(patchSpy).toHaveBeenCalledWith('marketing')})- it('recovery diff enqueues an immediate per-Host delete for a disappeared Host (#827 item 1)', async () => {- vi.clearAllMocks()+ it('preserves credentialsSecretRef from the watch event into the cache', async () => {const watcher = new McpServerWatcher()- const reconcileDelete = vi- .spyOn(watcher.getHostReconciler(), 'reconcileDelete')- .mockResolvedValue(undefined)- // Fresh authoritative snapshot holds only "live-host"; "gone-host" was in the- // pre-recovery inventory (previousNames) and has genuinely disappeared.- ;(watcher as any).enqueueRecoveredHostDeletes(- [{ name: 'live-host', namespace: 'mcp-host', uid: 'u1', spec: { host: 'live-host' } }],- new Set(['live-host', 'gone-host'])- )- await vi.waitFor(() =>- expect(reconcileDelete).toHaveBeenCalledWith('gone-host', 'mcp-host', expect.anything())+ markHostInventoryAuthoritative(watcher)+ const cache = (watcher as any).communicationChannels as Map<string, any>+ const hosts = (watcher as any).hosts as Map<string, any>+ hosts.set('marketing', {+ name: 'marketing',+ namespace: 'mcp-host',+ spec: { host: 'marketing', contextRef: 'c', secretRef: 's' },+ })+ vi.spyOn(watcher.getHostReconciler(), 'patchChannelReaderRevisionAnnotation').mockResolvedValue(+ undefined)- // A Host still present in the fresh snapshot is never deleted.- expect(reconcileDelete).not.toHaveBeenCalledWith('live-host', 'mcp-host', expect.anything())- watcher.stop()++ const watchCallback = (watcher as any).getCommunicationChannelWatchCallback()+ await watchCallback('ADDED', {+ metadata: { name: 'cc-marketing', namespace: 'channels' },+ spec: {+ hostRef: 'marketing',+ credentialsSecretRef: { name: 'cc-marketing-credentials' },+ },+ })++ const cached = cache.get('cc-marketing')+ expect(cached.spec.credentialsSecretRef).toEqual({ name: 'cc-marketing-credentials' })+ // findCommunicationChannelsByCredentialsSecretName must find the cached CC+ expect(+ watcher.findCommunicationChannelsByCredentialsSecretName('cc-marketing-credentials')+ ).toHaveLength(1)})- // The recovery diff resolves absence at LIST time. Between that diff and the- // moment the delete is admitted to the per-Host serializer, the recreation's- // ADDED can land: the watch callback sets `this.hosts` SYNCHRONOUSLY and- // enters the per-Host chain FIRST, while the fire-and-forget dispatch queues- // the delete SECOND. serializeByHost is strict FIFO, so without a fence the- // new bundle is created and then wiped. The fence must therefore read the- // LIVE cache at admission, not a value captured at enqueue time.- //- // This is the case the sibling recreation test does NOT cover: there the- // recreated Host is present in the snapshot, so enqueueRecoveredHostDeletes- // short-circuits on snapshotNames.has(name) and never dispatches at all.- it('recovery delete carries a live-cache fence for a Host recreated during admission (#827 F2 parity)', async () => {- vi.clearAllMocks()+ it('on CC MODIFIED with hostRef change, patches BOTH old and new hosts', async () => {const watcher = new McpServerWatcher()- const reconcileDelete = vi- .spyOn(watcher.getHostReconciler(), 'reconcileDelete')+ markHostInventoryAuthoritative(watcher)+ const cache = (watcher as any).communicationChannels as Map<string, any>+ cache.set('cc-1', {+ name: 'cc-1',+ namespace: 'channels',+ spec: { hostRef: 'old-host' },+ })+ const hosts = (watcher as any).hosts as Map<string, any>+ hosts.set('old-host', {+ name: 'old-host',+ namespace: 'mcp-host',+ spec: { host: 'old-host', contextRef: 'c', secretRef: 's' },+ })+ hosts.set('new-host', {+ name: 'new-host',+ namespace: 'mcp-host',+ spec: { host: 'new-host', contextRef: 'c', secretRef: 's' },+ })+ const patchSpy = vi+ .spyOn(watcher.getHostReconciler(), 'patchChannelReaderRevisionAnnotation').mockResolvedValue(undefined)- // "ghost" is absent from the fresh authoritative snapshot → genuinely- // disappeared → dispatched for delete.- ;(watcher as any).enqueueRecoveredHostDeletes([], new Set(['ghost']))- await vi.waitFor(() => expect(reconcileDelete).toHaveBeenCalled())- const [, , opts] = reconcileDelete.mock.calls[0] as [- string,- string,- { skipIf?: () => boolean } | undefined,- ]- expect(opts?.skipIf, 'the recovery delete must carry a skipIf fence').toBeTypeOf('function')+ const watchCallback = (watcher as any).getCommunicationChannelWatchCallback()+ await watchCallback('MODIFIED', {+ metadata: { name: 'cc-1', namespace: 'channels' },+ spec: { hostRef: 'new-host' },+ })- // Cache does not hold the name → the Host really is gone → do NOT skip.- expect(opts!.skipIf!()).toBe(false)+ expect(patchSpy).toHaveBeenCalledTimes(2)+ const targets = patchSpy.mock.calls.map(c => c[0] as string).sort()+ expect(targets).toEqual(['new-host', 'old-host'])+ })- // The recreation lands in the live cache during the admission window.- ;(watcher as any).hosts.set('ghost', {- name: 'ghost',+ it('on CC DELETED, patches the (former) host channel-reader revision annotation', async () => {+ const watcher = new McpServerWatcher()+ markHostInventoryAuthoritative(watcher)+ const cache = (watcher as any).communicationChannels as Map<string, any>+ cache.set('cc-marketing', {+ name: 'cc-marketing',+ namespace: 'channels',+ spec: { hostRef: 'marketing' },+ })+ const hosts = (watcher as any).hosts as Map<string, any>+ hosts.set('marketing', {+ name: 'marketing',namespace: 'mcp-host',- uid: 'uid-recreated',- generation: 1,- spec: { host: 'ghost' },+ spec: { host: 'marketing', contextRef: 'c', secretRef: 's' },})- // Same fence, re-evaluated at admission → skip the destructive delete.- expect(opts!.skipIf!()).toBe(true)- watcher.stop()+ const patchSpy = vi+ .spyOn(watcher.getHostReconciler(), 'patchChannelReaderRevisionAnnotation')+ .mockResolvedValue(undefined)++ const watchCallback = (watcher as any).getCommunicationChannelWatchCallback()+ await watchCallback('DELETED', {+ metadata: { name: 'cc-marketing', namespace: 'channels' },+ spec: { hostRef: 'marketing' },+ })++ expect(patchSpy).toHaveBeenCalledTimes(1)+ expect(patchSpy).toHaveBeenCalledWith('marketing')})- it('recovery diff reconciles (does NOT delete) a same-name Host recreated with a new uid (#827 items 1+2)', async () => {- vi.clearAllMocks()+ it('swallows patchChannelReaderRevisionAnnotation errors without breaking the watch loop', async () => {const watcher = new McpServerWatcher()- const reconcile = vi- .spyOn(watcher.getHostReconciler(), 'reconcile')- .mockResolvedValue(undefined)- const reconcileDelete = vi- .spyOn(watcher.getHostReconciler(), 'reconcileDelete')- .mockResolvedValue(undefined)- // The recreated Host is the CURRENT cache entry the urgent dispatch resolves.- ;(watcher as any).hosts.set('x', {- name: 'x',+ markHostInventoryAuthoritative(watcher)+ const hosts = (watcher as any).hosts as Map<string, any>+ hosts.set('marketing', {+ name: 'marketing',namespace: 'mcp-host',- uid: 'uid-B',- generation: 1,- spec: { host: 'x' },+ spec: { host: 'marketing', contextRef: 'c', secretRef: 's' },})- // Same name, same generation (reset to 1), but a DIFFERENT uid than before.- ;(watcher as any).enqueueRecoveredUrgentHosts(- [{ name: 'x', namespace: 'mcp-host', uid: 'uid-B', generation: 1, spec: { host: 'x' } }],- new Set(['x']),- new Map([['x', 1]]),- new Map([['x', 'uid-A']])- )- // The snapshot still contains "x" by name, so the recovery-delete diff never- // treats it as a disappearance.- ;(watcher as any).enqueueRecoveredHostDeletes(- [{ name: 'x', namespace: 'mcp-host', uid: 'uid-B', generation: 1, spec: { host: 'x' } }],- new Set(['x'])- )- await vi.waitFor(() =>- expect(reconcile).toHaveBeenCalledWith(expect.objectContaining({ name: 'x' }), 'urgent')+ vi.spyOn(watcher.getHostReconciler(), 'patchChannelReaderRevisionAnnotation').mockRejectedValue(+ new Error('boom'))- expect(reconcileDelete).not.toHaveBeenCalled()+ const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {})++ const watchCallback = (watcher as any).getCommunicationChannelWatchCallback()+ await expect(+ watchCallback('ADDED', {+ metadata: { name: 'cc-marketing', namespace: 'channels' },+ spec: { hostRef: 'marketing' },+ })+ ).resolves.toBeUndefined()+ expect(errorSpy).toHaveBeenCalled()++ errorSpy.mockRestore()+ })+ })++ describe('externalEgressResyncDelayMs (issue #299 H2)', () => {+ it('returns the configured interval (ms) when no TTL has been observed', () => {+ expect(externalEgressResyncDelayMs(60, 5)).toBe(60_000)+ expect(externalEgressResyncDelayMs(60, 5, Infinity)).toBe(60_000)+ })++ it('advances to TTL/2 when the observed TTL is faster than the interval', () => {+ expect(externalEgressResyncDelayMs(60, 5, 15_000)).toBe(7_500)+ })++ it('never drops below the floor even for a tiny TTL', () => {+ expect(externalEgressResyncDelayMs(60, 5, 6_000)).toBe(5_000)+ })++ it('keeps the configured interval when the observed TTL is large', () => {+ expect(externalEgressResyncDelayMs(60, 5, 600_000)).toBe(60_000)+ })++ it('clamps a below-floor configured interval up to the floor', () => {+ expect(externalEgressResyncDelayMs(3, 5)).toBe(5_000)+ })+ })+++describe('McpServerWatcher.hostFrontsOAuthServer (oauth:user-token scope probe)', () => {++ const hostFor = (contextRef: string): HostCRD =>++ ({++ name: 'h',++ namespace: 'mcp-host',++ spec: { host: 'h', contextRef, secretRef: 's' },++ }) as HostCRD++++ it('returns true when the Context fronts an enabled auth.type=oauth mcp-server', async () => {++ const watcher = new McpServerWatcher()++ vi.spyOn(watcher, 'getServerInfosByContext').mockResolvedValue([++ {++ name: 'notion',++ contextRef: 'ctx',++ transport: { type: 'streamableHttp' },++ enabled: true,++ auth: { type: 'oauth' },++ status: {},++ },++ ] as any)++ await expect((watcher as any).hostFrontsOAuthServer(hostFor('ctx'))).resolves.toBe(true)+ watcher.stop()+ })+- it('suppresses a watch event whose identity (uid) differs from the cached Host (#827 item 2)', () => {++ it('returns false for a non-oauth mcp-server', async () => {+ const watcher = new McpServerWatcher()- ;(watcher as any).hosts.set('x', {- name: 'x',- namespace: 'mcp-host',- uid: 'uid-B',- generation: 3,- })- ;(watcher as any).latestHostWatchEventRevisions.set('x', 7)- // A stale MODIFIED carrying the OLD identity must not act on the recreated one.- expect(- (watcher as any).isCurrentHostWatchEvent(- 'MODIFIED',- { name: 'x', namespace: 'mcp-host', uid: 'uid-A', generation: 3 },- 7- )- ).toBe(false)- // The current identity is admitted.- expect(- (watcher as any).isCurrentHostWatchEvent(- 'MODIFIED',- { name: 'x', namespace: 'mcp-host', uid: 'uid-B', generation: 3 },- 7- )- ).toBe(true)++ vi.spyOn(watcher, 'getServerInfosByContext').mockResolvedValue([++ {++ name: 'redis',++ contextRef: 'ctx',++ transport: { type: 'streamableHttp' },++ enabled: true,++ auth: { type: 'bearer' },++ status: {},++ },++ ] as any)++ await expect((watcher as any).hostFrontsOAuthServer(hostFor('ctx'))).resolves.toBe(false)+ watcher.stop()+ })+- it('recovery urgently admits a wake-pending Host that is otherwise unchanged (§10.2 step 7 wake branch)', async () => {- vi.clearAllMocks()++ it('returns false for a disabled oauth mcp-server', async () => {+ const watcher = new McpServerWatcher()- const reconcile = vi- .spyOn(watcher.getHostReconciler(), 'reconcile')- .mockResolvedValue(undefined)- // A Host unchanged by name, generation AND uid but carrying a pending wake- // annotation must still be admitted on the urgent lane after recovery, so a- // wake discovered during the LIST→WATCH gap is not deferred to the slow- // background fleet pass (A3: 2m47s admission vs 3s execution). This closes- // the recovery-time wake branch that the watch-time admission test does not- // exercise: deleting `|| wakePending` from enqueueRecoveredUrgentHosts()- // would make this assertion fail.- ;(watcher as any).hosts.set('waker', {- name: 'waker',- namespace: 'mcp-host',- uid: 'u1',- generation: 4,- annotations: { 'clerum.io/wake-requested': '7' },- spec: { host: 'waker' },- })- ;(watcher as any).enqueueRecoveredUrgentHosts(- [- {- name: 'waker',- namespace: 'mcp-host',- uid: 'u1',- generation: 4,- annotations: { 'clerum.io/wake-requested': '7' },- spec: { host: 'waker' },- },- ],- new Set(['waker']), // NOT new- new Map([['waker', 4]]), // NOT changed (same generation)- new Map([['waker', 'u1']]) // NOT recreated (same uid)- )- await vi.waitFor(() =>- expect(reconcile).toHaveBeenCalledWith(expect.objectContaining({ name: 'waker' }), 'urgent')- )++ vi.spyOn(watcher, 'getServerInfosByContext').mockResolvedValue([++ {++ name: 'notion',++ contextRef: 'ctx',++ transport: { type: 'streamableHttp' },++ enabled: false,++ auth: { type: 'oauth' },++ status: {},++ },++ ] as any)++ await expect((watcher as any).hostFrontsOAuthServer(hostFor('ctx'))).resolves.toBe(false)+ watcher.stop()+ })+- it('recovery does NOT urgently dispatch a fully-unchanged Host with no wake annotation', async () => {- vi.clearAllMocks()++ it('returns false when the Context fronts no servers', async () => {+ const watcher = new McpServerWatcher()- const reconcile = vi- .spyOn(watcher.getHostReconciler(), 'reconcile')- .mockResolvedValue(undefined)- // Same name, generation and uid as before recovery and no wake pending: the- // background fleet pass already covers it, so recovery must NOT enqueue a- // redundant urgent reconcile (guards the negative side of the predicate).- ;(watcher as any).hosts.set('steady', {- name: 'steady',- namespace: 'mcp-host',- uid: 'u1',- generation: 4,- spec: { host: 'steady' },- })- ;(watcher as any).enqueueRecoveredUrgentHosts(- [- {- name: 'steady',- namespace: 'mcp-host',- uid: 'u1',- generation: 4,- spec: { host: 'steady' },- },- ],- new Set(['steady']),- new Map([['steady', 4]]),- new Map([['steady', 'u1']])- )- // Let any erroneously-scheduled async dispatch run before asserting absence.- await Promise.resolve()- await Promise.resolve()- expect(reconcile).not.toHaveBeenCalled()++ vi.spyOn(watcher, 'getServerInfosByContext').mockResolvedValue([] as any)++ await expect((watcher as any).hostFrontsOAuthServer(hostFor('ctx'))).resolves.toBe(false)+ watcher.stop()+ })-- it('reconciles a retried Host watch event on the "retry" lane, not "urgent" (F1)', async () => {++})+ describe('McpServerWatcher readiness under sustained watch churn (GKE Premature-close regime)', () => {+ // Regression for the clerum-dev livelock. PR #205 coupled /ready to a safety+ // certificate whose validity was pinned to the watch GENERATION. On GKE the+ // apiserver closes long-lived watches ("Premature close") every few minutes;+ // each reconnect re-LISTs and bumps the generation, so every certification+ // pass aborted before it could persist and /ready stayed 503 forever — even+ // though the re-listed inventory was byte-identical. minikube never reproduced+ // this because its single local apiserver does not drop watches, which is+ // exactly why T2 stayed green while dev broke. This test injects the sustained+ // churn deterministically so the regression is caught in CI, not in production.+ it('certifies readiness under sustained Context watch churn when every re-list returns identical inventory', async () => {vi.useFakeTimers()- let hostWatchCallback: ((type: string, apiObj: any) => Promise<void>) | undefined- mocks.watch.mockImplementation(- async (- _path: string,- _params: object,- callback: (type: string, apiObj: any) => Promise<void>- ) => {- hostWatchCallback = callback- return { abort: vi.fn() }- }- )- const watcher = new McpServerWatcher()- const reconcile = vi- .spyOn(watcher.getHostReconciler(), 'reconcile')- .mockRejectedValueOnce(new Error('transient'))- .mockResolvedValue(undefined)- await (watcher as any).startHostWatch('test-host-rv')- if (!hostWatchCallback) throw new Error('Host watch callback was not installed')- await hostWatchCallback('MODIFIED', {- metadata: { name: 'retry-lane-host', namespace: 'mcp-host', generation: 2 },- spec: { host: 'retry-lane-host', lifecycle: { stateless: true } },+ // Stable desired state: the same Context in every LIST. Models clerum-dev,+ // where nothing changed — only the watch dropped and reconnected.+ const stableContexts = [+ {+ metadata: {+ name: 'churn-context',+ namespace: 'mcp-server',+ uid: 'churn-uid-1',+ generation: 1,+ },+ spec: { contextId: 'churn-context', mcpServers: [] },+ },+ ]+ mocks.listNamespacedCustomObject.mockImplementation(async ({ plural }: { plural: string }) => {+ if (plural === 'contexts') {+ return { metadata: { resourceVersion: 'ctx-rv-stable' }, items: stableContexts }+ }+ if (plural === 'mcpservers') {+ return { metadata: { resourceVersion: 'mcp-rv-stable' }, items: [] }+ }+ if (plural === 'hosts') {+ return { metadata: { resourceVersion: 'host-rv-stable' }, items: [] }+ }+ if (plural === 'communicationchannels') {+ return { metadata: { resourceVersion: 'cc-rv-stable' }, items: [] }+ }+ return { items: [] }})- // First attempt is the direct/urgent admission.- expect(reconcile).toHaveBeenNthCalledWith(- 1,- expect.objectContaining({ name: 'retry-lane-host' }),- 'urgent'- )- await vi.advanceTimersByTimeAsync(5000)- // The retry attempt populates the distinct "retry" lane.- expect(reconcile).toHaveBeenNthCalledWith(- 2,- expect.objectContaining({ name: 'retry-lane-host' }),- 'retry'- )- watcher.stop()- vi.useRealTimers()- })- it('automatically retries a transient Host watch reconciliation failure', async () => {- vi.useFakeTimers()- let hostWatchCallback: ((type: string, apiObj: any) => Promise<void>) | undefined- mocks.watch.mockImplementation(- async (- _path: string,- _params: object,- callback: (type: string, apiObj: any) => Promise<void>- ) => {- hostWatchCallback = callback- return { abort: vi.fn() }- }- )+ const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {})+ const warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => {})+ const logSpy = vi.spyOn(console, 'log').mockImplementation(() => {})const watcher = new McpServerWatcher()- const unrelatedHost = {- name: 'unrelated-host',- namespace: 'mcp-host',- generation: 4,- spec: { host: 'unrelated-host', lifecycle: { stateless: true } },- }- ;(watcher as any).hosts.set(unrelatedHost.name, unrelatedHost)- const reconcile = vi- .spyOn(watcher.getHostReconciler(), 'reconcile')- .mockRejectedValueOnce(Object.assign(new Error('disappeared'), { code: 404 }))- .mockResolvedValue(undefined)- await (watcher as any).startHostWatch('test-host-rv')- const watchGeneration = (watcher as any).hostWatchGeneration- if (!hostWatchCallback) throw new Error('Host watch callback was not installed')-- await hostWatchCallback('ADDED', {- metadata: { name: 'retry-host', namespace: 'mcp-host', generation: 1 },- spec: { host: 'retry-host', lifecycle: { stateless: true } },- })- expect(reconcile).toHaveBeenCalledTimes(1)- await vi.advanceTimersByTimeAsync(4999)- expect(reconcile).toHaveBeenCalledTimes(1)- await vi.advanceTimersByTimeAsync(1)- expect(reconcile).toHaveBeenCalledTimes(2)- expect((watcher as any).hostWatchRetryTimers.size).toBe(0)- expect((watcher as any).hostWatchRetryAttempts.size).toBe(0)- expect(- mocks.listNamespacedCustomObject.mock.calls.filter(([request]) => request.plural === 'hosts')- ).toHaveLength(0)- expect(mocks.watch).toHaveBeenCalledOnce()- expect(mocks.hostFullReconcile).not.toHaveBeenCalled()- expect((watcher as any).hostWatchGeneration).toBe(watchGeneration)- expect((watcher as any).hosts.get(unrelatedHost.name)).toEqual(unrelatedHost)- watcher.stop()- })+ vi.spyOn(watcher as any, 'startSharedFileSystemWatch').mockResolvedValue(undefined)+ vi.spyOn(watcher as any, 'startGlobalFileSystemWatch').mockResolvedValue(undefined)- it('bounds repeated Host watch retries and stops after exhaustion', async () => {- vi.useFakeTimers()- let hostWatchCallback: ((type: string, apiObj: any) => Promise<void>) | undefined- mocks.watch.mockImplementation(+ // Model a "Premature close" reconnect that lands DURING each authoritative+ // revocation pass: the pass captures its safety certificate, then the watch+ // reconnects and recovery re-LISTs the identical inventory — so the watch+ // GENERATION advances (channel identity) while the desired REVISION (content+ // identity) is unchanged and the cache stays authoritative. The #205 code+ // pinned the certificate to the generation, so it refused here on every pass+ // and livelocked; the fix pins it to the revision, so it certifies through+ // the churn.+ let passes = 0+ let reconnects = 0+ mocks.netPolFullReconcile.mockImplementation(async (- _path: string,- _params: object,- callback: (type: string, apiObj: any) => Promise<void>+ _ctxs: unknown,+ _servers: unknown,+ options: { onAuthoritativeRevocationComplete?: () => void } | undefined) => {- hostWatchCallback = callback- return { abort: vi.fn() }+ passes += 1+ ;(watcher as any).contextWatchGeneration += 2+ ;(watcher as any).mcpWatchGeneration += 2+ reconnects += 1+ options?.onAuthoritativeRevocationComplete?.()})- const watcher = new McpServerWatcher()- const reconcile = vi- .spyOn(watcher.getHostReconciler(), 'reconcile')- .mockRejectedValue(new Error('persistent failure'))- await (watcher as any).startHostWatch('test-host-rv')- if (!hostWatchCallback) throw new Error('Host watch callback was not installed')- await hostWatchCallback('ADDED', {- metadata: { name: 'bounded-host', namespace: 'mcp-host', generation: 1 },- spec: { host: 'bounded-host', lifecycle: { stateless: true } },- })- await vi.advanceTimersByTimeAsync(5000)- await vi.advanceTimersByTimeAsync(15000)- await vi.advanceTimersByTimeAsync(30000)- expect(reconcile).toHaveBeenCalledTimes(4)- await vi.advanceTimersByTimeAsync(300000)- expect(reconcile).toHaveBeenCalledTimes(4)- expect((watcher as any).hostWatchRetryTimers.size).toBe(0)- expect((watcher as any).hostWatchRetryAttempts.size).toBe(0)- watcher.stop()+ const server = new ContextMapperServer(watcher, 0, undefined, undefined, () =>+ watcher.isReadinessInventoryAuthoritative()+ )+ await server.start()++ try {+ await watcher.start()+ server.setReady(true)++ // Sustain the churn for a fixed number of authoritative passes. Each pass+ // takes a reconnect (generation bump) with a byte-identical re-LIST. We do+ // NOT stop early: the regime must run enough times that a livelock is+ // unmistakable, and the fix must hold readiness through all of it.+ const CHURN_PASSES = 5+ for (let i = 0; i < CHURN_PASSES; i += 1) {+ await (watcher as any).runInitialNetworkPolicyConvergence()+ await flushMicrotasks()+ }++ // The adverse regime actually ran (zero-tests-is-never-success).+ expect(passes).toBeGreaterThanOrEqual(CHURN_PASSES)+ expect(reconnects).toBeGreaterThanOrEqual(CHURN_PASSES)+ // The livelock assertion: an identical re-LIST must not keep readiness+ // down. #205 code → false (livelock). Fixed code → true.+ expect(+ watcher.isReadinessInventoryAuthoritative(),+ `livelock: ${passes} authoritative passes completed under sustained churn ` ++ `(${reconnects} reconnects, identical inventory on every re-LIST) ` ++ `and none certified the safety inventory`+ ).toBe(true)+ expect((await requestReadyOverHttp(server)).statusCode).toBe(200)+ } finally {+ await server.stop()+ await watcher.stop()+ errorSpy.mockRestore()+ warnSpy.mockRestore()+ logSpy.mockRestore()+ }})- it('cancels an older failed Host retry when a newer event succeeds', async () => {+ // R1 (fail-closed guardrail): the content-identity gate must NOT paper over a+ // genuine safety failure. If the authoritative pass could not revoke a real+ // stale allow (hasCertifiedSafetyInventory stays false), readiness must remain+ // 503 no matter how quiet the content is. This is the barrier that stops a+ // future "fix" from certifying on identical content when safety never certified.+ it('stays fail-closed under sustained churn when the safety pass never certified', async () => {vi.useFakeTimers()- let hostWatchCallback: ((type: string, apiObj: any) => Promise<void>) | undefined- mocks.watch.mockImplementation(- async (- _path: string,- _params: object,- callback: (type: string, apiObj: any) => Promise<void>- ) => {- hostWatchCallback = callback- return { abort: vi.fn() }+ mocks.listNamespacedCustomObject.mockImplementation(async ({ plural }: { plural: string }) => {+ if (plural === 'contexts') {+ return {+ metadata: { resourceVersion: 'ctx-rv-stable' },+ items: [+ {+ metadata: { name: 'fc-ctx', namespace: 'mcp-server', uid: 'fc-uid', generation: 1 },+ spec: { contextId: 'fc-ctx', mcpServers: [] },+ },+ ],+ }}+ if (plural === 'mcpservers') return { metadata: { resourceVersion: 'mcp-rv' }, items: [] }+ if (plural === 'hosts') return { metadata: { resourceVersion: 'host-rv' }, items: [] }+ if (plural === 'communicationchannels')+ return { metadata: { resourceVersion: 'cc-rv' }, items: [] }+ return { items: [] }+ })+ const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {})+ const warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => {})+ const logSpy = vi.spyOn(console, 'log').mockImplementation(() => {})+ const watcher = new McpServerWatcher()+ vi.spyOn(watcher as any, 'startSharedFileSystemWatch').mockResolvedValue(undefined)+ vi.spyOn(watcher as any, 'startGlobalFileSystemWatch').mockResolvedValue(undefined)+ // The authoritative pass aborts revoking a real stale allow: it never certifies.+ mocks.hasCertifiedSafetyInventory.mockReturnValue(false)+ let passes = 0+ mocks.netPolFullReconcile.mockImplementation(async () => {+ passes += 1+ ;(watcher as any).contextWatchGeneration += 2+ ;(watcher as any).mcpWatchGeneration += 2+ // No onAuthoritativeRevocationComplete: the revocation did not complete.+ })+ const server = new ContextMapperServer(watcher, 0, undefined, undefined, () =>+ watcher.isReadinessInventoryAuthoritative())+ await server.start()+ try {+ await watcher.start()+ server.setReady(true)+ const CHURN_PASSES = 5+ for (let i = 0; i < CHURN_PASSES; i += 1) {+ await (watcher as any).runInitialNetworkPolicyConvergence()+ await flushMicrotasks()+ }+ expect(passes).toBeGreaterThanOrEqual(CHURN_PASSES)+ expect(+ watcher.isReadinessInventoryAuthoritative(),+ 'fail-closed violated: readiness certified while the safety pass never did'+ ).toBe(false)+ expect((await requestReadyOverHttp(server)).statusCode).toBe(503)+ } finally {+ await server.stop()+ await watcher.stop()+ mocks.hasCertifiedSafetyInventory.mockReturnValue(true)+ errorSpy.mockRestore()+ warnSpy.mockRestore()+ logSpy.mockRestore()+ }+ })++ it('installContextSnapshot bumps the desired revision only when re-listed content changed', () => {const watcher = new McpServerWatcher()- const reconcile = vi- .spyOn(watcher.getHostReconciler(), 'reconcile')- .mockRejectedValueOnce(new Error('stale event failed'))- .mockResolvedValue(undefined)- await (watcher as any).startHostWatch('test-host-rv')- if (!hostWatchCallback) throw new Error('Host watch callback was not installed')+ const ctxA = {+ name: 'a',+ namespace: 'mcp-server',+ uid: 'ua',+ generation: 1,+ spec: { contextId: 'a', mcpServers: [] },+ }+ ;(watcher as any).installContextSnapshot({ contexts: [ctxA], resourceVersion: 'rv1' })+ const base = (watcher as any).contextDesiredRevision+ // Identical re-LIST (a Premature-close reconnect) must NOT bump.+ ;(watcher as any).installContextSnapshot({ contexts: [ctxA], resourceVersion: 'rv2' })+ expect((watcher as any).contextDesiredRevision).toBe(base)+ // A spec change on the same identity MUST bump.+ ;(watcher as any).installContextSnapshot({+ contexts: [{ ...ctxA, generation: 2, spec: { contextId: 'a', mcpServers: ['s1'] } }],+ resourceVersion: 'rv3',+ })+ expect((watcher as any).contextDesiredRevision).toBe(base + 1)+ // A removal MUST bump.+ ;(watcher as any).installContextSnapshot({ contexts: [], resourceVersion: 'rv4' })+ expect((watcher as any).contextDesiredRevision).toBe(base + 2)+ // An addition MUST bump.+ ;(watcher as any).installContextSnapshot({ contexts: [ctxA], resourceVersion: 'rv5' })+ expect((watcher as any).contextDesiredRevision).toBe(base + 3)+ })- await hostWatchCallback('ADDED', {- metadata: { name: 'superseded-host', namespace: 'mcp-host', generation: 1 },- spec: { host: 'superseded-host', lifecycle: { stateless: true } },- })- await hostWatchCallback('MODIFIED', {- metadata: { name: 'superseded-host', namespace: 'mcp-host', generation: 2 },- spec: { host: 'superseded-host', lifecycle: { stateless: true } },- })- await vi.advanceTimersByTimeAsync(60000)+ it('installMcpServerSnapshot bumps the desired revision only when re-listed content changed', () => {+ const watcher = new McpServerWatcher()+ const srvA = {+ name: 'a',+ namespace: 'mcp-server',+ uid: 'ua',+ generation: 1,+ spec: {+ contextRef: 'ctx',+ image: 'clerum/a:v1',+ transport: { type: 'streamableHttp' as const, port: 3000 },+ },+ }+ ;(watcher as any).installMcpServerSnapshot({ servers: [srvA], resourceVersion: 'rv1' })+ const base = (watcher as any).mcpServerDesiredRevision+ // Identical re-LIST must NOT bump.+ ;(watcher as any).installMcpServerSnapshot({ servers: [srvA], resourceVersion: 'rv2' })+ expect((watcher as any).mcpServerDesiredRevision).toBe(base)+ // A spec change MUST bump.+ ;(watcher as any).installMcpServerSnapshot({+ servers: [{ ...srvA, generation: 2, spec: { ...srvA.spec, image: 'clerum/a:v2' } }],+ resourceVersion: 'rv3',+ })+ expect((watcher as any).mcpServerDesiredRevision).toBe(base + 1)+ // A removal MUST bump.+ ;(watcher as any).installMcpServerSnapshot({ servers: [], resourceVersion: 'rv4' })+ expect((watcher as any).mcpServerDesiredRevision).toBe(base + 2)+ })- expect(reconcile).toHaveBeenCalledTimes(2)- expect(reconcile.mock.calls[1][0]).toMatchObject({ generation: 2 })- expect((watcher as any).hostWatchRetryTimers.size).toBe(0)- watcher.stop()+ it('installHostSnapshot bumps the desired revision only when re-listed content changed', () => {+ const watcher = new McpServerWatcher()+ const hostA = {+ name: 'a',+ namespace: 'mcp-host',+ uid: 'ua',+ generation: 1,+ spec: { contextRef: 'ctx' },+ } as unknown as HostCRD+ ;(watcher as any).installHostSnapshot({ hosts: [hostA], resourceVersion: 'rv1' })+ const base = (watcher as any).hostDesiredRevision+ // Identical re-LIST (a Premature-close reconnect) must NOT bump — this is+ // what stops the Host mutation-authority fence from starving every reconcile.+ ;(watcher as any).installHostSnapshot({ hosts: [hostA], resourceVersion: 'rv2' })+ expect((watcher as any).hostDesiredRevision).toBe(base)+ // A spec change (metadata.generation advances) MUST bump.+ ;(watcher as any).installHostSnapshot({+ hosts: [{ ...hostA, generation: 2 }],+ resourceVersion: 'rv3',+ })+ expect((watcher as any).hostDesiredRevision).toBe(base + 1)+ // A removal MUST bump.+ ;(watcher as any).installHostSnapshot({ hosts: [], resourceVersion: 'rv4' })+ expect((watcher as any).hostDesiredRevision).toBe(base + 2)})- })- describe('McpServerWatcher.startCommunicationChannelWatch', () => {- it('caches ADDED events and triggers Host reconcile', async () => {+ it('installHostSnapshot is deliberately blind to Host annotations/labels (pins the mutation-relevance invariant)', () => {+ // Guards the SCOPE note on sameHostDesiredRevision: the Host comparator is+ // uid+generation only, unlike the McpServer/Context comparators that hash+ // labels/annotations. That is correct ONLY while no Host annotation/label is+ // load-bearing for a mutation (the wake annotation drives dispatch, not+ // template content). This test pins that contract: annotation/label churn at+ // the SAME (uid, generation) — e.g. a Premature-close re-LIST — must NOT bump+ // the desired revision. If a Host annotation/label ever becomes mutation-+ // relevant, whoever extends the comparator to hash it will break this test and+ // be forced to re-examine the fail-open the SCOPE note warns about.const watcher = new McpServerWatcher()- const hosts = (watcher as any).hosts as Map<string, any>- hosts.set('marketing', {- name: 'marketing',+ const hostA = {+ name: 'a',namespace: 'mcp-host',- spec: { host: 'marketing', contextRef: 'c', secretRef: 's' },+ uid: 'ua',+ generation: 1,+ spec: { contextRef: 'ctx' },+ annotations: { 'clerum.io/wake': 'v1' },+ labels: { tier: 'a' },+ } as unknown as HostCRD+ ;(watcher as any).installHostSnapshot({ hosts: [hostA], resourceVersion: 'rv1' })+ const base = (watcher as any).hostDesiredRevision+ ;(watcher as any).installHostSnapshot({+ hosts: [+ {+ ...hostA,+ annotations: { 'clerum.io/wake': 'v2', 'clerum.io/added': 'x' },+ labels: { tier: 'b' },+ },+ ],+ resourceVersion: 'rv2',})- const reconcileSpy = vi- .spyOn(watcher.getHostReconciler(), 'reconcile')- .mockResolvedValue(undefined)+ expect((watcher as any).hostDesiredRevision).toBe(base)+ })+ })- const watchCallback = (watcher as any).getCommunicationChannelWatchCallback()- await watchCallback('ADDED', {- metadata: { name: 'cc-marketing', namespace: 'channels' },- spec: { hostRef: 'marketing' },- })+ describe('McpServerWatcher watch-close recovery latency (immediate first attempt + floor)', () => {+ // GKE closes long-lived watches ("Premature close") every few minutes. The+ // old contract armed a FIXED 5s timer before the first re-LIST, so every+ // isolated close cost ~5.5s of /ready 503. The hardened contract: the FIRST+ // recovery attempt after a close is IMMEDIATE (microtask, no timer), a paced+ // timer survives only as the retry-after-failure pacing (now the jittered+ // exponential backoff: first failure retries within [500ms, 1000ms]), and an+ // anti-busy-loop FLOOR (1s) demotes a close that arrives <1s after the last+ // successful recovery back onto the paced timer so a degraded apiserver is+ // never hammered with back-to-back re-LISTs.+ const inventoryLanes = [+ {+ kind: 'McpServer',+ plural: 'mcpservers',+ pathSuffix: '/mcpservers',+ restartMethod: 'restartMcpServerWatch',+ startSnapshot: { resourceVersion: 'mcp-start-rv', servers: [] },+ startRv: 'mcp-start-rv',+ recoveryRvPrefix: 'mcp-close-recovery-rv',+ timerField: 'mcpServerCacheRecoveryTimer',+ syncedField: 'mcpServerCacheSynced',+ peerSyncedField: 'contextCacheSynced',+ },+ {+ kind: 'Context',+ plural: 'contexts',+ pathSuffix: '/contexts',+ restartMethod: 'restartContextWatch',+ startSnapshot: { resourceVersion: 'context-start-rv', contexts: [] },+ startRv: 'context-start-rv',+ recoveryRvPrefix: 'context-close-recovery-rv',+ timerField: 'contextCacheRecoveryTimer',+ syncedField: 'contextCacheSynced',+ peerSyncedField: 'mcpServerCacheSynced',+ },+ ] as const- const cache = (watcher as any).communicationChannels as Map<string, any>- expect(cache.get('cc-marketing')?.spec.hostRef).toBe('marketing')- expect(reconcileSpy).toHaveBeenCalledWith(expect.objectContaining({ name: 'marketing' }))+ beforeEach(() => {+ vi.clearAllMocks()+ vi.useFakeTimers()+ mocks.watch.mockReset().mockResolvedValue({ abort: vi.fn() })+ mocks.listNamespacedCustomObject.mockReset().mockResolvedValue({ items: [] })})- it('removes from cache on DELETED and triggers Host reconcile', async () => {- const watcher = new McpServerWatcher()- const cache = (watcher as any).communicationChannels as Map<string, any>- cache.set('cc-marketing', {- name: 'cc-marketing',- namespace: 'channels',- spec: { hostRef: 'marketing' },+ afterEach(() => {+ vi.useRealTimers()+ })++ it.each(inventoryLanes)(+ 're-LISTs the $kind inventory immediately (microtask, no 5s timer) when its watch closes',+ async lane => {+ const doneCallbacks: Array<(error: Error | null) => void> = []+ const watchQueries: unknown[] = []+ let listCalls = 0+ mocks.watch.mockImplementation(async (path, options, _callback, done) => {+ if (path.endsWith(lane.pathSuffix)) {+ doneCallbacks.push(done)+ watchQueries.push(options)+ }+ return { abort: vi.fn() }+ })+ mocks.listNamespacedCustomObject.mockImplementation(+ async ({ plural }: { plural: string }) => {+ if (plural === lane.plural) {+ listCalls += 1+ return {+ metadata: { resourceVersion: `${lane.recoveryRvPrefix}-${listCalls}` },+ items: [],+ }+ }+ return { items: [] }+ }+ )+ const logSpy = vi.spyOn(console, 'log').mockImplementation(() => {})+ const watcher = new McpServerWatcher()+ ;(watcher as any)[lane.peerSyncedField] = true+ await (watcher as any)[lane.restartMethod](lane.startSnapshot)+ expect(watchQueries).toEqual([{ resourceVersion: lane.startRv }])++ doneCallbacks[0](Object.assign(new Error('Premature close'), { statusCode: 500 }))+ await flushMicrotasks(20)++ // Immediate first attempt: fresh LIST + fresh watch WITHOUT advancing+ // any timer. Under the old contract this stayed at 0 until +5000ms.+ expect(listCalls).toBe(1)+ expect(watchQueries).toEqual([+ { resourceVersion: lane.startRv },+ { resourceVersion: `${lane.recoveryRvPrefix}-1` },+ ])+ expect((watcher as any)[lane.syncedField]).toBe(true)+ expect((watcher as any)[lane.timerField]).toBeNull()++ // No retry timer was armed: +5s must not produce a duplicate re-LIST.+ await vi.advanceTimersByTimeAsync(5000)+ expect(listCalls).toBe(1)+ logSpy.mockRestore()+ await watcher.stop()+ }+ )++ it('re-LISTs the Host inventory immediately (microtask, no 5s timer) when its watch closes', async () => {+ const doneCallbacks: Array<(error: Error | null) => void> = []+ const watchQueries: unknown[] = []+ let hostListCalls = 0+ mocks.watch.mockImplementation(async (path, options, _callback, done) => {+ if (path.endsWith('/hosts')) {+ doneCallbacks.push(done)+ watchQueries.push(options)+ }+ return { abort: vi.fn() }})- const hosts = (watcher as any).hosts as Map<string, any>- hosts.set('marketing', {- name: 'marketing',- namespace: 'mcp-host',- spec: { host: 'marketing', contextRef: 'c', secretRef: 's' },+ mocks.listNamespacedCustomObject.mockImplementation(async ({ plural }: { plural: string }) => {+ if (plural === 'hosts') {+ hostListCalls += 1+ return {+ metadata: { resourceVersion: `host-close-recovery-rv-${hostListCalls}` },+ items: [],+ }+ }+ return { items: [] }})- const reconcileSpy = vi- .spyOn(watcher.getHostReconciler(), 'reconcile')+ const logSpy = vi.spyOn(console, 'log').mockImplementation(() => {})+ const watcher = newContextAuthoritativeWatcher()+ const fleetRequest = vi+ .spyOn(watcher as any, 'requestHostFleetReconcile').mockResolvedValue(undefined)+ await (watcher as any).startHostWatch('host-initial-rv')+ ;(watcher as any).hostCacheSynced = true- const watchCallback = (watcher as any).getCommunicationChannelWatchCallback()- await watchCallback('DELETED', {- metadata: { name: 'cc-marketing', namespace: 'channels' },- spec: { hostRef: 'marketing' },- })+ doneCallbacks[0](Object.assign(new Error('Premature close'), { statusCode: 500 }))+ expect((watcher as any).hostCacheSynced).toBe(false)+ await flushMicrotasks(20)- expect(cache.has('cc-marketing')).toBe(false)- expect(reconcileSpy).toHaveBeenCalled()+ // Immediate first attempt: fresh LIST + fresh watch WITHOUT advancing any+ // timer. Under the old contract this stayed at 0 until +5000ms.+ expect(hostListCalls).toBe(1)+ expect(watchQueries).toEqual([+ { resourceVersion: 'host-initial-rv' },+ { resourceVersion: 'host-close-recovery-rv-1' },+ ])+ expect((watcher as any).hostCacheSynced).toBe(true)+ expect((watcher as any).hostCacheRecoveryTimer).toBeNull()+ expect(fleetRequest).toHaveBeenCalledOnce()++ // No retry timer was armed: +5s must not produce a duplicate re-LIST.+ await vi.advanceTimersByTimeAsync(5000)+ expect(hostListCalls).toBe(1)+ logSpy.mockRestore()+ await watcher.stop()})- it('on MODIFIED with hostRef change, reconciles BOTH old and new hosts', async () => {- const watcher = new McpServerWatcher()- const cache = (watcher as any).communicationChannels as Map<string, any>- cache.set('cc-1', {- name: 'cc-1',- namespace: 'channels',- spec: { hostRef: 'old-host' },- })- const hosts = (watcher as any).hosts as Map<string, any>- hosts.set('old-host', {- name: 'old-host',- namespace: 'mcp-host',- spec: { host: 'old-host', contextRef: 'c', secretRef: 's' },+ it.each(inventoryLanes)(+ 'falls back to the paced backoff retry timer only after a failed immediate $kind recovery attempt',+ async lane => {+ const doneCallbacks: Array<(error: Error | null) => void> = []+ let listCalls = 0+ mocks.watch.mockImplementation(async (path, _options, _callback, done) => {+ if (path.endsWith(lane.pathSuffix)) doneCallbacks.push(done)+ return { abort: vi.fn() }+ })+ mocks.listNamespacedCustomObject.mockImplementation(+ async ({ plural }: { plural: string }) => {+ if (plural === lane.plural) {+ listCalls += 1+ if (listCalls === 1) throw new Error(`${lane.plural} re-LIST temporarily unavailable`)+ return {+ metadata: { resourceVersion: `${lane.recoveryRvPrefix}-${listCalls}` },+ items: [],+ }+ }+ return { items: [] }+ }+ )+ const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {})+ const logSpy = vi.spyOn(console, 'log').mockImplementation(() => {})+ const watcher = new McpServerWatcher()+ ;(watcher as any)[lane.peerSyncedField] = true+ await (watcher as any)[lane.restartMethod](lane.startSnapshot)++ doneCallbacks[0](Object.assign(new Error('Premature close'), { statusCode: 500 }))+ await flushMicrotasks(20)++ // The immediate attempt ran (and failed): exactly one LIST, retry armed.+ expect(listCalls).toBe(1)+ expect((watcher as any)[lane.syncedField]).toBe(false)+ expect((watcher as any)[lane.timerField]).not.toBeNull()++ // Retry-after-failure keeps its pacing: the first-failure backoff delay+ // is jittered within [500ms, 1000ms], so nothing may fire before the+ // 500ms floor and the retry must have fired by the 1000ms bound.+ await vi.advanceTimersByTimeAsync(499)+ expect(listCalls).toBe(1)+ await vi.advanceTimersByTimeAsync(501)+ await flushMicrotasks(20)+ expect(listCalls).toBe(2)+ expect((watcher as any)[lane.syncedField]).toBe(true)+ expect((watcher as any)[lane.timerField]).toBeNull()+ errorSpy.mockRestore()+ logSpy.mockRestore()+ await watcher.stop()+ }+ )++ it('falls back to the paced backoff retry timer only after a failed immediate Host recovery attempt', async () => {+ const doneCallbacks: Array<(error: Error | null) => void> = []+ let hostListCalls = 0+ mocks.watch.mockImplementation(async (path, _options, _callback, done) => {+ if (path.endsWith('/hosts')) doneCallbacks.push(done)+ return { abort: vi.fn() }})- hosts.set('new-host', {- name: 'new-host',- namespace: 'mcp-host',- spec: { host: 'new-host', contextRef: 'c', secretRef: 's' },+ mocks.listNamespacedCustomObject.mockImplementation(async ({ plural }: { plural: string }) => {+ if (plural === 'hosts') {+ hostListCalls += 1+ if (hostListCalls === 1) throw new Error('hosts re-LIST temporarily unavailable')+ return {+ metadata: { resourceVersion: `host-close-recovery-rv-${hostListCalls}` },+ items: [],+ }+ }+ return { items: [] }})- const reconcileSpy = vi- .spyOn(watcher.getHostReconciler(), 'reconcile')+ const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {})+ const logSpy = vi.spyOn(console, 'log').mockImplementation(() => {})+ const watcher = newContextAuthoritativeWatcher()+ const fleetRequest = vi+ .spyOn(watcher as any, 'requestHostFleetReconcile').mockResolvedValue(undefined)+ await (watcher as any).startHostWatch('host-initial-rv')+ ;(watcher as any).hostCacheSynced = true- const watchCallback = (watcher as any).getCommunicationChannelWatchCallback()- await watchCallback('MODIFIED', {- metadata: { name: 'cc-1', namespace: 'channels' },- spec: { hostRef: 'new-host' },- })+ doneCallbacks[0](Object.assign(new Error('Premature close'), { statusCode: 500 }))+ await flushMicrotasks(20)- expect(reconcileSpy).toHaveBeenCalledTimes(2)- const names = reconcileSpy.mock.calls.map(c => (c[0] as any).name).sort()- expect(names).toEqual(['new-host', 'old-host'])+ // The immediate attempt ran (and failed): exactly one LIST, retry armed+ // by performHostInventoryRecovery's existing failure path. The first-+ // failure backoff delay is jittered within [500ms, 1000ms].+ expect(hostListCalls).toBe(1)+ expect((watcher as any).hostCacheSynced).toBe(false)+ expect((watcher as any).hostCacheRecoveryTimer).not.toBeNull()++ await vi.advanceTimersByTimeAsync(499)+ expect(hostListCalls).toBe(1)+ await vi.advanceTimersByTimeAsync(501)+ await flushMicrotasks(20)+ expect(hostListCalls).toBe(2)+ expect((watcher as any).hostCacheSynced).toBe(true)+ expect((watcher as any).hostCacheRecoveryTimer).toBeNull()+ expect(fleetRequest).toHaveBeenCalled()+ errorSpy.mockRestore()+ logSpy.mockRestore()+ await watcher.stop()})- it('ignores unknown event types (e.g. BOOKMARK) without mutating cache or reconciling', async () => {- const watcher = new McpServerWatcher()- const cache = (watcher as any).communicationChannels as Map<string, any>- cache.set('cc-1', {- name: 'cc-1',- namespace: 'channels',- spec: { hostRef: 'marketing' },+ it.each(inventoryLanes)(+ 'demotes a $kind close arriving within the 1s floor of the last recovery onto the paced retry timer',+ async lane => {+ const doneCallbacks: Array<(error: Error | null) => void> = []+ let listCalls = 0+ mocks.watch.mockImplementation(async (path, _options, _callback, done) => {+ if (path.endsWith(lane.pathSuffix)) doneCallbacks.push(done)+ return { abort: vi.fn() }+ })+ mocks.listNamespacedCustomObject.mockImplementation(+ async ({ plural }: { plural: string }) => {+ if (plural === lane.plural) {+ listCalls += 1+ return {+ metadata: { resourceVersion: `${lane.recoveryRvPrefix}-${listCalls}` },+ items: [],+ }+ }+ return { items: [] }+ }+ )+ const logSpy = vi.spyOn(console, 'log').mockImplementation(() => {})+ const watcher = new McpServerWatcher()+ ;(watcher as any)[lane.peerSyncedField] = true+ await (watcher as any)[lane.restartMethod](lane.startSnapshot)++ // Isolated close: immediate recovery (LIST #1, watch #2).+ doneCallbacks[0](Object.assign(new Error('Premature close'), { statusCode: 500 }))+ await flushMicrotasks(20)+ expect(listCalls).toBe(1)+ expect(doneCallbacks).toHaveLength(2)++ // Burst close: the recovered watch dies again with ZERO fake-time+ // elapsed since the recovery that established it (< 1s floor). The+ // anti-busy-loop floor must demote this attempt onto the paced timer+ // instead of hammering the (visibly degraded) apiserver immediately.+ doneCallbacks[1](Object.assign(new Error('Premature close'), { statusCode: 500 }))+ await flushMicrotasks(20)+ expect(listCalls).toBe(1)+ expect((watcher as any)[lane.timerField]).not.toBeNull()+ expect((watcher as any)[lane.syncedField]).toBe(false)++ // The paced retry recovers it within the +5s advance.+ await vi.advanceTimersByTimeAsync(5000)+ await flushMicrotasks(20)+ expect(listCalls).toBe(2)+ expect((watcher as any)[lane.syncedField]).toBe(true)+ expect((watcher as any)[lane.timerField]).toBeNull()+ expect(doneCallbacks).toHaveLength(3)++ // Spaced close: once more than the 1s floor has elapsed since the last+ // recovery, a new close is immediate again (no timer).+ await vi.advanceTimersByTimeAsync(1500)+ doneCallbacks[2](Object.assign(new Error('Premature close'), { statusCode: 500 }))+ await flushMicrotasks(20)+ expect(listCalls).toBe(3)+ expect((watcher as any)[lane.timerField]).toBeNull()+ expect((watcher as any)[lane.syncedField]).toBe(true)+ logSpy.mockRestore()+ await watcher.stop()+ }+ )++ it('demotes a Host close arriving within the 1s floor of the last recovery onto the paced retry timer', async () => {+ const doneCallbacks: Array<(error: Error | null) => void> = []+ let hostListCalls = 0+ mocks.watch.mockImplementation(async (path, _options, _callback, done) => {+ if (path.endsWith('/hosts')) doneCallbacks.push(done)+ return { abort: vi.fn() }})- const hosts = (watcher as any).hosts as Map<string, any>- hosts.set('marketing', {- name: 'marketing',- namespace: 'mcp-host',- spec: { host: 'marketing', contextRef: 'c', secretRef: 's' },+ mocks.listNamespacedCustomObject.mockImplementation(async ({ plural }: { plural: string }) => {+ if (plural === 'hosts') {+ hostListCalls += 1+ return {+ metadata: { resourceVersion: `host-close-recovery-rv-${hostListCalls}` },+ items: [],+ }+ }+ return { items: [] }})- const reconcileSpy = vi- .spyOn(watcher.getHostReconciler(), 'reconcile')- .mockResolvedValue(undefined)+ const logSpy = vi.spyOn(console, 'log').mockImplementation(() => {})+ const watcher = newContextAuthoritativeWatcher()+ vi.spyOn(watcher as any, 'requestHostFleetReconcile').mockResolvedValue(undefined)+ await (watcher as any).startHostWatch('host-initial-rv')+ ;(watcher as any).hostCacheSynced = true- const watchCallback = (watcher as any).getCommunicationChannelWatchCallback()- await watchCallback('BOOKMARK', {- metadata: { name: 'cc-1', namespace: 'channels' },- spec: { hostRef: 'marketing' },- })+ // Isolated close: immediate recovery (LIST #1, watch #2).+ doneCallbacks[0](Object.assign(new Error('Premature close'), { statusCode: 500 }))+ await flushMicrotasks(20)+ expect(hostListCalls).toBe(1)+ expect(doneCallbacks).toHaveLength(2)+ expect((watcher as any).hostCacheSynced).toBe(true)++ // Burst close within the 1s floor: demoted onto the paced retry timer.+ doneCallbacks[1](Object.assign(new Error('Premature close'), { statusCode: 500 }))+ await flushMicrotasks(20)+ expect(hostListCalls).toBe(1)+ expect((watcher as any).hostCacheRecoveryTimer).not.toBeNull()+ expect((watcher as any).hostCacheSynced).toBe(false)- expect(cache.get('cc-1')?.spec.hostRef).toBe('marketing')- expect(reconcileSpy).not.toHaveBeenCalled()+ await vi.advanceTimersByTimeAsync(5000)+ await flushMicrotasks(20)+ expect(hostListCalls).toBe(2)+ expect((watcher as any).hostCacheSynced).toBe(true)+ expect((watcher as any).hostCacheRecoveryTimer).toBeNull()+ expect(doneCallbacks).toHaveLength(3)++ // Spaced close beyond the floor: immediate again.+ await vi.advanceTimersByTimeAsync(1500)+ doneCallbacks[2](Object.assign(new Error('Premature close'), { statusCode: 500 }))+ await flushMicrotasks(20)+ expect(hostListCalls).toBe(3)+ expect((watcher as any).hostCacheRecoveryTimer).toBeNull()+ expect((watcher as any).hostCacheSynced).toBe(true)+ logSpy.mockRestore()+ await watcher.stop()})})diff --cc host-context-controller/src/k8sClient.tsindex 46b588de9,ff4874570..085d4bd89--- a/host-context-controller/src/k8sClient.ts+++ b/host-context-controller/src/k8sClient.ts@@@ -1621,11 -2527,8 +2546,11 @@@ export class McpServerWatcher implementcontextRef: server.spec.contextRef,transport,auth: server.spec.auth,+ // Project the OAuth block (grantScope etc.) verbatim so mcp-host can+ // dispatch the per-connection partition. Token is NEVER mounted (O4).+ oauth: server.spec.oauth,enabled: server.spec.enabled !== false,- status: this.reconciler.getStatus(server.name),+ status: this.reconciler.getStatus(server),}}