Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript

Merge branch 'feat/mcp-server-oauth' into feat/mcp-server-oauth-host

Diego Zacariasa729a8fa5f29
diff --cc host-context-controller/src/hostReconciler.ts
index d891b7abf,94008e22f..203831263
--- a/host-context-controller/src/hostReconciler.ts
+++ b/host-context-controller/src/hostReconciler.ts
@@@ -3872,10 -4044,10 +4172,11 @@@ export class HostReconciler
const ensureCurrentRuntimeTokenScope = async (): Promise<void> => {
for (let attempt = 1; attempt <= 3; attempt++) {
+ revalidateHostMutationBoundary()
const currentScopeHash = HostReconciler.runtimeTokenScopeHash(
host,
- this.hasChannelIngress(host)
+ this.hasChannelIngress(host),
+ await this.frontsOAuthServer(host)
)
if (runtimeTokenProvision.scopeHash === currentScopeHash) return
@@@ -3884,10 -4056,10 +4185,11 @@@
targetSuspended:
lifecycle.effective.stateless && lifecycle.effective.state === 'suspended',
})
+ revalidateHostMutationBoundary()
const postProvisionScopeHash = HostReconciler.runtimeTokenScopeHash(
host,
- this.hasChannelIngress(host)
+ this.hasChannelIngress(host),
+ await this.frontsOAuthServer(host)
)
if (runtimeTokenProvision.scopeHash === postProvisionScopeHash) return
@@@ -3909,10 -4082,10 +4212,11 @@@
const effective = await resolveDeploymentLifecycle()
const currentScopeHash = HostReconciler.runtimeTokenScopeHash(
host,
- this.hasChannelIngress(host)
+ this.hasChannelIngress(host),
+ await this.frontsOAuthServer(host)
)
if (runtimeTokenProvision.scopeHash === currentScopeHash) {
+ revalidateHostMutationBoundary()
return {
lifecycle: effective,
runtimeTokenRevision: runtimeTokenProvision.revision,
diff --cc host-context-controller/src/k8sClient.test.ts
index 50b15a673,fb1ac6231..aa9159e07
--- a/host-context-controller/src/k8sClient.test.ts
+++ b/host-context-controller/src/k8sClient.test.ts
@@@ -3237,524 -8856,988 +8857,1051 @@@ describe('McpServerWatcher Host watch g
return { abort: vi.fn() }
}
)
- mocks.listNamespacedCustomObject.mockImplementation(async ({ plural }: { plural: string }) => {
- if (plural !== 'hosts') return { items: [] }
- return {
- metadata: { resourceVersion: 'opaque-rv-recreated' },
- items: [
- {
- metadata: { name: 'recreated-host', namespace: 'mcp-host', generation: 1 },
- spec: { host: 'recreated-host', lifecycle: { stateless: true } },
- },
- ],
- }
+ const watcher = newContextAuthoritativeWatcher()
+ const reconcile = vi
+ .spyOn(watcher.getHostReconciler(), 'reconcile')
+ .mockRejectedValueOnce(new Error('stale event failed'))
+ .mockResolvedValue(undefined)
+ await (watcher as any).startHostWatch('test-host-rv')
+ if (!hostWatchCallback) throw new Error('Host watch callback was not installed')
+ markHostInventoryAuthoritative(watcher)
+
+ await hostWatchCallback('ADDED', {
+ metadata: { name: 'superseded-host', namespace: 'mcp-host', generation: 1 },
+ spec: { host: 'superseded-host', lifecycle: { stateless: true } },
+ })
+ await hostWatchCallback('MODIFIED', {
+ metadata: { name: 'superseded-host', namespace: 'mcp-host', generation: 2 },
+ spec: { host: 'superseded-host', lifecycle: { stateless: true } },
+ })
+ await vi.advanceTimersByTimeAsync(60000)
+
+ expect(reconcile).toHaveBeenCalledTimes(2)
+ expect(reconcile.mock.calls[1][0]).toMatchObject({ generation: 2 })
+ expect((watcher as any).hostWatchRetryTimers.size).toBe(0)
+ watcher.stop()
+ })
+ })
+
+ describe('McpServerWatcher.startCommunicationChannelWatch', () => {
+ it('caches ADDED events and triggers Host reconcile', async () => {
+ const watcher = new McpServerWatcher()
+ markHostInventoryAuthoritative(watcher)
+ const hosts = (watcher as any).hosts as Map<string, any>
+ hosts.set('marketing', {
+ name: 'marketing',
+ namespace: 'mcp-host',
+ spec: { host: 'marketing', contextRef: 'c', secretRef: 's' },
+ })
+ const reconcileSpy = vi
+ .spyOn(watcher.getHostReconciler(), 'reconcile')
+ .mockResolvedValue(undefined)
+
+ const watchCallback = (watcher as any).getCommunicationChannelWatchCallback()
+ await watchCallback('ADDED', {
+ metadata: { name: 'cc-marketing', namespace: 'channels' },
+ spec: { hostRef: 'marketing' },
+ })
+
+ const cache = (watcher as any).communicationChannels as Map<string, any>
+ expect(cache.get('cc-marketing')?.spec.hostRef).toBe('marketing')
+ expect(reconcileSpy).toHaveBeenCalledWith(expect.objectContaining({ name: 'marketing' }))
+ })
+
+ it('removes from cache on DELETED and triggers Host reconcile', async () => {
+ const watcher = new McpServerWatcher()
+ markHostInventoryAuthoritative(watcher)
+ const cache = (watcher as any).communicationChannels as Map<string, any>
+ cache.set('cc-marketing', {
+ name: 'cc-marketing',
+ namespace: 'channels',
+ spec: { hostRef: 'marketing' },
+ })
+ const hosts = (watcher as any).hosts as Map<string, any>
+ hosts.set('marketing', {
+ name: 'marketing',
+ namespace: 'mcp-host',
+ spec: { host: 'marketing', contextRef: 'c', secretRef: 's' },
+ })
+ const reconcileSpy = vi
+ .spyOn(watcher.getHostReconciler(), 'reconcile')
+ .mockResolvedValue(undefined)
+
+ const watchCallback = (watcher as any).getCommunicationChannelWatchCallback()
+ await watchCallback('DELETED', {
+ metadata: { name: 'cc-marketing', namespace: 'channels' },
+ spec: { hostRef: 'marketing' },
+ })
+
+ expect(cache.has('cc-marketing')).toBe(false)
+ expect(reconcileSpy).toHaveBeenCalled()
+ })
+
+ it('on MODIFIED with hostRef change, reconciles BOTH old and new hosts', async () => {
+ const watcher = new McpServerWatcher()
+ markHostInventoryAuthoritative(watcher)
+ const cache = (watcher as any).communicationChannels as Map<string, any>
+ cache.set('cc-1', {
+ name: 'cc-1',
+ namespace: 'channels',
+ spec: { hostRef: 'old-host' },
+ })
+ const hosts = (watcher as any).hosts as Map<string, any>
+ hosts.set('old-host', {
+ name: 'old-host',
+ namespace: 'mcp-host',
+ spec: { host: 'old-host', contextRef: 'c', secretRef: 's' },
+ })
+ hosts.set('new-host', {
+ name: 'new-host',
+ namespace: 'mcp-host',
+ spec: { host: 'new-host', contextRef: 'c', secretRef: 's' },
+ })
+ const reconcileSpy = vi
+ .spyOn(watcher.getHostReconciler(), 'reconcile')
+ .mockResolvedValue(undefined)
+
+ const watchCallback = (watcher as any).getCommunicationChannelWatchCallback()
+ await watchCallback('MODIFIED', {
+ metadata: { name: 'cc-1', namespace: 'channels' },
+ spec: { hostRef: 'new-host' },
+ })
+
+ expect(reconcileSpy).toHaveBeenCalledTimes(2)
+ const names = reconcileSpy.mock.calls.map(c => (c[0] as any).name).sort()
+ expect(names).toEqual(['new-host', 'old-host'])
+ })
+
+ it('ignores unknown event types (e.g. BOOKMARK) without mutating cache or reconciling', async () => {
+ const watcher = new McpServerWatcher()
+ const cache = (watcher as any).communicationChannels as Map<string, any>
+ cache.set('cc-1', {
+ name: 'cc-1',
+ namespace: 'channels',
+ spec: { hostRef: 'marketing' },
+ })
+ const hosts = (watcher as any).hosts as Map<string, any>
+ hosts.set('marketing', {
+ name: 'marketing',
+ namespace: 'mcp-host',
+ spec: { host: 'marketing', contextRef: 'c', secretRef: 's' },
+ })
+ const reconcileSpy = vi
+ .spyOn(watcher.getHostReconciler(), 'reconcile')
+ .mockResolvedValue(undefined)
+
+ const watchCallback = (watcher as any).getCommunicationChannelWatchCallback()
+ await watchCallback('BOOKMARK', {
+ metadata: { name: 'cc-1', namespace: 'channels' },
+ spec: { hostRef: 'marketing' },
})
+
+ expect(cache.get('cc-1')?.spec.hostRef).toBe('marketing')
+ expect(reconcileSpy).not.toHaveBeenCalled()
+ })
+ })
+
+ describe('McpServerWatcher.startCommunicationChannelWatch — channel-reader revision rolls', () => {
+ it('on CC ADDED, patches the new host channel-reader revision annotation', async () => {
const watcher = new McpServerWatcher()
- const reconcileDelete = vi
- .spyOn(watcher.getHostReconciler(), 'reconcileDelete')
+ markHostInventoryAuthoritative(watcher)
+ const hosts = (watcher as any).hosts as Map<string, any>
+ hosts.set('marketing', {
+ name: 'marketing',
+ namespace: 'mcp-host',
+ spec: { host: 'marketing', contextRef: 'c', secretRef: 's' },
+ })
+ const patchSpy = vi
+ .spyOn(watcher.getHostReconciler(), 'patchChannelReaderRevisionAnnotation')
.mockResolvedValue(undefined)
- await (watcher as any).startHostWatch('test-host-rv')
- await (watcher as any).requestHostFleetReconcile('Periodic resync', undefined, 'full')
- await callbacks[0]('DELETED', {
- metadata: { name: 'recreated-host', namespace: 'mcp-host', generation: 8 },
- spec: { host: 'recreated-host', lifecycle: { stateless: false } },
+ const watchCallback = (watcher as any).getCommunicationChannelWatchCallback()
+ await watchCallback('ADDED', {
+ metadata: { name: 'cc-marketing', namespace: 'channels' },
+ spec: { hostRef: 'marketing' },
})
- expect(callbacks).toHaveLength(2)
- expect(reconcileDelete).not.toHaveBeenCalled()
- expect((watcher as any).hosts.get('recreated-host')).toMatchObject({
- spec: { lifecycle: { stateless: true } },
- })
- watcher.stop()
+ expect(patchSpy).toHaveBeenCalledTimes(1)
+ expect(patchSpy).toHaveBeenCalledWith('marketing')
})
- it('recovery diff enqueues an immediate per-Host delete for a disappeared Host (#827 item 1)', async () => {
- vi.clearAllMocks()
+ it('preserves credentialsSecretRef from the watch event into the cache', async () => {
const watcher = new McpServerWatcher()
- const reconcileDelete = vi
- .spyOn(watcher.getHostReconciler(), 'reconcileDelete')
- .mockResolvedValue(undefined)
- // Fresh authoritative snapshot holds only "live-host"; "gone-host" was in the
- // pre-recovery inventory (previousNames) and has genuinely disappeared.
- ;(watcher as any).enqueueRecoveredHostDeletes(
- [{ name: 'live-host', namespace: 'mcp-host', uid: 'u1', spec: { host: 'live-host' } }],
- new Set(['live-host', 'gone-host'])
- )
- await vi.waitFor(() =>
- expect(reconcileDelete).toHaveBeenCalledWith('gone-host', 'mcp-host', expect.anything())
+ markHostInventoryAuthoritative(watcher)
+ const cache = (watcher as any).communicationChannels as Map<string, any>
+ const hosts = (watcher as any).hosts as Map<string, any>
+ hosts.set('marketing', {
+ name: 'marketing',
+ namespace: 'mcp-host',
+ spec: { host: 'marketing', contextRef: 'c', secretRef: 's' },
+ })
+ vi.spyOn(watcher.getHostReconciler(), 'patchChannelReaderRevisionAnnotation').mockResolvedValue(
+ undefined
)
- // A Host still present in the fresh snapshot is never deleted.
- expect(reconcileDelete).not.toHaveBeenCalledWith('live-host', 'mcp-host', expect.anything())
- watcher.stop()
+
+ const watchCallback = (watcher as any).getCommunicationChannelWatchCallback()
+ await watchCallback('ADDED', {
+ metadata: { name: 'cc-marketing', namespace: 'channels' },
+ spec: {
+ hostRef: 'marketing',
+ credentialsSecretRef: { name: 'cc-marketing-credentials' },
+ },
+ })
+
+ const cached = cache.get('cc-marketing')
+ expect(cached.spec.credentialsSecretRef).toEqual({ name: 'cc-marketing-credentials' })
+ // findCommunicationChannelsByCredentialsSecretName must find the cached CC
+ expect(
+ watcher.findCommunicationChannelsByCredentialsSecretName('cc-marketing-credentials')
+ ).toHaveLength(1)
})
- // The recovery diff resolves absence at LIST time. Between that diff and the
- // moment the delete is admitted to the per-Host serializer, the recreation's
- // ADDED can land: the watch callback sets `this.hosts` SYNCHRONOUSLY and
- // enters the per-Host chain FIRST, while the fire-and-forget dispatch queues
- // the delete SECOND. serializeByHost is strict FIFO, so without a fence the
- // new bundle is created and then wiped. The fence must therefore read the
- // LIVE cache at admission, not a value captured at enqueue time.
- //
- // This is the case the sibling recreation test does NOT cover: there the
- // recreated Host is present in the snapshot, so enqueueRecoveredHostDeletes
- // short-circuits on snapshotNames.has(name) and never dispatches at all.
- it('recovery delete carries a live-cache fence for a Host recreated during admission (#827 F2 parity)', async () => {
- vi.clearAllMocks()
+ it('on CC MODIFIED with hostRef change, patches BOTH old and new hosts', async () => {
const watcher = new McpServerWatcher()
- const reconcileDelete = vi
- .spyOn(watcher.getHostReconciler(), 'reconcileDelete')
+ markHostInventoryAuthoritative(watcher)
+ const cache = (watcher as any).communicationChannels as Map<string, any>
+ cache.set('cc-1', {
+ name: 'cc-1',
+ namespace: 'channels',
+ spec: { hostRef: 'old-host' },
+ })
+ const hosts = (watcher as any).hosts as Map<string, any>
+ hosts.set('old-host', {
+ name: 'old-host',
+ namespace: 'mcp-host',
+ spec: { host: 'old-host', contextRef: 'c', secretRef: 's' },
+ })
+ hosts.set('new-host', {
+ name: 'new-host',
+ namespace: 'mcp-host',
+ spec: { host: 'new-host', contextRef: 'c', secretRef: 's' },
+ })
+ const patchSpy = vi
+ .spyOn(watcher.getHostReconciler(), 'patchChannelReaderRevisionAnnotation')
.mockResolvedValue(undefined)
- // "ghost" is absent from the fresh authoritative snapshot → genuinely
- // disappeared → dispatched for delete.
- ;(watcher as any).enqueueRecoveredHostDeletes([], new Set(['ghost']))
- await vi.waitFor(() => expect(reconcileDelete).toHaveBeenCalled())
- const [, , opts] = reconcileDelete.mock.calls[0] as [
- string,
- string,
- { skipIf?: () => boolean } | undefined,
- ]
- expect(opts?.skipIf, 'the recovery delete must carry a skipIf fence').toBeTypeOf('function')
+ const watchCallback = (watcher as any).getCommunicationChannelWatchCallback()
+ await watchCallback('MODIFIED', {
+ metadata: { name: 'cc-1', namespace: 'channels' },
+ spec: { hostRef: 'new-host' },
+ })
- // Cache does not hold the name → the Host really is gone → do NOT skip.
- expect(opts!.skipIf!()).toBe(false)
+ expect(patchSpy).toHaveBeenCalledTimes(2)
+ const targets = patchSpy.mock.calls.map(c => c[0] as string).sort()
+ expect(targets).toEqual(['new-host', 'old-host'])
+ })
- // The recreation lands in the live cache during the admission window.
- ;(watcher as any).hosts.set('ghost', {
- name: 'ghost',
+ it('on CC DELETED, patches the (former) host channel-reader revision annotation', async () => {
+ const watcher = new McpServerWatcher()
+ markHostInventoryAuthoritative(watcher)
+ const cache = (watcher as any).communicationChannels as Map<string, any>
+ cache.set('cc-marketing', {
+ name: 'cc-marketing',
+ namespace: 'channels',
+ spec: { hostRef: 'marketing' },
+ })
+ const hosts = (watcher as any).hosts as Map<string, any>
+ hosts.set('marketing', {
+ name: 'marketing',
namespace: 'mcp-host',
- uid: 'uid-recreated',
- generation: 1,
- spec: { host: 'ghost' },
+ spec: { host: 'marketing', contextRef: 'c', secretRef: 's' },
})
- // Same fence, re-evaluated at admission → skip the destructive delete.
- expect(opts!.skipIf!()).toBe(true)
- watcher.stop()
+ const patchSpy = vi
+ .spyOn(watcher.getHostReconciler(), 'patchChannelReaderRevisionAnnotation')
+ .mockResolvedValue(undefined)
+
+ const watchCallback = (watcher as any).getCommunicationChannelWatchCallback()
+ await watchCallback('DELETED', {
+ metadata: { name: 'cc-marketing', namespace: 'channels' },
+ spec: { hostRef: 'marketing' },
+ })
+
+ expect(patchSpy).toHaveBeenCalledTimes(1)
+ expect(patchSpy).toHaveBeenCalledWith('marketing')
})
- it('recovery diff reconciles (does NOT delete) a same-name Host recreated with a new uid (#827 items 1+2)', async () => {
- vi.clearAllMocks()
+ it('swallows patchChannelReaderRevisionAnnotation errors without breaking the watch loop', async () => {
const watcher = new McpServerWatcher()
- const reconcile = vi
- .spyOn(watcher.getHostReconciler(), 'reconcile')
- .mockResolvedValue(undefined)
- const reconcileDelete = vi
- .spyOn(watcher.getHostReconciler(), 'reconcileDelete')
- .mockResolvedValue(undefined)
- // The recreated Host is the CURRENT cache entry the urgent dispatch resolves.
- ;(watcher as any).hosts.set('x', {
- name: 'x',
+ markHostInventoryAuthoritative(watcher)
+ const hosts = (watcher as any).hosts as Map<string, any>
+ hosts.set('marketing', {
+ name: 'marketing',
namespace: 'mcp-host',
- uid: 'uid-B',
- generation: 1,
- spec: { host: 'x' },
+ spec: { host: 'marketing', contextRef: 'c', secretRef: 's' },
})
- // Same name, same generation (reset to 1), but a DIFFERENT uid than before.
- ;(watcher as any).enqueueRecoveredUrgentHosts(
- [{ name: 'x', namespace: 'mcp-host', uid: 'uid-B', generation: 1, spec: { host: 'x' } }],
- new Set(['x']),
- new Map([['x', 1]]),
- new Map([['x', 'uid-A']])
- )
- // The snapshot still contains "x" by name, so the recovery-delete diff never
- // treats it as a disappearance.
- ;(watcher as any).enqueueRecoveredHostDeletes(
- [{ name: 'x', namespace: 'mcp-host', uid: 'uid-B', generation: 1, spec: { host: 'x' } }],
- new Set(['x'])
- )
- await vi.waitFor(() =>
- expect(reconcile).toHaveBeenCalledWith(expect.objectContaining({ name: 'x' }), 'urgent')
+ vi.spyOn(watcher.getHostReconciler(), 'patchChannelReaderRevisionAnnotation').mockRejectedValue(
+ new Error('boom')
)
- expect(reconcileDelete).not.toHaveBeenCalled()
+ const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {})
+
+ const watchCallback = (watcher as any).getCommunicationChannelWatchCallback()
+ await expect(
+ watchCallback('ADDED', {
+ metadata: { name: 'cc-marketing', namespace: 'channels' },
+ spec: { hostRef: 'marketing' },
+ })
+ ).resolves.toBeUndefined()
+ expect(errorSpy).toHaveBeenCalled()
+
+ errorSpy.mockRestore()
+ })
+ })
+
+ describe('externalEgressResyncDelayMs (issue #299 H2)', () => {
+ it('returns the configured interval (ms) when no TTL has been observed', () => {
+ expect(externalEgressResyncDelayMs(60, 5)).toBe(60_000)
+ expect(externalEgressResyncDelayMs(60, 5, Infinity)).toBe(60_000)
+ })
+
+ it('advances to TTL/2 when the observed TTL is faster than the interval', () => {
+ expect(externalEgressResyncDelayMs(60, 5, 15_000)).toBe(7_500)
+ })
+
+ it('never drops below the floor even for a tiny TTL', () => {
+ expect(externalEgressResyncDelayMs(60, 5, 6_000)).toBe(5_000)
+ })
+
+ it('keeps the configured interval when the observed TTL is large', () => {
+ expect(externalEgressResyncDelayMs(60, 5, 600_000)).toBe(60_000)
+ })
+
+ it('clamps a below-floor configured interval up to the floor', () => {
+ expect(externalEgressResyncDelayMs(3, 5)).toBe(5_000)
+ })
+ })
+
++describe('McpServerWatcher.hostFrontsOAuthServer (oauth:user-token scope probe)', () => {
++ const hostFor = (contextRef: string): HostCRD =>
++ ({
++ name: 'h',
++ namespace: 'mcp-host',
++ spec: { host: 'h', contextRef, secretRef: 's' },
++ }) as HostCRD
++
++ it('returns true when the Context fronts an enabled auth.type=oauth mcp-server', async () => {
++ const watcher = new McpServerWatcher()
++ vi.spyOn(watcher, 'getServerInfosByContext').mockResolvedValue([
++ {
++ name: 'notion',
++ contextRef: 'ctx',
++ transport: { type: 'streamableHttp' },
++ enabled: true,
++ auth: { type: 'oauth' },
++ status: {},
++ },
++ ] as any)
++ await expect((watcher as any).hostFrontsOAuthServer(hostFor('ctx'))).resolves.toBe(true)
+ watcher.stop()
+ })
+
- it('suppresses a watch event whose identity (uid) differs from the cached Host (#827 item 2)', () => {
++ it('returns false for a non-oauth mcp-server', async () => {
+ const watcher = new McpServerWatcher()
- ;(watcher as any).hosts.set('x', {
- name: 'x',
- namespace: 'mcp-host',
- uid: 'uid-B',
- generation: 3,
- })
- ;(watcher as any).latestHostWatchEventRevisions.set('x', 7)
- // A stale MODIFIED carrying the OLD identity must not act on the recreated one.
- expect(
- (watcher as any).isCurrentHostWatchEvent(
- 'MODIFIED',
- { name: 'x', namespace: 'mcp-host', uid: 'uid-A', generation: 3 },
- 7
- )
- ).toBe(false)
- // The current identity is admitted.
- expect(
- (watcher as any).isCurrentHostWatchEvent(
- 'MODIFIED',
- { name: 'x', namespace: 'mcp-host', uid: 'uid-B', generation: 3 },
- 7
- )
- ).toBe(true)
++ vi.spyOn(watcher, 'getServerInfosByContext').mockResolvedValue([
++ {
++ name: 'redis',
++ contextRef: 'ctx',
++ transport: { type: 'streamableHttp' },
++ enabled: true,
++ auth: { type: 'bearer' },
++ status: {},
++ },
++ ] as any)
++ await expect((watcher as any).hostFrontsOAuthServer(hostFor('ctx'))).resolves.toBe(false)
+ watcher.stop()
+ })
+
- it('recovery urgently admits a wake-pending Host that is otherwise unchanged (§10.2 step 7 wake branch)', async () => {
- vi.clearAllMocks()
++ it('returns false for a disabled oauth mcp-server', async () => {
+ const watcher = new McpServerWatcher()
- const reconcile = vi
- .spyOn(watcher.getHostReconciler(), 'reconcile')
- .mockResolvedValue(undefined)
- // A Host unchanged by name, generation AND uid but carrying a pending wake
- // annotation must still be admitted on the urgent lane after recovery, so a
- // wake discovered during the LIST→WATCH gap is not deferred to the slow
- // background fleet pass (A3: 2m47s admission vs 3s execution). This closes
- // the recovery-time wake branch that the watch-time admission test does not
- // exercise: deleting `|| wakePending` from enqueueRecoveredUrgentHosts()
- // would make this assertion fail.
- ;(watcher as any).hosts.set('waker', {
- name: 'waker',
- namespace: 'mcp-host',
- uid: 'u1',
- generation: 4,
- annotations: { 'clerum.io/wake-requested': '7' },
- spec: { host: 'waker' },
- })
- ;(watcher as any).enqueueRecoveredUrgentHosts(
- [
- {
- name: 'waker',
- namespace: 'mcp-host',
- uid: 'u1',
- generation: 4,
- annotations: { 'clerum.io/wake-requested': '7' },
- spec: { host: 'waker' },
- },
- ],
- new Set(['waker']), // NOT new
- new Map([['waker', 4]]), // NOT changed (same generation)
- new Map([['waker', 'u1']]) // NOT recreated (same uid)
- )
- await vi.waitFor(() =>
- expect(reconcile).toHaveBeenCalledWith(expect.objectContaining({ name: 'waker' }), 'urgent')
- )
++ vi.spyOn(watcher, 'getServerInfosByContext').mockResolvedValue([
++ {
++ name: 'notion',
++ contextRef: 'ctx',
++ transport: { type: 'streamableHttp' },
++ enabled: false,
++ auth: { type: 'oauth' },
++ status: {},
++ },
++ ] as any)
++ await expect((watcher as any).hostFrontsOAuthServer(hostFor('ctx'))).resolves.toBe(false)
+ watcher.stop()
+ })
+
- it('recovery does NOT urgently dispatch a fully-unchanged Host with no wake annotation', async () => {
- vi.clearAllMocks()
++ it('returns false when the Context fronts no servers', async () => {
+ const watcher = new McpServerWatcher()
- const reconcile = vi
- .spyOn(watcher.getHostReconciler(), 'reconcile')
- .mockResolvedValue(undefined)
- // Same name, generation and uid as before recovery and no wake pending: the
- // background fleet pass already covers it, so recovery must NOT enqueue a
- // redundant urgent reconcile (guards the negative side of the predicate).
- ;(watcher as any).hosts.set('steady', {
- name: 'steady',
- namespace: 'mcp-host',
- uid: 'u1',
- generation: 4,
- spec: { host: 'steady' },
- })
- ;(watcher as any).enqueueRecoveredUrgentHosts(
- [
- {
- name: 'steady',
- namespace: 'mcp-host',
- uid: 'u1',
- generation: 4,
- spec: { host: 'steady' },
- },
- ],
- new Set(['steady']),
- new Map([['steady', 4]]),
- new Map([['steady', 'u1']])
- )
- // Let any erroneously-scheduled async dispatch run before asserting absence.
- await Promise.resolve()
- await Promise.resolve()
- expect(reconcile).not.toHaveBeenCalled()
++ vi.spyOn(watcher, 'getServerInfosByContext').mockResolvedValue([] as any)
++ await expect((watcher as any).hostFrontsOAuthServer(hostFor('ctx'))).resolves.toBe(false)
+ watcher.stop()
+ })
-
- it('reconciles a retried Host watch event on the "retry" lane, not "urgent" (F1)', async () => {
++})
+ describe('McpServerWatcher readiness under sustained watch churn (GKE Premature-close regime)', () => {
+ // Regression for the clerum-dev livelock. PR #205 coupled /ready to a safety
+ // certificate whose validity was pinned to the watch GENERATION. On GKE the
+ // apiserver closes long-lived watches ("Premature close") every few minutes;
+ // each reconnect re-LISTs and bumps the generation, so every certification
+ // pass aborted before it could persist and /ready stayed 503 forever — even
+ // though the re-listed inventory was byte-identical. minikube never reproduced
+ // this because its single local apiserver does not drop watches, which is
+ // exactly why T2 stayed green while dev broke. This test injects the sustained
+ // churn deterministically so the regression is caught in CI, not in production.
+ it('certifies readiness under sustained Context watch churn when every re-list returns identical inventory', async () => {
vi.useFakeTimers()
- let hostWatchCallback: ((type: string, apiObj: any) => Promise<void>) | undefined
- mocks.watch.mockImplementation(
- async (
- _path: string,
- _params: object,
- callback: (type: string, apiObj: any) => Promise<void>
- ) => {
- hostWatchCallback = callback
- return { abort: vi.fn() }
- }
- )
- const watcher = new McpServerWatcher()
- const reconcile = vi
- .spyOn(watcher.getHostReconciler(), 'reconcile')
- .mockRejectedValueOnce(new Error('transient'))
- .mockResolvedValue(undefined)
- await (watcher as any).startHostWatch('test-host-rv')
- if (!hostWatchCallback) throw new Error('Host watch callback was not installed')
- await hostWatchCallback('MODIFIED', {
- metadata: { name: 'retry-lane-host', namespace: 'mcp-host', generation: 2 },
- spec: { host: 'retry-lane-host', lifecycle: { stateless: true } },
+ // Stable desired state: the same Context in every LIST. Models clerum-dev,
+ // where nothing changed — only the watch dropped and reconnected.
+ const stableContexts = [
+ {
+ metadata: {
+ name: 'churn-context',
+ namespace: 'mcp-server',
+ uid: 'churn-uid-1',
+ generation: 1,
+ },
+ spec: { contextId: 'churn-context', mcpServers: [] },
+ },
+ ]
+ mocks.listNamespacedCustomObject.mockImplementation(async ({ plural }: { plural: string }) => {
+ if (plural === 'contexts') {
+ return { metadata: { resourceVersion: 'ctx-rv-stable' }, items: stableContexts }
+ }
+ if (plural === 'mcpservers') {
+ return { metadata: { resourceVersion: 'mcp-rv-stable' }, items: [] }
+ }
+ if (plural === 'hosts') {
+ return { metadata: { resourceVersion: 'host-rv-stable' }, items: [] }
+ }
+ if (plural === 'communicationchannels') {
+ return { metadata: { resourceVersion: 'cc-rv-stable' }, items: [] }
+ }
+ return { items: [] }
})
- // First attempt is the direct/urgent admission.
- expect(reconcile).toHaveBeenNthCalledWith(
- 1,
- expect.objectContaining({ name: 'retry-lane-host' }),
- 'urgent'
- )
- await vi.advanceTimersByTimeAsync(5000)
- // The retry attempt populates the distinct "retry" lane.
- expect(reconcile).toHaveBeenNthCalledWith(
- 2,
- expect.objectContaining({ name: 'retry-lane-host' }),
- 'retry'
- )
- watcher.stop()
- vi.useRealTimers()
- })
- it('automatically retries a transient Host watch reconciliation failure', async () => {
- vi.useFakeTimers()
- let hostWatchCallback: ((type: string, apiObj: any) => Promise<void>) | undefined
- mocks.watch.mockImplementation(
- async (
- _path: string,
- _params: object,
- callback: (type: string, apiObj: any) => Promise<void>
- ) => {
- hostWatchCallback = callback
- return { abort: vi.fn() }
- }
- )
+ const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {})
+ const warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => {})
+ const logSpy = vi.spyOn(console, 'log').mockImplementation(() => {})
const watcher = new McpServerWatcher()
- const unrelatedHost = {
- name: 'unrelated-host',
- namespace: 'mcp-host',
- generation: 4,
- spec: { host: 'unrelated-host', lifecycle: { stateless: true } },
- }
- ;(watcher as any).hosts.set(unrelatedHost.name, unrelatedHost)
- const reconcile = vi
- .spyOn(watcher.getHostReconciler(), 'reconcile')
- .mockRejectedValueOnce(Object.assign(new Error('disappeared'), { code: 404 }))
- .mockResolvedValue(undefined)
- await (watcher as any).startHostWatch('test-host-rv')
- const watchGeneration = (watcher as any).hostWatchGeneration
- if (!hostWatchCallback) throw new Error('Host watch callback was not installed')
-
- await hostWatchCallback('ADDED', {
- metadata: { name: 'retry-host', namespace: 'mcp-host', generation: 1 },
- spec: { host: 'retry-host', lifecycle: { stateless: true } },
- })
- expect(reconcile).toHaveBeenCalledTimes(1)
- await vi.advanceTimersByTimeAsync(4999)
- expect(reconcile).toHaveBeenCalledTimes(1)
- await vi.advanceTimersByTimeAsync(1)
- expect(reconcile).toHaveBeenCalledTimes(2)
- expect((watcher as any).hostWatchRetryTimers.size).toBe(0)
- expect((watcher as any).hostWatchRetryAttempts.size).toBe(0)
- expect(
- mocks.listNamespacedCustomObject.mock.calls.filter(([request]) => request.plural === 'hosts')
- ).toHaveLength(0)
- expect(mocks.watch).toHaveBeenCalledOnce()
- expect(mocks.hostFullReconcile).not.toHaveBeenCalled()
- expect((watcher as any).hostWatchGeneration).toBe(watchGeneration)
- expect((watcher as any).hosts.get(unrelatedHost.name)).toEqual(unrelatedHost)
- watcher.stop()
- })
+ vi.spyOn(watcher as any, 'startSharedFileSystemWatch').mockResolvedValue(undefined)
+ vi.spyOn(watcher as any, 'startGlobalFileSystemWatch').mockResolvedValue(undefined)
- it('bounds repeated Host watch retries and stops after exhaustion', async () => {
- vi.useFakeTimers()
- let hostWatchCallback: ((type: string, apiObj: any) => Promise<void>) | undefined
- mocks.watch.mockImplementation(
+ // Model a "Premature close" reconnect that lands DURING each authoritative
+ // revocation pass: the pass captures its safety certificate, then the watch
+ // reconnects and recovery re-LISTs the identical inventory — so the watch
+ // GENERATION advances (channel identity) while the desired REVISION (content
+ // identity) is unchanged and the cache stays authoritative. The #205 code
+ // pinned the certificate to the generation, so it refused here on every pass
+ // and livelocked; the fix pins it to the revision, so it certifies through
+ // the churn.
+ let passes = 0
+ let reconnects = 0
+ mocks.netPolFullReconcile.mockImplementation(
async (
- _path: string,
- _params: object,
- callback: (type: string, apiObj: any) => Promise<void>
+ _ctxs: unknown,
+ _servers: unknown,
+ options: { onAuthoritativeRevocationComplete?: () => void } | undefined
) => {
- hostWatchCallback = callback
- return { abort: vi.fn() }
+ passes += 1
+ ;(watcher as any).contextWatchGeneration += 2
+ ;(watcher as any).mcpWatchGeneration += 2
+ reconnects += 1
+ options?.onAuthoritativeRevocationComplete?.()
}
)
- const watcher = new McpServerWatcher()
- const reconcile = vi
- .spyOn(watcher.getHostReconciler(), 'reconcile')
- .mockRejectedValue(new Error('persistent failure'))
- await (watcher as any).startHostWatch('test-host-rv')
- if (!hostWatchCallback) throw new Error('Host watch callback was not installed')
- await hostWatchCallback('ADDED', {
- metadata: { name: 'bounded-host', namespace: 'mcp-host', generation: 1 },
- spec: { host: 'bounded-host', lifecycle: { stateless: true } },
- })
- await vi.advanceTimersByTimeAsync(5000)
- await vi.advanceTimersByTimeAsync(15000)
- await vi.advanceTimersByTimeAsync(30000)
- expect(reconcile).toHaveBeenCalledTimes(4)
- await vi.advanceTimersByTimeAsync(300000)
- expect(reconcile).toHaveBeenCalledTimes(4)
- expect((watcher as any).hostWatchRetryTimers.size).toBe(0)
- expect((watcher as any).hostWatchRetryAttempts.size).toBe(0)
- watcher.stop()
+ const server = new ContextMapperServer(watcher, 0, undefined, undefined, () =>
+ watcher.isReadinessInventoryAuthoritative()
+ )
+ await server.start()
+
+ try {
+ await watcher.start()
+ server.setReady(true)
+
+ // Sustain the churn for a fixed number of authoritative passes. Each pass
+ // takes a reconnect (generation bump) with a byte-identical re-LIST. We do
+ // NOT stop early: the regime must run enough times that a livelock is
+ // unmistakable, and the fix must hold readiness through all of it.
+ const CHURN_PASSES = 5
+ for (let i = 0; i < CHURN_PASSES; i += 1) {
+ await (watcher as any).runInitialNetworkPolicyConvergence()
+ await flushMicrotasks()
+ }
+
+ // The adverse regime actually ran (zero-tests-is-never-success).
+ expect(passes).toBeGreaterThanOrEqual(CHURN_PASSES)
+ expect(reconnects).toBeGreaterThanOrEqual(CHURN_PASSES)
+ // The livelock assertion: an identical re-LIST must not keep readiness
+ // down. #205 code → false (livelock). Fixed code → true.
+ expect(
+ watcher.isReadinessInventoryAuthoritative(),
+ `livelock: ${passes} authoritative passes completed under sustained churn ` +
+ `(${reconnects} reconnects, identical inventory on every re-LIST) ` +
+ `and none certified the safety inventory`
+ ).toBe(true)
+ expect((await requestReadyOverHttp(server)).statusCode).toBe(200)
+ } finally {
+ await server.stop()
+ await watcher.stop()
+ errorSpy.mockRestore()
+ warnSpy.mockRestore()
+ logSpy.mockRestore()
+ }
})
- it('cancels an older failed Host retry when a newer event succeeds', async () => {
+ // R1 (fail-closed guardrail): the content-identity gate must NOT paper over a
+ // genuine safety failure. If the authoritative pass could not revoke a real
+ // stale allow (hasCertifiedSafetyInventory stays false), readiness must remain
+ // 503 no matter how quiet the content is. This is the barrier that stops a
+ // future "fix" from certifying on identical content when safety never certified.
+ it('stays fail-closed under sustained churn when the safety pass never certified', async () => {
vi.useFakeTimers()
- let hostWatchCallback: ((type: string, apiObj: any) => Promise<void>) | undefined
- mocks.watch.mockImplementation(
- async (
- _path: string,
- _params: object,
- callback: (type: string, apiObj: any) => Promise<void>
- ) => {
- hostWatchCallback = callback
- return { abort: vi.fn() }
+ mocks.listNamespacedCustomObject.mockImplementation(async ({ plural }: { plural: string }) => {
+ if (plural === 'contexts') {
+ return {
+ metadata: { resourceVersion: 'ctx-rv-stable' },
+ items: [
+ {
+ metadata: { name: 'fc-ctx', namespace: 'mcp-server', uid: 'fc-uid', generation: 1 },
+ spec: { contextId: 'fc-ctx', mcpServers: [] },
+ },
+ ],
+ }
}
+ if (plural === 'mcpservers') return { metadata: { resourceVersion: 'mcp-rv' }, items: [] }
+ if (plural === 'hosts') return { metadata: { resourceVersion: 'host-rv' }, items: [] }
+ if (plural === 'communicationchannels')
+ return { metadata: { resourceVersion: 'cc-rv' }, items: [] }
+ return { items: [] }
+ })
+ const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {})
+ const warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => {})
+ const logSpy = vi.spyOn(console, 'log').mockImplementation(() => {})
+ const watcher = new McpServerWatcher()
+ vi.spyOn(watcher as any, 'startSharedFileSystemWatch').mockResolvedValue(undefined)
+ vi.spyOn(watcher as any, 'startGlobalFileSystemWatch').mockResolvedValue(undefined)
+ // The authoritative pass aborts revoking a real stale allow: it never certifies.
+ mocks.hasCertifiedSafetyInventory.mockReturnValue(false)
+ let passes = 0
+ mocks.netPolFullReconcile.mockImplementation(async () => {
+ passes += 1
+ ;(watcher as any).contextWatchGeneration += 2
+ ;(watcher as any).mcpWatchGeneration += 2
+ // No onAuthoritativeRevocationComplete: the revocation did not complete.
+ })
+ const server = new ContextMapperServer(watcher, 0, undefined, undefined, () =>
+ watcher.isReadinessInventoryAuthoritative()
)
+ await server.start()
+ try {
+ await watcher.start()
+ server.setReady(true)
+ const CHURN_PASSES = 5
+ for (let i = 0; i < CHURN_PASSES; i += 1) {
+ await (watcher as any).runInitialNetworkPolicyConvergence()
+ await flushMicrotasks()
+ }
+ expect(passes).toBeGreaterThanOrEqual(CHURN_PASSES)
+ expect(
+ watcher.isReadinessInventoryAuthoritative(),
+ 'fail-closed violated: readiness certified while the safety pass never did'
+ ).toBe(false)
+ expect((await requestReadyOverHttp(server)).statusCode).toBe(503)
+ } finally {
+ await server.stop()
+ await watcher.stop()
+ mocks.hasCertifiedSafetyInventory.mockReturnValue(true)
+ errorSpy.mockRestore()
+ warnSpy.mockRestore()
+ logSpy.mockRestore()
+ }
+ })
+
+ it('installContextSnapshot bumps the desired revision only when re-listed content changed', () => {
const watcher = new McpServerWatcher()
- const reconcile = vi
- .spyOn(watcher.getHostReconciler(), 'reconcile')
- .mockRejectedValueOnce(new Error('stale event failed'))
- .mockResolvedValue(undefined)
- await (watcher as any).startHostWatch('test-host-rv')
- if (!hostWatchCallback) throw new Error('Host watch callback was not installed')
+ const ctxA = {
+ name: 'a',
+ namespace: 'mcp-server',
+ uid: 'ua',
+ generation: 1,
+ spec: { contextId: 'a', mcpServers: [] },
+ }
+ ;(watcher as any).installContextSnapshot({ contexts: [ctxA], resourceVersion: 'rv1' })
+ const base = (watcher as any).contextDesiredRevision
+ // Identical re-LIST (a Premature-close reconnect) must NOT bump.
+ ;(watcher as any).installContextSnapshot({ contexts: [ctxA], resourceVersion: 'rv2' })
+ expect((watcher as any).contextDesiredRevision).toBe(base)
+ // A spec change on the same identity MUST bump.
+ ;(watcher as any).installContextSnapshot({
+ contexts: [{ ...ctxA, generation: 2, spec: { contextId: 'a', mcpServers: ['s1'] } }],
+ resourceVersion: 'rv3',
+ })
+ expect((watcher as any).contextDesiredRevision).toBe(base + 1)
+ // A removal MUST bump.
+ ;(watcher as any).installContextSnapshot({ contexts: [], resourceVersion: 'rv4' })
+ expect((watcher as any).contextDesiredRevision).toBe(base + 2)
+ // An addition MUST bump.
+ ;(watcher as any).installContextSnapshot({ contexts: [ctxA], resourceVersion: 'rv5' })
+ expect((watcher as any).contextDesiredRevision).toBe(base + 3)
+ })
- await hostWatchCallback('ADDED', {
- metadata: { name: 'superseded-host', namespace: 'mcp-host', generation: 1 },
- spec: { host: 'superseded-host', lifecycle: { stateless: true } },
- })
- await hostWatchCallback('MODIFIED', {
- metadata: { name: 'superseded-host', namespace: 'mcp-host', generation: 2 },
- spec: { host: 'superseded-host', lifecycle: { stateless: true } },
- })
- await vi.advanceTimersByTimeAsync(60000)
+ it('installMcpServerSnapshot bumps the desired revision only when re-listed content changed', () => {
+ const watcher = new McpServerWatcher()
+ const srvA = {
+ name: 'a',
+ namespace: 'mcp-server',
+ uid: 'ua',
+ generation: 1,
+ spec: {
+ contextRef: 'ctx',
+ image: 'clerum/a:v1',
+ transport: { type: 'streamableHttp' as const, port: 3000 },
+ },
+ }
+ ;(watcher as any).installMcpServerSnapshot({ servers: [srvA], resourceVersion: 'rv1' })
+ const base = (watcher as any).mcpServerDesiredRevision
+ // Identical re-LIST must NOT bump.
+ ;(watcher as any).installMcpServerSnapshot({ servers: [srvA], resourceVersion: 'rv2' })
+ expect((watcher as any).mcpServerDesiredRevision).toBe(base)
+ // A spec change MUST bump.
+ ;(watcher as any).installMcpServerSnapshot({
+ servers: [{ ...srvA, generation: 2, spec: { ...srvA.spec, image: 'clerum/a:v2' } }],
+ resourceVersion: 'rv3',
+ })
+ expect((watcher as any).mcpServerDesiredRevision).toBe(base + 1)
+ // A removal MUST bump.
+ ;(watcher as any).installMcpServerSnapshot({ servers: [], resourceVersion: 'rv4' })
+ expect((watcher as any).mcpServerDesiredRevision).toBe(base + 2)
+ })
- expect(reconcile).toHaveBeenCalledTimes(2)
- expect(reconcile.mock.calls[1][0]).toMatchObject({ generation: 2 })
- expect((watcher as any).hostWatchRetryTimers.size).toBe(0)
- watcher.stop()
+ it('installHostSnapshot bumps the desired revision only when re-listed content changed', () => {
+ const watcher = new McpServerWatcher()
+ const hostA = {
+ name: 'a',
+ namespace: 'mcp-host',
+ uid: 'ua',
+ generation: 1,
+ spec: { contextRef: 'ctx' },
+ } as unknown as HostCRD
+ ;(watcher as any).installHostSnapshot({ hosts: [hostA], resourceVersion: 'rv1' })
+ const base = (watcher as any).hostDesiredRevision
+ // Identical re-LIST (a Premature-close reconnect) must NOT bump — this is
+ // what stops the Host mutation-authority fence from starving every reconcile.
+ ;(watcher as any).installHostSnapshot({ hosts: [hostA], resourceVersion: 'rv2' })
+ expect((watcher as any).hostDesiredRevision).toBe(base)
+ // A spec change (metadata.generation advances) MUST bump.
+ ;(watcher as any).installHostSnapshot({
+ hosts: [{ ...hostA, generation: 2 }],
+ resourceVersion: 'rv3',
+ })
+ expect((watcher as any).hostDesiredRevision).toBe(base + 1)
+ // A removal MUST bump.
+ ;(watcher as any).installHostSnapshot({ hosts: [], resourceVersion: 'rv4' })
+ expect((watcher as any).hostDesiredRevision).toBe(base + 2)
})
- })
- describe('McpServerWatcher.startCommunicationChannelWatch', () => {
- it('caches ADDED events and triggers Host reconcile', async () => {
+ it('installHostSnapshot is deliberately blind to Host annotations/labels (pins the mutation-relevance invariant)', () => {
+ // Guards the SCOPE note on sameHostDesiredRevision: the Host comparator is
+ // uid+generation only, unlike the McpServer/Context comparators that hash
+ // labels/annotations. That is correct ONLY while no Host annotation/label is
+ // load-bearing for a mutation (the wake annotation drives dispatch, not
+ // template content). This test pins that contract: annotation/label churn at
+ // the SAME (uid, generation) — e.g. a Premature-close re-LIST — must NOT bump
+ // the desired revision. If a Host annotation/label ever becomes mutation-
+ // relevant, whoever extends the comparator to hash it will break this test and
+ // be forced to re-examine the fail-open the SCOPE note warns about.
const watcher = new McpServerWatcher()
- const hosts = (watcher as any).hosts as Map<string, any>
- hosts.set('marketing', {
- name: 'marketing',
+ const hostA = {
+ name: 'a',
namespace: 'mcp-host',
- spec: { host: 'marketing', contextRef: 'c', secretRef: 's' },
+ uid: 'ua',
+ generation: 1,
+ spec: { contextRef: 'ctx' },
+ annotations: { 'clerum.io/wake': 'v1' },
+ labels: { tier: 'a' },
+ } as unknown as HostCRD
+ ;(watcher as any).installHostSnapshot({ hosts: [hostA], resourceVersion: 'rv1' })
+ const base = (watcher as any).hostDesiredRevision
+ ;(watcher as any).installHostSnapshot({
+ hosts: [
+ {
+ ...hostA,
+ annotations: { 'clerum.io/wake': 'v2', 'clerum.io/added': 'x' },
+ labels: { tier: 'b' },
+ },
+ ],
+ resourceVersion: 'rv2',
})
- const reconcileSpy = vi
- .spyOn(watcher.getHostReconciler(), 'reconcile')
- .mockResolvedValue(undefined)
+ expect((watcher as any).hostDesiredRevision).toBe(base)
+ })
+ })
- const watchCallback = (watcher as any).getCommunicationChannelWatchCallback()
- await watchCallback('ADDED', {
- metadata: { name: 'cc-marketing', namespace: 'channels' },
- spec: { hostRef: 'marketing' },
- })
+ describe('McpServerWatcher watch-close recovery latency (immediate first attempt + floor)', () => {
+ // GKE closes long-lived watches ("Premature close") every few minutes. The
+ // old contract armed a FIXED 5s timer before the first re-LIST, so every
+ // isolated close cost ~5.5s of /ready 503. The hardened contract: the FIRST
+ // recovery attempt after a close is IMMEDIATE (microtask, no timer), a paced
+ // timer survives only as the retry-after-failure pacing (now the jittered
+ // exponential backoff: first failure retries within [500ms, 1000ms]), and an
+ // anti-busy-loop FLOOR (1s) demotes a close that arrives <1s after the last
+ // successful recovery back onto the paced timer so a degraded apiserver is
+ // never hammered with back-to-back re-LISTs.
+ const inventoryLanes = [
+ {
+ kind: 'McpServer',
+ plural: 'mcpservers',
+ pathSuffix: '/mcpservers',
+ restartMethod: 'restartMcpServerWatch',
+ startSnapshot: { resourceVersion: 'mcp-start-rv', servers: [] },
+ startRv: 'mcp-start-rv',
+ recoveryRvPrefix: 'mcp-close-recovery-rv',
+ timerField: 'mcpServerCacheRecoveryTimer',
+ syncedField: 'mcpServerCacheSynced',
+ peerSyncedField: 'contextCacheSynced',
+ },
+ {
+ kind: 'Context',
+ plural: 'contexts',
+ pathSuffix: '/contexts',
+ restartMethod: 'restartContextWatch',
+ startSnapshot: { resourceVersion: 'context-start-rv', contexts: [] },
+ startRv: 'context-start-rv',
+ recoveryRvPrefix: 'context-close-recovery-rv',
+ timerField: 'contextCacheRecoveryTimer',
+ syncedField: 'contextCacheSynced',
+ peerSyncedField: 'mcpServerCacheSynced',
+ },
+ ] as const
- const cache = (watcher as any).communicationChannels as Map<string, any>
- expect(cache.get('cc-marketing')?.spec.hostRef).toBe('marketing')
- expect(reconcileSpy).toHaveBeenCalledWith(expect.objectContaining({ name: 'marketing' }))
+ beforeEach(() => {
+ vi.clearAllMocks()
+ vi.useFakeTimers()
+ mocks.watch.mockReset().mockResolvedValue({ abort: vi.fn() })
+ mocks.listNamespacedCustomObject.mockReset().mockResolvedValue({ items: [] })
})
- it('removes from cache on DELETED and triggers Host reconcile', async () => {
- const watcher = new McpServerWatcher()
- const cache = (watcher as any).communicationChannels as Map<string, any>
- cache.set('cc-marketing', {
- name: 'cc-marketing',
- namespace: 'channels',
- spec: { hostRef: 'marketing' },
+ afterEach(() => {
+ vi.useRealTimers()
+ })
+
+ it.each(inventoryLanes)(
+ 're-LISTs the $kind inventory immediately (microtask, no 5s timer) when its watch closes',
+ async lane => {
+ const doneCallbacks: Array<(error: Error | null) => void> = []
+ const watchQueries: unknown[] = []
+ let listCalls = 0
+ mocks.watch.mockImplementation(async (path, options, _callback, done) => {
+ if (path.endsWith(lane.pathSuffix)) {
+ doneCallbacks.push(done)
+ watchQueries.push(options)
+ }
+ return { abort: vi.fn() }
+ })
+ mocks.listNamespacedCustomObject.mockImplementation(
+ async ({ plural }: { plural: string }) => {
+ if (plural === lane.plural) {
+ listCalls += 1
+ return {
+ metadata: { resourceVersion: `${lane.recoveryRvPrefix}-${listCalls}` },
+ items: [],
+ }
+ }
+ return { items: [] }
+ }
+ )
+ const logSpy = vi.spyOn(console, 'log').mockImplementation(() => {})
+ const watcher = new McpServerWatcher()
+ ;(watcher as any)[lane.peerSyncedField] = true
+ await (watcher as any)[lane.restartMethod](lane.startSnapshot)
+ expect(watchQueries).toEqual([{ resourceVersion: lane.startRv }])
+
+ doneCallbacks[0](Object.assign(new Error('Premature close'), { statusCode: 500 }))
+ await flushMicrotasks(20)
+
+ // Immediate first attempt: fresh LIST + fresh watch WITHOUT advancing
+ // any timer. Under the old contract this stayed at 0 until +5000ms.
+ expect(listCalls).toBe(1)
+ expect(watchQueries).toEqual([
+ { resourceVersion: lane.startRv },
+ { resourceVersion: `${lane.recoveryRvPrefix}-1` },
+ ])
+ expect((watcher as any)[lane.syncedField]).toBe(true)
+ expect((watcher as any)[lane.timerField]).toBeNull()
+
+ // No retry timer was armed: +5s must not produce a duplicate re-LIST.
+ await vi.advanceTimersByTimeAsync(5000)
+ expect(listCalls).toBe(1)
+ logSpy.mockRestore()
+ await watcher.stop()
+ }
+ )
+
+ it('re-LISTs the Host inventory immediately (microtask, no 5s timer) when its watch closes', async () => {
+ const doneCallbacks: Array<(error: Error | null) => void> = []
+ const watchQueries: unknown[] = []
+ let hostListCalls = 0
+ mocks.watch.mockImplementation(async (path, options, _callback, done) => {
+ if (path.endsWith('/hosts')) {
+ doneCallbacks.push(done)
+ watchQueries.push(options)
+ }
+ return { abort: vi.fn() }
})
- const hosts = (watcher as any).hosts as Map<string, any>
- hosts.set('marketing', {
- name: 'marketing',
- namespace: 'mcp-host',
- spec: { host: 'marketing', contextRef: 'c', secretRef: 's' },
+ mocks.listNamespacedCustomObject.mockImplementation(async ({ plural }: { plural: string }) => {
+ if (plural === 'hosts') {
+ hostListCalls += 1
+ return {
+ metadata: { resourceVersion: `host-close-recovery-rv-${hostListCalls}` },
+ items: [],
+ }
+ }
+ return { items: [] }
})
- const reconcileSpy = vi
- .spyOn(watcher.getHostReconciler(), 'reconcile')
+ const logSpy = vi.spyOn(console, 'log').mockImplementation(() => {})
+ const watcher = newContextAuthoritativeWatcher()
+ const fleetRequest = vi
+ .spyOn(watcher as any, 'requestHostFleetReconcile')
.mockResolvedValue(undefined)
+ await (watcher as any).startHostWatch('host-initial-rv')
+ ;(watcher as any).hostCacheSynced = true
- const watchCallback = (watcher as any).getCommunicationChannelWatchCallback()
- await watchCallback('DELETED', {
- metadata: { name: 'cc-marketing', namespace: 'channels' },
- spec: { hostRef: 'marketing' },
- })
+ doneCallbacks[0](Object.assign(new Error('Premature close'), { statusCode: 500 }))
+ expect((watcher as any).hostCacheSynced).toBe(false)
+ await flushMicrotasks(20)
- expect(cache.has('cc-marketing')).toBe(false)
- expect(reconcileSpy).toHaveBeenCalled()
+ // Immediate first attempt: fresh LIST + fresh watch WITHOUT advancing any
+ // timer. Under the old contract this stayed at 0 until +5000ms.
+ expect(hostListCalls).toBe(1)
+ expect(watchQueries).toEqual([
+ { resourceVersion: 'host-initial-rv' },
+ { resourceVersion: 'host-close-recovery-rv-1' },
+ ])
+ expect((watcher as any).hostCacheSynced).toBe(true)
+ expect((watcher as any).hostCacheRecoveryTimer).toBeNull()
+ expect(fleetRequest).toHaveBeenCalledOnce()
+
+ // No retry timer was armed: +5s must not produce a duplicate re-LIST.
+ await vi.advanceTimersByTimeAsync(5000)
+ expect(hostListCalls).toBe(1)
+ logSpy.mockRestore()
+ await watcher.stop()
})
- it('on MODIFIED with hostRef change, reconciles BOTH old and new hosts', async () => {
- const watcher = new McpServerWatcher()
- const cache = (watcher as any).communicationChannels as Map<string, any>
- cache.set('cc-1', {
- name: 'cc-1',
- namespace: 'channels',
- spec: { hostRef: 'old-host' },
- })
- const hosts = (watcher as any).hosts as Map<string, any>
- hosts.set('old-host', {
- name: 'old-host',
- namespace: 'mcp-host',
- spec: { host: 'old-host', contextRef: 'c', secretRef: 's' },
+ it.each(inventoryLanes)(
+ 'falls back to the paced backoff retry timer only after a failed immediate $kind recovery attempt',
+ async lane => {
+ const doneCallbacks: Array<(error: Error | null) => void> = []
+ let listCalls = 0
+ mocks.watch.mockImplementation(async (path, _options, _callback, done) => {
+ if (path.endsWith(lane.pathSuffix)) doneCallbacks.push(done)
+ return { abort: vi.fn() }
+ })
+ mocks.listNamespacedCustomObject.mockImplementation(
+ async ({ plural }: { plural: string }) => {
+ if (plural === lane.plural) {
+ listCalls += 1
+ if (listCalls === 1) throw new Error(`${lane.plural} re-LIST temporarily unavailable`)
+ return {
+ metadata: { resourceVersion: `${lane.recoveryRvPrefix}-${listCalls}` },
+ items: [],
+ }
+ }
+ return { items: [] }
+ }
+ )
+ const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {})
+ const logSpy = vi.spyOn(console, 'log').mockImplementation(() => {})
+ const watcher = new McpServerWatcher()
+ ;(watcher as any)[lane.peerSyncedField] = true
+ await (watcher as any)[lane.restartMethod](lane.startSnapshot)
+
+ doneCallbacks[0](Object.assign(new Error('Premature close'), { statusCode: 500 }))
+ await flushMicrotasks(20)
+
+ // The immediate attempt ran (and failed): exactly one LIST, retry armed.
+ expect(listCalls).toBe(1)
+ expect((watcher as any)[lane.syncedField]).toBe(false)
+ expect((watcher as any)[lane.timerField]).not.toBeNull()
+
+ // Retry-after-failure keeps its pacing: the first-failure backoff delay
+ // is jittered within [500ms, 1000ms], so nothing may fire before the
+ // 500ms floor and the retry must have fired by the 1000ms bound.
+ await vi.advanceTimersByTimeAsync(499)
+ expect(listCalls).toBe(1)
+ await vi.advanceTimersByTimeAsync(501)
+ await flushMicrotasks(20)
+ expect(listCalls).toBe(2)
+ expect((watcher as any)[lane.syncedField]).toBe(true)
+ expect((watcher as any)[lane.timerField]).toBeNull()
+ errorSpy.mockRestore()
+ logSpy.mockRestore()
+ await watcher.stop()
+ }
+ )
+
+ it('falls back to the paced backoff retry timer only after a failed immediate Host recovery attempt', async () => {
+ const doneCallbacks: Array<(error: Error | null) => void> = []
+ let hostListCalls = 0
+ mocks.watch.mockImplementation(async (path, _options, _callback, done) => {
+ if (path.endsWith('/hosts')) doneCallbacks.push(done)
+ return { abort: vi.fn() }
})
- hosts.set('new-host', {
- name: 'new-host',
- namespace: 'mcp-host',
- spec: { host: 'new-host', contextRef: 'c', secretRef: 's' },
+ mocks.listNamespacedCustomObject.mockImplementation(async ({ plural }: { plural: string }) => {
+ if (plural === 'hosts') {
+ hostListCalls += 1
+ if (hostListCalls === 1) throw new Error('hosts re-LIST temporarily unavailable')
+ return {
+ metadata: { resourceVersion: `host-close-recovery-rv-${hostListCalls}` },
+ items: [],
+ }
+ }
+ return { items: [] }
})
- const reconcileSpy = vi
- .spyOn(watcher.getHostReconciler(), 'reconcile')
+ const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {})
+ const logSpy = vi.spyOn(console, 'log').mockImplementation(() => {})
+ const watcher = newContextAuthoritativeWatcher()
+ const fleetRequest = vi
+ .spyOn(watcher as any, 'requestHostFleetReconcile')
.mockResolvedValue(undefined)
+ await (watcher as any).startHostWatch('host-initial-rv')
+ ;(watcher as any).hostCacheSynced = true
- const watchCallback = (watcher as any).getCommunicationChannelWatchCallback()
- await watchCallback('MODIFIED', {
- metadata: { name: 'cc-1', namespace: 'channels' },
- spec: { hostRef: 'new-host' },
- })
+ doneCallbacks[0](Object.assign(new Error('Premature close'), { statusCode: 500 }))
+ await flushMicrotasks(20)
- expect(reconcileSpy).toHaveBeenCalledTimes(2)
- const names = reconcileSpy.mock.calls.map(c => (c[0] as any).name).sort()
- expect(names).toEqual(['new-host', 'old-host'])
+ // The immediate attempt ran (and failed): exactly one LIST, retry armed
+ // by performHostInventoryRecovery's existing failure path. The first-
+ // failure backoff delay is jittered within [500ms, 1000ms].
+ expect(hostListCalls).toBe(1)
+ expect((watcher as any).hostCacheSynced).toBe(false)
+ expect((watcher as any).hostCacheRecoveryTimer).not.toBeNull()
+
+ await vi.advanceTimersByTimeAsync(499)
+ expect(hostListCalls).toBe(1)
+ await vi.advanceTimersByTimeAsync(501)
+ await flushMicrotasks(20)
+ expect(hostListCalls).toBe(2)
+ expect((watcher as any).hostCacheSynced).toBe(true)
+ expect((watcher as any).hostCacheRecoveryTimer).toBeNull()
+ expect(fleetRequest).toHaveBeenCalled()
+ errorSpy.mockRestore()
+ logSpy.mockRestore()
+ await watcher.stop()
})
- it('ignores unknown event types (e.g. BOOKMARK) without mutating cache or reconciling', async () => {
- const watcher = new McpServerWatcher()
- const cache = (watcher as any).communicationChannels as Map<string, any>
- cache.set('cc-1', {
- name: 'cc-1',
- namespace: 'channels',
- spec: { hostRef: 'marketing' },
+ it.each(inventoryLanes)(
+ 'demotes a $kind close arriving within the 1s floor of the last recovery onto the paced retry timer',
+ async lane => {
+ const doneCallbacks: Array<(error: Error | null) => void> = []
+ let listCalls = 0
+ mocks.watch.mockImplementation(async (path, _options, _callback, done) => {
+ if (path.endsWith(lane.pathSuffix)) doneCallbacks.push(done)
+ return { abort: vi.fn() }
+ })
+ mocks.listNamespacedCustomObject.mockImplementation(
+ async ({ plural }: { plural: string }) => {
+ if (plural === lane.plural) {
+ listCalls += 1
+ return {
+ metadata: { resourceVersion: `${lane.recoveryRvPrefix}-${listCalls}` },
+ items: [],
+ }
+ }
+ return { items: [] }
+ }
+ )
+ const logSpy = vi.spyOn(console, 'log').mockImplementation(() => {})
+ const watcher = new McpServerWatcher()
+ ;(watcher as any)[lane.peerSyncedField] = true
+ await (watcher as any)[lane.restartMethod](lane.startSnapshot)
+
+ // Isolated close: immediate recovery (LIST #1, watch #2).
+ doneCallbacks[0](Object.assign(new Error('Premature close'), { statusCode: 500 }))
+ await flushMicrotasks(20)
+ expect(listCalls).toBe(1)
+ expect(doneCallbacks).toHaveLength(2)
+
+ // Burst close: the recovered watch dies again with ZERO fake-time
+ // elapsed since the recovery that established it (< 1s floor). The
+ // anti-busy-loop floor must demote this attempt onto the paced timer
+ // instead of hammering the (visibly degraded) apiserver immediately.
+ doneCallbacks[1](Object.assign(new Error('Premature close'), { statusCode: 500 }))
+ await flushMicrotasks(20)
+ expect(listCalls).toBe(1)
+ expect((watcher as any)[lane.timerField]).not.toBeNull()
+ expect((watcher as any)[lane.syncedField]).toBe(false)
+
+ // The paced retry recovers it within the +5s advance.
+ await vi.advanceTimersByTimeAsync(5000)
+ await flushMicrotasks(20)
+ expect(listCalls).toBe(2)
+ expect((watcher as any)[lane.syncedField]).toBe(true)
+ expect((watcher as any)[lane.timerField]).toBeNull()
+ expect(doneCallbacks).toHaveLength(3)
+
+ // Spaced close: once more than the 1s floor has elapsed since the last
+ // recovery, a new close is immediate again (no timer).
+ await vi.advanceTimersByTimeAsync(1500)
+ doneCallbacks[2](Object.assign(new Error('Premature close'), { statusCode: 500 }))
+ await flushMicrotasks(20)
+ expect(listCalls).toBe(3)
+ expect((watcher as any)[lane.timerField]).toBeNull()
+ expect((watcher as any)[lane.syncedField]).toBe(true)
+ logSpy.mockRestore()
+ await watcher.stop()
+ }
+ )
+
+ it('demotes a Host close arriving within the 1s floor of the last recovery onto the paced retry timer', async () => {
+ const doneCallbacks: Array<(error: Error | null) => void> = []
+ let hostListCalls = 0
+ mocks.watch.mockImplementation(async (path, _options, _callback, done) => {
+ if (path.endsWith('/hosts')) doneCallbacks.push(done)
+ return { abort: vi.fn() }
})
- const hosts = (watcher as any).hosts as Map<string, any>
- hosts.set('marketing', {
- name: 'marketing',
- namespace: 'mcp-host',
- spec: { host: 'marketing', contextRef: 'c', secretRef: 's' },
+ mocks.listNamespacedCustomObject.mockImplementation(async ({ plural }: { plural: string }) => {
+ if (plural === 'hosts') {
+ hostListCalls += 1
+ return {
+ metadata: { resourceVersion: `host-close-recovery-rv-${hostListCalls}` },
+ items: [],
+ }
+ }
+ return { items: [] }
})
- const reconcileSpy = vi
- .spyOn(watcher.getHostReconciler(), 'reconcile')
- .mockResolvedValue(undefined)
+ const logSpy = vi.spyOn(console, 'log').mockImplementation(() => {})
+ const watcher = newContextAuthoritativeWatcher()
+ vi.spyOn(watcher as any, 'requestHostFleetReconcile').mockResolvedValue(undefined)
+ await (watcher as any).startHostWatch('host-initial-rv')
+ ;(watcher as any).hostCacheSynced = true
- const watchCallback = (watcher as any).getCommunicationChannelWatchCallback()
- await watchCallback('BOOKMARK', {
- metadata: { name: 'cc-1', namespace: 'channels' },
- spec: { hostRef: 'marketing' },
- })
+ // Isolated close: immediate recovery (LIST #1, watch #2).
+ doneCallbacks[0](Object.assign(new Error('Premature close'), { statusCode: 500 }))
+ await flushMicrotasks(20)
+ expect(hostListCalls).toBe(1)
+ expect(doneCallbacks).toHaveLength(2)
+ expect((watcher as any).hostCacheSynced).toBe(true)
+
+ // Burst close within the 1s floor: demoted onto the paced retry timer.
+ doneCallbacks[1](Object.assign(new Error('Premature close'), { statusCode: 500 }))
+ await flushMicrotasks(20)
+ expect(hostListCalls).toBe(1)
+ expect((watcher as any).hostCacheRecoveryTimer).not.toBeNull()
+ expect((watcher as any).hostCacheSynced).toBe(false)
- expect(cache.get('cc-1')?.spec.hostRef).toBe('marketing')
- expect(reconcileSpy).not.toHaveBeenCalled()
+ await vi.advanceTimersByTimeAsync(5000)
+ await flushMicrotasks(20)
+ expect(hostListCalls).toBe(2)
+ expect((watcher as any).hostCacheSynced).toBe(true)
+ expect((watcher as any).hostCacheRecoveryTimer).toBeNull()
+ expect(doneCallbacks).toHaveLength(3)
+
+ // Spaced close beyond the floor: immediate again.
+ await vi.advanceTimersByTimeAsync(1500)
+ doneCallbacks[2](Object.assign(new Error('Premature close'), { statusCode: 500 }))
+ await flushMicrotasks(20)
+ expect(hostListCalls).toBe(3)
+ expect((watcher as any).hostCacheRecoveryTimer).toBeNull()
+ expect((watcher as any).hostCacheSynced).toBe(true)
+ logSpy.mockRestore()
+ await watcher.stop()
})
})
diff --cc host-context-controller/src/k8sClient.ts
index 46b588de9,ff4874570..085d4bd89
--- a/host-context-controller/src/k8sClient.ts
+++ b/host-context-controller/src/k8sClient.ts
@@@ -1621,11 -2527,8 +2546,11 @@@ export class McpServerWatcher implement
contextRef: server.spec.contextRef,
transport,
auth: server.spec.auth,
+ // Project the OAuth block (grantScope etc.) verbatim so mcp-host can
+ // dispatch the per-connection partition. Token is NEVER mounted (O4).
+ oauth: server.spec.oauth,
enabled: server.spec.enabled !== false,
- status: this.reconciler.getStatus(server.name),
+ status: this.reconciler.getStatus(server),
}
}