Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript

perf(hcc): recover the inventory watch immediately on close, floor-paced

Alfredo Lopezd527543d7d4f
diff --git a/host-context-controller/src/k8sClient.test.ts b/host-context-controller/src/k8sClient.test.ts
index e0bba9a22..9e235f1cf 100644
--- a/host-context-controller/src/k8sClient.test.ts
+++ b/host-context-controller/src/k8sClient.test.ts
@@ -8199,8 +8199,11 @@ describe('McpServerWatcher Host watch generation', () => {
doneCallbacks[0](Object.assign(new Error('resource version expired'), { statusCode: 410 }))
expect((watcher as any).hostCacheSynced).toBe(false)
- expect((watcher as any).hostCacheRecoveryTimer).not.toBeNull()
- await vi.advanceTimersByTimeAsync(5000)
+ // Immediate-first-attempt contract: an isolated close (nothing recovered
+ // within the 1s floor) re-LISTs NOW in a microtask. The 5s timer stays
+ // disarmed and survives only as retry-after-failure pacing.
+ expect((watcher as any).hostCacheRecoveryTimer).toBeNull()
+ await flushMicrotasks(20)
await vi.waitFor(() => expect(mocks.hostFullReconcile).toHaveBeenCalledOnce())
expect(
@@ -8211,6 +8214,11 @@ describe('McpServerWatcher Host watch generation', () => {
{ resourceVersion: 'host-recovery-rv' },
])
expect((watcher as any).hostCacheSynced).toBe(true)
+ // The disarmed timer must not produce a duplicate re-LIST at +5s.
+ await vi.advanceTimersByTimeAsync(5000)
+ expect(
+ mocks.listNamespacedCustomObject.mock.calls.filter(([request]) => request.plural === 'hosts')
+ ).toHaveLength(1)
watcher.stop()
})
@@ -9423,3 +9431,371 @@ describe('McpServerWatcher readiness under sustained watch churn (GKE Premature-
expect((watcher as any).hostDesiredRevision).toBe(base + 2)
})
})
+
+describe('McpServerWatcher watch-close recovery latency (immediate first attempt + floor)', () => {
+ // GKE closes long-lived watches ("Premature close") every few minutes. The
+ // old contract armed a FIXED 5s timer before the first re-LIST, so every
+ // isolated close cost ~5.5s of /ready 503. The hardened contract: the FIRST
+ // recovery attempt after a close is IMMEDIATE (microtask, no timer), the 5s
+ // timer survives only as the retry-after-failure pacing, and an anti-busy-loop
+ // FLOOR (1s) demotes a close that arrives <1s after the last successful
+ // recovery back onto the paced timer so a degraded apiserver is never
+ // hammered with back-to-back re-LISTs.
+ const inventoryLanes = [
+ {
+ kind: 'McpServer',
+ plural: 'mcpservers',
+ pathSuffix: '/mcpservers',
+ restartMethod: 'restartMcpServerWatch',
+ startSnapshot: { resourceVersion: 'mcp-start-rv', servers: [] },
+ startRv: 'mcp-start-rv',
+ recoveryRvPrefix: 'mcp-close-recovery-rv',
+ timerField: 'mcpServerCacheRecoveryTimer',
+ syncedField: 'mcpServerCacheSynced',
+ peerSyncedField: 'contextCacheSynced',
+ },
+ {
+ kind: 'Context',
+ plural: 'contexts',
+ pathSuffix: '/contexts',
+ restartMethod: 'restartContextWatch',
+ startSnapshot: { resourceVersion: 'context-start-rv', contexts: [] },
+ startRv: 'context-start-rv',
+ recoveryRvPrefix: 'context-close-recovery-rv',
+ timerField: 'contextCacheRecoveryTimer',
+ syncedField: 'contextCacheSynced',
+ peerSyncedField: 'mcpServerCacheSynced',
+ },
+ ] as const
+
+ beforeEach(() => {
+ vi.clearAllMocks()
+ vi.useFakeTimers()
+ mocks.watch.mockReset().mockResolvedValue({ abort: vi.fn() })
+ mocks.listNamespacedCustomObject.mockReset().mockResolvedValue({ items: [] })
+ })
+
+ afterEach(() => {
+ vi.useRealTimers()
+ })
+
+ it.each(inventoryLanes)(
+ 're-LISTs the $kind inventory immediately (microtask, no 5s timer) when its watch closes',
+ async lane => {
+ const doneCallbacks: Array<(error: Error | null) => void> = []
+ const watchQueries: unknown[] = []
+ let listCalls = 0
+ mocks.watch.mockImplementation(async (path, options, _callback, done) => {
+ if (path.endsWith(lane.pathSuffix)) {
+ doneCallbacks.push(done)
+ watchQueries.push(options)
+ }
+ return { abort: vi.fn() }
+ })
+ mocks.listNamespacedCustomObject.mockImplementation(
+ async ({ plural }: { plural: string }) => {
+ if (plural === lane.plural) {
+ listCalls += 1
+ return {
+ metadata: { resourceVersion: `${lane.recoveryRvPrefix}-${listCalls}` },
+ items: [],
+ }
+ }
+ return { items: [] }
+ }
+ )
+ const logSpy = vi.spyOn(console, 'log').mockImplementation(() => {})
+ const watcher = new McpServerWatcher()
+ ;(watcher as any)[lane.peerSyncedField] = true
+ await (watcher as any)[lane.restartMethod](lane.startSnapshot)
+ expect(watchQueries).toEqual([{ resourceVersion: lane.startRv }])
+
+ doneCallbacks[0](Object.assign(new Error('Premature close'), { statusCode: 500 }))
+ await flushMicrotasks(20)
+
+ // Immediate first attempt: fresh LIST + fresh watch WITHOUT advancing
+ // any timer. Under the old contract this stayed at 0 until +5000ms.
+ expect(listCalls).toBe(1)
+ expect(watchQueries).toEqual([
+ { resourceVersion: lane.startRv },
+ { resourceVersion: `${lane.recoveryRvPrefix}-1` },
+ ])
+ expect((watcher as any)[lane.syncedField]).toBe(true)
+ expect((watcher as any)[lane.timerField]).toBeNull()
+
+ // No retry timer was armed: +5s must not produce a duplicate re-LIST.
+ await vi.advanceTimersByTimeAsync(5000)
+ expect(listCalls).toBe(1)
+ logSpy.mockRestore()
+ await watcher.stop()
+ }
+ )
+
+ it('re-LISTs the Host inventory immediately (microtask, no 5s timer) when its watch closes', async () => {
+ const doneCallbacks: Array<(error: Error | null) => void> = []
+ const watchQueries: unknown[] = []
+ let hostListCalls = 0
+ mocks.watch.mockImplementation(async (path, options, _callback, done) => {
+ if (path.endsWith('/hosts')) {
+ doneCallbacks.push(done)
+ watchQueries.push(options)
+ }
+ return { abort: vi.fn() }
+ })
+ mocks.listNamespacedCustomObject.mockImplementation(async ({ plural }: { plural: string }) => {
+ if (plural === 'hosts') {
+ hostListCalls += 1
+ return {
+ metadata: { resourceVersion: `host-close-recovery-rv-${hostListCalls}` },
+ items: [],
+ }
+ }
+ return { items: [] }
+ })
+ const logSpy = vi.spyOn(console, 'log').mockImplementation(() => {})
+ const watcher = newContextAuthoritativeWatcher()
+ const fleetRequest = vi
+ .spyOn(watcher as any, 'requestHostFleetReconcile')
+ .mockResolvedValue(undefined)
+ await (watcher as any).startHostWatch('host-initial-rv')
+ ;(watcher as any).hostCacheSynced = true
+
+ doneCallbacks[0](Object.assign(new Error('Premature close'), { statusCode: 500 }))
+ expect((watcher as any).hostCacheSynced).toBe(false)
+ await flushMicrotasks(20)
+
+ // Immediate first attempt: fresh LIST + fresh watch WITHOUT advancing any
+ // timer. Under the old contract this stayed at 0 until +5000ms.
+ expect(hostListCalls).toBe(1)
+ expect(watchQueries).toEqual([
+ { resourceVersion: 'host-initial-rv' },
+ { resourceVersion: 'host-close-recovery-rv-1' },
+ ])
+ expect((watcher as any).hostCacheSynced).toBe(true)
+ expect((watcher as any).hostCacheRecoveryTimer).toBeNull()
+ expect(fleetRequest).toHaveBeenCalledOnce()
+
+ // No retry timer was armed: +5s must not produce a duplicate re-LIST.
+ await vi.advanceTimersByTimeAsync(5000)
+ expect(hostListCalls).toBe(1)
+ logSpy.mockRestore()
+ await watcher.stop()
+ })
+
+ it.each(inventoryLanes)(
+ 'falls back to the 5s retry timer only after a failed immediate $kind recovery attempt',
+ async lane => {
+ const doneCallbacks: Array<(error: Error | null) => void> = []
+ let listCalls = 0
+ mocks.watch.mockImplementation(async (path, _options, _callback, done) => {
+ if (path.endsWith(lane.pathSuffix)) doneCallbacks.push(done)
+ return { abort: vi.fn() }
+ })
+ mocks.listNamespacedCustomObject.mockImplementation(
+ async ({ plural }: { plural: string }) => {
+ if (plural === lane.plural) {
+ listCalls += 1
+ if (listCalls === 1) throw new Error(`${lane.plural} re-LIST temporarily unavailable`)
+ return {
+ metadata: { resourceVersion: `${lane.recoveryRvPrefix}-${listCalls}` },
+ items: [],
+ }
+ }
+ return { items: [] }
+ }
+ )
+ const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {})
+ const logSpy = vi.spyOn(console, 'log').mockImplementation(() => {})
+ const watcher = new McpServerWatcher()
+ ;(watcher as any)[lane.peerSyncedField] = true
+ await (watcher as any)[lane.restartMethod](lane.startSnapshot)
+
+ doneCallbacks[0](Object.assign(new Error('Premature close'), { statusCode: 500 }))
+ await flushMicrotasks(20)
+
+ // The immediate attempt ran (and failed): exactly one LIST, retry armed.
+ expect(listCalls).toBe(1)
+ expect((watcher as any)[lane.syncedField]).toBe(false)
+ expect((watcher as any)[lane.timerField]).not.toBeNull()
+
+ // Retry-after-failure keeps the 5s pacing: nothing before the deadline.
+ await vi.advanceTimersByTimeAsync(4999)
+ expect(listCalls).toBe(1)
+ await vi.advanceTimersByTimeAsync(1)
+ await flushMicrotasks(20)
+ expect(listCalls).toBe(2)
+ expect((watcher as any)[lane.syncedField]).toBe(true)
+ expect((watcher as any)[lane.timerField]).toBeNull()
+ errorSpy.mockRestore()
+ logSpy.mockRestore()
+ await watcher.stop()
+ }
+ )
+
+ it('falls back to the 5s retry timer only after a failed immediate Host recovery attempt', async () => {
+ const doneCallbacks: Array<(error: Error | null) => void> = []
+ let hostListCalls = 0
+ mocks.watch.mockImplementation(async (path, _options, _callback, done) => {
+ if (path.endsWith('/hosts')) doneCallbacks.push(done)
+ return { abort: vi.fn() }
+ })
+ mocks.listNamespacedCustomObject.mockImplementation(async ({ plural }: { plural: string }) => {
+ if (plural === 'hosts') {
+ hostListCalls += 1
+ if (hostListCalls === 1) throw new Error('hosts re-LIST temporarily unavailable')
+ return {
+ metadata: { resourceVersion: `host-close-recovery-rv-${hostListCalls}` },
+ items: [],
+ }
+ }
+ return { items: [] }
+ })
+ const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {})
+ const logSpy = vi.spyOn(console, 'log').mockImplementation(() => {})
+ const watcher = newContextAuthoritativeWatcher()
+ const fleetRequest = vi
+ .spyOn(watcher as any, 'requestHostFleetReconcile')
+ .mockResolvedValue(undefined)
+ await (watcher as any).startHostWatch('host-initial-rv')
+ ;(watcher as any).hostCacheSynced = true
+
+ doneCallbacks[0](Object.assign(new Error('Premature close'), { statusCode: 500 }))
+ await flushMicrotasks(20)
+
+ // The immediate attempt ran (and failed): exactly one LIST, retry armed
+ // by performHostInventoryRecovery's existing failure path.
+ expect(hostListCalls).toBe(1)
+ expect((watcher as any).hostCacheSynced).toBe(false)
+ expect((watcher as any).hostCacheRecoveryTimer).not.toBeNull()
+
+ await vi.advanceTimersByTimeAsync(4999)
+ expect(hostListCalls).toBe(1)
+ await vi.advanceTimersByTimeAsync(1)
+ await flushMicrotasks(20)
+ expect(hostListCalls).toBe(2)
+ expect((watcher as any).hostCacheSynced).toBe(true)
+ expect((watcher as any).hostCacheRecoveryTimer).toBeNull()
+ expect(fleetRequest).toHaveBeenCalled()
+ errorSpy.mockRestore()
+ logSpy.mockRestore()
+ await watcher.stop()
+ })
+
+ it.each(inventoryLanes)(
+ 'demotes a $kind close arriving within the 1s floor of the last recovery onto the 5s timer',
+ async lane => {
+ const doneCallbacks: Array<(error: Error | null) => void> = []
+ let listCalls = 0
+ mocks.watch.mockImplementation(async (path, _options, _callback, done) => {
+ if (path.endsWith(lane.pathSuffix)) doneCallbacks.push(done)
+ return { abort: vi.fn() }
+ })
+ mocks.listNamespacedCustomObject.mockImplementation(
+ async ({ plural }: { plural: string }) => {
+ if (plural === lane.plural) {
+ listCalls += 1
+ return {
+ metadata: { resourceVersion: `${lane.recoveryRvPrefix}-${listCalls}` },
+ items: [],
+ }
+ }
+ return { items: [] }
+ }
+ )
+ const logSpy = vi.spyOn(console, 'log').mockImplementation(() => {})
+ const watcher = new McpServerWatcher()
+ ;(watcher as any)[lane.peerSyncedField] = true
+ await (watcher as any)[lane.restartMethod](lane.startSnapshot)
+
+ // Isolated close: immediate recovery (LIST #1, watch #2).
+ doneCallbacks[0](Object.assign(new Error('Premature close'), { statusCode: 500 }))
+ await flushMicrotasks(20)
+ expect(listCalls).toBe(1)
+ expect(doneCallbacks).toHaveLength(2)
+
+ // Burst close: the recovered watch dies again with ZERO fake-time
+ // elapsed since the recovery that established it (< 1s floor). The
+ // anti-busy-loop floor must demote this attempt onto the 5s timer
+ // instead of hammering the (visibly degraded) apiserver immediately.
+ doneCallbacks[1](Object.assign(new Error('Premature close'), { statusCode: 500 }))
+ await flushMicrotasks(20)
+ expect(listCalls).toBe(1)
+ expect((watcher as any)[lane.timerField]).not.toBeNull()
+ expect((watcher as any)[lane.syncedField]).toBe(false)
+
+ // The paced retry recovers it at +5s.
+ await vi.advanceTimersByTimeAsync(5000)
+ await flushMicrotasks(20)
+ expect(listCalls).toBe(2)
+ expect((watcher as any)[lane.syncedField]).toBe(true)
+ expect((watcher as any)[lane.timerField]).toBeNull()
+ expect(doneCallbacks).toHaveLength(3)
+
+ // Spaced close: once more than the 1s floor has elapsed since the last
+ // recovery, a new close is immediate again (no timer).
+ await vi.advanceTimersByTimeAsync(1500)
+ doneCallbacks[2](Object.assign(new Error('Premature close'), { statusCode: 500 }))
+ await flushMicrotasks(20)
+ expect(listCalls).toBe(3)
+ expect((watcher as any)[lane.timerField]).toBeNull()
+ expect((watcher as any)[lane.syncedField]).toBe(true)
+ logSpy.mockRestore()
+ await watcher.stop()
+ }
+ )
+
+ it('demotes a Host close arriving within the 1s floor of the last recovery onto the 5s timer', async () => {
+ const doneCallbacks: Array<(error: Error | null) => void> = []
+ let hostListCalls = 0
+ mocks.watch.mockImplementation(async (path, _options, _callback, done) => {
+ if (path.endsWith('/hosts')) doneCallbacks.push(done)
+ return { abort: vi.fn() }
+ })
+ mocks.listNamespacedCustomObject.mockImplementation(async ({ plural }: { plural: string }) => {
+ if (plural === 'hosts') {
+ hostListCalls += 1
+ return {
+ metadata: { resourceVersion: `host-close-recovery-rv-${hostListCalls}` },
+ items: [],
+ }
+ }
+ return { items: [] }
+ })
+ const logSpy = vi.spyOn(console, 'log').mockImplementation(() => {})
+ const watcher = newContextAuthoritativeWatcher()
+ vi.spyOn(watcher as any, 'requestHostFleetReconcile').mockResolvedValue(undefined)
+ await (watcher as any).startHostWatch('host-initial-rv')
+ ;(watcher as any).hostCacheSynced = true
+
+ // Isolated close: immediate recovery (LIST #1, watch #2).
+ doneCallbacks[0](Object.assign(new Error('Premature close'), { statusCode: 500 }))
+ await flushMicrotasks(20)
+ expect(hostListCalls).toBe(1)
+ expect(doneCallbacks).toHaveLength(2)
+ expect((watcher as any).hostCacheSynced).toBe(true)
+
+ // Burst close within the 1s floor: demoted onto the paced 5s timer.
+ doneCallbacks[1](Object.assign(new Error('Premature close'), { statusCode: 500 }))
+ await flushMicrotasks(20)
+ expect(hostListCalls).toBe(1)
+ expect((watcher as any).hostCacheRecoveryTimer).not.toBeNull()
+ expect((watcher as any).hostCacheSynced).toBe(false)
+
+ await vi.advanceTimersByTimeAsync(5000)
+ await flushMicrotasks(20)
+ expect(hostListCalls).toBe(2)
+ expect((watcher as any).hostCacheSynced).toBe(true)
+ expect((watcher as any).hostCacheRecoveryTimer).toBeNull()
+ expect(doneCallbacks).toHaveLength(3)
+
+ // Spaced close beyond the floor: immediate again.
+ await vi.advanceTimersByTimeAsync(1500)
+ doneCallbacks[2](Object.assign(new Error('Premature close'), { statusCode: 500 }))
+ await flushMicrotasks(20)
+ expect(hostListCalls).toBe(3)
+ expect((watcher as any).hostCacheRecoveryTimer).toBeNull()
+ expect((watcher as any).hostCacheSynced).toBe(true)
+ logSpy.mockRestore()
+ await watcher.stop()
+ })
+})
diff --git a/host-context-controller/src/k8sClient.ts b/host-context-controller/src/k8sClient.ts
index 680ceda74..a0c0da610 100644
--- a/host-context-controller/src/k8sClient.ts
+++ b/host-context-controller/src/k8sClient.ts
@@ -113,6 +113,13 @@ const INVENTORY_CACHE_RECOVERY_RETRY_MS = 5000
const COMMUNICATION_CHANNEL_CACHE_RECOVERY_RETRY_MS = 5000
const HOST_CACHE_RECOVERY_RETRY_MS = 5000
+// Anti-busy-loop floor for IMMEDIATE watch-close recovery. The first re-LIST
+// after an isolated watch close runs immediately (sub-second readiness blip
+// instead of the old fixed ~5.5s), but a close arriving within this floor of
+// the last successful recovery is demoted onto the paced retry timer above so
+// a degraded apiserver churning its watches is never hammered with
+// back-to-back LISTs.
+const WATCH_CLOSE_RECOVERY_FLOOR_MS = 1000
const HOST_WATCH_RECONCILE_RETRY_DELAYS_MS = [5000, 15000, 30000]
// Wake-pending Hosts get immediate per-Host admission after watch recovery
// (§10.2 step 7) rather than waiting for the background fleet pass.
@@ -661,6 +668,13 @@ export class McpServerWatcher implements McpServerProvider {
private contextCacheRecoveryTimer: ReturnType<typeof setTimeout> | null = null
private mcpServerCacheRecoveryInFlight: Promise<boolean> | null = null
private contextCacheRecoveryInFlight: Promise<boolean> | null = null
+ // Instant (Date.now) at which the last SUCCESSFUL recovery re-established
+ // each inventory watch. attempt*CacheRecovery compares against these to
+ // apply WATCH_CLOSE_RECOVERY_FLOOR_MS: a close spaced beyond the floor gets
+ // an immediate re-LIST; a burst close within it falls back to the timer.
+ private lastMcpServerWatchRecoveryAt = Number.NEGATIVE_INFINITY
+ private lastContextWatchRecoveryAt = Number.NEGATIVE_INFINITY
+ private lastHostWatchRecoveryAt = Number.NEGATIVE_INFINITY
// A watch callback can still settle after its stream reports completion.
// Host reconciles change the mcp-host pod template, so stale callbacks must
// never replay an older Host spec after a replacement watch is active.
@@ -1323,6 +1337,7 @@ export class McpServerWatcher implements McpServerProvider {
if (this.stopped) return false
await this.restartMcpServerWatch(snapshot)
if (this.stopped) return false
+ this.lastMcpServerWatchRecoveryAt = Date.now()
this.changeCallback?.()
if (this.contextCacheSynced) {
void this.runInitialNetworkPolicyConvergence()
@@ -1354,6 +1369,7 @@ export class McpServerWatcher implements McpServerProvider {
if (this.stopped) return false
await this.restartContextWatch(snapshot)
if (this.stopped) return false
+ this.lastContextWatchRecoveryAt = Date.now()
this.hostFleetScheduler.bumpInputRevision()
void this.runInitialNetworkPolicyConvergence()
void this.runInitialSharedFileSystemConvergence()
@@ -1385,13 +1401,58 @@ export class McpServerWatcher implements McpServerProvider {
return recovery
}
+ /**
+ * One recovery attempt NOW (immediate re-LIST on watch close), with the 5s
+ * timer kept purely as retry-after-failure pacing. The anti-busy-loop floor
+ * demotes a close that arrives within WATCH_CLOSE_RECOVERY_FLOOR_MS of the
+ * last successful recovery back onto the paced timer. Dedup against
+ * concurrent attempts lives in recoverMcpServerInventoryAndWatch
+ * (mcpServerCacheRecoveryInFlight).
+ */
+ private attemptMcpServerCacheRecovery(): void {
+ if (this.stopped) return
+ if (Date.now() - this.lastMcpServerWatchRecoveryAt < WATCH_CLOSE_RECOVERY_FLOOR_MS) {
+ this.scheduleMcpServerCacheRecovery()
+ return
+ }
+ void this.recoverMcpServerInventoryAndWatch().then(recovered => {
+ if (!recovered) this.scheduleMcpServerCacheRecovery()
+ })
+ }
+
+ /** Context analogue of attemptMcpServerCacheRecovery. */
+ private attemptContextCacheRecovery(): void {
+ if (this.stopped) return
+ if (Date.now() - this.lastContextWatchRecoveryAt < WATCH_CLOSE_RECOVERY_FLOOR_MS) {
+ this.scheduleContextCacheRecovery()
+ return
+ }
+ void this.recoverContextInventoryAndWatch().then(recovered => {
+ if (!recovered) this.scheduleContextCacheRecovery()
+ })
+ }
+
+ /**
+ * Host analogue: one attempt NOW unless inside the floor. The retry re-arm
+ * on failure lives in performHostInventoryRecovery (scheduleHostCacheRecovery
+ * before rethrow) and the intent/in-flight merge lives in
+ * recoverHostInventoryAndWatch, so this helper only decides immediate vs
+ * paced.
+ */
+ private attemptHostCacheRecovery(): void {
+ if (this.stopped) return
+ if (Date.now() - this.lastHostWatchRecoveryAt < WATCH_CLOSE_RECOVERY_FLOOR_MS) {
+ this.scheduleHostCacheRecovery()
+ return
+ }
+ void this.recoverHostInventoryAndWatch().catch(() => undefined)
+ }
+
private scheduleMcpServerCacheRecovery(): void {
if (this.stopped || this.mcpServerCacheRecoveryTimer) return
this.mcpServerCacheRecoveryTimer = setTimeout(() => {
this.mcpServerCacheRecoveryTimer = null
- void this.recoverMcpServerInventoryAndWatch().then(recovered => {
- if (!recovered) this.scheduleMcpServerCacheRecovery()
- })
+ this.attemptMcpServerCacheRecovery()
}, INVENTORY_CACHE_RECOVERY_RETRY_MS)
}
@@ -1399,9 +1460,7 @@ export class McpServerWatcher implements McpServerProvider {
if (this.stopped || this.contextCacheRecoveryTimer) return
this.contextCacheRecoveryTimer = setTimeout(() => {
this.contextCacheRecoveryTimer = null
- void this.recoverContextInventoryAndWatch().then(recovered => {
- if (!recovered) this.scheduleContextCacheRecovery()
- })
+ this.attemptContextCacheRecovery()
}, INVENTORY_CACHE_RECOVERY_RETRY_MS)
}
@@ -1616,6 +1675,7 @@ export class McpServerWatcher implements McpServerProvider {
}
// Authority is known ONLY after the WATCH is installed.
this.hostCacheSynced = true
+ this.lastHostWatchRecoveryAt = Date.now()
hostWatchRecoverySeconds.observe(
{ phase: 'watch', outcome: 'success' },
(Date.now() - watchStartedAt) / 1000
@@ -3667,7 +3727,7 @@ export class McpServerWatcher implements McpServerProvider {
console.error('[K8s] McpServer watch error:', err)
}
console.log('[K8s] McpServer watch ended; recovering authoritative inventory...')
- this.scheduleMcpServerCacheRecovery()
+ this.attemptMcpServerCacheRecovery()
}
const request = await this.watch.watch(path, { resourceVersion }, watchCallback, doneCallback)
@@ -3920,7 +3980,7 @@ export class McpServerWatcher implements McpServerProvider {
console.error('[K8s] Context watch error:', err)
}
console.log('[K8s] Context watch ended; recovering authoritative inventory...')
- this.scheduleContextCacheRecovery()
+ this.attemptContextCacheRecovery()
}
const request = await this.watch.watch(path, { resourceVersion }, watchCallback, doneCallback)
@@ -4019,7 +4079,7 @@ export class McpServerWatcher implements McpServerProvider {
console.error('[K8s] Host watch error:', err)
}
console.log('[K8s] Host watch ended; rebuilding the Host snapshot before watch recovery')
- this.scheduleHostCacheRecovery()
+ this.attemptHostCacheRecovery()
}
const request = await this.watch.watch(path, { resourceVersion }, watchCallback, doneCallback)