Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
fix(mcp-host): reconnect cr watches after a clean stream close
Andy Paviaf4e8559e0183
diff --git a/mcp-host/package-lock.json b/mcp-host/package-lock.jsonindex 7de42f4ec..db06dfb96 100644--- a/mcp-host/package-lock.json+++ b/mcp-host/package-lock.json@@ -1,12 +1,12 @@{"name": "clerum-mcp-host",- "version": "0.1.263",+ "version": "0.1.264","lockfileVersion": 3,"requires": true,"packages": {"": {"name": "clerum-mcp-host",- "version": "0.1.263",+ "version": "0.1.264","license": "SEE LICENSE IN LICENSE","dependencies": {"@anthropic-ai/sdk": "^0.32.0",diff --git a/mcp-host/package.json b/mcp-host/package.jsonindex c7b2a90dd..ec90473ca 100644--- a/mcp-host/package.json+++ b/mcp-host/package.json@@ -1,6 +1,6 @@{"name": "clerum-mcp-host",- "version": "0.1.263",+ "version": "0.1.264","description": "MCP Host - reads Host CRD and provides LLM access via OpenAI/Claude","main": "dist/main.js","engines": {diff --git a/mcp-host/src/__tests__/watchReconnector.test.ts b/mcp-host/src/__tests__/watchReconnector.test.tsnew file mode 100644index 000000000..ad3930fd4--- /dev/null+++ b/mcp-host/src/__tests__/watchReconnector.test.ts@@ -0,0 +1,95 @@+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'+import { WatchReconnector } from '../k8sClient.js'++describe('WatchReconnector', () => {+ beforeEach(() => {+ vi.useFakeTimers()+ })++ afterEach(() => {+ vi.useRealTimers()+ vi.restoreAllMocks()+ })++ // The regression this class exists for: @kubernetes/client-node reports the+ // apiserver's routine watch timeout as done(null), NOT an error. Reconnecting+ // only on a truthy error left the watcher dead for the life of the pod.+ it('reconnects after a close that reported no error', async () => {+ const r = new WatchReconnector()+ const restart = vi.fn().mockResolvedValue(undefined)+ r.begin()+ r.connected()++ r.schedule('watch closed', restart)+ expect(restart).not.toHaveBeenCalled()++ await vi.advanceTimersByTimeAsync(1_000)+ expect(restart).toHaveBeenCalledTimes(1)+ })++ it('backs off exponentially up to the 30s ceiling while reconnects fail', async () => {+ const r = new WatchReconnector()+ const restart = vi.fn().mockRejectedValue(new Error('apiserver unreachable'))+ r.begin()++ r.schedule('watch closed', restart)+ for (const delay of [1_000, 2_000, 4_000, 8_000, 16_000, 32_000, 32_000]) {+ await vi.advanceTimersByTimeAsync(delay)+ }++ // 1s, 2s, 4s, 8s, 16s, then capped at 30s — never a hot loop.+ expect(restart).toHaveBeenCalledTimes(7)+ })++ it('resets the backoff once a watch is re-established', async () => {+ const r = new WatchReconnector()+ const restart = vi.fn().mockResolvedValue(undefined)+ r.begin()++ r.schedule('watch closed', restart)+ await vi.advanceTimersByTimeAsync(1_000)+ r.schedule('watch closed', restart)+ await vi.advanceTimersByTimeAsync(2_000)+ expect(restart).toHaveBeenCalledTimes(2)++ // A successful (re)connection puts the next close back on the 1s floor.+ r.connected()+ r.schedule('watch closed', restart)+ await vi.advanceTimersByTimeAsync(1_000)+ expect(restart).toHaveBeenCalledTimes(3)+ })++ it('does not reconnect after cancel()', async () => {+ const r = new WatchReconnector()+ const restart = vi.fn().mockResolvedValue(undefined)+ r.begin()++ r.schedule('watch closed', restart)+ r.cancel()+ await vi.advanceTimersByTimeAsync(60_000)+ expect(restart).not.toHaveBeenCalled()+ })++ it('ignores a schedule() issued after cancel()', async () => {+ const r = new WatchReconnector()+ const restart = vi.fn().mockResolvedValue(undefined)+ r.begin()+ r.cancel()++ r.schedule('watch closed', restart)+ await vi.advanceTimersByTimeAsync(60_000)+ expect(restart).not.toHaveBeenCalled()+ })++ it('collapses concurrent schedules into one pending reconnect', async () => {+ const r = new WatchReconnector()+ const restart = vi.fn().mockResolvedValue(undefined)+ r.begin()++ r.schedule('watch closed', restart)+ r.schedule('watch closed', restart)+ r.schedule('watch closed', restart)+ await vi.advanceTimersByTimeAsync(1_000)+ expect(restart).toHaveBeenCalledTimes(1)+ })+})diff --git a/mcp-host/src/k8sClient.ts b/mcp-host/src/k8sClient.tsindex 280e140b0..ea0281478 100644--- a/mcp-host/src/k8sClient.ts+++ b/mcp-host/src/k8sClient.ts@@ -109,6 +109,66 @@ export async function getHost(name: string): Promise<HostCRD | null> {}}+/** Reconnect backoff bounds for the CR watchers. */+const WATCH_RECONNECT_MIN_MS = 1_000+const WATCH_RECONNECT_MAX_MS = 30_000++/**+ * Reconnect bookkeeping shared by the CR watchers below.+ *+ * The apiserver ends a watch NORMALLY on its own timeout (`--min-request-timeout`,+ * randomized, ~30-60 min), and `@kubernetes/client-node` reports that as+ * `done(null)` — not an error. Reconnecting only on a truthy error therefore left+ * a watcher permanently deaf after the first routine close: the CR kept changing+ * and the pod never heard about it again until it was rolled. Every reconnect+ * re-lists, so the first events after one are ADDED for the current objects —+ * which re-syncs whatever was missed while disconnected.+ */+export class WatchReconnector {+ private timer: ReturnType<typeof setTimeout> | null = null+ private attempt = 0+ private stopped = false++ /** Called from start(): a new connection attempt is being made. */+ begin(): void {+ this.stopped = false+ }++ /** Called once a watch is established — the next close starts from the floor. */+ connected(): void {+ this.attempt = 0+ }++ /** Called from stop(): no further reconnects until begin(). */+ cancel(): void {+ this.stopped = true+ if (this.timer) {+ clearTimeout(this.timer)+ this.timer = null+ }+ }++ /**+ * Schedule a reconnect unless stopped or one is already pending. Backoff runs+ * 1s → 30s so a routine close reconnects almost immediately while a genuinely+ * unreachable apiserver is not hot-looped.+ */+ schedule(label: string, restart: () => Promise<void>): void {+ if (this.stopped || this.timer) return+ const delayMs = Math.min(WATCH_RECONNECT_MAX_MS, WATCH_RECONNECT_MIN_MS * 2 ** this.attempt)+ this.attempt += 1+ console.log(`[K8s] ${label}; reconnecting in ${delayMs}ms`)+ this.timer = setTimeout(() => {+ this.timer = null+ if (this.stopped) return+ restart().catch(err => {+ console.error(`[K8s] ${label}: reconnect failed:`, err)+ this.schedule(label, restart)+ })+ }, delayMs)+ }+}+/*** Watch for changes to a specific Host CRD.*/@@ -116,6 +176,7 @@ export class HostWatcher {private name: stringprivate watch: k8s.Watchprivate watchRequest: { abort: () => void } | null = null+ private reconnector = new WatchReconnector()constructor(name: string) {this.name = name@@ -131,6 +192,7 @@ export class HostWatcher {const path = `/apis/${GROUP}/${VERSION}/namespaces/${config.namespace}/${HOSTS_PLURAL}`console.log(`[K8s] Starting watch on Host: ${this.name}`)+ this.reconnector.begin()const watchCallback = (type: string,@@ -153,12 +215,17 @@ export class HostWatcher {}}+ // `err` is null when the stream ended normally — the apiserver's routine watch+ // timeout. Both cases must reconnect (see WatchReconnector); treating only the+ // error case as recoverable is what left this watcher silently dead, freezing+ // every Host-driven update (guardrails, model, secretRef, personalization,+ // approval, failover) until the pod was rolled.const doneCallback = (err: Error | null) => {- if (err) {- console.error('[K8s] Watch error:', err)- // Restart watch after a delay- setTimeout(() => this.start(onChange, onDelete), 5000)- }+ if (err) console.error('[K8s] Watch error:', err)+ this.watchRequest = null+ this.reconnector.schedule(err ? 'Host watch errored' : 'Host watch closed', () =>+ this.start(onChange, onDelete)+ )}this.watchRequest = await this.watch.watch(@@ -167,12 +234,14 @@ export class HostWatcher {watchCallback,doneCallback)+ this.reconnector.connected()}/*** Stop watching.*/stop(): void {+ this.reconnector.cancel()if (this.watchRequest) {console.log('[K8s] Stopping Host watch')this.watchRequest.abort()@@ -189,6 +258,7 @@ export class HostWatcher {export class LlmHookWatcher {private watch: k8s.Watchprivate watchRequest: { abort: () => void } | null = null+ private reconnector = new WatchReconnector()constructor() {this.watch = new k8s.Watch(kc)@@ -197,6 +267,7 @@ export class LlmHookWatcher {async start(onChange: (name: string) => void): Promise<void> {const path = `/apis/${GROUP}/${VERSION}/namespaces/${config.llmHooksNamespace}/${LLMHOOKS_PLURAL}`console.log(`[K8s] Starting watch on LlmHooks (namespace ${config.llmHooksNamespace})`)+ this.reconnector.begin()const watchCallback = (type: string, apiObj: { metadata?: { name?: string } }) => {const name = apiObj?.metadata?.name@@ -207,17 +278,24 @@ export class LlmHookWatcher {}}+ // Same contract as the Host watch: a normal close reports `null`, and it must+ // reconnect too. The 5-minute guardrail re-resolve backstop in main.ts hides+ // this for hook-CR edits, but only because it polls — the watch itself was+ // gone for the life of the pod.const doneCallback = (err: Error | null) => {- if (err) {- console.error('[K8s] LlmHook watch error:', err)- setTimeout(() => this.start(onChange), 5000)- }+ if (err) console.error('[K8s] LlmHook watch error:', err)+ this.watchRequest = null+ this.reconnector.schedule(err ? 'LlmHook watch errored' : 'LlmHook watch closed', () =>+ this.start(onChange)+ )}this.watchRequest = await this.watch.watch(path, {}, watchCallback, doneCallback)+ this.reconnector.connected()}stop(): void {+ this.reconnector.cancel()if (this.watchRequest) {console.log('[K8s] Stopping LlmHook watch')this.watchRequest.abort()