Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript

fix(mcp-host): reconnect cr watches after a clean stream close

Andy Paviaf4e8559e0183
diff --git a/mcp-host/package-lock.json b/mcp-host/package-lock.json
index 7de42f4ec..db06dfb96 100644
--- a/mcp-host/package-lock.json
+++ b/mcp-host/package-lock.json
@@ -1,12 +1,12 @@
{
"name": "clerum-mcp-host",
- "version": "0.1.263",
+ "version": "0.1.264",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "clerum-mcp-host",
- "version": "0.1.263",
+ "version": "0.1.264",
"license": "SEE LICENSE IN LICENSE",
"dependencies": {
"@anthropic-ai/sdk": "^0.32.0",
diff --git a/mcp-host/package.json b/mcp-host/package.json
index c7b2a90dd..ec90473ca 100644
--- a/mcp-host/package.json
+++ b/mcp-host/package.json
@@ -1,6 +1,6 @@
{
"name": "clerum-mcp-host",
- "version": "0.1.263",
+ "version": "0.1.264",
"description": "MCP Host - reads Host CRD and provides LLM access via OpenAI/Claude",
"main": "dist/main.js",
"engines": {
diff --git a/mcp-host/src/__tests__/watchReconnector.test.ts b/mcp-host/src/__tests__/watchReconnector.test.ts
new file mode 100644
index 000000000..ad3930fd4
--- /dev/null
+++ b/mcp-host/src/__tests__/watchReconnector.test.ts
@@ -0,0 +1,95 @@
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
+import { WatchReconnector } from '../k8sClient.js'
+
+describe('WatchReconnector', () => {
+ beforeEach(() => {
+ vi.useFakeTimers()
+ })
+
+ afterEach(() => {
+ vi.useRealTimers()
+ vi.restoreAllMocks()
+ })
+
+ // The regression this class exists for: @kubernetes/client-node reports the
+ // apiserver's routine watch timeout as done(null), NOT an error. Reconnecting
+ // only on a truthy error left the watcher dead for the life of the pod.
+ it('reconnects after a close that reported no error', async () => {
+ const r = new WatchReconnector()
+ const restart = vi.fn().mockResolvedValue(undefined)
+ r.begin()
+ r.connected()
+
+ r.schedule('watch closed', restart)
+ expect(restart).not.toHaveBeenCalled()
+
+ await vi.advanceTimersByTimeAsync(1_000)
+ expect(restart).toHaveBeenCalledTimes(1)
+ })
+
+ it('backs off exponentially up to the 30s ceiling while reconnects fail', async () => {
+ const r = new WatchReconnector()
+ const restart = vi.fn().mockRejectedValue(new Error('apiserver unreachable'))
+ r.begin()
+
+ r.schedule('watch closed', restart)
+ for (const delay of [1_000, 2_000, 4_000, 8_000, 16_000, 32_000, 32_000]) {
+ await vi.advanceTimersByTimeAsync(delay)
+ }
+
+ // 1s, 2s, 4s, 8s, 16s, then capped at 30s — never a hot loop.
+ expect(restart).toHaveBeenCalledTimes(7)
+ })
+
+ it('resets the backoff once a watch is re-established', async () => {
+ const r = new WatchReconnector()
+ const restart = vi.fn().mockResolvedValue(undefined)
+ r.begin()
+
+ r.schedule('watch closed', restart)
+ await vi.advanceTimersByTimeAsync(1_000)
+ r.schedule('watch closed', restart)
+ await vi.advanceTimersByTimeAsync(2_000)
+ expect(restart).toHaveBeenCalledTimes(2)
+
+ // A successful (re)connection puts the next close back on the 1s floor.
+ r.connected()
+ r.schedule('watch closed', restart)
+ await vi.advanceTimersByTimeAsync(1_000)
+ expect(restart).toHaveBeenCalledTimes(3)
+ })
+
+ it('does not reconnect after cancel()', async () => {
+ const r = new WatchReconnector()
+ const restart = vi.fn().mockResolvedValue(undefined)
+ r.begin()
+
+ r.schedule('watch closed', restart)
+ r.cancel()
+ await vi.advanceTimersByTimeAsync(60_000)
+ expect(restart).not.toHaveBeenCalled()
+ })
+
+ it('ignores a schedule() issued after cancel()', async () => {
+ const r = new WatchReconnector()
+ const restart = vi.fn().mockResolvedValue(undefined)
+ r.begin()
+ r.cancel()
+
+ r.schedule('watch closed', restart)
+ await vi.advanceTimersByTimeAsync(60_000)
+ expect(restart).not.toHaveBeenCalled()
+ })
+
+ it('collapses concurrent schedules into one pending reconnect', async () => {
+ const r = new WatchReconnector()
+ const restart = vi.fn().mockResolvedValue(undefined)
+ r.begin()
+
+ r.schedule('watch closed', restart)
+ r.schedule('watch closed', restart)
+ r.schedule('watch closed', restart)
+ await vi.advanceTimersByTimeAsync(1_000)
+ expect(restart).toHaveBeenCalledTimes(1)
+ })
+})
diff --git a/mcp-host/src/k8sClient.ts b/mcp-host/src/k8sClient.ts
index 280e140b0..ea0281478 100644
--- a/mcp-host/src/k8sClient.ts
+++ b/mcp-host/src/k8sClient.ts
@@ -109,6 +109,66 @@ export async function getHost(name: string): Promise<HostCRD | null> {
}
}
+/** Reconnect backoff bounds for the CR watchers. */
+const WATCH_RECONNECT_MIN_MS = 1_000
+const WATCH_RECONNECT_MAX_MS = 30_000
+
+/**
+ * Reconnect bookkeeping shared by the CR watchers below.
+ *
+ * The apiserver ends a watch NORMALLY on its own timeout (`--min-request-timeout`,
+ * randomized, ~30-60 min), and `@kubernetes/client-node` reports that as
+ * `done(null)` — not an error. Reconnecting only on a truthy error therefore left
+ * a watcher permanently deaf after the first routine close: the CR kept changing
+ * and the pod never heard about it again until it was rolled. Every reconnect
+ * re-lists, so the first events after one are ADDED for the current objects —
+ * which re-syncs whatever was missed while disconnected.
+ */
+export class WatchReconnector {
+ private timer: ReturnType<typeof setTimeout> | null = null
+ private attempt = 0
+ private stopped = false
+
+ /** Called from start(): a new connection attempt is being made. */
+ begin(): void {
+ this.stopped = false
+ }
+
+ /** Called once a watch is established — the next close starts from the floor. */
+ connected(): void {
+ this.attempt = 0
+ }
+
+ /** Called from stop(): no further reconnects until begin(). */
+ cancel(): void {
+ this.stopped = true
+ if (this.timer) {
+ clearTimeout(this.timer)
+ this.timer = null
+ }
+ }
+
+ /**
+ * Schedule a reconnect unless stopped or one is already pending. Backoff runs
+ * 1s → 30s so a routine close reconnects almost immediately while a genuinely
+ * unreachable apiserver is not hot-looped.
+ */
+ schedule(label: string, restart: () => Promise<void>): void {
+ if (this.stopped || this.timer) return
+ const delayMs = Math.min(WATCH_RECONNECT_MAX_MS, WATCH_RECONNECT_MIN_MS * 2 ** this.attempt)
+ this.attempt += 1
+ console.log(`[K8s] ${label}; reconnecting in ${delayMs}ms`)
+ this.timer = setTimeout(() => {
+ this.timer = null
+ if (this.stopped) return
+ restart().catch(err => {
+ console.error(`[K8s] ${label}: reconnect failed:`, err)
+ this.schedule(label, restart)
+ })
+ }, delayMs)
+ }
+}
+
/**
* Watch for changes to a specific Host CRD.
*/
@@ -116,6 +176,7 @@ export class HostWatcher {
private name: string
private watch: k8s.Watch
private watchRequest: { abort: () => void } | null = null
+ private reconnector = new WatchReconnector()
constructor(name: string) {
this.name = name
@@ -131,6 +192,7 @@ export class HostWatcher {
const path = `/apis/${GROUP}/${VERSION}/namespaces/${config.namespace}/${HOSTS_PLURAL}`
console.log(`[K8s] Starting watch on Host: ${this.name}`)
+ this.reconnector.begin()
const watchCallback = (
type: string,
@@ -153,12 +215,17 @@ export class HostWatcher {
}
}
+ // `err` is null when the stream ended normally — the apiserver's routine watch
+ // timeout. Both cases must reconnect (see WatchReconnector); treating only the
+ // error case as recoverable is what left this watcher silently dead, freezing
+ // every Host-driven update (guardrails, model, secretRef, personalization,
+ // approval, failover) until the pod was rolled.
const doneCallback = (err: Error | null) => {
- if (err) {
- console.error('[K8s] Watch error:', err)
- // Restart watch after a delay
- setTimeout(() => this.start(onChange, onDelete), 5000)
- }
+ if (err) console.error('[K8s] Watch error:', err)
+ this.watchRequest = null
+ this.reconnector.schedule(err ? 'Host watch errored' : 'Host watch closed', () =>
+ this.start(onChange, onDelete)
+ )
}
this.watchRequest = await this.watch.watch(
@@ -167,12 +234,14 @@ export class HostWatcher {
watchCallback,
doneCallback
)
+ this.reconnector.connected()
}
/**
* Stop watching.
*/
stop(): void {
+ this.reconnector.cancel()
if (this.watchRequest) {
console.log('[K8s] Stopping Host watch')
this.watchRequest.abort()
@@ -189,6 +258,7 @@ export class HostWatcher {
export class LlmHookWatcher {
private watch: k8s.Watch
private watchRequest: { abort: () => void } | null = null
+ private reconnector = new WatchReconnector()
constructor() {
this.watch = new k8s.Watch(kc)
@@ -197,6 +267,7 @@ export class LlmHookWatcher {
async start(onChange: (name: string) => void): Promise<void> {
const path = `/apis/${GROUP}/${VERSION}/namespaces/${config.llmHooksNamespace}/${LLMHOOKS_PLURAL}`
console.log(`[K8s] Starting watch on LlmHooks (namespace ${config.llmHooksNamespace})`)
+ this.reconnector.begin()
const watchCallback = (type: string, apiObj: { metadata?: { name?: string } }) => {
const name = apiObj?.metadata?.name
@@ -207,17 +278,24 @@ export class LlmHookWatcher {
}
}
+ // Same contract as the Host watch: a normal close reports `null`, and it must
+ // reconnect too. The 5-minute guardrail re-resolve backstop in main.ts hides
+ // this for hook-CR edits, but only because it polls — the watch itself was
+ // gone for the life of the pod.
const doneCallback = (err: Error | null) => {
- if (err) {
- console.error('[K8s] LlmHook watch error:', err)
- setTimeout(() => this.start(onChange), 5000)
- }
+ if (err) console.error('[K8s] LlmHook watch error:', err)
+ this.watchRequest = null
+ this.reconnector.schedule(err ? 'LlmHook watch errored' : 'LlmHook watch closed', () =>
+ this.start(onChange)
+ )
}
this.watchRequest = await this.watch.watch(path, {}, watchCallback, doneCallback)
+ this.reconnector.connected()
}
stop(): void {
+ this.reconnector.cancel()
if (this.watchRequest) {
console.log('[K8s] Stopping LlmHook watch')
this.watchRequest.abort()