Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
perf(hcc): exponential backoff with jitter on watch-recovery retry
Alfredo Lopezab47032436ea
diff --git a/host-context-controller/src/k8sClient.test.ts b/host-context-controller/src/k8sClient.test.tsindex 9e235f1cf..344765841 100644--- a/host-context-controller/src/k8sClient.test.ts+++ b/host-context-controller/src/k8sClient.test.ts@@ -9436,11 +9436,12 @@ describe('McpServerWatcher watch-close recovery latency (immediate first attempt// GKE closes long-lived watches ("Premature close") every few minutes. The// old contract armed a FIXED 5s timer before the first re-LIST, so every// isolated close cost ~5.5s of /ready 503. The hardened contract: the FIRST- // recovery attempt after a close is IMMEDIATE (microtask, no timer), the 5s- // timer survives only as the retry-after-failure pacing, and an anti-busy-loop- // FLOOR (1s) demotes a close that arrives <1s after the last successful- // recovery back onto the paced timer so a degraded apiserver is never- // hammered with back-to-back re-LISTs.+ // recovery attempt after a close is IMMEDIATE (microtask, no timer), a paced+ // timer survives only as the retry-after-failure pacing (now the jittered+ // exponential backoff: first failure retries within [500ms, 1000ms]), and an+ // anti-busy-loop FLOOR (1s) demotes a close that arrives <1s after the last+ // successful recovery back onto the paced timer so a degraded apiserver is+ // never hammered with back-to-back re-LISTs.const inventoryLanes = [{kind: 'McpServer',@@ -9583,7 +9584,7 @@ describe('McpServerWatcher watch-close recovery latency (immediate first attempt})it.each(inventoryLanes)(- 'falls back to the 5s retry timer only after a failed immediate $kind recovery attempt',+ 'falls back to the paced backoff retry timer only after a failed immediate $kind recovery attempt',async lane => {const doneCallbacks: Array<(error: Error | null) => void> = []let listCalls = 0@@ -9618,10 +9619,12 @@ describe('McpServerWatcher watch-close recovery latency (immediate first attemptexpect((watcher as any)[lane.syncedField]).toBe(false)expect((watcher as any)[lane.timerField]).not.toBeNull()- // Retry-after-failure keeps the 5s pacing: nothing before the deadline.- await vi.advanceTimersByTimeAsync(4999)+ // Retry-after-failure keeps its pacing: the first-failure backoff delay+ // is jittered within [500ms, 1000ms], so nothing may fire before the+ // 500ms floor and the retry must have fired by the 1000ms bound.+ await vi.advanceTimersByTimeAsync(499)expect(listCalls).toBe(1)- await vi.advanceTimersByTimeAsync(1)+ await vi.advanceTimersByTimeAsync(501)await flushMicrotasks(20)expect(listCalls).toBe(2)expect((watcher as any)[lane.syncedField]).toBe(true)@@ -9632,7 +9635,7 @@ describe('McpServerWatcher watch-close recovery latency (immediate first attempt})- it('falls back to the 5s retry timer only after a failed immediate Host recovery attempt', async () => {+ it('falls back to the paced backoff retry timer only after a failed immediate Host recovery attempt', async () => {const doneCallbacks: Array<(error: Error | null) => void> = []let hostListCalls = 0mocks.watch.mockImplementation(async (path, _options, _callback, done) => {@@ -9663,14 +9666,15 @@ describe('McpServerWatcher watch-close recovery latency (immediate first attemptawait flushMicrotasks(20)// The immediate attempt ran (and failed): exactly one LIST, retry armed- // by performHostInventoryRecovery's existing failure path.+ // by performHostInventoryRecovery's existing failure path. The first-+ // failure backoff delay is jittered within [500ms, 1000ms].expect(hostListCalls).toBe(1)expect((watcher as any).hostCacheSynced).toBe(false)expect((watcher as any).hostCacheRecoveryTimer).not.toBeNull()- await vi.advanceTimersByTimeAsync(4999)+ await vi.advanceTimersByTimeAsync(499)expect(hostListCalls).toBe(1)- await vi.advanceTimersByTimeAsync(1)+ await vi.advanceTimersByTimeAsync(501)await flushMicrotasks(20)expect(hostListCalls).toBe(2)expect((watcher as any).hostCacheSynced).toBe(true)@@ -9682,7 +9686,7 @@ describe('McpServerWatcher watch-close recovery latency (immediate first attempt})it.each(inventoryLanes)(- 'demotes a $kind close arriving within the 1s floor of the last recovery onto the 5s timer',+ 'demotes a $kind close arriving within the 1s floor of the last recovery onto the paced retry timer',async lane => {const doneCallbacks: Array<(error: Error | null) => void> = []let listCalls = 0@@ -9715,7 +9719,7 @@ describe('McpServerWatcher watch-close recovery latency (immediate first attempt// Burst close: the recovered watch dies again with ZERO fake-time// elapsed since the recovery that established it (< 1s floor). The- // anti-busy-loop floor must demote this attempt onto the 5s timer+ // anti-busy-loop floor must demote this attempt onto the paced timer// instead of hammering the (visibly degraded) apiserver immediately.doneCallbacks[1](Object.assign(new Error('Premature close'), { statusCode: 500 }))await flushMicrotasks(20)@@ -9723,7 +9727,7 @@ describe('McpServerWatcher watch-close recovery latency (immediate first attemptexpect((watcher as any)[lane.timerField]).not.toBeNull()expect((watcher as any)[lane.syncedField]).toBe(false)- // The paced retry recovers it at +5s.+ // The paced retry recovers it within the +5s advance.await vi.advanceTimersByTimeAsync(5000)await flushMicrotasks(20)expect(listCalls).toBe(2)@@ -9744,7 +9748,7 @@ describe('McpServerWatcher watch-close recovery latency (immediate first attempt})- it('demotes a Host close arriving within the 1s floor of the last recovery onto the 5s timer', async () => {+ it('demotes a Host close arriving within the 1s floor of the last recovery onto the paced retry timer', async () => {const doneCallbacks: Array<(error: Error | null) => void> = []let hostListCalls = 0mocks.watch.mockImplementation(async (path, _options, _callback, done) => {@@ -9774,7 +9778,7 @@ describe('McpServerWatcher watch-close recovery latency (immediate first attemptexpect(doneCallbacks).toHaveLength(2)expect((watcher as any).hostCacheSynced).toBe(true)- // Burst close within the 1s floor: demoted onto the paced 5s timer.+ // Burst close within the 1s floor: demoted onto the paced retry timer.doneCallbacks[1](Object.assign(new Error('Premature close'), { statusCode: 500 }))await flushMicrotasks(20)expect(hostListCalls).toBe(1)@@ -9799,3 +9803,327 @@ describe('McpServerWatcher watch-close recovery latency (immediate first attemptawait watcher.stop()})})++describe('McpServerWatcher watch-recovery retry backoff (exponential, jittered, reset-on-success)', () => {+ // During a long apiserver outage (GKE zonal control-plane upgrade: minutes+ // of every watch down and every re-LIST failing) a FIXED retry interval+ // hammers the recovering apiserver, and — worse — the HCC's inventory+ // streams synchronize after a simultaneous cut and hit it in phase. The+ // ecosystem standard (client-go reflector) is exponential backoff with+ // jitter. Contract under test, per hardened lane (McpServer/Context/Host):+ // - retry-after-failure delay = min(BASE * 2^(failures-1), CAP) with FULL+ // jitter (random() * computed) floored at BASE/2 so jitter can never+ // reintroduce a 0-delay busy-loop;+ // - a SUCCESSFUL recovery resets the failure counter (next failure starts+ // the ladder again at ~BASE, not where the last outage left off);+ // - an HTTP 429 carrying Retry-After overrides the computed delay+ // (respects GKE APF), clamped into [BASE/2, CAP];+ // - the FIRST attempt after a close stays IMMEDIATE (hardening intact).+ const BACKOFF_BASE_MS = 1000+ const BACKOFF_CAP_MS = 30000+ const BACKOFF_MIN_MS = BACKOFF_BASE_MS / 2++ const inventoryLanes = [+ {+ kind: 'McpServer',+ plural: 'mcpservers',+ pathSuffix: '/mcpservers',+ restartMethod: 'restartMcpServerWatch',+ startSnapshot: { resourceVersion: 'mcp-backoff-start-rv', servers: [] },+ recoveryRvPrefix: 'mcp-backoff-recovery-rv',+ timerField: 'mcpServerCacheRecoveryTimer',+ syncedField: 'mcpServerCacheSynced',+ peerSyncedField: 'contextCacheSynced',+ },+ {+ kind: 'Context',+ plural: 'contexts',+ pathSuffix: '/contexts',+ restartMethod: 'restartContextWatch',+ startSnapshot: { resourceVersion: 'context-backoff-start-rv', contexts: [] },+ recoveryRvPrefix: 'context-backoff-recovery-rv',+ timerField: 'contextCacheRecoveryTimer',+ syncedField: 'contextCacheSynced',+ peerSyncedField: 'mcpServerCacheSynced',+ },+ ] as const++ let randomSpy: ReturnType<typeof vi.spyOn>+ let errorSpy: ReturnType<typeof vi.spyOn>+ let logSpy: ReturnType<typeof vi.spyOn>++ beforeEach(() => {+ vi.clearAllMocks()+ vi.useFakeTimers()+ mocks.watch.mockReset().mockResolvedValue({ abort: vi.fn() })+ mocks.listNamespacedCustomObject.mockReset().mockResolvedValue({ items: [] })+ randomSpy = vi.spyOn(Math, 'random')+ errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {})+ logSpy = vi.spyOn(console, 'log').mockImplementation(() => {})+ })++ afterEach(() => {+ randomSpy.mockRestore()+ errorSpy.mockRestore()+ logSpy.mockRestore()+ vi.useRealTimers()+ })++ /** Install the lane's watch and mock its LIST with a scriptable outcome. */+ async function setUpInventoryLane(+ lane: (typeof inventoryLanes)[number],+ listOutcome: (call: number) => 'fail' | 'fail-429' | { retryAfter: string } | 'succeed'+ ) {+ const doneCallbacks: Array<(error: Error | null) => void> = []+ let listCalls = 0+ mocks.watch.mockImplementation(async (path, _options, _callback, done) => {+ if (path.endsWith(lane.pathSuffix)) doneCallbacks.push(done)+ return { abort: vi.fn() }+ })+ mocks.listNamespacedCustomObject.mockImplementation(async ({ plural }: { plural: string }) => {+ if (plural !== lane.plural) return { items: [] }+ listCalls += 1+ const outcome = listOutcome(listCalls)+ if (outcome === 'fail') {+ throw Object.assign(new Error(`${lane.plural} re-LIST unavailable`), { code: 500 })+ }+ if (outcome === 'fail-429' || typeof outcome === 'object') {+ const retryAfter = outcome === 'fail-429' ? '7' : outcome.retryAfter+ throw Object.assign(new Error(`${lane.plural} re-LIST throttled`), {+ code: 429,+ headers: { 'retry-after': retryAfter },+ })+ }+ return {+ metadata: { resourceVersion: `${lane.recoveryRvPrefix}-${listCalls}` },+ items: [],+ }+ })+ const watcher = new McpServerWatcher()+ ;(watcher as any)[lane.peerSyncedField] = true+ await (watcher as any)[lane.restartMethod](lane.startSnapshot)+ return { watcher, doneCallbacks, getListCalls: () => listCalls }+ }++ /** Host analogue of setUpInventoryLane. */+ async function setUpHostLane(+ listOutcome: (call: number) => 'fail' | { retryAfter: string } | 'succeed'+ ) {+ const doneCallbacks: Array<(error: Error | null) => void> = []+ let listCalls = 0+ mocks.watch.mockImplementation(async (path, _options, _callback, done) => {+ if (path.endsWith('/hosts')) doneCallbacks.push(done)+ return { abort: vi.fn() }+ })+ mocks.listNamespacedCustomObject.mockImplementation(async ({ plural }: { plural: string }) => {+ if (plural !== 'hosts') return { items: [] }+ listCalls += 1+ const outcome = listOutcome(listCalls)+ if (outcome === 'fail') {+ throw Object.assign(new Error('hosts re-LIST unavailable'), { code: 500 })+ }+ if (typeof outcome === 'object') {+ throw Object.assign(new Error('hosts re-LIST throttled'), {+ code: 429,+ headers: { 'retry-after': outcome.retryAfter },+ })+ }+ return {+ metadata: { resourceVersion: `host-backoff-recovery-rv-${listCalls}` },+ items: [],+ }+ })+ const watcher = newContextAuthoritativeWatcher()+ vi.spyOn(watcher as any, 'requestHostFleetReconcile').mockResolvedValue(undefined)+ await (watcher as any).startHostWatch('host-backoff-initial-rv')+ ;(watcher as any).hostCacheSynced = true+ return { watcher, doneCallbacks, getListCalls: () => listCalls }+ }++ const closeError = () => Object.assign(new Error('Premature close'), { statusCode: 500 })++ /**+ * Advance to just before the expected fire instant (no re-LIST may have+ * happened) and then across it (exactly one more re-LIST must happen).+ */+ async function expectRetryAt(delayMs: number, getListCalls: () => number): Promise<void> {+ const before = getListCalls()+ await vi.advanceTimersByTimeAsync(delayMs - 1)+ expect(getListCalls()).toBe(before)+ await vi.advanceTimersByTimeAsync(1)+ await flushMicrotasks(20)+ expect(getListCalls()).toBe(before + 1)+ }++ it.each(inventoryLanes)(+ 'grows the $kind retry delay exponentially (1s, 2s, 4s, ... capped at 30s) under consecutive recovery failures',+ async lane => {+ randomSpy.mockReturnValue(1) // upper jitter bound: delay == computed backoff+ const { watcher, doneCallbacks, getListCalls } = await setUpInventoryLane(lane, () => 'fail')++ // First attempt after the close is immediate (failure #1)...+ doneCallbacks[0](closeError())+ await flushMicrotasks(20)+ expect(getListCalls()).toBe(1)+ expect((watcher as any)[lane.timerField]).not.toBeNull()++ // ...then each retry-after-failure doubles, bounded by the 30s cap.+ for (const expectedDelay of [1000, 2000, 4000, 8000, 16000, 30000, 30000]) {+ await expectRetryAt(expectedDelay, getListCalls)+ }+ await watcher.stop()+ }+ )++ it('grows the Host retry delay exponentially (1s, 2s, 4s, ... capped at 30s) under consecutive recovery failures', async () => {+ randomSpy.mockReturnValue(1)+ const { watcher, doneCallbacks, getListCalls } = await setUpHostLane(() => 'fail')++ doneCallbacks[0](closeError())+ await flushMicrotasks(20)+ expect(getListCalls()).toBe(1)+ expect((watcher as any).hostCacheRecoveryTimer).not.toBeNull()++ for (const expectedDelay of [1000, 2000, 4000, 8000, 16000, 30000, 30000]) {+ await expectRetryAt(expectedDelay, getListCalls)+ }+ await watcher.stop()+ })++ it.each(inventoryLanes)(+ 'resets the $kind backoff to the base delay after a successful recovery',+ async lane => {+ randomSpy.mockReturnValue(1)+ const { watcher, doneCallbacks, getListCalls } = await setUpInventoryLane(lane, call =>+ call === 3 || call === 5 ? 'succeed' : 'fail'+ )++ // Two consecutive failures escalate the ladder: immediate (#1), +1s (#2).+ doneCallbacks[0](closeError())+ await flushMicrotasks(20)+ expect(getListCalls()).toBe(1)+ await expectRetryAt(1000, getListCalls)+ // Third attempt (+2s) SUCCEEDS and must reset the failure counter.+ await expectRetryAt(2000, getListCalls)+ expect((watcher as any)[lane.syncedField]).toBe(true)+ expect(doneCallbacks).toHaveLength(2)++ // A later close (beyond the 1s floor) fails immediately again (#4). The+ // escalation must restart at the BASE delay (1s), not continue at 4s.+ await vi.advanceTimersByTimeAsync(1500)+ doneCallbacks[1](closeError())+ await flushMicrotasks(20)+ expect(getListCalls()).toBe(4)+ await expectRetryAt(1000, getListCalls)+ expect((watcher as any)[lane.syncedField]).toBe(true)+ await watcher.stop()+ }+ )++ it('resets the Host backoff to the base delay after a successful recovery', async () => {+ randomSpy.mockReturnValue(1)+ const { watcher, doneCallbacks, getListCalls } = await setUpHostLane(call =>+ call === 3 || call === 5 ? 'succeed' : 'fail'+ )++ doneCallbacks[0](closeError())+ await flushMicrotasks(20)+ expect(getListCalls()).toBe(1)+ await expectRetryAt(1000, getListCalls)+ await expectRetryAt(2000, getListCalls)+ expect((watcher as any).hostCacheSynced).toBe(true)+ expect(doneCallbacks).toHaveLength(2)++ await vi.advanceTimersByTimeAsync(1500)+ doneCallbacks[1](closeError())+ await flushMicrotasks(20)+ expect(getListCalls()).toBe(4)+ await expectRetryAt(1000, getListCalls)+ expect((watcher as any).hostCacheSynced).toBe(true)+ await watcher.stop()+ })++ it.each(inventoryLanes)(+ 'floors the jittered $kind delay at 500ms (random()=0 must never produce a 0-delay busy-loop)',+ async lane => {+ randomSpy.mockReturnValue(0) // lower jitter bound+ const { watcher, doneCallbacks, getListCalls } = await setUpInventoryLane(lane, call =>+ call === 1 ? 'fail' : 'succeed'+ )++ doneCallbacks[0](closeError())+ await flushMicrotasks(20)+ expect(getListCalls()).toBe(1)+ // Full jitter with random()=0 computes 0; the floor must hold it at+ // BASE/2 = 500ms — no instant (busy-loop) retry.+ await expectRetryAt(BACKOFF_MIN_MS, getListCalls)+ expect((watcher as any)[lane.syncedField]).toBe(true)+ await watcher.stop()+ }+ )++ it('floors the jittered Host delay at 500ms (random()=0 must never produce a 0-delay busy-loop)', async () => {+ randomSpy.mockReturnValue(0)+ const { watcher, doneCallbacks, getListCalls } = await setUpHostLane(call =>+ call === 1 ? 'fail' : 'succeed'+ )++ doneCallbacks[0](closeError())+ await flushMicrotasks(20)+ expect(getListCalls()).toBe(1)+ await expectRetryAt(BACKOFF_MIN_MS, getListCalls)+ expect((watcher as any).hostCacheSynced).toBe(true)+ await watcher.stop()+ })++ it.each(inventoryLanes)(+ 'honors a 429 Retry-After for the next $kind retry instead of the computed backoff',+ async lane => {+ randomSpy.mockReturnValue(1)+ const { watcher, doneCallbacks, getListCalls } = await setUpInventoryLane(lane, call =>+ call === 1 ? 'fail-429' : 'succeed'+ )++ doneCallbacks[0](closeError())+ await flushMicrotasks(20)+ expect(getListCalls()).toBe(1)+ // APF said Retry-After: 7 — the next retry must land at 7000ms, not at+ // the 1000ms first-failure backoff.+ await expectRetryAt(7000, getListCalls)+ expect((watcher as any)[lane.syncedField]).toBe(true)+ await watcher.stop()+ }+ )++ it('honors a 429 Retry-After for the next Host retry, clamped to the 30s cap', async () => {+ randomSpy.mockReturnValue(1)+ const { watcher, doneCallbacks, getListCalls } = await setUpHostLane(call =>+ call === 1 ? { retryAfter: '3600' } : 'succeed'+ )++ doneCallbacks[0](closeError())+ await flushMicrotasks(20)+ expect(getListCalls()).toBe(1)+ // Retry-After: 3600 (an hour) must not stall recovery: clamp to the cap.+ await expectRetryAt(BACKOFF_CAP_MS, getListCalls)+ expect((watcher as any).hostCacheSynced).toBe(true)+ await watcher.stop()+ })++ it.each(inventoryLanes)(+ 'keeps the first $kind attempt after a spaced close immediate (hardening not regressed by backoff)',+ async lane => {+ randomSpy.mockReturnValue(0) // worst-case jitter must not delay the immediate path+ const { watcher, doneCallbacks, getListCalls } = await setUpInventoryLane(lane, () => 'succeed')++ await vi.advanceTimersByTimeAsync(1500)+ doneCallbacks[0](closeError())+ await flushMicrotasks(20)+ // Immediate microtask re-LIST, no timer armed.+ expect(getListCalls()).toBe(1)+ expect((watcher as any)[lane.timerField]).toBeNull()+ expect((watcher as any)[lane.syncedField]).toBe(true)+ await watcher.stop()+ }+ )+})diff --git a/host-context-controller/src/k8sClient.ts b/host-context-controller/src/k8sClient.tsindex a0c0da610..465f120a8 100644--- a/host-context-controller/src/k8sClient.ts+++ b/host-context-controller/src/k8sClient.ts@@ -109,10 +109,21 @@ const PLURAL_COMMUNICATIONCHANNELS = 'communicationchannels'// sweep made provider.start() fail and Kubernetes restarted HCC. These retries// retain that convergence guarantee now that the sweeps run in the background.const INITIAL_CONVERGENCE_RETRY_DELAYS_MS = [5000, 15000, 30000, 60000, 300000]-const INVENTORY_CACHE_RECOVERY_RETRY_MS = 5000const COMMUNICATION_CHANNEL_CACHE_RECOVERY_RETRY_MS = 5000-const HOST_CACHE_RECOVERY_RETRY_MS = 5000+// Retry-after-failure pacing for the three hardened inventory lanes+// (McpServer/Context/Host) is exponential with full jitter, mirroring the+// client-go reflector standard. During a long apiserver outage (a GKE zonal+// control-plane upgrade: minutes of every watch down and every re-LIST+// failing) a fixed interval hammers the recovering apiserver, and — worse —+// the HCC's ~3-6 streams synchronize after a simultaneous cut and hit it in+// phase. delay = min(BASE * 2^(failures-1), CAP), then FULL jitter+// (random() * delay; AWS-style full jitter maximizes de-correlation across+// the streams, which is exactly the goal) floored at MIN so jitter can never+// reintroduce a 0-delay busy-loop. A successful recovery resets the ladder.+const WATCH_RECOVERY_BACKOFF_BASE_MS = 1000+const WATCH_RECOVERY_BACKOFF_CAP_MS = 30000+const WATCH_RECOVERY_BACKOFF_MIN_MS = WATCH_RECOVERY_BACKOFF_BASE_MS / 2// Anti-busy-loop floor for IMMEDIATE watch-close recovery. The first re-LIST// after an isolated watch close runs immediately (sub-second readiness blip// instead of the old fixed ~5.5s), but a close arriving within this floor of@@ -125,6 +136,52 @@ const HOST_WATCH_RECONCILE_RETRY_DELAYS_MS = [5000, 15000, 30000]// (§10.2 step 7) rather than waiting for the background fleet pass.const WAKE_REQUESTED_ANNOTATION = 'clerum.io/wake-requested'+/**+ * Next retry-after-failure delay for a hardened inventory lane. `retryAfterMs`+ * (from an HTTP 429 Retry-After, i.e. GKE API Priority & Fairness telling us+ * exactly when to come back) overrides the computed backoff, clamped into+ * [MIN, CAP] so a throttled lane can neither busy-loop nor stall for an hour+ * on a pathological header while readiness is failing closed.+ */+function computeWatchRecoveryRetryDelayMs(+ consecutiveFailures: number,+ retryAfterMs?: number+): number {+ if (retryAfterMs !== undefined) {+ return Math.min(+ Math.max(retryAfterMs, WATCH_RECOVERY_BACKOFF_MIN_MS),+ WATCH_RECOVERY_BACKOFF_CAP_MS+ )+ }+ const attempt = Math.max(1, consecutiveFailures)+ const computed = Math.min(+ WATCH_RECOVERY_BACKOFF_BASE_MS * 2 ** (attempt - 1),+ WATCH_RECOVERY_BACKOFF_CAP_MS+ )+ return Math.max(WATCH_RECOVERY_BACKOFF_MIN_MS, Math.random() * computed)+}++/**+ * Extract Retry-After (milliseconds) from an HTTP 429 error. The+ * @kubernetes/client-node ApiException carries the response headers as a+ * plain string map; header-name casing is transport-dependent, so the lookup+ * is case-insensitive. Only the delta-seconds form is honored — the HTTP-date+ * form is not parsed (APF emits delta-seconds; a date would need clock-skew+ * handling for marginal value) and falls through to the computed backoff.+ */+function getRetryAfterMs(error: unknown): number | undefined {+ if (getErrorCode(error) !== 429) return undefined+ const headers = (error as { headers?: unknown }).headers+ if (headers === null || typeof headers !== 'object') return undefined+ for (const [name, value] of Object.entries(headers)) {+ if (name.toLowerCase() !== 'retry-after') continue+ const seconds = Number(value)+ if (Number.isFinite(seconds) && seconds >= 0) return seconds * 1000+ return undefined+ }+ return undefined+}+type CommunicationChannelSnapshot = {channels: CommunicationChannelCRD[]resourceVersion?: string@@ -675,6 +732,17 @@ export class McpServerWatcher implements McpServerProvider {private lastMcpServerWatchRecoveryAt = Number.NEGATIVE_INFINITYprivate lastContextWatchRecoveryAt = Number.NEGATIVE_INFINITYprivate lastHostWatchRecoveryAt = Number.NEGATIVE_INFINITY+ // Consecutive recovery FAILURES per hardened lane, driving the exponential+ // backoff ladder in schedule*CacheRecovery. Reset to 0 by every successful+ // recovery so an isolated later close starts again at the base delay. The+ // paired retryAfterMs (captured from an HTTP 429 Retry-After) overrides the+ // computed delay for exactly the next scheduled retry.+ private mcpServerWatchRecoveryFailures = 0+ private mcpServerWatchRecoveryRetryAfterMs: number | undefined+ private contextWatchRecoveryFailures = 0+ private contextWatchRecoveryRetryAfterMs: number | undefined+ private hostWatchRecoveryFailures = 0+ private hostWatchRecoveryRetryAfterMs: number | undefined// A watch callback can still settle after its stream reports completion.// Host reconciles change the mcp-host pod template, so stale callbacks must// never replay an older Host spec after a replacement watch is active.@@ -1338,6 +1406,8 @@ export class McpServerWatcher implements McpServerProvider {await this.restartMcpServerWatch(snapshot)if (this.stopped) return falsethis.lastMcpServerWatchRecoveryAt = Date.now()+ this.mcpServerWatchRecoveryFailures = 0+ this.mcpServerWatchRecoveryRetryAfterMs = undefinedthis.changeCallback?.()if (this.contextCacheSynced) {void this.runInitialNetworkPolicyConvergence()@@ -1345,6 +1415,8 @@ export class McpServerWatcher implements McpServerProvider {return true} catch (error) {this.mcpServerCacheSynced = false+ this.mcpServerWatchRecoveryFailures += 1+ this.mcpServerWatchRecoveryRetryAfterMs = getRetryAfterMs(error)console.error('[K8s] McpServer cache recovery failed:', error)return false}@@ -1370,6 +1442,8 @@ export class McpServerWatcher implements McpServerProvider {await this.restartContextWatch(snapshot)if (this.stopped) return falsethis.lastContextWatchRecoveryAt = Date.now()+ this.contextWatchRecoveryFailures = 0+ this.contextWatchRecoveryRetryAfterMs = undefinedthis.hostFleetScheduler.bumpInputRevision()void this.runInitialNetworkPolicyConvergence()void this.runInitialSharedFileSystemConvergence()@@ -1387,6 +1461,8 @@ export class McpServerWatcher implements McpServerProvider {return true} catch (error) {this.contextCacheSynced = false+ this.contextWatchRecoveryFailures += 1+ this.contextWatchRecoveryRetryAfterMs = getRetryAfterMs(error)console.error('[K8s] Context cache recovery failed:', error)return false}@@ -1402,12 +1478,12 @@ export class McpServerWatcher implements McpServerProvider {}/**- * One recovery attempt NOW (immediate re-LIST on watch close), with the 5s- * timer kept purely as retry-after-failure pacing. The anti-busy-loop floor- * demotes a close that arrives within WATCH_CLOSE_RECOVERY_FLOOR_MS of the- * last successful recovery back onto the paced timer. Dedup against- * concurrent attempts lives in recoverMcpServerInventoryAndWatch- * (mcpServerCacheRecoveryInFlight).+ * One recovery attempt NOW (immediate re-LIST on watch close), with the+ * jittered exponential-backoff timer kept purely as retry-after-failure+ * pacing. The anti-busy-loop floor demotes a close that arrives within+ * WATCH_CLOSE_RECOVERY_FLOOR_MS of the last successful recovery back onto+ * the paced timer. Dedup against concurrent attempts lives in+ * recoverMcpServerInventoryAndWatch (mcpServerCacheRecoveryInFlight).*/private attemptMcpServerCacheRecovery(): void {if (this.stopped) return@@ -1450,18 +1526,30 @@ export class McpServerWatcher implements McpServerProvider {private scheduleMcpServerCacheRecovery(): void {if (this.stopped || this.mcpServerCacheRecoveryTimer) return+ const delayMs = computeWatchRecoveryRetryDelayMs(+ this.mcpServerWatchRecoveryFailures,+ this.mcpServerWatchRecoveryRetryAfterMs+ )+ // Retry-After applies to exactly the retry it throttled; the next failure+ // recaptures it (or falls back to the backoff ladder).+ this.mcpServerWatchRecoveryRetryAfterMs = undefinedthis.mcpServerCacheRecoveryTimer = setTimeout(() => {this.mcpServerCacheRecoveryTimer = nullthis.attemptMcpServerCacheRecovery()- }, INVENTORY_CACHE_RECOVERY_RETRY_MS)+ }, delayMs)}private scheduleContextCacheRecovery(): void {if (this.stopped || this.contextCacheRecoveryTimer) return+ const delayMs = computeWatchRecoveryRetryDelayMs(+ this.contextWatchRecoveryFailures,+ this.contextWatchRecoveryRetryAfterMs+ )+ this.contextWatchRecoveryRetryAfterMs = undefinedthis.contextCacheRecoveryTimer = setTimeout(() => {this.contextCacheRecoveryTimer = nullthis.attemptContextCacheRecovery()- }, INVENTORY_CACHE_RECOVERY_RETRY_MS)+ }, delayMs)}private installCommunicationChannelSnapshot(snapshot: CommunicationChannelSnapshot): void {@@ -1676,6 +1764,8 @@ export class McpServerWatcher implements McpServerProvider {// Authority is known ONLY after the WATCH is installed.this.hostCacheSynced = truethis.lastHostWatchRecoveryAt = Date.now()+ this.hostWatchRecoveryFailures = 0+ this.hostWatchRecoveryRetryAfterMs = undefinedhostWatchRecoverySeconds.observe({ phase: 'watch', outcome: 'success' },(Date.now() - watchStartedAt) / 1000@@ -1746,6 +1836,8 @@ export class McpServerWatcher implements McpServerProvider {(Date.now() - startedAt) / 1000)if (!this.stopped) {+ this.hostWatchRecoveryFailures += 1+ this.hostWatchRecoveryRetryAfterMs = getRetryAfterMs(error)console.error('[K8s] Host watch recovery failed:', error)this.scheduleHostCacheRecovery(request)}@@ -1864,6 +1956,11 @@ export class McpServerWatcher implements McpServerProvider {this.hostCacheRecoveryIntent = { ...requested }}if (this.hostCacheRecoveryTimer) return+ const delayMs = computeWatchRecoveryRetryDelayMs(+ this.hostWatchRecoveryFailures,+ this.hostWatchRecoveryRetryAfterMs+ )+ this.hostWatchRecoveryRetryAfterMs = undefinedthis.hostCacheRecoveryTimer = setTimeout(() => {this.hostCacheRecoveryTimer = nullconst request = this.hostCacheRecoveryIntent ?? requested@@ -1875,7 +1972,7 @@ export class McpServerWatcher implements McpServerProvider {request.ccLifecycleGeneration,request.cause).catch(() => undefined)- }, HOST_CACHE_RECOVERY_RETRY_MS)+ }, delayMs)}private isCurrentHostWatchEvent(