Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
fix(hcc): retry only failed McpServers instead of the whole fleet
Jose Gabriel Ramirez Encinase3e322531d5b
diff --git a/host-context-controller/package-lock.json b/host-context-controller/package-lock.jsonindex 29adc5140..5e911645c 100644--- a/host-context-controller/package-lock.json+++ b/host-context-controller/package-lock.json@@ -1,12 +1,12 @@{"name": "clerum-host-context-controller",- "version": "0.1.321",+ "version": "0.1.322","lockfileVersion": 3,"requires": true,"packages": {"": {"name": "clerum-host-context-controller",- "version": "0.1.321",+ "version": "0.1.322","license": "SEE LICENSE IN LICENSE","dependencies": {"@clerum/codex-catalog-projection": "file:../packages/codex-catalog-projection",diff --git a/host-context-controller/package.json b/host-context-controller/package.jsonindex c88b6747c..3d9e0bf25 100644--- a/host-context-controller/package.json+++ b/host-context-controller/package.json@@ -1,6 +1,6 @@{"name": "clerum-host-context-controller",- "version": "0.1.321",+ "version": "0.1.322","description": "Host Context Controller - provides McpServer CRDs to mcp-host via REST API","main": "dist/main.js","engines": {diff --git a/host-context-controller/src/k8sClient.runtimeRecovery.integration.test.ts b/host-context-controller/src/k8sClient.runtimeRecovery.integration.test.tsindex 3664e368a..7bedb4a0c 100644--- a/host-context-controller/src/k8sClient.runtimeRecovery.integration.test.ts+++ b/host-context-controller/src/k8sClient.runtimeRecovery.integration.test.ts@@ -20,10 +20,12 @@ const fixture = vi.hoisted(() => ({revision: 0,rolloutReady: true,failDeployment: false,+ failDeploymentName: undefined as string | undefined,serverExists: true,readInput: undefined as undefined | (() => Promise<unknown>),failPublication: undefined as undefined | ((body: any[]) => boolean),server: undefined as any,+ peers: [] as any[],readService: undefined as undefined | (() => Promise<void>),}))@@ -55,7 +57,10 @@ vi.mock('@kubernetes/client-node', async importOriginal => {return structuredClone(object)}const save = (kind: string, store: Map<string, any>, namespace: string, body: any) => {- if (kind === 'Deployment' && fixture.failDeployment)+ if (+ kind === 'Deployment' &&+ (fixture.failDeployment || fixture.failDeploymentName === body.metadata.name)+ )throw new Error('fixture deployment write unavailable')const object = structuredClone(body)object.metadata = {@@ -110,32 +115,35 @@ vi.mock('@kubernetes/client-node', async importOriginal => {replaceNamespacedDeployment: async ({ namespace, body }: any) =>save('Deployment', fixture.deployments, namespace, body),}+ const servers = () => [...(fixture.serverExists ? [fixture.server] : []), ...fixture.peers]+ const serverNamed = (name: string) => {+ const server = servers().find(candidate => candidate.metadata.name === name)+ if (!server) throw absent()+ return server+ }const custom = {listNamespacedCustomObject: async ({ plural }: any) => ({metadata: { resourceVersion: String(++fixture.revision) },- items:- plural === 'mcpservers' && fixture.serverExists ? [structuredClone(fixture.server)] : [],+ items: plural === 'mcpservers' ? servers().map(server => structuredClone(server)) : [],}),- getNamespacedCustomObject: async () => {- if (!fixture.serverExists) throw absent()- return structuredClone(fixture.server)- },- getNamespacedCustomObjectStatus: async () => structuredClone(fixture.server),- patchNamespacedCustomObjectStatus: async ({ body }: any) => {+ getNamespacedCustomObject: async ({ name }: any) => structuredClone(serverNamed(name)),+ getNamespacedCustomObjectStatus: async ({ name }: any) => structuredClone(serverNamed(name)),+ patchNamespacedCustomObjectStatus: async ({ name, body }: any) => {if (fixture.failPublication?.(body))throw Object.assign(new Error('fixture status unavailable'), { code: 503 })+ const server = serverNamed(name)for (const patch of body) {if (patch.op === 'test') {const field = patch.path.slice('/metadata/'.length)- if (fixture.server.metadata[field] !== patch.value)+ if (server.metadata[field] !== patch.value)throw Object.assign(new Error('fixture status conflict'), { code: 409 })- } else if (patch.path === '/status') fixture.server.status = structuredClone(patch.value)+ } else if (patch.path === '/status') server.status = structuredClone(patch.value)else if (patch.path === '/status/conditions')- fixture.server.status.conditions = structuredClone(patch.value)+ server.status.conditions = structuredClone(patch.value)else throw new Error(`Unexpected status patch ${patch.path}`)}- fixture.server.metadata.resourceVersion = String(++fixture.revision)- return structuredClone(fixture.server)+ server.metadata.resourceVersion = String(++fixture.revision)+ return structuredClone(server)},}return {@@ -201,6 +209,16 @@ function prepareWatcher(policyTail?: Promise<void>) {return { watcher, state }}+function recordReconciledNames(state: any): string[] {+ const names: string[] = []+ const reconcile = state.reconciler.reconcile.bind(state.reconciler)+ vi.spyOn(state.reconciler, 'reconcile').mockImplementation(async (server: any, options: any) => {+ names.push(server.name)+ return reconcile(server, options)+ })+ return names+}+async function skippedTickCount() {const metric = await registry.getSingleMetric('clerum_hcc_netpol_resync_ticks_skipped_total')!@@ -218,7 +236,9 @@ describe('A-T7 real MCP runtime repair after identical recovery', () => {fixture.revision = 0fixture.rolloutReady = truefixture.failDeployment = false+ fixture.failDeploymentName = undefinedfixture.serverExists = true+ fixture.peers = []fixture.readInput = undefinedfixture.failPublication = undefinedfixture.readService = undefined@@ -562,4 +582,80 @@ describe('A-T7 real MCP runtime repair after identical recovery', () => {expect(state.reconciler.hasPendingReconciliation()).toBe(false)expect(state.initialConvergenceRetryTimers.has('McpServer')).toBe(false)})++ describe('a persistently failing server alongside a healthy peer', () => {+ beforeEach(() => {+ const peer = structuredClone(fixture.server)+ peer.metadata = { ...peer.metadata, name: 'runtime-peer', uid: 'runtime-peer-uid' }+ fixture.peers = [peer]+ fixture.failDeploymentName = 'runtime-witness'+ })++ async function startWithScopedRetryArmed() {+ const harness = prepareWatcher()+ watcher = harness.watcher+ const { state } = harness+ await watcher.start()+ await drain(state)+ expect(fixture.deployments.has('runtime-peer')).toBe(true)+ expect(fixture.deployments.has('runtime-witness')).toBe(false)+ expect(state.reconciler.incompleteReconciliationNames()).toEqual(['runtime-witness'])+ expect(state.runtimeRepairPending).toBe(true)+ expect(state.initialConvergenceRetryTimers.has('McpServer')).toBe(true)+ return state+ }++ it('A-T7 regression: retries only the failing server, then certifies and skips identical recovery', async () => {+ const state = await startWithScopedRetryArmed()+ const reconciled = recordReconciledNames(state)+ await vi.advanceTimersByTimeAsync(5000)+ await drain(state)+ expect(reconciled).toEqual(['runtime-witness'])+ expect(state.runtimeRepairPending).toBe(true)+ expect(state.initialConvergenceRetryTimers.has('McpServer')).toBe(true)++ fixture.failDeploymentName = undefined+ await vi.advanceTimersByTimeAsync(15000)+ await drain(state)+ expect(reconciled).toEqual(['runtime-witness', 'runtime-witness'])+ expect(fixture.deployments.has('runtime-witness')).toBe(true)+ expect(state.reconciler.hasIncompleteReconciliation()).toBe(false)+ expect(state.runtimeRepairPending).toBe(false)+ expect(state.initialConvergenceRetryTimers.has('McpServer')).toBe(false)++ // Fold the status writes back through a real LIST once, then an+ // identical recovery must skip again.+ await state.recoverMcpServerInventoryAndWatch()+ await drain(state)+ const decisions = vi.spyOn(hccLogger, 'info')+ await state.recoverMcpServerInventoryAndWatch()+ await drain(state)+ expect(decisions).toHaveBeenCalledWith(+ expect.any(String),+ expect.objectContaining({ decision: 'skip', reason: 'identical-complete' })+ )+ })++ it('A-T7 control: a scoped retry widens to the fleet once inventory has moved', async () => {+ const state = await startWithScopedRetryArmed()+ state.mcpServerRetryRequest = {+ ...state.mcpServerRetryRequest,+ serverRevision: state.mcpServerRetryRequest.serverRevision - 1,+ }+ const reconciled = recordReconciledNames(state)+ await vi.advanceTimersByTimeAsync(5000)+ await drain(state)+ expect([...reconciled].sort()).toEqual(['runtime-peer', 'runtime-witness'])+ })++ it('A-T7 control: a full retry request widens an armed scoped retry', async () => {+ const state = await startWithScopedRetryArmed()+ state.scheduleInitialConvergenceRetry('McpServer')+ expect(state.mcpServerRetryRequest).toBe('full')+ const reconciled = recordReconciledNames(state)+ await vi.advanceTimersByTimeAsync(5000)+ await drain(state)+ expect([...reconciled].sort()).toEqual(['runtime-peer', 'runtime-witness'])+ })+ })})diff --git a/host-context-controller/src/k8sClient.ts b/host-context-controller/src/k8sClient.tsindex e2726a2d7..c96c3d04d 100644--- a/host-context-controller/src/k8sClient.ts+++ b/host-context-controller/src/k8sClient.ts@@ -244,6 +244,27 @@ type NetworkPolicyConvergenceCause =| 'context-change'| 'context-reconcile-failure'type CompletedInventoryRevision = { contextRevision: number; serverRevision: number }+// A scoped McpServer pass retries only names left incomplete by a pass whose+// effects all ran under this inventory revision. 'full' covers the fleet.+type McpServerConvergenceRequest =+ | 'full'+ | (CompletedInventoryRevision & { names: ReadonlySet<string> })++function widenMcpServerConvergence(+ current: McpServerConvergenceRequest | undefined,+ next: McpServerConvergenceRequest+): McpServerConvergenceRequest {+ if (current === undefined) return next+ if (+ current === 'full' ||+ next === 'full' ||+ current.contextRevision !== next.contextRevision ||+ current.serverRevision !== next.serverRevision+ ) {+ return 'full'+ }+ return { ...next, names: new Set([...current.names, ...next.names]) }+}type InitialConvergencePassResult =| 'certified'@@ -793,6 +814,8 @@ export class McpServerWatcher implements McpServerProvider {private completedRuntimeRevision: CompletedInventoryRevision | null = nullprivate networkPolicyRepairPending = falseprivate runtimeRepairPending = false+ private pendingMcpServerConvergence: McpServerConvergenceRequest | undefined+ private mcpServerRetryRequest: McpServerConvergenceRequest | undefinedprivate runtimeInventoryGeneration: number | null = nullprivate readonly pendingNetworkPolicyCauses = new Set<NetworkPolicyConvergenceCause>()private networkPolicyPassSequence = 0@@ -3105,7 +3128,13 @@ export class McpServerWatcher implements McpServerProvider {return request}- private runInitialMcpServerConvergence(): Promise<void> {+ private runInitialMcpServerConvergence(+ request: McpServerConvergenceRequest = 'full'+ ): Promise<void> {+ this.pendingMcpServerConvergence = widenMcpServerConvergence(+ this.pendingMcpServerConvergence,+ request+ )return this.runInitialConvergence('McpServer')}@@ -3161,12 +3190,18 @@ export class McpServerWatcher implements McpServerProvider {}private async runInitialMcpServerConvergenceCore(): Promise<void> {+ const request = this.pendingMcpServerConvergence ?? 'full'+ this.pendingMcpServerConvergence = undefinedconst safetyCertificate = this.currentNetworkPolicySafetyCertificate()// A retired run may settle normally without invoking work. Keep its repair// obligation until an authoritative pass actually completes every effect.this.runtimeRepairPending = trueif (safetyCertificate === null) returnlet effectsComplete = true+ // A scoped retry completes only the pass it came from. Once the inventory+ // has moved, the rest of the fleet may need work too, so run a full pass.+ const scope =+ request !== 'full' && this.matchesCurrentInventory(request) ? request.names : undefinedconst inventoryGeneration = this.mcpWatchGenerationthis.runtimeInventoryGeneration = inventoryGenerationconst inventoryAuthoritative = () =>@@ -3175,12 +3210,19 @@ export class McpServerWatcher implements McpServerProvider {try {const initialServers = [...this.servers.values()]const initialExternalEgressGates = this.externalEgressCoordinator.prepareStartupGates(- initialServers,+ scope ? initialServers.filter(server => scope.has(server.name)) : initialServers,inventoryAuthoritative)if (this.stopped) return- hccLogger.info('[K8s] Running initial McpServer background reconciliation...')+ if (scope) {+ hccLogger.info('[K8s] Running scoped McpServer background reconciliation retry...', {+ servers: [...scope].sort(),+ })+ } else {+ hccLogger.info('[K8s] Running initial McpServer background reconciliation...')+ }await this.reconciler.fullReconcile(initialServers, {+ scope,runEffect: async (serverName, work) => {const selected = this.servers.get(serverName)const laneOwner = selected ?? {@@ -3204,14 +3246,22 @@ export class McpServerWatcher implements McpServerProvider {})},})- if (- !effectsComplete ||- !inventoryAuthoritative() ||- this.reconciler.hasIncompleteReconciliation()- ) {+ if (!effectsComplete || !inventoryAuthoritative()) {this.scheduleInitialConvergenceRetry('McpServer')return}+ if (this.reconciler.hasIncompleteReconciliation()) {+ // Every offered effect ran under this certificate, so only individual+ // servers are unfinished (for example a Deployment write rejected by+ // admission). Retry just those names: one persistently failing server+ // must not re-reconcile the whole fleet at the retry cap.+ this.scheduleInitialConvergenceRetry('McpServer', {+ contextRevision: safetyCertificate.contextRevision,+ serverRevision: safetyCertificate.serverRevision,+ names: new Set(this.reconciler.incompleteReconciliationNames()),+ })+ return+ }this.completedRuntimeRevision = safetyCertificatethis.runtimeRepairPending = falseinitialConvergenceLastSuccessTimestampSeconds.set({ lane: 'McpServer' }, Date.now() / 1000)@@ -3536,10 +3586,22 @@ export class McpServerWatcher implements McpServerProvider {this.initialConvergenceRetryTimers.delete(lane)}this.initialConvergenceRetryAttempts.delete(lane)+ if (lane === 'McpServer') this.mcpServerRetryRequest = undefined}- private scheduleInitialConvergenceRetry(lane: InitialConvergenceLane): void {- if (this.stopped || this.initialConvergenceRetryTimers.has(lane)) return+ private scheduleInitialConvergenceRetry(+ lane: InitialConvergenceLane,+ mcpServerRequest: McpServerConvergenceRequest = 'full'+ ): void {+ if (this.stopped) return+ if (lane === 'McpServer') {+ // An armed retry widens to cover every request made before it fires.+ this.mcpServerRetryRequest = widenMcpServerConvergence(+ this.mcpServerRetryRequest,+ mcpServerRequest+ )+ }+ if (this.initialConvergenceRetryTimers.has(lane)) returnconst attempt = (this.initialConvergenceRetryAttempts.get(lane) ?? 0) + 1const delayMs =@@ -3556,7 +3618,9 @@ export class McpServerWatcher implements McpServerProvider {this.initialConvergenceRetryTimers.delete(lane)if (this.stopped) returnif (lane === 'McpServer') {- void this.runInitialMcpServerConvergence()+ const request = this.mcpServerRetryRequest ?? 'full'+ this.mcpServerRetryRequest = undefined+ void this.runInitialMcpServerConvergence(request)return}void this.runInitialNetworkPolicyConvergence({ cause: 'retry' })diff --git a/host-context-controller/src/reconciler.test.ts b/host-context-controller/src/reconciler.test.tsindex 7acd98b72..834926800 100644--- a/host-context-controller/src/reconciler.test.ts+++ b/host-context-controller/src/reconciler.test.ts@@ -1544,6 +1544,31 @@ describe('full reconciliation inventory authority', () => {warn.mockRestore()})+ it('limits a scoped pass to its names without treating out-of-scope servers as orphans', async () => {+ reconciler.setInventoryAuthority(() => ({ known: true, generation: 31 }))+ const failed = makeServer({ name: 'failed-server' })+ const healthy = makeServer({ name: 'healthy-server' })+ const reconcile = vi.spyOn(reconciler, 'reconcile').mockResolvedValue(undefined)+ appsApi.listNamespacedDeployment.mockResolvedValueOnce({+ items: [+ { metadata: { name: 'healthy-server', namespace: 'mcp-server' } },+ { metadata: { name: 'scoped-orphan', namespace: 'mcp-server' } },+ { metadata: { name: 'unscoped-orphan', namespace: 'mcp-server' } },+ ],+ })++ await reconciler.fullReconcile([failed, healthy], {+ scope: new Set(['failed-server', 'scoped-orphan']),+ })++ expect(reconcile.mock.calls.map(([server]) => server.name)).toEqual(['failed-server'])+ expect(appsApi.deleteNamespacedDeployment).toHaveBeenCalledOnce()+ expect(appsApi.deleteNamespacedDeployment).toHaveBeenCalledWith({+ name: 'scoped-orphan',+ namespace: 'mcp-server',+ })+ })+it('deletes an orphan when inventory authority remains known and stable', async () => {reconciler.setInventoryAuthority(() => ({ known: true, generation: 21 }))appsApi.listNamespacedDeployment.mockResolvedValueOnce({diff --git a/host-context-controller/src/reconciler.ts b/host-context-controller/src/reconciler.tsindex d47b16c29..64cd8ac76 100644--- a/host-context-controller/src/reconciler.ts+++ b/host-context-controller/src/reconciler.ts@@ -104,6 +104,10 @@ export type McpServerInventoryAuthoritySnapshot = {export type McpServerFullReconcileOptions = {runEffect?: (serverName: string, work: () => Promise<void>) => Promise<void>maxConcurrency?: number+ // Restrict create/update and orphan cleanup to these names. desiredServers+ // must still be the whole inventory so out-of-scope servers are never+ // mistaken for orphans.+ scope?: ReadonlySet<string>}export type McpServerMutationOptions = {@@ -316,6 +320,10 @@ export class McpServerReconciler {return this.pendingReconciliations.size > 0}+ incompleteReconciliationNames(): string[] {+ return [...this.pendingReconciliations.keys()]+ }+hasPendingReconciliation(): boolean {return (this.hasIncompleteReconciliation() ||@@ -2530,13 +2538,15 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolvconst runEffect =options.runEffect ?? ((_serverName: string, work: () => Promise<void>) => work())const maxConcurrency = options.maxConcurrency ?? DEFAULT_FULL_RECONCILE_MAX_CONCURRENCY+ const inScope = (name: string) => options.scope === undefined || options.scope.has(name)// Create or update desired servers independently. runEffect owns each// per-server lane, so a slow fleet member cannot become a process-wide// startup barrier. The finite worker pool admits no more than the configured// number of effects at once, while still waiting for every desired result// before reporting aggregated failures or moving to orphan cleanup.- const desiredResults = await mapSettledWithConcurrency(desiredServers, maxConcurrency, server =>+ const scopedServers = desiredServers.filter(server => inScope(server.name))+ const desiredResults = await mapSettledWithConcurrency(scopedServers, maxConcurrency, server =>runEffect(server.name, async () => {const current = this.resolveCurrentServer ? this.resolveCurrentServer(server.name) : server// Cache mutation happens synchronously before its keyed watch effect is@@ -2578,7 +2588,7 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolvif (!presentNames.has(name)) cleanupTargets.set(name, pending.namespace)}for (const [name, namespace] of cleanupTargets) {- if (!desiredNames.has(name)) {+ if (!desiredNames.has(name) && inScope(name)) {if (!presentNames.has(name)) this.pendingReconciliations.set(name, { namespace })await runEffect(name, async () => {if (!(await this.orphanDeleteAllowed(name, namespace, capturedAuthority))) {