Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript

fix(hcc): retry only failed McpServers instead of the whole fleet

Jose Gabriel Ramirez Encinase3e322531d5b
diff --git a/host-context-controller/package-lock.json b/host-context-controller/package-lock.json
index 29adc5140..5e911645c 100644
--- a/host-context-controller/package-lock.json
+++ b/host-context-controller/package-lock.json
@@ -1,12 +1,12 @@
{
"name": "clerum-host-context-controller",
- "version": "0.1.321",
+ "version": "0.1.322",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "clerum-host-context-controller",
- "version": "0.1.321",
+ "version": "0.1.322",
"license": "SEE LICENSE IN LICENSE",
"dependencies": {
"@clerum/codex-catalog-projection": "file:../packages/codex-catalog-projection",
diff --git a/host-context-controller/package.json b/host-context-controller/package.json
index c88b6747c..3d9e0bf25 100644
--- a/host-context-controller/package.json
+++ b/host-context-controller/package.json
@@ -1,6 +1,6 @@
{
"name": "clerum-host-context-controller",
- "version": "0.1.321",
+ "version": "0.1.322",
"description": "Host Context Controller - provides McpServer CRDs to mcp-host via REST API",
"main": "dist/main.js",
"engines": {
diff --git a/host-context-controller/src/k8sClient.runtimeRecovery.integration.test.ts b/host-context-controller/src/k8sClient.runtimeRecovery.integration.test.ts
index 3664e368a..7bedb4a0c 100644
--- a/host-context-controller/src/k8sClient.runtimeRecovery.integration.test.ts
+++ b/host-context-controller/src/k8sClient.runtimeRecovery.integration.test.ts
@@ -20,10 +20,12 @@ const fixture = vi.hoisted(() => ({
revision: 0,
rolloutReady: true,
failDeployment: false,
+ failDeploymentName: undefined as string | undefined,
serverExists: true,
readInput: undefined as undefined | (() => Promise<unknown>),
failPublication: undefined as undefined | ((body: any[]) => boolean),
server: undefined as any,
+ peers: [] as any[],
readService: undefined as undefined | (() => Promise<void>),
}))
@@ -55,7 +57,10 @@ vi.mock('@kubernetes/client-node', async importOriginal => {
return structuredClone(object)
}
const save = (kind: string, store: Map<string, any>, namespace: string, body: any) => {
- if (kind === 'Deployment' && fixture.failDeployment)
+ if (
+ kind === 'Deployment' &&
+ (fixture.failDeployment || fixture.failDeploymentName === body.metadata.name)
+ )
throw new Error('fixture deployment write unavailable')
const object = structuredClone(body)
object.metadata = {
@@ -110,32 +115,35 @@ vi.mock('@kubernetes/client-node', async importOriginal => {
replaceNamespacedDeployment: async ({ namespace, body }: any) =>
save('Deployment', fixture.deployments, namespace, body),
}
+ const servers = () => [...(fixture.serverExists ? [fixture.server] : []), ...fixture.peers]
+ const serverNamed = (name: string) => {
+ const server = servers().find(candidate => candidate.metadata.name === name)
+ if (!server) throw absent()
+ return server
+ }
const custom = {
listNamespacedCustomObject: async ({ plural }: any) => ({
metadata: { resourceVersion: String(++fixture.revision) },
- items:
- plural === 'mcpservers' && fixture.serverExists ? [structuredClone(fixture.server)] : [],
+ items: plural === 'mcpservers' ? servers().map(server => structuredClone(server)) : [],
}),
- getNamespacedCustomObject: async () => {
- if (!fixture.serverExists) throw absent()
- return structuredClone(fixture.server)
- },
- getNamespacedCustomObjectStatus: async () => structuredClone(fixture.server),
- patchNamespacedCustomObjectStatus: async ({ body }: any) => {
+ getNamespacedCustomObject: async ({ name }: any) => structuredClone(serverNamed(name)),
+ getNamespacedCustomObjectStatus: async ({ name }: any) => structuredClone(serverNamed(name)),
+ patchNamespacedCustomObjectStatus: async ({ name, body }: any) => {
if (fixture.failPublication?.(body))
throw Object.assign(new Error('fixture status unavailable'), { code: 503 })
+ const server = serverNamed(name)
for (const patch of body) {
if (patch.op === 'test') {
const field = patch.path.slice('/metadata/'.length)
- if (fixture.server.metadata[field] !== patch.value)
+ if (server.metadata[field] !== patch.value)
throw Object.assign(new Error('fixture status conflict'), { code: 409 })
- } else if (patch.path === '/status') fixture.server.status = structuredClone(patch.value)
+ } else if (patch.path === '/status') server.status = structuredClone(patch.value)
else if (patch.path === '/status/conditions')
- fixture.server.status.conditions = structuredClone(patch.value)
+ server.status.conditions = structuredClone(patch.value)
else throw new Error(`Unexpected status patch ${patch.path}`)
}
- fixture.server.metadata.resourceVersion = String(++fixture.revision)
- return structuredClone(fixture.server)
+ server.metadata.resourceVersion = String(++fixture.revision)
+ return structuredClone(server)
},
}
return {
@@ -201,6 +209,16 @@ function prepareWatcher(policyTail?: Promise<void>) {
return { watcher, state }
}
+function recordReconciledNames(state: any): string[] {
+ const names: string[] = []
+ const reconcile = state.reconciler.reconcile.bind(state.reconciler)
+ vi.spyOn(state.reconciler, 'reconcile').mockImplementation(async (server: any, options: any) => {
+ names.push(server.name)
+ return reconcile(server, options)
+ })
+ return names
+}
+
async function skippedTickCount() {
const metric = await registry
.getSingleMetric('clerum_hcc_netpol_resync_ticks_skipped_total')!
@@ -218,7 +236,9 @@ describe('A-T7 real MCP runtime repair after identical recovery', () => {
fixture.revision = 0
fixture.rolloutReady = true
fixture.failDeployment = false
+ fixture.failDeploymentName = undefined
fixture.serverExists = true
+ fixture.peers = []
fixture.readInput = undefined
fixture.failPublication = undefined
fixture.readService = undefined
@@ -562,4 +582,80 @@ describe('A-T7 real MCP runtime repair after identical recovery', () => {
expect(state.reconciler.hasPendingReconciliation()).toBe(false)
expect(state.initialConvergenceRetryTimers.has('McpServer')).toBe(false)
})
+
+ describe('a persistently failing server alongside a healthy peer', () => {
+ beforeEach(() => {
+ const peer = structuredClone(fixture.server)
+ peer.metadata = { ...peer.metadata, name: 'runtime-peer', uid: 'runtime-peer-uid' }
+ fixture.peers = [peer]
+ fixture.failDeploymentName = 'runtime-witness'
+ })
+
+ async function startWithScopedRetryArmed() {
+ const harness = prepareWatcher()
+ watcher = harness.watcher
+ const { state } = harness
+ await watcher.start()
+ await drain(state)
+ expect(fixture.deployments.has('runtime-peer')).toBe(true)
+ expect(fixture.deployments.has('runtime-witness')).toBe(false)
+ expect(state.reconciler.incompleteReconciliationNames()).toEqual(['runtime-witness'])
+ expect(state.runtimeRepairPending).toBe(true)
+ expect(state.initialConvergenceRetryTimers.has('McpServer')).toBe(true)
+ return state
+ }
+
+ it('A-T7 regression: retries only the failing server, then certifies and skips identical recovery', async () => {
+ const state = await startWithScopedRetryArmed()
+ const reconciled = recordReconciledNames(state)
+ await vi.advanceTimersByTimeAsync(5000)
+ await drain(state)
+ expect(reconciled).toEqual(['runtime-witness'])
+ expect(state.runtimeRepairPending).toBe(true)
+ expect(state.initialConvergenceRetryTimers.has('McpServer')).toBe(true)
+
+ fixture.failDeploymentName = undefined
+ await vi.advanceTimersByTimeAsync(15000)
+ await drain(state)
+ expect(reconciled).toEqual(['runtime-witness', 'runtime-witness'])
+ expect(fixture.deployments.has('runtime-witness')).toBe(true)
+ expect(state.reconciler.hasIncompleteReconciliation()).toBe(false)
+ expect(state.runtimeRepairPending).toBe(false)
+ expect(state.initialConvergenceRetryTimers.has('McpServer')).toBe(false)
+
+ // Fold the status writes back through a real LIST once, then an
+ // identical recovery must skip again.
+ await state.recoverMcpServerInventoryAndWatch()
+ await drain(state)
+ const decisions = vi.spyOn(hccLogger, 'info')
+ await state.recoverMcpServerInventoryAndWatch()
+ await drain(state)
+ expect(decisions).toHaveBeenCalledWith(
+ expect.any(String),
+ expect.objectContaining({ decision: 'skip', reason: 'identical-complete' })
+ )
+ })
+
+ it('A-T7 control: a scoped retry widens to the fleet once inventory has moved', async () => {
+ const state = await startWithScopedRetryArmed()
+ state.mcpServerRetryRequest = {
+ ...state.mcpServerRetryRequest,
+ serverRevision: state.mcpServerRetryRequest.serverRevision - 1,
+ }
+ const reconciled = recordReconciledNames(state)
+ await vi.advanceTimersByTimeAsync(5000)
+ await drain(state)
+ expect([...reconciled].sort()).toEqual(['runtime-peer', 'runtime-witness'])
+ })
+
+ it('A-T7 control: a full retry request widens an armed scoped retry', async () => {
+ const state = await startWithScopedRetryArmed()
+ state.scheduleInitialConvergenceRetry('McpServer')
+ expect(state.mcpServerRetryRequest).toBe('full')
+ const reconciled = recordReconciledNames(state)
+ await vi.advanceTimersByTimeAsync(5000)
+ await drain(state)
+ expect([...reconciled].sort()).toEqual(['runtime-peer', 'runtime-witness'])
+ })
+ })
})
diff --git a/host-context-controller/src/k8sClient.ts b/host-context-controller/src/k8sClient.ts
index e2726a2d7..c96c3d04d 100644
--- a/host-context-controller/src/k8sClient.ts
+++ b/host-context-controller/src/k8sClient.ts
@@ -244,6 +244,27 @@ type NetworkPolicyConvergenceCause =
| 'context-change'
| 'context-reconcile-failure'
type CompletedInventoryRevision = { contextRevision: number; serverRevision: number }
+// A scoped McpServer pass retries only names left incomplete by a pass whose
+// effects all ran under this inventory revision. 'full' covers the fleet.
+type McpServerConvergenceRequest =
+ | 'full'
+ | (CompletedInventoryRevision & { names: ReadonlySet<string> })
+
+function widenMcpServerConvergence(
+ current: McpServerConvergenceRequest | undefined,
+ next: McpServerConvergenceRequest
+): McpServerConvergenceRequest {
+ if (current === undefined) return next
+ if (
+ current === 'full' ||
+ next === 'full' ||
+ current.contextRevision !== next.contextRevision ||
+ current.serverRevision !== next.serverRevision
+ ) {
+ return 'full'
+ }
+ return { ...next, names: new Set([...current.names, ...next.names]) }
+}
type InitialConvergencePassResult =
| 'certified'
@@ -793,6 +814,8 @@ export class McpServerWatcher implements McpServerProvider {
private completedRuntimeRevision: CompletedInventoryRevision | null = null
private networkPolicyRepairPending = false
private runtimeRepairPending = false
+ private pendingMcpServerConvergence: McpServerConvergenceRequest | undefined
+ private mcpServerRetryRequest: McpServerConvergenceRequest | undefined
private runtimeInventoryGeneration: number | null = null
private readonly pendingNetworkPolicyCauses = new Set<NetworkPolicyConvergenceCause>()
private networkPolicyPassSequence = 0
@@ -3105,7 +3128,13 @@ export class McpServerWatcher implements McpServerProvider {
return request
}
- private runInitialMcpServerConvergence(): Promise<void> {
+ private runInitialMcpServerConvergence(
+ request: McpServerConvergenceRequest = 'full'
+ ): Promise<void> {
+ this.pendingMcpServerConvergence = widenMcpServerConvergence(
+ this.pendingMcpServerConvergence,
+ request
+ )
return this.runInitialConvergence('McpServer')
}
@@ -3161,12 +3190,18 @@ export class McpServerWatcher implements McpServerProvider {
}
private async runInitialMcpServerConvergenceCore(): Promise<void> {
+ const request = this.pendingMcpServerConvergence ?? 'full'
+ this.pendingMcpServerConvergence = undefined
const safetyCertificate = this.currentNetworkPolicySafetyCertificate()
// A retired run may settle normally without invoking work. Keep its repair
// obligation until an authoritative pass actually completes every effect.
this.runtimeRepairPending = true
if (safetyCertificate === null) return
let effectsComplete = true
+ // A scoped retry completes only the pass it came from. Once the inventory
+ // has moved, the rest of the fleet may need work too, so run a full pass.
+ const scope =
+ request !== 'full' && this.matchesCurrentInventory(request) ? request.names : undefined
const inventoryGeneration = this.mcpWatchGeneration
this.runtimeInventoryGeneration = inventoryGeneration
const inventoryAuthoritative = () =>
@@ -3175,12 +3210,19 @@ export class McpServerWatcher implements McpServerProvider {
try {
const initialServers = [...this.servers.values()]
const initialExternalEgressGates = this.externalEgressCoordinator.prepareStartupGates(
- initialServers,
+ scope ? initialServers.filter(server => scope.has(server.name)) : initialServers,
inventoryAuthoritative
)
if (this.stopped) return
- hccLogger.info('[K8s] Running initial McpServer background reconciliation...')
+ if (scope) {
+ hccLogger.info('[K8s] Running scoped McpServer background reconciliation retry...', {
+ servers: [...scope].sort(),
+ })
+ } else {
+ hccLogger.info('[K8s] Running initial McpServer background reconciliation...')
+ }
await this.reconciler.fullReconcile(initialServers, {
+ scope,
runEffect: async (serverName, work) => {
const selected = this.servers.get(serverName)
const laneOwner = selected ?? {
@@ -3204,14 +3246,22 @@ export class McpServerWatcher implements McpServerProvider {
})
},
})
- if (
- !effectsComplete ||
- !inventoryAuthoritative() ||
- this.reconciler.hasIncompleteReconciliation()
- ) {
+ if (!effectsComplete || !inventoryAuthoritative()) {
this.scheduleInitialConvergenceRetry('McpServer')
return
}
+ if (this.reconciler.hasIncompleteReconciliation()) {
+ // Every offered effect ran under this certificate, so only individual
+ // servers are unfinished (for example a Deployment write rejected by
+ // admission). Retry just those names: one persistently failing server
+ // must not re-reconcile the whole fleet at the retry cap.
+ this.scheduleInitialConvergenceRetry('McpServer', {
+ contextRevision: safetyCertificate.contextRevision,
+ serverRevision: safetyCertificate.serverRevision,
+ names: new Set(this.reconciler.incompleteReconciliationNames()),
+ })
+ return
+ }
this.completedRuntimeRevision = safetyCertificate
this.runtimeRepairPending = false
initialConvergenceLastSuccessTimestampSeconds.set({ lane: 'McpServer' }, Date.now() / 1000)
@@ -3536,10 +3586,22 @@ export class McpServerWatcher implements McpServerProvider {
this.initialConvergenceRetryTimers.delete(lane)
}
this.initialConvergenceRetryAttempts.delete(lane)
+ if (lane === 'McpServer') this.mcpServerRetryRequest = undefined
}
- private scheduleInitialConvergenceRetry(lane: InitialConvergenceLane): void {
- if (this.stopped || this.initialConvergenceRetryTimers.has(lane)) return
+ private scheduleInitialConvergenceRetry(
+ lane: InitialConvergenceLane,
+ mcpServerRequest: McpServerConvergenceRequest = 'full'
+ ): void {
+ if (this.stopped) return
+ if (lane === 'McpServer') {
+ // An armed retry widens to cover every request made before it fires.
+ this.mcpServerRetryRequest = widenMcpServerConvergence(
+ this.mcpServerRetryRequest,
+ mcpServerRequest
+ )
+ }
+ if (this.initialConvergenceRetryTimers.has(lane)) return
const attempt = (this.initialConvergenceRetryAttempts.get(lane) ?? 0) + 1
const delayMs =
@@ -3556,7 +3618,9 @@ export class McpServerWatcher implements McpServerProvider {
this.initialConvergenceRetryTimers.delete(lane)
if (this.stopped) return
if (lane === 'McpServer') {
- void this.runInitialMcpServerConvergence()
+ const request = this.mcpServerRetryRequest ?? 'full'
+ this.mcpServerRetryRequest = undefined
+ void this.runInitialMcpServerConvergence(request)
return
}
void this.runInitialNetworkPolicyConvergence({ cause: 'retry' })
diff --git a/host-context-controller/src/reconciler.test.ts b/host-context-controller/src/reconciler.test.ts
index 7acd98b72..834926800 100644
--- a/host-context-controller/src/reconciler.test.ts
+++ b/host-context-controller/src/reconciler.test.ts
@@ -1544,6 +1544,31 @@ describe('full reconciliation inventory authority', () => {
warn.mockRestore()
})
+ it('limits a scoped pass to its names without treating out-of-scope servers as orphans', async () => {
+ reconciler.setInventoryAuthority(() => ({ known: true, generation: 31 }))
+ const failed = makeServer({ name: 'failed-server' })
+ const healthy = makeServer({ name: 'healthy-server' })
+ const reconcile = vi.spyOn(reconciler, 'reconcile').mockResolvedValue(undefined)
+ appsApi.listNamespacedDeployment.mockResolvedValueOnce({
+ items: [
+ { metadata: { name: 'healthy-server', namespace: 'mcp-server' } },
+ { metadata: { name: 'scoped-orphan', namespace: 'mcp-server' } },
+ { metadata: { name: 'unscoped-orphan', namespace: 'mcp-server' } },
+ ],
+ })
+
+ await reconciler.fullReconcile([failed, healthy], {
+ scope: new Set(['failed-server', 'scoped-orphan']),
+ })
+
+ expect(reconcile.mock.calls.map(([server]) => server.name)).toEqual(['failed-server'])
+ expect(appsApi.deleteNamespacedDeployment).toHaveBeenCalledOnce()
+ expect(appsApi.deleteNamespacedDeployment).toHaveBeenCalledWith({
+ name: 'scoped-orphan',
+ namespace: 'mcp-server',
+ })
+ })
+
it('deletes an orphan when inventory authority remains known and stable', async () => {
reconciler.setInventoryAuthority(() => ({ known: true, generation: 21 }))
appsApi.listNamespacedDeployment.mockResolvedValueOnce({
diff --git a/host-context-controller/src/reconciler.ts b/host-context-controller/src/reconciler.ts
index d47b16c29..64cd8ac76 100644
--- a/host-context-controller/src/reconciler.ts
+++ b/host-context-controller/src/reconciler.ts
@@ -104,6 +104,10 @@ export type McpServerInventoryAuthoritySnapshot = {
export type McpServerFullReconcileOptions = {
runEffect?: (serverName: string, work: () => Promise<void>) => Promise<void>
maxConcurrency?: number
+ // Restrict create/update and orphan cleanup to these names. desiredServers
+ // must still be the whole inventory so out-of-scope servers are never
+ // mistaken for orphans.
+ scope?: ReadonlySet<string>
}
export type McpServerMutationOptions = {
@@ -316,6 +320,10 @@ export class McpServerReconciler {
return this.pendingReconciliations.size > 0
}
+ incompleteReconciliationNames(): string[] {
+ return [...this.pendingReconciliations.keys()]
+ }
+
hasPendingReconciliation(): boolean {
return (
this.hasIncompleteReconciliation() ||
@@ -2530,13 +2538,15 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolv
const runEffect =
options.runEffect ?? ((_serverName: string, work: () => Promise<void>) => work())
const maxConcurrency = options.maxConcurrency ?? DEFAULT_FULL_RECONCILE_MAX_CONCURRENCY
+ const inScope = (name: string) => options.scope === undefined || options.scope.has(name)
// Create or update desired servers independently. runEffect owns each
// per-server lane, so a slow fleet member cannot become a process-wide
// startup barrier. The finite worker pool admits no more than the configured
// number of effects at once, while still waiting for every desired result
// before reporting aggregated failures or moving to orphan cleanup.
- const desiredResults = await mapSettledWithConcurrency(desiredServers, maxConcurrency, server =>
+ const scopedServers = desiredServers.filter(server => inScope(server.name))
+ const desiredResults = await mapSettledWithConcurrency(scopedServers, maxConcurrency, server =>
runEffect(server.name, async () => {
const current = this.resolveCurrentServer ? this.resolveCurrentServer(server.name) : server
// Cache mutation happens synchronously before its keyed watch effect is
@@ -2578,7 +2588,7 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolv
if (!presentNames.has(name)) cleanupTargets.set(name, pending.namespace)
}
for (const [name, namespace] of cleanupTargets) {
- if (!desiredNames.has(name)) {
+ if (!desiredNames.has(name) && inScope(name)) {
if (!presentNames.has(name)) this.pendingReconciliations.set(name, { namespace })
await runEffect(name, async () => {
if (!(await this.orphanDeleteAllowed(name, namespace, capturedAuthority))) {