Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript

fix(hcc): withdraw fleet benign supersessions instead of failing the pass

Alfredo Lopezdb2e0f44a248
diff --git a/host-context-controller/package-lock.json b/host-context-controller/package-lock.json
index 1393c6c3b..eacc27b90 100644
--- a/host-context-controller/package-lock.json
+++ b/host-context-controller/package-lock.json
@@ -1,12 +1,12 @@
{
"name": "clerum-host-context-controller",
- "version": "0.1.266",
+ "version": "0.1.267",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "clerum-host-context-controller",
- "version": "0.1.266",
+ "version": "0.1.267",
"license": "SEE LICENSE IN LICENSE",
"dependencies": {
"@clerum/image-policy": "file:../packages/image-policy",
diff --git a/host-context-controller/package.json b/host-context-controller/package.json
index b178f5fd1..80eff8d53 100644
--- a/host-context-controller/package.json
+++ b/host-context-controller/package.json
@@ -1,6 +1,6 @@
{
"name": "clerum-host-context-controller",
- "version": "0.1.266",
+ "version": "0.1.267",
"description": "Host Context Controller - provides McpServer CRDs to mcp-host via REST API",
"main": "dist/main.js",
"engines": {
diff --git a/host-context-controller/src/hostReconciler.ts b/host-context-controller/src/hostReconciler.ts
index dcfdbb985..147411a33 100644
--- a/host-context-controller/src/hostReconciler.ts
+++ b/host-context-controller/src/hostReconciler.ts
@@ -25,6 +25,7 @@ import { issueMcpHostRuntimeTokens } from './mcpHostRuntimeTokenIssuerClient'
import {
hostCleanupDeferredTotal,
hostDeleteCleanupTotal,
+ hostFleetBenignSupersessionsTotal,
hostReconcileDurationSeconds,
hostReconcileInFlight,
hostReconcileQueueWaitSeconds,
@@ -149,7 +150,7 @@ const BENIGN_SUPERSESSION_ERROR_NAMES: ReadonlySet<string> = new Set([
'HostMutationDependencyChangedError',
])
-function isBenignSupersessionError(error: unknown): boolean {
+function isBenignSupersessionError(error: unknown): error is Error {
return error instanceof Error && BENIGN_SUPERSESSION_ERROR_NAMES.has(error.name)
}
@@ -4314,6 +4315,20 @@ export class HostReconciler {
try {
await this.reconcile(host, source)
} catch (error) {
+ if (isBenignSupersessionError(error)) {
+ // Per-host reconcile() already recorded outcome=superseded and
+ // rethrew. Withdraw from this fleet pass so HostFleetReconcileError,
+ // host_fleet_requests_total{failed}, and the lifecycle retry stay
+ // reserved for genuine host failures. Match by error.name — the
+ // classes are unexported and statelessLifecycleExecutor synthesizes
+ // name-equivalent plain Errors.
+ hostFleetBenignSupersessionsTotal.inc({ error: error.name })
+ hccLogger.warn('Fleet Host reconcile withdrawn as benign supersession', {
+ err: error,
+ errorName: error.name,
+ })
+ return
+ }
console.error(`[HostReconciler] Fleet reconcile failed for Host "${name}":`, error)
throw error
}
diff --git a/host-context-controller/src/k8sClient.test.ts b/host-context-controller/src/k8sClient.test.ts
index 16d1c2a8a..c4d2059b6 100644
--- a/host-context-controller/src/k8sClient.test.ts
+++ b/host-context-controller/src/k8sClient.test.ts
@@ -56,7 +56,9 @@ async function readLabeledConvergenceMetric(
| 'clerum_hcc_initial_convergence_effects_dropped_total'
| 'clerum_hcc_initial_convergence_pass_results_total'
| 'clerum_hcc_netpol_resync_ticks_skipped_total'
- | 'clerum_hcc_netpol_defaults_only_ticks_total',
+ | 'clerum_hcc_netpol_defaults_only_ticks_total'
+ | 'clerum_hcc_host_fleet_lifecycle_catch_total'
+ | 'clerum_hcc_host_fleet_requests_total',
labels: Record<string, string>
): Promise<number> {
const metric = registry.getSingleMetric(name)
@@ -7341,6 +7343,10 @@ describe('McpServerWatcher CommunicationChannel cache recovery', () => {
const watcher = newContextAuthoritativeWatcher()
const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {})
const lifecycleGeneration = (watcher as any).beginCommunicationChannelLifecycleTransition()
+ const appliedBefore = await readLabeledConvergenceMetric(
+ 'clerum_hcc_host_fleet_lifecycle_catch_total',
+ { decision: 'applied' }
+ )
try {
await (watcher as any).requestHostFleetReconcile(
@@ -7350,6 +7356,11 @@ describe('McpServerWatcher CommunicationChannel cache recovery', () => {
)
expect((watcher as any).ccAppliedLifecycleGeneration).toBe(lifecycleGeneration)
+ expect(
+ await readLabeledConvergenceMetric('clerum_hcc_host_fleet_lifecycle_catch_total', {
+ decision: 'applied',
+ })
+ ).toBe(appliedBefore + 1)
await vi.advanceTimersByTimeAsync(300000)
expect(mocks.hostFullReconcile).toHaveBeenCalledOnce()
} finally {
@@ -7374,6 +7385,10 @@ describe('McpServerWatcher CommunicationChannel cache recovery', () => {
;(watcher as any).ccCacheSynced = true
const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {})
const lifecycleGeneration = (watcher as any).beginCommunicationChannelLifecycleTransition()
+ const retryBefore = await readLabeledConvergenceMetric(
+ 'clerum_hcc_host_fleet_lifecycle_catch_total',
+ { decision: 'retry' }
+ )
try {
await (watcher as any).requestHostFleetReconcile(
@@ -7383,6 +7398,11 @@ describe('McpServerWatcher CommunicationChannel cache recovery', () => {
)
expect(mocks.hostFullReconcile).toHaveBeenCalledOnce()
expect(mocks.hostReconcileHosts).not.toHaveBeenCalled()
+ expect(
+ await readLabeledConvergenceMetric('clerum_hcc_host_fleet_lifecycle_catch_total', {
+ decision: 'retry',
+ })
+ ).toBe(retryBefore + 1)
await vi.advanceTimersByTimeAsync(5000)
await vi.waitFor(() => expect(mocks.hostReconcileHosts).toHaveBeenCalledOnce())
diff --git a/host-context-controller/src/k8sClient.ts b/host-context-controller/src/k8sClient.ts
index 0e48c58a7..6af38ff01 100644
--- a/host-context-controller/src/k8sClient.ts
+++ b/host-context-controller/src/k8sClient.ts
@@ -56,6 +56,7 @@ import {
} from './mcpServerSafety'
import {
hostDeleteCleanupTotal,
+ hostFleetLifecycleCatchTotal,
hostFleetRequestsTotal,
hostWatchRecoverySeconds,
initialConvergenceEffectsDroppedTotal,
@@ -2227,10 +2228,12 @@ export class McpServerWatcher implements McpServerProvider {
console.error(`[K8s] Host reconciliation after ${reason} failed:`, error)
if (!this.stopped && ccLifecycleGeneration !== undefined) {
if (error instanceof HostFleetReconcileError && error.hostFailures.length === 0) {
+ hostFleetLifecycleCatchTotal.inc({ decision: 'applied' })
this.markCommunicationChannelLifecycleApplied(ccLifecycleGeneration)
} else {
// Retry Host convergence without coupling the lifecycle ladder to
// orphan cleanup. A later periodic full pass retries cleanup work.
+ hostFleetLifecycleCatchTotal.inc({ decision: 'retry' })
this.scheduleCommunicationChannelFleetRetry({
...request,
mode: error instanceof HostFleetReconcileError ? 'lifecycle' : request.mode,
diff --git a/host-context-controller/src/metrics.ts b/host-context-controller/src/metrics.ts
index b599259e9..4f93c09ab 100644
--- a/host-context-controller/src/metrics.ts
+++ b/host-context-controller/src/metrics.ts
@@ -279,6 +279,23 @@ export const hostFleetRequestsTotal = counter({
labelNames: ['result'] as const,
})
+// #490: fleet-lane withdrawal of the four benign supersession names. Closed
+// `error` set — never a Host name. Distinct from host_fleet_requests_total
+// {failed} and from host_delete_cleanup_total{superseded}.
+export const hostFleetBenignSupersessionsTotal = counter({
+ name: 'clerum_hcc_host_fleet_benign_supersessions_total',
+ help: 'Host fleet workers withdrawn because a name-equivalent benign supersession retired the pass.',
+ labelNames: ['error'] as const,
+})
+
+// Maestro P-4: measure catch-path lifecycle decisions when a
+// CommunicationChannel generation is in flight. Do not treat rates as a claim.
+export const hostFleetLifecycleCatchTotal = counter({
+ name: 'clerum_hcc_host_fleet_lifecycle_catch_total',
+ help: 'Host fleet catch-path decisions while a CommunicationChannel lifecycle generation is in flight (applied when hostFailures is empty, retry otherwise).',
+ labelNames: ['decision'] as const,
+})
+
export const hostCleanupDeferredTotal = counter({
name: 'clerum_hcc_host_cleanup_deferred_total',
help: 'Orphan Host cleanup deferrals by bounded reason.',
diff --git a/host-context-controller/test/hostReconciler.test.ts b/host-context-controller/test/hostReconciler.test.ts
index 5a2650808..e411639f7 100644
--- a/host-context-controller/test/hostReconciler.test.ts
+++ b/host-context-controller/test/hostReconciler.test.ts
@@ -4,11 +4,13 @@ import type { AdministrativeOutcomeReporter } from '../src/administrativeOutcome
import { mintHostGfsToken } from '../src/gfsHostBinding'
import {
DEFAULT_FIRST_PARTY_WORKFLOW_CONTROL_SCOPES,
+ HostFleetReconcileError,
HostReconciler,
resolveWorkflowControlScopes,
} from '../src/hostReconciler'
import type { InfrastructureTelemetryReporter } from '../src/infrastructureTelemetryReporter'
import { issueMcpHostRuntimeTokens } from '../src/mcpHostRuntimeTokenIssuerClient'
+import { registry } from '../src/metrics'
import { HostCRD, type HostWorkflowControlScope } from '../src/types'
import {
asAppsApi,
@@ -1970,3 +1972,68 @@ describe('reconcileChannelReaderDeployment — B2: preserve replicas on unsynced
expect(replaceBody.spec?.replicas).toBe(1)
})
})
+
+async function readFleetBenignSupersessions(errorName: string): Promise<number> {
+ const metric = registry.getSingleMetric('clerum_hcc_host_fleet_benign_supersessions_total')
+ if (!metric) throw new Error('clerum_hcc_host_fleet_benign_supersessions_total is not registered')
+ const snapshot = await metric.get()
+ return snapshot.values.find(entry => entry.labels.error === errorName)?.value ?? 0
+}
+
+describe('collectHostReconcileFailures benign supersession (#490)', () => {
+ function nameEquivalent(name: string, message: string): Error {
+ const error = new Error(message)
+ error.name = name
+ return error
+ }
+
+ it('resolves reconcileHosts when every fleet error is a name-equivalent benign supersession', async () => {
+ const { reconciler } = createReconciler()
+ const withdrawn = nameEquivalent(
+ 'HostMutationSpecRevisionChangedError',
+ 'Host spec generation changed before Host "alpha-host" reconcile admission'
+ )
+ vi.spyOn(reconciler, 'reconcile').mockRejectedValue(withdrawn)
+ const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {})
+ const before = await readFleetBenignSupersessions('HostMutationSpecRevisionChangedError')
+
+ await expect(reconciler.reconcileHosts([makeHost()])).resolves.toBeUndefined()
+
+ expect(await readFleetBenignSupersessions('HostMutationSpecRevisionChangedError')).toBe(
+ before + 1
+ )
+ expect(
+ errorSpy.mock.calls.some(call => String(call[0]).includes('Fleet reconcile failed'))
+ ).toBe(false)
+ errorSpy.mockRestore()
+ })
+
+ it('still rejects HostFleetReconcileError for a non-benign fleet error', async () => {
+ const { reconciler } = createReconciler()
+ vi.spyOn(reconciler, 'reconcile').mockRejectedValue(new Error('boom'))
+
+ await expect(reconciler.reconcileHosts([makeHost()])).rejects.toBeInstanceOf(
+ HostFleetReconcileError
+ )
+ })
+
+ it('still rejects when a mixed pass has at least one genuine host failure', async () => {
+ const { reconciler } = createReconciler()
+ vi.spyOn(reconciler, 'reconcile').mockImplementation(async host => {
+ if (host.name === 'alpha-host') {
+ throw nameEquivalent(
+ 'HostMutationSpecRevisionChangedError',
+ 'Host spec generation changed before Host "alpha-host" reconcile admission'
+ )
+ }
+ throw new Error('boom')
+ })
+
+ await expect(
+ reconciler.reconcileHosts([makeHost(), makeHost({ name: 'beta-host' })])
+ ).rejects.toMatchObject({
+ name: 'HostFleetReconcileError',
+ hostFailures: [expect.objectContaining({ message: 'boom' })],
+ })
+ })
+})