Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
fix(hcc): withdraw fleet benign supersessions instead of failing the pass
Alfredo Lopezdb2e0f44a248
diff --git a/host-context-controller/package-lock.json b/host-context-controller/package-lock.jsonindex 1393c6c3b..eacc27b90 100644--- a/host-context-controller/package-lock.json+++ b/host-context-controller/package-lock.json@@ -1,12 +1,12 @@{"name": "clerum-host-context-controller",- "version": "0.1.266",+ "version": "0.1.267","lockfileVersion": 3,"requires": true,"packages": {"": {"name": "clerum-host-context-controller",- "version": "0.1.266",+ "version": "0.1.267","license": "SEE LICENSE IN LICENSE","dependencies": {"@clerum/image-policy": "file:../packages/image-policy",diff --git a/host-context-controller/package.json b/host-context-controller/package.jsonindex b178f5fd1..80eff8d53 100644--- a/host-context-controller/package.json+++ b/host-context-controller/package.json@@ -1,6 +1,6 @@{"name": "clerum-host-context-controller",- "version": "0.1.266",+ "version": "0.1.267","description": "Host Context Controller - provides McpServer CRDs to mcp-host via REST API","main": "dist/main.js","engines": {diff --git a/host-context-controller/src/hostReconciler.ts b/host-context-controller/src/hostReconciler.tsindex dcfdbb985..147411a33 100644--- a/host-context-controller/src/hostReconciler.ts+++ b/host-context-controller/src/hostReconciler.ts@@ -25,6 +25,7 @@ import { issueMcpHostRuntimeTokens } from './mcpHostRuntimeTokenIssuerClient'import {hostCleanupDeferredTotal,hostDeleteCleanupTotal,+ hostFleetBenignSupersessionsTotal,hostReconcileDurationSeconds,hostReconcileInFlight,hostReconcileQueueWaitSeconds,@@ -149,7 +150,7 @@ const BENIGN_SUPERSESSION_ERROR_NAMES: ReadonlySet<string> = new Set(['HostMutationDependencyChangedError',])-function isBenignSupersessionError(error: unknown): boolean {+function isBenignSupersessionError(error: unknown): error is Error {return error instanceof Error && BENIGN_SUPERSESSION_ERROR_NAMES.has(error.name)}@@ -4314,6 +4315,20 @@ export class HostReconciler {try {await this.reconcile(host, source)} catch (error) {+ if (isBenignSupersessionError(error)) {+ // Per-host reconcile() already recorded outcome=superseded and+ // rethrew. Withdraw from this fleet pass so HostFleetReconcileError,+ // host_fleet_requests_total{failed}, and the lifecycle retry stay+ // reserved for genuine host failures. Match by error.name — the+ // classes are unexported and statelessLifecycleExecutor synthesizes+ // name-equivalent plain Errors.+ hostFleetBenignSupersessionsTotal.inc({ error: error.name })+ hccLogger.warn('Fleet Host reconcile withdrawn as benign supersession', {+ err: error,+ errorName: error.name,+ })+ return+ }console.error(`[HostReconciler] Fleet reconcile failed for Host "${name}":`, error)throw error}diff --git a/host-context-controller/src/k8sClient.test.ts b/host-context-controller/src/k8sClient.test.tsindex 16d1c2a8a..c4d2059b6 100644--- a/host-context-controller/src/k8sClient.test.ts+++ b/host-context-controller/src/k8sClient.test.ts@@ -56,7 +56,9 @@ async function readLabeledConvergenceMetric(| 'clerum_hcc_initial_convergence_effects_dropped_total'| 'clerum_hcc_initial_convergence_pass_results_total'| 'clerum_hcc_netpol_resync_ticks_skipped_total'- | 'clerum_hcc_netpol_defaults_only_ticks_total',+ | 'clerum_hcc_netpol_defaults_only_ticks_total'+ | 'clerum_hcc_host_fleet_lifecycle_catch_total'+ | 'clerum_hcc_host_fleet_requests_total',labels: Record<string, string>): Promise<number> {const metric = registry.getSingleMetric(name)@@ -7341,6 +7343,10 @@ describe('McpServerWatcher CommunicationChannel cache recovery', () => {const watcher = newContextAuthoritativeWatcher()const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {})const lifecycleGeneration = (watcher as any).beginCommunicationChannelLifecycleTransition()+ const appliedBefore = await readLabeledConvergenceMetric(+ 'clerum_hcc_host_fleet_lifecycle_catch_total',+ { decision: 'applied' }+ )try {await (watcher as any).requestHostFleetReconcile(@@ -7350,6 +7356,11 @@ describe('McpServerWatcher CommunicationChannel cache recovery', () => {)expect((watcher as any).ccAppliedLifecycleGeneration).toBe(lifecycleGeneration)+ expect(+ await readLabeledConvergenceMetric('clerum_hcc_host_fleet_lifecycle_catch_total', {+ decision: 'applied',+ })+ ).toBe(appliedBefore + 1)await vi.advanceTimersByTimeAsync(300000)expect(mocks.hostFullReconcile).toHaveBeenCalledOnce()} finally {@@ -7374,6 +7385,10 @@ describe('McpServerWatcher CommunicationChannel cache recovery', () => {;(watcher as any).ccCacheSynced = trueconst errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {})const lifecycleGeneration = (watcher as any).beginCommunicationChannelLifecycleTransition()+ const retryBefore = await readLabeledConvergenceMetric(+ 'clerum_hcc_host_fleet_lifecycle_catch_total',+ { decision: 'retry' }+ )try {await (watcher as any).requestHostFleetReconcile(@@ -7383,6 +7398,11 @@ describe('McpServerWatcher CommunicationChannel cache recovery', () => {)expect(mocks.hostFullReconcile).toHaveBeenCalledOnce()expect(mocks.hostReconcileHosts).not.toHaveBeenCalled()+ expect(+ await readLabeledConvergenceMetric('clerum_hcc_host_fleet_lifecycle_catch_total', {+ decision: 'retry',+ })+ ).toBe(retryBefore + 1)await vi.advanceTimersByTimeAsync(5000)await vi.waitFor(() => expect(mocks.hostReconcileHosts).toHaveBeenCalledOnce())diff --git a/host-context-controller/src/k8sClient.ts b/host-context-controller/src/k8sClient.tsindex 0e48c58a7..6af38ff01 100644--- a/host-context-controller/src/k8sClient.ts+++ b/host-context-controller/src/k8sClient.ts@@ -56,6 +56,7 @@ import {} from './mcpServerSafety'import {hostDeleteCleanupTotal,+ hostFleetLifecycleCatchTotal,hostFleetRequestsTotal,hostWatchRecoverySeconds,initialConvergenceEffectsDroppedTotal,@@ -2227,10 +2228,12 @@ export class McpServerWatcher implements McpServerProvider {console.error(`[K8s] Host reconciliation after ${reason} failed:`, error)if (!this.stopped && ccLifecycleGeneration !== undefined) {if (error instanceof HostFleetReconcileError && error.hostFailures.length === 0) {+ hostFleetLifecycleCatchTotal.inc({ decision: 'applied' })this.markCommunicationChannelLifecycleApplied(ccLifecycleGeneration)} else {// Retry Host convergence without coupling the lifecycle ladder to// orphan cleanup. A later periodic full pass retries cleanup work.+ hostFleetLifecycleCatchTotal.inc({ decision: 'retry' })this.scheduleCommunicationChannelFleetRetry({...request,mode: error instanceof HostFleetReconcileError ? 'lifecycle' : request.mode,diff --git a/host-context-controller/src/metrics.ts b/host-context-controller/src/metrics.tsindex b599259e9..4f93c09ab 100644--- a/host-context-controller/src/metrics.ts+++ b/host-context-controller/src/metrics.ts@@ -279,6 +279,23 @@ export const hostFleetRequestsTotal = counter({labelNames: ['result'] as const,})+// #490: fleet-lane withdrawal of the four benign supersession names. Closed+// `error` set — never a Host name. Distinct from host_fleet_requests_total+// {failed} and from host_delete_cleanup_total{superseded}.+export const hostFleetBenignSupersessionsTotal = counter({+ name: 'clerum_hcc_host_fleet_benign_supersessions_total',+ help: 'Host fleet workers withdrawn because a name-equivalent benign supersession retired the pass.',+ labelNames: ['error'] as const,+})++// Maestro P-4: measure catch-path lifecycle decisions when a+// CommunicationChannel generation is in flight. Do not treat rates as a claim.+export const hostFleetLifecycleCatchTotal = counter({+ name: 'clerum_hcc_host_fleet_lifecycle_catch_total',+ help: 'Host fleet catch-path decisions while a CommunicationChannel lifecycle generation is in flight (applied when hostFailures is empty, retry otherwise).',+ labelNames: ['decision'] as const,+})+export const hostCleanupDeferredTotal = counter({name: 'clerum_hcc_host_cleanup_deferred_total',help: 'Orphan Host cleanup deferrals by bounded reason.',diff --git a/host-context-controller/test/hostReconciler.test.ts b/host-context-controller/test/hostReconciler.test.tsindex 5a2650808..e411639f7 100644--- a/host-context-controller/test/hostReconciler.test.ts+++ b/host-context-controller/test/hostReconciler.test.ts@@ -4,11 +4,13 @@ import type { AdministrativeOutcomeReporter } from '../src/administrativeOutcomeimport { mintHostGfsToken } from '../src/gfsHostBinding'import {DEFAULT_FIRST_PARTY_WORKFLOW_CONTROL_SCOPES,+ HostFleetReconcileError,HostReconciler,resolveWorkflowControlScopes,} from '../src/hostReconciler'import type { InfrastructureTelemetryReporter } from '../src/infrastructureTelemetryReporter'import { issueMcpHostRuntimeTokens } from '../src/mcpHostRuntimeTokenIssuerClient'+import { registry } from '../src/metrics'import { HostCRD, type HostWorkflowControlScope } from '../src/types'import {asAppsApi,@@ -1970,3 +1972,68 @@ describe('reconcileChannelReaderDeployment — B2: preserve replicas on unsyncedexpect(replaceBody.spec?.replicas).toBe(1)})})++async function readFleetBenignSupersessions(errorName: string): Promise<number> {+ const metric = registry.getSingleMetric('clerum_hcc_host_fleet_benign_supersessions_total')+ if (!metric) throw new Error('clerum_hcc_host_fleet_benign_supersessions_total is not registered')+ const snapshot = await metric.get()+ return snapshot.values.find(entry => entry.labels.error === errorName)?.value ?? 0+}++describe('collectHostReconcileFailures benign supersession (#490)', () => {+ function nameEquivalent(name: string, message: string): Error {+ const error = new Error(message)+ error.name = name+ return error+ }++ it('resolves reconcileHosts when every fleet error is a name-equivalent benign supersession', async () => {+ const { reconciler } = createReconciler()+ const withdrawn = nameEquivalent(+ 'HostMutationSpecRevisionChangedError',+ 'Host spec generation changed before Host "alpha-host" reconcile admission'+ )+ vi.spyOn(reconciler, 'reconcile').mockRejectedValue(withdrawn)+ const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {})+ const before = await readFleetBenignSupersessions('HostMutationSpecRevisionChangedError')++ await expect(reconciler.reconcileHosts([makeHost()])).resolves.toBeUndefined()++ expect(await readFleetBenignSupersessions('HostMutationSpecRevisionChangedError')).toBe(+ before + 1+ )+ expect(+ errorSpy.mock.calls.some(call => String(call[0]).includes('Fleet reconcile failed'))+ ).toBe(false)+ errorSpy.mockRestore()+ })++ it('still rejects HostFleetReconcileError for a non-benign fleet error', async () => {+ const { reconciler } = createReconciler()+ vi.spyOn(reconciler, 'reconcile').mockRejectedValue(new Error('boom'))++ await expect(reconciler.reconcileHosts([makeHost()])).rejects.toBeInstanceOf(+ HostFleetReconcileError+ )+ })++ it('still rejects when a mixed pass has at least one genuine host failure', async () => {+ const { reconciler } = createReconciler()+ vi.spyOn(reconciler, 'reconcile').mockImplementation(async host => {+ if (host.name === 'alpha-host') {+ throw nameEquivalent(+ 'HostMutationSpecRevisionChangedError',+ 'Host spec generation changed before Host "alpha-host" reconcile admission'+ )+ }+ throw new Error('boom')+ })++ await expect(+ reconciler.reconcileHosts([makeHost(), makeHost({ name: 'beta-host' })])+ ).rejects.toMatchObject({+ name: 'HostFleetReconcileError',+ hostFailures: [expect.objectContaining({ message: 'boom' })],+ })+ })+})