Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
fix(hcc): re-arm silent NetworkPolicy convergence sinks and name pass outcomes (#447)
Alfredo Lopez0692850fe2b8
diff --git a/host-context-controller/package-lock.json b/host-context-controller/package-lock.jsonindex 98f09c301..b75c7ef2b 100644--- a/host-context-controller/package-lock.json+++ b/host-context-controller/package-lock.json@@ -1,12 +1,12 @@{"name": "clerum-host-context-controller",- "version": "0.1.235",+ "version": "0.1.236","lockfileVersion": 3,"requires": true,"packages": {"": {"name": "clerum-host-context-controller",- "version": "0.1.235",+ "version": "0.1.236","license": "SEE LICENSE IN LICENSE","dependencies": {"@clerum/image-policy": "file:../packages/image-policy",diff --git a/host-context-controller/package.json b/host-context-controller/package.jsonindex 7fc86e36c..b02f8befc 100644--- a/host-context-controller/package.json+++ b/host-context-controller/package.json@@ -1,6 +1,6 @@{"name": "clerum-host-context-controller",- "version": "0.1.235",+ "version": "0.1.236","description": "Host Context Controller - provides McpServer CRDs to mcp-host via REST API","main": "dist/main.js","engines": {diff --git a/host-context-controller/src/k8sClient.test.ts b/host-context-controller/src/k8sClient.test.tsindex 47fa0acbc..336361dba 100644--- a/host-context-controller/src/k8sClient.test.ts+++ b/host-context-controller/src/k8sClient.test.ts@@ -49,6 +49,23 @@ async function readInitialConvergenceMetric(return snapshot.values.find(entry => entry.labels.lane === lane)?.value ?? 0}+async function readLabeledConvergenceMetric(+ name:+ | 'clerum_hcc_initial_convergence_swallowed_total'+ | 'clerum_hcc_initial_convergence_effects_dropped_total'+ | 'clerum_hcc_initial_convergence_pass_results_total',+ labels: Record<string, string>+): Promise<number> {+ const metric = registry.getSingleMetric(name)+ if (!metric) throw new Error(`${name} is not registered`)+ const snapshot = await metric.get()+ return (+ snapshot.values.find(entry =>+ Object.entries(labels).every(([key, value]) => entry.labels[key] === value)+ )?.value ?? 0+ )+}+async function requestReadyOverHttp(server: ContextMapperServer): Promise<{statusCode: number | undefinedbody: string@@ -3908,6 +3925,11 @@ describe('McpServerWatcher startup', () => {transport: { type: 'streamableHttp' as const, port: 3000 },},})+ const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {})+ const successTimestampBefore = await readInitialConvergenceMetric(+ 'clerum_hcc_initial_convergence_last_success_timestamp_seconds',+ 'NetworkPolicy'+ )mocks.netPolFullReconcile.mockImplementation(async (contexts, servers, options) => {pass += 1if (pass === 1) {@@ -3942,7 +3964,37 @@ describe('McpServerWatcher startup', () => {expect(appliedContexts).toEqual([])expect(appliedServers).toEqual([])- expect((watcher as any).initialConvergenceRetryAttempts.get('NetworkPolicy')).toBe(4)+ expect(+ errorSpy.mock.calls.some(call =>+ String(call[0]).includes('pass ended without certifying: inventory authority lost')+ )+ ).toBe(true)+ expect((watcher as any).initialConvergenceRetryAttempts.get('NetworkPolicy')).toBe(5)+ expect((watcher as any).initialConvergenceRetryTimers.has('NetworkPolicy')).toBe(true)+ expect(+ await readInitialConvergenceMetric(+ 'clerum_hcc_initial_convergence_last_success_timestamp_seconds',+ 'NetworkPolicy'+ )+ ).toBe(successTimestampBefore)+ expect(+ await readLabeledConvergenceMetric('clerum_hcc_initial_convergence_swallowed_total', {+ lane: 'NetworkPolicy',+ sink: 'authority-lost',+ })+ ).toBeGreaterThan(0)+ expect(+ await readLabeledConvergenceMetric('clerum_hcc_initial_convergence_pass_results_total', {+ lane: 'NetworkPolicy',+ result: 'aborted-authority',+ })+ ).toBeGreaterThan(0)+ expect(+ await readLabeledConvergenceMetric('clerum_hcc_initial_convergence_effects_dropped_total', {+ lane: 'NetworkPolicy',+ kind: 'context',+ })+ ).toBeGreaterThan(0);(watcher as any).contexts.clear();(watcher as any).servers.clear();(watcher as any).contexts.set('current-context', {@@ -3966,6 +4018,7 @@ describe('McpServerWatcher startup', () => {expect(appliedContexts).toEqual(['current-context'])expect(appliedServers).toEqual(['current-server'])expect((watcher as any).initialConvergenceRetryAttempts.has('NetworkPolicy')).toBe(false)+ errorSpy.mockRestore()await watcher.stop()})@@ -4298,6 +4351,7 @@ describe('McpServerWatcher startup', () => {})it('defers NetworkPolicy full convergence until both inventories are authoritative', async () => {+ const warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => {})const watcher = new McpServerWatcher();(watcher as any).contextCacheSynced = true;(watcher as any).mcpServerCacheSynced = false@@ -4306,7 +4360,21 @@ describe('McpServerWatcher startup', () => {await (watcher as any).runInitialNetworkPolicyConvergence()expect(mocks.netPolFullReconcile).not.toHaveBeenCalled()- expect((watcher as any).initialConvergenceRetryAttempts.get('NetworkPolicy')).toBe(2)+ expect(warnSpy.mock.calls.some(call => String(call[0]).includes('caches unsynced'))).toBe(true)+ expect((watcher as any).initialConvergenceRetryAttempts.get('NetworkPolicy')).toBe(3)+ expect((watcher as any).initialConvergenceRetryTimers.has('NetworkPolicy')).toBe(true)+ expect(+ await readLabeledConvergenceMetric('clerum_hcc_initial_convergence_swallowed_total', {+ lane: 'NetworkPolicy',+ sink: 'unsynced',+ })+ ).toBeGreaterThan(0)+ expect(+ await readLabeledConvergenceMetric('clerum_hcc_initial_convergence_pass_results_total', {+ lane: 'NetworkPolicy',+ result: 'deferred-unsynced',+ })+ ).toBeGreaterThan(0);(watcher as any).mcpServerCacheSynced = trueawait (watcher as any).runInitialNetworkPolicyConvergence()@@ -4322,6 +4390,94 @@ describe('McpServerWatcher startup', () => {}))expect((watcher as any).initialConvergenceRetryAttempts.has('NetworkPolicy')).toBe(false)+ warnSpy.mockRestore()+ await watcher.stop()+ })++ it('re-runs NetworkPolicy convergence from the retry ladder after a silent-sink miss without later watch events', async () => {+ vi.useFakeTimers()+ const warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => {})+ const watcher = new McpServerWatcher()+ ;(watcher as any).contextCacheSynced = true+ ;(watcher as any).mcpServerCacheSynced = false++ await (watcher as any).runInitialNetworkPolicyConvergence()++ expect(mocks.netPolFullReconcile).not.toHaveBeenCalled()+ expect((watcher as any).initialConvergenceRetryAttempts.get('NetworkPolicy')).toBe(1)+ expect((watcher as any).initialConvergenceRetryTimers.has('NetworkPolicy')).toBe(true)+ ;(watcher as any).mcpServerCacheSynced = true+ await vi.advanceTimersByTimeAsync(5000)++ expect(mocks.netPolFullReconcile).toHaveBeenCalledOnce()+ expect((watcher as any).initialConvergenceRetryAttempts.has('NetworkPolicy')).toBe(false)+ expect(+ await readLabeledConvergenceMetric('clerum_hcc_initial_convergence_pass_results_total', {+ lane: 'NetworkPolicy',+ result: 'certified',+ })+ ).toBeGreaterThan(0)+ warnSpy.mockRestore()+ await watcher.stop()+ })++ it('names a generic NetworkPolicy catch failure failed instead of aborted-bump', async () => {+ const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {})+ const watcher = new McpServerWatcher()+ ;(watcher as any).contextCacheSynced = true+ ;(watcher as any).mcpServerCacheSynced = true+ mocks.netPolFullReconcile.mockRejectedValueOnce(new Error('apiserver 5xx'))+ const failedBefore = await readLabeledConvergenceMetric(+ 'clerum_hcc_initial_convergence_pass_results_total',+ { lane: 'NetworkPolicy', result: 'failed' }+ )+ const bumpBefore = await readLabeledConvergenceMetric(+ 'clerum_hcc_initial_convergence_pass_results_total',+ { lane: 'NetworkPolicy', result: 'aborted-bump' }+ )++ await (watcher as any).runInitialNetworkPolicyConvergence()++ expect(+ await readLabeledConvergenceMetric('clerum_hcc_initial_convergence_pass_results_total', {+ lane: 'NetworkPolicy',+ result: 'failed',+ })+ ).toBe(failedBefore + 1)+ expect(+ await readLabeledConvergenceMetric('clerum_hcc_initial_convergence_pass_results_total', {+ lane: 'NetworkPolicy',+ result: 'aborted-bump',+ })+ ).toBe(bumpBefore)+ expect((watcher as any).initialConvergenceRetryAttempts.get('NetworkPolicy')).toBe(1)+ errorSpy.mockRestore()+ await watcher.stop()+ })++ it('names the inventory-changed throw aborted-bump', async () => {+ const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {})+ const watcher = new McpServerWatcher()+ ;(watcher as any).contextCacheSynced = true+ ;(watcher as any).mcpServerCacheSynced = true+ mocks.netPolFullReconcile.mockRejectedValueOnce(+ new Error('Desired NetworkPolicy inventory changed during authoritative revocation')+ )+ const bumpBefore = await readLabeledConvergenceMetric(+ 'clerum_hcc_initial_convergence_pass_results_total',+ { lane: 'NetworkPolicy', result: 'aborted-bump' }+ )++ await (watcher as any).runInitialNetworkPolicyConvergence()++ expect(+ await readLabeledConvergenceMetric('clerum_hcc_initial_convergence_pass_results_total', {+ lane: 'NetworkPolicy',+ result: 'aborted-bump',+ })+ ).toBe(bumpBefore + 1)+ expect((watcher as any).initialConvergenceRetryAttempts.get('NetworkPolicy')).toBe(1)+ errorSpy.mockRestore()await watcher.stop()})diff --git a/host-context-controller/src/k8sClient.ts b/host-context-controller/src/k8sClient.tsindex cd2475957..171c982a6 100644--- a/host-context-controller/src/k8sClient.ts+++ b/host-context-controller/src/k8sClient.ts@@ -58,8 +58,12 @@ import {hostDeleteCleanupTotal,hostFleetRequestsTotal,hostWatchRecoverySeconds,+ initialConvergenceEffectsDroppedTotal,initialConvergenceLastSuccessTimestampSeconds,+ initialConvergencePassDurationSeconds,+ initialConvergencePassResultsTotal,initialConvergenceRetriesTotal,+ initialConvergenceSwallowedTotal,} from './metrics'import { NetworkPolicyReconciler, sameContextDesiredRevision } from './networkPolicyReconciler'import type { ReadinessInventoryDetail } from './readinessGate'@@ -218,6 +222,24 @@ type HostSnapshot = {type HostInventoryRecoveryCause = 'cold-start' | 'watch-recovery'type HostWatchEventType = 'ADDED' | 'MODIFIED' | 'DELETED'type InitialConvergenceLane = 'McpServer' | 'NetworkPolicy'+type InitialConvergencePassResult =+ | 'certified'+ | 'aborted-authority'+ | 'deferred-unsynced'+ | 'aborted-bump'+ | 'failed'++const DESIRED_INVENTORY_CHANGED_MESSAGE =+ 'Desired NetworkPolicy inventory changed during authoritative revocation'++function observeInitialNetworkPolicyPass(+ startedAtMs: number,+ result: InitialConvergencePassResult+): void {+ const seconds = Math.max(0, (Date.now() - startedAtMs) / 1000)+ initialConvergencePassResultsTotal.inc({ lane: 'NetworkPolicy', result })+ initialConvergencePassDurationSeconds.observe({ lane: 'NetworkPolicy', result }, seconds)+}type NetworkPolicySafetyCertificate = {// VESTIGIAL — retained for record shape only, never read in any decision.@@ -3043,7 +3065,22 @@ export class McpServerWatcher implements McpServerProvider {// coordinator. Running safety from a partial inventory would be neither a// full convergence nor safe evidence of success. Recovery of either// missing LIST -> WATCH pair schedules a fresh current-cache pass.- if (!this.contextCacheSynced || !this.mcpServerCacheSynced) return+ const startedAtMs = Date.now()+ if (!this.contextCacheSynced || !this.mcpServerCacheSynced) {+ const unsynced = [+ !this.contextCacheSynced ? 'Context' : undefined,+ !this.mcpServerCacheSynced ? 'McpServer' : undefined,+ ]+ .filter((name): name is string => name !== undefined)+ .join(' and ')+ console.warn(+ `[K8s] NetworkPolicy convergence request deferred: caches unsynced (${unsynced})`+ )+ initialConvergenceSwallowedTotal.inc({ lane: 'NetworkPolicy', sink: 'unsynced' })+ observeInitialNetworkPolicyPass(startedAtMs, 'deferred-unsynced')+ this.scheduleInitialConvergenceRetry('NetworkPolicy')+ return+ }let authoritativeRevocationCompleted = falsetry {const initialContexts = [...this.contexts.values()]@@ -3069,7 +3106,13 @@ export class McpServerWatcher implements McpServerProvider {serverInventoryAuthoritative,runContextEffect: (contextId, work) =>this.enqueueContextReconciliation(contextId, async () => {- if (!contextInventoryAuthoritative() || !serverInventoryAuthoritative()) return+ if (!contextInventoryAuthoritative() || !serverInventoryAuthoritative()) {+ initialConvergenceEffectsDroppedTotal.inc({+ lane: 'NetworkPolicy',+ kind: 'context',+ })+ return+ }await work()}),runServerEffect: (serverName, work) => {@@ -3077,7 +3120,13 @@ export class McpServerWatcher implements McpServerProvider {return this.enqueueMcpServerReconciliation(selected ?? { name: serverName, namespace: config.namespace },async () => {- if (!serverInventoryAuthoritative()) return+ if (!serverInventoryAuthoritative()) {+ initialConvergenceEffectsDroppedTotal.inc({+ lane: 'NetworkPolicy',+ kind: 'server',+ })+ return+ }await work()})@@ -3105,12 +3154,19 @@ export class McpServerWatcher implements McpServerProvider {void this.runInitialMcpServerConvergence()},})- if (!contextInventoryAuthoritative() || !serverInventoryAuthoritative()) return+ if (!contextInventoryAuthoritative() || !serverInventoryAuthoritative()) {+ console.error('[K8s] pass ended without certifying: inventory authority lost')+ initialConvergenceSwallowedTotal.inc({ lane: 'NetworkPolicy', sink: 'authority-lost' })+ observeInitialNetworkPolicyPass(startedAtMs, 'aborted-authority')+ this.scheduleInitialConvergenceRetry('NetworkPolicy')+ return+ }initialConvergenceLastSuccessTimestampSeconds.set({ lane: 'NetworkPolicy' },Date.now() / 1000)this.clearInitialConvergenceRetry('NetworkPolicy')+ observeInitialNetworkPolicyPass(startedAtMs, 'certified')} catch (error) {console.error(authoritativeRevocationCompleted@@ -3118,6 +3174,9 @@ export class McpServerWatcher implements McpServerProvider {: '[K8s] Initial NetworkPolicy background reconciliation failed:',error)+ const abortedBump =+ error instanceof Error && error.message === DESIRED_INVENTORY_CHANGED_MESSAGE+ observeInitialNetworkPolicyPass(startedAtMs, abortedBump ? 'aborted-bump' : 'failed')this.scheduleInitialConvergenceRetry('NetworkPolicy')}}diff --git a/host-context-controller/src/metrics.ts b/host-context-controller/src/metrics.tsindex 5046b0ff6..eb61fee25 100644--- a/host-context-controller/src/metrics.ts+++ b/host-context-controller/src/metrics.ts@@ -91,6 +91,30 @@ export const initialConvergenceLastSuccessTimestampSeconds = gauge({labelNames: ['lane'] as const,})+export const initialConvergenceSwallowedTotal = counter({+ name: 'clerum_hcc_initial_convergence_swallowed_total',+ help: 'Initial background convergence requests that returned without certifying.',+ labelNames: ['lane', 'sink'] as const,+})++export const initialConvergenceEffectsDroppedTotal = counter({+ name: 'clerum_hcc_initial_convergence_effects_dropped_total',+ help: 'Positive NetworkPolicy effects dropped because the pass inventory lease was retired.',+ labelNames: ['lane', 'kind'] as const,+})++export const initialConvergencePassResultsTotal = counter({+ name: 'clerum_hcc_initial_convergence_pass_results_total',+ help: 'Named outcomes of initial background convergence passes.',+ labelNames: ['lane', 'result'] as const,+})++export const initialConvergencePassDurationSeconds = histogram({+ name: 'clerum_hcc_initial_convergence_pass_duration_seconds',+ help: 'Seconds spent in an initial background convergence pass, labeled by named result.',+ labelNames: ['lane', 'result'] as const,+})+export const networkPolicySafetyPassDurationSeconds = histogram({name: 'clerum_hcc_networkpolicy_safety_pass_duration_seconds',help: 'Seconds until an authoritative NetworkPolicy safety pass has revoked stale allows.',