Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript

fix(hcc): re-arm silent NetworkPolicy convergence sinks and name pass outcomes (#447)

Alfredo Lopez0692850fe2b8
diff --git a/host-context-controller/package-lock.json b/host-context-controller/package-lock.json
index 98f09c301..b75c7ef2b 100644
--- a/host-context-controller/package-lock.json
+++ b/host-context-controller/package-lock.json
@@ -1,12 +1,12 @@
{
"name": "clerum-host-context-controller",
- "version": "0.1.235",
+ "version": "0.1.236",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "clerum-host-context-controller",
- "version": "0.1.235",
+ "version": "0.1.236",
"license": "SEE LICENSE IN LICENSE",
"dependencies": {
"@clerum/image-policy": "file:../packages/image-policy",
diff --git a/host-context-controller/package.json b/host-context-controller/package.json
index 7fc86e36c..b02f8befc 100644
--- a/host-context-controller/package.json
+++ b/host-context-controller/package.json
@@ -1,6 +1,6 @@
{
"name": "clerum-host-context-controller",
- "version": "0.1.235",
+ "version": "0.1.236",
"description": "Host Context Controller - provides McpServer CRDs to mcp-host via REST API",
"main": "dist/main.js",
"engines": {
diff --git a/host-context-controller/src/k8sClient.test.ts b/host-context-controller/src/k8sClient.test.ts
index 47fa0acbc..336361dba 100644
--- a/host-context-controller/src/k8sClient.test.ts
+++ b/host-context-controller/src/k8sClient.test.ts
@@ -49,6 +49,23 @@ async function readInitialConvergenceMetric(
return snapshot.values.find(entry => entry.labels.lane === lane)?.value ?? 0
}
+async function readLabeledConvergenceMetric(
+ name:
+ | 'clerum_hcc_initial_convergence_swallowed_total'
+ | 'clerum_hcc_initial_convergence_effects_dropped_total'
+ | 'clerum_hcc_initial_convergence_pass_results_total',
+ labels: Record<string, string>
+): Promise<number> {
+ const metric = registry.getSingleMetric(name)
+ if (!metric) throw new Error(`${name} is not registered`)
+ const snapshot = await metric.get()
+ return (
+ snapshot.values.find(entry =>
+ Object.entries(labels).every(([key, value]) => entry.labels[key] === value)
+ )?.value ?? 0
+ )
+}
+
async function requestReadyOverHttp(server: ContextMapperServer): Promise<{
statusCode: number | undefined
body: string
@@ -3908,6 +3925,11 @@ describe('McpServerWatcher startup', () => {
transport: { type: 'streamableHttp' as const, port: 3000 },
},
})
+ const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {})
+ const successTimestampBefore = await readInitialConvergenceMetric(
+ 'clerum_hcc_initial_convergence_last_success_timestamp_seconds',
+ 'NetworkPolicy'
+ )
mocks.netPolFullReconcile.mockImplementation(async (contexts, servers, options) => {
pass += 1
if (pass === 1) {
@@ -3942,7 +3964,37 @@ describe('McpServerWatcher startup', () => {
expect(appliedContexts).toEqual([])
expect(appliedServers).toEqual([])
- expect((watcher as any).initialConvergenceRetryAttempts.get('NetworkPolicy')).toBe(4)
+ expect(
+ errorSpy.mock.calls.some(call =>
+ String(call[0]).includes('pass ended without certifying: inventory authority lost')
+ )
+ ).toBe(true)
+ expect((watcher as any).initialConvergenceRetryAttempts.get('NetworkPolicy')).toBe(5)
+ expect((watcher as any).initialConvergenceRetryTimers.has('NetworkPolicy')).toBe(true)
+ expect(
+ await readInitialConvergenceMetric(
+ 'clerum_hcc_initial_convergence_last_success_timestamp_seconds',
+ 'NetworkPolicy'
+ )
+ ).toBe(successTimestampBefore)
+ expect(
+ await readLabeledConvergenceMetric('clerum_hcc_initial_convergence_swallowed_total', {
+ lane: 'NetworkPolicy',
+ sink: 'authority-lost',
+ })
+ ).toBeGreaterThan(0)
+ expect(
+ await readLabeledConvergenceMetric('clerum_hcc_initial_convergence_pass_results_total', {
+ lane: 'NetworkPolicy',
+ result: 'aborted-authority',
+ })
+ ).toBeGreaterThan(0)
+ expect(
+ await readLabeledConvergenceMetric('clerum_hcc_initial_convergence_effects_dropped_total', {
+ lane: 'NetworkPolicy',
+ kind: 'context',
+ })
+ ).toBeGreaterThan(0)
;(watcher as any).contexts.clear()
;(watcher as any).servers.clear()
;(watcher as any).contexts.set('current-context', {
@@ -3966,6 +4018,7 @@ describe('McpServerWatcher startup', () => {
expect(appliedContexts).toEqual(['current-context'])
expect(appliedServers).toEqual(['current-server'])
expect((watcher as any).initialConvergenceRetryAttempts.has('NetworkPolicy')).toBe(false)
+ errorSpy.mockRestore()
await watcher.stop()
})
@@ -4298,6 +4351,7 @@ describe('McpServerWatcher startup', () => {
})
it('defers NetworkPolicy full convergence until both inventories are authoritative', async () => {
+ const warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => {})
const watcher = new McpServerWatcher()
;(watcher as any).contextCacheSynced = true
;(watcher as any).mcpServerCacheSynced = false
@@ -4306,7 +4360,21 @@ describe('McpServerWatcher startup', () => {
await (watcher as any).runInitialNetworkPolicyConvergence()
expect(mocks.netPolFullReconcile).not.toHaveBeenCalled()
- expect((watcher as any).initialConvergenceRetryAttempts.get('NetworkPolicy')).toBe(2)
+ expect(warnSpy.mock.calls.some(call => String(call[0]).includes('caches unsynced'))).toBe(true)
+ expect((watcher as any).initialConvergenceRetryAttempts.get('NetworkPolicy')).toBe(3)
+ expect((watcher as any).initialConvergenceRetryTimers.has('NetworkPolicy')).toBe(true)
+ expect(
+ await readLabeledConvergenceMetric('clerum_hcc_initial_convergence_swallowed_total', {
+ lane: 'NetworkPolicy',
+ sink: 'unsynced',
+ })
+ ).toBeGreaterThan(0)
+ expect(
+ await readLabeledConvergenceMetric('clerum_hcc_initial_convergence_pass_results_total', {
+ lane: 'NetworkPolicy',
+ result: 'deferred-unsynced',
+ })
+ ).toBeGreaterThan(0)
;(watcher as any).mcpServerCacheSynced = true
await (watcher as any).runInitialNetworkPolicyConvergence()
@@ -4322,6 +4390,94 @@ describe('McpServerWatcher startup', () => {
})
)
expect((watcher as any).initialConvergenceRetryAttempts.has('NetworkPolicy')).toBe(false)
+ warnSpy.mockRestore()
+ await watcher.stop()
+ })
+
+ it('re-runs NetworkPolicy convergence from the retry ladder after a silent-sink miss without later watch events', async () => {
+ vi.useFakeTimers()
+ const warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => {})
+ const watcher = new McpServerWatcher()
+ ;(watcher as any).contextCacheSynced = true
+ ;(watcher as any).mcpServerCacheSynced = false
+
+ await (watcher as any).runInitialNetworkPolicyConvergence()
+
+ expect(mocks.netPolFullReconcile).not.toHaveBeenCalled()
+ expect((watcher as any).initialConvergenceRetryAttempts.get('NetworkPolicy')).toBe(1)
+ expect((watcher as any).initialConvergenceRetryTimers.has('NetworkPolicy')).toBe(true)
+ ;(watcher as any).mcpServerCacheSynced = true
+ await vi.advanceTimersByTimeAsync(5000)
+
+ expect(mocks.netPolFullReconcile).toHaveBeenCalledOnce()
+ expect((watcher as any).initialConvergenceRetryAttempts.has('NetworkPolicy')).toBe(false)
+ expect(
+ await readLabeledConvergenceMetric('clerum_hcc_initial_convergence_pass_results_total', {
+ lane: 'NetworkPolicy',
+ result: 'certified',
+ })
+ ).toBeGreaterThan(0)
+ warnSpy.mockRestore()
+ await watcher.stop()
+ })
+
+ it('names a generic NetworkPolicy catch failure failed instead of aborted-bump', async () => {
+ const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {})
+ const watcher = new McpServerWatcher()
+ ;(watcher as any).contextCacheSynced = true
+ ;(watcher as any).mcpServerCacheSynced = true
+ mocks.netPolFullReconcile.mockRejectedValueOnce(new Error('apiserver 5xx'))
+ const failedBefore = await readLabeledConvergenceMetric(
+ 'clerum_hcc_initial_convergence_pass_results_total',
+ { lane: 'NetworkPolicy', result: 'failed' }
+ )
+ const bumpBefore = await readLabeledConvergenceMetric(
+ 'clerum_hcc_initial_convergence_pass_results_total',
+ { lane: 'NetworkPolicy', result: 'aborted-bump' }
+ )
+
+ await (watcher as any).runInitialNetworkPolicyConvergence()
+
+ expect(
+ await readLabeledConvergenceMetric('clerum_hcc_initial_convergence_pass_results_total', {
+ lane: 'NetworkPolicy',
+ result: 'failed',
+ })
+ ).toBe(failedBefore + 1)
+ expect(
+ await readLabeledConvergenceMetric('clerum_hcc_initial_convergence_pass_results_total', {
+ lane: 'NetworkPolicy',
+ result: 'aborted-bump',
+ })
+ ).toBe(bumpBefore)
+ expect((watcher as any).initialConvergenceRetryAttempts.get('NetworkPolicy')).toBe(1)
+ errorSpy.mockRestore()
+ await watcher.stop()
+ })
+
+ it('names the inventory-changed throw aborted-bump', async () => {
+ const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {})
+ const watcher = new McpServerWatcher()
+ ;(watcher as any).contextCacheSynced = true
+ ;(watcher as any).mcpServerCacheSynced = true
+ mocks.netPolFullReconcile.mockRejectedValueOnce(
+ new Error('Desired NetworkPolicy inventory changed during authoritative revocation')
+ )
+ const bumpBefore = await readLabeledConvergenceMetric(
+ 'clerum_hcc_initial_convergence_pass_results_total',
+ { lane: 'NetworkPolicy', result: 'aborted-bump' }
+ )
+
+ await (watcher as any).runInitialNetworkPolicyConvergence()
+
+ expect(
+ await readLabeledConvergenceMetric('clerum_hcc_initial_convergence_pass_results_total', {
+ lane: 'NetworkPolicy',
+ result: 'aborted-bump',
+ })
+ ).toBe(bumpBefore + 1)
+ expect((watcher as any).initialConvergenceRetryAttempts.get('NetworkPolicy')).toBe(1)
+ errorSpy.mockRestore()
await watcher.stop()
})
diff --git a/host-context-controller/src/k8sClient.ts b/host-context-controller/src/k8sClient.ts
index cd2475957..171c982a6 100644
--- a/host-context-controller/src/k8sClient.ts
+++ b/host-context-controller/src/k8sClient.ts
@@ -58,8 +58,12 @@ import {
hostDeleteCleanupTotal,
hostFleetRequestsTotal,
hostWatchRecoverySeconds,
+ initialConvergenceEffectsDroppedTotal,
initialConvergenceLastSuccessTimestampSeconds,
+ initialConvergencePassDurationSeconds,
+ initialConvergencePassResultsTotal,
initialConvergenceRetriesTotal,
+ initialConvergenceSwallowedTotal,
} from './metrics'
import { NetworkPolicyReconciler, sameContextDesiredRevision } from './networkPolicyReconciler'
import type { ReadinessInventoryDetail } from './readinessGate'
@@ -218,6 +222,24 @@ type HostSnapshot = {
type HostInventoryRecoveryCause = 'cold-start' | 'watch-recovery'
type HostWatchEventType = 'ADDED' | 'MODIFIED' | 'DELETED'
type InitialConvergenceLane = 'McpServer' | 'NetworkPolicy'
+type InitialConvergencePassResult =
+ | 'certified'
+ | 'aborted-authority'
+ | 'deferred-unsynced'
+ | 'aborted-bump'
+ | 'failed'
+
+const DESIRED_INVENTORY_CHANGED_MESSAGE =
+ 'Desired NetworkPolicy inventory changed during authoritative revocation'
+
+function observeInitialNetworkPolicyPass(
+ startedAtMs: number,
+ result: InitialConvergencePassResult
+): void {
+ const seconds = Math.max(0, (Date.now() - startedAtMs) / 1000)
+ initialConvergencePassResultsTotal.inc({ lane: 'NetworkPolicy', result })
+ initialConvergencePassDurationSeconds.observe({ lane: 'NetworkPolicy', result }, seconds)
+}
type NetworkPolicySafetyCertificate = {
// VESTIGIAL — retained for record shape only, never read in any decision.
@@ -3043,7 +3065,22 @@ export class McpServerWatcher implements McpServerProvider {
// coordinator. Running safety from a partial inventory would be neither a
// full convergence nor safe evidence of success. Recovery of either
// missing LIST -> WATCH pair schedules a fresh current-cache pass.
- if (!this.contextCacheSynced || !this.mcpServerCacheSynced) return
+ const startedAtMs = Date.now()
+ if (!this.contextCacheSynced || !this.mcpServerCacheSynced) {
+ const unsynced = [
+ !this.contextCacheSynced ? 'Context' : undefined,
+ !this.mcpServerCacheSynced ? 'McpServer' : undefined,
+ ]
+ .filter((name): name is string => name !== undefined)
+ .join(' and ')
+ console.warn(
+ `[K8s] NetworkPolicy convergence request deferred: caches unsynced (${unsynced})`
+ )
+ initialConvergenceSwallowedTotal.inc({ lane: 'NetworkPolicy', sink: 'unsynced' })
+ observeInitialNetworkPolicyPass(startedAtMs, 'deferred-unsynced')
+ this.scheduleInitialConvergenceRetry('NetworkPolicy')
+ return
+ }
let authoritativeRevocationCompleted = false
try {
const initialContexts = [...this.contexts.values()]
@@ -3069,7 +3106,13 @@ export class McpServerWatcher implements McpServerProvider {
serverInventoryAuthoritative,
runContextEffect: (contextId, work) =>
this.enqueueContextReconciliation(contextId, async () => {
- if (!contextInventoryAuthoritative() || !serverInventoryAuthoritative()) return
+ if (!contextInventoryAuthoritative() || !serverInventoryAuthoritative()) {
+ initialConvergenceEffectsDroppedTotal.inc({
+ lane: 'NetworkPolicy',
+ kind: 'context',
+ })
+ return
+ }
await work()
}),
runServerEffect: (serverName, work) => {
@@ -3077,7 +3120,13 @@ export class McpServerWatcher implements McpServerProvider {
return this.enqueueMcpServerReconciliation(
selected ?? { name: serverName, namespace: config.namespace },
async () => {
- if (!serverInventoryAuthoritative()) return
+ if (!serverInventoryAuthoritative()) {
+ initialConvergenceEffectsDroppedTotal.inc({
+ lane: 'NetworkPolicy',
+ kind: 'server',
+ })
+ return
+ }
await work()
}
)
@@ -3105,12 +3154,19 @@ export class McpServerWatcher implements McpServerProvider {
void this.runInitialMcpServerConvergence()
},
})
- if (!contextInventoryAuthoritative() || !serverInventoryAuthoritative()) return
+ if (!contextInventoryAuthoritative() || !serverInventoryAuthoritative()) {
+ console.error('[K8s] pass ended without certifying: inventory authority lost')
+ initialConvergenceSwallowedTotal.inc({ lane: 'NetworkPolicy', sink: 'authority-lost' })
+ observeInitialNetworkPolicyPass(startedAtMs, 'aborted-authority')
+ this.scheduleInitialConvergenceRetry('NetworkPolicy')
+ return
+ }
initialConvergenceLastSuccessTimestampSeconds.set(
{ lane: 'NetworkPolicy' },
Date.now() / 1000
)
this.clearInitialConvergenceRetry('NetworkPolicy')
+ observeInitialNetworkPolicyPass(startedAtMs, 'certified')
} catch (error) {
console.error(
authoritativeRevocationCompleted
@@ -3118,6 +3174,9 @@ export class McpServerWatcher implements McpServerProvider {
: '[K8s] Initial NetworkPolicy background reconciliation failed:',
error
)
+ const abortedBump =
+ error instanceof Error && error.message === DESIRED_INVENTORY_CHANGED_MESSAGE
+ observeInitialNetworkPolicyPass(startedAtMs, abortedBump ? 'aborted-bump' : 'failed')
this.scheduleInitialConvergenceRetry('NetworkPolicy')
}
}
diff --git a/host-context-controller/src/metrics.ts b/host-context-controller/src/metrics.ts
index 5046b0ff6..eb61fee25 100644
--- a/host-context-controller/src/metrics.ts
+++ b/host-context-controller/src/metrics.ts
@@ -91,6 +91,30 @@ export const initialConvergenceLastSuccessTimestampSeconds = gauge({
labelNames: ['lane'] as const,
})
+export const initialConvergenceSwallowedTotal = counter({
+ name: 'clerum_hcc_initial_convergence_swallowed_total',
+ help: 'Initial background convergence requests that returned without certifying.',
+ labelNames: ['lane', 'sink'] as const,
+})
+
+export const initialConvergenceEffectsDroppedTotal = counter({
+ name: 'clerum_hcc_initial_convergence_effects_dropped_total',
+ help: 'Positive NetworkPolicy effects dropped because the pass inventory lease was retired.',
+ labelNames: ['lane', 'kind'] as const,
+})
+
+export const initialConvergencePassResultsTotal = counter({
+ name: 'clerum_hcc_initial_convergence_pass_results_total',
+ help: 'Named outcomes of initial background convergence passes.',
+ labelNames: ['lane', 'result'] as const,
+})
+
+export const initialConvergencePassDurationSeconds = histogram({
+ name: 'clerum_hcc_initial_convergence_pass_duration_seconds',
+ help: 'Seconds spent in an initial background convergence pass, labeled by named result.',
+ labelNames: ['lane', 'result'] as const,
+})
+
export const networkPolicySafetyPassDurationSeconds = histogram({
name: 'clerum_hcc_networkpolicy_safety_pass_duration_seconds',
help: 'Seconds until an authoritative NetworkPolicy safety pass has revoked stale allows.',