Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript

fix(hcc): pin defaults-only write-set disjoint and tick observability

Alfredo Lopez32a4e8d1e12d
diff --git a/host-context-controller/package-lock.json b/host-context-controller/package-lock.json
index 77b3761be..1393c6c3b 100644
--- a/host-context-controller/package-lock.json
+++ b/host-context-controller/package-lock.json
@@ -1,12 +1,12 @@
{
"name": "clerum-host-context-controller",
- "version": "0.1.265",
+ "version": "0.1.266",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "clerum-host-context-controller",
- "version": "0.1.265",
+ "version": "0.1.266",
"license": "SEE LICENSE IN LICENSE",
"dependencies": {
"@clerum/image-policy": "file:../packages/image-policy",
diff --git a/host-context-controller/package.json b/host-context-controller/package.json
index f3e7f96c1..b178f5fd1 100644
--- a/host-context-controller/package.json
+++ b/host-context-controller/package.json
@@ -1,6 +1,6 @@
{
"name": "clerum-host-context-controller",
- "version": "0.1.265",
+ "version": "0.1.266",
"description": "Host Context Controller - provides McpServer CRDs to mcp-host via REST API",
"main": "dist/main.js",
"engines": {
diff --git a/host-context-controller/src/k8sClient.test.ts b/host-context-controller/src/k8sClient.test.ts
index 3a2265f62..16d1c2a8a 100644
--- a/host-context-controller/src/k8sClient.test.ts
+++ b/host-context-controller/src/k8sClient.test.ts
@@ -55,7 +55,8 @@ async function readLabeledConvergenceMetric(
| 'clerum_hcc_initial_convergence_swallowed_total'
| 'clerum_hcc_initial_convergence_effects_dropped_total'
| 'clerum_hcc_initial_convergence_pass_results_total'
- | 'clerum_hcc_netpol_resync_ticks_skipped_total',
+ | 'clerum_hcc_netpol_resync_ticks_skipped_total'
+ | 'clerum_hcc_netpol_defaults_only_ticks_total',
labels: Record<string, string>
): Promise<number> {
const metric = registry.getSingleMetric(name)
@@ -11532,9 +11533,18 @@ describe('McpServerWatcher NetworkPolicy defaults-only tick (#488)', () => {
)
).toBe(true)
+ const successBefore = await readLabeledConvergenceMetric(
+ 'clerum_hcc_netpol_defaults_only_ticks_total',
+ { result: 'success' }
+ )
await vi.advanceTimersByTimeAsync(60_000)
expect(mocks.ensureDefaultPolicies).toHaveBeenCalledTimes(2)
expect(mocks.netPolFullReconcile).toHaveBeenCalledTimes(1)
+ expect(
+ await readLabeledConvergenceMetric('clerum_hcc_netpol_defaults_only_ticks_total', {
+ result: 'success',
+ })
+ ).toBe(successBefore + 1)
logSpy.mockRestore()
await watcher.stop()
})
@@ -11639,6 +11649,11 @@ describe('McpServerWatcher NetworkPolicy defaults-only tick (#488)', () => {
)
).toBe(true)
expect((watcher as any).netPolDefaultsOnlyRun).toBeNull()
+ expect(
+ await readLabeledConvergenceMetric('clerum_hcc_netpol_defaults_only_ticks_total', {
+ result: 'error',
+ })
+ ).toBeGreaterThan(0)
expect(
await readLabeledConvergenceMetric('clerum_hcc_netpol_resync_ticks_skipped_total', {
reason: 'defaults-only-in-flight',
@@ -11723,7 +11738,31 @@ describe('McpServerWatcher NetworkPolicy defaults-only tick (#488)', () => {
await watcher.stop()
expect((watcher as any).netPolDefaultsResyncTimer).toBeNull()
+ expect((watcher as any).netPolDefaultsOnlyRun).toBeNull()
await vi.advanceTimersByTimeAsync(60_000)
expect(mocks.ensureDefaultPolicies).toHaveBeenCalledTimes(defaultsAfterStart)
})
+
+ it('stop() drops the in-flight lock so the next tick does not join a stale run', async () => {
+ vi.useFakeTimers()
+ mockConfig.netPolDefaultsResyncIntervalSec = 60
+ const watcher = await startWatcherForDefaultsOnlyTick()
+ const held = deferred()
+ mocks.ensureDefaultPolicies.mockImplementation(() => held.promise)
+ void (watcher as any).runNetworkPolicyDefaultsOnly()
+ await flushMicrotasks()
+ expect((watcher as any).netPolDefaultsOnlyRun).not.toBeNull()
+ const callsWhileHeld = mocks.ensureDefaultPolicies.mock.calls.length
+
+ await watcher.stop()
+ expect((watcher as any).netPolDefaultsOnlyRun).toBeNull()
+
+ mocks.ensureDefaultPolicies.mockResolvedValue(undefined)
+ await (watcher as any).runNetworkPolicyDefaultsOnly()
+ expect(mocks.ensureDefaultPolicies).toHaveBeenCalledTimes(callsWhileHeld + 1)
+ expect((watcher as any).netPolDefaultsOnlyRun).toBeNull()
+
+ held.resolve()
+ await flushMicrotasks()
+ })
})
diff --git a/host-context-controller/src/k8sClient.ts b/host-context-controller/src/k8sClient.ts
index 5ec53e040..0e48c58a7 100644
--- a/host-context-controller/src/k8sClient.ts
+++ b/host-context-controller/src/k8sClient.ts
@@ -64,6 +64,8 @@ import {
initialConvergencePassResultsTotal,
initialConvergenceRetriesTotal,
initialConvergenceSwallowedTotal,
+ netPolDefaultsOnlyTickDurationSeconds,
+ netPolDefaultsOnlyTicksTotal,
netPolResyncTicksSkippedTotal,
} from './metrics'
import {
@@ -3090,6 +3092,8 @@ export class McpServerWatcher implements McpServerProvider {
* Overlapping defaults-only ticks serialize: a second tick increments
* `netPolResyncTicksSkippedTotal{reason="defaults-only-in-flight"}` and
* joins the in-flight run. Never increment `pass-in-flight` from here.
+ * Each executed tick records `netPolDefaultsOnlyTicksTotal` and
+ * `netPolDefaultsOnlyTickDurationSeconds` with result success|error.
*/
private runNetworkPolicyDefaultsOnly(): Promise<void> {
if (this.netPolDefaultsOnlyRun) {
@@ -3097,10 +3101,17 @@ export class McpServerWatcher implements McpServerProvider {
return this.netPolDefaultsOnlyRun
}
const run = (async () => {
+ const startedAtMs = Date.now()
+ let result: 'success' | 'error' = 'success'
try {
await this.netPolReconciler.ensureDefaultPolicies()
} catch (error) {
+ result = 'error'
console.error('[K8s] NetworkPolicy defaults-only tick failed:', error)
+ } finally {
+ const seconds = Math.max(0, (Date.now() - startedAtMs) / 1000)
+ netPolDefaultsOnlyTicksTotal.inc({ result })
+ netPolDefaultsOnlyTickDurationSeconds.observe({ result }, seconds)
}
})()
this.netPolDefaultsOnlyRun = run
@@ -4647,6 +4658,7 @@ export class McpServerWatcher implements McpServerProvider {
clearInterval(this.netPolDefaultsResyncTimer)
this.netPolDefaultsResyncTimer = null
}
+ this.netPolDefaultsOnlyRun = null
this.netPolConvergenceEnsureDefaults = false
for (const timer of this.initialConvergenceRetryTimers.values()) {
clearTimeout(timer)
diff --git a/host-context-controller/src/metrics.ts b/host-context-controller/src/metrics.ts
index c6e0b26a1..b599259e9 100644
--- a/host-context-controller/src/metrics.ts
+++ b/host-context-controller/src/metrics.ts
@@ -155,6 +155,19 @@ export const netPolResyncTicksSkippedTotal = counter({
labelNames: ['reason'] as const,
})
+export const netPolDefaultsOnlyTicksTotal = counter({
+ name: 'clerum_hcc_netpol_defaults_only_ticks_total',
+ help: 'NetworkPolicy defaults-only ticks by named result (success/error).',
+ labelNames: ['result'] as const,
+})
+
+export const netPolDefaultsOnlyTickDurationSeconds = histogram({
+ name: 'clerum_hcc_netpol_defaults_only_tick_duration_seconds',
+ help: 'Seconds spent in a NetworkPolicy defaults-only tick, labeled by named result.',
+ labelNames: ['result'] as const,
+ buckets: NETWORKPOLICY_PASS_DURATION_BUCKETS,
+})
+
export const contextReconciliationsTotal = counter({
name: 'clerum_hcc_context_reconciliations_total',
help: 'Total Context CRD reconciliations',
diff --git a/host-context-controller/src/networkPolicyReconciler.test.ts b/host-context-controller/src/networkPolicyReconciler.test.ts
index ddf27b08a..6ba7d141a 100644
--- a/host-context-controller/src/networkPolicyReconciler.test.ts
+++ b/host-context-controller/src/networkPolicyReconciler.test.ts
@@ -6,6 +6,7 @@ import { readFileSync } from 'node:fs'
import { join } from 'node:path'
import { asApiserverNetworkPolicy } from './__tests__/asApiserverNetworkPolicy'
import { config } from './config'
+import { MANAGED_BY_LABEL, MANAGED_BY_VALUE, MCPSERVER_LABEL, POLICY_TYPE_LABEL } from './constants'
import { confirmAuthoritativeMcpServerAbsence } from './mcpServerSafety'
import {
networkPolicySafetyPassDurationSeconds,
@@ -5970,4 +5971,115 @@ describe('NetworkPolicyReconciler', () => {
expect(networkPolicySafetyPassDurationSeconds.observe).toHaveBeenCalledTimes(1)
})
})
+
+ describe('defaults-only write set vs typed sweep (#488)', () => {
+ const CONTEXT_LABEL = 'clerum.io/context'
+ const TYPED_SWEEP_TYPES = new Set(['context-allow', 'rpc-proxy-egress', 'external-egress'])
+
+ function policyMatchesLabelSelector(
+ policy: k8s.V1NetworkPolicy,
+ selector: string | undefined
+ ): boolean {
+ if (!selector) return true
+ const labels = policy.metadata?.labels ?? {}
+ return selector.split(',').every(clause => {
+ const trimmed = clause.trim()
+ const eq = trimmed.indexOf('=')
+ if (eq < 0) return false
+ return labels[trimmed.slice(0, eq)] === trimmed.slice(eq + 1)
+ })
+ }
+
+ it('ensureDefaultPolicies writes a {name, policy-type} set the typed sweep does not list or delete', async () => {
+ const store = new Map<string, k8s.V1NetworkPolicy>()
+ const deleted: string[] = []
+
+ mockApi.createNamespacedNetworkPolicy.mockImplementation(
+ async (args: { namespace: string; body: k8s.V1NetworkPolicy }) => {
+ const name = args.body.metadata?.name
+ if (!name) throw new Error('create is missing metadata.name')
+ const key = `${args.namespace}/${name}`
+ const stored: k8s.V1NetworkPolicy = {
+ ...args.body,
+ metadata: {
+ ...args.body.metadata,
+ name,
+ namespace: args.namespace,
+ uid: key,
+ resourceVersion: '1',
+ },
+ }
+ store.set(key, stored)
+ return stored
+ }
+ )
+ mockApi.readNamespacedNetworkPolicy.mockImplementation(
+ async (args: { name: string; namespace: string }) => {
+ const existing = store.get(`${args.namespace}/${args.name}`)
+ if (!existing) {
+ throw Object.assign(new Error('not found'), { code: 404 })
+ }
+ return existing
+ }
+ )
+ mockApi.listNamespacedNetworkPolicy.mockImplementation(
+ async (args: { namespace: string; labelSelector?: string }) => ({
+ items: [...store.values()].filter(
+ policy =>
+ policy.metadata?.namespace === args.namespace &&
+ policyMatchesLabelSelector(policy, args.labelSelector)
+ ),
+ })
+ )
+ mockApi.deleteNamespacedNetworkPolicy.mockImplementation(
+ async (args: { name: string; namespace: string }) => {
+ const key = `${args.namespace}/${args.name}`
+ deleted.push(key)
+ store.delete(key)
+ return {}
+ }
+ )
+ mockCustomApi.listNamespacedCustomObject.mockResolvedValue({ items: [] })
+
+ await reconciler.ensureDefaultPolicies()
+ expect(store.size).toBeGreaterThan(0)
+ const written = [...store.values()].map(policy => ({
+ key: `${policy.metadata?.namespace}/${policy.metadata?.name}`,
+ name: policy.metadata?.name ?? '',
+ type: policy.metadata?.labels?.[POLICY_TYPE_LABEL] ?? '',
+ }))
+ expect(written.every(item => !TYPED_SWEEP_TYPES.has(item.type))).toBe(true)
+ expect(new Set(written.map(item => item.type))).toEqual(
+ new Set(['allow-api', 'default-deny', 'infrastructure'])
+ )
+
+ const orphanKey = `${config.namespace}/ctx-gone-allow`
+ store.set(orphanKey, {
+ apiVersion: 'networking.k8s.io/v1',
+ kind: 'NetworkPolicy',
+ metadata: {
+ name: 'ctx-gone-allow',
+ namespace: config.namespace,
+ uid: orphanKey,
+ resourceVersion: '1',
+ labels: {
+ [MANAGED_BY_LABEL]: MANAGED_BY_VALUE,
+ [POLICY_TYPE_LABEL]: 'context-allow',
+ [CONTEXT_LABEL]: 'ctx-gone',
+ [MCPSERVER_LABEL]: 'srv-gone',
+ },
+ },
+ spec: { podSelector: {}, policyTypes: ['Ingress'] },
+ })
+
+ await reconciler.fullReconcile([], [], { ensureDefaults: false })
+
+ expect(deleted).toContain(orphanKey)
+ expect(deleted.filter(key => written.some(item => item.key === key))).toEqual([])
+ for (const item of written) {
+ expect(store.has(item.key)).toBe(true)
+ }
+ expect(store.has(orphanKey)).toBe(false)
+ })
+ })
})