Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
fix(hcc): pin defaults-only write-set disjoint and tick observability
Alfredo Lopez32a4e8d1e12d
diff --git a/host-context-controller/package-lock.json b/host-context-controller/package-lock.jsonindex 77b3761be..1393c6c3b 100644--- a/host-context-controller/package-lock.json+++ b/host-context-controller/package-lock.json@@ -1,12 +1,12 @@{"name": "clerum-host-context-controller",- "version": "0.1.265",+ "version": "0.1.266","lockfileVersion": 3,"requires": true,"packages": {"": {"name": "clerum-host-context-controller",- "version": "0.1.265",+ "version": "0.1.266","license": "SEE LICENSE IN LICENSE","dependencies": {"@clerum/image-policy": "file:../packages/image-policy",diff --git a/host-context-controller/package.json b/host-context-controller/package.jsonindex f3e7f96c1..b178f5fd1 100644--- a/host-context-controller/package.json+++ b/host-context-controller/package.json@@ -1,6 +1,6 @@{"name": "clerum-host-context-controller",- "version": "0.1.265",+ "version": "0.1.266","description": "Host Context Controller - provides McpServer CRDs to mcp-host via REST API","main": "dist/main.js","engines": {diff --git a/host-context-controller/src/k8sClient.test.ts b/host-context-controller/src/k8sClient.test.tsindex 3a2265f62..16d1c2a8a 100644--- a/host-context-controller/src/k8sClient.test.ts+++ b/host-context-controller/src/k8sClient.test.ts@@ -55,7 +55,8 @@ async function readLabeledConvergenceMetric(| 'clerum_hcc_initial_convergence_swallowed_total'| 'clerum_hcc_initial_convergence_effects_dropped_total'| 'clerum_hcc_initial_convergence_pass_results_total'- | 'clerum_hcc_netpol_resync_ticks_skipped_total',+ | 'clerum_hcc_netpol_resync_ticks_skipped_total'+ | 'clerum_hcc_netpol_defaults_only_ticks_total',labels: Record<string, string>): Promise<number> {const metric = registry.getSingleMetric(name)@@ -11532,9 +11533,18 @@ describe('McpServerWatcher NetworkPolicy defaults-only tick (#488)', () => {)).toBe(true)+ const successBefore = await readLabeledConvergenceMetric(+ 'clerum_hcc_netpol_defaults_only_ticks_total',+ { result: 'success' }+ )await vi.advanceTimersByTimeAsync(60_000)expect(mocks.ensureDefaultPolicies).toHaveBeenCalledTimes(2)expect(mocks.netPolFullReconcile).toHaveBeenCalledTimes(1)+ expect(+ await readLabeledConvergenceMetric('clerum_hcc_netpol_defaults_only_ticks_total', {+ result: 'success',+ })+ ).toBe(successBefore + 1)logSpy.mockRestore()await watcher.stop()})@@ -11639,6 +11649,11 @@ describe('McpServerWatcher NetworkPolicy defaults-only tick (#488)', () => {)).toBe(true)expect((watcher as any).netPolDefaultsOnlyRun).toBeNull()+ expect(+ await readLabeledConvergenceMetric('clerum_hcc_netpol_defaults_only_ticks_total', {+ result: 'error',+ })+ ).toBeGreaterThan(0)expect(await readLabeledConvergenceMetric('clerum_hcc_netpol_resync_ticks_skipped_total', {reason: 'defaults-only-in-flight',@@ -11723,7 +11738,31 @@ describe('McpServerWatcher NetworkPolicy defaults-only tick (#488)', () => {await watcher.stop()expect((watcher as any).netPolDefaultsResyncTimer).toBeNull()+ expect((watcher as any).netPolDefaultsOnlyRun).toBeNull()await vi.advanceTimersByTimeAsync(60_000)expect(mocks.ensureDefaultPolicies).toHaveBeenCalledTimes(defaultsAfterStart)})++ it('stop() drops the in-flight lock so the next tick does not join a stale run', async () => {+ vi.useFakeTimers()+ mockConfig.netPolDefaultsResyncIntervalSec = 60+ const watcher = await startWatcherForDefaultsOnlyTick()+ const held = deferred()+ mocks.ensureDefaultPolicies.mockImplementation(() => held.promise)+ void (watcher as any).runNetworkPolicyDefaultsOnly()+ await flushMicrotasks()+ expect((watcher as any).netPolDefaultsOnlyRun).not.toBeNull()+ const callsWhileHeld = mocks.ensureDefaultPolicies.mock.calls.length++ await watcher.stop()+ expect((watcher as any).netPolDefaultsOnlyRun).toBeNull()++ mocks.ensureDefaultPolicies.mockResolvedValue(undefined)+ await (watcher as any).runNetworkPolicyDefaultsOnly()+ expect(mocks.ensureDefaultPolicies).toHaveBeenCalledTimes(callsWhileHeld + 1)+ expect((watcher as any).netPolDefaultsOnlyRun).toBeNull()++ held.resolve()+ await flushMicrotasks()+ })})diff --git a/host-context-controller/src/k8sClient.ts b/host-context-controller/src/k8sClient.tsindex 5ec53e040..0e48c58a7 100644--- a/host-context-controller/src/k8sClient.ts+++ b/host-context-controller/src/k8sClient.ts@@ -64,6 +64,8 @@ import {initialConvergencePassResultsTotal,initialConvergenceRetriesTotal,initialConvergenceSwallowedTotal,+ netPolDefaultsOnlyTickDurationSeconds,+ netPolDefaultsOnlyTicksTotal,netPolResyncTicksSkippedTotal,} from './metrics'import {@@ -3090,6 +3092,8 @@ export class McpServerWatcher implements McpServerProvider {* Overlapping defaults-only ticks serialize: a second tick increments* `netPolResyncTicksSkippedTotal{reason="defaults-only-in-flight"}` and* joins the in-flight run. Never increment `pass-in-flight` from here.+ * Each executed tick records `netPolDefaultsOnlyTicksTotal` and+ * `netPolDefaultsOnlyTickDurationSeconds` with result success|error.*/private runNetworkPolicyDefaultsOnly(): Promise<void> {if (this.netPolDefaultsOnlyRun) {@@ -3097,10 +3101,17 @@ export class McpServerWatcher implements McpServerProvider {return this.netPolDefaultsOnlyRun}const run = (async () => {+ const startedAtMs = Date.now()+ let result: 'success' | 'error' = 'success'try {await this.netPolReconciler.ensureDefaultPolicies()} catch (error) {+ result = 'error'console.error('[K8s] NetworkPolicy defaults-only tick failed:', error)+ } finally {+ const seconds = Math.max(0, (Date.now() - startedAtMs) / 1000)+ netPolDefaultsOnlyTicksTotal.inc({ result })+ netPolDefaultsOnlyTickDurationSeconds.observe({ result }, seconds)}})()this.netPolDefaultsOnlyRun = run@@ -4647,6 +4658,7 @@ export class McpServerWatcher implements McpServerProvider {clearInterval(this.netPolDefaultsResyncTimer)this.netPolDefaultsResyncTimer = null}+ this.netPolDefaultsOnlyRun = nullthis.netPolConvergenceEnsureDefaults = falsefor (const timer of this.initialConvergenceRetryTimers.values()) {clearTimeout(timer)diff --git a/host-context-controller/src/metrics.ts b/host-context-controller/src/metrics.tsindex c6e0b26a1..b599259e9 100644--- a/host-context-controller/src/metrics.ts+++ b/host-context-controller/src/metrics.ts@@ -155,6 +155,19 @@ export const netPolResyncTicksSkippedTotal = counter({labelNames: ['reason'] as const,})+export const netPolDefaultsOnlyTicksTotal = counter({+ name: 'clerum_hcc_netpol_defaults_only_ticks_total',+ help: 'NetworkPolicy defaults-only ticks by named result (success/error).',+ labelNames: ['result'] as const,+})++export const netPolDefaultsOnlyTickDurationSeconds = histogram({+ name: 'clerum_hcc_netpol_defaults_only_tick_duration_seconds',+ help: 'Seconds spent in a NetworkPolicy defaults-only tick, labeled by named result.',+ labelNames: ['result'] as const,+ buckets: NETWORKPOLICY_PASS_DURATION_BUCKETS,+})+export const contextReconciliationsTotal = counter({name: 'clerum_hcc_context_reconciliations_total',help: 'Total Context CRD reconciliations',diff --git a/host-context-controller/src/networkPolicyReconciler.test.ts b/host-context-controller/src/networkPolicyReconciler.test.tsindex ddf27b08a..6ba7d141a 100644--- a/host-context-controller/src/networkPolicyReconciler.test.ts+++ b/host-context-controller/src/networkPolicyReconciler.test.ts@@ -6,6 +6,7 @@ import { readFileSync } from 'node:fs'import { join } from 'node:path'import { asApiserverNetworkPolicy } from './__tests__/asApiserverNetworkPolicy'import { config } from './config'+import { MANAGED_BY_LABEL, MANAGED_BY_VALUE, MCPSERVER_LABEL, POLICY_TYPE_LABEL } from './constants'import { confirmAuthoritativeMcpServerAbsence } from './mcpServerSafety'import {networkPolicySafetyPassDurationSeconds,@@ -5970,4 +5971,115 @@ describe('NetworkPolicyReconciler', () => {expect(networkPolicySafetyPassDurationSeconds.observe).toHaveBeenCalledTimes(1)})})++ describe('defaults-only write set vs typed sweep (#488)', () => {+ const CONTEXT_LABEL = 'clerum.io/context'+ const TYPED_SWEEP_TYPES = new Set(['context-allow', 'rpc-proxy-egress', 'external-egress'])++ function policyMatchesLabelSelector(+ policy: k8s.V1NetworkPolicy,+ selector: string | undefined+ ): boolean {+ if (!selector) return true+ const labels = policy.metadata?.labels ?? {}+ return selector.split(',').every(clause => {+ const trimmed = clause.trim()+ const eq = trimmed.indexOf('=')+ if (eq < 0) return false+ return labels[trimmed.slice(0, eq)] === trimmed.slice(eq + 1)+ })+ }++ it('ensureDefaultPolicies writes a {name, policy-type} set the typed sweep does not list or delete', async () => {+ const store = new Map<string, k8s.V1NetworkPolicy>()+ const deleted: string[] = []++ mockApi.createNamespacedNetworkPolicy.mockImplementation(+ async (args: { namespace: string; body: k8s.V1NetworkPolicy }) => {+ const name = args.body.metadata?.name+ if (!name) throw new Error('create is missing metadata.name')+ const key = `${args.namespace}/${name}`+ const stored: k8s.V1NetworkPolicy = {+ ...args.body,+ metadata: {+ ...args.body.metadata,+ name,+ namespace: args.namespace,+ uid: key,+ resourceVersion: '1',+ },+ }+ store.set(key, stored)+ return stored+ }+ )+ mockApi.readNamespacedNetworkPolicy.mockImplementation(+ async (args: { name: string; namespace: string }) => {+ const existing = store.get(`${args.namespace}/${args.name}`)+ if (!existing) {+ throw Object.assign(new Error('not found'), { code: 404 })+ }+ return existing+ }+ )+ mockApi.listNamespacedNetworkPolicy.mockImplementation(+ async (args: { namespace: string; labelSelector?: string }) => ({+ items: [...store.values()].filter(+ policy =>+ policy.metadata?.namespace === args.namespace &&+ policyMatchesLabelSelector(policy, args.labelSelector)+ ),+ })+ )+ mockApi.deleteNamespacedNetworkPolicy.mockImplementation(+ async (args: { name: string; namespace: string }) => {+ const key = `${args.namespace}/${args.name}`+ deleted.push(key)+ store.delete(key)+ return {}+ }+ )+ mockCustomApi.listNamespacedCustomObject.mockResolvedValue({ items: [] })++ await reconciler.ensureDefaultPolicies()+ expect(store.size).toBeGreaterThan(0)+ const written = [...store.values()].map(policy => ({+ key: `${policy.metadata?.namespace}/${policy.metadata?.name}`,+ name: policy.metadata?.name ?? '',+ type: policy.metadata?.labels?.[POLICY_TYPE_LABEL] ?? '',+ }))+ expect(written.every(item => !TYPED_SWEEP_TYPES.has(item.type))).toBe(true)+ expect(new Set(written.map(item => item.type))).toEqual(+ new Set(['allow-api', 'default-deny', 'infrastructure'])+ )++ const orphanKey = `${config.namespace}/ctx-gone-allow`+ store.set(orphanKey, {+ apiVersion: 'networking.k8s.io/v1',+ kind: 'NetworkPolicy',+ metadata: {+ name: 'ctx-gone-allow',+ namespace: config.namespace,+ uid: orphanKey,+ resourceVersion: '1',+ labels: {+ [MANAGED_BY_LABEL]: MANAGED_BY_VALUE,+ [POLICY_TYPE_LABEL]: 'context-allow',+ [CONTEXT_LABEL]: 'ctx-gone',+ [MCPSERVER_LABEL]: 'srv-gone',+ },+ },+ spec: { podSelector: {}, policyTypes: ['Ingress'] },+ })++ await reconciler.fullReconcile([], [], { ensureDefaults: false })++ expect(deleted).toContain(orphanKey)+ expect(deleted.filter(key => written.some(item => item.key === key))).toEqual([])+ for (const item of written) {+ expect(store.has(item.key)).toBe(true)+ }+ expect(store.has(orphanKey)).toBe(false)+ })+ })})