Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript

fix(hcc): abort the NetworkPolicy pass on content identity, not watch generation (#462)

Alfredo Lopeza5c73dbf1456
diff --git a/host-context-controller/package-lock.json b/host-context-controller/package-lock.json
index 1cd5126c3..1d7ff668d 100644
--- a/host-context-controller/package-lock.json
+++ b/host-context-controller/package-lock.json
@@ -1,12 +1,12 @@
{
"name": "clerum-host-context-controller",
- "version": "0.1.246",
+ "version": "0.1.247",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "clerum-host-context-controller",
- "version": "0.1.246",
+ "version": "0.1.247",
"license": "SEE LICENSE IN LICENSE",
"dependencies": {
"@clerum/image-policy": "file:../packages/image-policy",
diff --git a/host-context-controller/package.json b/host-context-controller/package.json
index b813d7c9f..44516712a 100644
--- a/host-context-controller/package.json
+++ b/host-context-controller/package.json
@@ -1,6 +1,6 @@
{
"name": "clerum-host-context-controller",
- "version": "0.1.246",
+ "version": "0.1.247",
"description": "Host Context Controller - provides McpServer CRDs to mcp-host via REST API",
"main": "dist/main.js",
"engines": {
diff --git a/host-context-controller/src/k8sClient.test.ts b/host-context-controller/src/k8sClient.test.ts
index a065444c1..252826d87 100644
--- a/host-context-controller/src/k8sClient.test.ts
+++ b/host-context-controller/src/k8sClient.test.ts
@@ -190,6 +190,27 @@ function markNetworkPolicyRevocationAuthoritative(watcher: McpServerWatcher): vo
).mcpServerDesiredRevision
}
+function seedNetworkPolicyPassInventory(watcher: McpServerWatcher): void {
+ ;(watcher as any).contextCacheSynced = true
+ ;(watcher as any).mcpServerCacheSynced = true
+ ;(watcher as any).contextWatchGeneration = 11
+ ;(watcher as any).mcpWatchGeneration = 13
+ ;(watcher as any).contexts.set('stale-context', {
+ name: 'stale-context',
+ namespace: 'mcp-server',
+ spec: { contextId: 'stale-context', mcpServers: ['stale-server'] },
+ })
+ ;(watcher as any).servers.set('stale-server', {
+ name: 'stale-server',
+ namespace: 'mcp-server',
+ spec: {
+ contextRef: 'stale-context',
+ image: 'clerum/stale-server:v1',
+ transport: { type: 'streamableHttp' as const, port: 3000 },
+ },
+ })
+}
+
function newContextAuthoritativeWatcher(): McpServerWatcher {
const watcher = new McpServerWatcher()
;(watcher as any).contextCacheSynced = true
@@ -4138,6 +4159,308 @@ describe('McpServerWatcher startup', () => {
await watcher.stop()
})
+ it('G1: certifies a NetworkPolicy pass when only watch generations recycle mid-flight', async () => {
+ const firstPassStarted = deferred()
+ const releaseFirstPass = deferred()
+ const appliedContexts: string[] = []
+ const appliedServers: string[] = []
+ let offeredContextEffects = 0
+ let offeredServerEffects = 0
+ const watcher = new McpServerWatcher()
+ seedNetworkPolicyPassInventory(watcher)
+ ;(watcher as any).initialConvergenceRetryAttempts.set('NetworkPolicy', 4)
+ const warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => {})
+ const successTimestampBefore = await readInitialConvergenceMetric(
+ 'clerum_hcc_initial_convergence_last_success_timestamp_seconds',
+ 'NetworkPolicy'
+ )
+ const swallowedBefore = await readLabeledConvergenceMetric(
+ 'clerum_hcc_initial_convergence_swallowed_total',
+ { lane: 'NetworkPolicy', sink: 'authority-lost' }
+ )
+ const abortedBefore = await readLabeledConvergenceMetric(
+ 'clerum_hcc_initial_convergence_pass_results_total',
+ { lane: 'NetworkPolicy', result: 'aborted-authority' }
+ )
+ const certifiedBefore = await readLabeledConvergenceMetric(
+ 'clerum_hcc_initial_convergence_pass_results_total',
+ { lane: 'NetworkPolicy', result: 'certified' }
+ )
+ mocks.netPolFullReconcile.mockImplementation(async (contexts, servers, options) => {
+ firstPassStarted.resolve(undefined)
+ await releaseFirstPass.promise
+ options.onAuthoritativeRevocationComplete?.()
+ await Promise.all([
+ ...contexts.map((context: { spec: { contextId: string } }) => {
+ offeredContextEffects += 1
+ return options.runContextEffect(context.spec.contextId, async () => {
+ appliedContexts.push(context.spec.contextId)
+ })
+ }),
+ ...servers.map((server: { name: string }) => {
+ offeredServerEffects += 1
+ return options.runServerEffect(server.name, async () => {
+ appliedServers.push(server.name)
+ })
+ }),
+ ])
+ })
+
+ const pass = (watcher as any).runInitialNetworkPolicyConvergence()
+ await firstPassStarted.promise
+ const contextRevision = (watcher as any).contextDesiredRevision
+ const serverRevision = (watcher as any).mcpServerDesiredRevision
+ ;(watcher as any).contextWatchGeneration += 2
+ ;(watcher as any).mcpWatchGeneration += 2
+ expect((watcher as any).contextCacheSynced).toBe(true)
+ expect((watcher as any).mcpServerCacheSynced).toBe(true)
+ expect((watcher as any).contextDesiredRevision).toBe(contextRevision)
+ expect((watcher as any).mcpServerDesiredRevision).toBe(serverRevision)
+ releaseFirstPass.resolve(undefined)
+ await pass
+
+ expect(offeredContextEffects).toBeGreaterThanOrEqual(1)
+ expect(offeredServerEffects).toBeGreaterThanOrEqual(1)
+ expect(appliedContexts).toEqual(['stale-context'])
+ expect(appliedServers).toEqual(['stale-server'])
+ expect(
+ warnSpy.mock.calls.some(
+ call => String(call[0]) === '[K8s] pass ended without certifying: inventory authority lost'
+ )
+ ).toBe(false)
+ expect((watcher as any).initialConvergenceRetryAttempts.has('NetworkPolicy')).toBe(false)
+ expect((watcher as any).initialConvergenceRetryTimers.has('NetworkPolicy')).toBe(false)
+ expect(
+ await readLabeledConvergenceMetric('clerum_hcc_initial_convergence_swallowed_total', {
+ lane: 'NetworkPolicy',
+ sink: 'authority-lost',
+ })
+ ).toBe(swallowedBefore)
+ expect(
+ await readLabeledConvergenceMetric('clerum_hcc_initial_convergence_pass_results_total', {
+ lane: 'NetworkPolicy',
+ result: 'aborted-authority',
+ })
+ ).toBe(abortedBefore)
+ expect(
+ await readLabeledConvergenceMetric('clerum_hcc_initial_convergence_pass_results_total', {
+ lane: 'NetworkPolicy',
+ result: 'certified',
+ })
+ ).toBe(certifiedBefore + 1)
+ expect(
+ await readInitialConvergenceMetric(
+ 'clerum_hcc_initial_convergence_last_success_timestamp_seconds',
+ 'NetworkPolicy'
+ )
+ ).toBeGreaterThan(successTimestampBefore)
+ warnSpy.mockRestore()
+ await watcher.stop()
+ })
+
+ it('G2: aborts a NetworkPolicy pass when contextDesiredRevision advances mid-flight', async () => {
+ const firstPassStarted = deferred()
+ const releaseFirstPass = deferred()
+ const watcher = new McpServerWatcher()
+ seedNetworkPolicyPassInventory(watcher)
+ ;(watcher as any).initialConvergenceRetryAttempts.set('NetworkPolicy', 4)
+ const warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => {})
+ const successTimestampBefore = await readInitialConvergenceMetric(
+ 'clerum_hcc_initial_convergence_last_success_timestamp_seconds',
+ 'NetworkPolicy'
+ )
+ const swallowedBefore = await readLabeledConvergenceMetric(
+ 'clerum_hcc_initial_convergence_swallowed_total',
+ { lane: 'NetworkPolicy', sink: 'authority-lost' }
+ )
+ const abortedBefore = await readLabeledConvergenceMetric(
+ 'clerum_hcc_initial_convergence_pass_results_total',
+ { lane: 'NetworkPolicy', result: 'aborted-authority' }
+ )
+ mocks.netPolFullReconcile.mockImplementation(async (_contexts, _servers, options) => {
+ firstPassStarted.resolve(undefined)
+ await releaseFirstPass.promise
+ options.onAuthoritativeRevocationComplete?.()
+ })
+
+ const pass = (watcher as any).runInitialNetworkPolicyConvergence()
+ await firstPassStarted.promise
+ const generationBefore = (watcher as any).contextWatchGeneration
+ ;(watcher as any).contextDesiredRevision += 1
+ expect((watcher as any).contextWatchGeneration).toBe(generationBefore)
+ releaseFirstPass.resolve(undefined)
+ await pass
+
+ expect(warnSpy).toHaveBeenCalledWith(
+ '[K8s] pass ended without certifying: inventory authority lost'
+ )
+ expect((watcher as any).initialConvergenceRetryAttempts.get('NetworkPolicy')).toBe(5)
+ expect((watcher as any).initialConvergenceRetryTimers.has('NetworkPolicy')).toBe(true)
+ expect(
+ await readInitialConvergenceMetric(
+ 'clerum_hcc_initial_convergence_last_success_timestamp_seconds',
+ 'NetworkPolicy'
+ )
+ ).toBe(successTimestampBefore)
+ expect(
+ await readLabeledConvergenceMetric('clerum_hcc_initial_convergence_swallowed_total', {
+ lane: 'NetworkPolicy',
+ sink: 'authority-lost',
+ })
+ ).toBe(swallowedBefore + 1)
+ expect(
+ await readLabeledConvergenceMetric('clerum_hcc_initial_convergence_pass_results_total', {
+ lane: 'NetworkPolicy',
+ result: 'aborted-authority',
+ })
+ ).toBe(abortedBefore + 1)
+ warnSpy.mockRestore()
+ await watcher.stop()
+ })
+
+ it('G3: aborts a NetworkPolicy pass when mcpServerDesiredRevision advances mid-flight', async () => {
+ const firstPassStarted = deferred()
+ const releaseFirstPass = deferred()
+ const watcher = new McpServerWatcher()
+ seedNetworkPolicyPassInventory(watcher)
+ ;(watcher as any).initialConvergenceRetryAttempts.set('NetworkPolicy', 4)
+ const warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => {})
+ const successTimestampBefore = await readInitialConvergenceMetric(
+ 'clerum_hcc_initial_convergence_last_success_timestamp_seconds',
+ 'NetworkPolicy'
+ )
+ const swallowedBefore = await readLabeledConvergenceMetric(
+ 'clerum_hcc_initial_convergence_swallowed_total',
+ { lane: 'NetworkPolicy', sink: 'authority-lost' }
+ )
+ const abortedBefore = await readLabeledConvergenceMetric(
+ 'clerum_hcc_initial_convergence_pass_results_total',
+ { lane: 'NetworkPolicy', result: 'aborted-authority' }
+ )
+ mocks.netPolFullReconcile.mockImplementation(async (_contexts, _servers, options) => {
+ firstPassStarted.resolve(undefined)
+ await releaseFirstPass.promise
+ options.onAuthoritativeRevocationComplete?.()
+ })
+
+ const pass = (watcher as any).runInitialNetworkPolicyConvergence()
+ await firstPassStarted.promise
+ const generationBefore = (watcher as any).mcpWatchGeneration
+ ;(watcher as any).mcpServerDesiredRevision += 1
+ expect((watcher as any).mcpWatchGeneration).toBe(generationBefore)
+ releaseFirstPass.resolve(undefined)
+ await pass
+
+ expect(warnSpy).toHaveBeenCalledWith(
+ '[K8s] pass ended without certifying: inventory authority lost'
+ )
+ expect((watcher as any).initialConvergenceRetryAttempts.get('NetworkPolicy')).toBe(5)
+ expect((watcher as any).initialConvergenceRetryTimers.has('NetworkPolicy')).toBe(true)
+ expect(
+ await readInitialConvergenceMetric(
+ 'clerum_hcc_initial_convergence_last_success_timestamp_seconds',
+ 'NetworkPolicy'
+ )
+ ).toBe(successTimestampBefore)
+ expect(
+ await readLabeledConvergenceMetric('clerum_hcc_initial_convergence_swallowed_total', {
+ lane: 'NetworkPolicy',
+ sink: 'authority-lost',
+ })
+ ).toBe(swallowedBefore + 1)
+ expect(
+ await readLabeledConvergenceMetric('clerum_hcc_initial_convergence_pass_results_total', {
+ lane: 'NetworkPolicy',
+ result: 'aborted-authority',
+ })
+ ).toBe(abortedBefore + 1)
+ warnSpy.mockRestore()
+ await watcher.stop()
+ })
+
+ it('G4: drops queued NetworkPolicy effects when a desired revision advances before they run', async () => {
+ const contextBlockerStarted = deferred()
+ const releaseContextBlocker = deferred()
+ const serverBlockerStarted = deferred()
+ const releaseServerBlocker = deferred()
+ const effectsOffered = deferred()
+ const contextWork = vi.fn()
+ const serverWork = vi.fn()
+ let passes = 0
+ let offeredContextEffects = 0
+ let offeredServerEffects = 0
+ const watcher = new McpServerWatcher()
+ seedNetworkPolicyPassInventory(watcher)
+ const droppedContextBefore = await readLabeledConvergenceMetric(
+ 'clerum_hcc_initial_convergence_effects_dropped_total',
+ { lane: 'NetworkPolicy', kind: 'context' }
+ )
+ const droppedServerBefore = await readLabeledConvergenceMetric(
+ 'clerum_hcc_initial_convergence_effects_dropped_total',
+ { lane: 'NetworkPolicy', kind: 'server' }
+ )
+ const contextBlocker = (watcher as any).enqueueContextReconciliation(
+ 'stale-context',
+ async () => {
+ contextBlockerStarted.resolve(undefined)
+ await releaseContextBlocker.promise
+ }
+ )
+ const serverBlocker = (watcher as any).enqueueMcpServerReconciliation(
+ { name: 'stale-server', namespace: 'mcp-server' },
+ async () => {
+ serverBlockerStarted.resolve(undefined)
+ await releaseServerBlocker.promise
+ }
+ )
+ await Promise.all([contextBlockerStarted.promise, serverBlockerStarted.promise])
+ mocks.netPolFullReconcile.mockImplementation(async (contexts, servers, options) => {
+ passes += 1
+ options.onAuthoritativeRevocationComplete?.()
+ const pending = [
+ ...contexts.map((context: { spec: { contextId: string } }) => {
+ offeredContextEffects += 1
+ return options.runContextEffect(context.spec.contextId, contextWork)
+ }),
+ ...servers.map((server: { name: string }) => {
+ offeredServerEffects += 1
+ return options.runServerEffect(server.name, serverWork)
+ }),
+ ]
+ effectsOffered.resolve(undefined)
+ await Promise.all(pending)
+ })
+
+ const pass = (watcher as any).runInitialNetworkPolicyConvergence()
+ await effectsOffered.promise
+ expect(passes).toBeGreaterThan(0)
+ expect(offeredContextEffects).toBeGreaterThanOrEqual(1)
+ expect(offeredServerEffects).toBeGreaterThanOrEqual(1)
+ expect(contextWork).not.toHaveBeenCalled()
+ expect(serverWork).not.toHaveBeenCalled()
+ ;(watcher as any).contextDesiredRevision += 1
+ ;(watcher as any).mcpServerDesiredRevision += 1
+ releaseContextBlocker.resolve(undefined)
+ releaseServerBlocker.resolve(undefined)
+ await Promise.all([contextBlocker, serverBlocker, pass])
+
+ expect(contextWork).not.toHaveBeenCalled()
+ expect(serverWork).not.toHaveBeenCalled()
+ expect(
+ await readLabeledConvergenceMetric('clerum_hcc_initial_convergence_effects_dropped_total', {
+ lane: 'NetworkPolicy',
+ kind: 'context',
+ })
+ ).toBe(droppedContextBefore + 1)
+ expect(
+ await readLabeledConvergenceMetric('clerum_hcc_initial_convergence_effects_dropped_total', {
+ lane: 'NetworkPolicy',
+ kind: 'server',
+ })
+ ).toBe(droppedServerBefore + 1)
+ await watcher.stop()
+ })
+
it('does not certify NetworkPolicy revocation after authority is lost at the callback boundary', async () => {
const watcher = new McpServerWatcher()
;(watcher as any).contextCacheSynced = true
@@ -4812,7 +5135,7 @@ describe('McpServerWatcher startup', () => {
await watcher.stop()
})
- it('fences NetworkPolicy orphan cleanup to the inventory generations captured by the pass', async () => {
+ it('fences NetworkPolicy orphan cleanup to the desired revisions captured by the pass', async () => {
const watcher = new McpServerWatcher()
;(watcher as any).contextCacheSynced = true
;(watcher as any).mcpServerCacheSynced = true
@@ -4842,11 +5165,20 @@ describe('McpServerWatcher startup', () => {
expect(options?.contextInventoryAuthoritative()).toBe(true)
expect(options?.serverInventoryAuthoritative()).toBe(true)
- // A recovered watch may already be authoritative again, but it is a
- // different snapshot generation and cannot authorize cleanup by this pass.
+ // A recovered watch that re-LISTs identical content advances generation
+ // (channel identity) but not the desired revision. That reconnect must
+ // not retire this pass's orphan-cleanup authority.
;(watcher as any).contextWatchGeneration = 18
;(watcher as any).mcpWatchGeneration = 24
+ expect(options?.contextInventoryAuthoritative()).toBe(true)
+ expect(options?.serverInventoryAuthoritative()).toBe(true)
+
+ // A real desired-state change advances the revision and must fence this
+ // in-flight pass immediately.
+ ;(watcher as any).contextDesiredRevision += 1
+ ;(watcher as any).mcpServerDesiredRevision += 1
+
expect(options?.contextInventoryAuthoritative()).toBe(false)
expect(options?.serverInventoryAuthoritative()).toBe(false)
await watcher.stop()
diff --git a/host-context-controller/src/k8sClient.ts b/host-context-controller/src/k8sClient.ts
index 54baa54af..a8a22bf91 100644
--- a/host-context-controller/src/k8sClient.ts
+++ b/host-context-controller/src/k8sClient.ts
@@ -3096,9 +3096,10 @@ export class McpServerWatcher implements McpServerProvider {
}
const serverInventoryComplete = this.mcpServerCacheSynced
const contextInventoryAuthoritative = () =>
- this.hasContextInventoryAuthority(contextInventoryGeneration)
+ this.contextCacheSynced && this.contextDesiredRevision === safetyCertificate.contextRevision
const serverInventoryAuthoritative = () =>
- this.hasMcpServerInventoryAuthority(serverInventoryGeneration)
+ this.mcpServerCacheSynced &&
+ this.mcpServerDesiredRevision === safetyCertificate.serverRevision
console.log('[K8s] Running initial NetworkPolicy background reconciliation...')
await this.netPolReconciler.fullReconcile(initialContexts, initialServers, {
serverInventoryComplete,
diff --git a/host-context-controller/src/mcpServerSafety.test.ts b/host-context-controller/src/mcpServerSafety.test.ts
index 19f75caf3..32a0329a0 100644
--- a/host-context-controller/src/mcpServerSafety.test.ts
+++ b/host-context-controller/src/mcpServerSafety.test.ts
@@ -1,4 +1,4 @@
-import { describe, expect, it, vi } from 'vitest'
+import { describe, expect, expectTypeOf, it, vi } from 'vitest'
import {
confirmAuthoritativeMcpServerAbsence,
isMcpServerStatusOnlyUpdate,
@@ -117,6 +117,53 @@ describe('sameMcpServerDesiredRevision', () => {
expect(sameMcpServerDesiredRevision(expected, current)).toBe(true)
expect(sameMcpServerDesiredRevision(current, expected)).toBe(true)
})
+
+ it('S3: rejects a transport port change', () => {
+ const expected = makeServer()
+ expect(
+ sameMcpServerDesiredRevision(
+ expected,
+ makeServer({
+ spec: { ...expected.spec, transport: { ...expected.spec.transport, port: 4000 } },
+ })
+ )
+ ).toBe(false)
+ })
+
+ it('S4: rejects an egressBindings change', () => {
+ const expected = makeServer({
+ spec: {
+ contextRef: 'default',
+ image: 'clerum/web-search:test',
+ transport: { type: 'streamableHttp', port: 3000 },
+ egressBindings: [{ cidr: '1.2.3.4/32', port: 443, protocol: 'TCP' }],
+ },
+ })
+ expect(
+ sameMcpServerDesiredRevision(
+ expected,
+ makeServer({
+ spec: {
+ ...expected.spec,
+ egressBindings: [{ cidr: '8.8.8.8/32', port: 443, protocol: 'TCP' }],
+ },
+ })
+ )
+ ).toBe(false)
+ })
+
+ it('G8: McpServerCRD compared fields have no escape hatch besides status', () => {
+ type McpServerCompared =
+ | 'name'
+ | 'namespace'
+ | 'uid'
+ | 'generation'
+ | 'spec'
+ | 'annotations'
+ | 'labels'
+ type McpServerEscapes = Exclude<keyof McpServerCRD, McpServerCompared | 'status'>
+ expectTypeOf<McpServerEscapes>().toEqualTypeOf<never>()
+ })
})
describe('isMcpServerStatusOnlyUpdate', () => {
diff --git a/host-context-controller/src/networkPolicyReconciler.test.ts b/host-context-controller/src/networkPolicyReconciler.test.ts
index bb393858f..02e0c80a3 100644
--- a/host-context-controller/src/networkPolicyReconciler.test.ts
+++ b/host-context-controller/src/networkPolicyReconciler.test.ts
@@ -1,4 +1,4 @@
-import { type Mock, beforeEach, describe, expect, it, vi } from 'vitest'
+import { type Mock, beforeEach, describe, expect, expectTypeOf, it, vi } from 'vitest'
import * as k8s from '@kubernetes/client-node'
import type { RecordWithTtl } from 'node:dns'
import * as dns from 'node:dns/promises'
@@ -215,6 +215,155 @@ describe('NetworkPolicyReconciler', () => {
expect(sameContextDesiredRevision(expected, { ...reordered, generation: 8 })).toBe(false)
})
+ describe('sameContextDesiredRevision mutation table (content identity)', () => {
+ const companionServer: McpServerCRD = {
+ name: 'alpha',
+ namespace: 'mcp-server',
+ spec: {
+ contextRef: 'default',
+ image: 'clerum/alpha:test',
+ transport: { type: 'streamableHttp', port: 3000 },
+ egressBindings: [{ cidr: '1.2.3.4/32', port: 443, protocol: 'TCP' }],
+ },
+ }
+ const base: ContextCRD = {
+ name: 'default',
+ namespace: 'mcp-server',
+ uid: 'context-uid',
+ generation: 7,
+ spec: {
+ contextId: 'default',
+ mcpServers: ['alpha'],
+ sharedFileSystems: [{ name: 'workspace', mountPath: '/workspace' }],
+ },
+ }
+
+ function stableJson(value: unknown): string {
+ if (Array.isArray(value)) return `[${value.map(stableJson).join(',')}]`
+ if (value && typeof value === 'object') {
+ const record = value as Record<string, unknown>
+ return `{${Object.keys(record)
+ .sort()
+ .map(key => `${JSON.stringify(key)}:${stableJson(record[key])}`)
+ .join(',')}}`
+ }
+ return JSON.stringify(value) ?? 'undefined'
+ }
+
+ function renderContextAllow(
+ context: ContextCRD,
+ server: McpServerCRD = companionServer
+ ): string {
+ return stableJson((reconciler as any).buildContextAllowPolicies(context, server))
+ }
+
+ function renderExactHostEgress(server: McpServerCRD): string {
+ const binding = server.spec.egressBindings?.[0]
+ if (!binding?.cidr) return 'none'
+ return stableJson(
+ (reconciler as any).buildExactHostEgressPolicy(server, 'exact-host-egress', binding, [
+ binding.cidr,
+ ])
+ )
+ }
+
+ it('G8: ContextCRD compared fields have no escape hatch besides status', () => {
+ type ContextCompared = 'name' | 'namespace' | 'uid' | 'generation' | 'spec'
+ type ContextEscapes = Exclude<keyof ContextCRD, ContextCompared | 'status'>
+ expectTypeOf<ContextEscapes>().toEqualTypeOf<never>()
+ })
+
+ it('C1: spec.contextId change fails the comparator and changes the rendered allow policies', () => {
+ const mutated: ContextCRD = {
+ ...base,
+ spec: { ...base.spec, contextId: 'other' },
+ }
+ expect(sameContextDesiredRevision(base, mutated)).toBe(false)
+ expect(renderContextAllow(mutated)).not.toBe(renderContextAllow(base))
+ })
+
+ it('C2: name change fails the comparator and changes the rendered allow policies', () => {
+ const mutated: ContextCRD = { ...base, name: 'other-context' }
+ expect(sameContextDesiredRevision(base, mutated)).toBe(false)
+ expect(renderContextAllow(mutated)).not.toBe(renderContextAllow(base))
+ })
+
+ it.each([
+ ['add', ['alpha', 'beta']],
+ ['remove', []],
+ ['rename', ['renamed']],
+ ] as const)('C3: spec.mcpServers %s fails the comparator', (_label, mcpServers) => {
+ const mutated: ContextCRD = {
+ ...base,
+ spec: { ...base.spec, mcpServers: [...mcpServers] },
+ }
+ expect(sameContextDesiredRevision(base, mutated)).toBe(false)
+ })
+
+ it('C4: status-only change matches and keeps the rendered allow policies identical', () => {
+ const mutated: ContextCRD = {
+ ...base,
+ status: {
+ sharedFileSystems: [{ name: 'workspace', mountPath: '/workspace', phase: 'Mounted' }],
+ },
+ }
+ expect(sameContextDesiredRevision(base, mutated)).toBe(true)
+ expect(renderContextAllow(mutated)).toBe(renderContextAllow(base))
+ })
+
+ it('C5: spec key reorder matches and keeps the rendered allow policies identical', () => {
+ const mutated: ContextCRD = {
+ name: base.name,
+ namespace: base.namespace,
+ uid: base.uid,
+ generation: base.generation,
+ spec: {
+ sharedFileSystems: [{ mountPath: '/workspace', name: 'workspace' }],
+ mcpServers: ['alpha'],
+ contextId: 'default',
+ },
+ }
+ expect(sameContextDesiredRevision(base, mutated)).toBe(true)
+ expect(renderContextAllow(mutated)).toBe(renderContextAllow(base))
+ })
+
+ it('C6: undefined-valued spec key matches and keeps the rendered allow policies identical', () => {
+ const mutated: ContextCRD = {
+ ...base,
+ spec: { ...base.spec, description: undefined },
+ }
+ expect(sameContextDesiredRevision(base, mutated)).toBe(true)
+ expect(renderContextAllow(mutated)).toBe(renderContextAllow(base))
+ })
+
+ it('S1: server name change changes the rendered allow policies', () => {
+ const mutated: McpServerCRD = { ...companionServer, name: 'beta' }
+ expect(renderContextAllow(base, mutated)).not.toBe(renderContextAllow(base))
+ })
+
+ it('S3: spec.transport.port change changes the rendered allow policies', () => {
+ const mutated: McpServerCRD = {
+ ...companionServer,
+ spec: {
+ ...companionServer.spec,
+ transport: { ...companionServer.spec.transport, port: 4000 },
+ },
+ }
+ expect(renderContextAllow(base, mutated)).not.toBe(renderContextAllow(base))
+ })
+
+ it('S4: egressBindings change changes the rendered exact-host egress policy', () => {
+ const mutated: McpServerCRD = {
+ ...companionServer,
+ spec: {
+ ...companionServer.spec,
+ egressBindings: [{ cidr: '8.8.8.8/32', port: 443, protocol: 'TCP' }],
+ },
+ }
+ expect(renderExactHostEgress(mutated)).not.toBe(renderExactHostEgress(companionServer))
+ })
+ })
+
it('records the authoritative safety-pass inventory, completed revocations, and duration', async () => {
mockApi.listNamespacedNetworkPolicy
.mockResolvedValueOnce({
diff --git a/host-context-controller/src/readinessSafetyFence.integration.test.ts b/host-context-controller/src/readinessSafetyFence.integration.test.ts
index dbdf29093..5bab48f71 100644
--- a/host-context-controller/src/readinessSafetyFence.integration.test.ts
+++ b/host-context-controller/src/readinessSafetyFence.integration.test.ts
@@ -18,6 +18,7 @@
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
import * as http from 'http'
import { McpServerWatcher } from './k8sClient'
+import { registry } from './metrics'
import { ContextMapperServer } from './server'
const mocks = vi.hoisted(() => ({
@@ -146,6 +147,27 @@ async function readyStatus(server: InstanceType<typeof ContextMapperServer>): Pr
}
/** Aligns every revocation counter so only the fence can gate readiness. */
+
+async function readLabeledMetric(name: string, labels: Record<string, string>): Promise<number> {
+ const metric = registry.getSingleMetric(name)
+ if (!metric) throw new Error(`${name} is not registered`)
+ const snapshot = await metric.get()
+ return (
+ snapshot.values.find(entry =>
+ Object.entries(labels).every(([key, value]) => entry.labels[key] === value)
+ )?.value ?? 0
+ )
+}
+
+async function readLaneSuccessTimestamp(lane: string): Promise<number> {
+ const metric = registry.getSingleMetric(
+ 'clerum_hcc_initial_convergence_last_success_timestamp_seconds'
+ )
+ if (!metric) throw new Error('last_success metric is not registered')
+ const snapshot = await metric.get()
+ return snapshot.values.find(entry => entry.labels.lane === lane)?.value ?? 0
+}
+
function alignRevocationCounters(watcher: InstanceType<typeof McpServerWatcher>): void {
const w = watcher as unknown as Record<string, unknown>
w.mcpServerCacheSynced = true
@@ -508,4 +530,86 @@ describe('readiness withdrawal on a real lost delete fence', () => {
warnSpy.mockRestore()
}
})
+
+ it('G9: certifies a real authoritative pass through watch-generation churn without authority-lost', async () => {
+ const w = watcher as unknown as Record<string, any>
+ w.contextCacheSynced = true
+ w.mcpServerCacheSynced = true
+ w.hostCacheSynced = true
+ w.contextWatchGeneration = 11
+ w.mcpWatchGeneration = 13
+ w.contexts.set('alpha', alphaContext)
+ const generationBefore = {
+ context: w.contextWatchGeneration,
+ server: w.mcpWatchGeneration,
+ }
+ const revisionBefore = {
+ context: w.contextDesiredRevision,
+ server: w.mcpServerDesiredRevision,
+ }
+ const warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => {})
+ const logSpy = vi.spyOn(console, 'log').mockImplementation(() => {})
+ const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {})
+ const swallowedBefore = await readLabeledMetric(
+ 'clerum_hcc_initial_convergence_swallowed_total',
+ { lane: 'NetworkPolicy', sink: 'authority-lost' }
+ )
+ const abortedBefore = await readLabeledMetric(
+ 'clerum_hcc_initial_convergence_pass_results_total',
+ { lane: 'NetworkPolicy', result: 'aborted-authority' }
+ )
+ const certifiedBefore = await readLabeledMetric(
+ 'clerum_hcc_initial_convergence_pass_results_total',
+ { lane: 'NetworkPolicy', result: 'certified' }
+ )
+ const successBefore = await readLaneSuccessTimestamp('NetworkPolicy')
+ let lists = 0
+ mocks.listNamespacedNetworkPolicy.mockImplementation(async () => {
+ lists += 1
+ if (lists === 1) {
+ w.contextWatchGeneration += 2
+ w.mcpWatchGeneration += 2
+ }
+ return { items: [] }
+ })
+ mocks.deleteNamespacedNetworkPolicy.mockResolvedValue({})
+
+ await w.runInitialNetworkPolicyConvergence()
+
+ expect(lists).toBeGreaterThan(0)
+ expect(w.contextWatchGeneration).toBe(generationBefore.context + 2)
+ expect(w.mcpWatchGeneration).toBe(generationBefore.server + 2)
+ expect(w.contextDesiredRevision).toBe(revisionBefore.context)
+ expect(w.mcpServerDesiredRevision).toBe(revisionBefore.server)
+ expect(
+ warnSpy.mock.calls.some(
+ call => String(call[0]) === '[K8s] pass ended without certifying: inventory authority lost'
+ )
+ ).toBe(false)
+ expect(
+ await readLabeledMetric('clerum_hcc_initial_convergence_swallowed_total', {
+ lane: 'NetworkPolicy',
+ sink: 'authority-lost',
+ })
+ ).toBe(swallowedBefore)
+ expect(
+ await readLabeledMetric('clerum_hcc_initial_convergence_pass_results_total', {
+ lane: 'NetworkPolicy',
+ result: 'aborted-authority',
+ })
+ ).toBe(abortedBefore)
+ expect(
+ await readLabeledMetric('clerum_hcc_initial_convergence_pass_results_total', {
+ lane: 'NetworkPolicy',
+ result: 'certified',
+ })
+ ).toBe(certifiedBefore + 1)
+ expect(await readLaneSuccessTimestamp('NetworkPolicy')).toBeGreaterThan(successBefore)
+ expect(w.networkPolicyRevocationContextRevision).toBe(w.contextDesiredRevision)
+ expect(w.networkPolicyRevocationServerRevision).toBe(w.mcpServerDesiredRevision)
+ expect(w.netPolReconciler.hasCertifiedSafetyInventory()).toBe(true)
+ warnSpy.mockRestore()
+ logSpy.mockRestore()
+ errorSpy.mockRestore()
+ })
})