Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
fix(hcc): pin the readiness safety certificate to content identity (GREEN)
Alfredo Lopez1a9c7363bbdf
diff --git a/host-context-controller/src/k8sClient.test.ts b/host-context-controller/src/k8sClient.test.tsindex 537ea7e26..ac4c23ca8 100644--- a/host-context-controller/src/k8sClient.test.ts+++ b/host-context-controller/src/k8sClient.test.ts@@ -100,10 +100,6 @@ function markMcpServerInventoryAuthoritative(watcher: McpServerWatcher): void {}function markNetworkPolicyRevocationAuthoritative(watcher: McpServerWatcher): void {- ;(watcher as any).networkPolicyRevocationContextGeneration = (- watcher as any- ).contextWatchGeneration- ;(watcher as any).networkPolicyRevocationServerGeneration = (watcher as any).mcpWatchGeneration;(watcher as any).networkPolicyRevocationContextRevision = (watcher as any).contextDesiredRevision;(watcher as any).networkPolicyRevocationServerRevision = (watcher as any@@ -3785,8 +3781,11 @@ describe('McpServerWatcher startup', () => {await (watcher as any).runInitialNetworkPolicyConvergence()- expect((watcher as any).networkPolicyRevocationContextGeneration).not.toBe(21)- expect((watcher as any).networkPolicyRevocationServerGeneration).not.toBe(34)+ // Authority lost at the callback boundary (contextCacheSynced=false) → record+ // refuses, so the content-identity revocation counters stay at the "never+ // certified" sentinel rather than adopting the captured revision.+ expect((watcher as any).networkPolicyRevocationContextRevision).toBe(-1)+ expect((watcher as any).networkPolicyRevocationServerRevision).toBe(-1)await watcher.stop()})@@ -3939,7 +3938,7 @@ describe('McpServerWatcher startup', () => {errorSpy.mockRestore()})- it('fences runtime effects when either generation retires the safety certificate', async () => {+ it('fences runtime effects when a desired-revision change retires the safety certificate', async () => {const selected = {name: 'certificate-fenced-server',namespace: 'mcp-server',@@ -3972,7 +3971,12 @@ describe('McpServerWatcher startup', () => {const convergence = (watcher as any).runInitialMcpServerConvergence()await egressStarted.promise- ;(watcher as any).contextWatchGeneration = 12+ // A real desired-state change lands mid-convergence: the content identity+ // moves, retiring the captured safety certificate. A watch reconnect that+ // did NOT change content (generation-only bump) would correctly NOT fence —+ // that is the whole point of the content-identity gate — so the fence must+ // key on the revision, which this change advances.+ ;(watcher as any).contextDesiredRevision += 1releaseEgress.resolve(undefined)await convergence@@ -9197,21 +9201,23 @@ describe('McpServerWatcher readiness under sustained watch churn (GKE Premature-return { items: [] }})- // Capture the real Context watch doneCallback so we can drive a Premature- // close through the exact production path (retireContextWatch bumps the- // generation synchronously, then schedules recovery).- let contextDone: ((err: Error | null) => void) | null = null- mocks.watch.mockImplementation(- async (path: string, _opts: unknown, _cb: unknown, done: (err: Error | null) => void) => {- if (path.endsWith('/contexts')) contextDone = done- return { abort: vi.fn() }- }- )+ const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {})+ const warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => {})+ const logSpy = vi.spyOn(console, 'log').mockImplementation(() => {})+ const watcher = new McpServerWatcher()+ vi.spyOn(watcher as any, 'startSharedFileSystemWatch').mockResolvedValue(undefined)+ vi.spyOn(watcher as any, 'startGlobalFileSystemWatch').mockResolvedValue(undefined)- // Every authoritative revocation pass takes a Premature close of the Context- // watch BEFORE it certifies — the production race, made deterministic.+ // Model a "Premature close" reconnect that lands DURING each authoritative+ // revocation pass: the pass captures its safety certificate, then the watch+ // reconnects and recovery re-LISTs the identical inventory — so the watch+ // GENERATION advances (channel identity) while the desired REVISION (content+ // identity) is unchanged and the cache stays authoritative. The #205 code+ // pinned the certificate to the generation, so it refused here on every pass+ // and livelocked; the fix pins it to the revision, so it certifies through+ // the churn.let passes = 0- let prematureCloses = 0+ let reconnects = 0mocks.netPolFullReconcile.mockImplementation(async (_ctxs: unknown,@@ -9219,22 +9225,13 @@ describe('McpServerWatcher readiness under sustained watch churn (GKE Premature-options: { onAuthoritativeRevocationComplete?: () => void } | undefined) => {passes += 1- if (contextDone) {- contextDone(new Error('Premature close'))- prematureCloses += 1- }- // Old code: record refuses (generation moved). Fixed code: record- // accepts (content revision is unchanged — identical re-LIST).+ ;(watcher as any).contextWatchGeneration += 2+ ;(watcher as any).mcpWatchGeneration += 2+ reconnects += 1options?.onAuthoritativeRevocationComplete?.()})- const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {})- const warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => {})- const logSpy = vi.spyOn(console, 'log').mockImplementation(() => {})- const watcher = new McpServerWatcher()- vi.spyOn(watcher as any, 'startSharedFileSystemWatch').mockResolvedValue(undefined)- vi.spyOn(watcher as any, 'startGlobalFileSystemWatch').mockResolvedValue(undefined)const server = new ContextMapperServer(watcher, 0, undefined, undefined, () =>watcher.isReadinessInventoryAuthoritative())@@ -9243,27 +9240,26 @@ describe('McpServerWatcher readiness under sustained watch churn (GKE Premature-try {await watcher.start()server.setReady(true)- await vi.waitFor(() => expect(mocks.netPolFullReconcile).toHaveBeenCalled())-- // Sustain the churn: each recovery cycle reinstates the watch and fires a- // fresh pass, which takes another Premature close. Bounded by MAX_PASSES so- // a livelock fails loudly by count instead of hanging.- const MAX_PASSES = 8- for (let i = 0; i < MAX_PASSES; i += 1) {- if (watcher.isReadinessInventoryAuthoritative()) break- // INVENTORY_CACHE_RECOVERY_RETRY_MS (production constant) is 5000ms.- await vi.advanceTimersByTimeAsync(5000)++ // Sustain the churn for a fixed number of authoritative passes. Each pass+ // takes a reconnect (generation bump) with a byte-identical re-LIST. We do+ // NOT stop early: the regime must run enough times that a livelock is+ // unmistakable, and the fix must hold readiness through all of it.+ const CHURN_PASSES = 5+ for (let i = 0; i < CHURN_PASSES; i += 1) {+ await (watcher as any).runInitialNetworkPolicyConvergence()await flushMicrotasks()}// The adverse regime actually ran (zero-tests-is-never-success).- expect(passes).toBeGreaterThanOrEqual(3)- expect(prematureCloses).toBeGreaterThanOrEqual(3)- // The livelock assertion: identical re-LIST must not keep readiness down.+ expect(passes).toBeGreaterThanOrEqual(CHURN_PASSES)+ expect(reconnects).toBeGreaterThanOrEqual(CHURN_PASSES)+ // The livelock assertion: an identical re-LIST must not keep readiness+ // down. #205 code → false (livelock). Fixed code → true.expect(watcher.isReadinessInventoryAuthoritative(),`livelock: ${passes} authoritative passes completed under sustained churn ` +- `(${prematureCloses} Premature close, identical inventory on every re-LIST) ` ++ `(${reconnects} reconnects, identical inventory on every re-LIST) ` +`and none certified the safety inventory`).toBe(true)expect((await requestReadyOverHttp(server)).statusCode).toBe(200)diff --git a/host-context-controller/src/k8sClient.ts b/host-context-controller/src/k8sClient.tsindex 1dc96adcf..37a89ebf6 100644--- a/host-context-controller/src/k8sClient.ts+++ b/host-context-controller/src/k8sClient.ts@@ -648,10 +648,12 @@ export class McpServerWatcher implements McpServerProvider {// Generations bind the completed safety sweep to the exact LIST -> WATCH// pair that supplied its absence decisions. A recovered watch invalidates// this marker until the new inventory's orphan-allow sweep completes.- private networkPolicyRevocationContextGeneration = 0- private networkPolicyRevocationServerGeneration = 0- private networkPolicyRevocationContextRevision = 0- private networkPolicyRevocationServerRevision = 0+ // Sentinel -1 = "no authoritative revocation has certified yet". A real+ // desired revision starts at 0, so the readiness gate stays closed until the+ // first recordNetworkPolicySafetyCertificate writes a real revision here —+ // the startup role the (now removed) generation equalities used to play.+ private networkPolicyRevocationContextRevision = -1+ private networkPolicyRevocationServerRevision = -1private mcpServerCacheSynced = falseprivate contextCacheSynced = falseprivate mcpServerCacheRecoveryTimer: ReturnType<typeof setTimeout> | null = null@@ -911,16 +913,19 @@ export class McpServerWatcher implements McpServerProvider {this.mcpServerCacheSynced &&this.contextCacheSynced &&this.hostCacheSynced &&- // The generation/revision equalities below prove no newer desired state- // is outstanding; they cannot prove the last pass actually revoked what- // it set out to revoke. Losing a delete fence bumps no generation and- // moves no revision, so it is invisible to every other term here while- // leaving live an allow the pass classified as stale. The reconciler- // degrades this fence at the point of loss; readiness reads it so the- // gate re-closes for the same reason it opened.+ // Certification is pinned to CONTENT identity (the desired-revision+ // counters), not CHANNEL identity (the watch-generation counters). A+ // "Premature close" reconnect re-LISTs the same inventory and bumps the+ // generation without changing desired state; gating readiness on the+ // generation there livelocked the certificate forever under sustained GKE+ // watch-churn (the clerum-dev incident) while the underlying safety+ // decision stayed valid. The revision only moves when the diffing snapshot+ // installer observes a real desired-state change, so a same-content+ // reconnect no longer re-closes the gate, while a genuine change still+ // forces re-certification. Losing a delete fence bumps no revision, so+ // hasCertifiedSafetyInventory() remains the fence for that one failure+ // mode that no equality below can see.this.netPolReconciler.hasCertifiedSafetyInventory() &&- this.networkPolicyRevocationContextGeneration === this.contextWatchGeneration &&- this.networkPolicyRevocationServerGeneration === this.mcpWatchGeneration &&this.networkPolicyRevocationContextRevision === this.contextDesiredRevision &&this.networkPolicyRevocationServerRevision === this.mcpServerDesiredRevision)@@ -1218,14 +1223,51 @@ export class McpServerWatcher implements McpServerProvider {return resourceVersion}+ // A re-LIST after a watch reconnect ("Premature close") reinstalls the whole+ // cache. Diff the incoming snapshot against the current cache BEFORE replacing+ // it and bump the desired revision iff the desired state actually changed+ // (an add, a removal, or a spec change under the canonical comparator). This+ // is the counterpart to the watch-event revision bump: it makes the desired+ // revision a monotonic CONTENT identity that survives channel loss, so the+ // safety certificate can gate on content instead of the churning generation.+ // Same-content reconnect → no bump → certification survives the churn. Real+ // change carried by the re-LIST → bump → forced re-certification (fail-closed).+ private mcpServerSnapshotChangesDesiredState(snapshot: McpServerSnapshot): boolean {+ if (snapshot.servers.length !== this.servers.size) return true+ for (const server of snapshot.servers) {+ const previous = this.servers.get(server.name)+ if (previous === undefined || !sameMcpServerDesiredRevision(previous, server)) {+ return true+ }+ }+ return false+ }+private installMcpServerSnapshot(snapshot: McpServerSnapshot): void {+ if (this.mcpServerSnapshotChangesDesiredState(snapshot)) {+ this.mcpServerDesiredRevision += 1+ }this.servers.clear()for (const server of snapshot.servers) {this.servers.set(server.name, server)}}+ private contextSnapshotChangesDesiredState(snapshot: ContextSnapshot): boolean {+ if (snapshot.contexts.length !== this.contexts.size) return true+ for (const context of snapshot.contexts) {+ const previous = this.contexts.get(context.name)+ if (previous === undefined || !sameContextDesiredRevision(previous, context)) {+ return true+ }+ }+ return false+ }+private installContextSnapshot(snapshot: ContextSnapshot): void {+ if (this.contextSnapshotChangesDesiredState(snapshot)) {+ this.contextDesiredRevision += 1+ }this.contexts.clear()for (const context of snapshot.contexts) {this.contexts.set(context.name, context)@@ -2596,8 +2638,7 @@ export class McpServerWatcher implements McpServerProvider {!this.contextCacheSynced ||!this.mcpServerCacheSynced ||!this.netPolReconciler.hasCertifiedSafetyInventory() ||- this.networkPolicyRevocationContextGeneration !== this.contextWatchGeneration ||- this.networkPolicyRevocationServerGeneration !== this.mcpWatchGeneration ||+ // Content identity, not channel identity — see isReadinessInventoryAuthoritative.this.networkPolicyRevocationContextRevision !== this.contextDesiredRevision ||this.networkPolicyRevocationServerRevision !== this.mcpServerDesiredRevision) {@@ -2618,15 +2659,13 @@ export class McpServerWatcher implements McpServerProvider {this.stopped ||!this.contextCacheSynced ||!this.mcpServerCacheSynced ||- this.contextWatchGeneration !== certificate.contextGeneration ||- this.mcpWatchGeneration !== certificate.serverGeneration ||+ // Content identity, not channel identity — a Premature-close reconnect+ // that re-LISTs the same inventory must not invalidate this certificate.this.contextDesiredRevision !== certificate.contextRevision ||this.mcpServerDesiredRevision !== certificate.serverRevision) {return false}- this.networkPolicyRevocationContextGeneration = certificate.contextGeneration- this.networkPolicyRevocationServerGeneration = certificate.serverGenerationthis.networkPolicyRevocationContextRevision = certificate.contextRevisionthis.networkPolicyRevocationServerRevision = certificate.serverRevisionreturn true@@ -2638,8 +2677,7 @@ export class McpServerWatcher implements McpServerProvider {const current = this.currentNetworkPolicySafetyCertificate()return (current !== null &&- current.contextGeneration === certificate.contextGeneration &&- current.serverGeneration === certificate.serverGeneration &&+ // Content identity, not channel identity — see isReadinessInventoryAuthoritative.current.contextRevision === certificate.contextRevision &¤t.serverRevision === certificate.serverRevision)