Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript

fix(hcc): pin the readiness safety certificate to content identity (GREEN)

Alfredo Lopez1a9c7363bbdf
diff --git a/host-context-controller/src/k8sClient.test.ts b/host-context-controller/src/k8sClient.test.ts
index 537ea7e26..ac4c23ca8 100644
--- a/host-context-controller/src/k8sClient.test.ts
+++ b/host-context-controller/src/k8sClient.test.ts
@@ -100,10 +100,6 @@ function markMcpServerInventoryAuthoritative(watcher: McpServerWatcher): void {
}
function markNetworkPolicyRevocationAuthoritative(watcher: McpServerWatcher): void {
- ;(watcher as any).networkPolicyRevocationContextGeneration = (
- watcher as any
- ).contextWatchGeneration
- ;(watcher as any).networkPolicyRevocationServerGeneration = (watcher as any).mcpWatchGeneration
;(watcher as any).networkPolicyRevocationContextRevision = (watcher as any).contextDesiredRevision
;(watcher as any).networkPolicyRevocationServerRevision = (
watcher as any
@@ -3785,8 +3781,11 @@ describe('McpServerWatcher startup', () => {
await (watcher as any).runInitialNetworkPolicyConvergence()
- expect((watcher as any).networkPolicyRevocationContextGeneration).not.toBe(21)
- expect((watcher as any).networkPolicyRevocationServerGeneration).not.toBe(34)
+ // Authority lost at the callback boundary (contextCacheSynced=false) → record
+ // refuses, so the content-identity revocation counters stay at the "never
+ // certified" sentinel rather than adopting the captured revision.
+ expect((watcher as any).networkPolicyRevocationContextRevision).toBe(-1)
+ expect((watcher as any).networkPolicyRevocationServerRevision).toBe(-1)
await watcher.stop()
})
@@ -3939,7 +3938,7 @@ describe('McpServerWatcher startup', () => {
errorSpy.mockRestore()
})
- it('fences runtime effects when either generation retires the safety certificate', async () => {
+ it('fences runtime effects when a desired-revision change retires the safety certificate', async () => {
const selected = {
name: 'certificate-fenced-server',
namespace: 'mcp-server',
@@ -3972,7 +3971,12 @@ describe('McpServerWatcher startup', () => {
const convergence = (watcher as any).runInitialMcpServerConvergence()
await egressStarted.promise
- ;(watcher as any).contextWatchGeneration = 12
+ // A real desired-state change lands mid-convergence: the content identity
+ // moves, retiring the captured safety certificate. A watch reconnect that
+ // did NOT change content (generation-only bump) would correctly NOT fence —
+ // that is the whole point of the content-identity gate — so the fence must
+ // key on the revision, which this change advances.
+ ;(watcher as any).contextDesiredRevision += 1
releaseEgress.resolve(undefined)
await convergence
@@ -9197,21 +9201,23 @@ describe('McpServerWatcher readiness under sustained watch churn (GKE Premature-
return { items: [] }
})
- // Capture the real Context watch doneCallback so we can drive a Premature
- // close through the exact production path (retireContextWatch bumps the
- // generation synchronously, then schedules recovery).
- let contextDone: ((err: Error | null) => void) | null = null
- mocks.watch.mockImplementation(
- async (path: string, _opts: unknown, _cb: unknown, done: (err: Error | null) => void) => {
- if (path.endsWith('/contexts')) contextDone = done
- return { abort: vi.fn() }
- }
- )
+ const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {})
+ const warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => {})
+ const logSpy = vi.spyOn(console, 'log').mockImplementation(() => {})
+ const watcher = new McpServerWatcher()
+ vi.spyOn(watcher as any, 'startSharedFileSystemWatch').mockResolvedValue(undefined)
+ vi.spyOn(watcher as any, 'startGlobalFileSystemWatch').mockResolvedValue(undefined)
- // Every authoritative revocation pass takes a Premature close of the Context
- // watch BEFORE it certifies — the production race, made deterministic.
+ // Model a "Premature close" reconnect that lands DURING each authoritative
+ // revocation pass: the pass captures its safety certificate, then the watch
+ // reconnects and recovery re-LISTs the identical inventory — so the watch
+ // GENERATION advances (channel identity) while the desired REVISION (content
+ // identity) is unchanged and the cache stays authoritative. The #205 code
+ // pinned the certificate to the generation, so it refused here on every pass
+ // and livelocked; the fix pins it to the revision, so it certifies through
+ // the churn.
let passes = 0
- let prematureCloses = 0
+ let reconnects = 0
mocks.netPolFullReconcile.mockImplementation(
async (
_ctxs: unknown,
@@ -9219,22 +9225,13 @@ describe('McpServerWatcher readiness under sustained watch churn (GKE Premature-
options: { onAuthoritativeRevocationComplete?: () => void } | undefined
) => {
passes += 1
- if (contextDone) {
- contextDone(new Error('Premature close'))
- prematureCloses += 1
- }
- // Old code: record refuses (generation moved). Fixed code: record
- // accepts (content revision is unchanged — identical re-LIST).
+ ;(watcher as any).contextWatchGeneration += 2
+ ;(watcher as any).mcpWatchGeneration += 2
+ reconnects += 1
options?.onAuthoritativeRevocationComplete?.()
}
)
- const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {})
- const warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => {})
- const logSpy = vi.spyOn(console, 'log').mockImplementation(() => {})
- const watcher = new McpServerWatcher()
- vi.spyOn(watcher as any, 'startSharedFileSystemWatch').mockResolvedValue(undefined)
- vi.spyOn(watcher as any, 'startGlobalFileSystemWatch').mockResolvedValue(undefined)
const server = new ContextMapperServer(watcher, 0, undefined, undefined, () =>
watcher.isReadinessInventoryAuthoritative()
)
@@ -9243,27 +9240,26 @@ describe('McpServerWatcher readiness under sustained watch churn (GKE Premature-
try {
await watcher.start()
server.setReady(true)
- await vi.waitFor(() => expect(mocks.netPolFullReconcile).toHaveBeenCalled())
-
- // Sustain the churn: each recovery cycle reinstates the watch and fires a
- // fresh pass, which takes another Premature close. Bounded by MAX_PASSES so
- // a livelock fails loudly by count instead of hanging.
- const MAX_PASSES = 8
- for (let i = 0; i < MAX_PASSES; i += 1) {
- if (watcher.isReadinessInventoryAuthoritative()) break
- // INVENTORY_CACHE_RECOVERY_RETRY_MS (production constant) is 5000ms.
- await vi.advanceTimersByTimeAsync(5000)
+
+ // Sustain the churn for a fixed number of authoritative passes. Each pass
+ // takes a reconnect (generation bump) with a byte-identical re-LIST. We do
+ // NOT stop early: the regime must run enough times that a livelock is
+ // unmistakable, and the fix must hold readiness through all of it.
+ const CHURN_PASSES = 5
+ for (let i = 0; i < CHURN_PASSES; i += 1) {
+ await (watcher as any).runInitialNetworkPolicyConvergence()
await flushMicrotasks()
}
// The adverse regime actually ran (zero-tests-is-never-success).
- expect(passes).toBeGreaterThanOrEqual(3)
- expect(prematureCloses).toBeGreaterThanOrEqual(3)
- // The livelock assertion: identical re-LIST must not keep readiness down.
+ expect(passes).toBeGreaterThanOrEqual(CHURN_PASSES)
+ expect(reconnects).toBeGreaterThanOrEqual(CHURN_PASSES)
+ // The livelock assertion: an identical re-LIST must not keep readiness
+ // down. #205 code → false (livelock). Fixed code → true.
expect(
watcher.isReadinessInventoryAuthoritative(),
`livelock: ${passes} authoritative passes completed under sustained churn ` +
- `(${prematureCloses} Premature close, identical inventory on every re-LIST) ` +
+ `(${reconnects} reconnects, identical inventory on every re-LIST) ` +
`and none certified the safety inventory`
).toBe(true)
expect((await requestReadyOverHttp(server)).statusCode).toBe(200)
diff --git a/host-context-controller/src/k8sClient.ts b/host-context-controller/src/k8sClient.ts
index 1dc96adcf..37a89ebf6 100644
--- a/host-context-controller/src/k8sClient.ts
+++ b/host-context-controller/src/k8sClient.ts
@@ -648,10 +648,12 @@ export class McpServerWatcher implements McpServerProvider {
// Generations bind the completed safety sweep to the exact LIST -> WATCH
// pair that supplied its absence decisions. A recovered watch invalidates
// this marker until the new inventory's orphan-allow sweep completes.
- private networkPolicyRevocationContextGeneration = 0
- private networkPolicyRevocationServerGeneration = 0
- private networkPolicyRevocationContextRevision = 0
- private networkPolicyRevocationServerRevision = 0
+ // Sentinel -1 = "no authoritative revocation has certified yet". A real
+ // desired revision starts at 0, so the readiness gate stays closed until the
+ // first recordNetworkPolicySafetyCertificate writes a real revision here —
+ // the startup role the (now removed) generation equalities used to play.
+ private networkPolicyRevocationContextRevision = -1
+ private networkPolicyRevocationServerRevision = -1
private mcpServerCacheSynced = false
private contextCacheSynced = false
private mcpServerCacheRecoveryTimer: ReturnType<typeof setTimeout> | null = null
@@ -911,16 +913,19 @@ export class McpServerWatcher implements McpServerProvider {
this.mcpServerCacheSynced &&
this.contextCacheSynced &&
this.hostCacheSynced &&
- // The generation/revision equalities below prove no newer desired state
- // is outstanding; they cannot prove the last pass actually revoked what
- // it set out to revoke. Losing a delete fence bumps no generation and
- // moves no revision, so it is invisible to every other term here while
- // leaving live an allow the pass classified as stale. The reconciler
- // degrades this fence at the point of loss; readiness reads it so the
- // gate re-closes for the same reason it opened.
+ // Certification is pinned to CONTENT identity (the desired-revision
+ // counters), not CHANNEL identity (the watch-generation counters). A
+ // "Premature close" reconnect re-LISTs the same inventory and bumps the
+ // generation without changing desired state; gating readiness on the
+ // generation there livelocked the certificate forever under sustained GKE
+ // watch-churn (the clerum-dev incident) while the underlying safety
+ // decision stayed valid. The revision only moves when the diffing snapshot
+ // installer observes a real desired-state change, so a same-content
+ // reconnect no longer re-closes the gate, while a genuine change still
+ // forces re-certification. Losing a delete fence bumps no revision, so
+ // hasCertifiedSafetyInventory() remains the fence for that one failure
+ // mode that no equality below can see.
this.netPolReconciler.hasCertifiedSafetyInventory() &&
- this.networkPolicyRevocationContextGeneration === this.contextWatchGeneration &&
- this.networkPolicyRevocationServerGeneration === this.mcpWatchGeneration &&
this.networkPolicyRevocationContextRevision === this.contextDesiredRevision &&
this.networkPolicyRevocationServerRevision === this.mcpServerDesiredRevision
)
@@ -1218,14 +1223,51 @@ export class McpServerWatcher implements McpServerProvider {
return resourceVersion
}
+ // A re-LIST after a watch reconnect ("Premature close") reinstalls the whole
+ // cache. Diff the incoming snapshot against the current cache BEFORE replacing
+ // it and bump the desired revision iff the desired state actually changed
+ // (an add, a removal, or a spec change under the canonical comparator). This
+ // is the counterpart to the watch-event revision bump: it makes the desired
+ // revision a monotonic CONTENT identity that survives channel loss, so the
+ // safety certificate can gate on content instead of the churning generation.
+ // Same-content reconnect → no bump → certification survives the churn. Real
+ // change carried by the re-LIST → bump → forced re-certification (fail-closed).
+ private mcpServerSnapshotChangesDesiredState(snapshot: McpServerSnapshot): boolean {
+ if (snapshot.servers.length !== this.servers.size) return true
+ for (const server of snapshot.servers) {
+ const previous = this.servers.get(server.name)
+ if (previous === undefined || !sameMcpServerDesiredRevision(previous, server)) {
+ return true
+ }
+ }
+ return false
+ }
+
private installMcpServerSnapshot(snapshot: McpServerSnapshot): void {
+ if (this.mcpServerSnapshotChangesDesiredState(snapshot)) {
+ this.mcpServerDesiredRevision += 1
+ }
this.servers.clear()
for (const server of snapshot.servers) {
this.servers.set(server.name, server)
}
}
+ private contextSnapshotChangesDesiredState(snapshot: ContextSnapshot): boolean {
+ if (snapshot.contexts.length !== this.contexts.size) return true
+ for (const context of snapshot.contexts) {
+ const previous = this.contexts.get(context.name)
+ if (previous === undefined || !sameContextDesiredRevision(previous, context)) {
+ return true
+ }
+ }
+ return false
+ }
+
private installContextSnapshot(snapshot: ContextSnapshot): void {
+ if (this.contextSnapshotChangesDesiredState(snapshot)) {
+ this.contextDesiredRevision += 1
+ }
this.contexts.clear()
for (const context of snapshot.contexts) {
this.contexts.set(context.name, context)
@@ -2596,8 +2638,7 @@ export class McpServerWatcher implements McpServerProvider {
!this.contextCacheSynced ||
!this.mcpServerCacheSynced ||
!this.netPolReconciler.hasCertifiedSafetyInventory() ||
- this.networkPolicyRevocationContextGeneration !== this.contextWatchGeneration ||
- this.networkPolicyRevocationServerGeneration !== this.mcpWatchGeneration ||
+ // Content identity, not channel identity — see isReadinessInventoryAuthoritative.
this.networkPolicyRevocationContextRevision !== this.contextDesiredRevision ||
this.networkPolicyRevocationServerRevision !== this.mcpServerDesiredRevision
) {
@@ -2618,15 +2659,13 @@ export class McpServerWatcher implements McpServerProvider {
this.stopped ||
!this.contextCacheSynced ||
!this.mcpServerCacheSynced ||
- this.contextWatchGeneration !== certificate.contextGeneration ||
- this.mcpWatchGeneration !== certificate.serverGeneration ||
+ // Content identity, not channel identity — a Premature-close reconnect
+ // that re-LISTs the same inventory must not invalidate this certificate.
this.contextDesiredRevision !== certificate.contextRevision ||
this.mcpServerDesiredRevision !== certificate.serverRevision
) {
return false
}
- this.networkPolicyRevocationContextGeneration = certificate.contextGeneration
- this.networkPolicyRevocationServerGeneration = certificate.serverGeneration
this.networkPolicyRevocationContextRevision = certificate.contextRevision
this.networkPolicyRevocationServerRevision = certificate.serverRevision
return true
@@ -2638,8 +2677,7 @@ export class McpServerWatcher implements McpServerProvider {
const current = this.currentNetworkPolicySafetyCertificate()
return (
current !== null &&
- current.contextGeneration === certificate.contextGeneration &&
- current.serverGeneration === certificate.serverGeneration &&
+ // Content identity, not channel identity — see isReadinessInventoryAuthoritative.
current.contextRevision === certificate.contextRevision &&
current.serverRevision === certificate.serverRevision
)