Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript

fix(hcc): apply content identity to the Host mutation-authority fence (phase 1b)

Alfredo Lopez6496fdfacf20
diff --git a/host-context-controller/src/hostReconciler.test.ts b/host-context-controller/src/hostReconciler.test.ts
index 53625ff43..6590681b1 100644
--- a/host-context-controller/src/hostReconciler.test.ts
+++ b/host-context-controller/src/hostReconciler.test.ts
@@ -1108,8 +1108,8 @@ describe('HostReconciler Host inventory mutation authority', () => {
const mutationAuthority = {
current: {
known: true,
- hostGeneration: 7,
- contextGeneration: 11,
+ hostRevision: 7,
+ contextRevision: 11,
},
}
;(reconciler as any).setHostMutationAuthority(() => mutationAuthority.current)
@@ -1126,13 +1126,15 @@ describe('HostReconciler Host inventory mutation authority', () => {
const fleet = reconciler.fullReconcile([hostA, hostB])
await vi.waitFor(() => expect(reconcileCore).toHaveBeenCalledWith(hostA, expect.any(Function)))
- // Context retired and recovered to a fresh LIST -> WATCH generation while
- // Host authority remained continuously known: a host-only lease would miss
- // this ABA transition and admit Host B with stale derived mounts.
+ // Context desired state changed (its content revision advanced) while Host
+ // authority remained continuously known: a host-only lease would miss this
+ // transition and admit Host B with stale derived mounts. A pure watch
+ // reconnect with identical content would NOT advance the revision and is
+ // correctly admitted — that is the point of the content-identity fence.
mutationAuthority.current = {
known: true,
- hostGeneration: 7,
- contextGeneration: 13,
+ hostRevision: 7,
+ contextRevision: 13,
}
releaseBlocker.resolve(undefined)
await occupied
@@ -1160,8 +1162,8 @@ describe('HostReconciler Host inventory mutation authority', () => {
reconciler.setResolveCurrentHost(name => (name === host.name ? host : undefined))
reconciler.setHostMutationAuthority(() => ({
known: true,
- hostGeneration: 7,
- contextGeneration: 11,
+ hostRevision: 7,
+ contextRevision: 11,
}))
reconciler.setResolveHostMutationDependencies(() => [selectedContext])
@@ -1216,8 +1218,8 @@ describe('HostReconciler Host inventory mutation authority', () => {
reconciler.setResolveCurrentHost(name => (name === host.name ? host : undefined))
reconciler.setHostMutationAuthority(() => ({
known: true,
- hostGeneration: 7,
- contextGeneration: 11,
+ hostRevision: 7,
+ contextRevision: 11,
}))
reconciler.setResolveHostMutationDependencies(() => [selectedContext])
@@ -1272,8 +1274,8 @@ describe('HostReconciler Host inventory mutation authority', () => {
reconciler.setResolveCurrentHost(name => (name === host.name ? host : undefined))
reconciler.setHostMutationAuthority(() => ({
known: true,
- hostGeneration: 7,
- contextGeneration: 11,
+ hostRevision: 7,
+ contextRevision: 11,
}))
reconciler.setResolveHostMutationDependencies(() => [context, selectedDependency])
diff --git a/host-context-controller/src/hostReconciler.ts b/host-context-controller/src/hostReconciler.ts
index 9e2f236d5..94008e22f 100644
--- a/host-context-controller/src/hostReconciler.ts
+++ b/host-context-controller/src/hostReconciler.ts
@@ -82,13 +82,17 @@ export type HostWatchAuthoritySnapshot = { known: boolean; generation: number }
/**
* Immutable lease for every inventory that contributes to a Host mutation.
* Host resources come from the Host watch; mounted SharedFileSystems come from
- * the Context watch. Keeping both generations prevents a false-stable
- * Host-only lease when Context retires and recovers during queued work.
+ * the Context watch. Both are CONTENT revisions (they move only on a real
+ * desired-state change), not channel generations — a watch reconnect that
+ * re-LISTs the identical inventory must not retire an in-flight Host lease, or
+ * sustained watch-churn starves every queued reconcile. Keeping both revisions
+ * still prevents a false-stable Host-only lease when Context desired state
+ * changes during queued work.
*/
export type HostMutationAuthoritySnapshot = {
known: boolean
- hostGeneration: number
- contextGeneration: number
+ hostRevision: number
+ contextRevision: number
}
class HostInventoryAuthorityUnavailableError extends Error {
@@ -99,9 +103,9 @@ class HostInventoryAuthorityUnavailableError extends Error {
) {
super(
`Host inventory authority changed before ${action} admission ` +
- `(captured known=${captured.known} hostGeneration=${captured.hostGeneration} ` +
- `contextGeneration=${captured.contextGeneration}, current known=${current.known} ` +
- `hostGeneration=${current.hostGeneration} contextGeneration=${current.contextGeneration})`
+ `(captured known=${captured.known} hostRevision=${captured.hostRevision} ` +
+ `contextRevision=${captured.contextRevision}, current known=${current.known} ` +
+ `hostRevision=${current.hostRevision} contextRevision=${current.contextRevision})`
)
this.name = 'HostInventoryAuthorityUnavailableError'
}
@@ -471,8 +475,11 @@ export class HostReconciler {
const host = this.hostWatchAuthority()
return {
known: host.known,
- hostGeneration: host.generation,
- contextGeneration: 0,
+ // Standalone contract: no separate desired-revision plumbing here, so the
+ // Host watch generation stands in as the Host-only lease identity. The
+ // McpServerWatcher wires the real composite content revisions in production.
+ hostRevision: host.generation,
+ contextRevision: 0,
}
}
/**
@@ -630,8 +637,8 @@ export class HostReconciler {
if (
!captured.known ||
!current.known ||
- captured.hostGeneration !== current.hostGeneration ||
- captured.contextGeneration !== current.contextGeneration
+ captured.hostRevision !== current.hostRevision ||
+ captured.contextRevision !== current.contextRevision
) {
throw new HostInventoryAuthorityUnavailableError(action, captured, current)
}
diff --git a/host-context-controller/src/k8sClient.test.ts b/host-context-controller/src/k8sClient.test.ts
index 81ee317d7..73b96ebf1 100644
--- a/host-context-controller/src/k8sClient.test.ts
+++ b/host-context-controller/src/k8sClient.test.ts
@@ -9395,4 +9395,30 @@ describe('McpServerWatcher readiness under sustained watch churn (GKE Premature-
;(watcher as any).installMcpServerSnapshot({ servers: [], resourceVersion: 'rv4' })
expect((watcher as any).mcpServerDesiredRevision).toBe(base + 2)
})
+
+ it('installHostSnapshot bumps the desired revision only when re-listed content changed', () => {
+ const watcher = new McpServerWatcher()
+ const hostA = {
+ name: 'a',
+ namespace: 'mcp-host',
+ uid: 'ua',
+ generation: 1,
+ spec: { contextRef: 'ctx' },
+ } as unknown as HostCRD
+ ;(watcher as any).installHostSnapshot({ hosts: [hostA], resourceVersion: 'rv1' })
+ const base = (watcher as any).hostDesiredRevision
+ // Identical re-LIST (a Premature-close reconnect) must NOT bump — this is
+ // what stops the Host mutation-authority fence from starving every reconcile.
+ ;(watcher as any).installHostSnapshot({ hosts: [hostA], resourceVersion: 'rv2' })
+ expect((watcher as any).hostDesiredRevision).toBe(base)
+ // A spec change (metadata.generation advances) MUST bump.
+ ;(watcher as any).installHostSnapshot({
+ hosts: [{ ...hostA, generation: 2 }],
+ resourceVersion: 'rv3',
+ })
+ expect((watcher as any).hostDesiredRevision).toBe(base + 1)
+ // A removal MUST bump.
+ ;(watcher as any).installHostSnapshot({ hosts: [], resourceVersion: 'rv4' })
+ expect((watcher as any).hostDesiredRevision).toBe(base + 2)
+ })
})
diff --git a/host-context-controller/src/k8sClient.ts b/host-context-controller/src/k8sClient.ts
index 37a89ebf6..680ceda74 100644
--- a/host-context-controller/src/k8sClient.ts
+++ b/host-context-controller/src/k8sClient.ts
@@ -644,6 +644,7 @@ export class McpServerWatcher implements McpServerProvider {
private contextWatchGeneration = 0
private mcpServerDesiredRevision = 0
private contextDesiredRevision = 0
+ private hostDesiredRevision = 0
// Readiness covers authoritative revocation, not additive fleet completion.
// Generations bind the completed safety sweep to the exact LIST -> WATCH
// pair that supplied its absence decisions. A recovered watch invalidates
@@ -840,8 +841,8 @@ export class McpServerWatcher implements McpServerProvider {
}))
this.hostReconciler.setHostMutationAuthority(() => ({
known: this.hostCacheSynced && this.contextCacheSynced,
- hostGeneration: this.hostWatchGeneration,
- contextGeneration: this.contextWatchGeneration,
+ hostRevision: this.hostDesiredRevision,
+ contextRevision: this.contextDesiredRevision,
}))
this.hostReconciler.setResolveHostMutationDependencies(host => {
const context = this.contexts.get(host.spec.contextRef)
@@ -1504,7 +1505,29 @@ export class McpServerWatcher implements McpServerProvider {
this.clearAllHostWatchRetries()
}
+ // Host content identity for the diffing installer and the watch-event bump.
+ // Kubernetes bumps metadata.generation only on a spec change (not on the
+ // status writes HCC itself makes), and the uid distinguishes a delete+recreate
+ // of the same name. Same (uid, generation) = same desired Host state.
+ private sameHostDesiredRevision(previous: HostCRD, current: HostCRD): boolean {
+ return previous.uid === current.uid && previous.generation === current.generation
+ }
+
+ private hostSnapshotChangesDesiredState(snapshot: HostSnapshot): boolean {
+ if (snapshot.hosts.length !== this.hosts.size) return true
+ for (const host of snapshot.hosts) {
+ const previous = this.hosts.get(host.name)
+ if (previous === undefined || !this.sameHostDesiredRevision(previous, host)) return true
+ }
+ return false
+ }
+
private installHostSnapshot(snapshot: HostSnapshot): void {
+ // Same diffing rule as the Context/McpServer installers: a Premature-close
+ // reconnect that re-LISTs the identical Host inventory leaves hostDesiredRevision
+ // untouched, so the Host mutation-authority fence no longer starves every
+ // queued reconcile with HostInventoryAuthorityUnavailableError under churn.
+ if (this.hostSnapshotChangesDesiredState(snapshot)) this.hostDesiredRevision += 1
this.hosts.clear()
for (const host of snapshot.hosts) this.hosts.set(host.name, host)
}
@@ -3959,11 +3982,19 @@ export class McpServerWatcher implements McpServerProvider {
this.clearHostWatchRetry(host.name)
const eventRevision = ++this.hostWatchRevision
this.latestHostWatchEventRevisions.set(host.name, eventRevision)
+ // Advance hostDesiredRevision only on a real desired-state change (the
+ // content identity the mutation-authority fence reads), separate from the
+ // per-event hostWatchRevision counter above.
+ const previousHost = this.hosts.get(host.name)
+ let hostDesiredStateChanged = false
if (eventType === 'ADDED' || eventType === 'MODIFIED') {
+ hostDesiredStateChanged =
+ previousHost === undefined || !this.sameHostDesiredRevision(previousHost, host)
this.hosts.set(host.name, host)
} else {
- this.hosts.delete(host.name)
+ hostDesiredStateChanged = this.hosts.delete(host.name)
}
+ if (hostDesiredStateChanged) this.hostDesiredRevision += 1
try {
// Direct dispatch (§10.3): reconcileHostWatchEvent enters the per-Host
diff --git a/host-context-controller/test/hostReconciler.heartbeat.test.ts b/host-context-controller/test/hostReconciler.heartbeat.test.ts
index 17ac53f0f..aac872754 100644
--- a/host-context-controller/test/hostReconciler.heartbeat.test.ts
+++ b/host-context-controller/test/hostReconciler.heartbeat.test.ts
@@ -945,8 +945,8 @@ describe('HostReconciler.suspendHostFromHeartbeat', () => {
reconciler.setResolveCurrentHost(name => (name === host.name ? host : undefined))
reconciler.setHostMutationAuthority(() => ({
known: true,
- hostGeneration: 1,
- contextGeneration: 1,
+ hostRevision: 1,
+ contextRevision: 1,
}))
const draining = freshHostRead({ state: 'draining', wakeHandledGeneration: 2 })
Object.assign(draining.metadata, {