Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
fix(hcc): apply content identity to the Host mutation-authority fence (phase 1b)
Alfredo Lopez6496fdfacf20
diff --git a/host-context-controller/src/hostReconciler.test.ts b/host-context-controller/src/hostReconciler.test.tsindex 53625ff43..6590681b1 100644--- a/host-context-controller/src/hostReconciler.test.ts+++ b/host-context-controller/src/hostReconciler.test.ts@@ -1108,8 +1108,8 @@ describe('HostReconciler Host inventory mutation authority', () => {const mutationAuthority = {current: {known: true,- hostGeneration: 7,- contextGeneration: 11,+ hostRevision: 7,+ contextRevision: 11,},};(reconciler as any).setHostMutationAuthority(() => mutationAuthority.current)@@ -1126,13 +1126,15 @@ describe('HostReconciler Host inventory mutation authority', () => {const fleet = reconciler.fullReconcile([hostA, hostB])await vi.waitFor(() => expect(reconcileCore).toHaveBeenCalledWith(hostA, expect.any(Function)))- // Context retired and recovered to a fresh LIST -> WATCH generation while- // Host authority remained continuously known: a host-only lease would miss- // this ABA transition and admit Host B with stale derived mounts.+ // Context desired state changed (its content revision advanced) while Host+ // authority remained continuously known: a host-only lease would miss this+ // transition and admit Host B with stale derived mounts. A pure watch+ // reconnect with identical content would NOT advance the revision and is+ // correctly admitted — that is the point of the content-identity fence.mutationAuthority.current = {known: true,- hostGeneration: 7,- contextGeneration: 13,+ hostRevision: 7,+ contextRevision: 13,}releaseBlocker.resolve(undefined)await occupied@@ -1160,8 +1162,8 @@ describe('HostReconciler Host inventory mutation authority', () => {reconciler.setResolveCurrentHost(name => (name === host.name ? host : undefined))reconciler.setHostMutationAuthority(() => ({known: true,- hostGeneration: 7,- contextGeneration: 11,+ hostRevision: 7,+ contextRevision: 11,}))reconciler.setResolveHostMutationDependencies(() => [selectedContext])@@ -1216,8 +1218,8 @@ describe('HostReconciler Host inventory mutation authority', () => {reconciler.setResolveCurrentHost(name => (name === host.name ? host : undefined))reconciler.setHostMutationAuthority(() => ({known: true,- hostGeneration: 7,- contextGeneration: 11,+ hostRevision: 7,+ contextRevision: 11,}))reconciler.setResolveHostMutationDependencies(() => [selectedContext])@@ -1272,8 +1274,8 @@ describe('HostReconciler Host inventory mutation authority', () => {reconciler.setResolveCurrentHost(name => (name === host.name ? host : undefined))reconciler.setHostMutationAuthority(() => ({known: true,- hostGeneration: 7,- contextGeneration: 11,+ hostRevision: 7,+ contextRevision: 11,}))reconciler.setResolveHostMutationDependencies(() => [context, selectedDependency])diff --git a/host-context-controller/src/hostReconciler.ts b/host-context-controller/src/hostReconciler.tsindex 9e2f236d5..94008e22f 100644--- a/host-context-controller/src/hostReconciler.ts+++ b/host-context-controller/src/hostReconciler.ts@@ -82,13 +82,17 @@ export type HostWatchAuthoritySnapshot = { known: boolean; generation: number }/*** Immutable lease for every inventory that contributes to a Host mutation.* Host resources come from the Host watch; mounted SharedFileSystems come from- * the Context watch. Keeping both generations prevents a false-stable- * Host-only lease when Context retires and recovers during queued work.+ * the Context watch. Both are CONTENT revisions (they move only on a real+ * desired-state change), not channel generations — a watch reconnect that+ * re-LISTs the identical inventory must not retire an in-flight Host lease, or+ * sustained watch-churn starves every queued reconcile. Keeping both revisions+ * still prevents a false-stable Host-only lease when Context desired state+ * changes during queued work.*/export type HostMutationAuthoritySnapshot = {known: boolean- hostGeneration: number- contextGeneration: number+ hostRevision: number+ contextRevision: number}class HostInventoryAuthorityUnavailableError extends Error {@@ -99,9 +103,9 @@ class HostInventoryAuthorityUnavailableError extends Error {) {super(`Host inventory authority changed before ${action} admission ` +- `(captured known=${captured.known} hostGeneration=${captured.hostGeneration} ` +- `contextGeneration=${captured.contextGeneration}, current known=${current.known} ` +- `hostGeneration=${current.hostGeneration} contextGeneration=${current.contextGeneration})`+ `(captured known=${captured.known} hostRevision=${captured.hostRevision} ` ++ `contextRevision=${captured.contextRevision}, current known=${current.known} ` ++ `hostRevision=${current.hostRevision} contextRevision=${current.contextRevision})`)this.name = 'HostInventoryAuthorityUnavailableError'}@@ -471,8 +475,11 @@ export class HostReconciler {const host = this.hostWatchAuthority()return {known: host.known,- hostGeneration: host.generation,- contextGeneration: 0,+ // Standalone contract: no separate desired-revision plumbing here, so the+ // Host watch generation stands in as the Host-only lease identity. The+ // McpServerWatcher wires the real composite content revisions in production.+ hostRevision: host.generation,+ contextRevision: 0,}}/**@@ -630,8 +637,8 @@ export class HostReconciler {if (!captured.known ||!current.known ||- captured.hostGeneration !== current.hostGeneration ||- captured.contextGeneration !== current.contextGeneration+ captured.hostRevision !== current.hostRevision ||+ captured.contextRevision !== current.contextRevision) {throw new HostInventoryAuthorityUnavailableError(action, captured, current)}diff --git a/host-context-controller/src/k8sClient.test.ts b/host-context-controller/src/k8sClient.test.tsindex 81ee317d7..73b96ebf1 100644--- a/host-context-controller/src/k8sClient.test.ts+++ b/host-context-controller/src/k8sClient.test.ts@@ -9395,4 +9395,30 @@ describe('McpServerWatcher readiness under sustained watch churn (GKE Premature-;(watcher as any).installMcpServerSnapshot({ servers: [], resourceVersion: 'rv4' })expect((watcher as any).mcpServerDesiredRevision).toBe(base + 2)})++ it('installHostSnapshot bumps the desired revision only when re-listed content changed', () => {+ const watcher = new McpServerWatcher()+ const hostA = {+ name: 'a',+ namespace: 'mcp-host',+ uid: 'ua',+ generation: 1,+ spec: { contextRef: 'ctx' },+ } as unknown as HostCRD+ ;(watcher as any).installHostSnapshot({ hosts: [hostA], resourceVersion: 'rv1' })+ const base = (watcher as any).hostDesiredRevision+ // Identical re-LIST (a Premature-close reconnect) must NOT bump — this is+ // what stops the Host mutation-authority fence from starving every reconcile.+ ;(watcher as any).installHostSnapshot({ hosts: [hostA], resourceVersion: 'rv2' })+ expect((watcher as any).hostDesiredRevision).toBe(base)+ // A spec change (metadata.generation advances) MUST bump.+ ;(watcher as any).installHostSnapshot({+ hosts: [{ ...hostA, generation: 2 }],+ resourceVersion: 'rv3',+ })+ expect((watcher as any).hostDesiredRevision).toBe(base + 1)+ // A removal MUST bump.+ ;(watcher as any).installHostSnapshot({ hosts: [], resourceVersion: 'rv4' })+ expect((watcher as any).hostDesiredRevision).toBe(base + 2)+ })})diff --git a/host-context-controller/src/k8sClient.ts b/host-context-controller/src/k8sClient.tsindex 37a89ebf6..680ceda74 100644--- a/host-context-controller/src/k8sClient.ts+++ b/host-context-controller/src/k8sClient.ts@@ -644,6 +644,7 @@ export class McpServerWatcher implements McpServerProvider {private contextWatchGeneration = 0private mcpServerDesiredRevision = 0private contextDesiredRevision = 0+ private hostDesiredRevision = 0// Readiness covers authoritative revocation, not additive fleet completion.// Generations bind the completed safety sweep to the exact LIST -> WATCH// pair that supplied its absence decisions. A recovered watch invalidates@@ -840,8 +841,8 @@ export class McpServerWatcher implements McpServerProvider {}))this.hostReconciler.setHostMutationAuthority(() => ({known: this.hostCacheSynced && this.contextCacheSynced,- hostGeneration: this.hostWatchGeneration,- contextGeneration: this.contextWatchGeneration,+ hostRevision: this.hostDesiredRevision,+ contextRevision: this.contextDesiredRevision,}))this.hostReconciler.setResolveHostMutationDependencies(host => {const context = this.contexts.get(host.spec.contextRef)@@ -1504,7 +1505,29 @@ export class McpServerWatcher implements McpServerProvider {this.clearAllHostWatchRetries()}+ // Host content identity for the diffing installer and the watch-event bump.+ // Kubernetes bumps metadata.generation only on a spec change (not on the+ // status writes HCC itself makes), and the uid distinguishes a delete+recreate+ // of the same name. Same (uid, generation) = same desired Host state.+ private sameHostDesiredRevision(previous: HostCRD, current: HostCRD): boolean {+ return previous.uid === current.uid && previous.generation === current.generation+ }++ private hostSnapshotChangesDesiredState(snapshot: HostSnapshot): boolean {+ if (snapshot.hosts.length !== this.hosts.size) return true+ for (const host of snapshot.hosts) {+ const previous = this.hosts.get(host.name)+ if (previous === undefined || !this.sameHostDesiredRevision(previous, host)) return true+ }+ return false+ }+private installHostSnapshot(snapshot: HostSnapshot): void {+ // Same diffing rule as the Context/McpServer installers: a Premature-close+ // reconnect that re-LISTs the identical Host inventory leaves hostDesiredRevision+ // untouched, so the Host mutation-authority fence no longer starves every+ // queued reconcile with HostInventoryAuthorityUnavailableError under churn.+ if (this.hostSnapshotChangesDesiredState(snapshot)) this.hostDesiredRevision += 1this.hosts.clear()for (const host of snapshot.hosts) this.hosts.set(host.name, host)}@@ -3959,11 +3982,19 @@ export class McpServerWatcher implements McpServerProvider {this.clearHostWatchRetry(host.name)const eventRevision = ++this.hostWatchRevisionthis.latestHostWatchEventRevisions.set(host.name, eventRevision)+ // Advance hostDesiredRevision only on a real desired-state change (the+ // content identity the mutation-authority fence reads), separate from the+ // per-event hostWatchRevision counter above.+ const previousHost = this.hosts.get(host.name)+ let hostDesiredStateChanged = falseif (eventType === 'ADDED' || eventType === 'MODIFIED') {+ hostDesiredStateChanged =+ previousHost === undefined || !this.sameHostDesiredRevision(previousHost, host)this.hosts.set(host.name, host)} else {- this.hosts.delete(host.name)+ hostDesiredStateChanged = this.hosts.delete(host.name)}+ if (hostDesiredStateChanged) this.hostDesiredRevision += 1try {// Direct dispatch (§10.3): reconcileHostWatchEvent enters the per-Hostdiff --git a/host-context-controller/test/hostReconciler.heartbeat.test.ts b/host-context-controller/test/hostReconciler.heartbeat.test.tsindex 17ac53f0f..aac872754 100644--- a/host-context-controller/test/hostReconciler.heartbeat.test.ts+++ b/host-context-controller/test/hostReconciler.heartbeat.test.ts@@ -945,8 +945,8 @@ describe('HostReconciler.suspendHostFromHeartbeat', () => {reconciler.setResolveCurrentHost(name => (name === host.name ? host : undefined))reconciler.setHostMutationAuthority(() => ({known: true,- hostGeneration: 1,- contextGeneration: 1,+ hostRevision: 1,+ contextRevision: 1,}))const draining = freshHostRead({ state: 'draining', wakeHandledGeneration: 2 })Object.assign(draining.metadata, {