Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
fix(hcc): name which inventory side aborted the NetworkPolicy pass (#462)
Alfredo Lopez1cdc679f62c9
diff --git a/host-context-controller/package-lock.json b/host-context-controller/package-lock.jsonindex 1d7ff668d..6055cad7b 100644--- a/host-context-controller/package-lock.json+++ b/host-context-controller/package-lock.json@@ -1,12 +1,12 @@{"name": "clerum-host-context-controller",- "version": "0.1.247",+ "version": "0.1.248","lockfileVersion": 3,"requires": true,"packages": {"": {"name": "clerum-host-context-controller",- "version": "0.1.247",+ "version": "0.1.248","license": "SEE LICENSE IN LICENSE","dependencies": {"@clerum/image-policy": "file:../packages/image-policy",diff --git a/host-context-controller/package.json b/host-context-controller/package.jsonindex 44516712a..b4d204505 100644--- a/host-context-controller/package.json+++ b/host-context-controller/package.json@@ -1,6 +1,6 @@{"name": "clerum-host-context-controller",- "version": "0.1.247",+ "version": "0.1.248","description": "Host Context Controller - provides McpServer CRDs to mcp-host via REST API","main": "dist/main.js","engines": {diff --git a/host-context-controller/src/k8sClient.test.ts b/host-context-controller/src/k8sClient.test.tsindex 252826d87..2687ec528 100644--- a/host-context-controller/src/k8sClient.test.ts+++ b/host-context-controller/src/k8sClient.test.ts@@ -4292,7 +4292,13 @@ describe('McpServerWatcher startup', () => {await passexpect(warnSpy).toHaveBeenCalledWith(- '[K8s] pass ended without certifying: inventory authority lost'+ '[K8s] pass ended without certifying: inventory authority lost',+ expect.objectContaining({+ contextMoved: true,+ serverMoved: false,+ contextCacheSynced: true,+ mcpServerCacheSynced: true,+ }))expect((watcher as any).initialConvergenceRetryAttempts.get('NetworkPolicy')).toBe(5)expect((watcher as any).initialConvergenceRetryTimers.has('NetworkPolicy')).toBe(true)@@ -4352,7 +4358,13 @@ describe('McpServerWatcher startup', () => {await passexpect(warnSpy).toHaveBeenCalledWith(- '[K8s] pass ended without certifying: inventory authority lost'+ '[K8s] pass ended without certifying: inventory authority lost',+ expect.objectContaining({+ contextMoved: false,+ serverMoved: true,+ contextCacheSynced: true,+ mcpServerCacheSynced: true,+ }))expect((watcher as any).initialConvergenceRetryAttempts.get('NetworkPolicy')).toBe(5)expect((watcher as any).initialConvergenceRetryTimers.has('NetworkPolicy')).toBe(true)@@ -5111,6 +5123,7 @@ describe('McpServerWatcher startup', () => {it('names the inventory-changed throw aborted-bump', async () => {const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {})+ const warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => {})const watcher = new McpServerWatcher();(watcher as any).contextCacheSynced = true;(watcher as any).mcpServerCacheSynced = true@@ -5131,7 +5144,17 @@ describe('McpServerWatcher startup', () => {})).toBe(bumpBefore + 1)expect((watcher as any).initialConvergenceRetryAttempts.get('NetworkPolicy')).toBe(1)+ expect(warnSpy).toHaveBeenCalledWith(+ '[K8s] pass ended without certifying: desired inventory changed',+ expect.objectContaining({+ contextMoved: false,+ serverMoved: false,+ contextCacheSynced: true,+ mcpServerCacheSynced: true,+ })+ )errorSpy.mockRestore()+ warnSpy.mockRestore()await watcher.stop()})diff --git a/host-context-controller/src/k8sClient.ts b/host-context-controller/src/k8sClient.tsindex a8a22bf91..4ecd88918 100644--- a/host-context-controller/src/k8sClient.ts+++ b/host-context-controller/src/k8sClient.ts@@ -3083,23 +3083,26 @@ export class McpServerWatcher implements McpServerProvider {return}let authoritativeRevocationCompleted = false+ let safetyCertificate: NetworkPolicySafetyCertificate | undefinedtry {const initialContexts = [...this.contexts.values()]const initialServers = [...this.servers.values()]const contextInventoryGeneration = this.contextWatchGenerationconst serverInventoryGeneration = this.mcpWatchGeneration- const safetyCertificate: NetworkPolicySafetyCertificate = {+ const capturedSafetyCertificate: NetworkPolicySafetyCertificate = {contextGeneration: contextInventoryGeneration,serverGeneration: serverInventoryGeneration,contextRevision: this.contextDesiredRevision,serverRevision: this.mcpServerDesiredRevision,}+ safetyCertificate = capturedSafetyCertificateconst serverInventoryComplete = this.mcpServerCacheSyncedconst contextInventoryAuthoritative = () =>- this.contextCacheSynced && this.contextDesiredRevision === safetyCertificate.contextRevision+ this.contextCacheSynced &&+ this.contextDesiredRevision === capturedSafetyCertificate.contextRevisionconst serverInventoryAuthoritative = () =>this.mcpServerCacheSynced &&- this.mcpServerDesiredRevision === safetyCertificate.serverRevision+ this.mcpServerDesiredRevision === capturedSafetyCertificate.serverRevisionconsole.log('[K8s] Running initial NetworkPolicy background reconciliation...')await this.netPolReconciler.fullReconcile(initialContexts, initialServers, {serverInventoryComplete,@@ -3147,7 +3150,7 @@ export class McpServerWatcher implements McpServerProvider {onExternalEgressRevoked: server =>this.externalEgressCoordinator.scheduleRetry('MODIFIED', server),onAuthoritativeRevocationComplete: () => {- if (!this.recordNetworkPolicySafetyCertificate(safetyCertificate)) return+ if (!this.recordNetworkPolicySafetyCertificate(capturedSafetyCertificate)) returnauthoritativeRevocationCompleted = true// Start runtime convergence at the safety boundary, before this full// pass enters additive Context work. Startup egress gates may now@@ -3157,7 +3160,10 @@ export class McpServerWatcher implements McpServerProvider {},})if (!contextInventoryAuthoritative() || !serverInventoryAuthoritative()) {- console.warn('[K8s] pass ended without certifying: inventory authority lost')+ console.warn(+ '[K8s] pass ended without certifying: inventory authority lost',+ this.networkPolicyInventoryMovement(capturedSafetyCertificate)+ )initialConvergenceSwallowedTotal.inc({ lane: 'NetworkPolicy', sink: 'authority-lost' })observeInitialNetworkPolicyPass(startedAtMs, 'aborted-authority')this.scheduleInitialConvergenceRetry('NetworkPolicy')@@ -3178,11 +3184,31 @@ export class McpServerWatcher implements McpServerProvider {)const abortedBump =error instanceof Error && error.message === DESIRED_NETWORKPOLICY_INVENTORY_CHANGED_MESSAGE+ if (abortedBump && safetyCertificate) {+ console.warn(+ '[K8s] pass ended without certifying: desired inventory changed',+ this.networkPolicyInventoryMovement(safetyCertificate)+ )+ }observeInitialNetworkPolicyPass(startedAtMs, abortedBump ? 'aborted-bump' : 'failed')this.scheduleInitialConvergenceRetry('NetworkPolicy')}}+ private networkPolicyInventoryMovement(certificate: NetworkPolicySafetyCertificate): {+ contextMoved: boolean+ serverMoved: boolean+ contextCacheSynced: boolean+ mcpServerCacheSynced: boolean+ } {+ return {+ contextMoved: this.contextDesiredRevision !== certificate.contextRevision,+ serverMoved: this.mcpServerDesiredRevision !== certificate.serverRevision,+ contextCacheSynced: this.contextCacheSynced,+ mcpServerCacheSynced: this.mcpServerCacheSynced,+ }+ }+private clearInitialConvergenceRetry(lane: InitialConvergenceLane): void {const timer = this.initialConvergenceRetryTimers.get(lane)if (timer) {diff --git a/host-context-controller/src/networkPolicyReconciler.ts b/host-context-controller/src/networkPolicyReconciler.tsindex b23a76c91..0c048d5b6 100644--- a/host-context-controller/src/networkPolicyReconciler.ts+++ b/host-context-controller/src/networkPolicyReconciler.ts@@ -391,8 +391,10 @@ export interface NetworkPolicyFullReconcileOptions {resolveCurrentServer?: (name: string) => McpServerCRD | undefined/*** Monotonic desired-state revisions fence ordinary watch events that arrive- * after an owner has completed its safety turn. Watch generations remain the- * separate authority fence for LIST -> WATCH recovery.+ * after an owner has completed its safety turn. The in-flight NetworkPolicy+ * pass also uses these revisions (plus cacheSynced) as its authority fence.+ * Watch generations remain the delete-confirm / watch-effect lease, not the+ * pass abort predicate.*/contextDesiredRevision?: () => numberserverDesiredRevision?: () => number