Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript

fix(hcc): name which inventory side aborted the NetworkPolicy pass (#462)

Alfredo Lopez1cdc679f62c9
diff --git a/host-context-controller/package-lock.json b/host-context-controller/package-lock.json
index 1d7ff668d..6055cad7b 100644
--- a/host-context-controller/package-lock.json
+++ b/host-context-controller/package-lock.json
@@ -1,12 +1,12 @@
{
"name": "clerum-host-context-controller",
- "version": "0.1.247",
+ "version": "0.1.248",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "clerum-host-context-controller",
- "version": "0.1.247",
+ "version": "0.1.248",
"license": "SEE LICENSE IN LICENSE",
"dependencies": {
"@clerum/image-policy": "file:../packages/image-policy",
diff --git a/host-context-controller/package.json b/host-context-controller/package.json
index 44516712a..b4d204505 100644
--- a/host-context-controller/package.json
+++ b/host-context-controller/package.json
@@ -1,6 +1,6 @@
{
"name": "clerum-host-context-controller",
- "version": "0.1.247",
+ "version": "0.1.248",
"description": "Host Context Controller - provides McpServer CRDs to mcp-host via REST API",
"main": "dist/main.js",
"engines": {
diff --git a/host-context-controller/src/k8sClient.test.ts b/host-context-controller/src/k8sClient.test.ts
index 252826d87..2687ec528 100644
--- a/host-context-controller/src/k8sClient.test.ts
+++ b/host-context-controller/src/k8sClient.test.ts
@@ -4292,7 +4292,13 @@ describe('McpServerWatcher startup', () => {
await pass
expect(warnSpy).toHaveBeenCalledWith(
- '[K8s] pass ended without certifying: inventory authority lost'
+ '[K8s] pass ended without certifying: inventory authority lost',
+ expect.objectContaining({
+ contextMoved: true,
+ serverMoved: false,
+ contextCacheSynced: true,
+ mcpServerCacheSynced: true,
+ })
)
expect((watcher as any).initialConvergenceRetryAttempts.get('NetworkPolicy')).toBe(5)
expect((watcher as any).initialConvergenceRetryTimers.has('NetworkPolicy')).toBe(true)
@@ -4352,7 +4358,13 @@ describe('McpServerWatcher startup', () => {
await pass
expect(warnSpy).toHaveBeenCalledWith(
- '[K8s] pass ended without certifying: inventory authority lost'
+ '[K8s] pass ended without certifying: inventory authority lost',
+ expect.objectContaining({
+ contextMoved: false,
+ serverMoved: true,
+ contextCacheSynced: true,
+ mcpServerCacheSynced: true,
+ })
)
expect((watcher as any).initialConvergenceRetryAttempts.get('NetworkPolicy')).toBe(5)
expect((watcher as any).initialConvergenceRetryTimers.has('NetworkPolicy')).toBe(true)
@@ -5111,6 +5123,7 @@ describe('McpServerWatcher startup', () => {
it('names the inventory-changed throw aborted-bump', async () => {
const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {})
+ const warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => {})
const watcher = new McpServerWatcher()
;(watcher as any).contextCacheSynced = true
;(watcher as any).mcpServerCacheSynced = true
@@ -5131,7 +5144,17 @@ describe('McpServerWatcher startup', () => {
})
).toBe(bumpBefore + 1)
expect((watcher as any).initialConvergenceRetryAttempts.get('NetworkPolicy')).toBe(1)
+ expect(warnSpy).toHaveBeenCalledWith(
+ '[K8s] pass ended without certifying: desired inventory changed',
+ expect.objectContaining({
+ contextMoved: false,
+ serverMoved: false,
+ contextCacheSynced: true,
+ mcpServerCacheSynced: true,
+ })
+ )
errorSpy.mockRestore()
+ warnSpy.mockRestore()
await watcher.stop()
})
diff --git a/host-context-controller/src/k8sClient.ts b/host-context-controller/src/k8sClient.ts
index a8a22bf91..4ecd88918 100644
--- a/host-context-controller/src/k8sClient.ts
+++ b/host-context-controller/src/k8sClient.ts
@@ -3083,23 +3083,26 @@ export class McpServerWatcher implements McpServerProvider {
return
}
let authoritativeRevocationCompleted = false
+ let safetyCertificate: NetworkPolicySafetyCertificate | undefined
try {
const initialContexts = [...this.contexts.values()]
const initialServers = [...this.servers.values()]
const contextInventoryGeneration = this.contextWatchGeneration
const serverInventoryGeneration = this.mcpWatchGeneration
- const safetyCertificate: NetworkPolicySafetyCertificate = {
+ const capturedSafetyCertificate: NetworkPolicySafetyCertificate = {
contextGeneration: contextInventoryGeneration,
serverGeneration: serverInventoryGeneration,
contextRevision: this.contextDesiredRevision,
serverRevision: this.mcpServerDesiredRevision,
}
+ safetyCertificate = capturedSafetyCertificate
const serverInventoryComplete = this.mcpServerCacheSynced
const contextInventoryAuthoritative = () =>
- this.contextCacheSynced && this.contextDesiredRevision === safetyCertificate.contextRevision
+ this.contextCacheSynced &&
+ this.contextDesiredRevision === capturedSafetyCertificate.contextRevision
const serverInventoryAuthoritative = () =>
this.mcpServerCacheSynced &&
- this.mcpServerDesiredRevision === safetyCertificate.serverRevision
+ this.mcpServerDesiredRevision === capturedSafetyCertificate.serverRevision
console.log('[K8s] Running initial NetworkPolicy background reconciliation...')
await this.netPolReconciler.fullReconcile(initialContexts, initialServers, {
serverInventoryComplete,
@@ -3147,7 +3150,7 @@ export class McpServerWatcher implements McpServerProvider {
onExternalEgressRevoked: server =>
this.externalEgressCoordinator.scheduleRetry('MODIFIED', server),
onAuthoritativeRevocationComplete: () => {
- if (!this.recordNetworkPolicySafetyCertificate(safetyCertificate)) return
+ if (!this.recordNetworkPolicySafetyCertificate(capturedSafetyCertificate)) return
authoritativeRevocationCompleted = true
// Start runtime convergence at the safety boundary, before this full
// pass enters additive Context work. Startup egress gates may now
@@ -3157,7 +3160,10 @@ export class McpServerWatcher implements McpServerProvider {
},
})
if (!contextInventoryAuthoritative() || !serverInventoryAuthoritative()) {
- console.warn('[K8s] pass ended without certifying: inventory authority lost')
+ console.warn(
+ '[K8s] pass ended without certifying: inventory authority lost',
+ this.networkPolicyInventoryMovement(capturedSafetyCertificate)
+ )
initialConvergenceSwallowedTotal.inc({ lane: 'NetworkPolicy', sink: 'authority-lost' })
observeInitialNetworkPolicyPass(startedAtMs, 'aborted-authority')
this.scheduleInitialConvergenceRetry('NetworkPolicy')
@@ -3178,11 +3184,31 @@ export class McpServerWatcher implements McpServerProvider {
)
const abortedBump =
error instanceof Error && error.message === DESIRED_NETWORKPOLICY_INVENTORY_CHANGED_MESSAGE
+ if (abortedBump && safetyCertificate) {
+ console.warn(
+ '[K8s] pass ended without certifying: desired inventory changed',
+ this.networkPolicyInventoryMovement(safetyCertificate)
+ )
+ }
observeInitialNetworkPolicyPass(startedAtMs, abortedBump ? 'aborted-bump' : 'failed')
this.scheduleInitialConvergenceRetry('NetworkPolicy')
}
}
+ private networkPolicyInventoryMovement(certificate: NetworkPolicySafetyCertificate): {
+ contextMoved: boolean
+ serverMoved: boolean
+ contextCacheSynced: boolean
+ mcpServerCacheSynced: boolean
+ } {
+ return {
+ contextMoved: this.contextDesiredRevision !== certificate.contextRevision,
+ serverMoved: this.mcpServerDesiredRevision !== certificate.serverRevision,
+ contextCacheSynced: this.contextCacheSynced,
+ mcpServerCacheSynced: this.mcpServerCacheSynced,
+ }
+ }
+
private clearInitialConvergenceRetry(lane: InitialConvergenceLane): void {
const timer = this.initialConvergenceRetryTimers.get(lane)
if (timer) {
diff --git a/host-context-controller/src/networkPolicyReconciler.ts b/host-context-controller/src/networkPolicyReconciler.ts
index b23a76c91..0c048d5b6 100644
--- a/host-context-controller/src/networkPolicyReconciler.ts
+++ b/host-context-controller/src/networkPolicyReconciler.ts
@@ -391,8 +391,10 @@ export interface NetworkPolicyFullReconcileOptions {
resolveCurrentServer?: (name: string) => McpServerCRD | undefined
/**
* Monotonic desired-state revisions fence ordinary watch events that arrive
- * after an owner has completed its safety turn. Watch generations remain the
- * separate authority fence for LIST -> WATCH recovery.
+ * after an owner has completed its safety turn. The in-flight NetworkPolicy
+ * pass also uses these revisions (plus cacheSynced) as its authority fence.
+ * Watch generations remain the delete-confirm / watch-effect lease, not the
+ * pass abort predicate.
*/
contextDesiredRevision?: () => number
serverDesiredRevision?: () => number