Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
feat(hcc): add read-first convergence for network policies
Alfredo Lopez2d98b4eab526
diff --git a/host-context-controller/package-lock.json b/host-context-controller/package-lock.jsonindex 4587b15c7..b89eaa68a 100644--- a/host-context-controller/package-lock.json+++ b/host-context-controller/package-lock.json@@ -1,12 +1,12 @@{"name": "clerum-host-context-controller",- "version": "0.1.303",+ "version": "0.1.304","lockfileVersion": 3,"requires": true,"packages": {"": {"name": "clerum-host-context-controller",- "version": "0.1.303",+ "version": "0.1.304","license": "SEE LICENSE IN LICENSE","dependencies": {"@clerum/codex-catalog-projection": "file:../packages/codex-catalog-projection",diff --git a/host-context-controller/package.json b/host-context-controller/package.jsonindex e3d11c180..5efd6438d 100644--- a/host-context-controller/package.json+++ b/host-context-controller/package.json@@ -1,6 +1,6 @@{"name": "clerum-host-context-controller",- "version": "0.1.303",+ "version": "0.1.304","description": "Host Context Controller - provides McpServer CRDs to mcp-host via REST API","main": "dist/main.js","engines": {diff --git a/host-context-controller/src/__tests__/applyNetworkPolicyNoopGate.test.ts b/host-context-controller/src/__tests__/applyNetworkPolicyNoopGate.test.tsindex 99e3ced92..37dc2ae3a 100644--- a/host-context-controller/src/__tests__/applyNetworkPolicyNoopGate.test.ts+++ b/host-context-controller/src/__tests__/applyNetworkPolicyNoopGate.test.ts@@ -34,11 +34,15 @@ function fakeNetworkingApi() {}describe('applyNetworkPolicy no-op gate', () => {- it('CREATE-NP-1: successful create never reads or replaces', async () => {+ it('CREATE-NP-1: absent policy creates after one GET and never replaces', async () => {const api = fakeNetworkingApi()+ api.readNamespacedNetworkPolicy.mockRejectedValueOnce(apiException(404))await applyNetworkPolicy(api as unknown as k8s.NetworkingV1Api, 'np', 'ns', desiredPolicy())expect(api.createNamespacedNetworkPolicy).toHaveBeenCalledOnce()- expect(api.readNamespacedNetworkPolicy).not.toHaveBeenCalled()+ expect(api.readNamespacedNetworkPolicy).toHaveBeenCalledExactlyOnceWith({+ name: 'np',+ namespace: 'ns',+ })expect(api.replaceNamespacedNetworkPolicy).not.toHaveBeenCalled()})@@ -47,11 +51,12 @@ describe('applyNetworkPolicy no-op gate', () => {const existing = asApiserverNetworkPolicy(desired)expect(existing.spec).toBeDefined()const api = fakeNetworkingApi()- api.createNamespacedNetworkPolicy.mockRejectedValue({ code: 409 })api.readNamespacedNetworkPolicy.mockResolvedValue(existing)const log = vi.spyOn(console, 'log')try {await applyNetworkPolicy(api as unknown as k8s.NetworkingV1Api, 'np', 'ns', desired)+ expect(api.createNamespacedNetworkPolicy).not.toHaveBeenCalled()+ expect(api.readNamespacedNetworkPolicy).toHaveBeenCalledOnce()expect(api.replaceNamespacedNetworkPolicy).not.toHaveBeenCalled()expect(updatedPolicyLogs(log, 'policy "np" in ns')).toEqual([])} finally {@@ -62,13 +67,14 @@ describe('applyNetworkPolicy no-op gate', () => {it('NOOP-NP-2 / LOG-NP-2: drift replaces once and logs once', async () => {const desired = desiredPolicy()const api = fakeNetworkingApi()- api.createNamespacedNetworkPolicy.mockRejectedValue({ code: 409 })api.readNamespacedNetworkPolicy.mockResolvedValue(asApiserverNetworkPolicy(desired, { port: 9090 }))const log = vi.spyOn(console, 'log')try {await applyNetworkPolicy(api as unknown as k8s.NetworkingV1Api, 'np', 'ns', desired)+ expect(api.createNamespacedNetworkPolicy).not.toHaveBeenCalled()+ expect(api.readNamespacedNetworkPolicy).toHaveBeenCalledOnce()expect(api.replaceNamespacedNetworkPolicy).toHaveBeenCalledOnce()expect(updatedPolicyLogs(log, 'policy "np" in ns')).toEqual(['[NetPol] Updated policy "np" in ns',@@ -81,7 +87,6 @@ describe('applyNetworkPolicy no-op gate', () => {it('ORDER-NP-1: validateExisting throws before isUpToDate; replace is not called', async () => {const desired = desiredPolicy()const api = fakeNetworkingApi()- api.createNamespacedNetworkPolicy.mockRejectedValue({ code: 409 })api.readNamespacedNetworkPolicy.mockResolvedValue(asApiserverNetworkPolicy(desired))const validateExisting = vi.fn(() => {throw new Error('NetworkPolicy "np" has conflicting ownership for the context-ingress lane')@@ -99,36 +104,44 @@ describe('applyNetworkPolicy no-op gate', () => {)).rejects.toThrow(/ownership/)- expect(validateExisting).toHaveBeenCalledOnce()+ expect(validateExisting).toHaveBeenCalledExactlyOnceWith(asApiserverNetworkPolicy(desired))+ expect(api.createNamespacedNetworkPolicy).not.toHaveBeenCalled()expect(api.replaceNamespacedNetworkPolicy).not.toHaveBeenCalled()})it.each([{ form: 'ApiException', error: apiException(404) },{ form: 'statusCode', error: { response: { statusCode: 404 } } },- ])('TOCTOU-NP-1: create 409 + read 404 ($form) returns without replace', async ({ error }) => {- const api = fakeNetworkingApi()- api.createNamespacedNetworkPolicy.mockRejectedValue({ code: 409 })- api.readNamespacedNetworkPolicy.mockRejectedValue(error)+ ])(+ 'TOCTOU-NP-1: GET404 + create409 + fresh GET404 ($form) returns without replace',+ async ({ error }) => {+ const api = fakeNetworkingApi()+ api.createNamespacedNetworkPolicy.mockRejectedValue({ code: 409 })+ api.readNamespacedNetworkPolicy+ .mockRejectedValueOnce(apiException(404))+ .mockRejectedValueOnce(error)- await expect(- applyNetworkPolicy(api as unknown as k8s.NetworkingV1Api, 'np', 'ns', desiredPolicy())- ).resolves.toBeUndefined()+ await expect(+ applyNetworkPolicy(api as unknown as k8s.NetworkingV1Api, 'np', 'ns', desiredPolicy())+ ).resolves.toBeUndefined()- expect(api.createNamespacedNetworkPolicy).toHaveBeenCalledOnce()- expect(api.readNamespacedNetworkPolicy).toHaveBeenCalledOnce()- expect(api.replaceNamespacedNetworkPolicy).not.toHaveBeenCalled()- })+ expect(api.createNamespacedNetworkPolicy).toHaveBeenCalledOnce()+ expect(api.readNamespacedNetworkPolicy).toHaveBeenCalledTimes(2)+ expect(api.replaceNamespacedNetworkPolicy).not.toHaveBeenCalled()+ }+ )it.each([{ form: 'ApiException', error: apiException(404) },{ form: 'statusCode', error: { response: { statusCode: 404 } } },])(- 'admission-sensitive create409/read404 ($form) rejects instead of certifying absence',+ 'admission-sensitive GET404/create409/fresh GET404 ($form) rejects instead of certifying absence',async ({ error }) => {const api = fakeNetworkingApi()api.createNamespacedNetworkPolicy.mockRejectedValue({ code: 409 })- api.readNamespacedNetworkPolicy.mockRejectedValue(error)+ api.readNamespacedNetworkPolicy+ .mockRejectedValueOnce(apiException(404))+ .mockRejectedValueOnce(error)await expect(applyNetworkPolicy(@@ -143,6 +156,8 @@ describe('applyNetworkPolicy no-op gate', () => {)).rejects.toBe(error)+ expect(api.createNamespacedNetworkPolicy).toHaveBeenCalledOnce()+ expect(api.readNamespacedNetworkPolicy).toHaveBeenCalledTimes(2)expect(api.replaceNamespacedNetworkPolicy).not.toHaveBeenCalled()})@@ -152,28 +167,36 @@ describe('applyNetworkPolicy no-op gate', () => {{ status: 403, form: 'statusCode', error: { response: { statusCode: 403 } } },{ status: 500, form: 'ApiException', error: apiException(500) },{ status: 500, form: 'statusCode', error: { response: { statusCode: 500 } } },- ])('TOCTOU-NP-2-APPLY: create 409 + read $status ($form) still throws', async ({ error }) => {- const api = fakeNetworkingApi()- api.createNamespacedNetworkPolicy.mockRejectedValue({ code: 409 })- api.readNamespacedNetworkPolicy.mockRejectedValue(error)+ ])(+ 'TOCTOU-NP-2-APPLY: GET404 + create409 + fresh GET$status ($form) still throws',+ async ({ error }) => {+ const api = fakeNetworkingApi()+ api.createNamespacedNetworkPolicy.mockRejectedValue({ code: 409 })+ api.readNamespacedNetworkPolicy+ .mockRejectedValueOnce(apiException(404))+ .mockRejectedValueOnce(error)- await expect(- applyNetworkPolicy(api as unknown as k8s.NetworkingV1Api, 'np', 'ns', desiredPolicy())- ).rejects.toBe(error)+ await expect(+ applyNetworkPolicy(api as unknown as k8s.NetworkingV1Api, 'np', 'ns', desiredPolicy())+ ).rejects.toBe(error)- expect(api.createNamespacedNetworkPolicy).toHaveBeenCalledOnce()- expect(api.readNamespacedNetworkPolicy).toHaveBeenCalledOnce()- expect(api.replaceNamespacedNetworkPolicy).not.toHaveBeenCalled()- })+ expect(api.createNamespacedNetworkPolicy).toHaveBeenCalledOnce()+ expect(api.readNamespacedNetworkPolicy).toHaveBeenCalledTimes(2)+ expect(api.replaceNamespacedNetworkPolicy).not.toHaveBeenCalled()+ }+ )- it('GATE-NP-1: mutationAllowed false after create-409 skips replace on drift', async () => {+ it('GATE-NP-1: expiry during existence GET suppresses POST and drift PUT', async () => {const desired = desiredPolicy()const drifted = asApiserverNetworkPolicy(desired, { port: 9090 })expect(networkPolicyMatchesDesired(desired, drifted)).toBe(false)const api = fakeNetworkingApi()- api.createNamespacedNetworkPolicy.mockRejectedValue({ code: 409 })- api.readNamespacedNetworkPolicy.mockResolvedValue(drifted)- const mutationAllowed = vi.fn().mockReturnValueOnce(true).mockReturnValueOnce(false)+ let active = true+ api.readNamespacedNetworkPolicy.mockImplementation(async () => {+ active = false+ return drifted+ })+ const mutationAllowed = vi.fn(() => active)await applyNetworkPolicy(api as unknown as k8s.NetworkingV1Api,@@ -184,7 +207,9 @@ describe('applyNetworkPolicy no-op gate', () => {mutationAllowed)- expect(mutationAllowed).toHaveBeenCalledTimes(2)+ expect(mutationAllowed.mock.results.map(result => result.value)).toEqual([true, false])+ expect(api.readNamespacedNetworkPolicy).toHaveBeenCalledOnce()+ expect(api.createNamespacedNetworkPolicy).not.toHaveBeenCalled()expect(api.replaceNamespacedNetworkPolicy).not.toHaveBeenCalled()})})diff --git a/host-context-controller/src/__tests__/hostNetworkPolicyNoopGate.test.ts b/host-context-controller/src/__tests__/hostNetworkPolicyNoopGate.test.tsindex 2fa4f8053..7981c5dbc 100644--- a/host-context-controller/src/__tests__/hostNetworkPolicyNoopGate.test.ts+++ b/host-context-controller/src/__tests__/hostNetworkPolicyNoopGate.test.ts@@ -51,14 +51,24 @@ describe('Host ensureMcpHostIngressNetworkPolicy inherits applyNetworkPolicy gatlet captured: k8s.V1NetworkPolicy | undefinednetworkingApi.createNamespacedNetworkPolicy.mockImplementation(async ({ body }) => {captured = body as k8s.V1NetworkPolicy- throw Object.assign(new Error('already exists'), { code: 409 })+ return body})networkingApi.readNamespacedNetworkPolicy.mockImplementation(async () => {- if (!captured?.spec) throw new Error('create did not capture a spec')+ if (!captured) throw Object.assign(new Error('not found'), { code: 404 })+ if (!captured.spec) throw new Error('create did not capture a spec')return asApiserverNetworkPolicy(captured)})await (reconciler as any).ensureMcpHostIngressNetworkPolicy(host)+ expect(networkingApi.readNamespacedNetworkPolicy).toHaveBeenCalledOnce()+ expect(networkingApi.createNamespacedNetworkPolicy).toHaveBeenCalledOnce()+ networkingApi.readNamespacedNetworkPolicy.mockClear()+ networkingApi.createNamespacedNetworkPolicy.mockClear()+ networkingApi.createNamespacedNetworkPolicy.mockRejectedValue({ code: 409 })++ await (reconciler as any).ensureMcpHostIngressNetworkPolicy(host)+ expect(networkingApi.readNamespacedNetworkPolicy).toHaveBeenCalledOnce()+ expect(networkingApi.createNamespacedNetworkPolicy).not.toHaveBeenCalled()expect(networkingApi.replaceNamespacedNetworkPolicy).not.toHaveBeenCalled()networkingApi.replaceNamespacedNetworkPolicy.mockClear()@@ -67,6 +77,8 @@ describe('Host ensureMcpHostIngressNetworkPolicy inherits applyNetworkPolicy gatreturn asApiserverNetworkPolicy(captured, { port: 9090 })})await (reconciler as any).ensureMcpHostIngressNetworkPolicy(host)+ expect(networkingApi.readNamespacedNetworkPolicy).toHaveBeenCalledTimes(2)+ expect(networkingApi.createNamespacedNetworkPolicy).not.toHaveBeenCalled()expect(networkingApi.replaceNamespacedNetworkPolicy).toHaveBeenCalledOnce()})})diff --git a/host-context-controller/src/__tests__/hostReconciler.mounts.test.ts b/host-context-controller/src/__tests__/hostReconciler.mounts.test.tsindex 9735bd350..9c51b6d4b 100644--- a/host-context-controller/src/__tests__/hostReconciler.mounts.test.ts+++ b/host-context-controller/src/__tests__/hostReconciler.mounts.test.ts@@ -1304,7 +1304,8 @@ describe('HostReconciler.reconcile — uses resolveContextMounts', () => {it('creates host-scoped rpc-proxy ingress and egress policies', async () => {const networkingApi = {createNamespacedNetworkPolicy: vi.fn(async () => ({})),- readNamespacedNetworkPolicy: vi.fn(async () => ({})),+ // This initial-provisioning case starts with no NetworkPolicies.+ readNamespacedNetworkPolicy: vi.fn().mockRejectedValue({ code: 404 }),replaceNamespacedNetworkPolicy: vi.fn(async () => ({})),}const desktopHost: HostCRD = {diff --git a/host-context-controller/src/__tests__/llmHookNetworkPolicyNoopGate.test.ts b/host-context-controller/src/__tests__/llmHookNetworkPolicyNoopGate.test.tsindex 6ffe58615..a51dcc3e7 100644--- a/host-context-controller/src/__tests__/llmHookNetworkPolicyNoopGate.test.ts+++ b/host-context-controller/src/__tests__/llmHookNetworkPolicyNoopGate.test.ts@@ -106,6 +106,11 @@ describe('LlmHook NetworkPolicy no-op gate', () => {const log = vi.spyOn(console, 'log')try {await (reconciler as any).ensureNetworkPolicy(podKey, [hook], [])+ expect(networkingApi.readNamespacedNetworkPolicy).toHaveBeenCalledExactlyOnceWith({+ name: desired.metadata!.name,+ namespace: 'llm-hooks',+ })+ expect(networkingApi.createNamespacedNetworkPolicy).not.toHaveBeenCalled()expect(networkingApi.replaceNamespacedNetworkPolicy).not.toHaveBeenCalled()expect(updatedPolicyLogs(log, `NetworkPolicy "${desired.metadata?.name}"`)).toEqual([])} finally {@@ -194,4 +199,40 @@ describe('LlmHook NetworkPolicy no-op gate', () => {log.mockRestore()}})++ it('preserves live annotations when a present policy needs convergence without POST', async () => {+ const hook = makeImageHook()+ const podKey = computePodKey(hook)!+ const desired = (reconciler as any).buildNetworkPolicy(+ podKey,+ [hook],+ []+ ) as k8s.V1NetworkPolicy+ const existing = asApiserverNetworkPolicy(desired)+ existing.metadata = {+ ...existing.metadata,+ annotations: { 'example.com/operator': 'retained' },+ }+ existing.spec = { ...existing.spec!, ingress: [{}] }+ networkingApi.readNamespacedNetworkPolicy.mockResolvedValue(existing)+ await (reconciler as any).ensureNetworkPolicy(podKey, [hook], [])+ expect(networkingApi.readNamespacedNetworkPolicy).toHaveBeenCalledTimes(1)+ expect(networkingApi.createNamespacedNetworkPolicy).not.toHaveBeenCalled()+ expect(networkingApi.replaceNamespacedNetworkPolicy).toHaveBeenCalledTimes(1)+ const body = networkingApi.replaceNamespacedNetworkPolicy.mock.calls[0][0].body+ expect(body.metadata?.annotations?.['example.com/operator']).toBe('retained')+ expect(body.spec?.ingress).toEqual(desired.spec?.ingress)+ })++ it('propagates GET 403 without creating or replacing a policy', async () => {+ const error = { code: 403 }+ networkingApi.readNamespacedNetworkPolicy.mockRejectedValue(error)+ const hook = makeImageHook()+ await expect(+ (reconciler as any).ensureNetworkPolicy(computePodKey(hook)!, [hook], [])+ ).rejects.toBe(error)+ expect(networkingApi.readNamespacedNetworkPolicy).toHaveBeenCalledTimes(1)+ expect(networkingApi.createNamespacedNetworkPolicy).not.toHaveBeenCalled()+ expect(networkingApi.replaceNamespacedNetworkPolicy).not.toHaveBeenCalled()+ })})diff --git a/host-context-controller/src/__tests__/llmHooksEffectiveEgress.test.ts b/host-context-controller/src/__tests__/llmHooksEffectiveEgress.test.tsindex d007b146d..ef478fc3a 100644--- a/host-context-controller/src/__tests__/llmHooksEffectiveEgress.test.ts+++ b/host-context-controller/src/__tests__/llmHooksEffectiveEgress.test.ts@@ -145,6 +145,26 @@ describe('llm-hooks effective pod egress (N5)', () => {coreApi = createMockCoreApi()customApi = createMockCustomApi()networkingApi = createMockNetworkingApi()+ // Start with an empty API inventory and retain real desired objects, including+ // their ownership labels, so subsequent reads model present policies.+ const policies = new Map<string, k8s.V1NetworkPolicy>()+ networkingApi.readNamespacedNetworkPolicy.mockImplementation(async ({ name, namespace }) => {+ const policy = policies.get(`${namespace}/${name}`)+ if (!policy) throw { code: 404 }+ return structuredClone(policy)+ })+ networkingApi.createNamespacedNetworkPolicy.mockImplementation(async ({ namespace, body }) => {+ const key = `${namespace}/${body.metadata!.name}`+ if (policies.has(key)) throw { code: 409 }+ policies.set(key, structuredClone(body))+ return body+ })+ networkingApi.replaceNamespacedNetworkPolicy.mockImplementation(+ async ({ name, namespace, body }) => {+ policies.set(`${namespace}/${name}`, structuredClone(body))+ return body+ }+ )customApi.getNamespacedCustomObjectStatus.mockResolvedValue({metadata: { resourceVersion: '1' },status: { conditions: [] },diff --git a/host-context-controller/src/__tests__/sharedFileSystemReconciler.test.ts b/host-context-controller/src/__tests__/sharedFileSystemReconciler.test.tsindex c724dfd39..c3f093563 100644--- a/host-context-controller/src/__tests__/sharedFileSystemReconciler.test.ts+++ b/host-context-controller/src/__tests__/sharedFileSystemReconciler.test.ts@@ -2,6 +2,8 @@ import { beforeEach, describe, expect, it, vi } from 'vitest'import {type SharedFileSystemFactoryConfig,WFC_MOUNT_PATH,+ buildEgressNetworkPolicy,+ buildIngressNetworkPolicy,pvcName,wfcDeploymentName,wfcEgressPolicyName,@@ -50,7 +52,13 @@ function err(code: number, msg = 'k8s error'): Error & { code: number } {const notFound = () => err(404, 'not found')const alreadyExists = () => err(409, 'already exists')-function makeMocks() {+function makeMocks(systems: SharedFileSystemCRD[] = [makeSfs()]) {+ // Status/cleanup tests use already-converged policies with real identity and+ // ownership. Cold-start tests explicitly arrange absence for their own reads.+ const policies = systems.flatMap(sfs => [+ buildIngressNetworkPolicy(sfs, factoryConfig),+ buildEgressNetworkPolicy(sfs, factoryConfig),+ ])const appsApi = {createNamespacedDeployment: vi.fn().mockResolvedValue({}),// Truthful status (#592) reads readyReplicas; conflict-retry reads metadata.@@ -82,7 +90,11 @@ function makeMocks() {}const networkingApi = {createNamespacedNetworkPolicy: vi.fn().mockResolvedValue({}),- readNamespacedNetworkPolicy: vi.fn().mockResolvedValue({ metadata: { resourceVersion: '1' } }),+ readNamespacedNetworkPolicy: vi.fn(async ({ name }: { name: string }) => {+ const policy = policies.find(item => item.metadata?.name === name)+ if (!policy) throw new Error(`Missing NetworkPolicy fixture for ${name}`)+ return { ...policy, metadata: { ...policy.metadata, resourceVersion: '1' } }+ }),replaceNamespacedNetworkPolicy: vi.fn().mockResolvedValue({}),deleteNamespacedNetworkPolicy: vi.fn().mockResolvedValue({}),}@@ -167,6 +179,9 @@ describe('SharedFileSystemReconciler — happy path', () => {it('creates PVC, Service, Deployment, and both NetworkPolicies — and NEVER an init Job', async () => {const sfs = makeSfs()+ mocks.networkingApi.readNamespacedNetworkPolicy+ .mockRejectedValueOnce(notFound())+ .mockRejectedValueOnce(notFound())await reconciler.reconcile(sfs)expect(mocks.coreApi.createNamespacedPersistentVolumeClaim).toHaveBeenCalledTimes(1)@@ -292,6 +307,23 @@ describe('SharedFileSystemReconciler — legacy init-Job cleanup', () => {})describe('SharedFileSystemReconciler — idempotence', () => {+ it('leaves existing equivalent NetworkPolicies in place without POST or PUT', async () => {+ const mocks = makeMocks()+ await makeReconciler(mocks).reconcile(makeSfs())+ expect(mocks.networkingApi.readNamespacedNetworkPolicy).toHaveBeenCalledTimes(2)+ expect(mocks.networkingApi.createNamespacedNetworkPolicy).not.toHaveBeenCalled()+ expect(mocks.networkingApi.replaceNamespacedNetworkPolicy).not.toHaveBeenCalled()+ })++ it('propagates a denied NetworkPolicy read without attempting creation', async () => {+ const mocks = makeMocks()+ const denied = err(403, 'policy read denied')+ mocks.networkingApi.readNamespacedNetworkPolicy.mockRejectedValueOnce(denied)+ await expect(makeReconciler(mocks).reconcile(makeSfs())).rejects.toBe(denied)+ expect(mocks.networkingApi.createNamespacedNetworkPolicy).not.toHaveBeenCalled()+ expect(mocks.networkingApi.replaceNamespacedNetworkPolicy).not.toHaveBeenCalled()+ })+it('PVC 409 (already exists) is treated as success — no replace attempt', async () => {const mocks = makeMocks()mocks.coreApi.createNamespacedPersistentVolumeClaim.mockRejectedValueOnce(alreadyExists())@@ -475,17 +507,17 @@ describe('SharedFileSystemReconciler — reconcileDelete', () => {describe('SharedFileSystemReconciler — fullReconcile', () => {it('reconciles every desired SFS and continues past errors on individual SFSes', async () => {- const mocks = makeMocks()- mocks.coreApi.createNamespacedPersistentVolumeClaim- .mockResolvedValueOnce({})- .mockRejectedValueOnce(err(500, 'cluster down'))- .mockResolvedValueOnce({})- const reconciler = makeReconciler(mocks)const desired = [makeSfs({ name: 'sfs-a' }),makeSfs({ name: 'sfs-b' }),makeSfs({ name: 'sfs-c' }),]+ const mocks = makeMocks(desired)+ mocks.coreApi.createNamespacedPersistentVolumeClaim+ .mockResolvedValueOnce({})+ .mockRejectedValueOnce(err(500, 'cluster down'))+ .mockResolvedValueOnce({})+ const reconciler = makeReconciler(mocks)await expect(reconciler.fullReconcile(desired)).resolves.toBeUndefined()expect(mocks.coreApi.createNamespacedPersistentVolumeClaim).toHaveBeenCalledTimes(3)expect(reconciler.getStatus(desired[0]).phase).toBe('Ready')diff --git a/host-context-controller/src/bindingPolicyReconciler.test.ts b/host-context-controller/src/bindingPolicyReconciler.test.tsindex 48d6fe191..3ba292a64 100644--- a/host-context-controller/src/bindingPolicyReconciler.test.ts+++ b/host-context-controller/src/bindingPolicyReconciler.test.ts@@ -13,7 +13,27 @@ function deferred<T = void>() {function makeMockNetworkingApi() {return {createNamespacedNetworkPolicy: vi.fn().mockResolvedValue({}),- readNamespacedNetworkPolicy: vi.fn().mockResolvedValue({ metadata: { resourceVersion: '1' } }),+ readNamespacedNetworkPolicy: vi.fn(+ async ({+ name,+ namespace,+ }: {+ name: string+ namespace: string+ }): Promise<k8s.V1NetworkPolicy> => ({+ metadata: {+ name,+ namespace,+ uid: `uid-${name}`,+ resourceVersion: '1',+ labels: {+ 'clerum.io/managed-by': 'host-context-controller',+ 'clerum.io/policy-type': 'binding-allow',+ 'clerum.io/recipe': 'my-recipe',+ },+ },+ })+ ),replaceNamespacedNetworkPolicy: vi.fn().mockResolvedValue({}),listNamespacedNetworkPolicy: vi.fn().mockResolvedValue({ items: [] }),deleteNamespacedNetworkPolicy: vi.fn().mockResolvedValue({}),@@ -48,10 +68,15 @@ describe('BindingPolicyReconciler', () => {const singleBinding: BindingDef[] = [{ from: 'redis-mcp', to: 'redis', port: 6379 }]it('creates egress + ingress policies for a binding', async () => {+ // Both policies are absent only in this initial-creation case.+ mockApi.readNamespacedNetworkPolicy+ .mockRejectedValueOnce({ code: 404 })+ .mockRejectedValueOnce({ code: 404 })await reconciler.reconcileBindings('my-recipe', singleBinding, 'redis-mcp', 'redis-mcp', {isCurrent: () => true,})+ expect(mockApi.readNamespacedNetworkPolicy).toHaveBeenCalledTimes(2)// 2 calls: 1 egress + 1 ingressexpect(mockApi.createNamespacedNetworkPolicy).toHaveBeenCalledTimes(2)@@ -89,7 +114,10 @@ describe('BindingPolicyReconciler', () => {isCurrent: () => true,})- for (const call of mockApi.createNamespacedNetworkPolicy.mock.calls) {+ expect(mockApi.readNamespacedNetworkPolicy).toHaveBeenCalledTimes(2)+ expect(mockApi.replaceNamespacedNetworkPolicy).toHaveBeenCalledTimes(2)+ expect(mockApi.createNamespacedNetworkPolicy).not.toHaveBeenCalled()+ for (const call of mockApi.replaceNamespacedNetworkPolicy.mock.calls) {const labels = (call[0] as { body: { metadata: { labels: Record<string, string> } } }).body.metadata.labelsexpect(labels['clerum.io/policy-type']).toBe('binding-allow')@@ -98,20 +126,33 @@ describe('BindingPolicyReconciler', () => {}})- it('handles 409 conflict with create-or-replace pattern', async () => {+ it('rereads and converges when a policy appears between GET and POST', async () => {+ const readPolicy = mockApi.readNamespacedNetworkPolicy.getMockImplementation()!+ const observedNames = new Set<string>()+ mockApi.readNamespacedNetworkPolicy.mockImplementation(async request => {+ if (!observedNames.has(request.name)) {+ observedNames.add(request.name)+ throw { code: 404 }+ }+ return readPolicy(request)+ })mockApi.createNamespacedNetworkPolicy.mockRejectedValue({ code: 409 })await reconciler.reconcileBindings('my-recipe', singleBinding, 'redis-mcp', 'redis-mcp', {isCurrent: () => true,})- expect(mockApi.readNamespacedNetworkPolicy).toHaveBeenCalled()- expect(mockApi.replaceNamespacedNetworkPolicy).toHaveBeenCalled()+ expect(mockApi.readNamespacedNetworkPolicy).toHaveBeenCalledTimes(4)+ expect(mockApi.createNamespacedNetworkPolicy).toHaveBeenCalledTimes(2)+ expect(mockApi.replaceNamespacedNetworkPolicy).toHaveBeenCalledTimes(2)+ for (const [request] of mockApi.replaceNamespacedNetworkPolicy.mock.calls) {+ expect(request.body.metadata.resourceVersion).toBe('1')+ }})it('does not replace or start another apply after its authority lease is retired', async () => {const readStarted = deferred()- const releaseRead = deferred<{ metadata: { resourceVersion: string } }>()+ const releaseRead = deferred<{ metadata: { name: string; resourceVersion: string } }>()mockApi.createNamespacedNetworkPolicy.mockRejectedValue({ code: 409 })mockApi.readNamespacedNetworkPolicy.mockImplementationOnce(async () => {readStarted.resolve()@@ -129,12 +170,14 @@ describe('BindingPolicyReconciler', () => {await readStarted.promisecurrent = false- releaseRead.resolve({ metadata: { resourceVersion: '1' } })+ releaseRead.resolve({+ metadata: { name: 'bind-my-recipe-redis-mcp-redis-egress', resourceVersion: '1' },+ })await reconcile- // The first create was authorized and returned 409 without mutating. Once- // the lease is retired, no replacement or second policy apply may start.- expect(mockApi.createNamespacedNetworkPolicy).toHaveBeenCalledTimes(1)+ // Retirement during the initial GET prevents every mutation and the next apply.+ expect(mockApi.readNamespacedNetworkPolicy).toHaveBeenCalledTimes(1)+ expect(mockApi.createNamespacedNetworkPolicy).not.toHaveBeenCalled()expect(mockApi.replaceNamespacedNetworkPolicy).not.toHaveBeenCalled()})@@ -211,7 +254,7 @@ describe('BindingPolicyReconciler', () => {isCurrent: () => true,})- const calls = mockApi.createNamespacedNetworkPolicy.mock.calls+ const calls = mockApi.replaceNamespacedNetworkPolicy.mock.callsconst ingressCall = calls.find((c: unknown[]) =>(c[0] as { body: { spec: { policyTypes: string[] } } }).body.spec.policyTypes[0] ===@@ -230,7 +273,7 @@ describe('BindingPolicyReconciler', () => {isCurrent: () => true,})- const calls = mockApi.createNamespacedNetworkPolicy.mock.calls+ const calls = mockApi.replaceNamespacedNetworkPolicy.mock.callsconst egressCall = calls.find((c: unknown[]) =>(c[0] as { body: { spec: { policyTypes: string[] } } }).body.spec.policyTypes[0] ===@@ -302,7 +345,7 @@ describe('BindingPolicyReconciler', () => {{ isCurrent: () => true })- const calls = mockApi.createNamespacedNetworkPolicy.mock.calls+ const calls = mockApi.replaceNamespacedNetworkPolicy.mock.callsconst egressCall = calls.find((c: unknown[]) =>(c[0] as { body: { spec: { policyTypes: string[] } } }).body.spec.policyTypes[0] ===diff --git a/host-context-controller/src/createMetrics.test.ts b/host-context-controller/src/createMetrics.test.tsindex 1ac18eb60..ba8a5464a 100644--- a/host-context-controller/src/createMetrics.test.ts+++ b/host-context-controller/src/createMetrics.test.ts@@ -214,7 +214,7 @@ describe('Kubernetes create instrumentation', () => {expect(await count('Deployment', 'skipped')).toBe(0)})- it('observes the POST-first conflict path without replacing an unchanged policy', async () => {+ it('counts a presence skip without POST or replacing an unchanged policy', async () => {const policy: k8s.V1NetworkPolicy = {kind: 'NetworkPolicy',metadata: { name: 'metrics-policy' },@@ -231,13 +231,13 @@ describe('Kubernetes create instrumentation', () => {'test',policy)- expect(api.createNamespacedNetworkPolicy).toHaveBeenCalledTimes(1)+ expect(api.createNamespacedNetworkPolicy).not.toHaveBeenCalled()expect(api.readNamespacedNetworkPolicy).toHaveBeenCalledTimes(1)expect(api.replaceNamespacedNetworkPolicy).not.toHaveBeenCalled()expect(await count('NetworkPolicy', 'created')).toBe(0)expect(await count('NetworkPolicy', 'error')).toBe(0)- expect(await count('NetworkPolicy', 'conflict')).toBe(1)- expect(await count('NetworkPolicy', 'skipped')).toBe(0)+ expect(await count('NetworkPolicy', 'conflict')).toBe(0)+ expect(await count('NetworkPolicy', 'skipped')).toBe(1)})it('does not count a create when its mutation fence has already expired', async () => {diff --git a/host-context-controller/src/ensureResource.test.ts b/host-context-controller/src/ensureResource.test.tsnew file mode 100644index 000000000..5a6694267--- /dev/null+++ b/host-context-controller/src/ensureResource.test.ts@@ -0,0 +1,238 @@+import { describe, expect, it, vi } from 'vitest'+import { ensureResource, replaceWithConflictRetry } from './utils'++const resource = (rv = '1') => ({+ kind: 'NetworkPolicy',+ metadata: { name: 'observed-policy', resourceVersion: rv },+})++describe('ensureResource', () => {+ it.each([{ code: 404 }, { response: { statusCode: 404 } }])(+ 'creates only after an actual absence read (%j)',+ async absent => {+ const events: string[] = []+ const read = vi.fn(async () => {+ events.push('GET')+ throw absent+ })+ const create = vi.fn(async () => {+ events.push('POST')+ return resource()+ })+ const converge = vi.fn()+ const onSkipped = vi.fn()+ await ensureResource({ read, create, converge, onSkipped })+ expect(events).toEqual(['GET', 'POST'])+ expect(create).toHaveBeenCalledTimes(1)+ expect(converge).not.toHaveBeenCalled()+ expect(onSkipped).not.toHaveBeenCalled()+ }+ )++ it('consumes an observed object once, then reads fresh on a replace conflict', async () => {+ const first = resource('1')+ const second = resource('2')+ const read = vi.fn().mockResolvedValueOnce(first).mockResolvedValueOnce(second)+ const create = vi.fn()+ const replace = vi.fn().mockRejectedValueOnce({ code: 409 }).mockResolvedValue(undefined)+ const onSkipped = vi.fn()+ const validated: unknown[] = []+ await ensureResource({+ read,+ create,+ onSkipped,+ converge: observedRead =>+ replaceWithConflictRetry({+ description: 'policy',+ logPrefix: '[Test]',+ body: resource(),+ read: observedRead,+ replace,+ validateExisting: value => {+ validated.push(value)+ },+ }),+ })+ expect(read).toHaveBeenCalledTimes(2)+ expect(validated).toEqual([first, second])+ expect(replace.mock.calls.map(([body]) => body.metadata.resourceVersion)).toEqual(['1', '2'])+ expect(create).not.toHaveBeenCalled()+ expect(onSkipped).toHaveBeenCalledTimes(1)+ })++ it('uses a supplied snapshot without an API read or a create', async () => {+ const existing = resource()+ const read = vi.fn()+ const create = vi.fn()+ const onSkipped = vi.fn()+ const converge = vi.fn(async (readOnce, observed) => {+ expect(observed).toBe(existing)+ expect(await readOnce()).toBe(existing)+ })+ await ensureResource({ existing, read, create, converge, onSkipped })+ expect(converge).toHaveBeenCalledTimes(1)+ expect(read).not.toHaveBeenCalled()+ expect(create).not.toHaveBeenCalled()+ expect(onSkipped).toHaveBeenCalledTimes(1)+ })++ it('discards observed absence after POST409 and retains the retry budget', async () => {+ const read = vi.fn().mockResolvedValueOnce(resource('2')).mockResolvedValueOnce(resource('3'))+ const create = vi.fn().mockRejectedValue({ code: 409 })+ const lastConflict = { code: 409, message: 'conflict remains' }+ const replace = vi.fn().mockRejectedValue(lastConflict)+ const onSkipped = vi.fn()+ await expect(+ ensureResource({+ existing: null,+ read,+ create,+ onSkipped,+ converge: observedRead =>+ replaceWithConflictRetry({+ description: 'policy',+ logPrefix: '[Test]',+ body: resource(),+ read: observedRead,+ replace,+ maxAttempts: 2,+ }),+ })+ ).rejects.toBe(lastConflict)+ expect(create).toHaveBeenCalledTimes(1)+ expect(read).toHaveBeenCalledTimes(2)+ expect(replace.mock.calls.map(([body]) => body.metadata.resourceVersion)).toEqual(['2', '3'])+ expect(onSkipped).not.toHaveBeenCalled()+ })++ it.each([403, 500, undefined])('propagates initial read failures (%s)', async code => {+ const error = { code, message: 'read failed' }+ const read = vi.fn().mockRejectedValue(error)+ const create = vi.fn()+ const converge = vi.fn()+ const onSkipped = vi.fn()+ await expect(ensureResource({ read, create, converge, onSkipped })).rejects.toBe(error)+ expect(read).toHaveBeenCalledTimes(1)+ expect(create).not.toHaveBeenCalled()+ expect(converge).not.toHaveBeenCalled()+ expect(onSkipped).not.toHaveBeenCalled()+ })++ it.each([undefined, null, {}, { metadata: {} }, { metadata: { name: '' } }])(+ 'rejects a resolved invalid read without interpreting it as absence (%j)',+ async value => {+ const read = vi.fn().mockResolvedValue(value)+ const create = vi.fn()+ const onSkipped = vi.fn()+ await expect(ensureResource({ read, create, converge: vi.fn(), onSkipped })).rejects.toThrow(+ 'metadata.name'+ )+ expect(read).toHaveBeenCalledTimes(1)+ expect(create).not.toHaveBeenCalled()+ expect(onSkipped).not.toHaveBeenCalled()+ }+ )++ it('rechecks the fence after a GET404 and never counts cancellation as a skip', async () => {+ let current = true+ const read = vi.fn(async () => {+ current = false+ throw { code: 404 }+ })+ const create = vi.fn()+ const onSkipped = vi.fn()+ await ensureResource({+ read,+ create,+ converge: vi.fn(),+ onSkipped,+ mutationAllowed: () => current,+ })+ expect(read).toHaveBeenCalledTimes(1)+ expect(create).not.toHaveBeenCalled()+ expect(onSkipped).not.toHaveBeenCalled()+ })++ it('retains the initial fence, with a live allowed invocation as control', async () => {+ const read = vi.fn().mockResolvedValue(resource())+ const create = vi.fn()+ const converge = vi.fn().mockResolvedValue(undefined)+ const onSkipped = vi.fn()+ await ensureResource({ read, create, converge, onSkipped, mutationAllowed: () => false })+ expect(read).not.toHaveBeenCalled()+ await ensureResource({ read, create, converge, onSkipped, mutationAllowed: () => true })+ expect(read).toHaveBeenCalledTimes(1)+ expect(converge).toHaveBeenCalledTimes(1)+ expect(onSkipped).toHaveBeenCalledTimes(1)+ })++ it('keeps missingIsError scoped to disappearance during convergence', async () => {+ const read = vi.fn().mockRejectedValue({ code: 404 })+ const create = vi.fn().mockRejectedValue({ code: 409 })+ const replace = vi.fn()+ const onSkipped = vi.fn()+ const converge =+ (missingIsError: boolean) => (readFresh: () => Promise<ReturnType<typeof resource>>) =>+ replaceWithConflictRetry({+ description: 'policy',+ logPrefix: '[Test]',+ body: resource(),+ read: readFresh,+ replace,+ missingIsError,+ })+ await ensureResource({ read, create, converge: converge(false), onSkipped })+ await expect(+ ensureResource({ read, create, converge: converge(true), onSkipped })+ ).rejects.toEqual({ code: 404 })+ expect(create).toHaveBeenCalledTimes(2)+ expect(read).toHaveBeenCalledTimes(4)+ expect(replace).not.toHaveBeenCalled()+ expect(onSkipped).not.toHaveBeenCalled()+ })++ it('does not count a rejected ownership check or retained failure as skipped', async () => {+ const read = vi.fn().mockResolvedValue(resource())+ const create = vi.fn()+ const onSkipped = vi.fn()+ const error = new Error('foreign ownership')+ const converge = vi.fn().mockRejectedValueOnce(error).mockResolvedValueOnce(false)+ await expect(ensureResource({ read, create, converge, onSkipped })).rejects.toBe(error)+ await ensureResource({ read, create, converge, onSkipped })+ expect(converge).toHaveBeenCalledTimes(2)+ expect(read).toHaveBeenCalledTimes(2)+ expect(create).not.toHaveBeenCalled()+ expect(onSkipped).not.toHaveBeenCalled()+ })++ it('does not count convergence cancelled during async body reconstruction', async () => {+ let current = true+ const read = vi.fn().mockResolvedValue(resource())+ const replace = vi.fn()+ const create = vi.fn()+ const onSkipped = vi.fn()+ await ensureResource({+ read,+ create,+ onSkipped,+ mutationAllowed: () => current,+ converge: readOnce =>+ replaceWithConflictRetry({+ description: 'policy',+ logPrefix: '[Test]',+ body: resource(),+ read: readOnce,+ replace,+ resolveBody: async () => {+ current = false+ return resource()+ },+ mutationAllowed: () => current,+ }),+ })+ expect(read).toHaveBeenCalledTimes(1)+ expect(replace).not.toHaveBeenCalled()+ expect(create).not.toHaveBeenCalled()+ expect(onSkipped).not.toHaveBeenCalled()+ })+})diff --git a/host-context-controller/src/existenceReadMetrics.test.ts b/host-context-controller/src/existenceReadMetrics.test.tsindex 808a99748..409fdeb04 100644--- a/host-context-controller/src/existenceReadMetrics.test.ts+++ b/host-context-controller/src/existenceReadMetrics.test.ts@@ -357,10 +357,10 @@ describe('Kubernetes existence-read instrumentation', () => {spec: { podSelector: {}, policyTypes: ['Ingress'], ingress: [] },}- it('preserves POST-first success without adding an existence request', async () => {+ it('observes GET404 before POST for an absent policy', async () => {const api = {createNamespacedNetworkPolicy: vi.fn().mockResolvedValue(policy),- readNamespacedNetworkPolicy: vi.fn(),+ readNamespacedNetworkPolicy: vi.fn().mockRejectedValue({ code: 404 }),}await applyNetworkPolicy(api as unknown as k8s.NetworkingV1Api,@@ -372,7 +372,11 @@ describe('Kubernetes existence-read instrumentation', () => {namespace: 'test',body: policy,})- expect(api.readNamespacedNetworkPolicy).not.toHaveBeenCalled()+ expect(api.readNamespacedNetworkPolicy).toHaveBeenCalledExactlyOnceWith({+ name: 'read-metrics-policy',+ namespace: 'test',+ })+ expect(await count('NetworkPolicy', 'absent')).toBe(1)expect(await count('NetworkPolicy', 'found')).toBe(0)})@@ -391,7 +395,7 @@ describe('Kubernetes existence-read instrumentation', () => {policy)).rejects.toBe(error)- expect(api.createNamespacedNetworkPolicy).toHaveBeenCalledTimes(1)+ expect(api.createNamespacedNetworkPolicy).not.toHaveBeenCalled()expect(api.readNamespacedNetworkPolicy).toHaveBeenCalledExactlyOnceWith({name: 'read-metrics-policy',namespace: 'test',@@ -403,7 +407,7 @@ describe('Kubernetes existence-read instrumentation', () => {)expect(samples.map(sample => [sample.labels.outcome, sample.value])).toEqual([['created', 0],- ['conflict', 1],+ ['conflict', 0],['error', 0],['skipped', 0],])@@ -437,7 +441,7 @@ describe('Kubernetes existence-read instrumentation', () => {'test',policy)- expect(events).toEqual(['POST', 'GET', 'PUT:1', 'GET', 'PUT:2'])+ expect(events).toEqual(['GET', 'PUT:1', 'GET', 'PUT:2'])expect(api.readNamespacedNetworkPolicy).toHaveBeenCalledTimes(2)expect(api.replaceNamespacedNetworkPolicy).toHaveBeenCalledTimes(2)expect(await count('NetworkPolicy', 'found')).toBe(2)@@ -445,7 +449,12 @@ describe('Kubernetes existence-read instrumentation', () => {const conflicts = (await createsTotal.get()).values.find(sample => sample.labels.kind === 'NetworkPolicy' && sample.labels.outcome === 'conflict')- expect(conflicts?.value).toBe(1)+ expect(conflicts?.value).toBe(0)+ expect(api.createNamespacedNetworkPolicy).not.toHaveBeenCalled()+ const skipped = (await createsTotal.get()).values.find(+ sample => sample.labels.kind === 'NetworkPolicy' && sample.labels.outcome === 'skipped'+ )+ expect(skipped?.value).toBe(1)})it('observes a read that expires the existing fence and prevents replace', async () => {@@ -468,7 +477,7 @@ describe('Kubernetes existence-read instrumentation', () => {allowed)expect(allowed.mock.results.map(result => result.value)).toEqual([true, false])- expect(api.createNamespacedNetworkPolicy).toHaveBeenCalledTimes(1)+ expect(api.createNamespacedNetworkPolicy).not.toHaveBeenCalled()expect(api.readNamespacedNetworkPolicy).toHaveBeenCalledTimes(1)expect(await count('NetworkPolicy', 'found')).toBe(1)expect(api.replaceNamespacedNetworkPolicy).not.toHaveBeenCalled()diff --git a/host-context-controller/src/hostReconciler.codexScopeProvenance.test.ts b/host-context-controller/src/hostReconciler.codexScopeProvenance.test.tsindex fde11b049..cde5f5035 100644--- a/host-context-controller/src/hostReconciler.codexScopeProvenance.test.ts+++ b/host-context-controller/src/hostReconciler.codexScopeProvenance.test.ts@@ -188,7 +188,7 @@ function scopeHashFromSecretWrites(coreApi: ReturnType<typeof createMockCoreApi>}function proxyPolicyBodies(networkingApi: ReturnType<typeof createMockNetworkingApi>) {- return networkingApi.createNamespacedNetworkPolicy.mock.calls+ return networkingApi.replaceNamespacedNetworkPolicy.mock.calls.map(call => call[0]?.body).filter(body => body?.metadata?.name === CODEX_PROXY_POLICY)}diff --git a/host-context-controller/src/llmHookReconciler.test.ts b/host-context-controller/src/llmHookReconciler.test.tsindex 09fd93068..7f9c673ee 100644--- a/host-context-controller/src/llmHookReconciler.test.ts+++ b/host-context-controller/src/llmHookReconciler.test.ts@@ -494,6 +494,8 @@ describe('LlmHookReconciler', () => {// ─── NetworkPolicy ingress reverse-index (§10) ─────────────────────it('NetworkPolicy ingress admits exactly the referencing Hosts', async () => {+ // This scenario starts without a policy; the API reports absence explicitly.+ networkingApi.readNamespacedNetworkPolicy.mockRejectedValue({ code: 404 })const a = makeHook({name: 'a',spec: { target: { image: { ref: IMG, port: 8080 } }, lifecyclePoints: ['preCall'] },@@ -517,6 +519,8 @@ describe('LlmHookReconciler', () => {})it('co-located members: NetworkPolicy label values are comma-free (member list is an annotation)', async () => {+ // This scenario starts without a policy; the API reports absence explicitly.+ networkingApi.readNamespacedNetworkPolicy.mockRejectedValue({ code: 404 })// Two hooks with the SAME image target share one pod key → one shared NP.const a = makeHook({name: 'a',@@ -551,6 +555,8 @@ describe('LlmHookReconciler', () => {})it('re-reconciles NetworkPolicy ingress on a Host reference change (fan-out)', async () => {+ // This scenario starts without a policy; the API reports absence explicitly.+ networkingApi.readNamespacedNetworkPolicy.mockRejectedValue({ code: 404 })const a = makeHook({name: 'a',spec: { target: { image: { ref: IMG, port: 8080 } }, lifecyclePoints: ['preCall'] },@@ -594,6 +600,8 @@ describe('LlmHookReconciler', () => {}it('creates an ingress NP selecting the Service pods and admitting only referencing hosts', async () => {+ // This scenario starts without a policy; the API reports absence explicitly.+ networkingApi.readNamespacedNetworkPolicy.mockRejectedValue({ code: 404 })withSelector({ app: 'ref-hook' })const s = makeServiceHook('s1')hooks.set('s1', s)@@ -619,6 +627,8 @@ describe('LlmHookReconciler', () => {})it('creates a deny-all (empty ingress) NP when no Host references the hook', async () => {+ // This scenario starts without a policy; the API reports absence explicitly.+ networkingApi.readNamespacedNetworkPolicy.mockRejectedValue({ code: 404 })withSelector({ app: 'ref-hook' })const s = makeServiceHook('s1')hooks.set('s1', s)@@ -646,6 +656,8 @@ describe('LlmHookReconciler', () => {})it('refreshes the admitted hosts on a Host reference change (fan-out)', async () => {+ // This scenario starts without a policy; the API reports absence explicitly.+ networkingApi.readNamespacedNetworkPolicy.mockRejectedValue({ code: 404 })withSelector({ app: 'ref-hook' })const s = makeServiceHook('s1')hooks.set('s1', s)@@ -675,6 +687,8 @@ describe('LlmHookReconciler', () => {describe('per-host egress (reconcileHostEgress)', () => {it('allows egress only to the referenced image hook pod-key + port', async () => {+ // This scenario starts without a policy; the API reports absence explicitly.+ networkingApi.readNamespacedNetworkPolicy.mockRejectedValue({ code: 404 })const hook = makeHook({ name: 'img1' }) // default image target, port 8080hooks.set('img1', hook)const host = makeHostRef('host-1', ['img1'])@@ -705,6 +719,8 @@ describe('LlmHookReconciler', () => {})it('uses the Service selector + port for a service-target hook', async () => {+ // This scenario starts without a policy; the API reports absence explicitly.+ networkingApi.readNamespacedNetworkPolicy.mockRejectedValue({ code: 404 })coreApi.readNamespacedService.mockResolvedValue({metadata: {},spec: { selector: { app: 'svc-hook' } },@@ -782,6 +798,8 @@ describe('LlmHookReconciler', () => {})it('grants scoped CoreDNS egress only when a hook declares egressBindings (N5)', async () => {+ // This scenario starts without a policy; the API reports absence explicitly.+ networkingApi.readNamespacedNetworkPolicy.mockRejectedValue({ code: 404 })const withEgress = makeHook({name: 'out',spec: {@@ -817,6 +835,8 @@ describe('LlmHookReconciler', () => {// "grants NO egress (not even DNS)" is what let namespace-wide DNS through// while the suite stayed green.it('declares no Egress in the per-pod-key policy for a pure responder (no egressBindings)', async () => {+ // This scenario starts without a policy; the API reports absence explicitly.+ networkingApi.readNamespacedNetworkPolicy.mockRejectedValue({ code: 404 })const responder = makeHook({ name: 'resp' }) // default image, no egressBindingshooks.set('resp', responder)await reconciler.reconcile(responder)@@ -829,6 +849,8 @@ describe('LlmHookReconciler', () => {})it('allows a valid public-CIDR egress binding as an Egress NetworkPolicy rule', async () => {+ // This scenario starts without a policy; the API reports absence explicitly.+ networkingApi.readNamespacedNetworkPolicy.mockRejectedValue({ code: 404 })const a = makeHook({name: 'a',spec: {diff --git a/host-context-controller/src/llmHookReconciler.ts b/host-context-controller/src/llmHookReconciler.tsindex e70e28e07..54e4ceb78 100644--- a/host-context-controller/src/llmHookReconciler.ts+++ b/host-context-controller/src/llmHookReconciler.ts@@ -35,10 +35,13 @@ import {MANAGED_BY_VALUE,POLICY_TYPE_LABEL,} from './constants'+import { hccLogger } from './logger'+import { createsTotal } from './metrics'import { isAllowedExternalEgressCidr, isPublicDnsHostname } from './networkPolicyReconciler'import { HostCRD, LlmHookCRD, LlmHookCondition, LlmHookImageTarget, LlmHookStatus } from './types'import {deploymentMatchesDesired,+ ensureResource,getErrorCode,networkPolicyMatchesDesired,observeCreate,@@ -825,30 +828,33 @@ export class LlmHookReconciler {await this.applyNetworkPolicy(this.buildNetworkPolicy(podKey, members, egressRules))}- /** Idempotent create-then-409-replace of a NetworkPolicy in its own namespace. */+ /** Read first, preserving the existing NetworkPolicy convergence policy. */private async applyNetworkPolicy(policy: k8s.V1NetworkPolicy): Promise<void> {const name = policy.metadata!.name!const namespace = policy.metadata!.namespace ?? config.llmHooksNamespace- try {- await observeCreate('NetworkPolicy', () =>- this.networkingApi.createNamespacedNetworkPolicy({ namespace, body: policy })- )- console.log(`${LOG} Created NetworkPolicy "${name}" (${namespace})`)- return- } catch (error) {- if (getErrorCode(error) !== 409) throw error- }- await replaceWithConflictRetry({- description: `NetworkPolicy "${name}"`,- logPrefix: LOG,- body: policy,- mergeExisting: preserveObjectAnnotations,- isUpToDate: networkPolicyMatchesDesired,+ await ensureResource({read: () =>observeExistenceRead('NetworkPolicy', () =>this.networkingApi.readNamespacedNetworkPolicy({ name, namespace })),- replace: body => this.networkingApi.replaceNamespacedNetworkPolicy({ name, namespace, body }),+ create: async () => {+ await observeCreate('NetworkPolicy', () =>+ this.networkingApi.createNamespacedNetworkPolicy({ namespace, body: policy })+ )+ hccLogger.info('NetworkPolicy created', { scope: LOG, policy: name, namespace })+ },+ converge: read =>+ replaceWithConflictRetry({+ description: `NetworkPolicy "${name}"`,+ logPrefix: LOG,+ body: policy,+ mergeExisting: preserveObjectAnnotations,+ isUpToDate: networkPolicyMatchesDesired,+ read,+ replace: body =>+ this.networkingApi.replaceNamespacedNetworkPolicy({ name, namespace, body }),+ }),+ onSkipped: () => createsTotal.inc({ kind: 'NetworkPolicy', outcome: 'skipped' }),})}diff --git a/host-context-controller/src/networkPolicyReconciler.egressObservation.test.ts b/host-context-controller/src/networkPolicyReconciler.egressObservation.test.tsindex dc1305840..9af1bef22 100644--- a/host-context-controller/src/networkPolicyReconciler.egressObservation.test.ts+++ b/host-context-controller/src/networkPolicyReconciler.egressObservation.test.ts@@ -316,7 +316,7 @@ describe('external egress observation failures do not decide authorization', ()expect(f.status()).toEqual(statusBefore)})- it('does not copy a newer resourceVersion onto a policy calculated before an apply conflict', async () => {+ it('does not copy a newer resourceVersion onto a policy calculated before a replace conflict', async () => {const f = fixture([currentDns])await f.run()const policy = f.policies.get(currentName)!@@ -325,8 +325,8 @@ describe('external egress observation failures do not decide authorization', ()ports: [{ port: 443, protocol: 'TCP' }],})f.clearCalls()- f.api.createNamespacedNetworkPolicy.mockImplementationOnce(async () => {- // The conflict retry reads this newer object after the final plan check.+ f.api.replaceNamespacedNetworkPolicy.mockImplementationOnce(async () => {+ // The first PUT races with a newer resourceVersion after the final plan check.// A matching owner alone cannot authorize reuse of the older DNS state.policy.metadata!.resourceVersion = 'changed-before-replace'throw apiError(409)@@ -334,20 +334,21 @@ describe('external egress observation failures do not decide authorization', ()await expect(f.run()).rejects.toThrow(/changed after the external-egress observation/)- expect(f.api.replaceNamespacedNetworkPolicy).not.toHaveBeenCalled()+ expect(f.api.replaceNamespacedNetworkPolicy).toHaveBeenCalledTimes(1)+ expect(f.api.createNamespacedNetworkPolicy).not.toHaveBeenCalled()expect(f.condition()?.status).toBe('False')await f.run()expect(f.condition()?.status).toBe('True')expect(f.policies.get(currentName)?.spec?.egress).toHaveLength(1)})- it('does not certify a policy deleted after create conflict and before replacement read, then recovers', async () => {+ it('does not certify a policy deleted during PUT409 and before the retry read, then recovers', async () => {const f = fixture([currentDns])await f.run()vi.mocked(dns.resolve4).mockResolvedValue([{ address: '8.8.8.8', ttl: 300 }] as never)- f.api.createNamespacedNetworkPolicy.mockImplementationOnce(async () => {- // POST sees the existing policy and conflicts. A concurrent deletion then- // wins before the helper's replacement GET, which observes real absence.+ f.api.replaceNamespacedNetworkPolicy.mockImplementationOnce(async () => {+ // A concurrent deletion wins during the first PUT. The retry GET must+ // observe absence and fail admission, rather than report success.f.policies.delete(currentName)throw apiError(409)})diff --git a/host-context-controller/src/networkPolicyReconciler.test.ts b/host-context-controller/src/networkPolicyReconciler.test.tsindex 50c58d777..3dc6a0ea8 100644--- a/host-context-controller/src/networkPolicyReconciler.test.ts+++ b/host-context-controller/src/networkPolicyReconciler.test.ts@@ -478,6 +478,8 @@ describe('NetworkPolicyReconciler', () => {describe('L0 — ensureDefaultDeny', () => {it('creates deny-all policies in every runtime namespace', async () => {+ // Bootstrap this scenario without policies; later LIST fixtures supply existing objects.+ mockApi.readNamespacedNetworkPolicy.mockRejectedValue({ code: 404 })await reconciler.ensureDefaultPolicies()const denyPolicies = mockApi.createNamespacedNetworkPolicy.mock.calls.filter(@@ -490,6 +492,8 @@ describe('NetworkPolicyReconciler', () => {})it('includes both Ingress and Egress in policyTypes', async () => {+ // Bootstrap this scenario without policies; later LIST fixtures supply existing objects.+ mockApi.readNamespacedNetworkPolicy.mockRejectedValue({ code: 404 })await reconciler.ensureDefaultPolicies()const denyPolicy = mockApi.createNamespacedNetworkPolicy.mock.calls.find(@@ -504,6 +508,8 @@ describe('NetworkPolicyReconciler', () => {})it('uses empty podSelector (selects all pods)', async () => {+ // Bootstrap this scenario without policies; later LIST fixtures supply existing objects.+ mockApi.readNamespacedNetworkPolicy.mockRejectedValue({ code: 404 })await reconciler.ensureDefaultPolicies()const denyPolicy = mockApi.createNamespacedNetworkPolicy.mock.calls.find(@@ -528,21 +534,8 @@ describe('NetworkPolicyReconciler', () => {])('refuses to adopt a foreign policy that collides with %s',async (targetName, policyType) => {- mockApi.createNamespacedNetworkPolicy.mockImplementation(- async ({ body }: { body: k8s.V1NetworkPolicy }) => {- if (body.metadata?.name === targetName) {- throw Object.assign(new Error('already exists'), { code: 409 })- }- return {}- }- )mockApi.readNamespacedNetworkPolicy.mockImplementation(async ({ name, namespace }) => {- if (- name === 'allow-desktop-egress-rpc-proxy' ||- name === 'allow-rpc-proxy-to-managed-mcp-servers'- ) {- throw Object.assign(new Error('not found'), { code: 404 })- }+ if (name !== targetName) throw { code: 404 }return {metadata: {name,@@ -558,6 +551,9 @@ describe('NetworkPolicyReconciler', () => {})await expect(reconciler.ensureDefaultPolicies()).rejects.toThrow(/ownership/i)+ expect(mockApi.readNamespacedNetworkPolicy).toHaveBeenCalledWith(+ expect.objectContaining({ name: targetName })+ )expect(mockApi.replaceNamespacedNetworkPolicy).not.toHaveBeenCalledWith(expect.objectContaining({ name: targetName }))@@ -565,6 +561,8 @@ describe('NetworkPolicyReconciler', () => {)it('creates DNS egress policies in every runtime namespace', async () => {+ // Bootstrap this scenario without policies; later LIST fixtures supply existing objects.+ mockApi.readNamespacedNetworkPolicy.mockRejectedValue({ code: 404 })await reconciler.ensureDefaultPolicies()const dnsPolicies = mockApi.createNamespacedNetworkPolicy.mock.calls.filter(@@ -577,6 +575,8 @@ describe('NetworkPolicyReconciler', () => {})it('DNS egress targets kube-system on port 53 UDP+TCP', async () => {+ // Bootstrap this scenario without policies; later LIST fixtures supply existing objects.+ mockApi.readNamespacedNetworkPolicy.mockRejectedValue({ code: 404 })await reconciler.ensureDefaultPolicies()const dnsPolicy = mockApi.createNamespacedNetworkPolicy.mock.calls.find((call: unknown[]) => {@@ -595,6 +595,8 @@ describe('NetworkPolicyReconciler', () => {})it('creates HCC API egress policies', async () => {+ // Bootstrap this scenario without policies; later LIST fixtures supply existing objects.+ mockApi.readNamespacedNetworkPolicy.mockRejectedValue({ code: 404 })await reconciler.ensureDefaultPolicies()const hccPolicies = mockApi.createNamespacedNetworkPolicy.mock.calls.filter(@@ -631,6 +633,8 @@ describe('NetworkPolicyReconciler', () => {})it('creates K8s API egress policies', async () => {+ // Bootstrap this scenario without policies; later LIST fixtures supply existing objects.+ mockApi.readNamespacedNetworkPolicy.mockRejectedValue({ code: 404 })await reconciler.ensureDefaultPolicies()const k8sPolicies = mockApi.createNamespacedNetworkPolicy.mock.calls.filter(@@ -643,6 +647,8 @@ describe('NetworkPolicyReconciler', () => {})it('scopes HCC API egress to the platform pods that need it', async () => {+ // Bootstrap this scenario without policies; later LIST fixtures supply existing objects.+ mockApi.readNamespacedNetworkPolicy.mockRejectedValue({ code: 404 })await reconciler.ensureDefaultPolicies()const policyByName = new Map(@@ -666,6 +672,8 @@ describe('NetworkPolicyReconciler', () => {})it('routes HCC API egress policies to the control-plane gateway', async () => {+ // Bootstrap this scenario without policies; later LIST fixtures supply existing objects.+ mockApi.readNamespacedNetworkPolicy.mockRejectedValue({ code: 404 })await reconciler.ensureDefaultPolicies()const hccPolicies = mockApi.createNamespacedNetworkPolicy.mock.calls@@ -686,6 +694,8 @@ describe('NetworkPolicyReconciler', () => {})it('does not grant Kubernetes API egress to runtime workload namespaces by default', async () => {+ // Bootstrap this scenario without policies; later LIST fixtures supply existing objects.+ mockApi.readNamespacedNetworkPolicy.mockRejectedValue({ code: 404 })await reconciler.ensureDefaultPolicies()const policyByName = new Map(@@ -709,6 +719,8 @@ describe('NetworkPolicyReconciler', () => {})it('infrastructure policies have correct policy-type label', async () => {+ // Bootstrap this scenario without policies; later LIST fixtures supply existing objects.+ mockApi.readNamespacedNetworkPolicy.mockRejectedValue({ code: 404 })await reconciler.ensureDefaultPolicies()const infraPolicies = mockApi.createNamespacedNetworkPolicy.mock.calls.filter(@@ -722,17 +734,25 @@ describe('NetworkPolicyReconciler', () => {})it('removes legacy static policies now replaced by generated scoped policies', async () => {- mockApi.readNamespacedNetworkPolicy.mockImplementation(async ({ name, namespace }) => ({- metadata: {- name,- namespace,- uid: `${name}-uid`,- resourceVersion: '7',- labels: {- 'clerum.io/managed-by': 'host-context-controller',+ mockApi.readNamespacedNetworkPolicy.mockImplementation(async ({ name, namespace }) => {+ if (+ !['allow-desktop-egress-rpc-proxy', 'allow-rpc-proxy-to-managed-mcp-servers'].includes(+ name+ )+ )+ throw { code: 404 }+ return {+ metadata: {+ name,+ namespace,+ uid: `${name}-uid`,+ resourceVersion: '7',+ labels: {+ 'clerum.io/managed-by': 'host-context-controller',+ },},- },- }))+ }+ })await reconciler.ensureDefaultPolicies()@@ -830,18 +850,21 @@ describe('NetworkPolicyReconciler', () => {return {}})- mockApi.readNamespacedNetworkPolicy.mockImplementation(async ({ name, namespace }) => ({- metadata: {- name,- namespace,- uid: 'existing-default-policy-uid',- resourceVersion: '7',- labels: {- 'clerum.io/managed-by': 'host-context-controller',- 'clerum.io/policy-type': 'allow-api',+ mockApi.readNamespacedNetworkPolicy.mockImplementation(async ({ name, namespace }) => {+ if (name !== 'allow-host-context-controller-api') throw { code: 404 }+ return {+ metadata: {+ name,+ namespace,+ uid: 'existing-default-policy-uid',+ resourceVersion: '7',+ labels: {+ 'clerum.io/managed-by': 'host-context-controller',+ 'clerum.io/policy-type': 'allow-api',+ },},- },- }))+ }+ })await expect(reconciler.ensureDefaultPolicies()).resolves.toBeUndefined()expect(mockApi.replaceNamespacedNetworkPolicy).toHaveBeenCalledWith(@@ -861,17 +884,20 @@ describe('NetworkPolicyReconciler', () => {return {}})- mockApi.readNamespacedNetworkPolicy.mockResolvedValue({- metadata: {- name: 'allow-host-context-controller-api',- namespace: 'mcp-server',- uid: 'foreign-default-policy-uid',- resourceVersion: '7',- labels: {- 'clerum.io/managed-by': 'foreign-controller',- 'clerum.io/policy-type': 'allow-api',+ mockApi.readNamespacedNetworkPolicy.mockImplementation(async ({ name }) => {+ if (name !== 'allow-host-context-controller-api') throw { code: 404 }+ return {+ metadata: {+ name: 'allow-host-context-controller-api',+ namespace: 'mcp-server',+ uid: 'foreign-default-policy-uid',+ resourceVersion: '7',+ labels: {+ 'clerum.io/managed-by': 'foreign-controller',+ 'clerum.io/policy-type': 'allow-api',+ },},- },+ }})await expect(reconciler.ensureDefaultPolicies()).rejects.toThrow(/ownership/i)@@ -887,7 +913,8 @@ describe('NetworkPolicyReconciler', () => {return {}})- mockApi.readNamespacedNetworkPolicy+ const readAllowApi = vi+ .fn().mockResolvedValueOnce({metadata: {name: 'allow-host-context-controller-api',@@ -912,6 +939,10 @@ describe('NetworkPolicyReconciler', () => {},},})+ mockApi.readNamespacedNetworkPolicy.mockImplementation(async ({ name }) => {+ if (name !== 'allow-host-context-controller-api') throw { code: 404 }+ return readAllowApi()+ })mockApi.replaceNamespacedNetworkPolicy.mockRejectedValueOnce(Object.assign(new Error('conflict'), { code: 409 }))@@ -921,6 +952,8 @@ describe('NetworkPolicyReconciler', () => {})it('creates allow-api ingress policy using _from (K8s client convention)', async () => {+ // Bootstrap this scenario without policies; later LIST fixtures supply existing objects.+ mockApi.readNamespacedNetworkPolicy.mockRejectedValue({ code: 404 })await reconciler.ensureDefaultPolicies()const apiPolicy = mockApi.createNamespacedNetworkPolicy.mock.calls.find((call: unknown[]) => {@@ -1020,6 +1053,8 @@ describe('NetworkPolicyReconciler', () => {})it('creates context-allow ingress policy per server in mcp-server namespace', async () => {+ // Bootstrap this scenario without policies; later LIST fixtures supply existing objects.+ mockApi.readNamespacedNetworkPolicy.mockRejectedValue({ code: 404 })const cache = new Map<string, McpServerCRD>()cache.set('mongo', {name: 'mongo',@@ -1081,6 +1116,8 @@ describe('NetworkPolicyReconciler', () => {})it('retains the ingress policy it just replaced when the authoritative server port changes', async () => {+ // Bootstrap this scenario without policies; later LIST fixtures supply existing objects.+ mockApi.readNamespacedNetworkPolicy.mockRejectedValue({ code: 404 })const oldServer: McpServerCRD = {name: 'mongo',namespace: 'mcp-server',@@ -1156,6 +1193,8 @@ describe('NetworkPolicyReconciler', () => {})it('stops remaining policy mutations after its combined authority lease is retired', async () => {+ // Bootstrap this scenario without policies; later LIST fixtures supply existing objects.+ mockApi.readNamespacedNetworkPolicy.mockRejectedValue({ code: 404 })const cache = new Map<string, McpServerCRD>()cache.set('mongo', {name: 'mongo',@@ -1202,6 +1241,8 @@ describe('NetworkPolicyReconciler', () => {})it('does not mutate a Context policy from a mixed McpServer cache revision', async () => {+ // Bootstrap this scenario without policies; later LIST fixtures supply existing objects.+ mockApi.readNamespacedNetworkPolicy.mockRejectedValue({ code: 404 })const cache = new Map<string, McpServerCRD>()cache.set('mongo', {name: 'mongo',@@ -1250,6 +1291,8 @@ describe('NetworkPolicyReconciler', () => {})it('continues a Context policy pass across a status-only McpServer replacement', async () => {+ // Bootstrap this scenario without policies; later LIST fixtures supply existing objects.+ mockApi.readNamespacedNetworkPolicy.mockRejectedValue({ code: 404 })const cache = new Map<string, McpServerCRD>()const selected: McpServerCRD = {name: 'mongo',@@ -1327,6 +1370,8 @@ describe('NetworkPolicyReconciler', () => {})it('creates L2 egress counterpart in mcp-host namespace (bidirectional)', async () => {+ // Bootstrap this scenario without policies; later LIST fixtures supply existing objects.+ mockApi.readNamespacedNetworkPolicy.mockRejectedValue({ code: 404 })const cache = new Map<string, McpServerCRD>()cache.set('mongo', {name: 'mongo',@@ -1384,6 +1429,8 @@ describe('NetworkPolicyReconciler', () => {})it('scopes L2 mcp-host egress by Context CRD name, not namespace-wide selector', async () => {+ // Bootstrap this scenario without policies; later LIST fixtures supply existing objects.+ mockApi.readNamespacedNetworkPolicy.mockRejectedValue({ code: 404 })const cache = new Map<string, McpServerCRD>()cache.set('mongo', {name: 'mongo',@@ -1418,6 +1465,8 @@ describe('NetworkPolicyReconciler', () => {})it('creates L2 rpc-proxy egress to one MCP server instead of relying on static namespace egress', async () => {+ // Bootstrap this scenario without policies; later LIST fixtures supply existing objects.+ mockApi.readNamespacedNetworkPolicy.mockRejectedValue({ code: 404 })const cache = new Map<string, McpServerCRD>()cache.set('mongo', {name: 'mongo',@@ -3205,6 +3254,8 @@ describe('NetworkPolicyReconciler', () => {})it('repairs one-marker live widened policies in every lane before certification', async () => {+ // Bootstrap this scenario without policies; later LIST fixtures supply existing objects.+ mockApi.readNamespacedNetworkPolicy.mockRejectedValue({ code: 404 })const server: McpServerCRD = {name: 'live-server',namespace: 'mcp-server',@@ -4030,6 +4081,8 @@ describe('NetworkPolicyReconciler', () => {})it('replaces same-name Context allow policies in every lane before certification', async () => {+ // Bootstrap this scenario without policies; later LIST fixtures supply existing objects.+ mockApi.readNamespacedNetworkPolicy.mockRejectedValue({ code: 404 })const oldServer: McpServerCRD = {name: 'live-server',namespace: 'mcp-server',@@ -4108,6 +4161,8 @@ describe('NetworkPolicyReconciler', () => {])('refuses a same-name %s safety replacement owned by another server',async (_lane, targetNamespace) => {+ // Create the original owned policies before replacing the listed owner.+ mockApi.readNamespacedNetworkPolicy.mockRejectedValue({ code: 404 })const oldServer: McpServerCRD = {name: 'live-server',namespace: 'mcp-server',@@ -4177,6 +4232,8 @@ describe('NetworkPolicyReconciler', () => {)it('fails the safety pass without certifying when a same-name Context replacement fails', async () => {+ // Bootstrap this scenario without policies; later LIST fixtures supply existing objects.+ mockApi.readNamespacedNetworkPolicy.mockRejectedValue({ code: 404 })const oldServer: McpServerCRD = {name: 'live-server',namespace: 'mcp-server',@@ -4242,6 +4299,8 @@ describe('NetworkPolicyReconciler', () => {})it('retains authoritative same-name policies when only API object key order differs', async () => {+ // Bootstrap this scenario without policies; later LIST fixtures supply existing objects.+ mockApi.readNamespacedNetworkPolicy.mockRejectedValue({ code: 404 })const server: McpServerCRD = {name: 'live-server',namespace: 'mcp-server',@@ -5169,6 +5228,8 @@ describe('NetworkPolicyReconciler', () => {})it('leaves startup external egress creation to its dedicated coordinator', async () => {+ // Bootstrap this scenario without policies; later LIST fixtures supply existing objects.+ mockApi.readNamespacedNetworkPolicy.mockRejectedValue({ code: 404 })const server: McpServerCRD = {name: 'airtable-mcp',namespace: 'mcp-server',@@ -5193,6 +5254,8 @@ describe('NetworkPolicyReconciler', () => {})it('deletes orphaned external egress policies for servers no longer present', async () => {+ // Bootstrap this scenario without policies; later LIST fixtures supply existing objects.+ mockApi.readNamespacedNetworkPolicy.mockRejectedValue({ code: 404 })mockApi.listNamespacedNetworkPolicy.mockImplementation(async ({ labelSelector }: { labelSelector?: string }) => {if (labelSelector?.includes('context-allow')) return { items: [] }@@ -5224,6 +5287,8 @@ describe('NetworkPolicyReconciler', () => {})it('fails closed by deleting HCC-managed policies with missing owner labels', async () => {+ // Bootstrap this scenario without policies; later LIST fixtures supply existing objects.+ mockApi.readNamespacedNetworkPolicy.mockRejectedValue({ code: 404 })mockApi.listNamespacedNetworkPolicy.mockImplementation(async ({ namespace, labelSelector }: { namespace?: string; labelSelector?: string }) => {const managedLabels = { 'clerum.io/managed-by': 'host-context-controller' }@@ -5303,6 +5368,8 @@ describe('NetworkPolicyReconciler', () => {})it('skips external egress orphan cleanup when startup server discovery is incomplete', async () => {+ // Bootstrap this scenario without policies; later LIST fixtures supply existing objects.+ mockApi.readNamespacedNetworkPolicy.mockRejectedValue({ code: 404 })mockApi.listNamespacedNetworkPolicy.mockImplementation(async ({ labelSelector }: { labelSelector?: string }) => {if (labelSelector?.includes('context-allow')) return { items: [] }diff --git a/host-context-controller/src/networkPolicyReconciler.ts b/host-context-controller/src/networkPolicyReconciler.tsindex d11a272be..a7ed8d507 100644--- a/host-context-controller/src/networkPolicyReconciler.ts+++ b/host-context-controller/src/networkPolicyReconciler.ts@@ -2889,7 +2889,8 @@ export class NetworkPolicyReconciler {convergence.policy,'external-egress',isCurrent,- convergence.observedPolicy+ convergence.observedPolicy,+ fresh.policy)} else {hccLogger.info('external egress policy unchanged', { policy: convergence.name })@@ -3336,7 +3337,8 @@ export class NetworkPolicyReconciler {policy: k8s.V1NetworkPolicy,lane: SafetyInventoryLane,isCurrent?: () => boolean,- expectedSnapshot?: k8s.V1NetworkPolicy | null+ expectedSnapshot?: k8s.V1NetworkPolicy | null,+ observedPolicy?: k8s.V1NetworkPolicy | null): Promise<void> {await applyNetworkPolicy(this.networkingApi,@@ -3353,7 +3355,8 @@ export class NetworkPolicyReconciler {},// Best-effort maintenance may skip a disappeared object. External-egress// admission must instead retry: POST409 followed by GET404 applied nothing.- lane === 'external-egress'+ lane === 'external-egress',+ observedPolicy)}diff --git a/host-context-controller/src/utils.ts b/host-context-controller/src/utils.tsindex c69ee9a06..068096412 100644--- a/host-context-controller/src/utils.ts+++ b/host-context-controller/src/utils.ts@@ -43,6 +43,68 @@ export async function observeExistenceRead<T>(}}+/**+ * Read before creating, then retain each caller's convergence policy. The first+ * observed object is consumed once by the existing retry loop; a conflict must+ * always reread the API. SDK calls and their metric labels stay at the call site.+ */+export async function ensureResource<T extends { metadata?: { name?: string } }>(opts: {+ read: () => Promise<T>+ create: () => Promise<unknown>+ existing?: T | null+ /** False preserves a caller's non-throwing failure without counting a skip. */+ converge: (read: () => Promise<T>, observed?: T) => Promise<void | false>+ onSkipped: () => void+ mutationAllowed?: () => boolean+}): Promise<void> {+ const { read, create, converge, onSkipped, mutationAllowed } = opts+ if (mutationAllowed && !mutationAllowed()) return++ const validate = (value: T): T => {+ if (!value || typeof value.metadata?.name !== 'string' || !value.metadata.name.trim()) {+ throw new Error('Kubernetes existence read must resolve to an object with metadata.name')+ }+ return value+ }+ const readFresh = async (): Promise<T> => validate(await read())+ let existing = opts.existing+ if (existing === undefined) {+ try {+ existing = await readFresh()+ } catch (error) {+ if (error == null || getErrorCode(error) !== 404) throw error+ existing = null+ }+ } else if (existing !== null) {+ validate(existing)+ }+ if (mutationAllowed && !mutationAllowed()) return++ if (existing !== null) {+ let snapshot: T | undefined = existing+ const readOnce = async (): Promise<T> => {+ if (snapshot !== undefined) {+ const observed = snapshot+ snapshot = undefined+ return observed+ }+ return readFresh()+ }+ const result = await converge(readOnce, existing)+ if (result !== false && (!mutationAllowed || mutationAllowed())) onSkipped()+ return+ }++ try {+ await create()+ } catch (error) {+ if (error == null || getErrorCode(error) !== 409) throw error+ if (mutationAllowed && !mutationAllowed()) return+ // Absence observed before the POST cannot survive a create conflict.+ await converge(readFresh)+ }+}+/*** Re-read + replace with optimistic-lock retry. K8s rejects a replace whose* `metadata.resourceVersion` does not match the current value; the only@@ -482,7 +544,7 @@ export function normalizeVolumeDefaults(volume: k8s.V1Volume): void {}}-/** Create-or-replace a NetworkPolicy (409 catch → conflict-retry replace). */+/** Read-first create or converge a NetworkPolicy with the existing retry policy. */export async function applyNetworkPolicy(api: k8s.NetworkingV1Api,name: string,@@ -491,33 +553,35 @@ export async function applyNetworkPolicy(logPrefix = '[NetPol]',mutationAllowed?: () => boolean,validateExisting?: (existing: k8s.V1NetworkPolicy) => void,- missingIsError = false+ missingIsError = false,+ existing?: k8s.V1NetworkPolicy | null): Promise<void> {- if (mutationAllowed && !mutationAllowed()) return- try {- await observeCreate('NetworkPolicy', () =>- api.createNamespacedNetworkPolicy({ namespace, body: policy })- )- hccLogger.info('NetworkPolicy created', { scope: logPrefix, policy: name, namespace })- return- } catch (error: unknown) {- if (getErrorCode(error) !== 409) {- throw error- }- }- await replaceWithConflictRetry<k8s.V1NetworkPolicy>({- description: `policy "${name}" in ${namespace}`,- logPrefix,- body: policy,+ await ensureResource<k8s.V1NetworkPolicy>({+ existing,+ mutationAllowed,read: () =>observeExistenceRead('NetworkPolicy', () =>api.readNamespacedNetworkPolicy({ name, namespace })),- replace: body => api.replaceNamespacedNetworkPolicy({ name, namespace, body }),- mutationAllowed,- validateExisting,- missingIsError,- isUpToDate: networkPolicyMatchesDesired,+ create: async () => {+ await observeCreate('NetworkPolicy', () =>+ api.createNamespacedNetworkPolicy({ namespace, body: policy })+ )+ hccLogger.info('NetworkPolicy created', { scope: logPrefix, policy: name, namespace })+ },+ onSkipped: () => createsTotal.inc({ kind: 'NetworkPolicy', outcome: 'skipped' }),+ converge: read =>+ replaceWithConflictRetry<k8s.V1NetworkPolicy>({+ description: `policy "${name}" in ${namespace}`,+ logPrefix,+ body: policy,+ read,+ replace: body => api.replaceNamespacedNetworkPolicy({ name, namespace, body }),+ mutationAllowed,+ validateExisting,+ missingIsError,+ isUpToDate: networkPolicyMatchesDesired,+ }),})}diff --git a/host-context-controller/test/hostReconciler.lifecycle.test.ts b/host-context-controller/test/hostReconciler.lifecycle.test.tsindex 7a799010f..7fceb76a3 100644--- a/host-context-controller/test/hostReconciler.lifecycle.test.ts+++ b/host-context-controller/test/hostReconciler.lifecycle.test.ts@@ -741,15 +741,17 @@ describe('HostReconciler stateless lifecycle — rejection matrix', () => {})const host = makeStatelessHost({ status: suspendedStatus(4) })customApi.getNamespacedCustomObject.mockImplementation(async () => hostApiObject(host))- networkingApi.createNamespacedNetworkPolicy.mockImplementation(async () => {+ const readPolicy = networkingApi.readNamespacedNetworkPolicy.getMockImplementation()!+ networkingApi.readNamespacedNetworkPolicy.mockImplementation(async request => {cacheSynced = false- return {}+ return readPolicy(request)})const provision = vi.spyOn(reconciler as any, 'provisionRuntimeTokenRevision').mockResolvedValue(runtimeTokenProvision(host))await reconciler.reconcile(host)+ expect(networkingApi.readNamespacedNetworkPolicy).toHaveBeenCalled()expect(provision).toHaveBeenCalledOnce()expect(provision).toHaveBeenCalledWith(@@ -779,12 +781,14 @@ describe('HostReconciler stateless lifecycle — rejection matrix', () => {})const host = makeStatelessHost({ status: suspendedStatus(5) })customApi.getNamespacedCustomObject.mockImplementation(async () => hostApiObject(host))- networkingApi.createNamespacedNetworkPolicy.mockImplementation(async () => {+ const readPolicy = networkingApi.readNamespacedNetworkPolicy.getMockImplementation()!+ networkingApi.readNamespacedNetworkPolicy.mockImplementation(async request => {channelCount = 1- return {}+ return readPolicy(request)})await reconciler.reconcile(host)+ expect(networkingApi.readNamespacedNetworkPolicy).toHaveBeenCalled()const deployment = hostDeploymentBody(appsApi, host.name)expect(deployment.spec?.replicas).toBe(1)@@ -814,9 +818,10 @@ describe('HostReconciler stateless lifecycle — rejection matrix', () => {status: { lifecycle: { state: 'active', wakeHandledGeneration: 1 } },})customApi.getNamespacedCustomObject.mockImplementation(async () => hostApiObject(host))- networkingApi.createNamespacedNetworkPolicy.mockImplementation(async () => {+ const readPolicy = networkingApi.readNamespacedNetworkPolicy.getMockImplementation()!+ networkingApi.readNamespacedNetworkPolicy.mockImplementation(async request => {channelCount = 1- return {}+ return readPolicy(request)})const issueTokens = vi.mocked(issueMcpHostRuntimeTokens)const mintGfs = vi.mocked(mintHostGfsToken)@@ -824,6 +829,7 @@ describe('HostReconciler stateless lifecycle — rejection matrix', () => {mintGfs.mockClear()await reconciler.reconcile(host)+ expect(networkingApi.readNamespacedNetworkPolicy).toHaveBeenCalled()expect(issueTokens).toHaveBeenCalledOnce()expect(mintGfs).toHaveBeenCalledOnce()@@ -1012,15 +1018,17 @@ describe('HostReconciler stateless lifecycle — rejection matrix', () => {status: { lifecycle: { state: 'active', wakeHandledGeneration: 1 } },})customApi.getNamespacedCustomObject.mockImplementation(async () => hostApiObject(host))- networkingApi.createNamespacedNetworkPolicy.mockImplementation(async () => {+ const readPolicy = networkingApi.readNamespacedNetworkPolicy.getMockImplementation()!+ networkingApi.readNamespacedNetworkPolicy.mockImplementation(async request => {channelCount = 1- return {}+ return readPolicy(request)})const provision = vi.spyOn(reconciler as any, 'provisionRuntimeTokenRevision').mockResolvedValue(runtimeTokenProvision(host, true))await reconciler.reconcile(host)+ expect(networkingApi.readNamespacedNetworkPolicy).toHaveBeenCalled()expect(provision).toHaveBeenCalledOnce()})@@ -1034,15 +1042,17 @@ describe('HostReconciler stateless lifecycle — rejection matrix', () => {status: { lifecycle: { state: 'active', wakeHandledGeneration: 1 } },})customApi.getNamespacedCustomObject.mockImplementation(async () => hostApiObject(host))- networkingApi.createNamespacedNetworkPolicy.mockImplementation(async () => {+ const readPolicy = networkingApi.readNamespacedNetworkPolicy.getMockImplementation()!+ networkingApi.readNamespacedNetworkPolicy.mockImplementation(async request => {cacheSynced = false- return {}+ return readPolicy(request)})const provision = vi.spyOn(reconciler as any, 'provisionRuntimeTokenRevision').mockResolvedValue(runtimeTokenProvision(host))await reconciler.reconcile(host)+ expect(networkingApi.readNamespacedNetworkPolicy).toHaveBeenCalled()expect(provision).toHaveBeenCalledOnce()const deployment = hostDeploymentBody(appsApi, host.name)diff --git a/host-context-controller/test/hostReconciler.networkpolicy.test.ts b/host-context-controller/test/hostReconciler.networkpolicy.test.tsindex f17fb1eb7..51774783e 100644--- a/host-context-controller/test/hostReconciler.networkpolicy.test.ts+++ b/host-context-controller/test/hostReconciler.networkpolicy.test.ts@@ -122,7 +122,10 @@ describe('HostReconciler.ensureChannelReaderEgressNetworkPolicy', () => {it('creates the egress NP in channels namespace with correct selectors', async () => {const { reconciler, mocks } = createReconciler()+ // This case exercises initial creation of an absent policy.+ mocks.networkingApi.readNamespacedNetworkPolicy.mockRejectedValueOnce({ code: 404 })await (reconciler as any).ensureChannelReaderEgressNetworkPolicy(HOST)+ expect(mocks.networkingApi.readNamespacedNetworkPolicy).toHaveBeenCalledTimes(1)expect(mocks.networkingApi.createNamespacedNetworkPolicy).toHaveBeenCalledTimes(1)const call = (mocks.networkingApi.createNamespacedNetworkPolicy as any).mock.calls[0][0]@@ -163,7 +166,10 @@ describe('HostReconciler.ensureMcpHostIngressNetworkPolicy', () => {it('creates the ingress NP in mcp-host namespace with correct selectors', async () => {const { reconciler, mocks } = createReconciler()+ // This case exercises initial creation of an absent policy.+ mocks.networkingApi.readNamespacedNetworkPolicy.mockRejectedValueOnce({ code: 404 })await (reconciler as any).ensureMcpHostIngressNetworkPolicy(HOST)+ expect(mocks.networkingApi.readNamespacedNetworkPolicy).toHaveBeenCalledTimes(1)expect(mocks.networkingApi.createNamespacedNetworkPolicy).toHaveBeenCalledTimes(1)const call = (mocks.networkingApi.createNamespacedNetworkPolicy as any).mock.calls[0][0]diff --git a/host-context-controller/test/hostReconciler.test.ts b/host-context-controller/test/hostReconciler.test.tsindex da634e350..f0dd97d86 100644--- a/host-context-controller/test/hostReconciler.test.ts+++ b/host-context-controller/test/hostReconciler.test.ts@@ -817,15 +817,17 @@ describe('HostReconciler — desktop support', () => {expect(resources.limits.cpu).toBe('1')})- it('creates desktop NetworkPolicy when desktop enabled', async () => {+ it('converges desktop NetworkPolicy when desktop enabled', async () => {const { reconciler, networkingApi } = createReconciler()const host = makeDesktopHost({ x11: true })await reconciler.reconcile(host)- // reconcile creates per-host ingress, rpc-proxy egress, desktop,+ // Reconcile updates existing per-host ingress, rpc-proxy egress, desktop,// GFS egress, channel-reader, and workflow-approval-reader policies.- expect(networkingApi.createNamespacedNetworkPolicy).toHaveBeenCalledTimes(8)- const names = networkingApi.createNamespacedNetworkPolicy.mock.calls.map(+ expect(networkingApi.replaceNamespacedNetworkPolicy).toHaveBeenCalledTimes(8)+ expect(networkingApi.createNamespacedNetworkPolicy).not.toHaveBeenCalled()+ expect(networkingApi.readNamespacedNetworkPolicy).toHaveBeenCalled()+ const names = networkingApi.replaceNamespacedNetworkPolicy.mock.calls.map((c: any[]) => c[0].body.metadata.name)expect(names).toEqual(@@ -835,25 +837,27 @@ describe('HostReconciler — desktop support', () => {'workflow-approval-reader-desktop-host-egress-mcp-host',]))- const call = networkingApi.createNamespacedNetworkPolicy.mock.calls.find(+ const call = networkingApi.replaceNamespacedNetworkPolicy.mock.calls.find((c: any[]) => c[0].body.metadata.name === 'allow-rpc-proxy-desktop-desktop-host')![0]expect(call.body.metadata.name).toBe('allow-rpc-proxy-desktop-desktop-host')expect(call.body.spec.podSelector).toEqual({ matchLabels: { app: 'desktop-host' } })})- it('does not create desktop NetworkPolicy for non-desktop hosts', async () => {+ it('does not converge desktop NetworkPolicy for non-desktop hosts', async () => {const { reconciler, networkingApi } = createReconciler()const host = makeHost()await reconciler.reconcile(host)- // reconcile creates 6 NPs for non-desktop hosts:+ // Reconcile updates 7 existing NPs for non-desktop hosts:// mcp-host ingress from channel-reader, mcp-host ingress from rpc-proxy,// mcp-host GFS egress, rpc-proxy host egress, channel-reader egress,// and workflow-approval-reader policies.// (no desktop NP)- expect(networkingApi.createNamespacedNetworkPolicy).toHaveBeenCalledTimes(7)- const names = networkingApi.createNamespacedNetworkPolicy.mock.calls.map(+ expect(networkingApi.replaceNamespacedNetworkPolicy).toHaveBeenCalledTimes(7)+ expect(networkingApi.createNamespacedNetworkPolicy).not.toHaveBeenCalled()+ expect(networkingApi.readNamespacedNetworkPolicy).toHaveBeenCalled()+ const names = networkingApi.replaceNamespacedNetworkPolicy.mock.calls.map((c: any[]) => c[0].body.metadata.name)expect(names).toEqual(