Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript

feat(hcc): add read-first convergence for network policies

Alfredo Lopez2d98b4eab526
diff --git a/host-context-controller/package-lock.json b/host-context-controller/package-lock.json
index 4587b15c7..b89eaa68a 100644
--- a/host-context-controller/package-lock.json
+++ b/host-context-controller/package-lock.json
@@ -1,12 +1,12 @@
{
"name": "clerum-host-context-controller",
- "version": "0.1.303",
+ "version": "0.1.304",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "clerum-host-context-controller",
- "version": "0.1.303",
+ "version": "0.1.304",
"license": "SEE LICENSE IN LICENSE",
"dependencies": {
"@clerum/codex-catalog-projection": "file:../packages/codex-catalog-projection",
diff --git a/host-context-controller/package.json b/host-context-controller/package.json
index e3d11c180..5efd6438d 100644
--- a/host-context-controller/package.json
+++ b/host-context-controller/package.json
@@ -1,6 +1,6 @@
{
"name": "clerum-host-context-controller",
- "version": "0.1.303",
+ "version": "0.1.304",
"description": "Host Context Controller - provides McpServer CRDs to mcp-host via REST API",
"main": "dist/main.js",
"engines": {
diff --git a/host-context-controller/src/__tests__/applyNetworkPolicyNoopGate.test.ts b/host-context-controller/src/__tests__/applyNetworkPolicyNoopGate.test.ts
index 99e3ced92..37dc2ae3a 100644
--- a/host-context-controller/src/__tests__/applyNetworkPolicyNoopGate.test.ts
+++ b/host-context-controller/src/__tests__/applyNetworkPolicyNoopGate.test.ts
@@ -34,11 +34,15 @@ function fakeNetworkingApi() {
}
describe('applyNetworkPolicy no-op gate', () => {
- it('CREATE-NP-1: successful create never reads or replaces', async () => {
+ it('CREATE-NP-1: absent policy creates after one GET and never replaces', async () => {
const api = fakeNetworkingApi()
+ api.readNamespacedNetworkPolicy.mockRejectedValueOnce(apiException(404))
await applyNetworkPolicy(api as unknown as k8s.NetworkingV1Api, 'np', 'ns', desiredPolicy())
expect(api.createNamespacedNetworkPolicy).toHaveBeenCalledOnce()
- expect(api.readNamespacedNetworkPolicy).not.toHaveBeenCalled()
+ expect(api.readNamespacedNetworkPolicy).toHaveBeenCalledExactlyOnceWith({
+ name: 'np',
+ namespace: 'ns',
+ })
expect(api.replaceNamespacedNetworkPolicy).not.toHaveBeenCalled()
})
@@ -47,11 +51,12 @@ describe('applyNetworkPolicy no-op gate', () => {
const existing = asApiserverNetworkPolicy(desired)
expect(existing.spec).toBeDefined()
const api = fakeNetworkingApi()
- api.createNamespacedNetworkPolicy.mockRejectedValue({ code: 409 })
api.readNamespacedNetworkPolicy.mockResolvedValue(existing)
const log = vi.spyOn(console, 'log')
try {
await applyNetworkPolicy(api as unknown as k8s.NetworkingV1Api, 'np', 'ns', desired)
+ expect(api.createNamespacedNetworkPolicy).not.toHaveBeenCalled()
+ expect(api.readNamespacedNetworkPolicy).toHaveBeenCalledOnce()
expect(api.replaceNamespacedNetworkPolicy).not.toHaveBeenCalled()
expect(updatedPolicyLogs(log, 'policy "np" in ns')).toEqual([])
} finally {
@@ -62,13 +67,14 @@ describe('applyNetworkPolicy no-op gate', () => {
it('NOOP-NP-2 / LOG-NP-2: drift replaces once and logs once', async () => {
const desired = desiredPolicy()
const api = fakeNetworkingApi()
- api.createNamespacedNetworkPolicy.mockRejectedValue({ code: 409 })
api.readNamespacedNetworkPolicy.mockResolvedValue(
asApiserverNetworkPolicy(desired, { port: 9090 })
)
const log = vi.spyOn(console, 'log')
try {
await applyNetworkPolicy(api as unknown as k8s.NetworkingV1Api, 'np', 'ns', desired)
+ expect(api.createNamespacedNetworkPolicy).not.toHaveBeenCalled()
+ expect(api.readNamespacedNetworkPolicy).toHaveBeenCalledOnce()
expect(api.replaceNamespacedNetworkPolicy).toHaveBeenCalledOnce()
expect(updatedPolicyLogs(log, 'policy "np" in ns')).toEqual([
'[NetPol] Updated policy "np" in ns',
@@ -81,7 +87,6 @@ describe('applyNetworkPolicy no-op gate', () => {
it('ORDER-NP-1: validateExisting throws before isUpToDate; replace is not called', async () => {
const desired = desiredPolicy()
const api = fakeNetworkingApi()
- api.createNamespacedNetworkPolicy.mockRejectedValue({ code: 409 })
api.readNamespacedNetworkPolicy.mockResolvedValue(asApiserverNetworkPolicy(desired))
const validateExisting = vi.fn(() => {
throw new Error('NetworkPolicy "np" has conflicting ownership for the context-ingress lane')
@@ -99,36 +104,44 @@ describe('applyNetworkPolicy no-op gate', () => {
)
).rejects.toThrow(/ownership/)
- expect(validateExisting).toHaveBeenCalledOnce()
+ expect(validateExisting).toHaveBeenCalledExactlyOnceWith(asApiserverNetworkPolicy(desired))
+ expect(api.createNamespacedNetworkPolicy).not.toHaveBeenCalled()
expect(api.replaceNamespacedNetworkPolicy).not.toHaveBeenCalled()
})
it.each([
{ form: 'ApiException', error: apiException(404) },
{ form: 'statusCode', error: { response: { statusCode: 404 } } },
- ])('TOCTOU-NP-1: create 409 + read 404 ($form) returns without replace', async ({ error }) => {
- const api = fakeNetworkingApi()
- api.createNamespacedNetworkPolicy.mockRejectedValue({ code: 409 })
- api.readNamespacedNetworkPolicy.mockRejectedValue(error)
+ ])(
+ 'TOCTOU-NP-1: GET404 + create409 + fresh GET404 ($form) returns without replace',
+ async ({ error }) => {
+ const api = fakeNetworkingApi()
+ api.createNamespacedNetworkPolicy.mockRejectedValue({ code: 409 })
+ api.readNamespacedNetworkPolicy
+ .mockRejectedValueOnce(apiException(404))
+ .mockRejectedValueOnce(error)
- await expect(
- applyNetworkPolicy(api as unknown as k8s.NetworkingV1Api, 'np', 'ns', desiredPolicy())
- ).resolves.toBeUndefined()
+ await expect(
+ applyNetworkPolicy(api as unknown as k8s.NetworkingV1Api, 'np', 'ns', desiredPolicy())
+ ).resolves.toBeUndefined()
- expect(api.createNamespacedNetworkPolicy).toHaveBeenCalledOnce()
- expect(api.readNamespacedNetworkPolicy).toHaveBeenCalledOnce()
- expect(api.replaceNamespacedNetworkPolicy).not.toHaveBeenCalled()
- })
+ expect(api.createNamespacedNetworkPolicy).toHaveBeenCalledOnce()
+ expect(api.readNamespacedNetworkPolicy).toHaveBeenCalledTimes(2)
+ expect(api.replaceNamespacedNetworkPolicy).not.toHaveBeenCalled()
+ }
+ )
it.each([
{ form: 'ApiException', error: apiException(404) },
{ form: 'statusCode', error: { response: { statusCode: 404 } } },
])(
- 'admission-sensitive create409/read404 ($form) rejects instead of certifying absence',
+ 'admission-sensitive GET404/create409/fresh GET404 ($form) rejects instead of certifying absence',
async ({ error }) => {
const api = fakeNetworkingApi()
api.createNamespacedNetworkPolicy.mockRejectedValue({ code: 409 })
- api.readNamespacedNetworkPolicy.mockRejectedValue(error)
+ api.readNamespacedNetworkPolicy
+ .mockRejectedValueOnce(apiException(404))
+ .mockRejectedValueOnce(error)
await expect(
applyNetworkPolicy(
@@ -143,6 +156,8 @@ describe('applyNetworkPolicy no-op gate', () => {
)
).rejects.toBe(error)
+ expect(api.createNamespacedNetworkPolicy).toHaveBeenCalledOnce()
+ expect(api.readNamespacedNetworkPolicy).toHaveBeenCalledTimes(2)
expect(api.replaceNamespacedNetworkPolicy).not.toHaveBeenCalled()
}
)
@@ -152,28 +167,36 @@ describe('applyNetworkPolicy no-op gate', () => {
{ status: 403, form: 'statusCode', error: { response: { statusCode: 403 } } },
{ status: 500, form: 'ApiException', error: apiException(500) },
{ status: 500, form: 'statusCode', error: { response: { statusCode: 500 } } },
- ])('TOCTOU-NP-2-APPLY: create 409 + read $status ($form) still throws', async ({ error }) => {
- const api = fakeNetworkingApi()
- api.createNamespacedNetworkPolicy.mockRejectedValue({ code: 409 })
- api.readNamespacedNetworkPolicy.mockRejectedValue(error)
+ ])(
+ 'TOCTOU-NP-2-APPLY: GET404 + create409 + fresh GET$status ($form) still throws',
+ async ({ error }) => {
+ const api = fakeNetworkingApi()
+ api.createNamespacedNetworkPolicy.mockRejectedValue({ code: 409 })
+ api.readNamespacedNetworkPolicy
+ .mockRejectedValueOnce(apiException(404))
+ .mockRejectedValueOnce(error)
- await expect(
- applyNetworkPolicy(api as unknown as k8s.NetworkingV1Api, 'np', 'ns', desiredPolicy())
- ).rejects.toBe(error)
+ await expect(
+ applyNetworkPolicy(api as unknown as k8s.NetworkingV1Api, 'np', 'ns', desiredPolicy())
+ ).rejects.toBe(error)
- expect(api.createNamespacedNetworkPolicy).toHaveBeenCalledOnce()
- expect(api.readNamespacedNetworkPolicy).toHaveBeenCalledOnce()
- expect(api.replaceNamespacedNetworkPolicy).not.toHaveBeenCalled()
- })
+ expect(api.createNamespacedNetworkPolicy).toHaveBeenCalledOnce()
+ expect(api.readNamespacedNetworkPolicy).toHaveBeenCalledTimes(2)
+ expect(api.replaceNamespacedNetworkPolicy).not.toHaveBeenCalled()
+ }
+ )
- it('GATE-NP-1: mutationAllowed false after create-409 skips replace on drift', async () => {
+ it('GATE-NP-1: expiry during existence GET suppresses POST and drift PUT', async () => {
const desired = desiredPolicy()
const drifted = asApiserverNetworkPolicy(desired, { port: 9090 })
expect(networkPolicyMatchesDesired(desired, drifted)).toBe(false)
const api = fakeNetworkingApi()
- api.createNamespacedNetworkPolicy.mockRejectedValue({ code: 409 })
- api.readNamespacedNetworkPolicy.mockResolvedValue(drifted)
- const mutationAllowed = vi.fn().mockReturnValueOnce(true).mockReturnValueOnce(false)
+ let active = true
+ api.readNamespacedNetworkPolicy.mockImplementation(async () => {
+ active = false
+ return drifted
+ })
+ const mutationAllowed = vi.fn(() => active)
await applyNetworkPolicy(
api as unknown as k8s.NetworkingV1Api,
@@ -184,7 +207,9 @@ describe('applyNetworkPolicy no-op gate', () => {
mutationAllowed
)
- expect(mutationAllowed).toHaveBeenCalledTimes(2)
+ expect(mutationAllowed.mock.results.map(result => result.value)).toEqual([true, false])
+ expect(api.readNamespacedNetworkPolicy).toHaveBeenCalledOnce()
+ expect(api.createNamespacedNetworkPolicy).not.toHaveBeenCalled()
expect(api.replaceNamespacedNetworkPolicy).not.toHaveBeenCalled()
})
})
diff --git a/host-context-controller/src/__tests__/hostNetworkPolicyNoopGate.test.ts b/host-context-controller/src/__tests__/hostNetworkPolicyNoopGate.test.ts
index 2fa4f8053..7981c5dbc 100644
--- a/host-context-controller/src/__tests__/hostNetworkPolicyNoopGate.test.ts
+++ b/host-context-controller/src/__tests__/hostNetworkPolicyNoopGate.test.ts
@@ -51,14 +51,24 @@ describe('Host ensureMcpHostIngressNetworkPolicy inherits applyNetworkPolicy gat
let captured: k8s.V1NetworkPolicy | undefined
networkingApi.createNamespacedNetworkPolicy.mockImplementation(async ({ body }) => {
captured = body as k8s.V1NetworkPolicy
- throw Object.assign(new Error('already exists'), { code: 409 })
+ return body
})
networkingApi.readNamespacedNetworkPolicy.mockImplementation(async () => {
- if (!captured?.spec) throw new Error('create did not capture a spec')
+ if (!captured) throw Object.assign(new Error('not found'), { code: 404 })
+ if (!captured.spec) throw new Error('create did not capture a spec')
return asApiserverNetworkPolicy(captured)
})
await (reconciler as any).ensureMcpHostIngressNetworkPolicy(host)
+ expect(networkingApi.readNamespacedNetworkPolicy).toHaveBeenCalledOnce()
+ expect(networkingApi.createNamespacedNetworkPolicy).toHaveBeenCalledOnce()
+ networkingApi.readNamespacedNetworkPolicy.mockClear()
+ networkingApi.createNamespacedNetworkPolicy.mockClear()
+ networkingApi.createNamespacedNetworkPolicy.mockRejectedValue({ code: 409 })
+
+ await (reconciler as any).ensureMcpHostIngressNetworkPolicy(host)
+ expect(networkingApi.readNamespacedNetworkPolicy).toHaveBeenCalledOnce()
+ expect(networkingApi.createNamespacedNetworkPolicy).not.toHaveBeenCalled()
expect(networkingApi.replaceNamespacedNetworkPolicy).not.toHaveBeenCalled()
networkingApi.replaceNamespacedNetworkPolicy.mockClear()
@@ -67,6 +77,8 @@ describe('Host ensureMcpHostIngressNetworkPolicy inherits applyNetworkPolicy gat
return asApiserverNetworkPolicy(captured, { port: 9090 })
})
await (reconciler as any).ensureMcpHostIngressNetworkPolicy(host)
+ expect(networkingApi.readNamespacedNetworkPolicy).toHaveBeenCalledTimes(2)
+ expect(networkingApi.createNamespacedNetworkPolicy).not.toHaveBeenCalled()
expect(networkingApi.replaceNamespacedNetworkPolicy).toHaveBeenCalledOnce()
})
})
diff --git a/host-context-controller/src/__tests__/hostReconciler.mounts.test.ts b/host-context-controller/src/__tests__/hostReconciler.mounts.test.ts
index 9735bd350..9c51b6d4b 100644
--- a/host-context-controller/src/__tests__/hostReconciler.mounts.test.ts
+++ b/host-context-controller/src/__tests__/hostReconciler.mounts.test.ts
@@ -1304,7 +1304,8 @@ describe('HostReconciler.reconcile — uses resolveContextMounts', () => {
it('creates host-scoped rpc-proxy ingress and egress policies', async () => {
const networkingApi = {
createNamespacedNetworkPolicy: vi.fn(async () => ({})),
- readNamespacedNetworkPolicy: vi.fn(async () => ({})),
+ // This initial-provisioning case starts with no NetworkPolicies.
+ readNamespacedNetworkPolicy: vi.fn().mockRejectedValue({ code: 404 }),
replaceNamespacedNetworkPolicy: vi.fn(async () => ({})),
}
const desktopHost: HostCRD = {
diff --git a/host-context-controller/src/__tests__/llmHookNetworkPolicyNoopGate.test.ts b/host-context-controller/src/__tests__/llmHookNetworkPolicyNoopGate.test.ts
index 6ffe58615..a51dcc3e7 100644
--- a/host-context-controller/src/__tests__/llmHookNetworkPolicyNoopGate.test.ts
+++ b/host-context-controller/src/__tests__/llmHookNetworkPolicyNoopGate.test.ts
@@ -106,6 +106,11 @@ describe('LlmHook NetworkPolicy no-op gate', () => {
const log = vi.spyOn(console, 'log')
try {
await (reconciler as any).ensureNetworkPolicy(podKey, [hook], [])
+ expect(networkingApi.readNamespacedNetworkPolicy).toHaveBeenCalledExactlyOnceWith({
+ name: desired.metadata!.name,
+ namespace: 'llm-hooks',
+ })
+ expect(networkingApi.createNamespacedNetworkPolicy).not.toHaveBeenCalled()
expect(networkingApi.replaceNamespacedNetworkPolicy).not.toHaveBeenCalled()
expect(updatedPolicyLogs(log, `NetworkPolicy "${desired.metadata?.name}"`)).toEqual([])
} finally {
@@ -194,4 +199,40 @@ describe('LlmHook NetworkPolicy no-op gate', () => {
log.mockRestore()
}
})
+
+ it('preserves live annotations when a present policy needs convergence without POST', async () => {
+ const hook = makeImageHook()
+ const podKey = computePodKey(hook)!
+ const desired = (reconciler as any).buildNetworkPolicy(
+ podKey,
+ [hook],
+ []
+ ) as k8s.V1NetworkPolicy
+ const existing = asApiserverNetworkPolicy(desired)
+ existing.metadata = {
+ ...existing.metadata,
+ annotations: { 'example.com/operator': 'retained' },
+ }
+ existing.spec = { ...existing.spec!, ingress: [{}] }
+ networkingApi.readNamespacedNetworkPolicy.mockResolvedValue(existing)
+ await (reconciler as any).ensureNetworkPolicy(podKey, [hook], [])
+ expect(networkingApi.readNamespacedNetworkPolicy).toHaveBeenCalledTimes(1)
+ expect(networkingApi.createNamespacedNetworkPolicy).not.toHaveBeenCalled()
+ expect(networkingApi.replaceNamespacedNetworkPolicy).toHaveBeenCalledTimes(1)
+ const body = networkingApi.replaceNamespacedNetworkPolicy.mock.calls[0][0].body
+ expect(body.metadata?.annotations?.['example.com/operator']).toBe('retained')
+ expect(body.spec?.ingress).toEqual(desired.spec?.ingress)
+ })
+
+ it('propagates GET 403 without creating or replacing a policy', async () => {
+ const error = { code: 403 }
+ networkingApi.readNamespacedNetworkPolicy.mockRejectedValue(error)
+ const hook = makeImageHook()
+ await expect(
+ (reconciler as any).ensureNetworkPolicy(computePodKey(hook)!, [hook], [])
+ ).rejects.toBe(error)
+ expect(networkingApi.readNamespacedNetworkPolicy).toHaveBeenCalledTimes(1)
+ expect(networkingApi.createNamespacedNetworkPolicy).not.toHaveBeenCalled()
+ expect(networkingApi.replaceNamespacedNetworkPolicy).not.toHaveBeenCalled()
+ })
})
diff --git a/host-context-controller/src/__tests__/llmHooksEffectiveEgress.test.ts b/host-context-controller/src/__tests__/llmHooksEffectiveEgress.test.ts
index d007b146d..ef478fc3a 100644
--- a/host-context-controller/src/__tests__/llmHooksEffectiveEgress.test.ts
+++ b/host-context-controller/src/__tests__/llmHooksEffectiveEgress.test.ts
@@ -145,6 +145,26 @@ describe('llm-hooks effective pod egress (N5)', () => {
coreApi = createMockCoreApi()
customApi = createMockCustomApi()
networkingApi = createMockNetworkingApi()
+ // Start with an empty API inventory and retain real desired objects, including
+ // their ownership labels, so subsequent reads model present policies.
+ const policies = new Map<string, k8s.V1NetworkPolicy>()
+ networkingApi.readNamespacedNetworkPolicy.mockImplementation(async ({ name, namespace }) => {
+ const policy = policies.get(`${namespace}/${name}`)
+ if (!policy) throw { code: 404 }
+ return structuredClone(policy)
+ })
+ networkingApi.createNamespacedNetworkPolicy.mockImplementation(async ({ namespace, body }) => {
+ const key = `${namespace}/${body.metadata!.name}`
+ if (policies.has(key)) throw { code: 409 }
+ policies.set(key, structuredClone(body))
+ return body
+ })
+ networkingApi.replaceNamespacedNetworkPolicy.mockImplementation(
+ async ({ name, namespace, body }) => {
+ policies.set(`${namespace}/${name}`, structuredClone(body))
+ return body
+ }
+ )
customApi.getNamespacedCustomObjectStatus.mockResolvedValue({
metadata: { resourceVersion: '1' },
status: { conditions: [] },
diff --git a/host-context-controller/src/__tests__/sharedFileSystemReconciler.test.ts b/host-context-controller/src/__tests__/sharedFileSystemReconciler.test.ts
index c724dfd39..c3f093563 100644
--- a/host-context-controller/src/__tests__/sharedFileSystemReconciler.test.ts
+++ b/host-context-controller/src/__tests__/sharedFileSystemReconciler.test.ts
@@ -2,6 +2,8 @@ import { beforeEach, describe, expect, it, vi } from 'vitest'
import {
type SharedFileSystemFactoryConfig,
WFC_MOUNT_PATH,
+ buildEgressNetworkPolicy,
+ buildIngressNetworkPolicy,
pvcName,
wfcDeploymentName,
wfcEgressPolicyName,
@@ -50,7 +52,13 @@ function err(code: number, msg = 'k8s error'): Error & { code: number } {
const notFound = () => err(404, 'not found')
const alreadyExists = () => err(409, 'already exists')
-function makeMocks() {
+function makeMocks(systems: SharedFileSystemCRD[] = [makeSfs()]) {
+ // Status/cleanup tests use already-converged policies with real identity and
+ // ownership. Cold-start tests explicitly arrange absence for their own reads.
+ const policies = systems.flatMap(sfs => [
+ buildIngressNetworkPolicy(sfs, factoryConfig),
+ buildEgressNetworkPolicy(sfs, factoryConfig),
+ ])
const appsApi = {
createNamespacedDeployment: vi.fn().mockResolvedValue({}),
// Truthful status (#592) reads readyReplicas; conflict-retry reads metadata.
@@ -82,7 +90,11 @@ function makeMocks() {
}
const networkingApi = {
createNamespacedNetworkPolicy: vi.fn().mockResolvedValue({}),
- readNamespacedNetworkPolicy: vi.fn().mockResolvedValue({ metadata: { resourceVersion: '1' } }),
+ readNamespacedNetworkPolicy: vi.fn(async ({ name }: { name: string }) => {
+ const policy = policies.find(item => item.metadata?.name === name)
+ if (!policy) throw new Error(`Missing NetworkPolicy fixture for ${name}`)
+ return { ...policy, metadata: { ...policy.metadata, resourceVersion: '1' } }
+ }),
replaceNamespacedNetworkPolicy: vi.fn().mockResolvedValue({}),
deleteNamespacedNetworkPolicy: vi.fn().mockResolvedValue({}),
}
@@ -167,6 +179,9 @@ describe('SharedFileSystemReconciler — happy path', () => {
it('creates PVC, Service, Deployment, and both NetworkPolicies — and NEVER an init Job', async () => {
const sfs = makeSfs()
+ mocks.networkingApi.readNamespacedNetworkPolicy
+ .mockRejectedValueOnce(notFound())
+ .mockRejectedValueOnce(notFound())
await reconciler.reconcile(sfs)
expect(mocks.coreApi.createNamespacedPersistentVolumeClaim).toHaveBeenCalledTimes(1)
@@ -292,6 +307,23 @@ describe('SharedFileSystemReconciler — legacy init-Job cleanup', () => {
})
describe('SharedFileSystemReconciler — idempotence', () => {
+ it('leaves existing equivalent NetworkPolicies in place without POST or PUT', async () => {
+ const mocks = makeMocks()
+ await makeReconciler(mocks).reconcile(makeSfs())
+ expect(mocks.networkingApi.readNamespacedNetworkPolicy).toHaveBeenCalledTimes(2)
+ expect(mocks.networkingApi.createNamespacedNetworkPolicy).not.toHaveBeenCalled()
+ expect(mocks.networkingApi.replaceNamespacedNetworkPolicy).not.toHaveBeenCalled()
+ })
+
+ it('propagates a denied NetworkPolicy read without attempting creation', async () => {
+ const mocks = makeMocks()
+ const denied = err(403, 'policy read denied')
+ mocks.networkingApi.readNamespacedNetworkPolicy.mockRejectedValueOnce(denied)
+ await expect(makeReconciler(mocks).reconcile(makeSfs())).rejects.toBe(denied)
+ expect(mocks.networkingApi.createNamespacedNetworkPolicy).not.toHaveBeenCalled()
+ expect(mocks.networkingApi.replaceNamespacedNetworkPolicy).not.toHaveBeenCalled()
+ })
+
it('PVC 409 (already exists) is treated as success — no replace attempt', async () => {
const mocks = makeMocks()
mocks.coreApi.createNamespacedPersistentVolumeClaim.mockRejectedValueOnce(alreadyExists())
@@ -475,17 +507,17 @@ describe('SharedFileSystemReconciler — reconcileDelete', () => {
describe('SharedFileSystemReconciler — fullReconcile', () => {
it('reconciles every desired SFS and continues past errors on individual SFSes', async () => {
- const mocks = makeMocks()
- mocks.coreApi.createNamespacedPersistentVolumeClaim
- .mockResolvedValueOnce({})
- .mockRejectedValueOnce(err(500, 'cluster down'))
- .mockResolvedValueOnce({})
- const reconciler = makeReconciler(mocks)
const desired = [
makeSfs({ name: 'sfs-a' }),
makeSfs({ name: 'sfs-b' }),
makeSfs({ name: 'sfs-c' }),
]
+ const mocks = makeMocks(desired)
+ mocks.coreApi.createNamespacedPersistentVolumeClaim
+ .mockResolvedValueOnce({})
+ .mockRejectedValueOnce(err(500, 'cluster down'))
+ .mockResolvedValueOnce({})
+ const reconciler = makeReconciler(mocks)
await expect(reconciler.fullReconcile(desired)).resolves.toBeUndefined()
expect(mocks.coreApi.createNamespacedPersistentVolumeClaim).toHaveBeenCalledTimes(3)
expect(reconciler.getStatus(desired[0]).phase).toBe('Ready')
diff --git a/host-context-controller/src/bindingPolicyReconciler.test.ts b/host-context-controller/src/bindingPolicyReconciler.test.ts
index 48d6fe191..3ba292a64 100644
--- a/host-context-controller/src/bindingPolicyReconciler.test.ts
+++ b/host-context-controller/src/bindingPolicyReconciler.test.ts
@@ -13,7 +13,27 @@ function deferred<T = void>() {
function makeMockNetworkingApi() {
return {
createNamespacedNetworkPolicy: vi.fn().mockResolvedValue({}),
- readNamespacedNetworkPolicy: vi.fn().mockResolvedValue({ metadata: { resourceVersion: '1' } }),
+ readNamespacedNetworkPolicy: vi.fn(
+ async ({
+ name,
+ namespace,
+ }: {
+ name: string
+ namespace: string
+ }): Promise<k8s.V1NetworkPolicy> => ({
+ metadata: {
+ name,
+ namespace,
+ uid: `uid-${name}`,
+ resourceVersion: '1',
+ labels: {
+ 'clerum.io/managed-by': 'host-context-controller',
+ 'clerum.io/policy-type': 'binding-allow',
+ 'clerum.io/recipe': 'my-recipe',
+ },
+ },
+ })
+ ),
replaceNamespacedNetworkPolicy: vi.fn().mockResolvedValue({}),
listNamespacedNetworkPolicy: vi.fn().mockResolvedValue({ items: [] }),
deleteNamespacedNetworkPolicy: vi.fn().mockResolvedValue({}),
@@ -48,10 +68,15 @@ describe('BindingPolicyReconciler', () => {
const singleBinding: BindingDef[] = [{ from: 'redis-mcp', to: 'redis', port: 6379 }]
it('creates egress + ingress policies for a binding', async () => {
+ // Both policies are absent only in this initial-creation case.
+ mockApi.readNamespacedNetworkPolicy
+ .mockRejectedValueOnce({ code: 404 })
+ .mockRejectedValueOnce({ code: 404 })
await reconciler.reconcileBindings('my-recipe', singleBinding, 'redis-mcp', 'redis-mcp', {
isCurrent: () => true,
})
+ expect(mockApi.readNamespacedNetworkPolicy).toHaveBeenCalledTimes(2)
// 2 calls: 1 egress + 1 ingress
expect(mockApi.createNamespacedNetworkPolicy).toHaveBeenCalledTimes(2)
@@ -89,7 +114,10 @@ describe('BindingPolicyReconciler', () => {
isCurrent: () => true,
})
- for (const call of mockApi.createNamespacedNetworkPolicy.mock.calls) {
+ expect(mockApi.readNamespacedNetworkPolicy).toHaveBeenCalledTimes(2)
+ expect(mockApi.replaceNamespacedNetworkPolicy).toHaveBeenCalledTimes(2)
+ expect(mockApi.createNamespacedNetworkPolicy).not.toHaveBeenCalled()
+ for (const call of mockApi.replaceNamespacedNetworkPolicy.mock.calls) {
const labels = (call[0] as { body: { metadata: { labels: Record<string, string> } } }).body
.metadata.labels
expect(labels['clerum.io/policy-type']).toBe('binding-allow')
@@ -98,20 +126,33 @@ describe('BindingPolicyReconciler', () => {
}
})
- it('handles 409 conflict with create-or-replace pattern', async () => {
+ it('rereads and converges when a policy appears between GET and POST', async () => {
+ const readPolicy = mockApi.readNamespacedNetworkPolicy.getMockImplementation()!
+ const observedNames = new Set<string>()
+ mockApi.readNamespacedNetworkPolicy.mockImplementation(async request => {
+ if (!observedNames.has(request.name)) {
+ observedNames.add(request.name)
+ throw { code: 404 }
+ }
+ return readPolicy(request)
+ })
mockApi.createNamespacedNetworkPolicy.mockRejectedValue({ code: 409 })
await reconciler.reconcileBindings('my-recipe', singleBinding, 'redis-mcp', 'redis-mcp', {
isCurrent: () => true,
})
- expect(mockApi.readNamespacedNetworkPolicy).toHaveBeenCalled()
- expect(mockApi.replaceNamespacedNetworkPolicy).toHaveBeenCalled()
+ expect(mockApi.readNamespacedNetworkPolicy).toHaveBeenCalledTimes(4)
+ expect(mockApi.createNamespacedNetworkPolicy).toHaveBeenCalledTimes(2)
+ expect(mockApi.replaceNamespacedNetworkPolicy).toHaveBeenCalledTimes(2)
+ for (const [request] of mockApi.replaceNamespacedNetworkPolicy.mock.calls) {
+ expect(request.body.metadata.resourceVersion).toBe('1')
+ }
})
it('does not replace or start another apply after its authority lease is retired', async () => {
const readStarted = deferred()
- const releaseRead = deferred<{ metadata: { resourceVersion: string } }>()
+ const releaseRead = deferred<{ metadata: { name: string; resourceVersion: string } }>()
mockApi.createNamespacedNetworkPolicy.mockRejectedValue({ code: 409 })
mockApi.readNamespacedNetworkPolicy.mockImplementationOnce(async () => {
readStarted.resolve()
@@ -129,12 +170,14 @@ describe('BindingPolicyReconciler', () => {
await readStarted.promise
current = false
- releaseRead.resolve({ metadata: { resourceVersion: '1' } })
+ releaseRead.resolve({
+ metadata: { name: 'bind-my-recipe-redis-mcp-redis-egress', resourceVersion: '1' },
+ })
await reconcile
- // The first create was authorized and returned 409 without mutating. Once
- // the lease is retired, no replacement or second policy apply may start.
- expect(mockApi.createNamespacedNetworkPolicy).toHaveBeenCalledTimes(1)
+ // Retirement during the initial GET prevents every mutation and the next apply.
+ expect(mockApi.readNamespacedNetworkPolicy).toHaveBeenCalledTimes(1)
+ expect(mockApi.createNamespacedNetworkPolicy).not.toHaveBeenCalled()
expect(mockApi.replaceNamespacedNetworkPolicy).not.toHaveBeenCalled()
})
@@ -211,7 +254,7 @@ describe('BindingPolicyReconciler', () => {
isCurrent: () => true,
})
- const calls = mockApi.createNamespacedNetworkPolicy.mock.calls
+ const calls = mockApi.replaceNamespacedNetworkPolicy.mock.calls
const ingressCall = calls.find(
(c: unknown[]) =>
(c[0] as { body: { spec: { policyTypes: string[] } } }).body.spec.policyTypes[0] ===
@@ -230,7 +273,7 @@ describe('BindingPolicyReconciler', () => {
isCurrent: () => true,
})
- const calls = mockApi.createNamespacedNetworkPolicy.mock.calls
+ const calls = mockApi.replaceNamespacedNetworkPolicy.mock.calls
const egressCall = calls.find(
(c: unknown[]) =>
(c[0] as { body: { spec: { policyTypes: string[] } } }).body.spec.policyTypes[0] ===
@@ -302,7 +345,7 @@ describe('BindingPolicyReconciler', () => {
{ isCurrent: () => true }
)
- const calls = mockApi.createNamespacedNetworkPolicy.mock.calls
+ const calls = mockApi.replaceNamespacedNetworkPolicy.mock.calls
const egressCall = calls.find(
(c: unknown[]) =>
(c[0] as { body: { spec: { policyTypes: string[] } } }).body.spec.policyTypes[0] ===
diff --git a/host-context-controller/src/createMetrics.test.ts b/host-context-controller/src/createMetrics.test.ts
index 1ac18eb60..ba8a5464a 100644
--- a/host-context-controller/src/createMetrics.test.ts
+++ b/host-context-controller/src/createMetrics.test.ts
@@ -214,7 +214,7 @@ describe('Kubernetes create instrumentation', () => {
expect(await count('Deployment', 'skipped')).toBe(0)
})
- it('observes the POST-first conflict path without replacing an unchanged policy', async () => {
+ it('counts a presence skip without POST or replacing an unchanged policy', async () => {
const policy: k8s.V1NetworkPolicy = {
kind: 'NetworkPolicy',
metadata: { name: 'metrics-policy' },
@@ -231,13 +231,13 @@ describe('Kubernetes create instrumentation', () => {
'test',
policy
)
- expect(api.createNamespacedNetworkPolicy).toHaveBeenCalledTimes(1)
+ expect(api.createNamespacedNetworkPolicy).not.toHaveBeenCalled()
expect(api.readNamespacedNetworkPolicy).toHaveBeenCalledTimes(1)
expect(api.replaceNamespacedNetworkPolicy).not.toHaveBeenCalled()
expect(await count('NetworkPolicy', 'created')).toBe(0)
expect(await count('NetworkPolicy', 'error')).toBe(0)
- expect(await count('NetworkPolicy', 'conflict')).toBe(1)
- expect(await count('NetworkPolicy', 'skipped')).toBe(0)
+ expect(await count('NetworkPolicy', 'conflict')).toBe(0)
+ expect(await count('NetworkPolicy', 'skipped')).toBe(1)
})
it('does not count a create when its mutation fence has already expired', async () => {
diff --git a/host-context-controller/src/ensureResource.test.ts b/host-context-controller/src/ensureResource.test.ts
new file mode 100644
index 000000000..5a6694267
--- /dev/null
+++ b/host-context-controller/src/ensureResource.test.ts
@@ -0,0 +1,238 @@
+import { describe, expect, it, vi } from 'vitest'
+import { ensureResource, replaceWithConflictRetry } from './utils'
+
+const resource = (rv = '1') => ({
+ kind: 'NetworkPolicy',
+ metadata: { name: 'observed-policy', resourceVersion: rv },
+})
+
+describe('ensureResource', () => {
+ it.each([{ code: 404 }, { response: { statusCode: 404 } }])(
+ 'creates only after an actual absence read (%j)',
+ async absent => {
+ const events: string[] = []
+ const read = vi.fn(async () => {
+ events.push('GET')
+ throw absent
+ })
+ const create = vi.fn(async () => {
+ events.push('POST')
+ return resource()
+ })
+ const converge = vi.fn()
+ const onSkipped = vi.fn()
+ await ensureResource({ read, create, converge, onSkipped })
+ expect(events).toEqual(['GET', 'POST'])
+ expect(create).toHaveBeenCalledTimes(1)
+ expect(converge).not.toHaveBeenCalled()
+ expect(onSkipped).not.toHaveBeenCalled()
+ }
+ )
+
+ it('consumes an observed object once, then reads fresh on a replace conflict', async () => {
+ const first = resource('1')
+ const second = resource('2')
+ const read = vi.fn().mockResolvedValueOnce(first).mockResolvedValueOnce(second)
+ const create = vi.fn()
+ const replace = vi.fn().mockRejectedValueOnce({ code: 409 }).mockResolvedValue(undefined)
+ const onSkipped = vi.fn()
+ const validated: unknown[] = []
+ await ensureResource({
+ read,
+ create,
+ onSkipped,
+ converge: observedRead =>
+ replaceWithConflictRetry({
+ description: 'policy',
+ logPrefix: '[Test]',
+ body: resource(),
+ read: observedRead,
+ replace,
+ validateExisting: value => {
+ validated.push(value)
+ },
+ }),
+ })
+ expect(read).toHaveBeenCalledTimes(2)
+ expect(validated).toEqual([first, second])
+ expect(replace.mock.calls.map(([body]) => body.metadata.resourceVersion)).toEqual(['1', '2'])
+ expect(create).not.toHaveBeenCalled()
+ expect(onSkipped).toHaveBeenCalledTimes(1)
+ })
+
+ it('uses a supplied snapshot without an API read or a create', async () => {
+ const existing = resource()
+ const read = vi.fn()
+ const create = vi.fn()
+ const onSkipped = vi.fn()
+ const converge = vi.fn(async (readOnce, observed) => {
+ expect(observed).toBe(existing)
+ expect(await readOnce()).toBe(existing)
+ })
+ await ensureResource({ existing, read, create, converge, onSkipped })
+ expect(converge).toHaveBeenCalledTimes(1)
+ expect(read).not.toHaveBeenCalled()
+ expect(create).not.toHaveBeenCalled()
+ expect(onSkipped).toHaveBeenCalledTimes(1)
+ })
+
+ it('discards observed absence after POST409 and retains the retry budget', async () => {
+ const read = vi.fn().mockResolvedValueOnce(resource('2')).mockResolvedValueOnce(resource('3'))
+ const create = vi.fn().mockRejectedValue({ code: 409 })
+ const lastConflict = { code: 409, message: 'conflict remains' }
+ const replace = vi.fn().mockRejectedValue(lastConflict)
+ const onSkipped = vi.fn()
+ await expect(
+ ensureResource({
+ existing: null,
+ read,
+ create,
+ onSkipped,
+ converge: observedRead =>
+ replaceWithConflictRetry({
+ description: 'policy',
+ logPrefix: '[Test]',
+ body: resource(),
+ read: observedRead,
+ replace,
+ maxAttempts: 2,
+ }),
+ })
+ ).rejects.toBe(lastConflict)
+ expect(create).toHaveBeenCalledTimes(1)
+ expect(read).toHaveBeenCalledTimes(2)
+ expect(replace.mock.calls.map(([body]) => body.metadata.resourceVersion)).toEqual(['2', '3'])
+ expect(onSkipped).not.toHaveBeenCalled()
+ })
+
+ it.each([403, 500, undefined])('propagates initial read failures (%s)', async code => {
+ const error = { code, message: 'read failed' }
+ const read = vi.fn().mockRejectedValue(error)
+ const create = vi.fn()
+ const converge = vi.fn()
+ const onSkipped = vi.fn()
+ await expect(ensureResource({ read, create, converge, onSkipped })).rejects.toBe(error)
+ expect(read).toHaveBeenCalledTimes(1)
+ expect(create).not.toHaveBeenCalled()
+ expect(converge).not.toHaveBeenCalled()
+ expect(onSkipped).not.toHaveBeenCalled()
+ })
+
+ it.each([undefined, null, {}, { metadata: {} }, { metadata: { name: '' } }])(
+ 'rejects a resolved invalid read without interpreting it as absence (%j)',
+ async value => {
+ const read = vi.fn().mockResolvedValue(value)
+ const create = vi.fn()
+ const onSkipped = vi.fn()
+ await expect(ensureResource({ read, create, converge: vi.fn(), onSkipped })).rejects.toThrow(
+ 'metadata.name'
+ )
+ expect(read).toHaveBeenCalledTimes(1)
+ expect(create).not.toHaveBeenCalled()
+ expect(onSkipped).not.toHaveBeenCalled()
+ }
+ )
+
+ it('rechecks the fence after a GET404 and never counts cancellation as a skip', async () => {
+ let current = true
+ const read = vi.fn(async () => {
+ current = false
+ throw { code: 404 }
+ })
+ const create = vi.fn()
+ const onSkipped = vi.fn()
+ await ensureResource({
+ read,
+ create,
+ converge: vi.fn(),
+ onSkipped,
+ mutationAllowed: () => current,
+ })
+ expect(read).toHaveBeenCalledTimes(1)
+ expect(create).not.toHaveBeenCalled()
+ expect(onSkipped).not.toHaveBeenCalled()
+ })
+
+ it('retains the initial fence, with a live allowed invocation as control', async () => {
+ const read = vi.fn().mockResolvedValue(resource())
+ const create = vi.fn()
+ const converge = vi.fn().mockResolvedValue(undefined)
+ const onSkipped = vi.fn()
+ await ensureResource({ read, create, converge, onSkipped, mutationAllowed: () => false })
+ expect(read).not.toHaveBeenCalled()
+ await ensureResource({ read, create, converge, onSkipped, mutationAllowed: () => true })
+ expect(read).toHaveBeenCalledTimes(1)
+ expect(converge).toHaveBeenCalledTimes(1)
+ expect(onSkipped).toHaveBeenCalledTimes(1)
+ })
+
+ it('keeps missingIsError scoped to disappearance during convergence', async () => {
+ const read = vi.fn().mockRejectedValue({ code: 404 })
+ const create = vi.fn().mockRejectedValue({ code: 409 })
+ const replace = vi.fn()
+ const onSkipped = vi.fn()
+ const converge =
+ (missingIsError: boolean) => (readFresh: () => Promise<ReturnType<typeof resource>>) =>
+ replaceWithConflictRetry({
+ description: 'policy',
+ logPrefix: '[Test]',
+ body: resource(),
+ read: readFresh,
+ replace,
+ missingIsError,
+ })
+ await ensureResource({ read, create, converge: converge(false), onSkipped })
+ await expect(
+ ensureResource({ read, create, converge: converge(true), onSkipped })
+ ).rejects.toEqual({ code: 404 })
+ expect(create).toHaveBeenCalledTimes(2)
+ expect(read).toHaveBeenCalledTimes(4)
+ expect(replace).not.toHaveBeenCalled()
+ expect(onSkipped).not.toHaveBeenCalled()
+ })
+
+ it('does not count a rejected ownership check or retained failure as skipped', async () => {
+ const read = vi.fn().mockResolvedValue(resource())
+ const create = vi.fn()
+ const onSkipped = vi.fn()
+ const error = new Error('foreign ownership')
+ const converge = vi.fn().mockRejectedValueOnce(error).mockResolvedValueOnce(false)
+ await expect(ensureResource({ read, create, converge, onSkipped })).rejects.toBe(error)
+ await ensureResource({ read, create, converge, onSkipped })
+ expect(converge).toHaveBeenCalledTimes(2)
+ expect(read).toHaveBeenCalledTimes(2)
+ expect(create).not.toHaveBeenCalled()
+ expect(onSkipped).not.toHaveBeenCalled()
+ })
+
+ it('does not count convergence cancelled during async body reconstruction', async () => {
+ let current = true
+ const read = vi.fn().mockResolvedValue(resource())
+ const replace = vi.fn()
+ const create = vi.fn()
+ const onSkipped = vi.fn()
+ await ensureResource({
+ read,
+ create,
+ onSkipped,
+ mutationAllowed: () => current,
+ converge: readOnce =>
+ replaceWithConflictRetry({
+ description: 'policy',
+ logPrefix: '[Test]',
+ body: resource(),
+ read: readOnce,
+ replace,
+ resolveBody: async () => {
+ current = false
+ return resource()
+ },
+ mutationAllowed: () => current,
+ }),
+ })
+ expect(read).toHaveBeenCalledTimes(1)
+ expect(replace).not.toHaveBeenCalled()
+ expect(create).not.toHaveBeenCalled()
+ expect(onSkipped).not.toHaveBeenCalled()
+ })
+})
diff --git a/host-context-controller/src/existenceReadMetrics.test.ts b/host-context-controller/src/existenceReadMetrics.test.ts
index 808a99748..409fdeb04 100644
--- a/host-context-controller/src/existenceReadMetrics.test.ts
+++ b/host-context-controller/src/existenceReadMetrics.test.ts
@@ -357,10 +357,10 @@ describe('Kubernetes existence-read instrumentation', () => {
spec: { podSelector: {}, policyTypes: ['Ingress'], ingress: [] },
}
- it('preserves POST-first success without adding an existence request', async () => {
+ it('observes GET404 before POST for an absent policy', async () => {
const api = {
createNamespacedNetworkPolicy: vi.fn().mockResolvedValue(policy),
- readNamespacedNetworkPolicy: vi.fn(),
+ readNamespacedNetworkPolicy: vi.fn().mockRejectedValue({ code: 404 }),
}
await applyNetworkPolicy(
api as unknown as k8s.NetworkingV1Api,
@@ -372,7 +372,11 @@ describe('Kubernetes existence-read instrumentation', () => {
namespace: 'test',
body: policy,
})
- expect(api.readNamespacedNetworkPolicy).not.toHaveBeenCalled()
+ expect(api.readNamespacedNetworkPolicy).toHaveBeenCalledExactlyOnceWith({
+ name: 'read-metrics-policy',
+ namespace: 'test',
+ })
+ expect(await count('NetworkPolicy', 'absent')).toBe(1)
expect(await count('NetworkPolicy', 'found')).toBe(0)
})
@@ -391,7 +395,7 @@ describe('Kubernetes existence-read instrumentation', () => {
policy
)
).rejects.toBe(error)
- expect(api.createNamespacedNetworkPolicy).toHaveBeenCalledTimes(1)
+ expect(api.createNamespacedNetworkPolicy).not.toHaveBeenCalled()
expect(api.readNamespacedNetworkPolicy).toHaveBeenCalledExactlyOnceWith({
name: 'read-metrics-policy',
namespace: 'test',
@@ -403,7 +407,7 @@ describe('Kubernetes existence-read instrumentation', () => {
)
expect(samples.map(sample => [sample.labels.outcome, sample.value])).toEqual([
['created', 0],
- ['conflict', 1],
+ ['conflict', 0],
['error', 0],
['skipped', 0],
])
@@ -437,7 +441,7 @@ describe('Kubernetes existence-read instrumentation', () => {
'test',
policy
)
- expect(events).toEqual(['POST', 'GET', 'PUT:1', 'GET', 'PUT:2'])
+ expect(events).toEqual(['GET', 'PUT:1', 'GET', 'PUT:2'])
expect(api.readNamespacedNetworkPolicy).toHaveBeenCalledTimes(2)
expect(api.replaceNamespacedNetworkPolicy).toHaveBeenCalledTimes(2)
expect(await count('NetworkPolicy', 'found')).toBe(2)
@@ -445,7 +449,12 @@ describe('Kubernetes existence-read instrumentation', () => {
const conflicts = (await createsTotal.get()).values.find(
sample => sample.labels.kind === 'NetworkPolicy' && sample.labels.outcome === 'conflict'
)
- expect(conflicts?.value).toBe(1)
+ expect(conflicts?.value).toBe(0)
+ expect(api.createNamespacedNetworkPolicy).not.toHaveBeenCalled()
+ const skipped = (await createsTotal.get()).values.find(
+ sample => sample.labels.kind === 'NetworkPolicy' && sample.labels.outcome === 'skipped'
+ )
+ expect(skipped?.value).toBe(1)
})
it('observes a read that expires the existing fence and prevents replace', async () => {
@@ -468,7 +477,7 @@ describe('Kubernetes existence-read instrumentation', () => {
allowed
)
expect(allowed.mock.results.map(result => result.value)).toEqual([true, false])
- expect(api.createNamespacedNetworkPolicy).toHaveBeenCalledTimes(1)
+ expect(api.createNamespacedNetworkPolicy).not.toHaveBeenCalled()
expect(api.readNamespacedNetworkPolicy).toHaveBeenCalledTimes(1)
expect(await count('NetworkPolicy', 'found')).toBe(1)
expect(api.replaceNamespacedNetworkPolicy).not.toHaveBeenCalled()
diff --git a/host-context-controller/src/hostReconciler.codexScopeProvenance.test.ts b/host-context-controller/src/hostReconciler.codexScopeProvenance.test.ts
index fde11b049..cde5f5035 100644
--- a/host-context-controller/src/hostReconciler.codexScopeProvenance.test.ts
+++ b/host-context-controller/src/hostReconciler.codexScopeProvenance.test.ts
@@ -188,7 +188,7 @@ function scopeHashFromSecretWrites(coreApi: ReturnType<typeof createMockCoreApi>
}
function proxyPolicyBodies(networkingApi: ReturnType<typeof createMockNetworkingApi>) {
- return networkingApi.createNamespacedNetworkPolicy.mock.calls
+ return networkingApi.replaceNamespacedNetworkPolicy.mock.calls
.map(call => call[0]?.body)
.filter(body => body?.metadata?.name === CODEX_PROXY_POLICY)
}
diff --git a/host-context-controller/src/llmHookReconciler.test.ts b/host-context-controller/src/llmHookReconciler.test.ts
index 09fd93068..7f9c673ee 100644
--- a/host-context-controller/src/llmHookReconciler.test.ts
+++ b/host-context-controller/src/llmHookReconciler.test.ts
@@ -494,6 +494,8 @@ describe('LlmHookReconciler', () => {
// ─── NetworkPolicy ingress reverse-index (§10) ─────────────────────
it('NetworkPolicy ingress admits exactly the referencing Hosts', async () => {
+ // This scenario starts without a policy; the API reports absence explicitly.
+ networkingApi.readNamespacedNetworkPolicy.mockRejectedValue({ code: 404 })
const a = makeHook({
name: 'a',
spec: { target: { image: { ref: IMG, port: 8080 } }, lifecyclePoints: ['preCall'] },
@@ -517,6 +519,8 @@ describe('LlmHookReconciler', () => {
})
it('co-located members: NetworkPolicy label values are comma-free (member list is an annotation)', async () => {
+ // This scenario starts without a policy; the API reports absence explicitly.
+ networkingApi.readNamespacedNetworkPolicy.mockRejectedValue({ code: 404 })
// Two hooks with the SAME image target share one pod key → one shared NP.
const a = makeHook({
name: 'a',
@@ -551,6 +555,8 @@ describe('LlmHookReconciler', () => {
})
it('re-reconciles NetworkPolicy ingress on a Host reference change (fan-out)', async () => {
+ // This scenario starts without a policy; the API reports absence explicitly.
+ networkingApi.readNamespacedNetworkPolicy.mockRejectedValue({ code: 404 })
const a = makeHook({
name: 'a',
spec: { target: { image: { ref: IMG, port: 8080 } }, lifecyclePoints: ['preCall'] },
@@ -594,6 +600,8 @@ describe('LlmHookReconciler', () => {
}
it('creates an ingress NP selecting the Service pods and admitting only referencing hosts', async () => {
+ // This scenario starts without a policy; the API reports absence explicitly.
+ networkingApi.readNamespacedNetworkPolicy.mockRejectedValue({ code: 404 })
withSelector({ app: 'ref-hook' })
const s = makeServiceHook('s1')
hooks.set('s1', s)
@@ -619,6 +627,8 @@ describe('LlmHookReconciler', () => {
})
it('creates a deny-all (empty ingress) NP when no Host references the hook', async () => {
+ // This scenario starts without a policy; the API reports absence explicitly.
+ networkingApi.readNamespacedNetworkPolicy.mockRejectedValue({ code: 404 })
withSelector({ app: 'ref-hook' })
const s = makeServiceHook('s1')
hooks.set('s1', s)
@@ -646,6 +656,8 @@ describe('LlmHookReconciler', () => {
})
it('refreshes the admitted hosts on a Host reference change (fan-out)', async () => {
+ // This scenario starts without a policy; the API reports absence explicitly.
+ networkingApi.readNamespacedNetworkPolicy.mockRejectedValue({ code: 404 })
withSelector({ app: 'ref-hook' })
const s = makeServiceHook('s1')
hooks.set('s1', s)
@@ -675,6 +687,8 @@ describe('LlmHookReconciler', () => {
describe('per-host egress (reconcileHostEgress)', () => {
it('allows egress only to the referenced image hook pod-key + port', async () => {
+ // This scenario starts without a policy; the API reports absence explicitly.
+ networkingApi.readNamespacedNetworkPolicy.mockRejectedValue({ code: 404 })
const hook = makeHook({ name: 'img1' }) // default image target, port 8080
hooks.set('img1', hook)
const host = makeHostRef('host-1', ['img1'])
@@ -705,6 +719,8 @@ describe('LlmHookReconciler', () => {
})
it('uses the Service selector + port for a service-target hook', async () => {
+ // This scenario starts without a policy; the API reports absence explicitly.
+ networkingApi.readNamespacedNetworkPolicy.mockRejectedValue({ code: 404 })
coreApi.readNamespacedService.mockResolvedValue({
metadata: {},
spec: { selector: { app: 'svc-hook' } },
@@ -782,6 +798,8 @@ describe('LlmHookReconciler', () => {
})
it('grants scoped CoreDNS egress only when a hook declares egressBindings (N5)', async () => {
+ // This scenario starts without a policy; the API reports absence explicitly.
+ networkingApi.readNamespacedNetworkPolicy.mockRejectedValue({ code: 404 })
const withEgress = makeHook({
name: 'out',
spec: {
@@ -817,6 +835,8 @@ describe('LlmHookReconciler', () => {
// "grants NO egress (not even DNS)" is what let namespace-wide DNS through
// while the suite stayed green.
it('declares no Egress in the per-pod-key policy for a pure responder (no egressBindings)', async () => {
+ // This scenario starts without a policy; the API reports absence explicitly.
+ networkingApi.readNamespacedNetworkPolicy.mockRejectedValue({ code: 404 })
const responder = makeHook({ name: 'resp' }) // default image, no egressBindings
hooks.set('resp', responder)
await reconciler.reconcile(responder)
@@ -829,6 +849,8 @@ describe('LlmHookReconciler', () => {
})
it('allows a valid public-CIDR egress binding as an Egress NetworkPolicy rule', async () => {
+ // This scenario starts without a policy; the API reports absence explicitly.
+ networkingApi.readNamespacedNetworkPolicy.mockRejectedValue({ code: 404 })
const a = makeHook({
name: 'a',
spec: {
diff --git a/host-context-controller/src/llmHookReconciler.ts b/host-context-controller/src/llmHookReconciler.ts
index e70e28e07..54e4ceb78 100644
--- a/host-context-controller/src/llmHookReconciler.ts
+++ b/host-context-controller/src/llmHookReconciler.ts
@@ -35,10 +35,13 @@ import {
MANAGED_BY_VALUE,
POLICY_TYPE_LABEL,
} from './constants'
+import { hccLogger } from './logger'
+import { createsTotal } from './metrics'
import { isAllowedExternalEgressCidr, isPublicDnsHostname } from './networkPolicyReconciler'
import { HostCRD, LlmHookCRD, LlmHookCondition, LlmHookImageTarget, LlmHookStatus } from './types'
import {
deploymentMatchesDesired,
+ ensureResource,
getErrorCode,
networkPolicyMatchesDesired,
observeCreate,
@@ -825,30 +828,33 @@ export class LlmHookReconciler {
await this.applyNetworkPolicy(this.buildNetworkPolicy(podKey, members, egressRules))
}
- /** Idempotent create-then-409-replace of a NetworkPolicy in its own namespace. */
+ /** Read first, preserving the existing NetworkPolicy convergence policy. */
private async applyNetworkPolicy(policy: k8s.V1NetworkPolicy): Promise<void> {
const name = policy.metadata!.name!
const namespace = policy.metadata!.namespace ?? config.llmHooksNamespace
- try {
- await observeCreate('NetworkPolicy', () =>
- this.networkingApi.createNamespacedNetworkPolicy({ namespace, body: policy })
- )
- console.log(`${LOG} Created NetworkPolicy "${name}" (${namespace})`)
- return
- } catch (error) {
- if (getErrorCode(error) !== 409) throw error
- }
- await replaceWithConflictRetry({
- description: `NetworkPolicy "${name}"`,
- logPrefix: LOG,
- body: policy,
- mergeExisting: preserveObjectAnnotations,
- isUpToDate: networkPolicyMatchesDesired,
+ await ensureResource({
read: () =>
observeExistenceRead('NetworkPolicy', () =>
this.networkingApi.readNamespacedNetworkPolicy({ name, namespace })
),
- replace: body => this.networkingApi.replaceNamespacedNetworkPolicy({ name, namespace, body }),
+ create: async () => {
+ await observeCreate('NetworkPolicy', () =>
+ this.networkingApi.createNamespacedNetworkPolicy({ namespace, body: policy })
+ )
+ hccLogger.info('NetworkPolicy created', { scope: LOG, policy: name, namespace })
+ },
+ converge: read =>
+ replaceWithConflictRetry({
+ description: `NetworkPolicy "${name}"`,
+ logPrefix: LOG,
+ body: policy,
+ mergeExisting: preserveObjectAnnotations,
+ isUpToDate: networkPolicyMatchesDesired,
+ read,
+ replace: body =>
+ this.networkingApi.replaceNamespacedNetworkPolicy({ name, namespace, body }),
+ }),
+ onSkipped: () => createsTotal.inc({ kind: 'NetworkPolicy', outcome: 'skipped' }),
})
}
diff --git a/host-context-controller/src/networkPolicyReconciler.egressObservation.test.ts b/host-context-controller/src/networkPolicyReconciler.egressObservation.test.ts
index dc1305840..9af1bef22 100644
--- a/host-context-controller/src/networkPolicyReconciler.egressObservation.test.ts
+++ b/host-context-controller/src/networkPolicyReconciler.egressObservation.test.ts
@@ -316,7 +316,7 @@ describe('external egress observation failures do not decide authorization', ()
expect(f.status()).toEqual(statusBefore)
})
- it('does not copy a newer resourceVersion onto a policy calculated before an apply conflict', async () => {
+ it('does not copy a newer resourceVersion onto a policy calculated before a replace conflict', async () => {
const f = fixture([currentDns])
await f.run()
const policy = f.policies.get(currentName)!
@@ -325,8 +325,8 @@ describe('external egress observation failures do not decide authorization', ()
ports: [{ port: 443, protocol: 'TCP' }],
})
f.clearCalls()
- f.api.createNamespacedNetworkPolicy.mockImplementationOnce(async () => {
- // The conflict retry reads this newer object after the final plan check.
+ f.api.replaceNamespacedNetworkPolicy.mockImplementationOnce(async () => {
+ // The first PUT races with a newer resourceVersion after the final plan check.
// A matching owner alone cannot authorize reuse of the older DNS state.
policy.metadata!.resourceVersion = 'changed-before-replace'
throw apiError(409)
@@ -334,20 +334,21 @@ describe('external egress observation failures do not decide authorization', ()
await expect(f.run()).rejects.toThrow(/changed after the external-egress observation/)
- expect(f.api.replaceNamespacedNetworkPolicy).not.toHaveBeenCalled()
+ expect(f.api.replaceNamespacedNetworkPolicy).toHaveBeenCalledTimes(1)
+ expect(f.api.createNamespacedNetworkPolicy).not.toHaveBeenCalled()
expect(f.condition()?.status).toBe('False')
await f.run()
expect(f.condition()?.status).toBe('True')
expect(f.policies.get(currentName)?.spec?.egress).toHaveLength(1)
})
- it('does not certify a policy deleted after create conflict and before replacement read, then recovers', async () => {
+ it('does not certify a policy deleted during PUT409 and before the retry read, then recovers', async () => {
const f = fixture([currentDns])
await f.run()
vi.mocked(dns.resolve4).mockResolvedValue([{ address: '8.8.8.8', ttl: 300 }] as never)
- f.api.createNamespacedNetworkPolicy.mockImplementationOnce(async () => {
- // POST sees the existing policy and conflicts. A concurrent deletion then
- // wins before the helper's replacement GET, which observes real absence.
+ f.api.replaceNamespacedNetworkPolicy.mockImplementationOnce(async () => {
+ // A concurrent deletion wins during the first PUT. The retry GET must
+ // observe absence and fail admission, rather than report success.
f.policies.delete(currentName)
throw apiError(409)
})
diff --git a/host-context-controller/src/networkPolicyReconciler.test.ts b/host-context-controller/src/networkPolicyReconciler.test.ts
index 50c58d777..3dc6a0ea8 100644
--- a/host-context-controller/src/networkPolicyReconciler.test.ts
+++ b/host-context-controller/src/networkPolicyReconciler.test.ts
@@ -478,6 +478,8 @@ describe('NetworkPolicyReconciler', () => {
describe('L0 — ensureDefaultDeny', () => {
it('creates deny-all policies in every runtime namespace', async () => {
+ // Bootstrap this scenario without policies; later LIST fixtures supply existing objects.
+ mockApi.readNamespacedNetworkPolicy.mockRejectedValue({ code: 404 })
await reconciler.ensureDefaultPolicies()
const denyPolicies = mockApi.createNamespacedNetworkPolicy.mock.calls.filter(
@@ -490,6 +492,8 @@ describe('NetworkPolicyReconciler', () => {
})
it('includes both Ingress and Egress in policyTypes', async () => {
+ // Bootstrap this scenario without policies; later LIST fixtures supply existing objects.
+ mockApi.readNamespacedNetworkPolicy.mockRejectedValue({ code: 404 })
await reconciler.ensureDefaultPolicies()
const denyPolicy = mockApi.createNamespacedNetworkPolicy.mock.calls.find(
@@ -504,6 +508,8 @@ describe('NetworkPolicyReconciler', () => {
})
it('uses empty podSelector (selects all pods)', async () => {
+ // Bootstrap this scenario without policies; later LIST fixtures supply existing objects.
+ mockApi.readNamespacedNetworkPolicy.mockRejectedValue({ code: 404 })
await reconciler.ensureDefaultPolicies()
const denyPolicy = mockApi.createNamespacedNetworkPolicy.mock.calls.find(
@@ -528,21 +534,8 @@ describe('NetworkPolicyReconciler', () => {
])(
'refuses to adopt a foreign policy that collides with %s',
async (targetName, policyType) => {
- mockApi.createNamespacedNetworkPolicy.mockImplementation(
- async ({ body }: { body: k8s.V1NetworkPolicy }) => {
- if (body.metadata?.name === targetName) {
- throw Object.assign(new Error('already exists'), { code: 409 })
- }
- return {}
- }
- )
mockApi.readNamespacedNetworkPolicy.mockImplementation(async ({ name, namespace }) => {
- if (
- name === 'allow-desktop-egress-rpc-proxy' ||
- name === 'allow-rpc-proxy-to-managed-mcp-servers'
- ) {
- throw Object.assign(new Error('not found'), { code: 404 })
- }
+ if (name !== targetName) throw { code: 404 }
return {
metadata: {
name,
@@ -558,6 +551,9 @@ describe('NetworkPolicyReconciler', () => {
})
await expect(reconciler.ensureDefaultPolicies()).rejects.toThrow(/ownership/i)
+ expect(mockApi.readNamespacedNetworkPolicy).toHaveBeenCalledWith(
+ expect.objectContaining({ name: targetName })
+ )
expect(mockApi.replaceNamespacedNetworkPolicy).not.toHaveBeenCalledWith(
expect.objectContaining({ name: targetName })
)
@@ -565,6 +561,8 @@ describe('NetworkPolicyReconciler', () => {
)
it('creates DNS egress policies in every runtime namespace', async () => {
+ // Bootstrap this scenario without policies; later LIST fixtures supply existing objects.
+ mockApi.readNamespacedNetworkPolicy.mockRejectedValue({ code: 404 })
await reconciler.ensureDefaultPolicies()
const dnsPolicies = mockApi.createNamespacedNetworkPolicy.mock.calls.filter(
@@ -577,6 +575,8 @@ describe('NetworkPolicyReconciler', () => {
})
it('DNS egress targets kube-system on port 53 UDP+TCP', async () => {
+ // Bootstrap this scenario without policies; later LIST fixtures supply existing objects.
+ mockApi.readNamespacedNetworkPolicy.mockRejectedValue({ code: 404 })
await reconciler.ensureDefaultPolicies()
const dnsPolicy = mockApi.createNamespacedNetworkPolicy.mock.calls.find((call: unknown[]) => {
@@ -595,6 +595,8 @@ describe('NetworkPolicyReconciler', () => {
})
it('creates HCC API egress policies', async () => {
+ // Bootstrap this scenario without policies; later LIST fixtures supply existing objects.
+ mockApi.readNamespacedNetworkPolicy.mockRejectedValue({ code: 404 })
await reconciler.ensureDefaultPolicies()
const hccPolicies = mockApi.createNamespacedNetworkPolicy.mock.calls.filter(
@@ -631,6 +633,8 @@ describe('NetworkPolicyReconciler', () => {
})
it('creates K8s API egress policies', async () => {
+ // Bootstrap this scenario without policies; later LIST fixtures supply existing objects.
+ mockApi.readNamespacedNetworkPolicy.mockRejectedValue({ code: 404 })
await reconciler.ensureDefaultPolicies()
const k8sPolicies = mockApi.createNamespacedNetworkPolicy.mock.calls.filter(
@@ -643,6 +647,8 @@ describe('NetworkPolicyReconciler', () => {
})
it('scopes HCC API egress to the platform pods that need it', async () => {
+ // Bootstrap this scenario without policies; later LIST fixtures supply existing objects.
+ mockApi.readNamespacedNetworkPolicy.mockRejectedValue({ code: 404 })
await reconciler.ensureDefaultPolicies()
const policyByName = new Map(
@@ -666,6 +672,8 @@ describe('NetworkPolicyReconciler', () => {
})
it('routes HCC API egress policies to the control-plane gateway', async () => {
+ // Bootstrap this scenario without policies; later LIST fixtures supply existing objects.
+ mockApi.readNamespacedNetworkPolicy.mockRejectedValue({ code: 404 })
await reconciler.ensureDefaultPolicies()
const hccPolicies = mockApi.createNamespacedNetworkPolicy.mock.calls
@@ -686,6 +694,8 @@ describe('NetworkPolicyReconciler', () => {
})
it('does not grant Kubernetes API egress to runtime workload namespaces by default', async () => {
+ // Bootstrap this scenario without policies; later LIST fixtures supply existing objects.
+ mockApi.readNamespacedNetworkPolicy.mockRejectedValue({ code: 404 })
await reconciler.ensureDefaultPolicies()
const policyByName = new Map(
@@ -709,6 +719,8 @@ describe('NetworkPolicyReconciler', () => {
})
it('infrastructure policies have correct policy-type label', async () => {
+ // Bootstrap this scenario without policies; later LIST fixtures supply existing objects.
+ mockApi.readNamespacedNetworkPolicy.mockRejectedValue({ code: 404 })
await reconciler.ensureDefaultPolicies()
const infraPolicies = mockApi.createNamespacedNetworkPolicy.mock.calls.filter(
@@ -722,17 +734,25 @@ describe('NetworkPolicyReconciler', () => {
})
it('removes legacy static policies now replaced by generated scoped policies', async () => {
- mockApi.readNamespacedNetworkPolicy.mockImplementation(async ({ name, namespace }) => ({
- metadata: {
- name,
- namespace,
- uid: `${name}-uid`,
- resourceVersion: '7',
- labels: {
- 'clerum.io/managed-by': 'host-context-controller',
+ mockApi.readNamespacedNetworkPolicy.mockImplementation(async ({ name, namespace }) => {
+ if (
+ !['allow-desktop-egress-rpc-proxy', 'allow-rpc-proxy-to-managed-mcp-servers'].includes(
+ name
+ )
+ )
+ throw { code: 404 }
+ return {
+ metadata: {
+ name,
+ namespace,
+ uid: `${name}-uid`,
+ resourceVersion: '7',
+ labels: {
+ 'clerum.io/managed-by': 'host-context-controller',
+ },
},
- },
- }))
+ }
+ })
await reconciler.ensureDefaultPolicies()
@@ -830,18 +850,21 @@ describe('NetworkPolicyReconciler', () => {
return {}
}
)
- mockApi.readNamespacedNetworkPolicy.mockImplementation(async ({ name, namespace }) => ({
- metadata: {
- name,
- namespace,
- uid: 'existing-default-policy-uid',
- resourceVersion: '7',
- labels: {
- 'clerum.io/managed-by': 'host-context-controller',
- 'clerum.io/policy-type': 'allow-api',
+ mockApi.readNamespacedNetworkPolicy.mockImplementation(async ({ name, namespace }) => {
+ if (name !== 'allow-host-context-controller-api') throw { code: 404 }
+ return {
+ metadata: {
+ name,
+ namespace,
+ uid: 'existing-default-policy-uid',
+ resourceVersion: '7',
+ labels: {
+ 'clerum.io/managed-by': 'host-context-controller',
+ 'clerum.io/policy-type': 'allow-api',
+ },
},
- },
- }))
+ }
+ })
await expect(reconciler.ensureDefaultPolicies()).resolves.toBeUndefined()
expect(mockApi.replaceNamespacedNetworkPolicy).toHaveBeenCalledWith(
@@ -861,17 +884,20 @@ describe('NetworkPolicyReconciler', () => {
return {}
}
)
- mockApi.readNamespacedNetworkPolicy.mockResolvedValue({
- metadata: {
- name: 'allow-host-context-controller-api',
- namespace: 'mcp-server',
- uid: 'foreign-default-policy-uid',
- resourceVersion: '7',
- labels: {
- 'clerum.io/managed-by': 'foreign-controller',
- 'clerum.io/policy-type': 'allow-api',
+ mockApi.readNamespacedNetworkPolicy.mockImplementation(async ({ name }) => {
+ if (name !== 'allow-host-context-controller-api') throw { code: 404 }
+ return {
+ metadata: {
+ name: 'allow-host-context-controller-api',
+ namespace: 'mcp-server',
+ uid: 'foreign-default-policy-uid',
+ resourceVersion: '7',
+ labels: {
+ 'clerum.io/managed-by': 'foreign-controller',
+ 'clerum.io/policy-type': 'allow-api',
+ },
},
- },
+ }
})
await expect(reconciler.ensureDefaultPolicies()).rejects.toThrow(/ownership/i)
@@ -887,7 +913,8 @@ describe('NetworkPolicyReconciler', () => {
return {}
}
)
- mockApi.readNamespacedNetworkPolicy
+ const readAllowApi = vi
+ .fn()
.mockResolvedValueOnce({
metadata: {
name: 'allow-host-context-controller-api',
@@ -912,6 +939,10 @@ describe('NetworkPolicyReconciler', () => {
},
},
})
+ mockApi.readNamespacedNetworkPolicy.mockImplementation(async ({ name }) => {
+ if (name !== 'allow-host-context-controller-api') throw { code: 404 }
+ return readAllowApi()
+ })
mockApi.replaceNamespacedNetworkPolicy.mockRejectedValueOnce(
Object.assign(new Error('conflict'), { code: 409 })
)
@@ -921,6 +952,8 @@ describe('NetworkPolicyReconciler', () => {
})
it('creates allow-api ingress policy using _from (K8s client convention)', async () => {
+ // Bootstrap this scenario without policies; later LIST fixtures supply existing objects.
+ mockApi.readNamespacedNetworkPolicy.mockRejectedValue({ code: 404 })
await reconciler.ensureDefaultPolicies()
const apiPolicy = mockApi.createNamespacedNetworkPolicy.mock.calls.find((call: unknown[]) => {
@@ -1020,6 +1053,8 @@ describe('NetworkPolicyReconciler', () => {
})
it('creates context-allow ingress policy per server in mcp-server namespace', async () => {
+ // Bootstrap this scenario without policies; later LIST fixtures supply existing objects.
+ mockApi.readNamespacedNetworkPolicy.mockRejectedValue({ code: 404 })
const cache = new Map<string, McpServerCRD>()
cache.set('mongo', {
name: 'mongo',
@@ -1081,6 +1116,8 @@ describe('NetworkPolicyReconciler', () => {
})
it('retains the ingress policy it just replaced when the authoritative server port changes', async () => {
+ // Bootstrap this scenario without policies; later LIST fixtures supply existing objects.
+ mockApi.readNamespacedNetworkPolicy.mockRejectedValue({ code: 404 })
const oldServer: McpServerCRD = {
name: 'mongo',
namespace: 'mcp-server',
@@ -1156,6 +1193,8 @@ describe('NetworkPolicyReconciler', () => {
})
it('stops remaining policy mutations after its combined authority lease is retired', async () => {
+ // Bootstrap this scenario without policies; later LIST fixtures supply existing objects.
+ mockApi.readNamespacedNetworkPolicy.mockRejectedValue({ code: 404 })
const cache = new Map<string, McpServerCRD>()
cache.set('mongo', {
name: 'mongo',
@@ -1202,6 +1241,8 @@ describe('NetworkPolicyReconciler', () => {
})
it('does not mutate a Context policy from a mixed McpServer cache revision', async () => {
+ // Bootstrap this scenario without policies; later LIST fixtures supply existing objects.
+ mockApi.readNamespacedNetworkPolicy.mockRejectedValue({ code: 404 })
const cache = new Map<string, McpServerCRD>()
cache.set('mongo', {
name: 'mongo',
@@ -1250,6 +1291,8 @@ describe('NetworkPolicyReconciler', () => {
})
it('continues a Context policy pass across a status-only McpServer replacement', async () => {
+ // Bootstrap this scenario without policies; later LIST fixtures supply existing objects.
+ mockApi.readNamespacedNetworkPolicy.mockRejectedValue({ code: 404 })
const cache = new Map<string, McpServerCRD>()
const selected: McpServerCRD = {
name: 'mongo',
@@ -1327,6 +1370,8 @@ describe('NetworkPolicyReconciler', () => {
})
it('creates L2 egress counterpart in mcp-host namespace (bidirectional)', async () => {
+ // Bootstrap this scenario without policies; later LIST fixtures supply existing objects.
+ mockApi.readNamespacedNetworkPolicy.mockRejectedValue({ code: 404 })
const cache = new Map<string, McpServerCRD>()
cache.set('mongo', {
name: 'mongo',
@@ -1384,6 +1429,8 @@ describe('NetworkPolicyReconciler', () => {
})
it('scopes L2 mcp-host egress by Context CRD name, not namespace-wide selector', async () => {
+ // Bootstrap this scenario without policies; later LIST fixtures supply existing objects.
+ mockApi.readNamespacedNetworkPolicy.mockRejectedValue({ code: 404 })
const cache = new Map<string, McpServerCRD>()
cache.set('mongo', {
name: 'mongo',
@@ -1418,6 +1465,8 @@ describe('NetworkPolicyReconciler', () => {
})
it('creates L2 rpc-proxy egress to one MCP server instead of relying on static namespace egress', async () => {
+ // Bootstrap this scenario without policies; later LIST fixtures supply existing objects.
+ mockApi.readNamespacedNetworkPolicy.mockRejectedValue({ code: 404 })
const cache = new Map<string, McpServerCRD>()
cache.set('mongo', {
name: 'mongo',
@@ -3205,6 +3254,8 @@ describe('NetworkPolicyReconciler', () => {
})
it('repairs one-marker live widened policies in every lane before certification', async () => {
+ // Bootstrap this scenario without policies; later LIST fixtures supply existing objects.
+ mockApi.readNamespacedNetworkPolicy.mockRejectedValue({ code: 404 })
const server: McpServerCRD = {
name: 'live-server',
namespace: 'mcp-server',
@@ -4030,6 +4081,8 @@ describe('NetworkPolicyReconciler', () => {
})
it('replaces same-name Context allow policies in every lane before certification', async () => {
+ // Bootstrap this scenario without policies; later LIST fixtures supply existing objects.
+ mockApi.readNamespacedNetworkPolicy.mockRejectedValue({ code: 404 })
const oldServer: McpServerCRD = {
name: 'live-server',
namespace: 'mcp-server',
@@ -4108,6 +4161,8 @@ describe('NetworkPolicyReconciler', () => {
])(
'refuses a same-name %s safety replacement owned by another server',
async (_lane, targetNamespace) => {
+ // Create the original owned policies before replacing the listed owner.
+ mockApi.readNamespacedNetworkPolicy.mockRejectedValue({ code: 404 })
const oldServer: McpServerCRD = {
name: 'live-server',
namespace: 'mcp-server',
@@ -4177,6 +4232,8 @@ describe('NetworkPolicyReconciler', () => {
)
it('fails the safety pass without certifying when a same-name Context replacement fails', async () => {
+ // Bootstrap this scenario without policies; later LIST fixtures supply existing objects.
+ mockApi.readNamespacedNetworkPolicy.mockRejectedValue({ code: 404 })
const oldServer: McpServerCRD = {
name: 'live-server',
namespace: 'mcp-server',
@@ -4242,6 +4299,8 @@ describe('NetworkPolicyReconciler', () => {
})
it('retains authoritative same-name policies when only API object key order differs', async () => {
+ // Bootstrap this scenario without policies; later LIST fixtures supply existing objects.
+ mockApi.readNamespacedNetworkPolicy.mockRejectedValue({ code: 404 })
const server: McpServerCRD = {
name: 'live-server',
namespace: 'mcp-server',
@@ -5169,6 +5228,8 @@ describe('NetworkPolicyReconciler', () => {
})
it('leaves startup external egress creation to its dedicated coordinator', async () => {
+ // Bootstrap this scenario without policies; later LIST fixtures supply existing objects.
+ mockApi.readNamespacedNetworkPolicy.mockRejectedValue({ code: 404 })
const server: McpServerCRD = {
name: 'airtable-mcp',
namespace: 'mcp-server',
@@ -5193,6 +5254,8 @@ describe('NetworkPolicyReconciler', () => {
})
it('deletes orphaned external egress policies for servers no longer present', async () => {
+ // Bootstrap this scenario without policies; later LIST fixtures supply existing objects.
+ mockApi.readNamespacedNetworkPolicy.mockRejectedValue({ code: 404 })
mockApi.listNamespacedNetworkPolicy.mockImplementation(
async ({ labelSelector }: { labelSelector?: string }) => {
if (labelSelector?.includes('context-allow')) return { items: [] }
@@ -5224,6 +5287,8 @@ describe('NetworkPolicyReconciler', () => {
})
it('fails closed by deleting HCC-managed policies with missing owner labels', async () => {
+ // Bootstrap this scenario without policies; later LIST fixtures supply existing objects.
+ mockApi.readNamespacedNetworkPolicy.mockRejectedValue({ code: 404 })
mockApi.listNamespacedNetworkPolicy.mockImplementation(
async ({ namespace, labelSelector }: { namespace?: string; labelSelector?: string }) => {
const managedLabels = { 'clerum.io/managed-by': 'host-context-controller' }
@@ -5303,6 +5368,8 @@ describe('NetworkPolicyReconciler', () => {
})
it('skips external egress orphan cleanup when startup server discovery is incomplete', async () => {
+ // Bootstrap this scenario without policies; later LIST fixtures supply existing objects.
+ mockApi.readNamespacedNetworkPolicy.mockRejectedValue({ code: 404 })
mockApi.listNamespacedNetworkPolicy.mockImplementation(
async ({ labelSelector }: { labelSelector?: string }) => {
if (labelSelector?.includes('context-allow')) return { items: [] }
diff --git a/host-context-controller/src/networkPolicyReconciler.ts b/host-context-controller/src/networkPolicyReconciler.ts
index d11a272be..a7ed8d507 100644
--- a/host-context-controller/src/networkPolicyReconciler.ts
+++ b/host-context-controller/src/networkPolicyReconciler.ts
@@ -2889,7 +2889,8 @@ export class NetworkPolicyReconciler {
convergence.policy,
'external-egress',
isCurrent,
- convergence.observedPolicy
+ convergence.observedPolicy,
+ fresh.policy
)
} else {
hccLogger.info('external egress policy unchanged', { policy: convergence.name })
@@ -3336,7 +3337,8 @@ export class NetworkPolicyReconciler {
policy: k8s.V1NetworkPolicy,
lane: SafetyInventoryLane,
isCurrent?: () => boolean,
- expectedSnapshot?: k8s.V1NetworkPolicy | null
+ expectedSnapshot?: k8s.V1NetworkPolicy | null,
+ observedPolicy?: k8s.V1NetworkPolicy | null
): Promise<void> {
await applyNetworkPolicy(
this.networkingApi,
@@ -3353,7 +3355,8 @@ export class NetworkPolicyReconciler {
},
// Best-effort maintenance may skip a disappeared object. External-egress
// admission must instead retry: POST409 followed by GET404 applied nothing.
- lane === 'external-egress'
+ lane === 'external-egress',
+ observedPolicy
)
}
diff --git a/host-context-controller/src/utils.ts b/host-context-controller/src/utils.ts
index c69ee9a06..068096412 100644
--- a/host-context-controller/src/utils.ts
+++ b/host-context-controller/src/utils.ts
@@ -43,6 +43,68 @@ export async function observeExistenceRead<T>(
}
}
+/**
+ * Read before creating, then retain each caller's convergence policy. The first
+ * observed object is consumed once by the existing retry loop; a conflict must
+ * always reread the API. SDK calls and their metric labels stay at the call site.
+ */
+export async function ensureResource<T extends { metadata?: { name?: string } }>(opts: {
+ read: () => Promise<T>
+ create: () => Promise<unknown>
+ existing?: T | null
+ /** False preserves a caller's non-throwing failure without counting a skip. */
+ converge: (read: () => Promise<T>, observed?: T) => Promise<void | false>
+ onSkipped: () => void
+ mutationAllowed?: () => boolean
+}): Promise<void> {
+ const { read, create, converge, onSkipped, mutationAllowed } = opts
+ if (mutationAllowed && !mutationAllowed()) return
+
+ const validate = (value: T): T => {
+ if (!value || typeof value.metadata?.name !== 'string' || !value.metadata.name.trim()) {
+ throw new Error('Kubernetes existence read must resolve to an object with metadata.name')
+ }
+ return value
+ }
+ const readFresh = async (): Promise<T> => validate(await read())
+ let existing = opts.existing
+ if (existing === undefined) {
+ try {
+ existing = await readFresh()
+ } catch (error) {
+ if (error == null || getErrorCode(error) !== 404) throw error
+ existing = null
+ }
+ } else if (existing !== null) {
+ validate(existing)
+ }
+ if (mutationAllowed && !mutationAllowed()) return
+
+ if (existing !== null) {
+ let snapshot: T | undefined = existing
+ const readOnce = async (): Promise<T> => {
+ if (snapshot !== undefined) {
+ const observed = snapshot
+ snapshot = undefined
+ return observed
+ }
+ return readFresh()
+ }
+ const result = await converge(readOnce, existing)
+ if (result !== false && (!mutationAllowed || mutationAllowed())) onSkipped()
+ return
+ }
+
+ try {
+ await create()
+ } catch (error) {
+ if (error == null || getErrorCode(error) !== 409) throw error
+ if (mutationAllowed && !mutationAllowed()) return
+ // Absence observed before the POST cannot survive a create conflict.
+ await converge(readFresh)
+ }
+}
+
/**
* Re-read + replace with optimistic-lock retry. K8s rejects a replace whose
* `metadata.resourceVersion` does not match the current value; the only
@@ -482,7 +544,7 @@ export function normalizeVolumeDefaults(volume: k8s.V1Volume): void {
}
}
-/** Create-or-replace a NetworkPolicy (409 catch → conflict-retry replace). */
+/** Read-first create or converge a NetworkPolicy with the existing retry policy. */
export async function applyNetworkPolicy(
api: k8s.NetworkingV1Api,
name: string,
@@ -491,33 +553,35 @@ export async function applyNetworkPolicy(
logPrefix = '[NetPol]',
mutationAllowed?: () => boolean,
validateExisting?: (existing: k8s.V1NetworkPolicy) => void,
- missingIsError = false
+ missingIsError = false,
+ existing?: k8s.V1NetworkPolicy | null
): Promise<void> {
- if (mutationAllowed && !mutationAllowed()) return
- try {
- await observeCreate('NetworkPolicy', () =>
- api.createNamespacedNetworkPolicy({ namespace, body: policy })
- )
- hccLogger.info('NetworkPolicy created', { scope: logPrefix, policy: name, namespace })
- return
- } catch (error: unknown) {
- if (getErrorCode(error) !== 409) {
- throw error
- }
- }
- await replaceWithConflictRetry<k8s.V1NetworkPolicy>({
- description: `policy "${name}" in ${namespace}`,
- logPrefix,
- body: policy,
+ await ensureResource<k8s.V1NetworkPolicy>({
+ existing,
+ mutationAllowed,
read: () =>
observeExistenceRead('NetworkPolicy', () =>
api.readNamespacedNetworkPolicy({ name, namespace })
),
- replace: body => api.replaceNamespacedNetworkPolicy({ name, namespace, body }),
- mutationAllowed,
- validateExisting,
- missingIsError,
- isUpToDate: networkPolicyMatchesDesired,
+ create: async () => {
+ await observeCreate('NetworkPolicy', () =>
+ api.createNamespacedNetworkPolicy({ namespace, body: policy })
+ )
+ hccLogger.info('NetworkPolicy created', { scope: logPrefix, policy: name, namespace })
+ },
+ onSkipped: () => createsTotal.inc({ kind: 'NetworkPolicy', outcome: 'skipped' }),
+ converge: read =>
+ replaceWithConflictRetry<k8s.V1NetworkPolicy>({
+ description: `policy "${name}" in ${namespace}`,
+ logPrefix,
+ body: policy,
+ read,
+ replace: body => api.replaceNamespacedNetworkPolicy({ name, namespace, body }),
+ mutationAllowed,
+ validateExisting,
+ missingIsError,
+ isUpToDate: networkPolicyMatchesDesired,
+ }),
})
}
diff --git a/host-context-controller/test/hostReconciler.lifecycle.test.ts b/host-context-controller/test/hostReconciler.lifecycle.test.ts
index 7a799010f..7fceb76a3 100644
--- a/host-context-controller/test/hostReconciler.lifecycle.test.ts
+++ b/host-context-controller/test/hostReconciler.lifecycle.test.ts
@@ -741,15 +741,17 @@ describe('HostReconciler stateless lifecycle — rejection matrix', () => {
})
const host = makeStatelessHost({ status: suspendedStatus(4) })
customApi.getNamespacedCustomObject.mockImplementation(async () => hostApiObject(host))
- networkingApi.createNamespacedNetworkPolicy.mockImplementation(async () => {
+ const readPolicy = networkingApi.readNamespacedNetworkPolicy.getMockImplementation()!
+ networkingApi.readNamespacedNetworkPolicy.mockImplementation(async request => {
cacheSynced = false
- return {}
+ return readPolicy(request)
})
const provision = vi
.spyOn(reconciler as any, 'provisionRuntimeTokenRevision')
.mockResolvedValue(runtimeTokenProvision(host))
await reconciler.reconcile(host)
+ expect(networkingApi.readNamespacedNetworkPolicy).toHaveBeenCalled()
expect(provision).toHaveBeenCalledOnce()
expect(provision).toHaveBeenCalledWith(
@@ -779,12 +781,14 @@ describe('HostReconciler stateless lifecycle — rejection matrix', () => {
})
const host = makeStatelessHost({ status: suspendedStatus(5) })
customApi.getNamespacedCustomObject.mockImplementation(async () => hostApiObject(host))
- networkingApi.createNamespacedNetworkPolicy.mockImplementation(async () => {
+ const readPolicy = networkingApi.readNamespacedNetworkPolicy.getMockImplementation()!
+ networkingApi.readNamespacedNetworkPolicy.mockImplementation(async request => {
channelCount = 1
- return {}
+ return readPolicy(request)
})
await reconciler.reconcile(host)
+ expect(networkingApi.readNamespacedNetworkPolicy).toHaveBeenCalled()
const deployment = hostDeploymentBody(appsApi, host.name)
expect(deployment.spec?.replicas).toBe(1)
@@ -814,9 +818,10 @@ describe('HostReconciler stateless lifecycle — rejection matrix', () => {
status: { lifecycle: { state: 'active', wakeHandledGeneration: 1 } },
})
customApi.getNamespacedCustomObject.mockImplementation(async () => hostApiObject(host))
- networkingApi.createNamespacedNetworkPolicy.mockImplementation(async () => {
+ const readPolicy = networkingApi.readNamespacedNetworkPolicy.getMockImplementation()!
+ networkingApi.readNamespacedNetworkPolicy.mockImplementation(async request => {
channelCount = 1
- return {}
+ return readPolicy(request)
})
const issueTokens = vi.mocked(issueMcpHostRuntimeTokens)
const mintGfs = vi.mocked(mintHostGfsToken)
@@ -824,6 +829,7 @@ describe('HostReconciler stateless lifecycle — rejection matrix', () => {
mintGfs.mockClear()
await reconciler.reconcile(host)
+ expect(networkingApi.readNamespacedNetworkPolicy).toHaveBeenCalled()
expect(issueTokens).toHaveBeenCalledOnce()
expect(mintGfs).toHaveBeenCalledOnce()
@@ -1012,15 +1018,17 @@ describe('HostReconciler stateless lifecycle — rejection matrix', () => {
status: { lifecycle: { state: 'active', wakeHandledGeneration: 1 } },
})
customApi.getNamespacedCustomObject.mockImplementation(async () => hostApiObject(host))
- networkingApi.createNamespacedNetworkPolicy.mockImplementation(async () => {
+ const readPolicy = networkingApi.readNamespacedNetworkPolicy.getMockImplementation()!
+ networkingApi.readNamespacedNetworkPolicy.mockImplementation(async request => {
channelCount = 1
- return {}
+ return readPolicy(request)
})
const provision = vi
.spyOn(reconciler as any, 'provisionRuntimeTokenRevision')
.mockResolvedValue(runtimeTokenProvision(host, true))
await reconciler.reconcile(host)
+ expect(networkingApi.readNamespacedNetworkPolicy).toHaveBeenCalled()
expect(provision).toHaveBeenCalledOnce()
})
@@ -1034,15 +1042,17 @@ describe('HostReconciler stateless lifecycle — rejection matrix', () => {
status: { lifecycle: { state: 'active', wakeHandledGeneration: 1 } },
})
customApi.getNamespacedCustomObject.mockImplementation(async () => hostApiObject(host))
- networkingApi.createNamespacedNetworkPolicy.mockImplementation(async () => {
+ const readPolicy = networkingApi.readNamespacedNetworkPolicy.getMockImplementation()!
+ networkingApi.readNamespacedNetworkPolicy.mockImplementation(async request => {
cacheSynced = false
- return {}
+ return readPolicy(request)
})
const provision = vi
.spyOn(reconciler as any, 'provisionRuntimeTokenRevision')
.mockResolvedValue(runtimeTokenProvision(host))
await reconciler.reconcile(host)
+ expect(networkingApi.readNamespacedNetworkPolicy).toHaveBeenCalled()
expect(provision).toHaveBeenCalledOnce()
const deployment = hostDeploymentBody(appsApi, host.name)
diff --git a/host-context-controller/test/hostReconciler.networkpolicy.test.ts b/host-context-controller/test/hostReconciler.networkpolicy.test.ts
index f17fb1eb7..51774783e 100644
--- a/host-context-controller/test/hostReconciler.networkpolicy.test.ts
+++ b/host-context-controller/test/hostReconciler.networkpolicy.test.ts
@@ -122,7 +122,10 @@ describe('HostReconciler.ensureChannelReaderEgressNetworkPolicy', () => {
it('creates the egress NP in channels namespace with correct selectors', async () => {
const { reconciler, mocks } = createReconciler()
+ // This case exercises initial creation of an absent policy.
+ mocks.networkingApi.readNamespacedNetworkPolicy.mockRejectedValueOnce({ code: 404 })
await (reconciler as any).ensureChannelReaderEgressNetworkPolicy(HOST)
+ expect(mocks.networkingApi.readNamespacedNetworkPolicy).toHaveBeenCalledTimes(1)
expect(mocks.networkingApi.createNamespacedNetworkPolicy).toHaveBeenCalledTimes(1)
const call = (mocks.networkingApi.createNamespacedNetworkPolicy as any).mock.calls[0][0]
@@ -163,7 +166,10 @@ describe('HostReconciler.ensureMcpHostIngressNetworkPolicy', () => {
it('creates the ingress NP in mcp-host namespace with correct selectors', async () => {
const { reconciler, mocks } = createReconciler()
+ // This case exercises initial creation of an absent policy.
+ mocks.networkingApi.readNamespacedNetworkPolicy.mockRejectedValueOnce({ code: 404 })
await (reconciler as any).ensureMcpHostIngressNetworkPolicy(HOST)
+ expect(mocks.networkingApi.readNamespacedNetworkPolicy).toHaveBeenCalledTimes(1)
expect(mocks.networkingApi.createNamespacedNetworkPolicy).toHaveBeenCalledTimes(1)
const call = (mocks.networkingApi.createNamespacedNetworkPolicy as any).mock.calls[0][0]
diff --git a/host-context-controller/test/hostReconciler.test.ts b/host-context-controller/test/hostReconciler.test.ts
index da634e350..f0dd97d86 100644
--- a/host-context-controller/test/hostReconciler.test.ts
+++ b/host-context-controller/test/hostReconciler.test.ts
@@ -817,15 +817,17 @@ describe('HostReconciler — desktop support', () => {
expect(resources.limits.cpu).toBe('1')
})
- it('creates desktop NetworkPolicy when desktop enabled', async () => {
+ it('converges desktop NetworkPolicy when desktop enabled', async () => {
const { reconciler, networkingApi } = createReconciler()
const host = makeDesktopHost({ x11: true })
await reconciler.reconcile(host)
- // reconcile creates per-host ingress, rpc-proxy egress, desktop,
+ // Reconcile updates existing per-host ingress, rpc-proxy egress, desktop,
// GFS egress, channel-reader, and workflow-approval-reader policies.
- expect(networkingApi.createNamespacedNetworkPolicy).toHaveBeenCalledTimes(8)
- const names = networkingApi.createNamespacedNetworkPolicy.mock.calls.map(
+ expect(networkingApi.replaceNamespacedNetworkPolicy).toHaveBeenCalledTimes(8)
+ expect(networkingApi.createNamespacedNetworkPolicy).not.toHaveBeenCalled()
+ expect(networkingApi.readNamespacedNetworkPolicy).toHaveBeenCalled()
+ const names = networkingApi.replaceNamespacedNetworkPolicy.mock.calls.map(
(c: any[]) => c[0].body.metadata.name
)
expect(names).toEqual(
@@ -835,25 +837,27 @@ describe('HostReconciler — desktop support', () => {
'workflow-approval-reader-desktop-host-egress-mcp-host',
])
)
- const call = networkingApi.createNamespacedNetworkPolicy.mock.calls.find(
+ const call = networkingApi.replaceNamespacedNetworkPolicy.mock.calls.find(
(c: any[]) => c[0].body.metadata.name === 'allow-rpc-proxy-desktop-desktop-host'
)![0]
expect(call.body.metadata.name).toBe('allow-rpc-proxy-desktop-desktop-host')
expect(call.body.spec.podSelector).toEqual({ matchLabels: { app: 'desktop-host' } })
})
- it('does not create desktop NetworkPolicy for non-desktop hosts', async () => {
+ it('does not converge desktop NetworkPolicy for non-desktop hosts', async () => {
const { reconciler, networkingApi } = createReconciler()
const host = makeHost()
await reconciler.reconcile(host)
- // reconcile creates 6 NPs for non-desktop hosts:
+ // Reconcile updates 7 existing NPs for non-desktop hosts:
// mcp-host ingress from channel-reader, mcp-host ingress from rpc-proxy,
// mcp-host GFS egress, rpc-proxy host egress, channel-reader egress,
// and workflow-approval-reader policies.
// (no desktop NP)
- expect(networkingApi.createNamespacedNetworkPolicy).toHaveBeenCalledTimes(7)
- const names = networkingApi.createNamespacedNetworkPolicy.mock.calls.map(
+ expect(networkingApi.replaceNamespacedNetworkPolicy).toHaveBeenCalledTimes(7)
+ expect(networkingApi.createNamespacedNetworkPolicy).not.toHaveBeenCalled()
+ expect(networkingApi.readNamespacedNetworkPolicy).toHaveBeenCalled()
+ const names = networkingApi.replaceNamespacedNetworkPolicy.mock.calls.map(
(c: any[]) => c[0].body.metadata.name
)
expect(names).toEqual(