Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript

fix(egress): enforce proven fail-closed convergence

Alfredo Lopezde776803800e
diff --git a/host-context-controller/package-lock.json b/host-context-controller/package-lock.json
index 4317836a5..c390d2654 100644
--- a/host-context-controller/package-lock.json
+++ b/host-context-controller/package-lock.json
@@ -1,12 +1,12 @@
{
"name": "clerum-host-context-controller",
- "version": "0.1.295",
+ "version": "0.1.296",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "clerum-host-context-controller",
- "version": "0.1.295",
+ "version": "0.1.296",
"license": "SEE LICENSE IN LICENSE",
"dependencies": {
"@clerum/codex-catalog-projection": "file:../packages/codex-catalog-projection",
diff --git a/host-context-controller/package.json b/host-context-controller/package.json
index a8980c5b6..dd067e323 100644
--- a/host-context-controller/package.json
+++ b/host-context-controller/package.json
@@ -1,6 +1,6 @@
{
"name": "clerum-host-context-controller",
- "version": "0.1.295",
+ "version": "0.1.296",
"description": "Host Context Controller - provides McpServer CRDs to mcp-host via REST API",
"main": "dist/main.js",
"engines": {
diff --git a/host-context-controller/src/externalEgressAccumulator.test.ts b/host-context-controller/src/externalEgressAccumulator.test.ts
index 63419c553..33084e5b1 100644
--- a/host-context-controller/src/externalEgressAccumulator.test.ts
+++ b/host-context-controller/src/externalEgressAccumulator.test.ts
@@ -153,4 +153,52 @@ describe('accumulateHostExactHostEgress — issue #299 sliding window (HCC)', ()
expect(r.annotations[STATE_ANNOTATION]).toBeTruthy()
expect(r.annotations[RESOLVED_AT_ANNOTATION]).toBe(new Date(T0).toISOString())
})
+
+ it('preserves resolved-at on a DNS failure instead of inventing a fresh resolution time', () => {
+ const r1 = accumulateHostExactHostEgress(baseInput({ resolution: ok(['140.82.112.3'], 15) }))
+ const r2 = accumulateHostExactHostEgress(
+ baseInput({
+ resolution: { kind: 'transient' },
+ previousAnnotations: r1.annotations,
+ now: T0 + 400_000,
+ })
+ )
+ expect(r2.annotations[RESOLVED_AT_ANNOTATION]).toBe(new Date(T0).toISOString())
+ expect(r2.resolvedAt).toBe(new Date(T0).toISOString())
+ })
+
+ it('derives resolved-at from proven structured state when the legacy timestamp is absent', () => {
+ const r1 = accumulateHostExactHostEgress(baseInput({ resolution: ok(['140.82.112.3'], 15) }))
+ const previousAnnotations = { ...r1.annotations }
+ delete previousAnnotations[RESOLVED_AT_ANNOTATION]
+ const r2 = accumulateHostExactHostEgress(
+ baseInput({
+ resolution: { kind: 'transient' },
+ previousAnnotations,
+ now: T0 + 400_000,
+ })
+ )
+ expect(r2.resolvedAt).toBe(new Date(T0).toISOString())
+ expect(r2.annotations[RESOLVED_AT_ANNOTATION]).toBe(new Date(T0).toISOString())
+ })
+
+ it('does not synthesize resolved-at for a bootstrap failure', () => {
+ const r = accumulateHostExactHostEgress(baseInput({ resolution: { kind: 'transient' } }))
+ expect(r.annotations[RESOLVED_AT_ANNOTATION]).toBeUndefined()
+ expect(r.resolvedAt).toBeUndefined()
+ })
+
+ it('does not claim a DNS timestamp for legacy portless state during a failure', () => {
+ const r = accumulateHostExactHostEgress(
+ baseInput({
+ resolution: { kind: 'transient' },
+ previousAnnotations: {
+ [TARGETS_ANNOTATION]: 'api.github.com=140.82.112.9/32',
+ },
+ })
+ )
+ expect(r.cidrs).toEqual(['140.82.112.9/32'])
+ expect(r.resolvedAt).toBeUndefined()
+ expect(r.annotations[RESOLVED_AT_ANNOTATION]).toBeUndefined()
+ })
})
diff --git a/host-context-controller/src/externalEgressAccumulator.ts b/host-context-controller/src/externalEgressAccumulator.ts
index d863df558..4dceda778 100644
--- a/host-context-controller/src/externalEgressAccumulator.ts
+++ b/host-context-controller/src/externalEgressAccumulator.ts
@@ -29,6 +29,7 @@ import {
RESOLVED_AT_ANNOTATION,
emptyState,
parseState,
+ parseStateStrict,
reconcileEgressState,
serializeState,
} from '@clerum/network-policy-core'
@@ -56,10 +57,14 @@ export interface AccumulateHostEgressInput {
}
export interface AccumulateHostEgressOutput {
+ /** Normalized state entries behind cidrs/annotations. */
+ entries: ReturnType<typeof emptyState>['entries']
/** The effective accumulated /32 set to render as ipBlocks (sorted, deduped). */
cidrs: string[]
/** Annotations to stamp on the policy: serialized state + resolved-at. */
annotations: Record<string, string>
+ /** Last accepted DNS observation represented by the effective set, if proven. */
+ resolvedAt?: string
/** True iff the (fqdn,ip,port,protocol) set changed vs the previous state. */
changed: boolean
/** True iff the persisted window is aging and must be re-persisted (audit M1). */
@@ -107,12 +112,35 @@ export function accumulateHostExactHostEgress(
const out = reconcileEgressState(previous, [observation], now, config)
+ const annotations = serializeState(out.state)
+ let resolvedAt: string | undefined
+ if (resolution.kind === 'ok') {
+ resolvedAt = new Date(now).toISOString()
+ } else {
+ const priorResolvedAt = previousAnnotations?.[RESOLVED_AT_ANNOTATION]
+ if (priorResolvedAt && Number.isFinite(Date.parse(priorResolvedAt))) {
+ resolvedAt = priorResolvedAt
+ } else if (previousAnnotations) {
+ const strictPrevious = parseStateStrict(previousAnnotations, config, [
+ { fqdn, port, protocol },
+ ])
+ if (strictPrevious.kind === 'valid' && strictPrevious.state.entries.length > 0) {
+ const latestObservedAt = Math.max(
+ ...strictPrevious.state.entries.map(entry => entry.lastObservedAt)
+ )
+ if (Number.isFinite(new Date(latestObservedAt).getTime())) {
+ resolvedAt = new Date(latestObservedAt).toISOString()
+ }
+ }
+ }
+ }
+ if (resolvedAt) annotations[RESOLVED_AT_ANNOTATION] = resolvedAt
+
return {
+ entries: out.entries,
cidrs: out.entries.map(entry => `${entry.ip}/32`),
- annotations: {
- ...serializeState(out.state),
- [RESOLVED_AT_ANNOTATION]: new Date(now).toISOString(),
- },
+ annotations,
+ resolvedAt,
changed: out.changed,
renewalDue: out.renewalDue,
frozen: out.frozenFqdns.includes(fqdn),
diff --git a/host-context-controller/src/networkPolicyReconciler.egressClassification.test.ts b/host-context-controller/src/networkPolicyReconciler.egressClassification.test.ts
index cc4bc902f..aacb2e14f 100644
--- a/host-context-controller/src/networkPolicyReconciler.egressClassification.test.ts
+++ b/host-context-controller/src/networkPolicyReconciler.egressClassification.test.ts
@@ -88,25 +88,42 @@ function makeReconciler(
currentContext: 'test',
})
const reconciler = new NetworkPolicyReconciler(kc, new Map())
+ const normalizePolicy = (
+ policy: k8s.V1NetworkPolicy,
+ namespace: string,
+ index = 0
+ ): k8s.V1NetworkPolicy => {
+ const name = policy.metadata?.name ?? `unnamed-${index}`
+ return {
+ ...policy,
+ metadata: {
+ ...policy.metadata,
+ name,
+ namespace: policy.metadata?.namespace ?? namespace,
+ uid: policy.metadata?.uid ?? `${namespace}:${name}:uid`,
+ resourceVersion: policy.metadata?.resourceVersion ?? '1',
+ },
+ }
+ }
const networkingApi = {
...mockApi,
listNamespacedNetworkPolicy: async (args: { namespace: string }) => {
const response = await mockApi.listNamespacedNetworkPolicy(args)
- const items = (response.items ?? []).map((policy: k8s.V1NetworkPolicy, index: number) => {
- const name = policy.metadata?.name ?? `unnamed-${index}`
- return {
- ...policy,
- metadata: {
- ...policy.metadata,
- name,
- namespace: policy.metadata?.namespace ?? args.namespace,
- uid: policy.metadata?.uid ?? `${args.namespace}:${name}:uid`,
- resourceVersion: policy.metadata?.resourceVersion ?? '1',
- },
- }
- })
+ const items = (response.items ?? []).map((policy: k8s.V1NetworkPolicy, index: number) =>
+ normalizePolicy(policy, args.namespace, index)
+ )
return { ...response, items }
},
+ readNamespacedNetworkPolicy: async (args: { name: string; namespace: string }) => {
+ const direct = (await mockApi.readNamespacedNetworkPolicy(args)) as k8s.V1NetworkPolicy
+ if (direct.metadata?.name || direct.spec) return normalizePolicy(direct, args.namespace)
+ const response = await mockApi.listNamespacedNetworkPolicy({ namespace: args.namespace })
+ const found = (response.items ?? []).find(
+ (policy: k8s.V1NetworkPolicy) => policy.metadata?.name === args.name
+ )
+ if (!found) throw Object.assign(new Error('not found'), { code: 404 })
+ return normalizePolicy(found, args.namespace)
+ },
}
;(reconciler as unknown as { networkingApi: unknown }).networkingApi = networkingApi
;(reconciler as unknown as { customApi: unknown }).customApi = mockCustomApi
@@ -184,6 +201,15 @@ describe('#513: a non-DNS exception inside the resolve block is not a DNS condit
])
}
+ function injectPostResolutionCodedError(code: string): void {
+ const record = Object.defineProperty({ ttl: 300 }, 'address', {
+ get() {
+ throw Object.assign(new Error(`post-resolution ${code}`), { code })
+ },
+ }) as RecordWithTtl
+ resolve4Mock.mockResolvedValueOnce([record])
+ }
+
type Condition = { type?: string; status?: string; reason?: string; message?: string }
/**
@@ -218,9 +244,11 @@ describe('#513: a non-DNS exception inside the resolve block is not a DNS condit
const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined)
injectPostResolutionTypeError()
- // No throw: there is a live policy, so the fault is served fail-static (T7).
- // Everything else this test pins is unchanged.
- await reconciler.reconcileExternalEgress(server, { isCurrent: () => true })
+ // A verified prior policy may remain for already-running pods, but the
+ // controller fault still blocks new/update runtime reconciliation.
+ await expect(
+ reconciler.reconcileExternalEgress(server, { isCurrent: () => true })
+ ).rejects.toThrow(TypeError)
// The operator must not be pointed at DNS for a fault in the reconciler.
expect(warn.mock.calls.flat().join('\n')).not.toContain('permanent DNS failure')
@@ -242,9 +270,9 @@ describe('#513: a non-DNS exception inside the resolve block is not a DNS condit
await seedAccumulatedPolicy({ lapsed: true })
injectPostResolutionTypeError()
- // No throw: the policy is live (its window lapsed, the object did not), so
- // the fault is served fail-static (T7). The prune assertion is the point.
- await reconciler.reconcileExternalEgress(server, { isCurrent: () => true })
+ await expect(
+ reconciler.reconcileExternalEgress(server, { isCurrent: () => true })
+ ).rejects.toThrow(TypeError)
// Pre-fix this prunes the lapsed set to zero, hits the `continue`, and the
// policy is swept as undesired. D4 fail-static must hold for a fault that is
@@ -255,6 +283,86 @@ describe('#513: a non-DNS exception inside the resolve block is not a DNS condit
expect(egress.at(-1)?.reason).toBe('ExternalEgressReconcileFailed')
})
+ it('T2b never promotes annotation-only state during transient fail-static', async () => {
+ await seedAccumulatedPolicy({ lapsed: false })
+ const listed = (await mockApi.listNamespacedNetworkPolicy({ namespace: 'mcp-server' })) as {
+ items: k8s.V1NetworkPolicy[]
+ }
+ const drifted = structuredClone(listed)
+ const state = JSON.parse(
+ drifted.items[0].metadata!.annotations!['clerum.io/egress-fqdn-state']
+ ) as Array<Record<string, unknown>>
+ state.push({
+ ...state[0],
+ ip: '8.8.8.8',
+ })
+ drifted.items[0].metadata!.annotations!['clerum.io/egress-fqdn-state'] = JSON.stringify(state)
+ mockApi.listNamespacedNetworkPolicy.mockResolvedValue(drifted)
+ resolve4Mock.mockRejectedValueOnce(
+ Object.assign(new Error('queryA ESERVFAIL api.example.com'), { code: 'ESERVFAIL' })
+ )
+
+ await expect(
+ reconciler.reconcileExternalEgress(server, { isCurrent: () => true })
+ ).rejects.toThrow(/Fail-static proof failed/)
+
+ expect(mockApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledWith(
+ expect.objectContaining({ name: drifted.items[0].metadata!.name })
+ )
+ const written = mockApi.replaceNamespacedNetworkPolicy.mock.calls
+ .map(call => JSON.stringify(call[0]?.body))
+ .join(' ')
+ expect(written).not.toContain('8.8.8.8')
+ const egress = statusConditions().filter(c => c.type === 'ExternalEgressReady')
+ expect(egress.at(-1)?.status).toBe('False')
+ expect(egress.at(-1)?.reason).toBe('ExternalEgressReconcileFailed')
+ expect(egress.at(-1)?.message).toContain('REVOKED')
+ })
+
+ it('T2c never promotes annotation-only state after an accepted DNS answer', async () => {
+ await seedAccumulatedPolicy({ lapsed: false })
+ const listed = (await mockApi.listNamespacedNetworkPolicy({ namespace: 'mcp-server' })) as {
+ items: k8s.V1NetworkPolicy[]
+ }
+ const drifted = structuredClone(listed)
+ const state = JSON.parse(
+ drifted.items[0].metadata!.annotations!['clerum.io/egress-fqdn-state']
+ ) as Array<Record<string, unknown>>
+ state.push({ ...state[0], ip: '8.8.8.8' })
+ drifted.items[0].metadata!.annotations!['clerum.io/egress-fqdn-state'] = JSON.stringify(state)
+ mockApi.listNamespacedNetworkPolicy.mockResolvedValue(drifted)
+
+ await reconciler.reconcileExternalEgress(server, { isCurrent: () => true })
+
+ const writes = [
+ ...mockApi.createNamespacedNetworkPolicy.mock.calls,
+ ...mockApi.replaceNamespacedNetworkPolicy.mock.calls,
+ ]
+ .map(call => JSON.stringify(call[0]?.body))
+ .join(' ')
+ expect(writes).not.toContain('8.8.8.8')
+ expect(writes).toContain('1.2.3.4')
+ expect(servedPayload()).not.toContain('8.8.8.8')
+ expect(servedPayload()).toContain('1.2.3.4')
+ })
+
+ it.each(['ETIMEDOUT', 'ENOTFOUND'])(
+ 'T2a keeps a post-resolution %s exception out of DNS classification',
+ async code => {
+ await seedAccumulatedPolicy({ lapsed: false })
+ injectPostResolutionCodedError(code)
+
+ await expect(
+ reconciler.reconcileExternalEgress(server, { isCurrent: () => true })
+ ).rejects.toThrow(`post-resolution ${code}`)
+
+ expect(mockApi.deleteNamespacedNetworkPolicy).not.toHaveBeenCalled()
+ const egress = statusConditions().filter(c => c.type === 'ExternalEgressReady')
+ expect(egress.at(-1)?.reason).toBe('ExternalEgressReconcileFailed')
+ expect(egress.at(-1)?.message).not.toContain('DNS failure')
+ }
+ )
+
it('T3 still prunes and fails loud on a genuine NXDOMAIN with a lapsed window', async () => {
await seedAccumulatedPolicy({ lapsed: true })
resolve4Mock.mockRejectedValueOnce(
@@ -272,6 +380,22 @@ describe('#513: a non-DNS exception inside the resolve block is not a DNS condit
expect(egress.at(-1)?.reason).toBe('ExternalEgressRejected')
})
+ it('T3a revokes and blocks on genuine NXDOMAIN even while the prior window is unexpired', async () => {
+ await seedAccumulatedPolicy({ lapsed: false })
+ resolve4Mock.mockRejectedValueOnce(
+ Object.assign(new Error('queryA ENOTFOUND api.example.com'), { code: 'ENOTFOUND' })
+ )
+
+ await expect(
+ reconciler.reconcileExternalEgress(server, { isCurrent: () => true })
+ ).rejects.toThrow(/failed to resolve hostname "api\.example\.com"/)
+
+ expect(mockApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledTimes(1)
+ const egress = statusConditions().filter(c => c.type === 'ExternalEgressReady')
+ expect(egress.at(-1)?.status).toBe('False')
+ expect(egress.at(-1)?.reason).toBe('ExternalEgressRejected')
+ })
+
it("T6 still deletes a de-authorized binding's policy when a sibling binding faults", async () => {
// Fail-loud must not cost fail-closed. `cleanupExternalEgress` runs AFTER the
// binding loop and is the only place that deletes policies for bindings no
@@ -305,11 +429,11 @@ describe('#513: a non-DNS exception inside the resolve block is not a DNS condit
// The operator removes a.example.com; b.example.com faults after resolution.
injectPostResolutionTypeError()
- // No throw: b.example.com has a live policy, so the fault is served
- // fail-static (T7). What this test pins is that the cleanup still ran.
- await reconciler.reconcileExternalEgress(bindings(['b.example.com']), {
- isCurrent: () => true,
- })
+ await expect(
+ reconciler.reconcileExternalEgress(bindings(['b.example.com']), {
+ isCurrent: () => true,
+ })
+ ).rejects.toThrow(TypeError)
// Exactly one delete, and it is A's. `toHaveBeenCalledWith` alone is a
// membership check, not an exclusivity one: it stays green when the cleanup
@@ -325,24 +449,13 @@ describe('#513: a non-DNS exception inside the resolve block is not a DNS condit
)
})
- it('T7 serves the frozen set and does NOT block the workload when a fault has a live policy', async () => {
- // Review of #567, major 3. The throw reaches k8sClient's catch, which logs
- // "runtime reconciliation blocked" and returns BEFORE the managed Deployment
- // is created — so a fault in ONE binding kept a whole server's pod down,
- // healthy siblings included. #513 never asked for that: its acceptance is "no
- // permanent DNS failure line, no prune". D4 fail-static is the house rule for
- // a condition we cannot resolve this pass, and the transient branch already
- // obeys it. When there is a live policy to serve, serve it, report the fault,
- // and let the HCC-managed Deployment start on the last known-good egress.
- //
- // Scoped to HCC on purpose. Every consumer of ExternalEgressReady treats a
- // False as terminal whatever the reason, so a WRC recipe depending on this
- // server still fails — see #577, which is a decision about what the
- // condition means, not something this test can assert away.
+ it('T7 retains a verified set for existing pods but blocks runtime on a controller fault', async () => {
await seedAccumulatedPolicy({ lapsed: false })
injectPostResolutionTypeError()
- await reconciler.reconcileExternalEgress(server, { isCurrent: () => true })
+ await expect(
+ reconciler.reconcileExternalEgress(server, { isCurrent: () => true })
+ ).rejects.toThrow(TypeError)
expect(mockApi.deleteNamespacedNetworkPolicy).not.toHaveBeenCalled()
expect(mockApi.replaceNamespacedNetworkPolicy).not.toHaveBeenCalled()
@@ -527,7 +640,7 @@ describe('#513: a non-DNS exception inside the resolve block is not a DNS condit
expect(egress.at(-1)?.message).toContain('REVOKED')
})
- it('T11c still serves a clean /32 — the guard narrowed, it did not close', async () => {
+ it('T11c retains a clean /32 for existing pods while blocking runtime', async () => {
// Anti-over-correction. T11/T11a/T11b all assert revocation, so a guard that
// simply declared everything untrustworthy would satisfy the three of them
// and destroy fail-static, which is the property five review rounds converged
@@ -535,14 +648,9 @@ describe('#513: a non-DNS exception inside the resolve block is not a DNS condit
await seedAccumulatedPolicy({ lapsed: false })
injectPostResolutionTypeError()
- // Fail-static in one assertion: a fault WITH something trustworthy to serve
- // completes instead of throwing. T11/T11a/T11b are the three shapes that must
- // block; this is the shape that must not, and it is also the liveness witness
- // — the pass reached the fault branch and chose to serve, rather than never
- // faulting at all. T8 pins the other side (nothing to serve ⇒ still throws).
await expect(
reconciler.reconcileExternalEgress(server, { isCurrent: () => true })
- ).resolves.toBeUndefined()
+ ).rejects.toThrow(TypeError)
// The clean policy is kept, not revoked …
expect(mockApi.deleteNamespacedNetworkPolicy).not.toHaveBeenCalled()
@@ -559,6 +667,145 @@ describe('#513: a non-DNS exception inside the resolve block is not a DNS condit
expect(served).toContain('1.2.3.4')
})
+ it('T11c1 revokes a canonical public /32 absent from persisted DNS state', async () => {
+ await seedAccumulatedPolicy({ lapsed: false })
+ const policyName = await driftLivePolicy(spec => {
+ spec.egress!.push({
+ to: [{ ipBlock: { cidr: '8.8.8.8/32' } }],
+ ports: [{ port: 443, protocol: 'TCP' }],
+ })
+ })
+ injectPostResolutionTypeError()
+
+ await expect(
+ reconciler.reconcileExternalEgress(server, { isCurrent: () => true })
+ ).rejects.toThrow(TypeError)
+
+ expect(mockApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledWith(
+ expect.objectContaining({ name: policyName })
+ )
+ expect(servedPayload()).not.toContain('8.8.8.8')
+ })
+
+ it('T11c2 rejects an otherwise canonical state above the configured cap', async () => {
+ await seedAccumulatedPolicy({ lapsed: false })
+ const items = (await mockApi.listNamespacedNetworkPolicy({ namespace: 'mcp-server' })) as {
+ items: k8s.V1NetworkPolicy[]
+ }
+ const drifted = structuredClone(items)
+ const now = Date.now()
+ const ips = Array.from({ length: 129 }, (_, index) => `11.0.0.${index + 1}`)
+ drifted.items[0].metadata!.annotations!['clerum.io/egress-fqdn-state'] = JSON.stringify(
+ ips.map(ip => ({
+ ip,
+ port: 443,
+ protocol: 'TCP',
+ fqdn: 'api.example.com',
+ expiresAt: now + 600_000,
+ lastObservedAt: now,
+ }))
+ )
+ drifted.items[0].spec!.egress = ips.map(ip => ({
+ to: [{ ipBlock: { cidr: `${ip}/32` } }],
+ ports: [{ port: 443, protocol: 'TCP' }],
+ }))
+ const policyName = drifted.items[0].metadata!.name!
+ mockApi.listNamespacedNetworkPolicy.mockResolvedValue(drifted)
+ injectPostResolutionTypeError()
+
+ await expect(
+ reconciler.reconcileExternalEgress(server, { isCurrent: () => true })
+ ).rejects.toThrow(TypeError)
+ expect(mockApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledWith(
+ expect.objectContaining({ name: policyName })
+ )
+ })
+
+ it('T11c3 judges a fresh GET and fences deletion to that resourceVersion', async () => {
+ await seedAccumulatedPolicy({ lapsed: false })
+ const listed = (await mockApi.listNamespacedNetworkPolicy({ namespace: 'mcp-server' })) as {
+ items: k8s.V1NetworkPolicy[]
+ }
+ const fresh = structuredClone(listed.items[0])
+ fresh.metadata = {
+ ...fresh.metadata,
+ name: fresh.metadata!.name!,
+ namespace: 'mcp-server',
+ uid: 'fresh-policy-uid',
+ resourceVersion: '99',
+ }
+ fresh.spec!.egress!.push({
+ to: [{ ipBlock: { cidr: '8.8.4.4/32' } }],
+ ports: [{ port: 443, protocol: 'TCP' }],
+ })
+ mockApi.readNamespacedNetworkPolicy.mockResolvedValue(fresh)
+ injectPostResolutionTypeError()
+
+ await expect(
+ reconciler.reconcileExternalEgress(server, { isCurrent: () => true })
+ ).rejects.toThrow(TypeError)
+
+ expect(mockApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledWith(
+ expect.objectContaining({
+ name: fresh.metadata!.name,
+ body: { preconditions: { uid: 'fresh-policy-uid', resourceVersion: '99' } },
+ })
+ )
+ })
+
+ it('T11c3a leaves a fresh homonymous policy untouched when HCC does not own it', async () => {
+ const policyName = 'ext-egress-classify-mcp-api.example.com-443'
+ mockApi.readNamespacedNetworkPolicy.mockResolvedValue({
+ metadata: {
+ name: policyName,
+ namespace: 'mcp-server',
+ uid: 'foreign-policy-uid',
+ resourceVersion: '77',
+ labels: {
+ 'clerum.io/managed-by': 'another-controller',
+ 'clerum.io/policy-type': 'external-egress',
+ 'clerum.io/mcpserver': server.name,
+ },
+ },
+ spec: {
+ podSelector: { matchLabels: { app: 'foreign-workload' } },
+ policyTypes: ['Egress'],
+ egress: [],
+ },
+ } satisfies k8s.V1NetworkPolicy)
+ injectPostResolutionTypeError()
+
+ await expect(
+ reconciler.reconcileExternalEgress(server, { isCurrent: () => true })
+ ).rejects.toThrow(/conflicting or incomplete ownership/)
+
+ // A fresh GET is authoritative for both integrity and ownership. A policy
+ // that merely occupies HCC's deterministic name is not ours to retain,
+ // rewrite, or revoke—even while the controller correctly blocks runtime.
+ expect(mockApi.deleteNamespacedNetworkPolicy).not.toHaveBeenCalled()
+ expect(mockApi.replaceNamespacedNetworkPolicy).not.toHaveBeenCalled()
+ const egress = statusConditions().filter(c => c.type === 'ExternalEgressReady')
+ expect(egress.at(-1)?.reason).toBe('ExternalEgressReconcileFailed')
+ expect(egress.at(-1)?.message).toContain('not HCC-owned')
+ expect(egress.at(-1)?.message).toContain('left untouched')
+ })
+
+ it('T11c4 preserves the last successful resolvedAt on a controller fault', async () => {
+ await seedAccumulatedPolicy({ lapsed: false })
+ const listed = (await mockApi.listNamespacedNetworkPolicy({ namespace: 'mcp-server' })) as {
+ items: k8s.V1NetworkPolicy[]
+ }
+ const oldResolvedAt = '2000-01-01T00:00:00.000Z'
+ listed.items[0].metadata!.annotations!['clerum.io/egress-fqdn-resolved-at'] = oldResolvedAt
+ mockApi.listNamespacedNetworkPolicy.mockResolvedValue(listed)
+ injectPostResolutionTypeError()
+
+ await expect(
+ reconciler.reconcileExternalEgress(server, { isCurrent: () => true })
+ ).rejects.toThrow(TypeError)
+ expect(servedPayload()).toContain(oldResolvedAt)
+ })
+
/** The live policy the fixture seeded, cloned and handed to a drift mutation. */
async function driftLivePolicy(mutate: (spec: k8s.V1NetworkPolicySpec) => void): Promise<string> {
const items = (await mockApi.listNamespacedNetworkPolicy({ namespace: 'mcp-server' })) as {
@@ -698,7 +945,7 @@ describe('#513: a non-DNS exception inside the resolve block is not a DNS condit
).toContain('REVOKED')
})
- it('T11g still serves a clean multi-CIDR UDP binding — the shape check reads the binding', async () => {
+ it('T11g retains a clean multi-CIDR UDP binding while blocking runtime', async () => {
// Anti-over-correction, the half T11c cannot cover. T11c serves ONE /32 on the
// default TCP/443 binding, so a verdict that rebuilt the shape with a
// hardcoded TCP, a hardcoded port, or only the first CIDR would still satisfy
@@ -720,12 +967,9 @@ describe('#513: a non-DNS exception inside the resolve block is not a DNS condit
await seedAccumulatedPolicy({ lapsed: false, srv: udpServer })
injectPostResolutionTypeError()
- // Fail-static in one assertion: a fault WITH something trustworthy to serve
- // completes instead of throwing. It is also the liveness witness — the pass
- // reached the fault branch and chose to serve, rather than never faulting.
await expect(
reconciler.reconcileExternalEgress(udpServer, { isCurrent: () => true })
- ).resolves.toBeUndefined()
+ ).rejects.toThrow(TypeError)
// Both accumulated addresses are still served, so a rebuild that kept only the
// first CIDR fails here rather than passing quietly.
@@ -737,7 +981,7 @@ describe('#513: a non-DNS exception inside the resolve block is not a DNS condit
statusConditions()
.filter(c => c.type === 'ExternalEgressReady')
.at(-1)?.message
- ).toContain('left untouched')
+ ).toContain('runtime remains blocked')
})
it('T11h revokes a live policy with no egress rules — a deny disguised as an allow', async () => {
diff --git a/host-context-controller/src/networkPolicyReconciler.egressFailStatic.test.ts b/host-context-controller/src/networkPolicyReconciler.egressFailStatic.test.ts
index f28a29320..79457d9a1 100644
--- a/host-context-controller/src/networkPolicyReconciler.egressFailStatic.test.ts
+++ b/host-context-controller/src/networkPolicyReconciler.egressFailStatic.test.ts
@@ -55,7 +55,9 @@ vi.mock('./metrics', () => ({
function makeMockNetworkingApi() {
return {
createNamespacedNetworkPolicy: vi.fn().mockResolvedValue({}),
- readNamespacedNetworkPolicy: vi.fn().mockResolvedValue({ metadata: { resourceVersion: '1' } }),
+ readNamespacedNetworkPolicy: vi
+ .fn()
+ .mockRejectedValue(Object.assign(new Error('not found'), { code: 404 })),
replaceNamespacedNetworkPolicy: vi.fn().mockResolvedValue({}),
listNamespacedNetworkPolicy: vi.fn().mockResolvedValue({ items: [] }),
deleteNamespacedNetworkPolicy: vi.fn().mockResolvedValue({}),
@@ -88,25 +90,46 @@ function makeReconciler(
currentContext: 'test',
})
const reconciler = new NetworkPolicyReconciler(kc, new Map())
+ const normalizePolicy = (
+ policy: k8s.V1NetworkPolicy,
+ namespace: string,
+ index = 0
+ ): k8s.V1NetworkPolicy => {
+ const name = policy.metadata?.name ?? `unnamed-${index}`
+ return {
+ ...policy,
+ metadata: {
+ ...policy.metadata,
+ name,
+ namespace: policy.metadata?.namespace ?? namespace,
+ uid: policy.metadata?.uid ?? `${namespace}:${name}:uid`,
+ resourceVersion: policy.metadata?.resourceVersion ?? '1',
+ },
+ }
+ }
const networkingApi = {
...mockApi,
listNamespacedNetworkPolicy: async (args: { namespace: string }) => {
const response = await mockApi.listNamespacedNetworkPolicy(args)
- const items = (response.items ?? []).map((policy: k8s.V1NetworkPolicy, index: number) => {
- const name = policy.metadata?.name ?? `unnamed-${index}`
- return {
- ...policy,
- metadata: {
- ...policy.metadata,
- name,
- namespace: policy.metadata?.namespace ?? args.namespace,
- uid: policy.metadata?.uid ?? `${args.namespace}:${name}:uid`,
- resourceVersion: policy.metadata?.resourceVersion ?? '1',
- },
- }
- })
+ const items = (response.items ?? []).map((policy: k8s.V1NetworkPolicy, index: number) =>
+ normalizePolicy(policy, args.namespace, index)
+ )
return { ...response, items }
},
+ readNamespacedNetworkPolicy: async (args: { name: string; namespace: string }) => {
+ try {
+ const direct = (await mockApi.readNamespacedNetworkPolicy(args)) as k8s.V1NetworkPolicy
+ if (direct.metadata?.name || direct.spec) return normalizePolicy(direct, args.namespace)
+ } catch (error: unknown) {
+ if ((error as { code?: unknown })?.code !== 404) throw error
+ }
+ const response = await mockApi.listNamespacedNetworkPolicy({ namespace: args.namespace })
+ const found = (response.items ?? []).find(
+ (policy: k8s.V1NetworkPolicy) => policy.metadata?.name === args.name
+ )
+ if (!found) throw Object.assign(new Error('not found'), { code: 404 })
+ return normalizePolicy(found, args.namespace)
+ },
}
;(reconciler as unknown as { networkingApi: unknown }).networkingApi = networkingApi
;(reconciler as unknown as { customApi: unknown }).customApi = mockCustomApi
@@ -191,6 +214,9 @@ describe('R1-B1 regression: DNS timeout must fail-static (freeze), not prune', (
// it issues no create/replace here.
expect(result.outcome).toBe('resolved')
expect(mockApi.deleteNamespacedNetworkPolicy).not.toHaveBeenCalled()
+ expect(JSON.stringify(mockCustomApi.patchNamespacedCustomObjectStatus.mock.calls)).toContain(
+ 'FailStatic'
+ )
// Defensive: should a future freeze ever re-persist instead of no-op'ing, any
// write it makes must still carry the accumulated /32 (never a pruned body).
const writes = [
diff --git a/host-context-controller/src/networkPolicyReconciler.test.ts b/host-context-controller/src/networkPolicyReconciler.test.ts
index 8682c09fe..0073d2b46 100644
--- a/host-context-controller/src/networkPolicyReconciler.test.ts
+++ b/host-context-controller/src/networkPolicyReconciler.test.ts
@@ -19,6 +19,7 @@ import {
sameContextDesiredRevision,
} from './networkPolicyReconciler'
import { ContextCRD, McpServerCRD } from './types'
+import { getErrorCode } from './utils'
function deferred<T = void>(): {
promise: Promise<T>
@@ -88,6 +89,7 @@ function makeMockNetworkingApi() {
createNamespacedNetworkPolicy: vi.fn().mockResolvedValue({}),
readNamespacedNetworkPolicy: vi.fn().mockImplementation(async ({ name }) => {
if (
+ name.startsWith('ext-egress-') ||
name === 'allow-desktop-egress-rpc-proxy' ||
name === 'allow-rpc-proxy-to-managed-mcp-servers'
) {
@@ -155,8 +157,14 @@ function makeReconciler(
return { ...response, items }
},
readNamespacedNetworkPolicy: async (args: { name: string; namespace: string }) => {
- const response = await mockApi.readNamespacedNetworkPolicy(args)
const listed = listedPolicies.get(`${args.namespace}/${args.name}`)
+ let response: k8s.V1NetworkPolicy
+ try {
+ response = await mockApi.readNamespacedNetworkPolicy(args)
+ } catch (error: unknown) {
+ if (getErrorCode(error) === 404 && listed) return listed
+ throw error
+ }
return {
...listed,
...response,
@@ -2164,6 +2172,72 @@ describe('NetworkPolicyReconciler', () => {
expect(mockApi.replaceNamespacedNetworkPolicy).not.toHaveBeenCalled()
})
+ it.each(['success', 'transient'] as const)(
+ 'uses a fresh GET before the %s DNS no-op decision and prevents a widened live policy',
+ async mode => {
+ const server: McpServerCRD = {
+ name: `fresh-${mode}-mcp`,
+ namespace: 'mcp-server',
+ spec: {
+ contextRef: 'dev',
+ image: 'fresh:test',
+ transport: { type: 'streamableHttp', url: 'http://fresh:3000', port: 3000 },
+ egressBindings: [{ dns: 'api.example.com', port: 443 }],
+ },
+ }
+ await reconciler.reconcileExternalEgress(server, { isCurrent: () => true })
+ const written = (
+ mockApi.createNamespacedNetworkPolicy.mock.calls[0][0] as {
+ body: k8s.V1NetworkPolicy
+ }
+ ).body
+ const listed = structuredClone(written)
+ const freshWidened = structuredClone(written)
+ freshWidened.metadata = {
+ ...freshWidened.metadata,
+ name: written.metadata!.name!,
+ namespace: 'mcp-server',
+ uid: `fresh-${mode}-uid`,
+ resourceVersion: '22',
+ }
+ freshWidened.spec!.egress!.push({
+ to: [{ ipBlock: { cidr: '8.8.8.8/32' } }],
+ ports: [{ port: 443, protocol: 'TCP' }],
+ })
+ mockApi.listNamespacedNetworkPolicy.mockResolvedValue({ items: [listed] })
+ mockApi.readNamespacedNetworkPolicy.mockResolvedValue(freshWidened)
+ mockApi.createNamespacedNetworkPolicy.mockReset()
+ mockApi.createNamespacedNetworkPolicy.mockRejectedValue({ code: 409 })
+ mockApi.replaceNamespacedNetworkPolicy.mockClear()
+ mockCustomApi.patchNamespacedCustomObjectStatus.mockClear()
+ if (mode === 'transient') {
+ resolve4Mock.mockRejectedValueOnce(
+ Object.assign(new Error('queryA ESERVFAIL api.example.com'), { code: 'ESERVFAIL' })
+ )
+ }
+
+ if (mode === 'transient') {
+ await expect(
+ reconciler.reconcileExternalEgress(server, { isCurrent: () => true })
+ ).rejects.toThrow(/Fail-static proof failed/)
+ expect(mockApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledWith(
+ expect.objectContaining({ name: written.metadata!.name })
+ )
+ expect(
+ JSON.stringify(mockCustomApi.patchNamespacedCustomObjectStatus.mock.calls)
+ ).toContain('ExternalEgressReconcileFailed')
+ } else {
+ await reconciler.reconcileExternalEgress(server, { isCurrent: () => true })
+ const replacement = mockApi.replaceNamespacedNetworkPolicy.mock.calls.at(-1)?.[0]
+ ?.body as k8s.V1NetworkPolicy
+ expect(replacement).toBeDefined()
+ }
+ expect(JSON.stringify(mockApi.replaceNamespacedNetworkPolicy.mock.calls)).not.toContain(
+ '8.8.8.8'
+ )
+ }
+ )
+
it('WRITES on resync when the persisted window is AGING even though the IP set is unchanged (issue #299 M1 renewalDue wiring)', async () => {
const server: McpServerCRD = {
name: 'renew-mcp',
@@ -2607,7 +2681,16 @@ describe('NetworkPolicyReconciler', () => {
it('deletes only orphaned external egress policies after applying the desired set', async () => {
mockApi.listNamespacedNetworkPolicy.mockResolvedValue({
items: [
- { metadata: { name: 'ext-egress-openai-mcp-api.openai.com-443' } },
+ {
+ metadata: {
+ name: 'ext-egress-openai-mcp-api.openai.com-443',
+ labels: {
+ [MANAGED_BY_LABEL]: MANAGED_BY_VALUE,
+ [POLICY_TYPE_LABEL]: 'external-egress',
+ [MCPSERVER_LABEL]: 'openai-mcp',
+ },
+ },
+ },
{ metadata: { name: 'ext-egress-openai-mcp-old-example-com-443' } },
],
})
@@ -2637,7 +2720,16 @@ describe('NetworkPolicyReconciler', () => {
it('fails closed when an orphaned external egress policy cannot be deleted', async () => {
mockApi.listNamespacedNetworkPolicy.mockResolvedValue({
items: [
- { metadata: { name: 'ext-egress-openai-mcp-api.openai.com-443' } },
+ {
+ metadata: {
+ name: 'ext-egress-openai-mcp-api.openai.com-443',
+ labels: {
+ [MANAGED_BY_LABEL]: MANAGED_BY_VALUE,
+ [POLICY_TYPE_LABEL]: 'external-egress',
+ [MCPSERVER_LABEL]: 'openai-mcp',
+ },
+ },
+ },
{ metadata: { name: 'ext-egress-openai-mcp-old-example-com-443' } },
],
})
@@ -2671,7 +2763,16 @@ describe('NetworkPolicyReconciler', () => {
)
mockApi.listNamespacedNetworkPolicy.mockResolvedValue({
items: [
- { metadata: { name: 'ext-egress-openai-mcp-api.openai.com-443' } },
+ {
+ metadata: {
+ name: 'ext-egress-openai-mcp-api.openai.com-443',
+ labels: {
+ [MANAGED_BY_LABEL]: MANAGED_BY_VALUE,
+ [POLICY_TYPE_LABEL]: 'external-egress',
+ [MCPSERVER_LABEL]: 'openai-mcp',
+ },
+ },
+ },
{ metadata: { name: 'ext-egress-openai-mcp-old-example-com-443' } },
],
})
diff --git a/host-context-controller/src/networkPolicyReconciler.ts b/host-context-controller/src/networkPolicyReconciler.ts
index c5ddb82b6..9ffb525bf 100644
--- a/host-context-controller/src/networkPolicyReconciler.ts
+++ b/host-context-controller/src/networkPolicyReconciler.ts
@@ -17,10 +17,12 @@ import * as k8s from '@kubernetes/client-node'
import * as dns from 'node:dns/promises'
import { isIP } from 'node:net'
import {
+ RESOLVED_AT_ANNOTATION,
STATE_ANNOTATION,
- classifyDnsError,
- dnsCodeOf,
- isPermanentDnsCode,
+ TARGETS_ANNOTATION,
+ classifyDnsRejection,
+ parseStateStrict,
+ serializeState,
} from '@clerum/network-policy-core'
import { config } from './config'
import {
@@ -32,6 +34,7 @@ import {
POLICY_TYPE_LABEL,
} from './constants'
import { accumulateHostExactHostEgress } from './externalEgressAccumulator'
+import { hccLogger } from './logger'
import {
confirmAuthoritativeMcpServerAbsence,
sameMcpServerDesiredRevision,
@@ -148,9 +151,21 @@ function sameNetworkPolicySpec(
*/
type LiveEgressVerdict =
| { state: 'none' }
- | { state: 'intact'; cidrs: string[] }
- | { state: 'drifted-shape' }
- | { state: 'drifted-cidr' }
+ | { state: 'foreign' }
+ | { state: 'intact'; cidrs: string[]; resolvedAt: string; policy: k8s.V1NetworkPolicy }
+ | { state: 'drifted-shape'; policy: k8s.V1NetworkPolicy }
+ | { state: 'drifted-cidr'; policy: k8s.V1NetworkPolicy }
+ | { state: 'untrusted-state'; policy: k8s.V1NetworkPolicy }
+
+type FreshExternalEgressPolicy = {
+ policy: k8s.V1NetworkPolicy | null
+ repairRequired: boolean
+}
+
+type ProvenDnsAnnotations = {
+ annotations: Record<string, string> | undefined
+ contracted: boolean
+}
/**
* What the ExternalEgressReady message says about the live policy on a non-DNS
@@ -159,10 +174,12 @@ type LiveEgressVerdict =
*/
const LIVE_EGRESS_FAULT_TEXT: Record<LiveEgressVerdict['state'], string> = {
none: 'no live policy to serve',
- intact: 'accumulated egress left untouched',
+ foreign: 'homonymous live policy is not HCC-owned and was left untouched',
+ intact: 'verified policy retained for existing pods; runtime remains blocked',
'drifted-cidr':
'live policy REVOKED: it enforced a CIDR outside the /32 allowlist this controller writes',
'drifted-shape': 'live policy REVOKED: its egress shape is not one this controller writes',
+ 'untrusted-state': 'live policy REVOKED: its DNS provenance was absent or invalid',
}
function hasExpectedPolicyOwnership(policy: k8s.V1NetworkPolicy, policyType: string): boolean {
@@ -1869,19 +1886,64 @@ export class NetworkPolicyReconciler {
* A rebuild-and-compare has no dimension to miss: anything the controller does
* not write differs from the rebuild, whatever it happens to allow.
*/
- private judgeLiveExactHostEgress(
+ private async judgeLiveExactHostEgress(
+ server: McpServerCRD,
+ name: string,
+ binding: EgressBinding
+ ): Promise<LiveEgressVerdict> {
+ let existingPolicy: k8s.V1NetworkPolicy
+ try {
+ existingPolicy = await this.networkingApi.readNamespacedNetworkPolicy({
+ name,
+ namespace: server.namespace,
+ })
+ } catch (error: unknown) {
+ if (getErrorCode(error) === 404) return { state: 'none' }
+ throw error
+ }
+ return this.judgeExactHostEgressPolicy(server, name, binding, existingPolicy)
+ }
+
+ private judgeExactHostEgressPolicy(
server: McpServerCRD,
name: string,
binding: EgressBinding,
- existingPolicy: k8s.V1NetworkPolicy | null
+ existingPolicy: k8s.V1NetworkPolicy
): LiveEgressVerdict {
- // Bootstrap: nothing live to judge, and nothing to serve.
- if (existingPolicy === null) return { state: 'none' }
+ const metadata = existingPolicy.metadata
+ const labels = metadata?.labels ?? {}
+ if (
+ metadata?.name !== name ||
+ metadata.namespace !== server.namespace ||
+ !metadata.uid ||
+ !metadata.resourceVersion
+ ) {
+ return { state: 'untrusted-state', policy: existingPolicy }
+ }
+ const ownsExpectedLane =
+ hasExpectedPolicyOwnership(existingPolicy, EXTERNAL_EGRESS_POLICY_TYPE) &&
+ labels[MCPSERVER_LABEL] === server.name
+ if (!ownsExpectedLane) return { state: 'foreign' }
- const cidrs = (existingPolicy.spec?.egress ?? [])
- .flatMap(rule => rule.to ?? [])
- .map(peer => peer.ipBlock?.cidr)
- .filter((cidr): cidr is string => typeof cidr === 'string')
+ const stateRead = parseStateStrict(
+ existingPolicy.metadata?.annotations,
+ {
+ overlapMs: config.externalEgressOverlapSec * 1000,
+ maxEntries: config.externalEgressMaxEntries,
+ },
+ [
+ {
+ fqdn: binding.dns!,
+ port: binding.port!,
+ protocol: binding.protocol ?? 'TCP',
+ },
+ ]
+ )
+ if (stateRead.kind !== 'valid' || stateRead.state.entries.length === 0) {
+ return { state: 'untrusted-state', policy: existingPolicy }
+ }
+
+ const cidrs = stateRead.state.entries.map(entry => `${entry.ip}/32`)
// A policy with no ipBlock peers is a deny disguised as an allow, and the
// rebuild from an empty CIDR list would compare equal to it. Gate first, the
@@ -1898,10 +1960,130 @@ export class NetworkPolicyReconciler {
cidr => cidr.endsWith('/32') && isAllowedExternalEgressCidr(cidr)
)
- if (shapeIntact && cidrsIntact) return { state: 'intact', cidrs }
+ if (shapeIntact && cidrsIntact) {
+ const annotatedResolvedAt = existingPolicy.metadata?.annotations?.[RESOLVED_AT_ANNOTATION]
+ const latestObservedAt = Math.max(
+ ...stateRead.state.entries.map(entry => entry.lastObservedAt)
+ )
+ const resolvedAt =
+ annotatedResolvedAt && Number.isFinite(Date.parse(annotatedResolvedAt))
+ ? annotatedResolvedAt
+ : new Date(latestObservedAt).toISOString()
+ return { state: 'intact', cidrs, resolvedAt, policy: existingPolicy }
+ }
// A bad CIDR is the more specific diagnosis and the one an operator acts on
// first, so it wins when a policy drifted on both axes.
- return { state: cidrsIntact ? 'drifted-shape' : 'drifted-cidr' }
+ return {
+ state: cidrsIntact ? 'drifted-shape' : 'drifted-cidr',
+ policy: existingPolicy,
+ }
+ }
+
+ /**
+ * Rehydrate a successful DNS observation only from structured entries that
+ * were already enforced by the fresh live spec. STATE is provenance, not an
+ * alternate allowlist: annotation-only entries are discarded before the
+ * accumulator can union them into the next policy. A fresh DNS answer may
+ * then add only the addresses it actually returned.
+ */
+ private provenPreviousDnsAnnotations(
+ server: McpServerCRD,
+ name: string,
+ binding: EgressBinding,
+ existingPolicy: k8s.V1NetworkPolicy | null
+ ): ProvenDnsAnnotations {
+ if (!existingPolicy) return { annotations: undefined, contracted: false }
+ const empty = serializeState({ entries: [] })
+ const finish = (annotations: Record<string, string>): ProvenDnsAnnotations => {
+ const current = existingPolicy.metadata?.annotations ?? {}
+ return {
+ annotations,
+ contracted: [STATE_ANNOTATION, TARGETS_ANNOTATION, RESOLVED_AT_ANNOTATION].some(
+ key => current[key] !== annotations[key]
+ ),
+ }
+ }
+ const stateRead = parseStateStrict(
+ existingPolicy.metadata?.annotations,
+ {
+ overlapMs: config.externalEgressOverlapSec * 1000,
+ maxEntries: config.externalEgressMaxEntries,
+ },
+ [
+ {
+ fqdn: binding.dns!,
+ port: binding.port!,
+ protocol: binding.protocol ?? 'TCP',
+ },
+ ]
+ )
+ if (stateRead.kind !== 'valid') return finish(empty)
+
+ const expectedShape = this.buildExactHostEgressPolicy(server, name, binding, [])
+ const liveShape: k8s.V1NetworkPolicy = {
+ ...existingPolicy,
+ spec: { ...existingPolicy.spec!, egress: [] },
+ }
+ if (!sameNetworkPolicySpec(liveShape, expectedShape)) return finish(empty)
+
+ const liveRules = new Set(
+ (existingPolicy.spec?.egress ?? []).map(rule =>
+ JSON.stringify(canonicalizeNetworkPolicyValue(rule))
+ )
+ )
+ const entries = stateRead.state.entries.filter(entry => {
+ const expectedRule = this.buildExactHostEgressPolicy(server, name, binding, [
+ `${entry.ip}/32`,
+ ]).spec?.egress?.[0]
+ return (
+ expectedRule !== undefined &&
+ liveRules.has(JSON.stringify(canonicalizeNetworkPolicyValue(expectedRule)))
+ )
+ })
+ const annotations = serializeState({ entries })
+ const priorResolvedAt = existingPolicy.metadata?.annotations?.[RESOLVED_AT_ANNOTATION]
+ if (entries.length > 0 && priorResolvedAt && Number.isFinite(Date.parse(priorResolvedAt))) {
+ annotations[RESOLVED_AT_ANNOTATION] = priorResolvedAt
+ }
+ return finish(annotations)
+ }
+
+ /**
+ * Re-read the exact external-egress object after DNS latency. LIST is only a
+ * discovery/cleanup snapshot; it cannot authorize a no-op or status claim.
+ * A single missing HCC marker remains repairable under the existing inventory
+ * contract, but foreign/ambiguous ownership and incomplete identity fail loud.
+ */
+ private async readFreshExternalEgressPolicy(
+ server: McpServerCRD,
+ name: string
+ ): Promise<FreshExternalEgressPolicy> {
+ let policy: k8s.V1NetworkPolicy
+ try {
+ policy = await this.networkingApi.readNamespacedNetworkPolicy({
+ name,
+ namespace: server.namespace,
+ })
+ } catch (error: unknown) {
+ if (getErrorCode(error) === 404) return { policy: null, repairRequired: false }
+ throw error
+ }
+
+ const classification = classifySafetyInventoryPolicy(policy, 'external-egress')
+ const metadata = policy.metadata
+ if (
+ metadata?.name !== name ||
+ metadata.namespace !== server.namespace ||
+ !metadata.uid ||
+ !metadata.resourceVersion ||
+ metadata.labels?.[MCPSERVER_LABEL] !== server.name ||
+ (classification !== 'owned' && classification !== 'repairable')
+ ) {
+ throw new Error(
+ `NetworkPolicy "${name}" has conflicting or incomplete ownership for the external-egress lane`
+ )
+ }
+ return { policy, repairRequired: classification === 'repairable' }
}
private async resolveExternalEgressDns(
@@ -1918,10 +2100,9 @@ export class NetworkPolicyReconciler {
timeout = setTimeout(
() =>
// A bounded-timeout DNS failure is a TRANSIENT resolver error, not a
- // permanent one. classifyDnsError keys on `.code`, and a codeless Error
- // falls into the 'permanent' (NXDOMAIN/expire) branch, which prunes the
- // accumulated egress instead of freezing it — breaking D4 fail-static.
- // Tag it ETIMEDOUT (already in TRANSIENT_DNS_CODES) so it freezes.
+ // permanent one. The resolver-boundary verdict keys on `.code`;
+ // tag the bounded timeout ETIMEDOUT so it is positively transient
+ // and freezes the accumulated set.
reject(
Object.assign(
new Error(
@@ -2164,11 +2345,56 @@ export class NetworkPolicyReconciler {
const reconcileFaults: Array<{
dns: string
err: unknown
- blocking: boolean
live: LiveEgressVerdict['state']
+ verificationError?: unknown
}> = []
+ const freshPolicySnapshots = new Map<string, k8s.V1NetworkPolicy>()
const resolvedAtMs = Date.now()
const resolvedAt = new Date(resolvedAtMs).toISOString()
+ let dnsFailStatic = false
+
+ const recordReconcileFault = async (
+ binding: EgressBinding,
+ name: string,
+ err: unknown
+ ): Promise<void> => {
+ if (!binding.dns) throw new Error('DNS reconcile fault requires a DNS binding')
+ hccLogger.error('external egress reconcile fault is not a DNS condition', {
+ serverName: server.name,
+ dns: binding.dns,
+ err,
+ })
+
+ let verdict: LiveEgressVerdict
+ try {
+ verdict = await this.judgeLiveExactHostEgress(server, name, binding)
+ } catch (verificationError: unknown) {
+ hccLogger.error('external egress live policy verification failed', {
+ serverName: server.name,
+ dns: binding.dns,
+ err: verificationError,
+ })
+ reconcileFaults.push({
+ dns: binding.dns,
+ err,
+ live: 'untrusted-state',
+ verificationError,
+ })
+ return
+ }
+
+ if (verdict.state === 'intact') {
+ desiredPolicyNames.add(name)
+ resolvedEgressIPs.push({
+ dns: binding.dns,
+ ips: verdict.cidrs.map(cidr => cidr.replace(/\/32$/, '')),
+ resolvedAt: verdict.resolvedAt,
+ })
+ } else if ('policy' in verdict) {
+ freshPolicySnapshots.set(name, verdict.policy)
+ }
+ reconcileFaults.push({ dns: binding.dns, err, live: verdict.state })
+ }
for (const binding of bindings) {
if (!isCurrent()) return
@@ -2178,7 +2404,7 @@ export class NetworkPolicyReconciler {
continue
}
- let cidrs: string[]
+ let cidrs: string[] = []
// Sliding-window state annotations to persist on the policy (issue #299).
// Only the DNS branch sets this; CIDR/public-web bindings leave it undefined.
let stateAnnotations: Record<string, string> | undefined
@@ -2191,6 +2417,9 @@ export class NetworkPolicyReconciler {
// accumulated.changed is over (fqdn,ip,port,protocol), so it forces the
// write that re-persists the identity and preserves the overlap grace.
let egressStateChanged = false
+ let resolvedStatus: McpServerResolvedEgressIP | undefined
+ let bindingDnsFailStatic = false
+ let ownershipRepairRequired = false
const name = this.externalEgressPolicyName(server.name, binding)
if (!name) {
failures.push('exact-host external egress bindings must declare dns or cidr')
@@ -2200,7 +2429,7 @@ export class NetworkPolicyReconciler {
// AND to decide whether a write is actually needed (audit F2/L1) — the
// write is gated on the ENFORCED spec.egress, so identical static policies
// never churn and out-of-band drift self-heals, for every egress class.
- const existingPolicy = existingPolicies.find(p => p.metadata?.name === name) ?? null
+ let existingPolicy = existingPolicies.find(p => p.metadata?.name === name) ?? null
if (egressClass === 'public-web') {
if (
@@ -2215,16 +2444,18 @@ export class NetworkPolicyReconciler {
continue
}
- desiredPolicyNames.add(name)
-
// public-web is a static policy. Preserve #205's ownership-fenced write
// (applyOwnedPolicy asserts ownership + honors the isCurrent generation
// fence) while keeping dev's F2 no-churn gate so the TTL-accelerated
// resync does not rewrite it every tick.
const policy = this.buildPublicWebEgressPolicy(server, name)
- if (this.externalEgressWriteNeeded(existingPolicy, policy)) {
+ const fresh = await this.readFreshExternalEgressPolicy(server, name)
+ existingPolicy = fresh.policy
+ if (fresh.repairRequired || this.externalEgressWriteNeeded(existingPolicy, policy)) {
await this.applyOwnedPolicy(name, server.namespace, policy, 'external-egress', isCurrent)
}
+ if (!isCurrent()) return
+ desiredPolicyNames.add(name)
continue
}
@@ -2264,217 +2495,199 @@ export class NetworkPolicyReconciler {
// resolver failure we fail-static: keep serving the accumulated set
// rather than deleting the policy (the pre-#299 catch dropped egress on
// any DNS hiccup).
- const previousAnnotations = existingPolicy?.metadata?.annotations
const now = Date.parse(resolvedAt)
const coreConfig = {
overlapMs: config.externalEgressOverlapSec * 1000,
maxEntries: config.externalEgressMaxEntries,
}
+ let records: Array<{ address: string; ttl: number }> | undefined
+ let resolverFailure: { error: unknown; kind: 'transient' | 'permanent' } | undefined
try {
- const records = await this.resolveExternalEgressDns(binding.dns)
- if (!isCurrent()) return
- const uniqueIps = [...new Set(records.map(r => r.address))].sort()
- if (uniqueIps.length === 0) {
- failures.push(`hostname "${binding.dns}" resolved to no IPv4 addresses`)
+ records = await this.resolveExternalEgressDns(binding.dns)
+ } catch (resolverError: unknown) {
+ const verdict = classifyDnsRejection(resolverError)
+ if (verdict.kind === 'fault') {
+ await recordReconcileFault(binding, name, resolverError)
continue
}
- const invalidIps = uniqueIps.filter(ip => isIP(ip) !== 4)
- if (invalidIps.length > 0) {
- failures.push(
- `hostname "${binding.dns}" resolved invalid IPv4 answer(s): ${invalidIps.join(', ')}`
- )
- continue
+ resolverFailure = {
+ error: resolverError,
+ kind: verdict.kind === 'transient' ? 'transient' : 'permanent',
}
- const freshCidrs = uniqueIps.map(ip => `${ip}/32`)
- const disallowedCidrs = freshCidrs.filter(cidr => !isAllowedExternalEgressCidr(cidr))
- if (disallowedCidrs.length > 0) {
+ }
+
+ let fresh: FreshExternalEgressPolicy
+ try {
+ fresh = await this.readFreshExternalEgressPolicy(server, name)
+ } catch (policyReadError: unknown) {
+ await recordReconcileFault(binding, name, policyReadError)
+ continue
+ }
+ existingPolicy = fresh.policy
+ ownershipRepairRequired = fresh.repairRequired
+ const liveAnnotations = existingPolicy?.metadata?.annotations
+
+ if (resolverFailure) {
+ if (fresh.policy) freshPolicySnapshots.set(name, fresh.policy)
+ if (resolverFailure.kind === 'permanent') {
failures.push(
- `hostname "${binding.dns}" resolved disallowed address(es): ${disallowedCidrs.join(', ')}`
+ `failed to resolve hostname "${binding.dns}": ${this.errorMessage(resolverFailure.error)}`
)
continue
}
- // Security gates above ran on the FRESH snapshot; only validated IPs
- // enter the accumulator, so the union can never contain a blocked IP.
- const ttlSeconds = records.length > 0 ? Math.min(...records.map(r => r.ttl)) : 0
- if (ttlSeconds > 0) {
- this.externalEgressMinObservedTtlMs = Math.min(
- this.externalEgressMinObservedTtlMs,
- ttlSeconds * 1000
- )
+ if (existingPolicy) {
+ const proof = this.judgeExactHostEgressPolicy(server, name, binding, existingPolicy)
+ if (proof.state !== 'intact') {
+ reconcileFaults.push({
+ dns: binding.dns,
+ err: new Error(
+ `Fail-static proof failed for "${binding.dns}": ${LIVE_EGRESS_FAULT_TEXT[proof.state]}`
+ ),
+ live: proof.state,
+ })
+ continue
+ }
}
const accumulated = accumulateHostExactHostEgress({
fqdn: binding.dns,
port: binding.port,
protocol,
- resolution: { kind: 'ok', ips: uniqueIps, ttlSeconds },
- previousAnnotations,
+ resolution: { kind: 'transient' },
+ previousAnnotations: liveAnnotations,
now,
config: coreConfig,
})
- // Defense-in-depth (audit M3): rehydrated IPs (from the policy's own
- // annotations) bypass the fresh-snapshot CIDR gate, so a tampered/corrupt
- // state annotation could union a blocked/private IP. Re-validate the
- // effective set against the blocked ranges before rendering.
+ if (accumulated.cidrs.length === 0) {
+ failures.push(
+ `failed to resolve hostname "${binding.dns}": ${this.errorMessage(resolverFailure.error)}`
+ )
+ continue
+ }
cidrs = accumulated.cidrs.filter(c => isAllowedExternalEgressCidr(c))
+ if (cidrs.length !== accumulated.cidrs.length) {
+ failures.push(
+ `persisted DNS state for hostname "${binding.dns}" contains a disallowed address`
+ )
+ continue
+ }
stateAnnotations = accumulated.annotations
egressRenewalDue = accumulated.renewalDue
egressStateChanged = accumulated.changed
- resolvedEgressIPs.push({
+ bindingDnsFailStatic = true
+ if (accumulated.resolvedAt) {
+ resolvedStatus = {
+ dns: binding.dns,
+ ips: cidrs.map(cidr => cidr.replace(/\/32$/, '')),
+ resolvedAt: accumulated.resolvedAt,
+ }
+ }
+ hccLogger.warn('external egress DNS failure', {
+ serverName: server.name,
dns: binding.dns,
- ips: cidrs.map(cidr => cidr.replace(/\/32$/, '')),
- resolvedAt,
+ kind: resolverFailure.kind,
+ accumulatedIps: cidrs.length,
+ err: resolverFailure.error,
})
- if (accumulated.overCap) {
- console.warn(
- `[NetPol] "${binding.dns}" egress set hit the cap (${config.externalEgressMaxEntries}); evicted ${accumulated.evicted} least-recently-observed IP(s)`
+ }
+
+ if (records) {
+ try {
+ if (!isCurrent()) return
+ const uniqueIps = [...new Set(records.map(record => record.address))].sort()
+ if (uniqueIps.length === 0) {
+ if (existingPolicy) freshPolicySnapshots.set(name, existingPolicy)
+ failures.push(`hostname "${binding.dns}" resolved to no IPv4 addresses`)
+ continue
+ }
+ const invalidIps = uniqueIps.filter(ip => isIP(ip) !== 4)
+ if (invalidIps.length > 0) {
+ if (existingPolicy) freshPolicySnapshots.set(name, existingPolicy)
+ failures.push(
+ `hostname "${binding.dns}" resolved invalid IPv4 answer(s): ${invalidIps.join(', ')}`
+ )
+ continue
+ }
+ const freshCidrs = uniqueIps.map(ip => `${ip}/32`)
+ const disallowedCidrs = freshCidrs.filter(cidr => !isAllowedExternalEgressCidr(cidr))
+ if (disallowedCidrs.length > 0) {
+ if (existingPolicy) freshPolicySnapshots.set(name, existingPolicy)
+ failures.push(
+ `hostname "${binding.dns}" resolved disallowed address(es): ${disallowedCidrs.join(', ')}`
+ )
+ continue
+ }
+ const ttlSeconds = Math.min(...records.map(record => record.ttl))
+ if (ttlSeconds > 0) {
+ this.externalEgressMinObservedTtlMs = Math.min(
+ this.externalEgressMinObservedTtlMs,
+ ttlSeconds * 1000
+ )
+ }
+ const provenPrevious = this.provenPreviousDnsAnnotations(
+ server,
+ name,
+ binding,
+ existingPolicy
)
- }
- console.log(`[NetPol] Resolved ${binding.dns} → [${cidrs.join(', ')}]`)
- } catch (err) {
- // Fail-static (H1): freeze the accumulated set on a transient resolver
- // failure; fail loud when there is nothing to serve (bootstrap, or a
- // permanent no-records answer).
- //
- // Issue #513: only a POSITIVELY identified DNS condition may reach the
- // sliding-window classifier. classifyDnsError defaults every unknown or
- // missing code to 'permanent' (network-policy-core index.cjs), and
- // 'permanent' PRUNES the accumulated set rather than freezing it. This
- // try covers the whole resolve-and-render block, so without this gate a
- // programming fault raised after resolution — a TypeError from a
- // contract violation, an ERR_* argument error — is laundered into the
- // prune branch and reported to the operator as a DNS failure, pointing
- // at the resolver while the fault is here. Transient codes are positive
- // by construction (TRANSIENT_DNS_CODES); 'permanent' is positive only for
- // PERMANENT_DNS_CODES — the two no-records answers plus EBADNAME, a name
- // c-ares refuses to query. EBADNAME cannot arrive from this controller's
- // own CRD field (isPublicDnsHostname rejects `..` and 64-octet labels
- // before we resolve), but the set is shared with WRC, whose ui lane
- // validates the fqdn with the CRD pattern alone; there it IS the
- // operator's misconfiguration, and it must keep pruning and reporting
- // ExternalEgressRejected rather than blame the controller.
- //
- // `dnsCodeOf` reads the code off any object, so a rejection that is not
- // an Error instance is still classified by its code rather than by how
- // it was thrown. It is deliberately not full parity with
- // classifyDnsError, which also accepts a bare code string: a rejection
- // whose value IS the string 'ENOTFOUND' classifies permanent there and
- // yields no code here, so it lands in the fault branch. `node:dns`
- // rejects with Error objects, so that shape does not occur in
- // production. Shared with WRC rather than unwrapped by hand in each
- // process — two hand-written extractions of the same field is the drift
- // surface network-policy-core exists to remove (#567 review, R1-M2).
- const kind = classifyDnsError(err)
- const code = dnsCodeOf(err)
- if (kind !== 'transient' && !isPermanentDnsCode(code)) {
- // Log before anything that can throw. writeExternalEgressStatus fails
- // on a stale uid/generation or a rejected patch, and the isCurrent
- // fence below returns without a word — either would swallow the very
- // fault this branch exists to surface.
- console.error(
- `[NetPol] External egress reconcile fault on binding "${binding.dns}" for "${server.name}" (not a DNS condition):`,
- err
+ const accumulated = accumulateHostExactHostEgress({
+ fqdn: binding.dns,
+ port: binding.port,
+ protocol,
+ resolution: { kind: 'ok', ips: uniqueIps, ttlSeconds },
+ previousAnnotations: provenPrevious.annotations,
+ now,
+ config: coreConfig,
+ })
+ const allowedEntries = accumulated.entries.filter(entry =>
+ isAllowedExternalEgressCidr(`${entry.ip}/32`)
)
- // Recorded, not thrown: the throw is deferred to after the loop so
- // cleanupExternalEgress still runs. It is the only place that deletes
- // policies for bindings the operator removed from the spec, and
- // desiredPolicyNames is only complete once the loop ends. Throwing
- // here would leave a de-authorized destination allowed for as long as
- // the fault persists — fail-loud paid for with fail-open.
- //
- // D4 fail-static, applied to a fault exactly as the transient branch
- // applies it to a resolver outage: when there is a live policy, its
- // rules are what is enforced right now, so report them and let the
- // HCC-managed Deployment start on the last known-good egress. Read
- // from the live object, not recomputed — recomputing is what faulted.
- // Only HCC: consumers of ExternalEgressReady treat any False as
- // terminal, so a dependent WRC recipe still fails (#577).
- const verdict = this.judgeLiveExactHostEgress(server, name, binding, existingPolicy)
-
- // Membership in `desiredPolicyNames` is the difference between "left
- // untouched" and "revoked": cleanup deletes every external-egress
- // policy NOT in this set, and it runs BEFORE the deferred throw, so a
- // revocation actually happens rather than being skipped by the
- // fail-loud exit. Revoking is fail-CLOSED — these are pure allow
- // policies under the namespace L0 deny-all, so removing one closes
- // that destination and cannot open one.
- //
- // Nothing here can rewrite a drifted policy instead: rebuilding means
- // recomputing the CIDR set, and recomputing is what faulted.
- //
- // `none` is bootstrap, and there the add is dataplane-NEUTRAL: the
- // policy came from `existingPolicies.find(...)`, cleanup filters that
- // same list by name, and a name that was not found has no live object
- // to delete. It stays explicit because the status has to tell "nothing
- // was live" apart from "what was live got revoked" — an earlier
- // revision of this branch asserted the opposite in a comment.
- const revoked = verdict.state === 'drifted-shape' || verdict.state === 'drifted-cidr'
- if (!revoked) desiredPolicyNames.add(name)
-
- if (verdict.state === 'intact') {
- // Without this the binding vanishes from status.resolvedEgressIPs —
- // the audit record an operator reads to learn what is allowed —
- // while its /32s stay enforced. The status message would claim
- // "accumulated egress left untouched" and the record would disagree.
- resolvedEgressIPs.push({
+ cidrs = allowedEntries.map(entry => `${entry.ip}/32`)
+ stateAnnotations =
+ allowedEntries.length === accumulated.entries.length
+ ? accumulated.annotations
+ : {
+ ...serializeState({ entries: allowedEntries }),
+ ...(accumulated.resolvedAt
+ ? { [RESOLVED_AT_ANNOTATION]: accumulated.resolvedAt }
+ : {}),
+ }
+ egressRenewalDue = accumulated.renewalDue
+ egressStateChanged =
+ accumulated.changed ||
+ provenPrevious.contracted ||
+ allowedEntries.length !== accumulated.entries.length
+ resolvedStatus = {
+ dns: binding.dns,
+ ips: cidrs.map(cidr => cidr.replace(/\/32$/, '')),
+ resolvedAt,
+ }
+ if (accumulated.overCap) {
+ hccLogger.warn('external egress set hit the cap', {
+ serverName: server.name,
dns: binding.dns,
- ips: verdict.cidrs.map(cidr => cidr.replace(/\/32$/, '')),
- resolvedAt,
+ maxEntries: config.externalEgressMaxEntries,
+ evicted: accumulated.evicted,
})
}
- reconcileFaults.push({
- // Inside `else if (binding.dns)`, so it is always a string here;
- // a `??` fallback would be an unreachable branch pretending to be a guard.
+ hccLogger.info('external egress DNS resolved', {
+ serverName: server.name,
dns: binding.dns,
- err,
- // Blocking unless there is a trustworthy live set to serve. A fault
- // on a binding that has never been written has no frozen set behind
- // it, so completing the pass would publish a server as reconciled
- // while one of its declared destinations was never enforced at all.
- blocking: verdict.state !== 'intact',
- // The operator message reads off this. A status that describes the
- // opposite of what the pass did is the same defect class this branch
- // exists to close.
- live: verdict.state,
+ cidrs,
})
+ } catch (processingError: unknown) {
+ await recordReconcileFault(binding, name, processingError)
continue
}
- const accumulated = accumulateHostExactHostEgress({
- fqdn: binding.dns,
- port: binding.port,
- protocol,
- resolution: { kind },
- previousAnnotations,
- now,
- config: coreConfig,
- })
- if (accumulated.cidrs.length === 0) {
- failures.push(`failed to resolve hostname "${binding.dns}": ${this.errorMessage(err)}`)
- continue
- }
- // Defense-in-depth (audit M3): rehydrated IPs (from the policy's own
- // annotations) bypass the fresh-snapshot CIDR gate, so a tampered/corrupt
- // state annotation could union a blocked/private IP. Re-validate the
- // effective set against the blocked ranges before rendering.
- cidrs = accumulated.cidrs.filter(c => isAllowedExternalEgressCidr(c))
- stateAnnotations = accumulated.annotations
- // A freeze keeps the SAME set as the live policy → changed=false → no
- // write (no churn while DNS is failing); a genuine change still writes.
- egressRenewalDue = accumulated.renewalDue
- egressStateChanged = accumulated.changed
- resolvedEgressIPs.push({
- dns: binding.dns,
- ips: cidrs.map(cidr => cidr.replace(/\/32$/, '')),
- resolvedAt,
- })
- console.warn(
- `[NetPol] ${kind} DNS failure for "${binding.dns}"; serving ${cidrs.length} accumulated IP(s) (fail-static, issue #299): ${this.errorMessage(err)}`
- )
}
} else {
continue
}
- desiredPolicyNames.add(name)
+ if (!binding.dns) {
+ const fresh = await this.readFreshExternalEgressPolicy(server, name)
+ existingPolicy = fresh.policy
+ ownershipRepairRequired = fresh.repairRequired
+ }
// Build the exact-host policy from #205's helper, then fold in the #299
// sliding-window state annotations so the next reconcile rehydrates the
@@ -2483,6 +2696,7 @@ export class NetworkPolicyReconciler {
if (stateAnnotations) {
policy.metadata = { ...(policy.metadata ?? {}), annotations: stateAnnotations }
}
+ if (existingPolicy) freshPolicySnapshots.set(name, existingPolicy)
// Write only when the enforced rules changed / the policy is new / drifted
// (audit F2 no-churn + L1 self-heal), OR when the DNS window is aging and
@@ -2490,6 +2704,7 @@ export class NetworkPolicyReconciler {
// Route the write through applyOwnedPolicy so #205's ownership assertion
// and isCurrent generation fence still guard the mutation.
if (
+ ownershipRepairRequired ||
this.externalEgressWriteNeeded(existingPolicy, policy) ||
egressRenewalDue ||
egressStateChanged
@@ -2498,16 +2713,32 @@ export class NetworkPolicyReconciler {
} else {
console.log(`[NetPol] External egress "${name}" unchanged — no-op (issue #299 no-churn)`)
}
+ if (!isCurrent()) return
+ desiredPolicyNames.add(name)
+ if (resolvedStatus) resolvedEgressIPs.push(resolvedStatus)
+ if (bindingDnsFailStatic) dnsFailStatic = true
+ }
+
+ const cleanupCandidates = existingPolicies
+ .filter(policy => {
+ const name = policy.metadata?.name
+ return Boolean(name) && !desiredPolicyNames.has(name!)
+ })
+ .map(policy => freshPolicySnapshots.get(policy.metadata?.name ?? '') ?? policy)
+ for (const [name, snapshot] of freshPolicySnapshots) {
+ if (
+ !desiredPolicyNames.has(name) &&
+ !cleanupCandidates.some(policy => policy.metadata?.name === name)
+ ) {
+ cleanupCandidates.push(snapshot)
+ }
}
try {
await this.cleanupExternalEgress(
server.name,
server.namespace,
- existingPolicies.filter(policy => {
- const name = policy.metadata?.name
- return Boolean(name) && !desiredPolicyNames.has(name!)
- }),
+ cleanupCandidates,
async () => isCurrent(),
options.onRevoked
)
@@ -2531,12 +2762,7 @@ export class NetworkPolicyReconciler {
// Reported only after cleanupExternalEgress above, so a binding removed from
// the spec is still de-authorized on a pass that ends in a fault.
if (reconcileFaults.length > 0) {
- // The blocking fault is the one that propagates, so it is the one the
- // status must name: reporting reconcileFaults[0] while throwing a different
- // error hands the operator a condition and a log line about two different
- // bindings. With no blocking fault, the first is the whole story.
- const blocking = reconcileFaults.find(fault => fault.blocking)
- const first = blocking ?? reconcileFaults[0]
+ const first = reconcileFaults[0]
const parts = [
`External egress reconciliation failed on binding "${first.dns}" with a non-DNS error ` +
`(not a resolver condition, ${LIVE_EGRESS_FAULT_TEXT[first.live]}): ${this.errorMessage(
@@ -2546,6 +2772,11 @@ export class NetworkPolicyReconciler {
if (reconcileFaults.length > 1) {
parts.push(`and ${reconcileFaults.length - 1} more binding(s) faulted`)
}
+ if (first.verificationError) {
+ parts.push(
+ `live policy verification also failed: ${this.errorMessage(first.verificationError)}`
+ )
+ }
// `failures` has exactly one sink — the ExternalEgressRejected message
// below — which this branch used to make unreachable. A pass carrying both
// then lost the operator's own config error completely, while cleanup had
@@ -2560,20 +2791,10 @@ export class NetworkPolicyReconciler {
parts.join('. '),
isCurrent
)
- // Fail-static, not fail-stop. The throw reaches k8sClient's catch, which
- // returns before the managed Deployment is created — so throwing for a
- // fault that HAS a frozen set to serve takes a whole server's pod down over
- // one binding, healthy siblings included, on a retry ladder that cannot fix
- // a deterministic fault. Throw only when nothing could be served.
- if (blocking) throw blocking.err
- // Rejected input stays blocking as it always was, now that it has been said
- // out loud in the message above.
- if (failures.length > 0) {
- throw new Error(
- `External egress reconciliation failed for "${server.name}": ${failures.join('; ')}`
- )
- }
- return
+ // The documented pre-start contract is fail-closed for controller faults:
+ // an exactly-proven policy may remain for already-running pods, but False
+ // never admits a new/update runtime and never completes the retry.
+ throw first.err
}
if (failures.length > 0) {
@@ -2593,10 +2814,12 @@ export class NetworkPolicyReconciler {
server,
this.sortResolvedEgressIPs(resolvedEgressIPs),
'True',
- 'Reconciled',
- bindings.some(binding => binding.egressClass === 'public-web')
- ? 'External egress policies reconciled; public-web allows public TCP 80/443 with private and special ranges excluded'
- : 'External egress policies reconciled',
+ dnsFailStatic ? 'FailStatic' : 'Reconciled',
+ dnsFailStatic
+ ? 'External egress retained from the last accepted DNS state while the resolver is temporarily unavailable'
+ : bindings.some(binding => binding.egressClass === 'public-web')
+ ? 'External egress policies reconciled; public-web allows public TCP 80/443 with private and special ranges excluded'
+ : 'External egress policies reconciled',
isCurrent
)
}
diff --git a/packages/network-policy-core/index.cjs b/packages/network-policy-core/index.cjs
index 27286dcba..4d2ed89bc 100644
--- a/packages/network-policy-core/index.cjs
+++ b/packages/network-policy-core/index.cjs
@@ -134,6 +134,31 @@ function dnsCodeOf(err) {
return typeof err.code === 'string' ? err.code : undefined
}
+/**
+ * Classify one value rejected by the DNS promise at the resolver boundary.
+ * Unlike classifyDnsError, this is a total trust-boundary verdict: an unknown,
+ * missing, or non-object rejection is a controller fault, never an implicit
+ * negative DNS answer. The caller must invoke this only in the catch that wraps
+ * the DNS operation itself; a code cannot prove where an exception originated.
+ */
+function classifyDnsRejection(value) {
+ const code = dnsCodeOf(value)
+ if (typeof code === 'string' && TRANSIENT_DNS_CODES.has(code)) {
+ return { kind: 'transient', code }
+ }
+ if (typeof code === 'string' && NO_RECORDS_DNS_CODES.includes(code)) {
+ return { kind: 'negative', reason: 'no-records', code }
+ }
+ if (code === 'EBADNAME') {
+ return { kind: 'negative', reason: 'invalid-name', code }
+ }
+ return {
+ kind: 'fault',
+ ...(typeof code === 'string' ? { code } : {}),
+ cause: value,
+ }
+}
+
/** True for the two genuine no-records answers, false for every other code. */
function isNoRecordsDnsCode(code) {
return typeof code === 'string' && NO_RECORDS_DNS_CODES.includes(code)
@@ -484,6 +509,87 @@ function parseState(annotations, now, config, declarations) {
return { entries: [] }
}
+/**
+ * Strict current-format parser for authorization decisions. Unlike parseState,
+ * this never falls back to legacy data and never drops malformed entries while
+ * returning success. It is intentionally separate so migration/availability
+ * callers can keep parseState's tolerant behavior without making corrupt or
+ * over-cap state authoritative.
+ */
+function parseStateStrict(annotations, config, declarations) {
+ const raw = annotations && annotations[STATE_ANNOTATION]
+ if (typeof raw !== 'string' || raw.length === 0) return { kind: 'absent' }
+
+ let value
+ try {
+ value = JSON.parse(raw)
+ } catch (_error) {
+ return { kind: 'invalid', reason: 'state annotation is not valid JSON' }
+ }
+ if (!Array.isArray(value)) {
+ return { kind: 'invalid', reason: 'state annotation is not an array' }
+ }
+ if (value.length > config.maxEntries) {
+ return { kind: 'invalid', reason: 'state annotation exceeds maxEntries' }
+ }
+
+ const declarationIds = Array.isArray(declarations)
+ ? new Set(
+ declarations
+ .filter(declaration => declaration && typeof declaration.fqdn === 'string')
+ .map(declaration =>
+ idOf(declaration.fqdn, declaration.port, declaration.protocol || DEFAULT_PROTOCOL)
+ )
+ )
+ : null
+ const entries = []
+ const seen = new Set()
+ for (const entry of value) {
+ if (
+ !entry ||
+ !isValidIpv4(entry.ip) ||
+ !Number.isInteger(entry.port) ||
+ entry.port < 1 ||
+ entry.port > 65535 ||
+ typeof entry.fqdn !== 'string' ||
+ entry.fqdn.length === 0 ||
+ typeof entry.protocol !== 'string' ||
+ entry.protocol.length === 0 ||
+ !Number.isFinite(entry.expiresAt) ||
+ !Number.isFinite(entry.lastObservedAt) ||
+ !Number.isFinite(new Date(entry.lastObservedAt).getTime())
+ ) {
+ return { kind: 'invalid', reason: 'state annotation contains a malformed entry' }
+ }
+ if (declarationIds && !declarationIds.has(idOf(entry.fqdn, entry.port, entry.protocol))) {
+ return { kind: 'invalid', reason: 'state annotation contains an undeclared identity' }
+ }
+ const normalized = {
+ ip: entry.ip,
+ port: entry.port,
+ protocol: entry.protocol,
+ fqdn: entry.fqdn,
+ expiresAt: entry.expiresAt,
+ lastObservedAt: entry.lastObservedAt,
+ }
+ const key = keyOf(normalized)
+ if (seen.has(key)) {
+ return { kind: 'invalid', reason: 'state annotation contains a duplicate entry' }
+ }
+ seen.add(key)
+ entries.push(normalized)
+ }
+ entries.sort(byKey)
+ const serialized = serializeState({ entries })
+ const combinedBytes =
+ Buffer.byteLength(serialized[STATE_ANNOTATION], 'utf8') +
+ Buffer.byteLength(serialized[TARGETS_ANNOTATION], 'utf8')
+ if (combinedBytes > MAX_ANNOTATION_BYTES) {
+ return { kind: 'invalid', reason: 'state annotation exceeds the byte budget' }
+ }
+ return { kind: 'valid', state: { entries } }
+}
+
module.exports = {
STATE_ANNOTATION,
TARGETS_ANNOTATION,
@@ -496,9 +602,11 @@ module.exports = {
stateHash,
serializeState,
parseState,
+ parseStateStrict,
classifyDnsError,
PERMANENT_DNS_CODES,
isPermanentDnsCode,
isNoRecordsDnsCode,
dnsCodeOf,
+ classifyDnsRejection,
}
diff --git a/packages/network-policy-core/index.d.ts b/packages/network-policy-core/index.d.ts
index d8b2fc3be..dcf79cea9 100644
--- a/packages/network-policy-core/index.d.ts
+++ b/packages/network-policy-core/index.d.ts
@@ -126,6 +126,21 @@ export declare function parseState(
declarations?: ReadonlyArray<{ fqdn: string; port: number; protocol?: string }>
): EgressState
+export type StrictStateRead =
+ | { kind: 'valid'; state: EgressState }
+ | { kind: 'absent' }
+ | { kind: 'invalid'; reason: string }
+
+/**
+ * Parse only the current structured state for authorization decisions. No
+ * legacy fallback and no silent dropping of malformed/over-cap entries.
+ */
+export declare function parseStateStrict(
+ annotations: Record<string, string> | undefined,
+ config: EgressCoreConfig,
+ declarations?: ReadonlyArray<{ fqdn: string; port: number; protocol?: string }>
+): StrictStateRead
+
export declare const STATE_ANNOTATION: string
export declare const TARGETS_ANNOTATION: string
export declare const RESOLVED_AT_ANNOTATION: string
@@ -174,6 +189,19 @@ export declare function isPermanentDnsCode(code: unknown): boolean
*/
export declare function dnsCodeOf(err: unknown): string | undefined
+export type DnsRejectionVerdict =
+ | { kind: 'transient'; code: string }
+ | { kind: 'negative'; reason: 'no-records' | 'invalid-name'; code: string }
+ | { kind: 'fault'; code?: string; cause: unknown }
+
+/**
+ * Total verdict for a value rejected by a DNS promise. Unknown, missing, and
+ * non-object values are faults rather than implicit negative DNS answers.
+ * Invoke only at the exact resolver rejection boundary: a code alone does not
+ * establish where an exception originated.
+ */
+export declare function classifyDnsRejection(value: unknown): DnsRejectionVerdict
+
/**
* True for the two genuine no-records answers (ENODATA, ENOTFOUND) and false for
* every other code, PERMANENT_DNS_CODES members included. Lets a caller word a
diff --git a/packages/network-policy-core/index.test.cjs b/packages/network-policy-core/index.test.cjs
index 14233ebcc..3868e53fd 100644
--- a/packages/network-policy-core/index.test.cjs
+++ b/packages/network-policy-core/index.test.cjs
@@ -415,6 +415,101 @@ test('#567: dnsCodeOf does NOT accept a bare code string, unlike classifyDnsErro
assert.equal(core.isPermanentDnsCode(core.dnsCodeOf('ENOTFOUND')), false)
})
+test('#567 remediation: classifyDnsRejection is total for falsy and non-Error values', () => {
+ for (const value of [undefined, null, false, 0, '', 'ENOTFOUND', new Error('no code')]) {
+ const verdict = core.classifyDnsRejection(value)
+ assert.equal(verdict.kind, 'fault', `${String(value)} must remain a fault`)
+ assert.equal(verdict.cause, value)
+ }
+ assert.deepEqual(core.classifyDnsRejection({ code: 7 }), {
+ kind: 'fault',
+ cause: { code: 7 },
+ })
+})
+
+test('#567 remediation: classifyDnsRejection distinguishes transient and negative DNS verdicts', () => {
+ assert.deepEqual(
+ core.classifyDnsRejection(Object.assign(new Error('servfail'), { code: 'ESERVFAIL' })),
+ { kind: 'transient', code: 'ESERVFAIL' }
+ )
+ assert.deepEqual(
+ core.classifyDnsRejection(Object.assign(new Error('not found'), { code: 'ENOTFOUND' })),
+ { kind: 'negative', reason: 'no-records', code: 'ENOTFOUND' }
+ )
+ assert.deepEqual(
+ core.classifyDnsRejection(Object.assign(new Error('bad name'), { code: 'EBADNAME' })),
+ { kind: 'negative', reason: 'invalid-name', code: 'EBADNAME' }
+ )
+})
+
+test('#567 remediation: unknown coded rejections remain faults with their code and cause', () => {
+ const err = Object.assign(new Error('bad family'), { code: 'EBADFAMILY' })
+ assert.deepEqual(core.classifyDnsRejection(err), {
+ kind: 'fault',
+ code: 'EBADFAMILY',
+ cause: err,
+ })
+})
+
+test('#567 remediation: parseStateStrict rejects absent, malformed, undeclared, duplicate, and over-cap state', () => {
+ assert.deepEqual(core.parseStateStrict({}, CFG), { kind: 'absent' })
+ assert.equal(core.parseStateStrict({ [core.STATE_ANNOTATION]: '{' }, CFG).kind, 'invalid')
+
+ const validEntry = {
+ ip: '1.2.3.4',
+ port: 443,
+ protocol: 'TCP',
+ fqdn: 'api.example.com',
+ expiresAt: 2000,
+ lastObservedAt: 1000,
+ }
+ const declaration = [{ fqdn: 'api.example.com', port: 443, protocol: 'TCP' }]
+ const undeclared = { ...validEntry, fqdn: 'other.example.com' }
+ assert.equal(
+ core.parseStateStrict(
+ { [core.STATE_ANNOTATION]: JSON.stringify([undeclared]) },
+ CFG,
+ declaration
+ ).kind,
+ 'invalid'
+ )
+ assert.equal(
+ core.parseStateStrict(
+ { [core.STATE_ANNOTATION]: JSON.stringify([validEntry, validEntry]) },
+ CFG,
+ declaration
+ ).kind,
+ 'invalid'
+ )
+ assert.equal(
+ core.parseStateStrict(
+ { [core.STATE_ANNOTATION]: JSON.stringify([validEntry, { ...validEntry, ip: '5.6.7.8' }]) },
+ { ...CFG, maxEntries: 1 },
+ declaration
+ ).kind,
+ 'invalid'
+ )
+})
+
+test('#567 remediation: parseStateStrict returns normalized current state only', () => {
+ const entry = {
+ ip: '1.2.3.4',
+ port: 443,
+ protocol: 'TCP',
+ fqdn: 'api.example.com',
+ expiresAt: 2000,
+ lastObservedAt: 1000,
+ }
+ assert.deepEqual(
+ core.parseStateStrict(
+ { [core.STATE_ANNOTATION]: JSON.stringify([entry]) },
+ CFG,
+ [{ fqdn: 'api.example.com', port: 443 }]
+ ),
+ { kind: 'valid', state: { entries: [entry] } }
+ )
+})
+
// ─── H-A: revocation/freeze keyed by (fqdn,port,protocol), not fqdn alone ──────
// The state map is keyed by (fqdn,ip,port,protocol) but classification used to be
// keyed by fqdn only, so an entry whose declared PORT was removed survived under
diff --git a/workflow-recipes/package-lock.json b/workflow-recipes/package-lock.json
index 50a6bcac6..e63ee916f 100644
--- a/workflow-recipes/package-lock.json
+++ b/workflow-recipes/package-lock.json
@@ -1,12 +1,12 @@
{
"name": "workload-recipes-controller",
- "version": "0.1.342",
+ "version": "0.1.343",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "workload-recipes-controller",
- "version": "0.1.342",
+ "version": "0.1.343",
"license": "SEE LICENSE IN LICENSE",
"dependencies": {
"@aws-sdk/client-s3": "^3.1009.0",
diff --git a/workflow-recipes/package.json b/workflow-recipes/package.json
index 03ba8569f..18fc8a122 100644
--- a/workflow-recipes/package.json
+++ b/workflow-recipes/package.json
@@ -1,6 +1,6 @@
{
"name": "workload-recipes-controller",
- "version": "0.1.342",
+ "version": "0.1.343",
"description": "Workload Recipes Controller (WRC) - WorkloadRecipe lifecycle management and MCP server interface",
"main": "dist/main.js",
"engines": {
diff --git a/workflow-recipes/src/reconciler/externalEgressAccumulator.test.ts b/workflow-recipes/src/reconciler/externalEgressAccumulator.test.ts
index c9a6d7426..d2285777a 100644
--- a/workflow-recipes/src/reconciler/externalEgressAccumulator.test.ts
+++ b/workflow-recipes/src/reconciler/externalEgressAccumulator.test.ts
@@ -7,7 +7,7 @@ import {
reconcileEgressState,
serializeState,
} from '@clerum/network-policy-core'
-import { accumulateExternalEgress } from './externalEgressAccumulator'
+import { accumulateExternalEgress, contractExternalEgress } from './externalEgressAccumulator'
import type { ResolveResult } from './fqdnResolver'
const CONFIG = { overlapMs: 300_000, maxEntries: 128 }
@@ -95,6 +95,7 @@ describe('accumulateExternalEgress — H1 fail-static under transient DNS failur
[{ fqdn: 'api.github.com', port: 443, ip: '140.82.112.3', ttlSeconds: 15 }],
NOW
)
+ prev[RESOLVED_AT_ANNOTATION] = '2000-01-01T00:00:00.000Z'
const resolveResult: ResolveResult = {
resolved: [],
failures: [{ fqdn: 'api.github.com', error: 'ESERVFAIL', retryable: true }],
@@ -110,6 +111,41 @@ describe('accumulateExternalEgress — H1 fail-static under transient DNS failur
// The IP is CONSERVED (frozen), not pruned.
expect(out.resolved.map(r => r.cidr)).toEqual(['140.82.112.3/32'])
expect(out.frozenFqdns).toContain('api.github.com')
+ expect(out.annotations[RESOLVED_AT_ANNOTATION]).toBe('2000-01-01T00:00:00.000Z')
+ })
+
+ it('preserves the last accepted timestamp on a permanent failure that removes state', () => {
+ const prev = previousAnnotations(
+ [{ fqdn: 'api.github.com', port: 443, ip: '140.82.112.3', ttlSeconds: 15 }],
+ NOW
+ )
+ prev[RESOLVED_AT_ANNOTATION] = '2000-01-01T00:00:00.000Z'
+ const out = accumulateExternalEgress({
+ externals: [{ fqdn: 'api.github.com', port: 443 }],
+ resolveResult: {
+ resolved: [],
+ failures: [{ fqdn: 'api.github.com', error: 'no A records', retryable: false }],
+ },
+ previousAnnotations: prev,
+ now: NOW + 1000,
+ config: CONFIG,
+ })
+ expect(out.entries).toEqual([])
+ expect(out.annotations[RESOLVED_AT_ANNOTATION]).toBe('2000-01-01T00:00:00.000Z')
+ })
+
+ it('does not synthesize a resolved timestamp for a bootstrap failure', () => {
+ const out = accumulateExternalEgress({
+ externals: [{ fqdn: 'api.github.com', port: 443 }],
+ resolveResult: {
+ resolved: [],
+ failures: [{ fqdn: 'api.github.com', error: 'ESERVFAIL', retryable: true }],
+ },
+ previousAnnotations: undefined,
+ now: NOW,
+ config: CONFIG,
+ })
+ expect(out.annotations[RESOLVED_AT_ANNOTATION]).toBeUndefined()
})
})
@@ -202,6 +238,64 @@ describe('accumulateExternalEgress — F1 revocation of a removed fqdn (audit #2
})
})
+describe('contractExternalEgress — subtractive phase before DNS', () => {
+ it('revokes undeclared identities, freezes current ones, and preserves resolved-at', () => {
+ const prev = previousAnnotations(
+ [
+ { fqdn: 'removed.example.com', port: 443, ip: '93.184.216.10', ttlSeconds: 30 },
+ { fqdn: 'current.example.com', port: 443, ip: '93.184.216.20', ttlSeconds: 30 },
+ ],
+ NOW
+ )
+ prev[RESOLVED_AT_ANNOTATION] = new Date(NOW).toISOString()
+
+ const out = contractExternalEgress({
+ externals: [{ fqdn: 'current.example.com', port: 443 }],
+ previousAnnotations: prev,
+ now: NOW + 30_000 + CONFIG.overlapMs + 60_000,
+ config: CONFIG,
+ })
+
+ expect(out.resolved.map(entry => entry.cidr)).toEqual(['93.184.216.20/32'])
+ expect(out.changed).toBe(true)
+ expect(out.annotations[RESOLVED_AT_ANNOTATION]).toBe(new Date(NOW).toISOString())
+ expect(out.annotations[TARGETS_ANNOTATION]).not.toContain('removed.example.com')
+ })
+
+ it('returns empty trusted state when prior provenance is absent', () => {
+ const out = contractExternalEgress({
+ externals: [{ fqdn: 'current.example.com', port: 443 }],
+ previousAnnotations: undefined,
+ now: NOW,
+ config: CONFIG,
+ })
+ expect(out.entries).toEqual([])
+ expect(out.resolved).toEqual([])
+ expect(out.annotations[STATE_ANNOTATION]).toBe('[]')
+ expect(out.annotations[RESOLVED_AT_ANNOTATION]).toBeUndefined()
+ })
+
+ it('removes persisted entries rejected by the lane safety filter from state and spec', () => {
+ const prev = previousAnnotations(
+ [
+ { fqdn: 'api.example.com', port: 443, ip: '10.0.0.5', ttlSeconds: 30 },
+ { fqdn: 'api.example.com', port: 443, ip: '93.184.216.20', ttlSeconds: 30 },
+ ],
+ NOW
+ )
+ const out = contractExternalEgress({
+ externals: [{ fqdn: 'api.example.com', port: 443 }],
+ previousAnnotations: prev,
+ now: NOW + 1000,
+ config: CONFIG,
+ isAllowedIp: ip => !ip.startsWith('10.'),
+ })
+ expect(out.resolved.map(entry => entry.cidr)).toEqual(['93.184.216.20/32'])
+ expect(out.annotations[STATE_ANNOTATION]).not.toContain('10.0.0.5')
+ expect(out.changed).toBe(true)
+ })
+})
+
describe('accumulateExternalEgress — permanent failure expires normally', () => {
it('does NOT freeze a fqdn that failed permanently (no records)', () => {
const prev = previousAnnotations(
diff --git a/workflow-recipes/src/reconciler/externalEgressAccumulator.ts b/workflow-recipes/src/reconciler/externalEgressAccumulator.ts
index 1323142a1..1b17c7e76 100644
--- a/workflow-recipes/src/reconciler/externalEgressAccumulator.ts
+++ b/workflow-recipes/src/reconciler/externalEgressAccumulator.ts
@@ -32,6 +32,7 @@ import {
RESOLVED_AT_ANNOTATION,
emptyState,
parseState,
+ parseStateStrict,
reconcileEgressState,
serializeState,
} from '@clerum/network-policy-core'
@@ -75,6 +76,24 @@ export interface AccumulateOutput {
frozenFqdns: string[]
}
+export interface ContractInput {
+ externals: DeclaredExternal[]
+ previousAnnotations: Record<string, string> | undefined
+ now: number
+ config: EgressCoreConfig
+ /** Optional lane-specific safety filter applied to persisted IPs. */
+ isAllowedIp?: (ip: string) => boolean
+}
+
+export interface ContractOutput {
+ entries: EgressEntry[]
+ resolved: ResolvedExternalEgressInput[]
+ annotations: Record<string, string>
+ changed: boolean
+ evicted: EgressEntry[]
+ overCap: boolean
+}
+
/**
* Turn the declared externals + this round's resolver output into normalized
* observations for the core. Iterating the DECLARED list (not the resolver
@@ -131,6 +150,22 @@ function buildObservations(
return observations
}
+function priorAcceptedResolvedAt(
+ previousAnnotations: Record<string, string> | undefined,
+ config: EgressCoreConfig,
+ externals: DeclaredExternal[]
+): string | undefined {
+ if (!previousAnnotations) return undefined
+ const annotated = previousAnnotations[RESOLVED_AT_ANNOTATION]
+ if (annotated && Number.isFinite(Date.parse(annotated))) return annotated
+ const strict = parseStateStrict(previousAnnotations, config, externals)
+ if (strict.kind !== 'valid' || strict.state.entries.length === 0) return undefined
+ const latestObservedAt = Math.max(...strict.state.entries.map(entry => entry.lastObservedAt))
+ return Number.isFinite(new Date(latestObservedAt).getTime())
+ ? new Date(latestObservedAt).toISOString()
+ : undefined
+}
+
/**
* Fold this reconcile's DNS snapshot into the accumulated egress set persisted
* on the policy's annotations, applying the sliding window, fail-static expiry,
@@ -147,26 +182,73 @@ export function accumulateExternalEgress(input: AccumulateInput): AccumulateOutp
const observations = buildObservations(externals, resolveResult)
const out = reconcileEgressState(previous, observations, now, config)
+ const permanentFqdns = new Set(
+ resolveResult.failures.filter(failure => !failure.retryable).map(failure => failure.fqdn)
+ )
+ // A conclusive negative answer blocks this reconcile and revokes that
+ // identity immediately. The shared core's normal TTL expiry remains useful
+ // for other consumers, but WRC must not keep a known-negative allow live.
+ const entries = out.entries.filter(entry => !permanentFqdns.has(entry.fqdn))
- const resolved: ResolvedExternalEgressInput[] = out.entries.map(entry => ({
+ const resolved: ResolvedExternalEgressInput[] = entries.map(entry => ({
cidr: `${entry.ip}/32`,
port: entry.port,
source: { kind: 'fqdn', fqdn: entry.fqdn },
}))
- const annotations: Record<string, string> = {
- ...serializeState(out.state),
- [RESOLVED_AT_ANNOTATION]: new Date(now).toISOString(),
- }
+ const annotations: Record<string, string> = serializeState({ entries })
+ const resolvedAt =
+ resolveResult.resolved.length > 0
+ ? new Date(now).toISOString()
+ : priorAcceptedResolvedAt(previousAnnotations, config, externals)
+ if (resolvedAt) annotations[RESOLVED_AT_ANNOTATION] = resolvedAt
return {
- entries: out.entries,
+ entries,
resolved,
annotations,
- changed: out.changed,
+ changed: out.changed || entries.length !== out.entries.length,
renewalDue: out.renewalDue,
evicted: out.evicted,
overCap: out.overCap,
frozenFqdns: out.frozenFqdns,
}
}
+
+/**
+ * Contract an existing aggregate policy to the declarations that still exist,
+ * without performing or pretending to perform a DNS observation. Current
+ * identities are frozen for this pure subtractive phase; undeclared identities
+ * are removed and the normal cap still applies. The last successful resolved-at
+ * timestamp is preserved because contraction is not a resolution event.
+ */
+export function contractExternalEgress(input: ContractInput): ContractOutput {
+ const { externals, previousAnnotations, now, config, isAllowedIp } = input
+ const previous = previousAnnotations
+ ? parseState(previousAnnotations, now, config, externals)
+ : emptyState()
+ const observations: Observation[] = externals.map(external => ({
+ fqdn: external.fqdn,
+ port: external.port,
+ kind: 'transient',
+ }))
+ const out = reconcileEgressState(previous, observations, now, config)
+ const entries = isAllowedIp ? out.entries.filter(entry => isAllowedIp(entry.ip)) : out.entries
+ const resolved: ResolvedExternalEgressInput[] = entries.map(entry => ({
+ cidr: `${entry.ip}/32`,
+ port: entry.port,
+ source: { kind: 'fqdn', fqdn: entry.fqdn },
+ }))
+ const annotations = serializeState({ entries })
+ const lastResolvedAt = priorAcceptedResolvedAt(previousAnnotations, config, externals)
+ if (lastResolvedAt) annotations[RESOLVED_AT_ANNOTATION] = lastResolvedAt
+
+ return {
+ entries,
+ resolved,
+ annotations,
+ changed: out.changed || entries.length !== out.entries.length,
+ evicted: out.evicted,
+ overCap: out.overCap,
+ }
+}
diff --git a/workflow-recipes/src/reconciler/fqdnResolver.test.ts b/workflow-recipes/src/reconciler/fqdnResolver.test.ts
index 182ba8320..a40ecc3af 100644
--- a/workflow-recipes/src/reconciler/fqdnResolver.test.ts
+++ b/workflow-recipes/src/reconciler/fqdnResolver.test.ts
@@ -168,16 +168,15 @@ describe('defaultFqdnLookup classification', () => {
resolve6Mock.mockReset()
})
- it('returns ok with A records even when AAAA has none', async () => {
+ it('returns ok with A records and does not query unenforceable AAAA records', async () => {
resolve4Mock.mockResolvedValue([{ address: '160.79.104.10', ttl: 300 }])
- resolve6Mock.mockRejectedValue(dnsError('ENODATA'))
const result = await defaultFqdnLookup('api.anthropic.com')
expect(result).toEqual({ kind: 'ok', ipv4: ['160.79.104.10'], ipv6: [], ttlSeconds: 300 })
+ expect(resolve6Mock).not.toHaveBeenCalled()
})
it('classifies SERVFAIL as a retryable error and surfaces the code', async () => {
resolve4Mock.mockRejectedValue(dnsError('ESERVFAIL'))
- resolve6Mock.mockRejectedValue(dnsError('ESERVFAIL'))
const result = await defaultFqdnLookup('huggingface.co')
expect(result.kind).toBe('error')
if (result.kind !== 'error') throw new Error('expected error')
@@ -188,7 +187,6 @@ describe('defaultFqdnLookup classification', () => {
it('classifies a timeout as retryable', async () => {
resolve4Mock.mockRejectedValue(dnsError('ETIMEOUT'))
- resolve6Mock.mockRejectedValue(dnsError('ETIMEOUT'))
const result = await defaultFqdnLookup('api.anthropic.com')
expect(result).toMatchObject({ kind: 'error', retryable: true })
})
@@ -197,7 +195,6 @@ describe('defaultFqdnLookup classification', () => {
'classifies operational DNS failure %s as retryable instead of no-records',
async code => {
resolve4Mock.mockRejectedValue(dnsError(code))
- resolve6Mock.mockRejectedValue(dnsError(code))
const result = await defaultFqdnLookup('resolver-broken.example.com')
expect(result).toMatchObject({ kind: 'error', retryable: true })
if (result.kind !== 'error') throw new Error('expected error')
@@ -207,24 +204,22 @@ describe('defaultFqdnLookup classification', () => {
it('treats a genuine no-records answer (ENODATA) as permanent', async () => {
resolve4Mock.mockRejectedValue(dnsError('ENODATA'))
- resolve6Mock.mockRejectedValue(dnsError('ENODATA'))
const result = await defaultFqdnLookup('no-records.example.com')
- expect(result).toEqual({ kind: 'error', error: 'no A or AAAA records' })
+ expect(result).toEqual({ kind: 'error', error: 'no A records for IPv4 egress enforcement' })
})
it('treats NXDOMAIN (ENOTFOUND) as permanent', async () => {
resolve4Mock.mockRejectedValue(dnsError('ENOTFOUND'))
- resolve6Mock.mockRejectedValue(dnsError('ENOTFOUND'))
const result = await defaultFqdnLookup('does-not-exist.example.com')
- expect(result).toEqual({ kind: 'error', error: 'no A or AAAA records' })
+ expect(result).toEqual({ kind: 'error', error: 'no A records for IPv4 egress enforcement' })
expect((result as { retryable?: boolean }).retryable).toBeFalsy()
})
- it('is retryable when any family hits a transient code even if the other is empty', async () => {
+ it('is retryable when the enforceable A query hits a transient code', async () => {
resolve4Mock.mockRejectedValue(dnsError('ECONNREFUSED'))
- resolve6Mock.mockRejectedValue(dnsError('ENODATA'))
const result = await defaultFqdnLookup('partial.example.com')
expect(result).toMatchObject({ kind: 'error', retryable: true })
+ expect(resolve6Mock).not.toHaveBeenCalled()
})
// ─── issue #513: only a positive DNS verdict may become a lookup result ──────
@@ -238,7 +233,6 @@ describe('defaultFqdnLookup classification', () => {
'T4a throws on %s: a complaint about our own query is not a DNS answer',
async code => {
resolve4Mock.mockRejectedValue(dnsError(code))
- resolve6Mock.mockRejectedValue(dnsError(code))
await expect(defaultFqdnLookup('h.example.com')).rejects.toThrow(code)
await expect(defaultFqdnLookup('h.example.com')).rejects.toThrow('"h.example.com"')
@@ -271,46 +265,48 @@ describe('defaultFqdnLookup classification', () => {
it('T4b throws when a rejection carries no code at all', async () => {
resolve4Mock.mockRejectedValue(new Error('boom'))
- resolve6Mock.mockRejectedValue(dnsError('ENODATA'))
await expect(defaultFqdnLookup('nocode.example.com')).rejects.toThrow(/no code/)
await expect(defaultFqdnLookup('nocode.example.com')).rejects.toThrow(/boom/)
})
+ it.each([undefined, null, false, 0, ''])(
+ 'T4b-total throws for a falsy rejection payload %s',
+ async value => {
+ resolve4Mock.mockRejectedValue(value)
+ await expect(defaultFqdnLookup('falsy.example.com')).rejects.toThrow(/non-DNS error/)
+ }
+ )
+
it('T4c reports EBADNAME as a malformed name, not as an empty answer', async () => {
// Permanent like NXDOMAIN, but the resolver never queried the name. Saying
// "no A or AAAA records" sends the operator hunting for missing records
// instead of at the typo in their own manifest.
resolve4Mock.mockRejectedValue(dnsError('EBADNAME'))
- resolve6Mock.mockRejectedValue(dnsError('EBADNAME'))
const result = await defaultFqdnLookup('foo..example.com')
expect(result).toMatchObject({ kind: 'error' })
expect((result as { error: string }).error).toContain('EBADNAME')
expect((result as { error: string }).error).toContain('foo..example.com')
- expect((result as { error: string }).error).not.toContain('no A or AAAA')
+ expect((result as { error: string }).error).not.toContain('no A records')
expect((result as { retryable?: boolean }).retryable).toBeFalsy()
})
it('T4d still reports a genuinely empty answer as no-records', async () => {
- // A family that RESOLVED to an empty list contributes no rejection at all.
- // The gate keys on rejections, so this must not be mistaken for an
+ // An empty successful A answer is a genuine negative observation, not an
// unclassified fault.
resolve4Mock.mockResolvedValue([])
- resolve6Mock.mockRejectedValue(dnsError('ENODATA'))
const result = await defaultFqdnLookup('empty.example.com')
- expect(result).toEqual({ kind: 'error', error: 'no A or AAAA records' })
+ expect(result).toEqual({ kind: 'error', error: 'no A records for IPv4 egress enforcement' })
})
- it('T4e lets a transient code win over an unclassified one', async () => {
- // Retrying is harmless, and if the resolver recovers the fault surfaces on
- // the next pass with a real answer to contrast it against.
+ it('T4e does not let an AAAA success hide an unclassified A fault', async () => {
resolve4Mock.mockRejectedValue(dnsError('EBADFAMILY'))
- resolve6Mock.mockRejectedValue(dnsError('ESERVFAIL'))
+ resolve6Mock.mockResolvedValue(['2001:4860:4860::8888'])
- const result = await defaultFqdnLookup('mixed.example.com')
- expect(result).toMatchObject({ kind: 'error', retryable: true })
+ await expect(defaultFqdnLookup('mixed.example.com')).rejects.toThrow('EBADFAMILY')
+ expect(resolve6Mock).not.toHaveBeenCalled()
})
})
@@ -323,7 +319,6 @@ describe('defaultFqdnLookup TTL propagation (issue #299)', () => {
it('requests per-record TTLs via resolve4({ ttl: true })', async () => {
resolve4Mock.mockResolvedValue([{ address: '140.82.112.3', ttl: 18 }])
- resolve6Mock.mockRejectedValue(dnsError('ENODATA'))
await defaultFqdnLookup('api.github.com')
expect(resolve4Mock).toHaveBeenCalledWith('api.github.com', { ttl: true })
})
@@ -333,7 +328,6 @@ describe('defaultFqdnLookup TTL propagation (issue #299)', () => {
{ address: '140.82.112.3', ttl: 18 },
{ address: '140.82.112.4', ttl: 12 },
])
- resolve6Mock.mockRejectedValue(dnsError('ENODATA'))
const result = await defaultFqdnLookup('api.github.com')
expect(result).toEqual({
kind: 'ok',
diff --git a/workflow-recipes/src/reconciler/fqdnResolver.ts b/workflow-recipes/src/reconciler/fqdnResolver.ts
index 30fcfaaf5..46069200a 100644
--- a/workflow-recipes/src/reconciler/fqdnResolver.ts
+++ b/workflow-recipes/src/reconciler/fqdnResolver.ts
@@ -1,11 +1,6 @@
import type { RecordWithTtl } from 'node:dns'
-import { resolve4, resolve6 } from 'node:dns/promises'
-import {
- classifyDnsError,
- dnsCodeOf,
- isNoRecordsDnsCode,
- isPermanentDnsCode,
-} from '@clerum/network-policy-core'
+import { resolve4 } from 'node:dns/promises'
+import { classifyDnsRejection } from '@clerum/network-policy-core'
import type { SandboxUiExternalEgress } from '../types'
const BLOCKED_EGRESS_CIDRS = [
@@ -90,12 +85,12 @@ export interface ResolvedExternalEgress {
export type FqdnLookupResult =
// `ttlSeconds` is the MINIMUM TTL (seconds) across the resolved A records —
// the binding window for the whole FQDN, consumed by the egress accumulator
- // (issue #299). It is 0 when only AAAA resolved (A-less), since no /32 is
- // emitted for an IPv4-only enforcement target in that case.
+ // (issue #299). `ipv6` remains as an empty compatibility field until the
+ // platform has an explicit dual-stack enforcement capability.
| { kind: 'ok'; ipv4: string[]; ipv6: string[]; ttlSeconds: number }
// `retryable` distinguishes a transient resolver failure (SERVFAIL, timeout,
// unreachable upstream) — worth retrying on a later reconcile — from a
- // permanent one (the name genuinely has no A/AAAA records, or is malformed).
+ // permanent one (the name has no enforceable A records, or is malformed).
// Absent ⇒ permanent. A failure that is not a DNS verdict at all never becomes
// a result: the lookup throws (issue #513).
| { kind: 'error'; error: string; retryable?: boolean }
@@ -104,126 +99,79 @@ export type FqdnLookup = (host: string) => Promise<FqdnLookupResult>
/**
* Transient-vs-permanent DNS classification is centralized in
- * `@clerum/network-policy-core` (`classifyDnsError`) so WRC and HCC agree on what
- * a transient failure is — a single source of truth avoids drift between the two
- * controllers' fail-static behavior (issue #299 audit F6). c-ares / system codes
- * meaning "resolver/upstream temporarily unavailable" map to `retryable: true`;
- * stable answers (ENODATA, ENOTFOUND/NXDOMAIN, empty) fail closed permanently.
+ * `@clerum/network-policy-core` (`classifyDnsRejection`) so WRC and HCC agree on
+ * a total resolver-boundary verdict. Unknown/missing values remain faults;
+ * c-ares/system outage codes map to `retryable: true`; stable negative answers
+ * fail closed permanently.
*/
/**
- * `settleV4`/`settleV6` cast whatever the promise rejected with to
- * `NodeJS.ErrnoException`; nothing guarantees it was an Error. Reading `.message`
- * off a rejected string or object then interpolates `undefined` into the message
- * an operator reads. Fall back to the value itself, never to a placeholder.
+ * Rejection payloads are unknown by construction. Keep diagnostics total so an
+ * unusual promise rejection cannot replace the controller fault it is reporting.
*/
-function describeRejection(err: NodeJS.ErrnoException): string {
- return typeof err?.message === 'string' ? err.message : String(err)
+function describeRejection(err: unknown): string {
+ try {
+ if (
+ typeof err === 'object' &&
+ err !== null &&
+ typeof (err as { message?: unknown }).message === 'string'
+ ) {
+ return (err as { message: string }).message
+ }
+ return String(err)
+ } catch {
+ // A hostile Proxy/toString must not hide the original fault classification.
+ return 'unprintable rejection value'
+ }
}
/**
- * Default FQDN lookup using node:dns/promises. Returns ipv4 and ipv6 record
- * sets independently; a missing record type for one family is fine as long as
- * the other resolves, so `api.example.com` (A only) and `ipv6-only.example.com`
- * both resolve cleanly.
+ * Default FQDN lookup using node:dns/promises. The current NetworkPolicy
+ * dataplane emits only IPv4 /32 entries, so A is the sole enforceable family.
+ * Querying AAAA and treating it as success would let an A failure erase/fail to
+ * freeze the actual IPv4 window. Dual-stack must arrive as an explicit future
+ * capability rather than an inert resolver side channel.
*
- * When neither family yields a record, the lookup inspects the underlying DNS
- * error code to decide whether the empty answer is permanent (no records) or a
- * transient resolver failure (SERVFAIL/timeout/unreachable upstream). The latter
- * is reported with `retryable: true` and the real code in the message, so the
- * reconciler can retry instead of collapsing every failure into the misleading
- * "no A or AAAA records".
+ * Only the catch around resolve4 classifies a rejection. Downstream validation
+ * faults occur outside that boundary and therefore cannot be laundered by a
+ * DNS-looking `.code`.
*/
export const defaultFqdnLookup: FqdnLookup = async host => {
- // A records are requested WITH per-record TTLs (`{ ttl: true }`) so the
- // accumulator can size the sliding window off the real DNS TTL (issue #299).
- // AAAA is resolved without TTLs: /128 blocks are dropped (IPv4-only targets),
- // so their TTL would be inert.
- const settleV4 = (p: Promise<RecordWithTtl[]>) =>
- p.then(
- records => ({ records }) as { records: RecordWithTtl[] },
- (err: NodeJS.ErrnoException) => ({ err }) as { err: NodeJS.ErrnoException }
- )
- const settleV6 = (p: Promise<string[]>) =>
- p.then(
- addrs => ({ addrs }) as { addrs: string[] },
- (err: NodeJS.ErrnoException) => ({ err }) as { err: NodeJS.ErrnoException }
- )
-
- const [v4, v6] = await Promise.all([
- settleV4(resolve4(host, { ttl: true })),
- settleV6(resolve6(host)),
- ])
-
- const v4Records = 'records' in v4 ? v4.records : []
- const ipv4 = v4Records.map(r => r.address)
- const ipv6 = 'addrs' in v6 ? v6.addrs : []
- if (ipv4.length > 0 || ipv6.length > 0) {
- // Minimum TTL across the A records = the window for the whole FQDN. 0 when
- // A-less (AAAA-only), where no /32 is emitted anyway.
- const ttlSeconds = v4Records.length > 0 ? Math.min(...v4Records.map(r => r.ttl)) : 0
- return { kind: 'ok', ipv4, ipv6, ttlSeconds }
- }
-
- const rejections: NodeJS.ErrnoException[] = []
- if ('err' in v4) rejections.push(v4.err)
- if ('err' in v6) rejections.push(v6.err)
-
- const transient = rejections.find(e => classifyDnsError(e) === 'transient')
- if (transient) {
- return {
- kind: 'error',
- // Read through `dnsCodeOf` like every other consumer below. A `transient`
- // verdict already guarantees a string code, so this is display-only — but
- // it is the last hand-rolled `.code` read left in the consumers, and the
- // point of the shared helper is that there is exactly one way to ask.
- error: `DNS resolution for "${host}" failed (${dnsCodeOf(transient) ?? 'unknown'}) — resolver or upstream unavailable`,
- retryable: true,
+ let records: RecordWithTtl[]
+ try {
+ records = await resolve4(host, { ttl: true })
+ } catch (error: unknown) {
+ const verdict = classifyDnsRejection(error)
+ if (verdict.kind === 'transient') {
+ return {
+ kind: 'error',
+ error: `DNS resolution for "${host}" failed (${verdict.code}) — resolver or upstream unavailable`,
+ retryable: true,
+ }
+ }
+ if (verdict.kind === 'negative') {
+ if (verdict.reason === 'invalid-name') {
+ return {
+ kind: 'error',
+ error: `malformed hostname "${host}" (${verdict.code}) — the resolver refused to query it`,
+ }
+ }
+ return { kind: 'error', error: 'no A records for IPv4 egress enforcement' }
}
- }
-
- // Issue #513: only a POSITIVE DNS verdict may become a lookup result. A family
- // that resolved to an empty list contributes no rejection at all; one that
- // rejected must carry a code in PERMANENT_DNS_CODES. Anything else — a
- // codeless rejection, or a c-ares complaint about the query WE built rather
- // than about the name — is not a DNS answer, and the old default branch turned
- // it into "no A or AAAA records": a reply that was never received, blamed on
- // the resolver, and routed on through egressResolutionError() as the
- // operator's permanent misconfiguration. Throw instead. The reconciler's catch
- // already routes a plain Error to the terminal `failed` phase with the real
- // message (walking `.cause` for socket codes), and this throw happens before
- // any policy write or delete, so the live NetworkPolicy is left untouched. The
- // message deliberately avoids the 'egress resolution failed' marker that catch
- // matches on: a raw fault is not a DNS classification worth protecting. That
- // absence is asserted in fqdnResolver.test.ts rather than left as prose.
- //
- // The code is read through the package's `dnsCodeOf` rather than off `.code`
- // directly, so both egress gates unwrap a rejection the same way (#567 review,
- // R1-M2). It also makes the read total: `rejections` is typed as
- // ErrnoException[] because the settle wrappers say so, not because anything
- // checked, so a non-object rejection would have thrown on the property access.
- const unclassified = rejections.find(e => !isPermanentDnsCode(dnsCodeOf(e)))
- if (unclassified) {
throw new Error(
- `DNS lookup for "${host}" failed with a non-DNS error (${dnsCodeOf(unclassified) ?? 'no code'}): ${describeRejection(unclassified)}`,
- { cause: unclassified }
+ `DNS lookup for "${host}" failed with a non-DNS error (${verdict.code ?? 'no code'}): ${describeRejection(verdict.cause)}`,
+ { cause: verdict.cause }
)
}
- // A permanent verdict that is NOT one of the two no-records answers means the
- // resolver refused to send the query at all — EBADNAME today. Report the real
- // code rather than testing for it: hardcoding 'EBADNAME' here would drift the
- // moment PERMANENT_DNS_CODES gains another member, and the message would fall
- // back to "no A or AAAA records" — the same lie this branch exists to remove.
- const refused = rejections.find(e => !isNoRecordsDnsCode(dnsCodeOf(e)))
- if (refused) {
- return {
- kind: 'error',
- error: `malformed hostname "${host}" (${dnsCodeOf(refused)}) — the resolver refused to query it`,
- }
+ // Everything below is downstream of a successful resolver call. Exceptions
+ // here are controller/contract faults even if they happen to carry a DNS code.
+ const ipv4 = records.map(record => record.address)
+ if (ipv4.length === 0) {
+ return { kind: 'error', error: 'no A records for IPv4 egress enforcement' }
}
-
- return { kind: 'error', error: 'no A or AAAA records' }
+ const ttlSeconds = Math.min(...records.map(record => record.ttl))
+ return { kind: 'ok', ipv4, ipv6: [], ttlSeconds }
}
export interface ResolveResult {
diff --git a/workflow-recipes/src/reconciler/mcpDelegation.test.ts b/workflow-recipes/src/reconciler/mcpDelegation.test.ts
index 929ff5419..67040a428 100644
--- a/workflow-recipes/src/reconciler/mcpDelegation.test.ts
+++ b/workflow-recipes/src/reconciler/mcpDelegation.test.ts
@@ -2062,6 +2062,60 @@ describe('waitForExternalEgressReady', () => {
expect(result.pending).toEqual(['web-search'])
expect(result.failed).toEqual([])
})
+
+ it('treats a current controller-fault condition as retryable pending', async () => {
+ const deps: DelegationDeps = {
+ customApi: {
+ getNamespacedCustomObject: vi.fn().mockResolvedValue({
+ metadata: { generation: 3 },
+ status: {
+ conditions: [
+ {
+ type: 'ExternalEgressReady',
+ status: 'False',
+ reason: 'ExternalEgressReconcileFailed',
+ message: 'controller fault; runtime remains blocked',
+ observedGeneration: 3,
+ },
+ ],
+ },
+ }),
+ } as any,
+ coreApi: {} as any,
+ }
+
+ const result = await waitForExternalEgressReady(deps, ['web-search'], 'mcp-server', 1)
+ expect(result).toEqual({ ready: false, pending: ['web-search'], failed: [] })
+ })
+
+ it('keeps a current rejected binding terminal and fail-closed', async () => {
+ const deps: DelegationDeps = {
+ customApi: {
+ getNamespacedCustomObject: vi.fn().mockResolvedValue({
+ metadata: { generation: 3 },
+ status: {
+ conditions: [
+ {
+ type: 'ExternalEgressReady',
+ status: 'False',
+ reason: 'ExternalEgressRejected',
+ message: 'hostname is not allowed',
+ observedGeneration: 3,
+ },
+ ],
+ },
+ }),
+ } as any,
+ coreApi: {} as any,
+ }
+
+ const result = await waitForExternalEgressReady(deps, ['web-search'], 'mcp-server', 1000)
+ expect(result).toEqual({
+ ready: false,
+ pending: ['web-search'],
+ failed: [{ name: 'web-search', message: 'hostname is not allowed' }],
+ })
+ })
})
// ─── Issue #408 — waitForNetworkReady must be generation-aware ─────────────────
diff --git a/workflow-recipes/src/reconciler/mcpDelegation.ts b/workflow-recipes/src/reconciler/mcpDelegation.ts
index 21b5f8168..521e15b46 100644
--- a/workflow-recipes/src/reconciler/mcpDelegation.ts
+++ b/workflow-recipes/src/reconciler/mcpDelegation.ts
@@ -37,6 +37,14 @@ import { effectiveWorkflowContextRef, privateWorkflowContextName } from './workf
const MCPSERVER_PLURAL = 'mcpservers'
const CONTEXT_PLURAL = 'contexts'
const EXTERNAL_EGRESS_READY_CONDITION = 'ExternalEgressReady'
+const RETRYABLE_EXTERNAL_EGRESS_REASONS = new Set([
+ 'ExternalEgressReconcileFailed',
+ 'InventoryListFailed',
+ 'CleanupFailed',
+ 'ExternalEgressProofFailed',
+ 'ExternalEgressUntrustedState',
+ 'StatusWriteFailed',
+])
const MCPSERVER_REPLACE_CONFLICT_RETRIES = 3
const CONTEXT_REPLACE_CONFLICT_RETRIES = 3
@@ -1041,6 +1049,7 @@ export async function waitForExternalEgressReady(
return {
name,
status: isFresh ? condition?.status : 'Unknown',
+ reason: isFresh ? condition?.reason : undefined,
message:
isFresh && condition
? (condition.message ??
@@ -1053,7 +1062,8 @@ export async function waitForExternalEgressReady(
} catch (error) {
return {
name,
- status: 'False',
+ status: getErrorCode(error) === 404 ? 'False' : 'Unknown',
+ reason: getErrorCode(error) === 404 ? 'McpServerDeleted' : 'InventoryReadFailed',
message:
getErrorCode(error) === 404
? 'McpServer was deleted before external egress became ready'
@@ -1068,7 +1078,9 @@ export async function waitForExternalEgressReady(
pending.delete(result.name)
console.log(`[MCP-Delegation] External egress ready for "${result.name}"`)
} else if (result.status === 'False') {
- failed.set(result.name, result.message ?? `${EXTERNAL_EGRESS_READY_CONDITION} is False`)
+ if (!result.reason || !RETRYABLE_EXTERNAL_EGRESS_REASONS.has(result.reason)) {
+ failed.set(result.name, result.message ?? `${EXTERNAL_EGRESS_READY_CONDITION} is False`)
+ }
}
}
diff --git a/workflow-recipes/src/reconciler/reconciler.test.ts b/workflow-recipes/src/reconciler/reconciler.test.ts
index b089a9871..c22ac5028 100644
--- a/workflow-recipes/src/reconciler/reconciler.test.ts
+++ b/workflow-recipes/src/reconciler/reconciler.test.ts
@@ -187,6 +187,23 @@ function snippetRun(code = 'return { ok: true }') {
return { type: 'snippet' as const, language: 'typescript' as const, code }
}
+function egressStateAnnotations(
+ entries: Array<{ fqdn: string; ip: string; port: number }>
+): Record<string, string> {
+ const now = Date.now()
+ return {
+ 'clerum.io/egress-fqdn-state': JSON.stringify(
+ entries.map(entry => ({
+ ...entry,
+ protocol: 'TCP',
+ expiresAt: now + 600_000,
+ lastObservedAt: now,
+ }))
+ ),
+ 'clerum.io/egress-fqdn-resolved-at': new Date(now).toISOString(),
+ }
+}
+
describe('WorkflowRecipeReconciler', () => {
let reconciler: WorkflowRecipeReconciler
@@ -249,18 +266,28 @@ describe('WorkflowRecipeReconciler', () => {
mockNetworkingApi.createNamespacedNetworkPolicy.mockReset()
mockNetworkingApi.createNamespacedNetworkPolicy.mockResolvedValue({})
mockNetworkingApi.readNamespacedNetworkPolicy.mockReset()
- mockNetworkingApi.readNamespacedNetworkPolicy.mockImplementation(({ name }: { name: string }) =>
- Promise.resolve({
- metadata: {
- name,
- uid: `uid-${name}`,
- resourceVersion: '1',
- labels: {
- 'clerum.io/managed-by': 'wrc',
- 'clerum.io/recipe': name.replace(/-coordinator-to-gfs$/, ''),
+ mockNetworkingApi.readNamespacedNetworkPolicy.mockImplementation(
+ ({ name }: { name: string }) => {
+ if (
+ name.startsWith('ui-egress-') ||
+ name.startsWith('wl-egress-') ||
+ name.startsWith('ui-ingress-') ||
+ name.startsWith('wl-ingress-')
+ ) {
+ return Promise.reject({ code: 404 })
+ }
+ return Promise.resolve({
+ metadata: {
+ name,
+ uid: `uid-${name}`,
+ resourceVersion: '1',
+ labels: {
+ 'clerum.io/managed-by': 'wrc',
+ 'clerum.io/recipe': name.replace(/-coordinator-to-gfs$/, ''),
+ },
},
- },
- })
+ })
+ }
)
mockNetworkingApi.replaceNamespacedNetworkPolicy.mockReset()
mockNetworkingApi.replaceNamespacedNetworkPolicy.mockResolvedValue({})
@@ -1683,16 +1710,28 @@ describe('WorkflowRecipeReconciler', () => {
it('fails closed instead of replacing an existing non-WRC internal-dependency policy', async () => {
mockNetworkingApi.createNamespacedNetworkPolicy.mockRejectedValueOnce({ code: 409 })
- mockNetworkingApi.readNamespacedNetworkPolicy.mockResolvedValueOnce({
- metadata: {
- resourceVersion: 'rv-hcc',
- labels: {
- 'clerum.io/managed-by': 'host-capability-controller',
- 'clerum.io/policy-type': 'binding-allow',
- 'clerum.io/recipe': 'test-recipe',
- },
- },
- })
+ mockNetworkingApi.readNamespacedNetworkPolicy.mockImplementation(
+ ({ name }: { name: string }) => {
+ if (
+ name.startsWith('ui-egress-') ||
+ name.startsWith('wl-egress-') ||
+ name.startsWith('ui-ingress-') ||
+ name.startsWith('wl-ingress-')
+ ) {
+ return Promise.reject({ code: 404 })
+ }
+ return Promise.resolve({
+ metadata: {
+ resourceVersion: 'rv-hcc',
+ labels: {
+ 'clerum.io/managed-by': 'host-capability-controller',
+ 'clerum.io/policy-type': 'binding-allow',
+ 'clerum.io/recipe': 'test-recipe',
+ },
+ },
+ })
+ }
+ )
const result = await reconciler.reconcile(
makeRecipe({
@@ -2488,9 +2527,6 @@ describe('WorkflowRecipeReconciler', () => {
resolve4Mock.mockRejectedValueOnce(
Object.assign(new Error('queryA EBADFAMILY'), { code: 'EBADFAMILY' })
)
- resolve6Mock.mockRejectedValueOnce(
- Object.assign(new Error('queryAaaa EBADFAMILY'), { code: 'EBADFAMILY' })
- )
const reconcilerWithLookup = new WorkflowRecipeReconciler(new k8s.KubeConfig(), undefined, {
fqdnLookup: host => defaultFqdnLookup(host),
})
@@ -2509,6 +2545,7 @@ describe('WorkflowRecipeReconciler', () => {
// returning an error instead of throwing — leaves the test green, because
// both routes end in `failed` with the code somewhere in the message.
expect(result.message).not.toContain('egress resolution failed')
+ expect(resolve6Mock).not.toHaveBeenCalled()
const egressPolicy = expect.objectContaining({
body: expect.objectContaining({
@@ -2522,6 +2559,374 @@ describe('WorkflowRecipeReconciler', () => {
)
})
+ it('#567 remediation: contracts a removed workload binding before a sibling lookup faults', async () => {
+ const policyName = 'wl-egress-test-recipe-worker'
+ let existing: k8s.V1NetworkPolicy = {
+ metadata: {
+ name: policyName,
+ namespace: 'sandbox-recipes',
+ uid: 'uid-workload-egress',
+ resourceVersion: '7',
+ labels: {
+ 'clerum.io/managed-by': 'workflow-recipes',
+ 'clerum.io/recipe': 'test-recipe',
+ 'clerum.io/workload': 'worker',
+ },
+ annotations: egressStateAnnotations([
+ { fqdn: 'removed.example.com', ip: '93.184.216.10', port: 443 },
+ { fqdn: 'current.example.com', ip: '93.184.216.20', port: 443 },
+ ]),
+ },
+ spec: {
+ podSelector: {
+ matchLabels: { 'clerum.io/recipe': 'test-recipe', 'clerum.io/workload': 'worker' },
+ },
+ policyTypes: ['Egress'],
+ egress: ['93.184.216.10/32', '93.184.216.20/32'].map(cidr => ({
+ to: [{ ipBlock: { cidr } }],
+ ports: [{ port: 443, protocol: 'TCP' }],
+ })),
+ },
+ }
+ mockNetworkingApi.readNamespacedNetworkPolicy.mockImplementation(
+ ({ name }: { name: string }) =>
+ name === policyName ? Promise.resolve(existing) : Promise.reject({ code: 404 })
+ )
+ mockNetworkingApi.replaceNamespacedNetworkPolicy.mockImplementation(
+ ({ body }: { body: k8s.V1NetworkPolicy }) => {
+ existing = {
+ ...body,
+ metadata: {
+ ...body.metadata,
+ name: policyName,
+ namespace: 'sandbox-recipes',
+ uid: 'uid-workload-egress',
+ resourceVersion: '8',
+ },
+ }
+ return Promise.resolve(existing)
+ }
+ )
+ const rec = new WorkflowRecipeReconciler(new k8s.KubeConfig(), undefined, {
+ fqdnLookup: async () => {
+ throw Object.assign(new Error('queryA EBADFAMILY'), { code: 'EBADFAMILY' })
+ },
+ })
+ const result = await rec.reconcile(
+ makeRecipe({
+ spec: {
+ contextRef: 'default',
+ workloads: [
+ {
+ id: 'worker',
+ type: 'deployment',
+ image: 'worker:latest',
+ port: 8080,
+ egressBindings: [{ dns: 'current.example.com', port: 443 }],
+ },
+ ],
+ },
+ })
+ )
+
+ expect(result.phase).toBe('failed')
+ expect(result.message).toContain('EBADFAMILY')
+ const replacement = mockNetworkingApi.replaceNamespacedNetworkPolicy.mock.calls.find(
+ call => call[0]?.name === policyName
+ )?.[0].body as k8s.V1NetworkPolicy
+ const cidrs = (replacement?.spec?.egress ?? []).flatMap(rule =>
+ (rule.to ?? []).map(peer => peer.ipBlock?.cidr)
+ )
+ if (replacement) {
+ expect(cidrs).toEqual(['93.184.216.20/32'])
+ expect(JSON.stringify(replacement.metadata?.annotations)).not.toContain('removed.example.com')
+ } else {
+ // If any hand-built live dimension cannot prove a subset, exact deletion
+ // is the safer contraction than retaining the removed binding.
+ expect(mockNetworkingApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledWith(
+ expect.objectContaining({ name: policyName, namespace: 'sandbox-recipes' })
+ )
+ }
+ })
+
+ it('#567 remediation: contracts workload authorization before a UI lookup can fail', async () => {
+ const policyName = 'wl-egress-test-recipe-worker'
+ const ingressPolicyName = 'wl-ingress-test-recipe-worker'
+ const events: string[] = []
+ mockNetworkingApi.readNamespacedNetworkPolicy.mockImplementation(
+ ({ name }: { name: string }) =>
+ name === policyName || name === ingressPolicyName
+ ? Promise.resolve({
+ metadata: {
+ name,
+ namespace: 'sandbox-recipes',
+ uid:
+ name === policyName ? 'uid-stale-workload-egress' : 'uid-stale-workload-ingress',
+ resourceVersion: '21',
+ labels: {
+ 'clerum.io/managed-by': 'workflow-recipes',
+ 'clerum.io/recipe': 'test-recipe',
+ },
+ },
+ spec:
+ name === policyName
+ ? { podSelector: {}, policyTypes: ['Egress'], egress: [] }
+ : { podSelector: {}, policyTypes: ['Ingress'], ingress: [] },
+ } satisfies k8s.V1NetworkPolicy)
+ : Promise.reject({ code: 404 })
+ )
+ mockNetworkingApi.deleteNamespacedNetworkPolicy.mockImplementation(({ name }) => {
+ events.push(`${name}-delete`)
+ return Promise.resolve({})
+ })
+ const rec = new WorkflowRecipeReconciler(new k8s.KubeConfig(), undefined, {
+ fqdnLookup: async () => {
+ events.push('ui-lookup')
+ return { kind: 'error', error: 'no A records', retryable: false }
+ },
+ })
+ const result = await rec.reconcile(
+ makeRecipe({
+ spec: {
+ workloads: [
+ { id: 'frontend', type: 'deployment', image: 'fe:1', port: 8080 },
+ { id: 'worker', type: 'deployment', image: 'worker:1', port: 3000 },
+ ],
+ ui: {
+ workloadRef: 'frontend',
+ port: 8080,
+ egress: { external: [{ fqdn: 'broken.example.com', port: 443 }] },
+ },
+ },
+ })
+ )
+
+ expect(result.phase).toBe('failed')
+ expect(events).toEqual([`${policyName}-delete`, `${ingressPolicyName}-delete`, 'ui-lookup'])
+ expect(mockNetworkingApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledWith({
+ name: policyName,
+ namespace: 'sandbox-recipes',
+ body: {
+ preconditions: {
+ uid: 'uid-stale-workload-egress',
+ resourceVersion: '21',
+ },
+ },
+ })
+ expect(mockNetworkingApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledWith({
+ name: ingressPolicyName,
+ namespace: 'sandbox-recipes',
+ body: {
+ preconditions: {
+ uid: 'uid-stale-workload-ingress',
+ resourceVersion: '21',
+ },
+ },
+ })
+ })
+
+ it('#567 remediation: treats a concurrent 404 contraction delete as already revoked', async () => {
+ const policyName = 'wl-egress-test-recipe-worker'
+ mockNetworkingApi.readNamespacedNetworkPolicy.mockImplementation(
+ ({ name }: { name: string }) =>
+ name === policyName
+ ? Promise.resolve({
+ metadata: {
+ name,
+ namespace: 'sandbox-recipes',
+ uid: 'uid-concurrent-delete',
+ resourceVersion: '31',
+ labels: {
+ 'clerum.io/managed-by': 'workflow-recipes',
+ 'clerum.io/recipe': 'test-recipe',
+ },
+ },
+ spec: { podSelector: {}, policyTypes: ['Egress'], egress: [] },
+ } satisfies k8s.V1NetworkPolicy)
+ : Promise.reject({ code: 404 })
+ )
+ mockNetworkingApi.deleteNamespacedNetworkPolicy.mockRejectedValueOnce({
+ code: 404,
+ message: 'already gone',
+ })
+
+ const result = await reconciler.reconcile(
+ makeRecipe({
+ spec: {
+ workloads: [{ id: 'worker', type: 'deployment', image: 'worker:1', port: 8080 }],
+ },
+ })
+ )
+
+ expect(result.phase).toBe('active')
+ expect(mockNetworkingApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledWith({
+ name: policyName,
+ namespace: 'sandbox-recipes',
+ body: { preconditions: { uid: 'uid-concurrent-delete', resourceVersion: '31' } },
+ })
+ })
+
+ it('#567 remediation: treats a 404 contraction read-back as a safe concurrent revocation', async () => {
+ const policyName = 'wl-egress-test-recipe-worker'
+ let replaced = false
+ const existing: k8s.V1NetworkPolicy = {
+ metadata: {
+ name: policyName,
+ namespace: 'sandbox-recipes',
+ uid: 'uid-readback-race',
+ resourceVersion: '41',
+ labels: {
+ 'clerum.io/managed-by': 'workflow-recipes',
+ 'clerum.io/recipe': 'test-recipe',
+ 'clerum.io/workload': 'worker',
+ },
+ annotations: egressStateAnnotations([
+ { fqdn: 'removed.example.com', ip: '93.184.216.10', port: 443 },
+ { fqdn: 'current.example.com', ip: '93.184.216.20', port: 443 },
+ ]),
+ },
+ spec: {
+ podSelector: {
+ matchLabels: { 'clerum.io/recipe': 'test-recipe', 'clerum.io/workload': 'worker' },
+ },
+ policyTypes: ['Egress'],
+ egress: ['93.184.216.10/32', '93.184.216.20/32'].map(cidr => ({
+ to: [{ ipBlock: { cidr } }],
+ ports: [{ port: 443, protocol: 'TCP' }],
+ })),
+ },
+ }
+ mockNetworkingApi.readNamespacedNetworkPolicy.mockImplementation(
+ ({ name }: { name: string }) => {
+ if (name !== policyName) return Promise.reject({ code: 404 })
+ return replaced ? Promise.reject({ code: 404 }) : Promise.resolve(existing)
+ }
+ )
+ mockNetworkingApi.replaceNamespacedNetworkPolicy.mockImplementation(() => {
+ replaced = true
+ return Promise.resolve({})
+ })
+ const rec = new WorkflowRecipeReconciler(new k8s.KubeConfig(), undefined, {
+ fqdnLookup: async () => {
+ throw Object.assign(new Error('queryA EBADFAMILY'), { code: 'EBADFAMILY' })
+ },
+ })
+ const result = await rec.reconcile(
+ makeRecipe({
+ spec: {
+ workloads: [
+ {
+ id: 'worker',
+ type: 'deployment',
+ image: 'worker:1',
+ port: 8080,
+ egressBindings: [{ dns: 'current.example.com', port: 443 }],
+ },
+ ],
+ },
+ })
+ )
+
+ expect(result.phase).toBe('failed')
+ expect(result.message).toContain('EBADFAMILY')
+ expect(mockNetworkingApi.deleteNamespacedNetworkPolicy).not.toHaveBeenCalledWith(
+ expect.objectContaining({ name: policyName })
+ )
+ })
+
+ it('#567 remediation: uses fresh post-replace identity for same-pass revocation', async () => {
+ const policyName = 'wl-egress-test-recipe-worker'
+ const now = Date.now()
+ let live: k8s.V1NetworkPolicy = {
+ metadata: {
+ name: policyName,
+ namespace: 'sandbox-recipes',
+ uid: 'uid-workload-egress',
+ resourceVersion: '7',
+ labels: {
+ 'clerum.io/managed-by': 'workflow-recipes',
+ 'clerum.io/recipe': 'test-recipe',
+ 'clerum.io/workload': 'worker',
+ },
+ annotations: {
+ 'clerum.io/egress-fqdn-state': JSON.stringify(
+ [
+ ['removed.example.com', '93.184.216.10'],
+ ['current.example.com', '93.184.216.20'],
+ ].map(([fqdn, ip]) => ({
+ fqdn,
+ ip,
+ port: 443,
+ protocol: 'TCP',
+ expiresAt: now - 60_000,
+ lastObservedAt: now - 120_000,
+ }))
+ ),
+ 'clerum.io/egress-fqdn-resolved-at': new Date(now - 120_000).toISOString(),
+ },
+ },
+ spec: {
+ podSelector: {
+ matchLabels: { 'clerum.io/recipe': 'test-recipe', 'clerum.io/workload': 'worker' },
+ },
+ policyTypes: ['Egress'],
+ egress: ['93.184.216.10/32', '93.184.216.20/32'].map(cidr => ({
+ to: [{ ipBlock: { cidr } }],
+ ports: [{ port: 443, protocol: 'TCP' }],
+ })),
+ },
+ }
+ mockNetworkingApi.readNamespacedNetworkPolicy.mockImplementation(
+ ({ name }: { name: string }) =>
+ name === policyName ? Promise.resolve(live) : Promise.reject({ code: 404 })
+ )
+ mockNetworkingApi.replaceNamespacedNetworkPolicy.mockImplementation(
+ ({ body }: { body: k8s.V1NetworkPolicy }) => {
+ live = {
+ ...body,
+ metadata: {
+ ...body.metadata,
+ name: policyName,
+ namespace: 'sandbox-recipes',
+ uid: 'uid-workload-egress',
+ resourceVersion: '8',
+ },
+ }
+ return Promise.resolve(live)
+ }
+ )
+ const rec = new WorkflowRecipeReconciler(new k8s.KubeConfig(), undefined, {
+ fqdnLookup: async () => ({ kind: 'error', error: 'no A records', retryable: false }),
+ })
+ const result = await rec.reconcile(
+ makeRecipe({
+ spec: {
+ workloads: [
+ {
+ id: 'worker',
+ type: 'deployment',
+ image: 'worker:1',
+ port: 8080,
+ egressBindings: [{ dns: 'current.example.com', port: 443 }],
+ },
+ ],
+ },
+ })
+ )
+
+ expect(result.phase).toBe('failed')
+ expect(mockNetworkingApi.replaceNamespacedNetworkPolicy).toHaveBeenCalledWith(
+ expect.objectContaining({ name: policyName })
+ )
+ expect(mockNetworkingApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledWith({
+ name: policyName,
+ namespace: 'sandbox-recipes',
+ body: {
+ preconditions: { uid: 'uid-workload-egress', resourceVersion: '8' },
+ },
+ })
+ })
+
it('returns failed phase when a workload egress binding uses wildcard DNS', async () => {
const recipe = makeRecipe({
spec: {
@@ -4033,9 +4438,6 @@ describe('WorkflowRecipeReconciler', () => {
resolve4Mock.mockRejectedValueOnce(
Object.assign(new Error('queryA EBADFAMILY'), { code: 'EBADFAMILY' })
)
- resolve6Mock.mockRejectedValueOnce(
- Object.assign(new Error('queryAaaa EBADFAMILY'), { code: 'EBADFAMILY' })
- )
const kc = new k8s.KubeConfig()
const rec = new WorkflowRecipeReconciler(kc, undefined, {
fqdnLookup: host => defaultFqdnLookup(host),
@@ -4057,6 +4459,7 @@ describe('WorkflowRecipeReconciler', () => {
// returning an error instead of throwing — leaves the test green, because
// both routes end in `failed` with the code somewhere in the message.
expect(result.message).not.toContain('egress resolution failed')
+ expect(resolve6Mock).not.toHaveBeenCalled()
expect(createdPolicy('ui-egress-test-recipe')).toBeUndefined()
expect(mockNetworkingApi.replaceNamespacedNetworkPolicy).not.toHaveBeenCalledWith(
@@ -4071,6 +4474,285 @@ describe('WorkflowRecipeReconciler', () => {
)
})
+ it('#567 remediation: contracts a removed UI binding before a sibling lookup faults', async () => {
+ const policyName = 'ui-egress-test-recipe'
+ const existing: k8s.V1NetworkPolicy = {
+ metadata: {
+ name: policyName,
+ namespace: 'sandbox-ui',
+ uid: 'uid-ui-egress',
+ resourceVersion: '8',
+ labels: {
+ 'clerum.io/managed-by': 'workflow-recipes',
+ 'clerum.io/recipe': 'test-recipe',
+ },
+ annotations: egressStateAnnotations([
+ { fqdn: 'removed.example.com', ip: '93.184.216.10', port: 443 },
+ { fqdn: 'current.example.com', ip: '93.184.216.20', port: 443 },
+ ]),
+ },
+ spec: {
+ podSelector: { matchLabels: { 'clerum.io/sandbox-ui': 'true' } },
+ policyTypes: ['Egress'],
+ egress: ['93.184.216.10/32', '93.184.216.20/32'].map(cidr => ({
+ to: [{ ipBlock: { cidr } }],
+ ports: [{ port: 443, protocol: 'TCP' }],
+ })),
+ },
+ }
+ mockNetworkingApi.readNamespacedNetworkPolicy.mockImplementation(
+ ({ name }: { name: string }) =>
+ name === policyName ? Promise.resolve(existing) : Promise.reject({ code: 404 })
+ )
+ const rec = new WorkflowRecipeReconciler(new k8s.KubeConfig(), undefined, {
+ fqdnLookup: async () => {
+ throw Object.assign(new Error('queryA EBADFAMILY'), { code: 'EBADFAMILY' })
+ },
+ })
+ const result = await rec.reconcile(
+ uiRecipeWithExternals([{ fqdn: 'current.example.com', port: 443 }])
+ )
+
+ expect(result.phase).toBe('failed')
+ expect(result.message).toContain('EBADFAMILY')
+ const replacement = mockNetworkingApi.replaceNamespacedNetworkPolicy.mock.calls.find(
+ call => call[0]?.name === policyName
+ )?.[0].body as k8s.V1NetworkPolicy
+ const cidrs = (replacement?.spec?.egress ?? []).flatMap(rule =>
+ (rule.to ?? []).map(peer => peer.ipBlock?.cidr)
+ )
+ if (replacement) {
+ expect(cidrs).toEqual(['93.184.216.20/32'])
+ expect(JSON.stringify(replacement.metadata?.annotations)).not.toContain('removed.example.com')
+ } else {
+ expect(mockNetworkingApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledWith(
+ expect.objectContaining({ name: policyName, namespace: 'sandbox-ui' })
+ )
+ }
+ })
+
+ it('#567 remediation: a transient lookup cannot promote annotation-only UI egress', async () => {
+ let transient = false
+ const rec = new WorkflowRecipeReconciler(new k8s.KubeConfig(), undefined, {
+ fqdnLookup: async () =>
+ transient
+ ? { kind: 'error', error: 'ESERVFAIL', retryable: true }
+ : { kind: 'ok', ipv4: ['93.184.216.20'], ipv6: [], ttlSeconds: 300 },
+ })
+ const recipe = uiRecipeWithExternals([{ fqdn: 'current.example.com', port: 443 }])
+ await rec.reconcile(recipe)
+ const written = createdPolicy('ui-egress-test-recipe')!
+ let liveWithoutRule: k8s.V1NetworkPolicy = {
+ ...written,
+ metadata: {
+ ...written.metadata,
+ uid: 'uid-ui-egress',
+ resourceVersion: '10',
+ },
+ spec: { ...written.spec!, egress: [] },
+ }
+ mockNetworkingApi.readNamespacedNetworkPolicy.mockImplementation(
+ ({ name }: { name: string }) =>
+ name === 'ui-egress-test-recipe'
+ ? Promise.resolve(liveWithoutRule)
+ : Promise.reject({ code: 404 })
+ )
+ mockNetworkingApi.replaceNamespacedNetworkPolicy.mockImplementation(
+ ({ name, body }: { name: string; body: k8s.V1NetworkPolicy }) => {
+ if (name === 'ui-egress-test-recipe') {
+ liveWithoutRule = {
+ ...body,
+ metadata: {
+ ...body.metadata,
+ name,
+ namespace: 'sandbox-ui',
+ uid: 'uid-ui-egress',
+ resourceVersion: '11',
+ },
+ }
+ return Promise.resolve(liveWithoutRule)
+ }
+ return Promise.resolve({})
+ }
+ )
+ mockNetworkingApi.createNamespacedNetworkPolicy.mockClear()
+ mockNetworkingApi.replaceNamespacedNetworkPolicy.mockClear()
+ transient = true
+
+ const result = await rec.reconcile(recipe)
+ expect(result.phase).toBe('degraded')
+ const writtenBodies = [
+ ...mockNetworkingApi.createNamespacedNetworkPolicy.mock.calls.map(call => call[0]?.body),
+ ...mockNetworkingApi.replaceNamespacedNetworkPolicy.mock.calls.map(call => call[0]?.body),
+ ].filter(body => body?.metadata?.name === 'ui-egress-test-recipe') as k8s.V1NetworkPolicy[]
+ for (const body of writtenBodies) {
+ const cidrs = (body.spec?.egress ?? []).flatMap(rule =>
+ (rule.to ?? []).map(peer => peer.ipBlock?.cidr)
+ )
+ expect(cidrs).not.toContain('93.184.216.20/32')
+ expect(body.metadata?.annotations?.['clerum.io/egress-fqdn-state']).toBe('[]')
+ }
+ })
+
+ it.each(['success', 'fault'] as const)(
+ '#567 remediation: re-contracts a UI policy widened during DNS %s latency',
+ async mode => {
+ let secondRound = false
+ let live: k8s.V1NetworkPolicy | undefined
+ const rec = new WorkflowRecipeReconciler(new k8s.KubeConfig(), undefined, {
+ fqdnLookup: async () => {
+ if (secondRound && live) {
+ live = structuredClone(live)
+ live.spec!.egress!.push({
+ to: [{ ipBlock: { cidr: '8.8.8.8/32' } }],
+ ports: [{ port: 443, protocol: 'TCP' }],
+ })
+ if (mode === 'fault') {
+ throw Object.assign(new Error('post-list EBADFAMILY'), { code: 'EBADFAMILY' })
+ }
+ }
+ return { kind: 'ok', ipv4: ['93.184.216.20'], ipv6: [], ttlSeconds: 300 }
+ },
+ })
+ const recipe = uiRecipeWithExternals([{ fqdn: 'current.example.com', port: 443 }])
+ await rec.reconcile(recipe)
+ const written = createdPolicy('ui-egress-test-recipe')!
+ live = {
+ ...structuredClone(written),
+ metadata: {
+ ...written.metadata,
+ name: 'ui-egress-test-recipe',
+ namespace: 'sandbox-ui',
+ uid: 'uid-ui-dns-race',
+ resourceVersion: '50',
+ },
+ }
+ mockNetworkingApi.readNamespacedNetworkPolicy.mockImplementation(
+ ({ name }: { name: string }) =>
+ name === 'ui-egress-test-recipe' && live
+ ? Promise.resolve(live)
+ : Promise.reject({ code: 404 })
+ )
+ mockNetworkingApi.replaceNamespacedNetworkPolicy.mockImplementation(
+ ({ body }: { body: k8s.V1NetworkPolicy }) => {
+ live = {
+ ...body,
+ metadata: {
+ ...body.metadata,
+ name: 'ui-egress-test-recipe',
+ namespace: 'sandbox-ui',
+ uid: 'uid-ui-dns-race',
+ resourceVersion: '51',
+ },
+ }
+ return Promise.resolve(live)
+ }
+ )
+ mockNetworkingApi.createNamespacedNetworkPolicy.mockClear()
+ mockNetworkingApi.replaceNamespacedNetworkPolicy.mockClear()
+ secondRound = true
+
+ const result = await rec.reconcile(recipe)
+
+ expect(result.phase).toBe(mode === 'fault' ? 'failed' : 'active')
+ const replacements = mockNetworkingApi.replaceNamespacedNetworkPolicy.mock.calls
+ .filter(call => call[0]?.name === 'ui-egress-test-recipe')
+ .map(call => call[0]?.body as k8s.V1NetworkPolicy)
+ expect(replacements.length).toBeGreaterThan(0)
+ for (const replacement of replacements) {
+ expect(JSON.stringify(replacement.spec?.egress)).not.toContain('8.8.8.8')
+ }
+ }
+ )
+
+ it('#567 remediation: persists the prior resolved-at when a permanent failure narrows the policy', async () => {
+ let permanentFailure = false
+ let live: k8s.V1NetworkPolicy | undefined
+ const rec = new WorkflowRecipeReconciler(new k8s.KubeConfig(), undefined, {
+ fqdnLookup: async () =>
+ permanentFailure
+ ? { kind: 'error', error: 'no A records', retryable: false }
+ : { kind: 'ok', ipv4: ['93.184.216.20'], ipv6: [], ttlSeconds: 30 },
+ })
+ const recipe = makeRecipe({
+ spec: {
+ workloads: [
+ { id: 'frontend', type: 'deployment', image: 'fe:1', port: 8080 },
+ { id: 'backend', type: 'deployment', image: 'backend:1', port: 8080 },
+ ],
+ ui: {
+ workloadRef: 'frontend',
+ port: 8080,
+ egress: {
+ internal: [{ workloadRef: 'backend', port: 8080 }],
+ external: [{ fqdn: 'api.example.com', port: 443 }],
+ },
+ },
+ },
+ })
+ await rec.reconcile(recipe)
+ const written = createdPolicy('ui-egress-test-recipe')!
+ const oldResolvedAt = '2000-01-01T00:00:00.000Z'
+ const state = JSON.parse(
+ written.metadata!.annotations!['clerum.io/egress-fqdn-state']
+ ) as Array<Record<string, unknown>>
+ for (const entry of state) entry.expiresAt = Date.now() + 600_000
+ live = {
+ ...structuredClone(written),
+ metadata: {
+ ...written.metadata,
+ name: 'ui-egress-test-recipe',
+ namespace: 'sandbox-ui',
+ uid: 'uid-ui-permanent-narrow',
+ resourceVersion: '60',
+ annotations: {
+ ...written.metadata!.annotations,
+ 'clerum.io/egress-fqdn-state': JSON.stringify(state),
+ 'clerum.io/egress-fqdn-resolved-at': oldResolvedAt,
+ },
+ },
+ }
+ mockNetworkingApi.readNamespacedNetworkPolicy.mockImplementation(
+ ({ name }: { name: string }) =>
+ name === 'ui-egress-test-recipe' && live
+ ? Promise.resolve(live)
+ : Promise.reject({ code: 404 })
+ )
+ mockNetworkingApi.replaceNamespacedNetworkPolicy.mockImplementation(
+ ({ name, body }: { name: string; body: k8s.V1NetworkPolicy }) => {
+ if (name === 'ui-egress-test-recipe') {
+ live = {
+ ...body,
+ metadata: {
+ ...body.metadata,
+ name,
+ namespace: 'sandbox-ui',
+ uid: 'uid-ui-permanent-narrow',
+ resourceVersion: '61',
+ },
+ }
+ return Promise.resolve(live)
+ }
+ return Promise.resolve({})
+ }
+ )
+ mockNetworkingApi.createNamespacedNetworkPolicy.mockClear()
+ mockNetworkingApi.replaceNamespacedNetworkPolicy.mockClear()
+ permanentFailure = true
+
+ const result = await rec.reconcile(recipe)
+
+ expect(result.phase).toBe('failed')
+ const replacement = mockNetworkingApi.replaceNamespacedNetworkPolicy.mock.calls
+ .filter(call => call[0]?.name === 'ui-egress-test-recipe')
+ .at(-1)?.[0].body as k8s.V1NetworkPolicy
+ expect(replacement.metadata?.annotations?.['clerum.io/egress-fqdn-resolved-at']).toBe(
+ oldResolvedAt
+ )
+ expect(replacement.metadata?.annotations?.['clerum.io/egress-fqdn-state']).toBe('[]')
+ expect(JSON.stringify(replacement.spec?.egress)).not.toContain('93.184.216.20')
+ })
+
it('records the ui lane egress set with the exact enforced cidrs and ports', async () => {
const cap = captureEgressRecords()
try {
@@ -4320,7 +5002,12 @@ describe('WorkflowRecipeReconciler', () => {
mockNetworkingApi.readNamespacedNetworkPolicy.mockResolvedValue({
metadata: {
name: 'ui-egress-test-recipe',
+ uid: 'uid-ui-egress-test-recipe',
resourceVersion: '1',
+ labels: {
+ 'clerum.io/managed-by': 'workflow-recipes',
+ 'clerum.io/recipe': 'test-recipe',
+ },
annotations: { 'clerum.io/egress-fqdn-state': JSON.stringify(rehydrated) },
},
})
@@ -4341,7 +5028,9 @@ describe('WorkflowRecipeReconciler', () => {
expect(cap.payloads()).toHaveLength(1)
const recorded = cap.payloads()[0].cidrs as string[]
expect(recorded).not.toContain('10.0.0.5/32')
- expect(recorded).toEqual(['93.184.216.10/32', '93.184.216.99/32'])
+ // The prior public address existed only in the annotation, not in a live
+ // rule. Contraction must not promote it during the next successful round.
+ expect(recorded).toEqual(['93.184.216.99/32'])
} finally {
cap.restore()
}
@@ -4384,17 +5073,22 @@ describe('WorkflowRecipeReconciler', () => {
rendered!.metadata!.annotations!['clerum.io/egress-fqdn-state']
) as Array<Record<string, unknown>>
const aging = persisted.map(e => ({ ...e, expiresAt: Date.now() + 30_000 }))
- mockNetworkingApi.readNamespacedNetworkPolicy.mockResolvedValue({
- ...rendered,
- metadata: {
- ...rendered!.metadata,
- resourceVersion: '1',
- annotations: {
- ...rendered!.metadata!.annotations,
- 'clerum.io/egress-fqdn-state': JSON.stringify(aging),
- },
- },
- })
+ mockNetworkingApi.readNamespacedNetworkPolicy.mockImplementation(
+ ({ name }: { name: string }) =>
+ name === 'ui-egress-test-recipe'
+ ? Promise.resolve({
+ ...rendered,
+ metadata: {
+ ...rendered!.metadata,
+ resourceVersion: '1',
+ annotations: {
+ ...rendered!.metadata!.annotations,
+ 'clerum.io/egress-fqdn-state': JSON.stringify(aging),
+ },
+ },
+ })
+ : Promise.reject({ code: 404 })
+ )
const cap = captureEgressRecords()
const noOpSpy = vi.spyOn(console, 'log').mockImplementation(() => {})
@@ -4435,22 +5129,33 @@ describe('WorkflowRecipeReconciler', () => {
// below does — no hand-built fixture.
const created = createdPolicy('ui-egress-test-recipe')
expect(created).toBeDefined()
- mockNetworkingApi.readNamespacedNetworkPolicy.mockResolvedValue(created!)
+ mockNetworkingApi.readNamespacedNetworkPolicy.mockResolvedValue({
+ ...created!,
+ metadata: {
+ ...created!.metadata,
+ uid: 'uid-ui-egress-test-recipe',
+ resourceVersion: '1',
+ },
+ })
const cap = captureEgressRecords()
- const noOpSpy = vi.spyOn(console, 'log').mockImplementation(() => {})
try {
+ mockNetworkingApi.createNamespacedNetworkPolicy.mockClear()
+ mockNetworkingApi.replaceNamespacedNetworkPolicy.mockClear()
await rec.reconcile(recipe)
expect(cap.payloads()).toHaveLength(0)
- // POSITIVE CONTROL. Without it this assertion also passes when the second
- // reconcile never reached the egress path at all, which would make the
- // test a tautology rather than a check of the change gate.
- expect(noOpSpy).toHaveBeenCalledWith(
- expect.stringContaining('ui-egress-test-recipe" in sandbox-ui egress set unchanged — no-op')
- )
+ expect(
+ mockNetworkingApi.createNamespacedNetworkPolicy.mock.calls.some(
+ call => call[0]?.body?.metadata?.name === 'ui-egress-test-recipe'
+ )
+ ).toBe(false)
+ expect(
+ mockNetworkingApi.replaceNamespacedNetworkPolicy.mock.calls.some(
+ call => call[0]?.name === 'ui-egress-test-recipe'
+ )
+ ).toBe(false)
} finally {
- noOpSpy.mockRestore()
cap.restore()
}
})
@@ -4488,7 +5193,16 @@ describe('WorkflowRecipeReconciler', () => {
// That policy is now live; the next reconcile renames the FQDN onto the SAME IP.
mockNetworkingApi.readNamespacedNetworkPolicy.mockImplementation(({ name }: { name: string }) =>
Promise.resolve(
- name === 'ui-egress-test-recipe' ? p1 : { metadata: { name, resourceVersion: '1' } }
+ name === 'ui-egress-test-recipe'
+ ? {
+ ...p1,
+ metadata: {
+ ...p1.metadata,
+ uid: 'uid-ui-egress-test-recipe',
+ resourceVersion: '1',
+ },
+ }
+ : { metadata: { name, resourceVersion: '1' } }
)
)
mockNetworkingApi.createNamespacedNetworkPolicy.mockClear()
@@ -4828,6 +5542,19 @@ describe('WorkflowRecipeReconciler', () => {
})
it('R.8.24 — no ui-egress NetworkPolicy is created when spec.ui is unset', async () => {
+ mockNetworkingApi.readNamespacedNetworkPolicy.mockResolvedValueOnce({
+ metadata: {
+ name: 'ui-egress-test-recipe',
+ namespace: 'sandbox-ui',
+ uid: 'uid-ui-egress-test-recipe',
+ resourceVersion: '7',
+ labels: {
+ 'clerum.io/managed-by': 'workflow-recipes',
+ 'clerum.io/recipe': 'test-recipe',
+ },
+ },
+ spec: { podSelector: {}, policyTypes: ['Egress'], egress: [] },
+ })
const recipe = makeRecipe({
spec: {
workloads: [{ id: 'app', type: 'deployment', image: 'app:1', port: 8080 }],
@@ -4835,8 +5562,8 @@ describe('WorkflowRecipeReconciler', () => {
})
await reconciler.reconcile(recipe)
expect(mockNetworkingApi.createNamespacedNetworkPolicy).not.toHaveBeenCalled()
- // Convergence path: stale policy from a prior reconcile (when ui was set)
- // is removed unconditionally. safeDelete swallows the 404 if none exists.
+ // Convergence path: an owned stale policy from a prior reconcile is removed
+ // with live identity preconditions.
expect(mockNetworkingApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledWith(
expect.objectContaining({ name: 'ui-egress-test-recipe', namespace: 'sandbox-ui' })
)
diff --git a/workflow-recipes/src/reconciler/workflowRecipeReconciler.ts b/workflow-recipes/src/reconciler/workflowRecipeReconciler.ts
index 790bee8f8..124843d72 100644
--- a/workflow-recipes/src/reconciler/workflowRecipeReconciler.ts
+++ b/workflow-recipes/src/reconciler/workflowRecipeReconciler.ts
@@ -14,7 +14,13 @@
* 11. Handle delete (reverse dependency order, skip PVCs)
*/
import * as k8s from '@kubernetes/client-node'
-import { STATE_ANNOTATION } from '@clerum/network-policy-core'
+import {
+ type EgressEntry,
+ RESOLVED_AT_ANNOTATION,
+ STATE_ANNOTATION,
+ TARGETS_ANNOTATION,
+ serializeState,
+} from '@clerum/network-policy-core'
import { OperatorConfig, loadConfig } from '../config'
import { getPool } from '../db'
import { createLogger } from '../observability/logger'
@@ -58,7 +64,11 @@ import {
import { evaluateComputedValues } from './computedValuesEvaluator'
import { CRD_GROUP, CRD_VERSION, WORKFLOWRECIPE_PLURAL } from './crdConstants'
import { sort as sortDependencies } from './dependencyGraph'
-import { type AccumulateOutput, accumulateExternalEgress } from './externalEgressAccumulator'
+import {
+ type AccumulateOutput,
+ accumulateExternalEgress,
+ contractExternalEgress,
+} from './externalEgressAccumulator'
import {
type FqdnLookup,
defaultFqdnLookup,
@@ -134,11 +144,11 @@ function externalEgressReadinessError(
): Error {
const pending = result.pending.map(name => `${name}: pending`)
const failed = result.failed.map(({ name, message }) => `${name}: ${message}`)
- return new Error(
+ const message =
`External egress policy readiness not achieved for WorkflowRecipe "${recipeName}". ` +
- `${[...pending, ...failed].join('; ')}. ` +
- `External egress bindings are not enforceable until HCC reports ExternalEgressReady=True.`
- )
+ `${[...pending, ...failed].join('; ')}. ` +
+ `External egress bindings are not enforceable until HCC reports ExternalEgressReady=True.`
+ return result.failed.length === 0 ? new RetryableReconcileError(message) : new Error(message)
}
function clusterNetworkPolicyEnforcementError(recipeName: string): Error {
@@ -720,6 +730,110 @@ export function egressSignature(policy: k8s.V1NetworkPolicy): string {
)
}
+function networkPolicySpecSignature(policy: k8s.V1NetworkPolicy): string {
+ return JSON.stringify(canonicalize(policy.spec ?? {}))
+}
+
+function contractionWriteNeeded(
+ existing: k8s.V1NetworkPolicy,
+ desired: k8s.V1NetworkPolicy
+): boolean {
+ if (networkPolicySpecSignature(existing) !== networkPolicySpecSignature(desired)) return true
+ const currentAnnotations = existing.metadata?.annotations ?? {}
+ const desiredAnnotations = desired.metadata?.annotations ?? {}
+ return [STATE_ANNOTATION, TARGETS_ANNOTATION, RESOLVED_AT_ANNOTATION].some(
+ key => currentAnnotations[key] !== desiredAnnotations[key]
+ )
+}
+
+function intersectRules<T>(existing: T[] | undefined, desired: T[] | undefined): T[] {
+ const available = new Map<string, number>()
+ for (const rule of existing ?? []) {
+ const signature = JSON.stringify(canonicalize(rule))
+ available.set(signature, (available.get(signature) ?? 0) + 1)
+ }
+ return (desired ?? []).filter(rule => {
+ const signature = JSON.stringify(canonicalize(rule))
+ const remaining = available.get(signature) ?? 0
+ if (remaining === 0) return false
+ available.set(signature, remaining - 1)
+ return true
+ })
+}
+
+/**
+ * Intersect a desired policy with the rules already enforced by the live
+ * snapshot. Used only by the pre-DNS contraction phase: it may remove rules but
+ * can never add one. Selector/policy-type drift cannot be proven subtractive,
+ * so the caller revokes the policy instead.
+ */
+function subtractiveNetworkPolicy(
+ existing: k8s.V1NetworkPolicy,
+ desired: k8s.V1NetworkPolicy | null
+): k8s.V1NetworkPolicy | null {
+ if (!desired) return null
+ const existingSelector = JSON.stringify(canonicalize(existing.spec?.podSelector ?? {}))
+ const desiredSelector = JSON.stringify(canonicalize(desired.spec?.podSelector ?? {}))
+ const existingTypes = JSON.stringify([...(existing.spec?.policyTypes ?? [])].sort())
+ const desiredTypes = JSON.stringify([...(desired.spec?.policyTypes ?? [])].sort())
+ if (existingSelector !== desiredSelector || existingTypes !== desiredTypes) return null
+
+ const contractedSpec: k8s.V1NetworkPolicySpec = { ...desired.spec! }
+ if (desired.spec?.egress !== undefined) {
+ contractedSpec.egress = intersectRules(existing.spec?.egress, desired.spec.egress)
+ }
+ if (desired.spec?.ingress !== undefined) {
+ contractedSpec.ingress = intersectRules(existing.spec?.ingress, desired.spec.ingress)
+ }
+ const contracted: k8s.V1NetworkPolicy = { ...desired, spec: contractedSpec }
+
+ // State is authorization provenance for later fail-static rounds. Keep only
+ // entries represented by a retained live ipBlock+port rule so a transient DNS
+ // failure cannot re-add an annotation-only destination that was not enforced.
+ const rawState = contracted.metadata?.annotations?.[STATE_ANNOTATION]
+ if (rawState) {
+ const livePairs = new Set<string>()
+ for (const rule of contractedSpec.egress ?? []) {
+ for (const peer of rule.to ?? []) {
+ const cidr = peer.ipBlock?.cidr
+ if (!cidr?.endsWith('/32')) continue
+ for (const port of rule.ports ?? []) {
+ livePairs.add(
+ `${cidr.slice(0, -3)}\u0000${String(port.port)}\u0000${port.protocol ?? 'TCP'}`
+ )
+ }
+ }
+ }
+ const entries = (JSON.parse(rawState) as EgressEntry[]).filter(entry =>
+ livePairs.has(`${entry.ip}\u0000${entry.port}\u0000${entry.protocol}`)
+ )
+ const priorResolvedAt = contracted.metadata?.annotations?.[RESOLVED_AT_ANNOTATION]
+ contracted.metadata = {
+ ...(contracted.metadata ?? {}),
+ annotations: {
+ ...(contracted.metadata?.annotations ?? {}),
+ ...serializeState({ entries }),
+ ...(priorResolvedAt ? { [RESOLVED_AT_ANNOTATION]: priorResolvedAt } : {}),
+ },
+ }
+ }
+ return contracted
+}
+
+interface WorkloadEgressContractionPlan {
+ plans: Array<{
+ workload: WorkloadDef
+ namespace: string
+ policyName: string
+ externalDeclared: Array<{ fqdn: string; port: number }>
+ }>
+ ingressSourcesByTarget: Map<string, rb.WorkloadIngressSource[]>
+}
+
+interface RecipeEgressContractionPlan {
+ workload: WorkloadEgressContractionPlan
+}
+
export class WorkflowRecipeReconciler {
private appsApi: k8s.AppsV1Api
private batchApi: k8s.BatchV1Api
@@ -2486,6 +2600,7 @@ export class WorkflowRecipeReconciler {
// Step 9b: Sandbox UI egress policy + symmetric ingress on each
// internal-egress target (without the ingress side the target stays
// behind deny-all-<ns> and UI→backend traffic is dropped).
+ const externalEgressPlan = await this.prepareRecipeEgressContractions(recipe)
await this.reconcileUiEgressPolicy(recipe)
await this.reconcileUiIngressPolicies(recipe)
// Egress for background-OAuth workloads → control-api broker route.
@@ -2497,7 +2612,7 @@ export class WorkflowRecipeReconciler {
// any sibling targets it points at. MCP workloads inherit egress
// through HCC via the McpServer CRD path; UI workloads use
// `spec.ui.egress.*` which is handled by reconcileUiEgressPolicy.
- await this.reconcileWorkloadEgressPolicies(recipe)
+ await this.reconcileWorkloadEgressPolicies(recipe, externalEgressPlan.workload)
// Step 9d: WRC-owned intra-recipe internal dependency policies.
// This lane is separate from HCC binding-allow and legacy wl-* policies.
@@ -3316,23 +3431,17 @@ export class WorkflowRecipeReconciler {
// Sandbox UI egress policy + symmetric ingress on internal-egress
// targets (no-op when spec.ui is unset; idempotent)
try {
+ const externalEgressPlan = await this.prepareRecipeEgressContractions(recipe)
await this.reconcileUiEgressPolicy(recipe)
await this.reconcileUiIngressPolicies(recipe)
await this.reconcileOAuthBrokerEgressPolicy(recipe)
- } catch (err) {
- console.error(
- `[WR-Reconciler] UI egress/ingress policies failed for workflow "${name}":`,
- err
- )
- throw err
- }
- // Per-workload egressBindings → NetworkPolicies (idempotent; no-op when
- // no workload declares egressBindings).
- try {
- await this.reconcileWorkloadEgressPolicies(recipe)
+ // Per-workload egressBindings → NetworkPolicies (idempotent; no-op when
+ // no workload declares egressBindings). Its subtractive phase already ran
+ // recipe-wide above, before the first DNS lookup.
+ await this.reconcileWorkloadEgressPolicies(recipe, externalEgressPlan.workload)
} catch (err) {
- console.error(`[WR-Reconciler] Workload egress policies failed for workflow "${name}":`, err)
+ createLogger('wrc', name).error('recipe network policy reconciliation failed', { err })
throw err
}
@@ -4092,36 +4201,97 @@ export class WorkflowRecipeReconciler {
}
/**
- * Reconcile the per-recipe `ui-egress-<recipeName>` NetworkPolicy in
- * `sandboxUiNamespace`. Ensures the policy converges with `recipe.spec.ui`:
- * - spec.ui set → create-or-replace the policy
- * - spec.ui unset → delete any leftover policy from a prior reconcile
- *
- * The policy lives in a different namespace from the WorkflowRecipe CRD, so
- * it carries no ownerReference and relies on this method (and reconcileDelete)
- * for cleanup.
+ * One recipe-wide subtractive barrier. Every WRC policy whose authorization
+ * derives from UI/workload egress intent is narrowed before any resolver call
+ * in any lane. Later phases may add current intent only after their own lookup
+ * succeeds; a UI fault can therefore no longer strand a removed workload or
+ * symmetric-ingress permission.
+ */
+ private async prepareRecipeEgressContractions(
+ recipe: WorkflowRecipeCRD
+ ): Promise<RecipeEgressContractionPlan> {
+ await this.contractUiEgressPolicy(recipe)
+ const workload = await this.contractWorkloadEgressPolicies(recipe)
+ await this.contractUiIngressPolicies(recipe)
+ await this.contractWorkloadIngressPolicies(recipe, workload.ingressSourcesByTarget)
+ return { workload }
+ }
+
+ private async contractUiEgressPolicy(
+ recipe: WorkflowRecipeCRD
+ ): Promise<k8s.V1NetworkPolicy | null> {
+ const namespace = this.config.sandboxUiNamespace
+ const policyName = `ui-egress-${recipe.metadata.name}`
+ const existing = await this.readNetworkPolicyOrNull(policyName, namespace)
+ if (!existing) return null
+
+ const externals = recipe.spec.ui?.egress?.external ?? []
+ const declarations = externals.map(entry => ({ fqdn: entry.fqdn, port: entry.port }))
+ const contraction = contractExternalEgress({
+ externals: declarations,
+ previousAnnotations: existing.metadata?.annotations,
+ now: Date.now(),
+ config: {
+ overlapMs: this.config.externalEgressOverlapSeconds * 1000,
+ maxEntries: this.config.externalEgressMaxEntries,
+ },
+ isAllowedIp: ip => !isBlockedExternalIPv4(ip),
+ })
+ const needsStateAnnotations =
+ declarations.length > 0 ||
+ contraction.entries.length > 0 ||
+ existing.metadata?.annotations?.[STATE_ANNOTATION] !== undefined
+ const desired = rb.buildUiEgressNetworkPolicy(
+ recipe,
+ namespace,
+ this.config.sandboxNamespace,
+ contraction.resolved,
+ needsStateAnnotations ? contraction.annotations : undefined
+ )
+ return this.applyNetworkPolicyContraction({
+ existing,
+ desired: subtractiveNetworkPolicy(existing, desired),
+ namespace,
+ recipeName: recipe.metadata.name,
+ })
+ }
+
+ /**
+ * Reconcile the additive/fail-static half of the per-recipe UI policy. The
+ * caller supplies the snapshot returned by the recipe-wide contraction pass,
+ * which has already narrowed UI, workload, and symmetric ingress policies
+ * before the first DNS lookup can fail.
*/
private async reconcileUiEgressPolicy(recipe: WorkflowRecipeCRD): Promise<void> {
const policyName = `ui-egress-${recipe.metadata.name}`
const ns = this.config.sandboxUiNamespace
const externals = recipe.spec.ui?.egress?.external ?? []
- const { resolved, failures } = await resolveExternalEgress(externals, this.fqdnLookup)
+ const externalDeclarations = externals.map(e => ({ fqdn: e.fqdn, port: e.port }))
+ const egressConfig = {
+ overlapMs: this.config.externalEgressOverlapSeconds * 1000,
+ maxEntries: this.config.externalEgressMaxEntries,
+ }
+
+ let resolution: Awaited<ReturnType<typeof resolveExternalEgress>>
+ try {
+ resolution = await resolveExternalEgress(externals, this.fqdnLookup)
+ } catch (error: unknown) {
+ // DNS/controller latency is a race window. Re-contract from a fresh read
+ // before propagating so a concurrent widening cannot survive merely
+ // because the lookup itself faulted.
+ await this.contractUiEgressPolicy(recipe)
+ throw error
+ }
+ const existing = await this.contractUiEgressPolicy(recipe)
+ const { resolved, failures } = resolution
this.recordExternalEgressTtl(resolved)
- // Permanent failures (no-records / blocked range) never author a partial
- // policy — fail exactly as the single-snapshot resolver did.
const permanentFailures = failures.filter(f => !f.retryable)
- if (permanentFailures.length > 0) {
- throw egressResolutionError(
- `WorkflowRecipe "${recipe.metadata.name}" ui external egress resolution failed`,
- permanentFailures
- )
- }
+ let blockingFailures = permanentFailures
let effectiveResolved: rb.ResolvedExternalEgressInput[] = resolved
let stateAnnotations: Record<string, string> | undefined
- let existing: k8s.V1NetworkPolicy | null = null
let egressRenewalDue = false
// H-E: the write gate compares rendered spec.egress (ipBlock cidr+port, no
// fqdn), so a rename old.example.com→new.example.com onto the SAME ip/port is
@@ -4129,32 +4299,20 @@ export class WorkflowRecipeReconciler {
// annotation and losing the overlap grace when the new name later rotates.
// acc.changed is over (fqdn,ip,port,protocol), so it catches the rename.
let egressStateChanged = false
- // R1-M2: read the live policy for ALL cases, not only external egress, so an
- // internal-only sibling policy (ui.egress.internal[] with no external[]) can
- // hit the no-op gate below instead of being rewritten on every reconcile —
- // which the 60s external-egress refresh loop amplifies for mixed recipes. For
- // external egress it also seeds the accumulator's rehydration (H5).
- existing = await this.readNetworkPolicyOrNull(policyName, ns)
if (externals.length > 0) {
// issue #299: fold this DNS snapshot into the accumulated sliding-window
// set persisted on the live policy's annotations (rehydrate H5).
const acc = accumulateExternalEgress({
- externals: externals.map(e => ({ fqdn: e.fqdn, port: e.port })),
+ externals: externalDeclarations,
resolveResult: { resolved, failures },
previousAnnotations: existing?.metadata?.annotations,
now: Date.now(),
- config: {
- overlapMs: this.config.externalEgressOverlapSeconds * 1000,
- maxEntries: this.config.externalEgressMaxEntries,
- },
+ config: egressConfig,
})
// Bootstrap fail-closed: a transient resolver failure with NOTHING to
// freeze (no rehydratable prior set) must not author an empty policy.
if (acc.entries.length === 0 && failures.length > 0) {
- throw egressResolutionError(
- `WorkflowRecipe "${recipe.metadata.name}" ui external egress resolution failed`,
- failures
- )
+ blockingFailures = failures
}
// Audit L3: a transient failure of a newly-added FQDN (with resolving
// siblings) does not throw and freezes nothing — surface it so the missing
@@ -4190,11 +4348,30 @@ export class WorkflowRecipeReconciler {
stateAnnotations
)
- if (!policy) {
- await this.safeDelete(
- () => this.networkingApi.deleteNamespacedNetworkPolicy({ name: policyName, namespace: ns }),
- `NetworkPolicy "${policyName}" in ${ns}`
+ if (blockingFailures.length > 0) {
+ if (existing) {
+ await this.applyNetworkPolicyContraction({
+ existing,
+ desired: subtractiveNetworkPolicy(existing, policy),
+ namespace: ns,
+ recipeName: recipe.metadata.name,
+ })
+ }
+ throw egressResolutionError(
+ `WorkflowRecipe "${recipe.metadata.name}" ui external egress resolution failed`,
+ blockingFailures
)
+ }
+
+ if (!policy) {
+ if (existing) {
+ await this.applyNetworkPolicyContraction({
+ existing,
+ desired: null,
+ namespace: ns,
+ recipeName: recipe.metadata.name,
+ })
+ }
return
}
@@ -4206,37 +4383,46 @@ export class WorkflowRecipeReconciler {
console.log(
`[WR-Reconciler] NetworkPolicy "${policyName}" in ${ns} egress set unchanged — no-op`
)
- return
+ } else {
+ await this.applyOwnedRecipeNetworkPolicy(policy, ns, recipe.metadata.name)
+ // #299: the policy has landed — record the set it actually enforces.
+ this.logResolvedEgressSet(
+ recipe.metadata.name,
+ policyName,
+ externals.map(e => e.fqdn),
+ effectiveResolved,
+ egressStateChanged
+ )
}
+ }
- await this.createOrReplace(
- () =>
- this.networkingApi.createNamespacedNetworkPolicy({
- namespace: ns,
- body: policy,
- }),
- async () => {
- const existing = await this.networkingApi.readNamespacedNetworkPolicy({
- name: policyName,
- namespace: ns,
- })
- policy.metadata!.resourceVersion = existing.metadata?.resourceVersion
- return this.networkingApi.replaceNamespacedNetworkPolicy({
- name: policyName,
- namespace: ns,
- body: policy,
- })
- },
- `NetworkPolicy "${policyName}" in ${ns}`
- )
- // #299: the policy has landed — record the set it actually enforces.
- this.logResolvedEgressSet(
- recipe.metadata.name,
- policyName,
- externals.map(e => e.fqdn),
- effectiveResolved,
- egressStateChanged
- )
+ private async contractUiIngressPolicies(recipe: WorkflowRecipeCRD): Promise<void> {
+ const namespace = this.config.sandboxNamespace
+ const portsByWorkload = new Map<string, Set<number>>()
+ for (const rule of recipe.spec.ui?.egress?.internal ?? []) {
+ const ports = portsByWorkload.get(rule.workloadRef) ?? new Set<number>()
+ ports.add(rule.port)
+ portsByWorkload.set(rule.workloadRef, ports)
+ }
+
+ for (const workload of recipe.spec.workloads ?? []) {
+ const name = rb.uiIngressPolicyName(recipe.metadata.name, workload.id)
+ const existing = await this.readNetworkPolicyOrNull(name, namespace)
+ if (!existing) continue
+ const desired = rb.buildUiIngressNetworkPolicy(
+ recipe,
+ workload.id,
+ [...(portsByWorkload.get(workload.id) ?? [])],
+ namespace,
+ this.config.sandboxUiNamespace
+ )
+ await this.applyNetworkPolicyContraction({
+ existing,
+ desired: subtractiveNetworkPolicy(existing, desired),
+ namespace,
+ recipeName: recipe.metadata.name,
+ })
+ }
}
/**
@@ -4280,7 +4466,7 @@ export class WorkflowRecipeReconciler {
await this.deleteUiIngressIfExists(recipe.metadata.name, w.id, ns)
continue
}
- await this.applyNetworkPolicy(policy, ns)
+ await this.applyOwnedRecipeNetworkPolicy(policy, ns, recipe.metadata.name)
}
}
@@ -4290,10 +4476,14 @@ export class WorkflowRecipeReconciler {
namespace: string
): Promise<void> {
const name = rb.uiIngressPolicyName(recipeName, workloadId)
- await this.safeDelete(
- () => this.networkingApi.deleteNamespacedNetworkPolicy({ name, namespace }),
- `NetworkPolicy "${name}" in ${namespace}`
- )
+ const existing = await this.readNetworkPolicyOrNull(name, namespace)
+ if (!existing) return
+ await this.applyNetworkPolicyContraction({
+ existing,
+ desired: null,
+ namespace,
+ recipeName,
+ })
}
private internalDependencyCondition(
@@ -4515,6 +4705,117 @@ export class WorkflowRecipeReconciler {
}
}
+ private async contractWorkloadEgressPolicies(
+ recipe: WorkflowRecipeCRD
+ ): Promise<WorkloadEgressContractionPlan> {
+ const workloads = recipe.spec.workloads ?? []
+ const uiWorkloadId = recipe.spec.ui?.workloadRef
+ const ingressSourcesByTarget = new Map<string, rb.WorkloadIngressSource[]>()
+ const plans: WorkloadEgressContractionPlan['plans'] = []
+
+ for (const workload of workloads) {
+ if (workload.transport) continue
+ if (uiWorkloadId && workload.id === uiWorkloadId) continue
+ const namespace = this.resolveWorkloadNamespace(workload, uiWorkloadId)
+ const bindings = workload.egressBindings ?? []
+ if (bindings.length === 0) {
+ await this.deleteWorkloadEgressIfExists(recipe.metadata.name, workload.id, namespace)
+ continue
+ }
+
+ const externalDeclared: Array<{ fqdn: string; port: number }> = []
+ for (const binding of bindings) {
+ if (!binding.dns || binding.port == null) continue
+ const resolved = rb.resolveClusterLocalBinding(binding.dns, recipe, namespace)
+ if (resolved?.kind === 'cluster-local') {
+ const sources = ingressSourcesByTarget.get(resolved.workloadId) ?? []
+ sources.push({
+ fromWorkloadId: workload.id,
+ fromNamespace: namespace,
+ port: binding.port,
+ protocol: (binding.protocol ?? 'TCP') as 'TCP' | 'UDP',
+ })
+ ingressSourcesByTarget.set(resolved.workloadId, sources)
+ } else if (!resolved) {
+ externalDeclared.push({ fqdn: binding.dns, port: binding.port })
+ }
+ }
+
+ const policyName = rb.workloadEgressPolicyName(recipe.metadata.name, workload.id)
+ await this.contractOneWorkloadEgressPolicy(recipe, workload, namespace, externalDeclared)
+ plans.push({ workload, namespace, policyName, externalDeclared })
+ }
+
+ return { plans, ingressSourcesByTarget }
+ }
+
+ private async contractOneWorkloadEgressPolicy(
+ recipe: WorkflowRecipeCRD,
+ workload: WorkloadDef,
+ namespace: string,
+ externalDeclared: Array<{ fqdn: string; port: number }>
+ ): Promise<k8s.V1NetworkPolicy | null> {
+ const policyName = rb.workloadEgressPolicyName(recipe.metadata.name, workload.id)
+ const existing = await this.readNetworkPolicyOrNull(policyName, namespace)
+ if (!existing) return null
+
+ const contraction = contractExternalEgress({
+ externals: externalDeclared,
+ previousAnnotations: existing.metadata?.annotations,
+ now: Date.now(),
+ config: {
+ overlapMs: this.config.externalEgressOverlapSeconds * 1000,
+ maxEntries: this.config.externalEgressMaxEntries,
+ },
+ isAllowedIp: ip => !isBlockedExternalIPv4(ip),
+ })
+ const needsStateAnnotations =
+ externalDeclared.length > 0 ||
+ contraction.entries.length > 0 ||
+ existing.metadata?.annotations?.[STATE_ANNOTATION] !== undefined
+ const desired = rb.buildWorkloadEgressNetworkPolicy(
+ workload,
+ recipe,
+ namespace,
+ contraction.resolved,
+ needsStateAnnotations ? contraction.annotations : undefined
+ )
+ return this.applyNetworkPolicyContraction({
+ existing,
+ desired: subtractiveNetworkPolicy(existing, desired),
+ namespace,
+ recipeName: recipe.metadata.name,
+ })
+ }
+
+ private async contractWorkloadIngressPolicies(
+ recipe: WorkflowRecipeCRD,
+ ingressSourcesByTarget: Map<string, rb.WorkloadIngressSource[]>
+ ): Promise<void> {
+ const workloads = recipe.spec.workloads ?? []
+ const uiWorkloadId = recipe.spec.ui?.workloadRef
+ for (const workload of workloads) {
+ if (workload.transport) continue
+ if (uiWorkloadId && workload.id === uiWorkloadId) continue
+ const namespace = this.resolveWorkloadNamespace(workload, uiWorkloadId)
+ const name = rb.workloadIngressPolicyName(recipe.metadata.name, workload.id)
+ const existing = await this.readNetworkPolicyOrNull(name, namespace)
+ if (!existing) continue
+ const desired = rb.buildWorkloadIngressNetworkPolicy(
+ workload,
+ recipe,
+ namespace,
+ ingressSourcesByTarget.get(workload.id) ?? []
+ )
+ await this.applyNetworkPolicyContraction({
+ existing,
+ desired: subtractiveNetworkPolicy(existing, desired),
+ namespace,
+ recipeName: recipe.metadata.name,
+ })
+ }
+ }
+
/**
* Reconcile per-workload egress NetworkPolicies for each workload that
* declares `egressBindings[]`. Also emits the symmetric ingress policy
@@ -4530,80 +4831,45 @@ export class WorkflowRecipeReconciler {
* silent authorization drift, so any unresolved or blocked hostname fails
* the reconcile before policy apply.
*/
- private async reconcileWorkloadEgressPolicies(recipe: WorkflowRecipeCRD): Promise<void> {
+ private async reconcileWorkloadEgressPolicies(
+ recipe: WorkflowRecipeCRD,
+ preparation: WorkloadEgressContractionPlan
+ ): Promise<void> {
const workloads = recipe.spec.workloads ?? []
if (workloads.length === 0) return
const uiWorkloadId = recipe.spec.ui?.workloadRef
+ const { plans, ingressSourcesByTarget } = preparation
- // Aggregate ingress sources keyed by target workload id. Filled as we
- // walk each source workload's cluster-local bindings.
- const ingressSourcesByTarget = new Map<string, rb.WorkloadIngressSource[]>()
-
- for (const w of workloads) {
- // Skip MCP + UI workloads — covered by other code paths.
- if (w.transport) continue
- if (uiWorkloadId && w.id === uiWorkloadId) continue
- const wlNs = this.resolveWorkloadNamespace(w, uiWorkloadId)
- const bindings = w.egressBindings ?? []
- if (bindings.length === 0) {
- // Reap any policy left over from a previous reconcile that DID
- // have bindings — keeps the policy set in lockstep with the spec.
- await this.deleteWorkloadEgressIfExists(recipe.metadata.name, w.id, wlNs)
- continue
- }
-
- // Split into cluster-local sibling targets vs external FQDNs.
- const externalDeclared: { fqdn: string; port: number }[] = []
- for (const b of bindings) {
- if (!b.dns || b.port == null) continue
- const port = b.port
- const resolved = rb.resolveClusterLocalBinding(b.dns, recipe, wlNs)
- if (resolved && resolved.kind === 'cluster-local') {
- const sources = ingressSourcesByTarget.get(resolved.workloadId) ?? []
- sources.push({
- fromWorkloadId: w.id,
- fromNamespace: wlNs,
- port,
- protocol: (b.protocol ?? 'TCP') as 'TCP' | 'UDP',
- })
- ingressSourcesByTarget.set(resolved.workloadId, sources)
- } else if (!resolved) {
- // null = treat as external FQDN
- externalDeclared.push({ fqdn: b.dns, port })
- }
- // resolved.kind === 'mismatch' is unreachable here — validation
- // ran upstream and would have thrown.
+ // Phase 2: resolve and add/freeze only the identities that survived phase 1.
+ for (const plan of plans) {
+ const { workload: w, namespace: wlNs, policyName: wlPolicyName, externalDeclared } = plan
+ let resolution: Awaited<ReturnType<typeof resolveExternalEgress>>
+ try {
+ resolution = await resolveExternalEgress(externalDeclared, this.fqdnLookup)
+ } catch (error: unknown) {
+ await this.contractOneWorkloadEgressPolicy(recipe, w, wlNs, externalDeclared)
+ throw error
}
-
- const { resolved: resolvedExternal, failures } = await resolveExternalEgress(
- externalDeclared.map(e => ({ fqdn: e.fqdn, port: e.port })),
- this.fqdnLookup
+ let existingWlPolicy = await this.contractOneWorkloadEgressPolicy(
+ recipe,
+ w,
+ wlNs,
+ externalDeclared
)
+ const { resolved: resolvedExternal, failures } = resolution
this.recordExternalEgressTtl(resolvedExternal)
- // Permanent failures never author a partial policy — fail as before.
const permanentFailures = failures.filter(f => !f.retryable)
- if (permanentFailures.length > 0) {
- throw egressResolutionError(
- `WorkflowRecipe "${recipe.metadata.name}" workload "${w.id}" egress resolution failed`,
- permanentFailures
- )
- }
+ let blockingFailures = permanentFailures
let effectiveExternal: rb.ResolvedExternalEgressInput[] = resolvedExternal
let wlStateAnnotations: Record<string, string> | undefined
- let existingWlPolicy: k8s.V1NetworkPolicy | null = null
let wlEgressRenewalDue = false
let wlEgressStateChanged = false // H-E: catch fqdn-attribution-only changes
- const wlPolicyName = rb.workloadEgressPolicyName(recipe.metadata.name, w.id)
- // R1-M2: read the live policy for ALL cases so an internal-only (cluster-
- // local) workload egress policy hits the no-op gate instead of churning
- // every reconcile; for external egress it also seeds rehydration (H5).
- existingWlPolicy = await this.readNetworkPolicyOrNull(wlPolicyName, wlNs)
if (externalDeclared.length > 0) {
// issue #299: accumulate the sliding-window egress set (rehydrate H5).
const acc = accumulateExternalEgress({
- externals: externalDeclared.map(e => ({ fqdn: e.fqdn, port: e.port })),
+ externals: externalDeclared,
resolveResult: { resolved: resolvedExternal, failures },
previousAnnotations: existingWlPolicy?.metadata?.annotations,
now: Date.now(),
@@ -4614,10 +4880,7 @@ export class WorkflowRecipeReconciler {
})
// Bootstrap fail-closed: transient failure with nothing to freeze.
if (acc.entries.length === 0 && failures.length > 0) {
- throw egressResolutionError(
- `WorkflowRecipe "${recipe.metadata.name}" workload "${w.id}" egress resolution failed`,
- failures
- )
+ blockingFailures = failures
}
this.warnEgressAccumulator(wlPolicyName, acc)
// Defense-in-depth (audit M3): re-validate rehydrated IPs vs blocked ranges.
@@ -4641,8 +4904,30 @@ export class WorkflowRecipeReconciler {
effectiveExternal,
wlStateAnnotations
)
+ if (blockingFailures.length > 0) {
+ if (existingWlPolicy) {
+ await this.applyNetworkPolicyContraction({
+ existing: existingWlPolicy,
+ desired: subtractiveNetworkPolicy(existingWlPolicy, policy),
+ namespace: wlNs,
+ recipeName: recipe.metadata.name,
+ })
+ }
+ throw egressResolutionError(
+ `WorkflowRecipe "${recipe.metadata.name}" workload "${w.id}" egress resolution failed`,
+ blockingFailures
+ )
+ }
if (!policy) {
- await this.deleteWorkloadEgressIfExists(recipe.metadata.name, w.id, wlNs)
+ if (existingWlPolicy) {
+ await this.applyNetworkPolicyContraction({
+ existing: existingWlPolicy,
+ desired: null,
+ namespace: wlNs,
+ recipeName: recipe.metadata.name,
+ })
+ existingWlPolicy = null
+ }
continue
}
// issue #299: NO-OP when the accumulated set and rules are already live —
@@ -4655,17 +4940,17 @@ export class WorkflowRecipeReconciler {
console.log(
`[WR-Reconciler] NetworkPolicy "${wlPolicyName}" in ${wlNs} egress set unchanged — no-op`
)
- continue
+ } else {
+ await this.applyOwnedRecipeNetworkPolicy(policy, wlNs, recipe.metadata.name)
+ // #299: the policy has landed — record the set it actually enforces.
+ this.logResolvedEgressSet(
+ recipe.metadata.name,
+ wlPolicyName,
+ externalDeclared.map(e => e.fqdn),
+ effectiveExternal,
+ wlEgressStateChanged
+ )
}
- await this.applyNetworkPolicy(policy, wlNs)
- // #299: the policy has landed — record the set it actually enforces.
- this.logResolvedEgressSet(
- recipe.metadata.name,
- wlPolicyName,
- externalDeclared.map(e => e.fqdn),
- effectiveExternal,
- wlEgressStateChanged
- )
}
// Now ingress side: for each target workload that any sibling pointed
@@ -4684,7 +4969,7 @@ export class WorkflowRecipeReconciler {
await this.deleteWorkloadIngressIfExists(recipe.metadata.name, w.id, wlNs)
continue
}
- await this.applyNetworkPolicy(policy, wlNs)
+ await this.applyOwnedRecipeNetworkPolicy(policy, wlNs, recipe.metadata.name)
}
}
@@ -4728,6 +5013,150 @@ export class WorkflowRecipeReconciler {
}
}
+ private assertRecipeNetworkPolicyOwnership(
+ existing: k8s.V1NetworkPolicy,
+ recipeName: string,
+ policyName: string,
+ namespace: string
+ ): void {
+ const labels = existing.metadata?.labels ?? {}
+ if (
+ labels['clerum.io/managed-by'] !== 'workflow-recipes' ||
+ labels['clerum.io/recipe'] !== recipeName
+ ) {
+ throw new NetworkPolicyOwnershipConflictError(
+ `Refusing to mutate NetworkPolicy "${policyName}" in ${namespace}: existing policy is not owned by WorkflowRecipe "${recipeName}"`
+ )
+ }
+ }
+
+ /**
+ * Apply a strictly subtractive pre-DNS transition against the exact snapshot
+ * that was read. This path never creates a policy and therefore cannot add an
+ * authorization before DNS succeeds. ResourceVersion/UID fencing makes a
+ * concurrent replacement fail loud instead of mutating a newer object.
+ */
+ private async applyNetworkPolicyContraction(input: {
+ existing: k8s.V1NetworkPolicy
+ desired: k8s.V1NetworkPolicy | null
+ namespace: string
+ recipeName: string
+ }): Promise<k8s.V1NetworkPolicy | null> {
+ const { existing, desired, namespace, recipeName } = input
+ const policyName = existing.metadata?.name
+ const resourceVersion = existing.metadata?.resourceVersion
+ if (!policyName || !resourceVersion) {
+ throw new Error('NetworkPolicy contraction requires policy name and resourceVersion')
+ }
+ this.assertRecipeNetworkPolicyOwnership(existing, recipeName, policyName, namespace)
+
+ if (desired === null) {
+ const uid = existing.metadata?.uid
+ if (!uid) throw new Error('NetworkPolicy contraction delete requires policy uid')
+ try {
+ await this.networkingApi.deleteNamespacedNetworkPolicy({
+ name: policyName,
+ namespace,
+ body: { preconditions: { uid, resourceVersion } },
+ })
+ } catch (error: unknown) {
+ if (getErrorCode(error) !== 404) throw error
+ }
+ createLogger('wrc', recipeName).info('contracted recipe network policy', {
+ policy: policyName,
+ namespace,
+ action: 'delete',
+ })
+ return null
+ }
+
+ if (!contractionWriteNeeded(existing, desired)) return existing
+ desired.metadata = { ...(desired.metadata ?? {}), resourceVersion }
+ await this.networkingApi.replaceNamespacedNetworkPolicy({
+ name: policyName,
+ namespace,
+ body: desired,
+ })
+ let live: k8s.V1NetworkPolicy
+ try {
+ live = await this.networkingApi.readNamespacedNetworkPolicy({
+ name: policyName,
+ namespace,
+ })
+ } catch (error: unknown) {
+ if (getErrorCode(error) === 404) return null
+ throw error
+ }
+ this.assertRecipeNetworkPolicyOwnership(live, recipeName, policyName, namespace)
+ const liveUid = live.metadata?.uid
+ const liveResourceVersion = live.metadata?.resourceVersion
+ if (
+ live.metadata?.name !== policyName ||
+ live.metadata?.namespace !== namespace ||
+ !liveUid ||
+ !liveResourceVersion ||
+ (existing.metadata?.uid && liveUid !== existing.metadata.uid)
+ ) {
+ throw new NetworkPolicyOwnershipConflictError(
+ `NetworkPolicy "${policyName}" in ${namespace} changed identity during contraction`
+ )
+ }
+ const provenLive = subtractiveNetworkPolicy(live, desired)
+ if (!provenLive || contractionWriteNeeded(live, provenLive)) {
+ // The apiserver read-back is broader or carries provenance that does not
+ // match its enforced rules. Do not let phase 2 treat the request body as
+ // reality; revoke this exact owned identity and force a clean re-plan.
+ await this.networkingApi.deleteNamespacedNetworkPolicy({
+ name: policyName,
+ namespace,
+ body: { preconditions: { uid: liveUid, resourceVersion: liveResourceVersion } },
+ })
+ throw new NetworkPolicyOwnershipConflictError(
+ `NetworkPolicy "${policyName}" in ${namespace} did not read back as the requested contraction`
+ )
+ }
+ createLogger('wrc', recipeName).info('contracted recipe network policy', {
+ policy: policyName,
+ namespace,
+ action: 'replace',
+ })
+ return live
+ }
+
+ private async applyOwnedRecipeNetworkPolicy(
+ policy: k8s.V1NetworkPolicy,
+ namespace: string,
+ recipeName: string
+ ): Promise<void> {
+ const policyName = policy.metadata?.name
+ if (!policyName) throw new Error('Recipe NetworkPolicy requires a name')
+ await this.createOrReplace(
+ () => this.networkingApi.createNamespacedNetworkPolicy({ namespace, body: policy }),
+ async () => {
+ const live = await this.networkingApi.readNamespacedNetworkPolicy({
+ name: policyName,
+ namespace,
+ })
+ this.assertRecipeNetworkPolicyOwnership(live, recipeName, policyName, namespace)
+ if (!live.metadata?.resourceVersion) {
+ throw new NetworkPolicyOwnershipConflictError(
+ `Refusing to replace NetworkPolicy "${policyName}" in ${namespace}: live resourceVersion is missing`
+ )
+ }
+ policy.metadata = {
+ ...(policy.metadata ?? {}),
+ resourceVersion: live.metadata?.resourceVersion,
+ }
+ return this.networkingApi.replaceNamespacedNetworkPolicy({
+ name: policyName,
+ namespace,
+ body: policy,
+ })
+ },
+ `NetworkPolicy "${policyName}" in ${namespace}`
+ )
+ }
+
/**
* True when the live policy differs from the desired one and must be written
* (issue #299 NO-OP gate). Compares the accumulated-state fingerprint AND the
@@ -5050,10 +5479,14 @@ export class WorkflowRecipeReconciler {
namespace: string
): Promise<void> {
const name = rb.workloadEgressPolicyName(recipeName, workloadId)
- await this.safeDelete(
- () => this.networkingApi.deleteNamespacedNetworkPolicy({ name, namespace }),
- `NetworkPolicy "${name}" in ${namespace}`
- )
+ const existing = await this.readNetworkPolicyOrNull(name, namespace)
+ if (!existing) return
+ await this.applyNetworkPolicyContraction({
+ existing,
+ desired: null,
+ namespace,
+ recipeName,
+ })
}
private async deleteWorkloadIngressIfExists(
@@ -5062,10 +5495,14 @@ export class WorkflowRecipeReconciler {
namespace: string
): Promise<void> {
const name = rb.workloadIngressPolicyName(recipeName, workloadId)
- await this.safeDelete(
- () => this.networkingApi.deleteNamespacedNetworkPolicy({ name, namespace }),
- `NetworkPolicy "${name}" in ${namespace}`
- )
+ const existing = await this.readNetworkPolicyOrNull(name, namespace)
+ if (!existing) return
+ await this.applyNetworkPolicyContraction({
+ existing,
+ desired: null,
+ namespace,
+ recipeName,
+ })
}
// ─── Workload Ensure Methods ──────────────────────────────────────