Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
fix(egress): enforce proven fail-closed convergence
Alfredo Lopezde776803800e
diff --git a/host-context-controller/package-lock.json b/host-context-controller/package-lock.jsonindex 4317836a5..c390d2654 100644--- a/host-context-controller/package-lock.json+++ b/host-context-controller/package-lock.json@@ -1,12 +1,12 @@{"name": "clerum-host-context-controller",- "version": "0.1.295",+ "version": "0.1.296","lockfileVersion": 3,"requires": true,"packages": {"": {"name": "clerum-host-context-controller",- "version": "0.1.295",+ "version": "0.1.296","license": "SEE LICENSE IN LICENSE","dependencies": {"@clerum/codex-catalog-projection": "file:../packages/codex-catalog-projection",diff --git a/host-context-controller/package.json b/host-context-controller/package.jsonindex a8980c5b6..dd067e323 100644--- a/host-context-controller/package.json+++ b/host-context-controller/package.json@@ -1,6 +1,6 @@{"name": "clerum-host-context-controller",- "version": "0.1.295",+ "version": "0.1.296","description": "Host Context Controller - provides McpServer CRDs to mcp-host via REST API","main": "dist/main.js","engines": {diff --git a/host-context-controller/src/externalEgressAccumulator.test.ts b/host-context-controller/src/externalEgressAccumulator.test.tsindex 63419c553..33084e5b1 100644--- a/host-context-controller/src/externalEgressAccumulator.test.ts+++ b/host-context-controller/src/externalEgressAccumulator.test.ts@@ -153,4 +153,52 @@ describe('accumulateHostExactHostEgress — issue #299 sliding window (HCC)', ()expect(r.annotations[STATE_ANNOTATION]).toBeTruthy()expect(r.annotations[RESOLVED_AT_ANNOTATION]).toBe(new Date(T0).toISOString())})++ it('preserves resolved-at on a DNS failure instead of inventing a fresh resolution time', () => {+ const r1 = accumulateHostExactHostEgress(baseInput({ resolution: ok(['140.82.112.3'], 15) }))+ const r2 = accumulateHostExactHostEgress(+ baseInput({+ resolution: { kind: 'transient' },+ previousAnnotations: r1.annotations,+ now: T0 + 400_000,+ })+ )+ expect(r2.annotations[RESOLVED_AT_ANNOTATION]).toBe(new Date(T0).toISOString())+ expect(r2.resolvedAt).toBe(new Date(T0).toISOString())+ })++ it('derives resolved-at from proven structured state when the legacy timestamp is absent', () => {+ const r1 = accumulateHostExactHostEgress(baseInput({ resolution: ok(['140.82.112.3'], 15) }))+ const previousAnnotations = { ...r1.annotations }+ delete previousAnnotations[RESOLVED_AT_ANNOTATION]+ const r2 = accumulateHostExactHostEgress(+ baseInput({+ resolution: { kind: 'transient' },+ previousAnnotations,+ now: T0 + 400_000,+ })+ )+ expect(r2.resolvedAt).toBe(new Date(T0).toISOString())+ expect(r2.annotations[RESOLVED_AT_ANNOTATION]).toBe(new Date(T0).toISOString())+ })++ it('does not synthesize resolved-at for a bootstrap failure', () => {+ const r = accumulateHostExactHostEgress(baseInput({ resolution: { kind: 'transient' } }))+ expect(r.annotations[RESOLVED_AT_ANNOTATION]).toBeUndefined()+ expect(r.resolvedAt).toBeUndefined()+ })++ it('does not claim a DNS timestamp for legacy portless state during a failure', () => {+ const r = accumulateHostExactHostEgress(+ baseInput({+ resolution: { kind: 'transient' },+ previousAnnotations: {+ [TARGETS_ANNOTATION]: 'api.github.com=140.82.112.9/32',+ },+ })+ )+ expect(r.cidrs).toEqual(['140.82.112.9/32'])+ expect(r.resolvedAt).toBeUndefined()+ expect(r.annotations[RESOLVED_AT_ANNOTATION]).toBeUndefined()+ })})diff --git a/host-context-controller/src/externalEgressAccumulator.ts b/host-context-controller/src/externalEgressAccumulator.tsindex d863df558..4dceda778 100644--- a/host-context-controller/src/externalEgressAccumulator.ts+++ b/host-context-controller/src/externalEgressAccumulator.ts@@ -29,6 +29,7 @@ import {RESOLVED_AT_ANNOTATION,emptyState,parseState,+ parseStateStrict,reconcileEgressState,serializeState,} from '@clerum/network-policy-core'@@ -56,10 +57,14 @@ export interface AccumulateHostEgressInput {}export interface AccumulateHostEgressOutput {+ /** Normalized state entries behind cidrs/annotations. */+ entries: ReturnType<typeof emptyState>['entries']/** The effective accumulated /32 set to render as ipBlocks (sorted, deduped). */cidrs: string[]/** Annotations to stamp on the policy: serialized state + resolved-at. */annotations: Record<string, string>+ /** Last accepted DNS observation represented by the effective set, if proven. */+ resolvedAt?: string/** True iff the (fqdn,ip,port,protocol) set changed vs the previous state. */changed: boolean/** True iff the persisted window is aging and must be re-persisted (audit M1). */@@ -107,12 +112,35 @@ export function accumulateHostExactHostEgress(const out = reconcileEgressState(previous, [observation], now, config)+ const annotations = serializeState(out.state)+ let resolvedAt: string | undefined+ if (resolution.kind === 'ok') {+ resolvedAt = new Date(now).toISOString()+ } else {+ const priorResolvedAt = previousAnnotations?.[RESOLVED_AT_ANNOTATION]+ if (priorResolvedAt && Number.isFinite(Date.parse(priorResolvedAt))) {+ resolvedAt = priorResolvedAt+ } else if (previousAnnotations) {+ const strictPrevious = parseStateStrict(previousAnnotations, config, [+ { fqdn, port, protocol },+ ])+ if (strictPrevious.kind === 'valid' && strictPrevious.state.entries.length > 0) {+ const latestObservedAt = Math.max(+ ...strictPrevious.state.entries.map(entry => entry.lastObservedAt)+ )+ if (Number.isFinite(new Date(latestObservedAt).getTime())) {+ resolvedAt = new Date(latestObservedAt).toISOString()+ }+ }+ }+ }+ if (resolvedAt) annotations[RESOLVED_AT_ANNOTATION] = resolvedAt+return {+ entries: out.entries,cidrs: out.entries.map(entry => `${entry.ip}/32`),- annotations: {- ...serializeState(out.state),- [RESOLVED_AT_ANNOTATION]: new Date(now).toISOString(),- },+ annotations,+ resolvedAt,changed: out.changed,renewalDue: out.renewalDue,frozen: out.frozenFqdns.includes(fqdn),diff --git a/host-context-controller/src/networkPolicyReconciler.egressClassification.test.ts b/host-context-controller/src/networkPolicyReconciler.egressClassification.test.tsindex cc4bc902f..aacb2e14f 100644--- a/host-context-controller/src/networkPolicyReconciler.egressClassification.test.ts+++ b/host-context-controller/src/networkPolicyReconciler.egressClassification.test.ts@@ -88,25 +88,42 @@ function makeReconciler(currentContext: 'test',})const reconciler = new NetworkPolicyReconciler(kc, new Map())+ const normalizePolicy = (+ policy: k8s.V1NetworkPolicy,+ namespace: string,+ index = 0+ ): k8s.V1NetworkPolicy => {+ const name = policy.metadata?.name ?? `unnamed-${index}`+ return {+ ...policy,+ metadata: {+ ...policy.metadata,+ name,+ namespace: policy.metadata?.namespace ?? namespace,+ uid: policy.metadata?.uid ?? `${namespace}:${name}:uid`,+ resourceVersion: policy.metadata?.resourceVersion ?? '1',+ },+ }+ }const networkingApi = {...mockApi,listNamespacedNetworkPolicy: async (args: { namespace: string }) => {const response = await mockApi.listNamespacedNetworkPolicy(args)- const items = (response.items ?? []).map((policy: k8s.V1NetworkPolicy, index: number) => {- const name = policy.metadata?.name ?? `unnamed-${index}`- return {- ...policy,- metadata: {- ...policy.metadata,- name,- namespace: policy.metadata?.namespace ?? args.namespace,- uid: policy.metadata?.uid ?? `${args.namespace}:${name}:uid`,- resourceVersion: policy.metadata?.resourceVersion ?? '1',- },- }- })+ const items = (response.items ?? []).map((policy: k8s.V1NetworkPolicy, index: number) =>+ normalizePolicy(policy, args.namespace, index)+ )return { ...response, items }},+ readNamespacedNetworkPolicy: async (args: { name: string; namespace: string }) => {+ const direct = (await mockApi.readNamespacedNetworkPolicy(args)) as k8s.V1NetworkPolicy+ if (direct.metadata?.name || direct.spec) return normalizePolicy(direct, args.namespace)+ const response = await mockApi.listNamespacedNetworkPolicy({ namespace: args.namespace })+ const found = (response.items ?? []).find(+ (policy: k8s.V1NetworkPolicy) => policy.metadata?.name === args.name+ )+ if (!found) throw Object.assign(new Error('not found'), { code: 404 })+ return normalizePolicy(found, args.namespace)+ },};(reconciler as unknown as { networkingApi: unknown }).networkingApi = networkingApi;(reconciler as unknown as { customApi: unknown }).customApi = mockCustomApi@@ -184,6 +201,15 @@ describe('#513: a non-DNS exception inside the resolve block is not a DNS condit])}+ function injectPostResolutionCodedError(code: string): void {+ const record = Object.defineProperty({ ttl: 300 }, 'address', {+ get() {+ throw Object.assign(new Error(`post-resolution ${code}`), { code })+ },+ }) as RecordWithTtl+ resolve4Mock.mockResolvedValueOnce([record])+ }+type Condition = { type?: string; status?: string; reason?: string; message?: string }/**@@ -218,9 +244,11 @@ describe('#513: a non-DNS exception inside the resolve block is not a DNS conditconst warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined)injectPostResolutionTypeError()- // No throw: there is a live policy, so the fault is served fail-static (T7).- // Everything else this test pins is unchanged.- await reconciler.reconcileExternalEgress(server, { isCurrent: () => true })+ // A verified prior policy may remain for already-running pods, but the+ // controller fault still blocks new/update runtime reconciliation.+ await expect(+ reconciler.reconcileExternalEgress(server, { isCurrent: () => true })+ ).rejects.toThrow(TypeError)// The operator must not be pointed at DNS for a fault in the reconciler.expect(warn.mock.calls.flat().join('\n')).not.toContain('permanent DNS failure')@@ -242,9 +270,9 @@ describe('#513: a non-DNS exception inside the resolve block is not a DNS conditawait seedAccumulatedPolicy({ lapsed: true })injectPostResolutionTypeError()- // No throw: the policy is live (its window lapsed, the object did not), so- // the fault is served fail-static (T7). The prune assertion is the point.- await reconciler.reconcileExternalEgress(server, { isCurrent: () => true })+ await expect(+ reconciler.reconcileExternalEgress(server, { isCurrent: () => true })+ ).rejects.toThrow(TypeError)// Pre-fix this prunes the lapsed set to zero, hits the `continue`, and the// policy is swept as undesired. D4 fail-static must hold for a fault that is@@ -255,6 +283,86 @@ describe('#513: a non-DNS exception inside the resolve block is not a DNS conditexpect(egress.at(-1)?.reason).toBe('ExternalEgressReconcileFailed')})+ it('T2b never promotes annotation-only state during transient fail-static', async () => {+ await seedAccumulatedPolicy({ lapsed: false })+ const listed = (await mockApi.listNamespacedNetworkPolicy({ namespace: 'mcp-server' })) as {+ items: k8s.V1NetworkPolicy[]+ }+ const drifted = structuredClone(listed)+ const state = JSON.parse(+ drifted.items[0].metadata!.annotations!['clerum.io/egress-fqdn-state']+ ) as Array<Record<string, unknown>>+ state.push({+ ...state[0],+ ip: '8.8.8.8',+ })+ drifted.items[0].metadata!.annotations!['clerum.io/egress-fqdn-state'] = JSON.stringify(state)+ mockApi.listNamespacedNetworkPolicy.mockResolvedValue(drifted)+ resolve4Mock.mockRejectedValueOnce(+ Object.assign(new Error('queryA ESERVFAIL api.example.com'), { code: 'ESERVFAIL' })+ )++ await expect(+ reconciler.reconcileExternalEgress(server, { isCurrent: () => true })+ ).rejects.toThrow(/Fail-static proof failed/)++ expect(mockApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledWith(+ expect.objectContaining({ name: drifted.items[0].metadata!.name })+ )+ const written = mockApi.replaceNamespacedNetworkPolicy.mock.calls+ .map(call => JSON.stringify(call[0]?.body))+ .join(' ')+ expect(written).not.toContain('8.8.8.8')+ const egress = statusConditions().filter(c => c.type === 'ExternalEgressReady')+ expect(egress.at(-1)?.status).toBe('False')+ expect(egress.at(-1)?.reason).toBe('ExternalEgressReconcileFailed')+ expect(egress.at(-1)?.message).toContain('REVOKED')+ })++ it('T2c never promotes annotation-only state after an accepted DNS answer', async () => {+ await seedAccumulatedPolicy({ lapsed: false })+ const listed = (await mockApi.listNamespacedNetworkPolicy({ namespace: 'mcp-server' })) as {+ items: k8s.V1NetworkPolicy[]+ }+ const drifted = structuredClone(listed)+ const state = JSON.parse(+ drifted.items[0].metadata!.annotations!['clerum.io/egress-fqdn-state']+ ) as Array<Record<string, unknown>>+ state.push({ ...state[0], ip: '8.8.8.8' })+ drifted.items[0].metadata!.annotations!['clerum.io/egress-fqdn-state'] = JSON.stringify(state)+ mockApi.listNamespacedNetworkPolicy.mockResolvedValue(drifted)++ await reconciler.reconcileExternalEgress(server, { isCurrent: () => true })++ const writes = [+ ...mockApi.createNamespacedNetworkPolicy.mock.calls,+ ...mockApi.replaceNamespacedNetworkPolicy.mock.calls,+ ]+ .map(call => JSON.stringify(call[0]?.body))+ .join(' ')+ expect(writes).not.toContain('8.8.8.8')+ expect(writes).toContain('1.2.3.4')+ expect(servedPayload()).not.toContain('8.8.8.8')+ expect(servedPayload()).toContain('1.2.3.4')+ })++ it.each(['ETIMEDOUT', 'ENOTFOUND'])(+ 'T2a keeps a post-resolution %s exception out of DNS classification',+ async code => {+ await seedAccumulatedPolicy({ lapsed: false })+ injectPostResolutionCodedError(code)++ await expect(+ reconciler.reconcileExternalEgress(server, { isCurrent: () => true })+ ).rejects.toThrow(`post-resolution ${code}`)++ expect(mockApi.deleteNamespacedNetworkPolicy).not.toHaveBeenCalled()+ const egress = statusConditions().filter(c => c.type === 'ExternalEgressReady')+ expect(egress.at(-1)?.reason).toBe('ExternalEgressReconcileFailed')+ expect(egress.at(-1)?.message).not.toContain('DNS failure')+ }+ )+it('T3 still prunes and fails loud on a genuine NXDOMAIN with a lapsed window', async () => {await seedAccumulatedPolicy({ lapsed: true })resolve4Mock.mockRejectedValueOnce(@@ -272,6 +380,22 @@ describe('#513: a non-DNS exception inside the resolve block is not a DNS conditexpect(egress.at(-1)?.reason).toBe('ExternalEgressRejected')})+ it('T3a revokes and blocks on genuine NXDOMAIN even while the prior window is unexpired', async () => {+ await seedAccumulatedPolicy({ lapsed: false })+ resolve4Mock.mockRejectedValueOnce(+ Object.assign(new Error('queryA ENOTFOUND api.example.com'), { code: 'ENOTFOUND' })+ )++ await expect(+ reconciler.reconcileExternalEgress(server, { isCurrent: () => true })+ ).rejects.toThrow(/failed to resolve hostname "api\.example\.com"/)++ expect(mockApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledTimes(1)+ const egress = statusConditions().filter(c => c.type === 'ExternalEgressReady')+ expect(egress.at(-1)?.status).toBe('False')+ expect(egress.at(-1)?.reason).toBe('ExternalEgressRejected')+ })+it("T6 still deletes a de-authorized binding's policy when a sibling binding faults", async () => {// Fail-loud must not cost fail-closed. `cleanupExternalEgress` runs AFTER the// binding loop and is the only place that deletes policies for bindings no@@ -305,11 +429,11 @@ describe('#513: a non-DNS exception inside the resolve block is not a DNS condit// The operator removes a.example.com; b.example.com faults after resolution.injectPostResolutionTypeError()- // No throw: b.example.com has a live policy, so the fault is served- // fail-static (T7). What this test pins is that the cleanup still ran.- await reconciler.reconcileExternalEgress(bindings(['b.example.com']), {- isCurrent: () => true,- })+ await expect(+ reconciler.reconcileExternalEgress(bindings(['b.example.com']), {+ isCurrent: () => true,+ })+ ).rejects.toThrow(TypeError)// Exactly one delete, and it is A's. `toHaveBeenCalledWith` alone is a// membership check, not an exclusivity one: it stays green when the cleanup@@ -325,24 +449,13 @@ describe('#513: a non-DNS exception inside the resolve block is not a DNS condit)})- it('T7 serves the frozen set and does NOT block the workload when a fault has a live policy', async () => {- // Review of #567, major 3. The throw reaches k8sClient's catch, which logs- // "runtime reconciliation blocked" and returns BEFORE the managed Deployment- // is created — so a fault in ONE binding kept a whole server's pod down,- // healthy siblings included. #513 never asked for that: its acceptance is "no- // permanent DNS failure line, no prune". D4 fail-static is the house rule for- // a condition we cannot resolve this pass, and the transient branch already- // obeys it. When there is a live policy to serve, serve it, report the fault,- // and let the HCC-managed Deployment start on the last known-good egress.- //- // Scoped to HCC on purpose. Every consumer of ExternalEgressReady treats a- // False as terminal whatever the reason, so a WRC recipe depending on this- // server still fails — see #577, which is a decision about what the- // condition means, not something this test can assert away.+ it('T7 retains a verified set for existing pods but blocks runtime on a controller fault', async () => {await seedAccumulatedPolicy({ lapsed: false })injectPostResolutionTypeError()- await reconciler.reconcileExternalEgress(server, { isCurrent: () => true })+ await expect(+ reconciler.reconcileExternalEgress(server, { isCurrent: () => true })+ ).rejects.toThrow(TypeError)expect(mockApi.deleteNamespacedNetworkPolicy).not.toHaveBeenCalled()expect(mockApi.replaceNamespacedNetworkPolicy).not.toHaveBeenCalled()@@ -527,7 +640,7 @@ describe('#513: a non-DNS exception inside the resolve block is not a DNS conditexpect(egress.at(-1)?.message).toContain('REVOKED')})- it('T11c still serves a clean /32 — the guard narrowed, it did not close', async () => {+ it('T11c retains a clean /32 for existing pods while blocking runtime', async () => {// Anti-over-correction. T11/T11a/T11b all assert revocation, so a guard that// simply declared everything untrustworthy would satisfy the three of them// and destroy fail-static, which is the property five review rounds converged@@ -535,14 +648,9 @@ describe('#513: a non-DNS exception inside the resolve block is not a DNS conditawait seedAccumulatedPolicy({ lapsed: false })injectPostResolutionTypeError()- // Fail-static in one assertion: a fault WITH something trustworthy to serve- // completes instead of throwing. T11/T11a/T11b are the three shapes that must- // block; this is the shape that must not, and it is also the liveness witness- // — the pass reached the fault branch and chose to serve, rather than never- // faulting at all. T8 pins the other side (nothing to serve ⇒ still throws).await expect(reconciler.reconcileExternalEgress(server, { isCurrent: () => true })- ).resolves.toBeUndefined()+ ).rejects.toThrow(TypeError)// The clean policy is kept, not revoked …expect(mockApi.deleteNamespacedNetworkPolicy).not.toHaveBeenCalled()@@ -559,6 +667,145 @@ describe('#513: a non-DNS exception inside the resolve block is not a DNS conditexpect(served).toContain('1.2.3.4')})+ it('T11c1 revokes a canonical public /32 absent from persisted DNS state', async () => {+ await seedAccumulatedPolicy({ lapsed: false })+ const policyName = await driftLivePolicy(spec => {+ spec.egress!.push({+ to: [{ ipBlock: { cidr: '8.8.8.8/32' } }],+ ports: [{ port: 443, protocol: 'TCP' }],+ })+ })+ injectPostResolutionTypeError()++ await expect(+ reconciler.reconcileExternalEgress(server, { isCurrent: () => true })+ ).rejects.toThrow(TypeError)++ expect(mockApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledWith(+ expect.objectContaining({ name: policyName })+ )+ expect(servedPayload()).not.toContain('8.8.8.8')+ })++ it('T11c2 rejects an otherwise canonical state above the configured cap', async () => {+ await seedAccumulatedPolicy({ lapsed: false })+ const items = (await mockApi.listNamespacedNetworkPolicy({ namespace: 'mcp-server' })) as {+ items: k8s.V1NetworkPolicy[]+ }+ const drifted = structuredClone(items)+ const now = Date.now()+ const ips = Array.from({ length: 129 }, (_, index) => `11.0.0.${index + 1}`)+ drifted.items[0].metadata!.annotations!['clerum.io/egress-fqdn-state'] = JSON.stringify(+ ips.map(ip => ({+ ip,+ port: 443,+ protocol: 'TCP',+ fqdn: 'api.example.com',+ expiresAt: now + 600_000,+ lastObservedAt: now,+ }))+ )+ drifted.items[0].spec!.egress = ips.map(ip => ({+ to: [{ ipBlock: { cidr: `${ip}/32` } }],+ ports: [{ port: 443, protocol: 'TCP' }],+ }))+ const policyName = drifted.items[0].metadata!.name!+ mockApi.listNamespacedNetworkPolicy.mockResolvedValue(drifted)+ injectPostResolutionTypeError()++ await expect(+ reconciler.reconcileExternalEgress(server, { isCurrent: () => true })+ ).rejects.toThrow(TypeError)+ expect(mockApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledWith(+ expect.objectContaining({ name: policyName })+ )+ })++ it('T11c3 judges a fresh GET and fences deletion to that resourceVersion', async () => {+ await seedAccumulatedPolicy({ lapsed: false })+ const listed = (await mockApi.listNamespacedNetworkPolicy({ namespace: 'mcp-server' })) as {+ items: k8s.V1NetworkPolicy[]+ }+ const fresh = structuredClone(listed.items[0])+ fresh.metadata = {+ ...fresh.metadata,+ name: fresh.metadata!.name!,+ namespace: 'mcp-server',+ uid: 'fresh-policy-uid',+ resourceVersion: '99',+ }+ fresh.spec!.egress!.push({+ to: [{ ipBlock: { cidr: '8.8.4.4/32' } }],+ ports: [{ port: 443, protocol: 'TCP' }],+ })+ mockApi.readNamespacedNetworkPolicy.mockResolvedValue(fresh)+ injectPostResolutionTypeError()++ await expect(+ reconciler.reconcileExternalEgress(server, { isCurrent: () => true })+ ).rejects.toThrow(TypeError)++ expect(mockApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledWith(+ expect.objectContaining({+ name: fresh.metadata!.name,+ body: { preconditions: { uid: 'fresh-policy-uid', resourceVersion: '99' } },+ })+ )+ })++ it('T11c3a leaves a fresh homonymous policy untouched when HCC does not own it', async () => {+ const policyName = 'ext-egress-classify-mcp-api.example.com-443'+ mockApi.readNamespacedNetworkPolicy.mockResolvedValue({+ metadata: {+ name: policyName,+ namespace: 'mcp-server',+ uid: 'foreign-policy-uid',+ resourceVersion: '77',+ labels: {+ 'clerum.io/managed-by': 'another-controller',+ 'clerum.io/policy-type': 'external-egress',+ 'clerum.io/mcpserver': server.name,+ },+ },+ spec: {+ podSelector: { matchLabels: { app: 'foreign-workload' } },+ policyTypes: ['Egress'],+ egress: [],+ },+ } satisfies k8s.V1NetworkPolicy)+ injectPostResolutionTypeError()++ await expect(+ reconciler.reconcileExternalEgress(server, { isCurrent: () => true })+ ).rejects.toThrow(/conflicting or incomplete ownership/)++ // A fresh GET is authoritative for both integrity and ownership. A policy+ // that merely occupies HCC's deterministic name is not ours to retain,+ // rewrite, or revoke—even while the controller correctly blocks runtime.+ expect(mockApi.deleteNamespacedNetworkPolicy).not.toHaveBeenCalled()+ expect(mockApi.replaceNamespacedNetworkPolicy).not.toHaveBeenCalled()+ const egress = statusConditions().filter(c => c.type === 'ExternalEgressReady')+ expect(egress.at(-1)?.reason).toBe('ExternalEgressReconcileFailed')+ expect(egress.at(-1)?.message).toContain('not HCC-owned')+ expect(egress.at(-1)?.message).toContain('left untouched')+ })++ it('T11c4 preserves the last successful resolvedAt on a controller fault', async () => {+ await seedAccumulatedPolicy({ lapsed: false })+ const listed = (await mockApi.listNamespacedNetworkPolicy({ namespace: 'mcp-server' })) as {+ items: k8s.V1NetworkPolicy[]+ }+ const oldResolvedAt = '2000-01-01T00:00:00.000Z'+ listed.items[0].metadata!.annotations!['clerum.io/egress-fqdn-resolved-at'] = oldResolvedAt+ mockApi.listNamespacedNetworkPolicy.mockResolvedValue(listed)+ injectPostResolutionTypeError()++ await expect(+ reconciler.reconcileExternalEgress(server, { isCurrent: () => true })+ ).rejects.toThrow(TypeError)+ expect(servedPayload()).toContain(oldResolvedAt)+ })+/** The live policy the fixture seeded, cloned and handed to a drift mutation. */async function driftLivePolicy(mutate: (spec: k8s.V1NetworkPolicySpec) => void): Promise<string> {const items = (await mockApi.listNamespacedNetworkPolicy({ namespace: 'mcp-server' })) as {@@ -698,7 +945,7 @@ describe('#513: a non-DNS exception inside the resolve block is not a DNS condit).toContain('REVOKED')})- it('T11g still serves a clean multi-CIDR UDP binding — the shape check reads the binding', async () => {+ it('T11g retains a clean multi-CIDR UDP binding while blocking runtime', async () => {// Anti-over-correction, the half T11c cannot cover. T11c serves ONE /32 on the// default TCP/443 binding, so a verdict that rebuilt the shape with a// hardcoded TCP, a hardcoded port, or only the first CIDR would still satisfy@@ -720,12 +967,9 @@ describe('#513: a non-DNS exception inside the resolve block is not a DNS conditawait seedAccumulatedPolicy({ lapsed: false, srv: udpServer })injectPostResolutionTypeError()- // Fail-static in one assertion: a fault WITH something trustworthy to serve- // completes instead of throwing. It is also the liveness witness — the pass- // reached the fault branch and chose to serve, rather than never faulting.await expect(reconciler.reconcileExternalEgress(udpServer, { isCurrent: () => true })- ).resolves.toBeUndefined()+ ).rejects.toThrow(TypeError)// Both accumulated addresses are still served, so a rebuild that kept only the// first CIDR fails here rather than passing quietly.@@ -737,7 +981,7 @@ describe('#513: a non-DNS exception inside the resolve block is not a DNS conditstatusConditions().filter(c => c.type === 'ExternalEgressReady').at(-1)?.message- ).toContain('left untouched')+ ).toContain('runtime remains blocked')})it('T11h revokes a live policy with no egress rules — a deny disguised as an allow', async () => {diff --git a/host-context-controller/src/networkPolicyReconciler.egressFailStatic.test.ts b/host-context-controller/src/networkPolicyReconciler.egressFailStatic.test.tsindex f28a29320..79457d9a1 100644--- a/host-context-controller/src/networkPolicyReconciler.egressFailStatic.test.ts+++ b/host-context-controller/src/networkPolicyReconciler.egressFailStatic.test.ts@@ -55,7 +55,9 @@ vi.mock('./metrics', () => ({function makeMockNetworkingApi() {return {createNamespacedNetworkPolicy: vi.fn().mockResolvedValue({}),- readNamespacedNetworkPolicy: vi.fn().mockResolvedValue({ metadata: { resourceVersion: '1' } }),+ readNamespacedNetworkPolicy: vi+ .fn()+ .mockRejectedValue(Object.assign(new Error('not found'), { code: 404 })),replaceNamespacedNetworkPolicy: vi.fn().mockResolvedValue({}),listNamespacedNetworkPolicy: vi.fn().mockResolvedValue({ items: [] }),deleteNamespacedNetworkPolicy: vi.fn().mockResolvedValue({}),@@ -88,25 +90,46 @@ function makeReconciler(currentContext: 'test',})const reconciler = new NetworkPolicyReconciler(kc, new Map())+ const normalizePolicy = (+ policy: k8s.V1NetworkPolicy,+ namespace: string,+ index = 0+ ): k8s.V1NetworkPolicy => {+ const name = policy.metadata?.name ?? `unnamed-${index}`+ return {+ ...policy,+ metadata: {+ ...policy.metadata,+ name,+ namespace: policy.metadata?.namespace ?? namespace,+ uid: policy.metadata?.uid ?? `${namespace}:${name}:uid`,+ resourceVersion: policy.metadata?.resourceVersion ?? '1',+ },+ }+ }const networkingApi = {...mockApi,listNamespacedNetworkPolicy: async (args: { namespace: string }) => {const response = await mockApi.listNamespacedNetworkPolicy(args)- const items = (response.items ?? []).map((policy: k8s.V1NetworkPolicy, index: number) => {- const name = policy.metadata?.name ?? `unnamed-${index}`- return {- ...policy,- metadata: {- ...policy.metadata,- name,- namespace: policy.metadata?.namespace ?? args.namespace,- uid: policy.metadata?.uid ?? `${args.namespace}:${name}:uid`,- resourceVersion: policy.metadata?.resourceVersion ?? '1',- },- }- })+ const items = (response.items ?? []).map((policy: k8s.V1NetworkPolicy, index: number) =>+ normalizePolicy(policy, args.namespace, index)+ )return { ...response, items }},+ readNamespacedNetworkPolicy: async (args: { name: string; namespace: string }) => {+ try {+ const direct = (await mockApi.readNamespacedNetworkPolicy(args)) as k8s.V1NetworkPolicy+ if (direct.metadata?.name || direct.spec) return normalizePolicy(direct, args.namespace)+ } catch (error: unknown) {+ if ((error as { code?: unknown })?.code !== 404) throw error+ }+ const response = await mockApi.listNamespacedNetworkPolicy({ namespace: args.namespace })+ const found = (response.items ?? []).find(+ (policy: k8s.V1NetworkPolicy) => policy.metadata?.name === args.name+ )+ if (!found) throw Object.assign(new Error('not found'), { code: 404 })+ return normalizePolicy(found, args.namespace)+ },};(reconciler as unknown as { networkingApi: unknown }).networkingApi = networkingApi;(reconciler as unknown as { customApi: unknown }).customApi = mockCustomApi@@ -191,6 +214,9 @@ describe('R1-B1 regression: DNS timeout must fail-static (freeze), not prune', (// it issues no create/replace here.expect(result.outcome).toBe('resolved')expect(mockApi.deleteNamespacedNetworkPolicy).not.toHaveBeenCalled()+ expect(JSON.stringify(mockCustomApi.patchNamespacedCustomObjectStatus.mock.calls)).toContain(+ 'FailStatic'+ )// Defensive: should a future freeze ever re-persist instead of no-op'ing, any// write it makes must still carry the accumulated /32 (never a pruned body).const writes = [diff --git a/host-context-controller/src/networkPolicyReconciler.test.ts b/host-context-controller/src/networkPolicyReconciler.test.tsindex 8682c09fe..0073d2b46 100644--- a/host-context-controller/src/networkPolicyReconciler.test.ts+++ b/host-context-controller/src/networkPolicyReconciler.test.ts@@ -19,6 +19,7 @@ import {sameContextDesiredRevision,} from './networkPolicyReconciler'import { ContextCRD, McpServerCRD } from './types'+import { getErrorCode } from './utils'function deferred<T = void>(): {promise: Promise<T>@@ -88,6 +89,7 @@ function makeMockNetworkingApi() {createNamespacedNetworkPolicy: vi.fn().mockResolvedValue({}),readNamespacedNetworkPolicy: vi.fn().mockImplementation(async ({ name }) => {if (+ name.startsWith('ext-egress-') ||name === 'allow-desktop-egress-rpc-proxy' ||name === 'allow-rpc-proxy-to-managed-mcp-servers') {@@ -155,8 +157,14 @@ function makeReconciler(return { ...response, items }},readNamespacedNetworkPolicy: async (args: { name: string; namespace: string }) => {- const response = await mockApi.readNamespacedNetworkPolicy(args)const listed = listedPolicies.get(`${args.namespace}/${args.name}`)+ let response: k8s.V1NetworkPolicy+ try {+ response = await mockApi.readNamespacedNetworkPolicy(args)+ } catch (error: unknown) {+ if (getErrorCode(error) === 404 && listed) return listed+ throw error+ }return {...listed,...response,@@ -2164,6 +2172,72 @@ describe('NetworkPolicyReconciler', () => {expect(mockApi.replaceNamespacedNetworkPolicy).not.toHaveBeenCalled()})+ it.each(['success', 'transient'] as const)(+ 'uses a fresh GET before the %s DNS no-op decision and prevents a widened live policy',+ async mode => {+ const server: McpServerCRD = {+ name: `fresh-${mode}-mcp`,+ namespace: 'mcp-server',+ spec: {+ contextRef: 'dev',+ image: 'fresh:test',+ transport: { type: 'streamableHttp', url: 'http://fresh:3000', port: 3000 },+ egressBindings: [{ dns: 'api.example.com', port: 443 }],+ },+ }+ await reconciler.reconcileExternalEgress(server, { isCurrent: () => true })+ const written = (+ mockApi.createNamespacedNetworkPolicy.mock.calls[0][0] as {+ body: k8s.V1NetworkPolicy+ }+ ).body+ const listed = structuredClone(written)+ const freshWidened = structuredClone(written)+ freshWidened.metadata = {+ ...freshWidened.metadata,+ name: written.metadata!.name!,+ namespace: 'mcp-server',+ uid: `fresh-${mode}-uid`,+ resourceVersion: '22',+ }+ freshWidened.spec!.egress!.push({+ to: [{ ipBlock: { cidr: '8.8.8.8/32' } }],+ ports: [{ port: 443, protocol: 'TCP' }],+ })+ mockApi.listNamespacedNetworkPolicy.mockResolvedValue({ items: [listed] })+ mockApi.readNamespacedNetworkPolicy.mockResolvedValue(freshWidened)+ mockApi.createNamespacedNetworkPolicy.mockReset()+ mockApi.createNamespacedNetworkPolicy.mockRejectedValue({ code: 409 })+ mockApi.replaceNamespacedNetworkPolicy.mockClear()+ mockCustomApi.patchNamespacedCustomObjectStatus.mockClear()+ if (mode === 'transient') {+ resolve4Mock.mockRejectedValueOnce(+ Object.assign(new Error('queryA ESERVFAIL api.example.com'), { code: 'ESERVFAIL' })+ )+ }++ if (mode === 'transient') {+ await expect(+ reconciler.reconcileExternalEgress(server, { isCurrent: () => true })+ ).rejects.toThrow(/Fail-static proof failed/)+ expect(mockApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledWith(+ expect.objectContaining({ name: written.metadata!.name })+ )+ expect(+ JSON.stringify(mockCustomApi.patchNamespacedCustomObjectStatus.mock.calls)+ ).toContain('ExternalEgressReconcileFailed')+ } else {+ await reconciler.reconcileExternalEgress(server, { isCurrent: () => true })+ const replacement = mockApi.replaceNamespacedNetworkPolicy.mock.calls.at(-1)?.[0]+ ?.body as k8s.V1NetworkPolicy+ expect(replacement).toBeDefined()+ }+ expect(JSON.stringify(mockApi.replaceNamespacedNetworkPolicy.mock.calls)).not.toContain(+ '8.8.8.8'+ )+ }+ )+it('WRITES on resync when the persisted window is AGING even though the IP set is unchanged (issue #299 M1 renewalDue wiring)', async () => {const server: McpServerCRD = {name: 'renew-mcp',@@ -2607,7 +2681,16 @@ describe('NetworkPolicyReconciler', () => {it('deletes only orphaned external egress policies after applying the desired set', async () => {mockApi.listNamespacedNetworkPolicy.mockResolvedValue({items: [- { metadata: { name: 'ext-egress-openai-mcp-api.openai.com-443' } },+ {+ metadata: {+ name: 'ext-egress-openai-mcp-api.openai.com-443',+ labels: {+ [MANAGED_BY_LABEL]: MANAGED_BY_VALUE,+ [POLICY_TYPE_LABEL]: 'external-egress',+ [MCPSERVER_LABEL]: 'openai-mcp',+ },+ },+ },{ metadata: { name: 'ext-egress-openai-mcp-old-example-com-443' } },],})@@ -2637,7 +2720,16 @@ describe('NetworkPolicyReconciler', () => {it('fails closed when an orphaned external egress policy cannot be deleted', async () => {mockApi.listNamespacedNetworkPolicy.mockResolvedValue({items: [- { metadata: { name: 'ext-egress-openai-mcp-api.openai.com-443' } },+ {+ metadata: {+ name: 'ext-egress-openai-mcp-api.openai.com-443',+ labels: {+ [MANAGED_BY_LABEL]: MANAGED_BY_VALUE,+ [POLICY_TYPE_LABEL]: 'external-egress',+ [MCPSERVER_LABEL]: 'openai-mcp',+ },+ },+ },{ metadata: { name: 'ext-egress-openai-mcp-old-example-com-443' } },],})@@ -2671,7 +2763,16 @@ describe('NetworkPolicyReconciler', () => {)mockApi.listNamespacedNetworkPolicy.mockResolvedValue({items: [- { metadata: { name: 'ext-egress-openai-mcp-api.openai.com-443' } },+ {+ metadata: {+ name: 'ext-egress-openai-mcp-api.openai.com-443',+ labels: {+ [MANAGED_BY_LABEL]: MANAGED_BY_VALUE,+ [POLICY_TYPE_LABEL]: 'external-egress',+ [MCPSERVER_LABEL]: 'openai-mcp',+ },+ },+ },{ metadata: { name: 'ext-egress-openai-mcp-old-example-com-443' } },],})diff --git a/host-context-controller/src/networkPolicyReconciler.ts b/host-context-controller/src/networkPolicyReconciler.tsindex c5ddb82b6..9ffb525bf 100644--- a/host-context-controller/src/networkPolicyReconciler.ts+++ b/host-context-controller/src/networkPolicyReconciler.ts@@ -17,10 +17,12 @@ import * as k8s from '@kubernetes/client-node'import * as dns from 'node:dns/promises'import { isIP } from 'node:net'import {+ RESOLVED_AT_ANNOTATION,STATE_ANNOTATION,- classifyDnsError,- dnsCodeOf,- isPermanentDnsCode,+ TARGETS_ANNOTATION,+ classifyDnsRejection,+ parseStateStrict,+ serializeState,} from '@clerum/network-policy-core'import { config } from './config'import {@@ -32,6 +34,7 @@ import {POLICY_TYPE_LABEL,} from './constants'import { accumulateHostExactHostEgress } from './externalEgressAccumulator'+import { hccLogger } from './logger'import {confirmAuthoritativeMcpServerAbsence,sameMcpServerDesiredRevision,@@ -148,9 +151,21 @@ function sameNetworkPolicySpec(*/type LiveEgressVerdict =| { state: 'none' }- | { state: 'intact'; cidrs: string[] }- | { state: 'drifted-shape' }- | { state: 'drifted-cidr' }+ | { state: 'foreign' }+ | { state: 'intact'; cidrs: string[]; resolvedAt: string; policy: k8s.V1NetworkPolicy }+ | { state: 'drifted-shape'; policy: k8s.V1NetworkPolicy }+ | { state: 'drifted-cidr'; policy: k8s.V1NetworkPolicy }+ | { state: 'untrusted-state'; policy: k8s.V1NetworkPolicy }++type FreshExternalEgressPolicy = {+ policy: k8s.V1NetworkPolicy | null+ repairRequired: boolean+}++type ProvenDnsAnnotations = {+ annotations: Record<string, string> | undefined+ contracted: boolean+}/*** What the ExternalEgressReady message says about the live policy on a non-DNS@@ -159,10 +174,12 @@ type LiveEgressVerdict =*/const LIVE_EGRESS_FAULT_TEXT: Record<LiveEgressVerdict['state'], string> = {none: 'no live policy to serve',- intact: 'accumulated egress left untouched',+ foreign: 'homonymous live policy is not HCC-owned and was left untouched',+ intact: 'verified policy retained for existing pods; runtime remains blocked','drifted-cidr':'live policy REVOKED: it enforced a CIDR outside the /32 allowlist this controller writes','drifted-shape': 'live policy REVOKED: its egress shape is not one this controller writes',+ 'untrusted-state': 'live policy REVOKED: its DNS provenance was absent or invalid',}function hasExpectedPolicyOwnership(policy: k8s.V1NetworkPolicy, policyType: string): boolean {@@ -1869,19 +1886,64 @@ export class NetworkPolicyReconciler {* A rebuild-and-compare has no dimension to miss: anything the controller does* not write differs from the rebuild, whatever it happens to allow.*/- private judgeLiveExactHostEgress(+ private async judgeLiveExactHostEgress(+ server: McpServerCRD,+ name: string,+ binding: EgressBinding+ ): Promise<LiveEgressVerdict> {+ let existingPolicy: k8s.V1NetworkPolicy+ try {+ existingPolicy = await this.networkingApi.readNamespacedNetworkPolicy({+ name,+ namespace: server.namespace,+ })+ } catch (error: unknown) {+ if (getErrorCode(error) === 404) return { state: 'none' }+ throw error+ }+ return this.judgeExactHostEgressPolicy(server, name, binding, existingPolicy)+ }++ private judgeExactHostEgressPolicy(server: McpServerCRD,name: string,binding: EgressBinding,- existingPolicy: k8s.V1NetworkPolicy | null+ existingPolicy: k8s.V1NetworkPolicy): LiveEgressVerdict {- // Bootstrap: nothing live to judge, and nothing to serve.- if (existingPolicy === null) return { state: 'none' }+ const metadata = existingPolicy.metadata+ const labels = metadata?.labels ?? {}+ if (+ metadata?.name !== name ||+ metadata.namespace !== server.namespace ||+ !metadata.uid ||+ !metadata.resourceVersion+ ) {+ return { state: 'untrusted-state', policy: existingPolicy }+ }+ const ownsExpectedLane =+ hasExpectedPolicyOwnership(existingPolicy, EXTERNAL_EGRESS_POLICY_TYPE) &&+ labels[MCPSERVER_LABEL] === server.name+ if (!ownsExpectedLane) return { state: 'foreign' }- const cidrs = (existingPolicy.spec?.egress ?? [])- .flatMap(rule => rule.to ?? [])- .map(peer => peer.ipBlock?.cidr)- .filter((cidr): cidr is string => typeof cidr === 'string')+ const stateRead = parseStateStrict(+ existingPolicy.metadata?.annotations,+ {+ overlapMs: config.externalEgressOverlapSec * 1000,+ maxEntries: config.externalEgressMaxEntries,+ },+ [+ {+ fqdn: binding.dns!,+ port: binding.port!,+ protocol: binding.protocol ?? 'TCP',+ },+ ]+ )+ if (stateRead.kind !== 'valid' || stateRead.state.entries.length === 0) {+ return { state: 'untrusted-state', policy: existingPolicy }+ }++ const cidrs = stateRead.state.entries.map(entry => `${entry.ip}/32`)// A policy with no ipBlock peers is a deny disguised as an allow, and the// rebuild from an empty CIDR list would compare equal to it. Gate first, the@@ -1898,10 +1960,130 @@ export class NetworkPolicyReconciler {cidr => cidr.endsWith('/32') && isAllowedExternalEgressCidr(cidr))- if (shapeIntact && cidrsIntact) return { state: 'intact', cidrs }+ if (shapeIntact && cidrsIntact) {+ const annotatedResolvedAt = existingPolicy.metadata?.annotations?.[RESOLVED_AT_ANNOTATION]+ const latestObservedAt = Math.max(+ ...stateRead.state.entries.map(entry => entry.lastObservedAt)+ )+ const resolvedAt =+ annotatedResolvedAt && Number.isFinite(Date.parse(annotatedResolvedAt))+ ? annotatedResolvedAt+ : new Date(latestObservedAt).toISOString()+ return { state: 'intact', cidrs, resolvedAt, policy: existingPolicy }+ }// A bad CIDR is the more specific diagnosis and the one an operator acts on// first, so it wins when a policy drifted on both axes.- return { state: cidrsIntact ? 'drifted-shape' : 'drifted-cidr' }+ return {+ state: cidrsIntact ? 'drifted-shape' : 'drifted-cidr',+ policy: existingPolicy,+ }+ }++ /**+ * Rehydrate a successful DNS observation only from structured entries that+ * were already enforced by the fresh live spec. STATE is provenance, not an+ * alternate allowlist: annotation-only entries are discarded before the+ * accumulator can union them into the next policy. A fresh DNS answer may+ * then add only the addresses it actually returned.+ */+ private provenPreviousDnsAnnotations(+ server: McpServerCRD,+ name: string,+ binding: EgressBinding,+ existingPolicy: k8s.V1NetworkPolicy | null+ ): ProvenDnsAnnotations {+ if (!existingPolicy) return { annotations: undefined, contracted: false }+ const empty = serializeState({ entries: [] })+ const finish = (annotations: Record<string, string>): ProvenDnsAnnotations => {+ const current = existingPolicy.metadata?.annotations ?? {}+ return {+ annotations,+ contracted: [STATE_ANNOTATION, TARGETS_ANNOTATION, RESOLVED_AT_ANNOTATION].some(+ key => current[key] !== annotations[key]+ ),+ }+ }+ const stateRead = parseStateStrict(+ existingPolicy.metadata?.annotations,+ {+ overlapMs: config.externalEgressOverlapSec * 1000,+ maxEntries: config.externalEgressMaxEntries,+ },+ [+ {+ fqdn: binding.dns!,+ port: binding.port!,+ protocol: binding.protocol ?? 'TCP',+ },+ ]+ )+ if (stateRead.kind !== 'valid') return finish(empty)++ const expectedShape = this.buildExactHostEgressPolicy(server, name, binding, [])+ const liveShape: k8s.V1NetworkPolicy = {+ ...existingPolicy,+ spec: { ...existingPolicy.spec!, egress: [] },+ }+ if (!sameNetworkPolicySpec(liveShape, expectedShape)) return finish(empty)++ const liveRules = new Set(+ (existingPolicy.spec?.egress ?? []).map(rule =>+ JSON.stringify(canonicalizeNetworkPolicyValue(rule))+ )+ )+ const entries = stateRead.state.entries.filter(entry => {+ const expectedRule = this.buildExactHostEgressPolicy(server, name, binding, [+ `${entry.ip}/32`,+ ]).spec?.egress?.[0]+ return (+ expectedRule !== undefined &&+ liveRules.has(JSON.stringify(canonicalizeNetworkPolicyValue(expectedRule)))+ )+ })+ const annotations = serializeState({ entries })+ const priorResolvedAt = existingPolicy.metadata?.annotations?.[RESOLVED_AT_ANNOTATION]+ if (entries.length > 0 && priorResolvedAt && Number.isFinite(Date.parse(priorResolvedAt))) {+ annotations[RESOLVED_AT_ANNOTATION] = priorResolvedAt+ }+ return finish(annotations)+ }++ /**+ * Re-read the exact external-egress object after DNS latency. LIST is only a+ * discovery/cleanup snapshot; it cannot authorize a no-op or status claim.+ * A single missing HCC marker remains repairable under the existing inventory+ * contract, but foreign/ambiguous ownership and incomplete identity fail loud.+ */+ private async readFreshExternalEgressPolicy(+ server: McpServerCRD,+ name: string+ ): Promise<FreshExternalEgressPolicy> {+ let policy: k8s.V1NetworkPolicy+ try {+ policy = await this.networkingApi.readNamespacedNetworkPolicy({+ name,+ namespace: server.namespace,+ })+ } catch (error: unknown) {+ if (getErrorCode(error) === 404) return { policy: null, repairRequired: false }+ throw error+ }++ const classification = classifySafetyInventoryPolicy(policy, 'external-egress')+ const metadata = policy.metadata+ if (+ metadata?.name !== name ||+ metadata.namespace !== server.namespace ||+ !metadata.uid ||+ !metadata.resourceVersion ||+ metadata.labels?.[MCPSERVER_LABEL] !== server.name ||+ (classification !== 'owned' && classification !== 'repairable')+ ) {+ throw new Error(+ `NetworkPolicy "${name}" has conflicting or incomplete ownership for the external-egress lane`+ )+ }+ return { policy, repairRequired: classification === 'repairable' }}private async resolveExternalEgressDns(@@ -1918,10 +2100,9 @@ export class NetworkPolicyReconciler {timeout = setTimeout(() =>// A bounded-timeout DNS failure is a TRANSIENT resolver error, not a- // permanent one. classifyDnsError keys on `.code`, and a codeless Error- // falls into the 'permanent' (NXDOMAIN/expire) branch, which prunes the- // accumulated egress instead of freezing it — breaking D4 fail-static.- // Tag it ETIMEDOUT (already in TRANSIENT_DNS_CODES) so it freezes.+ // permanent one. The resolver-boundary verdict keys on `.code`;+ // tag the bounded timeout ETIMEDOUT so it is positively transient+ // and freezes the accumulated set.reject(Object.assign(new Error(@@ -2164,11 +2345,56 @@ export class NetworkPolicyReconciler {const reconcileFaults: Array<{dns: stringerr: unknown- blocking: booleanlive: LiveEgressVerdict['state']+ verificationError?: unknown}> = []+ const freshPolicySnapshots = new Map<string, k8s.V1NetworkPolicy>()const resolvedAtMs = Date.now()const resolvedAt = new Date(resolvedAtMs).toISOString()+ let dnsFailStatic = false++ const recordReconcileFault = async (+ binding: EgressBinding,+ name: string,+ err: unknown+ ): Promise<void> => {+ if (!binding.dns) throw new Error('DNS reconcile fault requires a DNS binding')+ hccLogger.error('external egress reconcile fault is not a DNS condition', {+ serverName: server.name,+ dns: binding.dns,+ err,+ })++ let verdict: LiveEgressVerdict+ try {+ verdict = await this.judgeLiveExactHostEgress(server, name, binding)+ } catch (verificationError: unknown) {+ hccLogger.error('external egress live policy verification failed', {+ serverName: server.name,+ dns: binding.dns,+ err: verificationError,+ })+ reconcileFaults.push({+ dns: binding.dns,+ err,+ live: 'untrusted-state',+ verificationError,+ })+ return+ }++ if (verdict.state === 'intact') {+ desiredPolicyNames.add(name)+ resolvedEgressIPs.push({+ dns: binding.dns,+ ips: verdict.cidrs.map(cidr => cidr.replace(/\/32$/, '')),+ resolvedAt: verdict.resolvedAt,+ })+ } else if ('policy' in verdict) {+ freshPolicySnapshots.set(name, verdict.policy)+ }+ reconcileFaults.push({ dns: binding.dns, err, live: verdict.state })+ }for (const binding of bindings) {if (!isCurrent()) return@@ -2178,7 +2404,7 @@ export class NetworkPolicyReconciler {continue}- let cidrs: string[]+ let cidrs: string[] = []// Sliding-window state annotations to persist on the policy (issue #299).// Only the DNS branch sets this; CIDR/public-web bindings leave it undefined.let stateAnnotations: Record<string, string> | undefined@@ -2191,6 +2417,9 @@ export class NetworkPolicyReconciler {// accumulated.changed is over (fqdn,ip,port,protocol), so it forces the// write that re-persists the identity and preserves the overlap grace.let egressStateChanged = false+ let resolvedStatus: McpServerResolvedEgressIP | undefined+ let bindingDnsFailStatic = false+ let ownershipRepairRequired = falseconst name = this.externalEgressPolicyName(server.name, binding)if (!name) {failures.push('exact-host external egress bindings must declare dns or cidr')@@ -2200,7 +2429,7 @@ export class NetworkPolicyReconciler {// AND to decide whether a write is actually needed (audit F2/L1) — the// write is gated on the ENFORCED spec.egress, so identical static policies// never churn and out-of-band drift self-heals, for every egress class.- const existingPolicy = existingPolicies.find(p => p.metadata?.name === name) ?? null+ let existingPolicy = existingPolicies.find(p => p.metadata?.name === name) ?? nullif (egressClass === 'public-web') {if (@@ -2215,16 +2444,18 @@ export class NetworkPolicyReconciler {continue}- desiredPolicyNames.add(name)-// public-web is a static policy. Preserve #205's ownership-fenced write// (applyOwnedPolicy asserts ownership + honors the isCurrent generation// fence) while keeping dev's F2 no-churn gate so the TTL-accelerated// resync does not rewrite it every tick.const policy = this.buildPublicWebEgressPolicy(server, name)- if (this.externalEgressWriteNeeded(existingPolicy, policy)) {+ const fresh = await this.readFreshExternalEgressPolicy(server, name)+ existingPolicy = fresh.policy+ if (fresh.repairRequired || this.externalEgressWriteNeeded(existingPolicy, policy)) {await this.applyOwnedPolicy(name, server.namespace, policy, 'external-egress', isCurrent)}+ if (!isCurrent()) return+ desiredPolicyNames.add(name)continue}@@ -2264,217 +2495,199 @@ export class NetworkPolicyReconciler {// resolver failure we fail-static: keep serving the accumulated set// rather than deleting the policy (the pre-#299 catch dropped egress on// any DNS hiccup).- const previousAnnotations = existingPolicy?.metadata?.annotationsconst now = Date.parse(resolvedAt)const coreConfig = {overlapMs: config.externalEgressOverlapSec * 1000,maxEntries: config.externalEgressMaxEntries,}+ let records: Array<{ address: string; ttl: number }> | undefined+ let resolverFailure: { error: unknown; kind: 'transient' | 'permanent' } | undefinedtry {- const records = await this.resolveExternalEgressDns(binding.dns)- if (!isCurrent()) return- const uniqueIps = [...new Set(records.map(r => r.address))].sort()- if (uniqueIps.length === 0) {- failures.push(`hostname "${binding.dns}" resolved to no IPv4 addresses`)+ records = await this.resolveExternalEgressDns(binding.dns)+ } catch (resolverError: unknown) {+ const verdict = classifyDnsRejection(resolverError)+ if (verdict.kind === 'fault') {+ await recordReconcileFault(binding, name, resolverError)continue}- const invalidIps = uniqueIps.filter(ip => isIP(ip) !== 4)- if (invalidIps.length > 0) {- failures.push(- `hostname "${binding.dns}" resolved invalid IPv4 answer(s): ${invalidIps.join(', ')}`- )- continue+ resolverFailure = {+ error: resolverError,+ kind: verdict.kind === 'transient' ? 'transient' : 'permanent',}- const freshCidrs = uniqueIps.map(ip => `${ip}/32`)- const disallowedCidrs = freshCidrs.filter(cidr => !isAllowedExternalEgressCidr(cidr))- if (disallowedCidrs.length > 0) {+ }++ let fresh: FreshExternalEgressPolicy+ try {+ fresh = await this.readFreshExternalEgressPolicy(server, name)+ } catch (policyReadError: unknown) {+ await recordReconcileFault(binding, name, policyReadError)+ continue+ }+ existingPolicy = fresh.policy+ ownershipRepairRequired = fresh.repairRequired+ const liveAnnotations = existingPolicy?.metadata?.annotations++ if (resolverFailure) {+ if (fresh.policy) freshPolicySnapshots.set(name, fresh.policy)+ if (resolverFailure.kind === 'permanent') {failures.push(- `hostname "${binding.dns}" resolved disallowed address(es): ${disallowedCidrs.join(', ')}`+ `failed to resolve hostname "${binding.dns}": ${this.errorMessage(resolverFailure.error)}`)continue}- // Security gates above ran on the FRESH snapshot; only validated IPs- // enter the accumulator, so the union can never contain a blocked IP.- const ttlSeconds = records.length > 0 ? Math.min(...records.map(r => r.ttl)) : 0- if (ttlSeconds > 0) {- this.externalEgressMinObservedTtlMs = Math.min(- this.externalEgressMinObservedTtlMs,- ttlSeconds * 1000- )+ if (existingPolicy) {+ const proof = this.judgeExactHostEgressPolicy(server, name, binding, existingPolicy)+ if (proof.state !== 'intact') {+ reconcileFaults.push({+ dns: binding.dns,+ err: new Error(+ `Fail-static proof failed for "${binding.dns}": ${LIVE_EGRESS_FAULT_TEXT[proof.state]}`+ ),+ live: proof.state,+ })+ continue+ }}const accumulated = accumulateHostExactHostEgress({fqdn: binding.dns,port: binding.port,protocol,- resolution: { kind: 'ok', ips: uniqueIps, ttlSeconds },- previousAnnotations,+ resolution: { kind: 'transient' },+ previousAnnotations: liveAnnotations,now,config: coreConfig,})- // Defense-in-depth (audit M3): rehydrated IPs (from the policy's own- // annotations) bypass the fresh-snapshot CIDR gate, so a tampered/corrupt- // state annotation could union a blocked/private IP. Re-validate the- // effective set against the blocked ranges before rendering.+ if (accumulated.cidrs.length === 0) {+ failures.push(+ `failed to resolve hostname "${binding.dns}": ${this.errorMessage(resolverFailure.error)}`+ )+ continue+ }cidrs = accumulated.cidrs.filter(c => isAllowedExternalEgressCidr(c))+ if (cidrs.length !== accumulated.cidrs.length) {+ failures.push(+ `persisted DNS state for hostname "${binding.dns}" contains a disallowed address`+ )+ continue+ }stateAnnotations = accumulated.annotationsegressRenewalDue = accumulated.renewalDueegressStateChanged = accumulated.changed- resolvedEgressIPs.push({+ bindingDnsFailStatic = true+ if (accumulated.resolvedAt) {+ resolvedStatus = {+ dns: binding.dns,+ ips: cidrs.map(cidr => cidr.replace(/\/32$/, '')),+ resolvedAt: accumulated.resolvedAt,+ }+ }+ hccLogger.warn('external egress DNS failure', {+ serverName: server.name,dns: binding.dns,- ips: cidrs.map(cidr => cidr.replace(/\/32$/, '')),- resolvedAt,+ kind: resolverFailure.kind,+ accumulatedIps: cidrs.length,+ err: resolverFailure.error,})- if (accumulated.overCap) {- console.warn(- `[NetPol] "${binding.dns}" egress set hit the cap (${config.externalEgressMaxEntries}); evicted ${accumulated.evicted} least-recently-observed IP(s)`+ }++ if (records) {+ try {+ if (!isCurrent()) return+ const uniqueIps = [...new Set(records.map(record => record.address))].sort()+ if (uniqueIps.length === 0) {+ if (existingPolicy) freshPolicySnapshots.set(name, existingPolicy)+ failures.push(`hostname "${binding.dns}" resolved to no IPv4 addresses`)+ continue+ }+ const invalidIps = uniqueIps.filter(ip => isIP(ip) !== 4)+ if (invalidIps.length > 0) {+ if (existingPolicy) freshPolicySnapshots.set(name, existingPolicy)+ failures.push(+ `hostname "${binding.dns}" resolved invalid IPv4 answer(s): ${invalidIps.join(', ')}`+ )+ continue+ }+ const freshCidrs = uniqueIps.map(ip => `${ip}/32`)+ const disallowedCidrs = freshCidrs.filter(cidr => !isAllowedExternalEgressCidr(cidr))+ if (disallowedCidrs.length > 0) {+ if (existingPolicy) freshPolicySnapshots.set(name, existingPolicy)+ failures.push(+ `hostname "${binding.dns}" resolved disallowed address(es): ${disallowedCidrs.join(', ')}`+ )+ continue+ }+ const ttlSeconds = Math.min(...records.map(record => record.ttl))+ if (ttlSeconds > 0) {+ this.externalEgressMinObservedTtlMs = Math.min(+ this.externalEgressMinObservedTtlMs,+ ttlSeconds * 1000+ )+ }+ const provenPrevious = this.provenPreviousDnsAnnotations(+ server,+ name,+ binding,+ existingPolicy)- }- console.log(`[NetPol] Resolved ${binding.dns} → [${cidrs.join(', ')}]`)- } catch (err) {- // Fail-static (H1): freeze the accumulated set on a transient resolver- // failure; fail loud when there is nothing to serve (bootstrap, or a- // permanent no-records answer).- //- // Issue #513: only a POSITIVELY identified DNS condition may reach the- // sliding-window classifier. classifyDnsError defaults every unknown or- // missing code to 'permanent' (network-policy-core index.cjs), and- // 'permanent' PRUNES the accumulated set rather than freezing it. This- // try covers the whole resolve-and-render block, so without this gate a- // programming fault raised after resolution — a TypeError from a- // contract violation, an ERR_* argument error — is laundered into the- // prune branch and reported to the operator as a DNS failure, pointing- // at the resolver while the fault is here. Transient codes are positive- // by construction (TRANSIENT_DNS_CODES); 'permanent' is positive only for- // PERMANENT_DNS_CODES — the two no-records answers plus EBADNAME, a name- // c-ares refuses to query. EBADNAME cannot arrive from this controller's- // own CRD field (isPublicDnsHostname rejects `..` and 64-octet labels- // before we resolve), but the set is shared with WRC, whose ui lane- // validates the fqdn with the CRD pattern alone; there it IS the- // operator's misconfiguration, and it must keep pruning and reporting- // ExternalEgressRejected rather than blame the controller.- //- // `dnsCodeOf` reads the code off any object, so a rejection that is not- // an Error instance is still classified by its code rather than by how- // it was thrown. It is deliberately not full parity with- // classifyDnsError, which also accepts a bare code string: a rejection- // whose value IS the string 'ENOTFOUND' classifies permanent there and- // yields no code here, so it lands in the fault branch. `node:dns`- // rejects with Error objects, so that shape does not occur in- // production. Shared with WRC rather than unwrapped by hand in each- // process — two hand-written extractions of the same field is the drift- // surface network-policy-core exists to remove (#567 review, R1-M2).- const kind = classifyDnsError(err)- const code = dnsCodeOf(err)- if (kind !== 'transient' && !isPermanentDnsCode(code)) {- // Log before anything that can throw. writeExternalEgressStatus fails- // on a stale uid/generation or a rejected patch, and the isCurrent- // fence below returns without a word — either would swallow the very- // fault this branch exists to surface.- console.error(- `[NetPol] External egress reconcile fault on binding "${binding.dns}" for "${server.name}" (not a DNS condition):`,- err+ const accumulated = accumulateHostExactHostEgress({+ fqdn: binding.dns,+ port: binding.port,+ protocol,+ resolution: { kind: 'ok', ips: uniqueIps, ttlSeconds },+ previousAnnotations: provenPrevious.annotations,+ now,+ config: coreConfig,+ })+ const allowedEntries = accumulated.entries.filter(entry =>+ isAllowedExternalEgressCidr(`${entry.ip}/32`))- // Recorded, not thrown: the throw is deferred to after the loop so- // cleanupExternalEgress still runs. It is the only place that deletes- // policies for bindings the operator removed from the spec, and- // desiredPolicyNames is only complete once the loop ends. Throwing- // here would leave a de-authorized destination allowed for as long as- // the fault persists — fail-loud paid for with fail-open.- //- // D4 fail-static, applied to a fault exactly as the transient branch- // applies it to a resolver outage: when there is a live policy, its- // rules are what is enforced right now, so report them and let the- // HCC-managed Deployment start on the last known-good egress. Read- // from the live object, not recomputed — recomputing is what faulted.- // Only HCC: consumers of ExternalEgressReady treat any False as- // terminal, so a dependent WRC recipe still fails (#577).- const verdict = this.judgeLiveExactHostEgress(server, name, binding, existingPolicy)-- // Membership in `desiredPolicyNames` is the difference between "left- // untouched" and "revoked": cleanup deletes every external-egress- // policy NOT in this set, and it runs BEFORE the deferred throw, so a- // revocation actually happens rather than being skipped by the- // fail-loud exit. Revoking is fail-CLOSED — these are pure allow- // policies under the namespace L0 deny-all, so removing one closes- // that destination and cannot open one.- //- // Nothing here can rewrite a drifted policy instead: rebuilding means- // recomputing the CIDR set, and recomputing is what faulted.- //- // `none` is bootstrap, and there the add is dataplane-NEUTRAL: the- // policy came from `existingPolicies.find(...)`, cleanup filters that- // same list by name, and a name that was not found has no live object- // to delete. It stays explicit because the status has to tell "nothing- // was live" apart from "what was live got revoked" — an earlier- // revision of this branch asserted the opposite in a comment.- const revoked = verdict.state === 'drifted-shape' || verdict.state === 'drifted-cidr'- if (!revoked) desiredPolicyNames.add(name)-- if (verdict.state === 'intact') {- // Without this the binding vanishes from status.resolvedEgressIPs —- // the audit record an operator reads to learn what is allowed —- // while its /32s stay enforced. The status message would claim- // "accumulated egress left untouched" and the record would disagree.- resolvedEgressIPs.push({+ cidrs = allowedEntries.map(entry => `${entry.ip}/32`)+ stateAnnotations =+ allowedEntries.length === accumulated.entries.length+ ? accumulated.annotations+ : {+ ...serializeState({ entries: allowedEntries }),+ ...(accumulated.resolvedAt+ ? { [RESOLVED_AT_ANNOTATION]: accumulated.resolvedAt }+ : {}),+ }+ egressRenewalDue = accumulated.renewalDue+ egressStateChanged =+ accumulated.changed ||+ provenPrevious.contracted ||+ allowedEntries.length !== accumulated.entries.length+ resolvedStatus = {+ dns: binding.dns,+ ips: cidrs.map(cidr => cidr.replace(/\/32$/, '')),+ resolvedAt,+ }+ if (accumulated.overCap) {+ hccLogger.warn('external egress set hit the cap', {+ serverName: server.name,dns: binding.dns,- ips: verdict.cidrs.map(cidr => cidr.replace(/\/32$/, '')),- resolvedAt,+ maxEntries: config.externalEgressMaxEntries,+ evicted: accumulated.evicted,})}- reconcileFaults.push({- // Inside `else if (binding.dns)`, so it is always a string here;- // a `??` fallback would be an unreachable branch pretending to be a guard.+ hccLogger.info('external egress DNS resolved', {+ serverName: server.name,dns: binding.dns,- err,- // Blocking unless there is a trustworthy live set to serve. A fault- // on a binding that has never been written has no frozen set behind- // it, so completing the pass would publish a server as reconciled- // while one of its declared destinations was never enforced at all.- blocking: verdict.state !== 'intact',- // The operator message reads off this. A status that describes the- // opposite of what the pass did is the same defect class this branch- // exists to close.- live: verdict.state,+ cidrs,})+ } catch (processingError: unknown) {+ await recordReconcileFault(binding, name, processingError)continue}- const accumulated = accumulateHostExactHostEgress({- fqdn: binding.dns,- port: binding.port,- protocol,- resolution: { kind },- previousAnnotations,- now,- config: coreConfig,- })- if (accumulated.cidrs.length === 0) {- failures.push(`failed to resolve hostname "${binding.dns}": ${this.errorMessage(err)}`)- continue- }- // Defense-in-depth (audit M3): rehydrated IPs (from the policy's own- // annotations) bypass the fresh-snapshot CIDR gate, so a tampered/corrupt- // state annotation could union a blocked/private IP. Re-validate the- // effective set against the blocked ranges before rendering.- cidrs = accumulated.cidrs.filter(c => isAllowedExternalEgressCidr(c))- stateAnnotations = accumulated.annotations- // A freeze keeps the SAME set as the live policy → changed=false → no- // write (no churn while DNS is failing); a genuine change still writes.- egressRenewalDue = accumulated.renewalDue- egressStateChanged = accumulated.changed- resolvedEgressIPs.push({- dns: binding.dns,- ips: cidrs.map(cidr => cidr.replace(/\/32$/, '')),- resolvedAt,- })- console.warn(- `[NetPol] ${kind} DNS failure for "${binding.dns}"; serving ${cidrs.length} accumulated IP(s) (fail-static, issue #299): ${this.errorMessage(err)}`- )}} else {continue}- desiredPolicyNames.add(name)+ if (!binding.dns) {+ const fresh = await this.readFreshExternalEgressPolicy(server, name)+ existingPolicy = fresh.policy+ ownershipRepairRequired = fresh.repairRequired+ }// Build the exact-host policy from #205's helper, then fold in the #299// sliding-window state annotations so the next reconcile rehydrates the@@ -2483,6 +2696,7 @@ export class NetworkPolicyReconciler {if (stateAnnotations) {policy.metadata = { ...(policy.metadata ?? {}), annotations: stateAnnotations }}+ if (existingPolicy) freshPolicySnapshots.set(name, existingPolicy)// Write only when the enforced rules changed / the policy is new / drifted// (audit F2 no-churn + L1 self-heal), OR when the DNS window is aging and@@ -2490,6 +2704,7 @@ export class NetworkPolicyReconciler {// Route the write through applyOwnedPolicy so #205's ownership assertion// and isCurrent generation fence still guard the mutation.if (+ ownershipRepairRequired ||this.externalEgressWriteNeeded(existingPolicy, policy) ||egressRenewalDue ||egressStateChanged@@ -2498,16 +2713,32 @@ export class NetworkPolicyReconciler {} else {console.log(`[NetPol] External egress "${name}" unchanged — no-op (issue #299 no-churn)`)}+ if (!isCurrent()) return+ desiredPolicyNames.add(name)+ if (resolvedStatus) resolvedEgressIPs.push(resolvedStatus)+ if (bindingDnsFailStatic) dnsFailStatic = true+ }++ const cleanupCandidates = existingPolicies+ .filter(policy => {+ const name = policy.metadata?.name+ return Boolean(name) && !desiredPolicyNames.has(name!)+ })+ .map(policy => freshPolicySnapshots.get(policy.metadata?.name ?? '') ?? policy)+ for (const [name, snapshot] of freshPolicySnapshots) {+ if (+ !desiredPolicyNames.has(name) &&+ !cleanupCandidates.some(policy => policy.metadata?.name === name)+ ) {+ cleanupCandidates.push(snapshot)+ }}try {await this.cleanupExternalEgress(server.name,server.namespace,- existingPolicies.filter(policy => {- const name = policy.metadata?.name- return Boolean(name) && !desiredPolicyNames.has(name!)- }),+ cleanupCandidates,async () => isCurrent(),options.onRevoked)@@ -2531,12 +2762,7 @@ export class NetworkPolicyReconciler {// Reported only after cleanupExternalEgress above, so a binding removed from// the spec is still de-authorized on a pass that ends in a fault.if (reconcileFaults.length > 0) {- // The blocking fault is the one that propagates, so it is the one the- // status must name: reporting reconcileFaults[0] while throwing a different- // error hands the operator a condition and a log line about two different- // bindings. With no blocking fault, the first is the whole story.- const blocking = reconcileFaults.find(fault => fault.blocking)- const first = blocking ?? reconcileFaults[0]+ const first = reconcileFaults[0]const parts = [`External egress reconciliation failed on binding "${first.dns}" with a non-DNS error ` +`(not a resolver condition, ${LIVE_EGRESS_FAULT_TEXT[first.live]}): ${this.errorMessage(@@ -2546,6 +2772,11 @@ export class NetworkPolicyReconciler {if (reconcileFaults.length > 1) {parts.push(`and ${reconcileFaults.length - 1} more binding(s) faulted`)}+ if (first.verificationError) {+ parts.push(+ `live policy verification also failed: ${this.errorMessage(first.verificationError)}`+ )+ }// `failures` has exactly one sink — the ExternalEgressRejected message// below — which this branch used to make unreachable. A pass carrying both// then lost the operator's own config error completely, while cleanup had@@ -2560,20 +2791,10 @@ export class NetworkPolicyReconciler {parts.join('. '),isCurrent)- // Fail-static, not fail-stop. The throw reaches k8sClient's catch, which- // returns before the managed Deployment is created — so throwing for a- // fault that HAS a frozen set to serve takes a whole server's pod down over- // one binding, healthy siblings included, on a retry ladder that cannot fix- // a deterministic fault. Throw only when nothing could be served.- if (blocking) throw blocking.err- // Rejected input stays blocking as it always was, now that it has been said- // out loud in the message above.- if (failures.length > 0) {- throw new Error(- `External egress reconciliation failed for "${server.name}": ${failures.join('; ')}`- )- }- return+ // The documented pre-start contract is fail-closed for controller faults:+ // an exactly-proven policy may remain for already-running pods, but False+ // never admits a new/update runtime and never completes the retry.+ throw first.err}if (failures.length > 0) {@@ -2593,10 +2814,12 @@ export class NetworkPolicyReconciler {server,this.sortResolvedEgressIPs(resolvedEgressIPs),'True',- 'Reconciled',- bindings.some(binding => binding.egressClass === 'public-web')- ? 'External egress policies reconciled; public-web allows public TCP 80/443 with private and special ranges excluded'- : 'External egress policies reconciled',+ dnsFailStatic ? 'FailStatic' : 'Reconciled',+ dnsFailStatic+ ? 'External egress retained from the last accepted DNS state while the resolver is temporarily unavailable'+ : bindings.some(binding => binding.egressClass === 'public-web')+ ? 'External egress policies reconciled; public-web allows public TCP 80/443 with private and special ranges excluded'+ : 'External egress policies reconciled',isCurrent)}diff --git a/packages/network-policy-core/index.cjs b/packages/network-policy-core/index.cjsindex 27286dcba..4d2ed89bc 100644--- a/packages/network-policy-core/index.cjs+++ b/packages/network-policy-core/index.cjs@@ -134,6 +134,31 @@ function dnsCodeOf(err) {return typeof err.code === 'string' ? err.code : undefined}+/**+ * Classify one value rejected by the DNS promise at the resolver boundary.+ * Unlike classifyDnsError, this is a total trust-boundary verdict: an unknown,+ * missing, or non-object rejection is a controller fault, never an implicit+ * negative DNS answer. The caller must invoke this only in the catch that wraps+ * the DNS operation itself; a code cannot prove where an exception originated.+ */+function classifyDnsRejection(value) {+ const code = dnsCodeOf(value)+ if (typeof code === 'string' && TRANSIENT_DNS_CODES.has(code)) {+ return { kind: 'transient', code }+ }+ if (typeof code === 'string' && NO_RECORDS_DNS_CODES.includes(code)) {+ return { kind: 'negative', reason: 'no-records', code }+ }+ if (code === 'EBADNAME') {+ return { kind: 'negative', reason: 'invalid-name', code }+ }+ return {+ kind: 'fault',+ ...(typeof code === 'string' ? { code } : {}),+ cause: value,+ }+}+/** True for the two genuine no-records answers, false for every other code. */function isNoRecordsDnsCode(code) {return typeof code === 'string' && NO_RECORDS_DNS_CODES.includes(code)@@ -484,6 +509,87 @@ function parseState(annotations, now, config, declarations) {return { entries: [] }}+/**+ * Strict current-format parser for authorization decisions. Unlike parseState,+ * this never falls back to legacy data and never drops malformed entries while+ * returning success. It is intentionally separate so migration/availability+ * callers can keep parseState's tolerant behavior without making corrupt or+ * over-cap state authoritative.+ */+function parseStateStrict(annotations, config, declarations) {+ const raw = annotations && annotations[STATE_ANNOTATION]+ if (typeof raw !== 'string' || raw.length === 0) return { kind: 'absent' }++ let value+ try {+ value = JSON.parse(raw)+ } catch (_error) {+ return { kind: 'invalid', reason: 'state annotation is not valid JSON' }+ }+ if (!Array.isArray(value)) {+ return { kind: 'invalid', reason: 'state annotation is not an array' }+ }+ if (value.length > config.maxEntries) {+ return { kind: 'invalid', reason: 'state annotation exceeds maxEntries' }+ }++ const declarationIds = Array.isArray(declarations)+ ? new Set(+ declarations+ .filter(declaration => declaration && typeof declaration.fqdn === 'string')+ .map(declaration =>+ idOf(declaration.fqdn, declaration.port, declaration.protocol || DEFAULT_PROTOCOL)+ )+ )+ : null+ const entries = []+ const seen = new Set()+ for (const entry of value) {+ if (+ !entry ||+ !isValidIpv4(entry.ip) ||+ !Number.isInteger(entry.port) ||+ entry.port < 1 ||+ entry.port > 65535 ||+ typeof entry.fqdn !== 'string' ||+ entry.fqdn.length === 0 ||+ typeof entry.protocol !== 'string' ||+ entry.protocol.length === 0 ||+ !Number.isFinite(entry.expiresAt) ||+ !Number.isFinite(entry.lastObservedAt) ||+ !Number.isFinite(new Date(entry.lastObservedAt).getTime())+ ) {+ return { kind: 'invalid', reason: 'state annotation contains a malformed entry' }+ }+ if (declarationIds && !declarationIds.has(idOf(entry.fqdn, entry.port, entry.protocol))) {+ return { kind: 'invalid', reason: 'state annotation contains an undeclared identity' }+ }+ const normalized = {+ ip: entry.ip,+ port: entry.port,+ protocol: entry.protocol,+ fqdn: entry.fqdn,+ expiresAt: entry.expiresAt,+ lastObservedAt: entry.lastObservedAt,+ }+ const key = keyOf(normalized)+ if (seen.has(key)) {+ return { kind: 'invalid', reason: 'state annotation contains a duplicate entry' }+ }+ seen.add(key)+ entries.push(normalized)+ }+ entries.sort(byKey)+ const serialized = serializeState({ entries })+ const combinedBytes =+ Buffer.byteLength(serialized[STATE_ANNOTATION], 'utf8') ++ Buffer.byteLength(serialized[TARGETS_ANNOTATION], 'utf8')+ if (combinedBytes > MAX_ANNOTATION_BYTES) {+ return { kind: 'invalid', reason: 'state annotation exceeds the byte budget' }+ }+ return { kind: 'valid', state: { entries } }+}+module.exports = {STATE_ANNOTATION,TARGETS_ANNOTATION,@@ -496,9 +602,11 @@ module.exports = {stateHash,serializeState,parseState,+ parseStateStrict,classifyDnsError,PERMANENT_DNS_CODES,isPermanentDnsCode,isNoRecordsDnsCode,dnsCodeOf,+ classifyDnsRejection,}diff --git a/packages/network-policy-core/index.d.ts b/packages/network-policy-core/index.d.tsindex d8b2fc3be..dcf79cea9 100644--- a/packages/network-policy-core/index.d.ts+++ b/packages/network-policy-core/index.d.ts@@ -126,6 +126,21 @@ export declare function parseState(declarations?: ReadonlyArray<{ fqdn: string; port: number; protocol?: string }>): EgressState+export type StrictStateRead =+ | { kind: 'valid'; state: EgressState }+ | { kind: 'absent' }+ | { kind: 'invalid'; reason: string }++/**+ * Parse only the current structured state for authorization decisions. No+ * legacy fallback and no silent dropping of malformed/over-cap entries.+ */+export declare function parseStateStrict(+ annotations: Record<string, string> | undefined,+ config: EgressCoreConfig,+ declarations?: ReadonlyArray<{ fqdn: string; port: number; protocol?: string }>+): StrictStateRead+export declare const STATE_ANNOTATION: stringexport declare const TARGETS_ANNOTATION: stringexport declare const RESOLVED_AT_ANNOTATION: string@@ -174,6 +189,19 @@ export declare function isPermanentDnsCode(code: unknown): boolean*/export declare function dnsCodeOf(err: unknown): string | undefined+export type DnsRejectionVerdict =+ | { kind: 'transient'; code: string }+ | { kind: 'negative'; reason: 'no-records' | 'invalid-name'; code: string }+ | { kind: 'fault'; code?: string; cause: unknown }++/**+ * Total verdict for a value rejected by a DNS promise. Unknown, missing, and+ * non-object values are faults rather than implicit negative DNS answers.+ * Invoke only at the exact resolver rejection boundary: a code alone does not+ * establish where an exception originated.+ */+export declare function classifyDnsRejection(value: unknown): DnsRejectionVerdict+/*** True for the two genuine no-records answers (ENODATA, ENOTFOUND) and false for* every other code, PERMANENT_DNS_CODES members included. Lets a caller word adiff --git a/packages/network-policy-core/index.test.cjs b/packages/network-policy-core/index.test.cjsindex 14233ebcc..3868e53fd 100644--- a/packages/network-policy-core/index.test.cjs+++ b/packages/network-policy-core/index.test.cjs@@ -415,6 +415,101 @@ test('#567: dnsCodeOf does NOT accept a bare code string, unlike classifyDnsErroassert.equal(core.isPermanentDnsCode(core.dnsCodeOf('ENOTFOUND')), false)})+test('#567 remediation: classifyDnsRejection is total for falsy and non-Error values', () => {+ for (const value of [undefined, null, false, 0, '', 'ENOTFOUND', new Error('no code')]) {+ const verdict = core.classifyDnsRejection(value)+ assert.equal(verdict.kind, 'fault', `${String(value)} must remain a fault`)+ assert.equal(verdict.cause, value)+ }+ assert.deepEqual(core.classifyDnsRejection({ code: 7 }), {+ kind: 'fault',+ cause: { code: 7 },+ })+})++test('#567 remediation: classifyDnsRejection distinguishes transient and negative DNS verdicts', () => {+ assert.deepEqual(+ core.classifyDnsRejection(Object.assign(new Error('servfail'), { code: 'ESERVFAIL' })),+ { kind: 'transient', code: 'ESERVFAIL' }+ )+ assert.deepEqual(+ core.classifyDnsRejection(Object.assign(new Error('not found'), { code: 'ENOTFOUND' })),+ { kind: 'negative', reason: 'no-records', code: 'ENOTFOUND' }+ )+ assert.deepEqual(+ core.classifyDnsRejection(Object.assign(new Error('bad name'), { code: 'EBADNAME' })),+ { kind: 'negative', reason: 'invalid-name', code: 'EBADNAME' }+ )+})++test('#567 remediation: unknown coded rejections remain faults with their code and cause', () => {+ const err = Object.assign(new Error('bad family'), { code: 'EBADFAMILY' })+ assert.deepEqual(core.classifyDnsRejection(err), {+ kind: 'fault',+ code: 'EBADFAMILY',+ cause: err,+ })+})++test('#567 remediation: parseStateStrict rejects absent, malformed, undeclared, duplicate, and over-cap state', () => {+ assert.deepEqual(core.parseStateStrict({}, CFG), { kind: 'absent' })+ assert.equal(core.parseStateStrict({ [core.STATE_ANNOTATION]: '{' }, CFG).kind, 'invalid')++ const validEntry = {+ ip: '1.2.3.4',+ port: 443,+ protocol: 'TCP',+ fqdn: 'api.example.com',+ expiresAt: 2000,+ lastObservedAt: 1000,+ }+ const declaration = [{ fqdn: 'api.example.com', port: 443, protocol: 'TCP' }]+ const undeclared = { ...validEntry, fqdn: 'other.example.com' }+ assert.equal(+ core.parseStateStrict(+ { [core.STATE_ANNOTATION]: JSON.stringify([undeclared]) },+ CFG,+ declaration+ ).kind,+ 'invalid'+ )+ assert.equal(+ core.parseStateStrict(+ { [core.STATE_ANNOTATION]: JSON.stringify([validEntry, validEntry]) },+ CFG,+ declaration+ ).kind,+ 'invalid'+ )+ assert.equal(+ core.parseStateStrict(+ { [core.STATE_ANNOTATION]: JSON.stringify([validEntry, { ...validEntry, ip: '5.6.7.8' }]) },+ { ...CFG, maxEntries: 1 },+ declaration+ ).kind,+ 'invalid'+ )+})++test('#567 remediation: parseStateStrict returns normalized current state only', () => {+ const entry = {+ ip: '1.2.3.4',+ port: 443,+ protocol: 'TCP',+ fqdn: 'api.example.com',+ expiresAt: 2000,+ lastObservedAt: 1000,+ }+ assert.deepEqual(+ core.parseStateStrict(+ { [core.STATE_ANNOTATION]: JSON.stringify([entry]) },+ CFG,+ [{ fqdn: 'api.example.com', port: 443 }]+ ),+ { kind: 'valid', state: { entries: [entry] } }+ )+})+// ─── H-A: revocation/freeze keyed by (fqdn,port,protocol), not fqdn alone ──────// The state map is keyed by (fqdn,ip,port,protocol) but classification used to be// keyed by fqdn only, so an entry whose declared PORT was removed survived underdiff --git a/workflow-recipes/package-lock.json b/workflow-recipes/package-lock.jsonindex 50a6bcac6..e63ee916f 100644--- a/workflow-recipes/package-lock.json+++ b/workflow-recipes/package-lock.json@@ -1,12 +1,12 @@{"name": "workload-recipes-controller",- "version": "0.1.342",+ "version": "0.1.343","lockfileVersion": 3,"requires": true,"packages": {"": {"name": "workload-recipes-controller",- "version": "0.1.342",+ "version": "0.1.343","license": "SEE LICENSE IN LICENSE","dependencies": {"@aws-sdk/client-s3": "^3.1009.0",diff --git a/workflow-recipes/package.json b/workflow-recipes/package.jsonindex 03ba8569f..18fc8a122 100644--- a/workflow-recipes/package.json+++ b/workflow-recipes/package.json@@ -1,6 +1,6 @@{"name": "workload-recipes-controller",- "version": "0.1.342",+ "version": "0.1.343","description": "Workload Recipes Controller (WRC) - WorkloadRecipe lifecycle management and MCP server interface","main": "dist/main.js","engines": {diff --git a/workflow-recipes/src/reconciler/externalEgressAccumulator.test.ts b/workflow-recipes/src/reconciler/externalEgressAccumulator.test.tsindex c9a6d7426..d2285777a 100644--- a/workflow-recipes/src/reconciler/externalEgressAccumulator.test.ts+++ b/workflow-recipes/src/reconciler/externalEgressAccumulator.test.ts@@ -7,7 +7,7 @@ import {reconcileEgressState,serializeState,} from '@clerum/network-policy-core'-import { accumulateExternalEgress } from './externalEgressAccumulator'+import { accumulateExternalEgress, contractExternalEgress } from './externalEgressAccumulator'import type { ResolveResult } from './fqdnResolver'const CONFIG = { overlapMs: 300_000, maxEntries: 128 }@@ -95,6 +95,7 @@ describe('accumulateExternalEgress — H1 fail-static under transient DNS failur[{ fqdn: 'api.github.com', port: 443, ip: '140.82.112.3', ttlSeconds: 15 }],NOW)+ prev[RESOLVED_AT_ANNOTATION] = '2000-01-01T00:00:00.000Z'const resolveResult: ResolveResult = {resolved: [],failures: [{ fqdn: 'api.github.com', error: 'ESERVFAIL', retryable: true }],@@ -110,6 +111,41 @@ describe('accumulateExternalEgress — H1 fail-static under transient DNS failur// The IP is CONSERVED (frozen), not pruned.expect(out.resolved.map(r => r.cidr)).toEqual(['140.82.112.3/32'])expect(out.frozenFqdns).toContain('api.github.com')+ expect(out.annotations[RESOLVED_AT_ANNOTATION]).toBe('2000-01-01T00:00:00.000Z')+ })++ it('preserves the last accepted timestamp on a permanent failure that removes state', () => {+ const prev = previousAnnotations(+ [{ fqdn: 'api.github.com', port: 443, ip: '140.82.112.3', ttlSeconds: 15 }],+ NOW+ )+ prev[RESOLVED_AT_ANNOTATION] = '2000-01-01T00:00:00.000Z'+ const out = accumulateExternalEgress({+ externals: [{ fqdn: 'api.github.com', port: 443 }],+ resolveResult: {+ resolved: [],+ failures: [{ fqdn: 'api.github.com', error: 'no A records', retryable: false }],+ },+ previousAnnotations: prev,+ now: NOW + 1000,+ config: CONFIG,+ })+ expect(out.entries).toEqual([])+ expect(out.annotations[RESOLVED_AT_ANNOTATION]).toBe('2000-01-01T00:00:00.000Z')+ })++ it('does not synthesize a resolved timestamp for a bootstrap failure', () => {+ const out = accumulateExternalEgress({+ externals: [{ fqdn: 'api.github.com', port: 443 }],+ resolveResult: {+ resolved: [],+ failures: [{ fqdn: 'api.github.com', error: 'ESERVFAIL', retryable: true }],+ },+ previousAnnotations: undefined,+ now: NOW,+ config: CONFIG,+ })+ expect(out.annotations[RESOLVED_AT_ANNOTATION]).toBeUndefined()})})@@ -202,6 +238,64 @@ describe('accumulateExternalEgress — F1 revocation of a removed fqdn (audit #2})})+describe('contractExternalEgress — subtractive phase before DNS', () => {+ it('revokes undeclared identities, freezes current ones, and preserves resolved-at', () => {+ const prev = previousAnnotations(+ [+ { fqdn: 'removed.example.com', port: 443, ip: '93.184.216.10', ttlSeconds: 30 },+ { fqdn: 'current.example.com', port: 443, ip: '93.184.216.20', ttlSeconds: 30 },+ ],+ NOW+ )+ prev[RESOLVED_AT_ANNOTATION] = new Date(NOW).toISOString()++ const out = contractExternalEgress({+ externals: [{ fqdn: 'current.example.com', port: 443 }],+ previousAnnotations: prev,+ now: NOW + 30_000 + CONFIG.overlapMs + 60_000,+ config: CONFIG,+ })++ expect(out.resolved.map(entry => entry.cidr)).toEqual(['93.184.216.20/32'])+ expect(out.changed).toBe(true)+ expect(out.annotations[RESOLVED_AT_ANNOTATION]).toBe(new Date(NOW).toISOString())+ expect(out.annotations[TARGETS_ANNOTATION]).not.toContain('removed.example.com')+ })++ it('returns empty trusted state when prior provenance is absent', () => {+ const out = contractExternalEgress({+ externals: [{ fqdn: 'current.example.com', port: 443 }],+ previousAnnotations: undefined,+ now: NOW,+ config: CONFIG,+ })+ expect(out.entries).toEqual([])+ expect(out.resolved).toEqual([])+ expect(out.annotations[STATE_ANNOTATION]).toBe('[]')+ expect(out.annotations[RESOLVED_AT_ANNOTATION]).toBeUndefined()+ })++ it('removes persisted entries rejected by the lane safety filter from state and spec', () => {+ const prev = previousAnnotations(+ [+ { fqdn: 'api.example.com', port: 443, ip: '10.0.0.5', ttlSeconds: 30 },+ { fqdn: 'api.example.com', port: 443, ip: '93.184.216.20', ttlSeconds: 30 },+ ],+ NOW+ )+ const out = contractExternalEgress({+ externals: [{ fqdn: 'api.example.com', port: 443 }],+ previousAnnotations: prev,+ now: NOW + 1000,+ config: CONFIG,+ isAllowedIp: ip => !ip.startsWith('10.'),+ })+ expect(out.resolved.map(entry => entry.cidr)).toEqual(['93.184.216.20/32'])+ expect(out.annotations[STATE_ANNOTATION]).not.toContain('10.0.0.5')+ expect(out.changed).toBe(true)+ })+})+describe('accumulateExternalEgress — permanent failure expires normally', () => {it('does NOT freeze a fqdn that failed permanently (no records)', () => {const prev = previousAnnotations(diff --git a/workflow-recipes/src/reconciler/externalEgressAccumulator.ts b/workflow-recipes/src/reconciler/externalEgressAccumulator.tsindex 1323142a1..1b17c7e76 100644--- a/workflow-recipes/src/reconciler/externalEgressAccumulator.ts+++ b/workflow-recipes/src/reconciler/externalEgressAccumulator.ts@@ -32,6 +32,7 @@ import {RESOLVED_AT_ANNOTATION,emptyState,parseState,+ parseStateStrict,reconcileEgressState,serializeState,} from '@clerum/network-policy-core'@@ -75,6 +76,24 @@ export interface AccumulateOutput {frozenFqdns: string[]}+export interface ContractInput {+ externals: DeclaredExternal[]+ previousAnnotations: Record<string, string> | undefined+ now: number+ config: EgressCoreConfig+ /** Optional lane-specific safety filter applied to persisted IPs. */+ isAllowedIp?: (ip: string) => boolean+}++export interface ContractOutput {+ entries: EgressEntry[]+ resolved: ResolvedExternalEgressInput[]+ annotations: Record<string, string>+ changed: boolean+ evicted: EgressEntry[]+ overCap: boolean+}+/*** Turn the declared externals + this round's resolver output into normalized* observations for the core. Iterating the DECLARED list (not the resolver@@ -131,6 +150,22 @@ function buildObservations(return observations}+function priorAcceptedResolvedAt(+ previousAnnotations: Record<string, string> | undefined,+ config: EgressCoreConfig,+ externals: DeclaredExternal[]+): string | undefined {+ if (!previousAnnotations) return undefined+ const annotated = previousAnnotations[RESOLVED_AT_ANNOTATION]+ if (annotated && Number.isFinite(Date.parse(annotated))) return annotated+ const strict = parseStateStrict(previousAnnotations, config, externals)+ if (strict.kind !== 'valid' || strict.state.entries.length === 0) return undefined+ const latestObservedAt = Math.max(...strict.state.entries.map(entry => entry.lastObservedAt))+ return Number.isFinite(new Date(latestObservedAt).getTime())+ ? new Date(latestObservedAt).toISOString()+ : undefined+}+/*** Fold this reconcile's DNS snapshot into the accumulated egress set persisted* on the policy's annotations, applying the sliding window, fail-static expiry,@@ -147,26 +182,73 @@ export function accumulateExternalEgress(input: AccumulateInput): AccumulateOutpconst observations = buildObservations(externals, resolveResult)const out = reconcileEgressState(previous, observations, now, config)+ const permanentFqdns = new Set(+ resolveResult.failures.filter(failure => !failure.retryable).map(failure => failure.fqdn)+ )+ // A conclusive negative answer blocks this reconcile and revokes that+ // identity immediately. The shared core's normal TTL expiry remains useful+ // for other consumers, but WRC must not keep a known-negative allow live.+ const entries = out.entries.filter(entry => !permanentFqdns.has(entry.fqdn))- const resolved: ResolvedExternalEgressInput[] = out.entries.map(entry => ({+ const resolved: ResolvedExternalEgressInput[] = entries.map(entry => ({cidr: `${entry.ip}/32`,port: entry.port,source: { kind: 'fqdn', fqdn: entry.fqdn },}))- const annotations: Record<string, string> = {- ...serializeState(out.state),- [RESOLVED_AT_ANNOTATION]: new Date(now).toISOString(),- }+ const annotations: Record<string, string> = serializeState({ entries })+ const resolvedAt =+ resolveResult.resolved.length > 0+ ? new Date(now).toISOString()+ : priorAcceptedResolvedAt(previousAnnotations, config, externals)+ if (resolvedAt) annotations[RESOLVED_AT_ANNOTATION] = resolvedAtreturn {- entries: out.entries,+ entries,resolved,annotations,- changed: out.changed,+ changed: out.changed || entries.length !== out.entries.length,renewalDue: out.renewalDue,evicted: out.evicted,overCap: out.overCap,frozenFqdns: out.frozenFqdns,}}++/**+ * Contract an existing aggregate policy to the declarations that still exist,+ * without performing or pretending to perform a DNS observation. Current+ * identities are frozen for this pure subtractive phase; undeclared identities+ * are removed and the normal cap still applies. The last successful resolved-at+ * timestamp is preserved because contraction is not a resolution event.+ */+export function contractExternalEgress(input: ContractInput): ContractOutput {+ const { externals, previousAnnotations, now, config, isAllowedIp } = input+ const previous = previousAnnotations+ ? parseState(previousAnnotations, now, config, externals)+ : emptyState()+ const observations: Observation[] = externals.map(external => ({+ fqdn: external.fqdn,+ port: external.port,+ kind: 'transient',+ }))+ const out = reconcileEgressState(previous, observations, now, config)+ const entries = isAllowedIp ? out.entries.filter(entry => isAllowedIp(entry.ip)) : out.entries+ const resolved: ResolvedExternalEgressInput[] = entries.map(entry => ({+ cidr: `${entry.ip}/32`,+ port: entry.port,+ source: { kind: 'fqdn', fqdn: entry.fqdn },+ }))+ const annotations = serializeState({ entries })+ const lastResolvedAt = priorAcceptedResolvedAt(previousAnnotations, config, externals)+ if (lastResolvedAt) annotations[RESOLVED_AT_ANNOTATION] = lastResolvedAt++ return {+ entries,+ resolved,+ annotations,+ changed: out.changed || entries.length !== out.entries.length,+ evicted: out.evicted,+ overCap: out.overCap,+ }+}diff --git a/workflow-recipes/src/reconciler/fqdnResolver.test.ts b/workflow-recipes/src/reconciler/fqdnResolver.test.tsindex 182ba8320..a40ecc3af 100644--- a/workflow-recipes/src/reconciler/fqdnResolver.test.ts+++ b/workflow-recipes/src/reconciler/fqdnResolver.test.ts@@ -168,16 +168,15 @@ describe('defaultFqdnLookup classification', () => {resolve6Mock.mockReset()})- it('returns ok with A records even when AAAA has none', async () => {+ it('returns ok with A records and does not query unenforceable AAAA records', async () => {resolve4Mock.mockResolvedValue([{ address: '160.79.104.10', ttl: 300 }])- resolve6Mock.mockRejectedValue(dnsError('ENODATA'))const result = await defaultFqdnLookup('api.anthropic.com')expect(result).toEqual({ kind: 'ok', ipv4: ['160.79.104.10'], ipv6: [], ttlSeconds: 300 })+ expect(resolve6Mock).not.toHaveBeenCalled()})it('classifies SERVFAIL as a retryable error and surfaces the code', async () => {resolve4Mock.mockRejectedValue(dnsError('ESERVFAIL'))- resolve6Mock.mockRejectedValue(dnsError('ESERVFAIL'))const result = await defaultFqdnLookup('huggingface.co')expect(result.kind).toBe('error')if (result.kind !== 'error') throw new Error('expected error')@@ -188,7 +187,6 @@ describe('defaultFqdnLookup classification', () => {it('classifies a timeout as retryable', async () => {resolve4Mock.mockRejectedValue(dnsError('ETIMEOUT'))- resolve6Mock.mockRejectedValue(dnsError('ETIMEOUT'))const result = await defaultFqdnLookup('api.anthropic.com')expect(result).toMatchObject({ kind: 'error', retryable: true })})@@ -197,7 +195,6 @@ describe('defaultFqdnLookup classification', () => {'classifies operational DNS failure %s as retryable instead of no-records',async code => {resolve4Mock.mockRejectedValue(dnsError(code))- resolve6Mock.mockRejectedValue(dnsError(code))const result = await defaultFqdnLookup('resolver-broken.example.com')expect(result).toMatchObject({ kind: 'error', retryable: true })if (result.kind !== 'error') throw new Error('expected error')@@ -207,24 +204,22 @@ describe('defaultFqdnLookup classification', () => {it('treats a genuine no-records answer (ENODATA) as permanent', async () => {resolve4Mock.mockRejectedValue(dnsError('ENODATA'))- resolve6Mock.mockRejectedValue(dnsError('ENODATA'))const result = await defaultFqdnLookup('no-records.example.com')- expect(result).toEqual({ kind: 'error', error: 'no A or AAAA records' })+ expect(result).toEqual({ kind: 'error', error: 'no A records for IPv4 egress enforcement' })})it('treats NXDOMAIN (ENOTFOUND) as permanent', async () => {resolve4Mock.mockRejectedValue(dnsError('ENOTFOUND'))- resolve6Mock.mockRejectedValue(dnsError('ENOTFOUND'))const result = await defaultFqdnLookup('does-not-exist.example.com')- expect(result).toEqual({ kind: 'error', error: 'no A or AAAA records' })+ expect(result).toEqual({ kind: 'error', error: 'no A records for IPv4 egress enforcement' })expect((result as { retryable?: boolean }).retryable).toBeFalsy()})- it('is retryable when any family hits a transient code even if the other is empty', async () => {+ it('is retryable when the enforceable A query hits a transient code', async () => {resolve4Mock.mockRejectedValue(dnsError('ECONNREFUSED'))- resolve6Mock.mockRejectedValue(dnsError('ENODATA'))const result = await defaultFqdnLookup('partial.example.com')expect(result).toMatchObject({ kind: 'error', retryable: true })+ expect(resolve6Mock).not.toHaveBeenCalled()})// ─── issue #513: only a positive DNS verdict may become a lookup result ──────@@ -238,7 +233,6 @@ describe('defaultFqdnLookup classification', () => {'T4a throws on %s: a complaint about our own query is not a DNS answer',async code => {resolve4Mock.mockRejectedValue(dnsError(code))- resolve6Mock.mockRejectedValue(dnsError(code))await expect(defaultFqdnLookup('h.example.com')).rejects.toThrow(code)await expect(defaultFqdnLookup('h.example.com')).rejects.toThrow('"h.example.com"')@@ -271,46 +265,48 @@ describe('defaultFqdnLookup classification', () => {it('T4b throws when a rejection carries no code at all', async () => {resolve4Mock.mockRejectedValue(new Error('boom'))- resolve6Mock.mockRejectedValue(dnsError('ENODATA'))await expect(defaultFqdnLookup('nocode.example.com')).rejects.toThrow(/no code/)await expect(defaultFqdnLookup('nocode.example.com')).rejects.toThrow(/boom/)})+ it.each([undefined, null, false, 0, ''])(+ 'T4b-total throws for a falsy rejection payload %s',+ async value => {+ resolve4Mock.mockRejectedValue(value)+ await expect(defaultFqdnLookup('falsy.example.com')).rejects.toThrow(/non-DNS error/)+ }+ )+it('T4c reports EBADNAME as a malformed name, not as an empty answer', async () => {// Permanent like NXDOMAIN, but the resolver never queried the name. Saying// "no A or AAAA records" sends the operator hunting for missing records// instead of at the typo in their own manifest.resolve4Mock.mockRejectedValue(dnsError('EBADNAME'))- resolve6Mock.mockRejectedValue(dnsError('EBADNAME'))const result = await defaultFqdnLookup('foo..example.com')expect(result).toMatchObject({ kind: 'error' })expect((result as { error: string }).error).toContain('EBADNAME')expect((result as { error: string }).error).toContain('foo..example.com')- expect((result as { error: string }).error).not.toContain('no A or AAAA')+ expect((result as { error: string }).error).not.toContain('no A records')expect((result as { retryable?: boolean }).retryable).toBeFalsy()})it('T4d still reports a genuinely empty answer as no-records', async () => {- // A family that RESOLVED to an empty list contributes no rejection at all.- // The gate keys on rejections, so this must not be mistaken for an+ // An empty successful A answer is a genuine negative observation, not an// unclassified fault.resolve4Mock.mockResolvedValue([])- resolve6Mock.mockRejectedValue(dnsError('ENODATA'))const result = await defaultFqdnLookup('empty.example.com')- expect(result).toEqual({ kind: 'error', error: 'no A or AAAA records' })+ expect(result).toEqual({ kind: 'error', error: 'no A records for IPv4 egress enforcement' })})- it('T4e lets a transient code win over an unclassified one', async () => {- // Retrying is harmless, and if the resolver recovers the fault surfaces on- // the next pass with a real answer to contrast it against.+ it('T4e does not let an AAAA success hide an unclassified A fault', async () => {resolve4Mock.mockRejectedValue(dnsError('EBADFAMILY'))- resolve6Mock.mockRejectedValue(dnsError('ESERVFAIL'))+ resolve6Mock.mockResolvedValue(['2001:4860:4860::8888'])- const result = await defaultFqdnLookup('mixed.example.com')- expect(result).toMatchObject({ kind: 'error', retryable: true })+ await expect(defaultFqdnLookup('mixed.example.com')).rejects.toThrow('EBADFAMILY')+ expect(resolve6Mock).not.toHaveBeenCalled()})})@@ -323,7 +319,6 @@ describe('defaultFqdnLookup TTL propagation (issue #299)', () => {it('requests per-record TTLs via resolve4({ ttl: true })', async () => {resolve4Mock.mockResolvedValue([{ address: '140.82.112.3', ttl: 18 }])- resolve6Mock.mockRejectedValue(dnsError('ENODATA'))await defaultFqdnLookup('api.github.com')expect(resolve4Mock).toHaveBeenCalledWith('api.github.com', { ttl: true })})@@ -333,7 +328,6 @@ describe('defaultFqdnLookup TTL propagation (issue #299)', () => {{ address: '140.82.112.3', ttl: 18 },{ address: '140.82.112.4', ttl: 12 },])- resolve6Mock.mockRejectedValue(dnsError('ENODATA'))const result = await defaultFqdnLookup('api.github.com')expect(result).toEqual({kind: 'ok',diff --git a/workflow-recipes/src/reconciler/fqdnResolver.ts b/workflow-recipes/src/reconciler/fqdnResolver.tsindex 30fcfaaf5..46069200a 100644--- a/workflow-recipes/src/reconciler/fqdnResolver.ts+++ b/workflow-recipes/src/reconciler/fqdnResolver.ts@@ -1,11 +1,6 @@import type { RecordWithTtl } from 'node:dns'-import { resolve4, resolve6 } from 'node:dns/promises'-import {- classifyDnsError,- dnsCodeOf,- isNoRecordsDnsCode,- isPermanentDnsCode,-} from '@clerum/network-policy-core'+import { resolve4 } from 'node:dns/promises'+import { classifyDnsRejection } from '@clerum/network-policy-core'import type { SandboxUiExternalEgress } from '../types'const BLOCKED_EGRESS_CIDRS = [@@ -90,12 +85,12 @@ export interface ResolvedExternalEgress {export type FqdnLookupResult =// `ttlSeconds` is the MINIMUM TTL (seconds) across the resolved A records —// the binding window for the whole FQDN, consumed by the egress accumulator- // (issue #299). It is 0 when only AAAA resolved (A-less), since no /32 is- // emitted for an IPv4-only enforcement target in that case.+ // (issue #299). `ipv6` remains as an empty compatibility field until the+ // platform has an explicit dual-stack enforcement capability.| { kind: 'ok'; ipv4: string[]; ipv6: string[]; ttlSeconds: number }// `retryable` distinguishes a transient resolver failure (SERVFAIL, timeout,// unreachable upstream) — worth retrying on a later reconcile — from a- // permanent one (the name genuinely has no A/AAAA records, or is malformed).+ // permanent one (the name has no enforceable A records, or is malformed).// Absent ⇒ permanent. A failure that is not a DNS verdict at all never becomes// a result: the lookup throws (issue #513).| { kind: 'error'; error: string; retryable?: boolean }@@ -104,126 +99,79 @@ export type FqdnLookup = (host: string) => Promise<FqdnLookupResult>/*** Transient-vs-permanent DNS classification is centralized in- * `@clerum/network-policy-core` (`classifyDnsError`) so WRC and HCC agree on what- * a transient failure is — a single source of truth avoids drift between the two- * controllers' fail-static behavior (issue #299 audit F6). c-ares / system codes- * meaning "resolver/upstream temporarily unavailable" map to `retryable: true`;- * stable answers (ENODATA, ENOTFOUND/NXDOMAIN, empty) fail closed permanently.+ * `@clerum/network-policy-core` (`classifyDnsRejection`) so WRC and HCC agree on+ * a total resolver-boundary verdict. Unknown/missing values remain faults;+ * c-ares/system outage codes map to `retryable: true`; stable negative answers+ * fail closed permanently.*//**- * `settleV4`/`settleV6` cast whatever the promise rejected with to- * `NodeJS.ErrnoException`; nothing guarantees it was an Error. Reading `.message`- * off a rejected string or object then interpolates `undefined` into the message- * an operator reads. Fall back to the value itself, never to a placeholder.+ * Rejection payloads are unknown by construction. Keep diagnostics total so an+ * unusual promise rejection cannot replace the controller fault it is reporting.*/-function describeRejection(err: NodeJS.ErrnoException): string {- return typeof err?.message === 'string' ? err.message : String(err)+function describeRejection(err: unknown): string {+ try {+ if (+ typeof err === 'object' &&+ err !== null &&+ typeof (err as { message?: unknown }).message === 'string'+ ) {+ return (err as { message: string }).message+ }+ return String(err)+ } catch {+ // A hostile Proxy/toString must not hide the original fault classification.+ return 'unprintable rejection value'+ }}/**- * Default FQDN lookup using node:dns/promises. Returns ipv4 and ipv6 record- * sets independently; a missing record type for one family is fine as long as- * the other resolves, so `api.example.com` (A only) and `ipv6-only.example.com`- * both resolve cleanly.+ * Default FQDN lookup using node:dns/promises. The current NetworkPolicy+ * dataplane emits only IPv4 /32 entries, so A is the sole enforceable family.+ * Querying AAAA and treating it as success would let an A failure erase/fail to+ * freeze the actual IPv4 window. Dual-stack must arrive as an explicit future+ * capability rather than an inert resolver side channel.*- * When neither family yields a record, the lookup inspects the underlying DNS- * error code to decide whether the empty answer is permanent (no records) or a- * transient resolver failure (SERVFAIL/timeout/unreachable upstream). The latter- * is reported with `retryable: true` and the real code in the message, so the- * reconciler can retry instead of collapsing every failure into the misleading- * "no A or AAAA records".+ * Only the catch around resolve4 classifies a rejection. Downstream validation+ * faults occur outside that boundary and therefore cannot be laundered by a+ * DNS-looking `.code`.*/export const defaultFqdnLookup: FqdnLookup = async host => {- // A records are requested WITH per-record TTLs (`{ ttl: true }`) so the- // accumulator can size the sliding window off the real DNS TTL (issue #299).- // AAAA is resolved without TTLs: /128 blocks are dropped (IPv4-only targets),- // so their TTL would be inert.- const settleV4 = (p: Promise<RecordWithTtl[]>) =>- p.then(- records => ({ records }) as { records: RecordWithTtl[] },- (err: NodeJS.ErrnoException) => ({ err }) as { err: NodeJS.ErrnoException }- )- const settleV6 = (p: Promise<string[]>) =>- p.then(- addrs => ({ addrs }) as { addrs: string[] },- (err: NodeJS.ErrnoException) => ({ err }) as { err: NodeJS.ErrnoException }- )-- const [v4, v6] = await Promise.all([- settleV4(resolve4(host, { ttl: true })),- settleV6(resolve6(host)),- ])-- const v4Records = 'records' in v4 ? v4.records : []- const ipv4 = v4Records.map(r => r.address)- const ipv6 = 'addrs' in v6 ? v6.addrs : []- if (ipv4.length > 0 || ipv6.length > 0) {- // Minimum TTL across the A records = the window for the whole FQDN. 0 when- // A-less (AAAA-only), where no /32 is emitted anyway.- const ttlSeconds = v4Records.length > 0 ? Math.min(...v4Records.map(r => r.ttl)) : 0- return { kind: 'ok', ipv4, ipv6, ttlSeconds }- }-- const rejections: NodeJS.ErrnoException[] = []- if ('err' in v4) rejections.push(v4.err)- if ('err' in v6) rejections.push(v6.err)-- const transient = rejections.find(e => classifyDnsError(e) === 'transient')- if (transient) {- return {- kind: 'error',- // Read through `dnsCodeOf` like every other consumer below. A `transient`- // verdict already guarantees a string code, so this is display-only — but- // it is the last hand-rolled `.code` read left in the consumers, and the- // point of the shared helper is that there is exactly one way to ask.- error: `DNS resolution for "${host}" failed (${dnsCodeOf(transient) ?? 'unknown'}) — resolver or upstream unavailable`,- retryable: true,+ let records: RecordWithTtl[]+ try {+ records = await resolve4(host, { ttl: true })+ } catch (error: unknown) {+ const verdict = classifyDnsRejection(error)+ if (verdict.kind === 'transient') {+ return {+ kind: 'error',+ error: `DNS resolution for "${host}" failed (${verdict.code}) — resolver or upstream unavailable`,+ retryable: true,+ }+ }+ if (verdict.kind === 'negative') {+ if (verdict.reason === 'invalid-name') {+ return {+ kind: 'error',+ error: `malformed hostname "${host}" (${verdict.code}) — the resolver refused to query it`,+ }+ }+ return { kind: 'error', error: 'no A records for IPv4 egress enforcement' }}- }-- // Issue #513: only a POSITIVE DNS verdict may become a lookup result. A family- // that resolved to an empty list contributes no rejection at all; one that- // rejected must carry a code in PERMANENT_DNS_CODES. Anything else — a- // codeless rejection, or a c-ares complaint about the query WE built rather- // than about the name — is not a DNS answer, and the old default branch turned- // it into "no A or AAAA records": a reply that was never received, blamed on- // the resolver, and routed on through egressResolutionError() as the- // operator's permanent misconfiguration. Throw instead. The reconciler's catch- // already routes a plain Error to the terminal `failed` phase with the real- // message (walking `.cause` for socket codes), and this throw happens before- // any policy write or delete, so the live NetworkPolicy is left untouched. The- // message deliberately avoids the 'egress resolution failed' marker that catch- // matches on: a raw fault is not a DNS classification worth protecting. That- // absence is asserted in fqdnResolver.test.ts rather than left as prose.- //- // The code is read through the package's `dnsCodeOf` rather than off `.code`- // directly, so both egress gates unwrap a rejection the same way (#567 review,- // R1-M2). It also makes the read total: `rejections` is typed as- // ErrnoException[] because the settle wrappers say so, not because anything- // checked, so a non-object rejection would have thrown on the property access.- const unclassified = rejections.find(e => !isPermanentDnsCode(dnsCodeOf(e)))- if (unclassified) {throw new Error(- `DNS lookup for "${host}" failed with a non-DNS error (${dnsCodeOf(unclassified) ?? 'no code'}): ${describeRejection(unclassified)}`,- { cause: unclassified }+ `DNS lookup for "${host}" failed with a non-DNS error (${verdict.code ?? 'no code'}): ${describeRejection(verdict.cause)}`,+ { cause: verdict.cause })}- // A permanent verdict that is NOT one of the two no-records answers means the- // resolver refused to send the query at all — EBADNAME today. Report the real- // code rather than testing for it: hardcoding 'EBADNAME' here would drift the- // moment PERMANENT_DNS_CODES gains another member, and the message would fall- // back to "no A or AAAA records" — the same lie this branch exists to remove.- const refused = rejections.find(e => !isNoRecordsDnsCode(dnsCodeOf(e)))- if (refused) {- return {- kind: 'error',- error: `malformed hostname "${host}" (${dnsCodeOf(refused)}) — the resolver refused to query it`,- }+ // Everything below is downstream of a successful resolver call. Exceptions+ // here are controller/contract faults even if they happen to carry a DNS code.+ const ipv4 = records.map(record => record.address)+ if (ipv4.length === 0) {+ return { kind: 'error', error: 'no A records for IPv4 egress enforcement' }}-- return { kind: 'error', error: 'no A or AAAA records' }+ const ttlSeconds = Math.min(...records.map(record => record.ttl))+ return { kind: 'ok', ipv4, ipv6: [], ttlSeconds }}export interface ResolveResult {diff --git a/workflow-recipes/src/reconciler/mcpDelegation.test.ts b/workflow-recipes/src/reconciler/mcpDelegation.test.tsindex 929ff5419..67040a428 100644--- a/workflow-recipes/src/reconciler/mcpDelegation.test.ts+++ b/workflow-recipes/src/reconciler/mcpDelegation.test.ts@@ -2062,6 +2062,60 @@ describe('waitForExternalEgressReady', () => {expect(result.pending).toEqual(['web-search'])expect(result.failed).toEqual([])})++ it('treats a current controller-fault condition as retryable pending', async () => {+ const deps: DelegationDeps = {+ customApi: {+ getNamespacedCustomObject: vi.fn().mockResolvedValue({+ metadata: { generation: 3 },+ status: {+ conditions: [+ {+ type: 'ExternalEgressReady',+ status: 'False',+ reason: 'ExternalEgressReconcileFailed',+ message: 'controller fault; runtime remains blocked',+ observedGeneration: 3,+ },+ ],+ },+ }),+ } as any,+ coreApi: {} as any,+ }++ const result = await waitForExternalEgressReady(deps, ['web-search'], 'mcp-server', 1)+ expect(result).toEqual({ ready: false, pending: ['web-search'], failed: [] })+ })++ it('keeps a current rejected binding terminal and fail-closed', async () => {+ const deps: DelegationDeps = {+ customApi: {+ getNamespacedCustomObject: vi.fn().mockResolvedValue({+ metadata: { generation: 3 },+ status: {+ conditions: [+ {+ type: 'ExternalEgressReady',+ status: 'False',+ reason: 'ExternalEgressRejected',+ message: 'hostname is not allowed',+ observedGeneration: 3,+ },+ ],+ },+ }),+ } as any,+ coreApi: {} as any,+ }++ const result = await waitForExternalEgressReady(deps, ['web-search'], 'mcp-server', 1000)+ expect(result).toEqual({+ ready: false,+ pending: ['web-search'],+ failed: [{ name: 'web-search', message: 'hostname is not allowed' }],+ })+ })})// ─── Issue #408 — waitForNetworkReady must be generation-aware ─────────────────diff --git a/workflow-recipes/src/reconciler/mcpDelegation.ts b/workflow-recipes/src/reconciler/mcpDelegation.tsindex 21b5f8168..521e15b46 100644--- a/workflow-recipes/src/reconciler/mcpDelegation.ts+++ b/workflow-recipes/src/reconciler/mcpDelegation.ts@@ -37,6 +37,14 @@ import { effectiveWorkflowContextRef, privateWorkflowContextName } from './workfconst MCPSERVER_PLURAL = 'mcpservers'const CONTEXT_PLURAL = 'contexts'const EXTERNAL_EGRESS_READY_CONDITION = 'ExternalEgressReady'+const RETRYABLE_EXTERNAL_EGRESS_REASONS = new Set([+ 'ExternalEgressReconcileFailed',+ 'InventoryListFailed',+ 'CleanupFailed',+ 'ExternalEgressProofFailed',+ 'ExternalEgressUntrustedState',+ 'StatusWriteFailed',+])const MCPSERVER_REPLACE_CONFLICT_RETRIES = 3const CONTEXT_REPLACE_CONFLICT_RETRIES = 3@@ -1041,6 +1049,7 @@ export async function waitForExternalEgressReady(return {name,status: isFresh ? condition?.status : 'Unknown',+ reason: isFresh ? condition?.reason : undefined,message:isFresh && condition? (condition.message ??@@ -1053,7 +1062,8 @@ export async function waitForExternalEgressReady(} catch (error) {return {name,- status: 'False',+ status: getErrorCode(error) === 404 ? 'False' : 'Unknown',+ reason: getErrorCode(error) === 404 ? 'McpServerDeleted' : 'InventoryReadFailed',message:getErrorCode(error) === 404? 'McpServer was deleted before external egress became ready'@@ -1068,7 +1078,9 @@ export async function waitForExternalEgressReady(pending.delete(result.name)console.log(`[MCP-Delegation] External egress ready for "${result.name}"`)} else if (result.status === 'False') {- failed.set(result.name, result.message ?? `${EXTERNAL_EGRESS_READY_CONDITION} is False`)+ if (!result.reason || !RETRYABLE_EXTERNAL_EGRESS_REASONS.has(result.reason)) {+ failed.set(result.name, result.message ?? `${EXTERNAL_EGRESS_READY_CONDITION} is False`)+ }}}diff --git a/workflow-recipes/src/reconciler/reconciler.test.ts b/workflow-recipes/src/reconciler/reconciler.test.tsindex b089a9871..c22ac5028 100644--- a/workflow-recipes/src/reconciler/reconciler.test.ts+++ b/workflow-recipes/src/reconciler/reconciler.test.ts@@ -187,6 +187,23 @@ function snippetRun(code = 'return { ok: true }') {return { type: 'snippet' as const, language: 'typescript' as const, code }}+function egressStateAnnotations(+ entries: Array<{ fqdn: string; ip: string; port: number }>+): Record<string, string> {+ const now = Date.now()+ return {+ 'clerum.io/egress-fqdn-state': JSON.stringify(+ entries.map(entry => ({+ ...entry,+ protocol: 'TCP',+ expiresAt: now + 600_000,+ lastObservedAt: now,+ }))+ ),+ 'clerum.io/egress-fqdn-resolved-at': new Date(now).toISOString(),+ }+}+describe('WorkflowRecipeReconciler', () => {let reconciler: WorkflowRecipeReconciler@@ -249,18 +266,28 @@ describe('WorkflowRecipeReconciler', () => {mockNetworkingApi.createNamespacedNetworkPolicy.mockReset()mockNetworkingApi.createNamespacedNetworkPolicy.mockResolvedValue({})mockNetworkingApi.readNamespacedNetworkPolicy.mockReset()- mockNetworkingApi.readNamespacedNetworkPolicy.mockImplementation(({ name }: { name: string }) =>- Promise.resolve({- metadata: {- name,- uid: `uid-${name}`,- resourceVersion: '1',- labels: {- 'clerum.io/managed-by': 'wrc',- 'clerum.io/recipe': name.replace(/-coordinator-to-gfs$/, ''),+ mockNetworkingApi.readNamespacedNetworkPolicy.mockImplementation(+ ({ name }: { name: string }) => {+ if (+ name.startsWith('ui-egress-') ||+ name.startsWith('wl-egress-') ||+ name.startsWith('ui-ingress-') ||+ name.startsWith('wl-ingress-')+ ) {+ return Promise.reject({ code: 404 })+ }+ return Promise.resolve({+ metadata: {+ name,+ uid: `uid-${name}`,+ resourceVersion: '1',+ labels: {+ 'clerum.io/managed-by': 'wrc',+ 'clerum.io/recipe': name.replace(/-coordinator-to-gfs$/, ''),+ },},- },- })+ })+ })mockNetworkingApi.replaceNamespacedNetworkPolicy.mockReset()mockNetworkingApi.replaceNamespacedNetworkPolicy.mockResolvedValue({})@@ -1683,16 +1710,28 @@ describe('WorkflowRecipeReconciler', () => {it('fails closed instead of replacing an existing non-WRC internal-dependency policy', async () => {mockNetworkingApi.createNamespacedNetworkPolicy.mockRejectedValueOnce({ code: 409 })- mockNetworkingApi.readNamespacedNetworkPolicy.mockResolvedValueOnce({- metadata: {- resourceVersion: 'rv-hcc',- labels: {- 'clerum.io/managed-by': 'host-capability-controller',- 'clerum.io/policy-type': 'binding-allow',- 'clerum.io/recipe': 'test-recipe',- },- },- })+ mockNetworkingApi.readNamespacedNetworkPolicy.mockImplementation(+ ({ name }: { name: string }) => {+ if (+ name.startsWith('ui-egress-') ||+ name.startsWith('wl-egress-') ||+ name.startsWith('ui-ingress-') ||+ name.startsWith('wl-ingress-')+ ) {+ return Promise.reject({ code: 404 })+ }+ return Promise.resolve({+ metadata: {+ resourceVersion: 'rv-hcc',+ labels: {+ 'clerum.io/managed-by': 'host-capability-controller',+ 'clerum.io/policy-type': 'binding-allow',+ 'clerum.io/recipe': 'test-recipe',+ },+ },+ })+ }+ )const result = await reconciler.reconcile(makeRecipe({@@ -2488,9 +2527,6 @@ describe('WorkflowRecipeReconciler', () => {resolve4Mock.mockRejectedValueOnce(Object.assign(new Error('queryA EBADFAMILY'), { code: 'EBADFAMILY' }))- resolve6Mock.mockRejectedValueOnce(- Object.assign(new Error('queryAaaa EBADFAMILY'), { code: 'EBADFAMILY' })- )const reconcilerWithLookup = new WorkflowRecipeReconciler(new k8s.KubeConfig(), undefined, {fqdnLookup: host => defaultFqdnLookup(host),})@@ -2509,6 +2545,7 @@ describe('WorkflowRecipeReconciler', () => {// returning an error instead of throwing — leaves the test green, because// both routes end in `failed` with the code somewhere in the message.expect(result.message).not.toContain('egress resolution failed')+ expect(resolve6Mock).not.toHaveBeenCalled()const egressPolicy = expect.objectContaining({body: expect.objectContaining({@@ -2522,6 +2559,374 @@ describe('WorkflowRecipeReconciler', () => {)})+ it('#567 remediation: contracts a removed workload binding before a sibling lookup faults', async () => {+ const policyName = 'wl-egress-test-recipe-worker'+ let existing: k8s.V1NetworkPolicy = {+ metadata: {+ name: policyName,+ namespace: 'sandbox-recipes',+ uid: 'uid-workload-egress',+ resourceVersion: '7',+ labels: {+ 'clerum.io/managed-by': 'workflow-recipes',+ 'clerum.io/recipe': 'test-recipe',+ 'clerum.io/workload': 'worker',+ },+ annotations: egressStateAnnotations([+ { fqdn: 'removed.example.com', ip: '93.184.216.10', port: 443 },+ { fqdn: 'current.example.com', ip: '93.184.216.20', port: 443 },+ ]),+ },+ spec: {+ podSelector: {+ matchLabels: { 'clerum.io/recipe': 'test-recipe', 'clerum.io/workload': 'worker' },+ },+ policyTypes: ['Egress'],+ egress: ['93.184.216.10/32', '93.184.216.20/32'].map(cidr => ({+ to: [{ ipBlock: { cidr } }],+ ports: [{ port: 443, protocol: 'TCP' }],+ })),+ },+ }+ mockNetworkingApi.readNamespacedNetworkPolicy.mockImplementation(+ ({ name }: { name: string }) =>+ name === policyName ? Promise.resolve(existing) : Promise.reject({ code: 404 })+ )+ mockNetworkingApi.replaceNamespacedNetworkPolicy.mockImplementation(+ ({ body }: { body: k8s.V1NetworkPolicy }) => {+ existing = {+ ...body,+ metadata: {+ ...body.metadata,+ name: policyName,+ namespace: 'sandbox-recipes',+ uid: 'uid-workload-egress',+ resourceVersion: '8',+ },+ }+ return Promise.resolve(existing)+ }+ )+ const rec = new WorkflowRecipeReconciler(new k8s.KubeConfig(), undefined, {+ fqdnLookup: async () => {+ throw Object.assign(new Error('queryA EBADFAMILY'), { code: 'EBADFAMILY' })+ },+ })+ const result = await rec.reconcile(+ makeRecipe({+ spec: {+ contextRef: 'default',+ workloads: [+ {+ id: 'worker',+ type: 'deployment',+ image: 'worker:latest',+ port: 8080,+ egressBindings: [{ dns: 'current.example.com', port: 443 }],+ },+ ],+ },+ })+ )++ expect(result.phase).toBe('failed')+ expect(result.message).toContain('EBADFAMILY')+ const replacement = mockNetworkingApi.replaceNamespacedNetworkPolicy.mock.calls.find(+ call => call[0]?.name === policyName+ )?.[0].body as k8s.V1NetworkPolicy+ const cidrs = (replacement?.spec?.egress ?? []).flatMap(rule =>+ (rule.to ?? []).map(peer => peer.ipBlock?.cidr)+ )+ if (replacement) {+ expect(cidrs).toEqual(['93.184.216.20/32'])+ expect(JSON.stringify(replacement.metadata?.annotations)).not.toContain('removed.example.com')+ } else {+ // If any hand-built live dimension cannot prove a subset, exact deletion+ // is the safer contraction than retaining the removed binding.+ expect(mockNetworkingApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledWith(+ expect.objectContaining({ name: policyName, namespace: 'sandbox-recipes' })+ )+ }+ })++ it('#567 remediation: contracts workload authorization before a UI lookup can fail', async () => {+ const policyName = 'wl-egress-test-recipe-worker'+ const ingressPolicyName = 'wl-ingress-test-recipe-worker'+ const events: string[] = []+ mockNetworkingApi.readNamespacedNetworkPolicy.mockImplementation(+ ({ name }: { name: string }) =>+ name === policyName || name === ingressPolicyName+ ? Promise.resolve({+ metadata: {+ name,+ namespace: 'sandbox-recipes',+ uid:+ name === policyName ? 'uid-stale-workload-egress' : 'uid-stale-workload-ingress',+ resourceVersion: '21',+ labels: {+ 'clerum.io/managed-by': 'workflow-recipes',+ 'clerum.io/recipe': 'test-recipe',+ },+ },+ spec:+ name === policyName+ ? { podSelector: {}, policyTypes: ['Egress'], egress: [] }+ : { podSelector: {}, policyTypes: ['Ingress'], ingress: [] },+ } satisfies k8s.V1NetworkPolicy)+ : Promise.reject({ code: 404 })+ )+ mockNetworkingApi.deleteNamespacedNetworkPolicy.mockImplementation(({ name }) => {+ events.push(`${name}-delete`)+ return Promise.resolve({})+ })+ const rec = new WorkflowRecipeReconciler(new k8s.KubeConfig(), undefined, {+ fqdnLookup: async () => {+ events.push('ui-lookup')+ return { kind: 'error', error: 'no A records', retryable: false }+ },+ })+ const result = await rec.reconcile(+ makeRecipe({+ spec: {+ workloads: [+ { id: 'frontend', type: 'deployment', image: 'fe:1', port: 8080 },+ { id: 'worker', type: 'deployment', image: 'worker:1', port: 3000 },+ ],+ ui: {+ workloadRef: 'frontend',+ port: 8080,+ egress: { external: [{ fqdn: 'broken.example.com', port: 443 }] },+ },+ },+ })+ )++ expect(result.phase).toBe('failed')+ expect(events).toEqual([`${policyName}-delete`, `${ingressPolicyName}-delete`, 'ui-lookup'])+ expect(mockNetworkingApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledWith({+ name: policyName,+ namespace: 'sandbox-recipes',+ body: {+ preconditions: {+ uid: 'uid-stale-workload-egress',+ resourceVersion: '21',+ },+ },+ })+ expect(mockNetworkingApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledWith({+ name: ingressPolicyName,+ namespace: 'sandbox-recipes',+ body: {+ preconditions: {+ uid: 'uid-stale-workload-ingress',+ resourceVersion: '21',+ },+ },+ })+ })++ it('#567 remediation: treats a concurrent 404 contraction delete as already revoked', async () => {+ const policyName = 'wl-egress-test-recipe-worker'+ mockNetworkingApi.readNamespacedNetworkPolicy.mockImplementation(+ ({ name }: { name: string }) =>+ name === policyName+ ? Promise.resolve({+ metadata: {+ name,+ namespace: 'sandbox-recipes',+ uid: 'uid-concurrent-delete',+ resourceVersion: '31',+ labels: {+ 'clerum.io/managed-by': 'workflow-recipes',+ 'clerum.io/recipe': 'test-recipe',+ },+ },+ spec: { podSelector: {}, policyTypes: ['Egress'], egress: [] },+ } satisfies k8s.V1NetworkPolicy)+ : Promise.reject({ code: 404 })+ )+ mockNetworkingApi.deleteNamespacedNetworkPolicy.mockRejectedValueOnce({+ code: 404,+ message: 'already gone',+ })++ const result = await reconciler.reconcile(+ makeRecipe({+ spec: {+ workloads: [{ id: 'worker', type: 'deployment', image: 'worker:1', port: 8080 }],+ },+ })+ )++ expect(result.phase).toBe('active')+ expect(mockNetworkingApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledWith({+ name: policyName,+ namespace: 'sandbox-recipes',+ body: { preconditions: { uid: 'uid-concurrent-delete', resourceVersion: '31' } },+ })+ })++ it('#567 remediation: treats a 404 contraction read-back as a safe concurrent revocation', async () => {+ const policyName = 'wl-egress-test-recipe-worker'+ let replaced = false+ const existing: k8s.V1NetworkPolicy = {+ metadata: {+ name: policyName,+ namespace: 'sandbox-recipes',+ uid: 'uid-readback-race',+ resourceVersion: '41',+ labels: {+ 'clerum.io/managed-by': 'workflow-recipes',+ 'clerum.io/recipe': 'test-recipe',+ 'clerum.io/workload': 'worker',+ },+ annotations: egressStateAnnotations([+ { fqdn: 'removed.example.com', ip: '93.184.216.10', port: 443 },+ { fqdn: 'current.example.com', ip: '93.184.216.20', port: 443 },+ ]),+ },+ spec: {+ podSelector: {+ matchLabels: { 'clerum.io/recipe': 'test-recipe', 'clerum.io/workload': 'worker' },+ },+ policyTypes: ['Egress'],+ egress: ['93.184.216.10/32', '93.184.216.20/32'].map(cidr => ({+ to: [{ ipBlock: { cidr } }],+ ports: [{ port: 443, protocol: 'TCP' }],+ })),+ },+ }+ mockNetworkingApi.readNamespacedNetworkPolicy.mockImplementation(+ ({ name }: { name: string }) => {+ if (name !== policyName) return Promise.reject({ code: 404 })+ return replaced ? Promise.reject({ code: 404 }) : Promise.resolve(existing)+ }+ )+ mockNetworkingApi.replaceNamespacedNetworkPolicy.mockImplementation(() => {+ replaced = true+ return Promise.resolve({})+ })+ const rec = new WorkflowRecipeReconciler(new k8s.KubeConfig(), undefined, {+ fqdnLookup: async () => {+ throw Object.assign(new Error('queryA EBADFAMILY'), { code: 'EBADFAMILY' })+ },+ })+ const result = await rec.reconcile(+ makeRecipe({+ spec: {+ workloads: [+ {+ id: 'worker',+ type: 'deployment',+ image: 'worker:1',+ port: 8080,+ egressBindings: [{ dns: 'current.example.com', port: 443 }],+ },+ ],+ },+ })+ )++ expect(result.phase).toBe('failed')+ expect(result.message).toContain('EBADFAMILY')+ expect(mockNetworkingApi.deleteNamespacedNetworkPolicy).not.toHaveBeenCalledWith(+ expect.objectContaining({ name: policyName })+ )+ })++ it('#567 remediation: uses fresh post-replace identity for same-pass revocation', async () => {+ const policyName = 'wl-egress-test-recipe-worker'+ const now = Date.now()+ let live: k8s.V1NetworkPolicy = {+ metadata: {+ name: policyName,+ namespace: 'sandbox-recipes',+ uid: 'uid-workload-egress',+ resourceVersion: '7',+ labels: {+ 'clerum.io/managed-by': 'workflow-recipes',+ 'clerum.io/recipe': 'test-recipe',+ 'clerum.io/workload': 'worker',+ },+ annotations: {+ 'clerum.io/egress-fqdn-state': JSON.stringify(+ [+ ['removed.example.com', '93.184.216.10'],+ ['current.example.com', '93.184.216.20'],+ ].map(([fqdn, ip]) => ({+ fqdn,+ ip,+ port: 443,+ protocol: 'TCP',+ expiresAt: now - 60_000,+ lastObservedAt: now - 120_000,+ }))+ ),+ 'clerum.io/egress-fqdn-resolved-at': new Date(now - 120_000).toISOString(),+ },+ },+ spec: {+ podSelector: {+ matchLabels: { 'clerum.io/recipe': 'test-recipe', 'clerum.io/workload': 'worker' },+ },+ policyTypes: ['Egress'],+ egress: ['93.184.216.10/32', '93.184.216.20/32'].map(cidr => ({+ to: [{ ipBlock: { cidr } }],+ ports: [{ port: 443, protocol: 'TCP' }],+ })),+ },+ }+ mockNetworkingApi.readNamespacedNetworkPolicy.mockImplementation(+ ({ name }: { name: string }) =>+ name === policyName ? Promise.resolve(live) : Promise.reject({ code: 404 })+ )+ mockNetworkingApi.replaceNamespacedNetworkPolicy.mockImplementation(+ ({ body }: { body: k8s.V1NetworkPolicy }) => {+ live = {+ ...body,+ metadata: {+ ...body.metadata,+ name: policyName,+ namespace: 'sandbox-recipes',+ uid: 'uid-workload-egress',+ resourceVersion: '8',+ },+ }+ return Promise.resolve(live)+ }+ )+ const rec = new WorkflowRecipeReconciler(new k8s.KubeConfig(), undefined, {+ fqdnLookup: async () => ({ kind: 'error', error: 'no A records', retryable: false }),+ })+ const result = await rec.reconcile(+ makeRecipe({+ spec: {+ workloads: [+ {+ id: 'worker',+ type: 'deployment',+ image: 'worker:1',+ port: 8080,+ egressBindings: [{ dns: 'current.example.com', port: 443 }],+ },+ ],+ },+ })+ )++ expect(result.phase).toBe('failed')+ expect(mockNetworkingApi.replaceNamespacedNetworkPolicy).toHaveBeenCalledWith(+ expect.objectContaining({ name: policyName })+ )+ expect(mockNetworkingApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledWith({+ name: policyName,+ namespace: 'sandbox-recipes',+ body: {+ preconditions: { uid: 'uid-workload-egress', resourceVersion: '8' },+ },+ })+ })+it('returns failed phase when a workload egress binding uses wildcard DNS', async () => {const recipe = makeRecipe({spec: {@@ -4033,9 +4438,6 @@ describe('WorkflowRecipeReconciler', () => {resolve4Mock.mockRejectedValueOnce(Object.assign(new Error('queryA EBADFAMILY'), { code: 'EBADFAMILY' }))- resolve6Mock.mockRejectedValueOnce(- Object.assign(new Error('queryAaaa EBADFAMILY'), { code: 'EBADFAMILY' })- )const kc = new k8s.KubeConfig()const rec = new WorkflowRecipeReconciler(kc, undefined, {fqdnLookup: host => defaultFqdnLookup(host),@@ -4057,6 +4459,7 @@ describe('WorkflowRecipeReconciler', () => {// returning an error instead of throwing — leaves the test green, because// both routes end in `failed` with the code somewhere in the message.expect(result.message).not.toContain('egress resolution failed')+ expect(resolve6Mock).not.toHaveBeenCalled()expect(createdPolicy('ui-egress-test-recipe')).toBeUndefined()expect(mockNetworkingApi.replaceNamespacedNetworkPolicy).not.toHaveBeenCalledWith(@@ -4071,6 +4474,285 @@ describe('WorkflowRecipeReconciler', () => {)})+ it('#567 remediation: contracts a removed UI binding before a sibling lookup faults', async () => {+ const policyName = 'ui-egress-test-recipe'+ const existing: k8s.V1NetworkPolicy = {+ metadata: {+ name: policyName,+ namespace: 'sandbox-ui',+ uid: 'uid-ui-egress',+ resourceVersion: '8',+ labels: {+ 'clerum.io/managed-by': 'workflow-recipes',+ 'clerum.io/recipe': 'test-recipe',+ },+ annotations: egressStateAnnotations([+ { fqdn: 'removed.example.com', ip: '93.184.216.10', port: 443 },+ { fqdn: 'current.example.com', ip: '93.184.216.20', port: 443 },+ ]),+ },+ spec: {+ podSelector: { matchLabels: { 'clerum.io/sandbox-ui': 'true' } },+ policyTypes: ['Egress'],+ egress: ['93.184.216.10/32', '93.184.216.20/32'].map(cidr => ({+ to: [{ ipBlock: { cidr } }],+ ports: [{ port: 443, protocol: 'TCP' }],+ })),+ },+ }+ mockNetworkingApi.readNamespacedNetworkPolicy.mockImplementation(+ ({ name }: { name: string }) =>+ name === policyName ? Promise.resolve(existing) : Promise.reject({ code: 404 })+ )+ const rec = new WorkflowRecipeReconciler(new k8s.KubeConfig(), undefined, {+ fqdnLookup: async () => {+ throw Object.assign(new Error('queryA EBADFAMILY'), { code: 'EBADFAMILY' })+ },+ })+ const result = await rec.reconcile(+ uiRecipeWithExternals([{ fqdn: 'current.example.com', port: 443 }])+ )++ expect(result.phase).toBe('failed')+ expect(result.message).toContain('EBADFAMILY')+ const replacement = mockNetworkingApi.replaceNamespacedNetworkPolicy.mock.calls.find(+ call => call[0]?.name === policyName+ )?.[0].body as k8s.V1NetworkPolicy+ const cidrs = (replacement?.spec?.egress ?? []).flatMap(rule =>+ (rule.to ?? []).map(peer => peer.ipBlock?.cidr)+ )+ if (replacement) {+ expect(cidrs).toEqual(['93.184.216.20/32'])+ expect(JSON.stringify(replacement.metadata?.annotations)).not.toContain('removed.example.com')+ } else {+ expect(mockNetworkingApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledWith(+ expect.objectContaining({ name: policyName, namespace: 'sandbox-ui' })+ )+ }+ })++ it('#567 remediation: a transient lookup cannot promote annotation-only UI egress', async () => {+ let transient = false+ const rec = new WorkflowRecipeReconciler(new k8s.KubeConfig(), undefined, {+ fqdnLookup: async () =>+ transient+ ? { kind: 'error', error: 'ESERVFAIL', retryable: true }+ : { kind: 'ok', ipv4: ['93.184.216.20'], ipv6: [], ttlSeconds: 300 },+ })+ const recipe = uiRecipeWithExternals([{ fqdn: 'current.example.com', port: 443 }])+ await rec.reconcile(recipe)+ const written = createdPolicy('ui-egress-test-recipe')!+ let liveWithoutRule: k8s.V1NetworkPolicy = {+ ...written,+ metadata: {+ ...written.metadata,+ uid: 'uid-ui-egress',+ resourceVersion: '10',+ },+ spec: { ...written.spec!, egress: [] },+ }+ mockNetworkingApi.readNamespacedNetworkPolicy.mockImplementation(+ ({ name }: { name: string }) =>+ name === 'ui-egress-test-recipe'+ ? Promise.resolve(liveWithoutRule)+ : Promise.reject({ code: 404 })+ )+ mockNetworkingApi.replaceNamespacedNetworkPolicy.mockImplementation(+ ({ name, body }: { name: string; body: k8s.V1NetworkPolicy }) => {+ if (name === 'ui-egress-test-recipe') {+ liveWithoutRule = {+ ...body,+ metadata: {+ ...body.metadata,+ name,+ namespace: 'sandbox-ui',+ uid: 'uid-ui-egress',+ resourceVersion: '11',+ },+ }+ return Promise.resolve(liveWithoutRule)+ }+ return Promise.resolve({})+ }+ )+ mockNetworkingApi.createNamespacedNetworkPolicy.mockClear()+ mockNetworkingApi.replaceNamespacedNetworkPolicy.mockClear()+ transient = true++ const result = await rec.reconcile(recipe)+ expect(result.phase).toBe('degraded')+ const writtenBodies = [+ ...mockNetworkingApi.createNamespacedNetworkPolicy.mock.calls.map(call => call[0]?.body),+ ...mockNetworkingApi.replaceNamespacedNetworkPolicy.mock.calls.map(call => call[0]?.body),+ ].filter(body => body?.metadata?.name === 'ui-egress-test-recipe') as k8s.V1NetworkPolicy[]+ for (const body of writtenBodies) {+ const cidrs = (body.spec?.egress ?? []).flatMap(rule =>+ (rule.to ?? []).map(peer => peer.ipBlock?.cidr)+ )+ expect(cidrs).not.toContain('93.184.216.20/32')+ expect(body.metadata?.annotations?.['clerum.io/egress-fqdn-state']).toBe('[]')+ }+ })++ it.each(['success', 'fault'] as const)(+ '#567 remediation: re-contracts a UI policy widened during DNS %s latency',+ async mode => {+ let secondRound = false+ let live: k8s.V1NetworkPolicy | undefined+ const rec = new WorkflowRecipeReconciler(new k8s.KubeConfig(), undefined, {+ fqdnLookup: async () => {+ if (secondRound && live) {+ live = structuredClone(live)+ live.spec!.egress!.push({+ to: [{ ipBlock: { cidr: '8.8.8.8/32' } }],+ ports: [{ port: 443, protocol: 'TCP' }],+ })+ if (mode === 'fault') {+ throw Object.assign(new Error('post-list EBADFAMILY'), { code: 'EBADFAMILY' })+ }+ }+ return { kind: 'ok', ipv4: ['93.184.216.20'], ipv6: [], ttlSeconds: 300 }+ },+ })+ const recipe = uiRecipeWithExternals([{ fqdn: 'current.example.com', port: 443 }])+ await rec.reconcile(recipe)+ const written = createdPolicy('ui-egress-test-recipe')!+ live = {+ ...structuredClone(written),+ metadata: {+ ...written.metadata,+ name: 'ui-egress-test-recipe',+ namespace: 'sandbox-ui',+ uid: 'uid-ui-dns-race',+ resourceVersion: '50',+ },+ }+ mockNetworkingApi.readNamespacedNetworkPolicy.mockImplementation(+ ({ name }: { name: string }) =>+ name === 'ui-egress-test-recipe' && live+ ? Promise.resolve(live)+ : Promise.reject({ code: 404 })+ )+ mockNetworkingApi.replaceNamespacedNetworkPolicy.mockImplementation(+ ({ body }: { body: k8s.V1NetworkPolicy }) => {+ live = {+ ...body,+ metadata: {+ ...body.metadata,+ name: 'ui-egress-test-recipe',+ namespace: 'sandbox-ui',+ uid: 'uid-ui-dns-race',+ resourceVersion: '51',+ },+ }+ return Promise.resolve(live)+ }+ )+ mockNetworkingApi.createNamespacedNetworkPolicy.mockClear()+ mockNetworkingApi.replaceNamespacedNetworkPolicy.mockClear()+ secondRound = true++ const result = await rec.reconcile(recipe)++ expect(result.phase).toBe(mode === 'fault' ? 'failed' : 'active')+ const replacements = mockNetworkingApi.replaceNamespacedNetworkPolicy.mock.calls+ .filter(call => call[0]?.name === 'ui-egress-test-recipe')+ .map(call => call[0]?.body as k8s.V1NetworkPolicy)+ expect(replacements.length).toBeGreaterThan(0)+ for (const replacement of replacements) {+ expect(JSON.stringify(replacement.spec?.egress)).not.toContain('8.8.8.8')+ }+ }+ )++ it('#567 remediation: persists the prior resolved-at when a permanent failure narrows the policy', async () => {+ let permanentFailure = false+ let live: k8s.V1NetworkPolicy | undefined+ const rec = new WorkflowRecipeReconciler(new k8s.KubeConfig(), undefined, {+ fqdnLookup: async () =>+ permanentFailure+ ? { kind: 'error', error: 'no A records', retryable: false }+ : { kind: 'ok', ipv4: ['93.184.216.20'], ipv6: [], ttlSeconds: 30 },+ })+ const recipe = makeRecipe({+ spec: {+ workloads: [+ { id: 'frontend', type: 'deployment', image: 'fe:1', port: 8080 },+ { id: 'backend', type: 'deployment', image: 'backend:1', port: 8080 },+ ],+ ui: {+ workloadRef: 'frontend',+ port: 8080,+ egress: {+ internal: [{ workloadRef: 'backend', port: 8080 }],+ external: [{ fqdn: 'api.example.com', port: 443 }],+ },+ },+ },+ })+ await rec.reconcile(recipe)+ const written = createdPolicy('ui-egress-test-recipe')!+ const oldResolvedAt = '2000-01-01T00:00:00.000Z'+ const state = JSON.parse(+ written.metadata!.annotations!['clerum.io/egress-fqdn-state']+ ) as Array<Record<string, unknown>>+ for (const entry of state) entry.expiresAt = Date.now() + 600_000+ live = {+ ...structuredClone(written),+ metadata: {+ ...written.metadata,+ name: 'ui-egress-test-recipe',+ namespace: 'sandbox-ui',+ uid: 'uid-ui-permanent-narrow',+ resourceVersion: '60',+ annotations: {+ ...written.metadata!.annotations,+ 'clerum.io/egress-fqdn-state': JSON.stringify(state),+ 'clerum.io/egress-fqdn-resolved-at': oldResolvedAt,+ },+ },+ }+ mockNetworkingApi.readNamespacedNetworkPolicy.mockImplementation(+ ({ name }: { name: string }) =>+ name === 'ui-egress-test-recipe' && live+ ? Promise.resolve(live)+ : Promise.reject({ code: 404 })+ )+ mockNetworkingApi.replaceNamespacedNetworkPolicy.mockImplementation(+ ({ name, body }: { name: string; body: k8s.V1NetworkPolicy }) => {+ if (name === 'ui-egress-test-recipe') {+ live = {+ ...body,+ metadata: {+ ...body.metadata,+ name,+ namespace: 'sandbox-ui',+ uid: 'uid-ui-permanent-narrow',+ resourceVersion: '61',+ },+ }+ return Promise.resolve(live)+ }+ return Promise.resolve({})+ }+ )+ mockNetworkingApi.createNamespacedNetworkPolicy.mockClear()+ mockNetworkingApi.replaceNamespacedNetworkPolicy.mockClear()+ permanentFailure = true++ const result = await rec.reconcile(recipe)++ expect(result.phase).toBe('failed')+ const replacement = mockNetworkingApi.replaceNamespacedNetworkPolicy.mock.calls+ .filter(call => call[0]?.name === 'ui-egress-test-recipe')+ .at(-1)?.[0].body as k8s.V1NetworkPolicy+ expect(replacement.metadata?.annotations?.['clerum.io/egress-fqdn-resolved-at']).toBe(+ oldResolvedAt+ )+ expect(replacement.metadata?.annotations?.['clerum.io/egress-fqdn-state']).toBe('[]')+ expect(JSON.stringify(replacement.spec?.egress)).not.toContain('93.184.216.20')+ })+it('records the ui lane egress set with the exact enforced cidrs and ports', async () => {const cap = captureEgressRecords()try {@@ -4320,7 +5002,12 @@ describe('WorkflowRecipeReconciler', () => {mockNetworkingApi.readNamespacedNetworkPolicy.mockResolvedValue({metadata: {name: 'ui-egress-test-recipe',+ uid: 'uid-ui-egress-test-recipe',resourceVersion: '1',+ labels: {+ 'clerum.io/managed-by': 'workflow-recipes',+ 'clerum.io/recipe': 'test-recipe',+ },annotations: { 'clerum.io/egress-fqdn-state': JSON.stringify(rehydrated) },},})@@ -4341,7 +5028,9 @@ describe('WorkflowRecipeReconciler', () => {expect(cap.payloads()).toHaveLength(1)const recorded = cap.payloads()[0].cidrs as string[]expect(recorded).not.toContain('10.0.0.5/32')- expect(recorded).toEqual(['93.184.216.10/32', '93.184.216.99/32'])+ // The prior public address existed only in the annotation, not in a live+ // rule. Contraction must not promote it during the next successful round.+ expect(recorded).toEqual(['93.184.216.99/32'])} finally {cap.restore()}@@ -4384,17 +5073,22 @@ describe('WorkflowRecipeReconciler', () => {rendered!.metadata!.annotations!['clerum.io/egress-fqdn-state']) as Array<Record<string, unknown>>const aging = persisted.map(e => ({ ...e, expiresAt: Date.now() + 30_000 }))- mockNetworkingApi.readNamespacedNetworkPolicy.mockResolvedValue({- ...rendered,- metadata: {- ...rendered!.metadata,- resourceVersion: '1',- annotations: {- ...rendered!.metadata!.annotations,- 'clerum.io/egress-fqdn-state': JSON.stringify(aging),- },- },- })+ mockNetworkingApi.readNamespacedNetworkPolicy.mockImplementation(+ ({ name }: { name: string }) =>+ name === 'ui-egress-test-recipe'+ ? Promise.resolve({+ ...rendered,+ metadata: {+ ...rendered!.metadata,+ resourceVersion: '1',+ annotations: {+ ...rendered!.metadata!.annotations,+ 'clerum.io/egress-fqdn-state': JSON.stringify(aging),+ },+ },+ })+ : Promise.reject({ code: 404 })+ )const cap = captureEgressRecords()const noOpSpy = vi.spyOn(console, 'log').mockImplementation(() => {})@@ -4435,22 +5129,33 @@ describe('WorkflowRecipeReconciler', () => {// below does — no hand-built fixture.const created = createdPolicy('ui-egress-test-recipe')expect(created).toBeDefined()- mockNetworkingApi.readNamespacedNetworkPolicy.mockResolvedValue(created!)+ mockNetworkingApi.readNamespacedNetworkPolicy.mockResolvedValue({+ ...created!,+ metadata: {+ ...created!.metadata,+ uid: 'uid-ui-egress-test-recipe',+ resourceVersion: '1',+ },+ })const cap = captureEgressRecords()- const noOpSpy = vi.spyOn(console, 'log').mockImplementation(() => {})try {+ mockNetworkingApi.createNamespacedNetworkPolicy.mockClear()+ mockNetworkingApi.replaceNamespacedNetworkPolicy.mockClear()await rec.reconcile(recipe)expect(cap.payloads()).toHaveLength(0)- // POSITIVE CONTROL. Without it this assertion also passes when the second- // reconcile never reached the egress path at all, which would make the- // test a tautology rather than a check of the change gate.- expect(noOpSpy).toHaveBeenCalledWith(- expect.stringContaining('ui-egress-test-recipe" in sandbox-ui egress set unchanged — no-op')- )+ expect(+ mockNetworkingApi.createNamespacedNetworkPolicy.mock.calls.some(+ call => call[0]?.body?.metadata?.name === 'ui-egress-test-recipe'+ )+ ).toBe(false)+ expect(+ mockNetworkingApi.replaceNamespacedNetworkPolicy.mock.calls.some(+ call => call[0]?.name === 'ui-egress-test-recipe'+ )+ ).toBe(false)} finally {- noOpSpy.mockRestore()cap.restore()}})@@ -4488,7 +5193,16 @@ describe('WorkflowRecipeReconciler', () => {// That policy is now live; the next reconcile renames the FQDN onto the SAME IP.mockNetworkingApi.readNamespacedNetworkPolicy.mockImplementation(({ name }: { name: string }) =>Promise.resolve(- name === 'ui-egress-test-recipe' ? p1 : { metadata: { name, resourceVersion: '1' } }+ name === 'ui-egress-test-recipe'+ ? {+ ...p1,+ metadata: {+ ...p1.metadata,+ uid: 'uid-ui-egress-test-recipe',+ resourceVersion: '1',+ },+ }+ : { metadata: { name, resourceVersion: '1' } }))mockNetworkingApi.createNamespacedNetworkPolicy.mockClear()@@ -4828,6 +5542,19 @@ describe('WorkflowRecipeReconciler', () => {})it('R.8.24 — no ui-egress NetworkPolicy is created when spec.ui is unset', async () => {+ mockNetworkingApi.readNamespacedNetworkPolicy.mockResolvedValueOnce({+ metadata: {+ name: 'ui-egress-test-recipe',+ namespace: 'sandbox-ui',+ uid: 'uid-ui-egress-test-recipe',+ resourceVersion: '7',+ labels: {+ 'clerum.io/managed-by': 'workflow-recipes',+ 'clerum.io/recipe': 'test-recipe',+ },+ },+ spec: { podSelector: {}, policyTypes: ['Egress'], egress: [] },+ })const recipe = makeRecipe({spec: {workloads: [{ id: 'app', type: 'deployment', image: 'app:1', port: 8080 }],@@ -4835,8 +5562,8 @@ describe('WorkflowRecipeReconciler', () => {})await reconciler.reconcile(recipe)expect(mockNetworkingApi.createNamespacedNetworkPolicy).not.toHaveBeenCalled()- // Convergence path: stale policy from a prior reconcile (when ui was set)- // is removed unconditionally. safeDelete swallows the 404 if none exists.+ // Convergence path: an owned stale policy from a prior reconcile is removed+ // with live identity preconditions.expect(mockNetworkingApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledWith(expect.objectContaining({ name: 'ui-egress-test-recipe', namespace: 'sandbox-ui' }))diff --git a/workflow-recipes/src/reconciler/workflowRecipeReconciler.ts b/workflow-recipes/src/reconciler/workflowRecipeReconciler.tsindex 790bee8f8..124843d72 100644--- a/workflow-recipes/src/reconciler/workflowRecipeReconciler.ts+++ b/workflow-recipes/src/reconciler/workflowRecipeReconciler.ts@@ -14,7 +14,13 @@* 11. Handle delete (reverse dependency order, skip PVCs)*/import * as k8s from '@kubernetes/client-node'-import { STATE_ANNOTATION } from '@clerum/network-policy-core'+import {+ type EgressEntry,+ RESOLVED_AT_ANNOTATION,+ STATE_ANNOTATION,+ TARGETS_ANNOTATION,+ serializeState,+} from '@clerum/network-policy-core'import { OperatorConfig, loadConfig } from '../config'import { getPool } from '../db'import { createLogger } from '../observability/logger'@@ -58,7 +64,11 @@ import {import { evaluateComputedValues } from './computedValuesEvaluator'import { CRD_GROUP, CRD_VERSION, WORKFLOWRECIPE_PLURAL } from './crdConstants'import { sort as sortDependencies } from './dependencyGraph'-import { type AccumulateOutput, accumulateExternalEgress } from './externalEgressAccumulator'+import {+ type AccumulateOutput,+ accumulateExternalEgress,+ contractExternalEgress,+} from './externalEgressAccumulator'import {type FqdnLookup,defaultFqdnLookup,@@ -134,11 +144,11 @@ function externalEgressReadinessError(): Error {const pending = result.pending.map(name => `${name}: pending`)const failed = result.failed.map(({ name, message }) => `${name}: ${message}`)- return new Error(+ const message =`External egress policy readiness not achieved for WorkflowRecipe "${recipeName}". ` +- `${[...pending, ...failed].join('; ')}. ` +- `External egress bindings are not enforceable until HCC reports ExternalEgressReady=True.`- )+ `${[...pending, ...failed].join('; ')}. ` ++ `External egress bindings are not enforceable until HCC reports ExternalEgressReady=True.`+ return result.failed.length === 0 ? new RetryableReconcileError(message) : new Error(message)}function clusterNetworkPolicyEnforcementError(recipeName: string): Error {@@ -720,6 +730,110 @@ export function egressSignature(policy: k8s.V1NetworkPolicy): string {)}+function networkPolicySpecSignature(policy: k8s.V1NetworkPolicy): string {+ return JSON.stringify(canonicalize(policy.spec ?? {}))+}++function contractionWriteNeeded(+ existing: k8s.V1NetworkPolicy,+ desired: k8s.V1NetworkPolicy+): boolean {+ if (networkPolicySpecSignature(existing) !== networkPolicySpecSignature(desired)) return true+ const currentAnnotations = existing.metadata?.annotations ?? {}+ const desiredAnnotations = desired.metadata?.annotations ?? {}+ return [STATE_ANNOTATION, TARGETS_ANNOTATION, RESOLVED_AT_ANNOTATION].some(+ key => currentAnnotations[key] !== desiredAnnotations[key]+ )+}++function intersectRules<T>(existing: T[] | undefined, desired: T[] | undefined): T[] {+ const available = new Map<string, number>()+ for (const rule of existing ?? []) {+ const signature = JSON.stringify(canonicalize(rule))+ available.set(signature, (available.get(signature) ?? 0) + 1)+ }+ return (desired ?? []).filter(rule => {+ const signature = JSON.stringify(canonicalize(rule))+ const remaining = available.get(signature) ?? 0+ if (remaining === 0) return false+ available.set(signature, remaining - 1)+ return true+ })+}++/**+ * Intersect a desired policy with the rules already enforced by the live+ * snapshot. Used only by the pre-DNS contraction phase: it may remove rules but+ * can never add one. Selector/policy-type drift cannot be proven subtractive,+ * so the caller revokes the policy instead.+ */+function subtractiveNetworkPolicy(+ existing: k8s.V1NetworkPolicy,+ desired: k8s.V1NetworkPolicy | null+): k8s.V1NetworkPolicy | null {+ if (!desired) return null+ const existingSelector = JSON.stringify(canonicalize(existing.spec?.podSelector ?? {}))+ const desiredSelector = JSON.stringify(canonicalize(desired.spec?.podSelector ?? {}))+ const existingTypes = JSON.stringify([...(existing.spec?.policyTypes ?? [])].sort())+ const desiredTypes = JSON.stringify([...(desired.spec?.policyTypes ?? [])].sort())+ if (existingSelector !== desiredSelector || existingTypes !== desiredTypes) return null++ const contractedSpec: k8s.V1NetworkPolicySpec = { ...desired.spec! }+ if (desired.spec?.egress !== undefined) {+ contractedSpec.egress = intersectRules(existing.spec?.egress, desired.spec.egress)+ }+ if (desired.spec?.ingress !== undefined) {+ contractedSpec.ingress = intersectRules(existing.spec?.ingress, desired.spec.ingress)+ }+ const contracted: k8s.V1NetworkPolicy = { ...desired, spec: contractedSpec }++ // State is authorization provenance for later fail-static rounds. Keep only+ // entries represented by a retained live ipBlock+port rule so a transient DNS+ // failure cannot re-add an annotation-only destination that was not enforced.+ const rawState = contracted.metadata?.annotations?.[STATE_ANNOTATION]+ if (rawState) {+ const livePairs = new Set<string>()+ for (const rule of contractedSpec.egress ?? []) {+ for (const peer of rule.to ?? []) {+ const cidr = peer.ipBlock?.cidr+ if (!cidr?.endsWith('/32')) continue+ for (const port of rule.ports ?? []) {+ livePairs.add(+ `${cidr.slice(0, -3)}\u0000${String(port.port)}\u0000${port.protocol ?? 'TCP'}`+ )+ }+ }+ }+ const entries = (JSON.parse(rawState) as EgressEntry[]).filter(entry =>+ livePairs.has(`${entry.ip}\u0000${entry.port}\u0000${entry.protocol}`)+ )+ const priorResolvedAt = contracted.metadata?.annotations?.[RESOLVED_AT_ANNOTATION]+ contracted.metadata = {+ ...(contracted.metadata ?? {}),+ annotations: {+ ...(contracted.metadata?.annotations ?? {}),+ ...serializeState({ entries }),+ ...(priorResolvedAt ? { [RESOLVED_AT_ANNOTATION]: priorResolvedAt } : {}),+ },+ }+ }+ return contracted+}++interface WorkloadEgressContractionPlan {+ plans: Array<{+ workload: WorkloadDef+ namespace: string+ policyName: string+ externalDeclared: Array<{ fqdn: string; port: number }>+ }>+ ingressSourcesByTarget: Map<string, rb.WorkloadIngressSource[]>+}++interface RecipeEgressContractionPlan {+ workload: WorkloadEgressContractionPlan+}+export class WorkflowRecipeReconciler {private appsApi: k8s.AppsV1Apiprivate batchApi: k8s.BatchV1Api@@ -2486,6 +2600,7 @@ export class WorkflowRecipeReconciler {// Step 9b: Sandbox UI egress policy + symmetric ingress on each// internal-egress target (without the ingress side the target stays// behind deny-all-<ns> and UI→backend traffic is dropped).+ const externalEgressPlan = await this.prepareRecipeEgressContractions(recipe)await this.reconcileUiEgressPolicy(recipe)await this.reconcileUiIngressPolicies(recipe)// Egress for background-OAuth workloads → control-api broker route.@@ -2497,7 +2612,7 @@ export class WorkflowRecipeReconciler {// any sibling targets it points at. MCP workloads inherit egress// through HCC via the McpServer CRD path; UI workloads use// `spec.ui.egress.*` which is handled by reconcileUiEgressPolicy.- await this.reconcileWorkloadEgressPolicies(recipe)+ await this.reconcileWorkloadEgressPolicies(recipe, externalEgressPlan.workload)// Step 9d: WRC-owned intra-recipe internal dependency policies.// This lane is separate from HCC binding-allow and legacy wl-* policies.@@ -3316,23 +3431,17 @@ export class WorkflowRecipeReconciler {// Sandbox UI egress policy + symmetric ingress on internal-egress// targets (no-op when spec.ui is unset; idempotent)try {+ const externalEgressPlan = await this.prepareRecipeEgressContractions(recipe)await this.reconcileUiEgressPolicy(recipe)await this.reconcileUiIngressPolicies(recipe)await this.reconcileOAuthBrokerEgressPolicy(recipe)- } catch (err) {- console.error(- `[WR-Reconciler] UI egress/ingress policies failed for workflow "${name}":`,- err- )- throw err- }- // Per-workload egressBindings → NetworkPolicies (idempotent; no-op when- // no workload declares egressBindings).- try {- await this.reconcileWorkloadEgressPolicies(recipe)+ // Per-workload egressBindings → NetworkPolicies (idempotent; no-op when+ // no workload declares egressBindings). Its subtractive phase already ran+ // recipe-wide above, before the first DNS lookup.+ await this.reconcileWorkloadEgressPolicies(recipe, externalEgressPlan.workload)} catch (err) {- console.error(`[WR-Reconciler] Workload egress policies failed for workflow "${name}":`, err)+ createLogger('wrc', name).error('recipe network policy reconciliation failed', { err })throw err}@@ -4092,36 +4201,97 @@ export class WorkflowRecipeReconciler {}/**- * Reconcile the per-recipe `ui-egress-<recipeName>` NetworkPolicy in- * `sandboxUiNamespace`. Ensures the policy converges with `recipe.spec.ui`:- * - spec.ui set → create-or-replace the policy- * - spec.ui unset → delete any leftover policy from a prior reconcile- *- * The policy lives in a different namespace from the WorkflowRecipe CRD, so- * it carries no ownerReference and relies on this method (and reconcileDelete)- * for cleanup.+ * One recipe-wide subtractive barrier. Every WRC policy whose authorization+ * derives from UI/workload egress intent is narrowed before any resolver call+ * in any lane. Later phases may add current intent only after their own lookup+ * succeeds; a UI fault can therefore no longer strand a removed workload or+ * symmetric-ingress permission.+ */+ private async prepareRecipeEgressContractions(+ recipe: WorkflowRecipeCRD+ ): Promise<RecipeEgressContractionPlan> {+ await this.contractUiEgressPolicy(recipe)+ const workload = await this.contractWorkloadEgressPolicies(recipe)+ await this.contractUiIngressPolicies(recipe)+ await this.contractWorkloadIngressPolicies(recipe, workload.ingressSourcesByTarget)+ return { workload }+ }++ private async contractUiEgressPolicy(+ recipe: WorkflowRecipeCRD+ ): Promise<k8s.V1NetworkPolicy | null> {+ const namespace = this.config.sandboxUiNamespace+ const policyName = `ui-egress-${recipe.metadata.name}`+ const existing = await this.readNetworkPolicyOrNull(policyName, namespace)+ if (!existing) return null++ const externals = recipe.spec.ui?.egress?.external ?? []+ const declarations = externals.map(entry => ({ fqdn: entry.fqdn, port: entry.port }))+ const contraction = contractExternalEgress({+ externals: declarations,+ previousAnnotations: existing.metadata?.annotations,+ now: Date.now(),+ config: {+ overlapMs: this.config.externalEgressOverlapSeconds * 1000,+ maxEntries: this.config.externalEgressMaxEntries,+ },+ isAllowedIp: ip => !isBlockedExternalIPv4(ip),+ })+ const needsStateAnnotations =+ declarations.length > 0 ||+ contraction.entries.length > 0 ||+ existing.metadata?.annotations?.[STATE_ANNOTATION] !== undefined+ const desired = rb.buildUiEgressNetworkPolicy(+ recipe,+ namespace,+ this.config.sandboxNamespace,+ contraction.resolved,+ needsStateAnnotations ? contraction.annotations : undefined+ )+ return this.applyNetworkPolicyContraction({+ existing,+ desired: subtractiveNetworkPolicy(existing, desired),+ namespace,+ recipeName: recipe.metadata.name,+ })+ }++ /**+ * Reconcile the additive/fail-static half of the per-recipe UI policy. The+ * caller supplies the snapshot returned by the recipe-wide contraction pass,+ * which has already narrowed UI, workload, and symmetric ingress policies+ * before the first DNS lookup can fail.*/private async reconcileUiEgressPolicy(recipe: WorkflowRecipeCRD): Promise<void> {const policyName = `ui-egress-${recipe.metadata.name}`const ns = this.config.sandboxUiNamespaceconst externals = recipe.spec.ui?.egress?.external ?? []- const { resolved, failures } = await resolveExternalEgress(externals, this.fqdnLookup)+ const externalDeclarations = externals.map(e => ({ fqdn: e.fqdn, port: e.port }))+ const egressConfig = {+ overlapMs: this.config.externalEgressOverlapSeconds * 1000,+ maxEntries: this.config.externalEgressMaxEntries,+ }++ let resolution: Awaited<ReturnType<typeof resolveExternalEgress>>+ try {+ resolution = await resolveExternalEgress(externals, this.fqdnLookup)+ } catch (error: unknown) {+ // DNS/controller latency is a race window. Re-contract from a fresh read+ // before propagating so a concurrent widening cannot survive merely+ // because the lookup itself faulted.+ await this.contractUiEgressPolicy(recipe)+ throw error+ }+ const existing = await this.contractUiEgressPolicy(recipe)+ const { resolved, failures } = resolutionthis.recordExternalEgressTtl(resolved)- // Permanent failures (no-records / blocked range) never author a partial- // policy — fail exactly as the single-snapshot resolver did.const permanentFailures = failures.filter(f => !f.retryable)- if (permanentFailures.length > 0) {- throw egressResolutionError(- `WorkflowRecipe "${recipe.metadata.name}" ui external egress resolution failed`,- permanentFailures- )- }+ let blockingFailures = permanentFailureslet effectiveResolved: rb.ResolvedExternalEgressInput[] = resolvedlet stateAnnotations: Record<string, string> | undefined- let existing: k8s.V1NetworkPolicy | null = nulllet egressRenewalDue = false// H-E: the write gate compares rendered spec.egress (ipBlock cidr+port, no// fqdn), so a rename old.example.com→new.example.com onto the SAME ip/port is@@ -4129,32 +4299,20 @@ export class WorkflowRecipeReconciler {// annotation and losing the overlap grace when the new name later rotates.// acc.changed is over (fqdn,ip,port,protocol), so it catches the rename.let egressStateChanged = false- // R1-M2: read the live policy for ALL cases, not only external egress, so an- // internal-only sibling policy (ui.egress.internal[] with no external[]) can- // hit the no-op gate below instead of being rewritten on every reconcile —- // which the 60s external-egress refresh loop amplifies for mixed recipes. For- // external egress it also seeds the accumulator's rehydration (H5).- existing = await this.readNetworkPolicyOrNull(policyName, ns)if (externals.length > 0) {// issue #299: fold this DNS snapshot into the accumulated sliding-window// set persisted on the live policy's annotations (rehydrate H5).const acc = accumulateExternalEgress({- externals: externals.map(e => ({ fqdn: e.fqdn, port: e.port })),+ externals: externalDeclarations,resolveResult: { resolved, failures },previousAnnotations: existing?.metadata?.annotations,now: Date.now(),- config: {- overlapMs: this.config.externalEgressOverlapSeconds * 1000,- maxEntries: this.config.externalEgressMaxEntries,- },+ config: egressConfig,})// Bootstrap fail-closed: a transient resolver failure with NOTHING to// freeze (no rehydratable prior set) must not author an empty policy.if (acc.entries.length === 0 && failures.length > 0) {- throw egressResolutionError(- `WorkflowRecipe "${recipe.metadata.name}" ui external egress resolution failed`,- failures- )+ blockingFailures = failures}// Audit L3: a transient failure of a newly-added FQDN (with resolving// siblings) does not throw and freezes nothing — surface it so the missing@@ -4190,11 +4348,30 @@ export class WorkflowRecipeReconciler {stateAnnotations)- if (!policy) {- await this.safeDelete(- () => this.networkingApi.deleteNamespacedNetworkPolicy({ name: policyName, namespace: ns }),- `NetworkPolicy "${policyName}" in ${ns}`+ if (blockingFailures.length > 0) {+ if (existing) {+ await this.applyNetworkPolicyContraction({+ existing,+ desired: subtractiveNetworkPolicy(existing, policy),+ namespace: ns,+ recipeName: recipe.metadata.name,+ })+ }+ throw egressResolutionError(+ `WorkflowRecipe "${recipe.metadata.name}" ui external egress resolution failed`,+ blockingFailures)+ }++ if (!policy) {+ if (existing) {+ await this.applyNetworkPolicyContraction({+ existing,+ desired: null,+ namespace: ns,+ recipeName: recipe.metadata.name,+ })+ }return}@@ -4206,37 +4383,46 @@ export class WorkflowRecipeReconciler {console.log(`[WR-Reconciler] NetworkPolicy "${policyName}" in ${ns} egress set unchanged — no-op`)- return+ } else {+ await this.applyOwnedRecipeNetworkPolicy(policy, ns, recipe.metadata.name)+ // #299: the policy has landed — record the set it actually enforces.+ this.logResolvedEgressSet(+ recipe.metadata.name,+ policyName,+ externals.map(e => e.fqdn),+ effectiveResolved,+ egressStateChanged+ )}+ }- await this.createOrReplace(- () =>- this.networkingApi.createNamespacedNetworkPolicy({- namespace: ns,- body: policy,- }),- async () => {- const existing = await this.networkingApi.readNamespacedNetworkPolicy({- name: policyName,- namespace: ns,- })- policy.metadata!.resourceVersion = existing.metadata?.resourceVersion- return this.networkingApi.replaceNamespacedNetworkPolicy({- name: policyName,- namespace: ns,- body: policy,- })- },- `NetworkPolicy "${policyName}" in ${ns}`- )- // #299: the policy has landed — record the set it actually enforces.- this.logResolvedEgressSet(- recipe.metadata.name,- policyName,- externals.map(e => e.fqdn),- effectiveResolved,- egressStateChanged- )+ private async contractUiIngressPolicies(recipe: WorkflowRecipeCRD): Promise<void> {+ const namespace = this.config.sandboxNamespace+ const portsByWorkload = new Map<string, Set<number>>()+ for (const rule of recipe.spec.ui?.egress?.internal ?? []) {+ const ports = portsByWorkload.get(rule.workloadRef) ?? new Set<number>()+ ports.add(rule.port)+ portsByWorkload.set(rule.workloadRef, ports)+ }++ for (const workload of recipe.spec.workloads ?? []) {+ const name = rb.uiIngressPolicyName(recipe.metadata.name, workload.id)+ const existing = await this.readNetworkPolicyOrNull(name, namespace)+ if (!existing) continue+ const desired = rb.buildUiIngressNetworkPolicy(+ recipe,+ workload.id,+ [...(portsByWorkload.get(workload.id) ?? [])],+ namespace,+ this.config.sandboxUiNamespace+ )+ await this.applyNetworkPolicyContraction({+ existing,+ desired: subtractiveNetworkPolicy(existing, desired),+ namespace,+ recipeName: recipe.metadata.name,+ })+ }}/**@@ -4280,7 +4466,7 @@ export class WorkflowRecipeReconciler {await this.deleteUiIngressIfExists(recipe.metadata.name, w.id, ns)continue}- await this.applyNetworkPolicy(policy, ns)+ await this.applyOwnedRecipeNetworkPolicy(policy, ns, recipe.metadata.name)}}@@ -4290,10 +4476,14 @@ export class WorkflowRecipeReconciler {namespace: string): Promise<void> {const name = rb.uiIngressPolicyName(recipeName, workloadId)- await this.safeDelete(- () => this.networkingApi.deleteNamespacedNetworkPolicy({ name, namespace }),- `NetworkPolicy "${name}" in ${namespace}`- )+ const existing = await this.readNetworkPolicyOrNull(name, namespace)+ if (!existing) return+ await this.applyNetworkPolicyContraction({+ existing,+ desired: null,+ namespace,+ recipeName,+ })}private internalDependencyCondition(@@ -4515,6 +4705,117 @@ export class WorkflowRecipeReconciler {}}+ private async contractWorkloadEgressPolicies(+ recipe: WorkflowRecipeCRD+ ): Promise<WorkloadEgressContractionPlan> {+ const workloads = recipe.spec.workloads ?? []+ const uiWorkloadId = recipe.spec.ui?.workloadRef+ const ingressSourcesByTarget = new Map<string, rb.WorkloadIngressSource[]>()+ const plans: WorkloadEgressContractionPlan['plans'] = []++ for (const workload of workloads) {+ if (workload.transport) continue+ if (uiWorkloadId && workload.id === uiWorkloadId) continue+ const namespace = this.resolveWorkloadNamespace(workload, uiWorkloadId)+ const bindings = workload.egressBindings ?? []+ if (bindings.length === 0) {+ await this.deleteWorkloadEgressIfExists(recipe.metadata.name, workload.id, namespace)+ continue+ }++ const externalDeclared: Array<{ fqdn: string; port: number }> = []+ for (const binding of bindings) {+ if (!binding.dns || binding.port == null) continue+ const resolved = rb.resolveClusterLocalBinding(binding.dns, recipe, namespace)+ if (resolved?.kind === 'cluster-local') {+ const sources = ingressSourcesByTarget.get(resolved.workloadId) ?? []+ sources.push({+ fromWorkloadId: workload.id,+ fromNamespace: namespace,+ port: binding.port,+ protocol: (binding.protocol ?? 'TCP') as 'TCP' | 'UDP',+ })+ ingressSourcesByTarget.set(resolved.workloadId, sources)+ } else if (!resolved) {+ externalDeclared.push({ fqdn: binding.dns, port: binding.port })+ }+ }++ const policyName = rb.workloadEgressPolicyName(recipe.metadata.name, workload.id)+ await this.contractOneWorkloadEgressPolicy(recipe, workload, namespace, externalDeclared)+ plans.push({ workload, namespace, policyName, externalDeclared })+ }++ return { plans, ingressSourcesByTarget }+ }++ private async contractOneWorkloadEgressPolicy(+ recipe: WorkflowRecipeCRD,+ workload: WorkloadDef,+ namespace: string,+ externalDeclared: Array<{ fqdn: string; port: number }>+ ): Promise<k8s.V1NetworkPolicy | null> {+ const policyName = rb.workloadEgressPolicyName(recipe.metadata.name, workload.id)+ const existing = await this.readNetworkPolicyOrNull(policyName, namespace)+ if (!existing) return null++ const contraction = contractExternalEgress({+ externals: externalDeclared,+ previousAnnotations: existing.metadata?.annotations,+ now: Date.now(),+ config: {+ overlapMs: this.config.externalEgressOverlapSeconds * 1000,+ maxEntries: this.config.externalEgressMaxEntries,+ },+ isAllowedIp: ip => !isBlockedExternalIPv4(ip),+ })+ const needsStateAnnotations =+ externalDeclared.length > 0 ||+ contraction.entries.length > 0 ||+ existing.metadata?.annotations?.[STATE_ANNOTATION] !== undefined+ const desired = rb.buildWorkloadEgressNetworkPolicy(+ workload,+ recipe,+ namespace,+ contraction.resolved,+ needsStateAnnotations ? contraction.annotations : undefined+ )+ return this.applyNetworkPolicyContraction({+ existing,+ desired: subtractiveNetworkPolicy(existing, desired),+ namespace,+ recipeName: recipe.metadata.name,+ })+ }++ private async contractWorkloadIngressPolicies(+ recipe: WorkflowRecipeCRD,+ ingressSourcesByTarget: Map<string, rb.WorkloadIngressSource[]>+ ): Promise<void> {+ const workloads = recipe.spec.workloads ?? []+ const uiWorkloadId = recipe.spec.ui?.workloadRef+ for (const workload of workloads) {+ if (workload.transport) continue+ if (uiWorkloadId && workload.id === uiWorkloadId) continue+ const namespace = this.resolveWorkloadNamespace(workload, uiWorkloadId)+ const name = rb.workloadIngressPolicyName(recipe.metadata.name, workload.id)+ const existing = await this.readNetworkPolicyOrNull(name, namespace)+ if (!existing) continue+ const desired = rb.buildWorkloadIngressNetworkPolicy(+ workload,+ recipe,+ namespace,+ ingressSourcesByTarget.get(workload.id) ?? []+ )+ await this.applyNetworkPolicyContraction({+ existing,+ desired: subtractiveNetworkPolicy(existing, desired),+ namespace,+ recipeName: recipe.metadata.name,+ })+ }+ }+/*** Reconcile per-workload egress NetworkPolicies for each workload that* declares `egressBindings[]`. Also emits the symmetric ingress policy@@ -4530,80 +4831,45 @@ export class WorkflowRecipeReconciler {* silent authorization drift, so any unresolved or blocked hostname fails* the reconcile before policy apply.*/- private async reconcileWorkloadEgressPolicies(recipe: WorkflowRecipeCRD): Promise<void> {+ private async reconcileWorkloadEgressPolicies(+ recipe: WorkflowRecipeCRD,+ preparation: WorkloadEgressContractionPlan+ ): Promise<void> {const workloads = recipe.spec.workloads ?? []if (workloads.length === 0) returnconst uiWorkloadId = recipe.spec.ui?.workloadRef+ const { plans, ingressSourcesByTarget } = preparation- // Aggregate ingress sources keyed by target workload id. Filled as we- // walk each source workload's cluster-local bindings.- const ingressSourcesByTarget = new Map<string, rb.WorkloadIngressSource[]>()-- for (const w of workloads) {- // Skip MCP + UI workloads — covered by other code paths.- if (w.transport) continue- if (uiWorkloadId && w.id === uiWorkloadId) continue- const wlNs = this.resolveWorkloadNamespace(w, uiWorkloadId)- const bindings = w.egressBindings ?? []- if (bindings.length === 0) {- // Reap any policy left over from a previous reconcile that DID- // have bindings — keeps the policy set in lockstep with the spec.- await this.deleteWorkloadEgressIfExists(recipe.metadata.name, w.id, wlNs)- continue- }-- // Split into cluster-local sibling targets vs external FQDNs.- const externalDeclared: { fqdn: string; port: number }[] = []- for (const b of bindings) {- if (!b.dns || b.port == null) continue- const port = b.port- const resolved = rb.resolveClusterLocalBinding(b.dns, recipe, wlNs)- if (resolved && resolved.kind === 'cluster-local') {- const sources = ingressSourcesByTarget.get(resolved.workloadId) ?? []- sources.push({- fromWorkloadId: w.id,- fromNamespace: wlNs,- port,- protocol: (b.protocol ?? 'TCP') as 'TCP' | 'UDP',- })- ingressSourcesByTarget.set(resolved.workloadId, sources)- } else if (!resolved) {- // null = treat as external FQDN- externalDeclared.push({ fqdn: b.dns, port })- }- // resolved.kind === 'mismatch' is unreachable here — validation- // ran upstream and would have thrown.+ // Phase 2: resolve and add/freeze only the identities that survived phase 1.+ for (const plan of plans) {+ const { workload: w, namespace: wlNs, policyName: wlPolicyName, externalDeclared } = plan+ let resolution: Awaited<ReturnType<typeof resolveExternalEgress>>+ try {+ resolution = await resolveExternalEgress(externalDeclared, this.fqdnLookup)+ } catch (error: unknown) {+ await this.contractOneWorkloadEgressPolicy(recipe, w, wlNs, externalDeclared)+ throw error}-- const { resolved: resolvedExternal, failures } = await resolveExternalEgress(- externalDeclared.map(e => ({ fqdn: e.fqdn, port: e.port })),- this.fqdnLookup+ let existingWlPolicy = await this.contractOneWorkloadEgressPolicy(+ recipe,+ w,+ wlNs,+ externalDeclared)+ const { resolved: resolvedExternal, failures } = resolutionthis.recordExternalEgressTtl(resolvedExternal)- // Permanent failures never author a partial policy — fail as before.const permanentFailures = failures.filter(f => !f.retryable)- if (permanentFailures.length > 0) {- throw egressResolutionError(- `WorkflowRecipe "${recipe.metadata.name}" workload "${w.id}" egress resolution failed`,- permanentFailures- )- }+ let blockingFailures = permanentFailureslet effectiveExternal: rb.ResolvedExternalEgressInput[] = resolvedExternallet wlStateAnnotations: Record<string, string> | undefined- let existingWlPolicy: k8s.V1NetworkPolicy | null = nulllet wlEgressRenewalDue = falselet wlEgressStateChanged = false // H-E: catch fqdn-attribution-only changes- const wlPolicyName = rb.workloadEgressPolicyName(recipe.metadata.name, w.id)- // R1-M2: read the live policy for ALL cases so an internal-only (cluster-- // local) workload egress policy hits the no-op gate instead of churning- // every reconcile; for external egress it also seeds rehydration (H5).- existingWlPolicy = await this.readNetworkPolicyOrNull(wlPolicyName, wlNs)if (externalDeclared.length > 0) {// issue #299: accumulate the sliding-window egress set (rehydrate H5).const acc = accumulateExternalEgress({- externals: externalDeclared.map(e => ({ fqdn: e.fqdn, port: e.port })),+ externals: externalDeclared,resolveResult: { resolved: resolvedExternal, failures },previousAnnotations: existingWlPolicy?.metadata?.annotations,now: Date.now(),@@ -4614,10 +4880,7 @@ export class WorkflowRecipeReconciler {})// Bootstrap fail-closed: transient failure with nothing to freeze.if (acc.entries.length === 0 && failures.length > 0) {- throw egressResolutionError(- `WorkflowRecipe "${recipe.metadata.name}" workload "${w.id}" egress resolution failed`,- failures- )+ blockingFailures = failures}this.warnEgressAccumulator(wlPolicyName, acc)// Defense-in-depth (audit M3): re-validate rehydrated IPs vs blocked ranges.@@ -4641,8 +4904,30 @@ export class WorkflowRecipeReconciler {effectiveExternal,wlStateAnnotations)+ if (blockingFailures.length > 0) {+ if (existingWlPolicy) {+ await this.applyNetworkPolicyContraction({+ existing: existingWlPolicy,+ desired: subtractiveNetworkPolicy(existingWlPolicy, policy),+ namespace: wlNs,+ recipeName: recipe.metadata.name,+ })+ }+ throw egressResolutionError(+ `WorkflowRecipe "${recipe.metadata.name}" workload "${w.id}" egress resolution failed`,+ blockingFailures+ )+ }if (!policy) {- await this.deleteWorkloadEgressIfExists(recipe.metadata.name, w.id, wlNs)+ if (existingWlPolicy) {+ await this.applyNetworkPolicyContraction({+ existing: existingWlPolicy,+ desired: null,+ namespace: wlNs,+ recipeName: recipe.metadata.name,+ })+ existingWlPolicy = null+ }continue}// issue #299: NO-OP when the accumulated set and rules are already live —@@ -4655,17 +4940,17 @@ export class WorkflowRecipeReconciler {console.log(`[WR-Reconciler] NetworkPolicy "${wlPolicyName}" in ${wlNs} egress set unchanged — no-op`)- continue+ } else {+ await this.applyOwnedRecipeNetworkPolicy(policy, wlNs, recipe.metadata.name)+ // #299: the policy has landed — record the set it actually enforces.+ this.logResolvedEgressSet(+ recipe.metadata.name,+ wlPolicyName,+ externalDeclared.map(e => e.fqdn),+ effectiveExternal,+ wlEgressStateChanged+ )}- await this.applyNetworkPolicy(policy, wlNs)- // #299: the policy has landed — record the set it actually enforces.- this.logResolvedEgressSet(- recipe.metadata.name,- wlPolicyName,- externalDeclared.map(e => e.fqdn),- effectiveExternal,- wlEgressStateChanged- )}// Now ingress side: for each target workload that any sibling pointed@@ -4684,7 +4969,7 @@ export class WorkflowRecipeReconciler {await this.deleteWorkloadIngressIfExists(recipe.metadata.name, w.id, wlNs)continue}- await this.applyNetworkPolicy(policy, wlNs)+ await this.applyOwnedRecipeNetworkPolicy(policy, wlNs, recipe.metadata.name)}}@@ -4728,6 +5013,150 @@ export class WorkflowRecipeReconciler {}}+ private assertRecipeNetworkPolicyOwnership(+ existing: k8s.V1NetworkPolicy,+ recipeName: string,+ policyName: string,+ namespace: string+ ): void {+ const labels = existing.metadata?.labels ?? {}+ if (+ labels['clerum.io/managed-by'] !== 'workflow-recipes' ||+ labels['clerum.io/recipe'] !== recipeName+ ) {+ throw new NetworkPolicyOwnershipConflictError(+ `Refusing to mutate NetworkPolicy "${policyName}" in ${namespace}: existing policy is not owned by WorkflowRecipe "${recipeName}"`+ )+ }+ }++ /**+ * Apply a strictly subtractive pre-DNS transition against the exact snapshot+ * that was read. This path never creates a policy and therefore cannot add an+ * authorization before DNS succeeds. ResourceVersion/UID fencing makes a+ * concurrent replacement fail loud instead of mutating a newer object.+ */+ private async applyNetworkPolicyContraction(input: {+ existing: k8s.V1NetworkPolicy+ desired: k8s.V1NetworkPolicy | null+ namespace: string+ recipeName: string+ }): Promise<k8s.V1NetworkPolicy | null> {+ const { existing, desired, namespace, recipeName } = input+ const policyName = existing.metadata?.name+ const resourceVersion = existing.metadata?.resourceVersion+ if (!policyName || !resourceVersion) {+ throw new Error('NetworkPolicy contraction requires policy name and resourceVersion')+ }+ this.assertRecipeNetworkPolicyOwnership(existing, recipeName, policyName, namespace)++ if (desired === null) {+ const uid = existing.metadata?.uid+ if (!uid) throw new Error('NetworkPolicy contraction delete requires policy uid')+ try {+ await this.networkingApi.deleteNamespacedNetworkPolicy({+ name: policyName,+ namespace,+ body: { preconditions: { uid, resourceVersion } },+ })+ } catch (error: unknown) {+ if (getErrorCode(error) !== 404) throw error+ }+ createLogger('wrc', recipeName).info('contracted recipe network policy', {+ policy: policyName,+ namespace,+ action: 'delete',+ })+ return null+ }++ if (!contractionWriteNeeded(existing, desired)) return existing+ desired.metadata = { ...(desired.metadata ?? {}), resourceVersion }+ await this.networkingApi.replaceNamespacedNetworkPolicy({+ name: policyName,+ namespace,+ body: desired,+ })+ let live: k8s.V1NetworkPolicy+ try {+ live = await this.networkingApi.readNamespacedNetworkPolicy({+ name: policyName,+ namespace,+ })+ } catch (error: unknown) {+ if (getErrorCode(error) === 404) return null+ throw error+ }+ this.assertRecipeNetworkPolicyOwnership(live, recipeName, policyName, namespace)+ const liveUid = live.metadata?.uid+ const liveResourceVersion = live.metadata?.resourceVersion+ if (+ live.metadata?.name !== policyName ||+ live.metadata?.namespace !== namespace ||+ !liveUid ||+ !liveResourceVersion ||+ (existing.metadata?.uid && liveUid !== existing.metadata.uid)+ ) {+ throw new NetworkPolicyOwnershipConflictError(+ `NetworkPolicy "${policyName}" in ${namespace} changed identity during contraction`+ )+ }+ const provenLive = subtractiveNetworkPolicy(live, desired)+ if (!provenLive || contractionWriteNeeded(live, provenLive)) {+ // The apiserver read-back is broader or carries provenance that does not+ // match its enforced rules. Do not let phase 2 treat the request body as+ // reality; revoke this exact owned identity and force a clean re-plan.+ await this.networkingApi.deleteNamespacedNetworkPolicy({+ name: policyName,+ namespace,+ body: { preconditions: { uid: liveUid, resourceVersion: liveResourceVersion } },+ })+ throw new NetworkPolicyOwnershipConflictError(+ `NetworkPolicy "${policyName}" in ${namespace} did not read back as the requested contraction`+ )+ }+ createLogger('wrc', recipeName).info('contracted recipe network policy', {+ policy: policyName,+ namespace,+ action: 'replace',+ })+ return live+ }++ private async applyOwnedRecipeNetworkPolicy(+ policy: k8s.V1NetworkPolicy,+ namespace: string,+ recipeName: string+ ): Promise<void> {+ const policyName = policy.metadata?.name+ if (!policyName) throw new Error('Recipe NetworkPolicy requires a name')+ await this.createOrReplace(+ () => this.networkingApi.createNamespacedNetworkPolicy({ namespace, body: policy }),+ async () => {+ const live = await this.networkingApi.readNamespacedNetworkPolicy({+ name: policyName,+ namespace,+ })+ this.assertRecipeNetworkPolicyOwnership(live, recipeName, policyName, namespace)+ if (!live.metadata?.resourceVersion) {+ throw new NetworkPolicyOwnershipConflictError(+ `Refusing to replace NetworkPolicy "${policyName}" in ${namespace}: live resourceVersion is missing`+ )+ }+ policy.metadata = {+ ...(policy.metadata ?? {}),+ resourceVersion: live.metadata?.resourceVersion,+ }+ return this.networkingApi.replaceNamespacedNetworkPolicy({+ name: policyName,+ namespace,+ body: policy,+ })+ },+ `NetworkPolicy "${policyName}" in ${namespace}`+ )+ }+/*** True when the live policy differs from the desired one and must be written* (issue #299 NO-OP gate). Compares the accumulated-state fingerprint AND the@@ -5050,10 +5479,14 @@ export class WorkflowRecipeReconciler {namespace: string): Promise<void> {const name = rb.workloadEgressPolicyName(recipeName, workloadId)- await this.safeDelete(- () => this.networkingApi.deleteNamespacedNetworkPolicy({ name, namespace }),- `NetworkPolicy "${name}" in ${namespace}`- )+ const existing = await this.readNetworkPolicyOrNull(name, namespace)+ if (!existing) return+ await this.applyNetworkPolicyContraction({+ existing,+ desired: null,+ namespace,+ recipeName,+ })}private async deleteWorkloadIngressIfExists(@@ -5062,10 +5495,14 @@ export class WorkflowRecipeReconciler {namespace: string): Promise<void> {const name = rb.workloadIngressPolicyName(recipeName, workloadId)- await this.safeDelete(- () => this.networkingApi.deleteNamespacedNetworkPolicy({ name, namespace }),- `NetworkPolicy "${name}" in ${namespace}`- )+ const existing = await this.readNetworkPolicyOrNull(name, namespace)+ if (!existing) return+ await this.applyNetworkPolicyContraction({+ existing,+ desired: null,+ namespace,+ recipeName,+ })}// ─── Workload Ensure Methods ──────────────────────────────────────