Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
fix(egress): close Astra review findings and harden runtime proofs
Alfredo Lopezb4a78cb73037
diff --git a/.github/workflows/ci-public.yml b/.github/workflows/ci-public.ymlindex ac616d14e..c1b01c3ef 100644--- a/.github/workflows/ci-public.yml+++ b/.github/workflows/ci-public.yml@@ -413,6 +413,10 @@ jobs:with:fetch-depth: 0+ - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4+ with:+ node-version: '24'+- name: Run bash -n on all .sh filesrun: |fail=0@@ -461,9 +465,12 @@ jobs:- name: Test WRC internal dependency NetworkPolicy contractsrun: bash scripts/tests/test-wrc-internal-dependency-networkpolicy-gate.sh- - name: Test WRC external-egress degradation gate contract+ - name: Test WRC egress oracle, DNS and lifecycle (unit/integration, cluster-free)run: bash scripts/tests/test-wrc-egress-degradation-gate.sh+ - name: Test Minikube mutation and WRC recovery admission (hermetic, cluster-free)+ run: bash scripts/tests/test-minikube-mutation-boundary.sh+- name: Test minikube sandbox-ui NetworkPolicy lint (static-policy ownership + DNS)run: bash scripts/tests/test-lint-networkpolicies.shdiff --git a/Makefile b/Makefileindex ae1911166..de2acb4fe 100644--- a/Makefile+++ b/Makefile@@ -1265,7 +1265,18 @@ test-e2e-hcc-communicationchannel-watch-recovery: ## Run isolated minikube HCC wtest-e2e-wrc-egress-degradation: ## Prove valid UI/workload services survive transient DNS while removed/raced egress is contracted@echo "Running WRC external-egress degradation gate..."@test -n "$(E2E_EXPECTED_PRE_GATE_GATE)" || { echo "Set E2E_EXPECTED_PRE_GATE_GATE to the gate recorded by the branch-owned pre-gate sync" >&2; exit 1; }- E2E_WRC_EGRESS_FAULT_INJECTION=1 E2E_EXPECTED_PRE_GATE_GATE="$(E2E_EXPECTED_PRE_GATE_GATE)" MINIKUBE_PROFILE=$(E2E_KUBECONTEXT) KUBECONTEXT=$(E2E_KUBECONTEXT) bash scripts/e2e/e2e-wrc-egress-degradation.sh+ @T2_PROJECT_DIR="$(CURDIR)" T2_PROFILE="$(E2E_KUBECONTEXT)" T2_CONTEXT="$(E2E_KUBECONTEXT)" \+ T2_SKIP_LOCK="$(T2_SKIP_LOCK)" T2_LOCK_TOKEN="$(T2_LOCK_TOKEN)" \+ E2E_WRC_EGRESS_FAULT_INJECTION=1 E2E_EXPECTED_PRE_GATE_GATE="$(E2E_EXPECTED_PRE_GATE_GATE)" \+ MINIKUBE_PROFILE="$(E2E_KUBECONTEXT)" KUBECONTEXT="$(E2E_KUBECONTEXT)" \+ bash scripts/minikube/with-t2-mutation-lock.sh -- bash scripts/e2e/e2e-wrc-egress-degradation.sh++.PHONY: test-e2e-wrc-egress-recover+test-e2e-wrc-egress-recover: ## Restore only the recorded, owned WRC DNS fault intervention; never certify the interrupted gate+ @T2_PROJECT_DIR="$(CURDIR)" T2_PROFILE="$(E2E_KUBECONTEXT)" T2_CONTEXT="$(E2E_KUBECONTEXT)" \+ T2_SKIP_LOCK="$(T2_SKIP_LOCK)" T2_LOCK_TOKEN="$(T2_LOCK_TOKEN)" \+ MINIKUBE_PROFILE="$(E2E_KUBECONTEXT)" KUBECONTEXT="$(E2E_KUBECONTEXT)" \+ bash scripts/minikube/with-t2-mutation-lock.sh --recover-wrc-egress.PHONY: test-e2e-hcc-readiness-bootstraptest-e2e-hcc-readiness-bootstrap: ## Prove HCC readiness while its initial Host fleet pass remains activediff --git a/host-context-controller/package-lock.json b/host-context-controller/package-lock.jsonindex c390d2654..a62bb6812 100644--- a/host-context-controller/package-lock.json+++ b/host-context-controller/package-lock.json@@ -1,12 +1,12 @@{"name": "clerum-host-context-controller",- "version": "0.1.296",+ "version": "0.1.297","lockfileVersion": 3,"requires": true,"packages": {"": {"name": "clerum-host-context-controller",- "version": "0.1.296",+ "version": "0.1.297","license": "SEE LICENSE IN LICENSE","dependencies": {"@clerum/codex-catalog-projection": "file:../packages/codex-catalog-projection",diff --git a/host-context-controller/package.json b/host-context-controller/package.jsonindex dd067e323..55d3bb0c6 100644--- a/host-context-controller/package.json+++ b/host-context-controller/package.json@@ -1,6 +1,6 @@{"name": "clerum-host-context-controller",- "version": "0.1.296",+ "version": "0.1.297","description": "Host Context Controller - provides McpServer CRDs to mcp-host via REST API","main": "dist/main.js","engines": {diff --git a/host-context-controller/src/__tests__/applyNetworkPolicyNoopGate.test.ts b/host-context-controller/src/__tests__/applyNetworkPolicyNoopGate.test.tsindex f55690950..99e3ced92 100644--- a/host-context-controller/src/__tests__/applyNetworkPolicyNoopGate.test.ts+++ b/host-context-controller/src/__tests__/applyNetworkPolicyNoopGate.test.ts@@ -120,6 +120,33 @@ describe('applyNetworkPolicy no-op gate', () => {expect(api.replaceNamespacedNetworkPolicy).not.toHaveBeenCalled()})+ it.each([+ { form: 'ApiException', error: apiException(404) },+ { form: 'statusCode', error: { response: { statusCode: 404 } } },+ ])(+ 'admission-sensitive create409/read404 ($form) rejects instead of certifying absence',+ async ({ error }) => {+ const api = fakeNetworkingApi()+ api.createNamespacedNetworkPolicy.mockRejectedValue({ code: 409 })+ api.readNamespacedNetworkPolicy.mockRejectedValue(error)++ await expect(+ applyNetworkPolicy(+ api as unknown as k8s.NetworkingV1Api,+ 'np',+ 'ns',+ desiredPolicy(),+ '[NetPol]',+ undefined,+ undefined,+ true+ )+ ).rejects.toBe(error)++ expect(api.replaceNamespacedNetworkPolicy).not.toHaveBeenCalled()+ }+ )+it.each([{ status: 403, form: 'ApiException', error: apiException(403) },{ status: 403, form: 'statusCode', error: { response: { statusCode: 403 } } },diff --git a/host-context-controller/src/__tests__/replaceWithConflictRetryOrder.test.ts b/host-context-controller/src/__tests__/replaceWithConflictRetryOrder.test.tsindex fb195364d..b7688dc2b 100644--- a/host-context-controller/src/__tests__/replaceWithConflictRetryOrder.test.ts+++ b/host-context-controller/src/__tests__/replaceWithConflictRetryOrder.test.ts@@ -107,9 +107,16 @@ describe('replaceWithConflictRetry order and retry', () => {expect(replace.mock.calls[0][0].metadata?.resourceVersion).toBe('9')expect(replace.mock.calls[1][0].metadata?.resourceVersion).toBe('10')const updated = log.mock.calls- .map((call: unknown[]) => String(call[0]))- .filter((line: string) => line.includes('[Test] Updated Service "svc"'))- expect(updated).toEqual(['[Test] Updated Service "svc" (after 2 attempts)'])+ .map((call: unknown[]) => JSON.parse(String(call[0])))+ .filter(entry => entry.msg === 'Kubernetes resource updated')+ expect(updated).toEqual([+ expect.objectContaining({+ level: 'info',+ scope: '[Test]',+ description: 'Service "svc"',+ attempt: 2,+ }),+ ])} finally {log.mockRestore()}@@ -355,9 +362,13 @@ describe('replaceWithConflictRetry order and retry', () => {expect(replace).not.toHaveBeenCalled()expect(await readWritesTotal('Service')).toBe(before)- expect(warn).toHaveBeenCalledWith(- expect.stringMatching(/disappeared \(404\); skipping replace/)- )+ expect(warn).toHaveBeenCalledOnce()+ expect(JSON.parse(String(warn.mock.calls[0][0]))).toMatchObject({+ level: 'warn',+ msg: 'Kubernetes resource disappeared; skipping replace',+ scope: '[Test]',+ description: 'Service "svc"',+ })warn.mockRestore()})diff --git a/host-context-controller/src/networkPolicyReconciler.egressObservation.test.ts b/host-context-controller/src/networkPolicyReconciler.egressObservation.test.tsnew file mode 100644index 000000000..4d833a8e3--- /dev/null+++ b/host-context-controller/src/networkPolicyReconciler.egressObservation.test.ts@@ -0,0 +1,408 @@+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'+import * as k8s from '@kubernetes/client-node'+import * as dns from 'node:dns/promises'+import { ExternalEgressConvergenceCoordinator } from './externalEgressConvergenceCoordinator'+import { NetworkPolicyReconciler } from './networkPolicyReconciler'+import type { EgressBinding, McpServerCRD, McpServerCrdStatus } from './types'++vi.mock('./config', () => ({+ config: {+ namespace: 'mcp-server',+ controlPlaneNamespace: 'control-plane',+ hostNamespace: 'mcp-host',+ rpcProxyNamespace: 'rpc-proxy',+ runtimeNamespaces: ['mcp-server'],+ externalEgressDnsResolveTimeoutMs: 5000,+ externalEgressOverlapSec: 300,+ externalEgressMaxEntries: 128,+ },+}))+vi.mock('node:dns/promises', () => ({ resolve4: vi.fn() }))+vi.mock('./metrics', () => ({+ writesTotal: { inc: vi.fn() },+ writeSkipsTotal: { inc: vi.fn() },+ externalEgressRetriesAtCap: { set: vi.fn() },+}))++const currentDns: EgressBinding = { dns: 'current.example.com', port: 443 }+const removedDns: EgressBinding = { dns: 'removed.example.com', port: 8443 }+const currentName = 'ext-egress-observation-mcp-current.example.com-443'+const removedName = 'ext-egress-observation-mcp-removed.example.com-8443'+const apiError = (code: number) => Object.assign(new Error(`Kubernetes API ${code}`), { code })++function fixture(bindings: EgressBinding[]) {+ const server: McpServerCRD = {+ name: 'observation-mcp',+ namespace: 'mcp-server',+ uid: 'server-uid',+ generation: 7,+ spec: {+ contextRef: 'dev',+ image: 'observation:1',+ transport: { type: 'streamableHttp', port: 3000 },+ egressBindings: bindings,+ },+ }+ const policies = new Map<string, k8s.V1NetworkPolicy>()+ const unavailableReads = new Set<string>()+ const unavailableDeletes = new Set<string>()+ let status: McpServerCrdStatus = {}+ let version = 0+ const api = {+ listNamespacedNetworkPolicy: vi.fn(async () => ({+ items: [...policies.values()].map(policy => structuredClone(policy)),+ })),+ readNamespacedNetworkPolicy: vi.fn(async ({ name }: { name: string }) => {+ if (unavailableReads.has(name)) throw apiError(503)+ const policy = policies.get(name)+ if (!policy) throw apiError(404)+ return structuredClone(policy)+ }),+ createNamespacedNetworkPolicy: vi.fn(async ({ body }: { body: k8s.V1NetworkPolicy }) => {+ const name = body.metadata!.name!+ if (policies.has(name)) throw apiError(409)+ const policy = structuredClone(body)+ policy.metadata = { ...policy.metadata, uid: `uid-${name}`, resourceVersion: `${++version}` }+ policies.set(name, policy)+ return structuredClone(policy)+ }),+ replaceNamespacedNetworkPolicy: vi.fn(+ async ({ name, body }: { name: string; body: k8s.V1NetworkPolicy }) => {+ const current = policies.get(name)+ if (!current) throw apiError(404)+ if (body.metadata?.resourceVersion !== current.metadata?.resourceVersion)+ throw apiError(409)+ const policy = structuredClone(body)+ policy.metadata = {+ ...policy.metadata,+ uid: current.metadata!.uid,+ resourceVersion: `${++version}`,+ }+ policies.set(name, policy)+ return structuredClone(policy)+ }+ ),+ deleteNamespacedNetworkPolicy: vi.fn(+ async ({ name, body }: { name: string; body: k8s.V1DeleteOptions }) => {+ if (unavailableDeletes.has(name)) throw apiError(503)+ const current = policies.get(name)+ if (!current) throw apiError(404)+ if (+ body.preconditions?.uid !== current.metadata?.uid ||+ body.preconditions?.resourceVersion !== current.metadata?.resourceVersion+ )+ throw apiError(409)+ policies.delete(name)+ }+ ),+ }+ const customApi = {+ getNamespacedCustomObjectStatus: vi.fn(async () => ({+ metadata: { uid: server.uid, generation: server.generation, resourceVersion: 'status-rv' },+ status: structuredClone(status),+ })),+ patchNamespacedCustomObjectStatus: vi.fn(+ async ({ body }: { body: Array<{ op: string; path: string; value: unknown }> }) => {+ for (const operation of body) {+ if (operation.path === '/status') status = operation.value as McpServerCrdStatus+ if (operation.path === '/status/conditions')+ status.conditions = operation.value as McpServerCrdStatus['conditions']+ if (operation.path === '/status/resolvedEgressIPs')+ status.resolvedEgressIPs = operation.value as McpServerCrdStatus['resolvedEgressIPs']+ }+ }+ ),+ }+ const kc = new k8s.KubeConfig()+ kc.loadFromOptions({+ clusters: [{ name: 'unit', server: 'https://unit.invalid' }],+ users: [{ name: 'unit' }],+ contexts: [{ name: 'unit', cluster: 'unit', user: 'unit' }],+ currentContext: 'unit',+ })+ const reconciler = new NetworkPolicyReconciler(kc, new Map())+ Object.assign(reconciler, { networkingApi: api, customApi })+ return {+ server,+ policies,+ api,+ unavailableReads,+ unavailableDeletes,+ reconciler,+ status: () => status,+ condition: () => status.conditions?.find(condition => condition.type === 'ExternalEgressReady'),+ run: () => reconciler.reconcileExternalEgress(server, { isCurrent: () => true }),+ clearCalls: () => Object.values(api).forEach(mock => mock.mockClear()),+ }+}++describe('external egress observation failures do not decide authorization', () => {+ beforeEach(() => {+ vi.mocked(dns.resolve4).mockResolvedValue([{ address: '1.2.3.4', ttl: 300 }] as never)+ })+ afterEach(() => vi.useRealTimers())++ it('keeps the current policy unmodified and revokes a removed sibling when fresh DNS-policy reads are unavailable', async () => {+ const f = fixture([removedDns, currentDns])+ await f.run()+ const before = structuredClone(f.policies.get(currentName))+ const removed = f.policies.get(removedName)!+ f.server.spec.egressBindings = [currentDns]+ f.unavailableReads.add(currentName)+ f.clearCalls()++ await expect(f.run()).rejects.toThrow('Kubernetes API 503')++ expect(f.policies.get(currentName)).toEqual(before)+ expect(f.api.createNamespacedNetworkPolicy).not.toHaveBeenCalled()+ expect(f.api.replaceNamespacedNetworkPolicy).not.toHaveBeenCalled()+ expect(f.api.deleteNamespacedNetworkPolicy).toHaveBeenCalledExactlyOnceWith({+ name: removedName,+ namespace: f.server.namespace,+ body: {+ preconditions: {+ uid: removed.metadata!.uid,+ resourceVersion: removed.metadata!.resourceVersion,+ },+ },+ })+ expect(f.condition()).toMatchObject({+ status: 'False',+ reason: 'ExternalEgressReconcileFailed',+ })+ expect(f.condition()?.message).toMatch(/unavailable|could not be verified/)+ expect(f.condition()?.message).not.toMatch(/REVOKED|provenance was absent or invalid/)+ expect(f.status().resolvedEgressIPs).toEqual([])++ f.unavailableReads.clear()+ await f.run()+ expect(f.condition()).toMatchObject({ status: 'True', reason: 'Reconciled' })+ expect(f.policies.get(currentName)).toEqual(before)+ expect(f.policies.has(removedName)).toBe(false)+ })++ it.each([+ {+ binding: { cidr: '8.8.8.8/32', port: 9443 },+ name: 'ext-egress-observation-mcp-8-8-8-8-32-9443',+ },+ {+ binding: { egressClass: 'public-web' as const },+ name: 'ext-egress-observation-mcp-public-web',+ },+ ])(+ 'revokes an independent removed binding despite a $name read failure and recovers without a spec change',+ async ({ binding, name }) => {+ const f = fixture([removedDns, binding])+ await f.run()+ const before = structuredClone(f.policies.get(name))+ f.server.spec.egressBindings = [binding]+ f.unavailableReads.add(name)+ f.clearCalls()++ await expect(f.run()).rejects.toThrow('Kubernetes API 503')++ expect(f.policies.has(removedName)).toBe(false)+ expect(f.policies.get(name)).toEqual(before)+ expect(f.api.createNamespacedNetworkPolicy).not.toHaveBeenCalled()+ expect(f.api.replaceNamespacedNetworkPolicy).not.toHaveBeenCalled()+ expect(f.condition()).toMatchObject({+ status: 'False',+ reason: 'ExternalEgressReconcileFailed',+ })+ expect(f.status().resolvedEgressIPs).toEqual([])+ f.unavailableReads.clear()+ await f.run()+ expect(f.condition()).toMatchObject({ status: 'True', reason: 'Reconciled' })+ expect(f.policies.get(name)).toEqual(before)+ }+ )++ it('does not resolve or expand until the independent removal barrier succeeds', async () => {+ const f = fixture([removedDns])+ await f.run()+ f.server.spec.egressBindings = [currentDns]+ f.unavailableDeletes.add(removedName)+ f.clearCalls()+ vi.mocked(dns.resolve4).mockClear()++ await expect(f.run()).rejects.toThrow('Kubernetes API 503')++ expect(dns.resolve4).not.toHaveBeenCalled()+ expect(f.api.createNamespacedNetworkPolicy).not.toHaveBeenCalled()+ expect(f.condition()).toMatchObject({ status: 'False', reason: 'CleanupFailed' })+ f.unavailableDeletes.clear()+ await f.run()+ expect(f.policies.has(removedName)).toBe(false)+ expect(f.policies.has(currentName)).toBe(true)+ })++ it('finishes independent removals when one delete fails, without expanding another binding', async () => {+ const f = fixture([removedDns, currentDns])+ await f.run()+ f.server.spec.egressBindings = [{ dns: 'added.example.com', port: 443 }]+ f.unavailableDeletes.add(removedName)+ f.clearCalls()++ await expect(f.run()).rejects.toThrow('Kubernetes API 503')++ expect(f.policies.has(removedName)).toBe(true)+ expect(f.policies.has(currentName)).toBe(false)+ expect(f.api.createNamespacedNetworkPolicy).not.toHaveBeenCalled()+ expect(f.condition()?.reason).toBe('CleanupFailed')+ })++ it('does not add a sibling permission while a current policy observation is pending', async () => {+ const f = fixture([currentDns])+ await f.run()+ f.server.spec.egressBindings = [{ cidr: '8.8.8.8/32', port: 443 }, currentDns]+ f.unavailableReads.add(currentName)+ f.clearCalls()++ await expect(f.run()).rejects.toThrow('Kubernetes API 503')++ expect(f.api.createNamespacedNetworkPolicy).not.toHaveBeenCalled()+ expect(f.api.replaceNamespacedNetworkPolicy).not.toHaveBeenCalled()+ expect(f.policies.size).toBe(1)+ expect(f.status().resolvedEgressIPs).toEqual([])+ })++ it('rechecks a deferred no-op after sibling DNS latency and recovers from the changed snapshot', async () => {+ const sibling: EgressBinding = { dns: 'later.example.com', port: 443 }+ const f = fixture([currentDns, sibling])+ await f.run()+ f.clearCalls()+ vi.mocked(dns.resolve4).mockImplementation(async hostname => {+ if (hostname === sibling.dns) {+ const policy = f.policies.get(currentName)!+ policy.metadata!.resourceVersion = 'concurrent-rv'+ policy.spec!.egress!.push({+ to: [{ ipBlock: { cidr: '8.8.8.8/32' } }],+ ports: [{ port: 443, protocol: 'TCP' }],+ })+ }+ return [{ address: '1.2.3.4', ttl: 300 }] as never+ })++ await expect(f.run()).rejects.toThrow(/changed after the external-egress observation/)++ expect(f.condition()?.status).toBe('False')+ expect(f.status().resolvedEgressIPs).toEqual([])+ expect(f.api.replaceNamespacedNetworkPolicy).not.toHaveBeenCalled()+ vi.mocked(dns.resolve4).mockResolvedValue([{ address: '1.2.3.4', ttl: 300 }] as never)+ await f.run()+ expect(f.condition()?.status).toBe('True')+ expect(f.policies.get(currentName)?.spec?.egress).toHaveLength(1)+ })++ it('does not write or report status after the generation fence closes during an unavailable read', async () => {+ const f = fixture([currentDns])+ await f.run()+ f.clearCalls()+ const statusBefore = structuredClone(f.status())+ let current = true+ f.api.readNamespacedNetworkPolicy.mockImplementation(async () => {+ current = false+ throw apiError(503)+ })++ await f.reconciler.reconcileExternalEgress(f.server, { isCurrent: () => current })++ expect(f.api.deleteNamespacedNetworkPolicy).not.toHaveBeenCalled()+ expect(f.api.createNamespacedNetworkPolicy).not.toHaveBeenCalled()+ expect(f.api.replaceNamespacedNetworkPolicy).not.toHaveBeenCalled()+ expect(f.status()).toEqual(statusBefore)+ })++ it('does not copy a newer resourceVersion onto a policy calculated before an apply conflict', async () => {+ const f = fixture([currentDns])+ await f.run()+ const policy = f.policies.get(currentName)!+ policy.spec!.egress!.push({+ to: [{ ipBlock: { cidr: '8.8.8.8/32' } }],+ ports: [{ port: 443, protocol: 'TCP' }],+ })+ f.clearCalls()+ f.api.createNamespacedNetworkPolicy.mockImplementationOnce(async () => {+ // The conflict retry reads this newer object after the final plan check.+ // A matching owner alone cannot authorize reuse of the older DNS state.+ policy.metadata!.resourceVersion = 'changed-before-replace'+ throw apiError(409)+ })++ await expect(f.run()).rejects.toThrow(/changed after the external-egress observation/)++ expect(f.api.replaceNamespacedNetworkPolicy).not.toHaveBeenCalled()+ expect(f.condition()?.status).toBe('False')+ await f.run()+ expect(f.condition()?.status).toBe('True')+ expect(f.policies.get(currentName)?.spec?.egress).toHaveLength(1)+ })++ it('does not certify a policy deleted after create conflict and before replacement read, then recovers', async () => {+ const f = fixture([currentDns])+ await f.run()+ vi.mocked(dns.resolve4).mockResolvedValue([{ address: '8.8.8.8', ttl: 300 }] as never)+ f.api.createNamespacedNetworkPolicy.mockImplementationOnce(async () => {+ // POST sees the existing policy and conflicts. A concurrent deletion then+ // wins before the helper's replacement GET, which observes real absence.+ f.policies.delete(currentName)+ throw apiError(409)+ })++ await expect(f.run()).rejects.toThrow()++ expect(f.policies.has(currentName)).toBe(false)+ expect(f.condition()).toMatchObject({+ status: 'False',+ reason: 'ExternalEgressReconcileFailed',+ })+ expect(f.status().resolvedEgressIPs).toEqual([])+ await f.run()+ expect(f.condition()).toMatchObject({ status: 'True', reason: 'Reconciled' })+ expect(f.policies.get(currentName)?.spec?.egress).toEqual([+ { to: [{ ipBlock: { cidr: '8.8.8.8/32' } }], ports: [{ port: 443, protocol: 'TCP' }] },+ ])+ expect(f.status().resolvedEgressIPs?.[0].ips).toEqual(['8.8.8.8'])+ })++ it('keeps the existing retry coordinator pending until the API recovers and then admits the same generation once', async () => {+ vi.useFakeTimers()+ const f = fixture([currentDns])+ await f.run()+ f.unavailableReads.add(currentName)+ const admitted = vi.fn()+ let coordinator: ExternalEgressConvergenceCoordinator+ coordinator = new ExternalEgressConvergenceCoordinator({+ listServers: () => [f.server],+ getCurrentServer: () => f.server,+ inventoryAuthoritative: () => true,+ sameDesiredRevision: (left, right) =>+ left.uid === right.uid && left.generation === right.generation,+ enqueue: async (_server, work) => work(),+ mutate: async (_type, server, options) =>+ f.reconciler.reconcileExternalEgress(server, options),+ replay: async (type, server, retry) => {+ const completion = await coordinator.reconcile(type, server, { retry })+ admitted(server.generation)+ completion?.complete()+ },+ externalEgressRefreshMinTtlMs: () => f.reconciler.externalEgressRefreshMinTtlMs,+ })+ try {+ const gate = coordinator.prepareStartupGates([f.server])+ expect(await gate.waitFor(f.server)).toBe(false)+ await vi.advanceTimersByTimeAsync(5000)+ expect(admitted).not.toHaveBeenCalled()+ expect(f.condition()?.status).toBe('False')+ f.unavailableReads.clear()+ await vi.advanceTimersByTimeAsync(15000)+ expect(admitted).toHaveBeenCalledExactlyOnceWith(7)+ expect(f.condition()?.status).toBe('True')+ await vi.advanceTimersByTimeAsync(60000)+ expect(admitted).toHaveBeenCalledTimes(1)+ } finally {+ coordinator.stop()+ }+ })+})diff --git a/host-context-controller/src/networkPolicyReconciler.test.ts b/host-context-controller/src/networkPolicyReconciler.test.tsindex 0073d2b46..94c849a20 100644--- a/host-context-controller/src/networkPolicyReconciler.test.ts+++ b/host-context-controller/src/networkPolicyReconciler.test.ts@@ -4018,9 +4018,12 @@ describe('NetworkPolicyReconciler', () => {expect(error).toBeInstanceOf(AggregateError)expect((error as AggregateError).errors).toEqual([additiveFailure])expect(ordering).toEqual(['certify', 'context:aaa-poisoned', 'context:bbb-healthy'])- expect(errorLog).toHaveBeenCalledWith(- '[NetPol] Additive Context reconciliation failed for "aaa-poisoned":',- additiveFailure+ expect(errorLog.mock.calls.map(([line]) => JSON.parse(String(line)))).toContainEqual(+ expect.objectContaining({+ level: 'error',+ contextId: 'aaa-poisoned',+ err: { name: 'Error', message: additiveFailure.message },+ }))})diff --git a/host-context-controller/src/networkPolicyReconciler.ts b/host-context-controller/src/networkPolicyReconciler.tsindex 9ffb525bf..6788c3710 100644--- a/host-context-controller/src/networkPolicyReconciler.ts+++ b/host-context-controller/src/networkPolicyReconciler.ts@@ -167,12 +167,18 @@ type ProvenDnsAnnotations = {contracted: boolean}+type ExternalEgressDisposition =+ | { state: 'pending' }+ | { state: 'retained' }+ | { state: 'absent' }+ | { state: 'revoke'; policy: k8s.V1NetworkPolicy }+/*** What the ExternalEgressReady message says about the live policy on a non-DNS* fault. One entry per verdict state so a new state cannot be added without* deciding what the operator is told.*/-const LIVE_EGRESS_FAULT_TEXT: Record<LiveEgressVerdict['state'], string> = {+const LIVE_EGRESS_FAULT_TEXT: Record<LiveEgressVerdict['state'] | 'unavailable', string> = {none: 'no live policy to serve',foreign: 'homonymous live policy is not HCC-owned and was left untouched',intact: 'verified policy retained for existing pods; runtime remains blocked',@@ -180,6 +186,7 @@ const LIVE_EGRESS_FAULT_TEXT: Record<LiveEgressVerdict['state'], string> = {'live policy REVOKED: it enforced a CIDR outside the /32 allowlist this controller writes','drifted-shape': 'live policy REVOKED: its egress shape is not one this controller writes','untrusted-state': 'live policy REVOKED: its DNS provenance was absent or invalid',+ unavailable: 'live policy could not be verified and was left untouched; runtime remains blocked',}function hasExpectedPolicyOwnership(policy: k8s.V1NetworkPolicy, policyType: string): boolean {@@ -569,6 +576,40 @@ export class NetworkPolicyReconciler {return `ext-egress-${serverName}-${suffix}-${binding.port}`.slice(0, 253)}+ private externalEgressBindingError(binding: EgressBinding): string | null {+ const egressClass = binding.egressClass ?? 'exact-host'+ if (egressClass !== 'exact-host' && egressClass !== 'public-web') {+ return `egressClass "${String(binding.egressClass)}" is not supported`+ }+ if (egressClass === 'public-web') {+ return binding.dns ||+ binding.cidr ||+ binding.port !== undefined ||+ binding.protocol !== undefined+ ? 'public-web external egress bindings must not declare dns, cidr, port, or protocol'+ : null+ }+ if (!binding.dns && !binding.cidr) {+ return 'exact-host external egress bindings must declare dns or cidr'+ }+ if (+ binding.port === undefined ||+ !Number.isInteger(binding.port) ||+ binding.port < 1 ||+ binding.port > 65535+ ) {+ return 'exact-host external egress bindings must declare an integer port from 1-65535'+ }+ if (binding.cidr) {+ return isAllowedExternalEgressCidr(binding.cidr)+ ? null+ : `CIDR "${binding.cidr}" overlaps private, metadata, link-local, multicast, documentation, or reserved ranges`+ }+ return isPublicDnsHostname(binding.dns!)+ ? null+ : `hostname "${binding.dns}" is private, internal, metadata, local, or otherwise disallowed`+ }+// ─── Default Policies (L0 + L1) ─────────────────────────────────────/**@@ -583,7 +624,7 @@ export class NetworkPolicyReconciler {await this.ensureAllowContextMapperApi()await this.deleteLegacyStaticPolicy(config.rpcProxyNamespace, 'allow-desktop-egress-rpc-proxy')await this.deleteLegacyStaticPolicy(config.namespace, 'allow-rpc-proxy-to-managed-mcp-servers')- console.log('[NetPol] Default policies ensured')+ hccLogger.info('default network policies ensured')}/**@@ -970,18 +1011,17 @@ export class NetworkPolicyReconciler {})if (!isCurrent()) return false- console.log(- `[NetPol] Reconciling context "${contextId}" — allowed servers: [${allowedServers.join(', ')}]`- )+ hccLogger.info('reconciling context network policies', { contextId, allowedServers })// Create or update policies for each allowed server.for (const serverName of allowedServers) {if (!isCurrent()) return falseconst server = this.serverCache.get(serverName)if (!server) {- console.warn(- `[NetPol] McpServer "${serverName}" not found in cache — skipping policy for context "${contextId}"`- )+ hccLogger.warn('McpServer absent from cache; context policy skipped', {+ serverName,+ contextId,+ })continue}@@ -1109,14 +1149,14 @@ export class NetworkPolicyReconciler {const existingName = existing.metadata?.name || ''const desired = desiredIngress.get(existingName)if (!desired) {- console.log(`[NetPol] Deleting orphaned policy "${existingName}"`)+ hccLogger.info('deleting orphaned context policy', { policy: existingName })if (!(await revokeOrphanedPolicy(config.namespace, existing))) return falseonRevoked?.()} else if (!sameNetworkPolicySpec(existing, desired) ||!hasExpectedPolicyOwnership(existing, 'context-allow')) {- console.log(`[NetPol] Replacing stale same-name policy "${existingName}"`)+ hccLogger.info('replacing stale context policy', { policy: existingName })if (safetySnapshotProvided) {if (!(await this.replaceSafetyPolicySnapshot(@@ -1150,14 +1190,14 @@ export class NetworkPolicyReconciler {const existingName = existing.metadata?.name || ''const desired = desiredHostEgress.get(existingName)if (!desired) {- console.log(`[NetPol] Deleting orphaned L2 egress policy "${existingName}"`)+ hccLogger.info('deleting orphaned L2 egress policy', { policy: existingName })if (!(await revokeOrphanedPolicy(config.hostNamespace, existing))) return falseonRevoked?.()} else if (!sameNetworkPolicySpec(existing, desired) ||!hasExpectedPolicyOwnership(existing, 'context-allow')) {- console.log(`[NetPol] Replacing stale same-name L2 egress policy "${existingName}"`)+ hccLogger.info('replacing stale L2 egress policy', { policy: existingName })if (safetySnapshotProvided) {if (!(await this.replaceSafetyPolicySnapshot(@@ -1191,14 +1231,14 @@ export class NetworkPolicyReconciler {const existingName = existing.metadata?.name || ''const desired = desiredRpcProxyEgress.get(existingName)if (!desired) {- console.log(`[NetPol] Deleting orphaned rpc-proxy egress policy "${existingName}"`)+ hccLogger.info('deleting orphaned rpc-proxy egress policy', { policy: existingName })if (!(await revokeOrphanedPolicy(config.rpcProxyNamespace, existing))) return falseonRevoked?.()} else if (!sameNetworkPolicySpec(existing, desired) ||!hasExpectedPolicyOwnership(existing, RPC_PROXY_EGRESS_POLICY_TYPE)) {- console.log(`[NetPol] Replacing stale same-name rpc-proxy egress policy "${existingName}"`)+ hccLogger.info('replacing stale rpc-proxy egress policy', { policy: existingName })if (safetySnapshotProvided) {if (!(await this.replaceSafetyPolicySnapshot(@@ -1234,7 +1274,7 @@ export class NetworkPolicyReconciler {contextId: string,deleteAllowed?: () => Promise<boolean>): Promise<void> {- console.log(`[NetPol] Context "${contextId}" deleted — removing all policies`)+ hccLogger.info('removing policies for deleted context', { contextId })const policies = await this.listPoliciesForContext(contextId)for (const policy of policies) {@@ -1319,7 +1359,7 @@ export class NetworkPolicyReconciler {servers: McpServerCRD[] = [],options: NetworkPolicyFullReconcileOptions = {}): Promise<void> {- console.log(`[NetPol] Running full reconciliation for ${contexts.length} Context(s)`)+ hccLogger.info('running full network policy reconciliation', { contexts: contexts.length })// This is intentionally scoped to callers that use the authoritative// revocation callback. Ordinary additive reconciles must not be presented@@ -1362,7 +1402,7 @@ export class NetworkPolicyReconciler {}if (contextAuthorityLost && !contextAuthorityWarningLogged) {contextAuthorityWarningLogged = true- console.warn(+ hccLogger.warn('[NetPol] Skipping remaining Context policy orphan cleanup because Context inventory authority was lost')}@@ -1381,7 +1421,7 @@ export class NetworkPolicyReconciler {}if (serverAuthorityLost && !serverAuthorityWarningLogged) {serverAuthorityWarningLogged = true- console.warn(+ hccLogger.warn('[NetPol] Skipping remaining external egress orphan cleanup because McpServer inventory authority was lost')}@@ -1410,9 +1450,10 @@ export class NetworkPolicyReconciler {if (!name) {throw new Error('HCC-managed Context NetworkPolicy is missing its name and owner label')}- console.warn(- `[NetPol] Deleting malformed HCC-managed policy "${name}" without ${CONTEXT_LABEL}`- )+ hccLogger.warn('deleting malformed HCC-managed policy', {+ policy: name,+ missingLabel: CONTEXT_LABEL,+ })recordOrphanDelete(lane, await deletePolicy(policy))continue}@@ -1428,7 +1469,7 @@ export class NetworkPolicyReconciler {) {return}- console.log(describePolicy(name, contextId))+ hccLogger.info(describePolicy(name, contextId), { policy: name, contextId, lane })recordOrphanDelete(lane, await deletePolicy(policy))})}@@ -1524,9 +1565,12 @@ export class NetworkPolicyReconciler {const capReason = evaluateNetPolOrphanSweepCap(orphanCandidates.length, listedManagedFleet)if (capReason) {netPolOrphanSweepCappedTotal.inc({ reason: capReason })- console.warn(- `[NetPol] ${NETWORKPOLICY_ORPHAN_SWEEP_CAPPED_MESSAGE} (${capReason}): ${orphanCandidates.length} candidates of ${listedManagedFleet} listed managed policies; refusing deletes and continuing certification`- )+ hccLogger.warn(NETWORKPOLICY_ORPHAN_SWEEP_CAPPED_MESSAGE, {+ reason: capReason,+ candidates: orphanCandidates.length,+ listedManagedFleet,+ action: 'refusing deletes and continuing certification',+ })}// Delete orphaned Context policies across every L2 lane. Each lane uses@@ -1614,9 +1658,10 @@ export class NetworkPolicyReconciler {'HCC-managed external egress NetworkPolicy is missing its name and owner label')}- console.warn(- `[NetPol] Deleting malformed HCC-managed policy "${name}" without ${MCPSERVER_LABEL}`- )+ hccLogger.warn('deleting malformed HCC-managed policy', {+ policy: name,+ missingLabel: MCPSERVER_LABEL,+ })recordOrphanDelete('external-egress',await this.deleteSafetyPolicySnapshot(@@ -1646,7 +1691,7 @@ export class NetworkPolicyReconciler {) {return}- console.log(`[NetPol] Deleting orphaned external egress policy "${name}"`)+ hccLogger.info('deleting orphaned external egress policy', { policy: name })recordOrphanDelete('external-egress',await this.deleteSafetyPolicySnapshot(@@ -1659,7 +1704,7 @@ export class NetworkPolicyReconciler {}}} else if (options.serverInventoryComplete === false) {- console.warn(+ hccLogger.warn('[NetPol] Skipping external egress orphan cleanup because server inventory is incomplete')}@@ -1752,7 +1797,7 @@ export class NetworkPolicyReconciler {})} catch (error) {contextAdditiveFailures.push(error)- console.error(`[NetPol] Additive Context reconciliation failed for "${contextId}":`, error)+ hccLogger.error('additive context reconciliation failed', { contextId, err: error })}}@@ -1763,7 +1808,7 @@ export class NetworkPolicyReconciler {)}- console.log('[NetPol] Full reconciliation complete')+ hccLogger.info('full network policy reconciliation complete')}private async contextOrphanDeleteAllowed(@@ -2332,27 +2377,109 @@ export class NetworkPolicyReconciler {return}- console.log(- `[NetPol] Reconciling ${bindings.length} external egress binding(s) for "${server.name}"`- )+ hccLogger.info('reconciling external egress bindings', {+ serverName: server.name,+ bindings: bindings.length,+ })- const desiredPolicyNames = new Set<string>()const resolvedEgressIPs: McpServerResolvedEgressIP[] = []const failures: string[] = []+ const declaredBindings: Array<{ binding: EgressBinding; name: string }> = []+ const dispositions = new Map<string, ExternalEgressDisposition>()+ for (const binding of bindings) {+ const error = this.externalEgressBindingError(binding)+ const name = this.externalEgressPolicyName(server.name, binding)+ if (error || !name) {+ failures.push(error ?? 'exact-host external egress bindings must declare dns or cidr')+ continue+ }+ declaredBindings.push({ binding, name })+ dispositions.set(name, { state: 'pending' })+ }++ const cleanup = async (policies: k8s.V1NetworkPolicy[]): Promise<void> => {+ const errors: unknown[] = []+ for (const policy of policies) {+ if (!isCurrent()) return+ try {+ await this.cleanupExternalEgress(+ server.name,+ server.namespace,+ [policy],+ async () => isCurrent(),+ options.onRevoked+ )+ } catch (error: unknown) {+ // Independent removals remain authorized even when one snapshot loses+ // its fence or the API is temporarily unavailable for that object.+ errors.push(error)+ }+ }+ if (errors.length === 0 || !isCurrent()) return+ await this.writeExternalEgressStatus(+ server,+ resolvedEgressIPs,+ 'False',+ 'CleanupFailed',+ `Failed to delete stale external egress policies: ${errors.map(error => this.errorMessage(error)).join('; ')}`,+ isCurrent+ )+ throw errors[0]+ }++ // Current intent, not success/failure of a GET, determines removal. Complete+ // this barrier before DNS or additions; a failed removal never admits them.+ await cleanup(existingPolicies.filter(policy => !dispositions.has(policy.metadata?.name ?? '')))+ if (!isCurrent()) return+ existingPolicies = existingPolicies.filter(policy =>+ dispositions.has(policy.metadata?.name ?? '')+ )// Issue #513: faults that are NOT a DNS condition. Kept separate from// `failures` because those are rejected INPUT (the operator's) while these are// OUR bug, and the two earn different reasons and different messages.const reconcileFaults: Array<{- dns: string+ binding: stringerr: unknown- live: LiveEgressVerdict['state']+ live: LiveEgressVerdict['state'] | 'unavailable'verificationError?: unknown}> = []- const freshPolicySnapshots = new Map<string, k8s.V1NetworkPolicy>()+ const convergences: Array<{+ name: string+ policy: k8s.V1NetworkPolicy+ observedPolicy: k8s.V1NetworkPolicy | null+ writeRequired: boolean+ resolvedStatus?: McpServerResolvedEgressIP+ dnsFailStatic?: boolean+ }> = []const resolvedAtMs = Date.now()const resolvedAt = new Date(resolvedAtMs).toISOString()let dnsFailStatic = false+ const recordUnavailable = (+ binding: EgressBinding,+ name: string,+ err: unknown,+ verificationError?: unknown+ ): void => {+ dispositions.set(name, { state: 'pending' })+ hccLogger.error('external egress policy observation unavailable', {+ serverName: server.name,+ policy: name,+ err,+ verificationError,+ })+ reconcileFaults.push({+ binding: binding.dns ?? binding.cidr ?? 'public-web',+ err,+ live: 'unavailable',+ verificationError,+ })+ }++ const revoke = (name: string, policy: k8s.V1NetworkPolicy | null): void => {+ dispositions.set(name, policy ? { state: 'revoke', policy } : { state: 'absent' })+ }+const recordReconcileFault = async (binding: EgressBinding,name: string,@@ -2374,35 +2501,28 @@ export class NetworkPolicyReconciler {dns: binding.dns,err: verificationError,})- reconcileFaults.push({- dns: binding.dns,- err,- live: 'untrusted-state',- verificationError,- })+ recordUnavailable(binding, name, err, verificationError)return}if (verdict.state === 'intact') {- desiredPolicyNames.add(name)+ dispositions.set(name, { state: 'retained' })resolvedEgressIPs.push({dns: binding.dns,ips: verdict.cidrs.map(cidr => cidr.replace(/\/32$/, '')),resolvedAt: verdict.resolvedAt,})} else if ('policy' in verdict) {- freshPolicySnapshots.set(name, verdict.policy)+ revoke(name, verdict.policy)+ } else {+ dispositions.set(name, { state: verdict.state === 'none' ? 'absent' : 'pending' })}- reconcileFaults.push({ dns: binding.dns, err, live: verdict.state })+ reconcileFaults.push({ binding: binding.dns, err, live: verdict.state })}- for (const binding of bindings) {+ for (const { binding, name } of declaredBindings) {if (!isCurrent()) returnconst egressClass = binding.egressClass ?? 'exact-host'- if (egressClass !== 'exact-host' && egressClass !== 'public-web') {- failures.push(`egressClass "${String(binding.egressClass)}" is not supported`)- continue- }let cidrs: string[] = []// Sliding-window state annotations to persist on the policy (issue #299).@@ -2420,11 +2540,6 @@ export class NetworkPolicyReconciler {let resolvedStatus: McpServerResolvedEgressIP | undefinedlet bindingDnsFailStatic = falselet ownershipRepairRequired = false- const name = this.externalEgressPolicyName(server.name, binding)- if (!name) {- failures.push('exact-host external egress bindings must declare dns or cidr')- continue- }// Look up the live policy once: used to rehydrate the window (DNS branch)// AND to decide whether a write is actually needed (audit F2/L1) — the// write is gated on the ENFORCED spec.egress, so identical static policies@@ -2432,61 +2547,34 @@ export class NetworkPolicyReconciler {let existingPolicy = existingPolicies.find(p => p.metadata?.name === name) ?? nullif (egressClass === 'public-web') {- if (- binding.dns ||- binding.cidr ||- binding.port !== undefined ||- binding.protocol !== undefined- ) {- failures.push(- 'public-web external egress bindings must not declare dns, cidr, port, or protocol'- )- continue- }-// public-web is a static policy. Preserve #205's ownership-fenced write// (applyOwnedPolicy asserts ownership + honors the isCurrent generation// fence) while keeping dev's F2 no-churn gate so the TTL-accelerated// resync does not rewrite it every tick.const policy = this.buildPublicWebEgressPolicy(server, name)- const fresh = await this.readFreshExternalEgressPolicy(server, name)- existingPolicy = fresh.policy- if (fresh.repairRequired || this.externalEgressWriteNeeded(existingPolicy, policy)) {- await this.applyOwnedPolicy(name, server.namespace, policy, 'external-egress', isCurrent)+ let fresh: FreshExternalEgressPolicy+ try {+ fresh = await this.readFreshExternalEgressPolicy(server, name)+ } catch (error: unknown) {+ recordUnavailable(binding, name, error)+ continue}+ existingPolicy = fresh.policyif (!isCurrent()) return- desiredPolicyNames.add(name)+ convergences.push({+ name,+ policy,+ observedPolicy: existingPolicy,+ writeRequired:+ fresh.repairRequired || this.externalEgressWriteNeeded(existingPolicy, policy),+ })continue}const protocol = binding.protocol ?? 'TCP'- if (- binding.port === undefined ||- !Number.isInteger(binding.port) ||- binding.port < 1 ||- binding.port > 65535- ) {- failures.push(- 'exact-host external egress bindings must declare an integer port from 1-65535'- )- continue- }-if (binding.cidr) {- if (!isAllowedExternalEgressCidr(binding.cidr)) {- failures.push(- `CIDR "${binding.cidr}" overlaps private, metadata, link-local, multicast, documentation, or reserved ranges`- )- continue- }cidrs = [binding.cidr]} else if (binding.dns) {- if (!isPublicDnsHostname(binding.dns)) {- failures.push(- `hostname "${binding.dns}" is private, internal, metadata, local, or otherwise disallowed`- )- continue- }// Sliding-window accumulation (issue #299): a single-generation snapshot// pins one rotating IP of a provider that serves 1 A record/response, so// the pod's next resolution lands on an un-pinned IP and egress drops.@@ -2528,8 +2616,8 @@ export class NetworkPolicyReconciler {const liveAnnotations = existingPolicy?.metadata?.annotationsif (resolverFailure) {- if (fresh.policy) freshPolicySnapshots.set(name, fresh.policy)if (resolverFailure.kind === 'permanent') {+ revoke(name, existingPolicy)failures.push(`failed to resolve hostname "${binding.dns}": ${this.errorMessage(resolverFailure.error)}`)@@ -2538,8 +2626,9 @@ export class NetworkPolicyReconciler {if (existingPolicy) {const proof = this.judgeExactHostEgressPolicy(server, name, binding, existingPolicy)if (proof.state !== 'intact') {+ if ('policy' in proof) revoke(name, proof.policy)reconcileFaults.push({- dns: binding.dns,+ binding: binding.dns,err: new Error(`Fail-static proof failed for "${binding.dns}": ${LIVE_EGRESS_FAULT_TEXT[proof.state]}`),@@ -2550,7 +2639,7 @@ export class NetworkPolicyReconciler {}const accumulated = accumulateHostExactHostEgress({fqdn: binding.dns,- port: binding.port,+ port: binding.port!,protocol,resolution: { kind: 'transient' },previousAnnotations: liveAnnotations,@@ -2558,6 +2647,7 @@ export class NetworkPolicyReconciler {config: coreConfig,})if (accumulated.cidrs.length === 0) {+ revoke(name, existingPolicy)failures.push(`failed to resolve hostname "${binding.dns}": ${this.errorMessage(resolverFailure.error)}`)@@ -2565,6 +2655,7 @@ export class NetworkPolicyReconciler {}cidrs = accumulated.cidrs.filter(c => isAllowedExternalEgressCidr(c))if (cidrs.length !== accumulated.cidrs.length) {+ revoke(name, existingPolicy)failures.push(`persisted DNS state for hostname "${binding.dns}" contains a disallowed address`)@@ -2595,13 +2686,13 @@ export class NetworkPolicyReconciler {if (!isCurrent()) returnconst uniqueIps = [...new Set(records.map(record => record.address))].sort()if (uniqueIps.length === 0) {- if (existingPolicy) freshPolicySnapshots.set(name, existingPolicy)+ revoke(name, existingPolicy)failures.push(`hostname "${binding.dns}" resolved to no IPv4 addresses`)continue}const invalidIps = uniqueIps.filter(ip => isIP(ip) !== 4)if (invalidIps.length > 0) {- if (existingPolicy) freshPolicySnapshots.set(name, existingPolicy)+ revoke(name, existingPolicy)failures.push(`hostname "${binding.dns}" resolved invalid IPv4 answer(s): ${invalidIps.join(', ')}`)@@ -2610,7 +2701,7 @@ export class NetworkPolicyReconciler {const freshCidrs = uniqueIps.map(ip => `${ip}/32`)const disallowedCidrs = freshCidrs.filter(cidr => !isAllowedExternalEgressCidr(cidr))if (disallowedCidrs.length > 0) {- if (existingPolicy) freshPolicySnapshots.set(name, existingPolicy)+ revoke(name, existingPolicy)failures.push(`hostname "${binding.dns}" resolved disallowed address(es): ${disallowedCidrs.join(', ')}`)@@ -2631,7 +2722,7 @@ export class NetworkPolicyReconciler {)const accumulated = accumulateHostExactHostEgress({fqdn: binding.dns,- port: binding.port,+ port: binding.port!,protocol,resolution: { kind: 'ok', ips: uniqueIps, ttlSeconds },previousAnnotations: provenPrevious.annotations,@@ -2684,7 +2775,13 @@ export class NetworkPolicyReconciler {}if (!binding.dns) {- const fresh = await this.readFreshExternalEgressPolicy(server, name)+ let fresh: FreshExternalEgressPolicy+ try {+ fresh = await this.readFreshExternalEgressPolicy(server, name)+ } catch (error: unknown) {+ recordUnavailable(binding, name, error)+ continue+ }existingPolicy = fresh.policyownershipRepairRequired = fresh.repairRequired}@@ -2696,64 +2793,32 @@ export class NetworkPolicyReconciler {if (stateAnnotations) {policy.metadata = { ...(policy.metadata ?? {}), annotations: stateAnnotations }}- if (existingPolicy) freshPolicySnapshots.set(name, existingPolicy)// Write only when the enforced rules changed / the policy is new / drifted// (audit F2 no-churn + L1 self-heal), OR when the DNS window is aging and// must be re-persisted (audit M1). A pure timestamp refresh is a no-op.// Route the write through applyOwnedPolicy so #205's ownership assertion// and isCurrent generation fence still guard the mutation.- if (- ownershipRepairRequired ||- this.externalEgressWriteNeeded(existingPolicy, policy) ||- egressRenewalDue ||- egressStateChanged- ) {- await this.applyOwnedPolicy(name, server.namespace, policy, 'external-egress', isCurrent)- } else {- console.log(`[NetPol] External egress "${name}" unchanged — no-op (issue #299 no-churn)`)- }if (!isCurrent()) return- desiredPolicyNames.add(name)- if (resolvedStatus) resolvedEgressIPs.push(resolvedStatus)- if (bindingDnsFailStatic) dnsFailStatic = true- }-- const cleanupCandidates = existingPolicies- .filter(policy => {- const name = policy.metadata?.name- return Boolean(name) && !desiredPolicyNames.has(name!)+ convergences.push({+ name,+ policy,+ observedPolicy: existingPolicy,+ resolvedStatus,+ dnsFailStatic: bindingDnsFailStatic,+ writeRequired:+ ownershipRepairRequired ||+ this.externalEgressWriteNeeded(existingPolicy, policy) ||+ egressRenewalDue ||+ egressStateChanged,})- .map(policy => freshPolicySnapshots.get(policy.metadata?.name ?? '') ?? policy)- for (const [name, snapshot] of freshPolicySnapshots) {- if (- !desiredPolicyNames.has(name) &&- !cleanupCandidates.some(policy => policy.metadata?.name === name)- ) {- cleanupCandidates.push(snapshot)- }}- try {- await this.cleanupExternalEgress(- server.name,- server.namespace,- cleanupCandidates,- async () => isCurrent(),- options.onRevoked+ await cleanup(+ [...dispositions.values()].flatMap(disposition =>+ disposition.state === 'revoke' ? [disposition.policy] : [])- } catch (error) {- if (!isCurrent()) return- await this.writeExternalEgressStatus(- server,- resolvedEgressIPs,- 'False',- 'CleanupFailed',- `Failed to delete stale external egress policies: ${this.errorMessage(error)}`,- isCurrent- )- throw error- }+ )if (!isCurrent()) return// Issue #513: a controller fault outranks rejected input. `failures` means the@@ -2764,7 +2829,7 @@ export class NetworkPolicyReconciler {if (reconcileFaults.length > 0) {const first = reconcileFaults[0]const parts = [- `External egress reconciliation failed on binding "${first.dns}" with a non-DNS error ` ++ `External egress reconciliation failed on binding "${first.binding}" with a non-DNS error ` +`(not a resolver condition, ${LIVE_EGRESS_FAULT_TEXT[first.live]}): ${this.errorMessage(first.err)}`,@@ -2797,6 +2862,52 @@ export class NetworkPolicyReconciler {throw first.err}+ // No additive write is attempted until every observed revocation succeeded+ // and every policy observation completed. A pending observation contributes+ // neither a new permit nor a resolved-at/status proof.+ for (const convergence of convergences) {+ if (!isCurrent()) return+ try {+ // A sibling DNS lookup or cleanup may have taken time since this plan+ // was built. Revalidate even no-ops; changed snapshots require a new+ // observation rather than stamping a newer resourceVersion on old state.+ const fresh = await this.readFreshExternalEgressPolicy(server, convergence.name)+ this.assertExternalEgressSnapshot(+ convergence.name,+ convergence.observedPolicy,+ fresh.policy+ )+ if (!isCurrent()) return+ if (convergence.writeRequired) {+ await this.applyOwnedPolicy(+ convergence.name,+ server.namespace,+ convergence.policy,+ 'external-egress',+ isCurrent,+ convergence.observedPolicy+ )+ } else {+ hccLogger.info('external egress policy unchanged', { policy: convergence.name })+ }+ } catch (error: unknown) {+ if (!isCurrent()) return+ await this.writeExternalEgressStatus(+ server,+ this.sortResolvedEgressIPs(resolvedEgressIPs),+ 'False',+ 'ExternalEgressReconcileFailed',+ `Failed to apply external egress policy "${convergence.name}": ${this.errorMessage(error)}`,+ isCurrent+ )+ throw error+ }+ if (!isCurrent()) return+ dispositions.set(convergence.name, { state: 'retained' })+ if (convergence.resolvedStatus) resolvedEgressIPs.push(convergence.resolvedStatus)+ if (convergence.dnsFailStatic) dnsFailStatic = true+ }+if (failures.length > 0) {const message = failures.join('; ')await this.writeExternalEgressStatus(@@ -2843,12 +2954,12 @@ export class NetworkPolicyReconciler {try {if (deleteAllowed && !(await deleteAllowed())) returnawait this.deleteSafetyPolicySnapshot(namespace, policy, undefined, () => {- console.log(`[NetPol] Deleted external egress policy "${name}"`)+ hccLogger.info('deleted external egress policy', { policy: name })onDeleted?.()})} catch (error: unknown) {if (getErrorCode(error) !== 404) {- console.error(`[NetPol] Failed to delete external egress policy "${name}":`, error)+ hccLogger.error('failed to delete external egress policy', { policy: name, err: error })throw error}// A concurrent actor may already have removed the policy. The safety@@ -2969,15 +3080,15 @@ export class NetworkPolicyReconciler {currentStatus = current.status ?? {}} catch (error) {if (getErrorCode(error) === 404) {- console.warn(- `[NetPol] McpServer "${server.name}" deleted mid-reconcile — external egress status skipped`- )+ hccLogger.warn('McpServer deleted mid-reconcile; external egress status skipped', {+ serverName: server.name,+ })return}- console.warn(- `[NetPol] Failed to read external egress status for "${server.name}" — skipping status update:`,- error- )+ hccLogger.warn('failed to read external egress status; status update skipped', {+ serverName: server.name,+ err: error,+ })throw error}if (!isCurrent()) return@@ -3057,10 +3168,11 @@ export class NetworkPolicyReconciler {body: statusPatch,})} catch (error) {- console.warn(- `[NetPol] Failed to write ExternalEgressReady=${status} on "${server.name}":`,- error- )+ hccLogger.warn('failed to write ExternalEgressReady', {+ serverName: server.name,+ status,+ err: error,+ })throw error}}@@ -3081,7 +3193,7 @@ export class NetworkPolicyReconciler {})return response.items || []} catch (error) {- console.error(`[NetPol] Failed to list policies for context "${contextId}":`, error)+ hccLogger.error('failed to list context policies', { contextId, err: error })throw error}}@@ -3095,7 +3207,7 @@ export class NetworkPolicyReconciler {})return response.items || []} catch (error) {- console.error('[NetPol] Failed to list all context policies:', error)+ hccLogger.error('failed to list all context policies', { err: error })throw error}}@@ -3106,7 +3218,7 @@ export class NetworkPolicyReconciler {const response = await this.networkingApi.listNamespacedNetworkPolicy({ namespace })return response.items || []} catch (error) {- console.error(`[NetPol] Failed to list broad NetworkPolicy inventory in ${namespace}:`, error)+ hccLogger.error('failed to list broad NetworkPolicy inventory', { namespace, err: error })throw error}}@@ -3150,7 +3262,7 @@ export class NetworkPolicyReconciler {})return response.items || []} catch (error) {- console.error('[NetPol] Failed to list all Context egress policies:', error)+ hccLogger.error('failed to list all context egress policies', { err: error })throw error}}@@ -3164,7 +3276,7 @@ export class NetworkPolicyReconciler {})return response.items || []} catch (error) {- console.error('[NetPol] Failed to list all external egress policies:', error)+ hccLogger.error('failed to list all external egress policies', { err: error })throw error}}@@ -3219,7 +3331,8 @@ export class NetworkPolicyReconciler {namespace: string,policy: k8s.V1NetworkPolicy,lane: SafetyInventoryLane,- isCurrent?: () => boolean+ isCurrent?: () => boolean,+ expectedSnapshot?: k8s.V1NetworkPolicy | null): Promise<void> {await applyNetworkPolicy(this.networkingApi,@@ -3228,10 +3341,33 @@ export class NetworkPolicyReconciler {policy,'[NetPol]',isCurrent,- existing => this.assertPolicyOwnership(name, existing, policy, lane)+ existing => {+ this.assertPolicyOwnership(name, existing, policy, lane)+ if (expectedSnapshot !== undefined) {+ this.assertExternalEgressSnapshot(name, expectedSnapshot, existing)+ }+ },+ // Best-effort maintenance may skip a disappeared object. External-egress+ // admission must instead retry: POST409 followed by GET404 applied nothing.+ lane === 'external-egress')}+ private assertExternalEgressSnapshot(+ name: string,+ expected: k8s.V1NetworkPolicy | null,+ actual: k8s.V1NetworkPolicy | null+ ): void {+ if (+ expected?.metadata?.uid !== actual?.metadata?.uid ||+ expected?.metadata?.resourceVersion !== actual?.metadata?.resourceVersion+ ) {+ throw new Error(+ `NetworkPolicy "${name}" changed after the external-egress observation; reconciliation must retry`+ )+ }+ }+private assertPolicyOwnership(name: string,existing: k8s.V1NetworkPolicy,@@ -3287,12 +3423,12 @@ export class NetworkPolicyReconciler {namespace,body: { preconditions: { uid, resourceVersion } },})- console.log(`[NetPol] Deleted safety-inventory policy "${name}" in ${namespace}`)+ hccLogger.info('deleted safety-inventory policy', { policy: name, namespace })onDeleted?.()return true} catch (error: unknown) {if (getErrorCode(error) === 404) {- console.log(`[NetPol] Safety-inventory policy "${name}" already gone`)+ hccLogger.info('safety-inventory policy already gone', { policy: name, namespace })return true}if (getErrorCode(error) === 409 && lostFenceOutcome === 'report') {@@ -3302,15 +3438,17 @@ export class NetworkPolicyReconciler {// adopted. Callers that opt in act on this outcome by declining to// certify, which is the expected racy result rather than a failure.// Callers that cannot act on it keep the loud default.- console.warn(- `[NetPol] Lost the delete fence on safety-inventory policy "${name}" in ${namespace}; it changed identity after the authoritative inventory and was not revoked`- )+ hccLogger.warn('lost safety-inventory delete fence; changed policy was not revoked', {+ policy: name,+ namespace,+ })return false}- console.error(- `[NetPol] Failed identity-bound delete of safety-inventory policy "${name}" in ${namespace}:`,- error- )+ hccLogger.error('failed identity-bound delete of safety-inventory policy', {+ policy: name,+ namespace,+ err: error,+ })throw error}}@@ -3381,9 +3519,10 @@ export class NetworkPolicyReconciler {if (!name) {throw new Error(`Cannot converge a conflicting ${lane} policy without a desired name`)}- console.log(- `[NetPol] Safety policy "${name}" in ${namespace} already exists — converging it to the desired spec`- )+ hccLogger.info('converging existing safety policy to desired spec', {+ policy: name,+ namespace,+ })// Deliberately not delegating to applyOwnedPolicy here: that helper owns// the create-then-conflict flow, so calling it from inside our own// create's catch would issue a second create. Delegating from the top is@@ -3423,7 +3562,7 @@ export class NetworkPolicyReconciler {})return response.items || []} catch (error) {- console.error(`[NetPol] Failed to list egress policies for context "${contextId}":`, error)+ hccLogger.error('failed to list context egress policies', { contextId, err: error })throw error}}@@ -3481,10 +3620,10 @@ export class NetworkPolicyReconciler {await this.deleteSafetyPolicySnapshot(config.rpcProxyNamespace, policy)}} catch (error) {- console.error(- `[NetPol] Failed to delete rpc-proxy egress policies for context "${contextId}":`,- error- )+ hccLogger.error('failed to delete context rpc-proxy egress policies', {+ contextId,+ err: error,+ })throw error}}@@ -3500,10 +3639,7 @@ export class NetworkPolicyReconciler {})return response.items || []} catch (error) {- console.error(- `[NetPol] Failed to list rpc-proxy egress policies for context "${contextId}":`,- error- )+ hccLogger.error('failed to list context rpc-proxy egress policies', { contextId, err: error })throw error}}@@ -3517,7 +3653,7 @@ export class NetworkPolicyReconciler {})return response.items || []} catch (error) {- console.error('[NetPol] Failed to list all rpc-proxy egress policies:', error)+ hccLogger.error('failed to list all rpc-proxy egress policies', { err: error })throw error}}@@ -3535,6 +3671,6 @@ export class NetworkPolicyReconciler {throw new Error(`NetworkPolicy "${name}" has conflicting ownership for legacy cleanup`)}await this.deleteSafetyPolicySnapshot(namespace, existing)- console.log(`[NetPol] Deleted legacy static policy "${name}" in "${namespace}"`)+ hccLogger.info('deleted legacy static policy', { policy: name, namespace })}}diff --git a/host-context-controller/src/utils.ts b/host-context-controller/src/utils.tsindex 3ecea9520..e1bb34ef6 100644--- a/host-context-controller/src/utils.ts+++ b/host-context-controller/src/utils.ts@@ -1,4 +1,5 @@import * as k8s from '@kubernetes/client-node'+import { hccLogger } from './logger'import { writeSkipsTotal, writesTotal } from './metrics'/** Extract HTTP status code from a K8s client error (handles both error formats). */@@ -41,6 +42,8 @@ export async function replaceWithConflictRetry<validateExisting?: (existing: T) => void/** Rechecked immediately before every Kubernetes write attempt. */mutationAllowed?: () => boolean+ /** Admission-sensitive callers must not treat an absent object as applied. */+ missingIsError?: booleanmaxAttempts?: number}): Promise<void> {const {@@ -54,6 +57,7 @@ export async function replaceWithConflictRetry<isUpToDate,validateExisting,mutationAllowed,+ missingIsError = false,maxAttempts = 3,} = optsfor (let attempt = 1; attempt <= maxAttempts; attempt++) {@@ -62,7 +66,11 @@ export async function replaceWithConflictRetry<existing = await read()} catch (err) {if (getErrorCode(err) === 404) {- console.warn(`${logPrefix} ${description} disappeared (404); skipping replace`)+ if (missingIsError) throw err+ hccLogger.warn('Kubernetes resource disappeared; skipping replace', {+ scope: logPrefix,+ description,+ })return}throw err@@ -83,8 +91,7 @@ export async function replaceWithConflictRetry<try {await replace(next)writesTotal.inc({ kind: next.kind ?? 'unknown' })- const suffix = attempt > 1 ? ` (after ${attempt} attempts)` : ''- console.log(`${logPrefix} Updated ${description}${suffix}`)+ hccLogger.info('Kubernetes resource updated', { scope: logPrefix, description, attempt })return} catch (err) {if (getErrorCode(err) === 409 && attempt < maxAttempts) {@@ -448,12 +455,13 @@ export async function applyNetworkPolicy(policy: k8s.V1NetworkPolicy,logPrefix = '[NetPol]',mutationAllowed?: () => boolean,- validateExisting?: (existing: k8s.V1NetworkPolicy) => void+ validateExisting?: (existing: k8s.V1NetworkPolicy) => void,+ missingIsError = false): Promise<void> {if (mutationAllowed && !mutationAllowed()) returntry {await api.createNamespacedNetworkPolicy({ namespace, body: policy })- console.log(`${logPrefix} Created policy "${name}" in ${namespace}`)+ hccLogger.info('NetworkPolicy created', { scope: logPrefix, policy: name, namespace })return} catch (error: unknown) {if (getErrorCode(error) !== 409) {@@ -468,6 +476,7 @@ export async function applyNetworkPolicy(replace: body => api.replaceNamespacedNetworkPolicy({ name, namespace, body }),mutationAllowed,validateExisting,+ missingIsError,isUpToDate: networkPolicyMatchesDesired,})}diff --git a/host-context-controller/test/__fixtures__/updatedLogs.ts b/host-context-controller/test/__fixtures__/updatedLogs.tsindex eb765dcb1..1e29abfaa 100644--- a/host-context-controller/test/__fixtures__/updatedLogs.ts+++ b/host-context-controller/test/__fixtures__/updatedLogs.ts@@ -3,6 +3,21 @@ import { vi } from 'vitest'/** Shared log filter for the no-op-gate suites. Every needle must match. */export function updatedLogs(log: ReturnType<typeof vi.spyOn>, ...needles: string[]): string[] {return log.mock.calls- .map((call: unknown[]) => String(call[0]))+ .map((call: unknown[]) => {+ const line = String(call[0])+ // Some untouched producers still use the legacy presentation. Project the+ // structured shared-helper event into the same test vocabulary so both+ // no-op and real-write assertions keep measuring actual emitted events.+ try {+ const entry = JSON.parse(line)+ if (entry.msg === 'Kubernetes resource updated' && typeof entry.description === 'string') {+ const suffix = entry.attempt > 1 ? ` (after ${entry.attempt} attempts)` : ''+ return `${entry.scope} Updated ${entry.description}${suffix}`+ }+ } catch {+ /* Legacy plain-text producer. */+ }+ return line+ }).filter((line: string) => needles.every(needle => line.includes(needle)))}diff --git a/scripts/e2e/_lib/wrc-egress-gate.cjs b/scripts/e2e/_lib/wrc-egress-gate.cjsnew file mode 100644index 000000000..548bde157--- /dev/null+++ b/scripts/e2e/_lib/wrc-egress-gate.cjs@@ -0,0 +1,1222 @@+#!/usr/bin/env node+'use strict'++// Product/controller E2E, not a browser journey and not a T2 attestation.+// All mutation goes through the explicit-context client and inherited lease.+const fs = require('node:fs')+const path = require('node:path')+const dns = require('node:dns/promises')+const { BlockList, isIPv4 } = require('node:net')+const { randomUUID } = require('node:crypto')+const { execFileSync } = require('node:child_process')+const { assertPolicy } = require('./wrc-egress-proof.cjs')+const {+ Journal,+ journalPath,+ readJournal,+ assertJournalBinding,+ createOwned,+ deleteOwned,+ waitFor,+ createCommandRunner,+ createKubeClient,+ invariant,+ RUN_LABEL,+ resourceIdentity,+} = require('./wrc-egress-lifecycle.cjs')++const STATE = 'clerum.io/egress-fqdn-state'+const LOOPBACK = [127, 0, 0, 1].join('.')+const endpointHost = 'example.com' // Existing mock-MCP fixture public endpoint contract.+const deploymentReady = object =>+ Boolean(+ object &&+ object.spec?.replicas > 0 &&+ object.status?.observedGeneration >= object.metadata.generation &&+ object.status.readyReplicas === object.spec.replicas &&+ object.status.availableReplicas === object.spec.replicas+ )+const same = (left, right) => JSON.stringify(left) === JSON.stringify(right)+const id = (kind, namespace, name) => ({ kind, namespace, name })++// Called after the complete policy oracle succeeds. Fresh DNS traffic alone+// cannot satisfy recovery; every tuple needs a newer persisted observation.+function policyObservation(policy, newerThan) {+ const entries = JSON.parse(policy.metadata.annotations[STATE])+ invariant(+ entries.length > 0 &&+ entries.every(+ entry => Number.isFinite(entry.lastObservedAt) && Number.isFinite(entry.expiresAt)+ ),+ 'MISSING_ACCEPTED_DNS_OBSERVATION'+ )+ if (newerThan !== undefined && !entries.every(entry => entry.lastObservedAt > newerThan))+ return null+ return {+ lastAcceptedAt: Math.max(...entries.map(entry => entry.lastObservedAt)),+ latestExpiry: Math.max(...entries.map(entry => entry.expiresAt)),+ }+}++function acceptPolicyIntentChange(lanes, before, after, recipeUid, targets) {+ invariant(+ after.metadata?.uid === recipeUid &&+ before.metadata?.uid === recipeUid &&+ after.metadata.generation > before.metadata.generation,+ 'HOST_MIGRATION_NOT_APPLIED'+ )+ for (const lane of lanes) {+ const target = targets.find(target => target.lane === (lane.lane === 'ui' ? 'worker' : 'ui'))+ invariant(target, 'MISSING_MIGRATION_TARGET')+ lane.target = target+ lane.uid = undefined+ }+}++// A test setup restriction, not an alternate production address classifier.+const blocked = new BlockList()+for (const cidr of [+ '0.0.0.0/8',+ '10.0.0.0/8',+ '100.64.0.0/10',+ '127.0.0.0/8',+ '169.254.0.0/16',+ '172.16.0.0/12',+ '192.0.0.0/24',+ '192.0.2.0/24',+ '192.31.196.0/24',+ '192.52.193.0/24',+ '192.88.99.0/24',+ '192.168.0.0/16',+ '192.175.48.0/24',+ '198.18.0.0/15',+ '198.51.100.0/24',+ '203.0.113.0/24',+ '224.0.0.0/4',+ '240.0.0.0/4',+]) {+ const [address, prefix] = cidr.split('/')+ blocked.addSubnet(address, Number(prefix), 'ipv4')+}++const trafficCode = `+const net=require('node:net'),https=require('node:https');+const [ip,portText,host,mode]=process.argv.slice(1); const port=Number(portText);+if(mode==='https') {+ const request=https.get({hostname:ip,port,servername:host,path:'/',agent:false,+ headers:{host,connection:'close'}},response=>{+ let body=''; response.on('data',chunk=>{body+=chunk;if(body.length>65536)request.destroy(new Error('body limit'))});+ response.once('aborted',()=>{process.exitCode=43;});+ response.once('error',()=>{process.exitCode=43;});+ response.once('end',()=>{if(response.statusCode===200&&body.includes('Example Domain')){+ process.stdout.write(JSON.stringify({allowed:true,contentVerified:true}));+ }else process.exitCode=44;});+ });+ const timer=setTimeout(()=>request.destroy(new Error('deadline')),5000);+ request.once('close',()=>clearTimeout(timer)); request.once('error',()=>{process.exitCode=43;});+} else {+ const socket=net.connect({host:ip,port});+ const timer=setTimeout(()=>{socket.destroy();process.exitCode=42;},2500);+ socket.once('connect',()=>{clearTimeout(timer);socket.destroy();process.stdout.write('connected');});+ socket.once('error',()=>{clearTimeout(timer);process.exitCode=43;});+}`++const businessCode = `+const {Client}=require('@modelcontextprotocol/sdk/client/index.js');+const {StreamableHTTPClientTransport}=require('@modelcontextprotocol/sdk/client/streamableHttp.js');+const [url,key,value,operation]=process.argv.slice(1);+const client=new Client({name:'wrc-egress-business-witness',version:'1.0.0'},{capabilities:{}});+const timer=setTimeout(()=>process.exit(45),10000);+(async()=>{try{+ await client.connect(new StreamableHTTPClientTransport(new URL(url)));+ if(operation==='record'){+ const stored=await client.callTool({name:'record',arguments:{key,value}});+ if(stored.isError)throw new Error('record failed');+ }+ const response=await client.callTool({name:'recall',arguments:{key}});+ const payload=JSON.parse(response.content.find(item=>item.type==='text').text);+ if(response.isError||payload.key!==key||payload.value!==value)throw new Error('business state lost');+ process.stdout.write(JSON.stringify({businessVerified:true}));+}finally{await client.close();clearTimeout(timer);}})().catch(()=>{process.exitCode=46;});`++const dnsControlCode = `+const http=require('node:http'); const [host,route,method]=process.argv.slice(1);+const request=http.request({host,port:8090,path:route,method},response=>{+ let body='';response.on('data',chunk=>{body+=chunk;if(body.length>262144)request.destroy()});+ response.once('end',()=>{if(response.statusCode!==200)process.exitCode=47;else process.stdout.write(body);});+});+const timer=setTimeout(()=>request.destroy(new Error('deadline')),5000);+request.once('close',()=>clearTimeout(timer));request.once('error',()=>{process.exitCode=48;});request.end();`++function fixtureObjects(config, runId, wrcImage, answerIp, upstream) {+ const suffix = runId.slice(0, 12)+ const names = {+ proxy: `e2e-wrc-dns-${suffix}`,+ script: `e2e-wrc-dns-script-${suffix}`,+ policy: `e2e-wrc-dns-policy-${suffix}`,+ fromWrc: `e2e-wrc-from-${suffix}`,+ recipe: `e2e-wrc-${suffix}`,+ canary: `e2e-wrc-canary-${suffix}`,+ }+ const labels = { 'e2e.clerum.io/suite': 'wrc-egress-degradation', [RUN_LABEL]: runId }+ const metadata = (name, namespace = config.wrcNamespace) => ({ name, namespace, labels })+ const targets = [+ { lane: 'ui', fqdn: `ui-${suffix}.example.com`, ip: answerIp },+ { lane: 'worker', fqdn: `worker-${suffix}.example.com`, ip: answerIp },+ { lane: 'canary', fqdn: `canary-${suffix}.example.com`, ip: answerIp },+ ]+ const securityContext = {+ allowPrivilegeEscalation: false,+ capabilities: { drop: ['ALL'] },+ readOnlyRootFilesystem: true,+ runAsNonRoot: true,+ runAsUser: 1000,+ runAsGroup: 1000,+ seccompProfile: { type: 'RuntimeDefault' },+ }+ const ports = [+ { protocol: 'UDP', port: 53 },+ { protocol: 'TCP', port: 53 },+ { protocol: 'UDP', port: 8053 },+ { protocol: 'TCP', port: 8053 },+ ]+ const proxyPeer = { podSelector: { matchLabels: { app: names.proxy } } }+ const objects = [+ {+ apiVersion: 'v1',+ kind: 'ConfigMap',+ metadata: metadata(names.script),+ data: {+ 'server.cjs': fs.readFileSync(+ path.join(config.repository, 'tests/e2e/fixtures/wrc-egress-dns-proxy/server.cjs'),+ 'utf8'+ ),+ },+ },+ {+ apiVersion: 'networking.k8s.io/v1',+ kind: 'NetworkPolicy',+ metadata: metadata(names.policy),+ spec: {+ podSelector: proxyPeer.podSelector,+ policyTypes: ['Ingress', 'Egress'],+ ingress: [+ { from: [{ podSelector: { matchLabels: { app: config.wrcDeployment } } }], ports },+ ],+ egress: [+ {+ to: [+ {+ namespaceSelector: {+ matchLabels: { 'kubernetes.io/metadata.name': 'kube-system' },+ },+ podSelector: { matchLabels: { 'k8s-app': 'kube-dns' } },+ },+ ],+ ports: ports.slice(0, 2),+ },+ ],+ },+ },+ {+ apiVersion: 'networking.k8s.io/v1',+ kind: 'NetworkPolicy',+ metadata: metadata(names.fromWrc),+ spec: {+ podSelector: { matchLabels: { app: config.wrcDeployment } },+ policyTypes: ['Egress'],+ egress: [{ to: [proxyPeer], ports }],+ },+ },+ {+ apiVersion: 'v1',+ kind: 'Service',+ metadata: metadata(names.proxy),+ spec: {+ selector: { app: names.proxy },+ ports: [+ { name: 'dns-udp', protocol: 'UDP', port: 53, targetPort: 8053 },+ { name: 'dns-tcp', protocol: 'TCP', port: 53, targetPort: 8053 },+ ],+ },+ },+ {+ apiVersion: 'apps/v1',+ kind: 'Deployment',+ metadata: metadata(names.proxy),+ spec: {+ replicas: 1,+ selector: { matchLabels: { app: names.proxy } },+ template: {+ metadata: { labels: { ...labels, app: names.proxy } },+ spec: {+ automountServiceAccountToken: false,+ terminationGracePeriodSeconds: 1,+ containers: [+ {+ name: 'dns-proxy',+ image: wrcImage,+ imagePullPolicy: 'IfNotPresent',+ command: ['node', '/fixture/server.cjs'],+ env: [+ { name: 'DNS_TARGETS_JSON', value: JSON.stringify(targets) },+ { name: 'DNS_TTL_SECONDS', value: '5' },+ { name: 'UPSTREAM_DNS', value: upstream },+ ],+ ports: [+ { name: 'dns-udp', containerPort: 8053, protocol: 'UDP' },+ { name: 'dns-tcp', containerPort: 8053, protocol: 'TCP' },+ ],+ readinessProbe: {+ tcpSocket: { port: 8053 },+ periodSeconds: 1,+ failureThreshold: 10,+ },+ volumeMounts: [{ name: 'fixture', mountPath: '/fixture', readOnly: true }],+ resources: {+ requests: { cpu: '5m', memory: '20Mi' },+ limits: { cpu: '100m', memory: '96Mi' },+ },+ securityContext,+ },+ ],+ volumes: [{ name: 'fixture', configMap: { name: names.script } }],+ },+ },+ },+ },+ ]+ const workload = name => ({+ id: name,+ type: 'deployment',+ image: 'clerum/mock-mcp-server:test',+ port: 3000,+ healthCheck: { type: 'http', path: '/', port: 3001 },+ })+ const bindings = fqdn => [80, 443].map(port => ({ dns: fqdn, port, protocol: 'TCP' }))+ const recipe = {+ apiVersion: 'clerum.io/v1alpha1',+ kind: 'WorkflowRecipe',+ metadata: metadata(names.recipe, config.recipeNamespace),+ spec: {+ contextRef: 'context1',+ description: 'WRC authorization continuity fixture',+ workloads: [+ workload('frontend'),+ { ...workload('worker'), egressBindings: bindings(targets[1].fqdn) },+ ],+ ui: {+ workloadRef: 'frontend',+ port: 3000,+ title: 'WRC continuity',+ defaultPath: '/mcp',+ egress: {+ internal: [+ { workloadRef: 'worker', port: 3000 },+ { workloadRef: 'worker', port: 3001 },+ ],+ external: [80, 443].map(port => ({ fqdn: targets[0].fqdn, port })),+ },+ },+ security: { isolationLevel: 'minimal' },+ },+ }+ const canary = {+ apiVersion: 'clerum.io/v1alpha1',+ kind: 'WorkflowRecipe',+ metadata: metadata(names.canary, config.recipeNamespace),+ spec: {+ contextRef: 'context1',+ description: 'Independent WRC business witness',+ workloads: [{ ...workload('witness'), egressBindings: bindings(targets[2].fqdn) }],+ security: { isolationLevel: 'minimal' },+ },+ }+ return { names, targets, objects, recipe, canary }+}++async function readyPod(kube, namespace, recipe, workload, timeoutMs = 120_000) {+ let pod+ await waitFor('ready-fixture-pod', timeoutMs, async (remaining, signal) => {+ const pods = await kube.list(+ 'pods',+ namespace,+ `clerum.io/recipe=${recipe},clerum.io/workload=${workload}`,+ remaining,+ signal+ )+ const ready = pods.filter(+ item =>+ !item.metadata.deletionTimestamp &&+ item.status?.conditions?.some(+ condition => condition.type === 'Ready' && condition.status === 'True'+ )+ )+ invariant(ready.length <= 1, 'AMBIGUOUS_FIXTURE_POD')+ pod = ready[0]+ return Boolean(pod?.metadata.uid)+ })+ return pod+}++const identityKey = identity => `${identity.kind}/${identity.namespace}/${identity.name}`+const checkedIdentity = identity => resourceIdentity({ kind: identity.kind, metadata: identity })++function uiChildIdentities(ui) {+ const identities = []+ if (ui.physicalName !== null) {+ for (const kind of ['Deployment', 'Service']) {+ identities.push(checkedIdentity({ kind, namespace: ui.namespace, name: ui.physicalName }))+ }+ }+ identities.push(+ checkedIdentity({+ kind: 'NetworkPolicy',+ namespace: ui.namespace,+ name: `ui-egress-${ui.parent.name}`,+ })+ )+ return identities+}++function uiCleanupRecord(journal) {+ const ui = journal.state.uiCleanup+ // Journals from before UI capture cannot silently certify cross-namespace+ // teardown once a recipe was attempted. They require explicit inspection.+ invariant(+ ui !== undefined || !journal.state.resources.some(entry => entry.kind === 'WorkflowRecipe'),+ 'UI_CLEANUP_RECORD_MISSING'+ )+ if (ui === null || ui === undefined) return null+ invariant(+ ui.runId === journal.state.runId &&+ ui.parent?.kind === 'WorkflowRecipe' &&+ typeof ui.workload === 'string' &&+ /^[a-z0-9][a-z0-9.-]*$/.test(ui.workload) &&+ Array.isArray(ui.children) &&+ ui.children.length <= 3,+ 'INVALID_UI_CLEANUP_RECORD'+ )+ checkedIdentity(ui.parent)+ const allowed = new Set(uiChildIdentities(ui).map(identityKey))+ const seen = new Set()+ for (const child of ui.children) {+ const key = identityKey(checkedIdentity(child))+ invariant(+ allowed.has(key) &&+ !seen.has(key) &&+ (child.uid === null || (typeof child.uid === 'string' && child.uid.length > 0)),+ 'INVALID_UI_CHILD_RECORD'+ )+ seen.add(key)+ }+ return ui+}++function assertUiChild(object, identity, ui) {+ invariant(+ identityKey(resourceIdentity(object)) === identityKey(identity) &&+ typeof object.metadata.uid === 'string' &&+ object.metadata.uid.length > 0 &&+ typeof object.metadata.resourceVersion === 'string' &&+ object.metadata.resourceVersion.length > 0,+ 'UI_CHILD_IDENTITY_UNPROVEN'+ )+ const labels = object.metadata.labels ?? {}+ invariant(+ labels['clerum.io/managed-by'] === 'workflow-recipes' &&+ labels['clerum.io/recipe'] === ui.parent.name,+ 'UI_CHILD_OWNER_UNPROVEN'+ )+ invariant(+ (labels['clerum.io/recipe-name'] === undefined ||+ labels['clerum.io/recipe-name'] === ui.parent.name) &&+ (labels['clerum.io/recipe-namespace'] === undefined ||+ labels['clerum.io/recipe-namespace'] === ui.parent.namespace) &&+ (labels['clerum.io/sandbox-ui'] === undefined || labels['clerum.io/sandbox-ui'] === 'true'),+ 'UI_CHILD_SCOPE_CONFLICT'+ )+ if (identity.kind !== 'NetworkPolicy') {+ invariant(labels['clerum.io/workload'] === ui.workload, 'UI_CHILD_WORKLOAD_UNPROVEN')+ }+ // Deployment metadata may omit sandbox-ui labels; its physical status name+ // plus ordinary WRC recipe/workload labels supply the ownership proof.+ if (identity.kind !== 'Deployment') {+ invariant(+ labels['clerum.io/recipe-name'] === ui.parent.name &&+ labels['clerum.io/recipe-namespace'] === ui.parent.namespace,+ 'UI_CHILD_SCOPE_UNPROVEN'+ )+ }+ if (identity.kind === 'Service') {+ invariant(labels['clerum.io/sandbox-ui'] === 'true', 'UI_SERVICE_SCOPE_UNPROVEN')+ }+ for (const owner of object.metadata.ownerReferences ?? []) {+ invariant(+ owner.kind === 'WorkflowRecipe' &&+ owner.name === ui.parent.name &&+ owner.uid === ui.parentUid,+ 'UI_CHILD_HAS_FOREIGN_OWNER'+ )+ }+}++async function captureUiChildren(kube, journal) {+ const ui = uiCleanupRecord(journal)+ if (!ui) return null+ const parentIdentity = checkedIdentity(ui.parent)+ const parentRecord = journal.state.resources.find(+ entry => identityKey(entry) === identityKey(parentIdentity)+ )+ if (ui.parentUid !== null) {+ invariant(parentRecord?.uid === ui.parentUid, 'UI_PARENT_RECORD_MISMATCH')+ }+ const parent = await kube.get(parentIdentity)+ // On recovery, absence cannot authorize adopting a same-name child. Only+ // identities durably captured while the exact parent existed remain usable.+ if (!parent) return ui+ invariant(+ parentRecord?.uid &&+ identityKey(resourceIdentity(parent)) === identityKey(parentIdentity) &&+ parent.metadata.uid === parentRecord.uid &&+ parent.metadata.labels?.[RUN_LABEL] === ui.runId &&+ parent.spec?.ui?.workloadRef === ui.workload &&+ parent.spec.workloads?.some(+ workload => workload.id === ui.workload && workload.type === 'deployment'+ ),+ 'UI_PARENT_OWNERSHIP_UNPROVEN'+ )+ ui.parentUid = parentRecord.uid+ const physicalName = parent.status?.workloadInstances?.[ui.workload]+ if (physicalName !== undefined) {+ // Status is persisted before WRC creates workloads. Validate it before any+ // CLI argument; never substitute the raw logical workload ID when absent.+ checkedIdentity({ kind: 'Deployment', namespace: ui.namespace, name: physicalName })+ invariant(+ ui.physicalName === null || ui.physicalName === physicalName,+ 'UI_PHYSICAL_NAME_CHANGED'+ )+ ui.physicalName = physicalName+ }+ for (const identity of uiChildIdentities(ui)) {+ const recorded = ui.children.find(child => identityKey(child) === identityKey(identity))+ const object = await kube.get(identity)+ if (object) {+ assertUiChild(object, identity, ui)+ invariant(!recorded || recorded.uid === object.metadata.uid, 'UI_CHILD_IDENTITY_CHANGED')+ }+ if (!recorded) ui.children.push({ ...identity, uid: object?.metadata.uid ?? null })+ journal.save()+ }+ return ui+}++async function cleanupUiChildren(kube, ui) {+ invariant(!(await kube.get(checkedIdentity(ui.parent))), 'UI_PARENT_STILL_PRESENT')+ for (const child of ui.children) {+ const identity = checkedIdentity(child)+ const live = await kube.get(identity)+ if (!live) continue+ invariant(child.uid && live.metadata.uid === child.uid, 'UI_CHILD_IDENTITY_CHANGED')+ assertUiChild(live, identity, ui)+ try {+ await kube.delete(identity, {+ uid: child.uid,+ resourceVersion: live.metadata.resourceVersion,+ })+ } catch (error) {+ if (await kube.get(identity)) throw error+ }+ await waitFor('UI-child-absence', 90_000, async (remaining, signal) => {+ const current = await kube.get(identity, remaining, signal)+ if (!current) return true+ invariant(current.metadata.uid === child.uid, 'UI_CHILD_RECREATED_DURING_CLEANUP')+ return false+ })+ }+ // WRC's best-effort finalizer can delete a Deployment in Background. Its+ // ReplicaSets/Pods must also disappear before a restored verdict is valid.+ // Discovery is read-only: an uncaptured managed child is never adopted here.+ await waitFor('UI-resource-absence', 90_000, async (remaining, signal) => {+ const deadline = Date.now() + remaining+ let empty = true+ for (const kind of ['deployments', 'services', 'networkpolicies', 'replicasets', 'pods']) {+ const objects = await kube.list(+ kind,+ ui.namespace,+ `clerum.io/recipe=${ui.parent.name}`,+ deadline - Date.now(),+ signal+ )+ for (const object of objects) {+ if (!['replicasets', 'pods'].includes(kind)) {+ const identity = resourceIdentity(object)+ const recorded = ui.children.find(child => identityKey(child) === identityKey(identity))+ invariant(+ recorded?.uid && recorded.uid === object.metadata.uid,+ 'UI_CHILD_IDENTITY_NOT_CAPTURED'+ )+ }+ empty = false+ }+ }+ return empty+ })+}++async function restore(kube, journal) {+ await kube.setCleanup(true)+ journal.state.phase = 'restoring'+ journal.save()+ const failures = []+ let ui+ const wrc = journal.state.wrc+ if (wrc?.attempted) {+ try {+ const live = await kube.get(wrc.identity)+ invariant(+ live?.metadata.uid === wrc.uid && live.spec.replicas === wrc.replicas,+ 'WRC_RESTORE_IDENTITY_CHANGED'+ )+ const current = live.spec.template.spec+ const alreadyRestored =+ (current.dnsPolicy ?? 'ClusterFirst') === wrc.originalPolicy &&+ (current.dnsConfig ?? null) === null+ if (!alreadyRestored) {+ invariant(+ current.dnsPolicy === 'None' && same(current.dnsConfig, wrc.injectedConfig),+ 'WRC_DNS_CHANGED_BY_ANOTHER_OWNER'+ )+ await kube.patch(wrc.identity, [+ { op: 'test', path: '/metadata/uid', value: wrc.uid },+ { op: 'test', path: '/metadata/resourceVersion', value: live.metadata.resourceVersion },+ { op: 'replace', path: '/spec/template/spec/dnsPolicy', value: wrc.originalPolicy },+ { op: 'remove', path: '/spec/template/spec/dnsConfig' },+ ])+ }+ await kube.rollout(wrc.identity.namespace, wrc.identity.name)+ const restored = await kube.get(wrc.identity)+ invariant(+ restored?.metadata.uid === wrc.uid &&+ restored.spec.replicas === wrc.replicas &&+ deploymentReady(restored) &&+ (restored.spec.template.spec.dnsPolicy ?? 'ClusterFirst') === wrc.originalPolicy &&+ (restored.spec.template.spec.dnsConfig ?? null) === null,+ 'WRC_RESTORE_NOT_PROVEN'+ )+ journal.state.wrc.attempted = false+ journal.save()+ } catch (error) {+ failures.push(error)+ }+ }+ // Keep the DNS fixture if restoration is uncertain; deleting it would remove+ // the nameserver from a WRC Deployment that may still depend on it.+ if (failures.length === 0) {+ try {+ ui = await captureUiChildren(kube, journal)+ } catch (error) {+ failures.push(error)+ }+ }+ if (failures.length === 0) {+ for (const entry of [...journal.state.resources].reverse()) {+ try {+ await deleteOwned(kube, journal, entry)+ } catch (error) {+ failures.push(error)+ }+ }+ }+ if (failures.length === 0 && ui) {+ try {+ await cleanupUiChildren(kube, ui)+ } catch (error) {+ failures.push(error)+ }+ }+ if (failures.length) {+ journal.state.phase = 'recovery-required'+ journal.save()+ throw new AggregateError(failures, 'WRC_EGRESS_RECOVERY_REQUIRED')+ }+ journal.state.phase = 'restored'+ journal.save()+ journal.remove()+}++async function protectedRun(kube, journal, exercise, cancellation) {+ let original+ try {+ await exercise()+ } catch (error) {+ original = error+ }+ try {+ await restore(kube, journal)+ } catch (cleanupError) {+ throw new AggregateError(+ [...(original ? [original] : []), cleanupError],+ 'WRC_EGRESS_RECOVERY_REQUIRED'+ )+ }+ if (cancellation.signal.aborted) throw cancellation.signal.reason+ if (original) throw original+}++async function exercise(config, kube, journal) {+ const wrcIdentity = id('Deployment', config.wrcNamespace, config.wrcDeployment)+ const wrc = await kube.get(wrcIdentity)+ invariant(deploymentReady(wrc) && wrc.spec.replicas === 1, 'WRC_NOT_READY_OR_NOT_SINGLE_WRITER')+ invariant(+ (wrc.spec.template.spec.dnsPolicy ?? 'ClusterFirst') === 'ClusterFirst' &&+ (wrc.spec.template.spec.dnsConfig ?? null) === null,+ 'WRC_DNS_NOT_BASELINE'+ )+ const image = wrc.spec.template.spec.containers.find(+ container => container.name === 'workflow-recipes'+ )?.image+ invariant(image, 'MISSING_WRC_IMAGE')+ const upstream = (await kube.get(id('Service', 'kube-system', 'kube-dns')))?.spec.clusterIP+ invariant(upstream, 'MISSING_CLUSTER_DNS')+ const resolver = new dns.Resolver({ timeout: 5000, tries: 1 })+ const answers = await resolver.resolve4(endpointHost)+ const answerIp = answers.find(address => isIPv4(address) && !blocked.check(address, 'ipv4'))+ invariant(answerIp, 'PUBLIC_ENDPOINT_UNAVAILABLE')+ const fixture = fixtureObjects(config, journal.state.runId, image, answerIp, upstream)+ const { names, targets } = fixture+ for (const object of fixture.objects) await createOwned(kube, journal, object)+ await kube.rollout(config.wrcNamespace, names.proxy, 90_000)+ const proxy = (await kube.list('pods', config.wrcNamespace, `app=${names.proxy}`)).find(+ pod =>+ !pod.metadata.deletionTimestamp &&+ pod.status?.conditions?.some(c => c.type === 'Ready' && c.status === 'True')+ )+ invariant(proxy?.metadata.uid, 'DNS_PROXY_NOT_READY')+ const proxyIp = (await kube.get(id('Service', config.wrcNamespace, names.proxy)))?.spec.clusterIP+ invariant(proxyIp, 'DNS_PROXY_ADDRESS_MISSING')+ const dnsControl = async (route = '/state', method = 'GET', remaining = 15_000, signal) => {+ const state = JSON.parse(+ await kube.exec(+ config.wrcNamespace,+ proxy.metadata.name,+ dnsControlCode,+ [LOOPBACK, route, method],+ Math.min(15_000, remaining),+ signal+ )+ )+ invariant(state.fault === null && Array.isArray(state.lanes), 'DNS_FIXTURE_FAULT')+ return state+ }+ const setAffectedDns = async mode => {+ await dnsControl(`/mode/ui/${mode}`, 'POST')+ await dnsControl(`/mode/worker/${mode}`, 'POST')+ }+ await dnsControl()+ const injectedConfig = {+ nameservers: [proxyIp],+ options: [+ { name: 'ndots', value: '1' },+ { name: 'timeout', value: '2' },+ { name: 'attempts', value: '1' },+ ],+ }+ journal.state.wrc = {+ identity: wrcIdentity,+ uid: wrc.metadata.uid,+ replicas: 1,+ originalPolicy: 'ClusterFirst',+ injectedConfig,+ attempted: true,+ }+ journal.state.phase = 'DNS-injected'+ journal.save()+ const beforeInjection = await kube.get(wrcIdentity)+ invariant(+ beforeInjection?.metadata.uid === wrc.metadata.uid &&+ deploymentReady(beforeInjection) &&+ (beforeInjection.spec.template.spec.dnsPolicy ?? 'ClusterFirst') === 'ClusterFirst' &&+ (beforeInjection.spec.template.spec.dnsConfig ?? null) === null,+ 'WRC_CHANGED_BEFORE_INJECTION'+ )+ await kube.patch(wrcIdentity, [+ { op: 'test', path: '/metadata/uid', value: wrc.metadata.uid },+ {+ op: 'test',+ path: '/metadata/resourceVersion',+ value: beforeInjection.metadata.resourceVersion,+ },+ { op: 'add', path: '/spec/template/spec/dnsPolicy', value: 'None' },+ { op: 'add', path: '/spec/template/spec/dnsConfig', value: injectedConfig },+ ])+ await kube.rollout(config.wrcNamespace, config.wrcDeployment)+ journal.state.uiCleanup = {+ parent: resourceIdentity(fixture.recipe),+ parentUid: null,+ runId: journal.state.runId,+ namespace: config.uiNamespace,+ workload: fixture.recipe.spec.ui.workloadRef,+ physicalName: null,+ children: [],+ }+ journal.save()+ const recipe = await createOwned(kube, journal, fixture.recipe)+ await createOwned(kube, journal, fixture.canary)+ const recipeIdentity = id('WorkflowRecipe', config.recipeNamespace, names.recipe)+ const active = async (name, remaining, signal) => {+ const current = await kube.get(+ id('WorkflowRecipe', config.recipeNamespace, name),+ remaining,+ signal+ )+ return current?.status?.phase === 'active'+ }+ await waitFor('active-recipes', 180_000, async (remaining, signal) => {+ const deadline = Date.now() + remaining+ return (+ (await active(names.recipe, remaining, signal)) &&+ (await active(names.canary, deadline - Date.now(), signal))+ )+ })+ const frontend = await readyPod(kube, config.uiNamespace, names.recipe, 'frontend')+ const worker = await readyPod(kube, config.recipeNamespace, names.recipe, 'worker')+ const canary = await readyPod(kube, config.recipeNamespace, names.canary, 'witness')+ const lanes = [+ {+ lane: 'ui',+ workload: 'frontend',+ namespace: config.uiNamespace,+ pod: frontend,+ policy: `ui-egress-${names.recipe}`,+ target: targets[0],+ },+ {+ lane: 'workload',+ workload: 'worker',+ namespace: config.recipeNamespace,+ pod: worker,+ policy: `wl-egress-${names.recipe}-worker`,+ target: targets[1],+ },+ ]+ const policiesMatch = async (ports, internalPorts, remaining = 30_000, signal) => {+ const deadline = Date.now() + remaining+ for (const lane of lanes) {+ const policy = await kube.get(+ id('NetworkPolicy', lane.namespace, lane.policy),+ deadline - Date.now(),+ signal+ )+ if (!policy) return false+ try {+ assertPolicy(policy, {+ name: lane.policy,+ namespace: lane.namespace,+ recipe: names.recipe,+ recipeNamespace: config.recipeNamespace,+ lane: lane.lane,+ workload: lane.workload,+ podLabels: lane.pod.metadata.labels,+ uid: lane.uid,+ tuples: ports.map(port => ({+ fqdn: lane.target.fqdn,+ ip: answerIp,+ port,+ protocol: 'TCP',+ })),+ internal:+ lane.lane === 'ui'+ ? internalPorts.map(port => ({+ namespace: config.recipeNamespace,+ workload: 'worker',+ port,+ protocol: 'TCP',+ }))+ : [],+ })+ } catch {+ return false+ }+ const observation = policyObservation(policy, lane.recoveryAfter)+ if (!observation) return false+ Object.assign(lane, observation)+ lane.uid = policy.metadata.uid+ }+ return true+ }+ await waitFor('baseline-policy-proof', 120_000, (remaining, signal) =>+ policiesMatch([80, 443], [3000, 3001], remaining, signal)+ )+ const services = await kube.list(+ 'services',+ config.recipeNamespace,+ `clerum.io/recipe=${names.recipe},clerum.io/workload=worker`+ )+ invariant(+ services.length === 1 &&+ services[0].metadata.ownerReferences?.some(owner => owner.uid === recipe.metadata.uid),+ 'WORKER_SERVICE_OWNERSHIP_NOT_PROVEN'+ )+ const workerUrl = `http://${services[0].metadata.name}.${config.recipeNamespace}.svc:3000/mcp`+ const business = async (write = false, internal = true) => {+ for (const [namespace, pod, url, key] of [+ [config.uiNamespace, frontend, `http://${LOOPBACK}:3000/mcp`, 'frontend'],+ [config.recipeNamespace, canary, `http://${LOOPBACK}:3000/mcp`, 'canary'],+ ...(internal ? [[config.uiNamespace, frontend, workerUrl, 'worker']] : []),+ ])+ await kube.exec(+ namespace,+ pod.metadata.name,+ businessCode,+ [url, key, journal.state.runId, write ? 'record' : 'recall'],+ 15_000+ )+ }+ const traffic = async (namespace, pod, port, mode = 'tcp') => {+ const output = await kube.exec(+ namespace,+ pod.metadata.name,+ trafficCode,+ [answerIp, String(port), endpointHost, mode],+ 10_000+ )+ if (mode === 'https')+ invariant(+ same(JSON.parse(output), { allowed: true, contentVerified: true }),+ 'HTTPS_BUSINESS_CONTENT_NOT_PROVEN'+ )+ else invariant(output === 'connected', 'TCP_CONNECTION_NOT_PROVEN')+ }+ const proveTraffic = async removed => {+ // The canary proves the remote port is open at the moment of a negative+ // observation. Only a connection timeout counts as policy denial.+ await traffic(config.recipeNamespace, canary, 80)+ await traffic(config.recipeNamespace, canary, 443, 'https')+ for (const lane of lanes) {+ await traffic(lane.namespace, lane.pod, 443, 'https')+ if (!removed) await traffic(lane.namespace, lane.pod, 80)+ else {+ let denied = false+ try {+ await traffic(lane.namespace, lane.pod, 80)+ } catch (error) {+ denied = error.exitCode === 42+ }+ invariant(denied, 'REVOKED_TRAFFIC_NOT_PROVEN_DENIED')+ await traffic(config.recipeNamespace, canary, 80)+ }+ }+ }+ await proveTraffic(false)+ await business(true)+ // Internal health port is genuinely listening before its declaration is removed.+ invariant(worker.status?.podIP, 'WORKER_POD_ADDRESS_MISSING')+ await kube.exec(+ config.uiNamespace,+ frontend.metadata.name,+ trafficCode,+ [worker.status.podIP, '3001', endpointHost, 'tcp'],+ 10_000+ )+ journal.state.phase = 'exercising'+ journal.save()+ console.log('WRC_EGRESS_BASELINE_PROVEN')++ await setAffectedDns('hold')+ const heldBefore = await dnsControl()+ const current = await kube.get(recipeIdentity)+ const reduced = await kube.patch(recipeIdentity, [+ { op: 'test', path: '/metadata/uid', value: recipe.metadata.uid },+ { op: 'test', path: '/metadata/resourceVersion', value: current.metadata.resourceVersion },+ {+ op: 'replace',+ path: '/spec/ui/egress/external',+ value: [{ fqdn: targets[0].fqdn, port: 443 }],+ },+ {+ op: 'replace',+ path: '/spec/ui/egress/internal',+ value: [{ workloadRef: 'worker', port: 3000 }],+ },+ {+ op: 'replace',+ path: '/spec/workloads/1/egressBindings',+ value: [{ dns: targets[1].fqdn, port: 443, protocol: 'TCP' }],+ },+ ])+ const countsFor = (state, lane, mode) => {+ const count = state.lanes.find(item => item.lane === lane)?.received?.[mode]+ invariant(Number.isSafeInteger(count) && count >= 0, 'MISSING_UNIQUE_DNS_OBSERVATION')+ return count+ }+ await waitFor('DNS-hold-started', 60_000, async (remaining, signal) => {+ const state = await dnsControl('/state', 'GET', remaining, signal)+ return ['ui', 'worker'].some(+ lane => countsFor(state, lane, 'hold') > countsFor(heldBefore, lane, 'hold')+ )+ })+ // Runtime proves convergence/continuity while the resolver withholds replies.+ // Exact pre/post-DNS ordering and same-attempt races are proved by the+ // deterministic controller tests, not by historical packet counters here.+ await waitFor('contraction-with-resolver-unavailable', 60_000, (remaining, signal) =>+ policiesMatch([443], [3000], remaining, signal)+ )+ await business()+ await proveTraffic(true)+ let internalDenied = false+ try {+ await kube.exec(+ config.uiNamespace,+ frontend.metadata.name,+ trafficCode,+ [worker.status.podIP, '3001', endpointHost, 'tcp'],+ 10_000+ )+ } catch (error) {+ internalDenied = error.exitCode === 42+ }+ invariant(internalDenied, 'REMOVED_INTERNAL_PORT_NOT_DENIED')+ console.log('WRC_EGRESS_HELD_CONTINUITY_PROVEN')++ for (const lane of lanes) {+ const policyIdentity = id('NetworkPolicy', lane.namespace, lane.policy)+ const policy = await kube.get(policyIdentity)+ await kube.patch(policyIdentity, [+ { op: 'test', path: '/metadata/uid', value: lane.uid },+ { op: 'test', path: '/metadata/resourceVersion', value: policy.metadata.resourceVersion },+ {+ op: 'add',+ path: '/spec/egress/-',+ value: {+ to: [{ ipBlock: { cidr: `${answerIp}/32` } }],+ ports: [{ port: 80, protocol: 'TCP' }],+ },+ },+ ])+ }+ const failedBefore = await dnsControl()+ await setAffectedDns('servfail')+ await waitFor('both-DNS-lanes-SERVFAIL', 90_000, async (remaining, signal) => {+ const state = await dnsControl('/state', 'GET', remaining, signal)+ return ['ui', 'worker'].every(+ lane => countsFor(state, lane, 'servfail') > countsFor(failedBefore, lane, 'servfail')+ )+ })+ await waitFor('drift-convergence-under-SERVFAIL', 90_000, (remaining, signal) =>+ policiesMatch([443], [3000], remaining, signal)+ )+ await business()+ await proveTraffic(true)+ invariant(+ (await active(names.recipe)) && (await active(names.canary)),+ 'RECIPE_AVAILABILITY_LOST'+ )+ console.log('WRC_EGRESS_SERVFAIL_CONTINUITY_PROVEN')++ // A no-op refresh may legitimately avoid writing timestamps until renewal.+ // Bound that wait by the observed stored window instead of demanding churn+ // or mistaking a DNS packet / old active phase for accepted convergence.+ const latestExpiry = Math.max(...lanes.map(lane => lane.latestExpiry))+ invariant(Number.isFinite(latestExpiry), 'MISSING_ACCEPTED_DNS_WINDOW')+ const recoveryBudget = Math.max(90_000, latestExpiry - Date.now() + 90_000)+ invariant(recoveryBudget <= 600_000, 'DNS_WINDOW_EXCEEDS_GATE_RECOVERY_BUDGET')+ for (const lane of lanes) {+ invariant(Number.isFinite(lane.lastAcceptedAt), 'MISSING_ACCEPTED_DNS_OBSERVATION')+ lane.recoveryAfter = lane.lastAcceptedAt+ }+ const recoveredBefore = await dnsControl()+ await setAffectedDns('ok')+ await waitFor('autonomous-DNS-recovery', 180_000, async (remaining, signal) => {+ const state = await dnsControl('/state', 'GET', remaining, signal)+ return ['ui', 'worker'].every(+ lane => countsFor(state, lane, 'ok') > countsFor(recoveredBefore, lane, 'ok')+ )+ })+ const recovered = await kube.get(recipeIdentity)+ invariant(+ recovered.metadata.generation === reduced.metadata.generation,+ 'RECOVERY_NEEDED_SPEC_NUDGE'+ )+ await waitFor('accepted-recovered-policy-proof', recoveryBudget, async (remaining, signal) => {+ const deadline = Date.now() + remaining+ return (+ (await policiesMatch([443], [3000], remaining, signal)) &&+ (await active(names.recipe, deadline - Date.now(), signal)) &&+ (await active(names.canary, deadline - Date.now(), signal))+ )+ })+ await business()+ await proveTraffic(true)++ // Exercise the real API's empty-list serialization on a valid A→B edit.+ // Remove the internal route intentionally; no service-continuity claim is+ // made for that explicitly revoked route after this configuration change.+ const migration = await kube.get(recipeIdentity)+ const migrated = await kube.patch(recipeIdentity, [+ { op: 'test', path: '/metadata/uid', value: recipe.metadata.uid },+ { op: 'test', path: '/metadata/resourceVersion', value: migration.metadata.resourceVersion },+ {+ op: 'replace',+ path: '/spec/ui/egress',+ value: { external: [{ fqdn: targets[1].fqdn, port: 443 }] },+ },+ {+ op: 'replace',+ path: '/spec/workloads/1/egressBindings',+ value: [{ dns: targets[0].fqdn, port: 443, protocol: 'TCP' }],+ },+ ])+ // Contracting away the worker's last old FQDN legitimately deletes its+ // aggregate policy. Re-pin the recreated identity only after this confirmed+ // intent change and the complete ownership/spec/provenance proof below.+ // The baseline UID remains mandatory throughout the prior fault/race phases.+ acceptPolicyIntentChange(lanes, migration, migrated, recipe.metadata.uid, targets)+ await waitFor('valid-host-migration', 120_000, async (remaining, signal) => {+ const deadline = Date.now() + remaining+ return (+ (await policiesMatch([443], [], remaining, signal)) &&+ (await active(names.recipe, deadline - Date.now(), signal))+ )+ })+ await business(false, false)+ await proveTraffic(true)+ for (const name of [config.wrcDeployment, 'control-api', 'host-context-controller']) {+ invariant(+ deploymentReady(await kube.get(id('Deployment', config.wrcNamespace, name))),+ 'SIBLING_DEPLOYMENT_NOT_READY'+ )+ }+ for (const lane of [...lanes, { pod: canary, namespace: config.recipeNamespace }]) {+ const now = (+ await kube.list(+ 'pods',+ lane.namespace,+ `clerum.io/recipe=${lane.pod.metadata.labels['clerum.io/recipe']},clerum.io/workload=${lane.pod.metadata.labels['clerum.io/workload']}`+ )+ ).find(pod => pod.metadata.uid === lane.pod.metadata.uid)+ invariant(+ now &&+ !now.metadata.deletionTimestamp &&+ same(+ now.status?.containerStatuses?.map(c => c.restartCount),+ lane.pod.status?.containerStatuses?.map(c => c.restartCount)+ ),+ 'FIXTURE_POD_RESTARTED_OR_REPLACED'+ )+ }+ console.log('WRC_EGRESS_AUTONOMOUS_RECOVERY_AND_MIGRATION_PROVEN')+}++async function main(argv) {+ invariant(+ argv.length === 0 || (argv.length === 1 && argv[0] === '--recover'),+ 'INVALID_GATE_ARGUMENTS'+ )+ const repository = fs.realpathSync(path.join(__dirname, '../../..'))+ invariant(+ process.env.T2_PROJECT_DIR === repository &&+ process.env.T2_PROFILE === process.env.T2_CONTEXT &&+ process.env.KUBECONTEXT === process.env.T2_CONTEXT,+ 'INVALID_GATE_BINDING'+ )+ const binding = {+ repository,+ profile: process.env.T2_PROFILE,+ context: process.env.T2_CONTEXT,+ branch: execFileSync('git', ['branch', '--show-current'], {+ cwd: repository,+ encoding: 'utf8',+ timeout: 5000,+ }).trim(),+ head: execFileSync('git', ['rev-parse', 'HEAD'], {+ cwd: repository,+ encoding: 'utf8',+ timeout: 5000,+ }).trim(),+ }+ const cancellation = new AbortController()+ let signalCode = 0+ const signalHandlers = new Map(+ ['SIGINT', 'SIGTERM'].map(signal => [+ signal,+ () => {+ signalCode ||= signal === 'SIGINT' ? 130 : 143+ const error = new Error('WRC_GATE_INTERRUPTED')+ error.exitCode = signalCode+ cancellation.abort(error)+ },+ ])+ )+ for (const [signal, handler] of signalHandlers) process.on(signal, handler)+ const run = createCommandRunner(repository, cancellation)+ const kube = createKubeClient(repository, binding.context, run)+ try {+ // Protect direct Node invocation as well as the Bash entry point.+ await run(['bash', path.join(repository, 'scripts/minikube/require-t2-mutation-lock.sh')], {+ timeoutMs: 15_000,+ })+ const file = journalPath(repository, binding.profile)+ if (argv[0] === '--recover') {+ const state = readJournal(file)+ assertJournalBinding(state, binding)+ await restore(kube, new Journal(file, state, true))+ invariant(!signalCode, 'RECOVERY_INTERRUPTED')+ console.log('WRC_EGRESS_RECOVERY_COMPLETE') // Never the interrupted test's PASS.+ return+ }+ invariant(+ process.env.E2E_WRC_EGRESS_FAULT_INJECTION === '1',+ 'FAULT_INJECTION_NOT_ACKNOWLEDGED'+ )+ // Direct Node invocation cannot bypass exact-head/image/context verification.+ // Reuse the supported read-only preflight while inheriting the same lease.+ await run(+ [+ 'env',+ 'T2_PLAN_MODE=false',+ 'bash',+ path.join(repository, 'scripts/minikube/t2-preflight.sh'),+ ],+ { timeoutMs: 300_000 }+ )+ const journal = new Journal(file, {+ version: 1,+ binding,+ runId: randomUUID(),+ phase: 'validated',+ resources: [],+ uiCleanup: null,+ })+ const config = {+ repository,+ wrcNamespace: process.env.WRC_NAMESPACE || 'control-plane',+ wrcDeployment: process.env.WRC_DEPLOYMENT || 'workflow-recipes',+ uiNamespace: process.env.SANDBOX_UI_NS || 'sandbox-ui',+ recipeNamespace: process.env.WORKFLOW_RECIPE_NS || 'sandbox-recipes',+ }+ await protectedRun(kube, journal, () => exercise(config, kube, journal), cancellation)+ console.log('WRC_EGRESS_DEGRADATION_E2E_PASS')+ } catch (error) {+ process.stderr.write(+ `${error instanceof AggregateError ? 'WRC_EGRESS_RECOVERY_REQUIRED' : 'WRC_EGRESS_GATE_FAILED'}\n`+ )+ process.exitCode = signalCode || error.exitCode || 1+ } finally {+ for (const [signal, handler] of signalHandlers) process.off(signal, handler)+ }+}++module.exports = {+ restore,+ protectedRun,+ fixtureObjects,+ deploymentReady,+ policyObservation,+ acceptPolicyIntentChange,+ main,+}+if (require.main === module)+ main(process.argv.slice(2)).catch(() => {+ process.exitCode = 1+ })diff --git a/scripts/e2e/_lib/wrc-egress-lifecycle.cjs b/scripts/e2e/_lib/wrc-egress-lifecycle.cjsnew file mode 100644index 000000000..7eb938f2b--- /dev/null+++ b/scripts/e2e/_lib/wrc-egress-lifecycle.cjs@@ -0,0 +1,448 @@+'use strict'++// Development-only lifecycle for the disruptive WRC gate. The durable journal+// is deliberately outside the transient profile lease: a killed process must+// leave a recovery obligation that later certification cannot silently ignore.+const fs = require('node:fs')+const path = require('node:path')+const { randomUUID } = require('node:crypto')+const { spawn } = require('node:child_process')+const { setTimeout: delay } = require('node:timers/promises')++const RUN_LABEL = 'e2e.clerum.io/run'+const resourceRoutes = new Map([+ ['ConfigMap', ['v1', 'configmaps']],+ ['Service', ['v1', 'services']],+ ['Deployment', ['apps/v1', 'deployments']],+ ['NetworkPolicy', ['networking.k8s.io/v1', 'networkpolicies']],+ ['WorkflowRecipe', ['clerum.io/v1alpha1', 'workflowrecipes']],+])+const validName = value =>+ typeof value === 'string' && /^[a-z0-9](?:[a-z0-9.-]{0,251}[a-z0-9])?$/.test(value)+function invariant(value, code) {+ if (!value) throw new Error(code)+}++function journalPath(repository, profile) {+ invariant(validName(profile), 'INVALID_PROFILE')+ invariant(fs.realpathSync(repository) === repository, 'NONCANONICAL_REPOSITORY')+ return path.join(+ repository,+ '.local-notes',+ 'infra',+ 'runs',+ 'fault-injection',+ `wrc-${profile}.json`+ )+}++function safeParents(file, create = false, allowMissing = false) {+ let current = path.parse(file).root+ for (const part of path.dirname(file).slice(current.length).split(path.sep).filter(Boolean)) {+ current = path.join(current, part)+ if (create && !fs.existsSync(current)) fs.mkdirSync(current, { mode: 0o700 })+ const info = fs.lstatSync(current, { throwIfNoEntry: false })+ if (!info && allowMissing) return+ invariant(info, 'MISSING_JOURNAL_PARENT')+ invariant(info.isDirectory() && !info.isSymbolicLink(), 'UNSAFE_JOURNAL_PARENT')+ }+}++function readJournal(file) {+ safeParents(file)+ const info = fs.lstatSync(file)+ invariant(info.isFile() && info.nlink === 1 && (info.mode & 0o077) === 0, 'UNSAFE_JOURNAL_FILE')+ invariant(info.size > 0 && info.size <= 1024 * 1024, 'INVALID_JOURNAL_SIZE')+ const state = JSON.parse(fs.readFileSync(file, 'utf8'))+ invariant(+ state.version === 1 &&+ typeof state.runId === 'string' &&+ /^[a-f0-9-]{36}$/.test(state.runId) &&+ Array.isArray(state.resources),+ 'INVALID_JOURNAL'+ )+ return state+}++function assertJournalBinding(state, expected) {+ // HEAD is historical during recovery; stable owner identity must still match.+ for (const key of ['repository', 'branch', 'profile', 'context']) {+ invariant(+ typeof expected[key] === 'string' &&+ expected[key].length > 0 &&+ state.binding?.[key] === expected[key],+ 'JOURNAL_OWNER_MISMATCH'+ )+ }+}++class Journal {+ constructor(file, state, existing = false) {+ this.file = file+ this.state = state+ safeParents(file, true)+ if (!existing) {+ const fd = fs.openSync(file, 'wx', 0o600)+ try {+ fs.writeFileSync(fd, JSON.stringify(state))+ fs.fsyncSync(fd)+ } finally {+ fs.closeSync(fd)+ }+ } else readJournal(file)+ }+ save() {+ const current = readJournal(this.file)+ invariant(current.runId === this.state.runId, 'JOURNAL_REPLACED')+ const pending = `${this.file}.${randomUUID()}.new`+ const fd = fs.openSync(pending, 'wx', 0o600)+ try {+ fs.writeFileSync(fd, JSON.stringify(this.state))+ fs.fsyncSync(fd)+ } finally {+ fs.closeSync(fd)+ }+ fs.renameSync(pending, this.file)+ }+ remove() {+ invariant(readJournal(this.file).runId === this.state.runId, 'JOURNAL_REPLACED')+ fs.unlinkSync(this.file)+ }+}++function resourceIdentity(object) {+ const metadata = object?.metadata+ invariant(+ resourceRoutes.has(object?.kind) && validName(metadata?.name) && validName(metadata?.namespace),+ 'INVALID_RESOURCE_IDENTITY'+ )+ return { kind: object.kind, name: metadata.name, namespace: metadata.namespace }+}++function resourcePath(identity) {+ const { kind, name, namespace } = resourceIdentity({ kind: identity.kind, metadata: identity })+ const [version, plural] = resourceRoutes.get(kind)+ return `${version === 'v1' ? '/api' : '/apis'}/${version}/namespaces/${namespace}/${plural}/${name}`+}++async function createOwned(kube, journal, object) {+ const identity = resourceIdentity(object)+ invariant(object.metadata.labels?.[RUN_LABEL] === journal.state.runId, 'MISSING_RUN_OWNERSHIP')+ const entry = { ...identity, attempted: true, uid: null }+ journal.state.resources.push(entry)+ journal.save() // Record uncertain acknowledgement BEFORE issuing the request.+ const created = await kube.create(object)+ invariant(+ created?.metadata?.uid && created.metadata.labels?.[RUN_LABEL] === journal.state.runId,+ 'UNPROVEN_CREATED_RESOURCE'+ )+ invariant(+ JSON.stringify(resourceIdentity(created)) === JSON.stringify(identity),+ 'CREATED_IDENTITY_MISMATCH'+ )+ entry.uid = created.metadata.uid+ journal.save()+ return created+}++async function deleteOwned(kube, journal, entry) {+ const live = await kube.get(entry)+ if (!live) return // Only the client's authoritative NotFound produces null.+ invariant(+ entry.uid &&+ live.metadata?.uid === entry.uid &&+ live.metadata.labels?.[RUN_LABEL] === journal.state.runId,+ 'UNPROVEN_CLEANUP_OWNERSHIP'+ )+ invariant(live.metadata.resourceVersion, 'MISSING_DELETE_VERSION')+ try {+ await kube.delete(entry, { uid: entry.uid, resourceVersion: live.metadata.resourceVersion })+ } catch (error) {+ if (await kube.get(entry)) throw error+ }+ await waitFor('owned-resource-absence', 90_000, async (remaining, signal) => {+ const current = await kube.get(entry, remaining, signal)+ if (!current) return true+ invariant(current.metadata?.uid === entry.uid, 'RESOURCE_RECREATED_DURING_CLEANUP')+ return false+ })+}++async function waitFor(label, timeoutMs, predicate, options = {}) {+ const now = options.now ?? Date.now+ const sleep = options.sleep ?? delay+ const deadline = now() + timeoutMs+ const timeoutError = new Error(`TIMEOUT:${label}`)+ timeoutError.exitCode = 124+ while (now() < deadline) {+ if (options.signal?.aborted) throw options.signal.reason+ const remaining = deadline - now()+ // Predicates pass this remaining budget into all child I/O. Race also bounds+ // a misbehaving predicate; cancellation must terminate its actual process.+ const controller = new AbortController()+ let timer+ const cancel = () => controller.abort(options.signal.reason)+ options.signal?.addEventListener('abort', cancel, { once: true })+ const cancelled = new Promise((_, reject) => {+ controller.signal.addEventListener('abort', () => reject(controller.signal.reason), {+ once: true,+ })+ })+ const expired = new Promise((_, reject) => {+ timer = setTimeout(() => {+ controller.abort(timeoutError)+ reject(timeoutError)+ }, remaining)+ })+ let ready+ try {+ ready = await Promise.race([predicate(remaining, controller.signal), expired, cancelled])+ } finally {+ clearTimeout(timer)+ options.signal?.removeEventListener('abort', cancel)+ }+ if (options.signal?.aborted) throw options.signal.reason+ if (ready && now() < deadline) return+ if (now() >= deadline) break+ await sleep(Math.min(500, deadline - now()))+ }+ throw timeoutError+}++function createCommandRunner(repository, cancellation) {+ return (args, { input, timeoutMs = 30_000, signal, cleanup = false } = {}) =>+ new Promise((resolve, reject) => {+ if (!cleanup && cancellation.signal.aborted) return reject(cancellation.signal.reason)+ const effectiveSignal = cleanup+ ? signal+ : AbortSignal.any([cancellation.signal, ...(signal ? [signal] : [])])+ if (effectiveSignal?.aborted) return reject(effectiveSignal.reason)+ const timeoutSeconds = Math.max(1, Math.ceil(timeoutMs / 1000))+ const child = spawn(+ process.execPath,+ [+ path.join(repository, 'scripts/minikube/run-with-deadline.mjs'),+ '--timeout-seconds',+ String(timeoutSeconds),+ '--kill-grace-seconds',+ '2',+ '--heartbeat-seconds',+ '20',+ '--label',+ 'wrc-egress-operation',+ '--',+ ...args,+ ],+ { stdio: ['pipe', 'pipe', 'pipe'], env: process.env }+ )+ const started = Date.now()+ const stdout = []+ let bytes = 0+ // Never echo arbitrary API bodies, pod output, or access configuration.+ child.stderr.resume()+ const abort = () => child.kill('SIGTERM')+ effectiveSignal?.addEventListener('abort', abort, { once: true })+ // The reused process-group supervisor accepts seconds; this parent timer+ // enforces the remaining millisecond phase budget without granting another+ // rounded-up second in which a command could still perform work.+ let timedOut = false+ const timer = setTimeout(() => {+ timedOut = true+ abort()+ }, timeoutMs)+ child.stdout.on('data', chunk => {+ bytes += chunk.length+ if (bytes > 4 * 1024 * 1024) abort()+ else stdout.push(chunk)+ })+ child.once('error', error => {+ clearTimeout(timer)+ effectiveSignal?.removeEventListener('abort', abort)+ reject(error)+ })+ child.once('close', (code, signalName) => {+ clearTimeout(timer)+ effectiveSignal?.removeEventListener('abort', abort)+ if (+ timedOut ||+ code !== 0 ||+ signalName ||+ effectiveSignal?.aborted ||+ bytes > 4 * 1024 * 1024 ||+ Date.now() - started > timeoutMs+ ) {+ const error = new Error('WRC_GATE_OPERATION_FAILED')+ error.exitCode =+ timedOut || Date.now() - started > timeoutMs+ ? 124+ : effectiveSignal?.reason?.exitCode || code || 1+ reject(error)+ } else resolve(Buffer.concat(stdout).toString('utf8'))+ })+ child.stdin.on('error', () => {})+ child.stdin.end(input ?? '')+ })+}++function createKubeClient(repository, context, run) {+ let cleanup = false+ const pending = new Set()+ const readBudget = timeoutMs => {+ const budget = timeoutMs ?? 30_000+ if (!Number.isFinite(budget) || budget <= 0) {+ const error = new Error('TIMEOUT:wrc-read')+ error.exitCode = 124+ throw error+ }+ return Math.min(30_000, budget)+ }+ const command = async (args, input, timeoutMs = 30_000, signal) => {+ const operation = run(+ [+ 'kubectl',+ `--context=${context}`,+ `--request-timeout=${Math.max(1, Math.ceil(timeoutMs / 1000))}s`,+ ...args,+ ],+ { input, timeoutMs, signal, cleanup }+ )+ pending.add(operation)+ try {+ return await operation+ } finally {+ pending.delete(operation)+ }+ }+ const mutation = async (args, input) => {+ await run(['bash', path.join(repository, 'scripts/minikube/require-t2-mutation-lock.sh')], {+ timeoutMs: 15_000,+ cleanup,+ })+ return command(args, input)+ }+ return {+ async setCleanup(value) {+ cleanup = value+ // A polling deadline can abort a GET before its process-group supervisor+ // has finished reaping it. Drain those bounded operations before cleanup+ // mutates resources or releases the profile's ownership obligation.+ if (value) await Promise.allSettled([...pending])+ },+ async get(identity, timeoutMs, signal) {+ resourceIdentity({ kind: identity.kind, metadata: identity })+ const text = await command(+ [+ 'get',+ identity.kind,+ identity.name,+ '-n',+ identity.namespace,+ '--ignore-not-found=true',+ '-o',+ 'json',+ ],+ undefined,+ readBudget(timeoutMs),+ signal+ )+ return text.trim() === '' ? null : JSON.parse(text)+ },+ async list(kind, namespace, selector, timeoutMs, signal) {+ return JSON.parse(+ await command(+ ['get', kind, '-n', namespace, '-l', selector, '-o', 'json'],+ undefined,+ readBudget(timeoutMs),+ signal+ )+ ).items+ },+ async create(object) {+ resourceIdentity(object)+ return JSON.parse(await mutation(['create', '-f', '-', '-o', 'json'], JSON.stringify(object)))+ },+ async patch(identity, patch) {+ resourceIdentity({ kind: identity.kind, metadata: identity })+ return JSON.parse(+ await mutation([+ 'patch',+ identity.kind,+ identity.name,+ '-n',+ identity.namespace,+ '--type=json',+ '-p',+ JSON.stringify(patch),+ '-o',+ 'json',+ ])+ )+ },+ async delete(identity, preconditions) {+ // kubectl RawDelete accepts a DeleteOptions request body via stdin.+ return mutation(+ ['delete', `--raw=${resourcePath(identity)}`, '-f', '-'],+ JSON.stringify({+ apiVersion: 'v1',+ kind: 'DeleteOptions',+ preconditions,+ propagationPolicy: 'Foreground',+ })+ )+ },+ async exec(namespace, pod, code, args = [], timeoutMs = 20_000, signal) {+ return command(+ ['exec', pod, '-n', namespace, '--', 'node', '-e', code, ...args],+ undefined,+ timeoutMs,+ signal+ )+ },+ async rollout(namespace, name, timeoutMs = 180_000) {+ return command(+ [+ 'rollout',+ 'status',+ `deployment/${name}`,+ '-n',+ namespace,+ `--timeout=${Math.ceil(timeoutMs / 1000)}s`,+ ],+ undefined,+ timeoutMs+ )+ },+ }+}++module.exports = {+ Journal,+ journalPath,+ readJournal,+ assertJournalBinding,+ createOwned,+ deleteOwned,+ resourceIdentity,+ resourcePath,+ waitFor,+ createCommandRunner,+ createKubeClient,+ invariant,+ RUN_LABEL,+}++if (require.main === module) {+ const [repository, profile] = process.argv.slice(2)+ try {+ const file = journalPath(repository, profile)+ safeParents(file, false, true)+ if (fs.existsSync(file) || fs.lstatSync(file, { throwIfNoEntry: false })) {+ throw new Error('WRC_EGRESS_RECOVERY_REQUIRED')+ }+ } catch (error) {+ process.stderr.write(+ 'WRC_EGRESS_RECOVERY_REQUIRED: resolve the owned gate journal before certification\n'+ )+ process.exitCode = 1+ }+}diff --git a/scripts/e2e/_lib/wrc-egress-proof.cjs b/scripts/e2e/_lib/wrc-egress-proof.cjsnew file mode 100644index 000000000..2784937aa--- /dev/null+++ b/scripts/e2e/_lib/wrc-egress-proof.cjs@@ -0,0 +1,366 @@+#!/usr/bin/env node+'use strict'++// Test oracle for the WRC egress fixture. Deliberately independent of+// the production policy builders/comparators: compare complete authorization+// tuples only after rejecting every unsupported/wildcard shape.+const { isIPv4 } = require('node:net')++const STATE = 'clerum.io/egress-fqdn-state'+const TARGETS = 'clerum.io/egress-fqdn-targets'+const RESOLVED_AT = 'clerum.io/egress-fqdn-resolved-at'+const MAX_ENTRIES = 128+const MAX_INPUT_BYTES = 1024 * 1024++/**+ * @typedef {{fqdn:string, ip:string, port:number, protocol:'TCP'}} ExternalTuple+ * @typedef {{namespace:string, workload:string, port:number, protocol:'TCP'}} InternalTuple+ * @typedef {{name:string, namespace:string, recipe:string,+ * recipeNamespace:string, lane:'ui'|'workload', workload:string,+ * podLabels:Record<string,string>, tuples:ExternalTuple[], internal?:InternalTuple[], uid?:string}} Expectation+ */++class ProofError extends Error {+ constructor(code, input = false) {+ super(code)+ this.name = 'ProofError'+ this.code = code+ this.input = input+ }+}++const record = value => value !== null && typeof value === 'object' && !Array.isArray(value)+const hasKeys = (value, allowed) =>+ record(value) && Object.keys(value).every(key => allowed.includes(key))+const nonempty = value => typeof value === 'string' && value.length > 0+const validName = value =>+ typeof value === 'string' &&+ value.length <= 253 &&+ value.split('.').every(label => /^[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?$/.test(label))+const dateNumber = value => Number.isFinite(value) && Number.isFinite(new Date(value).getTime())+const sameSet = (left, right) =>+ left.size === right.size && [...left].every(value => right.has(value))+const tupleKey = entry => `${entry.fqdn}\0${entry.ip}\0${entry.port}\0${entry.protocol}`+const networkKey = entry => `${entry.ip}\0${entry.port}\0${entry.protocol}`+const internalKey = (entry, recipe) =>+ `internal\0${entry.namespace}\0${recipe}\0${entry.workload}\0${entry.port}\0${entry.protocol}`++function requireProof(condition, code, input = false) {+ if (!condition) throw new ProofError(code, input)+}++function checkTuple(entry, input) {+ requireProof(+ record(entry) &&+ validName(entry.fqdn) &&+ entry.fqdn.includes('.') &&+ typeof entry.ip === 'string' &&+ isIPv4(entry.ip) &&+ Number.isInteger(entry.port) &&+ entry.port >= 1 &&+ entry.port <= 65535 &&+ entry.protocol === 'TCP',+ 'INVALID_EXTERNAL_TUPLE',+ input+ )+}++/** @param {Expectation} expected */+function checkExpectation(expected) {+ requireProof(+ hasKeys(expected, [+ 'name',+ 'namespace',+ 'recipe',+ 'recipeNamespace',+ 'lane',+ 'workload',+ 'podLabels',+ 'tuples',+ 'internal',+ 'uid',+ ]),+ 'INVALID_EXPECTATION_FIELDS',+ true+ )+ for (const key of ['name', 'namespace', 'recipe', 'recipeNamespace', 'workload']) {+ requireProof(validName(expected[key]), 'INVALID_EXPECTATION_IDENTITY', true)+ }+ requireProof(+ expected.lane === 'ui' || expected.lane === 'workload',+ 'INVALID_EXPECTATION_LANE',+ true+ )+ requireProof(+ expected.uid === undefined || nonempty(expected.uid),+ 'INVALID_EXPECTATION_UID',+ true+ )+ requireProof(+ record(expected.podLabels) &&+ Object.values(expected.podLabels).every(value => typeof value === 'string'),+ 'INVALID_EXPECTATION_POD_LABELS',+ true+ )+ requireProof(+ Array.isArray(expected.tuples) && expected.tuples.length <= MAX_ENTRIES,+ 'INVALID_EXPECTATION_TUPLES',+ true+ )+ const identities = new Set()+ for (const entry of expected.tuples) {+ requireProof(+ hasKeys(entry, ['fqdn', 'ip', 'port', 'protocol']),+ 'INVALID_EXPECTATION_TUPLE_FIELDS',+ true+ )+ checkTuple(entry, true)+ requireProof(!identities.has(tupleKey(entry)), 'DUPLICATE_EXPECTATION_TUPLE', true)+ identities.add(tupleKey(entry))+ }+ requireProof(+ expected.internal === undefined ||+ (Array.isArray(expected.internal) &&+ expected.internal.length <= 20 &&+ (expected.internal.length === 0 || expected.lane === 'ui')),+ 'INVALID_EXPECTATION_INTERNAL',+ true+ )+ const internalIdentities = new Set()+ for (const entry of expected.internal ?? []) {+ requireProof(+ hasKeys(entry, ['namespace', 'workload', 'port', 'protocol']) &&+ validName(entry.namespace) &&+ validName(entry.workload) &&+ Number.isInteger(entry.port) &&+ entry.port >= 1 &&+ entry.port <= 65535 &&+ entry.protocol === 'TCP',+ 'INVALID_EXPECTATION_INTERNAL',+ true+ )+ const key = internalKey(entry, expected.recipe)+ requireProof(!internalIdentities.has(key), 'DUPLICATE_EXPECTATION_INTERNAL', true)+ internalIdentities.add(key)+ }+}++/** Throws a sanitized ProofError; never includes the supplied policy body.+ * @param {unknown} policy+ * @param {Expectation} expected+ */+function assertPolicy(policy, expected) {+ checkExpectation(expected)+ requireProof(+ record(policy) &&+ policy.apiVersion === 'networking.k8s.io/v1' &&+ policy.kind === 'NetworkPolicy',+ 'INVALID_POLICY_KIND'+ )+ const metadata = policy.metadata+ requireProof(+ record(metadata) &&+ metadata.name === expected.name &&+ metadata.namespace === expected.namespace &&+ nonempty(metadata.uid) &&+ nonempty(metadata.resourceVersion) &&+ !metadata.deletionTimestamp &&+ (expected.uid === undefined || metadata.uid === expected.uid),+ 'POLICY_IDENTITY_MISMATCH'+ )++ const labels = metadata.labels+ requireProof(+ record(labels) &&+ labels['clerum.io/managed-by'] === 'workflow-recipes' &&+ labels['clerum.io/recipe'] === expected.recipe &&+ labels['clerum.io/recipe-name'] === expected.recipe &&+ labels['clerum.io/recipe-namespace'] === expected.recipeNamespace &&+ (expected.lane !== 'workload' || labels['clerum.io/workload'] === expected.workload),+ 'POLICY_OWNERSHIP_MISMATCH'+ )++ const selector =+ expected.lane === 'ui'+ ? {+ 'clerum.io/sandbox-ui': 'true',+ 'clerum.io/recipe-namespace': expected.recipeNamespace,+ 'clerum.io/recipe-name': expected.recipe,+ }+ : { 'clerum.io/recipe': expected.recipe, 'clerum.io/workload': expected.workload }+ requireProof(+ expected.podLabels['clerum.io/recipe'] === expected.recipe &&+ expected.podLabels['clerum.io/workload'] === expected.workload &&+ Object.entries(selector).every(([key, value]) => expected.podLabels[key] === value),+ 'POD_IDENTITY_MISMATCH'+ )+ const spec = policy.spec+ requireProof(hasKeys(spec, ['podSelector', 'policyTypes', 'egress']), 'UNSUPPORTED_POLICY_SPEC')+ requireProof(+ Array.isArray(spec.policyTypes) &&+ spec.policyTypes.length === 1 &&+ spec.policyTypes[0] === 'Egress',+ 'POLICY_TYPES_MISMATCH'+ )+ requireProof(+ hasKeys(spec.podSelector, ['matchLabels']) &&+ record(spec.podSelector.matchLabels) &&+ Object.keys(spec.podSelector.matchLabels).length === Object.keys(selector).length &&+ Object.entries(selector).every(([key, value]) => spec.podSelector.matchLabels[key] === value),+ 'POLICY_SELECTOR_MISMATCH'+ )++ // Kubernetes may omit the top-level empty list. With explicit Egress isolation+ // this is deny-all; missing to/ports INSIDE a rule is unrestricted and rejected.+ requireProof(spec.egress === undefined || Array.isArray(spec.egress), 'INVALID_EGRESS_LIST')+ const actualNetwork = new Set()+ for (const rule of spec.egress ?? []) {+ requireProof(+ hasKeys(rule, ['to', 'ports']) &&+ Array.isArray(rule.to) &&+ rule.to.length > 0 &&+ Array.isArray(rule.ports) &&+ rule.ports.length > 0,+ 'UNRESTRICTED_OR_UNKNOWN_RULE'+ )+ for (const peer of rule.to) {+ let internal+ if (record(peer) && Object.hasOwn(peer, 'ipBlock')) {+ requireProof(+ hasKeys(peer, ['ipBlock']) &&+ hasKeys(peer.ipBlock, ['cidr']) &&+ typeof peer.ipBlock.cidr === 'string' &&+ peer.ipBlock.cidr.endsWith('/32') &&+ isIPv4(peer.ipBlock.cidr.slice(0, -3)),+ 'UNSUPPORTED_EXTERNAL_PEER'+ )+ } else {+ requireProof(+ expected.lane === 'ui' &&+ hasKeys(peer, ['namespaceSelector', 'podSelector']) &&+ hasKeys(peer.namespaceSelector, ['matchLabels']) &&+ hasKeys(peer.podSelector, ['matchLabels']) &&+ hasKeys(peer.namespaceSelector.matchLabels, ['kubernetes.io/metadata.name']) &&+ validName(peer.namespaceSelector.matchLabels['kubernetes.io/metadata.name']) &&+ hasKeys(peer.podSelector.matchLabels, ['clerum.io/recipe', 'clerum.io/workload']) &&+ peer.podSelector.matchLabels['clerum.io/recipe'] === expected.recipe &&+ validName(peer.podSelector.matchLabels['clerum.io/workload']),+ 'UNSUPPORTED_INTERNAL_PEER'+ )+ internal = {+ namespace: peer.namespaceSelector.matchLabels['kubernetes.io/metadata.name'],+ workload: peer.podSelector.matchLabels['clerum.io/workload'],+ }+ }+ for (const port of rule.ports) {+ requireProof(+ hasKeys(port, ['port', 'protocol']) &&+ Number.isInteger(port.port) &&+ port.port >= 1 &&+ port.port <= 65535 &&+ (port.protocol === undefined || port.protocol === 'TCP'),+ 'UNSUPPORTED_EXTERNAL_PORT'+ )+ actualNetwork.add(+ internal+ ? internalKey({ ...internal, port: port.port, protocol: 'TCP' }, expected.recipe)+ : networkKey({ ip: peer.ipBlock.cidr.slice(0, -3), port: port.port, protocol: 'TCP' })+ )+ }+ }+ }+ const expectedNetwork = new Set([+ ...expected.tuples.map(networkKey),+ ...(expected.internal ?? []).map(entry => internalKey(entry, expected.recipe)),+ ])+ requireProof(sameSet(actualNetwork, expectedNetwork), 'EXTERNAL_TUPLES_MISMATCH')++ const annotations = metadata.annotations+ requireProof(record(annotations) && typeof annotations[STATE] === 'string', 'MISSING_PROVENANCE')+ let entries+ try {+ entries = JSON.parse(annotations[STATE])+ } catch {+ throw new ProofError('INVALID_PROVENANCE_JSON')+ }+ requireProof(Array.isArray(entries) && entries.length <= MAX_ENTRIES, 'INVALID_PROVENANCE_LIST')+ const identities = new Set()+ for (const entry of entries) {+ requireProof(+ hasKeys(entry, ['fqdn', 'ip', 'port', 'protocol', 'expiresAt', 'lastObservedAt']),+ 'INVALID_PROVENANCE_FIELDS'+ )+ checkTuple(entry, false)+ requireProof(+ dateNumber(entry.expiresAt) &&+ dateNumber(entry.lastObservedAt) &&+ entry.expiresAt >= entry.lastObservedAt,+ 'INVALID_PROVENANCE_TIME'+ )+ requireProof(!identities.has(tupleKey(entry)), 'DUPLICATE_PROVENANCE')+ identities.add(tupleKey(entry))+ }+ requireProof(+ sameSet(identities, new Set(expected.tuples.map(tupleKey))),+ 'PROVENANCE_TUPLES_MISMATCH'+ )+ const expectedTargets = expected.tuples.map(entry => `${entry.fqdn}=${entry.ip}/32`).sort()+ const targets = annotations[TARGETS]+ requireProof(+ typeof targets === 'string' &&+ JSON.stringify(targets === '' ? [] : targets.split(',').sort()) ===+ JSON.stringify(expectedTargets),+ 'PROVENANCE_TARGETS_MISMATCH'+ )+ const resolvedAt = annotations[RESOLVED_AT]+ requireProof(+ (entries.length === 0 && resolvedAt === undefined) ||+ (typeof resolvedAt === 'string' && Number.isFinite(Date.parse(resolvedAt))),+ 'INVALID_RESOLVED_AT'+ )+ return { ok: true }+}++function validatePolicy(policy, expected) {+ try {+ return assertPolicy(policy, expected)+ } catch (error) {+ if (!(error instanceof ProofError)) throw error+ return { ok: false, code: error.code, input: error.input }+ }+}++async function main(argv) {+ requireProof(argv.length === 2 && argv[0] === '--expect-json', 'USAGE_EXPECT_JSON_REQUIRED', true)+ let expected+ try {+ expected = JSON.parse(argv[1])+ } catch {+ throw new ProofError('INVALID_EXPECTATION_JSON', true)+ }+ checkExpectation(expected)+ const chunks = []+ let bytes = 0+ for await (const chunk of process.stdin) {+ bytes += chunk.length+ requireProof(bytes <= MAX_INPUT_BYTES, 'POLICY_INPUT_TOO_LARGE', true)+ chunks.push(chunk)+ }+ let policy+ try {+ policy = JSON.parse(Buffer.concat(chunks).toString('utf8'))+ } catch {+ throw new ProofError('INVALID_POLICY_JSON', true)+ }+ process.stdout.write(`${JSON.stringify(assertPolicy(policy, expected))}\n`)+}++module.exports = { assertPolicy, validatePolicy, ProofError }+if (require.main === module) {+ main(process.argv.slice(2)).catch(error => {+ process.stderr.write(+ `${JSON.stringify({ ok: false, code: error instanceof ProofError ? error.code : 'PROOF_FAILED' })}\n`+ )+ process.exitCode = error instanceof ProofError && error.input ? 2 : 1+ })+}diff --git a/scripts/e2e/e2e-wrc-egress-degradation.sh b/scripts/e2e/e2e-wrc-egress-degradation.shindex 46f997be5..1bdbd7b94 100755--- a/scripts/e2e/e2e-wrc-egress-degradation.sh+++ b/scripts/e2e/e2e-wrc-egress-degradation.sh@@ -1,542 +1,42 @@#!/usr/bin/env bash-# Proves that a correctly configured, active WorkflowRecipe stays available-# while WRC contracts removed UI/workload egress authorization across a real-# transient DNS failure. The fixture also widens both live policies while DNS-# is held and proves the post-DNS fresh-read contraction removes that drift-# before the reconcile completes.+# Admission-only entry point. The Node runner owns the reversible lifecycle;+# this product E2E is not T2.set -euo pipefail--SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"-# shellcheck source=scripts/e2e/e2e-lib.sh-# shellcheck disable=SC1091+SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)"+ROOT="$(cd -- "${SCRIPT_DIR}/../.." && pwd -P)"+# shellcheck source=e2e-lib.shsource "${SCRIPT_DIR}/e2e-lib.sh"-# Reuse the exact-head/profile/fingerprint validator used by the disruptive HCC-# gates. Despite its historical name, this helper performs no HCC mutation.-# shellcheck source=scripts/e2e/_lib/hcc-watch-recovery-fixture.sh-# shellcheck disable=SC1091+# shellcheck source=_lib/hcc-watch-recovery-fixture.shsource "${SCRIPT_DIR}/_lib/hcc-watch-recovery-fixture.sh"--[ -n "$E2E_KUBECONTEXT" ] || {- echo "KUBECONTEXT or E2E_K8S_CONTEXT must select an explicit branch profile." >&2- exit 1-}-is_branch_scoped_e2e_context "$E2E_KUBECONTEXT" || {- echo "Refusing WRC fault injection on non-branch context '${E2E_KUBECONTEXT}'." >&2- exit 1-}+die() { printf '[wrc-egress-degradation] %s\n' "$*" >&2; exit 1; }+RECOVER=false+if [[ "${1:-}" == --recover && "$#" -eq 1 ]]; then+ RECOVER=true+elif [[ "$#" -ne 0 ]]; then+ die 'usage: e2e-wrc-egress-degradation.sh [--recover]'+fi+[[ -n "$E2E_KUBECONTEXT" && "${MINIKUBE_PROFILE:-}" == "$E2E_KUBECONTEXT" ]] || die 'explicit matching branch profile/context required'+is_branch_scoped_e2e_context "$E2E_KUBECONTEXT" || die 'non-branch context refused'require_safe_kube_context-[ "${E2E_WRC_EGRESS_FAULT_INJECTION:-0}" = 1 ] || {- echo "Set E2E_WRC_EGRESS_FAULT_INJECTION=1 to acknowledge reversible WRC DNS fault injection." >&2- exit 1-}-command -v jq >/dev/null 2>&1 || { echo "jq is required" >&2; exit 1; }-command -v node >/dev/null 2>&1 || { echo "node is required" >&2; exit 1; }--WRC_NS="${WRC_NAMESPACE:-control-plane}"-WRC_DEPLOY="${WRC_DEPLOYMENT:-workflow-recipes}"-UI_NS="${SANDBOX_UI_NS:-sandbox-ui}"-RECIPE_NAMESPACE="${WORKFLOW_RECIPE_NS:-sandbox-recipes}"-RUN_ID="$(date +%s)-$$"-RUN_LABEL="$(truncate_rfc1123 "$RUN_ID")"-SUITE_NAME=wrc-egress-degradation-RECIPE_NAME="$(truncate_rfc1123 "e2e-wrc-egress-${RUN_ID}")"-TARGET_DNS="$(truncate_rfc1123 "wrc-egress-${RUN_ID}").example.com"-ANSWER_IP=93.184.216.34-DNS_PROXY="$(truncate_rfc1123 "e2e-wrc-dns-${RUN_ID}")"-DNS_PROXY_CONFIGMAP="$(truncate_rfc1123 "${DNS_PROXY}-script")"-DNS_PROXY_POLICY="$(truncate_rfc1123 "${DNS_PROXY}-policy")"-WRC_DNS_POLICY="$(truncate_rfc1123 "${DNS_PROXY}-from-wrc")"-LOCK_NAME="e2e-wrc-egress-degradation-lock"-UI_POLICY="ui-egress-${RECIPE_NAME}"-WORKLOAD_POLICY="wl-egress-${RECIPE_NAME}-worker"-DNS_PROXY_SCRIPT="${SCRIPT_DIR}/../../tests/e2e/fixtures/wrc-egress-dns-proxy/server.cjs"--LOCK_UID=""-LOCK_ACQUIRED=0-PROXY_CREATED=0-WRC_MUTATED=0-RECIPE_CREATED=0-ORIGINAL_DNS_POLICY=""-ORIGINAL_DNS_CONFIG=""-ORIGINAL_REPLICAS=""-WRC_IMAGE=""-UPSTREAM_DNS_IP=""--die() {- echo "[wrc-egress-degradation] ERROR: $*" >&2- if [ "$PROXY_CREATED" = 1 ]; then- kctl logs "deployment/${DNS_PROXY}" -n "$WRC_NS" -c dns-proxy --tail=120 >&2 || true- fi- exit 1-}--wait_until() {- local timeout=$1 description=$2- shift 2- local deadline=$(( $(date +%s) + timeout ))- while [ "$(date +%s)" -lt "$deadline" ]; do- "$@" && return 0- sleep 1- done- echo "Timed out after ${timeout}s waiting for ${description}" >&2- return 1-}--deployment_ready() {- local namespace=$1 name=$2- local deployment- deployment="$(kctl get deployment "$name" -n "$namespace" -o json 2>/dev/null)" || return 1- jq -e '- (.spec.replicas // 0) > 0 and- (.status.observedGeneration // 0) >= (.metadata.generation // 0) and- (.status.readyReplicas // 0) == (.spec.replicas // 0) and- (.status.availableReplicas // 0) == (.spec.replicas // 0)- ' <<<"$deployment" >/dev/null-}--recipe_active() {- [ "$(kctl get workflowrecipe "$RECIPE_NAME" -n "$RECIPE_NAMESPACE" \- -o jsonpath='{.status.phase}' 2>/dev/null)" = active ]-}--policy_matches_ports() {- local namespace=$1 name=$2 expected_json=$3- local policy- policy="$(kctl get networkpolicy "$name" -n "$namespace" -o json 2>/dev/null)" || return 1- jq -e --arg answer "${ANSWER_IP}/32" --argjson expected "$expected_json" '- .metadata.labels["clerum.io/managed-by"] == "workflow-recipes" and- ([.spec.egress[]?.ports[]?.port] | unique | sort) == ($expected | unique | sort) and- ([.spec.egress[]?.to[]?.ipBlock.cidr] | unique) == [$answer] and- ((.metadata.annotations["clerum.io/egress-fqdn-state"] | fromjson |- map(.port) | unique | sort) == ($expected | unique | sort))- ' <<<"$policy" >/dev/null-}--policy_has_port() {- local namespace=$1 name=$2 port=$3- kctl get networkpolicy "$name" -n "$namespace" -o json 2>/dev/null |- jq -e --argjson port "$port" 'any(.spec.egress[]?.ports[]?; .port == $port)' >/dev/null-}--append_policy_port() {- local namespace=$1 name=$2 port=$3- local patch- patch="$(jq -cn --arg cidr "${ANSWER_IP}/32" --argjson port "$port" '- [{op:"add",path:"/spec/egress/-",value:{- to:[{ipBlock:{cidr:$cidr}}],ports:[{port:$port,protocol:"TCP"}]- }}]- ')"- kctl patch networkpolicy "$name" -n "$namespace" --type=json -p "$patch" >/dev/null-}--workload_health() {- local namespace=$1 workload=$2- local pod- pod="$(ready_pod_name "$namespace" \- "clerum.io/recipe=${RECIPE_NAME},clerum.io/workload=${workload}" 2>/dev/null)" || return 1- # loopback is intentionally assembled inside JavaScript for the repository's- # public/private evidence boundary; no shell expansion belongs in this body.- # shellcheck disable=SC2016- kctl exec "$pod" -n "$namespace" -- node -e '- const loopback = [127, 0, 0, 1].join(".");- fetch(`http://${loopback}:3001/`)- .then(async response => {- const body = await response.json();- if (response.status !== 200 || body.status !== "ok") process.exit(2);- })- .catch(() => process.exit(3));- ' >/dev/null 2>&1-}--dns_state() {- kctl exec "deployment/${DNS_PROXY}" -n "$WRC_NS" -c dns-proxy -- node -e '- const http = require("node:http");- http.get({host:"127.0.0.1",port:8090,path:"/state"}, response => {- let body=""; response.on("data", chunk => body += chunk);- response.on("end", () => { process.stdout.write(body); process.exit(response.statusCode === 200 ? 0 : 2); });- }).once("error", () => process.exit(3));- ' 2>/dev/null-}--dns_count() {- local key=$1- dns_state | jq -er --arg key "$key" '.counts[$key]'-}--set_dns_mode() {- local selected=$1- # The JavaScript reads DNS_MODE from the explicit `env` argument; shell- # expansion inside the program would be a quoting bug.- # shellcheck disable=SC2016- kctl exec "deployment/${DNS_PROXY}" -n "$WRC_NS" -c dns-proxy -- \- env DNS_MODE="$selected" node -e '- const http = require("node:http");- const request = http.request(- {host:"127.0.0.1",port:8090,path:`/mode/${process.env.DNS_MODE}`,method:"POST"},- response => { response.resume(); response.on("end", () => process.exit(response.statusCode === 200 ? 0 : 2)); }- );- request.setTimeout(5000, () => request.destroy());- request.once("error", () => process.exit(3));- request.end();- '-}--dns_count_at_least() {- local key=$1 minimum=$2 current- current="$(dns_count "$key" 2>/dev/null)" || return 1- [ "$current" -ge "$minimum" ]-}--acquire_lock() {- local created existing- if ! created="$(kctl create configmap "$LOCK_NAME" -n "$WRC_NS" \- --from-literal="holder=${RUN_ID}" \- --from-literal="context=${E2E_KUBECONTEXT}" \- --from-literal="gitHead=$(git -C "${SCRIPT_DIR}/../.." rev-parse HEAD)" \- -o json 2>/dev/null)"; then- existing="$(kctl get configmap "$LOCK_NAME" -n "$WRC_NS" -o json 2>/dev/null || true)"- die "another WRC egress gate owns ${WRC_NS}/${LOCK_NAME}: holder=$(jq -r '.data.holder // "unknown"' <<<"${existing:-{}}"), head=$(jq -r '.data.gitHead // "unknown"' <<<"${existing:-{}}")"- fi- LOCK_UID="$(jq -r '.metadata.uid // ""' <<<"$created")"- [ -n "$LOCK_UID" ] || die "created gate lock has no UID"- LOCK_ACQUIRED=1-}--release_lock() {- local current- [ "$LOCK_ACQUIRED" = 1 ] || return 0- current="$(kctl get configmap "$LOCK_NAME" -n "$WRC_NS" -o json 2>/dev/null)" || return 1- [ "$(jq -r '.metadata.uid // ""' <<<"$current")" = "$LOCK_UID" ] || return 1- [ "$(jq -r '.data.holder // ""' <<<"$current")" = "$RUN_ID" ] || return 1- kctl delete configmap "$LOCK_NAME" -n "$WRC_NS" --wait=true --timeout=30s >/dev/null || return 1- LOCK_ACQUIRED=0-}--wrc_restore_verified() {- local deployment- deployment="$(kctl get deployment "$WRC_DEPLOY" -n "$WRC_NS" -o json 2>/dev/null)" || return 1- jq -e --arg policy "$ORIGINAL_DNS_POLICY" --argjson replicas "$ORIGINAL_REPLICAS" '- .spec.replicas == $replicas and- (.status.readyReplicas // 0) == $replicas and- (.status.availableReplicas // 0) == $replicas and- (.spec.template.spec.dnsPolicy // "ClusterFirst") == $policy and- (.spec.template.spec.dnsConfig // null) == null- ' <<<"$deployment" >/dev/null-}--delete_proxy_resources() {- local failed=0- kctl delete networkpolicy "$WRC_DNS_POLICY" "$DNS_PROXY_POLICY" -n "$WRC_NS" \- --ignore-not-found --wait=true --timeout=30s >/dev/null 2>&1 || failed=1- kctl delete service "$DNS_PROXY" -n "$WRC_NS" \- --ignore-not-found --wait=true --timeout=30s >/dev/null 2>&1 || failed=1- kctl delete deployment "$DNS_PROXY" -n "$WRC_NS" \- --ignore-not-found --wait=true --timeout=60s >/dev/null 2>&1 || failed=1- kctl delete configmap "$DNS_PROXY_CONFIGMAP" -n "$WRC_NS" \- --ignore-not-found --wait=true --timeout=30s >/dev/null 2>&1 || failed=1- return "$failed"-}--cleanup() {- local status=$?- trap - EXIT INT TERM- local cleanup_failed=0 restore_ok=1-- if [ "$PROXY_CREATED" = 1 ]; then- set_dns_mode ok >/dev/null 2>&1 || cleanup_failed=1- fi- if [ "$RECIPE_CREATED" = 1 ]; then- kctl delete workflowrecipe "$RECIPE_NAME" -n "$RECIPE_NAMESPACE" \- --ignore-not-found --wait=false >/dev/null 2>&1 || cleanup_failed=1- wait_until 90 "fixture WorkflowRecipe deletion" \- bash -c "! kubectl --context='${E2E_KUBECONTEXT}' get workflowrecipe '${RECIPE_NAME}' -n '${RECIPE_NAMESPACE}' >/dev/null 2>&1" || cleanup_failed=1- fi- if [ "$WRC_MUTATED" = 1 ]; then- local restore_patch- restore_patch="$(jq -cn --arg policy "$ORIGINAL_DNS_POLICY" '- {spec:{template:{spec:{dnsPolicy:$policy,dnsConfig:null}}}}- ')"- kctl patch deployment "$WRC_DEPLOY" -n "$WRC_NS" --type=strategic \- -p "$restore_patch" >/dev/null 2>&1 || restore_ok=0- kctl rollout status deployment "$WRC_DEPLOY" -n "$WRC_NS" \- --timeout=180s >/dev/null 2>&1 || restore_ok=0- wrc_restore_verified || restore_ok=0- fi- if [ "$restore_ok" = 1 ]; then- delete_proxy_resources || cleanup_failed=1- else- cleanup_failed=1- fi- if [ "$cleanup_failed" = 0 ] && [ "$restore_ok" = 1 ]; then- release_lock || cleanup_failed=1- fi- if [ "$cleanup_failed" != 0 ]; then- echo "WRC egress E2E cleanup/restore failed; lock retained at ${WRC_NS}/${LOCK_NAME}." >&2- echo "Inspect: kubectl --context=${E2E_KUBECONTEXT} -n ${WRC_NS} get deployment ${WRC_DEPLOY} -o yaml" >&2- status=1- fi- exit "$status"-}-trap cleanup EXIT INT TERM--require_branch_owned_hcc_gate "$WRC_NS"-acquire_lock--[ -r "$DNS_PROXY_SCRIPT" ] || die "DNS proxy fixture is missing at ${DNS_PROXY_SCRIPT}"-deployment_ready "$WRC_NS" "$WRC_DEPLOY" || die "WRC is not Ready before fault injection"-deployment_ready "$WRC_NS" control-api || die "control-api is not Ready before fault injection"-deployment_ready "$WRC_NS" host-context-controller || die "HCC is not Ready before fault injection"--ORIGINAL_REPLICAS="$(kctl get deployment "$WRC_DEPLOY" -n "$WRC_NS" -o jsonpath='{.spec.replicas}')"-[ "$ORIGINAL_REPLICAS" = 1 ] || die "expected one WRC replica, found ${ORIGINAL_REPLICAS:-unknown}"-ORIGINAL_DNS_POLICY="$(kctl get deployment "$WRC_DEPLOY" -n "$WRC_NS" -o json |- jq -r '.spec.template.spec.dnsPolicy // "ClusterFirst"')"-[ "$ORIGINAL_DNS_POLICY" = ClusterFirst ] || die "WRC already has non-default dnsPolicy"-ORIGINAL_DNS_CONFIG="$(kctl get deployment "$WRC_DEPLOY" -n "$WRC_NS" -o json |- jq -c '.spec.template.spec.dnsConfig // null')"-[ "$ORIGINAL_DNS_CONFIG" = null ] || die "WRC already has a dnsConfig; refusing non-restorable mutation"-WRC_IMAGE="$(kctl get deployment "$WRC_DEPLOY" -n "$WRC_NS" \- -o jsonpath='{.spec.template.spec.containers[?(@.name=="workflow-recipes")].image}')"-[ -n "$WRC_IMAGE" ] || die "could not resolve WRC image"-UPSTREAM_DNS_IP="$(kctl get service kube-dns -n kube-system -o jsonpath='{.spec.clusterIP}')"-[ -n "$UPSTREAM_DNS_IP" ] || die "could not resolve kube-dns ClusterIP"--kctl create configmap "$DNS_PROXY_CONFIGMAP" -n "$WRC_NS" \- --from-file="server.cjs=${DNS_PROXY_SCRIPT}" --dry-run=client -o yaml |- kctl apply -f - >/dev/null-PROXY_CREATED=1--kctl apply -f - >/dev/null <<EOF-apiVersion: apps/v1-kind: Deployment-metadata:- name: ${DNS_PROXY}- namespace: ${WRC_NS}- labels: {e2e.clerum.io/suite: ${SUITE_NAME}, e2e.clerum.io/run: "${RUN_LABEL}"}-spec:- replicas: 1- selector: {matchLabels: {app: ${DNS_PROXY}}}- template:- metadata:- labels: {app: ${DNS_PROXY}, e2e.clerum.io/suite: ${SUITE_NAME}, e2e.clerum.io/run: "${RUN_LABEL}"}- spec:- automountServiceAccountToken: false- terminationGracePeriodSeconds: 1- containers:- - name: dns-proxy- image: ${WRC_IMAGE}- imagePullPolicy: IfNotPresent- command: [node, /fixture/server.cjs]- env:- - {name: TARGET_DNS, value: "${TARGET_DNS}"}- - {name: ANSWER_IP, value: "${ANSWER_IP}"}- - {name: UPSTREAM_DNS, value: "${UPSTREAM_DNS_IP}"}- ports:- - {name: dns-udp, containerPort: 8053, protocol: UDP}- - {name: dns-tcp, containerPort: 8053, protocol: TCP}- readinessProbe: {tcpSocket: {port: 8053}, periodSeconds: 1, failureThreshold: 10}- volumeMounts:- - {name: fixture, mountPath: /fixture, readOnly: true}- resources:- requests: {cpu: 5m, memory: 20Mi}- limits: {cpu: 75m, memory: 64Mi}- securityContext:- allowPrivilegeEscalation: false- capabilities: {drop: [ALL]}- readOnlyRootFilesystem: true- runAsNonRoot: true- runAsUser: 1000- runAsGroup: 1000- seccompProfile: {type: RuntimeDefault}- volumes:- - name: fixture- configMap: {name: ${DNS_PROXY_CONFIGMAP}}-----apiVersion: v1-kind: Service-metadata:- name: ${DNS_PROXY}- namespace: ${WRC_NS}-spec:- selector: {app: ${DNS_PROXY}}- ports:- - {name: dns-udp, protocol: UDP, port: 53, targetPort: 8053}- - {name: dns-tcp, protocol: TCP, port: 53, targetPort: 8053}-----apiVersion: networking.k8s.io/v1-kind: NetworkPolicy-metadata:- name: ${DNS_PROXY_POLICY}- namespace: ${WRC_NS}-spec:- podSelector: {matchLabels: {app: ${DNS_PROXY}}}- policyTypes: [Ingress, Egress]- ingress:- - from: [{podSelector: {matchLabels: {app: ${WRC_DEPLOY}}}}]- ports:- - {protocol: UDP, port: 53}- - {protocol: UDP, port: 8053}- - {protocol: TCP, port: 53}- - {protocol: TCP, port: 8053}- egress:- - to:- - namespaceSelector: {matchLabels: {kubernetes.io/metadata.name: kube-system}}- podSelector: {matchLabels: {k8s-app: kube-dns}}- ports:- - {protocol: UDP, port: 53}- - {protocol: TCP, port: 53}-----apiVersion: networking.k8s.io/v1-kind: NetworkPolicy-metadata:- name: ${WRC_DNS_POLICY}- namespace: ${WRC_NS}-spec:- podSelector: {matchLabels: {app: ${WRC_DEPLOY}}}- policyTypes: [Egress]- egress:- - to: [{podSelector: {matchLabels: {app: ${DNS_PROXY}}}}]- ports:- - {protocol: UDP, port: 53}- - {protocol: UDP, port: 8053}- - {protocol: TCP, port: 53}- - {protocol: TCP, port: 8053}-EOF--kctl rollout status deployment "$DNS_PROXY" -n "$WRC_NS" --timeout=90s >/dev/null ||- die "DNS proxy did not become Ready"-wait_until 30 "DNS proxy listener" \- bash -c "kubectl --context='${E2E_KUBECONTEXT}' logs deployment/'${DNS_PROXY}' -n '${WRC_NS}' -c dns-proxy | grep -Fq 'wrc egress DNS proxy ready'" ||- die "DNS proxy did not report readiness"-DNS_PROXY_IP="$(kctl get service "$DNS_PROXY" -n "$WRC_NS" -o jsonpath='{.spec.clusterIP}')"-[ -n "$DNS_PROXY_IP" ] || die "DNS proxy Service has no ClusterIP"--WRC_MUTATED=1-dns_patch="$(jq -cn --arg nameserver "$DNS_PROXY_IP" '- {spec:{template:{spec:{- dnsPolicy:"None",- dnsConfig:{nameservers:[$nameserver],options:[- {name:"ndots",value:"1"},{name:"timeout",value:"2"},{name:"attempts",value:"1"}- ]}- }}}}-')"-kctl patch deployment "$WRC_DEPLOY" -n "$WRC_NS" --type=strategic -p "$dns_patch" >/dev/null-kctl rollout status deployment "$WRC_DEPLOY" -n "$WRC_NS" --timeout=180s >/dev/null ||- die "WRC did not become Ready with the isolated DNS proxy"-deployment_ready "$WRC_NS" control-api || die "control-api lost readiness during WRC DNS setup"-deployment_ready "$WRC_NS" host-context-controller || die "HCC lost readiness during WRC DNS setup"-ok "isolated WRC DNS proxy is Ready without affecting control-api or HCC"--RECIPE_CREATED=1-kctl apply -f - >/dev/null <<EOF-apiVersion: clerum.io/v1alpha1-kind: WorkflowRecipe-metadata:- name: ${RECIPE_NAME}- namespace: ${RECIPE_NAMESPACE}- labels: {e2e.clerum.io/suite: ${SUITE_NAME}, e2e.clerum.io/run: "${RUN_LABEL}"}-spec:- description: WRC egress degradation E2E baseline- contextRef: context1- workloads:- - id: frontend- type: deployment- image: clerum/mock-mcp-server:test- port: 3001- healthCheck: {type: http, path: /, port: 3001}- - id: worker- type: deployment- image: clerum/mock-mcp-server:test- port: 3001- healthCheck: {type: http, path: /, port: 3001}- egressBindings:- - {dns: ${TARGET_DNS}, port: 443, protocol: TCP}- - {dns: ${TARGET_DNS}, port: 8443, protocol: TCP}- ui:- workloadRef: frontend- port: 3001- title: WRC egress degradation fixture- defaultPath: /- egress:- external:- - {fqdn: ${TARGET_DNS}, port: 443}- - {fqdn: ${TARGET_DNS}, port: 8443}- security: {isolationLevel: minimal}-EOF--wait_until 180 "valid fixture recipe to become active" recipe_active ||- die "fixture recipe did not become active"-wait_until 120 "initial UI egress policy" policy_matches_ports "$UI_NS" "$UI_POLICY" '[443,8443]' ||- die "initial UI policy did not enforce both declared ports"-wait_until 120 "initial workload egress policy" policy_matches_ports "$RECIPE_NAMESPACE" "$WORKLOAD_POLICY" '[443,8443]' ||- die "initial workload policy did not enforce both declared ports"-wait_until 120 "frontend business health" workload_health "$UI_NS" frontend ||- die "frontend health endpoint was unavailable"-wait_until 120 "worker business health" workload_health "$RECIPE_NAMESPACE" worker ||- die "worker health endpoint was unavailable"-ok "valid recipe is active, both workload health endpoints respond, and both policies enforce the declared set"--hold_before="$(dns_count hold)"-set_dns_mode hold || die "could not hold fixture DNS"-reduced_patch="$(jq -cn --arg dns "$TARGET_DNS" '- [- {op:"replace",path:"/spec/ui/egress/external",value:[{fqdn:$dns,port:443}]},- {op:"replace",path:"/spec/workloads/1/egressBindings",value:[{dns:$dns,port:443,protocol:"TCP"}]}- ]-')"-kctl patch workflowrecipe "$RECIPE_NAME" -n "$RECIPE_NAMESPACE" --type=json \- -p "$reduced_patch" >/dev/null-wait_until 60 "WRC DNS query to enter the deterministic hold" \- dns_count_at_least hold "$((hold_before + 1))" || die "WRC never reached the DNS hold"-wait_until 60 "pre-DNS UI contraction" policy_matches_ports "$UI_NS" "$UI_POLICY" '[443]' ||- die "UI port 8443 was not revoked before DNS completed"-wait_until 60 "pre-DNS workload contraction" policy_matches_ports "$RECIPE_NAMESPACE" "$WORKLOAD_POLICY" '[443]' ||- die "workload port 8443 was not revoked before DNS completed"-recipe_active || die "correctly configured recipe left active before the held DNS completed"-workload_health "$UI_NS" frontend || die "frontend stopped serving during held DNS"-workload_health "$RECIPE_NAMESPACE" worker || die "worker stopped serving during held DNS"-ok "removed UI/workload permissions contracted before DNS completed while both services stayed healthy"--append_policy_port "$UI_NS" "$UI_POLICY" 9443-append_policy_port "$RECIPE_NAMESPACE" "$WORKLOAD_POLICY" 9443-policy_has_port "$UI_NS" "$UI_POLICY" 9443 || die "could not inject UI policy race"-policy_has_port "$RECIPE_NAMESPACE" "$WORKLOAD_POLICY" 9443 || die "could not inject workload policy race"--servfail_before="$(dns_count servfail)"-set_dns_mode servfail || die "could not release held DNS as SERVFAIL"-wait_until 90 "both WRC lanes to observe SERVFAIL" \- dns_count_at_least servfail "$((servfail_before + 2))" ||- die "UI and workload lanes did not both observe the transient DNS failure"-wait_until 90 "post-DNS UI re-contraction" policy_matches_ports "$UI_NS" "$UI_POLICY" '[443]' ||- die "UI race widening survived transient DNS failure"-wait_until 90 "post-DNS workload re-contraction" policy_matches_ports "$RECIPE_NAMESPACE" "$WORKLOAD_POLICY" '[443]' ||- die "workload race widening survived transient DNS failure"-wait_until 60 "recipe to remain active after proven fail-static" recipe_active ||- die "correctly configured recipe was cut by transient DNS failure"-workload_health "$UI_NS" frontend || die "frontend health failed after transient DNS failure"-workload_health "$RECIPE_NAMESPACE" worker || die "worker health failed after transient DNS failure"-deployment_ready "$WRC_NS" "$WRC_DEPLOY" || die "WRC lost readiness during fault"-deployment_ready "$WRC_NS" control-api || die "control-api lost readiness during WRC fault"-deployment_ready "$WRC_NS" host-context-controller || die "HCC lost readiness during WRC fault"-ok "transient DNS failure removed concurrent widening, preserved proven port 443, and isolated blast radius"--ok_before="$(dns_count ok)"-set_dns_mode ok || die "could not restore successful DNS"-recovery_patch="$(jq -cn --arg description "WRC egress recovery ${RUN_ID}" \- '{spec:{description:$description}}')"-kctl patch workflowrecipe "$RECIPE_NAME" -n "$RECIPE_NAMESPACE" --type=merge \- -p "$recovery_patch" >/dev/null-wait_until 90 "both WRC lanes to resolve after recovery" \- dns_count_at_least ok "$((ok_before + 2))" || die "DNS recovery was not observed by both lanes"-wait_until 90 "recovered recipe active state" recipe_active || die "recipe did not recover active"-policy_matches_ports "$UI_NS" "$UI_POLICY" '[443]' || die "UI policy drifted after recovery"-policy_matches_ports "$RECIPE_NAMESPACE" "$WORKLOAD_POLICY" '[443]' ||- die "workload policy drifted after recovery"-workload_health "$UI_NS" frontend || die "frontend health failed after DNS recovery"-workload_health "$RECIPE_NAMESPACE" worker || die "worker health failed after DNS recovery"-ok "successful DNS recovery reconverged both lanes without restoring removed or raced permissions"--echo "WRC_EGRESS_DEGRADATION_E2E_PASS"-echo "recipe=${RECIPE_NAMESPACE}/${RECIPE_NAME}"-echo "uiPolicy=${UI_NS}/${UI_POLICY}"-echo "workloadPolicy=${RECIPE_NAMESPACE}/${WORKLOAD_POLICY}"+[[ "${T2_PROJECT_DIR:-}" == "$ROOT" && "${T2_CONTEXT:-}" == "$E2E_KUBECONTEXT" && "${T2_PROFILE:-}" == "$E2E_KUBECONTEXT" ]] || die 'inherited lease target differs from gate'+# No mutation or cleanup handler exists before the live lease is proved.+bash "${ROOT}/scripts/minikube/require-t2-mutation-lock.sh"+export KUBECONTEXT="$E2E_KUBECONTEXT"+if [[ "$RECOVER" == true ]]; then+ exec node "${SCRIPT_DIR}/_lib/wrc-egress-gate.cjs" --recover+fi+[[ "${E2E_WRC_EGRESS_FAULT_INJECTION:-0}" == 1 ]] || die 'explicit fault-injection acknowledgement required'+# Bound every read made by the historical exact-head validator.+kctl() {+ node "${ROOT}/scripts/minikube/run-with-deadline.mjs" \+ --timeout-seconds 20 --kill-grace-seconds 2 --heartbeat-seconds 20 \+ --label wrc-egress-admission -- kubectl --context="$E2E_KUBECONTEXT" --request-timeout=15s "$@"+}+require_branch_owned_hcc_gate "${WRC_NAMESPACE:-control-plane}"+# The old helper checks HEAD/fingerprint, but not image acquisition freshness.+manifest="${T2_IMAGE_MANIFEST:-${ROOT}/deploy/minikube/.image-manifest.json}"+[[ -r "$manifest" ]] || die 'image manifest missing'+manifest_stamp="$(jq -er '.generated | select(type == "string" and length > 0)' "$manifest")"+marker_stamp="$(jq -er '.data.imagesGeneratedAt | select(type == "string" and length > 0)' <<<"$HCC_BRANCH_GATE_SYNC_MARKER")"+[[ "$manifest_stamp" == "$marker_stamp" ]] || die 'image stamp differs from deployed marker'+exec node "${SCRIPT_DIR}/_lib/wrc-egress-gate.cjs"diff --git a/scripts/minikube/t2-common.sh b/scripts/minikube/t2-common.shindex ca46dfdb5..84e9b4840 100755--- a/scripts/minikube/t2-common.sh+++ b/scripts/minikube/t2-common.sh@@ -1097,6 +1097,15 @@ t2_mutation_lock() {elset2_lock_acquirefi+ # A disruptive local gate can outlive its process (SIGKILL/API outage). The+ # profile lease is transient; its durable recovery obligation is not. Never+ # certify or reconcile over an unfinished DNS intervention.+ if ! node "$T2_SCRIPT_DIR/../e2e/_lib/wrc-egress-lifecycle.cjs" "$T2_PROJECT_DIR" "$T2_PROFILE"; then+ t2_lock_release 0 || true+ T2_NEXT_COMMAND='run the owned test-e2e-wrc-egress-recover target, then retry the original operation'+ t2_fail WRC_EGRESS_RECOVERY_REQUIRED 'an unfinished WRC fault-injection journal prevents profile mutation or certification'+ return 1+ fi}t2_lock_release() {diff --git a/scripts/minikube/with-t2-mutation-lock.sh b/scripts/minikube/with-t2-mutation-lock.shindex 75504e155..280e38427 100755--- a/scripts/minikube/with-t2-mutation-lock.sh+++ b/scripts/minikube/with-t2-mutation-lock.sh@@ -4,7 +4,10 @@# of acquiring a second lock for the same profile.set -euo pipefail-if [[ "${1:-}" != "--" || "$#" -lt 2 ]]; then+WRC_RECOVERY_ONLY=false+if [[ "${1:-}" == "--recover-wrc-egress" && "$#" -eq 1 ]]; then+ WRC_RECOVERY_ONLY=true+elif [[ "${1:-}" != "--" || "$#" -lt 2 ]]; thenprintf 'usage: %s -- command [args...]\n' "${0##*/}" >&2exit 2fi@@ -60,7 +63,14 @@ if [[ "${T2_BOOTSTRAP_REQUIRED}" == true ]]; thenfit2_context_checkt2_profile_context_identity_check-t2_mutation_lock+if [[ "$WRC_RECOVERY_ONLY" == true ]]; then+ # Recovery is a dedicated command, not a general skip-guard environment flag.+ # It still requires the exact live profile lease; only the recovery body is+ # permitted to run while that profile has a pending fault-injection journal.+ if [[ "$T2_SKIP_LOCK" == true ]]; then t2_lock_validate_inherited; else t2_lock_acquire; fi+else+ t2_mutation_lock+fiexport T2_PROJECT_DIR T2_PROFILE T2_CONTEXT T2_PROFILE_ROOT T2_PROFILE_ENV T2_PORTS_ENV \T2_LOCK_ROOT T2_LOCK_TOKEN MINIKUBE_PROFILE CONTROL_API_REAL_PG_CONTEXT@@ -80,5 +90,9 @@ trap 'exit 130' INTtrap 'exit 143' TERMrun_status=0-"$@" || run_status=$?+if [[ "$WRC_RECOVERY_ONLY" == true ]]; then+ bash "$ROOT/scripts/e2e/e2e-wrc-egress-degradation.sh" --recover || run_status=$?+else+ "$@" || run_status=$?+fiexit "${run_status}"diff --git a/scripts/tests/test-minikube-mutation-boundary.sh b/scripts/tests/test-minikube-mutation-boundary.shindex 5912d3165..490b5c6e6 100755--- a/scripts/tests/test-minikube-mutation-boundary.sh+++ b/scripts/tests/test-minikube-mutation-boundary.sh@@ -6,20 +6,21 @@ MUTATION_WRAPPER="${ROOT}/scripts/minikube/with-t2-mutation-lock.sh"TMP_DIR="$(mktemp -d)"source "${ROOT}/scripts/tests/lib/minikube-fixture-repo.sh"-MINIKUBE_TEST_PROFILE="boundary-profile"-MINIKUBE_TEST_CONTEXT="boundary-profile"+MINIKUBE_TEST_PROFILE="clerum-boundary-1234abcd"+MINIKUBE_TEST_CONTEXT="${MINIKUBE_TEST_PROFILE}"minikube_test_fixture_repo_init "${ROOT}" "${TMP_DIR}"cleanup() {local status=$?+ trap - EXITif ! minikube_test_assert_host_unchanged; thenstatus=1firm -rf "${TMP_DIR}"- return "${status}"+ exit "${status}"}trap cleanup EXIT-PROFILE=boundary-profile+PROFILE="${MINIKUBE_TEST_PROFILE}"TOKEN=boundary-test-tokenLOCK_ROOT="${TMP_DIR}/locks"LOCK_DIR="${LOCK_ROOT}/${PROFILE}.lock"@@ -30,7 +31,9 @@ BRANCH="${MINIKUBE_TEST_BRANCH}"HEAD="${MINIKUBE_TEST_HEAD}"WORKTREE_ID="${MINIKUBE_TEST_WORKTREE_ID}"LOCK_KEY="${MINIKUBE_TEST_LOCK_KEY}"-cat >"${LOCK_DIR}/owner.env" <<EOF+write_owner() {+ local owner_pid="${1:-$$}"+ cat >"${LOCK_DIR}/owner.env" <<EOFREPOSITORY=${MINIKUBE_TEST_PROJECT_DIR}BRANCH=${BRANCH}HEAD=${HEAD}@@ -39,9 +42,11 @@ CONTEXT=${PROFILE}WORKTREE_ID=${WORKTREE_ID}LOCK_KEY=${LOCK_KEY}TOKEN=${TOKEN}-PID=$$+PID=${owner_pid}PROCESS_START=unavailableEOF+}+write_owner# GNU Make may execute recipe lines containing $(MAKE) during a dry run. Force# the dry-run flag into the child environment so with-t2-mutation-lock.sh@@ -51,8 +56,9 @@ dry_run_make() {if output="$(MAKEFLAGS=-n make -n -C "${ROOT}" "$@" 2>&1)"; thenprintf '%s\n' "${output}"return 0+ else+ status=$?fi- status=$?printf 'FAIL: dry-run make %s exited %s\n%s\n' "$*" "${status}" "${output}" >&2return "${status}"}@@ -64,8 +70,9 @@ run_child() {bash "${ROOT}/scripts/minikube/require-t2-mutation-lock.sh"; thenprintf 'mutation\n' >>"${LOG}"return 0+ else+ status=$?fi- status=$?printf 'FAIL: valid inherited lease child exited %s (branch=%s head=%s)\n' \"${status}" "${BRANCH}" "${HEAD}" >&2return "${status}"@@ -127,6 +134,137 @@ assert_makeflags_dry_run_matrix() {printf 'PASS: compact and long GNU Make dry-run flags stop the real wrapper before profile or lock mutation\n'}+assert_wrc_admission() {+ local fixture_scripts="${MINIKUBE_TEST_PROJECT_DIR}/scripts"+ local fixture_entry="${fixture_scripts}/e2e/e2e-wrc-egress-degradation.sh"+ local fixture_wrapper="${fixture_scripts}/minikube/with-t2-mutation-lock.sh"+ local fake_bin="${TMP_DIR}/wrc-admission-bin"+ local entry_log="${TMP_DIR}/wrc-entry.log"+ local cluster_log="${TMP_DIR}/wrc-cluster.log"+ local output="${TMP_DIR}/wrc-admission.out"+ local real_node owner_before journal status+ real_node="$(command -v node)"+ mkdir -p "${fixture_scripts}/e2e/_lib" "${fixture_scripts}/minikube" "${fake_bin}"+ # Keep the fixture repository clean for the real wrapper's Git admission.+ # No Git operation or exclusion is applied to the host checkout.+ printf '/scripts/\n/.local-notes/\n' >>"${MINIKUBE_TEST_PROJECT_DIR}/.git/info/exclude"+ cp "${ROOT}/scripts/e2e/e2e-wrc-egress-degradation.sh" "${fixture_entry}"+ cp "${ROOT}/scripts/e2e/e2e-lib.sh" "${fixture_scripts}/e2e/e2e-lib.sh"+ cp "${ROOT}/scripts/e2e/_lib/hcc-watch-recovery-fixture.sh" "${fixture_scripts}/e2e/_lib/hcc-watch-recovery-fixture.sh"+ cp "${MUTATION_WRAPPER}" "${fixture_wrapper}"+ cp "${ROOT}/scripts/minikube/require-t2-mutation-lock.sh" "${fixture_scripts}/minikube/require-t2-mutation-lock.sh"+ cp "${ROOT}/scripts/minikube/t2-common.sh" "${fixture_scripts}/minikube/t2-common.sh"+ # Only runtime discovery/health is faked. Git admission, lease validation,+ # journal refusal, recovery dispatch, and lease release execute the real code.+ cat >>"${fixture_scripts}/minikube/t2-common.sh" <<'SH'+t2_require_commands() { :; }+t2_profile_scope() { :; }+t2_profile_status() { T2_BOOTSTRAP_REQUIRED=false; }+t2_context_check() { :; }+t2_profile_context_identity_check() { :; }+SH+ cat >"${fake_bin}/node" <<'SH'+#!/usr/bin/env bash+set -euo pipefail+if [[ "$1" == "$WRC_ADMISSION_GUARD" && "$#" -eq 3 ]]; then+ exec "$WRC_ADMISSION_NODE" "$@"+fi+if [[ "$1" == "$WRC_ADMISSION_RUNNER" && "$#" -eq 2 && "$2" == --recover ]]; then+ printf '%s\n' "$2" >>"$WRC_ADMISSION_ENTRY_LOG"+ exit 0+fi+printf 'unexpected Node runtime dispatch\n' >>"$WRC_ADMISSION_ENTRY_LOG"+exit 99+SH+ cat >"${fake_bin}/kubectl" <<'SH'+#!/usr/bin/env bash+printf 'unexpected cluster operation\n' >>"$WRC_ADMISSION_CLUSTER_LOG"+exit 99+SH+ chmod +x "${fake_bin}/node" "${fake_bin}/kubectl"+ : >"${entry_log}"; : >"${cluster_log}"++ invoke_wrc() {+ local surface="$1" lease_token="$2" effective_context="$3" acknowledgement="$4"+ shift 4+ local script="${fixture_entry}"+ [[ "$surface" != wrapper ]] || script="${fixture_wrapper}"+ PATH="${fake_bin}:${PATH}" MAKEFLAGS= \+ T2_PROJECT_DIR="${MINIKUBE_TEST_PROJECT_DIR}" T2_SCRIPT_DIR="${ROOT}/scripts/minikube" \+ T2_PROFILE="${PROFILE}" T2_CONTEXT="${effective_context}" T2_LOCK_ROOT="${LOCK_ROOT}" \+ T2_LOCK_TOKEN="${lease_token}" T2_SKIP_LOCK=true T2_MUTATION_LOCK_WRAPPED=true \+ T2_PROFILE_ROOT="${TMP_DIR}/wrc-profiles" T2_GATE_ID=wrc-egress-recovery \+ MINIKUBE_PROFILE="${PROFILE}" KUBECONTEXT="${PROFILE}" \+ CONTROL_API_REAL_PG_CONTEXT="${effective_context}" \+ E2E_WRC_EGRESS_FAULT_INJECTION="${acknowledgement}" \+ WRC_ADMISSION_GUARD="${ROOT}/scripts/minikube/../e2e/_lib/wrc-egress-lifecycle.cjs" \+ WRC_ADMISSION_RUNNER="${fixture_scripts}/e2e/_lib/wrc-egress-gate.cjs" \+ WRC_ADMISSION_NODE="${real_node}" WRC_ADMISSION_ENTRY_LOG="${entry_log}" \+ WRC_ADMISSION_CLUSTER_LOG="${cluster_log}" bash "${script}" "$@"+ }+ expect_wrc_refusal() {+ local description="$1" reason="$2"+ shift 2+ : >"${entry_log}"; : >"${cluster_log}"+ status=0+ invoke_wrc "$@" >"${output}" 2>&1 || status=$?+ if [[ "$status" -eq 0 || -s "$entry_log" || -s "$cluster_log" ]] || ! grep -Fq "$reason" "$output"; then+ printf 'FAIL: %s did not refuse before runtime dispatch (exit=%s)\n' "$description" "$status" >&2+ cat "$output" >&2+ exit 1+ fi+ }++ expect_wrc_refusal 'direct gate without a lease' PROFILE_LOCK_REQUIRED entry '' "$PROFILE" 1+ expect_wrc_refusal 'wrapped boolean with false inherited token' PROFILE_LOCK_REQUIRED entry wrong-token "$PROFILE" 1+ expect_wrc_refusal 'direct recovery without a lease' PROFILE_LOCK_REQUIRED entry '' "$PROFILE" 0 --recover+ expect_wrc_refusal 'direct entry with mismatched context' 'inherited lease target differs' entry "$TOKEN" wrong-context 1+ expect_wrc_refusal 'valid lease without fault acknowledgement' 'explicit fault-injection acknowledgement' entry "$TOKEN" "$PROFILE" 0+ expect_wrc_refusal 'recovery with arbitrary extra command' usage wrapper "$TOKEN" "$PROFILE" 0 --recover-wrc-egress -- bash -c true+ expect_wrc_refusal 'wrapper recovery with mismatched context' PROFILE_LOCK_REQUIRED wrapper "$TOKEN" wrong-context 0 --recover-wrc-egress++ # This deliberately impossible PID cannot belong to another local process.+ if kill -0 2147483647 2>/dev/null; then+ printf 'FAIL: dead-owner PID fixture unexpectedly exists\n' >&2+ exit 1+ fi+ write_owner 2147483647+ expect_wrc_refusal 'dead inherited owner' PROFILE_LOCK_REQUIRED entry "$TOKEN" "$PROFILE" 0 --recover+ expect_wrc_refusal 'dedicated recovery with dead inherited owner' PROFILE_LOCK_REQUIRED wrapper "$TOKEN" "$PROFILE" 0 --recover-wrc-egress+ write_owner++ journal="$(node - "${ROOT}/scripts/e2e/_lib/wrc-egress-lifecycle.cjs" "${MINIKUBE_TEST_PROJECT_DIR}" "$PROFILE" <<'NODE'+const { Journal, journalPath } = require(process.argv[2])+const { randomUUID } = require('node:crypto')+const repository = process.argv[3], profile = process.argv[4]+const file = journalPath(repository, profile)+new Journal(file, { version: 1, runId: randomUUID(), phase: 'DNS-injected',+ binding: { repository, profile, context: profile, branch: process.env.MINIKUBE_TEST_BRANCH,+ head: process.env.MINIKUBE_TEST_HEAD }, resources: [] })+process.stdout.write(file)+NODE+ )"+ owner_before="$(shasum -a 256 "${LOCK_DIR}/owner.env")"+ expect_wrc_refusal 'ordinary mutation over pending journal' WRC_EGRESS_RECOVERY_REQUIRED wrapper "$TOKEN" "$PROFILE" 1 -- bash "$fixture_entry"+ expect_wrc_refusal 'recovery path with false inheritance' PROFILE_LOCK_REQUIRED wrapper wrong-token "$PROFILE" 0 --recover-wrc-egress+ for surface in entry wrapper; do+ : >"${entry_log}"; : >"${cluster_log}"+ status=0+ if [[ "$surface" == entry ]]; then+ invoke_wrc entry "$TOKEN" "$PROFILE" 0 --recover >"${output}" 2>&1 || status=$?+ else+ invoke_wrc wrapper "$TOKEN" "$PROFILE" 0 --recover-wrc-egress >"${output}" 2>&1 || status=$?+ fi+ if [[ "$status" -ne 0 || "$(cat "$entry_log")" != --recover || -s "$cluster_log" \+ || ! -f "$journal" || "$(shasum -a 256 "${LOCK_DIR}/owner.env")" != "$owner_before" ]]; then+ printf 'FAIL: dedicated %s recovery did not preserve the inherited lease and dispatch only recovery\n' "$surface" >&2+ cat "$output" >&2+ exit 1+ fi+ done+ printf 'PASS: WRC direct entry, pending journal and dedicated recovery enforce the real inherited lease\n'+}+run_child[[ "$(wc -l <"${LOG}" | tr -d ' ')" -eq 1 ]] || {echo 'FAIL: valid inherited lease did not reach the child boundary' >&2@@ -157,6 +295,8 @@ fiexit 1}+assert_wrc_admission+full_plan="$(dry_run_make minikube-build-images 2>&1)"full_body_plan="$(dry_run_make minikube-build-images-body 2>&1)"targeted_plan="$(dry_run_make minikube-deploy-service SVC=control-api NS=control-plane 2>&1)"@@ -166,6 +306,8 @@ restart_body_plan="$(dry_run_make minikube-restart-deploy-body SVC=control-api Ne2e_fixture_plan="$(dry_run_make minikube-build-e2e-fixtures 2>&1)"e2e_fixture_body_plan="$(dry_run_make minikube-build-e2e-fixtures-body 2>&1)"verify_plan="$(dry_run_make minikube-verify-images 2>&1)"+wrc_plan="$(dry_run_make test-e2e-wrc-egress-degradation E2E_KUBECONTEXT="$PROFILE" E2E_EXPECTED_PRE_GATE_GATE=unit-gate 2>&1)"+wrc_recovery_plan="$(dry_run_make test-e2e-wrc-egress-recover E2E_KUBECONTEXT="$PROFILE" 2>&1)"[[ "$full_plan" == *"with-t2-mutation-lock.sh"* \&& "$full_body_plan" == *"require-t2-mutation-lock.sh"* ]] || {@@ -197,6 +339,15 @@ verify_plan="$(dry_run_make minikube-verify-images 2>&1)"echo 'FAIL: read-only image verification was incorrectly placed behind a mutation lease' >&2exit 1}+[[ "$wrc_plan" == *"with-t2-mutation-lock.sh -- bash scripts/e2e/e2e-wrc-egress-degradation.sh"* \+ && "$wrc_plan" == *"T2_SKIP_LOCK="* && "$wrc_plan" == *"T2_LOCK_TOKEN="* \+ && "$wrc_plan" == *"T2_PROFILE=\"${PROFILE}\""* && "$wrc_plan" == *"T2_CONTEXT=\"${PROFILE}\""* \+ && "$wrc_recovery_plan" == *"with-t2-mutation-lock.sh --recover-wrc-egress"* \+ && "$wrc_recovery_plan" != *"--recover-wrc-egress --"* \+ && "$wrc_recovery_plan" == *"T2_SKIP_LOCK="* && "$wrc_recovery_plan" == *"T2_LOCK_TOKEN="* ]] || {+ echo 'FAIL: WRC gate/recovery Make targets do not preserve the canonical lease and dedicated recovery dispatch' >&2+ exit 1+}printf 'PASS: child mutation boundary requires the exact live profile lease and target context\n'printf 'PASS: image builds and targeted deployment mutations require the lease while verify-only stays read-only\n'diff --git a/scripts/tests/test-wrc-egress-degradation-gate.sh b/scripts/tests/test-wrc-egress-degradation-gate.shindex d2192efa7..74794f631 100755--- a/scripts/tests/test-wrc-egress-degradation-gate.sh+++ b/scripts/tests/test-wrc-egress-degradation-gate.sh@@ -1,76 +1,20 @@#!/usr/bin/env bash-# Literal source-contract patterns intentionally contain shell-looking text.-# shellcheck disable=SC2016+# Cluster-free unit/integration evidence for the WRC gate. Runtime dataplane,+# business continuity, browser journeys, and T2 remain separate evidence lanes.set -euo pipefail-ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"-GATE="${ROOT}/scripts/e2e/e2e-wrc-egress-degradation.sh"-FIXTURE="${ROOT}/tests/e2e/fixtures/wrc-egress-dns-proxy/server.cjs"-MAKEFILE="${ROOT}/Makefile"+ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd -P)"-fail() { echo "FAIL: $*" >&2; exit 1; }-contains() {- local file=$1 pattern=$2 description=$3- grep -Fq -- "$pattern" "$file" || fail "$description"-}+node -e 'if (Number(process.versions.node.split(".")[0]) !== 24) { process.stderr.write("WRC gate unit tests require Node 24\n"); process.exit(1) }'+bash -n "${ROOT}/scripts/e2e/e2e-wrc-egress-degradation.sh"-bash -n "$GATE"-node --check "$FIXTURE" >/dev/null+# These tests import the real pure oracle, DNS server, lifecycle, and runner.+# DNS uses ephemeral loopback sockets; Kubernetes is faked at its boundary.+# Executable entrypoint/lease refusal cases live in test-minikube-mutation-boundary.sh.+node --test --test-concurrency=1 \+ "${ROOT}/scripts/tests/wrc-egress-proof.test.cjs" \+ "${ROOT}/scripts/tests/wrc-egress-dns-proxy.test.cjs" \+ "${ROOT}/scripts/tests/wrc-egress-lifecycle.test.cjs" \+ "${ROOT}/scripts/tests/wrc-egress-gate.test.cjs"-contains "$GATE" 'is_branch_scoped_e2e_context "$E2E_KUBECONTEXT"' \- 'gate must refuse non-branch Kubernetes contexts'-contains "$GATE" 'E2E_WRC_EGRESS_FAULT_INJECTION' \- 'gate must require explicit fault-injection acknowledgement'-contains "$GATE" 'require_branch_owned_hcc_gate "$WRC_NS"' \- 'gate must verify exact HEAD, worktree, profile, fingerprint, and pre-gate marker'-contains "$GATE" 'acquire_lock' 'gate must acquire exclusive WRC fault-injection ownership'-contains "$GATE" 'wrc_restore_verified' 'gate must prove WRC DNS configuration restoration'-contains "$GATE" 'lock retained' 'failed restoration must retain the coordination lock'--contains "$GATE" 'id: frontend' 'fixture must exercise the sandbox UI workload'-contains "$GATE" 'id: worker' 'fixture must exercise a non-UI workload'-contains "$GATE" 'port: 8443' 'fixture must begin with a permission that will be removed'-contains "$GATE" 'set_dns_mode hold' 'gate must create a real DNS latency window'-contains "$GATE" 'pre-DNS UI contraction' 'gate must assert UI contraction before DNS completes'-contains "$GATE" 'pre-DNS workload contraction' \- 'gate must assert workload contraction before DNS completes'-contains "$GATE" 'append_policy_port "$UI_NS" "$UI_POLICY" 9443' \- 'gate must inject live UI drift during DNS latency'-contains "$GATE" 'append_policy_port "$RECIPE_NAMESPACE" "$WORKLOAD_POLICY" 9443' \- 'gate must inject live workload drift during DNS latency'-contains "$GATE" 'set_dns_mode servfail' 'gate must produce a real retryable resolver response'-contains "$GATE" 'post-DNS UI re-contraction' \- 'gate must assert the UI fresh-read contraction after DNS failure'-contains "$GATE" 'post-DNS workload re-contraction' \- 'gate must assert the workload fresh-read contraction after DNS failure'--contains "$GATE" 'workload_health "$UI_NS" frontend' \- 'gate must prove the UI workload still serves its health endpoint'-contains "$GATE" 'workload_health "$RECIPE_NAMESPACE" worker' \- 'gate must prove the workload still serves its health endpoint'-contains "$GATE" 'deployment_ready "$WRC_NS" control-api' \- 'gate must assert an unrelated production service remains Ready'-contains "$GATE" 'deployment_ready "$WRC_NS" host-context-controller' \- 'gate must assert HCC remains Ready'-contains "$GATE" 'set_dns_mode ok' 'gate must restore successful DNS'-contains "$GATE" 'WRC_EGRESS_DEGRADATION_E2E_PASS' \- 'gate must emit an unambiguous terminal success marker'--contains "$FIXTURE" 'match(/^\/mode\/(ok|hold|servfail)$/)' \- 'DNS fixture must expose exactly the required deterministic modes'-contains "$FIXTURE" '0x8180 | rcode' 'DNS fixture must emit an actual SERVFAIL response'-contains "$FIXTURE" 'forwardUdp' 'DNS fixture must forward unrelated UDP queries'-contains "$FIXTURE" 'forwardTcp' 'DNS fixture must forward unrelated TCP queries'--contains "$MAKEFILE" '.PHONY: test-e2e-wrc-egress-degradation' \- 'Makefile must expose the WRC egress degradation gate'-contains "$MAKEFILE" 'MINIKUBE_PROFILE=$(E2E_KUBECONTEXT) KUBECONTEXT=$(E2E_KUBECONTEXT) bash scripts/e2e/e2e-wrc-egress-degradation.sh' \- 'Make target must bind the mutation to the explicit profile/context'-if grep -Eq 'kubectl[[:space:]]+(get|apply|patch|delete|exec|logs)' "$GATE"; then- fail 'raw kubectl operations must use kctl or an explicit --context cleanup probe'-fi-if grep -Fq -- '--force' "$GATE"; then- fail 'gate must not force Kubernetes mutations'-fi--echo 'PASS: WRC egress degradation E2E guard contract'+printf 'WRC_EGRESS_UNIT_CONTRACT_PASS\n'diff --git a/scripts/tests/wrc-egress-dns-proxy.test.cjs b/scripts/tests/wrc-egress-dns-proxy.test.cjsnew file mode 100644index 000000000..b89bb7664--- /dev/null+++ b/scripts/tests/wrc-egress-dns-proxy.test.cjs@@ -0,0 +1,412 @@+'use strict'++const { test } = require('node:test')+const assert = require('node:assert/strict')+const { once } = require('node:events')+const dgram = require('node:dgram')+const { Resolver } = require('node:dns/promises')+const net = require('node:net')+const {+ createDnsProxy,+ readConfig,+ question,+ dnsResponse,+ tcpFrame,+} = require('../../tests/e2e/fixtures/wrc-egress-dns-proxy/server.cjs')++const targets = [+ { lane: 'ui', fqdn: 'ui.example.com', ip: '93.184.216.34' },+ { lane: 'worker', fqdn: 'worker.example.com', ip: '93.184.216.35' },+]+function query(fqdn, id = 7, type = 1) {+ const header = Buffer.alloc(12)+ header.writeUInt16BE(id, 0)+ header.writeUInt16BE(0x0100, 2)+ header.writeUInt16BE(1, 4)+ const name = fqdn.split('.').flatMap(label => [Buffer.from([label.length]), Buffer.from(label)])+ const tail = Buffer.from([0, 0, 0, 0, 1])+ tail.writeUInt16BE(type, 1)+ return Buffer.concat([header, ...name, tail])+}++async function udpClient(t) {+ const socket = dgram.createSocket('udp4')+ socket.bind(0, '127.0.0.1')+ await once(socket, 'listening')+ t.after(() => new Promise(resolve => socket.close(resolve)))+ return socket+}+async function exchangeUdp(socket, port, message) {+ const response = once(socket, 'message', { signal: AbortSignal.timeout(2000) })+ socket.send(message, port, '127.0.0.1')+ return (await response)[0]+}+async function exchangeTcp(port, message) {+ const socket = net.createConnection({ host: '127.0.0.1', port })+ await once(socket, 'connect', { signal: AbortSignal.timeout(2000) })+ try {+ const response = new Promise((resolve, reject) => {+ let pending = Buffer.alloc(0)+ socket.setTimeout(2000, () => reject(new Error('test TCP DNS deadline')))+ socket.once('error', reject)+ socket.on('data', chunk => {+ pending = Buffer.concat([pending, chunk])+ if (pending.length >= 2 && pending.length >= pending.readUInt16BE(0) + 2) {+ resolve(pending.subarray(2, pending.readUInt16BE(0) + 2))+ }+ })+ })+ const framed = tcpFrame(message)+ socket.write(framed.subarray(0, 1))+ socket.write(framed.subarray(1))+ return await response+ } finally {+ socket.destroy()+ }+}++async function fixture(t, configuredTargets = targets) {+ const upstream = await udpClient(t)+ const upstreamPort = upstream.address().port+ const respond = message => dnsResponse(message, question(message), 3, '203.0.113.7', 20)+ upstream.on('message', (message, rinfo) =>+ upstream.send(respond(message), rinfo.port, rinfo.address)+ )+ const tcpUpstream = net.createServer(socket => {+ let pending = Buffer.alloc(0)+ socket.on('data', chunk => {+ pending = Buffer.concat([pending, chunk])+ if (pending.length >= 2 && pending.length >= pending.readUInt16BE(0) + 2) {+ socket.end(tcpFrame(respond(pending.subarray(2, pending.readUInt16BE(0) + 2))))+ }+ })+ })+ tcpUpstream.listen(upstreamPort, '127.0.0.1')+ await once(tcpUpstream, 'listening')+ t.after(() => new Promise(resolve => tcpUpstream.close(resolve)))+ const proxy = createDnsProxy({+ targets: configuredTargets,+ upstream: '127.0.0.1',+ upstreamPort,+ listenHost: '127.0.0.1',+ dnsPort: 0,+ controlPort: 0,+ ttlSeconds: 10,+ })+ t.after(() => proxy.close())+ const ports = await proxy.start()+ const control = async (path, method = 'GET') => {+ const response = await fetch(`http://127.0.0.1:${ports.controlPort}${path}`, {+ method,+ signal: AbortSignal.timeout(2000),+ headers: { connection: 'close' },+ })+ return {+ status: response.status,+ body: response.status === 200 ? await response.json() : undefined,+ }+ }+ return { proxy, ports, control }+}++test('target configuration is explicit, bounded, unambiguous, and safe for arbitrary names', () => {+ const valid = { targets, upstream: '127.0.0.1' }+ assert.throws(() => readConfig({ TARGET_DNS: 'legacy.example.com', UPSTREAM_DNS: '127.0.0.1' }))+ assert.deepEqual(+ readConfig({ DNS_TARGETS_JSON: JSON.stringify(targets), UPSTREAM_DNS: '127.0.0.1' }),+ { ...valid, ttlSeconds: 300 }+ )+ for (const options of [+ { ...valid, targets: [] },+ { ...valid, targets: [targets[0], targets[0]] },+ { ...valid, targets: [targets[0], { ...targets[1], fqdn: targets[0].fqdn }] },+ { ...valid, targets: [{ ...targets[0], lane: '__proto__' }] },+ { ...valid, targets: [{ ...targets[0], ip: '999.1.2.3' }] },+ { ...valid, targets: [{ ...targets[0], fqdn: 'UPPER.example.com' }] },+ { ...valid, targets: [{ ...targets[0], fqdn: 'not-a-host' }] },+ { ...valid, targets: [{ ...targets[0], unknown: true }] },+ { ...valid, upstream: 'upstream.example.com' },+ { ...valid, ttlSeconds: 0 },+ { ...valid, ttlSeconds: 301 },+ { ...valid, dnsPort: -1 },+ { ...valid, upstreamPort: 0 },+ ])+ assert.throws(() => createDnsProxy(options))+ assert.equal({}.polluted, undefined)+})++test('parses exact DNS questions and rejects malformed or compressed question interpretation', () => {+ const wire = query('UI.example.com', 42)+ assert.deepEqual(question(wire), {+ name: 'ui.example.com',+ type: 1,+ queryClass: 1,+ id: 42,+ end: wire.length,+ })+ for (let length = 0; length < wire.length; length++)+ assert.equal(question(wire.subarray(0, length)), null)+ const compressed = Buffer.from(wire)+ compressed[12] = 0xc0+ assert.equal(question(compressed), null)+ const multiple = Buffer.from(wire)+ multiple.writeUInt16BE(2, 4)+ assert.equal(question(multiple), null)+})++test(+ 'UDP and TCP serve each lane independently with real A records and configured TTL',+ { timeout: 5000 },+ async t => {+ const { proxy, ports, control } = await fixture(t)+ const socket = await udpClient(t)+ for (const [index, target] of targets.entries()) {+ const message = query(target.fqdn, index + 20)+ const udp = await exchangeUdp(socket, ports.dnsPort, message)+ assert.deepEqual(udp, dnsResponse(message, question(message), 0, target.ip, 10))+ const tcp = await exchangeTcp(ports.dnsPort, message)+ assert.deepEqual(tcp, udp)+ }+ const state = (await control('/state')).body+ assert.equal(state.fault, null)+ assert.equal(state.counts.ok, 4)+ assert.deepEqual(+ state.lanes.map(lane => lane.received.ok),+ [2, 2]+ )+ assert.deepEqual(+ state.lanes.map(lane => lane.counts.ok),+ [2, 2]+ )+ assert.deepEqual(state.lanes[0].questions.map(value => value.transport).sort(), ['tcp', 'udp'])+ assert.equal(proxy.snapshot().lanes[1].unique.ok, 2)+ }+)++test(+ 'duplicate UI retries cannot prove a worker observation; releasing hold emits SERVFAIL',+ { timeout: 5000 },+ async t => {+ const { proxy, ports, control } = await fixture(t)+ const socket = await udpClient(t)+ assert.equal((await control('/mode/ui/hold', 'POST')).status, 200)+ const message = query(targets[0].fqdn, 89)+ socket.send(message, ports.dnsPort, '127.0.0.1')+ socket.send(message, ports.dnsPort, '127.0.0.1')+ const deadline = Date.now() + 1500+ let state+ do {+ state = (await control('/state')).body+ } while (state.held !== 2 && Date.now() < deadline)+ assert.equal(state.held, 2)+ assert.equal(state.lanes[0].counts.hold, 2)+ assert.equal(state.lanes[0].unique.hold, 1)+ assert.equal(state.lanes[0].received.hold, 1)+ assert.equal(state.lanes[1].counts.hold, 0)+ const workerQuery = query(targets[1].fqdn, 90)+ assert.equal((await exchangeUdp(socket, ports.dnsPort, workerQuery)).readUInt16BE(2) & 15, 0)+ const received = once(socket, 'message', { signal: AbortSignal.timeout(2000) })+ await control('/mode/ui/servfail', 'POST')+ assert.deepEqual(+ (await received)[0],+ dnsResponse(message, question(message), 2, targets[0].ip, 10)+ )+ state = proxy.snapshot()+ assert.equal(state.held, 0)+ assert.equal(state.lanes[0].counts.servfail, 2)+ assert.equal(state.lanes[0].unique.servfail, 1)+ assert.equal(state.lanes[0].received.servfail, 0)+ assert.equal(state.lanes[1].counts.servfail, 0)+ assert.equal(state.fault, null)+ }+)++test(+ 'global modes preserve per-lane counters and recover without changing targets',+ { timeout: 5000 },+ async t => {+ const { proxy, ports, control } = await fixture(t)+ const socket = await udpClient(t)+ await control('/mode/servfail', 'POST')+ for (const [index, target] of targets.entries()) {+ assert.equal(+ (await exchangeUdp(socket, ports.dnsPort, query(target.fqdn, index + 40))).readUInt16BE(2) &+ 15,+ 2+ )+ }+ await control('/mode/ok', 'POST')+ for (const [index, target] of targets.entries()) {+ assert.equal(+ (await exchangeUdp(socket, ports.dnsPort, query(target.fqdn, index + 40))).readUInt16BE(2) &+ 15,+ 0+ )+ }+ const state = proxy.snapshot()+ assert.deepEqual(+ state.lanes.map(value => ({+ fqdn: value.fqdn,+ ok: value.unique.ok,+ failed: value.unique.servfail,+ })),+ targets.map(value => ({ fqdn: value.fqdn, ok: 1, failed: 1 }))+ )+ assert.equal((await control('/mode/__proto__/hold', 'POST')).status, 404)+ assert.equal((await control('/mode/worker/unknown', 'POST')).status, 404)+ assert.equal((await control('/mode/hold')).status, 404)+ assert.throws(() => proxy.setMode('constructor'))+ }+)++for (const releaseMode of ['ok', 'servfail']) {+ test(+ `received DNS requests exclude an expired hold flushed as ${releaseMode}`,+ { timeout: 5000 },+ async t => {+ const { proxy, ports, control } = await fixture(t)+ const resolver = new Resolver({ timeout: 25, tries: 1 })+ resolver.setServers([`127.0.0.1:${ports.dnsPort}`])+ await control('/mode/ui/hold', 'POST')+ await assert.rejects(resolver.resolve4(targets[0].fqdn), { code: 'ETIMEOUT' })+ const before = proxy.snapshot().lanes.find(lane => lane.lane === 'ui')+ assert.equal(before.held, 1, 'the DNS client has finished but its datagram remains queued')+ assert.deepEqual(before.received, { ok: 0, hold: 1, servfail: 0 })+ await control(`/mode/ui/${releaseMode}`, 'POST')+ const released = proxy.snapshot().lanes.find(lane => lane.lane === 'ui')+ assert.equal(+ released.counts[releaseMode],+ 1,+ 'legacy response counters still include the queued response'+ )+ assert.deepEqual(released.received, before.received, 'flushing does not receive a new query')+ if (releaseMode === 'servfail') {+ await assert.rejects(resolver.resolve4(targets[0].fqdn), { code: 'ESERVFAIL' })+ } else {+ assert.deepEqual(await resolver.resolve4(targets[0].fqdn), [targets[0].ip])+ }+ const after = proxy.snapshot()+ const observed = after.lanes.find(lane => lane.lane === 'ui')+ assert.equal(+ observed.received[releaseMode],+ 1,+ 'only a request received in the new mode counts'+ )+ assert.equal(observed.received.hold, 1)+ assert.deepEqual(after.lanes.find(lane => lane.lane === 'worker').received, {+ ok: 0,+ hold: 0,+ servfail: 0,+ })+ assert.equal(after.fault, null)+ }+ )+}++test(+ 'forwards unrelated UDP/TCP and target AAAA byte-for-byte even while target A is held',+ { timeout: 5000 },+ async t => {+ const { proxy, ports, control } = await fixture(t)+ const socket = await udpClient(t)+ await control('/mode/hold', 'POST')+ for (const message of [query('unrelated.example.com', 77), query(targets[0].fqdn, 78, 28)]) {+ const expected = dnsResponse(message, question(message), 3, '203.0.113.7', 20)+ assert.deepEqual(await exchangeUdp(socket, ports.dnsPort, message), expected)+ assert.deepEqual(await exchangeTcp(ports.dnsPort, message), expected)+ }+ const state = proxy.snapshot()+ assert.equal(state.held, 0)+ assert.equal(state.counts.forwarded, 4)+ assert.equal(state.lanes[0].counts.forwarded, 2)+ assert.equal(+ state.lanes[0].questions.every(value => value.type === 28 && value.mode === 'forwarded'),+ true+ )+ assert.equal(state.lanes[1].questions.length, 0)+ assert.deepEqual(state.lanes[0].received, { ok: 0, hold: 0, servfail: 0 })+ }+)++test(+ 'scoped UI/worker outages keep a third canary lane resolvable throughout',+ { timeout: 5000 },+ async t => {+ const canary = { lane: 'canary', fqdn: 'canary.example.com', ip: '93.184.216.36' }+ const { proxy, ports, control } = await fixture(t, [...targets, canary])+ const socket = await udpClient(t)+ for (const mode of ['hold', 'servfail', 'ok']) {+ assert.equal((await control(`/mode/ui/${mode}`, 'POST')).status, 200)+ assert.equal((await control(`/mode/worker/${mode}`, 'POST')).status, 200)+ const wire = query(canary.fqdn, mode === 'hold' ? 111 : mode === 'servfail' ? 112 : 113)+ assert.deepEqual(+ await exchangeUdp(socket, ports.dnsPort, wire),+ dnsResponse(wire, question(wire), 0, canary.ip, 10)+ )+ const state = proxy.snapshot().lanes.find(lane => lane.lane === 'canary')+ assert.equal(state.mode, 'ok')+ assert.equal(state.counts.hold, 0)+ assert.equal(state.counts.servfail, 0)+ }+ assert.equal(proxy.snapshot().lanes.find(lane => lane.lane === 'canary').unique.ok, 3)+ assert.deepEqual(proxy.snapshot().lanes.find(lane => lane.lane === 'canary').received, {+ ok: 3,+ hold: 0,+ servfail: 0,+ })+ assert.equal((await control('/mode/canary/servfail', 'POST')).status, 200)+ assert.equal(+ (await exchangeUdp(socket, ports.dnsPort, query(canary.fqdn, 114))).readUInt16BE(2) & 15,+ 2+ )+ assert.equal(proxy.snapshot().fault, null)+ }+)++test(+ 'closing with held UDP/TCP queries releases all listeners and does not manufacture success',+ { timeout: 5000 },+ async t => {+ const { proxy, ports, control } = await fixture(t)+ const socket = await udpClient(t)+ await control('/mode/hold', 'POST')+ socket.send(query(targets[0].fqdn), ports.dnsPort, '127.0.0.1')+ const tcp = net.createConnection({ host: '127.0.0.1', port: ports.dnsPort })+ await once(tcp, 'connect')+ tcp.on('error', error => assert.equal(error.code, 'ECONNRESET'))+ const closed = new Promise(resolve => tcp.once('close', resolve))+ tcp.write(tcpFrame(query(targets[1].fqdn)))+ await proxy.close()+ await closed+ assert.equal(proxy.snapshot().held, 0)+ assert.equal(proxy.snapshot().counts.ok, 0)+ await assert.rejects(proxy.start(), /cannot be started twice/)+ }+)++test(+ 'held-query overflow is bounded and explicitly invalidates fixture evidence',+ { timeout: 5000 },+ async t => {+ const { proxy, ports, control } = await fixture(t)+ await control('/mode/ui/hold', 'POST')+ const socket = net.createConnection({ host: '127.0.0.1', port: ports.dnsPort })+ socket.on('error', error => assert.equal(error.code, 'ECONNRESET'))+ await once(socket, 'connect')+ t.after(() => socket.destroy())+ socket.write(+ Buffer.concat(Array.from({ length: 513 }, (_, id) => tcpFrame(query(targets[0].fqdn, id))))+ )+ const deadline = Date.now() + 1500+ let state+ do {+ state = (await control('/state')).body+ } while (state.fault === null && Date.now() < deadline)+ assert.equal(state.fault, 'HELD_QUERY_LIMIT')+ assert.equal(state.held, 512)+ assert.equal(state.lanes[0].counts.hold, 513)+ assert.equal(state.lanes[1].counts.hold, 0)+ assert.equal(state.counts.ok, 0)+ await proxy.close()+ }+)diff --git a/scripts/tests/wrc-egress-gate.test.cjs b/scripts/tests/wrc-egress-gate.test.cjsnew file mode 100644index 000000000..fcec1c59f--- /dev/null+++ b/scripts/tests/wrc-egress-gate.test.cjs@@ -0,0 +1,867 @@+'use strict'++// Integration contract for the gate's exported lifecycle. Kubernetes is a+// boundary fake; journals are real, private files in a test-owned directory.+const { test } = require('node:test')+const assert = require('node:assert/strict')+const fs = require('node:fs')+const os = require('node:os')+const path = require('node:path')+const { randomUUID } = require('node:crypto')+const {+ restore,+ protectedRun,+ fixtureObjects,+ deploymentReady,+ policyObservation,+ acceptPolicyIntentChange,+} = require('../e2e/_lib/wrc-egress-gate.cjs')+const {+ Journal,+ journalPath,+ readJournal,+ RUN_LABEL,+} = require('../e2e/_lib/wrc-egress-lifecycle.cjs')+const { readConfig } = require('../../tests/e2e/fixtures/wrc-egress-dns-proxy/server.cjs')++const source = fs.realpathSync(path.resolve(__dirname, '../..'))+const key = identity => `${identity.kind}/${identity.namespace}/${identity.name}`+const clone = value => (value === undefined ? undefined : structuredClone(value))+const apiError = code => Object.assign(new Error(`fixture API ${code}`), { code })++function fixture(t) {+ const directory = fs.mkdtempSync(path.join(fs.realpathSync(os.tmpdir()), 'wrc-gate-test-'))+ t.after(() => fs.rmSync(directory, { recursive: true, force: true }))+ const runId = randomUUID()+ const binding = {+ repository: directory,+ branch: 'fix/unit-gate',+ profile: 'unit-profile',+ context: 'unit-profile',+ head: 'unit-head',+ }+ const injectedConfig = { nameservers: ['10.96.0.20'], options: [{ name: 'timeout', value: '2' }] }+ const wrcIdentity = { kind: 'Deployment', namespace: 'control-plane', name: 'workflow-recipes' }+ const resources = [+ {+ kind: 'ConfigMap',+ namespace: 'control-plane',+ name: 'unit-script',+ attempted: true,+ uid: 'script-uid',+ },+ {+ kind: 'Service',+ namespace: 'control-plane',+ name: 'unit-dns',+ attempted: true,+ uid: 'dns-uid',+ },+ {+ kind: 'Deployment',+ namespace: 'control-plane',+ name: 'unit-proxy',+ attempted: true,+ uid: 'proxy-uid',+ },+ {+ kind: 'WorkflowRecipe',+ namespace: 'sandbox-recipes',+ name: 'unit-recipe',+ attempted: true,+ uid: 'recipe-uid',+ },+ ]+ const state = {+ version: 1,+ runId,+ binding,+ phase: 'exercising',+ resources,+ uiCleanup: null,+ wrc: {+ identity: wrcIdentity,+ uid: 'wrc-uid',+ replicas: 1,+ originalPolicy: 'ClusterFirst',+ injectedConfig,+ attempted: true,+ },+ }+ const file = journalPath(directory, binding.profile)+ const journal = new Journal(file, state)+ const objects = new Map(+ resources.map(entry => [+ key(entry),+ {+ apiVersion: 'v1',+ kind: entry.kind,+ metadata: {+ name: entry.name,+ namespace: entry.namespace,+ uid: entry.uid,+ resourceVersion: '1',+ labels: { [RUN_LABEL]: runId },+ },+ },+ ])+ )+ const wrc = {+ apiVersion: 'apps/v1',+ kind: 'Deployment',+ metadata: {+ name: wrcIdentity.name,+ namespace: wrcIdentity.namespace,+ uid: state.wrc.uid,+ resourceVersion: '7',+ generation: 2,+ },+ spec: {+ replicas: 1,+ template: { spec: { dnsPolicy: 'None', dnsConfig: clone(injectedConfig) } },+ },+ status: { observedGeneration: 2, readyReplicas: 1, availableReplicas: 1 },+ }+ objects.set(key(wrcIdentity), wrc)+ const events = []+ const hooks = {}+ const kube = {+ setCleanup(enabled) {+ events.push({ action: 'cleanup', enabled })+ },+ async get(identity) {+ events.push({ action: 'get', identity: clone(identity) })+ if (hooks.get) {+ const result = await hooks.get(identity)+ if (result !== undefined) return clone(result)+ }+ return clone(objects.get(key(identity)) ?? null)+ },+ async list(kind, namespace, selector) {+ events.push({ action: 'list', kind, namespace, selector })+ if (hooks.list) {+ const result = await hooks.list(kind, namespace, selector)+ if (result !== undefined) return clone(result)+ }+ const singular = new Map([+ ['deployments', 'Deployment'],+ ['services', 'Service'],+ ['networkpolicies', 'NetworkPolicy'],+ ['pods', 'Pod'],+ ['replicasets', 'ReplicaSet'],+ ]).get(kind)+ const labels = selector.split(',').map(pair => pair.split('='))+ return [...objects.values()]+ .filter(+ object =>+ object.kind === singular &&+ object.metadata.namespace === namespace &&+ labels.every(([name, value]) => object.metadata.labels?.[name] === value)+ )+ .map(clone)+ },+ async patch(identity, patch) {+ events.push({ action: 'patch', identity: clone(identity), patch: clone(patch) })+ if (hooks.patch) await hooks.patch(identity, patch)+ const live = objects.get(key(identity))+ for (const operation of patch) {+ if (operation.op === 'test' && operation.path === '/metadata/uid')+ assert.equal(operation.value, live.metadata.uid)+ else if (operation.op === 'test' && operation.path === '/metadata/resourceVersion')+ assert.equal(operation.value, live.metadata.resourceVersion)+ else if (operation.op === 'replace' && operation.path === '/spec/template/spec/dnsPolicy')+ live.spec.template.spec.dnsPolicy = operation.value+ else if (operation.op === 'remove' && operation.path === '/spec/template/spec/dnsConfig')+ delete live.spec.template.spec.dnsConfig+ else throw new Error('unexpected mutation in restoration contract')+ }+ live.metadata.resourceVersion = String(Number(live.metadata.resourceVersion) + 1)+ live.metadata.generation+++ return clone(live)+ },+ async rollout(namespace, name) {+ events.push({ action: 'rollout', namespace, name })+ const live = objects.get(key({ kind: 'Deployment', namespace, name }))+ if (hooks.rollout) await hooks.rollout(live)+ live.status = {+ observedGeneration: live.metadata.generation,+ readyReplicas: live.spec.replicas,+ availableReplicas: live.spec.replicas,+ }+ },+ async delete(identity, preconditions) {+ events.push({+ action: 'delete',+ identity: clone(identity),+ preconditions: clone(preconditions),+ })+ if (hooks.delete) await hooks.delete(identity, preconditions)+ const live = objects.get(key(identity))+ assert.equal(preconditions.uid, live.metadata.uid)+ assert.equal(preconditions.resourceVersion, live.metadata.resourceVersion)+ objects.delete(key(identity))+ },+ }+ return { directory, file, journal, state, wrc, objects, kube, hooks, events, wrcIdentity }+}++const mutations = events =>+ events.filter(event => event.action === 'patch' || event.action === 'delete')++function uiFixture(t, { assigned = true, children = true } = {}) {+ const fixtureState = fixture(t)+ const { state, objects, journal } = fixtureState+ const parent = state.resources.find(entry => entry.kind === 'WorkflowRecipe')+ const parentObject = objects.get(key(parent))+ parentObject.spec = {+ ui: { workloadRef: 'frontend' },+ workloads: [{ id: 'frontend', type: 'deployment', port: 3000 }],+ }+ const physicalName = 'wf-unit-recipe-frontend-a17c5e92'+ if (assigned)+ parentObject.status = { workloadInstances: { frontend: physicalName }, phase: 'candidate' }+ state.uiCleanup = {+ parent: { kind: parent.kind, name: parent.name, namespace: parent.namespace },+ parentUid: parent.uid,+ runId: state.runId,+ namespace: 'sandbox-ui',+ workload: 'frontend',+ physicalName: null,+ children: [],+ }+ const common = { 'clerum.io/managed-by': 'workflow-recipes', 'clerum.io/recipe': parent.name }+ const uiLabels = {+ 'clerum.io/sandbox-ui': 'true',+ 'clerum.io/recipe-name': parent.name,+ 'clerum.io/recipe-namespace': parent.namespace,+ }+ const uiObjects = [+ {+ kind: 'Deployment',+ metadata: {+ name: physicalName,+ namespace: 'sandbox-ui',+ uid: 'ui-deployment-uid',+ resourceVersion: '10',+ labels: { ...common, 'clerum.io/workload': 'frontend' },+ },+ },+ {+ kind: 'Service',+ metadata: {+ name: physicalName,+ namespace: 'sandbox-ui',+ uid: 'ui-service-uid',+ resourceVersion: '11',+ labels: { ...common, ...uiLabels, 'clerum.io/workload': 'frontend' },+ },+ },+ {+ kind: 'NetworkPolicy',+ metadata: {+ name: `ui-egress-${parent.name}`,+ namespace: 'sandbox-ui',+ uid: 'ui-policy-uid',+ resourceVersion: '12',+ labels: {+ ...common,+ 'clerum.io/recipe-name': parent.name,+ 'clerum.io/recipe-namespace': parent.namespace,+ },+ },+ },+ ]+ if (children)+ for (const object of uiObjects)+ objects.set(key({ kind: object.kind, ...object.metadata }), object)+ journal.save()+ return { ...fixtureState, parent, parentObject, physicalName, uiObjects }+}++test('foreground recipe deletion cannot leave UI children after its finalizer swallows DELETE503', async t => {+ const { kube, journal, objects, hooks, file, events, parent, physicalName, uiObjects } =+ uiFixture(t)+ const rawCollision = {+ kind: 'Deployment',+ metadata: {+ name: 'frontend',+ namespace: 'sandbox-ui',+ uid: 'unrelated-uid',+ resourceVersion: '1',+ labels: { 'clerum.io/recipe': 'unrelated' },+ },+ }+ objects.set('Deployment/sandbox-ui/frontend', rawCollision)+ let recordedBeforeParentDeletion = false+ hooks.delete = identity => {+ if (identity.kind === 'WorkflowRecipe') {+ const saved = readJournal(file).uiCleanup+ recordedBeforeParentDeletion =+ saved.physicalName === physicalName &&+ saved.parentUid === parent.uid &&+ uiObjects.every(object => saved.children.some(child => child.uid === object.metadata.uid))+ // Real WRC safeDelete absorbs the simulated UI DELETE503, removes its+ // finalizer, and allows parent deletion; the cross-namespace objects stay.+ }+ }+ await restore(kube, journal)+ assert.equal(recordedBeforeParentDeletion, true)+ assert.equal(fs.existsSync(file), false)+ assert.equal(objects.get('Deployment/sandbox-ui/frontend'), rawCollision)+ for (const object of uiObjects) {+ assert.equal(objects.has(key({ kind: object.kind, ...object.metadata })), false)+ const deletion = events.find(+ event =>+ event.action === 'delete' &&+ event.identity.kind === object.kind &&+ event.identity.namespace === 'sandbox-ui'+ )+ assert.deepEqual(deletion.preconditions, {+ uid: object.metadata.uid,+ resourceVersion: object.metadata.resourceVersion,+ })+ }+ assert.equal(+ events.some(+ event => event.identity?.namespace === 'sandbox-ui' && event.identity.name === 'frontend'+ ),+ false+ )+})++test('failed leftover delete preserves recorded UI identities for recovery after the parent is absent', async t => {+ const { kube, journal, hooks, file, objects, parent, uiObjects } = uiFixture(t)+ hooks.delete = identity => {+ if (identity.namespace === 'sandbox-ui') throw apiError(503)+ }+ await assert.rejects(restore(kube, journal), /WRC_EGRESS_RECOVERY_REQUIRED/)+ const saved = readJournal(file)+ assert.equal(saved.phase, 'recovery-required')+ assert.equal(objects.has(key(parent)), false)+ assert(saved.uiCleanup.children.some(child => child.uid === 'ui-deployment-uid'))+ hooks.delete = undefined+ await restore(kube, new Journal(file, saved, true))+ assert.equal(fs.existsSync(file), false)+ for (const object of uiObjects)+ assert.equal(objects.has(key({ kind: object.kind, ...object.metadata })), false)+})++test('a replaced UI UID is never adopted during recovery, even with matching name and labels', async t => {+ const { kube, journal, hooks, file, events, uiObjects } = uiFixture(t)+ hooks.delete = identity => {+ if (identity.kind === 'WorkflowRecipe') uiObjects[0].metadata.uid = 'replacement-uid'+ }+ await assert.rejects(restore(kube, journal), /WRC_EGRESS_RECOVERY_REQUIRED/)+ assert.equal(+ readJournal(file).uiCleanup.children.find(child => child.kind === 'Deployment').uid,+ 'ui-deployment-uid'+ )+ assert.equal(+ events.some(event => event.action === 'delete' && event.identity.namespace === 'sandbox-ui'),+ false+ )+ await assert.rejects(+ restore(kube, new Journal(file, readJournal(file), true)),+ /WRC_EGRESS_RECOVERY_REQUIRED/+ )+ assert.equal(+ events.some(event => event.action === 'delete' && event.identity.namespace === 'sandbox-ui'),+ false+ )+})++test('a foreign UI owner is refused before deleting its parent or any UI resource', async t => {+ const { kube, journal, uiObjects, events, file } = uiFixture(t)+ uiObjects[0].metadata.labels['clerum.io/recipe'] = 'foreign-recipe'+ await assert.rejects(restore(kube, journal), /WRC_EGRESS_RECOVERY_REQUIRED/)+ assert.equal(readJournal(file).phase, 'recovery-required')+ assert.equal(+ events.some(event => event.action === 'delete'),+ false+ )+})++test('optional Deployment UI labels may be absent but cannot contradict the parent scope', async t => {+ const { kube, journal, uiObjects, events, file } = uiFixture(t)+ uiObjects[0].metadata.labels['clerum.io/recipe-namespace'] = 'foreign-namespace'+ await assert.rejects(restore(kube, journal), /WRC_EGRESS_RECOVERY_REQUIRED/)+ assert.equal(readJournal(file).phase, 'recovery-required')+ assert.equal(+ events.some(event => event.action === 'delete'),+ false+ )+})++test('UI discovery read503 retains recovery-required; authoritative absence before Ready is a valid no-op', async t => {+ const failed = uiFixture(t)+ failed.hooks.get = identity => {+ if (identity.namespace === 'sandbox-ui') throw apiError(503)+ }+ await assert.rejects(restore(failed.kube, failed.journal), /WRC_EGRESS_RECOVERY_REQUIRED/)+ assert.equal(readJournal(failed.file).phase, 'recovery-required')+ assert.equal(+ failed.events.some(event => event.action === 'delete'),+ false+ )+ for (const assigned of [false, true]) {+ const absent = uiFixture(t, { assigned, children: false })+ await restore(absent.kube, absent.journal)+ assert.equal(fs.existsSync(absent.file), false)+ assert.equal(+ absent.events.some(+ event => event.action === 'delete' && event.identity.namespace === 'sandbox-ui'+ ),+ false+ )+ assert.equal(+ absent.events.some(event => event.identity?.name === 'frontend'),+ false+ )+ }+})++test('parent already absent without child UID capture cannot certify or adopt an orphan UI', async t => {+ const { kube, journal, parent, objects, file, events } = uiFixture(t)+ objects.delete(key(parent))+ await assert.rejects(restore(kube, journal), /WRC_EGRESS_RECOVERY_REQUIRED/)+ assert.equal(readJournal(file).phase, 'recovery-required')+ assert.equal(+ events.some(event => event.action === 'delete' && event.identity.namespace === 'sandbox-ui'),+ false+ )+ const empty = uiFixture(t, { assigned: false, children: false })+ empty.objects.delete(key(empty.parent))+ await restore(empty.kube, empty.journal)+ assert.equal(fs.existsSync(empty.file), false)+})++test('invalid physical names from status are refused before they reach a Kubernetes argument', async t => {+ const { kube, journal, parentObject, events, file } = uiFixture(t)+ parentObject.status.workloadInstances.frontend = '--context=foreign'+ await assert.rejects(restore(kube, journal), /WRC_EGRESS_RECOVERY_REQUIRED/)+ assert.equal(+ events.some(event => event.identity?.name === '--context=foreign'),+ false+ )+ assert.equal(+ events.some(event => event.action === 'delete'),+ false+ )+ assert.equal(readJournal(file).phase, 'recovery-required')+})++test('UI census errors cannot clear the journal after acknowledged child deletion', async t => {+ const { kube, journal, hooks, file } = uiFixture(t)+ hooks.list = () => {+ throw apiError(503)+ }+ await assert.rejects(restore(kube, journal), /WRC_EGRESS_RECOVERY_REQUIRED/)+ assert.equal(readJournal(file).phase, 'recovery-required')+})++test('an old journal with recipe deletion but no UI cleanup declaration cannot certify absence', async t => {+ const { kube, journal, state, file, events } = fixture(t)+ delete state.uiCleanup+ journal.save()+ await assert.rejects(restore(kube, journal), /WRC_EGRESS_RECOVERY_REQUIRED/)+ assert.equal(readJournal(file).phase, 'recovery-required')+ assert.equal(+ events.some(event => event.action === 'delete'),+ false+ )+})++test('a UI child first appearing after parent deletion has no adoptable recorded UID', async t => {+ const { kube, journal, hooks, objects, uiObjects, file, events } = uiFixture(t, {+ children: false,+ })+ hooks.delete = identity => {+ if (identity.kind === 'WorkflowRecipe') {+ objects.set(key({ kind: uiObjects[0].kind, ...uiObjects[0].metadata }), uiObjects[0])+ }+ }+ await assert.rejects(restore(kube, journal), /WRC_EGRESS_RECOVERY_REQUIRED/)+ assert.equal(+ readJournal(file).uiCleanup.children.find(child => child.kind === 'Deployment').uid,+ null+ )+ assert.equal(+ events.some(event => event.action === 'delete' && event.identity.namespace === 'sandbox-ui'),+ false+ )+})++test('a foreign parent UID cannot authorize capturing or deleting its UI children', async t => {+ const { kube, journal, parentObject, file, events } = uiFixture(t)+ parentObject.metadata.uid = 'foreign-parent-uid'+ await assert.rejects(restore(kube, journal), /WRC_EGRESS_RECOVERY_REQUIRED/)+ assert.equal(readJournal(file).phase, 'recovery-required')+ assert.equal(+ events.some(event => event.action === 'delete'),+ false+ )+})++test('UI background ReplicaSets and Pods must disappear before the journal is removed', async t => {+ const { kube, journal, hooks, objects, parent, physicalName, events, file } = uiFixture(t)+ for (const kind of ['ReplicaSet', 'Pod']) {+ const name = `${physicalName}-${kind.toLowerCase()}`+ objects.set(`${kind}/sandbox-ui/${name}`, {+ kind,+ metadata: {+ name,+ namespace: 'sandbox-ui',+ uid: `${kind}-uid`,+ labels: { 'clerum.io/recipe': parent.name },+ },+ })+ }+ hooks.list = kind => {+ const singular = kind === 'replicasets' ? 'ReplicaSet' : kind === 'pods' ? 'Pod' : null+ if (!singular) return undefined+ const lingering = [...objects.values()].filter(object => object.kind === singular)+ for (const object of lingering)+ objects.delete(`${object.kind}/${object.metadata.namespace}/${object.metadata.name}`)+ return lingering+ }+ await restore(kube, journal)+ assert.equal(fs.existsSync(file), false)+ assert(events.filter(event => event.action === 'list' && event.kind === 'pods').length >= 2)+ assert.equal(+ events.some(+ event => event.action === 'delete' && ['Pod', 'ReplicaSet'].includes(event.identity.kind)+ ),+ false+ )+})++test('recovery requires a newer controller observation for every retained tuple, not old policy or DNS counters', () => {+ const policy = entries => ({+ metadata: {+ annotations: {+ 'clerum.io/egress-fqdn-state': JSON.stringify(entries),+ },+ },+ })+ const retained = [+ { lastObservedAt: 100, expiresAt: 500 },+ { lastObservedAt: 101, expiresAt: 510 },+ ]+ assert.deepEqual(policyObservation(policy(retained)), { lastAcceptedAt: 101, latestExpiry: 510 })+ assert.equal(policyObservation(policy(retained), 100), null)+ assert.equal(policyObservation(policy(retained), 101), null)+ assert.deepEqual(policyObservation(policy([{ lastObservedAt: 102, expiresAt: 600 }]), 101), {+ lastAcceptedAt: 102,+ latestExpiry: 600,+ })+ assert.throws(() => policyObservation(policy([])), /MISSING_ACCEPTED/)+ assert.throws(+ () => policyObservation(policy([{ lastObservedAt: null, expiresAt: 600 }])),+ /MISSING_ACCEPTED/+ )+})++test('only confirmed same-recipe intent change releases policy UID pins for a fully reproven migration', () => {+ const targets = [+ { lane: 'ui', fqdn: 'ui.example.com' },+ { lane: 'worker', fqdn: 'worker.example.com' },+ ]+ const before = { metadata: { uid: 'recipe-uid', generation: 7 } }+ for (const after of [+ { metadata: { uid: 'recipe-uid', generation: 7 } },+ { metadata: { uid: 'foreign-uid', generation: 8 } },+ ]) {+ const lanes = [+ { lane: 'ui', uid: 'ui-policy-uid' },+ { lane: 'workload', uid: 'worker-policy-uid' },+ ]+ assert.throws(+ () => acceptPolicyIntentChange(lanes, before, after, 'recipe-uid', targets),+ /HOST_MIGRATION_NOT_APPLIED/+ )+ assert.deepEqual(+ lanes.map(lane => lane.uid),+ ['ui-policy-uid', 'worker-policy-uid']+ )+ }+ const lanes = [+ { lane: 'ui', uid: 'ui-policy-uid' },+ { lane: 'workload', uid: 'worker-policy-uid' },+ ]+ acceptPolicyIntentChange(+ lanes,+ before,+ { metadata: { uid: 'recipe-uid', generation: 8 } },+ 'recipe-uid',+ targets+ )+ assert.deepEqual(+ lanes.map(lane => lane.uid),+ [undefined, undefined]+ )+ assert.deepEqual(+ lanes.map(lane => lane.target.fqdn),+ ['worker.example.com', 'ui.example.com']+ )+})++test('restores exact DNS fields with CAS and proves rollout before deleting owned resources', async t => {+ const { kube, journal, file, wrc, events, state } = fixture(t)+ await restore(kube, journal)+ assert.equal(wrc.spec.template.spec.dnsPolicy, 'ClusterFirst')+ assert.equal(wrc.spec.template.spec.dnsConfig, undefined)+ assert.equal(deploymentReady(wrc), true)+ assert.equal(fs.existsSync(file), false)+ assert.equal(state.phase, 'restored')+ assert.deepEqual(events[0], { action: 'cleanup', enabled: true })+ const patch = events.find(event => event.action === 'patch').patch+ assert.deepEqual(patch.slice(0, 2), [+ { op: 'test', path: '/metadata/uid', value: 'wrc-uid' },+ { op: 'test', path: '/metadata/resourceVersion', value: '7' },+ ])+ assert.deepEqual(+ events.filter(event => event.action === 'delete').map(event => event.identity.name),+ ['unit-recipe', 'unit-proxy', 'unit-dns', 'unit-script']+ )+ const firstDelete = events.findIndex(event => event.action === 'delete')+ assert(events.slice(0, firstDelete).some(event => event.action === 'rollout'))+})++test('restoration recognizes already restored default DNS without rewriting it', async t => {+ const { kube, journal, wrc, events, file } = fixture(t)+ delete wrc.spec.template.spec.dnsPolicy+ delete wrc.spec.template.spec.dnsConfig+ await restore(kube, journal)+ assert.equal(+ events.some(event => event.action === 'patch'),+ false+ )+ assert.equal(fs.existsSync(file), false)+})++for (const scenario of [+ 'uid',+ 'replicas',+ 'dns',+ 'absent',+ 'read-error',+ 'patch-conflict',+ 'rollout-error',+ 'readback-drift',+]) {+ test(`uncertain ${scenario} restoration retains every DNS resource and the recovery journal`, async t => {+ const { kube, journal, wrc, events, hooks, file, wrcIdentity, objects } = fixture(t)+ if (scenario === 'uid') wrc.metadata.uid = 'foreign-wrc'+ if (scenario === 'replicas') wrc.spec.replicas = 2+ if (scenario === 'dns') wrc.spec.template.spec.dnsConfig = { nameservers: ['10.96.0.99'] }+ if (scenario === 'absent') objects.delete(key(wrcIdentity))+ if (scenario === 'read-error')+ hooks.get = () => {+ throw apiError(503)+ }+ if (scenario === 'patch-conflict')+ hooks.patch = () => {+ throw apiError(409)+ }+ if (scenario === 'rollout-error')+ hooks.rollout = () => {+ throw apiError(503)+ }+ if (scenario === 'readback-drift')+ hooks.rollout = live => {+ live.spec.template.spec.dnsConfig = { nameservers: ['10.96.0.99'] }+ }+ await assert.rejects(restore(kube, journal), /WRC_EGRESS_RECOVERY_REQUIRED/)+ assert.equal(readJournal(file).phase, 'recovery-required')+ assert.equal(readJournal(file).wrc.attempted, true)+ assert.equal(+ events.some(event => event.action === 'delete'),+ false+ )+ if (['uid', 'replicas', 'dns', 'absent', 'read-error'].includes(scenario))+ assert.equal(mutations(events).length, 0)+ })+}++test('replica changes during restoration readback cannot be certified as baseline restored', async t => {+ const { kube, journal, hooks, file, events } = fixture(t)+ hooks.rollout = live => {+ live.spec.replicas = 2+ }+ await assert.rejects(restore(kube, journal), /WRC_EGRESS_RECOVERY_REQUIRED/)+ assert.equal(readJournal(file).phase, 'recovery-required')+ assert.equal(+ events.some(event => event.action === 'delete'),+ false+ )+})++test('an unmutated failed preflight produces no resource write and removes only its own journal', async t => {+ const { kube, journal, state, file, events } = fixture(t)+ state.resources = []+ delete state.wrc+ journal.save()+ const original = new Error('ownership/precondition refused')+ await assert.rejects(+ protectedRun(+ kube,+ journal,+ async () => {+ throw original+ },+ new AbortController()+ ),+ error => error === original+ )+ assert.equal(mutations(events).length, 0)+ assert.equal(fs.existsSync(file), false)+})++test('partial creation before DNS injection deletes only acknowledged existing identities', async t => {+ const { kube, journal, state, objects, file, events } = fixture(t)+ state.wrc.attempted = false+ state.resources = state.resources.slice(0, 2)+ objects.delete(key(state.resources[1]))+ state.resources[1].uid = null+ journal.save()+ await restore(kube, journal)+ assert.deepEqual(+ mutations(events).map(event => `${event.action}:${event.identity.name}`),+ ['delete:unit-script']+ )+ assert.equal(fs.existsSync(file), false)+})++for (const conflict of ['uid', 'label', 'unacknowledged']) {+ test(`cleanup refuses ${conflict} ownership while preserving a recovery obligation`, async t => {+ const { kube, journal, state, objects, file, events } = fixture(t)+ state.wrc.attempted = false+ const foreign = state.resources[0]+ const live = objects.get(key(foreign))+ if (conflict === 'uid') live.metadata.uid = 'replacement-resource'+ if (conflict === 'label') live.metadata.labels[RUN_LABEL] = randomUUID()+ if (conflict === 'unacknowledged') foreign.uid = null+ journal.save()+ await assert.rejects(restore(kube, journal), /WRC_EGRESS_RECOVERY_REQUIRED/)+ assert.equal(+ events.some(event => event.action === 'delete' && event.identity.name === foreign.name),+ false+ )+ assert.equal(objects.has(key(foreign)), true)+ assert.equal(readJournal(file).phase, 'recovery-required')+ })+}++test('an unavailable cleanup GET cannot be confused with confirmed resource absence', async t => {+ const { kube, journal, hooks, state, file, events } = fixture(t)+ state.wrc.attempted = false+ hooks.get = identity => {+ if (identity.kind === 'Service') throw apiError(503)+ }+ await assert.rejects(restore(kube, journal), /WRC_EGRESS_RECOVERY_REQUIRED/)+ assert.equal(readJournal(file).phase, 'recovery-required')+ assert.equal(+ events.some(event => event.action === 'delete' && event.identity.kind === 'Service'),+ false+ )+})++test('failed exercise is preserved after successful restoration; cleanup failure includes both causes', async t => {+ const first = fixture(t)+ const original = new Error('business assertion failed')+ await assert.rejects(+ protectedRun(+ first.kube,+ first.journal,+ async () => {+ throw original+ },+ new AbortController()+ ),+ error => error === original+ )+ assert.equal(fs.existsSync(first.file), false)+ const second = fixture(t)+ second.hooks.rollout = () => {+ throw apiError(503)+ }+ await assert.rejects(+ protectedRun(+ second.kube,+ second.journal,+ async () => {+ throw original+ },+ new AbortController()+ ),+ error =>+ error instanceof AggregateError && error.errors[0] === original && error.errors.length === 2+ )+ assert.equal(readJournal(second.file).phase, 'recovery-required')+})++for (const exitCode of [130, 143]) {+ for (const timing of ['exercise', 'restoration']) {+ test(`interruption ${exitCode} during ${timing} cannot become a green protected run`, async t => {+ const { kube, journal, hooks, file, events } = fixture(t)+ const cancellation = new AbortController()+ const interruption = Object.assign(new Error('interrupted'), { exitCode })+ if (timing === 'restoration') hooks.rollout = () => cancellation.abort(interruption)+ await assert.rejects(+ protectedRun(+ kube,+ journal,+ async () => {+ if (timing === 'exercise') cancellation.abort(interruption)+ },+ cancellation+ ),+ error => error === interruption+ )+ assert.equal(fs.existsSync(file), false)+ assert.equal(events.filter(event => event.action === 'delete').length, 4)+ })+ }+}++test('fixture manifests preserve distinct DNS lanes, scoped policies and a valid independent exact-host canary', () => {+ const config = {+ repository: source,+ wrcNamespace: 'control-plane',+ wrcDeployment: 'workflow-recipes',+ recipeNamespace: 'sandbox-recipes',+ uiNamespace: 'sandbox-ui',+ }+ const runId = randomUUID()+ const result = fixtureObjects(config, runId, 'wrc:test', '93.184.216.34', '10.96.0.10')+ for (const object of [...result.objects, result.recipe, result.canary])+ assert.equal(object.metadata.labels[RUN_LABEL], runId)+ const proxy = result.objects.find(object => object.kind === 'Deployment')+ const container = proxy.spec.template.spec.containers[0]+ const env = Object.fromEntries(container.env.map(entry => [entry.name, entry.value]))+ const dnsConfig = readConfig(env)+ assert.deepEqual(dnsConfig.targets.map(target => target.lane).sort(), ['canary', 'ui', 'worker'])+ assert.equal(new Set(dnsConfig.targets.map(target => target.fqdn)).size, 3)+ assert.equal(proxy.spec.template.spec.automountServiceAccountToken, false)+ assert.equal(container.securityContext.allowPrivilegeEscalation, false)+ const witness = result.canary.spec.workloads[0]+ assert.equal(witness.transport, undefined)+ const canaryTarget = dnsConfig.targets.find(target => target.lane === 'canary')+ assert.deepEqual(+ witness.egressBindings,+ [80, 443].map(port => ({ dns: canaryTarget.fqdn, port, protocol: 'TCP' }))+ )+ assert.deepEqual(+ result.recipe.spec.ui.egress.internal,+ [3000, 3001].map(port => ({ workloadRef: 'worker', port }))+ )+ for (const workload of result.recipe.spec.workloads) {+ assert.equal(workload.port, 3000)+ assert.equal(workload.healthCheck.port, 3001)+ }+ const fromWrc = result.objects.find(object => object.metadata.name === result.names.fromWrc)+ assert.deepEqual(fromWrc.spec.podSelector, { matchLabels: { app: config.wrcDeployment } })+ assert.deepEqual(fromWrc.spec.egress[0].to, [+ { podSelector: { matchLabels: { app: result.names.proxy } } },+ ])+})diff --git a/scripts/tests/wrc-egress-lifecycle.test.cjs b/scripts/tests/wrc-egress-lifecycle.test.cjsnew file mode 100644index 000000000..4755699d4--- /dev/null+++ b/scripts/tests/wrc-egress-lifecycle.test.cjs@@ -0,0 +1,846 @@+'use strict'++// Unit/contract tests for the development-only lifecycle. Kubernetes is mocked+// only at its boundary; child-process supervision uses real local processes.+// These tests do not certify a browser journey or cluster/dataplane behavior.+const { test } = require('node:test')+const assert = require('node:assert/strict')+const fs = require('node:fs')+const os = require('node:os')+const path = require('node:path')+const { randomUUID } = require('node:crypto')+const { spawnSync } = require('node:child_process')+const {+ Journal,+ journalPath,+ readJournal,+ assertJournalBinding,+ createOwned,+ deleteOwned,+ waitFor,+ createCommandRunner,+ createKubeClient,+ resourcePath,+ RUN_LABEL,+} = require('../e2e/_lib/wrc-egress-lifecycle.cjs')++const repository = fs.realpathSync(path.resolve(__dirname, '../..'))+const lifecycle = path.join(repository, 'scripts/e2e/_lib/wrc-egress-lifecycle.cjs')+function temporary(t) {+ const directory = fs.mkdtempSync(path.join(fs.realpathSync(os.tmpdir()), 'wrc-lifecycle-test-'))+ t.after(() => fs.rmSync(directory, { recursive: true, force: true }))+ return directory+}+function fixture(t) {+ const directory = temporary(t)+ const binding = {+ repository: directory,+ branch: 'fix/unit-lifecycle',+ profile: 'unit-profile',+ context: 'unit-context',+ head: 'historical-head',+ }+ const state = { version: 1, runId: randomUUID(), binding, resources: [] }+ const file = journalPath(directory, binding.profile)+ const journal = new Journal(file, state)+ const object = {+ apiVersion: 'v1',+ kind: 'ConfigMap',+ metadata: {+ name: 'unit-resource',+ namespace: 'unit-namespace',+ labels: { [RUN_LABEL]: state.runId },+ },+ data: { fixture: 'unit' },+ }+ const created = structuredClone(object)+ created.metadata.uid = 'created-uid'+ created.metadata.resourceVersion = 'initial-rv'+ return { directory, binding, state, file, journal, object, created }+}+function apiError(code) {+ return Object.assign(new Error(`API ${code}`), { code })+}++test('journal creates private durable state, saves atomically, and removes only its own run', t => {+ const { journal, file, state } = fixture(t)+ assert.equal(fs.statSync(file).mode & 0o777, 0o600)+ assert.deepEqual(readJournal(file), state)+ state.phase = 'restoring'+ journal.save()+ assert.equal(readJournal(file).phase, 'restoring')+ assert.equal(fs.readdirSync(path.dirname(file)).length, 1)+ journal.remove()+ assert.equal(fs.existsSync(file), false)+})++test('journal binding permits historical HEAD but rejects every stable owner mismatch', t => {+ const { state, binding } = fixture(t)+ assert.doesNotThrow(() => assertJournalBinding(state, { ...binding, head: 'current-head' }))+ for (const key of ['repository', 'branch', 'profile', 'context']) {+ assert.throws(+ () => assertJournalBinding(state, { ...binding, [key]: `foreign-${key}` }),+ /JOURNAL_OWNER_MISMATCH/+ )+ assert.throws(+ () => assertJournalBinding(state, { ...binding, [key]: '' }),+ /JOURNAL_OWNER_MISMATCH/+ )+ }+})++test('journal refuses replacement, hard links, public permissions, invalid JSON and oversized state', t => {+ const { journal, file, state, directory } = fixture(t)+ fs.writeFileSync(file, JSON.stringify({ ...state, runId: randomUUID() }))+ assert.throws(() => journal.save(), /JOURNAL_REPLACED/)+ assert.throws(() => journal.remove(), /JOURNAL_REPLACED/)+ fs.writeFileSync(file, JSON.stringify(state))+ const link = path.join(directory, 'hardlink.json')+ fs.linkSync(file, link)+ assert.throws(() => readJournal(file), /UNSAFE_JOURNAL_FILE/)+ fs.unlinkSync(link)+ fs.chmodSync(file, 0o644)+ assert.throws(() => readJournal(file), /UNSAFE_JOURNAL_FILE/)+ fs.chmodSync(file, 0o600)+ fs.writeFileSync(file, '{')+ assert.throws(() => readJournal(file), SyntaxError)+ fs.writeFileSync(file, 'x'.repeat(1024 * 1024 + 1))+ assert.throws(() => readJournal(file), /INVALID_JOURNAL_SIZE/)+})++test('journal paths reject traversal, noncanonical roots, and symbolic files or ancestors', t => {+ const { directory, file, state } = fixture(t)+ for (const profile of ['../escape', '/absolute', 'nested/profile', '', '..']) {+ assert.throws(() => journalPath(directory, profile), /INVALID_PROFILE/)+ }+ const alias = path.join(directory, 'repository-alias')+ fs.symlinkSync(directory, alias)+ assert.throws(() => journalPath(alias, 'unit-profile'), /NONCANONICAL_REPOSITORY/)+ const target = path.join(directory, 'private-journal.json')+ fs.renameSync(file, target)+ fs.symlinkSync(target, file)+ assert.throws(() => readJournal(file), /UNSAFE_JOURNAL_FILE/)+ fs.unlinkSync(file)+ const parent = path.dirname(file)+ const parentTarget = path.join(directory, 'journal-target')+ fs.renameSync(parent, parentTarget)+ fs.symlinkSync(parentTarget, parent)+ assert.throws(() => new Journal(file, state), /UNSAFE_JOURNAL_PARENT/)+})++test('certification guard rejects an intermediate symlink even before any journal exists', t => {+ const directory = temporary(t)+ const outside = path.join(directory, 'outside')+ fs.mkdirSync(outside)+ fs.symlinkSync(outside, path.join(directory, '.local-notes'))+ const result = spawnSync(process.execPath, [lifecycle, directory, 'unit-profile'], {+ encoding: 'utf8',+ timeout: 3000,+ })+ assert.notEqual(result.status, 0, 'unsafe journal ancestry must not certify an empty lane')+ assert.match(result.stderr, /WRC_EGRESS_RECOVERY_REQUIRED/)+})++test('certification guard accepts genuine absence and rejects an outstanding valid journal', t => {+ const directory = temporary(t)+ const absent = spawnSync(process.execPath, [lifecycle, directory, 'unit-profile'], {+ encoding: 'utf8',+ timeout: 3000,+ })+ assert.equal(absent.status, 0)+ const { directory: owned } = fixture(t)+ const pending = spawnSync(process.execPath, [lifecycle, owned, 'unit-profile'], {+ encoding: 'utf8',+ timeout: 3000,+ })+ assert.notEqual(pending.status, 0)+ assert.match(pending.stderr, /WRC_EGRESS_RECOVERY_REQUIRED/)+})++test('creation persists its uncertain attempt before I/O and accepts only its acknowledged identity', async t => {+ const { journal, file, object, created } = fixture(t)+ let observedAttempt = false+ const result = await createOwned(+ {+ create: async body => {+ assert.deepEqual(body, object)+ observedAttempt = readJournal(file).resources[0].uid === null+ return created+ },+ },+ journal,+ object+ )+ assert.equal(observedAttempt, true)+ assert.deepEqual(result, created)+ assert.equal(readJournal(file).resources[0].uid, created.metadata.uid)+})++test('creation rejects missing run ownership before recording or issuing an attempt', async t => {+ const { journal, file, object } = fixture(t)+ delete object.metadata.labels[RUN_LABEL]+ let creates = 0+ await assert.rejects(+ createOwned(+ {+ create: async () => {+ creates+++ },+ },+ journal,+ object+ ),+ /MISSING_RUN_OWNERSHIP/+ )+ assert.equal(creates, 0)+ assert.deepEqual(readJournal(file).resources, [])+})++for (const code of [403, 409, 503]) {+ test(`rejected or ambiguous create ${code} never authorizes cleanup of an unacknowledged object`, async t => {+ const { journal, object, created } = fixture(t)+ const failure = apiError(code)+ await assert.rejects(+ createOwned(+ {+ create: async () => {+ throw failure+ },+ },+ journal,+ object+ ),+ error => error === failure+ )+ assert.equal(journal.state.resources[0].uid, null)+ let deletes = 0+ await assert.rejects(+ deleteOwned(+ {+ get: async () => created,+ delete: async () => {+ deletes+++ },+ },+ journal,+ journal.state.resources[0]+ ),+ /UNPROVEN_CLEANUP_OWNERSHIP/+ )+ assert.equal(deletes, 0)+ })+}++for (const [label, mutate] of [+ [+ 'missing UID',+ object => {+ delete object.metadata.uid+ },+ ],+ [+ 'foreign run',+ object => {+ object.metadata.labels[RUN_LABEL] = randomUUID()+ },+ ],+ [+ 'different name',+ object => {+ object.metadata.name = 'another-resource'+ },+ ],+ [+ 'different namespace',+ object => {+ object.metadata.namespace = 'another-namespace'+ },+ ],+]) {+ test(`creation refuses an acknowledgement with ${label}`, async t => {+ const { journal, object, created } = fixture(t)+ mutate(created)+ await assert.rejects(+ createOwned({ create: async () => created }, journal, object),+ /UNPROVEN_CREATED_RESOURCE|CREATED_IDENTITY_MISMATCH/+ )+ assert.equal(journal.state.resources[0].uid, null)+ })+}++test('cleanup uses fresh UID/resourceVersion preconditions and verifies authoritative absence', async t => {+ const { journal, created } = fixture(t)+ const entry = {+ kind: created.kind,+ name: created.metadata.name,+ namespace: created.metadata.namespace,+ uid: created.metadata.uid,+ }+ let live = { ...created, metadata: { ...created.metadata, resourceVersion: 'fresh-rv' } }+ const calls = []+ await deleteOwned(+ {+ get: async () => {+ calls.push('get')+ return live+ },+ delete: async (identity, fence) => {+ calls.push('delete')+ assert.deepEqual(identity, entry)+ assert.deepEqual(fence, { uid: entry.uid, resourceVersion: 'fresh-rv' })+ live = null+ },+ },+ journal,+ entry+ )+ assert.deepEqual(calls, ['get', 'delete', 'get'])+})++test('cleanup distinguishes failed observation, authoritative absence, and a foreign UID', async t => {+ const { journal, created } = fixture(t)+ const entry = {+ kind: created.kind,+ name: created.metadata.name,+ namespace: created.metadata.namespace,+ uid: created.metadata.uid,+ }+ let deletes = 0+ const del = async () => {+ deletes+++ }+ await assert.rejects(+ deleteOwned(+ {+ get: async () => {+ throw apiError(503)+ },+ delete: del,+ },+ journal,+ entry+ ),+ /API 503/+ )+ await deleteOwned({ get: async () => null, delete: del }, journal, entry)+ await assert.rejects(+ deleteOwned(+ {+ get: async () => ({ ...created, metadata: { ...created.metadata, uid: 'recreated-uid' } }),+ delete: del,+ },+ journal,+ entry+ ),+ /UNPROVEN_CLEANUP_OWNERSHIP/+ )+ assert.equal(deletes, 0)+})++test('cleanup forwards its bounded predicate signal to the authoritative absence GET', async t => {+ const { journal, created } = fixture(t)+ const entry = {+ kind: created.kind,+ name: created.metadata.name,+ namespace: created.metadata.namespace,+ uid: created.metadata.uid,+ }+ let gets = 0+ await deleteOwned(+ {+ get: async (_identity, remaining, signal) => {+ if (++gets === 1) return created+ assert(remaining > 0 && remaining <= 90000)+ assert(signal instanceof AbortSignal, 'the predicate signal must reach its real I/O')+ return null+ },+ delete: async () => {},+ },+ journal,+ entry+ )+})++test('cleanup does not retry a failed delete with an unfenced or newly recreated object', async t => {+ const { journal, created } = fixture(t)+ const entry = {+ kind: created.kind,+ name: created.metadata.name,+ namespace: created.metadata.namespace,+ uid: created.metadata.uid,+ }+ let gets = 0+ let deletes = 0+ const recreated = { ...created, metadata: { ...created.metadata, uid: 'recreated-uid' } }+ await assert.rejects(+ deleteOwned(+ {+ get: async () => (++gets === 1 ? created : recreated),+ delete: async () => {+ deletes+++ throw apiError(409)+ },+ },+ journal,+ entry+ ),+ /API 409|RESOURCE_RECREATED|UNPROVEN_CLEANUP/+ )+ assert.equal(deletes, 1)+ gets = 0+ await assert.rejects(+ deleteOwned(+ {+ get: async () => (++gets === 1 ? created : recreated),+ delete: async () => {},+ },+ journal,+ entry+ ),+ /RESOURCE_RECREATED_DURING_CLEANUP/+ )+})++test('delete failure is cleared only when a subsequent successful GET proves absence', async t => {+ const { journal, created } = fixture(t)+ const entry = {+ kind: created.kind,+ name: created.metadata.name,+ namespace: created.metadata.namespace,+ uid: created.metadata.uid,+ }+ let gets = 0+ await deleteOwned(+ {+ get: async () => (++gets === 1 ? created : null),+ delete: async () => {+ throw apiError(503)+ },+ },+ journal,+ entry+ )+ gets = 0+ await assert.rejects(+ deleteOwned(+ {+ get: async () => {+ if (++gets === 1) return created+ throw apiError(503)+ },+ delete: async () => {+ throw apiError(503)+ },+ },+ journal,+ entry+ ),+ /API 503/+ )+})++test('waitFor rejects a true predicate that finishes after its absolute deadline', async () => {+ let now = 100+ await assert.rejects(+ waitFor(+ 'late',+ 20,+ async remaining => {+ assert.equal(remaining, 20)+ now = 121+ return true+ },+ { now: () => now }+ ),+ error => /TIMEOUT:late/.test(error.message) && error.exitCode === 124+ )+})++test('waitFor does not accept true after cancellation during the predicate', async () => {+ const cancellation = new AbortController()+ const reason = new Error('cancelled during predicate')+ await assert.rejects(+ waitFor(+ 'cancelled',+ 1000,+ async () => {+ cancellation.abort(reason)+ return true+ },+ { signal: cancellation.signal }+ ),+ error => error === reason+ )+})++test('waitFor propagates parent cancellation to in-flight predicate I/O', async () => {+ const cancellation = new AbortController()+ const reason = new Error('cancelled in-flight')+ let receivedSignal+ await assert.rejects(+ waitFor(+ 'in-flight',+ 100,+ async (_remaining, signal) => {+ receivedSignal = signal+ cancellation.abort(reason)+ return new Promise(() => {})+ },+ { signal: cancellation.signal }+ ),+ error => error === reason+ )+ assert.equal(receivedSignal.aborted, true)+})++test('waitFor cancels a hung predicate at its own deadline and never treats an exception as ready', async () => {+ let signal+ await assert.rejects(+ waitFor('hung', 20, async (_remaining, current) => {+ signal = current+ return new Promise(() => {})+ }),+ /TIMEOUT:hung/+ )+ assert.equal(signal.aborted, true)+ const failure = new Error('predicate failed')+ await assert.rejects(+ waitFor('failed', 100, async () => {+ throw failure+ }),+ error => error === failure+ )+})++test('Kubernetes client distinguishes failed GET from authoritative empty NotFound and always supplies context', async () => {+ const calls = []+ const identity = { kind: 'ConfigMap', name: 'unit-resource', namespace: 'unit-namespace' }+ let response = ''+ const client = createKubeClient(repository, 'unit-context', async (args, options) => {+ calls.push({ args, options })+ if (response instanceof Error) throw response+ return response+ })+ assert.equal(await client.get(identity, 1234), null)+ assert(calls[0].args.includes('--context=unit-context'))+ assert(calls[0].args.includes('--ignore-not-found=true'))+ assert.equal(calls[0].options.timeoutMs, 1234)+ response = apiError(503)+ await assert.rejects(client.get(identity), /API 503/)+ response = '{'+ await assert.rejects(client.get(identity), SyntaxError)+})++test('Kubernetes reads cap their own request budget and refuse an expired phase before spawning', async () => {+ const calls = []+ const client = createKubeClient(repository, 'unit-profile', async (args, options) => {+ calls.push({ args, options })+ return args.includes('get') && args.includes('pods') ? '{"items":[]}' : ''+ })+ const identity = { kind: 'Service', namespace: 'unit', name: 'service' }+ await client.get(identity, 90_000)+ await client.list('pods', 'unit', 'app=unit', 900)+ assert.equal(calls[0].options.timeoutMs, 30_000)+ assert.equal(calls[1].options.timeoutMs, 900)+ await assert.rejects(client.get(identity, 0), error => error.exitCode === 124)+ assert.equal(calls.length, 2)+})++test('cleanup waits for aborted polling processes to finish before returning ownership to the restorer', async () => {+ let finish+ const client = createKubeClient(+ repository,+ 'unit-profile',+ () =>+ new Promise(resolve => {+ finish = resolve+ })+ )+ const observation = client.get({ kind: 'Service', namespace: 'unit', name: 'service' })+ let drained = false+ const cleanup = client.setCleanup(true).then(() => {+ drained = true+ })+ await Promise.resolve()+ assert.equal(drained, false)+ finish('')+ assert.equal(await observation, null)+ await cleanup+ assert.equal(drained, true)+})++test('Kubernetes resource operations reject invalid status-derived identities before dispatch', async () => {+ let calls = 0+ const client = createKubeClient(repository, 'unit-profile', async () => {+ calls+++ return '{}'+ })+ for (const identity of [+ { kind: 'Deployment', name: 'invalid name', namespace: 'unit' },+ { kind: 'Service', name: '-invalid', namespace: 'unit' },+ { kind: 'NetworkPolicy', name: 'valid', namespace: '' },+ { kind: 'Unknown', name: 'valid', namespace: 'unit' },+ ]) {+ await assert.rejects(client.get(identity), /INVALID_RESOURCE_IDENTITY/)+ await assert.rejects(client.patch(identity, []), /INVALID_RESOURCE_IDENTITY/)+ }+ assert.equal(calls, 0)+})++test('Kubernetes mutations require the live lease before create/patch/delete and preserve raw delete fencing', async () => {+ const calls = []+ const identity = { kind: 'ConfigMap', name: 'unit-resource', namespace: 'unit-namespace' }+ const client = createKubeClient(repository, 'unit-context', async (args, options) => {+ calls.push({ args, options })+ return '{}'+ })+ await client.create({ kind: 'ConfigMap', metadata: identity })+ await client.patch(identity, [{ op: 'test', path: '/metadata/uid', value: 'uid' }])+ await client.delete(identity, { uid: 'uid', resourceVersion: 'rv' })+ assert.equal(calls.length, 6)+ for (const index of [0, 2, 4]) {+ assert.equal(calls[index].args[0], 'bash')+ assert.match(calls[index].args[1], /require-t2-mutation-lock\.sh$/)+ assert(calls[index + 1].args.includes('--context=unit-context'))+ }+ assert(calls[5].args.includes(`--raw=${resourcePath(identity)}`))+ assert.deepEqual(JSON.parse(calls[5].options.input).preconditions, {+ uid: 'uid',+ resourceVersion: 'rv',+ })+ let mutations = 0+ const denied = createKubeClient(repository, 'unit-context', async args => {+ if (args[0] === 'bash') throw new Error('lease rejected')+ mutations+++ return '{}'+ })+ await assert.rejects(+ denied.delete(identity, { uid: 'uid', resourceVersion: 'rv' }),+ /lease rejected/+ )+ assert.equal(mutations, 0)+})++test(+ 'command runner returns successful output and keeps a separate cleanup budget after cancellation',+ { timeout: 5000 },+ async () => {+ const cancellation = new AbortController()+ const run = createCommandRunner(repository, cancellation)+ assert.equal(+ await run([process.execPath, '-e', 'process.stdin.pipe(process.stdout)'], {+ input: 'unit-input',+ timeoutMs: 2000,+ }),+ 'unit-input'+ )+ const reason = new Error('main run interrupted')+ cancellation.abort(reason)+ await assert.rejects(+ run([process.execPath, '-e', 'process.exit(0)']),+ error => error === reason+ )+ assert.equal(+ await run([process.execPath, '-e', 'process.stdout.write("restored")'], {+ cleanup: true,+ timeoutMs: 2000,+ }),+ 'restored'+ )+ }+)++for (const cleanup of [false, true]) {+ test(+ `command runner never starts an already-cancelled operation (cleanup=${cleanup})`,+ { timeout: 5000 },+ async t => {+ const directory = temporary(t)+ const marker = path.join(directory, 'must-not-run')+ const cancellation = new AbortController()+ const operation = new AbortController()+ operation.abort(new Error('operation already cancelled'))+ const run = createCommandRunner(repository, cancellation)+ await assert.rejects(+ run(+ [+ process.execPath,+ '-e',+ 'require("node:fs").writeFileSync(process.argv[1], "ran")',+ marker,+ ],+ {+ cleanup,+ signal: operation.signal,+ timeoutMs: 2000,+ }+ )+ )+ assert.equal(fs.existsSync(marker), false)+ }+ )+}++test(+ 'command runner enforces the requested millisecond budget before later side effects',+ { timeout: 5000 },+ async t => {+ const directory = temporary(t)+ const marker = path.join(directory, 'late-effect')+ const run = createCommandRunner(repository, new AbortController())+ await assert.rejects(+ run(+ [+ process.execPath,+ '-e',+ 'setTimeout(() => require("node:fs").writeFileSync(process.argv[1], "late"), 250)',+ marker,+ ],+ {+ timeoutMs: 40,+ }+ ),+ error => error.exitCode === 124+ )+ assert.equal(+ fs.existsSync(marker),+ false,+ 'work must terminate at the operation deadline, not merely be judged late after completion'+ )+ }+)++test(+ 'command runner bounds output and does not expose child output in its error',+ { timeout: 5000 },+ async () => {+ const run = createCommandRunner(repository, new AbortController())+ await assert.rejects(+ run([process.execPath, '-e', 'process.stdout.write("unit-payload".repeat(500000))'], {+ timeoutMs: 2000,+ }),+ error =>+ /WRC_GATE_OPERATION_FAILED/.test(error.message) && !error.message.includes('unit-payload')+ )+ }+)++function observeJsonFile(file, timeoutMs = 3000) {+ return new Promise((resolve, reject) => {+ // Watch an explicit readiness artifact. Stat polling avoids the platform's+ // finite native watcher quota without guessing when a child is ready.+ const changed = () => {+ if (!fs.existsSync(file)) return+ try {+ const value = JSON.parse(fs.readFileSync(file, 'utf8'))+ clearTimeout(timer)+ fs.unwatchFile(file, changed)+ resolve(value)+ } catch (error) {+ if (!(error instanceof SyntaxError)) {+ clearTimeout(timer)+ fs.unwatchFile(file, changed)+ reject(error)+ }+ }+ }+ fs.watchFile(file, { interval: 20 }, changed)+ const timer = setTimeout(() => {+ fs.unwatchFile(file, changed)+ reject(new Error('child readiness deadline'))+ }, timeoutMs)+ })+}++function processRunning(pid) {+ const result = spawnSync('ps', ['-p', String(pid), '-o', 'stat='], {+ encoding: 'utf8',+ timeout: 1000,+ })+ if (result.error) throw result.error+ assert.equal(result.stderr.trim(), '', 'failed process inspection cannot prove a child exited')+ return result.status === 0 && result.stdout.trim() !== '' && !result.stdout.trim().startsWith('Z')+}++test(+ 'command cancellation reaps a TERM-resistant descendant before resolving cleanup ownership',+ { timeout: 8000 },+ async t => {+ const directory = temporary(t)+ const marker = path.join(directory, 'children.json')+ const readiness = observeJsonFile(marker)+ const cancellation = new AbortController()+ const run = createCommandRunner(repository, cancellation)+ const grandchildCode =+ 'process.on("SIGTERM",()=>{});process.stdout.write("ready");setInterval(()=>{},1000)'+ const parentCode = `+ const child = require('node:child_process').spawn(process.execPath, ['-e', ${JSON.stringify(grandchildCode)}], {stdio:['ignore','pipe','ignore']});+ process.on('SIGTERM', () => process.exit(0));+ child.stdout.once('data', () => require('node:fs').writeFileSync(process.argv[1], JSON.stringify({parent:process.pid,descendant:child.pid})));+ setInterval(()=>{},1000);+ `+ const operation = run([process.execPath, '-e', parentCode, marker], { timeoutMs: 6000 })+ // Attach rejection immediately so an unexpected early child failure is never+ // an unhandled rejection while the independent readiness event is awaited.+ const outcome = operation.then(+ value => ({ value }),+ error => ({ error })+ )+ const pids = await readiness+ t.after(() => {+ for (const pid of [pids.parent, pids.descendant]) {+ if (processRunning(pid)) {+ try {+ process.kill(pid, 'SIGKILL')+ } catch (error) {+ if (error.code !== 'ESRCH') throw error+ }+ }+ }+ })+ assert(processRunning(pids.parent))+ assert(processRunning(pids.descendant))+ cancellation.abort(Object.assign(new Error('terminated by test'), { exitCode: 143 }))+ const result = await outcome+ assert(result.error, 'cancellation must never report success')+ assert.equal(result.error.exitCode, 143)+ assert.equal(processRunning(pids.parent), false)+ assert.equal(processRunning(pids.descendant), false)+ }+)++test(+ 'natural direct-child exit does not leave a daemonized descendant running',+ { timeout: 5000 },+ async t => {+ const directory = temporary(t)+ const marker = path.join(directory, 'daemon.json')+ const readiness = observeJsonFile(marker)+ const run = createCommandRunner(repository, new AbortController())+ const code = `+ const child = require('node:child_process').spawn(process.execPath, ['-e', 'process.stdout.write("ready");setInterval(()=>{},1000)'], {stdio:['ignore','pipe','ignore']});+ child.stdout.once('data', () => {+ require('node:fs').writeFileSync(process.argv[1], JSON.stringify({pid:child.pid}));+ process.exit(0);+ });+ `+ const operation = run([process.execPath, '-e', code, marker], { timeoutMs: 3000 })+ const outcome = operation.then(+ value => ({ value }),+ error => ({ error })+ )+ const { pid } = await readiness+ t.after(() => {+ if (processRunning(pid)) {+ try {+ process.kill(pid, 'SIGKILL')+ } catch (error) {+ if (error.code !== 'ESRCH') throw error+ }+ }+ })+ const result = await outcome+ assert.equal(result.error, undefined)+ assert.equal(processRunning(pid), false)+ }+)diff --git a/scripts/tests/wrc-egress-proof.test.cjs b/scripts/tests/wrc-egress-proof.test.cjsnew file mode 100644index 000000000..401d16fb3--- /dev/null+++ b/scripts/tests/wrc-egress-proof.test.cjs@@ -0,0 +1,445 @@+'use strict'++const { test } = require('node:test')+const assert = require('node:assert/strict')+const { spawnSync } = require('node:child_process')+const path = require('node:path')+const { assertPolicy, validatePolicy } = require('../e2e/_lib/wrc-egress-proof.cjs')++const STATE = 'clerum.io/egress-fqdn-state'+const TARGETS = 'clerum.io/egress-fqdn-targets'+const RESOLVED_AT = 'clerum.io/egress-fqdn-resolved-at'+function fixture(lane = 'ui') {+ const tuples = [{ fqdn: 'fixture.example.com', ip: '93.184.216.34', port: 443, protocol: 'TCP' }]+ const owner = {+ 'clerum.io/managed-by': 'workflow-recipes',+ 'clerum.io/recipe': 'recipe',+ 'clerum.io/recipe-name': 'recipe',+ 'clerum.io/recipe-namespace': 'sandbox-recipes',+ }+ const podLabels = {+ ...owner,+ 'clerum.io/workload': lane === 'ui' ? 'frontend' : 'worker',+ 'clerum.io/sandbox-ui': 'true',+ }+ const expected = {+ name: 'policy',+ namespace: lane === 'ui' ? 'sandbox-ui' : 'sandbox-recipes',+ recipe: 'recipe',+ recipeNamespace: 'sandbox-recipes',+ lane,+ workload: podLabels['clerum.io/workload'],+ podLabels,+ tuples,+ uid: 'policy-uid',+ }+ const policy = {+ apiVersion: 'networking.k8s.io/v1',+ kind: 'NetworkPolicy',+ metadata: {+ name: expected.name,+ namespace: expected.namespace,+ uid: expected.uid,+ resourceVersion: '1',+ labels: { ...owner, 'clerum.io/workload': expected.workload },+ annotations: {+ [STATE]: JSON.stringify(+ tuples.map(tuple => ({ ...tuple, expiresAt: 2000, lastObservedAt: 1000 }))+ ),+ [TARGETS]: 'fixture.example.com=93.184.216.34/32',+ [RESOLVED_AT]: new Date(1000).toISOString(),+ },+ },+ spec: {+ podSelector: {+ matchLabels:+ lane === 'ui'+ ? {+ 'clerum.io/sandbox-ui': 'true',+ 'clerum.io/recipe-name': 'recipe',+ 'clerum.io/recipe-namespace': 'sandbox-recipes',+ }+ : { 'clerum.io/recipe': 'recipe', 'clerum.io/workload': 'worker' },+ },+ policyTypes: ['Egress'],+ egress: [+ {+ to: [{ ipBlock: { cidr: '93.184.216.34/32' } }],+ ports: [{ port: 443, protocol: 'TCP' }],+ },+ ],+ },+ }+ return { expected, policy }+}++for (const lane of ['ui', 'workload']) {+ test(`accepts a complete ${lane} policy without treating expired fail-static state as invalid`, () => {+ const { policy, expected } = fixture(lane)+ assert.deepEqual(assertPolicy(policy, expected), { ok: true })+ expected.internal = []+ delete expected.uid+ assert.deepEqual(validatePolicy(policy, expected), { ok: true })+ })+}++const regressions = [+ [+ 'wrong selector',+ p => {+ p.spec.podSelector = { matchLabels: { app: 'nonexistent' } }+ },+ ],+ [+ 'UDP instead of TCP',+ p => {+ p.spec.egress[0].ports[0].protocol = 'UDP'+ },+ ],+ [+ 'extra allow-all rule',+ p => {+ p.spec.egress.push({})+ },+ ],+ [+ 'different provenance FQDN/IP/protocol',+ p => {+ p.metadata.annotations[STATE] = JSON.stringify([+ { fqdn: 'other.example.com', ip: '8.8.8.8', port: 443, protocol: 'UDP' },+ ])+ },+ ],+ [+ 'wrong recipe owner',+ p => {+ p.metadata.labels['clerum.io/recipe'] = 'foreign'+ },+ ],+ [+ 'extra peer',+ p => {+ p.spec.egress[0].to.push({ namespaceSelector: {} })+ },+ ],+ [+ 'no peers',+ p => {+ delete p.spec.egress[0].to+ },+ ],+ [+ 'empty peers',+ p => {+ p.spec.egress[0].to = []+ },+ ],+ [+ 'no ports',+ p => {+ delete p.spec.egress[0].ports+ },+ ],+ [+ 'empty ports',+ p => {+ p.spec.egress[0].ports = []+ },+ ],+ [+ 'port range',+ p => {+ p.spec.egress[0].ports[0].endPort = 444+ },+ ],+ [+ 'named port',+ p => {+ p.spec.egress[0].ports[0].port = 'https'+ },+ ],+ [+ 'out of range port',+ p => {+ p.spec.egress[0].ports[0].port = 65536+ },+ ],+ [+ 'null protocol',+ p => {+ p.spec.egress[0].ports[0].protocol = null+ },+ ],+ [+ 'CIDR wider than host',+ p => {+ p.spec.egress[0].to[0].ipBlock.cidr = '93.184.216.0/24'+ },+ ],+ [+ 'CIDR exception',+ p => {+ p.spec.egress[0].to[0].ipBlock.except = ['93.184.216.34/32']+ },+ ],+ [+ 'mixed peer selectors',+ p => {+ p.spec.egress[0].to[0].podSelector = {}+ },+ ],+ [+ 'unknown rule field',+ p => {+ p.spec.egress[0].unexpected = true+ },+ ],+ [+ 'unknown spec field',+ p => {+ p.spec.futureAllow = true+ },+ ],+ [+ 'ingress policy type',+ p => {+ p.spec.policyTypes.push('Ingress')+ },+ ],+ [+ 'omitted policy types',+ p => {+ delete p.spec.policyTypes+ },+ ],+ [+ 'selector expression',+ p => {+ p.spec.podSelector.matchExpressions = []+ },+ ],+ [+ 'wrong namespace',+ p => {+ p.metadata.namespace = 'other'+ },+ ],+ [+ 'wrong UID',+ p => {+ p.metadata.uid = 'replacement-uid'+ },+ ],+ [+ 'missing resource version',+ p => {+ delete p.metadata.resourceVersion+ },+ ],+ [+ 'deleting policy',+ p => {+ p.metadata.deletionTimestamp = new Date().toISOString()+ },+ ],+ [+ 'annotation-only IP',+ p => {+ const state = JSON.parse(p.metadata.annotations[STATE])+ state[0].ip = '8.8.8.8'+ p.metadata.annotations[STATE] = JSON.stringify(state)+ },+ ],+ [+ 'annotation-only FQDN',+ p => {+ const state = JSON.parse(p.metadata.annotations[STATE])+ state[0].fqdn = 'other.example.com'+ p.metadata.annotations[STATE] = JSON.stringify(state)+ },+ ],+ [+ 'duplicate provenance',+ p => {+ const state = JSON.parse(p.metadata.annotations[STATE])+ p.metadata.annotations[STATE] = JSON.stringify([...state, ...state])+ },+ ],+ [+ 'missing timestamp',+ p => {+ const state = JSON.parse(p.metadata.annotations[STATE])+ delete state[0].lastObservedAt+ p.metadata.annotations[STATE] = JSON.stringify(state)+ },+ ],+ [+ 'corrupt state JSON',+ p => {+ p.metadata.annotations[STATE] = '{'+ },+ ],+ [+ 'missing structured state',+ p => {+ delete p.metadata.annotations[STATE]+ },+ ],+ [+ 'wrong target summary',+ p => {+ p.metadata.annotations[TARGETS] = 'other.example.com=8.8.8.8/32'+ },+ ],+ [+ 'invalid resolution date',+ p => {+ p.metadata.annotations[RESOLVED_AT] = 'not-a-date'+ },+ ],+]+for (const [description, mutate] of regressions) {+ test(`rejects ${description}`, () => {+ const { policy, expected } = fixture()+ mutate(policy)+ assert.equal(validatePolicy(policy, expected).ok, false)+ })+}++test('accepts Kubernetes TCP default, reordered/grouped permissions, and harmless duplicate network rules', () => {+ const { policy, expected } = fixture()+ const extra = { ...expected.tuples[0], port: 80 }+ expected.tuples.push(extra)+ policy.spec.egress[0].ports.unshift({ port: 80 })+ delete policy.spec.egress[0].ports[1].protocol+ policy.spec.egress.push(structuredClone(policy.spec.egress[0]))+ policy.metadata.annotations[STATE] = JSON.stringify(+ expected.tuples.map(entry => ({ ...entry, expiresAt: 2000, lastObservedAt: 1000 }))+ )+ policy.metadata.annotations[TARGETS] += ',' + policy.metadata.annotations[TARGETS]+ assert.equal(validatePolicy(policy, expected).ok, true)+})++test('rejects unintended peer/port Cartesian product', () => {+ const { policy, expected } = fixture()+ expected.tuples.push({ fqdn: 'second.example.com', ip: '8.8.8.8', port: 80, protocol: 'TCP' })+ policy.spec.egress[0].to.push({ ipBlock: { cidr: '8.8.8.8/32' } })+ policy.spec.egress[0].ports.push({ port: 80 })+ assert.equal(validatePolicy(policy, expected).code, 'EXTERNAL_TUPLES_MISMATCH')+})++test('distinct FQDN provenance may legitimately share an enforced IP/port tuple', () => {+ const { policy, expected } = fixture()+ expected.tuples.push({ ...expected.tuples[0], fqdn: 'alias.example.com' })+ policy.metadata.annotations[STATE] = JSON.stringify(+ expected.tuples.map(entry => ({+ ...entry,+ expiresAt: 2000,+ lastObservedAt: 1000,+ }))+ )+ policy.metadata.annotations[TARGETS] += ',alias.example.com=93.184.216.34/32'+ assert.equal(validatePolicy(policy, expected).ok, true)+})++test('accepts only explicitly declared conjunctive UI-to-worker Service permissions', () => {+ const { policy, expected } = fixture()+ expected.internal = [+ { namespace: 'sandbox-recipes', workload: 'worker', port: 3000, protocol: 'TCP' },+ ]+ const rule = {+ to: [+ {+ namespaceSelector: { matchLabels: { 'kubernetes.io/metadata.name': 'sandbox-recipes' } },+ podSelector: {+ matchLabels: { 'clerum.io/recipe': 'recipe', 'clerum.io/workload': 'worker' },+ },+ },+ ],+ ports: [{ port: 3000, protocol: 'TCP' }],+ }+ policy.spec.egress.push(rule)+ assert.equal(validatePolicy(policy, expected).ok, true)+ for (const mutate of [+ p => {+ p.spec.egress[1].to[0].namespaceSelector.matchLabels['kubernetes.io/metadata.name'] =+ 'foreign'+ },+ p => {+ p.spec.egress[1].to[0].podSelector.matchLabels['clerum.io/recipe'] = 'foreign'+ },+ p => {+ p.spec.egress[1].to[0].podSelector.matchLabels['clerum.io/workload'] = 'other-worker'+ },+ p => {+ const peer = p.spec.egress[1].to[0]+ p.spec.egress[1].to = [+ { namespaceSelector: peer.namespaceSelector },+ { podSelector: peer.podSelector },+ ]+ },+ p => {+ p.spec.egress[1].to[0].namespaceSelector = {}+ },+ p => {+ p.spec.egress[1].to[0].podSelector.matchExpressions = []+ },+ p => {+ p.spec.egress[1].ports.push({ port: 3001 })+ },+ ]) {+ const candidate = structuredClone(policy)+ mutate(candidate)+ assert.equal(validatePolicy(candidate, expected).ok, false)+ }+ delete expected.internal+ assert.equal(validatePolicy(policy, expected).ok, false)+})++test('accepts empty or omitted top-level egress under explicit Egress isolation', () => {+ const { policy, expected } = fixture()+ expected.tuples = []+ policy.spec.egress = []+ policy.metadata.annotations = { [STATE]: '[]', [TARGETS]: '' }+ assert.equal(validatePolicy(policy, expected).ok, true)+ delete policy.spec.egress+ assert.equal(validatePolicy(policy, expected).ok, true)+ policy.spec.egress = [{}]+ assert.equal(validatePolicy(policy, expected).ok, false)+})++test('rejects unknown, defaulted, duplicate, and incorrectly bound expectations', () => {+ const { policy, expected } = fixture()+ for (const candidate of [+ undefined,+ {},+ { ...expected, lane: 'unknown' },+ { ...expected, extra: true },+ { ...expected, tuples: [...expected.tuples, ...expected.tuples] },+ { ...expected, tuples: [{ ...expected.tuples[0], protocol: undefined }] },+ ]) {+ assert.equal(validatePolicy(policy, candidate).input, true)+ }+ const wrongPod = {+ ...expected,+ podLabels: { ...expected.podLabels, 'clerum.io/workload': 'other' },+ }+ assert.equal(validatePolicy(policy, wrongPod).code, 'POD_IDENTITY_MISMATCH')+})++test('CLI distinguishes bad input from a failed proof and never echoes policy contents', () => {+ const { policy, expected } = fixture()+ const cli = path.join(__dirname, '../e2e/_lib/wrc-egress-proof.cjs')+ const run = (argv, input) =>+ spawnSync(process.execPath, [cli, ...argv], { input, encoding: 'utf8', timeout: 5000 })+ const args = ['--expect-json', JSON.stringify(expected)]+ assert.equal(run(args, JSON.stringify(policy)).status, 0)+ assert.equal(run([], JSON.stringify(policy)).status, 2)+ assert.equal(run(['--expect-json', '{'], '').status, 2)+ assert.equal(run(args, '{').status, 2)+ policy.spec.egress.push({})+ const failure = run(args, JSON.stringify(policy))+ assert.equal(failure.status, 1)+ assert.equal(failure.stdout, '')+ assert.deepEqual(JSON.parse(failure.stderr), { ok: false, code: 'UNRESTRICTED_OR_UNKNOWN_RULE' })+})diff --git a/tests/e2e/fixtures/package-lock.json b/tests/e2e/fixtures/package-lock.jsonindex f89b7ed67..40bfad0b9 100644--- a/tests/e2e/fixtures/package-lock.json+++ b/tests/e2e/fixtures/package-lock.json@@ -1,12 +1,12 @@{"name": "@clerum/e2e-test-fixtures",- "version": "1.0.70",+ "version": "1.0.71","lockfileVersion": 3,"requires": true,"packages": {"": {"name": "@clerum/e2e-test-fixtures",- "version": "1.0.70",+ "version": "1.0.71","dependencies": {"@kubernetes/client-node": "^1.0.0"},diff --git a/tests/e2e/fixtures/package.json b/tests/e2e/fixtures/package.jsonindex b6f32c58f..e471108f1 100644--- a/tests/e2e/fixtures/package.json+++ b/tests/e2e/fixtures/package.json@@ -1,6 +1,6 @@{"name": "@clerum/e2e-test-fixtures",- "version": "1.0.70",+ "version": "1.0.71","description": "E2E test fixtures and utilities for Clerum testing","type": "module","main": "index.ts",diff --git a/tests/e2e/fixtures/wrc-egress-dns-proxy/server.cjs b/tests/e2e/fixtures/wrc-egress-dns-proxy/server.cjsindex d09d60637..0ae585cee 100644--- a/tests/e2e/fixtures/wrc-egress-dns-proxy/server.cjs+++ b/tests/e2e/fixtures/wrc-egress-dns-proxy/server.cjs@@ -4,27 +4,45 @@ const dgram = require('node:dgram')const http = require('node:http')const net = require('node:net')-const target = String(process.env.TARGET_DNS || '').toLowerCase()-const upstream = process.env.UPSTREAM_DNS-const answerIp = String(process.env.ANSWER_IP || '93.184.216.34')-const answerOctets = answerIp.split('.').map(Number)+const MAX_HELD = 512+const MAX_OBSERVATIONS = 16+const MAX_QUERY_IDENTITIES = 1024+const MAX_RELAYS = 128+const MODES = new Set(['ok', 'hold', 'servfail'])-if (!target || !upstream || answerOctets.length !== 4 || answerOctets.some(Number.isNaN)) {- throw new Error('TARGET_DNS, UPSTREAM_DNS, and a valid IPv4 ANSWER_IP are required')+function validHostname(value) {+ return (+ typeof value === 'string' &&+ value.length <= 253 &&+ value.includes('.') &&+ value.split('.').every(label => /^[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?$/.test(label))+ )}-let mode = 'ok'-const held = []-const counts = { ok: 0, hold: 0, servfail: 0, forwarded: 0 }-const udp = dgram.createSocket('udp4')+function readConfig(env) {+ let targets+ try {+ targets = JSON.parse(env.DNS_TARGETS_JSON)+ } catch {+ throw new Error('DNS_TARGETS_JSON must contain the explicit UI/worker/canary target array')+ }+ return {+ targets,+ upstream: env.UPSTREAM_DNS,+ ttlSeconds: env.DNS_TTL_SECONDS === undefined ? 300 : Number(env.DNS_TTL_SECONDS),+ }+}function question(message) {+ if (message.length < 12 || message.readUInt16BE(4) !== 1 || message.readUInt16BE(2) & 0x8000)+ return nulllet offset = 12const labels = []while (offset < message.length) {const length = message[offset++]if (length === 0) break- if (offset + length > message.length) return null+ // Compressed questions are forwarded unchanged, not partially interpreted.+ if (length > 63 || offset + length > message.length) return nulllabels.push(message.subarray(offset, offset + length).toString('ascii'))offset += length}@@ -32,13 +50,16 @@ function question(message) {return {name: labels.join('.').toLowerCase(),type: message.readUInt16BE(offset),+ queryClass: message.readUInt16BE(offset + 2),end: offset + 4,+ id: message.readUInt16BE(0),}}-function dnsResponse(message, parsed, rcode) {+function dnsResponse(message, parsed, rcode, answerIp, ttlSeconds) {const header = Buffer.from(message.subarray(0, 12))header.writeUInt16BE(0x8180 | rcode, 2)+ header.writeUInt16BE(1, 4)header.writeUInt16BE(rcode === 0 ? 1 : 0, 6)header.writeUInt16BE(0, 8)header.writeUInt16BE(0, 10)@@ -47,18 +68,19 @@ function dnsResponse(message, parsed, rcode) {const record = Buffer.from([0xc0,0x0c,- 0x00,- 0x01,- 0x00,- 0x01,- 0x00,- 0x00,- 0x01,- 0x2c,- 0x00,- 0x04,- ...answerOctets,+ 0,+ 1,+ 0,+ 1,+ 0,+ 0,+ 0,+ 0,+ 0,+ 4,+ ...answerIp.split('.').map(Number),])+ record.writeUInt32BE(ttlSeconds, 6)return Buffer.concat([header, questionBytes, record])}@@ -69,126 +91,414 @@ function tcpFrame(message) {return frame}-function send(entry, selectedMode) {- const response = dnsResponse(entry.message, entry.parsed, selectedMode === 'ok' ? 0 : 2)- if (selectedMode === 'ok') counts.ok++- else counts.servfail++- if (entry.transport === 'udp') {- udp.send(response, entry.rinfo.port, entry.rinfo.address)- } else if (!entry.socket.destroyed) {- entry.socket.write(tcpFrame(response))- }- if (selectedMode === 'ok') {- console.log(entry.transport === 'udp' ? 'ok DNS A fixture via UDP' : 'ok DNS A fixture via TCP')- } else {- console.log(- entry.transport === 'udp'- ? 'servfail DNS A fixture via UDP'- : 'servfail DNS A fixture via TCP'+const emptyCounts = () => ({ ok: 0, hold: 0, servfail: 0, forwarded: 0 })+// Explicit writes keep DNS-derived names/types away from object-property writes.+function increment(counts, mode) {+ if (mode === 'ok') counts.ok+++ else if (mode === 'hold') counts.hold+++ else if (mode === 'servfail') counts.servfail+++ else if (mode === 'forwarded') counts.forwarded+++ else throw new Error('Invalid observation mode')+}++/** A real UDP/TCP DNS fixture; ephemeral loopback ports are used only by tests. */+function createDnsProxy({+ targets,+ upstream,+ ttlSeconds = 300,+ upstreamPort = 53,+ listenHost = '0.0.0.0',+ dnsPort = 8053,+ controlPort = 8090,+}) {+ if (+ !Array.isArray(targets) ||+ targets.length === 0 ||+ targets.length > 3 ||+ !net.isIPv4(upstream) ||+ !Number.isInteger(ttlSeconds) ||+ ttlSeconds < 1 ||+ ttlSeconds > 300 ||+ ![upstreamPort, dnsPort, controlPort].every(+ port => Number.isInteger(port) && port >= 0 && port <= 65535+ ) ||+ upstreamPort === 0+ ) {+ throw new Error(+ 'Explicit targets, IPv4 upstream, valid ports, and DNS TTL from 1 to 300 are required')}-}+ const lanes = new Map()+ const byName = new Map()+ for (const target of targets) {+ if (+ !target ||+ !['ui', 'worker', 'canary'].includes(target.lane) ||+ !validHostname(target.fqdn) ||+ typeof target.ip !== 'string' ||+ !net.isIPv4(target.ip) ||+ lanes.has(target.lane) ||+ byName.has(target.fqdn) ||+ Object.keys(target).some(key => !['lane', 'fqdn', 'ip'].includes(key))+ ) {+ throw new Error(+ 'DNS targets require distinct UI/worker/canary lanes, distinct lowercase FQDNs, and IPv4 answers'+ )+ }+ const state = {+ lane: target.lane,+ fqdn: target.fqdn,+ ip: target.ip,+ mode: 'ok',+ revision: 0,+ counts: emptyCounts(),+ unique: emptyCounts(),+ received: { ok: 0, hold: 0, servfail: 0 },+ receivedIdentities: new Map(),+ held: [],+ observations: new Map(),+ }+ lanes.set(target.lane, state)+ byName.set(target.fqdn, state)+ }-function forwardUdp(message, rinfo) {- counts.forwarded++- const relay = dgram.createSocket('udp4')- const timer = setTimeout(() => relay.close(), 5000)- relay.once('message', response => {- clearTimeout(timer)- udp.send(response, rinfo.port, rinfo.address)- relay.close()- })- relay.once('error', () => {- clearTimeout(timer)- relay.close()- })- relay.send(message, 53, upstream)-}+ const counts = emptyCounts()+ const udp = dgram.createSocket('udp4')+ const connections = new Set()+ const relays = new Set()+ let fault = null+ let started = false+ let closing = false-function forwardTcp(message, socket) {- counts.forwarded++- const relay = net.createConnection({ host: upstream, port: 53 }, () =>- relay.write(tcpFrame(message))- )- let pending = Buffer.alloc(0)- relay.setTimeout(5000, () => relay.destroy())- relay.on('data', chunk => {- pending = Buffer.concat([pending, chunk])- if (pending.length < 2) return- const length = pending.readUInt16BE(0)- if (pending.length < length + 2) return- if (!socket.destroyed) socket.write(pending.subarray(0, length + 2))- relay.destroy()- })- relay.once('error', () => relay.destroy())-}+ const queryIdentity = (state, entry) =>+ `${state.revision}:${entry.client}:${entry.parsed.id}:${entry.parsed.name}:${entry.parsed.queryClass}`-function handle(message, entry) {- const parsed = question(message)- if (parsed && parsed.name === target && parsed.type === 1) {- entry.message = message- entry.parsed = parsed- if (mode === 'hold') {- counts.hold++- held.push(entry)- console.log(- entry.transport === 'udp' ? 'hold DNS A fixture via UDP' : 'hold DNS A fixture via TCP'- )- } else {- send(entry, mode)+ // Receipt evidence concerns new IN/A requests in the current mode. Sending+ // an old held response (possibly after its client timed out) never enters+ // this path. The keys have only three modes and two server-owned transports;+ // each bucket has the same identity bound as the existing observation sets.+ function receive(state, entry) {+ const key = `${state.mode}:${entry.transport}`+ let identities = state.receivedIdentities.get(key)+ if (!identities) {+ identities = new Set()+ state.receivedIdentities.set(key, identities)}- return+ const identity = queryIdentity(state, entry)+ if (identities.has(identity)) return+ if (identities.size >= MAX_QUERY_IDENTITIES) {+ fault = 'RECEIVED_QUERY_IDENTITY_LIMIT'+ return+ }+ identities.add(identity)+ increment(state.received, state.mode)}- if (entry.transport === 'udp') forwardUdp(message, entry.rinfo)- else forwardTcp(message, entry.socket)-}-function setMode(nextMode) {- mode = nextMode- if (mode !== 'hold') {- while (held.length > 0) send(held.shift(), mode)+ function observe(state, entry, mode) {+ increment(counts, mode)+ if (!state) return+ increment(state.counts, mode)+ const key = `${entry.parsed.type}:${entry.transport}:${mode}`+ let observation = state.observations.get(key)+ if (!observation) {+ if (state.observations.size >= MAX_OBSERVATIONS) {+ fault = 'OBSERVATION_LIMIT'+ return+ }+ observation = {+ type: entry.parsed.type,+ transport: entry.transport,+ mode,+ count: 0,+ seen: new Set(),+ }+ state.observations.set(key, observation)+ }+ observation.count+++ const identity = queryIdentity(state, entry)+ if (!observation.seen.has(identity)) {+ if (observation.seen.size >= MAX_QUERY_IDENTITIES) {+ fault = 'QUERY_IDENTITY_LIMIT'+ return+ }+ observation.seen.add(identity)+ increment(state.unique, mode)+ }}-}-let listeners = 0-function listening() {- listeners++- if (listeners === 2) console.log('wrc egress DNS proxy ready')-}+ function send(entry, selectedMode) {+ if (entry.transport === 'tcp' && entry.socket.destroyed) return+ const state = entry.state+ const response = dnsResponse(+ entry.message,+ entry.parsed,+ selectedMode === 'ok' ? 0 : 2,+ state.ip,+ ttlSeconds+ )+ observe(state, entry, selectedMode)+ if (entry.transport === 'udp') {+ udp.send(response, entry.rinfo.port, entry.rinfo.address, error => {+ if (error) fault = 'DNS_SEND_FAILED'+ })+ } else if (!entry.socket.destroyed) {+ entry.socket.write(tcpFrame(response))+ }+ }-udp.on('message', (message, rinfo) => handle(message, { transport: 'udp', rinfo }))-udp.bind(8053, '0.0.0.0', listening)+ function forwardUdp(message, rinfo) {+ if (relays.size >= MAX_RELAYS) {+ fault = 'RELAY_LIMIT'+ return+ }+ const relay = dgram.createSocket('udp4')+ let done = false+ const close = () => {+ if (done) return+ done = true+ clearTimeout(timer)+ relays.delete(close)+ relay.close()+ }+ const timer = setTimeout(close, 5000)+ relays.add(close)+ relay.once('message', response => {+ udp.send(response, rinfo.port, rinfo.address, error => {+ if (error) fault = 'DNS_FORWARD_SEND_FAILED'+ })+ close()+ })+ relay.once('error', close)+ relay.send(message, upstreamPort, upstream, error => {+ if (error) close()+ })+ }-const tcp = net.createServer(socket => {- let pending = Buffer.alloc(0)- socket.on('data', chunk => {- pending = Buffer.concat([pending, chunk])- while (pending.length >= 2) {+ function forwardTcp(message, socket) {+ if (relays.size >= MAX_RELAYS) {+ fault = 'RELAY_LIMIT'+ return+ }+ const relay = net.createConnection({ host: upstream, port: upstreamPort }, () =>+ relay.write(tcpFrame(message))+ )+ const close = () => {+ relays.delete(close)+ relay.destroy()+ }+ relays.add(close)+ let pending = Buffer.alloc(0)+ relay.setTimeout(5000, close)+ relay.on('data', chunk => {+ pending = Buffer.concat([pending, chunk])+ if (pending.length > 65537) {+ close()+ return+ }+ if (pending.length < 2) returnconst length = pending.readUInt16BE(0)if (pending.length < length + 2) return- const message = pending.subarray(2, length + 2)- pending = pending.subarray(length + 2)- handle(message, { transport: 'tcp', socket })+ if (!socket.destroyed) socket.write(pending.subarray(0, length + 2))+ close()+ })+ relay.once('error', close)+ relay.once('close', () => relays.delete(close))+ }++ function handle(message, entry) {+ if (closing) return+ const parsed = question(message)+ const state = parsed && byName.get(parsed.name)+ entry.parsed = parsed+ if (state && parsed.type === 1 && parsed.queryClass === 1) {+ if (message.length > 4096) {+ fault = 'TARGET_QUERY_SIZE_LIMIT'+ return+ }+ entry.message = message+ entry.state = state+ receive(state, entry)+ if (state.mode === 'hold') {+ observe(state, entry, 'hold')+ if (state.held.length >= MAX_HELD) {+ fault = 'HELD_QUERY_LIMIT'+ return+ }+ state.held.push(entry)+ } else send(entry, state.mode)+ return+ }+ observe(state, entry, 'forwarded')+ if (entry.transport === 'udp') forwardUdp(message, entry.rinfo)+ else forwardTcp(message, entry.socket)+ }++ function setMode(nextMode, lane) {+ if (!MODES.has(nextMode) || (lane !== undefined && !lanes.has(lane)))+ throw new Error('Invalid DNS mode or lane')+ for (const state of lane === undefined ? lanes.values() : [lanes.get(lane)]) {+ if (state.mode !== nextMode) state.revision+++ state.mode = nextMode+ if (nextMode !== 'hold') {+ const held = state.held.splice(0)+ for (const entry of held) send(entry, nextMode)+ }+ }+ }++ function snapshot() {+ const states = [...lanes.values()]+ return {+ mode: states.every(state => state.mode === states[0].mode) ? states[0].mode : 'mixed',+ held: states.reduce((sum, state) => sum + state.held.length, 0),+ counts: { ...counts },+ fault,+ lanes: states.map(state => ({+ lane: state.lane,+ fqdn: state.fqdn,+ mode: state.mode,+ held: state.held.length,+ counts: { ...state.counts },+ unique: { ...state.unique },+ received: { ...state.received },+ questions: [...state.observations.values()].map(value => ({+ type: value.type,+ transport: value.transport,+ mode: value.mode,+ count: value.count,+ unique: value.seen.size,+ })),+ })),}+ }++ udp.on('message', (message, rinfo) =>+ handle(message, { transport: 'udp', rinfo, client: `${rinfo.address}:${rinfo.port}` })+ )+ udp.on('error', () => {+ fault = 'DNS_SOCKET_FAILED'+ })+ const tcp = net.createServer(socket => {+ connections.add(socket)+ socket.once('close', () => connections.delete(socket))+ socket.once('error', () => socket.destroy())+ socket.setTimeout(30000, () => socket.destroy())+ let pending = Buffer.alloc(0)+ socket.on('data', chunk => {+ pending = Buffer.concat([pending, chunk])+ if (pending.length > 65537) {+ socket.destroy()+ return+ }+ while (pending.length >= 2) {+ const length = pending.readUInt16BE(0)+ if (pending.length < length + 2) return+ const message = pending.subarray(2, length + 2)+ pending = pending.subarray(length + 2)+ handle(message, {+ transport: 'tcp',+ socket,+ client: `${socket.remoteAddress}:${socket.remotePort}`,+ })+ }+ })})-})-tcp.listen(8053, '0.0.0.0', listening)--http- .createServer((request, response) => {- const nextMode = request.url?.match(/^\/mode\/(ok|hold|servfail)$/)?.[1]- if (request.method === 'POST' && nextMode) {- setMode(nextMode)+ tcp.maxConnections = 64+ const control = http.createServer((request, response) => {+ const globalMode = request.url?.match(/^\/mode\/(ok|hold|servfail)$/)?.[1]+ const laneMode = request.url?.match(/^\/mode\/(ui|worker|canary)\/(ok|hold|servfail)$/)+ if (request.method === 'POST' && (globalMode || laneMode)) {+ const lane = laneMode?.[1]+ if (lane !== undefined && !lanes.has(lane)) {+ response.writeHead(404).end()+ return+ }+ setMode(globalMode ?? laneMode[2], lane)response.writeHead(200, { 'content-type': 'application/json' })- response.end(JSON.stringify({ mode, held: held.length, counts }))+ response.end(JSON.stringify(snapshot()))return}if (request.method === 'GET' && request.url === '/state') {response.writeHead(200, { 'content-type': 'application/json' })- response.end(JSON.stringify({ mode, held: held.length, counts }))+ response.end(JSON.stringify(snapshot()))return}- response.writeHead(404)- response.end()+ response.writeHead(404).end()})- .listen(8090, '127.0.0.1')+ control.requestTimeout = 5000+ control.headersTimeout = 5000++ const listen = (server, action) =>+ new Promise((resolve, reject) => {+ const failed = error => {+ server.removeListener('listening', ready)+ reject(error)+ }+ const ready = () => {+ server.removeListener('error', failed)+ resolve()+ }+ server.once('error', failed)+ server.once('listening', ready)+ action()+ })+ async function close() {+ if (closing) return+ closing = true+ for (const state of lanes.values()) state.held.length = 0+ for (const relayClose of [...relays]) relayClose()+ for (const socket of connections) socket.destroy()+ control.closeAllConnections()+ await Promise.all(+ [tcp, control].map(server => new Promise(resolve => server.close(() => resolve())))+ )+ await new Promise(resolve => {+ try {+ udp.close(resolve)+ } catch {+ resolve()+ }+ })+ }+ async function start() {+ if (started || closing) throw new Error('DNS fixture cannot be started twice')+ started = true+ try {+ await listen(udp, () => udp.bind(dnsPort, listenHost))+ await listen(tcp, () => tcp.listen(udp.address().port, listenHost))+ await listen(control, () => control.listen(controlPort, '127.0.0.1'))+ return { dnsPort: udp.address().port, controlPort: control.address().port }+ } catch (error) {+ await close()+ throw error+ }+ }+ return { start, close, snapshot, setMode }+}++module.exports = { createDnsProxy, readConfig, question, dnsResponse, tcpFrame }+if (require.main === module) {+ let proxy+ try {+ proxy = createDnsProxy(readConfig(process.env))+ } catch (error) {+ process.stderr.write(`${error.message}\n`)+ process.exitCode = 1+ }+ if (proxy) {+ for (const signal of ['SIGINT', 'SIGTERM']) {+ process.once(signal, () => {+ void proxy.close().then(() => {+ process.exitCode = signal === 'SIGINT' ? 130 : 143+ })+ })+ }+ proxy+ .start()+ .then(() => process.stdout.write('wrc egress DNS proxy ready\n'))+ .catch(() => {+ process.stderr.write('DNS fixture listeners could not start\n')+ process.exitCode = 1+ })+ }+}diff --git a/workflow-recipes/package-lock.json b/workflow-recipes/package-lock.jsonindex 543d4a0a6..a099d73c4 100644--- a/workflow-recipes/package-lock.json+++ b/workflow-recipes/package-lock.json@@ -1,12 +1,12 @@{"name": "workload-recipes-controller",- "version": "0.1.344",+ "version": "0.1.345","lockfileVersion": 3,"requires": true,"packages": {"": {"name": "workload-recipes-controller",- "version": "0.1.344",+ "version": "0.1.345","license": "SEE LICENSE IN LICENSE","dependencies": {"@aws-sdk/client-s3": "^3.1009.0",diff --git a/workflow-recipes/package.json b/workflow-recipes/package.jsonindex e2b3242cc..474d7ac77 100644--- a/workflow-recipes/package.json+++ b/workflow-recipes/package.json@@ -1,6 +1,6 @@{"name": "workload-recipes-controller",- "version": "0.1.344",+ "version": "0.1.345","description": "Workload Recipes Controller (WRC) - WorkloadRecipe lifecycle management and MCP server interface","main": "dist/main.js","engines": {diff --git a/workflow-recipes/src/k8sClient.test.ts b/workflow-recipes/src/k8sClient.test.tsindex 6daab5089..5a3e71402 100644--- a/workflow-recipes/src/k8sClient.test.ts+++ b/workflow-recipes/src/k8sClient.test.ts@@ -16,6 +16,7 @@ import {workflowNeedsWorkloadStatusRefresh,workflowRecipeFromWatchObject,} from './k8sClient'+import { captureLogger } from './reconciler/__tests__/captureLogger'import type { WorkflowRecipeCRD } from './types'function makeWorkflowRecipe(overrides: Partial<WorkflowRecipeCRD> = {}): WorkflowRecipeCRD {@@ -75,6 +76,74 @@ describe('workflowRecipeFromWatchObject', () => {})})+describe('persisted transport external-egress admission', () => {+ const pending = {+ type: 'TransportExternalEgressReady',+ status: 'False' as const,+ reason: 'ExternalEgressPending',+ message: 'Awaiting HCC admission',+ lastTransitionTime: '2026-09-05T00:00:00.000Z',+ }++ it('publishes explicit admission clearance without treating observation as clearance', () => {+ const recipe = makeWorkflowRecipe({+ spec: { steps: [] },+ status: { phase: 'active', message: 'Ready', conditions: [pending] },+ })+ const observed = { phase: 'active' as const, message: 'Ready', workloadStatuses: [] }+ expect(shouldPatchRecipeStatus(recipe, observed)).toBe(false)+ expect(shouldPatchRecipeStatus(recipe, { ...observed, transportNetworkConditions: [] })).toBe(+ true+ )+ expect(+ shouldPatchRecipeStatus(recipe, {+ ...observed,+ transportNetworkConditions: [+ { ...pending, lastTransitionTime: '2026-09-06T00:00:00.000Z' },+ ],+ })+ ).toBe(false)+ })++ it('restores retries only for live, pending admission rather than Pod-only degradation or terminal runs', () => {+ const recipe = makeWorkflowRecipe({+ spec: { steps: [] },+ status: { phase: 'degraded', conditions: [pending] },+ })+ expect(workflowNeedsInfrastructureReconcile(recipe)).toBe(true)+ expect(workflowNeedsInfrastructureReconcile({ ...recipe, status: { phase: 'degraded' } })).toBe(+ false+ )+ expect(+ workflowNeedsInfrastructureReconcile({+ ...recipe,+ status: { phase: 'failed', conditions: [pending] },+ })+ ).toBe(false)+ expect(+ workflowNeedsInfrastructureReconcile({+ ...recipe,+ metadata: { ...recipe.metadata, deletionTimestamp: '2026-09-05T00:00:00.000Z' },+ })+ ).toBe(false)+ const workflow = {+ ...recipe,+ spec: { ...recipe.spec, steps: [{ id: 's', instruction: 'run' }] },+ }+ expect(workflowNeedsInfrastructureReconcile(workflow)).toBe(true)+ expect(+ workflowNeedsInfrastructureReconcile({+ ...workflow,+ status: {+ phase: 'active',+ conditions: [pending],+ workflowExecution: { phase: 'completed' },+ },+ })+ ).toBe(false)+ })+})+describe('rotateBrokerTokensOnce', () => {function makeRecipe(name: string, withBackgroundAccess: boolean): WorkflowRecipeCRD {return {@@ -1397,6 +1466,55 @@ describe('workload status refresh loop helpers', () => {expect(reconcile).not.toHaveBeenCalled()})+ it.each([+ { from: 'failed', to: 'candidate', runs: 1 },+ { from: 'active', to: 'active', runs: 0 },+ { from: 'active', to: 'degraded', runs: 0 },+ { from: 'degraded', to: 'active', runs: 0 },+ { from: 'failed', to: 'failed', runs: 0 },+ { from: 'candidate', to: 'candidate', runs: 0 },+ ] as const)(+ 'handles the supported retry transition $from -> $to without a generation bump',+ async ({ from, to, runs }) => {+ const cached = makeWorkloadRecipe({+ metadata: { name: 'retry-recipe', namespace: 'sandbox-recipes', generation: 7 },+ status: { phase: from },+ })+ const requested = {+ ...cached,+ status: { phase: to, message: 'Manual retry requested by operator' },+ }+ const reconcile = vi.fn().mockResolvedValue({+ phase: 'active',+ message: 'All workloads deployed',+ workloadStatuses: [],+ })+ const watcher = Object.create(WorkflowRecipeWatcher.prototype) as {+ recipes: Map<string, WorkflowRecipeCRD>+ handleRecipeEvent: (type: string, recipe: WorkflowRecipeCRD) => Promise<void>+ }+ Object.assign(watcher, {+ recipes: new Map([[cached.metadata.name, cached]]),+ transientRetries: new Map(),+ stopped: false,+ traceReporter: null,+ dbRunProcessor: null,+ reconciler: {+ reconcile,+ isRecipeStillActive: vi.fn().mockResolvedValue(true),+ ensureFinalizer: vi.fn().mockResolvedValue(undefined),+ patchStatus: vi.fn().mockResolvedValue(undefined),+ },+ })+ await watcher.handleRecipeEvent('MODIFIED', requested)+ expect(reconcile).toHaveBeenCalledTimes(runs)+ expect(watcher.recipes.get(cached.metadata.name)).toEqual(requested)+ // Duplicate status events must not repeatedly execute the retry transition.+ await watcher.handleRecipeEvent('MODIFIED', requested)+ expect(reconcile).toHaveBeenCalledTimes(runs)+ }+ )+it('reconciles when Codex connection-ref changes without a generation bump', async () => {const cached = makeWorkloadRecipe({metadata: { name: 'active-recipe', namespace: 'sandbox-recipes', generation: 7 },@@ -1844,7 +1962,7 @@ describe('status-only events drive DB sync (terminal + heartbeat)', () => {const { internal, reconcile, sync } = makeInternal(cached, () =>Promise.reject(new Error('db down')))- const errSpy = vi.spyOn(console, 'error').mockImplementation(() => undefined)+ const errSpy = captureLogger('error').mockImplementation(() => undefined)await expect(internal.handleRecipeEvent('MODIFIED', completed)).resolves.toBeUndefined()@@ -1952,7 +2070,7 @@ describe('infrastructure telemetry enqueue isolation', () => {const enqueueInfrastructureTelemetry = vi.fn()const { internal } = makeInternal(recipe, { enqueueInfrastructureTelemetry })internal.reconciler.patchStatus.mockRejectedValue(new Error('patch failed'))- const errSpy = vi.spyOn(console, 'error').mockImplementation(() => undefined)+ const errSpy = captureLogger('error').mockImplementation(() => undefined)await internal.handleRecipeEvent('ADDED', recipe)@@ -1972,7 +2090,7 @@ describe('infrastructure telemetry enqueue isolation', () => {throw new Error('trace queue down')})const { internal } = makeInternal(recipe, { enqueueInfrastructureTelemetry })- const errSpy = vi.spyOn(console, 'error').mockImplementation(() => undefined)+ const errSpy = captureLogger('error').mockImplementation(() => undefined)await expect(internal.handleRecipeEvent('ADDED', recipe)).resolves.toBeUndefined()@@ -1980,7 +2098,7 @@ describe('infrastructure telemetry enqueue isolation', () => {expect(enqueueInfrastructureTelemetry).toHaveBeenCalled()expect(errSpy).toHaveBeenCalledWith('[WR-K8s] Infrastructure telemetry enqueue failed:',- expect.any(Error)+ expect.objectContaining({ err: expect.any(Error) }))errSpy.mockRestore()})@@ -2184,7 +2302,7 @@ describe('reconcile failure re-schedules the transient retry chain (issue #375 P}it('re-arms the transient retry after a non-404 reconcile failure (keeps the loud log)', async () => {- const errSpy = vi.spyOn(console, 'error').mockImplementation(() => undefined)+ const errSpy = captureLogger('error').mockImplementation(() => undefined)const recipe = makeWorkflowRecipe({ metadata: { name: 'flaky', namespace: 'sandbox-recipes' } })const internal = makeInternal(recipe, new Error('control-api unreachable'))try {@@ -2192,8 +2310,8 @@ describe('reconcile failure re-schedules the transient retry chain (issue #375 P// fail-loud preserved: the reconciliation-failed error is still logged.expect(errSpy).toHaveBeenCalledWith(- expect.stringContaining('Reconciliation failed for "flaky"'),- expect.anything()+ expect.stringContaining('Reconciliation failed'),+ expect.objectContaining({ name: 'flaky', err: expect.any(Error) }))// NEW: the 5s self-heal chain is re-armed instead of dying on first error.expect(internal.transientRetries.has('flaky')).toBe(true)@@ -2204,7 +2322,7 @@ describe('reconcile failure re-schedules the transient retry chain (issue #375 P})it('does not re-arm a retry when the recipe is already gone (404)', async () => {- const warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => undefined)+ const warnSpy = captureLogger('warn').mockImplementation(() => undefined)const recipe = makeWorkflowRecipe({ metadata: { name: 'gone', namespace: 'sandbox-recipes' } })const internal = makeInternal(recipe, { code: 404 })try {@@ -2272,7 +2390,7 @@ describe('Plugin Workload SDK computed-vs-published state divergence log (issue}it('logs the old→new SDK state and patchEmitted when the computed state diverges', async () => {- const logSpy = vi.spyOn(console, 'log').mockImplementation(() => undefined)+ const logSpy = captureLogger('info').mockImplementation(() => undefined)const recipe = sdkRecipe('awaiting_policy')const internal = makeInternal(recipe, {phase: 'active',@@ -2291,9 +2409,13 @@ describe('Plugin Workload SDK computed-vs-published state divergence log (issuetry {await internal.handleRecipeEvent('ADDED', recipe)expect(logSpy).toHaveBeenCalledWith(- expect.stringMatching(- /Plugin Workload SDK state.*sdk-recipe.*awaiting_policy -> validated.*patchEmitted=true/- )+ expect.stringContaining('Plugin Workload SDK state'),+ expect.objectContaining({+ name: 'sdk-recipe',+ value2: 'awaiting_policy',+ projectedSdkState: 'validated',+ willPatch: true,+ }))} finally {logSpy.mockRestore()@@ -2301,7 +2423,7 @@ describe('Plugin Workload SDK computed-vs-published state divergence log (issue})it('does not log a transition when the computed SDK state is unchanged', async () => {- const logSpy = vi.spyOn(console, 'log').mockImplementation(() => undefined)+ const logSpy = captureLogger('info').mockImplementation(() => undefined)const recipe = sdkRecipe('validated')const internal = makeInternal(recipe, {phase: 'active',@@ -2383,7 +2505,7 @@ describe('grant-update NOTIFY dispatch (issue #375 P3 wiring, H4/H5)', () => {})it('discards and warns on a namespace mismatch — no enqueue (H4)', () => {- const warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => undefined)+ const warnSpy = captureLogger('warn').mockImplementation(() => undefined)const recipe = makeWorkflowRecipe({metadata: { name: 'sdk-recipe', namespace: 'sandbox-recipes' },})@@ -2392,7 +2514,8 @@ describe('grant-update NOTIFY dispatch (issue #375 P3 wiring, H4/H5)', () => {internal.handleGrantUpdateNotification('other-namespace', 'sdk-recipe')expect(internal.eventQueue.enqueue).not.toHaveBeenCalled()expect(warnSpy).toHaveBeenCalledWith(- expect.stringContaining('Discarding grant-update NOTIFY for "other-namespace/sdk-recipe"')+ expect.stringContaining('Discarding grant-update NOTIFY'),+ expect.objectContaining({ recipeNamespace: 'other-namespace', recipeName: 'sdk-recipe' }))} finally {warnSpy.mockRestore()@@ -2400,12 +2523,15 @@ describe('grant-update NOTIFY dispatch (issue #375 P3 wiring, H4/H5)', () => {})it('discards and warns on an unknown recipe — no enqueue (H4)', () => {- const warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => undefined)+ const warnSpy = captureLogger('warn').mockImplementation(() => undefined)const { internal } = makeInternal([])try {internal.handleGrantUpdateNotification('sandbox-recipes', 'nope')expect(internal.eventQueue.enqueue).not.toHaveBeenCalled()- expect(warnSpy).toHaveBeenCalledWith(expect.stringContaining('unknown recipe'))+ expect(warnSpy).toHaveBeenCalledWith(+ expect.stringContaining('Discarding grant-update NOTIFY'),+ expect.objectContaining({ value3: 'unknown recipe' })+ )} finally {warnSpy.mockRestore()}diff --git a/workflow-recipes/src/k8sClient.ts b/workflow-recipes/src/k8sClient.tsindex 7d1715f70..a58690e47 100644--- a/workflow-recipes/src/k8sClient.ts+++ b/workflow-recipes/src/k8sClient.ts@@ -21,6 +21,7 @@ import {createGovernedTraceReporter,} from './governedTraceReporter'import { K8sSecretWatchLoop } from './k8sSecretWatchLoop'+import { createLogger } from './observability/logger'import { CRD_GROUP, CRD_VERSION, WORKFLOWRECIPE_PLURAL } from './reconciler/crdConstants'import {type ChildRecipeCreator,@@ -44,7 +45,9 @@ import { SecretWatcher } from './reconciler/secretWatcher'import {type ReconcileResult,TRANSIENT_REQUEUE_BASE_MS,+ TRANSPORT_NETWORK_CONDITION_TYPE,WorkflowRecipeReconciler,+ hasPendingTransportNetworkReadiness,} from './reconciler/workflowRecipeReconciler'import {type StatusCondition,@@ -148,6 +151,14 @@ export function shouldPatchRecipeStatus(// cleared. That still needs one status patch even though phase is unchanged.if (result.clearWorkflowExecution && recipe.status?.workflowExecution) return true+ if (+ result.transportNetworkConditions !== undefined &&+ ownedConditionsChanged(recipe.status?.conditions, result.transportNetworkConditions, [+ TRANSPORT_NETWORK_CONDITION_TYPE,+ ])+ )+ return true+return false}@@ -307,7 +318,10 @@ function enqueueInfrastructureTelemetryBestEffort(try {traceReporter.enqueueInfrastructureTelemetry(projection)} catch (error) {- console.error('[WR-K8s] Infrastructure telemetry enqueue failed:', error)+ createLogger('wrc', 'workflow-recipes').error(+ '[WR-K8s] Infrastructure telemetry enqueue failed:',+ { err: error }+ )}}@@ -405,6 +419,15 @@ export function workflowNeedsRuntimeCredentialRefresh(recipe: WorkflowRecipeCRD)export function workflowNeedsInfrastructureReconcile(recipe: WorkflowRecipeCRD): boolean {const isWorkflow = (recipe.spec.steps ?? []).length > 0if (recipe.metadata.deletionTimestamp) return false+ const workflowPhase = recipe.status?.workflowExecution?.phase+ // Restore the retry after a process restart, including stepless stdio recipes+ // whose egress is exclusively HCC-owned. Pod readiness is not admission proof.+ if (+ hasPendingTransportNetworkReadiness(recipe) &&+ !['failed', 'deprecated', 'rollback-failed'].includes(recipe.status?.phase ?? '') &&+ !['completed', 'failed', 'cancelled'].includes(workflowPhase ?? '')+ )+ return true// SDK-only recipes are deliberately stepless, but their eager mcp-host and// bootstrap proof still need the full reconciler lane. The generic workload@@ -414,7 +437,6 @@ export function workflowNeedsInfrastructureReconcile(recipe: WorkflowRecipeCRD):if (!isWorkflow && recipe.spec.pluginWorkloadSdk) return trueif (!isWorkflow) return false- const workflowPhase = recipe.status?.workflowExecution?.phaseif (workflowPhase === 'initializing' || workflowPhase === 'recovering') return true// Plugin Workload SDK eager path: the recipe sits in phase=active with no run@@ -638,7 +660,10 @@ export class WorkflowRecipeWatcher implements WorkflowRecipeProvider {private externalEgressRefreshTimer: ReturnType<typeof setTimeout> | null = nullprivate externalEgressRefreshRunning = falseprivate eventQueue = new RecipeEventQueue((key, error) => {- console.error(`[WR-K8s] Unhandled event failure for "${key}":`, error)+ createLogger('wrc', 'workflow-recipes').error('Unhandled recipe event failure', {+ key,+ err: error,+ })})private secretReverseIndex = new SecretReverseIndex()private secretWatchLoops: K8sSecretWatchLoop[] = []@@ -723,10 +748,13 @@ export class WorkflowRecipeWatcher implements WorkflowRecipeProvider {// independent env vars; if they ever diverge, 100% of NOTIFYs would be// dropped and the feature would silently degrade to 30s polling. Log names// only — no secrets.- console.warn(- `[WR-K8s] Discarding grant-update NOTIFY for "${recipeNamespace}/${recipeName}": ` +- (known ? `namespace mismatch (watching "${known.metadata.namespace}")` : 'unknown recipe')- )+ createLogger('wrc', recipeName).warn('Discarding grant-update NOTIFY', {+ recipeNamespace,+ recipeName,+ value3: known+ ? `namespace mismatch (watching "${known.metadata.namespace}")`+ : 'unknown recipe',+ })return}void this.eventQueue.enqueue(recipeName, () => {@@ -787,7 +815,7 @@ export class WorkflowRecipeWatcher implements WorkflowRecipeProvider {}async start(): Promise<void> {- console.log('[WR-K8s] Starting WorkflowRecipe watcher')+ createLogger('wrc', 'workflow-recipes').info('[WR-K8s] Starting WorkflowRecipe watcher')// Initialize workflow subsystem by reading the signing key from control-plane.// Non-fatal: if the Secret doesn't exist, workflow recipes will gracefully fail@@ -821,7 +849,10 @@ export class WorkflowRecipeWatcher implements WorkflowRecipeProvider {this.reconciler.ensureOAuthBrokerTokenSecret(recipe)),(recipe, err) => {- console.error(`[WR-K8s] Broker-token refresh failed for "${recipe.metadata.name}":`, err)+ createLogger('wrc', recipe.metadata.name).error('Broker-token refresh failed', {+ name: recipe.metadata.name,+ err,+ })})}@@ -895,14 +926,16 @@ export class WorkflowRecipeWatcher implements WorkflowRecipeProvider {const privateKeyB64 = secret.data?.['private.pem']const publicKeyB64 = secret.data?.['public.pem']if (!privateKeyB64) {- console.warn(- `[WR-K8s] Secret "${SIGNING_KEY_SECRET}" exists but missing private.pem key — workflow disabled`+ createLogger('wrc', 'workflow-recipes').warn(+ 'Signing Secret lacks private.pem; workflow disabled',+ { SIGNING_KEY_SECRET })return}if (!publicKeyB64) {- console.warn(- `[WR-K8s] Secret "${SIGNING_KEY_SECRET}" exists but missing public.pem key — JWT verification disabled`+ createLogger('wrc', 'workflow-recipes').warn(+ 'Signing Secret lacks public.pem; JWT verification disabled',+ { SIGNING_KEY_SECRET })}@@ -911,15 +944,21 @@ export class WorkflowRecipeWatcher implements WorkflowRecipeProvider {? Buffer.from(publicKeyB64, 'base64').toString('utf-8'): undefinedawait this.reconciler.initializeWorkflow(privateKeyPem, publicKeyPem)- console.log('[WR-K8s] Workflow subsystem initialized from signing key')+ createLogger('wrc', 'workflow-recipes').info(+ '[WR-K8s] Workflow subsystem initialized from signing key'+ )} catch (error: unknown) {const code = (error as { response?: { statusCode?: number } })?.response?.statusCodeif (code === 404 || code === 403) {- console.warn(- `[WR-K8s] Signing key Secret not found or forbidden (${code}) — workflow disabled`+ createLogger('wrc', 'workflow-recipes').warn(+ 'Signing Secret not found or forbidden; workflow disabled',+ { code })} else {- console.warn('[WR-K8s] Failed to read signing key — workflow disabled:', error)+ createLogger('wrc', 'workflow-recipes').warn(+ '[WR-K8s] Failed to read signing key — workflow disabled:',+ { err: error }+ )}}}@@ -933,7 +972,10 @@ export class WorkflowRecipeWatcher implements WorkflowRecipeProvider {const watchCallback = (type: string, apiObj: WorkflowRecipeWatchObject) => {const recipe = workflowRecipeFromWatchObject(apiObj, this.config.sandboxNamespace)- console.log(`[WR-K8s] Watch event: ${type} for "${recipe.metadata.name}"`)+ createLogger('wrc', recipe.metadata.name).info('Received recipe watch event', {+ type,+ name: recipe.metadata.name,+ })this.eventQueue.enqueue(recipe.metadata.name, () => this.handleRecipeEvent(type, recipe))}@@ -941,9 +983,9 @@ export class WorkflowRecipeWatcher implements WorkflowRecipeProvider {const doneCallback = (err: Error | null) => {if (this.stopped) returnif (err) {- console.error('[WR-K8s] Watch error:', err)+ createLogger('wrc', 'workflow-recipes').error('[WR-K8s] Watch error:', { err })}- console.log('[WR-K8s] Watch ended, restarting...')+ createLogger('wrc', 'workflow-recipes').info('[WR-K8s] Watch ended, restarting...')setTimeout(() => this.startWatch(), err ? 5000 : 1000)}@@ -959,17 +1001,23 @@ export class WorkflowRecipeWatcher implements WorkflowRecipeProvider {let dbSyncAttempted = falseif (recipe.metadata.deletionTimestamp) {// Resource is being deleted — cleanup cross-namespace resources, then remove finalizer.- console.log(`[WR-K8s] Finalizer cleanup for "${recipe.metadata.name}"`)+ createLogger('wrc', recipe.metadata.name).info('Cleaning up recipe finalizer', {+ name: recipe.metadata.name,+ })try {await this.reconciler.reconcileDelete(recipe)await this.reconciler.removeFinalizer(recipe)} catch (error) {if (getErrorCode(error) === 404) {- console.warn(- `[WR-K8s] Finalizer cleanup skipped for "${recipe.metadata.name}"; recipe already gone`+ createLogger('wrc', recipe.metadata.name).warn(+ 'Skipping finalizer cleanup because the recipe is gone',+ { name: recipe.metadata.name })} else {- console.error(`[WR-K8s] Finalizer cleanup failed for "${recipe.metadata.name}":`, error)+ createLogger('wrc', recipe.metadata.name).error('Finalizer cleanup failed', {+ name: recipe.metadata.name,+ err: error,+ })}}this.recipes.delete(recipe.metadata.name)@@ -997,17 +1045,18 @@ export class WorkflowRecipeWatcher implements WorkflowRecipeProvider {try {await this.dbRunProcessor.syncFromRecipeExecution(recipe)} catch (error) {- console.error(- `[WR-K8s] DB sync failed for status-only event "${recipe.metadata.name}":`,- error+ createLogger('wrc', recipe.metadata.name).error(+ 'Database sync failed for a status-only event',+ { name: recipe.metadata.name, err: error })} finally {dbSyncAttempted = true}}if (needsTerminalRunTeardown) {- console.log(- `[WR-K8s] Terminal run status-only event for "${recipe.metadata.name}" — forcing reconcile for compute teardown`+ createLogger('wrc', recipe.metadata.name).info(+ 'Forcing reconciliation of terminal run status for compute teardown',+ { name: recipe.metadata.name })} else {return@@ -1018,8 +1067,9 @@ export class WorkflowRecipeWatcher implements WorkflowRecipeProvider {this.recipes.set(recipe.metadata.name, recipe)try {if (!(await this.reconciler.isRecipeStillActive(recipe))) {- console.warn(- `[WR-K8s] Skipping stale event for "${recipe.metadata.name}"; recipe is gone or deleting`+ createLogger('wrc', recipe.metadata.name).warn(+ 'Skipping stale event because the recipe is gone or deleting',+ { name: recipe.metadata.name })this.recipes.delete(recipe.metadata.name)return@@ -1029,8 +1079,9 @@ export class WorkflowRecipeWatcher implements WorkflowRecipeProvider {}await this.reconciler.ensureFinalizer(recipe)if (!(await this.reconciler.isRecipeStillActive(recipe))) {- console.warn(- `[WR-K8s] Skipping reconcile for "${recipe.metadata.name}"; recipe started deleting after finalizer check`+ createLogger('wrc', recipe.metadata.name).warn(+ 'Skipping reconciliation because deletion began after the finalizer check',+ { name: recipe.metadata.name })this.recipes.delete(recipe.metadata.name)return@@ -1052,10 +1103,12 @@ export class WorkflowRecipeWatcher implements WorkflowRecipeProvider {// capability); its clearing PATCH is still emitted via// `pluginWorkloadSdkProjectionChanged`, so the publish is covered.if (projectedSdkState !== undefined && projectedSdkState !== persistedSdkState) {- console.log(- `[WR-K8s] Plugin Workload SDK state for "${recipe.metadata.name}": ` +- `${persistedSdkState ?? 'none'} -> ${projectedSdkState} (patchEmitted=${willPatch})`- )+ createLogger('wrc', recipe.metadata.name).info('Plugin Workload SDK state changed', {+ name: recipe.metadata.name,+ value2: persistedSdkState ?? 'none',+ projectedSdkState,+ willPatch,+ })}if (willPatch) {await this.reconciler.patchStatus(recipe, result)@@ -1078,13 +1131,17 @@ export class WorkflowRecipeWatcher implements WorkflowRecipeProvider {}} catch (error) {if (getErrorCode(error) === 404) {- console.warn(- `[WR-K8s] Reconciliation skipped for "${recipe.metadata.name}"; recipe already gone`+ createLogger('wrc', recipe.metadata.name).warn(+ 'Reconciliation skipped because the recipe is gone',+ { name: recipe.metadata.name })this.recipes.delete(recipe.metadata.name)return}- console.error(`[WR-K8s] Reconciliation failed for "${recipe.metadata.name}":`, error)+ createLogger('wrc', recipe.metadata.name).error('Reconciliation failed', {+ name: recipe.metadata.name,+ err: error,+ })enqueueInfrastructureTelemetryBestEffort(this.traceReporter,controllerErrorTelemetryProjection(recipe, error)@@ -1109,6 +1166,10 @@ export class WorkflowRecipeWatcher implements WorkflowRecipeProvider {private isStatusOnlyModifiedEvent(recipe: WorkflowRecipeCRD): boolean {const cached = this.recipes.get(recipe.metadata.name)+ // POST /admin/recipes/:name/retry requests the canonical failed -> candidate+ // transition through /status, without changing spec/generation. Run that+ // transition once; subsequent candidate/status updates remain coalesced.+ if (cached?.status?.phase === 'failed' && recipe.status?.phase === 'candidate') return falseconst cachedGeneration = cached?.metadata.generationconst nextGeneration = recipe.metadata.generationif (@@ -1165,7 +1226,12 @@ export class WorkflowRecipeWatcher implements WorkflowRecipeProvider {return}const label = fixedInterval ? 'progress' : 'transient'- console.log(`[WR-K8s] ${label} retry for "${name}" (attempt ${attempts + 1}, ${delay}ms)`)+ createLogger('wrc', 'workflow-recipes').info('Retrying recipe reconciliation', {+ label,+ name,+ value3: attempts + 1,+ delay,+ })void this.eventQueue.enqueue(name, () =>this.handleRecipeEvent('MODIFIED', cached, { forceReconcile: true }))@@ -1285,7 +1351,10 @@ export class WorkflowRecipeWatcher implements WorkflowRecipeProvider {)enqueued++} catch (error) {- console.error(`[WR-K8s] External egress refresh failed for "${name}":`, error)+ createLogger('wrc', 'workflow-recipes').error('External egress refresh failed', {+ name,+ err: error,+ })}})@@ -1296,8 +1365,9 @@ export class WorkflowRecipeWatcher implements WorkflowRecipeProvider {// where the set is decided and written instead — `logResolvedEgressSet` in// workflowRecipeReconciler.ts, one entry per FQDN, only when the set changed// and only once the policy has landed (#299). Secrets stay out of both.- console.log(- `[WR-K8s] External egress refresh: re-enqueued ${enqueued}/${selected.length} recipe(s) in ${Date.now() - started}ms`+ createLogger('wrc', 'workflow-recipes').info(+ 'Re-enqueued recipes for external egress refresh',+ { enqueued, length: selected.length, value3: Date.now() - started })}@@ -1319,7 +1389,10 @@ export class WorkflowRecipeWatcher implements WorkflowRecipeProvider {this.refreshWorkloadStatusOnly(latest, { ownershipOnly: !needsStatusRefresh }))} catch (error) {- console.error(`[WR-K8s] Workload status refresh failed for "${recipeName}":`, error)+ createLogger('wrc', recipeName).error('Workload status refresh failed', {+ recipeName,+ err: error,+ })}})@@ -1377,9 +1450,9 @@ export class WorkflowRecipeWatcher implements WorkflowRecipeProvider {)}} catch (error) {- console.error(- `[WR-K8s] Runtime credential refresh or infrastructure retry failed for "${recipeName}":`,- error+ createLogger('wrc', recipeName).error(+ 'Runtime credential refresh or infrastructure retry failed',+ { recipeName, err: error })}})@@ -1404,7 +1477,7 @@ export class WorkflowRecipeWatcher implements WorkflowRecipeProvider {this.externalEgressRefreshTimer = null}if (this.watchRequest) {- console.log('[WR-K8s] Stopping watch')+ createLogger('wrc', 'workflow-recipes').info('[WR-K8s] Stopping watch')this.watchRequest.abort()this.watchRequest = null}@@ -1431,9 +1504,14 @@ export class WorkflowRecipeWatcher implements WorkflowRecipeProvider {if (this.traceReporter) {const result = await this.traceReporter.stopAndDrain()if (!result.drained) {- console.warn('[WR-K8s] Governed trace reporter shutdown timed out', {- dropped: result.dropped,- })+ createLogger('wrc', 'workflow-recipes').warn(+ '[WR-K8s] Governed trace reporter shutdown timed out',+ {+ value1: {+ dropped: result.dropped,+ },+ }+ )}}}@@ -1464,7 +1542,9 @@ export class DevWorkflowRecipeProvider implements WorkflowRecipeProvider {async start(): Promise<void> {const raw = process.env.CLERUM_RECIPESif (!raw) {- console.log('[WR-Dev] No CLERUM_RECIPES env var — starting with empty recipe list')+ createLogger('wrc', 'workflow-recipes').info(+ '[WR-Dev] No CLERUM_RECIPES env var — starting with empty recipe list'+ )return}@@ -1472,16 +1552,22 @@ export class DevWorkflowRecipeProvider implements WorkflowRecipeProvider {const recipes = JSON.parse(raw) as WorkflowRecipeCRD[]for (const recipe of recipes) {this.recipes.set(recipe.metadata.name, recipe)- console.log(`[WR-Dev] Loaded recipe: ${recipe.metadata.name}`)+ createLogger('wrc', recipe.metadata.name).info('Loaded development recipe', {+ name: recipe.metadata.name,+ })}- console.log(`[WR-Dev] Loaded ${this.recipes.size} recipe(s)`)+ createLogger('wrc', 'workflow-recipes').info('Loaded development recipes', {+ size: this.recipes.size,+ })} catch (error) {- console.error('[WR-Dev] Failed to parse CLERUM_RECIPES:', error)+ createLogger('wrc', 'workflow-recipes').error('[WR-Dev] Failed to parse CLERUM_RECIPES:', {+ err: error,+ })}}async stop(): Promise<void> {- console.log('[WR-Dev] Stopping dev provider')+ createLogger('wrc', 'workflow-recipes').info('[WR-Dev] Stopping dev provider')}}@@ -1495,7 +1581,7 @@ export class DevWorkflowRecipeProvider implements WorkflowRecipeProvider {export function createWorkflowRecipeProvider(): WorkflowRecipeProvider {const config = loadConfig()if (config.devMode) {- console.log('[WR-Provider] Creating dev mode provider')+ createLogger('wrc', 'workflow-recipes').info('[WR-Provider] Creating dev mode provider')return new DevWorkflowRecipeProvider()}@@ -1507,9 +1593,9 @@ export function createWorkflowRecipeProvider(): WorkflowRecipeProvider {try {pool = initDb(config.db)} catch (err) {- console.warn(+ createLogger('wrc', 'workflow-recipes').warn('[WR-Provider] initDb() failed — DB run processor disabled:',- err instanceof Error ? err.message : err+ { err })pool = null}@@ -1525,6 +1611,6 @@ export function createWorkflowRecipeProvider(): WorkflowRecipeProvider {}}- console.log('[WR-Provider] Creating K8s watcher provider')+ createLogger('wrc', 'workflow-recipes').info('[WR-Provider] Creating K8s watcher provider')return new WorkflowRecipeWatcher(kc, pool)}diff --git a/workflow-recipes/src/observability/logger.ts b/workflow-recipes/src/observability/logger.tsindex a99c009c4..65de8c405 100644--- a/workflow-recipes/src/observability/logger.ts+++ b/workflow-recipes/src/observability/logger.ts@@ -4,6 +4,7 @@* Produces one JSON line per log entry to stdout. Cluster log collectors* (Loki, Fluent Bit) pick up stdout automatically.*/+import { isNativeError } from 'node:util/types'export interface Logger {debug(msg: string, fields?: Record<string, unknown>): void@@ -23,6 +24,9 @@ const LEVEL_PRIORITY: Record<LogLevel, number> = {}const REDACTED = '[REDACTED]'+// Error causes can contain arbitrary response objects or recurse indefinitely.+const MAX_ERROR_DEPTH = 6+const MAX_ERROR_TEXT_LENGTH = 16_384const SENSITIVE_KEY_PATTERNS = [/authorization/i,/cookie/i,@@ -69,16 +73,63 @@ function isSensitiveKey(key: string): boolean {return SENSITIVE_KEY_PATTERNS.some(pattern => pattern.test(key))}-function redactValue(value: unknown, key: string, seen: WeakSet<object>): unknown {+function redactErrorText(text: string): string {+ if (text.length > MAX_ERROR_TEXT_LENGTH) return '[Truncated]'+ // Newly exposed native diagnostics must not reintroduce access data through+ // messages/stack traces. URLs are removed in full, including private paths.+ return (+ text+ // ApiException duplicates JSON.stringify(body/headers) on these diagnostic+ // lines. Remove the entire sections, including fields not named sensitive,+ // while retaining the HTTP code, ordinary message and stack frames.+ .replace(/^(\s*(?:Body|Headers):)[^\r\n]*/gim, '$1 ' + REDACTED)+ // Generic messages can embed quoted JSON, including escaped JSON strings.+ // Its value may itself contain quotes/newlines, so do not guess its end:+ // preserve the diagnostic prefix and redact the remaining fragment.+ .replace(+ /(\b[\w.-]*(?:authorization|cookie|token|secret|password|credential|api[_-]?key)[\w.-]*\\*["']\s*:\s*)[\s\S]*/gi,+ '$1' + REDACTED+ )+ .replace(/\b[a-z][a-z\d+.-]*:\/\/[^\s"'<>]+/gi, REDACTED)+ .replace(/\b(?:authorization|set-cookie|cookie)\s*:\s*[^\r\n]+/gi, REDACTED)+ .replace(/\b(?:bearer|basic)\s+[^\s,;]+/gi, REDACTED)+ .replace(+ /(\b[\w.-]*(?:token|secret|password|credential|api[_-]?key)[\w.-]*\s*[:=]\s*)(?:"[^"]*"|'[^']*'|[^\s,;]+)/gi,+ `$1${REDACTED}`+ )+ )+}++function redactValue(value: unknown, key: string, seen: WeakSet<object>, errorDepth = 0): unknown {if (isSensitiveKey(key)) return REDACTED- if (Array.isArray(value)) return value.map(item => redactValue(item, '', seen))+ if (errorDepth > 0 && /^(body|headers|request|response)$/i.test(key)) return REDACTED+ if (errorDepth > 0 && typeof value === 'string') return redactErrorText(value)if (typeof value === 'object' && value !== null) {if (seen.has(value)) return '[Circular]'+ if (errorDepth >= MAX_ERROR_DEPTH) return '[Truncated]'seen.add(value)+ const nativeError = isNativeError(value)+ const nextDepth = errorDepth > 0 || nativeError ? errorDepth + 1 : 0+ if (Array.isArray(value)) return value.map(item => redactValue(item, '', seen, nextDepth))+ let fields: Record<string, unknown> = value as Record<string, unknown>+ if (nativeError) {+ try {+ fields = {+ ...value,+ name: value.name,+ message: value.message,+ stack: value.stack,+ ...('cause' in value ? { cause: value.cause } : {}),+ }+ } catch {+ // A custom Error accessor must not replace the failure being reported.+ return { name: 'Error', message: '[Unserializable]' }+ }+ }return Object.fromEntries(- Object.entries(value as Record<string, unknown>).map(([childKey, childValue]) => [+ Object.entries(fields).map(([childKey, childValue]) => [childKey,- redactValue(childValue, childKey, seen),+ redactValue(childValue, childKey, seen, nextDepth),]))}diff --git a/workflow-recipes/src/reconciler/__tests__/captureLogger.ts b/workflow-recipes/src/reconciler/__tests__/captureLogger.tsnew file mode 100644index 000000000..f880e1fac--- /dev/null+++ b/workflow-recipes/src/reconciler/__tests__/captureLogger.ts@@ -0,0 +1,16 @@+import { vi } from 'vitest'+import * as logging from '../../observability/logger'++/** Observe the service logging boundary without replacing other log levels. */+export function captureLogger(level: 'info' | 'warn' | 'error') {+ const original = logging.createLogger+ const calls = vi.fn()+ const factory = vi.spyOn(logging, 'createLogger').mockImplementation((...args) => ({+ ...original(...args),+ [level]: calls,+ }))+ calls.mockRestore = () => {+ factory.mockRestore()+ }+ return calls+}diff --git a/workflow-recipes/src/reconciler/networkPolicyContraction.test.ts b/workflow-recipes/src/reconciler/networkPolicyContraction.test.tsnew file mode 100644index 000000000..a9c3a8a90--- /dev/null+++ b/workflow-recipes/src/reconciler/networkPolicyContraction.test.ts@@ -0,0 +1,127 @@+import { describe, expect, it } from 'vitest'+import type * as k8s from '@kubernetes/client-node'+import { intersectNetworkPolicyRules, networkPolicySpecSignature } from './networkPolicyContraction'++const peer = {+ namespaceSelector: { matchLabels: { 'kubernetes.io/metadata.name': 'sandbox-ui' } },+ podSelector: { matchLabels: { 'clerum.io/recipe': 'one' } },+}+const ingress = (ports: number[]): k8s.V1NetworkPolicyIngressRule[] => [+ {+ _from: [peer],+ ports: ports.map(port => ({ port, protocol: 'TCP' })),+ },+]+const policy = (rules?: k8s.V1NetworkPolicyIngressRule[]): k8s.V1NetworkPolicy => ({+ spec: {+ podSelector: { matchLabels: { app: 'backend' } },+ policyTypes: ['Ingress'],+ ...(rules ? { ingress: rules } : {}),+ },+})++describe('WRC NetworkPolicy permission contraction', () => {+ it('compares API-omitted empty top-level lists as equivalent', () => {+ expect(networkPolicySpecSignature(policy())).toBe(networkPolicySpecSignature(policy([])))+ })++ it('compares grouped, split, duplicated and reordered permissions as equivalent', () => {+ expect(networkPolicySpecSignature(policy(ingress([8080, 9090])))).toBe(+ networkPolicySpecSignature(policy([...ingress([9090]), ...ingress([8080, 8080])]))+ )+ })++ it('preserves the namespace-and-pod conjunction while splitting peer alternatives', () => {+ const other = {+ namespaceSelector: peer.namespaceSelector,+ podSelector: { matchLabels: { 'clerum.io/recipe': 'two' } },+ }+ const result = intersectNetworkPolicyRules(+ [{ _from: [peer, other], ports: [{ port: 8080 }] }],+ ingress([8080]),+ '_from'+ )+ expect(result).toEqual(ingress([8080]))+ expect(result[0]._from).toEqual([peer])+ expect(+ intersectNetworkPolicyRules(+ [+ {+ _from: [+ { namespaceSelector: peer.namespaceSelector },+ { podSelector: peer.podSelector },+ ],+ ports: [{ port: 8080 }],+ },+ ],+ ingress([8080]),+ '_from'+ )+ ).toEqual([])+ })++ it('keeps TCP defaulting while rejecting different protocols', () => {+ expect(+ intersectNetworkPolicyRules(+ [{ _from: [peer], ports: [{ port: 8080 }] }],+ ingress([8080]),+ '_from'+ )+ ).toEqual(ingress([8080]))+ expect(+ intersectNetworkPolicyRules(+ [{ _from: [peer], ports: [{ port: 8080, protocol: 'UDP' }] }],+ ingress([8080]),+ '_from'+ )+ ).toEqual([])+ })++ it('does not equate inner omitted peers or ports to a deny rule', () => {+ expect(networkPolicySpecSignature(policy([{}]))).not.toBe(+ networkPolicySpecSignature(policy([]))+ )+ expect(networkPolicySpecSignature(policy([{ _from: [peer] }]))).not.toBe(+ networkPolicySpecSignature(policy(ingress([8080])))+ )+ expect(intersectNetworkPolicyRules([{}], ingress([8080]), '_from')).toEqual(ingress([8080]))+ expect(intersectNetworkPolicyRules(ingress([8080]), [{}], '_from')).toEqual(ingress([8080]))+ })++ it('does not ignore an IP exception or an endPort restriction', () => {+ const wanted = [+ { to: [{ ipBlock: { cidr: '8.8.8.0/24' } }], ports: [{ port: 443, protocol: 'TCP' }] },+ ]+ expect(+ intersectNetworkPolicyRules(+ [{ ...wanted[0], to: [{ ipBlock: { cidr: '8.8.8.0/24', except: ['8.8.8.8/32'] } }] }],+ wanted,+ 'to'+ )+ ).toEqual([])+ expect(+ intersectNetworkPolicyRules(+ [{ ...wanted[0], ports: [{ port: 443, endPort: 444, protocol: 'TCP' }] }],+ wanted,+ 'to'+ )+ ).toEqual([])+ })++ it('cannot prove an unknown rule restriction by dropping it', () => {+ const unknown = { ...ingress([8080])[0], futureRestriction: true }+ expect(intersectNetworkPolicyRules([unknown], ingress([8080]), '_from')).toEqual([])+ expect(networkPolicySpecSignature(policy([unknown]))).not.toBe(+ networkPolicySpecSignature(policy(ingress([8080])))+ )+ })++ it('preserves selectors and effective isolation types in equivalence checks', () => {+ expect(networkPolicySpecSignature(policy())).not.toBe(+ networkPolicySpecSignature({ spec: { ...policy().spec!, podSelector: {} } })+ )+ expect(networkPolicySpecSignature(policy())).not.toBe(+ networkPolicySpecSignature({ spec: { ...policy().spec!, policyTypes: ['Egress'] } })+ )+ })+})diff --git a/workflow-recipes/src/reconciler/networkPolicyContraction.ts b/workflow-recipes/src/reconciler/networkPolicyContraction.tsnew file mode 100644index 000000000..3a8a37842--- /dev/null+++ b/workflow-recipes/src/reconciler/networkPolicyContraction.ts@@ -0,0 +1,127 @@+import type * as k8s from '@kubernetes/client-node'++type Rule = k8s.V1NetworkPolicyEgressRule & k8s.V1NetworkPolicyIngressRule+type PeerKey = 'to' | '_from'++function canonical(value: unknown): unknown {+ if (Array.isArray(value)) return value.map(canonical)+ if (value && typeof value === 'object') {+ return Object.fromEntries(+ Object.entries(value)+ .sort(([a], [b]) => a.localeCompare(b))+ .map(([key, item]) => [key, canonical(item)])+ )+ }+ return value+}++function signature(value: unknown): string {+ return JSON.stringify(canonical(value))+}++function uniqueRules(rules: Rule[]): Rule[] {+ return [...new Map(rules.map(rule => [signature(rule), rule])).entries()]+ .sort(([a], [b]) => a.localeCompare(b))+ .map(([, rule]) => rule)+}++/** Split only the OR lists; namespaceSelector + podSelector stay one AND peer.+ * Empty/missing inner lists mean unrestricted, unlike empty top-level rules.+ * Keep all other fields when comparing; an unknown restriction must not vanish.+ */+function splitRules(rules: Rule[] | undefined, peerKey: PeerKey): Rule[] {+ return uniqueRules(+ (rules ?? []).flatMap(rule => {+ const peers = rule[peerKey]?.length ? rule[peerKey]! : [undefined]+ const ports = rule.ports?.length ? rule.ports : [undefined]+ return peers.flatMap(peer =>+ ports.map(port => ({+ ...rule,+ [peerKey]: peer ? [peer] : undefined,+ ports: port ? [{ ...port, protocol: port.protocol ?? 'TCP' }] : undefined,+ }))+ )+ })+ )+}++export function effectivePolicyTypes(spec: k8s.V1NetworkPolicySpec | undefined): string[] {+ return [+ ...(spec?.policyTypes?.length+ ? spec.policyTypes+ : ['Ingress', ...(spec?.egress?.length ? ['Egress'] : [])]),+ ].sort()+}++/** Normalize Kubernetes omitempty only at the rule-list boundary. Nested empty+ * peers/ports have different semantics and are handled by splitRules instead.+ */+export function networkPolicySpecSignature(policy: k8s.V1NetworkPolicy): string {+ return signature({+ ...policy.spec,+ podSelector: policy.spec?.podSelector ?? {},+ policyTypes: effectivePolicyTypes(policy.spec),+ egress: splitRules(policy.spec?.egress, 'to'),+ ingress: splitRules(policy.spec?.ingress, '_from'),+ })+}++function supportedRule(rule: Rule, peerKey: PeerKey): boolean {+ return (+ Object.entries(rule).every(+ ([key, value]) => value === undefined || key === peerKey || key === 'ports'+ ) &&+ (rule.ports ?? []).every(port =>+ Object.entries(port).every(+ ([key, value]) =>+ value === undefined || key === 'port' || key === 'protocol' || key === 'endPort'+ )+ )+ )+}++/** Intersection of the finite rule shapes emitted by WRC. Peers are matched as+ * whole conjunctions; ports preserve protocol and endPort. Unrestricted live+ * dimensions can be narrowed to current intent, never the reverse. Unrecognized+ * fields cannot prove a retained permission and are deliberately not copied.+ */+export function intersectNetworkPolicyRules(+ existing: Rule[] | undefined,+ desired: Rule[] | undefined,+ peerKey: PeerKey+): Rule[] {+ const liveRules = splitRules(existing, peerKey).filter(rule => supportedRule(rule, peerKey))+ const desiredRules = splitRules(desired, peerKey).filter(rule => supportedRule(rule, peerKey))+ const retained: Rule[] = []+ for (const wanted of desiredRules) {+ for (const live of liveRules) {+ const wantedPeer = wanted[peerKey]?.[0]+ const livePeer = live[peerKey]?.[0]+ if (wantedPeer && livePeer && signature(wantedPeer) !== signature(livePeer)) continue+ const wantedPort = wanted.ports?.[0]+ const livePort = live.ports?.[0]+ if (wantedPort && livePort) {+ if (wantedPort.protocol !== livePort.protocol) continue+ if (+ wantedPort.port !== undefined &&+ livePort.port !== undefined &&+ signature(wantedPort) !== signature(livePort)+ )+ continue+ // endPort is never discarded to pretend an unsupported range is all ports.+ if (+ (wantedPort.endPort !== undefined && wantedPort.port === undefined) ||+ (livePort.endPort !== undefined && livePort.port === undefined)+ )+ continue+ }+ const peer = wantedPeer ?? livePeer+ const port = wantedPort?.port !== undefined ? wantedPort : (livePort ?? wantedPort)+ retained.push({+ [peerKey]: peer ? [peer] : undefined,+ ports: port ? [port] : undefined,+ })+ }+ }+ return uniqueRules(retained)+}diff --git a/workflow-recipes/src/reconciler/reconciler.test.ts b/workflow-recipes/src/reconciler/reconciler.test.tsindex 5891f489c..b82cb9be2 100644--- a/workflow-recipes/src/reconciler/reconciler.test.ts+++ b/workflow-recipes/src/reconciler/reconciler.test.ts@@ -5,10 +5,11 @@ import { readFileSync } from 'node:fs'import { resolve } from 'node:path'import { EVENFIRE_REGISTRY_PULL_SECRET_NAME } from '@clerum/workflow-runtime-core'import { loadConfig } from '../config'-import { shouldPatchRecipeStatus } from '../k8sClient'+import { WorkflowRecipeWatcher, shouldPatchRecipeStatus } from '../k8sClient'import { CronJobDef, WorkflowRecipeCRD, WorkflowRecipeGfsIntentSpec, WorkloadDef } from '../types'import { INHERITED_PARENT_RESOURCES_ANNOTATION } from '../workflow/childRecipeFactory'import { buildCoordinatorGfsNetworkPolicy } from '../workflow/networkPolicyFactory'+import { captureLogger } from './__tests__/captureLogger'import { defaultFqdnLookup } from './fqdnResolver'import { isRetryableInfraError } from './k8sErrors'import * as brokerIssuer from './oauthBrokerTokenIssuerClient'@@ -303,6 +304,612 @@ describe('WorkflowRecipeReconciler', () => {// ─── Pipeline Tests ─────────────────────────────────────────────+ describe('transport admission pending survives observation and restart', () => {+ const conditionType = 'TransportExternalEgressReady'+ const pendingCondition = {+ type: conditionType,+ status: 'False' as const,+ reason: 'ExternalEgressPending',+ message: 'Transport external egress admission is pending',+ lastTransitionTime: '2026-09-05T00:00:00.000Z',+ }+ function transportRecipe(workflow = false): WorkflowRecipeCRD {+ return makeRecipe({+ metadata: {+ name: 'test-recipe',+ namespace: 'sandbox-recipes',+ uid: 'uid-123',+ generation: 7,+ },+ spec: {+ ...(workflow ? {} : { contextRef: 'context1' }),+ workloads: [+ {+ id: 'calculator',+ type: 'deployment',+ image: 'clerum/mock-stdio-mcp-server:test',+ port: 3000,+ transport: { type: 'stdio' },+ egressBindings: [{ dns: 'current.example.com', port: 443 }],+ },+ ],+ ...(workflow ? { steps: [{ id: 'step', run: snippetRun() }] } : {}),+ },+ status: { phase: 'approved' },+ })+ }+ function hccReadiness() {+ const healthyRead = mockCustomApi.getNamespacedCustomObject.getMockImplementation()!+ const state = { ready: false }+ mockCustomApi.getNamespacedCustomObject.mockImplementation(async request => {+ const response = await healthyRead(request)+ if (request.plural === 'mcpservers') {+ response.metadata.generation = 7+ response.status.conditions = [+ {+ type: 'ExternalEgressReady',+ status: state.ready ? 'True' : 'False',+ observedGeneration: 7,+ reason: state.ready ? 'Reconciled' : 'ExternalEgressReconcileFailed',+ message: 'Policy observation pending',+ },+ ]+ }+ return response+ })+ return state+ }+ function watcherFor(rec: WorkflowRecipeReconciler, recipe?: WorkflowRecipeCRD) {+ const watcher: any = Object.create(WorkflowRecipeWatcher.prototype)+ Object.assign(watcher, {+ recipes: new Map(recipe ? [[recipe.metadata.name, recipe]] : []),+ transientRetries: new Map(),+ stopped: false,+ traceReporter: null,+ dbRunProcessor: null,+ reconciler: rec,+ eventQueue: { enqueue: vi.fn((_key: string, work: () => Promise<void>) => work()) },+ config: loadConfig(),+ })+ return watcher+ }+ function stopWatcher(watcher: any) {+ watcher.stopped = true+ for (const { timer } of watcher.transientRetries.values()) clearTimeout(timer)+ watcher.transientRetries.clear()+ }+ function patchedStatus(previous?: WorkflowRecipeCRD['status']) {+ return {+ ...previous,+ ...mockCustomApi.patchNamespacedCustomObjectStatus.mock.calls.at(-1)![0].body.status,+ }+ }++ it('retries persisted stdio-only admission with backoff after restart and cannot become active from ready pods', async () => {+ const recipe = transportRecipe()+ const hcc = hccReadiness()+ const watcher = watcherFor(reconciler)+ let restarted: any+ vi.useFakeTimers()+ try {+ const firstRun = watcher.handleRecipeEvent('ADDED', recipe)+ await vi.advanceTimersByTimeAsync(31_000)+ await firstRun+ expect(watcher.transientRetries.get(recipe.metadata.name)?.attempts).toBe(1)+ const pendingRecipe = { ...recipe, status: patchedStatus() }+ expect(pendingRecipe.status.phase).toBe('degraded')+ expect(pendingRecipe.status.conditions).toContainEqual(+ expect.objectContaining({ type: conditionType, status: 'False' })+ )+ await watcher.handleRecipeEvent('MODIFIED', pendingRecipe)+ await watcher.refreshWorkloadStatusesForSteadyRecipes()+ const observedStatus = patchedStatus(pendingRecipe.status)+ expect(observedStatus.phase).toBe('degraded')+ expect(observedStatus.conditions).toContainEqual(+ expect.objectContaining({ type: conditionType, status: 'False' })+ )+ stopWatcher(watcher)++ // A new process has no previous retry timer; only persisted condition state remains.+ const restartRecipe = { ...recipe, status: observedStatus }+ const rec = new WorkflowRecipeReconciler(new k8s.KubeConfig())+ restarted = watcherFor(rec, restartRecipe)+ const run = vi.spyOn(rec, 'reconcile')+ const restartPass = restarted.refreshRuntimeCredentialsForInProgressRecipes()+ await vi.advanceTimersByTimeAsync(31_000)+ await restartPass+ expect(run).toHaveBeenCalledTimes(1)+ expect(restarted.transientRetries.get(recipe.metadata.name)?.attempts).toBe(1)+ await restarted.handleRecipeEvent('MODIFIED', { ...recipe, status: patchedStatus() })+ await vi.advanceTimersByTimeAsync(35_000)+ expect(run).toHaveBeenCalledTimes(2)+ expect(restarted.transientRetries.get(recipe.metadata.name)?.attempts).toBe(2)+ expect(patchedStatus().phase).toBe('degraded')+ hcc.ready = true+ await vi.advanceTimersByTimeAsync(10_000)+ expect(run).toHaveBeenCalledTimes(3)+ expect(patchedStatus().phase).toBe('active')+ expect(+ patchedStatus().conditions?.some(+ (condition: { type: string }) => condition.type === conditionType+ )+ ).not.toBe(true)+ expect(restarted.transientRetries.size).toBe(0)+ expect(recipe.metadata.generation).toBe(7)+ } finally {+ stopWatcher(watcher)+ if (restarted) stopWatcher(restarted)+ vi.useRealTimers()+ }+ })++ it('persists workflow admission pending and only starts the workflow after HCC heals', async () => {+ const recipe = transportRecipe(true)+ const hcc = hccReadiness()+ const workflow = vi+ .fn()+ .mockResolvedValue({ phase: 'active', workflowPhase: 'running', message: 'Workflow ready' })+ ;(reconciler as any).workflowReconciler = {+ reconcile: workflow,+ validateWorkflowSpec: () => undefined,+ }+ vi.useFakeTimers()+ try {+ const pending = reconciler.reconcile(recipe)+ await vi.advanceTimersByTimeAsync(31_000)+ const first = await pending+ expect(first.phase, first.message).toBe('degraded')+ expect(first.requeueAfterMs).toBe(TRANSIENT_REQUEUE_BASE_MS)+ expect(workflow).not.toHaveBeenCalled()+ await reconciler.patchStatus(recipe, first)+ const persisted = { ...recipe, status: patchedStatus() }+ expect(persisted.status.conditions).toContainEqual(+ expect.objectContaining({ type: conditionType, status: 'False' })+ )+ expect(persisted.status.workflowExecution?.phase).not.toBe('failed')+ hcc.ready = true+ const result = await reconciler.reconcile(persisted)+ expect(result.phase).toBe('active')+ expect(workflow).toHaveBeenCalledTimes(1)+ await reconciler.patchStatus(persisted, result)+ expect(+ patchedStatus().conditions?.some(+ (condition: { type: string }) => condition.type === conditionType+ )+ ).not.toBe(true)+ } finally {+ vi.useRealTimers()+ }+ })++ it('preserves pending admission through an earlier validation failure and clears it after transport removal', async () => {+ const recipe = transportRecipe()+ recipe.status = { phase: 'degraded', conditions: [pendingCondition] }+ recipe.spec.workloads![0].egressBindings![0].port = 0+ const failed = await reconciler.reconcile(recipe)+ expect(failed.phase).toBe('failed')+ await reconciler.patchStatus(recipe, failed)+ expect(patchedStatus(recipe.status).conditions).toContainEqual(pendingCondition)+ recipe.spec.workloads![0].egressBindings = []+ delete recipe.spec.workloads![0].transport+ const removed = await reconciler.reconcile(recipe)+ expect(removed.phase).toBe('active')+ await reconciler.patchStatus(recipe, removed)+ expect(+ patchedStatus(recipe.status).conditions?.some(+ (condition: { type: string }) => condition.type === conditionType+ )+ ).not.toBe(true)+ })++ it('preserves ordinary Pod-only downgrade and recovery when no admission is pending', async () => {+ const recipe = makeRecipe({ status: { phase: 'degraded' } })+ mockAppsApi.readNamespacedDeployment.mockResolvedValueOnce({+ spec: { replicas: 1 },+ status: { readyReplicas: 0, updatedReplicas: 0, availableReplicas: 0 },+ })+ expect((await reconciler.observeCurrentWorkloadStatus(recipe)).phase).toBe('degraded')+ expect((await reconciler.observeCurrentWorkloadStatus(recipe)).phase).toBe('active')+ })+ })++ describe('#567 regression: policy contraction', () => {+ function policyStore(policies: k8s.V1NetworkPolicy[]) {+ const live = new Map<string, k8s.V1NetworkPolicy>()+ let version = 0+ const persist = (body: k8s.V1NetworkPolicy) => {+ const policy = structuredClone(body)+ const name = policy.metadata!.name!+ policy.metadata = {+ ...policy.metadata,+ uid: `uid-${name}`,+ resourceVersion: String(++version),+ }+ // The API's omitempty serialization omits empty top-level rule lists.+ if (policy.spec?.egress?.length === 0) delete policy.spec.egress+ if (policy.spec?.ingress?.length === 0) delete policy.spec.ingress+ live.set(name, policy)+ return structuredClone(policy)+ }+ for (const policy of policies) persist(policy)+ mockNetworkingApi.readNamespacedNetworkPolicy.mockImplementation(async ({ name }) => {+ if (!live.has(name)) throw { code: 404 }+ return structuredClone(live.get(name))+ })+ mockNetworkingApi.createNamespacedNetworkPolicy.mockImplementation(async ({ body }) => {+ if (live.has(body.metadata.name)) throw { code: 409 }+ return persist(body)+ })+ mockNetworkingApi.replaceNamespacedNetworkPolicy.mockImplementation(+ async ({ name, body }) => {+ expect(body.metadata.resourceVersion).toBe(live.get(name)?.metadata?.resourceVersion)+ return persist(body)+ }+ )+ mockNetworkingApi.deleteNamespacedNetworkPolicy.mockImplementation(async ({ name, body }) => {+ if (body?.preconditions) {+ expect(body.preconditions).toEqual({+ uid: live.get(name)?.metadata?.uid,+ resourceVersion: live.get(name)?.metadata?.resourceVersion,+ })+ }+ live.delete(name)+ return {}+ })+ return live+ }++ function uiRecipe(ports = [8080, 9090]) {+ return makeRecipe({+ spec: {+ workloads: [+ { id: 'ui', type: 'deployment', image: 'ui:1', port: 8080 },+ { id: 'backend', type: 'deployment', image: 'backend:1', port: 8080 },+ ],+ ui: {+ workloadRef: 'ui',+ port: 8080,+ egress: {+ internal: ports.map(port => ({ workloadRef: 'backend', port })),+ external: [{ fqdn: 'current.example.com', port: 443 }],+ },+ },+ },+ })+ }++ it.each([[8080], [8080, 9090, 9443], [9090, 8080]])(+ 'preserves the intersection of grouped UI ports during a DNS fault: %j',+ async (...ports) => {+ const rb = await import('./resourceBuilder')+ const recipe = uiRecipe()+ const name = 'ui-ingress-test-recipe-backend'+ const live = policyStore([+ rb.buildUiIngressNetworkPolicy(+ recipe,+ 'backend',+ [8080, 9090],+ 'sandbox-recipes',+ 'sandbox-ui'+ )!,+ ])+ recipe.spec.ui!.egress!.internal = ports.map(port => ({ workloadRef: 'backend', port }))+ const rec = new WorkflowRecipeReconciler(new k8s.KubeConfig(), undefined, {+ fqdnLookup: async () => {+ throw new Error('resolver controller fault')+ },+ })+ const result = await rec.reconcile({ ...recipe, status: { phase: 'active' } })+ expect(result.phase).toBe('failed')+ const retained = (live.get(name)?.spec?.ingress ?? [])+ .flatMap(rule => rule.ports ?? [])+ .map(port => port.port)+ expect(retained.sort()).toEqual(ports.filter(port => port === 8080 || port === 9090).sort())+ }+ )++ it.each([false, true])(+ 'accepts empty UI readback before external replacement=%s',+ async replacement => {+ const rb = await import('./resourceBuilder')+ const recipe = uiRecipe([])+ const name = 'ui-egress-test-recipe'+ const live = policyStore([+ rb.buildUiEgressNetworkPolicy(+ recipe,+ 'sandbox-ui',+ 'sandbox-recipes',+ [+ {+ cidr: '93.184.216.20/32',+ port: 443,+ source: { kind: 'fqdn', fqdn: 'current.example.com' },+ },+ ],+ egressStateAnnotations([+ { fqdn: 'current.example.com', ip: '93.184.216.20', port: 443 },+ ])+ )!,+ ])+ recipe.spec.ui!.egress!.external = replacement+ ? [{ fqdn: 'new.example.com', port: 443 }]+ : []+ const lookup = vi.fn(async () => ({+ kind: 'ok' as const,+ ipv4: ['93.184.216.21'],+ ipv6: [],+ ttlSeconds: 300,+ }))+ const rec = new WorkflowRecipeReconciler(new k8s.KubeConfig(), undefined, {+ fqdnLookup: lookup,+ })+ const result = await rec.reconcile({ ...recipe, status: { phase: 'active' } })+ expect(result.phase).toBe('active')+ expect(live.has(name)).toBe(true)+ expect(lookup).toHaveBeenCalledTimes(replacement ? 1 : 0)+ expect(mockNetworkingApi.deleteNamespacedNetworkPolicy).not.toHaveBeenCalledWith(+ expect.objectContaining({ name })+ )+ }+ )++ it.each([+ { mode: 'broader-rules', phase: 'failed', deleted: true },+ { mode: 'annotation-only', phase: 'failed', deleted: true },+ { mode: 'replacement-uid', phase: 'degraded', deleted: false },+ { mode: 'foreign-owner', phase: 'failed', deleted: false },+ { mode: 'missing-uid', phase: 'failed', deleted: false },+ { mode: 'wrong-namespace', phase: 'failed', deleted: false },+ ] as const)('rejects unsafe readback: $mode', async ({ mode, phase, deleted }) => {+ const rb = await import('./resourceBuilder')+ const recipe = uiRecipe([])+ const name = 'ui-egress-test-recipe'+ const live = policyStore([+ rb.buildUiEgressNetworkPolicy(+ recipe,+ 'sandbox-ui',+ 'sandbox-recipes',+ [+ {+ cidr: '93.184.216.20/32',+ port: 443,+ source: { kind: 'fqdn', fqdn: 'current.example.com' },+ },+ ],+ egressStateAnnotations([{ fqdn: 'current.example.com', ip: '93.184.216.20', port: 443 }])+ )!,+ ])+ recipe.spec.ui!.egress!.external = []+ const replace = mockNetworkingApi.replaceNamespacedNetworkPolicy.getMockImplementation()!+ mockNetworkingApi.replaceNamespacedNetworkPolicy.mockImplementation(async request => {+ const result = await replace(request)+ if (request.name === name) {+ const observed = live.get(name)!+ if (mode === 'broader-rules')+ observed.spec!.egress = [+ {+ to: [{ ipBlock: { cidr: '8.8.8.8/32' } }],+ ports: [{ port: 443, protocol: 'TCP' }],+ },+ ]+ if (mode === 'annotation-only')+ observed.metadata!.annotations = egressStateAnnotations([+ { fqdn: 'removed.example.com', ip: '8.8.8.8', port: 443 },+ ])+ if (mode === 'replacement-uid') observed.metadata!.uid = 'uid-new-owned-object'+ if (mode === 'missing-uid') delete observed.metadata!.uid+ if (mode === 'wrong-namespace') observed.metadata!.namespace = 'another-namespace'+ if (mode === 'foreign-owner')+ observed.metadata!.labels!['clerum.io/managed-by'] = 'another-controller'+ }+ return result+ })+ const rec = new WorkflowRecipeReconciler(new k8s.KubeConfig(), undefined, {+ fqdnLookup: async () => {+ throw new Error('no external lookup is declared')+ },+ })+ const result = await rec.reconcile({ ...recipe, status: { phase: 'active' } })+ expect(result.phase).toBe(phase)+ expect(live.has(name)).toBe(!deleted)+ if (!deleted) expect(mockNetworkingApi.deleteNamespacedNetworkPolicy).not.toHaveBeenCalled()+ expect(result.requeueAfterMs).toBe(+ mode === 'replacement-uid' ? TRANSIENT_REQUEUE_BASE_MS : undefined+ )+ })++ it.each(['ui', 'workload'] as const)(+ 'replans additive %s conflicts and disappearance without latching a healthy recipe',+ async lane => {+ for (const failure of ['replace-conflict', 'read-disappearance', 'replace-disappearance']) {+ const recipe =+ lane === 'ui'+ ? uiRecipe([])+ : makeRecipe({+ spec: {+ workloads: [+ {+ id: 'worker',+ type: 'deployment',+ image: 'worker:1',+ port: 8080,+ egressBindings: [{ dns: 'current.example.com', port: 443 }],+ },+ ],+ },+ })+ const name = lane === 'ui' ? 'ui-egress-test-recipe' : 'wl-egress-test-recipe-worker'+ const live = policyStore([])+ let address = '93.184.216.20'+ const rec = new WorkflowRecipeReconciler(new k8s.KubeConfig(), undefined, {+ fqdnLookup: async () => ({ kind: 'ok', ipv4: [address], ipv6: [], ttlSeconds: 300 }),+ })+ expect((await rec.reconcile({ ...recipe, status: { phase: 'active' } })).phase).toBe(+ 'active'+ )+ const healthyReplace =+ mockNetworkingApi.replaceNamespacedNetworkPolicy.getMockImplementation()!+ const healthyCreate =+ mockNetworkingApi.createNamespacedNetworkPolicy.getMockImplementation()!+ address = '93.184.216.21'+ if (failure === 'read-disappearance') {+ mockNetworkingApi.createNamespacedNetworkPolicy.mockImplementation(async request => {+ if (request.body.metadata.name === name) {+ live.delete(name)+ throw Object.assign(new Error('object existed before replacement read'), {+ code: 409,+ })+ }+ return healthyCreate(request)+ })+ } else {+ mockNetworkingApi.replaceNamespacedNetworkPolicy.mockImplementation(async request => {+ if (request.name === name && JSON.stringify(request.body.spec).includes(address)) {+ if (failure === 'replace-disappearance') live.delete(name)+ throw Object.assign(new Error('object snapshot changed'), {+ code: failure === 'replace-conflict' ? 409 : 404,+ })+ }+ return healthyReplace(request)+ })+ }+ const first = await rec.reconcile({ ...recipe, status: { phase: 'active' } })+ expect(first.phase, `${lane}/${failure}`).toBe('degraded')+ expect(first.requeueAfterMs).toBe(TRANSIENT_REQUEUE_BASE_MS)+ await rec.patchStatus(recipe, first)+ const persisted =+ mockCustomApi.patchNamespacedCustomObjectStatus.mock.calls.at(-1)![0].body.status+ mockNetworkingApi.createNamespacedNetworkPolicy.mockImplementation(healthyCreate)+ mockNetworkingApi.replaceNamespacedNetworkPolicy.mockImplementation(healthyReplace)+ const recovered = await rec.reconcile({ ...recipe, status: persisted })+ expect(recovered.phase).toBe('active')+ expect(JSON.stringify(live.get(name)?.spec)).toContain(address)+ }+ }+ )++ it.each(['delete', 'replace'] as const)(+ 'retries a fenced %s conflict after the status roundtrip',+ async operation => {+ const rb = await import('./resourceBuilder')+ const recipe = makeRecipe({+ spec: {+ workloads: [+ {+ id: 'worker',+ type: 'deployment',+ image: 'worker:1',+ port: 8080,+ egressBindings: [+ { dns: 'current.example.com', port: 443 },+ { dns: 'removed.example.com', port: 443 },+ ],+ },+ {+ id: 'keeper',+ type: 'deployment',+ image: 'keeper:1',+ port: 8080,+ egressBindings: [{ dns: 'keeper.example.com', port: 443 }],+ },+ ],+ },+ })+ const name = 'wl-egress-test-recipe-worker'+ const entries = [+ { fqdn: 'current.example.com', ip: '93.184.216.20', port: 443 },+ { fqdn: 'removed.example.com', ip: '93.184.216.21', port: 443 },+ ]+ const live = policyStore([+ rb.buildWorkloadEgressNetworkPolicy(+ recipe.spec.workloads![0],+ recipe,+ 'sandbox-recipes',+ entries.map(entry => ({+ cidr: `${entry.ip}/32`,+ port: entry.port,+ source: { kind: 'fqdn', fqdn: entry.fqdn },+ })),+ egressStateAnnotations(entries)+ )!,+ ])+ recipe.spec.workloads![0].egressBindings =+ operation === 'delete' ? [] : [{ dns: 'current.example.com', port: 443 }]+ const mutation =+ operation === 'delete'+ ? mockNetworkingApi.deleteNamespacedNetworkPolicy+ : mockNetworkingApi.replaceNamespacedNetworkPolicy+ const healthyMutation = mutation.getMockImplementation()!+ mutation.mockImplementationOnce(async () => {+ throw Object.assign(new Error('object modified'), { code: 409 })+ })+ const rec = new WorkflowRecipeReconciler(new k8s.KubeConfig(), undefined, {+ fqdnLookup: async () => ({+ kind: 'ok',+ ipv4: ['93.184.216.20'],+ ipv6: [],+ ttlSeconds: 300,+ }),+ })+ type Watcher = {+ recipes: Map<string, WorkflowRecipeCRD>+ transientRetries: Map<string, { timer: ReturnType<typeof setTimeout>; attempts: number }>+ stopped: boolean+ traceReporter: null+ dbRunProcessor: null+ reconciler: WorkflowRecipeReconciler+ eventQueue: { enqueue: (key: string, task: () => Promise<void>) => Promise<void> }+ handleRecipeEvent: (type: string, recipe: WorkflowRecipeCRD) => Promise<void>+ }+ const watcher = Object.create(WorkflowRecipeWatcher.prototype) as Watcher+ Object.assign(watcher, {+ recipes: new Map(),+ transientRetries: new Map(),+ stopped: false,+ traceReporter: null,+ dbRunProcessor: null,+ reconciler: rec,+ eventQueue: { enqueue: vi.fn((_key: string, task: () => Promise<void>) => task()) },+ })+ const reconcile = vi.spyOn(rec, 'reconcile')+ const active = {+ ...recipe,+ metadata: { ...recipe.metadata, generation: 1 },+ status: { phase: 'active' as const },+ }+ vi.useFakeTimers()+ try {+ await watcher.handleRecipeEvent('ADDED', active)+ const first = await reconcile.mock.results[0].value+ expect(first.phase).toBe('degraded')+ expect(first.requeueAfterMs).toBe(TRANSIENT_REQUEUE_BASE_MS)+ const patched =+ mockCustomApi.patchNamespacedCustomObjectStatus.mock.calls.at(-1)![0].body.status+ // Deliver the real status-only watch event without changing generation.+ await watcher.handleRecipeEvent('MODIFIED', { ...active, status: patched })+ expect(reconcile).toHaveBeenCalledTimes(1)+ expect(watcher.transientRetries.get(recipe.metadata.name)?.attempts).toBe(1)+ mutation.mockImplementationOnce(async () => {+ throw Object.assign(new Error('object modified again'), { code: 409 })+ })+ await vi.advanceTimersByTimeAsync(TRANSIENT_REQUEUE_BASE_MS)+ expect(reconcile).toHaveBeenCalledTimes(2)+ expect(watcher.transientRetries.get(recipe.metadata.name)?.attempts).toBe(2)+ mutation.mockImplementation(healthyMutation)+ await vi.advanceTimersByTimeAsync(TRANSIENT_REQUEUE_BASE_MS * 2)+ const next = await reconcile.mock.results[2].value+ expect(next.phase).toBe('active')+ expect(watcher.transientRetries.size).toBe(0)+ expect(JSON.stringify(live.get(name)?.spec ?? {})).not.toContain('93.184.216.21')+ expect(live.has(name)).toBe(operation === 'replace')+ } finally {+ for (const { timer } of watcher.transientRetries.values()) clearTimeout(timer)+ vi.useRealTimers()+ }+ }+ )+ })+it('processes full pipeline successfully (3.11a)', async () => {const result = await reconciler.reconcile(makeRecipe())expect(result.phase).toBe('active')@@ -2940,9 +3547,9 @@ describe('WorkflowRecipeReconciler', () => {{resolverLabel: 'controller fault',resolverResult: 'throw' as const,- label: 'non-retryable identity conflict',+ label: 'non-retryable API access denial',contractionError: () =>- Object.assign(new Error('workload re-contraction conflict'), { code: 409 }),+ Object.assign(new Error('workload re-contraction access denied'), { code: 403 }),expectedPhase: 'failed' as const,retryable: false,},@@ -2958,9 +3565,9 @@ describe('WorkflowRecipeReconciler', () => {{resolverLabel: 'transient DNS result',resolverResult: 'transient' as const,- label: 'non-retryable identity conflict',+ label: 'non-retryable API access denial',contractionError: () =>- Object.assign(new Error('workload re-contraction conflict'), { code: 409 }),+ Object.assign(new Error('workload re-contraction access denied'), { code: 403 }),expectedPhase: 'failed' as const,retryable: false,},@@ -3030,7 +3637,7 @@ describe('WorkflowRecipeReconciler', () => {expect(result.requeueAfterMs).toBe(TRANSIENT_REQUEUE_BASE_MS)} else {expect(result.skipStatusPatch).not.toBe(true)- expect(result.message).toContain('workload re-contraction conflict')+ expect(result.message).toContain('workload re-contraction access denied')}expect(mockNetworkingApi.createNamespacedNetworkPolicy).not.toHaveBeenCalledWith(expect.objectContaining({@@ -4799,8 +5406,9 @@ describe('WorkflowRecipeReconciler', () => {{resolverLabel: 'controller fault',resolverResult: 'throw' as const,- label: 'non-retryable identity conflict',- contractionError: () => Object.assign(new Error('UI re-contraction conflict'), { code: 409 }),+ label: 'non-retryable API access denial',+ contractionError: () =>+ Object.assign(new Error('UI re-contraction access denied'), { code: 403 }),expectedPhase: 'failed' as const,retryable: false,},@@ -4816,8 +5424,9 @@ describe('WorkflowRecipeReconciler', () => {{resolverLabel: 'transient DNS result',resolverResult: 'transient' as const,- label: 'non-retryable identity conflict',- contractionError: () => Object.assign(new Error('UI re-contraction conflict'), { code: 409 }),+ label: 'non-retryable API access denial',+ contractionError: () =>+ Object.assign(new Error('UI re-contraction access denied'), { code: 403 }),expectedPhase: 'failed' as const,retryable: false,},@@ -4873,7 +5482,7 @@ describe('WorkflowRecipeReconciler', () => {expect(result.requeueAfterMs).toBe(TRANSIENT_REQUEUE_BASE_MS)} else {expect(result.skipStatusPatch).not.toBe(true)- expect(result.message).toContain('UI re-contraction conflict')+ expect(result.message).toContain('UI re-contraction access denied')}expect(mockNetworkingApi.createNamespacedNetworkPolicy).not.toHaveBeenCalledWith(expect.objectContaining({@@ -5317,7 +5926,6 @@ describe('WorkflowRecipeReconciler', () => {)const cap = captureEgressRecords()- const noOpSpy = vi.spyOn(console, 'log').mockImplementation(() => {})try {mockNetworkingApi.createNamespacedNetworkPolicy.mockClear()mockNetworkingApi.replaceNamespacedNetworkPolicy.mockClear()@@ -5331,14 +5939,9 @@ describe('WorkflowRecipeReconciler', () => {mockNetworkingApi.createNamespacedNetworkPolicy.mock.calls.length +mockNetworkingApi.replaceNamespacedNetworkPolicy.mock.calls.lengthexpect(wrote).toBeGreaterThan(0)- expect(noOpSpy).not.toHaveBeenCalledWith(- expect.stringContaining('ui-egress-test-recipe" in sandbox-ui egress set unchanged — no-op')- )-// ...and no record, because the resolved set did not change.expect(cap.payloads()).toHaveLength(0)} finally {- noOpSpy.mockRestore()cap.restore()}})@@ -11174,7 +11777,7 @@ describe('WorkflowRecipeReconciler', () => {vi.spyOn(brokerIssuer, 'issueOAuthBrokerToken').mockRejectedValue(new Error('OAuth broker token issuance timed out after 10000ms for recipe "test-recipe"'))- const warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => {})+ const warnSpy = captureLogger('warn')const result = await reconciler.reconcile(recipeWithBackgroundAccess())@@ -11182,10 +11785,12 @@ describe('WorkflowRecipeReconciler', () => {expect(result.phase).toBe('active')// Warning was emitted so the operator can see what happened.expect(warnSpy).toHaveBeenCalledWith(- expect.stringContaining('Broker-token issuance failed during reconcile')+ expect.stringContaining('Broker-token issuance failed during reconcile'),+ expect.objectContaining({ err: expect.any(Error) }))// Workloads still deployed.expect(mockAppsApi.createNamespacedDeployment).toHaveBeenCalledTimes(1)+ warnSpy.mockRestore()})})diff --git a/workflow-recipes/src/reconciler/workflowRecipeReconciler.ts b/workflow-recipes/src/reconciler/workflowRecipeReconciler.tsindex 124843d72..f6f7e0dbd 100644--- a/workflow-recipes/src/reconciler/workflowRecipeReconciler.ts+++ b/workflow-recipes/src/reconciler/workflowRecipeReconciler.ts@@ -99,6 +99,11 @@ import {waitForExternalEgressReady,waitForNetworkReady,} from './mcpDelegation'+import {+ effectivePolicyTypes,+ intersectNetworkPolicyRules,+ networkPolicySpecSignature,+} from './networkPolicyContraction'import { issueOAuthBrokerToken } from './oauthBrokerTokenIssuerClient'import {PLUGIN_WORKLOAD_SDK_CONDITION_TYPE,@@ -137,6 +142,36 @@ const PARENT_RECIPE_LABEL = 'clerum.io/parent-recipe'const WORKFLOW_RUN_ID_LABEL = 'clerum.io/workflow-run-id'const WORKFLOW_ACTOR_ID_LABEL = 'clerum.io/workflow-actor-id'const WORKFLOW_ACTOR_TYPE_LABEL = 'clerum.io/workflow-actor-type'+export const TRANSPORT_NETWORK_CONDITION_TYPE = 'TransportExternalEgressReady'+const TRANSPORT_NETWORK_CONDITION_TYPES = new Set([TRANSPORT_NETWORK_CONDITION_TYPE])++export function hasPendingTransportNetworkReadiness(recipe: WorkflowRecipeCRD): boolean {+ return (+ recipe.status?.conditions?.some(+ condition =>+ condition.type === TRANSPORT_NETWORK_CONDITION_TYPE && condition.status !== 'True'+ ) ?? false+ )+}++function pendingTransportNetworkConditions(+ recipe: WorkflowRecipeCRD,+ message: string+): StatusCondition[] {+ const previous = recipe.status?.conditions?.find(+ condition => condition.type === TRANSPORT_NETWORK_CONDITION_TYPE+ )+ return [+ {+ type: TRANSPORT_NETWORK_CONDITION_TYPE,+ status: 'False',+ reason: 'ExternalEgressPending',+ message,+ lastTransitionTime:+ previous?.status === 'False' ? previous.lastTransitionTime : new Date().toISOString(),+ },+ ]+}function externalEgressReadinessError(recipeName: string,@@ -148,7 +183,9 @@ function externalEgressReadinessError(`External egress policy readiness not achieved for WorkflowRecipe "${recipeName}". ` +`${[...pending, ...failed].join('; ')}. ` +`External egress bindings are not enforceable until HCC reports ExternalEgressReady=True.`- return result.failed.length === 0 ? new RetryableReconcileError(message) : new Error(message)+ return result.failed.length === 0+ ? new TransportNetworkReadinessPendingError(message)+ : new Error(message)}function clusterNetworkPolicyEnforcementError(recipeName: string): Error {@@ -482,6 +519,25 @@ class NetworkPolicyOwnershipConflictError extends Error {}}+/** A fenced mutation lost its snapshot; only a fresh reconciliation may retry. */+class NetworkPolicyReplanRequiredError extends RetryableReconcileError {+ override name = 'NetworkPolicyReplanRequiredError'+}++class TransportNetworkReadinessPendingError extends RetryableReconcileError {+ override name = 'TransportNetworkReadinessPendingError'+}++function throwNetworkPolicyMutationError(error: unknown, name: string, namespace: string): never {+ if (getErrorCode(error) === 409 || getErrorCode(error) === 404) {+ throw new NetworkPolicyReplanRequiredError(+ `NetworkPolicy "${name}" in ${namespace} changed during a fenced mutation; a fresh reconciliation is required`,+ { cause: error }+ )+ }+ throw error+}+class InternalDependencyReconcileError extends Error {constructor(message: string,@@ -541,6 +597,8 @@ export interface ReconcileResult {* UI/API-consumable reasons that should not be inferred from free-form messages.*/workloadConditions?: StatusCondition[]+ /** Undefined preserves admission state; [] clears it only after a full admission check. */+ transportNetworkConditions?: StatusCondition[]/** SDK-only eager-host provider health, kept separate from workflow phase. */pluginWorkloadSdkProviderUnavailable?: boolean/** SDK host identity is ready, but an operator prompt policy is not active yet. */@@ -730,10 +788,6 @@ export function egressSignature(policy: k8s.V1NetworkPolicy): string {)}-function networkPolicySpecSignature(policy: k8s.V1NetworkPolicy): string {- return JSON.stringify(canonicalize(policy.spec ?? {}))-}-function contractionWriteNeeded(existing: k8s.V1NetworkPolicy,desired: k8s.V1NetworkPolicy@@ -746,21 +800,6 @@ function contractionWriteNeeded()}-function intersectRules<T>(existing: T[] | undefined, desired: T[] | undefined): T[] {- const available = new Map<string, number>()- for (const rule of existing ?? []) {- const signature = JSON.stringify(canonicalize(rule))- available.set(signature, (available.get(signature) ?? 0) + 1)- }- return (desired ?? []).filter(rule => {- const signature = JSON.stringify(canonicalize(rule))- const remaining = available.get(signature) ?? 0- if (remaining === 0) return false- available.set(signature, remaining - 1)- return true- })-}-/*** Intersect a desired policy with the rules already enforced by the live* snapshot. Used only by the pre-DNS contraction phase: it may remove rules but@@ -774,16 +813,24 @@ function subtractiveNetworkPolicy(if (!desired) return nullconst existingSelector = JSON.stringify(canonicalize(existing.spec?.podSelector ?? {}))const desiredSelector = JSON.stringify(canonicalize(desired.spec?.podSelector ?? {}))- const existingTypes = JSON.stringify([...(existing.spec?.policyTypes ?? [])].sort())- const desiredTypes = JSON.stringify([...(desired.spec?.policyTypes ?? [])].sort())+ const existingTypes = JSON.stringify(effectivePolicyTypes(existing.spec))+ const desiredTypes = JSON.stringify(effectivePolicyTypes(desired.spec))if (existingSelector !== desiredSelector || existingTypes !== desiredTypes) return nullconst contractedSpec: k8s.V1NetworkPolicySpec = { ...desired.spec! }if (desired.spec?.egress !== undefined) {- contractedSpec.egress = intersectRules(existing.spec?.egress, desired.spec.egress)+ contractedSpec.egress = intersectNetworkPolicyRules(+ existing.spec?.egress,+ desired.spec.egress,+ 'to'+ )}if (desired.spec?.ingress !== undefined) {- contractedSpec.ingress = intersectRules(existing.spec?.ingress, desired.spec.ingress)+ contractedSpec.ingress = intersectNetworkPolicyRules(+ existing.spec?.ingress,+ desired.spec.ingress,+ '_from'+ )}const contracted: k8s.V1NetworkPolicy = { ...desired, spec: contractedSpec }@@ -910,9 +957,11 @@ export class WorkflowRecipeReconciler {if (controlApiPem) {const { initializeControlApiPublicKey } = await import('../workflow/restEndpoints')await initializeControlApiPublicKey(controlApiPem)- console.log('[WR-Reconciler] control-api JWT public key loaded — admin delegation enabled')+ createLogger('wrc', 'workflow-recipes').info(+ '[WR-Reconciler] control-api JWT public key loaded — admin delegation enabled'+ )} else {- console.warn(+ createLogger('wrc', 'workflow-recipes').warn('[WR-Reconciler] CONTROL_API_PUBLIC_KEY_PEM not set — admin artifact delegation will return 401')}@@ -972,7 +1021,7 @@ export class WorkflowRecipeReconciler {pluginWorkloadSdkRevocationClient: new HttpPluginWorkloadSdkRevocationClient(),}this.workflowReconciler = new WorkflowReconciler(deps)- console.log('[WR-Reconciler] Workflow subsystem initialized')+ createLogger('wrc', 'workflow-recipes').info('[WR-Reconciler] Workflow subsystem initialized')}private get delegationDeps(): DelegationDeps {@@ -1036,9 +1085,9 @@ export class WorkflowRecipeReconciler {namespace)if (!ready) {- console.warn(- `[WR-Reconciler] Generic network readiness not confirmed for: ${pending.join(', ')}`- )+ createLogger('wrc', recipe.metadata.name).warn('Generic network readiness not confirmed', {+ value1: pending.join(', '),+ })}}}@@ -1054,8 +1103,9 @@ export class WorkflowRecipeReconciler {throw clusterNetworkPolicyEnforcementError(recipe.metadata.name)}if (this.config.networkPolicyEnforcementMode === 'warn') {- console.warn(- `[WR-Reconciler] NetworkPolicy enforcement mode is "warn" for recipe "${recipe.metadata.name}". ${policyReadiness}, but packet-level cluster enforcement must still be validated before treating this as a security gate.`+ createLogger('wrc', recipe.metadata.name).warn(+ 'NetworkPolicy enforcement is in warn mode; packet-level enforcement still requires validation',+ { name: recipe.metadata.name, policyReadiness })}}@@ -1082,8 +1132,9 @@ export class WorkflowRecipeReconciler {parentLabel !== ownerRecipeName ||recipe.metadata.annotations?.[INHERITED_PARENT_RESOURCES_ANNOTATION] !== 'true') {- console.warn(- `[WR-Reconciler] Ignoring controller ownerReference "${ownerRecipeName}" on workflow "${recipe.metadata.name}" because normal DB-run inheritance metadata is incomplete or inconsistent`+ createLogger('wrc', recipe.metadata.name).warn(+ 'Ignoring controller ownerReference because DB-run inheritance metadata is inconsistent',+ { ownerRecipeName, name: recipe.metadata.name })return recipe.metadata.name}@@ -1095,8 +1146,9 @@ export class WorkflowRecipeReconciler {)if (!verifiedOwnerRecipeName) {if (parentLabel) {- console.warn(- `[WR-Reconciler] Ignoring ${PARENT_RECIPE_LABEL}="${parentLabel}" on workflow "${recipe.metadata.name}" because controller ownerReference "${ownerRecipeName}" could not be verified`+ createLogger('wrc', recipe.metadata.name).warn(+ 'Ignoring parent-recipe label because controller ownership could not be verified',+ { PARENT_RECIPE_LABEL, parentLabel, name: recipe.metadata.name, ownerRecipeName })}return recipe.metadata.name@@ -1116,16 +1168,17 @@ export class WorkflowRecipeReconciler {)}if (provenance === 'invalid') {- console.warn(- `[WR-Reconciler] Ignoring controller ownerReference "${ownerRecipeName}" on workflow "${recipe.metadata.name}" because workflow_runs does not bind run "${workflowRunId}" to this exact parent and child`+ createLogger('wrc', recipe.metadata.name).warn(+ 'Ignoring controller ownerReference because workflow_runs does not bind the exact parent and child',+ { ownerRecipeName, name: recipe.metadata.name, workflowRunId })return recipe.metadata.name}} catch (error) {if (error instanceof RuntimeScopeResolutionPendingError) throw error- console.warn(- `[WR-Reconciler] Deferring runtime scope resolution for workflow "${recipe.metadata.name}" because DB-run provenance is temporarily unavailable:`,- error+ createLogger('wrc', recipe.metadata.name).warn(+ 'Deferring runtime scope resolution because DB-run provenance is unavailable',+ { name: recipe.metadata.name, err: error })throw new RuntimeScopeResolutionPendingError(`DB-run provenance is temporarily unavailable for workflow "${recipe.metadata.name}"`,@@ -1137,8 +1190,9 @@ export class WorkflowRecipeReconciler {}if (parentLabel) {- console.warn(- `[WR-Reconciler] Ignoring ${PARENT_RECIPE_LABEL}="${parentLabel}" on workflow "${recipe.metadata.name}" because no controller WorkflowRecipe ownerReference is present`+ createLogger('wrc', recipe.metadata.name).warn(+ 'Ignoring parent-recipe label without a controller WorkflowRecipe ownerReference',+ { PARENT_RECIPE_LABEL, parentLabel, name: recipe.metadata.name })}return recipe.metadata.name@@ -1253,14 +1307,16 @@ export class WorkflowRecipeReconciler {const liveOwnerUid = liveOwner.metadata?.uidif (liveOwner.metadata?.deletionTimestamp) {- console.warn(- `[WR-Reconciler] Ignoring controller ownerReference "${ownerRecipeName}" on workflow "${recipe.metadata.name}" because the owner is deleting`+ createLogger('wrc', recipe.metadata.name).warn(+ 'Ignoring controller ownerReference because the owner is deleting',+ { ownerRecipeName, name: recipe.metadata.name })return null}if (liveOwnerUid !== ownerRecipeUid) {- console.warn(- `[WR-Reconciler] Ignoring controller ownerReference "${ownerRecipeName}" on workflow "${recipe.metadata.name}" because owner UID did not match the live WorkflowRecipe`+ createLogger('wrc', recipe.metadata.name).warn(+ 'Ignoring controller ownerReference because the live owner UID differs',+ { ownerRecipeName, name: recipe.metadata.name })return null}@@ -1268,14 +1324,15 @@ export class WorkflowRecipeReconciler {} catch (error) {const code = getErrorCode(error)if (code === 404) {- console.warn(- `[WR-Reconciler] Ignoring controller ownerReference "${ownerRecipeName}" on workflow "${recipe.metadata.name}" because the owner WorkflowRecipe was not found`+ createLogger('wrc', recipe.metadata.name).warn(+ 'Ignoring controller ownerReference because the owner was not found',+ { ownerRecipeName, name: recipe.metadata.name })return null}- console.warn(- `[WR-Reconciler] Deferring runtime scope resolution for workflow "${recipe.metadata.name}" because the owner WorkflowRecipe is temporarily unavailable:`,- error+ createLogger('wrc', recipe.metadata.name).warn(+ 'Deferring runtime scope resolution because the owner is unavailable',+ { name: recipe.metadata.name, err: error })throw new RuntimeScopeResolutionPendingError(`Owner WorkflowRecipe "${ownerRecipeName}" is temporarily unavailable`,@@ -1308,9 +1365,9 @@ export class WorkflowRecipeReconciler {runtimeScopeRecipeName)} catch (error) {- console.error(- `[WR-Reconciler] Failed to repair coordinator runtime credentials for steady-state workflow "${recipe.metadata.name}"; continuing without phase change:`,- error+ createLogger('wrc', recipe.metadata.name).error(+ 'Failed to repair coordinator runtime credentials; continuing without phase change',+ { name: recipe.metadata.name, err: error })}try {@@ -1322,9 +1379,9 @@ export class WorkflowRecipeReconciler {recipe.metadata.uid)} catch (error) {- console.error(- `[WR-Reconciler] Failed to repair mcpHost runtime credentials for steady-state workflow "${recipe.metadata.name}"; continuing without phase change:`,- error+ createLogger('wrc', recipe.metadata.name).error(+ 'Failed to repair mcpHost runtime credentials; continuing without phase change',+ { name: recipe.metadata.name, err: error })}try {@@ -1336,9 +1393,9 @@ export class WorkflowRecipeReconciler {runtimeScopeRecipeName)} catch (error) {- console.error(- `[WR-Reconciler] Failed to refresh runtime HTTP egress for steady-state workflow "${recipe.metadata.name}"; keeping last valid NetworkPolicy:`,- error+ createLogger('wrc', recipe.metadata.name).error(+ 'Failed to refresh runtime HTTP egress; keeping the last valid NetworkPolicy',+ { name: recipe.metadata.name, err: error })}}@@ -1357,8 +1414,9 @@ export class WorkflowRecipeReconciler {}private staleRecipeResult(recipe: WorkflowRecipeCRD, stage: string): ReconcileResult {- console.warn(- `[WR-Reconciler] Skipping stale reconcile for "${recipe.metadata.name}" at ${stage}; recipe is gone or deleting`+ createLogger('wrc', recipe.metadata.name).warn(+ 'Skipping stale reconcile because the recipe is gone or deleting',+ { name: recipe.metadata.name, stage })return {phase: recipe.status?.phase ?? 'candidate',@@ -1473,7 +1531,9 @@ export class WorkflowRecipeReconciler {name: recipe.metadata.name,body: patch,})- console.log(`[WR-Reconciler] Added finalizer to "${recipe.metadata.name}"`)+ createLogger('wrc', recipe.metadata.name).info('Added recipe finalizer', {+ name: recipe.metadata.name,+ })}async removeFinalizer(recipe: WorkflowRecipeCRD): Promise<void> {@@ -1492,14 +1552,17 @@ export class WorkflowRecipeReconciler {})} catch (error) {if (getErrorCode(error) === 404) {- console.warn(- `[WR-Reconciler] Finalizer already gone for "${recipe.metadata.name}" because the recipe no longer exists`+ createLogger('wrc', recipe.metadata.name).warn(+ 'Finalizer already gone because the recipe no longer exists',+ { name: recipe.metadata.name })return}throw error}- console.log(`[WR-Reconciler] Removed finalizer from "${recipe.metadata.name}"`)+ createLogger('wrc', recipe.metadata.name).info('Removed recipe finalizer', {+ name: recipe.metadata.name,+ })}// ─── Main Pipeline ────────────────────────────────────────────────@@ -1527,8 +1590,9 @@ export class WorkflowRecipeReconciler {// but a manually-applied CRD in another namespace must not be reconciled.const allowedNamespaces = [this.config.sandboxNamespace]if (!allowedNamespaces.includes(ns)) {- console.warn(- `[WR-Reconciler] Refusing to reconcile "${name}" — namespace "${ns}" is not in allowlist (${allowedNamespaces.join(', ')}). This recipe bypassed the admission layer; leaving untouched.`+ createLogger('wrc', recipe.metadata.name).warn(+ 'Refusing to reconcile a recipe outside the namespace allowlist',+ { name, ns, value3: allowedNamespaces.join(', ') })return {phase: 'failed',@@ -1538,9 +1602,7 @@ export class WorkflowRecipeReconciler {}}- console.log(- `[WR-Reconciler] Reconciling "${name}" in namespace "${ns}" (phase: ${currentPhase})`- )+ createLogger('wrc', recipe.metadata.name).info('Reconciling recipe', { name, ns, currentPhase })if (!(await this.recipeStillActive(recipe))) {return this.staleRecipeResult(recipe, 'initial check')@@ -1624,7 +1686,10 @@ export class WorkflowRecipeReconciler {if (violations.length > 0) {if (isWorkflow) await this.revokeCoordinatorGfsNetworkPolicy(recipe)const details = violations.map(v => `[${v.policy}] ${v.rule}: ${v.message}`).join('; ')- console.error(`[WR-Reconciler] Policy violation for "${name}": ${details}`)+ createLogger('wrc', recipe.metadata.name).error('Recipe policy violation', {+ name,+ details,+ })return {phase: 'failed',message: `Policy violation: ${details}`,@@ -1703,9 +1768,9 @@ export class WorkflowRecipeReconciler {approvalScopeRecipeName = await this.workflowRuntimeScopeRecipeName(recipe)} catch (error) {if (!(error instanceof RuntimeScopeResolutionPendingError)) throw error- console.warn(- `[WR-Reconciler] Runtime scope resolution pending for workflow "${name}"; keeping current state and requeueing:`,- error+ createLogger('wrc', recipe.metadata.name).warn(+ 'Runtime scope resolution pending; keeping current state and requeueing',+ { name, err: error })return {phase: currentPhase as RecipePhase,@@ -1721,7 +1786,12 @@ export class WorkflowRecipeReconciler {}}- if (currentPhase === 'active' && awaitsTriggeredRun && !wfExecPhase) {+ if (+ currentPhase === 'active' &&+ awaitsTriggeredRun &&+ !wfExecPhase &&+ !hasPendingTransportNetworkReadiness(recipe)+ ) {// Plugin Workload SDK recipes (BOTH families) keep an eager mcp-host and// must fall through to the inner reconcile: promptBridge so// ensureEagerSdkMcpHost retries the /configure (the short-circuit below@@ -1793,8 +1863,9 @@ export class WorkflowRecipeReconciler {skipStatusPatch: recipe.status?.message === derivedMessage,}}- console.log(- `[WR-Reconciler] Workflow "${name}" ${wfExecPhase} — transitioning recipe phase: ${currentPhase} → ${derivedPhase}`+ createLogger('wrc', recipe.metadata.name).info(+ 'Transitioning recipe phase from workflow execution state',+ { name, wfExecPhase, currentPhase, derivedPhase })return { phase: derivedPhase, message: derivedMessage, workloadStatuses: [] }}@@ -1812,18 +1883,24 @@ export class WorkflowRecipeReconciler {// EXCEPTION: "initializing" and "recovering" phases mean pods may not exist yet,// may be broken (e.g. CreateContainerConfigError), or were just deleted by crash// recovery. We MUST reconcile to allow infrastructure creation to complete.- if (wfInProgress && (currentPhase === 'deploying' || currentPhase === 'failed')) {+ if (+ wfInProgress &&+ (currentPhase === 'deploying' || currentPhase === 'failed') &&+ !hasPendingTransportNetworkReadiness(recipe)+ ) {if (wfExecPhase === 'initializing' || wfExecPhase === 'recovering') {- console.log(- `[WR-Reconciler] Workflow "${name}" ${wfExecPhase} — allowing reconcile for pod creation`+ createLogger('wrc', recipe.metadata.name).info(+ 'Allowing workflow reconciliation for pod creation',+ { name, wfExecPhase })// Fall through to WorkflowReconciler.reconcile() which will create/recreate pods} else {if (coordinatorGfsPolicyCanOpen) {await this.ensureCoordinatorGfsNetworkPolicyIfEnabled(recipe)}- console.log(- `[WR-Reconciler] Workflow "${name}" in-progress (wf: ${wfExecPhase}) — skipping reconcile`+ createLogger('wrc', recipe.metadata.name).info(+ 'Skipping reconciliation for an in-progress workflow',+ { name, wfExecPhase })await this.ensureSteadyWorkflowRuntimeCredentials(recipe, approvalScopeRecipeName)// Issue #637 — same revocation enforcement as the active short-circuit@@ -1846,7 +1923,11 @@ export class WorkflowRecipeReconciler {// "active" covers both completed workflow infrastructure and an already-active// recipe whose current execution is still running.- if (currentPhase === 'active' && !awaitsTriggeredRun) {+ if (+ currentPhase === 'active' &&+ !awaitsTriggeredRun &&+ !hasPendingTransportNetworkReadiness(recipe)+ ) {if (coordinatorGfsPolicyCanOpen) {await this.ensureCoordinatorGfsNetworkPolicyIfEnabled(recipe)}@@ -1913,8 +1994,9 @@ export class WorkflowRecipeReconciler {// otherwise it propagates out of the workflow branch where the watcher// swallows it with no deterministic retry (issue #571).if (isRetryableInfraError(error)) {- console.warn(- `[WR-Reconciler] Transient infra error assigning instances for "${recipe.metadata.name}" — will retry: ${String(error)}`+ createLogger('wrc', recipe.metadata.name).warn(+ 'Transient infrastructure error assigning workload instances; will retry',+ { name: recipe.metadata.name, err: error })return {phase: (recipe.status?.phase ?? 'deploying') as RecipePhase,@@ -2000,14 +2082,18 @@ export class WorkflowRecipeReconciler {let workflowInternalDependencyConditions: StatusCondition[] | undefinedlet workflowSecretOwnershipConditions: StatusCondition[] | undefinedlet workflowWorkloadConditions: StatusCondition[] | undefined+ let workflowTransportNetworkConditions: StatusCondition[] | undefined =+ workflowWorkloads.length === 0 ? [] : undefinedif (workflowWorkloads.length > 0) {- console.log(- `[WR-Reconciler] Deploying ${workflowWorkloads.length} workflow workload(s) for "${name}"`- )+ createLogger('wrc', recipe.metadata.name).info('Deploying workflow workloads', {+ length: workflowWorkloads.length,+ name,+ })try {const workflowDeploy = await this.deployWorkflowWorkloads(recipe, name)workflowInternalDependencyConditions = workflowDeploy.internalDependencyConditionsworkflowWorkloadConditions = workflowDeploy.workloadConditions+ workflowTransportNetworkConditions = workflowDeploy.transportNetworkConditions// Issue #637 — surface the EnvSecretOwnershipDenied condition from the// workflow build path too; previously it was computed and dropped, so a// denied workflow workload degraded silently with no status condition.@@ -2028,6 +2114,13 @@ export class WorkflowRecipeReconciler {workloadStatuses: [],internalDependencyConditions,workloadConditions,+ ...(error instanceof TransportNetworkReadinessPendingError+ ? { transportNetworkConditions: pendingTransportNetworkConditions(recipe, message) }+ : {}),+ ...(error instanceof NetworkPolicyReplanRequiredError ||+ error instanceof TransportNetworkReadinessPendingError+ ? { requeueAfterMs: TRANSIENT_REQUEUE_BASE_MS }+ : {}),}}return {@@ -2041,8 +2134,9 @@ export class WorkflowRecipeReconciler {}}- console.log(- `[WR-Reconciler] Workflow "${name}" first deploy (${recipe.spec.steps!.length} steps) — creating infrastructure`+ createLogger('wrc', recipe.metadata.name).info(+ 'Creating workflow infrastructure for first deployment',+ { name, length: recipe.spec.steps!.length })// Recovery handoff: when this workflow was latched `failed` by a transient// infra blip, hand the inner reconciler a `recovering` execution phase@@ -2092,6 +2186,7 @@ export class WorkflowRecipeReconciler {clearWorkflowExecution: result.clearWorkflowExecution,workflowConditions: result.workflowConditions,workloadConditions: workflowWorkloadConditions,+ transportNetworkConditions: workflowTransportNetworkConditions,pluginWorkloadSdkBootstrapProof: result.pluginWorkloadSdkBootstrapProof,pluginWorkloadSdkPolicyPending: result.pluginWorkloadSdkPolicyPending,// Thread the inner reconciler's transient-skip up so the watcher leaves@@ -2190,18 +2285,21 @@ export class WorkflowRecipeReconciler {(recipe.status?.workloads?.length ?? 0) > 0 &&(recipe.status?.workloads ?? []).every(w => w.ready === true)if (latchedByTransientError) {- console.log(- `[WR-Reconciler] Recipe "${name}" is failed with a transient infra message — re-reconciling to self-heal`+ createLogger('wrc', recipe.metadata.name).info(+ 'Retrying a recipe failed by transient infrastructure',+ { name })}if (latchedBySharedMcpInternalDependencyBoundary) {- console.log(- `[WR-Reconciler] Recipe "${name}" is failed by a shared mcp-server internal-dependency boundary decision — re-reconciling to self-heal`+ createLogger('wrc', recipe.metadata.name).info(+ 'Retrying a recipe failed by a shared MCP internal-dependency boundary',+ { name })}if (latchedDespiteHealthyWorkloads) {- console.log(- `[WR-Reconciler] Recipe "${name}" is failed but all observed workloads are ready — re-reconciling to re-derive phase from live health`+ createLogger('wrc', recipe.metadata.name).info(+ 'Retrying a failed recipe whose observed workloads are ready',+ { name })}if (@@ -2210,9 +2308,10 @@ export class WorkflowRecipeReconciler {!latchedBySharedMcpInternalDependencyBoundary &&!latchedDespiteHealthyWorkloads) {- console.log(- `[WR-Reconciler] Skipping non-deployable recipe "${name}" (phase: ${currentPhase})`- )+ createLogger('wrc', recipe.metadata.name).info('Skipping non-deployable recipe', {+ name,+ currentPhase,+ })return {phase: currentPhase as RecipePhase,message: recipe.status?.message ?? `Recipe is in ${currentPhase} state`,@@ -2380,8 +2479,9 @@ export class WorkflowRecipeReconciler {try {await this.ensureOAuthBrokerTokenSecret(recipe)} catch (err) {- console.warn(- `[WR-Reconciler] Broker-token issuance failed during reconcile for "${name}" (will retry via rotation loop): ${err instanceof Error ? err.message : String(err)}`+ createLogger('wrc', recipe.metadata.name).warn(+ 'Broker-token issuance failed during reconcile; the rotation loop will retry',+ { name, err })}@@ -2422,12 +2522,16 @@ export class WorkflowRecipeReconciler {secretKeys)if (preDeployedServers.length > 0) {- console.log(- `[WR-Reconciler] Pre-deployed ${preDeployedServers.length} McpServer(s) for network isolation`+ createLogger('wrc', recipe.metadata.name).info(+ 'Pre-deployed McpServers for network isolation',+ { length: preDeployedServers.length })}} catch (error) {- console.error(`[WR-Reconciler] Pre-deploy failed for "${name}":`, error)+ createLogger('wrc', recipe.metadata.name).error('Recipe pre-deploy failed', {+ name,+ err: error,+ })// The pre-deploy McpServer handshake is an eventually-consistent// wait on HCC reconciling the child McpServers + NetworkPolicies. A// failure here means HCC has not caught up yet, NOT that the recipe@@ -2447,7 +2551,9 @@ export class WorkflowRecipeReconciler {// Step 7c: Wait for HCC to confirm network isolationif (preDeployedServers.length > 0) {await this.waitForTransportNetworkReadiness(recipe, preDeployedServers, mcpBatchNs)- console.log(`[WR-Reconciler] Transport network readiness checked for MCP workloads`)+ createLogger('wrc', recipe.metadata.name).info(+ '[WR-Reconciler] Transport network readiness checked for MCP workloads'+ )}if (!(await this.recipeStillActive(recipe))) {@@ -2471,10 +2577,9 @@ export class WorkflowRecipeReconciler {// instance and surface the denial so the operator labels the Secret.await this.teardownDeniedWorkload(workload, recipe, secretKeys).catch(err => {deniedTeardownFailed = true- console.error(- `[WR-Reconciler] Issue #637: teardown of denied workload "${workload.id}" failed; ` +- `will requeue:`,- err+ createLogger('wrc', recipe.metadata.name).error(+ 'Teardown of denied workload failed; will requeue',+ { id: workload.id, err })})workloadStatuses.push({@@ -2654,12 +2759,16 @@ export class WorkflowRecipeReconciler {secretKeys)if (delegated.length > 0) {- console.log(- `[WR-Reconciler] Delegated ${delegated.length} MCP server(s): ${delegated.join(', ')}`- )+ createLogger('wrc', recipe.metadata.name).info('Delegated MCP servers', {+ length: delegated.length,+ value2: delegated.join(', '),+ })}} catch (error) {- console.error(`[WR-Reconciler] MCP delegation failed for "${name}":`, error)+ createLogger('wrc', recipe.metadata.name).error('MCP delegation failed', {+ name,+ err: error,+ })// Same eventually-consistent handshake rationale as the pre-deploy// step above: delegation depends on HCC having persisted the child// McpServers/Context. A failure is transient (HCC not caught up),@@ -2711,6 +2820,7 @@ export class WorkflowRecipeReconciler {internalDependencyConditions,secretOwnershipConditions: secretOwnership.conditions,workloadConditions,+ transportNetworkConditions: [],}}@@ -2763,6 +2873,7 @@ export class WorkflowRecipeReconciler {internalDependencyConditions,secretOwnershipConditions: secretOwnership.conditions,workloadConditions,+ transportNetworkConditions: [],pluginWorkloadSdkProviderUnavailable: sdkOnlyProviderUnavailable,pluginWorkloadSdkPolicyPending: sdkOnlyPolicyPending,pluginWorkloadSdkBootstrapProof: sdkOnlyRuntime?.pluginWorkloadSdkBootstrapProof,@@ -2791,13 +2902,26 @@ export class WorkflowRecipeReconciler {// every failure was transient; a permanent failure yields a plain// Error that must stay terminal (fail-closed) below.if (error instanceof RetryableReconcileError) {- console.warn(- `[WR-Reconciler] Transient failure reconciling "${name}" (will retry): ${error.message}`+ createLogger('wrc', recipe.metadata.name).warn(+ 'Transient reconciliation failure; will retry',+ { name, err: error })return {phase: 'degraded',message: error.message,workloadStatuses: [],+ ...(error instanceof TransportNetworkReadinessPendingError+ ? {+ transportNetworkConditions: pendingTransportNetworkConditions(+ recipe,+ error.message+ ),+ }+ : {}),+ ...(error instanceof NetworkPolicyReplanRequiredError ||+ error instanceof TransportNetworkReadinessPendingError+ ? { requeueAfterMs: TRANSIENT_REQUEUE_BASE_MS }+ : {}),}}// (2) A raw retryable infra error (connect ETIMEDOUT/ECONNRESET, 5xx,@@ -2816,8 +2940,9 @@ export class WorkflowRecipeReconciler {const isEgressResolutionError =error instanceof Error && error.message.includes('egress resolution failed')if (!isEgressResolutionError && isRetryableInfraError(error)) {- console.warn(- `[WR-Reconciler] Transient infra error reconciling "${name}" — keeping phase "${currentPhase}", will retry: ${String(error)}`+ createLogger('wrc', recipe.metadata.name).warn(+ 'Transient infrastructure error during reconciliation; keeping phase and retrying',+ { name, currentPhase, err: error })return {phase: currentPhase as RecipePhase,@@ -2835,7 +2960,10 @@ export class WorkflowRecipeReconciler {requeueAfterMs: TRANSIENT_REQUEUE_BASE_MS,}}- console.error(`[WR-Reconciler] Failed to reconcile "${name}":`, error)+ createLogger('wrc', recipe.metadata.name).error('Recipe reconciliation failed', {+ name,+ err: error,+ })return {phase: 'failed',message: String(error),@@ -3055,10 +3183,10 @@ export class WorkflowRecipeReconciler {const readyReplicas = dep.status?.readyReplicas ?? 0ready = readyReplicas >= 1} catch (error) {- console.warn(- `[WR-Reconciler] Could not read webhook-gateway Deployment for "${name}":`,- error- )+ createLogger('wrc', recipe.metadata.name).warn('Could not read webhook-gateway Deployment', {+ name,+ err: error,+ })}if (ready) {conditions.push(@@ -3230,6 +3358,7 @@ export class WorkflowRecipeReconciler {internalDependencyConditions: StatusCondition[]secretOwnershipConditions: StatusCondition[]workloadConditions: StatusCondition[]+ transportNetworkConditions?: StatusCondition[]}> {const workloads = recipe.spec.workloads ?? []if (workloads.length === 0)@@ -3239,8 +3368,9 @@ export class WorkflowRecipeReconciler {workloadConditions: [],}if (!(await this.recipeStillActive(recipe))) {- console.warn(- `[WR-Reconciler] Skipping workflow workload deploy for "${name}"; recipe is gone or deleting`+ createLogger('wrc', recipe.metadata.name).warn(+ 'Skipping workload deployment because the recipe is gone or deleting',+ { name })return {internalDependencyConditions: [],@@ -3283,7 +3413,10 @@ export class WorkflowRecipeReconciler {secretKeys)} catch (err) {- console.error(`[WR-Reconciler] Pre-deploy MCP failed for workflow "${name}":`, err)+ createLogger('wrc', recipe.metadata.name).error('MCP pre-deploy failed for workflow', {+ name,+ err,+ })// Eventually-consistent HCC handshake (see the non-workflow pre-deploy// step). Throw RetryableReconcileError so the workflow deploy path// (which maps it to `degraded`) retries instead of failing the run.@@ -3298,8 +3431,9 @@ export class WorkflowRecipeReconciler {await this.waitForTransportNetworkReadiness(recipe, preDeployedServers, mcpBatchNs)}if (!(await this.recipeStillActive(recipe))) {- console.warn(- `[WR-Reconciler] Skipping workflow workload materialization for "${name}"; recipe is gone or deleting`+ createLogger('wrc', recipe.metadata.name).warn(+ 'Skipping workload materialization because the recipe is gone or deleting',+ { name })return {internalDependencyConditions: [],@@ -3315,8 +3449,9 @@ export class WorkflowRecipeReconciler {try {await this.ensureOAuthBrokerTokenSecret(recipe)} catch (err) {- console.warn(- `[WR-Reconciler] Broker-token issuance failed during workflow reconcile for "${name}" (will retry via rotation loop): ${err instanceof Error ? err.message : String(err)}`+ createLogger('wrc', recipe.metadata.name).warn(+ 'Broker-token issuance failed during workflow reconciliation; rotation will retry',+ { name, err })}@@ -3329,9 +3464,13 @@ export class WorkflowRecipeReconciler {for (const workloadId of sortOrder) {const workload = workloads.find(w => w.id === workloadId)!if (secretOwnership.deniedWorkloadIds.has(workload.id)) {- console.error(- `[WR-Reconciler] EnvSecretOwnershipDenied for workflow workload "${workloadId}" in ` +- `"${name}": ${secretOwnership.messageByWorkload.get(workloadId) ?? 'foreign Secret reference'}`+ createLogger('wrc', recipe.metadata.name).error(+ 'Secret ownership denied for workflow workload',+ {+ workloadId,+ name,+ value3: secretOwnership.messageByWorkload.get(workloadId) ?? 'foreign Secret reference',+ })try {await this.teardownDeniedWorkload(workload, recipe, secretKeys)@@ -3381,10 +3520,11 @@ export class WorkflowRecipeReconciler {break}} catch (err) {- console.error(- `[WR-Reconciler] Failed to deploy workload "${workloadId}" for workflow "${name}":`,- err- )+ createLogger('wrc', recipe.metadata.name).error('Failed to deploy workflow workload', {+ workloadId,+ name,+ err,+ })if (workload.type === 'statefulset') {throw err}@@ -3421,9 +3561,9 @@ export class WorkflowRecipeReconciler {`Service "${svcName}"`)} catch (err) {- console.error(- `[WR-Reconciler] Service creation failed for workload "${workload.id}" in workflow "${name}":`,- err+ createLogger('wrc', recipe.metadata.name).error(+ 'Service creation failed for workflow workload',+ { id: workload.id, name, err })}}@@ -3459,8 +3599,9 @@ export class WorkflowRecipeReconciler {// Children live in `mcp-server` (mcpBatchNs), not the recipe's own ns.if (hasTransport) {if (!(await this.recipeStillActive(recipe))) {- console.warn(- `[WR-Reconciler] Skipping workflow MCP delegation for "${name}"; recipe is gone or deleting`+ createLogger('wrc', recipe.metadata.name).warn(+ 'Skipping MCP delegation because the recipe is gone or deleting',+ { name })return {internalDependencyConditions,@@ -3476,12 +3617,16 @@ export class WorkflowRecipeReconciler {secretKeys)if (delegated.length > 0) {- console.log(- `[WR-Reconciler] MCP delegation for workflow "${name}": ${delegated.join(', ')}`- )+ createLogger('wrc', recipe.metadata.name).info('Delegated workflow MCP servers', {+ name,+ value2: delegated.join(', '),+ })}} catch (err) {- console.error(`[WR-Reconciler] MCP delegation failed for workflow "${name}":`, err)+ createLogger('wrc', recipe.metadata.name).error('Workflow MCP delegation failed', {+ name,+ err,+ })// Same eventually-consistent HCC handshake as the non-workflow// delegation step: degrade + retry instead of failing the run.throw new RetryableReconcileError(@@ -3496,6 +3641,7 @@ export class WorkflowRecipeReconciler {internalDependencyConditions,secretOwnershipConditions: secretOwnership.conditions,workloadConditions: [],+ transportNetworkConditions: [],}}@@ -3504,7 +3650,7 @@ export class WorkflowRecipeReconciler {async reconcileDelete(recipe: WorkflowRecipeCRD): Promise<void> {const name = recipe.metadata.name- console.log(`[WR-Reconciler] Deleting resources for "${name}"`)+ createLogger('wrc', recipe.metadata.name).info('Deleting recipe resources', { name })this.secretReverseIndex?.delete(name)// ─── Workflow Delete (Stage 1) ────────────────────────────────────@@ -3541,7 +3687,10 @@ export class WorkflowRecipeReconciler {try {await cleanupDelegation(this.delegationDeps, recipe, this.config.namespace)} catch (error) {- console.error(`[WR-Reconciler] Delegation cleanup failed for "${name}":`, error)+ createLogger('wrc', recipe.metadata.name).error('Delegation cleanup failed', {+ name,+ err: error,+ })throw error}}@@ -3583,10 +3732,11 @@ export class WorkflowRecipeReconciler {completed.push('compute-pods(coordinator,mcp-host,snippet-runner)')} catch (error) {failures.push('compute-pods')- console.error(- `[WR-Reconciler] Terminal compute-pod teardown failed for "${name}" (run ${runId}):`,- error- )+ createLogger('wrc', recipe.metadata.name).error('Terminal compute-pod teardown failed', {+ name,+ runId,+ err: error,+ })}}@@ -3605,10 +3755,11 @@ export class WorkflowRecipeReconciler {}} catch (error) {failures.push('mcp-server-delegation')- console.error(- `[WR-Reconciler] Terminal delegation cleanup failed for "${name}" (run ${runId}):`,- error- )+ createLogger('wrc', recipe.metadata.name).error('Terminal delegation cleanup failed', {+ name,+ runId,+ err: error,+ })}// 3. WRC-owned transport Deployment(s) in mcp-server. See step 2 rationale.@@ -3619,16 +3770,20 @@ export class WorkflowRecipeReconciler {}} catch (error) {failures.push('transport-deployments')- console.error(- `[WR-Reconciler] Terminal transport-Deployment teardown failed for "${name}" (run ${runId}):`,- error+ createLogger('wrc', recipe.metadata.name).error(+ 'Terminal transport Deployment teardown failed',+ { name, runId, err: error })}- console.log(- `[WR-Reconciler] Workflow "${name}" terminal (run ${runId}) — cleanup attempted [${completed.join('; ') || 'nothing'}]` +- `${failures.length ? `; failed (left for archive-cron) [${failures.join(', ')}]` : ''}` +- `; preserved artifact-reader + output PVC + output-anchor + CR + DB`+ createLogger('wrc', recipe.metadata.name).info(+ 'Terminal workflow cleanup attempted; artifacts, output resources, recipe and database retained',+ {+ name,+ runId,+ value3: completed.join('; ') || 'nothing',+ value4: failures.length ? `; failed (left for archive-cron) [${failures.join(', ')}]` : '',+ })}@@ -3700,7 +3855,9 @@ export class WorkflowRecipeReconciler {// PVCs: honor clerum.io/pvc-retention annotation (default: "retain").const pvcRetention = recipe.metadata.annotations?.['clerum.io/pvc-retention'] ?? 'retain'if (await this.hasVerifiedInheritedParentResources(recipe)) {- console.log(`[WR-Reconciler] Skipping inherited parent resources for child "${name}"`)+ createLogger('wrc', recipe.metadata.name).info('Skipping inherited parent resources', {+ name,+ })return}@@ -3748,8 +3905,9 @@ export class WorkflowRecipeReconciler {`PVC "${res.id}" in ${resNs}`)} else {- console.log(- `[WR-Reconciler] Retaining PVC "${res.id}" (pvc-retention: ${pvcRetention})`+ createLogger('wrc', recipe.metadata.name).info(+ 'Retaining PVC according to retention policy',+ { id: res.id, pvcRetention })}break@@ -3849,22 +4007,31 @@ export class WorkflowRecipeReconciler {if (idempotency) stampSpecHash(idempotency.manifest)try {await createFn()- console.log(`[WR-Reconciler] Created ${label}`)+ createLogger('wrc', 'workflow-recipes').info('Created resource', { label })} catch (error: unknown) {if (getErrorCode(error) === 409) {if (idempotency && (await this.applyIsNoop(idempotency))) {- console.log(`[WR-Reconciler] ${label} unchanged (spec-hash match); skipping update`)+ createLogger('wrc', 'workflow-recipes').info(+ 'Resource spec hash unchanged; skipping update',+ { label }+ )return}try {await replaceFn()- console.log(`[WR-Reconciler] Updated ${label}`)+ createLogger('wrc', 'workflow-recipes').info('Updated resource', { label })} catch (updateError) {- console.error(`[WR-Reconciler] Failed to update ${label}:`, updateError)+ createLogger('wrc', 'workflow-recipes').error('Failed to update resource', {+ label,+ err: updateError,+ })throw updateError}} else {- console.error(`[WR-Reconciler] Failed to create ${label}:`, error)+ createLogger('wrc', 'workflow-recipes').error('Failed to create resource', {+ label,+ err: error,+ })throw error}}@@ -4083,22 +4250,24 @@ export class WorkflowRecipeReconciler {const desiredHash = stampSpecHash(statefulSet)try {await this.appsApi.createNamespacedStatefulSet({ namespace, body: statefulSet })- console.log(`[WR-Reconciler] Created StatefulSet "${name}" in ${namespace}`)+ createLogger('wrc', 'workflow-recipes').info('Created StatefulSet', { name, namespace })return} catch (error: unknown) {if (getErrorCode(error) !== 409) {- console.error(- `[WR-Reconciler] Failed to create StatefulSet "${name}" in ${namespace}:`,- error- )+ createLogger('wrc', 'workflow-recipes').error('Failed to create StatefulSet', {+ name,+ namespace,+ err: error,+ })throw error}}const existing = await this.appsApi.readNamespacedStatefulSet({ name, namespace })if (existing.metadata?.annotations?.[SPEC_HASH_ANNOTATION] === desiredHash) {- console.log(- `[WR-Reconciler] StatefulSet "${name}" in ${namespace} unchanged (spec-hash match); skipping update`+ createLogger('wrc', 'workflow-recipes').info(+ 'StatefulSet spec hash unchanged; skipping update',+ { name, namespace })return}@@ -4126,18 +4295,27 @@ export class WorkflowRecipeReconciler {})const patchKind = patchBody.spec ? 'metadata+mutable spec' : 'metadata'- console.log(`[WR-Reconciler] Patched StatefulSet "${name}" in ${namespace} (${patchKind})`)+ createLogger('wrc', 'workflow-recipes').info('Patched StatefulSet', {+ name,+ namespace,+ patchKind,+ })}private async safeDelete(deleteFn: () => Promise<unknown>, label: string): Promise<void> {try {await deleteFn()- console.log(`[WR-Reconciler] Deleted ${label}`)+ createLogger('wrc', 'workflow-recipes').info('Deleted resource', { label })} catch (error: unknown) {if (getErrorCode(error) === 404) {- console.log(`[WR-Reconciler] ${label} already gone`)+ createLogger('wrc', 'workflow-recipes').info('Resource already gone', {+ label,+ })} else {- console.error(`[WR-Reconciler] Failed to delete ${label}:`, error)+ createLogger('wrc', 'workflow-recipes').error('Failed to delete resource', {+ label,+ err: error,+ })}}}@@ -4152,13 +4330,18 @@ export class WorkflowRecipeReconciler {private async deleteOrThrow(deleteFn: () => Promise<unknown>, label: string): Promise<void> {try {await deleteFn()- console.log(`[WR-Reconciler] Deleted ${label}`)+ createLogger('wrc', 'workflow-recipes').info('Deleted resource', { label })} catch (error: unknown) {if (getErrorCode(error) === 404) {- console.log(`[WR-Reconciler] ${label} already gone`)+ createLogger('wrc', 'workflow-recipes').info('Resource already gone', {+ label,+ })return}- console.error(`[WR-Reconciler] Failed to delete ${label}:`, error)+ createLogger('wrc', 'workflow-recipes').error('Failed to delete resource', {+ label,+ err: error,+ })throw error}}@@ -4184,16 +4367,22 @@ export class WorkflowRecipeReconciler {existing = await read()} catch (error: unknown) {if (getErrorCode(error) === 404) {- console.log(`[WR-Reconciler] ${label} already gone`)+ createLogger('wrc', recipe.metadata.name).info('Resource already gone', {+ label,+ })} else {- console.error(`[WR-Reconciler] Skipping delete of ${label} — ownership read failed:`, error)+ createLogger('wrc', recipe.metadata.name).error(+ 'Skipping resource deletion because ownership could not be read',+ { label, err: error }+ )}return}const owner = existing.metadata?.labels?.['clerum.io/recipe']if (owner !== recipe.metadata.name) {- console.log(- `[WR-Reconciler] Skipping delete of ${label} — owned by "${owner ?? 'unset'}", not "${recipe.metadata.name}" (issue #571)`+ createLogger('wrc', recipe.metadata.name).info(+ 'Skipping resource deletion because another recipe owns it',+ { label, value2: owner ?? 'unset', name: recipe.metadata.name })return}@@ -4318,10 +4507,15 @@ export class WorkflowRecipeReconciler {// siblings) does not throw and freezes nothing — surface it so the missing// egress until the next refresh is not silent.if (failures.length > 0) {- console.warn(- `[WR-Reconciler] ${policyName}: ${failures.length} external egress FQDN(s) failed to resolve this round; policy written without them until the next refresh converges: ${failures- .map(f => `${f.fqdn} (${f.retryable ? 'transient' : 'permanent'})`)- .join(', ')}`+ createLogger('wrc', recipe.metadata.name).warn(+ 'External egress FQDN resolution failed; next refresh must converge the policy',+ {+ policyName,+ length: failures.length,+ value3: failures+ .map(f => `${f.fqdn} (${f.retryable ? 'transient' : 'permanent'})`)+ .join(', '),+ })}this.warnEgressAccumulator(policyName, acc)@@ -4380,9 +4574,10 @@ export class WorkflowRecipeReconciler {// write when the persisted window is aging (renewalDue, audit M1), even if// the set is unchanged, so a stable-then-rotated IP keeps its overlap grace.if (!this.egressWriteNeeded(existing, policy) && !egressRenewalDue && !egressStateChanged) {- console.log(- `[WR-Reconciler] NetworkPolicy "${policyName}" in ${ns} egress set unchanged — no-op`- )+ createLogger('wrc', recipe.metadata.name).info('NetworkPolicy egress set unchanged; no-op', {+ policyName,+ ns,+ })} else {await this.applyOwnedRecipeNetworkPolicy(policy, ns, recipe.metadata.name)// #299: the policy has landed — record the set it actually enforces.@@ -4695,8 +4890,9 @@ export class WorkflowRecipeReconciler {if (!name || desiredNames.has(name)) continuetry {await this.networkingApi.deleteNamespacedNetworkPolicy({ name, namespace })- console.log(- `[WR-Reconciler] Deleted stale internal-dependency NetworkPolicy "${name}" in ${namespace}`+ createLogger('wrc', recipe.metadata.name).info(+ 'Deleted stale internal-dependency NetworkPolicy',+ { name, namespace })} catch (error) {if (getErrorCode(error) !== 404) throw error@@ -4937,8 +5133,9 @@ export class WorkflowRecipeReconciler {!wlEgressRenewalDue &&!wlEgressStateChanged) {- console.log(- `[WR-Reconciler] NetworkPolicy "${wlPolicyName}" in ${wlNs} egress set unchanged — no-op`+ createLogger('wrc', recipe.metadata.name).info(+ 'NetworkPolicy egress set unchanged; no-op',+ { wlPolicyName, wlNs })} else {await this.applyOwnedRecipeNetworkPolicy(policy, wlNs, recipe.metadata.name)@@ -5060,7 +5257,9 @@ export class WorkflowRecipeReconciler {body: { preconditions: { uid, resourceVersion } },})} catch (error: unknown) {- if (getErrorCode(error) !== 404) throw error+ if (getErrorCode(error) !== 404) {+ throwNetworkPolicyMutationError(error, policyName, namespace)+ }}createLogger('wrc', recipeName).info('contracted recipe network policy', {policy: policyName,@@ -5072,11 +5271,15 @@ export class WorkflowRecipeReconciler {if (!contractionWriteNeeded(existing, desired)) return existingdesired.metadata = { ...(desired.metadata ?? {}), resourceVersion }- await this.networkingApi.replaceNamespacedNetworkPolicy({- name: policyName,- namespace,- body: desired,- })+ try {+ await this.networkingApi.replaceNamespacedNetworkPolicy({+ name: policyName,+ namespace,+ body: desired,+ })+ } catch (error: unknown) {+ throwNetworkPolicyMutationError(error, policyName, namespace)+ }let live: k8s.V1NetworkPolicytry {live = await this.networkingApi.readNamespacedNetworkPolicy({@@ -5094,10 +5297,14 @@ export class WorkflowRecipeReconciler {live.metadata?.name !== policyName ||live.metadata?.namespace !== namespace ||!liveUid ||- !liveResourceVersion ||- (existing.metadata?.uid && liveUid !== existing.metadata.uid)+ !liveResourceVersion) {throw new NetworkPolicyOwnershipConflictError(+ `NetworkPolicy "${policyName}" in ${namespace} has incomplete or inconsistent identity after contraction`+ )+ }+ if (existing.metadata?.uid && liveUid !== existing.metadata.uid) {+ throw new NetworkPolicyReplanRequiredError(`NetworkPolicy "${policyName}" in ${namespace} changed identity during contraction`)}@@ -5106,11 +5313,16 @@ export class WorkflowRecipeReconciler {// The apiserver read-back is broader or carries provenance that does not// match its enforced rules. Do not let phase 2 treat the request body as// reality; revoke this exact owned identity and force a clean re-plan.- await this.networkingApi.deleteNamespacedNetworkPolicy({- name: policyName,- namespace,- body: { preconditions: { uid: liveUid, resourceVersion: liveResourceVersion } },- })+ try {+ await this.networkingApi.deleteNamespacedNetworkPolicy({+ name: policyName,+ namespace,+ body: { preconditions: { uid: liveUid, resourceVersion: liveResourceVersion } },+ })+ } catch (error: unknown) {+ if (getErrorCode(error) !== 404)+ throwNetworkPolicyMutationError(error, policyName, namespace)+ }throw new NetworkPolicyOwnershipConflictError(`NetworkPolicy "${policyName}" in ${namespace} did not read back as the requested contraction`)@@ -5133,10 +5345,21 @@ export class WorkflowRecipeReconciler {await this.createOrReplace(() => this.networkingApi.createNamespacedNetworkPolicy({ namespace, body: policy }),async () => {- const live = await this.networkingApi.readNamespacedNetworkPolicy({- name: policyName,- namespace,- })+ let live: k8s.V1NetworkPolicy+ try {+ live = await this.networkingApi.readNamespacedNetworkPolicy({+ name: policyName,+ namespace,+ })+ } catch (error: unknown) {+ if (getErrorCode(error) === 404) {+ throw new NetworkPolicyReplanRequiredError(+ `NetworkPolicy "${policyName}" in ${namespace} disappeared after create conflict; a fresh reconciliation is required`,+ { cause: error }+ )+ }+ throw error+ }this.assertRecipeNetworkPolicyOwnership(live, recipeName, policyName, namespace)if (!live.metadata?.resourceVersion) {throw new NetworkPolicyOwnershipConflictError(@@ -5147,11 +5370,17 @@ export class WorkflowRecipeReconciler {...(policy.metadata ?? {}),resourceVersion: live.metadata?.resourceVersion,}- return this.networkingApi.replaceNamespacedNetworkPolicy({- name: policyName,- namespace,- body: policy,- })+ try {+ return await this.networkingApi.replaceNamespacedNetworkPolicy({+ name: policyName,+ namespace,+ body: policy,+ })+ } catch (error: unknown) {+ // This PUT is fenced by the freshly checked owner/resourceVersion.+ // Losing that snapshot is unfinished convergence, not bad user intent.+ throwNetworkPolicyMutationError(error, policyName, namespace)+ }},`NetworkPolicy "${policyName}" in ${namespace}`)@@ -5222,17 +5451,15 @@ export class WorkflowRecipeReconciler {*/private warnEgressAccumulator(policyName: string, acc: AccumulateOutput): void {if (acc.frozenFqdns.length > 0) {- console.warn(- `[WR-Reconciler] ${policyName}: egress set FROZEN (fail-static) for ${acc.frozenFqdns.join(- ', '- )} — DNS resolution is transiently failing; serving last-known IPs, not pruning.`+ createLogger('wrc', 'workflow-recipes').warn(+ 'External egress frozen during transient DNS failure; serving last accepted IPs',+ { policyName, value2: acc.frozenFqdns.join(', ') })}if (acc.overCap) {- console.warn(- `[WR-Reconciler] ${policyName}: egress set hit the maxEntries cap — evicted ${acc.evicted.length} least-recently-observed entr${- acc.evicted.length === 1 ? 'y' : 'ies'- } (never rejecting the policy).`+ createLogger('wrc', 'workflow-recipes').warn(+ 'External egress exceeded maxEntries; evicted least-recently-observed entries',+ { policyName, length: acc.evicted.length, value3: acc.evicted.length === 1 ? 'y' : 'ies' })}}@@ -5607,12 +5834,14 @@ export class WorkflowRecipeReconciler {)combined = combined ? combineSecretAccess(combined, access) : accessif (access.state === 'denied') {- console.error(- `[WR-Reconciler] Refusing to project Secret "${name}" (ns ${secretNs}) into recipe "${recipe.metadata.name}" — not owned or shared; label clerum.io/shared=true or clerum.io/owner-recipe=${recipe.metadata.name}`+ createLogger('wrc', recipe.metadata.name).error(+ 'Refusing to project a Secret that is neither owned nor shared',+ { name, secretNs, name3: recipe.metadata.name, name4: recipe.metadata.name })} else if (access.state === 'error') {- console.warn(- `[WR-Reconciler] Secret "${name}" (ns ${secretNs}) ownership could not be verified for recipe "${recipe.metadata.name}" — failing closed and requeuing`+ createLogger('wrc', recipe.metadata.name).warn(+ 'Secret ownership could not be verified; failing closed and requeueing',+ { name, secretNs, name3: recipe.metadata.name })}}@@ -5731,8 +5960,9 @@ export class WorkflowRecipeReconciler {// Read failed — do NOT silently skip (that would leave a foreign Secret// projected with a healthy status). Requeue and retry, matching the deploy// path's fail-closed contract for an unverifiable Secret.- console.warn(- `[WR-Reconciler] Issue #637: ownership recheck read failed for "${recipe.metadata.name}" — requeuing: ${String(error)}`+ createLogger('wrc', recipe.metadata.name).warn(+ 'Secret ownership recheck failed; requeueing',+ { name: recipe.metadata.name, err: error })return { ...clean, needsRequeue: true, requeueReason: 'Secret read failed' }}@@ -5755,11 +5985,13 @@ export class WorkflowRecipeReconciler {const failedTeardowns: string[] = []for (const workload of recipe.spec.workloads ?? []) {if (!secretOwnership.deniedWorkloadIds.has(workload.id)) continue- console.warn(- `[WR-Reconciler] Issue #637: revoking steady workflow workload "${workload.id}" of ` +- `"${recipe.metadata.name}" — ${- secretOwnership.messageByWorkload.get(workload.id) ?? 'foreign Secret ref'- }`+ createLogger('wrc', recipe.metadata.name).warn(+ 'Revoking steady workflow workload because Secret access was denied',+ {+ id: workload.id,+ name: recipe.metadata.name,+ value3: secretOwnership.messageByWorkload.get(workload.id) ?? 'foreign Secret ref',+ })try {await this.teardownDeniedWorkload(workload, recipe, secretKeys)@@ -5769,9 +6001,9 @@ export class WorkflowRecipeReconciler {// the next pass retries; the EnvSecretOwnershipDenied condition is written// only once teardown actually succeeds.failedTeardowns.push(workload.id)- console.error(- `[WR-Reconciler] Issue #637: teardown FAILED for denied workload "${workload.id}" of ` +- `"${recipe.metadata.name}" — requeuing: ${String(error)}`+ createLogger('wrc', recipe.metadata.name).error(+ 'Denied workload teardown failed; requeueing',+ { id: workload.id, name: recipe.metadata.name, err: error })}}@@ -5888,9 +6120,9 @@ export class WorkflowRecipeReconciler {namespace,labelSelector: `app=${resourceName}`,})- console.log(- `[WR-Reconciler] Issue #637: deleted StatefulSet "${resourceName}" managed Pods in ` +- `${namespace} (Secret ownership revoked); PVCs and StatefulSet preserved`+ createLogger('wrc', recipe.metadata.name).info(+ 'Deleted StatefulSet pods after Secret ownership revocation; preserving StatefulSet and PVCs',+ { resourceName, namespace })} catch (error) {if (getErrorCode(error) !== 404) throw error@@ -6249,7 +6481,18 @@ export class WorkflowRecipeReconciler {}}- return this.workloadStatusResult(workloadStatuses)+ const result = this.workloadStatusResult(workloadStatuses)+ if (hasPendingTransportNetworkReadiness(recipe)) {+ const condition = recipe.status?.conditions?.find(+ condition => condition.type === TRANSPORT_NETWORK_CONDITION_TYPE+ )+ return {+ ...result,+ phase: 'degraded',+ message: condition?.message ?? 'Transport external egress admission is pending',+ }+ }+ return result}private workloadStatusResult(@@ -6455,7 +6698,10 @@ export class WorkflowRecipeReconciler {// PVCs: create only (never replace — immutable after creation)try {await this.coreApi.createNamespacedPersistentVolumeClaim({ namespace: ns, body: manifest })- console.log(`[WR-Reconciler] Created PVC "${res.id}" in ${ns}`)+ createLogger('wrc', recipe.metadata.name).info('Created PVC', {+ id: res.id,+ ns,+ })} catch (error: unknown) {if (getErrorCode(error) === 409) {const existing = await this.coreApi.readNamespacedPersistentVolumeClaim({@@ -6463,7 +6709,10 @@ export class WorkflowRecipeReconciler {namespace: ns,})this.assertExistingResourcePvcOwnedByRecipe(existing, manifest, res, recipe, ns)- console.log(`[WR-Reconciler] PVC "${res.id}" already exists in ${ns} (owned, skip)`)+ createLogger('wrc', recipe.metadata.name).info(+ 'Owned PVC already exists; skipping creation',+ { id: res.id, ns }+ )} else {throw error}@@ -6617,16 +6866,19 @@ export class WorkflowRecipeReconciler {{ name: secretName, namespace: ns, body: { data: manifest.data } },{ middleware: [k8s.setHeaderMiddleware('Content-Type', 'application/merge-patch+json')] })- console.log(`[WR-Reconciler] Refreshed Secret "${secretName}" in ${ns}`)+ createLogger('wrc', recipe.metadata.name).info('Refreshed Secret', { secretName, ns })return}try {await this.coreApi.createNamespacedSecret({ namespace: ns, body: manifest })- console.log(`[WR-Reconciler] Created Secret "${secretName}" in ${ns}`)+ createLogger('wrc', recipe.metadata.name).info('Created Secret', { secretName, ns })} catch (err) {if (getErrorCode(err) !== 409) throw err- console.log(`[WR-Reconciler] Secret "${secretName}" already exists in ${ns} (skip)`)+ createLogger('wrc', recipe.metadata.name).info('Secret already exists; skipping creation', {+ secretName,+ ns,+ })}}@@ -6830,6 +7082,22 @@ export class WorkflowRecipeReconciler {recipe.status?.conditions,result.workloadConditions)+ const priorTransportConditions =+ workloadReconcileMergedConditions ??+ secretOwnershipMergedConditions ??+ internalDependencyMergedConditions ??+ workflowOutputMergedConditions ??+ webhookMergedConditions ??+ recipe.status?.conditions+ // Observers and earlier failures cannot discharge an admission requirement.+ const transportNetworkMergedConditions =+ result.transportNetworkConditions === undefined+ ? undefined+ : mergeOwnedConditions(+ priorTransportConditions,+ result.transportNetworkConditions,+ TRANSPORT_NETWORK_CONDITION_TYPES+ )// Plugin Workload SDK conditions are derived so every status patch carries a// consistent projection of spec.pluginWorkloadSdk + feature flag, while the// SDK-only provider health bit is propagated explicitly through@@ -6841,7 +7109,8 @@ export class WorkflowRecipeReconciler {const pluginSdkProjection =result.pluginWorkloadSdkProjection ?? this.projectPluginWorkloadSdk(recipe, result, now)const pluginSdkMergedConditions = mergePluginWorkloadSdkConditions(- workloadReconcileMergedConditions ??+ transportNetworkMergedConditions ??+ workloadReconcileMergedConditions ??secretOwnershipMergedConditions ??internalDependencyMergedConditions ??workflowOutputMergedConditions ??@@ -6851,6 +7120,7 @@ export class WorkflowRecipeReconciler {)const mergedConditions =pluginSdkMergedConditions ??+ transportNetworkMergedConditions ??workloadReconcileMergedConditions ??secretOwnershipMergedConditions ??internalDependencyMergedConditions ??@@ -6894,9 +7164,10 @@ export class WorkflowRecipeReconciler {{ middleware: [k8s.setHeaderMiddleware('Content-Type', 'application/merge-patch+json')] })- console.log(- `[WR-Reconciler] Status patched for "${recipe.metadata.name}": phase=${result.phase}`- )+ createLogger('wrc', recipe.metadata.name).info('Patched recipe status', {+ name: recipe.metadata.name,+ phase: result.phase,+ })}// ─── UUID Instance Assignment ──────────────────────────────────────@@ -6947,9 +7218,10 @@ export class WorkflowRecipeReconciler {},{ middleware: [k8s.setHeaderMiddleware('Content-Type', 'application/merge-patch+json')] })- console.log(- `[WR-Reconciler] Assigned workload instances for "${recipe.metadata.name}": ${JSON.stringify(instances)}`- )+ createLogger('wrc', recipe.metadata.name).info('Assigned workload instances', {+ name: recipe.metadata.name,+ value2: JSON.stringify(instances),+ })}return newlyAssigned@@ -6991,8 +7263,9 @@ export class WorkflowRecipeReconciler {) {// Logged once (only reached when no instance is recorded yet) — the single// observable marker that a fleet migration re-scoped this workload.- console.log(- `[WR-Reconciler] Migrating workload "${workload.id}" of recipe "${recipe.metadata.name}" to scoped name "${scopedName}" (issue #571)`+ createLogger('wrc', recipe.metadata.name).info(+ 'Migrating workload to its scoped resource name',+ { id: workload.id, name: recipe.metadata.name, scopedName })return scopedName}@@ -7061,9 +7334,10 @@ export class WorkflowRecipeReconciler {},{ middleware: [k8s.setHeaderMiddleware('Content-Type', 'application/merge-patch+json')] })- console.log(- `[WR-Reconciler] Assigned ${Object.keys(instances).length} resource instance(s) for "${recipe.metadata.name}"`- )+ createLogger('wrc', recipe.metadata.name).info('Assigned resource instances', {+ length: Object.keys(instances).length,+ name: recipe.metadata.name,+ })}return newlyAssigned@@ -7225,8 +7499,9 @@ export class WorkflowRecipeReconciler {if (!observedStatus?.ready) {if (anyOwned) {cleanupPending = true- console.log(- `[WR-Reconciler] Legacy StatefulSet cleanup for ${namespace}/${workload.id} deferred until scoped workload is ready`+ createLogger('wrc', recipe.metadata.name).info(+ 'Deferring legacy StatefulSet cleanup until scoped workload is ready',+ { namespace, id: workload.id })}continue@@ -7366,8 +7641,9 @@ export class WorkflowRecipeReconciler {if (!observedStatus?.ready) {if (anyOwned) {cleanupPending = true- console.log(- `[WR-Reconciler] Legacy Deployment cleanup for ${namespace}/${workload.id} deferred until scoped workload is ready`+ createLogger('wrc', recipe.metadata.name).info(+ 'Deferring legacy Deployment cleanup until scoped workload is ready',+ { namespace, id: workload.id })}continue@@ -7458,7 +7734,9 @@ export class WorkflowRecipeReconciler {): Promise<void> {try {await deleteFn()- console.log(`[WR-Reconciler] Deleted ${description}`)+ createLogger('wrc', 'workflow-recipes').info('Deleted workload resource', {+ description,+ })} catch (err: unknown) {if (getErrorCode(err) === 404) {returndiff --git a/workflow-recipes/tests/unit/observability/logger.test.ts b/workflow-recipes/tests/unit/observability/logger.test.tsindex 8066ce82c..4e076bcd5 100644--- a/workflow-recipes/tests/unit/observability/logger.test.ts+++ b/workflow-recipes/tests/unit/observability/logger.test.ts@@ -1,4 +1,6 @@import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'+import { ApiException } from '@kubernetes/client-node'+import { runInNewContext } from 'node:vm'import { createLogger } from '../../../src/observability/logger'describe('createLogger', () => {@@ -89,6 +91,150 @@ describe('createLogger', () => {expect(entry.durationMs).toBe(340)})+ it('serializes native Error diagnostics and cause into the emitted JSON', () => {+ const cause = new Error('upstream unavailable')+ const err = Object.assign(new TypeError('network policy readback failed', { cause }), {+ code: 503,+ })+ createLogger('wrc', 'r').error('reconcile failed', { err })+ const entry = JSON.parse(stdoutSpy.mock.calls[0][0])+ expect(entry.err).toMatchObject({+ name: 'TypeError',+ message: 'network policy readback failed',+ code: 503,+ cause: { name: 'Error', message: 'upstream unavailable' },+ })+ expect(entry.err.stack).toContain('TypeError: network policy readback failed')+ expect(entry.err.cause.stack).toContain('Error: upstream unavailable')+ })++ it('serializes Error values inside objects and arrays', () => {+ createLogger('wrc', 'r').error('batch failed', {+ results: [{ err: new Error('first failed') }, new Error('second failed')],+ })+ const entry = JSON.parse(stdoutSpy.mock.calls[0][0])+ expect(entry.results[0].err.message).toBe('first failed')+ expect(entry.results[1].message).toBe('second failed')+ })++ it('bounds cause depth and terminates cyclic Error causes', () => {+ const cycle = new Error('cyclic cause')+ cycle.cause = cycle+ let deep = new Error('deepest cause')+ for (let index = 0; index < 20; index++) deep = new Error(`cause ${index}`, { cause: deep })+ createLogger('wrc', 'r').error('failed', { cycle, deep })+ const entry = JSON.parse(stdoutSpy.mock.calls[0][0])+ expect(entry.cycle.cause).toBe('[Circular]')+ let current = entry.deep+ let depth = 0+ while (current && typeof current === 'object') {+ current = current.cause+ depth+++ }+ expect(depth).toBeLessThanOrEqual(6)+ expect(current).toBe('[Truncated]')+ })++ it('redacts sensitive Error properties without losing ordinary diagnostics', () => {+ // Single-character inert fixtures test field redaction; none is an access value.+ const err = Object.assign(new Error('request failed'), {+ authorization: 'x',+ body: { data: 'x' },+ safe: 'retained',+ cause: Object.assign(new Error('upstream unavailable'), { refreshToken: 'x' }),+ })+ createLogger('wrc', 'r').error('failed', { err })+ const entry = JSON.parse(stdoutSpy.mock.calls[0][0])+ expect(entry.err.message).toBe('request failed')+ expect(entry.err.authorization).toBe('[REDACTED]')+ expect(entry.err.body).toBe('[REDACTED]')+ expect(entry.err.cause.refreshToken).toBe('[REDACTED]')+ expect(entry.err.safe).toBe('retained')+ })++ it.each([false, true])(+ 'removes duplicated ApiException body and headers from diagnostics (wrapped=%s)',+ wrapped => {+ // Real client exception, with single-character inert values only.+ const body = { password: 'x', unrelatedBodyField: 'x' }+ const headers = { authorization: 'x', unrelatedHeader: 'x' }+ const apiError = new ApiException(503, 'network policy read failed', body, headers)+ const err = wrapped ? new Error('reconciliation failed', { cause: apiError }) : apiError+ createLogger('wrc', 'r').error('failed', { err })+ const entry = JSON.parse(stdoutSpy.mock.calls[0][0])+ const diagnostic = wrapped ? entry.err.cause : entry.err+ expect(diagnostic.code).toBe(503)+ expect(diagnostic.name).toBe('Error')+ expect(diagnostic.message).toContain('network policy read failed')+ expect(diagnostic.message).toContain('HTTP-Code: 503')+ expect(diagnostic.stack).toContain('at ')+ expect(diagnostic.body).toBe('[REDACTED]')+ expect(diagnostic.headers).toBe('[REDACTED]')+ for (const text of [diagnostic.message, diagnostic.stack]) {+ expect(text).not.toContain('unrelatedBodyField')+ expect(text).not.toContain('unrelatedHeader')+ expect(text).not.toContain(JSON.stringify(body))+ expect(text).not.toContain(JSON.stringify(headers))+ }+ }+ )++ it.each(['password', 'api_key', 'authorization', 'cookie'])(+ 'redacts quoted and escaped JSON %s keys in generic Error diagnostics',+ key => {+ // Serialization represents actual plain/escaped JSON diagnostics. The value+ // stays an inert single character; no credential is generated or encoded.+ for (const json of [+ JSON.stringify({ [key]: 'x' }),+ JSON.stringify(JSON.stringify({ [key]: 'x' })),+ ]) {+ stdoutSpy.mockClear()+ const err = new Error(`operation failed: ${json}`)+ createLogger('wrc', 'r').error('failed', { err })+ const diagnostic = JSON.parse(stdoutSpy.mock.calls[0][0]).err+ expect(diagnostic.message).toContain('operation failed')+ expect(diagnostic.message).toContain('[REDACTED]')+ expect(diagnostic.message).not.toContain('x')+ expect(diagnostic.stack).toContain('[REDACTED]')+ expect(diagnostic.stack).not.toContain(json)+ }+ }+ )++ it('serializes native errors from another JavaScript context', () => {+ const err: unknown = runInNewContext('new Error("isolated operation failed")')+ createLogger('wrc', 'r').error('failed', { err })+ expect(JSON.parse(stdoutSpy.mock.calls[0][0]).err.message).toBe('isolated operation failed')+ })++ it('bounds oversized diagnostic strings before serializing', () => {+ const err = new Error('a'.repeat(20_000))+ createLogger('wrc', 'r').error('failed', { err })+ const entry = JSON.parse(stdoutSpy.mock.calls[0][0])+ expect(entry.err.message).toBe('[Truncated]')+ expect(entry.err.stack).toBe('[Truncated]')+ })++ it('does not throw on a custom Error accessor or a cyclic cause array', () => {+ const unreadable = new Error('unreadable cause')+ Object.defineProperty(unreadable, 'cause', {+ get() {+ throw new Error('accessor failed')+ },+ })+ const cycle: unknown[] = []+ cycle.push(cycle)+ expect(() =>+ createLogger('wrc', 'r').error('failed', {+ unreadable,+ err: new Error('array cause', { cause: cycle }),+ })+ ).not.toThrow()+ const entry = JSON.parse(stdoutSpy.mock.calls[0][0])+ expect(entry.unreadable).toEqual({ name: 'Error', message: '[Unserializable]' })+ expect(entry.err.cause).toEqual(['[Circular]'])+ })+it('redacts sensitive top-level and nested fields', () => {const log = createLogger('mcp_host', 'r')log.info('request', {