Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript

fix(egress): close Astra review findings and harden runtime proofs

Alfredo Lopezb4a78cb73037
diff --git a/.github/workflows/ci-public.yml b/.github/workflows/ci-public.yml
index ac616d14e..c1b01c3ef 100644
--- a/.github/workflows/ci-public.yml
+++ b/.github/workflows/ci-public.yml
@@ -413,6 +413,10 @@ jobs:
with:
fetch-depth: 0
+ - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4
+ with:
+ node-version: '24'
+
- name: Run bash -n on all .sh files
run: |
fail=0
@@ -461,9 +465,12 @@ jobs:
- name: Test WRC internal dependency NetworkPolicy contracts
run: bash scripts/tests/test-wrc-internal-dependency-networkpolicy-gate.sh
- - name: Test WRC external-egress degradation gate contract
+ - name: Test WRC egress oracle, DNS and lifecycle (unit/integration, cluster-free)
run: bash scripts/tests/test-wrc-egress-degradation-gate.sh
+ - name: Test Minikube mutation and WRC recovery admission (hermetic, cluster-free)
+ run: bash scripts/tests/test-minikube-mutation-boundary.sh
+
- name: Test minikube sandbox-ui NetworkPolicy lint (static-policy ownership + DNS)
run: bash scripts/tests/test-lint-networkpolicies.sh
diff --git a/Makefile b/Makefile
index ae1911166..de2acb4fe 100644
--- a/Makefile
+++ b/Makefile
@@ -1265,7 +1265,18 @@ test-e2e-hcc-communicationchannel-watch-recovery: ## Run isolated minikube HCC w
test-e2e-wrc-egress-degradation: ## Prove valid UI/workload services survive transient DNS while removed/raced egress is contracted
@echo "Running WRC external-egress degradation gate..."
@test -n "$(E2E_EXPECTED_PRE_GATE_GATE)" || { echo "Set E2E_EXPECTED_PRE_GATE_GATE to the gate recorded by the branch-owned pre-gate sync" >&2; exit 1; }
- E2E_WRC_EGRESS_FAULT_INJECTION=1 E2E_EXPECTED_PRE_GATE_GATE="$(E2E_EXPECTED_PRE_GATE_GATE)" MINIKUBE_PROFILE=$(E2E_KUBECONTEXT) KUBECONTEXT=$(E2E_KUBECONTEXT) bash scripts/e2e/e2e-wrc-egress-degradation.sh
+ @T2_PROJECT_DIR="$(CURDIR)" T2_PROFILE="$(E2E_KUBECONTEXT)" T2_CONTEXT="$(E2E_KUBECONTEXT)" \
+ T2_SKIP_LOCK="$(T2_SKIP_LOCK)" T2_LOCK_TOKEN="$(T2_LOCK_TOKEN)" \
+ E2E_WRC_EGRESS_FAULT_INJECTION=1 E2E_EXPECTED_PRE_GATE_GATE="$(E2E_EXPECTED_PRE_GATE_GATE)" \
+ MINIKUBE_PROFILE="$(E2E_KUBECONTEXT)" KUBECONTEXT="$(E2E_KUBECONTEXT)" \
+ bash scripts/minikube/with-t2-mutation-lock.sh -- bash scripts/e2e/e2e-wrc-egress-degradation.sh
+
+.PHONY: test-e2e-wrc-egress-recover
+test-e2e-wrc-egress-recover: ## Restore only the recorded, owned WRC DNS fault intervention; never certify the interrupted gate
+ @T2_PROJECT_DIR="$(CURDIR)" T2_PROFILE="$(E2E_KUBECONTEXT)" T2_CONTEXT="$(E2E_KUBECONTEXT)" \
+ T2_SKIP_LOCK="$(T2_SKIP_LOCK)" T2_LOCK_TOKEN="$(T2_LOCK_TOKEN)" \
+ MINIKUBE_PROFILE="$(E2E_KUBECONTEXT)" KUBECONTEXT="$(E2E_KUBECONTEXT)" \
+ bash scripts/minikube/with-t2-mutation-lock.sh --recover-wrc-egress
.PHONY: test-e2e-hcc-readiness-bootstrap
test-e2e-hcc-readiness-bootstrap: ## Prove HCC readiness while its initial Host fleet pass remains active
diff --git a/host-context-controller/package-lock.json b/host-context-controller/package-lock.json
index c390d2654..a62bb6812 100644
--- a/host-context-controller/package-lock.json
+++ b/host-context-controller/package-lock.json
@@ -1,12 +1,12 @@
{
"name": "clerum-host-context-controller",
- "version": "0.1.296",
+ "version": "0.1.297",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "clerum-host-context-controller",
- "version": "0.1.296",
+ "version": "0.1.297",
"license": "SEE LICENSE IN LICENSE",
"dependencies": {
"@clerum/codex-catalog-projection": "file:../packages/codex-catalog-projection",
diff --git a/host-context-controller/package.json b/host-context-controller/package.json
index dd067e323..55d3bb0c6 100644
--- a/host-context-controller/package.json
+++ b/host-context-controller/package.json
@@ -1,6 +1,6 @@
{
"name": "clerum-host-context-controller",
- "version": "0.1.296",
+ "version": "0.1.297",
"description": "Host Context Controller - provides McpServer CRDs to mcp-host via REST API",
"main": "dist/main.js",
"engines": {
diff --git a/host-context-controller/src/__tests__/applyNetworkPolicyNoopGate.test.ts b/host-context-controller/src/__tests__/applyNetworkPolicyNoopGate.test.ts
index f55690950..99e3ced92 100644
--- a/host-context-controller/src/__tests__/applyNetworkPolicyNoopGate.test.ts
+++ b/host-context-controller/src/__tests__/applyNetworkPolicyNoopGate.test.ts
@@ -120,6 +120,33 @@ describe('applyNetworkPolicy no-op gate', () => {
expect(api.replaceNamespacedNetworkPolicy).not.toHaveBeenCalled()
})
+ it.each([
+ { form: 'ApiException', error: apiException(404) },
+ { form: 'statusCode', error: { response: { statusCode: 404 } } },
+ ])(
+ 'admission-sensitive create409/read404 ($form) rejects instead of certifying absence',
+ async ({ error }) => {
+ const api = fakeNetworkingApi()
+ api.createNamespacedNetworkPolicy.mockRejectedValue({ code: 409 })
+ api.readNamespacedNetworkPolicy.mockRejectedValue(error)
+
+ await expect(
+ applyNetworkPolicy(
+ api as unknown as k8s.NetworkingV1Api,
+ 'np',
+ 'ns',
+ desiredPolicy(),
+ '[NetPol]',
+ undefined,
+ undefined,
+ true
+ )
+ ).rejects.toBe(error)
+
+ expect(api.replaceNamespacedNetworkPolicy).not.toHaveBeenCalled()
+ }
+ )
+
it.each([
{ status: 403, form: 'ApiException', error: apiException(403) },
{ status: 403, form: 'statusCode', error: { response: { statusCode: 403 } } },
diff --git a/host-context-controller/src/__tests__/replaceWithConflictRetryOrder.test.ts b/host-context-controller/src/__tests__/replaceWithConflictRetryOrder.test.ts
index fb195364d..b7688dc2b 100644
--- a/host-context-controller/src/__tests__/replaceWithConflictRetryOrder.test.ts
+++ b/host-context-controller/src/__tests__/replaceWithConflictRetryOrder.test.ts
@@ -107,9 +107,16 @@ describe('replaceWithConflictRetry order and retry', () => {
expect(replace.mock.calls[0][0].metadata?.resourceVersion).toBe('9')
expect(replace.mock.calls[1][0].metadata?.resourceVersion).toBe('10')
const updated = log.mock.calls
- .map((call: unknown[]) => String(call[0]))
- .filter((line: string) => line.includes('[Test] Updated Service "svc"'))
- expect(updated).toEqual(['[Test] Updated Service "svc" (after 2 attempts)'])
+ .map((call: unknown[]) => JSON.parse(String(call[0])))
+ .filter(entry => entry.msg === 'Kubernetes resource updated')
+ expect(updated).toEqual([
+ expect.objectContaining({
+ level: 'info',
+ scope: '[Test]',
+ description: 'Service "svc"',
+ attempt: 2,
+ }),
+ ])
} finally {
log.mockRestore()
}
@@ -355,9 +362,13 @@ describe('replaceWithConflictRetry order and retry', () => {
expect(replace).not.toHaveBeenCalled()
expect(await readWritesTotal('Service')).toBe(before)
- expect(warn).toHaveBeenCalledWith(
- expect.stringMatching(/disappeared \(404\); skipping replace/)
- )
+ expect(warn).toHaveBeenCalledOnce()
+ expect(JSON.parse(String(warn.mock.calls[0][0]))).toMatchObject({
+ level: 'warn',
+ msg: 'Kubernetes resource disappeared; skipping replace',
+ scope: '[Test]',
+ description: 'Service "svc"',
+ })
warn.mockRestore()
})
diff --git a/host-context-controller/src/networkPolicyReconciler.egressObservation.test.ts b/host-context-controller/src/networkPolicyReconciler.egressObservation.test.ts
new file mode 100644
index 000000000..4d833a8e3
--- /dev/null
+++ b/host-context-controller/src/networkPolicyReconciler.egressObservation.test.ts
@@ -0,0 +1,408 @@
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
+import * as k8s from '@kubernetes/client-node'
+import * as dns from 'node:dns/promises'
+import { ExternalEgressConvergenceCoordinator } from './externalEgressConvergenceCoordinator'
+import { NetworkPolicyReconciler } from './networkPolicyReconciler'
+import type { EgressBinding, McpServerCRD, McpServerCrdStatus } from './types'
+
+vi.mock('./config', () => ({
+ config: {
+ namespace: 'mcp-server',
+ controlPlaneNamespace: 'control-plane',
+ hostNamespace: 'mcp-host',
+ rpcProxyNamespace: 'rpc-proxy',
+ runtimeNamespaces: ['mcp-server'],
+ externalEgressDnsResolveTimeoutMs: 5000,
+ externalEgressOverlapSec: 300,
+ externalEgressMaxEntries: 128,
+ },
+}))
+vi.mock('node:dns/promises', () => ({ resolve4: vi.fn() }))
+vi.mock('./metrics', () => ({
+ writesTotal: { inc: vi.fn() },
+ writeSkipsTotal: { inc: vi.fn() },
+ externalEgressRetriesAtCap: { set: vi.fn() },
+}))
+
+const currentDns: EgressBinding = { dns: 'current.example.com', port: 443 }
+const removedDns: EgressBinding = { dns: 'removed.example.com', port: 8443 }
+const currentName = 'ext-egress-observation-mcp-current.example.com-443'
+const removedName = 'ext-egress-observation-mcp-removed.example.com-8443'
+const apiError = (code: number) => Object.assign(new Error(`Kubernetes API ${code}`), { code })
+
+function fixture(bindings: EgressBinding[]) {
+ const server: McpServerCRD = {
+ name: 'observation-mcp',
+ namespace: 'mcp-server',
+ uid: 'server-uid',
+ generation: 7,
+ spec: {
+ contextRef: 'dev',
+ image: 'observation:1',
+ transport: { type: 'streamableHttp', port: 3000 },
+ egressBindings: bindings,
+ },
+ }
+ const policies = new Map<string, k8s.V1NetworkPolicy>()
+ const unavailableReads = new Set<string>()
+ const unavailableDeletes = new Set<string>()
+ let status: McpServerCrdStatus = {}
+ let version = 0
+ const api = {
+ listNamespacedNetworkPolicy: vi.fn(async () => ({
+ items: [...policies.values()].map(policy => structuredClone(policy)),
+ })),
+ readNamespacedNetworkPolicy: vi.fn(async ({ name }: { name: string }) => {
+ if (unavailableReads.has(name)) throw apiError(503)
+ const policy = policies.get(name)
+ if (!policy) throw apiError(404)
+ return structuredClone(policy)
+ }),
+ createNamespacedNetworkPolicy: vi.fn(async ({ body }: { body: k8s.V1NetworkPolicy }) => {
+ const name = body.metadata!.name!
+ if (policies.has(name)) throw apiError(409)
+ const policy = structuredClone(body)
+ policy.metadata = { ...policy.metadata, uid: `uid-${name}`, resourceVersion: `${++version}` }
+ policies.set(name, policy)
+ return structuredClone(policy)
+ }),
+ replaceNamespacedNetworkPolicy: vi.fn(
+ async ({ name, body }: { name: string; body: k8s.V1NetworkPolicy }) => {
+ const current = policies.get(name)
+ if (!current) throw apiError(404)
+ if (body.metadata?.resourceVersion !== current.metadata?.resourceVersion)
+ throw apiError(409)
+ const policy = structuredClone(body)
+ policy.metadata = {
+ ...policy.metadata,
+ uid: current.metadata!.uid,
+ resourceVersion: `${++version}`,
+ }
+ policies.set(name, policy)
+ return structuredClone(policy)
+ }
+ ),
+ deleteNamespacedNetworkPolicy: vi.fn(
+ async ({ name, body }: { name: string; body: k8s.V1DeleteOptions }) => {
+ if (unavailableDeletes.has(name)) throw apiError(503)
+ const current = policies.get(name)
+ if (!current) throw apiError(404)
+ if (
+ body.preconditions?.uid !== current.metadata?.uid ||
+ body.preconditions?.resourceVersion !== current.metadata?.resourceVersion
+ )
+ throw apiError(409)
+ policies.delete(name)
+ }
+ ),
+ }
+ const customApi = {
+ getNamespacedCustomObjectStatus: vi.fn(async () => ({
+ metadata: { uid: server.uid, generation: server.generation, resourceVersion: 'status-rv' },
+ status: structuredClone(status),
+ })),
+ patchNamespacedCustomObjectStatus: vi.fn(
+ async ({ body }: { body: Array<{ op: string; path: string; value: unknown }> }) => {
+ for (const operation of body) {
+ if (operation.path === '/status') status = operation.value as McpServerCrdStatus
+ if (operation.path === '/status/conditions')
+ status.conditions = operation.value as McpServerCrdStatus['conditions']
+ if (operation.path === '/status/resolvedEgressIPs')
+ status.resolvedEgressIPs = operation.value as McpServerCrdStatus['resolvedEgressIPs']
+ }
+ }
+ ),
+ }
+ const kc = new k8s.KubeConfig()
+ kc.loadFromOptions({
+ clusters: [{ name: 'unit', server: 'https://unit.invalid' }],
+ users: [{ name: 'unit' }],
+ contexts: [{ name: 'unit', cluster: 'unit', user: 'unit' }],
+ currentContext: 'unit',
+ })
+ const reconciler = new NetworkPolicyReconciler(kc, new Map())
+ Object.assign(reconciler, { networkingApi: api, customApi })
+ return {
+ server,
+ policies,
+ api,
+ unavailableReads,
+ unavailableDeletes,
+ reconciler,
+ status: () => status,
+ condition: () => status.conditions?.find(condition => condition.type === 'ExternalEgressReady'),
+ run: () => reconciler.reconcileExternalEgress(server, { isCurrent: () => true }),
+ clearCalls: () => Object.values(api).forEach(mock => mock.mockClear()),
+ }
+}
+
+describe('external egress observation failures do not decide authorization', () => {
+ beforeEach(() => {
+ vi.mocked(dns.resolve4).mockResolvedValue([{ address: '1.2.3.4', ttl: 300 }] as never)
+ })
+ afterEach(() => vi.useRealTimers())
+
+ it('keeps the current policy unmodified and revokes a removed sibling when fresh DNS-policy reads are unavailable', async () => {
+ const f = fixture([removedDns, currentDns])
+ await f.run()
+ const before = structuredClone(f.policies.get(currentName))
+ const removed = f.policies.get(removedName)!
+ f.server.spec.egressBindings = [currentDns]
+ f.unavailableReads.add(currentName)
+ f.clearCalls()
+
+ await expect(f.run()).rejects.toThrow('Kubernetes API 503')
+
+ expect(f.policies.get(currentName)).toEqual(before)
+ expect(f.api.createNamespacedNetworkPolicy).not.toHaveBeenCalled()
+ expect(f.api.replaceNamespacedNetworkPolicy).not.toHaveBeenCalled()
+ expect(f.api.deleteNamespacedNetworkPolicy).toHaveBeenCalledExactlyOnceWith({
+ name: removedName,
+ namespace: f.server.namespace,
+ body: {
+ preconditions: {
+ uid: removed.metadata!.uid,
+ resourceVersion: removed.metadata!.resourceVersion,
+ },
+ },
+ })
+ expect(f.condition()).toMatchObject({
+ status: 'False',
+ reason: 'ExternalEgressReconcileFailed',
+ })
+ expect(f.condition()?.message).toMatch(/unavailable|could not be verified/)
+ expect(f.condition()?.message).not.toMatch(/REVOKED|provenance was absent or invalid/)
+ expect(f.status().resolvedEgressIPs).toEqual([])
+
+ f.unavailableReads.clear()
+ await f.run()
+ expect(f.condition()).toMatchObject({ status: 'True', reason: 'Reconciled' })
+ expect(f.policies.get(currentName)).toEqual(before)
+ expect(f.policies.has(removedName)).toBe(false)
+ })
+
+ it.each([
+ {
+ binding: { cidr: '8.8.8.8/32', port: 9443 },
+ name: 'ext-egress-observation-mcp-8-8-8-8-32-9443',
+ },
+ {
+ binding: { egressClass: 'public-web' as const },
+ name: 'ext-egress-observation-mcp-public-web',
+ },
+ ])(
+ 'revokes an independent removed binding despite a $name read failure and recovers without a spec change',
+ async ({ binding, name }) => {
+ const f = fixture([removedDns, binding])
+ await f.run()
+ const before = structuredClone(f.policies.get(name))
+ f.server.spec.egressBindings = [binding]
+ f.unavailableReads.add(name)
+ f.clearCalls()
+
+ await expect(f.run()).rejects.toThrow('Kubernetes API 503')
+
+ expect(f.policies.has(removedName)).toBe(false)
+ expect(f.policies.get(name)).toEqual(before)
+ expect(f.api.createNamespacedNetworkPolicy).not.toHaveBeenCalled()
+ expect(f.api.replaceNamespacedNetworkPolicy).not.toHaveBeenCalled()
+ expect(f.condition()).toMatchObject({
+ status: 'False',
+ reason: 'ExternalEgressReconcileFailed',
+ })
+ expect(f.status().resolvedEgressIPs).toEqual([])
+ f.unavailableReads.clear()
+ await f.run()
+ expect(f.condition()).toMatchObject({ status: 'True', reason: 'Reconciled' })
+ expect(f.policies.get(name)).toEqual(before)
+ }
+ )
+
+ it('does not resolve or expand until the independent removal barrier succeeds', async () => {
+ const f = fixture([removedDns])
+ await f.run()
+ f.server.spec.egressBindings = [currentDns]
+ f.unavailableDeletes.add(removedName)
+ f.clearCalls()
+ vi.mocked(dns.resolve4).mockClear()
+
+ await expect(f.run()).rejects.toThrow('Kubernetes API 503')
+
+ expect(dns.resolve4).not.toHaveBeenCalled()
+ expect(f.api.createNamespacedNetworkPolicy).not.toHaveBeenCalled()
+ expect(f.condition()).toMatchObject({ status: 'False', reason: 'CleanupFailed' })
+ f.unavailableDeletes.clear()
+ await f.run()
+ expect(f.policies.has(removedName)).toBe(false)
+ expect(f.policies.has(currentName)).toBe(true)
+ })
+
+ it('finishes independent removals when one delete fails, without expanding another binding', async () => {
+ const f = fixture([removedDns, currentDns])
+ await f.run()
+ f.server.spec.egressBindings = [{ dns: 'added.example.com', port: 443 }]
+ f.unavailableDeletes.add(removedName)
+ f.clearCalls()
+
+ await expect(f.run()).rejects.toThrow('Kubernetes API 503')
+
+ expect(f.policies.has(removedName)).toBe(true)
+ expect(f.policies.has(currentName)).toBe(false)
+ expect(f.api.createNamespacedNetworkPolicy).not.toHaveBeenCalled()
+ expect(f.condition()?.reason).toBe('CleanupFailed')
+ })
+
+ it('does not add a sibling permission while a current policy observation is pending', async () => {
+ const f = fixture([currentDns])
+ await f.run()
+ f.server.spec.egressBindings = [{ cidr: '8.8.8.8/32', port: 443 }, currentDns]
+ f.unavailableReads.add(currentName)
+ f.clearCalls()
+
+ await expect(f.run()).rejects.toThrow('Kubernetes API 503')
+
+ expect(f.api.createNamespacedNetworkPolicy).not.toHaveBeenCalled()
+ expect(f.api.replaceNamespacedNetworkPolicy).not.toHaveBeenCalled()
+ expect(f.policies.size).toBe(1)
+ expect(f.status().resolvedEgressIPs).toEqual([])
+ })
+
+ it('rechecks a deferred no-op after sibling DNS latency and recovers from the changed snapshot', async () => {
+ const sibling: EgressBinding = { dns: 'later.example.com', port: 443 }
+ const f = fixture([currentDns, sibling])
+ await f.run()
+ f.clearCalls()
+ vi.mocked(dns.resolve4).mockImplementation(async hostname => {
+ if (hostname === sibling.dns) {
+ const policy = f.policies.get(currentName)!
+ policy.metadata!.resourceVersion = 'concurrent-rv'
+ policy.spec!.egress!.push({
+ to: [{ ipBlock: { cidr: '8.8.8.8/32' } }],
+ ports: [{ port: 443, protocol: 'TCP' }],
+ })
+ }
+ return [{ address: '1.2.3.4', ttl: 300 }] as never
+ })
+
+ await expect(f.run()).rejects.toThrow(/changed after the external-egress observation/)
+
+ expect(f.condition()?.status).toBe('False')
+ expect(f.status().resolvedEgressIPs).toEqual([])
+ expect(f.api.replaceNamespacedNetworkPolicy).not.toHaveBeenCalled()
+ vi.mocked(dns.resolve4).mockResolvedValue([{ address: '1.2.3.4', ttl: 300 }] as never)
+ await f.run()
+ expect(f.condition()?.status).toBe('True')
+ expect(f.policies.get(currentName)?.spec?.egress).toHaveLength(1)
+ })
+
+ it('does not write or report status after the generation fence closes during an unavailable read', async () => {
+ const f = fixture([currentDns])
+ await f.run()
+ f.clearCalls()
+ const statusBefore = structuredClone(f.status())
+ let current = true
+ f.api.readNamespacedNetworkPolicy.mockImplementation(async () => {
+ current = false
+ throw apiError(503)
+ })
+
+ await f.reconciler.reconcileExternalEgress(f.server, { isCurrent: () => current })
+
+ expect(f.api.deleteNamespacedNetworkPolicy).not.toHaveBeenCalled()
+ expect(f.api.createNamespacedNetworkPolicy).not.toHaveBeenCalled()
+ expect(f.api.replaceNamespacedNetworkPolicy).not.toHaveBeenCalled()
+ expect(f.status()).toEqual(statusBefore)
+ })
+
+ it('does not copy a newer resourceVersion onto a policy calculated before an apply conflict', async () => {
+ const f = fixture([currentDns])
+ await f.run()
+ const policy = f.policies.get(currentName)!
+ policy.spec!.egress!.push({
+ to: [{ ipBlock: { cidr: '8.8.8.8/32' } }],
+ ports: [{ port: 443, protocol: 'TCP' }],
+ })
+ f.clearCalls()
+ f.api.createNamespacedNetworkPolicy.mockImplementationOnce(async () => {
+ // The conflict retry reads this newer object after the final plan check.
+ // A matching owner alone cannot authorize reuse of the older DNS state.
+ policy.metadata!.resourceVersion = 'changed-before-replace'
+ throw apiError(409)
+ })
+
+ await expect(f.run()).rejects.toThrow(/changed after the external-egress observation/)
+
+ expect(f.api.replaceNamespacedNetworkPolicy).not.toHaveBeenCalled()
+ expect(f.condition()?.status).toBe('False')
+ await f.run()
+ expect(f.condition()?.status).toBe('True')
+ expect(f.policies.get(currentName)?.spec?.egress).toHaveLength(1)
+ })
+
+ it('does not certify a policy deleted after create conflict and before replacement read, then recovers', async () => {
+ const f = fixture([currentDns])
+ await f.run()
+ vi.mocked(dns.resolve4).mockResolvedValue([{ address: '8.8.8.8', ttl: 300 }] as never)
+ f.api.createNamespacedNetworkPolicy.mockImplementationOnce(async () => {
+ // POST sees the existing policy and conflicts. A concurrent deletion then
+ // wins before the helper's replacement GET, which observes real absence.
+ f.policies.delete(currentName)
+ throw apiError(409)
+ })
+
+ await expect(f.run()).rejects.toThrow()
+
+ expect(f.policies.has(currentName)).toBe(false)
+ expect(f.condition()).toMatchObject({
+ status: 'False',
+ reason: 'ExternalEgressReconcileFailed',
+ })
+ expect(f.status().resolvedEgressIPs).toEqual([])
+ await f.run()
+ expect(f.condition()).toMatchObject({ status: 'True', reason: 'Reconciled' })
+ expect(f.policies.get(currentName)?.spec?.egress).toEqual([
+ { to: [{ ipBlock: { cidr: '8.8.8.8/32' } }], ports: [{ port: 443, protocol: 'TCP' }] },
+ ])
+ expect(f.status().resolvedEgressIPs?.[0].ips).toEqual(['8.8.8.8'])
+ })
+
+ it('keeps the existing retry coordinator pending until the API recovers and then admits the same generation once', async () => {
+ vi.useFakeTimers()
+ const f = fixture([currentDns])
+ await f.run()
+ f.unavailableReads.add(currentName)
+ const admitted = vi.fn()
+ let coordinator: ExternalEgressConvergenceCoordinator
+ coordinator = new ExternalEgressConvergenceCoordinator({
+ listServers: () => [f.server],
+ getCurrentServer: () => f.server,
+ inventoryAuthoritative: () => true,
+ sameDesiredRevision: (left, right) =>
+ left.uid === right.uid && left.generation === right.generation,
+ enqueue: async (_server, work) => work(),
+ mutate: async (_type, server, options) =>
+ f.reconciler.reconcileExternalEgress(server, options),
+ replay: async (type, server, retry) => {
+ const completion = await coordinator.reconcile(type, server, { retry })
+ admitted(server.generation)
+ completion?.complete()
+ },
+ externalEgressRefreshMinTtlMs: () => f.reconciler.externalEgressRefreshMinTtlMs,
+ })
+ try {
+ const gate = coordinator.prepareStartupGates([f.server])
+ expect(await gate.waitFor(f.server)).toBe(false)
+ await vi.advanceTimersByTimeAsync(5000)
+ expect(admitted).not.toHaveBeenCalled()
+ expect(f.condition()?.status).toBe('False')
+ f.unavailableReads.clear()
+ await vi.advanceTimersByTimeAsync(15000)
+ expect(admitted).toHaveBeenCalledExactlyOnceWith(7)
+ expect(f.condition()?.status).toBe('True')
+ await vi.advanceTimersByTimeAsync(60000)
+ expect(admitted).toHaveBeenCalledTimes(1)
+ } finally {
+ coordinator.stop()
+ }
+ })
+})
diff --git a/host-context-controller/src/networkPolicyReconciler.test.ts b/host-context-controller/src/networkPolicyReconciler.test.ts
index 0073d2b46..94c849a20 100644
--- a/host-context-controller/src/networkPolicyReconciler.test.ts
+++ b/host-context-controller/src/networkPolicyReconciler.test.ts
@@ -4018,9 +4018,12 @@ describe('NetworkPolicyReconciler', () => {
expect(error).toBeInstanceOf(AggregateError)
expect((error as AggregateError).errors).toEqual([additiveFailure])
expect(ordering).toEqual(['certify', 'context:aaa-poisoned', 'context:bbb-healthy'])
- expect(errorLog).toHaveBeenCalledWith(
- '[NetPol] Additive Context reconciliation failed for "aaa-poisoned":',
- additiveFailure
+ expect(errorLog.mock.calls.map(([line]) => JSON.parse(String(line)))).toContainEqual(
+ expect.objectContaining({
+ level: 'error',
+ contextId: 'aaa-poisoned',
+ err: { name: 'Error', message: additiveFailure.message },
+ })
)
})
diff --git a/host-context-controller/src/networkPolicyReconciler.ts b/host-context-controller/src/networkPolicyReconciler.ts
index 9ffb525bf..6788c3710 100644
--- a/host-context-controller/src/networkPolicyReconciler.ts
+++ b/host-context-controller/src/networkPolicyReconciler.ts
@@ -167,12 +167,18 @@ type ProvenDnsAnnotations = {
contracted: boolean
}
+type ExternalEgressDisposition =
+ | { state: 'pending' }
+ | { state: 'retained' }
+ | { state: 'absent' }
+ | { state: 'revoke'; policy: k8s.V1NetworkPolicy }
+
/**
* What the ExternalEgressReady message says about the live policy on a non-DNS
* fault. One entry per verdict state so a new state cannot be added without
* deciding what the operator is told.
*/
-const LIVE_EGRESS_FAULT_TEXT: Record<LiveEgressVerdict['state'], string> = {
+const LIVE_EGRESS_FAULT_TEXT: Record<LiveEgressVerdict['state'] | 'unavailable', string> = {
none: 'no live policy to serve',
foreign: 'homonymous live policy is not HCC-owned and was left untouched',
intact: 'verified policy retained for existing pods; runtime remains blocked',
@@ -180,6 +186,7 @@ const LIVE_EGRESS_FAULT_TEXT: Record<LiveEgressVerdict['state'], string> = {
'live policy REVOKED: it enforced a CIDR outside the /32 allowlist this controller writes',
'drifted-shape': 'live policy REVOKED: its egress shape is not one this controller writes',
'untrusted-state': 'live policy REVOKED: its DNS provenance was absent or invalid',
+ unavailable: 'live policy could not be verified and was left untouched; runtime remains blocked',
}
function hasExpectedPolicyOwnership(policy: k8s.V1NetworkPolicy, policyType: string): boolean {
@@ -569,6 +576,40 @@ export class NetworkPolicyReconciler {
return `ext-egress-${serverName}-${suffix}-${binding.port}`.slice(0, 253)
}
+ private externalEgressBindingError(binding: EgressBinding): string | null {
+ const egressClass = binding.egressClass ?? 'exact-host'
+ if (egressClass !== 'exact-host' && egressClass !== 'public-web') {
+ return `egressClass "${String(binding.egressClass)}" is not supported`
+ }
+ if (egressClass === 'public-web') {
+ return binding.dns ||
+ binding.cidr ||
+ binding.port !== undefined ||
+ binding.protocol !== undefined
+ ? 'public-web external egress bindings must not declare dns, cidr, port, or protocol'
+ : null
+ }
+ if (!binding.dns && !binding.cidr) {
+ return 'exact-host external egress bindings must declare dns or cidr'
+ }
+ if (
+ binding.port === undefined ||
+ !Number.isInteger(binding.port) ||
+ binding.port < 1 ||
+ binding.port > 65535
+ ) {
+ return 'exact-host external egress bindings must declare an integer port from 1-65535'
+ }
+ if (binding.cidr) {
+ return isAllowedExternalEgressCidr(binding.cidr)
+ ? null
+ : `CIDR "${binding.cidr}" overlaps private, metadata, link-local, multicast, documentation, or reserved ranges`
+ }
+ return isPublicDnsHostname(binding.dns!)
+ ? null
+ : `hostname "${binding.dns}" is private, internal, metadata, local, or otherwise disallowed`
+ }
+
// ─── Default Policies (L0 + L1) ─────────────────────────────────────
/**
@@ -583,7 +624,7 @@ export class NetworkPolicyReconciler {
await this.ensureAllowContextMapperApi()
await this.deleteLegacyStaticPolicy(config.rpcProxyNamespace, 'allow-desktop-egress-rpc-proxy')
await this.deleteLegacyStaticPolicy(config.namespace, 'allow-rpc-proxy-to-managed-mcp-servers')
- console.log('[NetPol] Default policies ensured')
+ hccLogger.info('default network policies ensured')
}
/**
@@ -970,18 +1011,17 @@ export class NetworkPolicyReconciler {
})
if (!isCurrent()) return false
- console.log(
- `[NetPol] Reconciling context "${contextId}" — allowed servers: [${allowedServers.join(', ')}]`
- )
+ hccLogger.info('reconciling context network policies', { contextId, allowedServers })
// Create or update policies for each allowed server.
for (const serverName of allowedServers) {
if (!isCurrent()) return false
const server = this.serverCache.get(serverName)
if (!server) {
- console.warn(
- `[NetPol] McpServer "${serverName}" not found in cache — skipping policy for context "${contextId}"`
- )
+ hccLogger.warn('McpServer absent from cache; context policy skipped', {
+ serverName,
+ contextId,
+ })
continue
}
@@ -1109,14 +1149,14 @@ export class NetworkPolicyReconciler {
const existingName = existing.metadata?.name || ''
const desired = desiredIngress.get(existingName)
if (!desired) {
- console.log(`[NetPol] Deleting orphaned policy "${existingName}"`)
+ hccLogger.info('deleting orphaned context policy', { policy: existingName })
if (!(await revokeOrphanedPolicy(config.namespace, existing))) return false
onRevoked?.()
} else if (
!sameNetworkPolicySpec(existing, desired) ||
!hasExpectedPolicyOwnership(existing, 'context-allow')
) {
- console.log(`[NetPol] Replacing stale same-name policy "${existingName}"`)
+ hccLogger.info('replacing stale context policy', { policy: existingName })
if (safetySnapshotProvided) {
if (
!(await this.replaceSafetyPolicySnapshot(
@@ -1150,14 +1190,14 @@ export class NetworkPolicyReconciler {
const existingName = existing.metadata?.name || ''
const desired = desiredHostEgress.get(existingName)
if (!desired) {
- console.log(`[NetPol] Deleting orphaned L2 egress policy "${existingName}"`)
+ hccLogger.info('deleting orphaned L2 egress policy', { policy: existingName })
if (!(await revokeOrphanedPolicy(config.hostNamespace, existing))) return false
onRevoked?.()
} else if (
!sameNetworkPolicySpec(existing, desired) ||
!hasExpectedPolicyOwnership(existing, 'context-allow')
) {
- console.log(`[NetPol] Replacing stale same-name L2 egress policy "${existingName}"`)
+ hccLogger.info('replacing stale L2 egress policy', { policy: existingName })
if (safetySnapshotProvided) {
if (
!(await this.replaceSafetyPolicySnapshot(
@@ -1191,14 +1231,14 @@ export class NetworkPolicyReconciler {
const existingName = existing.metadata?.name || ''
const desired = desiredRpcProxyEgress.get(existingName)
if (!desired) {
- console.log(`[NetPol] Deleting orphaned rpc-proxy egress policy "${existingName}"`)
+ hccLogger.info('deleting orphaned rpc-proxy egress policy', { policy: existingName })
if (!(await revokeOrphanedPolicy(config.rpcProxyNamespace, existing))) return false
onRevoked?.()
} else if (
!sameNetworkPolicySpec(existing, desired) ||
!hasExpectedPolicyOwnership(existing, RPC_PROXY_EGRESS_POLICY_TYPE)
) {
- console.log(`[NetPol] Replacing stale same-name rpc-proxy egress policy "${existingName}"`)
+ hccLogger.info('replacing stale rpc-proxy egress policy', { policy: existingName })
if (safetySnapshotProvided) {
if (
!(await this.replaceSafetyPolicySnapshot(
@@ -1234,7 +1274,7 @@ export class NetworkPolicyReconciler {
contextId: string,
deleteAllowed?: () => Promise<boolean>
): Promise<void> {
- console.log(`[NetPol] Context "${contextId}" deleted — removing all policies`)
+ hccLogger.info('removing policies for deleted context', { contextId })
const policies = await this.listPoliciesForContext(contextId)
for (const policy of policies) {
@@ -1319,7 +1359,7 @@ export class NetworkPolicyReconciler {
servers: McpServerCRD[] = [],
options: NetworkPolicyFullReconcileOptions = {}
): Promise<void> {
- console.log(`[NetPol] Running full reconciliation for ${contexts.length} Context(s)`)
+ hccLogger.info('running full network policy reconciliation', { contexts: contexts.length })
// This is intentionally scoped to callers that use the authoritative
// revocation callback. Ordinary additive reconciles must not be presented
@@ -1362,7 +1402,7 @@ export class NetworkPolicyReconciler {
}
if (contextAuthorityLost && !contextAuthorityWarningLogged) {
contextAuthorityWarningLogged = true
- console.warn(
+ hccLogger.warn(
'[NetPol] Skipping remaining Context policy orphan cleanup because Context inventory authority was lost'
)
}
@@ -1381,7 +1421,7 @@ export class NetworkPolicyReconciler {
}
if (serverAuthorityLost && !serverAuthorityWarningLogged) {
serverAuthorityWarningLogged = true
- console.warn(
+ hccLogger.warn(
'[NetPol] Skipping remaining external egress orphan cleanup because McpServer inventory authority was lost'
)
}
@@ -1410,9 +1450,10 @@ export class NetworkPolicyReconciler {
if (!name) {
throw new Error('HCC-managed Context NetworkPolicy is missing its name and owner label')
}
- console.warn(
- `[NetPol] Deleting malformed HCC-managed policy "${name}" without ${CONTEXT_LABEL}`
- )
+ hccLogger.warn('deleting malformed HCC-managed policy', {
+ policy: name,
+ missingLabel: CONTEXT_LABEL,
+ })
recordOrphanDelete(lane, await deletePolicy(policy))
continue
}
@@ -1428,7 +1469,7 @@ export class NetworkPolicyReconciler {
) {
return
}
- console.log(describePolicy(name, contextId))
+ hccLogger.info(describePolicy(name, contextId), { policy: name, contextId, lane })
recordOrphanDelete(lane, await deletePolicy(policy))
})
}
@@ -1524,9 +1565,12 @@ export class NetworkPolicyReconciler {
const capReason = evaluateNetPolOrphanSweepCap(orphanCandidates.length, listedManagedFleet)
if (capReason) {
netPolOrphanSweepCappedTotal.inc({ reason: capReason })
- console.warn(
- `[NetPol] ${NETWORKPOLICY_ORPHAN_SWEEP_CAPPED_MESSAGE} (${capReason}): ${orphanCandidates.length} candidates of ${listedManagedFleet} listed managed policies; refusing deletes and continuing certification`
- )
+ hccLogger.warn(NETWORKPOLICY_ORPHAN_SWEEP_CAPPED_MESSAGE, {
+ reason: capReason,
+ candidates: orphanCandidates.length,
+ listedManagedFleet,
+ action: 'refusing deletes and continuing certification',
+ })
}
// Delete orphaned Context policies across every L2 lane. Each lane uses
@@ -1614,9 +1658,10 @@ export class NetworkPolicyReconciler {
'HCC-managed external egress NetworkPolicy is missing its name and owner label'
)
}
- console.warn(
- `[NetPol] Deleting malformed HCC-managed policy "${name}" without ${MCPSERVER_LABEL}`
- )
+ hccLogger.warn('deleting malformed HCC-managed policy', {
+ policy: name,
+ missingLabel: MCPSERVER_LABEL,
+ })
recordOrphanDelete(
'external-egress',
await this.deleteSafetyPolicySnapshot(
@@ -1646,7 +1691,7 @@ export class NetworkPolicyReconciler {
) {
return
}
- console.log(`[NetPol] Deleting orphaned external egress policy "${name}"`)
+ hccLogger.info('deleting orphaned external egress policy', { policy: name })
recordOrphanDelete(
'external-egress',
await this.deleteSafetyPolicySnapshot(
@@ -1659,7 +1704,7 @@ export class NetworkPolicyReconciler {
}
}
} else if (options.serverInventoryComplete === false) {
- console.warn(
+ hccLogger.warn(
'[NetPol] Skipping external egress orphan cleanup because server inventory is incomplete'
)
}
@@ -1752,7 +1797,7 @@ export class NetworkPolicyReconciler {
})
} catch (error) {
contextAdditiveFailures.push(error)
- console.error(`[NetPol] Additive Context reconciliation failed for "${contextId}":`, error)
+ hccLogger.error('additive context reconciliation failed', { contextId, err: error })
}
}
@@ -1763,7 +1808,7 @@ export class NetworkPolicyReconciler {
)
}
- console.log('[NetPol] Full reconciliation complete')
+ hccLogger.info('full network policy reconciliation complete')
}
private async contextOrphanDeleteAllowed(
@@ -2332,27 +2377,109 @@ export class NetworkPolicyReconciler {
return
}
- console.log(
- `[NetPol] Reconciling ${bindings.length} external egress binding(s) for "${server.name}"`
- )
+ hccLogger.info('reconciling external egress bindings', {
+ serverName: server.name,
+ bindings: bindings.length,
+ })
- const desiredPolicyNames = new Set<string>()
const resolvedEgressIPs: McpServerResolvedEgressIP[] = []
const failures: string[] = []
+ const declaredBindings: Array<{ binding: EgressBinding; name: string }> = []
+ const dispositions = new Map<string, ExternalEgressDisposition>()
+ for (const binding of bindings) {
+ const error = this.externalEgressBindingError(binding)
+ const name = this.externalEgressPolicyName(server.name, binding)
+ if (error || !name) {
+ failures.push(error ?? 'exact-host external egress bindings must declare dns or cidr')
+ continue
+ }
+ declaredBindings.push({ binding, name })
+ dispositions.set(name, { state: 'pending' })
+ }
+
+ const cleanup = async (policies: k8s.V1NetworkPolicy[]): Promise<void> => {
+ const errors: unknown[] = []
+ for (const policy of policies) {
+ if (!isCurrent()) return
+ try {
+ await this.cleanupExternalEgress(
+ server.name,
+ server.namespace,
+ [policy],
+ async () => isCurrent(),
+ options.onRevoked
+ )
+ } catch (error: unknown) {
+ // Independent removals remain authorized even when one snapshot loses
+ // its fence or the API is temporarily unavailable for that object.
+ errors.push(error)
+ }
+ }
+ if (errors.length === 0 || !isCurrent()) return
+ await this.writeExternalEgressStatus(
+ server,
+ resolvedEgressIPs,
+ 'False',
+ 'CleanupFailed',
+ `Failed to delete stale external egress policies: ${errors.map(error => this.errorMessage(error)).join('; ')}`,
+ isCurrent
+ )
+ throw errors[0]
+ }
+
+ // Current intent, not success/failure of a GET, determines removal. Complete
+ // this barrier before DNS or additions; a failed removal never admits them.
+ await cleanup(existingPolicies.filter(policy => !dispositions.has(policy.metadata?.name ?? '')))
+ if (!isCurrent()) return
+ existingPolicies = existingPolicies.filter(policy =>
+ dispositions.has(policy.metadata?.name ?? '')
+ )
// Issue #513: faults that are NOT a DNS condition. Kept separate from
// `failures` because those are rejected INPUT (the operator's) while these are
// OUR bug, and the two earn different reasons and different messages.
const reconcileFaults: Array<{
- dns: string
+ binding: string
err: unknown
- live: LiveEgressVerdict['state']
+ live: LiveEgressVerdict['state'] | 'unavailable'
verificationError?: unknown
}> = []
- const freshPolicySnapshots = new Map<string, k8s.V1NetworkPolicy>()
+ const convergences: Array<{
+ name: string
+ policy: k8s.V1NetworkPolicy
+ observedPolicy: k8s.V1NetworkPolicy | null
+ writeRequired: boolean
+ resolvedStatus?: McpServerResolvedEgressIP
+ dnsFailStatic?: boolean
+ }> = []
const resolvedAtMs = Date.now()
const resolvedAt = new Date(resolvedAtMs).toISOString()
let dnsFailStatic = false
+ const recordUnavailable = (
+ binding: EgressBinding,
+ name: string,
+ err: unknown,
+ verificationError?: unknown
+ ): void => {
+ dispositions.set(name, { state: 'pending' })
+ hccLogger.error('external egress policy observation unavailable', {
+ serverName: server.name,
+ policy: name,
+ err,
+ verificationError,
+ })
+ reconcileFaults.push({
+ binding: binding.dns ?? binding.cidr ?? 'public-web',
+ err,
+ live: 'unavailable',
+ verificationError,
+ })
+ }
+
+ const revoke = (name: string, policy: k8s.V1NetworkPolicy | null): void => {
+ dispositions.set(name, policy ? { state: 'revoke', policy } : { state: 'absent' })
+ }
+
const recordReconcileFault = async (
binding: EgressBinding,
name: string,
@@ -2374,35 +2501,28 @@ export class NetworkPolicyReconciler {
dns: binding.dns,
err: verificationError,
})
- reconcileFaults.push({
- dns: binding.dns,
- err,
- live: 'untrusted-state',
- verificationError,
- })
+ recordUnavailable(binding, name, err, verificationError)
return
}
if (verdict.state === 'intact') {
- desiredPolicyNames.add(name)
+ dispositions.set(name, { state: 'retained' })
resolvedEgressIPs.push({
dns: binding.dns,
ips: verdict.cidrs.map(cidr => cidr.replace(/\/32$/, '')),
resolvedAt: verdict.resolvedAt,
})
} else if ('policy' in verdict) {
- freshPolicySnapshots.set(name, verdict.policy)
+ revoke(name, verdict.policy)
+ } else {
+ dispositions.set(name, { state: verdict.state === 'none' ? 'absent' : 'pending' })
}
- reconcileFaults.push({ dns: binding.dns, err, live: verdict.state })
+ reconcileFaults.push({ binding: binding.dns, err, live: verdict.state })
}
- for (const binding of bindings) {
+ for (const { binding, name } of declaredBindings) {
if (!isCurrent()) return
const egressClass = binding.egressClass ?? 'exact-host'
- if (egressClass !== 'exact-host' && egressClass !== 'public-web') {
- failures.push(`egressClass "${String(binding.egressClass)}" is not supported`)
- continue
- }
let cidrs: string[] = []
// Sliding-window state annotations to persist on the policy (issue #299).
@@ -2420,11 +2540,6 @@ export class NetworkPolicyReconciler {
let resolvedStatus: McpServerResolvedEgressIP | undefined
let bindingDnsFailStatic = false
let ownershipRepairRequired = false
- const name = this.externalEgressPolicyName(server.name, binding)
- if (!name) {
- failures.push('exact-host external egress bindings must declare dns or cidr')
- continue
- }
// Look up the live policy once: used to rehydrate the window (DNS branch)
// AND to decide whether a write is actually needed (audit F2/L1) — the
// write is gated on the ENFORCED spec.egress, so identical static policies
@@ -2432,61 +2547,34 @@ export class NetworkPolicyReconciler {
let existingPolicy = existingPolicies.find(p => p.metadata?.name === name) ?? null
if (egressClass === 'public-web') {
- if (
- binding.dns ||
- binding.cidr ||
- binding.port !== undefined ||
- binding.protocol !== undefined
- ) {
- failures.push(
- 'public-web external egress bindings must not declare dns, cidr, port, or protocol'
- )
- continue
- }
-
// public-web is a static policy. Preserve #205's ownership-fenced write
// (applyOwnedPolicy asserts ownership + honors the isCurrent generation
// fence) while keeping dev's F2 no-churn gate so the TTL-accelerated
// resync does not rewrite it every tick.
const policy = this.buildPublicWebEgressPolicy(server, name)
- const fresh = await this.readFreshExternalEgressPolicy(server, name)
- existingPolicy = fresh.policy
- if (fresh.repairRequired || this.externalEgressWriteNeeded(existingPolicy, policy)) {
- await this.applyOwnedPolicy(name, server.namespace, policy, 'external-egress', isCurrent)
+ let fresh: FreshExternalEgressPolicy
+ try {
+ fresh = await this.readFreshExternalEgressPolicy(server, name)
+ } catch (error: unknown) {
+ recordUnavailable(binding, name, error)
+ continue
}
+ existingPolicy = fresh.policy
if (!isCurrent()) return
- desiredPolicyNames.add(name)
+ convergences.push({
+ name,
+ policy,
+ observedPolicy: existingPolicy,
+ writeRequired:
+ fresh.repairRequired || this.externalEgressWriteNeeded(existingPolicy, policy),
+ })
continue
}
const protocol = binding.protocol ?? 'TCP'
- if (
- binding.port === undefined ||
- !Number.isInteger(binding.port) ||
- binding.port < 1 ||
- binding.port > 65535
- ) {
- failures.push(
- 'exact-host external egress bindings must declare an integer port from 1-65535'
- )
- continue
- }
-
if (binding.cidr) {
- if (!isAllowedExternalEgressCidr(binding.cidr)) {
- failures.push(
- `CIDR "${binding.cidr}" overlaps private, metadata, link-local, multicast, documentation, or reserved ranges`
- )
- continue
- }
cidrs = [binding.cidr]
} else if (binding.dns) {
- if (!isPublicDnsHostname(binding.dns)) {
- failures.push(
- `hostname "${binding.dns}" is private, internal, metadata, local, or otherwise disallowed`
- )
- continue
- }
// Sliding-window accumulation (issue #299): a single-generation snapshot
// pins one rotating IP of a provider that serves 1 A record/response, so
// the pod's next resolution lands on an un-pinned IP and egress drops.
@@ -2528,8 +2616,8 @@ export class NetworkPolicyReconciler {
const liveAnnotations = existingPolicy?.metadata?.annotations
if (resolverFailure) {
- if (fresh.policy) freshPolicySnapshots.set(name, fresh.policy)
if (resolverFailure.kind === 'permanent') {
+ revoke(name, existingPolicy)
failures.push(
`failed to resolve hostname "${binding.dns}": ${this.errorMessage(resolverFailure.error)}`
)
@@ -2538,8 +2626,9 @@ export class NetworkPolicyReconciler {
if (existingPolicy) {
const proof = this.judgeExactHostEgressPolicy(server, name, binding, existingPolicy)
if (proof.state !== 'intact') {
+ if ('policy' in proof) revoke(name, proof.policy)
reconcileFaults.push({
- dns: binding.dns,
+ binding: binding.dns,
err: new Error(
`Fail-static proof failed for "${binding.dns}": ${LIVE_EGRESS_FAULT_TEXT[proof.state]}`
),
@@ -2550,7 +2639,7 @@ export class NetworkPolicyReconciler {
}
const accumulated = accumulateHostExactHostEgress({
fqdn: binding.dns,
- port: binding.port,
+ port: binding.port!,
protocol,
resolution: { kind: 'transient' },
previousAnnotations: liveAnnotations,
@@ -2558,6 +2647,7 @@ export class NetworkPolicyReconciler {
config: coreConfig,
})
if (accumulated.cidrs.length === 0) {
+ revoke(name, existingPolicy)
failures.push(
`failed to resolve hostname "${binding.dns}": ${this.errorMessage(resolverFailure.error)}`
)
@@ -2565,6 +2655,7 @@ export class NetworkPolicyReconciler {
}
cidrs = accumulated.cidrs.filter(c => isAllowedExternalEgressCidr(c))
if (cidrs.length !== accumulated.cidrs.length) {
+ revoke(name, existingPolicy)
failures.push(
`persisted DNS state for hostname "${binding.dns}" contains a disallowed address`
)
@@ -2595,13 +2686,13 @@ export class NetworkPolicyReconciler {
if (!isCurrent()) return
const uniqueIps = [...new Set(records.map(record => record.address))].sort()
if (uniqueIps.length === 0) {
- if (existingPolicy) freshPolicySnapshots.set(name, existingPolicy)
+ revoke(name, existingPolicy)
failures.push(`hostname "${binding.dns}" resolved to no IPv4 addresses`)
continue
}
const invalidIps = uniqueIps.filter(ip => isIP(ip) !== 4)
if (invalidIps.length > 0) {
- if (existingPolicy) freshPolicySnapshots.set(name, existingPolicy)
+ revoke(name, existingPolicy)
failures.push(
`hostname "${binding.dns}" resolved invalid IPv4 answer(s): ${invalidIps.join(', ')}`
)
@@ -2610,7 +2701,7 @@ export class NetworkPolicyReconciler {
const freshCidrs = uniqueIps.map(ip => `${ip}/32`)
const disallowedCidrs = freshCidrs.filter(cidr => !isAllowedExternalEgressCidr(cidr))
if (disallowedCidrs.length > 0) {
- if (existingPolicy) freshPolicySnapshots.set(name, existingPolicy)
+ revoke(name, existingPolicy)
failures.push(
`hostname "${binding.dns}" resolved disallowed address(es): ${disallowedCidrs.join(', ')}`
)
@@ -2631,7 +2722,7 @@ export class NetworkPolicyReconciler {
)
const accumulated = accumulateHostExactHostEgress({
fqdn: binding.dns,
- port: binding.port,
+ port: binding.port!,
protocol,
resolution: { kind: 'ok', ips: uniqueIps, ttlSeconds },
previousAnnotations: provenPrevious.annotations,
@@ -2684,7 +2775,13 @@ export class NetworkPolicyReconciler {
}
if (!binding.dns) {
- const fresh = await this.readFreshExternalEgressPolicy(server, name)
+ let fresh: FreshExternalEgressPolicy
+ try {
+ fresh = await this.readFreshExternalEgressPolicy(server, name)
+ } catch (error: unknown) {
+ recordUnavailable(binding, name, error)
+ continue
+ }
existingPolicy = fresh.policy
ownershipRepairRequired = fresh.repairRequired
}
@@ -2696,64 +2793,32 @@ export class NetworkPolicyReconciler {
if (stateAnnotations) {
policy.metadata = { ...(policy.metadata ?? {}), annotations: stateAnnotations }
}
- if (existingPolicy) freshPolicySnapshots.set(name, existingPolicy)
// Write only when the enforced rules changed / the policy is new / drifted
// (audit F2 no-churn + L1 self-heal), OR when the DNS window is aging and
// must be re-persisted (audit M1). A pure timestamp refresh is a no-op.
// Route the write through applyOwnedPolicy so #205's ownership assertion
// and isCurrent generation fence still guard the mutation.
- if (
- ownershipRepairRequired ||
- this.externalEgressWriteNeeded(existingPolicy, policy) ||
- egressRenewalDue ||
- egressStateChanged
- ) {
- await this.applyOwnedPolicy(name, server.namespace, policy, 'external-egress', isCurrent)
- } else {
- console.log(`[NetPol] External egress "${name}" unchanged — no-op (issue #299 no-churn)`)
- }
if (!isCurrent()) return
- desiredPolicyNames.add(name)
- if (resolvedStatus) resolvedEgressIPs.push(resolvedStatus)
- if (bindingDnsFailStatic) dnsFailStatic = true
- }
-
- const cleanupCandidates = existingPolicies
- .filter(policy => {
- const name = policy.metadata?.name
- return Boolean(name) && !desiredPolicyNames.has(name!)
+ convergences.push({
+ name,
+ policy,
+ observedPolicy: existingPolicy,
+ resolvedStatus,
+ dnsFailStatic: bindingDnsFailStatic,
+ writeRequired:
+ ownershipRepairRequired ||
+ this.externalEgressWriteNeeded(existingPolicy, policy) ||
+ egressRenewalDue ||
+ egressStateChanged,
})
- .map(policy => freshPolicySnapshots.get(policy.metadata?.name ?? '') ?? policy)
- for (const [name, snapshot] of freshPolicySnapshots) {
- if (
- !desiredPolicyNames.has(name) &&
- !cleanupCandidates.some(policy => policy.metadata?.name === name)
- ) {
- cleanupCandidates.push(snapshot)
- }
}
- try {
- await this.cleanupExternalEgress(
- server.name,
- server.namespace,
- cleanupCandidates,
- async () => isCurrent(),
- options.onRevoked
+ await cleanup(
+ [...dispositions.values()].flatMap(disposition =>
+ disposition.state === 'revoke' ? [disposition.policy] : []
)
- } catch (error) {
- if (!isCurrent()) return
- await this.writeExternalEgressStatus(
- server,
- resolvedEgressIPs,
- 'False',
- 'CleanupFailed',
- `Failed to delete stale external egress policies: ${this.errorMessage(error)}`,
- isCurrent
- )
- throw error
- }
+ )
if (!isCurrent()) return
// Issue #513: a controller fault outranks rejected input. `failures` means the
@@ -2764,7 +2829,7 @@ export class NetworkPolicyReconciler {
if (reconcileFaults.length > 0) {
const first = reconcileFaults[0]
const parts = [
- `External egress reconciliation failed on binding "${first.dns}" with a non-DNS error ` +
+ `External egress reconciliation failed on binding "${first.binding}" with a non-DNS error ` +
`(not a resolver condition, ${LIVE_EGRESS_FAULT_TEXT[first.live]}): ${this.errorMessage(
first.err
)}`,
@@ -2797,6 +2862,52 @@ export class NetworkPolicyReconciler {
throw first.err
}
+ // No additive write is attempted until every observed revocation succeeded
+ // and every policy observation completed. A pending observation contributes
+ // neither a new permit nor a resolved-at/status proof.
+ for (const convergence of convergences) {
+ if (!isCurrent()) return
+ try {
+ // A sibling DNS lookup or cleanup may have taken time since this plan
+ // was built. Revalidate even no-ops; changed snapshots require a new
+ // observation rather than stamping a newer resourceVersion on old state.
+ const fresh = await this.readFreshExternalEgressPolicy(server, convergence.name)
+ this.assertExternalEgressSnapshot(
+ convergence.name,
+ convergence.observedPolicy,
+ fresh.policy
+ )
+ if (!isCurrent()) return
+ if (convergence.writeRequired) {
+ await this.applyOwnedPolicy(
+ convergence.name,
+ server.namespace,
+ convergence.policy,
+ 'external-egress',
+ isCurrent,
+ convergence.observedPolicy
+ )
+ } else {
+ hccLogger.info('external egress policy unchanged', { policy: convergence.name })
+ }
+ } catch (error: unknown) {
+ if (!isCurrent()) return
+ await this.writeExternalEgressStatus(
+ server,
+ this.sortResolvedEgressIPs(resolvedEgressIPs),
+ 'False',
+ 'ExternalEgressReconcileFailed',
+ `Failed to apply external egress policy "${convergence.name}": ${this.errorMessage(error)}`,
+ isCurrent
+ )
+ throw error
+ }
+ if (!isCurrent()) return
+ dispositions.set(convergence.name, { state: 'retained' })
+ if (convergence.resolvedStatus) resolvedEgressIPs.push(convergence.resolvedStatus)
+ if (convergence.dnsFailStatic) dnsFailStatic = true
+ }
+
if (failures.length > 0) {
const message = failures.join('; ')
await this.writeExternalEgressStatus(
@@ -2843,12 +2954,12 @@ export class NetworkPolicyReconciler {
try {
if (deleteAllowed && !(await deleteAllowed())) return
await this.deleteSafetyPolicySnapshot(namespace, policy, undefined, () => {
- console.log(`[NetPol] Deleted external egress policy "${name}"`)
+ hccLogger.info('deleted external egress policy', { policy: name })
onDeleted?.()
})
} catch (error: unknown) {
if (getErrorCode(error) !== 404) {
- console.error(`[NetPol] Failed to delete external egress policy "${name}":`, error)
+ hccLogger.error('failed to delete external egress policy', { policy: name, err: error })
throw error
}
// A concurrent actor may already have removed the policy. The safety
@@ -2969,15 +3080,15 @@ export class NetworkPolicyReconciler {
currentStatus = current.status ?? {}
} catch (error) {
if (getErrorCode(error) === 404) {
- console.warn(
- `[NetPol] McpServer "${server.name}" deleted mid-reconcile — external egress status skipped`
- )
+ hccLogger.warn('McpServer deleted mid-reconcile; external egress status skipped', {
+ serverName: server.name,
+ })
return
}
- console.warn(
- `[NetPol] Failed to read external egress status for "${server.name}" — skipping status update:`,
- error
- )
+ hccLogger.warn('failed to read external egress status; status update skipped', {
+ serverName: server.name,
+ err: error,
+ })
throw error
}
if (!isCurrent()) return
@@ -3057,10 +3168,11 @@ export class NetworkPolicyReconciler {
body: statusPatch,
})
} catch (error) {
- console.warn(
- `[NetPol] Failed to write ExternalEgressReady=${status} on "${server.name}":`,
- error
- )
+ hccLogger.warn('failed to write ExternalEgressReady', {
+ serverName: server.name,
+ status,
+ err: error,
+ })
throw error
}
}
@@ -3081,7 +3193,7 @@ export class NetworkPolicyReconciler {
})
return response.items || []
} catch (error) {
- console.error(`[NetPol] Failed to list policies for context "${contextId}":`, error)
+ hccLogger.error('failed to list context policies', { contextId, err: error })
throw error
}
}
@@ -3095,7 +3207,7 @@ export class NetworkPolicyReconciler {
})
return response.items || []
} catch (error) {
- console.error('[NetPol] Failed to list all context policies:', error)
+ hccLogger.error('failed to list all context policies', { err: error })
throw error
}
}
@@ -3106,7 +3218,7 @@ export class NetworkPolicyReconciler {
const response = await this.networkingApi.listNamespacedNetworkPolicy({ namespace })
return response.items || []
} catch (error) {
- console.error(`[NetPol] Failed to list broad NetworkPolicy inventory in ${namespace}:`, error)
+ hccLogger.error('failed to list broad NetworkPolicy inventory', { namespace, err: error })
throw error
}
}
@@ -3150,7 +3262,7 @@ export class NetworkPolicyReconciler {
})
return response.items || []
} catch (error) {
- console.error('[NetPol] Failed to list all Context egress policies:', error)
+ hccLogger.error('failed to list all context egress policies', { err: error })
throw error
}
}
@@ -3164,7 +3276,7 @@ export class NetworkPolicyReconciler {
})
return response.items || []
} catch (error) {
- console.error('[NetPol] Failed to list all external egress policies:', error)
+ hccLogger.error('failed to list all external egress policies', { err: error })
throw error
}
}
@@ -3219,7 +3331,8 @@ export class NetworkPolicyReconciler {
namespace: string,
policy: k8s.V1NetworkPolicy,
lane: SafetyInventoryLane,
- isCurrent?: () => boolean
+ isCurrent?: () => boolean,
+ expectedSnapshot?: k8s.V1NetworkPolicy | null
): Promise<void> {
await applyNetworkPolicy(
this.networkingApi,
@@ -3228,10 +3341,33 @@ export class NetworkPolicyReconciler {
policy,
'[NetPol]',
isCurrent,
- existing => this.assertPolicyOwnership(name, existing, policy, lane)
+ existing => {
+ this.assertPolicyOwnership(name, existing, policy, lane)
+ if (expectedSnapshot !== undefined) {
+ this.assertExternalEgressSnapshot(name, expectedSnapshot, existing)
+ }
+ },
+ // Best-effort maintenance may skip a disappeared object. External-egress
+ // admission must instead retry: POST409 followed by GET404 applied nothing.
+ lane === 'external-egress'
)
}
+ private assertExternalEgressSnapshot(
+ name: string,
+ expected: k8s.V1NetworkPolicy | null,
+ actual: k8s.V1NetworkPolicy | null
+ ): void {
+ if (
+ expected?.metadata?.uid !== actual?.metadata?.uid ||
+ expected?.metadata?.resourceVersion !== actual?.metadata?.resourceVersion
+ ) {
+ throw new Error(
+ `NetworkPolicy "${name}" changed after the external-egress observation; reconciliation must retry`
+ )
+ }
+ }
+
private assertPolicyOwnership(
name: string,
existing: k8s.V1NetworkPolicy,
@@ -3287,12 +3423,12 @@ export class NetworkPolicyReconciler {
namespace,
body: { preconditions: { uid, resourceVersion } },
})
- console.log(`[NetPol] Deleted safety-inventory policy "${name}" in ${namespace}`)
+ hccLogger.info('deleted safety-inventory policy', { policy: name, namespace })
onDeleted?.()
return true
} catch (error: unknown) {
if (getErrorCode(error) === 404) {
- console.log(`[NetPol] Safety-inventory policy "${name}" already gone`)
+ hccLogger.info('safety-inventory policy already gone', { policy: name, namespace })
return true
}
if (getErrorCode(error) === 409 && lostFenceOutcome === 'report') {
@@ -3302,15 +3438,17 @@ export class NetworkPolicyReconciler {
// adopted. Callers that opt in act on this outcome by declining to
// certify, which is the expected racy result rather than a failure.
// Callers that cannot act on it keep the loud default.
- console.warn(
- `[NetPol] Lost the delete fence on safety-inventory policy "${name}" in ${namespace}; it changed identity after the authoritative inventory and was not revoked`
- )
+ hccLogger.warn('lost safety-inventory delete fence; changed policy was not revoked', {
+ policy: name,
+ namespace,
+ })
return false
}
- console.error(
- `[NetPol] Failed identity-bound delete of safety-inventory policy "${name}" in ${namespace}:`,
- error
- )
+ hccLogger.error('failed identity-bound delete of safety-inventory policy', {
+ policy: name,
+ namespace,
+ err: error,
+ })
throw error
}
}
@@ -3381,9 +3519,10 @@ export class NetworkPolicyReconciler {
if (!name) {
throw new Error(`Cannot converge a conflicting ${lane} policy without a desired name`)
}
- console.log(
- `[NetPol] Safety policy "${name}" in ${namespace} already exists — converging it to the desired spec`
- )
+ hccLogger.info('converging existing safety policy to desired spec', {
+ policy: name,
+ namespace,
+ })
// Deliberately not delegating to applyOwnedPolicy here: that helper owns
// the create-then-conflict flow, so calling it from inside our own
// create's catch would issue a second create. Delegating from the top is
@@ -3423,7 +3562,7 @@ export class NetworkPolicyReconciler {
})
return response.items || []
} catch (error) {
- console.error(`[NetPol] Failed to list egress policies for context "${contextId}":`, error)
+ hccLogger.error('failed to list context egress policies', { contextId, err: error })
throw error
}
}
@@ -3481,10 +3620,10 @@ export class NetworkPolicyReconciler {
await this.deleteSafetyPolicySnapshot(config.rpcProxyNamespace, policy)
}
} catch (error) {
- console.error(
- `[NetPol] Failed to delete rpc-proxy egress policies for context "${contextId}":`,
- error
- )
+ hccLogger.error('failed to delete context rpc-proxy egress policies', {
+ contextId,
+ err: error,
+ })
throw error
}
}
@@ -3500,10 +3639,7 @@ export class NetworkPolicyReconciler {
})
return response.items || []
} catch (error) {
- console.error(
- `[NetPol] Failed to list rpc-proxy egress policies for context "${contextId}":`,
- error
- )
+ hccLogger.error('failed to list context rpc-proxy egress policies', { contextId, err: error })
throw error
}
}
@@ -3517,7 +3653,7 @@ export class NetworkPolicyReconciler {
})
return response.items || []
} catch (error) {
- console.error('[NetPol] Failed to list all rpc-proxy egress policies:', error)
+ hccLogger.error('failed to list all rpc-proxy egress policies', { err: error })
throw error
}
}
@@ -3535,6 +3671,6 @@ export class NetworkPolicyReconciler {
throw new Error(`NetworkPolicy "${name}" has conflicting ownership for legacy cleanup`)
}
await this.deleteSafetyPolicySnapshot(namespace, existing)
- console.log(`[NetPol] Deleted legacy static policy "${name}" in "${namespace}"`)
+ hccLogger.info('deleted legacy static policy', { policy: name, namespace })
}
}
diff --git a/host-context-controller/src/utils.ts b/host-context-controller/src/utils.ts
index 3ecea9520..e1bb34ef6 100644
--- a/host-context-controller/src/utils.ts
+++ b/host-context-controller/src/utils.ts
@@ -1,4 +1,5 @@
import * as k8s from '@kubernetes/client-node'
+import { hccLogger } from './logger'
import { writeSkipsTotal, writesTotal } from './metrics'
/** Extract HTTP status code from a K8s client error (handles both error formats). */
@@ -41,6 +42,8 @@ export async function replaceWithConflictRetry<
validateExisting?: (existing: T) => void
/** Rechecked immediately before every Kubernetes write attempt. */
mutationAllowed?: () => boolean
+ /** Admission-sensitive callers must not treat an absent object as applied. */
+ missingIsError?: boolean
maxAttempts?: number
}): Promise<void> {
const {
@@ -54,6 +57,7 @@ export async function replaceWithConflictRetry<
isUpToDate,
validateExisting,
mutationAllowed,
+ missingIsError = false,
maxAttempts = 3,
} = opts
for (let attempt = 1; attempt <= maxAttempts; attempt++) {
@@ -62,7 +66,11 @@ export async function replaceWithConflictRetry<
existing = await read()
} catch (err) {
if (getErrorCode(err) === 404) {
- console.warn(`${logPrefix} ${description} disappeared (404); skipping replace`)
+ if (missingIsError) throw err
+ hccLogger.warn('Kubernetes resource disappeared; skipping replace', {
+ scope: logPrefix,
+ description,
+ })
return
}
throw err
@@ -83,8 +91,7 @@ export async function replaceWithConflictRetry<
try {
await replace(next)
writesTotal.inc({ kind: next.kind ?? 'unknown' })
- const suffix = attempt > 1 ? ` (after ${attempt} attempts)` : ''
- console.log(`${logPrefix} Updated ${description}${suffix}`)
+ hccLogger.info('Kubernetes resource updated', { scope: logPrefix, description, attempt })
return
} catch (err) {
if (getErrorCode(err) === 409 && attempt < maxAttempts) {
@@ -448,12 +455,13 @@ export async function applyNetworkPolicy(
policy: k8s.V1NetworkPolicy,
logPrefix = '[NetPol]',
mutationAllowed?: () => boolean,
- validateExisting?: (existing: k8s.V1NetworkPolicy) => void
+ validateExisting?: (existing: k8s.V1NetworkPolicy) => void,
+ missingIsError = false
): Promise<void> {
if (mutationAllowed && !mutationAllowed()) return
try {
await api.createNamespacedNetworkPolicy({ namespace, body: policy })
- console.log(`${logPrefix} Created policy "${name}" in ${namespace}`)
+ hccLogger.info('NetworkPolicy created', { scope: logPrefix, policy: name, namespace })
return
} catch (error: unknown) {
if (getErrorCode(error) !== 409) {
@@ -468,6 +476,7 @@ export async function applyNetworkPolicy(
replace: body => api.replaceNamespacedNetworkPolicy({ name, namespace, body }),
mutationAllowed,
validateExisting,
+ missingIsError,
isUpToDate: networkPolicyMatchesDesired,
})
}
diff --git a/host-context-controller/test/__fixtures__/updatedLogs.ts b/host-context-controller/test/__fixtures__/updatedLogs.ts
index eb765dcb1..1e29abfaa 100644
--- a/host-context-controller/test/__fixtures__/updatedLogs.ts
+++ b/host-context-controller/test/__fixtures__/updatedLogs.ts
@@ -3,6 +3,21 @@ import { vi } from 'vitest'
/** Shared log filter for the no-op-gate suites. Every needle must match. */
export function updatedLogs(log: ReturnType<typeof vi.spyOn>, ...needles: string[]): string[] {
return log.mock.calls
- .map((call: unknown[]) => String(call[0]))
+ .map((call: unknown[]) => {
+ const line = String(call[0])
+ // Some untouched producers still use the legacy presentation. Project the
+ // structured shared-helper event into the same test vocabulary so both
+ // no-op and real-write assertions keep measuring actual emitted events.
+ try {
+ const entry = JSON.parse(line)
+ if (entry.msg === 'Kubernetes resource updated' && typeof entry.description === 'string') {
+ const suffix = entry.attempt > 1 ? ` (after ${entry.attempt} attempts)` : ''
+ return `${entry.scope} Updated ${entry.description}${suffix}`
+ }
+ } catch {
+ /* Legacy plain-text producer. */
+ }
+ return line
+ })
.filter((line: string) => needles.every(needle => line.includes(needle)))
}
diff --git a/scripts/e2e/_lib/wrc-egress-gate.cjs b/scripts/e2e/_lib/wrc-egress-gate.cjs
new file mode 100644
index 000000000..548bde157
--- /dev/null
+++ b/scripts/e2e/_lib/wrc-egress-gate.cjs
@@ -0,0 +1,1222 @@
+#!/usr/bin/env node
+'use strict'
+
+// Product/controller E2E, not a browser journey and not a T2 attestation.
+// All mutation goes through the explicit-context client and inherited lease.
+const fs = require('node:fs')
+const path = require('node:path')
+const dns = require('node:dns/promises')
+const { BlockList, isIPv4 } = require('node:net')
+const { randomUUID } = require('node:crypto')
+const { execFileSync } = require('node:child_process')
+const { assertPolicy } = require('./wrc-egress-proof.cjs')
+const {
+ Journal,
+ journalPath,
+ readJournal,
+ assertJournalBinding,
+ createOwned,
+ deleteOwned,
+ waitFor,
+ createCommandRunner,
+ createKubeClient,
+ invariant,
+ RUN_LABEL,
+ resourceIdentity,
+} = require('./wrc-egress-lifecycle.cjs')
+
+const STATE = 'clerum.io/egress-fqdn-state'
+const LOOPBACK = [127, 0, 0, 1].join('.')
+const endpointHost = 'example.com' // Existing mock-MCP fixture public endpoint contract.
+const deploymentReady = object =>
+ Boolean(
+ object &&
+ object.spec?.replicas > 0 &&
+ object.status?.observedGeneration >= object.metadata.generation &&
+ object.status.readyReplicas === object.spec.replicas &&
+ object.status.availableReplicas === object.spec.replicas
+ )
+const same = (left, right) => JSON.stringify(left) === JSON.stringify(right)
+const id = (kind, namespace, name) => ({ kind, namespace, name })
+
+// Called after the complete policy oracle succeeds. Fresh DNS traffic alone
+// cannot satisfy recovery; every tuple needs a newer persisted observation.
+function policyObservation(policy, newerThan) {
+ const entries = JSON.parse(policy.metadata.annotations[STATE])
+ invariant(
+ entries.length > 0 &&
+ entries.every(
+ entry => Number.isFinite(entry.lastObservedAt) && Number.isFinite(entry.expiresAt)
+ ),
+ 'MISSING_ACCEPTED_DNS_OBSERVATION'
+ )
+ if (newerThan !== undefined && !entries.every(entry => entry.lastObservedAt > newerThan))
+ return null
+ return {
+ lastAcceptedAt: Math.max(...entries.map(entry => entry.lastObservedAt)),
+ latestExpiry: Math.max(...entries.map(entry => entry.expiresAt)),
+ }
+}
+
+function acceptPolicyIntentChange(lanes, before, after, recipeUid, targets) {
+ invariant(
+ after.metadata?.uid === recipeUid &&
+ before.metadata?.uid === recipeUid &&
+ after.metadata.generation > before.metadata.generation,
+ 'HOST_MIGRATION_NOT_APPLIED'
+ )
+ for (const lane of lanes) {
+ const target = targets.find(target => target.lane === (lane.lane === 'ui' ? 'worker' : 'ui'))
+ invariant(target, 'MISSING_MIGRATION_TARGET')
+ lane.target = target
+ lane.uid = undefined
+ }
+}
+
+// A test setup restriction, not an alternate production address classifier.
+const blocked = new BlockList()
+for (const cidr of [
+ '0.0.0.0/8',
+ '10.0.0.0/8',
+ '100.64.0.0/10',
+ '127.0.0.0/8',
+ '169.254.0.0/16',
+ '172.16.0.0/12',
+ '192.0.0.0/24',
+ '192.0.2.0/24',
+ '192.31.196.0/24',
+ '192.52.193.0/24',
+ '192.88.99.0/24',
+ '192.168.0.0/16',
+ '192.175.48.0/24',
+ '198.18.0.0/15',
+ '198.51.100.0/24',
+ '203.0.113.0/24',
+ '224.0.0.0/4',
+ '240.0.0.0/4',
+]) {
+ const [address, prefix] = cidr.split('/')
+ blocked.addSubnet(address, Number(prefix), 'ipv4')
+}
+
+const trafficCode = `
+const net=require('node:net'),https=require('node:https');
+const [ip,portText,host,mode]=process.argv.slice(1); const port=Number(portText);
+if(mode==='https') {
+ const request=https.get({hostname:ip,port,servername:host,path:'/',agent:false,
+ headers:{host,connection:'close'}},response=>{
+ let body=''; response.on('data',chunk=>{body+=chunk;if(body.length>65536)request.destroy(new Error('body limit'))});
+ response.once('aborted',()=>{process.exitCode=43;});
+ response.once('error',()=>{process.exitCode=43;});
+ response.once('end',()=>{if(response.statusCode===200&&body.includes('Example Domain')){
+ process.stdout.write(JSON.stringify({allowed:true,contentVerified:true}));
+ }else process.exitCode=44;});
+ });
+ const timer=setTimeout(()=>request.destroy(new Error('deadline')),5000);
+ request.once('close',()=>clearTimeout(timer)); request.once('error',()=>{process.exitCode=43;});
+} else {
+ const socket=net.connect({host:ip,port});
+ const timer=setTimeout(()=>{socket.destroy();process.exitCode=42;},2500);
+ socket.once('connect',()=>{clearTimeout(timer);socket.destroy();process.stdout.write('connected');});
+ socket.once('error',()=>{clearTimeout(timer);process.exitCode=43;});
+}`
+
+const businessCode = `
+const {Client}=require('@modelcontextprotocol/sdk/client/index.js');
+const {StreamableHTTPClientTransport}=require('@modelcontextprotocol/sdk/client/streamableHttp.js');
+const [url,key,value,operation]=process.argv.slice(1);
+const client=new Client({name:'wrc-egress-business-witness',version:'1.0.0'},{capabilities:{}});
+const timer=setTimeout(()=>process.exit(45),10000);
+(async()=>{try{
+ await client.connect(new StreamableHTTPClientTransport(new URL(url)));
+ if(operation==='record'){
+ const stored=await client.callTool({name:'record',arguments:{key,value}});
+ if(stored.isError)throw new Error('record failed');
+ }
+ const response=await client.callTool({name:'recall',arguments:{key}});
+ const payload=JSON.parse(response.content.find(item=>item.type==='text').text);
+ if(response.isError||payload.key!==key||payload.value!==value)throw new Error('business state lost');
+ process.stdout.write(JSON.stringify({businessVerified:true}));
+}finally{await client.close();clearTimeout(timer);}})().catch(()=>{process.exitCode=46;});`
+
+const dnsControlCode = `
+const http=require('node:http'); const [host,route,method]=process.argv.slice(1);
+const request=http.request({host,port:8090,path:route,method},response=>{
+ let body='';response.on('data',chunk=>{body+=chunk;if(body.length>262144)request.destroy()});
+ response.once('end',()=>{if(response.statusCode!==200)process.exitCode=47;else process.stdout.write(body);});
+});
+const timer=setTimeout(()=>request.destroy(new Error('deadline')),5000);
+request.once('close',()=>clearTimeout(timer));request.once('error',()=>{process.exitCode=48;});request.end();`
+
+function fixtureObjects(config, runId, wrcImage, answerIp, upstream) {
+ const suffix = runId.slice(0, 12)
+ const names = {
+ proxy: `e2e-wrc-dns-${suffix}`,
+ script: `e2e-wrc-dns-script-${suffix}`,
+ policy: `e2e-wrc-dns-policy-${suffix}`,
+ fromWrc: `e2e-wrc-from-${suffix}`,
+ recipe: `e2e-wrc-${suffix}`,
+ canary: `e2e-wrc-canary-${suffix}`,
+ }
+ const labels = { 'e2e.clerum.io/suite': 'wrc-egress-degradation', [RUN_LABEL]: runId }
+ const metadata = (name, namespace = config.wrcNamespace) => ({ name, namespace, labels })
+ const targets = [
+ { lane: 'ui', fqdn: `ui-${suffix}.example.com`, ip: answerIp },
+ { lane: 'worker', fqdn: `worker-${suffix}.example.com`, ip: answerIp },
+ { lane: 'canary', fqdn: `canary-${suffix}.example.com`, ip: answerIp },
+ ]
+ const securityContext = {
+ allowPrivilegeEscalation: false,
+ capabilities: { drop: ['ALL'] },
+ readOnlyRootFilesystem: true,
+ runAsNonRoot: true,
+ runAsUser: 1000,
+ runAsGroup: 1000,
+ seccompProfile: { type: 'RuntimeDefault' },
+ }
+ const ports = [
+ { protocol: 'UDP', port: 53 },
+ { protocol: 'TCP', port: 53 },
+ { protocol: 'UDP', port: 8053 },
+ { protocol: 'TCP', port: 8053 },
+ ]
+ const proxyPeer = { podSelector: { matchLabels: { app: names.proxy } } }
+ const objects = [
+ {
+ apiVersion: 'v1',
+ kind: 'ConfigMap',
+ metadata: metadata(names.script),
+ data: {
+ 'server.cjs': fs.readFileSync(
+ path.join(config.repository, 'tests/e2e/fixtures/wrc-egress-dns-proxy/server.cjs'),
+ 'utf8'
+ ),
+ },
+ },
+ {
+ apiVersion: 'networking.k8s.io/v1',
+ kind: 'NetworkPolicy',
+ metadata: metadata(names.policy),
+ spec: {
+ podSelector: proxyPeer.podSelector,
+ policyTypes: ['Ingress', 'Egress'],
+ ingress: [
+ { from: [{ podSelector: { matchLabels: { app: config.wrcDeployment } } }], ports },
+ ],
+ egress: [
+ {
+ to: [
+ {
+ namespaceSelector: {
+ matchLabels: { 'kubernetes.io/metadata.name': 'kube-system' },
+ },
+ podSelector: { matchLabels: { 'k8s-app': 'kube-dns' } },
+ },
+ ],
+ ports: ports.slice(0, 2),
+ },
+ ],
+ },
+ },
+ {
+ apiVersion: 'networking.k8s.io/v1',
+ kind: 'NetworkPolicy',
+ metadata: metadata(names.fromWrc),
+ spec: {
+ podSelector: { matchLabels: { app: config.wrcDeployment } },
+ policyTypes: ['Egress'],
+ egress: [{ to: [proxyPeer], ports }],
+ },
+ },
+ {
+ apiVersion: 'v1',
+ kind: 'Service',
+ metadata: metadata(names.proxy),
+ spec: {
+ selector: { app: names.proxy },
+ ports: [
+ { name: 'dns-udp', protocol: 'UDP', port: 53, targetPort: 8053 },
+ { name: 'dns-tcp', protocol: 'TCP', port: 53, targetPort: 8053 },
+ ],
+ },
+ },
+ {
+ apiVersion: 'apps/v1',
+ kind: 'Deployment',
+ metadata: metadata(names.proxy),
+ spec: {
+ replicas: 1,
+ selector: { matchLabels: { app: names.proxy } },
+ template: {
+ metadata: { labels: { ...labels, app: names.proxy } },
+ spec: {
+ automountServiceAccountToken: false,
+ terminationGracePeriodSeconds: 1,
+ containers: [
+ {
+ name: 'dns-proxy',
+ image: wrcImage,
+ imagePullPolicy: 'IfNotPresent',
+ command: ['node', '/fixture/server.cjs'],
+ env: [
+ { name: 'DNS_TARGETS_JSON', value: JSON.stringify(targets) },
+ { name: 'DNS_TTL_SECONDS', value: '5' },
+ { name: 'UPSTREAM_DNS', value: upstream },
+ ],
+ ports: [
+ { name: 'dns-udp', containerPort: 8053, protocol: 'UDP' },
+ { name: 'dns-tcp', containerPort: 8053, protocol: 'TCP' },
+ ],
+ readinessProbe: {
+ tcpSocket: { port: 8053 },
+ periodSeconds: 1,
+ failureThreshold: 10,
+ },
+ volumeMounts: [{ name: 'fixture', mountPath: '/fixture', readOnly: true }],
+ resources: {
+ requests: { cpu: '5m', memory: '20Mi' },
+ limits: { cpu: '100m', memory: '96Mi' },
+ },
+ securityContext,
+ },
+ ],
+ volumes: [{ name: 'fixture', configMap: { name: names.script } }],
+ },
+ },
+ },
+ },
+ ]
+ const workload = name => ({
+ id: name,
+ type: 'deployment',
+ image: 'clerum/mock-mcp-server:test',
+ port: 3000,
+ healthCheck: { type: 'http', path: '/', port: 3001 },
+ })
+ const bindings = fqdn => [80, 443].map(port => ({ dns: fqdn, port, protocol: 'TCP' }))
+ const recipe = {
+ apiVersion: 'clerum.io/v1alpha1',
+ kind: 'WorkflowRecipe',
+ metadata: metadata(names.recipe, config.recipeNamespace),
+ spec: {
+ contextRef: 'context1',
+ description: 'WRC authorization continuity fixture',
+ workloads: [
+ workload('frontend'),
+ { ...workload('worker'), egressBindings: bindings(targets[1].fqdn) },
+ ],
+ ui: {
+ workloadRef: 'frontend',
+ port: 3000,
+ title: 'WRC continuity',
+ defaultPath: '/mcp',
+ egress: {
+ internal: [
+ { workloadRef: 'worker', port: 3000 },
+ { workloadRef: 'worker', port: 3001 },
+ ],
+ external: [80, 443].map(port => ({ fqdn: targets[0].fqdn, port })),
+ },
+ },
+ security: { isolationLevel: 'minimal' },
+ },
+ }
+ const canary = {
+ apiVersion: 'clerum.io/v1alpha1',
+ kind: 'WorkflowRecipe',
+ metadata: metadata(names.canary, config.recipeNamespace),
+ spec: {
+ contextRef: 'context1',
+ description: 'Independent WRC business witness',
+ workloads: [{ ...workload('witness'), egressBindings: bindings(targets[2].fqdn) }],
+ security: { isolationLevel: 'minimal' },
+ },
+ }
+ return { names, targets, objects, recipe, canary }
+}
+
+async function readyPod(kube, namespace, recipe, workload, timeoutMs = 120_000) {
+ let pod
+ await waitFor('ready-fixture-pod', timeoutMs, async (remaining, signal) => {
+ const pods = await kube.list(
+ 'pods',
+ namespace,
+ `clerum.io/recipe=${recipe},clerum.io/workload=${workload}`,
+ remaining,
+ signal
+ )
+ const ready = pods.filter(
+ item =>
+ !item.metadata.deletionTimestamp &&
+ item.status?.conditions?.some(
+ condition => condition.type === 'Ready' && condition.status === 'True'
+ )
+ )
+ invariant(ready.length <= 1, 'AMBIGUOUS_FIXTURE_POD')
+ pod = ready[0]
+ return Boolean(pod?.metadata.uid)
+ })
+ return pod
+}
+
+const identityKey = identity => `${identity.kind}/${identity.namespace}/${identity.name}`
+const checkedIdentity = identity => resourceIdentity({ kind: identity.kind, metadata: identity })
+
+function uiChildIdentities(ui) {
+ const identities = []
+ if (ui.physicalName !== null) {
+ for (const kind of ['Deployment', 'Service']) {
+ identities.push(checkedIdentity({ kind, namespace: ui.namespace, name: ui.physicalName }))
+ }
+ }
+ identities.push(
+ checkedIdentity({
+ kind: 'NetworkPolicy',
+ namespace: ui.namespace,
+ name: `ui-egress-${ui.parent.name}`,
+ })
+ )
+ return identities
+}
+
+function uiCleanupRecord(journal) {
+ const ui = journal.state.uiCleanup
+ // Journals from before UI capture cannot silently certify cross-namespace
+ // teardown once a recipe was attempted. They require explicit inspection.
+ invariant(
+ ui !== undefined || !journal.state.resources.some(entry => entry.kind === 'WorkflowRecipe'),
+ 'UI_CLEANUP_RECORD_MISSING'
+ )
+ if (ui === null || ui === undefined) return null
+ invariant(
+ ui.runId === journal.state.runId &&
+ ui.parent?.kind === 'WorkflowRecipe' &&
+ typeof ui.workload === 'string' &&
+ /^[a-z0-9][a-z0-9.-]*$/.test(ui.workload) &&
+ Array.isArray(ui.children) &&
+ ui.children.length <= 3,
+ 'INVALID_UI_CLEANUP_RECORD'
+ )
+ checkedIdentity(ui.parent)
+ const allowed = new Set(uiChildIdentities(ui).map(identityKey))
+ const seen = new Set()
+ for (const child of ui.children) {
+ const key = identityKey(checkedIdentity(child))
+ invariant(
+ allowed.has(key) &&
+ !seen.has(key) &&
+ (child.uid === null || (typeof child.uid === 'string' && child.uid.length > 0)),
+ 'INVALID_UI_CHILD_RECORD'
+ )
+ seen.add(key)
+ }
+ return ui
+}
+
+function assertUiChild(object, identity, ui) {
+ invariant(
+ identityKey(resourceIdentity(object)) === identityKey(identity) &&
+ typeof object.metadata.uid === 'string' &&
+ object.metadata.uid.length > 0 &&
+ typeof object.metadata.resourceVersion === 'string' &&
+ object.metadata.resourceVersion.length > 0,
+ 'UI_CHILD_IDENTITY_UNPROVEN'
+ )
+ const labels = object.metadata.labels ?? {}
+ invariant(
+ labels['clerum.io/managed-by'] === 'workflow-recipes' &&
+ labels['clerum.io/recipe'] === ui.parent.name,
+ 'UI_CHILD_OWNER_UNPROVEN'
+ )
+ invariant(
+ (labels['clerum.io/recipe-name'] === undefined ||
+ labels['clerum.io/recipe-name'] === ui.parent.name) &&
+ (labels['clerum.io/recipe-namespace'] === undefined ||
+ labels['clerum.io/recipe-namespace'] === ui.parent.namespace) &&
+ (labels['clerum.io/sandbox-ui'] === undefined || labels['clerum.io/sandbox-ui'] === 'true'),
+ 'UI_CHILD_SCOPE_CONFLICT'
+ )
+ if (identity.kind !== 'NetworkPolicy') {
+ invariant(labels['clerum.io/workload'] === ui.workload, 'UI_CHILD_WORKLOAD_UNPROVEN')
+ }
+ // Deployment metadata may omit sandbox-ui labels; its physical status name
+ // plus ordinary WRC recipe/workload labels supply the ownership proof.
+ if (identity.kind !== 'Deployment') {
+ invariant(
+ labels['clerum.io/recipe-name'] === ui.parent.name &&
+ labels['clerum.io/recipe-namespace'] === ui.parent.namespace,
+ 'UI_CHILD_SCOPE_UNPROVEN'
+ )
+ }
+ if (identity.kind === 'Service') {
+ invariant(labels['clerum.io/sandbox-ui'] === 'true', 'UI_SERVICE_SCOPE_UNPROVEN')
+ }
+ for (const owner of object.metadata.ownerReferences ?? []) {
+ invariant(
+ owner.kind === 'WorkflowRecipe' &&
+ owner.name === ui.parent.name &&
+ owner.uid === ui.parentUid,
+ 'UI_CHILD_HAS_FOREIGN_OWNER'
+ )
+ }
+}
+
+async function captureUiChildren(kube, journal) {
+ const ui = uiCleanupRecord(journal)
+ if (!ui) return null
+ const parentIdentity = checkedIdentity(ui.parent)
+ const parentRecord = journal.state.resources.find(
+ entry => identityKey(entry) === identityKey(parentIdentity)
+ )
+ if (ui.parentUid !== null) {
+ invariant(parentRecord?.uid === ui.parentUid, 'UI_PARENT_RECORD_MISMATCH')
+ }
+ const parent = await kube.get(parentIdentity)
+ // On recovery, absence cannot authorize adopting a same-name child. Only
+ // identities durably captured while the exact parent existed remain usable.
+ if (!parent) return ui
+ invariant(
+ parentRecord?.uid &&
+ identityKey(resourceIdentity(parent)) === identityKey(parentIdentity) &&
+ parent.metadata.uid === parentRecord.uid &&
+ parent.metadata.labels?.[RUN_LABEL] === ui.runId &&
+ parent.spec?.ui?.workloadRef === ui.workload &&
+ parent.spec.workloads?.some(
+ workload => workload.id === ui.workload && workload.type === 'deployment'
+ ),
+ 'UI_PARENT_OWNERSHIP_UNPROVEN'
+ )
+ ui.parentUid = parentRecord.uid
+ const physicalName = parent.status?.workloadInstances?.[ui.workload]
+ if (physicalName !== undefined) {
+ // Status is persisted before WRC creates workloads. Validate it before any
+ // CLI argument; never substitute the raw logical workload ID when absent.
+ checkedIdentity({ kind: 'Deployment', namespace: ui.namespace, name: physicalName })
+ invariant(
+ ui.physicalName === null || ui.physicalName === physicalName,
+ 'UI_PHYSICAL_NAME_CHANGED'
+ )
+ ui.physicalName = physicalName
+ }
+ for (const identity of uiChildIdentities(ui)) {
+ const recorded = ui.children.find(child => identityKey(child) === identityKey(identity))
+ const object = await kube.get(identity)
+ if (object) {
+ assertUiChild(object, identity, ui)
+ invariant(!recorded || recorded.uid === object.metadata.uid, 'UI_CHILD_IDENTITY_CHANGED')
+ }
+ if (!recorded) ui.children.push({ ...identity, uid: object?.metadata.uid ?? null })
+ journal.save()
+ }
+ return ui
+}
+
+async function cleanupUiChildren(kube, ui) {
+ invariant(!(await kube.get(checkedIdentity(ui.parent))), 'UI_PARENT_STILL_PRESENT')
+ for (const child of ui.children) {
+ const identity = checkedIdentity(child)
+ const live = await kube.get(identity)
+ if (!live) continue
+ invariant(child.uid && live.metadata.uid === child.uid, 'UI_CHILD_IDENTITY_CHANGED')
+ assertUiChild(live, identity, ui)
+ try {
+ await kube.delete(identity, {
+ uid: child.uid,
+ resourceVersion: live.metadata.resourceVersion,
+ })
+ } catch (error) {
+ if (await kube.get(identity)) throw error
+ }
+ await waitFor('UI-child-absence', 90_000, async (remaining, signal) => {
+ const current = await kube.get(identity, remaining, signal)
+ if (!current) return true
+ invariant(current.metadata.uid === child.uid, 'UI_CHILD_RECREATED_DURING_CLEANUP')
+ return false
+ })
+ }
+ // WRC's best-effort finalizer can delete a Deployment in Background. Its
+ // ReplicaSets/Pods must also disappear before a restored verdict is valid.
+ // Discovery is read-only: an uncaptured managed child is never adopted here.
+ await waitFor('UI-resource-absence', 90_000, async (remaining, signal) => {
+ const deadline = Date.now() + remaining
+ let empty = true
+ for (const kind of ['deployments', 'services', 'networkpolicies', 'replicasets', 'pods']) {
+ const objects = await kube.list(
+ kind,
+ ui.namespace,
+ `clerum.io/recipe=${ui.parent.name}`,
+ deadline - Date.now(),
+ signal
+ )
+ for (const object of objects) {
+ if (!['replicasets', 'pods'].includes(kind)) {
+ const identity = resourceIdentity(object)
+ const recorded = ui.children.find(child => identityKey(child) === identityKey(identity))
+ invariant(
+ recorded?.uid && recorded.uid === object.metadata.uid,
+ 'UI_CHILD_IDENTITY_NOT_CAPTURED'
+ )
+ }
+ empty = false
+ }
+ }
+ return empty
+ })
+}
+
+async function restore(kube, journal) {
+ await kube.setCleanup(true)
+ journal.state.phase = 'restoring'
+ journal.save()
+ const failures = []
+ let ui
+ const wrc = journal.state.wrc
+ if (wrc?.attempted) {
+ try {
+ const live = await kube.get(wrc.identity)
+ invariant(
+ live?.metadata.uid === wrc.uid && live.spec.replicas === wrc.replicas,
+ 'WRC_RESTORE_IDENTITY_CHANGED'
+ )
+ const current = live.spec.template.spec
+ const alreadyRestored =
+ (current.dnsPolicy ?? 'ClusterFirst') === wrc.originalPolicy &&
+ (current.dnsConfig ?? null) === null
+ if (!alreadyRestored) {
+ invariant(
+ current.dnsPolicy === 'None' && same(current.dnsConfig, wrc.injectedConfig),
+ 'WRC_DNS_CHANGED_BY_ANOTHER_OWNER'
+ )
+ await kube.patch(wrc.identity, [
+ { op: 'test', path: '/metadata/uid', value: wrc.uid },
+ { op: 'test', path: '/metadata/resourceVersion', value: live.metadata.resourceVersion },
+ { op: 'replace', path: '/spec/template/spec/dnsPolicy', value: wrc.originalPolicy },
+ { op: 'remove', path: '/spec/template/spec/dnsConfig' },
+ ])
+ }
+ await kube.rollout(wrc.identity.namespace, wrc.identity.name)
+ const restored = await kube.get(wrc.identity)
+ invariant(
+ restored?.metadata.uid === wrc.uid &&
+ restored.spec.replicas === wrc.replicas &&
+ deploymentReady(restored) &&
+ (restored.spec.template.spec.dnsPolicy ?? 'ClusterFirst') === wrc.originalPolicy &&
+ (restored.spec.template.spec.dnsConfig ?? null) === null,
+ 'WRC_RESTORE_NOT_PROVEN'
+ )
+ journal.state.wrc.attempted = false
+ journal.save()
+ } catch (error) {
+ failures.push(error)
+ }
+ }
+ // Keep the DNS fixture if restoration is uncertain; deleting it would remove
+ // the nameserver from a WRC Deployment that may still depend on it.
+ if (failures.length === 0) {
+ try {
+ ui = await captureUiChildren(kube, journal)
+ } catch (error) {
+ failures.push(error)
+ }
+ }
+ if (failures.length === 0) {
+ for (const entry of [...journal.state.resources].reverse()) {
+ try {
+ await deleteOwned(kube, journal, entry)
+ } catch (error) {
+ failures.push(error)
+ }
+ }
+ }
+ if (failures.length === 0 && ui) {
+ try {
+ await cleanupUiChildren(kube, ui)
+ } catch (error) {
+ failures.push(error)
+ }
+ }
+ if (failures.length) {
+ journal.state.phase = 'recovery-required'
+ journal.save()
+ throw new AggregateError(failures, 'WRC_EGRESS_RECOVERY_REQUIRED')
+ }
+ journal.state.phase = 'restored'
+ journal.save()
+ journal.remove()
+}
+
+async function protectedRun(kube, journal, exercise, cancellation) {
+ let original
+ try {
+ await exercise()
+ } catch (error) {
+ original = error
+ }
+ try {
+ await restore(kube, journal)
+ } catch (cleanupError) {
+ throw new AggregateError(
+ [...(original ? [original] : []), cleanupError],
+ 'WRC_EGRESS_RECOVERY_REQUIRED'
+ )
+ }
+ if (cancellation.signal.aborted) throw cancellation.signal.reason
+ if (original) throw original
+}
+
+async function exercise(config, kube, journal) {
+ const wrcIdentity = id('Deployment', config.wrcNamespace, config.wrcDeployment)
+ const wrc = await kube.get(wrcIdentity)
+ invariant(deploymentReady(wrc) && wrc.spec.replicas === 1, 'WRC_NOT_READY_OR_NOT_SINGLE_WRITER')
+ invariant(
+ (wrc.spec.template.spec.dnsPolicy ?? 'ClusterFirst') === 'ClusterFirst' &&
+ (wrc.spec.template.spec.dnsConfig ?? null) === null,
+ 'WRC_DNS_NOT_BASELINE'
+ )
+ const image = wrc.spec.template.spec.containers.find(
+ container => container.name === 'workflow-recipes'
+ )?.image
+ invariant(image, 'MISSING_WRC_IMAGE')
+ const upstream = (await kube.get(id('Service', 'kube-system', 'kube-dns')))?.spec.clusterIP
+ invariant(upstream, 'MISSING_CLUSTER_DNS')
+ const resolver = new dns.Resolver({ timeout: 5000, tries: 1 })
+ const answers = await resolver.resolve4(endpointHost)
+ const answerIp = answers.find(address => isIPv4(address) && !blocked.check(address, 'ipv4'))
+ invariant(answerIp, 'PUBLIC_ENDPOINT_UNAVAILABLE')
+ const fixture = fixtureObjects(config, journal.state.runId, image, answerIp, upstream)
+ const { names, targets } = fixture
+ for (const object of fixture.objects) await createOwned(kube, journal, object)
+ await kube.rollout(config.wrcNamespace, names.proxy, 90_000)
+ const proxy = (await kube.list('pods', config.wrcNamespace, `app=${names.proxy}`)).find(
+ pod =>
+ !pod.metadata.deletionTimestamp &&
+ pod.status?.conditions?.some(c => c.type === 'Ready' && c.status === 'True')
+ )
+ invariant(proxy?.metadata.uid, 'DNS_PROXY_NOT_READY')
+ const proxyIp = (await kube.get(id('Service', config.wrcNamespace, names.proxy)))?.spec.clusterIP
+ invariant(proxyIp, 'DNS_PROXY_ADDRESS_MISSING')
+ const dnsControl = async (route = '/state', method = 'GET', remaining = 15_000, signal) => {
+ const state = JSON.parse(
+ await kube.exec(
+ config.wrcNamespace,
+ proxy.metadata.name,
+ dnsControlCode,
+ [LOOPBACK, route, method],
+ Math.min(15_000, remaining),
+ signal
+ )
+ )
+ invariant(state.fault === null && Array.isArray(state.lanes), 'DNS_FIXTURE_FAULT')
+ return state
+ }
+ const setAffectedDns = async mode => {
+ await dnsControl(`/mode/ui/${mode}`, 'POST')
+ await dnsControl(`/mode/worker/${mode}`, 'POST')
+ }
+ await dnsControl()
+ const injectedConfig = {
+ nameservers: [proxyIp],
+ options: [
+ { name: 'ndots', value: '1' },
+ { name: 'timeout', value: '2' },
+ { name: 'attempts', value: '1' },
+ ],
+ }
+ journal.state.wrc = {
+ identity: wrcIdentity,
+ uid: wrc.metadata.uid,
+ replicas: 1,
+ originalPolicy: 'ClusterFirst',
+ injectedConfig,
+ attempted: true,
+ }
+ journal.state.phase = 'DNS-injected'
+ journal.save()
+ const beforeInjection = await kube.get(wrcIdentity)
+ invariant(
+ beforeInjection?.metadata.uid === wrc.metadata.uid &&
+ deploymentReady(beforeInjection) &&
+ (beforeInjection.spec.template.spec.dnsPolicy ?? 'ClusterFirst') === 'ClusterFirst' &&
+ (beforeInjection.spec.template.spec.dnsConfig ?? null) === null,
+ 'WRC_CHANGED_BEFORE_INJECTION'
+ )
+ await kube.patch(wrcIdentity, [
+ { op: 'test', path: '/metadata/uid', value: wrc.metadata.uid },
+ {
+ op: 'test',
+ path: '/metadata/resourceVersion',
+ value: beforeInjection.metadata.resourceVersion,
+ },
+ { op: 'add', path: '/spec/template/spec/dnsPolicy', value: 'None' },
+ { op: 'add', path: '/spec/template/spec/dnsConfig', value: injectedConfig },
+ ])
+ await kube.rollout(config.wrcNamespace, config.wrcDeployment)
+ journal.state.uiCleanup = {
+ parent: resourceIdentity(fixture.recipe),
+ parentUid: null,
+ runId: journal.state.runId,
+ namespace: config.uiNamespace,
+ workload: fixture.recipe.spec.ui.workloadRef,
+ physicalName: null,
+ children: [],
+ }
+ journal.save()
+ const recipe = await createOwned(kube, journal, fixture.recipe)
+ await createOwned(kube, journal, fixture.canary)
+ const recipeIdentity = id('WorkflowRecipe', config.recipeNamespace, names.recipe)
+ const active = async (name, remaining, signal) => {
+ const current = await kube.get(
+ id('WorkflowRecipe', config.recipeNamespace, name),
+ remaining,
+ signal
+ )
+ return current?.status?.phase === 'active'
+ }
+ await waitFor('active-recipes', 180_000, async (remaining, signal) => {
+ const deadline = Date.now() + remaining
+ return (
+ (await active(names.recipe, remaining, signal)) &&
+ (await active(names.canary, deadline - Date.now(), signal))
+ )
+ })
+ const frontend = await readyPod(kube, config.uiNamespace, names.recipe, 'frontend')
+ const worker = await readyPod(kube, config.recipeNamespace, names.recipe, 'worker')
+ const canary = await readyPod(kube, config.recipeNamespace, names.canary, 'witness')
+ const lanes = [
+ {
+ lane: 'ui',
+ workload: 'frontend',
+ namespace: config.uiNamespace,
+ pod: frontend,
+ policy: `ui-egress-${names.recipe}`,
+ target: targets[0],
+ },
+ {
+ lane: 'workload',
+ workload: 'worker',
+ namespace: config.recipeNamespace,
+ pod: worker,
+ policy: `wl-egress-${names.recipe}-worker`,
+ target: targets[1],
+ },
+ ]
+ const policiesMatch = async (ports, internalPorts, remaining = 30_000, signal) => {
+ const deadline = Date.now() + remaining
+ for (const lane of lanes) {
+ const policy = await kube.get(
+ id('NetworkPolicy', lane.namespace, lane.policy),
+ deadline - Date.now(),
+ signal
+ )
+ if (!policy) return false
+ try {
+ assertPolicy(policy, {
+ name: lane.policy,
+ namespace: lane.namespace,
+ recipe: names.recipe,
+ recipeNamespace: config.recipeNamespace,
+ lane: lane.lane,
+ workload: lane.workload,
+ podLabels: lane.pod.metadata.labels,
+ uid: lane.uid,
+ tuples: ports.map(port => ({
+ fqdn: lane.target.fqdn,
+ ip: answerIp,
+ port,
+ protocol: 'TCP',
+ })),
+ internal:
+ lane.lane === 'ui'
+ ? internalPorts.map(port => ({
+ namespace: config.recipeNamespace,
+ workload: 'worker',
+ port,
+ protocol: 'TCP',
+ }))
+ : [],
+ })
+ } catch {
+ return false
+ }
+ const observation = policyObservation(policy, lane.recoveryAfter)
+ if (!observation) return false
+ Object.assign(lane, observation)
+ lane.uid = policy.metadata.uid
+ }
+ return true
+ }
+ await waitFor('baseline-policy-proof', 120_000, (remaining, signal) =>
+ policiesMatch([80, 443], [3000, 3001], remaining, signal)
+ )
+ const services = await kube.list(
+ 'services',
+ config.recipeNamespace,
+ `clerum.io/recipe=${names.recipe},clerum.io/workload=worker`
+ )
+ invariant(
+ services.length === 1 &&
+ services[0].metadata.ownerReferences?.some(owner => owner.uid === recipe.metadata.uid),
+ 'WORKER_SERVICE_OWNERSHIP_NOT_PROVEN'
+ )
+ const workerUrl = `http://${services[0].metadata.name}.${config.recipeNamespace}.svc:3000/mcp`
+ const business = async (write = false, internal = true) => {
+ for (const [namespace, pod, url, key] of [
+ [config.uiNamespace, frontend, `http://${LOOPBACK}:3000/mcp`, 'frontend'],
+ [config.recipeNamespace, canary, `http://${LOOPBACK}:3000/mcp`, 'canary'],
+ ...(internal ? [[config.uiNamespace, frontend, workerUrl, 'worker']] : []),
+ ])
+ await kube.exec(
+ namespace,
+ pod.metadata.name,
+ businessCode,
+ [url, key, journal.state.runId, write ? 'record' : 'recall'],
+ 15_000
+ )
+ }
+ const traffic = async (namespace, pod, port, mode = 'tcp') => {
+ const output = await kube.exec(
+ namespace,
+ pod.metadata.name,
+ trafficCode,
+ [answerIp, String(port), endpointHost, mode],
+ 10_000
+ )
+ if (mode === 'https')
+ invariant(
+ same(JSON.parse(output), { allowed: true, contentVerified: true }),
+ 'HTTPS_BUSINESS_CONTENT_NOT_PROVEN'
+ )
+ else invariant(output === 'connected', 'TCP_CONNECTION_NOT_PROVEN')
+ }
+ const proveTraffic = async removed => {
+ // The canary proves the remote port is open at the moment of a negative
+ // observation. Only a connection timeout counts as policy denial.
+ await traffic(config.recipeNamespace, canary, 80)
+ await traffic(config.recipeNamespace, canary, 443, 'https')
+ for (const lane of lanes) {
+ await traffic(lane.namespace, lane.pod, 443, 'https')
+ if (!removed) await traffic(lane.namespace, lane.pod, 80)
+ else {
+ let denied = false
+ try {
+ await traffic(lane.namespace, lane.pod, 80)
+ } catch (error) {
+ denied = error.exitCode === 42
+ }
+ invariant(denied, 'REVOKED_TRAFFIC_NOT_PROVEN_DENIED')
+ await traffic(config.recipeNamespace, canary, 80)
+ }
+ }
+ }
+ await proveTraffic(false)
+ await business(true)
+ // Internal health port is genuinely listening before its declaration is removed.
+ invariant(worker.status?.podIP, 'WORKER_POD_ADDRESS_MISSING')
+ await kube.exec(
+ config.uiNamespace,
+ frontend.metadata.name,
+ trafficCode,
+ [worker.status.podIP, '3001', endpointHost, 'tcp'],
+ 10_000
+ )
+ journal.state.phase = 'exercising'
+ journal.save()
+ console.log('WRC_EGRESS_BASELINE_PROVEN')
+
+ await setAffectedDns('hold')
+ const heldBefore = await dnsControl()
+ const current = await kube.get(recipeIdentity)
+ const reduced = await kube.patch(recipeIdentity, [
+ { op: 'test', path: '/metadata/uid', value: recipe.metadata.uid },
+ { op: 'test', path: '/metadata/resourceVersion', value: current.metadata.resourceVersion },
+ {
+ op: 'replace',
+ path: '/spec/ui/egress/external',
+ value: [{ fqdn: targets[0].fqdn, port: 443 }],
+ },
+ {
+ op: 'replace',
+ path: '/spec/ui/egress/internal',
+ value: [{ workloadRef: 'worker', port: 3000 }],
+ },
+ {
+ op: 'replace',
+ path: '/spec/workloads/1/egressBindings',
+ value: [{ dns: targets[1].fqdn, port: 443, protocol: 'TCP' }],
+ },
+ ])
+ const countsFor = (state, lane, mode) => {
+ const count = state.lanes.find(item => item.lane === lane)?.received?.[mode]
+ invariant(Number.isSafeInteger(count) && count >= 0, 'MISSING_UNIQUE_DNS_OBSERVATION')
+ return count
+ }
+ await waitFor('DNS-hold-started', 60_000, async (remaining, signal) => {
+ const state = await dnsControl('/state', 'GET', remaining, signal)
+ return ['ui', 'worker'].some(
+ lane => countsFor(state, lane, 'hold') > countsFor(heldBefore, lane, 'hold')
+ )
+ })
+ // Runtime proves convergence/continuity while the resolver withholds replies.
+ // Exact pre/post-DNS ordering and same-attempt races are proved by the
+ // deterministic controller tests, not by historical packet counters here.
+ await waitFor('contraction-with-resolver-unavailable', 60_000, (remaining, signal) =>
+ policiesMatch([443], [3000], remaining, signal)
+ )
+ await business()
+ await proveTraffic(true)
+ let internalDenied = false
+ try {
+ await kube.exec(
+ config.uiNamespace,
+ frontend.metadata.name,
+ trafficCode,
+ [worker.status.podIP, '3001', endpointHost, 'tcp'],
+ 10_000
+ )
+ } catch (error) {
+ internalDenied = error.exitCode === 42
+ }
+ invariant(internalDenied, 'REMOVED_INTERNAL_PORT_NOT_DENIED')
+ console.log('WRC_EGRESS_HELD_CONTINUITY_PROVEN')
+
+ for (const lane of lanes) {
+ const policyIdentity = id('NetworkPolicy', lane.namespace, lane.policy)
+ const policy = await kube.get(policyIdentity)
+ await kube.patch(policyIdentity, [
+ { op: 'test', path: '/metadata/uid', value: lane.uid },
+ { op: 'test', path: '/metadata/resourceVersion', value: policy.metadata.resourceVersion },
+ {
+ op: 'add',
+ path: '/spec/egress/-',
+ value: {
+ to: [{ ipBlock: { cidr: `${answerIp}/32` } }],
+ ports: [{ port: 80, protocol: 'TCP' }],
+ },
+ },
+ ])
+ }
+ const failedBefore = await dnsControl()
+ await setAffectedDns('servfail')
+ await waitFor('both-DNS-lanes-SERVFAIL', 90_000, async (remaining, signal) => {
+ const state = await dnsControl('/state', 'GET', remaining, signal)
+ return ['ui', 'worker'].every(
+ lane => countsFor(state, lane, 'servfail') > countsFor(failedBefore, lane, 'servfail')
+ )
+ })
+ await waitFor('drift-convergence-under-SERVFAIL', 90_000, (remaining, signal) =>
+ policiesMatch([443], [3000], remaining, signal)
+ )
+ await business()
+ await proveTraffic(true)
+ invariant(
+ (await active(names.recipe)) && (await active(names.canary)),
+ 'RECIPE_AVAILABILITY_LOST'
+ )
+ console.log('WRC_EGRESS_SERVFAIL_CONTINUITY_PROVEN')
+
+ // A no-op refresh may legitimately avoid writing timestamps until renewal.
+ // Bound that wait by the observed stored window instead of demanding churn
+ // or mistaking a DNS packet / old active phase for accepted convergence.
+ const latestExpiry = Math.max(...lanes.map(lane => lane.latestExpiry))
+ invariant(Number.isFinite(latestExpiry), 'MISSING_ACCEPTED_DNS_WINDOW')
+ const recoveryBudget = Math.max(90_000, latestExpiry - Date.now() + 90_000)
+ invariant(recoveryBudget <= 600_000, 'DNS_WINDOW_EXCEEDS_GATE_RECOVERY_BUDGET')
+ for (const lane of lanes) {
+ invariant(Number.isFinite(lane.lastAcceptedAt), 'MISSING_ACCEPTED_DNS_OBSERVATION')
+ lane.recoveryAfter = lane.lastAcceptedAt
+ }
+ const recoveredBefore = await dnsControl()
+ await setAffectedDns('ok')
+ await waitFor('autonomous-DNS-recovery', 180_000, async (remaining, signal) => {
+ const state = await dnsControl('/state', 'GET', remaining, signal)
+ return ['ui', 'worker'].every(
+ lane => countsFor(state, lane, 'ok') > countsFor(recoveredBefore, lane, 'ok')
+ )
+ })
+ const recovered = await kube.get(recipeIdentity)
+ invariant(
+ recovered.metadata.generation === reduced.metadata.generation,
+ 'RECOVERY_NEEDED_SPEC_NUDGE'
+ )
+ await waitFor('accepted-recovered-policy-proof', recoveryBudget, async (remaining, signal) => {
+ const deadline = Date.now() + remaining
+ return (
+ (await policiesMatch([443], [3000], remaining, signal)) &&
+ (await active(names.recipe, deadline - Date.now(), signal)) &&
+ (await active(names.canary, deadline - Date.now(), signal))
+ )
+ })
+ await business()
+ await proveTraffic(true)
+
+ // Exercise the real API's empty-list serialization on a valid A→B edit.
+ // Remove the internal route intentionally; no service-continuity claim is
+ // made for that explicitly revoked route after this configuration change.
+ const migration = await kube.get(recipeIdentity)
+ const migrated = await kube.patch(recipeIdentity, [
+ { op: 'test', path: '/metadata/uid', value: recipe.metadata.uid },
+ { op: 'test', path: '/metadata/resourceVersion', value: migration.metadata.resourceVersion },
+ {
+ op: 'replace',
+ path: '/spec/ui/egress',
+ value: { external: [{ fqdn: targets[1].fqdn, port: 443 }] },
+ },
+ {
+ op: 'replace',
+ path: '/spec/workloads/1/egressBindings',
+ value: [{ dns: targets[0].fqdn, port: 443, protocol: 'TCP' }],
+ },
+ ])
+ // Contracting away the worker's last old FQDN legitimately deletes its
+ // aggregate policy. Re-pin the recreated identity only after this confirmed
+ // intent change and the complete ownership/spec/provenance proof below.
+ // The baseline UID remains mandatory throughout the prior fault/race phases.
+ acceptPolicyIntentChange(lanes, migration, migrated, recipe.metadata.uid, targets)
+ await waitFor('valid-host-migration', 120_000, async (remaining, signal) => {
+ const deadline = Date.now() + remaining
+ return (
+ (await policiesMatch([443], [], remaining, signal)) &&
+ (await active(names.recipe, deadline - Date.now(), signal))
+ )
+ })
+ await business(false, false)
+ await proveTraffic(true)
+ for (const name of [config.wrcDeployment, 'control-api', 'host-context-controller']) {
+ invariant(
+ deploymentReady(await kube.get(id('Deployment', config.wrcNamespace, name))),
+ 'SIBLING_DEPLOYMENT_NOT_READY'
+ )
+ }
+ for (const lane of [...lanes, { pod: canary, namespace: config.recipeNamespace }]) {
+ const now = (
+ await kube.list(
+ 'pods',
+ lane.namespace,
+ `clerum.io/recipe=${lane.pod.metadata.labels['clerum.io/recipe']},clerum.io/workload=${lane.pod.metadata.labels['clerum.io/workload']}`
+ )
+ ).find(pod => pod.metadata.uid === lane.pod.metadata.uid)
+ invariant(
+ now &&
+ !now.metadata.deletionTimestamp &&
+ same(
+ now.status?.containerStatuses?.map(c => c.restartCount),
+ lane.pod.status?.containerStatuses?.map(c => c.restartCount)
+ ),
+ 'FIXTURE_POD_RESTARTED_OR_REPLACED'
+ )
+ }
+ console.log('WRC_EGRESS_AUTONOMOUS_RECOVERY_AND_MIGRATION_PROVEN')
+}
+
+async function main(argv) {
+ invariant(
+ argv.length === 0 || (argv.length === 1 && argv[0] === '--recover'),
+ 'INVALID_GATE_ARGUMENTS'
+ )
+ const repository = fs.realpathSync(path.join(__dirname, '../../..'))
+ invariant(
+ process.env.T2_PROJECT_DIR === repository &&
+ process.env.T2_PROFILE === process.env.T2_CONTEXT &&
+ process.env.KUBECONTEXT === process.env.T2_CONTEXT,
+ 'INVALID_GATE_BINDING'
+ )
+ const binding = {
+ repository,
+ profile: process.env.T2_PROFILE,
+ context: process.env.T2_CONTEXT,
+ branch: execFileSync('git', ['branch', '--show-current'], {
+ cwd: repository,
+ encoding: 'utf8',
+ timeout: 5000,
+ }).trim(),
+ head: execFileSync('git', ['rev-parse', 'HEAD'], {
+ cwd: repository,
+ encoding: 'utf8',
+ timeout: 5000,
+ }).trim(),
+ }
+ const cancellation = new AbortController()
+ let signalCode = 0
+ const signalHandlers = new Map(
+ ['SIGINT', 'SIGTERM'].map(signal => [
+ signal,
+ () => {
+ signalCode ||= signal === 'SIGINT' ? 130 : 143
+ const error = new Error('WRC_GATE_INTERRUPTED')
+ error.exitCode = signalCode
+ cancellation.abort(error)
+ },
+ ])
+ )
+ for (const [signal, handler] of signalHandlers) process.on(signal, handler)
+ const run = createCommandRunner(repository, cancellation)
+ const kube = createKubeClient(repository, binding.context, run)
+ try {
+ // Protect direct Node invocation as well as the Bash entry point.
+ await run(['bash', path.join(repository, 'scripts/minikube/require-t2-mutation-lock.sh')], {
+ timeoutMs: 15_000,
+ })
+ const file = journalPath(repository, binding.profile)
+ if (argv[0] === '--recover') {
+ const state = readJournal(file)
+ assertJournalBinding(state, binding)
+ await restore(kube, new Journal(file, state, true))
+ invariant(!signalCode, 'RECOVERY_INTERRUPTED')
+ console.log('WRC_EGRESS_RECOVERY_COMPLETE') // Never the interrupted test's PASS.
+ return
+ }
+ invariant(
+ process.env.E2E_WRC_EGRESS_FAULT_INJECTION === '1',
+ 'FAULT_INJECTION_NOT_ACKNOWLEDGED'
+ )
+ // Direct Node invocation cannot bypass exact-head/image/context verification.
+ // Reuse the supported read-only preflight while inheriting the same lease.
+ await run(
+ [
+ 'env',
+ 'T2_PLAN_MODE=false',
+ 'bash',
+ path.join(repository, 'scripts/minikube/t2-preflight.sh'),
+ ],
+ { timeoutMs: 300_000 }
+ )
+ const journal = new Journal(file, {
+ version: 1,
+ binding,
+ runId: randomUUID(),
+ phase: 'validated',
+ resources: [],
+ uiCleanup: null,
+ })
+ const config = {
+ repository,
+ wrcNamespace: process.env.WRC_NAMESPACE || 'control-plane',
+ wrcDeployment: process.env.WRC_DEPLOYMENT || 'workflow-recipes',
+ uiNamespace: process.env.SANDBOX_UI_NS || 'sandbox-ui',
+ recipeNamespace: process.env.WORKFLOW_RECIPE_NS || 'sandbox-recipes',
+ }
+ await protectedRun(kube, journal, () => exercise(config, kube, journal), cancellation)
+ console.log('WRC_EGRESS_DEGRADATION_E2E_PASS')
+ } catch (error) {
+ process.stderr.write(
+ `${error instanceof AggregateError ? 'WRC_EGRESS_RECOVERY_REQUIRED' : 'WRC_EGRESS_GATE_FAILED'}\n`
+ )
+ process.exitCode = signalCode || error.exitCode || 1
+ } finally {
+ for (const [signal, handler] of signalHandlers) process.off(signal, handler)
+ }
+}
+
+module.exports = {
+ restore,
+ protectedRun,
+ fixtureObjects,
+ deploymentReady,
+ policyObservation,
+ acceptPolicyIntentChange,
+ main,
+}
+if (require.main === module)
+ main(process.argv.slice(2)).catch(() => {
+ process.exitCode = 1
+ })
diff --git a/scripts/e2e/_lib/wrc-egress-lifecycle.cjs b/scripts/e2e/_lib/wrc-egress-lifecycle.cjs
new file mode 100644
index 000000000..7eb938f2b
--- /dev/null
+++ b/scripts/e2e/_lib/wrc-egress-lifecycle.cjs
@@ -0,0 +1,448 @@
+'use strict'
+
+// Development-only lifecycle for the disruptive WRC gate. The durable journal
+// is deliberately outside the transient profile lease: a killed process must
+// leave a recovery obligation that later certification cannot silently ignore.
+const fs = require('node:fs')
+const path = require('node:path')
+const { randomUUID } = require('node:crypto')
+const { spawn } = require('node:child_process')
+const { setTimeout: delay } = require('node:timers/promises')
+
+const RUN_LABEL = 'e2e.clerum.io/run'
+const resourceRoutes = new Map([
+ ['ConfigMap', ['v1', 'configmaps']],
+ ['Service', ['v1', 'services']],
+ ['Deployment', ['apps/v1', 'deployments']],
+ ['NetworkPolicy', ['networking.k8s.io/v1', 'networkpolicies']],
+ ['WorkflowRecipe', ['clerum.io/v1alpha1', 'workflowrecipes']],
+])
+const validName = value =>
+ typeof value === 'string' && /^[a-z0-9](?:[a-z0-9.-]{0,251}[a-z0-9])?$/.test(value)
+function invariant(value, code) {
+ if (!value) throw new Error(code)
+}
+
+function journalPath(repository, profile) {
+ invariant(validName(profile), 'INVALID_PROFILE')
+ invariant(fs.realpathSync(repository) === repository, 'NONCANONICAL_REPOSITORY')
+ return path.join(
+ repository,
+ '.local-notes',
+ 'infra',
+ 'runs',
+ 'fault-injection',
+ `wrc-${profile}.json`
+ )
+}
+
+function safeParents(file, create = false, allowMissing = false) {
+ let current = path.parse(file).root
+ for (const part of path.dirname(file).slice(current.length).split(path.sep).filter(Boolean)) {
+ current = path.join(current, part)
+ if (create && !fs.existsSync(current)) fs.mkdirSync(current, { mode: 0o700 })
+ const info = fs.lstatSync(current, { throwIfNoEntry: false })
+ if (!info && allowMissing) return
+ invariant(info, 'MISSING_JOURNAL_PARENT')
+ invariant(info.isDirectory() && !info.isSymbolicLink(), 'UNSAFE_JOURNAL_PARENT')
+ }
+}
+
+function readJournal(file) {
+ safeParents(file)
+ const info = fs.lstatSync(file)
+ invariant(info.isFile() && info.nlink === 1 && (info.mode & 0o077) === 0, 'UNSAFE_JOURNAL_FILE')
+ invariant(info.size > 0 && info.size <= 1024 * 1024, 'INVALID_JOURNAL_SIZE')
+ const state = JSON.parse(fs.readFileSync(file, 'utf8'))
+ invariant(
+ state.version === 1 &&
+ typeof state.runId === 'string' &&
+ /^[a-f0-9-]{36}$/.test(state.runId) &&
+ Array.isArray(state.resources),
+ 'INVALID_JOURNAL'
+ )
+ return state
+}
+
+function assertJournalBinding(state, expected) {
+ // HEAD is historical during recovery; stable owner identity must still match.
+ for (const key of ['repository', 'branch', 'profile', 'context']) {
+ invariant(
+ typeof expected[key] === 'string' &&
+ expected[key].length > 0 &&
+ state.binding?.[key] === expected[key],
+ 'JOURNAL_OWNER_MISMATCH'
+ )
+ }
+}
+
+class Journal {
+ constructor(file, state, existing = false) {
+ this.file = file
+ this.state = state
+ safeParents(file, true)
+ if (!existing) {
+ const fd = fs.openSync(file, 'wx', 0o600)
+ try {
+ fs.writeFileSync(fd, JSON.stringify(state))
+ fs.fsyncSync(fd)
+ } finally {
+ fs.closeSync(fd)
+ }
+ } else readJournal(file)
+ }
+ save() {
+ const current = readJournal(this.file)
+ invariant(current.runId === this.state.runId, 'JOURNAL_REPLACED')
+ const pending = `${this.file}.${randomUUID()}.new`
+ const fd = fs.openSync(pending, 'wx', 0o600)
+ try {
+ fs.writeFileSync(fd, JSON.stringify(this.state))
+ fs.fsyncSync(fd)
+ } finally {
+ fs.closeSync(fd)
+ }
+ fs.renameSync(pending, this.file)
+ }
+ remove() {
+ invariant(readJournal(this.file).runId === this.state.runId, 'JOURNAL_REPLACED')
+ fs.unlinkSync(this.file)
+ }
+}
+
+function resourceIdentity(object) {
+ const metadata = object?.metadata
+ invariant(
+ resourceRoutes.has(object?.kind) && validName(metadata?.name) && validName(metadata?.namespace),
+ 'INVALID_RESOURCE_IDENTITY'
+ )
+ return { kind: object.kind, name: metadata.name, namespace: metadata.namespace }
+}
+
+function resourcePath(identity) {
+ const { kind, name, namespace } = resourceIdentity({ kind: identity.kind, metadata: identity })
+ const [version, plural] = resourceRoutes.get(kind)
+ return `${version === 'v1' ? '/api' : '/apis'}/${version}/namespaces/${namespace}/${plural}/${name}`
+}
+
+async function createOwned(kube, journal, object) {
+ const identity = resourceIdentity(object)
+ invariant(object.metadata.labels?.[RUN_LABEL] === journal.state.runId, 'MISSING_RUN_OWNERSHIP')
+ const entry = { ...identity, attempted: true, uid: null }
+ journal.state.resources.push(entry)
+ journal.save() // Record uncertain acknowledgement BEFORE issuing the request.
+ const created = await kube.create(object)
+ invariant(
+ created?.metadata?.uid && created.metadata.labels?.[RUN_LABEL] === journal.state.runId,
+ 'UNPROVEN_CREATED_RESOURCE'
+ )
+ invariant(
+ JSON.stringify(resourceIdentity(created)) === JSON.stringify(identity),
+ 'CREATED_IDENTITY_MISMATCH'
+ )
+ entry.uid = created.metadata.uid
+ journal.save()
+ return created
+}
+
+async function deleteOwned(kube, journal, entry) {
+ const live = await kube.get(entry)
+ if (!live) return // Only the client's authoritative NotFound produces null.
+ invariant(
+ entry.uid &&
+ live.metadata?.uid === entry.uid &&
+ live.metadata.labels?.[RUN_LABEL] === journal.state.runId,
+ 'UNPROVEN_CLEANUP_OWNERSHIP'
+ )
+ invariant(live.metadata.resourceVersion, 'MISSING_DELETE_VERSION')
+ try {
+ await kube.delete(entry, { uid: entry.uid, resourceVersion: live.metadata.resourceVersion })
+ } catch (error) {
+ if (await kube.get(entry)) throw error
+ }
+ await waitFor('owned-resource-absence', 90_000, async (remaining, signal) => {
+ const current = await kube.get(entry, remaining, signal)
+ if (!current) return true
+ invariant(current.metadata?.uid === entry.uid, 'RESOURCE_RECREATED_DURING_CLEANUP')
+ return false
+ })
+}
+
+async function waitFor(label, timeoutMs, predicate, options = {}) {
+ const now = options.now ?? Date.now
+ const sleep = options.sleep ?? delay
+ const deadline = now() + timeoutMs
+ const timeoutError = new Error(`TIMEOUT:${label}`)
+ timeoutError.exitCode = 124
+ while (now() < deadline) {
+ if (options.signal?.aborted) throw options.signal.reason
+ const remaining = deadline - now()
+ // Predicates pass this remaining budget into all child I/O. Race also bounds
+ // a misbehaving predicate; cancellation must terminate its actual process.
+ const controller = new AbortController()
+ let timer
+ const cancel = () => controller.abort(options.signal.reason)
+ options.signal?.addEventListener('abort', cancel, { once: true })
+ const cancelled = new Promise((_, reject) => {
+ controller.signal.addEventListener('abort', () => reject(controller.signal.reason), {
+ once: true,
+ })
+ })
+ const expired = new Promise((_, reject) => {
+ timer = setTimeout(() => {
+ controller.abort(timeoutError)
+ reject(timeoutError)
+ }, remaining)
+ })
+ let ready
+ try {
+ ready = await Promise.race([predicate(remaining, controller.signal), expired, cancelled])
+ } finally {
+ clearTimeout(timer)
+ options.signal?.removeEventListener('abort', cancel)
+ }
+ if (options.signal?.aborted) throw options.signal.reason
+ if (ready && now() < deadline) return
+ if (now() >= deadline) break
+ await sleep(Math.min(500, deadline - now()))
+ }
+ throw timeoutError
+}
+
+function createCommandRunner(repository, cancellation) {
+ return (args, { input, timeoutMs = 30_000, signal, cleanup = false } = {}) =>
+ new Promise((resolve, reject) => {
+ if (!cleanup && cancellation.signal.aborted) return reject(cancellation.signal.reason)
+ const effectiveSignal = cleanup
+ ? signal
+ : AbortSignal.any([cancellation.signal, ...(signal ? [signal] : [])])
+ if (effectiveSignal?.aborted) return reject(effectiveSignal.reason)
+ const timeoutSeconds = Math.max(1, Math.ceil(timeoutMs / 1000))
+ const child = spawn(
+ process.execPath,
+ [
+ path.join(repository, 'scripts/minikube/run-with-deadline.mjs'),
+ '--timeout-seconds',
+ String(timeoutSeconds),
+ '--kill-grace-seconds',
+ '2',
+ '--heartbeat-seconds',
+ '20',
+ '--label',
+ 'wrc-egress-operation',
+ '--',
+ ...args,
+ ],
+ { stdio: ['pipe', 'pipe', 'pipe'], env: process.env }
+ )
+ const started = Date.now()
+ const stdout = []
+ let bytes = 0
+ // Never echo arbitrary API bodies, pod output, or access configuration.
+ child.stderr.resume()
+ const abort = () => child.kill('SIGTERM')
+ effectiveSignal?.addEventListener('abort', abort, { once: true })
+ // The reused process-group supervisor accepts seconds; this parent timer
+ // enforces the remaining millisecond phase budget without granting another
+ // rounded-up second in which a command could still perform work.
+ let timedOut = false
+ const timer = setTimeout(() => {
+ timedOut = true
+ abort()
+ }, timeoutMs)
+ child.stdout.on('data', chunk => {
+ bytes += chunk.length
+ if (bytes > 4 * 1024 * 1024) abort()
+ else stdout.push(chunk)
+ })
+ child.once('error', error => {
+ clearTimeout(timer)
+ effectiveSignal?.removeEventListener('abort', abort)
+ reject(error)
+ })
+ child.once('close', (code, signalName) => {
+ clearTimeout(timer)
+ effectiveSignal?.removeEventListener('abort', abort)
+ if (
+ timedOut ||
+ code !== 0 ||
+ signalName ||
+ effectiveSignal?.aborted ||
+ bytes > 4 * 1024 * 1024 ||
+ Date.now() - started > timeoutMs
+ ) {
+ const error = new Error('WRC_GATE_OPERATION_FAILED')
+ error.exitCode =
+ timedOut || Date.now() - started > timeoutMs
+ ? 124
+ : effectiveSignal?.reason?.exitCode || code || 1
+ reject(error)
+ } else resolve(Buffer.concat(stdout).toString('utf8'))
+ })
+ child.stdin.on('error', () => {})
+ child.stdin.end(input ?? '')
+ })
+}
+
+function createKubeClient(repository, context, run) {
+ let cleanup = false
+ const pending = new Set()
+ const readBudget = timeoutMs => {
+ const budget = timeoutMs ?? 30_000
+ if (!Number.isFinite(budget) || budget <= 0) {
+ const error = new Error('TIMEOUT:wrc-read')
+ error.exitCode = 124
+ throw error
+ }
+ return Math.min(30_000, budget)
+ }
+ const command = async (args, input, timeoutMs = 30_000, signal) => {
+ const operation = run(
+ [
+ 'kubectl',
+ `--context=${context}`,
+ `--request-timeout=${Math.max(1, Math.ceil(timeoutMs / 1000))}s`,
+ ...args,
+ ],
+ { input, timeoutMs, signal, cleanup }
+ )
+ pending.add(operation)
+ try {
+ return await operation
+ } finally {
+ pending.delete(operation)
+ }
+ }
+ const mutation = async (args, input) => {
+ await run(['bash', path.join(repository, 'scripts/minikube/require-t2-mutation-lock.sh')], {
+ timeoutMs: 15_000,
+ cleanup,
+ })
+ return command(args, input)
+ }
+ return {
+ async setCleanup(value) {
+ cleanup = value
+ // A polling deadline can abort a GET before its process-group supervisor
+ // has finished reaping it. Drain those bounded operations before cleanup
+ // mutates resources or releases the profile's ownership obligation.
+ if (value) await Promise.allSettled([...pending])
+ },
+ async get(identity, timeoutMs, signal) {
+ resourceIdentity({ kind: identity.kind, metadata: identity })
+ const text = await command(
+ [
+ 'get',
+ identity.kind,
+ identity.name,
+ '-n',
+ identity.namespace,
+ '--ignore-not-found=true',
+ '-o',
+ 'json',
+ ],
+ undefined,
+ readBudget(timeoutMs),
+ signal
+ )
+ return text.trim() === '' ? null : JSON.parse(text)
+ },
+ async list(kind, namespace, selector, timeoutMs, signal) {
+ return JSON.parse(
+ await command(
+ ['get', kind, '-n', namespace, '-l', selector, '-o', 'json'],
+ undefined,
+ readBudget(timeoutMs),
+ signal
+ )
+ ).items
+ },
+ async create(object) {
+ resourceIdentity(object)
+ return JSON.parse(await mutation(['create', '-f', '-', '-o', 'json'], JSON.stringify(object)))
+ },
+ async patch(identity, patch) {
+ resourceIdentity({ kind: identity.kind, metadata: identity })
+ return JSON.parse(
+ await mutation([
+ 'patch',
+ identity.kind,
+ identity.name,
+ '-n',
+ identity.namespace,
+ '--type=json',
+ '-p',
+ JSON.stringify(patch),
+ '-o',
+ 'json',
+ ])
+ )
+ },
+ async delete(identity, preconditions) {
+ // kubectl RawDelete accepts a DeleteOptions request body via stdin.
+ return mutation(
+ ['delete', `--raw=${resourcePath(identity)}`, '-f', '-'],
+ JSON.stringify({
+ apiVersion: 'v1',
+ kind: 'DeleteOptions',
+ preconditions,
+ propagationPolicy: 'Foreground',
+ })
+ )
+ },
+ async exec(namespace, pod, code, args = [], timeoutMs = 20_000, signal) {
+ return command(
+ ['exec', pod, '-n', namespace, '--', 'node', '-e', code, ...args],
+ undefined,
+ timeoutMs,
+ signal
+ )
+ },
+ async rollout(namespace, name, timeoutMs = 180_000) {
+ return command(
+ [
+ 'rollout',
+ 'status',
+ `deployment/${name}`,
+ '-n',
+ namespace,
+ `--timeout=${Math.ceil(timeoutMs / 1000)}s`,
+ ],
+ undefined,
+ timeoutMs
+ )
+ },
+ }
+}
+
+module.exports = {
+ Journal,
+ journalPath,
+ readJournal,
+ assertJournalBinding,
+ createOwned,
+ deleteOwned,
+ resourceIdentity,
+ resourcePath,
+ waitFor,
+ createCommandRunner,
+ createKubeClient,
+ invariant,
+ RUN_LABEL,
+}
+
+if (require.main === module) {
+ const [repository, profile] = process.argv.slice(2)
+ try {
+ const file = journalPath(repository, profile)
+ safeParents(file, false, true)
+ if (fs.existsSync(file) || fs.lstatSync(file, { throwIfNoEntry: false })) {
+ throw new Error('WRC_EGRESS_RECOVERY_REQUIRED')
+ }
+ } catch (error) {
+ process.stderr.write(
+ 'WRC_EGRESS_RECOVERY_REQUIRED: resolve the owned gate journal before certification\n'
+ )
+ process.exitCode = 1
+ }
+}
diff --git a/scripts/e2e/_lib/wrc-egress-proof.cjs b/scripts/e2e/_lib/wrc-egress-proof.cjs
new file mode 100644
index 000000000..2784937aa
--- /dev/null
+++ b/scripts/e2e/_lib/wrc-egress-proof.cjs
@@ -0,0 +1,366 @@
+#!/usr/bin/env node
+'use strict'
+
+// Test oracle for the WRC egress fixture. Deliberately independent of
+// the production policy builders/comparators: compare complete authorization
+// tuples only after rejecting every unsupported/wildcard shape.
+const { isIPv4 } = require('node:net')
+
+const STATE = 'clerum.io/egress-fqdn-state'
+const TARGETS = 'clerum.io/egress-fqdn-targets'
+const RESOLVED_AT = 'clerum.io/egress-fqdn-resolved-at'
+const MAX_ENTRIES = 128
+const MAX_INPUT_BYTES = 1024 * 1024
+
+/**
+ * @typedef {{fqdn:string, ip:string, port:number, protocol:'TCP'}} ExternalTuple
+ * @typedef {{namespace:string, workload:string, port:number, protocol:'TCP'}} InternalTuple
+ * @typedef {{name:string, namespace:string, recipe:string,
+ * recipeNamespace:string, lane:'ui'|'workload', workload:string,
+ * podLabels:Record<string,string>, tuples:ExternalTuple[], internal?:InternalTuple[], uid?:string}} Expectation
+ */
+
+class ProofError extends Error {
+ constructor(code, input = false) {
+ super(code)
+ this.name = 'ProofError'
+ this.code = code
+ this.input = input
+ }
+}
+
+const record = value => value !== null && typeof value === 'object' && !Array.isArray(value)
+const hasKeys = (value, allowed) =>
+ record(value) && Object.keys(value).every(key => allowed.includes(key))
+const nonempty = value => typeof value === 'string' && value.length > 0
+const validName = value =>
+ typeof value === 'string' &&
+ value.length <= 253 &&
+ value.split('.').every(label => /^[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?$/.test(label))
+const dateNumber = value => Number.isFinite(value) && Number.isFinite(new Date(value).getTime())
+const sameSet = (left, right) =>
+ left.size === right.size && [...left].every(value => right.has(value))
+const tupleKey = entry => `${entry.fqdn}\0${entry.ip}\0${entry.port}\0${entry.protocol}`
+const networkKey = entry => `${entry.ip}\0${entry.port}\0${entry.protocol}`
+const internalKey = (entry, recipe) =>
+ `internal\0${entry.namespace}\0${recipe}\0${entry.workload}\0${entry.port}\0${entry.protocol}`
+
+function requireProof(condition, code, input = false) {
+ if (!condition) throw new ProofError(code, input)
+}
+
+function checkTuple(entry, input) {
+ requireProof(
+ record(entry) &&
+ validName(entry.fqdn) &&
+ entry.fqdn.includes('.') &&
+ typeof entry.ip === 'string' &&
+ isIPv4(entry.ip) &&
+ Number.isInteger(entry.port) &&
+ entry.port >= 1 &&
+ entry.port <= 65535 &&
+ entry.protocol === 'TCP',
+ 'INVALID_EXTERNAL_TUPLE',
+ input
+ )
+}
+
+/** @param {Expectation} expected */
+function checkExpectation(expected) {
+ requireProof(
+ hasKeys(expected, [
+ 'name',
+ 'namespace',
+ 'recipe',
+ 'recipeNamespace',
+ 'lane',
+ 'workload',
+ 'podLabels',
+ 'tuples',
+ 'internal',
+ 'uid',
+ ]),
+ 'INVALID_EXPECTATION_FIELDS',
+ true
+ )
+ for (const key of ['name', 'namespace', 'recipe', 'recipeNamespace', 'workload']) {
+ requireProof(validName(expected[key]), 'INVALID_EXPECTATION_IDENTITY', true)
+ }
+ requireProof(
+ expected.lane === 'ui' || expected.lane === 'workload',
+ 'INVALID_EXPECTATION_LANE',
+ true
+ )
+ requireProof(
+ expected.uid === undefined || nonempty(expected.uid),
+ 'INVALID_EXPECTATION_UID',
+ true
+ )
+ requireProof(
+ record(expected.podLabels) &&
+ Object.values(expected.podLabels).every(value => typeof value === 'string'),
+ 'INVALID_EXPECTATION_POD_LABELS',
+ true
+ )
+ requireProof(
+ Array.isArray(expected.tuples) && expected.tuples.length <= MAX_ENTRIES,
+ 'INVALID_EXPECTATION_TUPLES',
+ true
+ )
+ const identities = new Set()
+ for (const entry of expected.tuples) {
+ requireProof(
+ hasKeys(entry, ['fqdn', 'ip', 'port', 'protocol']),
+ 'INVALID_EXPECTATION_TUPLE_FIELDS',
+ true
+ )
+ checkTuple(entry, true)
+ requireProof(!identities.has(tupleKey(entry)), 'DUPLICATE_EXPECTATION_TUPLE', true)
+ identities.add(tupleKey(entry))
+ }
+ requireProof(
+ expected.internal === undefined ||
+ (Array.isArray(expected.internal) &&
+ expected.internal.length <= 20 &&
+ (expected.internal.length === 0 || expected.lane === 'ui')),
+ 'INVALID_EXPECTATION_INTERNAL',
+ true
+ )
+ const internalIdentities = new Set()
+ for (const entry of expected.internal ?? []) {
+ requireProof(
+ hasKeys(entry, ['namespace', 'workload', 'port', 'protocol']) &&
+ validName(entry.namespace) &&
+ validName(entry.workload) &&
+ Number.isInteger(entry.port) &&
+ entry.port >= 1 &&
+ entry.port <= 65535 &&
+ entry.protocol === 'TCP',
+ 'INVALID_EXPECTATION_INTERNAL',
+ true
+ )
+ const key = internalKey(entry, expected.recipe)
+ requireProof(!internalIdentities.has(key), 'DUPLICATE_EXPECTATION_INTERNAL', true)
+ internalIdentities.add(key)
+ }
+}
+
+/** Throws a sanitized ProofError; never includes the supplied policy body.
+ * @param {unknown} policy
+ * @param {Expectation} expected
+ */
+function assertPolicy(policy, expected) {
+ checkExpectation(expected)
+ requireProof(
+ record(policy) &&
+ policy.apiVersion === 'networking.k8s.io/v1' &&
+ policy.kind === 'NetworkPolicy',
+ 'INVALID_POLICY_KIND'
+ )
+ const metadata = policy.metadata
+ requireProof(
+ record(metadata) &&
+ metadata.name === expected.name &&
+ metadata.namespace === expected.namespace &&
+ nonempty(metadata.uid) &&
+ nonempty(metadata.resourceVersion) &&
+ !metadata.deletionTimestamp &&
+ (expected.uid === undefined || metadata.uid === expected.uid),
+ 'POLICY_IDENTITY_MISMATCH'
+ )
+
+ const labels = metadata.labels
+ requireProof(
+ record(labels) &&
+ labels['clerum.io/managed-by'] === 'workflow-recipes' &&
+ labels['clerum.io/recipe'] === expected.recipe &&
+ labels['clerum.io/recipe-name'] === expected.recipe &&
+ labels['clerum.io/recipe-namespace'] === expected.recipeNamespace &&
+ (expected.lane !== 'workload' || labels['clerum.io/workload'] === expected.workload),
+ 'POLICY_OWNERSHIP_MISMATCH'
+ )
+
+ const selector =
+ expected.lane === 'ui'
+ ? {
+ 'clerum.io/sandbox-ui': 'true',
+ 'clerum.io/recipe-namespace': expected.recipeNamespace,
+ 'clerum.io/recipe-name': expected.recipe,
+ }
+ : { 'clerum.io/recipe': expected.recipe, 'clerum.io/workload': expected.workload }
+ requireProof(
+ expected.podLabels['clerum.io/recipe'] === expected.recipe &&
+ expected.podLabels['clerum.io/workload'] === expected.workload &&
+ Object.entries(selector).every(([key, value]) => expected.podLabels[key] === value),
+ 'POD_IDENTITY_MISMATCH'
+ )
+ const spec = policy.spec
+ requireProof(hasKeys(spec, ['podSelector', 'policyTypes', 'egress']), 'UNSUPPORTED_POLICY_SPEC')
+ requireProof(
+ Array.isArray(spec.policyTypes) &&
+ spec.policyTypes.length === 1 &&
+ spec.policyTypes[0] === 'Egress',
+ 'POLICY_TYPES_MISMATCH'
+ )
+ requireProof(
+ hasKeys(spec.podSelector, ['matchLabels']) &&
+ record(spec.podSelector.matchLabels) &&
+ Object.keys(spec.podSelector.matchLabels).length === Object.keys(selector).length &&
+ Object.entries(selector).every(([key, value]) => spec.podSelector.matchLabels[key] === value),
+ 'POLICY_SELECTOR_MISMATCH'
+ )
+
+ // Kubernetes may omit the top-level empty list. With explicit Egress isolation
+ // this is deny-all; missing to/ports INSIDE a rule is unrestricted and rejected.
+ requireProof(spec.egress === undefined || Array.isArray(spec.egress), 'INVALID_EGRESS_LIST')
+ const actualNetwork = new Set()
+ for (const rule of spec.egress ?? []) {
+ requireProof(
+ hasKeys(rule, ['to', 'ports']) &&
+ Array.isArray(rule.to) &&
+ rule.to.length > 0 &&
+ Array.isArray(rule.ports) &&
+ rule.ports.length > 0,
+ 'UNRESTRICTED_OR_UNKNOWN_RULE'
+ )
+ for (const peer of rule.to) {
+ let internal
+ if (record(peer) && Object.hasOwn(peer, 'ipBlock')) {
+ requireProof(
+ hasKeys(peer, ['ipBlock']) &&
+ hasKeys(peer.ipBlock, ['cidr']) &&
+ typeof peer.ipBlock.cidr === 'string' &&
+ peer.ipBlock.cidr.endsWith('/32') &&
+ isIPv4(peer.ipBlock.cidr.slice(0, -3)),
+ 'UNSUPPORTED_EXTERNAL_PEER'
+ )
+ } else {
+ requireProof(
+ expected.lane === 'ui' &&
+ hasKeys(peer, ['namespaceSelector', 'podSelector']) &&
+ hasKeys(peer.namespaceSelector, ['matchLabels']) &&
+ hasKeys(peer.podSelector, ['matchLabels']) &&
+ hasKeys(peer.namespaceSelector.matchLabels, ['kubernetes.io/metadata.name']) &&
+ validName(peer.namespaceSelector.matchLabels['kubernetes.io/metadata.name']) &&
+ hasKeys(peer.podSelector.matchLabels, ['clerum.io/recipe', 'clerum.io/workload']) &&
+ peer.podSelector.matchLabels['clerum.io/recipe'] === expected.recipe &&
+ validName(peer.podSelector.matchLabels['clerum.io/workload']),
+ 'UNSUPPORTED_INTERNAL_PEER'
+ )
+ internal = {
+ namespace: peer.namespaceSelector.matchLabels['kubernetes.io/metadata.name'],
+ workload: peer.podSelector.matchLabels['clerum.io/workload'],
+ }
+ }
+ for (const port of rule.ports) {
+ requireProof(
+ hasKeys(port, ['port', 'protocol']) &&
+ Number.isInteger(port.port) &&
+ port.port >= 1 &&
+ port.port <= 65535 &&
+ (port.protocol === undefined || port.protocol === 'TCP'),
+ 'UNSUPPORTED_EXTERNAL_PORT'
+ )
+ actualNetwork.add(
+ internal
+ ? internalKey({ ...internal, port: port.port, protocol: 'TCP' }, expected.recipe)
+ : networkKey({ ip: peer.ipBlock.cidr.slice(0, -3), port: port.port, protocol: 'TCP' })
+ )
+ }
+ }
+ }
+ const expectedNetwork = new Set([
+ ...expected.tuples.map(networkKey),
+ ...(expected.internal ?? []).map(entry => internalKey(entry, expected.recipe)),
+ ])
+ requireProof(sameSet(actualNetwork, expectedNetwork), 'EXTERNAL_TUPLES_MISMATCH')
+
+ const annotations = metadata.annotations
+ requireProof(record(annotations) && typeof annotations[STATE] === 'string', 'MISSING_PROVENANCE')
+ let entries
+ try {
+ entries = JSON.parse(annotations[STATE])
+ } catch {
+ throw new ProofError('INVALID_PROVENANCE_JSON')
+ }
+ requireProof(Array.isArray(entries) && entries.length <= MAX_ENTRIES, 'INVALID_PROVENANCE_LIST')
+ const identities = new Set()
+ for (const entry of entries) {
+ requireProof(
+ hasKeys(entry, ['fqdn', 'ip', 'port', 'protocol', 'expiresAt', 'lastObservedAt']),
+ 'INVALID_PROVENANCE_FIELDS'
+ )
+ checkTuple(entry, false)
+ requireProof(
+ dateNumber(entry.expiresAt) &&
+ dateNumber(entry.lastObservedAt) &&
+ entry.expiresAt >= entry.lastObservedAt,
+ 'INVALID_PROVENANCE_TIME'
+ )
+ requireProof(!identities.has(tupleKey(entry)), 'DUPLICATE_PROVENANCE')
+ identities.add(tupleKey(entry))
+ }
+ requireProof(
+ sameSet(identities, new Set(expected.tuples.map(tupleKey))),
+ 'PROVENANCE_TUPLES_MISMATCH'
+ )
+ const expectedTargets = expected.tuples.map(entry => `${entry.fqdn}=${entry.ip}/32`).sort()
+ const targets = annotations[TARGETS]
+ requireProof(
+ typeof targets === 'string' &&
+ JSON.stringify(targets === '' ? [] : targets.split(',').sort()) ===
+ JSON.stringify(expectedTargets),
+ 'PROVENANCE_TARGETS_MISMATCH'
+ )
+ const resolvedAt = annotations[RESOLVED_AT]
+ requireProof(
+ (entries.length === 0 && resolvedAt === undefined) ||
+ (typeof resolvedAt === 'string' && Number.isFinite(Date.parse(resolvedAt))),
+ 'INVALID_RESOLVED_AT'
+ )
+ return { ok: true }
+}
+
+function validatePolicy(policy, expected) {
+ try {
+ return assertPolicy(policy, expected)
+ } catch (error) {
+ if (!(error instanceof ProofError)) throw error
+ return { ok: false, code: error.code, input: error.input }
+ }
+}
+
+async function main(argv) {
+ requireProof(argv.length === 2 && argv[0] === '--expect-json', 'USAGE_EXPECT_JSON_REQUIRED', true)
+ let expected
+ try {
+ expected = JSON.parse(argv[1])
+ } catch {
+ throw new ProofError('INVALID_EXPECTATION_JSON', true)
+ }
+ checkExpectation(expected)
+ const chunks = []
+ let bytes = 0
+ for await (const chunk of process.stdin) {
+ bytes += chunk.length
+ requireProof(bytes <= MAX_INPUT_BYTES, 'POLICY_INPUT_TOO_LARGE', true)
+ chunks.push(chunk)
+ }
+ let policy
+ try {
+ policy = JSON.parse(Buffer.concat(chunks).toString('utf8'))
+ } catch {
+ throw new ProofError('INVALID_POLICY_JSON', true)
+ }
+ process.stdout.write(`${JSON.stringify(assertPolicy(policy, expected))}\n`)
+}
+
+module.exports = { assertPolicy, validatePolicy, ProofError }
+if (require.main === module) {
+ main(process.argv.slice(2)).catch(error => {
+ process.stderr.write(
+ `${JSON.stringify({ ok: false, code: error instanceof ProofError ? error.code : 'PROOF_FAILED' })}\n`
+ )
+ process.exitCode = error instanceof ProofError && error.input ? 2 : 1
+ })
+}
diff --git a/scripts/e2e/e2e-wrc-egress-degradation.sh b/scripts/e2e/e2e-wrc-egress-degradation.sh
index 46f997be5..1bdbd7b94 100755
--- a/scripts/e2e/e2e-wrc-egress-degradation.sh
+++ b/scripts/e2e/e2e-wrc-egress-degradation.sh
@@ -1,542 +1,42 @@
#!/usr/bin/env bash
-# Proves that a correctly configured, active WorkflowRecipe stays available
-# while WRC contracts removed UI/workload egress authorization across a real
-# transient DNS failure. The fixture also widens both live policies while DNS
-# is held and proves the post-DNS fresh-read contraction removes that drift
-# before the reconcile completes.
+# Admission-only entry point. The Node runner owns the reversible lifecycle;
+# this product E2E is not T2.
set -euo pipefail
-
-SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
-# shellcheck source=scripts/e2e/e2e-lib.sh
-# shellcheck disable=SC1091
+SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)"
+ROOT="$(cd -- "${SCRIPT_DIR}/../.." && pwd -P)"
+# shellcheck source=e2e-lib.sh
source "${SCRIPT_DIR}/e2e-lib.sh"
-# Reuse the exact-head/profile/fingerprint validator used by the disruptive HCC
-# gates. Despite its historical name, this helper performs no HCC mutation.
-# shellcheck source=scripts/e2e/_lib/hcc-watch-recovery-fixture.sh
-# shellcheck disable=SC1091
+# shellcheck source=_lib/hcc-watch-recovery-fixture.sh
source "${SCRIPT_DIR}/_lib/hcc-watch-recovery-fixture.sh"
-
-[ -n "$E2E_KUBECONTEXT" ] || {
- echo "KUBECONTEXT or E2E_K8S_CONTEXT must select an explicit branch profile." >&2
- exit 1
-}
-is_branch_scoped_e2e_context "$E2E_KUBECONTEXT" || {
- echo "Refusing WRC fault injection on non-branch context '${E2E_KUBECONTEXT}'." >&2
- exit 1
-}
+die() { printf '[wrc-egress-degradation] %s\n' "$*" >&2; exit 1; }
+RECOVER=false
+if [[ "${1:-}" == --recover && "$#" -eq 1 ]]; then
+ RECOVER=true
+elif [[ "$#" -ne 0 ]]; then
+ die 'usage: e2e-wrc-egress-degradation.sh [--recover]'
+fi
+[[ -n "$E2E_KUBECONTEXT" && "${MINIKUBE_PROFILE:-}" == "$E2E_KUBECONTEXT" ]] || die 'explicit matching branch profile/context required'
+is_branch_scoped_e2e_context "$E2E_KUBECONTEXT" || die 'non-branch context refused'
require_safe_kube_context
-[ "${E2E_WRC_EGRESS_FAULT_INJECTION:-0}" = 1 ] || {
- echo "Set E2E_WRC_EGRESS_FAULT_INJECTION=1 to acknowledge reversible WRC DNS fault injection." >&2
- exit 1
-}
-command -v jq >/dev/null 2>&1 || { echo "jq is required" >&2; exit 1; }
-command -v node >/dev/null 2>&1 || { echo "node is required" >&2; exit 1; }
-
-WRC_NS="${WRC_NAMESPACE:-control-plane}"
-WRC_DEPLOY="${WRC_DEPLOYMENT:-workflow-recipes}"
-UI_NS="${SANDBOX_UI_NS:-sandbox-ui}"
-RECIPE_NAMESPACE="${WORKFLOW_RECIPE_NS:-sandbox-recipes}"
-RUN_ID="$(date +%s)-$$"
-RUN_LABEL="$(truncate_rfc1123 "$RUN_ID")"
-SUITE_NAME=wrc-egress-degradation
-RECIPE_NAME="$(truncate_rfc1123 "e2e-wrc-egress-${RUN_ID}")"
-TARGET_DNS="$(truncate_rfc1123 "wrc-egress-${RUN_ID}").example.com"
-ANSWER_IP=93.184.216.34
-DNS_PROXY="$(truncate_rfc1123 "e2e-wrc-dns-${RUN_ID}")"
-DNS_PROXY_CONFIGMAP="$(truncate_rfc1123 "${DNS_PROXY}-script")"
-DNS_PROXY_POLICY="$(truncate_rfc1123 "${DNS_PROXY}-policy")"
-WRC_DNS_POLICY="$(truncate_rfc1123 "${DNS_PROXY}-from-wrc")"
-LOCK_NAME="e2e-wrc-egress-degradation-lock"
-UI_POLICY="ui-egress-${RECIPE_NAME}"
-WORKLOAD_POLICY="wl-egress-${RECIPE_NAME}-worker"
-DNS_PROXY_SCRIPT="${SCRIPT_DIR}/../../tests/e2e/fixtures/wrc-egress-dns-proxy/server.cjs"
-
-LOCK_UID=""
-LOCK_ACQUIRED=0
-PROXY_CREATED=0
-WRC_MUTATED=0
-RECIPE_CREATED=0
-ORIGINAL_DNS_POLICY=""
-ORIGINAL_DNS_CONFIG=""
-ORIGINAL_REPLICAS=""
-WRC_IMAGE=""
-UPSTREAM_DNS_IP=""
-
-die() {
- echo "[wrc-egress-degradation] ERROR: $*" >&2
- if [ "$PROXY_CREATED" = 1 ]; then
- kctl logs "deployment/${DNS_PROXY}" -n "$WRC_NS" -c dns-proxy --tail=120 >&2 || true
- fi
- exit 1
-}
-
-wait_until() {
- local timeout=$1 description=$2
- shift 2
- local deadline=$(( $(date +%s) + timeout ))
- while [ "$(date +%s)" -lt "$deadline" ]; do
- "$@" && return 0
- sleep 1
- done
- echo "Timed out after ${timeout}s waiting for ${description}" >&2
- return 1
-}
-
-deployment_ready() {
- local namespace=$1 name=$2
- local deployment
- deployment="$(kctl get deployment "$name" -n "$namespace" -o json 2>/dev/null)" || return 1
- jq -e '
- (.spec.replicas // 0) > 0 and
- (.status.observedGeneration // 0) >= (.metadata.generation // 0) and
- (.status.readyReplicas // 0) == (.spec.replicas // 0) and
- (.status.availableReplicas // 0) == (.spec.replicas // 0)
- ' <<<"$deployment" >/dev/null
-}
-
-recipe_active() {
- [ "$(kctl get workflowrecipe "$RECIPE_NAME" -n "$RECIPE_NAMESPACE" \
- -o jsonpath='{.status.phase}' 2>/dev/null)" = active ]
-}
-
-policy_matches_ports() {
- local namespace=$1 name=$2 expected_json=$3
- local policy
- policy="$(kctl get networkpolicy "$name" -n "$namespace" -o json 2>/dev/null)" || return 1
- jq -e --arg answer "${ANSWER_IP}/32" --argjson expected "$expected_json" '
- .metadata.labels["clerum.io/managed-by"] == "workflow-recipes" and
- ([.spec.egress[]?.ports[]?.port] | unique | sort) == ($expected | unique | sort) and
- ([.spec.egress[]?.to[]?.ipBlock.cidr] | unique) == [$answer] and
- ((.metadata.annotations["clerum.io/egress-fqdn-state"] | fromjson |
- map(.port) | unique | sort) == ($expected | unique | sort))
- ' <<<"$policy" >/dev/null
-}
-
-policy_has_port() {
- local namespace=$1 name=$2 port=$3
- kctl get networkpolicy "$name" -n "$namespace" -o json 2>/dev/null |
- jq -e --argjson port "$port" 'any(.spec.egress[]?.ports[]?; .port == $port)' >/dev/null
-}
-
-append_policy_port() {
- local namespace=$1 name=$2 port=$3
- local patch
- patch="$(jq -cn --arg cidr "${ANSWER_IP}/32" --argjson port "$port" '
- [{op:"add",path:"/spec/egress/-",value:{
- to:[{ipBlock:{cidr:$cidr}}],ports:[{port:$port,protocol:"TCP"}]
- }}]
- ')"
- kctl patch networkpolicy "$name" -n "$namespace" --type=json -p "$patch" >/dev/null
-}
-
-workload_health() {
- local namespace=$1 workload=$2
- local pod
- pod="$(ready_pod_name "$namespace" \
- "clerum.io/recipe=${RECIPE_NAME},clerum.io/workload=${workload}" 2>/dev/null)" || return 1
- # loopback is intentionally assembled inside JavaScript for the repository's
- # public/private evidence boundary; no shell expansion belongs in this body.
- # shellcheck disable=SC2016
- kctl exec "$pod" -n "$namespace" -- node -e '
- const loopback = [127, 0, 0, 1].join(".");
- fetch(`http://${loopback}:3001/`)
- .then(async response => {
- const body = await response.json();
- if (response.status !== 200 || body.status !== "ok") process.exit(2);
- })
- .catch(() => process.exit(3));
- ' >/dev/null 2>&1
-}
-
-dns_state() {
- kctl exec "deployment/${DNS_PROXY}" -n "$WRC_NS" -c dns-proxy -- node -e '
- const http = require("node:http");
- http.get({host:"127.0.0.1",port:8090,path:"/state"}, response => {
- let body=""; response.on("data", chunk => body += chunk);
- response.on("end", () => { process.stdout.write(body); process.exit(response.statusCode === 200 ? 0 : 2); });
- }).once("error", () => process.exit(3));
- ' 2>/dev/null
-}
-
-dns_count() {
- local key=$1
- dns_state | jq -er --arg key "$key" '.counts[$key]'
-}
-
-set_dns_mode() {
- local selected=$1
- # The JavaScript reads DNS_MODE from the explicit `env` argument; shell
- # expansion inside the program would be a quoting bug.
- # shellcheck disable=SC2016
- kctl exec "deployment/${DNS_PROXY}" -n "$WRC_NS" -c dns-proxy -- \
- env DNS_MODE="$selected" node -e '
- const http = require("node:http");
- const request = http.request(
- {host:"127.0.0.1",port:8090,path:`/mode/${process.env.DNS_MODE}`,method:"POST"},
- response => { response.resume(); response.on("end", () => process.exit(response.statusCode === 200 ? 0 : 2)); }
- );
- request.setTimeout(5000, () => request.destroy());
- request.once("error", () => process.exit(3));
- request.end();
- '
-}
-
-dns_count_at_least() {
- local key=$1 minimum=$2 current
- current="$(dns_count "$key" 2>/dev/null)" || return 1
- [ "$current" -ge "$minimum" ]
-}
-
-acquire_lock() {
- local created existing
- if ! created="$(kctl create configmap "$LOCK_NAME" -n "$WRC_NS" \
- --from-literal="holder=${RUN_ID}" \
- --from-literal="context=${E2E_KUBECONTEXT}" \
- --from-literal="gitHead=$(git -C "${SCRIPT_DIR}/../.." rev-parse HEAD)" \
- -o json 2>/dev/null)"; then
- existing="$(kctl get configmap "$LOCK_NAME" -n "$WRC_NS" -o json 2>/dev/null || true)"
- die "another WRC egress gate owns ${WRC_NS}/${LOCK_NAME}: holder=$(jq -r '.data.holder // "unknown"' <<<"${existing:-{}}"), head=$(jq -r '.data.gitHead // "unknown"' <<<"${existing:-{}}")"
- fi
- LOCK_UID="$(jq -r '.metadata.uid // ""' <<<"$created")"
- [ -n "$LOCK_UID" ] || die "created gate lock has no UID"
- LOCK_ACQUIRED=1
-}
-
-release_lock() {
- local current
- [ "$LOCK_ACQUIRED" = 1 ] || return 0
- current="$(kctl get configmap "$LOCK_NAME" -n "$WRC_NS" -o json 2>/dev/null)" || return 1
- [ "$(jq -r '.metadata.uid // ""' <<<"$current")" = "$LOCK_UID" ] || return 1
- [ "$(jq -r '.data.holder // ""' <<<"$current")" = "$RUN_ID" ] || return 1
- kctl delete configmap "$LOCK_NAME" -n "$WRC_NS" --wait=true --timeout=30s >/dev/null || return 1
- LOCK_ACQUIRED=0
-}
-
-wrc_restore_verified() {
- local deployment
- deployment="$(kctl get deployment "$WRC_DEPLOY" -n "$WRC_NS" -o json 2>/dev/null)" || return 1
- jq -e --arg policy "$ORIGINAL_DNS_POLICY" --argjson replicas "$ORIGINAL_REPLICAS" '
- .spec.replicas == $replicas and
- (.status.readyReplicas // 0) == $replicas and
- (.status.availableReplicas // 0) == $replicas and
- (.spec.template.spec.dnsPolicy // "ClusterFirst") == $policy and
- (.spec.template.spec.dnsConfig // null) == null
- ' <<<"$deployment" >/dev/null
-}
-
-delete_proxy_resources() {
- local failed=0
- kctl delete networkpolicy "$WRC_DNS_POLICY" "$DNS_PROXY_POLICY" -n "$WRC_NS" \
- --ignore-not-found --wait=true --timeout=30s >/dev/null 2>&1 || failed=1
- kctl delete service "$DNS_PROXY" -n "$WRC_NS" \
- --ignore-not-found --wait=true --timeout=30s >/dev/null 2>&1 || failed=1
- kctl delete deployment "$DNS_PROXY" -n "$WRC_NS" \
- --ignore-not-found --wait=true --timeout=60s >/dev/null 2>&1 || failed=1
- kctl delete configmap "$DNS_PROXY_CONFIGMAP" -n "$WRC_NS" \
- --ignore-not-found --wait=true --timeout=30s >/dev/null 2>&1 || failed=1
- return "$failed"
-}
-
-cleanup() {
- local status=$?
- trap - EXIT INT TERM
- local cleanup_failed=0 restore_ok=1
-
- if [ "$PROXY_CREATED" = 1 ]; then
- set_dns_mode ok >/dev/null 2>&1 || cleanup_failed=1
- fi
- if [ "$RECIPE_CREATED" = 1 ]; then
- kctl delete workflowrecipe "$RECIPE_NAME" -n "$RECIPE_NAMESPACE" \
- --ignore-not-found --wait=false >/dev/null 2>&1 || cleanup_failed=1
- wait_until 90 "fixture WorkflowRecipe deletion" \
- bash -c "! kubectl --context='${E2E_KUBECONTEXT}' get workflowrecipe '${RECIPE_NAME}' -n '${RECIPE_NAMESPACE}' >/dev/null 2>&1" || cleanup_failed=1
- fi
- if [ "$WRC_MUTATED" = 1 ]; then
- local restore_patch
- restore_patch="$(jq -cn --arg policy "$ORIGINAL_DNS_POLICY" '
- {spec:{template:{spec:{dnsPolicy:$policy,dnsConfig:null}}}}
- ')"
- kctl patch deployment "$WRC_DEPLOY" -n "$WRC_NS" --type=strategic \
- -p "$restore_patch" >/dev/null 2>&1 || restore_ok=0
- kctl rollout status deployment "$WRC_DEPLOY" -n "$WRC_NS" \
- --timeout=180s >/dev/null 2>&1 || restore_ok=0
- wrc_restore_verified || restore_ok=0
- fi
- if [ "$restore_ok" = 1 ]; then
- delete_proxy_resources || cleanup_failed=1
- else
- cleanup_failed=1
- fi
- if [ "$cleanup_failed" = 0 ] && [ "$restore_ok" = 1 ]; then
- release_lock || cleanup_failed=1
- fi
- if [ "$cleanup_failed" != 0 ]; then
- echo "WRC egress E2E cleanup/restore failed; lock retained at ${WRC_NS}/${LOCK_NAME}." >&2
- echo "Inspect: kubectl --context=${E2E_KUBECONTEXT} -n ${WRC_NS} get deployment ${WRC_DEPLOY} -o yaml" >&2
- status=1
- fi
- exit "$status"
-}
-trap cleanup EXIT INT TERM
-
-require_branch_owned_hcc_gate "$WRC_NS"
-acquire_lock
-
-[ -r "$DNS_PROXY_SCRIPT" ] || die "DNS proxy fixture is missing at ${DNS_PROXY_SCRIPT}"
-deployment_ready "$WRC_NS" "$WRC_DEPLOY" || die "WRC is not Ready before fault injection"
-deployment_ready "$WRC_NS" control-api || die "control-api is not Ready before fault injection"
-deployment_ready "$WRC_NS" host-context-controller || die "HCC is not Ready before fault injection"
-
-ORIGINAL_REPLICAS="$(kctl get deployment "$WRC_DEPLOY" -n "$WRC_NS" -o jsonpath='{.spec.replicas}')"
-[ "$ORIGINAL_REPLICAS" = 1 ] || die "expected one WRC replica, found ${ORIGINAL_REPLICAS:-unknown}"
-ORIGINAL_DNS_POLICY="$(kctl get deployment "$WRC_DEPLOY" -n "$WRC_NS" -o json |
- jq -r '.spec.template.spec.dnsPolicy // "ClusterFirst"')"
-[ "$ORIGINAL_DNS_POLICY" = ClusterFirst ] || die "WRC already has non-default dnsPolicy"
-ORIGINAL_DNS_CONFIG="$(kctl get deployment "$WRC_DEPLOY" -n "$WRC_NS" -o json |
- jq -c '.spec.template.spec.dnsConfig // null')"
-[ "$ORIGINAL_DNS_CONFIG" = null ] || die "WRC already has a dnsConfig; refusing non-restorable mutation"
-WRC_IMAGE="$(kctl get deployment "$WRC_DEPLOY" -n "$WRC_NS" \
- -o jsonpath='{.spec.template.spec.containers[?(@.name=="workflow-recipes")].image}')"
-[ -n "$WRC_IMAGE" ] || die "could not resolve WRC image"
-UPSTREAM_DNS_IP="$(kctl get service kube-dns -n kube-system -o jsonpath='{.spec.clusterIP}')"
-[ -n "$UPSTREAM_DNS_IP" ] || die "could not resolve kube-dns ClusterIP"
-
-kctl create configmap "$DNS_PROXY_CONFIGMAP" -n "$WRC_NS" \
- --from-file="server.cjs=${DNS_PROXY_SCRIPT}" --dry-run=client -o yaml |
- kctl apply -f - >/dev/null
-PROXY_CREATED=1
-
-kctl apply -f - >/dev/null <<EOF
-apiVersion: apps/v1
-kind: Deployment
-metadata:
- name: ${DNS_PROXY}
- namespace: ${WRC_NS}
- labels: {e2e.clerum.io/suite: ${SUITE_NAME}, e2e.clerum.io/run: "${RUN_LABEL}"}
-spec:
- replicas: 1
- selector: {matchLabels: {app: ${DNS_PROXY}}}
- template:
- metadata:
- labels: {app: ${DNS_PROXY}, e2e.clerum.io/suite: ${SUITE_NAME}, e2e.clerum.io/run: "${RUN_LABEL}"}
- spec:
- automountServiceAccountToken: false
- terminationGracePeriodSeconds: 1
- containers:
- - name: dns-proxy
- image: ${WRC_IMAGE}
- imagePullPolicy: IfNotPresent
- command: [node, /fixture/server.cjs]
- env:
- - {name: TARGET_DNS, value: "${TARGET_DNS}"}
- - {name: ANSWER_IP, value: "${ANSWER_IP}"}
- - {name: UPSTREAM_DNS, value: "${UPSTREAM_DNS_IP}"}
- ports:
- - {name: dns-udp, containerPort: 8053, protocol: UDP}
- - {name: dns-tcp, containerPort: 8053, protocol: TCP}
- readinessProbe: {tcpSocket: {port: 8053}, periodSeconds: 1, failureThreshold: 10}
- volumeMounts:
- - {name: fixture, mountPath: /fixture, readOnly: true}
- resources:
- requests: {cpu: 5m, memory: 20Mi}
- limits: {cpu: 75m, memory: 64Mi}
- securityContext:
- allowPrivilegeEscalation: false
- capabilities: {drop: [ALL]}
- readOnlyRootFilesystem: true
- runAsNonRoot: true
- runAsUser: 1000
- runAsGroup: 1000
- seccompProfile: {type: RuntimeDefault}
- volumes:
- - name: fixture
- configMap: {name: ${DNS_PROXY_CONFIGMAP}}
----
-apiVersion: v1
-kind: Service
-metadata:
- name: ${DNS_PROXY}
- namespace: ${WRC_NS}
-spec:
- selector: {app: ${DNS_PROXY}}
- ports:
- - {name: dns-udp, protocol: UDP, port: 53, targetPort: 8053}
- - {name: dns-tcp, protocol: TCP, port: 53, targetPort: 8053}
----
-apiVersion: networking.k8s.io/v1
-kind: NetworkPolicy
-metadata:
- name: ${DNS_PROXY_POLICY}
- namespace: ${WRC_NS}
-spec:
- podSelector: {matchLabels: {app: ${DNS_PROXY}}}
- policyTypes: [Ingress, Egress]
- ingress:
- - from: [{podSelector: {matchLabels: {app: ${WRC_DEPLOY}}}}]
- ports:
- - {protocol: UDP, port: 53}
- - {protocol: UDP, port: 8053}
- - {protocol: TCP, port: 53}
- - {protocol: TCP, port: 8053}
- egress:
- - to:
- - namespaceSelector: {matchLabels: {kubernetes.io/metadata.name: kube-system}}
- podSelector: {matchLabels: {k8s-app: kube-dns}}
- ports:
- - {protocol: UDP, port: 53}
- - {protocol: TCP, port: 53}
----
-apiVersion: networking.k8s.io/v1
-kind: NetworkPolicy
-metadata:
- name: ${WRC_DNS_POLICY}
- namespace: ${WRC_NS}
-spec:
- podSelector: {matchLabels: {app: ${WRC_DEPLOY}}}
- policyTypes: [Egress]
- egress:
- - to: [{podSelector: {matchLabels: {app: ${DNS_PROXY}}}}]
- ports:
- - {protocol: UDP, port: 53}
- - {protocol: UDP, port: 8053}
- - {protocol: TCP, port: 53}
- - {protocol: TCP, port: 8053}
-EOF
-
-kctl rollout status deployment "$DNS_PROXY" -n "$WRC_NS" --timeout=90s >/dev/null ||
- die "DNS proxy did not become Ready"
-wait_until 30 "DNS proxy listener" \
- bash -c "kubectl --context='${E2E_KUBECONTEXT}' logs deployment/'${DNS_PROXY}' -n '${WRC_NS}' -c dns-proxy | grep -Fq 'wrc egress DNS proxy ready'" ||
- die "DNS proxy did not report readiness"
-DNS_PROXY_IP="$(kctl get service "$DNS_PROXY" -n "$WRC_NS" -o jsonpath='{.spec.clusterIP}')"
-[ -n "$DNS_PROXY_IP" ] || die "DNS proxy Service has no ClusterIP"
-
-WRC_MUTATED=1
-dns_patch="$(jq -cn --arg nameserver "$DNS_PROXY_IP" '
- {spec:{template:{spec:{
- dnsPolicy:"None",
- dnsConfig:{nameservers:[$nameserver],options:[
- {name:"ndots",value:"1"},{name:"timeout",value:"2"},{name:"attempts",value:"1"}
- ]}
- }}}}
-')"
-kctl patch deployment "$WRC_DEPLOY" -n "$WRC_NS" --type=strategic -p "$dns_patch" >/dev/null
-kctl rollout status deployment "$WRC_DEPLOY" -n "$WRC_NS" --timeout=180s >/dev/null ||
- die "WRC did not become Ready with the isolated DNS proxy"
-deployment_ready "$WRC_NS" control-api || die "control-api lost readiness during WRC DNS setup"
-deployment_ready "$WRC_NS" host-context-controller || die "HCC lost readiness during WRC DNS setup"
-ok "isolated WRC DNS proxy is Ready without affecting control-api or HCC"
-
-RECIPE_CREATED=1
-kctl apply -f - >/dev/null <<EOF
-apiVersion: clerum.io/v1alpha1
-kind: WorkflowRecipe
-metadata:
- name: ${RECIPE_NAME}
- namespace: ${RECIPE_NAMESPACE}
- labels: {e2e.clerum.io/suite: ${SUITE_NAME}, e2e.clerum.io/run: "${RUN_LABEL}"}
-spec:
- description: WRC egress degradation E2E baseline
- contextRef: context1
- workloads:
- - id: frontend
- type: deployment
- image: clerum/mock-mcp-server:test
- port: 3001
- healthCheck: {type: http, path: /, port: 3001}
- - id: worker
- type: deployment
- image: clerum/mock-mcp-server:test
- port: 3001
- healthCheck: {type: http, path: /, port: 3001}
- egressBindings:
- - {dns: ${TARGET_DNS}, port: 443, protocol: TCP}
- - {dns: ${TARGET_DNS}, port: 8443, protocol: TCP}
- ui:
- workloadRef: frontend
- port: 3001
- title: WRC egress degradation fixture
- defaultPath: /
- egress:
- external:
- - {fqdn: ${TARGET_DNS}, port: 443}
- - {fqdn: ${TARGET_DNS}, port: 8443}
- security: {isolationLevel: minimal}
-EOF
-
-wait_until 180 "valid fixture recipe to become active" recipe_active ||
- die "fixture recipe did not become active"
-wait_until 120 "initial UI egress policy" policy_matches_ports "$UI_NS" "$UI_POLICY" '[443,8443]' ||
- die "initial UI policy did not enforce both declared ports"
-wait_until 120 "initial workload egress policy" policy_matches_ports "$RECIPE_NAMESPACE" "$WORKLOAD_POLICY" '[443,8443]' ||
- die "initial workload policy did not enforce both declared ports"
-wait_until 120 "frontend business health" workload_health "$UI_NS" frontend ||
- die "frontend health endpoint was unavailable"
-wait_until 120 "worker business health" workload_health "$RECIPE_NAMESPACE" worker ||
- die "worker health endpoint was unavailable"
-ok "valid recipe is active, both workload health endpoints respond, and both policies enforce the declared set"
-
-hold_before="$(dns_count hold)"
-set_dns_mode hold || die "could not hold fixture DNS"
-reduced_patch="$(jq -cn --arg dns "$TARGET_DNS" '
- [
- {op:"replace",path:"/spec/ui/egress/external",value:[{fqdn:$dns,port:443}]},
- {op:"replace",path:"/spec/workloads/1/egressBindings",value:[{dns:$dns,port:443,protocol:"TCP"}]}
- ]
-')"
-kctl patch workflowrecipe "$RECIPE_NAME" -n "$RECIPE_NAMESPACE" --type=json \
- -p "$reduced_patch" >/dev/null
-wait_until 60 "WRC DNS query to enter the deterministic hold" \
- dns_count_at_least hold "$((hold_before + 1))" || die "WRC never reached the DNS hold"
-wait_until 60 "pre-DNS UI contraction" policy_matches_ports "$UI_NS" "$UI_POLICY" '[443]' ||
- die "UI port 8443 was not revoked before DNS completed"
-wait_until 60 "pre-DNS workload contraction" policy_matches_ports "$RECIPE_NAMESPACE" "$WORKLOAD_POLICY" '[443]' ||
- die "workload port 8443 was not revoked before DNS completed"
-recipe_active || die "correctly configured recipe left active before the held DNS completed"
-workload_health "$UI_NS" frontend || die "frontend stopped serving during held DNS"
-workload_health "$RECIPE_NAMESPACE" worker || die "worker stopped serving during held DNS"
-ok "removed UI/workload permissions contracted before DNS completed while both services stayed healthy"
-
-append_policy_port "$UI_NS" "$UI_POLICY" 9443
-append_policy_port "$RECIPE_NAMESPACE" "$WORKLOAD_POLICY" 9443
-policy_has_port "$UI_NS" "$UI_POLICY" 9443 || die "could not inject UI policy race"
-policy_has_port "$RECIPE_NAMESPACE" "$WORKLOAD_POLICY" 9443 || die "could not inject workload policy race"
-
-servfail_before="$(dns_count servfail)"
-set_dns_mode servfail || die "could not release held DNS as SERVFAIL"
-wait_until 90 "both WRC lanes to observe SERVFAIL" \
- dns_count_at_least servfail "$((servfail_before + 2))" ||
- die "UI and workload lanes did not both observe the transient DNS failure"
-wait_until 90 "post-DNS UI re-contraction" policy_matches_ports "$UI_NS" "$UI_POLICY" '[443]' ||
- die "UI race widening survived transient DNS failure"
-wait_until 90 "post-DNS workload re-contraction" policy_matches_ports "$RECIPE_NAMESPACE" "$WORKLOAD_POLICY" '[443]' ||
- die "workload race widening survived transient DNS failure"
-wait_until 60 "recipe to remain active after proven fail-static" recipe_active ||
- die "correctly configured recipe was cut by transient DNS failure"
-workload_health "$UI_NS" frontend || die "frontend health failed after transient DNS failure"
-workload_health "$RECIPE_NAMESPACE" worker || die "worker health failed after transient DNS failure"
-deployment_ready "$WRC_NS" "$WRC_DEPLOY" || die "WRC lost readiness during fault"
-deployment_ready "$WRC_NS" control-api || die "control-api lost readiness during WRC fault"
-deployment_ready "$WRC_NS" host-context-controller || die "HCC lost readiness during WRC fault"
-ok "transient DNS failure removed concurrent widening, preserved proven port 443, and isolated blast radius"
-
-ok_before="$(dns_count ok)"
-set_dns_mode ok || die "could not restore successful DNS"
-recovery_patch="$(jq -cn --arg description "WRC egress recovery ${RUN_ID}" \
- '{spec:{description:$description}}')"
-kctl patch workflowrecipe "$RECIPE_NAME" -n "$RECIPE_NAMESPACE" --type=merge \
- -p "$recovery_patch" >/dev/null
-wait_until 90 "both WRC lanes to resolve after recovery" \
- dns_count_at_least ok "$((ok_before + 2))" || die "DNS recovery was not observed by both lanes"
-wait_until 90 "recovered recipe active state" recipe_active || die "recipe did not recover active"
-policy_matches_ports "$UI_NS" "$UI_POLICY" '[443]' || die "UI policy drifted after recovery"
-policy_matches_ports "$RECIPE_NAMESPACE" "$WORKLOAD_POLICY" '[443]' ||
- die "workload policy drifted after recovery"
-workload_health "$UI_NS" frontend || die "frontend health failed after DNS recovery"
-workload_health "$RECIPE_NAMESPACE" worker || die "worker health failed after DNS recovery"
-ok "successful DNS recovery reconverged both lanes without restoring removed or raced permissions"
-
-echo "WRC_EGRESS_DEGRADATION_E2E_PASS"
-echo "recipe=${RECIPE_NAMESPACE}/${RECIPE_NAME}"
-echo "uiPolicy=${UI_NS}/${UI_POLICY}"
-echo "workloadPolicy=${RECIPE_NAMESPACE}/${WORKLOAD_POLICY}"
+[[ "${T2_PROJECT_DIR:-}" == "$ROOT" && "${T2_CONTEXT:-}" == "$E2E_KUBECONTEXT" && "${T2_PROFILE:-}" == "$E2E_KUBECONTEXT" ]] || die 'inherited lease target differs from gate'
+# No mutation or cleanup handler exists before the live lease is proved.
+bash "${ROOT}/scripts/minikube/require-t2-mutation-lock.sh"
+export KUBECONTEXT="$E2E_KUBECONTEXT"
+if [[ "$RECOVER" == true ]]; then
+ exec node "${SCRIPT_DIR}/_lib/wrc-egress-gate.cjs" --recover
+fi
+[[ "${E2E_WRC_EGRESS_FAULT_INJECTION:-0}" == 1 ]] || die 'explicit fault-injection acknowledgement required'
+# Bound every read made by the historical exact-head validator.
+kctl() {
+ node "${ROOT}/scripts/minikube/run-with-deadline.mjs" \
+ --timeout-seconds 20 --kill-grace-seconds 2 --heartbeat-seconds 20 \
+ --label wrc-egress-admission -- kubectl --context="$E2E_KUBECONTEXT" --request-timeout=15s "$@"
+}
+require_branch_owned_hcc_gate "${WRC_NAMESPACE:-control-plane}"
+# The old helper checks HEAD/fingerprint, but not image acquisition freshness.
+manifest="${T2_IMAGE_MANIFEST:-${ROOT}/deploy/minikube/.image-manifest.json}"
+[[ -r "$manifest" ]] || die 'image manifest missing'
+manifest_stamp="$(jq -er '.generated | select(type == "string" and length > 0)' "$manifest")"
+marker_stamp="$(jq -er '.data.imagesGeneratedAt | select(type == "string" and length > 0)' <<<"$HCC_BRANCH_GATE_SYNC_MARKER")"
+[[ "$manifest_stamp" == "$marker_stamp" ]] || die 'image stamp differs from deployed marker'
+exec node "${SCRIPT_DIR}/_lib/wrc-egress-gate.cjs"
diff --git a/scripts/minikube/t2-common.sh b/scripts/minikube/t2-common.sh
index ca46dfdb5..84e9b4840 100755
--- a/scripts/minikube/t2-common.sh
+++ b/scripts/minikube/t2-common.sh
@@ -1097,6 +1097,15 @@ t2_mutation_lock() {
else
t2_lock_acquire
fi
+ # A disruptive local gate can outlive its process (SIGKILL/API outage). The
+ # profile lease is transient; its durable recovery obligation is not. Never
+ # certify or reconcile over an unfinished DNS intervention.
+ if ! node "$T2_SCRIPT_DIR/../e2e/_lib/wrc-egress-lifecycle.cjs" "$T2_PROJECT_DIR" "$T2_PROFILE"; then
+ t2_lock_release 0 || true
+ T2_NEXT_COMMAND='run the owned test-e2e-wrc-egress-recover target, then retry the original operation'
+ t2_fail WRC_EGRESS_RECOVERY_REQUIRED 'an unfinished WRC fault-injection journal prevents profile mutation or certification'
+ return 1
+ fi
}
t2_lock_release() {
diff --git a/scripts/minikube/with-t2-mutation-lock.sh b/scripts/minikube/with-t2-mutation-lock.sh
index 75504e155..280e38427 100755
--- a/scripts/minikube/with-t2-mutation-lock.sh
+++ b/scripts/minikube/with-t2-mutation-lock.sh
@@ -4,7 +4,10 @@
# of acquiring a second lock for the same profile.
set -euo pipefail
-if [[ "${1:-}" != "--" || "$#" -lt 2 ]]; then
+WRC_RECOVERY_ONLY=false
+if [[ "${1:-}" == "--recover-wrc-egress" && "$#" -eq 1 ]]; then
+ WRC_RECOVERY_ONLY=true
+elif [[ "${1:-}" != "--" || "$#" -lt 2 ]]; then
printf 'usage: %s -- command [args...]\n' "${0##*/}" >&2
exit 2
fi
@@ -60,7 +63,14 @@ if [[ "${T2_BOOTSTRAP_REQUIRED}" == true ]]; then
fi
t2_context_check
t2_profile_context_identity_check
-t2_mutation_lock
+if [[ "$WRC_RECOVERY_ONLY" == true ]]; then
+ # Recovery is a dedicated command, not a general skip-guard environment flag.
+ # It still requires the exact live profile lease; only the recovery body is
+ # permitted to run while that profile has a pending fault-injection journal.
+ if [[ "$T2_SKIP_LOCK" == true ]]; then t2_lock_validate_inherited; else t2_lock_acquire; fi
+else
+ t2_mutation_lock
+fi
export T2_PROJECT_DIR T2_PROFILE T2_CONTEXT T2_PROFILE_ROOT T2_PROFILE_ENV T2_PORTS_ENV \
T2_LOCK_ROOT T2_LOCK_TOKEN MINIKUBE_PROFILE CONTROL_API_REAL_PG_CONTEXT
@@ -80,5 +90,9 @@ trap 'exit 130' INT
trap 'exit 143' TERM
run_status=0
-"$@" || run_status=$?
+if [[ "$WRC_RECOVERY_ONLY" == true ]]; then
+ bash "$ROOT/scripts/e2e/e2e-wrc-egress-degradation.sh" --recover || run_status=$?
+else
+ "$@" || run_status=$?
+fi
exit "${run_status}"
diff --git a/scripts/tests/test-minikube-mutation-boundary.sh b/scripts/tests/test-minikube-mutation-boundary.sh
index 5912d3165..490b5c6e6 100755
--- a/scripts/tests/test-minikube-mutation-boundary.sh
+++ b/scripts/tests/test-minikube-mutation-boundary.sh
@@ -6,20 +6,21 @@ MUTATION_WRAPPER="${ROOT}/scripts/minikube/with-t2-mutation-lock.sh"
TMP_DIR="$(mktemp -d)"
source "${ROOT}/scripts/tests/lib/minikube-fixture-repo.sh"
-MINIKUBE_TEST_PROFILE="boundary-profile"
-MINIKUBE_TEST_CONTEXT="boundary-profile"
+MINIKUBE_TEST_PROFILE="clerum-boundary-1234abcd"
+MINIKUBE_TEST_CONTEXT="${MINIKUBE_TEST_PROFILE}"
minikube_test_fixture_repo_init "${ROOT}" "${TMP_DIR}"
cleanup() {
local status=$?
+ trap - EXIT
if ! minikube_test_assert_host_unchanged; then
status=1
fi
rm -rf "${TMP_DIR}"
- return "${status}"
+ exit "${status}"
}
trap cleanup EXIT
-PROFILE=boundary-profile
+PROFILE="${MINIKUBE_TEST_PROFILE}"
TOKEN=boundary-test-token
LOCK_ROOT="${TMP_DIR}/locks"
LOCK_DIR="${LOCK_ROOT}/${PROFILE}.lock"
@@ -30,7 +31,9 @@ BRANCH="${MINIKUBE_TEST_BRANCH}"
HEAD="${MINIKUBE_TEST_HEAD}"
WORKTREE_ID="${MINIKUBE_TEST_WORKTREE_ID}"
LOCK_KEY="${MINIKUBE_TEST_LOCK_KEY}"
-cat >"${LOCK_DIR}/owner.env" <<EOF
+write_owner() {
+ local owner_pid="${1:-$$}"
+ cat >"${LOCK_DIR}/owner.env" <<EOF
REPOSITORY=${MINIKUBE_TEST_PROJECT_DIR}
BRANCH=${BRANCH}
HEAD=${HEAD}
@@ -39,9 +42,11 @@ CONTEXT=${PROFILE}
WORKTREE_ID=${WORKTREE_ID}
LOCK_KEY=${LOCK_KEY}
TOKEN=${TOKEN}
-PID=$$
+PID=${owner_pid}
PROCESS_START=unavailable
EOF
+}
+write_owner
# GNU Make may execute recipe lines containing $(MAKE) during a dry run. Force
# the dry-run flag into the child environment so with-t2-mutation-lock.sh
@@ -51,8 +56,9 @@ dry_run_make() {
if output="$(MAKEFLAGS=-n make -n -C "${ROOT}" "$@" 2>&1)"; then
printf '%s\n' "${output}"
return 0
+ else
+ status=$?
fi
- status=$?
printf 'FAIL: dry-run make %s exited %s\n%s\n' "$*" "${status}" "${output}" >&2
return "${status}"
}
@@ -64,8 +70,9 @@ run_child() {
bash "${ROOT}/scripts/minikube/require-t2-mutation-lock.sh"; then
printf 'mutation\n' >>"${LOG}"
return 0
+ else
+ status=$?
fi
- status=$?
printf 'FAIL: valid inherited lease child exited %s (branch=%s head=%s)\n' \
"${status}" "${BRANCH}" "${HEAD}" >&2
return "${status}"
@@ -127,6 +134,137 @@ assert_makeflags_dry_run_matrix() {
printf 'PASS: compact and long GNU Make dry-run flags stop the real wrapper before profile or lock mutation\n'
}
+assert_wrc_admission() {
+ local fixture_scripts="${MINIKUBE_TEST_PROJECT_DIR}/scripts"
+ local fixture_entry="${fixture_scripts}/e2e/e2e-wrc-egress-degradation.sh"
+ local fixture_wrapper="${fixture_scripts}/minikube/with-t2-mutation-lock.sh"
+ local fake_bin="${TMP_DIR}/wrc-admission-bin"
+ local entry_log="${TMP_DIR}/wrc-entry.log"
+ local cluster_log="${TMP_DIR}/wrc-cluster.log"
+ local output="${TMP_DIR}/wrc-admission.out"
+ local real_node owner_before journal status
+ real_node="$(command -v node)"
+ mkdir -p "${fixture_scripts}/e2e/_lib" "${fixture_scripts}/minikube" "${fake_bin}"
+ # Keep the fixture repository clean for the real wrapper's Git admission.
+ # No Git operation or exclusion is applied to the host checkout.
+ printf '/scripts/\n/.local-notes/\n' >>"${MINIKUBE_TEST_PROJECT_DIR}/.git/info/exclude"
+ cp "${ROOT}/scripts/e2e/e2e-wrc-egress-degradation.sh" "${fixture_entry}"
+ cp "${ROOT}/scripts/e2e/e2e-lib.sh" "${fixture_scripts}/e2e/e2e-lib.sh"
+ cp "${ROOT}/scripts/e2e/_lib/hcc-watch-recovery-fixture.sh" "${fixture_scripts}/e2e/_lib/hcc-watch-recovery-fixture.sh"
+ cp "${MUTATION_WRAPPER}" "${fixture_wrapper}"
+ cp "${ROOT}/scripts/minikube/require-t2-mutation-lock.sh" "${fixture_scripts}/minikube/require-t2-mutation-lock.sh"
+ cp "${ROOT}/scripts/minikube/t2-common.sh" "${fixture_scripts}/minikube/t2-common.sh"
+ # Only runtime discovery/health is faked. Git admission, lease validation,
+ # journal refusal, recovery dispatch, and lease release execute the real code.
+ cat >>"${fixture_scripts}/minikube/t2-common.sh" <<'SH'
+t2_require_commands() { :; }
+t2_profile_scope() { :; }
+t2_profile_status() { T2_BOOTSTRAP_REQUIRED=false; }
+t2_context_check() { :; }
+t2_profile_context_identity_check() { :; }
+SH
+ cat >"${fake_bin}/node" <<'SH'
+#!/usr/bin/env bash
+set -euo pipefail
+if [[ "$1" == "$WRC_ADMISSION_GUARD" && "$#" -eq 3 ]]; then
+ exec "$WRC_ADMISSION_NODE" "$@"
+fi
+if [[ "$1" == "$WRC_ADMISSION_RUNNER" && "$#" -eq 2 && "$2" == --recover ]]; then
+ printf '%s\n' "$2" >>"$WRC_ADMISSION_ENTRY_LOG"
+ exit 0
+fi
+printf 'unexpected Node runtime dispatch\n' >>"$WRC_ADMISSION_ENTRY_LOG"
+exit 99
+SH
+ cat >"${fake_bin}/kubectl" <<'SH'
+#!/usr/bin/env bash
+printf 'unexpected cluster operation\n' >>"$WRC_ADMISSION_CLUSTER_LOG"
+exit 99
+SH
+ chmod +x "${fake_bin}/node" "${fake_bin}/kubectl"
+ : >"${entry_log}"; : >"${cluster_log}"
+
+ invoke_wrc() {
+ local surface="$1" lease_token="$2" effective_context="$3" acknowledgement="$4"
+ shift 4
+ local script="${fixture_entry}"
+ [[ "$surface" != wrapper ]] || script="${fixture_wrapper}"
+ PATH="${fake_bin}:${PATH}" MAKEFLAGS= \
+ T2_PROJECT_DIR="${MINIKUBE_TEST_PROJECT_DIR}" T2_SCRIPT_DIR="${ROOT}/scripts/minikube" \
+ T2_PROFILE="${PROFILE}" T2_CONTEXT="${effective_context}" T2_LOCK_ROOT="${LOCK_ROOT}" \
+ T2_LOCK_TOKEN="${lease_token}" T2_SKIP_LOCK=true T2_MUTATION_LOCK_WRAPPED=true \
+ T2_PROFILE_ROOT="${TMP_DIR}/wrc-profiles" T2_GATE_ID=wrc-egress-recovery \
+ MINIKUBE_PROFILE="${PROFILE}" KUBECONTEXT="${PROFILE}" \
+ CONTROL_API_REAL_PG_CONTEXT="${effective_context}" \
+ E2E_WRC_EGRESS_FAULT_INJECTION="${acknowledgement}" \
+ WRC_ADMISSION_GUARD="${ROOT}/scripts/minikube/../e2e/_lib/wrc-egress-lifecycle.cjs" \
+ WRC_ADMISSION_RUNNER="${fixture_scripts}/e2e/_lib/wrc-egress-gate.cjs" \
+ WRC_ADMISSION_NODE="${real_node}" WRC_ADMISSION_ENTRY_LOG="${entry_log}" \
+ WRC_ADMISSION_CLUSTER_LOG="${cluster_log}" bash "${script}" "$@"
+ }
+ expect_wrc_refusal() {
+ local description="$1" reason="$2"
+ shift 2
+ : >"${entry_log}"; : >"${cluster_log}"
+ status=0
+ invoke_wrc "$@" >"${output}" 2>&1 || status=$?
+ if [[ "$status" -eq 0 || -s "$entry_log" || -s "$cluster_log" ]] || ! grep -Fq "$reason" "$output"; then
+ printf 'FAIL: %s did not refuse before runtime dispatch (exit=%s)\n' "$description" "$status" >&2
+ cat "$output" >&2
+ exit 1
+ fi
+ }
+
+ expect_wrc_refusal 'direct gate without a lease' PROFILE_LOCK_REQUIRED entry '' "$PROFILE" 1
+ expect_wrc_refusal 'wrapped boolean with false inherited token' PROFILE_LOCK_REQUIRED entry wrong-token "$PROFILE" 1
+ expect_wrc_refusal 'direct recovery without a lease' PROFILE_LOCK_REQUIRED entry '' "$PROFILE" 0 --recover
+ expect_wrc_refusal 'direct entry with mismatched context' 'inherited lease target differs' entry "$TOKEN" wrong-context 1
+ expect_wrc_refusal 'valid lease without fault acknowledgement' 'explicit fault-injection acknowledgement' entry "$TOKEN" "$PROFILE" 0
+ expect_wrc_refusal 'recovery with arbitrary extra command' usage wrapper "$TOKEN" "$PROFILE" 0 --recover-wrc-egress -- bash -c true
+ expect_wrc_refusal 'wrapper recovery with mismatched context' PROFILE_LOCK_REQUIRED wrapper "$TOKEN" wrong-context 0 --recover-wrc-egress
+
+ # This deliberately impossible PID cannot belong to another local process.
+ if kill -0 2147483647 2>/dev/null; then
+ printf 'FAIL: dead-owner PID fixture unexpectedly exists\n' >&2
+ exit 1
+ fi
+ write_owner 2147483647
+ expect_wrc_refusal 'dead inherited owner' PROFILE_LOCK_REQUIRED entry "$TOKEN" "$PROFILE" 0 --recover
+ expect_wrc_refusal 'dedicated recovery with dead inherited owner' PROFILE_LOCK_REQUIRED wrapper "$TOKEN" "$PROFILE" 0 --recover-wrc-egress
+ write_owner
+
+ journal="$(node - "${ROOT}/scripts/e2e/_lib/wrc-egress-lifecycle.cjs" "${MINIKUBE_TEST_PROJECT_DIR}" "$PROFILE" <<'NODE'
+const { Journal, journalPath } = require(process.argv[2])
+const { randomUUID } = require('node:crypto')
+const repository = process.argv[3], profile = process.argv[4]
+const file = journalPath(repository, profile)
+new Journal(file, { version: 1, runId: randomUUID(), phase: 'DNS-injected',
+ binding: { repository, profile, context: profile, branch: process.env.MINIKUBE_TEST_BRANCH,
+ head: process.env.MINIKUBE_TEST_HEAD }, resources: [] })
+process.stdout.write(file)
+NODE
+ )"
+ owner_before="$(shasum -a 256 "${LOCK_DIR}/owner.env")"
+ expect_wrc_refusal 'ordinary mutation over pending journal' WRC_EGRESS_RECOVERY_REQUIRED wrapper "$TOKEN" "$PROFILE" 1 -- bash "$fixture_entry"
+ expect_wrc_refusal 'recovery path with false inheritance' PROFILE_LOCK_REQUIRED wrapper wrong-token "$PROFILE" 0 --recover-wrc-egress
+ for surface in entry wrapper; do
+ : >"${entry_log}"; : >"${cluster_log}"
+ status=0
+ if [[ "$surface" == entry ]]; then
+ invoke_wrc entry "$TOKEN" "$PROFILE" 0 --recover >"${output}" 2>&1 || status=$?
+ else
+ invoke_wrc wrapper "$TOKEN" "$PROFILE" 0 --recover-wrc-egress >"${output}" 2>&1 || status=$?
+ fi
+ if [[ "$status" -ne 0 || "$(cat "$entry_log")" != --recover || -s "$cluster_log" \
+ || ! -f "$journal" || "$(shasum -a 256 "${LOCK_DIR}/owner.env")" != "$owner_before" ]]; then
+ printf 'FAIL: dedicated %s recovery did not preserve the inherited lease and dispatch only recovery\n' "$surface" >&2
+ cat "$output" >&2
+ exit 1
+ fi
+ done
+ printf 'PASS: WRC direct entry, pending journal and dedicated recovery enforce the real inherited lease\n'
+}
+
run_child
[[ "$(wc -l <"${LOG}" | tr -d ' ')" -eq 1 ]] || {
echo 'FAIL: valid inherited lease did not reach the child boundary' >&2
@@ -157,6 +295,8 @@ fi
exit 1
}
+assert_wrc_admission
+
full_plan="$(dry_run_make minikube-build-images 2>&1)"
full_body_plan="$(dry_run_make minikube-build-images-body 2>&1)"
targeted_plan="$(dry_run_make minikube-deploy-service SVC=control-api NS=control-plane 2>&1)"
@@ -166,6 +306,8 @@ restart_body_plan="$(dry_run_make minikube-restart-deploy-body SVC=control-api N
e2e_fixture_plan="$(dry_run_make minikube-build-e2e-fixtures 2>&1)"
e2e_fixture_body_plan="$(dry_run_make minikube-build-e2e-fixtures-body 2>&1)"
verify_plan="$(dry_run_make minikube-verify-images 2>&1)"
+wrc_plan="$(dry_run_make test-e2e-wrc-egress-degradation E2E_KUBECONTEXT="$PROFILE" E2E_EXPECTED_PRE_GATE_GATE=unit-gate 2>&1)"
+wrc_recovery_plan="$(dry_run_make test-e2e-wrc-egress-recover E2E_KUBECONTEXT="$PROFILE" 2>&1)"
[[ "$full_plan" == *"with-t2-mutation-lock.sh"* \
&& "$full_body_plan" == *"require-t2-mutation-lock.sh"* ]] || {
@@ -197,6 +339,15 @@ verify_plan="$(dry_run_make minikube-verify-images 2>&1)"
echo 'FAIL: read-only image verification was incorrectly placed behind a mutation lease' >&2
exit 1
}
+[[ "$wrc_plan" == *"with-t2-mutation-lock.sh -- bash scripts/e2e/e2e-wrc-egress-degradation.sh"* \
+ && "$wrc_plan" == *"T2_SKIP_LOCK="* && "$wrc_plan" == *"T2_LOCK_TOKEN="* \
+ && "$wrc_plan" == *"T2_PROFILE=\"${PROFILE}\""* && "$wrc_plan" == *"T2_CONTEXT=\"${PROFILE}\""* \
+ && "$wrc_recovery_plan" == *"with-t2-mutation-lock.sh --recover-wrc-egress"* \
+ && "$wrc_recovery_plan" != *"--recover-wrc-egress --"* \
+ && "$wrc_recovery_plan" == *"T2_SKIP_LOCK="* && "$wrc_recovery_plan" == *"T2_LOCK_TOKEN="* ]] || {
+ echo 'FAIL: WRC gate/recovery Make targets do not preserve the canonical lease and dedicated recovery dispatch' >&2
+ exit 1
+}
printf 'PASS: child mutation boundary requires the exact live profile lease and target context\n'
printf 'PASS: image builds and targeted deployment mutations require the lease while verify-only stays read-only\n'
diff --git a/scripts/tests/test-wrc-egress-degradation-gate.sh b/scripts/tests/test-wrc-egress-degradation-gate.sh
index d2192efa7..74794f631 100755
--- a/scripts/tests/test-wrc-egress-degradation-gate.sh
+++ b/scripts/tests/test-wrc-egress-degradation-gate.sh
@@ -1,76 +1,20 @@
#!/usr/bin/env bash
-# Literal source-contract patterns intentionally contain shell-looking text.
-# shellcheck disable=SC2016
+# Cluster-free unit/integration evidence for the WRC gate. Runtime dataplane,
+# business continuity, browser journeys, and T2 remain separate evidence lanes.
set -euo pipefail
-ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
-GATE="${ROOT}/scripts/e2e/e2e-wrc-egress-degradation.sh"
-FIXTURE="${ROOT}/tests/e2e/fixtures/wrc-egress-dns-proxy/server.cjs"
-MAKEFILE="${ROOT}/Makefile"
+ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd -P)"
-fail() { echo "FAIL: $*" >&2; exit 1; }
-contains() {
- local file=$1 pattern=$2 description=$3
- grep -Fq -- "$pattern" "$file" || fail "$description"
-}
+node -e 'if (Number(process.versions.node.split(".")[0]) !== 24) { process.stderr.write("WRC gate unit tests require Node 24\n"); process.exit(1) }'
+bash -n "${ROOT}/scripts/e2e/e2e-wrc-egress-degradation.sh"
-bash -n "$GATE"
-node --check "$FIXTURE" >/dev/null
+# These tests import the real pure oracle, DNS server, lifecycle, and runner.
+# DNS uses ephemeral loopback sockets; Kubernetes is faked at its boundary.
+# Executable entrypoint/lease refusal cases live in test-minikube-mutation-boundary.sh.
+node --test --test-concurrency=1 \
+ "${ROOT}/scripts/tests/wrc-egress-proof.test.cjs" \
+ "${ROOT}/scripts/tests/wrc-egress-dns-proxy.test.cjs" \
+ "${ROOT}/scripts/tests/wrc-egress-lifecycle.test.cjs" \
+ "${ROOT}/scripts/tests/wrc-egress-gate.test.cjs"
-contains "$GATE" 'is_branch_scoped_e2e_context "$E2E_KUBECONTEXT"' \
- 'gate must refuse non-branch Kubernetes contexts'
-contains "$GATE" 'E2E_WRC_EGRESS_FAULT_INJECTION' \
- 'gate must require explicit fault-injection acknowledgement'
-contains "$GATE" 'require_branch_owned_hcc_gate "$WRC_NS"' \
- 'gate must verify exact HEAD, worktree, profile, fingerprint, and pre-gate marker'
-contains "$GATE" 'acquire_lock' 'gate must acquire exclusive WRC fault-injection ownership'
-contains "$GATE" 'wrc_restore_verified' 'gate must prove WRC DNS configuration restoration'
-contains "$GATE" 'lock retained' 'failed restoration must retain the coordination lock'
-
-contains "$GATE" 'id: frontend' 'fixture must exercise the sandbox UI workload'
-contains "$GATE" 'id: worker' 'fixture must exercise a non-UI workload'
-contains "$GATE" 'port: 8443' 'fixture must begin with a permission that will be removed'
-contains "$GATE" 'set_dns_mode hold' 'gate must create a real DNS latency window'
-contains "$GATE" 'pre-DNS UI contraction' 'gate must assert UI contraction before DNS completes'
-contains "$GATE" 'pre-DNS workload contraction' \
- 'gate must assert workload contraction before DNS completes'
-contains "$GATE" 'append_policy_port "$UI_NS" "$UI_POLICY" 9443' \
- 'gate must inject live UI drift during DNS latency'
-contains "$GATE" 'append_policy_port "$RECIPE_NAMESPACE" "$WORKLOAD_POLICY" 9443' \
- 'gate must inject live workload drift during DNS latency'
-contains "$GATE" 'set_dns_mode servfail' 'gate must produce a real retryable resolver response'
-contains "$GATE" 'post-DNS UI re-contraction' \
- 'gate must assert the UI fresh-read contraction after DNS failure'
-contains "$GATE" 'post-DNS workload re-contraction' \
- 'gate must assert the workload fresh-read contraction after DNS failure'
-
-contains "$GATE" 'workload_health "$UI_NS" frontend' \
- 'gate must prove the UI workload still serves its health endpoint'
-contains "$GATE" 'workload_health "$RECIPE_NAMESPACE" worker' \
- 'gate must prove the workload still serves its health endpoint'
-contains "$GATE" 'deployment_ready "$WRC_NS" control-api' \
- 'gate must assert an unrelated production service remains Ready'
-contains "$GATE" 'deployment_ready "$WRC_NS" host-context-controller' \
- 'gate must assert HCC remains Ready'
-contains "$GATE" 'set_dns_mode ok' 'gate must restore successful DNS'
-contains "$GATE" 'WRC_EGRESS_DEGRADATION_E2E_PASS' \
- 'gate must emit an unambiguous terminal success marker'
-
-contains "$FIXTURE" 'match(/^\/mode\/(ok|hold|servfail)$/)' \
- 'DNS fixture must expose exactly the required deterministic modes'
-contains "$FIXTURE" '0x8180 | rcode' 'DNS fixture must emit an actual SERVFAIL response'
-contains "$FIXTURE" 'forwardUdp' 'DNS fixture must forward unrelated UDP queries'
-contains "$FIXTURE" 'forwardTcp' 'DNS fixture must forward unrelated TCP queries'
-
-contains "$MAKEFILE" '.PHONY: test-e2e-wrc-egress-degradation' \
- 'Makefile must expose the WRC egress degradation gate'
-contains "$MAKEFILE" 'MINIKUBE_PROFILE=$(E2E_KUBECONTEXT) KUBECONTEXT=$(E2E_KUBECONTEXT) bash scripts/e2e/e2e-wrc-egress-degradation.sh' \
- 'Make target must bind the mutation to the explicit profile/context'
-if grep -Eq 'kubectl[[:space:]]+(get|apply|patch|delete|exec|logs)' "$GATE"; then
- fail 'raw kubectl operations must use kctl or an explicit --context cleanup probe'
-fi
-if grep -Fq -- '--force' "$GATE"; then
- fail 'gate must not force Kubernetes mutations'
-fi
-
-echo 'PASS: WRC egress degradation E2E guard contract'
+printf 'WRC_EGRESS_UNIT_CONTRACT_PASS\n'
diff --git a/scripts/tests/wrc-egress-dns-proxy.test.cjs b/scripts/tests/wrc-egress-dns-proxy.test.cjs
new file mode 100644
index 000000000..b89bb7664
--- /dev/null
+++ b/scripts/tests/wrc-egress-dns-proxy.test.cjs
@@ -0,0 +1,412 @@
+'use strict'
+
+const { test } = require('node:test')
+const assert = require('node:assert/strict')
+const { once } = require('node:events')
+const dgram = require('node:dgram')
+const { Resolver } = require('node:dns/promises')
+const net = require('node:net')
+const {
+ createDnsProxy,
+ readConfig,
+ question,
+ dnsResponse,
+ tcpFrame,
+} = require('../../tests/e2e/fixtures/wrc-egress-dns-proxy/server.cjs')
+
+const targets = [
+ { lane: 'ui', fqdn: 'ui.example.com', ip: '93.184.216.34' },
+ { lane: 'worker', fqdn: 'worker.example.com', ip: '93.184.216.35' },
+]
+function query(fqdn, id = 7, type = 1) {
+ const header = Buffer.alloc(12)
+ header.writeUInt16BE(id, 0)
+ header.writeUInt16BE(0x0100, 2)
+ header.writeUInt16BE(1, 4)
+ const name = fqdn.split('.').flatMap(label => [Buffer.from([label.length]), Buffer.from(label)])
+ const tail = Buffer.from([0, 0, 0, 0, 1])
+ tail.writeUInt16BE(type, 1)
+ return Buffer.concat([header, ...name, tail])
+}
+
+async function udpClient(t) {
+ const socket = dgram.createSocket('udp4')
+ socket.bind(0, '127.0.0.1')
+ await once(socket, 'listening')
+ t.after(() => new Promise(resolve => socket.close(resolve)))
+ return socket
+}
+async function exchangeUdp(socket, port, message) {
+ const response = once(socket, 'message', { signal: AbortSignal.timeout(2000) })
+ socket.send(message, port, '127.0.0.1')
+ return (await response)[0]
+}
+async function exchangeTcp(port, message) {
+ const socket = net.createConnection({ host: '127.0.0.1', port })
+ await once(socket, 'connect', { signal: AbortSignal.timeout(2000) })
+ try {
+ const response = new Promise((resolve, reject) => {
+ let pending = Buffer.alloc(0)
+ socket.setTimeout(2000, () => reject(new Error('test TCP DNS deadline')))
+ socket.once('error', reject)
+ socket.on('data', chunk => {
+ pending = Buffer.concat([pending, chunk])
+ if (pending.length >= 2 && pending.length >= pending.readUInt16BE(0) + 2) {
+ resolve(pending.subarray(2, pending.readUInt16BE(0) + 2))
+ }
+ })
+ })
+ const framed = tcpFrame(message)
+ socket.write(framed.subarray(0, 1))
+ socket.write(framed.subarray(1))
+ return await response
+ } finally {
+ socket.destroy()
+ }
+}
+
+async function fixture(t, configuredTargets = targets) {
+ const upstream = await udpClient(t)
+ const upstreamPort = upstream.address().port
+ const respond = message => dnsResponse(message, question(message), 3, '203.0.113.7', 20)
+ upstream.on('message', (message, rinfo) =>
+ upstream.send(respond(message), rinfo.port, rinfo.address)
+ )
+ const tcpUpstream = net.createServer(socket => {
+ let pending = Buffer.alloc(0)
+ socket.on('data', chunk => {
+ pending = Buffer.concat([pending, chunk])
+ if (pending.length >= 2 && pending.length >= pending.readUInt16BE(0) + 2) {
+ socket.end(tcpFrame(respond(pending.subarray(2, pending.readUInt16BE(0) + 2))))
+ }
+ })
+ })
+ tcpUpstream.listen(upstreamPort, '127.0.0.1')
+ await once(tcpUpstream, 'listening')
+ t.after(() => new Promise(resolve => tcpUpstream.close(resolve)))
+ const proxy = createDnsProxy({
+ targets: configuredTargets,
+ upstream: '127.0.0.1',
+ upstreamPort,
+ listenHost: '127.0.0.1',
+ dnsPort: 0,
+ controlPort: 0,
+ ttlSeconds: 10,
+ })
+ t.after(() => proxy.close())
+ const ports = await proxy.start()
+ const control = async (path, method = 'GET') => {
+ const response = await fetch(`http://127.0.0.1:${ports.controlPort}${path}`, {
+ method,
+ signal: AbortSignal.timeout(2000),
+ headers: { connection: 'close' },
+ })
+ return {
+ status: response.status,
+ body: response.status === 200 ? await response.json() : undefined,
+ }
+ }
+ return { proxy, ports, control }
+}
+
+test('target configuration is explicit, bounded, unambiguous, and safe for arbitrary names', () => {
+ const valid = { targets, upstream: '127.0.0.1' }
+ assert.throws(() => readConfig({ TARGET_DNS: 'legacy.example.com', UPSTREAM_DNS: '127.0.0.1' }))
+ assert.deepEqual(
+ readConfig({ DNS_TARGETS_JSON: JSON.stringify(targets), UPSTREAM_DNS: '127.0.0.1' }),
+ { ...valid, ttlSeconds: 300 }
+ )
+ for (const options of [
+ { ...valid, targets: [] },
+ { ...valid, targets: [targets[0], targets[0]] },
+ { ...valid, targets: [targets[0], { ...targets[1], fqdn: targets[0].fqdn }] },
+ { ...valid, targets: [{ ...targets[0], lane: '__proto__' }] },
+ { ...valid, targets: [{ ...targets[0], ip: '999.1.2.3' }] },
+ { ...valid, targets: [{ ...targets[0], fqdn: 'UPPER.example.com' }] },
+ { ...valid, targets: [{ ...targets[0], fqdn: 'not-a-host' }] },
+ { ...valid, targets: [{ ...targets[0], unknown: true }] },
+ { ...valid, upstream: 'upstream.example.com' },
+ { ...valid, ttlSeconds: 0 },
+ { ...valid, ttlSeconds: 301 },
+ { ...valid, dnsPort: -1 },
+ { ...valid, upstreamPort: 0 },
+ ])
+ assert.throws(() => createDnsProxy(options))
+ assert.equal({}.polluted, undefined)
+})
+
+test('parses exact DNS questions and rejects malformed or compressed question interpretation', () => {
+ const wire = query('UI.example.com', 42)
+ assert.deepEqual(question(wire), {
+ name: 'ui.example.com',
+ type: 1,
+ queryClass: 1,
+ id: 42,
+ end: wire.length,
+ })
+ for (let length = 0; length < wire.length; length++)
+ assert.equal(question(wire.subarray(0, length)), null)
+ const compressed = Buffer.from(wire)
+ compressed[12] = 0xc0
+ assert.equal(question(compressed), null)
+ const multiple = Buffer.from(wire)
+ multiple.writeUInt16BE(2, 4)
+ assert.equal(question(multiple), null)
+})
+
+test(
+ 'UDP and TCP serve each lane independently with real A records and configured TTL',
+ { timeout: 5000 },
+ async t => {
+ const { proxy, ports, control } = await fixture(t)
+ const socket = await udpClient(t)
+ for (const [index, target] of targets.entries()) {
+ const message = query(target.fqdn, index + 20)
+ const udp = await exchangeUdp(socket, ports.dnsPort, message)
+ assert.deepEqual(udp, dnsResponse(message, question(message), 0, target.ip, 10))
+ const tcp = await exchangeTcp(ports.dnsPort, message)
+ assert.deepEqual(tcp, udp)
+ }
+ const state = (await control('/state')).body
+ assert.equal(state.fault, null)
+ assert.equal(state.counts.ok, 4)
+ assert.deepEqual(
+ state.lanes.map(lane => lane.received.ok),
+ [2, 2]
+ )
+ assert.deepEqual(
+ state.lanes.map(lane => lane.counts.ok),
+ [2, 2]
+ )
+ assert.deepEqual(state.lanes[0].questions.map(value => value.transport).sort(), ['tcp', 'udp'])
+ assert.equal(proxy.snapshot().lanes[1].unique.ok, 2)
+ }
+)
+
+test(
+ 'duplicate UI retries cannot prove a worker observation; releasing hold emits SERVFAIL',
+ { timeout: 5000 },
+ async t => {
+ const { proxy, ports, control } = await fixture(t)
+ const socket = await udpClient(t)
+ assert.equal((await control('/mode/ui/hold', 'POST')).status, 200)
+ const message = query(targets[0].fqdn, 89)
+ socket.send(message, ports.dnsPort, '127.0.0.1')
+ socket.send(message, ports.dnsPort, '127.0.0.1')
+ const deadline = Date.now() + 1500
+ let state
+ do {
+ state = (await control('/state')).body
+ } while (state.held !== 2 && Date.now() < deadline)
+ assert.equal(state.held, 2)
+ assert.equal(state.lanes[0].counts.hold, 2)
+ assert.equal(state.lanes[0].unique.hold, 1)
+ assert.equal(state.lanes[0].received.hold, 1)
+ assert.equal(state.lanes[1].counts.hold, 0)
+ const workerQuery = query(targets[1].fqdn, 90)
+ assert.equal((await exchangeUdp(socket, ports.dnsPort, workerQuery)).readUInt16BE(2) & 15, 0)
+ const received = once(socket, 'message', { signal: AbortSignal.timeout(2000) })
+ await control('/mode/ui/servfail', 'POST')
+ assert.deepEqual(
+ (await received)[0],
+ dnsResponse(message, question(message), 2, targets[0].ip, 10)
+ )
+ state = proxy.snapshot()
+ assert.equal(state.held, 0)
+ assert.equal(state.lanes[0].counts.servfail, 2)
+ assert.equal(state.lanes[0].unique.servfail, 1)
+ assert.equal(state.lanes[0].received.servfail, 0)
+ assert.equal(state.lanes[1].counts.servfail, 0)
+ assert.equal(state.fault, null)
+ }
+)
+
+test(
+ 'global modes preserve per-lane counters and recover without changing targets',
+ { timeout: 5000 },
+ async t => {
+ const { proxy, ports, control } = await fixture(t)
+ const socket = await udpClient(t)
+ await control('/mode/servfail', 'POST')
+ for (const [index, target] of targets.entries()) {
+ assert.equal(
+ (await exchangeUdp(socket, ports.dnsPort, query(target.fqdn, index + 40))).readUInt16BE(2) &
+ 15,
+ 2
+ )
+ }
+ await control('/mode/ok', 'POST')
+ for (const [index, target] of targets.entries()) {
+ assert.equal(
+ (await exchangeUdp(socket, ports.dnsPort, query(target.fqdn, index + 40))).readUInt16BE(2) &
+ 15,
+ 0
+ )
+ }
+ const state = proxy.snapshot()
+ assert.deepEqual(
+ state.lanes.map(value => ({
+ fqdn: value.fqdn,
+ ok: value.unique.ok,
+ failed: value.unique.servfail,
+ })),
+ targets.map(value => ({ fqdn: value.fqdn, ok: 1, failed: 1 }))
+ )
+ assert.equal((await control('/mode/__proto__/hold', 'POST')).status, 404)
+ assert.equal((await control('/mode/worker/unknown', 'POST')).status, 404)
+ assert.equal((await control('/mode/hold')).status, 404)
+ assert.throws(() => proxy.setMode('constructor'))
+ }
+)
+
+for (const releaseMode of ['ok', 'servfail']) {
+ test(
+ `received DNS requests exclude an expired hold flushed as ${releaseMode}`,
+ { timeout: 5000 },
+ async t => {
+ const { proxy, ports, control } = await fixture(t)
+ const resolver = new Resolver({ timeout: 25, tries: 1 })
+ resolver.setServers([`127.0.0.1:${ports.dnsPort}`])
+ await control('/mode/ui/hold', 'POST')
+ await assert.rejects(resolver.resolve4(targets[0].fqdn), { code: 'ETIMEOUT' })
+ const before = proxy.snapshot().lanes.find(lane => lane.lane === 'ui')
+ assert.equal(before.held, 1, 'the DNS client has finished but its datagram remains queued')
+ assert.deepEqual(before.received, { ok: 0, hold: 1, servfail: 0 })
+ await control(`/mode/ui/${releaseMode}`, 'POST')
+ const released = proxy.snapshot().lanes.find(lane => lane.lane === 'ui')
+ assert.equal(
+ released.counts[releaseMode],
+ 1,
+ 'legacy response counters still include the queued response'
+ )
+ assert.deepEqual(released.received, before.received, 'flushing does not receive a new query')
+ if (releaseMode === 'servfail') {
+ await assert.rejects(resolver.resolve4(targets[0].fqdn), { code: 'ESERVFAIL' })
+ } else {
+ assert.deepEqual(await resolver.resolve4(targets[0].fqdn), [targets[0].ip])
+ }
+ const after = proxy.snapshot()
+ const observed = after.lanes.find(lane => lane.lane === 'ui')
+ assert.equal(
+ observed.received[releaseMode],
+ 1,
+ 'only a request received in the new mode counts'
+ )
+ assert.equal(observed.received.hold, 1)
+ assert.deepEqual(after.lanes.find(lane => lane.lane === 'worker').received, {
+ ok: 0,
+ hold: 0,
+ servfail: 0,
+ })
+ assert.equal(after.fault, null)
+ }
+ )
+}
+
+test(
+ 'forwards unrelated UDP/TCP and target AAAA byte-for-byte even while target A is held',
+ { timeout: 5000 },
+ async t => {
+ const { proxy, ports, control } = await fixture(t)
+ const socket = await udpClient(t)
+ await control('/mode/hold', 'POST')
+ for (const message of [query('unrelated.example.com', 77), query(targets[0].fqdn, 78, 28)]) {
+ const expected = dnsResponse(message, question(message), 3, '203.0.113.7', 20)
+ assert.deepEqual(await exchangeUdp(socket, ports.dnsPort, message), expected)
+ assert.deepEqual(await exchangeTcp(ports.dnsPort, message), expected)
+ }
+ const state = proxy.snapshot()
+ assert.equal(state.held, 0)
+ assert.equal(state.counts.forwarded, 4)
+ assert.equal(state.lanes[0].counts.forwarded, 2)
+ assert.equal(
+ state.lanes[0].questions.every(value => value.type === 28 && value.mode === 'forwarded'),
+ true
+ )
+ assert.equal(state.lanes[1].questions.length, 0)
+ assert.deepEqual(state.lanes[0].received, { ok: 0, hold: 0, servfail: 0 })
+ }
+)
+
+test(
+ 'scoped UI/worker outages keep a third canary lane resolvable throughout',
+ { timeout: 5000 },
+ async t => {
+ const canary = { lane: 'canary', fqdn: 'canary.example.com', ip: '93.184.216.36' }
+ const { proxy, ports, control } = await fixture(t, [...targets, canary])
+ const socket = await udpClient(t)
+ for (const mode of ['hold', 'servfail', 'ok']) {
+ assert.equal((await control(`/mode/ui/${mode}`, 'POST')).status, 200)
+ assert.equal((await control(`/mode/worker/${mode}`, 'POST')).status, 200)
+ const wire = query(canary.fqdn, mode === 'hold' ? 111 : mode === 'servfail' ? 112 : 113)
+ assert.deepEqual(
+ await exchangeUdp(socket, ports.dnsPort, wire),
+ dnsResponse(wire, question(wire), 0, canary.ip, 10)
+ )
+ const state = proxy.snapshot().lanes.find(lane => lane.lane === 'canary')
+ assert.equal(state.mode, 'ok')
+ assert.equal(state.counts.hold, 0)
+ assert.equal(state.counts.servfail, 0)
+ }
+ assert.equal(proxy.snapshot().lanes.find(lane => lane.lane === 'canary').unique.ok, 3)
+ assert.deepEqual(proxy.snapshot().lanes.find(lane => lane.lane === 'canary').received, {
+ ok: 3,
+ hold: 0,
+ servfail: 0,
+ })
+ assert.equal((await control('/mode/canary/servfail', 'POST')).status, 200)
+ assert.equal(
+ (await exchangeUdp(socket, ports.dnsPort, query(canary.fqdn, 114))).readUInt16BE(2) & 15,
+ 2
+ )
+ assert.equal(proxy.snapshot().fault, null)
+ }
+)
+
+test(
+ 'closing with held UDP/TCP queries releases all listeners and does not manufacture success',
+ { timeout: 5000 },
+ async t => {
+ const { proxy, ports, control } = await fixture(t)
+ const socket = await udpClient(t)
+ await control('/mode/hold', 'POST')
+ socket.send(query(targets[0].fqdn), ports.dnsPort, '127.0.0.1')
+ const tcp = net.createConnection({ host: '127.0.0.1', port: ports.dnsPort })
+ await once(tcp, 'connect')
+ tcp.on('error', error => assert.equal(error.code, 'ECONNRESET'))
+ const closed = new Promise(resolve => tcp.once('close', resolve))
+ tcp.write(tcpFrame(query(targets[1].fqdn)))
+ await proxy.close()
+ await closed
+ assert.equal(proxy.snapshot().held, 0)
+ assert.equal(proxy.snapshot().counts.ok, 0)
+ await assert.rejects(proxy.start(), /cannot be started twice/)
+ }
+)
+
+test(
+ 'held-query overflow is bounded and explicitly invalidates fixture evidence',
+ { timeout: 5000 },
+ async t => {
+ const { proxy, ports, control } = await fixture(t)
+ await control('/mode/ui/hold', 'POST')
+ const socket = net.createConnection({ host: '127.0.0.1', port: ports.dnsPort })
+ socket.on('error', error => assert.equal(error.code, 'ECONNRESET'))
+ await once(socket, 'connect')
+ t.after(() => socket.destroy())
+ socket.write(
+ Buffer.concat(Array.from({ length: 513 }, (_, id) => tcpFrame(query(targets[0].fqdn, id))))
+ )
+ const deadline = Date.now() + 1500
+ let state
+ do {
+ state = (await control('/state')).body
+ } while (state.fault === null && Date.now() < deadline)
+ assert.equal(state.fault, 'HELD_QUERY_LIMIT')
+ assert.equal(state.held, 512)
+ assert.equal(state.lanes[0].counts.hold, 513)
+ assert.equal(state.lanes[1].counts.hold, 0)
+ assert.equal(state.counts.ok, 0)
+ await proxy.close()
+ }
+)
diff --git a/scripts/tests/wrc-egress-gate.test.cjs b/scripts/tests/wrc-egress-gate.test.cjs
new file mode 100644
index 000000000..fcec1c59f
--- /dev/null
+++ b/scripts/tests/wrc-egress-gate.test.cjs
@@ -0,0 +1,867 @@
+'use strict'
+
+// Integration contract for the gate's exported lifecycle. Kubernetes is a
+// boundary fake; journals are real, private files in a test-owned directory.
+const { test } = require('node:test')
+const assert = require('node:assert/strict')
+const fs = require('node:fs')
+const os = require('node:os')
+const path = require('node:path')
+const { randomUUID } = require('node:crypto')
+const {
+ restore,
+ protectedRun,
+ fixtureObjects,
+ deploymentReady,
+ policyObservation,
+ acceptPolicyIntentChange,
+} = require('../e2e/_lib/wrc-egress-gate.cjs')
+const {
+ Journal,
+ journalPath,
+ readJournal,
+ RUN_LABEL,
+} = require('../e2e/_lib/wrc-egress-lifecycle.cjs')
+const { readConfig } = require('../../tests/e2e/fixtures/wrc-egress-dns-proxy/server.cjs')
+
+const source = fs.realpathSync(path.resolve(__dirname, '../..'))
+const key = identity => `${identity.kind}/${identity.namespace}/${identity.name}`
+const clone = value => (value === undefined ? undefined : structuredClone(value))
+const apiError = code => Object.assign(new Error(`fixture API ${code}`), { code })
+
+function fixture(t) {
+ const directory = fs.mkdtempSync(path.join(fs.realpathSync(os.tmpdir()), 'wrc-gate-test-'))
+ t.after(() => fs.rmSync(directory, { recursive: true, force: true }))
+ const runId = randomUUID()
+ const binding = {
+ repository: directory,
+ branch: 'fix/unit-gate',
+ profile: 'unit-profile',
+ context: 'unit-profile',
+ head: 'unit-head',
+ }
+ const injectedConfig = { nameservers: ['10.96.0.20'], options: [{ name: 'timeout', value: '2' }] }
+ const wrcIdentity = { kind: 'Deployment', namespace: 'control-plane', name: 'workflow-recipes' }
+ const resources = [
+ {
+ kind: 'ConfigMap',
+ namespace: 'control-plane',
+ name: 'unit-script',
+ attempted: true,
+ uid: 'script-uid',
+ },
+ {
+ kind: 'Service',
+ namespace: 'control-plane',
+ name: 'unit-dns',
+ attempted: true,
+ uid: 'dns-uid',
+ },
+ {
+ kind: 'Deployment',
+ namespace: 'control-plane',
+ name: 'unit-proxy',
+ attempted: true,
+ uid: 'proxy-uid',
+ },
+ {
+ kind: 'WorkflowRecipe',
+ namespace: 'sandbox-recipes',
+ name: 'unit-recipe',
+ attempted: true,
+ uid: 'recipe-uid',
+ },
+ ]
+ const state = {
+ version: 1,
+ runId,
+ binding,
+ phase: 'exercising',
+ resources,
+ uiCleanup: null,
+ wrc: {
+ identity: wrcIdentity,
+ uid: 'wrc-uid',
+ replicas: 1,
+ originalPolicy: 'ClusterFirst',
+ injectedConfig,
+ attempted: true,
+ },
+ }
+ const file = journalPath(directory, binding.profile)
+ const journal = new Journal(file, state)
+ const objects = new Map(
+ resources.map(entry => [
+ key(entry),
+ {
+ apiVersion: 'v1',
+ kind: entry.kind,
+ metadata: {
+ name: entry.name,
+ namespace: entry.namespace,
+ uid: entry.uid,
+ resourceVersion: '1',
+ labels: { [RUN_LABEL]: runId },
+ },
+ },
+ ])
+ )
+ const wrc = {
+ apiVersion: 'apps/v1',
+ kind: 'Deployment',
+ metadata: {
+ name: wrcIdentity.name,
+ namespace: wrcIdentity.namespace,
+ uid: state.wrc.uid,
+ resourceVersion: '7',
+ generation: 2,
+ },
+ spec: {
+ replicas: 1,
+ template: { spec: { dnsPolicy: 'None', dnsConfig: clone(injectedConfig) } },
+ },
+ status: { observedGeneration: 2, readyReplicas: 1, availableReplicas: 1 },
+ }
+ objects.set(key(wrcIdentity), wrc)
+ const events = []
+ const hooks = {}
+ const kube = {
+ setCleanup(enabled) {
+ events.push({ action: 'cleanup', enabled })
+ },
+ async get(identity) {
+ events.push({ action: 'get', identity: clone(identity) })
+ if (hooks.get) {
+ const result = await hooks.get(identity)
+ if (result !== undefined) return clone(result)
+ }
+ return clone(objects.get(key(identity)) ?? null)
+ },
+ async list(kind, namespace, selector) {
+ events.push({ action: 'list', kind, namespace, selector })
+ if (hooks.list) {
+ const result = await hooks.list(kind, namespace, selector)
+ if (result !== undefined) return clone(result)
+ }
+ const singular = new Map([
+ ['deployments', 'Deployment'],
+ ['services', 'Service'],
+ ['networkpolicies', 'NetworkPolicy'],
+ ['pods', 'Pod'],
+ ['replicasets', 'ReplicaSet'],
+ ]).get(kind)
+ const labels = selector.split(',').map(pair => pair.split('='))
+ return [...objects.values()]
+ .filter(
+ object =>
+ object.kind === singular &&
+ object.metadata.namespace === namespace &&
+ labels.every(([name, value]) => object.metadata.labels?.[name] === value)
+ )
+ .map(clone)
+ },
+ async patch(identity, patch) {
+ events.push({ action: 'patch', identity: clone(identity), patch: clone(patch) })
+ if (hooks.patch) await hooks.patch(identity, patch)
+ const live = objects.get(key(identity))
+ for (const operation of patch) {
+ if (operation.op === 'test' && operation.path === '/metadata/uid')
+ assert.equal(operation.value, live.metadata.uid)
+ else if (operation.op === 'test' && operation.path === '/metadata/resourceVersion')
+ assert.equal(operation.value, live.metadata.resourceVersion)
+ else if (operation.op === 'replace' && operation.path === '/spec/template/spec/dnsPolicy')
+ live.spec.template.spec.dnsPolicy = operation.value
+ else if (operation.op === 'remove' && operation.path === '/spec/template/spec/dnsConfig')
+ delete live.spec.template.spec.dnsConfig
+ else throw new Error('unexpected mutation in restoration contract')
+ }
+ live.metadata.resourceVersion = String(Number(live.metadata.resourceVersion) + 1)
+ live.metadata.generation++
+ return clone(live)
+ },
+ async rollout(namespace, name) {
+ events.push({ action: 'rollout', namespace, name })
+ const live = objects.get(key({ kind: 'Deployment', namespace, name }))
+ if (hooks.rollout) await hooks.rollout(live)
+ live.status = {
+ observedGeneration: live.metadata.generation,
+ readyReplicas: live.spec.replicas,
+ availableReplicas: live.spec.replicas,
+ }
+ },
+ async delete(identity, preconditions) {
+ events.push({
+ action: 'delete',
+ identity: clone(identity),
+ preconditions: clone(preconditions),
+ })
+ if (hooks.delete) await hooks.delete(identity, preconditions)
+ const live = objects.get(key(identity))
+ assert.equal(preconditions.uid, live.metadata.uid)
+ assert.equal(preconditions.resourceVersion, live.metadata.resourceVersion)
+ objects.delete(key(identity))
+ },
+ }
+ return { directory, file, journal, state, wrc, objects, kube, hooks, events, wrcIdentity }
+}
+
+const mutations = events =>
+ events.filter(event => event.action === 'patch' || event.action === 'delete')
+
+function uiFixture(t, { assigned = true, children = true } = {}) {
+ const fixtureState = fixture(t)
+ const { state, objects, journal } = fixtureState
+ const parent = state.resources.find(entry => entry.kind === 'WorkflowRecipe')
+ const parentObject = objects.get(key(parent))
+ parentObject.spec = {
+ ui: { workloadRef: 'frontend' },
+ workloads: [{ id: 'frontend', type: 'deployment', port: 3000 }],
+ }
+ const physicalName = 'wf-unit-recipe-frontend-a17c5e92'
+ if (assigned)
+ parentObject.status = { workloadInstances: { frontend: physicalName }, phase: 'candidate' }
+ state.uiCleanup = {
+ parent: { kind: parent.kind, name: parent.name, namespace: parent.namespace },
+ parentUid: parent.uid,
+ runId: state.runId,
+ namespace: 'sandbox-ui',
+ workload: 'frontend',
+ physicalName: null,
+ children: [],
+ }
+ const common = { 'clerum.io/managed-by': 'workflow-recipes', 'clerum.io/recipe': parent.name }
+ const uiLabels = {
+ 'clerum.io/sandbox-ui': 'true',
+ 'clerum.io/recipe-name': parent.name,
+ 'clerum.io/recipe-namespace': parent.namespace,
+ }
+ const uiObjects = [
+ {
+ kind: 'Deployment',
+ metadata: {
+ name: physicalName,
+ namespace: 'sandbox-ui',
+ uid: 'ui-deployment-uid',
+ resourceVersion: '10',
+ labels: { ...common, 'clerum.io/workload': 'frontend' },
+ },
+ },
+ {
+ kind: 'Service',
+ metadata: {
+ name: physicalName,
+ namespace: 'sandbox-ui',
+ uid: 'ui-service-uid',
+ resourceVersion: '11',
+ labels: { ...common, ...uiLabels, 'clerum.io/workload': 'frontend' },
+ },
+ },
+ {
+ kind: 'NetworkPolicy',
+ metadata: {
+ name: `ui-egress-${parent.name}`,
+ namespace: 'sandbox-ui',
+ uid: 'ui-policy-uid',
+ resourceVersion: '12',
+ labels: {
+ ...common,
+ 'clerum.io/recipe-name': parent.name,
+ 'clerum.io/recipe-namespace': parent.namespace,
+ },
+ },
+ },
+ ]
+ if (children)
+ for (const object of uiObjects)
+ objects.set(key({ kind: object.kind, ...object.metadata }), object)
+ journal.save()
+ return { ...fixtureState, parent, parentObject, physicalName, uiObjects }
+}
+
+test('foreground recipe deletion cannot leave UI children after its finalizer swallows DELETE503', async t => {
+ const { kube, journal, objects, hooks, file, events, parent, physicalName, uiObjects } =
+ uiFixture(t)
+ const rawCollision = {
+ kind: 'Deployment',
+ metadata: {
+ name: 'frontend',
+ namespace: 'sandbox-ui',
+ uid: 'unrelated-uid',
+ resourceVersion: '1',
+ labels: { 'clerum.io/recipe': 'unrelated' },
+ },
+ }
+ objects.set('Deployment/sandbox-ui/frontend', rawCollision)
+ let recordedBeforeParentDeletion = false
+ hooks.delete = identity => {
+ if (identity.kind === 'WorkflowRecipe') {
+ const saved = readJournal(file).uiCleanup
+ recordedBeforeParentDeletion =
+ saved.physicalName === physicalName &&
+ saved.parentUid === parent.uid &&
+ uiObjects.every(object => saved.children.some(child => child.uid === object.metadata.uid))
+ // Real WRC safeDelete absorbs the simulated UI DELETE503, removes its
+ // finalizer, and allows parent deletion; the cross-namespace objects stay.
+ }
+ }
+ await restore(kube, journal)
+ assert.equal(recordedBeforeParentDeletion, true)
+ assert.equal(fs.existsSync(file), false)
+ assert.equal(objects.get('Deployment/sandbox-ui/frontend'), rawCollision)
+ for (const object of uiObjects) {
+ assert.equal(objects.has(key({ kind: object.kind, ...object.metadata })), false)
+ const deletion = events.find(
+ event =>
+ event.action === 'delete' &&
+ event.identity.kind === object.kind &&
+ event.identity.namespace === 'sandbox-ui'
+ )
+ assert.deepEqual(deletion.preconditions, {
+ uid: object.metadata.uid,
+ resourceVersion: object.metadata.resourceVersion,
+ })
+ }
+ assert.equal(
+ events.some(
+ event => event.identity?.namespace === 'sandbox-ui' && event.identity.name === 'frontend'
+ ),
+ false
+ )
+})
+
+test('failed leftover delete preserves recorded UI identities for recovery after the parent is absent', async t => {
+ const { kube, journal, hooks, file, objects, parent, uiObjects } = uiFixture(t)
+ hooks.delete = identity => {
+ if (identity.namespace === 'sandbox-ui') throw apiError(503)
+ }
+ await assert.rejects(restore(kube, journal), /WRC_EGRESS_RECOVERY_REQUIRED/)
+ const saved = readJournal(file)
+ assert.equal(saved.phase, 'recovery-required')
+ assert.equal(objects.has(key(parent)), false)
+ assert(saved.uiCleanup.children.some(child => child.uid === 'ui-deployment-uid'))
+ hooks.delete = undefined
+ await restore(kube, new Journal(file, saved, true))
+ assert.equal(fs.existsSync(file), false)
+ for (const object of uiObjects)
+ assert.equal(objects.has(key({ kind: object.kind, ...object.metadata })), false)
+})
+
+test('a replaced UI UID is never adopted during recovery, even with matching name and labels', async t => {
+ const { kube, journal, hooks, file, events, uiObjects } = uiFixture(t)
+ hooks.delete = identity => {
+ if (identity.kind === 'WorkflowRecipe') uiObjects[0].metadata.uid = 'replacement-uid'
+ }
+ await assert.rejects(restore(kube, journal), /WRC_EGRESS_RECOVERY_REQUIRED/)
+ assert.equal(
+ readJournal(file).uiCleanup.children.find(child => child.kind === 'Deployment').uid,
+ 'ui-deployment-uid'
+ )
+ assert.equal(
+ events.some(event => event.action === 'delete' && event.identity.namespace === 'sandbox-ui'),
+ false
+ )
+ await assert.rejects(
+ restore(kube, new Journal(file, readJournal(file), true)),
+ /WRC_EGRESS_RECOVERY_REQUIRED/
+ )
+ assert.equal(
+ events.some(event => event.action === 'delete' && event.identity.namespace === 'sandbox-ui'),
+ false
+ )
+})
+
+test('a foreign UI owner is refused before deleting its parent or any UI resource', async t => {
+ const { kube, journal, uiObjects, events, file } = uiFixture(t)
+ uiObjects[0].metadata.labels['clerum.io/recipe'] = 'foreign-recipe'
+ await assert.rejects(restore(kube, journal), /WRC_EGRESS_RECOVERY_REQUIRED/)
+ assert.equal(readJournal(file).phase, 'recovery-required')
+ assert.equal(
+ events.some(event => event.action === 'delete'),
+ false
+ )
+})
+
+test('optional Deployment UI labels may be absent but cannot contradict the parent scope', async t => {
+ const { kube, journal, uiObjects, events, file } = uiFixture(t)
+ uiObjects[0].metadata.labels['clerum.io/recipe-namespace'] = 'foreign-namespace'
+ await assert.rejects(restore(kube, journal), /WRC_EGRESS_RECOVERY_REQUIRED/)
+ assert.equal(readJournal(file).phase, 'recovery-required')
+ assert.equal(
+ events.some(event => event.action === 'delete'),
+ false
+ )
+})
+
+test('UI discovery read503 retains recovery-required; authoritative absence before Ready is a valid no-op', async t => {
+ const failed = uiFixture(t)
+ failed.hooks.get = identity => {
+ if (identity.namespace === 'sandbox-ui') throw apiError(503)
+ }
+ await assert.rejects(restore(failed.kube, failed.journal), /WRC_EGRESS_RECOVERY_REQUIRED/)
+ assert.equal(readJournal(failed.file).phase, 'recovery-required')
+ assert.equal(
+ failed.events.some(event => event.action === 'delete'),
+ false
+ )
+ for (const assigned of [false, true]) {
+ const absent = uiFixture(t, { assigned, children: false })
+ await restore(absent.kube, absent.journal)
+ assert.equal(fs.existsSync(absent.file), false)
+ assert.equal(
+ absent.events.some(
+ event => event.action === 'delete' && event.identity.namespace === 'sandbox-ui'
+ ),
+ false
+ )
+ assert.equal(
+ absent.events.some(event => event.identity?.name === 'frontend'),
+ false
+ )
+ }
+})
+
+test('parent already absent without child UID capture cannot certify or adopt an orphan UI', async t => {
+ const { kube, journal, parent, objects, file, events } = uiFixture(t)
+ objects.delete(key(parent))
+ await assert.rejects(restore(kube, journal), /WRC_EGRESS_RECOVERY_REQUIRED/)
+ assert.equal(readJournal(file).phase, 'recovery-required')
+ assert.equal(
+ events.some(event => event.action === 'delete' && event.identity.namespace === 'sandbox-ui'),
+ false
+ )
+ const empty = uiFixture(t, { assigned: false, children: false })
+ empty.objects.delete(key(empty.parent))
+ await restore(empty.kube, empty.journal)
+ assert.equal(fs.existsSync(empty.file), false)
+})
+
+test('invalid physical names from status are refused before they reach a Kubernetes argument', async t => {
+ const { kube, journal, parentObject, events, file } = uiFixture(t)
+ parentObject.status.workloadInstances.frontend = '--context=foreign'
+ await assert.rejects(restore(kube, journal), /WRC_EGRESS_RECOVERY_REQUIRED/)
+ assert.equal(
+ events.some(event => event.identity?.name === '--context=foreign'),
+ false
+ )
+ assert.equal(
+ events.some(event => event.action === 'delete'),
+ false
+ )
+ assert.equal(readJournal(file).phase, 'recovery-required')
+})
+
+test('UI census errors cannot clear the journal after acknowledged child deletion', async t => {
+ const { kube, journal, hooks, file } = uiFixture(t)
+ hooks.list = () => {
+ throw apiError(503)
+ }
+ await assert.rejects(restore(kube, journal), /WRC_EGRESS_RECOVERY_REQUIRED/)
+ assert.equal(readJournal(file).phase, 'recovery-required')
+})
+
+test('an old journal with recipe deletion but no UI cleanup declaration cannot certify absence', async t => {
+ const { kube, journal, state, file, events } = fixture(t)
+ delete state.uiCleanup
+ journal.save()
+ await assert.rejects(restore(kube, journal), /WRC_EGRESS_RECOVERY_REQUIRED/)
+ assert.equal(readJournal(file).phase, 'recovery-required')
+ assert.equal(
+ events.some(event => event.action === 'delete'),
+ false
+ )
+})
+
+test('a UI child first appearing after parent deletion has no adoptable recorded UID', async t => {
+ const { kube, journal, hooks, objects, uiObjects, file, events } = uiFixture(t, {
+ children: false,
+ })
+ hooks.delete = identity => {
+ if (identity.kind === 'WorkflowRecipe') {
+ objects.set(key({ kind: uiObjects[0].kind, ...uiObjects[0].metadata }), uiObjects[0])
+ }
+ }
+ await assert.rejects(restore(kube, journal), /WRC_EGRESS_RECOVERY_REQUIRED/)
+ assert.equal(
+ readJournal(file).uiCleanup.children.find(child => child.kind === 'Deployment').uid,
+ null
+ )
+ assert.equal(
+ events.some(event => event.action === 'delete' && event.identity.namespace === 'sandbox-ui'),
+ false
+ )
+})
+
+test('a foreign parent UID cannot authorize capturing or deleting its UI children', async t => {
+ const { kube, journal, parentObject, file, events } = uiFixture(t)
+ parentObject.metadata.uid = 'foreign-parent-uid'
+ await assert.rejects(restore(kube, journal), /WRC_EGRESS_RECOVERY_REQUIRED/)
+ assert.equal(readJournal(file).phase, 'recovery-required')
+ assert.equal(
+ events.some(event => event.action === 'delete'),
+ false
+ )
+})
+
+test('UI background ReplicaSets and Pods must disappear before the journal is removed', async t => {
+ const { kube, journal, hooks, objects, parent, physicalName, events, file } = uiFixture(t)
+ for (const kind of ['ReplicaSet', 'Pod']) {
+ const name = `${physicalName}-${kind.toLowerCase()}`
+ objects.set(`${kind}/sandbox-ui/${name}`, {
+ kind,
+ metadata: {
+ name,
+ namespace: 'sandbox-ui',
+ uid: `${kind}-uid`,
+ labels: { 'clerum.io/recipe': parent.name },
+ },
+ })
+ }
+ hooks.list = kind => {
+ const singular = kind === 'replicasets' ? 'ReplicaSet' : kind === 'pods' ? 'Pod' : null
+ if (!singular) return undefined
+ const lingering = [...objects.values()].filter(object => object.kind === singular)
+ for (const object of lingering)
+ objects.delete(`${object.kind}/${object.metadata.namespace}/${object.metadata.name}`)
+ return lingering
+ }
+ await restore(kube, journal)
+ assert.equal(fs.existsSync(file), false)
+ assert(events.filter(event => event.action === 'list' && event.kind === 'pods').length >= 2)
+ assert.equal(
+ events.some(
+ event => event.action === 'delete' && ['Pod', 'ReplicaSet'].includes(event.identity.kind)
+ ),
+ false
+ )
+})
+
+test('recovery requires a newer controller observation for every retained tuple, not old policy or DNS counters', () => {
+ const policy = entries => ({
+ metadata: {
+ annotations: {
+ 'clerum.io/egress-fqdn-state': JSON.stringify(entries),
+ },
+ },
+ })
+ const retained = [
+ { lastObservedAt: 100, expiresAt: 500 },
+ { lastObservedAt: 101, expiresAt: 510 },
+ ]
+ assert.deepEqual(policyObservation(policy(retained)), { lastAcceptedAt: 101, latestExpiry: 510 })
+ assert.equal(policyObservation(policy(retained), 100), null)
+ assert.equal(policyObservation(policy(retained), 101), null)
+ assert.deepEqual(policyObservation(policy([{ lastObservedAt: 102, expiresAt: 600 }]), 101), {
+ lastAcceptedAt: 102,
+ latestExpiry: 600,
+ })
+ assert.throws(() => policyObservation(policy([])), /MISSING_ACCEPTED/)
+ assert.throws(
+ () => policyObservation(policy([{ lastObservedAt: null, expiresAt: 600 }])),
+ /MISSING_ACCEPTED/
+ )
+})
+
+test('only confirmed same-recipe intent change releases policy UID pins for a fully reproven migration', () => {
+ const targets = [
+ { lane: 'ui', fqdn: 'ui.example.com' },
+ { lane: 'worker', fqdn: 'worker.example.com' },
+ ]
+ const before = { metadata: { uid: 'recipe-uid', generation: 7 } }
+ for (const after of [
+ { metadata: { uid: 'recipe-uid', generation: 7 } },
+ { metadata: { uid: 'foreign-uid', generation: 8 } },
+ ]) {
+ const lanes = [
+ { lane: 'ui', uid: 'ui-policy-uid' },
+ { lane: 'workload', uid: 'worker-policy-uid' },
+ ]
+ assert.throws(
+ () => acceptPolicyIntentChange(lanes, before, after, 'recipe-uid', targets),
+ /HOST_MIGRATION_NOT_APPLIED/
+ )
+ assert.deepEqual(
+ lanes.map(lane => lane.uid),
+ ['ui-policy-uid', 'worker-policy-uid']
+ )
+ }
+ const lanes = [
+ { lane: 'ui', uid: 'ui-policy-uid' },
+ { lane: 'workload', uid: 'worker-policy-uid' },
+ ]
+ acceptPolicyIntentChange(
+ lanes,
+ before,
+ { metadata: { uid: 'recipe-uid', generation: 8 } },
+ 'recipe-uid',
+ targets
+ )
+ assert.deepEqual(
+ lanes.map(lane => lane.uid),
+ [undefined, undefined]
+ )
+ assert.deepEqual(
+ lanes.map(lane => lane.target.fqdn),
+ ['worker.example.com', 'ui.example.com']
+ )
+})
+
+test('restores exact DNS fields with CAS and proves rollout before deleting owned resources', async t => {
+ const { kube, journal, file, wrc, events, state } = fixture(t)
+ await restore(kube, journal)
+ assert.equal(wrc.spec.template.spec.dnsPolicy, 'ClusterFirst')
+ assert.equal(wrc.spec.template.spec.dnsConfig, undefined)
+ assert.equal(deploymentReady(wrc), true)
+ assert.equal(fs.existsSync(file), false)
+ assert.equal(state.phase, 'restored')
+ assert.deepEqual(events[0], { action: 'cleanup', enabled: true })
+ const patch = events.find(event => event.action === 'patch').patch
+ assert.deepEqual(patch.slice(0, 2), [
+ { op: 'test', path: '/metadata/uid', value: 'wrc-uid' },
+ { op: 'test', path: '/metadata/resourceVersion', value: '7' },
+ ])
+ assert.deepEqual(
+ events.filter(event => event.action === 'delete').map(event => event.identity.name),
+ ['unit-recipe', 'unit-proxy', 'unit-dns', 'unit-script']
+ )
+ const firstDelete = events.findIndex(event => event.action === 'delete')
+ assert(events.slice(0, firstDelete).some(event => event.action === 'rollout'))
+})
+
+test('restoration recognizes already restored default DNS without rewriting it', async t => {
+ const { kube, journal, wrc, events, file } = fixture(t)
+ delete wrc.spec.template.spec.dnsPolicy
+ delete wrc.spec.template.spec.dnsConfig
+ await restore(kube, journal)
+ assert.equal(
+ events.some(event => event.action === 'patch'),
+ false
+ )
+ assert.equal(fs.existsSync(file), false)
+})
+
+for (const scenario of [
+ 'uid',
+ 'replicas',
+ 'dns',
+ 'absent',
+ 'read-error',
+ 'patch-conflict',
+ 'rollout-error',
+ 'readback-drift',
+]) {
+ test(`uncertain ${scenario} restoration retains every DNS resource and the recovery journal`, async t => {
+ const { kube, journal, wrc, events, hooks, file, wrcIdentity, objects } = fixture(t)
+ if (scenario === 'uid') wrc.metadata.uid = 'foreign-wrc'
+ if (scenario === 'replicas') wrc.spec.replicas = 2
+ if (scenario === 'dns') wrc.spec.template.spec.dnsConfig = { nameservers: ['10.96.0.99'] }
+ if (scenario === 'absent') objects.delete(key(wrcIdentity))
+ if (scenario === 'read-error')
+ hooks.get = () => {
+ throw apiError(503)
+ }
+ if (scenario === 'patch-conflict')
+ hooks.patch = () => {
+ throw apiError(409)
+ }
+ if (scenario === 'rollout-error')
+ hooks.rollout = () => {
+ throw apiError(503)
+ }
+ if (scenario === 'readback-drift')
+ hooks.rollout = live => {
+ live.spec.template.spec.dnsConfig = { nameservers: ['10.96.0.99'] }
+ }
+ await assert.rejects(restore(kube, journal), /WRC_EGRESS_RECOVERY_REQUIRED/)
+ assert.equal(readJournal(file).phase, 'recovery-required')
+ assert.equal(readJournal(file).wrc.attempted, true)
+ assert.equal(
+ events.some(event => event.action === 'delete'),
+ false
+ )
+ if (['uid', 'replicas', 'dns', 'absent', 'read-error'].includes(scenario))
+ assert.equal(mutations(events).length, 0)
+ })
+}
+
+test('replica changes during restoration readback cannot be certified as baseline restored', async t => {
+ const { kube, journal, hooks, file, events } = fixture(t)
+ hooks.rollout = live => {
+ live.spec.replicas = 2
+ }
+ await assert.rejects(restore(kube, journal), /WRC_EGRESS_RECOVERY_REQUIRED/)
+ assert.equal(readJournal(file).phase, 'recovery-required')
+ assert.equal(
+ events.some(event => event.action === 'delete'),
+ false
+ )
+})
+
+test('an unmutated failed preflight produces no resource write and removes only its own journal', async t => {
+ const { kube, journal, state, file, events } = fixture(t)
+ state.resources = []
+ delete state.wrc
+ journal.save()
+ const original = new Error('ownership/precondition refused')
+ await assert.rejects(
+ protectedRun(
+ kube,
+ journal,
+ async () => {
+ throw original
+ },
+ new AbortController()
+ ),
+ error => error === original
+ )
+ assert.equal(mutations(events).length, 0)
+ assert.equal(fs.existsSync(file), false)
+})
+
+test('partial creation before DNS injection deletes only acknowledged existing identities', async t => {
+ const { kube, journal, state, objects, file, events } = fixture(t)
+ state.wrc.attempted = false
+ state.resources = state.resources.slice(0, 2)
+ objects.delete(key(state.resources[1]))
+ state.resources[1].uid = null
+ journal.save()
+ await restore(kube, journal)
+ assert.deepEqual(
+ mutations(events).map(event => `${event.action}:${event.identity.name}`),
+ ['delete:unit-script']
+ )
+ assert.equal(fs.existsSync(file), false)
+})
+
+for (const conflict of ['uid', 'label', 'unacknowledged']) {
+ test(`cleanup refuses ${conflict} ownership while preserving a recovery obligation`, async t => {
+ const { kube, journal, state, objects, file, events } = fixture(t)
+ state.wrc.attempted = false
+ const foreign = state.resources[0]
+ const live = objects.get(key(foreign))
+ if (conflict === 'uid') live.metadata.uid = 'replacement-resource'
+ if (conflict === 'label') live.metadata.labels[RUN_LABEL] = randomUUID()
+ if (conflict === 'unacknowledged') foreign.uid = null
+ journal.save()
+ await assert.rejects(restore(kube, journal), /WRC_EGRESS_RECOVERY_REQUIRED/)
+ assert.equal(
+ events.some(event => event.action === 'delete' && event.identity.name === foreign.name),
+ false
+ )
+ assert.equal(objects.has(key(foreign)), true)
+ assert.equal(readJournal(file).phase, 'recovery-required')
+ })
+}
+
+test('an unavailable cleanup GET cannot be confused with confirmed resource absence', async t => {
+ const { kube, journal, hooks, state, file, events } = fixture(t)
+ state.wrc.attempted = false
+ hooks.get = identity => {
+ if (identity.kind === 'Service') throw apiError(503)
+ }
+ await assert.rejects(restore(kube, journal), /WRC_EGRESS_RECOVERY_REQUIRED/)
+ assert.equal(readJournal(file).phase, 'recovery-required')
+ assert.equal(
+ events.some(event => event.action === 'delete' && event.identity.kind === 'Service'),
+ false
+ )
+})
+
+test('failed exercise is preserved after successful restoration; cleanup failure includes both causes', async t => {
+ const first = fixture(t)
+ const original = new Error('business assertion failed')
+ await assert.rejects(
+ protectedRun(
+ first.kube,
+ first.journal,
+ async () => {
+ throw original
+ },
+ new AbortController()
+ ),
+ error => error === original
+ )
+ assert.equal(fs.existsSync(first.file), false)
+ const second = fixture(t)
+ second.hooks.rollout = () => {
+ throw apiError(503)
+ }
+ await assert.rejects(
+ protectedRun(
+ second.kube,
+ second.journal,
+ async () => {
+ throw original
+ },
+ new AbortController()
+ ),
+ error =>
+ error instanceof AggregateError && error.errors[0] === original && error.errors.length === 2
+ )
+ assert.equal(readJournal(second.file).phase, 'recovery-required')
+})
+
+for (const exitCode of [130, 143]) {
+ for (const timing of ['exercise', 'restoration']) {
+ test(`interruption ${exitCode} during ${timing} cannot become a green protected run`, async t => {
+ const { kube, journal, hooks, file, events } = fixture(t)
+ const cancellation = new AbortController()
+ const interruption = Object.assign(new Error('interrupted'), { exitCode })
+ if (timing === 'restoration') hooks.rollout = () => cancellation.abort(interruption)
+ await assert.rejects(
+ protectedRun(
+ kube,
+ journal,
+ async () => {
+ if (timing === 'exercise') cancellation.abort(interruption)
+ },
+ cancellation
+ ),
+ error => error === interruption
+ )
+ assert.equal(fs.existsSync(file), false)
+ assert.equal(events.filter(event => event.action === 'delete').length, 4)
+ })
+ }
+}
+
+test('fixture manifests preserve distinct DNS lanes, scoped policies and a valid independent exact-host canary', () => {
+ const config = {
+ repository: source,
+ wrcNamespace: 'control-plane',
+ wrcDeployment: 'workflow-recipes',
+ recipeNamespace: 'sandbox-recipes',
+ uiNamespace: 'sandbox-ui',
+ }
+ const runId = randomUUID()
+ const result = fixtureObjects(config, runId, 'wrc:test', '93.184.216.34', '10.96.0.10')
+ for (const object of [...result.objects, result.recipe, result.canary])
+ assert.equal(object.metadata.labels[RUN_LABEL], runId)
+ const proxy = result.objects.find(object => object.kind === 'Deployment')
+ const container = proxy.spec.template.spec.containers[0]
+ const env = Object.fromEntries(container.env.map(entry => [entry.name, entry.value]))
+ const dnsConfig = readConfig(env)
+ assert.deepEqual(dnsConfig.targets.map(target => target.lane).sort(), ['canary', 'ui', 'worker'])
+ assert.equal(new Set(dnsConfig.targets.map(target => target.fqdn)).size, 3)
+ assert.equal(proxy.spec.template.spec.automountServiceAccountToken, false)
+ assert.equal(container.securityContext.allowPrivilegeEscalation, false)
+ const witness = result.canary.spec.workloads[0]
+ assert.equal(witness.transport, undefined)
+ const canaryTarget = dnsConfig.targets.find(target => target.lane === 'canary')
+ assert.deepEqual(
+ witness.egressBindings,
+ [80, 443].map(port => ({ dns: canaryTarget.fqdn, port, protocol: 'TCP' }))
+ )
+ assert.deepEqual(
+ result.recipe.spec.ui.egress.internal,
+ [3000, 3001].map(port => ({ workloadRef: 'worker', port }))
+ )
+ for (const workload of result.recipe.spec.workloads) {
+ assert.equal(workload.port, 3000)
+ assert.equal(workload.healthCheck.port, 3001)
+ }
+ const fromWrc = result.objects.find(object => object.metadata.name === result.names.fromWrc)
+ assert.deepEqual(fromWrc.spec.podSelector, { matchLabels: { app: config.wrcDeployment } })
+ assert.deepEqual(fromWrc.spec.egress[0].to, [
+ { podSelector: { matchLabels: { app: result.names.proxy } } },
+ ])
+})
diff --git a/scripts/tests/wrc-egress-lifecycle.test.cjs b/scripts/tests/wrc-egress-lifecycle.test.cjs
new file mode 100644
index 000000000..4755699d4
--- /dev/null
+++ b/scripts/tests/wrc-egress-lifecycle.test.cjs
@@ -0,0 +1,846 @@
+'use strict'
+
+// Unit/contract tests for the development-only lifecycle. Kubernetes is mocked
+// only at its boundary; child-process supervision uses real local processes.
+// These tests do not certify a browser journey or cluster/dataplane behavior.
+const { test } = require('node:test')
+const assert = require('node:assert/strict')
+const fs = require('node:fs')
+const os = require('node:os')
+const path = require('node:path')
+const { randomUUID } = require('node:crypto')
+const { spawnSync } = require('node:child_process')
+const {
+ Journal,
+ journalPath,
+ readJournal,
+ assertJournalBinding,
+ createOwned,
+ deleteOwned,
+ waitFor,
+ createCommandRunner,
+ createKubeClient,
+ resourcePath,
+ RUN_LABEL,
+} = require('../e2e/_lib/wrc-egress-lifecycle.cjs')
+
+const repository = fs.realpathSync(path.resolve(__dirname, '../..'))
+const lifecycle = path.join(repository, 'scripts/e2e/_lib/wrc-egress-lifecycle.cjs')
+function temporary(t) {
+ const directory = fs.mkdtempSync(path.join(fs.realpathSync(os.tmpdir()), 'wrc-lifecycle-test-'))
+ t.after(() => fs.rmSync(directory, { recursive: true, force: true }))
+ return directory
+}
+function fixture(t) {
+ const directory = temporary(t)
+ const binding = {
+ repository: directory,
+ branch: 'fix/unit-lifecycle',
+ profile: 'unit-profile',
+ context: 'unit-context',
+ head: 'historical-head',
+ }
+ const state = { version: 1, runId: randomUUID(), binding, resources: [] }
+ const file = journalPath(directory, binding.profile)
+ const journal = new Journal(file, state)
+ const object = {
+ apiVersion: 'v1',
+ kind: 'ConfigMap',
+ metadata: {
+ name: 'unit-resource',
+ namespace: 'unit-namespace',
+ labels: { [RUN_LABEL]: state.runId },
+ },
+ data: { fixture: 'unit' },
+ }
+ const created = structuredClone(object)
+ created.metadata.uid = 'created-uid'
+ created.metadata.resourceVersion = 'initial-rv'
+ return { directory, binding, state, file, journal, object, created }
+}
+function apiError(code) {
+ return Object.assign(new Error(`API ${code}`), { code })
+}
+
+test('journal creates private durable state, saves atomically, and removes only its own run', t => {
+ const { journal, file, state } = fixture(t)
+ assert.equal(fs.statSync(file).mode & 0o777, 0o600)
+ assert.deepEqual(readJournal(file), state)
+ state.phase = 'restoring'
+ journal.save()
+ assert.equal(readJournal(file).phase, 'restoring')
+ assert.equal(fs.readdirSync(path.dirname(file)).length, 1)
+ journal.remove()
+ assert.equal(fs.existsSync(file), false)
+})
+
+test('journal binding permits historical HEAD but rejects every stable owner mismatch', t => {
+ const { state, binding } = fixture(t)
+ assert.doesNotThrow(() => assertJournalBinding(state, { ...binding, head: 'current-head' }))
+ for (const key of ['repository', 'branch', 'profile', 'context']) {
+ assert.throws(
+ () => assertJournalBinding(state, { ...binding, [key]: `foreign-${key}` }),
+ /JOURNAL_OWNER_MISMATCH/
+ )
+ assert.throws(
+ () => assertJournalBinding(state, { ...binding, [key]: '' }),
+ /JOURNAL_OWNER_MISMATCH/
+ )
+ }
+})
+
+test('journal refuses replacement, hard links, public permissions, invalid JSON and oversized state', t => {
+ const { journal, file, state, directory } = fixture(t)
+ fs.writeFileSync(file, JSON.stringify({ ...state, runId: randomUUID() }))
+ assert.throws(() => journal.save(), /JOURNAL_REPLACED/)
+ assert.throws(() => journal.remove(), /JOURNAL_REPLACED/)
+ fs.writeFileSync(file, JSON.stringify(state))
+ const link = path.join(directory, 'hardlink.json')
+ fs.linkSync(file, link)
+ assert.throws(() => readJournal(file), /UNSAFE_JOURNAL_FILE/)
+ fs.unlinkSync(link)
+ fs.chmodSync(file, 0o644)
+ assert.throws(() => readJournal(file), /UNSAFE_JOURNAL_FILE/)
+ fs.chmodSync(file, 0o600)
+ fs.writeFileSync(file, '{')
+ assert.throws(() => readJournal(file), SyntaxError)
+ fs.writeFileSync(file, 'x'.repeat(1024 * 1024 + 1))
+ assert.throws(() => readJournal(file), /INVALID_JOURNAL_SIZE/)
+})
+
+test('journal paths reject traversal, noncanonical roots, and symbolic files or ancestors', t => {
+ const { directory, file, state } = fixture(t)
+ for (const profile of ['../escape', '/absolute', 'nested/profile', '', '..']) {
+ assert.throws(() => journalPath(directory, profile), /INVALID_PROFILE/)
+ }
+ const alias = path.join(directory, 'repository-alias')
+ fs.symlinkSync(directory, alias)
+ assert.throws(() => journalPath(alias, 'unit-profile'), /NONCANONICAL_REPOSITORY/)
+ const target = path.join(directory, 'private-journal.json')
+ fs.renameSync(file, target)
+ fs.symlinkSync(target, file)
+ assert.throws(() => readJournal(file), /UNSAFE_JOURNAL_FILE/)
+ fs.unlinkSync(file)
+ const parent = path.dirname(file)
+ const parentTarget = path.join(directory, 'journal-target')
+ fs.renameSync(parent, parentTarget)
+ fs.symlinkSync(parentTarget, parent)
+ assert.throws(() => new Journal(file, state), /UNSAFE_JOURNAL_PARENT/)
+})
+
+test('certification guard rejects an intermediate symlink even before any journal exists', t => {
+ const directory = temporary(t)
+ const outside = path.join(directory, 'outside')
+ fs.mkdirSync(outside)
+ fs.symlinkSync(outside, path.join(directory, '.local-notes'))
+ const result = spawnSync(process.execPath, [lifecycle, directory, 'unit-profile'], {
+ encoding: 'utf8',
+ timeout: 3000,
+ })
+ assert.notEqual(result.status, 0, 'unsafe journal ancestry must not certify an empty lane')
+ assert.match(result.stderr, /WRC_EGRESS_RECOVERY_REQUIRED/)
+})
+
+test('certification guard accepts genuine absence and rejects an outstanding valid journal', t => {
+ const directory = temporary(t)
+ const absent = spawnSync(process.execPath, [lifecycle, directory, 'unit-profile'], {
+ encoding: 'utf8',
+ timeout: 3000,
+ })
+ assert.equal(absent.status, 0)
+ const { directory: owned } = fixture(t)
+ const pending = spawnSync(process.execPath, [lifecycle, owned, 'unit-profile'], {
+ encoding: 'utf8',
+ timeout: 3000,
+ })
+ assert.notEqual(pending.status, 0)
+ assert.match(pending.stderr, /WRC_EGRESS_RECOVERY_REQUIRED/)
+})
+
+test('creation persists its uncertain attempt before I/O and accepts only its acknowledged identity', async t => {
+ const { journal, file, object, created } = fixture(t)
+ let observedAttempt = false
+ const result = await createOwned(
+ {
+ create: async body => {
+ assert.deepEqual(body, object)
+ observedAttempt = readJournal(file).resources[0].uid === null
+ return created
+ },
+ },
+ journal,
+ object
+ )
+ assert.equal(observedAttempt, true)
+ assert.deepEqual(result, created)
+ assert.equal(readJournal(file).resources[0].uid, created.metadata.uid)
+})
+
+test('creation rejects missing run ownership before recording or issuing an attempt', async t => {
+ const { journal, file, object } = fixture(t)
+ delete object.metadata.labels[RUN_LABEL]
+ let creates = 0
+ await assert.rejects(
+ createOwned(
+ {
+ create: async () => {
+ creates++
+ },
+ },
+ journal,
+ object
+ ),
+ /MISSING_RUN_OWNERSHIP/
+ )
+ assert.equal(creates, 0)
+ assert.deepEqual(readJournal(file).resources, [])
+})
+
+for (const code of [403, 409, 503]) {
+ test(`rejected or ambiguous create ${code} never authorizes cleanup of an unacknowledged object`, async t => {
+ const { journal, object, created } = fixture(t)
+ const failure = apiError(code)
+ await assert.rejects(
+ createOwned(
+ {
+ create: async () => {
+ throw failure
+ },
+ },
+ journal,
+ object
+ ),
+ error => error === failure
+ )
+ assert.equal(journal.state.resources[0].uid, null)
+ let deletes = 0
+ await assert.rejects(
+ deleteOwned(
+ {
+ get: async () => created,
+ delete: async () => {
+ deletes++
+ },
+ },
+ journal,
+ journal.state.resources[0]
+ ),
+ /UNPROVEN_CLEANUP_OWNERSHIP/
+ )
+ assert.equal(deletes, 0)
+ })
+}
+
+for (const [label, mutate] of [
+ [
+ 'missing UID',
+ object => {
+ delete object.metadata.uid
+ },
+ ],
+ [
+ 'foreign run',
+ object => {
+ object.metadata.labels[RUN_LABEL] = randomUUID()
+ },
+ ],
+ [
+ 'different name',
+ object => {
+ object.metadata.name = 'another-resource'
+ },
+ ],
+ [
+ 'different namespace',
+ object => {
+ object.metadata.namespace = 'another-namespace'
+ },
+ ],
+]) {
+ test(`creation refuses an acknowledgement with ${label}`, async t => {
+ const { journal, object, created } = fixture(t)
+ mutate(created)
+ await assert.rejects(
+ createOwned({ create: async () => created }, journal, object),
+ /UNPROVEN_CREATED_RESOURCE|CREATED_IDENTITY_MISMATCH/
+ )
+ assert.equal(journal.state.resources[0].uid, null)
+ })
+}
+
+test('cleanup uses fresh UID/resourceVersion preconditions and verifies authoritative absence', async t => {
+ const { journal, created } = fixture(t)
+ const entry = {
+ kind: created.kind,
+ name: created.metadata.name,
+ namespace: created.metadata.namespace,
+ uid: created.metadata.uid,
+ }
+ let live = { ...created, metadata: { ...created.metadata, resourceVersion: 'fresh-rv' } }
+ const calls = []
+ await deleteOwned(
+ {
+ get: async () => {
+ calls.push('get')
+ return live
+ },
+ delete: async (identity, fence) => {
+ calls.push('delete')
+ assert.deepEqual(identity, entry)
+ assert.deepEqual(fence, { uid: entry.uid, resourceVersion: 'fresh-rv' })
+ live = null
+ },
+ },
+ journal,
+ entry
+ )
+ assert.deepEqual(calls, ['get', 'delete', 'get'])
+})
+
+test('cleanup distinguishes failed observation, authoritative absence, and a foreign UID', async t => {
+ const { journal, created } = fixture(t)
+ const entry = {
+ kind: created.kind,
+ name: created.metadata.name,
+ namespace: created.metadata.namespace,
+ uid: created.metadata.uid,
+ }
+ let deletes = 0
+ const del = async () => {
+ deletes++
+ }
+ await assert.rejects(
+ deleteOwned(
+ {
+ get: async () => {
+ throw apiError(503)
+ },
+ delete: del,
+ },
+ journal,
+ entry
+ ),
+ /API 503/
+ )
+ await deleteOwned({ get: async () => null, delete: del }, journal, entry)
+ await assert.rejects(
+ deleteOwned(
+ {
+ get: async () => ({ ...created, metadata: { ...created.metadata, uid: 'recreated-uid' } }),
+ delete: del,
+ },
+ journal,
+ entry
+ ),
+ /UNPROVEN_CLEANUP_OWNERSHIP/
+ )
+ assert.equal(deletes, 0)
+})
+
+test('cleanup forwards its bounded predicate signal to the authoritative absence GET', async t => {
+ const { journal, created } = fixture(t)
+ const entry = {
+ kind: created.kind,
+ name: created.metadata.name,
+ namespace: created.metadata.namespace,
+ uid: created.metadata.uid,
+ }
+ let gets = 0
+ await deleteOwned(
+ {
+ get: async (_identity, remaining, signal) => {
+ if (++gets === 1) return created
+ assert(remaining > 0 && remaining <= 90000)
+ assert(signal instanceof AbortSignal, 'the predicate signal must reach its real I/O')
+ return null
+ },
+ delete: async () => {},
+ },
+ journal,
+ entry
+ )
+})
+
+test('cleanup does not retry a failed delete with an unfenced or newly recreated object', async t => {
+ const { journal, created } = fixture(t)
+ const entry = {
+ kind: created.kind,
+ name: created.metadata.name,
+ namespace: created.metadata.namespace,
+ uid: created.metadata.uid,
+ }
+ let gets = 0
+ let deletes = 0
+ const recreated = { ...created, metadata: { ...created.metadata, uid: 'recreated-uid' } }
+ await assert.rejects(
+ deleteOwned(
+ {
+ get: async () => (++gets === 1 ? created : recreated),
+ delete: async () => {
+ deletes++
+ throw apiError(409)
+ },
+ },
+ journal,
+ entry
+ ),
+ /API 409|RESOURCE_RECREATED|UNPROVEN_CLEANUP/
+ )
+ assert.equal(deletes, 1)
+ gets = 0
+ await assert.rejects(
+ deleteOwned(
+ {
+ get: async () => (++gets === 1 ? created : recreated),
+ delete: async () => {},
+ },
+ journal,
+ entry
+ ),
+ /RESOURCE_RECREATED_DURING_CLEANUP/
+ )
+})
+
+test('delete failure is cleared only when a subsequent successful GET proves absence', async t => {
+ const { journal, created } = fixture(t)
+ const entry = {
+ kind: created.kind,
+ name: created.metadata.name,
+ namespace: created.metadata.namespace,
+ uid: created.metadata.uid,
+ }
+ let gets = 0
+ await deleteOwned(
+ {
+ get: async () => (++gets === 1 ? created : null),
+ delete: async () => {
+ throw apiError(503)
+ },
+ },
+ journal,
+ entry
+ )
+ gets = 0
+ await assert.rejects(
+ deleteOwned(
+ {
+ get: async () => {
+ if (++gets === 1) return created
+ throw apiError(503)
+ },
+ delete: async () => {
+ throw apiError(503)
+ },
+ },
+ journal,
+ entry
+ ),
+ /API 503/
+ )
+})
+
+test('waitFor rejects a true predicate that finishes after its absolute deadline', async () => {
+ let now = 100
+ await assert.rejects(
+ waitFor(
+ 'late',
+ 20,
+ async remaining => {
+ assert.equal(remaining, 20)
+ now = 121
+ return true
+ },
+ { now: () => now }
+ ),
+ error => /TIMEOUT:late/.test(error.message) && error.exitCode === 124
+ )
+})
+
+test('waitFor does not accept true after cancellation during the predicate', async () => {
+ const cancellation = new AbortController()
+ const reason = new Error('cancelled during predicate')
+ await assert.rejects(
+ waitFor(
+ 'cancelled',
+ 1000,
+ async () => {
+ cancellation.abort(reason)
+ return true
+ },
+ { signal: cancellation.signal }
+ ),
+ error => error === reason
+ )
+})
+
+test('waitFor propagates parent cancellation to in-flight predicate I/O', async () => {
+ const cancellation = new AbortController()
+ const reason = new Error('cancelled in-flight')
+ let receivedSignal
+ await assert.rejects(
+ waitFor(
+ 'in-flight',
+ 100,
+ async (_remaining, signal) => {
+ receivedSignal = signal
+ cancellation.abort(reason)
+ return new Promise(() => {})
+ },
+ { signal: cancellation.signal }
+ ),
+ error => error === reason
+ )
+ assert.equal(receivedSignal.aborted, true)
+})
+
+test('waitFor cancels a hung predicate at its own deadline and never treats an exception as ready', async () => {
+ let signal
+ await assert.rejects(
+ waitFor('hung', 20, async (_remaining, current) => {
+ signal = current
+ return new Promise(() => {})
+ }),
+ /TIMEOUT:hung/
+ )
+ assert.equal(signal.aborted, true)
+ const failure = new Error('predicate failed')
+ await assert.rejects(
+ waitFor('failed', 100, async () => {
+ throw failure
+ }),
+ error => error === failure
+ )
+})
+
+test('Kubernetes client distinguishes failed GET from authoritative empty NotFound and always supplies context', async () => {
+ const calls = []
+ const identity = { kind: 'ConfigMap', name: 'unit-resource', namespace: 'unit-namespace' }
+ let response = ''
+ const client = createKubeClient(repository, 'unit-context', async (args, options) => {
+ calls.push({ args, options })
+ if (response instanceof Error) throw response
+ return response
+ })
+ assert.equal(await client.get(identity, 1234), null)
+ assert(calls[0].args.includes('--context=unit-context'))
+ assert(calls[0].args.includes('--ignore-not-found=true'))
+ assert.equal(calls[0].options.timeoutMs, 1234)
+ response = apiError(503)
+ await assert.rejects(client.get(identity), /API 503/)
+ response = '{'
+ await assert.rejects(client.get(identity), SyntaxError)
+})
+
+test('Kubernetes reads cap their own request budget and refuse an expired phase before spawning', async () => {
+ const calls = []
+ const client = createKubeClient(repository, 'unit-profile', async (args, options) => {
+ calls.push({ args, options })
+ return args.includes('get') && args.includes('pods') ? '{"items":[]}' : ''
+ })
+ const identity = { kind: 'Service', namespace: 'unit', name: 'service' }
+ await client.get(identity, 90_000)
+ await client.list('pods', 'unit', 'app=unit', 900)
+ assert.equal(calls[0].options.timeoutMs, 30_000)
+ assert.equal(calls[1].options.timeoutMs, 900)
+ await assert.rejects(client.get(identity, 0), error => error.exitCode === 124)
+ assert.equal(calls.length, 2)
+})
+
+test('cleanup waits for aborted polling processes to finish before returning ownership to the restorer', async () => {
+ let finish
+ const client = createKubeClient(
+ repository,
+ 'unit-profile',
+ () =>
+ new Promise(resolve => {
+ finish = resolve
+ })
+ )
+ const observation = client.get({ kind: 'Service', namespace: 'unit', name: 'service' })
+ let drained = false
+ const cleanup = client.setCleanup(true).then(() => {
+ drained = true
+ })
+ await Promise.resolve()
+ assert.equal(drained, false)
+ finish('')
+ assert.equal(await observation, null)
+ await cleanup
+ assert.equal(drained, true)
+})
+
+test('Kubernetes resource operations reject invalid status-derived identities before dispatch', async () => {
+ let calls = 0
+ const client = createKubeClient(repository, 'unit-profile', async () => {
+ calls++
+ return '{}'
+ })
+ for (const identity of [
+ { kind: 'Deployment', name: 'invalid name', namespace: 'unit' },
+ { kind: 'Service', name: '-invalid', namespace: 'unit' },
+ { kind: 'NetworkPolicy', name: 'valid', namespace: '' },
+ { kind: 'Unknown', name: 'valid', namespace: 'unit' },
+ ]) {
+ await assert.rejects(client.get(identity), /INVALID_RESOURCE_IDENTITY/)
+ await assert.rejects(client.patch(identity, []), /INVALID_RESOURCE_IDENTITY/)
+ }
+ assert.equal(calls, 0)
+})
+
+test('Kubernetes mutations require the live lease before create/patch/delete and preserve raw delete fencing', async () => {
+ const calls = []
+ const identity = { kind: 'ConfigMap', name: 'unit-resource', namespace: 'unit-namespace' }
+ const client = createKubeClient(repository, 'unit-context', async (args, options) => {
+ calls.push({ args, options })
+ return '{}'
+ })
+ await client.create({ kind: 'ConfigMap', metadata: identity })
+ await client.patch(identity, [{ op: 'test', path: '/metadata/uid', value: 'uid' }])
+ await client.delete(identity, { uid: 'uid', resourceVersion: 'rv' })
+ assert.equal(calls.length, 6)
+ for (const index of [0, 2, 4]) {
+ assert.equal(calls[index].args[0], 'bash')
+ assert.match(calls[index].args[1], /require-t2-mutation-lock\.sh$/)
+ assert(calls[index + 1].args.includes('--context=unit-context'))
+ }
+ assert(calls[5].args.includes(`--raw=${resourcePath(identity)}`))
+ assert.deepEqual(JSON.parse(calls[5].options.input).preconditions, {
+ uid: 'uid',
+ resourceVersion: 'rv',
+ })
+ let mutations = 0
+ const denied = createKubeClient(repository, 'unit-context', async args => {
+ if (args[0] === 'bash') throw new Error('lease rejected')
+ mutations++
+ return '{}'
+ })
+ await assert.rejects(
+ denied.delete(identity, { uid: 'uid', resourceVersion: 'rv' }),
+ /lease rejected/
+ )
+ assert.equal(mutations, 0)
+})
+
+test(
+ 'command runner returns successful output and keeps a separate cleanup budget after cancellation',
+ { timeout: 5000 },
+ async () => {
+ const cancellation = new AbortController()
+ const run = createCommandRunner(repository, cancellation)
+ assert.equal(
+ await run([process.execPath, '-e', 'process.stdin.pipe(process.stdout)'], {
+ input: 'unit-input',
+ timeoutMs: 2000,
+ }),
+ 'unit-input'
+ )
+ const reason = new Error('main run interrupted')
+ cancellation.abort(reason)
+ await assert.rejects(
+ run([process.execPath, '-e', 'process.exit(0)']),
+ error => error === reason
+ )
+ assert.equal(
+ await run([process.execPath, '-e', 'process.stdout.write("restored")'], {
+ cleanup: true,
+ timeoutMs: 2000,
+ }),
+ 'restored'
+ )
+ }
+)
+
+for (const cleanup of [false, true]) {
+ test(
+ `command runner never starts an already-cancelled operation (cleanup=${cleanup})`,
+ { timeout: 5000 },
+ async t => {
+ const directory = temporary(t)
+ const marker = path.join(directory, 'must-not-run')
+ const cancellation = new AbortController()
+ const operation = new AbortController()
+ operation.abort(new Error('operation already cancelled'))
+ const run = createCommandRunner(repository, cancellation)
+ await assert.rejects(
+ run(
+ [
+ process.execPath,
+ '-e',
+ 'require("node:fs").writeFileSync(process.argv[1], "ran")',
+ marker,
+ ],
+ {
+ cleanup,
+ signal: operation.signal,
+ timeoutMs: 2000,
+ }
+ )
+ )
+ assert.equal(fs.existsSync(marker), false)
+ }
+ )
+}
+
+test(
+ 'command runner enforces the requested millisecond budget before later side effects',
+ { timeout: 5000 },
+ async t => {
+ const directory = temporary(t)
+ const marker = path.join(directory, 'late-effect')
+ const run = createCommandRunner(repository, new AbortController())
+ await assert.rejects(
+ run(
+ [
+ process.execPath,
+ '-e',
+ 'setTimeout(() => require("node:fs").writeFileSync(process.argv[1], "late"), 250)',
+ marker,
+ ],
+ {
+ timeoutMs: 40,
+ }
+ ),
+ error => error.exitCode === 124
+ )
+ assert.equal(
+ fs.existsSync(marker),
+ false,
+ 'work must terminate at the operation deadline, not merely be judged late after completion'
+ )
+ }
+)
+
+test(
+ 'command runner bounds output and does not expose child output in its error',
+ { timeout: 5000 },
+ async () => {
+ const run = createCommandRunner(repository, new AbortController())
+ await assert.rejects(
+ run([process.execPath, '-e', 'process.stdout.write("unit-payload".repeat(500000))'], {
+ timeoutMs: 2000,
+ }),
+ error =>
+ /WRC_GATE_OPERATION_FAILED/.test(error.message) && !error.message.includes('unit-payload')
+ )
+ }
+)
+
+function observeJsonFile(file, timeoutMs = 3000) {
+ return new Promise((resolve, reject) => {
+ // Watch an explicit readiness artifact. Stat polling avoids the platform's
+ // finite native watcher quota without guessing when a child is ready.
+ const changed = () => {
+ if (!fs.existsSync(file)) return
+ try {
+ const value = JSON.parse(fs.readFileSync(file, 'utf8'))
+ clearTimeout(timer)
+ fs.unwatchFile(file, changed)
+ resolve(value)
+ } catch (error) {
+ if (!(error instanceof SyntaxError)) {
+ clearTimeout(timer)
+ fs.unwatchFile(file, changed)
+ reject(error)
+ }
+ }
+ }
+ fs.watchFile(file, { interval: 20 }, changed)
+ const timer = setTimeout(() => {
+ fs.unwatchFile(file, changed)
+ reject(new Error('child readiness deadline'))
+ }, timeoutMs)
+ })
+}
+
+function processRunning(pid) {
+ const result = spawnSync('ps', ['-p', String(pid), '-o', 'stat='], {
+ encoding: 'utf8',
+ timeout: 1000,
+ })
+ if (result.error) throw result.error
+ assert.equal(result.stderr.trim(), '', 'failed process inspection cannot prove a child exited')
+ return result.status === 0 && result.stdout.trim() !== '' && !result.stdout.trim().startsWith('Z')
+}
+
+test(
+ 'command cancellation reaps a TERM-resistant descendant before resolving cleanup ownership',
+ { timeout: 8000 },
+ async t => {
+ const directory = temporary(t)
+ const marker = path.join(directory, 'children.json')
+ const readiness = observeJsonFile(marker)
+ const cancellation = new AbortController()
+ const run = createCommandRunner(repository, cancellation)
+ const grandchildCode =
+ 'process.on("SIGTERM",()=>{});process.stdout.write("ready");setInterval(()=>{},1000)'
+ const parentCode = `
+ const child = require('node:child_process').spawn(process.execPath, ['-e', ${JSON.stringify(grandchildCode)}], {stdio:['ignore','pipe','ignore']});
+ process.on('SIGTERM', () => process.exit(0));
+ child.stdout.once('data', () => require('node:fs').writeFileSync(process.argv[1], JSON.stringify({parent:process.pid,descendant:child.pid})));
+ setInterval(()=>{},1000);
+ `
+ const operation = run([process.execPath, '-e', parentCode, marker], { timeoutMs: 6000 })
+ // Attach rejection immediately so an unexpected early child failure is never
+ // an unhandled rejection while the independent readiness event is awaited.
+ const outcome = operation.then(
+ value => ({ value }),
+ error => ({ error })
+ )
+ const pids = await readiness
+ t.after(() => {
+ for (const pid of [pids.parent, pids.descendant]) {
+ if (processRunning(pid)) {
+ try {
+ process.kill(pid, 'SIGKILL')
+ } catch (error) {
+ if (error.code !== 'ESRCH') throw error
+ }
+ }
+ }
+ })
+ assert(processRunning(pids.parent))
+ assert(processRunning(pids.descendant))
+ cancellation.abort(Object.assign(new Error('terminated by test'), { exitCode: 143 }))
+ const result = await outcome
+ assert(result.error, 'cancellation must never report success')
+ assert.equal(result.error.exitCode, 143)
+ assert.equal(processRunning(pids.parent), false)
+ assert.equal(processRunning(pids.descendant), false)
+ }
+)
+
+test(
+ 'natural direct-child exit does not leave a daemonized descendant running',
+ { timeout: 5000 },
+ async t => {
+ const directory = temporary(t)
+ const marker = path.join(directory, 'daemon.json')
+ const readiness = observeJsonFile(marker)
+ const run = createCommandRunner(repository, new AbortController())
+ const code = `
+ const child = require('node:child_process').spawn(process.execPath, ['-e', 'process.stdout.write("ready");setInterval(()=>{},1000)'], {stdio:['ignore','pipe','ignore']});
+ child.stdout.once('data', () => {
+ require('node:fs').writeFileSync(process.argv[1], JSON.stringify({pid:child.pid}));
+ process.exit(0);
+ });
+ `
+ const operation = run([process.execPath, '-e', code, marker], { timeoutMs: 3000 })
+ const outcome = operation.then(
+ value => ({ value }),
+ error => ({ error })
+ )
+ const { pid } = await readiness
+ t.after(() => {
+ if (processRunning(pid)) {
+ try {
+ process.kill(pid, 'SIGKILL')
+ } catch (error) {
+ if (error.code !== 'ESRCH') throw error
+ }
+ }
+ })
+ const result = await outcome
+ assert.equal(result.error, undefined)
+ assert.equal(processRunning(pid), false)
+ }
+)
diff --git a/scripts/tests/wrc-egress-proof.test.cjs b/scripts/tests/wrc-egress-proof.test.cjs
new file mode 100644
index 000000000..401d16fb3
--- /dev/null
+++ b/scripts/tests/wrc-egress-proof.test.cjs
@@ -0,0 +1,445 @@
+'use strict'
+
+const { test } = require('node:test')
+const assert = require('node:assert/strict')
+const { spawnSync } = require('node:child_process')
+const path = require('node:path')
+const { assertPolicy, validatePolicy } = require('../e2e/_lib/wrc-egress-proof.cjs')
+
+const STATE = 'clerum.io/egress-fqdn-state'
+const TARGETS = 'clerum.io/egress-fqdn-targets'
+const RESOLVED_AT = 'clerum.io/egress-fqdn-resolved-at'
+function fixture(lane = 'ui') {
+ const tuples = [{ fqdn: 'fixture.example.com', ip: '93.184.216.34', port: 443, protocol: 'TCP' }]
+ const owner = {
+ 'clerum.io/managed-by': 'workflow-recipes',
+ 'clerum.io/recipe': 'recipe',
+ 'clerum.io/recipe-name': 'recipe',
+ 'clerum.io/recipe-namespace': 'sandbox-recipes',
+ }
+ const podLabels = {
+ ...owner,
+ 'clerum.io/workload': lane === 'ui' ? 'frontend' : 'worker',
+ 'clerum.io/sandbox-ui': 'true',
+ }
+ const expected = {
+ name: 'policy',
+ namespace: lane === 'ui' ? 'sandbox-ui' : 'sandbox-recipes',
+ recipe: 'recipe',
+ recipeNamespace: 'sandbox-recipes',
+ lane,
+ workload: podLabels['clerum.io/workload'],
+ podLabels,
+ tuples,
+ uid: 'policy-uid',
+ }
+ const policy = {
+ apiVersion: 'networking.k8s.io/v1',
+ kind: 'NetworkPolicy',
+ metadata: {
+ name: expected.name,
+ namespace: expected.namespace,
+ uid: expected.uid,
+ resourceVersion: '1',
+ labels: { ...owner, 'clerum.io/workload': expected.workload },
+ annotations: {
+ [STATE]: JSON.stringify(
+ tuples.map(tuple => ({ ...tuple, expiresAt: 2000, lastObservedAt: 1000 }))
+ ),
+ [TARGETS]: 'fixture.example.com=93.184.216.34/32',
+ [RESOLVED_AT]: new Date(1000).toISOString(),
+ },
+ },
+ spec: {
+ podSelector: {
+ matchLabels:
+ lane === 'ui'
+ ? {
+ 'clerum.io/sandbox-ui': 'true',
+ 'clerum.io/recipe-name': 'recipe',
+ 'clerum.io/recipe-namespace': 'sandbox-recipes',
+ }
+ : { 'clerum.io/recipe': 'recipe', 'clerum.io/workload': 'worker' },
+ },
+ policyTypes: ['Egress'],
+ egress: [
+ {
+ to: [{ ipBlock: { cidr: '93.184.216.34/32' } }],
+ ports: [{ port: 443, protocol: 'TCP' }],
+ },
+ ],
+ },
+ }
+ return { expected, policy }
+}
+
+for (const lane of ['ui', 'workload']) {
+ test(`accepts a complete ${lane} policy without treating expired fail-static state as invalid`, () => {
+ const { policy, expected } = fixture(lane)
+ assert.deepEqual(assertPolicy(policy, expected), { ok: true })
+ expected.internal = []
+ delete expected.uid
+ assert.deepEqual(validatePolicy(policy, expected), { ok: true })
+ })
+}
+
+const regressions = [
+ [
+ 'wrong selector',
+ p => {
+ p.spec.podSelector = { matchLabels: { app: 'nonexistent' } }
+ },
+ ],
+ [
+ 'UDP instead of TCP',
+ p => {
+ p.spec.egress[0].ports[0].protocol = 'UDP'
+ },
+ ],
+ [
+ 'extra allow-all rule',
+ p => {
+ p.spec.egress.push({})
+ },
+ ],
+ [
+ 'different provenance FQDN/IP/protocol',
+ p => {
+ p.metadata.annotations[STATE] = JSON.stringify([
+ { fqdn: 'other.example.com', ip: '8.8.8.8', port: 443, protocol: 'UDP' },
+ ])
+ },
+ ],
+ [
+ 'wrong recipe owner',
+ p => {
+ p.metadata.labels['clerum.io/recipe'] = 'foreign'
+ },
+ ],
+ [
+ 'extra peer',
+ p => {
+ p.spec.egress[0].to.push({ namespaceSelector: {} })
+ },
+ ],
+ [
+ 'no peers',
+ p => {
+ delete p.spec.egress[0].to
+ },
+ ],
+ [
+ 'empty peers',
+ p => {
+ p.spec.egress[0].to = []
+ },
+ ],
+ [
+ 'no ports',
+ p => {
+ delete p.spec.egress[0].ports
+ },
+ ],
+ [
+ 'empty ports',
+ p => {
+ p.spec.egress[0].ports = []
+ },
+ ],
+ [
+ 'port range',
+ p => {
+ p.spec.egress[0].ports[0].endPort = 444
+ },
+ ],
+ [
+ 'named port',
+ p => {
+ p.spec.egress[0].ports[0].port = 'https'
+ },
+ ],
+ [
+ 'out of range port',
+ p => {
+ p.spec.egress[0].ports[0].port = 65536
+ },
+ ],
+ [
+ 'null protocol',
+ p => {
+ p.spec.egress[0].ports[0].protocol = null
+ },
+ ],
+ [
+ 'CIDR wider than host',
+ p => {
+ p.spec.egress[0].to[0].ipBlock.cidr = '93.184.216.0/24'
+ },
+ ],
+ [
+ 'CIDR exception',
+ p => {
+ p.spec.egress[0].to[0].ipBlock.except = ['93.184.216.34/32']
+ },
+ ],
+ [
+ 'mixed peer selectors',
+ p => {
+ p.spec.egress[0].to[0].podSelector = {}
+ },
+ ],
+ [
+ 'unknown rule field',
+ p => {
+ p.spec.egress[0].unexpected = true
+ },
+ ],
+ [
+ 'unknown spec field',
+ p => {
+ p.spec.futureAllow = true
+ },
+ ],
+ [
+ 'ingress policy type',
+ p => {
+ p.spec.policyTypes.push('Ingress')
+ },
+ ],
+ [
+ 'omitted policy types',
+ p => {
+ delete p.spec.policyTypes
+ },
+ ],
+ [
+ 'selector expression',
+ p => {
+ p.spec.podSelector.matchExpressions = []
+ },
+ ],
+ [
+ 'wrong namespace',
+ p => {
+ p.metadata.namespace = 'other'
+ },
+ ],
+ [
+ 'wrong UID',
+ p => {
+ p.metadata.uid = 'replacement-uid'
+ },
+ ],
+ [
+ 'missing resource version',
+ p => {
+ delete p.metadata.resourceVersion
+ },
+ ],
+ [
+ 'deleting policy',
+ p => {
+ p.metadata.deletionTimestamp = new Date().toISOString()
+ },
+ ],
+ [
+ 'annotation-only IP',
+ p => {
+ const state = JSON.parse(p.metadata.annotations[STATE])
+ state[0].ip = '8.8.8.8'
+ p.metadata.annotations[STATE] = JSON.stringify(state)
+ },
+ ],
+ [
+ 'annotation-only FQDN',
+ p => {
+ const state = JSON.parse(p.metadata.annotations[STATE])
+ state[0].fqdn = 'other.example.com'
+ p.metadata.annotations[STATE] = JSON.stringify(state)
+ },
+ ],
+ [
+ 'duplicate provenance',
+ p => {
+ const state = JSON.parse(p.metadata.annotations[STATE])
+ p.metadata.annotations[STATE] = JSON.stringify([...state, ...state])
+ },
+ ],
+ [
+ 'missing timestamp',
+ p => {
+ const state = JSON.parse(p.metadata.annotations[STATE])
+ delete state[0].lastObservedAt
+ p.metadata.annotations[STATE] = JSON.stringify(state)
+ },
+ ],
+ [
+ 'corrupt state JSON',
+ p => {
+ p.metadata.annotations[STATE] = '{'
+ },
+ ],
+ [
+ 'missing structured state',
+ p => {
+ delete p.metadata.annotations[STATE]
+ },
+ ],
+ [
+ 'wrong target summary',
+ p => {
+ p.metadata.annotations[TARGETS] = 'other.example.com=8.8.8.8/32'
+ },
+ ],
+ [
+ 'invalid resolution date',
+ p => {
+ p.metadata.annotations[RESOLVED_AT] = 'not-a-date'
+ },
+ ],
+]
+for (const [description, mutate] of regressions) {
+ test(`rejects ${description}`, () => {
+ const { policy, expected } = fixture()
+ mutate(policy)
+ assert.equal(validatePolicy(policy, expected).ok, false)
+ })
+}
+
+test('accepts Kubernetes TCP default, reordered/grouped permissions, and harmless duplicate network rules', () => {
+ const { policy, expected } = fixture()
+ const extra = { ...expected.tuples[0], port: 80 }
+ expected.tuples.push(extra)
+ policy.spec.egress[0].ports.unshift({ port: 80 })
+ delete policy.spec.egress[0].ports[1].protocol
+ policy.spec.egress.push(structuredClone(policy.spec.egress[0]))
+ policy.metadata.annotations[STATE] = JSON.stringify(
+ expected.tuples.map(entry => ({ ...entry, expiresAt: 2000, lastObservedAt: 1000 }))
+ )
+ policy.metadata.annotations[TARGETS] += ',' + policy.metadata.annotations[TARGETS]
+ assert.equal(validatePolicy(policy, expected).ok, true)
+})
+
+test('rejects unintended peer/port Cartesian product', () => {
+ const { policy, expected } = fixture()
+ expected.tuples.push({ fqdn: 'second.example.com', ip: '8.8.8.8', port: 80, protocol: 'TCP' })
+ policy.spec.egress[0].to.push({ ipBlock: { cidr: '8.8.8.8/32' } })
+ policy.spec.egress[0].ports.push({ port: 80 })
+ assert.equal(validatePolicy(policy, expected).code, 'EXTERNAL_TUPLES_MISMATCH')
+})
+
+test('distinct FQDN provenance may legitimately share an enforced IP/port tuple', () => {
+ const { policy, expected } = fixture()
+ expected.tuples.push({ ...expected.tuples[0], fqdn: 'alias.example.com' })
+ policy.metadata.annotations[STATE] = JSON.stringify(
+ expected.tuples.map(entry => ({
+ ...entry,
+ expiresAt: 2000,
+ lastObservedAt: 1000,
+ }))
+ )
+ policy.metadata.annotations[TARGETS] += ',alias.example.com=93.184.216.34/32'
+ assert.equal(validatePolicy(policy, expected).ok, true)
+})
+
+test('accepts only explicitly declared conjunctive UI-to-worker Service permissions', () => {
+ const { policy, expected } = fixture()
+ expected.internal = [
+ { namespace: 'sandbox-recipes', workload: 'worker', port: 3000, protocol: 'TCP' },
+ ]
+ const rule = {
+ to: [
+ {
+ namespaceSelector: { matchLabels: { 'kubernetes.io/metadata.name': 'sandbox-recipes' } },
+ podSelector: {
+ matchLabels: { 'clerum.io/recipe': 'recipe', 'clerum.io/workload': 'worker' },
+ },
+ },
+ ],
+ ports: [{ port: 3000, protocol: 'TCP' }],
+ }
+ policy.spec.egress.push(rule)
+ assert.equal(validatePolicy(policy, expected).ok, true)
+ for (const mutate of [
+ p => {
+ p.spec.egress[1].to[0].namespaceSelector.matchLabels['kubernetes.io/metadata.name'] =
+ 'foreign'
+ },
+ p => {
+ p.spec.egress[1].to[0].podSelector.matchLabels['clerum.io/recipe'] = 'foreign'
+ },
+ p => {
+ p.spec.egress[1].to[0].podSelector.matchLabels['clerum.io/workload'] = 'other-worker'
+ },
+ p => {
+ const peer = p.spec.egress[1].to[0]
+ p.spec.egress[1].to = [
+ { namespaceSelector: peer.namespaceSelector },
+ { podSelector: peer.podSelector },
+ ]
+ },
+ p => {
+ p.spec.egress[1].to[0].namespaceSelector = {}
+ },
+ p => {
+ p.spec.egress[1].to[0].podSelector.matchExpressions = []
+ },
+ p => {
+ p.spec.egress[1].ports.push({ port: 3001 })
+ },
+ ]) {
+ const candidate = structuredClone(policy)
+ mutate(candidate)
+ assert.equal(validatePolicy(candidate, expected).ok, false)
+ }
+ delete expected.internal
+ assert.equal(validatePolicy(policy, expected).ok, false)
+})
+
+test('accepts empty or omitted top-level egress under explicit Egress isolation', () => {
+ const { policy, expected } = fixture()
+ expected.tuples = []
+ policy.spec.egress = []
+ policy.metadata.annotations = { [STATE]: '[]', [TARGETS]: '' }
+ assert.equal(validatePolicy(policy, expected).ok, true)
+ delete policy.spec.egress
+ assert.equal(validatePolicy(policy, expected).ok, true)
+ policy.spec.egress = [{}]
+ assert.equal(validatePolicy(policy, expected).ok, false)
+})
+
+test('rejects unknown, defaulted, duplicate, and incorrectly bound expectations', () => {
+ const { policy, expected } = fixture()
+ for (const candidate of [
+ undefined,
+ {},
+ { ...expected, lane: 'unknown' },
+ { ...expected, extra: true },
+ { ...expected, tuples: [...expected.tuples, ...expected.tuples] },
+ { ...expected, tuples: [{ ...expected.tuples[0], protocol: undefined }] },
+ ]) {
+ assert.equal(validatePolicy(policy, candidate).input, true)
+ }
+ const wrongPod = {
+ ...expected,
+ podLabels: { ...expected.podLabels, 'clerum.io/workload': 'other' },
+ }
+ assert.equal(validatePolicy(policy, wrongPod).code, 'POD_IDENTITY_MISMATCH')
+})
+
+test('CLI distinguishes bad input from a failed proof and never echoes policy contents', () => {
+ const { policy, expected } = fixture()
+ const cli = path.join(__dirname, '../e2e/_lib/wrc-egress-proof.cjs')
+ const run = (argv, input) =>
+ spawnSync(process.execPath, [cli, ...argv], { input, encoding: 'utf8', timeout: 5000 })
+ const args = ['--expect-json', JSON.stringify(expected)]
+ assert.equal(run(args, JSON.stringify(policy)).status, 0)
+ assert.equal(run([], JSON.stringify(policy)).status, 2)
+ assert.equal(run(['--expect-json', '{'], '').status, 2)
+ assert.equal(run(args, '{').status, 2)
+ policy.spec.egress.push({})
+ const failure = run(args, JSON.stringify(policy))
+ assert.equal(failure.status, 1)
+ assert.equal(failure.stdout, '')
+ assert.deepEqual(JSON.parse(failure.stderr), { ok: false, code: 'UNRESTRICTED_OR_UNKNOWN_RULE' })
+})
diff --git a/tests/e2e/fixtures/package-lock.json b/tests/e2e/fixtures/package-lock.json
index f89b7ed67..40bfad0b9 100644
--- a/tests/e2e/fixtures/package-lock.json
+++ b/tests/e2e/fixtures/package-lock.json
@@ -1,12 +1,12 @@
{
"name": "@clerum/e2e-test-fixtures",
- "version": "1.0.70",
+ "version": "1.0.71",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "@clerum/e2e-test-fixtures",
- "version": "1.0.70",
+ "version": "1.0.71",
"dependencies": {
"@kubernetes/client-node": "^1.0.0"
},
diff --git a/tests/e2e/fixtures/package.json b/tests/e2e/fixtures/package.json
index b6f32c58f..e471108f1 100644
--- a/tests/e2e/fixtures/package.json
+++ b/tests/e2e/fixtures/package.json
@@ -1,6 +1,6 @@
{
"name": "@clerum/e2e-test-fixtures",
- "version": "1.0.70",
+ "version": "1.0.71",
"description": "E2E test fixtures and utilities for Clerum testing",
"type": "module",
"main": "index.ts",
diff --git a/tests/e2e/fixtures/wrc-egress-dns-proxy/server.cjs b/tests/e2e/fixtures/wrc-egress-dns-proxy/server.cjs
index d09d60637..0ae585cee 100644
--- a/tests/e2e/fixtures/wrc-egress-dns-proxy/server.cjs
+++ b/tests/e2e/fixtures/wrc-egress-dns-proxy/server.cjs
@@ -4,27 +4,45 @@ const dgram = require('node:dgram')
const http = require('node:http')
const net = require('node:net')
-const target = String(process.env.TARGET_DNS || '').toLowerCase()
-const upstream = process.env.UPSTREAM_DNS
-const answerIp = String(process.env.ANSWER_IP || '93.184.216.34')
-const answerOctets = answerIp.split('.').map(Number)
+const MAX_HELD = 512
+const MAX_OBSERVATIONS = 16
+const MAX_QUERY_IDENTITIES = 1024
+const MAX_RELAYS = 128
+const MODES = new Set(['ok', 'hold', 'servfail'])
-if (!target || !upstream || answerOctets.length !== 4 || answerOctets.some(Number.isNaN)) {
- throw new Error('TARGET_DNS, UPSTREAM_DNS, and a valid IPv4 ANSWER_IP are required')
+function validHostname(value) {
+ return (
+ typeof value === 'string' &&
+ value.length <= 253 &&
+ value.includes('.') &&
+ value.split('.').every(label => /^[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?$/.test(label))
+ )
}
-let mode = 'ok'
-const held = []
-const counts = { ok: 0, hold: 0, servfail: 0, forwarded: 0 }
-const udp = dgram.createSocket('udp4')
+function readConfig(env) {
+ let targets
+ try {
+ targets = JSON.parse(env.DNS_TARGETS_JSON)
+ } catch {
+ throw new Error('DNS_TARGETS_JSON must contain the explicit UI/worker/canary target array')
+ }
+ return {
+ targets,
+ upstream: env.UPSTREAM_DNS,
+ ttlSeconds: env.DNS_TTL_SECONDS === undefined ? 300 : Number(env.DNS_TTL_SECONDS),
+ }
+}
function question(message) {
+ if (message.length < 12 || message.readUInt16BE(4) !== 1 || message.readUInt16BE(2) & 0x8000)
+ return null
let offset = 12
const labels = []
while (offset < message.length) {
const length = message[offset++]
if (length === 0) break
- if (offset + length > message.length) return null
+ // Compressed questions are forwarded unchanged, not partially interpreted.
+ if (length > 63 || offset + length > message.length) return null
labels.push(message.subarray(offset, offset + length).toString('ascii'))
offset += length
}
@@ -32,13 +50,16 @@ function question(message) {
return {
name: labels.join('.').toLowerCase(),
type: message.readUInt16BE(offset),
+ queryClass: message.readUInt16BE(offset + 2),
end: offset + 4,
+ id: message.readUInt16BE(0),
}
}
-function dnsResponse(message, parsed, rcode) {
+function dnsResponse(message, parsed, rcode, answerIp, ttlSeconds) {
const header = Buffer.from(message.subarray(0, 12))
header.writeUInt16BE(0x8180 | rcode, 2)
+ header.writeUInt16BE(1, 4)
header.writeUInt16BE(rcode === 0 ? 1 : 0, 6)
header.writeUInt16BE(0, 8)
header.writeUInt16BE(0, 10)
@@ -47,18 +68,19 @@ function dnsResponse(message, parsed, rcode) {
const record = Buffer.from([
0xc0,
0x0c,
- 0x00,
- 0x01,
- 0x00,
- 0x01,
- 0x00,
- 0x00,
- 0x01,
- 0x2c,
- 0x00,
- 0x04,
- ...answerOctets,
+ 0,
+ 1,
+ 0,
+ 1,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 4,
+ ...answerIp.split('.').map(Number),
])
+ record.writeUInt32BE(ttlSeconds, 6)
return Buffer.concat([header, questionBytes, record])
}
@@ -69,126 +91,414 @@ function tcpFrame(message) {
return frame
}
-function send(entry, selectedMode) {
- const response = dnsResponse(entry.message, entry.parsed, selectedMode === 'ok' ? 0 : 2)
- if (selectedMode === 'ok') counts.ok++
- else counts.servfail++
- if (entry.transport === 'udp') {
- udp.send(response, entry.rinfo.port, entry.rinfo.address)
- } else if (!entry.socket.destroyed) {
- entry.socket.write(tcpFrame(response))
- }
- if (selectedMode === 'ok') {
- console.log(entry.transport === 'udp' ? 'ok DNS A fixture via UDP' : 'ok DNS A fixture via TCP')
- } else {
- console.log(
- entry.transport === 'udp'
- ? 'servfail DNS A fixture via UDP'
- : 'servfail DNS A fixture via TCP'
+const emptyCounts = () => ({ ok: 0, hold: 0, servfail: 0, forwarded: 0 })
+// Explicit writes keep DNS-derived names/types away from object-property writes.
+function increment(counts, mode) {
+ if (mode === 'ok') counts.ok++
+ else if (mode === 'hold') counts.hold++
+ else if (mode === 'servfail') counts.servfail++
+ else if (mode === 'forwarded') counts.forwarded++
+ else throw new Error('Invalid observation mode')
+}
+
+/** A real UDP/TCP DNS fixture; ephemeral loopback ports are used only by tests. */
+function createDnsProxy({
+ targets,
+ upstream,
+ ttlSeconds = 300,
+ upstreamPort = 53,
+ listenHost = '0.0.0.0',
+ dnsPort = 8053,
+ controlPort = 8090,
+}) {
+ if (
+ !Array.isArray(targets) ||
+ targets.length === 0 ||
+ targets.length > 3 ||
+ !net.isIPv4(upstream) ||
+ !Number.isInteger(ttlSeconds) ||
+ ttlSeconds < 1 ||
+ ttlSeconds > 300 ||
+ ![upstreamPort, dnsPort, controlPort].every(
+ port => Number.isInteger(port) && port >= 0 && port <= 65535
+ ) ||
+ upstreamPort === 0
+ ) {
+ throw new Error(
+ 'Explicit targets, IPv4 upstream, valid ports, and DNS TTL from 1 to 300 are required'
)
}
-}
+ const lanes = new Map()
+ const byName = new Map()
+ for (const target of targets) {
+ if (
+ !target ||
+ !['ui', 'worker', 'canary'].includes(target.lane) ||
+ !validHostname(target.fqdn) ||
+ typeof target.ip !== 'string' ||
+ !net.isIPv4(target.ip) ||
+ lanes.has(target.lane) ||
+ byName.has(target.fqdn) ||
+ Object.keys(target).some(key => !['lane', 'fqdn', 'ip'].includes(key))
+ ) {
+ throw new Error(
+ 'DNS targets require distinct UI/worker/canary lanes, distinct lowercase FQDNs, and IPv4 answers'
+ )
+ }
+ const state = {
+ lane: target.lane,
+ fqdn: target.fqdn,
+ ip: target.ip,
+ mode: 'ok',
+ revision: 0,
+ counts: emptyCounts(),
+ unique: emptyCounts(),
+ received: { ok: 0, hold: 0, servfail: 0 },
+ receivedIdentities: new Map(),
+ held: [],
+ observations: new Map(),
+ }
+ lanes.set(target.lane, state)
+ byName.set(target.fqdn, state)
+ }
-function forwardUdp(message, rinfo) {
- counts.forwarded++
- const relay = dgram.createSocket('udp4')
- const timer = setTimeout(() => relay.close(), 5000)
- relay.once('message', response => {
- clearTimeout(timer)
- udp.send(response, rinfo.port, rinfo.address)
- relay.close()
- })
- relay.once('error', () => {
- clearTimeout(timer)
- relay.close()
- })
- relay.send(message, 53, upstream)
-}
+ const counts = emptyCounts()
+ const udp = dgram.createSocket('udp4')
+ const connections = new Set()
+ const relays = new Set()
+ let fault = null
+ let started = false
+ let closing = false
-function forwardTcp(message, socket) {
- counts.forwarded++
- const relay = net.createConnection({ host: upstream, port: 53 }, () =>
- relay.write(tcpFrame(message))
- )
- let pending = Buffer.alloc(0)
- relay.setTimeout(5000, () => relay.destroy())
- relay.on('data', chunk => {
- pending = Buffer.concat([pending, chunk])
- if (pending.length < 2) return
- const length = pending.readUInt16BE(0)
- if (pending.length < length + 2) return
- if (!socket.destroyed) socket.write(pending.subarray(0, length + 2))
- relay.destroy()
- })
- relay.once('error', () => relay.destroy())
-}
+ const queryIdentity = (state, entry) =>
+ `${state.revision}:${entry.client}:${entry.parsed.id}:${entry.parsed.name}:${entry.parsed.queryClass}`
-function handle(message, entry) {
- const parsed = question(message)
- if (parsed && parsed.name === target && parsed.type === 1) {
- entry.message = message
- entry.parsed = parsed
- if (mode === 'hold') {
- counts.hold++
- held.push(entry)
- console.log(
- entry.transport === 'udp' ? 'hold DNS A fixture via UDP' : 'hold DNS A fixture via TCP'
- )
- } else {
- send(entry, mode)
+ // Receipt evidence concerns new IN/A requests in the current mode. Sending
+ // an old held response (possibly after its client timed out) never enters
+ // this path. The keys have only three modes and two server-owned transports;
+ // each bucket has the same identity bound as the existing observation sets.
+ function receive(state, entry) {
+ const key = `${state.mode}:${entry.transport}`
+ let identities = state.receivedIdentities.get(key)
+ if (!identities) {
+ identities = new Set()
+ state.receivedIdentities.set(key, identities)
}
- return
+ const identity = queryIdentity(state, entry)
+ if (identities.has(identity)) return
+ if (identities.size >= MAX_QUERY_IDENTITIES) {
+ fault = 'RECEIVED_QUERY_IDENTITY_LIMIT'
+ return
+ }
+ identities.add(identity)
+ increment(state.received, state.mode)
}
- if (entry.transport === 'udp') forwardUdp(message, entry.rinfo)
- else forwardTcp(message, entry.socket)
-}
-function setMode(nextMode) {
- mode = nextMode
- if (mode !== 'hold') {
- while (held.length > 0) send(held.shift(), mode)
+ function observe(state, entry, mode) {
+ increment(counts, mode)
+ if (!state) return
+ increment(state.counts, mode)
+ const key = `${entry.parsed.type}:${entry.transport}:${mode}`
+ let observation = state.observations.get(key)
+ if (!observation) {
+ if (state.observations.size >= MAX_OBSERVATIONS) {
+ fault = 'OBSERVATION_LIMIT'
+ return
+ }
+ observation = {
+ type: entry.parsed.type,
+ transport: entry.transport,
+ mode,
+ count: 0,
+ seen: new Set(),
+ }
+ state.observations.set(key, observation)
+ }
+ observation.count++
+ const identity = queryIdentity(state, entry)
+ if (!observation.seen.has(identity)) {
+ if (observation.seen.size >= MAX_QUERY_IDENTITIES) {
+ fault = 'QUERY_IDENTITY_LIMIT'
+ return
+ }
+ observation.seen.add(identity)
+ increment(state.unique, mode)
+ }
}
-}
-let listeners = 0
-function listening() {
- listeners++
- if (listeners === 2) console.log('wrc egress DNS proxy ready')
-}
+ function send(entry, selectedMode) {
+ if (entry.transport === 'tcp' && entry.socket.destroyed) return
+ const state = entry.state
+ const response = dnsResponse(
+ entry.message,
+ entry.parsed,
+ selectedMode === 'ok' ? 0 : 2,
+ state.ip,
+ ttlSeconds
+ )
+ observe(state, entry, selectedMode)
+ if (entry.transport === 'udp') {
+ udp.send(response, entry.rinfo.port, entry.rinfo.address, error => {
+ if (error) fault = 'DNS_SEND_FAILED'
+ })
+ } else if (!entry.socket.destroyed) {
+ entry.socket.write(tcpFrame(response))
+ }
+ }
-udp.on('message', (message, rinfo) => handle(message, { transport: 'udp', rinfo }))
-udp.bind(8053, '0.0.0.0', listening)
+ function forwardUdp(message, rinfo) {
+ if (relays.size >= MAX_RELAYS) {
+ fault = 'RELAY_LIMIT'
+ return
+ }
+ const relay = dgram.createSocket('udp4')
+ let done = false
+ const close = () => {
+ if (done) return
+ done = true
+ clearTimeout(timer)
+ relays.delete(close)
+ relay.close()
+ }
+ const timer = setTimeout(close, 5000)
+ relays.add(close)
+ relay.once('message', response => {
+ udp.send(response, rinfo.port, rinfo.address, error => {
+ if (error) fault = 'DNS_FORWARD_SEND_FAILED'
+ })
+ close()
+ })
+ relay.once('error', close)
+ relay.send(message, upstreamPort, upstream, error => {
+ if (error) close()
+ })
+ }
-const tcp = net.createServer(socket => {
- let pending = Buffer.alloc(0)
- socket.on('data', chunk => {
- pending = Buffer.concat([pending, chunk])
- while (pending.length >= 2) {
+ function forwardTcp(message, socket) {
+ if (relays.size >= MAX_RELAYS) {
+ fault = 'RELAY_LIMIT'
+ return
+ }
+ const relay = net.createConnection({ host: upstream, port: upstreamPort }, () =>
+ relay.write(tcpFrame(message))
+ )
+ const close = () => {
+ relays.delete(close)
+ relay.destroy()
+ }
+ relays.add(close)
+ let pending = Buffer.alloc(0)
+ relay.setTimeout(5000, close)
+ relay.on('data', chunk => {
+ pending = Buffer.concat([pending, chunk])
+ if (pending.length > 65537) {
+ close()
+ return
+ }
+ if (pending.length < 2) return
const length = pending.readUInt16BE(0)
if (pending.length < length + 2) return
- const message = pending.subarray(2, length + 2)
- pending = pending.subarray(length + 2)
- handle(message, { transport: 'tcp', socket })
+ if (!socket.destroyed) socket.write(pending.subarray(0, length + 2))
+ close()
+ })
+ relay.once('error', close)
+ relay.once('close', () => relays.delete(close))
+ }
+
+ function handle(message, entry) {
+ if (closing) return
+ const parsed = question(message)
+ const state = parsed && byName.get(parsed.name)
+ entry.parsed = parsed
+ if (state && parsed.type === 1 && parsed.queryClass === 1) {
+ if (message.length > 4096) {
+ fault = 'TARGET_QUERY_SIZE_LIMIT'
+ return
+ }
+ entry.message = message
+ entry.state = state
+ receive(state, entry)
+ if (state.mode === 'hold') {
+ observe(state, entry, 'hold')
+ if (state.held.length >= MAX_HELD) {
+ fault = 'HELD_QUERY_LIMIT'
+ return
+ }
+ state.held.push(entry)
+ } else send(entry, state.mode)
+ return
+ }
+ observe(state, entry, 'forwarded')
+ if (entry.transport === 'udp') forwardUdp(message, entry.rinfo)
+ else forwardTcp(message, entry.socket)
+ }
+
+ function setMode(nextMode, lane) {
+ if (!MODES.has(nextMode) || (lane !== undefined && !lanes.has(lane)))
+ throw new Error('Invalid DNS mode or lane')
+ for (const state of lane === undefined ? lanes.values() : [lanes.get(lane)]) {
+ if (state.mode !== nextMode) state.revision++
+ state.mode = nextMode
+ if (nextMode !== 'hold') {
+ const held = state.held.splice(0)
+ for (const entry of held) send(entry, nextMode)
+ }
+ }
+ }
+
+ function snapshot() {
+ const states = [...lanes.values()]
+ return {
+ mode: states.every(state => state.mode === states[0].mode) ? states[0].mode : 'mixed',
+ held: states.reduce((sum, state) => sum + state.held.length, 0),
+ counts: { ...counts },
+ fault,
+ lanes: states.map(state => ({
+ lane: state.lane,
+ fqdn: state.fqdn,
+ mode: state.mode,
+ held: state.held.length,
+ counts: { ...state.counts },
+ unique: { ...state.unique },
+ received: { ...state.received },
+ questions: [...state.observations.values()].map(value => ({
+ type: value.type,
+ transport: value.transport,
+ mode: value.mode,
+ count: value.count,
+ unique: value.seen.size,
+ })),
+ })),
}
+ }
+
+ udp.on('message', (message, rinfo) =>
+ handle(message, { transport: 'udp', rinfo, client: `${rinfo.address}:${rinfo.port}` })
+ )
+ udp.on('error', () => {
+ fault = 'DNS_SOCKET_FAILED'
+ })
+ const tcp = net.createServer(socket => {
+ connections.add(socket)
+ socket.once('close', () => connections.delete(socket))
+ socket.once('error', () => socket.destroy())
+ socket.setTimeout(30000, () => socket.destroy())
+ let pending = Buffer.alloc(0)
+ socket.on('data', chunk => {
+ pending = Buffer.concat([pending, chunk])
+ if (pending.length > 65537) {
+ socket.destroy()
+ return
+ }
+ while (pending.length >= 2) {
+ const length = pending.readUInt16BE(0)
+ if (pending.length < length + 2) return
+ const message = pending.subarray(2, length + 2)
+ pending = pending.subarray(length + 2)
+ handle(message, {
+ transport: 'tcp',
+ socket,
+ client: `${socket.remoteAddress}:${socket.remotePort}`,
+ })
+ }
+ })
})
-})
-tcp.listen(8053, '0.0.0.0', listening)
-
-http
- .createServer((request, response) => {
- const nextMode = request.url?.match(/^\/mode\/(ok|hold|servfail)$/)?.[1]
- if (request.method === 'POST' && nextMode) {
- setMode(nextMode)
+ tcp.maxConnections = 64
+ const control = http.createServer((request, response) => {
+ const globalMode = request.url?.match(/^\/mode\/(ok|hold|servfail)$/)?.[1]
+ const laneMode = request.url?.match(/^\/mode\/(ui|worker|canary)\/(ok|hold|servfail)$/)
+ if (request.method === 'POST' && (globalMode || laneMode)) {
+ const lane = laneMode?.[1]
+ if (lane !== undefined && !lanes.has(lane)) {
+ response.writeHead(404).end()
+ return
+ }
+ setMode(globalMode ?? laneMode[2], lane)
response.writeHead(200, { 'content-type': 'application/json' })
- response.end(JSON.stringify({ mode, held: held.length, counts }))
+ response.end(JSON.stringify(snapshot()))
return
}
if (request.method === 'GET' && request.url === '/state') {
response.writeHead(200, { 'content-type': 'application/json' })
- response.end(JSON.stringify({ mode, held: held.length, counts }))
+ response.end(JSON.stringify(snapshot()))
return
}
- response.writeHead(404)
- response.end()
+ response.writeHead(404).end()
})
- .listen(8090, '127.0.0.1')
+ control.requestTimeout = 5000
+ control.headersTimeout = 5000
+
+ const listen = (server, action) =>
+ new Promise((resolve, reject) => {
+ const failed = error => {
+ server.removeListener('listening', ready)
+ reject(error)
+ }
+ const ready = () => {
+ server.removeListener('error', failed)
+ resolve()
+ }
+ server.once('error', failed)
+ server.once('listening', ready)
+ action()
+ })
+ async function close() {
+ if (closing) return
+ closing = true
+ for (const state of lanes.values()) state.held.length = 0
+ for (const relayClose of [...relays]) relayClose()
+ for (const socket of connections) socket.destroy()
+ control.closeAllConnections()
+ await Promise.all(
+ [tcp, control].map(server => new Promise(resolve => server.close(() => resolve())))
+ )
+ await new Promise(resolve => {
+ try {
+ udp.close(resolve)
+ } catch {
+ resolve()
+ }
+ })
+ }
+ async function start() {
+ if (started || closing) throw new Error('DNS fixture cannot be started twice')
+ started = true
+ try {
+ await listen(udp, () => udp.bind(dnsPort, listenHost))
+ await listen(tcp, () => tcp.listen(udp.address().port, listenHost))
+ await listen(control, () => control.listen(controlPort, '127.0.0.1'))
+ return { dnsPort: udp.address().port, controlPort: control.address().port }
+ } catch (error) {
+ await close()
+ throw error
+ }
+ }
+ return { start, close, snapshot, setMode }
+}
+
+module.exports = { createDnsProxy, readConfig, question, dnsResponse, tcpFrame }
+if (require.main === module) {
+ let proxy
+ try {
+ proxy = createDnsProxy(readConfig(process.env))
+ } catch (error) {
+ process.stderr.write(`${error.message}\n`)
+ process.exitCode = 1
+ }
+ if (proxy) {
+ for (const signal of ['SIGINT', 'SIGTERM']) {
+ process.once(signal, () => {
+ void proxy.close().then(() => {
+ process.exitCode = signal === 'SIGINT' ? 130 : 143
+ })
+ })
+ }
+ proxy
+ .start()
+ .then(() => process.stdout.write('wrc egress DNS proxy ready\n'))
+ .catch(() => {
+ process.stderr.write('DNS fixture listeners could not start\n')
+ process.exitCode = 1
+ })
+ }
+}
diff --git a/workflow-recipes/package-lock.json b/workflow-recipes/package-lock.json
index 543d4a0a6..a099d73c4 100644
--- a/workflow-recipes/package-lock.json
+++ b/workflow-recipes/package-lock.json
@@ -1,12 +1,12 @@
{
"name": "workload-recipes-controller",
- "version": "0.1.344",
+ "version": "0.1.345",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "workload-recipes-controller",
- "version": "0.1.344",
+ "version": "0.1.345",
"license": "SEE LICENSE IN LICENSE",
"dependencies": {
"@aws-sdk/client-s3": "^3.1009.0",
diff --git a/workflow-recipes/package.json b/workflow-recipes/package.json
index e2b3242cc..474d7ac77 100644
--- a/workflow-recipes/package.json
+++ b/workflow-recipes/package.json
@@ -1,6 +1,6 @@
{
"name": "workload-recipes-controller",
- "version": "0.1.344",
+ "version": "0.1.345",
"description": "Workload Recipes Controller (WRC) - WorkloadRecipe lifecycle management and MCP server interface",
"main": "dist/main.js",
"engines": {
diff --git a/workflow-recipes/src/k8sClient.test.ts b/workflow-recipes/src/k8sClient.test.ts
index 6daab5089..5a3e71402 100644
--- a/workflow-recipes/src/k8sClient.test.ts
+++ b/workflow-recipes/src/k8sClient.test.ts
@@ -16,6 +16,7 @@ import {
workflowNeedsWorkloadStatusRefresh,
workflowRecipeFromWatchObject,
} from './k8sClient'
+import { captureLogger } from './reconciler/__tests__/captureLogger'
import type { WorkflowRecipeCRD } from './types'
function makeWorkflowRecipe(overrides: Partial<WorkflowRecipeCRD> = {}): WorkflowRecipeCRD {
@@ -75,6 +76,74 @@ describe('workflowRecipeFromWatchObject', () => {
})
})
+describe('persisted transport external-egress admission', () => {
+ const pending = {
+ type: 'TransportExternalEgressReady',
+ status: 'False' as const,
+ reason: 'ExternalEgressPending',
+ message: 'Awaiting HCC admission',
+ lastTransitionTime: '2026-09-05T00:00:00.000Z',
+ }
+
+ it('publishes explicit admission clearance without treating observation as clearance', () => {
+ const recipe = makeWorkflowRecipe({
+ spec: { steps: [] },
+ status: { phase: 'active', message: 'Ready', conditions: [pending] },
+ })
+ const observed = { phase: 'active' as const, message: 'Ready', workloadStatuses: [] }
+ expect(shouldPatchRecipeStatus(recipe, observed)).toBe(false)
+ expect(shouldPatchRecipeStatus(recipe, { ...observed, transportNetworkConditions: [] })).toBe(
+ true
+ )
+ expect(
+ shouldPatchRecipeStatus(recipe, {
+ ...observed,
+ transportNetworkConditions: [
+ { ...pending, lastTransitionTime: '2026-09-06T00:00:00.000Z' },
+ ],
+ })
+ ).toBe(false)
+ })
+
+ it('restores retries only for live, pending admission rather than Pod-only degradation or terminal runs', () => {
+ const recipe = makeWorkflowRecipe({
+ spec: { steps: [] },
+ status: { phase: 'degraded', conditions: [pending] },
+ })
+ expect(workflowNeedsInfrastructureReconcile(recipe)).toBe(true)
+ expect(workflowNeedsInfrastructureReconcile({ ...recipe, status: { phase: 'degraded' } })).toBe(
+ false
+ )
+ expect(
+ workflowNeedsInfrastructureReconcile({
+ ...recipe,
+ status: { phase: 'failed', conditions: [pending] },
+ })
+ ).toBe(false)
+ expect(
+ workflowNeedsInfrastructureReconcile({
+ ...recipe,
+ metadata: { ...recipe.metadata, deletionTimestamp: '2026-09-05T00:00:00.000Z' },
+ })
+ ).toBe(false)
+ const workflow = {
+ ...recipe,
+ spec: { ...recipe.spec, steps: [{ id: 's', instruction: 'run' }] },
+ }
+ expect(workflowNeedsInfrastructureReconcile(workflow)).toBe(true)
+ expect(
+ workflowNeedsInfrastructureReconcile({
+ ...workflow,
+ status: {
+ phase: 'active',
+ conditions: [pending],
+ workflowExecution: { phase: 'completed' },
+ },
+ })
+ ).toBe(false)
+ })
+})
+
describe('rotateBrokerTokensOnce', () => {
function makeRecipe(name: string, withBackgroundAccess: boolean): WorkflowRecipeCRD {
return {
@@ -1397,6 +1466,55 @@ describe('workload status refresh loop helpers', () => {
expect(reconcile).not.toHaveBeenCalled()
})
+ it.each([
+ { from: 'failed', to: 'candidate', runs: 1 },
+ { from: 'active', to: 'active', runs: 0 },
+ { from: 'active', to: 'degraded', runs: 0 },
+ { from: 'degraded', to: 'active', runs: 0 },
+ { from: 'failed', to: 'failed', runs: 0 },
+ { from: 'candidate', to: 'candidate', runs: 0 },
+ ] as const)(
+ 'handles the supported retry transition $from -> $to without a generation bump',
+ async ({ from, to, runs }) => {
+ const cached = makeWorkloadRecipe({
+ metadata: { name: 'retry-recipe', namespace: 'sandbox-recipes', generation: 7 },
+ status: { phase: from },
+ })
+ const requested = {
+ ...cached,
+ status: { phase: to, message: 'Manual retry requested by operator' },
+ }
+ const reconcile = vi.fn().mockResolvedValue({
+ phase: 'active',
+ message: 'All workloads deployed',
+ workloadStatuses: [],
+ })
+ const watcher = Object.create(WorkflowRecipeWatcher.prototype) as {
+ recipes: Map<string, WorkflowRecipeCRD>
+ handleRecipeEvent: (type: string, recipe: WorkflowRecipeCRD) => Promise<void>
+ }
+ Object.assign(watcher, {
+ recipes: new Map([[cached.metadata.name, cached]]),
+ transientRetries: new Map(),
+ stopped: false,
+ traceReporter: null,
+ dbRunProcessor: null,
+ reconciler: {
+ reconcile,
+ isRecipeStillActive: vi.fn().mockResolvedValue(true),
+ ensureFinalizer: vi.fn().mockResolvedValue(undefined),
+ patchStatus: vi.fn().mockResolvedValue(undefined),
+ },
+ })
+ await watcher.handleRecipeEvent('MODIFIED', requested)
+ expect(reconcile).toHaveBeenCalledTimes(runs)
+ expect(watcher.recipes.get(cached.metadata.name)).toEqual(requested)
+ // Duplicate status events must not repeatedly execute the retry transition.
+ await watcher.handleRecipeEvent('MODIFIED', requested)
+ expect(reconcile).toHaveBeenCalledTimes(runs)
+ }
+ )
+
it('reconciles when Codex connection-ref changes without a generation bump', async () => {
const cached = makeWorkloadRecipe({
metadata: { name: 'active-recipe', namespace: 'sandbox-recipes', generation: 7 },
@@ -1844,7 +1962,7 @@ describe('status-only events drive DB sync (terminal + heartbeat)', () => {
const { internal, reconcile, sync } = makeInternal(cached, () =>
Promise.reject(new Error('db down'))
)
- const errSpy = vi.spyOn(console, 'error').mockImplementation(() => undefined)
+ const errSpy = captureLogger('error').mockImplementation(() => undefined)
await expect(internal.handleRecipeEvent('MODIFIED', completed)).resolves.toBeUndefined()
@@ -1952,7 +2070,7 @@ describe('infrastructure telemetry enqueue isolation', () => {
const enqueueInfrastructureTelemetry = vi.fn()
const { internal } = makeInternal(recipe, { enqueueInfrastructureTelemetry })
internal.reconciler.patchStatus.mockRejectedValue(new Error('patch failed'))
- const errSpy = vi.spyOn(console, 'error').mockImplementation(() => undefined)
+ const errSpy = captureLogger('error').mockImplementation(() => undefined)
await internal.handleRecipeEvent('ADDED', recipe)
@@ -1972,7 +2090,7 @@ describe('infrastructure telemetry enqueue isolation', () => {
throw new Error('trace queue down')
})
const { internal } = makeInternal(recipe, { enqueueInfrastructureTelemetry })
- const errSpy = vi.spyOn(console, 'error').mockImplementation(() => undefined)
+ const errSpy = captureLogger('error').mockImplementation(() => undefined)
await expect(internal.handleRecipeEvent('ADDED', recipe)).resolves.toBeUndefined()
@@ -1980,7 +2098,7 @@ describe('infrastructure telemetry enqueue isolation', () => {
expect(enqueueInfrastructureTelemetry).toHaveBeenCalled()
expect(errSpy).toHaveBeenCalledWith(
'[WR-K8s] Infrastructure telemetry enqueue failed:',
- expect.any(Error)
+ expect.objectContaining({ err: expect.any(Error) })
)
errSpy.mockRestore()
})
@@ -2184,7 +2302,7 @@ describe('reconcile failure re-schedules the transient retry chain (issue #375 P
}
it('re-arms the transient retry after a non-404 reconcile failure (keeps the loud log)', async () => {
- const errSpy = vi.spyOn(console, 'error').mockImplementation(() => undefined)
+ const errSpy = captureLogger('error').mockImplementation(() => undefined)
const recipe = makeWorkflowRecipe({ metadata: { name: 'flaky', namespace: 'sandbox-recipes' } })
const internal = makeInternal(recipe, new Error('control-api unreachable'))
try {
@@ -2192,8 +2310,8 @@ describe('reconcile failure re-schedules the transient retry chain (issue #375 P
// fail-loud preserved: the reconciliation-failed error is still logged.
expect(errSpy).toHaveBeenCalledWith(
- expect.stringContaining('Reconciliation failed for "flaky"'),
- expect.anything()
+ expect.stringContaining('Reconciliation failed'),
+ expect.objectContaining({ name: 'flaky', err: expect.any(Error) })
)
// NEW: the 5s self-heal chain is re-armed instead of dying on first error.
expect(internal.transientRetries.has('flaky')).toBe(true)
@@ -2204,7 +2322,7 @@ describe('reconcile failure re-schedules the transient retry chain (issue #375 P
})
it('does not re-arm a retry when the recipe is already gone (404)', async () => {
- const warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => undefined)
+ const warnSpy = captureLogger('warn').mockImplementation(() => undefined)
const recipe = makeWorkflowRecipe({ metadata: { name: 'gone', namespace: 'sandbox-recipes' } })
const internal = makeInternal(recipe, { code: 404 })
try {
@@ -2272,7 +2390,7 @@ describe('Plugin Workload SDK computed-vs-published state divergence log (issue
}
it('logs the old→new SDK state and patchEmitted when the computed state diverges', async () => {
- const logSpy = vi.spyOn(console, 'log').mockImplementation(() => undefined)
+ const logSpy = captureLogger('info').mockImplementation(() => undefined)
const recipe = sdkRecipe('awaiting_policy')
const internal = makeInternal(recipe, {
phase: 'active',
@@ -2291,9 +2409,13 @@ describe('Plugin Workload SDK computed-vs-published state divergence log (issue
try {
await internal.handleRecipeEvent('ADDED', recipe)
expect(logSpy).toHaveBeenCalledWith(
- expect.stringMatching(
- /Plugin Workload SDK state.*sdk-recipe.*awaiting_policy -> validated.*patchEmitted=true/
- )
+ expect.stringContaining('Plugin Workload SDK state'),
+ expect.objectContaining({
+ name: 'sdk-recipe',
+ value2: 'awaiting_policy',
+ projectedSdkState: 'validated',
+ willPatch: true,
+ })
)
} finally {
logSpy.mockRestore()
@@ -2301,7 +2423,7 @@ describe('Plugin Workload SDK computed-vs-published state divergence log (issue
})
it('does not log a transition when the computed SDK state is unchanged', async () => {
- const logSpy = vi.spyOn(console, 'log').mockImplementation(() => undefined)
+ const logSpy = captureLogger('info').mockImplementation(() => undefined)
const recipe = sdkRecipe('validated')
const internal = makeInternal(recipe, {
phase: 'active',
@@ -2383,7 +2505,7 @@ describe('grant-update NOTIFY dispatch (issue #375 P3 wiring, H4/H5)', () => {
})
it('discards and warns on a namespace mismatch — no enqueue (H4)', () => {
- const warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => undefined)
+ const warnSpy = captureLogger('warn').mockImplementation(() => undefined)
const recipe = makeWorkflowRecipe({
metadata: { name: 'sdk-recipe', namespace: 'sandbox-recipes' },
})
@@ -2392,7 +2514,8 @@ describe('grant-update NOTIFY dispatch (issue #375 P3 wiring, H4/H5)', () => {
internal.handleGrantUpdateNotification('other-namespace', 'sdk-recipe')
expect(internal.eventQueue.enqueue).not.toHaveBeenCalled()
expect(warnSpy).toHaveBeenCalledWith(
- expect.stringContaining('Discarding grant-update NOTIFY for "other-namespace/sdk-recipe"')
+ expect.stringContaining('Discarding grant-update NOTIFY'),
+ expect.objectContaining({ recipeNamespace: 'other-namespace', recipeName: 'sdk-recipe' })
)
} finally {
warnSpy.mockRestore()
@@ -2400,12 +2523,15 @@ describe('grant-update NOTIFY dispatch (issue #375 P3 wiring, H4/H5)', () => {
})
it('discards and warns on an unknown recipe — no enqueue (H4)', () => {
- const warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => undefined)
+ const warnSpy = captureLogger('warn').mockImplementation(() => undefined)
const { internal } = makeInternal([])
try {
internal.handleGrantUpdateNotification('sandbox-recipes', 'nope')
expect(internal.eventQueue.enqueue).not.toHaveBeenCalled()
- expect(warnSpy).toHaveBeenCalledWith(expect.stringContaining('unknown recipe'))
+ expect(warnSpy).toHaveBeenCalledWith(
+ expect.stringContaining('Discarding grant-update NOTIFY'),
+ expect.objectContaining({ value3: 'unknown recipe' })
+ )
} finally {
warnSpy.mockRestore()
}
diff --git a/workflow-recipes/src/k8sClient.ts b/workflow-recipes/src/k8sClient.ts
index 7d1715f70..a58690e47 100644
--- a/workflow-recipes/src/k8sClient.ts
+++ b/workflow-recipes/src/k8sClient.ts
@@ -21,6 +21,7 @@ import {
createGovernedTraceReporter,
} from './governedTraceReporter'
import { K8sSecretWatchLoop } from './k8sSecretWatchLoop'
+import { createLogger } from './observability/logger'
import { CRD_GROUP, CRD_VERSION, WORKFLOWRECIPE_PLURAL } from './reconciler/crdConstants'
import {
type ChildRecipeCreator,
@@ -44,7 +45,9 @@ import { SecretWatcher } from './reconciler/secretWatcher'
import {
type ReconcileResult,
TRANSIENT_REQUEUE_BASE_MS,
+ TRANSPORT_NETWORK_CONDITION_TYPE,
WorkflowRecipeReconciler,
+ hasPendingTransportNetworkReadiness,
} from './reconciler/workflowRecipeReconciler'
import {
type StatusCondition,
@@ -148,6 +151,14 @@ export function shouldPatchRecipeStatus(
// cleared. That still needs one status patch even though phase is unchanged.
if (result.clearWorkflowExecution && recipe.status?.workflowExecution) return true
+ if (
+ result.transportNetworkConditions !== undefined &&
+ ownedConditionsChanged(recipe.status?.conditions, result.transportNetworkConditions, [
+ TRANSPORT_NETWORK_CONDITION_TYPE,
+ ])
+ )
+ return true
+
return false
}
@@ -307,7 +318,10 @@ function enqueueInfrastructureTelemetryBestEffort(
try {
traceReporter.enqueueInfrastructureTelemetry(projection)
} catch (error) {
- console.error('[WR-K8s] Infrastructure telemetry enqueue failed:', error)
+ createLogger('wrc', 'workflow-recipes').error(
+ '[WR-K8s] Infrastructure telemetry enqueue failed:',
+ { err: error }
+ )
}
}
@@ -405,6 +419,15 @@ export function workflowNeedsRuntimeCredentialRefresh(recipe: WorkflowRecipeCRD)
export function workflowNeedsInfrastructureReconcile(recipe: WorkflowRecipeCRD): boolean {
const isWorkflow = (recipe.spec.steps ?? []).length > 0
if (recipe.metadata.deletionTimestamp) return false
+ const workflowPhase = recipe.status?.workflowExecution?.phase
+ // Restore the retry after a process restart, including stepless stdio recipes
+ // whose egress is exclusively HCC-owned. Pod readiness is not admission proof.
+ if (
+ hasPendingTransportNetworkReadiness(recipe) &&
+ !['failed', 'deprecated', 'rollback-failed'].includes(recipe.status?.phase ?? '') &&
+ !['completed', 'failed', 'cancelled'].includes(workflowPhase ?? '')
+ )
+ return true
// SDK-only recipes are deliberately stepless, but their eager mcp-host and
// bootstrap proof still need the full reconciler lane. The generic workload
@@ -414,7 +437,6 @@ export function workflowNeedsInfrastructureReconcile(recipe: WorkflowRecipeCRD):
if (!isWorkflow && recipe.spec.pluginWorkloadSdk) return true
if (!isWorkflow) return false
- const workflowPhase = recipe.status?.workflowExecution?.phase
if (workflowPhase === 'initializing' || workflowPhase === 'recovering') return true
// Plugin Workload SDK eager path: the recipe sits in phase=active with no run
@@ -638,7 +660,10 @@ export class WorkflowRecipeWatcher implements WorkflowRecipeProvider {
private externalEgressRefreshTimer: ReturnType<typeof setTimeout> | null = null
private externalEgressRefreshRunning = false
private eventQueue = new RecipeEventQueue((key, error) => {
- console.error(`[WR-K8s] Unhandled event failure for "${key}":`, error)
+ createLogger('wrc', 'workflow-recipes').error('Unhandled recipe event failure', {
+ key,
+ err: error,
+ })
})
private secretReverseIndex = new SecretReverseIndex()
private secretWatchLoops: K8sSecretWatchLoop[] = []
@@ -723,10 +748,13 @@ export class WorkflowRecipeWatcher implements WorkflowRecipeProvider {
// independent env vars; if they ever diverge, 100% of NOTIFYs would be
// dropped and the feature would silently degrade to 30s polling. Log names
// only — no secrets.
- console.warn(
- `[WR-K8s] Discarding grant-update NOTIFY for "${recipeNamespace}/${recipeName}": ` +
- (known ? `namespace mismatch (watching "${known.metadata.namespace}")` : 'unknown recipe')
- )
+ createLogger('wrc', recipeName).warn('Discarding grant-update NOTIFY', {
+ recipeNamespace,
+ recipeName,
+ value3: known
+ ? `namespace mismatch (watching "${known.metadata.namespace}")`
+ : 'unknown recipe',
+ })
return
}
void this.eventQueue.enqueue(recipeName, () => {
@@ -787,7 +815,7 @@ export class WorkflowRecipeWatcher implements WorkflowRecipeProvider {
}
async start(): Promise<void> {
- console.log('[WR-K8s] Starting WorkflowRecipe watcher')
+ createLogger('wrc', 'workflow-recipes').info('[WR-K8s] Starting WorkflowRecipe watcher')
// Initialize workflow subsystem by reading the signing key from control-plane.
// Non-fatal: if the Secret doesn't exist, workflow recipes will gracefully fail
@@ -821,7 +849,10 @@ export class WorkflowRecipeWatcher implements WorkflowRecipeProvider {
this.reconciler.ensureOAuthBrokerTokenSecret(recipe)
),
(recipe, err) => {
- console.error(`[WR-K8s] Broker-token refresh failed for "${recipe.metadata.name}":`, err)
+ createLogger('wrc', recipe.metadata.name).error('Broker-token refresh failed', {
+ name: recipe.metadata.name,
+ err,
+ })
}
)
}
@@ -895,14 +926,16 @@ export class WorkflowRecipeWatcher implements WorkflowRecipeProvider {
const privateKeyB64 = secret.data?.['private.pem']
const publicKeyB64 = secret.data?.['public.pem']
if (!privateKeyB64) {
- console.warn(
- `[WR-K8s] Secret "${SIGNING_KEY_SECRET}" exists but missing private.pem key — workflow disabled`
+ createLogger('wrc', 'workflow-recipes').warn(
+ 'Signing Secret lacks private.pem; workflow disabled',
+ { SIGNING_KEY_SECRET }
)
return
}
if (!publicKeyB64) {
- console.warn(
- `[WR-K8s] Secret "${SIGNING_KEY_SECRET}" exists but missing public.pem key — JWT verification disabled`
+ createLogger('wrc', 'workflow-recipes').warn(
+ 'Signing Secret lacks public.pem; JWT verification disabled',
+ { SIGNING_KEY_SECRET }
)
}
@@ -911,15 +944,21 @@ export class WorkflowRecipeWatcher implements WorkflowRecipeProvider {
? Buffer.from(publicKeyB64, 'base64').toString('utf-8')
: undefined
await this.reconciler.initializeWorkflow(privateKeyPem, publicKeyPem)
- console.log('[WR-K8s] Workflow subsystem initialized from signing key')
+ createLogger('wrc', 'workflow-recipes').info(
+ '[WR-K8s] Workflow subsystem initialized from signing key'
+ )
} catch (error: unknown) {
const code = (error as { response?: { statusCode?: number } })?.response?.statusCode
if (code === 404 || code === 403) {
- console.warn(
- `[WR-K8s] Signing key Secret not found or forbidden (${code}) — workflow disabled`
+ createLogger('wrc', 'workflow-recipes').warn(
+ 'Signing Secret not found or forbidden; workflow disabled',
+ { code }
)
} else {
- console.warn('[WR-K8s] Failed to read signing key — workflow disabled:', error)
+ createLogger('wrc', 'workflow-recipes').warn(
+ '[WR-K8s] Failed to read signing key — workflow disabled:',
+ { err: error }
+ )
}
}
}
@@ -933,7 +972,10 @@ export class WorkflowRecipeWatcher implements WorkflowRecipeProvider {
const watchCallback = (type: string, apiObj: WorkflowRecipeWatchObject) => {
const recipe = workflowRecipeFromWatchObject(apiObj, this.config.sandboxNamespace)
- console.log(`[WR-K8s] Watch event: ${type} for "${recipe.metadata.name}"`)
+ createLogger('wrc', recipe.metadata.name).info('Received recipe watch event', {
+ type,
+ name: recipe.metadata.name,
+ })
this.eventQueue.enqueue(recipe.metadata.name, () => this.handleRecipeEvent(type, recipe))
}
@@ -941,9 +983,9 @@ export class WorkflowRecipeWatcher implements WorkflowRecipeProvider {
const doneCallback = (err: Error | null) => {
if (this.stopped) return
if (err) {
- console.error('[WR-K8s] Watch error:', err)
+ createLogger('wrc', 'workflow-recipes').error('[WR-K8s] Watch error:', { err })
}
- console.log('[WR-K8s] Watch ended, restarting...')
+ createLogger('wrc', 'workflow-recipes').info('[WR-K8s] Watch ended, restarting...')
setTimeout(() => this.startWatch(), err ? 5000 : 1000)
}
@@ -959,17 +1001,23 @@ export class WorkflowRecipeWatcher implements WorkflowRecipeProvider {
let dbSyncAttempted = false
if (recipe.metadata.deletionTimestamp) {
// Resource is being deleted — cleanup cross-namespace resources, then remove finalizer.
- console.log(`[WR-K8s] Finalizer cleanup for "${recipe.metadata.name}"`)
+ createLogger('wrc', recipe.metadata.name).info('Cleaning up recipe finalizer', {
+ name: recipe.metadata.name,
+ })
try {
await this.reconciler.reconcileDelete(recipe)
await this.reconciler.removeFinalizer(recipe)
} catch (error) {
if (getErrorCode(error) === 404) {
- console.warn(
- `[WR-K8s] Finalizer cleanup skipped for "${recipe.metadata.name}"; recipe already gone`
+ createLogger('wrc', recipe.metadata.name).warn(
+ 'Skipping finalizer cleanup because the recipe is gone',
+ { name: recipe.metadata.name }
)
} else {
- console.error(`[WR-K8s] Finalizer cleanup failed for "${recipe.metadata.name}":`, error)
+ createLogger('wrc', recipe.metadata.name).error('Finalizer cleanup failed', {
+ name: recipe.metadata.name,
+ err: error,
+ })
}
}
this.recipes.delete(recipe.metadata.name)
@@ -997,17 +1045,18 @@ export class WorkflowRecipeWatcher implements WorkflowRecipeProvider {
try {
await this.dbRunProcessor.syncFromRecipeExecution(recipe)
} catch (error) {
- console.error(
- `[WR-K8s] DB sync failed for status-only event "${recipe.metadata.name}":`,
- error
+ createLogger('wrc', recipe.metadata.name).error(
+ 'Database sync failed for a status-only event',
+ { name: recipe.metadata.name, err: error }
)
} finally {
dbSyncAttempted = true
}
}
if (needsTerminalRunTeardown) {
- console.log(
- `[WR-K8s] Terminal run status-only event for "${recipe.metadata.name}" — forcing reconcile for compute teardown`
+ createLogger('wrc', recipe.metadata.name).info(
+ 'Forcing reconciliation of terminal run status for compute teardown',
+ { name: recipe.metadata.name }
)
} else {
return
@@ -1018,8 +1067,9 @@ export class WorkflowRecipeWatcher implements WorkflowRecipeProvider {
this.recipes.set(recipe.metadata.name, recipe)
try {
if (!(await this.reconciler.isRecipeStillActive(recipe))) {
- console.warn(
- `[WR-K8s] Skipping stale event for "${recipe.metadata.name}"; recipe is gone or deleting`
+ createLogger('wrc', recipe.metadata.name).warn(
+ 'Skipping stale event because the recipe is gone or deleting',
+ { name: recipe.metadata.name }
)
this.recipes.delete(recipe.metadata.name)
return
@@ -1029,8 +1079,9 @@ export class WorkflowRecipeWatcher implements WorkflowRecipeProvider {
}
await this.reconciler.ensureFinalizer(recipe)
if (!(await this.reconciler.isRecipeStillActive(recipe))) {
- console.warn(
- `[WR-K8s] Skipping reconcile for "${recipe.metadata.name}"; recipe started deleting after finalizer check`
+ createLogger('wrc', recipe.metadata.name).warn(
+ 'Skipping reconciliation because deletion began after the finalizer check',
+ { name: recipe.metadata.name }
)
this.recipes.delete(recipe.metadata.name)
return
@@ -1052,10 +1103,12 @@ export class WorkflowRecipeWatcher implements WorkflowRecipeProvider {
// capability); its clearing PATCH is still emitted via
// `pluginWorkloadSdkProjectionChanged`, so the publish is covered.
if (projectedSdkState !== undefined && projectedSdkState !== persistedSdkState) {
- console.log(
- `[WR-K8s] Plugin Workload SDK state for "${recipe.metadata.name}": ` +
- `${persistedSdkState ?? 'none'} -> ${projectedSdkState} (patchEmitted=${willPatch})`
- )
+ createLogger('wrc', recipe.metadata.name).info('Plugin Workload SDK state changed', {
+ name: recipe.metadata.name,
+ value2: persistedSdkState ?? 'none',
+ projectedSdkState,
+ willPatch,
+ })
}
if (willPatch) {
await this.reconciler.patchStatus(recipe, result)
@@ -1078,13 +1131,17 @@ export class WorkflowRecipeWatcher implements WorkflowRecipeProvider {
}
} catch (error) {
if (getErrorCode(error) === 404) {
- console.warn(
- `[WR-K8s] Reconciliation skipped for "${recipe.metadata.name}"; recipe already gone`
+ createLogger('wrc', recipe.metadata.name).warn(
+ 'Reconciliation skipped because the recipe is gone',
+ { name: recipe.metadata.name }
)
this.recipes.delete(recipe.metadata.name)
return
}
- console.error(`[WR-K8s] Reconciliation failed for "${recipe.metadata.name}":`, error)
+ createLogger('wrc', recipe.metadata.name).error('Reconciliation failed', {
+ name: recipe.metadata.name,
+ err: error,
+ })
enqueueInfrastructureTelemetryBestEffort(
this.traceReporter,
controllerErrorTelemetryProjection(recipe, error)
@@ -1109,6 +1166,10 @@ export class WorkflowRecipeWatcher implements WorkflowRecipeProvider {
private isStatusOnlyModifiedEvent(recipe: WorkflowRecipeCRD): boolean {
const cached = this.recipes.get(recipe.metadata.name)
+ // POST /admin/recipes/:name/retry requests the canonical failed -> candidate
+ // transition through /status, without changing spec/generation. Run that
+ // transition once; subsequent candidate/status updates remain coalesced.
+ if (cached?.status?.phase === 'failed' && recipe.status?.phase === 'candidate') return false
const cachedGeneration = cached?.metadata.generation
const nextGeneration = recipe.metadata.generation
if (
@@ -1165,7 +1226,12 @@ export class WorkflowRecipeWatcher implements WorkflowRecipeProvider {
return
}
const label = fixedInterval ? 'progress' : 'transient'
- console.log(`[WR-K8s] ${label} retry for "${name}" (attempt ${attempts + 1}, ${delay}ms)`)
+ createLogger('wrc', 'workflow-recipes').info('Retrying recipe reconciliation', {
+ label,
+ name,
+ value3: attempts + 1,
+ delay,
+ })
void this.eventQueue.enqueue(name, () =>
this.handleRecipeEvent('MODIFIED', cached, { forceReconcile: true })
)
@@ -1285,7 +1351,10 @@ export class WorkflowRecipeWatcher implements WorkflowRecipeProvider {
)
enqueued++
} catch (error) {
- console.error(`[WR-K8s] External egress refresh failed for "${name}":`, error)
+ createLogger('wrc', 'workflow-recipes').error('External egress refresh failed', {
+ name,
+ err: error,
+ })
}
})
@@ -1296,8 +1365,9 @@ export class WorkflowRecipeWatcher implements WorkflowRecipeProvider {
// where the set is decided and written instead — `logResolvedEgressSet` in
// workflowRecipeReconciler.ts, one entry per FQDN, only when the set changed
// and only once the policy has landed (#299). Secrets stay out of both.
- console.log(
- `[WR-K8s] External egress refresh: re-enqueued ${enqueued}/${selected.length} recipe(s) in ${Date.now() - started}ms`
+ createLogger('wrc', 'workflow-recipes').info(
+ 'Re-enqueued recipes for external egress refresh',
+ { enqueued, length: selected.length, value3: Date.now() - started }
)
}
@@ -1319,7 +1389,10 @@ export class WorkflowRecipeWatcher implements WorkflowRecipeProvider {
this.refreshWorkloadStatusOnly(latest, { ownershipOnly: !needsStatusRefresh })
)
} catch (error) {
- console.error(`[WR-K8s] Workload status refresh failed for "${recipeName}":`, error)
+ createLogger('wrc', recipeName).error('Workload status refresh failed', {
+ recipeName,
+ err: error,
+ })
}
})
@@ -1377,9 +1450,9 @@ export class WorkflowRecipeWatcher implements WorkflowRecipeProvider {
)
}
} catch (error) {
- console.error(
- `[WR-K8s] Runtime credential refresh or infrastructure retry failed for "${recipeName}":`,
- error
+ createLogger('wrc', recipeName).error(
+ 'Runtime credential refresh or infrastructure retry failed',
+ { recipeName, err: error }
)
}
})
@@ -1404,7 +1477,7 @@ export class WorkflowRecipeWatcher implements WorkflowRecipeProvider {
this.externalEgressRefreshTimer = null
}
if (this.watchRequest) {
- console.log('[WR-K8s] Stopping watch')
+ createLogger('wrc', 'workflow-recipes').info('[WR-K8s] Stopping watch')
this.watchRequest.abort()
this.watchRequest = null
}
@@ -1431,9 +1504,14 @@ export class WorkflowRecipeWatcher implements WorkflowRecipeProvider {
if (this.traceReporter) {
const result = await this.traceReporter.stopAndDrain()
if (!result.drained) {
- console.warn('[WR-K8s] Governed trace reporter shutdown timed out', {
- dropped: result.dropped,
- })
+ createLogger('wrc', 'workflow-recipes').warn(
+ '[WR-K8s] Governed trace reporter shutdown timed out',
+ {
+ value1: {
+ dropped: result.dropped,
+ },
+ }
+ )
}
}
}
@@ -1464,7 +1542,9 @@ export class DevWorkflowRecipeProvider implements WorkflowRecipeProvider {
async start(): Promise<void> {
const raw = process.env.CLERUM_RECIPES
if (!raw) {
- console.log('[WR-Dev] No CLERUM_RECIPES env var — starting with empty recipe list')
+ createLogger('wrc', 'workflow-recipes').info(
+ '[WR-Dev] No CLERUM_RECIPES env var — starting with empty recipe list'
+ )
return
}
@@ -1472,16 +1552,22 @@ export class DevWorkflowRecipeProvider implements WorkflowRecipeProvider {
const recipes = JSON.parse(raw) as WorkflowRecipeCRD[]
for (const recipe of recipes) {
this.recipes.set(recipe.metadata.name, recipe)
- console.log(`[WR-Dev] Loaded recipe: ${recipe.metadata.name}`)
+ createLogger('wrc', recipe.metadata.name).info('Loaded development recipe', {
+ name: recipe.metadata.name,
+ })
}
- console.log(`[WR-Dev] Loaded ${this.recipes.size} recipe(s)`)
+ createLogger('wrc', 'workflow-recipes').info('Loaded development recipes', {
+ size: this.recipes.size,
+ })
} catch (error) {
- console.error('[WR-Dev] Failed to parse CLERUM_RECIPES:', error)
+ createLogger('wrc', 'workflow-recipes').error('[WR-Dev] Failed to parse CLERUM_RECIPES:', {
+ err: error,
+ })
}
}
async stop(): Promise<void> {
- console.log('[WR-Dev] Stopping dev provider')
+ createLogger('wrc', 'workflow-recipes').info('[WR-Dev] Stopping dev provider')
}
}
@@ -1495,7 +1581,7 @@ export class DevWorkflowRecipeProvider implements WorkflowRecipeProvider {
export function createWorkflowRecipeProvider(): WorkflowRecipeProvider {
const config = loadConfig()
if (config.devMode) {
- console.log('[WR-Provider] Creating dev mode provider')
+ createLogger('wrc', 'workflow-recipes').info('[WR-Provider] Creating dev mode provider')
return new DevWorkflowRecipeProvider()
}
@@ -1507,9 +1593,9 @@ export function createWorkflowRecipeProvider(): WorkflowRecipeProvider {
try {
pool = initDb(config.db)
} catch (err) {
- console.warn(
+ createLogger('wrc', 'workflow-recipes').warn(
'[WR-Provider] initDb() failed — DB run processor disabled:',
- err instanceof Error ? err.message : err
+ { err }
)
pool = null
}
@@ -1525,6 +1611,6 @@ export function createWorkflowRecipeProvider(): WorkflowRecipeProvider {
}
}
- console.log('[WR-Provider] Creating K8s watcher provider')
+ createLogger('wrc', 'workflow-recipes').info('[WR-Provider] Creating K8s watcher provider')
return new WorkflowRecipeWatcher(kc, pool)
}
diff --git a/workflow-recipes/src/observability/logger.ts b/workflow-recipes/src/observability/logger.ts
index a99c009c4..65de8c405 100644
--- a/workflow-recipes/src/observability/logger.ts
+++ b/workflow-recipes/src/observability/logger.ts
@@ -4,6 +4,7 @@
* Produces one JSON line per log entry to stdout. Cluster log collectors
* (Loki, Fluent Bit) pick up stdout automatically.
*/
+import { isNativeError } from 'node:util/types'
export interface Logger {
debug(msg: string, fields?: Record<string, unknown>): void
@@ -23,6 +24,9 @@ const LEVEL_PRIORITY: Record<LogLevel, number> = {
}
const REDACTED = '[REDACTED]'
+// Error causes can contain arbitrary response objects or recurse indefinitely.
+const MAX_ERROR_DEPTH = 6
+const MAX_ERROR_TEXT_LENGTH = 16_384
const SENSITIVE_KEY_PATTERNS = [
/authorization/i,
/cookie/i,
@@ -69,16 +73,63 @@ function isSensitiveKey(key: string): boolean {
return SENSITIVE_KEY_PATTERNS.some(pattern => pattern.test(key))
}
-function redactValue(value: unknown, key: string, seen: WeakSet<object>): unknown {
+function redactErrorText(text: string): string {
+ if (text.length > MAX_ERROR_TEXT_LENGTH) return '[Truncated]'
+ // Newly exposed native diagnostics must not reintroduce access data through
+ // messages/stack traces. URLs are removed in full, including private paths.
+ return (
+ text
+ // ApiException duplicates JSON.stringify(body/headers) on these diagnostic
+ // lines. Remove the entire sections, including fields not named sensitive,
+ // while retaining the HTTP code, ordinary message and stack frames.
+ .replace(/^(\s*(?:Body|Headers):)[^\r\n]*/gim, '$1 ' + REDACTED)
+ // Generic messages can embed quoted JSON, including escaped JSON strings.
+ // Its value may itself contain quotes/newlines, so do not guess its end:
+ // preserve the diagnostic prefix and redact the remaining fragment.
+ .replace(
+ /(\b[\w.-]*(?:authorization|cookie|token|secret|password|credential|api[_-]?key)[\w.-]*\\*["']\s*:\s*)[\s\S]*/gi,
+ '$1' + REDACTED
+ )
+ .replace(/\b[a-z][a-z\d+.-]*:\/\/[^\s"'<>]+/gi, REDACTED)
+ .replace(/\b(?:authorization|set-cookie|cookie)\s*:\s*[^\r\n]+/gi, REDACTED)
+ .replace(/\b(?:bearer|basic)\s+[^\s,;]+/gi, REDACTED)
+ .replace(
+ /(\b[\w.-]*(?:token|secret|password|credential|api[_-]?key)[\w.-]*\s*[:=]\s*)(?:"[^"]*"|'[^']*'|[^\s,;]+)/gi,
+ `$1${REDACTED}`
+ )
+ )
+}
+
+function redactValue(value: unknown, key: string, seen: WeakSet<object>, errorDepth = 0): unknown {
if (isSensitiveKey(key)) return REDACTED
- if (Array.isArray(value)) return value.map(item => redactValue(item, '', seen))
+ if (errorDepth > 0 && /^(body|headers|request|response)$/i.test(key)) return REDACTED
+ if (errorDepth > 0 && typeof value === 'string') return redactErrorText(value)
if (typeof value === 'object' && value !== null) {
if (seen.has(value)) return '[Circular]'
+ if (errorDepth >= MAX_ERROR_DEPTH) return '[Truncated]'
seen.add(value)
+ const nativeError = isNativeError(value)
+ const nextDepth = errorDepth > 0 || nativeError ? errorDepth + 1 : 0
+ if (Array.isArray(value)) return value.map(item => redactValue(item, '', seen, nextDepth))
+ let fields: Record<string, unknown> = value as Record<string, unknown>
+ if (nativeError) {
+ try {
+ fields = {
+ ...value,
+ name: value.name,
+ message: value.message,
+ stack: value.stack,
+ ...('cause' in value ? { cause: value.cause } : {}),
+ }
+ } catch {
+ // A custom Error accessor must not replace the failure being reported.
+ return { name: 'Error', message: '[Unserializable]' }
+ }
+ }
return Object.fromEntries(
- Object.entries(value as Record<string, unknown>).map(([childKey, childValue]) => [
+ Object.entries(fields).map(([childKey, childValue]) => [
childKey,
- redactValue(childValue, childKey, seen),
+ redactValue(childValue, childKey, seen, nextDepth),
])
)
}
diff --git a/workflow-recipes/src/reconciler/__tests__/captureLogger.ts b/workflow-recipes/src/reconciler/__tests__/captureLogger.ts
new file mode 100644
index 000000000..f880e1fac
--- /dev/null
+++ b/workflow-recipes/src/reconciler/__tests__/captureLogger.ts
@@ -0,0 +1,16 @@
+import { vi } from 'vitest'
+import * as logging from '../../observability/logger'
+
+/** Observe the service logging boundary without replacing other log levels. */
+export function captureLogger(level: 'info' | 'warn' | 'error') {
+ const original = logging.createLogger
+ const calls = vi.fn()
+ const factory = vi.spyOn(logging, 'createLogger').mockImplementation((...args) => ({
+ ...original(...args),
+ [level]: calls,
+ }))
+ calls.mockRestore = () => {
+ factory.mockRestore()
+ }
+ return calls
+}
diff --git a/workflow-recipes/src/reconciler/networkPolicyContraction.test.ts b/workflow-recipes/src/reconciler/networkPolicyContraction.test.ts
new file mode 100644
index 000000000..a9c3a8a90
--- /dev/null
+++ b/workflow-recipes/src/reconciler/networkPolicyContraction.test.ts
@@ -0,0 +1,127 @@
+import { describe, expect, it } from 'vitest'
+import type * as k8s from '@kubernetes/client-node'
+import { intersectNetworkPolicyRules, networkPolicySpecSignature } from './networkPolicyContraction'
+
+const peer = {
+ namespaceSelector: { matchLabels: { 'kubernetes.io/metadata.name': 'sandbox-ui' } },
+ podSelector: { matchLabels: { 'clerum.io/recipe': 'one' } },
+}
+const ingress = (ports: number[]): k8s.V1NetworkPolicyIngressRule[] => [
+ {
+ _from: [peer],
+ ports: ports.map(port => ({ port, protocol: 'TCP' })),
+ },
+]
+const policy = (rules?: k8s.V1NetworkPolicyIngressRule[]): k8s.V1NetworkPolicy => ({
+ spec: {
+ podSelector: { matchLabels: { app: 'backend' } },
+ policyTypes: ['Ingress'],
+ ...(rules ? { ingress: rules } : {}),
+ },
+})
+
+describe('WRC NetworkPolicy permission contraction', () => {
+ it('compares API-omitted empty top-level lists as equivalent', () => {
+ expect(networkPolicySpecSignature(policy())).toBe(networkPolicySpecSignature(policy([])))
+ })
+
+ it('compares grouped, split, duplicated and reordered permissions as equivalent', () => {
+ expect(networkPolicySpecSignature(policy(ingress([8080, 9090])))).toBe(
+ networkPolicySpecSignature(policy([...ingress([9090]), ...ingress([8080, 8080])]))
+ )
+ })
+
+ it('preserves the namespace-and-pod conjunction while splitting peer alternatives', () => {
+ const other = {
+ namespaceSelector: peer.namespaceSelector,
+ podSelector: { matchLabels: { 'clerum.io/recipe': 'two' } },
+ }
+ const result = intersectNetworkPolicyRules(
+ [{ _from: [peer, other], ports: [{ port: 8080 }] }],
+ ingress([8080]),
+ '_from'
+ )
+ expect(result).toEqual(ingress([8080]))
+ expect(result[0]._from).toEqual([peer])
+ expect(
+ intersectNetworkPolicyRules(
+ [
+ {
+ _from: [
+ { namespaceSelector: peer.namespaceSelector },
+ { podSelector: peer.podSelector },
+ ],
+ ports: [{ port: 8080 }],
+ },
+ ],
+ ingress([8080]),
+ '_from'
+ )
+ ).toEqual([])
+ })
+
+ it('keeps TCP defaulting while rejecting different protocols', () => {
+ expect(
+ intersectNetworkPolicyRules(
+ [{ _from: [peer], ports: [{ port: 8080 }] }],
+ ingress([8080]),
+ '_from'
+ )
+ ).toEqual(ingress([8080]))
+ expect(
+ intersectNetworkPolicyRules(
+ [{ _from: [peer], ports: [{ port: 8080, protocol: 'UDP' }] }],
+ ingress([8080]),
+ '_from'
+ )
+ ).toEqual([])
+ })
+
+ it('does not equate inner omitted peers or ports to a deny rule', () => {
+ expect(networkPolicySpecSignature(policy([{}]))).not.toBe(
+ networkPolicySpecSignature(policy([]))
+ )
+ expect(networkPolicySpecSignature(policy([{ _from: [peer] }]))).not.toBe(
+ networkPolicySpecSignature(policy(ingress([8080])))
+ )
+ expect(intersectNetworkPolicyRules([{}], ingress([8080]), '_from')).toEqual(ingress([8080]))
+ expect(intersectNetworkPolicyRules(ingress([8080]), [{}], '_from')).toEqual(ingress([8080]))
+ })
+
+ it('does not ignore an IP exception or an endPort restriction', () => {
+ const wanted = [
+ { to: [{ ipBlock: { cidr: '8.8.8.0/24' } }], ports: [{ port: 443, protocol: 'TCP' }] },
+ ]
+ expect(
+ intersectNetworkPolicyRules(
+ [{ ...wanted[0], to: [{ ipBlock: { cidr: '8.8.8.0/24', except: ['8.8.8.8/32'] } }] }],
+ wanted,
+ 'to'
+ )
+ ).toEqual([])
+ expect(
+ intersectNetworkPolicyRules(
+ [{ ...wanted[0], ports: [{ port: 443, endPort: 444, protocol: 'TCP' }] }],
+ wanted,
+ 'to'
+ )
+ ).toEqual([])
+ })
+
+ it('cannot prove an unknown rule restriction by dropping it', () => {
+ const unknown = { ...ingress([8080])[0], futureRestriction: true }
+ expect(intersectNetworkPolicyRules([unknown], ingress([8080]), '_from')).toEqual([])
+ expect(networkPolicySpecSignature(policy([unknown]))).not.toBe(
+ networkPolicySpecSignature(policy(ingress([8080])))
+ )
+ })
+
+ it('preserves selectors and effective isolation types in equivalence checks', () => {
+ expect(networkPolicySpecSignature(policy())).not.toBe(
+ networkPolicySpecSignature({ spec: { ...policy().spec!, podSelector: {} } })
+ )
+ expect(networkPolicySpecSignature(policy())).not.toBe(
+ networkPolicySpecSignature({ spec: { ...policy().spec!, policyTypes: ['Egress'] } })
+ )
+ })
+})
diff --git a/workflow-recipes/src/reconciler/networkPolicyContraction.ts b/workflow-recipes/src/reconciler/networkPolicyContraction.ts
new file mode 100644
index 000000000..3a8a37842
--- /dev/null
+++ b/workflow-recipes/src/reconciler/networkPolicyContraction.ts
@@ -0,0 +1,127 @@
+import type * as k8s from '@kubernetes/client-node'
+
+type Rule = k8s.V1NetworkPolicyEgressRule & k8s.V1NetworkPolicyIngressRule
+type PeerKey = 'to' | '_from'
+
+function canonical(value: unknown): unknown {
+ if (Array.isArray(value)) return value.map(canonical)
+ if (value && typeof value === 'object') {
+ return Object.fromEntries(
+ Object.entries(value)
+ .sort(([a], [b]) => a.localeCompare(b))
+ .map(([key, item]) => [key, canonical(item)])
+ )
+ }
+ return value
+}
+
+function signature(value: unknown): string {
+ return JSON.stringify(canonical(value))
+}
+
+function uniqueRules(rules: Rule[]): Rule[] {
+ return [...new Map(rules.map(rule => [signature(rule), rule])).entries()]
+ .sort(([a], [b]) => a.localeCompare(b))
+ .map(([, rule]) => rule)
+}
+
+/** Split only the OR lists; namespaceSelector + podSelector stay one AND peer.
+ * Empty/missing inner lists mean unrestricted, unlike empty top-level rules.
+ * Keep all other fields when comparing; an unknown restriction must not vanish.
+ */
+function splitRules(rules: Rule[] | undefined, peerKey: PeerKey): Rule[] {
+ return uniqueRules(
+ (rules ?? []).flatMap(rule => {
+ const peers = rule[peerKey]?.length ? rule[peerKey]! : [undefined]
+ const ports = rule.ports?.length ? rule.ports : [undefined]
+ return peers.flatMap(peer =>
+ ports.map(port => ({
+ ...rule,
+ [peerKey]: peer ? [peer] : undefined,
+ ports: port ? [{ ...port, protocol: port.protocol ?? 'TCP' }] : undefined,
+ }))
+ )
+ })
+ )
+}
+
+export function effectivePolicyTypes(spec: k8s.V1NetworkPolicySpec | undefined): string[] {
+ return [
+ ...(spec?.policyTypes?.length
+ ? spec.policyTypes
+ : ['Ingress', ...(spec?.egress?.length ? ['Egress'] : [])]),
+ ].sort()
+}
+
+/** Normalize Kubernetes omitempty only at the rule-list boundary. Nested empty
+ * peers/ports have different semantics and are handled by splitRules instead.
+ */
+export function networkPolicySpecSignature(policy: k8s.V1NetworkPolicy): string {
+ return signature({
+ ...policy.spec,
+ podSelector: policy.spec?.podSelector ?? {},
+ policyTypes: effectivePolicyTypes(policy.spec),
+ egress: splitRules(policy.spec?.egress, 'to'),
+ ingress: splitRules(policy.spec?.ingress, '_from'),
+ })
+}
+
+function supportedRule(rule: Rule, peerKey: PeerKey): boolean {
+ return (
+ Object.entries(rule).every(
+ ([key, value]) => value === undefined || key === peerKey || key === 'ports'
+ ) &&
+ (rule.ports ?? []).every(port =>
+ Object.entries(port).every(
+ ([key, value]) =>
+ value === undefined || key === 'port' || key === 'protocol' || key === 'endPort'
+ )
+ )
+ )
+}
+
+/** Intersection of the finite rule shapes emitted by WRC. Peers are matched as
+ * whole conjunctions; ports preserve protocol and endPort. Unrestricted live
+ * dimensions can be narrowed to current intent, never the reverse. Unrecognized
+ * fields cannot prove a retained permission and are deliberately not copied.
+ */
+export function intersectNetworkPolicyRules(
+ existing: Rule[] | undefined,
+ desired: Rule[] | undefined,
+ peerKey: PeerKey
+): Rule[] {
+ const liveRules = splitRules(existing, peerKey).filter(rule => supportedRule(rule, peerKey))
+ const desiredRules = splitRules(desired, peerKey).filter(rule => supportedRule(rule, peerKey))
+ const retained: Rule[] = []
+ for (const wanted of desiredRules) {
+ for (const live of liveRules) {
+ const wantedPeer = wanted[peerKey]?.[0]
+ const livePeer = live[peerKey]?.[0]
+ if (wantedPeer && livePeer && signature(wantedPeer) !== signature(livePeer)) continue
+ const wantedPort = wanted.ports?.[0]
+ const livePort = live.ports?.[0]
+ if (wantedPort && livePort) {
+ if (wantedPort.protocol !== livePort.protocol) continue
+ if (
+ wantedPort.port !== undefined &&
+ livePort.port !== undefined &&
+ signature(wantedPort) !== signature(livePort)
+ )
+ continue
+ // endPort is never discarded to pretend an unsupported range is all ports.
+ if (
+ (wantedPort.endPort !== undefined && wantedPort.port === undefined) ||
+ (livePort.endPort !== undefined && livePort.port === undefined)
+ )
+ continue
+ }
+ const peer = wantedPeer ?? livePeer
+ const port = wantedPort?.port !== undefined ? wantedPort : (livePort ?? wantedPort)
+ retained.push({
+ [peerKey]: peer ? [peer] : undefined,
+ ports: port ? [port] : undefined,
+ })
+ }
+ }
+ return uniqueRules(retained)
+}
diff --git a/workflow-recipes/src/reconciler/reconciler.test.ts b/workflow-recipes/src/reconciler/reconciler.test.ts
index 5891f489c..b82cb9be2 100644
--- a/workflow-recipes/src/reconciler/reconciler.test.ts
+++ b/workflow-recipes/src/reconciler/reconciler.test.ts
@@ -5,10 +5,11 @@ import { readFileSync } from 'node:fs'
import { resolve } from 'node:path'
import { EVENFIRE_REGISTRY_PULL_SECRET_NAME } from '@clerum/workflow-runtime-core'
import { loadConfig } from '../config'
-import { shouldPatchRecipeStatus } from '../k8sClient'
+import { WorkflowRecipeWatcher, shouldPatchRecipeStatus } from '../k8sClient'
import { CronJobDef, WorkflowRecipeCRD, WorkflowRecipeGfsIntentSpec, WorkloadDef } from '../types'
import { INHERITED_PARENT_RESOURCES_ANNOTATION } from '../workflow/childRecipeFactory'
import { buildCoordinatorGfsNetworkPolicy } from '../workflow/networkPolicyFactory'
+import { captureLogger } from './__tests__/captureLogger'
import { defaultFqdnLookup } from './fqdnResolver'
import { isRetryableInfraError } from './k8sErrors'
import * as brokerIssuer from './oauthBrokerTokenIssuerClient'
@@ -303,6 +304,612 @@ describe('WorkflowRecipeReconciler', () => {
// ─── Pipeline Tests ─────────────────────────────────────────────
+ describe('transport admission pending survives observation and restart', () => {
+ const conditionType = 'TransportExternalEgressReady'
+ const pendingCondition = {
+ type: conditionType,
+ status: 'False' as const,
+ reason: 'ExternalEgressPending',
+ message: 'Transport external egress admission is pending',
+ lastTransitionTime: '2026-09-05T00:00:00.000Z',
+ }
+ function transportRecipe(workflow = false): WorkflowRecipeCRD {
+ return makeRecipe({
+ metadata: {
+ name: 'test-recipe',
+ namespace: 'sandbox-recipes',
+ uid: 'uid-123',
+ generation: 7,
+ },
+ spec: {
+ ...(workflow ? {} : { contextRef: 'context1' }),
+ workloads: [
+ {
+ id: 'calculator',
+ type: 'deployment',
+ image: 'clerum/mock-stdio-mcp-server:test',
+ port: 3000,
+ transport: { type: 'stdio' },
+ egressBindings: [{ dns: 'current.example.com', port: 443 }],
+ },
+ ],
+ ...(workflow ? { steps: [{ id: 'step', run: snippetRun() }] } : {}),
+ },
+ status: { phase: 'approved' },
+ })
+ }
+ function hccReadiness() {
+ const healthyRead = mockCustomApi.getNamespacedCustomObject.getMockImplementation()!
+ const state = { ready: false }
+ mockCustomApi.getNamespacedCustomObject.mockImplementation(async request => {
+ const response = await healthyRead(request)
+ if (request.plural === 'mcpservers') {
+ response.metadata.generation = 7
+ response.status.conditions = [
+ {
+ type: 'ExternalEgressReady',
+ status: state.ready ? 'True' : 'False',
+ observedGeneration: 7,
+ reason: state.ready ? 'Reconciled' : 'ExternalEgressReconcileFailed',
+ message: 'Policy observation pending',
+ },
+ ]
+ }
+ return response
+ })
+ return state
+ }
+ function watcherFor(rec: WorkflowRecipeReconciler, recipe?: WorkflowRecipeCRD) {
+ const watcher: any = Object.create(WorkflowRecipeWatcher.prototype)
+ Object.assign(watcher, {
+ recipes: new Map(recipe ? [[recipe.metadata.name, recipe]] : []),
+ transientRetries: new Map(),
+ stopped: false,
+ traceReporter: null,
+ dbRunProcessor: null,
+ reconciler: rec,
+ eventQueue: { enqueue: vi.fn((_key: string, work: () => Promise<void>) => work()) },
+ config: loadConfig(),
+ })
+ return watcher
+ }
+ function stopWatcher(watcher: any) {
+ watcher.stopped = true
+ for (const { timer } of watcher.transientRetries.values()) clearTimeout(timer)
+ watcher.transientRetries.clear()
+ }
+ function patchedStatus(previous?: WorkflowRecipeCRD['status']) {
+ return {
+ ...previous,
+ ...mockCustomApi.patchNamespacedCustomObjectStatus.mock.calls.at(-1)![0].body.status,
+ }
+ }
+
+ it('retries persisted stdio-only admission with backoff after restart and cannot become active from ready pods', async () => {
+ const recipe = transportRecipe()
+ const hcc = hccReadiness()
+ const watcher = watcherFor(reconciler)
+ let restarted: any
+ vi.useFakeTimers()
+ try {
+ const firstRun = watcher.handleRecipeEvent('ADDED', recipe)
+ await vi.advanceTimersByTimeAsync(31_000)
+ await firstRun
+ expect(watcher.transientRetries.get(recipe.metadata.name)?.attempts).toBe(1)
+ const pendingRecipe = { ...recipe, status: patchedStatus() }
+ expect(pendingRecipe.status.phase).toBe('degraded')
+ expect(pendingRecipe.status.conditions).toContainEqual(
+ expect.objectContaining({ type: conditionType, status: 'False' })
+ )
+ await watcher.handleRecipeEvent('MODIFIED', pendingRecipe)
+ await watcher.refreshWorkloadStatusesForSteadyRecipes()
+ const observedStatus = patchedStatus(pendingRecipe.status)
+ expect(observedStatus.phase).toBe('degraded')
+ expect(observedStatus.conditions).toContainEqual(
+ expect.objectContaining({ type: conditionType, status: 'False' })
+ )
+ stopWatcher(watcher)
+
+ // A new process has no previous retry timer; only persisted condition state remains.
+ const restartRecipe = { ...recipe, status: observedStatus }
+ const rec = new WorkflowRecipeReconciler(new k8s.KubeConfig())
+ restarted = watcherFor(rec, restartRecipe)
+ const run = vi.spyOn(rec, 'reconcile')
+ const restartPass = restarted.refreshRuntimeCredentialsForInProgressRecipes()
+ await vi.advanceTimersByTimeAsync(31_000)
+ await restartPass
+ expect(run).toHaveBeenCalledTimes(1)
+ expect(restarted.transientRetries.get(recipe.metadata.name)?.attempts).toBe(1)
+ await restarted.handleRecipeEvent('MODIFIED', { ...recipe, status: patchedStatus() })
+ await vi.advanceTimersByTimeAsync(35_000)
+ expect(run).toHaveBeenCalledTimes(2)
+ expect(restarted.transientRetries.get(recipe.metadata.name)?.attempts).toBe(2)
+ expect(patchedStatus().phase).toBe('degraded')
+ hcc.ready = true
+ await vi.advanceTimersByTimeAsync(10_000)
+ expect(run).toHaveBeenCalledTimes(3)
+ expect(patchedStatus().phase).toBe('active')
+ expect(
+ patchedStatus().conditions?.some(
+ (condition: { type: string }) => condition.type === conditionType
+ )
+ ).not.toBe(true)
+ expect(restarted.transientRetries.size).toBe(0)
+ expect(recipe.metadata.generation).toBe(7)
+ } finally {
+ stopWatcher(watcher)
+ if (restarted) stopWatcher(restarted)
+ vi.useRealTimers()
+ }
+ })
+
+ it('persists workflow admission pending and only starts the workflow after HCC heals', async () => {
+ const recipe = transportRecipe(true)
+ const hcc = hccReadiness()
+ const workflow = vi
+ .fn()
+ .mockResolvedValue({ phase: 'active', workflowPhase: 'running', message: 'Workflow ready' })
+ ;(reconciler as any).workflowReconciler = {
+ reconcile: workflow,
+ validateWorkflowSpec: () => undefined,
+ }
+ vi.useFakeTimers()
+ try {
+ const pending = reconciler.reconcile(recipe)
+ await vi.advanceTimersByTimeAsync(31_000)
+ const first = await pending
+ expect(first.phase, first.message).toBe('degraded')
+ expect(first.requeueAfterMs).toBe(TRANSIENT_REQUEUE_BASE_MS)
+ expect(workflow).not.toHaveBeenCalled()
+ await reconciler.patchStatus(recipe, first)
+ const persisted = { ...recipe, status: patchedStatus() }
+ expect(persisted.status.conditions).toContainEqual(
+ expect.objectContaining({ type: conditionType, status: 'False' })
+ )
+ expect(persisted.status.workflowExecution?.phase).not.toBe('failed')
+ hcc.ready = true
+ const result = await reconciler.reconcile(persisted)
+ expect(result.phase).toBe('active')
+ expect(workflow).toHaveBeenCalledTimes(1)
+ await reconciler.patchStatus(persisted, result)
+ expect(
+ patchedStatus().conditions?.some(
+ (condition: { type: string }) => condition.type === conditionType
+ )
+ ).not.toBe(true)
+ } finally {
+ vi.useRealTimers()
+ }
+ })
+
+ it('preserves pending admission through an earlier validation failure and clears it after transport removal', async () => {
+ const recipe = transportRecipe()
+ recipe.status = { phase: 'degraded', conditions: [pendingCondition] }
+ recipe.spec.workloads![0].egressBindings![0].port = 0
+ const failed = await reconciler.reconcile(recipe)
+ expect(failed.phase).toBe('failed')
+ await reconciler.patchStatus(recipe, failed)
+ expect(patchedStatus(recipe.status).conditions).toContainEqual(pendingCondition)
+ recipe.spec.workloads![0].egressBindings = []
+ delete recipe.spec.workloads![0].transport
+ const removed = await reconciler.reconcile(recipe)
+ expect(removed.phase).toBe('active')
+ await reconciler.patchStatus(recipe, removed)
+ expect(
+ patchedStatus(recipe.status).conditions?.some(
+ (condition: { type: string }) => condition.type === conditionType
+ )
+ ).not.toBe(true)
+ })
+
+ it('preserves ordinary Pod-only downgrade and recovery when no admission is pending', async () => {
+ const recipe = makeRecipe({ status: { phase: 'degraded' } })
+ mockAppsApi.readNamespacedDeployment.mockResolvedValueOnce({
+ spec: { replicas: 1 },
+ status: { readyReplicas: 0, updatedReplicas: 0, availableReplicas: 0 },
+ })
+ expect((await reconciler.observeCurrentWorkloadStatus(recipe)).phase).toBe('degraded')
+ expect((await reconciler.observeCurrentWorkloadStatus(recipe)).phase).toBe('active')
+ })
+ })
+
+ describe('#567 regression: policy contraction', () => {
+ function policyStore(policies: k8s.V1NetworkPolicy[]) {
+ const live = new Map<string, k8s.V1NetworkPolicy>()
+ let version = 0
+ const persist = (body: k8s.V1NetworkPolicy) => {
+ const policy = structuredClone(body)
+ const name = policy.metadata!.name!
+ policy.metadata = {
+ ...policy.metadata,
+ uid: `uid-${name}`,
+ resourceVersion: String(++version),
+ }
+ // The API's omitempty serialization omits empty top-level rule lists.
+ if (policy.spec?.egress?.length === 0) delete policy.spec.egress
+ if (policy.spec?.ingress?.length === 0) delete policy.spec.ingress
+ live.set(name, policy)
+ return structuredClone(policy)
+ }
+ for (const policy of policies) persist(policy)
+ mockNetworkingApi.readNamespacedNetworkPolicy.mockImplementation(async ({ name }) => {
+ if (!live.has(name)) throw { code: 404 }
+ return structuredClone(live.get(name))
+ })
+ mockNetworkingApi.createNamespacedNetworkPolicy.mockImplementation(async ({ body }) => {
+ if (live.has(body.metadata.name)) throw { code: 409 }
+ return persist(body)
+ })
+ mockNetworkingApi.replaceNamespacedNetworkPolicy.mockImplementation(
+ async ({ name, body }) => {
+ expect(body.metadata.resourceVersion).toBe(live.get(name)?.metadata?.resourceVersion)
+ return persist(body)
+ }
+ )
+ mockNetworkingApi.deleteNamespacedNetworkPolicy.mockImplementation(async ({ name, body }) => {
+ if (body?.preconditions) {
+ expect(body.preconditions).toEqual({
+ uid: live.get(name)?.metadata?.uid,
+ resourceVersion: live.get(name)?.metadata?.resourceVersion,
+ })
+ }
+ live.delete(name)
+ return {}
+ })
+ return live
+ }
+
+ function uiRecipe(ports = [8080, 9090]) {
+ return makeRecipe({
+ spec: {
+ workloads: [
+ { id: 'ui', type: 'deployment', image: 'ui:1', port: 8080 },
+ { id: 'backend', type: 'deployment', image: 'backend:1', port: 8080 },
+ ],
+ ui: {
+ workloadRef: 'ui',
+ port: 8080,
+ egress: {
+ internal: ports.map(port => ({ workloadRef: 'backend', port })),
+ external: [{ fqdn: 'current.example.com', port: 443 }],
+ },
+ },
+ },
+ })
+ }
+
+ it.each([[8080], [8080, 9090, 9443], [9090, 8080]])(
+ 'preserves the intersection of grouped UI ports during a DNS fault: %j',
+ async (...ports) => {
+ const rb = await import('./resourceBuilder')
+ const recipe = uiRecipe()
+ const name = 'ui-ingress-test-recipe-backend'
+ const live = policyStore([
+ rb.buildUiIngressNetworkPolicy(
+ recipe,
+ 'backend',
+ [8080, 9090],
+ 'sandbox-recipes',
+ 'sandbox-ui'
+ )!,
+ ])
+ recipe.spec.ui!.egress!.internal = ports.map(port => ({ workloadRef: 'backend', port }))
+ const rec = new WorkflowRecipeReconciler(new k8s.KubeConfig(), undefined, {
+ fqdnLookup: async () => {
+ throw new Error('resolver controller fault')
+ },
+ })
+ const result = await rec.reconcile({ ...recipe, status: { phase: 'active' } })
+ expect(result.phase).toBe('failed')
+ const retained = (live.get(name)?.spec?.ingress ?? [])
+ .flatMap(rule => rule.ports ?? [])
+ .map(port => port.port)
+ expect(retained.sort()).toEqual(ports.filter(port => port === 8080 || port === 9090).sort())
+ }
+ )
+
+ it.each([false, true])(
+ 'accepts empty UI readback before external replacement=%s',
+ async replacement => {
+ const rb = await import('./resourceBuilder')
+ const recipe = uiRecipe([])
+ const name = 'ui-egress-test-recipe'
+ const live = policyStore([
+ rb.buildUiEgressNetworkPolicy(
+ recipe,
+ 'sandbox-ui',
+ 'sandbox-recipes',
+ [
+ {
+ cidr: '93.184.216.20/32',
+ port: 443,
+ source: { kind: 'fqdn', fqdn: 'current.example.com' },
+ },
+ ],
+ egressStateAnnotations([
+ { fqdn: 'current.example.com', ip: '93.184.216.20', port: 443 },
+ ])
+ )!,
+ ])
+ recipe.spec.ui!.egress!.external = replacement
+ ? [{ fqdn: 'new.example.com', port: 443 }]
+ : []
+ const lookup = vi.fn(async () => ({
+ kind: 'ok' as const,
+ ipv4: ['93.184.216.21'],
+ ipv6: [],
+ ttlSeconds: 300,
+ }))
+ const rec = new WorkflowRecipeReconciler(new k8s.KubeConfig(), undefined, {
+ fqdnLookup: lookup,
+ })
+ const result = await rec.reconcile({ ...recipe, status: { phase: 'active' } })
+ expect(result.phase).toBe('active')
+ expect(live.has(name)).toBe(true)
+ expect(lookup).toHaveBeenCalledTimes(replacement ? 1 : 0)
+ expect(mockNetworkingApi.deleteNamespacedNetworkPolicy).not.toHaveBeenCalledWith(
+ expect.objectContaining({ name })
+ )
+ }
+ )
+
+ it.each([
+ { mode: 'broader-rules', phase: 'failed', deleted: true },
+ { mode: 'annotation-only', phase: 'failed', deleted: true },
+ { mode: 'replacement-uid', phase: 'degraded', deleted: false },
+ { mode: 'foreign-owner', phase: 'failed', deleted: false },
+ { mode: 'missing-uid', phase: 'failed', deleted: false },
+ { mode: 'wrong-namespace', phase: 'failed', deleted: false },
+ ] as const)('rejects unsafe readback: $mode', async ({ mode, phase, deleted }) => {
+ const rb = await import('./resourceBuilder')
+ const recipe = uiRecipe([])
+ const name = 'ui-egress-test-recipe'
+ const live = policyStore([
+ rb.buildUiEgressNetworkPolicy(
+ recipe,
+ 'sandbox-ui',
+ 'sandbox-recipes',
+ [
+ {
+ cidr: '93.184.216.20/32',
+ port: 443,
+ source: { kind: 'fqdn', fqdn: 'current.example.com' },
+ },
+ ],
+ egressStateAnnotations([{ fqdn: 'current.example.com', ip: '93.184.216.20', port: 443 }])
+ )!,
+ ])
+ recipe.spec.ui!.egress!.external = []
+ const replace = mockNetworkingApi.replaceNamespacedNetworkPolicy.getMockImplementation()!
+ mockNetworkingApi.replaceNamespacedNetworkPolicy.mockImplementation(async request => {
+ const result = await replace(request)
+ if (request.name === name) {
+ const observed = live.get(name)!
+ if (mode === 'broader-rules')
+ observed.spec!.egress = [
+ {
+ to: [{ ipBlock: { cidr: '8.8.8.8/32' } }],
+ ports: [{ port: 443, protocol: 'TCP' }],
+ },
+ ]
+ if (mode === 'annotation-only')
+ observed.metadata!.annotations = egressStateAnnotations([
+ { fqdn: 'removed.example.com', ip: '8.8.8.8', port: 443 },
+ ])
+ if (mode === 'replacement-uid') observed.metadata!.uid = 'uid-new-owned-object'
+ if (mode === 'missing-uid') delete observed.metadata!.uid
+ if (mode === 'wrong-namespace') observed.metadata!.namespace = 'another-namespace'
+ if (mode === 'foreign-owner')
+ observed.metadata!.labels!['clerum.io/managed-by'] = 'another-controller'
+ }
+ return result
+ })
+ const rec = new WorkflowRecipeReconciler(new k8s.KubeConfig(), undefined, {
+ fqdnLookup: async () => {
+ throw new Error('no external lookup is declared')
+ },
+ })
+ const result = await rec.reconcile({ ...recipe, status: { phase: 'active' } })
+ expect(result.phase).toBe(phase)
+ expect(live.has(name)).toBe(!deleted)
+ if (!deleted) expect(mockNetworkingApi.deleteNamespacedNetworkPolicy).not.toHaveBeenCalled()
+ expect(result.requeueAfterMs).toBe(
+ mode === 'replacement-uid' ? TRANSIENT_REQUEUE_BASE_MS : undefined
+ )
+ })
+
+ it.each(['ui', 'workload'] as const)(
+ 'replans additive %s conflicts and disappearance without latching a healthy recipe',
+ async lane => {
+ for (const failure of ['replace-conflict', 'read-disappearance', 'replace-disappearance']) {
+ const recipe =
+ lane === 'ui'
+ ? uiRecipe([])
+ : makeRecipe({
+ spec: {
+ workloads: [
+ {
+ id: 'worker',
+ type: 'deployment',
+ image: 'worker:1',
+ port: 8080,
+ egressBindings: [{ dns: 'current.example.com', port: 443 }],
+ },
+ ],
+ },
+ })
+ const name = lane === 'ui' ? 'ui-egress-test-recipe' : 'wl-egress-test-recipe-worker'
+ const live = policyStore([])
+ let address = '93.184.216.20'
+ const rec = new WorkflowRecipeReconciler(new k8s.KubeConfig(), undefined, {
+ fqdnLookup: async () => ({ kind: 'ok', ipv4: [address], ipv6: [], ttlSeconds: 300 }),
+ })
+ expect((await rec.reconcile({ ...recipe, status: { phase: 'active' } })).phase).toBe(
+ 'active'
+ )
+ const healthyReplace =
+ mockNetworkingApi.replaceNamespacedNetworkPolicy.getMockImplementation()!
+ const healthyCreate =
+ mockNetworkingApi.createNamespacedNetworkPolicy.getMockImplementation()!
+ address = '93.184.216.21'
+ if (failure === 'read-disappearance') {
+ mockNetworkingApi.createNamespacedNetworkPolicy.mockImplementation(async request => {
+ if (request.body.metadata.name === name) {
+ live.delete(name)
+ throw Object.assign(new Error('object existed before replacement read'), {
+ code: 409,
+ })
+ }
+ return healthyCreate(request)
+ })
+ } else {
+ mockNetworkingApi.replaceNamespacedNetworkPolicy.mockImplementation(async request => {
+ if (request.name === name && JSON.stringify(request.body.spec).includes(address)) {
+ if (failure === 'replace-disappearance') live.delete(name)
+ throw Object.assign(new Error('object snapshot changed'), {
+ code: failure === 'replace-conflict' ? 409 : 404,
+ })
+ }
+ return healthyReplace(request)
+ })
+ }
+ const first = await rec.reconcile({ ...recipe, status: { phase: 'active' } })
+ expect(first.phase, `${lane}/${failure}`).toBe('degraded')
+ expect(first.requeueAfterMs).toBe(TRANSIENT_REQUEUE_BASE_MS)
+ await rec.patchStatus(recipe, first)
+ const persisted =
+ mockCustomApi.patchNamespacedCustomObjectStatus.mock.calls.at(-1)![0].body.status
+ mockNetworkingApi.createNamespacedNetworkPolicy.mockImplementation(healthyCreate)
+ mockNetworkingApi.replaceNamespacedNetworkPolicy.mockImplementation(healthyReplace)
+ const recovered = await rec.reconcile({ ...recipe, status: persisted })
+ expect(recovered.phase).toBe('active')
+ expect(JSON.stringify(live.get(name)?.spec)).toContain(address)
+ }
+ }
+ )
+
+ it.each(['delete', 'replace'] as const)(
+ 'retries a fenced %s conflict after the status roundtrip',
+ async operation => {
+ const rb = await import('./resourceBuilder')
+ const recipe = makeRecipe({
+ spec: {
+ workloads: [
+ {
+ id: 'worker',
+ type: 'deployment',
+ image: 'worker:1',
+ port: 8080,
+ egressBindings: [
+ { dns: 'current.example.com', port: 443 },
+ { dns: 'removed.example.com', port: 443 },
+ ],
+ },
+ {
+ id: 'keeper',
+ type: 'deployment',
+ image: 'keeper:1',
+ port: 8080,
+ egressBindings: [{ dns: 'keeper.example.com', port: 443 }],
+ },
+ ],
+ },
+ })
+ const name = 'wl-egress-test-recipe-worker'
+ const entries = [
+ { fqdn: 'current.example.com', ip: '93.184.216.20', port: 443 },
+ { fqdn: 'removed.example.com', ip: '93.184.216.21', port: 443 },
+ ]
+ const live = policyStore([
+ rb.buildWorkloadEgressNetworkPolicy(
+ recipe.spec.workloads![0],
+ recipe,
+ 'sandbox-recipes',
+ entries.map(entry => ({
+ cidr: `${entry.ip}/32`,
+ port: entry.port,
+ source: { kind: 'fqdn', fqdn: entry.fqdn },
+ })),
+ egressStateAnnotations(entries)
+ )!,
+ ])
+ recipe.spec.workloads![0].egressBindings =
+ operation === 'delete' ? [] : [{ dns: 'current.example.com', port: 443 }]
+ const mutation =
+ operation === 'delete'
+ ? mockNetworkingApi.deleteNamespacedNetworkPolicy
+ : mockNetworkingApi.replaceNamespacedNetworkPolicy
+ const healthyMutation = mutation.getMockImplementation()!
+ mutation.mockImplementationOnce(async () => {
+ throw Object.assign(new Error('object modified'), { code: 409 })
+ })
+ const rec = new WorkflowRecipeReconciler(new k8s.KubeConfig(), undefined, {
+ fqdnLookup: async () => ({
+ kind: 'ok',
+ ipv4: ['93.184.216.20'],
+ ipv6: [],
+ ttlSeconds: 300,
+ }),
+ })
+ type Watcher = {
+ recipes: Map<string, WorkflowRecipeCRD>
+ transientRetries: Map<string, { timer: ReturnType<typeof setTimeout>; attempts: number }>
+ stopped: boolean
+ traceReporter: null
+ dbRunProcessor: null
+ reconciler: WorkflowRecipeReconciler
+ eventQueue: { enqueue: (key: string, task: () => Promise<void>) => Promise<void> }
+ handleRecipeEvent: (type: string, recipe: WorkflowRecipeCRD) => Promise<void>
+ }
+ const watcher = Object.create(WorkflowRecipeWatcher.prototype) as Watcher
+ Object.assign(watcher, {
+ recipes: new Map(),
+ transientRetries: new Map(),
+ stopped: false,
+ traceReporter: null,
+ dbRunProcessor: null,
+ reconciler: rec,
+ eventQueue: { enqueue: vi.fn((_key: string, task: () => Promise<void>) => task()) },
+ })
+ const reconcile = vi.spyOn(rec, 'reconcile')
+ const active = {
+ ...recipe,
+ metadata: { ...recipe.metadata, generation: 1 },
+ status: { phase: 'active' as const },
+ }
+ vi.useFakeTimers()
+ try {
+ await watcher.handleRecipeEvent('ADDED', active)
+ const first = await reconcile.mock.results[0].value
+ expect(first.phase).toBe('degraded')
+ expect(first.requeueAfterMs).toBe(TRANSIENT_REQUEUE_BASE_MS)
+ const patched =
+ mockCustomApi.patchNamespacedCustomObjectStatus.mock.calls.at(-1)![0].body.status
+ // Deliver the real status-only watch event without changing generation.
+ await watcher.handleRecipeEvent('MODIFIED', { ...active, status: patched })
+ expect(reconcile).toHaveBeenCalledTimes(1)
+ expect(watcher.transientRetries.get(recipe.metadata.name)?.attempts).toBe(1)
+ mutation.mockImplementationOnce(async () => {
+ throw Object.assign(new Error('object modified again'), { code: 409 })
+ })
+ await vi.advanceTimersByTimeAsync(TRANSIENT_REQUEUE_BASE_MS)
+ expect(reconcile).toHaveBeenCalledTimes(2)
+ expect(watcher.transientRetries.get(recipe.metadata.name)?.attempts).toBe(2)
+ mutation.mockImplementation(healthyMutation)
+ await vi.advanceTimersByTimeAsync(TRANSIENT_REQUEUE_BASE_MS * 2)
+ const next = await reconcile.mock.results[2].value
+ expect(next.phase).toBe('active')
+ expect(watcher.transientRetries.size).toBe(0)
+ expect(JSON.stringify(live.get(name)?.spec ?? {})).not.toContain('93.184.216.21')
+ expect(live.has(name)).toBe(operation === 'replace')
+ } finally {
+ for (const { timer } of watcher.transientRetries.values()) clearTimeout(timer)
+ vi.useRealTimers()
+ }
+ }
+ )
+ })
+
it('processes full pipeline successfully (3.11a)', async () => {
const result = await reconciler.reconcile(makeRecipe())
expect(result.phase).toBe('active')
@@ -2940,9 +3547,9 @@ describe('WorkflowRecipeReconciler', () => {
{
resolverLabel: 'controller fault',
resolverResult: 'throw' as const,
- label: 'non-retryable identity conflict',
+ label: 'non-retryable API access denial',
contractionError: () =>
- Object.assign(new Error('workload re-contraction conflict'), { code: 409 }),
+ Object.assign(new Error('workload re-contraction access denied'), { code: 403 }),
expectedPhase: 'failed' as const,
retryable: false,
},
@@ -2958,9 +3565,9 @@ describe('WorkflowRecipeReconciler', () => {
{
resolverLabel: 'transient DNS result',
resolverResult: 'transient' as const,
- label: 'non-retryable identity conflict',
+ label: 'non-retryable API access denial',
contractionError: () =>
- Object.assign(new Error('workload re-contraction conflict'), { code: 409 }),
+ Object.assign(new Error('workload re-contraction access denied'), { code: 403 }),
expectedPhase: 'failed' as const,
retryable: false,
},
@@ -3030,7 +3637,7 @@ describe('WorkflowRecipeReconciler', () => {
expect(result.requeueAfterMs).toBe(TRANSIENT_REQUEUE_BASE_MS)
} else {
expect(result.skipStatusPatch).not.toBe(true)
- expect(result.message).toContain('workload re-contraction conflict')
+ expect(result.message).toContain('workload re-contraction access denied')
}
expect(mockNetworkingApi.createNamespacedNetworkPolicy).not.toHaveBeenCalledWith(
expect.objectContaining({
@@ -4799,8 +5406,9 @@ describe('WorkflowRecipeReconciler', () => {
{
resolverLabel: 'controller fault',
resolverResult: 'throw' as const,
- label: 'non-retryable identity conflict',
- contractionError: () => Object.assign(new Error('UI re-contraction conflict'), { code: 409 }),
+ label: 'non-retryable API access denial',
+ contractionError: () =>
+ Object.assign(new Error('UI re-contraction access denied'), { code: 403 }),
expectedPhase: 'failed' as const,
retryable: false,
},
@@ -4816,8 +5424,9 @@ describe('WorkflowRecipeReconciler', () => {
{
resolverLabel: 'transient DNS result',
resolverResult: 'transient' as const,
- label: 'non-retryable identity conflict',
- contractionError: () => Object.assign(new Error('UI re-contraction conflict'), { code: 409 }),
+ label: 'non-retryable API access denial',
+ contractionError: () =>
+ Object.assign(new Error('UI re-contraction access denied'), { code: 403 }),
expectedPhase: 'failed' as const,
retryable: false,
},
@@ -4873,7 +5482,7 @@ describe('WorkflowRecipeReconciler', () => {
expect(result.requeueAfterMs).toBe(TRANSIENT_REQUEUE_BASE_MS)
} else {
expect(result.skipStatusPatch).not.toBe(true)
- expect(result.message).toContain('UI re-contraction conflict')
+ expect(result.message).toContain('UI re-contraction access denied')
}
expect(mockNetworkingApi.createNamespacedNetworkPolicy).not.toHaveBeenCalledWith(
expect.objectContaining({
@@ -5317,7 +5926,6 @@ describe('WorkflowRecipeReconciler', () => {
)
const cap = captureEgressRecords()
- const noOpSpy = vi.spyOn(console, 'log').mockImplementation(() => {})
try {
mockNetworkingApi.createNamespacedNetworkPolicy.mockClear()
mockNetworkingApi.replaceNamespacedNetworkPolicy.mockClear()
@@ -5331,14 +5939,9 @@ describe('WorkflowRecipeReconciler', () => {
mockNetworkingApi.createNamespacedNetworkPolicy.mock.calls.length +
mockNetworkingApi.replaceNamespacedNetworkPolicy.mock.calls.length
expect(wrote).toBeGreaterThan(0)
- expect(noOpSpy).not.toHaveBeenCalledWith(
- expect.stringContaining('ui-egress-test-recipe" in sandbox-ui egress set unchanged — no-op')
- )
-
// ...and no record, because the resolved set did not change.
expect(cap.payloads()).toHaveLength(0)
} finally {
- noOpSpy.mockRestore()
cap.restore()
}
})
@@ -11174,7 +11777,7 @@ describe('WorkflowRecipeReconciler', () => {
vi.spyOn(brokerIssuer, 'issueOAuthBrokerToken').mockRejectedValue(
new Error('OAuth broker token issuance timed out after 10000ms for recipe "test-recipe"')
)
- const warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => {})
+ const warnSpy = captureLogger('warn')
const result = await reconciler.reconcile(recipeWithBackgroundAccess())
@@ -11182,10 +11785,12 @@ describe('WorkflowRecipeReconciler', () => {
expect(result.phase).toBe('active')
// Warning was emitted so the operator can see what happened.
expect(warnSpy).toHaveBeenCalledWith(
- expect.stringContaining('Broker-token issuance failed during reconcile')
+ expect.stringContaining('Broker-token issuance failed during reconcile'),
+ expect.objectContaining({ err: expect.any(Error) })
)
// Workloads still deployed.
expect(mockAppsApi.createNamespacedDeployment).toHaveBeenCalledTimes(1)
+ warnSpy.mockRestore()
})
})
diff --git a/workflow-recipes/src/reconciler/workflowRecipeReconciler.ts b/workflow-recipes/src/reconciler/workflowRecipeReconciler.ts
index 124843d72..f6f7e0dbd 100644
--- a/workflow-recipes/src/reconciler/workflowRecipeReconciler.ts
+++ b/workflow-recipes/src/reconciler/workflowRecipeReconciler.ts
@@ -99,6 +99,11 @@ import {
waitForExternalEgressReady,
waitForNetworkReady,
} from './mcpDelegation'
+import {
+ effectivePolicyTypes,
+ intersectNetworkPolicyRules,
+ networkPolicySpecSignature,
+} from './networkPolicyContraction'
import { issueOAuthBrokerToken } from './oauthBrokerTokenIssuerClient'
import {
PLUGIN_WORKLOAD_SDK_CONDITION_TYPE,
@@ -137,6 +142,36 @@ const PARENT_RECIPE_LABEL = 'clerum.io/parent-recipe'
const WORKFLOW_RUN_ID_LABEL = 'clerum.io/workflow-run-id'
const WORKFLOW_ACTOR_ID_LABEL = 'clerum.io/workflow-actor-id'
const WORKFLOW_ACTOR_TYPE_LABEL = 'clerum.io/workflow-actor-type'
+export const TRANSPORT_NETWORK_CONDITION_TYPE = 'TransportExternalEgressReady'
+const TRANSPORT_NETWORK_CONDITION_TYPES = new Set([TRANSPORT_NETWORK_CONDITION_TYPE])
+
+export function hasPendingTransportNetworkReadiness(recipe: WorkflowRecipeCRD): boolean {
+ return (
+ recipe.status?.conditions?.some(
+ condition =>
+ condition.type === TRANSPORT_NETWORK_CONDITION_TYPE && condition.status !== 'True'
+ ) ?? false
+ )
+}
+
+function pendingTransportNetworkConditions(
+ recipe: WorkflowRecipeCRD,
+ message: string
+): StatusCondition[] {
+ const previous = recipe.status?.conditions?.find(
+ condition => condition.type === TRANSPORT_NETWORK_CONDITION_TYPE
+ )
+ return [
+ {
+ type: TRANSPORT_NETWORK_CONDITION_TYPE,
+ status: 'False',
+ reason: 'ExternalEgressPending',
+ message,
+ lastTransitionTime:
+ previous?.status === 'False' ? previous.lastTransitionTime : new Date().toISOString(),
+ },
+ ]
+}
function externalEgressReadinessError(
recipeName: string,
@@ -148,7 +183,9 @@ function externalEgressReadinessError(
`External egress policy readiness not achieved for WorkflowRecipe "${recipeName}". ` +
`${[...pending, ...failed].join('; ')}. ` +
`External egress bindings are not enforceable until HCC reports ExternalEgressReady=True.`
- return result.failed.length === 0 ? new RetryableReconcileError(message) : new Error(message)
+ return result.failed.length === 0
+ ? new TransportNetworkReadinessPendingError(message)
+ : new Error(message)
}
function clusterNetworkPolicyEnforcementError(recipeName: string): Error {
@@ -482,6 +519,25 @@ class NetworkPolicyOwnershipConflictError extends Error {
}
}
+/** A fenced mutation lost its snapshot; only a fresh reconciliation may retry. */
+class NetworkPolicyReplanRequiredError extends RetryableReconcileError {
+ override name = 'NetworkPolicyReplanRequiredError'
+}
+
+class TransportNetworkReadinessPendingError extends RetryableReconcileError {
+ override name = 'TransportNetworkReadinessPendingError'
+}
+
+function throwNetworkPolicyMutationError(error: unknown, name: string, namespace: string): never {
+ if (getErrorCode(error) === 409 || getErrorCode(error) === 404) {
+ throw new NetworkPolicyReplanRequiredError(
+ `NetworkPolicy "${name}" in ${namespace} changed during a fenced mutation; a fresh reconciliation is required`,
+ { cause: error }
+ )
+ }
+ throw error
+}
+
class InternalDependencyReconcileError extends Error {
constructor(
message: string,
@@ -541,6 +597,8 @@ export interface ReconcileResult {
* UI/API-consumable reasons that should not be inferred from free-form messages.
*/
workloadConditions?: StatusCondition[]
+ /** Undefined preserves admission state; [] clears it only after a full admission check. */
+ transportNetworkConditions?: StatusCondition[]
/** SDK-only eager-host provider health, kept separate from workflow phase. */
pluginWorkloadSdkProviderUnavailable?: boolean
/** SDK host identity is ready, but an operator prompt policy is not active yet. */
@@ -730,10 +788,6 @@ export function egressSignature(policy: k8s.V1NetworkPolicy): string {
)
}
-function networkPolicySpecSignature(policy: k8s.V1NetworkPolicy): string {
- return JSON.stringify(canonicalize(policy.spec ?? {}))
-}
-
function contractionWriteNeeded(
existing: k8s.V1NetworkPolicy,
desired: k8s.V1NetworkPolicy
@@ -746,21 +800,6 @@ function contractionWriteNeeded(
)
}
-function intersectRules<T>(existing: T[] | undefined, desired: T[] | undefined): T[] {
- const available = new Map<string, number>()
- for (const rule of existing ?? []) {
- const signature = JSON.stringify(canonicalize(rule))
- available.set(signature, (available.get(signature) ?? 0) + 1)
- }
- return (desired ?? []).filter(rule => {
- const signature = JSON.stringify(canonicalize(rule))
- const remaining = available.get(signature) ?? 0
- if (remaining === 0) return false
- available.set(signature, remaining - 1)
- return true
- })
-}
-
/**
* Intersect a desired policy with the rules already enforced by the live
* snapshot. Used only by the pre-DNS contraction phase: it may remove rules but
@@ -774,16 +813,24 @@ function subtractiveNetworkPolicy(
if (!desired) return null
const existingSelector = JSON.stringify(canonicalize(existing.spec?.podSelector ?? {}))
const desiredSelector = JSON.stringify(canonicalize(desired.spec?.podSelector ?? {}))
- const existingTypes = JSON.stringify([...(existing.spec?.policyTypes ?? [])].sort())
- const desiredTypes = JSON.stringify([...(desired.spec?.policyTypes ?? [])].sort())
+ const existingTypes = JSON.stringify(effectivePolicyTypes(existing.spec))
+ const desiredTypes = JSON.stringify(effectivePolicyTypes(desired.spec))
if (existingSelector !== desiredSelector || existingTypes !== desiredTypes) return null
const contractedSpec: k8s.V1NetworkPolicySpec = { ...desired.spec! }
if (desired.spec?.egress !== undefined) {
- contractedSpec.egress = intersectRules(existing.spec?.egress, desired.spec.egress)
+ contractedSpec.egress = intersectNetworkPolicyRules(
+ existing.spec?.egress,
+ desired.spec.egress,
+ 'to'
+ )
}
if (desired.spec?.ingress !== undefined) {
- contractedSpec.ingress = intersectRules(existing.spec?.ingress, desired.spec.ingress)
+ contractedSpec.ingress = intersectNetworkPolicyRules(
+ existing.spec?.ingress,
+ desired.spec.ingress,
+ '_from'
+ )
}
const contracted: k8s.V1NetworkPolicy = { ...desired, spec: contractedSpec }
@@ -910,9 +957,11 @@ export class WorkflowRecipeReconciler {
if (controlApiPem) {
const { initializeControlApiPublicKey } = await import('../workflow/restEndpoints')
await initializeControlApiPublicKey(controlApiPem)
- console.log('[WR-Reconciler] control-api JWT public key loaded — admin delegation enabled')
+ createLogger('wrc', 'workflow-recipes').info(
+ '[WR-Reconciler] control-api JWT public key loaded — admin delegation enabled'
+ )
} else {
- console.warn(
+ createLogger('wrc', 'workflow-recipes').warn(
'[WR-Reconciler] CONTROL_API_PUBLIC_KEY_PEM not set — admin artifact delegation will return 401'
)
}
@@ -972,7 +1021,7 @@ export class WorkflowRecipeReconciler {
pluginWorkloadSdkRevocationClient: new HttpPluginWorkloadSdkRevocationClient(),
}
this.workflowReconciler = new WorkflowReconciler(deps)
- console.log('[WR-Reconciler] Workflow subsystem initialized')
+ createLogger('wrc', 'workflow-recipes').info('[WR-Reconciler] Workflow subsystem initialized')
}
private get delegationDeps(): DelegationDeps {
@@ -1036,9 +1085,9 @@ export class WorkflowRecipeReconciler {
namespace
)
if (!ready) {
- console.warn(
- `[WR-Reconciler] Generic network readiness not confirmed for: ${pending.join(', ')}`
- )
+ createLogger('wrc', recipe.metadata.name).warn('Generic network readiness not confirmed', {
+ value1: pending.join(', '),
+ })
}
}
}
@@ -1054,8 +1103,9 @@ export class WorkflowRecipeReconciler {
throw clusterNetworkPolicyEnforcementError(recipe.metadata.name)
}
if (this.config.networkPolicyEnforcementMode === 'warn') {
- console.warn(
- `[WR-Reconciler] NetworkPolicy enforcement mode is "warn" for recipe "${recipe.metadata.name}". ${policyReadiness}, but packet-level cluster enforcement must still be validated before treating this as a security gate.`
+ createLogger('wrc', recipe.metadata.name).warn(
+ 'NetworkPolicy enforcement is in warn mode; packet-level enforcement still requires validation',
+ { name: recipe.metadata.name, policyReadiness }
)
}
}
@@ -1082,8 +1132,9 @@ export class WorkflowRecipeReconciler {
parentLabel !== ownerRecipeName ||
recipe.metadata.annotations?.[INHERITED_PARENT_RESOURCES_ANNOTATION] !== 'true'
) {
- console.warn(
- `[WR-Reconciler] Ignoring controller ownerReference "${ownerRecipeName}" on workflow "${recipe.metadata.name}" because normal DB-run inheritance metadata is incomplete or inconsistent`
+ createLogger('wrc', recipe.metadata.name).warn(
+ 'Ignoring controller ownerReference because DB-run inheritance metadata is inconsistent',
+ { ownerRecipeName, name: recipe.metadata.name }
)
return recipe.metadata.name
}
@@ -1095,8 +1146,9 @@ export class WorkflowRecipeReconciler {
)
if (!verifiedOwnerRecipeName) {
if (parentLabel) {
- console.warn(
- `[WR-Reconciler] Ignoring ${PARENT_RECIPE_LABEL}="${parentLabel}" on workflow "${recipe.metadata.name}" because controller ownerReference "${ownerRecipeName}" could not be verified`
+ createLogger('wrc', recipe.metadata.name).warn(
+ 'Ignoring parent-recipe label because controller ownership could not be verified',
+ { PARENT_RECIPE_LABEL, parentLabel, name: recipe.metadata.name, ownerRecipeName }
)
}
return recipe.metadata.name
@@ -1116,16 +1168,17 @@ export class WorkflowRecipeReconciler {
)
}
if (provenance === 'invalid') {
- console.warn(
- `[WR-Reconciler] Ignoring controller ownerReference "${ownerRecipeName}" on workflow "${recipe.metadata.name}" because workflow_runs does not bind run "${workflowRunId}" to this exact parent and child`
+ createLogger('wrc', recipe.metadata.name).warn(
+ 'Ignoring controller ownerReference because workflow_runs does not bind the exact parent and child',
+ { ownerRecipeName, name: recipe.metadata.name, workflowRunId }
)
return recipe.metadata.name
}
} catch (error) {
if (error instanceof RuntimeScopeResolutionPendingError) throw error
- console.warn(
- `[WR-Reconciler] Deferring runtime scope resolution for workflow "${recipe.metadata.name}" because DB-run provenance is temporarily unavailable:`,
- error
+ createLogger('wrc', recipe.metadata.name).warn(
+ 'Deferring runtime scope resolution because DB-run provenance is unavailable',
+ { name: recipe.metadata.name, err: error }
)
throw new RuntimeScopeResolutionPendingError(
`DB-run provenance is temporarily unavailable for workflow "${recipe.metadata.name}"`,
@@ -1137,8 +1190,9 @@ export class WorkflowRecipeReconciler {
}
if (parentLabel) {
- console.warn(
- `[WR-Reconciler] Ignoring ${PARENT_RECIPE_LABEL}="${parentLabel}" on workflow "${recipe.metadata.name}" because no controller WorkflowRecipe ownerReference is present`
+ createLogger('wrc', recipe.metadata.name).warn(
+ 'Ignoring parent-recipe label without a controller WorkflowRecipe ownerReference',
+ { PARENT_RECIPE_LABEL, parentLabel, name: recipe.metadata.name }
)
}
return recipe.metadata.name
@@ -1253,14 +1307,16 @@ export class WorkflowRecipeReconciler {
const liveOwnerUid = liveOwner.metadata?.uid
if (liveOwner.metadata?.deletionTimestamp) {
- console.warn(
- `[WR-Reconciler] Ignoring controller ownerReference "${ownerRecipeName}" on workflow "${recipe.metadata.name}" because the owner is deleting`
+ createLogger('wrc', recipe.metadata.name).warn(
+ 'Ignoring controller ownerReference because the owner is deleting',
+ { ownerRecipeName, name: recipe.metadata.name }
)
return null
}
if (liveOwnerUid !== ownerRecipeUid) {
- console.warn(
- `[WR-Reconciler] Ignoring controller ownerReference "${ownerRecipeName}" on workflow "${recipe.metadata.name}" because owner UID did not match the live WorkflowRecipe`
+ createLogger('wrc', recipe.metadata.name).warn(
+ 'Ignoring controller ownerReference because the live owner UID differs',
+ { ownerRecipeName, name: recipe.metadata.name }
)
return null
}
@@ -1268,14 +1324,15 @@ export class WorkflowRecipeReconciler {
} catch (error) {
const code = getErrorCode(error)
if (code === 404) {
- console.warn(
- `[WR-Reconciler] Ignoring controller ownerReference "${ownerRecipeName}" on workflow "${recipe.metadata.name}" because the owner WorkflowRecipe was not found`
+ createLogger('wrc', recipe.metadata.name).warn(
+ 'Ignoring controller ownerReference because the owner was not found',
+ { ownerRecipeName, name: recipe.metadata.name }
)
return null
}
- console.warn(
- `[WR-Reconciler] Deferring runtime scope resolution for workflow "${recipe.metadata.name}" because the owner WorkflowRecipe is temporarily unavailable:`,
- error
+ createLogger('wrc', recipe.metadata.name).warn(
+ 'Deferring runtime scope resolution because the owner is unavailable',
+ { name: recipe.metadata.name, err: error }
)
throw new RuntimeScopeResolutionPendingError(
`Owner WorkflowRecipe "${ownerRecipeName}" is temporarily unavailable`,
@@ -1308,9 +1365,9 @@ export class WorkflowRecipeReconciler {
runtimeScopeRecipeName
)
} catch (error) {
- console.error(
- `[WR-Reconciler] Failed to repair coordinator runtime credentials for steady-state workflow "${recipe.metadata.name}"; continuing without phase change:`,
- error
+ createLogger('wrc', recipe.metadata.name).error(
+ 'Failed to repair coordinator runtime credentials; continuing without phase change',
+ { name: recipe.metadata.name, err: error }
)
}
try {
@@ -1322,9 +1379,9 @@ export class WorkflowRecipeReconciler {
recipe.metadata.uid
)
} catch (error) {
- console.error(
- `[WR-Reconciler] Failed to repair mcpHost runtime credentials for steady-state workflow "${recipe.metadata.name}"; continuing without phase change:`,
- error
+ createLogger('wrc', recipe.metadata.name).error(
+ 'Failed to repair mcpHost runtime credentials; continuing without phase change',
+ { name: recipe.metadata.name, err: error }
)
}
try {
@@ -1336,9 +1393,9 @@ export class WorkflowRecipeReconciler {
runtimeScopeRecipeName
)
} catch (error) {
- console.error(
- `[WR-Reconciler] Failed to refresh runtime HTTP egress for steady-state workflow "${recipe.metadata.name}"; keeping last valid NetworkPolicy:`,
- error
+ createLogger('wrc', recipe.metadata.name).error(
+ 'Failed to refresh runtime HTTP egress; keeping the last valid NetworkPolicy',
+ { name: recipe.metadata.name, err: error }
)
}
}
@@ -1357,8 +1414,9 @@ export class WorkflowRecipeReconciler {
}
private staleRecipeResult(recipe: WorkflowRecipeCRD, stage: string): ReconcileResult {
- console.warn(
- `[WR-Reconciler] Skipping stale reconcile for "${recipe.metadata.name}" at ${stage}; recipe is gone or deleting`
+ createLogger('wrc', recipe.metadata.name).warn(
+ 'Skipping stale reconcile because the recipe is gone or deleting',
+ { name: recipe.metadata.name, stage }
)
return {
phase: recipe.status?.phase ?? 'candidate',
@@ -1473,7 +1531,9 @@ export class WorkflowRecipeReconciler {
name: recipe.metadata.name,
body: patch,
})
- console.log(`[WR-Reconciler] Added finalizer to "${recipe.metadata.name}"`)
+ createLogger('wrc', recipe.metadata.name).info('Added recipe finalizer', {
+ name: recipe.metadata.name,
+ })
}
async removeFinalizer(recipe: WorkflowRecipeCRD): Promise<void> {
@@ -1492,14 +1552,17 @@ export class WorkflowRecipeReconciler {
})
} catch (error) {
if (getErrorCode(error) === 404) {
- console.warn(
- `[WR-Reconciler] Finalizer already gone for "${recipe.metadata.name}" because the recipe no longer exists`
+ createLogger('wrc', recipe.metadata.name).warn(
+ 'Finalizer already gone because the recipe no longer exists',
+ { name: recipe.metadata.name }
)
return
}
throw error
}
- console.log(`[WR-Reconciler] Removed finalizer from "${recipe.metadata.name}"`)
+ createLogger('wrc', recipe.metadata.name).info('Removed recipe finalizer', {
+ name: recipe.metadata.name,
+ })
}
// ─── Main Pipeline ────────────────────────────────────────────────
@@ -1527,8 +1590,9 @@ export class WorkflowRecipeReconciler {
// but a manually-applied CRD in another namespace must not be reconciled.
const allowedNamespaces = [this.config.sandboxNamespace]
if (!allowedNamespaces.includes(ns)) {
- console.warn(
- `[WR-Reconciler] Refusing to reconcile "${name}" — namespace "${ns}" is not in allowlist (${allowedNamespaces.join(', ')}). This recipe bypassed the admission layer; leaving untouched.`
+ createLogger('wrc', recipe.metadata.name).warn(
+ 'Refusing to reconcile a recipe outside the namespace allowlist',
+ { name, ns, value3: allowedNamespaces.join(', ') }
)
return {
phase: 'failed',
@@ -1538,9 +1602,7 @@ export class WorkflowRecipeReconciler {
}
}
- console.log(
- `[WR-Reconciler] Reconciling "${name}" in namespace "${ns}" (phase: ${currentPhase})`
- )
+ createLogger('wrc', recipe.metadata.name).info('Reconciling recipe', { name, ns, currentPhase })
if (!(await this.recipeStillActive(recipe))) {
return this.staleRecipeResult(recipe, 'initial check')
@@ -1624,7 +1686,10 @@ export class WorkflowRecipeReconciler {
if (violations.length > 0) {
if (isWorkflow) await this.revokeCoordinatorGfsNetworkPolicy(recipe)
const details = violations.map(v => `[${v.policy}] ${v.rule}: ${v.message}`).join('; ')
- console.error(`[WR-Reconciler] Policy violation for "${name}": ${details}`)
+ createLogger('wrc', recipe.metadata.name).error('Recipe policy violation', {
+ name,
+ details,
+ })
return {
phase: 'failed',
message: `Policy violation: ${details}`,
@@ -1703,9 +1768,9 @@ export class WorkflowRecipeReconciler {
approvalScopeRecipeName = await this.workflowRuntimeScopeRecipeName(recipe)
} catch (error) {
if (!(error instanceof RuntimeScopeResolutionPendingError)) throw error
- console.warn(
- `[WR-Reconciler] Runtime scope resolution pending for workflow "${name}"; keeping current state and requeueing:`,
- error
+ createLogger('wrc', recipe.metadata.name).warn(
+ 'Runtime scope resolution pending; keeping current state and requeueing',
+ { name, err: error }
)
return {
phase: currentPhase as RecipePhase,
@@ -1721,7 +1786,12 @@ export class WorkflowRecipeReconciler {
}
}
- if (currentPhase === 'active' && awaitsTriggeredRun && !wfExecPhase) {
+ if (
+ currentPhase === 'active' &&
+ awaitsTriggeredRun &&
+ !wfExecPhase &&
+ !hasPendingTransportNetworkReadiness(recipe)
+ ) {
// Plugin Workload SDK recipes (BOTH families) keep an eager mcp-host and
// must fall through to the inner reconcile: promptBridge so
// ensureEagerSdkMcpHost retries the /configure (the short-circuit below
@@ -1793,8 +1863,9 @@ export class WorkflowRecipeReconciler {
skipStatusPatch: recipe.status?.message === derivedMessage,
}
}
- console.log(
- `[WR-Reconciler] Workflow "${name}" ${wfExecPhase} — transitioning recipe phase: ${currentPhase} → ${derivedPhase}`
+ createLogger('wrc', recipe.metadata.name).info(
+ 'Transitioning recipe phase from workflow execution state',
+ { name, wfExecPhase, currentPhase, derivedPhase }
)
return { phase: derivedPhase, message: derivedMessage, workloadStatuses: [] }
}
@@ -1812,18 +1883,24 @@ export class WorkflowRecipeReconciler {
// EXCEPTION: "initializing" and "recovering" phases mean pods may not exist yet,
// may be broken (e.g. CreateContainerConfigError), or were just deleted by crash
// recovery. We MUST reconcile to allow infrastructure creation to complete.
- if (wfInProgress && (currentPhase === 'deploying' || currentPhase === 'failed')) {
+ if (
+ wfInProgress &&
+ (currentPhase === 'deploying' || currentPhase === 'failed') &&
+ !hasPendingTransportNetworkReadiness(recipe)
+ ) {
if (wfExecPhase === 'initializing' || wfExecPhase === 'recovering') {
- console.log(
- `[WR-Reconciler] Workflow "${name}" ${wfExecPhase} — allowing reconcile for pod creation`
+ createLogger('wrc', recipe.metadata.name).info(
+ 'Allowing workflow reconciliation for pod creation',
+ { name, wfExecPhase }
)
// Fall through to WorkflowReconciler.reconcile() which will create/recreate pods
} else {
if (coordinatorGfsPolicyCanOpen) {
await this.ensureCoordinatorGfsNetworkPolicyIfEnabled(recipe)
}
- console.log(
- `[WR-Reconciler] Workflow "${name}" in-progress (wf: ${wfExecPhase}) — skipping reconcile`
+ createLogger('wrc', recipe.metadata.name).info(
+ 'Skipping reconciliation for an in-progress workflow',
+ { name, wfExecPhase }
)
await this.ensureSteadyWorkflowRuntimeCredentials(recipe, approvalScopeRecipeName)
// Issue #637 — same revocation enforcement as the active short-circuit
@@ -1846,7 +1923,11 @@ export class WorkflowRecipeReconciler {
// "active" covers both completed workflow infrastructure and an already-active
// recipe whose current execution is still running.
- if (currentPhase === 'active' && !awaitsTriggeredRun) {
+ if (
+ currentPhase === 'active' &&
+ !awaitsTriggeredRun &&
+ !hasPendingTransportNetworkReadiness(recipe)
+ ) {
if (coordinatorGfsPolicyCanOpen) {
await this.ensureCoordinatorGfsNetworkPolicyIfEnabled(recipe)
}
@@ -1913,8 +1994,9 @@ export class WorkflowRecipeReconciler {
// otherwise it propagates out of the workflow branch where the watcher
// swallows it with no deterministic retry (issue #571).
if (isRetryableInfraError(error)) {
- console.warn(
- `[WR-Reconciler] Transient infra error assigning instances for "${recipe.metadata.name}" — will retry: ${String(error)}`
+ createLogger('wrc', recipe.metadata.name).warn(
+ 'Transient infrastructure error assigning workload instances; will retry',
+ { name: recipe.metadata.name, err: error }
)
return {
phase: (recipe.status?.phase ?? 'deploying') as RecipePhase,
@@ -2000,14 +2082,18 @@ export class WorkflowRecipeReconciler {
let workflowInternalDependencyConditions: StatusCondition[] | undefined
let workflowSecretOwnershipConditions: StatusCondition[] | undefined
let workflowWorkloadConditions: StatusCondition[] | undefined
+ let workflowTransportNetworkConditions: StatusCondition[] | undefined =
+ workflowWorkloads.length === 0 ? [] : undefined
if (workflowWorkloads.length > 0) {
- console.log(
- `[WR-Reconciler] Deploying ${workflowWorkloads.length} workflow workload(s) for "${name}"`
- )
+ createLogger('wrc', recipe.metadata.name).info('Deploying workflow workloads', {
+ length: workflowWorkloads.length,
+ name,
+ })
try {
const workflowDeploy = await this.deployWorkflowWorkloads(recipe, name)
workflowInternalDependencyConditions = workflowDeploy.internalDependencyConditions
workflowWorkloadConditions = workflowDeploy.workloadConditions
+ workflowTransportNetworkConditions = workflowDeploy.transportNetworkConditions
// Issue #637 — surface the EnvSecretOwnershipDenied condition from the
// workflow build path too; previously it was computed and dropped, so a
// denied workflow workload degraded silently with no status condition.
@@ -2028,6 +2114,13 @@ export class WorkflowRecipeReconciler {
workloadStatuses: [],
internalDependencyConditions,
workloadConditions,
+ ...(error instanceof TransportNetworkReadinessPendingError
+ ? { transportNetworkConditions: pendingTransportNetworkConditions(recipe, message) }
+ : {}),
+ ...(error instanceof NetworkPolicyReplanRequiredError ||
+ error instanceof TransportNetworkReadinessPendingError
+ ? { requeueAfterMs: TRANSIENT_REQUEUE_BASE_MS }
+ : {}),
}
}
return {
@@ -2041,8 +2134,9 @@ export class WorkflowRecipeReconciler {
}
}
- console.log(
- `[WR-Reconciler] Workflow "${name}" first deploy (${recipe.spec.steps!.length} steps) — creating infrastructure`
+ createLogger('wrc', recipe.metadata.name).info(
+ 'Creating workflow infrastructure for first deployment',
+ { name, length: recipe.spec.steps!.length }
)
// Recovery handoff: when this workflow was latched `failed` by a transient
// infra blip, hand the inner reconciler a `recovering` execution phase
@@ -2092,6 +2186,7 @@ export class WorkflowRecipeReconciler {
clearWorkflowExecution: result.clearWorkflowExecution,
workflowConditions: result.workflowConditions,
workloadConditions: workflowWorkloadConditions,
+ transportNetworkConditions: workflowTransportNetworkConditions,
pluginWorkloadSdkBootstrapProof: result.pluginWorkloadSdkBootstrapProof,
pluginWorkloadSdkPolicyPending: result.pluginWorkloadSdkPolicyPending,
// Thread the inner reconciler's transient-skip up so the watcher leaves
@@ -2190,18 +2285,21 @@ export class WorkflowRecipeReconciler {
(recipe.status?.workloads?.length ?? 0) > 0 &&
(recipe.status?.workloads ?? []).every(w => w.ready === true)
if (latchedByTransientError) {
- console.log(
- `[WR-Reconciler] Recipe "${name}" is failed with a transient infra message — re-reconciling to self-heal`
+ createLogger('wrc', recipe.metadata.name).info(
+ 'Retrying a recipe failed by transient infrastructure',
+ { name }
)
}
if (latchedBySharedMcpInternalDependencyBoundary) {
- console.log(
- `[WR-Reconciler] Recipe "${name}" is failed by a shared mcp-server internal-dependency boundary decision — re-reconciling to self-heal`
+ createLogger('wrc', recipe.metadata.name).info(
+ 'Retrying a recipe failed by a shared MCP internal-dependency boundary',
+ { name }
)
}
if (latchedDespiteHealthyWorkloads) {
- console.log(
- `[WR-Reconciler] Recipe "${name}" is failed but all observed workloads are ready — re-reconciling to re-derive phase from live health`
+ createLogger('wrc', recipe.metadata.name).info(
+ 'Retrying a failed recipe whose observed workloads are ready',
+ { name }
)
}
if (
@@ -2210,9 +2308,10 @@ export class WorkflowRecipeReconciler {
!latchedBySharedMcpInternalDependencyBoundary &&
!latchedDespiteHealthyWorkloads
) {
- console.log(
- `[WR-Reconciler] Skipping non-deployable recipe "${name}" (phase: ${currentPhase})`
- )
+ createLogger('wrc', recipe.metadata.name).info('Skipping non-deployable recipe', {
+ name,
+ currentPhase,
+ })
return {
phase: currentPhase as RecipePhase,
message: recipe.status?.message ?? `Recipe is in ${currentPhase} state`,
@@ -2380,8 +2479,9 @@ export class WorkflowRecipeReconciler {
try {
await this.ensureOAuthBrokerTokenSecret(recipe)
} catch (err) {
- console.warn(
- `[WR-Reconciler] Broker-token issuance failed during reconcile for "${name}" (will retry via rotation loop): ${err instanceof Error ? err.message : String(err)}`
+ createLogger('wrc', recipe.metadata.name).warn(
+ 'Broker-token issuance failed during reconcile; the rotation loop will retry',
+ { name, err }
)
}
@@ -2422,12 +2522,16 @@ export class WorkflowRecipeReconciler {
secretKeys
)
if (preDeployedServers.length > 0) {
- console.log(
- `[WR-Reconciler] Pre-deployed ${preDeployedServers.length} McpServer(s) for network isolation`
+ createLogger('wrc', recipe.metadata.name).info(
+ 'Pre-deployed McpServers for network isolation',
+ { length: preDeployedServers.length }
)
}
} catch (error) {
- console.error(`[WR-Reconciler] Pre-deploy failed for "${name}":`, error)
+ createLogger('wrc', recipe.metadata.name).error('Recipe pre-deploy failed', {
+ name,
+ err: error,
+ })
// The pre-deploy McpServer handshake is an eventually-consistent
// wait on HCC reconciling the child McpServers + NetworkPolicies. A
// failure here means HCC has not caught up yet, NOT that the recipe
@@ -2447,7 +2551,9 @@ export class WorkflowRecipeReconciler {
// Step 7c: Wait for HCC to confirm network isolation
if (preDeployedServers.length > 0) {
await this.waitForTransportNetworkReadiness(recipe, preDeployedServers, mcpBatchNs)
- console.log(`[WR-Reconciler] Transport network readiness checked for MCP workloads`)
+ createLogger('wrc', recipe.metadata.name).info(
+ '[WR-Reconciler] Transport network readiness checked for MCP workloads'
+ )
}
if (!(await this.recipeStillActive(recipe))) {
@@ -2471,10 +2577,9 @@ export class WorkflowRecipeReconciler {
// instance and surface the denial so the operator labels the Secret.
await this.teardownDeniedWorkload(workload, recipe, secretKeys).catch(err => {
deniedTeardownFailed = true
- console.error(
- `[WR-Reconciler] Issue #637: teardown of denied workload "${workload.id}" failed; ` +
- `will requeue:`,
- err
+ createLogger('wrc', recipe.metadata.name).error(
+ 'Teardown of denied workload failed; will requeue',
+ { id: workload.id, err }
)
})
workloadStatuses.push({
@@ -2654,12 +2759,16 @@ export class WorkflowRecipeReconciler {
secretKeys
)
if (delegated.length > 0) {
- console.log(
- `[WR-Reconciler] Delegated ${delegated.length} MCP server(s): ${delegated.join(', ')}`
- )
+ createLogger('wrc', recipe.metadata.name).info('Delegated MCP servers', {
+ length: delegated.length,
+ value2: delegated.join(', '),
+ })
}
} catch (error) {
- console.error(`[WR-Reconciler] MCP delegation failed for "${name}":`, error)
+ createLogger('wrc', recipe.metadata.name).error('MCP delegation failed', {
+ name,
+ err: error,
+ })
// Same eventually-consistent handshake rationale as the pre-deploy
// step above: delegation depends on HCC having persisted the child
// McpServers/Context. A failure is transient (HCC not caught up),
@@ -2711,6 +2820,7 @@ export class WorkflowRecipeReconciler {
internalDependencyConditions,
secretOwnershipConditions: secretOwnership.conditions,
workloadConditions,
+ transportNetworkConditions: [],
}
}
@@ -2763,6 +2873,7 @@ export class WorkflowRecipeReconciler {
internalDependencyConditions,
secretOwnershipConditions: secretOwnership.conditions,
workloadConditions,
+ transportNetworkConditions: [],
pluginWorkloadSdkProviderUnavailable: sdkOnlyProviderUnavailable,
pluginWorkloadSdkPolicyPending: sdkOnlyPolicyPending,
pluginWorkloadSdkBootstrapProof: sdkOnlyRuntime?.pluginWorkloadSdkBootstrapProof,
@@ -2791,13 +2902,26 @@ export class WorkflowRecipeReconciler {
// every failure was transient; a permanent failure yields a plain
// Error that must stay terminal (fail-closed) below.
if (error instanceof RetryableReconcileError) {
- console.warn(
- `[WR-Reconciler] Transient failure reconciling "${name}" (will retry): ${error.message}`
+ createLogger('wrc', recipe.metadata.name).warn(
+ 'Transient reconciliation failure; will retry',
+ { name, err: error }
)
return {
phase: 'degraded',
message: error.message,
workloadStatuses: [],
+ ...(error instanceof TransportNetworkReadinessPendingError
+ ? {
+ transportNetworkConditions: pendingTransportNetworkConditions(
+ recipe,
+ error.message
+ ),
+ }
+ : {}),
+ ...(error instanceof NetworkPolicyReplanRequiredError ||
+ error instanceof TransportNetworkReadinessPendingError
+ ? { requeueAfterMs: TRANSIENT_REQUEUE_BASE_MS }
+ : {}),
}
}
// (2) A raw retryable infra error (connect ETIMEDOUT/ECONNRESET, 5xx,
@@ -2816,8 +2940,9 @@ export class WorkflowRecipeReconciler {
const isEgressResolutionError =
error instanceof Error && error.message.includes('egress resolution failed')
if (!isEgressResolutionError && isRetryableInfraError(error)) {
- console.warn(
- `[WR-Reconciler] Transient infra error reconciling "${name}" — keeping phase "${currentPhase}", will retry: ${String(error)}`
+ createLogger('wrc', recipe.metadata.name).warn(
+ 'Transient infrastructure error during reconciliation; keeping phase and retrying',
+ { name, currentPhase, err: error }
)
return {
phase: currentPhase as RecipePhase,
@@ -2835,7 +2960,10 @@ export class WorkflowRecipeReconciler {
requeueAfterMs: TRANSIENT_REQUEUE_BASE_MS,
}
}
- console.error(`[WR-Reconciler] Failed to reconcile "${name}":`, error)
+ createLogger('wrc', recipe.metadata.name).error('Recipe reconciliation failed', {
+ name,
+ err: error,
+ })
return {
phase: 'failed',
message: String(error),
@@ -3055,10 +3183,10 @@ export class WorkflowRecipeReconciler {
const readyReplicas = dep.status?.readyReplicas ?? 0
ready = readyReplicas >= 1
} catch (error) {
- console.warn(
- `[WR-Reconciler] Could not read webhook-gateway Deployment for "${name}":`,
- error
- )
+ createLogger('wrc', recipe.metadata.name).warn('Could not read webhook-gateway Deployment', {
+ name,
+ err: error,
+ })
}
if (ready) {
conditions.push(
@@ -3230,6 +3358,7 @@ export class WorkflowRecipeReconciler {
internalDependencyConditions: StatusCondition[]
secretOwnershipConditions: StatusCondition[]
workloadConditions: StatusCondition[]
+ transportNetworkConditions?: StatusCondition[]
}> {
const workloads = recipe.spec.workloads ?? []
if (workloads.length === 0)
@@ -3239,8 +3368,9 @@ export class WorkflowRecipeReconciler {
workloadConditions: [],
}
if (!(await this.recipeStillActive(recipe))) {
- console.warn(
- `[WR-Reconciler] Skipping workflow workload deploy for "${name}"; recipe is gone or deleting`
+ createLogger('wrc', recipe.metadata.name).warn(
+ 'Skipping workload deployment because the recipe is gone or deleting',
+ { name }
)
return {
internalDependencyConditions: [],
@@ -3283,7 +3413,10 @@ export class WorkflowRecipeReconciler {
secretKeys
)
} catch (err) {
- console.error(`[WR-Reconciler] Pre-deploy MCP failed for workflow "${name}":`, err)
+ createLogger('wrc', recipe.metadata.name).error('MCP pre-deploy failed for workflow', {
+ name,
+ err,
+ })
// Eventually-consistent HCC handshake (see the non-workflow pre-deploy
// step). Throw RetryableReconcileError so the workflow deploy path
// (which maps it to `degraded`) retries instead of failing the run.
@@ -3298,8 +3431,9 @@ export class WorkflowRecipeReconciler {
await this.waitForTransportNetworkReadiness(recipe, preDeployedServers, mcpBatchNs)
}
if (!(await this.recipeStillActive(recipe))) {
- console.warn(
- `[WR-Reconciler] Skipping workflow workload materialization for "${name}"; recipe is gone or deleting`
+ createLogger('wrc', recipe.metadata.name).warn(
+ 'Skipping workload materialization because the recipe is gone or deleting',
+ { name }
)
return {
internalDependencyConditions: [],
@@ -3315,8 +3449,9 @@ export class WorkflowRecipeReconciler {
try {
await this.ensureOAuthBrokerTokenSecret(recipe)
} catch (err) {
- console.warn(
- `[WR-Reconciler] Broker-token issuance failed during workflow reconcile for "${name}" (will retry via rotation loop): ${err instanceof Error ? err.message : String(err)}`
+ createLogger('wrc', recipe.metadata.name).warn(
+ 'Broker-token issuance failed during workflow reconciliation; rotation will retry',
+ { name, err }
)
}
@@ -3329,9 +3464,13 @@ export class WorkflowRecipeReconciler {
for (const workloadId of sortOrder) {
const workload = workloads.find(w => w.id === workloadId)!
if (secretOwnership.deniedWorkloadIds.has(workload.id)) {
- console.error(
- `[WR-Reconciler] EnvSecretOwnershipDenied for workflow workload "${workloadId}" in ` +
- `"${name}": ${secretOwnership.messageByWorkload.get(workloadId) ?? 'foreign Secret reference'}`
+ createLogger('wrc', recipe.metadata.name).error(
+ 'Secret ownership denied for workflow workload',
+ {
+ workloadId,
+ name,
+ value3: secretOwnership.messageByWorkload.get(workloadId) ?? 'foreign Secret reference',
+ }
)
try {
await this.teardownDeniedWorkload(workload, recipe, secretKeys)
@@ -3381,10 +3520,11 @@ export class WorkflowRecipeReconciler {
break
}
} catch (err) {
- console.error(
- `[WR-Reconciler] Failed to deploy workload "${workloadId}" for workflow "${name}":`,
- err
- )
+ createLogger('wrc', recipe.metadata.name).error('Failed to deploy workflow workload', {
+ workloadId,
+ name,
+ err,
+ })
if (workload.type === 'statefulset') {
throw err
}
@@ -3421,9 +3561,9 @@ export class WorkflowRecipeReconciler {
`Service "${svcName}"`
)
} catch (err) {
- console.error(
- `[WR-Reconciler] Service creation failed for workload "${workload.id}" in workflow "${name}":`,
- err
+ createLogger('wrc', recipe.metadata.name).error(
+ 'Service creation failed for workflow workload',
+ { id: workload.id, name, err }
)
}
}
@@ -3459,8 +3599,9 @@ export class WorkflowRecipeReconciler {
// Children live in `mcp-server` (mcpBatchNs), not the recipe's own ns.
if (hasTransport) {
if (!(await this.recipeStillActive(recipe))) {
- console.warn(
- `[WR-Reconciler] Skipping workflow MCP delegation for "${name}"; recipe is gone or deleting`
+ createLogger('wrc', recipe.metadata.name).warn(
+ 'Skipping MCP delegation because the recipe is gone or deleting',
+ { name }
)
return {
internalDependencyConditions,
@@ -3476,12 +3617,16 @@ export class WorkflowRecipeReconciler {
secretKeys
)
if (delegated.length > 0) {
- console.log(
- `[WR-Reconciler] MCP delegation for workflow "${name}": ${delegated.join(', ')}`
- )
+ createLogger('wrc', recipe.metadata.name).info('Delegated workflow MCP servers', {
+ name,
+ value2: delegated.join(', '),
+ })
}
} catch (err) {
- console.error(`[WR-Reconciler] MCP delegation failed for workflow "${name}":`, err)
+ createLogger('wrc', recipe.metadata.name).error('Workflow MCP delegation failed', {
+ name,
+ err,
+ })
// Same eventually-consistent HCC handshake as the non-workflow
// delegation step: degrade + retry instead of failing the run.
throw new RetryableReconcileError(
@@ -3496,6 +3641,7 @@ export class WorkflowRecipeReconciler {
internalDependencyConditions,
secretOwnershipConditions: secretOwnership.conditions,
workloadConditions: [],
+ transportNetworkConditions: [],
}
}
@@ -3504,7 +3650,7 @@ export class WorkflowRecipeReconciler {
async reconcileDelete(recipe: WorkflowRecipeCRD): Promise<void> {
const name = recipe.metadata.name
- console.log(`[WR-Reconciler] Deleting resources for "${name}"`)
+ createLogger('wrc', recipe.metadata.name).info('Deleting recipe resources', { name })
this.secretReverseIndex?.delete(name)
// ─── Workflow Delete (Stage 1) ────────────────────────────────────
@@ -3541,7 +3687,10 @@ export class WorkflowRecipeReconciler {
try {
await cleanupDelegation(this.delegationDeps, recipe, this.config.namespace)
} catch (error) {
- console.error(`[WR-Reconciler] Delegation cleanup failed for "${name}":`, error)
+ createLogger('wrc', recipe.metadata.name).error('Delegation cleanup failed', {
+ name,
+ err: error,
+ })
throw error
}
}
@@ -3583,10 +3732,11 @@ export class WorkflowRecipeReconciler {
completed.push('compute-pods(coordinator,mcp-host,snippet-runner)')
} catch (error) {
failures.push('compute-pods')
- console.error(
- `[WR-Reconciler] Terminal compute-pod teardown failed for "${name}" (run ${runId}):`,
- error
- )
+ createLogger('wrc', recipe.metadata.name).error('Terminal compute-pod teardown failed', {
+ name,
+ runId,
+ err: error,
+ })
}
}
@@ -3605,10 +3755,11 @@ export class WorkflowRecipeReconciler {
}
} catch (error) {
failures.push('mcp-server-delegation')
- console.error(
- `[WR-Reconciler] Terminal delegation cleanup failed for "${name}" (run ${runId}):`,
- error
- )
+ createLogger('wrc', recipe.metadata.name).error('Terminal delegation cleanup failed', {
+ name,
+ runId,
+ err: error,
+ })
}
// 3. WRC-owned transport Deployment(s) in mcp-server. See step 2 rationale.
@@ -3619,16 +3770,20 @@ export class WorkflowRecipeReconciler {
}
} catch (error) {
failures.push('transport-deployments')
- console.error(
- `[WR-Reconciler] Terminal transport-Deployment teardown failed for "${name}" (run ${runId}):`,
- error
+ createLogger('wrc', recipe.metadata.name).error(
+ 'Terminal transport Deployment teardown failed',
+ { name, runId, err: error }
)
}
- console.log(
- `[WR-Reconciler] Workflow "${name}" terminal (run ${runId}) — cleanup attempted [${completed.join('; ') || 'nothing'}]` +
- `${failures.length ? `; failed (left for archive-cron) [${failures.join(', ')}]` : ''}` +
- `; preserved artifact-reader + output PVC + output-anchor + CR + DB`
+ createLogger('wrc', recipe.metadata.name).info(
+ 'Terminal workflow cleanup attempted; artifacts, output resources, recipe and database retained',
+ {
+ name,
+ runId,
+ value3: completed.join('; ') || 'nothing',
+ value4: failures.length ? `; failed (left for archive-cron) [${failures.join(', ')}]` : '',
+ }
)
}
@@ -3700,7 +3855,9 @@ export class WorkflowRecipeReconciler {
// PVCs: honor clerum.io/pvc-retention annotation (default: "retain").
const pvcRetention = recipe.metadata.annotations?.['clerum.io/pvc-retention'] ?? 'retain'
if (await this.hasVerifiedInheritedParentResources(recipe)) {
- console.log(`[WR-Reconciler] Skipping inherited parent resources for child "${name}"`)
+ createLogger('wrc', recipe.metadata.name).info('Skipping inherited parent resources', {
+ name,
+ })
return
}
@@ -3748,8 +3905,9 @@ export class WorkflowRecipeReconciler {
`PVC "${res.id}" in ${resNs}`
)
} else {
- console.log(
- `[WR-Reconciler] Retaining PVC "${res.id}" (pvc-retention: ${pvcRetention})`
+ createLogger('wrc', recipe.metadata.name).info(
+ 'Retaining PVC according to retention policy',
+ { id: res.id, pvcRetention }
)
}
break
@@ -3849,22 +4007,31 @@ export class WorkflowRecipeReconciler {
if (idempotency) stampSpecHash(idempotency.manifest)
try {
await createFn()
- console.log(`[WR-Reconciler] Created ${label}`)
+ createLogger('wrc', 'workflow-recipes').info('Created resource', { label })
} catch (error: unknown) {
if (getErrorCode(error) === 409) {
if (idempotency && (await this.applyIsNoop(idempotency))) {
- console.log(`[WR-Reconciler] ${label} unchanged (spec-hash match); skipping update`)
+ createLogger('wrc', 'workflow-recipes').info(
+ 'Resource spec hash unchanged; skipping update',
+ { label }
+ )
return
}
try {
await replaceFn()
- console.log(`[WR-Reconciler] Updated ${label}`)
+ createLogger('wrc', 'workflow-recipes').info('Updated resource', { label })
} catch (updateError) {
- console.error(`[WR-Reconciler] Failed to update ${label}:`, updateError)
+ createLogger('wrc', 'workflow-recipes').error('Failed to update resource', {
+ label,
+ err: updateError,
+ })
throw updateError
}
} else {
- console.error(`[WR-Reconciler] Failed to create ${label}:`, error)
+ createLogger('wrc', 'workflow-recipes').error('Failed to create resource', {
+ label,
+ err: error,
+ })
throw error
}
}
@@ -4083,22 +4250,24 @@ export class WorkflowRecipeReconciler {
const desiredHash = stampSpecHash(statefulSet)
try {
await this.appsApi.createNamespacedStatefulSet({ namespace, body: statefulSet })
- console.log(`[WR-Reconciler] Created StatefulSet "${name}" in ${namespace}`)
+ createLogger('wrc', 'workflow-recipes').info('Created StatefulSet', { name, namespace })
return
} catch (error: unknown) {
if (getErrorCode(error) !== 409) {
- console.error(
- `[WR-Reconciler] Failed to create StatefulSet "${name}" in ${namespace}:`,
- error
- )
+ createLogger('wrc', 'workflow-recipes').error('Failed to create StatefulSet', {
+ name,
+ namespace,
+ err: error,
+ })
throw error
}
}
const existing = await this.appsApi.readNamespacedStatefulSet({ name, namespace })
if (existing.metadata?.annotations?.[SPEC_HASH_ANNOTATION] === desiredHash) {
- console.log(
- `[WR-Reconciler] StatefulSet "${name}" in ${namespace} unchanged (spec-hash match); skipping update`
+ createLogger('wrc', 'workflow-recipes').info(
+ 'StatefulSet spec hash unchanged; skipping update',
+ { name, namespace }
)
return
}
@@ -4126,18 +4295,27 @@ export class WorkflowRecipeReconciler {
}
)
const patchKind = patchBody.spec ? 'metadata+mutable spec' : 'metadata'
- console.log(`[WR-Reconciler] Patched StatefulSet "${name}" in ${namespace} (${patchKind})`)
+ createLogger('wrc', 'workflow-recipes').info('Patched StatefulSet', {
+ name,
+ namespace,
+ patchKind,
+ })
}
private async safeDelete(deleteFn: () => Promise<unknown>, label: string): Promise<void> {
try {
await deleteFn()
- console.log(`[WR-Reconciler] Deleted ${label}`)
+ createLogger('wrc', 'workflow-recipes').info('Deleted resource', { label })
} catch (error: unknown) {
if (getErrorCode(error) === 404) {
- console.log(`[WR-Reconciler] ${label} already gone`)
+ createLogger('wrc', 'workflow-recipes').info('Resource already gone', {
+ label,
+ })
} else {
- console.error(`[WR-Reconciler] Failed to delete ${label}:`, error)
+ createLogger('wrc', 'workflow-recipes').error('Failed to delete resource', {
+ label,
+ err: error,
+ })
}
}
}
@@ -4152,13 +4330,18 @@ export class WorkflowRecipeReconciler {
private async deleteOrThrow(deleteFn: () => Promise<unknown>, label: string): Promise<void> {
try {
await deleteFn()
- console.log(`[WR-Reconciler] Deleted ${label}`)
+ createLogger('wrc', 'workflow-recipes').info('Deleted resource', { label })
} catch (error: unknown) {
if (getErrorCode(error) === 404) {
- console.log(`[WR-Reconciler] ${label} already gone`)
+ createLogger('wrc', 'workflow-recipes').info('Resource already gone', {
+ label,
+ })
return
}
- console.error(`[WR-Reconciler] Failed to delete ${label}:`, error)
+ createLogger('wrc', 'workflow-recipes').error('Failed to delete resource', {
+ label,
+ err: error,
+ })
throw error
}
}
@@ -4184,16 +4367,22 @@ export class WorkflowRecipeReconciler {
existing = await read()
} catch (error: unknown) {
if (getErrorCode(error) === 404) {
- console.log(`[WR-Reconciler] ${label} already gone`)
+ createLogger('wrc', recipe.metadata.name).info('Resource already gone', {
+ label,
+ })
} else {
- console.error(`[WR-Reconciler] Skipping delete of ${label} — ownership read failed:`, error)
+ createLogger('wrc', recipe.metadata.name).error(
+ 'Skipping resource deletion because ownership could not be read',
+ { label, err: error }
+ )
}
return
}
const owner = existing.metadata?.labels?.['clerum.io/recipe']
if (owner !== recipe.metadata.name) {
- console.log(
- `[WR-Reconciler] Skipping delete of ${label} — owned by "${owner ?? 'unset'}", not "${recipe.metadata.name}" (issue #571)`
+ createLogger('wrc', recipe.metadata.name).info(
+ 'Skipping resource deletion because another recipe owns it',
+ { label, value2: owner ?? 'unset', name: recipe.metadata.name }
)
return
}
@@ -4318,10 +4507,15 @@ export class WorkflowRecipeReconciler {
// siblings) does not throw and freezes nothing — surface it so the missing
// egress until the next refresh is not silent.
if (failures.length > 0) {
- console.warn(
- `[WR-Reconciler] ${policyName}: ${failures.length} external egress FQDN(s) failed to resolve this round; policy written without them until the next refresh converges: ${failures
- .map(f => `${f.fqdn} (${f.retryable ? 'transient' : 'permanent'})`)
- .join(', ')}`
+ createLogger('wrc', recipe.metadata.name).warn(
+ 'External egress FQDN resolution failed; next refresh must converge the policy',
+ {
+ policyName,
+ length: failures.length,
+ value3: failures
+ .map(f => `${f.fqdn} (${f.retryable ? 'transient' : 'permanent'})`)
+ .join(', '),
+ }
)
}
this.warnEgressAccumulator(policyName, acc)
@@ -4380,9 +4574,10 @@ export class WorkflowRecipeReconciler {
// write when the persisted window is aging (renewalDue, audit M1), even if
// the set is unchanged, so a stable-then-rotated IP keeps its overlap grace.
if (!this.egressWriteNeeded(existing, policy) && !egressRenewalDue && !egressStateChanged) {
- console.log(
- `[WR-Reconciler] NetworkPolicy "${policyName}" in ${ns} egress set unchanged — no-op`
- )
+ createLogger('wrc', recipe.metadata.name).info('NetworkPolicy egress set unchanged; no-op', {
+ policyName,
+ ns,
+ })
} else {
await this.applyOwnedRecipeNetworkPolicy(policy, ns, recipe.metadata.name)
// #299: the policy has landed — record the set it actually enforces.
@@ -4695,8 +4890,9 @@ export class WorkflowRecipeReconciler {
if (!name || desiredNames.has(name)) continue
try {
await this.networkingApi.deleteNamespacedNetworkPolicy({ name, namespace })
- console.log(
- `[WR-Reconciler] Deleted stale internal-dependency NetworkPolicy "${name}" in ${namespace}`
+ createLogger('wrc', recipe.metadata.name).info(
+ 'Deleted stale internal-dependency NetworkPolicy',
+ { name, namespace }
)
} catch (error) {
if (getErrorCode(error) !== 404) throw error
@@ -4937,8 +5133,9 @@ export class WorkflowRecipeReconciler {
!wlEgressRenewalDue &&
!wlEgressStateChanged
) {
- console.log(
- `[WR-Reconciler] NetworkPolicy "${wlPolicyName}" in ${wlNs} egress set unchanged — no-op`
+ createLogger('wrc', recipe.metadata.name).info(
+ 'NetworkPolicy egress set unchanged; no-op',
+ { wlPolicyName, wlNs }
)
} else {
await this.applyOwnedRecipeNetworkPolicy(policy, wlNs, recipe.metadata.name)
@@ -5060,7 +5257,9 @@ export class WorkflowRecipeReconciler {
body: { preconditions: { uid, resourceVersion } },
})
} catch (error: unknown) {
- if (getErrorCode(error) !== 404) throw error
+ if (getErrorCode(error) !== 404) {
+ throwNetworkPolicyMutationError(error, policyName, namespace)
+ }
}
createLogger('wrc', recipeName).info('contracted recipe network policy', {
policy: policyName,
@@ -5072,11 +5271,15 @@ export class WorkflowRecipeReconciler {
if (!contractionWriteNeeded(existing, desired)) return existing
desired.metadata = { ...(desired.metadata ?? {}), resourceVersion }
- await this.networkingApi.replaceNamespacedNetworkPolicy({
- name: policyName,
- namespace,
- body: desired,
- })
+ try {
+ await this.networkingApi.replaceNamespacedNetworkPolicy({
+ name: policyName,
+ namespace,
+ body: desired,
+ })
+ } catch (error: unknown) {
+ throwNetworkPolicyMutationError(error, policyName, namespace)
+ }
let live: k8s.V1NetworkPolicy
try {
live = await this.networkingApi.readNamespacedNetworkPolicy({
@@ -5094,10 +5297,14 @@ export class WorkflowRecipeReconciler {
live.metadata?.name !== policyName ||
live.metadata?.namespace !== namespace ||
!liveUid ||
- !liveResourceVersion ||
- (existing.metadata?.uid && liveUid !== existing.metadata.uid)
+ !liveResourceVersion
) {
throw new NetworkPolicyOwnershipConflictError(
+ `NetworkPolicy "${policyName}" in ${namespace} has incomplete or inconsistent identity after contraction`
+ )
+ }
+ if (existing.metadata?.uid && liveUid !== existing.metadata.uid) {
+ throw new NetworkPolicyReplanRequiredError(
`NetworkPolicy "${policyName}" in ${namespace} changed identity during contraction`
)
}
@@ -5106,11 +5313,16 @@ export class WorkflowRecipeReconciler {
// The apiserver read-back is broader or carries provenance that does not
// match its enforced rules. Do not let phase 2 treat the request body as
// reality; revoke this exact owned identity and force a clean re-plan.
- await this.networkingApi.deleteNamespacedNetworkPolicy({
- name: policyName,
- namespace,
- body: { preconditions: { uid: liveUid, resourceVersion: liveResourceVersion } },
- })
+ try {
+ await this.networkingApi.deleteNamespacedNetworkPolicy({
+ name: policyName,
+ namespace,
+ body: { preconditions: { uid: liveUid, resourceVersion: liveResourceVersion } },
+ })
+ } catch (error: unknown) {
+ if (getErrorCode(error) !== 404)
+ throwNetworkPolicyMutationError(error, policyName, namespace)
+ }
throw new NetworkPolicyOwnershipConflictError(
`NetworkPolicy "${policyName}" in ${namespace} did not read back as the requested contraction`
)
@@ -5133,10 +5345,21 @@ export class WorkflowRecipeReconciler {
await this.createOrReplace(
() => this.networkingApi.createNamespacedNetworkPolicy({ namespace, body: policy }),
async () => {
- const live = await this.networkingApi.readNamespacedNetworkPolicy({
- name: policyName,
- namespace,
- })
+ let live: k8s.V1NetworkPolicy
+ try {
+ live = await this.networkingApi.readNamespacedNetworkPolicy({
+ name: policyName,
+ namespace,
+ })
+ } catch (error: unknown) {
+ if (getErrorCode(error) === 404) {
+ throw new NetworkPolicyReplanRequiredError(
+ `NetworkPolicy "${policyName}" in ${namespace} disappeared after create conflict; a fresh reconciliation is required`,
+ { cause: error }
+ )
+ }
+ throw error
+ }
this.assertRecipeNetworkPolicyOwnership(live, recipeName, policyName, namespace)
if (!live.metadata?.resourceVersion) {
throw new NetworkPolicyOwnershipConflictError(
@@ -5147,11 +5370,17 @@ export class WorkflowRecipeReconciler {
...(policy.metadata ?? {}),
resourceVersion: live.metadata?.resourceVersion,
}
- return this.networkingApi.replaceNamespacedNetworkPolicy({
- name: policyName,
- namespace,
- body: policy,
- })
+ try {
+ return await this.networkingApi.replaceNamespacedNetworkPolicy({
+ name: policyName,
+ namespace,
+ body: policy,
+ })
+ } catch (error: unknown) {
+ // This PUT is fenced by the freshly checked owner/resourceVersion.
+ // Losing that snapshot is unfinished convergence, not bad user intent.
+ throwNetworkPolicyMutationError(error, policyName, namespace)
+ }
},
`NetworkPolicy "${policyName}" in ${namespace}`
)
@@ -5222,17 +5451,15 @@ export class WorkflowRecipeReconciler {
*/
private warnEgressAccumulator(policyName: string, acc: AccumulateOutput): void {
if (acc.frozenFqdns.length > 0) {
- console.warn(
- `[WR-Reconciler] ${policyName}: egress set FROZEN (fail-static) for ${acc.frozenFqdns.join(
- ', '
- )} — DNS resolution is transiently failing; serving last-known IPs, not pruning.`
+ createLogger('wrc', 'workflow-recipes').warn(
+ 'External egress frozen during transient DNS failure; serving last accepted IPs',
+ { policyName, value2: acc.frozenFqdns.join(', ') }
)
}
if (acc.overCap) {
- console.warn(
- `[WR-Reconciler] ${policyName}: egress set hit the maxEntries cap — evicted ${acc.evicted.length} least-recently-observed entr${
- acc.evicted.length === 1 ? 'y' : 'ies'
- } (never rejecting the policy).`
+ createLogger('wrc', 'workflow-recipes').warn(
+ 'External egress exceeded maxEntries; evicted least-recently-observed entries',
+ { policyName, length: acc.evicted.length, value3: acc.evicted.length === 1 ? 'y' : 'ies' }
)
}
}
@@ -5607,12 +5834,14 @@ export class WorkflowRecipeReconciler {
)
combined = combined ? combineSecretAccess(combined, access) : access
if (access.state === 'denied') {
- console.error(
- `[WR-Reconciler] Refusing to project Secret "${name}" (ns ${secretNs}) into recipe "${recipe.metadata.name}" — not owned or shared; label clerum.io/shared=true or clerum.io/owner-recipe=${recipe.metadata.name}`
+ createLogger('wrc', recipe.metadata.name).error(
+ 'Refusing to project a Secret that is neither owned nor shared',
+ { name, secretNs, name3: recipe.metadata.name, name4: recipe.metadata.name }
)
} else if (access.state === 'error') {
- console.warn(
- `[WR-Reconciler] Secret "${name}" (ns ${secretNs}) ownership could not be verified for recipe "${recipe.metadata.name}" — failing closed and requeuing`
+ createLogger('wrc', recipe.metadata.name).warn(
+ 'Secret ownership could not be verified; failing closed and requeueing',
+ { name, secretNs, name3: recipe.metadata.name }
)
}
}
@@ -5731,8 +5960,9 @@ export class WorkflowRecipeReconciler {
// Read failed — do NOT silently skip (that would leave a foreign Secret
// projected with a healthy status). Requeue and retry, matching the deploy
// path's fail-closed contract for an unverifiable Secret.
- console.warn(
- `[WR-Reconciler] Issue #637: ownership recheck read failed for "${recipe.metadata.name}" — requeuing: ${String(error)}`
+ createLogger('wrc', recipe.metadata.name).warn(
+ 'Secret ownership recheck failed; requeueing',
+ { name: recipe.metadata.name, err: error }
)
return { ...clean, needsRequeue: true, requeueReason: 'Secret read failed' }
}
@@ -5755,11 +5985,13 @@ export class WorkflowRecipeReconciler {
const failedTeardowns: string[] = []
for (const workload of recipe.spec.workloads ?? []) {
if (!secretOwnership.deniedWorkloadIds.has(workload.id)) continue
- console.warn(
- `[WR-Reconciler] Issue #637: revoking steady workflow workload "${workload.id}" of ` +
- `"${recipe.metadata.name}" — ${
- secretOwnership.messageByWorkload.get(workload.id) ?? 'foreign Secret ref'
- }`
+ createLogger('wrc', recipe.metadata.name).warn(
+ 'Revoking steady workflow workload because Secret access was denied',
+ {
+ id: workload.id,
+ name: recipe.metadata.name,
+ value3: secretOwnership.messageByWorkload.get(workload.id) ?? 'foreign Secret ref',
+ }
)
try {
await this.teardownDeniedWorkload(workload, recipe, secretKeys)
@@ -5769,9 +6001,9 @@ export class WorkflowRecipeReconciler {
// the next pass retries; the EnvSecretOwnershipDenied condition is written
// only once teardown actually succeeds.
failedTeardowns.push(workload.id)
- console.error(
- `[WR-Reconciler] Issue #637: teardown FAILED for denied workload "${workload.id}" of ` +
- `"${recipe.metadata.name}" — requeuing: ${String(error)}`
+ createLogger('wrc', recipe.metadata.name).error(
+ 'Denied workload teardown failed; requeueing',
+ { id: workload.id, name: recipe.metadata.name, err: error }
)
}
}
@@ -5888,9 +6120,9 @@ export class WorkflowRecipeReconciler {
namespace,
labelSelector: `app=${resourceName}`,
})
- console.log(
- `[WR-Reconciler] Issue #637: deleted StatefulSet "${resourceName}" managed Pods in ` +
- `${namespace} (Secret ownership revoked); PVCs and StatefulSet preserved`
+ createLogger('wrc', recipe.metadata.name).info(
+ 'Deleted StatefulSet pods after Secret ownership revocation; preserving StatefulSet and PVCs',
+ { resourceName, namespace }
)
} catch (error) {
if (getErrorCode(error) !== 404) throw error
@@ -6249,7 +6481,18 @@ export class WorkflowRecipeReconciler {
}
}
- return this.workloadStatusResult(workloadStatuses)
+ const result = this.workloadStatusResult(workloadStatuses)
+ if (hasPendingTransportNetworkReadiness(recipe)) {
+ const condition = recipe.status?.conditions?.find(
+ condition => condition.type === TRANSPORT_NETWORK_CONDITION_TYPE
+ )
+ return {
+ ...result,
+ phase: 'degraded',
+ message: condition?.message ?? 'Transport external egress admission is pending',
+ }
+ }
+ return result
}
private workloadStatusResult(
@@ -6455,7 +6698,10 @@ export class WorkflowRecipeReconciler {
// PVCs: create only (never replace — immutable after creation)
try {
await this.coreApi.createNamespacedPersistentVolumeClaim({ namespace: ns, body: manifest })
- console.log(`[WR-Reconciler] Created PVC "${res.id}" in ${ns}`)
+ createLogger('wrc', recipe.metadata.name).info('Created PVC', {
+ id: res.id,
+ ns,
+ })
} catch (error: unknown) {
if (getErrorCode(error) === 409) {
const existing = await this.coreApi.readNamespacedPersistentVolumeClaim({
@@ -6463,7 +6709,10 @@ export class WorkflowRecipeReconciler {
namespace: ns,
})
this.assertExistingResourcePvcOwnedByRecipe(existing, manifest, res, recipe, ns)
- console.log(`[WR-Reconciler] PVC "${res.id}" already exists in ${ns} (owned, skip)`)
+ createLogger('wrc', recipe.metadata.name).info(
+ 'Owned PVC already exists; skipping creation',
+ { id: res.id, ns }
+ )
} else {
throw error
}
@@ -6617,16 +6866,19 @@ export class WorkflowRecipeReconciler {
{ name: secretName, namespace: ns, body: { data: manifest.data } },
{ middleware: [k8s.setHeaderMiddleware('Content-Type', 'application/merge-patch+json')] }
)
- console.log(`[WR-Reconciler] Refreshed Secret "${secretName}" in ${ns}`)
+ createLogger('wrc', recipe.metadata.name).info('Refreshed Secret', { secretName, ns })
return
}
try {
await this.coreApi.createNamespacedSecret({ namespace: ns, body: manifest })
- console.log(`[WR-Reconciler] Created Secret "${secretName}" in ${ns}`)
+ createLogger('wrc', recipe.metadata.name).info('Created Secret', { secretName, ns })
} catch (err) {
if (getErrorCode(err) !== 409) throw err
- console.log(`[WR-Reconciler] Secret "${secretName}" already exists in ${ns} (skip)`)
+ createLogger('wrc', recipe.metadata.name).info('Secret already exists; skipping creation', {
+ secretName,
+ ns,
+ })
}
}
@@ -6830,6 +7082,22 @@ export class WorkflowRecipeReconciler {
recipe.status?.conditions,
result.workloadConditions
)
+ const priorTransportConditions =
+ workloadReconcileMergedConditions ??
+ secretOwnershipMergedConditions ??
+ internalDependencyMergedConditions ??
+ workflowOutputMergedConditions ??
+ webhookMergedConditions ??
+ recipe.status?.conditions
+ // Observers and earlier failures cannot discharge an admission requirement.
+ const transportNetworkMergedConditions =
+ result.transportNetworkConditions === undefined
+ ? undefined
+ : mergeOwnedConditions(
+ priorTransportConditions,
+ result.transportNetworkConditions,
+ TRANSPORT_NETWORK_CONDITION_TYPES
+ )
// Plugin Workload SDK conditions are derived so every status patch carries a
// consistent projection of spec.pluginWorkloadSdk + feature flag, while the
// SDK-only provider health bit is propagated explicitly through
@@ -6841,7 +7109,8 @@ export class WorkflowRecipeReconciler {
const pluginSdkProjection =
result.pluginWorkloadSdkProjection ?? this.projectPluginWorkloadSdk(recipe, result, now)
const pluginSdkMergedConditions = mergePluginWorkloadSdkConditions(
- workloadReconcileMergedConditions ??
+ transportNetworkMergedConditions ??
+ workloadReconcileMergedConditions ??
secretOwnershipMergedConditions ??
internalDependencyMergedConditions ??
workflowOutputMergedConditions ??
@@ -6851,6 +7120,7 @@ export class WorkflowRecipeReconciler {
)
const mergedConditions =
pluginSdkMergedConditions ??
+ transportNetworkMergedConditions ??
workloadReconcileMergedConditions ??
secretOwnershipMergedConditions ??
internalDependencyMergedConditions ??
@@ -6894,9 +7164,10 @@ export class WorkflowRecipeReconciler {
{ middleware: [k8s.setHeaderMiddleware('Content-Type', 'application/merge-patch+json')] }
)
- console.log(
- `[WR-Reconciler] Status patched for "${recipe.metadata.name}": phase=${result.phase}`
- )
+ createLogger('wrc', recipe.metadata.name).info('Patched recipe status', {
+ name: recipe.metadata.name,
+ phase: result.phase,
+ })
}
// ─── UUID Instance Assignment ──────────────────────────────────────
@@ -6947,9 +7218,10 @@ export class WorkflowRecipeReconciler {
},
{ middleware: [k8s.setHeaderMiddleware('Content-Type', 'application/merge-patch+json')] }
)
- console.log(
- `[WR-Reconciler] Assigned workload instances for "${recipe.metadata.name}": ${JSON.stringify(instances)}`
- )
+ createLogger('wrc', recipe.metadata.name).info('Assigned workload instances', {
+ name: recipe.metadata.name,
+ value2: JSON.stringify(instances),
+ })
}
return newlyAssigned
@@ -6991,8 +7263,9 @@ export class WorkflowRecipeReconciler {
) {
// Logged once (only reached when no instance is recorded yet) — the single
// observable marker that a fleet migration re-scoped this workload.
- console.log(
- `[WR-Reconciler] Migrating workload "${workload.id}" of recipe "${recipe.metadata.name}" to scoped name "${scopedName}" (issue #571)`
+ createLogger('wrc', recipe.metadata.name).info(
+ 'Migrating workload to its scoped resource name',
+ { id: workload.id, name: recipe.metadata.name, scopedName }
)
return scopedName
}
@@ -7061,9 +7334,10 @@ export class WorkflowRecipeReconciler {
},
{ middleware: [k8s.setHeaderMiddleware('Content-Type', 'application/merge-patch+json')] }
)
- console.log(
- `[WR-Reconciler] Assigned ${Object.keys(instances).length} resource instance(s) for "${recipe.metadata.name}"`
- )
+ createLogger('wrc', recipe.metadata.name).info('Assigned resource instances', {
+ length: Object.keys(instances).length,
+ name: recipe.metadata.name,
+ })
}
return newlyAssigned
@@ -7225,8 +7499,9 @@ export class WorkflowRecipeReconciler {
if (!observedStatus?.ready) {
if (anyOwned) {
cleanupPending = true
- console.log(
- `[WR-Reconciler] Legacy StatefulSet cleanup for ${namespace}/${workload.id} deferred until scoped workload is ready`
+ createLogger('wrc', recipe.metadata.name).info(
+ 'Deferring legacy StatefulSet cleanup until scoped workload is ready',
+ { namespace, id: workload.id }
)
}
continue
@@ -7366,8 +7641,9 @@ export class WorkflowRecipeReconciler {
if (!observedStatus?.ready) {
if (anyOwned) {
cleanupPending = true
- console.log(
- `[WR-Reconciler] Legacy Deployment cleanup for ${namespace}/${workload.id} deferred until scoped workload is ready`
+ createLogger('wrc', recipe.metadata.name).info(
+ 'Deferring legacy Deployment cleanup until scoped workload is ready',
+ { namespace, id: workload.id }
)
}
continue
@@ -7458,7 +7734,9 @@ export class WorkflowRecipeReconciler {
): Promise<void> {
try {
await deleteFn()
- console.log(`[WR-Reconciler] Deleted ${description}`)
+ createLogger('wrc', 'workflow-recipes').info('Deleted workload resource', {
+ description,
+ })
} catch (err: unknown) {
if (getErrorCode(err) === 404) {
return
diff --git a/workflow-recipes/tests/unit/observability/logger.test.ts b/workflow-recipes/tests/unit/observability/logger.test.ts
index 8066ce82c..4e076bcd5 100644
--- a/workflow-recipes/tests/unit/observability/logger.test.ts
+++ b/workflow-recipes/tests/unit/observability/logger.test.ts
@@ -1,4 +1,6 @@
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
+import { ApiException } from '@kubernetes/client-node'
+import { runInNewContext } from 'node:vm'
import { createLogger } from '../../../src/observability/logger'
describe('createLogger', () => {
@@ -89,6 +91,150 @@ describe('createLogger', () => {
expect(entry.durationMs).toBe(340)
})
+ it('serializes native Error diagnostics and cause into the emitted JSON', () => {
+ const cause = new Error('upstream unavailable')
+ const err = Object.assign(new TypeError('network policy readback failed', { cause }), {
+ code: 503,
+ })
+ createLogger('wrc', 'r').error('reconcile failed', { err })
+ const entry = JSON.parse(stdoutSpy.mock.calls[0][0])
+ expect(entry.err).toMatchObject({
+ name: 'TypeError',
+ message: 'network policy readback failed',
+ code: 503,
+ cause: { name: 'Error', message: 'upstream unavailable' },
+ })
+ expect(entry.err.stack).toContain('TypeError: network policy readback failed')
+ expect(entry.err.cause.stack).toContain('Error: upstream unavailable')
+ })
+
+ it('serializes Error values inside objects and arrays', () => {
+ createLogger('wrc', 'r').error('batch failed', {
+ results: [{ err: new Error('first failed') }, new Error('second failed')],
+ })
+ const entry = JSON.parse(stdoutSpy.mock.calls[0][0])
+ expect(entry.results[0].err.message).toBe('first failed')
+ expect(entry.results[1].message).toBe('second failed')
+ })
+
+ it('bounds cause depth and terminates cyclic Error causes', () => {
+ const cycle = new Error('cyclic cause')
+ cycle.cause = cycle
+ let deep = new Error('deepest cause')
+ for (let index = 0; index < 20; index++) deep = new Error(`cause ${index}`, { cause: deep })
+ createLogger('wrc', 'r').error('failed', { cycle, deep })
+ const entry = JSON.parse(stdoutSpy.mock.calls[0][0])
+ expect(entry.cycle.cause).toBe('[Circular]')
+ let current = entry.deep
+ let depth = 0
+ while (current && typeof current === 'object') {
+ current = current.cause
+ depth++
+ }
+ expect(depth).toBeLessThanOrEqual(6)
+ expect(current).toBe('[Truncated]')
+ })
+
+ it('redacts sensitive Error properties without losing ordinary diagnostics', () => {
+ // Single-character inert fixtures test field redaction; none is an access value.
+ const err = Object.assign(new Error('request failed'), {
+ authorization: 'x',
+ body: { data: 'x' },
+ safe: 'retained',
+ cause: Object.assign(new Error('upstream unavailable'), { refreshToken: 'x' }),
+ })
+ createLogger('wrc', 'r').error('failed', { err })
+ const entry = JSON.parse(stdoutSpy.mock.calls[0][0])
+ expect(entry.err.message).toBe('request failed')
+ expect(entry.err.authorization).toBe('[REDACTED]')
+ expect(entry.err.body).toBe('[REDACTED]')
+ expect(entry.err.cause.refreshToken).toBe('[REDACTED]')
+ expect(entry.err.safe).toBe('retained')
+ })
+
+ it.each([false, true])(
+ 'removes duplicated ApiException body and headers from diagnostics (wrapped=%s)',
+ wrapped => {
+ // Real client exception, with single-character inert values only.
+ const body = { password: 'x', unrelatedBodyField: 'x' }
+ const headers = { authorization: 'x', unrelatedHeader: 'x' }
+ const apiError = new ApiException(503, 'network policy read failed', body, headers)
+ const err = wrapped ? new Error('reconciliation failed', { cause: apiError }) : apiError
+ createLogger('wrc', 'r').error('failed', { err })
+ const entry = JSON.parse(stdoutSpy.mock.calls[0][0])
+ const diagnostic = wrapped ? entry.err.cause : entry.err
+ expect(diagnostic.code).toBe(503)
+ expect(diagnostic.name).toBe('Error')
+ expect(diagnostic.message).toContain('network policy read failed')
+ expect(diagnostic.message).toContain('HTTP-Code: 503')
+ expect(diagnostic.stack).toContain('at ')
+ expect(diagnostic.body).toBe('[REDACTED]')
+ expect(diagnostic.headers).toBe('[REDACTED]')
+ for (const text of [diagnostic.message, diagnostic.stack]) {
+ expect(text).not.toContain('unrelatedBodyField')
+ expect(text).not.toContain('unrelatedHeader')
+ expect(text).not.toContain(JSON.stringify(body))
+ expect(text).not.toContain(JSON.stringify(headers))
+ }
+ }
+ )
+
+ it.each(['password', 'api_key', 'authorization', 'cookie'])(
+ 'redacts quoted and escaped JSON %s keys in generic Error diagnostics',
+ key => {
+ // Serialization represents actual plain/escaped JSON diagnostics. The value
+ // stays an inert single character; no credential is generated or encoded.
+ for (const json of [
+ JSON.stringify({ [key]: 'x' }),
+ JSON.stringify(JSON.stringify({ [key]: 'x' })),
+ ]) {
+ stdoutSpy.mockClear()
+ const err = new Error(`operation failed: ${json}`)
+ createLogger('wrc', 'r').error('failed', { err })
+ const diagnostic = JSON.parse(stdoutSpy.mock.calls[0][0]).err
+ expect(diagnostic.message).toContain('operation failed')
+ expect(diagnostic.message).toContain('[REDACTED]')
+ expect(diagnostic.message).not.toContain('x')
+ expect(diagnostic.stack).toContain('[REDACTED]')
+ expect(diagnostic.stack).not.toContain(json)
+ }
+ }
+ )
+
+ it('serializes native errors from another JavaScript context', () => {
+ const err: unknown = runInNewContext('new Error("isolated operation failed")')
+ createLogger('wrc', 'r').error('failed', { err })
+ expect(JSON.parse(stdoutSpy.mock.calls[0][0]).err.message).toBe('isolated operation failed')
+ })
+
+ it('bounds oversized diagnostic strings before serializing', () => {
+ const err = new Error('a'.repeat(20_000))
+ createLogger('wrc', 'r').error('failed', { err })
+ const entry = JSON.parse(stdoutSpy.mock.calls[0][0])
+ expect(entry.err.message).toBe('[Truncated]')
+ expect(entry.err.stack).toBe('[Truncated]')
+ })
+
+ it('does not throw on a custom Error accessor or a cyclic cause array', () => {
+ const unreadable = new Error('unreadable cause')
+ Object.defineProperty(unreadable, 'cause', {
+ get() {
+ throw new Error('accessor failed')
+ },
+ })
+ const cycle: unknown[] = []
+ cycle.push(cycle)
+ expect(() =>
+ createLogger('wrc', 'r').error('failed', {
+ unreadable,
+ err: new Error('array cause', { cause: cycle }),
+ })
+ ).not.toThrow()
+ const entry = JSON.parse(stdoutSpy.mock.calls[0][0])
+ expect(entry.unreadable).toEqual({ name: 'Error', message: '[Unserializable]' })
+ expect(entry.err.cause).toEqual(['[Circular]'])
+ })
+
it('redacts sensitive top-level and nested fields', () => {
const log = createLogger('mcp_host', 'r')
log.info('request', {