Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
Merge origin/dev into fix/issue-575-wrc-networkpolicy-spec-hash-gate
Alfredo Lopez32087f6ef3df
diff --cc workflow-recipes/package-lock.jsonindex fde522079,8ac088539..a6a434abc--- a/workflow-recipes/package-lock.json+++ b/workflow-recipes/package-lock.json@@@ -1,12 -1,12 +1,12 @@@{"name": "workload-recipes-controller",- "version": "0.1.346",- "version": "0.1.345",++ "version": "0.1.347","lockfileVersion": 3,"requires": true,"packages": {"": {"name": "workload-recipes-controller",- "version": "0.1.346",- "version": "0.1.345",++ "version": "0.1.347","license": "SEE LICENSE IN LICENSE","dependencies": {"@aws-sdk/client-s3": "^3.1009.0",diff --cc workflow-recipes/package.jsonindex af9e20fcd,8d0e99308..a00abeb80--- a/workflow-recipes/package.json+++ b/workflow-recipes/package.json@@@ -1,6 -1,6 +1,6 @@@{"name": "workload-recipes-controller",- "version": "0.1.346",- "version": "0.1.345",++ "version": "0.1.347","description": "Workload Recipes Controller (WRC) - WorkloadRecipe lifecycle management and MCP server interface","main": "dist/main.js","engines": {diff --cc workflow-recipes/src/k8sClient.test.tsindex 62f474273,5a3e71402..af2d4f568--- a/workflow-recipes/src/k8sClient.test.ts+++ b/workflow-recipes/src/k8sClient.test.ts@@@ -1397,75 -1466,55 +1466,124 @@@ describe('workload status refresh loopexpect(reconcile).not.toHaveBeenCalled()})+ it('retries a transient degraded result even when the first pass patches status', async () => {+ vi.useFakeTimers()+ try {+ const recipe = makeWorkloadRecipe({+ metadata: { name: 'policy-race', namespace: 'sandbox-recipes', generation: 1 },+ status: { phase: 'candidate' },+ })+ const reconcile = vi+ .fn()+ .mockResolvedValueOnce({+ phase: 'degraded',+ message: 'NetworkPolicy changed during replace',+ workloadStatuses: [],+ requeueAfterMs: 5_000,+ })+ .mockResolvedValueOnce({+ phase: 'active',+ message: 'All workloads deployed',+ workloadStatuses: [],+ })+ const patchStatus = vi.fn().mockResolvedValue(undefined)+ const enqueue = vi.fn((_key: string, task: () => Promise<void>) => task())++ type InternalWatcher = {+ recipes: Map<string, WorkflowRecipeCRD>+ transientRetries: Map<string, { timer: ReturnType<typeof setTimeout>; attempts: number }>+ eventQueue: { enqueue: typeof enqueue }+ stopped: boolean+ traceReporter: null+ dbRunProcessor: null+ reconciler: {+ reconcile: typeof reconcile+ isRecipeStillActive: ReturnType<typeof vi.fn>+ ensureFinalizer: ReturnType<typeof vi.fn>+ patchStatus: typeof patchStatus+ }+ handleRecipeEvent: (type: string, recipe: WorkflowRecipeCRD) => Promise<void>+ }++ const internal = Object.create(WorkflowRecipeWatcher.prototype) as InternalWatcher+ internal.recipes = new Map([[recipe.metadata.name, recipe]])+ internal.transientRetries = new Map()+ internal.eventQueue = { enqueue }+ internal.stopped = false+ internal.traceReporter = null+ internal.dbRunProcessor = null+ internal.reconciler = {+ reconcile,+ isRecipeStillActive: vi.fn().mockResolvedValue(true),+ ensureFinalizer: vi.fn().mockResolvedValue(undefined),+ patchStatus,+ }++ await internal.handleRecipeEvent('ADDED', recipe)++ expect(patchStatus).toHaveBeenCalledTimes(1)+ expect(internal.transientRetries.has(recipe.metadata.name)).toBe(true)++ await vi.advanceTimersByTimeAsync(5_000)+ await Promise.resolve()++ expect(enqueue).toHaveBeenCalledWith(recipe.metadata.name, expect.any(Function))+ expect(reconcile).toHaveBeenCalledTimes(2)+ expect(internal.transientRetries.has(recipe.metadata.name)).toBe(false)+ } finally {+ vi.useRealTimers()+ }+ })++ it.each([+ { from: 'failed', to: 'candidate', runs: 1 },+ { from: 'active', to: 'active', runs: 0 },+ { from: 'active', to: 'degraded', runs: 0 },+ { from: 'degraded', to: 'active', runs: 0 },+ { from: 'failed', to: 'failed', runs: 0 },+ { from: 'candidate', to: 'candidate', runs: 0 },+ ] as const)(+ 'handles the supported retry transition $from -> $to without a generation bump',+ async ({ from, to, runs }) => {+ const cached = makeWorkloadRecipe({+ metadata: { name: 'retry-recipe', namespace: 'sandbox-recipes', generation: 7 },+ status: { phase: from },+ })+ const requested = {+ ...cached,+ status: { phase: to, message: 'Manual retry requested by operator' },+ }+ const reconcile = vi.fn().mockResolvedValue({+ phase: 'active',+ message: 'All workloads deployed',+ workloadStatuses: [],+ })+ const watcher = Object.create(WorkflowRecipeWatcher.prototype) as {+ recipes: Map<string, WorkflowRecipeCRD>+ handleRecipeEvent: (type: string, recipe: WorkflowRecipeCRD) => Promise<void>+ }+ Object.assign(watcher, {+ recipes: new Map([[cached.metadata.name, cached]]),+ transientRetries: new Map(),+ stopped: false,+ traceReporter: null,+ dbRunProcessor: null,+ reconciler: {+ reconcile,+ isRecipeStillActive: vi.fn().mockResolvedValue(true),+ ensureFinalizer: vi.fn().mockResolvedValue(undefined),+ patchStatus: vi.fn().mockResolvedValue(undefined),+ },+ })+ await watcher.handleRecipeEvent('MODIFIED', requested)+ expect(reconcile).toHaveBeenCalledTimes(runs)+ expect(watcher.recipes.get(cached.metadata.name)).toEqual(requested)+ // Duplicate status events must not repeatedly execute the retry transition.+ await watcher.handleRecipeEvent('MODIFIED', requested)+ expect(reconcile).toHaveBeenCalledTimes(runs)+ }+ )+it('reconciles when Codex connection-ref changes without a generation bump', async () => {const cached = makeWorkloadRecipe({metadata: { name: 'active-recipe', namespace: 'sandbox-recipes', generation: 7 },diff --cc workflow-recipes/src/reconciler/reconciler.test.tsindex 11747dabe,b82cb9be2..5e5bd0395--- a/workflow-recipes/src/reconciler/reconciler.test.ts+++ b/workflow-recipes/src/reconciler/reconciler.test.ts@@@ -9,8 -9,9 +9,10 @@@ import { WorkflowRecipeWatcher, shouldPimport { CronJobDef, WorkflowRecipeCRD, WorkflowRecipeGfsIntentSpec, WorkloadDef } from '../types'import { INHERITED_PARENT_RESOURCES_ANNOTATION } from '../workflow/childRecipeFactory'import { buildCoordinatorGfsNetworkPolicy } from '../workflow/networkPolicyFactory'+ import { captureLogger } from './__tests__/captureLogger'+ import { defaultFqdnLookup } from './fqdnResolver'import { isRetryableInfraError } from './k8sErrors'+import type { NetworkPolicyFamily } from './networkPolicyConvergence'import * as brokerIssuer from './oauthBrokerTokenIssuerClient'import { PLUGIN_WORKLOAD_SDK_PROVIDER_UNAVAILABLE_CONDITION_TYPE } from './pluginWorkloadSdkValidator'import {@@@ -254,6 -304,612 +298,615 @@@ describe('WorkflowRecipeReconciler', (// ─── Pipeline Tests ─────────────────────────────────────────────+ describe('transport admission pending survives observation and restart', () => {+ const conditionType = 'TransportExternalEgressReady'+ const pendingCondition = {+ type: conditionType,+ status: 'False' as const,+ reason: 'ExternalEgressPending',+ message: 'Transport external egress admission is pending',+ lastTransitionTime: '2026-09-05T00:00:00.000Z',+ }+ function transportRecipe(workflow = false): WorkflowRecipeCRD {+ return makeRecipe({+ metadata: {+ name: 'test-recipe',+ namespace: 'sandbox-recipes',+ uid: 'uid-123',+ generation: 7,+ },+ spec: {+ ...(workflow ? {} : { contextRef: 'context1' }),+ workloads: [+ {+ id: 'calculator',+ type: 'deployment',+ image: 'clerum/mock-stdio-mcp-server:test',+ port: 3000,+ transport: { type: 'stdio' },+ egressBindings: [{ dns: 'current.example.com', port: 443 }],+ },+ ],+ ...(workflow ? { steps: [{ id: 'step', run: snippetRun() }] } : {}),+ },+ status: { phase: 'approved' },+ })+ }+ function hccReadiness() {+ const healthyRead = mockCustomApi.getNamespacedCustomObject.getMockImplementation()!+ const state = { ready: false }+ mockCustomApi.getNamespacedCustomObject.mockImplementation(async request => {+ const response = await healthyRead(request)+ if (request.plural === 'mcpservers') {+ response.metadata.generation = 7+ response.status.conditions = [+ {+ type: 'ExternalEgressReady',+ status: state.ready ? 'True' : 'False',+ observedGeneration: 7,+ reason: state.ready ? 'Reconciled' : 'ExternalEgressReconcileFailed',+ message: 'Policy observation pending',+ },+ ]+ }+ return response+ })+ return state+ }+ function watcherFor(rec: WorkflowRecipeReconciler, recipe?: WorkflowRecipeCRD) {+ const watcher: any = Object.create(WorkflowRecipeWatcher.prototype)+ Object.assign(watcher, {+ recipes: new Map(recipe ? [[recipe.metadata.name, recipe]] : []),+ transientRetries: new Map(),+ stopped: false,+ traceReporter: null,+ dbRunProcessor: null,+ reconciler: rec,+ eventQueue: { enqueue: vi.fn((_key: string, work: () => Promise<void>) => work()) },+ config: loadConfig(),+ })+ return watcher+ }+ function stopWatcher(watcher: any) {+ watcher.stopped = true+ for (const { timer } of watcher.transientRetries.values()) clearTimeout(timer)+ watcher.transientRetries.clear()+ }+ function patchedStatus(previous?: WorkflowRecipeCRD['status']) {+ return {+ ...previous,+ ...mockCustomApi.patchNamespacedCustomObjectStatus.mock.calls.at(-1)![0].body.status,+ }+ }++ it('retries persisted stdio-only admission with backoff after restart and cannot become active from ready pods', async () => {+ const recipe = transportRecipe()+ const hcc = hccReadiness()+ const watcher = watcherFor(reconciler)+ let restarted: any+ vi.useFakeTimers()+ try {+ const firstRun = watcher.handleRecipeEvent('ADDED', recipe)+ await vi.advanceTimersByTimeAsync(31_000)+ await firstRun+ expect(watcher.transientRetries.get(recipe.metadata.name)?.attempts).toBe(1)+ const pendingRecipe = { ...recipe, status: patchedStatus() }+ expect(pendingRecipe.status.phase).toBe('degraded')+ expect(pendingRecipe.status.conditions).toContainEqual(+ expect.objectContaining({ type: conditionType, status: 'False' })+ )+ await watcher.handleRecipeEvent('MODIFIED', pendingRecipe)+ await watcher.refreshWorkloadStatusesForSteadyRecipes()+ const observedStatus = patchedStatus(pendingRecipe.status)+ expect(observedStatus.phase).toBe('degraded')+ expect(observedStatus.conditions).toContainEqual(+ expect.objectContaining({ type: conditionType, status: 'False' })+ )+ stopWatcher(watcher)++ // A new process has no previous retry timer; only persisted condition state remains.+ const restartRecipe = { ...recipe, status: observedStatus }+ const rec = new WorkflowRecipeReconciler(new k8s.KubeConfig())+ restarted = watcherFor(rec, restartRecipe)+ const run = vi.spyOn(rec, 'reconcile')+ const restartPass = restarted.refreshRuntimeCredentialsForInProgressRecipes()+ await vi.advanceTimersByTimeAsync(31_000)+ await restartPass+ expect(run).toHaveBeenCalledTimes(1)+ expect(restarted.transientRetries.get(recipe.metadata.name)?.attempts).toBe(1)+ await restarted.handleRecipeEvent('MODIFIED', { ...recipe, status: patchedStatus() })+ await vi.advanceTimersByTimeAsync(35_000)+ expect(run).toHaveBeenCalledTimes(2)+ expect(restarted.transientRetries.get(recipe.metadata.name)?.attempts).toBe(2)+ expect(patchedStatus().phase).toBe('degraded')+ hcc.ready = true+ await vi.advanceTimersByTimeAsync(10_000)+ expect(run).toHaveBeenCalledTimes(3)+ expect(patchedStatus().phase).toBe('active')+ expect(+ patchedStatus().conditions?.some(+ (condition: { type: string }) => condition.type === conditionType+ )+ ).not.toBe(true)+ expect(restarted.transientRetries.size).toBe(0)+ expect(recipe.metadata.generation).toBe(7)+ } finally {+ stopWatcher(watcher)+ if (restarted) stopWatcher(restarted)+ vi.useRealTimers()+ }+ })++ it('persists workflow admission pending and only starts the workflow after HCC heals', async () => {+ const recipe = transportRecipe(true)+ const hcc = hccReadiness()+ const workflow = vi+ .fn()+ .mockResolvedValue({ phase: 'active', workflowPhase: 'running', message: 'Workflow ready' })+ ;(reconciler as any).workflowReconciler = {+ reconcile: workflow,+ validateWorkflowSpec: () => undefined,+ }+ vi.useFakeTimers()+ try {+ const pending = reconciler.reconcile(recipe)+ await vi.advanceTimersByTimeAsync(31_000)+ const first = await pending+ expect(first.phase, first.message).toBe('degraded')+ expect(first.requeueAfterMs).toBe(TRANSIENT_REQUEUE_BASE_MS)+ expect(workflow).not.toHaveBeenCalled()+ await reconciler.patchStatus(recipe, first)+ const persisted = { ...recipe, status: patchedStatus() }+ expect(persisted.status.conditions).toContainEqual(+ expect.objectContaining({ type: conditionType, status: 'False' })+ )+ expect(persisted.status.workflowExecution?.phase).not.toBe('failed')+ hcc.ready = true+ const result = await reconciler.reconcile(persisted)+ expect(result.phase).toBe('active')+ expect(workflow).toHaveBeenCalledTimes(1)+ await reconciler.patchStatus(persisted, result)+ expect(+ patchedStatus().conditions?.some(+ (condition: { type: string }) => condition.type === conditionType+ )+ ).not.toBe(true)+ } finally {+ vi.useRealTimers()+ }+ })++ it('preserves pending admission through an earlier validation failure and clears it after transport removal', async () => {+ const recipe = transportRecipe()+ recipe.status = { phase: 'degraded', conditions: [pendingCondition] }+ recipe.spec.workloads![0].egressBindings![0].port = 0+ const failed = await reconciler.reconcile(recipe)+ expect(failed.phase).toBe('failed')+ await reconciler.patchStatus(recipe, failed)+ expect(patchedStatus(recipe.status).conditions).toContainEqual(pendingCondition)+ recipe.spec.workloads![0].egressBindings = []+ delete recipe.spec.workloads![0].transport+ const removed = await reconciler.reconcile(recipe)+ expect(removed.phase).toBe('active')+ await reconciler.patchStatus(recipe, removed)+ expect(+ patchedStatus(recipe.status).conditions?.some(+ (condition: { type: string }) => condition.type === conditionType+ )+ ).not.toBe(true)+ })++ it('preserves ordinary Pod-only downgrade and recovery when no admission is pending', async () => {+ const recipe = makeRecipe({ status: { phase: 'degraded' } })+ mockAppsApi.readNamespacedDeployment.mockResolvedValueOnce({+ spec: { replicas: 1 },+ status: { readyReplicas: 0, updatedReplicas: 0, availableReplicas: 0 },+ })+ expect((await reconciler.observeCurrentWorkloadStatus(recipe)).phase).toBe('degraded')+ expect((await reconciler.observeCurrentWorkloadStatus(recipe)).phase).toBe('active')+ })+ })++ describe('#567 regression: policy contraction', () => {+ function policyStore(policies: k8s.V1NetworkPolicy[]) {+ const live = new Map<string, k8s.V1NetworkPolicy>()+ let version = 0+ const persist = (body: k8s.V1NetworkPolicy) => {+ const policy = structuredClone(body)+ const name = policy.metadata!.name!+ policy.metadata = {+ ...policy.metadata,+ uid: `uid-${name}`,+ resourceVersion: String(++version),+ }+ // The API's omitempty serialization omits empty top-level rule lists.+ if (policy.spec?.egress?.length === 0) delete policy.spec.egress+ if (policy.spec?.ingress?.length === 0) delete policy.spec.ingress+ live.set(name, policy)+ return structuredClone(policy)+ }+ for (const policy of policies) persist(policy)+ mockNetworkingApi.readNamespacedNetworkPolicy.mockImplementation(async ({ name }) => {+ if (!live.has(name)) throw { code: 404 }+ return structuredClone(live.get(name))+ })+ mockNetworkingApi.createNamespacedNetworkPolicy.mockImplementation(async ({ body }) => {+ if (live.has(body.metadata.name)) throw { code: 409 }+ return persist(body)+ })+ mockNetworkingApi.replaceNamespacedNetworkPolicy.mockImplementation(+ async ({ name, body }) => {+ expect(body.metadata.resourceVersion).toBe(live.get(name)?.metadata?.resourceVersion)+ return persist(body)+ }+ )+ mockNetworkingApi.deleteNamespacedNetworkPolicy.mockImplementation(async ({ name, body }) => {+ if (body?.preconditions) {+ expect(body.preconditions).toEqual({+ uid: live.get(name)?.metadata?.uid,+ resourceVersion: live.get(name)?.metadata?.resourceVersion,+ })+ }+ live.delete(name)+ return {}+ })+ return live+ }++ function uiRecipe(ports = [8080, 9090]) {+ return makeRecipe({+ spec: {+ workloads: [+ { id: 'ui', type: 'deployment', image: 'ui:1', port: 8080 },+ { id: 'backend', type: 'deployment', image: 'backend:1', port: 8080 },+ ],+ ui: {+ workloadRef: 'ui',+ port: 8080,+ egress: {+ internal: ports.map(port => ({ workloadRef: 'backend', port })),+ external: [{ fqdn: 'current.example.com', port: 443 }],+ },+ },+ },+ })+ }++ it.each([[8080], [8080, 9090, 9443], [9090, 8080]])(+ 'preserves the intersection of grouped UI ports during a DNS fault: %j',+ async (...ports) => {+ const rb = await import('./resourceBuilder')+ const recipe = uiRecipe()+ const name = 'ui-ingress-test-recipe-backend'+ const live = policyStore([+ rb.buildUiIngressNetworkPolicy(+ recipe,+ 'backend',+ [8080, 9090],+ 'sandbox-recipes',+ 'sandbox-ui'+ )!,+ ])+ recipe.spec.ui!.egress!.internal = ports.map(port => ({ workloadRef: 'backend', port }))+ const rec = new WorkflowRecipeReconciler(new k8s.KubeConfig(), undefined, {+ fqdnLookup: async () => {+ throw new Error('resolver controller fault')+ },+ })+ const result = await rec.reconcile({ ...recipe, status: { phase: 'active' } })+ expect(result.phase).toBe('failed')+ const retained = (live.get(name)?.spec?.ingress ?? [])+ .flatMap(rule => rule.ports ?? [])+ .map(port => port.port)+ expect(retained.sort()).toEqual(ports.filter(port => port === 8080 || port === 9090).sort())+ }+ )++ it.each([false, true])(+ 'accepts empty UI readback before external replacement=%s',+ async replacement => {+ const rb = await import('./resourceBuilder')+ const recipe = uiRecipe([])+ const name = 'ui-egress-test-recipe'+ const live = policyStore([+ rb.buildUiEgressNetworkPolicy(+ recipe,+ 'sandbox-ui',+ 'sandbox-recipes',+ [+ {+ cidr: '93.184.216.20/32',+ port: 443,+ source: { kind: 'fqdn', fqdn: 'current.example.com' },+ },+ ],+ egressStateAnnotations([+ { fqdn: 'current.example.com', ip: '93.184.216.20', port: 443 },+ ])+ )!,+ ])+ recipe.spec.ui!.egress!.external = replacement+ ? [{ fqdn: 'new.example.com', port: 443 }]+ : []+ const lookup = vi.fn(async () => ({+ kind: 'ok' as const,+ ipv4: ['93.184.216.21'],+ ipv6: [],+ ttlSeconds: 300,+ }))+ const rec = new WorkflowRecipeReconciler(new k8s.KubeConfig(), undefined, {+ fqdnLookup: lookup,+ })+ const result = await rec.reconcile({ ...recipe, status: { phase: 'active' } })+ expect(result.phase).toBe('active')+ expect(live.has(name)).toBe(true)+ expect(lookup).toHaveBeenCalledTimes(replacement ? 1 : 0)+ expect(mockNetworkingApi.deleteNamespacedNetworkPolicy).not.toHaveBeenCalledWith(+ expect.objectContaining({ name })+ )+ }+ )++ it.each([+ { mode: 'broader-rules', phase: 'failed', deleted: true },+ { mode: 'annotation-only', phase: 'failed', deleted: true },+ { mode: 'replacement-uid', phase: 'degraded', deleted: false },+ { mode: 'foreign-owner', phase: 'failed', deleted: false },+ { mode: 'missing-uid', phase: 'failed', deleted: false },+ { mode: 'wrong-namespace', phase: 'failed', deleted: false },+ ] as const)('rejects unsafe readback: $mode', async ({ mode, phase, deleted }) => {+ const rb = await import('./resourceBuilder')+ const recipe = uiRecipe([])+ const name = 'ui-egress-test-recipe'+ const live = policyStore([+ rb.buildUiEgressNetworkPolicy(+ recipe,+ 'sandbox-ui',+ 'sandbox-recipes',+ [+ {+ cidr: '93.184.216.20/32',+ port: 443,+ source: { kind: 'fqdn', fqdn: 'current.example.com' },+ },+ ],+ egressStateAnnotations([{ fqdn: 'current.example.com', ip: '93.184.216.20', port: 443 }])+ )!,+ ])+ recipe.spec.ui!.egress!.external = []+ const replace = mockNetworkingApi.replaceNamespacedNetworkPolicy.getMockImplementation()!+ mockNetworkingApi.replaceNamespacedNetworkPolicy.mockImplementation(async request => {+ const result = await replace(request)+ if (request.name === name) {+ const observed = live.get(name)!+ if (mode === 'broader-rules')+ observed.spec!.egress = [+ {+ to: [{ ipBlock: { cidr: '8.8.8.8/32' } }],+ ports: [{ port: 443, protocol: 'TCP' }],+ },+ ]+ if (mode === 'annotation-only')+ observed.metadata!.annotations = egressStateAnnotations([+ { fqdn: 'removed.example.com', ip: '8.8.8.8', port: 443 },+ ])+ if (mode === 'replacement-uid') observed.metadata!.uid = 'uid-new-owned-object'+ if (mode === 'missing-uid') delete observed.metadata!.uid+ if (mode === 'wrong-namespace') observed.metadata!.namespace = 'another-namespace'+ if (mode === 'foreign-owner')+ observed.metadata!.labels!['clerum.io/managed-by'] = 'another-controller'+ }+ return result+ })+ const rec = new WorkflowRecipeReconciler(new k8s.KubeConfig(), undefined, {+ fqdnLookup: async () => {+ throw new Error('no external lookup is declared')+ },+ })+ const result = await rec.reconcile({ ...recipe, status: { phase: 'active' } })+ expect(result.phase).toBe(phase)+ expect(live.has(name)).toBe(!deleted)+ if (!deleted) expect(mockNetworkingApi.deleteNamespacedNetworkPolicy).not.toHaveBeenCalled()+ expect(result.requeueAfterMs).toBe(+ mode === 'replacement-uid' ? TRANSIENT_REQUEUE_BASE_MS : undefined+ )+ })++ it.each(['ui', 'workload'] as const)(+ 'replans additive %s conflicts and disappearance without latching a healthy recipe',+ async lane => {+ for (const failure of ['replace-conflict', 'read-disappearance', 'replace-disappearance']) {+ const recipe =+ lane === 'ui'+ ? uiRecipe([])+ : makeRecipe({+ spec: {+ workloads: [+ {+ id: 'worker',+ type: 'deployment',+ image: 'worker:1',+ port: 8080,+ egressBindings: [{ dns: 'current.example.com', port: 443 }],+ },+ ],+ },+ })+ const name = lane === 'ui' ? 'ui-egress-test-recipe' : 'wl-egress-test-recipe-worker'+ const live = policyStore([])+ let address = '93.184.216.20'+ const rec = new WorkflowRecipeReconciler(new k8s.KubeConfig(), undefined, {+ fqdnLookup: async () => ({ kind: 'ok', ipv4: [address], ipv6: [], ttlSeconds: 300 }),+ })+ expect((await rec.reconcile({ ...recipe, status: { phase: 'active' } })).phase).toBe(+ 'active'+ )+ const healthyReplace =+ mockNetworkingApi.replaceNamespacedNetworkPolicy.getMockImplementation()!+ const healthyCreate =+ mockNetworkingApi.createNamespacedNetworkPolicy.getMockImplementation()!+ address = '93.184.216.21'+ if (failure === 'read-disappearance') {++ // The workload lane is read-first: start absent so its create can++ // race with another creator, whose winner then disappears.++ if (lane === 'workload') live.delete(name)+ mockNetworkingApi.createNamespacedNetworkPolicy.mockImplementation(async request => {+ if (request.body.metadata.name === name) {+ live.delete(name)+ throw Object.assign(new Error('object existed before replacement read'), {+ code: 409,+ })+ }+ return healthyCreate(request)+ })+ } else {+ mockNetworkingApi.replaceNamespacedNetworkPolicy.mockImplementation(async request => {+ if (request.name === name && JSON.stringify(request.body.spec).includes(address)) {+ if (failure === 'replace-disappearance') live.delete(name)+ throw Object.assign(new Error('object snapshot changed'), {+ code: failure === 'replace-conflict' ? 409 : 404,+ })+ }+ return healthyReplace(request)+ })+ }+ const first = await rec.reconcile({ ...recipe, status: { phase: 'active' } })+ expect(first.phase, `${lane}/${failure}`).toBe('degraded')+ expect(first.requeueAfterMs).toBe(TRANSIENT_REQUEUE_BASE_MS)+ await rec.patchStatus(recipe, first)+ const persisted =+ mockCustomApi.patchNamespacedCustomObjectStatus.mock.calls.at(-1)![0].body.status+ mockNetworkingApi.createNamespacedNetworkPolicy.mockImplementation(healthyCreate)+ mockNetworkingApi.replaceNamespacedNetworkPolicy.mockImplementation(healthyReplace)+ const recovered = await rec.reconcile({ ...recipe, status: persisted })+ expect(recovered.phase).toBe('active')+ expect(JSON.stringify(live.get(name)?.spec)).toContain(address)+ }+ }+ )++ it.each(['delete', 'replace'] as const)(+ 'retries a fenced %s conflict after the status roundtrip',+ async operation => {+ const rb = await import('./resourceBuilder')+ const recipe = makeRecipe({+ spec: {+ workloads: [+ {+ id: 'worker',+ type: 'deployment',+ image: 'worker:1',+ port: 8080,+ egressBindings: [+ { dns: 'current.example.com', port: 443 },+ { dns: 'removed.example.com', port: 443 },+ ],+ },+ {+ id: 'keeper',+ type: 'deployment',+ image: 'keeper:1',+ port: 8080,+ egressBindings: [{ dns: 'keeper.example.com', port: 443 }],+ },+ ],+ },+ })+ const name = 'wl-egress-test-recipe-worker'+ const entries = [+ { fqdn: 'current.example.com', ip: '93.184.216.20', port: 443 },+ { fqdn: 'removed.example.com', ip: '93.184.216.21', port: 443 },+ ]+ const live = policyStore([+ rb.buildWorkloadEgressNetworkPolicy(+ recipe.spec.workloads![0],+ recipe,+ 'sandbox-recipes',+ entries.map(entry => ({+ cidr: `${entry.ip}/32`,+ port: entry.port,+ source: { kind: 'fqdn', fqdn: entry.fqdn },+ })),+ egressStateAnnotations(entries)+ )!,+ ])+ recipe.spec.workloads![0].egressBindings =+ operation === 'delete' ? [] : [{ dns: 'current.example.com', port: 443 }]+ const mutation =+ operation === 'delete'+ ? mockNetworkingApi.deleteNamespacedNetworkPolicy+ : mockNetworkingApi.replaceNamespacedNetworkPolicy+ const healthyMutation = mutation.getMockImplementation()!+ mutation.mockImplementationOnce(async () => {+ throw Object.assign(new Error('object modified'), { code: 409 })+ })+ const rec = new WorkflowRecipeReconciler(new k8s.KubeConfig(), undefined, {+ fqdnLookup: async () => ({+ kind: 'ok',+ ipv4: ['93.184.216.20'],+ ipv6: [],+ ttlSeconds: 300,+ }),+ })+ type Watcher = {+ recipes: Map<string, WorkflowRecipeCRD>+ transientRetries: Map<string, { timer: ReturnType<typeof setTimeout>; attempts: number }>+ stopped: boolean+ traceReporter: null+ dbRunProcessor: null+ reconciler: WorkflowRecipeReconciler+ eventQueue: { enqueue: (key: string, task: () => Promise<void>) => Promise<void> }+ handleRecipeEvent: (type: string, recipe: WorkflowRecipeCRD) => Promise<void>+ }+ const watcher = Object.create(WorkflowRecipeWatcher.prototype) as Watcher+ Object.assign(watcher, {+ recipes: new Map(),+ transientRetries: new Map(),+ stopped: false,+ traceReporter: null,+ dbRunProcessor: null,+ reconciler: rec,+ eventQueue: { enqueue: vi.fn((_key: string, task: () => Promise<void>) => task()) },+ })+ const reconcile = vi.spyOn(rec, 'reconcile')+ const active = {+ ...recipe,+ metadata: { ...recipe.metadata, generation: 1 },+ status: { phase: 'active' as const },+ }+ vi.useFakeTimers()+ try {+ await watcher.handleRecipeEvent('ADDED', active)+ const first = await reconcile.mock.results[0].value+ expect(first.phase).toBe('degraded')+ expect(first.requeueAfterMs).toBe(TRANSIENT_REQUEUE_BASE_MS)+ const patched =+ mockCustomApi.patchNamespacedCustomObjectStatus.mock.calls.at(-1)![0].body.status+ // Deliver the real status-only watch event without changing generation.+ await watcher.handleRecipeEvent('MODIFIED', { ...active, status: patched })+ expect(reconcile).toHaveBeenCalledTimes(1)+ expect(watcher.transientRetries.get(recipe.metadata.name)?.attempts).toBe(1)+ mutation.mockImplementationOnce(async () => {+ throw Object.assign(new Error('object modified again'), { code: 409 })+ })+ await vi.advanceTimersByTimeAsync(TRANSIENT_REQUEUE_BASE_MS)+ expect(reconcile).toHaveBeenCalledTimes(2)+ expect(watcher.transientRetries.get(recipe.metadata.name)?.attempts).toBe(2)+ mutation.mockImplementation(healthyMutation)+ await vi.advanceTimersByTimeAsync(TRANSIENT_REQUEUE_BASE_MS * 2)+ const next = await reconcile.mock.results[2].value+ expect(next.phase).toBe('active')+ expect(watcher.transientRetries.size).toBe(0)+ expect(JSON.stringify(live.get(name)?.spec ?? {})).not.toContain('93.184.216.21')+ expect(live.has(name)).toBe(operation === 'replace')+ } finally {+ for (const { timer } of watcher.transientRetries.values()) clearTimeout(timer)+ vi.useRealTimers()+ }+ }+ )+ })+it('processes full pipeline successfully (3.11a)', async () => {const result = await reconciler.reconcile(makeRecipe())expect(result.phase).toBe('active')@@@ -773,675 -1429,6 +1426,719 @@@])})+ // ─── issue #575: live NetworkPolicy convergence ────────────────────────+ describe('applyNetworkPolicy live convergence (issue #575)', () => {+ const SPEC_HASH = 'clerum.io/spec-hash'+ const NS = 'sandbox-recipes'+ const clone = <T>(value: T): T => structuredClone(value)+ type Priv = {+ applyNetworkPolicy: (+ policy: k8s.V1NetworkPolicy,+ namespace: string,- options: { family: NetworkPolicyFamily; existing?: k8s.V1NetworkPolicy | null }++ options: {++ family: NetworkPolicyFamily++ existing?: k8s.V1NetworkPolicy | null++ recipeName?: string++ }+ ) => Promise<void>+ }+ const apply = (+ policy: k8s.V1NetworkPolicy,+ family: NetworkPolicyFamily = 'workload-ingress',+ existing?: k8s.V1NetworkPolicy | null+ ) =>+ (reconciler as unknown as Priv).applyNetworkPolicy(policy, NS, {+ family,+ ...(existing === undefined ? {} : { existing }),+ })++ const plainPolicy = (): k8s.V1NetworkPolicy => ({+ apiVersion: 'networking.k8s.io/v1',+ kind: 'NetworkPolicy',+ metadata: {+ name: 'wl-ingress-r-w',+ namespace: NS,+ labels: { 'clerum.io/managed-by': 'workflow-recipes', 'clerum.io/recipe': 'r' },+ },+ spec: {+ podSelector: { matchLabels: { 'clerum.io/workload': 'w' } },+ policyTypes: ['Ingress'],+ ingress: [{ ports: [{ port: 8080, protocol: 'TCP' }] }],+ },+ })++ const intdepPolicy = (): k8s.V1NetworkPolicy => ({+ ...plainPolicy(),+ metadata: {+ name: 'wr-intdep-egress-r-w',+ namespace: NS,+ labels: {+ 'clerum.io/managed-by': 'workflow-recipes',+ 'clerum.io/recipe': 'r',+ 'clerum.io/policy-type': 'internal-dependency',+ },+ },+ })++ const gatewayPolicy = (ownerUid = 'uid-current'): k8s.V1NetworkPolicy => ({+ ...plainPolicy(),+ metadata: {+ name: 'allow-webhook-proxy-ingress-r',+ namespace: NS,+ labels: {+ 'clerum.io/managed-by': 'workflow-recipes',+ 'clerum.io/recipe-namespace': NS,+ 'clerum.io/recipe-name': 'r',+ 'clerum.io/webhook-gateway': 'true',+ },+ ownerReferences: [+ {+ apiVersion: 'clerum.io/v1alpha1',+ kind: 'WorkflowRecipe',+ name: 'r',+ uid: ownerUid,+ controller: true,+ blockOwnerDeletion: true,+ },+ ],+ },+ })++ const livePolicy = (+ desired: k8s.V1NetworkPolicy,+ metadata: Partial<k8s.V1ObjectMeta> = {}+ ): k8s.V1NetworkPolicy => ({+ ...clone(desired),+ metadata: {+ ...clone(desired.metadata ?? {}),+ resourceVersion: '9',+ ...metadata,+ },+ })++ const families: NetworkPolicyFamily[] = [+ 'ui-ingress',+ 'workload-ingress',+ 'workload-egress',+ 'internal-dependency',+ 'oauth-broker-egress',+ 'webhook-gateway',+ ]+ const policyForFamily = (family: NetworkPolicyFamily) =>+ family === 'webhook-gateway'+ ? gatewayPolicy()+ : family === 'internal-dependency'+ ? intdepPolicy()+ : plainPolicy()+++ it.each(['ui-ingress', 'workload-ingress', 'workload-egress'] as const)(++ 'preserves the dev recipe-ownership boundary on the supplied %s snapshot',++ async family => {++ const desired = plainPolicy()++ const live = livePolicy(desired)++ live.metadata!.labels!['clerum.io/recipe'] = 'another-recipe'++ mockNetworkingApi.readNamespacedNetworkPolicy.mockResolvedValue(livePolicy(desired))++ await expect(++ (reconciler as unknown as Priv).applyNetworkPolicy(desired, NS, {++ family,++ recipeName: 'r',++ existing: live,++ })++ ).rejects.toMatchObject({ name: 'NetworkPolicyOwnershipConflictError' })++ expect(mockNetworkingApi.readNamespacedNetworkPolicy).not.toHaveBeenCalled()++ expect(mockNetworkingApi.createNamespacedNetworkPolicy).not.toHaveBeenCalled()++ expect(mockNetworkingApi.replaceNamespacedNetworkPolicy).not.toHaveBeenCalled()++ }++ )++++ it.each(['ui-ingress', 'workload-ingress', 'workload-egress'] as const)(++ 'keeps owned %s read-only until real drift and replaces the validated version',++ async family => {++ const desired = plainPolicy()++ const live = livePolicy(desired)++ const options = { family, recipeName: 'r', existing: live }++ await (reconciler as unknown as Priv).applyNetworkPolicy(desired, NS, options)++ expect(mockNetworkingApi.createNamespacedNetworkPolicy).not.toHaveBeenCalled()++ expect(mockNetworkingApi.replaceNamespacedNetworkPolicy).not.toHaveBeenCalled()++ desired.spec!.podSelector = { matchLabels: { updated: 'selector' } }++ await (reconciler as unknown as Priv).applyNetworkPolicy(desired, NS, options)++ expect(mockNetworkingApi.readNamespacedNetworkPolicy).not.toHaveBeenCalled()++ expect(mockNetworkingApi.replaceNamespacedNetworkPolicy).toHaveBeenCalledTimes(1)++ expect(++ mockNetworkingApi.replaceNamespacedNetworkPolicy.mock.calls[0][0].body.metadata++ .resourceVersion++ ).toBe('9')++ }++ )+++ it.each(families)(+ 'preserves external metadata while repairing real %s spec drift',+ async family => {+ const desired = policyForFamily(family)+ desired.metadata!.labels!['clerum.io/test-contract'] = 'current'+ desired.metadata!.annotations = {+ 'clerum.io/test-contract': 'current',+ [SPEC_HASH]: 'obsolete',+ }+ const live = livePolicy(desired)+ live.metadata!.labels!['clerum.io/test-contract'] = 'stale'+ live.metadata!.labels!['admission.example/tier'] = 'audited'+ live.metadata!.annotations = {+ 'admission.example/audit': 'keep',+ 'clerum.io/test-contract': 'stale',+ [SPEC_HASH]: 'legacy',+ 'clerum.io/egress-fqdn-state': 'stale-state',+ 'clerum.io/egress-fqdn-targets': 'stale-targets',+ 'clerum.io/egress-fqdn-resolved-at': 'stale-time',+ }+ live.metadata!.finalizers = ['admission.example/cleanup']+ live.spec!.podSelector = {}+ const desiredBefore = clone(desired)+ const liveBefore = clone(live)+ mockNetworkingApi.readNamespacedNetworkPolicy.mockResolvedValue(live)++ await apply(desired, family)++ expect(mockNetworkingApi.createNamespacedNetworkPolicy).toHaveBeenCalledTimes(0)+ expect(mockNetworkingApi.replaceNamespacedNetworkPolicy).toHaveBeenCalledTimes(1)+ const body = mockNetworkingApi.replaceNamespacedNetworkPolicy.mock.calls[0][0].body+ expect(body.metadata.labels).toEqual({+ ...desired.metadata!.labels,+ 'admission.example/tier': 'audited',+ })+ expect(body.metadata.annotations).toEqual({+ 'admission.example/audit': 'keep',+ 'clerum.io/test-contract': 'current',+ })+ expect(body.metadata.finalizers).toEqual(['admission.example/cleanup'])+ expect(body.metadata.ownerReferences).toEqual(desired.metadata!.ownerReferences)+ expect(body.metadata.resourceVersion).toBe('9')+ expect(body.spec).toEqual(desired.spec)+ expect(mockNetworkingApi.readNamespacedNetworkPolicy).toHaveBeenCalledTimes(1)+ expect(desired).toEqual(desiredBefore)+ expect(live).toEqual(liveBefore)+ }+ )++ it.each(families)(+ 'performs zero create/replace calls for equivalent %s with external metadata',+ async family => {+ const desired = policyForFamily(family)+ const live = livePolicy(desired)+ live.metadata!.labels!['admission.example/tier'] = 'audited'+ live.metadata!.annotations = { 'admission.example/audit': 'keep', [SPEC_HASH]: 'legacy' }+ live.metadata!.finalizers = ['admission.example/cleanup']+ mockNetworkingApi.readNamespacedNetworkPolicy.mockResolvedValue(live)++ await apply(desired, family)++ expect(mockNetworkingApi.readNamespacedNetworkPolicy).toHaveBeenCalledTimes(1)+ expect(mockNetworkingApi.createNamespacedNetworkPolicy).toHaveBeenCalledTimes(0)+ expect(mockNetworkingApi.replaceNamespacedNetworkPolicy).toHaveBeenCalledTimes(0)+ }+ )++ it('vetoes an additional non-controller owner instead of removing it on gateway replace', async () => {+ const desired = gatewayPolicy()+ const live = livePolicy(desired)+ live.metadata!.ownerReferences!.push({+ apiVersion: 'v1',+ kind: 'ConfigMap',+ name: 'foreign',+ uid: 'foreign',+ })+ live.spec!.podSelector = {}+ mockNetworkingApi.readNamespacedNetworkPolicy.mockResolvedValue(live)++ await expect(apply(desired, 'webhook-gateway')).rejects.toMatchObject({+ name: 'NetworkPolicyOwnershipConflictError',+ })+ expect(mockNetworkingApi.createNamespacedNetworkPolicy).toHaveBeenCalledTimes(0)+ expect(mockNetworkingApi.replaceNamespacedNetworkPolicy).toHaveBeenCalledTimes(0)+ })++ it('removes retired workload-egress state once without churning external metadata', async () => {+ const desired = plainPolicy()+ desired.metadata!.name = 'wl-egress-r-w'+ const live = livePolicy(desired, {+ annotations: {+ 'admission.example/audit': 'keep',+ 'clerum.io/egress-fqdn-state': 'retired',+ 'clerum.io/egress-fqdn-targets': 'retired',+ 'clerum.io/egress-fqdn-resolved-at': 'retired',+ },+ })+ const priv = reconciler as unknown as {+ egressWriteNeeded: (existing: k8s.V1NetworkPolicy, desired: k8s.V1NetworkPolicy) => boolean+ }+ expect(priv.egressWriteNeeded(live, desired)).toBe(true)+ await apply(desired, 'workload-egress', live)+ expect(mockNetworkingApi.replaceNamespacedNetworkPolicy).toHaveBeenCalledTimes(1)+ const updated = mockNetworkingApi.replaceNamespacedNetworkPolicy.mock.calls[0][0].body+ expect(updated.metadata.annotations).toEqual({ 'admission.example/audit': 'keep' })+ mockNetworkingApi.replaceNamespacedNetworkPolicy.mockClear()++ expect(priv.egressWriteNeeded(updated, desired)).toBe(false)+ await apply(desired, 'workload-egress', updated)+ expect(mockNetworkingApi.createNamespacedNetworkPolicy).toHaveBeenCalledTimes(0)+ expect(mockNetworkingApi.replaceNamespacedNetworkPolicy).toHaveBeenCalledTimes(0)+ })++ function captureNetworkPolicyLogs(): {+ entries: Array<Record<string, unknown>>+ restore: () => void+ } {+ const previousLevel = process.env.LOG_LEVEL+ process.env.LOG_LEVEL = 'info'+ const entries: Array<Record<string, unknown>> = []+ const spy = vi.spyOn(process.stdout, 'write').mockImplementation(((chunk: unknown) => {+ const text = typeof chunk === 'string' ? chunk : Buffer.from(chunk as Uint8Array).toString()+ for (const line of text.split('\n')) {+ if (!line.startsWith('{')) continue+ const parsed = JSON.parse(line) as Record<string, unknown>+ if (String(parsed.msg).startsWith('network policy')) entries.push(parsed)+ }+ return true+ }) as unknown as typeof process.stdout.write)+ return {+ entries,+ restore: () => {+ spy.mockRestore()+ if (previousLevel === undefined) delete process.env.LOG_LEVEL+ else process.env.LOG_LEVEL = previousLevel+ },+ }+ }++ it('T1 skips a live-equivalent policy and emits a structured liveness witness', async () => {+ const desired = plainPolicy()+ const live = livePolicy(desired, { annotations: { [SPEC_HASH]: 'legacy-seal' } })+ live.spec!.ingress![0].ports![0].protocol = undefined+ mockNetworkingApi.readNamespacedNetworkPolicy.mockResolvedValue(live)+ const captured = captureNetworkPolicyLogs()+ try {+ await apply(desired)+ expect(mockNetworkingApi.readNamespacedNetworkPolicy).toHaveBeenCalledTimes(1)+ expect(mockNetworkingApi.createNamespacedNetworkPolicy).not.toHaveBeenCalled()+ expect(mockNetworkingApi.replaceNamespacedNetworkPolicy).not.toHaveBeenCalled()+ expect(captured.entries).toContainEqual(+ expect.objectContaining({+ level: 'info',+ msg: 'network policy unchanged; skipping update',+ policy: 'wl-ingress-r-w',+ namespace: NS,+ family: 'workload-ingress',+ })+ )+ } finally {+ captured.restore()+ }+ })++ it('T2 repairs live spec drift even when the legacy spec-hash is preserved', async () => {+ const desired = plainPolicy()+ const live = livePolicy(desired, { annotations: { [SPEC_HASH]: 'legacy-seal' } })+ live.spec!.podSelector = {}+ live.spec!.ingress = [{}]+ mockNetworkingApi.readNamespacedNetworkPolicy.mockResolvedValue(live)++ await apply(desired)++ expect(mockNetworkingApi.replaceNamespacedNetworkPolicy).toHaveBeenCalledTimes(1)+ const body = (+ mockNetworkingApi.replaceNamespacedNetworkPolicy.mock.calls[0][0] as {+ body: k8s.V1NetworkPolicy+ }+ ).body+ expect(body.metadata?.resourceVersion).toBe('9')+ expect(body.metadata?.annotations?.[SPEC_HASH]).toBeUndefined()+ expect(body.spec).toEqual(desired.spec)+ expect(mockNetworkingApi.readNamespacedNetworkPolicy).toHaveBeenCalledTimes(1)+ })++ it('does not let a supplied cluster-local wl-egress snapshot veto detected drift', async () => {+ const desired = plainPolicy()+ desired.metadata!.name = 'wl-egress-r-w'+ desired.spec = {+ podSelector: { matchLabels: { 'clerum.io/workload': 'w' } },+ policyTypes: ['Egress'],+ egress: [+ {+ to: [{ podSelector: { matchLabels: { 'clerum.io/workload': 'db' } } }],+ ports: [{ port: 5432, protocol: 'TCP' }],+ },+ ],+ }+ const live = livePolicy(desired, { annotations: { [SPEC_HASH]: 'legacy-seal' } })+ live.spec!.podSelector = {}++ await apply(desired, 'workload-egress', live)++ expect(mockNetworkingApi.readNamespacedNetworkPolicy).not.toHaveBeenCalled()+ expect(mockNetworkingApi.replaceNamespacedNetworkPolicy).toHaveBeenCalledTimes(1)+ expect(+ (+ mockNetworkingApi.replaceNamespacedNetworkPolicy.mock.calls[0][0] as {+ body: k8s.V1NetworkPolicy+ }+ ).body.spec?.podSelector+ ).toEqual(desired.spec.podSelector)+ })++ it('routes workload-egress label drift through the live apply prefilter', () => {+ const desired = plainPolicy()+ desired.metadata!.name = 'wl-egress-r-w'+ desired.spec = { podSelector: {}, policyTypes: ['Egress'], egress: [] }+ const live = livePolicy(desired)+ live.metadata!.labels = { ...live.metadata!.labels, 'clerum.io/recipe': 'wrong' }++ const writeNeeded = (+ reconciler as unknown as {+ egressWriteNeeded: (+ existing: k8s.V1NetworkPolicy | null,+ wanted: k8s.V1NetworkPolicy+ ) => boolean+ }+ ).egressWriteNeeded(live, desired)++ expect(writeNeeded).toBe(true)++ live.metadata!.labels = desired.metadata!.labels+ live.metadata!.ownerReferences = [+ {+ apiVersion: 'apps/v1',+ kind: 'Deployment',+ name: 'foreign',+ uid: 'foreign-uid',+ controller: true,+ },+ ]+ expect(+ (+ reconciler as unknown as {+ egressWriteNeeded: (+ existing: k8s.V1NetworkPolicy | null,+ wanted: k8s.V1NetworkPolicy+ ) => boolean+ }+ ).egressWriteNeeded(live, desired)+ ).toBe(true)+ })++ it('T3 creates on 404 and the next equivalent reconcile is read-only', async () => {+ const desired = plainPolicy()+ mockNetworkingApi.readNamespacedNetworkPolicy.mockRejectedValueOnce({ code: 404 })++ await apply(desired)+ expect(mockNetworkingApi.createNamespacedNetworkPolicy).toHaveBeenCalledTimes(1)+ const created = clone(+ (+ mockNetworkingApi.createNamespacedNetworkPolicy.mock.calls[0][0] as {+ body: k8s.V1NetworkPolicy+ }+ ).body+ )+ expect(created.metadata?.annotations?.[SPEC_HASH]).toBeUndefined()++ mockNetworkingApi.createNamespacedNetworkPolicy.mockClear()+ mockNetworkingApi.readNamespacedNetworkPolicy.mockResolvedValue(livePolicy(created))++ await apply(desired)+ expect(mockNetworkingApi.createNamespacedNetworkPolicy).not.toHaveBeenCalled()+ expect(mockNetworkingApi.replaceNamespacedNetworkPolicy).not.toHaveBeenCalled()+ })++ it('T4 propagates a foreign intdep ownership veto after exactly one read', async () => {+ const desired = intdepPolicy()+ mockNetworkingApi.readNamespacedNetworkPolicy+ .mockResolvedValueOnce({+ metadata: {+ name: 'wr-intdep-egress-r-w',+ resourceVersion: '9',+ labels: { 'clerum.io/managed-by': 'hcc' },+ },+ })+ .mockRejectedValueOnce({ code: 503 })++ const rejection = await apply(desired, 'internal-dependency').then(+ () => new Error('expected the ownership assertion to reject'),+ (error: unknown) => error as Error+ )+ expect(rejection.name).toBe('NetworkPolicyOwnershipConflictError')+ expect(rejection.message).toMatch(/Refusing to replace NetworkPolicy "wr-intdep-egress-r-w"/)+ expect(mockNetworkingApi.createNamespacedNetworkPolicy).not.toHaveBeenCalled()+ expect(mockNetworkingApi.replaceNamespacedNetworkPolicy).not.toHaveBeenCalled()+ expect(mockNetworkingApi.readNamespacedNetworkPolicy).toHaveBeenCalledTimes(1)+ })++ it('T4b lets an intdep policy WRC owns converge read-only', async () => {+ const desired = intdepPolicy()+ mockNetworkingApi.readNamespacedNetworkPolicy.mockResolvedValue(livePolicy(desired))++ await expect(apply(desired, 'internal-dependency')).resolves.toBeUndefined()++ expect(mockNetworkingApi.readNamespacedNetworkPolicy).toHaveBeenCalledTimes(1)+ expect(mockNetworkingApi.replaceNamespacedNetworkPolicy).not.toHaveBeenCalled()+ })++ it('T5 retries an HTTP read failure, replaces with the recovered RV, and logs safely', async () => {+ const desired = plainPolicy()+ mockNetworkingApi.readNamespacedNetworkPolicy+ .mockRejectedValueOnce({ code: 500, authorization: 'Bearer must-not-appear' })+ .mockResolvedValueOnce(livePolicy(desired))+ const captured = captureNetworkPolicyLogs()+ try {+ await apply(desired)++ expect(mockNetworkingApi.replaceNamespacedNetworkPolicy).toHaveBeenCalledTimes(1)+ expect(mockNetworkingApi.readNamespacedNetworkPolicy).toHaveBeenCalledTimes(2)+ expect(+ (+ mockNetworkingApi.replaceNamespacedNetworkPolicy.mock.calls[0][0] as {+ body: k8s.V1NetworkPolicy+ }+ ).body.metadata?.resourceVersion+ ).toBe('9')+ const warning = captured.entries.find(+ entry => entry.msg === 'network policy read unavailable; retrying once'+ )+ expect(warning).toEqual(+ expect.objectContaining({+ level: 'warn',+ errorName: 'NetworkPolicyInfraError',+ errorCode: 500,+ retryable: true,+ })+ )+ expect(JSON.stringify(warning)).not.toContain('must-not-appear')+ } finally {+ captured.restore()+ }+ })++ it('T5b retries a transport read failure once and replaces', async () => {+ const desired = plainPolicy()+ mockNetworkingApi.readNamespacedNetworkPolicy+ .mockRejectedValueOnce({ code: 'ECONNRESET' })+ .mockResolvedValueOnce(livePolicy(desired))++ await apply(desired)++ expect(mockNetworkingApi.replaceNamespacedNetworkPolicy).toHaveBeenCalledTimes(1)+ expect(mockNetworkingApi.readNamespacedNetworkPolicy).toHaveBeenCalledTimes(2)+ })++ it('T5c retries a codeless read failure once and replaces', async () => {+ const desired = plainPolicy()+ mockNetworkingApi.readNamespacedNetworkPolicy+ .mockRejectedValueOnce(new Error('transient read timeout'))+ .mockResolvedValueOnce(livePolicy(desired))++ await apply(desired)++ expect(mockNetworkingApi.replaceNamespacedNetworkPolicy).toHaveBeenCalledTimes(1)+ expect(mockNetworkingApi.readNamespacedNetworkPolicy).toHaveBeenCalledTimes(2)+ })++ it('repairs the gateway owner UID and refuses a foreign controller owner', async () => {+ const desired = gatewayPolicy('uid-current')+ mockNetworkingApi.readNamespacedNetworkPolicy.mockResolvedValue(+ livePolicy(gatewayPolicy('uid-old'), { annotations: { [SPEC_HASH]: 'legacy-seal' } })+ )++ await apply(desired, 'webhook-gateway')++ expect(mockNetworkingApi.replaceNamespacedNetworkPolicy).toHaveBeenCalledTimes(1)+ const body = (+ mockNetworkingApi.replaceNamespacedNetworkPolicy.mock.calls[0][0] as {+ body: k8s.V1NetworkPolicy+ }+ ).body+ expect(body.metadata?.ownerReferences?.[0].uid).toBe('uid-current')++ mockNetworkingApi.replaceNamespacedNetworkPolicy.mockClear()+ const foreign = livePolicy(desired)+ foreign.metadata!.ownerReferences = [+ {+ apiVersion: 'apps/v1',+ kind: 'Deployment',+ name: 'foreign',+ uid: 'foreign-uid',+ controller: true,+ },+ ]+ mockNetworkingApi.readNamespacedNetworkPolicy.mockResolvedValue(foreign)++ await expect(apply(desired, 'webhook-gateway')).rejects.toMatchObject({+ name: 'NetworkPolicyOwnershipConflictError',+ })+ expect(mockNetworkingApi.replaceNamespacedNetworkPolicy).not.toHaveBeenCalled()+ })++ it('handles a read/create race by re-reading instead of blind replacing', async () => {+ const desired = plainPolicy()+ mockNetworkingApi.readNamespacedNetworkPolicy+ .mockRejectedValueOnce({ code: 404 })+ .mockResolvedValueOnce(livePolicy(desired))+ mockNetworkingApi.createNamespacedNetworkPolicy.mockRejectedValueOnce({ code: 409 })++ await apply(desired)++ expect(mockNetworkingApi.readNamespacedNetworkPolicy).toHaveBeenCalledTimes(2)+ expect(mockNetworkingApi.createNamespacedNetworkPolicy).toHaveBeenCalledTimes(1)+ expect(mockNetworkingApi.replaceNamespacedNetworkPolicy).not.toHaveBeenCalled()+ })++ it('creates after a transient first read is followed by a confirmed 404', async () => {+ const desired = plainPolicy()+ mockNetworkingApi.readNamespacedNetworkPolicy+ .mockRejectedValueOnce({ code: 500 })+ .mockRejectedValueOnce({ code: 404 })++ await apply(desired)++ expect(mockNetworkingApi.readNamespacedNetworkPolicy).toHaveBeenCalledTimes(2)+ expect(mockNetworkingApi.createNamespacedNetworkPolicy).toHaveBeenCalledTimes(1)+ expect(mockNetworkingApi.replaceNamespacedNetworkPolicy).not.toHaveBeenCalled()+ })++ it('uses a supplied absent workload-egress snapshot without issuing another read', async () => {+ const desired = plainPolicy()+ desired.metadata!.name = 'wl-egress-r-w'++ await apply(desired, 'workload-egress', null)++ expect(mockNetworkingApi.readNamespacedNetworkPolicy).not.toHaveBeenCalled()+ expect(mockNetworkingApi.createNamespacedNetworkPolicy).toHaveBeenCalledTimes(1)+ })++ it('retries when a create-conflict winner is absent on the bounded re-read', async () => {+ const desired = plainPolicy()+ mockNetworkingApi.readNamespacedNetworkPolicy.mockRejectedValue({ code: 404 })+ mockNetworkingApi.createNamespacedNetworkPolicy.mockRejectedValueOnce({ code: 409 })++ await expect(apply(desired)).rejects.toMatchObject({+ name: 'RetryableReconcileError',+ cause: { code: 409 },+ })+ expect(mockNetworkingApi.readNamespacedNetworkPolicy).toHaveBeenCalledTimes(2)+ expect(mockNetworkingApi.createNamespacedNetworkPolicy).toHaveBeenCalledTimes(1)+ expect(mockNetworkingApi.replaceNamespacedNetworkPolicy).not.toHaveBeenCalled()+ })++ it('retries instead of blind replacing a drifted live policy without resourceVersion', async () => {+ const desired = plainPolicy()+ const live = livePolicy(desired)+ delete live.metadata!.resourceVersion+ live.spec!.podSelector = {}+ mockNetworkingApi.readNamespacedNetworkPolicy.mockResolvedValue(live)++ await expect(apply(desired)).rejects.toMatchObject({+ name: 'RetryableReconcileError',+ message: expect.stringMatching(/has no resourceVersion/),+ })+ expect(mockNetworkingApi.replaceNamespacedNetworkPolicy).not.toHaveBeenCalled()+ })++ it.each([404, 409])('classifies replace race status %s as retryable', async code => {+ const desired = plainPolicy()+ const live = livePolicy(desired)+ live.spec!.podSelector = {}+ mockNetworkingApi.readNamespacedNetworkPolicy.mockResolvedValue(live)+ mockNetworkingApi.replaceNamespacedNetworkPolicy.mockRejectedValueOnce({ code })++ await expect(apply(desired)).rejects.toMatchObject({+ name: 'RetryableReconcileError',+ cause: { code },+ })+ })++ it.each([500, 'ETIMEDOUT'])('preserves replace infrastructure failure %s', async code => {+ const desired = plainPolicy()+ const live = livePolicy(desired)+ live.spec!.podSelector = {}+ mockNetworkingApi.readNamespacedNetworkPolicy.mockResolvedValue(live)+ const apiError = Object.assign(new Error(`replace failed: ${code}`), { code })+ mockNetworkingApi.replaceNamespacedNetworkPolicy.mockRejectedValueOnce(apiError)++ await expect(apply(desired)).rejects.toMatchObject({+ name: 'NetworkPolicyInfraError',+ code,+ cause: apiError,+ })+ })++ it('propagates a permanent replace rejection without reclassifying it as transient', async () => {+ const desired = plainPolicy()+ const live = livePolicy(desired)+ live.spec!.podSelector = {}+ mockNetworkingApi.readNamespacedNetworkPolicy.mockResolvedValue(live)+ const permanent = { code: 422, reason: 'Invalid' }+ mockNetworkingApi.replaceNamespacedNetworkPolicy.mockRejectedValueOnce(permanent)++ await expect(apply(desired)).rejects.toBe(permanent)+ })++ it('propagates a permanent create rejection without attempting a replace', async () => {+ const desired = plainPolicy()+ const permanent = { code: 422, reason: 'Invalid' }+ mockNetworkingApi.readNamespacedNetworkPolicy.mockRejectedValueOnce({ code: 404 })+ mockNetworkingApi.createNamespacedNetworkPolicy.mockRejectedValueOnce(permanent)++ await expect(apply(desired)).rejects.toBe(permanent)+ expect(mockNetworkingApi.replaceNamespacedNetworkPolicy).not.toHaveBeenCalled()+ })++ it('propagates a permanent second-read rejection after the bounded retry', async () => {+ const desired = plainPolicy()+ const permanent = { code: 422, reason: 'Invalid' }+ mockNetworkingApi.readNamespacedNetworkPolicy+ .mockRejectedValueOnce({ code: 500 })+ .mockRejectedValueOnce(permanent)++ await expect(apply(desired)).rejects.toBe(permanent)+ expect(mockNetworkingApi.readNamespacedNetworkPolicy).toHaveBeenCalledTimes(2)+ expect(mockNetworkingApi.createNamespacedNetworkPolicy).not.toHaveBeenCalled()+ expect(mockNetworkingApi.replaceNamespacedNetworkPolicy).not.toHaveBeenCalled()+ })++ it('refuses to create a gateway policy without a complete desired owner', async () => {+ const desired = gatewayPolicy('')++ await expect(apply(desired, 'webhook-gateway')).rejects.toMatchObject({+ name: 'NetworkPolicyOwnershipConflictError',+ })+ expect(mockNetworkingApi.readNamespacedNetworkPolicy).not.toHaveBeenCalled()+ expect(mockNetworkingApi.createNamespacedNetworkPolicy).not.toHaveBeenCalled()+ })++ it('refuses a desired gateway policy with multiple controller owners before I/O', async () => {+ const desired = gatewayPolicy()+ desired.metadata!.ownerReferences!.push({+ apiVersion: 'apps/v1',+ kind: 'Deployment',+ name: 'second-controller',+ uid: 'second-controller-uid',+ controller: true,+ })++ await expect(apply(desired, 'webhook-gateway')).rejects.toMatchObject({+ name: 'NetworkPolicyOwnershipConflictError',+ })+ expect(mockNetworkingApi.readNamespacedNetworkPolicy).not.toHaveBeenCalled()+ expect(mockNetworkingApi.createNamespacedNetworkPolicy).not.toHaveBeenCalled()+ })++ it('refuses a desired policy without metadata.name before I/O', async () => {+ const desired = plainPolicy()+ delete desired.metadata!.name++ await expect(apply(desired)).rejects.toThrow(/without metadata\.name/)+ expect(mockNetworkingApi.readNamespacedNetworkPolicy).not.toHaveBeenCalled()+ expect(mockNetworkingApi.createNamespacedNetworkPolicy).not.toHaveBeenCalled()+ })+ })+describe('idempotent apply (stops generation churn)', () => {const SPEC_HASH = 'clerum.io/spec-hash'const clone = <T>(value: T): T => JSON.parse(JSON.stringify(value)) as T@@@ -3184,136 -3019,6 +3881,140 @@@)})+ describe('workload egress pre-read infrastructure failures', () => {- it.each([- { workflow: true, code: 503 },- { workflow: true, code: 'ECONNRESET' },- { workflow: false, code: 503 },- { workflow: false, code: 'ECONNRESET' },- { workflow: true, code: 403 },- { workflow: false, code: 403 },- ])('classifies $code after successful DNS (workflow=$workflow)', async ({ workflow, code }) => {++ it.each(++ (['before DNS', 'after DNS'] as const).flatMap(stage =>++ [++ { workflow: true, code: 503 },++ { workflow: true, code: 'ECONNRESET' },++ { workflow: false, code: 503 },++ { workflow: false, code: 'ECONNRESET' },++ { workflow: true, code: 403 },++ { workflow: false, code: 403 },++ ].map(scenario => ({ ...scenario, stage }))++ )++ )('classifies $code $stage (workflow=$workflow)', async ({ workflow, code, stage }) => {+ const recipe = makeRecipe({+ spec: {+ workloads: [+ {+ id: 'worker',+ type: 'deployment',+ image: 'worker:latest',+ port: 8080,+ egressBindings: [{ dns: 'api.example.com', port: 443 }],+ },+ ],+ ...(workflow ? { steps: [{ id: 'run', run: snippetRun() }] } : {}),+ },+ ...(!workflow+ ? {+ status: {+ phase: 'active' as const,+ message: 'Healthy before read failure',+ workloads: [],+ },+ }+ : {}),+ })+ const lookup = vi.fn(async () => ({+ kind: 'ok' as const,+ ipv4: ['93.184.216.10'],+ ipv6: [],+ ttlSeconds: 300,+ }))+ const rec = new WorkflowRecipeReconciler(new k8s.KubeConfig(), undefined, {+ fqdnLookup: lookup,+ })+ const workflowReconcile = vi.fn()+ ;(rec as unknown as { workflowReconciler: unknown }).workflowReconciler = {+ reconcile: workflowReconcile,+ validateWorkflowSpec: () => undefined,+ }+ const failure = { code }+ mockNetworkingApi.readNamespacedNetworkPolicy.mockImplementation(async ({ name }) => {+ if (name === 'wl-egress-test-recipe-worker') {- expect(lookup).toHaveBeenCalled()++ if (stage === 'after DNS' && lookup.mock.calls.length === 0) throw { code: 404 }+ throw failure+ }+ throw { code: 404 }+ })++ const result = await rec.reconcile(recipe)+- expect(lookup).toHaveBeenCalledTimes(1)++ expect(lookup).toHaveBeenCalledTimes(stage === 'after DNS' ? 1 : 0)+ if (code === 403) {+ expect(result.phase).toBe('failed')+ expect(result.requeueAfterMs).toBeUndefined()+ } else {+ expect(result.phase).toBe(workflow ? 'degraded' : 'active')+ expect(result.requeueAfterMs).toBe(TRANSIENT_REQUEUE_BASE_MS)+ if (!workflow) expect(result.skipStatusPatch).toBe(true)+ }+ expect(workflowReconcile).not.toHaveBeenCalled()+ expect(mockNetworkingApi.createNamespacedNetworkPolicy).toHaveBeenCalledTimes(0)+ expect(mockNetworkingApi.replaceNamespacedNetworkPolicy).toHaveBeenCalledTimes(0)+ })+ })++ describe('reconciliation completion receipt', () => {+ it.each(['success', 'reconcile-throw', 'projection-throw'] as const)(+ 'emits completion only after the full pipeline returns (%s)',+ async outcome => {+ const recipe = makeRecipe()+ recipe.metadata.generation = 7+ const previousLevel = process.env.LOG_LEVEL+ process.env.LOG_LEVEL = 'info'+ const receipts: Array<Record<string, unknown>> = []+ const write = vi.spyOn(process.stdout, 'write').mockImplementation(((chunk: unknown) => {+ for (const line of String(chunk).split('\n')) {+ if (!line.startsWith('{')) continue+ const record = JSON.parse(line)+ if (record.msg === 'recipe reconciliation completed') receipts.push(record)+ }+ return true+ }) as typeof process.stdout.write)+ const priv = reconciler as unknown as {+ reconcileInternal: () => Promise<unknown>+ projectPluginWorkloadSdk: () => unknown+ }+ const failure = new Error('pipeline failure')+ const injected =+ outcome === 'reconcile-throw'+ ? vi.spyOn(priv, 'reconcileInternal').mockRejectedValue(failure)+ : outcome === 'projection-throw'+ ? vi.spyOn(priv, 'projectPluginWorkloadSdk').mockImplementation(() => {+ throw failure+ })+ : undefined+ try {+ if (outcome === 'success') {+ const result = await reconciler.reconcile(recipe)+ expect(result.phase).toBe('active')+ expect(receipts).toEqual([+ expect.objectContaining({+ recipe: 'test-recipe',+ namespace: 'sandbox-recipes',+ uid: 'uid-123',+ generation: 7,+ phase: 'active',+ requeueAfterMs: 0,+ }),+ ])+ } else {+ await expect(reconciler.reconcile(recipe)).rejects.toBe(failure)+ expect(receipts).toEqual([])+ }+ } finally {+ injected?.mockRestore()+ write.mockRestore()+ if (previousLevel === undefined) delete process.env.LOG_LEVEL+ else process.env.LOG_LEVEL = previousLevel+ }+ }+ )+ })+it('fails non-transport external egress in required mode when cluster enforcement is not confirmed', async () => {const recipe = makeRecipe({spec: {@@@ -10160,76 -10890,6 +11888,76 @@@expect(allFalse).toBe(true)})+ it('T6 (#575) converges the 3 gateway NetworkPolicies through live reads', async () => {+ mockNetworkingApi.createNamespacedNetworkPolicy.mockClear()+ mockNetworkingApi.readNamespacedNetworkPolicy.mockRejectedValue({ code: 404 })+ await reconciler.reconcile(recipeWithWebhook())++ const sealed = new Map<string, unknown>()+ for (const call of mockNetworkingApi.createNamespacedNetworkPolicy.mock.calls) {+ const body = (call[0] as { body: k8s.V1NetworkPolicy }).body+ sealed.set(body.metadata!.name!, JSON.parse(JSON.stringify(body)))+ }+ // Which three, not just how many: cardinality alone stays green if a refactor+ // stops emitting gateway policies and three arrive from another family.+ expect([...sealed.keys()].sort()).toEqual([+ 'allow-gateway-egress-to-handler-wf-wh-recipe',+ 'allow-gateway-ingress-to-handler-wf-wh-recipe',+ 'allow-webhook-proxy-ingress-wf-wh-recipe',+ ])++ mockNetworkingApi.readNamespacedNetworkPolicy.mockImplementation(+ ({ name }: { name: string }) => {+ const previous = sealed.get(name) as k8s.V1NetworkPolicy | undefined- if (!previous) return Promise.resolve({ metadata: { name, resourceVersion: '1' } })++ if (!previous) return Promise.reject({ code: 404 })+ return Promise.resolve({+ ...previous,+ metadata: { ...previous.metadata, resourceVersion: '9' },+ })+ }+ )+ mockNetworkingApi.replaceNamespacedNetworkPolicy.mockClear()+ mockNetworkingApi.createNamespacedNetworkPolicy.mockClear()+ mockNetworkingApi.readNamespacedNetworkPolicy.mockClear()+ const previousLevel = process.env.LOG_LEVEL+ process.env.LOG_LEVEL = 'info'+ const logEntries: Array<Record<string, unknown>> = []+ const logSpy = vi.spyOn(process.stdout, 'write').mockImplementation(((chunk: unknown) => {+ const text = typeof chunk === 'string' ? chunk : Buffer.from(chunk as Uint8Array).toString()+ for (const line of text.split('\n')) {+ if (!line.startsWith('{')) continue+ const parsed = JSON.parse(line) as Record<string, unknown>+ if (parsed.msg === 'network policy unchanged; skipping update') logEntries.push(parsed)+ }+ return true+ }) as unknown as typeof process.stdout.write)++ try {+ await reconciler.reconcile(recipeWithWebhook())++ for (const name of sealed.keys()) {+ expect(mockNetworkingApi.replaceNamespacedNetworkPolicy).not.toHaveBeenCalledWith(+ expect.objectContaining({ name })+ )+ expect(mockNetworkingApi.readNamespacedNetworkPolicy).toHaveBeenCalledWith(+ expect.objectContaining({ name })+ )+ expect(logEntries).toContainEqual(+ expect.objectContaining({+ policy: name,+ family: 'webhook-gateway',+ msg: 'network policy unchanged; skipping update',+ })+ )+ }+ expect(mockNetworkingApi.createNamespacedNetworkPolicy).not.toHaveBeenCalled()+ } finally {+ logSpy.mockRestore()+ if (previousLevel === undefined) delete process.env.LOG_LEVEL+ else process.env.LOG_LEVEL = previousLevel+ }+ })+it('fail-closed when secretRef is missing: no gateway resources, condition set, phase degraded', async () => {mockAppsApi.createNamespacedDeployment.mockClear()mockCoreApi.createNamespacedService.mockClear()diff --cc workflow-recipes/src/reconciler/workflowRecipeReconciler.tsindex e364bb611,f6f7e0dbd..d0e2d294e--- a/workflow-recipes/src/reconciler/workflowRecipeReconciler.ts+++ b/workflow-recipes/src/reconciler/workflowRecipeReconciler.ts@@@ -89,12 -99,11 +99,17 @@@ importwaitForExternalEgressReady,waitForNetworkReady,} from './mcpDelegation'+ import {+ effectivePolicyTypes,+ intersectNetworkPolicyRules,+ networkPolicySpecSignature,+ } from './networkPolicyContraction'+import {+ type NetworkPolicyFamily,+ buildNetworkPolicyReplacement,+ decideNetworkPolicyConvergence,+ networkPolicyMetadataMatchesDesired,+} from './networkPolicyConvergence'import { issueOAuthBrokerToken } from './oauthBrokerTokenIssuerClient'import {PLUGIN_WORKLOAD_SDK_CONDITION_TYPE,@@@ -478,26 -519,25 +525,46 @@@ class NetworkPolicyOwnershipConflictErr}}+class NetworkPolicyInfraError extends Error {+ readonly code?: string | number++ constructor(message: string, cause: unknown, code?: string | number) {+ super(message, { cause })+ this.name = 'NetworkPolicyInfraError'+ this.code = code+ }+}++interface NetworkPolicyApplyOptions {+ family: NetworkPolicyFamily+ existing?: k8s.V1NetworkPolicy | null++ recipeName?: string+}++interface NetworkPolicyReadResult {+ existing: k8s.V1NetworkPolicy | null+ recoveredFromReadFailure: boolean+}++ /** A fenced mutation lost its snapshot; only a fresh reconciliation may retry. */+ class NetworkPolicyReplanRequiredError extends RetryableReconcileError {+ override name = 'NetworkPolicyReplanRequiredError'+ }++ class TransportNetworkReadinessPendingError extends RetryableReconcileError {+ override name = 'TransportNetworkReadinessPendingError'+ }++ function throwNetworkPolicyMutationError(error: unknown, name: string, namespace: string): never {+ if (getErrorCode(error) === 409 || getErrorCode(error) === 404) {+ throw new NetworkPolicyReplanRequiredError(+ `NetworkPolicy "${name}" in ${namespace} changed during a fenced mutation; a fresh reconciliation is required`,+ { cause: error }+ )+ }+ throw error+ }+class InternalDependencyReconcileError extends Error {constructor(message: string,@@@ -1958,7 -2114,13 +2159,10 @@@ export class WorkflowRecipeReconcilerworkloadStatuses: [],internalDependencyConditions,workloadConditions,+ requeueAfterMs: TRANSIENT_REQUEUE_BASE_MS,+ ...(error instanceof TransportNetworkReadinessPendingError+ ? { transportNetworkConditions: pendingTransportNetworkConditions(recipe, message) }+ : {}),- ...(error instanceof NetworkPolicyReplanRequiredError ||- error instanceof TransportNetworkReadinessPendingError- ? { requeueAfterMs: TRANSIENT_REQUEUE_BASE_MS }- : {}),}}return {@@@ -2728,7 -2910,18 +2952,15 @@@phase: 'degraded',message: error.message,workloadStatuses: [],+ requeueAfterMs: TRANSIENT_REQUEUE_BASE_MS,+ ...(error instanceof TransportNetworkReadinessPendingError+ ? {+ transportNetworkConditions: pendingTransportNetworkConditions(+ recipe,+ error.message+ ),+ }+ : {}),- ...(error instanceof NetworkPolicyReplanRequiredError ||- error instanceof TransportNetworkReadinessPendingError- ? { requeueAfterMs: TRANSIENT_REQUEUE_BASE_MS }- : {}),}}// (2) A raw retryable infra error (connect ETIMEDOUT/ECONNRESET, 5xx,@@@ -2746,12 -2939,10 +2978,13 @@@// decision and hiding the permanent misconfiguration.const isEgressResolutionError =error instanceof Error && error.message.includes('egress resolution failed')- if (!isEgressResolutionError && isRetryableInfraError(error)) {+ if (+ !isEgressResolutionError &&+ (error instanceof NetworkPolicyInfraError || isRetryableInfraError(error))+ ) {- console.warn(- `[WR-Reconciler] Transient infra error reconciling "${name}" — keeping phase "${currentPhase}", will retry: ${String(error)}`+ createLogger('wrc', recipe.metadata.name).warn(+ 'Transient infrastructure error during reconciliation; keeping phase and retrying',+ { name, currentPhase, err: error })return {phase: currentPhase as RecipePhase,@@@ -4317,7 -4661,7 +4691,10 @@@await this.deleteUiIngressIfExists(recipe.metadata.name, w.id, ns)continue}- await this.applyNetworkPolicy(policy, ns, { family: 'ui-ingress' })- await this.applyOwnedRecipeNetworkPolicy(policy, ns, recipe.metadata.name)++ await this.applyNetworkPolicy(policy, ns, {++ family: 'ui-ingress',++ recipeName: recipe.metadata.name,++ })}}@@@ -4693,23 -5134,20 +5170,24 @@@!wlEgressStateChanged) {createLogger('wrc', recipe.metadata.name).info(- 'NetworkPolicy egress set unchanged; no-op',- { wlPolicyName, wlNs }- )- } else {- await this.applyOwnedRecipeNetworkPolicy(policy, wlNs, recipe.metadata.name)- // #299: the policy has landed — record the set it actually enforces.- this.logResolvedEgressSet(- recipe.metadata.name,- wlPolicyName,- externalDeclared.map(e => e.fqdn),- effectiveExternal,- wlEgressStateChanged+ 'network policy egress set unchanged; skipping live apply',+ { policy: wlPolicyName, namespace: wlNs, family: 'workload-egress' })+ continue}+ await this.applyNetworkPolicy(policy, wlNs, {+ family: 'workload-egress',++ recipeName: recipe.metadata.name,+ existing: existingWlPolicy,+ })+ // #299: the policy has landed — record the set it actually enforces.+ this.logResolvedEgressSet(+ recipe.metadata.name,+ wlPolicyName,+ externalDeclared.map(e => e.fqdn),+ effectiveExternal,+ wlEgressStateChanged+ )}// Now ingress side: for each target workload that any sibling pointed@@@ -4728,7 -5166,7 +5206,10 @@@await this.deleteWorkloadIngressIfExists(recipe.metadata.name, w.id, wlNs)continue}- await this.applyNetworkPolicy(policy, wlNs, { family: 'workload-ingress' })- await this.applyOwnedRecipeNetworkPolicy(policy, wlNs, recipe.metadata.name)++ await this.applyNetworkPolicy(policy, wlNs, {++ family: 'workload-ingress',++ recipeName: recipe.metadata.name,++ })}}@@@ -4929,203 -5532,29 +5584,206 @@@}}- private async applyNetworkPolicy(policy: k8s.V1NetworkPolicy, namespace: string): Promise<void> {- const policyName = policy.metadata!.name!- await this.createOrReplace(- () =>- this.networkingApi.createNamespacedNetworkPolicy({- namespace,- body: policy,- }),- async () => {- const existing = await this.networkingApi.readNamespacedNetworkPolicy({- name: policyName,- namespace,- })- this.assertInternalDependencyPolicyOwnership(policy, existing, namespace)- policy.metadata!.resourceVersion = existing.metadata?.resourceVersion- return this.networkingApi.replaceNamespacedNetworkPolicy({- name: policyName,- namespace,- body: policy,- })- },- `NetworkPolicy "${policyName}" in ${namespace}`+ /**+ * Reconcile one WRC-authored NetworkPolicy from the live object, not from the+ * legacy spec-hash seal. NetworkPolicy is an enforcement resource: a stored+ * annotation can describe what WRC wrote previously while the live spec or+ * lifecycle ownership has since drifted. One validated live snapshot therefore+ * owns both the no-op decision and the resourceVersion used by a replace.+ */+ private async applyNetworkPolicy(+ policy: k8s.V1NetworkPolicy,+ namespace: string,+ options: NetworkPolicyApplyOptions+ ): Promise<void> {+ const desired = structuredClone(policy)+ const policyName = desired.metadata?.name+ if (!policyName) throw new Error('Cannot apply a NetworkPolicy without metadata.name')+ desired.metadata = desired.metadata ?? {}+ desired.metadata.namespace = desired.metadata.namespace ?? namespace+ this.assertDesiredNetworkPolicyOwner(desired, options.family, namespace)++ const recipeName =+ desired.metadata?.labels?.['clerum.io/recipe'] ??+ desired.metadata?.labels?.['clerum.io/recipe-name'] ??+ policyName+ const log = createLogger('wrc', recipeName)+ const fields = { policy: policyName, namespace, family: options.family }+ const hasSuppliedExisting = Object.prototype.hasOwnProperty.call(options, 'existing')+ let readResult: NetworkPolicyReadResult = hasSuppliedExisting+ ? { existing: options.existing ?? null, recoveredFromReadFailure: false }+ : await this.readNetworkPolicyForApply(policyName, namespace, log, fields)+ let existing = readResult.existing++ if (!existing) {+ try {+ await this.networkingApi.createNamespacedNetworkPolicy({ namespace, body: desired })+ log.info('network policy created', fields)+ return+ } catch (error: unknown) {+ if (getErrorCode(error) !== 409) {+ log.error('network policy create failed', {+ ...fields,+ ...this.safeInfrastructureErrorFields(error),+ })+ if (isRetryableInfraError(error)) {+ const safe = this.safeInfrastructureErrorFields(error)+ throw new NetworkPolicyInfraError(+ `NetworkPolicy "${policyName}" in ${namespace} create temporarily unavailable (${safe.errorCode ?? safe.errorName})`,+ error,+ safe.errorCode+ )+ }+ throw error+ }+ // A real create race is exceptional in the read-first path. Re-read once+ // and evaluate the winner from scratch; never blind-replace after 409.+ readResult = await this.readNetworkPolicyForApply(policyName, namespace, log, fields)+ existing = readResult.existing+ if (!existing) {+ throw new RetryableReconcileError(+ `NetworkPolicy "${policyName}" in ${namespace} still absent after create conflict`,+ { cause: error }+ )+ }+ }+ }++ // Deletion wins over ownership: a terminating object cannot be adopted or+ // safely classified from partial labels. Wait for 404, then recreate from the+ // current desired state.+ if (existing.metadata?.deletionTimestamp) {+ throw new RetryableReconcileError(+ `NetworkPolicy "${policyName}" in ${namespace} is terminating; retrying after deletion`+ )+ }++ // Domain invariants run outside the read catch. A confirmed veto propagates+ // immediately and can never be replaced by a second transport error.++ if (options.recipeName) {++ this.assertRecipeNetworkPolicyOwnership(existing, options.recipeName, policyName, namespace)++ }+ this.assertInternalDependencyPolicyOwnership(desired, existing, namespace)+ const decision = decideNetworkPolicyConvergence(options.family, desired, existing)+ if (decision.action === 'conflict') {+ throw new NetworkPolicyOwnershipConflictError(+ `Refusing to replace NetworkPolicy "${policyName}" in ${namespace}: ${decision.reason}`+ )+ }+ if (decision.action === 'retry') {+ throw new RetryableReconcileError(+ `NetworkPolicy "${policyName}" in ${namespace} is terminating; retrying after deletion`+ )+ }+ if (decision.action === 'unchanged' && !readResult.recoveredFromReadFailure) {+ log.info('network policy unchanged; skipping update', fields)+ return+ }++ const resourceVersion = existing.metadata?.resourceVersion+ if (!resourceVersion) {+ throw new RetryableReconcileError(+ `NetworkPolicy "${policyName}" in ${namespace} has no resourceVersion`+ )+ }+ const replacement = buildNetworkPolicyReplacement(desired, existing)+ try {+ await this.networkingApi.replaceNamespacedNetworkPolicy({+ name: policyName,+ namespace,+ body: replacement,+ })+ log.info('network policy replaced', {+ ...fields,+ reason: decision.action === 'unchanged' ? 'read-recovery' : decision.reason,+ })+ } catch (error: unknown) {+ log.error('network policy replace failed', {+ ...fields,+ ...this.safeInfrastructureErrorFields(error),+ })+ const errorCode = getErrorCode(error)+ if (errorCode === 404 || errorCode === 409) {+ throw new RetryableReconcileError(+ `NetworkPolicy "${policyName}" in ${namespace} ${errorCode === 404 ? 'disappeared' : 'changed'} during replace`,+ { cause: error }+ )+ }+ if (isRetryableInfraError(error)) {+ const safe = this.safeInfrastructureErrorFields(error)+ throw new NetworkPolicyInfraError(+ `NetworkPolicy "${policyName}" in ${namespace} replace temporarily unavailable (${safe.errorCode ?? safe.errorName})`,+ error,+ safe.errorCode+ )+ }+ throw error+ }+ }++ private assertDesiredNetworkPolicyOwner(+ desired: k8s.V1NetworkPolicy,+ family: NetworkPolicyFamily,+ namespace: string+ ): void {+ const controllerOwners = (desired.metadata?.ownerReferences ?? []).filter(+ owner => owner.controller === true)+ const invalidControllerOwner =+ controllerOwners.length > 1 || controllerOwners.some(owner => !owner.uid)+ const gatewayOwnerMissing = family === 'webhook-gateway' && controllerOwners.length !== 1+ if (invalidControllerOwner || gatewayOwnerMissing) {+ throw new NetworkPolicyOwnershipConflictError(+ `Refusing to apply NetworkPolicy "${desired.metadata?.name ?? '<unknown>'}" in ${namespace}: desired controller owner is incomplete`+ )+ }+ }++ private async readNetworkPolicyForApply(+ name: string,+ namespace: string,+ log: ReturnType<typeof createLogger>,+ fields: { policy: string; namespace: string; family: NetworkPolicyFamily }+ ): Promise<NetworkPolicyReadResult> {+ for (let attempt = 1; attempt <= 2; attempt += 1) {+ try {+ return {+ existing: await this.readNetworkPolicyOrNull(name, namespace),+ recoveredFromReadFailure: attempt > 1,+ }+ } catch (error: unknown) {+ if (attempt === 2) {+ throw error+ }+ log.warn('network policy read unavailable; retrying once', {+ ...fields,+ ...this.safeInfrastructureErrorFields(error),+ })+ }+ }+ return { existing: null, recoveredFromReadFailure: false }+ }++ private safeInfrastructureErrorFields(error: unknown): {+ errorName: string+ errorCode?: string | number+ retryable: boolean+ } {+ const candidateCode = (error as { code?: unknown } | null)?.code+ const candidateName = error instanceof Error ? error.name : 'UnknownError'+ const errorCode =+ typeof candidateCode === 'number'+ ? candidateCode+ : typeof candidateCode === 'string' && /^[A-Z0-9_]{1,64}$/.test(candidateCode)+ ? candidateCode+ : getErrorCode(error)+ return {+ errorName: /^[A-Za-z][A-Za-z0-9_.-]{0,63}$/.test(candidateName)+ ? candidateName+ : 'UnknownError',+ ...(errorCode === undefined ? {} : { errorCode }),+ retryable: isRetryableInfraError(error),+ }}private coordinatorGfsNetworkPolicy(recipe: WorkflowRecipeCRD): k8s.V1NetworkPolicy {