Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript

Merge origin/dev into fix/issue-575-wrc-networkpolicy-spec-hash-gate

Alfredo Lopez32087f6ef3df
diff --cc workflow-recipes/package-lock.json
index fde522079,8ac088539..a6a434abc
--- a/workflow-recipes/package-lock.json
+++ b/workflow-recipes/package-lock.json
@@@ -1,12 -1,12 +1,12 @@@
{
"name": "workload-recipes-controller",
- "version": "0.1.346",
- "version": "0.1.345",
++ "version": "0.1.347",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "workload-recipes-controller",
- "version": "0.1.346",
- "version": "0.1.345",
++ "version": "0.1.347",
"license": "SEE LICENSE IN LICENSE",
"dependencies": {
"@aws-sdk/client-s3": "^3.1009.0",
diff --cc workflow-recipes/package.json
index af9e20fcd,8d0e99308..a00abeb80
--- a/workflow-recipes/package.json
+++ b/workflow-recipes/package.json
@@@ -1,6 -1,6 +1,6 @@@
{
"name": "workload-recipes-controller",
- "version": "0.1.346",
- "version": "0.1.345",
++ "version": "0.1.347",
"description": "Workload Recipes Controller (WRC) - WorkloadRecipe lifecycle management and MCP server interface",
"main": "dist/main.js",
"engines": {
diff --cc workflow-recipes/src/k8sClient.test.ts
index 62f474273,5a3e71402..af2d4f568
--- a/workflow-recipes/src/k8sClient.test.ts
+++ b/workflow-recipes/src/k8sClient.test.ts
@@@ -1397,75 -1466,55 +1466,124 @@@ describe('workload status refresh loop
expect(reconcile).not.toHaveBeenCalled()
})
+ it('retries a transient degraded result even when the first pass patches status', async () => {
+ vi.useFakeTimers()
+ try {
+ const recipe = makeWorkloadRecipe({
+ metadata: { name: 'policy-race', namespace: 'sandbox-recipes', generation: 1 },
+ status: { phase: 'candidate' },
+ })
+ const reconcile = vi
+ .fn()
+ .mockResolvedValueOnce({
+ phase: 'degraded',
+ message: 'NetworkPolicy changed during replace',
+ workloadStatuses: [],
+ requeueAfterMs: 5_000,
+ })
+ .mockResolvedValueOnce({
+ phase: 'active',
+ message: 'All workloads deployed',
+ workloadStatuses: [],
+ })
+ const patchStatus = vi.fn().mockResolvedValue(undefined)
+ const enqueue = vi.fn((_key: string, task: () => Promise<void>) => task())
+
+ type InternalWatcher = {
+ recipes: Map<string, WorkflowRecipeCRD>
+ transientRetries: Map<string, { timer: ReturnType<typeof setTimeout>; attempts: number }>
+ eventQueue: { enqueue: typeof enqueue }
+ stopped: boolean
+ traceReporter: null
+ dbRunProcessor: null
+ reconciler: {
+ reconcile: typeof reconcile
+ isRecipeStillActive: ReturnType<typeof vi.fn>
+ ensureFinalizer: ReturnType<typeof vi.fn>
+ patchStatus: typeof patchStatus
+ }
+ handleRecipeEvent: (type: string, recipe: WorkflowRecipeCRD) => Promise<void>
+ }
+
+ const internal = Object.create(WorkflowRecipeWatcher.prototype) as InternalWatcher
+ internal.recipes = new Map([[recipe.metadata.name, recipe]])
+ internal.transientRetries = new Map()
+ internal.eventQueue = { enqueue }
+ internal.stopped = false
+ internal.traceReporter = null
+ internal.dbRunProcessor = null
+ internal.reconciler = {
+ reconcile,
+ isRecipeStillActive: vi.fn().mockResolvedValue(true),
+ ensureFinalizer: vi.fn().mockResolvedValue(undefined),
+ patchStatus,
+ }
+
+ await internal.handleRecipeEvent('ADDED', recipe)
+
+ expect(patchStatus).toHaveBeenCalledTimes(1)
+ expect(internal.transientRetries.has(recipe.metadata.name)).toBe(true)
+
+ await vi.advanceTimersByTimeAsync(5_000)
+ await Promise.resolve()
+
+ expect(enqueue).toHaveBeenCalledWith(recipe.metadata.name, expect.any(Function))
+ expect(reconcile).toHaveBeenCalledTimes(2)
+ expect(internal.transientRetries.has(recipe.metadata.name)).toBe(false)
+ } finally {
+ vi.useRealTimers()
+ }
+ })
+
+ it.each([
+ { from: 'failed', to: 'candidate', runs: 1 },
+ { from: 'active', to: 'active', runs: 0 },
+ { from: 'active', to: 'degraded', runs: 0 },
+ { from: 'degraded', to: 'active', runs: 0 },
+ { from: 'failed', to: 'failed', runs: 0 },
+ { from: 'candidate', to: 'candidate', runs: 0 },
+ ] as const)(
+ 'handles the supported retry transition $from -> $to without a generation bump',
+ async ({ from, to, runs }) => {
+ const cached = makeWorkloadRecipe({
+ metadata: { name: 'retry-recipe', namespace: 'sandbox-recipes', generation: 7 },
+ status: { phase: from },
+ })
+ const requested = {
+ ...cached,
+ status: { phase: to, message: 'Manual retry requested by operator' },
+ }
+ const reconcile = vi.fn().mockResolvedValue({
+ phase: 'active',
+ message: 'All workloads deployed',
+ workloadStatuses: [],
+ })
+ const watcher = Object.create(WorkflowRecipeWatcher.prototype) as {
+ recipes: Map<string, WorkflowRecipeCRD>
+ handleRecipeEvent: (type: string, recipe: WorkflowRecipeCRD) => Promise<void>
+ }
+ Object.assign(watcher, {
+ recipes: new Map([[cached.metadata.name, cached]]),
+ transientRetries: new Map(),
+ stopped: false,
+ traceReporter: null,
+ dbRunProcessor: null,
+ reconciler: {
+ reconcile,
+ isRecipeStillActive: vi.fn().mockResolvedValue(true),
+ ensureFinalizer: vi.fn().mockResolvedValue(undefined),
+ patchStatus: vi.fn().mockResolvedValue(undefined),
+ },
+ })
+ await watcher.handleRecipeEvent('MODIFIED', requested)
+ expect(reconcile).toHaveBeenCalledTimes(runs)
+ expect(watcher.recipes.get(cached.metadata.name)).toEqual(requested)
+ // Duplicate status events must not repeatedly execute the retry transition.
+ await watcher.handleRecipeEvent('MODIFIED', requested)
+ expect(reconcile).toHaveBeenCalledTimes(runs)
+ }
+ )
+
it('reconciles when Codex connection-ref changes without a generation bump', async () => {
const cached = makeWorkloadRecipe({
metadata: { name: 'active-recipe', namespace: 'sandbox-recipes', generation: 7 },
diff --cc workflow-recipes/src/reconciler/reconciler.test.ts
index 11747dabe,b82cb9be2..5e5bd0395
--- a/workflow-recipes/src/reconciler/reconciler.test.ts
+++ b/workflow-recipes/src/reconciler/reconciler.test.ts
@@@ -9,8 -9,9 +9,10 @@@ import { WorkflowRecipeWatcher, shouldP
import { CronJobDef, WorkflowRecipeCRD, WorkflowRecipeGfsIntentSpec, WorkloadDef } from '../types'
import { INHERITED_PARENT_RESOURCES_ANNOTATION } from '../workflow/childRecipeFactory'
import { buildCoordinatorGfsNetworkPolicy } from '../workflow/networkPolicyFactory'
+ import { captureLogger } from './__tests__/captureLogger'
+ import { defaultFqdnLookup } from './fqdnResolver'
import { isRetryableInfraError } from './k8sErrors'
+import type { NetworkPolicyFamily } from './networkPolicyConvergence'
import * as brokerIssuer from './oauthBrokerTokenIssuerClient'
import { PLUGIN_WORKLOAD_SDK_PROVIDER_UNAVAILABLE_CONDITION_TYPE } from './pluginWorkloadSdkValidator'
import {
@@@ -254,6 -304,612 +298,615 @@@ describe('WorkflowRecipeReconciler', (
// ─── Pipeline Tests ─────────────────────────────────────────────
+ describe('transport admission pending survives observation and restart', () => {
+ const conditionType = 'TransportExternalEgressReady'
+ const pendingCondition = {
+ type: conditionType,
+ status: 'False' as const,
+ reason: 'ExternalEgressPending',
+ message: 'Transport external egress admission is pending',
+ lastTransitionTime: '2026-09-05T00:00:00.000Z',
+ }
+ function transportRecipe(workflow = false): WorkflowRecipeCRD {
+ return makeRecipe({
+ metadata: {
+ name: 'test-recipe',
+ namespace: 'sandbox-recipes',
+ uid: 'uid-123',
+ generation: 7,
+ },
+ spec: {
+ ...(workflow ? {} : { contextRef: 'context1' }),
+ workloads: [
+ {
+ id: 'calculator',
+ type: 'deployment',
+ image: 'clerum/mock-stdio-mcp-server:test',
+ port: 3000,
+ transport: { type: 'stdio' },
+ egressBindings: [{ dns: 'current.example.com', port: 443 }],
+ },
+ ],
+ ...(workflow ? { steps: [{ id: 'step', run: snippetRun() }] } : {}),
+ },
+ status: { phase: 'approved' },
+ })
+ }
+ function hccReadiness() {
+ const healthyRead = mockCustomApi.getNamespacedCustomObject.getMockImplementation()!
+ const state = { ready: false }
+ mockCustomApi.getNamespacedCustomObject.mockImplementation(async request => {
+ const response = await healthyRead(request)
+ if (request.plural === 'mcpservers') {
+ response.metadata.generation = 7
+ response.status.conditions = [
+ {
+ type: 'ExternalEgressReady',
+ status: state.ready ? 'True' : 'False',
+ observedGeneration: 7,
+ reason: state.ready ? 'Reconciled' : 'ExternalEgressReconcileFailed',
+ message: 'Policy observation pending',
+ },
+ ]
+ }
+ return response
+ })
+ return state
+ }
+ function watcherFor(rec: WorkflowRecipeReconciler, recipe?: WorkflowRecipeCRD) {
+ const watcher: any = Object.create(WorkflowRecipeWatcher.prototype)
+ Object.assign(watcher, {
+ recipes: new Map(recipe ? [[recipe.metadata.name, recipe]] : []),
+ transientRetries: new Map(),
+ stopped: false,
+ traceReporter: null,
+ dbRunProcessor: null,
+ reconciler: rec,
+ eventQueue: { enqueue: vi.fn((_key: string, work: () => Promise<void>) => work()) },
+ config: loadConfig(),
+ })
+ return watcher
+ }
+ function stopWatcher(watcher: any) {
+ watcher.stopped = true
+ for (const { timer } of watcher.transientRetries.values()) clearTimeout(timer)
+ watcher.transientRetries.clear()
+ }
+ function patchedStatus(previous?: WorkflowRecipeCRD['status']) {
+ return {
+ ...previous,
+ ...mockCustomApi.patchNamespacedCustomObjectStatus.mock.calls.at(-1)![0].body.status,
+ }
+ }
+
+ it('retries persisted stdio-only admission with backoff after restart and cannot become active from ready pods', async () => {
+ const recipe = transportRecipe()
+ const hcc = hccReadiness()
+ const watcher = watcherFor(reconciler)
+ let restarted: any
+ vi.useFakeTimers()
+ try {
+ const firstRun = watcher.handleRecipeEvent('ADDED', recipe)
+ await vi.advanceTimersByTimeAsync(31_000)
+ await firstRun
+ expect(watcher.transientRetries.get(recipe.metadata.name)?.attempts).toBe(1)
+ const pendingRecipe = { ...recipe, status: patchedStatus() }
+ expect(pendingRecipe.status.phase).toBe('degraded')
+ expect(pendingRecipe.status.conditions).toContainEqual(
+ expect.objectContaining({ type: conditionType, status: 'False' })
+ )
+ await watcher.handleRecipeEvent('MODIFIED', pendingRecipe)
+ await watcher.refreshWorkloadStatusesForSteadyRecipes()
+ const observedStatus = patchedStatus(pendingRecipe.status)
+ expect(observedStatus.phase).toBe('degraded')
+ expect(observedStatus.conditions).toContainEqual(
+ expect.objectContaining({ type: conditionType, status: 'False' })
+ )
+ stopWatcher(watcher)
+
+ // A new process has no previous retry timer; only persisted condition state remains.
+ const restartRecipe = { ...recipe, status: observedStatus }
+ const rec = new WorkflowRecipeReconciler(new k8s.KubeConfig())
+ restarted = watcherFor(rec, restartRecipe)
+ const run = vi.spyOn(rec, 'reconcile')
+ const restartPass = restarted.refreshRuntimeCredentialsForInProgressRecipes()
+ await vi.advanceTimersByTimeAsync(31_000)
+ await restartPass
+ expect(run).toHaveBeenCalledTimes(1)
+ expect(restarted.transientRetries.get(recipe.metadata.name)?.attempts).toBe(1)
+ await restarted.handleRecipeEvent('MODIFIED', { ...recipe, status: patchedStatus() })
+ await vi.advanceTimersByTimeAsync(35_000)
+ expect(run).toHaveBeenCalledTimes(2)
+ expect(restarted.transientRetries.get(recipe.metadata.name)?.attempts).toBe(2)
+ expect(patchedStatus().phase).toBe('degraded')
+ hcc.ready = true
+ await vi.advanceTimersByTimeAsync(10_000)
+ expect(run).toHaveBeenCalledTimes(3)
+ expect(patchedStatus().phase).toBe('active')
+ expect(
+ patchedStatus().conditions?.some(
+ (condition: { type: string }) => condition.type === conditionType
+ )
+ ).not.toBe(true)
+ expect(restarted.transientRetries.size).toBe(0)
+ expect(recipe.metadata.generation).toBe(7)
+ } finally {
+ stopWatcher(watcher)
+ if (restarted) stopWatcher(restarted)
+ vi.useRealTimers()
+ }
+ })
+
+ it('persists workflow admission pending and only starts the workflow after HCC heals', async () => {
+ const recipe = transportRecipe(true)
+ const hcc = hccReadiness()
+ const workflow = vi
+ .fn()
+ .mockResolvedValue({ phase: 'active', workflowPhase: 'running', message: 'Workflow ready' })
+ ;(reconciler as any).workflowReconciler = {
+ reconcile: workflow,
+ validateWorkflowSpec: () => undefined,
+ }
+ vi.useFakeTimers()
+ try {
+ const pending = reconciler.reconcile(recipe)
+ await vi.advanceTimersByTimeAsync(31_000)
+ const first = await pending
+ expect(first.phase, first.message).toBe('degraded')
+ expect(first.requeueAfterMs).toBe(TRANSIENT_REQUEUE_BASE_MS)
+ expect(workflow).not.toHaveBeenCalled()
+ await reconciler.patchStatus(recipe, first)
+ const persisted = { ...recipe, status: patchedStatus() }
+ expect(persisted.status.conditions).toContainEqual(
+ expect.objectContaining({ type: conditionType, status: 'False' })
+ )
+ expect(persisted.status.workflowExecution?.phase).not.toBe('failed')
+ hcc.ready = true
+ const result = await reconciler.reconcile(persisted)
+ expect(result.phase).toBe('active')
+ expect(workflow).toHaveBeenCalledTimes(1)
+ await reconciler.patchStatus(persisted, result)
+ expect(
+ patchedStatus().conditions?.some(
+ (condition: { type: string }) => condition.type === conditionType
+ )
+ ).not.toBe(true)
+ } finally {
+ vi.useRealTimers()
+ }
+ })
+
+ it('preserves pending admission through an earlier validation failure and clears it after transport removal', async () => {
+ const recipe = transportRecipe()
+ recipe.status = { phase: 'degraded', conditions: [pendingCondition] }
+ recipe.spec.workloads![0].egressBindings![0].port = 0
+ const failed = await reconciler.reconcile(recipe)
+ expect(failed.phase).toBe('failed')
+ await reconciler.patchStatus(recipe, failed)
+ expect(patchedStatus(recipe.status).conditions).toContainEqual(pendingCondition)
+ recipe.spec.workloads![0].egressBindings = []
+ delete recipe.spec.workloads![0].transport
+ const removed = await reconciler.reconcile(recipe)
+ expect(removed.phase).toBe('active')
+ await reconciler.patchStatus(recipe, removed)
+ expect(
+ patchedStatus(recipe.status).conditions?.some(
+ (condition: { type: string }) => condition.type === conditionType
+ )
+ ).not.toBe(true)
+ })
+
+ it('preserves ordinary Pod-only downgrade and recovery when no admission is pending', async () => {
+ const recipe = makeRecipe({ status: { phase: 'degraded' } })
+ mockAppsApi.readNamespacedDeployment.mockResolvedValueOnce({
+ spec: { replicas: 1 },
+ status: { readyReplicas: 0, updatedReplicas: 0, availableReplicas: 0 },
+ })
+ expect((await reconciler.observeCurrentWorkloadStatus(recipe)).phase).toBe('degraded')
+ expect((await reconciler.observeCurrentWorkloadStatus(recipe)).phase).toBe('active')
+ })
+ })
+
+ describe('#567 regression: policy contraction', () => {
+ function policyStore(policies: k8s.V1NetworkPolicy[]) {
+ const live = new Map<string, k8s.V1NetworkPolicy>()
+ let version = 0
+ const persist = (body: k8s.V1NetworkPolicy) => {
+ const policy = structuredClone(body)
+ const name = policy.metadata!.name!
+ policy.metadata = {
+ ...policy.metadata,
+ uid: `uid-${name}`,
+ resourceVersion: String(++version),
+ }
+ // The API's omitempty serialization omits empty top-level rule lists.
+ if (policy.spec?.egress?.length === 0) delete policy.spec.egress
+ if (policy.spec?.ingress?.length === 0) delete policy.spec.ingress
+ live.set(name, policy)
+ return structuredClone(policy)
+ }
+ for (const policy of policies) persist(policy)
+ mockNetworkingApi.readNamespacedNetworkPolicy.mockImplementation(async ({ name }) => {
+ if (!live.has(name)) throw { code: 404 }
+ return structuredClone(live.get(name))
+ })
+ mockNetworkingApi.createNamespacedNetworkPolicy.mockImplementation(async ({ body }) => {
+ if (live.has(body.metadata.name)) throw { code: 409 }
+ return persist(body)
+ })
+ mockNetworkingApi.replaceNamespacedNetworkPolicy.mockImplementation(
+ async ({ name, body }) => {
+ expect(body.metadata.resourceVersion).toBe(live.get(name)?.metadata?.resourceVersion)
+ return persist(body)
+ }
+ )
+ mockNetworkingApi.deleteNamespacedNetworkPolicy.mockImplementation(async ({ name, body }) => {
+ if (body?.preconditions) {
+ expect(body.preconditions).toEqual({
+ uid: live.get(name)?.metadata?.uid,
+ resourceVersion: live.get(name)?.metadata?.resourceVersion,
+ })
+ }
+ live.delete(name)
+ return {}
+ })
+ return live
+ }
+
+ function uiRecipe(ports = [8080, 9090]) {
+ return makeRecipe({
+ spec: {
+ workloads: [
+ { id: 'ui', type: 'deployment', image: 'ui:1', port: 8080 },
+ { id: 'backend', type: 'deployment', image: 'backend:1', port: 8080 },
+ ],
+ ui: {
+ workloadRef: 'ui',
+ port: 8080,
+ egress: {
+ internal: ports.map(port => ({ workloadRef: 'backend', port })),
+ external: [{ fqdn: 'current.example.com', port: 443 }],
+ },
+ },
+ },
+ })
+ }
+
+ it.each([[8080], [8080, 9090, 9443], [9090, 8080]])(
+ 'preserves the intersection of grouped UI ports during a DNS fault: %j',
+ async (...ports) => {
+ const rb = await import('./resourceBuilder')
+ const recipe = uiRecipe()
+ const name = 'ui-ingress-test-recipe-backend'
+ const live = policyStore([
+ rb.buildUiIngressNetworkPolicy(
+ recipe,
+ 'backend',
+ [8080, 9090],
+ 'sandbox-recipes',
+ 'sandbox-ui'
+ )!,
+ ])
+ recipe.spec.ui!.egress!.internal = ports.map(port => ({ workloadRef: 'backend', port }))
+ const rec = new WorkflowRecipeReconciler(new k8s.KubeConfig(), undefined, {
+ fqdnLookup: async () => {
+ throw new Error('resolver controller fault')
+ },
+ })
+ const result = await rec.reconcile({ ...recipe, status: { phase: 'active' } })
+ expect(result.phase).toBe('failed')
+ const retained = (live.get(name)?.spec?.ingress ?? [])
+ .flatMap(rule => rule.ports ?? [])
+ .map(port => port.port)
+ expect(retained.sort()).toEqual(ports.filter(port => port === 8080 || port === 9090).sort())
+ }
+ )
+
+ it.each([false, true])(
+ 'accepts empty UI readback before external replacement=%s',
+ async replacement => {
+ const rb = await import('./resourceBuilder')
+ const recipe = uiRecipe([])
+ const name = 'ui-egress-test-recipe'
+ const live = policyStore([
+ rb.buildUiEgressNetworkPolicy(
+ recipe,
+ 'sandbox-ui',
+ 'sandbox-recipes',
+ [
+ {
+ cidr: '93.184.216.20/32',
+ port: 443,
+ source: { kind: 'fqdn', fqdn: 'current.example.com' },
+ },
+ ],
+ egressStateAnnotations([
+ { fqdn: 'current.example.com', ip: '93.184.216.20', port: 443 },
+ ])
+ )!,
+ ])
+ recipe.spec.ui!.egress!.external = replacement
+ ? [{ fqdn: 'new.example.com', port: 443 }]
+ : []
+ const lookup = vi.fn(async () => ({
+ kind: 'ok' as const,
+ ipv4: ['93.184.216.21'],
+ ipv6: [],
+ ttlSeconds: 300,
+ }))
+ const rec = new WorkflowRecipeReconciler(new k8s.KubeConfig(), undefined, {
+ fqdnLookup: lookup,
+ })
+ const result = await rec.reconcile({ ...recipe, status: { phase: 'active' } })
+ expect(result.phase).toBe('active')
+ expect(live.has(name)).toBe(true)
+ expect(lookup).toHaveBeenCalledTimes(replacement ? 1 : 0)
+ expect(mockNetworkingApi.deleteNamespacedNetworkPolicy).not.toHaveBeenCalledWith(
+ expect.objectContaining({ name })
+ )
+ }
+ )
+
+ it.each([
+ { mode: 'broader-rules', phase: 'failed', deleted: true },
+ { mode: 'annotation-only', phase: 'failed', deleted: true },
+ { mode: 'replacement-uid', phase: 'degraded', deleted: false },
+ { mode: 'foreign-owner', phase: 'failed', deleted: false },
+ { mode: 'missing-uid', phase: 'failed', deleted: false },
+ { mode: 'wrong-namespace', phase: 'failed', deleted: false },
+ ] as const)('rejects unsafe readback: $mode', async ({ mode, phase, deleted }) => {
+ const rb = await import('./resourceBuilder')
+ const recipe = uiRecipe([])
+ const name = 'ui-egress-test-recipe'
+ const live = policyStore([
+ rb.buildUiEgressNetworkPolicy(
+ recipe,
+ 'sandbox-ui',
+ 'sandbox-recipes',
+ [
+ {
+ cidr: '93.184.216.20/32',
+ port: 443,
+ source: { kind: 'fqdn', fqdn: 'current.example.com' },
+ },
+ ],
+ egressStateAnnotations([{ fqdn: 'current.example.com', ip: '93.184.216.20', port: 443 }])
+ )!,
+ ])
+ recipe.spec.ui!.egress!.external = []
+ const replace = mockNetworkingApi.replaceNamespacedNetworkPolicy.getMockImplementation()!
+ mockNetworkingApi.replaceNamespacedNetworkPolicy.mockImplementation(async request => {
+ const result = await replace(request)
+ if (request.name === name) {
+ const observed = live.get(name)!
+ if (mode === 'broader-rules')
+ observed.spec!.egress = [
+ {
+ to: [{ ipBlock: { cidr: '8.8.8.8/32' } }],
+ ports: [{ port: 443, protocol: 'TCP' }],
+ },
+ ]
+ if (mode === 'annotation-only')
+ observed.metadata!.annotations = egressStateAnnotations([
+ { fqdn: 'removed.example.com', ip: '8.8.8.8', port: 443 },
+ ])
+ if (mode === 'replacement-uid') observed.metadata!.uid = 'uid-new-owned-object'
+ if (mode === 'missing-uid') delete observed.metadata!.uid
+ if (mode === 'wrong-namespace') observed.metadata!.namespace = 'another-namespace'
+ if (mode === 'foreign-owner')
+ observed.metadata!.labels!['clerum.io/managed-by'] = 'another-controller'
+ }
+ return result
+ })
+ const rec = new WorkflowRecipeReconciler(new k8s.KubeConfig(), undefined, {
+ fqdnLookup: async () => {
+ throw new Error('no external lookup is declared')
+ },
+ })
+ const result = await rec.reconcile({ ...recipe, status: { phase: 'active' } })
+ expect(result.phase).toBe(phase)
+ expect(live.has(name)).toBe(!deleted)
+ if (!deleted) expect(mockNetworkingApi.deleteNamespacedNetworkPolicy).not.toHaveBeenCalled()
+ expect(result.requeueAfterMs).toBe(
+ mode === 'replacement-uid' ? TRANSIENT_REQUEUE_BASE_MS : undefined
+ )
+ })
+
+ it.each(['ui', 'workload'] as const)(
+ 'replans additive %s conflicts and disappearance without latching a healthy recipe',
+ async lane => {
+ for (const failure of ['replace-conflict', 'read-disappearance', 'replace-disappearance']) {
+ const recipe =
+ lane === 'ui'
+ ? uiRecipe([])
+ : makeRecipe({
+ spec: {
+ workloads: [
+ {
+ id: 'worker',
+ type: 'deployment',
+ image: 'worker:1',
+ port: 8080,
+ egressBindings: [{ dns: 'current.example.com', port: 443 }],
+ },
+ ],
+ },
+ })
+ const name = lane === 'ui' ? 'ui-egress-test-recipe' : 'wl-egress-test-recipe-worker'
+ const live = policyStore([])
+ let address = '93.184.216.20'
+ const rec = new WorkflowRecipeReconciler(new k8s.KubeConfig(), undefined, {
+ fqdnLookup: async () => ({ kind: 'ok', ipv4: [address], ipv6: [], ttlSeconds: 300 }),
+ })
+ expect((await rec.reconcile({ ...recipe, status: { phase: 'active' } })).phase).toBe(
+ 'active'
+ )
+ const healthyReplace =
+ mockNetworkingApi.replaceNamespacedNetworkPolicy.getMockImplementation()!
+ const healthyCreate =
+ mockNetworkingApi.createNamespacedNetworkPolicy.getMockImplementation()!
+ address = '93.184.216.21'
+ if (failure === 'read-disappearance') {
++ // The workload lane is read-first: start absent so its create can
++ // race with another creator, whose winner then disappears.
++ if (lane === 'workload') live.delete(name)
+ mockNetworkingApi.createNamespacedNetworkPolicy.mockImplementation(async request => {
+ if (request.body.metadata.name === name) {
+ live.delete(name)
+ throw Object.assign(new Error('object existed before replacement read'), {
+ code: 409,
+ })
+ }
+ return healthyCreate(request)
+ })
+ } else {
+ mockNetworkingApi.replaceNamespacedNetworkPolicy.mockImplementation(async request => {
+ if (request.name === name && JSON.stringify(request.body.spec).includes(address)) {
+ if (failure === 'replace-disappearance') live.delete(name)
+ throw Object.assign(new Error('object snapshot changed'), {
+ code: failure === 'replace-conflict' ? 409 : 404,
+ })
+ }
+ return healthyReplace(request)
+ })
+ }
+ const first = await rec.reconcile({ ...recipe, status: { phase: 'active' } })
+ expect(first.phase, `${lane}/${failure}`).toBe('degraded')
+ expect(first.requeueAfterMs).toBe(TRANSIENT_REQUEUE_BASE_MS)
+ await rec.patchStatus(recipe, first)
+ const persisted =
+ mockCustomApi.patchNamespacedCustomObjectStatus.mock.calls.at(-1)![0].body.status
+ mockNetworkingApi.createNamespacedNetworkPolicy.mockImplementation(healthyCreate)
+ mockNetworkingApi.replaceNamespacedNetworkPolicy.mockImplementation(healthyReplace)
+ const recovered = await rec.reconcile({ ...recipe, status: persisted })
+ expect(recovered.phase).toBe('active')
+ expect(JSON.stringify(live.get(name)?.spec)).toContain(address)
+ }
+ }
+ )
+
+ it.each(['delete', 'replace'] as const)(
+ 'retries a fenced %s conflict after the status roundtrip',
+ async operation => {
+ const rb = await import('./resourceBuilder')
+ const recipe = makeRecipe({
+ spec: {
+ workloads: [
+ {
+ id: 'worker',
+ type: 'deployment',
+ image: 'worker:1',
+ port: 8080,
+ egressBindings: [
+ { dns: 'current.example.com', port: 443 },
+ { dns: 'removed.example.com', port: 443 },
+ ],
+ },
+ {
+ id: 'keeper',
+ type: 'deployment',
+ image: 'keeper:1',
+ port: 8080,
+ egressBindings: [{ dns: 'keeper.example.com', port: 443 }],
+ },
+ ],
+ },
+ })
+ const name = 'wl-egress-test-recipe-worker'
+ const entries = [
+ { fqdn: 'current.example.com', ip: '93.184.216.20', port: 443 },
+ { fqdn: 'removed.example.com', ip: '93.184.216.21', port: 443 },
+ ]
+ const live = policyStore([
+ rb.buildWorkloadEgressNetworkPolicy(
+ recipe.spec.workloads![0],
+ recipe,
+ 'sandbox-recipes',
+ entries.map(entry => ({
+ cidr: `${entry.ip}/32`,
+ port: entry.port,
+ source: { kind: 'fqdn', fqdn: entry.fqdn },
+ })),
+ egressStateAnnotations(entries)
+ )!,
+ ])
+ recipe.spec.workloads![0].egressBindings =
+ operation === 'delete' ? [] : [{ dns: 'current.example.com', port: 443 }]
+ const mutation =
+ operation === 'delete'
+ ? mockNetworkingApi.deleteNamespacedNetworkPolicy
+ : mockNetworkingApi.replaceNamespacedNetworkPolicy
+ const healthyMutation = mutation.getMockImplementation()!
+ mutation.mockImplementationOnce(async () => {
+ throw Object.assign(new Error('object modified'), { code: 409 })
+ })
+ const rec = new WorkflowRecipeReconciler(new k8s.KubeConfig(), undefined, {
+ fqdnLookup: async () => ({
+ kind: 'ok',
+ ipv4: ['93.184.216.20'],
+ ipv6: [],
+ ttlSeconds: 300,
+ }),
+ })
+ type Watcher = {
+ recipes: Map<string, WorkflowRecipeCRD>
+ transientRetries: Map<string, { timer: ReturnType<typeof setTimeout>; attempts: number }>
+ stopped: boolean
+ traceReporter: null
+ dbRunProcessor: null
+ reconciler: WorkflowRecipeReconciler
+ eventQueue: { enqueue: (key: string, task: () => Promise<void>) => Promise<void> }
+ handleRecipeEvent: (type: string, recipe: WorkflowRecipeCRD) => Promise<void>
+ }
+ const watcher = Object.create(WorkflowRecipeWatcher.prototype) as Watcher
+ Object.assign(watcher, {
+ recipes: new Map(),
+ transientRetries: new Map(),
+ stopped: false,
+ traceReporter: null,
+ dbRunProcessor: null,
+ reconciler: rec,
+ eventQueue: { enqueue: vi.fn((_key: string, task: () => Promise<void>) => task()) },
+ })
+ const reconcile = vi.spyOn(rec, 'reconcile')
+ const active = {
+ ...recipe,
+ metadata: { ...recipe.metadata, generation: 1 },
+ status: { phase: 'active' as const },
+ }
+ vi.useFakeTimers()
+ try {
+ await watcher.handleRecipeEvent('ADDED', active)
+ const first = await reconcile.mock.results[0].value
+ expect(first.phase).toBe('degraded')
+ expect(first.requeueAfterMs).toBe(TRANSIENT_REQUEUE_BASE_MS)
+ const patched =
+ mockCustomApi.patchNamespacedCustomObjectStatus.mock.calls.at(-1)![0].body.status
+ // Deliver the real status-only watch event without changing generation.
+ await watcher.handleRecipeEvent('MODIFIED', { ...active, status: patched })
+ expect(reconcile).toHaveBeenCalledTimes(1)
+ expect(watcher.transientRetries.get(recipe.metadata.name)?.attempts).toBe(1)
+ mutation.mockImplementationOnce(async () => {
+ throw Object.assign(new Error('object modified again'), { code: 409 })
+ })
+ await vi.advanceTimersByTimeAsync(TRANSIENT_REQUEUE_BASE_MS)
+ expect(reconcile).toHaveBeenCalledTimes(2)
+ expect(watcher.transientRetries.get(recipe.metadata.name)?.attempts).toBe(2)
+ mutation.mockImplementation(healthyMutation)
+ await vi.advanceTimersByTimeAsync(TRANSIENT_REQUEUE_BASE_MS * 2)
+ const next = await reconcile.mock.results[2].value
+ expect(next.phase).toBe('active')
+ expect(watcher.transientRetries.size).toBe(0)
+ expect(JSON.stringify(live.get(name)?.spec ?? {})).not.toContain('93.184.216.21')
+ expect(live.has(name)).toBe(operation === 'replace')
+ } finally {
+ for (const { timer } of watcher.transientRetries.values()) clearTimeout(timer)
+ vi.useRealTimers()
+ }
+ }
+ )
+ })
+
it('processes full pipeline successfully (3.11a)', async () => {
const result = await reconciler.reconcile(makeRecipe())
expect(result.phase).toBe('active')
@@@ -773,675 -1429,6 +1426,719 @@@
])
})
+ // ─── issue #575: live NetworkPolicy convergence ────────────────────────
+ describe('applyNetworkPolicy live convergence (issue #575)', () => {
+ const SPEC_HASH = 'clerum.io/spec-hash'
+ const NS = 'sandbox-recipes'
+ const clone = <T>(value: T): T => structuredClone(value)
+ type Priv = {
+ applyNetworkPolicy: (
+ policy: k8s.V1NetworkPolicy,
+ namespace: string,
- options: { family: NetworkPolicyFamily; existing?: k8s.V1NetworkPolicy | null }
++ options: {
++ family: NetworkPolicyFamily
++ existing?: k8s.V1NetworkPolicy | null
++ recipeName?: string
++ }
+ ) => Promise<void>
+ }
+ const apply = (
+ policy: k8s.V1NetworkPolicy,
+ family: NetworkPolicyFamily = 'workload-ingress',
+ existing?: k8s.V1NetworkPolicy | null
+ ) =>
+ (reconciler as unknown as Priv).applyNetworkPolicy(policy, NS, {
+ family,
+ ...(existing === undefined ? {} : { existing }),
+ })
+
+ const plainPolicy = (): k8s.V1NetworkPolicy => ({
+ apiVersion: 'networking.k8s.io/v1',
+ kind: 'NetworkPolicy',
+ metadata: {
+ name: 'wl-ingress-r-w',
+ namespace: NS,
+ labels: { 'clerum.io/managed-by': 'workflow-recipes', 'clerum.io/recipe': 'r' },
+ },
+ spec: {
+ podSelector: { matchLabels: { 'clerum.io/workload': 'w' } },
+ policyTypes: ['Ingress'],
+ ingress: [{ ports: [{ port: 8080, protocol: 'TCP' }] }],
+ },
+ })
+
+ const intdepPolicy = (): k8s.V1NetworkPolicy => ({
+ ...plainPolicy(),
+ metadata: {
+ name: 'wr-intdep-egress-r-w',
+ namespace: NS,
+ labels: {
+ 'clerum.io/managed-by': 'workflow-recipes',
+ 'clerum.io/recipe': 'r',
+ 'clerum.io/policy-type': 'internal-dependency',
+ },
+ },
+ })
+
+ const gatewayPolicy = (ownerUid = 'uid-current'): k8s.V1NetworkPolicy => ({
+ ...plainPolicy(),
+ metadata: {
+ name: 'allow-webhook-proxy-ingress-r',
+ namespace: NS,
+ labels: {
+ 'clerum.io/managed-by': 'workflow-recipes',
+ 'clerum.io/recipe-namespace': NS,
+ 'clerum.io/recipe-name': 'r',
+ 'clerum.io/webhook-gateway': 'true',
+ },
+ ownerReferences: [
+ {
+ apiVersion: 'clerum.io/v1alpha1',
+ kind: 'WorkflowRecipe',
+ name: 'r',
+ uid: ownerUid,
+ controller: true,
+ blockOwnerDeletion: true,
+ },
+ ],
+ },
+ })
+
+ const livePolicy = (
+ desired: k8s.V1NetworkPolicy,
+ metadata: Partial<k8s.V1ObjectMeta> = {}
+ ): k8s.V1NetworkPolicy => ({
+ ...clone(desired),
+ metadata: {
+ ...clone(desired.metadata ?? {}),
+ resourceVersion: '9',
+ ...metadata,
+ },
+ })
+
+ const families: NetworkPolicyFamily[] = [
+ 'ui-ingress',
+ 'workload-ingress',
+ 'workload-egress',
+ 'internal-dependency',
+ 'oauth-broker-egress',
+ 'webhook-gateway',
+ ]
+ const policyForFamily = (family: NetworkPolicyFamily) =>
+ family === 'webhook-gateway'
+ ? gatewayPolicy()
+ : family === 'internal-dependency'
+ ? intdepPolicy()
+ : plainPolicy()
+
++ it.each(['ui-ingress', 'workload-ingress', 'workload-egress'] as const)(
++ 'preserves the dev recipe-ownership boundary on the supplied %s snapshot',
++ async family => {
++ const desired = plainPolicy()
++ const live = livePolicy(desired)
++ live.metadata!.labels!['clerum.io/recipe'] = 'another-recipe'
++ mockNetworkingApi.readNamespacedNetworkPolicy.mockResolvedValue(livePolicy(desired))
++ await expect(
++ (reconciler as unknown as Priv).applyNetworkPolicy(desired, NS, {
++ family,
++ recipeName: 'r',
++ existing: live,
++ })
++ ).rejects.toMatchObject({ name: 'NetworkPolicyOwnershipConflictError' })
++ expect(mockNetworkingApi.readNamespacedNetworkPolicy).not.toHaveBeenCalled()
++ expect(mockNetworkingApi.createNamespacedNetworkPolicy).not.toHaveBeenCalled()
++ expect(mockNetworkingApi.replaceNamespacedNetworkPolicy).not.toHaveBeenCalled()
++ }
++ )
++
++ it.each(['ui-ingress', 'workload-ingress', 'workload-egress'] as const)(
++ 'keeps owned %s read-only until real drift and replaces the validated version',
++ async family => {
++ const desired = plainPolicy()
++ const live = livePolicy(desired)
++ const options = { family, recipeName: 'r', existing: live }
++ await (reconciler as unknown as Priv).applyNetworkPolicy(desired, NS, options)
++ expect(mockNetworkingApi.createNamespacedNetworkPolicy).not.toHaveBeenCalled()
++ expect(mockNetworkingApi.replaceNamespacedNetworkPolicy).not.toHaveBeenCalled()
++ desired.spec!.podSelector = { matchLabels: { updated: 'selector' } }
++ await (reconciler as unknown as Priv).applyNetworkPolicy(desired, NS, options)
++ expect(mockNetworkingApi.readNamespacedNetworkPolicy).not.toHaveBeenCalled()
++ expect(mockNetworkingApi.replaceNamespacedNetworkPolicy).toHaveBeenCalledTimes(1)
++ expect(
++ mockNetworkingApi.replaceNamespacedNetworkPolicy.mock.calls[0][0].body.metadata
++ .resourceVersion
++ ).toBe('9')
++ }
++ )
++
+ it.each(families)(
+ 'preserves external metadata while repairing real %s spec drift',
+ async family => {
+ const desired = policyForFamily(family)
+ desired.metadata!.labels!['clerum.io/test-contract'] = 'current'
+ desired.metadata!.annotations = {
+ 'clerum.io/test-contract': 'current',
+ [SPEC_HASH]: 'obsolete',
+ }
+ const live = livePolicy(desired)
+ live.metadata!.labels!['clerum.io/test-contract'] = 'stale'
+ live.metadata!.labels!['admission.example/tier'] = 'audited'
+ live.metadata!.annotations = {
+ 'admission.example/audit': 'keep',
+ 'clerum.io/test-contract': 'stale',
+ [SPEC_HASH]: 'legacy',
+ 'clerum.io/egress-fqdn-state': 'stale-state',
+ 'clerum.io/egress-fqdn-targets': 'stale-targets',
+ 'clerum.io/egress-fqdn-resolved-at': 'stale-time',
+ }
+ live.metadata!.finalizers = ['admission.example/cleanup']
+ live.spec!.podSelector = {}
+ const desiredBefore = clone(desired)
+ const liveBefore = clone(live)
+ mockNetworkingApi.readNamespacedNetworkPolicy.mockResolvedValue(live)
+
+ await apply(desired, family)
+
+ expect(mockNetworkingApi.createNamespacedNetworkPolicy).toHaveBeenCalledTimes(0)
+ expect(mockNetworkingApi.replaceNamespacedNetworkPolicy).toHaveBeenCalledTimes(1)
+ const body = mockNetworkingApi.replaceNamespacedNetworkPolicy.mock.calls[0][0].body
+ expect(body.metadata.labels).toEqual({
+ ...desired.metadata!.labels,
+ 'admission.example/tier': 'audited',
+ })
+ expect(body.metadata.annotations).toEqual({
+ 'admission.example/audit': 'keep',
+ 'clerum.io/test-contract': 'current',
+ })
+ expect(body.metadata.finalizers).toEqual(['admission.example/cleanup'])
+ expect(body.metadata.ownerReferences).toEqual(desired.metadata!.ownerReferences)
+ expect(body.metadata.resourceVersion).toBe('9')
+ expect(body.spec).toEqual(desired.spec)
+ expect(mockNetworkingApi.readNamespacedNetworkPolicy).toHaveBeenCalledTimes(1)
+ expect(desired).toEqual(desiredBefore)
+ expect(live).toEqual(liveBefore)
+ }
+ )
+
+ it.each(families)(
+ 'performs zero create/replace calls for equivalent %s with external metadata',
+ async family => {
+ const desired = policyForFamily(family)
+ const live = livePolicy(desired)
+ live.metadata!.labels!['admission.example/tier'] = 'audited'
+ live.metadata!.annotations = { 'admission.example/audit': 'keep', [SPEC_HASH]: 'legacy' }
+ live.metadata!.finalizers = ['admission.example/cleanup']
+ mockNetworkingApi.readNamespacedNetworkPolicy.mockResolvedValue(live)
+
+ await apply(desired, family)
+
+ expect(mockNetworkingApi.readNamespacedNetworkPolicy).toHaveBeenCalledTimes(1)
+ expect(mockNetworkingApi.createNamespacedNetworkPolicy).toHaveBeenCalledTimes(0)
+ expect(mockNetworkingApi.replaceNamespacedNetworkPolicy).toHaveBeenCalledTimes(0)
+ }
+ )
+
+ it('vetoes an additional non-controller owner instead of removing it on gateway replace', async () => {
+ const desired = gatewayPolicy()
+ const live = livePolicy(desired)
+ live.metadata!.ownerReferences!.push({
+ apiVersion: 'v1',
+ kind: 'ConfigMap',
+ name: 'foreign',
+ uid: 'foreign',
+ })
+ live.spec!.podSelector = {}
+ mockNetworkingApi.readNamespacedNetworkPolicy.mockResolvedValue(live)
+
+ await expect(apply(desired, 'webhook-gateway')).rejects.toMatchObject({
+ name: 'NetworkPolicyOwnershipConflictError',
+ })
+ expect(mockNetworkingApi.createNamespacedNetworkPolicy).toHaveBeenCalledTimes(0)
+ expect(mockNetworkingApi.replaceNamespacedNetworkPolicy).toHaveBeenCalledTimes(0)
+ })
+
+ it('removes retired workload-egress state once without churning external metadata', async () => {
+ const desired = plainPolicy()
+ desired.metadata!.name = 'wl-egress-r-w'
+ const live = livePolicy(desired, {
+ annotations: {
+ 'admission.example/audit': 'keep',
+ 'clerum.io/egress-fqdn-state': 'retired',
+ 'clerum.io/egress-fqdn-targets': 'retired',
+ 'clerum.io/egress-fqdn-resolved-at': 'retired',
+ },
+ })
+ const priv = reconciler as unknown as {
+ egressWriteNeeded: (existing: k8s.V1NetworkPolicy, desired: k8s.V1NetworkPolicy) => boolean
+ }
+ expect(priv.egressWriteNeeded(live, desired)).toBe(true)
+ await apply(desired, 'workload-egress', live)
+ expect(mockNetworkingApi.replaceNamespacedNetworkPolicy).toHaveBeenCalledTimes(1)
+ const updated = mockNetworkingApi.replaceNamespacedNetworkPolicy.mock.calls[0][0].body
+ expect(updated.metadata.annotations).toEqual({ 'admission.example/audit': 'keep' })
+ mockNetworkingApi.replaceNamespacedNetworkPolicy.mockClear()
+
+ expect(priv.egressWriteNeeded(updated, desired)).toBe(false)
+ await apply(desired, 'workload-egress', updated)
+ expect(mockNetworkingApi.createNamespacedNetworkPolicy).toHaveBeenCalledTimes(0)
+ expect(mockNetworkingApi.replaceNamespacedNetworkPolicy).toHaveBeenCalledTimes(0)
+ })
+
+ function captureNetworkPolicyLogs(): {
+ entries: Array<Record<string, unknown>>
+ restore: () => void
+ } {
+ const previousLevel = process.env.LOG_LEVEL
+ process.env.LOG_LEVEL = 'info'
+ const entries: Array<Record<string, unknown>> = []
+ const spy = vi.spyOn(process.stdout, 'write').mockImplementation(((chunk: unknown) => {
+ const text = typeof chunk === 'string' ? chunk : Buffer.from(chunk as Uint8Array).toString()
+ for (const line of text.split('\n')) {
+ if (!line.startsWith('{')) continue
+ const parsed = JSON.parse(line) as Record<string, unknown>
+ if (String(parsed.msg).startsWith('network policy')) entries.push(parsed)
+ }
+ return true
+ }) as unknown as typeof process.stdout.write)
+ return {
+ entries,
+ restore: () => {
+ spy.mockRestore()
+ if (previousLevel === undefined) delete process.env.LOG_LEVEL
+ else process.env.LOG_LEVEL = previousLevel
+ },
+ }
+ }
+
+ it('T1 skips a live-equivalent policy and emits a structured liveness witness', async () => {
+ const desired = plainPolicy()
+ const live = livePolicy(desired, { annotations: { [SPEC_HASH]: 'legacy-seal' } })
+ live.spec!.ingress![0].ports![0].protocol = undefined
+ mockNetworkingApi.readNamespacedNetworkPolicy.mockResolvedValue(live)
+ const captured = captureNetworkPolicyLogs()
+ try {
+ await apply(desired)
+ expect(mockNetworkingApi.readNamespacedNetworkPolicy).toHaveBeenCalledTimes(1)
+ expect(mockNetworkingApi.createNamespacedNetworkPolicy).not.toHaveBeenCalled()
+ expect(mockNetworkingApi.replaceNamespacedNetworkPolicy).not.toHaveBeenCalled()
+ expect(captured.entries).toContainEqual(
+ expect.objectContaining({
+ level: 'info',
+ msg: 'network policy unchanged; skipping update',
+ policy: 'wl-ingress-r-w',
+ namespace: NS,
+ family: 'workload-ingress',
+ })
+ )
+ } finally {
+ captured.restore()
+ }
+ })
+
+ it('T2 repairs live spec drift even when the legacy spec-hash is preserved', async () => {
+ const desired = plainPolicy()
+ const live = livePolicy(desired, { annotations: { [SPEC_HASH]: 'legacy-seal' } })
+ live.spec!.podSelector = {}
+ live.spec!.ingress = [{}]
+ mockNetworkingApi.readNamespacedNetworkPolicy.mockResolvedValue(live)
+
+ await apply(desired)
+
+ expect(mockNetworkingApi.replaceNamespacedNetworkPolicy).toHaveBeenCalledTimes(1)
+ const body = (
+ mockNetworkingApi.replaceNamespacedNetworkPolicy.mock.calls[0][0] as {
+ body: k8s.V1NetworkPolicy
+ }
+ ).body
+ expect(body.metadata?.resourceVersion).toBe('9')
+ expect(body.metadata?.annotations?.[SPEC_HASH]).toBeUndefined()
+ expect(body.spec).toEqual(desired.spec)
+ expect(mockNetworkingApi.readNamespacedNetworkPolicy).toHaveBeenCalledTimes(1)
+ })
+
+ it('does not let a supplied cluster-local wl-egress snapshot veto detected drift', async () => {
+ const desired = plainPolicy()
+ desired.metadata!.name = 'wl-egress-r-w'
+ desired.spec = {
+ podSelector: { matchLabels: { 'clerum.io/workload': 'w' } },
+ policyTypes: ['Egress'],
+ egress: [
+ {
+ to: [{ podSelector: { matchLabels: { 'clerum.io/workload': 'db' } } }],
+ ports: [{ port: 5432, protocol: 'TCP' }],
+ },
+ ],
+ }
+ const live = livePolicy(desired, { annotations: { [SPEC_HASH]: 'legacy-seal' } })
+ live.spec!.podSelector = {}
+
+ await apply(desired, 'workload-egress', live)
+
+ expect(mockNetworkingApi.readNamespacedNetworkPolicy).not.toHaveBeenCalled()
+ expect(mockNetworkingApi.replaceNamespacedNetworkPolicy).toHaveBeenCalledTimes(1)
+ expect(
+ (
+ mockNetworkingApi.replaceNamespacedNetworkPolicy.mock.calls[0][0] as {
+ body: k8s.V1NetworkPolicy
+ }
+ ).body.spec?.podSelector
+ ).toEqual(desired.spec.podSelector)
+ })
+
+ it('routes workload-egress label drift through the live apply prefilter', () => {
+ const desired = plainPolicy()
+ desired.metadata!.name = 'wl-egress-r-w'
+ desired.spec = { podSelector: {}, policyTypes: ['Egress'], egress: [] }
+ const live = livePolicy(desired)
+ live.metadata!.labels = { ...live.metadata!.labels, 'clerum.io/recipe': 'wrong' }
+
+ const writeNeeded = (
+ reconciler as unknown as {
+ egressWriteNeeded: (
+ existing: k8s.V1NetworkPolicy | null,
+ wanted: k8s.V1NetworkPolicy
+ ) => boolean
+ }
+ ).egressWriteNeeded(live, desired)
+
+ expect(writeNeeded).toBe(true)
+
+ live.metadata!.labels = desired.metadata!.labels
+ live.metadata!.ownerReferences = [
+ {
+ apiVersion: 'apps/v1',
+ kind: 'Deployment',
+ name: 'foreign',
+ uid: 'foreign-uid',
+ controller: true,
+ },
+ ]
+ expect(
+ (
+ reconciler as unknown as {
+ egressWriteNeeded: (
+ existing: k8s.V1NetworkPolicy | null,
+ wanted: k8s.V1NetworkPolicy
+ ) => boolean
+ }
+ ).egressWriteNeeded(live, desired)
+ ).toBe(true)
+ })
+
+ it('T3 creates on 404 and the next equivalent reconcile is read-only', async () => {
+ const desired = plainPolicy()
+ mockNetworkingApi.readNamespacedNetworkPolicy.mockRejectedValueOnce({ code: 404 })
+
+ await apply(desired)
+ expect(mockNetworkingApi.createNamespacedNetworkPolicy).toHaveBeenCalledTimes(1)
+ const created = clone(
+ (
+ mockNetworkingApi.createNamespacedNetworkPolicy.mock.calls[0][0] as {
+ body: k8s.V1NetworkPolicy
+ }
+ ).body
+ )
+ expect(created.metadata?.annotations?.[SPEC_HASH]).toBeUndefined()
+
+ mockNetworkingApi.createNamespacedNetworkPolicy.mockClear()
+ mockNetworkingApi.readNamespacedNetworkPolicy.mockResolvedValue(livePolicy(created))
+
+ await apply(desired)
+ expect(mockNetworkingApi.createNamespacedNetworkPolicy).not.toHaveBeenCalled()
+ expect(mockNetworkingApi.replaceNamespacedNetworkPolicy).not.toHaveBeenCalled()
+ })
+
+ it('T4 propagates a foreign intdep ownership veto after exactly one read', async () => {
+ const desired = intdepPolicy()
+ mockNetworkingApi.readNamespacedNetworkPolicy
+ .mockResolvedValueOnce({
+ metadata: {
+ name: 'wr-intdep-egress-r-w',
+ resourceVersion: '9',
+ labels: { 'clerum.io/managed-by': 'hcc' },
+ },
+ })
+ .mockRejectedValueOnce({ code: 503 })
+
+ const rejection = await apply(desired, 'internal-dependency').then(
+ () => new Error('expected the ownership assertion to reject'),
+ (error: unknown) => error as Error
+ )
+ expect(rejection.name).toBe('NetworkPolicyOwnershipConflictError')
+ expect(rejection.message).toMatch(/Refusing to replace NetworkPolicy "wr-intdep-egress-r-w"/)
+ expect(mockNetworkingApi.createNamespacedNetworkPolicy).not.toHaveBeenCalled()
+ expect(mockNetworkingApi.replaceNamespacedNetworkPolicy).not.toHaveBeenCalled()
+ expect(mockNetworkingApi.readNamespacedNetworkPolicy).toHaveBeenCalledTimes(1)
+ })
+
+ it('T4b lets an intdep policy WRC owns converge read-only', async () => {
+ const desired = intdepPolicy()
+ mockNetworkingApi.readNamespacedNetworkPolicy.mockResolvedValue(livePolicy(desired))
+
+ await expect(apply(desired, 'internal-dependency')).resolves.toBeUndefined()
+
+ expect(mockNetworkingApi.readNamespacedNetworkPolicy).toHaveBeenCalledTimes(1)
+ expect(mockNetworkingApi.replaceNamespacedNetworkPolicy).not.toHaveBeenCalled()
+ })
+
+ it('T5 retries an HTTP read failure, replaces with the recovered RV, and logs safely', async () => {
+ const desired = plainPolicy()
+ mockNetworkingApi.readNamespacedNetworkPolicy
+ .mockRejectedValueOnce({ code: 500, authorization: 'Bearer must-not-appear' })
+ .mockResolvedValueOnce(livePolicy(desired))
+ const captured = captureNetworkPolicyLogs()
+ try {
+ await apply(desired)
+
+ expect(mockNetworkingApi.replaceNamespacedNetworkPolicy).toHaveBeenCalledTimes(1)
+ expect(mockNetworkingApi.readNamespacedNetworkPolicy).toHaveBeenCalledTimes(2)
+ expect(
+ (
+ mockNetworkingApi.replaceNamespacedNetworkPolicy.mock.calls[0][0] as {
+ body: k8s.V1NetworkPolicy
+ }
+ ).body.metadata?.resourceVersion
+ ).toBe('9')
+ const warning = captured.entries.find(
+ entry => entry.msg === 'network policy read unavailable; retrying once'
+ )
+ expect(warning).toEqual(
+ expect.objectContaining({
+ level: 'warn',
+ errorName: 'NetworkPolicyInfraError',
+ errorCode: 500,
+ retryable: true,
+ })
+ )
+ expect(JSON.stringify(warning)).not.toContain('must-not-appear')
+ } finally {
+ captured.restore()
+ }
+ })
+
+ it('T5b retries a transport read failure once and replaces', async () => {
+ const desired = plainPolicy()
+ mockNetworkingApi.readNamespacedNetworkPolicy
+ .mockRejectedValueOnce({ code: 'ECONNRESET' })
+ .mockResolvedValueOnce(livePolicy(desired))
+
+ await apply(desired)
+
+ expect(mockNetworkingApi.replaceNamespacedNetworkPolicy).toHaveBeenCalledTimes(1)
+ expect(mockNetworkingApi.readNamespacedNetworkPolicy).toHaveBeenCalledTimes(2)
+ })
+
+ it('T5c retries a codeless read failure once and replaces', async () => {
+ const desired = plainPolicy()
+ mockNetworkingApi.readNamespacedNetworkPolicy
+ .mockRejectedValueOnce(new Error('transient read timeout'))
+ .mockResolvedValueOnce(livePolicy(desired))
+
+ await apply(desired)
+
+ expect(mockNetworkingApi.replaceNamespacedNetworkPolicy).toHaveBeenCalledTimes(1)
+ expect(mockNetworkingApi.readNamespacedNetworkPolicy).toHaveBeenCalledTimes(2)
+ })
+
+ it('repairs the gateway owner UID and refuses a foreign controller owner', async () => {
+ const desired = gatewayPolicy('uid-current')
+ mockNetworkingApi.readNamespacedNetworkPolicy.mockResolvedValue(
+ livePolicy(gatewayPolicy('uid-old'), { annotations: { [SPEC_HASH]: 'legacy-seal' } })
+ )
+
+ await apply(desired, 'webhook-gateway')
+
+ expect(mockNetworkingApi.replaceNamespacedNetworkPolicy).toHaveBeenCalledTimes(1)
+ const body = (
+ mockNetworkingApi.replaceNamespacedNetworkPolicy.mock.calls[0][0] as {
+ body: k8s.V1NetworkPolicy
+ }
+ ).body
+ expect(body.metadata?.ownerReferences?.[0].uid).toBe('uid-current')
+
+ mockNetworkingApi.replaceNamespacedNetworkPolicy.mockClear()
+ const foreign = livePolicy(desired)
+ foreign.metadata!.ownerReferences = [
+ {
+ apiVersion: 'apps/v1',
+ kind: 'Deployment',
+ name: 'foreign',
+ uid: 'foreign-uid',
+ controller: true,
+ },
+ ]
+ mockNetworkingApi.readNamespacedNetworkPolicy.mockResolvedValue(foreign)
+
+ await expect(apply(desired, 'webhook-gateway')).rejects.toMatchObject({
+ name: 'NetworkPolicyOwnershipConflictError',
+ })
+ expect(mockNetworkingApi.replaceNamespacedNetworkPolicy).not.toHaveBeenCalled()
+ })
+
+ it('handles a read/create race by re-reading instead of blind replacing', async () => {
+ const desired = plainPolicy()
+ mockNetworkingApi.readNamespacedNetworkPolicy
+ .mockRejectedValueOnce({ code: 404 })
+ .mockResolvedValueOnce(livePolicy(desired))
+ mockNetworkingApi.createNamespacedNetworkPolicy.mockRejectedValueOnce({ code: 409 })
+
+ await apply(desired)
+
+ expect(mockNetworkingApi.readNamespacedNetworkPolicy).toHaveBeenCalledTimes(2)
+ expect(mockNetworkingApi.createNamespacedNetworkPolicy).toHaveBeenCalledTimes(1)
+ expect(mockNetworkingApi.replaceNamespacedNetworkPolicy).not.toHaveBeenCalled()
+ })
+
+ it('creates after a transient first read is followed by a confirmed 404', async () => {
+ const desired = plainPolicy()
+ mockNetworkingApi.readNamespacedNetworkPolicy
+ .mockRejectedValueOnce({ code: 500 })
+ .mockRejectedValueOnce({ code: 404 })
+
+ await apply(desired)
+
+ expect(mockNetworkingApi.readNamespacedNetworkPolicy).toHaveBeenCalledTimes(2)
+ expect(mockNetworkingApi.createNamespacedNetworkPolicy).toHaveBeenCalledTimes(1)
+ expect(mockNetworkingApi.replaceNamespacedNetworkPolicy).not.toHaveBeenCalled()
+ })
+
+ it('uses a supplied absent workload-egress snapshot without issuing another read', async () => {
+ const desired = plainPolicy()
+ desired.metadata!.name = 'wl-egress-r-w'
+
+ await apply(desired, 'workload-egress', null)
+
+ expect(mockNetworkingApi.readNamespacedNetworkPolicy).not.toHaveBeenCalled()
+ expect(mockNetworkingApi.createNamespacedNetworkPolicy).toHaveBeenCalledTimes(1)
+ })
+
+ it('retries when a create-conflict winner is absent on the bounded re-read', async () => {
+ const desired = plainPolicy()
+ mockNetworkingApi.readNamespacedNetworkPolicy.mockRejectedValue({ code: 404 })
+ mockNetworkingApi.createNamespacedNetworkPolicy.mockRejectedValueOnce({ code: 409 })
+
+ await expect(apply(desired)).rejects.toMatchObject({
+ name: 'RetryableReconcileError',
+ cause: { code: 409 },
+ })
+ expect(mockNetworkingApi.readNamespacedNetworkPolicy).toHaveBeenCalledTimes(2)
+ expect(mockNetworkingApi.createNamespacedNetworkPolicy).toHaveBeenCalledTimes(1)
+ expect(mockNetworkingApi.replaceNamespacedNetworkPolicy).not.toHaveBeenCalled()
+ })
+
+ it('retries instead of blind replacing a drifted live policy without resourceVersion', async () => {
+ const desired = plainPolicy()
+ const live = livePolicy(desired)
+ delete live.metadata!.resourceVersion
+ live.spec!.podSelector = {}
+ mockNetworkingApi.readNamespacedNetworkPolicy.mockResolvedValue(live)
+
+ await expect(apply(desired)).rejects.toMatchObject({
+ name: 'RetryableReconcileError',
+ message: expect.stringMatching(/has no resourceVersion/),
+ })
+ expect(mockNetworkingApi.replaceNamespacedNetworkPolicy).not.toHaveBeenCalled()
+ })
+
+ it.each([404, 409])('classifies replace race status %s as retryable', async code => {
+ const desired = plainPolicy()
+ const live = livePolicy(desired)
+ live.spec!.podSelector = {}
+ mockNetworkingApi.readNamespacedNetworkPolicy.mockResolvedValue(live)
+ mockNetworkingApi.replaceNamespacedNetworkPolicy.mockRejectedValueOnce({ code })
+
+ await expect(apply(desired)).rejects.toMatchObject({
+ name: 'RetryableReconcileError',
+ cause: { code },
+ })
+ })
+
+ it.each([500, 'ETIMEDOUT'])('preserves replace infrastructure failure %s', async code => {
+ const desired = plainPolicy()
+ const live = livePolicy(desired)
+ live.spec!.podSelector = {}
+ mockNetworkingApi.readNamespacedNetworkPolicy.mockResolvedValue(live)
+ const apiError = Object.assign(new Error(`replace failed: ${code}`), { code })
+ mockNetworkingApi.replaceNamespacedNetworkPolicy.mockRejectedValueOnce(apiError)
+
+ await expect(apply(desired)).rejects.toMatchObject({
+ name: 'NetworkPolicyInfraError',
+ code,
+ cause: apiError,
+ })
+ })
+
+ it('propagates a permanent replace rejection without reclassifying it as transient', async () => {
+ const desired = plainPolicy()
+ const live = livePolicy(desired)
+ live.spec!.podSelector = {}
+ mockNetworkingApi.readNamespacedNetworkPolicy.mockResolvedValue(live)
+ const permanent = { code: 422, reason: 'Invalid' }
+ mockNetworkingApi.replaceNamespacedNetworkPolicy.mockRejectedValueOnce(permanent)
+
+ await expect(apply(desired)).rejects.toBe(permanent)
+ })
+
+ it('propagates a permanent create rejection without attempting a replace', async () => {
+ const desired = plainPolicy()
+ const permanent = { code: 422, reason: 'Invalid' }
+ mockNetworkingApi.readNamespacedNetworkPolicy.mockRejectedValueOnce({ code: 404 })
+ mockNetworkingApi.createNamespacedNetworkPolicy.mockRejectedValueOnce(permanent)
+
+ await expect(apply(desired)).rejects.toBe(permanent)
+ expect(mockNetworkingApi.replaceNamespacedNetworkPolicy).not.toHaveBeenCalled()
+ })
+
+ it('propagates a permanent second-read rejection after the bounded retry', async () => {
+ const desired = plainPolicy()
+ const permanent = { code: 422, reason: 'Invalid' }
+ mockNetworkingApi.readNamespacedNetworkPolicy
+ .mockRejectedValueOnce({ code: 500 })
+ .mockRejectedValueOnce(permanent)
+
+ await expect(apply(desired)).rejects.toBe(permanent)
+ expect(mockNetworkingApi.readNamespacedNetworkPolicy).toHaveBeenCalledTimes(2)
+ expect(mockNetworkingApi.createNamespacedNetworkPolicy).not.toHaveBeenCalled()
+ expect(mockNetworkingApi.replaceNamespacedNetworkPolicy).not.toHaveBeenCalled()
+ })
+
+ it('refuses to create a gateway policy without a complete desired owner', async () => {
+ const desired = gatewayPolicy('')
+
+ await expect(apply(desired, 'webhook-gateway')).rejects.toMatchObject({
+ name: 'NetworkPolicyOwnershipConflictError',
+ })
+ expect(mockNetworkingApi.readNamespacedNetworkPolicy).not.toHaveBeenCalled()
+ expect(mockNetworkingApi.createNamespacedNetworkPolicy).not.toHaveBeenCalled()
+ })
+
+ it('refuses a desired gateway policy with multiple controller owners before I/O', async () => {
+ const desired = gatewayPolicy()
+ desired.metadata!.ownerReferences!.push({
+ apiVersion: 'apps/v1',
+ kind: 'Deployment',
+ name: 'second-controller',
+ uid: 'second-controller-uid',
+ controller: true,
+ })
+
+ await expect(apply(desired, 'webhook-gateway')).rejects.toMatchObject({
+ name: 'NetworkPolicyOwnershipConflictError',
+ })
+ expect(mockNetworkingApi.readNamespacedNetworkPolicy).not.toHaveBeenCalled()
+ expect(mockNetworkingApi.createNamespacedNetworkPolicy).not.toHaveBeenCalled()
+ })
+
+ it('refuses a desired policy without metadata.name before I/O', async () => {
+ const desired = plainPolicy()
+ delete desired.metadata!.name
+
+ await expect(apply(desired)).rejects.toThrow(/without metadata\.name/)
+ expect(mockNetworkingApi.readNamespacedNetworkPolicy).not.toHaveBeenCalled()
+ expect(mockNetworkingApi.createNamespacedNetworkPolicy).not.toHaveBeenCalled()
+ })
+ })
+
describe('idempotent apply (stops generation churn)', () => {
const SPEC_HASH = 'clerum.io/spec-hash'
const clone = <T>(value: T): T => JSON.parse(JSON.stringify(value)) as T
@@@ -3184,136 -3019,6 +3881,140 @@@
)
})
+ describe('workload egress pre-read infrastructure failures', () => {
- it.each([
- { workflow: true, code: 503 },
- { workflow: true, code: 'ECONNRESET' },
- { workflow: false, code: 503 },
- { workflow: false, code: 'ECONNRESET' },
- { workflow: true, code: 403 },
- { workflow: false, code: 403 },
- ])('classifies $code after successful DNS (workflow=$workflow)', async ({ workflow, code }) => {
++ it.each(
++ (['before DNS', 'after DNS'] as const).flatMap(stage =>
++ [
++ { workflow: true, code: 503 },
++ { workflow: true, code: 'ECONNRESET' },
++ { workflow: false, code: 503 },
++ { workflow: false, code: 'ECONNRESET' },
++ { workflow: true, code: 403 },
++ { workflow: false, code: 403 },
++ ].map(scenario => ({ ...scenario, stage }))
++ )
++ )('classifies $code $stage (workflow=$workflow)', async ({ workflow, code, stage }) => {
+ const recipe = makeRecipe({
+ spec: {
+ workloads: [
+ {
+ id: 'worker',
+ type: 'deployment',
+ image: 'worker:latest',
+ port: 8080,
+ egressBindings: [{ dns: 'api.example.com', port: 443 }],
+ },
+ ],
+ ...(workflow ? { steps: [{ id: 'run', run: snippetRun() }] } : {}),
+ },
+ ...(!workflow
+ ? {
+ status: {
+ phase: 'active' as const,
+ message: 'Healthy before read failure',
+ workloads: [],
+ },
+ }
+ : {}),
+ })
+ const lookup = vi.fn(async () => ({
+ kind: 'ok' as const,
+ ipv4: ['93.184.216.10'],
+ ipv6: [],
+ ttlSeconds: 300,
+ }))
+ const rec = new WorkflowRecipeReconciler(new k8s.KubeConfig(), undefined, {
+ fqdnLookup: lookup,
+ })
+ const workflowReconcile = vi.fn()
+ ;(rec as unknown as { workflowReconciler: unknown }).workflowReconciler = {
+ reconcile: workflowReconcile,
+ validateWorkflowSpec: () => undefined,
+ }
+ const failure = { code }
+ mockNetworkingApi.readNamespacedNetworkPolicy.mockImplementation(async ({ name }) => {
+ if (name === 'wl-egress-test-recipe-worker') {
- expect(lookup).toHaveBeenCalled()
++ if (stage === 'after DNS' && lookup.mock.calls.length === 0) throw { code: 404 }
+ throw failure
+ }
+ throw { code: 404 }
+ })
+
+ const result = await rec.reconcile(recipe)
+
- expect(lookup).toHaveBeenCalledTimes(1)
++ expect(lookup).toHaveBeenCalledTimes(stage === 'after DNS' ? 1 : 0)
+ if (code === 403) {
+ expect(result.phase).toBe('failed')
+ expect(result.requeueAfterMs).toBeUndefined()
+ } else {
+ expect(result.phase).toBe(workflow ? 'degraded' : 'active')
+ expect(result.requeueAfterMs).toBe(TRANSIENT_REQUEUE_BASE_MS)
+ if (!workflow) expect(result.skipStatusPatch).toBe(true)
+ }
+ expect(workflowReconcile).not.toHaveBeenCalled()
+ expect(mockNetworkingApi.createNamespacedNetworkPolicy).toHaveBeenCalledTimes(0)
+ expect(mockNetworkingApi.replaceNamespacedNetworkPolicy).toHaveBeenCalledTimes(0)
+ })
+ })
+
+ describe('reconciliation completion receipt', () => {
+ it.each(['success', 'reconcile-throw', 'projection-throw'] as const)(
+ 'emits completion only after the full pipeline returns (%s)',
+ async outcome => {
+ const recipe = makeRecipe()
+ recipe.metadata.generation = 7
+ const previousLevel = process.env.LOG_LEVEL
+ process.env.LOG_LEVEL = 'info'
+ const receipts: Array<Record<string, unknown>> = []
+ const write = vi.spyOn(process.stdout, 'write').mockImplementation(((chunk: unknown) => {
+ for (const line of String(chunk).split('\n')) {
+ if (!line.startsWith('{')) continue
+ const record = JSON.parse(line)
+ if (record.msg === 'recipe reconciliation completed') receipts.push(record)
+ }
+ return true
+ }) as typeof process.stdout.write)
+ const priv = reconciler as unknown as {
+ reconcileInternal: () => Promise<unknown>
+ projectPluginWorkloadSdk: () => unknown
+ }
+ const failure = new Error('pipeline failure')
+ const injected =
+ outcome === 'reconcile-throw'
+ ? vi.spyOn(priv, 'reconcileInternal').mockRejectedValue(failure)
+ : outcome === 'projection-throw'
+ ? vi.spyOn(priv, 'projectPluginWorkloadSdk').mockImplementation(() => {
+ throw failure
+ })
+ : undefined
+ try {
+ if (outcome === 'success') {
+ const result = await reconciler.reconcile(recipe)
+ expect(result.phase).toBe('active')
+ expect(receipts).toEqual([
+ expect.objectContaining({
+ recipe: 'test-recipe',
+ namespace: 'sandbox-recipes',
+ uid: 'uid-123',
+ generation: 7,
+ phase: 'active',
+ requeueAfterMs: 0,
+ }),
+ ])
+ } else {
+ await expect(reconciler.reconcile(recipe)).rejects.toBe(failure)
+ expect(receipts).toEqual([])
+ }
+ } finally {
+ injected?.mockRestore()
+ write.mockRestore()
+ if (previousLevel === undefined) delete process.env.LOG_LEVEL
+ else process.env.LOG_LEVEL = previousLevel
+ }
+ }
+ )
+ })
+
it('fails non-transport external egress in required mode when cluster enforcement is not confirmed', async () => {
const recipe = makeRecipe({
spec: {
@@@ -10160,76 -10890,6 +11888,76 @@@
expect(allFalse).toBe(true)
})
+ it('T6 (#575) converges the 3 gateway NetworkPolicies through live reads', async () => {
+ mockNetworkingApi.createNamespacedNetworkPolicy.mockClear()
+ mockNetworkingApi.readNamespacedNetworkPolicy.mockRejectedValue({ code: 404 })
+ await reconciler.reconcile(recipeWithWebhook())
+
+ const sealed = new Map<string, unknown>()
+ for (const call of mockNetworkingApi.createNamespacedNetworkPolicy.mock.calls) {
+ const body = (call[0] as { body: k8s.V1NetworkPolicy }).body
+ sealed.set(body.metadata!.name!, JSON.parse(JSON.stringify(body)))
+ }
+ // Which three, not just how many: cardinality alone stays green if a refactor
+ // stops emitting gateway policies and three arrive from another family.
+ expect([...sealed.keys()].sort()).toEqual([
+ 'allow-gateway-egress-to-handler-wf-wh-recipe',
+ 'allow-gateway-ingress-to-handler-wf-wh-recipe',
+ 'allow-webhook-proxy-ingress-wf-wh-recipe',
+ ])
+
+ mockNetworkingApi.readNamespacedNetworkPolicy.mockImplementation(
+ ({ name }: { name: string }) => {
+ const previous = sealed.get(name) as k8s.V1NetworkPolicy | undefined
- if (!previous) return Promise.resolve({ metadata: { name, resourceVersion: '1' } })
++ if (!previous) return Promise.reject({ code: 404 })
+ return Promise.resolve({
+ ...previous,
+ metadata: { ...previous.metadata, resourceVersion: '9' },
+ })
+ }
+ )
+ mockNetworkingApi.replaceNamespacedNetworkPolicy.mockClear()
+ mockNetworkingApi.createNamespacedNetworkPolicy.mockClear()
+ mockNetworkingApi.readNamespacedNetworkPolicy.mockClear()
+ const previousLevel = process.env.LOG_LEVEL
+ process.env.LOG_LEVEL = 'info'
+ const logEntries: Array<Record<string, unknown>> = []
+ const logSpy = vi.spyOn(process.stdout, 'write').mockImplementation(((chunk: unknown) => {
+ const text = typeof chunk === 'string' ? chunk : Buffer.from(chunk as Uint8Array).toString()
+ for (const line of text.split('\n')) {
+ if (!line.startsWith('{')) continue
+ const parsed = JSON.parse(line) as Record<string, unknown>
+ if (parsed.msg === 'network policy unchanged; skipping update') logEntries.push(parsed)
+ }
+ return true
+ }) as unknown as typeof process.stdout.write)
+
+ try {
+ await reconciler.reconcile(recipeWithWebhook())
+
+ for (const name of sealed.keys()) {
+ expect(mockNetworkingApi.replaceNamespacedNetworkPolicy).not.toHaveBeenCalledWith(
+ expect.objectContaining({ name })
+ )
+ expect(mockNetworkingApi.readNamespacedNetworkPolicy).toHaveBeenCalledWith(
+ expect.objectContaining({ name })
+ )
+ expect(logEntries).toContainEqual(
+ expect.objectContaining({
+ policy: name,
+ family: 'webhook-gateway',
+ msg: 'network policy unchanged; skipping update',
+ })
+ )
+ }
+ expect(mockNetworkingApi.createNamespacedNetworkPolicy).not.toHaveBeenCalled()
+ } finally {
+ logSpy.mockRestore()
+ if (previousLevel === undefined) delete process.env.LOG_LEVEL
+ else process.env.LOG_LEVEL = previousLevel
+ }
+ })
+
it('fail-closed when secretRef is missing: no gateway resources, condition set, phase degraded', async () => {
mockAppsApi.createNamespacedDeployment.mockClear()
mockCoreApi.createNamespacedService.mockClear()
diff --cc workflow-recipes/src/reconciler/workflowRecipeReconciler.ts
index e364bb611,f6f7e0dbd..d0e2d294e
--- a/workflow-recipes/src/reconciler/workflowRecipeReconciler.ts
+++ b/workflow-recipes/src/reconciler/workflowRecipeReconciler.ts
@@@ -89,12 -99,11 +99,17 @@@ import
waitForExternalEgressReady,
waitForNetworkReady,
} from './mcpDelegation'
+ import {
+ effectivePolicyTypes,
+ intersectNetworkPolicyRules,
+ networkPolicySpecSignature,
+ } from './networkPolicyContraction'
+import {
+ type NetworkPolicyFamily,
+ buildNetworkPolicyReplacement,
+ decideNetworkPolicyConvergence,
+ networkPolicyMetadataMatchesDesired,
+} from './networkPolicyConvergence'
import { issueOAuthBrokerToken } from './oauthBrokerTokenIssuerClient'
import {
PLUGIN_WORKLOAD_SDK_CONDITION_TYPE,
@@@ -478,26 -519,25 +525,46 @@@ class NetworkPolicyOwnershipConflictErr
}
}
+class NetworkPolicyInfraError extends Error {
+ readonly code?: string | number
+
+ constructor(message: string, cause: unknown, code?: string | number) {
+ super(message, { cause })
+ this.name = 'NetworkPolicyInfraError'
+ this.code = code
+ }
+}
+
+interface NetworkPolicyApplyOptions {
+ family: NetworkPolicyFamily
+ existing?: k8s.V1NetworkPolicy | null
++ recipeName?: string
+}
+
+interface NetworkPolicyReadResult {
+ existing: k8s.V1NetworkPolicy | null
+ recoveredFromReadFailure: boolean
+}
+
+ /** A fenced mutation lost its snapshot; only a fresh reconciliation may retry. */
+ class NetworkPolicyReplanRequiredError extends RetryableReconcileError {
+ override name = 'NetworkPolicyReplanRequiredError'
+ }
+
+ class TransportNetworkReadinessPendingError extends RetryableReconcileError {
+ override name = 'TransportNetworkReadinessPendingError'
+ }
+
+ function throwNetworkPolicyMutationError(error: unknown, name: string, namespace: string): never {
+ if (getErrorCode(error) === 409 || getErrorCode(error) === 404) {
+ throw new NetworkPolicyReplanRequiredError(
+ `NetworkPolicy "${name}" in ${namespace} changed during a fenced mutation; a fresh reconciliation is required`,
+ { cause: error }
+ )
+ }
+ throw error
+ }
+
class InternalDependencyReconcileError extends Error {
constructor(
message: string,
@@@ -1958,7 -2114,13 +2159,10 @@@ export class WorkflowRecipeReconciler
workloadStatuses: [],
internalDependencyConditions,
workloadConditions,
+ requeueAfterMs: TRANSIENT_REQUEUE_BASE_MS,
+ ...(error instanceof TransportNetworkReadinessPendingError
+ ? { transportNetworkConditions: pendingTransportNetworkConditions(recipe, message) }
+ : {}),
- ...(error instanceof NetworkPolicyReplanRequiredError ||
- error instanceof TransportNetworkReadinessPendingError
- ? { requeueAfterMs: TRANSIENT_REQUEUE_BASE_MS }
- : {}),
}
}
return {
@@@ -2728,7 -2910,18 +2952,15 @@@
phase: 'degraded',
message: error.message,
workloadStatuses: [],
+ requeueAfterMs: TRANSIENT_REQUEUE_BASE_MS,
+ ...(error instanceof TransportNetworkReadinessPendingError
+ ? {
+ transportNetworkConditions: pendingTransportNetworkConditions(
+ recipe,
+ error.message
+ ),
+ }
+ : {}),
- ...(error instanceof NetworkPolicyReplanRequiredError ||
- error instanceof TransportNetworkReadinessPendingError
- ? { requeueAfterMs: TRANSIENT_REQUEUE_BASE_MS }
- : {}),
}
}
// (2) A raw retryable infra error (connect ETIMEDOUT/ECONNRESET, 5xx,
@@@ -2746,12 -2939,10 +2978,13 @@@
// decision and hiding the permanent misconfiguration.
const isEgressResolutionError =
error instanceof Error && error.message.includes('egress resolution failed')
- if (!isEgressResolutionError && isRetryableInfraError(error)) {
+ if (
+ !isEgressResolutionError &&
+ (error instanceof NetworkPolicyInfraError || isRetryableInfraError(error))
+ ) {
- console.warn(
- `[WR-Reconciler] Transient infra error reconciling "${name}" — keeping phase "${currentPhase}", will retry: ${String(error)}`
+ createLogger('wrc', recipe.metadata.name).warn(
+ 'Transient infrastructure error during reconciliation; keeping phase and retrying',
+ { name, currentPhase, err: error }
)
return {
phase: currentPhase as RecipePhase,
@@@ -4317,7 -4661,7 +4691,10 @@@
await this.deleteUiIngressIfExists(recipe.metadata.name, w.id, ns)
continue
}
- await this.applyNetworkPolicy(policy, ns, { family: 'ui-ingress' })
- await this.applyOwnedRecipeNetworkPolicy(policy, ns, recipe.metadata.name)
++ await this.applyNetworkPolicy(policy, ns, {
++ family: 'ui-ingress',
++ recipeName: recipe.metadata.name,
++ })
}
}
@@@ -4693,23 -5134,20 +5170,24 @@@
!wlEgressStateChanged
) {
createLogger('wrc', recipe.metadata.name).info(
- 'NetworkPolicy egress set unchanged; no-op',
- { wlPolicyName, wlNs }
- )
- } else {
- await this.applyOwnedRecipeNetworkPolicy(policy, wlNs, recipe.metadata.name)
- // #299: the policy has landed — record the set it actually enforces.
- this.logResolvedEgressSet(
- recipe.metadata.name,
- wlPolicyName,
- externalDeclared.map(e => e.fqdn),
- effectiveExternal,
- wlEgressStateChanged
+ 'network policy egress set unchanged; skipping live apply',
+ { policy: wlPolicyName, namespace: wlNs, family: 'workload-egress' }
)
+ continue
}
+ await this.applyNetworkPolicy(policy, wlNs, {
+ family: 'workload-egress',
++ recipeName: recipe.metadata.name,
+ existing: existingWlPolicy,
+ })
+ // #299: the policy has landed — record the set it actually enforces.
+ this.logResolvedEgressSet(
+ recipe.metadata.name,
+ wlPolicyName,
+ externalDeclared.map(e => e.fqdn),
+ effectiveExternal,
+ wlEgressStateChanged
+ )
}
// Now ingress side: for each target workload that any sibling pointed
@@@ -4728,7 -5166,7 +5206,10 @@@
await this.deleteWorkloadIngressIfExists(recipe.metadata.name, w.id, wlNs)
continue
}
- await this.applyNetworkPolicy(policy, wlNs, { family: 'workload-ingress' })
- await this.applyOwnedRecipeNetworkPolicy(policy, wlNs, recipe.metadata.name)
++ await this.applyNetworkPolicy(policy, wlNs, {
++ family: 'workload-ingress',
++ recipeName: recipe.metadata.name,
++ })
}
}
@@@ -4929,203 -5532,29 +5584,206 @@@
}
}
- private async applyNetworkPolicy(policy: k8s.V1NetworkPolicy, namespace: string): Promise<void> {
- const policyName = policy.metadata!.name!
- await this.createOrReplace(
- () =>
- this.networkingApi.createNamespacedNetworkPolicy({
- namespace,
- body: policy,
- }),
- async () => {
- const existing = await this.networkingApi.readNamespacedNetworkPolicy({
- name: policyName,
- namespace,
- })
- this.assertInternalDependencyPolicyOwnership(policy, existing, namespace)
- policy.metadata!.resourceVersion = existing.metadata?.resourceVersion
- return this.networkingApi.replaceNamespacedNetworkPolicy({
- name: policyName,
- namespace,
- body: policy,
- })
- },
- `NetworkPolicy "${policyName}" in ${namespace}`
+ /**
+ * Reconcile one WRC-authored NetworkPolicy from the live object, not from the
+ * legacy spec-hash seal. NetworkPolicy is an enforcement resource: a stored
+ * annotation can describe what WRC wrote previously while the live spec or
+ * lifecycle ownership has since drifted. One validated live snapshot therefore
+ * owns both the no-op decision and the resourceVersion used by a replace.
+ */
+ private async applyNetworkPolicy(
+ policy: k8s.V1NetworkPolicy,
+ namespace: string,
+ options: NetworkPolicyApplyOptions
+ ): Promise<void> {
+ const desired = structuredClone(policy)
+ const policyName = desired.metadata?.name
+ if (!policyName) throw new Error('Cannot apply a NetworkPolicy without metadata.name')
+ desired.metadata = desired.metadata ?? {}
+ desired.metadata.namespace = desired.metadata.namespace ?? namespace
+ this.assertDesiredNetworkPolicyOwner(desired, options.family, namespace)
+
+ const recipeName =
+ desired.metadata?.labels?.['clerum.io/recipe'] ??
+ desired.metadata?.labels?.['clerum.io/recipe-name'] ??
+ policyName
+ const log = createLogger('wrc', recipeName)
+ const fields = { policy: policyName, namespace, family: options.family }
+ const hasSuppliedExisting = Object.prototype.hasOwnProperty.call(options, 'existing')
+ let readResult: NetworkPolicyReadResult = hasSuppliedExisting
+ ? { existing: options.existing ?? null, recoveredFromReadFailure: false }
+ : await this.readNetworkPolicyForApply(policyName, namespace, log, fields)
+ let existing = readResult.existing
+
+ if (!existing) {
+ try {
+ await this.networkingApi.createNamespacedNetworkPolicy({ namespace, body: desired })
+ log.info('network policy created', fields)
+ return
+ } catch (error: unknown) {
+ if (getErrorCode(error) !== 409) {
+ log.error('network policy create failed', {
+ ...fields,
+ ...this.safeInfrastructureErrorFields(error),
+ })
+ if (isRetryableInfraError(error)) {
+ const safe = this.safeInfrastructureErrorFields(error)
+ throw new NetworkPolicyInfraError(
+ `NetworkPolicy "${policyName}" in ${namespace} create temporarily unavailable (${safe.errorCode ?? safe.errorName})`,
+ error,
+ safe.errorCode
+ )
+ }
+ throw error
+ }
+ // A real create race is exceptional in the read-first path. Re-read once
+ // and evaluate the winner from scratch; never blind-replace after 409.
+ readResult = await this.readNetworkPolicyForApply(policyName, namespace, log, fields)
+ existing = readResult.existing
+ if (!existing) {
+ throw new RetryableReconcileError(
+ `NetworkPolicy "${policyName}" in ${namespace} still absent after create conflict`,
+ { cause: error }
+ )
+ }
+ }
+ }
+
+ // Deletion wins over ownership: a terminating object cannot be adopted or
+ // safely classified from partial labels. Wait for 404, then recreate from the
+ // current desired state.
+ if (existing.metadata?.deletionTimestamp) {
+ throw new RetryableReconcileError(
+ `NetworkPolicy "${policyName}" in ${namespace} is terminating; retrying after deletion`
+ )
+ }
+
+ // Domain invariants run outside the read catch. A confirmed veto propagates
+ // immediately and can never be replaced by a second transport error.
++ if (options.recipeName) {
++ this.assertRecipeNetworkPolicyOwnership(existing, options.recipeName, policyName, namespace)
++ }
+ this.assertInternalDependencyPolicyOwnership(desired, existing, namespace)
+ const decision = decideNetworkPolicyConvergence(options.family, desired, existing)
+ if (decision.action === 'conflict') {
+ throw new NetworkPolicyOwnershipConflictError(
+ `Refusing to replace NetworkPolicy "${policyName}" in ${namespace}: ${decision.reason}`
+ )
+ }
+ if (decision.action === 'retry') {
+ throw new RetryableReconcileError(
+ `NetworkPolicy "${policyName}" in ${namespace} is terminating; retrying after deletion`
+ )
+ }
+ if (decision.action === 'unchanged' && !readResult.recoveredFromReadFailure) {
+ log.info('network policy unchanged; skipping update', fields)
+ return
+ }
+
+ const resourceVersion = existing.metadata?.resourceVersion
+ if (!resourceVersion) {
+ throw new RetryableReconcileError(
+ `NetworkPolicy "${policyName}" in ${namespace} has no resourceVersion`
+ )
+ }
+ const replacement = buildNetworkPolicyReplacement(desired, existing)
+ try {
+ await this.networkingApi.replaceNamespacedNetworkPolicy({
+ name: policyName,
+ namespace,
+ body: replacement,
+ })
+ log.info('network policy replaced', {
+ ...fields,
+ reason: decision.action === 'unchanged' ? 'read-recovery' : decision.reason,
+ })
+ } catch (error: unknown) {
+ log.error('network policy replace failed', {
+ ...fields,
+ ...this.safeInfrastructureErrorFields(error),
+ })
+ const errorCode = getErrorCode(error)
+ if (errorCode === 404 || errorCode === 409) {
+ throw new RetryableReconcileError(
+ `NetworkPolicy "${policyName}" in ${namespace} ${errorCode === 404 ? 'disappeared' : 'changed'} during replace`,
+ { cause: error }
+ )
+ }
+ if (isRetryableInfraError(error)) {
+ const safe = this.safeInfrastructureErrorFields(error)
+ throw new NetworkPolicyInfraError(
+ `NetworkPolicy "${policyName}" in ${namespace} replace temporarily unavailable (${safe.errorCode ?? safe.errorName})`,
+ error,
+ safe.errorCode
+ )
+ }
+ throw error
+ }
+ }
+
+ private assertDesiredNetworkPolicyOwner(
+ desired: k8s.V1NetworkPolicy,
+ family: NetworkPolicyFamily,
+ namespace: string
+ ): void {
+ const controllerOwners = (desired.metadata?.ownerReferences ?? []).filter(
+ owner => owner.controller === true
)
+ const invalidControllerOwner =
+ controllerOwners.length > 1 || controllerOwners.some(owner => !owner.uid)
+ const gatewayOwnerMissing = family === 'webhook-gateway' && controllerOwners.length !== 1
+ if (invalidControllerOwner || gatewayOwnerMissing) {
+ throw new NetworkPolicyOwnershipConflictError(
+ `Refusing to apply NetworkPolicy "${desired.metadata?.name ?? '<unknown>'}" in ${namespace}: desired controller owner is incomplete`
+ )
+ }
+ }
+
+ private async readNetworkPolicyForApply(
+ name: string,
+ namespace: string,
+ log: ReturnType<typeof createLogger>,
+ fields: { policy: string; namespace: string; family: NetworkPolicyFamily }
+ ): Promise<NetworkPolicyReadResult> {
+ for (let attempt = 1; attempt <= 2; attempt += 1) {
+ try {
+ return {
+ existing: await this.readNetworkPolicyOrNull(name, namespace),
+ recoveredFromReadFailure: attempt > 1,
+ }
+ } catch (error: unknown) {
+ if (attempt === 2) {
+ throw error
+ }
+ log.warn('network policy read unavailable; retrying once', {
+ ...fields,
+ ...this.safeInfrastructureErrorFields(error),
+ })
+ }
+ }
+ return { existing: null, recoveredFromReadFailure: false }
+ }
+
+ private safeInfrastructureErrorFields(error: unknown): {
+ errorName: string
+ errorCode?: string | number
+ retryable: boolean
+ } {
+ const candidateCode = (error as { code?: unknown } | null)?.code
+ const candidateName = error instanceof Error ? error.name : 'UnknownError'
+ const errorCode =
+ typeof candidateCode === 'number'
+ ? candidateCode
+ : typeof candidateCode === 'string' && /^[A-Z0-9_]{1,64}$/.test(candidateCode)
+ ? candidateCode
+ : getErrorCode(error)
+ return {
+ errorName: /^[A-Za-z][A-Za-z0-9_.-]{0,63}$/.test(candidateName)
+ ? candidateName
+ : 'UnknownError',
+ ...(errorCode === undefined ? {} : { errorCode }),
+ retryable: isRetryableInfraError(error),
+ }
}
private coordinatorGfsNetworkPolicy(recipe: WorkflowRecipeCRD): k8s.V1NetworkPolicy {