Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
fix(workflow-recipes): retry the legacy delete before the short-circuits with a backoff (R4-L4)
Alfredo Lopez2dc56e1062cc
diff --git a/workflow-recipes/package-lock.json b/workflow-recipes/package-lock.jsonindex 7986c86dc..2812e742e 100644--- a/workflow-recipes/package-lock.json+++ b/workflow-recipes/package-lock.json@@ -1,12 +1,12 @@{"name": "workload-recipes-controller",- "version": "0.1.444",+ "version": "0.1.445","lockfileVersion": 3,"requires": true,"packages": {"": {"name": "workload-recipes-controller",- "version": "0.1.444",+ "version": "0.1.445","license": "SEE LICENSE IN LICENSE","dependencies": {"@aws-sdk/client-s3": "^3.1009.0",diff --git a/workflow-recipes/package.json b/workflow-recipes/package.jsonindex ffaed2bab..8a02d1241 100644--- a/workflow-recipes/package.json+++ b/workflow-recipes/package.json@@ -1,6 +1,6 @@{"name": "workload-recipes-controller",- "version": "0.1.444",+ "version": "0.1.445","description": "Workload Recipes Controller (WRC) - WorkloadRecipe lifecycle management and MCP server interface","main": "dist/main.js","engines": {diff --git a/workflow-recipes/src/reconciler/reconciler.test.ts b/workflow-recipes/src/reconciler/reconciler.test.tsindex 7b19eed95..eb8765ef8 100644--- a/workflow-recipes/src/reconciler/reconciler.test.ts+++ b/workflow-recipes/src/reconciler/reconciler.test.ts@@ -16653,6 +16653,43 @@ describe('WorkflowRecipeReconciler', () => {expect(legacyDeletes()).toBe(0)expect(result.requeueAfterMs).toBeUndefined()})++ it('R4-L4: keeps the requeue a short-circuit already asked for while DeletePending stays published', async () => {+ const inner = installRealInner()+ stubShortCircuits(inner)+ const retry = vi+ .spyOn(inner, 'retryRunLaneNetworkPolicies')+ .mockResolvedValue({ conflicts: [], retryPending: true })+ legacyDeleteAnswers({ code: 403, message: 'forbidden' })+ const conditions = networkPolicyMarkerConditions(+ {+ kind: 'reconcile',+ summary: { conflicts: [], retryPending: true, prune: 'converged', legacy: 'pending' },+ },+ [],+ '2026-09-23T09:00:00.000Z'+ )+ expect(ofType(conditions, MARKER_TYPE)).toMatchObject([{ reason: 'RetryPending' }])+ expect(ofType(conditions, LEGACY_TYPE)).toMatchObject([{ reason: 'DeletePending' }])+ const active = runScopedRecipe({+ phase: 'active',+ message: 'Workflow running',+ workflowExecution: { phase: 'running' },+ conditions,+ })++ const result = await reconciler.reconcile(active)++ // Witnesses: the apply retry ran and the legacy DELETE was sent.+ expect(retry).toHaveBeenCalledTimes(1)+ expect(legacyDeletes()).toBe(1)+ expect(ofType(active.status?.conditions, LEGACY_TYPE)).toMatchObject([+ { reason: 'DeletePending' },+ ])+ // The apply retry's progress requeue wins over the 60 s legacy window.+ expect(result.requeueAfterMs).toBe(WORKFLOW_PROGRESS_REQUEUE_BASE_MS)+ expect(result.requeueFixedInterval).toBe(false)+ })})// A reconcile() prune whose DELETE did not land publishesdiff --git a/workflow-recipes/src/reconciler/workflowRecipeReconciler.ts b/workflow-recipes/src/reconciler/workflowRecipeReconciler.tsindex 2bca25ed1..4636e391d 100644--- a/workflow-recipes/src/reconciler/workflowRecipeReconciler.ts+++ b/workflow-recipes/src/reconciler/workflowRecipeReconciler.ts@@ -69,6 +69,7 @@ import {WORKFLOW_OUTPUT_CONDITION_TYPES,WorkflowReconciler,WorkflowReconcilerDeps,+ hasLegacyNetworkPolicyDeletePendingMarker,hasNetworkPolicyRetryPendingMarker,networkPolicyConditionsChanged,networkPolicyMarkerConditions,@@ -1525,6 +1526,54 @@ export class WorkflowRecipeReconciler {return summary.retryPending ? WORKFLOW_PROGRESS_REQUEUE_BASE_MS : undefined}+ /**+ * R4-L4: the `legacy-retry` writer. Enters only while the published group+ * says the legacy mcp-servers internet policy is pending a delete. The+ * delete shares the reconcile() pass's per-recipe backoff, so inside the+ * window it sends no request. `removed` clears the legacy fact with a+ * conditions-only patch and carries the apply and prune facts; `pending`+ * writes nothing, and the `reconcile()` wrapper requeues for the rest of+ * the window.+ */+ private async retryPendingLegacyNetworkPolicyDelete(recipe: WorkflowRecipeCRD): Promise<void> {+ if (+ !this.workflowReconciler ||+ !hasLegacyNetworkPolicyDeletePendingMarker(recipe.status?.conditions)+ ) {+ return+ }+ const legacy = await this.workflowReconciler.retryLegacyMcpServersInternetEgressDelete(+ recipe.metadata.name,+ recipe.metadata.uid ?? ''+ )+ if (legacy === 'pending') return+ await this.publishNetworkPolicyMarkerWriter(recipe, { kind: 'legacy-retry', legacy })+ }++ /**+ * R4-L4: a pass that leaves `DeletePending` published and asks for no other+ * requeue is requeued at a fixed interval for the rest of the legacy+ * delete's backoff window, at least 1 s. The group the pass publishes wins+ * over the one it found; a pass that publishes no group keeps the found+ * one, including a legacy-retry patch made during the pass.+ */+ private requeueForPendingLegacyNetworkPolicyDelete(+ recipe: WorkflowRecipeCRD,+ result: ReconcileResult+ ): void {+ if (!this.workflowReconciler || result.requeueAfterMs !== undefined) return+ const conditions = result.networkPolicyOwnershipConditions ?? recipe.status?.conditions+ if (!hasLegacyNetworkPolicyDeletePendingMarker(conditions)) return+ result.requeueAfterMs = Math.max(+ 1_000,+ this.workflowReconciler.legacyMcpServersInternetEgressRetryDelayMs(+ recipe.metadata.name,+ recipe.metadata.uid ?? ''+ )+ )+ result.requeueFixedInterval = true+ }+/*** Settles the marker once the run is terminal. The run no longer needs its* policies, so a pending apply is dropped: there is nothing left to retry.@@ -1755,6 +1804,7 @@ export class WorkflowRecipeReconciler {// (e.g. observeCurrentWorkloadStatus) leave the projection undefined, which// both consumers treat as "no SDK opinion this pass".const result = await this.reconcileInternal(recipe)+ this.requeueForPendingLegacyNetworkPolicyDelete(recipe, result)result.pluginWorkloadSdkProjection = this.projectPluginWorkloadSdk(recipe, result)createLogger('wrc', recipe.metadata.name).info('recipe reconciliation completed', {recipe: recipe.metadata.name,@@ -2001,6 +2051,11 @@ export class WorkflowRecipeReconciler {await this.ensureOAuthBrokerTokenSecret(recipe)}+ // R4-L4: the short-circuits below never reach the inner reconcile(), so+ // a legacy mcp-servers internet policy pending a delete is retried here,+ // before all of them (the `legacy-retry` writer).+ await this.retryPendingLegacyNetworkPolicyDelete(recipe)+if (currentPhase === 'active' &&awaitsTriggeredRun &&diff --git a/workflow-recipes/src/workflow/workflowReconciler.ts b/workflow-recipes/src/workflow/workflowReconciler.tsindex 28c8ae5b1..caae3a1e5 100644--- a/workflow-recipes/src/workflow/workflowReconciler.ts+++ b/workflow-recipes/src/workflow/workflowReconciler.ts@@ -432,6 +432,36 @@ export function hasNetworkPolicyRetryPendingMarker(return networkPolicyMarkerFacts(conditions).apply === 'pending'}+/**+ * True while the published group says the legacy mcp-servers internet policy+ * is pending a delete (`WorkflowLegacyNetworkPolicyRemoved=False/DeletePending`).+ * Only this reason enters the `legacy-retry` writer; a reason this controller+ * did not write is never interpreted.+ */+export function hasLegacyNetworkPolicyDeletePendingMarker(+ conditions: StatusCondition[] | undefined+): boolean {+ const published = publishedMarker(+ conditions,+ LEGACY_NETWORK_POLICY_REMOVED_CONDITION_TYPE,+ isLegacyNetworkPolicyReason+ )+ return (+ published.kind === 'known' &&+ LEGACY_NETWORK_POLICY_REASONS[published.condition.reason as LegacyNetworkPolicyReason].fact ===+ 'pending'+ )+}++/**+ * Backoff between two DELETEs of the legacy mcp-servers internet policy for+ * one recipe after a DELETE that did not land: 60 s, doubling up to 1 h. A+ * transient 403 does not leave a false DeletePending for hours, a sustained+ * one costs one DELETE per hour per recipe, and a converged recipe costs none.+ */+export const LEGACY_NETWORK_POLICY_DELETE_BACKOFF_BASE_MS = 60_000+export const LEGACY_NETWORK_POLICY_DELETE_BACKOFF_MAX_MS = 3_600_000+/*** Reads the facts a published marker carries. This and* `buildNetworkPolicyConvergedCondition` are the only places that encode what@@ -554,6 +584,9 @@ function buildLegacyNetworkPolicyCondition(* summary). Observes the apply, the prune and the legacy delete.* - `apply-retry`: the mid-run retry. Observes the apply; carries the prune* and legacy facts.+ * - `legacy-retry`: the legacy delete the WRC retries before its+ * short-circuits while `DeletePending` is published. Observes the legacy+ * fact; carries the apply and prune facts and the ownership condition.* - `terminal`: the run ended, so a pending apply is dropped; carries the* prune and legacy facts and the ownership condition.* - `unmanaged`: the kill switch, the SDK-only lane without a runtime and@@ -563,6 +596,7 @@ function buildLegacyNetworkPolicyCondition(export type NetworkPolicyMarkerWriter =| { kind: 'reconcile'; summary: NetworkPolicyPassSummary }| { kind: 'apply-retry'; summary: WorkflowNetworkPolicyApplySummary }+ | { kind: 'legacy-retry'; legacy: NetworkPolicyLegacyFact }| { kind: 'terminal' }| { kind: 'unmanaged' }@@ -685,6 +719,19 @@ export function networkPolicyMarkerConditions(carriedLegacyMarker(writer.kind, existingConditions),])}+ case 'legacy-retry':+ return presentConditions([+ ...(existingConditions ?? []).filter(+ c => c.type === NETWORK_POLICY_OWNERSHIP_CONDITION_TYPE+ ),+ carriedConvergedMarker(+ writer.kind,+ networkPolicyMarkerFacts(existingConditions).apply,+ existingConditions,+ now+ ),+ buildLegacyNetworkPolicyCondition(writer.legacy, now, existingConditions),+ ])case 'terminal':return presentConditions([...(existingConditions ?? []).filter(@@ -1471,6 +1518,14 @@ export class WorkflowReconciler {private readonly codexContexts = new Map<string, CodexReconcileContext>()/** B3(c): one process-lifetime DELETE of the leftover mcp-servers internet NP. Keyed by recipe uid. */private readonly prunedLegacyMcpServersInternetEgress = new Set<string>()+ /**+ * R4-L4: the backoff after a legacy DELETE that did not land, keyed like the+ * gone set. `notBeforeMs` is the earliest time the next DELETE may be sent.+ */+ private readonly legacyMcpServersInternetEgressBackoff = new Map<+ string,+ { delayMs: number; notBeforeMs: number }+ >()constructor(private readonly deps: WorkflowReconcilerDeps) {this.pluginWorkloadSdkProvisioner = new PluginWorkloadSdkProvisioner({@@ -3346,6 +3401,7 @@ export class WorkflowReconciler {* | path | proxy required | proxy uncertain | proxy retired | other lane, not desired | legacy |* | reconcile() pass | applied | kept if live | pruned | pruned | deleted |* | mid-run retry (this) | applied | kept if live | kept | kept | kept |+ * | legacy retry | untouched | untouched | untouched | untouched | deleted, backoff |* | terminal teardown | kept | kept | kept | kept | kept |* | finalizer | deleted | deleted | deleted | deleted | only if labeled |*@@ -3370,10 +3426,21 @@ export class WorkflowReconciler {* | `reconcile` (run lane, eager, SDK with a summary) | observed | observed | observed | what converged |* | SDK with a runtime and no summary | not written | not written | not written | nothing |* | `apply-retry` (this, in-progress and active) | observed | carried | carried | A only |+ * | `legacy-retry` (WRC, before the short-circuits) | carried | carried | observed, backoff | L only |* | `terminal` | forced converged | carried | carried | A |* | `unmanaged` (kill switch, SDK lane without a runtime, capability removal) | converged | converged | removed | everything |* | finalizer | the object is deleted | | | |*+ * The legacy retry runs in the WRC after the early token reap and before+ * every short-circuit, only while `DeletePending` is published. It and the+ * reconcile() pass share one backoff per recipe: after a DELETE that did+ * not land, the next one waits `min(prev * 2 || 60 s, 1 h)`, and inside+ * that window the delete answers `pending` without a request. A `removed`+ * answer clears L with a conditions-only patch; `pending` writes nothing.+ * The WRC `reconcile()` wrapper requeues a pass that leaves L pending and+ * asks for no other requeue after `max(1 s, the rest of the window)`, at a+ * fixed interval.+ ** A carrying writer keeps a marker whose reason it did not write as it is* and warns; `reconcile` overwrites it. `unmanaged` is written only after* `cleanupPluginWorkloadSdk` without `preserveWorkflowRuntime` succeeded:@@ -3626,7 +3693,37 @@ export class WorkflowReconciler {private forgetLegacyMcpServersInternetEgress(recipeName: string, recipeUid?: string): void {this.prunedLegacyMcpServersInternetEgress.delete(recipeName)- if (recipeUid) this.prunedLegacyMcpServersInternetEgress.delete(recipeUid)+ this.legacyMcpServersInternetEgressBackoff.delete(recipeName)+ if (recipeUid) {+ this.prunedLegacyMcpServersInternetEgress.delete(recipeUid)+ this.legacyMcpServersInternetEgressBackoff.delete(recipeUid)+ }+ }++ /**+ * R4-L4: the legacy delete the WRC retries before its short-circuits while+ * `DeletePending` is published (the `legacy-retry` writer). The same delete+ * and backoff as the reconcile() pass: inside the backoff window it answers+ * `pending` without a request.+ */+ async retryLegacyMcpServersInternetEgressDelete(+ recipeName: string,+ recipeUid: string+ ): Promise<NetworkPolicyLegacyFact> {+ return this.pruneLegacyMcpServersInternetEgressPolicy(recipeName, recipeUid)+ }++ /**+ * R4-L4: how long until the next legacy DELETE may be sent for this recipe;+ * 0 when no backoff is running. The WRC requeues a pass that leaves+ * `DeletePending` published after this delay.+ */+ legacyMcpServersInternetEgressRetryDelayMs(recipeName: string, recipeUid: string): number {+ const backoff = this.legacyMcpServersInternetEgressBackoff.get(+ this.legacyMcpServersInternetEgressKey(recipeName, recipeUid)+ )+ if (backoff === undefined) return 0+ return Math.max(0, backoff.notBeforeMs - Date.now())}private async pruneLegacyMcpServersInternetEgressPolicy(@@ -3640,6 +3737,14 @@ export class WorkflowReconciler {})return 'removed'}+ const backoff = this.legacyMcpServersInternetEgressBackoff.get(key)+ if (backoff !== undefined && Date.now() < backoff.notBeforeMs) {+ this.log.debug('Skipping legacy mcp-servers internet NP delete; inside its backoff window', {+ recipe: recipeName,+ retryInMs: backoff.notBeforeMs - Date.now(),+ })+ return 'pending'+ }const outcome = await observeNamespacedDelete(() =>this.deps.networkingApi.deleteNamespacedNetworkPolicy({name: `${recipeName}-mcp-servers-egress-internet`,@@ -3647,13 +3752,23 @@ export class WorkflowReconciler {}))if (outcome.kind === 'failed') {+ const delayMs = Math.min(+ backoff === undefined ? LEGACY_NETWORK_POLICY_DELETE_BACKOFF_BASE_MS : backoff.delayMs * 2,+ LEGACY_NETWORK_POLICY_DELETE_BACKOFF_MAX_MS+ )+ this.legacyMcpServersInternetEgressBackoff.set(key, {+ delayMs,+ notBeforeMs: Date.now() + delayMs,+ })this.log.error('Legacy mcp-servers internet NP delete failed', {recipe: recipeName,...deleteOutcomeFields(outcome),+ retryInMs: delayMs,err: outcome.error,})return 'pending'}+ this.legacyMcpServersInternetEgressBackoff.delete(key)this.log.info('Legacy mcp-servers internet NP delete', {recipe: recipeName,...deleteOutcomeFields(outcome),diff --git a/workflow-recipes/tests/unit/workflow/workflowReconciler.test.ts b/workflow-recipes/tests/unit/workflow/workflowReconciler.test.tsindex e1705a836..7a8764f13 100644--- a/workflow-recipes/tests/unit/workflow/workflowReconciler.test.ts+++ b/workflow-recipes/tests/unit/workflow/workflowReconciler.test.ts@@ -41,6 +41,7 @@ import {type WorkflowReconcilerDeps,buildNetworkPolicyConvergedCondition,buildNetworkPolicyOwnershipConditions,+ hasLegacyNetworkPolicyDeletePendingMarker,networkPolicyConditionsChanged,networkPolicyMarkerConditions,networkPolicyMarkerFacts,@@ -3034,7 +3035,9 @@ describe('WorkflowReconciler — reconcile loop', () => {})})- it('G3: does not skip the legacy internet NP after DELETE 403', async () => {+ // R4-L4: a 403 does not mark the policy gone; the next DELETE waits for the+ // end of its backoff window (60 s after the first failure).+ it('G3: does not skip the legacy internet NP after DELETE 403, and retries it once its backoff window ends', async () => {const networkingApi = makeNetworkingApi() as ReturnType<typeof makeNetworkingApi>networkingApi.deleteNamespacedNetworkPolicy.mockImplementation(async ({ name, namespace }) => {if (name === 'test-wf-mcp-servers-egress-internet' && namespace === 'mcp-server') {@@ -3047,25 +3050,115 @@ describe('WorkflowReconciler — reconcile loop', () => {networkingApi: networkingApi as never,}))+ const legacyDeletes = () =>+ networkingApi.deleteNamespacedNetworkPolicy.mock.calls.filter(+ ([arg]) =>+ arg.name === 'test-wf-mcp-servers-egress-internet' && arg.namespace === 'mcp-server'+ )+ const t0 = new Date('2026-09-23T10:00:00.000Z').getTime()+ vi.useFakeTimers({ toFake: ['Date'] })+ try {+ vi.setSystemTime(t0)+ const first = await reconciler.reconcile(+ 'test-wf',+ 'uid-123',+ 'sandbox-recipes',+ makeSpec({ mcpServers: ['redis-mcp'] })+ )+ expect(legacyDeletes()).toHaveLength(1)+ expect(+ (first.networkPolicyOwnershipConditions ?? []).filter(+ c => c.type === 'WorkflowLegacyNetworkPolicyRemoved'+ )+ ).toMatchObject([{ status: 'False', reason: 'DeletePending' }])- await reconciler.reconcile(- 'test-wf',- 'uid-123',- 'sandbox-recipes',- makeSpec({ mcpServers: ['redis-mcp'] })- )- await reconciler.reconcile(- 'test-wf',- 'uid-123',- 'sandbox-recipes',- makeSpec({ mcpServers: ['redis-mcp'] })- )+ vi.setSystemTime(t0 + 30_000)+ const inside = await reconciler.reconcile(+ 'test-wf',+ 'uid-123',+ 'sandbox-recipes',+ makeSpec({ mcpServers: ['redis-mcp'] })+ )+ // Liveness witness: the pass reached the prune (the legacy fact is+ // published), and inside the window it sent no DELETE.+ expect(+ (inside.networkPolicyOwnershipConditions ?? []).filter(+ c => c.type === 'WorkflowLegacyNetworkPolicyRemoved'+ )+ ).toMatchObject([{ status: 'False', reason: 'DeletePending' }])+ expect(legacyDeletes()).toHaveLength(1)+ expect(reconciler.legacyMcpServersInternetEgressRetryDelayMs('test-wf', 'uid-123')).toBe(+ 30_000+ )++ vi.setSystemTime(t0 + 60_000)+ await reconciler.reconcile(+ 'test-wf',+ 'uid-123',+ 'sandbox-recipes',+ makeSpec({ mcpServers: ['redis-mcp'] })+ )+ expect(legacyDeletes()).toHaveLength(2)+ // The second failure doubles the window.+ expect(reconciler.legacyMcpServersInternetEgressRetryDelayMs('test-wf', 'uid-123')).toBe(+ 120_000+ )+ } finally {+ vi.useRealTimers()+ }+ })- const legacyDeletes = networkingApi.deleteNamespacedNetworkPolicy.mock.calls.filter(- ([arg]) =>- arg.name === 'test-wf-mcp-servers-egress-internet' && arg.namespace === 'mcp-server'+ it('R4-L4: doubles the legacy DELETE backoff up to 1 h, and a 404 ends it', async () => {+ const LEGACY = 'test-wf-mcp-servers-egress-internet'+ const networkingApi = makeNetworkingApi() as ReturnType<typeof makeNetworkingApi>+ let legacyAnswer = { code: 403 }+ networkingApi.deleteNamespacedNetworkPolicy.mockImplementation(async ({ name, namespace }) => {+ if (name === LEGACY && namespace === 'mcp-server') throw legacyAnswer+ return {}+ })+ const reconciler = new WorkflowReconciler(+ makeDeps({+ networkingApi: networkingApi as never,+ }))- expect(legacyDeletes).toHaveLength(2)+ const legacyDeletes = () =>+ networkingApi.deleteNamespacedNetworkPolicy.mock.calls.filter(+ ([arg]) => arg.name === LEGACY && arg.namespace === 'mcp-server'+ )+ vi.useFakeTimers({ toFake: ['Date'] })+ try {+ let now = new Date('2026-09-23T10:00:00.000Z').getTime()+ const windows: number[] = []+ for (let attempt = 0; attempt < 8; attempt++) {+ vi.setSystemTime(now)+ expect(+ await reconciler.retryLegacyMcpServersInternetEgressDelete('test-wf', 'uid-123')+ ).toBe('pending')+ const delay = reconciler.legacyMcpServersInternetEgressRetryDelayMs('test-wf', 'uid-123')+ windows.push(delay)+ now += delay+ }++ expect(windows).toEqual([+ 60_000, 120_000, 240_000, 480_000, 960_000, 1_920_000, 3_600_000, 3_600_000,+ ])+ expect(legacyDeletes()).toHaveLength(8)++ legacyAnswer = { code: 404 }+ vi.setSystemTime(now)+ expect(await reconciler.retryLegacyMcpServersInternetEgressDelete('test-wf', 'uid-123')).toBe(+ 'removed'+ )+ expect(reconciler.legacyMcpServersInternetEgressRetryDelayMs('test-wf', 'uid-123')).toBe(0)+ expect(legacyDeletes()).toHaveLength(9)+ // Observed gone: a later retry answers from the gone set, with no DELETE.+ expect(await reconciler.retryLegacyMcpServersInternetEgressDelete('test-wf', 'uid-123')).toBe(+ 'removed'+ )+ expect(legacyDeletes()).toHaveLength(9)+ } finally {+ vi.useRealTimers()+ }})it('clears the legacy internet NP process set on recipe delete so a recreate can delete again', async () => {@@ -5582,52 +5675,66 @@ describe('WorkflowReconciler — reconcile loop', () => {})// The legacy mcp-servers internet policy is deleted by name, apart- // from the catalog prune: only reconcile() deletes it. A failed delete- // is its own fact (R4-L3), DeletePending on its own condition, with no- // retry flag and the catalog prune converged, and the next reconcile()- // pass deletes it again.+ // from the catalog prune: reconcile() deletes it, and the WRC's+ // legacy-retry writer retries it (R4-L4). A failed delete is its own+ // fact (R4-L3), DeletePending on its own condition, with no retry flag+ // and the catalog prune converged, and the next reconcile() pass after+ // the backoff window deletes it again.const LEGACY = 'test-wf-mcp-servers-egress-internet'const legacyDeletes = (apiserver: Converged) =>apiserver.api.deleteNamespacedNetworkPolicy.mock.calls.filter(([arg]) => arg.name === LEGACY)- it('reports a failed legacy internet policy delete from reconcile() as DeletePending, and deletes it on the next reconcile()', async () => {- const apiserver = makeApiserverNetworkingApi()- const reconciler = new WorkflowReconciler(- makeDeps({ networkingApi: apiserver.api as never })- )- const liveDelete = apiserver.api.deleteNamespacedNetworkPolicy.getMockImplementation()!- apiserver.api.deleteNamespacedNetworkPolicy.mockImplementation(- async (arg: { name: string; namespace: string }) => {- if (arg.name === LEGACY && legacyDeletes(apiserver).length === 1) {- throw { code: 403, message: 'forbidden' }+ it('reports a failed legacy internet policy delete from reconcile() as DeletePending, and deletes it on the next reconcile() after its backoff window', async () => {+ const t0 = new Date('2026-09-23T10:00:00.000Z').getTime()+ vi.useFakeTimers({ toFake: ['Date'] })+ vi.setSystemTime(t0)+ try {+ const apiserver = makeApiserverNetworkingApi()+ const reconciler = new WorkflowReconciler(+ makeDeps({ networkingApi: apiserver.api as never })+ )+ const liveDelete = apiserver.api.deleteNamespacedNetworkPolicy.getMockImplementation()!+ apiserver.api.deleteNamespacedNetworkPolicy.mockImplementation(+ async (arg: { name: string; namespace: string }) => {+ if (arg.name === LEGACY && legacyDeletes(apiserver).length === 1) {+ throw { code: 403, message: 'forbidden' }+ }+ return liveDelete(arg)}- return liveDelete(arg)- }- )+ )- const failed = await reconciler.reconcile('test-wf', 'uid-123', 'sandbox-recipes', spec())- expect(legacyDeletes(apiserver)).toHaveLength(1)- expect(failed.networkPolicyRetryPending).toBeFalsy()- const published = failed.networkPolicyOwnershipConditions ?? []- expect(published.filter(c => c.type === 'WorkflowNetworkPoliciesConverged')).toEqual([])- expect(- published.filter(c => c.type === 'WorkflowLegacyNetworkPolicyRemoved')- ).toMatchObject([{ status: 'False', reason: 'DeletePending' }])+ const failed = await reconciler.reconcile(+ 'test-wf',+ 'uid-123',+ 'sandbox-recipes',+ spec()+ )+ expect(legacyDeletes(apiserver)).toHaveLength(1)+ expect(failed.networkPolicyRetryPending).toBeFalsy()+ const published = failed.networkPolicyOwnershipConditions ?? []+ expect(published.filter(c => c.type === 'WorkflowNetworkPoliciesConverged')).toEqual([])+ expect(+ published.filter(c => c.type === 'WorkflowLegacyNetworkPolicyRemoved')+ ).toMatchObject([{ status: 'False', reason: 'DeletePending' }])- const pruned = await reconciler.reconcile(- 'test-wf',- 'uid-123',- 'sandbox-recipes',- spec(),- {- conditions: published,- }- )- expect(legacyDeletes(apiserver)).toHaveLength(2)- expect(pruned.networkPolicyRetryPending).toBeFalsy()- expect(pruned.networkPolicyOwnershipConditions).toEqual([])+ vi.setSystemTime(t0 + 60_000)+ const pruned = await reconciler.reconcile(+ 'test-wf',+ 'uid-123',+ 'sandbox-recipes',+ spec(),+ {+ conditions: published,+ }+ )+ expect(legacyDeletes(apiserver)).toHaveLength(2)+ expect(pruned.networkPolicyRetryPending).toBeFalsy()+ expect(pruned.networkPolicyOwnershipConditions).toEqual([])+ } finally {+ vi.useRealTimers()+ }})it('sends no legacy internet policy DELETE from the retry, even one no pass has seen gone', async () => {@@ -9451,6 +9558,57 @@ describe('networkPolicyMarkerConditions', () => {})})+ describe('legacy-retry', () => {+ it('R4-L4: clears the legacy fact once removed and carries the apply, the prune and the ownership condition', () => {+ const existing = published({+ conflicts: [conflict],+ retryPending: true,+ prune: 'unevaluated',+ legacy: 'pending',+ })+ // Liveness witness: all three conditions were published.+ expect(existing).toHaveLength(3)++ const settled = networkPolicyMarkerConditions(+ { kind: 'legacy-retry', legacy: 'removed' },+ existing,+ now+ )++ expect(reasons(settled)).toEqual([+ 'WorkflowNetworkPolicyOwnership/OwnershipConflict',+ 'WorkflowNetworkPoliciesConverged/RetryAndPruneUnevaluated',+ ])+ expect(settled.every(c => c.lastTransitionTime === earlier)).toBe(true)+ })++ it('R4-L4: keeps DeletePending and its transition time while the delete stays pending', () => {+ const existing = published({ prune: 'pending', legacy: 'pending' })+ expect(existing).toHaveLength(2)++ expect(+ networkPolicyMarkerConditions({ kind: 'legacy-retry', legacy: 'pending' }, existing, now)+ ).toEqual(existing)+ })++ it('R4-L4: enters only for a published DeletePending', () => {+ expect(hasLegacyNetworkPolicyDeletePendingMarker(published({ legacy: 'pending' }))).toBe(true)+ expect(hasLegacyNetworkPolicyDeletePendingMarker(published({ prune: 'pending' }))).toBe(false)+ expect(hasLegacyNetworkPolicyDeletePendingMarker(undefined)).toBe(false)+ expect(+ hasLegacyNetworkPolicyDeletePendingMarker([+ {+ type: 'WorkflowLegacyNetworkPolicyRemoved',+ status: 'False',+ reason: 'SomethingElse',+ message: 'written by another head',+ lastTransitionTime: earlier,+ },+ ])+ ).toBe(false)+ })+ })+describe('terminal', () => {it('R4-L3: drops a pending apply and carries the prune, the legacy fact and the ownership condition', () => {const existing = published({