Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript

fix(workflow-recipes): retry the legacy delete before the short-circuits with a backoff (R4-L4)

Alfredo Lopez2dc56e1062cc
diff --git a/workflow-recipes/package-lock.json b/workflow-recipes/package-lock.json
index 7986c86dc..2812e742e 100644
--- a/workflow-recipes/package-lock.json
+++ b/workflow-recipes/package-lock.json
@@ -1,12 +1,12 @@
{
"name": "workload-recipes-controller",
- "version": "0.1.444",
+ "version": "0.1.445",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "workload-recipes-controller",
- "version": "0.1.444",
+ "version": "0.1.445",
"license": "SEE LICENSE IN LICENSE",
"dependencies": {
"@aws-sdk/client-s3": "^3.1009.0",
diff --git a/workflow-recipes/package.json b/workflow-recipes/package.json
index ffaed2bab..8a02d1241 100644
--- a/workflow-recipes/package.json
+++ b/workflow-recipes/package.json
@@ -1,6 +1,6 @@
{
"name": "workload-recipes-controller",
- "version": "0.1.444",
+ "version": "0.1.445",
"description": "Workload Recipes Controller (WRC) - WorkloadRecipe lifecycle management and MCP server interface",
"main": "dist/main.js",
"engines": {
diff --git a/workflow-recipes/src/reconciler/reconciler.test.ts b/workflow-recipes/src/reconciler/reconciler.test.ts
index 7b19eed95..eb8765ef8 100644
--- a/workflow-recipes/src/reconciler/reconciler.test.ts
+++ b/workflow-recipes/src/reconciler/reconciler.test.ts
@@ -16653,6 +16653,43 @@ describe('WorkflowRecipeReconciler', () => {
expect(legacyDeletes()).toBe(0)
expect(result.requeueAfterMs).toBeUndefined()
})
+
+ it('R4-L4: keeps the requeue a short-circuit already asked for while DeletePending stays published', async () => {
+ const inner = installRealInner()
+ stubShortCircuits(inner)
+ const retry = vi
+ .spyOn(inner, 'retryRunLaneNetworkPolicies')
+ .mockResolvedValue({ conflicts: [], retryPending: true })
+ legacyDeleteAnswers({ code: 403, message: 'forbidden' })
+ const conditions = networkPolicyMarkerConditions(
+ {
+ kind: 'reconcile',
+ summary: { conflicts: [], retryPending: true, prune: 'converged', legacy: 'pending' },
+ },
+ [],
+ '2026-09-23T09:00:00.000Z'
+ )
+ expect(ofType(conditions, MARKER_TYPE)).toMatchObject([{ reason: 'RetryPending' }])
+ expect(ofType(conditions, LEGACY_TYPE)).toMatchObject([{ reason: 'DeletePending' }])
+ const active = runScopedRecipe({
+ phase: 'active',
+ message: 'Workflow running',
+ workflowExecution: { phase: 'running' },
+ conditions,
+ })
+
+ const result = await reconciler.reconcile(active)
+
+ // Witnesses: the apply retry ran and the legacy DELETE was sent.
+ expect(retry).toHaveBeenCalledTimes(1)
+ expect(legacyDeletes()).toBe(1)
+ expect(ofType(active.status?.conditions, LEGACY_TYPE)).toMatchObject([
+ { reason: 'DeletePending' },
+ ])
+ // The apply retry's progress requeue wins over the 60 s legacy window.
+ expect(result.requeueAfterMs).toBe(WORKFLOW_PROGRESS_REQUEUE_BASE_MS)
+ expect(result.requeueFixedInterval).toBe(false)
+ })
})
// A reconcile() prune whose DELETE did not land publishes
diff --git a/workflow-recipes/src/reconciler/workflowRecipeReconciler.ts b/workflow-recipes/src/reconciler/workflowRecipeReconciler.ts
index 2bca25ed1..4636e391d 100644
--- a/workflow-recipes/src/reconciler/workflowRecipeReconciler.ts
+++ b/workflow-recipes/src/reconciler/workflowRecipeReconciler.ts
@@ -69,6 +69,7 @@ import {
WORKFLOW_OUTPUT_CONDITION_TYPES,
WorkflowReconciler,
WorkflowReconcilerDeps,
+ hasLegacyNetworkPolicyDeletePendingMarker,
hasNetworkPolicyRetryPendingMarker,
networkPolicyConditionsChanged,
networkPolicyMarkerConditions,
@@ -1525,6 +1526,54 @@ export class WorkflowRecipeReconciler {
return summary.retryPending ? WORKFLOW_PROGRESS_REQUEUE_BASE_MS : undefined
}
+ /**
+ * R4-L4: the `legacy-retry` writer. Enters only while the published group
+ * says the legacy mcp-servers internet policy is pending a delete. The
+ * delete shares the reconcile() pass's per-recipe backoff, so inside the
+ * window it sends no request. `removed` clears the legacy fact with a
+ * conditions-only patch and carries the apply and prune facts; `pending`
+ * writes nothing, and the `reconcile()` wrapper requeues for the rest of
+ * the window.
+ */
+ private async retryPendingLegacyNetworkPolicyDelete(recipe: WorkflowRecipeCRD): Promise<void> {
+ if (
+ !this.workflowReconciler ||
+ !hasLegacyNetworkPolicyDeletePendingMarker(recipe.status?.conditions)
+ ) {
+ return
+ }
+ const legacy = await this.workflowReconciler.retryLegacyMcpServersInternetEgressDelete(
+ recipe.metadata.name,
+ recipe.metadata.uid ?? ''
+ )
+ if (legacy === 'pending') return
+ await this.publishNetworkPolicyMarkerWriter(recipe, { kind: 'legacy-retry', legacy })
+ }
+
+ /**
+ * R4-L4: a pass that leaves `DeletePending` published and asks for no other
+ * requeue is requeued at a fixed interval for the rest of the legacy
+ * delete's backoff window, at least 1 s. The group the pass publishes wins
+ * over the one it found; a pass that publishes no group keeps the found
+ * one, including a legacy-retry patch made during the pass.
+ */
+ private requeueForPendingLegacyNetworkPolicyDelete(
+ recipe: WorkflowRecipeCRD,
+ result: ReconcileResult
+ ): void {
+ if (!this.workflowReconciler || result.requeueAfterMs !== undefined) return
+ const conditions = result.networkPolicyOwnershipConditions ?? recipe.status?.conditions
+ if (!hasLegacyNetworkPolicyDeletePendingMarker(conditions)) return
+ result.requeueAfterMs = Math.max(
+ 1_000,
+ this.workflowReconciler.legacyMcpServersInternetEgressRetryDelayMs(
+ recipe.metadata.name,
+ recipe.metadata.uid ?? ''
+ )
+ )
+ result.requeueFixedInterval = true
+ }
+
/**
* Settles the marker once the run is terminal. The run no longer needs its
* policies, so a pending apply is dropped: there is nothing left to retry.
@@ -1755,6 +1804,7 @@ export class WorkflowRecipeReconciler {
// (e.g. observeCurrentWorkloadStatus) leave the projection undefined, which
// both consumers treat as "no SDK opinion this pass".
const result = await this.reconcileInternal(recipe)
+ this.requeueForPendingLegacyNetworkPolicyDelete(recipe, result)
result.pluginWorkloadSdkProjection = this.projectPluginWorkloadSdk(recipe, result)
createLogger('wrc', recipe.metadata.name).info('recipe reconciliation completed', {
recipe: recipe.metadata.name,
@@ -2001,6 +2051,11 @@ export class WorkflowRecipeReconciler {
await this.ensureOAuthBrokerTokenSecret(recipe)
}
+ // R4-L4: the short-circuits below never reach the inner reconcile(), so
+ // a legacy mcp-servers internet policy pending a delete is retried here,
+ // before all of them (the `legacy-retry` writer).
+ await this.retryPendingLegacyNetworkPolicyDelete(recipe)
+
if (
currentPhase === 'active' &&
awaitsTriggeredRun &&
diff --git a/workflow-recipes/src/workflow/workflowReconciler.ts b/workflow-recipes/src/workflow/workflowReconciler.ts
index 28c8ae5b1..caae3a1e5 100644
--- a/workflow-recipes/src/workflow/workflowReconciler.ts
+++ b/workflow-recipes/src/workflow/workflowReconciler.ts
@@ -432,6 +432,36 @@ export function hasNetworkPolicyRetryPendingMarker(
return networkPolicyMarkerFacts(conditions).apply === 'pending'
}
+/**
+ * True while the published group says the legacy mcp-servers internet policy
+ * is pending a delete (`WorkflowLegacyNetworkPolicyRemoved=False/DeletePending`).
+ * Only this reason enters the `legacy-retry` writer; a reason this controller
+ * did not write is never interpreted.
+ */
+export function hasLegacyNetworkPolicyDeletePendingMarker(
+ conditions: StatusCondition[] | undefined
+): boolean {
+ const published = publishedMarker(
+ conditions,
+ LEGACY_NETWORK_POLICY_REMOVED_CONDITION_TYPE,
+ isLegacyNetworkPolicyReason
+ )
+ return (
+ published.kind === 'known' &&
+ LEGACY_NETWORK_POLICY_REASONS[published.condition.reason as LegacyNetworkPolicyReason].fact ===
+ 'pending'
+ )
+}
+
+/**
+ * Backoff between two DELETEs of the legacy mcp-servers internet policy for
+ * one recipe after a DELETE that did not land: 60 s, doubling up to 1 h. A
+ * transient 403 does not leave a false DeletePending for hours, a sustained
+ * one costs one DELETE per hour per recipe, and a converged recipe costs none.
+ */
+export const LEGACY_NETWORK_POLICY_DELETE_BACKOFF_BASE_MS = 60_000
+export const LEGACY_NETWORK_POLICY_DELETE_BACKOFF_MAX_MS = 3_600_000
+
/**
* Reads the facts a published marker carries. This and
* `buildNetworkPolicyConvergedCondition` are the only places that encode what
@@ -554,6 +584,9 @@ function buildLegacyNetworkPolicyCondition(
* summary). Observes the apply, the prune and the legacy delete.
* - `apply-retry`: the mid-run retry. Observes the apply; carries the prune
* and legacy facts.
+ * - `legacy-retry`: the legacy delete the WRC retries before its
+ * short-circuits while `DeletePending` is published. Observes the legacy
+ * fact; carries the apply and prune facts and the ownership condition.
* - `terminal`: the run ended, so a pending apply is dropped; carries the
* prune and legacy facts and the ownership condition.
* - `unmanaged`: the kill switch, the SDK-only lane without a runtime and
@@ -563,6 +596,7 @@ function buildLegacyNetworkPolicyCondition(
export type NetworkPolicyMarkerWriter =
| { kind: 'reconcile'; summary: NetworkPolicyPassSummary }
| { kind: 'apply-retry'; summary: WorkflowNetworkPolicyApplySummary }
+ | { kind: 'legacy-retry'; legacy: NetworkPolicyLegacyFact }
| { kind: 'terminal' }
| { kind: 'unmanaged' }
@@ -685,6 +719,19 @@ export function networkPolicyMarkerConditions(
carriedLegacyMarker(writer.kind, existingConditions),
])
}
+ case 'legacy-retry':
+ return presentConditions([
+ ...(existingConditions ?? []).filter(
+ c => c.type === NETWORK_POLICY_OWNERSHIP_CONDITION_TYPE
+ ),
+ carriedConvergedMarker(
+ writer.kind,
+ networkPolicyMarkerFacts(existingConditions).apply,
+ existingConditions,
+ now
+ ),
+ buildLegacyNetworkPolicyCondition(writer.legacy, now, existingConditions),
+ ])
case 'terminal':
return presentConditions([
...(existingConditions ?? []).filter(
@@ -1471,6 +1518,14 @@ export class WorkflowReconciler {
private readonly codexContexts = new Map<string, CodexReconcileContext>()
/** B3(c): one process-lifetime DELETE of the leftover mcp-servers internet NP. Keyed by recipe uid. */
private readonly prunedLegacyMcpServersInternetEgress = new Set<string>()
+ /**
+ * R4-L4: the backoff after a legacy DELETE that did not land, keyed like the
+ * gone set. `notBeforeMs` is the earliest time the next DELETE may be sent.
+ */
+ private readonly legacyMcpServersInternetEgressBackoff = new Map<
+ string,
+ { delayMs: number; notBeforeMs: number }
+ >()
constructor(private readonly deps: WorkflowReconcilerDeps) {
this.pluginWorkloadSdkProvisioner = new PluginWorkloadSdkProvisioner({
@@ -3346,6 +3401,7 @@ export class WorkflowReconciler {
* | path | proxy required | proxy uncertain | proxy retired | other lane, not desired | legacy |
* | reconcile() pass | applied | kept if live | pruned | pruned | deleted |
* | mid-run retry (this) | applied | kept if live | kept | kept | kept |
+ * | legacy retry | untouched | untouched | untouched | untouched | deleted, backoff |
* | terminal teardown | kept | kept | kept | kept | kept |
* | finalizer | deleted | deleted | deleted | deleted | only if labeled |
*
@@ -3370,10 +3426,21 @@ export class WorkflowReconciler {
* | `reconcile` (run lane, eager, SDK with a summary) | observed | observed | observed | what converged |
* | SDK with a runtime and no summary | not written | not written | not written | nothing |
* | `apply-retry` (this, in-progress and active) | observed | carried | carried | A only |
+ * | `legacy-retry` (WRC, before the short-circuits) | carried | carried | observed, backoff | L only |
* | `terminal` | forced converged | carried | carried | A |
* | `unmanaged` (kill switch, SDK lane without a runtime, capability removal) | converged | converged | removed | everything |
* | finalizer | the object is deleted | | | |
*
+ * The legacy retry runs in the WRC after the early token reap and before
+ * every short-circuit, only while `DeletePending` is published. It and the
+ * reconcile() pass share one backoff per recipe: after a DELETE that did
+ * not land, the next one waits `min(prev * 2 || 60 s, 1 h)`, and inside
+ * that window the delete answers `pending` without a request. A `removed`
+ * answer clears L with a conditions-only patch; `pending` writes nothing.
+ * The WRC `reconcile()` wrapper requeues a pass that leaves L pending and
+ * asks for no other requeue after `max(1 s, the rest of the window)`, at a
+ * fixed interval.
+ *
* A carrying writer keeps a marker whose reason it did not write as it is
* and warns; `reconcile` overwrites it. `unmanaged` is written only after
* `cleanupPluginWorkloadSdk` without `preserveWorkflowRuntime` succeeded:
@@ -3626,7 +3693,37 @@ export class WorkflowReconciler {
private forgetLegacyMcpServersInternetEgress(recipeName: string, recipeUid?: string): void {
this.prunedLegacyMcpServersInternetEgress.delete(recipeName)
- if (recipeUid) this.prunedLegacyMcpServersInternetEgress.delete(recipeUid)
+ this.legacyMcpServersInternetEgressBackoff.delete(recipeName)
+ if (recipeUid) {
+ this.prunedLegacyMcpServersInternetEgress.delete(recipeUid)
+ this.legacyMcpServersInternetEgressBackoff.delete(recipeUid)
+ }
+ }
+
+ /**
+ * R4-L4: the legacy delete the WRC retries before its short-circuits while
+ * `DeletePending` is published (the `legacy-retry` writer). The same delete
+ * and backoff as the reconcile() pass: inside the backoff window it answers
+ * `pending` without a request.
+ */
+ async retryLegacyMcpServersInternetEgressDelete(
+ recipeName: string,
+ recipeUid: string
+ ): Promise<NetworkPolicyLegacyFact> {
+ return this.pruneLegacyMcpServersInternetEgressPolicy(recipeName, recipeUid)
+ }
+
+ /**
+ * R4-L4: how long until the next legacy DELETE may be sent for this recipe;
+ * 0 when no backoff is running. The WRC requeues a pass that leaves
+ * `DeletePending` published after this delay.
+ */
+ legacyMcpServersInternetEgressRetryDelayMs(recipeName: string, recipeUid: string): number {
+ const backoff = this.legacyMcpServersInternetEgressBackoff.get(
+ this.legacyMcpServersInternetEgressKey(recipeName, recipeUid)
+ )
+ if (backoff === undefined) return 0
+ return Math.max(0, backoff.notBeforeMs - Date.now())
}
private async pruneLegacyMcpServersInternetEgressPolicy(
@@ -3640,6 +3737,14 @@ export class WorkflowReconciler {
})
return 'removed'
}
+ const backoff = this.legacyMcpServersInternetEgressBackoff.get(key)
+ if (backoff !== undefined && Date.now() < backoff.notBeforeMs) {
+ this.log.debug('Skipping legacy mcp-servers internet NP delete; inside its backoff window', {
+ recipe: recipeName,
+ retryInMs: backoff.notBeforeMs - Date.now(),
+ })
+ return 'pending'
+ }
const outcome = await observeNamespacedDelete(() =>
this.deps.networkingApi.deleteNamespacedNetworkPolicy({
name: `${recipeName}-mcp-servers-egress-internet`,
@@ -3647,13 +3752,23 @@ export class WorkflowReconciler {
})
)
if (outcome.kind === 'failed') {
+ const delayMs = Math.min(
+ backoff === undefined ? LEGACY_NETWORK_POLICY_DELETE_BACKOFF_BASE_MS : backoff.delayMs * 2,
+ LEGACY_NETWORK_POLICY_DELETE_BACKOFF_MAX_MS
+ )
+ this.legacyMcpServersInternetEgressBackoff.set(key, {
+ delayMs,
+ notBeforeMs: Date.now() + delayMs,
+ })
this.log.error('Legacy mcp-servers internet NP delete failed', {
recipe: recipeName,
...deleteOutcomeFields(outcome),
+ retryInMs: delayMs,
err: outcome.error,
})
return 'pending'
}
+ this.legacyMcpServersInternetEgressBackoff.delete(key)
this.log.info('Legacy mcp-servers internet NP delete', {
recipe: recipeName,
...deleteOutcomeFields(outcome),
diff --git a/workflow-recipes/tests/unit/workflow/workflowReconciler.test.ts b/workflow-recipes/tests/unit/workflow/workflowReconciler.test.ts
index e1705a836..7a8764f13 100644
--- a/workflow-recipes/tests/unit/workflow/workflowReconciler.test.ts
+++ b/workflow-recipes/tests/unit/workflow/workflowReconciler.test.ts
@@ -41,6 +41,7 @@ import {
type WorkflowReconcilerDeps,
buildNetworkPolicyConvergedCondition,
buildNetworkPolicyOwnershipConditions,
+ hasLegacyNetworkPolicyDeletePendingMarker,
networkPolicyConditionsChanged,
networkPolicyMarkerConditions,
networkPolicyMarkerFacts,
@@ -3034,7 +3035,9 @@ describe('WorkflowReconciler — reconcile loop', () => {
})
})
- it('G3: does not skip the legacy internet NP after DELETE 403', async () => {
+ // R4-L4: a 403 does not mark the policy gone; the next DELETE waits for the
+ // end of its backoff window (60 s after the first failure).
+ it('G3: does not skip the legacy internet NP after DELETE 403, and retries it once its backoff window ends', async () => {
const networkingApi = makeNetworkingApi() as ReturnType<typeof makeNetworkingApi>
networkingApi.deleteNamespacedNetworkPolicy.mockImplementation(async ({ name, namespace }) => {
if (name === 'test-wf-mcp-servers-egress-internet' && namespace === 'mcp-server') {
@@ -3047,25 +3050,115 @@ describe('WorkflowReconciler — reconcile loop', () => {
networkingApi: networkingApi as never,
})
)
+ const legacyDeletes = () =>
+ networkingApi.deleteNamespacedNetworkPolicy.mock.calls.filter(
+ ([arg]) =>
+ arg.name === 'test-wf-mcp-servers-egress-internet' && arg.namespace === 'mcp-server'
+ )
+ const t0 = new Date('2026-09-23T10:00:00.000Z').getTime()
+ vi.useFakeTimers({ toFake: ['Date'] })
+ try {
+ vi.setSystemTime(t0)
+ const first = await reconciler.reconcile(
+ 'test-wf',
+ 'uid-123',
+ 'sandbox-recipes',
+ makeSpec({ mcpServers: ['redis-mcp'] })
+ )
+ expect(legacyDeletes()).toHaveLength(1)
+ expect(
+ (first.networkPolicyOwnershipConditions ?? []).filter(
+ c => c.type === 'WorkflowLegacyNetworkPolicyRemoved'
+ )
+ ).toMatchObject([{ status: 'False', reason: 'DeletePending' }])
- await reconciler.reconcile(
- 'test-wf',
- 'uid-123',
- 'sandbox-recipes',
- makeSpec({ mcpServers: ['redis-mcp'] })
- )
- await reconciler.reconcile(
- 'test-wf',
- 'uid-123',
- 'sandbox-recipes',
- makeSpec({ mcpServers: ['redis-mcp'] })
- )
+ vi.setSystemTime(t0 + 30_000)
+ const inside = await reconciler.reconcile(
+ 'test-wf',
+ 'uid-123',
+ 'sandbox-recipes',
+ makeSpec({ mcpServers: ['redis-mcp'] })
+ )
+ // Liveness witness: the pass reached the prune (the legacy fact is
+ // published), and inside the window it sent no DELETE.
+ expect(
+ (inside.networkPolicyOwnershipConditions ?? []).filter(
+ c => c.type === 'WorkflowLegacyNetworkPolicyRemoved'
+ )
+ ).toMatchObject([{ status: 'False', reason: 'DeletePending' }])
+ expect(legacyDeletes()).toHaveLength(1)
+ expect(reconciler.legacyMcpServersInternetEgressRetryDelayMs('test-wf', 'uid-123')).toBe(
+ 30_000
+ )
+
+ vi.setSystemTime(t0 + 60_000)
+ await reconciler.reconcile(
+ 'test-wf',
+ 'uid-123',
+ 'sandbox-recipes',
+ makeSpec({ mcpServers: ['redis-mcp'] })
+ )
+ expect(legacyDeletes()).toHaveLength(2)
+ // The second failure doubles the window.
+ expect(reconciler.legacyMcpServersInternetEgressRetryDelayMs('test-wf', 'uid-123')).toBe(
+ 120_000
+ )
+ } finally {
+ vi.useRealTimers()
+ }
+ })
- const legacyDeletes = networkingApi.deleteNamespacedNetworkPolicy.mock.calls.filter(
- ([arg]) =>
- arg.name === 'test-wf-mcp-servers-egress-internet' && arg.namespace === 'mcp-server'
+ it('R4-L4: doubles the legacy DELETE backoff up to 1 h, and a 404 ends it', async () => {
+ const LEGACY = 'test-wf-mcp-servers-egress-internet'
+ const networkingApi = makeNetworkingApi() as ReturnType<typeof makeNetworkingApi>
+ let legacyAnswer = { code: 403 }
+ networkingApi.deleteNamespacedNetworkPolicy.mockImplementation(async ({ name, namespace }) => {
+ if (name === LEGACY && namespace === 'mcp-server') throw legacyAnswer
+ return {}
+ })
+ const reconciler = new WorkflowReconciler(
+ makeDeps({
+ networkingApi: networkingApi as never,
+ })
)
- expect(legacyDeletes).toHaveLength(2)
+ const legacyDeletes = () =>
+ networkingApi.deleteNamespacedNetworkPolicy.mock.calls.filter(
+ ([arg]) => arg.name === LEGACY && arg.namespace === 'mcp-server'
+ )
+ vi.useFakeTimers({ toFake: ['Date'] })
+ try {
+ let now = new Date('2026-09-23T10:00:00.000Z').getTime()
+ const windows: number[] = []
+ for (let attempt = 0; attempt < 8; attempt++) {
+ vi.setSystemTime(now)
+ expect(
+ await reconciler.retryLegacyMcpServersInternetEgressDelete('test-wf', 'uid-123')
+ ).toBe('pending')
+ const delay = reconciler.legacyMcpServersInternetEgressRetryDelayMs('test-wf', 'uid-123')
+ windows.push(delay)
+ now += delay
+ }
+
+ expect(windows).toEqual([
+ 60_000, 120_000, 240_000, 480_000, 960_000, 1_920_000, 3_600_000, 3_600_000,
+ ])
+ expect(legacyDeletes()).toHaveLength(8)
+
+ legacyAnswer = { code: 404 }
+ vi.setSystemTime(now)
+ expect(await reconciler.retryLegacyMcpServersInternetEgressDelete('test-wf', 'uid-123')).toBe(
+ 'removed'
+ )
+ expect(reconciler.legacyMcpServersInternetEgressRetryDelayMs('test-wf', 'uid-123')).toBe(0)
+ expect(legacyDeletes()).toHaveLength(9)
+ // Observed gone: a later retry answers from the gone set, with no DELETE.
+ expect(await reconciler.retryLegacyMcpServersInternetEgressDelete('test-wf', 'uid-123')).toBe(
+ 'removed'
+ )
+ expect(legacyDeletes()).toHaveLength(9)
+ } finally {
+ vi.useRealTimers()
+ }
})
it('clears the legacy internet NP process set on recipe delete so a recreate can delete again', async () => {
@@ -5582,52 +5675,66 @@ describe('WorkflowReconciler — reconcile loop', () => {
})
// The legacy mcp-servers internet policy is deleted by name, apart
- // from the catalog prune: only reconcile() deletes it. A failed delete
- // is its own fact (R4-L3), DeletePending on its own condition, with no
- // retry flag and the catalog prune converged, and the next reconcile()
- // pass deletes it again.
+ // from the catalog prune: reconcile() deletes it, and the WRC's
+ // legacy-retry writer retries it (R4-L4). A failed delete is its own
+ // fact (R4-L3), DeletePending on its own condition, with no retry flag
+ // and the catalog prune converged, and the next reconcile() pass after
+ // the backoff window deletes it again.
const LEGACY = 'test-wf-mcp-servers-egress-internet'
const legacyDeletes = (apiserver: Converged) =>
apiserver.api.deleteNamespacedNetworkPolicy.mock.calls.filter(
([arg]) => arg.name === LEGACY
)
- it('reports a failed legacy internet policy delete from reconcile() as DeletePending, and deletes it on the next reconcile()', async () => {
- const apiserver = makeApiserverNetworkingApi()
- const reconciler = new WorkflowReconciler(
- makeDeps({ networkingApi: apiserver.api as never })
- )
- const liveDelete = apiserver.api.deleteNamespacedNetworkPolicy.getMockImplementation()!
- apiserver.api.deleteNamespacedNetworkPolicy.mockImplementation(
- async (arg: { name: string; namespace: string }) => {
- if (arg.name === LEGACY && legacyDeletes(apiserver).length === 1) {
- throw { code: 403, message: 'forbidden' }
+ it('reports a failed legacy internet policy delete from reconcile() as DeletePending, and deletes it on the next reconcile() after its backoff window', async () => {
+ const t0 = new Date('2026-09-23T10:00:00.000Z').getTime()
+ vi.useFakeTimers({ toFake: ['Date'] })
+ vi.setSystemTime(t0)
+ try {
+ const apiserver = makeApiserverNetworkingApi()
+ const reconciler = new WorkflowReconciler(
+ makeDeps({ networkingApi: apiserver.api as never })
+ )
+ const liveDelete = apiserver.api.deleteNamespacedNetworkPolicy.getMockImplementation()!
+ apiserver.api.deleteNamespacedNetworkPolicy.mockImplementation(
+ async (arg: { name: string; namespace: string }) => {
+ if (arg.name === LEGACY && legacyDeletes(apiserver).length === 1) {
+ throw { code: 403, message: 'forbidden' }
+ }
+ return liveDelete(arg)
}
- return liveDelete(arg)
- }
- )
+ )
- const failed = await reconciler.reconcile('test-wf', 'uid-123', 'sandbox-recipes', spec())
- expect(legacyDeletes(apiserver)).toHaveLength(1)
- expect(failed.networkPolicyRetryPending).toBeFalsy()
- const published = failed.networkPolicyOwnershipConditions ?? []
- expect(published.filter(c => c.type === 'WorkflowNetworkPoliciesConverged')).toEqual([])
- expect(
- published.filter(c => c.type === 'WorkflowLegacyNetworkPolicyRemoved')
- ).toMatchObject([{ status: 'False', reason: 'DeletePending' }])
+ const failed = await reconciler.reconcile(
+ 'test-wf',
+ 'uid-123',
+ 'sandbox-recipes',
+ spec()
+ )
+ expect(legacyDeletes(apiserver)).toHaveLength(1)
+ expect(failed.networkPolicyRetryPending).toBeFalsy()
+ const published = failed.networkPolicyOwnershipConditions ?? []
+ expect(published.filter(c => c.type === 'WorkflowNetworkPoliciesConverged')).toEqual([])
+ expect(
+ published.filter(c => c.type === 'WorkflowLegacyNetworkPolicyRemoved')
+ ).toMatchObject([{ status: 'False', reason: 'DeletePending' }])
- const pruned = await reconciler.reconcile(
- 'test-wf',
- 'uid-123',
- 'sandbox-recipes',
- spec(),
- {
- conditions: published,
- }
- )
- expect(legacyDeletes(apiserver)).toHaveLength(2)
- expect(pruned.networkPolicyRetryPending).toBeFalsy()
- expect(pruned.networkPolicyOwnershipConditions).toEqual([])
+ vi.setSystemTime(t0 + 60_000)
+ const pruned = await reconciler.reconcile(
+ 'test-wf',
+ 'uid-123',
+ 'sandbox-recipes',
+ spec(),
+ {
+ conditions: published,
+ }
+ )
+ expect(legacyDeletes(apiserver)).toHaveLength(2)
+ expect(pruned.networkPolicyRetryPending).toBeFalsy()
+ expect(pruned.networkPolicyOwnershipConditions).toEqual([])
+ } finally {
+ vi.useRealTimers()
+ }
})
it('sends no legacy internet policy DELETE from the retry, even one no pass has seen gone', async () => {
@@ -9451,6 +9558,57 @@ describe('networkPolicyMarkerConditions', () => {
})
})
+ describe('legacy-retry', () => {
+ it('R4-L4: clears the legacy fact once removed and carries the apply, the prune and the ownership condition', () => {
+ const existing = published({
+ conflicts: [conflict],
+ retryPending: true,
+ prune: 'unevaluated',
+ legacy: 'pending',
+ })
+ // Liveness witness: all three conditions were published.
+ expect(existing).toHaveLength(3)
+
+ const settled = networkPolicyMarkerConditions(
+ { kind: 'legacy-retry', legacy: 'removed' },
+ existing,
+ now
+ )
+
+ expect(reasons(settled)).toEqual([
+ 'WorkflowNetworkPolicyOwnership/OwnershipConflict',
+ 'WorkflowNetworkPoliciesConverged/RetryAndPruneUnevaluated',
+ ])
+ expect(settled.every(c => c.lastTransitionTime === earlier)).toBe(true)
+ })
+
+ it('R4-L4: keeps DeletePending and its transition time while the delete stays pending', () => {
+ const existing = published({ prune: 'pending', legacy: 'pending' })
+ expect(existing).toHaveLength(2)
+
+ expect(
+ networkPolicyMarkerConditions({ kind: 'legacy-retry', legacy: 'pending' }, existing, now)
+ ).toEqual(existing)
+ })
+
+ it('R4-L4: enters only for a published DeletePending', () => {
+ expect(hasLegacyNetworkPolicyDeletePendingMarker(published({ legacy: 'pending' }))).toBe(true)
+ expect(hasLegacyNetworkPolicyDeletePendingMarker(published({ prune: 'pending' }))).toBe(false)
+ expect(hasLegacyNetworkPolicyDeletePendingMarker(undefined)).toBe(false)
+ expect(
+ hasLegacyNetworkPolicyDeletePendingMarker([
+ {
+ type: 'WorkflowLegacyNetworkPolicyRemoved',
+ status: 'False',
+ reason: 'SomethingElse',
+ message: 'written by another head',
+ lastTransitionTime: earlier,
+ },
+ ])
+ ).toBe(false)
+ })
+ })
+
describe('terminal', () => {
it('R4-L3: drops a pending apply and carries the prune, the legacy fact and the ownership condition', () => {
const existing = published({