Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript

test(workflow-recipes): retry the legacy delete before the short-circuits (R4-L4)

Alfredo Lopez530f75ce1472
diff --git a/workflow-recipes/package-lock.json b/workflow-recipes/package-lock.json
index 78f6e0d43..7986c86dc 100644
--- a/workflow-recipes/package-lock.json
+++ b/workflow-recipes/package-lock.json
@@ -1,12 +1,12 @@
{
"name": "workload-recipes-controller",
- "version": "0.1.443",
+ "version": "0.1.444",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "workload-recipes-controller",
- "version": "0.1.443",
+ "version": "0.1.444",
"license": "SEE LICENSE IN LICENSE",
"dependencies": {
"@aws-sdk/client-s3": "^3.1009.0",
diff --git a/workflow-recipes/package.json b/workflow-recipes/package.json
index 549c90249..ffaed2bab 100644
--- a/workflow-recipes/package.json
+++ b/workflow-recipes/package.json
@@ -1,6 +1,6 @@
{
"name": "workload-recipes-controller",
- "version": "0.1.443",
+ "version": "0.1.444",
"description": "Workload Recipes Controller (WRC) - WorkloadRecipe lifecycle management and MCP server interface",
"main": "dist/main.js",
"engines": {
diff --git a/workflow-recipes/src/reconciler/reconciler.test.ts b/workflow-recipes/src/reconciler/reconciler.test.ts
index a822ff9c2..7b19eed95 100644
--- a/workflow-recipes/src/reconciler/reconciler.test.ts
+++ b/workflow-recipes/src/reconciler/reconciler.test.ts
@@ -16463,6 +16463,198 @@ describe('WorkflowRecipeReconciler', () => {
})
})
+ // R4-L4: a legacy mcp-servers internet policy DELETE that did not land is
+ // published as WorkflowLegacyNetworkPolicyRemoved=False/DeletePending. The
+ // in-progress, active and terminal short-circuits never reach reconcile(),
+ // so the legacy-retry writer runs before them: one DELETE per backoff window
+ // (60 s, doubling up to 1 h), a fixed requeue for the rest of the window,
+ // and the marker cleared once the policy is gone.
+ describe('R4-L4: legacy internet policy DELETE retried before the short-circuits', () => {
+ const RECIPE = 'lg-recipe'
+ const RUN_ID = 'run-lg'
+ const LEGACY = `${RECIPE}-mcp-servers-egress-internet`
+ const LEGACY_TYPE = 'WorkflowLegacyNetworkPolicyRemoved'
+ const MARKER_TYPE = 'WorkflowNetworkPoliciesConverged'
+ const T0 = new Date('2026-09-23T10:00:00.000Z').getTime()
+
+ beforeEach(() => {
+ vi.useFakeTimers({ toFake: ['Date'] })
+ vi.setSystemTime(T0)
+ })
+
+ afterEach(() => {
+ vi.useRealTimers()
+ })
+
+ function runScopedRecipe(status: Record<string, unknown>): WorkflowRecipeCRD {
+ return makeRecipe({
+ metadata: {
+ name: RECIPE,
+ namespace: 'sandbox-recipes',
+ uid: 'uid-lg',
+ labels: { 'clerum.io/workflow-run-id': RUN_ID },
+ },
+ spec: {
+ agent: { provider: 'openai', model: 'gpt-4o' },
+ steps: [{ id: 'research', instruction: 'run' }],
+ },
+ status: status as WorkflowRecipeCRD['status'],
+ })
+ }
+
+ function publishedFacts(prune: 'converged' | 'pending'): StatusCondition[] {
+ return networkPolicyMarkerConditions(
+ {
+ kind: 'reconcile',
+ summary: { conflicts: [], retryPending: false, prune, legacy: 'pending' },
+ },
+ [],
+ '2026-09-23T09:00:00.000Z'
+ )
+ }
+
+ function ofType(conditions: StatusCondition[] | undefined, type: string): StatusCondition[] {
+ return (conditions ?? []).filter(c => c.type === type)
+ }
+
+ function legacyDeletes(): number {
+ return mockNetworkingApi.deleteNamespacedNetworkPolicy.mock.calls.filter(
+ ([arg]) => (arg as { name: string }).name === LEGACY
+ ).length
+ }
+
+ function legacyDeleteAnswers(error: { code: number; message: string }) {
+ mockNetworkingApi.deleteNamespacedNetworkPolicy.mockImplementation(
+ async ({ name }: { name: string }) => {
+ if (name === LEGACY) throw error
+ return {}
+ }
+ )
+ }
+
+ function stubShortCircuits(inner: WorkflowReconciler) {
+ const ensureCredentials = vi
+ .spyOn(inner, 'ensureMcpHostRuntimeCredentials')
+ .mockResolvedValue(undefined)
+ vi.spyOn(inner, 'refreshRuntimeHttpEgressNetworkPolicies').mockResolvedValue({
+ conflicts: [],
+ retryPending: false,
+ })
+ const teardown = vi.spyOn(inner, 'teardownComputePodsForTerminalRun').mockResolvedValue()
+ const innerReconcile = vi.spyOn(inner, 'reconcile')
+ return { ensureCredentials, teardown, innerReconcile }
+ }
+
+ it('R4-L4: sends one legacy DELETE per backoff window from the in-progress, active and terminal short-circuits', async () => {
+ const inner = installRealInner()
+ const { ensureCredentials, teardown, innerReconcile } = stubShortCircuits(inner)
+ legacyDeleteAnswers({ code: 403, message: 'forbidden' })
+ const conditions = publishedFacts('converged')
+ // Liveness witness: the fixture publishes the legacy fact alone.
+ expect(conditions).toMatchObject([
+ { type: LEGACY_TYPE, status: 'False', reason: 'DeletePending' },
+ ])
+
+ const inProgress = runScopedRecipe({
+ phase: 'deploying',
+ message: 'Workflow running',
+ workflowExecution: { phase: 'running' },
+ conditions,
+ })
+ const first = await reconciler.reconcile(inProgress)
+
+ // Witness: the in-progress short-circuit ran.
+ expect(ensureCredentials).toHaveBeenCalled()
+ expect(legacyDeletes()).toBe(1)
+ expect(first.requeueAfterMs).toBe(60_000)
+ expect(first.requeueFixedInterval).toBe(true)
+ expect(ofType(inProgress.status?.conditions, LEGACY_TYPE)).toMatchObject([
+ { status: 'False', reason: 'DeletePending' },
+ ])
+
+ vi.setSystemTime(T0 + 20_000)
+ const active = runScopedRecipe({
+ phase: 'active',
+ message: 'Workflow running',
+ workflowExecution: { phase: 'running' },
+ conditions,
+ })
+ const second = await reconciler.reconcile(active)
+
+ // Inside the window: no DELETE, and a requeue for the 40 s left.
+ expect(legacyDeletes()).toBe(1)
+ expect(second.requeueAfterMs).toBe(40_000)
+ expect(second.requeueFixedInterval).toBe(true)
+
+ vi.setSystemTime(T0 + 61_000)
+ const terminal = runScopedRecipe({
+ phase: 'active',
+ message: 'Workflow completed',
+ workflowExecution: { phase: 'completed' },
+ conditions,
+ })
+ const third = await reconciler.reconcile(terminal)
+
+ // Witness: the terminal branch tore the run's compute down.
+ expect(teardown).toHaveBeenCalledWith(RECIPE)
+ // Past the window: one more DELETE, and the window doubles.
+ expect(legacyDeletes()).toBe(2)
+ expect(third.requeueAfterMs).toBe(120_000)
+ expect(third.requeueFixedInterval).toBe(true)
+ expect(ofType(terminal.status?.conditions, LEGACY_TYPE)).toMatchObject([
+ { status: 'False', reason: 'DeletePending' },
+ ])
+ expect(innerReconcile).not.toHaveBeenCalled()
+ })
+
+ it('R4-L4: clears DeletePending from the active short-circuit once the legacy DELETE answers 404, and carries the prune fact', async () => {
+ const inner = installRealInner()
+ const { ensureCredentials } = stubShortCircuits(inner)
+ legacyDeleteAnswers({ code: 404, message: 'not found' })
+ const conditions = publishedFacts('pending')
+ expect(ofType(conditions, MARKER_TYPE)).toMatchObject([{ reason: 'PrunePending' }])
+ mockCustomApi.patchNamespacedCustomObjectStatus.mockClear()
+ const active = runScopedRecipe({
+ phase: 'active',
+ message: 'Workflow running',
+ workflowExecution: { phase: 'running' },
+ conditions,
+ })
+
+ const result = await reconciler.reconcile(active)
+
+ // Witness: the active short-circuit ran and sent the DELETE.
+ expect(ensureCredentials).toHaveBeenCalled()
+ expect(legacyDeletes()).toBe(1)
+ expect(ofType(active.status?.conditions, LEGACY_TYPE)).toEqual([])
+ expect(ofType(active.status?.conditions, MARKER_TYPE)).toEqual(
+ ofType(conditions, MARKER_TYPE)
+ )
+ expect(mockCustomApi.patchNamespacedCustomObjectStatus).toHaveBeenCalledTimes(1)
+ expect(result.requeueAfterMs).toBeUndefined()
+ })
+
+ it('R4-L4: sends no legacy DELETE from a short-circuit when no DeletePending marker is published', async () => {
+ const inner = installRealInner()
+ const { ensureCredentials } = stubShortCircuits(inner)
+ legacyDeleteAnswers({ code: 403, message: 'forbidden' })
+ const active = runScopedRecipe({
+ phase: 'active',
+ message: 'Workflow running',
+ workflowExecution: { phase: 'running' },
+ conditions: [],
+ })
+
+ const result = await reconciler.reconcile(active)
+
+ // Witness: the active short-circuit ran.
+ expect(ensureCredentials).toHaveBeenCalled()
+ expect(result.message).toBe('Workflow running')
+ expect(legacyDeletes()).toBe(0)
+ expect(result.requeueAfterMs).toBeUndefined()
+ })
+ })
+
// A reconcile() prune whose DELETE did not land publishes
// WorkflowNetworkPoliciesConverged=False/PrunePending. That fact is not a
// pending retry: the mid-run retry prunes nothing, so the short-circuits