Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript

fix(wrc): requeue while a run-lane NetworkPolicy is being deleted

Alfredo Lopez6f854f18bb96
diff --git a/workflow-recipes/package-lock.json b/workflow-recipes/package-lock.json
index 09b8aeca9..3da8ec4dd 100644
--- a/workflow-recipes/package-lock.json
+++ b/workflow-recipes/package-lock.json
@@ -1,12 +1,12 @@
{
"name": "workload-recipes-controller",
- "version": "0.1.374",
+ "version": "0.1.375",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "workload-recipes-controller",
- "version": "0.1.374",
+ "version": "0.1.375",
"license": "SEE LICENSE IN LICENSE",
"dependencies": {
"@aws-sdk/client-s3": "^3.1009.0",
diff --git a/workflow-recipes/package.json b/workflow-recipes/package.json
index 5d5a68519..76e8ba257 100644
--- a/workflow-recipes/package.json
+++ b/workflow-recipes/package.json
@@ -1,6 +1,6 @@
{
"name": "workload-recipes-controller",
- "version": "0.1.374",
+ "version": "0.1.375",
"description": "Workload Recipes Controller (WRC) - WorkloadRecipe lifecycle management and MCP server interface",
"main": "dist/main.js",
"engines": {
diff --git a/workflow-recipes/src/reconciler/reconciler.test.ts b/workflow-recipes/src/reconciler/reconciler.test.ts
index a2c99d6fe..4d774c046 100644
--- a/workflow-recipes/src/reconciler/reconciler.test.ts
+++ b/workflow-recipes/src/reconciler/reconciler.test.ts
@@ -1549,6 +1549,39 @@ describe('WorkflowRecipeReconciler', () => {
expect(result.phase).toBe('failed')
expect(result.workflowConditions).toEqual([ownershipCondition])
})
+
+ // A policy being deleted is retried on the transient path. The twin with
+ // retryPending false shows that no other input of the requeue expression is
+ // set, so the first requeue can only come from the pending retry.
+ it('requeues on the transient path while a policy is being deleted', async () => {
+ const pending = stubSdkOnly({
+ phase: 'active',
+ message: 'Plugin Workload SDK mcp-host registered',
+ pluginWorkloadSdkBootstrapProof: bootstrapProof,
+ networkPolicies: { conflicts: [], retryPending: true },
+ })
+
+ const retrying = await reconciler.reconcile(sdkOnlyRecipe())
+
+ expect(pending).toHaveBeenCalledTimes(1)
+ expect(retrying.phase).toBe('active')
+ expect(retrying.requeueAfterMs).toBe(TRANSIENT_REQUEUE_BASE_MS)
+ expect(retrying.requeueFixedInterval).toBe(false)
+ expect(retrying.workflowConditions).toEqual([])
+
+ const settled = stubSdkOnly({
+ phase: 'active',
+ message: 'Plugin Workload SDK mcp-host registered',
+ pluginWorkloadSdkBootstrapProof: bootstrapProof,
+ networkPolicies: { conflicts: [], retryPending: false },
+ })
+
+ const steady = await reconciler.reconcile(sdkOnlyRecipe())
+
+ expect(settled).toHaveBeenCalledTimes(1)
+ expect(steady.phase).toBe('active')
+ expect(steady.requeueAfterMs).toBeUndefined()
+ })
})
// awaiting_policy waits for an operator grant. The grant arrives by event
@@ -9311,6 +9344,83 @@ describe('WorkflowRecipeReconciler', () => {
expect(result.requeueFixedInterval).toBeFalsy()
})
+ // A running workflow has no periodic pass of its own, and its status-only
+ // MODIFIED events are absorbed, so a policy being deleted needs a timer to be
+ // written once the deletion completes. It rides the progress interval, but
+ // with backoff: nothing bounds how long the deletion takes.
+ it('requeues an active workflow on the backoff path while a run-lane policy is being deleted', async () => {
+ const workflowReconcile = vi.fn().mockResolvedValue({
+ phase: 'active',
+ message: 'Workflow running',
+ workflowPhase: 'running',
+ networkPolicyRetryPending: true,
+ })
+ ;(
+ reconciler as unknown as {
+ workflowReconciler: {
+ reconcile: typeof workflowReconcile
+ validateWorkflowSpec: () => undefined
+ }
+ }
+ ).workflowReconciler = { reconcile: workflowReconcile, validateWorkflowSpec: () => undefined }
+
+ const recipe = makeRecipe({
+ spec: {
+ agent: { provider: 'zai', model: 'glm-4.7' },
+ steps: [{ id: 'research', instruction: 'run' }],
+ },
+ status: { phase: 'candidate' },
+ })
+
+ const result = await reconciler.reconcile(recipe)
+
+ expect(workflowReconcile).toHaveBeenCalledTimes(1)
+ expect(result.phase).toBe('active')
+ expect(result.requeueAfterMs).toBe(WORKFLOW_PROGRESS_REQUEUE_BASE_MS)
+ expect(result.requeueFixedInterval).toBe(false)
+ })
+
+ // An ownership conflict waits for an operator, so it publishes its condition
+ // and does not requeue.
+ it('does not requeue a run-lane ownership conflict but still publishes it', async () => {
+ const conflict = {
+ type: 'WorkflowNetworkPolicyOwnership',
+ status: 'False' as const,
+ reason: 'OwnershipConflict',
+ message:
+ 'NetworkPolicy ownership conflict: test-recipe-coord-to-wrc (owner-reference-mismatch)',
+ lastTransitionTime: '2026-09-23T10:00:00.000Z',
+ }
+ const workflowReconcile = vi.fn().mockResolvedValue({
+ phase: 'active',
+ message: 'Workflow running',
+ workflowPhase: 'running',
+ workflowConditions: [conflict],
+ })
+ ;(
+ reconciler as unknown as {
+ workflowReconciler: {
+ reconcile: typeof workflowReconcile
+ validateWorkflowSpec: () => undefined
+ }
+ }
+ ).workflowReconciler = { reconcile: workflowReconcile, validateWorkflowSpec: () => undefined }
+
+ const recipe = makeRecipe({
+ spec: {
+ agent: { provider: 'zai', model: 'glm-4.7' },
+ steps: [{ id: 'research', instruction: 'run' }],
+ },
+ status: { phase: 'candidate' },
+ })
+
+ const result = await reconciler.reconcile(recipe)
+
+ expect(workflowReconcile).toHaveBeenCalledTimes(1)
+ expect(result.workflowConditions).toEqual([conflict])
+ expect(result.requeueAfterMs).toBeUndefined()
+ })
+
// §5 priority: a transient ERROR (skipStatusPatch) wins over PROGRESS even when
// the phase is `deploying`. The error path keeps the TRANSIENT base + backoff
// (requeueFixedInterval false), so genuine flakiness still backs off; it must
diff --git a/workflow-recipes/src/reconciler/workflowRecipeReconciler.ts b/workflow-recipes/src/reconciler/workflowRecipeReconciler.ts
index 193160302..76161d6b7 100644
--- a/workflow-recipes/src/reconciler/workflowRecipeReconciler.ts
+++ b/workflow-recipes/src/reconciler/workflowRecipeReconciler.ts
@@ -2269,9 +2269,16 @@ export class WorkflowRecipeReconciler {
// path: it waits for an operator grant, which arrives by event or by the
// 30s credential-refresh floor. Polling cannot advance it, and it has no
// deadline that bounds a fixed-interval loop.
+ //
+ // A NetworkPolicy left unwritten because it is being deleted requeues on
+ // the backoff path too: a running workflow has no periodic pass, and
+ // nothing bounds how long the deletion takes. An ownership conflict does
+ // not requeue; it waits for an operator and is published as a condition.
requeueAfterMs: result.skipStatusPatch
? TRANSIENT_REQUEUE_BASE_MS
- : result.phase === 'deploying' || result.pluginWorkloadSdkPolicyPending
+ : result.phase === 'deploying' ||
+ result.pluginWorkloadSdkPolicyPending ||
+ result.networkPolicyRetryPending
? WORKFLOW_PROGRESS_REQUEUE_BASE_MS
: undefined,
requeueFixedInterval: !result.skipStatusPatch && result.phase === 'deploying',
@@ -2950,13 +2957,16 @@ export class WorkflowRecipeReconciler {
pluginWorkloadSdkBootstrapProof: sdkOnlyRuntime?.pluginWorkloadSdkBootstrapProof,
// Issue #637 — requeue if a denied workload's teardown failed (deniedTeardownFailed),
// so the revocation is retried rather than left to the next event.
+ // A NetworkPolicy left unwritten because it is being deleted is retried
+ // the same way; an ownership conflict is not.
requeueAfterMs:
legacyRawCleanupPending ||
deniedTeardownFailed ||
sdkOnlyRuntime?.phase === 'deploying' ||
sdkOnlyRuntime?.phase === 'provider_unavailable' ||
sdkOnlyPolicyPending ||
- sdkOnlyBootstrapPending
+ sdkOnlyBootstrapPending ||
+ sdkOnlyRuntime?.networkPolicies.retryPending
? TRANSIENT_REQUEUE_BASE_MS
: undefined,
// Policy-pending waits for an operator grant (event or the 30s refresh
diff --git a/workflow-recipes/src/workflow/workflowReconciler.ts b/workflow-recipes/src/workflow/workflowReconciler.ts
index 1ce078758..d4874af83 100644
--- a/workflow-recipes/src/workflow/workflowReconciler.ts
+++ b/workflow-recipes/src/workflow/workflowReconciler.ts
@@ -791,6 +791,11 @@ export interface WorkflowReconcileResult {
pluginWorkloadSdkBootstrapProof?: EagerSdkBootstrapProof
/** Eager host identity is ready, but prompt policy awaits operator action. */
pluginWorkloadSdkPolicyPending?: boolean
+ /**
+ * A NetworkPolicy this pass had to leave unwritten because it is being
+ * deleted. Nothing else re-runs a steady pass, so the caller requeues.
+ */
+ networkPolicyRetryPending?: boolean
}
function mcpHostReadinessMessage(readiness: PodReadiness): string {
@@ -1623,9 +1628,11 @@ export class WorkflowReconciler {
let networkPolicyOwnershipConditions = carriedNetworkPolicyOwnershipConditions(
currentStatus?.conditions
)
+ let networkPolicyRetryPending = false
const withWorkflowConditions = (result: WorkflowReconcileResult): WorkflowReconcileResult => ({
...result,
workflowConditions: [...workflowConditions, ...networkPolicyOwnershipConditions],
+ ...(networkPolicyRetryPending ? { networkPolicyRetryPending } : {}),
})
const outputAnchorPodName = runtime.output.anchorRequired
? buildWorkflowOutputAnchorPodName(runtimeScopeRecipeName)
@@ -1774,6 +1781,7 @@ export class WorkflowReconciler {
new Date().toISOString(),
currentStatus?.conditions
)
+ networkPolicyRetryPending = eagerNetworkPolicies.retryPending
}
if (eagerStatus === 'failed') {
return withWorkflowConditions({
@@ -1794,6 +1802,7 @@ export class WorkflowReconciler {
phase: 'active',
message: providerUnavailableMessage,
clearWorkflowExecution: true,
+ ...(networkPolicyRetryPending ? { networkPolicyRetryPending } : {}),
workflowConditions: [
...workflowConditions,
...networkPolicyOwnershipConditions,
@@ -2239,6 +2248,7 @@ export class WorkflowReconciler {
new Date().toISOString(),
currentStatus?.conditions
)
+ networkPolicyRetryPending = runLaneNetworkPolicies.retryPending
// 6. Create Pods — mcp-host FIRST, then coordinator. If the coordinator
// resolves DNS before mcp-host's EndpointSlice exists, undici caches the
diff --git a/workflow-recipes/tests/unit/workflow/workflowReconciler.test.ts b/workflow-recipes/tests/unit/workflow/workflowReconciler.test.ts
index 3fb69ed47..47ce85064 100644
--- a/workflow-recipes/tests/unit/workflow/workflowReconciler.test.ts
+++ b/workflow-recipes/tests/unit/workflow/workflowReconciler.test.ts
@@ -3050,6 +3050,48 @@ describe('WorkflowReconciler — reconcile loop', () => {
([arg]) => arg.name === 'test-wf-coord-to-wrc'
)
).toHaveLength(0)
+ // A conflict is not transient: it waits for an operator, not a timer.
+ expect(second.networkPolicyRetryPending).toBeFalsy()
+ })
+
+ it('flags a retry for a run-lane policy that is being deleted and leaves it unwritten', async () => {
+ const { api, live, key } = makeApiserverNetworkingApi()
+ const reconciler = new WorkflowReconciler(makeDeps({ networkingApi: api as never }))
+ const spec = makeSpec({ agent: undefined, steps: [{ id: 'prepare', run: snippetRun() }] })
+
+ const first = await reconciler.reconcile('test-wf', 'uid-123', 'sandbox-recipes', spec)
+ expect(first.workflowPhase).not.toBe('failed')
+ expect(first.networkPolicyRetryPending).toBeFalsy()
+ const terminating = live.get(key('sandbox-recipes', 'test-wf-coord-to-wrc'))
+ expect(terminating).toBeDefined()
+ // Drift the spec too, so that without the deletion check the pass would PUT.
+ terminating!.spec = {
+ ...terminating!.spec!,
+ podSelector: { matchLabels: { drifted: 'yes' } },
+ }
+ terminating!.metadata = {
+ ...terminating!.metadata,
+ deletionTimestamp: new Date('2026-09-23T10:00:00Z'),
+ }
+ api.readNamespacedNetworkPolicy.mockClear()
+ api.replaceNamespacedNetworkPolicy.mockClear()
+
+ const second = await reconciler.reconcile('test-wf', 'uid-123', 'sandbox-recipes', spec)
+
+ expect(readPolicyNames(api)).toContain('test-wf-coord-to-wrc')
+ expect(second.networkPolicyRetryPending).toBe(true)
+ expect(second.phase).not.toBe('failed')
+ expect(second.workflowPhase).not.toBe('failed')
+ expect(
+ second.workflowConditions?.find(
+ condition => condition.type === 'WorkflowNetworkPolicyOwnership'
+ )
+ ).toBeUndefined()
+ expect(
+ api.replaceNamespacedNetworkPolicy.mock.calls.filter(
+ ([arg]) => arg.name === 'test-wf-coord-to-wrc'
+ )
+ ).toHaveLength(0)
})
})