Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
fix(wrc): requeue while a run-lane NetworkPolicy is being deleted
Alfredo Lopez6f854f18bb96
diff --git a/workflow-recipes/package-lock.json b/workflow-recipes/package-lock.jsonindex 09b8aeca9..3da8ec4dd 100644--- a/workflow-recipes/package-lock.json+++ b/workflow-recipes/package-lock.json@@ -1,12 +1,12 @@{"name": "workload-recipes-controller",- "version": "0.1.374",+ "version": "0.1.375","lockfileVersion": 3,"requires": true,"packages": {"": {"name": "workload-recipes-controller",- "version": "0.1.374",+ "version": "0.1.375","license": "SEE LICENSE IN LICENSE","dependencies": {"@aws-sdk/client-s3": "^3.1009.0",diff --git a/workflow-recipes/package.json b/workflow-recipes/package.jsonindex 5d5a68519..76e8ba257 100644--- a/workflow-recipes/package.json+++ b/workflow-recipes/package.json@@ -1,6 +1,6 @@{"name": "workload-recipes-controller",- "version": "0.1.374",+ "version": "0.1.375","description": "Workload Recipes Controller (WRC) - WorkloadRecipe lifecycle management and MCP server interface","main": "dist/main.js","engines": {diff --git a/workflow-recipes/src/reconciler/reconciler.test.ts b/workflow-recipes/src/reconciler/reconciler.test.tsindex a2c99d6fe..4d774c046 100644--- a/workflow-recipes/src/reconciler/reconciler.test.ts+++ b/workflow-recipes/src/reconciler/reconciler.test.ts@@ -1549,6 +1549,39 @@ describe('WorkflowRecipeReconciler', () => {expect(result.phase).toBe('failed')expect(result.workflowConditions).toEqual([ownershipCondition])})++ // A policy being deleted is retried on the transient path. The twin with+ // retryPending false shows that no other input of the requeue expression is+ // set, so the first requeue can only come from the pending retry.+ it('requeues on the transient path while a policy is being deleted', async () => {+ const pending = stubSdkOnly({+ phase: 'active',+ message: 'Plugin Workload SDK mcp-host registered',+ pluginWorkloadSdkBootstrapProof: bootstrapProof,+ networkPolicies: { conflicts: [], retryPending: true },+ })++ const retrying = await reconciler.reconcile(sdkOnlyRecipe())++ expect(pending).toHaveBeenCalledTimes(1)+ expect(retrying.phase).toBe('active')+ expect(retrying.requeueAfterMs).toBe(TRANSIENT_REQUEUE_BASE_MS)+ expect(retrying.requeueFixedInterval).toBe(false)+ expect(retrying.workflowConditions).toEqual([])++ const settled = stubSdkOnly({+ phase: 'active',+ message: 'Plugin Workload SDK mcp-host registered',+ pluginWorkloadSdkBootstrapProof: bootstrapProof,+ networkPolicies: { conflicts: [], retryPending: false },+ })++ const steady = await reconciler.reconcile(sdkOnlyRecipe())++ expect(settled).toHaveBeenCalledTimes(1)+ expect(steady.phase).toBe('active')+ expect(steady.requeueAfterMs).toBeUndefined()+ })})// awaiting_policy waits for an operator grant. The grant arrives by event@@ -9311,6 +9344,83 @@ describe('WorkflowRecipeReconciler', () => {expect(result.requeueFixedInterval).toBeFalsy()})+ // A running workflow has no periodic pass of its own, and its status-only+ // MODIFIED events are absorbed, so a policy being deleted needs a timer to be+ // written once the deletion completes. It rides the progress interval, but+ // with backoff: nothing bounds how long the deletion takes.+ it('requeues an active workflow on the backoff path while a run-lane policy is being deleted', async () => {+ const workflowReconcile = vi.fn().mockResolvedValue({+ phase: 'active',+ message: 'Workflow running',+ workflowPhase: 'running',+ networkPolicyRetryPending: true,+ })+ ;(+ reconciler as unknown as {+ workflowReconciler: {+ reconcile: typeof workflowReconcile+ validateWorkflowSpec: () => undefined+ }+ }+ ).workflowReconciler = { reconcile: workflowReconcile, validateWorkflowSpec: () => undefined }++ const recipe = makeRecipe({+ spec: {+ agent: { provider: 'zai', model: 'glm-4.7' },+ steps: [{ id: 'research', instruction: 'run' }],+ },+ status: { phase: 'candidate' },+ })++ const result = await reconciler.reconcile(recipe)++ expect(workflowReconcile).toHaveBeenCalledTimes(1)+ expect(result.phase).toBe('active')+ expect(result.requeueAfterMs).toBe(WORKFLOW_PROGRESS_REQUEUE_BASE_MS)+ expect(result.requeueFixedInterval).toBe(false)+ })++ // An ownership conflict waits for an operator, so it publishes its condition+ // and does not requeue.+ it('does not requeue a run-lane ownership conflict but still publishes it', async () => {+ const conflict = {+ type: 'WorkflowNetworkPolicyOwnership',+ status: 'False' as const,+ reason: 'OwnershipConflict',+ message:+ 'NetworkPolicy ownership conflict: test-recipe-coord-to-wrc (owner-reference-mismatch)',+ lastTransitionTime: '2026-09-23T10:00:00.000Z',+ }+ const workflowReconcile = vi.fn().mockResolvedValue({+ phase: 'active',+ message: 'Workflow running',+ workflowPhase: 'running',+ workflowConditions: [conflict],+ })+ ;(+ reconciler as unknown as {+ workflowReconciler: {+ reconcile: typeof workflowReconcile+ validateWorkflowSpec: () => undefined+ }+ }+ ).workflowReconciler = { reconcile: workflowReconcile, validateWorkflowSpec: () => undefined }++ const recipe = makeRecipe({+ spec: {+ agent: { provider: 'zai', model: 'glm-4.7' },+ steps: [{ id: 'research', instruction: 'run' }],+ },+ status: { phase: 'candidate' },+ })++ const result = await reconciler.reconcile(recipe)++ expect(workflowReconcile).toHaveBeenCalledTimes(1)+ expect(result.workflowConditions).toEqual([conflict])+ expect(result.requeueAfterMs).toBeUndefined()+ })+// §5 priority: a transient ERROR (skipStatusPatch) wins over PROGRESS even when// the phase is `deploying`. The error path keeps the TRANSIENT base + backoff// (requeueFixedInterval false), so genuine flakiness still backs off; it mustdiff --git a/workflow-recipes/src/reconciler/workflowRecipeReconciler.ts b/workflow-recipes/src/reconciler/workflowRecipeReconciler.tsindex 193160302..76161d6b7 100644--- a/workflow-recipes/src/reconciler/workflowRecipeReconciler.ts+++ b/workflow-recipes/src/reconciler/workflowRecipeReconciler.ts@@ -2269,9 +2269,16 @@ export class WorkflowRecipeReconciler {// path: it waits for an operator grant, which arrives by event or by the// 30s credential-refresh floor. Polling cannot advance it, and it has no// deadline that bounds a fixed-interval loop.+ //+ // A NetworkPolicy left unwritten because it is being deleted requeues on+ // the backoff path too: a running workflow has no periodic pass, and+ // nothing bounds how long the deletion takes. An ownership conflict does+ // not requeue; it waits for an operator and is published as a condition.requeueAfterMs: result.skipStatusPatch? TRANSIENT_REQUEUE_BASE_MS- : result.phase === 'deploying' || result.pluginWorkloadSdkPolicyPending+ : result.phase === 'deploying' ||+ result.pluginWorkloadSdkPolicyPending ||+ result.networkPolicyRetryPending? WORKFLOW_PROGRESS_REQUEUE_BASE_MS: undefined,requeueFixedInterval: !result.skipStatusPatch && result.phase === 'deploying',@@ -2950,13 +2957,16 @@ export class WorkflowRecipeReconciler {pluginWorkloadSdkBootstrapProof: sdkOnlyRuntime?.pluginWorkloadSdkBootstrapProof,// Issue #637 — requeue if a denied workload's teardown failed (deniedTeardownFailed),// so the revocation is retried rather than left to the next event.+ // A NetworkPolicy left unwritten because it is being deleted is retried+ // the same way; an ownership conflict is not.requeueAfterMs:legacyRawCleanupPending ||deniedTeardownFailed ||sdkOnlyRuntime?.phase === 'deploying' ||sdkOnlyRuntime?.phase === 'provider_unavailable' ||sdkOnlyPolicyPending ||- sdkOnlyBootstrapPending+ sdkOnlyBootstrapPending ||+ sdkOnlyRuntime?.networkPolicies.retryPending? TRANSIENT_REQUEUE_BASE_MS: undefined,// Policy-pending waits for an operator grant (event or the 30s refreshdiff --git a/workflow-recipes/src/workflow/workflowReconciler.ts b/workflow-recipes/src/workflow/workflowReconciler.tsindex 1ce078758..d4874af83 100644--- a/workflow-recipes/src/workflow/workflowReconciler.ts+++ b/workflow-recipes/src/workflow/workflowReconciler.ts@@ -791,6 +791,11 @@ export interface WorkflowReconcileResult {pluginWorkloadSdkBootstrapProof?: EagerSdkBootstrapProof/** Eager host identity is ready, but prompt policy awaits operator action. */pluginWorkloadSdkPolicyPending?: boolean+ /**+ * A NetworkPolicy this pass had to leave unwritten because it is being+ * deleted. Nothing else re-runs a steady pass, so the caller requeues.+ */+ networkPolicyRetryPending?: boolean}function mcpHostReadinessMessage(readiness: PodReadiness): string {@@ -1623,9 +1628,11 @@ export class WorkflowReconciler {let networkPolicyOwnershipConditions = carriedNetworkPolicyOwnershipConditions(currentStatus?.conditions)+ let networkPolicyRetryPending = falseconst withWorkflowConditions = (result: WorkflowReconcileResult): WorkflowReconcileResult => ({...result,workflowConditions: [...workflowConditions, ...networkPolicyOwnershipConditions],+ ...(networkPolicyRetryPending ? { networkPolicyRetryPending } : {}),})const outputAnchorPodName = runtime.output.anchorRequired? buildWorkflowOutputAnchorPodName(runtimeScopeRecipeName)@@ -1774,6 +1781,7 @@ export class WorkflowReconciler {new Date().toISOString(),currentStatus?.conditions)+ networkPolicyRetryPending = eagerNetworkPolicies.retryPending}if (eagerStatus === 'failed') {return withWorkflowConditions({@@ -1794,6 +1802,7 @@ export class WorkflowReconciler {phase: 'active',message: providerUnavailableMessage,clearWorkflowExecution: true,+ ...(networkPolicyRetryPending ? { networkPolicyRetryPending } : {}),workflowConditions: [...workflowConditions,...networkPolicyOwnershipConditions,@@ -2239,6 +2248,7 @@ export class WorkflowReconciler {new Date().toISOString(),currentStatus?.conditions)+ networkPolicyRetryPending = runLaneNetworkPolicies.retryPending// 6. Create Pods — mcp-host FIRST, then coordinator. If the coordinator// resolves DNS before mcp-host's EndpointSlice exists, undici caches thediff --git a/workflow-recipes/tests/unit/workflow/workflowReconciler.test.ts b/workflow-recipes/tests/unit/workflow/workflowReconciler.test.tsindex 3fb69ed47..47ce85064 100644--- a/workflow-recipes/tests/unit/workflow/workflowReconciler.test.ts+++ b/workflow-recipes/tests/unit/workflow/workflowReconciler.test.ts@@ -3050,6 +3050,48 @@ describe('WorkflowReconciler — reconcile loop', () => {([arg]) => arg.name === 'test-wf-coord-to-wrc')).toHaveLength(0)+ // A conflict is not transient: it waits for an operator, not a timer.+ expect(second.networkPolicyRetryPending).toBeFalsy()+ })++ it('flags a retry for a run-lane policy that is being deleted and leaves it unwritten', async () => {+ const { api, live, key } = makeApiserverNetworkingApi()+ const reconciler = new WorkflowReconciler(makeDeps({ networkingApi: api as never }))+ const spec = makeSpec({ agent: undefined, steps: [{ id: 'prepare', run: snippetRun() }] })++ const first = await reconciler.reconcile('test-wf', 'uid-123', 'sandbox-recipes', spec)+ expect(first.workflowPhase).not.toBe('failed')+ expect(first.networkPolicyRetryPending).toBeFalsy()+ const terminating = live.get(key('sandbox-recipes', 'test-wf-coord-to-wrc'))+ expect(terminating).toBeDefined()+ // Drift the spec too, so that without the deletion check the pass would PUT.+ terminating!.spec = {+ ...terminating!.spec!,+ podSelector: { matchLabels: { drifted: 'yes' } },+ }+ terminating!.metadata = {+ ...terminating!.metadata,+ deletionTimestamp: new Date('2026-09-23T10:00:00Z'),+ }+ api.readNamespacedNetworkPolicy.mockClear()+ api.replaceNamespacedNetworkPolicy.mockClear()++ const second = await reconciler.reconcile('test-wf', 'uid-123', 'sandbox-recipes', spec)++ expect(readPolicyNames(api)).toContain('test-wf-coord-to-wrc')+ expect(second.networkPolicyRetryPending).toBe(true)+ expect(second.phase).not.toBe('failed')+ expect(second.workflowPhase).not.toBe('failed')+ expect(+ second.workflowConditions?.find(+ condition => condition.type === 'WorkflowNetworkPolicyOwnership'+ )+ ).toBeUndefined()+ expect(+ api.replaceNamespacedNetworkPolicy.mock.calls.filter(+ ([arg]) => arg.name === 'test-wf-coord-to-wrc'+ )+ ).toHaveLength(0)})})