Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
fix(workflow-recipes): stop the mid-run NetworkPolicy retry from pruning live lanes
Alfredo Lopez9ad28c10e124
diff --git a/workflow-recipes/package-lock.json b/workflow-recipes/package-lock.jsonindex 80033b5e9..9504adcc3 100644--- a/workflow-recipes/package-lock.json+++ b/workflow-recipes/package-lock.json@@ -1,12 +1,12 @@{"name": "workload-recipes-controller",- "version": "0.1.419",+ "version": "0.1.420","lockfileVersion": 3,"requires": true,"packages": {"": {"name": "workload-recipes-controller",- "version": "0.1.419",+ "version": "0.1.420","license": "SEE LICENSE IN LICENSE","dependencies": {"@aws-sdk/client-s3": "^3.1009.0",diff --git a/workflow-recipes/package.json b/workflow-recipes/package.jsonindex fe094a437..7cd8721da 100644--- a/workflow-recipes/package.json+++ b/workflow-recipes/package.json@@ -1,6 +1,6 @@{"name": "workload-recipes-controller",- "version": "0.1.419",+ "version": "0.1.420","description": "Workload Recipes Controller (WRC) - WorkloadRecipe lifecycle management and MCP server interface","main": "dist/main.js","engines": {diff --git a/workflow-recipes/src/reconciler/reconciler.test.ts b/workflow-recipes/src/reconciler/reconciler.test.tsindex e9801d2cc..d4f78f909 100644--- a/workflow-recipes/src/reconciler/reconciler.test.ts+++ b/workflow-recipes/src/reconciler/reconciler.test.ts@@ -16129,20 +16129,26 @@ describe('WorkflowRecipeReconciler', () => {'run-g2')- // Liveness witness: the retry ran the prune and deleted an undesired- // catalog sibling, so the absence of a GFS DELETE below is the catalog- // gate holding, not a retry that never reached the prune.- expect(mockNetworkingApi.listNamespacedNetworkPolicy).toHaveBeenCalled()- expect(mockNetworkingApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledWith(- expect.objectContaining({ name: `${RECIPE}-snippet-runner-egress` })- )- expect(mockNetworkingApi.deleteNamespacedNetworkPolicy).not.toHaveBeenCalledWith(- expect.objectContaining({ name: GFS })- )- // The retry runs mid-run, so it never revokes a Codex/Grok proxy.- expect(mockNetworkingApi.deleteNamespacedNetworkPolicy).not.toHaveBeenCalledWith(- expect.objectContaining({ name: `${RECIPE}-mcp-host-to-grok-proxy` })- )+ // Liveness witness: the retry built this agent spec and applied a lane+ // it wants, so the absent DELETEs below are the retry declining to+ // prune mid-run, not a retry that never ran.+ expect(+ mockNetworkingApi.createNamespacedNetworkPolicy.mock.calls.some(+ ([arg]) => arg.body?.metadata?.name === `${RECIPE}-coord-to-mcp-host`+ )+ ).toBe(true)+ // The run's pods are live, so the retry revokes no lane: not the+ // reserved GFS policy, not a sibling the spec no longer wants, and not+ // a Codex/Grok proxy.+ for (const name of [+ GFS,+ `${RECIPE}-snippet-runner-egress`,+ `${RECIPE}-mcp-host-to-grok-proxy`,+ ]) {+ expect(mockNetworkingApi.deleteNamespacedNetworkPolicy).not.toHaveBeenCalledWith(+ expect.objectContaining({ name })+ )+ }})})diff --git a/workflow-recipes/src/workflow/workflowReconciler.ts b/workflow-recipes/src/workflow/workflowReconciler.tsindex 3a92ce2d9..74995487d 100644--- a/workflow-recipes/src/workflow/workflowReconciler.ts+++ b/workflow-recipes/src/workflow/workflowReconciler.ts@@ -233,8 +233,8 @@ type RunLaneNetworkPolicyApplyResult =| { policy: string; action: 'conflict'; reason: NetworkPolicyConflictReason }/** Why a Codex/Grok proxy policy survives a prune that does not desire it. */interface RunLaneProxyKeepReasons {- codex?: 'uncertain' | 'run-in-progress'- grok?: 'uncertain' | 'run-in-progress'+ codex?: 'uncertain'+ grok?: 'uncertain'}// Read-then-write rounds per NetworkPolicy and pass. Three rounds is at most// three writes, the same as the create plus two replaces it replaced.@@ -2993,16 +2993,26 @@ export class WorkflowReconciler {/*** Reapply the run-lane NetworkPolicies outside reconcile(). The running and* active short-circuits return before reconcile(), so a policy a previous- * pass left pending (terminating, contended, or a prune that failed) would- * otherwise stay as it is until the run ends. Runs the same build, apply and- * prune unit reconcile() runs, with a verdict computed now, with one- * difference: the Codex and Grok proxy policies are never revoked here.- * The verdict can change while a step is using the proxy, so a revoke- * mid-run would cut its egress; a proxy the verdict still requires is- * created (an allow), and one it no longer wants is pruned by the first- * reconcile() after the run. The catalog gate keeps everything the run lane- * does not own. A prune that failed sets retryPending, so without the prune- * here the marker would requeue forever and never reach the leftover.+ * pass left pending (terminating or contended) would otherwise stay as it is+ * until the run ends. Builds the desired set with a verdict computed now and+ * applies it, but prunes no run-lane policy: the run's pods are live and may+ * still use any lane the spec wanted when they were created, so a revoke here+ * cuts their traffic. Only the legacy mcp-servers internet policy, which no+ * spec wants, is still deleted. retryPending reflects the applies and that+ * delete, so a marker a failed reconcile() prune set clears here and the+ * leftover waits for the next reconcile() pass or the finalizer sweep.+ *+ * What each path may delete (verdict = Codex/Grok eligibility; it governs+ * only the proxy lanes, every other catalog lane follows the spec alone):+ *+ * | path | proxy required | proxy uncertain | proxy retired | other lane, not desired |+ * | reconcile() pass | applied | kept if live | pruned | pruned |+ * | mid-run retry (this) | applied | kept if live | kept | kept |+ * | terminal teardown | kept | kept | kept | kept |+ *+ * A desired lane is applied on both apply paths; an uncertain verdict never+ * creates a proxy. Terminal teardown deletes compute pods, not policies; the+ * finalizer deletes every recipe policy.*/async retryRunLaneNetworkPolicies(recipeName: string,@@ -3027,7 +3037,7 @@ export class WorkflowReconciler {runtimeScopeRecipeName,codexView)- const { policies, catalog } = await this.buildWorkflowNetworkPoliciesForSpec(+ const { policies } = await this.buildWorkflowNetworkPoliciesForSpec(recipeName,recipeUid,spec,@@ -3037,10 +3047,10 @@ export class WorkflowReconciler {false,codexVerdict.grokProjection)- return this.applyAndPruneRunLaneNetworkPolicies(recipeName, recipeUid, policies, catalog, {- codex: 'run-in-progress',- grok: 'run-in-progress',- })+ const summary = await this.applyNetworkPolicyList(policies)+ const legacy = await this.pruneLegacyMcpServersInternetEgressPolicy(recipeName, recipeUid)+ if (legacy.retryPending) summary.retryPending = true+ return summary}async ensureCoordinatorRuntimeCredentials(@@ -3479,10 +3489,11 @@ export class WorkflowReconciler {* Two-term selector only (`managed-by=wrc`): a single-term recipe selector* would also return the recipes-lane policies in the same namespace.* Universe is the factory catalog (GFS never in it), not LIST \ desired.- * A Codex/Grok proxy with a keep reason (an uncertain verdict, or a run in- * progress on the retry path) survives even when it is absent from this- * pass's desired set. A listed policy with an ownerReference is skipped,- * matching the apply's ownership veto.+ * A Codex/Grok proxy with a keep reason (an uncertain verdict) survives even+ * when it is absent from this pass's desired set. A listed policy with an+ * ownerReference is skipped, matching the apply's ownership veto. Only+ * reconcile() prunes; the mid-run retry never calls this (see+ * retryRunLaneNetworkPolicies for the per-path table).*/private async pruneUndesiredRunLaneNetworkPolicies(recipeName: string,diff --git a/workflow-recipes/tests/unit/workflow/workflowReconciler.test.ts b/workflow-recipes/tests/unit/workflow/workflowReconciler.test.tsindex 6b19ca87a..503aef755 100644--- a/workflow-recipes/tests/unit/workflow/workflowReconciler.test.ts+++ b/workflow-recipes/tests/unit/workflow/workflowReconciler.test.ts@@ -5297,12 +5297,89 @@ describe('WorkflowReconciler — reconcile loop', () => {expect(apiserver.api.deleteNamespacedNetworkPolicy).toHaveBeenCalledTimes(0)})- // A prune failure sets retryPending, and this pass is the only one that- // runs while the run is in progress or active. It has to prune again, or- // the marker requeues forever without ever reaching the leftover.+ // The retry runs only from the in-progress and active short-circuits,+ // where the run's pods are live and may still use any lane the spec+ // wanted when they were created. It applies what the current spec wants+ // and prunes nothing; the next reconcile() pass prunes the rest.+ describe('when the spec no longer wants a live lane', () => {+ const SNIPPET_LANES = [+ 'test-wf-coord-to-snippet-runner',+ 'test-wf-coord-to-snippet-runner-ingress',+ 'test-wf-snippet-runner-egress',+ ]+ const AGENT_LANE = 'test-wf-coord-to-mcp-host'++ it('keeps it on a mid-run retry and still applies the lanes the spec wants', async () => {+ const { apiserver, reconciler } = await convergedRunLane()+ apiserver.api.listNamespacedNetworkPolicy.mockClear()++ // An agent recipe with a run id wants the mcp-host lanes and none of+ // the snippet lanes the converged snippet recipe created.+ const summary = await reconciler.retryRunLaneNetworkPolicies(+ 'test-wf',+ 'uid-123',+ makeSpec(),+ 'test-wf',+ 'run-1'+ )++ // Liveness witness: the retry built the agent lane and created it.+ expect(+ apiserver.api.createNamespacedNetworkPolicy.mock.calls.map(+ ([arg]) => arg.body.metadata?.name+ )+ ).toContain(AGENT_LANE)+ expect(apiserver.live.has(apiserver.key('sandbox-recipes', AGENT_LANE))).toBe(true)+ for (const name of SNIPPET_LANES) {+ expect(apiserver.live.has(apiserver.key('sandbox-recipes', name)), name).toBe(true)+ }+ expect(apiserver.api.listNamespacedNetworkPolicy).toHaveBeenCalledTimes(0)+ expect(apiserver.api.deleteNamespacedNetworkPolicy).toHaveBeenCalledTimes(0)+ expect(summary).toEqual({ conflicts: [], retryPending: false })+ })++ it('prunes it on the next reconcile() pass', async () => {+ const { apiserver, reconciler } = await convergedRunLane()+ await reconciler.retryRunLaneNetworkPolicies(+ 'test-wf',+ 'uid-123',+ makeSpec(),+ 'test-wf',+ 'run-1'+ )+ expect(+ apiserver.live.has(apiserver.key('sandbox-recipes', 'test-wf-snippet-runner-egress'))+ ).toBe(true)++ const result = await reconciler.reconcile(+ 'test-wf',+ 'uid-123',+ 'sandbox-recipes',+ makeSpec(),+ undefined,+ undefined,+ 'test-wf',+ 'run-1'+ )++ expect(result.workflowPhase).not.toBe('failed')+ const deleted = apiserver.api.deleteNamespacedNetworkPolicy.mock.calls.map(+ ([arg]) => arg.name+ )+ for (const name of SNIPPET_LANES) {+ expect(deleted, name).toContain(name)+ expect(apiserver.live.has(apiserver.key('sandbox-recipes', name)), name).toBe(false)+ }+ expect(apiserver.live.has(apiserver.key('sandbox-recipes', AGENT_LANE))).toBe(true)+ })+ })++ // A reconcile() prune that failed sets the retry marker. The retry+ // does not prune, so it clears the marker once its applies converge,+ // and the leftover waits for the next reconcile() pass.describe('when a prune left a leftover behind', () => {- // A catalog member the snippet lane does not want and that no eligibility- // verdict governs: the retry must prune it so the marker can clear.+ // A catalog member the snippet lane does not want and that no+ // eligibility verdict governs.const LEFTOVER = 'test-wf-coord-to-mcp-host'const GROK_PROXY = 'test-wf-mcp-host-to-grok-proxy'const TWO_TERM_SELECTOR = 'clerum.io/recipe=test-wf,clerum.io/managed-by=wrc'@@ -5321,59 +5398,45 @@ describe('WorkflowReconciler — reconcile loop', () => {})}- it('deletes the leftover on the retry pass', async () => {- const { apiserver, reconciler } = await convergedRunLane()- apiserver.api.listNamespacedNetworkPolicy.mockClear()- seedLeftover(apiserver)-- const summary = await reconciler.retryRunLaneNetworkPolicies('test-wf', 'uid-123', spec())-- expect(apiserver.api.listNamespacedNetworkPolicy).toHaveBeenCalledWith({- namespace: 'sandbox-recipes',- labelSelector: TWO_TERM_SELECTOR,- })- expect(summary).toEqual({ conflicts: [], retryPending: false })- expect(apiserver.live.has(apiserver.key('sandbox-recipes', LEFTOVER))).toBe(false)- expect(- apiserver.api.deleteNamespacedNetworkPolicy.mock.calls.map(([arg]) => arg.name)- ).toEqual([LEFTOVER])- })-- it('keeps reporting the retry until the leftover delete succeeds', async () => {+ it('keeps it through the retry, which clears the marker, and prunes it on the next reconcile()', async () => {const { apiserver, reconciler } = await convergedRunLane()- apiserver.api.listNamespacedNetworkPolicy.mockClear()seedLeftover(apiserver)apiserver.api.deleteNamespacedNetworkPolicy.mockRejectedValueOnce({code: 403,message: 'forbidden',})- const first = await reconciler.retryRunLaneNetworkPolicies('test-wf', 'uid-123', spec())- expect(apiserver.api.listNamespacedNetworkPolicy).toHaveBeenCalledTimes(1)- expect(first.retryPending).toBe(true)+ const failed = await reconciler.reconcile('test-wf', 'uid-123', 'sandbox-recipes', spec())+ expect(failed.networkPolicyRetryPending).toBe(true)expect(apiserver.live.has(apiserver.key('sandbox-recipes', LEFTOVER))).toBe(true)- const second = await reconciler.retryRunLaneNetworkPolicies('test-wf', 'uid-123', spec())- expect(second).toEqual({ conflicts: [], retryPending: false })- expect(apiserver.live.has(apiserver.key('sandbox-recipes', LEFTOVER))).toBe(false)- })-- it('keeps reporting the retry while the prune LIST fails', async () => {- const { apiserver, reconciler } = await convergedRunLane()apiserver.api.listNamespacedNetworkPolicy.mockClear()- seedLeftover(apiserver)- apiserver.api.listNamespacedNetworkPolicy.mockRejectedValueOnce({- code: 500,- message: 'boom',- })-- const first = await reconciler.retryRunLaneNetworkPolicies('test-wf', 'uid-123', spec())- expect(apiserver.api.listNamespacedNetworkPolicy).toHaveBeenCalledTimes(1)- expect(first.retryPending).toBe(true)+ apiserver.api.createNamespacedNetworkPolicy.mockClear()+ apiserver.api.deleteNamespacedNetworkPolicy.mockClear()+ expect(+ apiserver.live.delete(apiserver.key('sandbox-recipes', 'test-wf-coord-to-wrc'))+ ).toBe(true)+ const retried = await reconciler.retryRunLaneNetworkPolicies('test-wf', 'uid-123', spec())+ // Liveness witness: the retry applied the lane the spec wants.+ expect(+ apiserver.api.createNamespacedNetworkPolicy.mock.calls.map(+ ([arg]) => arg.body.metadata?.name+ )+ ).toEqual(['test-wf-coord-to-wrc'])+ expect(retried).toEqual({ conflicts: [], retryPending: false })+ expect(apiserver.api.listNamespacedNetworkPolicy).toHaveBeenCalledTimes(0)+ expect(apiserver.api.deleteNamespacedNetworkPolicy).toHaveBeenCalledTimes(0)expect(apiserver.live.has(apiserver.key('sandbox-recipes', LEFTOVER))).toBe(true)- const second = await reconciler.retryRunLaneNetworkPolicies('test-wf', 'uid-123', spec())- expect(second.retryPending).toBe(false)+ const pruned = await reconciler.reconcile('test-wf', 'uid-123', 'sandbox-recipes', spec())+ expect(pruned.networkPolicyRetryPending).toBeFalsy()+ expect(apiserver.api.listNamespacedNetworkPolicy).toHaveBeenCalledWith({+ namespace: 'sandbox-recipes',+ labelSelector: TWO_TERM_SELECTOR,+ })+ expect(+ apiserver.api.deleteNamespacedNetworkPolicy.mock.calls.map(([arg]) => arg.name)+ ).toEqual([LEFTOVER])expect(apiserver.live.has(apiserver.key('sandbox-recipes', LEFTOVER))).toBe(false)})@@ -5400,15 +5463,13 @@ describe('WorkflowReconciler — reconcile loop', () => {expect(second.retryPending).toBe(false)})- // The retry computes its own verdict. These tests pin it: a proxy the- // fresh verdict still wants (or cannot decide about) survives, and one- // it no longer wants goes. Every case also asserts the delete of a- // snippet-lane sibling the agent spec no longer wants- // (`expectSnippetSiblingPruned`): that is the liveness witness proving- // the prune loop iterated the same LIST, so "no proxy was deleted"- // cannot pass because the loop never ran.+ // The retry computes its own verdict. Every case also asserts the+ // create of an agent lane the spec wants (`expectAgentLaneApplied`): that+ // is the liveness witness proving the retry built and applied this spec,+ // so "no proxy was deleted" cannot pass because the retry never ran.describe('with a fresh Codex/Grok verdict', () => {const SNIPPET_SIBLING = 'test-wf-snippet-runner-egress'+ const AGENT_LANE = 'test-wf-coord-to-mcp-host'const CODEX_LIVE = 'test-wf-mcp-host-to-codex-proxy'const projection = (overrides: Record<string, unknown>) => ({targets: [],@@ -5466,20 +5527,20 @@ describe('WorkflowReconciler — reconcile loop', () => {}}- // The baseline lane is the snippet one, so switching to an agent spec also- // prunes its snippet policies. Only the proxy policies are under test here.+ // The baseline lane is the snippet one. Switching to an agent spec+ // mid-run keeps its snippet policies; only the proxies are under test.const deletedNames = (apiserver: Converged) =>apiserver.api.deleteNamespacedNetworkPolicy.mock.calls.map(([arg]) => arg.name).filter(name => name.endsWith('-proxy'))- function expectSnippetSiblingPruned(apiserver: Converged): void {+ function expectAgentLaneApplied(apiserver: Converged): void {expect(- apiserver.api.deleteNamespacedNetworkPolicy.mock.calls.map(([arg]) => arg.name)- ).toContain(SNIPPET_SIBLING)- expect(apiserver.live.has(apiserver.key('sandbox-recipes', SNIPPET_SIBLING))).toBe(- false- )+ apiserver.api.createNamespacedNetworkPolicy.mock.calls.map(+ ([arg]) => arg.body.metadata?.name+ )+ ).toContain(AGENT_LANE)+ expect(apiserver.live.has(apiserver.key('sandbox-recipes', SNIPPET_SIBLING))).toBe(true)}it('keeps a proxy the fresh verdict still requires, and the undesired twin', async () => {@@ -5495,8 +5556,8 @@ describe('WorkflowReconciler — reconcile loop', () => {[CODEX_LIVE, GROK_PROXY])- expect(apiserver.api.listNamespacedNetworkPolicy).toHaveBeenCalledTimes(1)- expectSnippetSiblingPruned(apiserver)+ expect(apiserver.api.listNamespacedNetworkPolicy).toHaveBeenCalledTimes(0)+ expectAgentLaneApplied(apiserver)expect(deletedNames(apiserver)).toEqual([])expect(apiserver.live.has(apiserver.key('sandbox-recipes', CODEX_LIVE))).toBe(true)expect(apiserver.live.has(apiserver.key('sandbox-recipes', GROK_PROXY))).toBe(true)@@ -5512,8 +5573,8 @@ describe('WorkflowReconciler — reconcile loop', () => {[CODEX_LIVE, GROK_PROXY])- expect(apiserver.api.listNamespacedNetworkPolicy).toHaveBeenCalledTimes(1)- expectSnippetSiblingPruned(apiserver)+ expect(apiserver.api.listNamespacedNetworkPolicy).toHaveBeenCalledTimes(0)+ expectAgentLaneApplied(apiserver)expect(deletedNames(apiserver)).toEqual([])expect(apiserver.live.has(apiserver.key('sandbox-recipes', CODEX_LIVE))).toBe(true)expect(apiserver.live.has(apiserver.key('sandbox-recipes', GROK_PROXY))).toBe(true)@@ -5530,8 +5591,8 @@ describe('WorkflowReconciler — reconcile loop', () => {GROK_PROXY,])- expect(apiserver.api.listNamespacedNetworkPolicy).toHaveBeenCalledTimes(1)- expectSnippetSiblingPruned(apiserver)+ expect(apiserver.api.listNamespacedNetworkPolicy).toHaveBeenCalledTimes(0)+ expectAgentLaneApplied(apiserver)expect(deletedNames(apiserver)).toEqual([])expect(apiserver.live.has(apiserver.key('sandbox-recipes', CODEX_LIVE))).toBe(true)expect(apiserver.live.has(apiserver.key('sandbox-recipes', GROK_PROXY))).toBe(true)