Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript

fix(workflow-recipes): stop the mid-run NetworkPolicy retry from pruning live lanes

Alfredo Lopez9ad28c10e124
diff --git a/workflow-recipes/package-lock.json b/workflow-recipes/package-lock.json
index 80033b5e9..9504adcc3 100644
--- a/workflow-recipes/package-lock.json
+++ b/workflow-recipes/package-lock.json
@@ -1,12 +1,12 @@
{
"name": "workload-recipes-controller",
- "version": "0.1.419",
+ "version": "0.1.420",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "workload-recipes-controller",
- "version": "0.1.419",
+ "version": "0.1.420",
"license": "SEE LICENSE IN LICENSE",
"dependencies": {
"@aws-sdk/client-s3": "^3.1009.0",
diff --git a/workflow-recipes/package.json b/workflow-recipes/package.json
index fe094a437..7cd8721da 100644
--- a/workflow-recipes/package.json
+++ b/workflow-recipes/package.json
@@ -1,6 +1,6 @@
{
"name": "workload-recipes-controller",
- "version": "0.1.419",
+ "version": "0.1.420",
"description": "Workload Recipes Controller (WRC) - WorkloadRecipe lifecycle management and MCP server interface",
"main": "dist/main.js",
"engines": {
diff --git a/workflow-recipes/src/reconciler/reconciler.test.ts b/workflow-recipes/src/reconciler/reconciler.test.ts
index e9801d2cc..d4f78f909 100644
--- a/workflow-recipes/src/reconciler/reconciler.test.ts
+++ b/workflow-recipes/src/reconciler/reconciler.test.ts
@@ -16129,20 +16129,26 @@ describe('WorkflowRecipeReconciler', () => {
'run-g2'
)
- // Liveness witness: the retry ran the prune and deleted an undesired
- // catalog sibling, so the absence of a GFS DELETE below is the catalog
- // gate holding, not a retry that never reached the prune.
- expect(mockNetworkingApi.listNamespacedNetworkPolicy).toHaveBeenCalled()
- expect(mockNetworkingApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledWith(
- expect.objectContaining({ name: `${RECIPE}-snippet-runner-egress` })
- )
- expect(mockNetworkingApi.deleteNamespacedNetworkPolicy).not.toHaveBeenCalledWith(
- expect.objectContaining({ name: GFS })
- )
- // The retry runs mid-run, so it never revokes a Codex/Grok proxy.
- expect(mockNetworkingApi.deleteNamespacedNetworkPolicy).not.toHaveBeenCalledWith(
- expect.objectContaining({ name: `${RECIPE}-mcp-host-to-grok-proxy` })
- )
+ // Liveness witness: the retry built this agent spec and applied a lane
+ // it wants, so the absent DELETEs below are the retry declining to
+ // prune mid-run, not a retry that never ran.
+ expect(
+ mockNetworkingApi.createNamespacedNetworkPolicy.mock.calls.some(
+ ([arg]) => arg.body?.metadata?.name === `${RECIPE}-coord-to-mcp-host`
+ )
+ ).toBe(true)
+ // The run's pods are live, so the retry revokes no lane: not the
+ // reserved GFS policy, not a sibling the spec no longer wants, and not
+ // a Codex/Grok proxy.
+ for (const name of [
+ GFS,
+ `${RECIPE}-snippet-runner-egress`,
+ `${RECIPE}-mcp-host-to-grok-proxy`,
+ ]) {
+ expect(mockNetworkingApi.deleteNamespacedNetworkPolicy).not.toHaveBeenCalledWith(
+ expect.objectContaining({ name })
+ )
+ }
})
})
diff --git a/workflow-recipes/src/workflow/workflowReconciler.ts b/workflow-recipes/src/workflow/workflowReconciler.ts
index 3a92ce2d9..74995487d 100644
--- a/workflow-recipes/src/workflow/workflowReconciler.ts
+++ b/workflow-recipes/src/workflow/workflowReconciler.ts
@@ -233,8 +233,8 @@ type RunLaneNetworkPolicyApplyResult =
| { policy: string; action: 'conflict'; reason: NetworkPolicyConflictReason }
/** Why a Codex/Grok proxy policy survives a prune that does not desire it. */
interface RunLaneProxyKeepReasons {
- codex?: 'uncertain' | 'run-in-progress'
- grok?: 'uncertain' | 'run-in-progress'
+ codex?: 'uncertain'
+ grok?: 'uncertain'
}
// Read-then-write rounds per NetworkPolicy and pass. Three rounds is at most
// three writes, the same as the create plus two replaces it replaced.
@@ -2993,16 +2993,26 @@ export class WorkflowReconciler {
/**
* Reapply the run-lane NetworkPolicies outside reconcile(). The running and
* active short-circuits return before reconcile(), so a policy a previous
- * pass left pending (terminating, contended, or a prune that failed) would
- * otherwise stay as it is until the run ends. Runs the same build, apply and
- * prune unit reconcile() runs, with a verdict computed now, with one
- * difference: the Codex and Grok proxy policies are never revoked here.
- * The verdict can change while a step is using the proxy, so a revoke
- * mid-run would cut its egress; a proxy the verdict still requires is
- * created (an allow), and one it no longer wants is pruned by the first
- * reconcile() after the run. The catalog gate keeps everything the run lane
- * does not own. A prune that failed sets retryPending, so without the prune
- * here the marker would requeue forever and never reach the leftover.
+ * pass left pending (terminating or contended) would otherwise stay as it is
+ * until the run ends. Builds the desired set with a verdict computed now and
+ * applies it, but prunes no run-lane policy: the run's pods are live and may
+ * still use any lane the spec wanted when they were created, so a revoke here
+ * cuts their traffic. Only the legacy mcp-servers internet policy, which no
+ * spec wants, is still deleted. retryPending reflects the applies and that
+ * delete, so a marker a failed reconcile() prune set clears here and the
+ * leftover waits for the next reconcile() pass or the finalizer sweep.
+ *
+ * What each path may delete (verdict = Codex/Grok eligibility; it governs
+ * only the proxy lanes, every other catalog lane follows the spec alone):
+ *
+ * | path | proxy required | proxy uncertain | proxy retired | other lane, not desired |
+ * | reconcile() pass | applied | kept if live | pruned | pruned |
+ * | mid-run retry (this) | applied | kept if live | kept | kept |
+ * | terminal teardown | kept | kept | kept | kept |
+ *
+ * A desired lane is applied on both apply paths; an uncertain verdict never
+ * creates a proxy. Terminal teardown deletes compute pods, not policies; the
+ * finalizer deletes every recipe policy.
*/
async retryRunLaneNetworkPolicies(
recipeName: string,
@@ -3027,7 +3037,7 @@ export class WorkflowReconciler {
runtimeScopeRecipeName,
codexView
)
- const { policies, catalog } = await this.buildWorkflowNetworkPoliciesForSpec(
+ const { policies } = await this.buildWorkflowNetworkPoliciesForSpec(
recipeName,
recipeUid,
spec,
@@ -3037,10 +3047,10 @@ export class WorkflowReconciler {
false,
codexVerdict.grokProjection
)
- return this.applyAndPruneRunLaneNetworkPolicies(recipeName, recipeUid, policies, catalog, {
- codex: 'run-in-progress',
- grok: 'run-in-progress',
- })
+ const summary = await this.applyNetworkPolicyList(policies)
+ const legacy = await this.pruneLegacyMcpServersInternetEgressPolicy(recipeName, recipeUid)
+ if (legacy.retryPending) summary.retryPending = true
+ return summary
}
async ensureCoordinatorRuntimeCredentials(
@@ -3479,10 +3489,11 @@ export class WorkflowReconciler {
* Two-term selector only (`managed-by=wrc`): a single-term recipe selector
* would also return the recipes-lane policies in the same namespace.
* Universe is the factory catalog (GFS never in it), not LIST \ desired.
- * A Codex/Grok proxy with a keep reason (an uncertain verdict, or a run in
- * progress on the retry path) survives even when it is absent from this
- * pass's desired set. A listed policy with an ownerReference is skipped,
- * matching the apply's ownership veto.
+ * A Codex/Grok proxy with a keep reason (an uncertain verdict) survives even
+ * when it is absent from this pass's desired set. A listed policy with an
+ * ownerReference is skipped, matching the apply's ownership veto. Only
+ * reconcile() prunes; the mid-run retry never calls this (see
+ * retryRunLaneNetworkPolicies for the per-path table).
*/
private async pruneUndesiredRunLaneNetworkPolicies(
recipeName: string,
diff --git a/workflow-recipes/tests/unit/workflow/workflowReconciler.test.ts b/workflow-recipes/tests/unit/workflow/workflowReconciler.test.ts
index 6b19ca87a..503aef755 100644
--- a/workflow-recipes/tests/unit/workflow/workflowReconciler.test.ts
+++ b/workflow-recipes/tests/unit/workflow/workflowReconciler.test.ts
@@ -5297,12 +5297,89 @@ describe('WorkflowReconciler — reconcile loop', () => {
expect(apiserver.api.deleteNamespacedNetworkPolicy).toHaveBeenCalledTimes(0)
})
- // A prune failure sets retryPending, and this pass is the only one that
- // runs while the run is in progress or active. It has to prune again, or
- // the marker requeues forever without ever reaching the leftover.
+ // The retry runs only from the in-progress and active short-circuits,
+ // where the run's pods are live and may still use any lane the spec
+ // wanted when they were created. It applies what the current spec wants
+ // and prunes nothing; the next reconcile() pass prunes the rest.
+ describe('when the spec no longer wants a live lane', () => {
+ const SNIPPET_LANES = [
+ 'test-wf-coord-to-snippet-runner',
+ 'test-wf-coord-to-snippet-runner-ingress',
+ 'test-wf-snippet-runner-egress',
+ ]
+ const AGENT_LANE = 'test-wf-coord-to-mcp-host'
+
+ it('keeps it on a mid-run retry and still applies the lanes the spec wants', async () => {
+ const { apiserver, reconciler } = await convergedRunLane()
+ apiserver.api.listNamespacedNetworkPolicy.mockClear()
+
+ // An agent recipe with a run id wants the mcp-host lanes and none of
+ // the snippet lanes the converged snippet recipe created.
+ const summary = await reconciler.retryRunLaneNetworkPolicies(
+ 'test-wf',
+ 'uid-123',
+ makeSpec(),
+ 'test-wf',
+ 'run-1'
+ )
+
+ // Liveness witness: the retry built the agent lane and created it.
+ expect(
+ apiserver.api.createNamespacedNetworkPolicy.mock.calls.map(
+ ([arg]) => arg.body.metadata?.name
+ )
+ ).toContain(AGENT_LANE)
+ expect(apiserver.live.has(apiserver.key('sandbox-recipes', AGENT_LANE))).toBe(true)
+ for (const name of SNIPPET_LANES) {
+ expect(apiserver.live.has(apiserver.key('sandbox-recipes', name)), name).toBe(true)
+ }
+ expect(apiserver.api.listNamespacedNetworkPolicy).toHaveBeenCalledTimes(0)
+ expect(apiserver.api.deleteNamespacedNetworkPolicy).toHaveBeenCalledTimes(0)
+ expect(summary).toEqual({ conflicts: [], retryPending: false })
+ })
+
+ it('prunes it on the next reconcile() pass', async () => {
+ const { apiserver, reconciler } = await convergedRunLane()
+ await reconciler.retryRunLaneNetworkPolicies(
+ 'test-wf',
+ 'uid-123',
+ makeSpec(),
+ 'test-wf',
+ 'run-1'
+ )
+ expect(
+ apiserver.live.has(apiserver.key('sandbox-recipes', 'test-wf-snippet-runner-egress'))
+ ).toBe(true)
+
+ const result = await reconciler.reconcile(
+ 'test-wf',
+ 'uid-123',
+ 'sandbox-recipes',
+ makeSpec(),
+ undefined,
+ undefined,
+ 'test-wf',
+ 'run-1'
+ )
+
+ expect(result.workflowPhase).not.toBe('failed')
+ const deleted = apiserver.api.deleteNamespacedNetworkPolicy.mock.calls.map(
+ ([arg]) => arg.name
+ )
+ for (const name of SNIPPET_LANES) {
+ expect(deleted, name).toContain(name)
+ expect(apiserver.live.has(apiserver.key('sandbox-recipes', name)), name).toBe(false)
+ }
+ expect(apiserver.live.has(apiserver.key('sandbox-recipes', AGENT_LANE))).toBe(true)
+ })
+ })
+
+ // A reconcile() prune that failed sets the retry marker. The retry
+ // does not prune, so it clears the marker once its applies converge,
+ // and the leftover waits for the next reconcile() pass.
describe('when a prune left a leftover behind', () => {
- // A catalog member the snippet lane does not want and that no eligibility
- // verdict governs: the retry must prune it so the marker can clear.
+ // A catalog member the snippet lane does not want and that no
+ // eligibility verdict governs.
const LEFTOVER = 'test-wf-coord-to-mcp-host'
const GROK_PROXY = 'test-wf-mcp-host-to-grok-proxy'
const TWO_TERM_SELECTOR = 'clerum.io/recipe=test-wf,clerum.io/managed-by=wrc'
@@ -5321,59 +5398,45 @@ describe('WorkflowReconciler — reconcile loop', () => {
})
}
- it('deletes the leftover on the retry pass', async () => {
- const { apiserver, reconciler } = await convergedRunLane()
- apiserver.api.listNamespacedNetworkPolicy.mockClear()
- seedLeftover(apiserver)
-
- const summary = await reconciler.retryRunLaneNetworkPolicies('test-wf', 'uid-123', spec())
-
- expect(apiserver.api.listNamespacedNetworkPolicy).toHaveBeenCalledWith({
- namespace: 'sandbox-recipes',
- labelSelector: TWO_TERM_SELECTOR,
- })
- expect(summary).toEqual({ conflicts: [], retryPending: false })
- expect(apiserver.live.has(apiserver.key('sandbox-recipes', LEFTOVER))).toBe(false)
- expect(
- apiserver.api.deleteNamespacedNetworkPolicy.mock.calls.map(([arg]) => arg.name)
- ).toEqual([LEFTOVER])
- })
-
- it('keeps reporting the retry until the leftover delete succeeds', async () => {
+ it('keeps it through the retry, which clears the marker, and prunes it on the next reconcile()', async () => {
const { apiserver, reconciler } = await convergedRunLane()
- apiserver.api.listNamespacedNetworkPolicy.mockClear()
seedLeftover(apiserver)
apiserver.api.deleteNamespacedNetworkPolicy.mockRejectedValueOnce({
code: 403,
message: 'forbidden',
})
- const first = await reconciler.retryRunLaneNetworkPolicies('test-wf', 'uid-123', spec())
- expect(apiserver.api.listNamespacedNetworkPolicy).toHaveBeenCalledTimes(1)
- expect(first.retryPending).toBe(true)
+ const failed = await reconciler.reconcile('test-wf', 'uid-123', 'sandbox-recipes', spec())
+ expect(failed.networkPolicyRetryPending).toBe(true)
expect(apiserver.live.has(apiserver.key('sandbox-recipes', LEFTOVER))).toBe(true)
- const second = await reconciler.retryRunLaneNetworkPolicies('test-wf', 'uid-123', spec())
- expect(second).toEqual({ conflicts: [], retryPending: false })
- expect(apiserver.live.has(apiserver.key('sandbox-recipes', LEFTOVER))).toBe(false)
- })
-
- it('keeps reporting the retry while the prune LIST fails', async () => {
- const { apiserver, reconciler } = await convergedRunLane()
apiserver.api.listNamespacedNetworkPolicy.mockClear()
- seedLeftover(apiserver)
- apiserver.api.listNamespacedNetworkPolicy.mockRejectedValueOnce({
- code: 500,
- message: 'boom',
- })
-
- const first = await reconciler.retryRunLaneNetworkPolicies('test-wf', 'uid-123', spec())
- expect(apiserver.api.listNamespacedNetworkPolicy).toHaveBeenCalledTimes(1)
- expect(first.retryPending).toBe(true)
+ apiserver.api.createNamespacedNetworkPolicy.mockClear()
+ apiserver.api.deleteNamespacedNetworkPolicy.mockClear()
+ expect(
+ apiserver.live.delete(apiserver.key('sandbox-recipes', 'test-wf-coord-to-wrc'))
+ ).toBe(true)
+ const retried = await reconciler.retryRunLaneNetworkPolicies('test-wf', 'uid-123', spec())
+ // Liveness witness: the retry applied the lane the spec wants.
+ expect(
+ apiserver.api.createNamespacedNetworkPolicy.mock.calls.map(
+ ([arg]) => arg.body.metadata?.name
+ )
+ ).toEqual(['test-wf-coord-to-wrc'])
+ expect(retried).toEqual({ conflicts: [], retryPending: false })
+ expect(apiserver.api.listNamespacedNetworkPolicy).toHaveBeenCalledTimes(0)
+ expect(apiserver.api.deleteNamespacedNetworkPolicy).toHaveBeenCalledTimes(0)
expect(apiserver.live.has(apiserver.key('sandbox-recipes', LEFTOVER))).toBe(true)
- const second = await reconciler.retryRunLaneNetworkPolicies('test-wf', 'uid-123', spec())
- expect(second.retryPending).toBe(false)
+ const pruned = await reconciler.reconcile('test-wf', 'uid-123', 'sandbox-recipes', spec())
+ expect(pruned.networkPolicyRetryPending).toBeFalsy()
+ expect(apiserver.api.listNamespacedNetworkPolicy).toHaveBeenCalledWith({
+ namespace: 'sandbox-recipes',
+ labelSelector: TWO_TERM_SELECTOR,
+ })
+ expect(
+ apiserver.api.deleteNamespacedNetworkPolicy.mock.calls.map(([arg]) => arg.name)
+ ).toEqual([LEFTOVER])
expect(apiserver.live.has(apiserver.key('sandbox-recipes', LEFTOVER))).toBe(false)
})
@@ -5400,15 +5463,13 @@ describe('WorkflowReconciler — reconcile loop', () => {
expect(second.retryPending).toBe(false)
})
- // The retry computes its own verdict. These tests pin it: a proxy the
- // fresh verdict still wants (or cannot decide about) survives, and one
- // it no longer wants goes. Every case also asserts the delete of a
- // snippet-lane sibling the agent spec no longer wants
- // (`expectSnippetSiblingPruned`): that is the liveness witness proving
- // the prune loop iterated the same LIST, so "no proxy was deleted"
- // cannot pass because the loop never ran.
+ // The retry computes its own verdict. Every case also asserts the
+ // create of an agent lane the spec wants (`expectAgentLaneApplied`): that
+ // is the liveness witness proving the retry built and applied this spec,
+ // so "no proxy was deleted" cannot pass because the retry never ran.
describe('with a fresh Codex/Grok verdict', () => {
const SNIPPET_SIBLING = 'test-wf-snippet-runner-egress'
+ const AGENT_LANE = 'test-wf-coord-to-mcp-host'
const CODEX_LIVE = 'test-wf-mcp-host-to-codex-proxy'
const projection = (overrides: Record<string, unknown>) => ({
targets: [],
@@ -5466,20 +5527,20 @@ describe('WorkflowReconciler — reconcile loop', () => {
}
}
- // The baseline lane is the snippet one, so switching to an agent spec also
- // prunes its snippet policies. Only the proxy policies are under test here.
+ // The baseline lane is the snippet one. Switching to an agent spec
+ // mid-run keeps its snippet policies; only the proxies are under test.
const deletedNames = (apiserver: Converged) =>
apiserver.api.deleteNamespacedNetworkPolicy.mock.calls
.map(([arg]) => arg.name)
.filter(name => name.endsWith('-proxy'))
- function expectSnippetSiblingPruned(apiserver: Converged): void {
+ function expectAgentLaneApplied(apiserver: Converged): void {
expect(
- apiserver.api.deleteNamespacedNetworkPolicy.mock.calls.map(([arg]) => arg.name)
- ).toContain(SNIPPET_SIBLING)
- expect(apiserver.live.has(apiserver.key('sandbox-recipes', SNIPPET_SIBLING))).toBe(
- false
- )
+ apiserver.api.createNamespacedNetworkPolicy.mock.calls.map(
+ ([arg]) => arg.body.metadata?.name
+ )
+ ).toContain(AGENT_LANE)
+ expect(apiserver.live.has(apiserver.key('sandbox-recipes', SNIPPET_SIBLING))).toBe(true)
}
it('keeps a proxy the fresh verdict still requires, and the undesired twin', async () => {
@@ -5495,8 +5556,8 @@ describe('WorkflowReconciler — reconcile loop', () => {
[CODEX_LIVE, GROK_PROXY]
)
- expect(apiserver.api.listNamespacedNetworkPolicy).toHaveBeenCalledTimes(1)
- expectSnippetSiblingPruned(apiserver)
+ expect(apiserver.api.listNamespacedNetworkPolicy).toHaveBeenCalledTimes(0)
+ expectAgentLaneApplied(apiserver)
expect(deletedNames(apiserver)).toEqual([])
expect(apiserver.live.has(apiserver.key('sandbox-recipes', CODEX_LIVE))).toBe(true)
expect(apiserver.live.has(apiserver.key('sandbox-recipes', GROK_PROXY))).toBe(true)
@@ -5512,8 +5573,8 @@ describe('WorkflowReconciler — reconcile loop', () => {
[CODEX_LIVE, GROK_PROXY]
)
- expect(apiserver.api.listNamespacedNetworkPolicy).toHaveBeenCalledTimes(1)
- expectSnippetSiblingPruned(apiserver)
+ expect(apiserver.api.listNamespacedNetworkPolicy).toHaveBeenCalledTimes(0)
+ expectAgentLaneApplied(apiserver)
expect(deletedNames(apiserver)).toEqual([])
expect(apiserver.live.has(apiserver.key('sandbox-recipes', CODEX_LIVE))).toBe(true)
expect(apiserver.live.has(apiserver.key('sandbox-recipes', GROK_PROXY))).toBe(true)
@@ -5530,8 +5591,8 @@ describe('WorkflowReconciler — reconcile loop', () => {
GROK_PROXY,
])
- expect(apiserver.api.listNamespacedNetworkPolicy).toHaveBeenCalledTimes(1)
- expectSnippetSiblingPruned(apiserver)
+ expect(apiserver.api.listNamespacedNetworkPolicy).toHaveBeenCalledTimes(0)
+ expectAgentLaneApplied(apiserver)
expect(deletedNames(apiserver)).toEqual([])
expect(apiserver.live.has(apiserver.key('sandbox-recipes', CODEX_LIVE))).toBe(true)
expect(apiserver.live.has(apiserver.key('sandbox-recipes', GROK_PROXY))).toBe(true)