Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript

fix(workflow-recipes): skip terminal compute-pod DELETE after same-pass 404

Alfredo Lopez07ac854e8e17
diff --git a/workflow-recipes/package-lock.json b/workflow-recipes/package-lock.json
index 9f46506db..b5d1dab41 100644
--- a/workflow-recipes/package-lock.json
+++ b/workflow-recipes/package-lock.json
@@ -1,12 +1,12 @@
{
"name": "workload-recipes-controller",
- "version": "0.1.407",
+ "version": "0.1.408",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "workload-recipes-controller",
- "version": "0.1.407",
+ "version": "0.1.408",
"license": "SEE LICENSE IN LICENSE",
"dependencies": {
"@aws-sdk/client-s3": "^3.1009.0",
diff --git a/workflow-recipes/package.json b/workflow-recipes/package.json
index a661ef485..5fd7b25f3 100644
--- a/workflow-recipes/package.json
+++ b/workflow-recipes/package.json
@@ -1,6 +1,6 @@
{
"name": "workload-recipes-controller",
- "version": "0.1.407",
+ "version": "0.1.408",
"description": "Workload Recipes Controller (WRC) - WorkloadRecipe lifecycle management and MCP server interface",
"main": "dist/main.js",
"engines": {
diff --git a/workflow-recipes/src/workflow/networkPolicyFactory.gfs.test.ts b/workflow-recipes/src/workflow/networkPolicyFactory.gfs.test.ts
index 85812030c..9271a3071 100644
--- a/workflow-recipes/src/workflow/networkPolicyFactory.gfs.test.ts
+++ b/workflow-recipes/src/workflow/networkPolicyFactory.gfs.test.ts
@@ -166,4 +166,27 @@ describe('workflow GFS NetworkPolicy', () => {
expect(catalog.has('daily-report-wrc-to-artifact-reader')).toBe(true)
expect(catalog.has('daily-report-coordinator-to-gfs')).toBe(false)
})
+
+ it('forces unused lanes into the catalog so a pass-config prune would not go empty', () => {
+ const applyConfig: NetworkPolicyConfig = {
+ ...baseConfig,
+ includeCoordinator: true,
+ includeMcpHost: false,
+ includeCodexProxyEgress: false,
+ includeGrokProxyEgress: false,
+ includeArtifactReader: false,
+ includeSnippetRunner: false,
+ }
+ const desired = new Set(
+ buildWorkflowNetworkPolicies(applyConfig)
+ .map(policy => policy.metadata?.name)
+ .filter((name): name is string => Boolean(name))
+ )
+ const catalog = buildRunLaneNetworkPolicyCatalog(applyConfig)
+ expect(desired.has('daily-report-coord-to-wrc')).toBe(true)
+ expect(desired.has('daily-report-coord-to-mcp-host')).toBe(false)
+ expect(catalog.has('daily-report-coord-to-mcp-host')).toBe(true)
+ expect(catalog.has('daily-report-mcp-host-to-grok-proxy')).toBe(true)
+ expect(catalog.has('daily-report-coordinator-to-gfs')).toBe(false)
+ })
})
diff --git a/workflow-recipes/src/workflow/networkPolicyFactory.ts b/workflow-recipes/src/workflow/networkPolicyFactory.ts
index 87f916806..91840fe9d 100644
--- a/workflow-recipes/src/workflow/networkPolicyFactory.ts
+++ b/workflow-recipes/src/workflow/networkPolicyFactory.ts
@@ -1075,6 +1075,12 @@ const RUN_LANE_CATALOG_PLACEHOLDER_SERVER = '_catalog'
* Names the run-lane factory can emit for this recipe. Outer-owned
* `${recipe}-coordinator-to-gfs` is never included (`includeCoordinatorGfs: false`).
* Catalog membership — not a GFS-name keep-list — is the prune universe.
+ *
+ * Lane flags are forced on so leftovers of unused lanes stay in the universe
+ * when this pass's applyConfig turned them off. Building the catalog from the
+ * pass config alone makes catalog == desired and prune never deletes a leftover.
+ * Awaiting-trigger already tears down the mcp-host pod unless eager SDK; pruning
+ * those NPs is the same "mcp-host is not live" decision, not an accidental retire.
*/
export function buildRunLaneNetworkPolicyCatalog(
applyConfig: NetworkPolicyConfig,
diff --git a/workflow-recipes/src/workflow/workflowReconciler.test.ts b/workflow-recipes/src/workflow/workflowReconciler.test.ts
index 533360030..b68603042 100644
--- a/workflow-recipes/src/workflow/workflowReconciler.test.ts
+++ b/workflow-recipes/src/workflow/workflowReconciler.test.ts
@@ -1,4 +1,4 @@
-import { beforeEach, describe, expect, it, vi } from 'vitest'
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
import { buildMcpHostHeadlessService } from './podFactory'
import { buildMcpHostServiceName } from './resourceNames'
import { WorkflowReconciler, type WorkflowReconcilerDeps } from './workflowReconciler'
@@ -1276,7 +1276,28 @@ describe('WorkflowReconciler.reconcileDelete — orphaned Service cleanup', () =
})
describe('teardownComputePodsForTerminalRun', () => {
+ afterEach(() => {
+ crashRecoveryMocks.getPodPresence.mockImplementation(
+ async (api: unknown, name: string, namespace: string) => {
+ const phase = await crashRecoveryMocks.getPodPhase(api, name, namespace)
+ return phase === undefined
+ ? { kind: 'absent' as const }
+ : { kind: 'present' as const, phase }
+ }
+ )
+ })
+
it('deletes coordinator, mcp-host, and snippet-runner but PRESERVES artifact-reader', async () => {
+ crashRecoveryMocks.getPodPhase.mockImplementation(async (_api, name: string) => {
+ if (
+ name === 'wf-run-7-coordinator' ||
+ name === 'wf-run-7-mcp-host' ||
+ name === 'wf-run-7-snippet-runner'
+ ) {
+ return 'Succeeded'
+ }
+ return undefined
+ })
const reconciler = new WorkflowReconciler(deps)
await reconciler.teardownComputePodsForTerminalRun('wf-run-7')
@@ -1295,10 +1316,36 @@ describe('WorkflowReconciler.reconcileDelete — orphaned Service cleanup', () =
}
})
+ it('skips DELETE when the same-pass GET is already 404', async () => {
+ crashRecoveryMocks.getPodPhase.mockResolvedValue(undefined)
+ const reconciler = new WorkflowReconciler(deps)
+
+ await reconciler.teardownComputePodsForTerminalRun('wf-run-7')
+
+ expect(crashRecoveryMocks.deletePodIfExists).not.toHaveBeenCalled()
+ })
+
+ it('still deletes a live pod whose status.phase is empty', async () => {
+ crashRecoveryMocks.getPodPresence.mockResolvedValue({ kind: 'present' })
+ const reconciler = new WorkflowReconciler(deps)
+
+ await reconciler.teardownComputePodsForTerminalRun('wf-run-7')
+
+ const deletedPods = crashRecoveryMocks.deletePodIfExists.mock.calls.map(
+ call => call[1] as string
+ )
+ expect(deletedPods).toEqual([
+ 'wf-run-7-coordinator',
+ 'wf-run-7-mcp-host',
+ 'wf-run-7-snippet-runner',
+ ])
+ })
+
it('is idempotent: re-running after pods are gone is a no-op (404-safe)', async () => {
+ crashRecoveryMocks.getPodPhase.mockResolvedValue(undefined)
const reconciler = new WorkflowReconciler(deps)
- // deletePodIfExists already swallows 404 internally; a second pass must not throw.
await reconciler.teardownComputePodsForTerminalRun('wf-run-7')
+ expect(crashRecoveryMocks.deletePodIfExists).not.toHaveBeenCalled()
await expect(
reconciler.teardownComputePodsForTerminalRun('wf-run-7')
).resolves.toBeUndefined()
diff --git a/workflow-recipes/src/workflow/workflowReconciler.ts b/workflow-recipes/src/workflow/workflowReconciler.ts
index 15a07ec94..93412c5c2 100644
--- a/workflow-recipes/src/workflow/workflowReconciler.ts
+++ b/workflow-recipes/src/workflow/workflowReconciler.ts
@@ -2929,8 +2929,9 @@ export class WorkflowReconciler {
* so dropping mcp-host/coordinator/snippet-runner here frees CPU/RAM without
* losing any artifacts.
*
- * Idempotent: deleting an already-gone pod is a 404 no-op (deletePodIfExists).
- * Deliberately EXCLUDES `workflow-artifact-reader`.
+ * Skip DELETE when the same-pass GET is already 404. A live pod with an empty
+ * status.phase is present and is still deleted. Deliberately EXCLUDES
+ * `workflow-artifact-reader`.
*
* The component list is an explicit literal (not derived from
* `runtime.cleanup.*`) on purpose: the "artifact-reader is preserved"
@@ -2945,7 +2946,24 @@ export class WorkflowReconciler {
'workflow-mcp-host',
'workflow-snippet-runner',
]
+ const namespace = this.deps.config.sandboxNamespace
+ const podSuffixByComponent: Record<WorkflowRuntimeComponent, string> = {
+ 'workflow-coordinator': 'coordinator',
+ 'workflow-mcp-host': 'mcp-host',
+ 'workflow-artifact-reader': 'artifact-reader',
+ 'workflow-snippet-runner': 'snippet-runner',
+ }
for (const component of computeComponents) {
+ const podName = `${recipeName}-${podSuffixByComponent[component]}`
+ const presence = await getPodPresence(this.deps.coreApi, podName, namespace)
+ if (presence.kind === 'absent') {
+ this.log.info('Skipping terminal compute-pod DELETE; same-pass GET was absent', {
+ recipe: recipeName,
+ component,
+ presence: presence.kind,
+ })
+ continue
+ }
await this.deleteRuntimeComponentIfExists(recipeName, component)
}
}