Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
fix(workflow-recipes): revert terminal GET-before-DELETE and fail-loud prune errors
Alfredo Lopezbf7f949f06e1
diff --git a/.cursor/skills/minikube-t0-t1-t2/reference.md b/.cursor/skills/minikube-t0-t1-t2/reference.mdindex 59910e8b4..69344352e 100644--- a/.cursor/skills/minikube-t0-t1-t2/reference.md+++ b/.cursor/skills/minikube-t0-t1-t2/reference.md@@ -14,7 +14,8 @@ Companion to `SKILL.md`. Source of truth: `scripts/minikube/t2.sh`,- **Do not re-run full `make minikube-t2` to "close T2"** when T0 and T1 arealready green on the same HEAD — use `make minikube-t2-runtime`.- **Do not derive a new profile from the current HEAD.** Resolve the profile- through the primary checkout helper. Worktree + branch own profile identity;+ through this worktree's `scripts/minikube-profiles/branch.mk`. Worktree ++ branch own profile identity;the exact-head marker owns deployed freshness and `ports.env` is persistent.- **Do not `ls`/`cat` `~/.cache/clerum/minikube-profiles/`.** Private profilestate (ports, pids, markers). The harness is the only reader.@@ -128,9 +129,9 @@ Forbidden: new profile, PVC reset, `docker desktop restart`, `docker run`probes, treating Error 127 as GFS/`PROFILE_UNHEALTHY`.`PROFILE_METADATA_MISSING` (unreadable `profile.env` / ports) is raised as-`PROFILE_OWNERSHIP_MISMATCH`. Resolve or generate metadata via the primary-checkout `branch.mk` (`branch-profile-start`). Never invent ports or-`ls`/`cat` `~/.cache/clerum/minikube-profiles/`.+`PROFILE_OWNERSHIP_MISMATCH`. Resolve or generate metadata via this+worktree's `scripts/minikube-profiles/branch.mk` (`branch-profile-start`).+Never invent ports or `ls`/`cat` `~/.cache/clerum/minikube-profiles/`.## Stable failure codesdiff --git a/docs/testing/minikube-t2-runbook.md b/docs/testing/minikube-t2-runbook.mdindex 7c9438cc5..3459e6f70 100644--- a/docs/testing/minikube-t2-runbook.md+++ b/docs/testing/minikube-t2-runbook.md@@ -46,7 +46,7 @@ matches HEAD and the current source fingerprint. `make minikube-pre-gate-sync`reconciles the profile; it does not emit a T2 verdict.The profile helper that generated the profile remains the source of truth for-the profile metadata and random localhost port mapping. Resolve it from the+the profile metadata and random localhost port mapping. Resolve it fromthis worktree's `scripts/minikube-profiles/branch.mk`. A legacy creationSHA is historical metadata; it does not override stable worktree+branchownership. Persisted `ports.env` is allocated once. Missing, corrupt, ordiff --git a/scripts/dev/repo-intake-packet.sh b/scripts/dev/repo-intake-packet.shindex 671ce62b0..2db8e6f16 100755--- a/scripts/dev/repo-intake-packet.sh+++ b/scripts/dev/repo-intake-packet.sh@@ -266,13 +266,13 @@ PROFILE_OWNER_SCRIPT="${SCRIPT_ROOT}/scripts/minikube/profile-owner.sh"PROFILE_OWNER_AVAILABLE="no"[[ -f "${LOCAL_PROFILE_HELPER}" ]] && LOCAL_HELPER_EXISTS="yes"-if [[ -f "${PRIMARY_PROFILE_HELPER}" ]]; then- PRIMARY_HELPER_EXISTS="yes"- PROFILE_HELPER_EXISTS="yes"- PROFILE_HELPER_COMMAND="make -f ${PRIMARY_PROFILE_HELPER} branch-profile-info"-elif [[ "${LOCAL_HELPER_EXISTS}" == "yes" ]]; then+[[ -f "${PRIMARY_PROFILE_HELPER}" ]] && PRIMARY_HELPER_EXISTS="yes"+if [[ "${LOCAL_HELPER_EXISTS}" == "yes" ]]; thenPROFILE_HELPER_EXISTS="yes"PROFILE_HELPER_COMMAND="make -f ${LOCAL_PROFILE_HELPER} branch-profile-info"+elif [[ "${PRIMARY_HELPER_EXISTS}" == "yes" ]]; then+ PROFILE_HELPER_EXISTS="yes"+ PROFILE_HELPER_COMMAND="make -f ${PRIMARY_PROFILE_HELPER} branch-profile-info"elseBLOCKERS+=("missing_branch_profile_helper")fidiff --git a/scripts/minikube-profiles/branch-profile.sh b/scripts/minikube-profiles/branch-profile.shindex 09239bef7..b5f4b648d 100755--- a/scripts/minikube-profiles/branch-profile.sh+++ b/scripts/minikube-profiles/branch-profile.sh@@ -592,7 +592,8 @@ load_port_forward_owner() {pf_owner_process_command \pf_owner_command_matches \pf_owner_abort_child \- pf_owner_pause; do+ pf_owner_pause \+ pf_owner_remove_dead_record; dodeclare -F "${function_name}" >/dev/null ||die "PORT_FORWARD_OWNER_API_INVALID: missing ${function_name} in ${PORT_FORWARD_OWNER_SCRIPT}"done@@ -793,21 +794,32 @@ cmd_prepare_shims() {require_command git shasum perlpersist_statemkdir -p "${CACHE_DIR}/scripts"+ if [[ -L "${SHIMS_DIR}" || -L "${DEPLOY_SHIM_DIR}" ]]; then+ die "BRANCH_PROFILE_SHIM_SYMLINK: refusing to replace symlink shim dirs"+ fi+ umask 077rm -rf "${SHIMS_DIR}" "${DEPLOY_SHIM_DIR}"cp -R "${REPO_DIR}/scripts/minikube" "${SHIMS_DIR}"cp -R "${REPO_DIR}/deploy" "${DEPLOY_SHIM_DIR}"local filewhile IFS= read -r file; do- perl -0pi -e 's#PROJECT_DIR="\$\(cd "\$SCRIPT_DIR/\.\./\.\." && pwd\)"#PROJECT_DIR="\${CLERUM_PROJECT_DIR:-\$(cd \"\$SCRIPT_DIR/../..\" && pwd)}"#g; s#PROJECT_DIR="\$\(cd "\$\{SCRIPT_DIR\}/\.\./\.\." && pwd\)"#PROJECT_DIR="\${CLERUM_PROJECT_DIR:-\$(cd \"\${SCRIPT_DIR}/../..\" && pwd)}"#g; s#PROFILE="clerum-test"#PROFILE="\${MINIKUBE_PROFILE:-clerum-test}"#g; s#--context=clerum-test#--context=\${PROFILE}#g' "${file}"+ perl -0pi -e 's#PROJECT_DIR="\$\(cd "\$SCRIPT_DIR/\.\./\.\." && pwd\)"#PROJECT_DIR="\${CLERUM_PROJECT_DIR:-\$(cd \"\$SCRIPT_DIR/../..\" && pwd)}"#g; s#PROJECT_DIR="\$\(cd "\$\{SCRIPT_DIR\}/\.\./\.\." && pwd\)"#PROJECT_DIR="\${CLERUM_PROJECT_DIR:-\$(cd \"\${SCRIPT_DIR}/../..\" && pwd)}"#g; s#PROFILE="clerum-test"#PROFILE="\${MINIKUBE_PROFILE:-clerum-test}"#g; s#--context=clerum-test#--context=\${PROFILE}#g' "${file}" ||+ die "BRANCH_PROFILE_SHIM_REWRITE_FAILED: ${file}"done < <(find "${SHIMS_DIR}" -type f -name '*.sh' | sort)- perl -0pi -e 's#OUTPUT="\$\{PROJECT_DIR\}/deploy/minikube/secrets/jwt-signing-keys.yaml"#OUTPUT="\${BRANCH_PROFILE_DEPLOY_DIR:-\${PROJECT_DIR}/deploy}/minikube/secrets/jwt-signing-keys.yaml"#g' "${SHIMS_DIR}/generate-keys.sh"- perl -0pi -e 's#REPO_ROOT="\$\(cd "\$\{SCRIPT_DIR\}/\.\./\.\." && pwd\)"#REPO_ROOT="\${CLERUM_PROJECT_DIR:-\$(cd \"\${SCRIPT_DIR}/../..\" && pwd)}"#g' "${SHIMS_DIR}/seed-test-data.sh"- perl -0pi -e 's#: "\$\{CONTEXT:=\$\(kubectl config current-context\)\}"#: "\${CONTEXT:=\${MINIKUBE_PROFILE:-\$(kubectl config current-context)}}"#g; s#export ADMIN_PASSWORD E2E_TEST_EMAIL E2E_TEST_PASSWORD CONTEXT#: "\${ALLOWED_CONTEXTS:=\${CONTEXT}}"\nexport ADMIN_PASSWORD E2E_TEST_EMAIL E2E_TEST_PASSWORD CONTEXT ALLOWED_CONTEXTS#g; s#export ADMIN_PASSWORD E2E_DEV_LOGIN_EMAIL CONTEXT#: "\${ALLOWED_CONTEXTS:=\${CONTEXT}}"\nexport ADMIN_PASSWORD E2E_DEV_LOGIN_EMAIL CONTEXT ALLOWED_CONTEXTS#g' "${SHIMS_DIR}/seed-test-data.sh"- perl -0pi -e 's#MANIFEST_FILE="\$\{PROJECT_DIR\}/deploy/minikube/\.image-manifest.json"#MANIFEST_FILE="\${BRANCH_PROFILE_DEPLOY_DIR:-\${PROJECT_DIR}/deploy}/minikube/.image-manifest.json"#g' "${SHIMS_DIR}/build-images.sh"- perl -0pi -e 's#BASE_MINIKUBE_KUSTOMIZE_DIR="\$\{PROJECT_DIR\}/deploy/overlays/minikube"#BASE_MINIKUBE_KUSTOMIZE_DIR="\${BRANCH_PROFILE_DEPLOY_DIR:-\${PROJECT_DIR}/deploy}/overlays/minikube"#g; s#LOCAL_MEMBER_REGISTRATION_KUSTOMIZE_DIR="\$\{PROJECT_DIR\}/deploy/overlays/minikube-local-member-registration"#LOCAL_MEMBER_REGISTRATION_KUSTOMIZE_DIR="\${BRANCH_PROFILE_DEPLOY_DIR:-\${PROJECT_DIR}/deploy}/overlays/minikube-local-member-registration"#g' "${SHIMS_DIR}/full-setup.sh"- perl -0pi -e 's#CONTEXT="\$\{PROFILE\}" "\$\{PROJECT_DIR\}/deploy/scripts/minikube-detect-k8s-api-ip.sh"#CONTEXT="\${PROFILE}" OVERLAY_DIR="\${BRANCH_PROFILE_DEPLOY_DIR:-\${PROJECT_DIR}/deploy}/overlays/minikube" "\${PROJECT_DIR}/deploy/scripts/minikube-detect-k8s-api-ip.sh"#g; s#kubectl kustomize "\$\{PROJECT_DIR\}/deploy/overlays/minikube"#kubectl kustomize "\${BRANCH_PROFILE_DEPLOY_DIR:-\${PROJECT_DIR}/deploy}/overlays/minikube"#g' "${SHIMS_DIR}/full-setup.sh"+ perl -0pi -e 's#OUTPUT="\$\{PROJECT_DIR\}/deploy/minikube/secrets/jwt-signing-keys.yaml"#OUTPUT="\${BRANCH_PROFILE_DEPLOY_DIR:-\${PROJECT_DIR}/deploy}/minikube/secrets/jwt-signing-keys.yaml"#g' "${SHIMS_DIR}/generate-keys.sh" ||+ die "BRANCH_PROFILE_SHIM_REWRITE_FAILED: generate-keys.sh"+ perl -0pi -e 's#REPO_ROOT="\$\(cd "\$\{SCRIPT_DIR\}/\.\./\.\." && pwd\)"#REPO_ROOT="\${CLERUM_PROJECT_DIR:-\$(cd \"\${SCRIPT_DIR}/../..\" && pwd)}"#g' "${SHIMS_DIR}/seed-test-data.sh" ||+ die "BRANCH_PROFILE_SHIM_REWRITE_FAILED: seed-test-data.sh REPO_ROOT"+ perl -0pi -e 's#: "\$\{CONTEXT:=\$\(kubectl config current-context\)\}"#: "\${CONTEXT:=\${MINIKUBE_PROFILE:-\$(kubectl config current-context)}}"#g; s#export ADMIN_PASSWORD E2E_TEST_EMAIL E2E_TEST_PASSWORD CONTEXT#: "\${ALLOWED_CONTEXTS:=\${CONTEXT}}"\nexport ADMIN_PASSWORD E2E_TEST_EMAIL E2E_TEST_PASSWORD CONTEXT ALLOWED_CONTEXTS#g; s#export ADMIN_PASSWORD E2E_DEV_LOGIN_EMAIL CONTEXT#: "\${ALLOWED_CONTEXTS:=\${CONTEXT}}"\nexport ADMIN_PASSWORD E2E_DEV_LOGIN_EMAIL CONTEXT ALLOWED_CONTEXTS#g' "${SHIMS_DIR}/seed-test-data.sh" ||+ die "BRANCH_PROFILE_SHIM_REWRITE_FAILED: seed-test-data.sh CONTEXT"+ perl -0pi -e 's#MANIFEST_FILE="\$\{PROJECT_DIR\}/deploy/minikube/\.image-manifest.json"#MANIFEST_FILE="\${BRANCH_PROFILE_DEPLOY_DIR:-\${PROJECT_DIR}/deploy}/minikube/.image-manifest.json"#g' "${SHIMS_DIR}/build-images.sh" ||+ die "BRANCH_PROFILE_SHIM_REWRITE_FAILED: build-images.sh"+ perl -0pi -e 's#BASE_MINIKUBE_KUSTOMIZE_DIR="\$\{PROJECT_DIR\}/deploy/overlays/minikube"#BASE_MINIKUBE_KUSTOMIZE_DIR="\${BRANCH_PROFILE_DEPLOY_DIR:-\${PROJECT_DIR}/deploy}/overlays/minikube"#g; s#LOCAL_MEMBER_REGISTRATION_KUSTOMIZE_DIR="\$\{PROJECT_DIR\}/deploy/overlays/minikube-local-member-registration"#LOCAL_MEMBER_REGISTRATION_KUSTOMIZE_DIR="\${BRANCH_PROFILE_DEPLOY_DIR:-\${PROJECT_DIR}/deploy}/overlays/minikube-local-member-registration"#g' "${SHIMS_DIR}/full-setup.sh" ||+ die "BRANCH_PROFILE_SHIM_REWRITE_FAILED: full-setup.sh overlays"+ perl -0pi -e 's#CONTEXT="\$\{PROFILE\}" "\$\{PROJECT_DIR\}/deploy/scripts/minikube-detect-k8s-api-ip.sh"#CONTEXT="\${PROFILE}" OVERLAY_DIR="\${BRANCH_PROFILE_DEPLOY_DIR:-\${PROJECT_DIR}/deploy}/overlays/minikube" "\${PROJECT_DIR}/deploy/scripts/minikube-detect-k8s-api-ip.sh"#g; s#kubectl kustomize "\$\{PROJECT_DIR\}/deploy/overlays/minikube"#kubectl kustomize "\${BRANCH_PROFILE_DEPLOY_DIR:-\${PROJECT_DIR}/deploy}/overlays/minikube"#g' "${SHIMS_DIR}/full-setup.sh" ||+ die "BRANCH_PROFILE_SHIM_REWRITE_FAILED: full-setup.sh detect"chmod +x "${SHIMS_DIR}"/*.shcat >"${SHIM_ENV}" <<EOF_SHIMSdiff --git a/scripts/tests/test-repo-intake-packet.sh b/scripts/tests/test-repo-intake-packet.shindex 0034e8c2a..9d21919b6 100755--- a/scripts/tests/test-repo-intake-packet.sh+++ b/scripts/tests/test-repo-intake-packet.sh@@ -122,7 +122,7 @@ git -C "${MAIN_REPO}" config user.name "Clerum Test"printf '{"scripts":{"test":"true"}}\n' >"${MAIN_REPO}/package.json"mkdir -p "${MAIN_REPO}/scripts/minikube-profiles"-printf '# test helper\n' >"${MAIN_REPO}/scripts/minikube-profiles/branch.mk"+printf '# worktree-local helper (scripts/minikube-profiles)\n' >"${MAIN_REPO}/scripts/minikube-profiles/branch.mk"git -C "${MAIN_REPO}" add package.jsongit -C "${MAIN_REPO}" commit -q -m "initial"git -C "${MAIN_REPO}" update-ref refs/remotes/origin/dev HEADdiff --git a/workflow-recipes/package-lock.json b/workflow-recipes/package-lock.jsonindex b5d1dab41..7d7266deb 100644--- a/workflow-recipes/package-lock.json+++ b/workflow-recipes/package-lock.json@@ -1,12 +1,12 @@{"name": "workload-recipes-controller",- "version": "0.1.408",+ "version": "0.1.409","lockfileVersion": 3,"requires": true,"packages": {"": {"name": "workload-recipes-controller",- "version": "0.1.408",+ "version": "0.1.409","license": "SEE LICENSE IN LICENSE","dependencies": {"@aws-sdk/client-s3": "^3.1009.0",diff --git a/workflow-recipes/package.json b/workflow-recipes/package.jsonindex 5fd7b25f3..8b8931026 100644--- a/workflow-recipes/package.json+++ b/workflow-recipes/package.json@@ -1,6 +1,6 @@{"name": "workload-recipes-controller",- "version": "0.1.408",+ "version": "0.1.409","description": "Workload Recipes Controller (WRC) - WorkloadRecipe lifecycle management and MCP server interface","main": "dist/main.js","engines": {diff --git a/workflow-recipes/src/k8sClient.ts b/workflow-recipes/src/k8sClient.tsindex cbca042b3..bb09f0cfe 100644--- a/workflow-recipes/src/k8sClient.ts+++ b/workflow-recipes/src/k8sClient.ts@@ -911,8 +911,8 @@ export class WorkflowRecipeWatcher implements WorkflowRecipeProvider {this.secretReverseIndex,name => this.triggerSecretDrivenReconcile(name),10_000,- // Invalidate only. SecretWatcher debounces the recipe enqueue so a- // reconnect ADDED replay cannot force an immediate undebounced storm.+ // Invalidate only. SecretWatcher must not enqueue on oauth ADDED:+ // a watch reconnect replays ADDED for every broker token.recipeName => {this.reconciler.invalidateOAuthBrokerDeleteLedger(recipeName)}diff --git a/workflow-recipes/src/reconciler/secretWatcher.test.ts b/workflow-recipes/src/reconciler/secretWatcher.test.tsindex 6e0bd28f2..dd9f5e85c 100644--- a/workflow-recipes/src/reconciler/secretWatcher.test.ts+++ b/workflow-recipes/src/reconciler/secretWatcher.test.ts@@ -246,7 +246,8 @@ describe('SecretWatcher', () => {expect(invalidated).toEqual(['test-recipe', 'test-recipe'])expect(enqueued).toEqual([])vi.advanceTimersByTime(DEBOUNCE_MS)- expect(enqueued).toEqual(['test-recipe'])+ // E.4(a): invalidate only. A watch replay must not enqueue every recipe.+ expect(enqueued).toEqual([])watcher.stop()})diff --git a/workflow-recipes/src/reconciler/secretWatcher.ts b/workflow-recipes/src/reconciler/secretWatcher.tsindex e4518d634..e36325db3 100644--- a/workflow-recipes/src/reconciler/secretWatcher.ts+++ b/workflow-recipes/src/reconciler/secretWatcher.ts@@ -62,13 +62,13 @@ export class SecretWatcher {// B3(a): ADDED of the canonical broker token invalidates the ledger even// when the key-set matches a previous observation (dedup would drop it).- // Enqueue the recipe through the same debounce as key-set fan-out so a- // reconnect replay cannot force an immediate undebounced storm.+ // Invalidate only — a watch reconnect replays ADDED for every Secret and+ // must not force a reconcile storm. The 30s ownership backstop (and any+ // later key-set/ownership fan-out) issues the next delete.if (type === 'ADDED') {const recipeName = resolveOAuthBrokerTokenWatchRecipe(name, secret.metadata?.labels)if (recipeName) {this.onOAuthBrokerTokenAdded?.(recipeName)- this.scheduleRecipeReconcile(recipeName)}}@@ -133,19 +133,6 @@ export class SecretWatcher {}, this.debounceMs)this.debounceTimers.set(secretName, timer)}-- /** Debounce enqueue by recipe name (oauth-broker ADDED is not reverse-indexed). */- private scheduleRecipeReconcile(recipeName: string): void {- const key = `oauth-broker:${recipeName}`- const existing = this.debounceTimers.get(key)- if (existing) clearTimeout(existing)-- const timer = setTimeout(() => {- this.debounceTimers.delete(key)- this.enqueueReconcile(recipeName)- }, this.debounceMs)- this.debounceTimers.set(key, timer)- }}function setsEqual(a: ReadonlySet<string>, b: ReadonlySet<string>): boolean {diff --git a/workflow-recipes/src/reconciler/workflowRecipeReconciler.ts b/workflow-recipes/src/reconciler/workflowRecipeReconciler.tsindex 61bca6a7e..2d19154f6 100644--- a/workflow-recipes/src/reconciler/workflowRecipeReconciler.ts+++ b/workflow-recipes/src/reconciler/workflowRecipeReconciler.ts@@ -3940,7 +3940,12 @@ export class WorkflowRecipeReconciler {if (recipe.spec.pluginWorkloadSdk || recipe.status?.pluginWorkloadSdk) {await this.cleanupPluginWorkloadSdkOrThrow(name, { preserveWorkflowRuntime: true })}- await this.workflowReconciler.reconcileDelete(name, recipe.metadata.namespace, recipe.spec)+ await this.workflowReconciler.reconcileDelete(+ name,+ recipe.metadata.namespace,+ recipe.spec,+ recipe.metadata.uid+ )await this.cleanupDelegationIfNeeded(recipe)await this.cleanupDeclaredRuntimeResources(recipe)returndiff --git a/workflow-recipes/src/workflow/crashRecovery.test.ts b/workflow-recipes/src/workflow/crashRecovery.test.tsindex b01ca851a..572a42c88 100644--- a/workflow-recipes/src/workflow/crashRecovery.test.ts+++ b/workflow-recipes/src/workflow/crashRecovery.test.ts@@ -135,7 +135,7 @@ describe('getPodPresence', () => {})describe('waitForPodDeletion', () => {- it('returns true once getPodPhase reports the pod is gone', async () => {+ it('returns true once getPodPresence reports the pod is gone', async () => {const readNamespacedPod = vi.fn().mockResolvedValueOnce({ status: { phase: 'Running' } })@@ -152,4 +152,20 @@ describe('waitForPodDeletion', () => {).resolves.toBe(true)expect(readNamespacedPod).toHaveBeenCalledTimes(2)})++ it('does not treat a GET 200 with empty status.phase as deleted', async () => {+ const readNamespacedPod = vi+ .fn()+ .mockResolvedValue({ metadata: { name: 'recipe-mcp-host' }, status: {} })+ const coreApi = {+ readNamespacedPod,+ } as unknown as Parameters<typeof waitForPodDeletion>[0]++ await expect(+ waitForPodDeletion(coreApi, 'recipe-mcp-host', 'sandbox-recipes', {+ timeoutMs: 20,+ pollIntervalMs: 5,+ })+ ).resolves.toBe(false)+ })})diff --git a/workflow-recipes/src/workflow/runtimePlan.ts b/workflow-recipes/src/workflow/runtimePlan.tsindex 63f2f3d08..8cce99456 100644--- a/workflow-recipes/src/workflow/runtimePlan.ts+++ b/workflow-recipes/src/workflow/runtimePlan.ts@@ -370,6 +370,8 @@ export function deriveWorkflowRuntimePlan(: undefined,},cleanup: {+ // Awaiting-trigger tears down only coordinator + mcp-host. Snippet-runner+ // is run-scoped and is not in this list (B3(d) does not skip it here).deleteBeforeTriggeredRun: ['workflow-coordinator',...(needsMcpHost ? (['workflow-mcp-host'] as const) : []),diff --git a/workflow-recipes/src/workflow/workflowReconciler.test.ts b/workflow-recipes/src/workflow/workflowReconciler.test.tsindex b68603042..96a1b3079 100644--- a/workflow-recipes/src/workflow/workflowReconciler.test.ts+++ b/workflow-recipes/src/workflow/workflowReconciler.test.ts@@ -1276,28 +1276,7 @@ describe('WorkflowReconciler.reconcileDelete — orphaned Service cleanup', () =})describe('teardownComputePodsForTerminalRun', () => {- afterEach(() => {- crashRecoveryMocks.getPodPresence.mockImplementation(- async (api: unknown, name: string, namespace: string) => {- const phase = await crashRecoveryMocks.getPodPhase(api, name, namespace)- return phase === undefined- ? { kind: 'absent' as const }- : { kind: 'present' as const, phase }- }- )- })-it('deletes coordinator, mcp-host, and snippet-runner but PRESERVES artifact-reader', async () => {- crashRecoveryMocks.getPodPhase.mockImplementation(async (_api, name: string) => {- if (- name === 'wf-run-7-coordinator' ||- name === 'wf-run-7-mcp-host' ||- name === 'wf-run-7-snippet-runner'- ) {- return 'Succeeded'- }- return undefined- })const reconciler = new WorkflowReconciler(deps)await reconciler.teardownComputePodsForTerminalRun('wf-run-7')@@ -1314,41 +1293,27 @@ describe('WorkflowReconciler.reconcileDelete — orphaned Service cleanup', () =for (const call of crashRecoveryMocks.deletePodIfExists.mock.calls) {expect(call[2]).toBe(sandboxNamespace)}+ expect(crashRecoveryMocks.getPodPresence).not.toHaveBeenCalled()})- it('skips DELETE when the same-pass GET is already 404', async () => {- crashRecoveryMocks.getPodPhase.mockResolvedValue(undefined)- const reconciler = new WorkflowReconciler(deps)-- await reconciler.teardownComputePodsForTerminalRun('wf-run-7')-- expect(crashRecoveryMocks.deletePodIfExists).not.toHaveBeenCalled()- })-- it('still deletes a live pod whose status.phase is empty', async () => {- crashRecoveryMocks.getPodPresence.mockResolvedValue({ kind: 'present' })+ it('is DELETE-first and 404-safe even when pods are already gone', async () => {+ crashRecoveryMocks.deletePodIfExists.mockResolvedValue(undefined)const reconciler = new WorkflowReconciler(deps)await reconciler.teardownComputePodsForTerminalRun('wf-run-7')- const deletedPods = crashRecoveryMocks.deletePodIfExists.mock.calls.map(- call => call[1] as string- )- expect(deletedPods).toEqual([- 'wf-run-7-coordinator',- 'wf-run-7-mcp-host',- 'wf-run-7-snippet-runner',- ])+ expect(crashRecoveryMocks.deletePodIfExists).toHaveBeenCalledTimes(3)+ expect(crashRecoveryMocks.getPodPresence).not.toHaveBeenCalled()})- it('is idempotent: re-running after pods are gone is a no-op (404-safe)', async () => {- crashRecoveryMocks.getPodPhase.mockResolvedValue(undefined)+ it('is idempotent: a second pass still DELETE-first (404-safe)', async () => {const reconciler = new WorkflowReconciler(deps)await reconciler.teardownComputePodsForTerminalRun('wf-run-7')- expect(crashRecoveryMocks.deletePodIfExists).not.toHaveBeenCalled()+ crashRecoveryMocks.deletePodIfExists.mockClear()await expect(reconciler.teardownComputePodsForTerminalRun('wf-run-7')).resolves.toBeUndefined()+ expect(crashRecoveryMocks.deletePodIfExists).toHaveBeenCalledTimes(3)})})})diff --git a/workflow-recipes/src/workflow/workflowReconciler.ts b/workflow-recipes/src/workflow/workflowReconciler.tsindex 93412c5c2..7d19661e0 100644--- a/workflow-recipes/src/workflow/workflowReconciler.ts+++ b/workflow-recipes/src/workflow/workflowReconciler.ts@@ -1131,7 +1131,7 @@ export class WorkflowReconciler {snapshot: CodexCatalogSnapshot} = { snapshot: { flagEnabled: false } }private readonly codexContexts = new Map<string, CodexReconcileContext>()- /** B3(c): one process-lifetime DELETE of the leftover mcp-servers internet NP. */+ /** B3(c): one process-lifetime DELETE of the leftover mcp-servers internet NP. Keyed by recipe uid. */private readonly prunedLegacyMcpServersInternetEgress = new Set<string>()constructor(private readonly deps: WorkflowReconcilerDeps) {@@ -1886,7 +1886,7 @@ export class WorkflowReconciler {// pod with an empty status.phase is present and must still be deleted.const presence = samePassPresence[component]if (presence?.kind === 'absent') {- log.info('Skipping runtime component DELETE; same-pass GET was absent', {+ log.debug('Skipping runtime component DELETE; same-pass GET was absent', {recipe: recipeName,component,presence: presence.kind,@@ -2929,8 +2929,9 @@ export class WorkflowReconciler {* so dropping mcp-host/coordinator/snippet-runner here frees CPU/RAM without* losing any artifacts.*- * Skip DELETE when the same-pass GET is already 404. A live pod with an empty- * status.phase is present and is still deleted. Deliberately EXCLUDES+ * DELETE-first and 404-safe (`deletePodIfExists`). This path is outside B3+ * (E.4(e), §7.16): do not add a GET just to skip the DELETE. The archive+ * TTL still owns leftover objects. Deliberately EXCLUDES* `workflow-artifact-reader`.** The component list is an explicit literal (not derived from@@ -2946,24 +2947,7 @@ export class WorkflowReconciler {'workflow-mcp-host','workflow-snippet-runner',]- const namespace = this.deps.config.sandboxNamespace- const podSuffixByComponent: Record<WorkflowRuntimeComponent, string> = {- 'workflow-coordinator': 'coordinator',- 'workflow-mcp-host': 'mcp-host',- 'workflow-artifact-reader': 'artifact-reader',- 'workflow-snippet-runner': 'snippet-runner',- }for (const component of computeComponents) {- const podName = `${recipeName}-${podSuffixByComponent[component]}`- const presence = await getPodPresence(this.deps.coreApi, podName, namespace)- if (presence.kind === 'absent') {- this.log.info('Skipping terminal compute-pod DELETE; same-pass GET was absent', {- recipe: recipeName,- component,- presence: presence.kind,- })- continue- }await this.deleteRuntimeComponentIfExists(recipeName, component)}}@@ -3222,9 +3206,17 @@ export class WorkflowReconciler {}}- private async pruneLegacyMcpServersInternetEgressPolicy(recipeName: string): Promise<void> {- if (this.prunedLegacyMcpServersInternetEgress.has(recipeName)) {- this.log.info('Skipping legacy mcp-servers internet NP delete; already observed gone', {+ private legacyMcpServersInternetEgressKey(recipeName: string, recipeUid?: string): string {+ return recipeUid && recipeUid.length > 0 ? recipeUid : recipeName+ }++ private async pruneLegacyMcpServersInternetEgressPolicy(+ recipeName: string,+ recipeUid?: string+ ): Promise<void> {+ const key = this.legacyMcpServersInternetEgressKey(recipeName, recipeUid)+ if (this.prunedLegacyMcpServersInternetEgress.has(key)) {+ this.log.debug('Skipping legacy mcp-servers internet NP delete; already observed gone', {recipe: recipeName,})return@@ -3235,12 +3227,20 @@ export class WorkflowReconciler {namespace: this.deps.config.mcpServerNamespace,}))+ if (outcome.kind === 'failed') {+ this.log.error('Legacy mcp-servers internet NP delete failed', {+ recipe: recipeName,+ ...deleteOutcomeFields(outcome),+ error: outcome.error instanceof Error ? outcome.error.message : String(outcome.error),+ })+ return+ }this.log.info('Legacy mcp-servers internet NP delete', {recipe: recipeName,...deleteOutcomeFields(outcome),})if (shouldRecordDelete(outcome)) {- this.prunedLegacyMcpServersInternetEgress.add(recipeName)+ this.prunedLegacyMcpServersInternetEgress.add(key)}}@@ -3420,11 +3420,17 @@ export class WorkflowReconciler {)const summary = await this.applyNetworkPolicyList(policies)const policyNames = new Set(policies.map(policy => policy.metadata?.name))- await this.pruneUndesiredRunLaneNetworkPolicies(recipeName, policyNames, catalog, {- skipCodexProxy: codexProjection.eligibility === 'uncertain',- skipGrokProxy: grokProjection?.eligibility === 'uncertain',- })- await this.pruneLegacyMcpServersInternetEgressPolicy(recipeName)+ const prune = await this.pruneUndesiredRunLaneNetworkPolicies(+ recipeName,+ policyNames,+ catalog,+ {+ skipCodexProxy: codexProjection.eligibility === 'uncertain',+ skipGrokProxy: grokProjection?.eligibility === 'uncertain',+ }+ )+ if (prune.retryPending) summary.retryPending = true+ await this.pruneLegacyMcpServersInternetEgressPolicy(recipeName, recipeUid)return summary}@@ -3441,20 +3447,30 @@ export class WorkflowReconciler {desiredNames: Set<string | undefined>,catalog: Set<string>,uncertain: { skipCodexProxy: boolean; skipGrokProxy: boolean }- ): Promise<void> {+ ): Promise<{ retryPending: boolean }> {const namespace = this.deps.config.sandboxNamespaceconst labelSelector = `clerum.io/recipe=${recipeName},clerum.io/managed-by=wrc`- const list = await this.deps.networkingApi.listNamespacedNetworkPolicy({- namespace,- labelSelector,- })+ let list: { items?: Array<{ metadata?: { name?: string } }> }+ try {+ list = await this.deps.networkingApi.listNamespacedNetworkPolicy({+ namespace,+ labelSelector,+ })+ } catch (error: unknown) {+ this.log.error('Run-lane NP prune LIST failed; apply outcome is kept', {+ recipe: recipeName,+ error: error instanceof Error ? error.message : String(error),+ })+ return { retryPending: true }+ }const codexProxyPolicyName = `${recipeName}-mcp-host-to-codex-proxy`const grokProxyPolicyName = `${recipeName}-mcp-host-to-grok-proxy`+ let retryPending = falsefor (const policy of list.items ?? []) {const name = policy.metadata?.nameif (!name || desiredNames.has(name)) continueif (name === codexProxyPolicyName && uncertain.skipCodexProxy) {- this.log.info('Skipping run-lane NP prune', {+ this.log.debug('Skipping run-lane NP prune', {recipe: recipeName,policy: name,reason: 'uncertain',@@ -3462,7 +3478,7 @@ export class WorkflowReconciler {continue}if (name === grokProxyPolicyName && uncertain.skipGrokProxy) {- this.log.info('Skipping run-lane NP prune', {+ this.log.debug('Skipping run-lane NP prune', {recipe: recipeName,policy: name,reason: 'uncertain',@@ -3470,17 +3486,27 @@ export class WorkflowReconciler {continue}if (!catalog.has(name)) {- this.log.info('Skipping run-lane NP prune', {+ this.log.debug('Skipping run-lane NP prune', {recipe: recipeName,policy: name,reason: 'not-in-catalog',})continue}- await this.safeDelete(() =>+ const outcome = await observeNamespacedDelete(() =>this.deps.networkingApi.deleteNamespacedNetworkPolicy({ name, namespace }))+ if (outcome.kind === 'failed') {+ this.log.error('Run-lane NP prune delete failed', {+ recipe: recipeName,+ policy: name,+ ...deleteOutcomeFields(outcome),+ error: outcome.error instanceof Error ? outcome.error.message : String(outcome.error),+ })+ retryPending = true+ }}+ return { retryPending }}/**@@ -3744,12 +3770,15 @@ export class WorkflowReconciler {async reconcileDelete(recipeName: string,recipeNamespace?: string,- spec?: WorkflowRecipeSpec+ spec?: WorkflowRecipeSpec,+ recipeUid?: string): Promise<void> {const ns = this.deps.config.sandboxNamespaceconst log = createLogger('wrc', recipeName)log.info(`Deleting workflow resources`)this.pluginWorkloadSdkProvisioner.clearRecipeState(recipeName)+ this.prunedLegacyMcpServersInternetEgress.delete(recipeName)+ if (recipeUid) this.prunedLegacyMcpServersInternetEgress.delete(recipeUid)// Best-effort cleanup of the recipe's subdirectory on the workflow output PVC.// Must run BEFORE pod deletion while mcp-host is still reachable; failuresdiff --git a/workflow-recipes/tests/unit/workflow/workflowReconciler.test.ts b/workflow-recipes/tests/unit/workflow/workflowReconciler.test.tsindex f38a8f772..4ff44c5ad 100644--- a/workflow-recipes/tests/unit/workflow/workflowReconciler.test.ts+++ b/workflow-recipes/tests/unit/workflow/workflowReconciler.test.ts@@ -3062,6 +3062,34 @@ describe('WorkflowReconciler — reconcile loop', () => {expect(legacyDeletes).toHaveLength(2)})+ it('clears the legacy internet NP process set on recipe delete so a recreate can delete again', async () => {+ const networkingApi = makeNetworkingApi() as ReturnType<typeof makeNetworkingApi>+ const reconciler = new WorkflowReconciler(+ makeDeps({+ networkingApi: networkingApi as never,+ })+ )++ await reconciler.reconcile(+ 'test-wf',+ 'uid-123',+ 'sandbox-recipes',+ makeSpec({ mcpServers: ['redis-mcp'] })+ )+ networkingApi.deleteNamespacedNetworkPolicy.mockClear()+ await reconciler.reconcileDelete('test-wf', 'sandbox-recipes', makeSpec(), 'uid-123')+ await reconciler.reconcile(+ 'test-wf',+ 'uid-123',+ 'sandbox-recipes',+ makeSpec({ mcpServers: ['redis-mcp'] })+ )+ expect(networkingApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledWith({+ name: 'test-wf-mcp-servers-egress-internet',+ namespace: 'mcp-server',+ })+ })+describe('B2 desired-set NetworkPolicy prune', () => {const RECIPE = 'test-wf'const SANDBOX = 'sandbox-recipes'@@ -3069,6 +3097,7 @@ describe('WorkflowReconciler — reconcile loop', () => {const CODEX_PROXY = `${RECIPE}-mcp-host-to-codex-proxy`const GROK_PROXY = `${RECIPE}-mcp-host-to-grok-proxy`const COORD_TO_WRC = `${RECIPE}-coord-to-wrc`+ const COORD_TO_MCP_HOST = `${RECIPE}-coord-to-mcp-host`const WRC_LABELS = {'clerum.io/recipe': RECIPE,'clerum.io/managed-by': 'wrc',@@ -3126,8 +3155,9 @@ describe('WorkflowReconciler — reconcile loop', () => {codex?: Partial<CodexExecutionProjection>grok?: Partial<CodexExecutionProjection> & { requiresGrokProxyEgress?: boolean }awaitsTriggeredRun?: boolean+ eagerSdkMcpHost?: boolean} = {}- ): Promise<void> {+ ): Promise<{ retryPending: boolean }> {const spec = makeSpec()const runtime = deriveWorkflowRuntimePlan(spec, {recipeName: RECIPE,@@ -3144,7 +3174,7 @@ describe('WorkflowReconciler — reconcile loop', () => {requiresGrokProxyEgress: false,...opts.grok,}- await (+ return (await (reconciler as unknown as {applyWorkflowNetworkPolicies: (recipeName: string,@@ -3155,7 +3185,7 @@ describe('WorkflowReconciler — reconcile loop', () => {codexProjection: CodexExecutionProjection,eagerSdkMcpHost: boolean,grokProjection: typeof grok- ) => Promise<unknown>+ ) => Promise<{ retryPending: boolean }>}).applyWorkflowNetworkPolicies(RECIPE,@@ -3164,9 +3194,9 @@ describe('WorkflowReconciler — reconcile loop', () => {runtime,opts.awaitsTriggeredRun === true,codex,- false,+ opts.eagerSdkMcpHost === true,grok- )+ )) as { retryPending: boolean }}function expectListWitness(@@ -3367,6 +3397,98 @@ describe('WorkflowReconciler — reconcile loop', () => {expectListWitness(apiserver.api)expect(sandboxDeletes(apiserver.api)).toEqual([])})++ it('keeps an unlisted-as-desired Grok proxy while eligibility is uncertain, and deletes the twin', async () => {+ const uncertain = makeApiserverNetworkingApi()+ seedLivePolicy(uncertain, GROK_PROXY, WRC_LABELS)+ const uncertainReconciler = new WorkflowReconciler(+ makeDeps({ networkingApi: uncertain.api as never })+ )+ await applyPolicies(uncertainReconciler, {+ grok: { eligibility: 'uncertain', requiresGrokProxyEgress: false, reason: 'forbidden' },+ })+ expectListWitness(uncertain.api)+ expect(sandboxDeletes(uncertain.api).map(d => d.name)).not.toContain(GROK_PROXY)+ expect(uncertain.live.has(uncertain.key(SANDBOX, GROK_PROXY))).toBe(true)++ const decided = makeApiserverNetworkingApi()+ seedLivePolicy(decided, GROK_PROXY, WRC_LABELS)+ const decidedReconciler = new WorkflowReconciler(+ makeDeps({ networkingApi: decided.api as never })+ )+ await applyPolicies(decidedReconciler, {+ grok: { eligibility: 'ineligible', requiresGrokProxyEgress: false },+ })+ expectListWitness(decided.api)+ expect(sandboxDeletes(decided.api).map(d => d.name)).toContain(GROK_PROXY)+ })++ it('does not prune a catalog-named recipes-lane policy (two-term selector)', async () => {+ const apiserver = makeApiserverNetworkingApi()+ seedLivePolicy(apiserver, COORD_TO_MCP_HOST, {+ 'clerum.io/recipe': RECIPE,+ 'clerum.io/managed-by': 'workflow-recipes',+ })+ const reconciler = new WorkflowReconciler(makeDeps({ networkingApi: apiserver.api as never }))++ await applyPolicies(reconciler, { awaitsTriggeredRun: true })++ expectListWitness(apiserver.api)+ expect(sandboxDeletes(apiserver.api).map(d => d.name)).not.toContain(COORD_TO_MCP_HOST)+ expect(apiserver.live.has(apiserver.key(SANDBOX, COORD_TO_MCP_HOST))).toBe(true)+ })++ it('deletes a leftover coord-to-mcp-host when awaiting trigger', async () => {+ const apiserver = makeApiserverNetworkingApi()+ seedLivePolicy(apiserver, COORD_TO_MCP_HOST, WRC_LABELS)+ const reconciler = new WorkflowReconciler(makeDeps({ networkingApi: apiserver.api as never }))++ await applyPolicies(reconciler, { awaitsTriggeredRun: true })++ expectListWitness(apiserver.api)+ expect(sandboxDeletes(apiserver.api).map(d => d.name)).toContain(COORD_TO_MCP_HOST)+ expect(apiserver.live.has(apiserver.key(SANDBOX, COORD_TO_MCP_HOST))).toBe(false)+ })++ it('eager SDK path still prunes a leftover Grok proxy', async () => {+ const apiserver = makeApiserverNetworkingApi()+ seedLivePolicy(apiserver, GROK_PROXY, WRC_LABELS)+ const reconciler = new WorkflowReconciler(makeDeps({ networkingApi: apiserver.api as never }))++ await applyPolicies(reconciler, { awaitsTriggeredRun: true, eagerSdkMcpHost: true })++ expectListWitness(apiserver.api)+ expect(sandboxDeletes(apiserver.api).map(d => d.name)).toContain(GROK_PROXY)+ })++ it('marks retryPending when a leftover prune DELETE fails', async () => {+ const apiserver = makeApiserverNetworkingApi()+ seedLivePolicy(apiserver, GROK_PROXY, WRC_LABELS)+ apiserver.api.deleteNamespacedNetworkPolicy.mockImplementation(+ async ({ name, namespace }: { name: string; namespace: string }) => {+ if (name === GROK_PROXY) throw { code: 403, message: 'forbidden' }+ if (!apiserver.live.delete(apiserver.key(namespace, name))) throw { code: 404 }+ return {}+ }+ )+ const reconciler = new WorkflowReconciler(makeDeps({ networkingApi: apiserver.api as never }))++ const summary = await applyPolicies(reconciler)+ expect(summary.retryPending).toBe(true)+ expect(apiserver.live.has(apiserver.key(SANDBOX, GROK_PROXY))).toBe(true)+ })++ it('keeps the apply outcome when the prune LIST fails', async () => {+ const apiserver = makeApiserverNetworkingApi()+ seedLivePolicy(apiserver, GROK_PROXY, WRC_LABELS)+ apiserver.api.listNamespacedNetworkPolicy.mockRejectedValue({ code: 500, message: 'boom' })+ const reconciler = new WorkflowReconciler(makeDeps({ networkingApi: apiserver.api as never }))++ const summary = await applyPolicies(reconciler)+ expect(summary.retryPending).toBe(true)+ expect(sandboxDeletes(apiserver.api)).toEqual([])+ expect(apiserver.live.has(apiserver.key(SANDBOX, GROK_PROXY))).toBe(true)+ })})describe('B3(d) skip coordinator DELETE after same-pass 404', () => {@@ -3389,7 +3511,7 @@ describe('WorkflowReconciler — reconcile loop', () => {const result = await reconciler.reconcile('test-wf', 'uid-123', SANDBOX, makeSpec())expect(result.workflowPhase).not.toBe('failed')- expect(podReads(coreApi, COORDINATOR)).toBeGreaterThan(0)+ expect(podReads(coreApi, COORDINATOR)).toBe(2)expect(podDeletes(coreApi, COORDINATOR)).toBe(0)})@@ -3403,7 +3525,7 @@ describe('WorkflowReconciler — reconcile loop', () => {await reconciler.reconcile('test-wf', 'uid-123', SANDBOX, makeSpec())- expect(podReads(coreApi, COORDINATOR)).toBeGreaterThan(0)+ expect(podReads(coreApi, COORDINATOR)).toBe(2)expect(podDeletes(coreApi, COORDINATOR)).toBe(1)})@@ -3436,7 +3558,7 @@ describe('WorkflowReconciler — reconcile loop', () => {'run-1')- expect(podReads(coreApi, COORDINATOR)).toBeGreaterThan(0)+ expect(podReads(coreApi, COORDINATOR)).toBe(3)expect(podDeletes(coreApi, COORDINATOR)).toBe(0)})@@ -3446,7 +3568,7 @@ describe('WorkflowReconciler — reconcile loop', () => {await reconciler.reconcile('test-wf', 'uid-123', SANDBOX, makeSpec())- expect(podReads(coreApi, COORDINATOR)).toBeGreaterThan(0)+ expect(podReads(coreApi, COORDINATOR)).toBe(3)expect(podDeletes(coreApi, COORDINATOR)).toBe(1)expect(coreApi.deleteNamespacedPod).toHaveBeenCalledWith({name: COORDINATOR,@@ -3479,6 +3601,21 @@ describe('WorkflowReconciler — reconcile loop', () => {expect(podReads(coreApi, MCP_HOST)).toBe(0)expect(podDeletes(coreApi, MCP_HOST)).toBe(0)+ // Snippet-only has needsMcpHost=false, so awaitsTriggeredRun is false and+ // B3(d) does not run. The coordinator is still read for crash recovery.+ expect(podReads(coreApi, COORDINATOR)).toBe(2)+ expect(podDeletes(coreApi, COORDINATOR)).toBe(0)+ })++ it('does not delete mcp-host when same-pass GET is 404', async () => {+ const coreApi = makeCoreApi(false)+ const reconciler = new WorkflowReconciler(makeDeps({ coreApi: coreApi as never }))++ const result = await reconciler.reconcile('test-wf', 'uid-123', SANDBOX, makeSpec())++ expect(result.workflowPhase).not.toBe('failed')+ expect(podReads(coreApi, MCP_HOST)).toBe(2)+ expect(podDeletes(coreApi, MCP_HOST)).toBe(0)})})