Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
fix(workflow-recipes): address PR #855 review findings
Alfredo Lopez1958d798749a
diff --git a/scripts/minikube-profiles/branch-profile.sh b/scripts/minikube-profiles/branch-profile.shindex b5f4b648d..fd3f533c5 100755--- a/scripts/minikube-profiles/branch-profile.sh+++ b/scripts/minikube-profiles/branch-profile.sh@@ -790,6 +790,19 @@ cmd_preflight() {printf ' %s\n' "${PORTS_ENV}"}+# perl -0pi exits 0 when its pattern matches nothing, so an upstream edit to a+# shimmed script would leave the shim un-rewritten and the profile bound to the+# wrong paths or context with no error. Fail unless the rewrite changed the file.+rewrite_shim() {+ local file="$1" expr="$2" label="$3"+ local before after+ before="$(shasum "${file}")" || die "BRANCH_PROFILE_SHIM_REWRITE_FAILED: ${label}"+ perl -0pi -e "${expr}" "${file}" || die "BRANCH_PROFILE_SHIM_REWRITE_FAILED: ${label}"+ after="$(shasum "${file}")" || die "BRANCH_PROFILE_SHIM_REWRITE_FAILED: ${label}"+ [[ "${before}" != "${after}" ]] ||+ die "BRANCH_PROFILE_SHIM_REWRITE_NOOP: ${label} matched nothing; the upstream script changed"+}+cmd_prepare_shims() {require_command git shasum perlpersist_state@@ -808,18 +821,12 @@ cmd_prepare_shims() {die "BRANCH_PROFILE_SHIM_REWRITE_FAILED: ${file}"done < <(find "${SHIMS_DIR}" -type f -name '*.sh' | sort)- perl -0pi -e 's#OUTPUT="\$\{PROJECT_DIR\}/deploy/minikube/secrets/jwt-signing-keys.yaml"#OUTPUT="\${BRANCH_PROFILE_DEPLOY_DIR:-\${PROJECT_DIR}/deploy}/minikube/secrets/jwt-signing-keys.yaml"#g' "${SHIMS_DIR}/generate-keys.sh" ||- die "BRANCH_PROFILE_SHIM_REWRITE_FAILED: generate-keys.sh"- perl -0pi -e 's#REPO_ROOT="\$\(cd "\$\{SCRIPT_DIR\}/\.\./\.\." && pwd\)"#REPO_ROOT="\${CLERUM_PROJECT_DIR:-\$(cd \"\${SCRIPT_DIR}/../..\" && pwd)}"#g' "${SHIMS_DIR}/seed-test-data.sh" ||- die "BRANCH_PROFILE_SHIM_REWRITE_FAILED: seed-test-data.sh REPO_ROOT"- perl -0pi -e 's#: "\$\{CONTEXT:=\$\(kubectl config current-context\)\}"#: "\${CONTEXT:=\${MINIKUBE_PROFILE:-\$(kubectl config current-context)}}"#g; s#export ADMIN_PASSWORD E2E_TEST_EMAIL E2E_TEST_PASSWORD CONTEXT#: "\${ALLOWED_CONTEXTS:=\${CONTEXT}}"\nexport ADMIN_PASSWORD E2E_TEST_EMAIL E2E_TEST_PASSWORD CONTEXT ALLOWED_CONTEXTS#g; s#export ADMIN_PASSWORD E2E_DEV_LOGIN_EMAIL CONTEXT#: "\${ALLOWED_CONTEXTS:=\${CONTEXT}}"\nexport ADMIN_PASSWORD E2E_DEV_LOGIN_EMAIL CONTEXT ALLOWED_CONTEXTS#g' "${SHIMS_DIR}/seed-test-data.sh" ||- die "BRANCH_PROFILE_SHIM_REWRITE_FAILED: seed-test-data.sh CONTEXT"- perl -0pi -e 's#MANIFEST_FILE="\$\{PROJECT_DIR\}/deploy/minikube/\.image-manifest.json"#MANIFEST_FILE="\${BRANCH_PROFILE_DEPLOY_DIR:-\${PROJECT_DIR}/deploy}/minikube/.image-manifest.json"#g' "${SHIMS_DIR}/build-images.sh" ||- die "BRANCH_PROFILE_SHIM_REWRITE_FAILED: build-images.sh"- perl -0pi -e 's#BASE_MINIKUBE_KUSTOMIZE_DIR="\$\{PROJECT_DIR\}/deploy/overlays/minikube"#BASE_MINIKUBE_KUSTOMIZE_DIR="\${BRANCH_PROFILE_DEPLOY_DIR:-\${PROJECT_DIR}/deploy}/overlays/minikube"#g; s#LOCAL_MEMBER_REGISTRATION_KUSTOMIZE_DIR="\$\{PROJECT_DIR\}/deploy/overlays/minikube-local-member-registration"#LOCAL_MEMBER_REGISTRATION_KUSTOMIZE_DIR="\${BRANCH_PROFILE_DEPLOY_DIR:-\${PROJECT_DIR}/deploy}/overlays/minikube-local-member-registration"#g' "${SHIMS_DIR}/full-setup.sh" ||- die "BRANCH_PROFILE_SHIM_REWRITE_FAILED: full-setup.sh overlays"- perl -0pi -e 's#CONTEXT="\$\{PROFILE\}" "\$\{PROJECT_DIR\}/deploy/scripts/minikube-detect-k8s-api-ip.sh"#CONTEXT="\${PROFILE}" OVERLAY_DIR="\${BRANCH_PROFILE_DEPLOY_DIR:-\${PROJECT_DIR}/deploy}/overlays/minikube" "\${PROJECT_DIR}/deploy/scripts/minikube-detect-k8s-api-ip.sh"#g; s#kubectl kustomize "\$\{PROJECT_DIR\}/deploy/overlays/minikube"#kubectl kustomize "\${BRANCH_PROFILE_DEPLOY_DIR:-\${PROJECT_DIR}/deploy}/overlays/minikube"#g' "${SHIMS_DIR}/full-setup.sh" ||- die "BRANCH_PROFILE_SHIM_REWRITE_FAILED: full-setup.sh detect"+ rewrite_shim "${SHIMS_DIR}/generate-keys.sh" 's#OUTPUT="\$\{PROJECT_DIR\}/deploy/minikube/secrets/jwt-signing-keys.yaml"#OUTPUT="\${BRANCH_PROFILE_DEPLOY_DIR:-\${PROJECT_DIR}/deploy}/minikube/secrets/jwt-signing-keys.yaml"#g' "generate-keys.sh"+ rewrite_shim "${SHIMS_DIR}/seed-test-data.sh" 's#REPO_ROOT="\$\(cd "\$\{SCRIPT_DIR\}/\.\./\.\." && pwd\)"#REPO_ROOT="\${CLERUM_PROJECT_DIR:-\$(cd \"\${SCRIPT_DIR}/../..\" && pwd)}"#g' "seed-test-data.sh REPO_ROOT"+ rewrite_shim "${SHIMS_DIR}/seed-test-data.sh" 's#: "\$\{CONTEXT:=\$\(kubectl config current-context\)\}"#: "\${CONTEXT:=\${MINIKUBE_PROFILE:-\$(kubectl config current-context)}}"#g; s#export ADMIN_PASSWORD E2E_TEST_EMAIL E2E_TEST_PASSWORD CONTEXT#: "\${ALLOWED_CONTEXTS:=\${CONTEXT}}"\nexport ADMIN_PASSWORD E2E_TEST_EMAIL E2E_TEST_PASSWORD CONTEXT ALLOWED_CONTEXTS#g; s#export ADMIN_PASSWORD E2E_DEV_LOGIN_EMAIL CONTEXT#: "\${ALLOWED_CONTEXTS:=\${CONTEXT}}"\nexport ADMIN_PASSWORD E2E_DEV_LOGIN_EMAIL CONTEXT ALLOWED_CONTEXTS#g' "seed-test-data.sh CONTEXT"+ rewrite_shim "${SHIMS_DIR}/build-images.sh" 's#MANIFEST_FILE="\$\{PROJECT_DIR\}/deploy/minikube/\.image-manifest.json"#MANIFEST_FILE="\${BRANCH_PROFILE_DEPLOY_DIR:-\${PROJECT_DIR}/deploy}/minikube/.image-manifest.json"#g' "build-images.sh"+ # full-setup.sh resolves BRANCH_PROFILE_DEPLOY_DIR natively (ACTIVE_MINIKUBE_*),+ # so it needs no path rewrite; rewrite_shim would fail on it as a no-op.chmod +x "${SHIMS_DIR}"/*.shcat >"${SHIM_ENV}" <<EOF_SHIMSdiff --git a/scripts/minikube-profiles/branch.mk b/scripts/minikube-profiles/branch.mkindex cfa7abe03..4ef18628f 100644--- a/scripts/minikube-profiles/branch.mk+++ b/scripts/minikube-profiles/branch.mk@@ -15,18 +15,13 @@ ARGS ?=.DEFAULT_GOAL := branch-profile-info+# Exported, not interpolated into the recipe: a value containing a quote, `$`+# or a backtick would otherwise be re-parsed by the shell.+export HOST CACHE_ROOT MINIKUBE_MEMORY MINIKUBE_CPUS MINIKUBE_CNI MINIKUBE_DRIVER+export CONFIRM_DELETE CONFIRM_PROFILE BRANCH_PROFILE_PROFILE ARGS+define run_branch_profile-@HOST="$(HOST)" \-CACHE_ROOT="$(CACHE_ROOT)" \-MINIKUBE_MEMORY="$(MINIKUBE_MEMORY)" \-MINIKUBE_CPUS="$(MINIKUBE_CPUS)" \-MINIKUBE_CNI="$(MINIKUBE_CNI)" \-MINIKUBE_DRIVER="$(MINIKUBE_DRIVER)" \-CONFIRM_DELETE="$(CONFIRM_DELETE)" \-CONFIRM_PROFILE="$(CONFIRM_PROFILE)" \-BRANCH_PROFILE_PROFILE="$(BRANCH_PROFILE_PROFILE)" \-ARGS="$(ARGS)" \-"$(BRANCH_PROFILE_SCRIPT)" "$(1)"+@"$(BRANCH_PROFILE_SCRIPT)" "$(1)"endef.PHONY: branch-profile-resolvediff --git a/workflow-recipes/package-lock.json b/workflow-recipes/package-lock.jsonindex ffa689614..f3a166a25 100644--- a/workflow-recipes/package-lock.json+++ b/workflow-recipes/package-lock.json@@ -1,12 +1,12 @@{"name": "workload-recipes-controller",- "version": "0.1.411",+ "version": "0.1.412","lockfileVersion": 3,"requires": true,"packages": {"": {"name": "workload-recipes-controller",- "version": "0.1.411",+ "version": "0.1.412","license": "SEE LICENSE IN LICENSE","dependencies": {"@aws-sdk/client-s3": "^3.1009.0",diff --git a/workflow-recipes/package.json b/workflow-recipes/package.jsonindex ffaa72798..341e136e6 100644--- a/workflow-recipes/package.json+++ b/workflow-recipes/package.json@@ -1,6 +1,6 @@{"name": "workload-recipes-controller",- "version": "0.1.411",+ "version": "0.1.412","description": "Workload Recipes Controller (WRC) - WorkloadRecipe lifecycle management and MCP server interface","main": "dist/main.js","engines": {diff --git a/workflow-recipes/src/k8sClient.test.ts b/workflow-recipes/src/k8sClient.test.tsindex 2028305f6..5545401de 100644--- a/workflow-recipes/src/k8sClient.test.ts+++ b/workflow-recipes/src/k8sClient.test.ts@@ -3251,12 +3251,77 @@ describe('WorkflowRecipeWatcher wiring — grant-update listener lifecycle (issu})})-describe('startSecretWatch oauth ADDED callback', () => {- it('4th SecretWatcher argument only invalidates the ledger', () => {- const source = (- WorkflowRecipeWatcher.prototype as unknown as { startSecretWatch: () => Promise<void> }- ).startSecretWatch.toString()- expect(source).toContain('invalidateOAuthBrokerDeleteLedger')- expect(source).not.toContain('triggerSecretDrivenReconcile(recipeName)')+describe('handleOAuthBrokerTokenAdded', () => {+ type BrokerAddedInternal = {+ config: { sandboxNamespace: string }+ reconciler: { invalidateOAuthBrokerDeleteLedger: (recipeName: string) => void }+ recipes: Map<string, WorkflowRecipeCRD>+ eventQueue: { enqueue: (key: string, task: () => Promise<void>) => unknown }+ handleOAuthBrokerTokenAdded: (namespace: string, recipeName: string) => void+ }++ function recipeWith(name: string, backgroundAccess: boolean): WorkflowRecipeCRD {+ const recipe = makeWorkflowRecipe({ metadata: { name, namespace: 'sandbox-recipes' } })+ recipe.spec = {+ ...recipe.spec,+ oauthClients: backgroundAccess+ ? [+ {+ id: 'gmail',+ provider: 'google',+ clientIdRef: { name: 'creds', key: 'client-id' },+ clientSecretRef: { name: 'creds', key: 'client-secret' },+ scopes: ['https://www.googleapis.com/auth/gmail.readonly'],+ backgroundAccess: true,+ },+ ]+ : [],+ } as WorkflowRecipeCRD['spec']+ return recipe+ }++ function setup() {+ const watcher = new WorkflowRecipeWatcher(+ {+ makeApiClient: vi.fn(() => ({})),+ } as unknown as import('@kubernetes/client-node').KubeConfig,+ null+ )+ const internal = watcher as unknown as BrokerAddedInternal+ const invalidate = vi.fn()+ const enqueue = vi.fn()+ internal.reconciler = { invalidateOAuthBrokerDeleteLedger: invalidate }+ internal.eventQueue = { enqueue }+ internal.recipes = new Map([+ ['no-bg', recipeWith('no-bg', false)],+ ['with-bg', recipeWith('with-bg', true)],+ ])+ return { internal, invalidate, enqueue }+ }++ it('re-arms the ledger and re-reconciles a recipe that must not have a broker token', () => {+ const { internal, invalidate, enqueue } = setup()+ internal.handleOAuthBrokerTokenAdded(internal.config.sandboxNamespace, 'no-bg')+ expect(invalidate).toHaveBeenCalledWith('no-bg')+ expect(enqueue).toHaveBeenCalledTimes(1)+ expect(enqueue.mock.calls[0]?.[0]).toBe('no-bg')+ })++ it('only invalidates for an opted-in recipe: a watch reconnect replays ADDED for every token', () => {+ const { internal, invalidate, enqueue } = setup()+ internal.handleOAuthBrokerTokenAdded(internal.config.sandboxNamespace, 'with-bg')+ expect(invalidate).toHaveBeenCalledWith('with-bg')+ expect(enqueue).not.toHaveBeenCalled()+ })++ it('ignores a same-named Secret outside the sandbox namespace', () => {+ const { internal, invalidate, enqueue } = setup()+ // Liveness witness: the same call in the sandbox namespace does act.+ internal.handleOAuthBrokerTokenAdded(internal.config.sandboxNamespace, 'no-bg')+ expect(invalidate).toHaveBeenCalledTimes(1)+ expect(enqueue).toHaveBeenCalledTimes(1)+ internal.handleOAuthBrokerTokenAdded('mcp-server', 'no-bg')+ expect(invalidate).toHaveBeenCalledTimes(1)+ expect(enqueue).toHaveBeenCalledTimes(1)})})diff --git a/workflow-recipes/src/k8sClient.ts b/workflow-recipes/src/k8sClient.tsindex bb09f0cfe..9468489be 100644--- a/workflow-recipes/src/k8sClient.ts+++ b/workflow-recipes/src/k8sClient.ts@@ -911,11 +911,7 @@ export class WorkflowRecipeWatcher implements WorkflowRecipeProvider {this.secretReverseIndex,name => this.triggerSecretDrivenReconcile(name),10_000,- // Invalidate only. SecretWatcher must not enqueue on oauth ADDED:- // a watch reconnect replays ADDED for every broker token.- recipeName => {- this.reconciler.invalidateOAuthBrokerDeleteLedger(recipeName)- }+ recipeName => this.handleOAuthBrokerTokenAdded(namespace, recipeName))const loop = new K8sSecretWatchLoop(this.kc, namespace, handler)this.secretWatchLoops.push(loop)@@ -923,6 +919,27 @@ export class WorkflowRecipeWatcher implements WorkflowRecipeProvider {}}+ /**+ * B3(a): a broker token Secret appeared. The token lives only in the sandbox+ * namespace, so an ADDED for a same-named Secret elsewhere is ignored.+ *+ * The delete ledger is always re-armed. A recipe with no backgroundAccess+ * client must not have the token at all, so it is also re-reconciled to reap+ * it: the broker rotation loop skips such recipes and the ownership backstop+ * only covers recipes that reference named Secrets, so nothing else would+ * issue the delete before the next CRD event. An opted-in recipe legitimately+ * owns its token and only invalidates, because a watch reconnect replays+ * ADDED for every broker token and must not force a reconcile storm.+ */+ private handleOAuthBrokerTokenAdded(namespace: string, recipeName: string): void {+ if (namespace !== this.config.sandboxNamespace) return+ this.reconciler.invalidateOAuthBrokerDeleteLedger(recipeName)+ const recipe = this.recipes.get(recipeName)+ if (recipe && !recipeHasBackgroundAccessClient(recipe)) {+ this.triggerSecretDrivenReconcile(recipeName)+ }+ }+/*** Enqueue a re-reconcile for a recipe whose referenced Secret's key-set* just changed. The cached recipe object is reused — no extra GET againstdiff --git a/workflow-recipes/src/reconciler/reconciler.test.ts b/workflow-recipes/src/reconciler/reconciler.test.tsindex cc810de4e..095f06493 100644--- a/workflow-recipes/src/reconciler/reconciler.test.ts+++ b/workflow-recipes/src/reconciler/reconciler.test.ts@@ -15698,6 +15698,10 @@ describe('WorkflowRecipeReconciler', () => {'run-g2')+ // Liveness witness: the retry ran the prune LIST that listed GFS, so the+ // absence of a GFS DELETE below is the catalog gate holding, not a retry+ // that never reached the prune.+ expect(mockNetworkingApi.listNamespacedNetworkPolicy).toHaveBeenCalled()expect(mockNetworkingApi.deleteNamespacedNetworkPolicy).not.toHaveBeenCalledWith(expect.objectContaining({ name: GFS }))diff --git a/workflow-recipes/src/reconciler/secretWatcher.test.ts b/workflow-recipes/src/reconciler/secretWatcher.test.tsindex dd9f5e85c..38296147d 100644--- a/workflow-recipes/src/reconciler/secretWatcher.test.ts+++ b/workflow-recipes/src/reconciler/secretWatcher.test.ts@@ -286,6 +286,13 @@ describe('SecretWatcher', () => {data: { 'broker-token': 'eA==' },})expect(invalidated).toEqual([])+ // Liveness witness: the same watcher does invalidate for the canonical+ // ADDED, so the empty list above is not the callback being unwired.+ watcher.handleEvent('ADDED', {+ metadata: { name: 'wf-test-recipe-oauth-broker-token', labels },+ data: { 'broker-token': 'eA==' },+ })+ expect(invalidated).toEqual(['test-recipe'])vi.advanceTimersByTime(DEBOUNCE_MS)expect(enqueued).toEqual([])watcher.stop()diff --git a/workflow-recipes/src/reconciler/secretWatcher.ts b/workflow-recipes/src/reconciler/secretWatcher.tsindex e36325db3..535e22cf7 100644--- a/workflow-recipes/src/reconciler/secretWatcher.ts+++ b/workflow-recipes/src/reconciler/secretWatcher.ts@@ -60,11 +60,10 @@ export class SecretWatcher {const name = secret.metadata?.nameif (!name) return- // B3(a): ADDED of the canonical broker token invalidates the ledger even- // when the key-set matches a previous observation (dedup would drop it).- // Invalidate only — a watch reconnect replays ADDED for every Secret and- // must not force a reconcile storm. The 30s ownership backstop (and any- // later key-set/ownership fan-out) issues the next delete.+ // B3(a): ADDED of the canonical broker token is reported even when the+ // key-set matches a previous observation (dedup would drop it). This+ // watcher never enqueues on it: a watch reconnect replays ADDED for every+ // Secret, so the callback owner decides whether a reconcile is needed.if (type === 'ADDED') {const recipeName = resolveOAuthBrokerTokenWatchRecipe(name, secret.metadata?.labels)if (recipeName) {diff --git a/workflow-recipes/src/reconciler/workflowRecipeReconciler.ts b/workflow-recipes/src/reconciler/workflowRecipeReconciler.tsindex 2d19154f6..a74afaf3c 100644--- a/workflow-recipes/src/reconciler/workflowRecipeReconciler.ts+++ b/workflow-recipes/src/reconciler/workflowRecipeReconciler.ts@@ -1089,7 +1089,7 @@ export class WorkflowRecipeReconciler {*/private async cleanupPluginWorkloadSdkOrThrow(recipeName: string,- options: { preserveWorkflowRuntime?: boolean } = {}+ options: { preserveWorkflowRuntime?: boolean; recipeUid?: string } = {}): Promise<void> {if (!this.workflowReconciler) {throw new Error('workflow subsystem is not initialized; SDK cleanup cannot be confirmed')@@ -1469,9 +1469,9 @@ export class WorkflowRecipeReconciler {* Applies the run-lane NetworkPolicies again from a short-circuit when the* published status carries the retry marker. Those short-circuits never* reach `WorkflowReconciler.reconcile()`, so without this a policy left- * pending a retry would stay unapplied until the run ends. Nothing is- * pruned: the prune decisions depend on the eligibility verdicts that only- * the full pass computes.+ * pending a retry would stay unapplied until the run ends. The retry+ * re-prunes as well, with a verdict computed for this pass: a prune that+ * failed is what set the marker, so an apply-only retry could never clear it.** Returns the requeue delay the short-circuit must add: the progress base* while a policy is still pending, the transient base when the apply threw@@ -3955,7 +3955,7 @@ export class WorkflowRecipeReconciler {// a capability may have been removed in an update before Kubernetes emits// the final delete event, and cleanup must still revoke its host tokens.if (recipe.spec.pluginWorkloadSdk || recipe.status?.pluginWorkloadSdk) {- await this.cleanupPluginWorkloadSdkOrThrow(name)+ await this.cleanupPluginWorkloadSdkOrThrow(name, { recipeUid: recipe.metadata.uid })}await this.cleanupDelegationIfNeeded(recipe)diff --git a/workflow-recipes/src/workflow/workflowReconciler.test.ts b/workflow-recipes/src/workflow/workflowReconciler.test.tsindex 0fc956ec2..c6e1f818d 100644--- a/workflow-recipes/src/workflow/workflowReconciler.test.ts+++ b/workflow-recipes/src/workflow/workflowReconciler.test.ts@@ -1302,8 +1302,11 @@ describe('WorkflowReconciler.reconcileDelete — orphaned Service cleanup', () =await reconciler.teardownComputePodsForTerminalRun('wf-run-7')+ // Witness: the teardown ran (three DELETEs) and none of them was+ // preceded by a raw pod GET, which getPodPresence alone would not show.expect(crashRecoveryMocks.deletePodIfExists).toHaveBeenCalledTimes(3)expect(crashRecoveryMocks.getPodPresence).not.toHaveBeenCalled()+ expect(mockCoreApi.readNamespacedPod).not.toHaveBeenCalled()})it('is idempotent: a second pass still DELETE-first (404-safe)', async () => {@@ -2120,6 +2123,40 @@ describe('WorkflowReconciler — Plugin Workload SDK eager mcp-host', () => {expect(mockCoreApi.createNamespacedPod).not.toHaveBeenCalled()})+ it('forgets the legacy internet policy ledger entry when an SDK-only recipe is cleaned up', async () => {+ const reconciler = new WorkflowReconciler(makeDeps())+ const legacyDeletes = () =>+ mockNetworkingApi.deleteNamespacedNetworkPolicy.mock.calls.filter(+ ([arg]) => arg.name === 'sdk-only-mcp-servers-egress-internet'+ )+ const reconcileSdkOnly = () =>+ reconciler.reconcilePluginWorkloadSdkOnly(+ 'sdk-only',+ 'uid-sdk-only',+ sandboxNamespace,+ sdkSpec({+ agent: undefined,+ steps: undefined,+ pluginWorkloadSdk: {+ clientNotifications: { allowedEventTypes: ['e2e.test'] },+ allowedCallers: ['sdk-caller'],+ },+ })+ )++ await reconcileSdkOnly()+ // The witness: the first pass reached the legacy prune and recorded it, so+ // a second pass skips the DELETE.+ expect(legacyDeletes()).toHaveLength(1)+ await reconcileSdkOnly()+ expect(legacyDeletes()).toHaveLength(1)++ await reconciler.cleanupPluginWorkloadSdk('sdk-only', { recipeUid: 'uid-sdk-only' })+ await reconcileSdkOnly()++ expect(legacyDeletes()).toHaveLength(2)+ })+it('defers absence reads until pod deletion completes so 404s cannot become unhandled rejections', async () => {const events: string[] = []crashRecoveryMocks.waitForPodDeletion.mockImplementationOnce(async () => {diff --git a/workflow-recipes/src/workflow/workflowReconciler.ts b/workflow-recipes/src/workflow/workflowReconciler.tsindex c13509a84..79d94d3cb 100644--- a/workflow-recipes/src/workflow/workflowReconciler.ts+++ b/workflow-recipes/src/workflow/workflowReconciler.ts@@ -254,6 +254,12 @@ export interface PluginWorkloadSdkCleanupOptions {* remains the owner of shared runtime resources.*/preserveWorkflowRuntime?: boolean+ /**+ * Set when the recipe itself is being deleted. An SDK-only recipe never+ * reaches `reconcileDelete`, so this is the only place its legacy internet+ * policy ledger entry can be forgotten.+ */+ recipeUid?: string}export const NETWORK_POLICY_OWNERSHIP_CONDITION_TYPE = 'WorkflowNetworkPolicyOwnership'@@ -1465,6 +1471,9 @@ export class WorkflowReconciler {const ns = this.deps.config.sandboxNamespaceconst preserveWorkflowRuntime = options.preserveWorkflowRuntime === truethis.pluginWorkloadSdkProvisioner.clearRecipeState(recipeName)+ if (options.recipeUid) {+ this.forgetLegacyMcpServersInternetEgress(recipeName, options.recipeUid)+ }const sdkNetworkPolicyNames = [`${recipeName}-workload-to-mcp-host-sdk-ingress`,`${recipeName}-workload-to-mcp-host-sdk-egress`,@@ -2979,10 +2988,13 @@ export class WorkflowReconciler {/*** Reapply the run-lane NetworkPolicies outside reconcile(). The running and* active short-circuits return before reconcile(), so a policy a previous- * pass left pending (terminating or contended) would otherwise stay as it is- * until the run ends. Builds the same set reconcile() applies and only- * applies it: no codex, grok or legacy prune, because those follow an- * eligibility verdict that can change while a run is in progress.+ * pass left pending (terminating, contended, or a prune that failed) would+ * otherwise stay as it is until the run ends. Runs the same build, apply and+ * prune unit reconcile() runs, with a verdict computed now: an uncertain+ * Codex or Grok verdict keeps its proxy policy, and the catalog gate keeps+ * everything the run lane does not own. A prune that failed sets+ * retryPending, so without the prune here the marker would requeue forever+ * and never reach the leftover.*/async retryRunLaneNetworkPolicies(recipeName: string,@@ -3007,7 +3019,7 @@ export class WorkflowReconciler {runtimeScopeRecipeName,codexView)- const { policies } = await this.buildWorkflowNetworkPoliciesForSpec(+ return this.applyWorkflowNetworkPolicies(recipeName,recipeUid,spec,@@ -3017,7 +3029,6 @@ export class WorkflowReconciler {false,codexVerdict.grokProjection)- return this.applyNetworkPolicyList(policies)}async ensureCoordinatorRuntimeCredentials(@@ -3210,6 +3221,11 @@ export class WorkflowReconciler {return recipeUid && recipeUid.length > 0 ? recipeUid : recipeName}+ private forgetLegacyMcpServersInternetEgress(recipeName: string, recipeUid?: string): void {+ this.prunedLegacyMcpServersInternetEgress.delete(recipeName)+ if (recipeUid) this.prunedLegacyMcpServersInternetEgress.delete(recipeUid)+ }+private async pruneLegacyMcpServersInternetEgressPolicy(recipeName: string,recipeUid?: string@@ -3779,8 +3795,7 @@ export class WorkflowReconciler {const log = createLogger('wrc', recipeName)log.info(`Deleting workflow resources`)this.pluginWorkloadSdkProvisioner.clearRecipeState(recipeName)- this.prunedLegacyMcpServersInternetEgress.delete(recipeName)- if (recipeUid) this.prunedLegacyMcpServersInternetEgress.delete(recipeUid)+ this.forgetLegacyMcpServersInternetEgress(recipeName, recipeUid)// Best-effort cleanup of the recipe's subdirectory on the workflow output PVC.// Must run BEFORE pod deletion while mcp-host is still reachable; failuresdiff --git a/workflow-recipes/tests/unit/workflow/workflowReconciler.test.ts b/workflow-recipes/tests/unit/workflow/workflowReconciler.test.tsindex bec33d15e..f9087cac6 100644--- a/workflow-recipes/tests/unit/workflow/workflowReconciler.test.ts+++ b/workflow-recipes/tests/unit/workflow/workflowReconciler.test.ts@@ -3090,6 +3090,37 @@ describe('WorkflowReconciler — reconcile loop', () => {})})+ it('keys the legacy internet NP ledger by recipe uid, so a recreated recipe deletes again', async () => {+ const networkingApi = makeNetworkingApi() as ReturnType<typeof makeNetworkingApi>+ const reconciler = new WorkflowReconciler(+ makeDeps({+ networkingApi: networkingApi as never,+ })+ )+ const legacyDeletes = () =>+ networkingApi.deleteNamespacedNetworkPolicy.mock.calls.filter(+ ([arg]) =>+ arg.name === 'test-wf-mcp-servers-egress-internet' && arg.namespace === 'mcp-server'+ )+ const reconcileWithUid = (uid: string) =>+ reconciler.reconcile(+ 'test-wf',+ uid,+ 'sandbox-recipes',+ makeSpec({ mcpServers: ['redis-mcp'] })+ )++ await reconcileWithUid('uid-a')+ await reconcileWithUid('uid-a')+ // Witness: the ledger suppresses the repeat for the same uid.+ expect(legacyDeletes()).toHaveLength(1)++ // Same name, new uid, and no reconcileDelete seen by this process (another+ // replica handled the delete): a new identity must not be suppressed.+ await reconcileWithUid('uid-b')+ expect(legacyDeletes()).toHaveLength(2)+ })+describe('B2 desired-set NetworkPolicy prune', () => {const RECIPE = 'test-wf'const SANDBOX = 'sandbox-recipes'@@ -3333,7 +3364,7 @@ describe('WorkflowReconciler — reconcile loop', () => {expect(apiserver.live.has(apiserver.key(SANDBOX, COORD_TO_WRC))).toBe(true)})- it('does not prune a recipes-lane policy; LIST uses the two-term selector once', async () => {+ it('does not prune an uncatalogued policy carrying the recipe label pair; LIST is issued once', async () => {const apiserver = makeApiserverNetworkingApi()seedLivePolicy(apiserver, `${RECIPE}-oauth-broker-egress`, {'clerum.io/recipe': RECIPE,@@ -3505,6 +3536,14 @@ describe('WorkflowReconciler — reconcile loop', () => {expect(summary.retryPending).toBe(true)expect(sandboxDeletes(apiserver.api)).toEqual([])expect(apiserver.live.has(apiserver.key(SANDBOX, GROK_PROXY))).toBe(true)+ // The apply outcome survives: the legacy internet policy prune that runs+ // after the catalog prune is still attempted despite the failed LIST.+ expect(apiserver.api.listNamespacedNetworkPolicy).toHaveBeenCalled()+ expect(+ apiserver.api.deleteNamespacedNetworkPolicy.mock.calls.filter(+ ([arg]: [{ name: string }]) => arg.name === `${RECIPE}-mcp-servers-egress-internet`+ )+ ).toHaveLength(1)})})@@ -5121,6 +5160,107 @@ describe('WorkflowReconciler — reconcile loop', () => {expect(apiserver.api.replaceNamespacedNetworkPolicy).toHaveBeenCalledTimes(0)expect(apiserver.api.deleteNamespacedNetworkPolicy).toHaveBeenCalledTimes(0)})++ // A prune failure sets retryPending, and this pass is the only one that+ // runs while the run is in progress or active. It has to prune again, or+ // the marker requeues forever without ever reaching the leftover.+ describe('when a prune left a leftover behind', () => {+ const LEFTOVER = 'test-wf-mcp-host-to-grok-proxy'+ const TWO_TERM_SELECTOR = 'clerum.io/recipe=test-wf,clerum.io/managed-by=wrc'+ type Converged = Awaited<ReturnType<typeof convergedRunLane>>['apiserver']++ function seedLeftover(apiserver: Converged) {+ apiserver.live.set(apiserver.key('sandbox-recipes', LEFTOVER), {+ apiVersion: 'networking.k8s.io/v1',+ kind: 'NetworkPolicy',+ metadata: {+ name: LEFTOVER,+ namespace: 'sandbox-recipes',+ labels: { 'clerum.io/recipe': 'test-wf', 'clerum.io/managed-by': 'wrc' },+ },+ spec: { podSelector: {}, policyTypes: ['Egress'] },+ })+ }++ it('deletes the leftover on the retry pass', async () => {+ const { apiserver, reconciler } = await convergedRunLane()+ apiserver.api.listNamespacedNetworkPolicy.mockClear()+ seedLeftover(apiserver)++ const summary = await reconciler.retryRunLaneNetworkPolicies('test-wf', 'uid-123', spec())++ expect(apiserver.api.listNamespacedNetworkPolicy).toHaveBeenCalledWith({+ namespace: 'sandbox-recipes',+ labelSelector: TWO_TERM_SELECTOR,+ })+ expect(summary).toEqual({ conflicts: [], retryPending: false })+ expect(apiserver.live.has(apiserver.key('sandbox-recipes', LEFTOVER))).toBe(false)+ expect(+ apiserver.api.deleteNamespacedNetworkPolicy.mock.calls.map(([arg]) => arg.name)+ ).toEqual([LEFTOVER])+ })++ it('keeps reporting the retry until the leftover delete succeeds', async () => {+ const { apiserver, reconciler } = await convergedRunLane()+ apiserver.api.listNamespacedNetworkPolicy.mockClear()+ seedLeftover(apiserver)+ apiserver.api.deleteNamespacedNetworkPolicy.mockRejectedValueOnce({+ code: 403,+ message: 'forbidden',+ })++ const first = await reconciler.retryRunLaneNetworkPolicies('test-wf', 'uid-123', spec())+ expect(apiserver.api.listNamespacedNetworkPolicy).toHaveBeenCalledTimes(1)+ expect(first.retryPending).toBe(true)+ expect(apiserver.live.has(apiserver.key('sandbox-recipes', LEFTOVER))).toBe(true)++ const second = await reconciler.retryRunLaneNetworkPolicies('test-wf', 'uid-123', spec())+ expect(second).toEqual({ conflicts: [], retryPending: false })+ expect(apiserver.live.has(apiserver.key('sandbox-recipes', LEFTOVER))).toBe(false)+ })++ it('keeps reporting the retry while the prune LIST fails', async () => {+ const { apiserver, reconciler } = await convergedRunLane()+ apiserver.api.listNamespacedNetworkPolicy.mockClear()+ seedLeftover(apiserver)+ apiserver.api.listNamespacedNetworkPolicy.mockRejectedValueOnce({+ code: 500,+ message: 'boom',+ })++ const first = await reconciler.retryRunLaneNetworkPolicies('test-wf', 'uid-123', spec())+ expect(apiserver.api.listNamespacedNetworkPolicy).toHaveBeenCalledTimes(1)+ expect(first.retryPending).toBe(true)+ expect(apiserver.live.has(apiserver.key('sandbox-recipes', LEFTOVER))).toBe(true)++ const second = await reconciler.retryRunLaneNetworkPolicies('test-wf', 'uid-123', spec())+ expect(second.retryPending).toBe(false)+ expect(apiserver.live.has(apiserver.key('sandbox-recipes', LEFTOVER))).toBe(false)+ })++ it('retries a failed legacy internet policy delete', async () => {+ const apiserver = makeApiserverNetworkingApi()+ const reconciler = new WorkflowReconciler(+ makeDeps({ networkingApi: apiserver.api as never })+ )+ apiserver.api.deleteNamespacedNetworkPolicy.mockRejectedValueOnce({+ code: 403,+ message: 'forbidden',+ })+ const legacyDeletes = () =>+ apiserver.api.deleteNamespacedNetworkPolicy.mock.calls.filter(+ ([arg]) => arg.name === 'test-wf-mcp-servers-egress-internet'+ )++ const first = await reconciler.retryRunLaneNetworkPolicies('test-wf', 'uid-123', spec())+ expect(legacyDeletes()).toHaveLength(1)+ expect(first.retryPending).toBe(true)++ const second = await reconciler.retryRunLaneNetworkPolicies('test-wf', 'uid-123', spec())+ expect(legacyDeletes()).toHaveLength(2)+ expect(second.retryPending).toBe(false)+ })+ })})})