Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript

fix(workflow-recipes): address PR #855 review findings

Alfredo Lopez1958d798749a
diff --git a/scripts/minikube-profiles/branch-profile.sh b/scripts/minikube-profiles/branch-profile.sh
index b5f4b648d..fd3f533c5 100755
--- a/scripts/minikube-profiles/branch-profile.sh
+++ b/scripts/minikube-profiles/branch-profile.sh
@@ -790,6 +790,19 @@ cmd_preflight() {
printf ' %s\n' "${PORTS_ENV}"
}
+# perl -0pi exits 0 when its pattern matches nothing, so an upstream edit to a
+# shimmed script would leave the shim un-rewritten and the profile bound to the
+# wrong paths or context with no error. Fail unless the rewrite changed the file.
+rewrite_shim() {
+ local file="$1" expr="$2" label="$3"
+ local before after
+ before="$(shasum "${file}")" || die "BRANCH_PROFILE_SHIM_REWRITE_FAILED: ${label}"
+ perl -0pi -e "${expr}" "${file}" || die "BRANCH_PROFILE_SHIM_REWRITE_FAILED: ${label}"
+ after="$(shasum "${file}")" || die "BRANCH_PROFILE_SHIM_REWRITE_FAILED: ${label}"
+ [[ "${before}" != "${after}" ]] ||
+ die "BRANCH_PROFILE_SHIM_REWRITE_NOOP: ${label} matched nothing; the upstream script changed"
+}
+
cmd_prepare_shims() {
require_command git shasum perl
persist_state
@@ -808,18 +821,12 @@ cmd_prepare_shims() {
die "BRANCH_PROFILE_SHIM_REWRITE_FAILED: ${file}"
done < <(find "${SHIMS_DIR}" -type f -name '*.sh' | sort)
- perl -0pi -e 's#OUTPUT="\$\{PROJECT_DIR\}/deploy/minikube/secrets/jwt-signing-keys.yaml"#OUTPUT="\${BRANCH_PROFILE_DEPLOY_DIR:-\${PROJECT_DIR}/deploy}/minikube/secrets/jwt-signing-keys.yaml"#g' "${SHIMS_DIR}/generate-keys.sh" ||
- die "BRANCH_PROFILE_SHIM_REWRITE_FAILED: generate-keys.sh"
- perl -0pi -e 's#REPO_ROOT="\$\(cd "\$\{SCRIPT_DIR\}/\.\./\.\." && pwd\)"#REPO_ROOT="\${CLERUM_PROJECT_DIR:-\$(cd \"\${SCRIPT_DIR}/../..\" && pwd)}"#g' "${SHIMS_DIR}/seed-test-data.sh" ||
- die "BRANCH_PROFILE_SHIM_REWRITE_FAILED: seed-test-data.sh REPO_ROOT"
- perl -0pi -e 's#: "\$\{CONTEXT:=\$\(kubectl config current-context\)\}"#: "\${CONTEXT:=\${MINIKUBE_PROFILE:-\$(kubectl config current-context)}}"#g; s#export ADMIN_PASSWORD E2E_TEST_EMAIL E2E_TEST_PASSWORD CONTEXT#: "\${ALLOWED_CONTEXTS:=\${CONTEXT}}"\nexport ADMIN_PASSWORD E2E_TEST_EMAIL E2E_TEST_PASSWORD CONTEXT ALLOWED_CONTEXTS#g; s#export ADMIN_PASSWORD E2E_DEV_LOGIN_EMAIL CONTEXT#: "\${ALLOWED_CONTEXTS:=\${CONTEXT}}"\nexport ADMIN_PASSWORD E2E_DEV_LOGIN_EMAIL CONTEXT ALLOWED_CONTEXTS#g' "${SHIMS_DIR}/seed-test-data.sh" ||
- die "BRANCH_PROFILE_SHIM_REWRITE_FAILED: seed-test-data.sh CONTEXT"
- perl -0pi -e 's#MANIFEST_FILE="\$\{PROJECT_DIR\}/deploy/minikube/\.image-manifest.json"#MANIFEST_FILE="\${BRANCH_PROFILE_DEPLOY_DIR:-\${PROJECT_DIR}/deploy}/minikube/.image-manifest.json"#g' "${SHIMS_DIR}/build-images.sh" ||
- die "BRANCH_PROFILE_SHIM_REWRITE_FAILED: build-images.sh"
- perl -0pi -e 's#BASE_MINIKUBE_KUSTOMIZE_DIR="\$\{PROJECT_DIR\}/deploy/overlays/minikube"#BASE_MINIKUBE_KUSTOMIZE_DIR="\${BRANCH_PROFILE_DEPLOY_DIR:-\${PROJECT_DIR}/deploy}/overlays/minikube"#g; s#LOCAL_MEMBER_REGISTRATION_KUSTOMIZE_DIR="\$\{PROJECT_DIR\}/deploy/overlays/minikube-local-member-registration"#LOCAL_MEMBER_REGISTRATION_KUSTOMIZE_DIR="\${BRANCH_PROFILE_DEPLOY_DIR:-\${PROJECT_DIR}/deploy}/overlays/minikube-local-member-registration"#g' "${SHIMS_DIR}/full-setup.sh" ||
- die "BRANCH_PROFILE_SHIM_REWRITE_FAILED: full-setup.sh overlays"
- perl -0pi -e 's#CONTEXT="\$\{PROFILE\}" "\$\{PROJECT_DIR\}/deploy/scripts/minikube-detect-k8s-api-ip.sh"#CONTEXT="\${PROFILE}" OVERLAY_DIR="\${BRANCH_PROFILE_DEPLOY_DIR:-\${PROJECT_DIR}/deploy}/overlays/minikube" "\${PROJECT_DIR}/deploy/scripts/minikube-detect-k8s-api-ip.sh"#g; s#kubectl kustomize "\$\{PROJECT_DIR\}/deploy/overlays/minikube"#kubectl kustomize "\${BRANCH_PROFILE_DEPLOY_DIR:-\${PROJECT_DIR}/deploy}/overlays/minikube"#g' "${SHIMS_DIR}/full-setup.sh" ||
- die "BRANCH_PROFILE_SHIM_REWRITE_FAILED: full-setup.sh detect"
+ rewrite_shim "${SHIMS_DIR}/generate-keys.sh" 's#OUTPUT="\$\{PROJECT_DIR\}/deploy/minikube/secrets/jwt-signing-keys.yaml"#OUTPUT="\${BRANCH_PROFILE_DEPLOY_DIR:-\${PROJECT_DIR}/deploy}/minikube/secrets/jwt-signing-keys.yaml"#g' "generate-keys.sh"
+ rewrite_shim "${SHIMS_DIR}/seed-test-data.sh" 's#REPO_ROOT="\$\(cd "\$\{SCRIPT_DIR\}/\.\./\.\." && pwd\)"#REPO_ROOT="\${CLERUM_PROJECT_DIR:-\$(cd \"\${SCRIPT_DIR}/../..\" && pwd)}"#g' "seed-test-data.sh REPO_ROOT"
+ rewrite_shim "${SHIMS_DIR}/seed-test-data.sh" 's#: "\$\{CONTEXT:=\$\(kubectl config current-context\)\}"#: "\${CONTEXT:=\${MINIKUBE_PROFILE:-\$(kubectl config current-context)}}"#g; s#export ADMIN_PASSWORD E2E_TEST_EMAIL E2E_TEST_PASSWORD CONTEXT#: "\${ALLOWED_CONTEXTS:=\${CONTEXT}}"\nexport ADMIN_PASSWORD E2E_TEST_EMAIL E2E_TEST_PASSWORD CONTEXT ALLOWED_CONTEXTS#g; s#export ADMIN_PASSWORD E2E_DEV_LOGIN_EMAIL CONTEXT#: "\${ALLOWED_CONTEXTS:=\${CONTEXT}}"\nexport ADMIN_PASSWORD E2E_DEV_LOGIN_EMAIL CONTEXT ALLOWED_CONTEXTS#g' "seed-test-data.sh CONTEXT"
+ rewrite_shim "${SHIMS_DIR}/build-images.sh" 's#MANIFEST_FILE="\$\{PROJECT_DIR\}/deploy/minikube/\.image-manifest.json"#MANIFEST_FILE="\${BRANCH_PROFILE_DEPLOY_DIR:-\${PROJECT_DIR}/deploy}/minikube/.image-manifest.json"#g' "build-images.sh"
+ # full-setup.sh resolves BRANCH_PROFILE_DEPLOY_DIR natively (ACTIVE_MINIKUBE_*),
+ # so it needs no path rewrite; rewrite_shim would fail on it as a no-op.
chmod +x "${SHIMS_DIR}"/*.sh
cat >"${SHIM_ENV}" <<EOF_SHIMS
diff --git a/scripts/minikube-profiles/branch.mk b/scripts/minikube-profiles/branch.mk
index cfa7abe03..4ef18628f 100644
--- a/scripts/minikube-profiles/branch.mk
+++ b/scripts/minikube-profiles/branch.mk
@@ -15,18 +15,13 @@ ARGS ?=
.DEFAULT_GOAL := branch-profile-info
+# Exported, not interpolated into the recipe: a value containing a quote, `$`
+# or a backtick would otherwise be re-parsed by the shell.
+export HOST CACHE_ROOT MINIKUBE_MEMORY MINIKUBE_CPUS MINIKUBE_CNI MINIKUBE_DRIVER
+export CONFIRM_DELETE CONFIRM_PROFILE BRANCH_PROFILE_PROFILE ARGS
+
define run_branch_profile
-@HOST="$(HOST)" \
-CACHE_ROOT="$(CACHE_ROOT)" \
-MINIKUBE_MEMORY="$(MINIKUBE_MEMORY)" \
-MINIKUBE_CPUS="$(MINIKUBE_CPUS)" \
-MINIKUBE_CNI="$(MINIKUBE_CNI)" \
-MINIKUBE_DRIVER="$(MINIKUBE_DRIVER)" \
-CONFIRM_DELETE="$(CONFIRM_DELETE)" \
-CONFIRM_PROFILE="$(CONFIRM_PROFILE)" \
-BRANCH_PROFILE_PROFILE="$(BRANCH_PROFILE_PROFILE)" \
-ARGS="$(ARGS)" \
-"$(BRANCH_PROFILE_SCRIPT)" "$(1)"
+@"$(BRANCH_PROFILE_SCRIPT)" "$(1)"
endef
.PHONY: branch-profile-resolve
diff --git a/workflow-recipes/package-lock.json b/workflow-recipes/package-lock.json
index ffa689614..f3a166a25 100644
--- a/workflow-recipes/package-lock.json
+++ b/workflow-recipes/package-lock.json
@@ -1,12 +1,12 @@
{
"name": "workload-recipes-controller",
- "version": "0.1.411",
+ "version": "0.1.412",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "workload-recipes-controller",
- "version": "0.1.411",
+ "version": "0.1.412",
"license": "SEE LICENSE IN LICENSE",
"dependencies": {
"@aws-sdk/client-s3": "^3.1009.0",
diff --git a/workflow-recipes/package.json b/workflow-recipes/package.json
index ffaa72798..341e136e6 100644
--- a/workflow-recipes/package.json
+++ b/workflow-recipes/package.json
@@ -1,6 +1,6 @@
{
"name": "workload-recipes-controller",
- "version": "0.1.411",
+ "version": "0.1.412",
"description": "Workload Recipes Controller (WRC) - WorkloadRecipe lifecycle management and MCP server interface",
"main": "dist/main.js",
"engines": {
diff --git a/workflow-recipes/src/k8sClient.test.ts b/workflow-recipes/src/k8sClient.test.ts
index 2028305f6..5545401de 100644
--- a/workflow-recipes/src/k8sClient.test.ts
+++ b/workflow-recipes/src/k8sClient.test.ts
@@ -3251,12 +3251,77 @@ describe('WorkflowRecipeWatcher wiring — grant-update listener lifecycle (issu
})
})
-describe('startSecretWatch oauth ADDED callback', () => {
- it('4th SecretWatcher argument only invalidates the ledger', () => {
- const source = (
- WorkflowRecipeWatcher.prototype as unknown as { startSecretWatch: () => Promise<void> }
- ).startSecretWatch.toString()
- expect(source).toContain('invalidateOAuthBrokerDeleteLedger')
- expect(source).not.toContain('triggerSecretDrivenReconcile(recipeName)')
+describe('handleOAuthBrokerTokenAdded', () => {
+ type BrokerAddedInternal = {
+ config: { sandboxNamespace: string }
+ reconciler: { invalidateOAuthBrokerDeleteLedger: (recipeName: string) => void }
+ recipes: Map<string, WorkflowRecipeCRD>
+ eventQueue: { enqueue: (key: string, task: () => Promise<void>) => unknown }
+ handleOAuthBrokerTokenAdded: (namespace: string, recipeName: string) => void
+ }
+
+ function recipeWith(name: string, backgroundAccess: boolean): WorkflowRecipeCRD {
+ const recipe = makeWorkflowRecipe({ metadata: { name, namespace: 'sandbox-recipes' } })
+ recipe.spec = {
+ ...recipe.spec,
+ oauthClients: backgroundAccess
+ ? [
+ {
+ id: 'gmail',
+ provider: 'google',
+ clientIdRef: { name: 'creds', key: 'client-id' },
+ clientSecretRef: { name: 'creds', key: 'client-secret' },
+ scopes: ['https://www.googleapis.com/auth/gmail.readonly'],
+ backgroundAccess: true,
+ },
+ ]
+ : [],
+ } as WorkflowRecipeCRD['spec']
+ return recipe
+ }
+
+ function setup() {
+ const watcher = new WorkflowRecipeWatcher(
+ {
+ makeApiClient: vi.fn(() => ({})),
+ } as unknown as import('@kubernetes/client-node').KubeConfig,
+ null
+ )
+ const internal = watcher as unknown as BrokerAddedInternal
+ const invalidate = vi.fn()
+ const enqueue = vi.fn()
+ internal.reconciler = { invalidateOAuthBrokerDeleteLedger: invalidate }
+ internal.eventQueue = { enqueue }
+ internal.recipes = new Map([
+ ['no-bg', recipeWith('no-bg', false)],
+ ['with-bg', recipeWith('with-bg', true)],
+ ])
+ return { internal, invalidate, enqueue }
+ }
+
+ it('re-arms the ledger and re-reconciles a recipe that must not have a broker token', () => {
+ const { internal, invalidate, enqueue } = setup()
+ internal.handleOAuthBrokerTokenAdded(internal.config.sandboxNamespace, 'no-bg')
+ expect(invalidate).toHaveBeenCalledWith('no-bg')
+ expect(enqueue).toHaveBeenCalledTimes(1)
+ expect(enqueue.mock.calls[0]?.[0]).toBe('no-bg')
+ })
+
+ it('only invalidates for an opted-in recipe: a watch reconnect replays ADDED for every token', () => {
+ const { internal, invalidate, enqueue } = setup()
+ internal.handleOAuthBrokerTokenAdded(internal.config.sandboxNamespace, 'with-bg')
+ expect(invalidate).toHaveBeenCalledWith('with-bg')
+ expect(enqueue).not.toHaveBeenCalled()
+ })
+
+ it('ignores a same-named Secret outside the sandbox namespace', () => {
+ const { internal, invalidate, enqueue } = setup()
+ // Liveness witness: the same call in the sandbox namespace does act.
+ internal.handleOAuthBrokerTokenAdded(internal.config.sandboxNamespace, 'no-bg')
+ expect(invalidate).toHaveBeenCalledTimes(1)
+ expect(enqueue).toHaveBeenCalledTimes(1)
+ internal.handleOAuthBrokerTokenAdded('mcp-server', 'no-bg')
+ expect(invalidate).toHaveBeenCalledTimes(1)
+ expect(enqueue).toHaveBeenCalledTimes(1)
})
})
diff --git a/workflow-recipes/src/k8sClient.ts b/workflow-recipes/src/k8sClient.ts
index bb09f0cfe..9468489be 100644
--- a/workflow-recipes/src/k8sClient.ts
+++ b/workflow-recipes/src/k8sClient.ts
@@ -911,11 +911,7 @@ export class WorkflowRecipeWatcher implements WorkflowRecipeProvider {
this.secretReverseIndex,
name => this.triggerSecretDrivenReconcile(name),
10_000,
- // Invalidate only. SecretWatcher must not enqueue on oauth ADDED:
- // a watch reconnect replays ADDED for every broker token.
- recipeName => {
- this.reconciler.invalidateOAuthBrokerDeleteLedger(recipeName)
- }
+ recipeName => this.handleOAuthBrokerTokenAdded(namespace, recipeName)
)
const loop = new K8sSecretWatchLoop(this.kc, namespace, handler)
this.secretWatchLoops.push(loop)
@@ -923,6 +919,27 @@ export class WorkflowRecipeWatcher implements WorkflowRecipeProvider {
}
}
+ /**
+ * B3(a): a broker token Secret appeared. The token lives only in the sandbox
+ * namespace, so an ADDED for a same-named Secret elsewhere is ignored.
+ *
+ * The delete ledger is always re-armed. A recipe with no backgroundAccess
+ * client must not have the token at all, so it is also re-reconciled to reap
+ * it: the broker rotation loop skips such recipes and the ownership backstop
+ * only covers recipes that reference named Secrets, so nothing else would
+ * issue the delete before the next CRD event. An opted-in recipe legitimately
+ * owns its token and only invalidates, because a watch reconnect replays
+ * ADDED for every broker token and must not force a reconcile storm.
+ */
+ private handleOAuthBrokerTokenAdded(namespace: string, recipeName: string): void {
+ if (namespace !== this.config.sandboxNamespace) return
+ this.reconciler.invalidateOAuthBrokerDeleteLedger(recipeName)
+ const recipe = this.recipes.get(recipeName)
+ if (recipe && !recipeHasBackgroundAccessClient(recipe)) {
+ this.triggerSecretDrivenReconcile(recipeName)
+ }
+ }
+
/**
* Enqueue a re-reconcile for a recipe whose referenced Secret's key-set
* just changed. The cached recipe object is reused — no extra GET against
diff --git a/workflow-recipes/src/reconciler/reconciler.test.ts b/workflow-recipes/src/reconciler/reconciler.test.ts
index cc810de4e..095f06493 100644
--- a/workflow-recipes/src/reconciler/reconciler.test.ts
+++ b/workflow-recipes/src/reconciler/reconciler.test.ts
@@ -15698,6 +15698,10 @@ describe('WorkflowRecipeReconciler', () => {
'run-g2'
)
+ // Liveness witness: the retry ran the prune LIST that listed GFS, so the
+ // absence of a GFS DELETE below is the catalog gate holding, not a retry
+ // that never reached the prune.
+ expect(mockNetworkingApi.listNamespacedNetworkPolicy).toHaveBeenCalled()
expect(mockNetworkingApi.deleteNamespacedNetworkPolicy).not.toHaveBeenCalledWith(
expect.objectContaining({ name: GFS })
)
diff --git a/workflow-recipes/src/reconciler/secretWatcher.test.ts b/workflow-recipes/src/reconciler/secretWatcher.test.ts
index dd9f5e85c..38296147d 100644
--- a/workflow-recipes/src/reconciler/secretWatcher.test.ts
+++ b/workflow-recipes/src/reconciler/secretWatcher.test.ts
@@ -286,6 +286,13 @@ describe('SecretWatcher', () => {
data: { 'broker-token': 'eA==' },
})
expect(invalidated).toEqual([])
+ // Liveness witness: the same watcher does invalidate for the canonical
+ // ADDED, so the empty list above is not the callback being unwired.
+ watcher.handleEvent('ADDED', {
+ metadata: { name: 'wf-test-recipe-oauth-broker-token', labels },
+ data: { 'broker-token': 'eA==' },
+ })
+ expect(invalidated).toEqual(['test-recipe'])
vi.advanceTimersByTime(DEBOUNCE_MS)
expect(enqueued).toEqual([])
watcher.stop()
diff --git a/workflow-recipes/src/reconciler/secretWatcher.ts b/workflow-recipes/src/reconciler/secretWatcher.ts
index e36325db3..535e22cf7 100644
--- a/workflow-recipes/src/reconciler/secretWatcher.ts
+++ b/workflow-recipes/src/reconciler/secretWatcher.ts
@@ -60,11 +60,10 @@ export class SecretWatcher {
const name = secret.metadata?.name
if (!name) return
- // B3(a): ADDED of the canonical broker token invalidates the ledger even
- // when the key-set matches a previous observation (dedup would drop it).
- // Invalidate only — a watch reconnect replays ADDED for every Secret and
- // must not force a reconcile storm. The 30s ownership backstop (and any
- // later key-set/ownership fan-out) issues the next delete.
+ // B3(a): ADDED of the canonical broker token is reported even when the
+ // key-set matches a previous observation (dedup would drop it). This
+ // watcher never enqueues on it: a watch reconnect replays ADDED for every
+ // Secret, so the callback owner decides whether a reconcile is needed.
if (type === 'ADDED') {
const recipeName = resolveOAuthBrokerTokenWatchRecipe(name, secret.metadata?.labels)
if (recipeName) {
diff --git a/workflow-recipes/src/reconciler/workflowRecipeReconciler.ts b/workflow-recipes/src/reconciler/workflowRecipeReconciler.ts
index 2d19154f6..a74afaf3c 100644
--- a/workflow-recipes/src/reconciler/workflowRecipeReconciler.ts
+++ b/workflow-recipes/src/reconciler/workflowRecipeReconciler.ts
@@ -1089,7 +1089,7 @@ export class WorkflowRecipeReconciler {
*/
private async cleanupPluginWorkloadSdkOrThrow(
recipeName: string,
- options: { preserveWorkflowRuntime?: boolean } = {}
+ options: { preserveWorkflowRuntime?: boolean; recipeUid?: string } = {}
): Promise<void> {
if (!this.workflowReconciler) {
throw new Error('workflow subsystem is not initialized; SDK cleanup cannot be confirmed')
@@ -1469,9 +1469,9 @@ export class WorkflowRecipeReconciler {
* Applies the run-lane NetworkPolicies again from a short-circuit when the
* published status carries the retry marker. Those short-circuits never
* reach `WorkflowReconciler.reconcile()`, so without this a policy left
- * pending a retry would stay unapplied until the run ends. Nothing is
- * pruned: the prune decisions depend on the eligibility verdicts that only
- * the full pass computes.
+ * pending a retry would stay unapplied until the run ends. The retry
+ * re-prunes as well, with a verdict computed for this pass: a prune that
+ * failed is what set the marker, so an apply-only retry could never clear it.
*
* Returns the requeue delay the short-circuit must add: the progress base
* while a policy is still pending, the transient base when the apply threw
@@ -3955,7 +3955,7 @@ export class WorkflowRecipeReconciler {
// a capability may have been removed in an update before Kubernetes emits
// the final delete event, and cleanup must still revoke its host tokens.
if (recipe.spec.pluginWorkloadSdk || recipe.status?.pluginWorkloadSdk) {
- await this.cleanupPluginWorkloadSdkOrThrow(name)
+ await this.cleanupPluginWorkloadSdkOrThrow(name, { recipeUid: recipe.metadata.uid })
}
await this.cleanupDelegationIfNeeded(recipe)
diff --git a/workflow-recipes/src/workflow/workflowReconciler.test.ts b/workflow-recipes/src/workflow/workflowReconciler.test.ts
index 0fc956ec2..c6e1f818d 100644
--- a/workflow-recipes/src/workflow/workflowReconciler.test.ts
+++ b/workflow-recipes/src/workflow/workflowReconciler.test.ts
@@ -1302,8 +1302,11 @@ describe('WorkflowReconciler.reconcileDelete — orphaned Service cleanup', () =
await reconciler.teardownComputePodsForTerminalRun('wf-run-7')
+ // Witness: the teardown ran (three DELETEs) and none of them was
+ // preceded by a raw pod GET, which getPodPresence alone would not show.
expect(crashRecoveryMocks.deletePodIfExists).toHaveBeenCalledTimes(3)
expect(crashRecoveryMocks.getPodPresence).not.toHaveBeenCalled()
+ expect(mockCoreApi.readNamespacedPod).not.toHaveBeenCalled()
})
it('is idempotent: a second pass still DELETE-first (404-safe)', async () => {
@@ -2120,6 +2123,40 @@ describe('WorkflowReconciler — Plugin Workload SDK eager mcp-host', () => {
expect(mockCoreApi.createNamespacedPod).not.toHaveBeenCalled()
})
+ it('forgets the legacy internet policy ledger entry when an SDK-only recipe is cleaned up', async () => {
+ const reconciler = new WorkflowReconciler(makeDeps())
+ const legacyDeletes = () =>
+ mockNetworkingApi.deleteNamespacedNetworkPolicy.mock.calls.filter(
+ ([arg]) => arg.name === 'sdk-only-mcp-servers-egress-internet'
+ )
+ const reconcileSdkOnly = () =>
+ reconciler.reconcilePluginWorkloadSdkOnly(
+ 'sdk-only',
+ 'uid-sdk-only',
+ sandboxNamespace,
+ sdkSpec({
+ agent: undefined,
+ steps: undefined,
+ pluginWorkloadSdk: {
+ clientNotifications: { allowedEventTypes: ['e2e.test'] },
+ allowedCallers: ['sdk-caller'],
+ },
+ })
+ )
+
+ await reconcileSdkOnly()
+ // The witness: the first pass reached the legacy prune and recorded it, so
+ // a second pass skips the DELETE.
+ expect(legacyDeletes()).toHaveLength(1)
+ await reconcileSdkOnly()
+ expect(legacyDeletes()).toHaveLength(1)
+
+ await reconciler.cleanupPluginWorkloadSdk('sdk-only', { recipeUid: 'uid-sdk-only' })
+ await reconcileSdkOnly()
+
+ expect(legacyDeletes()).toHaveLength(2)
+ })
+
it('defers absence reads until pod deletion completes so 404s cannot become unhandled rejections', async () => {
const events: string[] = []
crashRecoveryMocks.waitForPodDeletion.mockImplementationOnce(async () => {
diff --git a/workflow-recipes/src/workflow/workflowReconciler.ts b/workflow-recipes/src/workflow/workflowReconciler.ts
index c13509a84..79d94d3cb 100644
--- a/workflow-recipes/src/workflow/workflowReconciler.ts
+++ b/workflow-recipes/src/workflow/workflowReconciler.ts
@@ -254,6 +254,12 @@ export interface PluginWorkloadSdkCleanupOptions {
* remains the owner of shared runtime resources.
*/
preserveWorkflowRuntime?: boolean
+ /**
+ * Set when the recipe itself is being deleted. An SDK-only recipe never
+ * reaches `reconcileDelete`, so this is the only place its legacy internet
+ * policy ledger entry can be forgotten.
+ */
+ recipeUid?: string
}
export const NETWORK_POLICY_OWNERSHIP_CONDITION_TYPE = 'WorkflowNetworkPolicyOwnership'
@@ -1465,6 +1471,9 @@ export class WorkflowReconciler {
const ns = this.deps.config.sandboxNamespace
const preserveWorkflowRuntime = options.preserveWorkflowRuntime === true
this.pluginWorkloadSdkProvisioner.clearRecipeState(recipeName)
+ if (options.recipeUid) {
+ this.forgetLegacyMcpServersInternetEgress(recipeName, options.recipeUid)
+ }
const sdkNetworkPolicyNames = [
`${recipeName}-workload-to-mcp-host-sdk-ingress`,
`${recipeName}-workload-to-mcp-host-sdk-egress`,
@@ -2979,10 +2988,13 @@ export class WorkflowReconciler {
/**
* Reapply the run-lane NetworkPolicies outside reconcile(). The running and
* active short-circuits return before reconcile(), so a policy a previous
- * pass left pending (terminating or contended) would otherwise stay as it is
- * until the run ends. Builds the same set reconcile() applies and only
- * applies it: no codex, grok or legacy prune, because those follow an
- * eligibility verdict that can change while a run is in progress.
+ * pass left pending (terminating, contended, or a prune that failed) would
+ * otherwise stay as it is until the run ends. Runs the same build, apply and
+ * prune unit reconcile() runs, with a verdict computed now: an uncertain
+ * Codex or Grok verdict keeps its proxy policy, and the catalog gate keeps
+ * everything the run lane does not own. A prune that failed sets
+ * retryPending, so without the prune here the marker would requeue forever
+ * and never reach the leftover.
*/
async retryRunLaneNetworkPolicies(
recipeName: string,
@@ -3007,7 +3019,7 @@ export class WorkflowReconciler {
runtimeScopeRecipeName,
codexView
)
- const { policies } = await this.buildWorkflowNetworkPoliciesForSpec(
+ return this.applyWorkflowNetworkPolicies(
recipeName,
recipeUid,
spec,
@@ -3017,7 +3029,6 @@ export class WorkflowReconciler {
false,
codexVerdict.grokProjection
)
- return this.applyNetworkPolicyList(policies)
}
async ensureCoordinatorRuntimeCredentials(
@@ -3210,6 +3221,11 @@ export class WorkflowReconciler {
return recipeUid && recipeUid.length > 0 ? recipeUid : recipeName
}
+ private forgetLegacyMcpServersInternetEgress(recipeName: string, recipeUid?: string): void {
+ this.prunedLegacyMcpServersInternetEgress.delete(recipeName)
+ if (recipeUid) this.prunedLegacyMcpServersInternetEgress.delete(recipeUid)
+ }
+
private async pruneLegacyMcpServersInternetEgressPolicy(
recipeName: string,
recipeUid?: string
@@ -3779,8 +3795,7 @@ export class WorkflowReconciler {
const log = createLogger('wrc', recipeName)
log.info(`Deleting workflow resources`)
this.pluginWorkloadSdkProvisioner.clearRecipeState(recipeName)
- this.prunedLegacyMcpServersInternetEgress.delete(recipeName)
- if (recipeUid) this.prunedLegacyMcpServersInternetEgress.delete(recipeUid)
+ this.forgetLegacyMcpServersInternetEgress(recipeName, recipeUid)
// Best-effort cleanup of the recipe's subdirectory on the workflow output PVC.
// Must run BEFORE pod deletion while mcp-host is still reachable; failures
diff --git a/workflow-recipes/tests/unit/workflow/workflowReconciler.test.ts b/workflow-recipes/tests/unit/workflow/workflowReconciler.test.ts
index bec33d15e..f9087cac6 100644
--- a/workflow-recipes/tests/unit/workflow/workflowReconciler.test.ts
+++ b/workflow-recipes/tests/unit/workflow/workflowReconciler.test.ts
@@ -3090,6 +3090,37 @@ describe('WorkflowReconciler — reconcile loop', () => {
})
})
+ it('keys the legacy internet NP ledger by recipe uid, so a recreated recipe deletes again', async () => {
+ const networkingApi = makeNetworkingApi() as ReturnType<typeof makeNetworkingApi>
+ const reconciler = new WorkflowReconciler(
+ makeDeps({
+ networkingApi: networkingApi as never,
+ })
+ )
+ const legacyDeletes = () =>
+ networkingApi.deleteNamespacedNetworkPolicy.mock.calls.filter(
+ ([arg]) =>
+ arg.name === 'test-wf-mcp-servers-egress-internet' && arg.namespace === 'mcp-server'
+ )
+ const reconcileWithUid = (uid: string) =>
+ reconciler.reconcile(
+ 'test-wf',
+ uid,
+ 'sandbox-recipes',
+ makeSpec({ mcpServers: ['redis-mcp'] })
+ )
+
+ await reconcileWithUid('uid-a')
+ await reconcileWithUid('uid-a')
+ // Witness: the ledger suppresses the repeat for the same uid.
+ expect(legacyDeletes()).toHaveLength(1)
+
+ // Same name, new uid, and no reconcileDelete seen by this process (another
+ // replica handled the delete): a new identity must not be suppressed.
+ await reconcileWithUid('uid-b')
+ expect(legacyDeletes()).toHaveLength(2)
+ })
+
describe('B2 desired-set NetworkPolicy prune', () => {
const RECIPE = 'test-wf'
const SANDBOX = 'sandbox-recipes'
@@ -3333,7 +3364,7 @@ describe('WorkflowReconciler — reconcile loop', () => {
expect(apiserver.live.has(apiserver.key(SANDBOX, COORD_TO_WRC))).toBe(true)
})
- it('does not prune a recipes-lane policy; LIST uses the two-term selector once', async () => {
+ it('does not prune an uncatalogued policy carrying the recipe label pair; LIST is issued once', async () => {
const apiserver = makeApiserverNetworkingApi()
seedLivePolicy(apiserver, `${RECIPE}-oauth-broker-egress`, {
'clerum.io/recipe': RECIPE,
@@ -3505,6 +3536,14 @@ describe('WorkflowReconciler — reconcile loop', () => {
expect(summary.retryPending).toBe(true)
expect(sandboxDeletes(apiserver.api)).toEqual([])
expect(apiserver.live.has(apiserver.key(SANDBOX, GROK_PROXY))).toBe(true)
+ // The apply outcome survives: the legacy internet policy prune that runs
+ // after the catalog prune is still attempted despite the failed LIST.
+ expect(apiserver.api.listNamespacedNetworkPolicy).toHaveBeenCalled()
+ expect(
+ apiserver.api.deleteNamespacedNetworkPolicy.mock.calls.filter(
+ ([arg]: [{ name: string }]) => arg.name === `${RECIPE}-mcp-servers-egress-internet`
+ )
+ ).toHaveLength(1)
})
})
@@ -5121,6 +5160,107 @@ describe('WorkflowReconciler — reconcile loop', () => {
expect(apiserver.api.replaceNamespacedNetworkPolicy).toHaveBeenCalledTimes(0)
expect(apiserver.api.deleteNamespacedNetworkPolicy).toHaveBeenCalledTimes(0)
})
+
+ // A prune failure sets retryPending, and this pass is the only one that
+ // runs while the run is in progress or active. It has to prune again, or
+ // the marker requeues forever without ever reaching the leftover.
+ describe('when a prune left a leftover behind', () => {
+ const LEFTOVER = 'test-wf-mcp-host-to-grok-proxy'
+ const TWO_TERM_SELECTOR = 'clerum.io/recipe=test-wf,clerum.io/managed-by=wrc'
+ type Converged = Awaited<ReturnType<typeof convergedRunLane>>['apiserver']
+
+ function seedLeftover(apiserver: Converged) {
+ apiserver.live.set(apiserver.key('sandbox-recipes', LEFTOVER), {
+ apiVersion: 'networking.k8s.io/v1',
+ kind: 'NetworkPolicy',
+ metadata: {
+ name: LEFTOVER,
+ namespace: 'sandbox-recipes',
+ labels: { 'clerum.io/recipe': 'test-wf', 'clerum.io/managed-by': 'wrc' },
+ },
+ spec: { podSelector: {}, policyTypes: ['Egress'] },
+ })
+ }
+
+ it('deletes the leftover on the retry pass', async () => {
+ const { apiserver, reconciler } = await convergedRunLane()
+ apiserver.api.listNamespacedNetworkPolicy.mockClear()
+ seedLeftover(apiserver)
+
+ const summary = await reconciler.retryRunLaneNetworkPolicies('test-wf', 'uid-123', spec())
+
+ expect(apiserver.api.listNamespacedNetworkPolicy).toHaveBeenCalledWith({
+ namespace: 'sandbox-recipes',
+ labelSelector: TWO_TERM_SELECTOR,
+ })
+ expect(summary).toEqual({ conflicts: [], retryPending: false })
+ expect(apiserver.live.has(apiserver.key('sandbox-recipes', LEFTOVER))).toBe(false)
+ expect(
+ apiserver.api.deleteNamespacedNetworkPolicy.mock.calls.map(([arg]) => arg.name)
+ ).toEqual([LEFTOVER])
+ })
+
+ it('keeps reporting the retry until the leftover delete succeeds', async () => {
+ const { apiserver, reconciler } = await convergedRunLane()
+ apiserver.api.listNamespacedNetworkPolicy.mockClear()
+ seedLeftover(apiserver)
+ apiserver.api.deleteNamespacedNetworkPolicy.mockRejectedValueOnce({
+ code: 403,
+ message: 'forbidden',
+ })
+
+ const first = await reconciler.retryRunLaneNetworkPolicies('test-wf', 'uid-123', spec())
+ expect(apiserver.api.listNamespacedNetworkPolicy).toHaveBeenCalledTimes(1)
+ expect(first.retryPending).toBe(true)
+ expect(apiserver.live.has(apiserver.key('sandbox-recipes', LEFTOVER))).toBe(true)
+
+ const second = await reconciler.retryRunLaneNetworkPolicies('test-wf', 'uid-123', spec())
+ expect(second).toEqual({ conflicts: [], retryPending: false })
+ expect(apiserver.live.has(apiserver.key('sandbox-recipes', LEFTOVER))).toBe(false)
+ })
+
+ it('keeps reporting the retry while the prune LIST fails', async () => {
+ const { apiserver, reconciler } = await convergedRunLane()
+ apiserver.api.listNamespacedNetworkPolicy.mockClear()
+ seedLeftover(apiserver)
+ apiserver.api.listNamespacedNetworkPolicy.mockRejectedValueOnce({
+ code: 500,
+ message: 'boom',
+ })
+
+ const first = await reconciler.retryRunLaneNetworkPolicies('test-wf', 'uid-123', spec())
+ expect(apiserver.api.listNamespacedNetworkPolicy).toHaveBeenCalledTimes(1)
+ expect(first.retryPending).toBe(true)
+ expect(apiserver.live.has(apiserver.key('sandbox-recipes', LEFTOVER))).toBe(true)
+
+ const second = await reconciler.retryRunLaneNetworkPolicies('test-wf', 'uid-123', spec())
+ expect(second.retryPending).toBe(false)
+ expect(apiserver.live.has(apiserver.key('sandbox-recipes', LEFTOVER))).toBe(false)
+ })
+
+ it('retries a failed legacy internet policy delete', async () => {
+ const apiserver = makeApiserverNetworkingApi()
+ const reconciler = new WorkflowReconciler(
+ makeDeps({ networkingApi: apiserver.api as never })
+ )
+ apiserver.api.deleteNamespacedNetworkPolicy.mockRejectedValueOnce({
+ code: 403,
+ message: 'forbidden',
+ })
+ const legacyDeletes = () =>
+ apiserver.api.deleteNamespacedNetworkPolicy.mock.calls.filter(
+ ([arg]) => arg.name === 'test-wf-mcp-servers-egress-internet'
+ )
+
+ const first = await reconciler.retryRunLaneNetworkPolicies('test-wf', 'uid-123', spec())
+ expect(legacyDeletes()).toHaveLength(1)
+ expect(first.retryPending).toBe(true)
+
+ const second = await reconciler.retryRunLaneNetworkPolicies('test-wf', 'uid-123', spec())
+ expect(legacyDeletes()).toHaveLength(2)
+ expect(second.retryPending).toBe(false)
+ })
+ })
})
})