Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
fix(workflow-recipes): stop WRC write-storm DELETE 404s on leftover policies
Alfredo Lopez219930f8393b
diff --git a/workflow-recipes/package-lock.json b/workflow-recipes/package-lock.jsonindex 2b9c3013c..c6c0eb7d4 100644--- a/workflow-recipes/package-lock.json+++ b/workflow-recipes/package-lock.json@@ -1,12 +1,12 @@{"name": "workload-recipes-controller",- "version": "0.1.405",+ "version": "0.1.406","lockfileVersion": 3,"requires": true,"packages": {"": {"name": "workload-recipes-controller",- "version": "0.1.405",+ "version": "0.1.406","license": "SEE LICENSE IN LICENSE","dependencies": {"@aws-sdk/client-s3": "^3.1009.0",diff --git a/workflow-recipes/package.json b/workflow-recipes/package.jsonindex c43d9d0b3..1086afb5c 100644--- a/workflow-recipes/package.json+++ b/workflow-recipes/package.json@@ -1,6 +1,6 @@{"name": "workload-recipes-controller",- "version": "0.1.405",+ "version": "0.1.406","description": "Workload Recipes Controller (WRC) - WorkloadRecipe lifecycle management and MCP server interface","main": "dist/main.js","engines": {diff --git a/workflow-recipes/src/k8sClient.ts b/workflow-recipes/src/k8sClient.tsindex 3239a9ff6..7a3a6afe9 100644--- a/workflow-recipes/src/k8sClient.ts+++ b/workflow-recipes/src/k8sClient.ts@@ -907,8 +907,14 @@ export class WorkflowRecipeWatcher implements WorkflowRecipeProvider {]),]for (const namespace of watchedNamespaces) {- const handler = new SecretWatcher(this.secretReverseIndex, name =>- this.triggerSecretDrivenReconcile(name)+ const handler = new SecretWatcher(+ this.secretReverseIndex,+ name => this.triggerSecretDrivenReconcile(name),+ 10_000,+ recipeName => {+ this.reconciler.invalidateOAuthBrokerDeleteLedger(recipeName)+ this.triggerSecretDrivenReconcile(recipeName)+ })const loop = new K8sSecretWatchLoop(this.kc, namespace, handler)this.secretWatchLoops.push(loop)diff --git a/workflow-recipes/src/reconciler/oauthBrokerDeleteLedger.ts b/workflow-recipes/src/reconciler/oauthBrokerDeleteLedger.tsnew file mode 100644index 000000000..76f3a5d4a--- /dev/null+++ b/workflow-recipes/src/reconciler/oauthBrokerDeleteLedger.ts@@ -0,0 +1,48 @@+/**+ * In-process B3(a)(b) ledger for oauth-broker Secret / NetworkPolicy deletes.+ * Transition is (recipe, metadata.generation). The NetworkPolicy side adds a+ * 1h TTL so a later pass can reap a leftover without a watch. Indexing only+ * by recipe name is the E.6 vacuity mutation: a generation bump must miss.+ */++export const OAUTH_BROKER_NP_TTL_MS = 60 * 60 * 1000++function ledgerKey(recipeName: string, generation: number | undefined): string {+ return `${recipeName}:${generation ?? 0}`+}++export class OAuthBrokerDeleteLedger {+ private readonly secrets = new Set<string>()+ private readonly policies = new Map<string, number>()++ shouldDeleteSecret(recipeName: string, generation: number | undefined): boolean {+ return !this.secrets.has(ledgerKey(recipeName, generation))+ }++ recordSecretDelete(recipeName: string, generation: number | undefined): void {+ this.secrets.add(ledgerKey(recipeName, generation))+ }++ shouldDeletePolicy(+ recipeName: string,+ generation: number | undefined,+ nowMs = Date.now()+ ): boolean {+ const expiresAt = this.policies.get(ledgerKey(recipeName, generation))+ return expiresAt === undefined || nowMs >= expiresAt+ }++ recordPolicyDelete(recipeName: string, generation: number | undefined, nowMs = Date.now()): void {+ this.policies.set(ledgerKey(recipeName, generation), nowMs + OAUTH_BROKER_NP_TTL_MS)+ }++ invalidate(recipeName: string): void {+ const prefix = `${recipeName}:`+ for (const key of [...this.secrets]) {+ if (key.startsWith(prefix)) this.secrets.delete(key)+ }+ for (const key of [...this.policies.keys()]) {+ if (key.startsWith(prefix)) this.policies.delete(key)+ }+ }+}diff --git a/workflow-recipes/src/reconciler/reconciler.test.ts b/workflow-recipes/src/reconciler/reconciler.test.tsindex f5853340b..1f18b266a 100644--- a/workflow-recipes/src/reconciler/reconciler.test.ts+++ b/workflow-recipes/src/reconciler/reconciler.test.ts@@ -15368,6 +15368,121 @@ describe('WorkflowRecipeReconciler', () => {})})+ describe('B3(a)(b) oauth-broker delete-on-transition ledger', () => {+ const SECRET_NAME = 'wf-test-recipe-oauth-broker-token'+ const POLICY_NAME = 'wf-test-recipe-oauth-broker-egress'++ function reapRecipe(generation: number): WorkflowRecipeCRD {+ return makeRecipe({+ metadata: {+ name: 'test-recipe',+ namespace: 'sandbox-recipes',+ uid: 'uid-123',+ generation,+ },+ })+ }++ function secretDeletes(): number {+ return mockCoreApi.deleteNamespacedSecret.mock.calls.filter(+ ([arg]) => arg.name === SECRET_NAME+ ).length+ }++ function policyDeletes(): number {+ return mockNetworkingApi.deleteNamespacedNetworkPolicy.mock.calls.filter(+ ([arg]) => arg.name === POLICY_NAME+ ).length+ }++ async function reapPolicy(recipe: WorkflowRecipeCRD): Promise<void> {+ await (+ reconciler as unknown as {+ reconcileOAuthBrokerEgressPolicy: (next: WorkflowRecipeCRD) => Promise<void>+ }+ ).reconcileOAuthBrokerEgressPolicy(recipe)+ }++ it('deletes the Secret once for the same generation and logs the skip', async () => {+ const recipe = reapRecipe(4)+ const infoSpy = captureLogger('info')++ await reconciler.ensureOAuthBrokerTokenSecret(recipe)+ await reconciler.ensureOAuthBrokerTokenSecret(recipe)++ expect(secretDeletes()).toBe(1)+ expect(infoSpy).toHaveBeenCalledWith(+ expect.stringContaining('Skipping oauth-broker-token delete'),+ expect.objectContaining({ recipe: 'test-recipe', generation: 4 })+ )+ infoSpy.mockRestore()+ })++ it('deletes the Secret again when metadata.generation changes', async () => {+ await reconciler.ensureOAuthBrokerTokenSecret(reapRecipe(4))+ await reconciler.ensureOAuthBrokerTokenSecret(reapRecipe(4))+ expect(secretDeletes()).toBe(1)++ await reconciler.ensureOAuthBrokerTokenSecret(reapRecipe(5))+ expect(secretDeletes()).toBe(2)+ })++ it('deletes the Secret again on a new process (fresh reconciler)', async () => {+ await reconciler.ensureOAuthBrokerTokenSecret(reapRecipe(4))+ expect(secretDeletes()).toBe(1)++ const next = new WorkflowRecipeReconciler(new k8s.KubeConfig(), undefined, {+ verifyWorkflowRunProvenance: mockVerifyWorkflowRunProvenance,+ })+ await next.ensureOAuthBrokerTokenSecret(reapRecipe(4))+ expect(secretDeletes()).toBe(2)+ })++ it('deletes the Secret again after ADDED invalidation', async () => {+ const recipe = reapRecipe(4)+ await reconciler.ensureOAuthBrokerTokenSecret(recipe)+ await reconciler.ensureOAuthBrokerTokenSecret(recipe)+ expect(secretDeletes()).toBe(1)++ reconciler.invalidateOAuthBrokerDeleteLedger('test-recipe')+ await reconciler.ensureOAuthBrokerTokenSecret(recipe)+ expect(secretDeletes()).toBe(2)+ })++ it('deletes the Secret again after reconcileDelete invalidates the ledger', async () => {+ const recipe = reapRecipe(4)+ await reconciler.ensureOAuthBrokerTokenSecret(recipe)+ await reconciler.ensureOAuthBrokerTokenSecret(recipe)+ expect(secretDeletes()).toBe(1)++ await reconciler.reconcileDelete(recipe)+ mockCoreApi.deleteNamespacedSecret.mockClear()+ await reconciler.ensureOAuthBrokerTokenSecret(recipe)+ expect(secretDeletes()).toBe(1)+ })++ it('deletes the NetworkPolicy once for the same generation, then again after the 1h TTL', async () => {+ vi.useFakeTimers()+ vi.setSystemTime(new Date('2026-09-25T00:00:00.000Z'))+ const recipe = reapRecipe(4)+ const infoSpy = captureLogger('info')++ await reapPolicy(recipe)+ await reapPolicy(recipe)+ expect(policyDeletes()).toBe(1)+ expect(infoSpy).toHaveBeenCalledWith(+ expect.stringContaining('Skipping oauth-broker-egress delete'),+ expect.objectContaining({ recipe: 'test-recipe', generation: 4 })+ )++ vi.setSystemTime(new Date('2026-09-25T01:00:00.000Z'))+ await reapPolicy(recipe)+ expect(policyDeletes()).toBe(2)+ infoSpy.mockRestore()+ vi.useRealTimers()+ })+ })+// ─── Broker-token issuance failure is non-fatal ─────────────────────────//// A transient control-api blip during reconcile (e.g. it is mid-restart)diff --git a/workflow-recipes/src/reconciler/secretWatcher.test.ts b/workflow-recipes/src/reconciler/secretWatcher.test.tsindex 7f7e07997..2a9ff5425 100644--- a/workflow-recipes/src/reconciler/secretWatcher.test.ts+++ b/workflow-recipes/src/reconciler/secretWatcher.test.ts@@ -215,4 +215,35 @@ describe('SecretWatcher', () => {vi.advanceTimersByTime(DEBOUNCE_MS)expect(enqueued.sort()).toEqual(['recipe-a', 'recipe-b'])})++ it('invalidates the oauth-broker-token ledger on ADDED before key-set dedup', () => {+ const reverseIndex = new SecretReverseIndex()+ const enqueued: string[] = []+ const invalidated: string[] = []+ const watcher = new SecretWatcher(+ reverseIndex,+ name => {+ enqueued.push(name)+ },+ DEBOUNCE_MS,+ recipeName => {+ invalidated.push(recipeName)+ }+ )+ const brokerSecret = {+ metadata: {+ name: 'wf-test-recipe-oauth-broker-token',+ labels: {+ 'clerum.io/component': 'oauth-broker-token',+ 'clerum.io/recipe': 'test-recipe',+ },+ },+ data: { 'broker-token': 'eA==' },+ }++ watcher.handleEvent('ADDED', brokerSecret)+ watcher.handleEvent('ADDED', brokerSecret)+ expect(invalidated).toEqual(['test-recipe', 'test-recipe'])+ watcher.stop()+ })})diff --git a/workflow-recipes/src/reconciler/secretWatcher.ts b/workflow-recipes/src/reconciler/secretWatcher.tsindex c1725b6ae..deb026968 100644--- a/workflow-recipes/src/reconciler/secretWatcher.ts+++ b/workflow-recipes/src/reconciler/secretWatcher.ts@@ -41,7 +41,8 @@ export class SecretWatcher {constructor(private readonly reverseIndex: SecretReverseIndex,private readonly enqueueReconcile: (recipeName: string) => void,- private readonly debounceMs: number = 10_000+ private readonly debounceMs: number = 10_000,+ private readonly onOAuthBrokerTokenAdded?: (recipeName: string) => void) {}/**@@ -58,6 +59,16 @@ export class SecretWatcher {const name = secret.metadata?.nameif (!name) return+ // B3(a): ADDED of the broker token must invalidate the ledger even when+ // the key-set matches a previous observation (dedup would otherwise drop it).+ if (type === 'ADDED') {+ const labels = secret.metadata?.labels ?? {}+ if (labels['clerum.io/component'] === 'oauth-broker-token') {+ const recipeName = labels['clerum.io/recipe']+ if (recipeName) this.onOAuthBrokerTokenAdded?.(recipeName)+ }+ }+const newKeys =type === 'DELETED'? new Set<string>()diff --git a/workflow-recipes/src/reconciler/workflowRecipeReconciler.ts b/workflow-recipes/src/reconciler/workflowRecipeReconciler.tsindex c73653edf..ff367c206 100644--- a/workflow-recipes/src/reconciler/workflowRecipeReconciler.ts+++ b/workflow-recipes/src/reconciler/workflowRecipeReconciler.ts@@ -127,6 +127,7 @@ import {decideNetworkPolicyConvergence,networkPolicyMetadataMatchesDesired,} from './networkPolicyConvergence'+import { OAuthBrokerDeleteLedger } from './oauthBrokerDeleteLedger'import { issueOAuthBrokerToken } from './oauthBrokerTokenIssuerClient'import {PLUGIN_WORKLOAD_SDK_CONDITION_TYPE,@@ -941,10 +942,15 @@ export class WorkflowRecipeReconciler {// F2 gate) and the safe direction; under-refreshing would reopen #299.private externalEgressMinObservedTtlMs = Infinityprivate secretReverseIndex: SecretReverseIndex | null+ private readonly oauthBrokerDeleteLedger = new OAuthBrokerDeleteLedger()private verifyWorkflowRunProvenance: NonNullable<WorkflowRecipeReconcilerDeps['verifyWorkflowRunProvenance']>+ invalidateOAuthBrokerDeleteLedger(recipeName: string): void {+ this.oauthBrokerDeleteLedger.invalidate(recipeName)+ }+constructor(kc: k8s.KubeConfig, config?: OperatorConfig, deps?: WorkflowRecipeReconcilerDeps) {this.appsApi = kc.makeApiClient(k8s.AppsV1Api)this.batchApi = kc.makeApiClient(k8s.BatchV1Api)@@ -3922,6 +3928,7 @@ export class WorkflowRecipeReconciler {createLogger('wrc', recipe.metadata.name).info('Deleting recipe resources', { name })this.secretReverseIndex?.delete(name)+ this.oauthBrokerDeleteLedger.invalidate(name)// ─── Workflow Delete (Stage 1) ────────────────────────────────────const isWorkflow = recipe.spec.steps !== undefined && recipe.spec.steps.length > 0@@ -7499,10 +7506,19 @@ export class WorkflowRecipeReconciler {const secretName = rb.oauthBrokerTokenSecretName(recipeName)if (!rb.recipeHasBackgroundAccessClient(recipe)) {+ const generation = recipe.metadata.generation+ if (!this.oauthBrokerDeleteLedger.shouldDeleteSecret(recipeName, generation)) {+ createLogger('wrc', recipeName).info(+ 'Skipping oauth-broker-token delete; generation already seen',+ { recipe: recipeName, generation }+ )+ return+ }await this.safeDelete(() => this.coreApi.deleteNamespacedSecret({ name: secretName, namespace: ns }),`Secret "${secretName}" in ${ns}`)+ this.oauthBrokerDeleteLedger.recordSecretDelete(recipeName, generation)return}@@ -7565,10 +7581,20 @@ export class WorkflowRecipeReconciler {this.config.controlPlaneNamespace)if (!policy) {+ const recipeName = recipe.metadata.name+ const generation = recipe.metadata.generation+ if (!this.oauthBrokerDeleteLedger.shouldDeletePolicy(recipeName, generation)) {+ createLogger('wrc', recipeName).info(+ 'Skipping oauth-broker-egress delete; generation already seen',+ { recipe: recipeName, generation }+ )+ return+ }await this.safeDelete(() => this.networkingApi.deleteNamespacedNetworkPolicy({ name: policyName, namespace: ns }),`NetworkPolicy "${policyName}" in ${ns}`)+ this.oauthBrokerDeleteLedger.recordPolicyDelete(recipeName, generation)return}await this.applyNetworkPolicy(policy, ns, { family: 'oauth-broker-egress' })diff --git a/workflow-recipes/src/workflow/workflowReconciler.codexScopeProvenance.test.ts b/workflow-recipes/src/workflow/workflowReconciler.codexScopeProvenance.test.tsindex 5cf95ac09..8083ea7f4 100644--- a/workflow-recipes/src/workflow/workflowReconciler.codexScopeProvenance.test.ts+++ b/workflow-recipes/src/workflow/workflowReconciler.codexScopeProvenance.test.ts@@ -381,6 +381,22 @@ describe('WorkflowReconciler Codex scope provenance', () => {})runtimeTokenIssuerMocks.issueMcpHostRuntimeTokens.mockClear()crashRecoveryMocks.deletePodIfExists.mockClear()+ // B2: withdraw is LIST membership of the leftover proxy, not a named DELETE+ // of an absent object. Seed the leftover created on the first pass.+ networkingApi.listNamespacedNetworkPolicy.mockResolvedValue({+ items: [+ {+ metadata: {+ name: CODEX_PROXY_POLICY,+ namespace: sandboxNamespace,+ labels: {+ 'clerum.io/recipe': 'codex-recipe',+ 'clerum.io/managed-by': 'wrc',+ },+ },+ },+ ],+ })await reconcileRecipe(reconciler,@@ -400,6 +416,12 @@ describe('WorkflowReconciler Codex scope provenance', () => {expect(remintPatch.body?.metadata?.annotations?.['clerum.io/mcp-host-runtime-token-generation']).toBe('1')+ expect(networkingApi.listNamespacedNetworkPolicy).toHaveBeenCalledWith(+ expect.objectContaining({+ namespace: sandboxNamespace,+ labelSelector: 'clerum.io/recipe=codex-recipe,clerum.io/managed-by=wrc',+ })+ )expect(networkingApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledWith({name: CODEX_PROXY_POLICY,namespace: sandboxNamespace,@@ -444,6 +466,21 @@ describe('WorkflowReconciler Codex scope provenance', () => {})runtimeTokenIssuerMocks.issueMcpHostRuntimeTokens.mockClear()crashRecoveryMocks.deletePodIfExists.mockClear()+ // B2: leftover proxy stays listed so withdraw prunes by membership, not by name.+ networkingApi.listNamespacedNetworkPolicy.mockResolvedValue({+ items: [+ {+ metadata: {+ name: CODEX_PROXY_POLICY,+ namespace: sandboxNamespace,+ labels: {+ 'clerum.io/recipe': 'codex-recipe',+ 'clerum.io/managed-by': 'wrc',+ },+ },+ },+ ],+ })await reconcileRecipe(reconciler, makeCodexSpec())expect(issueMcpHostRuntimeTokens).toHaveBeenCalled()@@ -453,6 +490,12 @@ describe('WorkflowReconciler Codex scope provenance', () => {'codex-recipe-mcp-host',sandboxNamespace)+ expect(networkingApi.listNamespacedNetworkPolicy).toHaveBeenCalledWith(+ expect.objectContaining({+ namespace: sandboxNamespace,+ labelSelector: 'clerum.io/recipe=codex-recipe,clerum.io/managed-by=wrc',+ })+ )expect(networkingApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledWith({name: CODEX_PROXY_POLICY,namespace: sandboxNamespace,diff --git a/workflow-recipes/src/workflow/workflowReconciler.grokWrcFlag.test.ts b/workflow-recipes/src/workflow/workflowReconciler.grokWrcFlag.test.tsindex 8e02c5524..c5535ed92 100644--- a/workflow-recipes/src/workflow/workflowReconciler.grokWrcFlag.test.ts+++ b/workflow-recipes/src/workflow/workflowReconciler.grokWrcFlag.test.ts@@ -294,7 +294,15 @@ describe('WorkflowReconciler Grok WRC switch', () => {expect(runtimeTokenIssuerMocks.issueMcpHostRuntimeTokens).toHaveBeenCalled()expect(issuedScopes()).not.toContain('llm:grok:execute')expect(createdPolicyNames(networkingApi)).not.toContain(GROK_PROXY_POLICY)- expect(networkingApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledWith(+ // B2: withheld proxy is not in the desired apply set; LIST membership is+ // empty, so there is no leftover DELETE of the named Grok policy.+ expect(networkingApi.listNamespacedNetworkPolicy).toHaveBeenCalledWith(+ expect.objectContaining({+ namespace: sandboxNamespace,+ labelSelector: `clerum.io/recipe=${RECIPE},clerum.io/managed-by=wrc`,+ })+ )+ expect(networkingApi.deleteNamespacedNetworkPolicy).not.toHaveBeenCalledWith(expect.objectContaining({ name: GROK_PROXY_POLICY }))const env = mcpHostEnvNames(coreApi)diff --git a/workflow-recipes/src/workflow/workflowReconciler.ts b/workflow-recipes/src/workflow/workflowReconciler.tsindex 0ca2a594c..e3f52de39 100644--- a/workflow-recipes/src/workflow/workflowReconciler.ts+++ b/workflow-recipes/src/workflow/workflowReconciler.ts@@ -1120,6 +1120,8 @@ export class WorkflowReconciler {snapshot: CodexCatalogSnapshot} = { snapshot: { flagEnabled: false } }private readonly codexContexts = new Map<string, CodexReconcileContext>()+ /** B3(c): one process-lifetime DELETE of the leftover mcp-servers internet NP. */+ private readonly prunedLegacyMcpServersInternetEgress = new Set<string>()constructor(private readonly deps: WorkflowReconcilerDeps) {this.pluginWorkloadSdkProvisioner = new PluginWorkloadSdkProvisioner({@@ -1858,11 +1860,18 @@ export class WorkflowReconciler {}if (awaitsTriggeredRun) {+ const samePassPhase: Partial<Record<WorkflowRuntimeComponent, string | undefined>> = {+ 'workflow-coordinator': coordPhase,+ 'workflow-mcp-host': mcpHostPhase,+ }for (const component of runtime.cleanup.deleteBeforeTriggeredRun) {// Plugin Workload SDK: keep the mcp-host alive across the// awaiting-trigger window — it hosts the always-on SDK server.// Only the coordinator (and run-scoped pods) are torn down here.if (eagerMcpHostForSdk && component === 'workflow-mcp-host') continue+ // B3(d): same-pass getPodPhase undefined is the 404 we already paid.+ // Do not DELETE again. Unread mcp-host is not in this list (needsMcpHost).+ if (samePassPhase[component] === undefined) continueawait this.deleteRuntimeComponentIfExists(recipeName, component)}@@ -3175,12 +3184,14 @@ export class WorkflowReconciler {}private async pruneLegacyMcpServersInternetEgressPolicy(recipeName: string): Promise<void> {+ if (this.prunedLegacyMcpServersInternetEgress.has(recipeName)) returnawait this.safeDelete(() =>this.deps.networkingApi.deleteNamespacedNetworkPolicy({name: `${recipeName}-mcp-servers-egress-internet`,namespace: this.deps.config.mcpServerNamespace,}))+ this.prunedLegacyMcpServersInternetEgress.add(recipeName)}private needsRuntimeHttpEgressRefresh(spec: WorkflowRecipeSpec): boolean {@@ -3354,27 +3365,43 @@ export class WorkflowReconciler {)const summary = await this.applyNetworkPolicyList(policies)const policyNames = new Set(policies.map(policy => policy.metadata?.name))+ await this.pruneUndesiredRunLaneNetworkPolicies(recipeName, policyNames, {+ skipCodexProxy: codexProjection.eligibility === 'uncertain',+ skipGrokProxy: grokProjection?.eligibility === 'uncertain',+ })+ await this.pruneLegacyMcpServersInternetEgressPolicy(recipeName)+ return summary+ }++ /**+ * Drop run-lane NetworkPolicies that exist and are no longer desired.+ * Two-term selector only (`managed-by=wrc`): a single-term recipe selector+ * would also return the recipes-lane policies in the same namespace.+ * Uncertain Codex/Grok eligibility keeps those proxy names even when they+ * are absent from this pass's desired set.+ */+ private async pruneUndesiredRunLaneNetworkPolicies(+ recipeName: string,+ desiredNames: Set<string | undefined>,+ uncertain: { skipCodexProxy: boolean; skipGrokProxy: boolean }+ ): Promise<void> {+ const namespace = this.deps.config.sandboxNamespace+ const labelSelector = `clerum.io/recipe=${recipeName},clerum.io/managed-by=wrc`+ const list = await this.deps.networkingApi.listNamespacedNetworkPolicy({+ namespace,+ labelSelector,+ })const codexProxyPolicyName = `${recipeName}-mcp-host-to-codex-proxy`const grokProxyPolicyName = `${recipeName}-mcp-host-to-grok-proxy`-- if (!policyNames.has(codexProxyPolicyName) && codexProjection.eligibility !== 'uncertain') {- await this.safeDelete(() =>- this.deps.networkingApi.deleteNamespacedNetworkPolicy({- name: codexProxyPolicyName,- namespace: this.deps.config.sandboxNamespace,- })- )- }- if (!policyNames.has(grokProxyPolicyName) && grokProjection?.eligibility !== 'uncertain') {+ for (const policy of list.items ?? []) {+ const name = policy.metadata?.name+ if (!name || desiredNames.has(name)) continue+ if (name === codexProxyPolicyName && uncertain.skipCodexProxy) continue+ if (name === grokProxyPolicyName && uncertain.skipGrokProxy) continueawait this.safeDelete(() =>- this.deps.networkingApi.deleteNamespacedNetworkPolicy({- name: grokProxyPolicyName,- namespace: this.deps.config.sandboxNamespace,- })+ this.deps.networkingApi.deleteNamespacedNetworkPolicy({ name, namespace }))}- await this.pruneLegacyMcpServersInternetEgressPolicy(recipeName)- return summary}/**diff --git a/workflow-recipes/tests/unit/workflow/workflowReconciler.test.ts b/workflow-recipes/tests/unit/workflow/workflowReconciler.test.tsindex fcc659b32..0645c615b 100644--- a/workflow-recipes/tests/unit/workflow/workflowReconciler.test.ts+++ b/workflow-recipes/tests/unit/workflow/workflowReconciler.test.ts@@ -8,6 +8,7 @@*/import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'import type * as k8s from '@kubernetes/client-node'+import type { CodexExecutionProjection } from '@clerum/codex-catalog-projection'import { mintRecipeHostGfsToken } from '../../../src/gfsBinding'import {resolveStatefulSetHeadlessServiceName,@@ -30,6 +31,7 @@ import {buildMcpHostServiceName,buildSnippetRunnerServiceName,} from '../../../src/workflow/resourceNames'+import { deriveWorkflowRuntimePlan } from '../../../src/workflow/runtimePlan'import {type WorkflowRecipeSpec,WorkflowReconciler,@@ -428,11 +430,35 @@ function makeApiserverNetworkingApi() {return {}}),- listNamespacedNetworkPolicy: vi.fn().mockResolvedValue({ items: [] }),+ listNamespacedNetworkPolicy: vi.fn(+ async ({ namespace, labelSelector }: { namespace: string; labelSelector?: string }) => ({+ items: [...live.values()]+ .filter(+ policy =>+ (policy.metadata?.namespace ?? namespace) === namespace &&+ policyMatchesLabelSelector(policy.metadata?.labels, labelSelector)+ )+ .map(policy => structuredClone(policy)),+ })+ ),}return { api, live, key }}+function policyMatchesLabelSelector(+ labels: Record<string, string> | undefined,+ labelSelector: string | undefined+): boolean {+ if (!labelSelector) return true+ const present = labels ?? {}+ return labelSelector.split(',').every(term => {+ const trimmed = term.trim()+ const eq = trimmed.indexOf('=')+ if (eq < 0) return false+ return present[trimmed.slice(0, eq)] === trimmed.slice(eq + 1)+ })+}+function createdPolicyNames(networkingApi: ReturnType<typeof makeNetworkingApi>): string[] {return networkingApi.createNamespacedNetworkPolicy.mock.calls.map(([arg]) => (arg as { body?: k8s.V1NetworkPolicy }).body?.metadata?.name ?? ''@@ -2985,6 +3011,295 @@ describe('WorkflowReconciler — reconcile loop', () => {([arg]) => arg.body?.metadata?.name === 'test-wf-mcp-servers-egress-internet')).toBe(false)++ networkingApi.deleteNamespacedNetworkPolicy.mockClear()+ networkingApi.listNamespacedNetworkPolicy.mockClear()+ const second = await reconciler.reconcile(+ 'test-wf',+ 'uid-123',+ 'sandbox-recipes',+ makeSpec({ mcpServers: ['redis-mcp'] })+ )+ expect(second.workflowPhase).not.toBe('failed')+ expect(networkingApi.listNamespacedNetworkPolicy).toHaveBeenCalled()+ expect(networkingApi.deleteNamespacedNetworkPolicy).not.toHaveBeenCalledWith({+ name: 'test-wf-mcp-servers-egress-internet',+ namespace: 'mcp-server',+ })+ })++ describe('B2 desired-set NetworkPolicy prune', () => {+ const RECIPE = 'test-wf'+ const SANDBOX = 'sandbox-recipes'+ const TWO_TERM_SELECTOR = `clerum.io/recipe=${RECIPE},clerum.io/managed-by=wrc`+ const CODEX_PROXY = `${RECIPE}-mcp-host-to-codex-proxy`+ const GROK_PROXY = `${RECIPE}-mcp-host-to-grok-proxy`+ const COORD_TO_WRC = `${RECIPE}-coord-to-wrc`+ const WRC_LABELS = {+ 'clerum.io/recipe': RECIPE,+ 'clerum.io/managed-by': 'wrc',+ }++ function ineligibleProjection(+ overrides: Partial<CodexExecutionProjection> = {}+ ): CodexExecutionProjection {+ return {+ targets: [],+ eligibleTargets: [],+ derivedScopes: [],+ requiresCodexProxyEgress: false,+ driftHashInput: '',+ catalogContentHash: null,+ catalogRevision: null,+ connectionRevision: null,+ eligibility: 'ineligible',+ reason: 'static_only',+ ...overrides,+ }+ }++ function seedLivePolicy(+ apiserver: ReturnType<typeof makeApiserverNetworkingApi>,+ name: string,+ labels: Record<string, string>,+ namespace = SANDBOX+ ): void {+ apiserver.live.set(apiserver.key(namespace, name), {+ apiVersion: 'networking.k8s.io/v1',+ kind: 'NetworkPolicy',+ metadata: { name, namespace, labels },+ spec: { podSelector: {}, policyTypes: ['Egress'] },+ })+ }++ function listCalls(+ api: ReturnType<typeof makeApiserverNetworkingApi>['api']+ ): Array<{ namespace: string; labelSelector?: string }> {+ return api.listNamespacedNetworkPolicy.mock.calls.map(([arg]) => arg)+ }++ function sandboxDeletes(+ api: ReturnType<typeof makeApiserverNetworkingApi>['api']+ ): Array<{ name: string; namespace: string }> {+ return api.deleteNamespacedNetworkPolicy.mock.calls+ .map(([arg]) => arg)+ .filter(arg => arg.namespace === SANDBOX)+ }++ async function applyPolicies(+ reconciler: WorkflowReconciler,+ opts: {+ codex?: Partial<CodexExecutionProjection>+ grok?: Partial<CodexExecutionProjection> & { requiresGrokProxyEgress?: boolean }+ } = {}+ ): Promise<void> {+ const spec = makeSpec()+ const runtime = deriveWorkflowRuntimePlan(spec, {+ recipeName: RECIPE,+ runtimeScopeRecipeName: RECIPE,+ workflowRunId: 'run-1',+ })+ const codex = ineligibleProjection({+ requiresCodexProxyEgress: false,+ ...opts.codex,+ })+ const grok = {+ ...ineligibleProjection(),+ requiresCodexProxyEgress: false,+ requiresGrokProxyEgress: false,+ ...opts.grok,+ }+ await (+ reconciler as unknown as {+ applyWorkflowNetworkPolicies: (+ recipeName: string,+ recipeUid: string,+ spec: WorkflowRecipeSpec,+ runtime: ReturnType<typeof deriveWorkflowRuntimePlan>,+ awaitsTriggeredRun: boolean,+ codexProjection: CodexExecutionProjection,+ eagerSdkMcpHost: boolean,+ grokProjection: typeof grok+ ) => Promise<unknown>+ }+ ).applyWorkflowNetworkPolicies(RECIPE, 'uid-123', spec, runtime, false, codex, false, grok)+ }++ function expectListWitness(+ api: ReturnType<typeof makeApiserverNetworkingApi>['api'],+ times = 1+ ): void {+ expect(listCalls(api)).toEqual(+ Array.from({ length: times }, () => ({+ namespace: SANDBOX,+ labelSelector: TWO_TERM_SELECTOR,+ }))+ )+ }++ it('deletes exactly one listed obsolete policy that is not in the desired set', async () => {+ const apiserver = makeApiserverNetworkingApi()+ seedLivePolicy(apiserver, `${RECIPE}-obsolete-wrc`, WRC_LABELS)+ const reconciler = new WorkflowReconciler(makeDeps({ networkingApi: apiserver.api as never }))++ await applyPolicies(reconciler)++ expectListWitness(apiserver.api)+ expect(+ sandboxDeletes(apiserver.api).filter(d => d.name === `${RECIPE}-obsolete-wrc`)+ ).toEqual([{ name: `${RECIPE}-obsolete-wrc`, namespace: SANDBOX }])+ })++ it('does not delete in the sandbox namespace when the LIST is empty', async () => {+ const apiserver = makeApiserverNetworkingApi()+ const reconciler = new WorkflowReconciler(makeDeps({ networkingApi: apiserver.api as never }))++ await applyPolicies(reconciler)++ expectListWitness(apiserver.api)+ expect(sandboxDeletes(apiserver.api)).toEqual([])+ })++ it('does not delete a listed policy that is in the desired set', async () => {+ const apiserver = makeApiserverNetworkingApi()+ seedLivePolicy(apiserver, COORD_TO_WRC, WRC_LABELS)+ const reconciler = new WorkflowReconciler(makeDeps({ networkingApi: apiserver.api as never }))++ await applyPolicies(reconciler)++ expect(sandboxDeletes(apiserver.api).filter(d => d.name === COORD_TO_WRC)).toEqual([])+ expect(apiserver.live.has(apiserver.key(SANDBOX, COORD_TO_WRC))).toBe(true)+ })++ it('does not prune a recipes-lane policy; LIST uses the two-term selector once', async () => {+ const apiserver = makeApiserverNetworkingApi()+ seedLivePolicy(apiserver, `${RECIPE}-oauth-broker-egress`, {+ 'clerum.io/recipe': RECIPE,+ 'clerum.io/managed-by': 'workflow-recipes',+ })+ const reconciler = new WorkflowReconciler(makeDeps({ networkingApi: apiserver.api as never }))++ await applyPolicies(reconciler)++ expectListWitness(apiserver.api)+ expect(sandboxDeletes(apiserver.api).map(d => d.name)).not.toContain(+ `${RECIPE}-oauth-broker-egress`+ )+ expect(apiserver.live.has(apiserver.key(SANDBOX, `${RECIPE}-oauth-broker-egress`))).toBe(true)+ })++ it('keeps an unlisted-as-desired proxy while eligibility is uncertain, and deletes the twin', async () => {+ const uncertain = makeApiserverNetworkingApi()+ seedLivePolicy(uncertain, CODEX_PROXY, WRC_LABELS)+ const uncertainReconciler = new WorkflowReconciler(+ makeDeps({ networkingApi: uncertain.api as never })+ )+ await applyPolicies(uncertainReconciler, {+ codex: { eligibility: 'uncertain', requiresCodexProxyEgress: false, reason: 'forbidden' },+ })+ expectListWitness(uncertain.api)+ expect(sandboxDeletes(uncertain.api).map(d => d.name)).not.toContain(CODEX_PROXY)+ expect(uncertain.live.has(uncertain.key(SANDBOX, CODEX_PROXY))).toBe(true)++ const decided = makeApiserverNetworkingApi()+ seedLivePolicy(decided, CODEX_PROXY, WRC_LABELS)+ const decidedReconciler = new WorkflowReconciler(+ makeDeps({ networkingApi: decided.api as never })+ )+ await applyPolicies(decidedReconciler, {+ codex: { eligibility: 'ineligible', requiresCodexProxyEgress: false },+ })+ expectListWitness(decided.api)+ expect(sandboxDeletes(decided.api).map(d => d.name)).toContain(CODEX_PROXY)+ })++ it('keeps a desired live Codex proxy and deletes an undesired Grok proxy only on the first pass', async () => {+ const apiserver = makeApiserverNetworkingApi()+ seedLivePolicy(apiserver, CODEX_PROXY, WRC_LABELS)+ seedLivePolicy(apiserver, GROK_PROXY, WRC_LABELS)+ const reconciler = new WorkflowReconciler(makeDeps({ networkingApi: apiserver.api as never }))++ await applyPolicies(reconciler, {+ codex: { eligibility: 'eligible', requiresCodexProxyEgress: true, reason: 'granted' },+ })+ expectListWitness(apiserver.api)+ expect(sandboxDeletes(apiserver.api).map(d => d.name)).toEqual([GROK_PROXY])+ expect(apiserver.live.has(apiserver.key(SANDBOX, CODEX_PROXY))).toBe(true)+ expect(apiserver.live.has(apiserver.key(SANDBOX, GROK_PROXY))).toBe(false)++ apiserver.api.listNamespacedNetworkPolicy.mockClear()+ apiserver.api.deleteNamespacedNetworkPolicy.mockClear()+ await applyPolicies(reconciler, {+ codex: { eligibility: 'eligible', requiresCodexProxyEgress: true, reason: 'granted' },+ })+ expectListWitness(apiserver.api)+ expect(sandboxDeletes(apiserver.api)).toEqual([])+ })+ })++ describe('B3(d) skip coordinator DELETE after same-pass 404', () => {+ const COORDINATOR = 'test-wf-coordinator'+ const MCP_HOST = 'test-wf-mcp-host'+ const SANDBOX = 'sandbox-recipes'++ function podReads(coreApi: ReturnType<typeof makeCoreApi>, name: string): number {+ return coreApi.readNamespacedPod.mock.calls.filter(([arg]) => arg.name === name).length+ }++ function podDeletes(coreApi: ReturnType<typeof makeCoreApi>, name: string): number {+ return coreApi.deleteNamespacedPod.mock.calls.filter(([arg]) => arg.name === name).length+ }++ it('does not delete the coordinator when same-pass getPodPhase is undefined', async () => {+ const coreApi = makeCoreApi(false)+ const reconciler = new WorkflowReconciler(makeDeps({ coreApi: coreApi as never }))++ await reconciler.reconcile('test-wf', 'uid-123', SANDBOX, makeSpec())++ expect(podReads(coreApi, COORDINATOR)).toBeGreaterThan(0)+ expect(podDeletes(coreApi, COORDINATOR)).toBe(0)+ })++ it('deletes the coordinator when same-pass getPodPhase is Running', async () => {+ const coreApi = makeCoreApi(true)+ const reconciler = new WorkflowReconciler(makeDeps({ coreApi: coreApi as never }))++ await reconciler.reconcile('test-wf', 'uid-123', SANDBOX, makeSpec())++ expect(podReads(coreApi, COORDINATOR)).toBeGreaterThan(0)+ expect(podDeletes(coreApi, COORDINATOR)).toBe(1)+ expect(coreApi.deleteNamespacedPod).toHaveBeenCalledWith({+ name: COORDINATOR,+ namespace: SANDBOX,+ })+ })++ it('neither reads nor deletes mcp-host when needsMcpHost is false', async () => {+ const coreApi = makeCoreApi(false)+ const reconciler = new WorkflowReconciler(+ makeDeps({+ coreApi: coreApi as never,+ config: {+ ...makeConfig(),+ enableSnippetRuntime: true,+ snippetRunnerImage: 'clerum/workflow-snippet-runner:test',+ } as never,+ })+ )++ await reconciler.reconcile(+ 'test-wf',+ 'uid-123',+ SANDBOX,+ makeSpec({+ agent: undefined,+ steps: [{ id: 'snippet', run: snippetRun() }],+ })+ )++ expect(podReads(coreApi, MCP_HOST)).toBe(0)+ expect(podDeletes(coreApi, MCP_HOST)).toBe(0)+ })})it('retries NetworkPolicy replace once when resourceVersion is stale', async () => {@@ -4403,9 +4718,9 @@ describe('WorkflowReconciler — reconcile loop', () => {const first = await reconciler.reconcile('test-wf', 'uid-123', 'sandbox-recipes', spec())expect(first.workflowPhase).not.toBe('failed')expect([...apiserver.live.keys()].sort()).toEqual(RUN_LANE_POLICY_NAMES)- // reconcile() also prunes the codex, grok and legacy policies, so a- // zero delete count below means the retry skipped them, not that no- // prune exists for this spec.+ // reconcile() LISTs leftover run-lane policies and still DELETEs the+ // legacy mcp-servers-egress-internet once (B3(c)). A zero delete count+ // below means the retry skipped those paths, not that no prune exists.expect(apiserver.api.deleteNamespacedNetworkPolicy).toHaveBeenCalled()apiserver.api.createNamespacedNetworkPolicy.mockClear()apiserver.api.readNamespacedNetworkPolicy.mockClear()