Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript

fix(workflow-recipes): stop WRC write-storm DELETE 404s on leftover policies

Alfredo Lopez219930f8393b
diff --git a/workflow-recipes/package-lock.json b/workflow-recipes/package-lock.json
index 2b9c3013c..c6c0eb7d4 100644
--- a/workflow-recipes/package-lock.json
+++ b/workflow-recipes/package-lock.json
@@ -1,12 +1,12 @@
{
"name": "workload-recipes-controller",
- "version": "0.1.405",
+ "version": "0.1.406",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "workload-recipes-controller",
- "version": "0.1.405",
+ "version": "0.1.406",
"license": "SEE LICENSE IN LICENSE",
"dependencies": {
"@aws-sdk/client-s3": "^3.1009.0",
diff --git a/workflow-recipes/package.json b/workflow-recipes/package.json
index c43d9d0b3..1086afb5c 100644
--- a/workflow-recipes/package.json
+++ b/workflow-recipes/package.json
@@ -1,6 +1,6 @@
{
"name": "workload-recipes-controller",
- "version": "0.1.405",
+ "version": "0.1.406",
"description": "Workload Recipes Controller (WRC) - WorkloadRecipe lifecycle management and MCP server interface",
"main": "dist/main.js",
"engines": {
diff --git a/workflow-recipes/src/k8sClient.ts b/workflow-recipes/src/k8sClient.ts
index 3239a9ff6..7a3a6afe9 100644
--- a/workflow-recipes/src/k8sClient.ts
+++ b/workflow-recipes/src/k8sClient.ts
@@ -907,8 +907,14 @@ export class WorkflowRecipeWatcher implements WorkflowRecipeProvider {
]),
]
for (const namespace of watchedNamespaces) {
- const handler = new SecretWatcher(this.secretReverseIndex, name =>
- this.triggerSecretDrivenReconcile(name)
+ const handler = new SecretWatcher(
+ this.secretReverseIndex,
+ name => this.triggerSecretDrivenReconcile(name),
+ 10_000,
+ recipeName => {
+ this.reconciler.invalidateOAuthBrokerDeleteLedger(recipeName)
+ this.triggerSecretDrivenReconcile(recipeName)
+ }
)
const loop = new K8sSecretWatchLoop(this.kc, namespace, handler)
this.secretWatchLoops.push(loop)
diff --git a/workflow-recipes/src/reconciler/oauthBrokerDeleteLedger.ts b/workflow-recipes/src/reconciler/oauthBrokerDeleteLedger.ts
new file mode 100644
index 000000000..76f3a5d4a
--- /dev/null
+++ b/workflow-recipes/src/reconciler/oauthBrokerDeleteLedger.ts
@@ -0,0 +1,48 @@
+/**
+ * In-process B3(a)(b) ledger for oauth-broker Secret / NetworkPolicy deletes.
+ * Transition is (recipe, metadata.generation). The NetworkPolicy side adds a
+ * 1h TTL so a later pass can reap a leftover without a watch. Indexing only
+ * by recipe name is the E.6 vacuity mutation: a generation bump must miss.
+ */
+
+export const OAUTH_BROKER_NP_TTL_MS = 60 * 60 * 1000
+
+function ledgerKey(recipeName: string, generation: number | undefined): string {
+ return `${recipeName}:${generation ?? 0}`
+}
+
+export class OAuthBrokerDeleteLedger {
+ private readonly secrets = new Set<string>()
+ private readonly policies = new Map<string, number>()
+
+ shouldDeleteSecret(recipeName: string, generation: number | undefined): boolean {
+ return !this.secrets.has(ledgerKey(recipeName, generation))
+ }
+
+ recordSecretDelete(recipeName: string, generation: number | undefined): void {
+ this.secrets.add(ledgerKey(recipeName, generation))
+ }
+
+ shouldDeletePolicy(
+ recipeName: string,
+ generation: number | undefined,
+ nowMs = Date.now()
+ ): boolean {
+ const expiresAt = this.policies.get(ledgerKey(recipeName, generation))
+ return expiresAt === undefined || nowMs >= expiresAt
+ }
+
+ recordPolicyDelete(recipeName: string, generation: number | undefined, nowMs = Date.now()): void {
+ this.policies.set(ledgerKey(recipeName, generation), nowMs + OAUTH_BROKER_NP_TTL_MS)
+ }
+
+ invalidate(recipeName: string): void {
+ const prefix = `${recipeName}:`
+ for (const key of [...this.secrets]) {
+ if (key.startsWith(prefix)) this.secrets.delete(key)
+ }
+ for (const key of [...this.policies.keys()]) {
+ if (key.startsWith(prefix)) this.policies.delete(key)
+ }
+ }
+}
diff --git a/workflow-recipes/src/reconciler/reconciler.test.ts b/workflow-recipes/src/reconciler/reconciler.test.ts
index f5853340b..1f18b266a 100644
--- a/workflow-recipes/src/reconciler/reconciler.test.ts
+++ b/workflow-recipes/src/reconciler/reconciler.test.ts
@@ -15368,6 +15368,121 @@ describe('WorkflowRecipeReconciler', () => {
})
})
+ describe('B3(a)(b) oauth-broker delete-on-transition ledger', () => {
+ const SECRET_NAME = 'wf-test-recipe-oauth-broker-token'
+ const POLICY_NAME = 'wf-test-recipe-oauth-broker-egress'
+
+ function reapRecipe(generation: number): WorkflowRecipeCRD {
+ return makeRecipe({
+ metadata: {
+ name: 'test-recipe',
+ namespace: 'sandbox-recipes',
+ uid: 'uid-123',
+ generation,
+ },
+ })
+ }
+
+ function secretDeletes(): number {
+ return mockCoreApi.deleteNamespacedSecret.mock.calls.filter(
+ ([arg]) => arg.name === SECRET_NAME
+ ).length
+ }
+
+ function policyDeletes(): number {
+ return mockNetworkingApi.deleteNamespacedNetworkPolicy.mock.calls.filter(
+ ([arg]) => arg.name === POLICY_NAME
+ ).length
+ }
+
+ async function reapPolicy(recipe: WorkflowRecipeCRD): Promise<void> {
+ await (
+ reconciler as unknown as {
+ reconcileOAuthBrokerEgressPolicy: (next: WorkflowRecipeCRD) => Promise<void>
+ }
+ ).reconcileOAuthBrokerEgressPolicy(recipe)
+ }
+
+ it('deletes the Secret once for the same generation and logs the skip', async () => {
+ const recipe = reapRecipe(4)
+ const infoSpy = captureLogger('info')
+
+ await reconciler.ensureOAuthBrokerTokenSecret(recipe)
+ await reconciler.ensureOAuthBrokerTokenSecret(recipe)
+
+ expect(secretDeletes()).toBe(1)
+ expect(infoSpy).toHaveBeenCalledWith(
+ expect.stringContaining('Skipping oauth-broker-token delete'),
+ expect.objectContaining({ recipe: 'test-recipe', generation: 4 })
+ )
+ infoSpy.mockRestore()
+ })
+
+ it('deletes the Secret again when metadata.generation changes', async () => {
+ await reconciler.ensureOAuthBrokerTokenSecret(reapRecipe(4))
+ await reconciler.ensureOAuthBrokerTokenSecret(reapRecipe(4))
+ expect(secretDeletes()).toBe(1)
+
+ await reconciler.ensureOAuthBrokerTokenSecret(reapRecipe(5))
+ expect(secretDeletes()).toBe(2)
+ })
+
+ it('deletes the Secret again on a new process (fresh reconciler)', async () => {
+ await reconciler.ensureOAuthBrokerTokenSecret(reapRecipe(4))
+ expect(secretDeletes()).toBe(1)
+
+ const next = new WorkflowRecipeReconciler(new k8s.KubeConfig(), undefined, {
+ verifyWorkflowRunProvenance: mockVerifyWorkflowRunProvenance,
+ })
+ await next.ensureOAuthBrokerTokenSecret(reapRecipe(4))
+ expect(secretDeletes()).toBe(2)
+ })
+
+ it('deletes the Secret again after ADDED invalidation', async () => {
+ const recipe = reapRecipe(4)
+ await reconciler.ensureOAuthBrokerTokenSecret(recipe)
+ await reconciler.ensureOAuthBrokerTokenSecret(recipe)
+ expect(secretDeletes()).toBe(1)
+
+ reconciler.invalidateOAuthBrokerDeleteLedger('test-recipe')
+ await reconciler.ensureOAuthBrokerTokenSecret(recipe)
+ expect(secretDeletes()).toBe(2)
+ })
+
+ it('deletes the Secret again after reconcileDelete invalidates the ledger', async () => {
+ const recipe = reapRecipe(4)
+ await reconciler.ensureOAuthBrokerTokenSecret(recipe)
+ await reconciler.ensureOAuthBrokerTokenSecret(recipe)
+ expect(secretDeletes()).toBe(1)
+
+ await reconciler.reconcileDelete(recipe)
+ mockCoreApi.deleteNamespacedSecret.mockClear()
+ await reconciler.ensureOAuthBrokerTokenSecret(recipe)
+ expect(secretDeletes()).toBe(1)
+ })
+
+ it('deletes the NetworkPolicy once for the same generation, then again after the 1h TTL', async () => {
+ vi.useFakeTimers()
+ vi.setSystemTime(new Date('2026-09-25T00:00:00.000Z'))
+ const recipe = reapRecipe(4)
+ const infoSpy = captureLogger('info')
+
+ await reapPolicy(recipe)
+ await reapPolicy(recipe)
+ expect(policyDeletes()).toBe(1)
+ expect(infoSpy).toHaveBeenCalledWith(
+ expect.stringContaining('Skipping oauth-broker-egress delete'),
+ expect.objectContaining({ recipe: 'test-recipe', generation: 4 })
+ )
+
+ vi.setSystemTime(new Date('2026-09-25T01:00:00.000Z'))
+ await reapPolicy(recipe)
+ expect(policyDeletes()).toBe(2)
+ infoSpy.mockRestore()
+ vi.useRealTimers()
+ })
+ })
+
// ─── Broker-token issuance failure is non-fatal ─────────────────────────
//
// A transient control-api blip during reconcile (e.g. it is mid-restart)
diff --git a/workflow-recipes/src/reconciler/secretWatcher.test.ts b/workflow-recipes/src/reconciler/secretWatcher.test.ts
index 7f7e07997..2a9ff5425 100644
--- a/workflow-recipes/src/reconciler/secretWatcher.test.ts
+++ b/workflow-recipes/src/reconciler/secretWatcher.test.ts
@@ -215,4 +215,35 @@ describe('SecretWatcher', () => {
vi.advanceTimersByTime(DEBOUNCE_MS)
expect(enqueued.sort()).toEqual(['recipe-a', 'recipe-b'])
})
+
+ it('invalidates the oauth-broker-token ledger on ADDED before key-set dedup', () => {
+ const reverseIndex = new SecretReverseIndex()
+ const enqueued: string[] = []
+ const invalidated: string[] = []
+ const watcher = new SecretWatcher(
+ reverseIndex,
+ name => {
+ enqueued.push(name)
+ },
+ DEBOUNCE_MS,
+ recipeName => {
+ invalidated.push(recipeName)
+ }
+ )
+ const brokerSecret = {
+ metadata: {
+ name: 'wf-test-recipe-oauth-broker-token',
+ labels: {
+ 'clerum.io/component': 'oauth-broker-token',
+ 'clerum.io/recipe': 'test-recipe',
+ },
+ },
+ data: { 'broker-token': 'eA==' },
+ }
+
+ watcher.handleEvent('ADDED', brokerSecret)
+ watcher.handleEvent('ADDED', brokerSecret)
+ expect(invalidated).toEqual(['test-recipe', 'test-recipe'])
+ watcher.stop()
+ })
})
diff --git a/workflow-recipes/src/reconciler/secretWatcher.ts b/workflow-recipes/src/reconciler/secretWatcher.ts
index c1725b6ae..deb026968 100644
--- a/workflow-recipes/src/reconciler/secretWatcher.ts
+++ b/workflow-recipes/src/reconciler/secretWatcher.ts
@@ -41,7 +41,8 @@ export class SecretWatcher {
constructor(
private readonly reverseIndex: SecretReverseIndex,
private readonly enqueueReconcile: (recipeName: string) => void,
- private readonly debounceMs: number = 10_000
+ private readonly debounceMs: number = 10_000,
+ private readonly onOAuthBrokerTokenAdded?: (recipeName: string) => void
) {}
/**
@@ -58,6 +59,16 @@ export class SecretWatcher {
const name = secret.metadata?.name
if (!name) return
+ // B3(a): ADDED of the broker token must invalidate the ledger even when
+ // the key-set matches a previous observation (dedup would otherwise drop it).
+ if (type === 'ADDED') {
+ const labels = secret.metadata?.labels ?? {}
+ if (labels['clerum.io/component'] === 'oauth-broker-token') {
+ const recipeName = labels['clerum.io/recipe']
+ if (recipeName) this.onOAuthBrokerTokenAdded?.(recipeName)
+ }
+ }
+
const newKeys =
type === 'DELETED'
? new Set<string>()
diff --git a/workflow-recipes/src/reconciler/workflowRecipeReconciler.ts b/workflow-recipes/src/reconciler/workflowRecipeReconciler.ts
index c73653edf..ff367c206 100644
--- a/workflow-recipes/src/reconciler/workflowRecipeReconciler.ts
+++ b/workflow-recipes/src/reconciler/workflowRecipeReconciler.ts
@@ -127,6 +127,7 @@ import {
decideNetworkPolicyConvergence,
networkPolicyMetadataMatchesDesired,
} from './networkPolicyConvergence'
+import { OAuthBrokerDeleteLedger } from './oauthBrokerDeleteLedger'
import { issueOAuthBrokerToken } from './oauthBrokerTokenIssuerClient'
import {
PLUGIN_WORKLOAD_SDK_CONDITION_TYPE,
@@ -941,10 +942,15 @@ export class WorkflowRecipeReconciler {
// F2 gate) and the safe direction; under-refreshing would reopen #299.
private externalEgressMinObservedTtlMs = Infinity
private secretReverseIndex: SecretReverseIndex | null
+ private readonly oauthBrokerDeleteLedger = new OAuthBrokerDeleteLedger()
private verifyWorkflowRunProvenance: NonNullable<
WorkflowRecipeReconcilerDeps['verifyWorkflowRunProvenance']
>
+ invalidateOAuthBrokerDeleteLedger(recipeName: string): void {
+ this.oauthBrokerDeleteLedger.invalidate(recipeName)
+ }
+
constructor(kc: k8s.KubeConfig, config?: OperatorConfig, deps?: WorkflowRecipeReconcilerDeps) {
this.appsApi = kc.makeApiClient(k8s.AppsV1Api)
this.batchApi = kc.makeApiClient(k8s.BatchV1Api)
@@ -3922,6 +3928,7 @@ export class WorkflowRecipeReconciler {
createLogger('wrc', recipe.metadata.name).info('Deleting recipe resources', { name })
this.secretReverseIndex?.delete(name)
+ this.oauthBrokerDeleteLedger.invalidate(name)
// ─── Workflow Delete (Stage 1) ────────────────────────────────────
const isWorkflow = recipe.spec.steps !== undefined && recipe.spec.steps.length > 0
@@ -7499,10 +7506,19 @@ export class WorkflowRecipeReconciler {
const secretName = rb.oauthBrokerTokenSecretName(recipeName)
if (!rb.recipeHasBackgroundAccessClient(recipe)) {
+ const generation = recipe.metadata.generation
+ if (!this.oauthBrokerDeleteLedger.shouldDeleteSecret(recipeName, generation)) {
+ createLogger('wrc', recipeName).info(
+ 'Skipping oauth-broker-token delete; generation already seen',
+ { recipe: recipeName, generation }
+ )
+ return
+ }
await this.safeDelete(
() => this.coreApi.deleteNamespacedSecret({ name: secretName, namespace: ns }),
`Secret "${secretName}" in ${ns}`
)
+ this.oauthBrokerDeleteLedger.recordSecretDelete(recipeName, generation)
return
}
@@ -7565,10 +7581,20 @@ export class WorkflowRecipeReconciler {
this.config.controlPlaneNamespace
)
if (!policy) {
+ const recipeName = recipe.metadata.name
+ const generation = recipe.metadata.generation
+ if (!this.oauthBrokerDeleteLedger.shouldDeletePolicy(recipeName, generation)) {
+ createLogger('wrc', recipeName).info(
+ 'Skipping oauth-broker-egress delete; generation already seen',
+ { recipe: recipeName, generation }
+ )
+ return
+ }
await this.safeDelete(
() => this.networkingApi.deleteNamespacedNetworkPolicy({ name: policyName, namespace: ns }),
`NetworkPolicy "${policyName}" in ${ns}`
)
+ this.oauthBrokerDeleteLedger.recordPolicyDelete(recipeName, generation)
return
}
await this.applyNetworkPolicy(policy, ns, { family: 'oauth-broker-egress' })
diff --git a/workflow-recipes/src/workflow/workflowReconciler.codexScopeProvenance.test.ts b/workflow-recipes/src/workflow/workflowReconciler.codexScopeProvenance.test.ts
index 5cf95ac09..8083ea7f4 100644
--- a/workflow-recipes/src/workflow/workflowReconciler.codexScopeProvenance.test.ts
+++ b/workflow-recipes/src/workflow/workflowReconciler.codexScopeProvenance.test.ts
@@ -381,6 +381,22 @@ describe('WorkflowReconciler Codex scope provenance', () => {
})
runtimeTokenIssuerMocks.issueMcpHostRuntimeTokens.mockClear()
crashRecoveryMocks.deletePodIfExists.mockClear()
+ // B2: withdraw is LIST membership of the leftover proxy, not a named DELETE
+ // of an absent object. Seed the leftover created on the first pass.
+ networkingApi.listNamespacedNetworkPolicy.mockResolvedValue({
+ items: [
+ {
+ metadata: {
+ name: CODEX_PROXY_POLICY,
+ namespace: sandboxNamespace,
+ labels: {
+ 'clerum.io/recipe': 'codex-recipe',
+ 'clerum.io/managed-by': 'wrc',
+ },
+ },
+ },
+ ],
+ })
await reconcileRecipe(
reconciler,
@@ -400,6 +416,12 @@ describe('WorkflowReconciler Codex scope provenance', () => {
expect(
remintPatch.body?.metadata?.annotations?.['clerum.io/mcp-host-runtime-token-generation']
).toBe('1')
+ expect(networkingApi.listNamespacedNetworkPolicy).toHaveBeenCalledWith(
+ expect.objectContaining({
+ namespace: sandboxNamespace,
+ labelSelector: 'clerum.io/recipe=codex-recipe,clerum.io/managed-by=wrc',
+ })
+ )
expect(networkingApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledWith({
name: CODEX_PROXY_POLICY,
namespace: sandboxNamespace,
@@ -444,6 +466,21 @@ describe('WorkflowReconciler Codex scope provenance', () => {
})
runtimeTokenIssuerMocks.issueMcpHostRuntimeTokens.mockClear()
crashRecoveryMocks.deletePodIfExists.mockClear()
+ // B2: leftover proxy stays listed so withdraw prunes by membership, not by name.
+ networkingApi.listNamespacedNetworkPolicy.mockResolvedValue({
+ items: [
+ {
+ metadata: {
+ name: CODEX_PROXY_POLICY,
+ namespace: sandboxNamespace,
+ labels: {
+ 'clerum.io/recipe': 'codex-recipe',
+ 'clerum.io/managed-by': 'wrc',
+ },
+ },
+ },
+ ],
+ })
await reconcileRecipe(reconciler, makeCodexSpec())
expect(issueMcpHostRuntimeTokens).toHaveBeenCalled()
@@ -453,6 +490,12 @@ describe('WorkflowReconciler Codex scope provenance', () => {
'codex-recipe-mcp-host',
sandboxNamespace
)
+ expect(networkingApi.listNamespacedNetworkPolicy).toHaveBeenCalledWith(
+ expect.objectContaining({
+ namespace: sandboxNamespace,
+ labelSelector: 'clerum.io/recipe=codex-recipe,clerum.io/managed-by=wrc',
+ })
+ )
expect(networkingApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledWith({
name: CODEX_PROXY_POLICY,
namespace: sandboxNamespace,
diff --git a/workflow-recipes/src/workflow/workflowReconciler.grokWrcFlag.test.ts b/workflow-recipes/src/workflow/workflowReconciler.grokWrcFlag.test.ts
index 8e02c5524..c5535ed92 100644
--- a/workflow-recipes/src/workflow/workflowReconciler.grokWrcFlag.test.ts
+++ b/workflow-recipes/src/workflow/workflowReconciler.grokWrcFlag.test.ts
@@ -294,7 +294,15 @@ describe('WorkflowReconciler Grok WRC switch', () => {
expect(runtimeTokenIssuerMocks.issueMcpHostRuntimeTokens).toHaveBeenCalled()
expect(issuedScopes()).not.toContain('llm:grok:execute')
expect(createdPolicyNames(networkingApi)).not.toContain(GROK_PROXY_POLICY)
- expect(networkingApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledWith(
+ // B2: withheld proxy is not in the desired apply set; LIST membership is
+ // empty, so there is no leftover DELETE of the named Grok policy.
+ expect(networkingApi.listNamespacedNetworkPolicy).toHaveBeenCalledWith(
+ expect.objectContaining({
+ namespace: sandboxNamespace,
+ labelSelector: `clerum.io/recipe=${RECIPE},clerum.io/managed-by=wrc`,
+ })
+ )
+ expect(networkingApi.deleteNamespacedNetworkPolicy).not.toHaveBeenCalledWith(
expect.objectContaining({ name: GROK_PROXY_POLICY })
)
const env = mcpHostEnvNames(coreApi)
diff --git a/workflow-recipes/src/workflow/workflowReconciler.ts b/workflow-recipes/src/workflow/workflowReconciler.ts
index 0ca2a594c..e3f52de39 100644
--- a/workflow-recipes/src/workflow/workflowReconciler.ts
+++ b/workflow-recipes/src/workflow/workflowReconciler.ts
@@ -1120,6 +1120,8 @@ export class WorkflowReconciler {
snapshot: CodexCatalogSnapshot
} = { snapshot: { flagEnabled: false } }
private readonly codexContexts = new Map<string, CodexReconcileContext>()
+ /** B3(c): one process-lifetime DELETE of the leftover mcp-servers internet NP. */
+ private readonly prunedLegacyMcpServersInternetEgress = new Set<string>()
constructor(private readonly deps: WorkflowReconcilerDeps) {
this.pluginWorkloadSdkProvisioner = new PluginWorkloadSdkProvisioner({
@@ -1858,11 +1860,18 @@ export class WorkflowReconciler {
}
if (awaitsTriggeredRun) {
+ const samePassPhase: Partial<Record<WorkflowRuntimeComponent, string | undefined>> = {
+ 'workflow-coordinator': coordPhase,
+ 'workflow-mcp-host': mcpHostPhase,
+ }
for (const component of runtime.cleanup.deleteBeforeTriggeredRun) {
// Plugin Workload SDK: keep the mcp-host alive across the
// awaiting-trigger window — it hosts the always-on SDK server.
// Only the coordinator (and run-scoped pods) are torn down here.
if (eagerMcpHostForSdk && component === 'workflow-mcp-host') continue
+ // B3(d): same-pass getPodPhase undefined is the 404 we already paid.
+ // Do not DELETE again. Unread mcp-host is not in this list (needsMcpHost).
+ if (samePassPhase[component] === undefined) continue
await this.deleteRuntimeComponentIfExists(recipeName, component)
}
@@ -3175,12 +3184,14 @@ export class WorkflowReconciler {
}
private async pruneLegacyMcpServersInternetEgressPolicy(recipeName: string): Promise<void> {
+ if (this.prunedLegacyMcpServersInternetEgress.has(recipeName)) return
await this.safeDelete(() =>
this.deps.networkingApi.deleteNamespacedNetworkPolicy({
name: `${recipeName}-mcp-servers-egress-internet`,
namespace: this.deps.config.mcpServerNamespace,
})
)
+ this.prunedLegacyMcpServersInternetEgress.add(recipeName)
}
private needsRuntimeHttpEgressRefresh(spec: WorkflowRecipeSpec): boolean {
@@ -3354,27 +3365,43 @@ export class WorkflowReconciler {
)
const summary = await this.applyNetworkPolicyList(policies)
const policyNames = new Set(policies.map(policy => policy.metadata?.name))
+ await this.pruneUndesiredRunLaneNetworkPolicies(recipeName, policyNames, {
+ skipCodexProxy: codexProjection.eligibility === 'uncertain',
+ skipGrokProxy: grokProjection?.eligibility === 'uncertain',
+ })
+ await this.pruneLegacyMcpServersInternetEgressPolicy(recipeName)
+ return summary
+ }
+
+ /**
+ * Drop run-lane NetworkPolicies that exist and are no longer desired.
+ * Two-term selector only (`managed-by=wrc`): a single-term recipe selector
+ * would also return the recipes-lane policies in the same namespace.
+ * Uncertain Codex/Grok eligibility keeps those proxy names even when they
+ * are absent from this pass's desired set.
+ */
+ private async pruneUndesiredRunLaneNetworkPolicies(
+ recipeName: string,
+ desiredNames: Set<string | undefined>,
+ uncertain: { skipCodexProxy: boolean; skipGrokProxy: boolean }
+ ): Promise<void> {
+ const namespace = this.deps.config.sandboxNamespace
+ const labelSelector = `clerum.io/recipe=${recipeName},clerum.io/managed-by=wrc`
+ const list = await this.deps.networkingApi.listNamespacedNetworkPolicy({
+ namespace,
+ labelSelector,
+ })
const codexProxyPolicyName = `${recipeName}-mcp-host-to-codex-proxy`
const grokProxyPolicyName = `${recipeName}-mcp-host-to-grok-proxy`
-
- if (!policyNames.has(codexProxyPolicyName) && codexProjection.eligibility !== 'uncertain') {
- await this.safeDelete(() =>
- this.deps.networkingApi.deleteNamespacedNetworkPolicy({
- name: codexProxyPolicyName,
- namespace: this.deps.config.sandboxNamespace,
- })
- )
- }
- if (!policyNames.has(grokProxyPolicyName) && grokProjection?.eligibility !== 'uncertain') {
+ for (const policy of list.items ?? []) {
+ const name = policy.metadata?.name
+ if (!name || desiredNames.has(name)) continue
+ if (name === codexProxyPolicyName && uncertain.skipCodexProxy) continue
+ if (name === grokProxyPolicyName && uncertain.skipGrokProxy) continue
await this.safeDelete(() =>
- this.deps.networkingApi.deleteNamespacedNetworkPolicy({
- name: grokProxyPolicyName,
- namespace: this.deps.config.sandboxNamespace,
- })
+ this.deps.networkingApi.deleteNamespacedNetworkPolicy({ name, namespace })
)
}
- await this.pruneLegacyMcpServersInternetEgressPolicy(recipeName)
- return summary
}
/**
diff --git a/workflow-recipes/tests/unit/workflow/workflowReconciler.test.ts b/workflow-recipes/tests/unit/workflow/workflowReconciler.test.ts
index fcc659b32..0645c615b 100644
--- a/workflow-recipes/tests/unit/workflow/workflowReconciler.test.ts
+++ b/workflow-recipes/tests/unit/workflow/workflowReconciler.test.ts
@@ -8,6 +8,7 @@
*/
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
import type * as k8s from '@kubernetes/client-node'
+import type { CodexExecutionProjection } from '@clerum/codex-catalog-projection'
import { mintRecipeHostGfsToken } from '../../../src/gfsBinding'
import {
resolveStatefulSetHeadlessServiceName,
@@ -30,6 +31,7 @@ import {
buildMcpHostServiceName,
buildSnippetRunnerServiceName,
} from '../../../src/workflow/resourceNames'
+import { deriveWorkflowRuntimePlan } from '../../../src/workflow/runtimePlan'
import {
type WorkflowRecipeSpec,
WorkflowReconciler,
@@ -428,11 +430,35 @@ function makeApiserverNetworkingApi() {
return {}
}
),
- listNamespacedNetworkPolicy: vi.fn().mockResolvedValue({ items: [] }),
+ listNamespacedNetworkPolicy: vi.fn(
+ async ({ namespace, labelSelector }: { namespace: string; labelSelector?: string }) => ({
+ items: [...live.values()]
+ .filter(
+ policy =>
+ (policy.metadata?.namespace ?? namespace) === namespace &&
+ policyMatchesLabelSelector(policy.metadata?.labels, labelSelector)
+ )
+ .map(policy => structuredClone(policy)),
+ })
+ ),
}
return { api, live, key }
}
+function policyMatchesLabelSelector(
+ labels: Record<string, string> | undefined,
+ labelSelector: string | undefined
+): boolean {
+ if (!labelSelector) return true
+ const present = labels ?? {}
+ return labelSelector.split(',').every(term => {
+ const trimmed = term.trim()
+ const eq = trimmed.indexOf('=')
+ if (eq < 0) return false
+ return present[trimmed.slice(0, eq)] === trimmed.slice(eq + 1)
+ })
+}
+
function createdPolicyNames(networkingApi: ReturnType<typeof makeNetworkingApi>): string[] {
return networkingApi.createNamespacedNetworkPolicy.mock.calls.map(
([arg]) => (arg as { body?: k8s.V1NetworkPolicy }).body?.metadata?.name ?? ''
@@ -2985,6 +3011,295 @@ describe('WorkflowReconciler — reconcile loop', () => {
([arg]) => arg.body?.metadata?.name === 'test-wf-mcp-servers-egress-internet'
)
).toBe(false)
+
+ networkingApi.deleteNamespacedNetworkPolicy.mockClear()
+ networkingApi.listNamespacedNetworkPolicy.mockClear()
+ const second = await reconciler.reconcile(
+ 'test-wf',
+ 'uid-123',
+ 'sandbox-recipes',
+ makeSpec({ mcpServers: ['redis-mcp'] })
+ )
+ expect(second.workflowPhase).not.toBe('failed')
+ expect(networkingApi.listNamespacedNetworkPolicy).toHaveBeenCalled()
+ expect(networkingApi.deleteNamespacedNetworkPolicy).not.toHaveBeenCalledWith({
+ name: 'test-wf-mcp-servers-egress-internet',
+ namespace: 'mcp-server',
+ })
+ })
+
+ describe('B2 desired-set NetworkPolicy prune', () => {
+ const RECIPE = 'test-wf'
+ const SANDBOX = 'sandbox-recipes'
+ const TWO_TERM_SELECTOR = `clerum.io/recipe=${RECIPE},clerum.io/managed-by=wrc`
+ const CODEX_PROXY = `${RECIPE}-mcp-host-to-codex-proxy`
+ const GROK_PROXY = `${RECIPE}-mcp-host-to-grok-proxy`
+ const COORD_TO_WRC = `${RECIPE}-coord-to-wrc`
+ const WRC_LABELS = {
+ 'clerum.io/recipe': RECIPE,
+ 'clerum.io/managed-by': 'wrc',
+ }
+
+ function ineligibleProjection(
+ overrides: Partial<CodexExecutionProjection> = {}
+ ): CodexExecutionProjection {
+ return {
+ targets: [],
+ eligibleTargets: [],
+ derivedScopes: [],
+ requiresCodexProxyEgress: false,
+ driftHashInput: '',
+ catalogContentHash: null,
+ catalogRevision: null,
+ connectionRevision: null,
+ eligibility: 'ineligible',
+ reason: 'static_only',
+ ...overrides,
+ }
+ }
+
+ function seedLivePolicy(
+ apiserver: ReturnType<typeof makeApiserverNetworkingApi>,
+ name: string,
+ labels: Record<string, string>,
+ namespace = SANDBOX
+ ): void {
+ apiserver.live.set(apiserver.key(namespace, name), {
+ apiVersion: 'networking.k8s.io/v1',
+ kind: 'NetworkPolicy',
+ metadata: { name, namespace, labels },
+ spec: { podSelector: {}, policyTypes: ['Egress'] },
+ })
+ }
+
+ function listCalls(
+ api: ReturnType<typeof makeApiserverNetworkingApi>['api']
+ ): Array<{ namespace: string; labelSelector?: string }> {
+ return api.listNamespacedNetworkPolicy.mock.calls.map(([arg]) => arg)
+ }
+
+ function sandboxDeletes(
+ api: ReturnType<typeof makeApiserverNetworkingApi>['api']
+ ): Array<{ name: string; namespace: string }> {
+ return api.deleteNamespacedNetworkPolicy.mock.calls
+ .map(([arg]) => arg)
+ .filter(arg => arg.namespace === SANDBOX)
+ }
+
+ async function applyPolicies(
+ reconciler: WorkflowReconciler,
+ opts: {
+ codex?: Partial<CodexExecutionProjection>
+ grok?: Partial<CodexExecutionProjection> & { requiresGrokProxyEgress?: boolean }
+ } = {}
+ ): Promise<void> {
+ const spec = makeSpec()
+ const runtime = deriveWorkflowRuntimePlan(spec, {
+ recipeName: RECIPE,
+ runtimeScopeRecipeName: RECIPE,
+ workflowRunId: 'run-1',
+ })
+ const codex = ineligibleProjection({
+ requiresCodexProxyEgress: false,
+ ...opts.codex,
+ })
+ const grok = {
+ ...ineligibleProjection(),
+ requiresCodexProxyEgress: false,
+ requiresGrokProxyEgress: false,
+ ...opts.grok,
+ }
+ await (
+ reconciler as unknown as {
+ applyWorkflowNetworkPolicies: (
+ recipeName: string,
+ recipeUid: string,
+ spec: WorkflowRecipeSpec,
+ runtime: ReturnType<typeof deriveWorkflowRuntimePlan>,
+ awaitsTriggeredRun: boolean,
+ codexProjection: CodexExecutionProjection,
+ eagerSdkMcpHost: boolean,
+ grokProjection: typeof grok
+ ) => Promise<unknown>
+ }
+ ).applyWorkflowNetworkPolicies(RECIPE, 'uid-123', spec, runtime, false, codex, false, grok)
+ }
+
+ function expectListWitness(
+ api: ReturnType<typeof makeApiserverNetworkingApi>['api'],
+ times = 1
+ ): void {
+ expect(listCalls(api)).toEqual(
+ Array.from({ length: times }, () => ({
+ namespace: SANDBOX,
+ labelSelector: TWO_TERM_SELECTOR,
+ }))
+ )
+ }
+
+ it('deletes exactly one listed obsolete policy that is not in the desired set', async () => {
+ const apiserver = makeApiserverNetworkingApi()
+ seedLivePolicy(apiserver, `${RECIPE}-obsolete-wrc`, WRC_LABELS)
+ const reconciler = new WorkflowReconciler(makeDeps({ networkingApi: apiserver.api as never }))
+
+ await applyPolicies(reconciler)
+
+ expectListWitness(apiserver.api)
+ expect(
+ sandboxDeletes(apiserver.api).filter(d => d.name === `${RECIPE}-obsolete-wrc`)
+ ).toEqual([{ name: `${RECIPE}-obsolete-wrc`, namespace: SANDBOX }])
+ })
+
+ it('does not delete in the sandbox namespace when the LIST is empty', async () => {
+ const apiserver = makeApiserverNetworkingApi()
+ const reconciler = new WorkflowReconciler(makeDeps({ networkingApi: apiserver.api as never }))
+
+ await applyPolicies(reconciler)
+
+ expectListWitness(apiserver.api)
+ expect(sandboxDeletes(apiserver.api)).toEqual([])
+ })
+
+ it('does not delete a listed policy that is in the desired set', async () => {
+ const apiserver = makeApiserverNetworkingApi()
+ seedLivePolicy(apiserver, COORD_TO_WRC, WRC_LABELS)
+ const reconciler = new WorkflowReconciler(makeDeps({ networkingApi: apiserver.api as never }))
+
+ await applyPolicies(reconciler)
+
+ expect(sandboxDeletes(apiserver.api).filter(d => d.name === COORD_TO_WRC)).toEqual([])
+ expect(apiserver.live.has(apiserver.key(SANDBOX, COORD_TO_WRC))).toBe(true)
+ })
+
+ it('does not prune a recipes-lane policy; LIST uses the two-term selector once', async () => {
+ const apiserver = makeApiserverNetworkingApi()
+ seedLivePolicy(apiserver, `${RECIPE}-oauth-broker-egress`, {
+ 'clerum.io/recipe': RECIPE,
+ 'clerum.io/managed-by': 'workflow-recipes',
+ })
+ const reconciler = new WorkflowReconciler(makeDeps({ networkingApi: apiserver.api as never }))
+
+ await applyPolicies(reconciler)
+
+ expectListWitness(apiserver.api)
+ expect(sandboxDeletes(apiserver.api).map(d => d.name)).not.toContain(
+ `${RECIPE}-oauth-broker-egress`
+ )
+ expect(apiserver.live.has(apiserver.key(SANDBOX, `${RECIPE}-oauth-broker-egress`))).toBe(true)
+ })
+
+ it('keeps an unlisted-as-desired proxy while eligibility is uncertain, and deletes the twin', async () => {
+ const uncertain = makeApiserverNetworkingApi()
+ seedLivePolicy(uncertain, CODEX_PROXY, WRC_LABELS)
+ const uncertainReconciler = new WorkflowReconciler(
+ makeDeps({ networkingApi: uncertain.api as never })
+ )
+ await applyPolicies(uncertainReconciler, {
+ codex: { eligibility: 'uncertain', requiresCodexProxyEgress: false, reason: 'forbidden' },
+ })
+ expectListWitness(uncertain.api)
+ expect(sandboxDeletes(uncertain.api).map(d => d.name)).not.toContain(CODEX_PROXY)
+ expect(uncertain.live.has(uncertain.key(SANDBOX, CODEX_PROXY))).toBe(true)
+
+ const decided = makeApiserverNetworkingApi()
+ seedLivePolicy(decided, CODEX_PROXY, WRC_LABELS)
+ const decidedReconciler = new WorkflowReconciler(
+ makeDeps({ networkingApi: decided.api as never })
+ )
+ await applyPolicies(decidedReconciler, {
+ codex: { eligibility: 'ineligible', requiresCodexProxyEgress: false },
+ })
+ expectListWitness(decided.api)
+ expect(sandboxDeletes(decided.api).map(d => d.name)).toContain(CODEX_PROXY)
+ })
+
+ it('keeps a desired live Codex proxy and deletes an undesired Grok proxy only on the first pass', async () => {
+ const apiserver = makeApiserverNetworkingApi()
+ seedLivePolicy(apiserver, CODEX_PROXY, WRC_LABELS)
+ seedLivePolicy(apiserver, GROK_PROXY, WRC_LABELS)
+ const reconciler = new WorkflowReconciler(makeDeps({ networkingApi: apiserver.api as never }))
+
+ await applyPolicies(reconciler, {
+ codex: { eligibility: 'eligible', requiresCodexProxyEgress: true, reason: 'granted' },
+ })
+ expectListWitness(apiserver.api)
+ expect(sandboxDeletes(apiserver.api).map(d => d.name)).toEqual([GROK_PROXY])
+ expect(apiserver.live.has(apiserver.key(SANDBOX, CODEX_PROXY))).toBe(true)
+ expect(apiserver.live.has(apiserver.key(SANDBOX, GROK_PROXY))).toBe(false)
+
+ apiserver.api.listNamespacedNetworkPolicy.mockClear()
+ apiserver.api.deleteNamespacedNetworkPolicy.mockClear()
+ await applyPolicies(reconciler, {
+ codex: { eligibility: 'eligible', requiresCodexProxyEgress: true, reason: 'granted' },
+ })
+ expectListWitness(apiserver.api)
+ expect(sandboxDeletes(apiserver.api)).toEqual([])
+ })
+ })
+
+ describe('B3(d) skip coordinator DELETE after same-pass 404', () => {
+ const COORDINATOR = 'test-wf-coordinator'
+ const MCP_HOST = 'test-wf-mcp-host'
+ const SANDBOX = 'sandbox-recipes'
+
+ function podReads(coreApi: ReturnType<typeof makeCoreApi>, name: string): number {
+ return coreApi.readNamespacedPod.mock.calls.filter(([arg]) => arg.name === name).length
+ }
+
+ function podDeletes(coreApi: ReturnType<typeof makeCoreApi>, name: string): number {
+ return coreApi.deleteNamespacedPod.mock.calls.filter(([arg]) => arg.name === name).length
+ }
+
+ it('does not delete the coordinator when same-pass getPodPhase is undefined', async () => {
+ const coreApi = makeCoreApi(false)
+ const reconciler = new WorkflowReconciler(makeDeps({ coreApi: coreApi as never }))
+
+ await reconciler.reconcile('test-wf', 'uid-123', SANDBOX, makeSpec())
+
+ expect(podReads(coreApi, COORDINATOR)).toBeGreaterThan(0)
+ expect(podDeletes(coreApi, COORDINATOR)).toBe(0)
+ })
+
+ it('deletes the coordinator when same-pass getPodPhase is Running', async () => {
+ const coreApi = makeCoreApi(true)
+ const reconciler = new WorkflowReconciler(makeDeps({ coreApi: coreApi as never }))
+
+ await reconciler.reconcile('test-wf', 'uid-123', SANDBOX, makeSpec())
+
+ expect(podReads(coreApi, COORDINATOR)).toBeGreaterThan(0)
+ expect(podDeletes(coreApi, COORDINATOR)).toBe(1)
+ expect(coreApi.deleteNamespacedPod).toHaveBeenCalledWith({
+ name: COORDINATOR,
+ namespace: SANDBOX,
+ })
+ })
+
+ it('neither reads nor deletes mcp-host when needsMcpHost is false', async () => {
+ const coreApi = makeCoreApi(false)
+ const reconciler = new WorkflowReconciler(
+ makeDeps({
+ coreApi: coreApi as never,
+ config: {
+ ...makeConfig(),
+ enableSnippetRuntime: true,
+ snippetRunnerImage: 'clerum/workflow-snippet-runner:test',
+ } as never,
+ })
+ )
+
+ await reconciler.reconcile(
+ 'test-wf',
+ 'uid-123',
+ SANDBOX,
+ makeSpec({
+ agent: undefined,
+ steps: [{ id: 'snippet', run: snippetRun() }],
+ })
+ )
+
+ expect(podReads(coreApi, MCP_HOST)).toBe(0)
+ expect(podDeletes(coreApi, MCP_HOST)).toBe(0)
+ })
})
it('retries NetworkPolicy replace once when resourceVersion is stale', async () => {
@@ -4403,9 +4718,9 @@ describe('WorkflowReconciler — reconcile loop', () => {
const first = await reconciler.reconcile('test-wf', 'uid-123', 'sandbox-recipes', spec())
expect(first.workflowPhase).not.toBe('failed')
expect([...apiserver.live.keys()].sort()).toEqual(RUN_LANE_POLICY_NAMES)
- // reconcile() also prunes the codex, grok and legacy policies, so a
- // zero delete count below means the retry skipped them, not that no
- // prune exists for this spec.
+ // reconcile() LISTs leftover run-lane policies and still DELETEs the
+ // legacy mcp-servers-egress-internet once (B3(c)). A zero delete count
+ // below means the retry skipped those paths, not that no prune exists.
expect(apiserver.api.deleteNamespacedNetworkPolicy).toHaveBeenCalled()
apiserver.api.createNamespacedNetworkPolicy.mockClear()
apiserver.api.readNamespacedNetworkPolicy.mockClear()