Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript

fix(workflow-recipes): prune only run-lane catalog leftovers and record deletes after 200/404

Alfredo Lopez7d33f8a435b2
diff --git a/workflow-recipes/package-lock.json b/workflow-recipes/package-lock.json
index c6c0eb7d4..9f46506db 100644
--- a/workflow-recipes/package-lock.json
+++ b/workflow-recipes/package-lock.json
@@ -1,12 +1,12 @@
{
"name": "workload-recipes-controller",
- "version": "0.1.406",
+ "version": "0.1.407",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "workload-recipes-controller",
- "version": "0.1.406",
+ "version": "0.1.407",
"license": "SEE LICENSE IN LICENSE",
"dependencies": {
"@aws-sdk/client-s3": "^3.1009.0",
diff --git a/workflow-recipes/package.json b/workflow-recipes/package.json
index 1086afb5c..a661ef485 100644
--- a/workflow-recipes/package.json
+++ b/workflow-recipes/package.json
@@ -1,6 +1,6 @@
{
"name": "workload-recipes-controller",
- "version": "0.1.406",
+ "version": "0.1.407",
"description": "Workload Recipes Controller (WRC) - WorkloadRecipe lifecycle management and MCP server interface",
"main": "dist/main.js",
"engines": {
diff --git a/workflow-recipes/src/k8sClient.test.ts b/workflow-recipes/src/k8sClient.test.ts
index 3a5a2d96c..2028305f6 100644
--- a/workflow-recipes/src/k8sClient.test.ts
+++ b/workflow-recipes/src/k8sClient.test.ts
@@ -3250,3 +3250,13 @@ describe('WorkflowRecipeWatcher wiring — grant-update listener lifecycle (issu
expect(poolEnd).toHaveBeenCalledTimes(1)
})
})
+
+describe('startSecretWatch oauth ADDED callback', () => {
+ it('4th SecretWatcher argument only invalidates the ledger', () => {
+ const source = (
+ WorkflowRecipeWatcher.prototype as unknown as { startSecretWatch: () => Promise<void> }
+ ).startSecretWatch.toString()
+ expect(source).toContain('invalidateOAuthBrokerDeleteLedger')
+ expect(source).not.toContain('triggerSecretDrivenReconcile(recipeName)')
+ })
+})
diff --git a/workflow-recipes/src/k8sClient.ts b/workflow-recipes/src/k8sClient.ts
index 7a3a6afe9..cbca042b3 100644
--- a/workflow-recipes/src/k8sClient.ts
+++ b/workflow-recipes/src/k8sClient.ts
@@ -911,9 +911,10 @@ export class WorkflowRecipeWatcher implements WorkflowRecipeProvider {
this.secretReverseIndex,
name => this.triggerSecretDrivenReconcile(name),
10_000,
+ // Invalidate only. SecretWatcher debounces the recipe enqueue so a
+ // reconnect ADDED replay cannot force an immediate undebounced storm.
recipeName => {
this.reconciler.invalidateOAuthBrokerDeleteLedger(recipeName)
- this.triggerSecretDrivenReconcile(recipeName)
}
)
const loop = new K8sSecretWatchLoop(this.kc, namespace, handler)
diff --git a/workflow-recipes/src/reconciler/deleteOutcome.ts b/workflow-recipes/src/reconciler/deleteOutcome.ts
new file mode 100644
index 000000000..a66f500aa
--- /dev/null
+++ b/workflow-recipes/src/reconciler/deleteOutcome.ts
@@ -0,0 +1,54 @@
+import { getErrorCode } from './k8sErrors'
+
+/**
+ * Observed result of a delete against the apiserver.
+ * Record "gone" only after DELETE 200 or DELETE 404 — never after a swallowed 403/5xx.
+ */
+export type DeleteOutcome =
+ | { kind: 'deleted' }
+ | { kind: 'gone' }
+ | { kind: 'failed'; code?: number; error: unknown }
+
+export async function observeNamespacedDelete(
+ deleteFn: () => Promise<unknown>
+): Promise<DeleteOutcome> {
+ try {
+ await deleteFn()
+ return { kind: 'deleted' }
+ } catch (error: unknown) {
+ const code = getErrorCode(error)
+ if (code === 404) return { kind: 'gone' }
+ return { kind: 'failed', code, error }
+ }
+}
+
+export function shouldRecordDelete(outcome: DeleteOutcome): boolean {
+ switch (outcome.kind) {
+ case 'deleted':
+ case 'gone':
+ return true
+ case 'failed':
+ return false
+ default: {
+ const _exhaustive: never = outcome
+ return _exhaustive
+ }
+ }
+}
+
+export function deleteOutcomeFields(outcome: DeleteOutcome): {
+ outcome: DeleteOutcome['kind']
+ code?: number
+} {
+ switch (outcome.kind) {
+ case 'deleted':
+ case 'gone':
+ return { outcome: outcome.kind }
+ case 'failed':
+ return { outcome: outcome.kind, code: outcome.code }
+ default: {
+ const _exhaustive: never = outcome
+ return _exhaustive
+ }
+ }
+}
diff --git a/workflow-recipes/src/reconciler/reconciler.test.ts b/workflow-recipes/src/reconciler/reconciler.test.ts
index 1f18b266a..938ff2dea 100644
--- a/workflow-recipes/src/reconciler/reconciler.test.ts
+++ b/workflow-recipes/src/reconciler/reconciler.test.ts
@@ -15,7 +15,10 @@ import {
WorkloadDef,
} from '../types'
import { INHERITED_PARENT_RESOURCES_ANNOTATION } from '../workflow/childRecipeFactory'
+import type { CodexExecutionProjection } from '../workflow/codexExecutionProjection'
import { buildCoordinatorGfsNetworkPolicy } from '../workflow/networkPolicyFactory'
+import { deriveWorkflowRuntimePlan } from '../workflow/runtimePlan'
+import { WorkflowReconciler } from '../workflow/workflowReconciler'
import { captureLogger } from './__tests__/captureLogger'
import { defaultFqdnLookup } from './fqdnResolver'
import { isRetryableInfraError } from './k8sErrors'
@@ -15461,6 +15464,70 @@ describe('WorkflowRecipeReconciler', () => {
expect(secretDeletes()).toBe(1)
})
+ it('G3: does not record a Secret delete after DELETE 403 or 500', async () => {
+ const recipe = reapRecipe(4)
+ mockCoreApi.deleteNamespacedSecret.mockRejectedValue({ code: 403 })
+ await reconciler.ensureOAuthBrokerTokenSecret(recipe)
+ await reconciler.ensureOAuthBrokerTokenSecret(recipe)
+ expect(secretDeletes()).toBe(2)
+
+ mockCoreApi.deleteNamespacedSecret.mockRejectedValue({ code: 500 })
+ mockCoreApi.deleteNamespacedSecret.mockClear()
+ await reconciler.ensureOAuthBrokerTokenSecret(recipe)
+ await reconciler.ensureOAuthBrokerTokenSecret(recipe)
+ expect(secretDeletes()).toBe(2)
+ mockCoreApi.deleteNamespacedSecret.mockResolvedValue({})
+ })
+
+ it('G3: records a Secret delete after DELETE 404 and skips the next same-generation call', async () => {
+ mockCoreApi.deleteNamespacedSecret.mockRejectedValue({ code: 404 })
+ const recipe = reapRecipe(7)
+ await reconciler.ensureOAuthBrokerTokenSecret(recipe)
+ await reconciler.ensureOAuthBrokerTokenSecret(recipe)
+ expect(secretDeletes()).toBe(1)
+ mockCoreApi.deleteNamespacedSecret.mockResolvedValue({})
+ })
+
+ it('G3: does not start the NP TTL after DELETE 403', async () => {
+ vi.useFakeTimers()
+ vi.setSystemTime(new Date('2026-09-25T00:00:00.000Z'))
+ mockNetworkingApi.deleteNamespacedNetworkPolicy.mockRejectedValue({ code: 403 })
+ const recipe = reapRecipe(4)
+ await reapPolicy(recipe)
+ await reapPolicy(recipe)
+ expect(policyDeletes()).toBe(2)
+ vi.setSystemTime(new Date('2026-09-25T01:00:00.000Z'))
+ await reapPolicy(recipe)
+ expect(policyDeletes()).toBe(3)
+ mockNetworkingApi.deleteNamespacedNetworkPolicy.mockResolvedValue({})
+ vi.useRealTimers()
+ })
+
+ it('c3: deletes the NetworkPolicy again when metadata.generation changes', async () => {
+ await reapPolicy(reapRecipe(4))
+ await reapPolicy(reapRecipe(4))
+ expect(policyDeletes()).toBe(1)
+ await reapPolicy(reapRecipe(5))
+ expect(policyDeletes()).toBe(2)
+ })
+
+ it('c3: generation 0 on one recipe does not skip another recipe with undefined generation', async () => {
+ await reconciler.ensureOAuthBrokerTokenSecret(reapRecipe(0))
+ const other = makeRecipe({
+ metadata: {
+ name: 'other-recipe',
+ namespace: 'sandbox-recipes',
+ uid: 'uid-other',
+ },
+ })
+ await reconciler.ensureOAuthBrokerTokenSecret(other)
+ expect(
+ mockCoreApi.deleteNamespacedSecret.mock.calls.filter(
+ ([arg]) => arg.name === 'wf-other-recipe-oauth-broker-token'
+ )
+ ).toHaveLength(1)
+ })
+
it('deletes the NetworkPolicy once for the same generation, then again after the 1h TTL', async () => {
vi.useFakeTimers()
vi.setSystemTime(new Date('2026-09-25T00:00:00.000Z'))
@@ -15483,6 +15550,150 @@ describe('WorkflowRecipeReconciler', () => {
})
})
+ describe('G2 skipStatusPatch must not leave GFS deleted', () => {
+ const RECIPE = 'g2-recipe'
+ const GFS = `${RECIPE}-coordinator-to-gfs`
+
+ function ineligibleProjection(): CodexExecutionProjection {
+ return {
+ targets: [],
+ eligibleTargets: [],
+ derivedScopes: [],
+ requiresCodexProxyEgress: false,
+ driftHashInput: '',
+ catalogContentHash: null,
+ catalogRevision: null,
+ connectionRevision: null,
+ eligibility: 'ineligible',
+ reason: 'static_only',
+ }
+ }
+
+ function installRealInner(): WorkflowReconciler {
+ const inner = new WorkflowReconciler({
+ coreApi: mockCoreApi,
+ customApi: mockCustomApi,
+ networkingApi: mockNetworkingApi,
+ config: {
+ coordinatorImage: 'coordinator:test',
+ mcpHostImage: 'mcp-host:test',
+ wrcEndpoint: 'http://wrc.example/api',
+ sandboxNamespace: 'sandbox-recipes',
+ mcpServerNamespace: 'mcp-server',
+ imagePullPolicy: 'IfNotPresent',
+ maxWorkflowSteps: 100,
+ runtimeTokenTtlSeconds: 3600,
+ runtimeTokenRefreshBeforeSeconds: 300,
+ },
+ tokenFactory: {
+ signWrcArtifactDeleteToken: vi.fn().mockResolvedValue('t'),
+ signCoordinatorToMcpHostToken: vi.fn().mockResolvedValue('t'),
+ signCustomCoordinatorToWrcToken: vi.fn().mockResolvedValue('t'),
+ signCoordinatorToWrcToken: vi.fn().mockResolvedValue('t'),
+ },
+ pluginWorkloadSdkRevocationClient: {
+ revoke: vi.fn().mockResolvedValue({ state: 'missing', revoked: 0, fencedInvocations: 0 }),
+ finalize: vi.fn(),
+ },
+ } as never)
+ ;(reconciler as unknown as { workflowReconciler: WorkflowReconciler }).workflowReconciler =
+ inner
+ return inner
+ }
+
+ it('inner apply does not delete reserved GFS and outer skips ensure on skipStatusPatch', async () => {
+ mockNetworkingApi.listNamespacedNetworkPolicy.mockResolvedValue({
+ items: [
+ {
+ metadata: {
+ name: GFS,
+ namespace: 'sandbox-recipes',
+ labels: { 'clerum.io/recipe': RECIPE, 'clerum.io/managed-by': 'wrc' },
+ },
+ },
+ ],
+ })
+ const inner = installRealInner()
+ const apply = (
+ inner as unknown as {
+ applyWorkflowNetworkPolicies: (...args: unknown[]) => Promise<unknown>
+ }
+ ).applyWorkflowNetworkPolicies.bind(inner)
+ vi.spyOn(inner, 'reconcile').mockImplementation(async (recipeName, recipeUid, _ns, spec) => {
+ const runtime = deriveWorkflowRuntimePlan(spec as never, {
+ recipeName: String(recipeName),
+ runtimeScopeRecipeName: String(recipeName),
+ workflowRunId: 'run-g2',
+ })
+ const projection = ineligibleProjection()
+ await apply(recipeName, recipeUid, spec, runtime, false, projection, false, {
+ ...projection,
+ requiresGrokProxyEgress: false,
+ })
+ return {
+ phase: 'active',
+ message: 'transient',
+ workflowPhase: 'running',
+ skipStatusPatch: true,
+ }
+ })
+
+ const result = await reconciler.reconcile(
+ makeRecipe({
+ metadata: {
+ name: RECIPE,
+ namespace: 'sandbox-recipes',
+ uid: 'uid-g2',
+ labels: { 'clerum.io/workflow-run-id': 'run-g2' },
+ },
+ spec: {
+ agent: { provider: 'openai', model: 'gpt-4o' },
+ steps: [{ id: 'publish', instruction: 'publish' }],
+ gfs: { publishTargets: [{ drive: 'main', target: 'out' }] },
+ },
+ status: { phase: 'pending' },
+ })
+ )
+
+ expect(result, result.message).toMatchObject({ skipStatusPatch: true })
+ expect(mockNetworkingApi.deleteNamespacedNetworkPolicy).not.toHaveBeenCalledWith(
+ expect.objectContaining({ name: GFS })
+ )
+ expect(
+ mockNetworkingApi.createNamespacedNetworkPolicy.mock.calls.some(
+ ([arg]) => arg.body?.metadata?.name === GFS
+ )
+ ).toBe(false)
+ })
+
+ it('retryRunLaneNetworkPolicies does not prune reserved GFS', async () => {
+ mockNetworkingApi.listNamespacedNetworkPolicy.mockResolvedValue({
+ items: [
+ {
+ metadata: {
+ name: GFS,
+ namespace: 'sandbox-recipes',
+ labels: { 'clerum.io/recipe': RECIPE, 'clerum.io/managed-by': 'wrc' },
+ },
+ },
+ ],
+ })
+ const inner = installRealInner()
+
+ await inner.retryRunLaneNetworkPolicies(
+ RECIPE,
+ 'uid-g2',
+ { steps: [{ id: 'publish', instruction: 'publish' }] },
+ RECIPE,
+ 'run-g2'
+ )
+
+ expect(mockNetworkingApi.deleteNamespacedNetworkPolicy).not.toHaveBeenCalledWith(
+ expect.objectContaining({ name: GFS })
+ )
+ })
+ })
+
// ─── Broker-token issuance failure is non-fatal ─────────────────────────
//
// A transient control-api blip during reconcile (e.g. it is mid-restart)
diff --git a/workflow-recipes/src/reconciler/resourceBuilder.test.ts b/workflow-recipes/src/reconciler/resourceBuilder.test.ts
index 4886756cd..7b48c62a6 100644
--- a/workflow-recipes/src/reconciler/resourceBuilder.test.ts
+++ b/workflow-recipes/src/reconciler/resourceBuilder.test.ts
@@ -25,8 +25,10 @@ import {
buildUiEgressNetworkPolicy,
oauthBrokerEgressPolicyName,
oauthBrokerTokenSecretName,
+ parseOAuthBrokerTokenSecretRecipe,
recipeHasBackgroundAccessClient,
resolveCronJobResourceName,
+ resolveOAuthBrokerTokenWatchRecipe,
resolveResourceName,
resolveScopedCronJobResourceName,
resolveScopedResourceName,
@@ -2078,6 +2080,36 @@ describe('recipe OAuth broker token', () => {
expect(workloadUsesBackgroundOauth(danglingRef, recipe)).toBe(false)
})
+ it('resolves watch ADDED only for the canonical oauth-broker-token Secret name', () => {
+ const recipe = 'test-recipe'
+ const canonical = oauthBrokerTokenSecretName(recipe)
+ expect(parseOAuthBrokerTokenSecretRecipe(canonical)).toBe(recipe)
+ expect(parseOAuthBrokerTokenSecretRecipe('wf--oauth-broker-token')).toBeUndefined()
+ expect(
+ resolveOAuthBrokerTokenWatchRecipe(canonical, {
+ 'clerum.io/component': 'oauth-broker-token',
+ 'clerum.io/recipe': recipe,
+ })
+ ).toBe(recipe)
+ expect(
+ resolveOAuthBrokerTokenWatchRecipe(canonical, {
+ 'clerum.io/component': 'oauth-broker-token',
+ })
+ ).toBe(recipe)
+ expect(
+ resolveOAuthBrokerTokenWatchRecipe('forged-name', {
+ 'clerum.io/component': 'oauth-broker-token',
+ 'clerum.io/recipe': recipe,
+ })
+ ).toBeUndefined()
+ expect(
+ resolveOAuthBrokerTokenWatchRecipe(canonical, {
+ 'clerum.io/component': 'other',
+ 'clerum.io/recipe': recipe,
+ })
+ ).toBeUndefined()
+ })
+
it('buildOAuthBrokerTokenSecret stores the token base64-encoded under broker-token', () => {
const secret = buildOAuthBrokerTokenSecret('test-recipe', 'JWT.VALUE', 'sandbox-recipes')
expect(secret.metadata?.name).toBe(oauthBrokerTokenSecretName('test-recipe'))
diff --git a/workflow-recipes/src/reconciler/resourceBuilder.ts b/workflow-recipes/src/reconciler/resourceBuilder.ts
index 7b7dc447b..caadd50ad 100644
--- a/workflow-recipes/src/reconciler/resourceBuilder.ts
+++ b/workflow-recipes/src/reconciler/resourceBuilder.ts
@@ -435,6 +435,41 @@ export function oauthBrokerTokenSecretName(recipeName: string): string {
return `wf-${recipeName}-oauth-broker-token`
}
+const OAUTH_BROKER_TOKEN_SECRET_PREFIX = 'wf-'
+const OAUTH_BROKER_TOKEN_SECRET_SUFFIX = '-oauth-broker-token'
+
+/** Parse `wf-<recipe>-oauth-broker-token`. Rejects an empty recipe segment. */
+export function parseOAuthBrokerTokenSecretRecipe(secretName: string): string | undefined {
+ if (
+ !secretName.startsWith(OAUTH_BROKER_TOKEN_SECRET_PREFIX) ||
+ !secretName.endsWith(OAUTH_BROKER_TOKEN_SECRET_SUFFIX)
+ ) {
+ return undefined
+ }
+ const recipe = secretName.slice(
+ OAUTH_BROKER_TOKEN_SECRET_PREFIX.length,
+ secretName.length - OAUTH_BROKER_TOKEN_SECRET_SUFFIX.length
+ )
+ return recipe.length > 0 ? recipe : undefined
+}
+
+/**
+ * Watch-bind for B3(a) invalidation. Requires `clerum.io/component=oauth-broker-token`
+ * and `metadata.name === wf-${recipe}-oauth-broker-token`. A forged recipe label
+ * on a different Secret name must not invalidate.
+ */
+export function resolveOAuthBrokerTokenWatchRecipe(
+ secretName: string,
+ labels: Record<string, string> | undefined
+): string | undefined {
+ if (labels?.['clerum.io/component'] !== 'oauth-broker-token') return undefined
+ const labeled = labels['clerum.io/recipe']
+ if (labeled) {
+ return secretName === oauthBrokerTokenSecretName(labeled) ? labeled : undefined
+ }
+ return parseOAuthBrokerTokenSecretRecipe(secretName)
+}
+
export function recipeHasBackgroundAccessClient(recipe: WorkflowRecipeCRD): boolean {
return (recipe.spec.oauthClients ?? []).some(c => c.backgroundAccess === true)
}
diff --git a/workflow-recipes/src/reconciler/secretWatcher.test.ts b/workflow-recipes/src/reconciler/secretWatcher.test.ts
index 2a9ff5425..6e0bd28f2 100644
--- a/workflow-recipes/src/reconciler/secretWatcher.test.ts
+++ b/workflow-recipes/src/reconciler/secretWatcher.test.ts
@@ -244,6 +244,49 @@ describe('SecretWatcher', () => {
watcher.handleEvent('ADDED', brokerSecret)
watcher.handleEvent('ADDED', brokerSecret)
expect(invalidated).toEqual(['test-recipe', 'test-recipe'])
+ expect(enqueued).toEqual([])
+ vi.advanceTimersByTime(DEBOUNCE_MS)
+ expect(enqueued).toEqual(['test-recipe'])
+ watcher.stop()
+ })
+
+ it('does not invalidate on forged name, wrong component, or MODIFIED', () => {
+ const reverseIndex = new SecretReverseIndex()
+ const enqueued: string[] = []
+ const invalidated: string[] = []
+ const watcher = new SecretWatcher(
+ reverseIndex,
+ name => {
+ enqueued.push(name)
+ },
+ DEBOUNCE_MS,
+ recipeName => {
+ invalidated.push(recipeName)
+ }
+ )
+ const labels = {
+ 'clerum.io/component': 'oauth-broker-token',
+ 'clerum.io/recipe': 'test-recipe',
+ }
+
+ watcher.handleEvent('ADDED', {
+ metadata: { name: 'forged-oauth-broker-token', labels },
+ data: { 'broker-token': 'eA==' },
+ })
+ watcher.handleEvent('ADDED', {
+ metadata: {
+ name: 'wf-test-recipe-oauth-broker-token',
+ labels: { 'clerum.io/component': 'other', 'clerum.io/recipe': 'test-recipe' },
+ },
+ data: { 'broker-token': 'eA==' },
+ })
+ watcher.handleEvent('MODIFIED', {
+ metadata: { name: 'wf-test-recipe-oauth-broker-token', labels },
+ data: { 'broker-token': 'eA==' },
+ })
+ expect(invalidated).toEqual([])
+ vi.advanceTimersByTime(DEBOUNCE_MS)
+ expect(enqueued).toEqual([])
watcher.stop()
})
})
diff --git a/workflow-recipes/src/reconciler/secretWatcher.ts b/workflow-recipes/src/reconciler/secretWatcher.ts
index deb026968..e4518d634 100644
--- a/workflow-recipes/src/reconciler/secretWatcher.ts
+++ b/workflow-recipes/src/reconciler/secretWatcher.ts
@@ -1,3 +1,4 @@
+import { resolveOAuthBrokerTokenWatchRecipe } from './resourceBuilder'
import { OWNER_RECIPE_LABEL_KEY, SHARED_LABEL_KEY } from './secretOwnership'
import { SecretReverseIndex } from './secretReverseIndex'
@@ -59,13 +60,15 @@ export class SecretWatcher {
const name = secret.metadata?.name
if (!name) return
- // B3(a): ADDED of the broker token must invalidate the ledger even when
- // the key-set matches a previous observation (dedup would otherwise drop it).
+ // B3(a): ADDED of the canonical broker token invalidates the ledger even
+ // when the key-set matches a previous observation (dedup would drop it).
+ // Enqueue the recipe through the same debounce as key-set fan-out so a
+ // reconnect replay cannot force an immediate undebounced storm.
if (type === 'ADDED') {
- const labels = secret.metadata?.labels ?? {}
- if (labels['clerum.io/component'] === 'oauth-broker-token') {
- const recipeName = labels['clerum.io/recipe']
- if (recipeName) this.onOAuthBrokerTokenAdded?.(recipeName)
+ const recipeName = resolveOAuthBrokerTokenWatchRecipe(name, secret.metadata?.labels)
+ if (recipeName) {
+ this.onOAuthBrokerTokenAdded?.(recipeName)
+ this.scheduleRecipeReconcile(recipeName)
}
}
@@ -130,6 +133,19 @@ export class SecretWatcher {
}, this.debounceMs)
this.debounceTimers.set(secretName, timer)
}
+
+ /** Debounce enqueue by recipe name (oauth-broker ADDED is not reverse-indexed). */
+ private scheduleRecipeReconcile(recipeName: string): void {
+ const key = `oauth-broker:${recipeName}`
+ const existing = this.debounceTimers.get(key)
+ if (existing) clearTimeout(existing)
+
+ const timer = setTimeout(() => {
+ this.debounceTimers.delete(key)
+ this.enqueueReconcile(recipeName)
+ }, this.debounceMs)
+ this.debounceTimers.set(key, timer)
+ }
}
function setsEqual(a: ReadonlySet<string>, b: ReadonlySet<string>): boolean {
diff --git a/workflow-recipes/src/reconciler/workflowRecipeReconciler.ts b/workflow-recipes/src/reconciler/workflowRecipeReconciler.ts
index ff367c206..61bca6a7e 100644
--- a/workflow-recipes/src/reconciler/workflowRecipeReconciler.ts
+++ b/workflow-recipes/src/reconciler/workflowRecipeReconciler.ts
@@ -74,6 +74,7 @@ import {
} from '../workflow/workflowReconciler'
import { evaluateComputedValues } from './computedValuesEvaluator'
import { CRD_GROUP, CRD_VERSION, WORKFLOWRECIPE_PLURAL } from './crdConstants'
+import { deleteOutcomeFields, observeNamespacedDelete, shouldRecordDelete } from './deleteOutcome'
import { sort as sortDependencies } from './dependencyGraph'
import {
type AccumulateOutput,
@@ -7514,11 +7515,27 @@ export class WorkflowRecipeReconciler {
)
return
}
- await this.safeDelete(
- () => this.coreApi.deleteNamespacedSecret({ name: secretName, namespace: ns }),
- `Secret "${secretName}" in ${ns}`
+ const outcome = await observeNamespacedDelete(() =>
+ this.coreApi.deleteNamespacedSecret({ name: secretName, namespace: ns })
)
- this.oauthBrokerDeleteLedger.recordSecretDelete(recipeName, generation)
+ const log = createLogger('wrc', recipeName)
+ if (outcome.kind === 'failed') {
+ log.error('Failed to delete oauth-broker-token Secret', {
+ secretName,
+ ns,
+ ...deleteOutcomeFields(outcome),
+ err: outcome.error,
+ })
+ } else {
+ log.info('oauth-broker-token Secret delete', {
+ secretName,
+ ns,
+ ...deleteOutcomeFields(outcome),
+ })
+ }
+ if (shouldRecordDelete(outcome)) {
+ this.oauthBrokerDeleteLedger.recordSecretDelete(recipeName, generation)
+ }
return
}
@@ -7590,11 +7607,27 @@ export class WorkflowRecipeReconciler {
)
return
}
- await this.safeDelete(
- () => this.networkingApi.deleteNamespacedNetworkPolicy({ name: policyName, namespace: ns }),
- `NetworkPolicy "${policyName}" in ${ns}`
+ const outcome = await observeNamespacedDelete(() =>
+ this.networkingApi.deleteNamespacedNetworkPolicy({ name: policyName, namespace: ns })
)
- this.oauthBrokerDeleteLedger.recordPolicyDelete(recipeName, generation)
+ const log = createLogger('wrc', recipeName)
+ if (outcome.kind === 'failed') {
+ log.error('Failed to delete oauth-broker-egress NetworkPolicy', {
+ policyName,
+ ns,
+ ...deleteOutcomeFields(outcome),
+ err: outcome.error,
+ })
+ } else {
+ log.info('oauth-broker-egress NetworkPolicy delete', {
+ policyName,
+ ns,
+ ...deleteOutcomeFields(outcome),
+ })
+ }
+ if (shouldRecordDelete(outcome)) {
+ this.oauthBrokerDeleteLedger.recordPolicyDelete(recipeName, generation)
+ }
return
}
await this.applyNetworkPolicy(policy, ns, { family: 'oauth-broker-egress' })
diff --git a/workflow-recipes/src/workflow/crashRecovery.test.ts b/workflow-recipes/src/workflow/crashRecovery.test.ts
index 7a0816e87..b01ca851a 100644
--- a/workflow-recipes/src/workflow/crashRecovery.test.ts
+++ b/workflow-recipes/src/workflow/crashRecovery.test.ts
@@ -2,6 +2,7 @@ import { describe, expect, it, vi } from 'vitest'
import {
evaluateCompletedRuntimePodRecovery,
evaluateCrashRecovery,
+ getPodPresence,
inspectPodReadiness,
waitForPodDeletion,
} from './crashRecovery'
@@ -111,6 +112,28 @@ describe('crash recovery pod readiness classification', () => {
})
})
+describe('getPodPresence', () => {
+ it('returns absent on GET 404 and present on GET 200 with empty status', async () => {
+ const readNamespacedPod = vi
+ .fn()
+ .mockRejectedValueOnce({ code: 404 })
+ .mockResolvedValueOnce({ metadata: { name: 'recipe-coordinator' }, status: {} })
+ const coreApi = { readNamespacedPod } as unknown as Parameters<typeof getPodPresence>[0]
+
+ await expect(getPodPresence(coreApi, 'recipe-coordinator', 'sandbox-recipes')).resolves.toEqual(
+ {
+ kind: 'absent',
+ }
+ )
+ await expect(getPodPresence(coreApi, 'recipe-coordinator', 'sandbox-recipes')).resolves.toEqual(
+ {
+ kind: 'present',
+ phase: undefined,
+ }
+ )
+ })
+})
+
describe('waitForPodDeletion', () => {
it('returns true once getPodPhase reports the pod is gone', async () => {
const readNamespacedPod = vi
diff --git a/workflow-recipes/src/workflow/crashRecovery.ts b/workflow-recipes/src/workflow/crashRecovery.ts
index 2b5aabce3..10cbdbdac 100644
--- a/workflow-recipes/src/workflow/crashRecovery.ts
+++ b/workflow-recipes/src/workflow/crashRecovery.ts
@@ -148,26 +148,56 @@ export async function waitForPodDeletion(
const pollIntervalMs = options.pollIntervalMs ?? POD_DELETE_POLL_MS
const deadline = Date.now() + timeoutMs
while (Date.now() < deadline) {
- if ((await getPodPhase(coreApi, name, namespace)) === undefined) return true
+ if ((await getPodPresence(coreApi, name, namespace)).kind === 'absent') return true
await new Promise(resolve => setTimeout(resolve, pollIntervalMs))
}
- return (await getPodPhase(coreApi, name, namespace)) === undefined
+ return (await getPodPresence(coreApi, name, namespace)).kind === 'absent'
}
-export async function getPodPhase(
+/**
+ * Discriminated GET result. `unread` is only for callers that skipped the GET;
+ * `getPodPresence` never returns it. Skip DELETE only on `absent` — a live
+ * object with an empty `status.phase` is `present`.
+ */
+export type PodPresence =
+ | { kind: 'absent' }
+ | { kind: 'present'; phase?: string }
+ | { kind: 'unread' }
+
+export type ObservedPodPresence = Exclude<PodPresence, { kind: 'unread' }>
+
+export async function getPodPresence(
coreApi: k8s.CoreV1Api,
name: string,
namespace: string
-): Promise<string | undefined> {
+): Promise<ObservedPodPresence> {
try {
const pod = await coreApi.readNamespacedPod({ name, namespace })
- return pod.status?.phase
+ return { kind: 'present', phase: pod.status?.phase }
} catch (error: unknown) {
- if (getErrorCode(error) === 404) return undefined
+ if (getErrorCode(error) === 404) return { kind: 'absent' }
throw error
}
}
+export async function getPodPhase(
+ coreApi: k8s.CoreV1Api,
+ name: string,
+ namespace: string
+): Promise<string | undefined> {
+ const presence = await getPodPresence(coreApi, name, namespace)
+ switch (presence.kind) {
+ case 'absent':
+ return undefined
+ case 'present':
+ return presence.phase
+ default: {
+ const _exhaustive: never = presence
+ return _exhaustive
+ }
+ }
+}
+
export async function getPodReadiness(
coreApi: k8s.CoreV1Api,
name: string,
diff --git a/workflow-recipes/src/workflow/networkPolicyFactory.gfs.test.ts b/workflow-recipes/src/workflow/networkPolicyFactory.gfs.test.ts
index 17d2ed201..85812030c 100644
--- a/workflow-recipes/src/workflow/networkPolicyFactory.gfs.test.ts
+++ b/workflow-recipes/src/workflow/networkPolicyFactory.gfs.test.ts
@@ -2,6 +2,7 @@ import { describe, expect, it } from 'vitest'
import {
type NetworkPolicyConfig,
buildCoordinatorGfsNetworkPolicy,
+ buildRunLaneNetworkPolicyCatalog,
buildWorkflowNetworkPolicies,
} from './networkPolicyFactory'
@@ -154,4 +155,15 @@ describe('workflow GFS NetworkPolicy', () => {
},
})
})
+
+ it('run-lane catalog includes factory leftovers and never coordinator-to-gfs', () => {
+ const catalog = buildRunLaneNetworkPolicyCatalog({
+ ...baseConfig,
+ includeCoordinatorGfs: true,
+ })
+ expect(catalog.has('daily-report-mcp-host-to-grok-proxy')).toBe(true)
+ expect(catalog.has('daily-report-mcp-host-to-codex-proxy')).toBe(true)
+ expect(catalog.has('daily-report-wrc-to-artifact-reader')).toBe(true)
+ expect(catalog.has('daily-report-coordinator-to-gfs')).toBe(false)
+ })
})
diff --git a/workflow-recipes/src/workflow/networkPolicyFactory.ts b/workflow-recipes/src/workflow/networkPolicyFactory.ts
index 8e3895eda..87f916806 100644
--- a/workflow-recipes/src/workflow/networkPolicyFactory.ts
+++ b/workflow-recipes/src/workflow/networkPolicyFactory.ts
@@ -1068,3 +1068,35 @@ export function buildWorkflowNetworkPolicies(
...pluginWorkloadSdkPolicies,
])
}
+
+const RUN_LANE_CATALOG_PLACEHOLDER_SERVER = '_catalog'
+
+/**
+ * Names the run-lane factory can emit for this recipe. Outer-owned
+ * `${recipe}-coordinator-to-gfs` is never included (`includeCoordinatorGfs: false`).
+ * Catalog membership — not a GFS-name keep-list — is the prune universe.
+ */
+export function buildRunLaneNetworkPolicyCatalog(
+ applyConfig: NetworkPolicyConfig,
+ mcpServerNames: string[] = [],
+ snippetMcpServerNames: string[] = []
+): Set<string> {
+ const catalogConfig: NetworkPolicyConfig = {
+ ...applyConfig,
+ includeCoordinator: true,
+ includeMcpHost: true,
+ includeCodexProxyEgress: true,
+ includeGrokProxyEgress: true,
+ includeCoordinatorGfs: false,
+ includeArtifactReader: true,
+ includeSnippetRunner: true,
+ pluginWorkloadSdkSandboxAccess: true,
+ }
+ const serverNames =
+ mcpServerNames.length > 0 ? mcpServerNames : [RUN_LANE_CATALOG_PLACEHOLDER_SERVER]
+ return new Set(
+ buildWorkflowNetworkPolicies(catalogConfig, serverNames, snippetMcpServerNames)
+ .map(policy => policy.metadata?.name)
+ .filter((name): name is string => Boolean(name))
+ )
+}
diff --git a/workflow-recipes/src/workflow/workflowReconciler.codexScopeProvenance.test.ts b/workflow-recipes/src/workflow/workflowReconciler.codexScopeProvenance.test.ts
index 8083ea7f4..7e77b9f52 100644
--- a/workflow-recipes/src/workflow/workflowReconciler.codexScopeProvenance.test.ts
+++ b/workflow-recipes/src/workflow/workflowReconciler.codexScopeProvenance.test.ts
@@ -24,6 +24,10 @@ const crashRecoveryMocks = vi.hoisted(() => ({
evaluateCrashRecovery: vi.fn().mockReturnValue({ action: 'none', message: 'Pod is healthy' }),
getContainerWaitingReason: vi.fn().mockResolvedValue(undefined),
getPodPhase: vi.fn().mockResolvedValue(undefined),
+ getPodPresence: vi.fn(async (api: unknown, name: string, namespace: string) => {
+ const phase = await crashRecoveryMocks.getPodPhase(api, name, namespace)
+ return phase === undefined ? { kind: 'absent' as const } : { kind: 'present' as const, phase }
+ }),
getPodReadiness: vi.fn().mockResolvedValue({ ready: true, phase: 'Running', uid: 'pod-uid-1' }),
isRecoverableContainerWaitingReason: vi.fn(() => false),
}))
diff --git a/workflow-recipes/src/workflow/workflowReconciler.codexUncertainScope.test.ts b/workflow-recipes/src/workflow/workflowReconciler.codexUncertainScope.test.ts
index 8949f7c36..9ea5cacbb 100644
--- a/workflow-recipes/src/workflow/workflowReconciler.codexUncertainScope.test.ts
+++ b/workflow-recipes/src/workflow/workflowReconciler.codexUncertainScope.test.ts
@@ -16,6 +16,10 @@ const crashRecoveryMocks = vi.hoisted(() => ({
evaluateCrashRecovery: vi.fn().mockReturnValue({ action: 'none', message: 'Pod is healthy' }),
getContainerWaitingReason: vi.fn().mockResolvedValue(undefined),
getPodPhase: vi.fn().mockResolvedValue(undefined),
+ getPodPresence: vi.fn(async (api: unknown, name: string, namespace: string) => {
+ const phase = await crashRecoveryMocks.getPodPhase(api, name, namespace)
+ return phase === undefined ? { kind: 'absent' as const } : { kind: 'present' as const, phase }
+ }),
getPodReadiness: vi.fn().mockResolvedValue({ ready: true, phase: 'Running', uid: 'pod-uid-1' }),
isRecoverableContainerWaitingReason: vi.fn(() => false),
}))
diff --git a/workflow-recipes/src/workflow/workflowReconciler.grokWrcFlag.test.ts b/workflow-recipes/src/workflow/workflowReconciler.grokWrcFlag.test.ts
index c5535ed92..740674edf 100644
--- a/workflow-recipes/src/workflow/workflowReconciler.grokWrcFlag.test.ts
+++ b/workflow-recipes/src/workflow/workflowReconciler.grokWrcFlag.test.ts
@@ -23,6 +23,10 @@ const crashRecoveryMocks = vi.hoisted(() => ({
evaluateCrashRecovery: vi.fn().mockReturnValue({ action: 'none', message: 'Pod is healthy' }),
getContainerWaitingReason: vi.fn().mockResolvedValue(undefined),
getPodPhase: vi.fn().mockResolvedValue(undefined),
+ getPodPresence: vi.fn(async (api: unknown, name: string, namespace: string) => {
+ const phase = await crashRecoveryMocks.getPodPhase(api, name, namespace)
+ return phase === undefined ? { kind: 'absent' as const } : { kind: 'present' as const, phase }
+ }),
getPodReadiness: vi.fn().mockResolvedValue({ ready: true, phase: 'Running', uid: 'pod-uid-1' }),
isRecoverableContainerWaitingReason: vi.fn(() => false),
}))
@@ -310,6 +314,30 @@ describe('WorkflowReconciler Grok WRC switch', () => {
expect(env).not.toContain('GROK_LLM_PROXY_RUNTIME_URL')
})
+ it(`deletes a seeded leftover Grok proxy when the WRC flag is off for a ${shape} recipe`, async () => {
+ const { reconciler, networkingApi } = createHarness(false)
+ bindGrokGrant(reconciler)
+ networkingApi.listNamespacedNetworkPolicy.mockResolvedValue({
+ items: [
+ {
+ metadata: {
+ name: GROK_PROXY_POLICY,
+ namespace: sandboxNamespace,
+ labels: { 'clerum.io/recipe': RECIPE, 'clerum.io/managed-by': 'wrc' },
+ },
+ },
+ ],
+ })
+
+ await reconcileRecipe(reconciler, spec())
+
+ expect(createdPolicyNames(networkingApi)).not.toContain(GROK_PROXY_POLICY)
+ expect(networkingApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledWith({
+ name: GROK_PROXY_POLICY,
+ namespace: sandboxNamespace,
+ })
+ })
+
it(`keeps Grok scope, proxy egress and pod env for a ${shape} recipe when the WRC flag is on`, async () => {
const { reconciler, coreApi, networkingApi } = createHarness(true)
bindGrokGrant(reconciler)
diff --git a/workflow-recipes/src/workflow/workflowReconciler.test.ts b/workflow-recipes/src/workflow/workflowReconciler.test.ts
index c64c120dd..533360030 100644
--- a/workflow-recipes/src/workflow/workflowReconciler.test.ts
+++ b/workflow-recipes/src/workflow/workflowReconciler.test.ts
@@ -16,6 +16,10 @@ const crashRecoveryMocks = vi.hoisted(() => ({
evaluateCrashRecovery: vi.fn().mockReturnValue({ action: 'none', message: 'Pod is healthy' }),
getContainerWaitingReason: vi.fn().mockResolvedValue(undefined),
getPodPhase: vi.fn().mockResolvedValue(undefined),
+ getPodPresence: vi.fn(async (api: unknown, name: string, namespace: string) => {
+ const phase = await crashRecoveryMocks.getPodPhase(api, name, namespace)
+ return phase === undefined ? { kind: 'absent' as const } : { kind: 'present' as const, phase }
+ }),
getPodReadiness: vi.fn().mockResolvedValue({ ready: true, phase: 'Running', uid: 'pod-uid-1' }),
isRecoverableContainerWaitingReason: vi.fn(
(phase: string | undefined, reason: string | undefined) =>
diff --git a/workflow-recipes/src/workflow/workflowReconciler.ts b/workflow-recipes/src/workflow/workflowReconciler.ts
index e3f52de39..15a07ec94 100644
--- a/workflow-recipes/src/workflow/workflowReconciler.ts
+++ b/workflow-recipes/src/workflow/workflowReconciler.ts
@@ -25,6 +25,11 @@ import {
import { mintRecipeHostGfsToken } from '../gfsBinding'
import { createLogger } from '../observability/logger'
import { CRD_GROUP, CRD_VERSION, WORKFLOWRECIPE_PLURAL } from '../reconciler/crdConstants'
+import {
+ deleteOutcomeFields,
+ observeNamespacedDelete,
+ shouldRecordDelete,
+} from '../reconciler/deleteOutcome'
import {
ResourceVanishedAfterConflictError,
RetryableReconcileError,
@@ -64,12 +69,14 @@ import {
projectCodexRecipeVerdict,
} from './codexRecipeVerdict'
import {
+ type PodPresence,
type PodReadiness,
deletePodIfExists,
evaluateCompletedRuntimePodRecovery,
evaluateCrashRecovery,
getContainerWaitingReason,
getPodPhase,
+ getPodPresence,
getPodReadiness,
isRecoverableContainerWaitingReason,
waitForPodDeletion,
@@ -89,7 +96,11 @@ import {
issueMcpHostWorkflowControlToken,
} from './mcpHostRuntimeTokenIssuerClient'
import type { ModelConfigHandler } from './modelConfigHandler'
-import { NetworkPolicyConfig, buildWorkflowNetworkPolicies } from './networkPolicyFactory'
+import {
+ NetworkPolicyConfig,
+ buildRunLaneNetworkPolicyCatalog,
+ buildWorkflowNetworkPolicies,
+} from './networkPolicyFactory'
import { ObjectStorageClient, StorageCredentials, StorageRef } from './objectStorageClient'
import {
type EagerSdkBootstrapProof,
@@ -1778,11 +1789,12 @@ export class WorkflowReconciler {
try {
// Check crash recovery for existing Pods
- let coordPhase = await getPodPhase(
+ const coordPresence = await getPodPresence(
this.deps.coreApi,
`${recipeName}-coordinator`,
this.deps.config.sandboxNamespace
)
+ let coordPhase = coordPresence.kind === 'present' ? coordPresence.phase : undefined
const coordWaitingReason = await getContainerWaitingReason(
this.deps.coreApi,
`${recipeName}-coordinator`,
@@ -1802,13 +1814,14 @@ export class WorkflowReconciler {
this.deps.config.sandboxNamespace
)
: undefined
- let mcpHostPhase = needsMcpHost
- ? await getPodPhase(
+ const mcpHostPresence: PodPresence = needsMcpHost
+ ? await getPodPresence(
this.deps.coreApi,
`${recipeName}-mcp-host`,
this.deps.config.sandboxNamespace
)
- : undefined
+ : { kind: 'unread' }
+ let mcpHostPhase = mcpHostPresence.kind === 'present' ? mcpHostPresence.phase : undefined
let artifactReaderPhase = needsArtifactReader
? await getPodPhase(
this.deps.coreApi,
@@ -1860,18 +1873,26 @@ export class WorkflowReconciler {
}
if (awaitsTriggeredRun) {
- const samePassPhase: Partial<Record<WorkflowRuntimeComponent, string | undefined>> = {
- 'workflow-coordinator': coordPhase,
- 'workflow-mcp-host': mcpHostPhase,
+ const samePassPresence: Partial<Record<WorkflowRuntimeComponent, PodPresence>> = {
+ 'workflow-coordinator': coordPresence,
+ ...(mcpHostPresence.kind !== 'unread' ? { 'workflow-mcp-host': mcpHostPresence } : {}),
}
for (const component of runtime.cleanup.deleteBeforeTriggeredRun) {
// Plugin Workload SDK: keep the mcp-host alive across the
// awaiting-trigger window — it hosts the always-on SDK server.
// Only the coordinator (and run-scoped pods) are torn down here.
if (eagerMcpHostForSdk && component === 'workflow-mcp-host') continue
- // B3(d): same-pass getPodPhase undefined is the 404 we already paid.
- // Do not DELETE again. Unread mcp-host is not in this list (needsMcpHost).
- if (samePassPhase[component] === undefined) continue
+ // B3(d): skip DELETE only when the same-pass GET was a 404. A live
+ // pod with an empty status.phase is present and must still be deleted.
+ const presence = samePassPresence[component]
+ if (presence?.kind === 'absent') {
+ log.info('Skipping runtime component DELETE; same-pass GET was absent', {
+ recipe: recipeName,
+ component,
+ presence: presence.kind,
+ })
+ continue
+ }
await this.deleteRuntimeComponentIfExists(recipeName, component)
}
@@ -2984,7 +3005,7 @@ export class WorkflowReconciler {
runtimeScopeRecipeName,
codexView
)
- const policies = await this.buildWorkflowNetworkPoliciesForSpec(
+ const { policies } = await this.buildWorkflowNetworkPoliciesForSpec(
recipeName,
recipeUid,
spec,
@@ -3184,14 +3205,25 @@ export class WorkflowReconciler {
}
private async pruneLegacyMcpServersInternetEgressPolicy(recipeName: string): Promise<void> {
- if (this.prunedLegacyMcpServersInternetEgress.has(recipeName)) return
- await this.safeDelete(() =>
+ if (this.prunedLegacyMcpServersInternetEgress.has(recipeName)) {
+ this.log.info('Skipping legacy mcp-servers internet NP delete; already observed gone', {
+ recipe: recipeName,
+ })
+ return
+ }
+ const outcome = await observeNamespacedDelete(() =>
this.deps.networkingApi.deleteNamespacedNetworkPolicy({
name: `${recipeName}-mcp-servers-egress-internet`,
namespace: this.deps.config.mcpServerNamespace,
})
)
- this.prunedLegacyMcpServersInternetEgress.add(recipeName)
+ this.log.info('Legacy mcp-servers internet NP delete', {
+ recipe: recipeName,
+ ...deleteOutcomeFields(outcome),
+ })
+ if (shouldRecordDelete(outcome)) {
+ this.prunedLegacyMcpServersInternetEgress.add(recipeName)
+ }
}
private needsRuntimeHttpEgressRefresh(spec: WorkflowRecipeSpec): boolean {
@@ -3226,7 +3258,7 @@ export class WorkflowReconciler {
eligibility: 'ineligible',
reason: 'static_only',
}
- ): Promise<k8s.V1NetworkPolicy[]> {
+ ): Promise<{ policies: k8s.V1NetworkPolicy[]; catalog: Set<string> }> {
const runtimeHttpEgressPolicyNames = this.runtimeHttpEgressPolicyNames(recipeName, spec)
const runtimeHttpEgressState =
runtimeHttpEgressPolicyNames.length > 0 && runtimeHttpEgressClass(spec) === 'exact-host'
@@ -3333,7 +3365,12 @@ export class WorkflowReconciler {
runtimeHttpEgressState.annotations
)
}
- return policies
+ const catalog = buildRunLaneNetworkPolicyCatalog(
+ npConfig,
+ mcpServerFullNames,
+ snippetMcpServerFullNames
+ )
+ return { policies, catalog }
}
/**
@@ -3353,7 +3390,7 @@ export class WorkflowReconciler {
eagerSdkMcpHost = false,
grokProjection?: CodexExecutionProjection & { requiresGrokProxyEgress?: boolean }
): Promise<WorkflowNetworkPolicyApplySummary> {
- const policies = await this.buildWorkflowNetworkPoliciesForSpec(
+ const { policies, catalog } = await this.buildWorkflowNetworkPoliciesForSpec(
recipeName,
recipeUid,
spec,
@@ -3365,7 +3402,7 @@ export class WorkflowReconciler {
)
const summary = await this.applyNetworkPolicyList(policies)
const policyNames = new Set(policies.map(policy => policy.metadata?.name))
- await this.pruneUndesiredRunLaneNetworkPolicies(recipeName, policyNames, {
+ await this.pruneUndesiredRunLaneNetworkPolicies(recipeName, policyNames, catalog, {
skipCodexProxy: codexProjection.eligibility === 'uncertain',
skipGrokProxy: grokProjection?.eligibility === 'uncertain',
})
@@ -3377,12 +3414,14 @@ export class WorkflowReconciler {
* Drop run-lane NetworkPolicies that exist and are no longer desired.
* Two-term selector only (`managed-by=wrc`): a single-term recipe selector
* would also return the recipes-lane policies in the same namespace.
+ * Universe is the factory catalog (GFS never in it), not LIST \ desired.
* Uncertain Codex/Grok eligibility keeps those proxy names even when they
* are absent from this pass's desired set.
*/
private async pruneUndesiredRunLaneNetworkPolicies(
recipeName: string,
desiredNames: Set<string | undefined>,
+ catalog: Set<string>,
uncertain: { skipCodexProxy: boolean; skipGrokProxy: boolean }
): Promise<void> {
const namespace = this.deps.config.sandboxNamespace
@@ -3396,8 +3435,30 @@ export class WorkflowReconciler {
for (const policy of list.items ?? []) {
const name = policy.metadata?.name
if (!name || desiredNames.has(name)) continue
- if (name === codexProxyPolicyName && uncertain.skipCodexProxy) continue
- if (name === grokProxyPolicyName && uncertain.skipGrokProxy) continue
+ if (name === codexProxyPolicyName && uncertain.skipCodexProxy) {
+ this.log.info('Skipping run-lane NP prune', {
+ recipe: recipeName,
+ policy: name,
+ reason: 'uncertain',
+ })
+ continue
+ }
+ if (name === grokProxyPolicyName && uncertain.skipGrokProxy) {
+ this.log.info('Skipping run-lane NP prune', {
+ recipe: recipeName,
+ policy: name,
+ reason: 'uncertain',
+ })
+ continue
+ }
+ if (!catalog.has(name)) {
+ this.log.info('Skipping run-lane NP prune', {
+ recipe: recipeName,
+ policy: name,
+ reason: 'not-in-catalog',
+ })
+ continue
+ }
await this.safeDelete(() =>
this.deps.networkingApi.deleteNamespacedNetworkPolicy({ name, namespace })
)
diff --git a/workflow-recipes/tests/unit/workflow/workflowReconciler.test.ts b/workflow-recipes/tests/unit/workflow/workflowReconciler.test.ts
index 0645c615b..f38a8f772 100644
--- a/workflow-recipes/tests/unit/workflow/workflowReconciler.test.ts
+++ b/workflow-recipes/tests/unit/workflow/workflowReconciler.test.ts
@@ -3028,6 +3028,40 @@ describe('WorkflowReconciler — reconcile loop', () => {
})
})
+ it('G3: does not skip the legacy internet NP after DELETE 403', async () => {
+ const networkingApi = makeNetworkingApi() as ReturnType<typeof makeNetworkingApi>
+ networkingApi.deleteNamespacedNetworkPolicy.mockImplementation(async ({ name, namespace }) => {
+ if (name === 'test-wf-mcp-servers-egress-internet' && namespace === 'mcp-server') {
+ throw { code: 403 }
+ }
+ return {}
+ })
+ const reconciler = new WorkflowReconciler(
+ makeDeps({
+ networkingApi: networkingApi as never,
+ })
+ )
+
+ await reconciler.reconcile(
+ 'test-wf',
+ 'uid-123',
+ 'sandbox-recipes',
+ makeSpec({ mcpServers: ['redis-mcp'] })
+ )
+ await reconciler.reconcile(
+ 'test-wf',
+ 'uid-123',
+ 'sandbox-recipes',
+ makeSpec({ mcpServers: ['redis-mcp'] })
+ )
+
+ const legacyDeletes = networkingApi.deleteNamespacedNetworkPolicy.mock.calls.filter(
+ ([arg]) =>
+ arg.name === 'test-wf-mcp-servers-egress-internet' && arg.namespace === 'mcp-server'
+ )
+ expect(legacyDeletes).toHaveLength(2)
+ })
+
describe('B2 desired-set NetworkPolicy prune', () => {
const RECIPE = 'test-wf'
const SANDBOX = 'sandbox-recipes'
@@ -3091,13 +3125,14 @@ describe('WorkflowReconciler — reconcile loop', () => {
opts: {
codex?: Partial<CodexExecutionProjection>
grok?: Partial<CodexExecutionProjection> & { requiresGrokProxyEgress?: boolean }
+ awaitsTriggeredRun?: boolean
} = {}
): Promise<void> {
const spec = makeSpec()
const runtime = deriveWorkflowRuntimePlan(spec, {
recipeName: RECIPE,
runtimeScopeRecipeName: RECIPE,
- workflowRunId: 'run-1',
+ workflowRunId: opts.awaitsTriggeredRun ? undefined : 'run-1',
})
const codex = ineligibleProjection({
requiresCodexProxyEgress: false,
@@ -3122,7 +3157,16 @@ describe('WorkflowReconciler — reconcile loop', () => {
grokProjection: typeof grok
) => Promise<unknown>
}
- ).applyWorkflowNetworkPolicies(RECIPE, 'uid-123', spec, runtime, false, codex, false, grok)
+ ).applyWorkflowNetworkPolicies(
+ RECIPE,
+ 'uid-123',
+ spec,
+ runtime,
+ opts.awaitsTriggeredRun === true,
+ codex,
+ false,
+ grok
+ )
}
function expectListWitness(
@@ -3137,7 +3181,7 @@ describe('WorkflowReconciler — reconcile loop', () => {
)
}
- it('deletes exactly one listed obsolete policy that is not in the desired set', async () => {
+ it('does not delete a listed name that is outside the run-lane catalog', async () => {
const apiserver = makeApiserverNetworkingApi()
seedLivePolicy(apiserver, `${RECIPE}-obsolete-wrc`, WRC_LABELS)
const reconciler = new WorkflowReconciler(makeDeps({ networkingApi: apiserver.api as never }))
@@ -3145,9 +3189,97 @@ describe('WorkflowReconciler — reconcile loop', () => {
await applyPolicies(reconciler)
expectListWitness(apiserver.api)
- expect(
- sandboxDeletes(apiserver.api).filter(d => d.name === `${RECIPE}-obsolete-wrc`)
- ).toEqual([{ name: `${RECIPE}-obsolete-wrc`, namespace: SANDBOX }])
+ expect(sandboxDeletes(apiserver.api).map(d => d.name)).not.toContain(`${RECIPE}-obsolete-wrc`)
+ expect(apiserver.live.has(apiserver.key(SANDBOX, `${RECIPE}-obsolete-wrc`))).toBe(true)
+ })
+
+ it('G1: reserved coordinator-to-gfs survives a run-lane apply', async () => {
+ const apiserver = makeApiserverNetworkingApi()
+ const gfsName = `${RECIPE}-coordinator-to-gfs`
+ seedLivePolicy(apiserver, gfsName, WRC_LABELS)
+ seedLivePolicy(apiserver, `${RECIPE}-obsolete-wrc`, WRC_LABELS)
+ seedLivePolicy(apiserver, COORD_TO_WRC, WRC_LABELS)
+ seedLivePolicy(apiserver, `${RECIPE}-oauth-broker-egress`, {
+ 'clerum.io/recipe': RECIPE,
+ 'clerum.io/managed-by': 'workflow-recipes',
+ })
+ seedLivePolicy(apiserver, `${RECIPE}-outer-unknown-wrc`, WRC_LABELS)
+ seedLivePolicy(apiserver, 'other-recipe-coordinator-to-gfs', {
+ 'clerum.io/recipe': 'other-recipe',
+ 'clerum.io/managed-by': 'wrc',
+ })
+ seedLivePolicy(apiserver, GROK_PROXY, WRC_LABELS)
+ const reconciler = new WorkflowReconciler(makeDeps({ networkingApi: apiserver.api as never }))
+
+ await applyPolicies(reconciler)
+
+ expectListWitness(apiserver.api)
+ expect(sandboxDeletes(apiserver.api).map(d => d.name)).toEqual([GROK_PROXY])
+ expect(apiserver.live.has(apiserver.key(SANDBOX, gfsName))).toBe(true)
+ expect(apiserver.live.has(apiserver.key(SANDBOX, `${RECIPE}-obsolete-wrc`))).toBe(true)
+ expect(apiserver.live.has(apiserver.key(SANDBOX, COORD_TO_WRC))).toBe(true)
+ expect(apiserver.live.has(apiserver.key(SANDBOX, `${RECIPE}-oauth-broker-egress`))).toBe(true)
+ expect(apiserver.live.has(apiserver.key(SANDBOX, `${RECIPE}-outer-unknown-wrc`))).toBe(true)
+ expect(apiserver.live.has(apiserver.key(SANDBOX, 'other-recipe-coordinator-to-gfs'))).toBe(
+ true
+ )
+ expect(apiserver.live.has(apiserver.key(SANDBOX, GROK_PROXY))).toBe(false)
+ })
+
+ it('deletes a seeded leftover Grok proxy and does not name-delete when unseeded', async () => {
+ const seeded = makeApiserverNetworkingApi()
+ seedLivePolicy(seeded, GROK_PROXY, WRC_LABELS)
+ const seededReconciler = new WorkflowReconciler(
+ makeDeps({ networkingApi: seeded.api as never })
+ )
+ await applyPolicies(seededReconciler)
+ expectListWitness(seeded.api)
+ expect(sandboxDeletes(seeded.api).map(d => d.name)).toEqual([GROK_PROXY])
+
+ const unseeded = makeApiserverNetworkingApi()
+ const unseededReconciler = new WorkflowReconciler(
+ makeDeps({ networkingApi: unseeded.api as never })
+ )
+ await applyPolicies(unseededReconciler)
+ expectListWitness(unseeded.api)
+ expect(sandboxDeletes(unseeded.api).map(d => d.name)).not.toContain(GROK_PROXY)
+ })
+
+ it('deletes a seeded leftover Codex proxy and does not name-delete when unseeded', async () => {
+ const seeded = makeApiserverNetworkingApi()
+ seedLivePolicy(seeded, CODEX_PROXY, WRC_LABELS)
+ const seededReconciler = new WorkflowReconciler(
+ makeDeps({ networkingApi: seeded.api as never })
+ )
+ await applyPolicies(seededReconciler)
+ expect(sandboxDeletes(seeded.api).map(d => d.name)).toEqual([CODEX_PROXY])
+
+ const unseeded = makeApiserverNetworkingApi()
+ const unseededReconciler = new WorkflowReconciler(
+ makeDeps({ networkingApi: unseeded.api as never })
+ )
+ await applyPolicies(unseededReconciler)
+ expect(sandboxDeletes(unseeded.api).map(d => d.name)).not.toContain(CODEX_PROXY)
+ })
+
+ it('c2: awaiting-trigger prunes a run-only leftover that apply(false) would keep', async () => {
+ const leftover = `${RECIPE}-wrc-to-artifact-reader`
+ const awaiting = makeApiserverNetworkingApi()
+ seedLivePolicy(awaiting, leftover, WRC_LABELS)
+ const awaitingReconciler = new WorkflowReconciler(
+ makeDeps({ networkingApi: awaiting.api as never })
+ )
+ await applyPolicies(awaitingReconciler, { awaitsTriggeredRun: true })
+ expect(sandboxDeletes(awaiting.api).map(d => d.name)).toEqual([leftover])
+
+ const running = makeApiserverNetworkingApi()
+ seedLivePolicy(running, leftover, WRC_LABELS)
+ const runningReconciler = new WorkflowReconciler(
+ makeDeps({ networkingApi: running.api as never })
+ )
+ await applyPolicies(runningReconciler, { awaitsTriggeredRun: false })
+ expect(sandboxDeletes(running.api).map(d => d.name)).not.toContain(leftover)
+ expect(running.live.has(running.key(SANDBOX, leftover))).toBe(true)
})
it('does not delete in the sandbox namespace when the LIST is empty', async () => {
@@ -3250,12 +3382,60 @@ describe('WorkflowReconciler — reconcile loop', () => {
return coreApi.deleteNamespacedPod.mock.calls.filter(([arg]) => arg.name === name).length
}
- it('does not delete the coordinator when same-pass getPodPhase is undefined', async () => {
+ it('does not delete the coordinator when same-pass GET is 404', async () => {
const coreApi = makeCoreApi(false)
const reconciler = new WorkflowReconciler(makeDeps({ coreApi: coreApi as never }))
+ const result = await reconciler.reconcile('test-wf', 'uid-123', SANDBOX, makeSpec())
+
+ expect(result.workflowPhase).not.toBe('failed')
+ expect(podReads(coreApi, COORDINATOR)).toBeGreaterThan(0)
+ expect(podDeletes(coreApi, COORDINATOR)).toBe(0)
+ })
+
+ it('G4: deletes the coordinator when GET 200 has an empty status.phase', async () => {
+ const coreApi = makeCoreApi(false)
+ coreApi.readNamespacedPod.mockImplementation(async ({ name }: { name: string }) => {
+ if (name === COORDINATOR) return { metadata: { name }, status: {} }
+ throw { code: 404 }
+ })
+ const reconciler = new WorkflowReconciler(makeDeps({ coreApi: coreApi as never }))
+
await reconciler.reconcile('test-wf', 'uid-123', SANDBOX, makeSpec())
+ expect(podReads(coreApi, COORDINATOR)).toBeGreaterThan(0)
+ expect(podDeletes(coreApi, COORDINATOR)).toBe(1)
+ })
+
+ it('G4: a 403 on coordinator GET does not skip as 404', async () => {
+ const coreApi = makeCoreApi(false)
+ coreApi.readNamespacedPod.mockImplementation(async ({ name }: { name: string }) => {
+ if (name === COORDINATOR) throw { code: 403, message: 'forbidden' }
+ throw { code: 404 }
+ })
+ const reconciler = new WorkflowReconciler(makeDeps({ coreApi: coreApi as never }))
+
+ const result = await reconciler.reconcile('test-wf', 'uid-123', SANDBOX, makeSpec())
+
+ expect(result.workflowPhase).toBe('failed')
+ expect(podDeletes(coreApi, COORDINATOR)).toBe(0)
+ })
+
+ it('c2: does not delete a Running coordinator when workflowRunId is set', async () => {
+ const coreApi = makeCoreApi(true)
+ const reconciler = new WorkflowReconciler(makeDeps({ coreApi: coreApi as never }))
+
+ await reconciler.reconcile(
+ 'test-wf',
+ 'uid-123',
+ SANDBOX,
+ makeSpec(),
+ undefined,
+ undefined,
+ undefined,
+ 'run-1'
+ )
+
expect(podReads(coreApi, COORDINATOR)).toBeGreaterThan(0)
expect(podDeletes(coreApi, COORDINATOR)).toBe(0)
})