Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
fix(workflow-recipes): prune only run-lane catalog leftovers and record deletes after 200/404
Alfredo Lopez7d33f8a435b2
diff --git a/workflow-recipes/package-lock.json b/workflow-recipes/package-lock.jsonindex c6c0eb7d4..9f46506db 100644--- a/workflow-recipes/package-lock.json+++ b/workflow-recipes/package-lock.json@@ -1,12 +1,12 @@{"name": "workload-recipes-controller",- "version": "0.1.406",+ "version": "0.1.407","lockfileVersion": 3,"requires": true,"packages": {"": {"name": "workload-recipes-controller",- "version": "0.1.406",+ "version": "0.1.407","license": "SEE LICENSE IN LICENSE","dependencies": {"@aws-sdk/client-s3": "^3.1009.0",diff --git a/workflow-recipes/package.json b/workflow-recipes/package.jsonindex 1086afb5c..a661ef485 100644--- a/workflow-recipes/package.json+++ b/workflow-recipes/package.json@@ -1,6 +1,6 @@{"name": "workload-recipes-controller",- "version": "0.1.406",+ "version": "0.1.407","description": "Workload Recipes Controller (WRC) - WorkloadRecipe lifecycle management and MCP server interface","main": "dist/main.js","engines": {diff --git a/workflow-recipes/src/k8sClient.test.ts b/workflow-recipes/src/k8sClient.test.tsindex 3a5a2d96c..2028305f6 100644--- a/workflow-recipes/src/k8sClient.test.ts+++ b/workflow-recipes/src/k8sClient.test.ts@@ -3250,3 +3250,13 @@ describe('WorkflowRecipeWatcher wiring — grant-update listener lifecycle (issuexpect(poolEnd).toHaveBeenCalledTimes(1)})})++describe('startSecretWatch oauth ADDED callback', () => {+ it('4th SecretWatcher argument only invalidates the ledger', () => {+ const source = (+ WorkflowRecipeWatcher.prototype as unknown as { startSecretWatch: () => Promise<void> }+ ).startSecretWatch.toString()+ expect(source).toContain('invalidateOAuthBrokerDeleteLedger')+ expect(source).not.toContain('triggerSecretDrivenReconcile(recipeName)')+ })+})diff --git a/workflow-recipes/src/k8sClient.ts b/workflow-recipes/src/k8sClient.tsindex 7a3a6afe9..cbca042b3 100644--- a/workflow-recipes/src/k8sClient.ts+++ b/workflow-recipes/src/k8sClient.ts@@ -911,9 +911,10 @@ export class WorkflowRecipeWatcher implements WorkflowRecipeProvider {this.secretReverseIndex,name => this.triggerSecretDrivenReconcile(name),10_000,+ // Invalidate only. SecretWatcher debounces the recipe enqueue so a+ // reconnect ADDED replay cannot force an immediate undebounced storm.recipeName => {this.reconciler.invalidateOAuthBrokerDeleteLedger(recipeName)- this.triggerSecretDrivenReconcile(recipeName)})const loop = new K8sSecretWatchLoop(this.kc, namespace, handler)diff --git a/workflow-recipes/src/reconciler/deleteOutcome.ts b/workflow-recipes/src/reconciler/deleteOutcome.tsnew file mode 100644index 000000000..a66f500aa--- /dev/null+++ b/workflow-recipes/src/reconciler/deleteOutcome.ts@@ -0,0 +1,54 @@+import { getErrorCode } from './k8sErrors'++/**+ * Observed result of a delete against the apiserver.+ * Record "gone" only after DELETE 200 or DELETE 404 — never after a swallowed 403/5xx.+ */+export type DeleteOutcome =+ | { kind: 'deleted' }+ | { kind: 'gone' }+ | { kind: 'failed'; code?: number; error: unknown }++export async function observeNamespacedDelete(+ deleteFn: () => Promise<unknown>+): Promise<DeleteOutcome> {+ try {+ await deleteFn()+ return { kind: 'deleted' }+ } catch (error: unknown) {+ const code = getErrorCode(error)+ if (code === 404) return { kind: 'gone' }+ return { kind: 'failed', code, error }+ }+}++export function shouldRecordDelete(outcome: DeleteOutcome): boolean {+ switch (outcome.kind) {+ case 'deleted':+ case 'gone':+ return true+ case 'failed':+ return false+ default: {+ const _exhaustive: never = outcome+ return _exhaustive+ }+ }+}++export function deleteOutcomeFields(outcome: DeleteOutcome): {+ outcome: DeleteOutcome['kind']+ code?: number+} {+ switch (outcome.kind) {+ case 'deleted':+ case 'gone':+ return { outcome: outcome.kind }+ case 'failed':+ return { outcome: outcome.kind, code: outcome.code }+ default: {+ const _exhaustive: never = outcome+ return _exhaustive+ }+ }+}diff --git a/workflow-recipes/src/reconciler/reconciler.test.ts b/workflow-recipes/src/reconciler/reconciler.test.tsindex 1f18b266a..938ff2dea 100644--- a/workflow-recipes/src/reconciler/reconciler.test.ts+++ b/workflow-recipes/src/reconciler/reconciler.test.ts@@ -15,7 +15,10 @@ import {WorkloadDef,} from '../types'import { INHERITED_PARENT_RESOURCES_ANNOTATION } from '../workflow/childRecipeFactory'+import type { CodexExecutionProjection } from '../workflow/codexExecutionProjection'import { buildCoordinatorGfsNetworkPolicy } from '../workflow/networkPolicyFactory'+import { deriveWorkflowRuntimePlan } from '../workflow/runtimePlan'+import { WorkflowReconciler } from '../workflow/workflowReconciler'import { captureLogger } from './__tests__/captureLogger'import { defaultFqdnLookup } from './fqdnResolver'import { isRetryableInfraError } from './k8sErrors'@@ -15461,6 +15464,70 @@ describe('WorkflowRecipeReconciler', () => {expect(secretDeletes()).toBe(1)})+ it('G3: does not record a Secret delete after DELETE 403 or 500', async () => {+ const recipe = reapRecipe(4)+ mockCoreApi.deleteNamespacedSecret.mockRejectedValue({ code: 403 })+ await reconciler.ensureOAuthBrokerTokenSecret(recipe)+ await reconciler.ensureOAuthBrokerTokenSecret(recipe)+ expect(secretDeletes()).toBe(2)++ mockCoreApi.deleteNamespacedSecret.mockRejectedValue({ code: 500 })+ mockCoreApi.deleteNamespacedSecret.mockClear()+ await reconciler.ensureOAuthBrokerTokenSecret(recipe)+ await reconciler.ensureOAuthBrokerTokenSecret(recipe)+ expect(secretDeletes()).toBe(2)+ mockCoreApi.deleteNamespacedSecret.mockResolvedValue({})+ })++ it('G3: records a Secret delete after DELETE 404 and skips the next same-generation call', async () => {+ mockCoreApi.deleteNamespacedSecret.mockRejectedValue({ code: 404 })+ const recipe = reapRecipe(7)+ await reconciler.ensureOAuthBrokerTokenSecret(recipe)+ await reconciler.ensureOAuthBrokerTokenSecret(recipe)+ expect(secretDeletes()).toBe(1)+ mockCoreApi.deleteNamespacedSecret.mockResolvedValue({})+ })++ it('G3: does not start the NP TTL after DELETE 403', async () => {+ vi.useFakeTimers()+ vi.setSystemTime(new Date('2026-09-25T00:00:00.000Z'))+ mockNetworkingApi.deleteNamespacedNetworkPolicy.mockRejectedValue({ code: 403 })+ const recipe = reapRecipe(4)+ await reapPolicy(recipe)+ await reapPolicy(recipe)+ expect(policyDeletes()).toBe(2)+ vi.setSystemTime(new Date('2026-09-25T01:00:00.000Z'))+ await reapPolicy(recipe)+ expect(policyDeletes()).toBe(3)+ mockNetworkingApi.deleteNamespacedNetworkPolicy.mockResolvedValue({})+ vi.useRealTimers()+ })++ it('c3: deletes the NetworkPolicy again when metadata.generation changes', async () => {+ await reapPolicy(reapRecipe(4))+ await reapPolicy(reapRecipe(4))+ expect(policyDeletes()).toBe(1)+ await reapPolicy(reapRecipe(5))+ expect(policyDeletes()).toBe(2)+ })++ it('c3: generation 0 on one recipe does not skip another recipe with undefined generation', async () => {+ await reconciler.ensureOAuthBrokerTokenSecret(reapRecipe(0))+ const other = makeRecipe({+ metadata: {+ name: 'other-recipe',+ namespace: 'sandbox-recipes',+ uid: 'uid-other',+ },+ })+ await reconciler.ensureOAuthBrokerTokenSecret(other)+ expect(+ mockCoreApi.deleteNamespacedSecret.mock.calls.filter(+ ([arg]) => arg.name === 'wf-other-recipe-oauth-broker-token'+ )+ ).toHaveLength(1)+ })+it('deletes the NetworkPolicy once for the same generation, then again after the 1h TTL', async () => {vi.useFakeTimers()vi.setSystemTime(new Date('2026-09-25T00:00:00.000Z'))@@ -15483,6 +15550,150 @@ describe('WorkflowRecipeReconciler', () => {})})+ describe('G2 skipStatusPatch must not leave GFS deleted', () => {+ const RECIPE = 'g2-recipe'+ const GFS = `${RECIPE}-coordinator-to-gfs`++ function ineligibleProjection(): CodexExecutionProjection {+ return {+ targets: [],+ eligibleTargets: [],+ derivedScopes: [],+ requiresCodexProxyEgress: false,+ driftHashInput: '',+ catalogContentHash: null,+ catalogRevision: null,+ connectionRevision: null,+ eligibility: 'ineligible',+ reason: 'static_only',+ }+ }++ function installRealInner(): WorkflowReconciler {+ const inner = new WorkflowReconciler({+ coreApi: mockCoreApi,+ customApi: mockCustomApi,+ networkingApi: mockNetworkingApi,+ config: {+ coordinatorImage: 'coordinator:test',+ mcpHostImage: 'mcp-host:test',+ wrcEndpoint: 'http://wrc.example/api',+ sandboxNamespace: 'sandbox-recipes',+ mcpServerNamespace: 'mcp-server',+ imagePullPolicy: 'IfNotPresent',+ maxWorkflowSteps: 100,+ runtimeTokenTtlSeconds: 3600,+ runtimeTokenRefreshBeforeSeconds: 300,+ },+ tokenFactory: {+ signWrcArtifactDeleteToken: vi.fn().mockResolvedValue('t'),+ signCoordinatorToMcpHostToken: vi.fn().mockResolvedValue('t'),+ signCustomCoordinatorToWrcToken: vi.fn().mockResolvedValue('t'),+ signCoordinatorToWrcToken: vi.fn().mockResolvedValue('t'),+ },+ pluginWorkloadSdkRevocationClient: {+ revoke: vi.fn().mockResolvedValue({ state: 'missing', revoked: 0, fencedInvocations: 0 }),+ finalize: vi.fn(),+ },+ } as never)+ ;(reconciler as unknown as { workflowReconciler: WorkflowReconciler }).workflowReconciler =+ inner+ return inner+ }++ it('inner apply does not delete reserved GFS and outer skips ensure on skipStatusPatch', async () => {+ mockNetworkingApi.listNamespacedNetworkPolicy.mockResolvedValue({+ items: [+ {+ metadata: {+ name: GFS,+ namespace: 'sandbox-recipes',+ labels: { 'clerum.io/recipe': RECIPE, 'clerum.io/managed-by': 'wrc' },+ },+ },+ ],+ })+ const inner = installRealInner()+ const apply = (+ inner as unknown as {+ applyWorkflowNetworkPolicies: (...args: unknown[]) => Promise<unknown>+ }+ ).applyWorkflowNetworkPolicies.bind(inner)+ vi.spyOn(inner, 'reconcile').mockImplementation(async (recipeName, recipeUid, _ns, spec) => {+ const runtime = deriveWorkflowRuntimePlan(spec as never, {+ recipeName: String(recipeName),+ runtimeScopeRecipeName: String(recipeName),+ workflowRunId: 'run-g2',+ })+ const projection = ineligibleProjection()+ await apply(recipeName, recipeUid, spec, runtime, false, projection, false, {+ ...projection,+ requiresGrokProxyEgress: false,+ })+ return {+ phase: 'active',+ message: 'transient',+ workflowPhase: 'running',+ skipStatusPatch: true,+ }+ })++ const result = await reconciler.reconcile(+ makeRecipe({+ metadata: {+ name: RECIPE,+ namespace: 'sandbox-recipes',+ uid: 'uid-g2',+ labels: { 'clerum.io/workflow-run-id': 'run-g2' },+ },+ spec: {+ agent: { provider: 'openai', model: 'gpt-4o' },+ steps: [{ id: 'publish', instruction: 'publish' }],+ gfs: { publishTargets: [{ drive: 'main', target: 'out' }] },+ },+ status: { phase: 'pending' },+ })+ )++ expect(result, result.message).toMatchObject({ skipStatusPatch: true })+ expect(mockNetworkingApi.deleteNamespacedNetworkPolicy).not.toHaveBeenCalledWith(+ expect.objectContaining({ name: GFS })+ )+ expect(+ mockNetworkingApi.createNamespacedNetworkPolicy.mock.calls.some(+ ([arg]) => arg.body?.metadata?.name === GFS+ )+ ).toBe(false)+ })++ it('retryRunLaneNetworkPolicies does not prune reserved GFS', async () => {+ mockNetworkingApi.listNamespacedNetworkPolicy.mockResolvedValue({+ items: [+ {+ metadata: {+ name: GFS,+ namespace: 'sandbox-recipes',+ labels: { 'clerum.io/recipe': RECIPE, 'clerum.io/managed-by': 'wrc' },+ },+ },+ ],+ })+ const inner = installRealInner()++ await inner.retryRunLaneNetworkPolicies(+ RECIPE,+ 'uid-g2',+ { steps: [{ id: 'publish', instruction: 'publish' }] },+ RECIPE,+ 'run-g2'+ )++ expect(mockNetworkingApi.deleteNamespacedNetworkPolicy).not.toHaveBeenCalledWith(+ expect.objectContaining({ name: GFS })+ )+ })+ })+// ─── Broker-token issuance failure is non-fatal ─────────────────────────//// A transient control-api blip during reconcile (e.g. it is mid-restart)diff --git a/workflow-recipes/src/reconciler/resourceBuilder.test.ts b/workflow-recipes/src/reconciler/resourceBuilder.test.tsindex 4886756cd..7b48c62a6 100644--- a/workflow-recipes/src/reconciler/resourceBuilder.test.ts+++ b/workflow-recipes/src/reconciler/resourceBuilder.test.ts@@ -25,8 +25,10 @@ import {buildUiEgressNetworkPolicy,oauthBrokerEgressPolicyName,oauthBrokerTokenSecretName,+ parseOAuthBrokerTokenSecretRecipe,recipeHasBackgroundAccessClient,resolveCronJobResourceName,+ resolveOAuthBrokerTokenWatchRecipe,resolveResourceName,resolveScopedCronJobResourceName,resolveScopedResourceName,@@ -2078,6 +2080,36 @@ describe('recipe OAuth broker token', () => {expect(workloadUsesBackgroundOauth(danglingRef, recipe)).toBe(false)})+ it('resolves watch ADDED only for the canonical oauth-broker-token Secret name', () => {+ const recipe = 'test-recipe'+ const canonical = oauthBrokerTokenSecretName(recipe)+ expect(parseOAuthBrokerTokenSecretRecipe(canonical)).toBe(recipe)+ expect(parseOAuthBrokerTokenSecretRecipe('wf--oauth-broker-token')).toBeUndefined()+ expect(+ resolveOAuthBrokerTokenWatchRecipe(canonical, {+ 'clerum.io/component': 'oauth-broker-token',+ 'clerum.io/recipe': recipe,+ })+ ).toBe(recipe)+ expect(+ resolveOAuthBrokerTokenWatchRecipe(canonical, {+ 'clerum.io/component': 'oauth-broker-token',+ })+ ).toBe(recipe)+ expect(+ resolveOAuthBrokerTokenWatchRecipe('forged-name', {+ 'clerum.io/component': 'oauth-broker-token',+ 'clerum.io/recipe': recipe,+ })+ ).toBeUndefined()+ expect(+ resolveOAuthBrokerTokenWatchRecipe(canonical, {+ 'clerum.io/component': 'other',+ 'clerum.io/recipe': recipe,+ })+ ).toBeUndefined()+ })+it('buildOAuthBrokerTokenSecret stores the token base64-encoded under broker-token', () => {const secret = buildOAuthBrokerTokenSecret('test-recipe', 'JWT.VALUE', 'sandbox-recipes')expect(secret.metadata?.name).toBe(oauthBrokerTokenSecretName('test-recipe'))diff --git a/workflow-recipes/src/reconciler/resourceBuilder.ts b/workflow-recipes/src/reconciler/resourceBuilder.tsindex 7b7dc447b..caadd50ad 100644--- a/workflow-recipes/src/reconciler/resourceBuilder.ts+++ b/workflow-recipes/src/reconciler/resourceBuilder.ts@@ -435,6 +435,41 @@ export function oauthBrokerTokenSecretName(recipeName: string): string {return `wf-${recipeName}-oauth-broker-token`}+const OAUTH_BROKER_TOKEN_SECRET_PREFIX = 'wf-'+const OAUTH_BROKER_TOKEN_SECRET_SUFFIX = '-oauth-broker-token'++/** Parse `wf-<recipe>-oauth-broker-token`. Rejects an empty recipe segment. */+export function parseOAuthBrokerTokenSecretRecipe(secretName: string): string | undefined {+ if (+ !secretName.startsWith(OAUTH_BROKER_TOKEN_SECRET_PREFIX) ||+ !secretName.endsWith(OAUTH_BROKER_TOKEN_SECRET_SUFFIX)+ ) {+ return undefined+ }+ const recipe = secretName.slice(+ OAUTH_BROKER_TOKEN_SECRET_PREFIX.length,+ secretName.length - OAUTH_BROKER_TOKEN_SECRET_SUFFIX.length+ )+ return recipe.length > 0 ? recipe : undefined+}++/**+ * Watch-bind for B3(a) invalidation. Requires `clerum.io/component=oauth-broker-token`+ * and `metadata.name === wf-${recipe}-oauth-broker-token`. A forged recipe label+ * on a different Secret name must not invalidate.+ */+export function resolveOAuthBrokerTokenWatchRecipe(+ secretName: string,+ labels: Record<string, string> | undefined+): string | undefined {+ if (labels?.['clerum.io/component'] !== 'oauth-broker-token') return undefined+ const labeled = labels['clerum.io/recipe']+ if (labeled) {+ return secretName === oauthBrokerTokenSecretName(labeled) ? labeled : undefined+ }+ return parseOAuthBrokerTokenSecretRecipe(secretName)+}+export function recipeHasBackgroundAccessClient(recipe: WorkflowRecipeCRD): boolean {return (recipe.spec.oauthClients ?? []).some(c => c.backgroundAccess === true)}diff --git a/workflow-recipes/src/reconciler/secretWatcher.test.ts b/workflow-recipes/src/reconciler/secretWatcher.test.tsindex 2a9ff5425..6e0bd28f2 100644--- a/workflow-recipes/src/reconciler/secretWatcher.test.ts+++ b/workflow-recipes/src/reconciler/secretWatcher.test.ts@@ -244,6 +244,49 @@ describe('SecretWatcher', () => {watcher.handleEvent('ADDED', brokerSecret)watcher.handleEvent('ADDED', brokerSecret)expect(invalidated).toEqual(['test-recipe', 'test-recipe'])+ expect(enqueued).toEqual([])+ vi.advanceTimersByTime(DEBOUNCE_MS)+ expect(enqueued).toEqual(['test-recipe'])+ watcher.stop()+ })++ it('does not invalidate on forged name, wrong component, or MODIFIED', () => {+ const reverseIndex = new SecretReverseIndex()+ const enqueued: string[] = []+ const invalidated: string[] = []+ const watcher = new SecretWatcher(+ reverseIndex,+ name => {+ enqueued.push(name)+ },+ DEBOUNCE_MS,+ recipeName => {+ invalidated.push(recipeName)+ }+ )+ const labels = {+ 'clerum.io/component': 'oauth-broker-token',+ 'clerum.io/recipe': 'test-recipe',+ }++ watcher.handleEvent('ADDED', {+ metadata: { name: 'forged-oauth-broker-token', labels },+ data: { 'broker-token': 'eA==' },+ })+ watcher.handleEvent('ADDED', {+ metadata: {+ name: 'wf-test-recipe-oauth-broker-token',+ labels: { 'clerum.io/component': 'other', 'clerum.io/recipe': 'test-recipe' },+ },+ data: { 'broker-token': 'eA==' },+ })+ watcher.handleEvent('MODIFIED', {+ metadata: { name: 'wf-test-recipe-oauth-broker-token', labels },+ data: { 'broker-token': 'eA==' },+ })+ expect(invalidated).toEqual([])+ vi.advanceTimersByTime(DEBOUNCE_MS)+ expect(enqueued).toEqual([])watcher.stop()})})diff --git a/workflow-recipes/src/reconciler/secretWatcher.ts b/workflow-recipes/src/reconciler/secretWatcher.tsindex deb026968..e4518d634 100644--- a/workflow-recipes/src/reconciler/secretWatcher.ts+++ b/workflow-recipes/src/reconciler/secretWatcher.ts@@ -1,3 +1,4 @@+import { resolveOAuthBrokerTokenWatchRecipe } from './resourceBuilder'import { OWNER_RECIPE_LABEL_KEY, SHARED_LABEL_KEY } from './secretOwnership'import { SecretReverseIndex } from './secretReverseIndex'@@ -59,13 +60,15 @@ export class SecretWatcher {const name = secret.metadata?.nameif (!name) return- // B3(a): ADDED of the broker token must invalidate the ledger even when- // the key-set matches a previous observation (dedup would otherwise drop it).+ // B3(a): ADDED of the canonical broker token invalidates the ledger even+ // when the key-set matches a previous observation (dedup would drop it).+ // Enqueue the recipe through the same debounce as key-set fan-out so a+ // reconnect replay cannot force an immediate undebounced storm.if (type === 'ADDED') {- const labels = secret.metadata?.labels ?? {}- if (labels['clerum.io/component'] === 'oauth-broker-token') {- const recipeName = labels['clerum.io/recipe']- if (recipeName) this.onOAuthBrokerTokenAdded?.(recipeName)+ const recipeName = resolveOAuthBrokerTokenWatchRecipe(name, secret.metadata?.labels)+ if (recipeName) {+ this.onOAuthBrokerTokenAdded?.(recipeName)+ this.scheduleRecipeReconcile(recipeName)}}@@ -130,6 +133,19 @@ export class SecretWatcher {}, this.debounceMs)this.debounceTimers.set(secretName, timer)}++ /** Debounce enqueue by recipe name (oauth-broker ADDED is not reverse-indexed). */+ private scheduleRecipeReconcile(recipeName: string): void {+ const key = `oauth-broker:${recipeName}`+ const existing = this.debounceTimers.get(key)+ if (existing) clearTimeout(existing)++ const timer = setTimeout(() => {+ this.debounceTimers.delete(key)+ this.enqueueReconcile(recipeName)+ }, this.debounceMs)+ this.debounceTimers.set(key, timer)+ }}function setsEqual(a: ReadonlySet<string>, b: ReadonlySet<string>): boolean {diff --git a/workflow-recipes/src/reconciler/workflowRecipeReconciler.ts b/workflow-recipes/src/reconciler/workflowRecipeReconciler.tsindex ff367c206..61bca6a7e 100644--- a/workflow-recipes/src/reconciler/workflowRecipeReconciler.ts+++ b/workflow-recipes/src/reconciler/workflowRecipeReconciler.ts@@ -74,6 +74,7 @@ import {} from '../workflow/workflowReconciler'import { evaluateComputedValues } from './computedValuesEvaluator'import { CRD_GROUP, CRD_VERSION, WORKFLOWRECIPE_PLURAL } from './crdConstants'+import { deleteOutcomeFields, observeNamespacedDelete, shouldRecordDelete } from './deleteOutcome'import { sort as sortDependencies } from './dependencyGraph'import {type AccumulateOutput,@@ -7514,11 +7515,27 @@ export class WorkflowRecipeReconciler {)return}- await this.safeDelete(- () => this.coreApi.deleteNamespacedSecret({ name: secretName, namespace: ns }),- `Secret "${secretName}" in ${ns}`+ const outcome = await observeNamespacedDelete(() =>+ this.coreApi.deleteNamespacedSecret({ name: secretName, namespace: ns }))- this.oauthBrokerDeleteLedger.recordSecretDelete(recipeName, generation)+ const log = createLogger('wrc', recipeName)+ if (outcome.kind === 'failed') {+ log.error('Failed to delete oauth-broker-token Secret', {+ secretName,+ ns,+ ...deleteOutcomeFields(outcome),+ err: outcome.error,+ })+ } else {+ log.info('oauth-broker-token Secret delete', {+ secretName,+ ns,+ ...deleteOutcomeFields(outcome),+ })+ }+ if (shouldRecordDelete(outcome)) {+ this.oauthBrokerDeleteLedger.recordSecretDelete(recipeName, generation)+ }return}@@ -7590,11 +7607,27 @@ export class WorkflowRecipeReconciler {)return}- await this.safeDelete(- () => this.networkingApi.deleteNamespacedNetworkPolicy({ name: policyName, namespace: ns }),- `NetworkPolicy "${policyName}" in ${ns}`+ const outcome = await observeNamespacedDelete(() =>+ this.networkingApi.deleteNamespacedNetworkPolicy({ name: policyName, namespace: ns }))- this.oauthBrokerDeleteLedger.recordPolicyDelete(recipeName, generation)+ const log = createLogger('wrc', recipeName)+ if (outcome.kind === 'failed') {+ log.error('Failed to delete oauth-broker-egress NetworkPolicy', {+ policyName,+ ns,+ ...deleteOutcomeFields(outcome),+ err: outcome.error,+ })+ } else {+ log.info('oauth-broker-egress NetworkPolicy delete', {+ policyName,+ ns,+ ...deleteOutcomeFields(outcome),+ })+ }+ if (shouldRecordDelete(outcome)) {+ this.oauthBrokerDeleteLedger.recordPolicyDelete(recipeName, generation)+ }return}await this.applyNetworkPolicy(policy, ns, { family: 'oauth-broker-egress' })diff --git a/workflow-recipes/src/workflow/crashRecovery.test.ts b/workflow-recipes/src/workflow/crashRecovery.test.tsindex 7a0816e87..b01ca851a 100644--- a/workflow-recipes/src/workflow/crashRecovery.test.ts+++ b/workflow-recipes/src/workflow/crashRecovery.test.ts@@ -2,6 +2,7 @@ import { describe, expect, it, vi } from 'vitest'import {evaluateCompletedRuntimePodRecovery,evaluateCrashRecovery,+ getPodPresence,inspectPodReadiness,waitForPodDeletion,} from './crashRecovery'@@ -111,6 +112,28 @@ describe('crash recovery pod readiness classification', () => {})})+describe('getPodPresence', () => {+ it('returns absent on GET 404 and present on GET 200 with empty status', async () => {+ const readNamespacedPod = vi+ .fn()+ .mockRejectedValueOnce({ code: 404 })+ .mockResolvedValueOnce({ metadata: { name: 'recipe-coordinator' }, status: {} })+ const coreApi = { readNamespacedPod } as unknown as Parameters<typeof getPodPresence>[0]++ await expect(getPodPresence(coreApi, 'recipe-coordinator', 'sandbox-recipes')).resolves.toEqual(+ {+ kind: 'absent',+ }+ )+ await expect(getPodPresence(coreApi, 'recipe-coordinator', 'sandbox-recipes')).resolves.toEqual(+ {+ kind: 'present',+ phase: undefined,+ }+ )+ })+})+describe('waitForPodDeletion', () => {it('returns true once getPodPhase reports the pod is gone', async () => {const readNamespacedPod = vidiff --git a/workflow-recipes/src/workflow/crashRecovery.ts b/workflow-recipes/src/workflow/crashRecovery.tsindex 2b5aabce3..10cbdbdac 100644--- a/workflow-recipes/src/workflow/crashRecovery.ts+++ b/workflow-recipes/src/workflow/crashRecovery.ts@@ -148,26 +148,56 @@ export async function waitForPodDeletion(const pollIntervalMs = options.pollIntervalMs ?? POD_DELETE_POLL_MSconst deadline = Date.now() + timeoutMswhile (Date.now() < deadline) {- if ((await getPodPhase(coreApi, name, namespace)) === undefined) return true+ if ((await getPodPresence(coreApi, name, namespace)).kind === 'absent') return trueawait new Promise(resolve => setTimeout(resolve, pollIntervalMs))}- return (await getPodPhase(coreApi, name, namespace)) === undefined+ return (await getPodPresence(coreApi, name, namespace)).kind === 'absent'}-export async function getPodPhase(+/**+ * Discriminated GET result. `unread` is only for callers that skipped the GET;+ * `getPodPresence` never returns it. Skip DELETE only on `absent` — a live+ * object with an empty `status.phase` is `present`.+ */+export type PodPresence =+ | { kind: 'absent' }+ | { kind: 'present'; phase?: string }+ | { kind: 'unread' }++export type ObservedPodPresence = Exclude<PodPresence, { kind: 'unread' }>++export async function getPodPresence(coreApi: k8s.CoreV1Api,name: string,namespace: string-): Promise<string | undefined> {+): Promise<ObservedPodPresence> {try {const pod = await coreApi.readNamespacedPod({ name, namespace })- return pod.status?.phase+ return { kind: 'present', phase: pod.status?.phase }} catch (error: unknown) {- if (getErrorCode(error) === 404) return undefined+ if (getErrorCode(error) === 404) return { kind: 'absent' }throw error}}+export async function getPodPhase(+ coreApi: k8s.CoreV1Api,+ name: string,+ namespace: string+): Promise<string | undefined> {+ const presence = await getPodPresence(coreApi, name, namespace)+ switch (presence.kind) {+ case 'absent':+ return undefined+ case 'present':+ return presence.phase+ default: {+ const _exhaustive: never = presence+ return _exhaustive+ }+ }+}+export async function getPodReadiness(coreApi: k8s.CoreV1Api,name: string,diff --git a/workflow-recipes/src/workflow/networkPolicyFactory.gfs.test.ts b/workflow-recipes/src/workflow/networkPolicyFactory.gfs.test.tsindex 17d2ed201..85812030c 100644--- a/workflow-recipes/src/workflow/networkPolicyFactory.gfs.test.ts+++ b/workflow-recipes/src/workflow/networkPolicyFactory.gfs.test.ts@@ -2,6 +2,7 @@ import { describe, expect, it } from 'vitest'import {type NetworkPolicyConfig,buildCoordinatorGfsNetworkPolicy,+ buildRunLaneNetworkPolicyCatalog,buildWorkflowNetworkPolicies,} from './networkPolicyFactory'@@ -154,4 +155,15 @@ describe('workflow GFS NetworkPolicy', () => {},})})++ it('run-lane catalog includes factory leftovers and never coordinator-to-gfs', () => {+ const catalog = buildRunLaneNetworkPolicyCatalog({+ ...baseConfig,+ includeCoordinatorGfs: true,+ })+ expect(catalog.has('daily-report-mcp-host-to-grok-proxy')).toBe(true)+ expect(catalog.has('daily-report-mcp-host-to-codex-proxy')).toBe(true)+ expect(catalog.has('daily-report-wrc-to-artifact-reader')).toBe(true)+ expect(catalog.has('daily-report-coordinator-to-gfs')).toBe(false)+ })})diff --git a/workflow-recipes/src/workflow/networkPolicyFactory.ts b/workflow-recipes/src/workflow/networkPolicyFactory.tsindex 8e3895eda..87f916806 100644--- a/workflow-recipes/src/workflow/networkPolicyFactory.ts+++ b/workflow-recipes/src/workflow/networkPolicyFactory.ts@@ -1068,3 +1068,35 @@ export function buildWorkflowNetworkPolicies(...pluginWorkloadSdkPolicies,])}++const RUN_LANE_CATALOG_PLACEHOLDER_SERVER = '_catalog'++/**+ * Names the run-lane factory can emit for this recipe. Outer-owned+ * `${recipe}-coordinator-to-gfs` is never included (`includeCoordinatorGfs: false`).+ * Catalog membership — not a GFS-name keep-list — is the prune universe.+ */+export function buildRunLaneNetworkPolicyCatalog(+ applyConfig: NetworkPolicyConfig,+ mcpServerNames: string[] = [],+ snippetMcpServerNames: string[] = []+): Set<string> {+ const catalogConfig: NetworkPolicyConfig = {+ ...applyConfig,+ includeCoordinator: true,+ includeMcpHost: true,+ includeCodexProxyEgress: true,+ includeGrokProxyEgress: true,+ includeCoordinatorGfs: false,+ includeArtifactReader: true,+ includeSnippetRunner: true,+ pluginWorkloadSdkSandboxAccess: true,+ }+ const serverNames =+ mcpServerNames.length > 0 ? mcpServerNames : [RUN_LANE_CATALOG_PLACEHOLDER_SERVER]+ return new Set(+ buildWorkflowNetworkPolicies(catalogConfig, serverNames, snippetMcpServerNames)+ .map(policy => policy.metadata?.name)+ .filter((name): name is string => Boolean(name))+ )+}diff --git a/workflow-recipes/src/workflow/workflowReconciler.codexScopeProvenance.test.ts b/workflow-recipes/src/workflow/workflowReconciler.codexScopeProvenance.test.tsindex 8083ea7f4..7e77b9f52 100644--- a/workflow-recipes/src/workflow/workflowReconciler.codexScopeProvenance.test.ts+++ b/workflow-recipes/src/workflow/workflowReconciler.codexScopeProvenance.test.ts@@ -24,6 +24,10 @@ const crashRecoveryMocks = vi.hoisted(() => ({evaluateCrashRecovery: vi.fn().mockReturnValue({ action: 'none', message: 'Pod is healthy' }),getContainerWaitingReason: vi.fn().mockResolvedValue(undefined),getPodPhase: vi.fn().mockResolvedValue(undefined),+ getPodPresence: vi.fn(async (api: unknown, name: string, namespace: string) => {+ const phase = await crashRecoveryMocks.getPodPhase(api, name, namespace)+ return phase === undefined ? { kind: 'absent' as const } : { kind: 'present' as const, phase }+ }),getPodReadiness: vi.fn().mockResolvedValue({ ready: true, phase: 'Running', uid: 'pod-uid-1' }),isRecoverableContainerWaitingReason: vi.fn(() => false),}))diff --git a/workflow-recipes/src/workflow/workflowReconciler.codexUncertainScope.test.ts b/workflow-recipes/src/workflow/workflowReconciler.codexUncertainScope.test.tsindex 8949f7c36..9ea5cacbb 100644--- a/workflow-recipes/src/workflow/workflowReconciler.codexUncertainScope.test.ts+++ b/workflow-recipes/src/workflow/workflowReconciler.codexUncertainScope.test.ts@@ -16,6 +16,10 @@ const crashRecoveryMocks = vi.hoisted(() => ({evaluateCrashRecovery: vi.fn().mockReturnValue({ action: 'none', message: 'Pod is healthy' }),getContainerWaitingReason: vi.fn().mockResolvedValue(undefined),getPodPhase: vi.fn().mockResolvedValue(undefined),+ getPodPresence: vi.fn(async (api: unknown, name: string, namespace: string) => {+ const phase = await crashRecoveryMocks.getPodPhase(api, name, namespace)+ return phase === undefined ? { kind: 'absent' as const } : { kind: 'present' as const, phase }+ }),getPodReadiness: vi.fn().mockResolvedValue({ ready: true, phase: 'Running', uid: 'pod-uid-1' }),isRecoverableContainerWaitingReason: vi.fn(() => false),}))diff --git a/workflow-recipes/src/workflow/workflowReconciler.grokWrcFlag.test.ts b/workflow-recipes/src/workflow/workflowReconciler.grokWrcFlag.test.tsindex c5535ed92..740674edf 100644--- a/workflow-recipes/src/workflow/workflowReconciler.grokWrcFlag.test.ts+++ b/workflow-recipes/src/workflow/workflowReconciler.grokWrcFlag.test.ts@@ -23,6 +23,10 @@ const crashRecoveryMocks = vi.hoisted(() => ({evaluateCrashRecovery: vi.fn().mockReturnValue({ action: 'none', message: 'Pod is healthy' }),getContainerWaitingReason: vi.fn().mockResolvedValue(undefined),getPodPhase: vi.fn().mockResolvedValue(undefined),+ getPodPresence: vi.fn(async (api: unknown, name: string, namespace: string) => {+ const phase = await crashRecoveryMocks.getPodPhase(api, name, namespace)+ return phase === undefined ? { kind: 'absent' as const } : { kind: 'present' as const, phase }+ }),getPodReadiness: vi.fn().mockResolvedValue({ ready: true, phase: 'Running', uid: 'pod-uid-1' }),isRecoverableContainerWaitingReason: vi.fn(() => false),}))@@ -310,6 +314,30 @@ describe('WorkflowReconciler Grok WRC switch', () => {expect(env).not.toContain('GROK_LLM_PROXY_RUNTIME_URL')})+ it(`deletes a seeded leftover Grok proxy when the WRC flag is off for a ${shape} recipe`, async () => {+ const { reconciler, networkingApi } = createHarness(false)+ bindGrokGrant(reconciler)+ networkingApi.listNamespacedNetworkPolicy.mockResolvedValue({+ items: [+ {+ metadata: {+ name: GROK_PROXY_POLICY,+ namespace: sandboxNamespace,+ labels: { 'clerum.io/recipe': RECIPE, 'clerum.io/managed-by': 'wrc' },+ },+ },+ ],+ })++ await reconcileRecipe(reconciler, spec())++ expect(createdPolicyNames(networkingApi)).not.toContain(GROK_PROXY_POLICY)+ expect(networkingApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledWith({+ name: GROK_PROXY_POLICY,+ namespace: sandboxNamespace,+ })+ })+it(`keeps Grok scope, proxy egress and pod env for a ${shape} recipe when the WRC flag is on`, async () => {const { reconciler, coreApi, networkingApi } = createHarness(true)bindGrokGrant(reconciler)diff --git a/workflow-recipes/src/workflow/workflowReconciler.test.ts b/workflow-recipes/src/workflow/workflowReconciler.test.tsindex c64c120dd..533360030 100644--- a/workflow-recipes/src/workflow/workflowReconciler.test.ts+++ b/workflow-recipes/src/workflow/workflowReconciler.test.ts@@ -16,6 +16,10 @@ const crashRecoveryMocks = vi.hoisted(() => ({evaluateCrashRecovery: vi.fn().mockReturnValue({ action: 'none', message: 'Pod is healthy' }),getContainerWaitingReason: vi.fn().mockResolvedValue(undefined),getPodPhase: vi.fn().mockResolvedValue(undefined),+ getPodPresence: vi.fn(async (api: unknown, name: string, namespace: string) => {+ const phase = await crashRecoveryMocks.getPodPhase(api, name, namespace)+ return phase === undefined ? { kind: 'absent' as const } : { kind: 'present' as const, phase }+ }),getPodReadiness: vi.fn().mockResolvedValue({ ready: true, phase: 'Running', uid: 'pod-uid-1' }),isRecoverableContainerWaitingReason: vi.fn((phase: string | undefined, reason: string | undefined) =>diff --git a/workflow-recipes/src/workflow/workflowReconciler.ts b/workflow-recipes/src/workflow/workflowReconciler.tsindex e3f52de39..15a07ec94 100644--- a/workflow-recipes/src/workflow/workflowReconciler.ts+++ b/workflow-recipes/src/workflow/workflowReconciler.ts@@ -25,6 +25,11 @@ import {import { mintRecipeHostGfsToken } from '../gfsBinding'import { createLogger } from '../observability/logger'import { CRD_GROUP, CRD_VERSION, WORKFLOWRECIPE_PLURAL } from '../reconciler/crdConstants'+import {+ deleteOutcomeFields,+ observeNamespacedDelete,+ shouldRecordDelete,+} from '../reconciler/deleteOutcome'import {ResourceVanishedAfterConflictError,RetryableReconcileError,@@ -64,12 +69,14 @@ import {projectCodexRecipeVerdict,} from './codexRecipeVerdict'import {+ type PodPresence,type PodReadiness,deletePodIfExists,evaluateCompletedRuntimePodRecovery,evaluateCrashRecovery,getContainerWaitingReason,getPodPhase,+ getPodPresence,getPodReadiness,isRecoverableContainerWaitingReason,waitForPodDeletion,@@ -89,7 +96,11 @@ import {issueMcpHostWorkflowControlToken,} from './mcpHostRuntimeTokenIssuerClient'import type { ModelConfigHandler } from './modelConfigHandler'-import { NetworkPolicyConfig, buildWorkflowNetworkPolicies } from './networkPolicyFactory'+import {+ NetworkPolicyConfig,+ buildRunLaneNetworkPolicyCatalog,+ buildWorkflowNetworkPolicies,+} from './networkPolicyFactory'import { ObjectStorageClient, StorageCredentials, StorageRef } from './objectStorageClient'import {type EagerSdkBootstrapProof,@@ -1778,11 +1789,12 @@ export class WorkflowReconciler {try {// Check crash recovery for existing Pods- let coordPhase = await getPodPhase(+ const coordPresence = await getPodPresence(this.deps.coreApi,`${recipeName}-coordinator`,this.deps.config.sandboxNamespace)+ let coordPhase = coordPresence.kind === 'present' ? coordPresence.phase : undefinedconst coordWaitingReason = await getContainerWaitingReason(this.deps.coreApi,`${recipeName}-coordinator`,@@ -1802,13 +1814,14 @@ export class WorkflowReconciler {this.deps.config.sandboxNamespace): undefined- let mcpHostPhase = needsMcpHost- ? await getPodPhase(+ const mcpHostPresence: PodPresence = needsMcpHost+ ? await getPodPresence(this.deps.coreApi,`${recipeName}-mcp-host`,this.deps.config.sandboxNamespace)- : undefined+ : { kind: 'unread' }+ let mcpHostPhase = mcpHostPresence.kind === 'present' ? mcpHostPresence.phase : undefinedlet artifactReaderPhase = needsArtifactReader? await getPodPhase(this.deps.coreApi,@@ -1860,18 +1873,26 @@ export class WorkflowReconciler {}if (awaitsTriggeredRun) {- const samePassPhase: Partial<Record<WorkflowRuntimeComponent, string | undefined>> = {- 'workflow-coordinator': coordPhase,- 'workflow-mcp-host': mcpHostPhase,+ const samePassPresence: Partial<Record<WorkflowRuntimeComponent, PodPresence>> = {+ 'workflow-coordinator': coordPresence,+ ...(mcpHostPresence.kind !== 'unread' ? { 'workflow-mcp-host': mcpHostPresence } : {}),}for (const component of runtime.cleanup.deleteBeforeTriggeredRun) {// Plugin Workload SDK: keep the mcp-host alive across the// awaiting-trigger window — it hosts the always-on SDK server.// Only the coordinator (and run-scoped pods) are torn down here.if (eagerMcpHostForSdk && component === 'workflow-mcp-host') continue- // B3(d): same-pass getPodPhase undefined is the 404 we already paid.- // Do not DELETE again. Unread mcp-host is not in this list (needsMcpHost).- if (samePassPhase[component] === undefined) continue+ // B3(d): skip DELETE only when the same-pass GET was a 404. A live+ // pod with an empty status.phase is present and must still be deleted.+ const presence = samePassPresence[component]+ if (presence?.kind === 'absent') {+ log.info('Skipping runtime component DELETE; same-pass GET was absent', {+ recipe: recipeName,+ component,+ presence: presence.kind,+ })+ continue+ }await this.deleteRuntimeComponentIfExists(recipeName, component)}@@ -2984,7 +3005,7 @@ export class WorkflowReconciler {runtimeScopeRecipeName,codexView)- const policies = await this.buildWorkflowNetworkPoliciesForSpec(+ const { policies } = await this.buildWorkflowNetworkPoliciesForSpec(recipeName,recipeUid,spec,@@ -3184,14 +3205,25 @@ export class WorkflowReconciler {}private async pruneLegacyMcpServersInternetEgressPolicy(recipeName: string): Promise<void> {- if (this.prunedLegacyMcpServersInternetEgress.has(recipeName)) return- await this.safeDelete(() =>+ if (this.prunedLegacyMcpServersInternetEgress.has(recipeName)) {+ this.log.info('Skipping legacy mcp-servers internet NP delete; already observed gone', {+ recipe: recipeName,+ })+ return+ }+ const outcome = await observeNamespacedDelete(() =>this.deps.networkingApi.deleteNamespacedNetworkPolicy({name: `${recipeName}-mcp-servers-egress-internet`,namespace: this.deps.config.mcpServerNamespace,}))- this.prunedLegacyMcpServersInternetEgress.add(recipeName)+ this.log.info('Legacy mcp-servers internet NP delete', {+ recipe: recipeName,+ ...deleteOutcomeFields(outcome),+ })+ if (shouldRecordDelete(outcome)) {+ this.prunedLegacyMcpServersInternetEgress.add(recipeName)+ }}private needsRuntimeHttpEgressRefresh(spec: WorkflowRecipeSpec): boolean {@@ -3226,7 +3258,7 @@ export class WorkflowReconciler {eligibility: 'ineligible',reason: 'static_only',}- ): Promise<k8s.V1NetworkPolicy[]> {+ ): Promise<{ policies: k8s.V1NetworkPolicy[]; catalog: Set<string> }> {const runtimeHttpEgressPolicyNames = this.runtimeHttpEgressPolicyNames(recipeName, spec)const runtimeHttpEgressState =runtimeHttpEgressPolicyNames.length > 0 && runtimeHttpEgressClass(spec) === 'exact-host'@@ -3333,7 +3365,12 @@ export class WorkflowReconciler {runtimeHttpEgressState.annotations)}- return policies+ const catalog = buildRunLaneNetworkPolicyCatalog(+ npConfig,+ mcpServerFullNames,+ snippetMcpServerFullNames+ )+ return { policies, catalog }}/**@@ -3353,7 +3390,7 @@ export class WorkflowReconciler {eagerSdkMcpHost = false,grokProjection?: CodexExecutionProjection & { requiresGrokProxyEgress?: boolean }): Promise<WorkflowNetworkPolicyApplySummary> {- const policies = await this.buildWorkflowNetworkPoliciesForSpec(+ const { policies, catalog } = await this.buildWorkflowNetworkPoliciesForSpec(recipeName,recipeUid,spec,@@ -3365,7 +3402,7 @@ export class WorkflowReconciler {)const summary = await this.applyNetworkPolicyList(policies)const policyNames = new Set(policies.map(policy => policy.metadata?.name))- await this.pruneUndesiredRunLaneNetworkPolicies(recipeName, policyNames, {+ await this.pruneUndesiredRunLaneNetworkPolicies(recipeName, policyNames, catalog, {skipCodexProxy: codexProjection.eligibility === 'uncertain',skipGrokProxy: grokProjection?.eligibility === 'uncertain',})@@ -3377,12 +3414,14 @@ export class WorkflowReconciler {* Drop run-lane NetworkPolicies that exist and are no longer desired.* Two-term selector only (`managed-by=wrc`): a single-term recipe selector* would also return the recipes-lane policies in the same namespace.+ * Universe is the factory catalog (GFS never in it), not LIST \ desired.* Uncertain Codex/Grok eligibility keeps those proxy names even when they* are absent from this pass's desired set.*/private async pruneUndesiredRunLaneNetworkPolicies(recipeName: string,desiredNames: Set<string | undefined>,+ catalog: Set<string>,uncertain: { skipCodexProxy: boolean; skipGrokProxy: boolean }): Promise<void> {const namespace = this.deps.config.sandboxNamespace@@ -3396,8 +3435,30 @@ export class WorkflowReconciler {for (const policy of list.items ?? []) {const name = policy.metadata?.nameif (!name || desiredNames.has(name)) continue- if (name === codexProxyPolicyName && uncertain.skipCodexProxy) continue- if (name === grokProxyPolicyName && uncertain.skipGrokProxy) continue+ if (name === codexProxyPolicyName && uncertain.skipCodexProxy) {+ this.log.info('Skipping run-lane NP prune', {+ recipe: recipeName,+ policy: name,+ reason: 'uncertain',+ })+ continue+ }+ if (name === grokProxyPolicyName && uncertain.skipGrokProxy) {+ this.log.info('Skipping run-lane NP prune', {+ recipe: recipeName,+ policy: name,+ reason: 'uncertain',+ })+ continue+ }+ if (!catalog.has(name)) {+ this.log.info('Skipping run-lane NP prune', {+ recipe: recipeName,+ policy: name,+ reason: 'not-in-catalog',+ })+ continue+ }await this.safeDelete(() =>this.deps.networkingApi.deleteNamespacedNetworkPolicy({ name, namespace }))diff --git a/workflow-recipes/tests/unit/workflow/workflowReconciler.test.ts b/workflow-recipes/tests/unit/workflow/workflowReconciler.test.tsindex 0645c615b..f38a8f772 100644--- a/workflow-recipes/tests/unit/workflow/workflowReconciler.test.ts+++ b/workflow-recipes/tests/unit/workflow/workflowReconciler.test.ts@@ -3028,6 +3028,40 @@ describe('WorkflowReconciler — reconcile loop', () => {})})+ it('G3: does not skip the legacy internet NP after DELETE 403', async () => {+ const networkingApi = makeNetworkingApi() as ReturnType<typeof makeNetworkingApi>+ networkingApi.deleteNamespacedNetworkPolicy.mockImplementation(async ({ name, namespace }) => {+ if (name === 'test-wf-mcp-servers-egress-internet' && namespace === 'mcp-server') {+ throw { code: 403 }+ }+ return {}+ })+ const reconciler = new WorkflowReconciler(+ makeDeps({+ networkingApi: networkingApi as never,+ })+ )++ await reconciler.reconcile(+ 'test-wf',+ 'uid-123',+ 'sandbox-recipes',+ makeSpec({ mcpServers: ['redis-mcp'] })+ )+ await reconciler.reconcile(+ 'test-wf',+ 'uid-123',+ 'sandbox-recipes',+ makeSpec({ mcpServers: ['redis-mcp'] })+ )++ const legacyDeletes = networkingApi.deleteNamespacedNetworkPolicy.mock.calls.filter(+ ([arg]) =>+ arg.name === 'test-wf-mcp-servers-egress-internet' && arg.namespace === 'mcp-server'+ )+ expect(legacyDeletes).toHaveLength(2)+ })+describe('B2 desired-set NetworkPolicy prune', () => {const RECIPE = 'test-wf'const SANDBOX = 'sandbox-recipes'@@ -3091,13 +3125,14 @@ describe('WorkflowReconciler — reconcile loop', () => {opts: {codex?: Partial<CodexExecutionProjection>grok?: Partial<CodexExecutionProjection> & { requiresGrokProxyEgress?: boolean }+ awaitsTriggeredRun?: boolean} = {}): Promise<void> {const spec = makeSpec()const runtime = deriveWorkflowRuntimePlan(spec, {recipeName: RECIPE,runtimeScopeRecipeName: RECIPE,- workflowRunId: 'run-1',+ workflowRunId: opts.awaitsTriggeredRun ? undefined : 'run-1',})const codex = ineligibleProjection({requiresCodexProxyEgress: false,@@ -3122,7 +3157,16 @@ describe('WorkflowReconciler — reconcile loop', () => {grokProjection: typeof grok) => Promise<unknown>}- ).applyWorkflowNetworkPolicies(RECIPE, 'uid-123', spec, runtime, false, codex, false, grok)+ ).applyWorkflowNetworkPolicies(+ RECIPE,+ 'uid-123',+ spec,+ runtime,+ opts.awaitsTriggeredRun === true,+ codex,+ false,+ grok+ )}function expectListWitness(@@ -3137,7 +3181,7 @@ describe('WorkflowReconciler — reconcile loop', () => {)}- it('deletes exactly one listed obsolete policy that is not in the desired set', async () => {+ it('does not delete a listed name that is outside the run-lane catalog', async () => {const apiserver = makeApiserverNetworkingApi()seedLivePolicy(apiserver, `${RECIPE}-obsolete-wrc`, WRC_LABELS)const reconciler = new WorkflowReconciler(makeDeps({ networkingApi: apiserver.api as never }))@@ -3145,9 +3189,97 @@ describe('WorkflowReconciler — reconcile loop', () => {await applyPolicies(reconciler)expectListWitness(apiserver.api)- expect(- sandboxDeletes(apiserver.api).filter(d => d.name === `${RECIPE}-obsolete-wrc`)- ).toEqual([{ name: `${RECIPE}-obsolete-wrc`, namespace: SANDBOX }])+ expect(sandboxDeletes(apiserver.api).map(d => d.name)).not.toContain(`${RECIPE}-obsolete-wrc`)+ expect(apiserver.live.has(apiserver.key(SANDBOX, `${RECIPE}-obsolete-wrc`))).toBe(true)+ })++ it('G1: reserved coordinator-to-gfs survives a run-lane apply', async () => {+ const apiserver = makeApiserverNetworkingApi()+ const gfsName = `${RECIPE}-coordinator-to-gfs`+ seedLivePolicy(apiserver, gfsName, WRC_LABELS)+ seedLivePolicy(apiserver, `${RECIPE}-obsolete-wrc`, WRC_LABELS)+ seedLivePolicy(apiserver, COORD_TO_WRC, WRC_LABELS)+ seedLivePolicy(apiserver, `${RECIPE}-oauth-broker-egress`, {+ 'clerum.io/recipe': RECIPE,+ 'clerum.io/managed-by': 'workflow-recipes',+ })+ seedLivePolicy(apiserver, `${RECIPE}-outer-unknown-wrc`, WRC_LABELS)+ seedLivePolicy(apiserver, 'other-recipe-coordinator-to-gfs', {+ 'clerum.io/recipe': 'other-recipe',+ 'clerum.io/managed-by': 'wrc',+ })+ seedLivePolicy(apiserver, GROK_PROXY, WRC_LABELS)+ const reconciler = new WorkflowReconciler(makeDeps({ networkingApi: apiserver.api as never }))++ await applyPolicies(reconciler)++ expectListWitness(apiserver.api)+ expect(sandboxDeletes(apiserver.api).map(d => d.name)).toEqual([GROK_PROXY])+ expect(apiserver.live.has(apiserver.key(SANDBOX, gfsName))).toBe(true)+ expect(apiserver.live.has(apiserver.key(SANDBOX, `${RECIPE}-obsolete-wrc`))).toBe(true)+ expect(apiserver.live.has(apiserver.key(SANDBOX, COORD_TO_WRC))).toBe(true)+ expect(apiserver.live.has(apiserver.key(SANDBOX, `${RECIPE}-oauth-broker-egress`))).toBe(true)+ expect(apiserver.live.has(apiserver.key(SANDBOX, `${RECIPE}-outer-unknown-wrc`))).toBe(true)+ expect(apiserver.live.has(apiserver.key(SANDBOX, 'other-recipe-coordinator-to-gfs'))).toBe(+ true+ )+ expect(apiserver.live.has(apiserver.key(SANDBOX, GROK_PROXY))).toBe(false)+ })++ it('deletes a seeded leftover Grok proxy and does not name-delete when unseeded', async () => {+ const seeded = makeApiserverNetworkingApi()+ seedLivePolicy(seeded, GROK_PROXY, WRC_LABELS)+ const seededReconciler = new WorkflowReconciler(+ makeDeps({ networkingApi: seeded.api as never })+ )+ await applyPolicies(seededReconciler)+ expectListWitness(seeded.api)+ expect(sandboxDeletes(seeded.api).map(d => d.name)).toEqual([GROK_PROXY])++ const unseeded = makeApiserverNetworkingApi()+ const unseededReconciler = new WorkflowReconciler(+ makeDeps({ networkingApi: unseeded.api as never })+ )+ await applyPolicies(unseededReconciler)+ expectListWitness(unseeded.api)+ expect(sandboxDeletes(unseeded.api).map(d => d.name)).not.toContain(GROK_PROXY)+ })++ it('deletes a seeded leftover Codex proxy and does not name-delete when unseeded', async () => {+ const seeded = makeApiserverNetworkingApi()+ seedLivePolicy(seeded, CODEX_PROXY, WRC_LABELS)+ const seededReconciler = new WorkflowReconciler(+ makeDeps({ networkingApi: seeded.api as never })+ )+ await applyPolicies(seededReconciler)+ expect(sandboxDeletes(seeded.api).map(d => d.name)).toEqual([CODEX_PROXY])++ const unseeded = makeApiserverNetworkingApi()+ const unseededReconciler = new WorkflowReconciler(+ makeDeps({ networkingApi: unseeded.api as never })+ )+ await applyPolicies(unseededReconciler)+ expect(sandboxDeletes(unseeded.api).map(d => d.name)).not.toContain(CODEX_PROXY)+ })++ it('c2: awaiting-trigger prunes a run-only leftover that apply(false) would keep', async () => {+ const leftover = `${RECIPE}-wrc-to-artifact-reader`+ const awaiting = makeApiserverNetworkingApi()+ seedLivePolicy(awaiting, leftover, WRC_LABELS)+ const awaitingReconciler = new WorkflowReconciler(+ makeDeps({ networkingApi: awaiting.api as never })+ )+ await applyPolicies(awaitingReconciler, { awaitsTriggeredRun: true })+ expect(sandboxDeletes(awaiting.api).map(d => d.name)).toEqual([leftover])++ const running = makeApiserverNetworkingApi()+ seedLivePolicy(running, leftover, WRC_LABELS)+ const runningReconciler = new WorkflowReconciler(+ makeDeps({ networkingApi: running.api as never })+ )+ await applyPolicies(runningReconciler, { awaitsTriggeredRun: false })+ expect(sandboxDeletes(running.api).map(d => d.name)).not.toContain(leftover)+ expect(running.live.has(running.key(SANDBOX, leftover))).toBe(true)})it('does not delete in the sandbox namespace when the LIST is empty', async () => {@@ -3250,12 +3382,60 @@ describe('WorkflowReconciler — reconcile loop', () => {return coreApi.deleteNamespacedPod.mock.calls.filter(([arg]) => arg.name === name).length}- it('does not delete the coordinator when same-pass getPodPhase is undefined', async () => {+ it('does not delete the coordinator when same-pass GET is 404', async () => {const coreApi = makeCoreApi(false)const reconciler = new WorkflowReconciler(makeDeps({ coreApi: coreApi as never }))+ const result = await reconciler.reconcile('test-wf', 'uid-123', SANDBOX, makeSpec())++ expect(result.workflowPhase).not.toBe('failed')+ expect(podReads(coreApi, COORDINATOR)).toBeGreaterThan(0)+ expect(podDeletes(coreApi, COORDINATOR)).toBe(0)+ })++ it('G4: deletes the coordinator when GET 200 has an empty status.phase', async () => {+ const coreApi = makeCoreApi(false)+ coreApi.readNamespacedPod.mockImplementation(async ({ name }: { name: string }) => {+ if (name === COORDINATOR) return { metadata: { name }, status: {} }+ throw { code: 404 }+ })+ const reconciler = new WorkflowReconciler(makeDeps({ coreApi: coreApi as never }))+await reconciler.reconcile('test-wf', 'uid-123', SANDBOX, makeSpec())+ expect(podReads(coreApi, COORDINATOR)).toBeGreaterThan(0)+ expect(podDeletes(coreApi, COORDINATOR)).toBe(1)+ })++ it('G4: a 403 on coordinator GET does not skip as 404', async () => {+ const coreApi = makeCoreApi(false)+ coreApi.readNamespacedPod.mockImplementation(async ({ name }: { name: string }) => {+ if (name === COORDINATOR) throw { code: 403, message: 'forbidden' }+ throw { code: 404 }+ })+ const reconciler = new WorkflowReconciler(makeDeps({ coreApi: coreApi as never }))++ const result = await reconciler.reconcile('test-wf', 'uid-123', SANDBOX, makeSpec())++ expect(result.workflowPhase).toBe('failed')+ expect(podDeletes(coreApi, COORDINATOR)).toBe(0)+ })++ it('c2: does not delete a Running coordinator when workflowRunId is set', async () => {+ const coreApi = makeCoreApi(true)+ const reconciler = new WorkflowReconciler(makeDeps({ coreApi: coreApi as never }))++ await reconciler.reconcile(+ 'test-wf',+ 'uid-123',+ SANDBOX,+ makeSpec(),+ undefined,+ undefined,+ undefined,+ 'run-1'+ )+expect(podReads(coreApi, COORDINATOR)).toBeGreaterThan(0)expect(podDeletes(coreApi, COORDINATOR)).toBe(0)})