Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
fix(workflow-recipes): enforce policy lifecycle and retry ownership
Alfredo Lopez5ab5316fed01
diff --git a/workflow-recipes/package-lock.json b/workflow-recipes/package-lock.jsonindex 35ca72de4..fabbdb163 100644--- a/workflow-recipes/package-lock.json+++ b/workflow-recipes/package-lock.json@@ -1,12 +1,12 @@{"name": "workload-recipes-controller",- "version": "0.1.453",+ "version": "0.1.454","lockfileVersion": 3,"requires": true,"packages": {"": {"name": "workload-recipes-controller",- "version": "0.1.453",+ "version": "0.1.454","license": "SEE LICENSE IN LICENSE","dependencies": {"@aws-sdk/client-s3": "^3.1009.0",diff --git a/workflow-recipes/package.json b/workflow-recipes/package.jsonindex f95e71b5f..c38b2ccc6 100644--- a/workflow-recipes/package.json+++ b/workflow-recipes/package.json@@ -1,6 +1,6 @@{"name": "workload-recipes-controller",- "version": "0.1.453",+ "version": "0.1.454","description": "Workload Recipes Controller (WRC) - WorkloadRecipe lifecycle management and MCP server interface","main": "dist/main.js","engines": {diff --git a/workflow-recipes/src/k8sClient.test.ts b/workflow-recipes/src/k8sClient.test.tsindex 4c54ce30c..812aa1b6b 100644--- a/workflow-recipes/src/k8sClient.test.ts+++ b/workflow-recipes/src/k8sClient.test.ts@@ -1553,10 +1553,12 @@ describe('workload status refresh loop helpers', () => {type InternalWatcher = {recipes: Map<string, WorkflowRecipeCRD>reconciler: { reconcile: typeof reconcile }+ transientRetries: Map<string, { timer: ReturnType<typeof setTimeout>; attempts: number }>handleRecipeEvent: (type: string, recipe: WorkflowRecipeCRD) => Promise<void>}const internal = Object.create(WorkflowRecipeWatcher.prototype) as InternalWatcherinternal.recipes = new Map([[cached.metadata.name, cached]])+ internal.transientRetries = new Map()internal.reconciler = { reconcile }await internal.handleRecipeEvent('MODIFIED', statusOnly)diff --git a/workflow-recipes/src/k8sClient.ts b/workflow-recipes/src/k8sClient.tsindex d7450be8b..f7b6eead6 100644--- a/workflow-recipes/src/k8sClient.ts+++ b/workflow-recipes/src/k8sClient.ts@@ -701,7 +701,11 @@ export class WorkflowRecipeWatcher implements WorkflowRecipeProvider {// result or DELETE.private transientRetries = new Map<string,- { timer: ReturnType<typeof setTimeout>; attempts: number }+ {+ timer: ReturnType<typeof setTimeout>+ attempts: number+ recipeUid?: string+ }>()private static readonly TRANSIENT_RETRY_MAX_MS = 60_000@@ -1055,8 +1059,13 @@ export class WorkflowRecipeWatcher implements WorkflowRecipeProvider {options: { forceReconcile?: boolean } = {}): Promise<void> {if (type === 'ADDED' || type === 'MODIFIED') {+ const scheduledUid = this.transientRetries.get(recipe.metadata.name)?.recipeUid+ if (scheduledUid && scheduledUid !== recipe.metadata.uid) {+ this.clearTransientRetry(recipe.metadata.name)+ }let dbSyncAttempted = falseif (recipe.metadata.deletionTimestamp) {+ this.clearTransientRetry(recipe.metadata.name)// Resource is being deleted — cleanup cross-namespace resources, then remove finalizer.createLogger('wrc', recipe.metadata.name).info('Cleaning up recipe finalizer', {name: recipe.metadata.name,@@ -1181,7 +1190,8 @@ export class WorkflowRecipeWatcher implements WorkflowRecipeProvider {this.scheduleTransientRetry(recipe.metadata.name,result.requeueAfterMs,- result.requeueFixedInterval ?? false+ result.requeueFixedInterval ?? false,+ result.requeueOperation)} else {this.clearTransientRetry(recipe.metadata.name)@@ -1266,17 +1276,38 @@ export class WorkflowRecipeWatcher implements WorkflowRecipeProvider {** Exposed for tests via the watcher's event path.*/- private scheduleTransientRetry(name: string, baseMs: number, fixedInterval = false): void {+ private scheduleTransientRetry(+ name: string,+ baseMs: number,+ fixedInterval = false,+ operation?: ReconcileResult['requeueOperation']+ ): void {const existing = this.transientRetries.get(name)// Fixed-interval (progress) requeues reset the backoff counter; transient// (error) requeues grow it.const attempts = fixedInterval ? 0 : (existing?.attempts ?? 0)if (existing) clearTimeout(existing.timer)+ const owner = operation ? this.recipes.get(name) : undefined+ if (operation && (!owner?.metadata.uid || owner.metadata.deletionTimestamp)) {+ this.transientRetries.delete(name)+ return+ }const delay = fixedInterval? baseMs: Math.min(baseMs * 2 ** attempts, WorkflowRecipeWatcher.TRANSIENT_RETRY_MAX_MS)const timer = setTimeout(() => {if (this.stopped) return+ if (operation && owner) {+ void this.eventQueue.enqueue(name, () =>+ this.retryPendingLegacyNetworkPolicyDelete(+ name,+ owner.metadata.namespace,+ owner.metadata.uid!,+ timer+ )+ )+ return+ }const cached = this.recipes.get(name)if (!cached) {this.transientRetries.delete(name)@@ -1301,7 +1332,81 @@ export class WorkflowRecipeWatcher implements WorkflowRecipeProvider {// reconcile advances past the waiting state once the awaited Pod succeeds,// and the 240s readiness deadline / dbRunProcessor max-duration paths// terminate any run permanently stuck in deploying, so this is bounded.- this.transientRetries.set(name, { timer, attempts: fixedInterval ? 0 : attempts + 1 })+ this.transientRetries.set(name, {+ timer,+ attempts: fixedInterval ? 0 : attempts + 1,+ ...(operation ? { recipeUid: owner!.metadata.uid } : {}),+ })+ }++ /**+ * Runs after the recipe's queue wait. The API read is authoritative for UID,+ * deletion and policy markers; this callback never invokes the full recipe+ * pipeline, token reap or terminal runtime teardown.+ */+ private async retryPendingLegacyNetworkPolicyDelete(+ name: string,+ namespace: string,+ uid: string,+ timer: ReturnType<typeof setTimeout>+ ): Promise<void> {+ const current = () => !this.stopped && this.transientRetries.get(name)?.timer === timer+ if (!current()) return+ try {+ const apiObject = (await this.customApi.getNamespacedCustomObject({+ group: CRD_GROUP,+ version: CRD_VERSION,+ namespace,+ plural: WORKFLOWRECIPE_PLURAL,+ name,+ })) as WorkflowRecipeWatchObject+ if (!current()) return+ if (+ apiObject.metadata.name !== name ||+ apiObject.metadata.namespace !== namespace ||+ apiObject.metadata.uid !== uid ||+ apiObject.metadata.deletionTimestamp+ ) {+ this.clearTransientRetry(name)+ return+ }+ const latest = workflowRecipeFromWatchObject(apiObject, namespace)+ const result = await this.reconciler.retryPendingLegacyNetworkPolicyDeleteOnly(latest)+ if (!current()) return+ const cached = this.recipes.get(name)+ if (!cached || cached.metadata.uid !== uid || cached.metadata.deletionTimestamp) {+ this.clearTransientRetry(name)+ return+ }+ if (result.requeueAfterMs) {+ this.scheduleTransientRetry(+ name,+ result.requeueAfterMs,+ true,+ 'legacy-network-policy-delete'+ )+ } else {+ this.clearTransientRetry(name)+ }+ } catch (error) {+ if (!current()) return+ if (getErrorCode(error) === 404) {+ this.clearTransientRetry(name)+ return+ }+ createLogger('wrc', name).error('Legacy NetworkPolicy delete retry failed', {+ name,+ err: error,+ })+ // A status conflict or transient API failure keeps this same narrow+ // operation on the existing bounded error backoff; it cannot replay runtime cleanup.+ this.scheduleTransientRetry(+ name,+ TRANSIENT_REQUEUE_BASE_MS,+ false,+ 'legacy-network-policy-delete'+ )+ }}private clearTransientRetry(name: string): void {diff --git a/workflow-recipes/src/reconciler/reconciler.test.ts b/workflow-recipes/src/reconciler/reconciler.test.tsindex 7e5805cf7..d6f00f38e 100644--- a/workflow-recipes/src/reconciler/reconciler.test.ts+++ b/workflow-recipes/src/reconciler/reconciler.test.ts@@ -20,6 +20,7 @@ import { buildCoordinatorGfsNetworkPolicy } from '../workflow/networkPolicyFactoimport type { NetworkPolicyPassSummary } from '../workflow/pluginWorkloadSdkProvisioner'import { deriveWorkflowRuntimePlan } from '../workflow/runtimePlan'import {+ LegacyNetworkPolicyDeletePendingError,WorkflowReconciler,buildNetworkPolicyConvergedCondition,networkPolicyMarkerConditions,@@ -33,6 +34,7 @@ import {PLUGIN_WORKLOAD_SDK_POLICY_PENDING_CONDITION_TYPE,PLUGIN_WORKLOAD_SDK_PROVIDER_UNAVAILABLE_CONDITION_TYPE,} from './pluginWorkloadSdkValidator'+import { RecipeEventQueue } from './recipeEventQueue'import {buildUiIngressNetworkPolicy,buildWorkloadIngressNetworkPolicy,@@ -88,6 +90,7 @@ const mockCoreApi = {readNamespacedPersistentVolumeClaim: vi.fn().mockRejectedValue({ code: 404 }),deleteNamespacedPersistentVolumeClaim: vi.fn().mockResolvedValue({}),deleteCollectionNamespacedPod: vi.fn().mockResolvedValue({}),+ deleteNamespacedPod: vi.fn().mockResolvedValue({}),createNamespacedSecret: vi.fn().mockResolvedValue({}),readNamespacedSecret: vi.fn().mockResolvedValue({ metadata: { resourceVersion: '1' } }),replaceNamespacedSecret: vi.fn().mockResolvedValue({}),@@ -1774,6 +1777,7 @@ describe('WorkflowRecipeReconciler', () => {expect(cleanupPluginWorkloadSdk).toHaveBeenCalledTimes(1)expect(cleanupPluginWorkloadSdk).toHaveBeenCalledWith('test-recipe', {preserveWorkflowRuntime: false,+ recipeUid: 'uid-123',})expect(sdkOnly).not.toHaveBeenCalled()expect(result.phase).not.toBe('failed')@@ -1827,6 +1831,55 @@ describe('WorkflowRecipeReconciler', () => {return cleanupPluginWorkloadSdk}+ it.each(['SDK-never-published', 'steps-removed'] as const)(+ 'R5-L1: %s without an SDK runtime or teardown keeps its published apply, prune and legacy facts',+ async history => {+ const cleanup = stubCapabilityRemovalTeardown()+ const inner = (reconciler as unknown as { workflowReconciler: WorkflowReconciler })+ .workflowReconciler+ const retryLegacy = vi.fn().mockResolvedValue('pending')+ inner.retryLegacyMcpServersInternetEgressDelete = retryLegacy+ inner.legacyMcpServersInternetEgressRetryDelayMs = vi.fn().mockReturnValue(60_000)+ const facts = networkPolicyMarkerConditions(+ {+ kind: 'reconcile',+ summary: { conflicts: [], retryPending: true, prune: 'pending', legacy: 'pending' },+ },+ [],+ '2026-09-23T10:00:00.000Z'+ )+ const recipe = makeRecipe({+ spec: {+ workloads: [+ { id: 'app', type: 'deployment', image: 'nginx:1.30.1-alpine', port: 8080 },+ ],+ ...(history === 'steps-removed' ? { steps: [] } : {}),+ },+ status: {+ phase: 'active',+ conditions: [ownershipCondition, ...facts],+ ...(history === 'steps-removed' ? { workflowExecution: { phase: 'completed' } } : {}),+ },+ })+ expect(recipe.spec.pluginWorkloadSdk).toBeUndefined()+ expect(recipe.status?.pluginWorkloadSdk).toBeUndefined()++ const result = await reconciler.reconcile(recipe)++ expect(retryLegacy).toHaveBeenCalledWith('test-recipe', 'uid-123')+ expect(cleanup).not.toHaveBeenCalled()+ expect(result.networkPolicyOwnershipConditions).toBeUndefined()+ expect(result.requeueAfterMs).toBe(60_000)+ expect(result.requeueOperation).toBe('legacy-network-policy-delete')+ await reconciler.patchStatus(recipe, result)+ const published = mockCustomApi.patchNamespacedCustomObjectStatus.mock.calls.at(-1)![0].body+ .status.conditions as StatusCondition[]+ for (const fact of [ownershipCondition, ...facts]) {+ expect(published).toContainEqual(fact)+ }+ }+ )+it('R4-L3: clears a published PrunePending marker once the SDK capability is removed from the spec', async () => {const cleanupPluginWorkloadSdk = stubCapabilityRemovalTeardown()const recipe = capabilityRemovedRecipe('active', [ownershipCondition, prunePendingMarker()])@@ -1836,6 +1889,7 @@ describe('WorkflowRecipeReconciler', () => {// Liveness witness: the capability-removal teardown ran.expect(cleanupPluginWorkloadSdk).toHaveBeenCalledWith('test-recipe', {preserveWorkflowRuntime: false,+ recipeUid: 'uid-123',})expect(result.phase).not.toBe('failed')expect(result.networkPolicyOwnershipConditions).toEqual([])@@ -1850,6 +1904,7 @@ describe('WorkflowRecipeReconciler', () => {// Liveness witness: the capability-removal teardown ran.expect(cleanupPluginWorkloadSdk).toHaveBeenCalledWith('test-recipe', {preserveWorkflowRuntime: false,+ recipeUid: 'uid-123',})const conditionPatches = mockCustomApi.patchNamespacedCustomObjectStatus.mock.calls.map(call => call[0].body.status)@@ -1953,6 +2008,7 @@ describe('WorkflowRecipeReconciler', () => {expect(result.phase).toBe('active')expect(cleanupPluginWorkloadSdk).toHaveBeenCalledWith('test-recipe', {preserveWorkflowRuntime: false,+ recipeUid: 'uid-123',})expect(workflowReconcile).not.toHaveBeenCalled()})@@ -1983,6 +2039,7 @@ describe('WorkflowRecipeReconciler', () => {expect(result.phase).toBe(phase)expect(cleanupPluginWorkloadSdk).toHaveBeenCalledWith('test-recipe', {preserveWorkflowRuntime: false,+ recipeUid: 'uid-123',})})@@ -2028,10 +2085,69 @@ describe('WorkflowRecipeReconciler', () => {})expect(cleanupPluginWorkloadSdk).toHaveBeenCalledWith('test-recipe', {preserveWorkflowRuntime: false,+ recipeUid: 'uid-123',})expect(workflowReconcile).not.toHaveBeenCalled()})+ it.each(['kill-switch', 'capability-removal'] as const)(+ 'R5-L2: %s cleanup passes the live UID and preserves the exact legacy backoff',+ async branch => {+ const cleanup = vi.fn()+ const retry = vi.fn()+ const delay = vi.fn()+ const internal = reconciler as unknown as {+ config: { pluginWorkloadSdkEnabled: boolean }+ workflowReconciler: unknown+ }+ internal.config.pluginWorkloadSdkEnabled = branch !== 'kill-switch'+ internal.workflowReconciler = {+ cleanupPluginWorkloadSdk: cleanup,+ retryLegacyMcpServersInternetEgressDelete: retry,+ legacyMcpServersInternetEgressRetryDelayMs: delay,+ }+ const conditions = networkPolicyMarkerConditions(+ {+ kind: 'reconcile',+ summary: { conflicts: [], retryPending: true, prune: 'pending', legacy: 'pending' },+ },+ [],+ '2026-09-23T10:00:00.000Z'+ )+ const recipe = makeRecipe({+ spec: {+ workloads: [],+ ...(branch === 'kill-switch' ? { pluginWorkloadSdk: { allowedCallers: [] } } : {}),+ },+ status: {+ phase: 'active',+ conditions,+ pluginWorkloadSdk: { state: 'validated', promptBridge: true, clientNotifications: false },+ },+ })+ for (const remaining of [60_000, 40_123, 999]) {+ cleanup.mockRejectedValue(+ new LegacyNetworkPolicyDeletePendingError('test-recipe', remaining)+ )+ const result = await reconciler.reconcile(recipe)+ expect(cleanup).toHaveBeenLastCalledWith('test-recipe', {+ preserveWorkflowRuntime: false,+ recipeUid: 'uid-123',+ })+ expect(result.requeueAfterMs).toBe(remaining)+ expect(result.requeueFixedInterval).toBe(true)+ expect(result.requeueOperation).toBeUndefined()+ expect(result.skipStatusPatch).toBe(true)+ expect(result.networkPolicyOwnershipConditions).toBeUndefined()+ expect(result.pluginWorkloadSdkTeardownConfirmed).toBeUndefined()+ expect(recipe.status?.conditions).toEqual(conditions)+ }+ expect(retry).not.toHaveBeenCalled()+ expect(delay).not.toHaveBeenCalled()+ expect(mockCustomApi.patchNamespacedCustomObjectStatus).not.toHaveBeenCalled()+ }+ )+// The kill switch returns before the SDK-only lane, which is the only path// that clears the ownership condition. With the runtime torn down no policy// is managed, so a conflict published earlier is stale.@@ -7600,34 +7716,40 @@ describe('WorkflowRecipeReconciler', () => {)})- // The legacy internet NP ledger is keyed by recipeUid and cleared from- // cleanupPluginWorkloadSdk only when the uid is passed. The SDK-only delete- // path is the one that has to hand it over.- it('reconcileDelete hands the recipe uid to the SDK cleanup on the SDK-only path', async () => {- const cleanupPluginWorkloadSdk = vi.fn().mockResolvedValue(undefined)- ;(- reconciler as unknown as {- workflowReconciler: { cleanupPluginWorkloadSdk: typeof cleanupPluginWorkloadSdk }- }- ).workflowReconciler = { cleanupPluginWorkloadSdk }- const recipe = makeRecipe({- metadata: { name: 'test-recipe', namespace: 'sandbox-recipes', uid: 'uid-sdk-delete' },- spec: {- workloads: [{ id: 'app', type: 'deployment', image: 'nginx:1.30.1-alpine', port: 8080 }],- },- status: {- phase: 'active',- pluginWorkloadSdk: { state: 'validated', promptBridge: true, clientNotifications: false },- },- } as Partial<WorkflowRecipeCRD>)+ // UID identifies the shared ledger. Only a real finalizer may forget it after+ // the last-chance cleanup succeeds; a live teardown must retain that history.+ it.each([false, true])(+ 'R5-L2: reconcileDelete identifies finalizer SDK cleanup for workflow=%s',+ async workflow => {+ const cleanupPluginWorkloadSdk = vi.fn().mockResolvedValue(undefined)+ const deleteWorkflow = vi.fn().mockResolvedValue(undefined)+ ;(+ reconciler as unknown as {+ workflowReconciler: { cleanupPluginWorkloadSdk: typeof cleanupPluginWorkloadSdk }+ }+ ).workflowReconciler = { cleanupPluginWorkloadSdk, reconcileDelete: deleteWorkflow }+ const recipe = makeRecipe({+ metadata: { name: 'test-recipe', namespace: 'sandbox-recipes', uid: 'uid-sdk-delete' },+ spec: {+ workloads: [{ id: 'app', type: 'deployment', image: 'nginx:1.30.1-alpine', port: 8080 }],+ ...(workflow ? { steps: [{ id: 'task', instruction: 'run' }] } : {}),+ },+ status: {+ phase: 'active',+ pluginWorkloadSdk: { state: 'validated', promptBridge: true, clientNotifications: false },+ },+ } as Partial<WorkflowRecipeCRD>)- await reconciler.reconcileDelete(recipe)+ await reconciler.reconcileDelete(recipe)- expect(cleanupPluginWorkloadSdk).toHaveBeenCalledTimes(1)- expect(cleanupPluginWorkloadSdk).toHaveBeenCalledWith('test-recipe', {- recipeUid: 'uid-sdk-delete',- })- })+ expect(cleanupPluginWorkloadSdk).toHaveBeenCalledTimes(1)+ expect(cleanupPluginWorkloadSdk).toHaveBeenCalledWith('test-recipe', {+ recipeUid: 'uid-sdk-delete',+ recipeDeleted: true,+ ...(workflow ? { preserveWorkflowRuntime: true } : {}),+ })+ }+ )// ─── Phase 8: Namespace Splitting ─────────────────────────────────@@ -15731,53 +15853,74 @@ describe('WorkflowRecipeReconciler', () => {})}- it('R4-L1: after the finalizer deleted the token, a recreated recipe sends no DELETE until a new token ADDED, then even below the old watermark', async () => {- const recipe = reapRecipe(4)- tokenSeen()- await reconciler.ensureOAuthBrokerTokenSecret(recipe)- await reconciler.ensureOAuthBrokerTokenSecret(recipe)- expect(secretDeletes()).toBe(1)+ it.each(['2xx', '404'] as const)(+ 'R5-L6: after the finalizer observed token %s, a recreated recipe sends no DELETE until a new token ADDED, then even below the old watermark',+ async outcome => {+ const recipe = reapRecipe(4)+ tokenSeen()+ await reconciler.ensureOAuthBrokerTokenSecret(recipe)+ await reconciler.ensureOAuthBrokerTokenSecret(recipe)+ expect(secretDeletes()).toBe(1)- await reconciler.reconcileDelete(recipe)- // Witness for the premise: the finalizer sent the token DELETE, and the- // default mock answers it 2xx.- expect(secretDeletes()).toBe(2)- mockCoreApi.deleteNamespacedSecret.mockClear()+ if (outcome === '404') failTokenDelete(404)+ try {+ await reconciler.reconcileDelete(recipe)+ } finally {+ mockCoreApi.deleteNamespacedSecret.mockResolvedValue({})+ }+ // The real finalizer sent the token DELETE and mapped its observed result.+ expect(secretDeletes()).toBe(2)+ mockCoreApi.deleteNamespacedSecret.mockClear()- const debugSpy = captureLogger('debug')- try {- await reconciler.ensureOAuthBrokerTokenSecret(recreatedRecipe())- await reconciler.ensureOAuthBrokerTokenSecret(recreatedRecipe())- expect(secretDeletes()).toBe(0)- // Liveness witness: both passes reached the ledger and it declined.- expect(- debugSpy.mock.calls.filter(- ([message, fields]) =>- typeof message === 'string' &&- message.startsWith('Skipping oauth-broker-token delete') &&- (fields as { generation?: number }).generation === 1- )- ).toHaveLength(2)- } finally {- debugSpy.mockRestore()- }+ const debugSpy = captureLogger('debug')+ try {+ await reconciler.ensureOAuthBrokerTokenSecret(recreatedRecipe())+ await reconciler.ensureOAuthBrokerTokenSecret(recreatedRecipe())+ expect(secretDeletes()).toBe(0)+ // Liveness witness: both passes reached the ledger and it declined.+ expect(+ debugSpy.mock.calls.filter(+ ([message, fields]) =>+ typeof message === 'string' &&+ message.startsWith('Skipping oauth-broker-token delete') &&+ (fields as { generation?: number }).generation === 1+ )+ ).toHaveLength(2)+ } finally {+ debugSpy.mockRestore()+ }- // A token ADDED for a recipe under the same name: the old watermark is- // gone, so a lower generation deletes.- tokenSeen()- await reconciler.ensureOAuthBrokerTokenSecret(reapRecipe(3))- expect(secretDeletes()).toBe(1)- })+ // A token ADDED for a recipe under the same name: the old watermark is+ // gone, so a lower generation deletes.+ tokenSeen()+ await reconciler.ensureOAuthBrokerTokenSecret(reapRecipe(3))+ expect(secretDeletes()).toBe(1)+ }+ )- it('R4-L1: a finalizer whose token DELETE failed keeps the token seen, so a recreated recipe deletes it once', async () => {+ it('R4-L1/R5-L7: a failed finalizer token DELETE preserves its error object and keeps the token seen', async () => {tokenSeen()- failTokenDelete(500)+ const error = { code: 500, message: 'delete failed' }+ const errorLog = captureLogger('error')+ mockCoreApi.deleteNamespacedSecret.mockImplementation(async ({ name }: { name: string }) => {+ if (name === SECRET_NAME) throw error+ return {}+ })try {await reconciler.reconcileDelete(reapRecipe(4))// Witness: the finalizer sent the token DELETE and it failed.expect(secretDeletes()).toBe(1)+ expect(errorLog).toHaveBeenCalledWith('Failed to delete resource', {+ label: `Secret "${SECRET_NAME}" in sandbox-recipes`,+ err: error,+ })+ const failure = errorLog.mock.calls.find(+ ([message]) => message === 'Failed to delete resource'+ )+ expect((failure?.[1] as { err?: unknown }).err).toBe(error)} finally {mockCoreApi.deleteNamespacedSecret.mockResolvedValue({})+ errorLog.mockRestore()}mockCoreApi.deleteNamespacedSecret.mockClear()@@ -15790,8 +15933,8 @@ describe('WorkflowRecipeReconciler', () => {it('R4-L1: a token ADDED while the finalizer DELETE is in flight keeps the token seen', async () => {tokenSeen()mockCoreApi.deleteNamespacedSecret.mockImplementation(async ({ name }: { name: string }) => {- // The recipe was recreated and its token ADDED was handled before the- // finalizer's DELETE of the old token returned.+ // A late ADDED delivery (or a restart's relist) was handled before the+ // finalizer's DELETE returned; it does not imply a concurrent CR recreation.if (name === SECRET_NAME) reconciler.invalidateOAuthBrokerSecretLedger('test-recipe')return {}})@@ -16494,6 +16637,7 @@ describe('WorkflowRecipeReconciler', () => {afterEach(() => {vi.useRealTimers()+ mockCustomApi.patchNamespacedCustomObjectStatus.mockReset().mockResolvedValue({})})function runScopedRecipe(status: Record<string, unknown>): WorkflowRecipeCRD {@@ -16736,6 +16880,393 @@ describe('WorkflowRecipeReconciler', () => {expect(result.requeueAfterMs).toBe(60_000)expect(result.requeueFixedInterval).toBe(true)})++ function realWatcherBridge(recipe: WorkflowRecipeCRD) {+ // The surrounding ledger-only cases fake Date; this bridge drives real timers too.+ vi.useRealTimers()+ vi.useFakeTimers()+ vi.setSystemTime(T0)+ let live: WorkflowRecipeCRD | undefined = structuredClone(recipe)+ const previousGet = mockCustomApi.getNamespacedCustomObject.getMockImplementation()!+ mockCustomApi.getNamespacedCustomObject.mockImplementation(async args => {+ if (args.plural !== 'workflowrecipes' || args.name !== RECIPE) return previousGet(args)+ if (!live) throw { code: 404 }+ return structuredClone(live)+ })+ mockCustomApi.patchNamespacedCustomObjectStatus.mockImplementation(async ({ body }) => {+ if (!live) throw { code: 404 }+ if (+ body.metadata &&+ (body.metadata.uid !== live.metadata.uid ||+ body.metadata.resourceVersion !== live.metadata.resourceVersion)+ ) {+ throw { code: 409 }+ }+ live.status = { ...live.status, ...body.status }+ live.metadata.resourceVersion = String(Number(live.metadata.resourceVersion) + 1)+ return structuredClone(live)+ })+ const queueFailures = vi.fn()+ const queue = new RecipeEventQueue(queueFailures)+ const watcher = Object.create(WorkflowRecipeWatcher.prototype) as {+ recipes: Map<string, WorkflowRecipeCRD>+ eventQueue: RecipeEventQueue+ transientRetries: Map<string, { timer: ReturnType<typeof setTimeout>; attempts: number }>+ handleRecipeEvent: (type: string, recipe: WorkflowRecipeCRD) => Promise<void>+ clearTransientRetry: (name: string) => void+ }+ Object.assign(watcher, {+ recipes: new Map(),+ eventQueue: queue,+ transientRetries: new Map(),+ stopped: false,+ traceReporter: null,+ dbRunProcessor: null,+ reconciler,+ customApi: mockCustomApi,+ })+ return {+ watcher,+ queue,+ queueFailures,+ current: () => live,+ replace: (next: WorkflowRecipeCRD | undefined) => {+ live = next+ },+ drain: () => queue.enqueue(RECIPE, async () => {}),+ }+ }++ function terminalRetryRecipe(): WorkflowRecipeCRD {+ const recipe = runScopedRecipe({+ phase: 'active',+ message: 'Workflow completed',+ workflowExecution: { phase: 'completed' },+ conditions: publishedFacts('converged'),+ workloadInstances: { 'web-search': `${RECIPE}-web-search-run` },+ })+ recipe.metadata.resourceVersion = '1'+ recipe.metadata.generation = 1+ recipe.spec.workloads = [+ {+ id: 'web-search',+ type: 'deployment',+ image: 'web-search:test',+ transport: { type: 'streamableHttp', path: '/mcp' },+ },+ ]+ return recipe+ }++ it('R5-L3: the dedicated retry retains the pipeline namespace allowlist', async () => {+ const inner = installRealInner()+ const retry = vi.spyOn(inner, 'retryLegacyMcpServersInternetEgressDelete')+ const recipe = terminalRetryRecipe()+ recipe.metadata.namespace = 'other-namespace'+ expect(ofType(recipe.status?.conditions, LEGACY_TYPE)).toHaveLength(1)+ mockCustomApi.getNamespacedCustomObject.mockClear()+ const result = await reconciler.retryPendingLegacyNetworkPolicyDeleteOnly(recipe)+ expect(result).toEqual({})+ expect(retry).not.toHaveBeenCalled()+ expect(mockCustomApi.getNamespacedCustomObject).not.toHaveBeenCalled()+ expect(legacyDeletes()).toBe(0)+ })++ it('R5-L3: the real watcher, WRC and event queue retry only legacy DELETE across 60/120/240 s windows', async () => {+ vi.useFakeTimers()+ const inner = installRealInner()+ vi.spyOn(inner, 'ensureMcpHostRuntimeCredentials').mockResolvedValue()+ const teardown = vi.spyOn(inner, 'teardownComputePodsForTerminalRun')+ const fullReconcile = vi.spyOn(reconciler, 'reconcile')+ legacyDeleteAnswers({ code: 403, message: 'forbidden' })+ const bridge = realWatcherBridge(terminalRetryRecipe())++ await bridge.queue.enqueue(RECIPE, () =>+ bridge.watcher.handleRecipeEvent('ADDED', bridge.current()!)+ )++ expect(teardown).toHaveBeenCalledTimes(1)+ expect(legacyDeletes()).toBe(1)+ const initialEffects = {+ pods: mockCoreApi.deleteNamespacedPod.mock.calls.length,+ transport: mockCustomApi.deleteNamespacedCustomObject.mock.calls.filter(+ ([arg]) => arg.plural === 'mcpservers'+ ).length,+ deployments: mockAppsApi.deleteNamespacedDeployment.mock.calls.length,+ gfs: mockNetworkingApi.deleteNamespacedNetworkPolicy.mock.calls.filter(([arg]) =>+ arg.name.endsWith('-coordinator-to-gfs')+ ).length,+ }+ expect(initialEffects.pods).toBe(3)+ expect(initialEffects.transport).toBeGreaterThan(0)+ expect(initialEffects.deployments).toBeGreaterThan(0)+ expect(initialEffects.gfs).toBeGreaterThan(0)++ for (const [index, delay] of [60_000, 120_000, 240_000].entries()) {+ await vi.advanceTimersByTimeAsync(delay)+ await bridge.drain()+ expect(legacyDeletes()).toBe(index + 2)+ expect(fullReconcile).toHaveBeenCalledTimes(1)+ expect(teardown).toHaveBeenCalledTimes(1)+ expect(mockCoreApi.deleteNamespacedPod).toHaveBeenCalledTimes(initialEffects.pods)+ expect(mockAppsApi.deleteNamespacedDeployment).toHaveBeenCalledTimes(+ initialEffects.deployments+ )+ expect(+ mockCustomApi.deleteNamespacedCustomObject.mock.calls.filter(+ ([arg]) => arg.plural === 'mcpservers'+ )+ ).toHaveLength(initialEffects.transport)+ expect(+ mockNetworkingApi.deleteNamespacedNetworkPolicy.mock.calls.filter(([arg]) =>+ arg.name.endsWith('-coordinator-to-gfs')+ )+ ).toHaveLength(initialEffects.gfs)+ }++ // Recovery performs a conditions-only patch, preserving the workflow/SDK state.+ const beforeRecovery = structuredClone(bridge.current()!.status)+ legacyDeleteAnswers({ code: 404, message: 'not found' })+ mockCustomApi.patchNamespacedCustomObjectStatus.mockClear()+ await vi.advanceTimersByTimeAsync(480_000)+ await bridge.drain()+ expect(legacyDeletes()).toBe(5)+ expect(fullReconcile).toHaveBeenCalledTimes(1)+ expect(bridge.current()!.status?.workflowExecution).toEqual(beforeRecovery?.workflowExecution)+ expect(bridge.current()!.status?.pluginWorkloadSdk).toEqual(beforeRecovery?.pluginWorkloadSdk)+ expect(ofType(bridge.current()!.status?.conditions, LEGACY_TYPE)).toEqual([])+ expect(mockCustomApi.patchNamespacedCustomObjectStatus).toHaveBeenCalledWith(+ expect.objectContaining({+ body: {+ metadata: { uid: 'uid-lg', resourceVersion: expect.any(String) },+ status: { conditions: expect.any(Array) },+ },+ }),+ expect.anything()+ )+ expect(bridge.queueFailures).not.toHaveBeenCalled()+ expect(vi.getTimerCount()).toBe(0)++ // Deliver the actual accepted PATCH response through the ordinary watch+ // path. Its unchanged generation makes this a status-only MODIFIED, whose+ // terminal cleanup remains required by the existing recovery contract.+ expect(mockCustomApi.patchNamespacedCustomObjectStatus).toHaveBeenCalledTimes(1)+ const recoveryWatchObject = (await mockCustomApi.patchNamespacedCustomObjectStatus.mock+ .results[0].value) as WorkflowRecipeCRD+ expect(recoveryWatchObject).toEqual(bridge.current())+ expect(recoveryWatchObject.metadata.resourceVersion).toBe('2')+ expect(recoveryWatchObject.metadata.generation).toBe(1)+ await bridge.queue.enqueue(RECIPE, () =>+ bridge.watcher.handleRecipeEvent('MODIFIED', recoveryWatchObject)+ )+ await bridge.drain()+ expect(fullReconcile).toHaveBeenCalledTimes(2)+ expect(teardown).toHaveBeenCalledTimes(2)+ expect(mockCoreApi.deleteNamespacedPod).toHaveBeenCalledTimes(initialEffects.pods * 2)+ expect(mockAppsApi.deleteNamespacedDeployment).toHaveBeenCalledTimes(+ initialEffects.deployments * 2+ )+ expect(+ mockCustomApi.deleteNamespacedCustomObject.mock.calls.filter(+ ([arg]) => arg.plural === 'mcpservers'+ )+ ).toHaveLength(initialEffects.transport * 2)+ expect(+ mockNetworkingApi.deleteNamespacedNetworkPolicy.mock.calls.filter(([arg]) =>+ arg.name.endsWith('-coordinator-to-gfs')+ )+ ).toHaveLength(initialEffects.gfs * 2)+ expect(legacyDeletes()).toBe(5)+ // That ordinary pass publishes nothing, so it cannot drive another watch+ // event. No legacy timer or continuing status-write loop remains either.+ expect(mockCustomApi.patchNamespacedCustomObjectStatus).toHaveBeenCalledTimes(1)+ expect(vi.getTimerCount()).toBe(0)+ await vi.advanceTimersByTimeAsync(3_600_000)+ await bridge.drain()+ expect(fullReconcile).toHaveBeenCalledTimes(2)+ expect(teardown).toHaveBeenCalledTimes(2)+ expect(legacyDeletes()).toBe(5)+ expect(mockCustomApi.patchNamespacedCustomObjectStatus).toHaveBeenCalledTimes(1)+ expect(bridge.queueFailures).not.toHaveBeenCalled()+ bridge.watcher.clearTransientRetry(RECIPE)+ })++ it.each([+ 'recreated',+ 'deleted',+ 'deleting',+ 'marker-cleared',+ 'namespace-changed',+ 'name-changed',+ ] as const)(+ 'R5-L3: a queued legacy retry drops a %s recipe after its fresh API read',+ async change => {+ vi.useFakeTimers()+ const inner = installRealInner()+ vi.spyOn(inner, 'ensureMcpHostRuntimeCredentials').mockResolvedValue()+ const teardown = vi.spyOn(inner, 'teardownComputePodsForTerminalRun')+ const fullReconcile = vi.spyOn(reconciler, 'reconcile')+ legacyDeleteAnswers({ code: 403, message: 'forbidden' })+ const bridge = realWatcherBridge(terminalRetryRecipe())+ await bridge.queue.enqueue(RECIPE, () =>+ bridge.watcher.handleRecipeEvent('ADDED', bridge.current()!)+ )+ expect(legacyDeletes()).toBe(1)+ expect(vi.getTimerCount()).toBe(1)+ let release!: () => void+ const blocker = bridge.queue.enqueue(+ RECIPE,+ () =>+ new Promise<void>(resolve => {+ release = resolve+ })+ )+ await Promise.resolve()+ await vi.advanceTimersByTimeAsync(60_000)+ const next = structuredClone(bridge.current()!)+ if (change === 'deleted') bridge.replace(undefined)+ else {+ if (change === 'recreated') next.metadata.uid = 'uid-new'+ if (change === 'deleting') next.metadata.deletionTimestamp = new Date().toISOString()+ if (change === 'marker-cleared') next.status!.conditions = []+ if (change === 'namespace-changed') next.metadata.namespace = 'other-namespace'+ if (change === 'name-changed') next.metadata.name = 'other-recipe'+ bridge.replace(next)+ }+ // Leave the watcher's cache stale: the API read after the queue wait is authoritative.+ release()+ await blocker+ await bridge.drain()+ expect(legacyDeletes()).toBe(1)+ expect(fullReconcile).toHaveBeenCalledTimes(1)+ expect(teardown).toHaveBeenCalledTimes(1)+ expect(bridge.queueFailures).not.toHaveBeenCalled()+ expect(vi.getTimerCount()).toBe(0)+ bridge.watcher.clearTransientRetry(RECIPE)+ }+ )++ it.each(['recreated', 'deleting', 'marker-cleared'] as const)(+ 'R5-L3: the post-DELETE read refuses to publish on a %s recipe',+ async change => {+ const inner = installRealInner()+ vi.spyOn(inner, 'ensureMcpHostRuntimeCredentials').mockResolvedValue()+ const fullReconcile = vi.spyOn(reconciler, 'reconcile')+ legacyDeleteAnswers({ code: 403, message: 'forbidden' })+ const bridge = realWatcherBridge(terminalRetryRecipe())+ await bridge.queue.enqueue(RECIPE, () =>+ bridge.watcher.handleRecipeEvent('ADDED', bridge.current()!)+ )+ expect(legacyDeletes()).toBe(1)+ mockCustomApi.patchNamespacedCustomObjectStatus.mockClear()+ mockNetworkingApi.deleteNamespacedNetworkPolicy.mockImplementation(async ({ name }) => {+ if (name === LEGACY) {+ const next = structuredClone(bridge.current()!)+ if (change === 'recreated') next.metadata.uid = 'uid-new'+ if (change === 'deleting') next.metadata.deletionTimestamp = new Date().toISOString()+ if (change === 'marker-cleared') next.status!.conditions = []+ bridge.replace(next)+ }+ return {}+ })+ await vi.advanceTimersByTimeAsync(60_000)+ await bridge.drain()+ expect(legacyDeletes()).toBe(2)+ expect(fullReconcile).toHaveBeenCalledTimes(1)+ expect(mockCustomApi.patchNamespacedCustomObjectStatus).not.toHaveBeenCalled()+ expect(vi.getTimerCount()).toBe(0)+ expect(bridge.queueFailures).not.toHaveBeenCalled()+ }+ )++ it.each(['recreated', 'newer-policy-facts'] as const)(+ 'R5-L3: a status conflict caused by %s retries only the fenced conditions patch',+ async change => {+ const inner = installRealInner()+ vi.spyOn(inner, 'ensureMcpHostRuntimeCredentials').mockResolvedValue()+ const fullReconcile = vi.spyOn(reconciler, 'reconcile')+ legacyDeleteAnswers({ code: 403, message: 'forbidden' })+ const bridge = realWatcherBridge(terminalRetryRecipe())+ await bridge.queue.enqueue(RECIPE, () =>+ bridge.watcher.handleRecipeEvent('ADDED', bridge.current()!)+ )+ mockCustomApi.patchNamespacedCustomObjectStatus.mockClear()+ const newerFacts = networkPolicyMarkerConditions(+ {+ kind: 'reconcile',+ summary: { conflicts: [], retryPending: true, prune: 'pending', legacy: 'pending' },+ },+ [],+ new Date().toISOString()+ )+ mockCustomApi.patchNamespacedCustomObjectStatus.mockImplementationOnce(async ({ body }) => {+ expect(body.metadata).toEqual({ uid: 'uid-lg', resourceVersion: '1' })+ const next = structuredClone(bridge.current()!)+ next.metadata.resourceVersion = '2'+ if (change === 'recreated') next.metadata.uid = 'uid-new'+ else next.status!.conditions = newerFacts+ bridge.replace(next)+ throw { code: 409 }+ })+ legacyDeleteAnswers({ code: 404, message: 'not found' })+ await vi.advanceTimersByTimeAsync(60_000)+ await bridge.drain()+ expect(legacyDeletes()).toBe(2)+ expect(mockCustomApi.patchNamespacedCustomObjectStatus).toHaveBeenCalledTimes(1)+ await vi.advanceTimersByTimeAsync(4_999)+ await bridge.drain()+ expect(mockCustomApi.patchNamespacedCustomObjectStatus).toHaveBeenCalledTimes(1)+ await vi.advanceTimersByTimeAsync(1)+ await bridge.drain()+ expect(legacyDeletes()).toBe(2)+ expect(fullReconcile).toHaveBeenCalledTimes(1)+ if (change === 'recreated') {+ expect(mockCustomApi.patchNamespacedCustomObjectStatus).toHaveBeenCalledTimes(1)+ expect(bridge.current()!.metadata.uid).toBe('uid-new')+ } else {+ expect(mockCustomApi.patchNamespacedCustomObjectStatus).toHaveBeenCalledTimes(2)+ expect(+ mockCustomApi.patchNamespacedCustomObjectStatus.mock.calls[1][0].body.metadata+ ).toEqual({ uid: 'uid-lg', resourceVersion: '2' })+ expect(ofType(bridge.current()!.status?.conditions, LEGACY_TYPE)).toEqual([])+ expect(ofType(bridge.current()!.status?.conditions, MARKER_TYPE)).toEqual(+ ofType(newerFacts, MARKER_TYPE)+ )+ expect(bridge.current()!.status?.workflowExecution).toEqual({ phase: 'completed' })+ }+ expect(vi.getTimerCount()).toBe(0)+ expect(bridge.queueFailures).not.toHaveBeenCalled()+ }+ )++ it('R5-L3: clearing the retry while its callback waits in the real queue cancels its API work', async () => {+ const inner = installRealInner()+ vi.spyOn(inner, 'ensureMcpHostRuntimeCredentials').mockResolvedValue()+ legacyDeleteAnswers({ code: 403, message: 'forbidden' })+ const bridge = realWatcherBridge(terminalRetryRecipe())+ await bridge.queue.enqueue(RECIPE, () =>+ bridge.watcher.handleRecipeEvent('ADDED', bridge.current()!)+ )+ let release!: () => void+ const blocker = bridge.queue.enqueue(+ RECIPE,+ () =>+ new Promise<void>(resolve => {+ release = resolve+ })+ )+ await Promise.resolve()+ await vi.advanceTimersByTimeAsync(60_000)+ bridge.watcher.clearTransientRetry(RECIPE)+ mockCustomApi.getNamespacedCustomObject.mockClear()+ release()+ await blocker+ await bridge.drain()+ expect(mockCustomApi.getNamespacedCustomObject).not.toHaveBeenCalled()+ expect(legacyDeletes()).toBe(1)+ expect(vi.getTimerCount()).toBe(0)+ expect(bridge.queueFailures).not.toHaveBeenCalled()+ })})// A reconcile() prune whose DELETE did not land publishesdiff --git a/workflow-recipes/src/reconciler/workflowRecipeReconciler.ts b/workflow-recipes/src/reconciler/workflowRecipeReconciler.tsindex 8ce033537..1a716ffab 100644--- a/workflow-recipes/src/reconciler/workflowRecipeReconciler.ts+++ b/workflow-recipes/src/reconciler/workflowRecipeReconciler.ts@@ -64,6 +64,7 @@ import { HttpPluginWorkloadSdkRevocationClient } from '../workflow/pluginWorkloaimport { deriveWorkflowRuntimePlan } from '../workflow/runtimePlan'import { validateWorkflowRecipeLimits } from '../workflow/workflowLimits'import {+ LegacyNetworkPolicyDeletePendingError,NETWORK_POLICY_OWNERSHIP_CONDITION_TYPES,type NetworkPolicyMarkerWriter,WORKFLOW_OUTPUT_CONDITION_TYPES,@@ -689,8 +690,18 @@ export interface ReconcileResult {* false/undefined so genuine flakiness still backs off.*/requeueFixedInterval?: boolean+ /** A legacy-only timer must not repeat the recipe's runtime lifecycle. */+ requeueOperation?: 'legacy-network-policy-delete'}+type LegacyNetworkPolicyDeleteRetryResult = Pick<+ ReconcileResult,+ | 'requeueAfterMs'+ | 'requeueFixedInterval'+ | 'requeueOperation'+ | 'networkPolicyOwnershipConditions'+>+/*** Base requeue delay for a transient (skipStatusPatch) reconcile result. The* watcher backs off exponentially from here up to its own cap.@@ -1097,7 +1108,7 @@ export class WorkflowRecipeReconciler {*/private async cleanupPluginWorkloadSdkOrThrow(recipeName: string,- options: { preserveWorkflowRuntime?: boolean; recipeUid?: string } = {}+ options: { preserveWorkflowRuntime?: boolean; recipeUid?: string; recipeDeleted?: boolean } = {}): Promise<void> {if (!this.workflowReconciler) {throw new Error('workflow subsystem is not initialized; SDK cleanup cannot be confirmed')@@ -1550,6 +1561,90 @@ export class WorkflowRecipeReconciler {await this.publishNetworkPolicyMarkerWriter(recipe, { kind: 'legacy-retry', legacy })}+ /**+ * The timer's only operation: retry the shared legacy DELETE and publish its+ * condition group. Reread after the asynchronous DELETE so a replacement CR,+ * deletion or newer policy facts cannot receive this snapshot's status.+ * Resource-version conflicts retry through the watcher's bounded timer; the+ * shared gone ledger prevents another DELETE while that patch converges.+ */+ async retryPendingLegacyNetworkPolicyDeleteOnly(+ recipe: WorkflowRecipeCRD+ ): Promise<LegacyNetworkPolicyDeleteRetryResult> {+ if (+ !this.workflowReconciler ||+ !recipe.metadata.uid ||+ recipe.metadata.namespace !== this.config.sandboxNamespace ||+ recipe.metadata.deletionTimestamp ||+ !hasLegacyNetworkPolicyDeletePendingMarker(recipe.status?.conditions)+ )+ return {}++ const legacy = await this.workflowReconciler.retryLegacyMcpServersInternetEgressDelete(+ recipe.metadata.name,+ recipe.metadata.uid+ )+ let live: WorkflowRecipeCRD+ try {+ live = (await this.customApi.getNamespacedCustomObject({+ group: CRD_GROUP,+ version: CRD_VERSION,+ namespace: recipe.metadata.namespace,+ plural: WORKFLOWRECIPE_PLURAL,+ name: recipe.metadata.name,+ })) as WorkflowRecipeCRD+ } catch (error) {+ if (getErrorCode(error) === 404) return {}+ throw error+ }+ if (+ live.metadata.uid !== recipe.metadata.uid ||+ live.metadata.name !== recipe.metadata.name ||+ live.metadata.namespace !== recipe.metadata.namespace ||+ live.metadata.deletionTimestamp ||+ !hasLegacyNetworkPolicyDeletePendingMarker(live.status?.conditions)+ )+ return {}++ if (legacy === 'pending') {+ const result: LegacyNetworkPolicyDeleteRetryResult = {}+ this.requeueForPendingLegacyNetworkPolicyDelete(live, result)+ return result+ }+ const fresh = networkPolicyMarkerConditions(+ { kind: 'legacy-retry', legacy },+ live.status?.conditions,+ new Date().toISOString()+ )+ const conditions =+ mergeOwnedConditions(+ live.status?.conditions,+ fresh,+ NETWORK_POLICY_OWNERSHIP_CONDITION_TYPES+ ) ?? []+ if (!live.metadata.resourceVersion) {+ throw new RetryableReconcileError(+ 'Cannot publish legacy policy retry without recipe resourceVersion'+ )+ }+ await this.customApi.patchNamespacedCustomObjectStatus(+ {+ group: CRD_GROUP,+ version: CRD_VERSION,+ namespace: live.metadata.namespace,+ plural: WORKFLOWRECIPE_PLURAL,+ name: live.metadata.name,+ body: {+ metadata: { uid: live.metadata.uid, resourceVersion: live.metadata.resourceVersion },+ status: { conditions },+ },+ },+ { middleware: [k8s.setHeaderMiddleware('Content-Type', 'application/merge-patch+json')] }+ )+ if (recipe.status) recipe.status.conditions = conditions+ return {}+ }+/*** R4-L4: a pass that leaves `DeletePending` published and asks for no other* requeue is requeued at a fixed interval for the rest of the legacy@@ -1559,9 +1654,15 @@ export class WorkflowRecipeReconciler {*/private requeueForPendingLegacyNetworkPolicyDelete(recipe: WorkflowRecipeCRD,- result: ReconcileResult+ result: LegacyNetworkPolicyDeleteRetryResult): void {- if (!this.workflowReconciler || result.requeueAfterMs !== undefined) return+ if (+ !this.workflowReconciler ||+ recipe.metadata.namespace !== this.config.sandboxNamespace ||+ recipe.metadata.deletionTimestamp ||+ result.requeueAfterMs !== undefined+ )+ returnconst conditions = result.networkPolicyOwnershipConditions ?? recipe.status?.conditionsif (!hasLegacyNetworkPolicyDeletePendingMarker(conditions)) returnresult.requeueAfterMs = Math.max(@@ -1572,6 +1673,7 @@ export class WorkflowRecipeReconciler {))result.requeueFixedInterval = true+ result.requeueOperation = 'legacy-network-policy-delete'}/**@@ -1822,6 +1924,7 @@ export class WorkflowRecipeReconciler {const ns = recipe.metadata.namespaceconst currentPhase = recipe.status?.phase ?? 'candidate'let sdkOnlyProviderUnavailable = false+ let sdkOnlyCapabilityRemovalTeardownConfirmed = false// Defense-in-depth: the VAP + admin-API already enforce this invariant,// but a manually-applied CRD in another namespace must not be reconciled.@@ -1855,6 +1958,7 @@ export class WorkflowRecipeReconciler {try {await this.cleanupPluginWorkloadSdkOrThrow(name, {preserveWorkflowRuntime: isWorkflow,+ recipeUid: recipe.metadata.uid,})} catch (error) {return {@@ -1862,7 +1966,11 @@ export class WorkflowRecipeReconciler {message: `Plugin Workload SDK teardown failed while the feature flag is disabled: ${String(error)}`,workloadStatuses: [],skipStatusPatch: true,- requeueAfterMs: TRANSIENT_REQUEUE_BASE_MS,+ requeueAfterMs:+ error instanceof LegacyNetworkPolicyDeletePendingError+ ? error.retryAfterMs+ : TRANSIENT_REQUEUE_BASE_MS,+ requeueFixedInterval: error instanceof LegacyNetworkPolicyDeletePendingError,}}return {@@ -1894,6 +2002,7 @@ export class WorkflowRecipeReconciler {try {await this.cleanupPluginWorkloadSdkOrThrow(name, {preserveWorkflowRuntime: isWorkflow,+ recipeUid: recipe.metadata.uid,})} catch (error) {return {@@ -1901,7 +2010,11 @@ export class WorkflowRecipeReconciler {message: `Plugin Workload SDK teardown failed after capability removal: ${String(error)}`,workloadStatuses: [],skipStatusPatch: true,- requeueAfterMs: TRANSIENT_REQUEUE_BASE_MS,+ requeueAfterMs:+ error instanceof LegacyNetworkPolicyDeletePendingError+ ? error.retryAfterMs+ : TRANSIENT_REQUEUE_BASE_MS,+ requeueFixedInterval: error instanceof LegacyNetworkPolicyDeletePendingError,}}// A stepless recipe managed policies only through the SDK, and the@@ -1909,6 +2022,7 @@ export class WorkflowRecipeReconciler {// terminal and validation returns below never reach the SDK-only lane.// A workflow recipe keeps its run lane, which still manages policies.if (!isWorkflow) {+ sdkOnlyCapabilityRemovalTeardownConfirmed = trueawait this.publishNetworkPolicyMarkerWriter(recipe, { kind: 'unmanaged' })}}@@ -3147,12 +3261,9 @@ export class WorkflowRecipeReconciler {sdkOnlyRuntime?.phase === 'active' &&recipe.spec.pluginWorkloadSdk !== undefined &&sdkOnlyRuntime.pluginWorkloadSdkBootstrapProof?.ready !== true- // Without an SDK runtime this lane reaches no policy: the spec has no- // SDK block, and the capability-removal teardown deleted every SDK- // policy, the legacy one by name included. That is the unmanaged- // writer, which clears the whole group. A runtime without a summary did- // not evaluate the policies (the host returned before the apply), so- // the field stays absent and patchStatus keeps what was published.+ // Only a completed capability-removal teardown proves this lane manages+ // no policy. A missing runtime alone evaluated nothing and must preserve+ // the published apply, prune, ownership and legacy-delete facts.const now = new Date().toISOString()const {networkPolicyRetryPending: sdkOnlyNetworkPolicyRetryPending,@@ -3163,13 +3274,15 @@ export class WorkflowRecipeReconciler {recipe.status?.conditions,now)- : {- networkPolicyOwnershipConditions: networkPolicyMarkerConditions(- { kind: 'unmanaged' },- recipe.status?.conditions,- now- ),- }+ : sdkOnlyCapabilityRemovalTeardownConfirmed+ ? {+ networkPolicyOwnershipConditions: networkPolicyMarkerConditions(+ { kind: 'unmanaged' },+ recipe.status?.conditions,+ now+ ),+ }+ : {}if (sdkOnlyRuntime?.phase === 'failed') {return {phase: 'failed',@@ -4059,7 +4172,11 @@ export class WorkflowRecipeReconciler {// the shared mcp-host/token resources. Hybrid recipes must not bypass// broker revocation merely because they also have coordinator state.if (recipe.spec.pluginWorkloadSdk || recipe.status?.pluginWorkloadSdk) {- await this.cleanupPluginWorkloadSdkOrThrow(name, { preserveWorkflowRuntime: true })+ await this.cleanupPluginWorkloadSdkOrThrow(name, {+ preserveWorkflowRuntime: true,+ recipeUid: recipe.metadata.uid,+ recipeDeleted: true,+ })}await this.workflowReconciler.reconcileDelete(name,@@ -4076,7 +4193,10 @@ export class WorkflowRecipeReconciler {// a capability may have been removed in an update before Kubernetes emits// the final delete event, and cleanup must still revoke its host tokens.if (recipe.spec.pluginWorkloadSdk || recipe.status?.pluginWorkloadSdk) {- await this.cleanupPluginWorkloadSdkOrThrow(name, { recipeUid: recipe.metadata.uid })+ await this.cleanupPluginWorkloadSdkOrThrow(name, {+ recipeUid: recipe.metadata.uid,+ recipeDeleted: true,+ })}await this.cleanupDelegationIfNeeded(recipe)diff --git a/workflow-recipes/src/workflow/workflowReconciler.ts b/workflow-recipes/src/workflow/workflowReconciler.tsindex 3aee38c43..6b0058524 100644--- a/workflow-recipes/src/workflow/workflowReconciler.ts+++ b/workflow-recipes/src/workflow/workflowReconciler.ts@@ -266,11 +266,30 @@ export interface PluginWorkloadSdkCleanupOptions {*/preserveWorkflowRuntime?: boolean/**- * Set when the recipe itself is being deleted. An SDK-only recipe never- * reaches `reconcileDelete`, so this is the only place its legacy internet- * policy ledger entry can be forgotten.+ * Identity for the live legacy-delete ledger. It shares the gone-set and+ * backoff with `reconcile()` and the WRC retry; it never erases either.*/recipeUid?: string+ /**+ * True only while the recipe itself is being deleted. A completed cleanup+ * may then drop the process-local legacy ledger entry; a pending cleanup+ * must retain it so the finalizer retry resumes from the same window.+ */+ recipeDeleted?: boolean+}++export class LegacyNetworkPolicyDeletePendingError extends Error {+ constructor(+ recipeName: string,+ readonly retryAfterMs: number+ ) {+ super(`Legacy mcp-servers internet egress NetworkPolicy delete is pending for ${recipeName}`)+ this.name = 'LegacyNetworkPolicyDeletePendingError'+ }+}++function buildLegacyMcpServersInternetEgressPolicyName(recipeName: string): string {+ return `${recipeName}-mcp-servers-egress-internet`}export const NETWORK_POLICY_OWNERSHIP_CONDITION_TYPE = 'WorkflowNetworkPolicyOwnership'@@ -293,8 +312,8 @@ export const NETWORK_POLICY_OWNERSHIP_CONDITION_TYPE = 'WorkflowNetworkPolicyOwn** Each combination with a fact that is not converged has its own reason* (`RetryPending`, `PrunePending`, `PruneUnevaluated`, `RetryAndPrunePending`,- * `RetryAndPruneUnevaluated`), and the facts are read back from the reason- * alone.+ * `RetryAndPruneUnevaluated`), and current facts are read back from the reason.+ * The frozen historical combined marker is normalized by the decoder below.*/export const NETWORK_POLICIES_CONVERGED_CONDITION_TYPE = 'WorkflowNetworkPoliciesConverged'@@ -310,8 +329,9 @@ const CONVERGED_NETWORK_POLICY_MARKER_FACTS = {} as const satisfies NetworkPolicyMarkerFacts/**- * One reason per combination of facts. The facts are read back from the- * reason alone; `message` is prose for operators and is never compared.+ * One current reason per combination of facts. Current messages are prose for+ * operators. The decoder separately recognizes the exact historical combined+ * marker emitted before those facts had distinct reasons.*/const NETWORK_POLICY_MARKER_REASONS = {RetryPending: {@@ -340,6 +360,15 @@ const NETWORK_POLICY_MARKER_REASONS = {},} as const satisfies Record<string, { facts: NetworkPolicyMarkerFacts; message: string }>+/**+ * The only historical message this controller interprets. Before every fact+ * had its own reason, `af0152924` encoded apply-plus-prune as `RetryPending`+ * with exactly this prose. Equality is deliberate: substring or reworded+ * messages do not gain a prune fact.+ */+const HISTORICAL_RETRY_AND_PRUNE_PENDING_MESSAGE =+ 'One or more run-lane NetworkPolicies are pending a retry (terminating or contended), and one or more the spec no longer wants are pending a delete'+type NetworkPolicyMarkerReason = keyof typeof NETWORK_POLICY_MARKER_REASONSfunction isNetworkPolicyMarkerReason(@@ -477,6 +506,12 @@ export function networkPolicyMarkerFacts(if (!isNetworkPolicyMarkerReason(marker?.reason)) {return { ...CONVERGED_NETWORK_POLICY_MARKER_FACTS }}+ if (+ marker.reason === 'RetryPending' &&+ marker.message === HISTORICAL_RETRY_AND_PRUNE_PENDING_MESSAGE+ ) {+ return { apply: 'pending', prune: 'pending' }+ }return { ...NETWORK_POLICY_MARKER_REASONS[marker.reason].facts }}@@ -1858,9 +1893,6 @@ export class WorkflowReconciler {const ns = this.deps.config.sandboxNamespaceconst preserveWorkflowRuntime = options.preserveWorkflowRuntime === truethis.pluginWorkloadSdkProvisioner.clearRecipeState(recipeName)- if (options.recipeUid) {- this.forgetLegacyMcpServersInternetEgress(recipeName, options.recipeUid)- }const sdkNetworkPolicyNames = [`${recipeName}-workload-to-mcp-host-sdk-ingress`,`${recipeName}-workload-to-mcp-host-sdk-egress`,@@ -1936,24 +1968,24 @@ export class WorkflowReconciler {this.teardownDelete(() =>this.deleteNetworkPoliciesByLabelSelector(ns,- `clerum.io/recipe=${recipeName},clerum.io/managed-by=wrc`+ `clerum.io/recipe=${recipeName},clerum.io/managed-by=wrc`,+ buildLegacyMcpServersInternetEgressPolicyName(recipeName))),this.teardownDelete(() =>this.deleteNetworkPoliciesByLabelSelector(this.deps.config.mcpServerNamespace,- `clerum.io/recipe=${recipeName},clerum.io/managed-by=wrc`+ `clerum.io/recipe=${recipeName},clerum.io/managed-by=wrc`,+ buildLegacyMcpServersInternetEgressPolicyName(recipeName))),- // The legacy internet policy may carry no labels, so the sweep- // above cannot be trusted to remove it. Deleting it by name is- // what lets the caller publish the unmanaged marker writer after- // this teardown: any non-404 failure rejects the teardown.- this.teardownDelete(() =>- this.deps.networkingApi.deleteNamespacedNetworkPolicy({- name: `${recipeName}-mcp-servers-egress-internet`,- namespace: this.deps.config.mcpServerNamespace,- })+ // The legacy internet policy may carry no labels, while a labeled+ // copy is excluded from the sweeps above. Both forms go through+ // the one live ledger so a completed cleanup can prove `unmanaged`.+ this.cleanupLegacyMcpServersInternetEgress(+ recipeName,+ options.recipeUid,+ options.recipeDeleted === true),]),])@@ -2009,6 +2041,9 @@ export class WorkflowReconciler {)}}+ if (options.recipeDeleted === true) {+ this.forgetLegacyMcpServersInternetEgress(recipeName, options.recipeUid)+ }}private async assertPluginWorkloadSdkResourcesAbsent(@@ -3390,7 +3425,7 @@ export class WorkflowReconciler {* applies it, but deletes nothing: the run's pods are live and may still use* any lane the spec wanted when they were created, so a revoke here cuts* their traffic. That includes the legacy mcp-servers internet policy, which- * only a reconcile() pass deletes. The summary is apply-only, so this path+ * this path keeps. The summary is apply-only, so this path* is the `apply-retry` writer of Table A below: it observes the apply and* carries the published prune and legacy facts.*@@ -3431,26 +3466,37 @@ export class WorkflowReconciler {* | `unmanaged` (kill switch, SDK lane without a runtime, capability removal) | converged | converged | removed | everything |* | finalizer | the object is deleted | | | |*- * The legacy retry runs in the WRC after the early token reap and before- * every short-circuit, only while `DeletePending` is published. It and the- * reconcile() pass share one backoff per recipe: after a DELETE that did+ * The legacy retry runs in the WRC before limit validation, every+ * short-circuit and the early token reap, only while `DeletePending` is+ * published. It, the reconcile() pass and a full SDK cleanup share one+ * backoff per recipe: after a DELETE that did* not land, the next one waits `min(prev * 2 || 60 s, 1 h)`, and inside* that window the delete answers `pending` without a request. A `removed`* answer clears L with a conditions-only patch; `pending` writes nothing.- * The WRC `reconcile()` wrapper requeues a pass that leaves L pending and- * asks for no other requeue after `max(1 s, the rest of the window)`, at a- * fixed interval.+ * The watcher's legacy-only callback requeues after the rest of the window;+ * it does not force a full reconcile. A cleanup that must remove the legacy+ * policy but observes `pending` fails before `unmanaged` and requeues the+ * full cleanup at the same window. Finalizer cleanup is the last chance:+ * it sends through the same ledger even inside a window, then records and+ * forgets state only after the complete cleanup succeeds.** A carrying writer keeps a marker whose reason it did not write as it is* and warns; `reconcile` overwrites it. `unmanaged` is written only after* `cleanupPluginWorkloadSdk` without `preserveWorkflowRuntime` succeeded:- * it sweeps the recipe labels and deletes the legacy policy by name, and a- * failure there fails the pass before the writer runs. A pending or+ * its label sweeps exclude the legacy name, and the shared live ledger+ * deletes that name once; a failure or backoff fails the pass before the+ * writer runs. A pending or* unevaluated prune is not retried outside reconcile(): only reconcile()* can compute the desired set, and the finalizer's label sweep removes* every run-lane leftover. On the finalizer, a non-404 DELETE is logged by* `safeDelete` and not rethrown, so the finalizer completes.*+ * Historical `af0152924` markers are read only from their reason, except+ * the one exact `RetryPending` combined message defined beside the decoder.+ * Its ambiguous `PrunePending` cannot be safely promoted to L: current+ * ordinary prune failures use the same reason, so they require a real+ * reconcile() observation before the legacy retry can run.+ ** A desired lane is applied on both apply paths; an uncertain verdict never* creates a proxy. Terminal teardown deletes a run-scoped run's compute pods* and its cross-namespace transport workloads, and, for every terminal@@ -3700,6 +3746,24 @@ export class WorkflowReconciler {}}+ private async cleanupLegacyMcpServersInternetEgress(+ recipeName: string,+ recipeUid?: string,+ lastChance = false+ ): Promise<void> {+ const fact = await this.pruneLegacyMcpServersInternetEgressPolicy(+ recipeName,+ recipeUid,+ lastChance+ )+ if (fact === 'pending') {+ throw new LegacyNetworkPolicyDeletePendingError(+ recipeName,+ Math.max(1, this.legacyMcpServersInternetEgressRetryDelayMs(recipeName, recipeUid ?? ''))+ )+ }+ }+/*** R4-L4: the legacy delete the WRC retries before its short-circuits while* `DeletePending` is published (the `legacy-retry` writer). The same delete@@ -3728,7 +3792,8 @@ export class WorkflowReconciler {private async pruneLegacyMcpServersInternetEgressPolicy(recipeName: string,- recipeUid?: string+ recipeUid?: string,+ lastChance = false): Promise<NetworkPolicyLegacyFact> {const key = this.legacyMcpServersInternetEgressKey(recipeName, recipeUid)if (this.prunedLegacyMcpServersInternetEgress.has(key)) {@@ -3738,7 +3803,7 @@ export class WorkflowReconciler {return 'removed'}const backoff = this.legacyMcpServersInternetEgressBackoff.get(key)- if (backoff !== undefined && Date.now() < backoff.notBeforeMs) {+ if (backoff !== undefined && !lastChance && Date.now() < backoff.notBeforeMs) {this.log.debug('Skipping legacy mcp-servers internet NP delete; inside its backoff window', {recipe: recipeName,retryInMs: backoff.notBeforeMs - Date.now(),@@ -3747,7 +3812,7 @@ export class WorkflowReconciler {}const outcome = await observeNamespacedDelete(() =>this.deps.networkingApi.deleteNamespacedNetworkPolicy({- name: `${recipeName}-mcp-servers-egress-internet`,+ name: buildLegacyMcpServersInternetEgressPolicyName(recipeName),namespace: this.deps.config.mcpServerNamespace,}))@@ -4453,7 +4518,7 @@ export class WorkflowReconciler {// List-by-label is the cleanest cleanup — all managed NPs have recipe label.// Per-mcp-server NPs are deleted via label selector below. The legacy// internet policy predates those labels, so it is deleted by name (R4-L4).- const mcpServerNpNames: string[] = [`${recipeName}-mcp-servers-egress-internet`]+ const mcpServerNpNames: string[] = [buildLegacyMcpServersInternetEgressPolicyName(recipeName)]const mcpNs = this.deps.config.mcpServerNamespaceconst cleanupTasks: Array<{ label: string; run: () => Promise<void> }> = [@@ -4672,7 +4737,8 @@ export class WorkflowReconciler {private async deleteNetworkPoliciesByLabelSelector(namespace: string,- labelSelector: string+ labelSelector: string,+ excludeName?: string): Promise<void> {const list = await this.deps.networkingApi.listNamespacedNetworkPolicy({namespace,@@ -4682,6 +4748,7 @@ export class WorkflowReconciler {(list.items ?? []).map(policy => policy.metadata?.name).filter((name): name is string => Boolean(name))+ .filter(name => name !== excludeName).map(async name => {try {await this.deps.networkingApi.deleteNamespacedNetworkPolicy({ name, namespace })diff --git a/workflow-recipes/tests/unit/workflow/workflowReconciler.test.ts b/workflow-recipes/tests/unit/workflow/workflowReconciler.test.tsindex 461e25314..1acac866d 100644--- a/workflow-recipes/tests/unit/workflow/workflowReconciler.test.ts+++ b/workflow-recipes/tests/unit/workflow/workflowReconciler.test.ts@@ -36,6 +36,8 @@ import {} from '../../../src/workflow/resourceNames'import { deriveWorkflowRuntimePlan } from '../../../src/workflow/runtimePlan'import {+ LEGACY_NETWORK_POLICY_DELETE_BACKOFF_BASE_MS,+ LegacyNetworkPolicyDeletePendingError,type WorkflowRecipeSpec,WorkflowReconciler,type WorkflowReconcilerDeps,@@ -9223,6 +9225,152 @@ describe('WorkflowReconciler — reconcile loop', () => {})})+describe('cleanupPluginWorkloadSdk legacy internet policy ownership', () => {+ const RECIPE = 'sdk-only'+ const RECIPE_UID = 'uid-sdk-only'+ const LEGACY = `${RECIPE}-mcp-servers-egress-internet`++ function legacyDeletes(networkingApi: ReturnType<typeof makeNetworkingApi>): number {+ return networkingApi.deleteNamespacedNetworkPolicy.mock.calls.filter(+ ([arg]) => arg.name === LEGACY+ ).length+ }++ function makeCleanupCoreApi(): ReturnType<typeof makeCoreApi> {+ const coreApi = makeCoreApi() as ReturnType<typeof makeCoreApi>+ coreApi.readNamespacedEndpoints = vi.fn().mockRejectedValue({ code: 404 })+ return coreApi+ }++ it('routes a labeled legacy policy through the shared live ledger instead of the label sweep', async () => {+ const networkingApi = makeNetworkingApi() as ReturnType<typeof makeNetworkingApi>+ let legacyPresent = true+ networkingApi.listNamespacedNetworkPolicy.mockImplementation(+ async ({ namespace }: { namespace: string }) => ({+ items:+ namespace === 'mcp-server'+ ? legacyPresent+ ? [+ {+ metadata: {+ name: LEGACY,+ namespace,+ labels: { 'clerum.io/recipe': RECIPE, 'clerum.io/managed-by': 'wrc' },+ },+ },+ ]+ : []+ : [],+ })+ )+ networkingApi.deleteNamespacedNetworkPolicy.mockImplementation(async ({ name }) => {+ if (name === LEGACY) legacyPresent = false+ return {}+ })+ const coreApi = makeCoreApi() as ReturnType<typeof makeCoreApi>+ coreApi.readNamespacedEndpoints = vi.fn().mockRejectedValue({ code: 404 })+ const reconciler = new WorkflowReconciler(+ makeDeps({+ coreApi: coreApi as never,+ networkingApi: networkingApi as never,+ pluginWorkloadSdkRevocationClient: {+ revoke: vi.fn().mockResolvedValue({ state: 'disabled' }),+ } as never,+ })+ )++ await reconciler.cleanupPluginWorkloadSdk(RECIPE, { recipeUid: RECIPE_UID })++ // Liveness witness: both label-sweep namespaces were evaluated, and the+ // shared by-name path sent the sole legacy DELETE.+ expect(networkingApi.listNamespacedNetworkPolicy).toHaveBeenCalledWith(+ expect.objectContaining({+ namespace: 'mcp-server',+ labelSelector: `clerum.io/recipe=${RECIPE},clerum.io/managed-by=wrc`,+ })+ )+ expect(legacyDeletes(networkingApi)).toBe(1)++ await reconciler.retryLegacyMcpServersInternetEgressDelete(RECIPE, RECIPE_UID)+ expect(legacyDeletes(networkingApi)).toBe(1)+ })++ it('keeps live cleanup on the shared backoff while finalizer cleanup gets one last chance', async () => {+ vi.useFakeTimers({ toFake: ['Date'] })+ const startedAt = Date.now()+ try {+ const networkingApi = makeNetworkingApi() as ReturnType<typeof makeNetworkingApi>+ let legacyDeleteFails = true+ networkingApi.deleteNamespacedNetworkPolicy.mockImplementation(async ({ name }) => {+ if (name === LEGACY && legacyDeleteFails) throw { code: 403, message: 'forbidden' }+ return {}+ })+ const reconciler = new WorkflowReconciler(+ makeDeps({+ coreApi: makeCleanupCoreApi() as never,+ networkingApi: networkingApi as never,+ pluginWorkloadSdkRevocationClient: {+ revoke: vi.fn().mockResolvedValue({ state: 'disabled' }),+ } as never,+ })+ )++ await expect(+ reconciler.retryLegacyMcpServersInternetEgressDelete(RECIPE, RECIPE_UID)+ ).resolves.toBe('pending')+ expect(legacyDeletes(networkingApi)).toBe(1)++ const pending = await reconciler+ .cleanupPluginWorkloadSdk(RECIPE, {+ recipeUid: RECIPE_UID,+ })+ .catch(error => error)+ expect(pending).toBeInstanceOf(LegacyNetworkPolicyDeletePendingError)+ expect((pending as LegacyNetworkPolicyDeletePendingError).retryAfterMs).toBe(+ LEGACY_NETWORK_POLICY_DELETE_BACKOFF_BASE_MS+ )+ expect(legacyDeletes(networkingApi)).toBe(1)++ legacyDeleteFails = false+ await reconciler.cleanupPluginWorkloadSdk(RECIPE, {+ recipeUid: RECIPE_UID,+ recipeDeleted: true,+ })+ expect(legacyDeletes(networkingApi)).toBe(2)++ await reconciler.retryLegacyMcpServersInternetEgressDelete(RECIPE, RECIPE_UID)+ expect(legacyDeletes(networkingApi)).toBe(3)+ } finally {+ vi.useRealTimers()+ }+ })++ it('keeps the legacy policy live when cleanup preserves the workflow runtime', async () => {+ const networkingApi = makeNetworkingApi() as ReturnType<typeof makeNetworkingApi>+ const reconciler = new WorkflowReconciler(+ makeDeps({+ coreApi: makeCleanupCoreApi() as never,+ networkingApi: networkingApi as never,+ pluginWorkloadSdkRevocationClient: {+ revoke: vi.fn().mockResolvedValue({ state: 'disabled' }),+ } as never,+ })+ )++ await reconciler.cleanupPluginWorkloadSdk('hybrid', {+ recipeUid: 'uid-hybrid',+ preserveWorkflowRuntime: true,+ })++ expect(+ networkingApi.deleteNamespacedNetworkPolicy.mock.calls.filter(+ ([arg]) => arg.name === 'hybrid-mcp-servers-egress-internet'+ )+ ).toHaveLength(0)+ expect(networkingApi.listNamespacedNetworkPolicy).not.toHaveBeenCalled()+ })+})+describe('buildNetworkPolicyOwnershipConditions', () => {it('names the conflicting policies in sorted order, whatever order the apply produced them in', () => {const conditions = buildNetworkPolicyOwnershipConditions(@@ -9791,6 +9939,41 @@ describe('networkPolicyMarkerFacts', () => {)expect(new Set(reasons).size).toBe(combinations.length)})++ it('reads the one historical RetryPending message that encoded a pending prune', () => {+ const historical = {+ type: 'WorkflowNetworkPoliciesConverged',+ status: 'False' as const,+ reason: 'RetryPending',+ message:+ 'One or more run-lane NetworkPolicies are pending a retry (terminating or contended), and one or more the spec no longer wants are pending a delete',+ lastTransitionTime: now,+ }++ expect(networkPolicyMarkerFacts([historical])).toStrictEqual({+ apply: 'pending',+ prune: 'pending',+ })+ expect(+ networkPolicyMarkerFacts([{ ...historical, message: `${historical.message} ` }])+ ).toStrictEqual({+ apply: 'pending',+ prune: 'converged',+ })++ const settled = networkPolicyMarkerConditions(+ { kind: 'apply-retry', summary: { conflicts: [], retryPending: false } },+ [historical],+ now+ )+ expect(settled).toMatchObject([+ {+ type: 'WorkflowNetworkPoliciesConverged',+ status: 'False',+ reason: 'PrunePending',+ },+ ])+ })})describe('networkPolicyConditionsChanged', () => {