Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript

fix(workflow-recipes): enforce policy lifecycle and retry ownership

Alfredo Lopez5ab5316fed01
diff --git a/workflow-recipes/package-lock.json b/workflow-recipes/package-lock.json
index 35ca72de4..fabbdb163 100644
--- a/workflow-recipes/package-lock.json
+++ b/workflow-recipes/package-lock.json
@@ -1,12 +1,12 @@
{
"name": "workload-recipes-controller",
- "version": "0.1.453",
+ "version": "0.1.454",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "workload-recipes-controller",
- "version": "0.1.453",
+ "version": "0.1.454",
"license": "SEE LICENSE IN LICENSE",
"dependencies": {
"@aws-sdk/client-s3": "^3.1009.0",
diff --git a/workflow-recipes/package.json b/workflow-recipes/package.json
index f95e71b5f..c38b2ccc6 100644
--- a/workflow-recipes/package.json
+++ b/workflow-recipes/package.json
@@ -1,6 +1,6 @@
{
"name": "workload-recipes-controller",
- "version": "0.1.453",
+ "version": "0.1.454",
"description": "Workload Recipes Controller (WRC) - WorkloadRecipe lifecycle management and MCP server interface",
"main": "dist/main.js",
"engines": {
diff --git a/workflow-recipes/src/k8sClient.test.ts b/workflow-recipes/src/k8sClient.test.ts
index 4c54ce30c..812aa1b6b 100644
--- a/workflow-recipes/src/k8sClient.test.ts
+++ b/workflow-recipes/src/k8sClient.test.ts
@@ -1553,10 +1553,12 @@ describe('workload status refresh loop helpers', () => {
type InternalWatcher = {
recipes: Map<string, WorkflowRecipeCRD>
reconciler: { reconcile: typeof reconcile }
+ transientRetries: Map<string, { timer: ReturnType<typeof setTimeout>; attempts: number }>
handleRecipeEvent: (type: string, recipe: WorkflowRecipeCRD) => Promise<void>
}
const internal = Object.create(WorkflowRecipeWatcher.prototype) as InternalWatcher
internal.recipes = new Map([[cached.metadata.name, cached]])
+ internal.transientRetries = new Map()
internal.reconciler = { reconcile }
await internal.handleRecipeEvent('MODIFIED', statusOnly)
diff --git a/workflow-recipes/src/k8sClient.ts b/workflow-recipes/src/k8sClient.ts
index d7450be8b..f7b6eead6 100644
--- a/workflow-recipes/src/k8sClient.ts
+++ b/workflow-recipes/src/k8sClient.ts
@@ -701,7 +701,11 @@ export class WorkflowRecipeWatcher implements WorkflowRecipeProvider {
// result or DELETE.
private transientRetries = new Map<
string,
- { timer: ReturnType<typeof setTimeout>; attempts: number }
+ {
+ timer: ReturnType<typeof setTimeout>
+ attempts: number
+ recipeUid?: string
+ }
>()
private static readonly TRANSIENT_RETRY_MAX_MS = 60_000
@@ -1055,8 +1059,13 @@ export class WorkflowRecipeWatcher implements WorkflowRecipeProvider {
options: { forceReconcile?: boolean } = {}
): Promise<void> {
if (type === 'ADDED' || type === 'MODIFIED') {
+ const scheduledUid = this.transientRetries.get(recipe.metadata.name)?.recipeUid
+ if (scheduledUid && scheduledUid !== recipe.metadata.uid) {
+ this.clearTransientRetry(recipe.metadata.name)
+ }
let dbSyncAttempted = false
if (recipe.metadata.deletionTimestamp) {
+ this.clearTransientRetry(recipe.metadata.name)
// Resource is being deleted — cleanup cross-namespace resources, then remove finalizer.
createLogger('wrc', recipe.metadata.name).info('Cleaning up recipe finalizer', {
name: recipe.metadata.name,
@@ -1181,7 +1190,8 @@ export class WorkflowRecipeWatcher implements WorkflowRecipeProvider {
this.scheduleTransientRetry(
recipe.metadata.name,
result.requeueAfterMs,
- result.requeueFixedInterval ?? false
+ result.requeueFixedInterval ?? false,
+ result.requeueOperation
)
} else {
this.clearTransientRetry(recipe.metadata.name)
@@ -1266,17 +1276,38 @@ export class WorkflowRecipeWatcher implements WorkflowRecipeProvider {
*
* Exposed for tests via the watcher's event path.
*/
- private scheduleTransientRetry(name: string, baseMs: number, fixedInterval = false): void {
+ private scheduleTransientRetry(
+ name: string,
+ baseMs: number,
+ fixedInterval = false,
+ operation?: ReconcileResult['requeueOperation']
+ ): void {
const existing = this.transientRetries.get(name)
// Fixed-interval (progress) requeues reset the backoff counter; transient
// (error) requeues grow it.
const attempts = fixedInterval ? 0 : (existing?.attempts ?? 0)
if (existing) clearTimeout(existing.timer)
+ const owner = operation ? this.recipes.get(name) : undefined
+ if (operation && (!owner?.metadata.uid || owner.metadata.deletionTimestamp)) {
+ this.transientRetries.delete(name)
+ return
+ }
const delay = fixedInterval
? baseMs
: Math.min(baseMs * 2 ** attempts, WorkflowRecipeWatcher.TRANSIENT_RETRY_MAX_MS)
const timer = setTimeout(() => {
if (this.stopped) return
+ if (operation && owner) {
+ void this.eventQueue.enqueue(name, () =>
+ this.retryPendingLegacyNetworkPolicyDelete(
+ name,
+ owner.metadata.namespace,
+ owner.metadata.uid!,
+ timer
+ )
+ )
+ return
+ }
const cached = this.recipes.get(name)
if (!cached) {
this.transientRetries.delete(name)
@@ -1301,7 +1332,81 @@ export class WorkflowRecipeWatcher implements WorkflowRecipeProvider {
// reconcile advances past the waiting state once the awaited Pod succeeds,
// and the 240s readiness deadline / dbRunProcessor max-duration paths
// terminate any run permanently stuck in deploying, so this is bounded.
- this.transientRetries.set(name, { timer, attempts: fixedInterval ? 0 : attempts + 1 })
+ this.transientRetries.set(name, {
+ timer,
+ attempts: fixedInterval ? 0 : attempts + 1,
+ ...(operation ? { recipeUid: owner!.metadata.uid } : {}),
+ })
+ }
+
+ /**
+ * Runs after the recipe's queue wait. The API read is authoritative for UID,
+ * deletion and policy markers; this callback never invokes the full recipe
+ * pipeline, token reap or terminal runtime teardown.
+ */
+ private async retryPendingLegacyNetworkPolicyDelete(
+ name: string,
+ namespace: string,
+ uid: string,
+ timer: ReturnType<typeof setTimeout>
+ ): Promise<void> {
+ const current = () => !this.stopped && this.transientRetries.get(name)?.timer === timer
+ if (!current()) return
+ try {
+ const apiObject = (await this.customApi.getNamespacedCustomObject({
+ group: CRD_GROUP,
+ version: CRD_VERSION,
+ namespace,
+ plural: WORKFLOWRECIPE_PLURAL,
+ name,
+ })) as WorkflowRecipeWatchObject
+ if (!current()) return
+ if (
+ apiObject.metadata.name !== name ||
+ apiObject.metadata.namespace !== namespace ||
+ apiObject.metadata.uid !== uid ||
+ apiObject.metadata.deletionTimestamp
+ ) {
+ this.clearTransientRetry(name)
+ return
+ }
+ const latest = workflowRecipeFromWatchObject(apiObject, namespace)
+ const result = await this.reconciler.retryPendingLegacyNetworkPolicyDeleteOnly(latest)
+ if (!current()) return
+ const cached = this.recipes.get(name)
+ if (!cached || cached.metadata.uid !== uid || cached.metadata.deletionTimestamp) {
+ this.clearTransientRetry(name)
+ return
+ }
+ if (result.requeueAfterMs) {
+ this.scheduleTransientRetry(
+ name,
+ result.requeueAfterMs,
+ true,
+ 'legacy-network-policy-delete'
+ )
+ } else {
+ this.clearTransientRetry(name)
+ }
+ } catch (error) {
+ if (!current()) return
+ if (getErrorCode(error) === 404) {
+ this.clearTransientRetry(name)
+ return
+ }
+ createLogger('wrc', name).error('Legacy NetworkPolicy delete retry failed', {
+ name,
+ err: error,
+ })
+ // A status conflict or transient API failure keeps this same narrow
+ // operation on the existing bounded error backoff; it cannot replay runtime cleanup.
+ this.scheduleTransientRetry(
+ name,
+ TRANSIENT_REQUEUE_BASE_MS,
+ false,
+ 'legacy-network-policy-delete'
+ )
+ }
}
private clearTransientRetry(name: string): void {
diff --git a/workflow-recipes/src/reconciler/reconciler.test.ts b/workflow-recipes/src/reconciler/reconciler.test.ts
index 7e5805cf7..d6f00f38e 100644
--- a/workflow-recipes/src/reconciler/reconciler.test.ts
+++ b/workflow-recipes/src/reconciler/reconciler.test.ts
@@ -20,6 +20,7 @@ import { buildCoordinatorGfsNetworkPolicy } from '../workflow/networkPolicyFacto
import type { NetworkPolicyPassSummary } from '../workflow/pluginWorkloadSdkProvisioner'
import { deriveWorkflowRuntimePlan } from '../workflow/runtimePlan'
import {
+ LegacyNetworkPolicyDeletePendingError,
WorkflowReconciler,
buildNetworkPolicyConvergedCondition,
networkPolicyMarkerConditions,
@@ -33,6 +34,7 @@ import {
PLUGIN_WORKLOAD_SDK_POLICY_PENDING_CONDITION_TYPE,
PLUGIN_WORKLOAD_SDK_PROVIDER_UNAVAILABLE_CONDITION_TYPE,
} from './pluginWorkloadSdkValidator'
+import { RecipeEventQueue } from './recipeEventQueue'
import {
buildUiIngressNetworkPolicy,
buildWorkloadIngressNetworkPolicy,
@@ -88,6 +90,7 @@ const mockCoreApi = {
readNamespacedPersistentVolumeClaim: vi.fn().mockRejectedValue({ code: 404 }),
deleteNamespacedPersistentVolumeClaim: vi.fn().mockResolvedValue({}),
deleteCollectionNamespacedPod: vi.fn().mockResolvedValue({}),
+ deleteNamespacedPod: vi.fn().mockResolvedValue({}),
createNamespacedSecret: vi.fn().mockResolvedValue({}),
readNamespacedSecret: vi.fn().mockResolvedValue({ metadata: { resourceVersion: '1' } }),
replaceNamespacedSecret: vi.fn().mockResolvedValue({}),
@@ -1774,6 +1777,7 @@ describe('WorkflowRecipeReconciler', () => {
expect(cleanupPluginWorkloadSdk).toHaveBeenCalledTimes(1)
expect(cleanupPluginWorkloadSdk).toHaveBeenCalledWith('test-recipe', {
preserveWorkflowRuntime: false,
+ recipeUid: 'uid-123',
})
expect(sdkOnly).not.toHaveBeenCalled()
expect(result.phase).not.toBe('failed')
@@ -1827,6 +1831,55 @@ describe('WorkflowRecipeReconciler', () => {
return cleanupPluginWorkloadSdk
}
+ it.each(['SDK-never-published', 'steps-removed'] as const)(
+ 'R5-L1: %s without an SDK runtime or teardown keeps its published apply, prune and legacy facts',
+ async history => {
+ const cleanup = stubCapabilityRemovalTeardown()
+ const inner = (reconciler as unknown as { workflowReconciler: WorkflowReconciler })
+ .workflowReconciler
+ const retryLegacy = vi.fn().mockResolvedValue('pending')
+ inner.retryLegacyMcpServersInternetEgressDelete = retryLegacy
+ inner.legacyMcpServersInternetEgressRetryDelayMs = vi.fn().mockReturnValue(60_000)
+ const facts = networkPolicyMarkerConditions(
+ {
+ kind: 'reconcile',
+ summary: { conflicts: [], retryPending: true, prune: 'pending', legacy: 'pending' },
+ },
+ [],
+ '2026-09-23T10:00:00.000Z'
+ )
+ const recipe = makeRecipe({
+ spec: {
+ workloads: [
+ { id: 'app', type: 'deployment', image: 'nginx:1.30.1-alpine', port: 8080 },
+ ],
+ ...(history === 'steps-removed' ? { steps: [] } : {}),
+ },
+ status: {
+ phase: 'active',
+ conditions: [ownershipCondition, ...facts],
+ ...(history === 'steps-removed' ? { workflowExecution: { phase: 'completed' } } : {}),
+ },
+ })
+ expect(recipe.spec.pluginWorkloadSdk).toBeUndefined()
+ expect(recipe.status?.pluginWorkloadSdk).toBeUndefined()
+
+ const result = await reconciler.reconcile(recipe)
+
+ expect(retryLegacy).toHaveBeenCalledWith('test-recipe', 'uid-123')
+ expect(cleanup).not.toHaveBeenCalled()
+ expect(result.networkPolicyOwnershipConditions).toBeUndefined()
+ expect(result.requeueAfterMs).toBe(60_000)
+ expect(result.requeueOperation).toBe('legacy-network-policy-delete')
+ await reconciler.patchStatus(recipe, result)
+ const published = mockCustomApi.patchNamespacedCustomObjectStatus.mock.calls.at(-1)![0].body
+ .status.conditions as StatusCondition[]
+ for (const fact of [ownershipCondition, ...facts]) {
+ expect(published).toContainEqual(fact)
+ }
+ }
+ )
+
it('R4-L3: clears a published PrunePending marker once the SDK capability is removed from the spec', async () => {
const cleanupPluginWorkloadSdk = stubCapabilityRemovalTeardown()
const recipe = capabilityRemovedRecipe('active', [ownershipCondition, prunePendingMarker()])
@@ -1836,6 +1889,7 @@ describe('WorkflowRecipeReconciler', () => {
// Liveness witness: the capability-removal teardown ran.
expect(cleanupPluginWorkloadSdk).toHaveBeenCalledWith('test-recipe', {
preserveWorkflowRuntime: false,
+ recipeUid: 'uid-123',
})
expect(result.phase).not.toBe('failed')
expect(result.networkPolicyOwnershipConditions).toEqual([])
@@ -1850,6 +1904,7 @@ describe('WorkflowRecipeReconciler', () => {
// Liveness witness: the capability-removal teardown ran.
expect(cleanupPluginWorkloadSdk).toHaveBeenCalledWith('test-recipe', {
preserveWorkflowRuntime: false,
+ recipeUid: 'uid-123',
})
const conditionPatches = mockCustomApi.patchNamespacedCustomObjectStatus.mock.calls
.map(call => call[0].body.status)
@@ -1953,6 +2008,7 @@ describe('WorkflowRecipeReconciler', () => {
expect(result.phase).toBe('active')
expect(cleanupPluginWorkloadSdk).toHaveBeenCalledWith('test-recipe', {
preserveWorkflowRuntime: false,
+ recipeUid: 'uid-123',
})
expect(workflowReconcile).not.toHaveBeenCalled()
})
@@ -1983,6 +2039,7 @@ describe('WorkflowRecipeReconciler', () => {
expect(result.phase).toBe(phase)
expect(cleanupPluginWorkloadSdk).toHaveBeenCalledWith('test-recipe', {
preserveWorkflowRuntime: false,
+ recipeUid: 'uid-123',
})
}
)
@@ -2028,10 +2085,69 @@ describe('WorkflowRecipeReconciler', () => {
})
expect(cleanupPluginWorkloadSdk).toHaveBeenCalledWith('test-recipe', {
preserveWorkflowRuntime: false,
+ recipeUid: 'uid-123',
})
expect(workflowReconcile).not.toHaveBeenCalled()
})
+ it.each(['kill-switch', 'capability-removal'] as const)(
+ 'R5-L2: %s cleanup passes the live UID and preserves the exact legacy backoff',
+ async branch => {
+ const cleanup = vi.fn()
+ const retry = vi.fn()
+ const delay = vi.fn()
+ const internal = reconciler as unknown as {
+ config: { pluginWorkloadSdkEnabled: boolean }
+ workflowReconciler: unknown
+ }
+ internal.config.pluginWorkloadSdkEnabled = branch !== 'kill-switch'
+ internal.workflowReconciler = {
+ cleanupPluginWorkloadSdk: cleanup,
+ retryLegacyMcpServersInternetEgressDelete: retry,
+ legacyMcpServersInternetEgressRetryDelayMs: delay,
+ }
+ const conditions = networkPolicyMarkerConditions(
+ {
+ kind: 'reconcile',
+ summary: { conflicts: [], retryPending: true, prune: 'pending', legacy: 'pending' },
+ },
+ [],
+ '2026-09-23T10:00:00.000Z'
+ )
+ const recipe = makeRecipe({
+ spec: {
+ workloads: [],
+ ...(branch === 'kill-switch' ? { pluginWorkloadSdk: { allowedCallers: [] } } : {}),
+ },
+ status: {
+ phase: 'active',
+ conditions,
+ pluginWorkloadSdk: { state: 'validated', promptBridge: true, clientNotifications: false },
+ },
+ })
+ for (const remaining of [60_000, 40_123, 999]) {
+ cleanup.mockRejectedValue(
+ new LegacyNetworkPolicyDeletePendingError('test-recipe', remaining)
+ )
+ const result = await reconciler.reconcile(recipe)
+ expect(cleanup).toHaveBeenLastCalledWith('test-recipe', {
+ preserveWorkflowRuntime: false,
+ recipeUid: 'uid-123',
+ })
+ expect(result.requeueAfterMs).toBe(remaining)
+ expect(result.requeueFixedInterval).toBe(true)
+ expect(result.requeueOperation).toBeUndefined()
+ expect(result.skipStatusPatch).toBe(true)
+ expect(result.networkPolicyOwnershipConditions).toBeUndefined()
+ expect(result.pluginWorkloadSdkTeardownConfirmed).toBeUndefined()
+ expect(recipe.status?.conditions).toEqual(conditions)
+ }
+ expect(retry).not.toHaveBeenCalled()
+ expect(delay).not.toHaveBeenCalled()
+ expect(mockCustomApi.patchNamespacedCustomObjectStatus).not.toHaveBeenCalled()
+ }
+ )
+
// The kill switch returns before the SDK-only lane, which is the only path
// that clears the ownership condition. With the runtime torn down no policy
// is managed, so a conflict published earlier is stale.
@@ -7600,34 +7716,40 @@ describe('WorkflowRecipeReconciler', () => {
)
})
- // The legacy internet NP ledger is keyed by recipeUid and cleared from
- // cleanupPluginWorkloadSdk only when the uid is passed. The SDK-only delete
- // path is the one that has to hand it over.
- it('reconcileDelete hands the recipe uid to the SDK cleanup on the SDK-only path', async () => {
- const cleanupPluginWorkloadSdk = vi.fn().mockResolvedValue(undefined)
- ;(
- reconciler as unknown as {
- workflowReconciler: { cleanupPluginWorkloadSdk: typeof cleanupPluginWorkloadSdk }
- }
- ).workflowReconciler = { cleanupPluginWorkloadSdk }
- const recipe = makeRecipe({
- metadata: { name: 'test-recipe', namespace: 'sandbox-recipes', uid: 'uid-sdk-delete' },
- spec: {
- workloads: [{ id: 'app', type: 'deployment', image: 'nginx:1.30.1-alpine', port: 8080 }],
- },
- status: {
- phase: 'active',
- pluginWorkloadSdk: { state: 'validated', promptBridge: true, clientNotifications: false },
- },
- } as Partial<WorkflowRecipeCRD>)
+ // UID identifies the shared ledger. Only a real finalizer may forget it after
+ // the last-chance cleanup succeeds; a live teardown must retain that history.
+ it.each([false, true])(
+ 'R5-L2: reconcileDelete identifies finalizer SDK cleanup for workflow=%s',
+ async workflow => {
+ const cleanupPluginWorkloadSdk = vi.fn().mockResolvedValue(undefined)
+ const deleteWorkflow = vi.fn().mockResolvedValue(undefined)
+ ;(
+ reconciler as unknown as {
+ workflowReconciler: { cleanupPluginWorkloadSdk: typeof cleanupPluginWorkloadSdk }
+ }
+ ).workflowReconciler = { cleanupPluginWorkloadSdk, reconcileDelete: deleteWorkflow }
+ const recipe = makeRecipe({
+ metadata: { name: 'test-recipe', namespace: 'sandbox-recipes', uid: 'uid-sdk-delete' },
+ spec: {
+ workloads: [{ id: 'app', type: 'deployment', image: 'nginx:1.30.1-alpine', port: 8080 }],
+ ...(workflow ? { steps: [{ id: 'task', instruction: 'run' }] } : {}),
+ },
+ status: {
+ phase: 'active',
+ pluginWorkloadSdk: { state: 'validated', promptBridge: true, clientNotifications: false },
+ },
+ } as Partial<WorkflowRecipeCRD>)
- await reconciler.reconcileDelete(recipe)
+ await reconciler.reconcileDelete(recipe)
- expect(cleanupPluginWorkloadSdk).toHaveBeenCalledTimes(1)
- expect(cleanupPluginWorkloadSdk).toHaveBeenCalledWith('test-recipe', {
- recipeUid: 'uid-sdk-delete',
- })
- })
+ expect(cleanupPluginWorkloadSdk).toHaveBeenCalledTimes(1)
+ expect(cleanupPluginWorkloadSdk).toHaveBeenCalledWith('test-recipe', {
+ recipeUid: 'uid-sdk-delete',
+ recipeDeleted: true,
+ ...(workflow ? { preserveWorkflowRuntime: true } : {}),
+ })
+ }
+ )
// ─── Phase 8: Namespace Splitting ─────────────────────────────────
@@ -15731,53 +15853,74 @@ describe('WorkflowRecipeReconciler', () => {
})
}
- it('R4-L1: after the finalizer deleted the token, a recreated recipe sends no DELETE until a new token ADDED, then even below the old watermark', async () => {
- const recipe = reapRecipe(4)
- tokenSeen()
- await reconciler.ensureOAuthBrokerTokenSecret(recipe)
- await reconciler.ensureOAuthBrokerTokenSecret(recipe)
- expect(secretDeletes()).toBe(1)
+ it.each(['2xx', '404'] as const)(
+ 'R5-L6: after the finalizer observed token %s, a recreated recipe sends no DELETE until a new token ADDED, then even below the old watermark',
+ async outcome => {
+ const recipe = reapRecipe(4)
+ tokenSeen()
+ await reconciler.ensureOAuthBrokerTokenSecret(recipe)
+ await reconciler.ensureOAuthBrokerTokenSecret(recipe)
+ expect(secretDeletes()).toBe(1)
- await reconciler.reconcileDelete(recipe)
- // Witness for the premise: the finalizer sent the token DELETE, and the
- // default mock answers it 2xx.
- expect(secretDeletes()).toBe(2)
- mockCoreApi.deleteNamespacedSecret.mockClear()
+ if (outcome === '404') failTokenDelete(404)
+ try {
+ await reconciler.reconcileDelete(recipe)
+ } finally {
+ mockCoreApi.deleteNamespacedSecret.mockResolvedValue({})
+ }
+ // The real finalizer sent the token DELETE and mapped its observed result.
+ expect(secretDeletes()).toBe(2)
+ mockCoreApi.deleteNamespacedSecret.mockClear()
- const debugSpy = captureLogger('debug')
- try {
- await reconciler.ensureOAuthBrokerTokenSecret(recreatedRecipe())
- await reconciler.ensureOAuthBrokerTokenSecret(recreatedRecipe())
- expect(secretDeletes()).toBe(0)
- // Liveness witness: both passes reached the ledger and it declined.
- expect(
- debugSpy.mock.calls.filter(
- ([message, fields]) =>
- typeof message === 'string' &&
- message.startsWith('Skipping oauth-broker-token delete') &&
- (fields as { generation?: number }).generation === 1
- )
- ).toHaveLength(2)
- } finally {
- debugSpy.mockRestore()
- }
+ const debugSpy = captureLogger('debug')
+ try {
+ await reconciler.ensureOAuthBrokerTokenSecret(recreatedRecipe())
+ await reconciler.ensureOAuthBrokerTokenSecret(recreatedRecipe())
+ expect(secretDeletes()).toBe(0)
+ // Liveness witness: both passes reached the ledger and it declined.
+ expect(
+ debugSpy.mock.calls.filter(
+ ([message, fields]) =>
+ typeof message === 'string' &&
+ message.startsWith('Skipping oauth-broker-token delete') &&
+ (fields as { generation?: number }).generation === 1
+ )
+ ).toHaveLength(2)
+ } finally {
+ debugSpy.mockRestore()
+ }
- // A token ADDED for a recipe under the same name: the old watermark is
- // gone, so a lower generation deletes.
- tokenSeen()
- await reconciler.ensureOAuthBrokerTokenSecret(reapRecipe(3))
- expect(secretDeletes()).toBe(1)
- })
+ // A token ADDED for a recipe under the same name: the old watermark is
+ // gone, so a lower generation deletes.
+ tokenSeen()
+ await reconciler.ensureOAuthBrokerTokenSecret(reapRecipe(3))
+ expect(secretDeletes()).toBe(1)
+ }
+ )
- it('R4-L1: a finalizer whose token DELETE failed keeps the token seen, so a recreated recipe deletes it once', async () => {
+ it('R4-L1/R5-L7: a failed finalizer token DELETE preserves its error object and keeps the token seen', async () => {
tokenSeen()
- failTokenDelete(500)
+ const error = { code: 500, message: 'delete failed' }
+ const errorLog = captureLogger('error')
+ mockCoreApi.deleteNamespacedSecret.mockImplementation(async ({ name }: { name: string }) => {
+ if (name === SECRET_NAME) throw error
+ return {}
+ })
try {
await reconciler.reconcileDelete(reapRecipe(4))
// Witness: the finalizer sent the token DELETE and it failed.
expect(secretDeletes()).toBe(1)
+ expect(errorLog).toHaveBeenCalledWith('Failed to delete resource', {
+ label: `Secret "${SECRET_NAME}" in sandbox-recipes`,
+ err: error,
+ })
+ const failure = errorLog.mock.calls.find(
+ ([message]) => message === 'Failed to delete resource'
+ )
+ expect((failure?.[1] as { err?: unknown }).err).toBe(error)
} finally {
mockCoreApi.deleteNamespacedSecret.mockResolvedValue({})
+ errorLog.mockRestore()
}
mockCoreApi.deleteNamespacedSecret.mockClear()
@@ -15790,8 +15933,8 @@ describe('WorkflowRecipeReconciler', () => {
it('R4-L1: a token ADDED while the finalizer DELETE is in flight keeps the token seen', async () => {
tokenSeen()
mockCoreApi.deleteNamespacedSecret.mockImplementation(async ({ name }: { name: string }) => {
- // The recipe was recreated and its token ADDED was handled before the
- // finalizer's DELETE of the old token returned.
+ // A late ADDED delivery (or a restart's relist) was handled before the
+ // finalizer's DELETE returned; it does not imply a concurrent CR recreation.
if (name === SECRET_NAME) reconciler.invalidateOAuthBrokerSecretLedger('test-recipe')
return {}
})
@@ -16494,6 +16637,7 @@ describe('WorkflowRecipeReconciler', () => {
afterEach(() => {
vi.useRealTimers()
+ mockCustomApi.patchNamespacedCustomObjectStatus.mockReset().mockResolvedValue({})
})
function runScopedRecipe(status: Record<string, unknown>): WorkflowRecipeCRD {
@@ -16736,6 +16880,393 @@ describe('WorkflowRecipeReconciler', () => {
expect(result.requeueAfterMs).toBe(60_000)
expect(result.requeueFixedInterval).toBe(true)
})
+
+ function realWatcherBridge(recipe: WorkflowRecipeCRD) {
+ // The surrounding ledger-only cases fake Date; this bridge drives real timers too.
+ vi.useRealTimers()
+ vi.useFakeTimers()
+ vi.setSystemTime(T0)
+ let live: WorkflowRecipeCRD | undefined = structuredClone(recipe)
+ const previousGet = mockCustomApi.getNamespacedCustomObject.getMockImplementation()!
+ mockCustomApi.getNamespacedCustomObject.mockImplementation(async args => {
+ if (args.plural !== 'workflowrecipes' || args.name !== RECIPE) return previousGet(args)
+ if (!live) throw { code: 404 }
+ return structuredClone(live)
+ })
+ mockCustomApi.patchNamespacedCustomObjectStatus.mockImplementation(async ({ body }) => {
+ if (!live) throw { code: 404 }
+ if (
+ body.metadata &&
+ (body.metadata.uid !== live.metadata.uid ||
+ body.metadata.resourceVersion !== live.metadata.resourceVersion)
+ ) {
+ throw { code: 409 }
+ }
+ live.status = { ...live.status, ...body.status }
+ live.metadata.resourceVersion = String(Number(live.metadata.resourceVersion) + 1)
+ return structuredClone(live)
+ })
+ const queueFailures = vi.fn()
+ const queue = new RecipeEventQueue(queueFailures)
+ const watcher = Object.create(WorkflowRecipeWatcher.prototype) as {
+ recipes: Map<string, WorkflowRecipeCRD>
+ eventQueue: RecipeEventQueue
+ transientRetries: Map<string, { timer: ReturnType<typeof setTimeout>; attempts: number }>
+ handleRecipeEvent: (type: string, recipe: WorkflowRecipeCRD) => Promise<void>
+ clearTransientRetry: (name: string) => void
+ }
+ Object.assign(watcher, {
+ recipes: new Map(),
+ eventQueue: queue,
+ transientRetries: new Map(),
+ stopped: false,
+ traceReporter: null,
+ dbRunProcessor: null,
+ reconciler,
+ customApi: mockCustomApi,
+ })
+ return {
+ watcher,
+ queue,
+ queueFailures,
+ current: () => live,
+ replace: (next: WorkflowRecipeCRD | undefined) => {
+ live = next
+ },
+ drain: () => queue.enqueue(RECIPE, async () => {}),
+ }
+ }
+
+ function terminalRetryRecipe(): WorkflowRecipeCRD {
+ const recipe = runScopedRecipe({
+ phase: 'active',
+ message: 'Workflow completed',
+ workflowExecution: { phase: 'completed' },
+ conditions: publishedFacts('converged'),
+ workloadInstances: { 'web-search': `${RECIPE}-web-search-run` },
+ })
+ recipe.metadata.resourceVersion = '1'
+ recipe.metadata.generation = 1
+ recipe.spec.workloads = [
+ {
+ id: 'web-search',
+ type: 'deployment',
+ image: 'web-search:test',
+ transport: { type: 'streamableHttp', path: '/mcp' },
+ },
+ ]
+ return recipe
+ }
+
+ it('R5-L3: the dedicated retry retains the pipeline namespace allowlist', async () => {
+ const inner = installRealInner()
+ const retry = vi.spyOn(inner, 'retryLegacyMcpServersInternetEgressDelete')
+ const recipe = terminalRetryRecipe()
+ recipe.metadata.namespace = 'other-namespace'
+ expect(ofType(recipe.status?.conditions, LEGACY_TYPE)).toHaveLength(1)
+ mockCustomApi.getNamespacedCustomObject.mockClear()
+ const result = await reconciler.retryPendingLegacyNetworkPolicyDeleteOnly(recipe)
+ expect(result).toEqual({})
+ expect(retry).not.toHaveBeenCalled()
+ expect(mockCustomApi.getNamespacedCustomObject).not.toHaveBeenCalled()
+ expect(legacyDeletes()).toBe(0)
+ })
+
+ it('R5-L3: the real watcher, WRC and event queue retry only legacy DELETE across 60/120/240 s windows', async () => {
+ vi.useFakeTimers()
+ const inner = installRealInner()
+ vi.spyOn(inner, 'ensureMcpHostRuntimeCredentials').mockResolvedValue()
+ const teardown = vi.spyOn(inner, 'teardownComputePodsForTerminalRun')
+ const fullReconcile = vi.spyOn(reconciler, 'reconcile')
+ legacyDeleteAnswers({ code: 403, message: 'forbidden' })
+ const bridge = realWatcherBridge(terminalRetryRecipe())
+
+ await bridge.queue.enqueue(RECIPE, () =>
+ bridge.watcher.handleRecipeEvent('ADDED', bridge.current()!)
+ )
+
+ expect(teardown).toHaveBeenCalledTimes(1)
+ expect(legacyDeletes()).toBe(1)
+ const initialEffects = {
+ pods: mockCoreApi.deleteNamespacedPod.mock.calls.length,
+ transport: mockCustomApi.deleteNamespacedCustomObject.mock.calls.filter(
+ ([arg]) => arg.plural === 'mcpservers'
+ ).length,
+ deployments: mockAppsApi.deleteNamespacedDeployment.mock.calls.length,
+ gfs: mockNetworkingApi.deleteNamespacedNetworkPolicy.mock.calls.filter(([arg]) =>
+ arg.name.endsWith('-coordinator-to-gfs')
+ ).length,
+ }
+ expect(initialEffects.pods).toBe(3)
+ expect(initialEffects.transport).toBeGreaterThan(0)
+ expect(initialEffects.deployments).toBeGreaterThan(0)
+ expect(initialEffects.gfs).toBeGreaterThan(0)
+
+ for (const [index, delay] of [60_000, 120_000, 240_000].entries()) {
+ await vi.advanceTimersByTimeAsync(delay)
+ await bridge.drain()
+ expect(legacyDeletes()).toBe(index + 2)
+ expect(fullReconcile).toHaveBeenCalledTimes(1)
+ expect(teardown).toHaveBeenCalledTimes(1)
+ expect(mockCoreApi.deleteNamespacedPod).toHaveBeenCalledTimes(initialEffects.pods)
+ expect(mockAppsApi.deleteNamespacedDeployment).toHaveBeenCalledTimes(
+ initialEffects.deployments
+ )
+ expect(
+ mockCustomApi.deleteNamespacedCustomObject.mock.calls.filter(
+ ([arg]) => arg.plural === 'mcpservers'
+ )
+ ).toHaveLength(initialEffects.transport)
+ expect(
+ mockNetworkingApi.deleteNamespacedNetworkPolicy.mock.calls.filter(([arg]) =>
+ arg.name.endsWith('-coordinator-to-gfs')
+ )
+ ).toHaveLength(initialEffects.gfs)
+ }
+
+ // Recovery performs a conditions-only patch, preserving the workflow/SDK state.
+ const beforeRecovery = structuredClone(bridge.current()!.status)
+ legacyDeleteAnswers({ code: 404, message: 'not found' })
+ mockCustomApi.patchNamespacedCustomObjectStatus.mockClear()
+ await vi.advanceTimersByTimeAsync(480_000)
+ await bridge.drain()
+ expect(legacyDeletes()).toBe(5)
+ expect(fullReconcile).toHaveBeenCalledTimes(1)
+ expect(bridge.current()!.status?.workflowExecution).toEqual(beforeRecovery?.workflowExecution)
+ expect(bridge.current()!.status?.pluginWorkloadSdk).toEqual(beforeRecovery?.pluginWorkloadSdk)
+ expect(ofType(bridge.current()!.status?.conditions, LEGACY_TYPE)).toEqual([])
+ expect(mockCustomApi.patchNamespacedCustomObjectStatus).toHaveBeenCalledWith(
+ expect.objectContaining({
+ body: {
+ metadata: { uid: 'uid-lg', resourceVersion: expect.any(String) },
+ status: { conditions: expect.any(Array) },
+ },
+ }),
+ expect.anything()
+ )
+ expect(bridge.queueFailures).not.toHaveBeenCalled()
+ expect(vi.getTimerCount()).toBe(0)
+
+ // Deliver the actual accepted PATCH response through the ordinary watch
+ // path. Its unchanged generation makes this a status-only MODIFIED, whose
+ // terminal cleanup remains required by the existing recovery contract.
+ expect(mockCustomApi.patchNamespacedCustomObjectStatus).toHaveBeenCalledTimes(1)
+ const recoveryWatchObject = (await mockCustomApi.patchNamespacedCustomObjectStatus.mock
+ .results[0].value) as WorkflowRecipeCRD
+ expect(recoveryWatchObject).toEqual(bridge.current())
+ expect(recoveryWatchObject.metadata.resourceVersion).toBe('2')
+ expect(recoveryWatchObject.metadata.generation).toBe(1)
+ await bridge.queue.enqueue(RECIPE, () =>
+ bridge.watcher.handleRecipeEvent('MODIFIED', recoveryWatchObject)
+ )
+ await bridge.drain()
+ expect(fullReconcile).toHaveBeenCalledTimes(2)
+ expect(teardown).toHaveBeenCalledTimes(2)
+ expect(mockCoreApi.deleteNamespacedPod).toHaveBeenCalledTimes(initialEffects.pods * 2)
+ expect(mockAppsApi.deleteNamespacedDeployment).toHaveBeenCalledTimes(
+ initialEffects.deployments * 2
+ )
+ expect(
+ mockCustomApi.deleteNamespacedCustomObject.mock.calls.filter(
+ ([arg]) => arg.plural === 'mcpservers'
+ )
+ ).toHaveLength(initialEffects.transport * 2)
+ expect(
+ mockNetworkingApi.deleteNamespacedNetworkPolicy.mock.calls.filter(([arg]) =>
+ arg.name.endsWith('-coordinator-to-gfs')
+ )
+ ).toHaveLength(initialEffects.gfs * 2)
+ expect(legacyDeletes()).toBe(5)
+ // That ordinary pass publishes nothing, so it cannot drive another watch
+ // event. No legacy timer or continuing status-write loop remains either.
+ expect(mockCustomApi.patchNamespacedCustomObjectStatus).toHaveBeenCalledTimes(1)
+ expect(vi.getTimerCount()).toBe(0)
+ await vi.advanceTimersByTimeAsync(3_600_000)
+ await bridge.drain()
+ expect(fullReconcile).toHaveBeenCalledTimes(2)
+ expect(teardown).toHaveBeenCalledTimes(2)
+ expect(legacyDeletes()).toBe(5)
+ expect(mockCustomApi.patchNamespacedCustomObjectStatus).toHaveBeenCalledTimes(1)
+ expect(bridge.queueFailures).not.toHaveBeenCalled()
+ bridge.watcher.clearTransientRetry(RECIPE)
+ })
+
+ it.each([
+ 'recreated',
+ 'deleted',
+ 'deleting',
+ 'marker-cleared',
+ 'namespace-changed',
+ 'name-changed',
+ ] as const)(
+ 'R5-L3: a queued legacy retry drops a %s recipe after its fresh API read',
+ async change => {
+ vi.useFakeTimers()
+ const inner = installRealInner()
+ vi.spyOn(inner, 'ensureMcpHostRuntimeCredentials').mockResolvedValue()
+ const teardown = vi.spyOn(inner, 'teardownComputePodsForTerminalRun')
+ const fullReconcile = vi.spyOn(reconciler, 'reconcile')
+ legacyDeleteAnswers({ code: 403, message: 'forbidden' })
+ const bridge = realWatcherBridge(terminalRetryRecipe())
+ await bridge.queue.enqueue(RECIPE, () =>
+ bridge.watcher.handleRecipeEvent('ADDED', bridge.current()!)
+ )
+ expect(legacyDeletes()).toBe(1)
+ expect(vi.getTimerCount()).toBe(1)
+ let release!: () => void
+ const blocker = bridge.queue.enqueue(
+ RECIPE,
+ () =>
+ new Promise<void>(resolve => {
+ release = resolve
+ })
+ )
+ await Promise.resolve()
+ await vi.advanceTimersByTimeAsync(60_000)
+ const next = structuredClone(bridge.current()!)
+ if (change === 'deleted') bridge.replace(undefined)
+ else {
+ if (change === 'recreated') next.metadata.uid = 'uid-new'
+ if (change === 'deleting') next.metadata.deletionTimestamp = new Date().toISOString()
+ if (change === 'marker-cleared') next.status!.conditions = []
+ if (change === 'namespace-changed') next.metadata.namespace = 'other-namespace'
+ if (change === 'name-changed') next.metadata.name = 'other-recipe'
+ bridge.replace(next)
+ }
+ // Leave the watcher's cache stale: the API read after the queue wait is authoritative.
+ release()
+ await blocker
+ await bridge.drain()
+ expect(legacyDeletes()).toBe(1)
+ expect(fullReconcile).toHaveBeenCalledTimes(1)
+ expect(teardown).toHaveBeenCalledTimes(1)
+ expect(bridge.queueFailures).not.toHaveBeenCalled()
+ expect(vi.getTimerCount()).toBe(0)
+ bridge.watcher.clearTransientRetry(RECIPE)
+ }
+ )
+
+ it.each(['recreated', 'deleting', 'marker-cleared'] as const)(
+ 'R5-L3: the post-DELETE read refuses to publish on a %s recipe',
+ async change => {
+ const inner = installRealInner()
+ vi.spyOn(inner, 'ensureMcpHostRuntimeCredentials').mockResolvedValue()
+ const fullReconcile = vi.spyOn(reconciler, 'reconcile')
+ legacyDeleteAnswers({ code: 403, message: 'forbidden' })
+ const bridge = realWatcherBridge(terminalRetryRecipe())
+ await bridge.queue.enqueue(RECIPE, () =>
+ bridge.watcher.handleRecipeEvent('ADDED', bridge.current()!)
+ )
+ expect(legacyDeletes()).toBe(1)
+ mockCustomApi.patchNamespacedCustomObjectStatus.mockClear()
+ mockNetworkingApi.deleteNamespacedNetworkPolicy.mockImplementation(async ({ name }) => {
+ if (name === LEGACY) {
+ const next = structuredClone(bridge.current()!)
+ if (change === 'recreated') next.metadata.uid = 'uid-new'
+ if (change === 'deleting') next.metadata.deletionTimestamp = new Date().toISOString()
+ if (change === 'marker-cleared') next.status!.conditions = []
+ bridge.replace(next)
+ }
+ return {}
+ })
+ await vi.advanceTimersByTimeAsync(60_000)
+ await bridge.drain()
+ expect(legacyDeletes()).toBe(2)
+ expect(fullReconcile).toHaveBeenCalledTimes(1)
+ expect(mockCustomApi.patchNamespacedCustomObjectStatus).not.toHaveBeenCalled()
+ expect(vi.getTimerCount()).toBe(0)
+ expect(bridge.queueFailures).not.toHaveBeenCalled()
+ }
+ )
+
+ it.each(['recreated', 'newer-policy-facts'] as const)(
+ 'R5-L3: a status conflict caused by %s retries only the fenced conditions patch',
+ async change => {
+ const inner = installRealInner()
+ vi.spyOn(inner, 'ensureMcpHostRuntimeCredentials').mockResolvedValue()
+ const fullReconcile = vi.spyOn(reconciler, 'reconcile')
+ legacyDeleteAnswers({ code: 403, message: 'forbidden' })
+ const bridge = realWatcherBridge(terminalRetryRecipe())
+ await bridge.queue.enqueue(RECIPE, () =>
+ bridge.watcher.handleRecipeEvent('ADDED', bridge.current()!)
+ )
+ mockCustomApi.patchNamespacedCustomObjectStatus.mockClear()
+ const newerFacts = networkPolicyMarkerConditions(
+ {
+ kind: 'reconcile',
+ summary: { conflicts: [], retryPending: true, prune: 'pending', legacy: 'pending' },
+ },
+ [],
+ new Date().toISOString()
+ )
+ mockCustomApi.patchNamespacedCustomObjectStatus.mockImplementationOnce(async ({ body }) => {
+ expect(body.metadata).toEqual({ uid: 'uid-lg', resourceVersion: '1' })
+ const next = structuredClone(bridge.current()!)
+ next.metadata.resourceVersion = '2'
+ if (change === 'recreated') next.metadata.uid = 'uid-new'
+ else next.status!.conditions = newerFacts
+ bridge.replace(next)
+ throw { code: 409 }
+ })
+ legacyDeleteAnswers({ code: 404, message: 'not found' })
+ await vi.advanceTimersByTimeAsync(60_000)
+ await bridge.drain()
+ expect(legacyDeletes()).toBe(2)
+ expect(mockCustomApi.patchNamespacedCustomObjectStatus).toHaveBeenCalledTimes(1)
+ await vi.advanceTimersByTimeAsync(4_999)
+ await bridge.drain()
+ expect(mockCustomApi.patchNamespacedCustomObjectStatus).toHaveBeenCalledTimes(1)
+ await vi.advanceTimersByTimeAsync(1)
+ await bridge.drain()
+ expect(legacyDeletes()).toBe(2)
+ expect(fullReconcile).toHaveBeenCalledTimes(1)
+ if (change === 'recreated') {
+ expect(mockCustomApi.patchNamespacedCustomObjectStatus).toHaveBeenCalledTimes(1)
+ expect(bridge.current()!.metadata.uid).toBe('uid-new')
+ } else {
+ expect(mockCustomApi.patchNamespacedCustomObjectStatus).toHaveBeenCalledTimes(2)
+ expect(
+ mockCustomApi.patchNamespacedCustomObjectStatus.mock.calls[1][0].body.metadata
+ ).toEqual({ uid: 'uid-lg', resourceVersion: '2' })
+ expect(ofType(bridge.current()!.status?.conditions, LEGACY_TYPE)).toEqual([])
+ expect(ofType(bridge.current()!.status?.conditions, MARKER_TYPE)).toEqual(
+ ofType(newerFacts, MARKER_TYPE)
+ )
+ expect(bridge.current()!.status?.workflowExecution).toEqual({ phase: 'completed' })
+ }
+ expect(vi.getTimerCount()).toBe(0)
+ expect(bridge.queueFailures).not.toHaveBeenCalled()
+ }
+ )
+
+ it('R5-L3: clearing the retry while its callback waits in the real queue cancels its API work', async () => {
+ const inner = installRealInner()
+ vi.spyOn(inner, 'ensureMcpHostRuntimeCredentials').mockResolvedValue()
+ legacyDeleteAnswers({ code: 403, message: 'forbidden' })
+ const bridge = realWatcherBridge(terminalRetryRecipe())
+ await bridge.queue.enqueue(RECIPE, () =>
+ bridge.watcher.handleRecipeEvent('ADDED', bridge.current()!)
+ )
+ let release!: () => void
+ const blocker = bridge.queue.enqueue(
+ RECIPE,
+ () =>
+ new Promise<void>(resolve => {
+ release = resolve
+ })
+ )
+ await Promise.resolve()
+ await vi.advanceTimersByTimeAsync(60_000)
+ bridge.watcher.clearTransientRetry(RECIPE)
+ mockCustomApi.getNamespacedCustomObject.mockClear()
+ release()
+ await blocker
+ await bridge.drain()
+ expect(mockCustomApi.getNamespacedCustomObject).not.toHaveBeenCalled()
+ expect(legacyDeletes()).toBe(1)
+ expect(vi.getTimerCount()).toBe(0)
+ expect(bridge.queueFailures).not.toHaveBeenCalled()
+ })
})
// A reconcile() prune whose DELETE did not land publishes
diff --git a/workflow-recipes/src/reconciler/workflowRecipeReconciler.ts b/workflow-recipes/src/reconciler/workflowRecipeReconciler.ts
index 8ce033537..1a716ffab 100644
--- a/workflow-recipes/src/reconciler/workflowRecipeReconciler.ts
+++ b/workflow-recipes/src/reconciler/workflowRecipeReconciler.ts
@@ -64,6 +64,7 @@ import { HttpPluginWorkloadSdkRevocationClient } from '../workflow/pluginWorkloa
import { deriveWorkflowRuntimePlan } from '../workflow/runtimePlan'
import { validateWorkflowRecipeLimits } from '../workflow/workflowLimits'
import {
+ LegacyNetworkPolicyDeletePendingError,
NETWORK_POLICY_OWNERSHIP_CONDITION_TYPES,
type NetworkPolicyMarkerWriter,
WORKFLOW_OUTPUT_CONDITION_TYPES,
@@ -689,8 +690,18 @@ export interface ReconcileResult {
* false/undefined so genuine flakiness still backs off.
*/
requeueFixedInterval?: boolean
+ /** A legacy-only timer must not repeat the recipe's runtime lifecycle. */
+ requeueOperation?: 'legacy-network-policy-delete'
}
+type LegacyNetworkPolicyDeleteRetryResult = Pick<
+ ReconcileResult,
+ | 'requeueAfterMs'
+ | 'requeueFixedInterval'
+ | 'requeueOperation'
+ | 'networkPolicyOwnershipConditions'
+>
+
/**
* Base requeue delay for a transient (skipStatusPatch) reconcile result. The
* watcher backs off exponentially from here up to its own cap.
@@ -1097,7 +1108,7 @@ export class WorkflowRecipeReconciler {
*/
private async cleanupPluginWorkloadSdkOrThrow(
recipeName: string,
- options: { preserveWorkflowRuntime?: boolean; recipeUid?: string } = {}
+ options: { preserveWorkflowRuntime?: boolean; recipeUid?: string; recipeDeleted?: boolean } = {}
): Promise<void> {
if (!this.workflowReconciler) {
throw new Error('workflow subsystem is not initialized; SDK cleanup cannot be confirmed')
@@ -1550,6 +1561,90 @@ export class WorkflowRecipeReconciler {
await this.publishNetworkPolicyMarkerWriter(recipe, { kind: 'legacy-retry', legacy })
}
+ /**
+ * The timer's only operation: retry the shared legacy DELETE and publish its
+ * condition group. Reread after the asynchronous DELETE so a replacement CR,
+ * deletion or newer policy facts cannot receive this snapshot's status.
+ * Resource-version conflicts retry through the watcher's bounded timer; the
+ * shared gone ledger prevents another DELETE while that patch converges.
+ */
+ async retryPendingLegacyNetworkPolicyDeleteOnly(
+ recipe: WorkflowRecipeCRD
+ ): Promise<LegacyNetworkPolicyDeleteRetryResult> {
+ if (
+ !this.workflowReconciler ||
+ !recipe.metadata.uid ||
+ recipe.metadata.namespace !== this.config.sandboxNamespace ||
+ recipe.metadata.deletionTimestamp ||
+ !hasLegacyNetworkPolicyDeletePendingMarker(recipe.status?.conditions)
+ )
+ return {}
+
+ const legacy = await this.workflowReconciler.retryLegacyMcpServersInternetEgressDelete(
+ recipe.metadata.name,
+ recipe.metadata.uid
+ )
+ let live: WorkflowRecipeCRD
+ try {
+ live = (await this.customApi.getNamespacedCustomObject({
+ group: CRD_GROUP,
+ version: CRD_VERSION,
+ namespace: recipe.metadata.namespace,
+ plural: WORKFLOWRECIPE_PLURAL,
+ name: recipe.metadata.name,
+ })) as WorkflowRecipeCRD
+ } catch (error) {
+ if (getErrorCode(error) === 404) return {}
+ throw error
+ }
+ if (
+ live.metadata.uid !== recipe.metadata.uid ||
+ live.metadata.name !== recipe.metadata.name ||
+ live.metadata.namespace !== recipe.metadata.namespace ||
+ live.metadata.deletionTimestamp ||
+ !hasLegacyNetworkPolicyDeletePendingMarker(live.status?.conditions)
+ )
+ return {}
+
+ if (legacy === 'pending') {
+ const result: LegacyNetworkPolicyDeleteRetryResult = {}
+ this.requeueForPendingLegacyNetworkPolicyDelete(live, result)
+ return result
+ }
+ const fresh = networkPolicyMarkerConditions(
+ { kind: 'legacy-retry', legacy },
+ live.status?.conditions,
+ new Date().toISOString()
+ )
+ const conditions =
+ mergeOwnedConditions(
+ live.status?.conditions,
+ fresh,
+ NETWORK_POLICY_OWNERSHIP_CONDITION_TYPES
+ ) ?? []
+ if (!live.metadata.resourceVersion) {
+ throw new RetryableReconcileError(
+ 'Cannot publish legacy policy retry without recipe resourceVersion'
+ )
+ }
+ await this.customApi.patchNamespacedCustomObjectStatus(
+ {
+ group: CRD_GROUP,
+ version: CRD_VERSION,
+ namespace: live.metadata.namespace,
+ plural: WORKFLOWRECIPE_PLURAL,
+ name: live.metadata.name,
+ body: {
+ metadata: { uid: live.metadata.uid, resourceVersion: live.metadata.resourceVersion },
+ status: { conditions },
+ },
+ },
+ { middleware: [k8s.setHeaderMiddleware('Content-Type', 'application/merge-patch+json')] }
+ )
+ if (recipe.status) recipe.status.conditions = conditions
+ return {}
+ }
+
/**
* R4-L4: a pass that leaves `DeletePending` published and asks for no other
* requeue is requeued at a fixed interval for the rest of the legacy
@@ -1559,9 +1654,15 @@ export class WorkflowRecipeReconciler {
*/
private requeueForPendingLegacyNetworkPolicyDelete(
recipe: WorkflowRecipeCRD,
- result: ReconcileResult
+ result: LegacyNetworkPolicyDeleteRetryResult
): void {
- if (!this.workflowReconciler || result.requeueAfterMs !== undefined) return
+ if (
+ !this.workflowReconciler ||
+ recipe.metadata.namespace !== this.config.sandboxNamespace ||
+ recipe.metadata.deletionTimestamp ||
+ result.requeueAfterMs !== undefined
+ )
+ return
const conditions = result.networkPolicyOwnershipConditions ?? recipe.status?.conditions
if (!hasLegacyNetworkPolicyDeletePendingMarker(conditions)) return
result.requeueAfterMs = Math.max(
@@ -1572,6 +1673,7 @@ export class WorkflowRecipeReconciler {
)
)
result.requeueFixedInterval = true
+ result.requeueOperation = 'legacy-network-policy-delete'
}
/**
@@ -1822,6 +1924,7 @@ export class WorkflowRecipeReconciler {
const ns = recipe.metadata.namespace
const currentPhase = recipe.status?.phase ?? 'candidate'
let sdkOnlyProviderUnavailable = false
+ let sdkOnlyCapabilityRemovalTeardownConfirmed = false
// Defense-in-depth: the VAP + admin-API already enforce this invariant,
// but a manually-applied CRD in another namespace must not be reconciled.
@@ -1855,6 +1958,7 @@ export class WorkflowRecipeReconciler {
try {
await this.cleanupPluginWorkloadSdkOrThrow(name, {
preserveWorkflowRuntime: isWorkflow,
+ recipeUid: recipe.metadata.uid,
})
} catch (error) {
return {
@@ -1862,7 +1966,11 @@ export class WorkflowRecipeReconciler {
message: `Plugin Workload SDK teardown failed while the feature flag is disabled: ${String(error)}`,
workloadStatuses: [],
skipStatusPatch: true,
- requeueAfterMs: TRANSIENT_REQUEUE_BASE_MS,
+ requeueAfterMs:
+ error instanceof LegacyNetworkPolicyDeletePendingError
+ ? error.retryAfterMs
+ : TRANSIENT_REQUEUE_BASE_MS,
+ requeueFixedInterval: error instanceof LegacyNetworkPolicyDeletePendingError,
}
}
return {
@@ -1894,6 +2002,7 @@ export class WorkflowRecipeReconciler {
try {
await this.cleanupPluginWorkloadSdkOrThrow(name, {
preserveWorkflowRuntime: isWorkflow,
+ recipeUid: recipe.metadata.uid,
})
} catch (error) {
return {
@@ -1901,7 +2010,11 @@ export class WorkflowRecipeReconciler {
message: `Plugin Workload SDK teardown failed after capability removal: ${String(error)}`,
workloadStatuses: [],
skipStatusPatch: true,
- requeueAfterMs: TRANSIENT_REQUEUE_BASE_MS,
+ requeueAfterMs:
+ error instanceof LegacyNetworkPolicyDeletePendingError
+ ? error.retryAfterMs
+ : TRANSIENT_REQUEUE_BASE_MS,
+ requeueFixedInterval: error instanceof LegacyNetworkPolicyDeletePendingError,
}
}
// A stepless recipe managed policies only through the SDK, and the
@@ -1909,6 +2022,7 @@ export class WorkflowRecipeReconciler {
// terminal and validation returns below never reach the SDK-only lane.
// A workflow recipe keeps its run lane, which still manages policies.
if (!isWorkflow) {
+ sdkOnlyCapabilityRemovalTeardownConfirmed = true
await this.publishNetworkPolicyMarkerWriter(recipe, { kind: 'unmanaged' })
}
}
@@ -3147,12 +3261,9 @@ export class WorkflowRecipeReconciler {
sdkOnlyRuntime?.phase === 'active' &&
recipe.spec.pluginWorkloadSdk !== undefined &&
sdkOnlyRuntime.pluginWorkloadSdkBootstrapProof?.ready !== true
- // Without an SDK runtime this lane reaches no policy: the spec has no
- // SDK block, and the capability-removal teardown deleted every SDK
- // policy, the legacy one by name included. That is the unmanaged
- // writer, which clears the whole group. A runtime without a summary did
- // not evaluate the policies (the host returned before the apply), so
- // the field stays absent and patchStatus keeps what was published.
+ // Only a completed capability-removal teardown proves this lane manages
+ // no policy. A missing runtime alone evaluated nothing and must preserve
+ // the published apply, prune, ownership and legacy-delete facts.
const now = new Date().toISOString()
const {
networkPolicyRetryPending: sdkOnlyNetworkPolicyRetryPending,
@@ -3163,13 +3274,15 @@ export class WorkflowRecipeReconciler {
recipe.status?.conditions,
now
)
- : {
- networkPolicyOwnershipConditions: networkPolicyMarkerConditions(
- { kind: 'unmanaged' },
- recipe.status?.conditions,
- now
- ),
- }
+ : sdkOnlyCapabilityRemovalTeardownConfirmed
+ ? {
+ networkPolicyOwnershipConditions: networkPolicyMarkerConditions(
+ { kind: 'unmanaged' },
+ recipe.status?.conditions,
+ now
+ ),
+ }
+ : {}
if (sdkOnlyRuntime?.phase === 'failed') {
return {
phase: 'failed',
@@ -4059,7 +4172,11 @@ export class WorkflowRecipeReconciler {
// the shared mcp-host/token resources. Hybrid recipes must not bypass
// broker revocation merely because they also have coordinator state.
if (recipe.spec.pluginWorkloadSdk || recipe.status?.pluginWorkloadSdk) {
- await this.cleanupPluginWorkloadSdkOrThrow(name, { preserveWorkflowRuntime: true })
+ await this.cleanupPluginWorkloadSdkOrThrow(name, {
+ preserveWorkflowRuntime: true,
+ recipeUid: recipe.metadata.uid,
+ recipeDeleted: true,
+ })
}
await this.workflowReconciler.reconcileDelete(
name,
@@ -4076,7 +4193,10 @@ export class WorkflowRecipeReconciler {
// a capability may have been removed in an update before Kubernetes emits
// the final delete event, and cleanup must still revoke its host tokens.
if (recipe.spec.pluginWorkloadSdk || recipe.status?.pluginWorkloadSdk) {
- await this.cleanupPluginWorkloadSdkOrThrow(name, { recipeUid: recipe.metadata.uid })
+ await this.cleanupPluginWorkloadSdkOrThrow(name, {
+ recipeUid: recipe.metadata.uid,
+ recipeDeleted: true,
+ })
}
await this.cleanupDelegationIfNeeded(recipe)
diff --git a/workflow-recipes/src/workflow/workflowReconciler.ts b/workflow-recipes/src/workflow/workflowReconciler.ts
index 3aee38c43..6b0058524 100644
--- a/workflow-recipes/src/workflow/workflowReconciler.ts
+++ b/workflow-recipes/src/workflow/workflowReconciler.ts
@@ -266,11 +266,30 @@ export interface PluginWorkloadSdkCleanupOptions {
*/
preserveWorkflowRuntime?: boolean
/**
- * Set when the recipe itself is being deleted. An SDK-only recipe never
- * reaches `reconcileDelete`, so this is the only place its legacy internet
- * policy ledger entry can be forgotten.
+ * Identity for the live legacy-delete ledger. It shares the gone-set and
+ * backoff with `reconcile()` and the WRC retry; it never erases either.
*/
recipeUid?: string
+ /**
+ * True only while the recipe itself is being deleted. A completed cleanup
+ * may then drop the process-local legacy ledger entry; a pending cleanup
+ * must retain it so the finalizer retry resumes from the same window.
+ */
+ recipeDeleted?: boolean
+}
+
+export class LegacyNetworkPolicyDeletePendingError extends Error {
+ constructor(
+ recipeName: string,
+ readonly retryAfterMs: number
+ ) {
+ super(`Legacy mcp-servers internet egress NetworkPolicy delete is pending for ${recipeName}`)
+ this.name = 'LegacyNetworkPolicyDeletePendingError'
+ }
+}
+
+function buildLegacyMcpServersInternetEgressPolicyName(recipeName: string): string {
+ return `${recipeName}-mcp-servers-egress-internet`
}
export const NETWORK_POLICY_OWNERSHIP_CONDITION_TYPE = 'WorkflowNetworkPolicyOwnership'
@@ -293,8 +312,8 @@ export const NETWORK_POLICY_OWNERSHIP_CONDITION_TYPE = 'WorkflowNetworkPolicyOwn
*
* Each combination with a fact that is not converged has its own reason
* (`RetryPending`, `PrunePending`, `PruneUnevaluated`, `RetryAndPrunePending`,
- * `RetryAndPruneUnevaluated`), and the facts are read back from the reason
- * alone.
+ * `RetryAndPruneUnevaluated`), and current facts are read back from the reason.
+ * The frozen historical combined marker is normalized by the decoder below.
*/
export const NETWORK_POLICIES_CONVERGED_CONDITION_TYPE = 'WorkflowNetworkPoliciesConverged'
@@ -310,8 +329,9 @@ const CONVERGED_NETWORK_POLICY_MARKER_FACTS = {
} as const satisfies NetworkPolicyMarkerFacts
/**
- * One reason per combination of facts. The facts are read back from the
- * reason alone; `message` is prose for operators and is never compared.
+ * One current reason per combination of facts. Current messages are prose for
+ * operators. The decoder separately recognizes the exact historical combined
+ * marker emitted before those facts had distinct reasons.
*/
const NETWORK_POLICY_MARKER_REASONS = {
RetryPending: {
@@ -340,6 +360,15 @@ const NETWORK_POLICY_MARKER_REASONS = {
},
} as const satisfies Record<string, { facts: NetworkPolicyMarkerFacts; message: string }>
+/**
+ * The only historical message this controller interprets. Before every fact
+ * had its own reason, `af0152924` encoded apply-plus-prune as `RetryPending`
+ * with exactly this prose. Equality is deliberate: substring or reworded
+ * messages do not gain a prune fact.
+ */
+const HISTORICAL_RETRY_AND_PRUNE_PENDING_MESSAGE =
+ 'One or more run-lane NetworkPolicies are pending a retry (terminating or contended), and one or more the spec no longer wants are pending a delete'
+
type NetworkPolicyMarkerReason = keyof typeof NETWORK_POLICY_MARKER_REASONS
function isNetworkPolicyMarkerReason(
@@ -477,6 +506,12 @@ export function networkPolicyMarkerFacts(
if (!isNetworkPolicyMarkerReason(marker?.reason)) {
return { ...CONVERGED_NETWORK_POLICY_MARKER_FACTS }
}
+ if (
+ marker.reason === 'RetryPending' &&
+ marker.message === HISTORICAL_RETRY_AND_PRUNE_PENDING_MESSAGE
+ ) {
+ return { apply: 'pending', prune: 'pending' }
+ }
return { ...NETWORK_POLICY_MARKER_REASONS[marker.reason].facts }
}
@@ -1858,9 +1893,6 @@ export class WorkflowReconciler {
const ns = this.deps.config.sandboxNamespace
const preserveWorkflowRuntime = options.preserveWorkflowRuntime === true
this.pluginWorkloadSdkProvisioner.clearRecipeState(recipeName)
- if (options.recipeUid) {
- this.forgetLegacyMcpServersInternetEgress(recipeName, options.recipeUid)
- }
const sdkNetworkPolicyNames = [
`${recipeName}-workload-to-mcp-host-sdk-ingress`,
`${recipeName}-workload-to-mcp-host-sdk-egress`,
@@ -1936,24 +1968,24 @@ export class WorkflowReconciler {
this.teardownDelete(() =>
this.deleteNetworkPoliciesByLabelSelector(
ns,
- `clerum.io/recipe=${recipeName},clerum.io/managed-by=wrc`
+ `clerum.io/recipe=${recipeName},clerum.io/managed-by=wrc`,
+ buildLegacyMcpServersInternetEgressPolicyName(recipeName)
)
),
this.teardownDelete(() =>
this.deleteNetworkPoliciesByLabelSelector(
this.deps.config.mcpServerNamespace,
- `clerum.io/recipe=${recipeName},clerum.io/managed-by=wrc`
+ `clerum.io/recipe=${recipeName},clerum.io/managed-by=wrc`,
+ buildLegacyMcpServersInternetEgressPolicyName(recipeName)
)
),
- // The legacy internet policy may carry no labels, so the sweep
- // above cannot be trusted to remove it. Deleting it by name is
- // what lets the caller publish the unmanaged marker writer after
- // this teardown: any non-404 failure rejects the teardown.
- this.teardownDelete(() =>
- this.deps.networkingApi.deleteNamespacedNetworkPolicy({
- name: `${recipeName}-mcp-servers-egress-internet`,
- namespace: this.deps.config.mcpServerNamespace,
- })
+ // The legacy internet policy may carry no labels, while a labeled
+ // copy is excluded from the sweeps above. Both forms go through
+ // the one live ledger so a completed cleanup can prove `unmanaged`.
+ this.cleanupLegacyMcpServersInternetEgress(
+ recipeName,
+ options.recipeUid,
+ options.recipeDeleted === true
),
]),
])
@@ -2009,6 +2041,9 @@ export class WorkflowReconciler {
)
}
}
+ if (options.recipeDeleted === true) {
+ this.forgetLegacyMcpServersInternetEgress(recipeName, options.recipeUid)
+ }
}
private async assertPluginWorkloadSdkResourcesAbsent(
@@ -3390,7 +3425,7 @@ export class WorkflowReconciler {
* applies it, but deletes nothing: the run's pods are live and may still use
* any lane the spec wanted when they were created, so a revoke here cuts
* their traffic. That includes the legacy mcp-servers internet policy, which
- * only a reconcile() pass deletes. The summary is apply-only, so this path
+ * this path keeps. The summary is apply-only, so this path
* is the `apply-retry` writer of Table A below: it observes the apply and
* carries the published prune and legacy facts.
*
@@ -3431,26 +3466,37 @@ export class WorkflowReconciler {
* | `unmanaged` (kill switch, SDK lane without a runtime, capability removal) | converged | converged | removed | everything |
* | finalizer | the object is deleted | | | |
*
- * The legacy retry runs in the WRC after the early token reap and before
- * every short-circuit, only while `DeletePending` is published. It and the
- * reconcile() pass share one backoff per recipe: after a DELETE that did
+ * The legacy retry runs in the WRC before limit validation, every
+ * short-circuit and the early token reap, only while `DeletePending` is
+ * published. It, the reconcile() pass and a full SDK cleanup share one
+ * backoff per recipe: after a DELETE that did
* not land, the next one waits `min(prev * 2 || 60 s, 1 h)`, and inside
* that window the delete answers `pending` without a request. A `removed`
* answer clears L with a conditions-only patch; `pending` writes nothing.
- * The WRC `reconcile()` wrapper requeues a pass that leaves L pending and
- * asks for no other requeue after `max(1 s, the rest of the window)`, at a
- * fixed interval.
+ * The watcher's legacy-only callback requeues after the rest of the window;
+ * it does not force a full reconcile. A cleanup that must remove the legacy
+ * policy but observes `pending` fails before `unmanaged` and requeues the
+ * full cleanup at the same window. Finalizer cleanup is the last chance:
+ * it sends through the same ledger even inside a window, then records and
+ * forgets state only after the complete cleanup succeeds.
*
* A carrying writer keeps a marker whose reason it did not write as it is
* and warns; `reconcile` overwrites it. `unmanaged` is written only after
* `cleanupPluginWorkloadSdk` without `preserveWorkflowRuntime` succeeded:
- * it sweeps the recipe labels and deletes the legacy policy by name, and a
- * failure there fails the pass before the writer runs. A pending or
+ * its label sweeps exclude the legacy name, and the shared live ledger
+ * deletes that name once; a failure or backoff fails the pass before the
+ * writer runs. A pending or
* unevaluated prune is not retried outside reconcile(): only reconcile()
* can compute the desired set, and the finalizer's label sweep removes
* every run-lane leftover. On the finalizer, a non-404 DELETE is logged by
* `safeDelete` and not rethrown, so the finalizer completes.
*
+ * Historical `af0152924` markers are read only from their reason, except
+ * the one exact `RetryPending` combined message defined beside the decoder.
+ * Its ambiguous `PrunePending` cannot be safely promoted to L: current
+ * ordinary prune failures use the same reason, so they require a real
+ * reconcile() observation before the legacy retry can run.
+ *
* A desired lane is applied on both apply paths; an uncertain verdict never
* creates a proxy. Terminal teardown deletes a run-scoped run's compute pods
* and its cross-namespace transport workloads, and, for every terminal
@@ -3700,6 +3746,24 @@ export class WorkflowReconciler {
}
}
+ private async cleanupLegacyMcpServersInternetEgress(
+ recipeName: string,
+ recipeUid?: string,
+ lastChance = false
+ ): Promise<void> {
+ const fact = await this.pruneLegacyMcpServersInternetEgressPolicy(
+ recipeName,
+ recipeUid,
+ lastChance
+ )
+ if (fact === 'pending') {
+ throw new LegacyNetworkPolicyDeletePendingError(
+ recipeName,
+ Math.max(1, this.legacyMcpServersInternetEgressRetryDelayMs(recipeName, recipeUid ?? ''))
+ )
+ }
+ }
+
/**
* R4-L4: the legacy delete the WRC retries before its short-circuits while
* `DeletePending` is published (the `legacy-retry` writer). The same delete
@@ -3728,7 +3792,8 @@ export class WorkflowReconciler {
private async pruneLegacyMcpServersInternetEgressPolicy(
recipeName: string,
- recipeUid?: string
+ recipeUid?: string,
+ lastChance = false
): Promise<NetworkPolicyLegacyFact> {
const key = this.legacyMcpServersInternetEgressKey(recipeName, recipeUid)
if (this.prunedLegacyMcpServersInternetEgress.has(key)) {
@@ -3738,7 +3803,7 @@ export class WorkflowReconciler {
return 'removed'
}
const backoff = this.legacyMcpServersInternetEgressBackoff.get(key)
- if (backoff !== undefined && Date.now() < backoff.notBeforeMs) {
+ if (backoff !== undefined && !lastChance && Date.now() < backoff.notBeforeMs) {
this.log.debug('Skipping legacy mcp-servers internet NP delete; inside its backoff window', {
recipe: recipeName,
retryInMs: backoff.notBeforeMs - Date.now(),
@@ -3747,7 +3812,7 @@ export class WorkflowReconciler {
}
const outcome = await observeNamespacedDelete(() =>
this.deps.networkingApi.deleteNamespacedNetworkPolicy({
- name: `${recipeName}-mcp-servers-egress-internet`,
+ name: buildLegacyMcpServersInternetEgressPolicyName(recipeName),
namespace: this.deps.config.mcpServerNamespace,
})
)
@@ -4453,7 +4518,7 @@ export class WorkflowReconciler {
// List-by-label is the cleanest cleanup — all managed NPs have recipe label.
// Per-mcp-server NPs are deleted via label selector below. The legacy
// internet policy predates those labels, so it is deleted by name (R4-L4).
- const mcpServerNpNames: string[] = [`${recipeName}-mcp-servers-egress-internet`]
+ const mcpServerNpNames: string[] = [buildLegacyMcpServersInternetEgressPolicyName(recipeName)]
const mcpNs = this.deps.config.mcpServerNamespace
const cleanupTasks: Array<{ label: string; run: () => Promise<void> }> = [
@@ -4672,7 +4737,8 @@ export class WorkflowReconciler {
private async deleteNetworkPoliciesByLabelSelector(
namespace: string,
- labelSelector: string
+ labelSelector: string,
+ excludeName?: string
): Promise<void> {
const list = await this.deps.networkingApi.listNamespacedNetworkPolicy({
namespace,
@@ -4682,6 +4748,7 @@ export class WorkflowReconciler {
(list.items ?? [])
.map(policy => policy.metadata?.name)
.filter((name): name is string => Boolean(name))
+ .filter(name => name !== excludeName)
.map(async name => {
try {
await this.deps.networkingApi.deleteNamespacedNetworkPolicy({ name, namespace })
diff --git a/workflow-recipes/tests/unit/workflow/workflowReconciler.test.ts b/workflow-recipes/tests/unit/workflow/workflowReconciler.test.ts
index 461e25314..1acac866d 100644
--- a/workflow-recipes/tests/unit/workflow/workflowReconciler.test.ts
+++ b/workflow-recipes/tests/unit/workflow/workflowReconciler.test.ts
@@ -36,6 +36,8 @@ import {
} from '../../../src/workflow/resourceNames'
import { deriveWorkflowRuntimePlan } from '../../../src/workflow/runtimePlan'
import {
+ LEGACY_NETWORK_POLICY_DELETE_BACKOFF_BASE_MS,
+ LegacyNetworkPolicyDeletePendingError,
type WorkflowRecipeSpec,
WorkflowReconciler,
type WorkflowReconcilerDeps,
@@ -9223,6 +9225,152 @@ describe('WorkflowReconciler — reconcile loop', () => {
})
})
+describe('cleanupPluginWorkloadSdk legacy internet policy ownership', () => {
+ const RECIPE = 'sdk-only'
+ const RECIPE_UID = 'uid-sdk-only'
+ const LEGACY = `${RECIPE}-mcp-servers-egress-internet`
+
+ function legacyDeletes(networkingApi: ReturnType<typeof makeNetworkingApi>): number {
+ return networkingApi.deleteNamespacedNetworkPolicy.mock.calls.filter(
+ ([arg]) => arg.name === LEGACY
+ ).length
+ }
+
+ function makeCleanupCoreApi(): ReturnType<typeof makeCoreApi> {
+ const coreApi = makeCoreApi() as ReturnType<typeof makeCoreApi>
+ coreApi.readNamespacedEndpoints = vi.fn().mockRejectedValue({ code: 404 })
+ return coreApi
+ }
+
+ it('routes a labeled legacy policy through the shared live ledger instead of the label sweep', async () => {
+ const networkingApi = makeNetworkingApi() as ReturnType<typeof makeNetworkingApi>
+ let legacyPresent = true
+ networkingApi.listNamespacedNetworkPolicy.mockImplementation(
+ async ({ namespace }: { namespace: string }) => ({
+ items:
+ namespace === 'mcp-server'
+ ? legacyPresent
+ ? [
+ {
+ metadata: {
+ name: LEGACY,
+ namespace,
+ labels: { 'clerum.io/recipe': RECIPE, 'clerum.io/managed-by': 'wrc' },
+ },
+ },
+ ]
+ : []
+ : [],
+ })
+ )
+ networkingApi.deleteNamespacedNetworkPolicy.mockImplementation(async ({ name }) => {
+ if (name === LEGACY) legacyPresent = false
+ return {}
+ })
+ const coreApi = makeCoreApi() as ReturnType<typeof makeCoreApi>
+ coreApi.readNamespacedEndpoints = vi.fn().mockRejectedValue({ code: 404 })
+ const reconciler = new WorkflowReconciler(
+ makeDeps({
+ coreApi: coreApi as never,
+ networkingApi: networkingApi as never,
+ pluginWorkloadSdkRevocationClient: {
+ revoke: vi.fn().mockResolvedValue({ state: 'disabled' }),
+ } as never,
+ })
+ )
+
+ await reconciler.cleanupPluginWorkloadSdk(RECIPE, { recipeUid: RECIPE_UID })
+
+ // Liveness witness: both label-sweep namespaces were evaluated, and the
+ // shared by-name path sent the sole legacy DELETE.
+ expect(networkingApi.listNamespacedNetworkPolicy).toHaveBeenCalledWith(
+ expect.objectContaining({
+ namespace: 'mcp-server',
+ labelSelector: `clerum.io/recipe=${RECIPE},clerum.io/managed-by=wrc`,
+ })
+ )
+ expect(legacyDeletes(networkingApi)).toBe(1)
+
+ await reconciler.retryLegacyMcpServersInternetEgressDelete(RECIPE, RECIPE_UID)
+ expect(legacyDeletes(networkingApi)).toBe(1)
+ })
+
+ it('keeps live cleanup on the shared backoff while finalizer cleanup gets one last chance', async () => {
+ vi.useFakeTimers({ toFake: ['Date'] })
+ const startedAt = Date.now()
+ try {
+ const networkingApi = makeNetworkingApi() as ReturnType<typeof makeNetworkingApi>
+ let legacyDeleteFails = true
+ networkingApi.deleteNamespacedNetworkPolicy.mockImplementation(async ({ name }) => {
+ if (name === LEGACY && legacyDeleteFails) throw { code: 403, message: 'forbidden' }
+ return {}
+ })
+ const reconciler = new WorkflowReconciler(
+ makeDeps({
+ coreApi: makeCleanupCoreApi() as never,
+ networkingApi: networkingApi as never,
+ pluginWorkloadSdkRevocationClient: {
+ revoke: vi.fn().mockResolvedValue({ state: 'disabled' }),
+ } as never,
+ })
+ )
+
+ await expect(
+ reconciler.retryLegacyMcpServersInternetEgressDelete(RECIPE, RECIPE_UID)
+ ).resolves.toBe('pending')
+ expect(legacyDeletes(networkingApi)).toBe(1)
+
+ const pending = await reconciler
+ .cleanupPluginWorkloadSdk(RECIPE, {
+ recipeUid: RECIPE_UID,
+ })
+ .catch(error => error)
+ expect(pending).toBeInstanceOf(LegacyNetworkPolicyDeletePendingError)
+ expect((pending as LegacyNetworkPolicyDeletePendingError).retryAfterMs).toBe(
+ LEGACY_NETWORK_POLICY_DELETE_BACKOFF_BASE_MS
+ )
+ expect(legacyDeletes(networkingApi)).toBe(1)
+
+ legacyDeleteFails = false
+ await reconciler.cleanupPluginWorkloadSdk(RECIPE, {
+ recipeUid: RECIPE_UID,
+ recipeDeleted: true,
+ })
+ expect(legacyDeletes(networkingApi)).toBe(2)
+
+ await reconciler.retryLegacyMcpServersInternetEgressDelete(RECIPE, RECIPE_UID)
+ expect(legacyDeletes(networkingApi)).toBe(3)
+ } finally {
+ vi.useRealTimers()
+ }
+ })
+
+ it('keeps the legacy policy live when cleanup preserves the workflow runtime', async () => {
+ const networkingApi = makeNetworkingApi() as ReturnType<typeof makeNetworkingApi>
+ const reconciler = new WorkflowReconciler(
+ makeDeps({
+ coreApi: makeCleanupCoreApi() as never,
+ networkingApi: networkingApi as never,
+ pluginWorkloadSdkRevocationClient: {
+ revoke: vi.fn().mockResolvedValue({ state: 'disabled' }),
+ } as never,
+ })
+ )
+
+ await reconciler.cleanupPluginWorkloadSdk('hybrid', {
+ recipeUid: 'uid-hybrid',
+ preserveWorkflowRuntime: true,
+ })
+
+ expect(
+ networkingApi.deleteNamespacedNetworkPolicy.mock.calls.filter(
+ ([arg]) => arg.name === 'hybrid-mcp-servers-egress-internet'
+ )
+ ).toHaveLength(0)
+ expect(networkingApi.listNamespacedNetworkPolicy).not.toHaveBeenCalled()
+ })
+})
+
describe('buildNetworkPolicyOwnershipConditions', () => {
it('names the conflicting policies in sorted order, whatever order the apply produced them in', () => {
const conditions = buildNetworkPolicyOwnershipConditions(
@@ -9791,6 +9939,41 @@ describe('networkPolicyMarkerFacts', () => {
)
expect(new Set(reasons).size).toBe(combinations.length)
})
+
+ it('reads the one historical RetryPending message that encoded a pending prune', () => {
+ const historical = {
+ type: 'WorkflowNetworkPoliciesConverged',
+ status: 'False' as const,
+ reason: 'RetryPending',
+ message:
+ 'One or more run-lane NetworkPolicies are pending a retry (terminating or contended), and one or more the spec no longer wants are pending a delete',
+ lastTransitionTime: now,
+ }
+
+ expect(networkPolicyMarkerFacts([historical])).toStrictEqual({
+ apply: 'pending',
+ prune: 'pending',
+ })
+ expect(
+ networkPolicyMarkerFacts([{ ...historical, message: `${historical.message} ` }])
+ ).toStrictEqual({
+ apply: 'pending',
+ prune: 'converged',
+ })
+
+ const settled = networkPolicyMarkerConditions(
+ { kind: 'apply-retry', summary: { conflicts: [], retryPending: false } },
+ [historical],
+ now
+ )
+ expect(settled).toMatchObject([
+ {
+ type: 'WorkflowNetworkPoliciesConverged',
+ status: 'False',
+ reason: 'PrunePending',
+ },
+ ])
+ })
})
describe('networkPolicyConditionsChanged', () => {