Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
fix(workflow-recipes): address PR #855 review round 2 findings
Alfredo Lopezb64325095ec6
diff --git a/workflow-recipes/package-lock.json b/workflow-recipes/package-lock.jsonindex 5fb7841e7..486a03e71 100644--- a/workflow-recipes/package-lock.json+++ b/workflow-recipes/package-lock.json@@ -1,12 +1,12 @@{"name": "workload-recipes-controller",- "version": "0.1.413",+ "version": "0.1.414","lockfileVersion": 3,"requires": true,"packages": {"": {"name": "workload-recipes-controller",- "version": "0.1.413",+ "version": "0.1.414","license": "SEE LICENSE IN LICENSE","dependencies": {"@aws-sdk/client-s3": "^3.1009.0",diff --git a/workflow-recipes/package.json b/workflow-recipes/package.jsonindex 171c101d4..b7207d280 100644--- a/workflow-recipes/package.json+++ b/workflow-recipes/package.json@@ -1,6 +1,6 @@{"name": "workload-recipes-controller",- "version": "0.1.413",+ "version": "0.1.414","description": "Workload Recipes Controller (WRC) - WorkloadRecipe lifecycle management and MCP server interface","main": "dist/main.js","engines": {diff --git a/workflow-recipes/src/k8sClient.test.ts b/workflow-recipes/src/k8sClient.test.tsindex 271a46756..3438c3fb8 100644--- a/workflow-recipes/src/k8sClient.test.ts+++ b/workflow-recipes/src/k8sClient.test.ts@@ -17,6 +17,7 @@ import {workflowRecipeFromWatchObject,} from './k8sClient'import { captureLogger } from './reconciler/__tests__/captureLogger'+import { buildOAuthBrokerTokenSecret } from './reconciler/resourceBuilder'import type { SecretEventType, SecretLike } from './reconciler/secretWatcher'import type { WorkflowRecipeCRD } from './types'@@ -3345,13 +3346,12 @@ describe('handleOAuthBrokerTokenAdded', () => {secretLoops.length = 0await (internal as unknown as { startSecretWatch: () => Promise<void> }).startSecretWatch()- const brokerToken = {- metadata: {- name: 'wf-no-bg-oauth-broker-token',- labels: { 'clerum.io/component': 'oauth-broker-token', 'clerum.io/recipe': 'no-bg' },- },- data: { token: 'x' },- }+ // The Secret WRC itself writes, not a hand-written copy of its labels.+ const brokerToken = buildOAuthBrokerTokenSecret(+ 'no-bg',+ 'jwt',+ internal.config.sandboxNamespace+ )// Liveness witness: the sandbox-namespace watcher exists and reaches the handler.const sandbox = secretLoops.find(l => l.namespace === internal.config.sandboxNamespace)diff --git a/workflow-recipes/src/reconciler/oauthBrokerDeleteLedger.test.ts b/workflow-recipes/src/reconciler/oauthBrokerDeleteLedger.test.tsindex b3d94a744..a918e2f55 100644--- a/workflow-recipes/src/reconciler/oauthBrokerDeleteLedger.test.ts+++ b/workflow-recipes/src/reconciler/oauthBrokerDeleteLedger.test.ts@@ -1,68 +1,136 @@import { describe, expect, it } from 'vitest'-import { OAUTH_BROKER_NP_TTL_MS, OAuthBrokerDeleteLedger } from './oauthBrokerDeleteLedger'+import {+ OAUTH_BROKER_NP_TTL_MS,+ OAuthBrokerDeleteLedger,+ type OAuthBrokerLedgerRecipe,+} from './oauthBrokerDeleteLedger'++function ref(name: string, generation: number, uid = `uid-${name}`): OAuthBrokerLedgerRecipe {+ return { name, uid, generation }+}++function recordSecret(ledger: OAuthBrokerDeleteLedger, recipe: OAuthBrokerLedgerRecipe): void {+ expect(ledger.recordSecretDelete(recipe, ledger.secretEpoch(recipe.name))).toBe(true)+}describe('OAuthBrokerDeleteLedger', () => {it('remembers a Secret delete for the recorded generation only', () => {const ledger = new OAuthBrokerDeleteLedger()- expect(ledger.shouldDeleteSecret('r', 4)).toBe(true)- ledger.recordSecretDelete('r', 4)- expect(ledger.shouldDeleteSecret('r', 4)).toBe(false)- expect(ledger.shouldDeleteSecret('r', 5)).toBe(true)+ expect(ledger.shouldDeleteSecret(ref('r', 4))).toBe(true)+ recordSecret(ledger, ref('r', 4))+ expect(ledger.shouldDeleteSecret(ref('r', 4))).toBe(false)+ expect(ledger.shouldDeleteSecret(ref('r', 5))).toBe(true)})- it('keeps one entry per recipe: a newer generation replaces the older one', () => {+ it('keeps the highest generation per recipe: an older generation arriving late is skipped', () => {const ledger = new OAuthBrokerDeleteLedger()- ledger.recordSecretDelete('r', 4)- ledger.recordSecretDelete('r', 5)- // Generations only increase, so the older key is dead weight. Not remembering it- // is what bounds the ledger at one entry per recipe instead of one per spec edit.- expect(ledger.shouldDeleteSecret('r', 5)).toBe(false)- expect(ledger.shouldDeleteSecret('r', 4)).toBe(true)-- ledger.recordPolicyDelete('r', 4, 0)- ledger.recordPolicyDelete('r', 5, 0)- expect(ledger.shouldDeletePolicy('r', 5, 1)).toBe(false)- expect(ledger.shouldDeletePolicy('r', 4, 1)).toBe(true)+ recordSecret(ledger, ref('r', 4))+ recordSecret(ledger, ref('r', 5))+ expect(ledger.shouldDeleteSecret(ref('r', 5))).toBe(false)+ expect(ledger.shouldDeleteSecret(ref('r', 4))).toBe(false)+ expect(ledger.shouldDeleteSecret(ref('r', 6))).toBe(true)++ ledger.recordPolicyDelete(ref('r', 4), 0)+ ledger.recordPolicyDelete(ref('r', 5), 0)+ expect(ledger.shouldDeletePolicy(ref('r', 5), 1)).toBe(false)+ expect(ledger.shouldDeletePolicy(ref('r', 4), 1)).toBe(false)+ expect(ledger.shouldDeletePolicy(ref('r', 6), 1)).toBe(true)+ })++ it('an older generation recorded late does not lower the recorded generation', () => {+ const ledger = new OAuthBrokerDeleteLedger()+ recordSecret(ledger, ref('r', 5))+ recordSecret(ledger, ref('r', 3))+ expect(ledger.shouldDeleteSecret(ref('r', 4))).toBe(false)++ ledger.recordPolicyDelete(ref('r', 5), 0)+ ledger.recordPolicyDelete(ref('r', 3), 0)+ expect(ledger.shouldDeletePolicy(ref('r', 4), 1)).toBe(false)+ })++ it('a recipe recreated under the same name (new uid) deletes again at any generation', () => {+ const ledger = new OAuthBrokerDeleteLedger()+ recordSecret(ledger, ref('r', 5, 'uid-old'))+ ledger.recordPolicyDelete(ref('r', 5, 'uid-old'), 0)+ expect(ledger.shouldDeleteSecret(ref('r', 1, 'uid-old'))).toBe(false)+ expect(ledger.shouldDeletePolicy(ref('r', 1, 'uid-old'), 1)).toBe(false)++ expect(ledger.shouldDeleteSecret(ref('r', 1, 'uid-new'))).toBe(true)+ expect(ledger.shouldDeletePolicy(ref('r', 1, 'uid-new'), 1)).toBe(true)++ // Recording the recreated recipe replaces the old uid's higher generation.+ recordSecret(ledger, ref('r', 1, 'uid-new'))+ expect(ledger.shouldDeleteSecret(ref('r', 1, 'uid-new'))).toBe(false)+ expect(ledger.shouldDeleteSecret(ref('r', 2, 'uid-new'))).toBe(true)})it('does not let one recipe suppress another', () => {const ledger = new OAuthBrokerDeleteLedger()- ledger.recordSecretDelete('a', 1)- expect(ledger.shouldDeleteSecret('a', 1)).toBe(false)- expect(ledger.shouldDeleteSecret('a-b', 1)).toBe(true)- expect(ledger.shouldDeleteSecret('b', 1)).toBe(true)+ recordSecret(ledger, ref('a', 1))+ expect(ledger.shouldDeleteSecret(ref('a', 1))).toBe(false)+ expect(ledger.shouldDeleteSecret(ref('a-b', 1))).toBe(true)+ expect(ledger.shouldDeleteSecret(ref('b', 1))).toBe(true)})it('expires the NetworkPolicy entry after the TTL', () => {const ledger = new OAuthBrokerDeleteLedger()- ledger.recordPolicyDelete('r', 4, 1_000)- expect(ledger.shouldDeletePolicy('r', 4, 1_000 + OAUTH_BROKER_NP_TTL_MS - 1)).toBe(false)- expect(ledger.shouldDeletePolicy('r', 4, 1_000 + OAUTH_BROKER_NP_TTL_MS)).toBe(true)+ ledger.recordPolicyDelete(ref('r', 4), 1_000)+ expect(ledger.shouldDeletePolicy(ref('r', 4), 1_000 + OAUTH_BROKER_NP_TTL_MS - 1)).toBe(false)+ expect(ledger.shouldDeletePolicy(ref('r', 4), 1_000 + OAUTH_BROKER_NP_TTL_MS)).toBe(true)})it('invalidateSecret re-arms only the Secret side; the NetworkPolicy TTL survives', () => {const ledger = new OAuthBrokerDeleteLedger()- ledger.recordSecretDelete('r', 4)- ledger.recordPolicyDelete('r', 4, 0)- expect(ledger.shouldDeleteSecret('r', 4)).toBe(false)- expect(ledger.shouldDeletePolicy('r', 4, 1)).toBe(false)+ recordSecret(ledger, ref('r', 4))+ ledger.recordPolicyDelete(ref('r', 4), 0)+ expect(ledger.shouldDeleteSecret(ref('r', 4))).toBe(false)+ expect(ledger.shouldDeletePolicy(ref('r', 4), 1)).toBe(false)++ ledger.invalidateSecret('r')+ expect(ledger.shouldDeleteSecret(ref('r', 4))).toBe(true)+ expect(ledger.shouldDeletePolicy(ref('r', 4), 1)).toBe(false)+ })++ it('does not record a Secret delete when an invalidation landed after the epoch was read', () => {+ const ledger = new OAuthBrokerDeleteLedger()+ const epochBeforeDelete = ledger.secretEpoch('r')+ ledger.invalidateSecret('r')+ expect(ledger.recordSecretDelete(ref('r', 4), epochBeforeDelete)).toBe(false)+ expect(ledger.shouldDeleteSecret(ref('r', 4))).toBe(true)++ // Liveness witness: with a fresh epoch the same record is accepted.+ expect(ledger.recordSecretDelete(ref('r', 4), ledger.secretEpoch('r'))).toBe(true)+ expect(ledger.shouldDeleteSecret(ref('r', 4))).toBe(false)+ })+ it('an invalidation for one recipe does not reject another recipe record', () => {+ const ledger = new OAuthBrokerDeleteLedger()+ const epochBeforeDelete = ledger.secretEpoch('r2')ledger.invalidateSecret('r')- expect(ledger.shouldDeleteSecret('r', 4)).toBe(true)- expect(ledger.shouldDeletePolicy('r', 4, 1)).toBe(false)+ expect(ledger.recordSecretDelete(ref('r2', 4), epochBeforeDelete)).toBe(true)+ expect(ledger.shouldDeleteSecret(ref('r2', 4))).toBe(false)})it('invalidate drops both sides for the named recipe and no other', () => {const ledger = new OAuthBrokerDeleteLedger()- ledger.recordSecretDelete('r', 4)- ledger.recordPolicyDelete('r', 4, 0)- ledger.recordSecretDelete('r2', 4)- ledger.recordPolicyDelete('r2', 4, 0)+ recordSecret(ledger, ref('r', 4))+ ledger.recordPolicyDelete(ref('r', 4), 0)+ recordSecret(ledger, ref('r2', 4))+ ledger.recordPolicyDelete(ref('r2', 4), 0)++ ledger.invalidate('r')+ expect(ledger.shouldDeleteSecret(ref('r', 4))).toBe(true)+ expect(ledger.shouldDeletePolicy(ref('r', 4), 1)).toBe(true)+ expect(ledger.shouldDeleteSecret(ref('r2', 4))).toBe(false)+ expect(ledger.shouldDeletePolicy(ref('r2', 4), 1)).toBe(false)+ })+ it('a DELETE in flight across a recipe deletion that followed an ADDED is not recorded', () => {+ const ledger = new OAuthBrokerDeleteLedger()+ ledger.invalidateSecret('r')+ const epochBeforeDelete = ledger.secretEpoch('r')ledger.invalidate('r')- expect(ledger.shouldDeleteSecret('r', 4)).toBe(true)- expect(ledger.shouldDeletePolicy('r', 4, 1)).toBe(true)- expect(ledger.shouldDeleteSecret('r2', 4)).toBe(false)- expect(ledger.shouldDeletePolicy('r2', 4, 1)).toBe(false)+ expect(ledger.recordSecretDelete(ref('r', 4), epochBeforeDelete)).toBe(false)+ expect(ledger.shouldDeleteSecret(ref('r', 4))).toBe(true)})})diff --git a/workflow-recipes/src/reconciler/oauthBrokerDeleteLedger.ts b/workflow-recipes/src/reconciler/oauthBrokerDeleteLedger.tsindex 0f1667c0b..3b9068f98 100644--- a/workflow-recipes/src/reconciler/oauthBrokerDeleteLedger.ts+++ b/workflow-recipes/src/reconciler/oauthBrokerDeleteLedger.ts@@ -4,15 +4,33 @@* 1h TTL so a later pass can reap a leftover without a watch. Indexing only* by recipe name is the E.6 vacuity mutation: a generation bump must miss.*- * One entry per recipe (the last generation wins): generations only increase,- * so older entries can never match again and keeping them would grow the- * ledger by one key per spec edit for the life of the process.+ * One entry per recipe, holding the highest generation recorded for the+ * recipe's current uid. A pass whose generation is at or below that value is+ * skipped: a queued pass still carrying an older object must not delete what+ * a newer generation provisioned. A different uid is a recipe recreated under+ * the same name, whose generations restart at 1, so it always deletes and+ * replaces the entry even when the recipe DELETE event was missed.+ *+ * `invalidateSecret` bumps a per-recipe epoch. A Secret delete is recorded+ * only when the epoch is unchanged since before the DELETE was sent, so an+ * ADDED handled while the DELETE is in flight is not overwritten by it.*/export const OAUTH_BROKER_NP_TTL_MS = 60 * 60 * 1000-interface PolicyEntry {+/** The recipe identity the ledger keys on; a recipe's `metadata` satisfies it. */+export interface OAuthBrokerLedgerRecipe {+ name: string+ uid?: string+ generation?: number+}++interface SecretEntry {+ uid: string | undefinedgeneration: number+}++interface PolicyEntry extends SecretEntry {expiresAt: number}@@ -20,34 +38,56 @@ function normalizeGeneration(generation: number | undefined): number {return generation ?? 0}+function coversPass(entry: SecretEntry | undefined, recipe: OAuthBrokerLedgerRecipe): boolean {+ return (+ entry !== undefined &&+ entry.uid === recipe.uid &&+ normalizeGeneration(recipe.generation) <= entry.generation+ )+}++function nextEntry(entry: SecretEntry | undefined, recipe: OAuthBrokerLedgerRecipe): SecretEntry {+ const generation = normalizeGeneration(recipe.generation)+ if (entry !== undefined && entry.uid === recipe.uid) {+ return { uid: recipe.uid, generation: Math.max(entry.generation, generation) }+ }+ return { uid: recipe.uid, generation }+}+export class OAuthBrokerDeleteLedger {- private readonly secrets = new Map<string, number>()+ private readonly secrets = new Map<string, SecretEntry>()private readonly policies = new Map<string, PolicyEntry>()+ private readonly secretEpochs = new Map<string, number>()+ private epochClock = 0++ shouldDeleteSecret(recipe: OAuthBrokerLedgerRecipe): boolean {+ return !coversPass(this.secrets.get(recipe.name), recipe)+ }- shouldDeleteSecret(recipeName: string, generation: number | undefined): boolean {- return this.secrets.get(recipeName) !== normalizeGeneration(generation)+ /** Read before sending the DELETE and hand back to `recordSecretDelete`. */+ secretEpoch(recipeName: string): number {+ return this.secretEpochs.get(recipeName) ?? 0}- recordSecretDelete(recipeName: string, generation: number | undefined): void {- this.secrets.set(recipeName, normalizeGeneration(generation))+ /**+ * Returns false, recording nothing, when an invalidation landed after+ * `epochBeforeDelete` was read.+ */+ recordSecretDelete(recipe: OAuthBrokerLedgerRecipe, epochBeforeDelete: number): boolean {+ if (this.secretEpoch(recipe.name) !== epochBeforeDelete) return false+ this.secrets.set(recipe.name, nextEntry(this.secrets.get(recipe.name), recipe))+ return true}- shouldDeletePolicy(- recipeName: string,- generation: number | undefined,- nowMs = Date.now()- ): boolean {- const entry = this.policies.get(recipeName)- return (- entry === undefined ||- entry.generation !== normalizeGeneration(generation) ||- nowMs >= entry.expiresAt- )+ shouldDeletePolicy(recipe: OAuthBrokerLedgerRecipe, nowMs = Date.now()): boolean {+ const entry = this.policies.get(recipe.name)+ if (entry === undefined || !coversPass(entry, recipe)) return true+ return nowMs >= entry.expiresAt}- recordPolicyDelete(recipeName: string, generation: number | undefined, nowMs = Date.now()): void {- this.policies.set(recipeName, {- generation: normalizeGeneration(generation),+ recordPolicyDelete(recipe: OAuthBrokerLedgerRecipe, nowMs = Date.now()): void {+ this.policies.set(recipe.name, {+ ...nextEntry(this.policies.get(recipe.name), recipe),expiresAt: nowMs + OAUTH_BROKER_NP_TTL_MS,})}@@ -60,11 +100,17 @@ export class OAuthBrokerDeleteLedger {*/invalidateSecret(recipeName: string): void {this.secrets.delete(recipeName)+ this.epochClock += 1+ this.secretEpochs.set(recipeName, this.epochClock)}- /** Recipe deletion drops both sides. */+ /**+ * Recipe deletion drops both sides. The epoch entry goes too; the clock is+ * global, so a DELETE in flight that read a non-zero epoch still mismatches.+ */invalidate(recipeName: string): void {this.secrets.delete(recipeName)this.policies.delete(recipeName)+ this.secretEpochs.delete(recipeName)}}diff --git a/workflow-recipes/src/reconciler/reconciler.test.ts b/workflow-recipes/src/reconciler/reconciler.test.tsindex 21fe2ca2f..e1d2a5549 100644--- a/workflow-recipes/src/reconciler/reconciler.test.ts+++ b/workflow-recipes/src/reconciler/reconciler.test.ts@@ -15603,6 +15603,131 @@ describe('WorkflowRecipeReconciler', () => {infoSpy.mockRestore()vi.useRealTimers()})++ it('R1-L2: a late pass carrying an older generation does not re-delete the Secret', async () => {+ const infoSpy = captureLogger('info')+ await reconciler.ensureOAuthBrokerTokenSecret(reapRecipe(4))+ await reconciler.ensureOAuthBrokerTokenSecret(reapRecipe(5))+ // Liveness witness: both newer-generation deletes really ran.+ expect(secretDeletes()).toBe(2)++ await reconciler.ensureOAuthBrokerTokenSecret(reapRecipe(4))+ expect(secretDeletes()).toBe(2)+ expect(infoSpy).toHaveBeenCalledWith(+ expect.stringContaining('Skipping oauth-broker-token delete'),+ expect.objectContaining({ recipe: 'test-recipe', generation: 4 })+ )+ infoSpy.mockRestore()+ })++ it('R1-L2: a late pass carrying an older generation does not re-delete the NetworkPolicy', async () => {+ const infoSpy = captureLogger('info')+ await reapPolicy(reapRecipe(4))+ await reapPolicy(reapRecipe(5))+ // Liveness witness: both newer-generation deletes really ran.+ expect(policyDeletes()).toBe(2)++ await reapPolicy(reapRecipe(4))+ expect(policyDeletes()).toBe(2)+ expect(infoSpy).toHaveBeenCalledWith(+ expect.stringContaining('Skipping oauth-broker-egress delete'),+ expect.objectContaining({ recipe: 'test-recipe', generation: 4 })+ )+ infoSpy.mockRestore()+ })++ it('R1-L2: a recipe recreated under the same name is reaped even at a lower generation', async () => {+ await reconciler.ensureOAuthBrokerTokenSecret(reapRecipe(5))+ await reapPolicy(reapRecipe(5))+ expect(secretDeletes()).toBe(1)+ expect(policyDeletes()).toBe(1)++ // The recipe DELETE event was missed, so nothing invalidated the ledger.+ const recreated = makeRecipe({+ metadata: {+ name: 'test-recipe',+ namespace: 'sandbox-recipes',+ uid: 'uid-recreated',+ generation: 1,+ },+ })+ await reconciler.ensureOAuthBrokerTokenSecret(recreated)+ await reapPolicy(recreated)+ expect(secretDeletes()).toBe(2)+ expect(policyDeletes()).toBe(2)+ })++ it('R1-L1: an ADDED invalidation that lands while the DELETE is in flight is not overwritten', async () => {+ const recipe = reapRecipe(4)+ mockCoreApi.deleteNamespacedSecret.mockImplementationOnce(async () => {+ // Another writer recreated the Secret and its ADDED was handled before+ // this DELETE returned.+ reconciler.invalidateOAuthBrokerSecretLedger('test-recipe')+ throw { code: 404 }+ })+ await reconciler.ensureOAuthBrokerTokenSecret(recipe)+ expect(secretDeletes()).toBe(1)++ await reconciler.ensureOAuthBrokerTokenSecret(recipe)+ expect(secretDeletes()).toBe(2)++ // Liveness witness for the ledger itself: with no racing ADDED the+ // second delete is recorded and the next same-generation pass skips.+ await reconciler.ensureOAuthBrokerTokenSecret(recipe)+ expect(secretDeletes()).toBe(2)+ })++ describe.each([+ ['active', 'running'],+ ['terminal', 'completed'],+ ] as const)('R1-L13: the %s workflow short-circuit', (_label, execPhase) => {+ it('reaps a token Secret the recipe must not have, without the inner reconcile', async () => {+ const workflowReconcile = vi.fn()+ ;(+ reconciler as unknown as {+ workflowReconciler: {+ reconcile: typeof workflowReconcile+ validateWorkflowSpec: () => undefined+ ensureMcpHostRuntimeCredentials?: () => Promise<void>+ }+ }+ ).workflowReconciler = {+ reconcile: workflowReconcile,+ validateWorkflowSpec: () => undefined,+ ensureMcpHostRuntimeCredentials: vi.fn().mockResolvedValue(undefined),+ }+ const recipe = makeRecipe({+ spec: {+ agent: { provider: 'zai', model: 'glm-4.7' },+ workloads: [],+ steps: [{ id: 'run-qa', instruction: 'Validate the QA API workload.' }],+ },+ // Triggered run: awaitsTriggeredRun=false, so the pass stops at the+ // active (running) or terminal (completed) short-circuit.+ metadata: {+ name: 'test-recipe',+ namespace: 'sandbox-recipes',+ uid: 'uid-123',+ generation: 4,+ labels: { 'clerum.io/workflow-run-id': 'run-123' },+ },+ status: { phase: 'active', workflowExecution: { phase: execPhase } },+ })++ // The watch ADDED re-arms the ledger and enqueues this reconcile.+ reconciler.invalidateOAuthBrokerSecretLedger('test-recipe')+ const result = await reconciler.reconcile(recipe)++ // Witness that the short-circuit was taken, not the first-deploy path.+ expect(workflowReconcile).not.toHaveBeenCalled()+ expect(result.phase).toBe('active')+ expect(secretDeletes()).toBe(1)++ // The ledger still bounds it: a second pass of the same generation skips.+ await reconciler.reconcile(recipe)+ expect(secretDeletes()).toBe(1)+ })+ })})describe('G2 skipStatusPatch must not leave GFS deleted', () => {@@ -15744,6 +15869,13 @@ describe('WorkflowRecipeReconciler', () => {labels: { 'clerum.io/recipe': RECIPE, 'clerum.io/managed-by': 'wrc' },},},+ {+ metadata: {+ name: `${RECIPE}-snippet-runner-egress`,+ namespace: 'sandbox-recipes',+ labels: { 'clerum.io/recipe': RECIPE, 'clerum.io/managed-by': 'wrc' },+ },+ },{metadata: {name: `${RECIPE}-mcp-host-to-grok-proxy`,@@ -15763,16 +15895,20 @@ describe('WorkflowRecipeReconciler', () => {'run-g2')- // Liveness witness: the retry ran the prune LIST that listed GFS, so the- // absence of a GFS DELETE below is the catalog gate holding, not a retry- // that never reached the prune.+ // Liveness witness: the retry ran the prune and deleted an undesired+ // catalog sibling, so the absence of a GFS DELETE below is the catalog+ // gate holding, not a retry that never reached the prune.expect(mockNetworkingApi.listNamespacedNetworkPolicy).toHaveBeenCalled()expect(mockNetworkingApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledWith(- expect.objectContaining({ name: `${RECIPE}-mcp-host-to-grok-proxy` })+ expect.objectContaining({ name: `${RECIPE}-snippet-runner-egress` }))expect(mockNetworkingApi.deleteNamespacedNetworkPolicy).not.toHaveBeenCalledWith(expect.objectContaining({ name: GFS }))+ // The retry runs mid-run, so it never revokes a Codex/Grok proxy.+ expect(mockNetworkingApi.deleteNamespacedNetworkPolicy).not.toHaveBeenCalledWith(+ expect.objectContaining({ name: `${RECIPE}-mcp-host-to-grok-proxy` })+ )})})diff --git a/workflow-recipes/src/reconciler/resourceBuilder.test.ts b/workflow-recipes/src/reconciler/resourceBuilder.test.tsindex 7b48c62a6..719a8bfd3 100644--- a/workflow-recipes/src/reconciler/resourceBuilder.test.ts+++ b/workflow-recipes/src/reconciler/resourceBuilder.test.ts@@ -2082,30 +2082,24 @@ describe('recipe OAuth broker token', () => {it('resolves watch ADDED only for the canonical oauth-broker-token Secret name', () => {const recipe = 'test-recipe'- const canonical = oauthBrokerTokenSecretName(recipe)+ // The Secret WRC itself writes: the resolver must bind it, so a drift in+ // the builder's name or labels fails here.+ const built = buildOAuthBrokerTokenSecret(recipe, 'jwt', 'sandbox-recipes')+ const canonical = built.metadata?.name ?? ''+ const labels = built.metadata?.labels ?? {}+ expect(canonical).toBe(oauthBrokerTokenSecretName(recipe))expect(parseOAuthBrokerTokenSecretRecipe(canonical)).toBe(recipe)expect(parseOAuthBrokerTokenSecretRecipe('wf--oauth-broker-token')).toBeUndefined()+ expect(resolveOAuthBrokerTokenWatchRecipe(canonical, labels)).toBe(recipe)+ const withoutRecipeLabel = Object.fromEntries(+ Object.entries(labels).filter(([key]) => key !== 'clerum.io/recipe')+ )+ expect(resolveOAuthBrokerTokenWatchRecipe(canonical, withoutRecipeLabel)).toBe(recipe)+ expect(resolveOAuthBrokerTokenWatchRecipe('forged-name', labels)).toBeUndefined()expect(resolveOAuthBrokerTokenWatchRecipe(canonical, {- 'clerum.io/component': 'oauth-broker-token',- 'clerum.io/recipe': recipe,- })- ).toBe(recipe)- expect(- resolveOAuthBrokerTokenWatchRecipe(canonical, {- 'clerum.io/component': 'oauth-broker-token',- })- ).toBe(recipe)- expect(- resolveOAuthBrokerTokenWatchRecipe('forged-name', {- 'clerum.io/component': 'oauth-broker-token',- 'clerum.io/recipe': recipe,- })- ).toBeUndefined()- expect(- resolveOAuthBrokerTokenWatchRecipe(canonical, {+ ...labels,'clerum.io/component': 'other',- 'clerum.io/recipe': recipe,})).toBeUndefined()})diff --git a/workflow-recipes/src/reconciler/secretWatcher.test.ts b/workflow-recipes/src/reconciler/secretWatcher.test.tsindex 38296147d..2188b6fa6 100644--- a/workflow-recipes/src/reconciler/secretWatcher.test.ts+++ b/workflow-recipes/src/reconciler/secretWatcher.test.ts@@ -1,4 +1,5 @@import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'+import { buildOAuthBrokerTokenSecret } from './resourceBuilder'import { OWNER_RECIPE_LABEL_KEY, SHARED_LABEL_KEY } from './secretOwnership'import { SecretReverseIndex } from './secretReverseIndex'import { SecretWatcher } from './secretWatcher'@@ -230,16 +231,9 @@ describe('SecretWatcher', () => {invalidated.push(recipeName)})- const brokerSecret = {- metadata: {- name: 'wf-test-recipe-oauth-broker-token',- labels: {- 'clerum.io/component': 'oauth-broker-token',- 'clerum.io/recipe': 'test-recipe',- },- },- data: { 'broker-token': 'eA==' },- }+ // The Secret WRC itself writes, so a drift in the builder's labels or+ // name breaks this test instead of passing against a hand-written copy.+ const brokerSecret = buildOAuthBrokerTokenSecret('test-recipe', 'jwt', 'sandbox-recipes')watcher.handleEvent('ADDED', brokerSecret)watcher.handleEvent('ADDED', brokerSecret)@@ -265,33 +259,24 @@ describe('SecretWatcher', () => {invalidated.push(recipeName)})- const labels = {- 'clerum.io/component': 'oauth-broker-token',- 'clerum.io/recipe': 'test-recipe',- }+ const canonical = buildOAuthBrokerTokenSecret('test-recipe', 'jwt', 'sandbox-recipes')watcher.handleEvent('ADDED', {- metadata: { name: 'forged-oauth-broker-token', labels },- data: { 'broker-token': 'eA==' },+ ...canonical,+ metadata: { ...canonical.metadata, name: 'forged-oauth-broker-token' },})watcher.handleEvent('ADDED', {+ ...canonical,metadata: {- name: 'wf-test-recipe-oauth-broker-token',- labels: { 'clerum.io/component': 'other', 'clerum.io/recipe': 'test-recipe' },+ ...canonical.metadata,+ labels: { ...canonical.metadata?.labels, 'clerum.io/component': 'other' },},- data: { 'broker-token': 'eA==' },- })- watcher.handleEvent('MODIFIED', {- metadata: { name: 'wf-test-recipe-oauth-broker-token', labels },- data: { 'broker-token': 'eA==' },})+ watcher.handleEvent('MODIFIED', canonical)expect(invalidated).toEqual([])// Liveness witness: the same watcher does invalidate for the canonical// ADDED, so the empty list above is not the callback being unwired.- watcher.handleEvent('ADDED', {- metadata: { name: 'wf-test-recipe-oauth-broker-token', labels },- data: { 'broker-token': 'eA==' },- })+ watcher.handleEvent('ADDED', canonical)expect(invalidated).toEqual(['test-recipe'])vi.advanceTimersByTime(DEBOUNCE_MS)expect(enqueued).toEqual([])diff --git a/workflow-recipes/src/reconciler/workflowRecipeReconciler.ts b/workflow-recipes/src/reconciler/workflowRecipeReconciler.tsindex 894bb5165..f484840f6 100644--- a/workflow-recipes/src/reconciler/workflowRecipeReconciler.ts+++ b/workflow-recipes/src/reconciler/workflowRecipeReconciler.ts@@ -1477,6 +1477,8 @@ export class WorkflowRecipeReconciler {* pending a retry would stay unapplied until the run ends. The retry* re-prunes as well, with a verdict computed for this pass: a prune that* failed is what set the marker, so an apply-only retry could never clear it.+ * The Codex and Grok proxy policies are never revoked by this retry, since+ * a run may be using them; the first reconcile() after the run prunes them.** Returns the requeue delay the short-circuit must add: the progress base* while a policy is still pending, the transient base when the apply threw@@ -1964,6 +1966,16 @@ export class WorkflowRecipeReconciler {}}+ // B3(a): the awaiting-trigger, terminal, in-progress and active+ // short-circuits below return before the first-deploy path that calls+ // ensureOAuthBrokerTokenSecret, so the token-ADDED re-reconcile would+ // never reach the reap. Only the delete branch runs here (issuance stays+ // on the first-deploy path and the rotation loop), and the delete ledger+ // bounds it to one DELETE per generation or ADDED.+ if (!rb.recipeHasBackgroundAccessClient(recipe)) {+ await this.ensureOAuthBrokerTokenSecret(recipe)+ }+if (currentPhase === 'active' &&awaitsTriggeredRun &&@@ -7518,13 +7530,14 @@ export class WorkflowRecipeReconciler {if (!rb.recipeHasBackgroundAccessClient(recipe)) {const generation = recipe.metadata.generation- if (!this.oauthBrokerDeleteLedger.shouldDeleteSecret(recipeName, generation)) {+ if (!this.oauthBrokerDeleteLedger.shouldDeleteSecret(recipe.metadata)) {createLogger('wrc', recipeName).info('Skipping oauth-broker-token delete; generation already seen',{ recipe: recipeName, generation })return}+ const epochBeforeDelete = this.oauthBrokerDeleteLedger.secretEpoch(recipeName)const outcome = await observeNamespacedDelete(() =>this.coreApi.deleteNamespacedSecret({ name: secretName, namespace: ns }))@@ -7543,8 +7556,14 @@ export class WorkflowRecipeReconciler {...deleteOutcomeFields(outcome),})}- if (shouldRecordDelete(outcome)) {- this.oauthBrokerDeleteLedger.recordSecretDelete(recipeName, generation)+ if (+ shouldRecordDelete(outcome) &&+ !this.oauthBrokerDeleteLedger.recordSecretDelete(recipe.metadata, epochBeforeDelete)+ ) {+ log.info('oauth-broker-token Secret re-added during delete; not recording', {+ recipe: recipeName,+ generation,+ })}return}@@ -7610,7 +7629,7 @@ export class WorkflowRecipeReconciler {if (!policy) {const recipeName = recipe.metadata.nameconst generation = recipe.metadata.generation- if (!this.oauthBrokerDeleteLedger.shouldDeletePolicy(recipeName, generation)) {+ if (!this.oauthBrokerDeleteLedger.shouldDeletePolicy(recipe.metadata)) {createLogger('wrc', recipeName).info('Skipping oauth-broker-egress delete; generation already seen',{ recipe: recipeName, generation }@@ -7636,7 +7655,7 @@ export class WorkflowRecipeReconciler {})}if (shouldRecordDelete(outcome)) {- this.oauthBrokerDeleteLedger.recordPolicyDelete(recipeName, generation)+ this.oauthBrokerDeleteLedger.recordPolicyDelete(recipe.metadata)}return}diff --git a/workflow-recipes/src/workflow/workflowReconciler.ts b/workflow-recipes/src/workflow/workflowReconciler.tsindex 671e407e0..3a92ce2d9 100644--- a/workflow-recipes/src/workflow/workflowReconciler.ts+++ b/workflow-recipes/src/workflow/workflowReconciler.ts@@ -231,6 +231,11 @@ type RunLaneNetworkPolicyApplyResult =reason: 'terminating' | 'contended'}| { policy: string; action: 'conflict'; reason: NetworkPolicyConflictReason }+/** Why a Codex/Grok proxy policy survives a prune that does not desire it. */+interface RunLaneProxyKeepReasons {+ codex?: 'uncertain' | 'run-in-progress'+ grok?: 'uncertain' | 'run-in-progress'+}// Read-then-write rounds per NetworkPolicy and pass. Three rounds is at most// three writes, the same as the create plus two replaces it replaced.const NETWORK_POLICY_APPLY_ROUNDS = 3@@ -2990,11 +2995,14 @@ export class WorkflowReconciler {* active short-circuits return before reconcile(), so a policy a previous* pass left pending (terminating, contended, or a prune that failed) would* otherwise stay as it is until the run ends. Runs the same build, apply and- * prune unit reconcile() runs, with a verdict computed now: an uncertain- * Codex or Grok verdict keeps its proxy policy, and the catalog gate keeps- * everything the run lane does not own. A prune that failed sets- * retryPending, so without the prune here the marker would requeue forever- * and never reach the leftover.+ * prune unit reconcile() runs, with a verdict computed now, with one+ * difference: the Codex and Grok proxy policies are never revoked here.+ * The verdict can change while a step is using the proxy, so a revoke+ * mid-run would cut its egress; a proxy the verdict still requires is+ * created (an allow), and one it no longer wants is pruned by the first+ * reconcile() after the run. The catalog gate keeps everything the run lane+ * does not own. A prune that failed sets retryPending, so without the prune+ * here the marker would requeue forever and never reach the leftover.*/async retryRunLaneNetworkPolicies(recipeName: string,@@ -3019,7 +3027,7 @@ export class WorkflowReconciler {runtimeScopeRecipeName,codexView)- return this.applyWorkflowNetworkPolicies(+ const { policies, catalog } = await this.buildWorkflowNetworkPoliciesForSpec(recipeName,recipeUid,spec,@@ -3029,6 +3037,10 @@ export class WorkflowReconciler {false,codexVerdict.grokProjection)+ return this.applyAndPruneRunLaneNetworkPolicies(recipeName, recipeUid, policies, catalog, {+ codex: 'run-in-progress',+ grok: 'run-in-progress',+ })}async ensureCoordinatorRuntimeCredentials(@@ -3435,16 +3447,26 @@ export class WorkflowReconciler {eagerSdkMcpHost,grokProjection)+ return this.applyAndPruneRunLaneNetworkPolicies(recipeName, recipeUid, policies, catalog, {+ codex: codexProjection.eligibility === 'uncertain' ? 'uncertain' : undefined,+ grok: grokProjection?.eligibility === 'uncertain' ? 'uncertain' : undefined,+ })+ }++ private async applyAndPruneRunLaneNetworkPolicies(+ recipeName: string,+ recipeUid: string,+ policies: k8s.V1NetworkPolicy[],+ catalog: Set<string>,+ keepProxies: RunLaneProxyKeepReasons+ ): Promise<WorkflowNetworkPolicyApplySummary> {const summary = await this.applyNetworkPolicyList(policies)const policyNames = new Set(policies.map(policy => policy.metadata?.name))const prune = await this.pruneUndesiredRunLaneNetworkPolicies(recipeName,policyNames,catalog,- {- skipCodexProxy: codexProjection.eligibility === 'uncertain',- skipGrokProxy: grokProjection?.eligibility === 'uncertain',- }+ keepProxies)if (prune.retryPending) summary.retryPending = trueconst legacy = await this.pruneLegacyMcpServersInternetEgressPolicy(recipeName, recipeUid)@@ -3457,18 +3479,20 @@ export class WorkflowReconciler {* Two-term selector only (`managed-by=wrc`): a single-term recipe selector* would also return the recipes-lane policies in the same namespace.* Universe is the factory catalog (GFS never in it), not LIST \ desired.- * Uncertain Codex/Grok eligibility keeps those proxy names even when they- * are absent from this pass's desired set.+ * A Codex/Grok proxy with a keep reason (an uncertain verdict, or a run in+ * progress on the retry path) survives even when it is absent from this+ * pass's desired set. A listed policy with an ownerReference is skipped,+ * matching the apply's ownership veto.*/private async pruneUndesiredRunLaneNetworkPolicies(recipeName: string,desiredNames: Set<string | undefined>,catalog: Set<string>,- uncertain: { skipCodexProxy: boolean; skipGrokProxy: boolean }+ keepProxies: RunLaneProxyKeepReasons): Promise<{ retryPending: boolean }> {const namespace = this.deps.config.sandboxNamespaceconst labelSelector = `clerum.io/recipe=${recipeName},clerum.io/managed-by=wrc`- let list: { items?: Array<{ metadata?: { name?: string } }> }+ let list: { items?: k8s.V1NetworkPolicy[] }try {list = await this.deps.networkingApi.listNamespacedNetworkPolicy({namespace,@@ -3487,27 +3511,37 @@ export class WorkflowReconciler {for (const policy of list.items ?? []) {const name = policy.metadata?.nameif (!name || desiredNames.has(name)) continue- if (name === codexProxyPolicyName && uncertain.skipCodexProxy) {+ const keepReason =+ name === codexProxyPolicyName+ ? keepProxies.codex+ : name === grokProxyPolicyName+ ? keepProxies.grok+ : undefined+ if (keepReason) {this.log.debug('Skipping run-lane NP prune', {recipe: recipeName,policy: name,- reason: 'uncertain',+ reason: keepReason,})continue}- if (name === grokProxyPolicyName && uncertain.skipGrokProxy) {+ if (!catalog.has(name)) {this.log.debug('Skipping run-lane NP prune', {recipe: recipeName,policy: name,- reason: 'uncertain',+ reason: 'not-in-catalog',})continue}- if (!catalog.has(name)) {- this.log.debug('Skipping run-lane NP prune', {+ // Same ownership veto the run-lane apply uses: a policy under our name+ // that carries an ownerReference belongs to someone else, and the apply+ // already refuses to replace it.+ const ownership = classifyOwnerlessNetworkPolicyOwnership(policy)+ if (ownership.kind !== 'owned') {+ this.log.warn('Skipping run-lane NP prune', {recipe: recipeName,policy: name,- reason: 'not-in-catalog',+ reason: ownership.kind === 'conflict' ? ownership.reason : ownership.kind,})continue}diff --git a/workflow-recipes/tests/unit/workflow/workflowReconciler.test.ts b/workflow-recipes/tests/unit/workflow/workflowReconciler.test.tsindex 5a7865788..6b19ca87a 100644--- a/workflow-recipes/tests/unit/workflow/workflowReconciler.test.ts+++ b/workflow-recipes/tests/unit/workflow/workflowReconciler.test.ts@@ -3507,6 +3507,46 @@ describe('WorkflowReconciler — reconcile loop', () => {expect(apiserver.live.has(apiserver.key(SANDBOX, COORD_TO_MCP_HOST))).toBe(false)})+ it('R1-M1: does not prune a catalog policy that carries an ownerReference (apply refuses it too)', async () => {+ const apiserver = makeApiserverNetworkingApi()+ seedLivePolicy(apiserver, COORD_TO_MCP_HOST, WRC_LABELS)+ const seeded = apiserver.live.get(apiserver.key(SANDBOX, COORD_TO_MCP_HOST))!+ seeded.metadata = {+ ...seeded.metadata,+ ownerReferences: [+ {+ apiVersion: 'apps/v1',+ kind: 'Deployment',+ name: 'someone-else',+ uid: 'uid-deployment',+ controller: true,+ },+ ],+ }+ seedPrunableSibling(apiserver)+ const warnLog = captureLogger('warn')+ try {+ const reconciler = new WorkflowReconciler(+ makeDeps({ networkingApi: apiserver.api as never })+ )++ await applyPolicies(reconciler, { awaitsTriggeredRun: true })++ expectListWitness(apiserver.api)+ // Liveness witness: the WRC-owned sibling in the same LIST was pruned.+ expectPrunableSiblingDeleted(apiserver)+ expect(sandboxDeletes(apiserver.api).map(d => d.name)).not.toContain(COORD_TO_MCP_HOST)+ expect(apiserver.live.has(apiserver.key(SANDBOX, COORD_TO_MCP_HOST))).toBe(true)+ expect(warnLog).toHaveBeenCalledWith('Skipping run-lane NP prune', {+ recipe: RECIPE,+ policy: COORD_TO_MCP_HOST,+ reason: 'owner-reference-mismatch',+ })+ } finally {+ warnLog.mockRestore()+ }+ })+it('eager SDK path still prunes a leftover Grok proxy', async () => {const apiserver = makeApiserverNetworkingApi()seedLivePolicy(apiserver, GROK_PROXY, WRC_LABELS)@@ -5261,7 +5301,10 @@ describe('WorkflowReconciler — reconcile loop', () => {// runs while the run is in progress or active. It has to prune again, or// the marker requeues forever without ever reaching the leftover.describe('when a prune left a leftover behind', () => {- const LEFTOVER = 'test-wf-mcp-host-to-grok-proxy'+ // A catalog member the snippet lane does not want and that no eligibility+ // verdict governs: the retry must prune it so the marker can clear.+ const LEFTOVER = 'test-wf-coord-to-mcp-host'+ const GROK_PROXY = 'test-wf-mcp-host-to-grok-proxy'const TWO_TERM_SELECTOR = 'clerum.io/recipe=test-wf,clerum.io/managed-by=wrc'type Converged = Awaited<ReturnType<typeof convergedRunLane>>['apiserver']@@ -5359,9 +5402,13 @@ describe('WorkflowReconciler — reconcile loop', () => {// The retry computes its own verdict. These tests pin it: a proxy the// fresh verdict still wants (or cannot decide about) survives, and one- // it no longer wants goes. The sibling delete is the liveness witness- // in every case: the prune loop demonstrably iterated the same LIST.+ // it no longer wants goes. Every case also asserts the delete of a+ // snippet-lane sibling the agent spec no longer wants+ // (`expectSnippetSiblingPruned`): that is the liveness witness proving+ // the prune loop iterated the same LIST, so "no proxy was deleted"+ // cannot pass because the loop never ran.describe('with a fresh Codex/Grok verdict', () => {+ const SNIPPET_SIBLING = 'test-wf-snippet-runner-egress'const CODEX_LIVE = 'test-wf-mcp-host-to-codex-proxy'const projection = (overrides: Record<string, unknown>) => ({targets: [],@@ -5426,7 +5473,16 @@ describe('WorkflowReconciler — reconcile loop', () => {.map(([arg]) => arg.name).filter(name => name.endsWith('-proxy'))- it('keeps a proxy the fresh verdict still requires and prunes the undesired twin', async () => {+ function expectSnippetSiblingPruned(apiserver: Converged): void {+ expect(+ apiserver.api.deleteNamespacedNetworkPolicy.mock.calls.map(([arg]) => arg.name)+ ).toContain(SNIPPET_SIBLING)+ expect(apiserver.live.has(apiserver.key('sandbox-recipes', SNIPPET_SIBLING))).toBe(+ false+ )+ }++ it('keeps a proxy the fresh verdict still requires, and the undesired twin', async () => {const { apiserver, summary } = await retryWith({codex: {@@ -5436,12 +5492,14 @@ describe('WorkflowReconciler — reconcile loop', () => {},grok: {},},- [CODEX_LIVE, LEFTOVER]+ [CODEX_LIVE, GROK_PROXY])expect(apiserver.api.listNamespacedNetworkPolicy).toHaveBeenCalledTimes(1)- expect(deletedNames(apiserver)).toEqual([LEFTOVER])+ expectSnippetSiblingPruned(apiserver)+ expect(deletedNames(apiserver)).toEqual([])expect(apiserver.live.has(apiserver.key('sandbox-recipes', CODEX_LIVE))).toBe(true)+ expect(apiserver.live.has(apiserver.key('sandbox-recipes', GROK_PROXY))).toBe(true)expect(summary.retryPending).toBe(false)})@@ -5451,25 +5509,55 @@ describe('WorkflowReconciler — reconcile loop', () => {codex: { eligibility: 'uncertain', reason: 'forbidden' },grok: { eligibility: 'uncertain', reason: 'forbidden' },},- [CODEX_LIVE, LEFTOVER]+ [CODEX_LIVE, GROK_PROXY])expect(apiserver.api.listNamespacedNetworkPolicy).toHaveBeenCalledTimes(1)+ expectSnippetSiblingPruned(apiserver)expect(deletedNames(apiserver)).toEqual([])expect(apiserver.live.has(apiserver.key('sandbox-recipes', CODEX_LIVE))).toBe(true)- expect(apiserver.live.has(apiserver.key('sandbox-recipes', LEFTOVER))).toBe(true)+ expect(apiserver.live.has(apiserver.key('sandbox-recipes', GROK_PROXY))).toBe(true)expect(summary.retryPending).toBe(false)})- it('prunes a proxy once the fresh verdict decides it is no longer wanted', async () => {+ // Decision: the retry never revokes a Codex/Grok proxy. It runs only+ // while a run is in progress or active, the path without the marker+ // never prunes proxies mid-run, and a step of the running run may still+ // be calling the proxy. The next reconcile() after the run prunes it.+ it('never revokes a proxy mid-run, even when the fresh verdict no longer wants it', async () => {const { apiserver, summary } = await retryWith({ codex: {}, grok: {} }, [CODEX_LIVE,- LEFTOVER,+ GROK_PROXY,])expect(apiserver.api.listNamespacedNetworkPolicy).toHaveBeenCalledTimes(1)- expect(deletedNames(apiserver).sort()).toEqual([CODEX_LIVE, LEFTOVER].sort())- expect(apiserver.live.has(apiserver.key('sandbox-recipes', CODEX_LIVE))).toBe(false)+ expectSnippetSiblingPruned(apiserver)+ expect(deletedNames(apiserver)).toEqual([])+ expect(apiserver.live.has(apiserver.key('sandbox-recipes', CODEX_LIVE))).toBe(true)+ expect(apiserver.live.has(apiserver.key('sandbox-recipes', GROK_PROXY))).toBe(true)+ // A kept proxy is not a pending retry, so the marker can clear.+ expect(summary.retryPending).toBe(false)+ })++ it('still creates a proxy the fresh verdict requires (an allow, never a revoke)', async () => {+ const { apiserver, summary } = await retryWith(+ {+ codex: {+ eligibility: 'eligible',+ requiresCodexProxyEgress: true,+ reason: 'granted',+ },+ grok: {},+ },+ []+ )++ expect(+ apiserver.api.createNamespacedNetworkPolicy.mock.calls.map(+ ([arg]) => arg.body.metadata?.name+ )+ ).toContain(CODEX_LIVE)+ expect(apiserver.live.has(apiserver.key('sandbox-recipes', CODEX_LIVE))).toBe(true)expect(summary.retryPending).toBe(false)})})