Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript

fix(workflow-recipes): address PR #855 review round 2 findings

Alfredo Lopezb64325095ec6
diff --git a/workflow-recipes/package-lock.json b/workflow-recipes/package-lock.json
index 5fb7841e7..486a03e71 100644
--- a/workflow-recipes/package-lock.json
+++ b/workflow-recipes/package-lock.json
@@ -1,12 +1,12 @@
{
"name": "workload-recipes-controller",
- "version": "0.1.413",
+ "version": "0.1.414",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "workload-recipes-controller",
- "version": "0.1.413",
+ "version": "0.1.414",
"license": "SEE LICENSE IN LICENSE",
"dependencies": {
"@aws-sdk/client-s3": "^3.1009.0",
diff --git a/workflow-recipes/package.json b/workflow-recipes/package.json
index 171c101d4..b7207d280 100644
--- a/workflow-recipes/package.json
+++ b/workflow-recipes/package.json
@@ -1,6 +1,6 @@
{
"name": "workload-recipes-controller",
- "version": "0.1.413",
+ "version": "0.1.414",
"description": "Workload Recipes Controller (WRC) - WorkloadRecipe lifecycle management and MCP server interface",
"main": "dist/main.js",
"engines": {
diff --git a/workflow-recipes/src/k8sClient.test.ts b/workflow-recipes/src/k8sClient.test.ts
index 271a46756..3438c3fb8 100644
--- a/workflow-recipes/src/k8sClient.test.ts
+++ b/workflow-recipes/src/k8sClient.test.ts
@@ -17,6 +17,7 @@ import {
workflowRecipeFromWatchObject,
} from './k8sClient'
import { captureLogger } from './reconciler/__tests__/captureLogger'
+import { buildOAuthBrokerTokenSecret } from './reconciler/resourceBuilder'
import type { SecretEventType, SecretLike } from './reconciler/secretWatcher'
import type { WorkflowRecipeCRD } from './types'
@@ -3345,13 +3346,12 @@ describe('handleOAuthBrokerTokenAdded', () => {
secretLoops.length = 0
await (internal as unknown as { startSecretWatch: () => Promise<void> }).startSecretWatch()
- const brokerToken = {
- metadata: {
- name: 'wf-no-bg-oauth-broker-token',
- labels: { 'clerum.io/component': 'oauth-broker-token', 'clerum.io/recipe': 'no-bg' },
- },
- data: { token: 'x' },
- }
+ // The Secret WRC itself writes, not a hand-written copy of its labels.
+ const brokerToken = buildOAuthBrokerTokenSecret(
+ 'no-bg',
+ 'jwt',
+ internal.config.sandboxNamespace
+ )
// Liveness witness: the sandbox-namespace watcher exists and reaches the handler.
const sandbox = secretLoops.find(l => l.namespace === internal.config.sandboxNamespace)
diff --git a/workflow-recipes/src/reconciler/oauthBrokerDeleteLedger.test.ts b/workflow-recipes/src/reconciler/oauthBrokerDeleteLedger.test.ts
index b3d94a744..a918e2f55 100644
--- a/workflow-recipes/src/reconciler/oauthBrokerDeleteLedger.test.ts
+++ b/workflow-recipes/src/reconciler/oauthBrokerDeleteLedger.test.ts
@@ -1,68 +1,136 @@
import { describe, expect, it } from 'vitest'
-import { OAUTH_BROKER_NP_TTL_MS, OAuthBrokerDeleteLedger } from './oauthBrokerDeleteLedger'
+import {
+ OAUTH_BROKER_NP_TTL_MS,
+ OAuthBrokerDeleteLedger,
+ type OAuthBrokerLedgerRecipe,
+} from './oauthBrokerDeleteLedger'
+
+function ref(name: string, generation: number, uid = `uid-${name}`): OAuthBrokerLedgerRecipe {
+ return { name, uid, generation }
+}
+
+function recordSecret(ledger: OAuthBrokerDeleteLedger, recipe: OAuthBrokerLedgerRecipe): void {
+ expect(ledger.recordSecretDelete(recipe, ledger.secretEpoch(recipe.name))).toBe(true)
+}
describe('OAuthBrokerDeleteLedger', () => {
it('remembers a Secret delete for the recorded generation only', () => {
const ledger = new OAuthBrokerDeleteLedger()
- expect(ledger.shouldDeleteSecret('r', 4)).toBe(true)
- ledger.recordSecretDelete('r', 4)
- expect(ledger.shouldDeleteSecret('r', 4)).toBe(false)
- expect(ledger.shouldDeleteSecret('r', 5)).toBe(true)
+ expect(ledger.shouldDeleteSecret(ref('r', 4))).toBe(true)
+ recordSecret(ledger, ref('r', 4))
+ expect(ledger.shouldDeleteSecret(ref('r', 4))).toBe(false)
+ expect(ledger.shouldDeleteSecret(ref('r', 5))).toBe(true)
})
- it('keeps one entry per recipe: a newer generation replaces the older one', () => {
+ it('keeps the highest generation per recipe: an older generation arriving late is skipped', () => {
const ledger = new OAuthBrokerDeleteLedger()
- ledger.recordSecretDelete('r', 4)
- ledger.recordSecretDelete('r', 5)
- // Generations only increase, so the older key is dead weight. Not remembering it
- // is what bounds the ledger at one entry per recipe instead of one per spec edit.
- expect(ledger.shouldDeleteSecret('r', 5)).toBe(false)
- expect(ledger.shouldDeleteSecret('r', 4)).toBe(true)
-
- ledger.recordPolicyDelete('r', 4, 0)
- ledger.recordPolicyDelete('r', 5, 0)
- expect(ledger.shouldDeletePolicy('r', 5, 1)).toBe(false)
- expect(ledger.shouldDeletePolicy('r', 4, 1)).toBe(true)
+ recordSecret(ledger, ref('r', 4))
+ recordSecret(ledger, ref('r', 5))
+ expect(ledger.shouldDeleteSecret(ref('r', 5))).toBe(false)
+ expect(ledger.shouldDeleteSecret(ref('r', 4))).toBe(false)
+ expect(ledger.shouldDeleteSecret(ref('r', 6))).toBe(true)
+
+ ledger.recordPolicyDelete(ref('r', 4), 0)
+ ledger.recordPolicyDelete(ref('r', 5), 0)
+ expect(ledger.shouldDeletePolicy(ref('r', 5), 1)).toBe(false)
+ expect(ledger.shouldDeletePolicy(ref('r', 4), 1)).toBe(false)
+ expect(ledger.shouldDeletePolicy(ref('r', 6), 1)).toBe(true)
+ })
+
+ it('an older generation recorded late does not lower the recorded generation', () => {
+ const ledger = new OAuthBrokerDeleteLedger()
+ recordSecret(ledger, ref('r', 5))
+ recordSecret(ledger, ref('r', 3))
+ expect(ledger.shouldDeleteSecret(ref('r', 4))).toBe(false)
+
+ ledger.recordPolicyDelete(ref('r', 5), 0)
+ ledger.recordPolicyDelete(ref('r', 3), 0)
+ expect(ledger.shouldDeletePolicy(ref('r', 4), 1)).toBe(false)
+ })
+
+ it('a recipe recreated under the same name (new uid) deletes again at any generation', () => {
+ const ledger = new OAuthBrokerDeleteLedger()
+ recordSecret(ledger, ref('r', 5, 'uid-old'))
+ ledger.recordPolicyDelete(ref('r', 5, 'uid-old'), 0)
+ expect(ledger.shouldDeleteSecret(ref('r', 1, 'uid-old'))).toBe(false)
+ expect(ledger.shouldDeletePolicy(ref('r', 1, 'uid-old'), 1)).toBe(false)
+
+ expect(ledger.shouldDeleteSecret(ref('r', 1, 'uid-new'))).toBe(true)
+ expect(ledger.shouldDeletePolicy(ref('r', 1, 'uid-new'), 1)).toBe(true)
+
+ // Recording the recreated recipe replaces the old uid's higher generation.
+ recordSecret(ledger, ref('r', 1, 'uid-new'))
+ expect(ledger.shouldDeleteSecret(ref('r', 1, 'uid-new'))).toBe(false)
+ expect(ledger.shouldDeleteSecret(ref('r', 2, 'uid-new'))).toBe(true)
})
it('does not let one recipe suppress another', () => {
const ledger = new OAuthBrokerDeleteLedger()
- ledger.recordSecretDelete('a', 1)
- expect(ledger.shouldDeleteSecret('a', 1)).toBe(false)
- expect(ledger.shouldDeleteSecret('a-b', 1)).toBe(true)
- expect(ledger.shouldDeleteSecret('b', 1)).toBe(true)
+ recordSecret(ledger, ref('a', 1))
+ expect(ledger.shouldDeleteSecret(ref('a', 1))).toBe(false)
+ expect(ledger.shouldDeleteSecret(ref('a-b', 1))).toBe(true)
+ expect(ledger.shouldDeleteSecret(ref('b', 1))).toBe(true)
})
it('expires the NetworkPolicy entry after the TTL', () => {
const ledger = new OAuthBrokerDeleteLedger()
- ledger.recordPolicyDelete('r', 4, 1_000)
- expect(ledger.shouldDeletePolicy('r', 4, 1_000 + OAUTH_BROKER_NP_TTL_MS - 1)).toBe(false)
- expect(ledger.shouldDeletePolicy('r', 4, 1_000 + OAUTH_BROKER_NP_TTL_MS)).toBe(true)
+ ledger.recordPolicyDelete(ref('r', 4), 1_000)
+ expect(ledger.shouldDeletePolicy(ref('r', 4), 1_000 + OAUTH_BROKER_NP_TTL_MS - 1)).toBe(false)
+ expect(ledger.shouldDeletePolicy(ref('r', 4), 1_000 + OAUTH_BROKER_NP_TTL_MS)).toBe(true)
})
it('invalidateSecret re-arms only the Secret side; the NetworkPolicy TTL survives', () => {
const ledger = new OAuthBrokerDeleteLedger()
- ledger.recordSecretDelete('r', 4)
- ledger.recordPolicyDelete('r', 4, 0)
- expect(ledger.shouldDeleteSecret('r', 4)).toBe(false)
- expect(ledger.shouldDeletePolicy('r', 4, 1)).toBe(false)
+ recordSecret(ledger, ref('r', 4))
+ ledger.recordPolicyDelete(ref('r', 4), 0)
+ expect(ledger.shouldDeleteSecret(ref('r', 4))).toBe(false)
+ expect(ledger.shouldDeletePolicy(ref('r', 4), 1)).toBe(false)
+
+ ledger.invalidateSecret('r')
+ expect(ledger.shouldDeleteSecret(ref('r', 4))).toBe(true)
+ expect(ledger.shouldDeletePolicy(ref('r', 4), 1)).toBe(false)
+ })
+
+ it('does not record a Secret delete when an invalidation landed after the epoch was read', () => {
+ const ledger = new OAuthBrokerDeleteLedger()
+ const epochBeforeDelete = ledger.secretEpoch('r')
+ ledger.invalidateSecret('r')
+ expect(ledger.recordSecretDelete(ref('r', 4), epochBeforeDelete)).toBe(false)
+ expect(ledger.shouldDeleteSecret(ref('r', 4))).toBe(true)
+
+ // Liveness witness: with a fresh epoch the same record is accepted.
+ expect(ledger.recordSecretDelete(ref('r', 4), ledger.secretEpoch('r'))).toBe(true)
+ expect(ledger.shouldDeleteSecret(ref('r', 4))).toBe(false)
+ })
+ it('an invalidation for one recipe does not reject another recipe record', () => {
+ const ledger = new OAuthBrokerDeleteLedger()
+ const epochBeforeDelete = ledger.secretEpoch('r2')
ledger.invalidateSecret('r')
- expect(ledger.shouldDeleteSecret('r', 4)).toBe(true)
- expect(ledger.shouldDeletePolicy('r', 4, 1)).toBe(false)
+ expect(ledger.recordSecretDelete(ref('r2', 4), epochBeforeDelete)).toBe(true)
+ expect(ledger.shouldDeleteSecret(ref('r2', 4))).toBe(false)
})
it('invalidate drops both sides for the named recipe and no other', () => {
const ledger = new OAuthBrokerDeleteLedger()
- ledger.recordSecretDelete('r', 4)
- ledger.recordPolicyDelete('r', 4, 0)
- ledger.recordSecretDelete('r2', 4)
- ledger.recordPolicyDelete('r2', 4, 0)
+ recordSecret(ledger, ref('r', 4))
+ ledger.recordPolicyDelete(ref('r', 4), 0)
+ recordSecret(ledger, ref('r2', 4))
+ ledger.recordPolicyDelete(ref('r2', 4), 0)
+
+ ledger.invalidate('r')
+ expect(ledger.shouldDeleteSecret(ref('r', 4))).toBe(true)
+ expect(ledger.shouldDeletePolicy(ref('r', 4), 1)).toBe(true)
+ expect(ledger.shouldDeleteSecret(ref('r2', 4))).toBe(false)
+ expect(ledger.shouldDeletePolicy(ref('r2', 4), 1)).toBe(false)
+ })
+ it('a DELETE in flight across a recipe deletion that followed an ADDED is not recorded', () => {
+ const ledger = new OAuthBrokerDeleteLedger()
+ ledger.invalidateSecret('r')
+ const epochBeforeDelete = ledger.secretEpoch('r')
ledger.invalidate('r')
- expect(ledger.shouldDeleteSecret('r', 4)).toBe(true)
- expect(ledger.shouldDeletePolicy('r', 4, 1)).toBe(true)
- expect(ledger.shouldDeleteSecret('r2', 4)).toBe(false)
- expect(ledger.shouldDeletePolicy('r2', 4, 1)).toBe(false)
+ expect(ledger.recordSecretDelete(ref('r', 4), epochBeforeDelete)).toBe(false)
+ expect(ledger.shouldDeleteSecret(ref('r', 4))).toBe(true)
})
})
diff --git a/workflow-recipes/src/reconciler/oauthBrokerDeleteLedger.ts b/workflow-recipes/src/reconciler/oauthBrokerDeleteLedger.ts
index 0f1667c0b..3b9068f98 100644
--- a/workflow-recipes/src/reconciler/oauthBrokerDeleteLedger.ts
+++ b/workflow-recipes/src/reconciler/oauthBrokerDeleteLedger.ts
@@ -4,15 +4,33 @@
* 1h TTL so a later pass can reap a leftover without a watch. Indexing only
* by recipe name is the E.6 vacuity mutation: a generation bump must miss.
*
- * One entry per recipe (the last generation wins): generations only increase,
- * so older entries can never match again and keeping them would grow the
- * ledger by one key per spec edit for the life of the process.
+ * One entry per recipe, holding the highest generation recorded for the
+ * recipe's current uid. A pass whose generation is at or below that value is
+ * skipped: a queued pass still carrying an older object must not delete what
+ * a newer generation provisioned. A different uid is a recipe recreated under
+ * the same name, whose generations restart at 1, so it always deletes and
+ * replaces the entry even when the recipe DELETE event was missed.
+ *
+ * `invalidateSecret` bumps a per-recipe epoch. A Secret delete is recorded
+ * only when the epoch is unchanged since before the DELETE was sent, so an
+ * ADDED handled while the DELETE is in flight is not overwritten by it.
*/
export const OAUTH_BROKER_NP_TTL_MS = 60 * 60 * 1000
-interface PolicyEntry {
+/** The recipe identity the ledger keys on; a recipe's `metadata` satisfies it. */
+export interface OAuthBrokerLedgerRecipe {
+ name: string
+ uid?: string
+ generation?: number
+}
+
+interface SecretEntry {
+ uid: string | undefined
generation: number
+}
+
+interface PolicyEntry extends SecretEntry {
expiresAt: number
}
@@ -20,34 +38,56 @@ function normalizeGeneration(generation: number | undefined): number {
return generation ?? 0
}
+function coversPass(entry: SecretEntry | undefined, recipe: OAuthBrokerLedgerRecipe): boolean {
+ return (
+ entry !== undefined &&
+ entry.uid === recipe.uid &&
+ normalizeGeneration(recipe.generation) <= entry.generation
+ )
+}
+
+function nextEntry(entry: SecretEntry | undefined, recipe: OAuthBrokerLedgerRecipe): SecretEntry {
+ const generation = normalizeGeneration(recipe.generation)
+ if (entry !== undefined && entry.uid === recipe.uid) {
+ return { uid: recipe.uid, generation: Math.max(entry.generation, generation) }
+ }
+ return { uid: recipe.uid, generation }
+}
+
export class OAuthBrokerDeleteLedger {
- private readonly secrets = new Map<string, number>()
+ private readonly secrets = new Map<string, SecretEntry>()
private readonly policies = new Map<string, PolicyEntry>()
+ private readonly secretEpochs = new Map<string, number>()
+ private epochClock = 0
+
+ shouldDeleteSecret(recipe: OAuthBrokerLedgerRecipe): boolean {
+ return !coversPass(this.secrets.get(recipe.name), recipe)
+ }
- shouldDeleteSecret(recipeName: string, generation: number | undefined): boolean {
- return this.secrets.get(recipeName) !== normalizeGeneration(generation)
+ /** Read before sending the DELETE and hand back to `recordSecretDelete`. */
+ secretEpoch(recipeName: string): number {
+ return this.secretEpochs.get(recipeName) ?? 0
}
- recordSecretDelete(recipeName: string, generation: number | undefined): void {
- this.secrets.set(recipeName, normalizeGeneration(generation))
+ /**
+ * Returns false, recording nothing, when an invalidation landed after
+ * `epochBeforeDelete` was read.
+ */
+ recordSecretDelete(recipe: OAuthBrokerLedgerRecipe, epochBeforeDelete: number): boolean {
+ if (this.secretEpoch(recipe.name) !== epochBeforeDelete) return false
+ this.secrets.set(recipe.name, nextEntry(this.secrets.get(recipe.name), recipe))
+ return true
}
- shouldDeletePolicy(
- recipeName: string,
- generation: number | undefined,
- nowMs = Date.now()
- ): boolean {
- const entry = this.policies.get(recipeName)
- return (
- entry === undefined ||
- entry.generation !== normalizeGeneration(generation) ||
- nowMs >= entry.expiresAt
- )
+ shouldDeletePolicy(recipe: OAuthBrokerLedgerRecipe, nowMs = Date.now()): boolean {
+ const entry = this.policies.get(recipe.name)
+ if (entry === undefined || !coversPass(entry, recipe)) return true
+ return nowMs >= entry.expiresAt
}
- recordPolicyDelete(recipeName: string, generation: number | undefined, nowMs = Date.now()): void {
- this.policies.set(recipeName, {
- generation: normalizeGeneration(generation),
+ recordPolicyDelete(recipe: OAuthBrokerLedgerRecipe, nowMs = Date.now()): void {
+ this.policies.set(recipe.name, {
+ ...nextEntry(this.policies.get(recipe.name), recipe),
expiresAt: nowMs + OAUTH_BROKER_NP_TTL_MS,
})
}
@@ -60,11 +100,17 @@ export class OAuthBrokerDeleteLedger {
*/
invalidateSecret(recipeName: string): void {
this.secrets.delete(recipeName)
+ this.epochClock += 1
+ this.secretEpochs.set(recipeName, this.epochClock)
}
- /** Recipe deletion drops both sides. */
+ /**
+ * Recipe deletion drops both sides. The epoch entry goes too; the clock is
+ * global, so a DELETE in flight that read a non-zero epoch still mismatches.
+ */
invalidate(recipeName: string): void {
this.secrets.delete(recipeName)
this.policies.delete(recipeName)
+ this.secretEpochs.delete(recipeName)
}
}
diff --git a/workflow-recipes/src/reconciler/reconciler.test.ts b/workflow-recipes/src/reconciler/reconciler.test.ts
index 21fe2ca2f..e1d2a5549 100644
--- a/workflow-recipes/src/reconciler/reconciler.test.ts
+++ b/workflow-recipes/src/reconciler/reconciler.test.ts
@@ -15603,6 +15603,131 @@ describe('WorkflowRecipeReconciler', () => {
infoSpy.mockRestore()
vi.useRealTimers()
})
+
+ it('R1-L2: a late pass carrying an older generation does not re-delete the Secret', async () => {
+ const infoSpy = captureLogger('info')
+ await reconciler.ensureOAuthBrokerTokenSecret(reapRecipe(4))
+ await reconciler.ensureOAuthBrokerTokenSecret(reapRecipe(5))
+ // Liveness witness: both newer-generation deletes really ran.
+ expect(secretDeletes()).toBe(2)
+
+ await reconciler.ensureOAuthBrokerTokenSecret(reapRecipe(4))
+ expect(secretDeletes()).toBe(2)
+ expect(infoSpy).toHaveBeenCalledWith(
+ expect.stringContaining('Skipping oauth-broker-token delete'),
+ expect.objectContaining({ recipe: 'test-recipe', generation: 4 })
+ )
+ infoSpy.mockRestore()
+ })
+
+ it('R1-L2: a late pass carrying an older generation does not re-delete the NetworkPolicy', async () => {
+ const infoSpy = captureLogger('info')
+ await reapPolicy(reapRecipe(4))
+ await reapPolicy(reapRecipe(5))
+ // Liveness witness: both newer-generation deletes really ran.
+ expect(policyDeletes()).toBe(2)
+
+ await reapPolicy(reapRecipe(4))
+ expect(policyDeletes()).toBe(2)
+ expect(infoSpy).toHaveBeenCalledWith(
+ expect.stringContaining('Skipping oauth-broker-egress delete'),
+ expect.objectContaining({ recipe: 'test-recipe', generation: 4 })
+ )
+ infoSpy.mockRestore()
+ })
+
+ it('R1-L2: a recipe recreated under the same name is reaped even at a lower generation', async () => {
+ await reconciler.ensureOAuthBrokerTokenSecret(reapRecipe(5))
+ await reapPolicy(reapRecipe(5))
+ expect(secretDeletes()).toBe(1)
+ expect(policyDeletes()).toBe(1)
+
+ // The recipe DELETE event was missed, so nothing invalidated the ledger.
+ const recreated = makeRecipe({
+ metadata: {
+ name: 'test-recipe',
+ namespace: 'sandbox-recipes',
+ uid: 'uid-recreated',
+ generation: 1,
+ },
+ })
+ await reconciler.ensureOAuthBrokerTokenSecret(recreated)
+ await reapPolicy(recreated)
+ expect(secretDeletes()).toBe(2)
+ expect(policyDeletes()).toBe(2)
+ })
+
+ it('R1-L1: an ADDED invalidation that lands while the DELETE is in flight is not overwritten', async () => {
+ const recipe = reapRecipe(4)
+ mockCoreApi.deleteNamespacedSecret.mockImplementationOnce(async () => {
+ // Another writer recreated the Secret and its ADDED was handled before
+ // this DELETE returned.
+ reconciler.invalidateOAuthBrokerSecretLedger('test-recipe')
+ throw { code: 404 }
+ })
+ await reconciler.ensureOAuthBrokerTokenSecret(recipe)
+ expect(secretDeletes()).toBe(1)
+
+ await reconciler.ensureOAuthBrokerTokenSecret(recipe)
+ expect(secretDeletes()).toBe(2)
+
+ // Liveness witness for the ledger itself: with no racing ADDED the
+ // second delete is recorded and the next same-generation pass skips.
+ await reconciler.ensureOAuthBrokerTokenSecret(recipe)
+ expect(secretDeletes()).toBe(2)
+ })
+
+ describe.each([
+ ['active', 'running'],
+ ['terminal', 'completed'],
+ ] as const)('R1-L13: the %s workflow short-circuit', (_label, execPhase) => {
+ it('reaps a token Secret the recipe must not have, without the inner reconcile', async () => {
+ const workflowReconcile = vi.fn()
+ ;(
+ reconciler as unknown as {
+ workflowReconciler: {
+ reconcile: typeof workflowReconcile
+ validateWorkflowSpec: () => undefined
+ ensureMcpHostRuntimeCredentials?: () => Promise<void>
+ }
+ }
+ ).workflowReconciler = {
+ reconcile: workflowReconcile,
+ validateWorkflowSpec: () => undefined,
+ ensureMcpHostRuntimeCredentials: vi.fn().mockResolvedValue(undefined),
+ }
+ const recipe = makeRecipe({
+ spec: {
+ agent: { provider: 'zai', model: 'glm-4.7' },
+ workloads: [],
+ steps: [{ id: 'run-qa', instruction: 'Validate the QA API workload.' }],
+ },
+ // Triggered run: awaitsTriggeredRun=false, so the pass stops at the
+ // active (running) or terminal (completed) short-circuit.
+ metadata: {
+ name: 'test-recipe',
+ namespace: 'sandbox-recipes',
+ uid: 'uid-123',
+ generation: 4,
+ labels: { 'clerum.io/workflow-run-id': 'run-123' },
+ },
+ status: { phase: 'active', workflowExecution: { phase: execPhase } },
+ })
+
+ // The watch ADDED re-arms the ledger and enqueues this reconcile.
+ reconciler.invalidateOAuthBrokerSecretLedger('test-recipe')
+ const result = await reconciler.reconcile(recipe)
+
+ // Witness that the short-circuit was taken, not the first-deploy path.
+ expect(workflowReconcile).not.toHaveBeenCalled()
+ expect(result.phase).toBe('active')
+ expect(secretDeletes()).toBe(1)
+
+ // The ledger still bounds it: a second pass of the same generation skips.
+ await reconciler.reconcile(recipe)
+ expect(secretDeletes()).toBe(1)
+ })
+ })
})
describe('G2 skipStatusPatch must not leave GFS deleted', () => {
@@ -15744,6 +15869,13 @@ describe('WorkflowRecipeReconciler', () => {
labels: { 'clerum.io/recipe': RECIPE, 'clerum.io/managed-by': 'wrc' },
},
},
+ {
+ metadata: {
+ name: `${RECIPE}-snippet-runner-egress`,
+ namespace: 'sandbox-recipes',
+ labels: { 'clerum.io/recipe': RECIPE, 'clerum.io/managed-by': 'wrc' },
+ },
+ },
{
metadata: {
name: `${RECIPE}-mcp-host-to-grok-proxy`,
@@ -15763,16 +15895,20 @@ describe('WorkflowRecipeReconciler', () => {
'run-g2'
)
- // Liveness witness: the retry ran the prune LIST that listed GFS, so the
- // absence of a GFS DELETE below is the catalog gate holding, not a retry
- // that never reached the prune.
+ // Liveness witness: the retry ran the prune and deleted an undesired
+ // catalog sibling, so the absence of a GFS DELETE below is the catalog
+ // gate holding, not a retry that never reached the prune.
expect(mockNetworkingApi.listNamespacedNetworkPolicy).toHaveBeenCalled()
expect(mockNetworkingApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledWith(
- expect.objectContaining({ name: `${RECIPE}-mcp-host-to-grok-proxy` })
+ expect.objectContaining({ name: `${RECIPE}-snippet-runner-egress` })
)
expect(mockNetworkingApi.deleteNamespacedNetworkPolicy).not.toHaveBeenCalledWith(
expect.objectContaining({ name: GFS })
)
+ // The retry runs mid-run, so it never revokes a Codex/Grok proxy.
+ expect(mockNetworkingApi.deleteNamespacedNetworkPolicy).not.toHaveBeenCalledWith(
+ expect.objectContaining({ name: `${RECIPE}-mcp-host-to-grok-proxy` })
+ )
})
})
diff --git a/workflow-recipes/src/reconciler/resourceBuilder.test.ts b/workflow-recipes/src/reconciler/resourceBuilder.test.ts
index 7b48c62a6..719a8bfd3 100644
--- a/workflow-recipes/src/reconciler/resourceBuilder.test.ts
+++ b/workflow-recipes/src/reconciler/resourceBuilder.test.ts
@@ -2082,30 +2082,24 @@ describe('recipe OAuth broker token', () => {
it('resolves watch ADDED only for the canonical oauth-broker-token Secret name', () => {
const recipe = 'test-recipe'
- const canonical = oauthBrokerTokenSecretName(recipe)
+ // The Secret WRC itself writes: the resolver must bind it, so a drift in
+ // the builder's name or labels fails here.
+ const built = buildOAuthBrokerTokenSecret(recipe, 'jwt', 'sandbox-recipes')
+ const canonical = built.metadata?.name ?? ''
+ const labels = built.metadata?.labels ?? {}
+ expect(canonical).toBe(oauthBrokerTokenSecretName(recipe))
expect(parseOAuthBrokerTokenSecretRecipe(canonical)).toBe(recipe)
expect(parseOAuthBrokerTokenSecretRecipe('wf--oauth-broker-token')).toBeUndefined()
+ expect(resolveOAuthBrokerTokenWatchRecipe(canonical, labels)).toBe(recipe)
+ const withoutRecipeLabel = Object.fromEntries(
+ Object.entries(labels).filter(([key]) => key !== 'clerum.io/recipe')
+ )
+ expect(resolveOAuthBrokerTokenWatchRecipe(canonical, withoutRecipeLabel)).toBe(recipe)
+ expect(resolveOAuthBrokerTokenWatchRecipe('forged-name', labels)).toBeUndefined()
expect(
resolveOAuthBrokerTokenWatchRecipe(canonical, {
- 'clerum.io/component': 'oauth-broker-token',
- 'clerum.io/recipe': recipe,
- })
- ).toBe(recipe)
- expect(
- resolveOAuthBrokerTokenWatchRecipe(canonical, {
- 'clerum.io/component': 'oauth-broker-token',
- })
- ).toBe(recipe)
- expect(
- resolveOAuthBrokerTokenWatchRecipe('forged-name', {
- 'clerum.io/component': 'oauth-broker-token',
- 'clerum.io/recipe': recipe,
- })
- ).toBeUndefined()
- expect(
- resolveOAuthBrokerTokenWatchRecipe(canonical, {
+ ...labels,
'clerum.io/component': 'other',
- 'clerum.io/recipe': recipe,
})
).toBeUndefined()
})
diff --git a/workflow-recipes/src/reconciler/secretWatcher.test.ts b/workflow-recipes/src/reconciler/secretWatcher.test.ts
index 38296147d..2188b6fa6 100644
--- a/workflow-recipes/src/reconciler/secretWatcher.test.ts
+++ b/workflow-recipes/src/reconciler/secretWatcher.test.ts
@@ -1,4 +1,5 @@
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
+import { buildOAuthBrokerTokenSecret } from './resourceBuilder'
import { OWNER_RECIPE_LABEL_KEY, SHARED_LABEL_KEY } from './secretOwnership'
import { SecretReverseIndex } from './secretReverseIndex'
import { SecretWatcher } from './secretWatcher'
@@ -230,16 +231,9 @@ describe('SecretWatcher', () => {
invalidated.push(recipeName)
}
)
- const brokerSecret = {
- metadata: {
- name: 'wf-test-recipe-oauth-broker-token',
- labels: {
- 'clerum.io/component': 'oauth-broker-token',
- 'clerum.io/recipe': 'test-recipe',
- },
- },
- data: { 'broker-token': 'eA==' },
- }
+ // The Secret WRC itself writes, so a drift in the builder's labels or
+ // name breaks this test instead of passing against a hand-written copy.
+ const brokerSecret = buildOAuthBrokerTokenSecret('test-recipe', 'jwt', 'sandbox-recipes')
watcher.handleEvent('ADDED', brokerSecret)
watcher.handleEvent('ADDED', brokerSecret)
@@ -265,33 +259,24 @@ describe('SecretWatcher', () => {
invalidated.push(recipeName)
}
)
- const labels = {
- 'clerum.io/component': 'oauth-broker-token',
- 'clerum.io/recipe': 'test-recipe',
- }
+ const canonical = buildOAuthBrokerTokenSecret('test-recipe', 'jwt', 'sandbox-recipes')
watcher.handleEvent('ADDED', {
- metadata: { name: 'forged-oauth-broker-token', labels },
- data: { 'broker-token': 'eA==' },
+ ...canonical,
+ metadata: { ...canonical.metadata, name: 'forged-oauth-broker-token' },
})
watcher.handleEvent('ADDED', {
+ ...canonical,
metadata: {
- name: 'wf-test-recipe-oauth-broker-token',
- labels: { 'clerum.io/component': 'other', 'clerum.io/recipe': 'test-recipe' },
+ ...canonical.metadata,
+ labels: { ...canonical.metadata?.labels, 'clerum.io/component': 'other' },
},
- data: { 'broker-token': 'eA==' },
- })
- watcher.handleEvent('MODIFIED', {
- metadata: { name: 'wf-test-recipe-oauth-broker-token', labels },
- data: { 'broker-token': 'eA==' },
})
+ watcher.handleEvent('MODIFIED', canonical)
expect(invalidated).toEqual([])
// Liveness witness: the same watcher does invalidate for the canonical
// ADDED, so the empty list above is not the callback being unwired.
- watcher.handleEvent('ADDED', {
- metadata: { name: 'wf-test-recipe-oauth-broker-token', labels },
- data: { 'broker-token': 'eA==' },
- })
+ watcher.handleEvent('ADDED', canonical)
expect(invalidated).toEqual(['test-recipe'])
vi.advanceTimersByTime(DEBOUNCE_MS)
expect(enqueued).toEqual([])
diff --git a/workflow-recipes/src/reconciler/workflowRecipeReconciler.ts b/workflow-recipes/src/reconciler/workflowRecipeReconciler.ts
index 894bb5165..f484840f6 100644
--- a/workflow-recipes/src/reconciler/workflowRecipeReconciler.ts
+++ b/workflow-recipes/src/reconciler/workflowRecipeReconciler.ts
@@ -1477,6 +1477,8 @@ export class WorkflowRecipeReconciler {
* pending a retry would stay unapplied until the run ends. The retry
* re-prunes as well, with a verdict computed for this pass: a prune that
* failed is what set the marker, so an apply-only retry could never clear it.
+ * The Codex and Grok proxy policies are never revoked by this retry, since
+ * a run may be using them; the first reconcile() after the run prunes them.
*
* Returns the requeue delay the short-circuit must add: the progress base
* while a policy is still pending, the transient base when the apply threw
@@ -1964,6 +1966,16 @@ export class WorkflowRecipeReconciler {
}
}
+ // B3(a): the awaiting-trigger, terminal, in-progress and active
+ // short-circuits below return before the first-deploy path that calls
+ // ensureOAuthBrokerTokenSecret, so the token-ADDED re-reconcile would
+ // never reach the reap. Only the delete branch runs here (issuance stays
+ // on the first-deploy path and the rotation loop), and the delete ledger
+ // bounds it to one DELETE per generation or ADDED.
+ if (!rb.recipeHasBackgroundAccessClient(recipe)) {
+ await this.ensureOAuthBrokerTokenSecret(recipe)
+ }
+
if (
currentPhase === 'active' &&
awaitsTriggeredRun &&
@@ -7518,13 +7530,14 @@ export class WorkflowRecipeReconciler {
if (!rb.recipeHasBackgroundAccessClient(recipe)) {
const generation = recipe.metadata.generation
- if (!this.oauthBrokerDeleteLedger.shouldDeleteSecret(recipeName, generation)) {
+ if (!this.oauthBrokerDeleteLedger.shouldDeleteSecret(recipe.metadata)) {
createLogger('wrc', recipeName).info(
'Skipping oauth-broker-token delete; generation already seen',
{ recipe: recipeName, generation }
)
return
}
+ const epochBeforeDelete = this.oauthBrokerDeleteLedger.secretEpoch(recipeName)
const outcome = await observeNamespacedDelete(() =>
this.coreApi.deleteNamespacedSecret({ name: secretName, namespace: ns })
)
@@ -7543,8 +7556,14 @@ export class WorkflowRecipeReconciler {
...deleteOutcomeFields(outcome),
})
}
- if (shouldRecordDelete(outcome)) {
- this.oauthBrokerDeleteLedger.recordSecretDelete(recipeName, generation)
+ if (
+ shouldRecordDelete(outcome) &&
+ !this.oauthBrokerDeleteLedger.recordSecretDelete(recipe.metadata, epochBeforeDelete)
+ ) {
+ log.info('oauth-broker-token Secret re-added during delete; not recording', {
+ recipe: recipeName,
+ generation,
+ })
}
return
}
@@ -7610,7 +7629,7 @@ export class WorkflowRecipeReconciler {
if (!policy) {
const recipeName = recipe.metadata.name
const generation = recipe.metadata.generation
- if (!this.oauthBrokerDeleteLedger.shouldDeletePolicy(recipeName, generation)) {
+ if (!this.oauthBrokerDeleteLedger.shouldDeletePolicy(recipe.metadata)) {
createLogger('wrc', recipeName).info(
'Skipping oauth-broker-egress delete; generation already seen',
{ recipe: recipeName, generation }
@@ -7636,7 +7655,7 @@ export class WorkflowRecipeReconciler {
})
}
if (shouldRecordDelete(outcome)) {
- this.oauthBrokerDeleteLedger.recordPolicyDelete(recipeName, generation)
+ this.oauthBrokerDeleteLedger.recordPolicyDelete(recipe.metadata)
}
return
}
diff --git a/workflow-recipes/src/workflow/workflowReconciler.ts b/workflow-recipes/src/workflow/workflowReconciler.ts
index 671e407e0..3a92ce2d9 100644
--- a/workflow-recipes/src/workflow/workflowReconciler.ts
+++ b/workflow-recipes/src/workflow/workflowReconciler.ts
@@ -231,6 +231,11 @@ type RunLaneNetworkPolicyApplyResult =
reason: 'terminating' | 'contended'
}
| { policy: string; action: 'conflict'; reason: NetworkPolicyConflictReason }
+/** Why a Codex/Grok proxy policy survives a prune that does not desire it. */
+interface RunLaneProxyKeepReasons {
+ codex?: 'uncertain' | 'run-in-progress'
+ grok?: 'uncertain' | 'run-in-progress'
+}
// Read-then-write rounds per NetworkPolicy and pass. Three rounds is at most
// three writes, the same as the create plus two replaces it replaced.
const NETWORK_POLICY_APPLY_ROUNDS = 3
@@ -2990,11 +2995,14 @@ export class WorkflowReconciler {
* active short-circuits return before reconcile(), so a policy a previous
* pass left pending (terminating, contended, or a prune that failed) would
* otherwise stay as it is until the run ends. Runs the same build, apply and
- * prune unit reconcile() runs, with a verdict computed now: an uncertain
- * Codex or Grok verdict keeps its proxy policy, and the catalog gate keeps
- * everything the run lane does not own. A prune that failed sets
- * retryPending, so without the prune here the marker would requeue forever
- * and never reach the leftover.
+ * prune unit reconcile() runs, with a verdict computed now, with one
+ * difference: the Codex and Grok proxy policies are never revoked here.
+ * The verdict can change while a step is using the proxy, so a revoke
+ * mid-run would cut its egress; a proxy the verdict still requires is
+ * created (an allow), and one it no longer wants is pruned by the first
+ * reconcile() after the run. The catalog gate keeps everything the run lane
+ * does not own. A prune that failed sets retryPending, so without the prune
+ * here the marker would requeue forever and never reach the leftover.
*/
async retryRunLaneNetworkPolicies(
recipeName: string,
@@ -3019,7 +3027,7 @@ export class WorkflowReconciler {
runtimeScopeRecipeName,
codexView
)
- return this.applyWorkflowNetworkPolicies(
+ const { policies, catalog } = await this.buildWorkflowNetworkPoliciesForSpec(
recipeName,
recipeUid,
spec,
@@ -3029,6 +3037,10 @@ export class WorkflowReconciler {
false,
codexVerdict.grokProjection
)
+ return this.applyAndPruneRunLaneNetworkPolicies(recipeName, recipeUid, policies, catalog, {
+ codex: 'run-in-progress',
+ grok: 'run-in-progress',
+ })
}
async ensureCoordinatorRuntimeCredentials(
@@ -3435,16 +3447,26 @@ export class WorkflowReconciler {
eagerSdkMcpHost,
grokProjection
)
+ return this.applyAndPruneRunLaneNetworkPolicies(recipeName, recipeUid, policies, catalog, {
+ codex: codexProjection.eligibility === 'uncertain' ? 'uncertain' : undefined,
+ grok: grokProjection?.eligibility === 'uncertain' ? 'uncertain' : undefined,
+ })
+ }
+
+ private async applyAndPruneRunLaneNetworkPolicies(
+ recipeName: string,
+ recipeUid: string,
+ policies: k8s.V1NetworkPolicy[],
+ catalog: Set<string>,
+ keepProxies: RunLaneProxyKeepReasons
+ ): Promise<WorkflowNetworkPolicyApplySummary> {
const summary = await this.applyNetworkPolicyList(policies)
const policyNames = new Set(policies.map(policy => policy.metadata?.name))
const prune = await this.pruneUndesiredRunLaneNetworkPolicies(
recipeName,
policyNames,
catalog,
- {
- skipCodexProxy: codexProjection.eligibility === 'uncertain',
- skipGrokProxy: grokProjection?.eligibility === 'uncertain',
- }
+ keepProxies
)
if (prune.retryPending) summary.retryPending = true
const legacy = await this.pruneLegacyMcpServersInternetEgressPolicy(recipeName, recipeUid)
@@ -3457,18 +3479,20 @@ export class WorkflowReconciler {
* Two-term selector only (`managed-by=wrc`): a single-term recipe selector
* would also return the recipes-lane policies in the same namespace.
* Universe is the factory catalog (GFS never in it), not LIST \ desired.
- * Uncertain Codex/Grok eligibility keeps those proxy names even when they
- * are absent from this pass's desired set.
+ * A Codex/Grok proxy with a keep reason (an uncertain verdict, or a run in
+ * progress on the retry path) survives even when it is absent from this
+ * pass's desired set. A listed policy with an ownerReference is skipped,
+ * matching the apply's ownership veto.
*/
private async pruneUndesiredRunLaneNetworkPolicies(
recipeName: string,
desiredNames: Set<string | undefined>,
catalog: Set<string>,
- uncertain: { skipCodexProxy: boolean; skipGrokProxy: boolean }
+ keepProxies: RunLaneProxyKeepReasons
): Promise<{ retryPending: boolean }> {
const namespace = this.deps.config.sandboxNamespace
const labelSelector = `clerum.io/recipe=${recipeName},clerum.io/managed-by=wrc`
- let list: { items?: Array<{ metadata?: { name?: string } }> }
+ let list: { items?: k8s.V1NetworkPolicy[] }
try {
list = await this.deps.networkingApi.listNamespacedNetworkPolicy({
namespace,
@@ -3487,27 +3511,37 @@ export class WorkflowReconciler {
for (const policy of list.items ?? []) {
const name = policy.metadata?.name
if (!name || desiredNames.has(name)) continue
- if (name === codexProxyPolicyName && uncertain.skipCodexProxy) {
+ const keepReason =
+ name === codexProxyPolicyName
+ ? keepProxies.codex
+ : name === grokProxyPolicyName
+ ? keepProxies.grok
+ : undefined
+ if (keepReason) {
this.log.debug('Skipping run-lane NP prune', {
recipe: recipeName,
policy: name,
- reason: 'uncertain',
+ reason: keepReason,
})
continue
}
- if (name === grokProxyPolicyName && uncertain.skipGrokProxy) {
+ if (!catalog.has(name)) {
this.log.debug('Skipping run-lane NP prune', {
recipe: recipeName,
policy: name,
- reason: 'uncertain',
+ reason: 'not-in-catalog',
})
continue
}
- if (!catalog.has(name)) {
- this.log.debug('Skipping run-lane NP prune', {
+ // Same ownership veto the run-lane apply uses: a policy under our name
+ // that carries an ownerReference belongs to someone else, and the apply
+ // already refuses to replace it.
+ const ownership = classifyOwnerlessNetworkPolicyOwnership(policy)
+ if (ownership.kind !== 'owned') {
+ this.log.warn('Skipping run-lane NP prune', {
recipe: recipeName,
policy: name,
- reason: 'not-in-catalog',
+ reason: ownership.kind === 'conflict' ? ownership.reason : ownership.kind,
})
continue
}
diff --git a/workflow-recipes/tests/unit/workflow/workflowReconciler.test.ts b/workflow-recipes/tests/unit/workflow/workflowReconciler.test.ts
index 5a7865788..6b19ca87a 100644
--- a/workflow-recipes/tests/unit/workflow/workflowReconciler.test.ts
+++ b/workflow-recipes/tests/unit/workflow/workflowReconciler.test.ts
@@ -3507,6 +3507,46 @@ describe('WorkflowReconciler — reconcile loop', () => {
expect(apiserver.live.has(apiserver.key(SANDBOX, COORD_TO_MCP_HOST))).toBe(false)
})
+ it('R1-M1: does not prune a catalog policy that carries an ownerReference (apply refuses it too)', async () => {
+ const apiserver = makeApiserverNetworkingApi()
+ seedLivePolicy(apiserver, COORD_TO_MCP_HOST, WRC_LABELS)
+ const seeded = apiserver.live.get(apiserver.key(SANDBOX, COORD_TO_MCP_HOST))!
+ seeded.metadata = {
+ ...seeded.metadata,
+ ownerReferences: [
+ {
+ apiVersion: 'apps/v1',
+ kind: 'Deployment',
+ name: 'someone-else',
+ uid: 'uid-deployment',
+ controller: true,
+ },
+ ],
+ }
+ seedPrunableSibling(apiserver)
+ const warnLog = captureLogger('warn')
+ try {
+ const reconciler = new WorkflowReconciler(
+ makeDeps({ networkingApi: apiserver.api as never })
+ )
+
+ await applyPolicies(reconciler, { awaitsTriggeredRun: true })
+
+ expectListWitness(apiserver.api)
+ // Liveness witness: the WRC-owned sibling in the same LIST was pruned.
+ expectPrunableSiblingDeleted(apiserver)
+ expect(sandboxDeletes(apiserver.api).map(d => d.name)).not.toContain(COORD_TO_MCP_HOST)
+ expect(apiserver.live.has(apiserver.key(SANDBOX, COORD_TO_MCP_HOST))).toBe(true)
+ expect(warnLog).toHaveBeenCalledWith('Skipping run-lane NP prune', {
+ recipe: RECIPE,
+ policy: COORD_TO_MCP_HOST,
+ reason: 'owner-reference-mismatch',
+ })
+ } finally {
+ warnLog.mockRestore()
+ }
+ })
+
it('eager SDK path still prunes a leftover Grok proxy', async () => {
const apiserver = makeApiserverNetworkingApi()
seedLivePolicy(apiserver, GROK_PROXY, WRC_LABELS)
@@ -5261,7 +5301,10 @@ describe('WorkflowReconciler — reconcile loop', () => {
// runs while the run is in progress or active. It has to prune again, or
// the marker requeues forever without ever reaching the leftover.
describe('when a prune left a leftover behind', () => {
- const LEFTOVER = 'test-wf-mcp-host-to-grok-proxy'
+ // A catalog member the snippet lane does not want and that no eligibility
+ // verdict governs: the retry must prune it so the marker can clear.
+ const LEFTOVER = 'test-wf-coord-to-mcp-host'
+ const GROK_PROXY = 'test-wf-mcp-host-to-grok-proxy'
const TWO_TERM_SELECTOR = 'clerum.io/recipe=test-wf,clerum.io/managed-by=wrc'
type Converged = Awaited<ReturnType<typeof convergedRunLane>>['apiserver']
@@ -5359,9 +5402,13 @@ describe('WorkflowReconciler — reconcile loop', () => {
// The retry computes its own verdict. These tests pin it: a proxy the
// fresh verdict still wants (or cannot decide about) survives, and one
- // it no longer wants goes. The sibling delete is the liveness witness
- // in every case: the prune loop demonstrably iterated the same LIST.
+ // it no longer wants goes. Every case also asserts the delete of a
+ // snippet-lane sibling the agent spec no longer wants
+ // (`expectSnippetSiblingPruned`): that is the liveness witness proving
+ // the prune loop iterated the same LIST, so "no proxy was deleted"
+ // cannot pass because the loop never ran.
describe('with a fresh Codex/Grok verdict', () => {
+ const SNIPPET_SIBLING = 'test-wf-snippet-runner-egress'
const CODEX_LIVE = 'test-wf-mcp-host-to-codex-proxy'
const projection = (overrides: Record<string, unknown>) => ({
targets: [],
@@ -5426,7 +5473,16 @@ describe('WorkflowReconciler — reconcile loop', () => {
.map(([arg]) => arg.name)
.filter(name => name.endsWith('-proxy'))
- it('keeps a proxy the fresh verdict still requires and prunes the undesired twin', async () => {
+ function expectSnippetSiblingPruned(apiserver: Converged): void {
+ expect(
+ apiserver.api.deleteNamespacedNetworkPolicy.mock.calls.map(([arg]) => arg.name)
+ ).toContain(SNIPPET_SIBLING)
+ expect(apiserver.live.has(apiserver.key('sandbox-recipes', SNIPPET_SIBLING))).toBe(
+ false
+ )
+ }
+
+ it('keeps a proxy the fresh verdict still requires, and the undesired twin', async () => {
const { apiserver, summary } = await retryWith(
{
codex: {
@@ -5436,12 +5492,14 @@ describe('WorkflowReconciler — reconcile loop', () => {
},
grok: {},
},
- [CODEX_LIVE, LEFTOVER]
+ [CODEX_LIVE, GROK_PROXY]
)
expect(apiserver.api.listNamespacedNetworkPolicy).toHaveBeenCalledTimes(1)
- expect(deletedNames(apiserver)).toEqual([LEFTOVER])
+ expectSnippetSiblingPruned(apiserver)
+ expect(deletedNames(apiserver)).toEqual([])
expect(apiserver.live.has(apiserver.key('sandbox-recipes', CODEX_LIVE))).toBe(true)
+ expect(apiserver.live.has(apiserver.key('sandbox-recipes', GROK_PROXY))).toBe(true)
expect(summary.retryPending).toBe(false)
})
@@ -5451,25 +5509,55 @@ describe('WorkflowReconciler — reconcile loop', () => {
codex: { eligibility: 'uncertain', reason: 'forbidden' },
grok: { eligibility: 'uncertain', reason: 'forbidden' },
},
- [CODEX_LIVE, LEFTOVER]
+ [CODEX_LIVE, GROK_PROXY]
)
expect(apiserver.api.listNamespacedNetworkPolicy).toHaveBeenCalledTimes(1)
+ expectSnippetSiblingPruned(apiserver)
expect(deletedNames(apiserver)).toEqual([])
expect(apiserver.live.has(apiserver.key('sandbox-recipes', CODEX_LIVE))).toBe(true)
- expect(apiserver.live.has(apiserver.key('sandbox-recipes', LEFTOVER))).toBe(true)
+ expect(apiserver.live.has(apiserver.key('sandbox-recipes', GROK_PROXY))).toBe(true)
expect(summary.retryPending).toBe(false)
})
- it('prunes a proxy once the fresh verdict decides it is no longer wanted', async () => {
+ // Decision: the retry never revokes a Codex/Grok proxy. It runs only
+ // while a run is in progress or active, the path without the marker
+ // never prunes proxies mid-run, and a step of the running run may still
+ // be calling the proxy. The next reconcile() after the run prunes it.
+ it('never revokes a proxy mid-run, even when the fresh verdict no longer wants it', async () => {
const { apiserver, summary } = await retryWith({ codex: {}, grok: {} }, [
CODEX_LIVE,
- LEFTOVER,
+ GROK_PROXY,
])
expect(apiserver.api.listNamespacedNetworkPolicy).toHaveBeenCalledTimes(1)
- expect(deletedNames(apiserver).sort()).toEqual([CODEX_LIVE, LEFTOVER].sort())
- expect(apiserver.live.has(apiserver.key('sandbox-recipes', CODEX_LIVE))).toBe(false)
+ expectSnippetSiblingPruned(apiserver)
+ expect(deletedNames(apiserver)).toEqual([])
+ expect(apiserver.live.has(apiserver.key('sandbox-recipes', CODEX_LIVE))).toBe(true)
+ expect(apiserver.live.has(apiserver.key('sandbox-recipes', GROK_PROXY))).toBe(true)
+ // A kept proxy is not a pending retry, so the marker can clear.
+ expect(summary.retryPending).toBe(false)
+ })
+
+ it('still creates a proxy the fresh verdict requires (an allow, never a revoke)', async () => {
+ const { apiserver, summary } = await retryWith(
+ {
+ codex: {
+ eligibility: 'eligible',
+ requiresCodexProxyEgress: true,
+ reason: 'granted',
+ },
+ grok: {},
+ },
+ []
+ )
+
+ expect(
+ apiserver.api.createNamespacedNetworkPolicy.mock.calls.map(
+ ([arg]) => arg.body.metadata?.name
+ )
+ ).toContain(CODEX_LIVE)
+ expect(apiserver.live.has(apiserver.key('sandbox-recipes', CODEX_LIVE))).toBe(true)
expect(summary.retryPending).toBe(false)
})
})