Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
fix(workflow-recipes): send no oauth-broker token DELETE for a token never seen
Alfredo Lopez1c228b2a2c5b
diff --git a/workflow-recipes/package-lock.json b/workflow-recipes/package-lock.jsonindex 1268ec7db..a7d261f86 100644--- a/workflow-recipes/package-lock.json+++ b/workflow-recipes/package-lock.json@@ -1,12 +1,12 @@{"name": "workload-recipes-controller",- "version": "0.1.428",+ "version": "0.1.429","lockfileVersion": 3,"requires": true,"packages": {"": {"name": "workload-recipes-controller",- "version": "0.1.428",+ "version": "0.1.429","license": "SEE LICENSE IN LICENSE","dependencies": {"@aws-sdk/client-s3": "^3.1009.0",diff --git a/workflow-recipes/package.json b/workflow-recipes/package.jsonindex 85237ed10..bf3b3bbcc 100644--- a/workflow-recipes/package.json+++ b/workflow-recipes/package.json@@ -1,6 +1,6 @@{"name": "workload-recipes-controller",- "version": "0.1.428",+ "version": "0.1.429","description": "Workload Recipes Controller (WRC) - WorkloadRecipe lifecycle management and MCP server interface","main": "dist/main.js","engines": {diff --git a/workflow-recipes/src/reconciler/__tests__/captureLogger.ts b/workflow-recipes/src/reconciler/__tests__/captureLogger.tsindex 4dab2d0e7..cdfa3df8c 100644--- a/workflow-recipes/src/reconciler/__tests__/captureLogger.ts+++ b/workflow-recipes/src/reconciler/__tests__/captureLogger.ts@@ -1,8 +1,10 @@-import { vi } from 'vitest'+import { type Mock, vi } from 'vitest'import * as logging from '../../observability/logger'+type CapturedLevel = 'debug' | 'info' | 'warn' | 'error'+/** Observe the service logging boundary without replacing other log levels. */-export function captureLogger(level: 'debug' | 'info' | 'warn' | 'error') {+export function captureLogger(level: CapturedLevel) {const original = logging.createLoggerconst calls = vi.fn()const factory = vi.spyOn(logging, 'createLogger').mockImplementation((...args) => ({@@ -14,3 +16,19 @@ export function captureLogger(level: 'debug' | 'info' | 'warn' | 'error') {}return calls}++/**+ * Observe several levels through one spy on the logger factory; a second+ * `captureLogger` call would replace the first one's implementation.+ */+export function captureLoggerLevels<L extends CapturedLevel>(+ levels: readonly L[]+): { calls: Record<L, Mock>; restore: () => void } {+ const original = logging.createLogger+ const calls = Object.fromEntries(levels.map(level => [level, vi.fn()])) as Record<L, Mock>+ const factory = vi.spyOn(logging, 'createLogger').mockImplementation((...args) => ({+ ...original(...args),+ ...calls,+ }))+ return { calls, restore: () => factory.mockRestore() }+}diff --git a/workflow-recipes/src/reconciler/oauthBrokerDeleteLedger.property.test.ts b/workflow-recipes/src/reconciler/oauthBrokerDeleteLedger.property.test.tsindex ea84830dc..5e4d2c6b8 100644--- a/workflow-recipes/src/reconciler/oauthBrokerDeleteLedger.property.test.ts+++ b/workflow-recipes/src/reconciler/oauthBrokerDeleteLedger.property.test.ts@@ -64,12 +64,14 @@ function apply(ledger: OAuthBrokerDeleteLedger, op: Op): void {}/** A recorded delete and a provisioned token raise the watermark the same way. */-function applyToModel(model: Map<string, ModelEntry>, op: Op): void {+function applyToModel(model: Map<string, ModelEntry>, seen: Set<string>, op: Op): void {if (op.kind === 'invalidate') {model.delete(op.name)+ seen.delete(op.name)return}if (op.kind === 'invalidateSecret') {+ seen.add(op.name)const entry = model.get(op.name)if (entry) entry.rearmed = truereturn@@ -83,13 +85,23 @@ function applyToModel(model: Map<string, ModelEntry>, op: Op): void {}}-function checkAgainstModel(ledger: OAuthBrokerDeleteLedger, model: Map<string, ModelEntry>): void {+function checkAgainstModel(+ ledger: OAuthBrokerDeleteLedger,+ model: Map<string, ModelEntry>,+ seen: Set<string>+): void {for (const query of QUERIES) {const entry = model.get(query.name)const generation = query.generation ?? 0const actual = ledger.shouldDeleteSecret(query)const where = `${query.name}/${query.uid}/gen${generation} model=${JSON.stringify(entry)}`+ // No token ADDED since the process started or the recipe was deleted:+ // there is no token to reap.+ if (!seen.has(query.name)) {+ expect(actual, `no token seen must skip: ${where}`).toBe(false)+ continue+ }// A recipe never recorded, or recreated under the same name, always deletes.if (!entry || entry.uid !== query.uid) {expect(actual, `uid change must delete: ${where}`).toBe(true)@@ -108,15 +120,16 @@ function checkAgainstModel(ledger: OAuthBrokerDeleteLedger, model: Map<string, M}describe('OAuthBrokerDeleteLedger Secret-side properties', () => {- it('matches the uid / watermark / re-arm model over record, provision, invalidateSecret and invalidate', () => {+ it('matches the token-seen / uid / watermark / re-arm model over record, provision, invalidateSecret and invalidate', () => {fc.assert(fc.property(fc.array(opArb, { maxLength: 30 }), ops => {const ledger = new OAuthBrokerDeleteLedger()const model = new Map<string, ModelEntry>()+ const seen = new Set<string>()for (const op of ops) {apply(ledger, op)- applyToModel(model, op)- checkAgainstModel(ledger, model)+ applyToModel(model, seen, op)+ checkAgainstModel(ledger, model, seen)}}),{ numRuns: 5000 }@@ -146,6 +159,7 @@ describe('OAuthBrokerDeleteLedger Secret-side properties', () => {fc.integer({ min: 1, max: MAX_GENERATION }),(uid, watermark) => {const ledger = new OAuthBrokerDeleteLedger()+ ledger.invalidateSecret('a')record(ledger, { name: 'a', uid, generation: watermark })ledger.invalidateSecret('a')for (let generation = 0; generation <= MAX_GENERATION; generation++) {diff --git a/workflow-recipes/src/reconciler/oauthBrokerDeleteLedger.test.ts b/workflow-recipes/src/reconciler/oauthBrokerDeleteLedger.test.tsindex 469e2e0a0..deeb563a8 100644--- a/workflow-recipes/src/reconciler/oauthBrokerDeleteLedger.test.ts+++ b/workflow-recipes/src/reconciler/oauthBrokerDeleteLedger.test.ts@@ -13,9 +13,33 @@ function recordSecret(ledger: OAuthBrokerDeleteLedger, recipe: OAuthBrokerLedgerexpect(ledger.recordSecretDelete(recipe, ledger.secretEpoch(recipe.name))).toBe(true)}+/**+ * The Secret watch observed the recipe's token (an ADDED, or the relist that+ * replays one). Every Secret-side case below starts here: without it the+ * ledger answers false for every pass, so a false would prove nothing.+ */+function tokenSeen(ledger: OAuthBrokerDeleteLedger, name: string): void {+ ledger.invalidateSecret(name)+}+describe('OAuthBrokerDeleteLedger', () => {+ it('never deletes a Secret it never saw a token for', () => {+ const ledger = new OAuthBrokerDeleteLedger()+ for (const generation of [0, 1, 4]) {+ expect(ledger.shouldDeleteSecret(ref('r', generation))).toBe(false)+ }+ // A recipe recreated under the same name does not bypass it either.+ expect(ledger.shouldDeleteSecret(ref('r', 1, 'uid-recreated'))).toBe(false)++ // Liveness witness: the token ADDED arms the delete, for that recipe only.+ tokenSeen(ledger, 'r')+ expect(ledger.shouldDeleteSecret(ref('r', 4))).toBe(true)+ expect(ledger.shouldDeleteSecret(ref('r2', 4))).toBe(false)+ })+it('remembers a Secret delete for the recorded generation only', () => {const ledger = new OAuthBrokerDeleteLedger()+ tokenSeen(ledger, 'r')expect(ledger.shouldDeleteSecret(ref('r', 4))).toBe(true)recordSecret(ledger, ref('r', 4))expect(ledger.shouldDeleteSecret(ref('r', 4))).toBe(false)@@ -24,6 +48,7 @@ describe('OAuthBrokerDeleteLedger', () => {it('keeps the highest generation per recipe: an older generation arriving late is skipped', () => {const ledger = new OAuthBrokerDeleteLedger()+ tokenSeen(ledger, 'r')recordSecret(ledger, ref('r', 4))recordSecret(ledger, ref('r', 5))expect(ledger.shouldDeleteSecret(ref('r', 5))).toBe(false)@@ -39,9 +64,12 @@ describe('OAuthBrokerDeleteLedger', () => {it('an older generation recorded late does not lower the recorded generation', () => {const ledger = new OAuthBrokerDeleteLedger()+ tokenSeen(ledger, 'r')recordSecret(ledger, ref('r', 5))recordSecret(ledger, ref('r', 3))expect(ledger.shouldDeleteSecret(ref('r', 4))).toBe(false)+ // Liveness witness: the watermark is 5, not unset.+ expect(ledger.shouldDeleteSecret(ref('r', 6))).toBe(true)ledger.recordPolicyDelete(ref('r', 5), 0)ledger.recordPolicyDelete(ref('r', 3), 0)@@ -50,6 +78,7 @@ describe('OAuthBrokerDeleteLedger', () => {it('a recipe recreated under the same name (new uid) deletes again at any generation', () => {const ledger = new OAuthBrokerDeleteLedger()+ tokenSeen(ledger, 'r')recordSecret(ledger, ref('r', 5, 'uid-old'))ledger.recordPolicyDelete(ref('r', 5, 'uid-old'), 0)expect(ledger.shouldDeleteSecret(ref('r', 1, 'uid-old'))).toBe(false)@@ -66,6 +95,9 @@ describe('OAuthBrokerDeleteLedger', () => {it('does not let one recipe suppress another', () => {const ledger = new OAuthBrokerDeleteLedger()+ tokenSeen(ledger, 'a')+ tokenSeen(ledger, 'a-b')+ tokenSeen(ledger, 'b')recordSecret(ledger, ref('a', 1))expect(ledger.shouldDeleteSecret(ref('a', 1))).toBe(false)expect(ledger.shouldDeleteSecret(ref('a-b', 1))).toBe(true)@@ -79,8 +111,15 @@ describe('OAuthBrokerDeleteLedger', () => {expect(ledger.shouldDeletePolicy(ref('r', 4), 1_000 + OAUTH_BROKER_NP_TTL_MS)).toBe(true)})+ it('never saw a token: the NetworkPolicy side still deletes, it is not tied to the Secret', () => {+ const ledger = new OAuthBrokerDeleteLedger()+ expect(ledger.shouldDeleteSecret(ref('r', 4))).toBe(false)+ expect(ledger.shouldDeletePolicy(ref('r', 4), 1)).toBe(true)+ })+it('invalidateSecret re-arms only the Secret side; the NetworkPolicy TTL survives', () => {const ledger = new OAuthBrokerDeleteLedger()+ tokenSeen(ledger, 'r')recordSecret(ledger, ref('r', 4))ledger.recordPolicyDelete(ref('r', 4), 0)expect(ledger.shouldDeleteSecret(ref('r', 4))).toBe(false)@@ -93,6 +132,7 @@ describe('OAuthBrokerDeleteLedger', () => {it('R2-L1: an ADDED after a newer generation provisioned the token does not re-arm an older pass', () => {const ledger = new OAuthBrokerDeleteLedger()+ tokenSeen(ledger, 'r')// gen4 has no backgroundAccess: the Secret is reaped and recorded.recordSecret(ledger, ref('r', 4))// gen5 turns backgroundAccess on and the token is issued.@@ -108,6 +148,7 @@ describe('OAuthBrokerDeleteLedger', () => {it('R2-L1: invalidateSecret keeps the watermark: below it stays skipped, at it re-arms', () => {const ledger = new OAuthBrokerDeleteLedger()+ tokenSeen(ledger, 'r')recordSecret(ledger, ref('r', 5))ledger.invalidateSecret('r')expect(ledger.shouldDeleteSecret(ref('r', 4))).toBe(false)@@ -132,6 +173,7 @@ describe('OAuthBrokerDeleteLedger', () => {it('an invalidation for one recipe does not reject another recipe record', () => {const ledger = new OAuthBrokerDeleteLedger()+ tokenSeen(ledger, 'r2')const epochBeforeDelete = ledger.secretEpoch('r2')ledger.invalidateSecret('r')expect(ledger.recordSecretDelete(ref('r2', 4), epochBeforeDelete)).toBe(true)@@ -140,16 +182,24 @@ describe('OAuthBrokerDeleteLedger', () => {it('invalidate drops both sides for the named recipe and no other', () => {const ledger = new OAuthBrokerDeleteLedger()+ tokenSeen(ledger, 'r')+ tokenSeen(ledger, 'r2')recordSecret(ledger, ref('r', 4))ledger.recordPolicyDelete(ref('r', 4), 0)recordSecret(ledger, ref('r2', 4))ledger.recordPolicyDelete(ref('r2', 4), 0)ledger.invalidate('r')- expect(ledger.shouldDeleteSecret(ref('r', 4))).toBe(true)+ // The recipe is gone with its token: nothing to delete until a new ADDED.+ expect(ledger.shouldDeleteSecret(ref('r', 5))).toBe(false)expect(ledger.shouldDeletePolicy(ref('r', 4), 1)).toBe(true)expect(ledger.shouldDeleteSecret(ref('r2', 4))).toBe(false)expect(ledger.shouldDeletePolicy(ref('r2', 4), 1)).toBe(false)++ // The watermark went too: after the next ADDED a generation below the+ // old one deletes.+ tokenSeen(ledger, 'r')+ expect(ledger.shouldDeleteSecret(ref('r', 3))).toBe(true)})it('a DELETE in flight across a recipe deletion that followed an ADDED is not recorded', () => {@@ -158,6 +208,10 @@ describe('OAuthBrokerDeleteLedger', () => {const epochBeforeDelete = ledger.secretEpoch('r')ledger.invalidate('r')expect(ledger.recordSecretDelete(ref('r', 4), epochBeforeDelete)).toBe(false)- expect(ledger.shouldDeleteSecret(ref('r', 4))).toBe(true)++ // Nothing was recorded: after the next ADDED a lower generation deletes,+ // which a watermark at 4 would have skipped.+ tokenSeen(ledger, 'r')+ expect(ledger.shouldDeleteSecret(ref('r', 3))).toBe(true)})})diff --git a/workflow-recipes/src/reconciler/oauthBrokerDeleteLedger.ts b/workflow-recipes/src/reconciler/oauthBrokerDeleteLedger.tsindex bcb1c5667..34e12a1e1 100644--- a/workflow-recipes/src/reconciler/oauthBrokerDeleteLedger.ts+++ b/workflow-recipes/src/reconciler/oauthBrokerDeleteLedger.ts@@ -13,6 +13,9 @@* always deletes and replaces the entry even when the recipe DELETE event was* missed.*+ * The Secret side deletes only after the Secret watch observed the recipe's+ * token (a non-zero epoch): a recipe that never had one gets no DELETE.+ ** The Secret watermark rises on a recorded delete and on* `noteSecretProvisioned`, which the backgroundAccess branch calls for the* generation that wants the token.@@ -77,6 +80,10 @@ export class OAuthBrokerDeleteLedger {private epochClock = 0shouldDeleteSecret(recipe: OAuthBrokerLedgerRecipe): boolean {+ // No token ADDED since this process started or the recipe was deleted:+ // the Secret watch's initial list replays ADDED for every token that+ // exists, so there is no token to reap and a DELETE would only 404.+ if (this.secretEpoch(recipe.name) === 0) return falseconst entry = this.secrets.get(recipe.name)if (entry === undefined || entry.uid !== recipe.uid) return trueconst generation = normalizeGeneration(recipe.generation)diff --git a/workflow-recipes/src/reconciler/reconciler.test.ts b/workflow-recipes/src/reconciler/reconciler.test.tsindex 4fc76ae9c..284037cca 100644--- a/workflow-recipes/src/reconciler/reconciler.test.ts+++ b/workflow-recipes/src/reconciler/reconciler.test.ts@@ -23,7 +23,7 @@ import {WorkflowReconciler,translateNetworkPolicyApplySummary,} from '../workflow/workflowReconciler'-import { captureLogger } from './__tests__/captureLogger'+import { captureLogger, captureLoggerLevels } from './__tests__/captureLogger'import { defaultFqdnLookup } from './fqdnResolver'import { isRetryableInfraError } from './k8sErrors'import type { NetworkPolicyFamily } from './networkPolicyConvergence'@@ -15510,22 +15510,33 @@ describe('WorkflowRecipeReconciler', () => {).reconcileOAuthBrokerEgressPolicy(recipe)}+ /**+ * The Secret watch observed the recipe's token (an ADDED, or the relist+ * that replays one). The Secret-side cases start here: without it the+ * ledger reaps nothing, so a skipped DELETE would prove nothing.+ */+ function tokenSeen(recipeName = 'test-recipe'): void {+ reconciler.invalidateOAuthBrokerSecretLedger(recipeName)+ }+it('deletes the Secret once for the same generation and logs the skip', async () => {const recipe = reapRecipe(4)- const infoSpy = captureLogger('info')+ tokenSeen()+ const debugSpy = captureLogger('debug')await reconciler.ensureOAuthBrokerTokenSecret(recipe)await reconciler.ensureOAuthBrokerTokenSecret(recipe)expect(secretDeletes()).toBe(1)- expect(infoSpy).toHaveBeenCalledWith(+ expect(debugSpy).toHaveBeenCalledWith(expect.stringContaining('Skipping oauth-broker-token delete'),expect.objectContaining({ recipe: 'test-recipe', generation: 4 }))- infoSpy.mockRestore()+ debugSpy.mockRestore()})it('deletes the Secret again when metadata.generation changes', async () => {+ tokenSeen()await reconciler.ensureOAuthBrokerTokenSecret(reapRecipe(4))await reconciler.ensureOAuthBrokerTokenSecret(reapRecipe(4))expect(secretDeletes()).toBe(1)@@ -15534,7 +15545,8 @@ describe('WorkflowRecipeReconciler', () => {expect(secretDeletes()).toBe(2)})- it('deletes the Secret again on a new process (fresh reconciler)', async () => {+ it('a new process sends no token DELETE until the watch replays the token ADDED, then one', async () => {+ tokenSeen()await reconciler.ensureOAuthBrokerTokenSecret(reapRecipe(4))expect(secretDeletes()).toBe(1)@@ -15542,11 +15554,17 @@ describe('WorkflowRecipeReconciler', () => {verifyWorkflowRunProvenance: mockVerifyWorkflowRunProvenance,})await next.ensureOAuthBrokerTokenSecret(reapRecipe(4))+ expect(secretDeletes()).toBe(1)++ // The Secret watch's initial list replays ADDED for a token that exists.+ next.invalidateOAuthBrokerSecretLedger('test-recipe')+ await next.ensureOAuthBrokerTokenSecret(reapRecipe(4))expect(secretDeletes()).toBe(2)})it('deletes the Secret again after ADDED invalidation', async () => {const recipe = reapRecipe(4)+ tokenSeen()await reconciler.ensureOAuthBrokerTokenSecret(recipe)await reconciler.ensureOAuthBrokerTokenSecret(recipe)expect(secretDeletes()).toBe(1)@@ -15558,6 +15576,7 @@ describe('WorkflowRecipeReconciler', () => {it('ADDED invalidation re-arms the Secret delete but keeps the NetworkPolicy TTL', async () => {const recipe = reapRecipe(4)+ tokenSeen()await reconciler.ensureOAuthBrokerTokenSecret(recipe)await reapPolicy(recipe)expect(secretDeletes()).toBe(1)@@ -15570,20 +15589,29 @@ describe('WorkflowRecipeReconciler', () => {expect(policyDeletes()).toBe(1)})- it('deletes the Secret again after reconcileDelete invalidates the ledger', async () => {+ it('reconcileDelete drops the ledger: no DELETE until a new token ADDED, then even below the old watermark', async () => {const recipe = reapRecipe(4)+ tokenSeen()await reconciler.ensureOAuthBrokerTokenSecret(recipe)await reconciler.ensureOAuthBrokerTokenSecret(recipe)expect(secretDeletes()).toBe(1)await reconciler.reconcileDelete(recipe)mockCoreApi.deleteNamespacedSecret.mockClear()+ // The finalizer deleted the token with the recipe; nothing is left to reap.await reconciler.ensureOAuthBrokerTokenSecret(recipe)+ expect(secretDeletes()).toBe(0)++ // A token ADDED for a recipe under the same name: the old watermark is+ // gone, so a lower generation deletes.+ tokenSeen()+ await reconciler.ensureOAuthBrokerTokenSecret(reapRecipe(3))expect(secretDeletes()).toBe(1)})it('G3: does not record a Secret delete after DELETE 403 or 500', async () => {const recipe = reapRecipe(4)+ tokenSeen()mockCoreApi.deleteNamespacedSecret.mockRejectedValue({ code: 403 })await reconciler.ensureOAuthBrokerTokenSecret(recipe)await reconciler.ensureOAuthBrokerTokenSecret(recipe)@@ -15600,6 +15628,7 @@ describe('WorkflowRecipeReconciler', () => {it('G3: records a Secret delete after DELETE 404 and skips the next same-generation call', async () => {mockCoreApi.deleteNamespacedSecret.mockRejectedValue({ code: 404 })const recipe = reapRecipe(7)+ tokenSeen()await reconciler.ensureOAuthBrokerTokenSecret(recipe)await reconciler.ensureOAuthBrokerTokenSecret(recipe)expect(secretDeletes()).toBe(1)@@ -15630,6 +15659,8 @@ describe('WorkflowRecipeReconciler', () => {})it('c3: a recorded delete for one recipe does not skip another recipe', async () => {+ tokenSeen()+ tokenSeen('other-recipe')await reconciler.ensureOAuthBrokerTokenSecret(reapRecipe(0))// Liveness witness: the first recipe's delete really ran.expect(secretDeletes()).toBe(1)@@ -15670,7 +15701,8 @@ describe('WorkflowRecipeReconciler', () => {})it('R1-L2: a late pass carrying an older generation does not re-delete the Secret', async () => {- const infoSpy = captureLogger('info')+ tokenSeen()+ const debugSpy = captureLogger('debug')await reconciler.ensureOAuthBrokerTokenSecret(reapRecipe(4))await reconciler.ensureOAuthBrokerTokenSecret(reapRecipe(5))// Liveness witness: both newer-generation deletes really ran.@@ -15678,11 +15710,11 @@ describe('WorkflowRecipeReconciler', () => {await reconciler.ensureOAuthBrokerTokenSecret(reapRecipe(4))expect(secretDeletes()).toBe(2)- expect(infoSpy).toHaveBeenCalledWith(+ expect(debugSpy).toHaveBeenCalledWith(expect.stringContaining('Skipping oauth-broker-token delete'),expect.objectContaining({ recipe: 'test-recipe', generation: 4 }))- infoSpy.mockRestore()+ debugSpy.mockRestore()})it('R1-L2: a late pass carrying an older generation does not re-delete the NetworkPolicy', async () => {@@ -15702,6 +15734,7 @@ describe('WorkflowRecipeReconciler', () => {})it('R1-L2: a recipe recreated under the same name is reaped even at a lower generation', async () => {+ tokenSeen()await reconciler.ensureOAuthBrokerTokenSecret(reapRecipe(5))await reapPolicy(reapRecipe(5))expect(secretDeletes()).toBe(1)@@ -15724,6 +15757,7 @@ describe('WorkflowRecipeReconciler', () => {it('R1-L1: an ADDED invalidation that lands while the DELETE is in flight is not overwritten', async () => {const recipe = reapRecipe(4)+ tokenSeen()mockCoreApi.deleteNamespacedSecret.mockImplementationOnce(async () => {// Another writer recreated the Secret and its ADDED was handled before// this DELETE returned.@@ -15748,8 +15782,9 @@ describe('WorkflowRecipeReconciler', () => {.mockResolvedValue({ brokerToken: 'issued-token' } as Awaited<ReturnType<typeof brokerIssuer.issueOAuthBrokerToken>>)- const infoSpy = captureLogger('info')+ const debugSpy = captureLogger('debug')try {+ tokenSeen()// gen4 has no backgroundAccess: the Secret is reaped and recorded.await reconciler.ensureOAuthBrokerTokenSecret(reapRecipe(4))expect(secretDeletes()).toBe(1)@@ -15792,7 +15827,7 @@ describe('WorkflowRecipeReconciler', () => {reconciler.invalidateOAuthBrokerSecretLedger('test-recipe')await reconciler.ensureOAuthBrokerTokenSecret(reapRecipe(4))expect(secretDeletes()).toBe(1)- expect(infoSpy).toHaveBeenCalledWith(+ expect(debugSpy).toHaveBeenCalledWith(expect.stringContaining('Skipping oauth-broker-token delete'),expect.objectContaining({ recipe: 'test-recipe', generation: 4 }))@@ -15801,7 +15836,7 @@ describe('WorkflowRecipeReconciler', () => {await reconciler.ensureOAuthBrokerTokenSecret(reapRecipe(6))expect(secretDeletes()).toBe(2)} finally {- infoSpy.mockRestore()+ debugSpy.mockRestore()issue.mockRestore()}})@@ -15987,13 +16022,32 @@ describe('WorkflowRecipeReconciler', () => {return { recipe, workflowReconcile }}- it('R2-L3: without a token ADDED the first-deploy pass still sends its one DELETE', async () => {+ it('R3-L11: without a token ADDED the first-deploy pass sends no DELETE and logs the skip once, at debug', async () => {const { recipe, workflowReconcile } = installFirstDeployWorkflow()+ const skips = (spy: ReturnType<typeof vi.fn>) =>+ spy.mock.calls.filter(+ ([message]) =>+ typeof message === 'string' && message.startsWith('Skipping oauth-broker-token delete')+ )+ const logs = captureLoggerLevels(['debug', 'info'] as const)+ try {+ await reconciler.reconcile(recipe)- await reconciler.reconcile(recipe)+ // Witness that the pass went through deployWorkflowWorkloads.+ expect(workflowReconcile).toHaveBeenCalledTimes(1)+ // Witness that the reap ran and the ledger declined it, once per pass.+ expect(skips(logs.calls.debug)).toHaveLength(1)+ expect(skips(logs.calls.info)).toHaveLength(0)+ expect(secretDeletes()).toBe(0)+ } finally {+ logs.restore()+ }- // Witness that the pass went through deployWorkflowWorkloads.- expect(workflowReconcile).toHaveBeenCalledTimes(1)+ // Liveness witness: once the watch replays the token ADDED, the next+ // pass sends its one DELETE.+ tokenSeen()+ await reconciler.reconcile(recipe)+ expect(workflowReconcile).toHaveBeenCalledTimes(2)expect(secretDeletes()).toBe(1)})diff --git a/workflow-recipes/src/reconciler/workflowRecipeReconciler.ts b/workflow-recipes/src/reconciler/workflowRecipeReconciler.tsindex 8995960cf..9f3a2be89 100644--- a/workflow-recipes/src/reconciler/workflowRecipeReconciler.ts+++ b/workflow-recipes/src/reconciler/workflowRecipeReconciler.ts@@ -1986,15 +1986,11 @@ export class WorkflowRecipeReconciler {// ensureOAuthBrokerTokenSecret, so the token-ADDED re-reconcile would// never reach the reap. Only the delete branch runs here (issuance stays// on the first-deploy path and the rotation loop), and the delete ledger- // bounds it to one DELETE per generation or ADDED. It runs only once a- // token ADDED was observed for the recipe: a fresh process would otherwise- // send one DELETE per recipe, and the Secret watch's initial list replays- // ADDED for every token that exists. The first-deploy path below skips+ // decides: no DELETE until a token ADDED was observed for the recipe,+ // then one per generation or ADDED. The first-deploy path below skips// its own call when this one ran, so a failed DELETE is not repeated in// the same pass.- const oauthBrokerTokenReapedEarly =- !rb.recipeHasBackgroundAccessClient(recipe) &&- this.oauthBrokerDeleteLedger.secretEpoch(recipe.metadata.name) !== 0+ const oauthBrokerTokenReapedEarly = !rb.recipeHasBackgroundAccessClient(recipe)if (oauthBrokerTokenReapedEarly) {await this.ensureOAuthBrokerTokenSecret(recipe)}@@ -7565,8 +7561,9 @@ export class WorkflowRecipeReconciler {if (!rb.recipeHasBackgroundAccessClient(recipe)) {const generation = recipe.metadata.generationif (!this.oauthBrokerDeleteLedger.shouldDeleteSecret(recipe.metadata)) {- createLogger('wrc', recipeName).info(- 'Skipping oauth-broker-token delete; generation already seen',+ // Every non-backgroundAccess pass reaches this, so it stays at debug.+ createLogger('wrc', recipeName).debug(+ 'Skipping oauth-broker-token delete; no token seen, or this generation already reaped it',{ recipe: recipeName, generation })return