Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript

fix(workflow-recipes): send no oauth-broker token DELETE for a token never seen

Alfredo Lopez1c228b2a2c5b
diff --git a/workflow-recipes/package-lock.json b/workflow-recipes/package-lock.json
index 1268ec7db..a7d261f86 100644
--- a/workflow-recipes/package-lock.json
+++ b/workflow-recipes/package-lock.json
@@ -1,12 +1,12 @@
{
"name": "workload-recipes-controller",
- "version": "0.1.428",
+ "version": "0.1.429",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "workload-recipes-controller",
- "version": "0.1.428",
+ "version": "0.1.429",
"license": "SEE LICENSE IN LICENSE",
"dependencies": {
"@aws-sdk/client-s3": "^3.1009.0",
diff --git a/workflow-recipes/package.json b/workflow-recipes/package.json
index 85237ed10..bf3b3bbcc 100644
--- a/workflow-recipes/package.json
+++ b/workflow-recipes/package.json
@@ -1,6 +1,6 @@
{
"name": "workload-recipes-controller",
- "version": "0.1.428",
+ "version": "0.1.429",
"description": "Workload Recipes Controller (WRC) - WorkloadRecipe lifecycle management and MCP server interface",
"main": "dist/main.js",
"engines": {
diff --git a/workflow-recipes/src/reconciler/__tests__/captureLogger.ts b/workflow-recipes/src/reconciler/__tests__/captureLogger.ts
index 4dab2d0e7..cdfa3df8c 100644
--- a/workflow-recipes/src/reconciler/__tests__/captureLogger.ts
+++ b/workflow-recipes/src/reconciler/__tests__/captureLogger.ts
@@ -1,8 +1,10 @@
-import { vi } from 'vitest'
+import { type Mock, vi } from 'vitest'
import * as logging from '../../observability/logger'
+type CapturedLevel = 'debug' | 'info' | 'warn' | 'error'
+
/** Observe the service logging boundary without replacing other log levels. */
-export function captureLogger(level: 'debug' | 'info' | 'warn' | 'error') {
+export function captureLogger(level: CapturedLevel) {
const original = logging.createLogger
const calls = vi.fn()
const factory = vi.spyOn(logging, 'createLogger').mockImplementation((...args) => ({
@@ -14,3 +16,19 @@ export function captureLogger(level: 'debug' | 'info' | 'warn' | 'error') {
}
return calls
}
+
+/**
+ * Observe several levels through one spy on the logger factory; a second
+ * `captureLogger` call would replace the first one's implementation.
+ */
+export function captureLoggerLevels<L extends CapturedLevel>(
+ levels: readonly L[]
+): { calls: Record<L, Mock>; restore: () => void } {
+ const original = logging.createLogger
+ const calls = Object.fromEntries(levels.map(level => [level, vi.fn()])) as Record<L, Mock>
+ const factory = vi.spyOn(logging, 'createLogger').mockImplementation((...args) => ({
+ ...original(...args),
+ ...calls,
+ }))
+ return { calls, restore: () => factory.mockRestore() }
+}
diff --git a/workflow-recipes/src/reconciler/oauthBrokerDeleteLedger.property.test.ts b/workflow-recipes/src/reconciler/oauthBrokerDeleteLedger.property.test.ts
index ea84830dc..5e4d2c6b8 100644
--- a/workflow-recipes/src/reconciler/oauthBrokerDeleteLedger.property.test.ts
+++ b/workflow-recipes/src/reconciler/oauthBrokerDeleteLedger.property.test.ts
@@ -64,12 +64,14 @@ function apply(ledger: OAuthBrokerDeleteLedger, op: Op): void {
}
/** A recorded delete and a provisioned token raise the watermark the same way. */
-function applyToModel(model: Map<string, ModelEntry>, op: Op): void {
+function applyToModel(model: Map<string, ModelEntry>, seen: Set<string>, op: Op): void {
if (op.kind === 'invalidate') {
model.delete(op.name)
+ seen.delete(op.name)
return
}
if (op.kind === 'invalidateSecret') {
+ seen.add(op.name)
const entry = model.get(op.name)
if (entry) entry.rearmed = true
return
@@ -83,13 +85,23 @@ function applyToModel(model: Map<string, ModelEntry>, op: Op): void {
}
}
-function checkAgainstModel(ledger: OAuthBrokerDeleteLedger, model: Map<string, ModelEntry>): void {
+function checkAgainstModel(
+ ledger: OAuthBrokerDeleteLedger,
+ model: Map<string, ModelEntry>,
+ seen: Set<string>
+): void {
for (const query of QUERIES) {
const entry = model.get(query.name)
const generation = query.generation ?? 0
const actual = ledger.shouldDeleteSecret(query)
const where = `${query.name}/${query.uid}/gen${generation} model=${JSON.stringify(entry)}`
+ // No token ADDED since the process started or the recipe was deleted:
+ // there is no token to reap.
+ if (!seen.has(query.name)) {
+ expect(actual, `no token seen must skip: ${where}`).toBe(false)
+ continue
+ }
// A recipe never recorded, or recreated under the same name, always deletes.
if (!entry || entry.uid !== query.uid) {
expect(actual, `uid change must delete: ${where}`).toBe(true)
@@ -108,15 +120,16 @@ function checkAgainstModel(ledger: OAuthBrokerDeleteLedger, model: Map<string, M
}
describe('OAuthBrokerDeleteLedger Secret-side properties', () => {
- it('matches the uid / watermark / re-arm model over record, provision, invalidateSecret and invalidate', () => {
+ it('matches the token-seen / uid / watermark / re-arm model over record, provision, invalidateSecret and invalidate', () => {
fc.assert(
fc.property(fc.array(opArb, { maxLength: 30 }), ops => {
const ledger = new OAuthBrokerDeleteLedger()
const model = new Map<string, ModelEntry>()
+ const seen = new Set<string>()
for (const op of ops) {
apply(ledger, op)
- applyToModel(model, op)
- checkAgainstModel(ledger, model)
+ applyToModel(model, seen, op)
+ checkAgainstModel(ledger, model, seen)
}
}),
{ numRuns: 5000 }
@@ -146,6 +159,7 @@ describe('OAuthBrokerDeleteLedger Secret-side properties', () => {
fc.integer({ min: 1, max: MAX_GENERATION }),
(uid, watermark) => {
const ledger = new OAuthBrokerDeleteLedger()
+ ledger.invalidateSecret('a')
record(ledger, { name: 'a', uid, generation: watermark })
ledger.invalidateSecret('a')
for (let generation = 0; generation <= MAX_GENERATION; generation++) {
diff --git a/workflow-recipes/src/reconciler/oauthBrokerDeleteLedger.test.ts b/workflow-recipes/src/reconciler/oauthBrokerDeleteLedger.test.ts
index 469e2e0a0..deeb563a8 100644
--- a/workflow-recipes/src/reconciler/oauthBrokerDeleteLedger.test.ts
+++ b/workflow-recipes/src/reconciler/oauthBrokerDeleteLedger.test.ts
@@ -13,9 +13,33 @@ function recordSecret(ledger: OAuthBrokerDeleteLedger, recipe: OAuthBrokerLedger
expect(ledger.recordSecretDelete(recipe, ledger.secretEpoch(recipe.name))).toBe(true)
}
+/**
+ * The Secret watch observed the recipe's token (an ADDED, or the relist that
+ * replays one). Every Secret-side case below starts here: without it the
+ * ledger answers false for every pass, so a false would prove nothing.
+ */
+function tokenSeen(ledger: OAuthBrokerDeleteLedger, name: string): void {
+ ledger.invalidateSecret(name)
+}
+
describe('OAuthBrokerDeleteLedger', () => {
+ it('never deletes a Secret it never saw a token for', () => {
+ const ledger = new OAuthBrokerDeleteLedger()
+ for (const generation of [0, 1, 4]) {
+ expect(ledger.shouldDeleteSecret(ref('r', generation))).toBe(false)
+ }
+ // A recipe recreated under the same name does not bypass it either.
+ expect(ledger.shouldDeleteSecret(ref('r', 1, 'uid-recreated'))).toBe(false)
+
+ // Liveness witness: the token ADDED arms the delete, for that recipe only.
+ tokenSeen(ledger, 'r')
+ expect(ledger.shouldDeleteSecret(ref('r', 4))).toBe(true)
+ expect(ledger.shouldDeleteSecret(ref('r2', 4))).toBe(false)
+ })
+
it('remembers a Secret delete for the recorded generation only', () => {
const ledger = new OAuthBrokerDeleteLedger()
+ tokenSeen(ledger, 'r')
expect(ledger.shouldDeleteSecret(ref('r', 4))).toBe(true)
recordSecret(ledger, ref('r', 4))
expect(ledger.shouldDeleteSecret(ref('r', 4))).toBe(false)
@@ -24,6 +48,7 @@ describe('OAuthBrokerDeleteLedger', () => {
it('keeps the highest generation per recipe: an older generation arriving late is skipped', () => {
const ledger = new OAuthBrokerDeleteLedger()
+ tokenSeen(ledger, 'r')
recordSecret(ledger, ref('r', 4))
recordSecret(ledger, ref('r', 5))
expect(ledger.shouldDeleteSecret(ref('r', 5))).toBe(false)
@@ -39,9 +64,12 @@ describe('OAuthBrokerDeleteLedger', () => {
it('an older generation recorded late does not lower the recorded generation', () => {
const ledger = new OAuthBrokerDeleteLedger()
+ tokenSeen(ledger, 'r')
recordSecret(ledger, ref('r', 5))
recordSecret(ledger, ref('r', 3))
expect(ledger.shouldDeleteSecret(ref('r', 4))).toBe(false)
+ // Liveness witness: the watermark is 5, not unset.
+ expect(ledger.shouldDeleteSecret(ref('r', 6))).toBe(true)
ledger.recordPolicyDelete(ref('r', 5), 0)
ledger.recordPolicyDelete(ref('r', 3), 0)
@@ -50,6 +78,7 @@ describe('OAuthBrokerDeleteLedger', () => {
it('a recipe recreated under the same name (new uid) deletes again at any generation', () => {
const ledger = new OAuthBrokerDeleteLedger()
+ tokenSeen(ledger, 'r')
recordSecret(ledger, ref('r', 5, 'uid-old'))
ledger.recordPolicyDelete(ref('r', 5, 'uid-old'), 0)
expect(ledger.shouldDeleteSecret(ref('r', 1, 'uid-old'))).toBe(false)
@@ -66,6 +95,9 @@ describe('OAuthBrokerDeleteLedger', () => {
it('does not let one recipe suppress another', () => {
const ledger = new OAuthBrokerDeleteLedger()
+ tokenSeen(ledger, 'a')
+ tokenSeen(ledger, 'a-b')
+ tokenSeen(ledger, 'b')
recordSecret(ledger, ref('a', 1))
expect(ledger.shouldDeleteSecret(ref('a', 1))).toBe(false)
expect(ledger.shouldDeleteSecret(ref('a-b', 1))).toBe(true)
@@ -79,8 +111,15 @@ describe('OAuthBrokerDeleteLedger', () => {
expect(ledger.shouldDeletePolicy(ref('r', 4), 1_000 + OAUTH_BROKER_NP_TTL_MS)).toBe(true)
})
+ it('never saw a token: the NetworkPolicy side still deletes, it is not tied to the Secret', () => {
+ const ledger = new OAuthBrokerDeleteLedger()
+ expect(ledger.shouldDeleteSecret(ref('r', 4))).toBe(false)
+ expect(ledger.shouldDeletePolicy(ref('r', 4), 1)).toBe(true)
+ })
+
it('invalidateSecret re-arms only the Secret side; the NetworkPolicy TTL survives', () => {
const ledger = new OAuthBrokerDeleteLedger()
+ tokenSeen(ledger, 'r')
recordSecret(ledger, ref('r', 4))
ledger.recordPolicyDelete(ref('r', 4), 0)
expect(ledger.shouldDeleteSecret(ref('r', 4))).toBe(false)
@@ -93,6 +132,7 @@ describe('OAuthBrokerDeleteLedger', () => {
it('R2-L1: an ADDED after a newer generation provisioned the token does not re-arm an older pass', () => {
const ledger = new OAuthBrokerDeleteLedger()
+ tokenSeen(ledger, 'r')
// gen4 has no backgroundAccess: the Secret is reaped and recorded.
recordSecret(ledger, ref('r', 4))
// gen5 turns backgroundAccess on and the token is issued.
@@ -108,6 +148,7 @@ describe('OAuthBrokerDeleteLedger', () => {
it('R2-L1: invalidateSecret keeps the watermark: below it stays skipped, at it re-arms', () => {
const ledger = new OAuthBrokerDeleteLedger()
+ tokenSeen(ledger, 'r')
recordSecret(ledger, ref('r', 5))
ledger.invalidateSecret('r')
expect(ledger.shouldDeleteSecret(ref('r', 4))).toBe(false)
@@ -132,6 +173,7 @@ describe('OAuthBrokerDeleteLedger', () => {
it('an invalidation for one recipe does not reject another recipe record', () => {
const ledger = new OAuthBrokerDeleteLedger()
+ tokenSeen(ledger, 'r2')
const epochBeforeDelete = ledger.secretEpoch('r2')
ledger.invalidateSecret('r')
expect(ledger.recordSecretDelete(ref('r2', 4), epochBeforeDelete)).toBe(true)
@@ -140,16 +182,24 @@ describe('OAuthBrokerDeleteLedger', () => {
it('invalidate drops both sides for the named recipe and no other', () => {
const ledger = new OAuthBrokerDeleteLedger()
+ tokenSeen(ledger, 'r')
+ tokenSeen(ledger, 'r2')
recordSecret(ledger, ref('r', 4))
ledger.recordPolicyDelete(ref('r', 4), 0)
recordSecret(ledger, ref('r2', 4))
ledger.recordPolicyDelete(ref('r2', 4), 0)
ledger.invalidate('r')
- expect(ledger.shouldDeleteSecret(ref('r', 4))).toBe(true)
+ // The recipe is gone with its token: nothing to delete until a new ADDED.
+ expect(ledger.shouldDeleteSecret(ref('r', 5))).toBe(false)
expect(ledger.shouldDeletePolicy(ref('r', 4), 1)).toBe(true)
expect(ledger.shouldDeleteSecret(ref('r2', 4))).toBe(false)
expect(ledger.shouldDeletePolicy(ref('r2', 4), 1)).toBe(false)
+
+ // The watermark went too: after the next ADDED a generation below the
+ // old one deletes.
+ tokenSeen(ledger, 'r')
+ expect(ledger.shouldDeleteSecret(ref('r', 3))).toBe(true)
})
it('a DELETE in flight across a recipe deletion that followed an ADDED is not recorded', () => {
@@ -158,6 +208,10 @@ describe('OAuthBrokerDeleteLedger', () => {
const epochBeforeDelete = ledger.secretEpoch('r')
ledger.invalidate('r')
expect(ledger.recordSecretDelete(ref('r', 4), epochBeforeDelete)).toBe(false)
- expect(ledger.shouldDeleteSecret(ref('r', 4))).toBe(true)
+
+ // Nothing was recorded: after the next ADDED a lower generation deletes,
+ // which a watermark at 4 would have skipped.
+ tokenSeen(ledger, 'r')
+ expect(ledger.shouldDeleteSecret(ref('r', 3))).toBe(true)
})
})
diff --git a/workflow-recipes/src/reconciler/oauthBrokerDeleteLedger.ts b/workflow-recipes/src/reconciler/oauthBrokerDeleteLedger.ts
index bcb1c5667..34e12a1e1 100644
--- a/workflow-recipes/src/reconciler/oauthBrokerDeleteLedger.ts
+++ b/workflow-recipes/src/reconciler/oauthBrokerDeleteLedger.ts
@@ -13,6 +13,9 @@
* always deletes and replaces the entry even when the recipe DELETE event was
* missed.
*
+ * The Secret side deletes only after the Secret watch observed the recipe's
+ * token (a non-zero epoch): a recipe that never had one gets no DELETE.
+ *
* The Secret watermark rises on a recorded delete and on
* `noteSecretProvisioned`, which the backgroundAccess branch calls for the
* generation that wants the token.
@@ -77,6 +80,10 @@ export class OAuthBrokerDeleteLedger {
private epochClock = 0
shouldDeleteSecret(recipe: OAuthBrokerLedgerRecipe): boolean {
+ // No token ADDED since this process started or the recipe was deleted:
+ // the Secret watch's initial list replays ADDED for every token that
+ // exists, so there is no token to reap and a DELETE would only 404.
+ if (this.secretEpoch(recipe.name) === 0) return false
const entry = this.secrets.get(recipe.name)
if (entry === undefined || entry.uid !== recipe.uid) return true
const generation = normalizeGeneration(recipe.generation)
diff --git a/workflow-recipes/src/reconciler/reconciler.test.ts b/workflow-recipes/src/reconciler/reconciler.test.ts
index 4fc76ae9c..284037cca 100644
--- a/workflow-recipes/src/reconciler/reconciler.test.ts
+++ b/workflow-recipes/src/reconciler/reconciler.test.ts
@@ -23,7 +23,7 @@ import {
WorkflowReconciler,
translateNetworkPolicyApplySummary,
} from '../workflow/workflowReconciler'
-import { captureLogger } from './__tests__/captureLogger'
+import { captureLogger, captureLoggerLevels } from './__tests__/captureLogger'
import { defaultFqdnLookup } from './fqdnResolver'
import { isRetryableInfraError } from './k8sErrors'
import type { NetworkPolicyFamily } from './networkPolicyConvergence'
@@ -15510,22 +15510,33 @@ describe('WorkflowRecipeReconciler', () => {
).reconcileOAuthBrokerEgressPolicy(recipe)
}
+ /**
+ * The Secret watch observed the recipe's token (an ADDED, or the relist
+ * that replays one). The Secret-side cases start here: without it the
+ * ledger reaps nothing, so a skipped DELETE would prove nothing.
+ */
+ function tokenSeen(recipeName = 'test-recipe'): void {
+ reconciler.invalidateOAuthBrokerSecretLedger(recipeName)
+ }
+
it('deletes the Secret once for the same generation and logs the skip', async () => {
const recipe = reapRecipe(4)
- const infoSpy = captureLogger('info')
+ tokenSeen()
+ const debugSpy = captureLogger('debug')
await reconciler.ensureOAuthBrokerTokenSecret(recipe)
await reconciler.ensureOAuthBrokerTokenSecret(recipe)
expect(secretDeletes()).toBe(1)
- expect(infoSpy).toHaveBeenCalledWith(
+ expect(debugSpy).toHaveBeenCalledWith(
expect.stringContaining('Skipping oauth-broker-token delete'),
expect.objectContaining({ recipe: 'test-recipe', generation: 4 })
)
- infoSpy.mockRestore()
+ debugSpy.mockRestore()
})
it('deletes the Secret again when metadata.generation changes', async () => {
+ tokenSeen()
await reconciler.ensureOAuthBrokerTokenSecret(reapRecipe(4))
await reconciler.ensureOAuthBrokerTokenSecret(reapRecipe(4))
expect(secretDeletes()).toBe(1)
@@ -15534,7 +15545,8 @@ describe('WorkflowRecipeReconciler', () => {
expect(secretDeletes()).toBe(2)
})
- it('deletes the Secret again on a new process (fresh reconciler)', async () => {
+ it('a new process sends no token DELETE until the watch replays the token ADDED, then one', async () => {
+ tokenSeen()
await reconciler.ensureOAuthBrokerTokenSecret(reapRecipe(4))
expect(secretDeletes()).toBe(1)
@@ -15542,11 +15554,17 @@ describe('WorkflowRecipeReconciler', () => {
verifyWorkflowRunProvenance: mockVerifyWorkflowRunProvenance,
})
await next.ensureOAuthBrokerTokenSecret(reapRecipe(4))
+ expect(secretDeletes()).toBe(1)
+
+ // The Secret watch's initial list replays ADDED for a token that exists.
+ next.invalidateOAuthBrokerSecretLedger('test-recipe')
+ await next.ensureOAuthBrokerTokenSecret(reapRecipe(4))
expect(secretDeletes()).toBe(2)
})
it('deletes the Secret again after ADDED invalidation', async () => {
const recipe = reapRecipe(4)
+ tokenSeen()
await reconciler.ensureOAuthBrokerTokenSecret(recipe)
await reconciler.ensureOAuthBrokerTokenSecret(recipe)
expect(secretDeletes()).toBe(1)
@@ -15558,6 +15576,7 @@ describe('WorkflowRecipeReconciler', () => {
it('ADDED invalidation re-arms the Secret delete but keeps the NetworkPolicy TTL', async () => {
const recipe = reapRecipe(4)
+ tokenSeen()
await reconciler.ensureOAuthBrokerTokenSecret(recipe)
await reapPolicy(recipe)
expect(secretDeletes()).toBe(1)
@@ -15570,20 +15589,29 @@ describe('WorkflowRecipeReconciler', () => {
expect(policyDeletes()).toBe(1)
})
- it('deletes the Secret again after reconcileDelete invalidates the ledger', async () => {
+ it('reconcileDelete drops the ledger: no DELETE until a new token ADDED, then even below the old watermark', async () => {
const recipe = reapRecipe(4)
+ tokenSeen()
await reconciler.ensureOAuthBrokerTokenSecret(recipe)
await reconciler.ensureOAuthBrokerTokenSecret(recipe)
expect(secretDeletes()).toBe(1)
await reconciler.reconcileDelete(recipe)
mockCoreApi.deleteNamespacedSecret.mockClear()
+ // The finalizer deleted the token with the recipe; nothing is left to reap.
await reconciler.ensureOAuthBrokerTokenSecret(recipe)
+ expect(secretDeletes()).toBe(0)
+
+ // A token ADDED for a recipe under the same name: the old watermark is
+ // gone, so a lower generation deletes.
+ tokenSeen()
+ await reconciler.ensureOAuthBrokerTokenSecret(reapRecipe(3))
expect(secretDeletes()).toBe(1)
})
it('G3: does not record a Secret delete after DELETE 403 or 500', async () => {
const recipe = reapRecipe(4)
+ tokenSeen()
mockCoreApi.deleteNamespacedSecret.mockRejectedValue({ code: 403 })
await reconciler.ensureOAuthBrokerTokenSecret(recipe)
await reconciler.ensureOAuthBrokerTokenSecret(recipe)
@@ -15600,6 +15628,7 @@ describe('WorkflowRecipeReconciler', () => {
it('G3: records a Secret delete after DELETE 404 and skips the next same-generation call', async () => {
mockCoreApi.deleteNamespacedSecret.mockRejectedValue({ code: 404 })
const recipe = reapRecipe(7)
+ tokenSeen()
await reconciler.ensureOAuthBrokerTokenSecret(recipe)
await reconciler.ensureOAuthBrokerTokenSecret(recipe)
expect(secretDeletes()).toBe(1)
@@ -15630,6 +15659,8 @@ describe('WorkflowRecipeReconciler', () => {
})
it('c3: a recorded delete for one recipe does not skip another recipe', async () => {
+ tokenSeen()
+ tokenSeen('other-recipe')
await reconciler.ensureOAuthBrokerTokenSecret(reapRecipe(0))
// Liveness witness: the first recipe's delete really ran.
expect(secretDeletes()).toBe(1)
@@ -15670,7 +15701,8 @@ describe('WorkflowRecipeReconciler', () => {
})
it('R1-L2: a late pass carrying an older generation does not re-delete the Secret', async () => {
- const infoSpy = captureLogger('info')
+ tokenSeen()
+ const debugSpy = captureLogger('debug')
await reconciler.ensureOAuthBrokerTokenSecret(reapRecipe(4))
await reconciler.ensureOAuthBrokerTokenSecret(reapRecipe(5))
// Liveness witness: both newer-generation deletes really ran.
@@ -15678,11 +15710,11 @@ describe('WorkflowRecipeReconciler', () => {
await reconciler.ensureOAuthBrokerTokenSecret(reapRecipe(4))
expect(secretDeletes()).toBe(2)
- expect(infoSpy).toHaveBeenCalledWith(
+ expect(debugSpy).toHaveBeenCalledWith(
expect.stringContaining('Skipping oauth-broker-token delete'),
expect.objectContaining({ recipe: 'test-recipe', generation: 4 })
)
- infoSpy.mockRestore()
+ debugSpy.mockRestore()
})
it('R1-L2: a late pass carrying an older generation does not re-delete the NetworkPolicy', async () => {
@@ -15702,6 +15734,7 @@ describe('WorkflowRecipeReconciler', () => {
})
it('R1-L2: a recipe recreated under the same name is reaped even at a lower generation', async () => {
+ tokenSeen()
await reconciler.ensureOAuthBrokerTokenSecret(reapRecipe(5))
await reapPolicy(reapRecipe(5))
expect(secretDeletes()).toBe(1)
@@ -15724,6 +15757,7 @@ describe('WorkflowRecipeReconciler', () => {
it('R1-L1: an ADDED invalidation that lands while the DELETE is in flight is not overwritten', async () => {
const recipe = reapRecipe(4)
+ tokenSeen()
mockCoreApi.deleteNamespacedSecret.mockImplementationOnce(async () => {
// Another writer recreated the Secret and its ADDED was handled before
// this DELETE returned.
@@ -15748,8 +15782,9 @@ describe('WorkflowRecipeReconciler', () => {
.mockResolvedValue({ brokerToken: 'issued-token' } as Awaited<
ReturnType<typeof brokerIssuer.issueOAuthBrokerToken>
>)
- const infoSpy = captureLogger('info')
+ const debugSpy = captureLogger('debug')
try {
+ tokenSeen()
// gen4 has no backgroundAccess: the Secret is reaped and recorded.
await reconciler.ensureOAuthBrokerTokenSecret(reapRecipe(4))
expect(secretDeletes()).toBe(1)
@@ -15792,7 +15827,7 @@ describe('WorkflowRecipeReconciler', () => {
reconciler.invalidateOAuthBrokerSecretLedger('test-recipe')
await reconciler.ensureOAuthBrokerTokenSecret(reapRecipe(4))
expect(secretDeletes()).toBe(1)
- expect(infoSpy).toHaveBeenCalledWith(
+ expect(debugSpy).toHaveBeenCalledWith(
expect.stringContaining('Skipping oauth-broker-token delete'),
expect.objectContaining({ recipe: 'test-recipe', generation: 4 })
)
@@ -15801,7 +15836,7 @@ describe('WorkflowRecipeReconciler', () => {
await reconciler.ensureOAuthBrokerTokenSecret(reapRecipe(6))
expect(secretDeletes()).toBe(2)
} finally {
- infoSpy.mockRestore()
+ debugSpy.mockRestore()
issue.mockRestore()
}
})
@@ -15987,13 +16022,32 @@ describe('WorkflowRecipeReconciler', () => {
return { recipe, workflowReconcile }
}
- it('R2-L3: without a token ADDED the first-deploy pass still sends its one DELETE', async () => {
+ it('R3-L11: without a token ADDED the first-deploy pass sends no DELETE and logs the skip once, at debug', async () => {
const { recipe, workflowReconcile } = installFirstDeployWorkflow()
+ const skips = (spy: ReturnType<typeof vi.fn>) =>
+ spy.mock.calls.filter(
+ ([message]) =>
+ typeof message === 'string' && message.startsWith('Skipping oauth-broker-token delete')
+ )
+ const logs = captureLoggerLevels(['debug', 'info'] as const)
+ try {
+ await reconciler.reconcile(recipe)
- await reconciler.reconcile(recipe)
+ // Witness that the pass went through deployWorkflowWorkloads.
+ expect(workflowReconcile).toHaveBeenCalledTimes(1)
+ // Witness that the reap ran and the ledger declined it, once per pass.
+ expect(skips(logs.calls.debug)).toHaveLength(1)
+ expect(skips(logs.calls.info)).toHaveLength(0)
+ expect(secretDeletes()).toBe(0)
+ } finally {
+ logs.restore()
+ }
- // Witness that the pass went through deployWorkflowWorkloads.
- expect(workflowReconcile).toHaveBeenCalledTimes(1)
+ // Liveness witness: once the watch replays the token ADDED, the next
+ // pass sends its one DELETE.
+ tokenSeen()
+ await reconciler.reconcile(recipe)
+ expect(workflowReconcile).toHaveBeenCalledTimes(2)
expect(secretDeletes()).toBe(1)
})
diff --git a/workflow-recipes/src/reconciler/workflowRecipeReconciler.ts b/workflow-recipes/src/reconciler/workflowRecipeReconciler.ts
index 8995960cf..9f3a2be89 100644
--- a/workflow-recipes/src/reconciler/workflowRecipeReconciler.ts
+++ b/workflow-recipes/src/reconciler/workflowRecipeReconciler.ts
@@ -1986,15 +1986,11 @@ export class WorkflowRecipeReconciler {
// ensureOAuthBrokerTokenSecret, so the token-ADDED re-reconcile would
// never reach the reap. Only the delete branch runs here (issuance stays
// on the first-deploy path and the rotation loop), and the delete ledger
- // bounds it to one DELETE per generation or ADDED. It runs only once a
- // token ADDED was observed for the recipe: a fresh process would otherwise
- // send one DELETE per recipe, and the Secret watch's initial list replays
- // ADDED for every token that exists. The first-deploy path below skips
+ // decides: no DELETE until a token ADDED was observed for the recipe,
+ // then one per generation or ADDED. The first-deploy path below skips
// its own call when this one ran, so a failed DELETE is not repeated in
// the same pass.
- const oauthBrokerTokenReapedEarly =
- !rb.recipeHasBackgroundAccessClient(recipe) &&
- this.oauthBrokerDeleteLedger.secretEpoch(recipe.metadata.name) !== 0
+ const oauthBrokerTokenReapedEarly = !rb.recipeHasBackgroundAccessClient(recipe)
if (oauthBrokerTokenReapedEarly) {
await this.ensureOAuthBrokerTokenSecret(recipe)
}
@@ -7565,8 +7561,9 @@ export class WorkflowRecipeReconciler {
if (!rb.recipeHasBackgroundAccessClient(recipe)) {
const generation = recipe.metadata.generation
if (!this.oauthBrokerDeleteLedger.shouldDeleteSecret(recipe.metadata)) {
- createLogger('wrc', recipeName).info(
- 'Skipping oauth-broker-token delete; generation already seen',
+ // Every non-backgroundAccess pass reaches this, so it stays at debug.
+ createLogger('wrc', recipeName).debug(
+ 'Skipping oauth-broker-token delete; no token seen, or this generation already reaped it',
{ recipe: recipeName, generation }
)
return